diff --git a/.claude/skills/hugagent-backend-dev/references/architecture.md b/.claude/skills/hugagent-backend-dev/references/architecture.md index 306f4283..e71773af 100644 --- a/.claude/skills/hugagent-backend-dev/references/architecture.md +++ b/.claude/skills/hugagent-backend-dev/references/architecture.md @@ -34,7 +34,7 @@ edition_ee ──► core / features (CE 派生树中不 | 子包 | 职责 | |---|---| -| `agent_runtime/` | 一次普通智能体回合:`runner.astream_chat_workflow` 选主智能体 / 直连路径;`prepare_*`(建 agent、组装输入)、`stream_*`(模型与工具事件 → 既有事件协议)、`*_state`(单轮有类型状态)、`integrations/`(记忆、压缩、本体证据/修复、演化) | +| `agent_runtime/` | 一次普通智能体回合:`runner.astream_chat_workflow` 选主智能体 / 直连路径;`prepare_*`(建 agent、组装输入)、`stream_*`(模型与工具事件 → 既有事件协议)、`*_state`(单轮有类型状态)、`integrations/`(记忆、压缩、演化) | | `runs/` | 持久化执行所有权:租约 / 心跳、取消、事件日志与续播(`events/`)、看门狗、工具副作用账本(`tool_effects/`)、恢复 | | `chat/` | 会话与消息读写、模式(modes)、分叉、引用、追问、压缩;`runtime/`(聊天 worker:消费回合事件、在 runs 租约下持久化输出、排队 / 插话 / 暂停 / 恢复);`operations/`、`projection/` | | `llm/` | AgentScope 适配与智能体装配:`factory/`(build / assembly / tools / prompts)、`runtime/`(流式循环、钩子适配、工具执行、人机交互)、`citations/`(锚点发号)、`mcp/`、`subagents/`、`tools/`、`tool_permissions/`、提示词版本池 | @@ -48,7 +48,6 @@ edition_ee ──► core / features (CE 派生树中不 | `files/` | 产物(artifacts)、内容解析(content)、空间文件(spaces)、存储协议(storage:local / s3 / oss) | | `projects/` | 项目、项目环境、指令、本地工作区、个人项目作用域 | | `memory/` | 分层记忆(L1 画像 / L2 向量 / L3 图谱)、抽取流水线、outbox、脱敏、TTL | -| `ontology/` | 本体治理:schema、校验、运行时闸门、输出评审、修订 | | `evolution/` | 协同进化共享契约与证据(控制面在 EE) | | `jobs/` | 后台作业运行时(作业 SDK、监督、通知、文件) | | `sandbox/` | 沙箱协议 / 工厂 / providers(script_runner、opensandbox、cube[EE])、runner 进程服务、进程 / 会话 / 工作区 / 权限 / OS 隔离 | @@ -118,7 +117,7 @@ POST /v1/agents/responses api/routes/v1/agent_responses.py → core/llm/factory/build.py AgentScope agent:模型选择、工具集、MCP(core/llm/mcp)、技能、提示词 → stream_main / stream_direct 模型与工具事件 → 事件协议(core/llm/runtime/streaming.py、stream_events.py) → core/llm/citations/ 引用锚点发号(e1/e2…) - → 结束后:本体评审、记忆 / 演化结算、追问(core/chat/runtime/followups.py) + → 结束后:记忆 / 演化结算、追问(core/chat/runtime/followups.py) → SSE 输出 api/transports/run_sse.py(跟随事件日志;续播 GET /v1/chats/stream/{run_id}/subscription) ``` @@ -150,7 +149,7 @@ POST /v1/agents/responses api/routes/v1/agent_responses.py |---|---| | 应用组装 | `api/app.py`、`api/lifecycle.py`、`api/startup/` | | 路由注册表 | `api/routes/v1/__init__.py`(CE)、`edition_ee/routes/registry.py`(EE)、`core/harness/registry`(模块路由) | -| 依赖注入 | `api/deps.py`(require_admin / require_config / require_admin_or_config / require_system_settings / require_ontology_governance / require_super_admin / console_actor) | +| 依赖注入 | `api/deps.py`(require_admin / require_config / require_admin_or_config / require_system_settings / require_super_admin / console_actor) | | 认证 | `core/auth/backend.py`(get_current_user / UserContext)、`core/auth/bearer.py` | | DB | `core/db/engine.py`(get_db)、`core/db/threads.py`、`core/db/models/`、`core/db/repository/` | | 信封 / 异常 | `core/infra/responses.py`、`core/infra/exceptions.py`、`api/middleware/error_handler.py` | diff --git a/.claude/skills/hugagent-frontend-dev/SKILL.md b/.claude/skills/hugagent-frontend-dev/SKILL.md index 2806bfc7..d0743241 100644 --- a/.claude/skills/hugagent-frontend-dev/SKILL.md +++ b/.claude/skills/hugagent-frontend-dev/SKILL.md @@ -85,8 +85,7 @@ src/frontend/src/ │ ├── useStreaming.ts # SSE 流式处理、文件上传、发送逻辑、断线续播 │ ├── usePlanMode.ts # Plan 模式 │ ├── usePageConfig.ts # 页面配置 -│ ├── useDelayedFlag.ts # 延迟标志位 -│ └── useStallDetector.ts # 流式停滞检测 +│ └── useDelayedFlag.ts # 延迟标志位 │ ├── stores/ # 18 个 Zustand Store │ ├── authStore.ts # 认证状态 diff --git a/.claude/skills/hugagent-frontend-dev/references/sse-events.md b/.claude/skills/hugagent-frontend-dev/references/sse-events.md index 4b31de68..27df8082 100644 --- a/.claude/skills/hugagent-frontend-dev/references/sse-events.md +++ b/.claude/skills/hugagent-frontend-dev/references/sse-events.md @@ -50,7 +50,7 @@ data: [DONE] # 流终止 |---|---| | `content`(别名 `ai_message` / `text` / `delta`) | 取 `delta || content || text` 追加;内嵌 `` 的模型由 parseBuffer 拆出思考 | | `thinking`(别名 `thought`) | 带 `delta` 时按增量追加到最近的 thinking 段,并标记 structuredReasoning | -| `content_replace` | **整体替换**当前正文(本体治理重写答案) | +| `content_replace` | **整体替换**当前正文 | ## 工具 @@ -93,11 +93,6 @@ data: [DONE] # 流终止 - Plan 模式(`plan_step_*` / `plan_complete` 等)与自主循环(`loop_*`)事件经 `ChatStreamOptions.onEvent` 钩子交给 `usePlanMode` / `useLoopMode` 先处理,钩子返回 true 则不再进通用 reducer。 -## 本体治理 - -`ontology_activation` / `ontology_gate` / `ontology_review` / `ontology_repair` / `ontology_revision` / -`ontology_revision_thinking`;`scope === 'ontology_revision'` 的工具事件单独分流(重写结果经 `content_replace` 落地)。 - ## 已退役(不要再处理或产出) - `batch_confirm`(批量执行已退役) diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index 0d2447be..a3290500 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -43,7 +43,6 @@ body: - Knowledge base & RAG / 知识库 - Memory / 记忆 - Sandbox execution / 沙箱执行 - - Ontology & governance / 本体与治理 - Deployment / 部署 - Frontend / UI - Documentation / 文档 diff --git a/.gitignore b/.gitignore index 007313d7..b61e1e82 100755 --- a/.gitignore +++ b/.gitignore @@ -164,7 +164,11 @@ docker/dbhub/dbhub.toml docker/dbhub/es.env license-keys/ -deploy_kit/ +deploy_kit/* +!deploy_kit/kubernetes/ +deploy_kit/kubernetes/* +!deploy_kit/kubernetes/space_mounts.py +!deploy_kit/kubernetes/patch_upstream.py node_modules/ # 建站模板必须完整入库(覆盖上面的 lib/ 与 AGENTS.md 规则),但依赖树除外 diff --git a/README.md b/README.md index d95b57cd..0239008e 100644 --- a/README.md +++ b/README.md @@ -192,8 +192,6 @@ quietly learn the wrong things. - **Dreaming for Personal Agent**: a daily job consolidates logs into `MEMORY.md` / `USER.md`, with `DREAMS.md` recording sources, deduplication and conflicts; only what you explicitly asked it to remember is promoted automatically. -- **Traceable ontology evolution**: approvals, rejections, evidence and outcomes are - recorded and organized into versioned ontology proposals that apply after human review. ## 🧰 More harness capabilities @@ -205,7 +203,6 @@ quietly learn the wrong things. | 🔧 **MCP, Skills and plugins** | Built-in web search, page fetching, knowledge retrieval, charts and automation tools; an MCP marketplace, private MCP servers, Agent Skills and plugins | | 🧪 **Sandbox and artifacts** | Run code in OpenSandbox or a local subprocess sandbox, producing charts, reports, Office files, web pages and data canvases | | 📚 **Private knowledge-base RAG** | Document chunking, hybrid vector and keyword retrieval, optional reranking | -| 🛡️ **Optional ontology control plane** | With a Domain Pack, deterministic rule checks run before tool execution and high-risk answers are reviewed by an independent reviewer or committee; switched off, the original flow is unchanged | | 🔌 **Replaceable models** | Connect cloud or local OpenAI-compatible models through one model service; reviewer, planner, memory and other roles can each use their own model | ## Quick start @@ -267,7 +264,7 @@ troubleshooting. ## Architecture HugAgentOS separates user channels, agent workflows, reusable capability engines, -ontology contracts, data governance, and infrastructure into clear layers. Durable +data governance, and infrastructure into clear layers. Durable runs, leases and recovery run through the execution path, while security and platform governance span the complete stack. @@ -327,7 +324,6 @@ feature boundary and upgrade path. | Memory and self-evolution | [Memory](./document/en/modules/memory.md) | [记忆系统](./document/zh-CN/modules/memory.md) | | Configure a deployment | [Deployment](./document/en/deployment/README.md) | [部署指南](./document/zh-CN/deployment/README.md) | | Explore the system design | [Architecture](./document/en/architecture/overview.md) | [架构总览](./document/zh-CN/architecture/overview.md) | -| The ontology control plane | [Domain ontology quickstart](./document/en/getting-started/domain-ontology-quickstart.md) | [快速构建领域本体](./document/zh-CN/getting-started/domain-ontology-quickstart.md) | | Build backend or frontend features | [Development](./document/en/README.md#development) | [开发指南](./document/zh-CN/README.md#开发指南) | Start from [document/README.md](./document/README.md) to browse every guide. diff --git a/README_CN.md b/README_CN.md index 3545dd22..c64d59f1 100644 --- a/README_CN.md +++ b/README_CN.md @@ -167,7 +167,6 @@ HugAgentOS 不让智能体每次从零开始,也不让它「悄悄学坏」。 - **逐条审批、可回滚**:记忆与技能的进化建议以卡片形式出现,经你确认才生效,可随时关闭。 - **Personal Agent 的 Dreaming**:每天定时把日志整理为 `MEMORY.md` / `USER.md` 的更新, `DREAMS.md` 记录来源、去重与冲突;只自动提升你明确说「记住」的内容。 -- **可溯的本体进化**:审批、驳回、证据与执行结果留痕,整理为版本化的本体建议,人工审核后生效。 ## 🧰 更多 Harness 能力 @@ -179,7 +178,6 @@ HugAgentOS 不让智能体每次从零开始,也不让它「悄悄学坏」。 | 🔧 **MCP · Skills · 插件** | 内置联网搜索、网页抓取、知识检索、图表、自动化等工具;MCP 市场、私有 MCP、Agent Skills 与插件 | | 🧪 **沙箱与产物** | OpenSandbox 或子进程沙箱执行代码,产出图表、报告、Office 文件、网页与数据画布 | | 📚 **私有知识库 RAG** | 文档分块,向量与关键词混合检索,可选重排 | -| 🛡️ **可选的本体控制平面** | 导入 Domain Pack 后,工具执行前做确定性规则检查,高风险答案由独立评审员或委员会复核;关闭时完全不改变原流程 | | 🔌 **模型可替换** | 统一的模型服务配置接入云端或本地 OpenAI 兼容模型,评审、规划、记忆等角色可分别指定模型 | ## 快速开始 @@ -232,7 +230,7 @@ curl -fsSL https://raw.githubusercontent.com/ZJU-REAL/HugAgentOS/main/install.sh ## 系统架构 -HugAgentOS 将用户渠道、智能体工作流、可复用能力引擎、本体契约、数据治理和 +HugAgentOS 将用户渠道、智能体工作流、可复用能力引擎、数据治理和 基础设施分层组织;Durable Run、租约与恢复机制贯穿执行链路,安全管理与平台监管 贯穿完整技术栈。 @@ -288,7 +286,6 @@ HugAgentOS 将用户渠道、智能体工作流、可复用能力引擎、本体 | 记忆与自进化 | [记忆系统](./document/zh-CN/modules/memory.md) | [Memory](./document/en/modules/memory.md) | | 生产部署与环境变量 | [部署指南](./document/zh-CN/deployment/README.md) | [Deployment](./document/en/deployment/README.md) | | 系统设计与请求链路 | [架构总览](./document/zh-CN/architecture/overview.md) | [Architecture](./document/en/architecture/overview.md) | -| 领域本体控制平面 | [领域本体快速入门](./document/zh-CN/getting-started/domain-ontology-quickstart.md) | [Domain ontology quickstart](./document/en/getting-started/domain-ontology-quickstart.md) | | 后端与前端开发 | [开发指南](./document/zh-CN/README.md#开发指南) | [Development](./document/en/README.md#development) | 所有文档入口见 [document/README.md](./document/README.md)。 diff --git a/assets/hugagentos-architecture-en.svg b/assets/hugagentos-architecture-en.svg index ff3c1158..d48f1f3e 100644 --- a/assets/hugagentos-architecture-en.svg +++ b/assets/hugagentos-architecture-en.svg @@ -1,6 +1,6 @@ - + HugAgentOS layered architecture - Platform, agent, capability component, ontology, data governance, data foundation, and security governance layers. + Platform, agent, capability component, data governance, data foundation, and security governance layers. - + @@ -132,70 +130,46 @@ Action-driven capability assembly: Generate Plan → Approval & Validation → Gated Execution - - - - - - - - - - Domain Ontology - - Entities · Relations - · Invariants - - Action Contracts - - Role & Permission - Derivation - - Capability Graph · Memory Graph - - - - Data Governance - - - - - - - Data Ingestion - Metadata Scan - Data Profiling - Ontology Proposals - Semantic Mapping - Governance Approval + + + Data Governance + + + + + + + Data Ingestion + Metadata Scan + Data Profiling + Quality Checks + Semantic Mapping + Governance Approval - - - - - - - - - - PostgreSQL - Redis - Milvus - Neo4j - Object Storage - Sandbox Runtime + + + + + + + + + + PostgreSQL + Redis + Milvus + Neo4j + Object Storage + Sandbox Runtime - - - - - - + + + - + Security Management Platform Governance diff --git a/assets/hugagentos-architecture-zh.png b/assets/hugagentos-architecture-zh.png index 23f1f7e7..21d278a4 100644 Binary files a/assets/hugagentos-architecture-zh.png and b/assets/hugagentos-architecture-zh.png differ diff --git a/deploy_kit/kubernetes/patch_upstream.py b/deploy_kit/kubernetes/patch_upstream.py new file mode 100644 index 00000000..fcd00a02 --- /dev/null +++ b/deploy_kit/kubernetes/patch_upstream.py @@ -0,0 +1,86 @@ +"""Hash-guarded native-runtime patches; Docker runtime behavior is untouched.""" +import argparse +import hashlib +from pathlib import Path + +ORIGINAL = { + "server-volumes": ("opensandbox_server/services/k8s/volume_helper.py", "4eb95e848454e1b7a5dc3685c57b271dbed006b4ad464badc10a59acd5452dc8"), + "committer-security": ("kubernetes/cmd/image-committer/main.go", "eaf884ddfc989bfaaf78069650b8abd25ad5ca5418260ebb11746f2472c74c15"), + "snapshot-trust": ("kubernetes/internal/controller/sandboxsnapshot_lifecycle.go", "a0ac9c2f4d9793cf85a572ed5dd21d0e7e02b1ad39611591a8d80d362c4f0427"), + "controller": ("kubernetes/cmd/controller/main.go", "b3d01d62c2a8e73056bceac4266e78ac41b81e64baa9b646bf0efcd2eff23dc7"), + "server-security": ("opensandbox_server/services/k8s/security_context.py", "91a87c7d74a02dec2bce0b40bacdf454a67435a6e3c5a920cf1161850587c83f"), + "server": ("opensandbox_server/services/k8s/provider_common.py", "e90a2e50f5592104313fa0c4be103d85f5ed96baeed53ce070b669fac5e1223e"), +} + + + +def patch_identity(): + root = Path(__file__).resolve().parent + return hashlib.sha256((root / "patch_upstream.py").read_bytes() + + (root / "space_mounts.py").read_bytes()).hexdigest() + + +def replace_once(text, before, after): + if text.count(before) != 1: + raise ValueError("Pinned source patch anchor changed") + return text.replace(before, after) + + +def patched(text, component): + if hashlib.sha256(text.encode()).hexdigest() != ORIGINAL[component][1]: + raise ValueError("Pinned upstream source hash changed: " + component) + if component == "server-volumes": + text = replace_once(text, ' containers = pod_spec.get("containers", [])', ' from opensandbox_server.services.k8s.space_mounts import apply_object_volumes\n volumes = apply_object_volumes(pod_spec, volumes)\n containers = pod_spec.get("containers", [])') + elif component == "controller": + text = replace_once(text, 'ctrl "sigs.k8s.io/controller-runtime"', 'ctrl "sigs.k8s.io/controller-runtime"\n\t"sigs.k8s.io/controller-runtime/pkg/cache"') + text = replace_once(text, '\tmgr, err := ctrl.NewManager(config, ctrl.Options{', '\tworkNamespace := os.Getenv("HUGAGENT_SANDBOX_NAMESPACE")\n\tif workNamespace == "" {\n\t\tsetupLog.Error(fmt.Errorf("missing namespace"), "HUGAGENT_SANDBOX_NAMESPACE is required")\n\t\tos.Exit(1)\n\t}\n\tmgr, err := ctrl.NewManager(config, ctrl.Options{\n\t\tCache: cache.Options{DefaultNamespaces: map[string]cache.Config{workNamespace: {}}},\n\t\tLeaderElectionNamespace: os.Getenv("POD_NAMESPACE"),') + text = replace_once(text, 'LeaderElectionID: "2fa1c467.opensandbox.io"', 'LeaderElectionID: workNamespace + ".opensandbox.io"') + text = replace_once(text, 'profileStore.SetupWithManager(mgr, os.Getenv("POD_NAMESPACE"))', 'profileStore.SetupWithManager(mgr, workNamespace)') + elif component == "snapshot-trust": + text = replace_once(text, '\tif r.SnapshotPushSecret != "" {', """\tvolumes = append(volumes, corev1.Volume{ + Name: "registry-trust", + VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: "registry-trust"}, + }}, + }) + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: "registry-trust", MountPath: "/var/run/hugagent/registry-trust", ReadOnly: true, + }) + if r.SnapshotPushSecret != "" {""") + text = replace_once(text, + 'env := []corev1.EnvVar{{Name: "CONTAINERD_SOCKET", Value: ContainerdSocketPath}}', + 'env := []corev1.EnvVar{{Name: "CONTAINERD_SOCKET", Value: ContainerdSocketPath}, {Name: "SSL_CERT_FILE", Value: "/var/run/hugagent/registry-trust/ca.crt"}}') + text = replace_once(text, + '\tif r.SnapshotRegistryInsecure {\n\t\tenv = append(env, corev1.EnvVar{Name: "SNAPSHOT_REGISTRY_INSECURE", Value: "true"})\n\t}', + '\tenv = append(env, corev1.EnvVar{Name: "SNAPSHOT_REGISTRY_INSECURE", Value: fmt.Sprintf("%t", r.SnapshotRegistryInsecure)})') + elif component == "committer-security": + text = replace_once(text, 'nerdctlBaseArgs(), "login", "-u", username, "-p", password', + 'nerdctlBaseArgs(), "login", "-u", username, "--password-stdin"') + text = replace_once(text, 'cmd := exec.Command("nerdctl", loginOpts...)', + 'cmd := exec.Command("nerdctl", loginOpts...)\n\tcmd.Stdin = strings.NewReader(password)') + elif component == "server-security": + text = replace_once(text, "from kubernetes.client import V1SecurityContext, V1Capabilities", "from kubernetes.client import V1SecurityContext, V1Capabilities, V1SeccompProfile") + text = replace_once(text, " privileged=privileged,", " privileged=privileged,\n allow_privilege_escalation=security_context_dict.get('allowPrivilegeEscalation'),\n seccomp_profile=V1SeccompProfile(**security_context_dict['seccompProfile']) if 'seccompProfile' in security_context_dict else None,") + text = replace_once(text, " return result if result else None", " if security_context.allow_privilege_escalation is not None:\n result['allowPrivilegeEscalation'] = security_context.allow_privilege_escalation\n if security_context.seccomp_profile is not None:\n result['seccompProfile'] = {'type': security_context.seccomp_profile.type}\n return result if result else None") + else: + before = " security_context = None\n if has_network_policy:\n security_context_dict = build_security_context_for_sandbox_container(True)\n security_context = build_security_context_from_dict(security_context_dict)" + after = """ # execd changes uid/gid and owns workspace files; retain only those + # runtime capabilities. User command children run as the requested uid. + security_context = build_security_context_from_dict({ + "privileged": False, + "allowPrivilegeEscalation": False, + "capabilities": {"drop": ["ALL"], "add": [ + "CHOWN", "DAC_OVERRIDE", "FOWNER", "SETGID", "SETUID", "KILL"]}, + "seccompProfile": {"type": "RuntimeDefault"}, + })""" + text = replace_once(text, before, after) + return text + + +if __name__ == "__main__": + p = argparse.ArgumentParser() + p.add_argument("--root", type=Path, required=True) + p.add_argument("--component", choices=ORIGINAL, required=True) + a = p.parse_args() + path = a.root / ORIGINAL[a.component][0] + path.write_text(patched(path.read_text(), a.component)) diff --git a/deploy_kit/kubernetes/space_mounts.py b/deploy_kit/kubernetes/space_mounts.py new file mode 100644 index 00000000..38dfdfb3 --- /dev/null +++ b/deploy_kit/kubernetes/space_mounts.py @@ -0,0 +1,67 @@ +"""Inject credential-isolated object mount helper into each sandbox Pod.""" +import json +import os +import re + +def apply_object_volumes(pod_spec, volumes): + objects = [v for v in volumes if v.ossfs is not None] + if not objects: + return volumes + names = ("OPENSANDBOX_SPACE_MOUNT_IMAGE", "HUGAGENT_SPACE_SECRET_NAME", + "HUGAGENT_SPACE_ENDPOINT", "HUGAGENT_SPACE_BUCKET", "HUGAGENT_SPACE_PREFIX") + config = {k: os.environ.get(k, "") for k in names} + if not all(config.values()): + raise ValueError("Controlled object mount configuration is incomplete") + if pod_spec.get("shareProcessNamespace") or pod_spec.get("hostPID"): + raise ValueError("Object mounts require private process namespaces") + pod_spec["shareProcessNamespace"] = False + pod_spec["automountServiceAccountToken"] = False + specs, roots = [], set() + for volume in objects: + obj = volume.ossfs + match = re.fullmatch(r"/workspace/(myspace|teamspace)/([A-Za-z0-9_-]{1,128})", volume.mount_path) + prefix = volume.sub_path or "" + if (not match or not prefix.startswith(config["HUGAGENT_SPACE_PREFIX"].rstrip("/") + "/spaces/") + or any(p in ("", ".", "..") for p in prefix.split("/"))): + raise ValueError("Invalid authorized space path") + if (obj.bucket != config["HUGAGENT_SPACE_BUCKET"] or obj.endpoint != config["HUGAGENT_SPACE_ENDPOINT"] + or obj.access_key_id != "server-managed" or obj.access_key_secret != "server-managed" + or obj.options): + raise ValueError("Object mounts require fixed server credentials and options") + roots.add(match[1]) + specs.append({"path": volume.mount_path, "prefix": prefix, "read_only": volume.read_only}) + if len({s["path"] for s in specs}) != len(specs): + raise ValueError("Duplicate object mount destination") + mounts = pod_spec["containers"][0].setdefault("volumeMounts", []) + if any(m["mountPath"].rstrip("/") in ("/workspace", "/workspace/myspace", "/workspace/teamspace") for m in mounts): + raise ValueError("Space root conflicts with another volume") + pod_volumes = pod_spec.setdefault("volumes", []) + helper_mounts = [] + for root in sorted(roots): + name, path = "hugagent-" + root, "/workspace/" + root + if any(v["name"] == name for v in pod_volumes): + raise ValueError("Reserved space volume name") + pod_volumes.append({"name": name, "emptyDir": {}}) + mounts.append({"name": name, "mountPath": path, "mountPropagation": "HostToContainer"}) + helper_mounts.append({"name": name, "mountPath": path, "mountPropagation": "Bidirectional"}) + pod_volumes.append({"name": "hugagent-space-credentials", "secret": { + "secretName": config["HUGAGENT_SPACE_SECRET_NAME"], "defaultMode": 256, + "items": [{"key": "accessKeyID", "path": "access"}, {"key": "secretAccessKey", "path": "secret"}]}}) + helper_mounts.append({"name": "hugagent-space-credentials", "mountPath": "/run/space-credentials", "readOnly": True}) + pod_spec.setdefault("initContainers", []).append({ + "restartPolicy": "Always", + "name": "space-mount", "image": config["OPENSANDBOX_SPACE_MOUNT_IMAGE"], + "env": [{"name": "SPACE_MOUNTS", "value": json.dumps(specs)}, + {"name": "S3_ENDPOINT", "value": config["HUGAGENT_SPACE_ENDPOINT"]}, + {"name": "S3_BUCKET", "value": config["HUGAGENT_SPACE_BUCKET"]}, + {"name": "S3_REGION", "value": os.environ.get("HUGAGENT_SPACE_REGION", "us-east-1")}], + "volumeMounts": helper_mounts, + # Kubernetes requires privileged for Bidirectional propagation. + "securityContext": {"privileged": True, "runAsUser": 0, "runAsGroup": 0}, + "resources": {"requests": {"cpu": "50m", "memory": "64Mi"}, + "limits": {"cpu": "1", "memory": "512Mi"}}, + "startupProbe": {"exec": {"command": ["python3", "/opt/space-mount.py", "check"]}, + "periodSeconds": 2, "timeoutSeconds": 2, "failureThreshold": 90}, + "readinessProbe": {"exec": {"command": ["python3", "/opt/space-mount.py", "check"]}, + "periodSeconds": 2, "timeoutSeconds": 2, "failureThreshold": 1}}) + return [v for v in volumes if v.ossfs is None] diff --git a/desktop/shared/pages/setup-html.html b/desktop/shared/pages/setup-html.html index 2d948ea1..eb8e37df 100644 --- a/desktop/shared/pages/setup-html.html +++ b/desktop/shared/pages/setup-html.html @@ -196,7 +196,12 @@

在这台电脑上开始使用

document.getElementById('visual').classList.add('ready'); document.getElementById('message').innerHTML='本机服务已就绪'; document.getElementById('install').style.display='none'; - if(s.active_local && !manage){ setTimeout(function(){location.replace('/__desktop/login')},450);return; } + if(s.active_local && !manage){ + // 启动时壳正在校验已保存的会话:结论由壳导航(有效进应用,失效转登录卡片)。 + if(s.session_restoring){ document.getElementById('message').textContent='本机服务已就绪,正在恢复登录…'; return; } + if(s.has_session){ location.replace('/'); return; } + setTimeout(function(){location.replace('/__desktop/login')},450);return; + } if(!s.active_local && !manage){ if(dual){ document.getElementById('message').textContent='本机服务已就绪,正在返回…'; diff --git a/desktop/src-tauri/src/auth.rs b/desktop/src-tauri/src/auth.rs index bb9b2f05..00afc942 100644 --- a/desktop/src-tauri/src/auth.rs +++ b/desktop/src-tauri/src/auth.rs @@ -124,6 +124,17 @@ pub async fn validate( cookie_name: &str, token: &str, ) -> bool { + check(http, server_base, cookie_name, token).await == Some(true) +} + +/// `Some(false)` only when the server rejected the session (401/403); `None` when +/// it could not answer (timeout, connection error, 5xx), so callers can keep the token. +pub async fn check( + http: &reqwest::Client, + server_base: &str, + cookie_name: &str, + token: &str, +) -> Option { let url = format!( "{}/api/v1/auth/session/check", server_base.trim_end_matches('/') @@ -140,8 +151,16 @@ pub async fn validate( .send() .await { - Ok(resp) => resp.status().is_success(), - Err(_) => false, + Ok(resp) if resp.status().is_success() => Some(true), + Ok(resp) + if matches!( + resp.status(), + reqwest::StatusCode::UNAUTHORIZED | reqwest::StatusCode::FORBIDDEN + ) => + { + Some(false) + } + _ => None, } } diff --git a/desktop/src-tauri/src/bootstrap.rs b/desktop/src-tauri/src/bootstrap.rs index 3a5fad01..143a938f 100644 --- a/desktop/src-tauri/src/bootstrap.rs +++ b/desktop/src-tauri/src/bootstrap.rs @@ -1,6 +1,6 @@ //! Assemble configured services and the initial window; runtime event wiring lives in app. use crate::display::sanitize_zoom; -use crate::session::{handle_deep_link, spawn_startup_probe}; +use crate::session::{handle_deep_link, spawn_local_session_restore, spawn_startup_probe}; use crate::tray::build_tray; use crate::windows::build_window; use crate::{ @@ -248,6 +248,9 @@ pub(crate) fn initialize(app: &mut tauri::App) -> Result<(), Box bool { + self.status.read().await.ready + } + pub async fn snapshot(&self) -> LocalServerStatus { let mut value = self.status.read().await.clone(); if !self.install_running.load(Ordering::SeqCst) && value.installed && self.is_ready().await diff --git a/desktop/src-tauri/src/proxy/state.rs b/desktop/src-tauri/src/proxy/state.rs index f625457e..98eeb48f 100644 --- a/desktop/src-tauri/src/proxy/state.rs +++ b/desktop/src-tauri/src/proxy/state.rs @@ -23,6 +23,10 @@ pub(super) struct SetupStatus { /// 避免把随启动变化的反代随机端口写进可分享的 URL。 local_server_base: String, provision_mode: ProvisionMode, + /// 本机模式:壳持有已保存的会话,就绪后直接进入应用,不跳登录卡片。 + has_session: bool, + /// 启动时壳还在向本机服务校验这份会话;结论由壳导航,进度页只等待。 + session_restoring: bool, /// 双模式:云端身份 / 模型拓扑是否已推到本机执行面。 bridge: crate::hybrid::BridgeSync, update: crate::update::UpdateStatus, @@ -35,6 +39,11 @@ pub(super) async fn setup_status_value(state: &ProxyState) -> SetupStatus { current_server_base: state.server_base.clone(), local_server_base: state.local_base.clone(), provision_mode: state.provision_mode.clone(), + has_session: state.session.token().await.is_some(), + session_restoring: state + .session + .restoring + .load(std::sync::atomic::Ordering::SeqCst), bridge: state.session.snapshot().await.bridge, update: crate::update::status(), } diff --git a/desktop/src-tauri/src/session.rs b/desktop/src-tauri/src/session.rs index 8ede98d8..a8f9d3b6 100644 --- a/desktop/src-tauri/src/session.rs +++ b/desktop/src-tauri/src/session.rs @@ -1,5 +1,6 @@ use crate::windows::{navigate_session_windows, show_main_window}; use crate::{auth, device_login, hybrid, local_server, Shared}; +use std::sync::atomic::Ordering; use tauri::Manager; use tauri_plugin_opener::OpenerExt; /// Only the authenticated host proxy calls this; no custom IPC command is registered. @@ -106,6 +107,79 @@ pub(crate) fn spawn_startup_probe(app: tauri::AppHandle, server_base: String, co }); } +/// LocalOnly: the backend is the local server this process is still starting, +/// so the saved session can only be checked once it is up. A valid session goes +/// straight to the app; a rejected one is cleared and lands on the login card. +pub(crate) fn spawn_local_session_restore(app: tauri::AppHandle) { + /// The setup page waits while this is set; clearing it wakes the page to route itself. + struct Restoring(tauri::AppHandle); + impl Drop for Restoring { + fn drop(&mut self) { + let shared = self.0.state::(); + shared.session.restoring.store(false, Ordering::SeqCst); + shared.local_server.notify_changed(); + } + } + if !app.state::().session.has_token() { + return; + } + app.state::() + .session + .restoring + .store(true, Ordering::SeqCst); + tauri::async_runtime::spawn(async move { + let _restoring = Restoring(app.clone()); + let shared = app.state::(); + let epoch = shared.session.epoch.current(); + let Some(token) = shared.session.token().await else { + return; + }; + let mut changes = shared.local_server.subscribe(); + while !shared.local_server.reported_ready().await { + if !shared.session.epoch.matches(epoch) || changes.changed().await.is_err() { + return; + } + } + // A server that has only just come up may still be slow to answer; only an + // explicit rejection costs the user their login. + let mut verdict = None; + for attempt in 0..3 { + if attempt > 0 { + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + } + verdict = auth::check( + &shared.http, + &shared.server_base, + &shared.cookie_name, + &token, + ) + .await; + if verdict.is_some() || !shared.session.epoch.matches(epoch) { + break; + } + } + if !shared.session.epoch.matches(epoch) { + return; + } + match verdict { + Some(true) => navigate_session_windows(&app, &shared.home_url()), + // Undecided: keep the token, so the next launch can try again. + None => navigate_session_windows(&app, &shared.login_idle_url()), + Some(false) => { + let Some(expected) = shared.session.epoch.advance_if_current(epoch) else { + return; + }; + cancel_device_login(&shared); + clear_invalidated_session(&shared, expected).await; + if !shared.session.epoch.matches(expected) { + return; + } + navigate_session_windows(&app, &shared.login_idle_url()); + } + } + }); +} + /// Cancel immediately in memory; server cleanup cannot block logout/navigation. pub(crate) fn cancel_device_login(shared: &Shared) { let cancelled_generation = shared.device_login.generation(); diff --git a/desktop/src-tauri/src/session_state.rs b/desktop/src-tauri/src/session_state.rs index 2561830d..2dcbb922 100644 --- a/desktop/src-tauri/src/session_state.rs +++ b/desktop/src-tauri/src/session_state.rs @@ -1,5 +1,6 @@ //! Owns the in-memory account and bridge state. Callers never receive write locks. use crate::{auth::SessionEpoch, hybrid::BridgeSync}; +use std::sync::atomic::AtomicBool; use tokio::sync::RwLock; #[derive(Default, Clone)] @@ -11,6 +12,9 @@ pub(crate) struct Snapshot { pub(crate) struct SessionState { pub epoch: SessionEpoch, + /// LocalOnly startup: the saved token is still being checked against the + /// local server, so the setup page waits instead of routing on its own. + pub restoring: AtomicBool, state: RwLock, } @@ -18,6 +22,7 @@ impl SessionState { pub fn new(token: Option) -> Self { Self { epoch: SessionEpoch::default(), + restoring: AtomicBool::new(false), state: RwLock::new(Snapshot { token, ..Default::default() diff --git a/docker/Dockerfile.space-mount b/docker/Dockerfile.space-mount new file mode 100644 index 00000000..32bb17d1 --- /dev/null +++ b/docker/Dockerfile.space-mount @@ -0,0 +1,4 @@ +FROM python:3.12-slim-bookworm +RUN apt-get update && apt-get install -y --no-install-recommends s3fs fuse ca-certificates && rm -rf /var/lib/apt/lists/* && echo user_allow_other >> /etc/fuse.conf +COPY docker/space-mount.py /opt/space-mount.py +ENTRYPOINT ["python3", "/opt/space-mount.py"] diff --git a/docker/space-mount.py b/docker/space-mount.py new file mode 100644 index 00000000..20dd6a14 --- /dev/null +++ b/docker/space-mount.py @@ -0,0 +1,78 @@ +"""Supervise S3FS. An absent mount never becomes a writable fallback.""" +import json +import os +from pathlib import Path +import signal +import subprocess +import sys +import time + +def mounted(path): + for line in Path("/proc/self/mountinfo").read_text().splitlines(): + before, after = line.split(" - ", 1) + if before.split()[4] == path and after.split()[0] == "fuse.s3fs": + return True + return False + +def prepare_mount(path): + path.parent.mkdir(parents=True, exist_ok=True) + path.parent.chmod(0o755) + # A killed FUSE process leaves a disconnected mount. Detach it + # before stat/mkdir, which would otherwise fail with ENOTCONN. + if mounted(str(path)): + subprocess.run(["fusermount", "-u", str(path)], check=True) + path.mkdir(exist_ok=True) + path.chmod(0) + +def main(): + specs = json.loads(os.environ["SPACE_MOUNTS"]) + if len(sys.argv) > 1 and sys.argv[1] == "check": + return 0 if all(mounted(s["path"]) for s in specs) else 1 + credential = Path("/run/s3fs.passwd") + access = Path("/run/space-credentials/access").read_text().strip() + secret = Path("/run/space-credentials/secret").read_text().strip() + if any(c in access + secret for c in "\r\n:"): + raise ValueError("Unsupported S3 credential format") + fd = os.open(credential, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w") as stream: + stream.write(access + ":" + secret) + children = [] + stopping = False + def stop(*_): + nonlocal stopping + stopping = True + signal.signal(signal.SIGTERM, stop) + signal.signal(signal.SIGINT, stop) + try: + for spec in specs: + path = Path(spec["path"]) + prepare_mount(path) + options = ["passwd_file=" + str(credential), "url=" + os.environ["S3_ENDPOINT"], + "endpoint=" + os.environ["S3_REGION"], "use_path_request_style", "sigv4", + "allow_other", "uid=1000", "gid=1000", "umask=0022", + "stat_cache_expire=0", "attr_timeout=0", "entry_timeout=0", + "negative_timeout=0", "nosuid", "nodev"] + if spec["read_only"]: + options.append("ro") + command = ["s3fs", os.environ["S3_BUCKET"] + ":/" + spec["prefix"], str(path), "-f"] + for option in options: + command.extend(["-o", option]) + children.append(subprocess.Popen(command)) + while not stopping: + if any(child.poll() is not None for child in children): + raise RuntimeError("Object filesystem exited") + time.sleep(0.5) + return 0 + finally: + for child in children: + child.terminate() + for child in children: + try: + child.wait(timeout=10) + except subprocess.TimeoutExpired: + child.kill() + child.wait() + credential.unlink(missing_ok=True) + +if __name__ == "__main__": + sys.exit(main()) diff --git a/document/README.md b/document/README.md index f06d0e45..55c0cf4c 100644 --- a/document/README.md +++ b/document/README.md @@ -18,11 +18,11 @@ HugAgentOS is an open-source, self-hosted agent harness: your request keeps runn ``` document/ ├── zh-CN/ · en/ # 双语产品文档(镜像) -│ ├── getting-started/ # 产品简介、快速开始、领域本体快速入门 +│ ├── getting-started/ # 产品简介、快速开始 │ ├── deployment/ # 部署总览、Docker Compose、桌面本机模式、一键安装、Windows / macOS / UOS、环境变量 │ ├── architecture/ # 总体架构、后端、前端、数据模型 │ ├── modules/ # 功能模块:对话与 Run、Personal Agent、自主循环、后台作业、MCP、技能、沙箱、 -│ │ # 记忆、本体治理、知识库、站点与应用托管、存储、项目空间、认证、管理台等 +│ │ # 记忆、知识库、站点与应用托管、存储、项目空间、认证、管理台等 │ ├── api/ # API 总览(信封 / 鉴权 / SSE / 路由清单)、错误码 │ ├── editions/ # 社区版 vs 商业版、License 机制、CE 构建管线 │ └── development/ # 后端 / 前端开发、界面定制、工作台(Workbench)协议与 SDK diff --git a/document/en/README.md b/document/en/README.md index 6c80cf3b..b84fdd5f 100644 --- a/document/en/README.md +++ b/document/en/README.md @@ -14,7 +14,6 @@ product documentation for operators, users, and integrators. |------|------| | [Introduction](getting-started/introduction.md) | The durable agent, runtime forms, feature overview, tech stack and architecture | | [Quick Start](getting-started/quick-start.md) | Choose HugAgentOS Cloud, the desktop client or the one-command installer, and enable Personal Agent | -| [Build a Domain Ontology Quickly](getting-started/domain-ontology-quickstart.md) | Validate, publish, test, and bind tools with a minimal Domain Pack | ## Deployment @@ -37,7 +36,7 @@ Start with the [Deployment Guide (choosing a method)](deployment/README.md), the | Document | Description | |------|------| -| [Overview](architecture/overview.md) | Layered architecture, ontology control plane, full request lifecycle, container topology | +| [Overview](architecture/overview.md) | Layered architecture, full request lifecycle, container topology | | [Backend](architecture/backend.md) | `src/backend/` walkthrough and router registry | | [Frontend](architecture/frontend.md) | App entries, component groups, Zustand stores, module pages and the app center | | [Data Model](architecture/data-model.md) | Tables by domain, migration chains, CE/EE table boundary | @@ -71,7 +70,6 @@ Start with the [Deployment Guide (choosing a method)](deployment/README.md), the | Document | Description | |------|------| | [Memory System](modules/memory.md) | L1/L2/L3 layered memory, durable outbox, procedural learning, sanitization and audit | -| [Domain Ontology Governance and Validation](modules/ontology-harness.md) | Domain Packs, tool gates, pre-delivery review, human review of evolution drafts | | [Knowledge Base](modules/knowledge-base.md) | Self-hosted knowledge bases (hybrid retrieval) and external sources | | [Knowledge Base Wiki](modules/knowledge-base-wiki.md) | Extracted entities and concepts as interlinked pages and a concept graph | diff --git a/document/en/api/overview.md b/document/en/api/overview.md index dff5828d..d841c4cc 100644 --- a/document/en/api/overview.md +++ b/document/en/api/overview.md @@ -282,7 +282,6 @@ Auth column legend: "User" = session cookie or personal API key (`get_current_us | Personal space | `myspace_folders.py` | `/v1/myspace/folders` | `GET/POST /`, `POST /move-artifact` | User | | Personal space | `artifacts.py` | `/v1/artifacts` | `GET /`, `GET /favorites`, `DELETE /{artifact_id}` | User | | Memory | `memories.py` | `/v1/memories` | `GET /`, `DELETE /{memory_id}`, `GET/PATCH /settings`, `GET /profile`, `GET /graph` | User | -| Domain ontology | `ontologies.py` | `/v1/ontologies`, `/v1/admin/ontologies` | User setting/runtime preview; Domain Pack versions, build preflight, gate/review evidence, metrics, and evolution-draft governance | User / ADMIN | | Capability catalog | `catalog.py` | `/v1/catalog` | `GET /` (capability catalog), `PATCH /{kind}/{id}` | User | | Capability catalog | `kb.py` | `/v1/catalog/kb` | `POST /`, `POST /{kb_id}/documents`, `GET /{kb_id}/chunks` (personal knowledge bases) | User | | Capability catalog | `marketplace.py` | `/v1/marketplace` | `GET /skills`, `POST /install`, `POST /submissions` (skill marketplace) | User | diff --git a/document/en/architecture/backend.md b/document/en/architecture/backend.md index 18029b3a..39258481 100644 --- a/document/en/architecture/backend.md +++ b/document/en/architecture/backend.md @@ -11,7 +11,7 @@ src/backend/ ├── api/ # FastAPI app, middleware, and CE routes ├── edition_ee/ # EE routes, License, Team/RBAC, ORM, Dify, and service implementations ├── orchestration/ # Chat orchestration: Run executor, workflow, strategy, citations, schedulers -├── core/ # Domain core: 17 submodules (auth/llm/db/ontology/services/...) +├── core/ # Domain core: 17 submodules (auth/llm/db/services/...) ├── prompts/ # Prompt assembly runtime + file fallback texts ├── extensions/ # Bundled extensions │ ├── skills/ # default / marketplace @@ -91,8 +91,6 @@ User domain: `user_service`, `local_user_service` (register / login / password c Config domain: `model_config` (DB-backed model config, cached), `system_config` (service config), `mcp_service` (MCP server config), `log_service` (async observability log writer). -Ontology domain: `ontology_service` (user setting, versions, and runtime cropping) and `ontology_evolution_service` (evidence prefiltering, sanitization, human-review drafts, and inactive-version materialization); `core/ontology/` contains the four-layer schema, build gate, deterministic runtime gate, tool filter, and prompt renderer. - Enterprise Edition (EE): `team_service` / `team_folder_service` / `sso_sync` (teams and SSO sync), `distillation_service` (skill distillation), `sandbox_rebuild_service` + `cube_template_builder` (persistent-sandbox template rebuilds), `security_service` (read-only security-console aggregation). ### core/memory — the three-layer memory system @@ -188,7 +186,7 @@ CE routes live under `api/routes/v1/`; physically split commercial routes live u |---|---| | Chat and streaming | `chats.py` (the SSE streaming entry point), `chat_runs.py`, `chat_shares.py`, `summary.py`, `classify.py`, `memories.py` | | Content and files | `content.py`, `file_upload.py`, `file_parse.py`, `artifacts.py`, `myspace_folders.py`, `kb.py` (+ `kb_models.py`, Pydantic models only), `projects.py` | -| Capabilities and config | `catalog.py`, `models.py`, `config.py`, `marketplace.py`, `me_capabilities.py` (self-service capability center), `ontologies.py` (user setting, version governance, evidence loop), `agents.py`, `plans.py`, `automations.py`, `meta.py` (edition / feature probe, unauthenticated) | +| Capabilities and config | `catalog.py`, `models.py`, `config.py`, `marketplace.py`, `me_capabilities.py` (self-service capability center), `agents.py`, `plans.py`, `automations.py`, `meta.py` (edition / feature probe, unauthenticated) | | Users and auth | `users.py`, `me.py`, `api_keys.py`, `mock_sso.py` (dev) | | Content console (Enterprise Edition, EE; backend of `/admin`) | `admin_skills.py`, `admin_prompts.py`, `admin_kb.py`, `admin_agents.py`, `admin_mcp_servers.py`, `admin_marketplace.py`, `admin_skill_drafts.py`, `admin_sandbox.py`, `admin_logs.py`, `admin_usage_logs.py`, `admin_billing.py`, `admin_chat_history.py` | | System console (Enterprise Edition, EE; backend of `/config`) | `config_users.py`, `config_teams.py`, `config_invites.py`, `config_security.py`, `config_verify.py`, `config_license.py`, `service_configs.py` | diff --git a/document/en/architecture/data-model.md b/document/en/architecture/data-model.md index 95f89d71..6e2b54d3 100644 --- a/document/en/architecture/data-model.md +++ b/document/en/architecture/data-model.md @@ -20,8 +20,7 @@ core/db/ │ ├── logs.py # Tool/sub-agent/skill call logs, audit │ ├── memory.py # Profile memory, memory audit, sanitizer rules │ ├── datasource.py # Data sources, metadata governance, golden SQL -│ ├── site.py # Sites, connectors, and channel configuration -│ └── ontology.py # Domain Pack versions, gates, committees, evolution drafts +│ └── site.py # Sites, connectors, and channel configuration ├── repository/ # Repository layer: agent/artifact/audit/catalog/chat/kb/team/user ├── model_repository.py # Repository for model providers / role assignments └── edition_tables.py # Single source of truth for the CE/EE table boundary @@ -135,16 +134,6 @@ Tables marked "(Enterprise Edition, EE)" belong to the `EE_ONLY_TABLES` set and | `skill_call_logs` | Skill triggers (view / run_script / auto_load) | | `audit_logs` (Enterprise Edition, EE) | User-facing audit of critical operations | -### Domain ontology (models/ontology.py) - -| Table | Purpose | -|---|---| -| `ontology_packs` | Domain Pack identity, enable/default flags, and active-version pointer | -| `ontology_pack_versions` | Immutable four-layer ontology content and validation report | -| `ontology_enforcement_events` | Append-only evidence from build, tool, and output gates | -| `ontology_review_runs` | Checkpoint/committee verdicts, evidence, and latency | -| `ontology_drafts` | Human-reviewed evolution candidates and their materialized-version pointer | - ## Alembic Migrations - **EE main chain**: 93 migrations under `src/backend/alembic/versions/`, evolving from the initial schema (including structural moves such as MCP-to-streamable-http and the retirement of the office MCPs in favor of skills). Common commands: `alembic upgrade head`, `make migrate-new msg="..."` (autogenerate is driven by `core/db/models` metadata); @@ -178,7 +167,6 @@ A few tables that *look* EE but are required by CE are deliberately excluded fro | Topic | Path | |---|---| | Shared ORM / EE ORM | `src/backend/core/db/models/`, `src/backend/edition_ee/db/models/` | -| Ontology repository | `src/backend/core/db/repository/ontology.py` | | Feedback model / repository | `src/backend/edition_ee/db/models/feedback.py`, `edition_ee/db/repository/feedback.py` | | Engine and startup table creation | `src/backend/core/db/engine.py` | | Repository layer | `src/backend/core/db/repository/` | diff --git a/document/en/architecture/frontend.md b/document/en/architecture/frontend.md index 84a01684..b6163d63 100644 --- a/document/en/architecture/frontend.md +++ b/document/en/architecture/frontend.md @@ -103,7 +103,6 @@ SSE streams bypass the JSON channel of `api.ts`; they are consumed directly from | `useChatInit` | Session initialization and active-run recovery on app start | | `usePlanMode` | Plan-mode SSE consumer (shared by first execution and resume) | | `usePageConfig` | Reads page config by dot-path (e.g. `branding.product_name`) | -| `useStallDetector` | Detects a stream gone silent past a threshold (stall hint) | | `useDelayedFlag` | Anti-flicker delayed boolean (delayed show + minimum hold) | ## Styles and Other Directories diff --git a/document/en/architecture/overview.md b/document/en/architecture/overview.md index 3fc350d6..5b9c5622 100644 --- a/document/en/architecture/overview.md +++ b/document/en/architecture/overview.md @@ -4,7 +4,7 @@ HugAgentOS is an open-source, self-hosted agent harness that runs as HugAgentOS Cloud (a fully containerized server) or HugAgentOS Local (on the desktop). A React single-page application provides the user experience, an async FastAPI service runs the backend, AgentScope 2.0's ReActAgent supplies the agent runtime, independent MCP (Model Context Protocol) processes expose tools, and pluggable sandbox providers isolate code execution. The platform ships in two editions — an open-source Community Edition (CE) and a commercial Enterprise Edition (EE) — built from a single codebase and separated by three runtime seams: the router registry, the table-creation boundary, and license feature gates (see [Editions](../editions/overview.md)). -This page describes the system from five angles: the layered architecture, the ontology-grounded enterprise trust plane, the full lifecycle of one chat request, the container topology, and the key design decisions. For deeper dives, see [Backend Architecture](./backend.md), [Frontend Architecture](./frontend.md), and the [Data Model](./data-model.md). +This page describes the system from four angles: the layered architecture, the full lifecycle of one chat request, the container topology, and the key design decisions. For deeper dives, see [Backend Architecture](./backend.md), [Frontend Architecture](./frontend.md), and the [Data Model](./data-model.md). ## Technology Stack @@ -44,7 +44,7 @@ This page describes the system from five angles: the layered architecture, the o ┌──────────────────────────────▼─────────────────────────────────────┐ │ Agent execution │ │ core/agent_runtime: runner.astream_chat_workflow, routing, │ -│ memory / ontology / evolution integration │ +│ memory / evolution integration │ │ core/llm: factory/build → AgentScope ReActAgent, tools, context │ │ orchestration/: plans · loops (autonomous loop) · automations │ │ personal_agent/: identity, task queue, scheduling, channels, │ @@ -61,74 +61,6 @@ This page describes the system from five angles: the layered architecture, the o Dependencies point strictly downward: `api → features / orchestration / personal_agent → core`; `core` never depends on upper layers, and `integrations/` holds connectors to external systems such as DingTalk, Feishu and email. MCP servers are independent processes that talk to the backend exclusively over HTTP, so a crash on either side never takes down the other. -## Ontology-Grounded Enterprise Trust Plane - -HugAgentOS extends the runtime into a trustworthy enterprise AgentOS by treating -domain ontology as an executable control plane rather than only a knowledge -store. A versioned Domain Pack can progress from controlled concepts and -relationships to invariants, Action contracts, workflows, roles, permissions, -and evidence requirements. The skill, orchestration, and memory engines -therefore reason against the same business semantics. - -The diagram below is the target enterprise architecture being integrated -incrementally on top of the current Harness. Existing confirmation gates, -structured review verdicts, audit records, and versioning provide the base; -ontology-specific assets and enforcement are added without claiming that every -free-text hallucination can be eliminated. - -```mermaid -flowchart TB - U[Enterprise users and channels] --> API[Web / Desktop / API] - API --> RUN[ChatRun and streaming workflow] - - ONTO[Domain ontology control plane
Concepts · Relationships · Invariants
Action contracts · Roles · Permissions] - ONTO --> BUILD[Build-time validation
Skills · Tools · Sub-agents] - ONTO --> BOOT[Domain bootstrap
Semantic alignment · Relevant rules] - - subgraph HARNESS[Three-engine Agent Harness] - direction LR - SKILL[Skill engine] <--> ORCH[Orchestration engine] - ORCH <--> MEMORY[Memory engine] - end - - RUN --> ORCH - BUILD --> SKILL - BOOT --> ORCH - BOOT --> MEMORY - ORCH --> PLAN[Candidate plan or action] - - PLAN --> RULE[Deterministic ontology rule gate] - ONTO --> RULE - RULE -->|Low risk and compliant| EXEC[Gated execution] - RULE -->|Checkpoint or high risk| REVIEW[Evidence-backed review] - REVIEW -->|Approved| EXEC - RULE -->|Violation| REVISE[Reject with evidence and repair guidance] - REVIEW -->|Revise or escalate| REVISE - REVISE --> ORCH - - EXEC --> CAP[MCP · Skills · Sandbox · RAG] - EXEC --> AUDIT[Traceable audit and replay] - REVISE --> AUDIT - AUDIT --> EVOLVE[Governed ontology proposals] - EVOLVE -. Human review · Versioning · Rollback .-> ONTO -``` - -Three principles keep the trust plane useful without turning every step into -an expensive committee review: - -1. **Deterministic before probabilistic.** Machine-checkable invariants, - permissions, and Action contracts run first. -2. **Review according to risk.** Only checkpoints, high-risk actions, or - uncertain evidence invoke deeper review; compliant low-risk actions proceed - through the execution gate. -3. **Evidence before evolution.** Violations, approvals, and outcomes enter the - audit trail. Proposed ontology changes require human approval, a new - version, and a rollback path. - -Ontology and RAG remain complementary: ontology governs what an agent may do -and which constraints apply, while RAG supplies the documents and evidence -used to support a decision. - ## Lifecycle of a Chat Request The entry point is `POST /v1/agents/responses` (`src/backend/api/routes/v1/chats/__init__.py`), which returns `text/event-stream`. diff --git a/document/en/deployment/README.md b/document/en/deployment/README.md index f8fdfab2..7975e857 100644 --- a/document/en/deployment/README.md +++ b/document/en/deployment/README.md @@ -57,3 +57,5 @@ A `{"status":"healthy",...}` response means the backend is up; then open the cor ## Local skill and plugin management [Local skill and plugin management](local-capability-management.md) + +For existing PostgreSQL, Redis and MinIO without new persistent volumes, see [External-services Kubernetes deployment (EE)](external-services-kubernetes.md). Run the live mount and network preflight before installation. diff --git a/document/en/deployment/desktop-local-mode.md b/document/en/deployment/desktop-local-mode.md index b56d5d1d..0f52fa5f 100644 --- a/document/en/deployment/desktop-local-mode.md +++ b/document/en/deployment/desktop-local-mode.md @@ -25,6 +25,16 @@ selected by the run. Skills do not prescribe deployment directories: scripts run and leave outputs in the working directory. Windows drive letters, Unicode and spaces are handled as command arguments. Container sandboxes retain separate mount and My Space policies. +## Hybrid mode: connecting to the cloud + +In hybrid mode the local backend shares one connection pool for fetching cloud capability +manifests, downloading skill and plugin bundles, and syncing work copies, so consecutive requests +reuse an established TLS connection. TCP connect plus the TLS handshake get 15 seconds by default, +adjustable with `HUGAGENT_CLOUD_CONNECT_TIMEOUT_S`. A connect or handshake timeout is retried once +(nothing has been sent yet, so repeating is safe); a read timeout is not. If it still fails, the +sync status shows a readable message such as "connection to the cloud timed out, check your network +or proxy", and the raw exception goes only to the local backend log. + ## Verification Validate installation and upgrades on each supported native platform. Workspace regression checks cover diff --git a/document/en/deployment/external-services-kubernetes.md b/document/en/deployment/external-services-kubernetes.md new file mode 100644 index 00000000..de700391 --- /dev/null +++ b/document/en/deployment/external-services-kubernetes.md @@ -0,0 +1,80 @@ +# Kubernetes with external services (EE) + +Use an existing PostgreSQL database, Redis service, and MinIO bucket. This deployment does not create databases, buckets, persistent volumes, CSI, or internal storage services. It excludes Personal Agent, LiteLLM/model gateway, internal knowledge-base services, and snapshot-committer. Memory uses pgvector in the same PostgreSQL database. Configure reachable model and embedding endpoints with the correct embedding dimensions. + +## Platform requirements + +- An existing namespace, registry pull secret, ingress domain and TLS secret; images pinned by digest for the target architecture. +- Kubernetes 1.29 or newer with native sidecar support, installed OpenSandbox CRDs, and permission to manage namespaced workloads, service accounts, RBAC, secrets and network policies. +- An existing PostgreSQL database with the vector extension installed. The owner account must be able to run Alembic migrations and read/write business tables. +- Authenticated Redis access and MinIO permissions to list, read, write and delete within the application prefix. +- A CNI that actually enforces NetworkPolicy. Sandbox sessions must not reach another session's execution or desktop endpoints. +- A controlled privileged helper with FUSE and Bidirectional mount propagation. The code execution container remains unprivileged; only the helper receives storage credentials. +- Platforms that forbid privileged containers set `HUGAGENT_SPACE_MOUNT_MODE=sync` (renderer: `space_transport: "sync"`). Nothing is mounted; the backend synchronizes space files over the sandbox's HTTP file API before and after each command. No privileged helper or FUSE is needed; consistency holds at command boundaries. See `internal design docs`. + +Ordinary Pod permissions alone cannot satisfy live filesystem mounts if the platform prohibits the helper capabilities. The deployment rejects missing mounts rather than substituting downloaded copies. + +## Configuration without an in-container .env + +The deployment tooling lives on the deploy branch at deploy_kit/kubernetes/external.py. Initialize inputs in a private directory outside the repository: + +```bash +python3 deploy_kit/kubernetes/external.py init --directory /secure/hugagent-input +``` + +For an external-services image pack, use registry-site.json containing registry, release and architecture: + +```bash +python3 deploy_kit/kubernetes/cloud.py import-images --external-services \ + --directory KIT --config registry-site.json --output images.json +``` + +For browser upload, use console-images --external-services. Copy the resulting image mapping into site.json, checking digests and architecture. Fill site.json with non-secret service settings and secrets.json with credentials and stable internal keys. Use separate strong values for ADMIN_TOKEN, CONFIG_TOKEN, BACKEND_INTERNAL_TOKEN, OPENSANDBOX_API_KEY and PLUGIN_RESOURCE_SECRET_KEY. Preserve them on upgrades. Enter PostgreSQL and Redis as the separate fields the administrator returns: DB_HOST, DB_PORT, DB_NAME, DB_USER, DB_PASSWORD and REDIS_HOST, REDIS_PORT, REDIS_PASSWORD, REDIS_DB, optionally DB_SSLMODE (disable, allow, prefer or require; CA-verifying modes are not supported yet), REDIS_TLS (true/false) and REDIS_USERNAME. The renderer encodes special characters and builds DATABASE_URL and REDIS_URL; complete URLs are still accepted. The application does not prefix Redis keys, so use a dedicated Redis database number that no other system shares. Do not embed credentials in images or command arguments. + +```bash +python3 deploy_kit/kubernetes/external.py render \ + --config /secure/hugagent-input/site.json \ + --secrets /secure/hugagent-input/secrets.json \ + --license /secure/license.lic \ + --output /secure/hugagent-rendered +``` + +The generated 00-config.json contains secrets. Keep the directory private (0700) and files restricted (0600). Containers receive ConfigMap/Secret values through envFrom; PYTHON_DOTENV_DISABLED=1 prevents dotenv loading. The renderer sets DEPLOY_PROFILE=kubernetes, STORAGE_TYPE=s3, SESSION_STORE=redis, MEM0_VECTOR_PROVIDER=pgvector, HUGAGENT_SPACE_MOUNT_MODE=sidecar, MODEL_GATEWAY_ENABLED=false and PERSONAL_AGENT_ENABLED=false. Use the renderer's complete configuration rather than this abbreviated list. + +The license Secret is a read-only seed. Verified renewals and clock state persist in PostgreSQL. Each process re-reads them every LICENSE_DB_REFRESH_SECONDS (default 30); requests do not query the database. During a short outage the last confirmed license stays valid for up to LICENSE_DB_OUTAGE_GRACE_SECONDS (default 300); a cold start or a longer outage fails closed. Artifact contents and metadata, manuals and branding assets persist in object storage; catalog enablement and ordering persist in PostgreSQL. Re-render after configuration changes: the configuration digest triggers Pod replacement. Do not edit live container files. + +## Preflight and installation order + +Back up existing data before upgrades and select the correct Kubernetes context. Preflight writes configuration and temporary probes into the target namespace. It checks real PG vector operations, Redis writes/reads, MinIO writes/reads, FUSE read/write propagation, read-only mounts and credential isolation. It also tests trusted access, denied cross-session access, then trusted access again using direct Pod IPs. Probe resources are cleaned up; the isolation policy and empty audit marker directories remain. It does not create a database, bucket or storage volume. + +```bash +python3 deploy_kit/kubernetes/external.py preflight \ + --directory /secure/hugagent-rendered --context TARGET_CONTEXT +``` + +Only proceed after all probes pass. Apply 03-control.json, then 04-migrate.json and wait for the migration Job to complete. Stop on migration failure. Apply 05-applications.json, 06-services.json and 07-ingress.json, then wait for backend and mcp rollouts. For example, replacing the namespace and release-specific Job name: + +```bash +kubectl --context TARGET_CONTEXT -n hugagent apply -f /secure/hugagent-rendered/03-control.json +kubectl --context TARGET_CONTEXT -n hugagent apply -f /secure/hugagent-rendered/04-migrate.json +kubectl --context TARGET_CONTEXT -n hugagent wait --for=condition=complete job/migrate-external-01 --timeout=1800s +kubectl --context TARGET_CONTEXT -n hugagent apply -f /secure/hugagent-rendered/05-applications.json +kubectl --context TARGET_CONTEXT -n hugagent apply -f /secure/hugagent-rendered/06-services.json +kubectl --context TARGET_CONTEXT -n hugagent apply -f /secure/hugagent-rendered/07-ingress.json +kubectl --context TARGET_CONTEXT -n hugagent rollout status deployment/backend --timeout=600s +kubectl --context TARGET_CONTEXT -n hugagent rollout status deployment/mcp --timeout=600s +``` + +Use a new release for each deployment; do not replace an active migration Job. Browser console uploads can follow these groups, but uploading static probe Jobs cannot replace the dynamic network preflight. An administrator or workstation with cluster access must run it. + +## Runtime behavior and acceptance + +Each session has a separate temporary execution directory. The same user's sessions share authorized persistent My Space files through live mounts. Team mounts follow existing authorization and read-only grants. Mount loss stops execution. Skills are checked against the current authorized view before each command and re-sent when the view changes or the sandbox is reset for a new session, including relative-path execution and removal of disabled/deleted packages. Revocation does not erase copies that a user previously saved elsewhere. Host-backed third-party credential directories and desktop capability publication are unsupported in this Kubernetes mode and fail explicitly. + +Without persistent volumes, control-plane SQLite state is temporary. Snapshots are disabled; no snapshot registry or node runtime socket is needed. Do not assume all active task state survives control-plane replacement. Persistent space files remain in object storage. + +Local isolated tests cover real database authorization, Bash execution, session isolation, live space sharing, private skill execution/revocation, read-only mounts and mount failure/recovery. These results do not establish target-platform permissions or model availability. Before release, test login, a real model reply, code in two sessions, cross-user isolation, persistent file recovery after Pod replacement and network isolation on the target platform. + +## Source builds + +Build backend, mcp and frontend from this source and supply their digests to the deploy branch renderer. Use the matching patched sandbox server and space-mount helper. The source branch includes shared mount helpers, not the full delivery CLI. Preserve the current source branch admission and session runtime. Rebuild affected images: already deployed images do not contain uncommitted source changes. diff --git a/document/en/development/frontend.md b/document/en/development/frontend.md index 715a69c5..bd43e717 100644 --- a/document/en/development/frontend.md +++ b/document/en/development/frontend.md @@ -44,7 +44,7 @@ src/frontend/src/ │ ├── citation/ common/ config/ docs/ file/ kb/ │ └── lab/ memory/ myspace/ projects/ settings/ share/ sidebar/ tool/ apidoc/ ├── hooks/ # useChatInit / useChatActions / useStreaming / usePlanMode -│ # usePageConfig / useDelayedFlag / useStallDetector +│ # usePageConfig / useDelayedFlag ├── stores/ # Zustand stores (authStore, chatStore, catalogStore, │ # editionStore, uiStore, settingsStore, fileStore, kbStore, │ # agentStore, automationStore, canvasStore, diff --git a/document/en/editions/build-ce.md b/document/en/editions/build-ce.md index 15150f24..a83e47ef 100644 --- a/document/en/editions/build-ce.md +++ b/document/en/editions/build-ce.md @@ -30,8 +30,7 @@ Glob patterns (relative to the repo root); a match means the file is never copie - **EE routes**: `api/routes/v1/admin_*.py`, `config_*.py`, `audit.py`, `auth.py`, `team_files.py`, `service_configs.py`, `data_sources.py`, `db_metadata.py`, `gateway_*.py`; - **Industry MCP servers**: `extensions/mcp_servers/query_database_mcp/**`, `ai_chain_information_mcp/**`; - **Industry-specific frontend code and seeds**: the industry-chain Canvas, - news and company-profile renderers, industry API client, icon assets, and the - bundled ontology example that depends on company-profile tools; + news and company-profile renderers, industry API client, and icon assets; - **The entire main-repo alembic chain** (`alembic/versions/**` — CE uses an independent chain from the overlay, see below); - **10 industry/branded skills** (under `extensions/skills/marketplace/`; the first 5 hard-depend on EE industry MCPs, the other 5 contain branded domain copy); - **EE-coupled tests** and `tests/licensing/**`; diff --git a/document/en/getting-started/domain-ontology-quickstart.md b/document/en/getting-started/domain-ontology-quickstart.md deleted file mode 100644 index 90f256e4..00000000 --- a/document/en/getting-started/domain-ontology-quickstart.md +++ /dev/null @@ -1,237 +0,0 @@ -# Build a domain ontology quickly - -> Last updated: July 21, 2026 | -> [简体中文](../../zh-CN/getting-started/domain-ontology-quickstart.md) - -This guide helps you build and publish a first Domain Pack in about 20 minutes. -HugAgentOS then uses its controlled concepts, relationships, constraints, and -workflows to validate domain tasks. The example has no external MCP dependency, -so you can verify the complete loop before binding production tools and skills. - -## What a Domain Pack contains - -A HugAgentOS domain ontology is an executable business contract, not only a -glossary for retrieval. A Domain Pack contains these parts: - -- **Metadata and configuration** define the pack ID, version, injection budget, - and circuit-breaker thresholds. -- **Concepts** define domain objects, aliases, meanings, hierarchies, controlled - values, and risk levels. -- **Relationships** define allowed or forbidden links and cardinality rules. -- **Constraints** apply JSON Schema, evidence, and prerequisite requirements to - tool arguments or final outputs. -- **Workflows** define text and asset triggers, required or forbidden tools, - output tags, and review levels. - -> [!TIP] -> Keep your first pack focused on one high-value task. Start with two to five -> concepts, one workflow, and one executable constraint, then expand it after -> the full loop works. - -## Step 1: choose a testable task - -Write the business goal as one sentence that you can test. For example: -“Analyze supplier risk in a procurement request and deliver a complete risk -summary.” Then define these three elements: - -1. List the objects the task must understand, such as a procurement request - and supplier risk. -2. State the invariant the result must satisfy, such as a minimum amount of - information in the risk summary. -3. Select a review level: `none` adds no review, `checkpoint` uses one reviewer, - and `committee` uses multiple reviewers for high-risk work. - -## Step 2: import a minimal Domain Pack - -Sign in as an instance administrator, open **Settings → Ontology Governance**, -and select **Import Domain Pack**. Paste the JSON below. This example activates -on procurement-risk text or a governed skill tag, then requires the final -summary to contain at least 120 characters. - -```json -{ - "schema_version": "1.0", - "pack_id": "procurement_risk", - "name": "Procurement Risk Domain Pack", - "version": "1.0.0", - "domain": "procurement-risk", - "description": "Governs supplier-risk identification and summaries for procurement requests.", - "config": { - "injection_enabled": true, - "max_concepts": 8, - "token_budget": 1600, - "committee_size": 3, - "repeated_denial_threshold": 2, - "circuit_breaker_threshold": 5, - "allow_unresolved_tools": false - }, - "concepts": [ - { - "id": "ProcurementRequest", - "name": "Procurement Request", - "aliases": ["Purchase Request"], - "definition": "A business request containing the purchase, budget, requesting unit, and candidate suppliers.", - "tags": ["procurement"], - "risk": "low" - }, - { - "id": "SupplierRisk", - "name": "Supplier Risk", - "aliases": ["Supply Risk"], - "definition": "A risk that can affect a supplier's delivery, compliance, or continued operation.", - "closed_values": ["low", "medium", "high", "unverified"], - "tags": ["risk"], - "risk": "medium" - } - ], - "relations": [ - { - "id": "request_has_supplier_risk", - "subject": "ProcurementRequest", - "predicate": "has", - "object": "SupplierRisk", - "description": "A procurement request can be associated with one or more supplier risks.", - "min_cardinality": 0, - "forbidden": false - } - ], - "constraints": [ - { - "id": "procurement_summary_complete", - "name": "Procurement risk summary must be complete", - "target": { - "kind": "output", - "output_tag": "procurement_risk_summary" - }, - "schema": { - "type": "string", - "minLength": 120 - }, - "concept_id": "SupplierRisk", - "requires_citations": false, - "prerequisite_tools": [], - "mode": "enforce", - "risk": "medium", - "message": "The procurement risk summary is too short for business review.", - "suggestion": "Add risk facts, impact, unknowns, and recommended verification steps.", - "enabled": true - } - ], - "workflows": [ - { - "id": "procurement_risk_review", - "name": "Procurement Risk Review", - "triggers": [ - "procurement risk", - "supplier risk", - "采购风险", - "供应商风险" - ], - "asset_triggers": [ - { - "kind": "skill", - "tags_any": ["ontology:SupplierRisk"] - } - ], - "required_tools": [], - "forbidden_tools": [], - "output_tags": ["procurement_risk_summary"], - "review_level": "checkpoint", - "risk": "medium" - } - ] -} -``` - -Leave **Publish immediately after import** off, then select **Validate and -Import**. HugAgentOS checks identifiers, versions, concept references, JSON -Schema, and tool references before it creates the working draft. Validation -errors include the exact field path. - -## Step 3: review and publish the version - -Review the working draft before it enters the runtime. Complete these steps in -order: - -1. Open **Details** and inspect the overview, concepts, relationships, - constraints, and workflows. -2. Open **Version Management** and publish the `1.0.0` working draft. -3. Return to the pack list and turn on **Enabled**. -4. If this pack must be the runtime default, turn on **Default**. - -A published version becomes read-only. Later edits create a new working draft -and don't overwrite the active version. - -## Step 4: enable and test ontology validation - -In **Settings → Ontology Governance**, turn on **Use domain ontology -validation**. Start a new chat and enter this test prompt: - -```text -Analyze supplier risk in this procurement request and provide a risk summary. -``` - -When the request matches `procurement risk` or `supplier risk`, the chat shows -a workflow-activation event and runs a `checkpoint` review after the answer. -The Ontology Governance page records the gate event and review result. - -Run two negative tests to confirm that the policy is scoped correctly: - -- Ask an unrelated question and confirm that the workflow doesn't activate. -- Request an extremely short procurement-risk conclusion and confirm that the - output constraint is reported or a revision is produced. - -## Step 5: bind production tools and skills - -After the minimal example works, extend the ontology from output validation to -action gating. First confirm the real tool IDs and argument schemas in MCP -management, then make these changes: - -1. Add every mandatory tool to the workflow's `required_tools`. -2. Set `target.tool` on a constraint to a real tool ID, and constrain its input - with `schema`. -3. Add tools to `prerequisite_tools` when the workflow requires an execution - order. -4. Activate workflows by tool ID or `ontology:ConceptId` in `asset_triggers`. -5. Bind the same controlled tag through the **Ontology governance tags** - selector when you edit a skill or sub-agent. - -Validation fails by default when a pack references an unregistered tool. Set -`allow_unresolved_tools` to `true` only during a staged migration, and restore -strict validation before production publication. - -## Pre-publication checklist - -A production-ready Domain Pack must pass at least these checks: - -- Every concept has a precise definition, and aliases don't conflict. -- Every relationship references existing concepts, and the hierarchy has no - cycles. -- Tool constraints use real tool IDs and argument names from the runtime. -- Every workflow has at least one text trigger or asset trigger. -- Required and forbidden tool sets don't overlap. -- Ordinary prompts don't activate the pack, while high-risk prompts reach the - intended review level. -- Violation messages explain why execution failed, and repair guidance tells - the agent what to do next. -- Administrators test a working draft before explicitly publishing it. - -## Troubleshooting - -The following table covers the most common problems in a first Domain Pack. - -| Symptom | Resolution | -|---|---| -| `unknown tool reference` | Confirm the discovered MCP tool ID, or remove the tool dependency until the minimal loop works. | -| A tag is missing from the skill selector | Define its concept and reference the tag in `asset_triggers.tags_any` for that asset kind. | -| A chat doesn't activate the workflow | Confirm that the pack is published, enabled, and default, then check whether a `triggers` value appears in the prompt. | -| Runtime behavior doesn't change after an edit | Published versions are read-only. Edit a working draft and publish the new version. | -| Rules block too many requests | Narrow the triggers, lower the mode or risk level, and use audit records to identify false matches. | - -## Next steps - -Read [Domain Ontology Governance and Validation](../modules/ontology-harness.md) -for tool gates, asset tags, committee review, version governance, and governed -evolution. To bind external capabilities, continue with the -[MCP Tool System](../modules/mcp-tools.md) and -[Agent Skills](../modules/agent-skills.md). diff --git a/document/en/getting-started/introduction.md b/document/en/getting-started/introduction.md index 74117c6e..0875c042 100644 --- a/document/en/getting-started/introduction.md +++ b/document/en/getting-started/introduction.md @@ -43,7 +43,6 @@ takeover and is meant for trials only. | Sandbox execution | OpenSandbox sessions with environment reuse and snapshot recovery, or a local subprocess sandbox; workflow jobs run as managed processes that can be taken over by lease | CE (CubeSandbox and team mounts are EE) | | Memory and personal evolution | L1 profile, L2 procedural memory (Milvus, optional), L3 relationship graph (Neo4j, optional); durable outbox writes; evolution proposals approved one by one | CE (memory auditing is EE) | | Knowledge base | Document chunking, hybrid vector and keyword retrieval, private knowledge bases, knowledge-base wiki | CE (public KB administration and Dify integration are EE) | -| Domain ontology governance | Domain Pack import and versioned publishing, rule checks before tool execution, independent review of high-risk answers, human review of evolution drafts | CE | | Automation | Scheduled tasks in natural language, cron scheduling, retries on failure | CE | | Sites and application hosting | Build a site in chat and publish it in one click; separate application databases and MCP hosting | CE (site collaboration after project conversion is EE) | | Data canvas and artifacts | Univer spreadsheets, charts, reports, Office files, web pages and an artifact center | CE personal editing; real-time collaboration is EE | @@ -78,7 +77,7 @@ takeover and is meant for trials only. │ ▼ Agent execution (core/agent_runtime · core/llm) - ReActAgent · per-step context assembly and compaction · memory / ontology / evolution integration + ReActAgent · per-step context assembly and compaction · memory / evolution integration Outer orchestration (orchestration/): Plan Mode · autonomous loop · automation │ ┌────────────────┼──────────────────┐ @@ -116,7 +115,6 @@ a task queue, scheduling, channels, device links and workspace files. See the | Plan Mode, autonomous loop, automation | `src/backend/orchestration/plans/`, `orchestration/loops/`, `orchestration/automations/` | | Personal Agent | `src/backend/personal_agent/` | | Memory and evolution | `src/backend/core/memory/`, `src/backend/core/evolution/` | -| Domain ontology | `src/backend/core/ontology/` | | Sandbox | `src/backend/core/sandbox/` | | Built-in MCP, skills, plugins and agents | `src/backend/extensions/` | | Frontend entry | `src/frontend/src/main.tsx`, `App.tsx` | diff --git a/document/en/getting-started/quick-start.md b/document/en/getting-started/quick-start.md index a6e25a9a..fc6fda5d 100644 --- a/document/en/getting-started/quick-start.md +++ b/document/en/getting-started/quick-start.md @@ -147,7 +147,5 @@ Continue with these guides after your first successful login: - Explore [MCP tools](../modules/mcp-tools.md), [Agent Skills](../modules/agent-skills.md), and [private knowledge bases](../modules/knowledge-base.md). -- Build a governed workflow with the - [domain ontology quickstart](domain-ontology-quickstart.md). - Read the [complete no-Docker installation guide](../deployment/quick-install.md) for installer options, capability boundaries, and troubleshooting. diff --git a/document/en/modules/admin-console.md b/document/en/modules/admin-console.md index 141faa56..3601d2b8 100644 --- a/document/en/modules/admin-console.md +++ b/document/en/modules/admin-console.md @@ -26,7 +26,6 @@ The CE/EE assignment of backend admin routes has one composed source of truth | Pending drafts | `SkillDraftsPanel` | `admin_skill_drafts.py` | | Sandbox dependencies | `SandboxDepsManager` | `admin_sandbox.py` | | Knowledge base management | `KnowledgeBaseManager` | `admin_kb.py` | -| Ontology governance | `OntologyManager` | `ontologies.py` | | Prompt hub | `PromptHubEditor` | `content.py` (`prompt_hub` content block) | | Sub-agents | `AdminAgentManager` | `admin_agents.py` | | User manual | `ManualEditor` | `content.py` (manual PDF upload) | @@ -136,7 +135,7 @@ There is also an API-oriented global audit query at `/v1/audit` (`api/routes/v1/ | Route group | File | Credential | Edition | Description | |---|---|---|---|---| -| `/v1/config/users` | `config_users.py` | `CONFIG_TOKEN` | Enterprise Edition (multi_tenancy) | User list / detail / status, password reset, deletion, and feature permissions such as app visibility and ontology validation — see [Authentication & Permissions](auth.md) | +| `/v1/config/users` | `config_users.py` | `CONFIG_TOKEN` | Enterprise Edition (multi_tenancy) | User list / detail / status, password reset, deletion, and feature permissions such as app visibility — see [Authentication & Permissions](auth.md) | | `/v1/config/teams` | `config_teams.py` | `CONFIG_TOKEN` | Enterprise Edition (multi_tenancy) | Team CRUD, member add/remove, role assignment | | `/v1/config/invites` | `config_invites.py` | `CONFIG_TOKEN` | Enterprise Edition (multi_tenancy) | Invite code batch generation / list / revoke / delete | | `/v1/config/security` | `config_security.py` | `CONFIG_TOKEN` | Enterprise Edition (system_config) | Security console (read-only): sandbox, audit logs, system health snapshot | @@ -152,7 +151,7 @@ There is also an API-oriented global audit query at `/v1/audit` (`api/routes/v1/ Aligned with chapter 4 of the productization plan and the route registry: - **Community Edition (CE) keeps**: `content.py` content-block management (rebrandable), `models.py` model management, and login infrastructure (`auth.py` session endpoints, mock SSO). -- **Enterprise Edition (EE)**: the full content console (feedback review / skills / plugins / drafts / marketplace / KB / ontology / sub-agents / prompts / MCP / sandbox dependencies, `content_admin`), the system console (service configs + security console, `system_config`), audit & chat review (`audit`), team billing & usage (`billing`), and users / teams / invite codes (`multi_tenancy`). +- **Enterprise Edition (EE)**: the full content console (feedback review / skills / plugins / drafts / marketplace / KB / sub-agents / prompts / MCP / sandbox dependencies, `content_admin`), the system console (service configs + security console, `system_config`), audit & chat review (`audit`), team billing & usage (`billing`), and users / teams / invite codes (`multi_tenancy`). - `config_license`, `config_verify`, and `auth` are explicitly exempt from the license guard so the "402 → replace license" escape hatch is always reachable. ## Source map diff --git a/document/en/modules/auth.md b/document/en/modules/auth.md index 08641a54..1035a79f 100644 --- a/document/en/modules/auth.md +++ b/document/en/modules/auth.md @@ -41,8 +41,7 @@ Password hashing lives in `core/auth/password.py`; the minimum length is control When you sign in to a CE instance with the default `admin/admin` credentials, you must replace the temporary password first. The application then opens a full-screen setup flow instead of the workspace. The flow covers display -language, models, internet search, document parsing, persistent memory, and -ontology validation. +language, models, internet search, document parsing, and persistent memory. - The model step includes a required primary chat model and optional embedding and reranker models. Saving tests every completed model configuration and @@ -55,11 +54,8 @@ ontology validation. can leave either retrieval model blank without blocking setup. - Internet search and external document parsing are optional. You can add them later under **Settings → System**. -- Memory and ontology switches check whether the instance has the required - service or a published Domain Pack. Unavailable features remain off. -- On a new CE deployment, **Show Dispatch Process** defaults to on when the - browser has no saved preference. If you turn it off under **Settings → Chat - Settings**, the browser preserves your choice. +- The memory switch checks whether the instance has the required service. + Unavailable features remain off. - The application stores a versioned completion marker in the administrator's `users_shadow.metadata`. Refreshing or signing in again doesn't repeat a completed setup flow. @@ -152,7 +148,6 @@ User-granular feature switches are stored in the `users_shadow.metadata` JSON co | `can_use_api_key` | off | `PATCH /v1/config/users/{id}/api-key-permission` | Creating / using personal API keys; turning it off immediately invalidates existing keys | | `can_add_skill` | off | `PATCH /v1/config/users/{id}/skill-permission` | Self-service private skill upload / authoring in the capability center (`api/routes/v1/me_capabilities.py`) | | `can_add_mcp` | off | `PATCH /v1/config/users/{id}/mcp-permission` | Self-service private remote MCP servers (HTTP/SSE) | -| `can_use_ontology_validation` | off | `PATCH /v1/config/users/{id}/ontology-validation-permission` | The ontology-validation Settings entry, personal switch, and runtime validation; without permission the entry is hidden and a previously enabled preference is ignored | | `lab_enabled` | **on** | `PATCH /v1/config/users/{id}/lab-permission` | Lab module entry and access | | `allowed_apps` | unrestricted | `PATCH /v1/config/users/{id}/app-permissions` | App visibility; `None` = all enabled apps, list = allowlist (empty list = block all) | | `role: super_admin` | none | (written directly into metadata) | Passes the `require_super_admin` dependency; bypasses team role checks | diff --git a/document/en/modules/chat.md b/document/en/modules/chat.md index 0500ddd6..3f2fe4c7 100644 --- a/document/en/modules/chat.md +++ b/document/en/modules/chat.md @@ -25,7 +25,7 @@ The scheduled-task setup template is inserted only when its conversation is crea User messages use rounded bubbles. Only newly sent messages play the entrance animation. History loading and returning to a conversation do not replay it. Message actions remain available on hover, keyboard focus, and touch devices. -Standalone reasoning retains its original inline summary. Reasoning within tool steps retains the timeline style. Click the summary or press Enter / Space while focused to expand or collapse it. Streaming updates preserve this choice. +Reasoning and tool calls appear in a collapsible execution card. Reasoning inside it keeps the timeline style. Click the card header or press Enter / Space while focused to expand or collapse it. Streaming updates preserve this choice. Composer attachments use compact cards that retain filename extensions and omit the bottom file-type label. Failed uploads keep an inline error and a Retry action. Removing an attachment or switching drafts during an upload cannot place the result in another draft. Completed attachments retain preview and download actions. The document/image filter in My Space supports arrow keys, Home / End, and selected-state feedback. @@ -176,7 +176,6 @@ Trusted unattended entry points (channel bots, scheduled automation) take the `d |---|---|---| | `thinking` | Reasoning (delta or stage hint) | `delta` / `message` | | `content` | Answer text delta | `event: "ai_message"`, `delta`, `chat_id` | -| `content_replace` | Replaces the streamed draft in place when ontology review revises the final answer | `content`, `reason: "ontology_review"`, `chat_id` | | `tool_call_start` | Tool-call construction starts; the frontend opens one card by stable ID | `tool_name`, `tool_display_name`, `tool_id` | | `tool_call_delta` | Incremental argument JSON appended to the same card | `tool_name`, `tool_id`, `arguments_delta` | | `tool_call` | Arguments are complete and execution is about to start | `tool_name`, `tool_display_name`, `tool_args`, `tool_id`, plus `subagent_name` for sub-agent calls | @@ -184,14 +183,13 @@ Trusted unattended entry points (channel bots, scheduled automation) take the `d | `steer_applied` | A mid-run instruction entered the ReAct context | `steer_id`, `message`, `message_id`, `chat_id` | | `queued_run_started` | A transactionally committed `followUp` / `nextRun`, or a `steer` that missed the final safe boundary, starts as a child and the frontend follows it immediately | `run_id`, `message_id`, `user_message_id`, `message`, `queue_id`, `steer_id`, `delivery_mode` | | `subagent_event` | Child execution details nested under the parent `call_subagent` card | `parent_tool_id`, `sub_type`, `agent_name`, plus child tool or content fields | -| `ontology_activation` / `ontology_gate` / `ontology_review` | Ontology-governance state, separate from model reasoning | workflow activation, gate decision, and committee status or verdict | | `tool_pending` | Waiting fallback when the provider exposes no parseable argument deltas | `reason` | | `file_confirm` | A tool is suspended awaiting confirmation of a MySpace write | confirmation context; the tool resumes in place after an out-of-band `POST /v1/chats/{chat_id}/file-confirm` | | `user_question` | `ask_user_question` is suspended awaiting the chat owner's answer | `request_id`, `questions[]`, `created_at`, `expires_at`, `chat_id` | | `user_question_resolved` | The server settled an answer, cancellation, or timeout | `request_id`, `outcome`, `chat_id` | | `context_usage` | Context snapshot for the latest primary-model call; the total is measured when the provider returns usage | `source`, `exact`, `prompt_tokens`, `completion_tokens`, `used_tokens`, `context_window`, `breakdown` | | `compaction_notice` | A new context-compaction checkpoint was created after the previous turn | `chat_id`, `context_compaction` (coverage boundary and replacement-summary token count) | -| `meta` | End-of-turn metadata | `route`, `citations[]`, `sources`, `artifacts`, `workspace_files`, `ontology_governance`, `warnings`, `is_markdown`, `message_id`, `usage`, `context_usage`, `compaction_pending` | +| `meta` | End-of-turn metadata | `route`, `citations[]`, `sources`, `artifacts`, `workspace_files`, `warnings`, `is_markdown`, `message_id`, `usage`, `context_usage`, `compaction_pending` | | `error` | Failure (mapped to a user-friendly message) | `error`, `chat_id` | | `heartbeat` | Heartbeat (event-level; a `: heartbeat` comment line also exists) | — | @@ -221,13 +219,7 @@ After `meta`, `chat_run_executor.py` persists the assistant message, backfills a and launches a background follow-up-question generator (`core/chat/followups.py`; results land in the message's `extra_data.follow_up_questions` and are fetched through -`GET /v1/chats/{chat_id}/messages/{message_id}/followups`). The frontend -collects ontology events in a standalone **Domain Ontology Governance** module -instead of model reasoning. The model draft continues to stream token by token. -If the committee changes the answer, the backend sends one `content_replace` -event, the frontend replaces the body in place, and the database stores only -the reviewed final answer. It persists `ontology_governance` with the assistant -message so the module remains available after a history refresh. +`GET /v1/chats/{chat_id}/messages/{message_id}/followups`). `usage` is cumulative billing data across all model calls in the turn. The context gauge instead consumes `context_usage`: `prompt_tokens + @@ -568,7 +560,7 @@ receiver reconciles bindings about every three seconds. Adding, disabling or del Subagent invocation rows show a status indicator, the agent avatar, its name and elapsed time. Clicking the row opens that invocation in a dedicated right-side Canvas tab instead of expanding its process beneath the message. The running animation is separate from the avatar. Ordinary tools retain their existing expand/collapse behavior. -The detail view reuses the main transcript’s tool batches, thinking and answer UI, including the same process visibility setting and collapsed defaults. It adds no separate parameter or execution-process sections. Completed invocation rows have no checkmark. It follows live updates until the reader scrolls up. Repeated invocations remain separate and coexist with file, ontology and plugin tabs. History uses saved data: at most 200 sub-steps and approximately 16,000 characters per string tool output; already truncated data cannot be recovered. +The detail view reuses the main transcript’s tool batches, thinking and answer UI, including the same process visibility setting and collapsed defaults. It adds no separate parameter or execution-process sections. Completed invocation rows have no checkmark. It follows live updates until the reader scrolls up. Repeated invocations remain separate and coexist with file and plugin tabs. History uses saved data: at most 200 sub-steps and approximately 16,000 characters per string tool output; already truncated data cannot be recovered. Skill loading, plugin loading and tools with an unambiguous connector owner display the corresponding capability-center logo. Plugin-contributed tool icons take precedence. The running indicator remains separate from the logo, with existing click and collapse behavior unchanged. Ordinary tools retain their category icons. diff --git a/document/en/modules/knowledge-base.md b/document/en/modules/knowledge-base.md index 32edc8ed..bad2cd7a 100644 --- a/document/en/modules/knowledge-base.md +++ b/document/en/modules/knowledge-base.md @@ -104,8 +104,8 @@ Figures inside documents are no longer discarded. Indexing runs through protocols with structured-output degradation, and the hard constraint that only a multimodal provider can be assigned to that role. Mapping: `summary` → `caption`, `ocr.full_text` merges into `text_content` (alongside the document's own caption), and - entities / relations / uncertainty are kept in `metadata.vision` for the ontology layer - and human review. **Leaving `vision` unassigned (with a text-only main model) is a + entities / relations / uncertainty are kept in `metadata.vision` for human + review. **Leaving `vision` unassigned (with a text-only main model) is a normal state**: images stay retrievable through their caption text, and configuring the role later backfills descriptions without re-parsing any document. 3. **Indexing** — each asset becomes one `row_type="image"` row whose `parent_chunk_id` diff --git a/document/en/modules/mcp-tools.md b/document/en/modules/mcp-tools.md index ce5d7776..9b337e42 100644 --- a/document/en/modules/mcp-tools.md +++ b/document/en/modules/mcp-tools.md @@ -261,7 +261,7 @@ MCP execution budgets exclude time waiting for user answers. Valid questions pau directory grants, dangerous-command policy, My Space write confirmation, or generic tool approval; - an allowed call receives a short-lived ticket bound to its tool name, call - id, argument hash, and spec key. `OntologyFilteredToolkit.call_tool()` checks + id, argument hash, and spec key. The toolkit's final `call_tool()` checks it again, while the real Read/Write/Edit/Bash host boundary consumes its path or command constraints; - a registered call that needs confirmation is denied in batch, sub-agent, diff --git a/document/en/modules/ontology-harness.md b/document/en/modules/ontology-harness.md deleted file mode 100644 index b2564a95..00000000 --- a/document/en/modules/ontology-harness.md +++ /dev/null @@ -1,292 +0,0 @@ -# Domain Ontology Governance and Validation - -Domain ontology validation adds an optional domain contract to the agent. It checks tools and -workflows before execution and independently reviews high-risk answers after they are generated. -Turning it off preserves the original chat path without ontology injection or gates. Thinking -status, content deltas, and tool events stream in real time whether or not review is activated. - -## User Setting - -In EE, an administrator must first grant `can_use_ontology_validation` through -**System Configuration → User Management / Team Management / Role Permissions**. -Without it, the ontology-validation entry is hidden and direct calls to the -personal settings endpoint are rejected. Open **Settings → Ontology Validation** after signing in. CE instance -administrators see this entry as **Ontology Governance**. The same page provides -the personal validation switch and global Domain Pack management. When an -active default Domain Pack is available, enable **Use Domain Ontology -Validation**. - -- Off: use the existing flow without ontology injection or gates. -- On with no matching workflow: unrelated domain rules are not applied. -- On with a checkpoint match: one independent reviewer checks the answer after the draft streams. -- On with a committee match: independent reviewers vote to pass, revise, or escalate. - -When review starts, the right panel opens automatically and streams validation -status and the revised answer while the original draft remains on the left. -Revision thinking is collapsed by default, and you can expand its tool calls -and new citations. Use the top-right panel button to collapse or reopen the -panel, or use the ontology-validation entry below a historical message to -revisit that result. The system doesn't overwrite the draft until you select -**Replace Original**. - -## Domain Packs - -A Domain Pack is a JSON domain contract with four layers: concepts, relations, -executable constraints, and workflows. CE instance administrators manage packs -under **Settings → Ontology Governance**. EE administrators continue to use -**Content Management → Ontology Governance**. - -An imported version starts as a working draft unless explicitly activated. Each Domain Pack can -have only one working draft, which administrators can update repeatedly. Publishing locks the -draft as an official version and activates it. Activation does not rewrite historical audit -evidence. CE doesn't bundle a business-domain pack, so a general-purpose -distribution never carries tool IDs or workflows for one industry. Instance -administrators import a Domain Pack that matches their installed tools. EE can -ship packs for its commercial capabilities. - -A workflow can define both kinds of entry point: - -- `triggers` activates it when the user's text matches a configured phrase. -- `asset_triggers` activates it when an invoked tool, skill, or sub-agent matches an asset ID or a - governed ontology tag. - -Governed asset tags use `ontology:ConceptId`. For example, tagging a data-check tool or report -skill with `ontology:QualityReport` activates the corresponding workflow when that asset is actually -invoked, even if the user never says a configured phrase such as "quality review." Ordinary display -tags do not activate ontology workflows. - -### Provide selectable tags - -For a tag to appear in the skill or sub-agent selector, the Domain Pack must declare both the -concept and its asset-trigger relationship. Defining the concept alone does not make it a runtime -trigger tag. - -1. Define the concept in `concepts`, such as `QualityReport`. -2. Select the asset kind in the target workflow's `asset_triggers`, and add - `ontology:QualityReport` to `tags_any`. -3. Activate the Domain Pack version. The skill and sub-agent forms then display the tag, linked - workflows, and review levels. - -The following example makes one tag available to both skills and sub-agents: - -```json -{ - "concepts": [ - {"id": "QualityReport", "name": "Quality report", "definition": "Data quality review report"} - ], - "workflows": [ - { - "id": "quality_review", - "review_level": "committee", - "asset_triggers": [ - {"kind": "skill", "tags_any": ["ontology:QualityReport"]}, - {"kind": "subagent", "tags_any": ["ontology:QualityReport"]} - ] - } - ] -} -``` - -## Runtime Validation - -Tool calls first pass through a deterministic gate that does not use an LLM. The gate can enforce -JSON Schema, prerequisite tools, and forbidden tools. A denial returns the exact reason and a -correction hint to the agent. Repeated denials require a strategy change and eventually trip a -circuit breaker. - -Runtime policy escalates monotonically. A turn can move from no match to checkpoint and then to -committee, but it cannot downgrade within the same run. An asset must be invoked to activate its -workflow; merely appearing in the catalog is not enough. The chat displays active workflows, -activation sources, tool-gate results, and committee progress, and the backend stores them as -audit evidence. - -Each governed turn receives one `governance_run_id`. The parent agent, children -created through `call_subagent`, and their tools share the same runtime object. -The same `pack_id + workflow_id` is activated once, whether text matches it -first or an asset tag matches it later. Every governed tool call still passes -its own deterministic gate because parameters and prerequisite evidence can -differ. The outer workflow exclusively claims final-answer review, and a child -doesn't start its own committee. Successful child tool calls join the outer -evidence trace with their `agent_id`, `sub_run_id`, and `parent_tool_id`, and -produce the same traceable citations as ordinary tool calls. A trusted child -can therefore satisfy required workflow tools without the outer trace -incorrectly reducing the execution to `call_subagent` alone. - -High-risk output also checks required workflow tools, content format, and citation evidence. The -committee runs only for workflows marked as high-risk in a Domain Pack; it is never invoked for -every tool call. - -The system runs deterministic output checks before the only checkpoint or -committee review round. If required tools, citations, or output structure are -missing, it sends structured violations back to the original agent first. The -agent continues in the same ReAct context, searches for missing evidence, calls -real tools, and returns one complete revised answer. The revision instruction -requires an evidence-gap comparison. If a new claim needs external facts, the -agent must retrieve them instead of only paraphrasing the draft. - -Each answer can produce at most one automatic revision and runs only one -independent review round. When the committee returns `revise`, the original -agent can use the remaining revision opportunity, but the system doesn't start -a second committee loop. Revision thinking streams into a separate, collapsed -thinking area. Tool calls, tool results, and revised content use distinct event -channels and don't enter the original answer. - -When human judgment is still required, the backend returns structured -`manual_review` JSON with the affected claim, rule ID, risk, and required human -check. The frontend renders these fields as **Domain Ontology Human Review** -cards instead of appending notice text to the answer. The revision remains -available as a working draft, but pending items aren't presented as -domain-approved conclusions. - -The original draft stays on the left. The revised answer, collapsed thinking, -additional tool calls, and human-review cards appear in the right panel, which -follows incoming SSE deltas to the bottom. If you scroll up, the panel stops -following so it doesn't interrupt reading. The system doesn't overwrite the -draft automatically. After you select **Replace Original**, it atomically -updates the message and preserves its accepted state in chat history. - -The same setting applies to regular chats, sub-agents, channel bots, automation, plans, batch -items, and autonomous loops. If the user opted in but the runtime policy cannot be built, the run -stops with an error instead of silently bypassing validation. - -When a user explicitly selects an `@sub-agent` in the composer, or enters the -strict Chinese command syntax `调用` or `请调用` + one unique complete sub-agent -name + an explicit task, the turn goes directly to that target. The main agent -doesn't run the domain workflow and dispatch a second child first. The request -boundary still creates the ontology runtime and activates it from the -sub-agent ID or tags. Skills and tools invoked inside the target share that -runtime, pass deterministic gates, and complete one checkpoint or committee -review at the highest level reached in the turn. - -## Build-time Validation - -Skills, MCP tools, plugins, and sub-agents are checked against active Domain Packs before they are -created, imported, installed, published, or saved from distillation. The gate validates tool -dependencies, forbidden tools, parameter schemas, and output contracts. Governed vocabulary tags -use `ontology:ConceptId`; unknown concepts are reported before saving, while ordinary tags remain -unrestricted. - -When an asset lacks a low-level tool required by a workflow, the validation report looks up the -bindable global MCP servers that provide it. The report displays the MCP and tool display names, -using Chinese names when configured, while retaining raw tool IDs in the complete report for -administrator troubleshooting. - -When you create or edit a skill, select MCP services under **Bound Tools (MCP)** instead of -entering low-level tool IDs. The backend reads the MCP's discovered `tools_json` and uses those -tools for build-time validation. A successful save records `ontology_tags`, -`ontology_workflows`, `mcp_servers`, and `allowed_tools` in `SKILL.md`. These bindings remain with -the skill when you edit, export, publish, or import it again. When the skill is enabled, its -declared MCP services are merged into that execution's capability scope without enabling other -unbound MCP services. - -The skill and sub-agent forms separate general tags from **Ontology governance tags**. The -ontology selector only lists tags from active Domain Packs that are linked to a workflow through -`asset_triggers.tags_any` for that asset kind. After selection, the form displays the workflows and -review levels that an actual invocation will trigger. - -MCP tool tags can come from a tool's `tags` or `ontology_tags`, or from its MCP server -configuration. Skills use governed tags in their own metadata. The same tags participate in -build-time checks and runtime `asset_triggers`. - -Administrators can set server-level tags in the MCP editor or assign tool-level tags using -`tool_name=ontology:ConceptId`. User and administrator forms for skills and sub-agents provide a -controlled ontology governance tag selector. - -## Governance - -CE instance administrators can complete governance under **Settings → Ontology -Governance** without the separate Content Management console, which isn't part -of CE. EE continues to expose **Content Management → Ontology Governance**. -Both entries use the same governance components and service logic, with these -operations: - -- import, validate, export, activate, and select default Domain Packs; -- inspect gate events, reviewer evidence, and latency; -- inspect closed-loop metrics for gates, reviews, drafts, candidate merges, and acceptance by source; -- automatically prefilter evolution drafts from repeated denials or review revisions, with a - manual trigger as well; -- add explicit user corrections on ontology-matched answers to the human-review queue; -- approve or reject evolution candidates and merge structured candidates into the working draft. - -### Edit a version from its details - -Open a Domain Pack's **Details**, and select **Edit This Module** from the -Overview, Concepts, Relations, Constraints, or Workflows tab. Each editor -changes only its selected module. All other modules are copied from the -version you are viewing. - -The editor uses structured forms, so you don't need to write JSON: - -- The overview form manages the pack description, injection budget, and - review thresholds. -- The concept form manages definitions, aliases, hierarchy, tags, and - controlled values item by item. -- The relation form connects existing concepts and manages predicates, - cardinality, and forbidden relations. -- The constraint form manages targets, messages, and correction guidance. It - builds JSON Schema from data types, required fields, enumerations, lengths, - and numeric ranges. -- The workflow form manages text and asset activation, required and forbidden - tools, output tags, and review levels. - -Before saving, the console validates the assembled Domain Pack and lists each -error at its exact path. The first edit creates one working draft with the -version number you confirm. Later edits to any module update that same draft, -so each save doesn't add another version. If a working draft already exists, -starting an edit from an official or archived version switches to the existing -draft and preserves changes in its other modules. - -Publishing locks the working draft as an official version and immediately -replaces the runtime version. Official and archived versions are read-only. To -make more changes, create the next working draft from the version you're -viewing. Version Management displays 10 rows per page by default and lets you -select 20 or 50 rows. Pagination never deletes history; the backend retains all -official versions. You can also discard an unwanted working draft before -publishing it. The Raw JSON tab remains a read-only verification and migration -view. - -Evolution never modifies or publishes the production version automatically. -An administrator reviews each candidate, merges an approved structured -candidate into the working draft, and then publishes that draft explicitly. -Conversation evidence and candidates are rejected when classified and are -sanitized for sensitive data and injection content before storage. A candidate -can't be merged twice. - -## API - -The principal endpoints are listed below. The CE Settings endpoints require -instance-administrator permission, with `ADMIN_TOKEN` as an operational -fallback. EE `/v1/admin/ontologies` endpoints continue to require `ADMIN_TOKEN` -or an equivalent content-management permission. - -| Method and path | Purpose | -|---|---| -| `GET/PATCH /v1/ontologies/settings` | Read or update the user opt-in setting | -| `GET /v1/ontologies/runtime/preview` | Preview rules matched by a task | -| `GET /v1/ontologies/tags?asset_kind=skill` | List selectable asset-trigger tags | -| `GET /v1/ontologies/governance/access` | Check whether the current CE user can manage global ontology assets | -| `/v1/ontologies/governance/*` | Manage CE packs, versions, evidence, and evolution from Settings | -| `GET /v1/admin/ontologies` | List Domain Packs and versions | -| `GET /v1/admin/ontologies/tags?asset_kind=skill` | List administrator asset-trigger tags | -| `GET /v1/admin/ontologies/metrics` | Inspect closed-loop governance metrics | -| `POST /v1/admin/ontologies/validate` | Validate Domain Pack JSON | -| `POST /v1/admin/ontologies/versions` | Import a new version | -| `PUT /v1/admin/ontologies/{pack_id}/draft` | Create or update the only working draft | -| `DELETE /v1/admin/ontologies/{pack_id}/draft/{version_id}` | Discard an unpublished working draft | -| `POST .../activate` | Publish a draft or activate a historical version | -| `POST /v1/admin/ontologies/build/validate` | Preflight a skill, tool, or sub-agent asset | -| `GET .../events`, `GET .../reviews`, `GET .../drafts` | Inspect runtime and evolution evidence | -| `POST /v1/chats/messages/{message_id}/ontology-revision/accept` | Replace the original message with its ontology revision | - -## Operations - -Run the database migration before starting a release that contains this feature. -If the user page shows that no pack is available, a CE instance administrator -must verify under **Settings → Ontology Governance** that a pack is enabled, -selected as the default, and has an active version. EE administrators perform -the same check in Content Management. - -CE upgrades don't write or replace business-domain packs. Imported Domain Pack -updates still preserve immutable versions: an administrator reviews the working -draft and explicitly publishes it before its asset triggers enter runtime. Packs -shipped with EE use the same draft and publication flow. diff --git a/document/en/modules/projects-myspace.md b/document/en/modules/projects-myspace.md index 2435daf7..ad5faf7d 100644 --- a/document/en/modules/projects-myspace.md +++ b/document/en/modules/projects-myspace.md @@ -65,7 +65,7 @@ The conversation takes the main width. A 360–440 px resource rail shows projec Scheduled tasks are filtered by the selected project and execution location. Each task has one conversation entry. Clicking it retrieves the latest run again; tasks with no runs show an empty state without starting execution. The settings button opens the existing task management page. Earlier runs remain available in task history. ### Page loading -While page code loads, the project list shows skeletons for the heading, actions, search field, and project cards. The first data request uses the same card skeletons; refreshing an existing list keeps its content visible. Selecting a project opens the shared conversation interface and loads project resources separately. Skeletons adapt to narrow screens and dark themes, and freeze the shimmer when the system requests reduced motion. +While page code loads, the project list shows skeletons for the heading, actions, search field, and project cards. The first data request uses the same card skeletons; refreshing an existing list keeps its content visible. Selecting a project opens the shared conversation interface and loads project resources separately: while they load, the project title and the resource rail show skeletons that match the final layout, and the instructions, files, memory, and scheduled-task cards each show line skeletons while their own data loads; if loading fails, the rail shows the reason and a retry button. Skeletons adapt to narrow screens and dark themes, and freeze the shimmer when the system requests reduced motion. In the project overview, the model and thinking-effort menu always opens downward. Its height is limited to the space below the trigger; longer lists do not flip it upward. The ordinary bottom composer chooses its direction from available space. Long lists scroll inside the menu. Resizing, viewport zoom and scrolling recalculate the bounds for all model and effort options. diff --git a/document/en/modules/sandbox.md b/document/en/modules/sandbox.md index 1fcf16e4..f49c2dbf 100644 --- a/document/en/modules/sandbox.md +++ b/document/en/modules/sandbox.md @@ -122,6 +122,7 @@ User returns → restore the session snapshot first → declare mounts again → ``` - A conversation reuses container files. Each Python/JS command starts a separate process and retains no variables. +- Every container is created with `EXECD_API_GRACE_SHUTDOWN=0s`. execd's default of 1 s holds a foreground command's stream open after it completes, which made each file write (its `chown`) and each setup command wait a full second. Events are flushed as they are written, so nothing is lost. Background status reads that get `command not found` within 2 s of launch are treated as "not started yet": execd before v1.0.22 replies before it registers the command. - A user-bound container can only be reused by the same user, after it is reset: leftover processes of the sandbox user are killed and everything outside mount points under `/workspace`, `$HOME`, `/tmp`, `/var/tmp` and `/dev/shm` is deleted, including execd's old command output; `scratch` and the home skeleton are rebuilt. Mount points (My Space, team spaces, skills, account credentials under `$HOME`) are detected from `/proc/self/mountinfo` and never entered. If the reset fails the container is destroyed instead. - The base runtime preserves personal and skill mounts. EE adds team mounts and permission checks. - Nothing is prewarmed. A user's container is created on the first tool call that needs one, then reused for that user's later chats once reset. Historical jupyter setting names refer to the user-session pool; no Jupyter server starts. diff --git a/document/zh-CN/README.md b/document/zh-CN/README.md index dd1765e7..6be29910 100644 --- a/document/zh-CN/README.md +++ b/document/zh-CN/README.md @@ -13,7 +13,6 @@ HugAgentOS 是一个开源、可自托管的 Agent Harness:你提出的问题 |------|------| | [产品简介](getting-started/introduction.md) | Durable Agent、运行形态、能力总览、技术栈与整体架构 | | [快速开始](getting-started/quick-start.md) | 选择 HugAgentOS Cloud、桌面客户端或一键安装,并启用 Personal Agent | -| [快速构建领域本体](getting-started/domain-ontology-quickstart.md) | 用最小 Domain Pack 跑通校验、发布、测试和工具绑定 | ## 部署 @@ -36,7 +35,7 @@ HugAgentOS 是一个开源、可自托管的 Agent Harness:你提出的问题 | 文档 | 说明 | |------|------| -| [总体架构](architecture/overview.md) | 分层架构、本体控制平面、一次对话的完整生命周期、容器拓扑 | +| [总体架构](architecture/overview.md) | 分层架构、一次对话的完整生命周期、容器拓扑 | | [后端架构](architecture/backend.md) | `src/backend/` 目录详解与路由注册表 | | [前端架构](architecture/frontend.md) | 入口分发、组件组、Zustand store、模块页面与应用中心 | | [数据模型](architecture/data-model.md) | 表分组速览、迁移链、CE/EE 表边界 | @@ -70,7 +69,6 @@ HugAgentOS 是一个开源、可自托管的 Agent Harness:你提出的问题 | 文档 | 说明 | |------|------| | [记忆系统](modules/memory.md) | L1/L2/L3 三层记忆、持久 Outbox、做法沉淀、脱敏与审计 | -| [领域本体治理与校验](modules/ontology-harness.md) | Domain Pack、工具门禁、交付前评审、演进草案人工审核 | | [知识库](modules/knowledge-base.md) | 自建知识库(混合检索)与外接知识库 | | [知识库 Wiki](modules/knowledge-base-wiki.md) | 抽取实体与概念,生成互链条目页与概念图谱 | diff --git a/document/zh-CN/api/overview.md b/document/zh-CN/api/overview.md index 8583aa0c..53971107 100644 --- a/document/zh-CN/api/overview.md +++ b/document/zh-CN/api/overview.md @@ -263,7 +263,6 @@ curl http://localhost:3000/api/v1/chat-runs/run_9f8e7d/steers \ | 个人空间 | `myspace_folders.py` | `/v1/myspace/folders` | `GET/POST /`、`POST /move-artifact` | 用户 | | 个人空间 | `artifacts.py` | `/v1/artifacts` | `GET /`、`GET /favorites`、`DELETE /{artifact_id}` | 用户 | | 记忆 | `memories.py` | `/v1/memories` | `GET /`、`DELETE /{memory_id}`、`GET/PATCH /settings`、`GET /profile`、`GET /graph` | 用户 | -| 领域本体 | `ontologies.py` | `/v1/ontologies`、`/v1/admin/ontologies` | 用户设置/运行时预览;Domain Pack 版本、构建预检、门禁/评审证据、闭环指标和演进草案治理 | 用户 / ADMIN | | 能力目录 | `catalog.py` | `/v1/catalog` | `GET /`(能力目录)、`PATCH /{kind}/{id}` | 用户 | | 能力目录 | `kb.py` | `/v1/catalog/kb` | `POST /`、`POST /{kb_id}/documents`、`GET /{kb_id}/chunks`(个人知识库) | 用户 | | 能力目录 | `marketplace.py` | `/v1/marketplace` | `GET /skills`、`POST /install`、`POST /submissions`(技能市场) | 用户 | diff --git a/document/zh-CN/architecture/backend.md b/document/zh-CN/architecture/backend.md index beb8c6bb..9bac1986 100644 --- a/document/zh-CN/architecture/backend.md +++ b/document/zh-CN/architecture/backend.md @@ -11,7 +11,7 @@ src/backend/ ├── api/ # FastAPI 应用、中间件与 CE 路由 ├── edition_ee/ # 商业版路由、License、Team/RBAC、ORM、Dify 与服务实现 ├── orchestration/ # 对话编排:Run 执行器、工作流、策略、引用、调度器 -├── core/ # 领域核心:17 个子模块(auth/llm/db/ontology/services/...) +├── core/ # 领域核心:17 个子模块(auth/llm/db/services/...) ├── prompts/ # 提示词装配运行时 + 文件兜底文本 ├── extensions/ # 随包扩展能力 │ ├── skills/ # default / marketplace @@ -91,8 +91,6 @@ src/backend/ 配置域:`model_config`(DB 化模型配置,带缓存)、`system_config`(服务配置)、`mcp_service`(MCP 服务器配置)、`log_service`(可观测性日志异步落库)。 -本体域:`ontology_service`(用户开关、版本与运行时裁剪)、`ontology_evolution_service`(证据预筛、脱敏、人审草案与未激活版本物化);`core/ontology/` 收口四层 Schema、构建校验、确定性门禁、工具过滤和提示词渲染。 - 商业版 EE:`team_service` / `team_folder_service` / `sso_sync`(团队与 SSO 同步)、`distillation_service`(技能蒸馏)、`sandbox_rebuild_service` + `cube_template_builder`(持久沙箱模板重建)、`security_service`(安全管理台只读聚合)。 ### core/memory — 三层记忆系统 @@ -188,7 +186,7 @@ CE 路由位于 `api/routes/v1/`;已物理拆分的商业路由位于 `edition |---|---| | 会话与流式 | `chats.py`(流式 SSE 主入口)、`chat_runs.py`、`chat_shares.py`、`summary.py`、`classify.py`、`memories.py` | | 内容与文件 | `content.py`、`file_upload.py`、`file_parse.py`、`artifacts.py`、`myspace_folders.py`、`kb.py`(+ `kb_models.py` 纯 Pydantic 模型)、`projects.py` | -| 能力与配置 | `catalog.py`、`models.py`、`config.py`、`marketplace.py`、`me_capabilities.py`(能力中心自助)、`ontologies.py`(用户开关、版本治理与证据闭环)、`agents.py`、`plans.py`、`automations.py`、`meta.py`(版本 / 能力位探针,无鉴权) | +| 能力与配置 | `catalog.py`、`models.py`、`config.py`、`marketplace.py`、`me_capabilities.py`(能力中心自助)、`agents.py`、`plans.py`、`automations.py`、`meta.py`(版本 / 能力位探针,无鉴权) | | 用户与认证 | `users.py`、`me.py`、`api_keys.py`、`mock_sso.py`(开发态) | | 内容管理台(商业版 EE,`/admin` 后端) | `admin_skills.py`、`admin_prompts.py`、`admin_kb.py`、`admin_agents.py`、`admin_mcp_servers.py`、`admin_marketplace.py`、`admin_skill_drafts.py`、`admin_sandbox.py`、`admin_logs.py`、`admin_usage_logs.py`、`admin_billing.py`、`admin_chat_history.py` | | 系统管理台(商业版 EE,`/config` 后端) | `config_users.py`、`config_teams.py`、`config_invites.py`、`config_security.py`、`config_verify.py`、`config_license.py`、`service_configs.py` | diff --git a/document/zh-CN/architecture/data-model.md b/document/zh-CN/architecture/data-model.md index 126931cd..d1ddaa9e 100644 --- a/document/zh-CN/architecture/data-model.md +++ b/document/zh-CN/architecture/data-model.md @@ -20,8 +20,7 @@ core/db/ │ ├── logs.py # 工具/子智能体/技能调用日志、审计 │ ├── memory.py # 画像记忆、记忆审计、脱敏规则 │ ├── datasource.py # 数据源、元数据治理与黄金 SQL -│ ├── site.py # 站点、连接器与渠道配置 -│ └── ontology.py # Domain Pack 版本、门禁、委员会与演进草案 +│ └── site.py # 站点、连接器与渠道配置 ├── repository/ # 仓储层:agent/artifact/audit/catalog/chat/kb/team/user ├── model_repository.py # 模型供应商/角色指派仓储 └── edition_tables.py # CE/EE 建表边界单一真源 @@ -135,16 +134,6 @@ core/db/ | `skill_call_logs` | 技能触发记录(view / run_script / auto_load) | | `audit_logs`(商业版 EE) | 用户态关键操作审计 | -### 领域本体(models/ontology.py) - -| 表 | 用途 | -|---|---| -| `ontology_packs` | Domain Pack 身份、启用/默认标记与激活版本指针 | -| `ontology_pack_versions` | 不可变的四层本体版本内容与校验报告 | -| `ontology_enforcement_events` | 构建、工具和输出门禁的追加式证据 | -| `ontology_review_runs` | checkpoint / committee 评审裁决、证据与延迟 | -| `ontology_drafts` | 待人工审查的演进候选及物化版本指针 | - ## Alembic 迁移机制 - **商业版主链**:`src/backend/alembic/versions/` 下 93 个迁移,从初始建表一路演进(含 MCP 迁往 streamable-http、办公 MCP 下线改技能等结构性变更)。常用命令:`alembic upgrade head`、`make migrate-new msg="..."`(autogenerate 基于 `core/db/models` 元数据); @@ -178,7 +167,6 @@ user_feedback # 问题反馈审核 / | 主题 | 路径 | |---|---| | 共享 ORM / EE ORM | `src/backend/core/db/models/`、`src/backend/edition_ee/db/models/` | -| 本体仓储 | `src/backend/core/db/repository/ontology.py` | | 问题反馈模型 / 仓储 | `src/backend/edition_ee/db/models/feedback.py`、`edition_ee/db/repository/feedback.py` | | 引擎与启动建表 | `src/backend/core/db/engine.py` | | 仓储层 | `src/backend/core/db/repository/` | diff --git a/document/zh-CN/architecture/frontend.md b/document/zh-CN/architecture/frontend.md index 3726f7d9..9733f6da 100644 --- a/document/zh-CN/architecture/frontend.md +++ b/document/zh-CN/architecture/frontend.md @@ -103,7 +103,6 @@ SSE 流式不走 `api.ts` 的 JSON 通道,由 `hooks/useStreaming.ts` 直接 | `useChatInit` | 应用启动时的会话初始化与活动 run 恢复 | | `usePlanMode` | 计划模式 SSE 消费器(首次执行与续播共用) | | `usePageConfig` | 按 dot-path 读取页面配置(如 `branding.product_name`) | -| `useStallDetector` | 流静默超时检测(卡顿提示) | | `useDelayedFlag` | 防闪烁的延迟布尔量(延迟显示 + 最短保持) | ## 流式正文显示 diff --git a/document/zh-CN/architecture/overview.md b/document/zh-CN/architecture/overview.md index 06677e4c..137abae0 100644 --- a/document/zh-CN/architecture/overview.md +++ b/document/zh-CN/architecture/overview.md @@ -4,7 +4,7 @@ HugAgentOS 是一个开源、可自托管的 Agent Harness,提供 HugAgentOS Cloud(全容器化服务端)与 HugAgentOS Local(桌面本机)两种运行形态。前端为 React 单页应用,后端为 FastAPI 异步服务,智能体底座为 AgentScope 2.0 的 ReActAgent,工具生态通过 MCP(Model Context Protocol)以独立进程提供,代码执行落在可插拔的沙箱驱动上。平台以「开源社区版(CE)+ 商业版(EE)」双形态发行,两版共用一套代码,通过路由注册表、建表边界与 license 能力位三道接缝裁剪(详见 [版本与授权](../editions/overview.md))。 -本文从五个视角描述系统:分层架构、本体驱动的企业可信控制平面、一次对话请求的完整生命周期、容器拓扑、关键设计决策。更深入的模块拆解见 [后端架构](./backend.md)、[前端架构](./frontend.md) 与 [数据模型](./data-model.md)。 +本文从四个视角描述系统:分层架构、一次对话请求的完整生命周期、容器拓扑、关键设计决策。更深入的模块拆解见 [后端架构](./backend.md)、[前端架构](./frontend.md) 与 [数据模型](./data-model.md)。 ## 技术栈一览 @@ -40,7 +40,7 @@ HugAgentOS 是一个开源、可自托管的 Agent Harness,提供 HugAgentOS C ┌──────────────────────────────▼─────────────────────────────────────┐ │ 智能体执行 │ │ core/agent_runtime:runner.astream_chat_workflow、路由策略、 │ -│ 记忆 / 本体 / 进化集成 │ +│ 记忆 / 进化集成 │ │ core/llm:factory/build → AgentScope ReActAgent、工具、上下文、引用 │ │ orchestration/:plans 计划模式 · loops 自主循环 · automations 自动化 │ │ personal_agent/:身份、任务队列、调度、渠道、设备连接、工作区 │ @@ -56,66 +56,6 @@ HugAgentOS 是一个开源、可自托管的 Agent Harness,提供 HugAgentOS C 依赖方向自上而下单向:`api → features / orchestration / personal_agent → core`;`core` 不依赖上层,`integrations/` 承载钉钉、飞书、邮箱等外部系统接入。MCP 服务器是独立进程,与后端只通过 HTTP 通信,崩溃互不影响。 -## 领域本体驱动的企业可信控制平面 - -HugAgentOS 将领域本体作为可执行的控制平面,而不只是一个知识库, -从而把智能体运行时扩展为面向企业的可信 AgentOS。版本化的领域包 -(Domain Pack)可从受控概念和关系,逐步延伸到不变式、Action 契约、 -工作流、角色、权限和证据要求,让技能、编排和记忆三个引擎使用同一套 -业务语义。 - -下图是在当前 Harness 基础上渐进集成的目标企业架构。现有的确认门、 -结构化审查结论、审计记录与版本管理构成底座,本体资产和执行约束则逐步 -接入;这一机制用于增强合规与证据驱动审查,不宣称能消除所有自由文本幻觉。 - -```mermaid -flowchart TB - U[企业用户与多端渠道] --> API[Web / 桌面端 / API] - API --> RUN[ChatRun 与流式工作流] - - ONTO[领域本体控制平面
概念 · 关系 · 不变式
Action 契约 · 角色 · 权限] - ONTO --> BUILD[构建时校验
技能 · 工具 · 子智能体] - ONTO --> BOOT[领域启动注入
语义对齐 · 相关规则] - - subgraph HARNESS[三引擎 Agent Harness] - direction LR - SKILL[技能引擎] <--> ORCH[编排引擎] - ORCH <--> MEMORY[记忆引擎] - end - - RUN --> ORCH - BUILD --> SKILL - BOOT --> ORCH - BOOT --> MEMORY - ORCH --> PLAN[候选计划或 Action] - - PLAN --> RULE[确定性本体规则门] - ONTO --> RULE - RULE -->|低风险且合规| EXEC[受控执行] - RULE -->|检查点或高风险| REVIEW[证据驱动复核] - REVIEW -->|通过| EXEC - RULE -->|违规| REVISE[拒绝并返回证据与修复指引] - REVIEW -->|修订或升级| REVISE - REVISE --> ORCH - - EXEC --> CAP[MCP · Skills · 沙箱 · RAG] - EXEC --> AUDIT[可追溯审计与回放] - REVISE --> AUDIT - AUDIT --> EVOLVE[受治理的本体演进提案] - EVOLVE -. 人工审批 · 版本化 · 可回滚 .-> ONTO -``` - -可信控制平面遵循三项原则,避免把每一步都变成高成本的委员会审查: - -1. **确定性优先。** 先执行可机械判定的不变式、权限和 Action 契约。 -2. **按风险复核。** 仅检查点、高风险动作或证据不确定时进入深度复核; - 合规的低风险动作可直接通过执行门。 -3. **有证据再演进。** 违规、审批和执行结果进入审计链;本体变更提案必须经过 - 人工批准、生成新版本并保留回滚路径。 - -本体与 RAG 相互补充:本体规定智能体可以做什么、需要满足哪些约束, -RAG 则提供支撑决策所需的文档与证据。 - ## 一次对话请求的完整生命周期 入口为 `POST /v1/agents/responses`(`src/backend/api/routes/v1/chats/__init__.py`),返回 `text/event-stream`。 diff --git a/document/zh-CN/deployment/README.md b/document/zh-CN/deployment/README.md index cb1ce781..3dbd1907 100644 --- a/document/zh-CN/deployment/README.md +++ b/document/zh-CN/deployment/README.md @@ -12,6 +12,8 @@ HugAgentOS 支持多种部署方式,从「个人单机零依赖尝鲜」到「 | **无 Docker 一键安装** | 个人单机尝鲜、二次开发体验;一条命令装好即用 | 不需要 | SQLite | 否(单用户) | [quick-install.md](quick-install.md) | | **Docker Compose** | 团队 / 生产的标准形态,多用户、全功能 | 需要 | PostgreSQL | 是 | [docker-compose.md](docker-compose.md) | +已有外部 PostgreSQL、Redis、MinIO 且不能新增持久卷的容器云,使用[外部服务 Kubernetes 部署](external-services-kubernetes.md)。真实空间挂载需要先通过平台权限及网络预检。 + 跨平台与参考: | 文档 | 说明 | diff --git a/document/zh-CN/deployment/desktop-local-mode.md b/document/zh-CN/deployment/desktop-local-mode.md index 2459a1ff..3f6b9294 100644 --- a/document/zh-CN/deployment/desktop-local-mode.md +++ b/document/zh-CN/deployment/desktop-local-mode.md @@ -43,6 +43,14 @@ JX_DEFAULT_SERVER_BASE=https://你的后端 npm run build:hybrid-only 进程,后端不再写死端口。升级换端口后,旧端口上的本机服务按记录的进程号与安装目录回收, 不会误伤同机其它产品。 +## 混合模式:本机到云端的连接 + +混合模式下,本机后端拉取云端能力清单、下载技能与插件包、同步工作副本时共用一个连接池, +相邻请求复用已建立的 TLS 连接。TCP 连接加 TLS 握手默认给 15 秒,可用环境变量 +`HUGAGENT_CLOUD_CONNECT_TIMEOUT_S` 调整;连接或握手超时会自动重试一次(此时请求尚未发出, +重试是安全的),读取超时不重试。仍然失败时,同步状态显示「连接云端超时,请检查网络或代理 +设置后重试」这类提示,原始异常只写入本机后端日志。 + ## 发行目标 | 目标 | 私有运行时 | 业务数据 | diff --git a/document/zh-CN/deployment/external-services-kubernetes.md b/document/zh-CN/deployment/external-services-kubernetes.md new file mode 100644 index 00000000..8c6aaf49 --- /dev/null +++ b/document/zh-CN/deployment/external-services-kubernetes.md @@ -0,0 +1,29 @@ +# 源码版容器云部署:外部服务与真实空间挂载(商业版 EE) + +HugAgentOS 源码支持外部 PostgreSQL、Redis、MinIO,以及 Kubernetes OpenSandbox。部署清单和精简镜像交付入口由 `deploy` 分支的 `deploy_kit/kubernetes/external.py` 维护,完整步骤见该分支同名部署文档。不要使用全栈 Compose 清单在受限平台创建数据库或存储卷。 + +## 配置和运行 + +使用 deploy 分支的 external init/render/preflight 流程生成配置,将源码构建的 backend、mcp、frontend 镜像 digest 填入 site.json;沙箱 server 和 space-mount 使用同套经 patch_upstream 校验的版本。不可只替换未经适配的上游 server 镜像。 + +容器通过 Kubernetes ConfigMap/Secret 的 envFrom 获得环境变量,不需要镜像内的 .env。设置 PYTHON_DOTENV_DISABLED=1;真实凭据不进入源码或镜像。主要配置是 DATABASE_URL、REDIS_URL、S3 端点/桶/凭据、DEPLOY_PROFILE=kubernetes、STORAGE_TYPE=s3、SESSION_STORE=redis、MEM0_VECTOR_PROVIDER=pgvector、HUGAGENT_SPACE_MOUNT_MODE=sidecar,以及所需模型端点。PERSONAL_AGENT_ENABLED=false、MODEL_GATEWAY_ENABLED=false;本部署不包含知识库组件或模型网关。使用 external 渲染器生成完整变量,避免只复制此处摘要漏项。 + +secrets.json 中 PostgreSQL 和 Redis 按管理员交回的字段分别填写:DB_HOST、DB_PORT、DB_NAME、DB_USER、DB_PASSWORD,REDIS_HOST、REDIS_PORT、REDIS_PASSWORD、REDIS_DB;按需加 DB_SSLMODE(disable/allow/prefer/require;需要 CA 证书的校验模式暂不支持)、REDIS_TLS(true/false)、REDIS_USERNAME。渲染器对特殊字符编码后生成 DATABASE_URL、REDIS_URL,密码无需手工转义;直接填写完整 URL 仍然兼容。应用不给 Redis 键加前缀,必须使用独立库号,不能与其他系统共用同一个库号。 + +单个已有 PostgreSQL 数据库承载业务表、许可证状态和 pgvector 记忆(启动时自动建表和按用户检索所需的索引);需要预装 vector 扩展并允许 owner 执行迁移。MinIO 保存文件、artifact 元数据和公开手册/品牌资源。更新 Secret/ConfigMap 后重新渲染并滚动替换 Pod,不在容器内手改配置。 + +许可证存于 PostgreSQL。每个进程每 LICENSE_DB_REFRESH_SECONDS(默认 30 秒)重新读取一次,请求本身不访问数据库;数据库短暂不可用时沿用最近一次确认的结果,至多 LICENSE_DB_OUTAGE_GRACE_SECONDS(默认 300 秒),冷启动或更长的中断仍拒绝放行。 + +每个 session 的沙箱临时目录独立。同一用户的“我的空间”通过真实 FUSE 挂载共享持久文件;用户和团队空间授权沿用现有权限检查。挂载丢失时拒绝执行,不降级成下载副本。技能在每次命令前按当前完整授权视图核对,视图有变化才重新下发,沙箱复用给新会话时也重新下发,保留执行权限;相对路径可直接使用,禁用、删除或撤权的包从当前技能目录清除,未变目录和文件保持不动。撤权不追溯删除用户之前复制到其他位置的数据。Kubernetes 使用云端技能来源,不支持混用桌面 capability publication。Kubernetes 不支持现有宿主机第三方凭据目录挂载;检测到这种配置会明确报错,相关第三方集成未纳入本次部署验收。 + +## 平台必须实际通过的条件 + +需要 Kubernetes 1.29+、OpenSandbox CRD/namespace 管理权限、有效的 NetworkPolicy、受控 helper 的 privileged/FUSE/Bidirectional mountPropagation。平台禁止特权时设 `HUGAGENT_SPACE_MOUNT_MODE=sync`(deploy 渲染器 `space_transport: "sync"`):不挂载,backend 在命令前后经沙箱 HTTP 文件接口增量同步空间文件,不需要 privileged/FUSE,一致性降为命令边界一致,见 `internal design docs`。运行代码的主容器保持非特权,MinIO 凭据只进入 helper。没有持久卷并不等于不需要这些挂载权限;若平台只有普通 Pod 权限且禁止 helper 能力,就无法满足真实挂载要求。 + +先运行 external preflight,真实检查 PG/Redis/MinIO、FUSE 读写/只读/凭据隔离、会话 NetworkPolicy,再执行迁移和应用上线。生产平台内网目前不可访问,权限未确认;本地验证不能代替目标平台预检。 + +## 本次本地验证 + +已在隔离 Kubernetes 环境验证两个分支的真实 PG 授权→产品 Bash→沙箱执行、会话临时文件隔离、共享个人空间和复用检查。源码版额外验证真实数据库私有技能发布/执行,以及拒绝访问其他用户私有技能。还验证挂载 helper/控制面重启、故障拒写、恢复、只读空间和非特权执行容器。 + +这些验证不等同于目标平台已部署,也未验证目标模型端点的真实回复。上线前仍需在目标环境完成登录、模型回复、两个 session 执行代码、文件持久化与隔离验收。代码未自动提交或推送。 diff --git a/document/zh-CN/development/frontend.md b/document/zh-CN/development/frontend.md index 9c2a8d93..3849dffa 100644 --- a/document/zh-CN/development/frontend.md +++ b/document/zh-CN/development/frontend.md @@ -44,7 +44,7 @@ src/frontend/src/ │ ├── citation/ common/ config/ docs/ file/ kb/ │ └── lab/ memory/ myspace/ projects/ settings/ share/ sidebar/ tool/ apidoc/ ├── hooks/ # useChatInit / useChatActions / useStreaming / usePlanMode -│ # usePageConfig / useDelayedFlag / useStallDetector +│ # usePageConfig / useDelayedFlag ├── stores/ # Zustand store(authStore、chatStore、catalogStore、 │ # editionStore、uiStore、settingsStore、fileStore、kbStore、 │ # agentStore、automationStore、canvasStore、 diff --git a/document/zh-CN/editions/build-ce.md b/document/zh-CN/editions/build-ce.md index 966063fa..df0c5e3e 100644 --- a/document/zh-CN/editions/build-ce.md +++ b/document/zh-CN/editions/build-ce.md @@ -29,8 +29,7 @@ glob 模式(相对仓库根),命中即不拷贝。覆盖: - **后端 EE 模块**:完整的 `edition_ee/**` 实现根(团队/RBAC、SSO、License 验签与闸门、EE ORM、Dify 集成),以及云存储(`core/files/storage/s3.py`、`oss.py`)、Cube 沙箱 provider、记忆审计、技能蒸馏等 EE 服务; - **EE 路由**:`api/routes/v1/admin_*.py`、`config_*.py`、`audit.py`、`auth.py`、`team_files.py`、`service_configs.py`、`data_sources.py`、`db_metadata.py`、`gateway_*.py`; - **行业 MCP**:`extensions/mcp_servers/query_database_mcp/**`、`ai_chain_information_mcp/**`; -- **行业专属前端与种子**:产业链 Canvas、资讯 / 企业画像渲染器、行业 API 客户端、图标资源, - 以及依赖企业画像工具的内置本体示例包; +- **行业专属前端与种子**:产业链 Canvas、资讯 / 企业画像渲染器、行业 API 客户端、图标资源; - **主仓 alembic 链整体**(`alembic/versions/**`,CE 用 overlay 的独立链,见下文); - **10 个行业 / 品牌技能**(`extensions/skills/marketplace/` 下,前 5 个硬依赖 EE 行业 MCP,后 5 个含品牌域文案); - **EE 强耦合测试**、`tests/licensing/**`; @@ -118,7 +117,7 @@ CE 前端不按工具名猜测远程 MCP 的业务结构。用户接入的远程 测试目录只允许保存“不得出现”的负向契约断言 [4/7] 二进制门禁 PNG/PDF/DOCX 必须匹配经人工/OCR 审阅后的 path + SHA-256 白名单 [5/7] 品牌门禁 文本逐行正则 0 命中 + 全量文件「路径」扫描(含二进制资产文件名; - 另有路径专用模式拦商业字体文件本体) + 另有路径专用模式拦商业字体文件) [6/7] LICENSE 闸门 overlay LICENSE 仍是占位文本(含 NOTE TO MAINTAINERS 标记)时拒绝生成 [7/7] 自检 --import-check / --pytest-check / --frontend-check(可选) [8/8] 清残留 自检留下的 __pycache__ / .pytest_cache / ce_selfcheck.db / node_modules / dist / 再生 lock diff --git a/document/zh-CN/getting-started/domain-ontology-quickstart.md b/document/zh-CN/getting-started/domain-ontology-quickstart.md deleted file mode 100644 index e2324675..00000000 --- a/document/zh-CN/getting-started/domain-ontology-quickstart.md +++ /dev/null @@ -1,212 +0,0 @@ -# 快速构建领域本体 - -> 最后更新:2026-07-21 | -> [English](../../en/getting-started/domain-ontology-quickstart.md) - -本指南帮助你在 20 分钟内构建并发布第一个 Domain Pack,让 HugAgentOS 能够用 -受控概念、关系、约束和工作流校验领域任务。示例不依赖外部 MCP,适合先验证 -完整闭环,再逐步绑定真实工具和技能。 - -## Domain Pack 包含什么 - -HugAgentOS 的领域本体是可执行的业务契约,而不是只用于检索的术语表。一个 -Domain Pack 由以下部分组成: - -- **基本信息与配置**:领域包 ID、版本、注入预算、熔断阈值等运行参数; -- **概念**:领域对象、别名、定义、层级、受控取值和风险等级; -- **关系**:概念之间允许或禁止的连接,以及基数要求; -- **约束**:工具参数或最终输出必须满足的 JSON Schema、证据和前置工具要求; -- **工作流**:文本或资产触发条件、必需/禁止工具、输出标签和评审级别。 - -> [!TIP] -> 第一个领域包只覆盖一个高价值任务。先用 2–5 个概念、1 个工作流和 1 条 -> 可执行约束跑通闭环,再扩展词表和规则。 - -## 第一步:选择一个可验收的任务 - -先把业务目标写成一句可以测试的话。例如:“分析一次采购申请中的供应商风险, -并给出结构完整的风险摘要。”随后明确以下三类内容: - -1. 列出任务必须理解的对象,例如采购申请、供应商风险; -2. 写出必须满足的铁律,例如风险摘要不得少于必要的说明; -3. 选择评审级别:`none` 不额外评审,`checkpoint` 使用单评审员, - `committee` 用于高风险、多评审员场景。 - -## 第二步:导入最小 Domain Pack - -使用管理员账号登录后,打开「设置 → 本体治理」,选择「导入 Domain Pack」, -粘贴下面的 JSON。这个示例用文本和技能标签触发采购风险工作流,并要求最终摘要 -至少包含 120 个字符。 - -```json -{ - "schema_version": "1.0", - "pack_id": "procurement_risk", - "name": "采购风险领域包", - "version": "1.0.0", - "domain": "procurement-risk", - "description": "约束采购申请中的供应商风险识别与摘要交付。", - "config": { - "injection_enabled": true, - "max_concepts": 8, - "token_budget": 1600, - "committee_size": 3, - "repeated_denial_threshold": 2, - "circuit_breaker_threshold": 5, - "allow_unresolved_tools": false - }, - "concepts": [ - { - "id": "ProcurementRequest", - "name": "采购申请", - "aliases": ["采购单"], - "definition": "包含采购标的、预算、申请部门和候选供应商的业务申请。", - "tags": ["采购"], - "risk": "low" - }, - { - "id": "SupplierRisk", - "name": "供应商风险", - "aliases": ["供应风险"], - "definition": "可能影响供应商履约、合规或持续经营能力的风险。", - "closed_values": ["低", "中", "高", "待核验"], - "tags": ["风险"], - "risk": "medium" - } - ], - "relations": [ - { - "id": "request_has_supplier_risk", - "subject": "ProcurementRequest", - "predicate": "包含", - "object": "SupplierRisk", - "description": "采购申请可以关联一个或多个供应商风险。", - "min_cardinality": 0, - "forbidden": false - } - ], - "constraints": [ - { - "id": "procurement_summary_complete", - "name": "采购风险摘要必须完整", - "target": { - "kind": "output", - "output_tag": "procurement_risk_summary" - }, - "schema": { - "type": "string", - "minLength": 120 - }, - "concept_id": "SupplierRisk", - "requires_citations": false, - "prerequisite_tools": [], - "mode": "enforce", - "risk": "medium", - "message": "采购风险摘要过短,无法支持业务复核。", - "suggestion": "补充风险事实、影响、未知项和建议的核验动作。", - "enabled": true - } - ], - "workflows": [ - { - "id": "procurement_risk_review", - "name": "采购风险评审", - "triggers": [ - "采购风险", - "供应商风险", - "procurement risk", - "supplier risk" - ], - "asset_triggers": [ - { - "kind": "skill", - "tags_any": ["ontology:SupplierRisk"] - } - ], - "required_tools": [], - "forbidden_tools": [], - "output_tags": ["procurement_risk_summary"], - "review_level": "checkpoint", - "risk": "medium" - } - ] -} -``` - -保留「导入后立即发布」为关闭状态,然后选择「校验并导入」。系统会先检查 ID、 -版本、概念引用、JSON Schema 和工具引用;有错误时会显示精确字段路径。 - -## 第三步:检查并发布版本 - -导入成功后,先检查工作草稿,再把它发布到运行时。按以下顺序操作: - -1. 在领域包列表中打开「详情」,核对总览、概念、关系、约束和工作流; -2. 打开「版本管理」,发布 `1.0.0` 工作草稿; -3. 回到领域包列表,打开「启用」开关; -4. 需要让它成为默认领域包时,再打开「默认」开关。 - -正式版本发布后会变为只读。后续修改会创建新的工作草稿,不会覆盖正在运行的 -版本。 - -## 第四步:启用并测试本体校验 - -在「设置 → 本体治理」打开「使用领域本体校验」,然后新建会话并输入下面的测试 -问题: - -```text -请分析这次采购申请的供应商风险,并输出一份风险摘要。 -``` - -任务命中 `采购风险` 或 `供应商风险` 后,聊天界面会显示工作流激活事件,并在 -回答完成后执行 `checkpoint` 评审。你可以在本体治理页面查看门禁事件和评审记录。 - -再执行两组负向测试,确认规则只在正确范围内生效: - -- 输入与采购无关的问题,确认不会加载该工作流; -- 要求极短的采购风险结论,确认系统能够指出输出约束或生成优化稿。 - -## 第五步:绑定真实工具和技能 - -最小示例跑通后,把本体从“输出检查”扩展到“行动门控”。开始前先在 MCP 管理中 -确认真实工具 ID 和参数 Schema,再进行以下配置: - -1. 在工作流的 `required_tools` 中加入完成任务必须调用的工具; -2. 在约束的 `target.tool` 中填写真实工具 ID,并用 `schema` 约束输入参数; -3. 需要固定执行顺序时,在 `prerequisite_tools` 中声明前置工具; -4. 在 `asset_triggers` 中用工具 ID 或 `ontology:ConceptId` 标签触发工作流; -5. 编辑技能或子智能体时,从「本体治理标签」选择器绑定相同标签。 - -如果领域包引用了尚未注册的工具,默认校验会失败。仅在迁移或分阶段接入时临时将 -`allow_unresolved_tools` 设为 `true`,并在生产发布前恢复严格校验。 - -## 发布前验收清单 - -一个可投入使用的领域包至少应通过以下检查: - -- 每个概念都有清晰定义,别名不会与其他概念冲突; -- 每条关系引用的起点和终点概念都存在,层级没有循环; -- 工具约束使用运行环境中的真实工具 ID 和参数名; -- 工作流至少有一个文本触发器或资产触发器; -- 必需工具和禁止工具没有交集; -- 普通问题不会误触发,高风险问题能进入预期评审级别; -- 违规消息说明“为什么失败”,修正建议说明“下一步怎么做”; -- 新版本先以工作草稿验证,再由管理员显式发布。 - -## 常见问题 - -下面列出第一次构建 Domain Pack 时最常见的失败原因和处理方式。 - -| 现象 | 处理方式 | -|---|---| -| 提示 `unknown tool reference` | 核对 MCP 已发现的真实工具 ID,或先移除工具依赖完成最小闭环 | -| 标签没有出现在技能选择器 | 同时定义对应概念,并在工作流 `asset_triggers.tags_any` 中使用该标签 | -| 对话没有命中工作流 | 确认领域包已发布、启用且设为默认,并检查 `triggers` 是否出现在用户问题中 | -| 修改后运行时没有变化 | 正式版本只读;编辑工作草稿并发布新版本 | -| 规则频繁误拦截 | 收窄触发词、降低规则模式或风险级别,并用审计记录定位误命中来源 | - -## 下一步 - -完成第一个领域包后,继续阅读[领域本体治理与校验](../modules/ontology-harness.md), -了解工具门禁、资产标签、委员会评审、版本治理和本体演进闭环。要绑定外部能力, -再阅读 [MCP 工具系统](../modules/mcp-tools.md)和 -[技能系统](../modules/agent-skills.md)。 diff --git a/document/zh-CN/getting-started/introduction.md b/document/zh-CN/getting-started/introduction.md index 8193115f..d4da9899 100644 --- a/document/zh-CN/getting-started/introduction.md +++ b/document/zh-CN/getting-started/introduction.md @@ -39,7 +39,6 @@ Personal Agent;商业版补充团队协作、SSO、审计合规、行业数据 | 沙箱执行 | OpenSandbox 会话、环境复用与快照恢复,或本地子进程沙箱;工作流后台作业以受管进程运行,可按租约接管 | CE(CubeSandbox 与团队挂载为 EE) | | 记忆与个人进化 | L1 画像、L2 做法记忆(Milvus,可选)、L3 关系图谱(Neo4j,可选);持久 Outbox 写入;进化建议逐条审批 | CE(记忆审计为 EE) | | 知识库 | 文档分块、向量 + 关键词混合检索、私有知识库、知识库 Wiki | CE(公共库管理与 Dify 对接为 EE) | -| 领域本体治理 | Domain Pack 导入与版本发布、工具执行前规则检查、高风险答案独立评审、演进草案人工审核 | CE | | 自动化 | 自然语言创建定时任务、Cron 调度、失败重试 | CE | | 站点与应用托管 | 对话建站并一键发布;应用独立数据库与 MCP 托管 | CE(项目转换后的站点协作为 EE) | | 数据画布与产物 | Univer 在线表格、图表、报告、Office 文件、网页等产物与产物中心 | CE 个人编辑;多人协同为 EE | @@ -74,7 +73,7 @@ Personal Agent;商业版补充团队协作、SSO、审计合规、行业数据 │ ▼ 智能体执行(core/agent_runtime · core/llm) - ReActAgent · 上下文按步装配与压缩 · 记忆 / 本体 / 进化集成 + ReActAgent · 上下文按步装配与压缩 · 记忆 / 进化集成 外层编排(orchestration/):计划模式 · 自主循环 · 自动化 │ ┌────────────────┼──────────────────┐ @@ -111,7 +110,6 @@ Personal Agent(`personal_agent/`)复用同一套 Run、租约与沙箱机制 | 计划模式、自主循环、自动化 | `src/backend/orchestration/plans/`、`orchestration/loops/`、`orchestration/automations/` | | Personal Agent | `src/backend/personal_agent/` | | 记忆与进化 | `src/backend/core/memory/`、`src/backend/core/evolution/` | -| 领域本体 | `src/backend/core/ontology/` | | 沙箱 | `src/backend/core/sandbox/` | | 内置 MCP、技能、插件与智能体 | `src/backend/extensions/` | | 前端入口 | `src/frontend/src/main.tsx`、`App.tsx` | diff --git a/document/zh-CN/getting-started/quick-start.md b/document/zh-CN/getting-started/quick-start.md index c4053d7b..95924f7b 100644 --- a/document/zh-CN/getting-started/quick-start.md +++ b/document/zh-CN/getting-started/quick-start.md @@ -137,5 +137,4 @@ export PATH="$HOME/.hugagent/venv/bin:$PATH" - [MCP 工具](../modules/mcp-tools.md)、 [Agent Skills](../modules/agent-skills.md)和 [私有知识库](../modules/knowledge-base.md); -- [领域本体快速入门](domain-ontology-quickstart.md):用最小 Domain Pack 构建受治理工作流; - [完整无 Docker 安装指南](../deployment/quick-install.md):安装选项、能力边界和故障排查。 diff --git a/document/zh-CN/modules/admin-console.md b/document/zh-CN/modules/admin-console.md index a90ed117..170900b6 100644 --- a/document/zh-CN/modules/admin-console.md +++ b/document/zh-CN/modules/admin-console.md @@ -26,7 +26,6 @@ HugAgentOS 提供**两个相互独立的管理入口**,分别面向内容运 | 待审草稿 | `SkillDraftsPanel` | `admin_skill_drafts.py` | | 沙盒依赖 | `SandboxDepsManager` | `admin_sandbox.py` | | 知识库管理 | `KnowledgeBaseManager` | `admin_kb.py` | -| 本体治理 | `OntologyManager` | `ontologies.py` | | 提示词中心 | `PromptHubEditor` | `content.py`(`prompt_hub` 内容块) | | 子智能体 | `AdminAgentManager` | `admin_agents.py` | | 操作手册 | `ManualEditor` | `content.py`(manual PDF 上传) | @@ -136,7 +135,7 @@ GitHub 绑定默认在此面板维护,也可用 `FEEDBACK_GITHUB_*` 环境变 | 路由组 | 文件 | 凭证 | 版本 | 说明 | |---|---|---|---|---| -| `/v1/config/users` | `config_users.py` | `CONFIG_TOKEN` | 商业版 EE(multi_tenancy) | 用户列表 / 详情 / 状态、重置密码、删除,以及应用可见范围、本体校验等功能权限位,见 [认证与权限](auth.md) | +| `/v1/config/users` | `config_users.py` | `CONFIG_TOKEN` | 商业版 EE(multi_tenancy) | 用户列表 / 详情 / 状态、重置密码、删除,以及应用可见范围等功能权限位,见 [认证与权限](auth.md) | | `/v1/config/teams` | `config_teams.py` | `CONFIG_TOKEN` | 商业版 EE(multi_tenancy) | 团队 CRUD、成员增删、角色设置 | | `/v1/config/invites` | `config_invites.py` | `CONFIG_TOKEN` | 商业版 EE(multi_tenancy) | 注册码批量生成 / 列表 / 吊销 / 删除 | | `/v1/config/security` | `config_security.py` | `CONFIG_TOKEN` | 商业版 EE(system_config) | 安全管理台(只读):沙盒、审计日志、系统健康快照 | @@ -152,7 +151,7 @@ GitHub 绑定默认在此面板维护,也可用 `FEEDBACK_GITHUB_*` 环境变 按《开源与商业化产品方案》第四章与路由注册表对齐: - **社区版(CE)保留**:`content.py` 内容块管理(品牌可定制)、`models.py` 模型管理、登录基础设施(`auth.py` 会话端点、mock SSO)。 -- **商业版(EE)**:完整内容管理台(问题反馈处置 / 技能 / 插件 / 草稿 / 市场 / 知识库 / 本体 / 子智能体 / 提示词 / MCP / 沙盒依赖,`content_admin`)、系统管理台(服务配置 + 安全台,`system_config`)、审计与会话审查(`audit`)、团队计费与用量(`billing`)、用户 / 团队 / 注册码(`multi_tenancy`)。 +- **商业版(EE)**:完整内容管理台(问题反馈处置 / 技能 / 插件 / 草稿 / 市场 / 知识库 / 子智能体 / 提示词 / MCP / 沙盒依赖,`content_admin`)、系统管理台(服务配置 + 安全台,`system_config`)、审计与会话审查(`audit`)、团队计费与用量(`billing`)、用户 / 团队 / 注册码(`multi_tenancy`)。 - `config_license`、`config_verify`、`auth` 显式豁免 License 守卫,保证「402 → 换证」的逃生通道始终可达。 ## 相关源码 diff --git a/document/zh-CN/modules/auth.md b/document/zh-CN/modules/auth.md index b916e5a6..05490c7b 100644 --- a/document/zh-CN/modules/auth.md +++ b/document/zh-CN/modules/auth.md @@ -39,8 +39,7 @@ HugAgentOS 的认证与权限体系覆盖三条互相独立的链路:**终端 ### CE 首次初始化 使用默认 `admin/admin` 登录的 CE 实例在修改临时密码后,会进入全屏首次初始化向导, -而不是直接进入工作台。向导依次设置界面语言、模型、互联网搜索、文件解析、永久记忆 -和本体核验。 +而不是直接进入工作台。向导依次设置界面语言、模型、互联网搜索、文件解析和永久记忆。 - 模型步骤包含必填的主对话模型,以及可选的索引(embedding)模型和重排 (reranker)模型。保存时后端会实测所有已填写模型的连通性,并分别指派对话、 @@ -49,10 +48,8 @@ HugAgentOS 的认证与权限体系覆盖三条互相独立的链路:**终端 - 配置并指派索引模型后,后续步骤会立即重新检测依赖并开放永久记忆开关;重排模型 用于提高长期记忆与知识库检索结果的相关性。两个检索模型留空均不阻塞初始化。 - 互联网搜索和外部文件解析均可跳过,以后可在**设置 → 系统管理**中补充。 -- 记忆和本体开关会先检查实例是否具备对应服务或已发布领域包;不可用时保持关闭, +- 记忆开关会先检查实例是否具备对应服务;不可用时保持关闭, 不会制造“已开启但运行时无效”的状态。 -- CE 新部署在浏览器没有保存过偏好时默认开启**显示调度过程**;用户在**设置 → - 会话设置**中关闭后会保留该选择。 - 完成状态存放在当前管理员的 `users_shadow.metadata` 中,并带版本号。刷新或重新登录 后不会重复进入已完成的向导。 @@ -143,7 +140,6 @@ SSO 提供方响应必须是 JSON 对象,且成功码必须为有限数值 0 | `can_use_api_key` | 关 | `PATCH /v1/config/users/{id}/api-key-permission` | 是否可创建 / 使用个人 API-Key;关闭后已有 Key 立即失效 | | `can_add_skill` | 关 | `PATCH /v1/config/users/{id}/skill-permission` | 能力中心自助上传 / 手写私有技能(`api/routes/v1/me_capabilities.py`) | | `can_add_mcp` | 关 | `PATCH /v1/config/users/{id}/mcp-permission` | 能力中心自助添加私有远程 MCP(HTTP/SSE) | -| `can_use_ontology_validation` | 关 | `PATCH /v1/config/users/{id}/ontology-validation-permission` | 设置中心的本体校验入口、个人开关与运行时本体校验;无权限时入口隐藏,已有个人开关也不生效 | | `lab_enabled` | **开** | `PATCH /v1/config/users/{id}/lab-permission` | 实验室模块入口与访问 | | `allowed_apps` | 无限制 | `PATCH /v1/config/users/{id}/app-permissions` | 应用可见范围;`None`=全部启用应用,列表=白名单(空列表=全屏蔽) | | `role: super_admin` | 无 | (直接写 metadata) | `require_super_admin` 依赖放行;可绕过团队角色检查 | diff --git a/document/zh-CN/modules/chat.md b/document/zh-CN/modules/chat.md index 068a8022..fe5c67a6 100644 --- a/document/zh-CN/modules/chat.md +++ b/document/zh-CN/modules/chat.md @@ -25,7 +25,7 @@ 用户消息使用统一圆角气泡。只有本次新发送的消息播放入场动画,加载历史或返回会话不会重播。消息操作在鼠标悬停、键盘聚焦和触屏场景下可见。 -独立思考沿用原有行内摘要,工具步骤中的思考保留时间线样式。点击摘要,或聚焦后按 Enter / Space,可展开和收起;流式更新保留展开选择。 +思考和工具调用统一显示在可折叠的执行卡中,其中的思考保留时间线样式。点击执行卡标题,或聚焦后按 Enter / Space,可展开和收起;流式更新保留展开选择。 输入区附件使用紧凑卡片,保留文件名扩展名,不显示底部文件类型文字。上传失败会在卡片中持续显示,并可点击“重试”;上传期间移除或切换草稿不会把异步结果写入其他草稿。已完成的附件仍可预览和下载。“我的空间”的文档/图片筛选支持方向键、Home / End 和选中状态提示。 @@ -159,7 +159,6 @@ SSE follower:chat_run_executor.follow_run_as_sse() |---|---|---| | `thinking` | 思考过程(增量或阶段提示) | `delta` / `message` | | `content` | 回答正文增量 | `event: "ai_message"`, `delta`, `chat_id` | -| `content_replace` | 本体评审修订了已流式展示的草稿时,原位替换最终答案 | `content`, `reason: "ontology_review"`, `chat_id` | | `tool_call_start` | 开始构造一次工具调用;前端按稳定 ID 创建一张卡片 | `tool_name`, `tool_display_name`, `tool_id` | | `tool_call_delta` | 工具参数 JSON 增量;前端在同一卡片中追加 | `tool_name`, `tool_id`, `arguments_delta` | | `tool_call` | 工具参数已完整、即将执行 | `tool_name`, `tool_display_name`, `tool_args`, `tool_id`,调子智能体时附 `subagent_name` | @@ -167,14 +166,13 @@ SSE follower:chat_run_executor.follow_run_as_sse() | `steer_applied` | 运行中追加指令已注入 ReAct 上下文 | `steer_id`, `message`, `message_id`, `chat_id` | | `queued_run_started` | 已原子提交的 `followUp` / `nextRun`,或错过最后安全边界的 `steer` 子运行开始,前端立即接力续播 | `run_id`, `message_id`, `user_message_id`, `message`, `queue_id`, `steer_id`, `delivery_mode` | | `subagent_event` | 子智能体内部过程,挂在父 `call_subagent` 卡片下 | `parent_tool_id`, `sub_type`, `agent_name`,以及内部工具或内容字段 | -| `ontology_activation` / `ontology_gate` / `ontology_review` | 本体治理状态,不属于模型思考 | 工作流、门禁决策、委员会状态与结论 | | `tool_pending` | 提供商没有暴露可解析参数增量时的等待兜底 | `reason` | | `file_confirm` | 工具挂起等待用户确认「我的空间」写操作 | 确认上下文;用户带外 `POST /v1/chats/{chat_id}/file-confirm` 后工具原地续跑 | | `user_question` | `ask_user_question` 已挂起并等待会话属主回答 | `request_id`, `questions[]`, `created_at`, `expires_at`, `chat_id` | | `user_question_resolved` | 回答、取消或超时已由服务端裁决 | `request_id`, `outcome`, `chat_id` | | `context_usage` | 最近一次主模型调用的上下文占用快照;供应商返回 usage 时总数为实测值 | `source`, `exact`, `prompt_tokens`, `completion_tokens`, `used_tokens`, `context_window`, `breakdown` | | `compaction_notice` | 上一轮结束后已生成新的上下文压缩检查点 | `chat_id`, `context_compaction`(覆盖边界、摘要基线 token 数) | -| `meta` | 回合收尾元数据 | `route`, `citations[]`, `sources`, `artifacts`, `workspace_files`, `ontology_governance`, `warnings`, `is_markdown`, `message_id`, `usage`, `context_usage`, `compaction_pending` | +| `meta` | 回合收尾元数据 | `route`, `citations[]`, `sources`, `artifacts`, `workspace_files`, `warnings`, `is_markdown`, `message_id`, `usage`, `context_usage`, `compaction_pending` | | `error` | 出错(已映射为用户友好中文文案) | `error`, `chat_id` | | `heartbeat` | 心跳(事件级;另有 `: heartbeat` 注释行) | — | @@ -200,7 +198,7 @@ data: {"type":"meta","route":"main","citations":[...],"usage":{"prompt_tokens":3 data: [DONE] ``` -`meta` 之后,`chat_run_executor.py` 持久化助手消息、回填 artifact,并起后台任务生成追问问题(`core/chat/followups.py`,结果写进消息 `extra_data.follow_up_questions`,前端经 `GET /v1/chats/{chat_id}/messages/{message_id}/followups` 拉取)。本体事件在前端汇总为独立的“领域本体治理”模块,不再写入或显示在“思考过程”中。模型草稿保持逐 token 流式展示;委员会仅在实际修订答案时发送一次 `content_replace`,前端原位替换正文,数据库只保存评审后的最终答案。`ontology_governance` 随助手消息持久化,刷新历史会话后仍可回显。 +`meta` 之后,`chat_run_executor.py` 持久化助手消息、回填 artifact,并起后台任务生成追问问题(`core/chat/followups.py`,结果写进消息 `extra_data.follow_up_questions`,前端经 `GET /v1/chats/{chat_id}/messages/{message_id}/followups` 拉取)。 `usage` 是本轮所有模型调用的累计计费数据;上下文仪表使用的是 `context_usage`,即最近一次主模型调用的 `prompt_tokens + completion_tokens`, @@ -444,7 +442,7 @@ MCP 一样,使用当前账号授权的云端绑定,不再设按工具名称 主对话中的子智能体调用行展示运行状态、智能体 Logo、名称与耗时。点击整行会在右侧 Canvas 打开本次调用的独立页签;内部过程不再在消息下方展开。运行状态动画与 Logo 分开显示。普通工具保持原来的折叠交互。 -详情直接复用主对话的工具批次、思考和正文 UI,遵循相同的过程显示设置与默认折叠交互,不额外增加参数或执行过程分组。完成后调用行不显示对勾。面板实时跟随当前调用;向上滚动阅读时暂停自动跟随。同一智能体的多次调用分别定位,并可与文件、本体、插件页签共存。历史详情使用已保存的数据:子步骤最多 200 项,单条字符串工具输出约 16,000 字符,已截断内容无法恢复。 +详情直接复用主对话的工具批次、思考和正文 UI,遵循相同的过程显示设置与默认折叠交互,不额外增加参数或执行过程分组。完成后调用行不显示对勾。面板实时跟随当前调用;向上滚动阅读时暂停自动跟随。同一智能体的多次调用分别定位,并可与文件、插件页签共存。历史详情使用已保存的数据:子步骤最多 200 项,单条字符串工具输出约 16,000 字符,已截断内容无法恢复。 技能加载、插件加载以及可唯一确定所属连接器的工具调用行,展示能力中心中对应的 Logo。插件声明的工具专属图标优先使用。运行动画与 Logo 分开显示,点击行为和折叠布局保持不变;普通工具保留原分类图标。 diff --git a/document/zh-CN/modules/knowledge-base.md b/document/zh-CN/modules/knowledge-base.md index 5dd8ba6b..ecedcd00 100644 --- a/document/zh-CN/modules/knowledge-base.md +++ b/document/zh-CN/modules/knowledge-base.md @@ -89,20 +89,20 @@ Wiki 相关的三张表(`kb_wiki_pages` / `kb_wiki_folders` / `kb_wiki_jobs` 文档里的图不再被丢弃。索引时走 `features/knowledge/kb_assets.py`: -1. **抽图**——PDF 经外部解析服务的版面切分拿回图片本体、图注、页码与 bbox;单独上传的 +1. **抽图**——PDF 经外部解析服务的版面切分拿回图片原图、图注、页码与 bbox;单独上传的 图片自身即一个资产(此前会被当作纯文本硬解成乱码入库,现已修复)。字节存对象存储, 正文里原本指向解析服务临时路径的死链被改写成真实资产 URL。 2. **图像理解**——复用[视觉桥](./model-providers.md#视觉桥让纯文本模型能读图)(`core/models/vision`),和对话里「纯文本主模型读图」 走的是同一条通路、同一个 `vision` 模型角色,因此白得三样东西:按图片内容哈希的**证据缓存** (重新索引同一篇文档不用再付一次钱)、三协议与结构化输出降级、以及「该角色只能指派多模态模型」 的硬约束。产出映射:`summary` → `caption`,`ocr.full_text` 并入 `text_content`(与原图注共存), - 实体 / 关系 / 不确定项留档进 `metadata.vision` 供本体层与人工复核使用。 + 实体 / 关系 / 不确定项留档进 `metadata.vision` 供人工复核使用。 **未指派 `vision` 角色(且主模型不识图)是正常状态**:图片仍以图注入索引,之后配好再回填即可, 无需重新解析文档。 3. **入索引**——每个资产写一条 `row_type="image"` 的检索行,`parent_chunk_id` 指向包含它 的父块,因此去重、回表取父块原文、重排、按文档删除等既有链路全部无改动复用。 4. **回传**——检索结果里该片段会带 `images: [{asset_id, url, caption}]`,供模型据描述作答、 - 供前端渲染缩略图。图片本体经 `GET /v1/catalog/kb/assets/{asset_id}` 读取,可见性完全 + 供前端渲染缩略图。图片原图经 `GET /v1/catalog/kb/assets/{asset_id}` 读取,可见性完全 跟随所属知识库空间。 开关与调优:`indexing_config.multimodal_indexing`(单库)优先于全局 `KB_MULTIMODAL_INDEXING`。 diff --git a/document/zh-CN/modules/mcp-tools.md b/document/zh-CN/modules/mcp-tools.md index d1ca4408..d858c758 100644 --- a/document/zh-CN/modules/mcp-tools.md +++ b/document/zh-CN/modules/mcp-tools.md @@ -179,7 +179,7 @@ extensions/mcp_servers/_mcp/ ```json { - "result": "……业务数据本体……", + "result": "……业务数据……", "__citations__": [ {"title": "来源标题", "url": "https://…", "snippet": "关键摘录", "source_type": "internet"}, {"title": "第二个来源"} @@ -221,7 +221,7 @@ MCP 工具的执行预算不包含等待用户回答的时间。有效提问会 - 工具名命中 Registry 时,Middleware 在副作用和真实分发之前解析调用参数,依次执行 本机目录 Grant、危险命令策略、MySpace 写确认或通用工具确认; - 通过后产生绑定 `tool name + call id + arguments hash + spec key` 的短期票据;最终 - `OntologyFilteredToolkit.call_tool()` 会再次核验票据,Read/Write/Edit/Bash 的真实 + 工具集分发入口 `call_tool()` 会再次核验票据,Read/Write/Edit/Bash 的真实 宿主 I/O 边界还会消费票据内的路径或命令约束; - 需要确认的受控工具在批处理、子智能体、计划模式、自动化运行或无确认 UI 的渠道运行中 一律拒绝,不会再以“记录审计后继续执行”降级; diff --git a/document/zh-CN/modules/ontology-harness.md b/document/zh-CN/modules/ontology-harness.md deleted file mode 100644 index a0855dff..00000000 --- a/document/zh-CN/modules/ontology-harness.md +++ /dev/null @@ -1,225 +0,0 @@ -# 领域本体治理与校验 - -领域本体校验为智能体增加可选的领域契约。它在工具执行前检查参数和流程,在高风险答案 -生成后运行独立评审,同时保留关闭后完全不改变原对话流程的兼容路径。无论是否触发评审, -思考状态、正文增量和工具事件都会实时流式发送。 - -## 用户开关 - -商业版 EE 中,管理员需先在「系统配置 → 用户管理 / 团队管理 / 角色权限」授予 -`can_use_ontology_validation` 权限;未获授权的用户不会在设置栏看到本体校验入口,直接调用 -个人设置接口也会被拒绝。登录后打开「设置 → 本体校验」。CE 实例管理员看到的入口名称为「本体治理」,同一页面 -同时提供个人校验开关和全局 Domain Pack 管理。系统有已激活的默认 Domain Pack 时,可以 -开启「使用领域本体校验」。 - -- 关闭:不注入本体、不拦截工具,使用原有流程。 -- 开启但任务未命中领域工作流:不会套用无关规则。 -- 开启且命中 checkpoint:首稿流式展示后,由一位独立评审员核验。 -- 开启且命中 committee:多位独立委员按多数票判断通过、修订或转人工。 - -评审启动时,聊天右侧面板会自动展开,并单独流式展示本体校验状态与优化稿;首稿继续 -保留在左侧。修订过程中的模型思考默认折叠,工具调用和新增引用仍可展开查看。用户可以 -使用右上角的侧边栏按钮收起或重新展开面板,也可以通过历史消息下方的本体校验入口回看 -对应结果。优化稿生成完成后,用户可以选择「替换原文」,系统不会在用户确认前覆盖首稿。 - -## Domain Pack - -Domain Pack 是 JSON 格式的领域契约,包含四层内容:概念、关系、可执行约束和工作流。 -CE 实例管理员在「设置 → 本体治理」中导入和管理版本;商业版 EE 管理员继续使用 -「内容管理 → 本体治理」。 - -新版本导入后默认为工作草稿。每个领域包同时只能有一份工作草稿,管理员可以反复修改; -只有显式发布后,草稿才会锁定为正式版本并进入运行时。切换激活版本不修改历史审计记录。 -CE 不预装业务领域包,避免把某个行业的工具 ID 和工作流带入通用发行版;实例管理员按实际 -接入的工具导入自己的 Domain Pack。商业版 EE 可以随版本提供商业能力对应的领域包。 - -工作流可以同时配置两类入口: - -- `triggers`:用户问题命中文本触发词时激活; -- `asset_triggers`:工具、技能或子智能体实际被调用时,按资产 ID 或受控本体标签激活。 - -资产标签使用 `ontology:ConceptId` 格式。例如,把数据检查工具或质量报告技能标记为 -`ontology:QualityReport`,即使用户没有说出“质量检查”等关键词,运行时实际调用该资产后也会 -进入对应的本体工作流。普通展示标签不会触发本体流程。 - -### 预置可选标签 - -要让一个标签出现在技能或子智能体的选择器中,Domain Pack 必须同时声明概念和资产触发 -关系。仅定义概念不会让它成为运行时触发标签。 - -1. 在 `concepts` 中定义概念,例如 `QualityReport`。 -2. 在目标工作流的 `asset_triggers` 中选择资产类型,并把 - `ontology:QualityReport` 加入 `tags_any`。 -3. 激活该 Domain Pack 版本。技能和子智能体表单随后会显示标签、关联工作流和评审级别。 - -以下示例让同一个标签可用于技能和子智能体: - -```json -{ - "concepts": [ - {"id": "QualityReport", "name": "质量报告", "definition": "数据质量检查报告"} - ], - "workflows": [ - { - "id": "quality_review", - "review_level": "committee", - "asset_triggers": [ - {"kind": "skill", "tags_any": ["ontology:QualityReport"]}, - {"kind": "subagent", "tags_any": ["ontology:QualityReport"]} - ] - } - ] -} -``` - -## 运行时校验 - -工具调用先经过不使用大模型的确定性门禁。门禁可检查 JSON Schema、必需的前置工具和 -禁用工具。违规结果会连同原因和修正建议返回智能体;重复违规会要求改变策略并最终熔断。 - -运行时采用单调升级:一次调用可以从“未命中”升级到 checkpoint,再升级到 committee, -但不会在同一轮中降级。资产必须实际调用后才激活,系统不会仅因某个带标签的资产出现在 -目录里就套用规则。已激活工作流、触发来源、工具门禁和委员会进度会作为运行事件显示在 -对话中,并写入审计证据。 - -每轮本体治理生成唯一 `governance_run_id`。父智能体、`call_subagent` 子智能体和目标内部工具 -共享同一个运行时对象;同一 `pack_id + workflow_id` 无论先由文本命中还是后由资产标签命中, -只记录一次激活。每次受治理的工具调用仍独立通过确定性门禁,因为参数和前置证据可能不同。 -最终答案评审由最外层工作流唯一认领,子智能体不会自行启动委员会。子智能体内部成功完成的 -工具调用会连同 `agent_id`、`sub_run_id` 和 `parent_tool_id` 归并到外层证据轨迹,并生成与普通 -工具调用相同的可追溯引用。因此,领域工作流要求的工具可以由受信任的子智能体完成,不会因 -外层只看到 `call_subagent` 而被误判为缺失。 - -高风险输出会同时检查工作流必需工具、内容长度或格式以及引用证据。委员会只在 Domain -Pack 明确配置的高风险工作流上运行,不会对每个工具调用启动大模型。 - -系统先运行确定性输出检查,再启动唯一一轮 checkpoint 或 committee 评审。若发现缺少必需 -工具、引用或输出结构,系统会先把结构化违规信息返回给生成答案的原智能体,让它在同一个 -ReAct 上下文中继续调用真实工具、搜索缺失证据并生成完整优化稿。修订指令明确要求先比较 -证据缺口;如果新结论需要外部事实,必须优先补充搜索或工具调用,不能只做同义改写。 - -每次回答最多生成一版自动优化稿,也只执行一轮独立评审。委员会返回 `revise` 时,原智能体 -仍可使用剩余的一次修订机会补充证据,但系统不会循环启动第二轮委员会。修订过程中的模型 -思考单独进入默认折叠的思考区,工具调用、工具结果和优化稿正文分别流式发送,不会混入原文。 - -若仍需人工判断,后端会输出结构化 `manual_review` JSON,其中包含相关原句或结论、规则 ID、 -风险原因和人工核对动作。前端把这些字段渲染为「领域本体人工复核」卡片,而不是把提示文本 -拼接到答案末尾。用户可以继续把优化稿作为工作草稿,但不能将待人工复核内容视为已通过领域 -治理的最终结论。 - -首稿始终按生成顺序展示在左侧;优化稿、折叠思考、补充工具调用和人工复核卡片显示在右侧 -面板,并随 SSE 增量自动跟随到底部。用户主动向上滚动时,面板停止自动跟随,避免打断阅读。 -优化稿不会自动覆盖首稿。用户点击「替换原文」后,系统才会原子更新消息正文,并在历史会话中 -保留已采用状态。 - -同一开关适用于普通对话、子智能体、渠道机器人、自动化、计划、批处理和自主循环。 -用户已开启本体但运行时策略不可用时,本次执行会停止并提示重试,不会静默绕过校验。 - -用户通过输入框显式 `@子智能体`,或者输入“调用/请调用 + 唯一完整子智能体名称 + 明确任务” -时,本轮直接进入目标子智能体,不会先让主智能体执行领域工作流再二次派发。本体运行时仍在 -请求边界创建,并按子智能体 ID / 标签立即激活;随后目标子智能体内部的技能和工具调用继续 -共享同一运行时,执行确定性门禁,最终输出按升级后的最高评审级别完成一次 checkpoint 或 -committee 评审。 - -## 构建时校验 - -技能、MCP 工具、插件和子智能体在创建、导入、安装、上架或蒸馏保存时,会先按当前激活的 -Domain Pack 检查工具依赖、禁用工具、参数 Schema 与输出契约。受控词表标签使用 -`ontology:ConceptId` 格式;未知概念会在保存前被指出,普通标签不受影响。 - -当资产缺少工作流要求的底层工具时,校验报告会反查当前可绑定的全局 MCP,并优先显示 MCP -中文名称和工具中文名称。例如,报告可以提示绑定“数据质量检查”,同时列出它可补齐的 -“字段校验”“重复记录检查”等能力。底层工具 ID 仍保留在完整报告中,便于管理员排查。 - -新建或编辑技能时,可以在“绑定工具 (MCP)”中直接选择 MCP,不需要手工填写底层工具 ID。 -后端会读取该 MCP 已发现的 `tools_json`,把工具清单用于本体构建校验。保存成功后,生成的 -`SKILL.md` 会同时记录 `ontology_tags`、`ontology_workflows`、`mcp_servers` 和 -`allowed_tools`。编辑、导出、上架或重新导入技能时,这些绑定关系会随技能保留。技能启用 -后,其声明的 MCP 会合并到本次技能执行的能力范围,不会同时开启其他未绑定 MCP。 - -技能和子智能体表单把普通标签与“本体治理标签”分开。本体标签只能从当前激活领域包中 -选择,并且只显示通过对应资产类型的 `asset_triggers.tags_any` 关联到工作流的标签。选择后, -表单会直接显示实际调用该资产时将触发的工作流和评审级别。 - -MCP 工具标签可以放在工具的 `tags` 或 `ontology_tags` 中,也可以由 MCP 服务配置统一提供。 -技能使用自身元数据中的受控标签。这些标签既参加构建时检查,也可供 `asset_triggers` 在 -运行时匹配。 - -管理员可以在 MCP 服务编辑页配置服务级标签,也可以按 -`tool_name=ontology:ConceptId` 的格式配置工具级标签。用户端和管理端的技能、子智能体表单 -都提供受控的本体治理标签选择器。 - -## 本体治理 - -CE 实例管理员可直接在「设置 → 本体治理」完成治理,不依赖 CE 中未提供的独立内容管理台。 -商业版 EE 仍从「内容管理 → 本体治理」进入。两个入口使用同一套治理组件和服务逻辑,并提供 -以下管理能力: - -- 导入、校验、导出、激活和切换默认 Domain Pack; -- 查看工具门禁事件、评审证据和延迟; -- 查看门禁、评审、草案、候选合并与各来源采纳率的闭环指标; -- 从重复违规或评审修订证据自动预筛演进草案,也可手动触发; -- 把命中本体工作流后的用户点踩纠正纳入人审草案; -- 人工批准或驳回草案,并将结构化草案合并到当前工作草稿。 - -### 在详情页编辑版本 - -管理员可以打开任一领域包的「详情」,直接在总览、概念、关系、约束和工作流页签中选择 -「编辑此模块」。每个页签只修改对应模块,其他模块沿用当前版本内容。 - -编辑器使用结构化表单,不要求管理员编写 JSON: - -- 总览表单维护领域包说明、注入预算和评审阈值; -- 概念表单逐项维护定义、别名、层级、标签和受控取值; -- 关系表单从已有概念中选择起点和终点,并维护谓词、基数和禁止关系; -- 约束表单维护目标、提示和修正建议,并用字段类型、必填项、枚举、长度和数值范围配置 - JSON Schema; -- 工作流表单维护文本或资产触发条件、必需/禁止工具、输出标签和评审级别。 - -保存前,管理台会对组合后的完整 Domain Pack 做校验,并在表单中列出具体错误位置。首次 -编辑使用管理员确认的新版本号创建一份工作草稿;后续编辑任何模块都更新同一草稿,不会为 -每次保存增加版本。若领域包已有草稿,从正式版本或历史版本进入编辑时,管理台会自动切换到 -该草稿继续修改,避免覆盖草稿中其他模块的变更。 - -工作草稿发布后会锁定为正式版本,并立即替换当前运行版本。正式版本和归档版本均为只读; -要继续修改,需从所查看的版本创建下一份工作草稿。版本管理弹窗默认每页显示 10 条,可以 -切换为每页 20 条或 50 条;后台保留全部正式历史版本,不会因为分页而自动删除。管理员也可在 -发布前明确放弃不再需要的工作草稿。原始 JSON 页签保留为只读核对和迁移入口。 - -本体演进永远不会自动修改或发布生产版本。管理员必须先审查候选,再将通过的结构化候选 -合并到工作草稿,最后显式发布。会话证据和候选在入库前会经过密级拒绝、敏感信息脱敏与 -注入内容净化;同一候选不能重复合并。 - -## API - -常用接口如下。CE 设置页治理接口需要实例管理员权限;`ADMIN_TOKEN` 可作为运维回退。 -商业版 EE 的 `/v1/admin/ontologies` 接口继续接受 `ADMIN_TOKEN` 或等价内容管理权限。 - -| 方法与路径 | 用途 | -|---|---| -| `GET/PATCH /v1/ontologies/settings` | 用户读取或更新本体开关 | -| `GET /v1/ontologies/runtime/preview` | 预览某个任务命中的规则 | -| `GET /v1/ontologies/tags?asset_kind=skill` | 读取当前可选的资产触发标签 | -| `GET /v1/ontologies/governance/access` | 探测当前 CE 用户是否可管理全局本体 | -| `/v1/ontologies/governance/*` | CE 设置页的领域包、版本、证据和演进治理 | -| `GET /v1/admin/ontologies` | 领域包与版本列表 | -| `GET /v1/admin/ontologies/tags?asset_kind=skill` | 管理端读取可选资产触发标签 | -| `GET /v1/admin/ontologies/metrics` | 闭环治理指标 | -| `POST /v1/admin/ontologies/validate` | 校验 Domain Pack JSON | -| `POST /v1/admin/ontologies/versions` | 导入新版本 | -| `PUT /v1/admin/ontologies/{pack_id}/draft` | 创建或更新唯一工作草稿 | -| `DELETE /v1/admin/ontologies/{pack_id}/draft/{version_id}` | 放弃未发布工作草稿 | -| `POST .../activate` | 发布草稿或激活指定历史版本 | -| `POST /v1/admin/ontologies/build/validate` | 预检技能、工具或子智能体资产 | -| `GET .../events`、`GET .../reviews`、`GET .../drafts` | 查看运行与演进证据 | -| `POST /v1/chats/messages/{message_id}/ontology-revision/accept` | 用本体优化稿替换原消息正文 | - -## 运维提示 - -升级到包含本功能的版本时必须先运行数据库迁移。若用户界面显示“暂无可用本体包”,CE -实例管理员应在「设置 → 本体治理」确认至少一个包同时处于启用、默认和已有激活版本状态; -商业版 EE 管理员在内容管理台执行相同检查。 - -CE 升级不会自动写入或替换业务领域包。导入新版 Domain Pack 时仍遵循版本不可变规则: -管理员先审查工作草稿,再在「本体治理」中显式发布,资产触发规则才会进入运行时。 -商业版 EE 随版本提供的领域包也遵循同一套草稿和发布流程。 diff --git a/document/zh-CN/modules/projects-myspace.md b/document/zh-CN/modules/projects-myspace.md index d79a3b0e..7f009df6 100644 --- a/document/zh-CN/modules/projects-myspace.md +++ b/document/zh-CN/modules/projects-myspace.md @@ -29,7 +29,7 @@ teams ─────────┬── team_members(role: owner/admin/memb | `Project` | `projects` | `kind`(personal/team)、`instructions`、`linked_folder_id` / `linked_team_folder_id` 互斥挂钩、`pinned`、`metadata`(含项目级记忆开关);CHECK 约束保证 kind 与 team_id 匹配 | | `ProjectFavorite` | `project_favorites` | 每人独立 star,不影响他人视图 | | `UserFolder` | `user_folders` | 个人文件夹树;命名安全约束(禁 `/`、`.`、`..`) | -| `Artifact` | `artifacts` | 文件本体:`storage_key`(对象存储)、`user_folder_id` 与 `team_folder_id` 互斥、`parsed_text` / `summary` 跨轮读取缓存、软删 `deleted_at` | +| `Artifact` | `artifacts` | 文件实体:`storage_key`(对象存储)、`user_folder_id` 与 `team_folder_id` 互斥、`parsed_text` / `summary` 跨轮读取缓存、软删 `deleted_at` | | `Team` / `TeamMember` | `teams` / `team_members` | 团队与成员(商业版 EE);可由外部 SSO 部门自动建立(`source=sso_auto`) | | `TeamFolder` | `team_folders` | 团队文件夹树(商业版 EE) | @@ -51,7 +51,7 @@ teams ─────────┬── team_members(role: owner/admin/memb | GET | `/v1/projects/{id}/chats` | 项目内会话列表(团队项目可见共享会话) | ### 页面加载 -项目列表在页面代码加载时显示标题、操作栏、搜索框和项目卡片骨架。首次请求项目数据时复用卡片骨架;已有列表刷新时保留原内容。选择项目后进入统一会话界面,项目资料单独加载。骨架适配窄屏和深色主题,并在系统启用减弱动态效果时停止微光动画。 +项目列表在页面代码加载时显示标题、操作栏、搜索框和项目卡片骨架。首次请求项目数据时复用卡片骨架;已有列表刷新时保留原内容。选择项目后进入统一会话界面,项目资料单独加载:加载期间项目标题和右侧资料栏显示与正式布局一致的骨架,指令、文件、记忆、定时任务卡片各自加载时用行骨架占位;加载失败时资料栏显示原因和重试。骨架适配窄屏和深色主题,并在系统启用减弱动态效果时停止微光动画。 ### 项目会话与定时任务 diff --git a/document/zh-CN/modules/sandbox.md b/document/zh-CN/modules/sandbox.md index 6a8a9e77..b9c8feaf 100644 --- a/document/zh-CN/modules/sandbox.md +++ b/document/zh-CN/modules/sandbox.md @@ -119,6 +119,7 @@ artifact 送入沙盒、`Bash` 执行后 `/myspace` 写回同步;backend 与 s ``` - 一个会话复用同一容器中的文件。Python/JS 每次启动独立进程,不保留内存变量。 +- 每个容器创建时都设置 `EXECD_API_GRACE_SHUTDOWN=0s`。execd 默认会在前台命令结束后让流再挂 1 秒,导致每次写文件(其后的 `chown`)和每条初始化命令都多等 1 秒;事件本来就是逐条写出并刷新的,不会丢输出。后台命令启动后 2 秒内查状态遇到 `command not found` 时按"尚未登记"处理:v1.0.22 之前的 execd 会先回复、后登记命令。 - 用户绑定的容器仅能交给同一用户,复用前清理上一会话的工作区。 - 基础实现保留个人空间和技能挂载。EE 扩展额外保留团队挂载并复核权限。 - 不预热空闲容器。用户的容器在第一次需要它的工具调用时创建,之后重置干净再给这个用户的后续会话复用。历史配置名中的 jupyter 表示用户会话池,不会启动 Jupyter。 diff --git a/document/zh-CN/modules/sites.md b/document/zh-CN/modules/sites.md index 185fb08a..69f9d88d 100644 --- a/document/zh-CN/modules/sites.md +++ b/document/zh-CN/modules/sites.md @@ -57,7 +57,7 @@ ## 限制 -- 站点本体为静态文件;动态能力由应用 SQL 数据接口与声明式 MCP 提供,不支持自定义服务端逻辑。 +- 站点内容为静态文件;动态能力由应用 SQL 数据接口与声明式 MCP 提供,不支持自定义服务端逻辑。 - 单站 ≤ 300 个文件、总量 ≤ 30MB、单文件 ≤ 10MB;单用户 ≤ 50 个站点。 - 内网部署时外部 CDN 资源可能不可达,样式脚本建议内联或本地化(智能体默认遵循此约定)。 diff --git a/requirements-mem0.txt b/requirements-mem0.txt index 8528f063..b403d004 100644 --- a/requirements-mem0.txt +++ b/requirements-mem0.txt @@ -1,4 +1,4 @@ # 可选依赖:L2 向量 / L3 图谱记忆(compose 的 mem0 profile) # 安装:pip install -r requirements-mem0.txt -mem0ai>=0.1.50 +mem0ai==2.2.1 neo4j>=5.15.0 diff --git a/requirements.txt b/requirements.txt index 0eaea501..23e02f0c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -57,7 +57,7 @@ fakeredis>=2.21.0 # Database psycopg2-binary>=2.9.9 # Some deployments use explicit postgresql+psycopg URLs. -psycopg[binary]>=3.2,<4 +psycopg[binary,pool]>=3.2,<4 sqlalchemy>=2.0.25 alembic>=1.13.0 diff --git a/src/backend/alembic/env.py b/src/backend/alembic/env.py index a75537aa..b0dd09eb 100644 --- a/src/backend/alembic/env.py +++ b/src/backend/alembic/env.py @@ -31,7 +31,9 @@ load_models(INSTALLED_FEATURES) -config.set_main_option("sqlalchemy.url", settings.db.url) +# Alembic stores options in ConfigParser, which treats "%" as interpolation; +# URL-encoded credentials (e.g. p%40ss) must be escaped to survive the round trip. +config.set_main_option("sqlalchemy.url", settings.db.url.replace("%", "%%")) # Set target metadata target_metadata = Base.metadata diff --git a/src/backend/alembic/versions/ce_0035_remove_ontology.py b/src/backend/alembic/versions/ce_0035_remove_ontology.py new file mode 100644 index 00000000..21ea182f --- /dev/null +++ b/src/backend/alembic/versions/ce_0035_remove_ontology.py @@ -0,0 +1,27 @@ +"""Remove domain ontology governance and delete its persisted data. + +Revision ID: ce_0035 +Revises: ce_0034 + +Back up the database before upgrading. Data rollback requires restoring that +backup; conversations and every other feature are retained. +""" +from alembic import op + +from core.db.ontology_retirement import retire_ontology + +revision = "ce_0035" +down_revision = "ce_0034" +branch_labels = None +depends_on = None + + +def upgrade(): + retire_ontology(op.get_bind()) + + +def downgrade(): + raise RuntimeError( + "Ontology data was deleted. Restore the pre-upgrade database backup " + "together with the previous application version to roll back." + ) diff --git a/src/backend/api/deps.py b/src/backend/api/deps.py index 237b2882..e71db7cc 100644 --- a/src/backend/api/deps.py +++ b/src/backend/api/deps.py @@ -254,21 +254,6 @@ def user_can_manage_system_settings(db: Session, user_id: Optional[str]) -> bool return settings.edition.edition == "ce" and settings.auth.mode == "mock" -def user_can_manage_ontology_governance(db: Session, user_id: Optional[str]) -> bool: - """Whether a signed-in CE instance owner may manage global ontology assets. - - The management surface is intentionally delegated only in CE, where the - separate Admin console is not shipped. A CE user must still be either a - content/system administrator (super-admin implies both) or the sole mock - user in the single-trust-domain development mode. - """ - if not user_id or settings.edition.edition != "ce": - return False - return _user_has_meta_flag( - db, user_id, "can_content_manage" - ) or user_can_manage_system_settings(db, user_id) - - def _authorize_delegated( db: Session, request: Request, @@ -338,28 +323,6 @@ async def require_system_settings( ) -async def require_ontology_governance( - request: Request, - user: Optional[UserContext] = Depends(require_auth(False)), -) -> str: - """Authorize the CE settings-page ontology governance surface. - - ``ADMIN_TOKEN`` remains an operational fallback. Browser access is limited - to users accepted by :func:`user_can_manage_ontology_governance`. - """ - return await run_in_session( - _authorize_delegated, - request, - user, - token=os.getenv("ADMIN_TOKEN") or getattr(settings.auth, "admin_token", ""), - token_grant="admin_token", - allowed=user_can_manage_ontology_governance, - action="ontology_governance", - required="ce_ontology_governance", - forbidden="需要 CE 本体治理权限", - ) - - def _authorize_super_admin(db: Session, request: Request) -> str: user_id = _resolve_current_user_id(request) if not user_id: diff --git a/src/backend/api/lifecycle.py b/src/backend/api/lifecycle.py index f2dca54e..b574418c 100644 --- a/src/backend/api/lifecycle.py +++ b/src/backend/api/lifecycle.py @@ -32,6 +32,7 @@ ) from api.startup.runtime import ( _startup_thread_budget, + _startup_validate_space_transport, _startup_watch_capability_changes, _startup_recover_chat_runs, _startup_resume_loops, @@ -126,6 +127,7 @@ def _startup_steps(): return ( # (step, stop, gate, roles, scope) (_startup_thread_budget, None, True, _ALL_ROLES, _PER_WORKER), + (_startup_validate_space_transport, None, True, _ALL_ROLES, _PER_WORKER), (_startup_ensure_tables, None, True, _ALL_ROLES, _SINGLETON), (_startup_watch_capability_changes, None, True, _ALL_ROLES, _PER_WORKER), (_startup_desktop_observability, None, True, _ALL_ROLES, _PER_WORKER), diff --git a/src/backend/api/openapi_data_schemas.py b/src/backend/api/openapi_data_schemas.py index f32cc698..a91692e6 100644 --- a/src/backend/api/openapi_data_schemas.py +++ b/src/backend/api/openapi_data_schemas.py @@ -262,7 +262,6 @@ "allowed_apps": {"type": "array", "items": {"type": "string"}}, "lab_enabled": {"type": "boolean"}, "can_use_voice": {"type": "boolean", "default": False}, - "can_use_ontology_validation": {"type": "boolean"}, }, }, "CurrentUserInfo": { diff --git a/src/backend/api/personal_agents.py b/src/backend/api/personal_agents.py index ebc1e61e..c2d2dd42 100644 --- a/src/backend/api/personal_agents.py +++ b/src/backend/api/personal_agents.py @@ -45,7 +45,6 @@ class AgentCreateRequest(BaseModel): max_tokens: Optional[int] = None max_iters: Optional[int] = 10 timeout: Optional[int] = 120 - ontology_tags: Optional[List[str]] = Field(default_factory=list) extra_config: Optional[Dict[str, Any]] = Field(default_factory=dict) @@ -66,7 +65,6 @@ class AgentUpdateRequest(BaseModel): max_iters: Optional[int] = None timeout: Optional[int] = None is_enabled: Optional[bool] = None - ontology_tags: Optional[List[str]] = None extra_config: Optional[Dict[str, Any]] = None diff --git a/src/backend/api/routes/v1/__init__.py b/src/backend/api/routes/v1/__init__.py index 6bffc440..6a538318 100644 --- a/src/backend/api/routes/v1/__init__.py +++ b/src/backend/api/routes/v1/__init__.py @@ -20,7 +20,6 @@ ("file_upload", "router"), ("content", "router"), ("memories", "router"), - ("ontologies", "router"), ("models", "router"), ("audio", "router"), ("prompt_management", "router"), diff --git a/src/backend/api/routes/v1/chats/feedback.py b/src/backend/api/routes/v1/chats/feedback.py index 67020df4..b43d045b 100644 --- a/src/backend/api/routes/v1/chats/feedback.py +++ b/src/backend/api/routes/v1/chats/feedback.py @@ -1,4 +1,4 @@ -"""Ontology acceptance and assistant feedback endpoints.""" +"""Assistant feedback endpoints.""" import core.chat.operations.schemas as chat_models import core.chat.operations.session_context as chat_session_context @@ -20,37 +20,6 @@ logger = get_logger(__name__) -@router.post( - "/messages/{message_id}/ontology-revision/accept", - summary="采用领域本体优化稿", -) -def accept_ontology_revision( - message_id: str, - user: UserContext = Depends(auth_backend.get_current_user), - db: Session = Depends(db_engine.get_db), -): - """Atomically replace one assistant message with its reviewed candidate answer.""" - chat_service = chat_services_ChatService_module.ChatService(db) - target = chat_service.get_message_by_id(message_id) - if not target or target.role != "assistant": - raise ResourceNotFoundError(resource_type="chat_message", resource_id=message_id) - db_user_id = chat_context.resolve_db_user_id( - db, chat_session_context.authenticated_user_id(user) - ) - access = chat_service.get_session_with_access(target.chat_id, db_user_id) - if access is None: - raise ResourceNotFoundError(resource_type="chat_message", resource_id=message_id) - if access[1] not in ("admin", "edit"): - raise HTTPException(status_code=403, detail="只读共享会话不可替换消息正文") - updated = chat_service.accept_ontology_revision(message_id) - if updated is None: - raise HTTPException(status_code=409, detail="当前消息没有可采用的本体优化稿") - return success_response( - data={"message_id": message_id, "content": updated.content}, - message="已采用本体优化稿", - ) - - @router.post("/messages/{message_id}/feedback", summary="消息反馈") def submit_feedback( message_id: str, @@ -94,18 +63,4 @@ def submit_feedback( db.commit() db.refresh(record) - if body.rating == "dislike" and (body.comment or "").strip() and record.chat_id: - try: - from core.ontology.evolution import OntologyEvolutionService - - OntologyEvolutionService(db).ingest_user_correction( - user_id=db_user_id, - chat_id=record.chat_id, - message_id=message_id, - feedback_id=str(record.feedback_id), - comment=(body.comment or "").strip(), - ) - except Exception: # noqa: BLE001 - feedback persistence must remain available - logger.warning("ontology user-correction ingestion failed", exc_info=True) - return {"ok": True, "feedback_id": record.feedback_id, "rating": record.rating} diff --git a/src/backend/api/routes/v1/evolution.py b/src/backend/api/routes/v1/evolution.py index d9bd40c6..2bfc2842 100644 --- a/src/backend/api/routes/v1/evolution.py +++ b/src/backend/api/routes/v1/evolution.py @@ -123,8 +123,6 @@ async def approve_my_candidate( class PrefsPatch(BaseModel): enabled: Optional[bool] = None - ontology_mode: Optional[str] = Field(None, description="none | any | specific") - ontology_pack_id: Optional[str] = None mechanisms: Optional[list] = None min_support: Optional[int] = Field(None, ge=2, le=20) auto_approve_low_risk: Optional[bool] = None @@ -132,19 +130,12 @@ class PrefsPatch(BaseModel): @router.get("/prefs", summary="获取我的进化配置") async def get_prefs(user: UserContext = Depends(get_current_user)): - """Preferences plus the domain packs available to bind to. - - The pack list ships with the prefs so the UI can render "no packs - configured" as a visible fact rather than hiding the setting — an absent - control looks like a missing feature, which is a different message. - """ - from core.evolution.profiles.prefs import ONTOLOGY_MODES, available_ontology_packs, resolve_prefs + """The current user's evolution preferences.""" + from core.evolution.profiles.prefs import resolve_prefs return success_response( data={ "prefs": resolve_prefs(user.user_id), - "ontology_packs": available_ontology_packs(), - "ontology_modes": list(ONTOLOGY_MODES), } ) diff --git a/src/backend/api/routes/v1/ontologies.py b/src/backend/api/routes/v1/ontologies.py deleted file mode 100644 index 3bd748e1..00000000 --- a/src/backend/api/routes/v1/ontologies.py +++ /dev/null @@ -1,725 +0,0 @@ -"""Ontology harness APIs: user opt-in plus Admin/CE asset governance.""" - -from __future__ import annotations -from core.infra.time import as_utc -from core.infra.time import utc_now - -from collections import Counter, defaultdict -from datetime import datetime, timedelta -from typing import Any, Literal - -from api.deps import require_admin, require_ontology_governance, user_can_manage_ontology_governance -from core.auth.backend import UserContext, get_current_user, require_auth -from core.config.settings import settings -from core.db.engine import get_db -from core.db.models import OntologyDraft, OntologyEnforcementEvent, OntologyReviewRun -from core.infra.exceptions import BadRequestError, ResourceNotFoundError -from core.infra.responses import success_response -from core.auth.users import UserService -from core.ontology.policy import plugin_import_build_validation_forced, require_ontology_validation_permission, set_plugin_import_build_validation_forced, user_can_use_ontology_validation -from core.ontology.service import OntologyService -from fastapi import APIRouter, Depends, Query -from pydantic import BaseModel, Field -from sqlalchemy import func -from sqlalchemy.orm import Session - -router = APIRouter(tags=["Ontologies"]) - - -class OntologySettingsRequest(BaseModel): - ontology_enabled: bool - ontology_pack_ids: list[str] | None = Field(default=None, max_length=20) - - -class OntologyGovernancePolicyRequest(BaseModel): - force_plugin_import_build_validation: bool - - -class OntologyVersionRequest(BaseModel): - document: dict[str, Any] - activate: bool = False - - -class OntologyWorkingDraftRequest(BaseModel): - document: dict[str, Any] - draft_version_id: str | None = Field(default=None, max_length=64) - expected_checksum: str | None = Field(default=None, min_length=64, max_length=64) - - -class OntologyPackFlagsRequest(BaseModel): - is_enabled: bool | None = None - is_default: bool | None = None - - -class OntologyDraftReviewRequest(BaseModel): - approved: bool - comment: str = Field(default="", max_length=2000) - - -class OntologyEvolutionGenerateRequest(BaseModel): - min_occurrences: int = Field(default=3, ge=2, le=20) - limit: int = Field(default=500, ge=10, le=2000) - model_name: str | None = Field(default=None, max_length=255) - - -class OntologyBuildValidationRequest(BaseModel): - asset_type: str = Field(pattern="^(skill|tool|subagent)$") - name: str = Field(min_length=1, max_length=255) - description: str = Field(default="", max_length=4000) - instructions: str = Field(default="", max_length=20000) - tool_names: list[str] = Field(default_factory=list, max_length=200) - mcp_server_ids: list[str] = Field(default_factory=list, max_length=100) - skill_ids: list[str] = Field(default_factory=list, max_length=100) - plugin_ids: list[str] = Field(default_factory=list, max_length=100) - ontology_tags: list[str] = Field(default_factory=list, max_length=100) - output_schema: dict[str, Any] | None = None - tool_schemas: dict[str, dict[str, Any]] = Field(default_factory=dict) - - -def _pack_dict(row, versions: list | None = None) -> dict[str, Any]: - data = { - "pack_id": row.pack_id, - "name": row.name, - "domain": row.domain, - "description": row.description, - "is_enabled": row.is_enabled, - "is_default": row.is_default, - "active_version_id": row.active_version_id, - "created_at": row.created_at.isoformat() if row.created_at else None, - "updated_at": row.updated_at.isoformat() if row.updated_at else None, - } - if versions is not None: - data["versions"] = [_version_dict(item) for item in versions] - working_draft = next((item for item in versions if item.status == "draft"), None) - data["working_draft_version_id"] = ( - working_draft.version_id if working_draft is not None else None - ) - return data - - -def _version_dict(row, *, include_content: bool = False) -> dict[str, Any]: - data = { - "version_id": row.version_id, - "pack_id": row.pack_id, - "version": row.version, - "checksum": row.checksum, - "status": row.status, - "validation_report": row.validation_report or {}, - "created_at": row.created_at.isoformat() if row.created_at else None, - "updated_at": row.updated_at.isoformat() if row.updated_at else None, - "activated_at": row.activated_at.isoformat() if row.activated_at else None, - } - if include_content: - data["content"] = row.content - return data - - -@router.get("/v1/ontologies/settings", summary="获取当前用户的本体校验设置") -def get_ontology_settings( - user: UserContext = Depends(get_current_user), - db: Session = Depends(get_db), -): - user_settings = UserService(db).get_user_settings(str(user.user_id)) - allowed = user_can_use_ontology_validation(db, str(user.user_id)) - selected = user_settings.get("ontology_pack_ids") or [] - active = OntologyService(db).repo.get_active_versions(selected or None) - return success_response( - data={ - "allowed": allowed, - "ontology_enabled": bool(allowed and user_settings.get("ontology_enabled", False)), - "ontology_pack_ids": selected, - "available": bool(active), - "plugin_import_build_validation_forced": (plugin_import_build_validation_forced(db)), - "active_packs": [ - {"pack_id": row.pack_id, "version_id": row.version_id, "version": row.version} - for row in active - ], - } - ) - - -@router.patch("/v1/ontologies/settings", summary="更新当前用户的本体校验设置") -def update_ontology_settings( - body: OntologySettingsRequest, - user: UserContext = Depends(get_current_user), - db: Session = Depends(get_db), -): - require_ontology_validation_permission(db, str(user.user_id)) - service = OntologyService(db) - pack_ids = body.ontology_pack_ids or [] - missing = [pack_id for pack_id in pack_ids if service.repo.get_pack(pack_id) is None] - if missing: - raise BadRequestError("包含不存在的 Domain Pack", data={"pack_ids": missing}) - active = service.repo.get_active_versions(pack_ids or None) - if body.ontology_enabled and not active: - raise BadRequestError("当前没有可启用的已激活 Domain Pack") - patch: dict[str, Any] = {"ontology_enabled": body.ontology_enabled} - if body.ontology_pack_ids is not None: - patch["ontology_pack_ids"] = pack_ids - UserService(db).update_user_metadata(str(user.user_id), patch) - return success_response( - data={ - **patch, - "available": bool(active), - "active_pack_count": len(active), - } - ) - - -@router.get("/v1/ontologies/runtime/preview", summary="预览当前用户本轮将注入的本体策略") -def preview_ontology_runtime( - task: str = Query(..., min_length=1, max_length=4000), - user: UserContext = Depends(get_current_user), - db: Session = Depends(get_db), -): - if not user_can_use_ontology_validation(db, str(user.user_id)): - return success_response(data={"enabled": False, "packs": [], "review_level": "none"}) - settings = UserService(db).get_user_settings(str(user.user_id)) - if not settings.get("ontology_enabled", False): - return success_response(data={"enabled": False, "packs": [], "review_level": "none"}) - from core.ontology.runtime_policy import build_user_ontology_runtime - - _, runtime = build_user_ontology_runtime( - user_id=str(user.user_id), - task=task, - db=db, - ) - return success_response(data=runtime) - - -@router.get("/v1/ontologies/tags", summary="获取可选的受控本体标签") -def list_ontology_tag_options( - asset_kind: Literal["tool", "skill", "subagent"] = Query(...), - _: UserContext = Depends(get_current_user), - db: Session = Depends(get_db), -): - items = OntologyService(db).list_asset_tag_options(asset_kind) - return success_response(data={"asset_kind": asset_kind, "items": items}) - - -@router.get("/v1/ontologies/governance/access", summary="CE 本体治理访问探针") -def ontology_governance_access( - user: UserContext | None = Depends(require_auth(False)), - db: Session = Depends(get_db), -): - """Let the settings page hide global governance controls from unauthorized users.""" - allowed = bool(user) and user_can_manage_ontology_governance(db, user.user_id) - return success_response(data={"allowed": allowed, "edition": settings.edition.edition}) - - -@router.get( - "/v1/ontologies/governance/policy", - dependencies=[Depends(require_ontology_governance)], - summary="CE 设置页获取本体治理策略", -) -@router.get( - "/v1/admin/ontologies/policy", - dependencies=[Depends(require_admin)], - summary="获取本体治理策略", -) -def get_ontology_governance_policy(db: Session = Depends(get_db)): - return success_response( - data={"force_plugin_import_build_validation": (plugin_import_build_validation_forced(db))} - ) - - -@router.patch( - "/v1/ontologies/governance/policy", - dependencies=[Depends(require_ontology_governance)], - summary="CE 设置页更新本体治理策略", -) -@router.patch( - "/v1/admin/ontologies/policy", - dependencies=[Depends(require_admin)], - summary="更新本体治理策略", -) -def update_ontology_governance_policy( - body: OntologyGovernancePolicyRequest, - db: Session = Depends(get_db), -): - enabled = set_plugin_import_build_validation_forced( - db, - body.force_plugin_import_build_validation, - ) - return success_response(data={"force_plugin_import_build_validation": enabled}) - - -@router.get( - "/v1/ontologies/governance", - dependencies=[Depends(require_ontology_governance)], - summary="CE 设置页列出 Domain Pack 与版本", -) -@router.get( - "/v1/admin/ontologies", - dependencies=[Depends(require_admin)], - summary="列出 Domain Pack 与版本", -) -def list_ontology_packs(db: Session = Depends(get_db)): - service = OntologyService(db) - return success_response( - data={ - "items": [ - _pack_dict(row, service.repo.list_versions(row.pack_id)) - for row in service.repo.list_packs() - ] - } - ) - - -@router.get( - "/v1/ontologies/governance/tags", - dependencies=[Depends(require_ontology_governance)], - summary="CE 设置页获取受控本体标签", -) -@router.get( - "/v1/admin/ontologies/tags", - dependencies=[Depends(require_admin)], - summary="获取管理端可选的受控本体标签", -) -def list_admin_ontology_tag_options( - asset_kind: Literal["tool", "skill", "subagent"] = Query(...), - db: Session = Depends(get_db), -): - items = OntologyService(db).list_asset_tag_options(asset_kind) - return success_response(data={"asset_kind": asset_kind, "items": items}) - - -@router.get( - "/v1/ontologies/governance/metrics", - dependencies=[Depends(require_ontology_governance)], - summary="CE 设置页查询本体闭环治理指标", -) -@router.get( - "/v1/admin/ontologies/metrics", - dependencies=[Depends(require_admin)], - summary="查询本体闭环治理指标", -) -def get_ontology_metrics(db: Session = Depends(get_db)): - event_decisions = Counter( - { - decision: count - for decision, count in db.query( - OntologyEnforcementEvent.decision, - func.count(OntologyEnforcementEvent.event_id), - ) - .group_by(OntologyEnforcementEvent.decision) - .all() - } - ) - review_verdicts = Counter( - { - verdict: count - for verdict, count in db.query( - OntologyReviewRun.verdict, - func.count(OntologyReviewRun.review_id), - ) - .group_by(OntologyReviewRun.verdict) - .all() - } - ) - drafts = db.query(OntologyDraft).all() - draft_statuses = Counter(row.review_status for row in drafts) - source_stats: dict[str, Counter] = defaultdict(Counter) - for draft in drafts: - source_stats[draft.source_type][draft.review_status] += 1 - source_acceptance = {} - for source, counts in source_stats.items(): - decided = counts["approved"] + counts["rejected"] - source_acceptance[source] = { - "pending": counts["pending"], - "approved": counts["approved"], - "rejected": counts["rejected"], - "acceptance_rate": round(counts["approved"] / decided, 4) if decided else None, - } - - cutoff = utc_now() - timedelta(days=30) - daily: dict[str, Counter] = defaultdict(Counter) - recent_events = ( - db.query(OntologyEnforcementEvent.decision, OntologyEnforcementEvent.created_at) - .filter(OntologyEnforcementEvent.created_at >= cutoff) - .all() - ) - recent_reviews = ( - db.query(OntologyReviewRun.verdict, OntologyReviewRun.created_at) - .filter(OntologyReviewRun.created_at >= cutoff) - .all() - ) - for decision, created_at in recent_events: - if created_at and as_utc(created_at) >= cutoff: - daily[created_at.date().isoformat()][f"event_{decision}"] += 1 - for verdict, created_at in recent_reviews: - if created_at and as_utc(created_at) >= cutoff: - daily[created_at.date().isoformat()][f"review_{verdict}"] += 1 - - return success_response( - data={ - "events_total": sum(event_decisions.values()), - "event_decisions": dict(event_decisions), - "reviews_total": sum(review_verdicts.values()), - "review_verdicts": dict(review_verdicts), - "drafts_total": len(drafts), - "draft_statuses": dict(draft_statuses), - "materialized_total": sum( - 1 for row in drafts if (row.proposal or {}).get("materialized_version_id") - ), - "source_acceptance": source_acceptance, - "daily_30d": [{"date": date, **dict(counts)} for date, counts in sorted(daily.items())], - } - ) - - -@router.post( - "/v1/ontologies/governance/build/validate", - dependencies=[Depends(require_ontology_governance)], - summary="CE 设置页校验技能、工具或子智能体定义", -) -@router.post( - "/v1/admin/ontologies/build/validate", - dependencies=[Depends(require_admin)], - summary="用激活的 Domain Pack 校验技能、工具或子智能体定义", -) -def validate_ontology_build_asset( - body: OntologyBuildValidationRequest, - db: Session = Depends(get_db), -): - from core.ontology.build_validator import OntologyBuildValidator - - report = OntologyBuildValidator(db).validate( - asset_type=body.asset_type, # type: ignore[arg-type] - name=body.name, - description=body.description, - instructions=body.instructions, - tool_names=body.tool_names, - mcp_server_ids=body.mcp_server_ids, - skill_ids=body.skill_ids, - plugin_ids=body.plugin_ids, - ontology_tags=body.ontology_tags, - output_schema=body.output_schema, - tool_schemas=body.tool_schemas, - ) - return success_response(data=report.as_dict()) - - -@router.post( - "/v1/ontologies/governance/validate", - dependencies=[Depends(require_ontology_governance)], - summary="CE 设置页校验 Domain Pack", -) -@router.post( - "/v1/admin/ontologies/validate", - dependencies=[Depends(require_admin)], - summary="校验 Domain Pack", -) -def validate_ontology_pack(body: dict[str, Any], db: Session = Depends(get_db)): - _, report = OntologyService(db).validate_document(body) - return success_response(data=report) - - -@router.post( - "/v1/ontologies/governance/versions", - dependencies=[Depends(require_ontology_governance)], - summary="CE 设置页导入 Domain Pack 新版本", -) -@router.post( - "/v1/admin/ontologies/versions", - dependencies=[Depends(require_admin)], - summary="导入 Domain Pack 新版本", -) -def create_ontology_version( - body: OntologyVersionRequest, - db: Session = Depends(get_db), -): - row = OntologyService(db).create_version(body.document, activate=body.activate) - return success_response(data=_version_dict(row, include_content=True)) - - -@router.put( - "/v1/ontologies/governance/{pack_id}/draft", - dependencies=[Depends(require_ontology_governance)], - summary="CE 设置页创建或更新 Domain Pack 工作草稿", -) -@router.put( - "/v1/admin/ontologies/{pack_id}/draft", - dependencies=[Depends(require_admin)], - summary="创建或更新 Domain Pack 工作草稿", -) -def save_ontology_working_draft( - pack_id: str, - body: OntologyWorkingDraftRequest, - db: Session = Depends(get_db), -): - row, created = OntologyService(db).save_working_draft( - pack_id, - body.document, - draft_version_id=body.draft_version_id, - expected_checksum=body.expected_checksum, - ) - return success_response(data={**_version_dict(row, include_content=True), "created": created}) - - -@router.delete( - "/v1/ontologies/governance/{pack_id}/draft/{version_id}", - dependencies=[Depends(require_ontology_governance)], - summary="CE 设置页放弃 Domain Pack 工作草稿", -) -@router.delete( - "/v1/admin/ontologies/{pack_id}/draft/{version_id}", - dependencies=[Depends(require_admin)], - summary="放弃 Domain Pack 工作草稿", -) -def discard_ontology_working_draft( - pack_id: str, - version_id: str, - db: Session = Depends(get_db), -): - OntologyService(db).discard_working_draft(pack_id, version_id) - return success_response(data={"pack_id": pack_id, "version_id": version_id}) - - -@router.post( - "/v1/ontologies/governance/{pack_id}/versions/{version_id}/activate", - dependencies=[Depends(require_ontology_governance)], - summary="CE 设置页激活 Domain Pack 版本", -) -@router.post( - "/v1/admin/ontologies/{pack_id}/versions/{version_id}/activate", - dependencies=[Depends(require_admin)], - summary="激活 Domain Pack 版本", -) -def activate_ontology_version( - pack_id: str, - version_id: str, - db: Session = Depends(get_db), -): - row = OntologyService(db).activate(pack_id, version_id) - return success_response(data=_version_dict(row, include_content=True)) - - -@router.patch( - "/v1/ontologies/governance/{pack_id}", - dependencies=[Depends(require_ontology_governance)], - summary="CE 设置页更新 Domain Pack 启用/默认状态", -) -@router.patch( - "/v1/admin/ontologies/{pack_id}", - dependencies=[Depends(require_admin)], - summary="更新 Domain Pack 启用/默认状态", -) -def update_ontology_pack( - pack_id: str, - body: OntologyPackFlagsRequest, - db: Session = Depends(get_db), -): - row = OntologyService(db).set_pack_flags( - pack_id, - is_enabled=body.is_enabled, - is_default=body.is_default, - ) - return success_response(data=_pack_dict(row)) - - -@router.get( - "/v1/ontologies/governance/{pack_id}/versions/{version_id}/export", - dependencies=[Depends(require_ontology_governance)], - summary="CE 设置页导出 Domain Pack 版本", -) -@router.get( - "/v1/admin/ontologies/{pack_id}/versions/{version_id}/export", - dependencies=[Depends(require_admin)], - summary="导出 Domain Pack 版本", -) -def export_ontology_version( - pack_id: str, - version_id: str, - db: Session = Depends(get_db), -): - row = OntologyService(db).repo.get_version(version_id) - if not row or row.pack_id != pack_id: - raise ResourceNotFoundError("ontology_pack_version", version_id) - return success_response(data=row.content) - - -@router.get( - "/v1/ontologies/governance/events", - dependencies=[Depends(require_ontology_governance)], - summary="CE 设置页查询本体门禁审计事件", -) -@router.get( - "/v1/admin/ontologies/events", - dependencies=[Depends(require_admin)], - summary="查询本体门禁审计事件", -) -def list_ontology_events( - limit: int = Query(100, ge=1, le=500), - chat_id: str | None = None, - decision: str | None = None, - db: Session = Depends(get_db), -): - rows = OntologyService(db).repo.list_events(limit=limit, chat_id=chat_id, decision=decision) - return success_response( - data={ - "items": [ - { - "event_id": row.event_id, - "user_id": row.user_id, - "chat_id": row.chat_id, - "pack_id": row.pack_id, - "version_id": row.version_id, - "rule_id": row.rule_id, - "stage": row.stage, - "event_type": row.event_type, - "decision": row.decision, - "mode": row.mode, - "target": row.target, - "latency_ms": row.latency_ms, - "details": row.details or {}, - "created_at": row.created_at.isoformat() if row.created_at else None, - } - for row in rows - ] - } - ) - - -@router.get( - "/v1/ontologies/governance/reviews", - dependencies=[Depends(require_ontology_governance)], - summary="CE 设置页查询本体评审记录", -) -@router.get( - "/v1/admin/ontologies/reviews", - dependencies=[Depends(require_admin)], - summary="查询本体评审记录", -) -def list_ontology_reviews( - limit: int = Query(100, ge=1, le=500), - db: Session = Depends(get_db), -): - rows = OntologyService(db).repo.list_reviews(limit=limit) - return success_response( - data={ - "items": [ - { - "review_id": row.review_id, - "chat_id": row.chat_id, - "level": row.level, - "subject_type": row.subject_type, - "verdict": row.verdict, - "evidence": row.evidence or [], - "feedback": row.feedback, - "reviewers": row.reviewers or [], - "latency_ms": row.latency_ms, - "created_at": row.created_at.isoformat() if row.created_at else None, - } - for row in rows - ] - } - ) - - -@router.get( - "/v1/ontologies/governance/drafts", - dependencies=[Depends(require_ontology_governance)], - summary="CE 设置页查询本体演进草案", -) -@router.get( - "/v1/admin/ontologies/drafts", - dependencies=[Depends(require_admin)], - summary="查询待人工审查的本体演进草案", -) -def list_ontology_drafts( - status: str | None = Query(None, pattern="^(pending|approved|rejected)$"), - limit: int = Query(100, ge=1, le=500), - db: Session = Depends(get_db), -): - rows = OntologyService(db).repo.list_drafts(status=status, limit=limit) - return success_response( - data={ - "items": [ - { - "draft_id": row.draft_id, - "pack_id": row.pack_id, - "source_type": row.source_type, - "candidate_type": row.candidate_type, - "proposal": row.proposal, - "evidence": row.evidence, - "value_score": row.value_score, - "review_status": row.review_status, - "reviewer_comment": row.reviewer_comment, - "created_at": row.created_at.isoformat() if row.created_at else None, - } - for row in rows - ] - } - ) - - -@router.post( - "/v1/ontologies/governance/drafts/{draft_id}/review", - dependencies=[Depends(require_ontology_governance)], - summary="CE 设置页审核本体演进草案", -) -@router.post( - "/v1/admin/ontologies/drafts/{draft_id}/review", - dependencies=[Depends(require_admin)], - summary="人工通过或驳回本体演进草案", -) -def review_ontology_draft( - draft_id: str, - body: OntologyDraftReviewRequest, - db: Session = Depends(get_db), -): - row = OntologyService(db).repo.get_draft(draft_id) - if not row: - raise ResourceNotFoundError("ontology_draft", draft_id) - if row.review_status != "pending": - raise BadRequestError("该演进草案已经完成审核,不能重复修改裁决") - row.review_status = "approved" if body.approved else "rejected" - row.reviewer_comment = body.comment - row.reviewed_at = utc_now() - row.updated_at = utc_now() - db.commit() - return success_response(data={"draft_id": row.draft_id, "review_status": row.review_status}) - - -@router.post( - "/v1/ontologies/governance/evolution/generate", - dependencies=[Depends(require_ontology_governance)], - summary="CE 设置页生成本体演进草案", -) -@router.post( - "/v1/admin/ontologies/evolution/generate", - dependencies=[Depends(require_admin)], - summary="从重复门禁证据生成本体演进草案", -) -async def generate_ontology_evolution_drafts( - body: OntologyEvolutionGenerateRequest, -): - from core.ontology.evolution import generate_candidates - - result = await generate_candidates( - min_occurrences=body.min_occurrences, - limit=body.limit, - model_name=body.model_name, - ) - return success_response(data=result) - - -@router.post( - "/v1/ontologies/governance/drafts/{draft_id}/materialize", - dependencies=[Depends(require_ontology_governance)], - summary="CE 设置页物化已批准的本体演进草案", -) -@router.post( - "/v1/admin/ontologies/drafts/{draft_id}/materialize", - dependencies=[Depends(require_admin)], - summary="把已批准演进草案物化为未激活的新版本", -) -def materialize_ontology_draft( - draft_id: str, - db: Session = Depends(get_db), -): - from core.ontology.evolution import OntologyEvolutionService - - row = OntologyEvolutionService(db).materialize_approved_draft(draft_id) - return success_response(data=_version_dict(row, include_content=True)) diff --git a/src/backend/api/routes/v1/users.py b/src/backend/api/routes/v1/users.py index 9947ba99..3e65b2f0 100644 --- a/src/backend/api/routes/v1/users.py +++ b/src/backend/api/routes/v1/users.py @@ -181,7 +181,7 @@ def complete_my_onboarding( ): """Complete the CE bootstrap-owner setup after a usable main model exists. - Language, service endpoints, memory, and ontology preferences are saved by + Language, service endpoints, and memory preferences are saved by their existing APIs as each wizard step advances. This endpoint is the final, idempotent checkpoint that unlocks the normal application shell. """ diff --git a/src/backend/api/startup/runtime.py b/src/backend/api/startup/runtime.py index 0a77b653..2876dbe6 100644 --- a/src/backend/api/startup/runtime.py +++ b/src/backend/api/startup/runtime.py @@ -15,6 +15,13 @@ async def _startup_thread_budget(): apply_thread_budget() +async def _startup_validate_space_transport(): + """Refuse to serve with a space transport the deployment cannot provide.""" + from core.files.spaces.transport import validate_space_transport + + logger.info("[startup] space transport: %s", validate_space_transport()) + + async def _startup_watch_capability_changes(): """智能体 / 技能 / 连接器 / 插件被增删时递增能力变更号,桌面端据此同步一次。""" from features.desktop.capabilities.change_signal import watch_capability_tables diff --git a/src/backend/cli.py b/src/backend/cli.py index c39813c4..e643853a 100644 --- a/src/backend/cli.py +++ b/src/backend/cli.py @@ -132,6 +132,12 @@ def apply_local_env(port: int) -> dict: # orchestration.run_event_stream, selected by this being empty. "REDIS_URL": "", "SESSION_STORE": "memory", + # The desktop app restarts this process on every launch; keep logins in a + # file so the user is not sent back through browser confirmation each time. + # One local user on their own machine: sessions last as long as the + # default "remember me" window instead of a shared server's 8 hours. + "SESSION_PERSIST_PATH": str(dd / "sessions.json"), + "SESSION_TTL_HOURS": "720", "AUTH_MODE": "session", "SSO_LOGIN_MODE": "local", "LOCAL_AUTH_ENABLED": "true", diff --git a/src/backend/core/agent_runtime/integrations/ontology_repair.py b/src/backend/core/agent_runtime/integrations/ontology_repair.py index 78a01e25..c79e0083 100644 --- a/src/backend/core/agent_runtime/integrations/ontology_repair.py +++ b/src/backend/core/agent_runtime/integrations/ontology_repair.py @@ -1,366 +1,9 @@ -"""Re-run the originating agent to produce a revision when ontology review rejects an answer.""" +"""Answer repair only follows a claimed review, which never happens in CE.""" from __future__ import annotations -from core.llm.citations.allocator import collect_citation_dicts -import json -import logging -import typing as _typing -from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple +from typing import Any -import core.agent_runtime.events as agent_runtime_events -import core.agent_runtime.integrations.ontology_evidence as ontology_evidence -import core.config.display_names as display_names -import core.ontology.revision as revision -from core.llm.citations.allocator import AnchorAllocator -from core.llm.runtime.streaming import StreamingAgent -logger = logging.getLogger(__name__) - - -def _ontology_repair_prompt(payload: Dict[str, Any]) -> str: - violations = payload.get("violations") or [] - feedback = payload.get("feedback") or [] - affected_claims = payload.get("affected_claims") or [] - original_task = str(payload.get("original_task") or "").strip() - current_answer = str(payload.get("answer") or "").strip() - citations = [ - { - "id": item.get("id"), - "tool_name": item.get("tool_name"), - "title": item.get("title"), - } - for item in payload.get("citations") or [] - if isinstance(item, dict) - ] - return ( - "\n" - "这是同一任务唯一一次领域本体修复,不是新的用户任务。" - "优先级强制规则:已经激活的 enforce 级领域本体约束及本修复指令,高于用户原始提示中" - "与其冲突的‘不要调用工具’、‘原样照抄’或类似限制;这些限制只能约束正常生成阶段," - "不能阻止本体修复取证和纠错。若‘确定性违规’包含‘未调用’某些必需工具,本轮必须" - "实际调用这些工具后才能形成终稿;以用户禁止工具为理由跳过调用、保留原错误断言," - "或再次原样输出待修订内容,均视为修复失败。" - "先比较现有证据与评审问题,明确是否出现了需要重新搜索的新内容或可补充调用的新工具。" - "有证据缺口时必须调用真实工具补齐;现有证据已足够时才可直接精准修订。" - "禁止只做同义改写,也不得虚构工具调用或引用。" - "证据核对、逐项判定、原文对比和修改说明都属于内部校验过程,只能放在" - " thinking/reasoning 通道;除非用户最初明确要求核验报告,否则不得把这些内容写入" - "最终回复或生成的文件。" - "必须继续遵守用户最初指令中的输出与交付形式;如果原任务要求生成 Word、PPT、" - "表格、网页或其他文件,修订时仍须调用相应工具生成该类产物,不得降级为纯文字替代。" - "最终交付必须是利用核验结论直接修正后的完整润色稿,而不是关于原答案的评估报告。" - "保留原任务要求的文体、篇幅、结构和语气,只改正有问题的事实、推断与表述;" - "以待修订原始输出为底稿做最小必要修改,不要因为查到更多资料就加入与原主张无关的" - "企业背景、资质或延伸分析。句子级任务应尽量保留原句的主语与分句顺序,逐个修正" - "不成立的断言,形成简洁、自然、可直接交付的一句话终稿。" - "用户只要求一句话时仍输出一句话,不得自行扩写成包含‘原始语句’、‘逐项核验’、" - "‘主张评估’或‘修改说明’等章节的报告。若交付物是文件,文件正文也必须是润色终稿," - "并重新生成、交付修正后的文件。没有获得证据的风险项只能表述为‘待核验’、" - "‘暂无数据支撑’或‘无法判断’,绝不能反向断言为‘不存在’、‘没有风险’或‘风险为零’。" - "在输出终稿和生成文件前,必须在内部逐条检查‘确定性违规’中的每个条件:若要求" - "最低长度,终稿必须达到该长度;若要求引用,相关事实后必须包含真实引用标记" - "(从工具结果复制 cite_id,写成 `[锚文本](cite:eN)`)" - "且按规则补齐参考资料;若要求区分事实、推断和待核验项,可在同一句中用分号和简短标签" - "表达,不得省略。先确定唯一的合格终稿,再把完全相同的正文写入用户要求的文件," - "不得先生成文件后又输出一份更短、缺引用或结论不同的候选正文。" - "思考过程只放在模型的 thinking/reasoning 通道。最终正文必须且只能放在" - " `完整候选答案` 中。" - "在正式输出最终正文前,不要复述、引用或示例化这组标签,也不要用省略号代替正文。" - "标签内必须是可独立阅读、内容完整的修订答案;若无法修订,则原样放入当前完整答案。" - "事实、推断和待核验项必须明确区分;引用已有或新增工具结果时,把工具结果里" - "标注的 cite_id 原样复制进 `[锚文本](cite:eN)`,禁止自行编号。\n" - f"用户最初指令:{json.dumps(original_task, ensure_ascii=False)[:12000]}\n" - f"待修订原始输出:{json.dumps(current_answer, ensure_ascii=False)[:12000]}\n" - f"修复轮次:{payload.get('attempt', 1)}\n" - f"问题来源:{payload.get('source', 'review')}\n" - f"确定性违规:{json.dumps(violations, ensure_ascii=False)[:8000]}\n" - f"委员会意见:{json.dumps(feedback, ensure_ascii=False)[:8000]}\n" - f"受影响内容:{json.dumps(affected_claims, ensure_ascii=False)[:5000]}\n" - f"当前可用引用:{json.dumps(citations, ensure_ascii=False)[:5000]}\n" - "直接开始补充工具或输出修复后的完整答案。\n" - "" - ) - - -async def _run_ontology_repair_round( - *, - streaming_agent: StreamingAgent, - context: Dict[str, Any], - payload: Dict[str, Any], - runtime: Dict[str, Any], - trace: List[Dict[str, Any]], - citations: List[Dict[str, Any]], - allocator: Optional[AnchorAllocator] = None, - event_cursor: int = 0, - subagent_log_id: Optional[str] = None, - event_sink: Optional[Callable[[Dict[str, Any]], Awaitable[None]]] = None, -) -> Tuple[str, List[Dict[str, Any]], int, int]: - """Continue the originating agent and stream a separate revision candidate.""" - answer = "" - events: _typing.List[_typing.Dict[str, _typing.Any]] = [] - tool_count = 0 - text_buffer = "" - raw_text = "" - revision_open = False - revision_closed = False - thinking_line_prefix = "" - - async def _publish(event: Dict[str, Any]) -> None: - events.append(event) - if event_sink is not None: - await event_sink(event) - - async def _publish_text_thinking(delta: str) -> None: - """Publish text-channel reasoning and remember its current line prefix.""" - nonlocal thinking_line_prefix - if not delta: - return - if "\n" in delta: - thinking_line_prefix = delta.rsplit("\n", 1)[-1] - else: - thinking_line_prefix = (thinking_line_prefix + delta)[-256:] - await _publish({"type": "ontology_revision_thinking", "delta": delta}) - - def _is_revision_opening_boundary(line_before_open: str) -> bool: - """Allow a revision wrapper after whitespace or a reasoning close tag.""" - - prefix = line_before_open.rstrip() - return not prefix or prefix.endswith(("
", "")) - - await _publish( - { - "type": "ontology_repair", - "status": "started", - "attempt": payload.get("attempt", 1), - "source": payload.get("source", "review"), - } - ) - - stream = streaming_agent.stream( - [{"role": "user", "content": _ontology_repair_prompt(payload)}], - context, - ).__aiter__() - while True: - try: - if subagent_log_id: - event_type, event_payload = await agent_runtime_events._anext_in_subagent_log_scope( - stream, - subagent_log_id, - ) - else: - event_type, event_payload = await anext(stream) - except StopAsyncIteration: - break - - state_runtime = getattr(streaming_agent.agent.state, "ontology_runtime", None) - if isinstance(state_runtime, dict): - runtime = state_runtime - context["ontology_runtime"] = state_runtime - pending_events = runtime.get("runtime_events", [])[event_cursor:] - for item in pending_events: - await _publish(dict(item)) - event_cursor += len(pending_events) - - if event_type == "vision_progress": - await _publish({"type": "vision_progress", **(event_payload or {})}) - continue - - if event_type == "text_delta": - delta = str(event_payload or "") - if not delta: - continue - raw_text += delta - if revision_closed: - await _publish_text_thinking(delta) - continue - text_buffer += delta - while text_buffer: - if not revision_open: - open_idx = text_buffer.find("") - if open_idx < 0: - keep = min(len("") - 1, len(text_buffer)) - safe_len = len(text_buffer) - keep - if safe_len > 0: - await _publish_text_thinking(text_buffer[:safe_len]) - text_buffer = text_buffer[safe_len:] - break - before_open = text_buffer[:open_idx] - line_before_open = (thinking_line_prefix + before_open).rsplit("\n", 1)[-1] - if not _is_revision_opening_boundary(line_before_open): - # Reasoning models sometimes describe the contract as - # `...`. A real - # wrapper starts a new line or immediately follows the - # model's reasoning close tag. Other inline occurrences - # remain folded as examples rather than candidate text. - ignored = text_buffer[: open_idx + len("")] - await _publish_text_thinking(ignored) - text_buffer = text_buffer[open_idx + len("") :] - continue - if open_idx > 0: - await _publish_text_thinking(before_open) - text_buffer = text_buffer[open_idx + len("") :] - revision_open = True - continue - close_idx = text_buffer.find("") - if close_idx < 0: - keep = min(len("") - 1, len(text_buffer)) - safe_len = len(text_buffer) - keep - if safe_len > 0: - candidate_delta = text_buffer[:safe_len] - answer += candidate_delta - await _publish({"type": "ontology_revision", "delta": candidate_delta}) - text_buffer = text_buffer[safe_len:] - break - if close_idx > 0: - candidate_delta = text_buffer[:close_idx] - answer += candidate_delta - await _publish({"type": "ontology_revision", "delta": candidate_delta}) - text_buffer = text_buffer[close_idx + len("") :] - revision_closed = True - break - elif event_type == "thinking_delta": - await _publish( - { - "type": "ontology_revision_thinking", - "delta": str(event_payload or ""), - } - ) - elif event_type == "tool_call_start": - tool_name = str(event_payload.get("name") or "unknown") - if tool_name != "update_plan": - await _publish( - { - "type": "tool_call_start", - "tool_name": tool_name, - "tool_display_name": display_names.TOOL_DISPLAY_NAMES.get( - tool_name, tool_name - ), - "tool_id": str(event_payload.get("id") or ""), - "scope": "ontology_revision", - } - ) - elif event_type == "tool_call_delta": - tool_name = str(event_payload.get("name") or "unknown") - if tool_name != "update_plan" and event_payload.get("delta"): - await _publish( - { - "type": "tool_call_delta", - "tool_name": tool_name, - "tool_id": str(event_payload.get("id") or ""), - "arguments_delta": str(event_payload.get("delta") or ""), - "scope": "ontology_revision", - } - ) - elif event_type == "tool_call": - tool_name = str(event_payload.get("name") or "unknown") - tool_id = str(event_payload.get("id") or "") - tool_args = event_payload.get("args") or {} - trace.append( - { - "type": "tool_call", - "tool_id": tool_id, - "tool_name": tool_name, - "input": ontology_evidence._bounded_trace_value(tool_args), - "source": "ontology_repair", - } - ) - await _publish( - { - "type": "tool_call", - "tool_name": tool_name, - "tool_display_name": display_names.TOOL_DISPLAY_NAMES.get(tool_name, tool_name), - "tool_args": tool_args, - "input": tool_args, - "tool_id": tool_id, - "scope": "ontology_revision", - } - ) - elif event_type == "tool_result": - tool_count += 1 - tool_name = str(event_payload.get("name") or "unknown") - tool_id = str(event_payload.get("id") or "") - result = ontology_evidence._parse_tool_result_value(event_payload.get("content")) - cit_dicts = collect_citation_dicts(tool_id, allocator) - citations.extend(cit_dicts) - trace.append( - { - "type": "tool_result", - "tool_id": tool_id, - "tool_name": tool_name, - "result": ontology_evidence._bounded_trace_value(result), - "citations": cit_dicts, - "source": "ontology_repair", - } - ) - await _publish( - { - "type": "tool_result", - "tool_name": tool_name, - "result": result, - "tool_id": tool_id, - "citations": cit_dicts, - "scope": "ontology_revision", - } - ) - elif event_type == "subagent_event": - nested_citations = ontology_evidence._capture_nested_ontology_evidence( - event_payload or {}, - trace, - citations, - allocator, - ) - sub_type = str((event_payload or {}).get("sub_type") or "") - if sub_type in { - "start", - "thinking", - "content", - "tool_call", - "tool_call_delta", - "tool_result", - "end", - }: - await _publish( - { - "type": "subagent_event", - **(event_payload or {}), - "scope": "ontology_revision", - **({"citations": nested_citations} if nested_citations else {}), - } - ) - elif event_type == "tool_pending": - await _publish( - { - "type": "tool_pending", - **(event_payload or {}), - "scope": "ontology_revision", - } - ) - elif event_type in {"file_confirm", "design_pick"}: - raise RuntimeError("本体自动修复轮不支持等待交互确认,请转人工复核") - elif event_type == "error": - if isinstance(event_payload, BaseException): - raise event_payload - raise RuntimeError(str(event_payload)) - - if revision_open and not revision_closed and text_buffer: - answer += text_buffer - await _publish({"type": "ontology_revision", "delta": text_buffer}) - elif not revision_open and text_buffer.strip(): - # Compatibility fallback for models that ignored the wrapper. When - # embedded text exists, only the content after the last closing - # marker can be a candidate; never promote the hidden reasoning itself. - fallback = revision.normalize_revision_candidate(raw_text.rsplit("", 1)[-1]) - if revision.is_substantive_revision(fallback): - answer = fallback - await _publish({"type": "ontology_revision", "delta": answer}) - - await _publish( - { - "type": "ontology_repair", - "status": "completed", - "attempt": payload.get("attempt", 1), - "tool_calls": tool_count, - } - ) - return revision.normalize_revision_candidate(answer), events, event_cursor, tool_count +async def _run_ontology_repair_round(*_args: Any, **_kwargs: Any): + raise RuntimeError("Community edition has no ontology review to repair") diff --git a/src/backend/core/auth/capabilities.py b/src/backend/core/auth/capabilities.py index fa7a05f4..09b6d9e1 100644 --- a/src/backend/core/auth/capabilities.py +++ b/src/backend/core/auth/capabilities.py @@ -12,7 +12,6 @@ | ``can_import_plugin``| bool| ``False`` | Install/import plugins | | ``can_add_agent`` | bool | ``False`` | Build own sub-agents / install & publish on sub-agent marketplace | | ``can_switch_model`` | bool | ``False`` | User-side chat model switching | -| ``can_use_ontology_validation`` | bool | ``False`` | Configure and use personal ontology validation | | ``can_use_personal_agent`` | bool | ``False`` | EE account permission for the shared Personal Agent module | | ``allowed_apps`` | list/None | ``None`` | App visibility whitelist (``None`` = all enabled apps) | @@ -55,7 +54,6 @@ "can_create_channel_bot": False, "can_switch_model": False, "can_use_voice": False, # Dictation is opt-in, including for administrators. - "can_use_ontology_validation": False, "can_run_autonomous_loop": True, # long-running autonomous loop (open in CE, enabled by default, can be disabled per user/team) # 自定义对话模式:授予后「设置 → 模式选择」出现,可自建仅本人可见的模式 "can_manage_chat_modes": False, diff --git a/src/backend/core/auth/local_users_constants.py b/src/backend/core/auth/local_users_constants.py index 20170f71..dfef22a6 100644 --- a/src/backend/core/auth/local_users_constants.py +++ b/src/backend/core/auth/local_users_constants.py @@ -31,7 +31,6 @@ "can_create_public_kb": False, "can_create_channel_bot": True, "can_switch_model": True, - "can_use_ontology_validation": True, "can_run_autonomous_loop": True, "can_manage_chat_modes": False, "can_system_config": True, diff --git a/src/backend/core/auth/session.py b/src/backend/core/auth/session.py index 2df60e4a..f0cda9ee 100644 --- a/src/backend/core/auth/session.py +++ b/src/backend/core/auth/session.py @@ -3,14 +3,23 @@ Key format: jx:session:{sha256(token)} Value: JSON-encoded user data TTL: SESSION_TTL_HOURS (default 8h) + +``SESSION_STORE=memory`` keeps the table in process instead. With +``SESSION_PERSIST_PATH`` set, that table is mirrored to a file so the desktop +local server, which restarts with the desktop app, keeps its logins. """ from __future__ import annotations +import asyncio import hashlib import json +import os import secrets +import threading +import time from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone +from pathlib import Path from typing import Any, Dict, Optional from core.config.settings import settings @@ -21,6 +30,11 @@ SESSION_KEY_PREFIX = "jx:session:" +# Sliding renewal touches every request; the file only needs to follow it often +# enough that a restart does not cost a session its last few minutes of life. +_PERSIST_RENEWAL_INTERVAL_SECONDS = 300.0 +_PERSIST_WRITE_LOCK = threading.Lock() + def _ttl_seconds() -> int: return int(settings.session.ttl_hours * 3600) @@ -63,15 +77,88 @@ def _use_memory_store() -> bool: return settings.session.store_type == "memory" +def _persist_path() -> Optional[Path]: + raw = settings.session.persist_path + return Path(raw) if raw else None + + +def _read_persisted(path: Path) -> dict[str, dict[str, Any]]: + try: + raw = json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError: + return {} + except (OSError, ValueError) as exc: + logger.warning("session_persist_unreadable", path=str(path), error=str(exc)) + return {} + now = datetime.now(timezone.utc) + sessions: dict[str, dict[str, Any]] = {} + for token_hash, entry in (raw.items() if isinstance(raw, dict) else ()): + try: + payload = entry["payload"] + expires_at = datetime.fromisoformat(entry["expires_at"]) + except (KeyError, TypeError, ValueError): + continue + if isinstance(payload, dict) and expires_at > now: + sessions[token_hash] = {"payload": payload, "expires_at": expires_at} + return sessions + + +def _memory_sessions() -> dict[str, dict[str, Any]]: + """The in-process session table, loaded from the persist file on first use.""" + state = process_state() + path = _persist_path() + if path is not None and not state.persist_loaded: + state.persist_loaded = True + state.MEMORY_SESSIONS.update(_read_persisted(path)) + return state.MEMORY_SESSIONS + + +def _write_persisted(path: Path, snapshot: dict[str, Any], generation: int) -> None: + with _PERSIST_WRITE_LOCK: + state = process_state() + # Snapshots are taken in order on the event loop but written from worker + # threads; never let an older one overwrite a newer one. + if generation <= state.persist_written_generation: + return + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.tmp") + fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as handle: + json.dump(snapshot, handle, ensure_ascii=False) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + state.persist_written_generation = generation + + +async def _persist_memory_sessions() -> None: + path = _persist_path() + if path is None: + return + state = process_state() + state.persist_generation += 1 + state.persisted_at = time.monotonic() + snapshot = { + token_hash: {"payload": entry["payload"], "expires_at": entry["expires_at"].isoformat()} + for token_hash, entry in _memory_sessions().items() + } + try: + await asyncio.to_thread(_write_persisted, path, snapshot, state.persist_generation) + except OSError as exc: + # The session itself is valid in memory; only surviving a restart is lost. + logger.warning("session_persist_failed", path=str(path), error=str(exc)) + + def _prune_expired_memory_sessions() -> None: now = datetime.now(timezone.utc) + sessions = _memory_sessions() expired = [ key - for key, value in process_state().MEMORY_SESSIONS.items() + for key, value in sessions.items() if value.get("expires_at") and value["expires_at"] <= now ] for key in expired: - process_state().MEMORY_SESSIONS.pop(key, None) + sessions.pop(key, None) def generate_token() -> str: @@ -113,10 +200,11 @@ async def create_session( if _use_memory_store(): _prune_expired_memory_sessions() - process_state().MEMORY_SESSIONS[token_hash] = { + _memory_sessions()[token_hash] = { "payload": payload, "expires_at": datetime.now(timezone.utc) + timedelta(seconds=ttl), } + await _persist_memory_sessions() logger.info("session_created", user_id=payload["user_id"], ttl=ttl, store="memory") return token @@ -142,12 +230,15 @@ async def validate_session(token: str) -> Optional[Dict[str, Any]]: if _use_memory_store(): _prune_expired_memory_sessions() - entry = process_state().MEMORY_SESSIONS.get(token_hash) + entry = _memory_sessions().get(token_hash) if entry is None: return None # Sliding renewal uses the session's own TTL (a long "remember me" session is not truncated to the default duration) renew_ttl = _resolve_ttl(entry["payload"].get("ttl_seconds")) entry["expires_at"] = datetime.now(timezone.utc) + timedelta(seconds=renew_ttl) + elapsed = time.monotonic() - process_state().persisted_at + if elapsed >= _PERSIST_RENEWAL_INTERVAL_SECONDS: + await _persist_memory_sessions() return dict(entry["payload"]) r = get_redis() @@ -179,7 +270,7 @@ async def find_session_by_hash(token_hash: str) -> Optional[Dict[str, Any]]: return None if _use_memory_store(): _prune_expired_memory_sessions() - entry = process_state().MEMORY_SESSIONS.get(token_hash) + entry = _memory_sessions().get(token_hash) return dict(entry["payload"]) if entry else None raw = await get_redis().get(f"{SESSION_KEY_PREFIX}{token_hash}") if raw is None: @@ -200,7 +291,9 @@ async def revoke_session(token: str) -> bool: token_hash = _hash_token(token) data = await find_session_by_hash(token_hash) if _use_memory_store(): - deleted = process_state().MEMORY_SESSIONS.pop(token_hash, None) is not None + deleted = _memory_sessions().pop(token_hash, None) is not None + if deleted: + await _persist_memory_sessions() else: deleted = bool(await get_redis().delete(f"{SESSION_KEY_PREFIX}{token_hash}")) if deleted: @@ -247,6 +340,10 @@ def expires_at_iso(ttl_seconds: Optional[int] = None) -> str: @dataclass class SessionState: MEMORY_SESSIONS: dict[str, dict[str, Any]] = field(default_factory=lambda: {}) + persist_loaded: bool = False + persisted_at: float = 0.0 + persist_generation: int = 0 + persist_written_generation: int = 0 _STATE = SessionState() diff --git a/src/backend/core/capabilities/catalog/api.py b/src/backend/core/capabilities/catalog/api.py index e425de07..4c3bd7a0 100644 --- a/src/backend/core/capabilities/catalog/api.py +++ b/src/backend/core/capabilities/catalog/api.py @@ -41,13 +41,14 @@ # ── Public API ───────────────────────────────────────────────────────────── -def get_catalog(*, include_runtime_details: bool = True) -> Dict[str, Any]: +def get_catalog(*, include_runtime_details: bool = True, fresh: bool = False) -> Dict[str, Any]: """Load catalog.json; if missing or invalid, recreate defaults. Results are cached in-memory with a short TTL to avoid repeated - disk I/O and dynamic source loading on every request. + disk I/O and dynamic source loading on every request; ``fresh`` reads + the latest state (and refreshes the cache) instead. """ - return load_catalog(include_runtime_details=include_runtime_details) + return load_catalog(include_runtime_details=include_runtime_details, fresh=fresh) def is_enabled(kind: str, item_id: str) -> bool: diff --git a/src/backend/core/capabilities/catalog/loader.py b/src/backend/core/capabilities/catalog/loader.py index 42a3d628..93752733 100644 --- a/src/backend/core/capabilities/catalog/loader.py +++ b/src/backend/core/capabilities/catalog/loader.py @@ -61,9 +61,13 @@ def invalidate_catalog_cache() -> None: def _write_catalog(data: Dict[str, Any]) -> None: - catalog_common.CATALOG_PATH.write_text( - json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" - ) + from core.capabilities.catalog import persistence + if persistence.enabled(): + persistence.save(data) + else: + catalog_common.CATALOG_PATH.write_text( + json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) invalidate_catalog_cache() @@ -250,6 +254,9 @@ def _filter_to_static_defaults(data: Dict[str, Any], defaults: Dict[str, Any]) - def ensure_default_catalog() -> Dict[str, Any]: """Create catalog.json with repository defaults if missing; return loaded catalog.""" + from core.capabilities.catalog import persistence + if persistence.enabled(): + return load_catalog() if not catalog_common.CATALOG_PATH.exists(): cat = _default_catalog() catalog_common.CATALOG_PATH.write_text( @@ -261,32 +268,29 @@ def ensure_default_catalog() -> Dict[str, Any]: return load_catalog() -def load_catalog(*, include_runtime_details: bool = True) -> Dict[str, Any]: +def load_catalog(*, include_runtime_details: bool = True, fresh: bool = False) -> Dict[str, Any]: """Load catalog.json; if missing or invalid, recreate defaults. Results are cached in-memory for up to ``_CATALOG_CACHE_TTL`` seconds to avoid repeated disk I/O and dynamic source loading on every request. + ``fresh`` skips the cached copy (and refreshes it) for callers that must + observe the latest administrator choices. """ from core.capabilities.catalog.migration import _migrate_legacy_shape now = monotonic() cached_time = process_state().CATALOG_CACHE_TIME.get(include_runtime_details, 0.0) - if include_runtime_details in process_state().CATALOG_CACHE and (now - cached_time) < _CATALOG_CACHE_TTL: + if (not fresh and include_runtime_details in process_state().CATALOG_CACHE + and (now - cached_time) < _CATALOG_CACHE_TTL): return copy.deepcopy(process_state().CATALOG_CACHE[include_runtime_details]) - if not catalog_common.CATALOG_PATH.exists(): - result = _default_catalog() - process_state().CATALOG_CACHE[include_runtime_details] = copy.deepcopy(result) - process_state().CATALOG_CACHE_TIME[include_runtime_details] = monotonic() - return result - - try: - raw = _read_raw_catalog() - data = _migrate_legacy_shape(raw) - except Exception: - # Return defaults on any parse/shape error without mutating repo files. + if catalog_common.CATALOG_PATH.exists(): + try: + data = _migrate_legacy_shape(_read_raw_catalog()) + except (OSError, ValueError, TypeError): + data = _default_catalog() + else: data = _default_catalog() - return data # Ensure required top-level keys exist and keep arrays. defaults = _default_catalog() @@ -300,6 +304,9 @@ def load_catalog(*, include_runtime_details: bool = True) -> Dict[str, Any]: # Do not persist static detail fields for skills/mcp. This cleanup is kept # in-memory here; explicit admin operations remain the only catalog writers. _strip_static_detail_fields(data) + from core.capabilities.catalog import persistence + if persistence.enabled(): + persistence.apply(data) if include_runtime_details: from core.capabilities.catalog.details import _attach_runtime_details diff --git a/src/backend/core/capabilities/catalog/persistence.py b/src/backend/core/capabilities/catalog/persistence.py new file mode 100644 index 00000000..14dcca3a --- /dev/null +++ b/src/backend/core/capabilities/catalog/persistence.py @@ -0,0 +1,54 @@ +"""Durable administrator choices for catalogs using object storage.""" +from __future__ import annotations + +import os +from copy import deepcopy + +BLOCK_ID = "catalog_admin_choices" +KINDS = ("skills", "agents", "mcp", "kb") + + +def enabled() -> bool: + return os.getenv("STORAGE_TYPE", "local").lower() in ("s3", "oss", "obs") + + +def save(data): + from core.db.engine import SessionLocal + from core.db.models import ContentBlock + from core.infra.time import utc_now + + # Descriptions and executable configuration remain owned by their sources. + choices = { + kind: [{"id": str(item["id"]), "enabled": bool(item.get("enabled"))} + for item in data.get(kind, []) if isinstance(item, dict) and item.get("id")] + for kind in KINDS + } + with SessionLocal() as db: + row = db.get(ContentBlock, BLOCK_ID) + if row is None: + db.add(ContentBlock(id=BLOCK_ID, payload=choices, updated_at=utc_now())) + else: + row.payload = choices + row.updated_at = utc_now() + db.commit() + + +def apply(data): + from core.db.engine import SessionLocal + from core.db.models import ContentBlock + + with SessionLocal() as db: + row = db.get(ContentBlock, BLOCK_ID) + choices = deepcopy(row.payload) if row else {} + if not isinstance(choices, dict): + raise ValueError("Invalid persisted catalog choices") + for kind in KINDS: + by_id = {str(item["id"]): item for item in data.get(kind, [])} + ordered = [] + for choice in choices.get(kind, []): + item = by_id.pop(str(choice["id"]), None) + if item is not None: + item["enabled"] = bool(choice["enabled"]) + ordered.append(item) + data[kind] = ordered + list(by_id.values()) + return data diff --git a/src/backend/core/capabilities/mcp/management.py b/src/backend/core/capabilities/mcp/management.py index dfb0aaa8..a19a31d2 100644 --- a/src/backend/core/capabilities/mcp/management.py +++ b/src/backend/core/capabilities/mcp/management.py @@ -410,7 +410,7 @@ async def _do_probe() -> None: ) if not report.valid: messages = "; ".join(item.message for item in report.errors) - raise RuntimeError(f"Domain Pack 构建校验失败:{messages}") + raise RuntimeError(f"构建校验失败:{messages}") finally: await resource.close() diff --git a/src/backend/core/chat/message_writes.py b/src/backend/core/chat/message_writes.py index c2e69cd8..f522e28d 100644 --- a/src/backend/core/chat/message_writes.py +++ b/src/backend/core/chat/message_writes.py @@ -6,7 +6,6 @@ from core.db.models import ChatCompactionState, ChatMessage, ChatRun, ChatSession from core.db.repository.chat import _strip_nul from sqlalchemy import update -from core.ontology.revision import is_substantive_revision, normalize_revision_candidate from core.chat import limits def add_message( @@ -260,40 +259,3 @@ def update_message_extra_data( ) -> bool: """Merge *patch* into a message's extra_data. Returns True on success.""" return self.message_repo.update_extra_data(message_id, patch) is not None - - -def accept_ontology_revision(self, message_id: str) -> Optional[ChatMessage]: - """Replace an assistant message with its persisted ontology revision candidate.""" - message = self.message_repo.get_by_id(message_id) - if not message or message.role != "assistant": - return None - extra_data = dict(message.extra_data or {}) - governance = extra_data.get("ontology_governance") - if not isinstance(governance, dict): - return None - review = governance.get("review") - if not isinstance(review, dict): - return None - candidate = normalize_revision_candidate(review.get("candidate_answer")) - if not is_substantive_revision(candidate): - return None - updated_review = {**review, "candidate_answer": candidate, "accepted": True} - extra_data["ontology_governance"] = {**governance, "review": updated_review} - # Some pure unit adapters exercise this method without a database. - if not hasattr(self, "db"): - return self.message_repo.update( - message_id, - {"content": candidate, "extra_data": extra_data}, - ) - - updated = self.message_repo.update( - message_id, - {"content": candidate, "extra_data": extra_data}, - commit=False, - ) - self._invalidate_compaction_lineage_for_rewrite( - message.chat_id, - rewritten_seq=int(message.chat_seq), - ) - self.db.commit() - return updated diff --git a/src/backend/core/chat/service.py b/src/backend/core/chat/service.py index 0d9349ea..972e0825 100644 --- a/src/backend/core/chat/service.py +++ b/src/backend/core/chat/service.py @@ -242,8 +242,6 @@ def update_message_extra_data( ) -> bool: return message_writes.update_message_extra_data(self, message_id, patch) - def accept_ontology_revision(self, message_id: str) -> Optional[ChatMessage]: - return message_writes.accept_ontology_revision(self, message_id) def search_sessions( self, diff --git a/src/backend/core/config/environment.py b/src/backend/core/config/environment.py index 5f981998..3526831f 100644 --- a/src/backend/core/config/environment.py +++ b/src/backend/core/config/environment.py @@ -23,6 +23,8 @@ def _load_env_files() -> None: 2. Env-specific file such as .env.dev 3. Base .env """ + if os.getenv("PYTHON_DOTENV_DISABLED", "").strip().lower() in {"1", "true", "t", "yes", "y"}: + return base_env_path = _REPO_ROOT / ".env" base_values = dotenv_values(base_env_path) if base_env_path.exists() else {} diff --git a/src/backend/core/config/identity_settings.py b/src/backend/core/config/identity_settings.py index 50e9e938..17f5eca0 100644 --- a/src/backend/core/config/identity_settings.py +++ b/src/backend/core/config/identity_settings.py @@ -83,6 +83,10 @@ class SessionSettings: ) ttl_hours: float = field(default_factory=lambda: float(_env("SESSION_TTL_HOURS", "8"))) store_type: str = field(default_factory=lambda: _env("SESSION_STORE", "memory").lower().strip()) + # Memory store only: mirror the in-process session table to this file so a + # single-process install (the desktop local server) keeps its logins across + # restarts. Empty keeps sessions purely in memory. + persist_path: str = field(default_factory=lambda: _env("SESSION_PERSIST_PATH", "").strip()) @dataclass(frozen=True) class OASsoSettings: diff --git a/src/backend/core/config/runtime_settings.py b/src/backend/core/config/runtime_settings.py index 117efe56..bf0f0d04 100644 --- a/src/backend/core/config/runtime_settings.py +++ b/src/backend/core/config/runtime_settings.py @@ -30,6 +30,7 @@ class MemorySettings: api_key: str = field( default_factory=lambda: _env("MEMORY_API_KEY", _env("API_KEY", "sk-placeholder")) ) + vector_provider: str = field(default_factory=lambda: _env("MEM0_VECTOR_PROVIDER", "milvus").strip().lower()) milvus_url: str = field(default_factory=lambda: _env("MILVUS_URL", "http://milvus:19530")) milvus_token: str = field(default_factory=lambda: _env("MILVUS_TOKEN", "")) neo4j_url: str = field(default_factory=lambda: _env("NEO4J_URL", "bolt://neo4j:7687")) diff --git a/src/backend/core/config/sandbox_settings.py b/src/backend/core/config/sandbox_settings.py index b140eadf..d2ed43f7 100644 --- a/src/backend/core/config/sandbox_settings.py +++ b/src/backend/core/config/sandbox_settings.py @@ -34,6 +34,9 @@ class SandboxSettings: idle_ttl_s: int = field( default_factory=lambda: max(60, _int(_env("SANDBOX_IDLE_TTL_S", "3600"), 3600)) ) + # Per-sandbox resource limits for Kubernetes workers (e.g. "1", "500m" / "1Gi"). + cpu: str = field(default_factory=lambda: _env("SANDBOX_CPU", "").strip()) + memory: str = field(default_factory=lambda: _env("SANDBOX_MEMORY", "").strip()) artifact_max_bytes: int = field( default_factory=lambda: max( 1, diff --git a/src/backend/core/config/store/system_config_constants.py b/src/backend/core/config/store/system_config_constants.py index 1588799d..d45956e6 100644 --- a/src/backend/core/config/store/system_config_constants.py +++ b/src/backend/core/config/store/system_config_constants.py @@ -112,17 +112,6 @@ "sandbox", False, ), - # ontology —— independent enterprise governance gate for plugin imports. - # Unlike the user's ontology preference, this switch cannot be bypassed by - # turning off ontology validation in personal settings. - ( - "ontology.force_plugin_import_build_validation", - "false", - "强制构建时本体校验", - "插件导入或安装时必须通过激活 Domain Pack 的构建校验;个人本体开关不能关闭此门禁。", - "ontology", - False, - ), # context —— the single compaction trigger ratio. Read per turn by # compaction_service.resolve_trigger_ratio and applied by every phase (mid-turn at the ReAct step # boundary, pre-turn before a turn samples, post-turn background pre-warm) — it is no longer an diff --git a/src/backend/core/db/engine.py b/src/backend/core/db/engine.py index 57153719..d5df7034 100644 --- a/src/backend/core/db/engine.py +++ b/src/backend/core/db/engine.py @@ -201,6 +201,12 @@ def init_db(): retire_site_storage(engine, directory) initialize_local_schema() if settings.edition.edition == "ce": + from core.db.ontology_retirement import has_ontology_data, retire_ontology + + # Local SQLite has no Alembic: retire ontology data here (ce_0035 elsewhere). + with engine.begin() as connection: + if has_ontology_data(connection): + logger.info("Ontology data retired: %s", retire_ontology(connection)) if settings.deploy.is_local: from core.db.local_schema_upgrade import reconcile_local_chat_sequences diff --git a/src/backend/core/db/models/__init__.py b/src/backend/core/db/models/__init__.py index dc5d9ca0..beb8116f 100644 --- a/src/backend/core/db/models/__init__.py +++ b/src/backend/core/db/models/__init__.py @@ -55,13 +55,6 @@ MemorySanitizerRule, ProfileMemory, ) -from core.db.models.ontology import ( - OntologyDraft, - OntologyEnforcementEvent, - OntologyPack, - OntologyPackVersion, - OntologyReviewRun, -) from core.db.models.command import CommandTask from core.db.models.job import JOB_LIVE_STATUSES, JOB_TERMINAL_STATUSES, Job, JobCall, JobItem from core.db.edition import EDITION_MODEL_NAMES, edition_model_exports @@ -147,11 +140,6 @@ def __getattr__(name): 'EvolutionEvidencePack', 'EvolutionCreditDecision', 'EvolutionPromotionLink', - 'OntologyPack', - 'OntologyPackVersion', - 'OntologyEnforcementEvent', - 'OntologyReviewRun', - 'OntologyDraft', *EDITION_MODEL_NAMES, ] diff --git a/src/backend/core/db/models/ontology.py b/src/backend/core/db/models/ontology.py deleted file mode 100644 index 8f85169b..00000000 --- a/src/backend/core/db/models/ontology.py +++ /dev/null @@ -1,239 +0,0 @@ -"""SQLAlchemy ORM models for ontology-governed agent execution.""" -from core.db.utc_datetime import UTCDateTime -from core.infra.time import utc_now - - -from core.db.engine import Base -from sqlalchemy import ( - Boolean, - CheckConstraint, - Column, - ForeignKey, - Index, - Integer, - String, - Text, - UniqueConstraint, - text, -) -from core.db.models.column_types import JSONType - - -class OntologyPack(Base): - """A versioned Domain Pack and its currently active version pointer.""" - - __tablename__ = "ontology_packs" - - pack_id = Column(String(64), primary_key=True) - name = Column(String(255), nullable=False) - domain = Column(String(128), nullable=False, default="general") - description = Column(Text, nullable=False, default="") - is_enabled = Column(Boolean, nullable=False, default=True) - is_default = Column(Boolean, nullable=False, default=False) - active_version_id = Column(String(64), nullable=True) - created_by = Column(String(64), nullable=True) - created_at = Column(UTCDateTime(timezone=True), nullable=False, default=utc_now) - updated_at = Column( - UTCDateTime(timezone=True), - nullable=False, - default=utc_now, - onupdate=utc_now, - ) - deleted_at = Column(UTCDateTime(timezone=True), nullable=True) - - __table_args__ = ( - Index("idx_ontology_packs_enabled", "is_enabled", "is_default"), - Index("idx_ontology_packs_domain", "domain"), - Index("idx_ontology_packs_deleted", "deleted_at"), - ) - - -class OntologyPackVersion(Base): - """Domain Pack version; drafts are mutable, while published versions are immutable.""" - - __tablename__ = "ontology_pack_versions" - - version_id = Column(String(64), primary_key=True) - pack_id = Column( - String(64), - ForeignKey("ontology_packs.pack_id", ondelete="CASCADE"), - nullable=False, - ) - version = Column(String(32), nullable=False) - content = Column(JSONType, nullable=False, default=dict) - checksum = Column(String(64), nullable=False) - status = Column(String(16), nullable=False, default="draft") - validation_report = Column(JSONType, nullable=False, default=dict) - created_by = Column(String(64), nullable=True) - created_at = Column(UTCDateTime(timezone=True), nullable=False, default=utc_now) - updated_at = Column( - UTCDateTime(timezone=True), - nullable=False, - default=utc_now, - onupdate=utc_now, - ) - activated_at = Column(UTCDateTime(timezone=True), nullable=True) - - __table_args__ = ( - CheckConstraint( - "status IN ('draft', 'active', 'retired')", - name="ontology_pack_versions_status_check", - ), - UniqueConstraint("pack_id", "version", name="uq_ontology_pack_version"), - Index("idx_ontology_pack_versions_pack", "pack_id", "created_at"), - Index("idx_ontology_pack_versions_status", "status"), - Index( - "uq_ontology_pack_versions_working_draft", - "pack_id", - unique=True, - postgresql_where=text("status = 'draft'"), - sqlite_where=text("status = 'draft'"), - ), - ) - - -class OntologyEnforcementEvent(Base): - """Append-only evidence emitted by the deterministic gate and reviewers.""" - - __tablename__ = "ontology_enforcement_events" - - event_id = Column(String(64), primary_key=True) - user_id = Column( - String(64), - ForeignKey("users_shadow.user_id", ondelete="SET NULL"), - nullable=True, - ) - chat_id = Column( - String(64), - ForeignKey("chat_sessions.chat_id", ondelete="SET NULL"), - nullable=True, - ) - pack_id = Column( - String(64), - ForeignKey("ontology_packs.pack_id", ondelete="SET NULL"), - nullable=True, - ) - version_id = Column( - String(64), - ForeignKey("ontology_pack_versions.version_id", ondelete="SET NULL"), - nullable=True, - ) - rule_id = Column(String(128), nullable=True) - stage = Column(String(24), nullable=False) - event_type = Column(String(32), nullable=False) - decision = Column(String(24), nullable=False) - mode = Column(String(16), nullable=False, default="enforce") - target = Column(String(255), nullable=True) - latency_ms = Column(Integer, nullable=True) - details = Column(JSONType, nullable=False, default=dict) - created_at = Column(UTCDateTime(timezone=True), nullable=False, default=utc_now) - - __table_args__ = ( - CheckConstraint( - "stage IN ('tool', 'checkpoint', 'output', 'evolution', 'build')", - name="ontology_enforcement_events_stage_check", - ), - CheckConstraint( - "decision IN ('pass', 'log', 'deny', 'revise', 'escalate', 'error')", - name="ontology_enforcement_events_decision_check", - ), - CheckConstraint( - "mode IN ('log', 'enforce')", - name="ontology_enforcement_events_mode_check", - ), - Index("idx_ontology_events_chat", "chat_id", "created_at"), - Index("idx_ontology_events_rule", "rule_id", "created_at"), - Index("idx_ontology_events_pack", "pack_id", "created_at"), - Index("idx_ontology_events_decision", "decision", "created_at"), - ) - - -class OntologyReviewRun(Base): - """Auditable L-b/L-c reviewer verdict with evidence and latency.""" - - __tablename__ = "ontology_review_runs" - - review_id = Column(String(64), primary_key=True) - user_id = Column( - String(64), - ForeignKey("users_shadow.user_id", ondelete="SET NULL"), - nullable=True, - ) - chat_id = Column( - String(64), - ForeignKey("chat_sessions.chat_id", ondelete="SET NULL"), - nullable=True, - ) - pack_version_ids = Column(JSONType, nullable=False, default=list) - level = Column(String(16), nullable=False) - subject_type = Column(String(32), nullable=False, default="final_answer") - verdict = Column(String(16), nullable=False) - evidence = Column(JSONType, nullable=False, default=list) - feedback = Column(Text, nullable=False, default="") - reviewers = Column(JSONType, nullable=False, default=list) - latency_ms = Column(Integer, nullable=True) - created_at = Column(UTCDateTime(timezone=True), nullable=False, default=utc_now) - - __table_args__ = ( - CheckConstraint( - "level IN ('checkpoint', 'committee')", - name="ontology_review_runs_level_check", - ), - CheckConstraint( - "verdict IN ('pass', 'revise', 'escalate', 'error')", - name="ontology_review_runs_verdict_check", - ), - Index("idx_ontology_reviews_chat", "chat_id", "created_at"), - Index("idx_ontology_reviews_verdict", "verdict", "created_at"), - ) - - -class OntologyDraft(Base): - """Human-in-the-loop ontology evolution candidate; never auto-activates.""" - - __tablename__ = "ontology_drafts" - - draft_id = Column(String(64), primary_key=True) - pack_id = Column( - String(64), - ForeignKey("ontology_packs.pack_id", ondelete="CASCADE"), - nullable=False, - ) - source_type = Column(String(24), nullable=False) - candidate_type = Column(String(24), nullable=False) - proposal = Column(JSONType, nullable=False, default=dict) - evidence = Column(JSONType, nullable=False, default=list) - source_event_ids = Column(JSONType, nullable=False, default=list) - value_score = Column(Integer, nullable=False, default=0) - review_status = Column(String(16), nullable=False, default="pending") - reviewer_id = Column(String(64), nullable=True) - reviewer_comment = Column(Text, nullable=True) - reviewed_at = Column(UTCDateTime(timezone=True), nullable=True) - created_at = Column(UTCDateTime(timezone=True), nullable=False, default=utc_now) - updated_at = Column( - UTCDateTime(timezone=True), - nullable=False, - default=utc_now, - onupdate=utc_now, - ) - - __table_args__ = ( - CheckConstraint( - "source_type IN ('enforcement', 'review', 'user_correction', 'document')", - name="ontology_drafts_source_type_check", - ), - CheckConstraint( - "candidate_type IN ('term', 'relation', 'constraint', 'false_positive')", - name="ontology_drafts_candidate_type_check", - ), - CheckConstraint( - "review_status IN ('pending', 'approved', 'rejected')", - name="ontology_drafts_status_check", - ), - CheckConstraint( - "value_score >= 0 AND value_score <= 100", - name="ontology_drafts_value_score_check", - ), - Index("idx_ontology_drafts_status", "review_status", "value_score", "created_at"), - Index("idx_ontology_drafts_pack", "pack_id", "created_at"), - ) diff --git a/src/backend/core/db/ontology_retirement.py b/src/backend/core/db/ontology_retirement.py new file mode 100644 index 00000000..963b34c6 --- /dev/null +++ b/src/backend/core/db/ontology_retirement.py @@ -0,0 +1,77 @@ +"""Remove domain ontology governance data from a community database. + +Used by the CE Alembic chain and by local SQLite startup, which has no Alembic. +Conversations keep their text; only ontology packs, reviews, settings and +bindings are deleted. Restoring them needs a backup taken before upgrading. +""" +from __future__ import annotations + +import json +from typing import Any + +import sqlalchemy as sa +from sqlalchemy.engine import Connection + +# Children before parents so the drops work without relying on FK cascades. +ONTOLOGY_TABLES = ( + "ontology_enforcement_events", + "ontology_review_runs", + "ontology_drafts", + "ontology_pack_versions", + "ontology_packs", +) +_USER_KEYS = ("ontology_enabled", "ontology_pack_ids", "can_use_ontology_validation") + + +def _json(value: Any) -> Any: + if isinstance(value, str): + try: + return json.loads(value) + except ValueError: + return value + return value + + +def _strip_keys(bind: Connection, table: sa.Table, key: str, column: str, names: tuple[str, ...]): + changed = 0 + for row in bind.execute(sa.select(table.c[key], table.c[column])): + value = _json(row[1]) + if not isinstance(value, dict) or not any(name in value for name in names): + continue + cleaned = {k: v for k, v in value.items() if k not in names} + # Reflected SQLite JSON columns are plain text; write back the stored form. + stored = json.dumps(cleaned, ensure_ascii=False) if isinstance(row[1], str) else cleaned + bind.execute(table.update().where(table.c[key] == row[0]).values({column: stored})) + changed += 1 + return changed + + +def has_ontology_data(bind: Connection) -> bool: + return bool(set(ONTOLOGY_TABLES) & set(sa.inspect(bind).get_table_names())) + + +def retire_ontology(bind: Connection) -> dict[str, int]: + """Delete ontology-owned rows and settings in the caller's transaction.""" + names = set(sa.inspect(bind).get_table_names()) + metadata = sa.MetaData() + report: dict[str, int] = {} + + def table(name: str): + return sa.Table(name, metadata, autoload_with=bind) if name in names else None + + configs = table("system_configs") + if configs is not None: + report["system_configs"] = bind.execute( + configs.delete().where(configs.c.config_key.like("ontology.%")) + ).rowcount + users = table("users_shadow") + if users is not None and "metadata" in users.c: + report["users_shadow"] = _strip_keys(bind, users, "user_id", "metadata", _USER_KEYS) + agents = table("user_agents") + if agents is not None and "extra_config" in agents.c: + report["user_agents"] = _strip_keys(bind, agents, "agent_id", "extra_config", ("ontology_tags",)) + for name in ONTOLOGY_TABLES: + if name in names: + bind.execute(sa.text(f'DROP TABLE "{name}"')) + report[name] = 1 + return report diff --git a/src/backend/core/db/repository/__init__.py b/src/backend/core/db/repository/__init__.py index 4cda10c1..1c8b61ee 100644 --- a/src/backend/core/db/repository/__init__.py +++ b/src/backend/core/db/repository/__init__.py @@ -12,7 +12,6 @@ from core.db.repository.chat import ChatMessageRepository, ChatSessionRepository -from core.db.repository.ontology import OntologyRepository from core.db.repository.user import LocalUserRepository, UserRepository @@ -29,5 +28,4 @@ "ROOT_FOLDER_SENTINEL", "AuditLogRepository", "UserAgentRepository", - "OntologyRepository", ] diff --git a/src/backend/core/db/repository/ontology.py b/src/backend/core/db/repository/ontology.py deleted file mode 100644 index f701f6a8..00000000 --- a/src/backend/core/db/repository/ontology.py +++ /dev/null @@ -1,193 +0,0 @@ -"""Repository for versioned ontology assets and runtime audit evidence.""" - -from __future__ import annotations -from core.infra.time import utc_now - -from datetime import datetime -from typing import Any - -from core.db.models import ( - OntologyDraft, - OntologyEnforcementEvent, - OntologyPack, - OntologyPackVersion, - OntologyReviewRun, -) -from sqlalchemy import desc -from sqlalchemy.orm import Session - - -class OntologyRepository: - def __init__(self, db: Session): - self.db = db - - def list_packs(self, *, enabled_only: bool = False) -> list[OntologyPack]: - query = self.db.query(OntologyPack).filter(OntologyPack.deleted_at.is_(None)) - if enabled_only: - query = query.filter(OntologyPack.is_enabled.is_(True)) - return query.order_by(desc(OntologyPack.is_default), OntologyPack.name).all() - - def get_pack(self, pack_id: str) -> OntologyPack | None: - return ( - self.db.query(OntologyPack) - .filter(OntologyPack.pack_id == pack_id, OntologyPack.deleted_at.is_(None)) - .first() - ) - - def create_pack(self, data: dict[str, Any]) -> OntologyPack: - row = OntologyPack(**data) - self.db.add(row) - self.db.flush() - return row - - def update_pack(self, row: OntologyPack, data: dict[str, Any]) -> OntologyPack: - for key, value in data.items(): - setattr(row, key, value) - row.updated_at = utc_now() - self.db.flush() - return row - - def list_versions(self, pack_id: str) -> list[OntologyPackVersion]: - return ( - self.db.query(OntologyPackVersion) - .filter(OntologyPackVersion.pack_id == pack_id) - .order_by(desc(OntologyPackVersion.created_at)) - .all() - ) - - def get_version(self, version_id: str) -> OntologyPackVersion | None: - return ( - self.db.query(OntologyPackVersion) - .filter(OntologyPackVersion.version_id == version_id) - .first() - ) - - def get_pack_version(self, pack_id: str, version: str) -> OntologyPackVersion | None: - return ( - self.db.query(OntologyPackVersion) - .filter( - OntologyPackVersion.pack_id == pack_id, - OntologyPackVersion.version == version, - ) - .first() - ) - - def get_working_draft(self, pack_id: str) -> OntologyPackVersion | None: - return ( - self.db.query(OntologyPackVersion) - .filter( - OntologyPackVersion.pack_id == pack_id, - OntologyPackVersion.status == "draft", - ) - .order_by( - desc(OntologyPackVersion.updated_at), - desc(OntologyPackVersion.created_at), - ) - .first() - ) - - def get_active_versions( - self, - pack_ids: list[str] | None = None, - ) -> list[OntologyPackVersion]: - query = ( - self.db.query(OntologyPackVersion) - .join(OntologyPack, OntologyPack.active_version_id == OntologyPackVersion.version_id) - .filter( - OntologyPack.deleted_at.is_(None), - OntologyPack.is_enabled.is_(True), - OntologyPackVersion.status == "active", - ) - ) - if pack_ids: - query = query.filter(OntologyPack.pack_id.in_(pack_ids)) - else: - query = query.filter(OntologyPack.is_default.is_(True)) - return query.order_by(OntologyPack.name).all() - - def create_version(self, data: dict[str, Any]) -> OntologyPackVersion: - row = OntologyPackVersion(**data) - self.db.add(row) - self.db.flush() - return row - - def update_working_draft( - self, - row: OntologyPackVersion, - data: dict[str, Any], - ) -> OntologyPackVersion: - for key, value in data.items(): - setattr(row, key, value) - row.updated_at = utc_now() - self.db.flush() - return row - - def delete_working_draft(self, row: OntologyPackVersion) -> None: - self.db.delete(row) - self.db.flush() - - def activate(self, pack: OntologyPack, version: OntologyPackVersion) -> None: - self.db.query(OntologyPackVersion).filter( - OntologyPackVersion.pack_id == pack.pack_id, - OntologyPackVersion.status == "active", - ).update({OntologyPackVersion.status: "retired"}, synchronize_session=False) - version.status = "active" - version.activated_at = utc_now() - version.updated_at = utc_now() - pack.active_version_id = version.version_id - pack.is_enabled = True - pack.updated_at = utc_now() - self.db.flush() - - def create_event(self, data: dict[str, Any]) -> OntologyEnforcementEvent: - row = OntologyEnforcementEvent(**data) - self.db.add(row) - self.db.commit() - return row - - def list_events( - self, - *, - limit: int = 100, - chat_id: str | None = None, - decision: str | None = None, - ) -> list[OntologyEnforcementEvent]: - query = self.db.query(OntologyEnforcementEvent) - if chat_id: - query = query.filter(OntologyEnforcementEvent.chat_id == chat_id) - if decision: - query = query.filter(OntologyEnforcementEvent.decision == decision) - return query.order_by(desc(OntologyEnforcementEvent.created_at)).limit(limit).all() - - def create_review(self, data: dict[str, Any]) -> OntologyReviewRun: - row = OntologyReviewRun(**data) - self.db.add(row) - self.db.commit() - return row - - def list_reviews(self, *, limit: int = 100) -> list[OntologyReviewRun]: - return ( - self.db.query(OntologyReviewRun) - .order_by(desc(OntologyReviewRun.created_at)) - .limit(limit) - .all() - ) - - def list_drafts(self, *, status: str | None = None, limit: int = 100) -> list[OntologyDraft]: - query = self.db.query(OntologyDraft) - if status: - query = query.filter(OntologyDraft.review_status == status) - return ( - query.order_by(desc(OntologyDraft.value_score), desc(OntologyDraft.created_at)) - .limit(limit) - .all() - ) - - def get_draft(self, draft_id: str) -> OntologyDraft | None: - return self.db.query(OntologyDraft).filter(OntologyDraft.draft_id == draft_id).first() - - def create_draft(self, data: dict[str, Any]) -> OntologyDraft: - row = OntologyDraft(**data) - self.db.add(row) - self.db.commit() - return row diff --git a/src/backend/core/evolution/cycle/runtime_binding.py b/src/backend/core/evolution/cycle/runtime_binding.py index dcc87496..4e66cc0a 100644 --- a/src/backend/core/evolution/cycle/runtime_binding.py +++ b/src/backend/core/evolution/cycle/runtime_binding.py @@ -341,7 +341,6 @@ def bind_runtime_assets( refs += _safe( lambda: _skill_refs(skill_ids, run_id, capability_scope=capability_scope), "skill", degraded ) - refs += _safe(lambda: _ontology_refs(), "ontology", degraded) refs += _safe(lambda: _memory_refs(memory_enabled, workspace_id), "memory", degraded) refs += _safe(lambda: _model_refs(model_name, model_provider_id), "model", degraded) refs += _safe(lambda: _kb_refs(kb_ids), "kb", degraded) diff --git a/src/backend/core/evolution/generators/ontology_bridge.py b/src/backend/core/evolution/generators/ontology_bridge.py deleted file mode 100644 index 8dae5358..00000000 --- a/src/backend/core/evolution/generators/ontology_bridge.py +++ /dev/null @@ -1,186 +0,0 @@ -"""Bridge ontology drafts into the shared candidate ledger (GCE ticket 22). - -Ontology governance is already the most mature safe-evolution path in this -codebase: runtime evidence can only create a draft, an approved draft only -materialises as an *inactive* version, and an administrator must activate it -explicitly. - -So this bridge deliberately flows in one direction. Ontology adopts the shared -ledger — the same candidate list, evidence view and version lineage as every -other asset — and gives up **nothing**. The unified framework inherits -ontology's discipline; ontology does not inherit the framework's faster paths. - -Concretely, and non-negotiably: - -* an ontology candidate never activates automatically, whatever the evidence; -* a new constraint runs in log-only mode first, so a false positive shows up as - an observation rather than as blocked business; -* a hard constraint cannot be weakened by a low-risk candidate. -""" - -from __future__ import annotations - -import logging -from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional, Sequence, Tuple - -logger = logging.getLogger(__name__) - -MODE_LOG_ONLY = "log_only" -MODE_ENFORCE = "enforce" - -# How long a new constraint must be observed before it may enforce, and the -# false-positive rate above which it is recommended for withdrawal instead. -MIN_LOG_ONLY_OBSERVATIONS = 20 -MAX_FALSE_POSITIVE_RATE = 0.10 - - -@dataclass -class OntologyCandidate: - """An ontology draft expressed in the shared envelope.""" - - draft_id: str - pack_id: str - operation: str - evidence_event_ids: List[str] = field(default_factory=list) - user_corrections: int = 0 - denial_count: int = 0 - proposal: Dict[str, Any] = field(default_factory=dict) - mode: str = MODE_LOG_ONLY - - @property - def auto_activatable(self) -> bool: - """Always false. Present as executable documentation of the invariant.""" - return False - - def to_ir_payload(self) -> Dict[str, Any]: - return { - "target_kind": "ontology", - "target_asset_id": self.pack_id, - "operation": self.operation, - "changes": [self.proposal], - "evidence_refs": list(self.evidence_event_ids), - # Highest tier by construction: ontology binds every other asset, so - # a mistake here is not contained to one skill. - "risk_tier": "high", - "mode": self.mode, - } - - -@dataclass -class ConsistencyProblem: - code: str - detail: str - - def to_dict(self) -> Dict[str, Any]: - return {"code": self.code, "detail": self.detail} - - -def check_consistency( - candidate: OntologyCandidate, - *, - known_concepts: Sequence[str], - referenced_tools: Sequence[str], - existing_workflows: Sequence[str], - hard_constraints: Sequence[str] = (), -) -> Tuple[bool, List[ConsistencyProblem]]: - """Structural checks before a draft may enter the ledger. - - A broken pack degrades every asset that depends on it, so these run before - a human is asked to look at anything. - """ - problems: List[ConsistencyProblem] = [] - proposal = candidate.proposal or {} - - concept = str(proposal.get("concept") or "") - if concept and concept not in known_concepts: - problems.append( - ConsistencyProblem("dangling_concept", f"引用了不存在的概念 {concept}") - ) - - for tool in proposal.get("tools") or []: - if tool not in referenced_tools: - problems.append( - ConsistencyProblem("unknown_tool", f"引用了不存在的工具 {tool}") - ) - - for workflow in proposal.get("breaks_workflows") or []: - if workflow in existing_workflows: - problems.append( - ConsistencyProblem("breaks_workflow", f"会破坏既有工作流 {workflow}") - ) - - weakened = proposal.get("weakens_constraint") - if weakened and weakened in hard_constraints: - problems.append( - ConsistencyProblem( - "hard_constraint_weakened", - f"试图弱化硬约束 {weakened}——任何候选都不允许", - ) - ) - - return (not problems), problems - - -def aggregate_false_positive_evidence( - *, - denials: Sequence[Dict[str, Any]], - corrections: Sequence[Dict[str, Any]], - min_denials: int = 2, -) -> Optional[Dict[str, Any]]: - """Bundle repeated denials plus user corrections into one evidence packet. - - Requires *both* signals. Repeated denials alone may simply mean the rule is - working; it is the human corrections that turn "the gate fired a lot" into - "the gate is wrong". - """ - by_rule: Dict[str, List[Dict[str, Any]]] = {} - for denial in denials: - rule_id = str(denial.get("rule_id") or "") - if rule_id: - by_rule.setdefault(rule_id, []).append(denial) - - for rule_id, items in by_rule.items(): - if len(items) < min_denials: - continue - related = [ - c for c in corrections if str(c.get("rule_id") or "") == rule_id - ] - if not related: - continue - return { - "rule_id": rule_id, - "denial_count": len(items), - "correction_count": len(related), - "event_ids": [str(i.get("event_id") or "") for i in items], - } - return None - - -def promote_to_enforce( - *, observations: int, false_positives: int -) -> Tuple[bool, str]: - """Whether a log-only constraint has earned the right to enforce. - - Below the observation floor there is no basis for the decision; above the - false-positive ceiling the recommendation is withdrawal, not enforcement. - """ - if observations < MIN_LOG_ONLY_OBSERVATIONS: - return False, f"observations {observations} < {MIN_LOG_ONLY_OBSERVATIONS}" - rate = false_positives / observations if observations else 1.0 - if rate > MAX_FALSE_POSITIVE_RATE: - return False, f"false_positive_rate {rate:.2%} > {MAX_FALSE_POSITIVE_RATE:.0%} — 建议撤回" - return True, "" - - -def activation_is_permitted(candidate: OntologyCandidate, *, approver: Optional[str]) -> Tuple[bool, str]: - """Ontology activation always needs an explicit human act. - - There is no evidence threshold that unlocks automation here. That is the - invariant the rest of the framework inherits, not an exception to it. - """ - if candidate.auto_activatable: # pragma: no cover - defensive, always False - return False, "auto_activation_forbidden" - if not approver: - return False, "human_approval_required" - return True, "" diff --git a/src/backend/core/files/artifacts/store.py b/src/backend/core/files/artifacts/store.py index 3e245fb2..b63fd74e 100644 --- a/src/backend/core/files/artifacts/store.py +++ b/src/backend/core/files/artifacts/store.py @@ -146,12 +146,12 @@ def _write_record_local(item: Dict[str, Any]) -> str: def _write_record(item: Dict[str, Any]) -> None: file_id = str(item["file_id"]) - text = _write_record_local(item) if _storage_type() in ("s3", "oss", "obs"): - try: - _get_oss_storage().upload_bytes(text.encode("utf-8"), _record_oss_key(file_id)) - except Exception as exc: # noqa: BLE001 — the local record is authoritative - logger.warning("Failed to mirror artifact record %s to OSS: %s", file_id, exc) + # The object record is durable; a local cache must never acknowledge a + # failed remote write. Leave an existing cached record unchanged on error. + text = json.dumps(item, ensure_ascii=False) + _get_oss_storage().upload_bytes(text.encode("utf-8"), _record_oss_key(file_id)) + _write_record_local(item) def migrate_legacy_index() -> int: @@ -405,7 +405,18 @@ def get_artifact(file_id: str) -> Optional[Dict[str, Any]]: """Look up artifact metadata by file_id.""" if not file_id: return None + # IDs form object keys and local filenames; do not allow path traversal. + if any(c in file_id for c in ("/", "\\", "\x00")) or file_id in (".", ".."): + return None item = _read_record(file_id) + if item is None and _storage_type() in ("s3", "oss", "obs"): + storage = _get_oss_storage() + key = _record_oss_key(file_id) + if storage.exists(key): + item = json.loads(storage.download_bytes(key).decode("utf-8")) + if not isinstance(item, dict) or item.get("file_id") != file_id: + raise ValueError("Invalid persisted artifact record") + _write_record_local(item) if item is None: return None diff --git a/src/backend/core/files/content/public_assets.py b/src/backend/core/files/content/public_assets.py new file mode 100644 index 00000000..fb36b910 --- /dev/null +++ b/src/backend/core/files/content/public_assets.py @@ -0,0 +1,114 @@ +"""Public branding and manual assets, persisted with the configured storage.""" +from __future__ import annotations + +import hashlib +import os +import re +import tempfile +import uuid +from pathlib import Path + +from fastapi import HTTPException +from fastapi.responses import FileResponse +from starlette.background import BackgroundTask + +from core.files.storage import get_storage + +PREFIX = "public-content" + + +def remote() -> bool: + return os.getenv("STORAGE_TYPE", "local").lower() in ("s3", "oss", "obs") + + +def key(kind: str, filename: str) -> str: + manual = kind == "manual" and filename == "manual.pdf" + image = kind == "page_config" and re.fullmatch( + r"(?:logo|favicon|app_icon|mcp_icon)_[0-9a-f]{12}\.(?:png|jpg|svg|webp|ico)", filename + ) + if not (manual or image): + raise HTTPException(404, "Asset not found") + return f"{PREFIX}/{kind}/{filename}" + + +def url(kind: str, filename: str) -> str: + key(kind, filename) + return f"/api/v1/content/public-assets/{kind}/{filename}" + + +def upload(kind: str, filename: str, content: bytes) -> str: + get_storage().upload_bytes(content, key(kind, filename)) + return url(kind, filename) + + +def manual_info(): + item = get_storage().stat_object(key("manual", "manual.pdf")) + if item is None: + return {"exists": False} + return {"exists": True, "filename": "manual.pdf", "size": item.size, + "uploaded_at": item.modified_at.isoformat() if item.modified_at else None, + "url": url("manual", "manual.pdf")} + + +def _cache_root() -> Path: + return Path(tempfile.gettempdir()) / "hugagent-public-assets" + + +def _cached_copy(storage, storage_key: str, item) -> Path: + """Return a local copy of the current object version, downloading it at most once. + + The copy is a disposable cache in the container's temporary directory, keyed + by the object's version metadata; object storage stays the only durable store. + Range requests are then answered from the local file instead of re-downloading + the whole object for every slice a PDF viewer asks for. + """ + root = _cache_root() + root.mkdir(mode=0o700, exist_ok=True) + stem = hashlib.sha256(storage_key.encode()).hexdigest()[:32] + version = item.etag or f"{item.size}:{item.modified_at.isoformat() if item.modified_at else ''}" + tag = hashlib.sha256(f"{version}\0{item.size}".encode()).hexdigest()[:32] + path = root / f"{stem}-{tag}{Path(storage_key).suffix}" + if path.is_file() and path.stat().st_size == item.size: + return path + handle = tempfile.NamedTemporaryFile(dir=root, prefix=".download-", delete=False) + temporary = Path(handle.name) + handle.close() + try: + storage.download(storage_key, str(temporary)) + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + for stale in root.glob(f"{stem}-*"): + if stale != path: + stale.unlink(missing_ok=True) + return path + + +def response(kind: str, filename: str): + storage_key = key(kind, filename) + storage = get_storage() + item = storage.stat_object(storage_key) + if item is None: + raise HTTPException(404, "Asset not found") + # Each response reads its own hard link, so replacing the cached version + # while a slow download is still in flight never breaks that download. + for _ in range(2): + path = _cached_copy(storage, storage_key, item) + serving = path.with_name(f".serving-{uuid.uuid4().hex}{path.suffix}") + try: + os.link(path, serving) + break + except FileNotFoundError: # a newer version replaced it just now + item = storage.stat_object(storage_key) + if item is None: + raise HTTPException(404, "Asset not found") + else: + raise HTTPException(503, "Asset is being replaced, retry") + headers = {"X-Content-Type-Options": "nosniff", + "Cache-Control": "no-cache" if kind == "manual" else "public, max-age=31536000, immutable"} + if kind != "manual": + # Uploaded SVG images must never run script. The manual keeps the headers it + # had as a static file: browsers refuse to open a PDF under a CSP sandbox. + headers["Content-Security-Policy"] = "sandbox; default-src 'none'" + return FileResponse(serving, filename=filename, content_disposition_type="inline", headers=headers, + background=BackgroundTask(serving.unlink, missing_ok=True)) diff --git a/src/backend/core/files/spaces/access.py b/src/backend/core/files/spaces/access.py index 0c44f5cb..83f7c4ac 100644 --- a/src/backend/core/files/spaces/access.py +++ b/src/backend/core/files/spaces/access.py @@ -52,6 +52,12 @@ def resolve_grants(db, actor: str) -> tuple[SpaceGrant, ...]: from core.files.edition import team_grants return (personal_grant(actor), *team_grants(db, actor)) +def grants_for(actor: str) -> tuple[SpaceGrant, ...]: + """``resolve_grants`` in its own database session.""" + from core.db.engine import SessionLocal + with SessionLocal() as db: + return resolve_grants(db, actor) + def require_grant(db, actor: str, space_id: str, *, write=False) -> SpaceGrant: grant = next((g for g in resolve_grants(db, actor) if g.space_id == space_id), None) if grant is None: diff --git a/src/backend/core/files/spaces/mounts.py b/src/backend/core/files/spaces/mounts.py index dc7c0c5d..89a4ad8a 100644 --- a/src/backend/core/files/spaces/mounts.py +++ b/src/backend/core/files/spaces/mounts.py @@ -1,10 +1,8 @@ """Create provider volumes from authorized space grants.""" import os import base64 -from pathlib import Path from fastapi import HTTPException from core.files.spaces.access import fingerprint, resolve_grants -from core.files.storage.entries import validate_key MOUNT_REVISION_TAG = "hugagent-space-revision" @@ -20,6 +18,9 @@ def mount_revision(grants): prefix + "_ACCESS_KEY_ID", prefix + "_ACCESS_KEY_SECRET", "OSS_KEY_PREFIX", )} + if os.getenv("DEPLOY_PROFILE") == "kubernetes": + config["mount_mode"] = os.getenv("HUGAGENT_SPACE_MOUNT_MODE", "csi") + config["claim"] = os.getenv("HUGAGENT_SPACE_CLAIM", "") if backend == "local": config["path"] = str(settings.storage.root.resolve()) config["host_path"] = settings.sandbox.opensandbox_host_storage_path @@ -29,53 +30,16 @@ def mount_revision(grants): def provider_volume(grant): - from opensandbox.models.sandboxes import Host, OSSFS, Volume - from core.config.settings import settings - - backend = os.getenv("STORAGE_TYPE", "local").lower() - name = "space-" + fingerprint((grant,))[:24] - if backend == "local": - root = settings.storage.root.resolve() - path = root / grant.prefix - if path.is_symlink() or not path.resolve().is_relative_to(root): - raise HTTPException(503, "Invalid local space root") - host_root = settings.sandbox.opensandbox_host_storage_path - if not host_root: - raise HTTPException(503, "HOST_STORAGE_PATH is required for space volumes") - return Volume(name=name, host=Host(path=str(Path(host_root) / grant.prefix)), - mount_path=grant.mount_path, read_only=not grant.writable) - if backend not in ("s3", "oss", "obs"): - raise HTTPException(503, "Unsupported storage provider") - prefix = backend.upper() - endpoint = os.getenv(prefix + "_ENDPOINT", "") - bucket = os.getenv(prefix + "_BUCKET", "") - access = os.getenv(prefix + ("_ACCESS_KEY_ID" if backend == "oss" else "_ACCESS_KEY"), "") - secret = os.getenv(prefix + ("_ACCESS_KEY_SECRET" if backend == "oss" else "_SECRET_KEY"), "") - if not all((endpoint, bucket, access, secret)): - raise HTTPException(503, "Object mount configuration is incomplete") - key_prefix = os.getenv("OSS_KEY_PREFIX", "").strip("/") if backend == "oss" else "" - path = "/".join(part for part in (key_prefix, grant.prefix.rstrip("/")) if part) - validate_key(path) - options = ["hugagent_driver=s3fs", "allow_other", "uid=1000", "gid=1000", - "umask=0022", "stat_cache_expire=0", "attr_timeout=0", "entry_timeout=0", - "negative_timeout=0"] - if backend == "oss": - options.append("sigv2") - else: - options.extend(["sigv4", "endpoint=" + os.getenv(prefix + "_REGION", "us-east-1")]) - if backend == "s3": - options.append("use_path_request_style") - if not grant.writable: - options.append("ro") - return Volume(name=name, mount_path=grant.mount_path, read_only=not grant.writable, - sub_path=path, ossfs=OSSFS(bucket=bucket, endpoint=endpoint, version="1.0", - access_key_id=access, access_key_secret=secret, options=options)) + """The provider volume for one grant, or None when the transport copies files.""" + from core.files.spaces.transport import get_space_transport + return get_space_transport().volume(grant) def prepare_volumes(actor): from core.db.engine import SessionLocal with SessionLocal() as db: grants = resolve_grants(db, actor) - return [provider_volume(grant) for grant in grants], {MOUNT_REVISION_TAG: mount_revision(grants)} + volumes = [provider_volume(grant) for grant in grants] + return [v for v in volumes if v is not None], {MOUNT_REVISION_TAG: mount_revision(grants)} def provision_volumes(actor): diff --git a/src/backend/core/files/spaces/recovery.py b/src/backend/core/files/spaces/recovery.py index b7604d6f..89f07cf6 100644 --- a/src/backend/core/files/spaces/recovery.py +++ b/src/backend/core/files/spaces/recovery.py @@ -1,41 +1,9 @@ -"""Restore authorized mounts in the existing sandbox after controller restart.""" -import asyncio -import os +"""Verify an existing sandbox still exposes the account's current grants. -import httpx -from fastapi import HTTPException +The checks themselves belong to each transport (``core.files.spaces.transport``). +""" -from core.files.spaces.leases import async_space_lock -from core.files.spaces.mounts import MOUNT_REVISION_TAG, prepare_volumes - -EPOCH_TAG = "hugagent-mount-epoch" async def ensure_mounts(sandbox, actor, info, *, domain, api_key): - if os.getenv("STORAGE_TYPE", "local").lower() == "local": - return - metadata = info.metadata or {} - epoch = metadata.get(EPOCH_TAG) - if not epoch: - raise HTTPException(503, "挂载控制器尚未就绪,工作区已保留") - # Controller identity does not imply sandbox or FUSE-owner health. - # Every use checks the live mounts, including after a sandbox restart. - async with async_space_lock(actor): - # Re-resolve while holding the same lock as membership changes. Never - # recover volumes from a historical permission snapshot. - volumes, current = await asyncio.to_thread(prepare_volumes, actor) - if current[MOUNT_REVISION_TAG] != metadata.get(MOUNT_REVISION_TAG): - raise HTTPException(409, "文件权限已变更,挂载未恢复") - base = domain if "://" in domain else "http://" + domain - headers = {"OPEN-SANDBOX-API-KEY": api_key} if api_key else {} - async with httpx.AsyncClient(timeout=90, trust_env=False) as client: - response = await client.post( - base.rstrip("/") + "/v1/sandboxes/" + str(sandbox.id) + "/space-mounts/ensure", - headers=headers, json={"actor": actor, "revision": current[MOUNT_REVISION_TAG], - "volumes": [v.model_dump(mode="json", by_alias=True) for v in volumes]}, - ) - if response.status_code == 409: - raise HTTPException(409, "文件挂载权限或配置已变更,沙盒和工作区已保留,请联系管理员") - if response.status_code != 200: - raise HTTPException(503, "文件挂载恢复失败,沙盒和工作区已保留,请重试") - if response.json().get("epoch") != epoch: - raise HTTPException(503, "挂载控制器正在重启,请重试") + from core.files.spaces.transport import get_space_transport + await get_space_transport().ensure(sandbox, actor, info, domain=domain, api_key=api_key) diff --git a/src/backend/core/files/spaces/sandbox.py b/src/backend/core/files/spaces/sandbox.py index 7940d6c4..31885d0e 100644 --- a/src/backend/core/files/spaces/sandbox.py +++ b/src/backend/core/files/spaces/sandbox.py @@ -15,6 +15,10 @@ def current_revision(actor): class SpaceSandboxMixin: from core.files.edition import workspace_mount_names + from core.files.spaces.transport.sync.sandbox_io import STATE_DIR_NAME + # A synchronized space copy keeps its baseline across a reset for reuse. + workspace_mount_names = (*workspace_mount_names, STATE_DIR_NAME) + del STATE_DIR_NAME def _session_entrypoint(self, user_id): from core.files.edition import session_entrypoint @@ -61,4 +65,7 @@ async def _renew_in_background(self, session_id, session): ).get(MOUNT_REVISION_TAG): await self._destroy_session(session_id, session, reuse_sandbox=False) return + from core.files.spaces.transport import get_space_transport + # Files written after a command returned (background processes). + await get_space_transport().flush(session.sandbox, session.user_id) await super()._renew_in_background(session_id, session) diff --git a/src/backend/core/files/spaces/transport/__init__.py b/src/backend/core/files/spaces/transport/__init__.py new file mode 100644 index 00000000..5d499099 --- /dev/null +++ b/src/backend/core/files/spaces/transport/__init__.py @@ -0,0 +1,48 @@ +"""Select the space transport for this deployment. + +Docker deployments follow ``STORAGE_TYPE``. Kubernetes deployments follow +``HUGAGENT_SPACE_MOUNT_MODE``: ``csi`` (default), ``sidecar`` or ``sync``. +Configuration is read on every call so a changed environment cannot leave a +stale transport behind; instances are stateless. +""" +import os + +from core.files.spaces.transport.base import SpaceTransport, SyncReport + +KUBERNETES_MODES = ("csi", "sidecar", "sync") + + +def kubernetes_mode() -> str: + mode = os.getenv("HUGAGENT_SPACE_MOUNT_MODE", "csi").strip().lower() or "csi" + if mode not in KUBERNETES_MODES: + raise RuntimeError( + f"HUGAGENT_SPACE_MOUNT_MODE={mode!r} is not supported; use one of {', '.join(KUBERNETES_MODES)}") + return mode + + +def get_space_transport() -> SpaceTransport: + if os.getenv("DEPLOY_PROFILE") == "kubernetes": + mode = kubernetes_mode() + if mode == "sidecar": + from core.files.spaces.transport.k8s_sidecar import KubernetesSidecarTransport + return KubernetesSidecarTransport() + if mode == "sync": + from core.files.spaces.transport.sync import SyncTransport + return SyncTransport() + from core.files.spaces.transport.k8s_csi import KubernetesCsiTransport + return KubernetesCsiTransport() + if os.getenv("STORAGE_TYPE", "local").lower() == "local": + from core.files.spaces.transport.host import HostTransport + return HostTransport() + from core.files.spaces.transport.object_fs import ObjectFsTransport + return ObjectFsTransport() + + +def validate_space_transport() -> str: + """Fail at startup on a configuration the selected transport cannot serve.""" + transport = get_space_transport() + transport.validate() + return transport.name + + +__all__ = ["SpaceTransport", "SyncReport", "get_space_transport", "validate_space_transport"] diff --git a/src/backend/core/files/spaces/transport/base.py b/src/backend/core/files/spaces/transport/base.py new file mode 100644 index 00000000..640def1e --- /dev/null +++ b/src/backend/core/files/spaces/transport/base.py @@ -0,0 +1,79 @@ +"""How authorized space files reach a sandbox. + +A transport either mounts the space (the kernel keeps it consistent) or keeps +a synchronized copy inside the sandbox. Sandbox providers only call this +interface; they never branch on the deployment mode themselves. +""" +from __future__ import annotations + +from dataclasses import dataclass, field + + +@dataclass +class SyncReport: + """What a synchronization could not apply as-is, shown after the command.""" + + conflicts: list[tuple[str, str]] = field(default_factory=list) + rejected: list[str] = field(default_factory=list) + skipped: list[str] = field(default_factory=list) + errors: list[str] = field(default_factory=list) + + def __bool__(self) -> bool: + return bool(self.conflicts or self.rejected or self.skipped or self.errors) + + def text(self, limit: int = 20) -> str: + if not self: + return "" + lines = ["[空间同步]"] + for path, copy in self.conflicts[:limit]: + lines.append(f"冲突:{path} 已被他人修改,保留远端版本;沙盒中的版本另存为 {copy}") + for path in self.rejected[:limit]: + lines.append(f"只读:{path} 所在空间不可写,修改未保存,已恢复为远端版本") + for path in self.skipped[:limit]: + lines.append(f"跳过:{path} 超出同步大小上限,未放入沙盒") + for message in self.errors[:limit]: + lines.append(f"失败:{message}") + hidden = sum(max(0, len(items) - limit) for items in + (self.conflicts, self.rejected, self.skipped, self.errors)) + if hidden: + lines.append(f"另有 {hidden} 条未列出") + return "\n".join(lines) + + +class SpaceTransport: + """Mount transports keep every hook below a no-op.""" + + name = "base" + # True when the sandbox holds a copy that the hooks below must keep in step. + synchronizes = False + + def validate(self) -> None: + """Fail at startup when the deployment cannot serve this transport.""" + + def volume(self, grant): + """The provider volume for one grant, or None when nothing is mounted.""" + raise NotImplementedError + + @staticmethod + def _volume(grant, **source): + from opensandbox.models.sandboxes import Volume + from core.files.spaces.access import fingerprint + return Volume(name="space-" + fingerprint((grant,))[:24], mount_path=grant.mount_path, + read_only=not grant.writable, **source) + + async def ensure(self, sandbox, actor, info, *, domain, api_key) -> None: + """Verify an existing sandbox still exposes the current grants.""" + + async def before_use(self, sandbox, actor, *, path=None) -> SyncReport: + """Bring remote changes into the sandbox before code reads them. + + ``path`` limits the work to the space holding that sandbox path.""" + return SyncReport() + + async def after_use(self, sandbox, actor, *, path=None) -> SyncReport: + """Persist sandbox changes after code wrote them.""" + return SyncReport() + + async def flush(self, sandbox, actor, *, force=False) -> SyncReport: + """Best-effort persistence outside a command (renewal, release).""" + return SyncReport() diff --git a/src/backend/core/files/spaces/transport/host.py b/src/backend/core/files/spaces/transport/host.py new file mode 100644 index 00000000..529f3117 --- /dev/null +++ b/src/backend/core/files/spaces/transport/host.py @@ -0,0 +1,23 @@ +"""Docker deployments with local storage bind the host directory directly.""" +from pathlib import Path + +from fastapi import HTTPException + +from core.files.spaces.transport.base import SpaceTransport + + +class HostTransport(SpaceTransport): + name = "host" + + def volume(self, grant): + from opensandbox.models.sandboxes import Host + from core.config.settings import settings + + root = settings.storage.root.resolve() + path = root / grant.prefix + if path.is_symlink() or not path.resolve().is_relative_to(root): + raise HTTPException(503, "Invalid local space root") + host_root = settings.sandbox.opensandbox_host_storage_path + if not host_root: + raise HTTPException(503, "HOST_STORAGE_PATH is required for space volumes") + return self._volume(grant, host=Host(path=str(Path(host_root) / grant.prefix))) diff --git a/src/backend/core/files/spaces/transport/k8s_csi.py b/src/backend/core/files/spaces/transport/k8s_csi.py new file mode 100644 index 00000000..71cf49bb --- /dev/null +++ b/src/backend/core/files/spaces/transport/k8s_csi.py @@ -0,0 +1,21 @@ +"""Kubernetes: mount authorized S3 prefixes through a prebound, namespace-local CSI claim.""" +import os +import re + +from fastapi import HTTPException + +from core.files.spaces.transport.k8s_mount import KubernetesMountTransport + + +class KubernetesCsiTransport(KubernetesMountTransport): + name = "k8s-csi" + + def volume(self, grant): + from opensandbox.models.sandboxes import PVC + if os.getenv("STORAGE_TYPE", "").lower() != "s3": + raise HTTPException(503, "Native space mounts require the configured S3 CSI storage") + claim = os.environ.get("HUGAGENT_SPACE_CLAIM", "") + if not re.fullmatch(r"[a-z0-9][a-z0-9.-]{0,62}", claim): + raise HTTPException(503, "Configure the namespace-local space CSI claim") + return self._volume(grant, pvc=PVC(claim_name=claim, create_if_not_exists=False), + sub_path=self._prepared_prefix(grant)) diff --git a/src/backend/core/files/spaces/transport/k8s_mount.py b/src/backend/core/files/spaces/transport/k8s_mount.py new file mode 100644 index 00000000..d95ec66e --- /dev/null +++ b/src/backend/core/files/spaces/transport/k8s_mount.py @@ -0,0 +1,22 @@ +"""Shared preparation of a grant prefix for Kubernetes mount transports.""" +import os + +from core.files.spaces.transport.base import SpaceTransport +from core.files.storage.entries import validate_key + + +class KubernetesMountTransport(SpaceTransport): + def validate(self): + if os.getenv("STORAGE_TYPE", "").lower() != "s3": + raise RuntimeError("Kubernetes space transports require STORAGE_TYPE=s3") + + @staticmethod + def _prepared_prefix(grant) -> str: + """The grant prefix as a subPath; the trusted backend creates its directory + marker, since Kubernetes subPath needs a real directory even for a + read-only member's first access.""" + from core.files.storage import get_storage + prefix = grant.prefix.rstrip("/") + validate_key(prefix) + get_storage().make_directory(grant.prefix) + return prefix diff --git a/src/backend/core/files/spaces/transport/k8s_sidecar.py b/src/backend/core/files/spaces/transport/k8s_sidecar.py new file mode 100644 index 00000000..5ad745f6 --- /dev/null +++ b/src/backend/core/files/spaces/transport/k8s_sidecar.py @@ -0,0 +1,62 @@ +"""Kubernetes: a privileged helper container mounts object storage into the sandbox Pod.""" +import asyncio +import json +import os +import shlex +from datetime import timedelta + +from fastapi import HTTPException + +from core.files.spaces.leases import async_space_lock +from core.files.spaces.mounts import MOUNT_REVISION_TAG, prepare_volumes +from core.files.spaces.transport.k8s_mount import KubernetesMountTransport + +_CHECK_SCRIPT = """import json,os +from pathlib import Path +expected=json.loads(EXPECTED) +mounts={} +for line in Path('/proc/self/mountinfo').read_text().splitlines(): + before,after=line.split(' - ',1) + fields=before.split() + if after.split()[0]=='fuse.s3fs': + mounts[fields[4]]=fields[5].split(',') +for path,readonly in expected: + assert path in mounts, 'missing object filesystem' + assert ('ro' in mounts[path]) == readonly, 'incorrect mount permission' + os.stat(path) +""" + + +class KubernetesSidecarTransport(KubernetesMountTransport): + name = "k8s-sidecar" + + def volume(self, grant): + from opensandbox.models.sandboxes import OSSFS + if os.getenv("STORAGE_TYPE", "").lower() != "s3": + raise HTTPException(503, "Kubernetes spaces require external S3 storage") + endpoint, bucket = os.getenv("S3_ENDPOINT", ""), os.getenv("S3_BUCKET", "") + if not endpoint or not bucket: + raise HTTPException(503, "External object mount configuration is incomplete") + return self._volume(grant, sub_path=self._prepared_prefix(grant), + ossfs=OSSFS(bucket=bucket, endpoint=endpoint, version="1.0", + access_key_id="server-managed", access_key_secret="server-managed")) + + async def ensure(self, sandbox, actor, info, *, domain, api_key): + """Check kernel mounts in the code container before every workspace use.""" + from opensandbox.models.execd import RunCommandOpts + async with async_space_lock(actor): + volumes, current = await asyncio.to_thread(prepare_volumes, actor) + if current[MOUNT_REVISION_TAG] != (info.metadata or {}).get(MOUNT_REVISION_TAG): + raise HTTPException(409, "文件权限已变更,挂载未恢复") + expected = [(v.mount_path, v.read_only) for v in volumes] + if not expected: + return + script = _CHECK_SCRIPT.replace("EXPECTED", repr(json.dumps(expected))) + try: + result = await asyncio.wait_for(sandbox.commands.run( + "python3 -c " + shlex.quote(script), + opts=RunCommandOpts(timeout=timedelta(seconds=8))), timeout=12) + if result.exit_code != 0: + raise RuntimeError("Space filesystem is not mounted") + except Exception as exc: + raise HTTPException(503, "文件挂载未就绪,工作区已保留,请重试") from exc diff --git a/src/backend/core/files/spaces/transport/object_fs.py b/src/backend/core/files/spaces/transport/object_fs.py new file mode 100644 index 00000000..182e17ec --- /dev/null +++ b/src/backend/core/files/spaces/transport/object_fs.py @@ -0,0 +1,75 @@ +"""Docker deployments with object storage mount through the controller's FUSE worker.""" +import asyncio +import os + +import httpx +from fastapi import HTTPException + +from core.files.spaces.leases import async_space_lock +from core.files.spaces.mounts import MOUNT_REVISION_TAG, prepare_volumes +from core.files.spaces.transport.base import SpaceTransport +from core.files.storage.entries import validate_key + +EPOCH_TAG = "hugagent-mount-epoch" + + +class ObjectFsTransport(SpaceTransport): + name = "object-fs" + + def volume(self, grant): + from opensandbox.models.sandboxes import OSSFS + + backend = os.getenv("STORAGE_TYPE", "local").lower() + if backend not in ("s3", "oss", "obs"): + raise HTTPException(503, "Unsupported storage provider") + prefix = backend.upper() + endpoint = os.getenv(prefix + "_ENDPOINT", "") + bucket = os.getenv(prefix + "_BUCKET", "") + access = os.getenv(prefix + ("_ACCESS_KEY_ID" if backend == "oss" else "_ACCESS_KEY"), "") + secret = os.getenv(prefix + ("_ACCESS_KEY_SECRET" if backend == "oss" else "_SECRET_KEY"), "") + if not all((endpoint, bucket, access, secret)): + raise HTTPException(503, "Object mount configuration is incomplete") + key_prefix = os.getenv("OSS_KEY_PREFIX", "").strip("/") if backend == "oss" else "" + path = "/".join(part for part in (key_prefix, grant.prefix.rstrip("/")) if part) + validate_key(path) + options = ["hugagent_driver=s3fs", "allow_other", "uid=1000", "gid=1000", + "umask=0022", "stat_cache_expire=0", "attr_timeout=0", "entry_timeout=0", + "negative_timeout=0"] + if backend == "oss": + options.append("sigv2") + else: + options.extend(["sigv4", "endpoint=" + os.getenv(prefix + "_REGION", "us-east-1")]) + if backend == "s3": + options.append("use_path_request_style") + if not grant.writable: + options.append("ro") + return self._volume(grant, sub_path=path, ossfs=OSSFS(bucket=bucket, endpoint=endpoint, + version="1.0", access_key_id=access, access_key_secret=secret, options=options)) + + async def ensure(self, sandbox, actor, info, *, domain, api_key): + metadata = info.metadata or {} + epoch = metadata.get(EPOCH_TAG) + if not epoch: + raise HTTPException(503, "挂载控制器尚未就绪,工作区已保留") + # Controller identity does not imply sandbox or FUSE-owner health. + # Every use checks the live mounts, including after a sandbox restart. + async with async_space_lock(actor): + # Re-resolve while holding the same lock as membership changes. Never + # recover volumes from a historical permission snapshot. + volumes, current = await asyncio.to_thread(prepare_volumes, actor) + if current[MOUNT_REVISION_TAG] != metadata.get(MOUNT_REVISION_TAG): + raise HTTPException(409, "文件权限已变更,挂载未恢复") + base = domain if "://" in domain else "http://" + domain + headers = {"OPEN-SANDBOX-API-KEY": api_key} if api_key else {} + async with httpx.AsyncClient(timeout=90, trust_env=False) as client: + response = await client.post( + base.rstrip("/") + "/v1/sandboxes/" + str(sandbox.id) + "/space-mounts/ensure", + headers=headers, json={"actor": actor, "revision": current[MOUNT_REVISION_TAG], + "volumes": [v.model_dump(mode="json", by_alias=True) for v in volumes]}, + ) + if response.status_code == 409: + raise HTTPException(409, "文件挂载权限或配置已变更,沙盒和工作区已保留,请联系管理员") + if response.status_code != 200: + raise HTTPException(503, "文件挂载恢复失败,沙盒和工作区已保留,请重试") + if response.json().get("epoch") != epoch: + raise HTTPException(503, "挂载控制器正在重启,请重试") diff --git a/src/backend/core/files/spaces/transport/sync/__init__.py b/src/backend/core/files/spaces/transport/sync/__init__.py new file mode 100644 index 00000000..0e834fcd --- /dev/null +++ b/src/backend/core/files/spaces/transport/sync/__init__.py @@ -0,0 +1,259 @@ +"""Kubernetes without privileged mounts: keep a synchronized copy of each space. + +Each use compares object storage, the sandbox copy and the baseline recorded +at the last synchronization (``manifest.plan``), then moves only what changed +in either direction over execd's HTTP file API. No volume is mounted, so the +sandbox Pod needs no FUSE, privileged helper or extra RBAC. + +Consistency holds at command boundaries: remote changes appear before a +command starts, sandbox changes are stored after it ends (and periodically +while the session is kept alive). A change made on both sides is never lost; +the sandbox version is stored next to the remote one under a conflict name. +""" +from __future__ import annotations + +import asyncio +import hashlib +import logging +import os +import tempfile +import time +import uuid +from datetime import datetime + +from core.files.spaces.access import grants_for +from core.files.spaces.transport.base import SpaceTransport, SyncReport +from core.files.spaces.transport.sync import sandbox_io +from core.files.spaces.transport.sync.config import sync_limits +from core.files.spaces.transport.sync.manifest import ( + Baseline, Base, Local, conflict_name, locally_changed, plan, +) +from core.files.spaces.transport.sync.remote import PreconditionFailed, RemoteSpace + +logger = logging.getLogger(__name__) + +_LOCK_ATTR = "_jx_space_sync_lock" +_FLUSHED_ATTR = "_jx_space_flushed_at" +_DOWNLOAD_BATCH = 64 + + +def state_path(grant) -> str: + digest = hashlib.sha256(grant.prefix.encode()).hexdigest()[:32] + return f"{sandbox_io.STATE_DIR}/{digest}.json" + + +def _containing(grants, actor, path): + """The personal grants whose copy holds ``path``.""" + for alias, target in (("/myspace", f"/workspace/myspace/{actor}"),): + if path == alias or path.startswith(alias + "/"): + path = target + path[len(alias):] + return tuple(g for g in grants if path == g.mount_path or path.startswith(g.mount_path + "/")) + + +def _spooled(): + return tempfile.SpooledTemporaryFile(max_size=8 * 1024 * 1024) + + +class SyncTransport(SpaceTransport): + name = "k8s-sync" + synchronizes = True + + def validate(self): + if os.getenv("STORAGE_TYPE", "").lower() != "s3": + raise RuntimeError("Kubernetes space transports require STORAGE_TYPE=s3") + if not os.getenv("S3_BUCKET"): + raise RuntimeError("HUGAGENT_SPACE_MOUNT_MODE=sync requires S3_BUCKET") + sync_limits() + + def volume(self, grant): + return None + + async def before_use(self, sandbox, actor, *, path=None) -> SyncReport: + from core.sandbox.errors import SandboxError + try: + return await self.synchronize(sandbox, actor, pull=True, path=path) + except Exception as exc: + logger.warning("[space-sync] pull failed sandbox=%s: %r", getattr(sandbox, "id", "?"), exc) + raise SandboxError("空间文件同步失败,命令未执行,工作区已保留,请重试") from exc + + async def after_use(self, sandbox, actor, *, path=None) -> SyncReport: + try: + return await self.synchronize(sandbox, actor, pull=False, path=path) + except Exception as exc: # noqa: BLE001 - the command already ran; report, retry later + logger.warning("[space-sync] push failed sandbox=%s: %r", getattr(sandbox, "id", "?"), exc) + return SyncReport(errors=["空间写回失败,修改仍保留在沙盒中,将在下次命令前重试"]) + + async def flush(self, sandbox, actor, *, force=False) -> SyncReport: + if not force and time.monotonic() - getattr(sandbox, _FLUSHED_ATTR, 0.0) < sync_limits().flush_interval_s: + return SyncReport() + report = await self.after_use(sandbox, actor) + # Also after a failure, which ``synchronize`` does not record: throttles retries. + setattr(sandbox, _FLUSHED_ATTR, time.monotonic()) + return report + + async def synchronize(self, sandbox, actor, *, pull: bool, path=None) -> SyncReport: + """Bring each grant's copy and its objects to the same state. + + Without ``pull`` (after a command) a grant whose copy is unchanged since + the last sync is skipped without listing storage; remote changes reach it + before the next command.""" + report = SyncReport() + if not actor: + return report + grants = await asyncio.to_thread(grants_for, actor) + if path is not None: + grants = _containing(grants, actor, str(path)) + if not grants: + return report + limits = sync_limits() + from core.files.storage import get_storage + from core.sandbox.providers.opensandbox.runtime import sandbox_lock + storage = get_storage() + async with sandbox_lock(sandbox, _LOCK_ATTR): + roots = [{"root": g.mount_path, "state": state_path(g)} for g in grants] + scanned = await sandbox_io.scan(sandbox, roots, md5_limit=limits.max_file_bytes) + # Grants own disjoint roots and prefixes. + await asyncio.gather(*( + _GrantSync(sandbox, grant, RemoteSpace(storage, grant.prefix), scanned[grant.mount_path], + limits, report).run(pull=pull) + for grant in grants)) + setattr(sandbox, _FLUSHED_ATTR, time.monotonic()) + return report + + +class _GrantSync: + """One synchronization of one grant: plan, transfer, then record the baseline.""" + + def __init__(self, sandbox, grant, remote: RemoteSpace, scanned: dict, limits, report: SyncReport): + self.sandbox, self.grant, self.remote, self.limits = sandbox, grant, remote, limits + self.local = {rel: Local(v[0], v[1], v[2]) for rel, v in scanned["files"].items()} + self.local_dirs = set(scanned["dirs"]) + self.has_baseline = scanned["baseline"] is not None + self.baseline = Baseline.load(scanned["baseline"]) + self.slots = asyncio.Semaphore(limits.concurrency) + self.stamp = datetime.now().strftime("%Y%m%d-%H%M%S") + self.report = report + + async def run(self, *, pull: bool) -> None: + if not pull and self.has_baseline and not locally_changed(self.local, self.local_dirs, self.baseline): + return + remote, remote_dirs = await asyncio.to_thread(self.remote.listing, limit=self.limits.max_entries) + result = plan(remote, remote_dirs, self.local, self.local_dirs, self.baseline, + writable=self.grant.writable, max_file_bytes=self.limits.max_file_bytes, + max_total_bytes=self.limits.max_total_bytes) + if result.is_empty() and self.has_baseline: + return + files = dict(self.baseline.files) + downloads, local_deletes = list(result.downloads), list(result.local_deletes) + + async def resolve_conflict(rel): + await self._keep_sandbox_copy(rel) + (downloads if rel in remote and rel not in result.skipped else local_deletes).append(rel) + + async def upload(rel, expected): + item = self.local[rel] + try: + etag = await self._push(rel, rel, expected) + except PreconditionFailed: + await self._keep_sandbox_copy(rel) + downloads.append(rel) + return + # The scanned state, not a fresh stat: a later edit must still look changed. + files[rel] = Base(etag, item.size, item.mtime_ns) + + async def delete(rel, expected): + try: + await asyncio.to_thread(self.remote.delete, rel, expected) + files.pop(rel, None) + except PreconditionFailed: + downloads.append(rel) + + await asyncio.gather( + *(resolve_conflict(rel) for rel in result.conflicts), + *(upload(rel, expected) for rel, expected in result.uploads.items()), + *(delete(rel, expected) for rel, expected in result.remote_deletes.items()), + *(asyncio.to_thread(self.remote.make_directory, rel) for rel in result.remote_mkdirs)) + for rel, etag in result.adopt.items(): + item = self.local[rel] + files[rel] = Base(etag, item.size, item.mtime_ns) + for rel in result.forget: + files.pop(rel, None) + # Deepest first: a parent is only empty once its children are gone. + removed_remote = {rel for rel in result.remote_rmdirs + if await asyncio.to_thread(self.remote.remove_directory, rel)} + dirs = ((remote_dirs | set(result.remote_mkdirs)) - removed_remote) & \ + ((self.local_dirs | set(result.local_mkdirs)) - set(result.local_rmdirs)) + + self.report.rejected += [self._display(rel) for rel in result.rejected] + self.report.skipped += [self._display(rel) for rel in result.newly_skipped] + + writes = await self._stage_downloads(sorted(set(downloads))) + outcome = await sandbox_io.apply(self.sandbox, { + "root": self.grant.mount_path, "state": state_path(self.grant), + "writable": self.grant.writable, "baseline": Baseline(files, dirs, result.skipped).dump(), + "writes": writes, + "deletes": [{"rel": rel, "expect": self._expect(rel)} for rel in sorted(set(local_deletes)) + if rel in self.local], + "mkdirs": sorted(set(result.local_mkdirs)), "rmdirs": result.local_rmdirs, + }) + if outcome.get("raced"): + logger.info("[space-sync] %d path(s) changed during sync; retried next time", + len(outcome["raced"])) + + def _display(self, rel: str) -> str: + return self.grant.mount_path + "/" + rel + + def _expect(self, rel): + item = self.local.get(rel) + return None if item is None else [item.size, item.mtime_ns] + + async def _push(self, rel: str, key: str, expected: str | None) -> str: + """Upload the sandbox file ``rel`` to ``key`` if the object still has ``expected``.""" + async with self.slots: + stream = _spooled() + try: + await sandbox_io.read_into(self.sandbox, self._display(rel), stream) + stream.seek(0) + return await asyncio.to_thread(self.remote.upload, key, stream, expected) + finally: + stream.close() + + async def _keep_sandbox_copy(self, rel: str) -> None: + """Store the sandbox version under a conflict name that does not exist yet.""" + for attempt in range(1, 100): + copy = conflict_name(rel, self.stamp if attempt == 1 else f"{self.stamp}-{attempt}") + try: + await self._push(rel, copy, None) + except PreconditionFailed: + continue + self.report.conflicts.append((self._display(rel), self._display(copy))) + return + raise RuntimeError("No free conflict name for " + rel) + + async def _stage_downloads(self, rels: list[str]) -> list[dict]: + writes = [] + token = uuid.uuid4().hex + for start in range(0, len(rels), _DOWNLOAD_BATCH): + chunk = rels[start:start + _DOWNLOAD_BATCH] + streams = [_spooled() for _ in chunk] + try: + async def fetch(rel, stream): + async with self.slots: + return await asyncio.to_thread(self.remote.download, rel, stream) + etags = await asyncio.gather(*(fetch(r, s) for r, s in zip(chunk, streams))) + staged = [] + for index, (rel, stream, etag) in enumerate(zip(chunk, streams, etags)): + if etag is None: + continue + path = f"/tmp/.jx-space-stage-{token}-{start + index}" + staged.append((path, stream)) + writes.append({"rel": rel, "staged": path, "etag": etag, + "expect": self._expect(rel)}) + await sandbox_io.stage(self.sandbox, staged) + finally: + for stream in streams: + stream.close() + return writes + + +__all__ = ["SyncTransport"] diff --git a/src/backend/core/files/spaces/transport/sync/config.py b/src/backend/core/files/spaces/transport/sync/config.py new file mode 100644 index 00000000..9d21a127 --- /dev/null +++ b/src/backend/core/files/spaces/transport/sync/config.py @@ -0,0 +1,35 @@ +"""Limits for the synchronized space copy, read from the environment on use.""" +import os +from dataclasses import dataclass + + +@dataclass(frozen=True) +class SyncLimits: + max_file_bytes: int + max_total_bytes: int + max_entries: int + concurrency: int + flush_interval_s: float + + +def _positive(name, default): + raw = os.getenv(name, "").strip() + if not raw: + return default + try: + value = int(raw) + except ValueError: + raise RuntimeError(f"{name} must be an integer") from None + if value <= 0: + raise RuntimeError(f"{name} must be positive") + return value + + +def sync_limits() -> SyncLimits: + return SyncLimits( + max_file_bytes=_positive("HUGAGENT_SPACE_SYNC_MAX_FILE_BYTES", 100 * 1024 * 1024), + max_total_bytes=_positive("HUGAGENT_SPACE_SYNC_MAX_TOTAL_BYTES", 2 * 1024 * 1024 * 1024), + max_entries=_positive("HUGAGENT_SPACE_SYNC_MAX_ENTRIES", 20000), + concurrency=_positive("HUGAGENT_SPACE_SYNC_CONCURRENCY", 8), + flush_interval_s=float(_positive("HUGAGENT_SPACE_SYNC_FLUSH_INTERVAL_S", 60)), + ) diff --git a/src/backend/core/files/spaces/transport/sync/manifest.py b/src/backend/core/files/spaces/transport/sync/manifest.py new file mode 100644 index 00000000..d1d6136c --- /dev/null +++ b/src/backend/core/files/spaces/transport/sync/manifest.py @@ -0,0 +1,216 @@ +"""Three-way comparison of object storage, the sandbox copy and the last sync. + +Pure functions only: every decision about which side wins lives here, so the +table in internal design docs is tested +without a sandbox or a bucket. +""" +from __future__ import annotations + +from dataclasses import dataclass, field + + +@dataclass(frozen=True) +class Remote: + etag: str + size: int + modified: float = 0.0 + + @property + def md5(self) -> str | None: + """S3 ETag of a single-part upload is the content MD5; multipart ETags are not.""" + value = self.etag.strip('"') + return None if "-" in value else value.lower() + + +@dataclass(frozen=True) +class Local: + size: int + mtime_ns: int + md5: str | None = None + + +@dataclass(frozen=True) +class Base: + etag: str + size: int + mtime_ns: int + + +@dataclass +class Baseline: + files: dict[str, Base] = field(default_factory=dict) + dirs: set[str] = field(default_factory=set) + # Remote files deliberately kept out of the copy, reported once each. + skipped: dict[str, str] = field(default_factory=dict) + + @classmethod + def load(cls, data) -> "Baseline": + if not isinstance(data, dict) or data.get("version") != 1: + return cls() + files = {} + for rel, item in (data.get("files") or {}).items(): + try: + files[rel] = Base(str(item["etag"]), int(item["size"]), int(item["mtime_ns"])) + except (KeyError, TypeError, ValueError): + continue + return cls(files, set(data.get("dirs") or ()), dict(data.get("skipped") or {})) + + def dump(self) -> dict: + return {"version": 1, + "files": {rel: {"etag": b.etag, "size": b.size, "mtime_ns": b.mtime_ns} + for rel, b in sorted(self.files.items())}, + "dirs": sorted(self.dirs), "skipped": dict(sorted(self.skipped.items()))} + + +@dataclass +class Plan: + # Remote content replaces the sandbox file (expected local state is recorded). + downloads: list[str] = field(default_factory=list) + # Sandbox content replaces the object; value is the ETag it must still have + # (None: the object must not exist yet). + uploads: dict[str, str | None] = field(default_factory=dict) + remote_deletes: dict[str, str] = field(default_factory=dict) + local_deletes: list[str] = field(default_factory=list) + # Both sides changed: keep the sandbox version under a new name, then download. + conflicts: list[str] = field(default_factory=list) + # Both sides hold identical new content; only the baseline is recorded. + adopt: dict[str, str] = field(default_factory=dict) + forget: list[str] = field(default_factory=list) + rejected: list[str] = field(default_factory=list) + newly_skipped: dict[str, str] = field(default_factory=dict) + skipped: dict[str, str] = field(default_factory=dict) + remote_mkdirs: list[str] = field(default_factory=list) + remote_rmdirs: list[str] = field(default_factory=list) + local_mkdirs: list[str] = field(default_factory=list) + local_rmdirs: list[str] = field(default_factory=list) + + def is_empty(self) -> bool: + return not any((self.downloads, self.uploads, self.remote_deletes, self.local_deletes, + self.conflicts, self.adopt, self.forget, self.remote_mkdirs, + self.remote_rmdirs, self.local_mkdirs, self.local_rmdirs, + self.newly_skipped)) + + +def local_changed(local: Local | None, base: Base | None) -> bool: + if local is None or base is None: + return (local is None) != (base is None) + return local.size != base.size or local.mtime_ns != base.mtime_ns + + +def remote_changed(remote: Remote | None, base: Base | None) -> bool: + if remote is None or base is None: + return (remote is None) != (base is None) + return remote.etag != base.etag + + +def locally_changed(local: dict[str, Local], local_dirs: set[str], baseline: Baseline) -> bool: + """Whether the sandbox copy differs from the last sync at all.""" + if local_dirs != baseline.dirs or set(local) - set(baseline.files) - set(baseline.skipped): + return True + return any(local_changed(local.get(rel), base) for rel, base in baseline.files.items()) + + +def eligible(remote: dict[str, Remote], *, max_file_bytes: int, max_total_bytes: int) -> set[str]: + """Newest files first until the copy reaches its total size limit.""" + chosen, total = set(), 0 + for rel, item in sorted(remote.items(), key=lambda kv: (-kv[1].modified, kv[0])): + if item.size > max_file_bytes or total + item.size > max_total_bytes: + continue + chosen.add(rel) + total += item.size + return chosen + + +def plan(remote: dict[str, Remote], remote_dirs: set[str], local: dict[str, Local], + local_dirs: set[str], baseline: Baseline, *, writable: bool, + max_file_bytes: int, max_total_bytes: int) -> Plan: + result = Plan() + allowed = eligible(remote, max_file_bytes=max_file_bytes, max_total_bytes=max_total_bytes) + for rel in sorted(set(remote) | set(local) | set(baseline.files)): + r, l, b = remote.get(rel), local.get(rel), baseline.files.get(rel) + lc, rc = local_changed(l, b), remote_changed(r, b) + too_big_local = l is not None and l.size > max_file_bytes + if r is not None and rel not in allowed: + result.skipped[rel] = r.etag + if baseline.skipped.get(rel) != r.etag: + result.newly_skipped[rel] = r.etag + if l is not None and lc and writable and not too_big_local: + # A local file at a path we never mirrored: keep both versions. + result.conflicts.append(rel) + elif b is not None and l is not None and not lc: + result.local_deletes.append(rel) + elif b is not None: + result.forget.append(rel) + continue + if not lc and not rc: + continue + if not writable: + if lc: + result.rejected.append(rel) + if r is not None: + result.downloads.append(rel) + elif l is not None: + result.local_deletes.append(rel) + else: + result.forget.append(rel) + continue + if too_big_local and lc: + result.skipped[rel] = "local" + if baseline.skipped.get(rel) != "local": + result.newly_skipped[rel] = "local" + continue + if lc and not rc: + if l is None: + result.remote_deletes[rel] = b.etag + else: + result.uploads[rel] = b.etag if b is not None else None + elif rc and not lc: + if r is None: + result.local_deletes.append(rel) + else: + result.downloads.append(rel) + elif r is None and l is None: + result.forget.append(rel) + elif r is None: + # Deleted remotely, edited here: the edit survives. + result.uploads[rel] = None + elif l is None: + # Deleted here, edited remotely: the remote edit survives. + result.downloads.append(rel) + elif b is None and l.md5 is not None and l.md5 == r.md5: + result.adopt[rel] = r.etag + else: + result.conflicts.append(rel) + + created_remote = remote_dirs - baseline.dirs + created_local = local_dirs - baseline.dirs + for rel in sorted(created_remote - local_dirs): + result.local_mkdirs.append(rel) + for rel in sorted(baseline.dirs): + in_r, in_l = rel in remote_dirs, rel in local_dirs + if in_r and not in_l: + if writable: + result.remote_rmdirs.append(rel) + else: + result.local_mkdirs.append(rel) + elif in_l and not in_r: + result.local_rmdirs.append(rel) + if writable: + for rel in sorted(created_local - remote_dirs): + result.remote_mkdirs.append(rel) + else: + for rel in sorted(created_local - remote_dirs): + result.local_rmdirs.append(rel) + result.local_rmdirs.sort(key=lambda p: (-p.count("/"), p)) + result.remote_rmdirs.sort(key=lambda p: (-p.count("/"), p)) + return result + + +def conflict_name(rel: str, stamp: str) -> str: + """``dir/report.xlsx`` -> ``dir/report (沙盒冲突 20261029-101500).xlsx``.""" + head, _, name = rel.rpartition("/") + stem, dot, ext = name.rpartition(".") + if not stem or not dot: + stem, ext, dot = name, "", "" + renamed = f"{stem} (沙盒冲突 {stamp}){dot}{ext}" + return f"{head}/{renamed}" if head else renamed diff --git a/src/backend/core/files/spaces/transport/sync/remote.py b/src/backend/core/files/spaces/transport/sync/remote.py new file mode 100644 index 00000000..63ef9192 --- /dev/null +++ b/src/backend/core/files/spaces/transport/sync/remote.py @@ -0,0 +1,139 @@ +"""Object storage side of a synchronized space: one grant prefix in an S3 bucket. + +Writes carry the ETag the sandbox copy was based on (``If-Match`` / +``If-None-Match``), so another writer's change is detected by the store +instead of being overwritten. Older S3-compatible stores (MinIO before 2024) +silently ignore those headers; a one-time probe detects that and the write +then re-checks the ETag immediately before storing, which narrows but does +not close the race. Deletes always use that re-check. +""" +from __future__ import annotations + +import os +import threading +import uuid + +from fastapi import HTTPException + +from core.files.spaces.access import identifier +from core.files.spaces.transport.sync.manifest import Remote +from core.files.storage.entries import validate_key +from core.files.storage.s3_directory import is_not_found, is_precondition_failed + +_CONDITIONAL: dict[tuple, bool] = {} +_CONDITIONAL_LOCK = threading.Lock() + + +class PreconditionFailed(Exception): + """The object changed since the sandbox copy was taken.""" + + +class RemoteSpace: + def __init__(self, storage, prefix: str): + if not hasattr(storage, "s3_client"): + raise RuntimeError("HUGAGENT_SPACE_MOUNT_MODE=sync requires S3-compatible storage") + self.storage = storage + self.client = storage.s3_client + self.bucket = storage.bucket + self.prefix = validate_key(prefix, directory=True) + + def key(self, rel: str, *, directory=False) -> str: + return self.prefix + validate_key(rel, directory=directory) + + def listing(self, *, limit: int) -> tuple[dict[str, Remote], set[str]]: + """Every object under the prefix in one flat pass (1000 keys per request). + + Directories are the marker objects plus the parents of every key, which + is what a delimited, per-directory walk would report.""" + files, dirs, count = {}, set(), 0 + pages = self.client.get_paginator("list_objects_v2").paginate( + Bucket=self.bucket, Prefix=self.prefix) + for page in pages: + for item in page.get("Contents", []): + rel = item["Key"][len(self.prefix):] + if not rel: + continue + count += 1 + if count > limit: + raise HTTPException(413, "Directory exceeds operation limit") + parts = rel.rstrip("/").split("/") + dirs.update("/".join(parts[:index]) for index in range(1, len(parts))) + if rel.endswith("/"): + dirs.add(rel.rstrip("/")) + else: + validate_key(rel) + modified = item["LastModified"].timestamp() if item.get("LastModified") else 0.0 + files[rel] = Remote(item.get("ETag") or "", int(item.get("Size") or 0), modified) + return files, dirs + + def download(self, rel: str, stream) -> str | None: + """Copy the object into ``stream``; return the ETag of exactly that content, + or None when it was deleted meanwhile (the next sync removes it).""" + try: + response = self.client.get_object(Bucket=self.bucket, Key=self.key(rel)) + except self.storage.ClientError as exc: + if is_not_found(exc): + return None + raise + body = response["Body"] + try: + for chunk in iter(lambda: body.read(1024 * 1024), b""): + stream.write(chunk) + finally: + body.close() + return response["ETag"] + + def conditional_writes(self) -> bool: + identity = (getattr(self.client.meta, "endpoint_url", ""), self.bucket) + with _CONDITIONAL_LOCK: + if identity not in _CONDITIONAL: + _CONDITIONAL[identity] = self._probe() + return _CONDITIONAL[identity] + + def _probe(self) -> bool: + environment = identifier(os.getenv("ENVIRONMENT", os.getenv("ENV", "dev"))) + key = f"{environment}/.hugagent-sync-probe/{uuid.uuid4().hex}" + self.client.put_object(Bucket=self.bucket, Key=key, Body=b"probe") + try: + self.client.put_object(Bucket=self.bucket, Key=key, Body=b"probe", IfNoneMatch="*") + return False + except self.storage.ClientError as exc: + if is_precondition_failed(exc): + return True + raise + finally: + self.client.delete_object(Bucket=self.bucket, Key=key) + + def upload(self, rel: str, stream, expected: str | None) -> str: + condition = {"IfNoneMatch": "*"} if expected is None else {"IfMatch": expected} + if not self.conditional_writes(): + current = self.storage.stat_object(self.key(rel)) + if (current.etag if current else None) != expected: + raise PreconditionFailed(rel) + condition = {} + try: + response = self.client.put_object(Bucket=self.bucket, Key=self.key(rel), Body=stream, + **condition) + except self.storage.ClientError as exc: + if is_precondition_failed(exc): + raise PreconditionFailed(rel) from exc + raise + return response["ETag"] + + def delete(self, rel: str, expected: str) -> None: + current = self.storage.stat_object(self.key(rel)) + if current is None: + return + if current.etag != expected: + raise PreconditionFailed(rel) + self.client.delete_object(Bucket=self.bucket, Key=self.key(rel)) + + def make_directory(self, rel: str) -> None: + self.storage.make_directory(self.key(rel, directory=True)) + + def remove_directory(self, rel: str) -> bool: + try: + self.storage.remove_directory(self.key(rel, directory=True)) + except OSError: + return False + return True diff --git a/src/backend/core/files/spaces/transport/sync/sandbox_io.py b/src/backend/core/files/spaces/transport/sync/sandbox_io.py new file mode 100644 index 00000000..ff43aa11 --- /dev/null +++ b/src/backend/core/files/spaces/transport/sync/sandbox_io.py @@ -0,0 +1,198 @@ +"""Sandbox side of a synchronized space: scan the copy and apply a plan. + +Both scripts run as root through execd. Inputs and outputs travel as files +under ``/tmp`` instead of argv/stdout, so a large tree is not limited by the +command line or the log stream. The baseline lives under +``/workspace/.hugagent-sync`` (root, 0700), outside every space copy, and +survives the workspace reset that precedes reuse by the same account. +""" +from __future__ import annotations + +import asyncio +import json +import shlex +import uuid +from datetime import timedelta + +from core.sandbox.workspace.identity import SANDBOX_RUN_GID, SANDBOX_RUN_UID + +STATE_DIR_NAME = ".hugagent-sync" +STATE_DIR = "/workspace/" + STATE_DIR_NAME + +_SCAN = r""" +import hashlib, json, os, stat, sys +spec = json.load(open(sys.argv[1])) +out = {} +def valid(rel): + return rel and "\\" not in rel and all(ord(c) >= 32 for c in rel) and \ + all(p not in ("", ".", "..") for p in rel.split("/")) +for item in spec["roots"]: + root, state = item["root"], item["state"] + try: + baseline = json.load(open(state)) + except (OSError, ValueError): + baseline = None + known = set((baseline or {}).get("files") or ()) + files, dirs = {}, [] + if os.path.isdir(root) and not os.path.islink(root): + for base, children, names in os.walk(root, followlinks=False): + rel_base = os.path.relpath(base, root) + rel_base = "" if rel_base == "." else rel_base + for name in list(children): + rel = rel_base + "/" + name if rel_base else name + path = os.path.join(base, name) + if os.path.islink(path) or not valid(rel): + children.remove(name) + else: + dirs.append(rel) + for name in names: + rel = rel_base + "/" + name if rel_base else name + info = os.lstat(os.path.join(base, name)) + if not stat.S_ISREG(info.st_mode) or not valid(rel): + continue + digest = None + if rel not in known and info.st_size <= spec["md5_limit"]: + h = hashlib.md5() + with open(os.path.join(base, name), "rb") as stream: + for chunk in iter(lambda: stream.read(1 << 20), b""): + h.update(chunk) + digest = h.hexdigest() + files[rel] = [info.st_size, info.st_mtime_ns, digest] + out[root] = {"files": files, "dirs": dirs, "baseline": baseline} +with open(sys.argv[2], "w") as stream: + json.dump(out, stream) +""" + +_APPLY = r""" +import json, os, shutil, sys +plan = json.load(open(sys.argv[1])) +root, writable = plan["root"], plan["writable"] +uid, gid = (plan["uid"], plan["gid"]) if writable else (0, 0) +baseline = plan["baseline"] +raced = [] + +def own(path, mode): + os.chown(path, uid, gid, follow_symlinks=False) + os.chmod(path, mode) + +def ensure_dir(path): + missing = [] + while path != root and not os.path.isdir(path): + missing.append(path) + path = os.path.dirname(path) + for item in reversed(missing): + if os.path.lexists(item): + os.unlink(item) + os.mkdir(item) + own(item, 0o755) + +def current(path): + try: + info = os.lstat(path) + except FileNotFoundError: + return None + return [info.st_size, info.st_mtime_ns] + +os.makedirs(root, exist_ok=True) +if os.path.islink(root): + raise SystemExit("space root must not be a symlink") +own(root, 0o755) +for rel in plan["mkdirs"]: + ensure_dir(os.path.join(root, rel)) +for item in plan["writes"]: + path = os.path.join(root, item["rel"]) + if current(path) != item["expect"]: + raced.append(item["rel"]) + os.unlink(item["staged"]) + continue + ensure_dir(os.path.dirname(path)) + if os.path.isdir(path) and not os.path.islink(path): + shutil.rmtree(path) + shutil.move(item["staged"], path) + own(path, 0o644) + info = os.lstat(path) + baseline["files"][item["rel"]] = {"etag": item["etag"], "size": info.st_size, + "mtime_ns": info.st_mtime_ns} +for item in plan["deletes"]: + path = os.path.join(root, item["rel"]) + state = current(path) + if state is not None and state != item["expect"]: + raced.append(item["rel"]) + continue + if state is not None: + os.unlink(path) + baseline["files"].pop(item["rel"], None) +for rel in plan["rmdirs"]: + try: + os.rmdir(os.path.join(root, rel)) + except OSError: + pass +os.makedirs(os.path.dirname(plan["state"]), mode=0o700, exist_ok=True) +temporary = plan["state"] + ".tmp" +with open(temporary, "w") as stream: + json.dump(baseline, stream) +os.chmod(temporary, 0o600) +os.replace(temporary, plan["state"]) +with open(sys.argv[2], "w") as stream: + json.dump({"raced": raced}, stream) +""" + + +async def _run(sandbox, script: str, payload: dict, *, timeout_s: int) -> dict: + from opensandbox.models.execd import RunCommandOpts + token = uuid.uuid4().hex + source, target = f"/tmp/.jx-space-{token}.in", f"/tmp/.jx-space-{token}.out" + await sandbox.files.write_file(source, json.dumps(payload).encode(), mode=600) + try: + result = await asyncio.wait_for(sandbox.commands.run( + "python3 -c " + shlex.quote(script) + " " + source + " " + target, + opts=RunCommandOpts(timeout=timedelta(seconds=timeout_s))), timeout=timeout_s + 10) + if getattr(result, "exit_code", None) != 0: + from core.sandbox.providers.opensandbox.runtime import _join_logs + stderr = _join_logs(getattr(getattr(result, "logs", None), "stderr", None) or []) + raise RuntimeError(stderr.strip()[-500:] or "space script failed") + raw = await sandbox.files.read_bytes(target) + return json.loads(raw if isinstance(raw, (bytes, str)) else bytes(raw)) + finally: + try: + await sandbox.commands.run("rm -f -- " + source + " " + target, + opts=RunCommandOpts(timeout=timedelta(seconds=10))) + except Exception: # noqa: BLE001 - /tmp is cleared on reset anyway + pass + + +async def scan(sandbox, roots: list[dict], *, md5_limit: int) -> dict: + return await _run(sandbox, _SCAN, {"roots": roots, "md5_limit": md5_limit}, timeout_s=120) + + +async def apply(sandbox, plan: dict) -> dict: + payload = dict(plan, uid=SANDBOX_RUN_UID, gid=SANDBOX_RUN_GID) + return await _run(sandbox, _APPLY, payload, timeout_s=300) + + +async def stage(sandbox, items: list[tuple[str, object]]) -> None: + """Upload downloaded objects to staging paths in as few requests as practical.""" + from opensandbox.models.filesystem import WriteEntry + batch, size = [], 0 + for path, stream in items: + stream.seek(0, 2) + length = stream.tell() + stream.seek(0) + batch.append(WriteEntry(path=path, data=stream, mode=600)) + size += length + if len(batch) >= 32 or size >= 32 * 1024 * 1024: + await sandbox.files.write_files(batch) + batch, size = [], 0 + if batch: + await sandbox.files.write_files(batch) + + +async def read_into(sandbox, path: str, stream) -> None: + chunks = await sandbox.files.read_bytes_stream(path, chunk_size=1024 * 1024) + try: + async for chunk in chunks: + stream.write(chunk) + finally: + close = getattr(chunks, "aclose", None) + if close is not None: + await close() diff --git a/src/backend/core/files/storage/s3_directory.py b/src/backend/core/files/storage/s3_directory.py index f61b80bf..15408636 100644 --- a/src/backend/core/files/storage/s3_directory.py +++ b/src/backend/core/files/storage/s3_directory.py @@ -1,6 +1,17 @@ """S3 directory operations shared by S3-compatible services.""" from core.files.storage.entries import DirectoryPage, StorageEntry, page_limit, validate_key +def error_code(exc) -> str: + return str(getattr(exc, "response", {}).get("Error", {}).get("Code")) + +def is_not_found(exc) -> bool: + return error_code(exc) in ("404", "NoSuchKey", "NotFound") + +def is_precondition_failed(exc) -> bool: + status = getattr(exc, "response", {}).get("ResponseMetadata", {}).get("HTTPStatusCode") + return error_code(exc) in ("412", "PreconditionFailed", "ConditionalRequestConflict") \ + or status in (409, 412) + class S3DirectoryOperations: def list_directory(self, prefix, *, cursor=None, limit=200): prefix = validate_key(prefix, directory=True) @@ -26,7 +37,7 @@ def stat_object(self, key): try: result = self.s3_client.head_object(Bucket=self.bucket, Key=key) except self.ClientError as exc: - if str(exc.response.get("Error", {}).get("Code")) in ("404", "NoSuchKey", "NotFound"): + if is_not_found(exc): return None raise return StorageEntry(key.rstrip("/").rsplit("/", 1)[-1], key, key.endswith("/"), diff --git a/src/backend/core/memory/access.py b/src/backend/core/memory/access.py index ebbaca5e..c14e750d 100644 --- a/src/backend/core/memory/access.py +++ b/src/backend/core/memory/access.py @@ -36,6 +36,13 @@ async def authorized_memory_context(memory_id: str, actor: str) -> Optional[Memo memory = await asyncio.to_thread(backend._get_memory) if memory is None: raise RuntimeError("memory store unavailable") + from core.memory.pgvector_store import is_pgvector + if is_pgvector(memory): + from uuid import UUID + try: + UUID(memory_id) + except (ValueError, TypeError, AttributeError): + return None try: item = await asyncio.to_thread(memory.get, memory_id) except IndexError: diff --git a/src/backend/core/memory/backend.py b/src/backend/core/memory/backend.py index e2d787d7..d819a04b 100644 --- a/src/backend/core/memory/backend.py +++ b/src/backend/core/memory/backend.py @@ -2,6 +2,7 @@ from __future__ import annotations import logging +import os import threading from dataclasses import dataclass, field from typing import Optional @@ -18,6 +19,11 @@ _MIGRATION_MAX_ROWS = 10_000 +# mem0 reads MEM0_TELEMETRY once, when first imported (only from this module). +# Its default reports usage to an external service and opens a second vector-store +# client with an extra "mem0migrations" collection/table. Operators may still opt in. +os.environ.setdefault("MEM0_TELEMETRY", "false") + def _patch_mem0_embedding() -> None: """Patch mem0 OpenAIEmbedding to remove dimensions param. @@ -120,6 +126,16 @@ def _build_mem0_config() -> dict: # exists. } + provider = settings.memory.vector_provider + if provider == "pgvector": + from core.db.engine import DATABASE_URL + from core.memory.pgvector_store import vector_config + config["vector_store"] = vector_config(DATABASE_URL, embed_dims) + # Product history and replay receipts live in PostgreSQL. mem0's private + # diagnostic history is not consumed by the product and is disposable. + config["history_db_path"] = ":memory:" + elif provider != "milvus": + raise ValueError("MEM0_VECTOR_PROVIDER must be milvus or pgvector") return config @@ -184,6 +200,8 @@ def _reconcile_vector_collection(cfg: dict) -> Optional[list]: Reconcile failures only warn — init proceeds exactly as before. """ + if cfg["vector_store"]["provider"] == "pgvector": + return None vs = cfg["vector_store"]["config"] client = None try: @@ -331,23 +349,19 @@ def _desktop_memory_epoch() -> str: def _close_failed_memory(instance) -> None: """Close clients owned by an unpublished mem0 instance, preserving stored data.""" - from mem0.memory.main import MEM0_TELEMETRY - - closers = [ - instance.vector_store.client.close, - instance.embedding_model.client.close, - instance.llm.client.close, - instance.close, - ] - if MEM0_TELEMETRY: - closers.append(instance._telemetry_vector_store.client.close) + from core.memory.pgvector_store import close_vector_store + + closers = [lambda: close_vector_store(instance.vector_store), + instance.embedding_model.client.close, instance.llm.client.close, + instance.close] + telemetry = getattr(instance, "_telemetry_vector_store", None) + if telemetry is not None: + closers.append(lambda: close_vector_store(telemetry)) for close in closers: try: close() except Exception as exc: - logger.warning( - "[MemoryService] failed initialization cleanup error_type=%s", type(exc).__name__ - ) + logger.warning("[MemoryService] cleanup error_type=%s", type(exc).__name__) def _get_memory() -> Optional[object]: @@ -383,7 +397,11 @@ def _get_memory() -> Optional[object]: ) instance = Memory.from_config(cfg) # Existing collections remain released after a local database restart. - instance.vector_store.client.load_collection(collection_name=MEMORY_COLLECTION_NAME) + if cfg["vector_store"]["provider"] == "pgvector": + from core.memory.pgvector_store import prepare_collection + prepare_collection(instance.vector_store) + else: + instance.vector_store.client.load_collection(collection_name=MEMORY_COLLECTION_NAME) from core.harness.registry.model_credentials import is_reference, sync_client_kwargs from openai import OpenAI diff --git a/src/backend/core/memory/pgvector_store.py b/src/backend/core/memory/pgvector_store.py new file mode 100644 index 00000000..399d3d7c --- /dev/null +++ b/src/backend/core/memory/pgvector_store.py @@ -0,0 +1,106 @@ +"""Product-specific PostgreSQL operations around mem0's pgvector adapter. + +CRUD and similarity search remain in mem0. These operations enforce schema +identity and query durable effect receipts without a capped result scan. +""" +from __future__ import annotations + +from sqlalchemy.engine import make_url + +COLLECTION = "hugagent_memories" + + +def vector_config(database_url: str, dimensions: int) -> dict: + url = make_url(database_url) + if url.get_backend_name() != "postgresql": + raise ValueError("pgvector memory requires the platform PostgreSQL database") + if not 1 <= dimensions <= 16000: + raise ValueError("Memory vector dimension must be between 1 and 16000") + # psycopg expects a libpq URI, not SQLAlchemy's driver suffix. + uri = url.set(drivername="postgresql").update_query_dict({"connect_timeout": "10"}).render_as_string(hide_password=False) + return {"provider": "pgvector", "config": { + "connection_string": uri, "collection_name": COLLECTION, + "embedding_model_dims": dimensions, "minconn": 1, "maxconn": 10, + # Exact cosine search supports model widths beyond ANN index limits. + "hnsw": False, "diskann": False, + }} + + +def is_pgvector(memory) -> bool: + config = getattr(getattr(memory, "config", None), "vector_store", None) + return getattr(config, "provider", None) == "pgvector" + + +def prepare_collection(store) -> None: + """Create only a table; never install extensions or replace existing vectors.""" + if store.collection_name != COLLECTION: + raise ValueError("Unexpected memory collection") + dimensions = int(store.embedding_model_dims) + if not 1 <= dimensions <= 16000: + raise ValueError("Invalid memory vector dimension") + with store._get_cursor(commit=True) as cursor: + cursor.execute("SELECT pg_advisory_xact_lock(hashtext('hugagent.memory.schema'))") + cursor.execute("SELECT 1 FROM pg_extension WHERE extname = 'vector'") + if cursor.fetchone() is None: + raise RuntimeError("The platform database must have the vector extension installed") + cursor.execute( + "CREATE TABLE IF NOT EXISTS hugagent_memories " + f"(id UUID PRIMARY KEY, vector vector({dimensions}), payload JSONB)" + ) + cursor.execute( + "SELECT format_type(atttypid, atttypmod) FROM pg_attribute " + "WHERE attrelid = 'hugagent_memories'::regclass " + "AND attname = 'vector' AND NOT attisdropped" + ) + row = cursor.fetchone() + if row is None or row[0] != f"vector({dimensions})": + raise RuntimeError("Memory embedding dimension differs from stored vectors; existing data retained") + cursor.execute("CREATE INDEX IF NOT EXISTS hugagent_memories_payload_idx " + "ON hugagent_memories USING gin(payload)") + # mem0 and the product queries filter with payload->>'key' = value, which + # the GIN index cannot serve; without these every search scans all users. + cursor.execute("CREATE INDEX IF NOT EXISTS hugagent_memories_user_idx " + "ON hugagent_memories ((payload->>'user_id'))") + cursor.execute("CREATE INDEX IF NOT EXISTS hugagent_memories_expiry_idx " + "ON hugagent_memories ((payload->>'expiration_date'), id) " + "WHERE payload->>'layer' = 'L2'") + store._collection_ensured = True + + +def receipt_rows(store, user_id: str, workspace_id: str | None, effect_id: str) -> list[dict]: + params = [user_id, effect_id, effect_id] + query = ("SELECT id, payload FROM hugagent_memories WHERE payload->>'user_id' = %s " + "AND (payload->>'outbox_effect_id' = %s " + "OR COALESCE(payload->'outbox_effect_ids', '[]'::jsonb) ? %s)") + if workspace_id: + query += " AND payload->>'workspace_id' = %s" + params.append(workspace_id) + query += " ORDER BY id LIMIT 5" + with store._get_cursor() as cursor: + cursor.execute(query, params) + return [{"id": str(row[0]), "metadata": row[1]} for row in cursor.fetchall()] + + +def expiry_candidates(store, scan_limit: int, today: str) -> list[dict]: + if scan_limit < 1: + return [] + with store._get_cursor() as cursor: + # Expiry is checked again by the common Python policy. SQL does not cast + # untrusted metadata to dates; malformed dates must never trigger deletion. + cursor.execute( + "SELECT id, payload FROM hugagent_memories WHERE payload->>'layer' = 'L2' " + "AND (payload->>'expiration_date' IS NULL OR payload->>'expiration_date' <= %s " + "OR payload->>'expiration_date' !~ '^\\d{4}-\\d{2}-\\d{2}$') " + "ORDER BY payload->>'expiration_date' NULLS LAST, id LIMIT %s", (today, scan_limit)) + return [{"id": str(row[0]), "metadata": row[1]} for row in cursor.fetchall()] + + +def close_vector_store(store) -> None: + pool = getattr(store, "connection_pool", None) + if pool is not None: + close = getattr(pool, "close", None) or pool.closeall + close() + else: + client = getattr(store, "client", None) + if client is not None: + client.close() diff --git a/src/backend/core/memory/procedures.py b/src/backend/core/memory/procedures.py index d3bd5705..3cbe73d8 100644 --- a/src/backend/core/memory/procedures.py +++ b/src/backend/core/memory/procedures.py @@ -112,6 +112,21 @@ async def find_procedure_by_effect_id( # directly, with Strong consistency, so the result is exact and observes a # receipt committed immediately before a crash. vector_store = getattr(memory, "vector_store", None) + from core.memory.pgvector_store import is_pgvector, receipt_rows + if is_pgvector(memory): + try: + rows = await run_effect(receipt_rows, vector_store, + ctx.effective_scope_user_id, ctx.workspace_id, effect_id) + except Exception as exc: + if strict: + raise RuntimeError("procedure idempotency lookup failed") from exc + logger.debug("[MemoryService] exact PG effect lookup failed: %s", type(exc).__name__) + return None + if not rows: + return None + item = rows[0] + meta = item["metadata"] or {} + return {"id": item["id"], "memory": meta.get("data") or "", "metadata": meta} client = getattr(vector_store, "client", None) collection_name = getattr(vector_store, "collection_name", None) if client is not None and collection_name: diff --git a/src/backend/core/memory/ttl_sweeper.py b/src/backend/core/memory/ttl_sweeper.py index ce70b143..d42315ac 100644 --- a/src/backend/core/memory/ttl_sweeper.py +++ b/src/backend/core/memory/ttl_sweeper.py @@ -90,15 +90,20 @@ async def sweep_expired_memories(scan_limit: int = DEFAULT_SCAN_LIMIT) -> dict: # whole mem0 payload. Scoped to L2 — L1 lives in the DB profile and TASK in # chat metadata; neither is swept here. try: - rows = await loop.run_in_executor( - None, - lambda: memory.vector_store.client.query( - collection_name=memory.vector_store.collection_name, - filter='metadata["layer"] == "L2"', - output_fields=["id", "metadata"], - limit=scan_limit, - ), - ) + from core.memory.pgvector_store import is_pgvector, expiry_candidates + if is_pgvector(memory): + rows = await loop.run_in_executor( + None, lambda: expiry_candidates(memory.vector_store, scan_limit, utc_now().date().isoformat())) + else: + rows = await loop.run_in_executor( + None, + lambda: memory.vector_store.client.query( + collection_name=memory.vector_store.collection_name, + filter='metadata["layer"] == "L2"', + output_fields=["id", "metadata"], + limit=scan_limit, + ), + ) except Exception as exc: logger.warning("[memory-ttl] candidate query failed: %s", exc) return stats diff --git a/src/backend/core/ontology/__init__.py b/src/backend/core/ontology/__init__.py index b3cc274c..685ef522 100644 --- a/src/backend/core/ontology/__init__.py +++ b/src/backend/core/ontology/__init__.py @@ -1,7 +1,5 @@ -"""Ontology harness: Domain Pack validation, runtime selection, and gates.""" +"""Community edition has no domain ontology governance. -from core.ontology.schemas import OntologyPackDocument -from core.ontology.validator import DomainPackValidator -from core.ontology.validation_models import OntologyGateDecision - -__all__ = ["DomainPackValidator", "OntologyGateDecision", "OntologyPackDocument"] +The shared runtime still imports these module names; every function here is +the "governance off" answer, so no pack, review, gate or prompt section exists. +""" diff --git a/src/backend/core/ontology/audit_events.py b/src/backend/core/ontology/audit_events.py deleted file mode 100644 index 284501f4..00000000 --- a/src/backend/core/ontology/audit_events.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Persist ontology enforcement, activation and review events.""" -from __future__ import annotations -from core.infra.time import utc_now -import uuid -from typing import Any -from core.db.repository import OntologyRepository - -def record_enforcement_event(data: dict[str, Any]) -> None: - """Persist an event from middleware/reviewer using an isolated session.""" - from core.db.engine import SessionLocal - - db = SessionLocal() - try: - payload = dict(data) - payload.setdefault("event_id", f"ontoe_{uuid.uuid4().hex[:16]}") - OntologyRepository(db).create_event(payload) - except Exception: - db.rollback() - raise - finally: - db.close() - - -def record_runtime_activation( - event: dict[str, Any], - runtime: dict[str, Any], - *, - user_id: str | None, - chat_id: str | None, -) -> None: - """Persist one asset-driven workflow activation as append-only evidence.""" - - version_id = next( - ( - pack.get("version_id") - for pack in runtime.get("packs", []) - if pack.get("pack_id") == event.get("pack_id") - ), - None, - ) - record_enforcement_event( - { - "user_id": user_id, - "chat_id": chat_id, - "pack_id": event.get("pack_id"), - "version_id": version_id, - "rule_id": f"activation:{event.get('workflow_id')}", - "stage": "checkpoint", - "event_type": "runtime_activation", - "decision": "pass", - "mode": "enforce", - "target": f"{event.get('asset_kind')}:{event.get('asset_id')}", - "details": event, - } - ) - - -def record_review_run(data: dict[str, Any]) -> None: - from core.db.engine import SessionLocal - - db = SessionLocal() - try: - payload = dict(data) - payload.setdefault("review_id", f"ontor_{uuid.uuid4().hex[:16]}") - payload.setdefault("created_at", utc_now()) - OntologyRepository(db).create_review(payload) - except Exception: - db.rollback() - raise - finally: - db.close() diff --git a/src/backend/core/ontology/build_validator.py b/src/backend/core/ontology/build_validator.py index 0b553776..2c137648 100644 --- a/src/backend/core/ontology/build_validator.py +++ b/src/backend/core/ontology/build_validator.py @@ -1,446 +1,17 @@ -"""Ontology-assisted validation for skills, tools, and sub-agent definitions.""" +"""Capability builds are never checked against Domain Packs.""" from __future__ import annotations -from dataclasses import asdict, dataclass, field -from typing import Any, Literal - -from core.config.display_names import MCP_SERVER_DISPLAY_NAMES, TOOL_DISPLAY_NAMES -from core.db.models import AdminMcpServer, AdminSkill, InstalledPlugin -from core.infra.exceptions import BadRequestError -from core.ontology.schemas import OntologyPackDocument -from core.ontology.runtime_pack import activate_runtime_for_asset -from sqlalchemy.orm import Session - - -@dataclass(slots=True) -class BuildIssue: - severity: str - code: str - message: str - workflow_id: str | None = None - details: dict[str, Any] = field(default_factory=dict) +from dataclasses import dataclass, field +from typing import Any @dataclass(slots=True) class BuildValidationReport: valid: bool - matched_workflows: list[str] = field(default_factory=list) - resolved_tools: list[str] = field(default_factory=list) - resolved_tool_details: list[dict[str, str]] = field(default_factory=list) - errors: list[BuildIssue] = field(default_factory=list) - warnings: list[BuildIssue] = field(default_factory=list) - suggestions: list[str] = field(default_factory=list) - - def as_dict(self) -> dict[str, Any]: - return { - "valid": self.valid, - "matched_workflows": self.matched_workflows, - "resolved_tools": self.resolved_tools, - "resolved_tool_details": self.resolved_tool_details, - "errors": [asdict(item) for item in self.errors], - "warnings": [asdict(item) for item in self.warnings], - "suggestions": self.suggestions, - } - - -class OntologyBuildValidator: - """Validate an agent asset against active default Domain Packs.""" - - def __init__(self, db: Session): - self.db = db - - def validate( - self, - *, - asset_type: Literal["skill", "tool", "subagent"], - name: str, - description: str = "", - instructions: str = "", - tool_names: list[str] | None = None, - mcp_server_ids: list[str] | None = None, - skill_ids: list[str] | None = None, - plugin_ids: list[str] | None = None, - ontology_tags: list[str] | None = None, - output_schema: dict[str, Any] | None = None, - tool_schemas: dict[str, dict[str, Any]] | None = None, - ) -> BuildValidationReport: - from core.ontology.service import OntologyService - - task = "\n".join(item for item in (name, description, instructions[:4000]) if item) - runtime = OntologyService(self.db).build_runtime(task=task) - resolved_tools = set(tool_names or []) - resolved_mcp_ids = set(mcp_server_ids or []) - resolved_skill_ids = set(skill_ids or []) - if plugin_ids: - plugin_rows = ( - self.db.query(InstalledPlugin) - .filter(InstalledPlugin.install_id.in_(plugin_ids)) - .all() - ) - for plugin in plugin_rows: - components = plugin.component_ids or {} - resolved_mcp_ids.update(components.get("mcp") or []) - resolved_skill_ids.update(components.get("skills") or []) - if resolved_skill_ids: - skill_rows = ( - self.db.query(AdminSkill).filter(AdminSkill.skill_id.in_(resolved_skill_ids)).all() - ) - for row in skill_rows: - resolved_tools.update(str(item) for item in row.allowed_tools or [] if item) - if resolved_mcp_ids: - rows = ( - self.db.query(AdminMcpServer) - .filter(AdminMcpServer.server_id.in_(resolved_mcp_ids)) - .all() - ) - for row in rows: - resolved_tools.update( - str(item["name"]) - for item in row.tools_json or [] - if isinstance(item, dict) and item.get("name") - ) - - errors: list[BuildIssue] = [] - warnings: list[BuildIssue] = [] - matched: list[str] = [] - suggestions: list[str] = [] - declared_ontology_tags = { - tag.split(":", 1)[1].strip() - for tag in (ontology_tags or []) - if isinstance(tag, str) and tag.startswith("ontology:") and tag.split(":", 1)[1].strip() - } - if declared_ontology_tags: - controlled_ids: set[str] = set() - for version in OntologyService(self.db).repo.get_active_versions(): - try: - document = OntologyPackDocument.model_validate(version.content) - except Exception: - continue - controlled_ids.update(concept.id for concept in document.concepts) - unknown_tags = sorted(declared_ontology_tags - controlled_ids) - if controlled_ids and unknown_tags: - errors.append( - BuildIssue( - "error", - "unknown_ontology_tags", - "资产声明了不在激活领域词表中的本体标签: " + ", ".join(unknown_tags), - ) - ) - suggestions.append( - "改用受控本体标签: " - + ", ".join(f"ontology:{item}" for item in sorted(controlled_ids)[:20]) - ) - if asset_type == "tool": - for tool_name in resolved_tools: - activate_runtime_for_asset( - runtime, - kind="tool", - asset_id=tool_name, - tags=ontology_tags or [], - ) - else: - activate_runtime_for_asset( - runtime, - kind=asset_type, - asset_id=name, - tags=ontology_tags or [], - ) - for pack in runtime.get("packs", []): - for workflow in pack.get("workflows", []): - workflow_id = workflow.get("id") - if workflow_id: - matched.append(f"{pack.get('pack_id')}:{workflow_id}") - missing = sorted(set(workflow.get("required_tools", [])) - resolved_tools) - forbidden = sorted(set(workflow.get("forbidden_tools", [])) & resolved_tools) - # A skill/subagent must carry the complete workflow. A tool - # asset is only one building block, so requiring every sibling - # tool on the same MCP server would reject valid modular tools. - if missing and asset_type != "tool": - guidance = self._mcp_binding_guidance(missing) - missing_names = "、".join( - item["display_name"] for item in guidance["missing_tools"] - ) - recommended_names = "、".join( - f"“{item['display_name']}”" for item in guidance["recommended_mcp_servers"] - ) - asset_label = {"skill": "技能", "subagent": "子智能体"}[asset_type] - message = f"{asset_label}缺少领域工作流要求的能力:{missing_names}。" - if recommended_names: - message += f"请在“绑定工具 (MCP)”中选择 {recommended_names}。" - else: - message += "当前未找到可绑定的 MCP,请联系管理员配置对应能力。" - errors.append( - BuildIssue( - "error", - "missing_required_tools", - message, - workflow_id, - guidance, - ) - ) - for server in guidance["recommended_mcp_servers"]: - provided = "、".join( - item["display_name"] for item in server["provided_tools"] - ) - suggestions.append( - f"绑定 MCP“{server['display_name']}”(提供:{provided})" - ) - if guidance["unmapped_tools"]: - suggestions.append( - "请联系管理员配置可提供以下能力的 MCP:" - + "、".join(item["display_name"] for item in guidance["unmapped_tools"]) - ) - if forbidden: - errors.append( - BuildIssue( - "error", - "forbidden_tools_bound", - f"{asset_type} 绑定了领域工作流禁止的工具: {', '.join(forbidden)}", - workflow_id, - ) - ) - suggestions.append("移除禁止工具: " + ", ".join(forbidden)) - - output_rules = [ - rule - for rule in pack.get("constraints", []) - if (rule.get("target") or {}).get("kind") == "output" - ] - if output_rules and output_schema is None: - warnings.append( - BuildIssue( - "warning", - "missing_output_contract", - "资产命中输出约束,但未声明结构化 output_schema;运行时仍会校验文本输出。", - ) - ) - - if asset_type == "tool": - self._validate_tool_contracts( - resolved_tools=resolved_tools, - tool_schemas=tool_schemas or {}, - errors=errors, - warnings=warnings, - matched=matched, - suggestions=suggestions, - ) - - return BuildValidationReport( - valid=not errors, - matched_workflows=matched, - resolved_tools=sorted(resolved_tools), - resolved_tool_details=[ - self._tool_detail(tool_name) for tool_name in sorted(resolved_tools) - ], - errors=errors, - warnings=warnings, - suggestions=sorted(set(suggestions)), - ) - - @staticmethod - def _tool_detail(tool_name: str) -> dict[str, str]: - return { - "name": tool_name, - "display_name": TOOL_DISPLAY_NAMES.get(tool_name, tool_name), - } - - @staticmethod - def _mcp_display_name(server: AdminMcpServer) -> str: - configured = str(server.display_name or "").strip() - fallback = MCP_SERVER_DISPLAY_NAMES.get(server.server_id, server.server_id) - return configured if configured and configured != server.server_id else fallback - - def _mcp_binding_guidance(self, missing_tools: list[str]) -> dict[str, Any]: - """Map low-level required tools to bindable global MCP services. - - Suggestions intentionally exclude private and plugin-owned MCP rows: those are not - universally visible in the direct MCP selector, and exposing them could leak another - user's private capability names. - """ - missing = set(missing_tools) - candidates: dict[str, dict[str, Any]] = {} - tool_servers: dict[str, list[dict[str, str]]] = {name: [] for name in missing} - rows = ( - self.db.query(AdminMcpServer) - .filter( - AdminMcpServer.owner_user_id.is_(None), - AdminMcpServer.source_plugin.is_(None), - AdminMcpServer.is_enabled.is_(True), - ) - .all() - ) - for server in rows: - provided = { - str(item.get("name") or "").strip() - for item in (server.tools_json or []) - if isinstance(item, dict) and str(item.get("name") or "").strip() - } - covered = provided & missing - if not covered: - continue - server_info = { - "server_id": server.server_id, - "display_name": self._mcp_display_name(server), - } - for tool_name in covered: - tool_servers[tool_name].append(server_info) - candidates[server.server_id] = { - **server_info, - "tool_names": covered, - } - - # Recommend the smallest practical set of MCPs using a deterministic greedy cover. - uncovered = set(missing) - recommended: list[dict[str, Any]] = [] - remaining = dict(candidates) - while uncovered and remaining: - best = max( - remaining.values(), - key=lambda item: ( - len(item["tool_names"] & uncovered), - item["display_name"], - item["server_id"], - ), - ) - covered = best["tool_names"] & uncovered - if not covered: - break - recommended.append( - { - "server_id": best["server_id"], - "display_name": best["display_name"], - "provided_tools": [ - self._tool_detail(tool_name) for tool_name in sorted(covered) - ], - } - ) - uncovered -= covered - remaining.pop(best["server_id"], None) - - missing_details = [] - for tool_name in sorted(missing): - detail: dict[str, Any] = self._tool_detail(tool_name) - detail["mcp_servers"] = sorted( - tool_servers[tool_name], - key=lambda item: (item["display_name"], item["server_id"]), - ) - missing_details.append(detail) - return { - "missing_tools": missing_details, - "recommended_mcp_servers": recommended, - "unmapped_tools": [self._tool_detail(tool_name) for tool_name in sorted(uncovered)], - } - - def _validate_tool_contracts( - self, - *, - resolved_tools: set[str], - tool_schemas: dict[str, dict[str, Any]], - errors: list[BuildIssue], - warnings: list[BuildIssue], - matched: list[str], - suggestions: list[str], - ) -> None: - """Check discovered tool inputs directly against active pack rules.""" - from core.ontology.service import OntologyService - - service = OntologyService(self.db) - for version in service.repo.get_active_versions(): - try: - document = OntologyPackDocument.model_validate(version.content) - except Exception: - continue - for workflow in document.workflows: - if resolved_tools & set(workflow.required_tools + workflow.forbidden_tools): - key = f"{document.pack_id}:{workflow.id}" - if key not in matched: - matched.append(key) - for rule in document.constraints: - target = rule.target - tool_name = target.tool - if not rule.enabled or not tool_name or tool_name not in resolved_tools: - continue - schema = tool_schemas.get(tool_name) or {} - properties = self._schema_properties(schema) - if not schema: - warnings.append( - BuildIssue( - "warning", - "missing_tool_input_schema", - f"工具 {tool_name} 命中本体规则 {rule.id},但没有可校验的 inputSchema。", - rule.id, - ) - ) - continue - required_parameters: set[str] = set() - if target.kind == "tool_parameter" and target.parameter: - required_parameters.add(target.parameter) - elif target.kind == "tool": - required_parameters.update( - str(item) for item in rule.schema_.get("required", []) - ) - required_parameters.update( - str(item) for item in rule.schema_.get("properties", {}).keys() - ) - missing_parameters = sorted(required_parameters - set(properties)) - if missing_parameters: - errors.append( - BuildIssue( - "error", - "missing_ontology_parameters", - f"工具 {tool_name} 的 inputSchema 缺少本体规则要求的参数: " - + ", ".join(missing_parameters), - rule.id, - ) - ) - suggestions.append(f"为 {tool_name} 补充参数: " + ", ".join(missing_parameters)) - - @staticmethod - def _schema_properties(schema: dict[str, Any]) -> dict[str, Any]: - function = schema.get("function") if isinstance(schema, dict) else None - if isinstance(function, dict): - schema = function.get("parameters") or function.get("inputSchema") or {} - else: - schema = schema.get("inputSchema") or schema.get("parameters") or schema - properties = schema.get("properties") if isinstance(schema, dict) else None - return properties if isinstance(properties, dict) else {} + errors: list[Any] = field(default_factory=list) + warnings: list[Any] = field(default_factory=list) -def ensure_ontology_build_valid( - db: Session, - *, - asset_type: Literal["skill", "tool", "subagent"], - name: str, - description: str = "", - instructions: str = "", - tool_names: list[str] | None = None, - mcp_server_ids: list[str] | None = None, - skill_ids: list[str] | None = None, - plugin_ids: list[str] | None = None, - ontology_tags: list[str] | None = None, - output_schema: dict[str, Any] | None = None, - tool_schemas: dict[str, dict[str, Any]] | None = None, - message: str | None = None, -) -> BuildValidationReport: - """Run the common reconstruction-layer gate and reject invalid assets.""" - report = OntologyBuildValidator(db).validate( - asset_type=asset_type, - name=name, - description=description, - instructions=instructions, - tool_names=tool_names, - mcp_server_ids=mcp_server_ids, - skill_ids=skill_ids, - plugin_ids=plugin_ids, - ontology_tags=ontology_tags, - output_schema=output_schema, - tool_schemas=tool_schemas, - ) - if not report.valid: - labels = {"skill": "技能", "tool": "工具", "subagent": "子智能体"} - raise BadRequestError( - message=message or f"{labels[asset_type]}未通过本体构建校验", - data=report.as_dict(), - ) - return report +def ensure_ontology_build_valid(*_args: Any, **_kwargs: Any) -> BuildValidationReport: + return BuildValidationReport(valid=True) diff --git a/src/backend/core/ontology/evolution.py b/src/backend/core/ontology/evolution.py deleted file mode 100644 index d438370c..00000000 --- a/src/backend/core/ontology/evolution.py +++ /dev/null @@ -1,508 +0,0 @@ -"""Human-governed ontology evolution from runtime evidence. - -The service only creates candidates and draft pack versions. It never changes -an active-version pointer; activation remains an explicit administrator action. -""" - -from __future__ import annotations - -import copy -import json -import logging -import re -import uuid -from collections import defaultdict -from dataclasses import dataclass, field -from threading import Lock -from typing import Any - -from core.db.models import ChatMessage, OntologyEnforcementEvent -from core.infra.background import spawn -from core.infra.exceptions import BadRequestError, ResourceNotFoundError -from core.memory.sanitizer import sanitize -from core.ontology.service import OntologyService -from sqlalchemy.orm import Session - -_SECTIONS = {"term": "concepts", "relation": "relations", "constraint": "constraints"} -logger = logging.getLogger(__name__) - - -def schedule_ontology_evolution(*, user_id: str) -> bool: - """Debounce the low-priority evidence prefilter on the current event loop.""" - with process_state().SCHEDULE_LOCK: - if process_state().EVOLUTION_RUNNING: - return False - process_state().EVOLUTION_RUNNING = True - - async def _run() -> None: - try: - await generate_candidates(min_occurrences=3, limit=500, user_id=user_id or "system") - except Exception: # noqa: BLE001 - a background learner must never affect delivery - logger.warning("ontology evolution background pass failed", exc_info=True) - finally: - with process_state().SCHEDULE_LOCK: - process_state().EVOLUTION_RUNNING = False - - spawn(_run(), name="ontology-evolution-prefilter") - return True - - -async def generate_candidates( - *, - min_occurrences: int = 3, - limit: int = 500, - user_id: str = "system", - model_name: str | None = None, -) -> dict[str, Any]: - """Turn recurring gate evidence into pending Domain Pack drafts. - - The evidence is read and the drafts are written in two separate sessions; - the model call that proposes them in between holds no connection. - """ - from core.db.threads import run_in_session - - recurring, summaries = await run_in_session( - lambda db: OntologyEvolutionService(db).evidence_summaries( - min_occurrences=min_occurrences, limit=limit - ) - ) - if not summaries: - return {"created": 0, "prefiltered_groups": recurring, "items": []} - proposals = await OntologyEvolutionService._llm_proposals( - summaries, user_id=user_id, model_name=model_name - ) - created = await run_in_session( - lambda db: OntologyEvolutionService(db).record_drafts(summaries, proposals) - ) - return {"created": len(created), "prefiltered_groups": recurring, "items": created} - - -def _parse_json_array_lenient(text: str) -> list[Any] | None: - """Parse a JSON array without coupling the service layer to orchestration.""" - if not text: - return None - candidates = [text.strip()] - fenced = re.search(r"```(?:json)?\s*\n?(.*?)\n?```", text, re.DOTALL) - if fenced: - candidates.append(fenced.group(1).strip()) - start, end = text.find("["), text.rfind("]") - if start != -1 and end > start: - candidates.append(text[start : end + 1]) - for candidate in candidates: - try: - parsed = json.loads(candidate) - except (json.JSONDecodeError, ValueError): - continue - if isinstance(parsed, list): - return parsed - return None - - -class OntologyEvolutionService: - def __init__(self, db: Session): - self.db = db - self.ontology = OntologyService(db) - - def evidence_summaries( - self, *, min_occurrences: int, limit: int - ) -> tuple[int, list[dict[str, Any]]]: - """Recurring gate decisions grouped by rule, with sanitized conversation samples. - - Returns ``(recurring group count, sanitized summaries)``. - """ - rows = ( - self.db.query(OntologyEnforcementEvent) - .filter( - OntologyEnforcementEvent.pack_id.isnot(None), - OntologyEnforcementEvent.decision.in_(("deny", "revise", "escalate")), - ) - .order_by(OntologyEnforcementEvent.created_at.desc()) - .limit(limit) - .all() - ) - groups: dict[tuple[str, str, str], list[Any]] = defaultdict(list) - for row in rows: - key = (row.pack_id or "", row.rule_id or "", row.target or "") - groups[key].append(row) - recurring = [ - (key, items) - for key, items in groups.items() - if key[0] and len(items) >= min_occurrences - ] - if not recurring: - return 0, [] - - chat_ids = list( - dict.fromkeys(row.chat_id for _, items in recurring for row in items if row.chat_id) - )[:50] - messages_by_chat: dict[str, list[dict[str, str]]] = defaultdict(list) - if chat_ids: - message_rows = ( - self.db.query(ChatMessage) - .filter(ChatMessage.chat_id.in_(chat_ids)) - .order_by(ChatMessage.chat_seq.desc()) - .limit(300) - .all() - ) - for message in message_rows: - bucket = messages_by_chat[str(message.chat_id)] - if len(bucket) < 6 and message.role in {"user", "assistant"}: - bucket.append( - { - "role": str(message.role), - "content": str(message.content or "")[:500], - } - ) - - summaries = [ - { - "pack_id": key[0], - "rule_id": key[1], - "target": key[2], - "occurrences": len(items), - "event_ids": [item.event_id for item in items[:20]], - "samples": [(item.details or {}).get("violations", []) for item in items[:3]], - "conversation_samples": [ - {"chat_id": chat_id, "messages": list(reversed(messages_by_chat[chat_id]))} - for chat_id in dict.fromkeys( - str(item.chat_id) for item in items if item.chat_id - ) - if messages_by_chat.get(chat_id) - ][:3], - } - for key, items in recurring - ] - sanitized_summaries = [ - item - for summary in summaries - if isinstance((item := self._sanitize_json_value(summary)), dict) - ] - return len(recurring), sanitized_summaries - - def record_drafts( - self, summaries: list[dict[str, Any]], proposals: list[dict[str, Any]] - ) -> list[str]: - """Store each summary's sanitized proposals (or a rule review) as pending drafts.""" - created = [] - for summary in summaries: - matching = [ - item - for item in proposals - if item.get("pack_id") == summary["pack_id"] - and item.get("source_rule_id", "") == summary["rule_id"] - ] - if not matching: - matching = [ - { - "pack_id": summary["pack_id"], - "source_rule_id": summary["rule_id"], - "candidate_type": "false_positive", - "proposal": { - "operation": "review_rule", - "rule_id": summary["rule_id"], - "reason": "同一门禁规则在近期重复触发,需人工判断规则过严或任务策略有误。", - }, - "evidence": [ - f"{summary['occurrences']} 次重复触发,目标 {summary['target']}" - ], - } - ] - for proposal in matching: - sanitized = self._sanitize_proposal(proposal, summary) - if sanitized is None or self._already_drafted(summary["event_ids"]): - continue - row = self.ontology.repo.create_draft( - { - "draft_id": f"ontod_{uuid.uuid4().hex[:16]}", - "pack_id": sanitized["pack_id"], - "source_type": "enforcement", - "candidate_type": sanitized["candidate_type"], - "proposal": sanitized["proposal"], - "evidence": sanitized["evidence"], - "source_event_ids": summary["event_ids"], - "value_score": min(100, 40 + summary["occurrences"] * 10), - "review_status": "pending", - } - ) - created.append(row.draft_id) - return created - - def ingest_user_correction( - self, - *, - user_id: str, - chat_id: str, - message_id: str, - feedback_id: str, - comment: str, - ) -> list[str]: - """Turn explicit, domain-matched user correction into human-review drafts. - - The user's text is evidence, not an ontology mutation instruction. It - therefore enters the queue as a diagnostic candidate and can never be - materialized or activated automatically. - """ - from core.db.models import ChatMessage - from core.ontology.runtime_policy import build_user_ontology_runtime - - assistant = ( - self.db.query(ChatMessage) - .filter( - ChatMessage.message_id == message_id, - ChatMessage.chat_id == chat_id, - ChatMessage.role == "assistant", - ) - .first() - ) - if assistant is None: - return [] - prompt_row = ( - self.db.query(ChatMessage) - .filter( - ChatMessage.chat_id == chat_id, - ChatMessage.role == "user", - ChatMessage.chat_seq <= assistant.chat_seq, - ) - .order_by(ChatMessage.chat_seq.desc()) - .first() - ) - task = prompt_row.content if prompt_row else comment - opted_in, runtime = build_user_ontology_runtime( - user_id=user_id, - task=task, - db=self.db, - ) - if not opted_in: - return [] - source_id = f"feedback:{feedback_id}" - if self._already_drafted([source_id]): - return [] - evidence_texts: list[str] = [] - for text in ( - f"用户纠正:{comment[:500]}", - f"原任务:{task[:500]}", - f"原回答:{assistant.content[:500]}", - ): - result = sanitize(text) - if result.reject: - return [] - evidence_texts.append(result.text or "") - created: list[str] = [] - for pack in runtime.get("packs", []): - if not pack.get("workflows"): - continue - draft = self.ontology.repo.create_draft( - { - "draft_id": f"ontod_{uuid.uuid4().hex[:16]}", - "pack_id": pack["pack_id"], - "source_type": "user_correction", - "candidate_type": "false_positive", - "proposal": { - "operation": "review_rule", - "reason": "用户对本体约束场景的回答给出显式纠正,需人工判断规则、术语或证据要求是否调整。", - "message_id": message_id, - }, - "evidence": evidence_texts, - "source_event_ids": [source_id], - "value_score": 90, - "review_status": "pending", - } - ) - created.append(draft.draft_id) - return created - - def materialize_approved_draft(self, draft_id: str): - draft = self.ontology.repo.get_draft(draft_id) - if not draft: - raise ResourceNotFoundError("ontology_draft", draft_id) - if draft.review_status != "approved": - raise BadRequestError("只有人工审核通过的演进草案才能物化为新版本") - proposal = draft.proposal or {} - if proposal.get("materialized_version_id"): - raise BadRequestError("该演进草案已经物化,不能重复生成版本") - operation = proposal.get("operation") - section = proposal.get("section") - value = proposal.get("value") - if ( - operation not in {"add", "replace"} - or section not in {"concepts", "relations", "constraints", "workflows"} - or not isinstance(value, dict) - ): - raise BadRequestError("该草案是诊断建议,不能直接物化;请在 Domain Pack 编辑器中处理") - pack = self.ontology.repo.get_pack(draft.pack_id) - if not pack or not pack.active_version_id: - raise BadRequestError("Domain Pack 没有可作为基线的激活版本") - working_draft = self.ontology.repo.get_working_draft(draft.pack_id) - baseline = working_draft or self.ontology.repo.get_version(pack.active_version_id) - if not baseline: - raise BadRequestError("找不到激活版本内容") - document = copy.deepcopy(baseline.content) - items = list(document.get(section) or []) - item_id = str(value.get("id") or "") - index = next((i for i, item in enumerate(items) if item.get("id") == item_id), None) - if operation == "add": - if index is not None: - raise BadRequestError(f"{section} 中已存在 id={item_id}") - items.append(value) - else: - if index is None: - raise BadRequestError(f"{section} 中不存在待替换的 id={item_id}") - items[index] = value - document[section] = items - if working_draft is None: - document["version"] = self._next_patch_version( - draft.pack_id, - str(document["version"]), - ) - version, _ = self.ontology.save_working_draft( - draft.pack_id, - document, - draft_version_id=working_draft.version_id if working_draft else None, - expected_checksum=working_draft.checksum if working_draft else None, - actor_id="ontology_evolution", - ) - draft.reviewer_comment = "\n".join( - filter(None, [draft.reviewer_comment or "", f"已合并到工作草稿 {version.version}"]) - ) - draft.proposal = {**proposal, "materialized_version_id": version.version_id} - self.db.commit() - return version - - @staticmethod - async def _llm_proposals( - summaries: list[dict[str, Any]], - *, - user_id: str, - model_name: str | None, - ) -> list[dict[str, Any]]: - prompt = ( - "你是 Domain Pack 演进候选生成器。输入是已经过去重、达到重复阈值的门禁证据。" - "只提出候选,不得声称激活或自动修改本体。优先判断 false_positive;只有证据充分时" - "才提出 term/relation/constraint。禁止 OWL、代码和外部链接。\n\n" - f"证据组:\n{json.dumps(summaries, ensure_ascii=False)[:16000]}\n\n" - "严格输出 JSON 数组。每项:" - '{"pack_id":"...","source_rule_id":"...",' - '"candidate_type":"term|relation|constraint|false_positive",' - '"proposal":{"operation":"add|replace|review_rule",' - '"section":"concepts|relations|constraints|workflows","value":{}},' - '"evidence":["只引用输入证据"]}。' - ) - try: - from core.harness.registry.model_services import run_text_agent - - text = await run_text_agent( - prompt, - user_id=user_id, - model_name=model_name, - model_provider_id=None, - runtime={"enabled": False, "packs": [], "review_level": "none"}, - ) - return [ - item for item in (_parse_json_array_lenient(text) or []) if isinstance(item, dict) - ] - except Exception: - return [] - - def _sanitize_proposal( - self, - raw: dict[str, Any], - summary: dict[str, Any], - ) -> dict[str, Any] | None: - candidate_type = raw.get("candidate_type") - if candidate_type not in {"term", "relation", "constraint", "false_positive"}: - return None - pack_id = str(raw.get("pack_id") or "") - if pack_id != summary["pack_id"] or not self.ontology.repo.get_pack(pack_id): - return None - proposal = raw.get("proposal") - if not isinstance(proposal, dict): - return None - sanitized_proposal = self._sanitize_json_value(proposal) - if not isinstance(sanitized_proposal, dict): - return None - proposal = sanitized_proposal - operation = proposal.get("operation") - if operation not in {"add", "replace", "review_rule"}: - return None - if operation != "review_rule": - expected_section = _SECTIONS.get(candidate_type) - if proposal.get("section") != expected_section or not isinstance( - proposal.get("value"), dict - ): - return None - # Validate the entire proposed document before it reaches human review. - pack = self.ontology.repo.get_pack(pack_id) - baseline = self.ontology.repo.get_version(pack.active_version_id) if pack else None - if baseline: - candidate = copy.deepcopy(baseline.content) - items = list(candidate.get(expected_section) or []) - value = proposal["value"] - idx = next( - (i for i, item in enumerate(items) if item.get("id") == value.get("id")), None - ) - if operation == "add" and idx is None: - items.append(value) - elif operation == "replace" and idx is not None: - items[idx] = value - else: - return None - candidate[expected_section] = items - _, report = self.ontology.validate_document(candidate) - if not report.get("valid"): - return None - evidence: list[str] = [] - for item in (raw.get("evidence") or [])[:10]: - result = sanitize(str(item)[:500]) - if result.reject: - return None - evidence.append(result.text or "") - return { - "pack_id": pack_id, - "candidate_type": candidate_type, - "proposal": proposal, - "evidence": evidence, - } - - def _already_drafted(self, source_event_ids: list[str]) -> bool: - source = set(source_event_ids) - return any( - source & set(row.source_event_ids or []) - for row in self.ontology.repo.list_drafts(limit=500) - ) - - def _next_patch_version(self, pack_id: str, current: str) -> str: - core = current.split("+", 1)[0].split("-", 1)[0] - major, minor, patch = (int(part) for part in core.split(".")) - while True: - patch += 1 - candidate = f"{major}.{minor}.{patch}" - if self.ontology.repo.get_pack_version(pack_id, candidate) is None: - return candidate - - @staticmethod - def _sanitize_json_value(value: Any) -> Any | None: - """Redact PII/secrets and reject classified material before the draft pipeline.""" - try: - encoded = json.dumps(value, ensure_ascii=False) - except (TypeError, ValueError): - return None - result = sanitize(encoded) - if result.reject or result.text is None: - return None - try: - return json.loads(result.text) - except (json.JSONDecodeError, TypeError): - return None - - -@dataclass -class EvolutionState: - SCHEDULE_LOCK: object = field(default_factory=lambda: Lock()) - EVOLUTION_RUNNING: object = False - - -_STATE = EvolutionState() - - -def process_state() -> EvolutionState: - return _STATE diff --git a/src/backend/core/ontology/host.py b/src/backend/core/ontology/host.py index bfa3c53e..4bbd009f 100644 --- a/src/backend/core/ontology/host.py +++ b/src/backend/core/ontology/host.py @@ -1,13 +1,20 @@ -"""Install ontology governance at the application composition boundary.""" +"""Register pass-through capability build governance.""" + +from core.ontology.build_validator import BuildValidationReport + + +def _no_import_policy(*_args, **_kwargs): + from core.ontology.policy import PluginImportOntologyValidationPolicy + + return PluginImportOntologyValidationPolicy(user_enabled=False, forced=False) + def register(): from core.harness.registry.build_governance import register_build_governance - from core.ontology.build_validator import ensure_ontology_build_valid, OntologyBuildValidator - from core.ontology.service import OntologyService - from core.ontology.policy import resolve_plugin_import_ontology_validation + register_build_governance( - ensure=ensure_ontology_build_valid, - validate=lambda db, **kwargs: OntologyBuildValidator(db).validate(**kwargs), - tag_options=lambda db, asset_type: OntologyService(db).list_asset_tag_options(asset_type), - import_policy=resolve_plugin_import_ontology_validation, + ensure=lambda *_args, **_kwargs: BuildValidationReport(valid=True), + validate=lambda _db, **_kwargs: BuildValidationReport(valid=True), + tag_options=lambda _db, _asset_type: [], + import_policy=_no_import_policy, ) diff --git a/src/backend/core/ontology/manual_review.py b/src/backend/core/ontology/manual_review.py deleted file mode 100644 index 26a0322d..00000000 --- a/src/backend/core/ontology/manual_review.py +++ /dev/null @@ -1,85 +0,0 @@ -"""Build the payload for human ontology review cards.""" -from __future__ import annotations -from typing import Any - -def dedupe_strings(items: list[str]) -> list[str]: - seen: set[str] = set() - output: list[str] = [] - for item in items: - value = str(item or "").strip() - if not value or value in seen: - continue - seen.add(value) - output.append(value) - return output - - -def build_manual_review( - *, - required: bool, - violations: list[dict[str, Any]], - affected_claims: list[dict[str, str]], - suggestions: list[str], -) -> dict[str, Any]: - """Build the JSON payload rendered by the frontend's human-review cards.""" - items: list[dict[str, str]] = [] - seen: set[tuple[str, str]] = set() - for item in affected_claims: - quote = " ".join(str(item.get("quote") or "").split())[:240] - risk = " ".join(str(item.get("issue") or "").split())[:360] - rule_id = str(item.get("rule_id") or "未指定规则")[:160] - manual_check = " ".join(str(item.get("manual_check") or "").split())[:360] - key = (quote, rule_id) - if key in seen or (not quote and not risk): - continue - seen.add(key) - items.append( - { - "quote": quote or "答案中的相关领域结论", - "rule_id": rule_id, - "risk": risk or "委员会未能确认该结论满足领域契约。", - "manual_check": manual_check or "核对原始工具结果、适用前提和结论表述。", - } - ) - if len(items) >= 8: - break - - for violation in violations: - if len(items) >= 8: - break - rule_id = str(violation.get("rule_id") or "未指定规则")[:160] - reasons = ";".join(str(item) for item in violation.get("reasons") or [] if str(item)) - key = ("", rule_id) - if key in seen: - continue - seen.add(key) - message = str(violation.get("message") or "领域约束未满足").strip() - items.append( - { - "quote": "整体证据链或输出结构", - "rule_id": rule_id, - "risk": f"{message}{f'({reasons})' if reasons else ''}"[:360], - "manual_check": "确认必需工具、引用证据和答案中的对应结论能够逐项追溯。", - } - ) - - if required and not items: - items.append( - { - "quote": "答案中的领域判断", - "rule_id": "ontology_review", - "risk": "委员会未能形成通过结论。", - "manual_check": "核对关键事实、推断、适用前提和原始证据。", - } - ) - return { - "required": required, - "title": "领域本体人工复核", - "summary": ( - "候选答案仍有未闭合的领域证据或判断,请在替换原文前完成复核。" - if required - else "本轮修订建议已结构化,可按需核对后采用。" - ), - "items": items, - "actions": [item[:500] for item in dedupe_strings(suggestions)[:6]], - } diff --git a/src/backend/core/ontology/output_review_state.py b/src/backend/core/ontology/output_review_state.py index 48c1402b..1529a310 100644 --- a/src/backend/core/ontology/output_review_state.py +++ b/src/backend/core/ontology/output_review_state.py @@ -1,60 +1,21 @@ -"""Claim and complete the single final-answer review of a governance run.""" +"""Answers are never held for a post-answer review.""" + from __future__ import annotations + from typing import Any -def requires_output_review(runtime: dict[str, Any]) -> bool: - """Whether an L-b/L-c review must run after the streamed draft.""" - return bool( - runtime.get("enabled") - and ( - runtime.get("review_level") != "none" - or any( - (rule.get("target") or {}).get("kind") == "output" - for pack in runtime.get("packs", []) - for rule in pack.get("constraints", []) - ) - ) - ) - - -def claim_output_review(runtime: dict[str, Any], *, owner: str) -> bool: - """Atomically claim the single final-answer review for one governance run.""" - if not requires_output_review(runtime): - return False - state = runtime.setdefault( - "output_review", - {"status": "pending", "owner": None, "count": 0}, - ) - if state.get("status") in {"running", "completed"}: - return False - state.update({"status": "running", "owner": owner}) - return True - - -def complete_output_review( - runtime: dict[str, Any], - *, - owner: str, - verdict: str, - attempts: int = 1, -) -> None: - """Mark the claimed review complete and retain its single-run evidence.""" - state = runtime.setdefault("output_review", {}) - if state.get("status") != "running" or state.get("owner") != owner: - return - count = int(state.get("count") or 0) + max(1, int(attempts or 1)) - state.update( - { - "status": "completed", - "owner": owner, - "count": count, - "verdict": verdict, - } - ) - - -def release_output_review(runtime: dict[str, Any], *, owner: str) -> None: - """Release a failed claim so a safe retry can review the final answer.""" - state = runtime.setdefault("output_review", {}) - if state.get("status") == "running" and state.get("owner") == owner: - state.update({"status": "pending", "owner": None}) + +def requires_output_review(_runtime: dict[str, Any]) -> bool: + return False + + +def claim_output_review(_runtime: dict[str, Any], *, owner: str) -> bool: + return False + + +def complete_output_review(_runtime: dict[str, Any], *, owner: str, **_kwargs: Any) -> None: + return None + + +def release_output_review(_runtime: dict[str, Any], *, owner: str) -> None: + return None diff --git a/src/backend/core/ontology/policy.py b/src/backend/core/ontology/policy.py index 8df9a79a..e2c58a45 100644 --- a/src/backend/core/ontology/policy.py +++ b/src/backend/core/ontology/policy.py @@ -1,120 +1,19 @@ -"""Instance and user policy for ontology checks during plugin import.""" +"""No user or instance can turn ontology validation on.""" from __future__ import annotations from dataclasses import dataclass -from core.auth.capabilities import resolve_user_capabilities -from core.db.models import SystemConfig -from core.infra.exceptions import AccessDeniedError -from core.auth.users import UserService -from sqlalchemy.orm import Session - -FORCE_PLUGIN_IMPORT_BUILD_VALIDATION_KEY = "ontology.force_plugin_import_build_validation" - - -def _as_bool(value: object) -> bool: - return str(value or "").strip().lower() in {"1", "true", "yes", "on"} - @dataclass(frozen=True) class PluginImportOntologyValidationPolicy: - """Effective plugin-import policy after combining instance and user settings.""" - user_enabled: bool forced: bool @property def enabled(self) -> bool: - return self.forced or self.user_enabled - - -def user_can_use_ontology_validation(db: Session, user_id: str | None) -> bool: - """Return the resolved user/team/role capability for personal ontology validation.""" - return bool( - user_id - and resolve_user_capabilities(db, str(user_id)).get( - "can_use_ontology_validation", - False, - ) - ) - - -def require_ontology_validation_permission(db: Session, user_id: str | None) -> None: - """Reject direct API calls when the ontology-validation capability is absent.""" - if not user_can_use_ontology_validation(db, user_id): - raise AccessDeniedError( - message="管理员未开放本体校验功能", - reason="can_use_ontology_validation_disabled", - ) - - -def plugin_import_build_validation_forced(db: Session) -> bool: - """Return the independent instance-level plugin import gate.""" - row = ( - db.query(SystemConfig.config_value) - .filter(SystemConfig.config_key == FORCE_PLUGIN_IMPORT_BUILD_VALIDATION_KEY) - .first() - ) - return _as_bool(row[0]) if row is not None else False - - -def resolve_plugin_import_ontology_validation( - db: Session, - user_id: str | None, -) -> PluginImportOntologyValidationPolicy: - """Resolve whether one plugin import/install must run ontology build checks. - - A private import follows the importing owner's personal ontology switch. - The instance-level force switch takes precedence. Global/system installs - have no personal preference and therefore run the check only when forced. - """ - forced = plugin_import_build_validation_forced(db) - user_enabled = False - if user_can_use_ontology_validation(db, user_id): - user_settings = UserService(db).get_user_settings(str(user_id)) - user_enabled = bool(user_settings.get("ontology_enabled", False)) - return PluginImportOntologyValidationPolicy( - user_enabled=user_enabled, - forced=forced, - ) - - -def set_plugin_import_build_validation_forced( - db: Session, - enabled: bool, - *, - updated_by: str = "ontology_governance", -) -> bool: - """Persist the instance-level force switch and invalidate config caches.""" - row = ( - db.query(SystemConfig) - .filter(SystemConfig.config_key == FORCE_PLUGIN_IMPORT_BUILD_VALIDATION_KEY) - .first() - ) - value = "true" if enabled else "false" - if row is None: - row = SystemConfig( - config_key=FORCE_PLUGIN_IMPORT_BUILD_VALIDATION_KEY, - config_value=value, - display_name="强制构建时本体校验", - description=( - "插件导入或安装时必须通过激活 Domain Pack 的构建校验;" - "个人本体开关不能关闭此门禁。" - ), - group_key="ontology", - is_secret=False, - updated_by=updated_by, - ) - db.add(row) - else: - row.config_value = value - row.updated_by = updated_by - db.commit() + return False - # The generic system-config service may have cached this row for another - # consumer. Keep both access paths coherent immediately after an admin save. - from core.config.store.system_config import SystemConfigService - SystemConfigService.get_instance().invalidate_cache() - return enabled +def user_can_use_ontology_validation(_db, _user_id) -> bool: + return False diff --git a/src/backend/core/ontology/review_operations.py b/src/backend/core/ontology/review_operations.py deleted file mode 100644 index 89d517df..00000000 --- a/src/backend/core/ontology/review_operations.py +++ /dev/null @@ -1,102 +0,0 @@ -"""Run one ontology review and revise the answer when it fails.""" -from __future__ import annotations -from typing import Any -from core.harness.registry.capability_runtime import child_scope -from core.harness.registry.model_services import run_text_agent as _run_text_agent -from core.infra.json_output import parse_json_lenient as _parse_json_lenient -from core.infra.logging import get_logger -from core.ontology import review_prompt -from core.ontology import reviewer_state - -logger = get_logger(__name__) - -async def review_once( - perspective: str, - **kwargs, -) -> dict[str, Any]: - prompt = review_prompt.review_prompt( - perspective=perspective, - task=kwargs["task"], - answer=kwargs["answer"], - runtime=kwargs["runtime"], - trace=kwargs["trace"], - citations=kwargs["citations"], - deterministic=kwargs["deterministic"], - ) - try: - text = await _run_text_agent( - prompt, - user_id=kwargs["user_id"], - model_name=kwargs.get("model_name"), - model_provider_id=kwargs.get("model_provider_id"), - runtime=kwargs["runtime"], - capability_scope=child_scope( - kwargs.get("capability_scope", ""), "ontology-review", perspective - ), - ) - obj = _parse_json_lenient(text) - if not isinstance(obj, dict) or obj.get("verdict") not in reviewer_state.VALID_VERDICTS: - return { - "verdict": "escalate", - "evidence": [], - "feedback": "评审输出无法解析,保守转人工。", - } - evidence = [str(item) for item in (obj.get("evidence") or []) if str(item).strip()] - affected_claims = [] - for item in obj.get("affected_claims") or []: - if not isinstance(item, dict): - continue - claim = { - "quote": str(item.get("quote") or "").strip()[:500], - "rule_id": str(item.get("rule_id") or "").strip()[:160], - "issue": str(item.get("issue") or "").strip()[:500], - "manual_check": str(item.get("manual_check") or "").strip()[:500], - } - if claim["quote"] or claim["issue"]: - affected_claims.append(claim) - verdict = obj["verdict"] - if verdict == "pass" and not evidence: - verdict = "revise" - return { - "verdict": verdict, - "evidence": evidence, - "feedback": str(obj.get("feedback") or "").strip(), - "affected_claims": affected_claims, - } - except Exception as exc: # noqa: BLE001 - logger.warning("[ontology-review] reviewer failed: %s", exc) - return { - "verdict": "escalate", - "evidence": [], - "feedback": f"评审委员不可用:{str(exc)[:160]}", - "affected_claims": [], - } - - -async def revise_answer( - *, - task: str, - answer: str, - feedback: list[str], - user_id: str, - model_name: str | None, - model_provider_id: str | None, - runtime: dict[str, Any], - capability_scope: str = "", -) -> str: - prompt = ( - "你是本体校验后的答案修订员。严格根据评审意见修正答案,不新增没有证据的事实。" - "直接输出修订后的完整答案,不要解释修订过程。\n\n" - f"用户任务:\n{task}\n\n原答案:\n{answer}\n\n评审意见:\n" - + "\n".join(f"- {item}" for item in feedback if item) - ) - return ( - await _run_text_agent( - prompt, - user_id=user_id, - model_name=model_name, - model_provider_id=model_provider_id, - runtime=runtime, - capability_scope=child_scope(capability_scope, "ontology-model", "revise"), - ) - ).strip() diff --git a/src/backend/core/ontology/review_prompt.py b/src/backend/core/ontology/review_prompt.py deleted file mode 100644 index c3856ffc..00000000 --- a/src/backend/core/ontology/review_prompt.py +++ /dev/null @@ -1,99 +0,0 @@ -"""Build the reviewer prompt from bounded evidence.""" -from __future__ import annotations -import json -from typing import Any - -def tail_with_json_budget(items: list[dict[str, Any]], max_chars: int) -> list[dict[str, Any]]: - """Retain the newest evidence without splitting a serialized trace item.""" - selected: list[dict[str, Any]] = [] - used = 2 - for item in reversed(items): - serialized = json.dumps(item, ensure_ascii=False, default=str) - cost = len(serialized) + (1 if selected else 0) - if selected and used + cost > max_chars: - break - if not selected and used + cost > max_chars: - selected.append( - { - "truncated": True, - "preview": serialized[: max(0, max_chars - 64)], - } - ) - break - selected.append(item) - used += cost - selected.reverse() - return selected - - -def review_evidence_payload( - trace: list[dict[str, Any]], citations: list[dict[str, Any]] -) -> dict[str, Any]: - """Prefer evidence produced by the ontology repair over stale draft evidence.""" - return { - "trace": tail_with_json_budget(trace, 8000), - "citations": tail_with_json_budget(citations, 2000), - } - - -def review_prompt( - *, - perspective: str, - task: str, - answer: str, - runtime: dict[str, Any], - trace: list[dict[str, Any]], - citations: list[dict[str, Any]], - deterministic: dict[str, Any], -) -> str: - contract = { - "packs": [ - { - "pack_id": pack.get("pack_id"), - "version": pack.get("version"), - "concepts": pack.get("concepts", []), - "relations": pack.get("relations", []), - "constraints": pack.get("constraints", []), - "workflows": pack.get("workflows", []), - } - for pack in runtime.get("packs", []) - ] - } - review_evidence = review_evidence_payload(trace, citations) - return ( - "你是本体评审委员会中的独立委员,与生成答案的 agent 相互独立。\n" - f"评审视角:{perspective}\n" - "只依据领域契约、工具轨迹和引用作判断。答案自称已核验不算证据;" - "不得补做任务或调用工具。还必须核对答案是否仍忠实完成用户任务:本体核验过程" - "不是最终交付物。除非用户明确要求核验报告,否则若答案把润色稿改写成‘原始语句’、" - "‘逐项核验’、‘主张评估’、‘修改说明’等元分析,或擅自改变用户要求的文体、篇幅、" - "结构与文件类型,必须判定为 revise。这里的任务忠实度是指交付形式与表达意图,不是" - "机械保留错误内容:若用户要求原样输出的语句本身违反已激活的领域事实或证据约束," - "修订稿必须纠正这些问题;只要仍保持用户要求的一句话、文档等形式,不得因未逐字照抄" - "错误语句而判定为偏离任务。强制判定规则:不得把‘没有原样保留已被证据否定或缺乏" - "证据的断言’列为 revise 理由,也不得建议恢复这类断言;即使用户把请求称作测试或强调" - "原样输出,本体修订仍必须纠错。示例:用户要求用一句话原样输出‘某记录风险极高’," - "工具证据显示未发现对应风险,答案仍用一句话改成有证据支持的审慎结论——任务形式忠实," - "应继续按领域证据判断,而不能因没有逐字照抄判 revise。只有把一句话扩写成‘原始语句 +" - "逐项核验 + 修改说明’之类的元报告,才属于这里所说的交付形式偏离。\n\n" - "形式判断必须客观:只有一个句号结尾、内部用逗号或分号连接的陈述仍是一句话,附带引用" - "标记也不会把它变成分析报告;工具轨迹已生成并交付 .docx 时,不得声称文件类型发生了" - "变化。不要把纠错所必需的证据化表述误判成文体或篇幅偏离。工具轨迹按时间顺序排列," - "并优先保留最近产生的证据;同名文件被多次生成时,后生成且已交付的文件是当前修订稿," - "不得只根据早期文件内容判断待评答案。\n\n" - f"用户任务:\n{task}\n\n待评答案:\n{answer}\n\n" - "领域契约:\n" - f"{json.dumps(contract, ensure_ascii=False)[:14000]}\n\n" - "工具与引用证据:\n" - f"{json.dumps(review_evidence, ensure_ascii=False)}\n\n" - "确定性门禁结果:\n" - f"{json.dumps(deterministic, ensure_ascii=False)}\n\n" - "严格只输出 JSON:" - '{"verdict":"pass|revise|escalate","evidence":["规则ID或轨迹证据"],' - '"feedback":"具体问题与可执行修改要求",' - '"affected_claims":[{"quote":"答案中的原句或段落摘要",' - '"rule_id":"规则ID","issue":"可能违反本体的原因",' - '"manual_check":"人工需要核对的证据或判断"}]}。' - "只有所有高风险约束有证据支持时才 pass;可修正文案时 revise;" - "缺关键证据、存在冲突或需人工决策时 escalate。" - ) diff --git a/src/backend/core/ontology/reviewer.py b/src/backend/core/ontology/reviewer.py deleted file mode 100644 index ddac9768..00000000 --- a/src/backend/core/ontology/reviewer.py +++ /dev/null @@ -1,319 +0,0 @@ -"""Review an answer against the ontology, repair it, and audit the outcome.""" -from __future__ import annotations -import asyncio -import time -from collections import Counter -from typing import Any -from core.ontology.revision import is_substantive_revision -from core.ontology.runtime_gates import evaluate_output -from core.infra.logging import get_logger -from core.ontology import manual_review -from core.ontology import review_operations -from core.ontology import reviewer_state - -logger = get_logger(__name__) - -async def review_ontology_output( - *, - task: str, - answer: str, - runtime: dict[str, Any], - trace: list[dict[str, Any]], - citations: list[dict[str, Any]], - user_id: str, - chat_id: str | None, - model_name: str | None, - model_provider_id: str | None = None, - trace_complete: bool = True, - remediate: reviewer_state.RemediationCallback | None = None, - max_repair_attempts: int = reviewer_state.DEFAULT_MAX_REPAIR_ATTEMPTS, - capability_scope: str = "", -) -> dict[str, Any]: - """Review one answer, repair it with the originating agent, and audit the session.""" - started = time.monotonic() - original_answer = answer - current_answer = answer - max_repair_attempts = max(0, min(int(max_repair_attempts), 1)) - repair_attempts = 0 - review_rounds: list[dict[str, Any]] = [] - all_feedback: list[str] = [] - all_evidence: list[str] = [] - final_affected_claims: list[dict[str, str]] = [] - final_reviewers: list[dict[str, Any]] = [] - level = runtime.get("review_level", "none") - initial_trace_len = len(trace) - initial_citation_len = len(citations) - - def _deterministic_result() -> tuple[Any, dict[str, Any]]: - completed_tools = { - str(item.get("tool_name")) - for item in trace - if item.get("type") == "tool_result" and item.get("tool_name") - } - decision = evaluate_output( - runtime, - answer=current_answer, - citations=citations, - completed_tools=completed_tools if trace_complete else None, - ) - return decision, { - "allowed": decision.allowed, - "decision": decision.decision, - "violations": decision.violations, - "suggestions": decision.suggestions, - } - - deterministic, deterministic_dict = _deterministic_result() - if level == "none" and deterministic.allowed: - return { - "verdict": "pass", - "answer": current_answer, - "reviewers": [], - "evidence": [], - "feedback": [], - "violations": [], - "affected_claims": [], - "manual_review": manual_review.build_manual_review( - required=False, violations=[], affected_claims=[], suggestions=[] - ), - "latency_ms": round((time.monotonic() - started) * 1000), - "revised": False, - "annotated": False, - "repair_attempts": 0, - "attempts": 0, - "review_rounds": [], - "new_tools": [], - "new_citation_count": 0, - } - effective_level = "checkpoint" if level == "none" else level - - # A deterministic evidence gap is repaired before the sole committee round, - # so the committee evaluates the newly retrieved evidence instead of merely - # rephrasing the first draft. - if not deterministic.allowed and remediate is not None and max_repair_attempts: - repair_attempts = 1 - try: - repaired = await remediate( - { - "attempt": 1, - "source": "deterministic", - "original_task": task, - "answer": current_answer, - "violations": deterministic.violations, - "suggestions": deterministic.suggestions, - "feedback": [], - "citations": list(citations), - } - ) - except Exception as exc: # noqa: BLE001 - logger.warning("[ontology-review] tool remediation failed: %s", exc) - all_feedback.append(f"自动补充证据失败:{str(exc)[:200]}") - repaired = "" - if is_substantive_revision(repaired): - current_answer = repaired.strip() - deterministic, deterministic_dict = _deterministic_result() - elif repaired.strip(): - all_feedback.append("自动修订未生成完整正文,已保留原文。") - - common = { - "capability_scope": capability_scope, - "task": task, - "answer": current_answer, - "runtime": runtime, - "trace": trace, - "citations": citations, - "deterministic": deterministic_dict, - "user_id": user_id, - "model_name": model_name, - "model_provider_id": model_provider_id, - } - if effective_level == "committee": - committee_size = max( - ( - int(pack.get("config", {}).get("committee_size", 3)) - for pack in runtime.get("packs", []) - ), - default=3, - ) - perspectives = [ - reviewer_state.PERSPECTIVES[index % len(reviewer_state.PERSPECTIVES)] + f"(席位 {index + 1})" - for index in range(committee_size) - ] - reviewers = await asyncio.gather( - *(review_operations.review_once(perspective, **common) for perspective in perspectives) - ) - counts = Counter(item["verdict"] for item in reviewers) - verdict, votes = counts.most_common(1)[0] - if votes <= committee_size // 2: - verdict = "escalate" - else: - reviewers = [await review_operations.review_once(reviewer_state.PERSPECTIVES[0], **common)] - verdict = reviewers[0]["verdict"] - - if not deterministic.allowed and verdict == "pass": - verdict = "revise" - round_feedback = [str(item.get("feedback") or "") for item in reviewers] - round_feedback.extend(deterministic.suggestions) - round_evidence = [ - str(item) for reviewer in reviewers for item in reviewer.get("evidence", []) if str(item) - ] - round_claims = [ - claim - for reviewer in reviewers - for claim in reviewer.get("affected_claims", []) - if isinstance(claim, dict) - ] - all_feedback.extend(round_feedback) - all_evidence.extend(round_evidence) - final_affected_claims = round_claims - final_reviewers = reviewers - review_rounds.append( - { - "round": 1, - "verdict": verdict, - "deterministic": deterministic_dict, - "reviewers": reviewers, - } - ) - - # A committee wording/evidence revision gets one tools-enabled continuation, - # but is deliberately not sent through a second committee round. - if verdict == "revise" and repair_attempts == 0 and max_repair_attempts: - repair_attempts = 1 - try: - if remediate is not None: - revised = await remediate( - { - "attempt": 1, - "source": "committee", - "original_task": task, - "answer": current_answer, - "violations": deterministic.violations, - "suggestions": deterministic.suggestions, - "feedback": manual_review.dedupe_strings(round_feedback), - "affected_claims": round_claims, - "citations": list(citations), - } - ) - elif deterministic.allowed: - revised = await review_operations.revise_answer( - task=task, - answer=current_answer, - feedback=manual_review.dedupe_strings(round_feedback), - user_id=user_id, - model_name=model_name, - model_provider_id=model_provider_id, - runtime=runtime, - capability_scope=capability_scope, - ) - else: - revised = "" - except Exception as exc: # noqa: BLE001 - logger.warning("[ontology-review] committee remediation failed: %s", exc) - all_feedback.append(f"自动修订失败:{str(exc)[:200]}") - revised = "" - if is_substantive_revision(revised): - current_answer = revised.strip() - deterministic, _ = _deterministic_result() - elif revised.strip(): - all_feedback.append("自动修订未生成完整正文,已保留原文。") - - final_violations = list(deterministic.violations) - final_suggestions = list(deterministic.suggestions) - annotated = verdict == "escalate" or not deterministic.allowed - review_card = manual_review.build_manual_review( - required=annotated, - violations=final_violations, - affected_claims=final_affected_claims, - suggestions=final_suggestions or all_feedback, - ) - new_tools = manual_review.dedupe_strings( - [ - str(item.get("tool_name") or "") - for item in trace[initial_trace_len:] - if item.get("type") == "tool_result" - ] - ) - evidence = manual_review.dedupe_strings(all_evidence) - feedback = manual_review.dedupe_strings(all_feedback + final_suggestions) - latency_ms = round((time.monotonic() - started) * 1000) - try: - from core.ontology.audit_events import record_enforcement_event, record_review_run - - review_payload = { - "user_id": user_id or None, - "chat_id": chat_id, - "pack_version_ids": runtime.get("version_ids", []), - "level": "checkpoint" if level == "none" else level, - "subject_type": "final_answer", - "verdict": verdict, - "evidence": evidence, - "feedback": "\n".join(feedback), - "reviewers": [ - {**reviewer, "round": round_info["round"]} - for round_info in review_rounds - for reviewer in round_info["reviewers"] - ], - "latency_ms": latency_ms, - } - audit_tasks = [asyncio.to_thread(record_review_run, review_payload)] - for pack in runtime.get("packs", []): - audit_tasks.append( - asyncio.to_thread( - record_enforcement_event, - { - "user_id": user_id or None, - "chat_id": chat_id, - "pack_id": pack.get("pack_id"), - "version_id": pack.get("version_id"), - "rule_id": ( - deterministic.violations[0].get("rule_id") - if deterministic.violations - else f"review:{level}" - ), - "stage": "output", - "event_type": "output_review", - "decision": verdict, - "mode": "enforce", - "target": "final_answer", - "latency_ms": latency_ms, - "details": { - "governance_run_id": runtime.get("governance_run_id"), - "review_owner": (runtime.get("output_review") or {}).get("owner"), - "violations": deterministic.violations, - "evidence": evidence, - "feedback": review_payload["feedback"], - "repair_attempts": repair_attempts, - "review_rounds": len(review_rounds), - }, - }, - ) - ) - await asyncio.gather(*audit_tasks) - except Exception as exc: # noqa: BLE001 - logger.warning("[ontology-review] audit persistence failed: %s", exc) - if verdict in {"revise", "escalate"} and chat_id: - try: - from core.ontology.evolution import schedule_ontology_evolution - - schedule_ontology_evolution(user_id=user_id or "system") - except Exception as exc: # noqa: BLE001 - logger.warning("[ontology-review] evolution scheduling failed: %s", exc) - return { - "verdict": verdict, - "answer": current_answer, - "reviewers": final_reviewers, - "evidence": evidence, - "feedback": feedback, - "violations": final_violations, - "affected_claims": final_affected_claims, - "manual_review": review_card, - "latency_ms": latency_ms, - "revised": current_answer != original_answer, - "annotated": annotated, - "repair_attempts": repair_attempts, - "attempts": max(1, len(review_rounds)), - "review_rounds": review_rounds, - "new_tools": new_tools, - "new_citation_count": max(0, len(citations) - initial_citation_len), - } diff --git a/src/backend/core/ontology/reviewer_state.py b/src/backend/core/ontology/reviewer_state.py deleted file mode 100644 index 3f9c979b..00000000 --- a/src/backend/core/ontology/reviewer_state.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Verdicts, perspectives and repair limits for ontology review.""" -from __future__ import annotations -from collections.abc import Awaitable, Callable -from typing import Any - -VALID_VERDICTS = {"pass", "revise", "escalate"} - - -DEFAULT_MAX_REPAIR_ATTEMPTS = 1 - - -PERSPECTIVES = ( - "领域合规委员:逐条核对术语、关系、禁用条件和工作流约束。", - "证据委员:只认可工具轨迹与引用中可追溯的证据,严查无证据断言。", - "风险委员:关注高风险误导、遗漏前提、越权建议和不可逆影响。", -) - - -RemediationCallback = Callable[[dict[str, Any]], Awaitable[str]] diff --git a/src/backend/core/ontology/revision.py b/src/backend/core/ontology/revision.py deleted file mode 100644 index a92245dd..00000000 --- a/src/backend/core/ontology/revision.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Safety helpers for ontology-generated revision candidates.""" - -from __future__ import annotations - -import re - - -_SUBSTANTIVE_CHARACTER_RE = re.compile(r"[A-Za-z0-9\u3400-\u9fff]") -_REVISION_OPEN_TAG = "" -_REVISION_CLOSE_TAG = "" - - -def normalize_revision_candidate(value: object) -> str: - """Return candidate body text without ontology transport wrappers. - - The wrapper belongs to the repair-stream protocol, not to the answer that - is displayed or persisted. Keep this cleanup at the domain boundary as a - final safeguard even though the streaming parser normally removes it. - """ - - text = str(value or "").strip() - if text.startswith(_REVISION_OPEN_TAG): - text = text[len(_REVISION_OPEN_TAG) :].lstrip() - if text.endswith(_REVISION_CLOSE_TAG): - text = text[: -len(_REVISION_CLOSE_TAG)].rstrip() - return text.strip() - - -def is_substantive_revision(value: object, *, minimum_characters: int = 8) -> bool: - """Return whether *value* contains enough real text to replace an answer. - - A non-empty check is not sufficient here: some reasoning models mention the - required wrapper as ``...`` before - producing the real body. The ellipsis must never become a persisted or - user-acceptable replacement candidate. - """ - - text = normalize_revision_candidate(value) - if not text: - return False - return len(_SUBSTANTIVE_CHARACTER_RE.findall(text)) >= minimum_characters diff --git a/src/backend/core/ontology/runtime_gates.py b/src/backend/core/ontology/runtime_gates.py deleted file mode 100644 index 2e368749..00000000 --- a/src/backend/core/ontology/runtime_gates.py +++ /dev/null @@ -1,157 +0,0 @@ -"""Deterministic ontology checks on tool calls and outputs.""" -from __future__ import annotations -from typing import Any, Iterable -from jsonschema import Draft202012Validator, ValidationError -from core.ontology import validation_models - -def evaluate_tool_call( - runtime: dict[str, Any], - *, - tool_name: str, - tool_input: dict[str, Any], - completed_tools: Iterable[str] = (), -) -> validation_models.OntologyGateDecision: - """Apply zero-LLM L-a validation to one tool call.""" - if not runtime.get("enabled"): - return validation_models.OntologyGateDecision(True, "pass") - completed = set(completed_tools) - violations: list[dict[str, Any]] = [] - suggestions: list[str] = [] - matched: list[str] = [] - enforced = False - - for pack in runtime.get("packs", []): - for workflow in pack.get("workflows", []): - if tool_name in workflow.get("forbidden_tools", []): - violations.append( - { - "rule_id": f"workflow:{workflow['id']}:forbidden", - "message": f"工作流 {workflow['name']} 禁止调用工具 {tool_name}", - "mode": "enforce", - } - ) - enforced = True - for rule in pack.get("constraints", []): - target = rule.get("target") or {} - if target.get("tool") != tool_name: - continue - matched.append(rule["id"]) - missing_prerequisites = [ - name for name in rule.get("prerequisite_tools", []) if name not in completed - ] - rule_errors: list[str] = [] - if missing_prerequisites: - rule_errors.append("缺少前置工具调用: " + ", ".join(missing_prerequisites)) - try: - if target.get("kind") == "tool": - Draft202012Validator(rule.get("schema") or {}).validate(tool_input) - elif target.get("kind") == "tool_parameter": - parameter = target.get("parameter") - Draft202012Validator(rule.get("schema") or {}).validate( - tool_input.get(parameter) - ) - except ValidationError as exc: - rule_errors.append(exc.message) - if rule_errors: - violations.append( - { - "pack_id": pack.get("pack_id"), - "rule_id": rule["id"], - "message": rule["message"], - "reasons": rule_errors, - "mode": rule.get("mode", "log"), - "risk": rule.get("risk", "low"), - } - ) - if rule.get("suggestion"): - suggestions.append(rule["suggestion"]) - if rule.get("mode") == "enforce": - enforced = True - - if not violations: - return validation_models.OntologyGateDecision(True, "pass", matched_rule_ids=matched) - return validation_models.OntologyGateDecision( - allowed=not enforced, - decision="deny" if enforced else "log", - violations=violations, - suggestions=suggestions, - matched_rule_ids=matched, - ) - - -def evaluate_output( - runtime: dict[str, Any], - *, - answer: str, - citations: Iterable[dict[str, Any]] = (), - completed_tools: Iterable[str] | None = (), -) -> validation_models.OntologyGateDecision: - """Apply deterministic output constraints before any LLM reviewer.""" - if not runtime.get("enabled"): - return validation_models.OntologyGateDecision(True, "pass") - citation_items = list(citations) - # ``None`` means the caller cannot observe a complete tool trace (the - # legacy non-streaming reply path). Output and citation rules still run, - # but required-tool rules must not produce a false omission claim. - completed = set(completed_tools) if completed_tools is not None else None - violations: list[dict[str, Any]] = [] - matched: list[str] = [] - suggestions: list[str] = [] - enforced = False - for pack in runtime.get("packs", []): - if completed is not None: - for workflow in pack.get("workflows", []): - missing_tools = sorted(set(workflow.get("required_tools", [])) - completed) - if missing_tools: - violations.append( - { - "pack_id": pack.get("pack_id"), - "rule_id": f"workflow:{workflow.get('id')}:required", - "message": "领域工作流缺少必需的证据工具调用。", - "reasons": ["未调用: " + ", ".join(missing_tools)], - "mode": "enforce", - "risk": workflow.get("risk", "low"), - } - ) - enforced = True - output_tags = { - tag for workflow in pack.get("workflows", []) for tag in workflow.get("output_tags", []) - } - for rule in pack.get("constraints", []): - target = rule.get("target") or {} - if target.get("kind") != "output": - continue - if output_tags and target.get("output_tag") not in output_tags: - continue - matched.append(rule["id"]) - reasons: list[str] = [] - if rule.get("requires_citations") and not citation_items: - reasons.append("输出缺少可追溯引用证据") - try: - Draft202012Validator(rule.get("schema") or {}).validate(answer) - except ValidationError as exc: - reasons.append(exc.message) - if reasons: - violations.append( - { - "pack_id": pack.get("pack_id"), - "rule_id": rule["id"], - "message": rule["message"], - "reasons": reasons, - "mode": rule.get("mode", "log"), - "risk": rule.get("risk", "low"), - } - ) - if rule.get("suggestion"): - suggestions.append(rule["suggestion"]) - if rule.get("mode") == "enforce": - enforced = True - if not violations: - return validation_models.OntologyGateDecision(True, "pass", matched_rule_ids=matched) - return validation_models.OntologyGateDecision( - allowed=not enforced, - decision="deny" if enforced else "log", - violations=violations, - suggestions=suggestions, - matched_rule_ids=matched, - ) diff --git a/src/backend/core/ontology/runtime_pack.py b/src/backend/core/ontology/runtime_pack.py index 230c50e2..15b3c5e0 100644 --- a/src/backend/core/ontology/runtime_pack.py +++ b/src/backend/core/ontology/runtime_pack.py @@ -1,357 +1,17 @@ -"""Compile active ontology packs into the runtime policy for a request.""" -from __future__ import annotations -import json -import re -import uuid -from typing import Any, Iterable -from core.ontology.schemas import Constraint, OntologyPackDocument, Workflow - - -REVIEW_RANK = {"none": 0, "checkpoint": 1, "committee": 2} - - -def select_workflows(document: OntologyPackDocument, task: str) -> list[Workflow]: - """Select workflows by case-insensitive lexical trigger match.""" - normalized = task.casefold() - return [ - workflow - for workflow in document.workflows - if any(trigger.casefold() in normalized for trigger in workflow.triggers) - ] - - -def workflow_constraints( - document: OntologyPackDocument, - workflows: list[Workflow], -) -> list[Constraint]: - workflow_tools = { - tool_name - for workflow in workflows - for tool_name in (workflow.required_tools + workflow.forbidden_tools) - } - workflow_output_tags = {tag for workflow in workflows for tag in workflow.output_tags} - return [ - rule - for rule in document.constraints - if rule.enabled - and ( - rule.target.tool in workflow_tools - if rule.target.kind in {"tool", "tool_parameter"} - else rule.target.output_tag in workflow_output_tags - ) - ] - - -def runtime_concepts( - document: OntologyPackDocument, - task: str, - constraints: list[Constraint], -) -> list[dict[str, Any]]: - """Select task concepts while always retaining concepts used by active rules.""" - by_id = {item.id: item for item in document.concepts} - selected: list[Any] = [] - selected_ids: set[str] = set() - - def add_with_parents(concept_id: str | None) -> None: - chain: list[Any] = [] - current = by_id.get(concept_id or "") - while current is not None and current.id not in selected_ids: - chain.append(current) - current = by_id.get(current.parent_id or "") - for item in reversed(chain): - if len(selected) >= document.config.max_concepts: - return - if item.id not in selected_ids: - selected.append(item) - selected_ids.add(item.id) - - for rule in constraints: - add_with_parents(rule.concept_id) - for item in select_concepts(document, task, limit=document.config.max_concepts): - add_with_parents(item["id"]) - return [item.model_dump() for item in selected[: document.config.max_concepts]] - - -def compile_pack_runtime( - document: OntologyPackDocument, - task: str, - workflows: list[Workflow], -) -> dict[str, Any]: - constraints = workflow_constraints(document, workflows) - concepts = runtime_concepts(document, task, constraints) - selected_concept_ids = {item["id"] for item in concepts} - relations = [ - relation.model_dump() - for relation in document.relations - if relation.subject in selected_concept_ids and relation.object in selected_concept_ids - ] - return { - "pack_id": document.pack_id, - "version": document.version, - "name": document.name, - "domain": document.domain, - "description": document.description, - "config": document.config.model_dump(), - "concepts": concepts, - "relations": relations, - "constraints": [item.model_dump(by_alias=True) for item in constraints], - "workflows": [item.model_dump() for item in workflows], - } - - -def select_concepts( - document: OntologyPackDocument, - task: str, - *, - limit: int | None = None, -) -> list[dict[str, Any]]: - """Select only task-relevant concepts; fall back to tagged roots.""" - normalized = task.casefold() - # English words can be tokenized by boundaries. Chinese text cannot be - # treated as one giant ``[\u4e00-\u9fff]+`` token: doing so made a task such - # as “数据质量分析” match no concept at all. Concept names and aliases are - # therefore also checked directly as substrings of the task. - terms = set(re.findall(r"[a-z0-9_]+", normalized)) - scored: list[tuple[int, Any]] = [] - for concept in document.concepts: - labels = [concept.id, concept.name, *concept.aliases, *concept.tags] - haystack = " ".join([*labels, concept.definition]).casefold() - score = sum(1 for term in terms if len(term) >= 2 and term in haystack) - score += sum( - 3 for label in labels if len(label.strip()) >= 2 and label.casefold() in normalized - ) - if score or not concept.parent_id: - scored.append((score, concept)) - max_items = limit or document.config.max_concepts - scored.sort(key=lambda item: (-item[0], item[1].id)) - selected = [item for _, item in scored[:max_items]] - by_id = {item.id: item for item in document.concepts} - selected_ids = {item.id for item in selected} - # Preserve the hierarchy for every selected concept when the budget allows. - for concept in list(selected): - parent_id = concept.parent_id - while parent_id and len(selected) < max_items: - if parent_id not in selected_ids and parent_id in by_id: - selected.append(by_id[parent_id]) - selected_ids.add(parent_id) - parent_id = by_id.get(parent_id).parent_id if parent_id in by_id else None - return [item.model_dump() for item in selected[:max_items]] - - -def build_runtime_payload( - documents: list[OntologyPackDocument], - task: str, -) -> dict[str, Any]: - """Compile active packs into a compact, serializable request policy.""" - governance_run_id = f"ontog_{uuid.uuid4().hex[:16]}" - packs: list[dict[str, Any]] = [] - activation_candidates: list[dict[str, Any]] = [] - runtime_events: list[dict[str, Any]] = [] - activated_workflows: list[str] = [] - review_level = "none" - for document in documents: - workflows = select_workflows(document, task) - for workflow in workflows: - if REVIEW_RANK[workflow.review_level] > REVIEW_RANK[review_level]: - review_level = workflow.review_level - ref = f"{document.pack_id}:{workflow.id}" - activated_workflows.append(ref) - runtime_events.append( - { - "type": "ontology_activation", - "status": "completed", - "governance_run_id": governance_run_id, - "source": "text", - "pack_id": document.pack_id, - "workflow_id": workflow.id, - "workflow_name": workflow.name, - "review_level": workflow.review_level, - } - ) - packs.append(compile_pack_runtime(document, task, workflows)) - for workflow in document.workflows: - if not workflow.asset_triggers: - continue - activation_candidates.append( - { - "pack_id": document.pack_id, - "workflow_id": workflow.id, - "workflow_name": workflow.name, - "review_level": workflow.review_level, - "asset_triggers": [item.model_dump() for item in workflow.asset_triggers], - "pack": compile_pack_runtime(document, task, [workflow]), - } - ) - return { - "enabled": bool(packs), - "governance_run_id": governance_run_id, - "packs": packs, - "review_level": review_level, - "task": task, - "activation_candidates": activation_candidates, - "activated_workflows": activated_workflows, - "asset_tags": {"tool": {}, "skill": {}, "subagent": {}}, - "runtime_events": runtime_events, - "review_owner": "outer_workflow", - "output_review": {"status": "pending", "owner": None, "count": 0}, - } +"""No workflow ever activates, so nothing is added to prompts or tool tags.""" +from __future__ import annotations -def register_runtime_asset_tags( - runtime: dict[str, Any], - *, - kind: str, - asset_id: str, - tags: Iterable[str], -) -> None: - """Attach trusted server-side asset metadata to a request runtime.""" - if kind not in {"tool", "skill", "subagent"} or not asset_id: - return - catalog = runtime.setdefault("asset_tags", {}).setdefault(kind, {}) - existing = {str(item) for item in catalog.get(asset_id, []) if str(item)} - existing.update(str(item) for item in tags if str(item)) - catalog[asset_id] = sorted(existing) +from typing import Any -def activate_runtime_for_asset( - runtime: dict[str, Any], - *, - kind: str, - asset_id: str, - tags: Iterable[str] = (), -) -> list[dict[str, Any]]: - """Monotonically merge workflow policy when an actual asset is invoked.""" - if not runtime.get("enabled") or kind not in {"tool", "skill", "subagent"}: - return [] - known_tags = set(str(item) for item in tags if str(item)) - known_tags.update( - str(item) - for item in runtime.get("asset_tags", {}).get(kind, {}).get(asset_id, []) - if str(item) - ) - activated = set(runtime.setdefault("activated_workflows", [])) - events: list[dict[str, Any]] = [] - for candidate in runtime.get("activation_candidates", []): - ref = f"{candidate.get('pack_id')}:{candidate.get('workflow_id')}" - if ref in activated: - continue - matched_tags: set[str] = set() - matched = False - for trigger in candidate.get("asset_triggers", []): - if trigger.get("kind") != kind: - continue - if asset_id in set(trigger.get("ids", [])): - matched = True - tag_hits = known_tags & set(trigger.get("tags_any", [])) - if tag_hits: - matched = True - matched_tags.update(tag_hits) - if not matched: - continue +def register_runtime_asset_tags(_runtime: dict[str, Any], **_kwargs: Any) -> None: + return None - fragment = candidate.get("pack") or {} - current = next( - ( - pack - for pack in runtime.get("packs", []) - if pack.get("pack_id") == fragment.get("pack_id") - ), - None, - ) - if current is None: - current = fragment - runtime.setdefault("packs", []).append(current) - else: - for field, key in ( - ("workflows", "id"), - ("constraints", "id"), - ("concepts", "id"), - ("relations", "id"), - ): - existing_ids = {item.get(key) for item in current.get(field, [])} - current.setdefault(field, []).extend( - item for item in fragment.get(field, []) if item.get(key) not in existing_ids - ) - if fragment.get("version_id"): - current["version_id"] = fragment["version_id"] - level = str(candidate.get("review_level") or "none") - if REVIEW_RANK.get(level, 0) > REVIEW_RANK.get(runtime.get("review_level"), 0): - runtime["review_level"] = level - activated.add(ref) - runtime["activated_workflows"] = sorted(activated) - event = { - "type": "ontology_activation", - "status": "completed", - "governance_run_id": runtime.get("governance_run_id"), - "source": kind, - "asset_kind": kind, - "asset_id": asset_id, - "asset_tags": sorted(known_tags), - "matched_tags": sorted(matched_tags), - "pack_id": candidate.get("pack_id"), - "workflow_id": candidate.get("workflow_id"), - "workflow_name": candidate.get("workflow_name"), - "review_level": level, - } - runtime.setdefault("runtime_events", []).append(event) - events.append(event) - return events +def activate_runtime_for_asset(_runtime: dict[str, Any], **_kwargs: Any) -> list[dict[str, Any]]: + return [] -def render_runtime_prompt(runtime: dict[str, Any]) -> str: - """Render the bounded ontology segment injected into the system prompt.""" - if not runtime.get("enabled"): - return "" - compact_packs = [] - for pack in runtime.get("packs", []): - compact_pack = { - "id": pack["pack_id"], - "version": pack["version"], - "concepts": ( - list(pack.get("concepts", [])) - if pack.get("config", {}).get("injection_enabled", True) - else [] - ), - "relations": ( - list(pack.get("relations", [])) - if pack.get("config", {}).get("injection_enabled", True) - else [] - ), - "workflows": pack.get("workflows", []), - "constraints": [ - { - "id": rule["id"], - "target": rule["target"], - "mode": rule["mode"], - "message": rule["message"], - "suggestion": rule.get("suggestion", ""), - "prerequisite_tools": rule.get("prerequisite_tools", []), - } - for rule in pack.get("constraints", []) - ], - } - char_budget = int(pack.get("config", {}).get("token_budget", 2500)) * 4 - while len(json.dumps(compact_pack, ensure_ascii=False)) > char_budget: - if compact_pack["relations"]: - compact_pack["relations"].pop() - continue - if compact_pack["concepts"]: - compact_pack["concepts"].pop() - remaining = {item["id"] for item in compact_pack["concepts"]} - compact_pack["relations"] = [ - relation - for relation in compact_pack["relations"] - if relation["subject"] in remaining and relation["object"] in remaining - ] - continue - break - compact_packs.append(compact_pack) - compact = {"packs": compact_packs} - return ( - "\n" - "本轮已启用领域本体校验。以下内容是可执行的领域契约;工具门禁会独立执行," - "请在调用前主动满足参数、前置证据与工作流要求。不要虚构证据。\n" - + json.dumps(compact, ensure_ascii=False, separators=(",", ":")) - + "\n" - ) +def render_runtime_prompt(_runtime: dict[str, Any]) -> str: + return "" diff --git a/src/backend/core/ontology/runtime_policy.py b/src/backend/core/ontology/runtime_policy.py index e0f56b8d..09654199 100644 --- a/src/backend/core/ontology/runtime_policy.py +++ b/src/backend/core/ontology/runtime_policy.py @@ -1,158 +1,21 @@ -"""Resolve a user's ontology preference into a runtime policy and asset tags.""" +"""Every execution path gets the disabled runtime.""" + from __future__ import annotations -from core.ontology import service as ontology_service + from typing import Any -from core.infra.exceptions import ServiceUnavailableError -from core.ontology.runtime_pack import register_runtime_asset_tags -from sqlalchemy.orm import Session -import logging -logger = logging.getLogger(__name__) + def disabled_ontology_runtime() -> dict[str, Any]: - """Return the canonical pass-through policy used when the user opts out.""" return {"enabled": False, "packs": [], "review_level": "none"} -def build_ontology_runtime_for_preference( - *, - enabled: bool, - task: str, - db: Session, - pack_ids: list[str] | None = None, -) -> tuple[bool, dict[str, Any]]: - """Resolve the effective ontology switch and compile its runtime policy. - - A user's stored opt-in is subordinate to the administrator-controlled pack - availability. Disabling the selected/default packs is therefore an - intentional global off switch, not a runtime-policy failure. Once an - active pack exists, compilation remains fail-closed so malformed policy - data cannot silently bypass validation. - """ - if not enabled: - return False, disabled_ontology_runtime() - - service = ontology_service.OntologyService(db) - selected = pack_ids or None - if not service.repo.get_active_versions(selected): - return False, disabled_ontology_runtime() - - runtime = service.build_runtime(task=task, pack_ids=selected) - if not runtime.get("enabled"): - raise ServiceUnavailableError("本体校验运行时策略编译失败") - return True, runtime - - -def runtime_needs_asset_tag_lookup( - runtime: dict[str, Any], - *, - kind: str, - asset_id: str, -) -> bool: - """Return whether tags could activate an otherwise-unmatched workflow.""" - activated = set(runtime.get("activated_workflows", [])) - for candidate in runtime.get("activation_candidates", []): - ref = f"{candidate.get('pack_id')}:{candidate.get('workflow_id')}" - if ref in activated: - continue - triggers = [ - trigger - for trigger in candidate.get("asset_triggers", []) - if trigger.get("kind") == kind - ] - if not triggers: - continue - if any(asset_id in set(trigger.get("ids", [])) for trigger in triggers): - continue - if any(trigger.get("tags_any") for trigger in triggers): - return True - return False - - -def resolve_runtime_asset_tags( - *, - runtime: dict[str, Any], - kind: str, - asset_id: str, - user_id: str, -) -> list[str]: - """Resolve and request-cache one invoked asset's trusted ontology tags. - - The runtime catalog doubles as a per-turn positive/negative cache. Metadata - failures are fail-closed so an asset cannot execute without the governance - policy that its server-authoritative tags may activate. - """ - if not runtime.get("enabled") or kind not in {"tool", "skill", "subagent"} or not asset_id: - return [] - catalog = runtime.setdefault("asset_tags", {}).setdefault(kind, {}) - if asset_id in catalog: - return list(catalog[asset_id]) - if not runtime_needs_asset_tag_lookup(runtime, kind=kind, asset_id=asset_id): - register_runtime_asset_tags(runtime, kind=kind, asset_id=asset_id, tags=[]) - return [] - - try: - from core.db.engine import SessionLocal - - with SessionLocal() as db: - tags = ontology_service.OntologyService(db).resolve_asset_tags( - kind=kind, - asset_id=asset_id, - user_id=user_id, - ) - except Exception as exc: # noqa: BLE001 - logger.warning( - "[ontology] lazy asset tag lookup failed kind=%s asset_id=%s", - kind, - asset_id, - exc_info=True, - ) - raise ServiceUnavailableError("本体资产标签读取失败,本次资产调用已停止") from exc - - register_runtime_asset_tags(runtime, kind=kind, asset_id=asset_id, tags=tags) - return list(runtime["asset_tags"][kind][asset_id]) - - -def build_user_ontology_runtime( - *, - user_id: str, - task: str, - db: Session | None = None, -) -> tuple[bool, dict[str, Any]]: - """Resolve one user's opt-in and compile the task-scoped runtime policy. +def build_ontology_runtime_for_preference(**_kwargs: Any) -> tuple[bool, dict[str, Any]]: + return False, disabled_ontology_runtime() - Web chat, channels, automation, batch, plan mode, and autonomous loops all - use this resolver so independent execution paths cannot silently bypass the - setting selected by the user. - """ - from core.auth.users import UserService - owns_session = db is None - if owns_session: - from core.db.engine import SessionLocal +def build_user_ontology_runtime(**_kwargs: Any) -> tuple[bool, dict[str, Any]]: + return False, disabled_ontology_runtime() - db = SessionLocal() - assert db is not None - try: - from core.ontology.policy import user_can_use_ontology_validation - if not user_can_use_ontology_validation(db, user_id): - return False, disabled_ontology_runtime() - settings = UserService(db).get_user_settings(user_id) - opted_in = bool(settings.get("ontology_enabled", False)) - if not opted_in: - return False, disabled_ontology_runtime() - pack_ids = settings.get("ontology_pack_ids") - selected = ( - [str(item) for item in pack_ids if str(item).strip()] - if isinstance(pack_ids, list) - else None - ) - return build_ontology_runtime_for_preference( - enabled=opted_in, - task=task, - db=db, - pack_ids=selected or None, - ) - finally: - if owns_session: - db.close() +def resolve_runtime_asset_tags(**_kwargs: Any) -> list[str]: + return [] diff --git a/src/backend/core/ontology/schemas.py b/src/backend/core/ontology/schemas.py deleted file mode 100644 index 0791a902..00000000 --- a/src/backend/core/ontology/schemas.py +++ /dev/null @@ -1,190 +0,0 @@ -"""Typed, JSON-serializable schema for ontology Domain Packs. - -The schema deliberately stays lightweight. A Domain Pack is an operational -contract for an agent harness, not an OWL reasoner: concepts and relations make -the vocabulary explicit, constraints are executable JSON-Schema fragments, -and workflows decide which gates/review levels apply to a task. -""" - -from __future__ import annotations - -from typing import Any, Literal - -from pydantic import BaseModel, ConfigDict, Field, model_validator - -Identifier = str -RiskLevel = Literal["low", "medium", "high"] -RuleMode = Literal["log", "enforce"] -ReviewLevel = Literal["none", "checkpoint", "committee"] -AssetKind = Literal["tool", "skill", "subagent"] - - -class StrictModel(BaseModel): - model_config = ConfigDict(extra="forbid", str_strip_whitespace=True) - - -class Concept(StrictModel): - id: Identifier = Field(pattern=r"^[A-Za-z][A-Za-z0-9_.:-]{0,127}$") - name: str = Field(min_length=1, max_length=255) - aliases: list[str] = Field(default_factory=list) - definition: str = Field(min_length=1, max_length=4000) - parent_id: Identifier | None = None - closed_values: list[str] = Field(default_factory=list) - tags: list[str] = Field(default_factory=list) - risk: RiskLevel = "low" - - -class Relation(StrictModel): - id: Identifier = Field(pattern=r"^[A-Za-z][A-Za-z0-9_.:-]{0,127}$") - subject: Identifier - predicate: str = Field(min_length=1, max_length=128) - object: Identifier - description: str = Field(default="", max_length=2000) - min_cardinality: int | None = Field(default=None, ge=0) - max_cardinality: int | None = Field(default=None, ge=0) - forbidden: bool = False - - @model_validator(mode="after") - def validate_cardinality(self) -> "Relation": - if ( - self.min_cardinality is not None - and self.max_cardinality is not None - and self.min_cardinality > self.max_cardinality - ): - raise ValueError("min_cardinality cannot exceed max_cardinality") - return self - - -class ConstraintTarget(StrictModel): - kind: Literal["tool", "tool_parameter", "output"] - tool: str | None = Field(default=None, max_length=255) - parameter: str | None = Field(default=None, max_length=255) - output_tag: str | None = Field(default=None, max_length=128) - - @model_validator(mode="after") - def validate_target(self) -> "ConstraintTarget": - if self.kind in {"tool", "tool_parameter"} and not self.tool: - raise ValueError(f"{self.kind} target requires tool") - if self.kind == "tool_parameter" and not self.parameter: - raise ValueError("tool_parameter target requires parameter") - if self.kind == "output" and not self.output_tag: - raise ValueError("output target requires output_tag") - return self - - -class Constraint(StrictModel): - id: Identifier = Field(pattern=r"^[A-Za-z][A-Za-z0-9_.:-]{0,127}$") - name: str = Field(min_length=1, max_length=255) - target: ConstraintTarget - schema_: dict[str, Any] = Field(default_factory=dict, alias="schema") - concept_id: Identifier | None = None - requires_citations: bool = False - prerequisite_tools: list[str] = Field(default_factory=list) - mode: RuleMode = "log" - risk: RiskLevel = "low" - message: str = Field(min_length=1, max_length=2000) - suggestion: str = Field(default="", max_length=2000) - enabled: bool = True - - -class AssetTrigger(StrictModel): - """Activate a workflow when a governed runtime asset is actually invoked.""" - - kind: AssetKind - ids: list[str] = Field(default_factory=list) - tags_any: list[str] = Field(default_factory=list) - - @model_validator(mode="after") - def validate_matcher(self) -> "AssetTrigger": - self.ids = [item for item in self.ids if item] - self.tags_any = [item for item in self.tags_any if item] - if not self.ids and not self.tags_any: - raise ValueError("asset trigger requires ids or tags_any") - return self - - -class Workflow(StrictModel): - id: Identifier = Field(pattern=r"^[A-Za-z][A-Za-z0-9_.:-]{0,127}$") - name: str = Field(min_length=1, max_length=255) - triggers: list[str] = Field(default_factory=list) - asset_triggers: list[AssetTrigger] = Field(default_factory=list) - required_tools: list[str] = Field(default_factory=list) - forbidden_tools: list[str] = Field(default_factory=list) - output_tags: list[str] = Field(default_factory=list) - review_level: ReviewLevel = "none" - risk: RiskLevel = "low" - - @model_validator(mode="after") - def validate_activation(self) -> "Workflow": - if not self.triggers and not self.asset_triggers: - raise ValueError("workflow requires text triggers or asset_triggers") - return self - - -class PackConfig(StrictModel): - injection_enabled: bool = True - max_concepts: int = Field(default=12, ge=1, le=50) - token_budget: int = Field(default=2500, ge=256, le=16000) - committee_size: int = Field(default=3, ge=2, le=5) - repeated_denial_threshold: int = Field(default=2, ge=1, le=10) - circuit_breaker_threshold: int = Field(default=5, ge=2, le=50) - allow_unresolved_tools: bool = False - - -class OntologyPackDocument(StrictModel): - schema_version: Literal["1.0"] = "1.0" - pack_id: Identifier = Field(pattern=r"^[a-z][a-z0-9_-]{1,63}$") - name: str = Field(min_length=1, max_length=255) - version: str = Field(pattern=r"^[0-9]+\.[0-9]+\.[0-9]+(?:[-+][A-Za-z0-9.-]+)?$") - domain: str = Field(min_length=1, max_length=128) - description: str = Field(default="", max_length=4000) - config: PackConfig = Field(default_factory=PackConfig) - concepts: list[Concept] = Field(default_factory=list) - relations: list[Relation] = Field(default_factory=list) - constraints: list[Constraint] = Field(default_factory=list) - workflows: list[Workflow] = Field(default_factory=list) - - @model_validator(mode="after") - def validate_references(self) -> "OntologyPackDocument": - self._require_unique("concept", [item.id for item in self.concepts]) - self._require_unique("relation", [item.id for item in self.relations]) - self._require_unique("constraint", [item.id for item in self.constraints]) - self._require_unique("workflow", [item.id for item in self.workflows]) - - concept_ids = {item.id for item in self.concepts} - for concept in self.concepts: - if concept.parent_id and concept.parent_id not in concept_ids: - raise ValueError( - f"concept {concept.id} references unknown parent {concept.parent_id}" - ) - for relation in self.relations: - for endpoint in (relation.subject, relation.object): - if endpoint not in concept_ids: - raise ValueError( - f"relation {relation.id} references unknown concept {endpoint}" - ) - for constraint in self.constraints: - if constraint.concept_id and constraint.concept_id not in concept_ids: - raise ValueError( - f"constraint {constraint.id} references unknown concept " - f"{constraint.concept_id}" - ) - self._validate_parent_cycles() - return self - - @staticmethod - def _require_unique(kind: str, ids: list[str]) -> None: - duplicates = sorted({item for item in ids if ids.count(item) > 1}) - if duplicates: - raise ValueError(f"duplicate {kind} ids: {', '.join(duplicates)}") - - def _validate_parent_cycles(self) -> None: - parent_by_id = {item.id: item.parent_id for item in self.concepts} - for concept_id in parent_by_id: - seen: set[str] = set() - current: str | None = concept_id - while current: - if current in seen: - raise ValueError(f"concept hierarchy contains a cycle at {current}") - seen.add(current) - current = parent_by_id.get(current) diff --git a/src/backend/core/ontology/service.py b/src/backend/core/ontology/service.py deleted file mode 100644 index 462b3166..00000000 --- a/src/backend/core/ontology/service.py +++ /dev/null @@ -1,440 +0,0 @@ -"""Manage ontology packs, drafts and versions, and resolve asset tags.""" -from __future__ import annotations -from core.infra.time import utc_now -import hashlib -import json -import uuid -from typing import Any -from core.config.display_names import TOOL_DISPLAY_NAMES -from core.db.models import AdminMcpServer, AdminSkill, OntologyDraft, OntologyPack, OntologyPackVersion -from core.db.repository import OntologyRepository -from core.infra.exceptions import BadRequestError, ResourceNotFoundError -from core.ontology.schemas import OntologyPackDocument -from core.ontology.validator import DomainPackValidator -from core.ontology.runtime_pack import build_runtime_payload -from sqlalchemy import or_ -from sqlalchemy.orm import Session - - -def normalize_ontology_tags(value: Any) -> list[str]: - if isinstance(value, str): - items = value.replace(",", ",").split(",") - elif isinstance(value, (list, tuple, set)): - items = value - else: - return [] - return sorted({str(item).strip() for item in items if str(item).strip()}) - - -class OntologyService: - def __init__(self, db: Session): - self.db = db - self.repo = OntologyRepository(db) - self.validator = DomainPackValidator() - - def validate_document( - self, - payload: dict[str, Any], - ) -> tuple[OntologyPackDocument | None, dict[str, Any]]: - schemas, known = self._known_tool_schemas() - document, report = self.validator.validate( - payload, - tool_schemas=schemas, - known_tools=known, - ) - return document, report.as_dict() - - def create_version( - self, - payload: dict[str, Any], - *, - actor_id: str | None = None, - activate: bool = False, - ): - document, report = self.validate_document(payload) - if document is None or not report["valid"]: - raise BadRequestError("Domain Pack 校验失败", data=report) - existing = self.repo.get_pack_version(document.pack_id, document.version) - if existing: - raise BadRequestError( - f"Domain Pack {document.pack_id} 的版本 {document.version} 已存在" - ) - pack = self.repo.get_pack(document.pack_id) - if pack is not None and self.repo.get_working_draft(document.pack_id) is not None: - raise BadRequestError("该领域包已有工作草稿,请先发布或放弃草稿后再导入新版本") - if pack is None: - pack = self.repo.create_pack( - { - "pack_id": document.pack_id, - "name": document.name, - "domain": document.domain, - "description": document.description, - "is_default": False, - "created_by": actor_id, - } - ) - canonical = document.model_dump(mode="json", by_alias=True) - checksum = self._checksum(canonical) - version = self.repo.create_version( - { - "version_id": f"ontov_{uuid.uuid4().hex[:16]}", - "pack_id": document.pack_id, - "version": document.version, - "content": canonical, - "checksum": checksum, - "status": "draft", - "validation_report": report, - "created_by": actor_id, - } - ) - if activate: - self._sync_pack_metadata(pack, document) - self.repo.activate(pack, version) - self.db.commit() - self.db.refresh(version) - return version - - def save_working_draft( - self, - pack_id: str, - payload: dict[str, Any], - *, - draft_version_id: str | None = None, - expected_checksum: str | None = None, - actor_id: str | None = None, - ) -> tuple[OntologyPackVersion, bool]: - """Create the only working draft for a pack, or update that draft in place.""" - - pack = self.repo.get_pack(pack_id) - if pack is None: - raise ResourceNotFoundError("ontology_pack", pack_id) - document, report = self.validate_document(payload) - if document is None or not report["valid"]: - raise BadRequestError("Domain Pack 校验失败", data=report) - if document.pack_id != pack_id: - raise BadRequestError("请求路径与 Domain Pack 文档中的 pack_id 不一致") - - canonical = document.model_dump(mode="json", by_alias=True) - checksum = self._checksum(canonical) - working_draft = self.repo.get_working_draft(pack_id) - created = working_draft is None - - if working_draft is None: - if draft_version_id is not None: - raise BadRequestError("工作草稿已不存在,请刷新后重试") - existing = self.repo.get_pack_version(pack_id, document.version) - if existing is not None: - raise BadRequestError(f"Domain Pack {pack_id} 的版本 {document.version} 已存在") - working_draft = self.repo.create_version( - { - "version_id": f"ontov_{uuid.uuid4().hex[:16]}", - "pack_id": pack_id, - "version": document.version, - "content": canonical, - "checksum": checksum, - "status": "draft", - "validation_report": report, - "created_by": actor_id, - } - ) - else: - if draft_version_id != working_draft.version_id: - raise BadRequestError("该领域包已有工作草稿,请刷新后在现有草稿上继续编辑") - if not expected_checksum: - raise BadRequestError("更新工作草稿时必须提供最新校验和") - if expected_checksum != working_draft.checksum: - raise BadRequestError("工作草稿已被其他管理员更新,请刷新后重新编辑") - if document.version != working_draft.version: - raise BadRequestError("工作草稿创建后不能修改版本号") - self.repo.update_working_draft( - working_draft, - { - "content": canonical, - "checksum": checksum, - "validation_report": report, - }, - ) - - if pack.active_version_id is None: - self._sync_pack_metadata(pack, document) - self.db.commit() - self.db.refresh(working_draft) - return working_draft, created - - def discard_working_draft(self, pack_id: str, version_id: str) -> None: - """Discard an unpublished draft and release linked evolution candidates.""" - - pack = self.repo.get_pack(pack_id) - if pack is None: - raise ResourceNotFoundError("ontology_pack", pack_id) - working_draft = self.repo.get_working_draft(pack_id) - if working_draft is None or working_draft.version_id != version_id: - raise ResourceNotFoundError("ontology_working_draft", version_id) - - now = utc_now() - linked_candidates = ( - self.db.query(OntologyDraft).filter(OntologyDraft.pack_id == pack_id).all() - ) - for candidate in linked_candidates: - proposal = dict(candidate.proposal or {}) - if proposal.get("materialized_version_id") != version_id: - continue - proposal.pop("materialized_version_id", None) - candidate.proposal = proposal - candidate.updated_at = now - self.repo.delete_working_draft(working_draft) - self.db.commit() - - def activate(self, pack_id: str, version_id: str): - pack = self.repo.get_pack(pack_id) - if not pack: - raise ResourceNotFoundError("ontology_pack", pack_id) - version = self.repo.get_version(version_id) - if not version or version.pack_id != pack_id: - raise ResourceNotFoundError("ontology_pack_version", version_id) - if not (version.validation_report or {}).get("valid", False): - raise BadRequestError("未通过校验的 Domain Pack 版本不能激活") - if version.status == "active" and pack.active_version_id == version_id: - return version - document = OntologyPackDocument.model_validate(version.content) - self._sync_pack_metadata(pack, document) - self.repo.activate(pack, version) - self.db.commit() - return version - - @staticmethod - def _checksum(canonical: dict[str, Any]) -> str: - return hashlib.sha256( - json.dumps(canonical, sort_keys=True, ensure_ascii=False).encode("utf-8") - ).hexdigest() - - def _sync_pack_metadata(self, pack: OntologyPack, document: OntologyPackDocument) -> None: - self.repo.update_pack( - pack, - { - "name": document.name, - "domain": document.domain, - "description": document.description, - }, - ) - - def set_pack_flags( - self, - pack_id: str, - *, - is_enabled: bool | None = None, - is_default: bool | None = None, - ): - pack = self.repo.get_pack(pack_id) - if not pack: - raise ResourceNotFoundError("ontology_pack", pack_id) - if is_default: - self.db.query(type(pack)).filter(type(pack).pack_id != pack_id).update( - {type(pack).is_default: False}, synchronize_session=False - ) - data: dict[str, Any] = {} - if is_enabled is not None: - data["is_enabled"] = is_enabled - if is_default is not None: - data["is_default"] = is_default - self.repo.update_pack(pack, data) - self.db.commit() - self.db.refresh(pack) - return pack - - def build_runtime( - self, - *, - task: str, - pack_ids: list[str] | None = None, - ) -> dict[str, Any]: - documents: list[OntologyPackDocument] = [] - version_ids: list[str] = [] - version_id_by_pack: dict[str, str] = {} - for version in self.repo.get_active_versions(pack_ids): - try: - document = OntologyPackDocument.model_validate(version.content) - documents.append(document) - version_ids.append(version.version_id) - version_id_by_pack[document.pack_id] = version.version_id - except Exception: - continue - runtime = build_runtime_payload(documents, task) - runtime["version_ids"] = version_ids - for pack in runtime.get("packs", []): - pack["version_id"] = version_id_by_pack.get(pack.get("pack_id")) - for candidate in runtime.get("activation_candidates", []): - version_id = version_id_by_pack.get(candidate.get("pack_id")) - candidate["version_id"] = version_id - if candidate.get("pack") is not None: - candidate["pack"]["version_id"] = version_id - return runtime - - def list_asset_tag_options(self, asset_kind: str) -> list[dict[str, Any]]: - """List controlled concept tags that actually activate workflows for one asset kind. - - A concept existing in a Domain Pack does not by itself imply runtime activation. Only - ``workflow.asset_triggers[].tags_any`` creates that link, so the form selector must be - built from those declarations instead of exposing every concept indiscriminately. - """ - if asset_kind not in {"tool", "skill", "subagent"}: - raise BadRequestError("不支持的本体资产类型") - - entries: dict[str, dict[str, Any]] = {} - for version in self.repo.get_active_versions(): - try: - document = OntologyPackDocument.model_validate(version.content) - except Exception: - continue - concepts = {f"ontology:{concept.id}": concept for concept in document.concepts} - for workflow in document.workflows: - for trigger in workflow.asset_triggers: - if trigger.kind != asset_kind: - continue - for value in trigger.tags_any: - concept = concepts.get(value) - if concept is None: - continue - entry = entries.setdefault( - value, - { - "value": value, - "concept_id": concept.id, - "concept_name": concept.name, - "definition": concept.definition, - "risk": concept.risk, - "packs": [], - "workflows": [], - }, - ) - pack_info = { - "pack_id": document.pack_id, - "pack_name": document.name, - "domain": document.domain, - "version": document.version, - } - if pack_info not in entry["packs"]: - entry["packs"].append(pack_info) - workflow_info = { - "workflow_ref": f"{document.pack_id}:{workflow.id}", - "workflow_name": workflow.name, - "review_level": workflow.review_level, - "risk": workflow.risk, - } - if workflow_info not in entry["workflows"]: - entry["workflows"].append(workflow_info) - - for entry in entries.values(): - entry["packs"].sort(key=lambda item: (item["pack_name"], item["version"])) - entry["workflows"].sort(key=lambda item: item["workflow_ref"]) - return sorted(entries.values(), key=lambda item: (item["concept_name"], item["value"])) - - def resolve_asset_tags( - self, - *, - kind: str, - asset_id: str, - user_id: str, - ) -> list[str]: - """Resolve trusted ontology tags for one asset immediately before use. - - Only metadata columns are selected. Global assets must be enabled; - private assets are visible only to their owner and may still be used by - an explicit binding even when their personal catalog switch is off. - """ - if kind == "skill": - owner_filter = ( - or_( - AdminSkill.owner_user_id.is_(None), - AdminSkill.owner_user_id == user_id, - ) - if user_id - else AdminSkill.owner_user_id.is_(None) - ) - enabled_filter = or_( - AdminSkill.owner_user_id.isnot(None), - AdminSkill.is_enabled.is_(True), - ) - row = ( - self.db.query(AdminSkill.tags) - .filter( - AdminSkill.skill_id == asset_id, - owner_filter, - enabled_filter, - ) - .one_or_none() - ) - if row: - return normalize_ontology_tags(row.tags) - # A DB row that exists but is outside this user's scope must never - # fall through to the process-global loader (which indexes private - # skills for materialization). Only true filesystem/built-in skills - # may use the loader metadata fallback. - db_row_exists = ( - self.db.query(AdminSkill.skill_id).filter(AdminSkill.skill_id == asset_id).first() - is not None - ) - if db_row_exists: - return [] - from core.capabilities.skills.loader_factory import get_skill_loader - - metadata = get_skill_loader().load_all_metadata().get(asset_id) - return normalize_ontology_tags(getattr(metadata, "tags", [])) - - if kind == "tool": - owner_filter = ( - or_( - AdminMcpServer.owner_user_id.is_(None), - AdminMcpServer.owner_user_id == user_id, - ) - if user_id - else AdminMcpServer.owner_user_id.is_(None) - ) - enabled_filter = or_( - AdminMcpServer.owner_user_id.isnot(None), - AdminMcpServer.is_enabled.is_(True), - ) - rows = ( - self.db.query(AdminMcpServer.extra_config, AdminMcpServer.tools_json) - .filter(owner_filter, enabled_filter) - .all() - ) - tags: set[str] = set() - for row in rows: - extra = row.extra_config or {} - tool_tag_map = ( - extra.get("tool_tags") if isinstance(extra.get("tool_tags"), dict) else {} - ) - for tool in row.tools_json or []: - if not isinstance(tool, dict) or str(tool.get("name") or "") != asset_id: - continue - tags.update(normalize_ontology_tags(extra.get("ontology_tags"))) - tags.update(normalize_ontology_tags(tool.get("tags"))) - tags.update(normalize_ontology_tags(tool.get("ontology_tags"))) - tags.update(normalize_ontology_tags(tool_tag_map.get(asset_id))) - return sorted(tags) - - if kind == "subagent": - from core.capabilities.agents.edition import UserAgentService - - try: - agent = UserAgentService(self.db).get_by_id(asset_id, user_id=user_id or None) - except (LookupError, PermissionError): - return [] - return normalize_ontology_tags(agent.get("ontology_tags")) - - return [] - - def _known_tool_schemas(self) -> tuple[dict[str, dict[str, Any]], set[str]]: - known = set(TOOL_DISPLAY_NAMES) - schemas: dict[str, dict[str, Any]] = {} - rows = self.db.query(AdminMcpServer).all() - for row in rows: - for tool in row.tools_json or []: - if not isinstance(tool, dict) or not tool.get("name"): - continue - name = str(tool["name"]) - known.add(name) - if tool.get("inputSchema"): - schemas[name] = {"inputSchema": tool["inputSchema"]} - return schemas, known diff --git a/src/backend/core/ontology/tool_hook.py b/src/backend/core/ontology/tool_hook.py index 1aaa00ad..fec454f9 100644 --- a/src/backend/core/ontology/tool_hook.py +++ b/src/backend/core/ontology/tool_hook.py @@ -1,183 +1,5 @@ -"""Ontology tool admission and audit through neutral execution hooks.""" -import asyncio -import json -import logging -import time -from pathlib import Path -from typing import Any -from core.harness.context_items import reminder_item -from core.harness.events import thaw_value -from core.harness.hooks import Decision, DecisionKind, HookSpec, HookStage +"""No tool-call gate is registered.""" -logger = logging.getLogger(__name__) - -class OntologyToolHook: - def __init__(self, runtime=None): - self.runtime = dict(runtime or {}) - self.completed_tools = set() - self.denial_counts = {} - - async def __call__(self, invocation): - if invocation.stage == HookStage.AFTER_TOOL: - result = invocation.data.get("tool_result") or {} - if result.get("state") not in {"error", "denied", "interrupted"}: - self.completed_tools.add(invocation.operation_name) - return Decision() - from core.ontology.runtime_pack import activate_runtime_for_asset, render_runtime_prompt - from core.ontology.runtime_gates import evaluate_tool_call - from core.ontology.runtime_policy import resolve_runtime_asset_tags - runtime = thaw_value(invocation.data.get("ontology_runtime") or self.runtime) - if not runtime.get("enabled"): - return Decision() - identity = invocation.data.get("identity", {}) - started = time.monotonic() - tool_name = invocation.operation_name - tool_input = thaw_value(invocation.data.get("tool_args") or {}) - asset_kind, asset_id = self._resolve_invoked_asset(tool_name, tool_input) - await asyncio.to_thread(resolve_runtime_asset_tags, runtime=runtime, kind=asset_kind, - asset_id=asset_id, user_id=str(identity.get("user_id") or "")) - activations = activate_runtime_for_asset(runtime, kind=asset_kind, asset_id=asset_id) - items = list(invocation.data.get("context_items") or ()) - if activations: - contract = render_runtime_prompt(runtime) - if contract: - items.append(reminder_item( - "检测到受领域本体治理的运行时资产,策略已升级且本轮不可降级。\n" + contract, - "harness:ontology_gate")) - await self._audit_activations(identity, runtime, activations) - decision = evaluate_tool_call(runtime, tool_name=tool_name, tool_input=tool_input, - completed_tools=self.completed_tools) - patch = {"ontology_runtime": runtime, "context_items": items} - if decision.allowed: - await self._audit(identity, runtime, tool_name, decision, started) - return Decision.modify(patch) - key = ",".join(decision.matched_rule_ids) or tool_name - count = self.denial_counts[key] = self.denial_counts.get(key, 0) + 1 - configs = [pack.get("config", {}) for pack in runtime.get("packs", [])] - strategy = min((int(item.get("repeated_denial_threshold", 2)) for item in configs), default=2) - breaker = min((int(item.get("circuit_breaker_threshold", 5)) for item in configs), default=5) - guidance = list(decision.suggestions) - if count >= strategy: - guidance.append("同一规则已重复拦截,请改变执行策略,不要原样重试。") - if count >= breaker: - guidance.append("本体门禁已触发熔断;停止调用该工具,并向用户说明缺失条件。") - await self._audit(identity, runtime, tool_name, decision, started, - denial_count=count, circuit_breaker=count >= breaker) - if count == strategy and identity.get("chat_id"): - try: - from core.ontology.evolution import schedule_ontology_evolution - schedule_ontology_evolution(user_id=str(identity.get("user_id") or "system")) - except Exception as exc: - logger.warning("[ontology] evolution scheduling failed: %s", exc) - payload = {"code": "ONTOLOGY_GATE_DENIED", "tool": tool_name, - "violations": decision.violations, "suggestions": guidance, - "denial_count": count, "circuit_breaker": count >= breaker} - patch["tool_response"] = {"content": [{"type": "text", "text": json.dumps(payload, ensure_ascii=False)}], - "state": "denied", "metadata": {"ontology_gate": payload}} - return Decision(DecisionKind.REJECT, patch=patch, reason="ontology gate denied tool") - - async def _audit( - self, - identity: dict, - runtime: dict[str, Any], - tool_name: str, - decision, - started: float, - *, - denial_count: int = 0, - circuit_breaker: bool = False, - ) -> None: # noqa: ANN001 - if not decision.violations and not decision.matched_rule_ids: - return - runtime.setdefault("runtime_events", []).append( - { - "type": "ontology_gate", - "status": "completed", - "governance_run_id": runtime.get("governance_run_id"), - "stage": "tool", - "decision": decision.decision, - "tool_name": tool_name, - "matched_rule_ids": list(decision.matched_rule_ids), - "violations": list(decision.violations), - "suggestions": list(decision.suggestions), - "denial_count": denial_count, - "circuit_breaker": circuit_breaker, - } - ) - try: - from core.ontology.audit_events import record_enforcement_event - - first = decision.violations[0] if decision.violations else {} - pack_id = first.get("pack_id") - version_id = None - for pack in runtime.get("packs", []): - if pack.get("pack_id") == pack_id: - version_id = pack.get("version_id") - break - payload = { - "user_id": identity.get("user_id"), - "chat_id": identity.get("chat_id"), - "pack_id": pack_id, - "version_id": version_id, - "rule_id": first.get("rule_id") - or (decision.matched_rule_ids[0] if decision.matched_rule_ids else None), - "stage": "tool", - "event_type": "tool_call_gate", - "decision": decision.decision, - "mode": first.get("mode", "enforce"), - "target": tool_name, - "latency_ms": round((time.monotonic() - started) * 1000), - "details": { - "governance_run_id": runtime.get("governance_run_id"), - "violations": decision.violations, - "matched_rule_ids": decision.matched_rule_ids, - "suggestions": decision.suggestions, - "denial_count": denial_count, - "circuit_breaker": circuit_breaker, - }, - } - await asyncio.to_thread(record_enforcement_event, payload) - except Exception as exc: # noqa: BLE001 - logger.warning("[ontology] audit event persistence failed: %s", exc) - - - @staticmethod - def _resolve_invoked_asset(tool_name: str, tool_input: dict[str, Any]) -> tuple[str, str]: - if tool_name == "call_subagent" and tool_input.get("agent_id"): - return "subagent", str(tool_input["agent_id"]) - if tool_name == "view_text_file": - file_path = str(tool_input.get("file_path") or "") - path = Path(file_path) - if path.name == "SKILL.md" and path.parent.name: - return "skill", path.parent.name - return "tool", tool_name - - - @staticmethod - async def _audit_activations( - identity: dict, - runtime: dict[str, Any], - activations: list[dict[str, Any]], - ) -> None: - try: - from core.ontology.audit_events import record_runtime_activation - - for event in activations: - await asyncio.to_thread( - record_runtime_activation, - event, - runtime, - user_id=identity.get("user_id"), - chat_id=identity.get("chat_id"), - ) - except Exception as exc: # noqa: BLE001 - logger.warning("[ontology] activation audit persistence failed: %s", exc) - - - -def ontology_tool_hooks(runtime): - policy = OntologyToolHook(runtime) - return (HookSpec("ontology_admission", HookStage.BEFORE_TOOL, policy, priority=500, - timeout_ms=30000, mutable_fields=frozenset({"ontology_runtime", "context_items", "tool_response"})), - HookSpec("ontology_completion", HookStage.AFTER_TOOL, policy, priority=300)) +def ontology_tool_hooks(_runtime): + return [] diff --git a/src/backend/core/ontology/validation_models.py b/src/backend/core/ontology/validation_models.py deleted file mode 100644 index cda206d4..00000000 --- a/src/backend/core/ontology/validation_models.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Validation issues, reports and gate decisions.""" -from __future__ import annotations -from dataclasses import asdict, dataclass, field -from typing import Any - -@dataclass(slots=True) -class ValidationIssue: - severity: str - path: str - message: str - - -@dataclass(slots=True) -class ValidationReport: - valid: bool - errors: list[ValidationIssue] = field(default_factory=list) - warnings: list[ValidationIssue] = field(default_factory=list) - - def as_dict(self) -> dict[str, Any]: - return { - "valid": self.valid, - "errors": [asdict(item) for item in self.errors], - "warnings": [asdict(item) for item in self.warnings], - } - - -@dataclass(slots=True) -class OntologyGateDecision: - allowed: bool - decision: str - violations: list[dict[str, Any]] = field(default_factory=list) - suggestions: list[str] = field(default_factory=list) - matched_rule_ids: list[str] = field(default_factory=list) diff --git a/src/backend/core/ontology/validator.py b/src/backend/core/ontology/validator.py deleted file mode 100644 index dcf9092b..00000000 --- a/src/backend/core/ontology/validator.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Validate ontology packs before activation.""" -from __future__ import annotations -from typing import Any, Iterable -from core.ontology.schemas import OntologyPackDocument -from jsonschema import Draft202012Validator -from pydantic import ValidationError as PydanticValidationError -from core.ontology import validation_models - -class DomainPackValidator: - """Validate structure, JSON Schema, and tool references before activation.""" - - def validate( - self, - payload: dict[str, Any], - *, - tool_schemas: dict[str, dict[str, Any]] | None = None, - known_tools: Iterable[str] = (), - ) -> tuple[OntologyPackDocument | None, validation_models.ValidationReport]: - errors: list[validation_models.ValidationIssue] = [] - warnings: list[validation_models.ValidationIssue] = [] - try: - document = OntologyPackDocument.model_validate(payload) - except PydanticValidationError as exc: - for item in exc.errors(include_url=False): - errors.append( - validation_models.ValidationIssue( - "error", - ".".join(str(part) for part in item["loc"]), - item["msg"], - ) - ) - return None, validation_models.ValidationReport(False, errors, warnings) - - schemas = tool_schemas or {} - known = set(known_tools) | set(schemas) - for index, rule in enumerate(document.constraints): - path = f"constraints.{index}" - if rule.schema_: - try: - Draft202012Validator.check_schema(rule.schema_) - except Exception as exc: # SchemaError is version-specific - errors.append(validation_models.ValidationIssue("error", f"{path}.schema", str(exc))) - tool_name = rule.target.tool - if not tool_name: - continue - if tool_name not in known: - issue = validation_models.ValidationIssue( - "warning" if document.config.allow_unresolved_tools else "error", - f"{path}.target.tool", - f"unknown tool reference: {tool_name}", - ) - (warnings if document.config.allow_unresolved_tools else errors).append(issue) - continue - if rule.target.kind == "tool_parameter" and tool_name in schemas: - properties = tool_input_schema(schemas[tool_name]).get("properties", {}) - if rule.target.parameter not in properties: - errors.append( - validation_models.ValidationIssue( - "error", - f"{path}.target.parameter", - f"unknown parameter {rule.target.parameter!r} for tool {tool_name}", - ) - ) - - for index, workflow in enumerate(document.workflows): - for tool_name in set(workflow.required_tools + workflow.forbidden_tools): - if tool_name not in known: - issue = validation_models.ValidationIssue( - "warning" if document.config.allow_unresolved_tools else "error", - f"workflows.{index}", - f"unknown tool reference: {tool_name}", - ) - (warnings if document.config.allow_unresolved_tools else errors).append(issue) - overlap = set(workflow.required_tools) & set(workflow.forbidden_tools) - if overlap: - errors.append( - validation_models.ValidationIssue( - "error", - f"workflows.{index}", - f"tools cannot be both required and forbidden: {sorted(overlap)}", - ) - ) - for trigger_index, trigger in enumerate(workflow.asset_triggers): - if trigger.kind != "tool": - continue - for tool_name in trigger.ids: - if tool_name in known: - continue - issue = validation_models.ValidationIssue( - "warning" if document.config.allow_unresolved_tools else "error", - f"workflows.{index}.asset_triggers.{trigger_index}.ids", - f"unknown tool reference: {tool_name}", - ) - (warnings if document.config.allow_unresolved_tools else errors).append(issue) - - return document, validation_models.ValidationReport(not errors, errors, warnings) - - -def tool_input_schema(schema: dict[str, Any]) -> dict[str, Any]: - function = schema.get("function") if isinstance(schema, dict) else None - if isinstance(function, dict): - return function.get("parameters") or function.get("inputSchema") or {} - return schema.get("inputSchema") or schema.get("parameters") or schema diff --git a/src/backend/core/sandbox/commands/store.py b/src/backend/core/sandbox/commands/store.py index 2ddf87a5..493f56ba 100644 --- a/src/backend/core/sandbox/commands/store.py +++ b/src/backend/core/sandbox/commands/store.py @@ -173,7 +173,8 @@ def list_owned(user_id, chat_id): rows = query.filter(CommandTask.status.in_(LIVE)).all() rows += query.filter(CommandTask.status.in_(TERMINAL)).limit(50).all() return [{"task_id": r.task_id, "command": r.command[:200], "status": r.status, - "exit_code": r.exit_code, "notice_run_id": r.notice_run_id, "created_at": r.created_at.isoformat()} for r in rows] + "exit_code": r.exit_code, "notice_run_id": r.notice_run_id, "created_at": r.created_at.isoformat(), + "background": bool(r.background)} for r in rows] def pending_notices(chat_id=None, after=None): with SessionLocal() as db: diff --git a/src/backend/core/sandbox/permissions/approval.py b/src/backend/core/sandbox/permissions/approval.py index 2737d0e8..f79aad8a 100644 --- a/src/backend/core/sandbox/permissions/approval.py +++ b/src/backend/core/sandbox/permissions/approval.py @@ -54,20 +54,6 @@ def _preset_answers(mode: str, *, dangerous: bool) -> bool: return mode == APPROVAL_AUTO and not dangerous -def preset_answers_for_user( - user_id: Optional[str], *, op: str = "", dangerous: bool = False -) -> bool: - """执行期复查:这个用户的权限档是不是已经替他答了这次确认。 - - 供分派中间件之外的确认调用使用。按账号读取同一权限档, - 不依赖当前异步任务的上下文。 - """ - return _preset_answers( - resolve_approval_mode(None, user_id=user_id), - dangerous=dangerous or op in DESTRUCTIVE_OPS, - ) - - async def refresh_runtime(runtime): """Resolve one immutable dispatch policy; a failed read is an error, never a prompt.""" if not runtime.account_approval or not runtime.user_id or runtime.default_allow: diff --git a/src/backend/core/sandbox/processes/cloud.py b/src/backend/core/sandbox/processes/cloud.py index 34bb99b8..4b474924 100644 --- a/src/backend/core/sandbox/processes/cloud.py +++ b/src/backend/core/sandbox/processes/cloud.py @@ -152,7 +152,10 @@ async def _spawn(self, req): ) if not execution.id: raise SandboxError("Background command returned no execution ID") - return OpenSandboxHandle(provider, req, sess.sandbox.commands, execution.id) + return OpenSandboxHandle( + provider, req, sess.sandbox.commands, execution.id, + launched_at=time.monotonic(), sandbox=sess.sandbox, + ) # Cube uses the E2B-compatible background command handle. async with await provider._get_session_lock(req.session_id): sbx = await provider._acquire_persistent(req.session_id, req.user_id) @@ -215,19 +218,57 @@ async def touch(self): class OpenSandboxHandle(CloudHandle): - def __init__(self, provider, req, commands, execution_id): + # execd before v1.0.22 answers a background run before it registers the command, + # so a status read right after launch can get "command not found" for a command + # that is already running. Read that as "not started yet" for a short window + # after our own launch; a recovered handle (no launched_at) gets no such window. + registration_window_s = 2.0 + + def __init__(self, provider, req, commands, execution_id, *, launched_at=None, sandbox=None): super().__init__(provider, req) self.commands = commands self.execution_id = execution_id + self.launched_at = launched_at self.cursor = 0 self.metadata["execution_cursor"] = 0 self.detached = False + # Present for commands launched here; a space copy is then stored before the exit. + self.sandbox = sandbox + self._exit_code = None + self._writeback = None async def poll(self): + # A synchronized space copy is stored before the exit is reported, so the + # next command (and the user) sees what this one wrote. The upload runs + # as a task: each poll stays short under the session's status timeout. + if self._writeback is not None: + if not self._writeback.done(): + return "", "", None + self.finished = True + report = self._writeback.result() + text = report.text() if report else "" + return ("\n" + text + "\n" if text else ""), "", self._exit_code + out, err, code = await self._poll_command() + if code is None or self.sandbox is None or not self.req.user_id: + return out, err, code + from core.files.spaces.transport import get_space_transport + transport = get_space_transport() + if not transport.synchronizes: + return out, err, code + self._exit_code = code + self._writeback = asyncio.create_task(transport.after_use(self.sandbox, self.req.user_id)) + return out, err, None + + async def _poll_command(self): await self.touch() # Read status before logs: when exit is observed, the subsequent log # read includes the final chunk rather than dropping it in a race. - status = await self.commands.get_command_status(self.execution_id) + try: + status = await self.commands.get_command_status(self.execution_id) + except Exception as exc: + if self._not_registered_yet(exc): + return "", "", None + raise logs = await self.commands.get_background_command_logs( self.execution_id, cursor=self.cursor ) @@ -248,11 +289,20 @@ async def poll(self): # OpenSandbox's background API merges stdout/stderr. return logs.content or "", status.error or "", code + def _not_registered_yet(self, exc): + return ( + self.launched_at is not None + and time.monotonic() - self.launched_at < self.registration_window_s + and "command not found" in str(exc) + ) + async def interrupt(self): await self.commands.interrupt(self.execution_id) async def close(self): - if not self.finished and not self.detached: + # After the exit only the write-back may still run; it must finish + # rather than be cancelled with the handle. + if not self.finished and not self.detached and self._writeback is None: await self.interrupt() diff --git a/src/backend/core/sandbox/providers/opensandbox/connection.py b/src/backend/core/sandbox/providers/opensandbox/connection.py index 351843a5..d6c3be4f 100644 --- a/src/backend/core/sandbox/providers/opensandbox/connection.py +++ b/src/backend/core/sandbox/providers/opensandbox/connection.py @@ -3,6 +3,7 @@ from __future__ import annotations from typing import Any +import os from core.config.settings import settings from core.sandbox.providers.opensandbox.runtime import logger @@ -29,7 +30,7 @@ def _install_endpoint_fastpath(self) -> None: failure is only logged and falls back to the SDK default, without affecting functionality. """ - if not settings.sandbox.opensandbox_endpoint_fastpath_enabled: + if os.getenv("DEPLOY_PROFILE") == "kubernetes" or not settings.sandbox.opensandbox_endpoint_fastpath_enabled: return if self._api_key: logger.info( diff --git a/src/backend/core/sandbox/providers/opensandbox/creation.py b/src/backend/core/sandbox/providers/opensandbox/creation.py index d4d24b86..62386aa0 100644 --- a/src/backend/core/sandbox/providers/opensandbox/creation.py +++ b/src/backend/core/sandbox/providers/opensandbox/creation.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import os import time from datetime import timedelta from typing import Any, Optional @@ -17,6 +18,7 @@ # deferred to runtime to avoid cross-module circular dependencies at startup # (models.py also imports core.db.engine). from core.sandbox.providers.opensandbox.runtime import ( + _EXECD_ENVS, _INIT_ENTRYPOINT, _SETUP_ENTRYPOINT, logger, @@ -51,6 +53,16 @@ async def _create_jupyter_sandbox( # selected skill packages are staged explicitly into this container. if str(user_id or "").startswith("api_ak_"): volumes = [] + elif os.getenv("DEPLOY_PROFILE") == "kubernetes": + # Pod workers cannot bind directories from the API container's host. + # Skills are delivered explicitly at execution time; spaces remain + # real authorized object-storage mounts. + credentials = credential_mounts(user_id) + if any(getattr(volume, "host", None) is not None for volume in credentials): + raise SandboxConnectError( + "Kubernetes does not support host credential mounts; configure a native credential provider" + ) + volumes = extra_volumes + credentials else: skills_vols = _make_skills_volumes(user_id) volumes = skills_vols + extra_volumes + credential_mounts(user_id) @@ -61,7 +73,7 @@ async def _create_jupyter_sandbox( # skip_health_check=True: execd's own readiness suffices, no waiting for Jupyter. # The metadata tag is for cross-process adopt (backend restart reclaiming its own sandboxes, see _adopt_existing_sandboxes). # 目录约定(scratch + /myspace 软链)随容器启动一起做完,首条命令不必再花一次 - # execd 往返(实测 ~1s)。uid 走环境变量,entrypoint 保持常量。 + # execd 往返。uid 走环境变量,entrypoint 保持常量。 setup_uid = safe_user_id(user_id) if user_id else "" common_kwargs = dict( timeout=timedelta(seconds=self._session_expiry_s()), @@ -70,9 +82,16 @@ async def _create_jupyter_sandbox( skip_health_check=True, volumes=volumes or None, metadata={**_pool_metadata("jupyter", user_id=user_id), **extra_metadata}, + env=dict(_EXECD_ENVS), ) + if os.getenv("DEPLOY_PROFILE") == "kubernetes": + if not settings.sandbox.cpu or not settings.sandbox.memory: + raise SandboxConnectError( + "Kubernetes sandboxes require SANDBOX_CPU and SANDBOX_MEMORY resource limits" + ) + common_kwargs["resource"] = {"cpu": settings.sandbox.cpu, "memory": settings.sandbox.memory} if setup_uid: - common_kwargs["env"] = {"JX_MYSPACE_UID": setup_uid} + common_kwargs["env"]["JX_MYSPACE_UID"] = setup_uid try: if from_snapshot_id: sbx = await self._create_owned_sandbox(snapshot_id=from_snapshot_id, **common_kwargs) @@ -109,7 +128,7 @@ async def _repoint_execd_direct(self, sbx: Any) -> bool: non-compose deployments, zero functionality loss). Returns whether we switched to direct connection. """ - if not settings.sandbox.opensandbox_direct_execd_enabled: + if os.getenv("DEPLOY_PROFILE") == "kubernetes" or not settings.sandbox.opensandbox_direct_execd_enabled: return False # This process has already determined direct connection is unreachable → skip probing; don't let every sandbox burn another 6s probe budget. if getattr(self, "_direct_execd_unreachable", False): diff --git a/src/backend/core/sandbox/providers/opensandbox/execution.py b/src/backend/core/sandbox/providers/opensandbox/execution.py index b3d5b60e..7ffcc039 100644 --- a/src/backend/core/sandbox/providers/opensandbox/execution.py +++ b/src/backend/core/sandbox/providers/opensandbox/execution.py @@ -4,6 +4,7 @@ import asyncio import base64 +import os from datetime import timedelta from shlex import quote as shell_escape from typing import Any @@ -70,6 +71,11 @@ async def reap_idle_sessions(self) -> int: async def _sync_inputs(self, sess: _Session, req: ProcessRequest) -> None: await self._persistent_setup(sess, req) + if os.getenv("DEPLOY_PROFILE") == "kubernetes" and not str(req.user_id or "").startswith("api_ak_"): + from core.sandbox.providers.opensandbox.skills import stage_skill_view + await stage_skill_view(sess.sandbox, req) + from core.files.spaces.transport import get_space_transport + await get_space_transport().before_use(sess.sandbox, req.user_id) await self._sync_inputs_to_sandbox( sess.sandbox, req, diff --git a/src/backend/core/sandbox/providers/opensandbox/provider.py b/src/backend/core/sandbox/providers/opensandbox/provider.py index 78c6bef1..72af4698 100644 --- a/src/backend/core/sandbox/providers/opensandbox/provider.py +++ b/src/backend/core/sandbox/providers/opensandbox/provider.py @@ -258,6 +258,9 @@ async def put_file( await write_file(sess.sandbox, path, content) except Exception as e: # noqa: BLE001 raise SandboxError(f"write_file {path} 失败: {e}") from e + from core.files.spaces.transport import get_space_transport + # A failed write-back keeps the file in the copy for the next sync. + await get_space_transport().after_use(sess.sandbox, getattr(sess, "user_id", None), path=path) @_on_service_loop async def get_file( @@ -270,6 +273,8 @@ async def get_file( if not session_id: raise SandboxError("sandbox get_file 需要 session_id(持久会话);当前请求未绑定会话") async with self._session_use(session_id, user_id) as sess: + from core.files.spaces.transport import get_space_transport + await get_space_transport().before_use(sess.sandbox, getattr(sess, "user_id", None), path=path) try: raw = await sess.sandbox.files.read_bytes(path) except Exception as e: # noqa: BLE001 @@ -292,6 +297,8 @@ async def get_file_to_path( if not session_id: raise SandboxError("sandbox get_file 需要 session_id(持久会话);当前请求未绑定会话") async with self._session_use(session_id, user_id) as sess: + from core.files.spaces.transport import get_space_transport + await get_space_transport().before_use(sess.sandbox, getattr(sess, "user_id", None), path=path) try: info_by_path = await sess.sandbox.files.get_file_info([path]) info = info_by_path.get(path) if isinstance(info_by_path, dict) else None diff --git a/src/backend/core/sandbox/providers/opensandbox/reclamation.py b/src/backend/core/sandbox/providers/opensandbox/reclamation.py index e0a767f4..222fac2e 100644 --- a/src/backend/core/sandbox/providers/opensandbox/reclamation.py +++ b/src/backend/core/sandbox/providers/opensandbox/reclamation.py @@ -2,6 +2,8 @@ from __future__ import annotations +import os + from core.config.settings import settings import core.sandbox.sessions.registry as session_registry @@ -49,6 +51,24 @@ async def protect(chat_id, sess): await asyncio.gather(*(protect(chat_id, sess) for chat_id, sess in entries)) + async def _flush_space_copy(self, sess) -> None: + """Store a synchronized space copy before the sandbox is reset or destroyed. + + A failure keeps the copy: a reused sandbox still holds it for the same + account's next session; only a destroyed one loses unsaved edits. + """ + from core.files.spaces.transport import get_space_transport + import asyncio + + try: + report = await asyncio.wait_for( + get_space_transport().flush(sess.sandbox, sess.user_id, force=True), timeout=120) + if report.errors: + logger.warning("[opensandbox] space copy not fully stored before release: %s", + report.text()) + except Exception as exc: # noqa: BLE001 - release must proceed + logger.warning("[opensandbox] space copy flush before release failed: %r", exc) + async def _destroy_session(self, session_id, sess, *, reuse_sandbox=False): from core.capabilities.plugins.resources.prewarm import closing, stop import asyncio @@ -98,6 +118,8 @@ async def _release_session( # harmless and cannot pop a replacement created while a cleanup was queued. if self._sessions.get(session_id) is not sess: return + if not detach_only: + await self._flush_space_copy(sess) # Choose the target pool by sess.pool_source — recorded at acquire time, always accurate is_user_pool = sess.pool_source == "user" and bool(sess.user_id) reuse_ok = ( @@ -111,6 +133,14 @@ async def _release_session( # 结果、解压出来的原料会原样留着,下一个会话一 ls 就看见、会当成自己的上文。 # 擦不干净就不复用 —— 宁可多花十秒新建,也不能把脏容器递出去。 reuse_ok = await self._wipe_workspace(sess) + if reuse_ok and os.getenv("DEPLOY_PROFILE") == "kubernetes": + # 下一个会话重新下发一次技能视图,不沿用上一会话里可能被改动的副本。 + from core.sandbox.providers.opensandbox.skills import forget_skill_view + try: + await forget_skill_view(sess.sandbox) + except Exception as exc: # noqa: BLE001 — 记不下就不复用,宁可新建 + logger.warning("[opensandbox] 清除技能视图记录失败,放弃复用: %r", exc) + reuse_ok = False # 先解绑再入池:容器一旦进了空闲队列,归属判定就会在 acquire / reap 时被问到, # 这时登记还指着它的话,池子会以为它仍有主而拒绝复用和回收,白白漏掉一台。 # 撤销只在登记仍指向这一个沙箱时生效,免得抹掉别的 worker 刚建的那条。 diff --git a/src/backend/core/sandbox/providers/opensandbox/runtime.py b/src/backend/core/sandbox/providers/opensandbox/runtime.py index df3e5685..5d9f8281 100644 --- a/src/backend/core/sandbox/providers/opensandbox/runtime.py +++ b/src/backend/core/sandbox/providers/opensandbox/runtime.py @@ -73,9 +73,19 @@ async def write_file(sbx: Any, path: str, content: Any) -> None: # what user commands, which run as SANDBOX_RUN_UID, leave behind. _INIT_ENTRYPOINT = ["sandbox-init", str(SANDBOX_RUN_UID)] +# 容器环境变量,execd 由 bootstrap.sh 启动时继承。 +# execd 在前台命令发完 execution_complete 后还要 sleep EXECD_API_GRACE_SHUTDOWN(默认 +# 1s)才关 SSE 流,SDK 的前台 commands.run 要等流关闭才返回,于是每次写文件后的 chown、 +# _persistent_setup、复用前的重置都白等 1 秒。这段 sleep 保护不了任何输出:每个事件都是 +# 同步写出并 flush 的,前台命令也是把 stdout/stderr 读空之后才发 complete(execd +# runtime/command.go runCommand;v1.0.17 到 v1.1.0 都如此)。它另外是 Jupyter 执行等 +# 结果的上限,我们不启动 Jupyter。实测 0s 后前台命令 1015ms → 3ms,空行、20 万行输出、 +# stderr 与退出码均不变。用户代码也能看到这个变量,无害。 +_EXECD_ENVS: dict[str, str] = {"EXECD_API_GRACE_SHUTDOWN": "0s"} + # 用户绑定沙箱的 entrypoint:容器启动时就建好目录约定(scratch + /myspace 软链), -# 首条命令不必再为此花一次 execd 往返(实测固定 ~1s,而这段 shell 只要 2ms)。uid 走 -# 环境变量,entrypoint 保持常量,adopt 仍能按它认出自己创建的沙箱。 +# 首条命令不必再为此花一次 execd 往返。uid 走环境变量,entrypoint 保持常量,adopt +# 仍能按它认出自己创建的沙箱。 # # 两个约束:``/myspace`` 已是真目录时不能直接 ln(软链会建到目录里面),所以先判断; # 整段不许中途退出,任何一步失败都要走到 exec sandbox-init,否则容器直接死掉。 @@ -95,6 +105,15 @@ async def write_file(sbx: Any, path: str, content: Any) -> None: ] +def sandbox_lock(sandbox: Any, attr: str) -> asyncio.Lock: + """A lock stored on the SDK sandbox object, serializing one kind of staging per sandbox.""" + lock = getattr(sandbox, attr, None) + if lock is None: + lock = asyncio.Lock() + setattr(sandbox, attr, lock) + return lock + + def _join_logs(parts: list) -> str: """The SDK's result.logs.stdout/stderr is list[StreamItem]; StreamItem.text is a string. diff --git a/src/backend/core/sandbox/providers/opensandbox/skills.py b/src/backend/core/sandbox/providers/opensandbox/skills.py new file mode 100644 index 00000000..c1d79a4c --- /dev/null +++ b/src/backend/core/sandbox/providers/opensandbox/skills.py @@ -0,0 +1,185 @@ +"""Synchronize the current authorized skill view into Kubernetes sandboxes.""" +from __future__ import annotations + +import asyncio +from datetime import timedelta +import hashlib +import io +from pathlib import Path +import shlex +import tarfile +import uuid + +from core.sandbox.errors import SandboxError +from core.sandbox.providers.opensandbox.runtime import logger +from core.sandbox.workspace.identity import SANDBOX_RUN_UID, SANDBOX_RUN_GID + +# Serializes staging per sandbox object inside one process. +_LOCK_ATTR = "_jx_skill_view_lock" + + +def _database_visible(skill_ids, user_id): + """Read mutable enablement/ownership freshly; loader listings include disabled rows.""" + from core.db.engine import SessionLocal + from core.db.models import AdminSkill + + allowed = set(skill_ids) + with SessionLocal() as db: + rows = db.query(AdminSkill.skill_id, AdminSkill.is_enabled, AdminSkill.owner_user_id, + AdminSkill.dep_status).filter(AdminSkill.skill_id.in_(allowed)).all() + for row in rows: + if not row.is_enabled or row.dep_status != "ready" or (row.owner_user_id and row.owner_user_id != user_id): + allowed.discard(row.skill_id) + return allowed + + +def _view_files(user_id: str | None) -> list[tuple[str, bytes, int]]: + """Freeze and validate the whole current view, independent of shell syntax.""" + from core.harness.registry.run_capabilities import has_run_capabilities + if has_run_capabilities(): + raise SandboxError("Kubernetes skill views require cloud sources; desktop capability publication is unsupported") + from core.capabilities.skills.publication import prepare_skill_view, publication_lock + from core.capabilities.skills.config import get_sandbox_skills_dir + from core.llm.api_runtime.agent_skills import _skill_files + from core.capabilities.catalog.api import get_catalog + + # Read the latest administrator choices without discarding the shared cache. + catalog = {str(item["id"]): bool(item.get("enabled")) + for item in get_catalog(include_runtime_details=False, fresh=True).get("skills", [])} + with publication_lock(): + view = prepare_skill_view(user_id) + root = view if view is not None else get_sandbox_skills_dir() + selected = {path.name: str(path) for path in sorted(root.iterdir()) + if path.is_dir() and catalog.get(path.name, True)} + visible = _database_visible(selected, user_id) + selected = {name: path for name, path in selected.items() if name in visible} + # Reuse explicit API package validation, including symlink rejection and + # the bounded file closure. Publication errors stop the new command. + files = [] + for target, content in _skill_files(selected): + relative = target.removeprefix("/workspace/skills/") + skill_id, name = relative.split("/", 1) + mode = 0o755 if (Path(selected[skill_id]) / name).stat().st_mode & 0o111 else 0o644 + files.append((relative, content, mode)) + return files + + +def _view_digest(user_id: str | None, files: list[tuple[str, bytes, int]]) -> str: + digest = hashlib.sha256(str(user_id or "").encode() + b"\0") + for relative, content, mode in files: + digest.update(relative.encode() + b"\0" + str(mode).encode() + b"\0") + digest.update(hashlib.sha256(content).digest()) + return digest.hexdigest() + + +def _view_archive(files: list[tuple[str, bytes, int]]) -> bytes: + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode="w:gz") as archive: + for relative, content, mode in files: + entry = tarfile.TarInfo(relative) + entry.size = len(content) + entry.mode = mode + archive.addfile(entry, io.BytesIO(content)) + return buffer.getvalue() + + +# Runs as uid 1000. Preserve unchanged directories and files so concurrent +# commands do not lose their working directory merely because another command +# starts. Changed files are replaced atomically; revoked files are removed. +_SYNC_VIEW = r""" +import os,shutil,sys,tarfile,tempfile +from pathlib import Path,PurePosixPath +root=Path('/workspace/skills') +if root.is_symlink(): + raise RuntimeError('Skill view must not be a symlink') +root.mkdir(parents=True,exist_ok=True) +files={} +with tarfile.open(sys.argv[1],'r:gz') as archive: + for entry in archive.getmembers(): + name=PurePosixPath(entry.name) + if not entry.isfile() or name.is_absolute() or '..' in name.parts or len(name.parts)<2: + raise RuntimeError('Invalid skill archive member') + files[name.as_posix()]=(archive.extractfile(entry).read(),entry.mode & 0o755) +dirs={parent.as_posix() for name in files for parent in PurePosixPath(name).parents if str(parent)!='.'} +# Remove only entries in this sandbox's skill view, never following symlinks. +for base,children,names in os.walk(root,topdown=False,followlinks=False): + for name in names+children: + path=Path(base)/name + rel=path.relative_to(root).as_posix() + if path.is_symlink() or (rel not in files and rel not in dirs): + if path.is_symlink() or not path.is_dir(): path.unlink() + else: shutil.rmtree(path) +for name in sorted(dirs,key=lambda value:(len(PurePosixPath(value).parts),value)): + path=root/name + if path.exists() and not path.is_dir(): path.unlink() + path.mkdir(exist_ok=True) +for name,(content,mode) in files.items(): + path=root/name + if path.is_dir(): shutil.rmtree(path) + if path.is_file() and path.read_bytes()==content: + path.chmod(mode) + continue + with tempfile.NamedTemporaryFile(dir=path.parent,delete=False) as stream: + temporary=Path(stream.name) + stream.write(content) + try: + temporary.chmod(mode) + os.replace(temporary,path) + finally: + temporary.unlink(missing_ok=True) +""" + + +def _installed_key(sandbox) -> str: + return "sandbox:skill-view:" + str(sandbox.id) + + +async def _installed_digest(sandbox) -> str | None: + from core.infra.ephemeral import get_ephemeral_state + try: + return await get_ephemeral_state().get(_installed_key(sandbox)) + except Exception as exc: # noqa: BLE001 - unknown state only costs a re-send + logger.warning("[opensandbox] skill view state unavailable: %s", type(exc).__name__) + return None + + +async def stage_skill_view(sandbox, request): + """Bring the sandbox's skill directory to the current authorized view. + + The view is recomputed for every command, so revocations apply immediately. + It is only transferred when it differs from the view last installed in this + sandbox. That record lives in the shared ephemeral store keyed by sandbox id, + so every worker attached to the sandbox sees the same installed view; it is + cleared before a transfer and written only after the transfer succeeds. + """ + from core.config.settings import settings + from core.infra.ephemeral import get_ephemeral_state + + from core.sandbox.providers.opensandbox.runtime import sandbox_lock + async with sandbox_lock(sandbox, _LOCK_ATTR): + files = await asyncio.to_thread(_view_files, request.user_id) + digest = _view_digest(request.user_id, files) + if await _installed_digest(sandbox) == digest: + return + state = get_ephemeral_state() + await state.drop(_installed_key(sandbox)) + content = await asyncio.to_thread(_view_archive, files) + from opensandbox.models.execd import RunCommandOpts + + source = "/tmp/.jx-skills-" + uuid.uuid4().hex + ".tgz" + await sandbox.files.write_file(source, content) + try: + result = await sandbox.commands.run( + "python3 -c " + shlex.quote(_SYNC_VIEW) + " " + shlex.quote(source), + opts=RunCommandOpts(uid=str(SANDBOX_RUN_UID), gid=str(SANDBOX_RUN_GID), timeout=timedelta(seconds=60))) + if getattr(result, "exit_code", None) != 0: + raise SandboxError("Failed to synchronize authorized skill view") + finally: + await sandbox.commands.run("rm -f -- " + shlex.quote(source), opts=RunCommandOpts(timeout=timedelta(seconds=10))) + await state.put(_installed_key(sandbox), digest, ttl=settings.sandbox.idle_ttl_s) + + +async def forget_skill_view(sandbox) -> None: + """Force the next command to re-send the view (e.g. after a sandbox is reset for reuse).""" + from core.infra.ephemeral import get_ephemeral_state + await get_ephemeral_state().drop(_installed_key(sandbox)) diff --git a/src/backend/extensions/mcp_servers/agent_manager_mcp/impl.py b/src/backend/extensions/mcp_servers/agent_manager_mcp/impl.py index 070b16b7..d45dbce7 100644 --- a/src/backend/extensions/mcp_servers/agent_manager_mcp/impl.py +++ b/src/backend/extensions/mcp_servers/agent_manager_mcp/impl.py @@ -400,11 +400,9 @@ def create_agent( } def _friendly_error(action: str, exc: Exception) -> str: - """把 service 层异常转成人话。本体校验失败要说清楚是本体没过,而不是甩一句失败。""" + """把 service 层异常转成人话。""" text = str(exc) or exc.__class__.__name__ low = text.lower() - if "ontology" in low or "本体" in text: - return f"❌ {action}失败:没通过本体校验——{text}。请按提示补齐名称/描述/正文或本体标签后重试。" if isinstance(exc, PermissionError): return f"❌ {action}失败:只能操作你自己的子智能体。" if isinstance(exc, LookupError): diff --git a/src/backend/features/desktop/bridge/bundles.py b/src/backend/features/desktop/bridge/bundles.py index ab6578de..890bc78d 100644 --- a/src/backend/features/desktop/bridge/bundles.py +++ b/src/backend/features/desktop/bridge/bundles.py @@ -46,8 +46,8 @@ def _headers(state: Dict[str, Any]) -> Dict[str, str]: def _fetch(kind: str, state: Dict[str, Any]) -> Dict[str, Any]: - import httpx from core.capabilities.manifest_protocol import validate_entity_manifest + from features.desktop.bridge import cloud_http ticket = manifest_order.begin(kind, _profile(state)) with _lock: @@ -55,10 +55,10 @@ def _fetch(kind: str, state: Dict[str, Any]) -> Dict[str, Any]: headers = {**_headers(state), "Cache-Control": "no-cache"} if current: headers["If-None-Match"] = f'"{current["revision"]}"' - resp = httpx.get( + resp = cloud_http.get( f"{state['cloud_base']}/api/v1/desktop/capability/{_PATHS[kind]}/manifest", headers=headers, - timeout=httpx.Timeout(10.0, connect=5.0), + timeout=10.0, ) if resp.status_code == 304 and current: return manifest_order.stamp(current, ticket) @@ -70,16 +70,16 @@ def _fetch(kind: str, state: Dict[str, Any]) -> Dict[str, Any]: def _download(kind: str, state: Dict[str, Any], ident: str) -> bytes: - import httpx + from features.desktop.bridge import cloud_http try: - resp = httpx.get( + resp = cloud_http.get( f"{state['cloud_base']}/api/v1/desktop/capability/{_PATHS[kind]}/{ident}/bundle", headers=_headers(state), - timeout=httpx.Timeout(30.0, connect=5.0), + timeout=30.0, ) - except httpx.HTTPError as exc: - raise CloudUnavailable(f"{kind} bundle download failed: {exc}", ref=ident) from exc + except CloudUnavailable as exc: + raise CloudUnavailable(exc.message, ref=ident) from exc if resp.status_code >= 400: raise CloudUnavailable(f"{kind} bundle download HTTP {resp.status_code}", ref=ident) return resp.content @@ -260,8 +260,10 @@ def sync_kind(kind: str, state: Dict[str, Any]) -> bool: note_rejected_credential(state, exc) if _state_fingerprint(get_state()) != _state_fingerprint(state): return False + from features.desktop.bridge.cloud_http import describe + with _lock: - _errors[kind] = str(exc) + _errors[kind] = describe(exc) logger.warning("[cloud-%s] sync failed: %s", kind, exc) return False if changed or prepared: diff --git a/src/backend/features/desktop/bridge/client.py b/src/backend/features/desktop/bridge/client.py index 863d1f10..3652bbfe 100644 --- a/src/backend/features/desktop/bridge/client.py +++ b/src/backend/features/desktop/bridge/client.py @@ -86,8 +86,8 @@ def sync_tool_manifest(st: Dict[str, Any]) -> None: bridge_identity.require_current_account(st) url = f"{st['cloud_base']}/api/v1/desktop/capability/manifest" - import httpx from core.capabilities.manifest_protocol import validate_manifest + from features.desktop.bridge import cloud_http headers = {**bridge_identity.cloud_headers(st), "Cache-Control": "no-cache"} with bridge_client_state.manifest_lock: @@ -95,11 +95,7 @@ def sync_tool_manifest(st: Dict[str, Any]) -> None: if current_revision: headers["If-None-Match"] = f'"{current_revision}"' - resp = httpx.get( - url, - headers=headers, - timeout=httpx.Timeout(10.0, connect=5.0), - ) + resp = cloud_http.get(url, headers=headers, timeout=10.0) bridge_identity.require_current_account(st) if resp.status_code in (401, 403): with bridge_identity.account_scope(st): @@ -129,6 +125,8 @@ def sync_tool_manifest(st: Dict[str, Any]) -> None: def fetch_manifest_blocking(st: Dict[str, Any]) -> None: pass # State is owned by its responsibility module. + from features.desktop.bridge import cloud_http + refresh_replacement = False try: sync_tool_manifest_blocking(st) @@ -142,7 +140,7 @@ def fetch_manifest_blocking(st: Dict[str, Any]) -> None: return with bridge_client_state.manifest_lock: bridge_client_state.manifest_ts = time.monotonic() - bridge_client_state.manifest_error = str(exc) + bridge_client_state.manifest_error = cloud_http.describe(exc) logger.warning("[cloud-bridge] manifest 拉取失败: %s", exc) finally: with bridge_client_state.manifest_lock: diff --git a/src/backend/features/desktop/bridge/cloud_http.py b/src/backend/features/desktop/bridge/cloud_http.py new file mode 100644 index 00000000..e99c2aeb --- /dev/null +++ b/src/backend/features/desktop/bridge/cloud_http.py @@ -0,0 +1,101 @@ +"""Pooled, patient HTTP for the local plane's synchronous calls to the cloud. + +Every manifest poll, bundle download and work-copy sync used to go out through a +one-off ``httpx.get``: a fresh connection pool per call, so each call paid DNS, +TCP and a full TLS handshake, with five seconds for all of it. Sign-in fires +several of these at once, and on a laptop behind a VPN or TUN proxy a cold +handshake regularly runs past five seconds. The raw ``_ssl.c:999: The handshake +operation timed out`` then sat in the sync status, and the UI showed it the +moment a page opened, long after the background attempt had failed. + +One shared client keeps connections warm between calls, the connect budget is +configurable, and a connect timeout gets exactly one more try: nothing has been +sent at that point, so repeating is safe for any method. Transport failures +surface as ``CloudUnavailable`` with a message written for the person; the raw +exception goes to the log. +""" + +from __future__ import annotations + +import logging +import os +import threading +from typing import Any, Dict, Optional + +import httpx + +from core.harness.capability_errors import CapabilityError, CloudUnavailable + +logger = logging.getLogger(__name__) + +CONNECT_TIMEOUT_ENV = "HUGAGENT_CLOUD_CONNECT_TIMEOUT_S" +DEFAULT_CONNECT_TIMEOUT_S = 15.0 + +# Retire pooled sockets before the gateway nginx's 65 s keep-alive closes them, +# so a reused connection is never one the server already dropped. +_LIMITS = httpx.Limits(max_connections=32, max_keepalive_connections=8, keepalive_expiry=55.0) + +_lock = threading.Lock() +_client: Optional[httpx.Client] = None + + +def connect_timeout_s() -> float: + """TCP connect plus TLS handshake budget; bad or non-positive values fall back.""" + raw = os.getenv(CONNECT_TIMEOUT_ENV, "").strip() + try: + value = float(raw) if raw else DEFAULT_CONNECT_TIMEOUT_S + except ValueError: + value = DEFAULT_CONNECT_TIMEOUT_S + return value if value > 0 else DEFAULT_CONNECT_TIMEOUT_S + + +def _shared() -> httpx.Client: + global _client + with _lock: + if _client is None or _client.is_closed: + _client = httpx.Client(limits=_LIMITS) + return _client + + +def request( + method: str, + url: str, + *, + headers: Optional[Dict[str, str]] = None, + json: Any = None, + timeout: float = 10.0, +) -> httpx.Response: + """Send one cloud request; ``timeout`` bounds reads, the connect budget is shared.""" + budget = httpx.Timeout(timeout, connect=connect_timeout_s()) + try: + try: + return _shared().request(method, url, headers=headers, json=json, timeout=budget) + except httpx.ConnectTimeout as first: + logger.info("[cloud-http] %s %s connect timed out, retrying once: %s", method, url, first) + return _shared().request(method, url, headers=headers, json=json, timeout=budget) + except httpx.TransportError as exc: + logger.warning("[cloud-http] %s %s failed: %r", method, url, exc) + raise CloudUnavailable(_transport_message(exc)) from exc + + +def get(url: str, *, headers: Optional[Dict[str, str]] = None, timeout: float = 10.0) -> httpx.Response: + return request("GET", url, headers=headers, timeout=timeout) + + +def _transport_message(exc: httpx.TransportError) -> str: + if isinstance(exc, httpx.TimeoutException): + return "连接云端超时,请检查网络或代理设置后重试" + if isinstance(exc, httpx.ConnectError): + return "无法连接云端,请检查网络或代理设置后重试" + return "与云端的连接中断,请稍后重试" + + +def describe(exc: BaseException) -> str: + """The sync error a person sees: no URLs, no ``_ssl.c`` internals.""" + if isinstance(exc, CapabilityError): + return exc.message + if isinstance(exc, httpx.HTTPStatusError): + return f"云端暂时不可用(HTTP {exc.response.status_code}),请稍后重试" + if isinstance(exc, httpx.TransportError): + return _transport_message(exc) + return str(exc) diff --git a/src/backend/features/desktop/bridge/resource_plugins.py b/src/backend/features/desktop/bridge/resource_plugins.py index c913f803..278edc20 100644 --- a/src/backend/features/desktop/bridge/resource_plugins.py +++ b/src/backend/features/desktop/bridge/resource_plugins.py @@ -74,7 +74,11 @@ def local_config(server, user_id, profile): "components", {} ).get("mcp", []): continue - comp = store.get("plugin", profile, inst.key, inst.resolved_revision) + comp = ( + store.get("plugin", profile, inst.key, inst.resolved_revision) + if inst.resolved_revision + else None + ) if comp is None: raise HTTPException(409, "plugin_package_missing") definition = load_manifest(comp) diff --git a/src/backend/features/desktop/bridge/skills.py b/src/backend/features/desktop/bridge/skills.py index e983ec26..c01a89bc 100644 --- a/src/backend/features/desktop/bridge/skills.py +++ b/src/backend/features/desktop/bridge/skills.py @@ -57,8 +57,8 @@ def _headers(state: Dict[str, Any]) -> Dict[str, str]: def _fetch_manifest(state: Dict[str, Any]) -> Dict[str, Any]: - import httpx from core.capabilities.manifest_protocol import validate_skill_manifest + from features.desktop.bridge import cloud_http ticket = manifest_order.begin(KIND_SKILL, _profile(state)) with _lock: @@ -66,10 +66,10 @@ def _fetch_manifest(state: Dict[str, Any]) -> Dict[str, Any]: headers = {**_headers(state), "Cache-Control": "no-cache"} if current: headers["If-None-Match"] = f'"{current["revision"]}"' - resp = httpx.get( + resp = cloud_http.get( f"{state['cloud_base']}/api/v1/desktop/capability/skills/manifest", headers=headers, - timeout=httpx.Timeout(10.0, connect=5.0), + timeout=10.0, ) if resp.status_code == 304 and current: return manifest_order.stamp(current, ticket) @@ -153,8 +153,10 @@ def sync_blocking(state: Dict[str, Any]) -> None: note_rejected_credential(state, exc) if _state_fingerprint(get_state()) != _state_fingerprint(state): return + from features.desktop.bridge.cloud_http import describe + with _lock: - _error = str(exc) + _error = describe(exc) logger.warning("[cloud-skills] sync failed: %s", exc) return prepared = prepare(state, pending) if pending else [] @@ -174,16 +176,16 @@ def sync_blocking(state: Dict[str, Any]) -> None: def _download(state: Dict[str, Any], skill_id: str) -> bytes: - import httpx + from features.desktop.bridge import cloud_http try: - resp = httpx.get( + resp = cloud_http.get( f"{state['cloud_base']}/api/v1/desktop/capability/skills/{skill_id}/bundle", headers=_headers(state), - timeout=httpx.Timeout(60.0, connect=5.0), + timeout=60.0, ) - except httpx.HTTPError as exc: - raise CloudUnavailable(f"bundle download failed: {exc}", ref=skill_id) from exc + except CloudUnavailable as exc: + raise CloudUnavailable(exc.message, ref=skill_id) from exc if resp.status_code == 404: raise PackageMissing("cloud no longer offers this skill", ref=skill_id) if resp.status_code >= 400: diff --git a/src/backend/features/desktop/capabilities/change_sync.py b/src/backend/features/desktop/capabilities/change_sync.py index 3e63c8da..a4d3fd71 100644 --- a/src/backend/features/desktop/capabilities/change_sync.py +++ b/src/backend/features/desktop/capabilities/change_sync.py @@ -8,7 +8,6 @@ import uuid from pathlib import Path from urllib.parse import quote -import httpx from fastapi import HTTPException from core.db.models import ContentBlock from core.capabilities.skills.binary_files import encode_upload @@ -19,7 +18,8 @@ from core.capabilities.identity_rules import LOCAL_PROFILE, BUILTIN_PROFILE from features.desktop.capabilities.paths import assert_managed_path from features.desktop.capabilities.change_merge import plan, revision, validate_files, sensitive_paths -from core.harness.capability_errors import CapabilityError +from core.harness.capability_errors import CapabilityError, CloudUnavailable +from features.desktop.bridge import cloud_http from features.desktop.capabilities.mcp_json import McpJsonError from sqlalchemy.exc import SQLAlchemyError from core.infra.crypto import encrypt_secret, decrypt_secret @@ -60,8 +60,8 @@ def cloud_request(state, method, kind, key, body=None): + quote(key, safe="") ) try: - response = httpx.request(method, url, headers=cloud_headers(state), json=body, timeout=30) - except httpx.HTTPError as exc: + response = cloud_http.request(method, url, headers=cloud_headers(state), json=body, timeout=30.0) + except CloudUnavailable as exc: raise HTTPException(502, detail="云端暂不可用,本地修改已保留") from exc require_current_account(state) if response.status_code >= 400: diff --git a/src/backend/features/desktop/capabilities/device_plugin_catalog.py b/src/backend/features/desktop/capabilities/device_plugin_catalog.py index d5b383ac..26ef6558 100644 --- a/src/backend/features/desktop/capabilities/device_plugin_catalog.py +++ b/src/backend/features/desktop/capabilities/device_plugin_catalog.py @@ -129,7 +129,12 @@ def plugin_detail(install_id_or_slug: str, user_id=None) -> Optional[Dict[str, A from core.capabilities.plugins.connections import manifest_connection from features.desktop.capabilities import plugins, store - comp = store.get("plugin", inst.profile_id, inst.key, inst.resolved_revision) + # 包还没下载好(同步中、失败或已移除)时没有修订号,详情照样给登记表那份。 + comp = ( + store.get("plugin", inst.profile_id, inst.key, inst.resolved_revision) + if inst.resolved_revision + else None + ) manifest = plugins.load_manifest(comp) if comp else {} profile = inst.profile_id components = dict(inst.payload.get("components") or {}) diff --git a/src/backend/features/desktop/capability/sync_check.py b/src/backend/features/desktop/capability/sync_check.py index 86fc3926..cae0f924 100644 --- a/src/backend/features/desktop/capability/sync_check.py +++ b/src/backend/features/desktop/capability/sync_check.py @@ -12,9 +12,8 @@ import time from typing import Any -import httpx - from features.desktop.bridge import client as bridge +from features.desktop.bridge import cloud_http from features.desktop.bridge import bundles as bundles from features.desktop.bridge import skills as skills from core.capabilities.manifest_protocol import canonical_hash, validate_manifest, validate_skill_manifest, validate_entity_manifest @@ -51,14 +50,14 @@ def _fetch(st, kind, known): "agent": "agents/manifest", "plugin": "plugins/manifest", }[kind] - response = httpx.get( + response = cloud_http.get( f"{st['cloud_base']}/api/v1/desktop/capability/{path}", headers={ **bridge_identity.cloud_headers(st), "Cache-Control": "no-cache", "If-None-Match": f'"{known["revision"]}"', }, - timeout=httpx.Timeout(10.0, connect=5.0), + timeout=10.0, ) bridge_identity.require_current_account(st) if response.status_code == 304: diff --git a/src/backend/personal_agent/background.py b/src/backend/personal_agent/background.py index 06eb7c35..460bc0e3 100644 --- a/src/backend/personal_agent/background.py +++ b/src/backend/personal_agent/background.py @@ -106,7 +106,7 @@ def _computer_bindings(db): bindings = await run_in_session(_computer_bindings) if not bindings: return - from personal_agent.computer.sandbox import get_personal_sandbox_provider + from personal_agent.computer.sandbox import ProvisionInProgress, get_personal_sandbox_provider for ident, box, uid in bindings: try: @@ -114,7 +114,11 @@ def _computer_bindings(db): provider = get_personal_sandbox_provider() await provider.ensure_user_workspace(box, uid) await provider.touch_session(box) - except Exception: + except ProvisionInProgress: + # Another worker is creating it and records the outcome. + continue + except Exception as exc: + log.warning("Personal computer %s maintenance failed: %s: %s", ident, type(exc).__name__, exc) def _mark_degraded(db): row = db.get(PersonalAgent, ident) if row: diff --git a/src/backend/personal_agent/computer/sandbox.py b/src/backend/personal_agent/computer/sandbox.py index 9bed3bb5..6daf969a 100644 --- a/src/backend/personal_agent/computer/sandbox.py +++ b/src/backend/personal_agent/computer/sandbox.py @@ -62,6 +62,16 @@ async def install_once(provider, box, uid, name, digest, command): def _computer_owner(db, session_id): return db.scalar(select(PersonalAgent.user_id).where(PersonalAgent.sandbox_session_id == session_id)) +class ComputerGone(SandboxError): + """The bound instance was killed or retired; a fresh instance may replace it.""" + +class ProvisionInProgress(SandboxError): + """Another worker holds the creation lock; the computer is starting, not failing.""" + +def _gone(exc): + # Only a confirmed missing or retired instance permits recreation. Network errors preserve identity. + return isinstance(exc, ComputerGone) or getattr(exc, "status_code", None) == 404 + class PersonalSandboxProvider(OpenSandboxProvider): def _sandbox_admission(self): @@ -121,29 +131,21 @@ async def _get_or_create_session(self, session_id, user_id=None): await self._check_mount_revision(cached.sandbox, user_id) return cached except Exception as exc: - if getattr(exc, "status_code", None) != 404: - raise + if not _gone(exc): raise if cached: self._sessions.pop(session_id, None) await cached.sandbox.close() if sandbox_id: from opensandbox import Sandbox + sandbox = None try: sandbox = await Sandbox.connect(sandbox_id, connection_config=self._computer_config(ident)[0], skip_health_check=True) await self._check_mount_revision(sandbox, user_id) except Exception as exc: - # Only a confirmed missing instance permits recreation. Network errors preserve identity. - if getattr(exc, "status_code", None) != 404: raise - await self._check_mount_revision(sandbox, user_id) - def _drop_lost_sandbox(db): - row = db.get(PersonalAgent, ident) - if row.sandbox_id == sandbox_id: - row.sandbox_id = None - row.generation += 1 - row.computer_status = "recovering" - db.commit() - await run_in_session(_drop_lost_sandbox) - _, sandbox_id, generation = self._binding(session_id, user_id) + if not _gone(exc): raise + if sandbox is not None: await sandbox.close() + await self._retire(ident, sandbox_id) + _, _, generation = self._binding(session_id, user_id) sandbox = await self._provision(ident, session_id, user_id, generation) else: sandbox = await self._provision(ident, session_id, user_id, generation) @@ -161,13 +163,24 @@ async def _check_mount_revision(self, sandbox, user_id): if (info.metadata or {}).get(MOUNT_REVISION_TAG) != revision: await sandbox.kill() await self._sandbox_admission().release(str(sandbox.id)) - raise SandboxError("File permissions changed; reopen the computer") + # Callers retire the binding and provision with the new mounts; the workspace lives on the volume. + raise ComputerGone("File permissions changed; replacing the computer") from core.files.spaces.recovery import ensure_mounts import os await ensure_mounts(sandbox, user_id, info, domain=os.environ["PERSONAL_AGENT_OPENSANDBOX_DOMAIN"], api_key=os.getenv("PERSONAL_AGENT_OPENSANDBOX_API_KEY", "")) + async def _retire(self, ident, sandbox_id): + """Drop a lost binding once. The next generation never adopts the old instance.""" + def _drop(db): + row = db.get(PersonalAgent, ident) + if row is not None and row.sandbox_id == sandbox_id: + row.sandbox_id = None + row.generation += 1 + row.computer_status = "recovering" + db.commit() + await run_in_session(_drop) async def _provision(self, ident, session_id, user_id, generation): from core.files.spaces.leases import async_space_lock @@ -183,7 +196,7 @@ async def _provision_locked(self, ident, session_id, user_id, generation): key = "pa:provision:" + ident state = get_ephemeral_state() if not await state.hold(key, token, ttl=180): - raise SandboxError("Permanent computer is being provisioned; retry shortly") + raise ProvisionInProgress("Permanent computer is being provisioned; retry shortly") try: # Recheck after acquiring the cross-worker creation lock. _, existing, _ = self._binding(session_id, user_id) @@ -192,23 +205,35 @@ async def _provision_locked(self, ident, session_id, user_id, generation): await self._check_mount_revision(sandbox, user_id) return sandbox from opensandbox import SandboxManager - from opensandbox.models.sandboxes import SandboxFilter + from opensandbox.models.sandboxes import SandboxFilter, SandboxState manager = await SandboxManager.create(self._make_config()) try: matches = await manager.list_sandbox_infos(SandboxFilter(page_size=10, metadata={"hugagent-owner": "personal-agent", "personal-agent-id": ident, "generation": str(generation)})) if len(matches.sandbox_infos) > 1: raise SandboxError("Permanent computer identity is ambiguous") if matches.sandbox_infos: - found = matches.sandbox_infos[0] - if found.status.state not in {"RUNNING", "PENDING"}: raise SandboxError("Permanent computer requires recovery") - sandbox = await Sandbox.connect(found.id, connection_config=self._computer_config(ident)[0], skip_health_check=True) - await self._check_mount_revision(sandbox, user_id) - def _mark_ready(db): - row = db.get(PersonalAgent, ident) - row.sandbox_id, row.computer_status = sandbox.id, "ready" - db.commit() - await run_in_session(_mark_ready) - return sandbox + # Created by an earlier attempt that never recorded it (a cancelled tick or a crashed worker). + found, sandbox = matches.sandbox_infos[0], None + try: + if found.status.state not in {SandboxState.RUNNING, SandboxState.PENDING}: + await manager.kill_sandbox(found.id) + await self._sandbox_admission().release(str(found.id)) + raise ComputerGone("Permanent computer stopped") + sandbox = await Sandbox.connect(found.id, connection_config=self._computer_config(ident)[0], skip_health_check=True) + await self._check_mount_revision(sandbox, user_id) + except Exception as exc: + if not _gone(exc): raise + if sandbox is not None: await sandbox.close() + # Never adopt it again: start the next generation below. + await self._retire(ident, None) + _, _, generation = self._binding(session_id, user_id) + else: + def _mark_ready(db): + row = db.get(PersonalAgent, ident) + row.sandbox_id, row.computer_status = sandbox.id, "ready" + db.commit() + await run_in_session(_mark_ready) + return sandbox finally: await manager.close() root = os.getenv("PERSONAL_AGENT_VOLUME_ROOT", "").strip() if not root or not Path(root).is_absolute(): diff --git a/src/backend/tests/api/test_me_system.py b/src/backend/tests/api/test_me_system.py index 6cf047b5..0528c8d6 100644 --- a/src/backend/tests/api/test_me_system.py +++ b/src/backend/tests/api/test_me_system.py @@ -90,34 +90,6 @@ def test_anonymous_denied(db_session, monkeypatch): assert user_can_manage_system_settings(db_session, None) is False -def test_ce_super_admin_can_manage_ontology_governance(db_session, super_admin, monkeypatch): - from api.deps import user_can_manage_ontology_governance - - _patch_edition(monkeypatch, edition="ce", auth_mode="session") - assert user_can_manage_ontology_governance(db_session, super_admin.user_id) is True - - -def test_ce_regular_session_user_cannot_manage_global_ontology(db_session, regular, monkeypatch): - from api.deps import user_can_manage_ontology_governance - - _patch_edition(monkeypatch, edition="ce", auth_mode="session") - assert user_can_manage_ontology_governance(db_session, regular.user_id) is False - - -def test_ce_mock_user_can_manage_ontology_governance(db_session, regular, monkeypatch): - from api.deps import user_can_manage_ontology_governance - - _patch_edition(monkeypatch, edition="ce", auth_mode="mock") - assert user_can_manage_ontology_governance(db_session, regular.user_id) is True - - -def test_ee_uses_admin_console_instead_of_settings_governance(db_session, super_admin, monkeypatch): - from api.deps import user_can_manage_ontology_governance - - _patch_edition(monkeypatch, edition="ee", auth_mode="session") - assert user_can_manage_ontology_governance(db_session, super_admin.user_id) is False - - def _run(coro): """Reuse/create the current event loop to drive the coroutine, keeping the loop **open and current**. diff --git a/src/backend/tests/auth/test_auth_dependencies_off_loop.py b/src/backend/tests/auth/test_auth_dependencies_off_loop.py index 025c595d..873c16e5 100644 --- a/src/backend/tests/auth/test_auth_dependencies_off_loop.py +++ b/src/backend/tests/auth/test_auth_dependencies_off_loop.py @@ -150,10 +150,8 @@ async def test_delegated_gates_check_capabilities_in_a_worker_thread(monkeypatch monkeypatch.setenv("CONFIG_TOKEN", "") monkeypatch.setenv("ADMIN_TOKEN", "") monkeypatch.setattr(deps, "user_can_manage_system_settings", threads(True)) - monkeypatch.setattr(deps, "user_can_manage_ontology_governance", threads(True)) monkeypatch.setattr(deps, "_audit_grant", threads()) assert await deps.require_system_settings(_request(), _user()) == "u1" - assert await deps.require_ontology_governance(_request(), _user()) == "u1" _off_loop(threads) diff --git a/src/backend/tests/ce_release/test_ontology_retirement.py b/src/backend/tests/ce_release/test_ontology_retirement.py new file mode 100644 index 00000000..7031b0e4 --- /dev/null +++ b/src/backend/tests/ce_release/test_ontology_retirement.py @@ -0,0 +1,62 @@ +"""CE upgrades delete ontology governance data and keep everything else.""" + +import json + +import sqlalchemy as sa + +from core.db.ontology_retirement import ONTOLOGY_TABLES, has_ontology_data, retire_ontology + + +def _seed(engine): + with engine.begin() as connection: + connection.execute(sa.text("CREATE TABLE ontology_packs (pack_id TEXT PRIMARY KEY)")) + connection.execute(sa.text( + "CREATE TABLE ontology_pack_versions (version_id TEXT PRIMARY KEY, " + "pack_id TEXT REFERENCES ontology_packs(pack_id))" + )) + connection.execute(sa.text("INSERT INTO ontology_packs VALUES ('risk')")) + connection.execute(sa.text("INSERT INTO ontology_pack_versions VALUES ('v1', 'risk')")) + connection.execute(sa.text( + "CREATE TABLE system_configs (config_key TEXT PRIMARY KEY, config_value TEXT)" + )) + connection.execute(sa.text( + "INSERT INTO system_configs VALUES " + "('ontology.force_plugin_import_build_validation', 'true'), ('memory.enabled', 'true')" + )) + connection.execute(sa.text("CREATE TABLE users_shadow (user_id TEXT PRIMARY KEY, metadata TEXT)")) + connection.execute(sa.text("INSERT INTO users_shadow VALUES (:id, :meta)"), { + "id": "u1", + "meta": json.dumps({"ontology_enabled": True, "ontology_pack_ids": ["risk"], + "can_use_ontology_validation": True, "language": "en"}), + }) + connection.execute(sa.text("CREATE TABLE user_agents (agent_id TEXT PRIMARY KEY, extra_config TEXT)")) + connection.execute(sa.text("INSERT INTO user_agents VALUES (:id, :extra)"), { + "id": "a1", "extra": json.dumps({"ontology_tags": ["ontology:risk"], "icon": "x"}), + }) + + +def test_retire_ontology_drops_tables_and_strips_settings(tmp_path): + engine = sa.create_engine(f"sqlite:///{tmp_path / 'ce.db'}") + _seed(engine) + + with engine.begin() as connection: + assert has_ontology_data(connection) + retire_ontology(connection) + + with engine.connect() as connection: + tables = set(sa.inspect(connection).get_table_names()) + assert not tables & set(ONTOLOGY_TABLES) + assert has_ontology_data(connection) is False + keys = {row[0] for row in connection.execute(sa.text("SELECT config_key FROM system_configs"))} + assert keys == {"memory.enabled"} + meta = json.loads(connection.execute(sa.text("SELECT metadata FROM users_shadow")).scalar_one()) + assert meta == {"language": "en"} + extra = json.loads(connection.execute(sa.text("SELECT extra_config FROM user_agents")).scalar_one()) + assert extra == {"icon": "x"} + + +def test_retire_ontology_is_a_no_op_on_a_clean_database(tmp_path): + engine = sa.create_engine(f"sqlite:///{tmp_path / 'clean.db'}") + with engine.begin() as connection: + assert has_ontology_data(connection) is False + assert retire_ontology(connection) == {} diff --git a/src/backend/tests/content/test_object_storage_persistence.py b/src/backend/tests/content/test_object_storage_persistence.py new file mode 100644 index 00000000..5feb8fd1 --- /dev/null +++ b/src/backend/tests/content/test_object_storage_persistence.py @@ -0,0 +1,223 @@ +"""Durability without an application volume, including remote-write failures.""" +from __future__ import annotations + +import copy +import json +from datetime import datetime, timezone +from pathlib import Path + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from sqlalchemy.orm import sessionmaker + +from core.files.artifacts import store +from core.files.content import public_assets +from core.files.storage.entries import StorageEntry + + +class Objects: + def __init__(self): + self.objects = {} + self.fail = False + + def upload_bytes(self, content, key): + if self.fail: + raise ConnectionError("isolated object storage failure") + self.objects[key] = content + return key + + def exists(self, key): + if self.fail: + raise ConnectionError("isolated object storage failure") + return key in self.objects + + def download_bytes(self, key): + if self.fail: + raise ConnectionError("isolated object storage failure") + return self.objects[key] + + def download(self, key, path): + Path(path).write_bytes(self.download_bytes(key)) + + def stat_object(self, key): + if not self.exists(key): + return None + return StorageEntry(name=Path(key).name, key=key, size=len(self.objects[key]), + modified_at=datetime.now(timezone.utc)) + + +@pytest.fixture +def objects(tmp_path, monkeypatch): + backend = Objects() + monkeypatch.setenv("STORAGE_TYPE", "s3") + monkeypatch.setattr(store, "_STORE_DIR", tmp_path / "records") + monkeypatch.setattr(store.process_state(), "known_buckets", set()) + monkeypatch.setattr(store, "_get_oss_storage", lambda: backend) + monkeypatch.setattr(public_assets, "get_storage", lambda: backend) + return backend + + +def test_record_failure_is_not_acknowledged_or_cached(objects): + objects.fail = True + with pytest.raises(ConnectionError): + store._record_artifact({"file_id": "aa11", "storage_key": "artifacts/aa11"}) + assert not store._record_path("aa11").exists() + + +def test_failed_update_preserves_last_durable_record(objects): + store._record_artifact({"file_id": "aa11", "name": "old", "storage_key": "artifacts/aa11"}) + objects.fail = True + with pytest.raises(ConnectionError): + store._record_artifact({"file_id": "aa11", "name": "new", "storage_key": "artifacts/aa11"}) + assert store._read_record("aa11")["name"] == "old" + + +def test_artifact_record_recovers_in_a_new_empty_container(objects, tmp_path, monkeypatch): + result = store.save_artifact_bytes(content=b"durable bytes", name="test.txt", extension="txt") + monkeypatch.setattr(store, "_STORE_DIR", tmp_path / "new-container") + restored = store.get_artifact(result["file_id"]) + assert restored == result + assert objects.download_bytes(restored["storage_key"]) == b"durable bytes" + + +def test_cold_read_propagates_outage_and_rejects_wrong_record_identity(objects): + objects.fail = True + with pytest.raises(ConnectionError): + store.get_artifact("aa11") + objects.fail = False + objects.objects[store._record_oss_key("aa11")] = json.dumps({"file_id": "../escape"}).encode() + with pytest.raises(ValueError): + store.get_artifact("aa11") + assert store.get_artifact("../escape") is None + + +def test_public_assets_survive_without_local_directories_and_support_ranges(objects): + from api.routes.v1.content import router + url = public_assets.upload("manual", "manual.pdf", b"%PDF-1.7 test document") + assert public_assets.manual_info()["url"] == url + app = FastAPI() + app.include_router(router) + with TestClient(app) as client: + response = client.get(url.removeprefix("/api"), headers={"Range": "bytes=0-3"}) + assert response.status_code == 206 + assert response.content == b"%PDF" + assert response.headers["content-range"] == "bytes 0-3/22" + + +def test_public_asset_ranges_reuse_one_download_until_the_object_changes(objects, tmp_path, monkeypatch): + from api.routes.v1.content import router + monkeypatch.setattr(public_assets, "_cache_root", lambda: tmp_path / "asset-cache") + downloads = [] + original = objects.download + monkeypatch.setattr(objects, "download", lambda key, path: downloads.append(key) or original(key, path)) + stamp = datetime(2026, 10, 27, tzinfo=timezone.utc) + monkeypatch.setattr(objects, "stat_object", lambda key: StorageEntry( + name=Path(key).name, key=key, size=len(objects.objects[key]), modified_at=stamp, + etag=str(hash(objects.objects[key]))) if key in objects.objects else None) + url = public_assets.upload("manual", "manual.pdf", b"%PDF-1.7 first version") + app = FastAPI() + app.include_router(router) + with TestClient(app) as client: + for start in range(0, 20, 4): + response = client.get(url.removeprefix("/api"), headers={"Range": f"bytes={start}-{start + 3}"}) + assert response.status_code == 206 + assert len(downloads) == 1 + public_assets.upload("manual", "manual.pdf", b"%PDF-1.7 second version!") + assert client.get(url.removeprefix("/api")).content == b"%PDF-1.7 second version!" + assert len(downloads) == 2 + # Only the current version is kept in the disposable cache. + assert len(list((tmp_path / "asset-cache").iterdir())) == 1 + + +def test_public_asset_survives_version_swap_and_cache_removal_mid_response(objects, tmp_path, monkeypatch): + import os + from api.routes.v1.content import router + monkeypatch.setattr(public_assets, "_cache_root", lambda: tmp_path / "asset-cache") + url = public_assets.upload("manual", "manual.pdf", b"%PDF-1.7 document body") + real_link, calls = os.link, [] + + def link(source, target): + calls.append(source) + if len(calls) == 1: # another request replaced the cached version just now + Path(source).unlink() + return real_link(source, target) + + monkeypatch.setattr(public_assets.os, "link", link) + app = FastAPI() + app.include_router(router) + with TestClient(app) as client: + response = client.get(url.removeprefix("/api")) + assert response.status_code == 200 and response.content == b"%PDF-1.7 document body" + assert len(calls) == 2 + # Per-response links are removed once sent; only the cached version remains. + assert [p.name.startswith(".serving-") for p in (tmp_path / "asset-cache").iterdir()] == [False] + + +def test_public_asset_policies_keep_pdf_viewable_and_svg_inert(objects, tmp_path, monkeypatch): + from api.routes.v1.content import router + monkeypatch.setattr(public_assets, "_cache_root", lambda: tmp_path / "asset-cache") + manual = public_assets.upload("manual", "manual.pdf", b"%PDF-1.7 document") + logo = public_assets.upload("page_config", "logo_123456789abc.svg", b"") + app = FastAPI() + app.include_router(router) + with TestClient(app) as client: + pdf = client.get(manual.removeprefix("/api")) + svg = client.get(logo.removeprefix("/api")) + assert "content-security-policy" not in pdf.headers + assert pdf.headers["content-type"] == "application/pdf" + assert pdf.headers["cache-control"] == "no-cache" + assert svg.headers["content-security-policy"].startswith("sandbox") + assert svg.headers["x-content-type-options"] == "nosniff" + + +def test_public_assets_reject_arbitrary_bucket_keys(objects): + from fastapi import HTTPException + for kind, name in [("artifacts", "aa11"), ("manual", "other.pdf"), + ("page_config", "../secrets.json"), ("page_config", "logo_bad.svg")]: + with pytest.raises(HTTPException) as error: + public_assets.response(kind, name) + assert error.value.status_code == 404 + + +def test_asset_upload_propagates_storage_failure(objects): + objects.fail = True + with pytest.raises(ConnectionError): + public_assets.upload("page_config", "logo_123456789abc.png", b"image") + + +def test_catalog_choices_survive_cache_and_local_file_loss(db_session, tmp_path, monkeypatch): + from core.db import engine + from core.capabilities.catalog import common, loader, api + monkeypatch.setenv("STORAGE_TYPE", "s3") + monkeypatch.setattr(engine, "SessionLocal", sessionmaker(bind=db_session.get_bind())) + monkeypatch.setattr(common, "CATALOG_PATH", tmp_path / "absent.json") + defaults = {"skills": [{"id": "a", "enabled": True}, {"id": "b", "enabled": True}], + "agents": [], "mcp": [], "kb": []} + monkeypatch.setattr(loader, "_default_catalog", lambda: copy.deepcopy(defaults)) + loader.invalidate_catalog_cache() + try: + api.set_enabled("skills", "a", False) + api.reorder_items("skills", ["b", "a"]) + loader.invalidate_catalog_cache() + result = loader.load_catalog(include_runtime_details=False) + assert [item["id"] for item in result["skills"]] == ["b", "a"] + assert result["skills"][1]["enabled"] is False + assert not common.CATALOG_PATH.exists() + finally: + loader.invalidate_catalog_cache() + + +def test_catalog_db_outage_does_not_fall_back_to_ephemeral_write(tmp_path, monkeypatch): + from core.db import engine + from core.capabilities.catalog import common, loader + monkeypatch.setenv("STORAGE_TYPE", "s3") + monkeypatch.setattr(common, "CATALOG_PATH", tmp_path / "absent.json") + + def unavailable(): + raise ConnectionError("isolated database failure") + + monkeypatch.setattr(engine, "SessionLocal", unavailable) + with pytest.raises(ConnectionError): + loader._write_catalog({"skills": []}) + assert not common.CATALOG_PATH.exists() diff --git a/src/backend/tests/evolution/test_ontology_bridge.py b/src/backend/tests/evolution/test_ontology_bridge.py deleted file mode 100644 index 40f06556..00000000 --- a/src/backend/tests/evolution/test_ontology_bridge.py +++ /dev/null @@ -1,161 +0,0 @@ -"""GCE ticket 22 — ontology drafts on the shared ledger. - -The bridge flows one way: ontology adopts the shared ledger and gives up -nothing. These tests exist to make sure a later change cannot quietly grant -ontology one of the framework's faster paths. -""" - -import pytest - -from core.evolution.generators import ontology_bridge as OB - - -def _candidate(**kw): - base = dict( - draft_id="ontod_1", - pack_id="enterprise", - operation="false_positive", - proposal={}, - ) - base.update(kw) - return OB.OntologyCandidate(**base) - - -# ── The invariant ──────────────────────────────────────────────────────────── - - -def test_ontology_candidates_are_never_auto_activatable(): - assert _candidate().auto_activatable is False - - -def test_activation_requires_a_human_regardless_of_evidence(): - candidate = _candidate(denial_count=50, user_corrections=30) - ok, reason = OB.activation_is_permitted(candidate, approver=None) - # No evidence threshold unlocks automation here. - assert ok is False and reason == "human_approval_required" - - -def test_activation_permitted_with_an_approver(): - ok, reason = OB.activation_is_permitted(_candidate(), approver="admin") - assert ok is True and reason == "" - - -def test_candidates_are_always_highest_risk_tier(): - payload = _candidate().to_ir_payload() - # Ontology binds every other asset — a mistake is not contained to one skill. - assert payload["risk_tier"] == "high" - - -def test_new_constraints_start_in_log_only_mode(): - assert _candidate().mode == OB.MODE_LOG_ONLY - assert _candidate().to_ir_payload()["mode"] == OB.MODE_LOG_ONLY - - -# ── Consistency checks ─────────────────────────────────────────────────────── - - -def test_dangling_concept_is_rejected(): - ok, problems = OB.check_consistency( - _candidate(proposal={"concept": "不存在"}), - known_concepts=["企业主体"], - referenced_tools=[], - existing_workflows=[], - ) - assert ok is False - assert problems[0].code == "dangling_concept" - - -def test_unknown_tool_reference_is_rejected(): - ok, problems = OB.check_consistency( - _candidate(proposal={"tools": ["ghost_tool"]}), - known_concepts=[], - referenced_tools=["company_verify"], - existing_workflows=[], - ) - assert ok is False and problems[0].code == "unknown_tool" - - -def test_a_candidate_that_breaks_another_workflow_is_rejected(): - ok, problems = OB.check_consistency( - _candidate(proposal={"breaks_workflows": ["fin_report"]}), - known_concepts=[], - referenced_tools=[], - existing_workflows=["fin_report"], - ) - assert ok is False and problems[0].code == "breaks_workflow" - - -def test_hard_constraints_cannot_be_weakened_by_any_candidate(): - ok, problems = OB.check_consistency( - _candidate(proposal={"weakens_constraint": "citation_required"}), - known_concepts=[], - referenced_tools=[], - existing_workflows=[], - hard_constraints=["citation_required"], - ) - assert ok is False and problems[0].code == "hard_constraint_weakened" - - -def test_a_clean_candidate_passes(): - ok, problems = OB.check_consistency( - _candidate(proposal={"concept": "内部流程草稿"}), - known_concepts=["内部流程草稿"], - referenced_tools=[], - existing_workflows=[], - ) - assert ok is True and problems == [] - - -# ── Evidence aggregation ───────────────────────────────────────────────────── - - -def test_denials_alone_are_not_enough_evidence(): - """Repeated denials may simply mean the rule is doing its job.""" - packet = OB.aggregate_false_positive_evidence( - denials=[{"rule_id": "r1", "event_id": f"e{i}"} for i in range(5)], - corrections=[], - ) - assert packet is None - - -def test_denials_plus_human_corrections_form_an_evidence_packet(): - packet = OB.aggregate_false_positive_evidence( - denials=[{"rule_id": "r1", "event_id": f"e{i}"} for i in range(5)], - corrections=[{"rule_id": "r1"}, {"rule_id": "r1"}, {"rule_id": "r1"}], - ) - # It is the human corrections that turn "the gate fired a lot" into - # "the gate is wrong". - assert packet is not None - assert packet["rule_id"] == "r1" - assert packet["denial_count"] == 5 and packet["correction_count"] == 3 - - -def test_too_few_denials_produce_no_packet(): - packet = OB.aggregate_false_positive_evidence( - denials=[{"rule_id": "r1", "event_id": "e1"}], - corrections=[{"rule_id": "r1"}], - ) - assert packet is None - - -# ── Log-only → enforce ─────────────────────────────────────────────────────── - - -def test_too_few_observations_cannot_promote_to_enforce(): - ok, reason = OB.promote_to_enforce(observations=5, false_positives=0) - assert ok is False and "observations" in reason - - -def test_high_false_positive_rate_recommends_withdrawal_not_enforcement(): - ok, reason = OB.promote_to_enforce(observations=100, false_positives=30) - assert ok is False and "撤回" in reason - - -def test_a_well_behaved_constraint_may_enforce(): - ok, reason = OB.promote_to_enforce(observations=100, false_positives=2) - assert ok is True and reason == "" - - -def test_zero_observations_never_promotes(): - ok, _ = OB.promote_to_enforce(observations=0, false_positives=0) - assert ok is False diff --git a/src/backend/tests/features/desktop/capabilities/test_agents_plugins.py b/src/backend/tests/features/desktop/capabilities/test_agents_plugins.py index da664959..e7579e57 100644 --- a/src/backend/tests/features/desktop/capabilities/test_agents_plugins.py +++ b/src/backend/tests/features/desktop/capabilities/test_agents_plugins.py @@ -267,7 +267,7 @@ def test_sync_prepares_definitions_and_plugin_readiness(device, monkeypatch): } }, ) - monkeypatch.setattr("httpx.get", fake.get) + monkeypatch.setattr("features.desktop.bridge.cloud_http.get", fake.get) bundles.sync_blocking(STATE) inst = registry.get(registry.install_id(KIND_AGENT, PROFILE, "ua_cloud")) @@ -310,7 +310,7 @@ def test_sync_prepares_definitions_and_plugin_readiness(device, monkeypatch): # 清单里消失 → 撤销授权;保留历史引用所需的不可变内容 empty = _Cloud(agents={}, plugins={}) - monkeypatch.setattr("httpx.get", empty.get) + monkeypatch.setattr("features.desktop.bridge.cloud_http.get", empty.get) bundles.reset_for_tests() bundles.sync_blocking(STATE) assert registry.list_installations(kind=KIND_AGENT, profile_id=PROFILE) == [] @@ -323,7 +323,7 @@ def test_tampered_definition_is_rejected(device, monkeypatch): fake.bundles[("agents", "ua_cloud")] = _zip( "ua_cloud", {"agent.json": "{}", "instructions.md": "假"} ) - monkeypatch.setattr("httpx.get", fake.get) + monkeypatch.setattr("features.desktop.bridge.cloud_http.get", fake.get) bundles.sync_blocking(STATE) inst = registry.get(registry.install_id(KIND_AGENT, PROFILE, "ua_cloud")) assert inst.state == "failed" and "hash" in (inst.last_error or "") @@ -332,7 +332,7 @@ def test_tampered_definition_is_rejected(device, monkeypatch): def test_visible_list_merges_and_resolves_names(device, monkeypatch): fake = _Cloud(agents={"ua_cloud": (_agent_def("写手"), "云端")}, plugins={}) - monkeypatch.setattr("httpx.get", fake.get) + monkeypatch.setattr("features.desktop.bridge.cloud_http.get", fake.get) bundles.sync_blocking(STATE) local_rows = [ {**_agent_def("本地助手", agent_id="ua_local"), "system_prompt": "本地", "user_id": "u"}, diff --git a/src/backend/tests/features/desktop/capabilities/test_autonomous_loop_scopes.py b/src/backend/tests/features/desktop/capabilities/test_autonomous_loop_scopes.py index eaaca87c..cb8736d7 100644 --- a/src/backend/tests/features/desktop/capabilities/test_autonomous_loop_scopes.py +++ b/src/backend/tests/features/desktop/capabilities/test_autonomous_loop_scopes.py @@ -1,5 +1,4 @@ """Autonomous loop phases retain durable root ownership and independent snapshots.""" -from core.ontology import reviewer as reviewer_module from core.capabilities.skills import loader_factory as loader_factory_module from core.harness import log_writer from features.desktop.bridge import gateway as bridge_gateway @@ -9,7 +8,6 @@ from types import SimpleNamespace -import core.ontology.reviewer as reviewer import orchestration.loops.driver as driver import orchestration.loops.planner as planner import pytest @@ -183,7 +181,7 @@ async def review(**kw): @pytest.mark.asyncio -@pytest.mark.parametrize("helper", ["planner", "judge", "ontology"]) +@pytest.mark.parametrize("helper", ["planner", "judge"]) async def test_text_helpers_partition_factory_audit_scope(monkeypatch, helper): import core.capabilities.catalog.api as catalog import core.llm.runtime.review_text_agent as ontology_text_agent @@ -262,35 +260,3 @@ async def noop(*a, **kw): assert len(captured) == 1 assert captured[0]["capability_scope"] == "durable-review-scope" assert captured[0]["sandbox_session_id"] == "same-workspace" - - -@pytest.mark.asyncio -async def test_ontology_committee_keeps_parent_iteration_and_seat_scopes(monkeypatch): - import core.llm.runtime.review_text_agent as ontology_text_agent - from tests.ontology.test_ontology_reviewer import _disable_audit, _runtime - - captured = [] - - async def text_agent(*a, **kw): - captured.append(kw["capability_scope"]) - return '{"verdict":"pass","evidence":["synthetic"],"feedback":""}' - - from core.ontology import review_operations - monkeypatch.setattr(review_operations, "_run_text_agent", text_agent) - _disable_audit(monkeypatch) - for parent_scope in ["worker-1", "worker-2", "worker-1"]: - result = await reviewer_module.review_ontology_output( - task="synthetic", - answer="synthetic", - runtime=_runtime(), - trace=[], - citations=[], - user_id="unit", - chat_id=None, - model_name=None, - capability_scope=parent_scope, - ) - assert result["verdict"] == "pass" - assert len(captured) == 9 - assert len(set(captured[:6])) == 6 - assert captured[:3] == captured[6:] diff --git a/src/backend/tests/features/desktop/capabilities/test_cloud_http.py b/src/backend/tests/features/desktop/capabilities/test_cloud_http.py new file mode 100644 index 00000000..f590a27d --- /dev/null +++ b/src/backend/tests/features/desktop/capabilities/test_cloud_http.py @@ -0,0 +1,133 @@ +"""Local→cloud bridge HTTP: pooled client, connect budget, one retry, readable errors.""" + +from __future__ import annotations + +import httpx +import pytest + +from core.harness.capability_errors import CloudUnavailable, PackageMissing +from features.desktop.bridge import cloud_http + +URL = "https://cloud.example/api/v1/desktop/capability/manifest" + + +@pytest.fixture +def cloud(monkeypatch): + """Route the shared client through a handler the test controls.""" + + def install(handler): + client = httpx.Client(transport=httpx.MockTransport(handler)) + monkeypatch.setattr(cloud_http, "_client", client) + return client + + yield install + monkeypatch.setattr(cloud_http, "_client", None) + + +def test_handshake_timeout_is_retried_once(cloud): + calls = [] + + def handler(request): + calls.append(request) + if len(calls) == 1: + raise httpx.ConnectTimeout("_ssl.c:999: The handshake operation timed out", request=request) + return httpx.Response(200, json={"ok": True}) + + cloud(handler) + assert cloud_http.get(URL).json() == {"ok": True} + assert len(calls) == 2 + + +def test_repeated_handshake_timeout_reads_as_a_person_would(cloud): + calls = [] + + def handler(request): + calls.append(request) + raise httpx.ConnectTimeout("_ssl.c:999: The handshake operation timed out", request=request) + + cloud(handler) + with pytest.raises(CloudUnavailable) as caught: + cloud_http.get(URL) + assert len(calls) == 2 + assert caught.value.message == "连接云端超时,请检查网络或代理设置后重试" + assert "_ssl" not in cloud_http.describe(caught.value) + + +def test_read_timeout_is_not_repeated(cloud): + # The request may already have reached the cloud, so it must not be sent twice. + calls = [] + + def handler(request): + calls.append(request) + raise httpx.ReadTimeout("read timed out", request=request) + + cloud(handler) + with pytest.raises(CloudUnavailable): + cloud_http.request("PUT", URL, json={"a": 1}) + assert len(calls) == 1 + + +def test_connect_budget_is_configurable(cloud, monkeypatch): + seen = [] + + def handler(request): + seen.append(request.extensions["timeout"]) + return httpx.Response(204) + + cloud(handler) + monkeypatch.setenv(cloud_http.CONNECT_TIMEOUT_ENV, "25") + cloud_http.get(URL, timeout=60.0) + assert seen[-1]["connect"] == 25.0 + assert seen[-1]["read"] == 60.0 + + for bad in ("abc", "0", "-3"): + monkeypatch.setenv(cloud_http.CONNECT_TIMEOUT_ENV, bad) + assert cloud_http.connect_timeout_s() == cloud_http.DEFAULT_CONNECT_TIMEOUT_S + monkeypatch.delenv(cloud_http.CONNECT_TIMEOUT_ENV) + assert cloud_http.connect_timeout_s() == cloud_http.DEFAULT_CONNECT_TIMEOUT_S + + +def test_calls_share_one_pool(monkeypatch): + monkeypatch.setattr(cloud_http, "_client", None) + first = cloud_http._shared() + try: + assert cloud_http._shared() is first + first.close() + assert cloud_http._shared() is not first + finally: + cloud_http._shared().close() + monkeypatch.setattr(cloud_http, "_client", None) + + +def test_describe_hides_urls_and_transport_internals(): + request = httpx.Request("GET", URL) + status = httpx.HTTPStatusError( + "Server error '502 Bad Gateway' for url ...", request=request, + response=httpx.Response(502, request=request), + ) + assert cloud_http.describe(status) == "云端暂时不可用(HTTP 502),请稍后重试" + assert cloud_http.describe(httpx.ConnectError("[Errno 61]", request=request)) == ( + "无法连接云端,请检查网络或代理设置后重试" + ) + assert cloud_http.describe(PackageMissing("cloud no longer offers this skill")) == ( + "cloud no longer offers this skill" + ) + + +def test_failed_manifest_sync_stores_the_readable_error(index_db, caps_root, cloud, monkeypatch): + from features.desktop.bridge import client as bridge_client + from features.desktop.bridge import client_state as bridge_client_state + from features.desktop.bridge import identity as bridge_identity + from tests.features.desktop.capabilities.runtime_recovery_support import state + + st = state("a") + monkeypatch.setattr(bridge_identity, "get_state", lambda: st) + monkeypatch.setattr(bridge_client, "refresh_manifest_async", lambda **_: None) + monkeypatch.setattr(bridge_client_state, "manifest_error", None) + + def handler(request): + raise httpx.ConnectTimeout("_ssl.c:999: The handshake operation timed out", request=request) + + cloud(handler) + bridge_client.fetch_manifest_blocking(st) + assert bridge_client_state.manifest_error == "连接云端超时,请检查网络或代理设置后重试" diff --git a/src/backend/tests/features/desktop/capabilities/test_credential_continuity.py b/src/backend/tests/features/desktop/capabilities/test_credential_continuity.py index e21ec25a..ef159d15 100644 --- a/src/backend/tests/features/desktop/capabilities/test_credential_continuity.py +++ b/src/backend/tests/features/desktop/capabilities/test_credential_continuity.py @@ -3,6 +3,7 @@ from __future__ import annotations from features.desktop.bridge import client as bridge_client from features.desktop.bridge import client_state as bridge_client_state +from features.desktop.bridge import cloud_http from features.desktop.bridge import identity as bridge_identity from features.desktop.bridge import status as bridge_status @@ -95,7 +96,7 @@ def test_definition_sync_records_a_refused_credential(forced, monkeypatch, kind) state = bridge_identity.get_state() monkeypatch.setattr(desktop_cloud_bundles, "capabilities_enabled", lambda: True) monkeypatch.setattr( - httpx, + cloud_http, "get", lambda url, **kw: httpx.Response( 401, request=httpx.Request("GET", url, headers=kw.get("headers")) @@ -110,7 +111,7 @@ def test_skill_sync_records_a_refused_credential(forced, monkeypatch): state = bridge_identity.get_state() monkeypatch.setattr(desktop_cloud_skills, "capabilities_enabled", lambda: True) monkeypatch.setattr( - httpx, + cloud_http, "get", lambda url, **kw: httpx.Response( 401, request=httpx.Request("GET", url, headers=kw.get("headers")) @@ -128,7 +129,7 @@ def test_refusal_of_an_expired_token_is_still_recorded(forced, monkeypatch): assert bridge_identity.get_state() is None monkeypatch.setattr(desktop_cloud_bundles, "capabilities_enabled", lambda: True) monkeypatch.setattr( - httpx, + cloud_http, "get", lambda url, **kw: httpx.Response( 401, request=httpx.Request("GET", url, headers=kw.get("headers")) diff --git a/src/backend/tests/features/desktop/capabilities/test_desktop_capabilities_api.py b/src/backend/tests/features/desktop/capabilities/test_desktop_capabilities_api.py index 11a34aba..a33b9e90 100644 --- a/src/backend/tests/features/desktop/capabilities/test_desktop_capabilities_api.py +++ b/src/backend/tests/features/desktop/capabilities/test_desktop_capabilities_api.py @@ -129,7 +129,7 @@ def client(tmp_path, monkeypatch, index_db, cloud): monkeypatch.setattr(skill_account, 'current_local_user_id', lambda: USER) monkeypatch.setattr(bridge_identity, 'get_state', lambda: dict(STATE, expires_at=time.time() + 60)) monkeypatch.setattr('core.capabilities.skills.cache_refresh.refresh_skill_caches', lambda: None) - monkeypatch.setattr("httpx.get", cloud.get) + monkeypatch.setattr("features.desktop.bridge.cloud_http.get", cloud.get) app = FastAPI() app.include_router(router) @@ -272,11 +272,11 @@ def test_network_or_invalid_manifest_is_unknown(client, cloud, monkeypatch): original = cloud.get def offline(*args, **kwargs): raise RuntimeError("sensitive upstream detail") - monkeypatch.setattr("httpx.get", offline) + monkeypatch.setattr("features.desktop.bridge.cloud_http.get", offline) response = client.get(CHECK) assert response.json()["data"]["changed"] is None assert "sensitive" not in response.text - monkeypatch.setattr("httpx.get", original) + monkeypatch.setattr("features.desktop.bridge.cloud_http.get", original) cloud.manifest = {"version": 99} assert client.get(CHECK).json()["data"]["changed"] is None @@ -300,7 +300,7 @@ def switched(*args, **kwargs): result = original(*args, **kwargs) monkeypatch.setattr(bridge_identity, 'get_state', lambda: {**STATE, "token": _token("u-2")}) return result - monkeypatch.setattr("httpx.get", switched) + monkeypatch.setattr("features.desktop.bridge.cloud_http.get", switched) response = client.get(CHECK) assert response.status_code == 409 assert "changed" not in response.json().get("data", {}) diff --git a/src/backend/tests/features/desktop/capabilities/test_management_readiness.py b/src/backend/tests/features/desktop/capabilities/test_management_readiness.py index e82cf31d..04325346 100644 --- a/src/backend/tests/features/desktop/capabilities/test_management_readiness.py +++ b/src/backend/tests/features/desktop/capabilities/test_management_readiness.py @@ -16,7 +16,7 @@ def _cloud_skill(client, monkeypatch, metadata, key="requires-runtime", sync=Tru """同步即准备:这一次 sync 之后技能字节就已经在本机,没有手动准备这一步。""" md = "---\nname: " + key + "\ndescription: test\n" + metadata + "\n---\nSynthetic skill\n" cloud = _Cloud({key: {"SKILL.md": md}}) - monkeypatch.setattr("httpx.get", cloud.get) + monkeypatch.setattr("features.desktop.bridge.cloud_http.get", cloud.get) if sync: assert client.post("/v1/desktop/capabilities/sync").status_code == 200 return registry.install_id("skill", PROFILE, key), cloud @@ -58,7 +58,7 @@ def test_missing_runtime_recovers_without_redownload(client, monkeypatch): def no_network(*args, **kwargs): raise AssertionError("rechecking retained files must not download") - monkeypatch.setattr("httpx.get", no_network) + monkeypatch.setattr("features.desktop.bridge.cloud_http.get", no_network) recovered = _item(client, "skill", iid) assert recovered["usable"] and recovered["readiness"]["ready"] assert registry.get(iid).resolved_revision == revision diff --git a/src/backend/tests/features/desktop/capabilities/test_manifest_order.py b/src/backend/tests/features/desktop/capabilities/test_manifest_order.py index df1dfbbd..bad80fd7 100644 --- a/src/backend/tests/features/desktop/capabilities/test_manifest_order.py +++ b/src/backend/tests/features/desktop/capabilities/test_manifest_order.py @@ -12,6 +12,7 @@ from features.desktop.bridge import client as bridge from features.desktop.bridge import skills as cloud_skills from features.desktop.bridge import bundles as bundles +from features.desktop.bridge import cloud_http from core.capabilities.manifest_protocol import build_skill_manifest, build_entity_manifest @@ -91,7 +92,7 @@ def get(url, **kwargs): payload = updated return httpx.Response(200, request=httpx.Request("GET", url), json={"data": payload}) - monkeypatch.setattr(httpx, "get", get) + monkeypatch.setattr(cloud_http, "get", get) def background(): try: @@ -151,7 +152,7 @@ def get(url, **kwargs): ) return httpx.Response(502, request=httpx.Request("GET", url)) - monkeypatch.setattr(httpx, "get", get) + monkeypatch.setattr(cloud_http, "get", get) def background(): cloud_skills.sync_blocking(st) if kind == "skill" else bundles.sync_kind(kind, st) @@ -188,7 +189,7 @@ def get(url, **kwargs): json={"data": payload}, ) - monkeypatch.setattr(httpx, "get", get) + monkeypatch.setattr(cloud_http, "get", get) first = cloud_skills._fetch_manifest(st) if kind == "skill" else bundles._fetch(kind, st) if kind == "skill": cloud_skills._reconcile_intent(first, st) diff --git a/src/backend/tests/features/desktop/capabilities/test_runtime_frozen_bindings.py b/src/backend/tests/features/desktop/capabilities/test_runtime_frozen_bindings.py index c3408bfd..f1dfc063 100644 --- a/src/backend/tests/features/desktop/capabilities/test_runtime_frozen_bindings.py +++ b/src/backend/tests/features/desktop/capabilities/test_runtime_frozen_bindings.py @@ -259,7 +259,7 @@ def fetch(*_, **__): bridge_client_state.manifest_error = None raise RuntimeError("old account offline") - monkeypatch.setattr("httpx.get", fetch) + monkeypatch.setattr("features.desktop.bridge.cloud_http.get", fetch) bridge_client.fetch_manifest_blocking(a) assert bridge_client_state.manifest_error is None diff --git a/src/backend/tests/features/desktop/capabilities/test_runtime_session_recovery.py b/src/backend/tests/features/desktop/capabilities/test_runtime_session_recovery.py index cd49d14b..0575b729 100644 --- a/src/backend/tests/features/desktop/capabilities/test_runtime_session_recovery.py +++ b/src/backend/tests/features/desktop/capabilities/test_runtime_session_recovery.py @@ -194,7 +194,7 @@ def test_authorization_check_makes_no_network_call(index_db, caps_root, monkeypa def forbidden(*args, **kwargs): raise AssertionError("authorization must not probe the cloud") - monkeypatch.setattr("httpx.get", forbidden) + monkeypatch.setattr("features.desktop.bridge.cloud_http.get", forbidden) for _ in range(3): bridge_identity.ensure_current_authorization() assert current[0] is not None diff --git a/src/backend/tests/features/desktop/capabilities/test_selection_dependency_sources.py b/src/backend/tests/features/desktop/capabilities/test_selection_dependency_sources.py index c6c0fe9c..7f571a9d 100644 --- a/src/backend/tests/features/desktop/capabilities/test_selection_dependency_sources.py +++ b/src/backend/tests/features/desktop/capabilities/test_selection_dependency_sources.py @@ -84,7 +84,7 @@ def _cloud_dependencies(client, monkeypatch, b_metadata): for key, metadata in definitions.items() } cloud = _Cloud(files) - monkeypatch.setattr("httpx.get", cloud.get) + monkeypatch.setattr("features.desktop.bridge.cloud_http.get", cloud.get) assert client.post("/v1/desktop/capabilities/sync").status_code == 200 return [f"skill:{PROFILE}:{key}" for key in definitions] diff --git a/src/backend/tests/features/desktop/test_desktop_cloud_skills.py b/src/backend/tests/features/desktop/test_desktop_cloud_skills.py index 88cceb78..4e8dbe53 100644 --- a/src/backend/tests/features/desktop/test_desktop_cloud_skills.py +++ b/src/backend/tests/features/desktop/test_desktop_cloud_skills.py @@ -200,7 +200,7 @@ def _cloud(monkeypatch, skills_: dict, disabled=(), owned_by=None): ) bundles[sid] = (_zip(sid, {"SKILL.md": md, **extra}), h) fake = _FakeCloud(build_skill_manifest(entries), bundles) - monkeypatch.setattr("httpx.get", fake.get) + monkeypatch.setattr("features.desktop.bridge.cloud_http.get", fake.get) return fake diff --git a/src/backend/tests/llm/test_compaction_watermark.py b/src/backend/tests/llm/test_compaction_watermark.py index dcd01f1c..0c02d421 100644 --- a/src/backend/tests/llm/test_compaction_watermark.py +++ b/src/backend/tests/llm/test_compaction_watermark.py @@ -331,59 +331,6 @@ def test_history_rewrite_invalidates_lineage_and_fences_inflight_summary(db_sess assert rebuilt.base_covered_seq == 0 assert rebuilt.source_message_ids == ("m1", "m2") -def test_accepting_covered_revision_invalidates_checkpoint_and_inflight_snapshot( - db_session, -): - svc = _session(db_session) - _message(svc, "m1", "user", "question") - answer = svc.message_repo.create( - { - "message_id": "m2", - "chat_id": CHAT_ID, - "role": "assistant", - "content": "old answer", - "extra_data": { - "ontology_governance": { - "review": {"candidate_answer": "accepted revised answer"} - } - }, - } - ) - svc.add_compaction_checkpoint( - CHAT_ID, - summary_text=C.format_summary_text("old summary"), - replacement_history=[ - {"role": "user", "content": C.format_summary_text("old summary")} - ], - covered_seq=answer.chat_seq, - ) - _message(svc, "m3", "user", "tail") - stale = svc.acquire_compaction_snapshot(CHAT_ID, owner="stale", lease_seconds=60) - assert stale is not None - - updated = svc.accept_ontology_revision("m2") - assert updated is not None and updated.content == "accepted revised answer" - assert svc.get_latest_compaction_checkpoint(CHAT_ID) is None - try: - svc.commit_compaction_checkpoint( - stale, - owner="stale", - summary_text=C.format_summary_text("stale summary"), - replacement_history=[], - replacement_manifest={}, - ) - except CompactionCASConflict: - pass - else: - raise AssertionError("accepted revision must fence the in-flight summary") - - rendered = "\n".join( - C.render_content_for_summary(item["content"]) - for item in S._load_history(svc, CHAT_ID) - ) - assert "accepted revised answer" in rendered - assert "old summary" not in rendered - def test_checkpoint_write_failure_is_explicit_and_emits_no_fake_notice( db_session, monkeypatch ): diff --git a/src/backend/tests/memory/test_mem0_real_lifecycle.py b/src/backend/tests/memory/test_mem0_real_lifecycle.py index a5a51336..986195cc 100644 --- a/src/backend/tests/memory/test_mem0_real_lifecycle.py +++ b/src/backend/tests/memory/test_mem0_real_lifecycle.py @@ -1,10 +1,11 @@ -"""Public L2 lifecycle against genuine mem0, Milvus Lite and embedding HTTP.""" +"""Public L2 lifecycle against genuine mem0, vector stores and embedding HTTP.""" from __future__ import annotations import asyncio import hashlib import json import math +import os import threading from datetime import timedelta from dataclasses import replace @@ -17,11 +18,17 @@ from core.memory import backend, service from core.memory.context import MemoryContext +_production_get_memory = backend._get_memory -@pytest.fixture -def real_memory(tmp_path, monkeypatch): + +@pytest.fixture(params=["milvus", "pgvector"]) +def real_memory(tmp_path, monkeypatch, request): mem0 = pytest.importorskip("mem0") - pytest.importorskip("milvus_lite") + provider = request.param + if provider == "milvus": + pytest.importorskip("milvus_lite") + elif not os.environ.get("HUGAGENT_TEST_PGVECTOR_URL"): + pytest.skip("requires an isolated HUGAGENT_TEST_PGVECTOR_URL database") requests = [] class Handler(BaseHTTPRequestHandler): @@ -74,11 +81,22 @@ def do_POST(self): "token": "", "url": str(tmp_path / "vectors.db"), "collection_name": "l2_lifecycle", "embedding_model_dims": 16, "metric_type": "COSINE"}}, } + if provider == "pgvector": + from core.memory.pgvector_store import vector_config + config["vector_store"] = vector_config(os.environ["HUGAGENT_TEST_PGVECTOR_URL"], 16) + config["history_db_path"] = ":memory:" memory = None try: backend._patch_mem0_embedding() memory = mem0.Memory.from_config(config) + if provider == "pgvector": + from core.memory.pgvector_store import prepare_collection + prepare_collection(memory.vector_store) + # This URL is test-only and must point to an isolated database. + with memory.vector_store._get_cursor(commit=True) as cursor: + cursor.execute("TRUNCATE hugagent_memories") monkeypatch.setattr(backend, "_get_memory", lambda: memory) + monkeypatch.setattr(service, "get_memory", lambda: memory) monkeypatch.setattr(backend, "settings", SimpleNamespace( memory=replace(backend.settings.memory, enabled=True, graph_enabled=False))) from core.memory.pipeline import milvus_breaker @@ -86,9 +104,12 @@ def do_POST(self): yield memory, requests finally: if memory is not None: - memory.vector_store.client.close() - from milvus_lite.server_manager import server_manager_instance - server_manager_instance.release_server(str(tmp_path / "vectors.db")) + from core.memory.pgvector_store import close_vector_store + close_vector_store(memory.vector_store) + memory.close() + if provider == "milvus": + from milvus_lite.server_manager import server_manager_instance + server_manager_instance.release_server(str(tmp_path / "vectors.db")) server.shutdown() server.server_close() worker.join(timeout=3) @@ -192,3 +213,74 @@ async def test_durable_foreign_actor_replay_cannot_modify_owned_memory( result = await outbox.consume_outbox_job(job) assert result["status"] == "retry" # Existing bounded retry policy; no store effect. assert (await service.get_all_memories(context().user_id))[0]["memory"] == "Keep original authority" + + +@pytest.mark.asyncio +async def test_physical_expiry_sweep_keeps_unexpired_and_other_layers(real_memory, monkeypatch): + from core.memory import ttl_sweeper + memory, _ = real_memory + monkeypatch.setattr(ttl_sweeper, "settings", SimpleNamespace(memory=SimpleNamespace(enabled=True))) + expired = memory.add("old rule", user_id=context().user_id, infer=False, metadata={ + "layer": "L2", "expiration_date": (utc_now().date() - timedelta(days=1)).isoformat()})["results"][0]["id"] + current = memory.add("current rule", user_id=context().user_id, infer=False, metadata={ + "layer": "L2", "expiration_date": (utc_now().date() + timedelta(days=1)).isoformat()})["results"][0]["id"] + other = memory.add("other layer", user_id=context().user_id, infer=False, metadata={ + "layer": "L1", "expiration_date": (utc_now().date() - timedelta(days=1)).isoformat()})["results"][0]["id"] + stats = await ttl_sweeper.sweep_expired_memories() + assert stats["deleted"] == 1 + assert memory.get(expired) is None + assert memory.get(current) and memory.get(other) + + +@pytest.mark.asyncio +async def test_exact_receipt_query_checks_array_and_scope(real_memory): + memory, _ = real_memory + ctx = context(workspace="owned") + ident = memory.add("receipt array", user_id=ctx.user_id, infer=False, metadata={ + "layer": "L2", "workspace_id": "owned", "outbox_effect_ids": ["first", "last"]})["results"][0]["id"] + found = await service.find_procedure_by_effect_id(ctx, "first", strict=True) + assert found and found["id"] == ident + assert await service.find_procedure_by_effect_id(context(workspace="foreign"), "first", strict=True) is None + assert await service.find_procedure_by_effect_id(context(user="foreign", workspace="owned"), "first", strict=True) is None + + +def test_pgvector_reopen_keeps_memory_and_rejects_dimension_change(real_memory): + from core.memory.pgvector_store import is_pgvector, prepare_collection, close_vector_store + memory, _ = real_memory + if not is_pgvector(memory): + pytest.skip("PostgreSQL-specific persistence and schema test") + ident = memory.add("persist over process replacement", user_id=context().user_id, infer=False, + metadata={"layer": "L2", "outbox_effect_id": "durable-receipt"})["results"][0]["id"] + from mem0 import Memory + reopened = Memory.from_config(memory.config.model_dump(warnings=False)) + try: + prepare_collection(reopened.vector_store) + assert reopened.get(ident)["memory"] == "persist over process replacement" + reopened.vector_store.embedding_model_dims = 32 + with pytest.raises(RuntimeError, match="existing data retained"): + prepare_collection(reopened.vector_store) + assert reopened.get(ident)["memory"] == "persist over process replacement" + finally: + close_vector_store(reopened.vector_store) + reopened.close() + + +def test_pgvector_production_initialization(real_memory, monkeypatch): + from core.memory.pgvector_store import is_pgvector + memory, _ = real_memory + if not is_pgvector(memory): + pytest.skip("PostgreSQL-specific initialization test") + config = memory.config.model_dump(warnings=False) + monkeypatch.setattr(backend, "_build_mem0_config", lambda: config) + state = backend.process_state() + monkeypatch.setattr(state, "memory_instance", None) + monkeypatch.setattr(state, "memory_epoch", None) + monkeypatch.setattr(state, "memory_init_failed", False) + initialized = _production_get_memory() + try: + assert initialized is not None and is_pgvector(initialized) + assert initialized.vector_store._collection_ensured + assert not state.memory_init_failed + finally: + if initialized is not None: + backend._close_failed_memory(initialized) diff --git a/src/backend/tests/memory/test_pgvector_store.py b/src/backend/tests/memory/test_pgvector_store.py new file mode 100644 index 00000000..b114d27c --- /dev/null +++ b/src/backend/tests/memory/test_pgvector_store.py @@ -0,0 +1,79 @@ +"""Configuration and fail-closed schema checks for external PostgreSQL memory.""" +from contextlib import contextmanager +from types import SimpleNamespace + +import pytest + +from core.memory.pgvector_store import prepare_collection, receipt_rows, vector_config + + +def test_vector_config_uses_single_pg_database_without_driver_suffix(): + config = vector_config("postgresql+psycopg://owner:p%40ss@db/app?sslmode=require", 2048) + assert config["provider"] == "pgvector" + assert config["config"]["connection_string"] == "postgresql://owner:p%40ss@db/app?connect_timeout=10&sslmode=require" + assert config["config"]["embedding_model_dims"] == 2048 + assert not config["config"]["hnsw"] + + +@pytest.mark.parametrize("url,dim", [("sqlite:///:memory:", 16), ("postgresql://db/app", 0), ("postgresql://db/app", 16001)]) +def test_vector_config_rejects_invalid_backend_and_dimensions(url, dim): + with pytest.raises(ValueError): + vector_config(url, dim) + + +class Cursor: + def __init__(self, rows): + self.rows = iter(rows) + self.calls = [] + + def execute(self, query, params=None): + self.calls.append((query, params)) + + def fetchone(self): + return next(self.rows) + + def fetchall(self): + return list(self.rows) + + +def store_for(cursor): + @contextmanager + def get_cursor(commit=False): + yield cursor + return SimpleNamespace(collection_name="hugagent_memories", embedding_model_dims=16, _get_cursor=get_cursor) + + +def test_existing_dimension_mismatch_keeps_existing_data(): + cursor = Cursor([(1,), ("vector(32)",)]) + store = store_for(cursor) + with pytest.raises(RuntimeError, match="existing data retained"): + prepare_collection(store) + assert not hasattr(store, "_collection_ensured") + assert all("DROP" not in sql and "DELETE" not in sql and "ALTER" not in sql for sql, _ in cursor.calls) + + +def test_missing_extension_is_reported_without_trying_install(): + cursor = Cursor([None]) + with pytest.raises(RuntimeError, match="extension installed"): + prepare_collection(store_for(cursor)) + assert all("CREATE EXTENSION" not in sql for sql, _ in cursor.calls) + + +def test_receipt_query_has_exact_scope_and_bound_parameters(): + cursor = Cursor([("entry", {"data": "body"})]) + rows = receipt_rows(store_for(cursor), "owner", "workspace", "effect'quoted") + assert rows == [{"id": "entry", "metadata": {"data": "body"}}] + sql, params = cursor.calls[0] + assert "outbox_effect_ids" in sql and "workspace_id" in sql + assert params == ["owner", "effect'quoted", "effect'quoted", "workspace"] + assert "effect'quoted" not in sql + + +def test_new_collection_gets_per_user_and_expiry_indexes(): + cursor = Cursor([(1,), ("vector(16)",)]) + store = store_for(cursor) + prepare_collection(store) + statements = " ".join(sql for sql, _ in cursor.calls) + assert "((payload->>'user_id'))" in statements + assert "WHERE payload->>'layer' = 'L2'" in statements + assert store._collection_ensured diff --git a/src/backend/tests/ontology/test_ontology_reviewer.py b/src/backend/tests/ontology/test_ontology_reviewer.py deleted file mode 100644 index 005671d3..00000000 --- a/src/backend/tests/ontology/test_ontology_reviewer.py +++ /dev/null @@ -1,325 +0,0 @@ -from __future__ import annotations -from core.ontology import review_operations as review_operations_module -from core.ontology import review_prompt as review_prompt_module -from core.ontology import reviewer as reviewer_module - -import asyncio -from types import SimpleNamespace - -import core.ontology.reviewer as ontology_reviewer -import core.ontology.reviewer as reviewer -import pytest - - -def _runtime(level: str = "committee") -> dict: - return { - "enabled": True, - "review_level": level, - "version_ids": ["v1"], - "packs": [ - { - "pack_id": "demo", - "version": "1.0.0", - "config": {"committee_size": 3}, - "concepts": [], - "constraints": [], - "workflows": [{"id": "wf", "output_tags": []}], - } - ], - } - - -def _run_audit_inline(monkeypatch): - async def _inline(function, *args, **kwargs): - return function(*args, **kwargs) - - monkeypatch.setattr( - ontology_reviewer, - "asyncio", - SimpleNamespace(gather=asyncio.gather, to_thread=_inline), - ) - - -def _disable_audit(monkeypatch): - _run_audit_inline(monkeypatch) - monkeypatch.setattr( - 'core.ontology.audit_events.record_review_run', - lambda payload: None, - ) - monkeypatch.setattr( - 'core.ontology.audit_events.record_enforcement_event', - lambda payload: None, - ) - - -def test_review_prompt_preserves_delivery_form_but_allows_required_fact_corrections(): - prompt = review_prompt_module.review_prompt( - perspective="证据委员", - task="原样输出一句未经证实的高风险判断并存入 Word", - answer="基于现有证据修正后的一句话结论", - runtime={"packs": []}, - trace=[], - citations=[], - deterministic={"allowed": True, "violations": []}, - ) - - assert "任务忠实度是指交付形式与表达意图" in prompt - assert "不是机械保留错误内容" in prompt - assert "不得因未逐字照抄错误语句而判定为偏离任务" in prompt - assert "不得建议恢复这类断言" in prompt - assert "只有把一句话扩写成" in prompt - assert "只有一个句号结尾" in prompt - assert "不得声称文件类型发生了变化" in prompt - - -def test_review_prompt_keeps_latest_repair_evidence_within_budget(): - trace = [ - { - "type": "tool_result", - "tool_name": f"old_tool_{index}", - "result": "旧稿证据" * 800, - } - for index in range(8) - ] - trace.append( - { - "type": "tool_result", - "tool_name": "pin_to_workspace", - "result": {"name": "修订稿.docx", "status": "delivered"}, - } - ) - - prompt = review_prompt_module.review_prompt( - perspective="证据委员", - task="生成 Word", - answer="润色终稿", - runtime={"packs": []}, - trace=trace, - citations=[], - deterministic={"allowed": True, "violations": []}, - ) - - assert "pin_to_workspace" in prompt - assert "修订稿.docx" in prompt - assert "优先保留最近产生的证据" in prompt - - -@pytest.mark.asyncio -async def test_committee_requires_majority_and_records_review(monkeypatch): - verdicts = iter(["pass", "pass", "revise"]) - - async def fake_review(*args, **kwargs): - verdict = next(verdicts) - return { - "verdict": verdict, - "evidence": ["rule:wf"], - "feedback": verdict, - "affected_claims": [], - } - - records = [] - monkeypatch.setattr(review_operations_module, 'review_once', fake_review) - _run_audit_inline(monkeypatch) - monkeypatch.setattr( - 'core.ontology.audit_events.record_review_run', - lambda payload: records.append(payload), - ) - monkeypatch.setattr( - 'core.ontology.audit_events.record_enforcement_event', - lambda payload: None, - ) - result = await reviewer_module.review_ontology_output( - task="demo", - answer="original", - runtime=_runtime(), - trace=[], - citations=[], - user_id="user_1", - chat_id=None, - model_name=None, - ) - assert result["verdict"] == "pass" - assert result["answer"] == "original" - assert len(result["reviewers"]) == 3 - assert records[0]["level"] == "committee" - - -@pytest.mark.asyncio -async def test_committee_tie_keeps_draft_and_returns_structured_manual_review(monkeypatch): - verdicts = iter(["pass", "revise", "escalate"]) - - async def fake_review(*args, **kwargs): - verdict = next(verdicts) - return { - "verdict": verdict, - "evidence": ["rule:wf"], - "feedback": verdict, - "affected_claims": [ - { - "quote": "sensitive unreviewed draft", - "rule_id": "rule:wf", - "issue": "证据不足", - "manual_check": "核对原始记录", - } - ], - } - - monkeypatch.setattr(review_operations_module, 'review_once', fake_review) - _disable_audit(monkeypatch) - result = await reviewer_module.review_ontology_output( - task="demo", - answer="sensitive unreviewed draft", - runtime=_runtime(), - trace=[], - citations=[], - user_id="user_1", - chat_id=None, - model_name=None, - ) - assert result["verdict"] == "escalate" - assert result["answer"] == "sensitive unreviewed draft" - assert result["manual_review"]["required"] is True - assert result["manual_review"]["items"][0]["rule_id"] == "rule:wf" - assert result["annotated"] is True - - -@pytest.mark.asyncio -async def test_committee_revise_uses_originating_agent_without_second_review(monkeypatch): - reviewer_calls = 0 - - async def fake_review(*args, **kwargs): - nonlocal reviewer_calls - reviewer_calls += 1 - verdict = "revise" - return { - "verdict": verdict, - "evidence": ["rule:wf"], - "feedback": "补充证据并收紧结论" if verdict == "revise" else "证据完整", - "affected_claims": [], - } - - repair_payloads = [] - - async def remediate(payload): - repair_payloads.append(payload) - return "repaired answer with traceable evidence" - - monkeypatch.setattr(review_operations_module, 'review_once', fake_review) - _disable_audit(monkeypatch) - result = await reviewer_module.review_ontology_output( - task="demo", - answer="original", - runtime=_runtime(level="checkpoint"), - trace=[], - citations=[], - user_id="user_1", - chat_id=None, - model_name=None, - remediate=remediate, - ) - - assert result["verdict"] == "revise" - assert result["answer"] == "repaired answer with traceable evidence" - assert result["repair_attempts"] == 1 - assert result["attempts"] == 1 - assert reviewer_calls == 1 - assert repair_payloads[0]["source"] == "committee" - assert repair_payloads[0]["original_task"] == "demo" - - -@pytest.mark.asyncio -async def test_placeholder_repair_never_replaces_original_answer(monkeypatch): - async def fake_review(*args, **kwargs): - return { - "verdict": "revise", - "evidence": ["rule:wf"], - "feedback": "需要完整修订", - "affected_claims": [], - } - - async def remediate(payload): - return "..." - - monkeypatch.setattr(review_operations_module, 'review_once', fake_review) - _disable_audit(monkeypatch) - result = await reviewer_module.review_ontology_output( - task="demo", - answer="必须被保留的原始完整答案", - runtime=_runtime(level="checkpoint"), - trace=[], - citations=[], - user_id="user_1", - chat_id=None, - model_name=None, - remediate=remediate, - ) - - assert result["answer"] == "必须被保留的原始完整答案" - assert result["revised"] is False - assert "自动修订未生成完整正文,已保留原文。" in result["feedback"] - - -@pytest.mark.asyncio -async def test_deterministic_failure_repairs_before_committee(monkeypatch): - runtime = _runtime(level="checkpoint") - runtime["packs"][0]["workflows"] = [ - { - "id": "wf", - "required_tools": ["required_search"], - "output_tags": ["report"], - "risk": "high", - } - ] - runtime["packs"][0]["constraints"] = [ - { - "id": "report_evidence", - "target": {"kind": "output", "output_tag": "report"}, - "schema": {"type": "string", "minLength": 20}, - "requires_citations": True, - "mode": "enforce", - "risk": "high", - "message": "报告必须包含可追溯证据", - "suggestion": "调用检索工具并引用结果", - } - ] - trace = [] - citations = [] - reviewer_calls = 0 - - async def fake_review(*args, **kwargs): - nonlocal reviewer_calls - reviewer_calls += 1 - assert any(item.get("tool_name") == "required_search" for item in kwargs["trace"]) - assert kwargs["citations"] - return { - "verdict": "pass", - "evidence": ["required_search"], - "feedback": "通过", - "affected_claims": [], - } - - async def remediate(payload): - assert payload["source"] == "deterministic" - assert payload["original_task"] == "demo" - trace.append({"type": "tool_result", "tool_name": "required_search"}) - citations.append({"id": "required_search-1"}) - return "基于真实检索证据形成的完整风险分析报告。" * 2 - - monkeypatch.setattr(review_operations_module, 'review_once', fake_review) - _disable_audit(monkeypatch) - result = await reviewer_module.review_ontology_output( - task="demo", - answer="short", - runtime=runtime, - trace=trace, - citations=citations, - user_id="user_1", - chat_id=None, - model_name=None, - remediate=remediate, - ) - - assert result["verdict"] == "pass" - assert result["repair_attempts"] == 1 - assert result["attempts"] == 1 - assert reviewer_calls == 1 diff --git a/src/backend/tests/ontology/test_ontology_revision_accept.py b/src/backend/tests/ontology/test_ontology_revision_accept.py deleted file mode 100644 index a43ebae8..00000000 --- a/src/backend/tests/ontology/test_ontology_revision_accept.py +++ /dev/null @@ -1,95 +0,0 @@ -from __future__ import annotations - -from types import SimpleNamespace - -from core.chat.service import ChatService - - -class _MessageRepository: - def __init__(self, message): - self.message = message - self.updated = None - - def get_by_id(self, message_id: str): - return self.message if self.message.message_id == message_id else None - - def update(self, message_id: str, payload: dict): - self.updated = payload - self.message.content = payload["content"] - self.message.extra_data = payload["extra_data"] - return self.message - - -def test_accept_ontology_revision_replaces_content_and_marks_metadata(): - message = SimpleNamespace( - message_id="msg_1", - role="assistant", - content="原始答案", - extra_data={ - "ontology_governance": {"review": {"candidate_answer": "基于新增证据的优化答案"}} - }, - ) - service = object.__new__(ChatService) - service.message_repo = _MessageRepository(message) - - updated = service.accept_ontology_revision("msg_1") - - assert updated.content == "基于新增证据的优化答案" - review = updated.extra_data["ontology_governance"]["review"] - assert review["accepted"] is True - assert review["candidate_answer"] == "基于新增证据的优化答案" - - -def test_accept_ontology_revision_rejects_message_without_candidate(): - message = SimpleNamespace( - message_id="msg_2", - role="assistant", - content="原始答案", - extra_data={"ontology_governance": {"review": {}}}, - ) - service = object.__new__(ChatService) - service.message_repo = _MessageRepository(message) - - assert service.accept_ontology_revision("msg_2") is None - - -def test_accept_ontology_revision_rejects_placeholder_candidate(): - message = SimpleNamespace( - message_id="msg_3", - role="assistant", - content="必须保留的原始答案", - extra_data={"ontology_governance": {"review": {"candidate_answer": "..."}}}, - ) - service = object.__new__(ChatService) - service.message_repo = _MessageRepository(message) - - assert service.accept_ontology_revision("msg_3") is None - assert message.content == "必须保留的原始答案" - - -def test_accept_ontology_revision_removes_transport_wrapper(): - message = SimpleNamespace( - message_id="msg_4", - role="assistant", - content="原始答案", - extra_data={ - "ontology_governance": { - "review": { - "candidate_answer": ( - "\n基于补充证据的完整优化答案\n" - "" - ) - } - } - }, - ) - service = object.__new__(ChatService) - service.message_repo = _MessageRepository(message) - - updated = service.accept_ontology_revision("msg_4") - - assert updated.content == "基于补充证据的完整优化答案" - assert ( - updated.extra_data["ontology_governance"]["review"]["candidate_answer"] - == "基于补充证据的完整优化答案" - ) diff --git a/src/backend/tests/ontology/test_ontology_workflow_evidence.py b/src/backend/tests/ontology/test_ontology_workflow_evidence.py deleted file mode 100644 index 13bffb61..00000000 --- a/src/backend/tests/ontology/test_ontology_workflow_evidence.py +++ /dev/null @@ -1,302 +0,0 @@ -from __future__ import annotations - -import json -from types import SimpleNamespace - -import pytest -from core.agent_runtime.direct import _astream_subagent_direct -from core.agent_runtime.integrations.ontology_evidence import ( - _capture_nested_ontology_evidence, - _ontology_review_failure_result, -) -from core.agent_runtime.integrations.ontology_repair import ( - _ontology_repair_prompt, - _run_ontology_repair_round, -) -from core.agent_runtime.runner import astream_chat_workflow -from core.llm.citations.allocator import AnchorAllocator -from core.llm.citations.anchor import annotate_tool_result - - -def test_repair_prompt_requires_polished_delivery_instead_of_validation_report(): - prompt = _ontology_repair_prompt( - { - "original_task": "只输出一句润色后的风险说明,并保存到 Word。", - "answer": "企业风险极高。", - "violations": [{"rule_id": "risk_report_requires_evidence"}], - } - ) - - assert "只输出一句润色后的风险说明,并保存到 Word。" in prompt - assert "待修订原始输出" in prompt - assert "企业风险极高。" in prompt - assert "最终交付必须是利用核验结论直接修正后的完整润色稿" in prompt - assert "证据核对、逐项判定、原文对比和修改说明都属于内部校验过程" in prompt - assert "以待修订原始输出为底稿做最小必要修改" in prompt - assert "不要因为查到更多资料就加入与原主张无关" in prompt - assert "用户只要求一句话时仍输出一句话" in prompt - assert "绝不能反向断言为‘不存在’" in prompt - assert "必须在内部逐条检查‘确定性违规’中的每个条件" in prompt - assert "把完全相同的正文写入用户要求的文件" in prompt - assert "已经激活的 enforce 级领域本体约束及本修复指令" in prompt - assert "若‘确定性违规’包含‘未调用’某些必需工具" in prompt - assert "再次原样输出待修订内容,均视为修复失败" in prompt - - -def test_streaming_workflows_do_not_shadow_global_asyncio_import(): - assert "asyncio" not in astream_chat_workflow.__code__.co_varnames - assert "asyncio" not in _astream_subagent_direct.__code__.co_varnames - - -def test_review_failure_preserves_original_and_escalates_instead_of_aborting(): - result = _ontology_review_failure_result("已经完成的原文", RuntimeError("review failed")) - - assert result["answer"] == "已经完成的原文" - assert result["verdict"] == "escalate" - assert result["revised"] is False - assert result["manual_review"]["required"] is True - - -def test_nested_subagent_tool_results_join_outer_trace_and_citations(): - trace = [] - citations = [] - allocator = AnchorAllocator() - tools = [ - ( - "search_company", - { - "items": [ - { - "企业名称": "杭州量知数据科技有限公司", - "企业状态": "存续", - } - ] - }, - ), - ("get_company_base_info", {"企业名称": "杭州量知数据科技有限公司"}), - ("get_company_risk_warning", {"经营司法风险": {"风险总数": 0}}), - ] - - for index, (tool_name, output) in enumerate(tools, 1): - _, items = annotate_tool_result( - tool_name, f"child-tool-{index}", json.dumps(output), allocator - ) - allocator.register(f"child-tool-{index}", items) - extracted = _capture_nested_ontology_evidence( - { - "sub_type": "tool_result", - "tool_name": tool_name, - "tool_id": f"child-tool-{index}", - "output": output, - "status": "success", - "agent_id": "risk-agent", - "sub_run_id": "sub-run-1", - "parent_tool_id": "call-subagent-1", - }, - trace, - citations, - allocator, - ) - assert extracted - - assert {item["tool_name"] for item in trace} == { - "search_company", - "get_company_base_info", - "get_company_risk_warning", - } - assert all(item["source"] == "subagent" for item in trace) - assert {item["tool_name"] for item in citations} == { - "search_company", - "get_company_base_info", - "get_company_risk_warning", - } - assert {item["tool_id"] for item in citations} == { - "child-tool-1", - "child-tool-2", - "child-tool-3", - } - - -@pytest.mark.asyncio -async def test_repair_round_continues_agent_and_streams_candidate_after_draft(): - runtime = {"enabled": True, "runtime_events": []} - allocator = AnchorAllocator() - - class FakeStreamingAgent: - def __init__(self): - self.agent = SimpleNamespace(state=SimpleNamespace(ontology_runtime=runtime)) - self.messages = None - - async def stream(self, messages, context): - self.messages = messages - yield "tool_pending", {"reason": "tool_args_streaming"} - yield "tool_call", { - "name": "get_company_risk_warning", - "id": "repair-tool-1", - "args": {"company_id": "company-1"}, - } - _, items = annotate_tool_result( - "get_company_risk_warning", - "repair-tool-1", - json.dumps({"经营司法风险": {"风险总数": 0}}), - allocator, - ) - allocator.register("repair-tool-1", items) - yield "tool_result", { - "name": "get_company_risk_warning", - "id": "repair-tool-1", - "content": '{"经营司法风险":{"风险总数":0}}', - } - yield "thinking_delta", "正在核对新增证据" - yield "text_delta", "修复后的" - yield "text_delta", "完整答案" - - streaming_agent = FakeStreamingAgent() - trace = [] - citations = [] - streamed_events = [] - - async def _capture_event(event): - streamed_events.append(event) - - answer, events, cursor, tool_count = await _run_ontology_repair_round( - streaming_agent=streaming_agent, - context={"ontology_runtime": runtime}, - payload={ - "attempt": 1, - "source": "deterministic", - "original_task": "请生成 Word 格式的企业风险分析报告", - "violations": [{"rule_id": "risk_report_requires_evidence"}], - "citations": [], - }, - runtime=runtime, - trace=trace, - citations=citations, - allocator=allocator, - event_cursor=0, - event_sink=_capture_event, - ) - - assert answer == "修复后的完整答案" - assert tool_count == 1 - assert cursor == 0 - assert streaming_agent.messages[0]["role"] == "user" - assert "同一任务唯一一次领域本体修复" in streaming_agent.messages[0]["content"] - assert "请生成 Word 格式的企业风险分析报告" in streaming_agent.messages[0]["content"] - assert "不得降级为纯文字替代" in streaming_agent.messages[0]["content"] - assert any(item.get("tool_name") == "get_company_risk_warning" for item in trace) - assert citations[0]["id"] == "e1" - assert any(item.get("type") == "ontology_revision_thinking" for item in streamed_events) - pending_event = next(item for item in streamed_events if item.get("type") == "tool_pending") - assert pending_event["reason"] == "tool_args_streaming" - assert pending_event["scope"] == "ontology_revision" - repair_tool_events = [ - item for item in streamed_events if item.get("type") in {"tool_call", "tool_result"} - ] - assert repair_tool_events - assert all(item.get("scope") == "ontology_revision" for item in repair_tool_events) - assert streamed_events[0] == { - "type": "ontology_repair", - "status": "started", - "attempt": 1, - "source": "deterministic", - } - assert streamed_events[-1]["type"] == "ontology_repair" - assert streamed_events[-1]["status"] == "completed" - content_events = [item for item in streamed_events if item.get("type") == "ontology_revision"] - assert "".join(item["delta"] for item in content_events) == "修复后的完整答案" - assert not any(item.get("type") == "content_replace" for item in events) - - -@pytest.mark.asyncio -async def test_repair_round_ignores_inline_wrapper_example_before_real_revision(): - runtime = {"enabled": True, "runtime_events": []} - allocator = AnchorAllocator() - - class FakeStreamingAgent: - def __init__(self): - self.agent = SimpleNamespace(state=SimpleNamespace(ontology_runtime=runtime)) - - async def stream(self, messages, context): - yield "text_delta", "正文应放在 `...` 中。" - yield "text_delta", "\n\n\n" - yield "text_delta", "这是包含真实证据与完整结论的修订正文。" - yield "text_delta", "\n" - - streamed_events = [] - - async def _capture_event(event): - streamed_events.append(event) - - answer, _, _, _ = await _run_ontology_repair_round( - streaming_agent=FakeStreamingAgent(), - context={"ontology_runtime": runtime}, - payload={"attempt": 1, "source": "deterministic", "citations": []}, - runtime=runtime, - trace=[], - citations=[], - allocator=allocator, - event_cursor=0, - event_sink=_capture_event, - ) - - assert answer == "这是包含真实证据与完整结论的修订正文。" - content = "".join( - item.get("delta", "") for item in streamed_events if item.get("type") == "ontology_revision" - ) - thinking = "".join( - item.get("delta", "") - for item in streamed_events - if item.get("type") == "ontology_revision_thinking" - ) - assert content.strip() == answer - assert "..." in thinking - - -@pytest.mark.asyncio -async def test_repair_round_streams_revision_adjacent_to_reasoning_close_tag(): - runtime = {"enabled": True, "runtime_events": []} - allocator = AnchorAllocator() - - class FakeStreamingAgent: - def __init__(self): - self.agent = SimpleNamespace(state=SimpleNamespace(ontology_runtime=runtime)) - - async def stream(self, messages, context): - yield "text_delta", "已完成证据核验,开始输出修订稿。第一段修订正文包含充分事实依据和明确结论," - yield "text_delta", "第二段继续说明证据边界并给出审慎判断。" - yield "text_delta", "" - - streamed_events = [] - - async def _capture_event(event): - streamed_events.append(event) - - answer, _, _, _ = await _run_ontology_repair_round( - streaming_agent=FakeStreamingAgent(), - context={"ontology_runtime": runtime}, - payload={"attempt": 1, "source": "review", "citations": []}, - runtime=runtime, - trace=[], - citations=[], - allocator=allocator, - event_cursor=0, - event_sink=_capture_event, - ) - - content_events = [item for item in streamed_events if item.get("type") == "ontology_revision"] - thinking = "".join( - item.get("delta", "") - for item in streamed_events - if item.get("type") == "ontology_revision_thinking" - ) - - assert answer == ( - "第一段修订正文包含充分事实依据和明确结论," "第二段继续说明证据边界并给出审慎判断。" - ) - assert len(content_events) >= 2 - assert "".join(item["delta"] for item in content_events) == answer - assert "第一段修订正文" not in thinking - assert "" not in answer - assert "" not in answer diff --git a/src/backend/tests/orchestration/test_autonomous_loop_driver.py b/src/backend/tests/orchestration/test_autonomous_loop_driver.py index 51bdfea0..db4100a1 100644 --- a/src/backend/tests/orchestration/test_autonomous_loop_driver.py +++ b/src/backend/tests/orchestration/test_autonomous_loop_driver.py @@ -107,8 +107,6 @@ async def _criteria(**kw): monkeypatch.setattr(al, "asyncio", SimpleNamespace( sleep=self._fast_sleep, gather=asyncio.gather, CancelledError=asyncio.CancelledError)) - # ontology 解析不碰 DB - import core.ontology.service as osvc monkeypatch.setattr(runtime_policy, 'build_user_ontology_runtime', lambda **kw: (False, {"enabled": False, "packs": [], diff --git a/src/backend/tests/orchestration/test_chat_run_restart_recovery.py b/src/backend/tests/orchestration/test_chat_run_restart_recovery.py index 064ef537..ffc68d20 100644 --- a/src/backend/tests/orchestration/test_chat_run_restart_recovery.py +++ b/src/backend/tests/orchestration/test_chat_run_restart_recovery.py @@ -29,7 +29,6 @@ async def test_automation_prompt_injects_durable_binding_and_recovery_surface( ): sessions = recovery_env from core.db import engine as db_engine - from core.ontology import service as ontology_service from core.models import selection as user_model_selection pass from orchestration.automations.automation_conversation import execute_prompt diff --git a/src/backend/tests/orchestration/test_command_recovery.py b/src/backend/tests/orchestration/test_command_recovery.py index 6537e3a5..3e116445 100644 --- a/src/backend/tests/orchestration/test_command_recovery.py +++ b/src/backend/tests/orchestration/test_command_recovery.py @@ -141,6 +141,8 @@ def test_list_never_hides_live_tasks_behind_display_limit(command_db): rows = store.list_owned("u", "c") assert len(rows) == 51 assert all(row["status"] == "starting" for row in rows) + # The run monitor lists only commands that were handed to the background. + assert all(row["background"] is False for row in rows) async def test_slow_launch_keeps_lease_without_execution_deadline(command_db, monkeypatch): import core.sandbox.commands.runtime as module diff --git a/src/backend/tests/orchestration/test_runtime_phase_lifecycle.py b/src/backend/tests/orchestration/test_runtime_phase_lifecycle.py deleted file mode 100644 index 823e405c..00000000 --- a/src/backend/tests/orchestration/test_runtime_phase_lifecycle.py +++ /dev/null @@ -1,203 +0,0 @@ -"""Exercise real preparation and stream adapters at consumer exit boundaries.""" -from core.agent_runtime.integrations import memory_identity as memory_identity_module -from core.agent_runtime.integrations import memory_session as memory_session_module -from core.ontology import output_review_state as output_review_state_module -from core.ontology import reviewer as reviewer_module -from core.llm.runtime import streaming as streaming_module -from core.llm.citations import allocator as allocator_module -from core.harness import log_writer - -import asyncio -from types import SimpleNamespace - -import pytest -from core.agent_runtime import budget, runner, targets -from core.agent_runtime.integrations import evolution, memory -from core.capabilities.catalog import resolver as catalog_resolver -from core.llm.citations import anchor as citation_anchor -from core.llm import factory -from core.llm.runtime import streaming -from core.ontology import reviewer, validator -from core.harness import log_writer as log_service - - -@pytest.fixture -def runtime(monkeypatch): - observed = SimpleNamespace( - messages=None, - created=0, - closed=0, - stream_closed=0, - claimed=False, - logs=[], - review_started=0, - ) - - async def empty(*args, **kwargs): - return "" - - async def session(**kwargs): - return memory_session_module.SessionMemory( - chat_id="", scope_user_id="", workspace_id="default", memory_enabled=False - ) - - async def create(**kwargs): - observed.created += 1 - return ( - SimpleNamespace( - state=SimpleNamespace(ontology_runtime=kwargs["ontology_runtime"]), model=None - ), - [], - ) - - class Adapter: - def __init__(self, agent, clients): - self.agent = agent - - async def stream(self, messages, context, **kwargs): - observed.messages = messages - try: - yield "text_delta", "Draft" - yield "user_question", {"id": "question"} - finally: - observed.stream_closed += 1 - - async def shutdown(self): - observed.closed += 1 - - async def aget_usage(self): - return {} - - def get_context_usage(self, usage): - return {} - - def claim(runtime, **kwargs): - observed.claimed = True - return True - - def release(runtime, **kwargs): - observed.claimed = False - - async def review(**kwargs): - observed.review_started += 1 - return {"verdict": "pass", "answer": kwargs["answer"], "attempts": 1} - - async def finish(*args, **kwargs): - observed.logs.append(kwargs["status"]) - - monkeypatch.setattr(factory, "create_agent_executor", create) - monkeypatch.setattr(streaming_module, 'StreamingAgent', Adapter) - monkeypatch.setattr(memory_session_module, 'open_session_memory', session) - monkeypatch.setattr(memory_session_module, 'resolve_session_memory', empty) - monkeypatch.setattr(memory_identity_module, 'build_user_identity_block', empty) - monkeypatch.setattr(allocator_module, 'anchor_start_for_chat', lambda *_: 0) - monkeypatch.setattr(targets, "_resolve_mode_spec", lambda *_: None) - monkeypatch.setattr(targets, "_runtime_visible_subagents", lambda *_: []) - monkeypatch.setattr( - targets, - "_load_direct_user_agent", - lambda *_: SimpleNamespace(name="Direct", extra_config={}), - ) - for name in ( - "enabled_skill_ids_from_context", - "enabled_mcp_ids_from_context", - "enabled_kb_ids_from_context", - ): - monkeypatch.setattr(catalog_resolver, name, lambda *_: []) - monkeypatch.setattr(budget, "_resolve_agent_model_runtime", lambda *_: ("test", 32000)) - monkeypatch.setattr(budget, "_compaction_budget_inputs", lambda *_: {}) - monkeypatch.setattr(output_review_state_module, 'claim_output_review', claim) - monkeypatch.setattr(output_review_state_module, 'release_output_review', release) - monkeypatch.setattr(output_review_state_module, 'complete_output_review', release) - monkeypatch.setattr(reviewer_module, 'review_ontology_output', review) - monkeypatch.setattr(log_writer, 'start_subagent_log', empty) - monkeypatch.setattr(log_writer, 'finish_subagent_log', finish) - monkeypatch.setattr(evolution, "_assemble_episode_background", lambda **_: None) - return observed - - -@pytest.mark.asyncio -@pytest.mark.parametrize("direct", [False, True]) -@pytest.mark.parametrize("stop", ["thinking", "user_question", "ontology_review", "meta", None]) -async def test_real_phases_close_stream_and_review_claim(runtime, direct, stop): - context = {"user_id": "phase-user", "memory_enabled": False} - if direct: - context["agent_id"] = "phase-agent" - stream = runner.astream_chat_workflow( - session_messages=[{"role": "user", "content": "Question"}], - user_message="Question", - context=context, - ) - events = [] - try: - async for event in stream: - events.append(event) - if event["type"] == stop: - break - finally: - await stream.aclose() - assert not runtime.claimed - if stop == "thinking": - assert runtime.created == 0 - assert runtime.closed == 0 - else: - assert runtime.created == 1 - assert runtime.messages[-1]["content"] == "Question" - assert runtime.stream_closed == 1 - assert runtime.closed >= 1 - if stop == "ontology_review": - assert runtime.review_started == 0 - assert events[-1]["status"] == "started" - if direct: - assert len(runtime.logs) == 1 - assert runtime.logs[0] == ("success" if stop in ("meta", None) else "cancelled") - if stop is None: - assert any(event["type"] == "meta" for event in events) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("direct", [False, True]) -async def test_cancel_while_review_is_running_joins_task(runtime, monkeypatch, direct): - started = asyncio.Event() - stopped = asyncio.Event() - - async def blocking_review(**kwargs): - started.set() - try: - await asyncio.Event().wait() - finally: - stopped.set() - - monkeypatch.setattr(reviewer_module, 'review_ontology_output', blocking_review) - context = {"user_id": "phase-user", "memory_enabled": False} - if direct: - context["agent_id"] = "phase-agent" - - async def consume(): - stream = runner.astream_chat_workflow( - session_messages=[{"role": "user", "content": "Question"}], - user_message="Question", - context=context, - ) - try: - async for _ in stream: - pass - finally: - await stream.aclose() - - consumer = asyncio.create_task(consume()) - try: - await asyncio.wait_for(started.wait(), 2) - consumer.cancel() - with pytest.raises(asyncio.CancelledError): - await asyncio.wait_for(consumer, 2) - assert stopped.is_set() - assert not runtime.claimed - assert runtime.stream_closed == 1 - assert runtime.closed >= 1 - if direct: - assert runtime.logs == ["cancelled"] - finally: - if not consumer.done(): - consumer.cancel() - await asyncio.gather(consumer, return_exceptions=True) diff --git a/src/backend/tests/sandbox/test_execd_latency_guards.py b/src/backend/tests/sandbox/test_execd_latency_guards.py new file mode 100644 index 00000000..d5bcf354 --- /dev/null +++ b/src/backend/tests/sandbox/test_execd_latency_guards.py @@ -0,0 +1,72 @@ +"""execd v1.0.17 workarounds: no post-completion stream delay, and the launch race.""" + +import time +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from core.sandbox.processes.cloud import OpenSandboxHandle +from core.sandbox.providers.opensandbox.provider import OpenSandboxProvider + + +class _Commands: + def __init__(self, misses): + self.misses = misses + + async def get_command_status(self, _id): + if self.misses: + self.misses -= 1 + raise RuntimeError("Get command status failed: command not found: abc") + return SimpleNamespace(running=False, exit_code=0, error=None) + + async def get_background_command_logs(self, _id, cursor=0): + return SimpleNamespace(cursor=3, content="ok\n") + + +def _handle(commands, launched_at): + handle = OpenSandboxHandle( + SimpleNamespace(name="opensandbox"), SimpleNamespace(session_id="s"), + commands, "abc", launched_at=launched_at, + ) + handle.touch = AsyncMock() + return handle + + +async def test_status_miss_right_after_launch_reads_as_still_starting(): + handle = _handle(_Commands(misses=1), launched_at=time.monotonic()) + assert await handle.poll() == ("", "", None) + assert await handle.poll() == ("ok\n", "", 0) + + +async def test_status_miss_after_the_window_is_an_error(): + handle = _handle(_Commands(misses=1), launched_at=time.monotonic() - 10) + with pytest.raises(RuntimeError, match="command not found"): + await handle.poll() + + +async def test_recovered_handle_gets_no_window(): + handle = _handle(_Commands(misses=1), launched_at=None) + with pytest.raises(RuntimeError, match="command not found"): + await handle.poll() + + +@pytest.mark.parametrize("user_id", [None, "u1"]) +async def test_every_sandbox_starts_execd_without_the_stream_grace_period(user_id): + provider = OpenSandboxProvider.__new__(OpenSandboxProvider) + provider._make_config = lambda: None + provider._session_expiry_s = lambda: 60 + provider._image = "image" + provider._additional_workspace_mounts = AsyncMock(return_value=([], {})) + provider._create_owned_sandbox = AsyncMock(return_value=SimpleNamespace(id="sbx")) + import core.sandbox.providers.opensandbox.creation as creation + + original = creation._make_skills_volumes + creation._make_skills_volumes = lambda _uid: [] + try: + await provider._create_jupyter_sandbox(user_id=user_id) + finally: + creation._make_skills_volumes = original + env = provider._create_owned_sandbox.await_args.kwargs["env"] + assert env["EXECD_API_GRACE_SHUTDOWN"] == "0s" + assert ("JX_MYSPACE_UID" in env) is bool(user_id) diff --git a/src/backend/tests/sandbox/test_opensandbox_kubernetes_staging.py b/src/backend/tests/sandbox/test_opensandbox_kubernetes_staging.py new file mode 100644 index 00000000..b4b13909 --- /dev/null +++ b/src/backend/tests/sandbox/test_opensandbox_kubernetes_staging.py @@ -0,0 +1,312 @@ +"""Kubernetes skill delivery keeps account mounts and native process admission.""" +from contextlib import nullcontext +from types import SimpleNamespace +from unittest.mock import AsyncMock +import io +import tarfile + +import pytest + +from core.sandbox.protocol import ProcessRequest +from core.sandbox.providers.opensandbox import creation, skills, connection +from core.sandbox.errors import SandboxError, SandboxConnectError + + +@pytest.mark.asyncio +async def test_kubernetes_creation_preserves_space_mounts_env_and_resource(monkeypatch): + from dataclasses import replace + monkeypatch.setenv("DEPLOY_PROFILE", "kubernetes") + monkeypatch.setattr(creation, "settings", replace( + creation.settings, sandbox=replace(creation.settings.sandbox, cpu="750m", memory="1536Mi"))) + monkeypatch.setattr(creation, "_make_skills_volumes", lambda _: pytest.fail("host skill bind requested")) + monkeypatch.setattr(creation, "credential_mounts", lambda _: []) + host = creation._OpenSandboxCreationMixin() + host._make_config = lambda: "native-config" + host._session_expiry_s = lambda: 600 + mount = object() + host._additional_workspace_mounts = AsyncMock(return_value=([mount], {"mount-revision": "current"})) + host._create_owned_sandbox = AsyncMock(return_value=SimpleNamespace(id="native")) + host._image = "native-image" + await host._create_jupyter_sandbox(user_id="alice") + args, kwargs = host._create_owned_sandbox.call_args + assert args == ("native-image",) + assert kwargs["volumes"] == [mount] + assert kwargs["resource"] == {"cpu": "750m", "memory": "1536Mi"} + assert kwargs["env"]["EXECD_API_GRACE_SHUTDOWN"] == "0s" + assert kwargs["env"]["JX_MYSPACE_UID"] == "alice" + assert kwargs["metadata"]["mount-revision"] == "current" + # Snapshot creation receives the same resources and grants. + await host._create_jupyter_sandbox(user_id="alice", from_snapshot_id="snapshot") + assert host._create_owned_sandbox.call_args.kwargs["snapshot_id"] == "snapshot" + assert host._create_owned_sandbox.call_args.kwargs["resource"] == kwargs["resource"] + + +@pytest.mark.asyncio +async def test_kubernetes_creation_requires_configured_resources(monkeypatch): + from dataclasses import replace + monkeypatch.setenv("DEPLOY_PROFILE", "kubernetes") + monkeypatch.setattr(creation, "settings", replace( + creation.settings, sandbox=replace(creation.settings.sandbox, cpu="", memory="1Gi"))) + monkeypatch.setattr(creation, "credential_mounts", lambda _: []) + host = creation._OpenSandboxCreationMixin() + host._make_config = lambda: None + host._session_expiry_s = lambda: 600 + host._additional_workspace_mounts = AsyncMock(return_value=([], {})) + host._create_owned_sandbox = AsyncMock() + host._image = "native-image" + with pytest.raises(SandboxConnectError, match="SANDBOX_CPU and SANDBOX_MEMORY"): + await host._create_jupyter_sandbox(user_id="alice") + host._create_owned_sandbox.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_configured_host_credentials_are_rejected_not_discarded(monkeypatch): + monkeypatch.setenv("DEPLOY_PROFILE", "kubernetes") + monkeypatch.setattr(creation, "credential_mounts", lambda _: [SimpleNamespace(host=object())]) + host = creation._OpenSandboxCreationMixin() + host._make_config = lambda: None + host._additional_workspace_mounts = AsyncMock(return_value=([], {})) + with pytest.raises(SandboxConnectError, match="host credential"): + await host._create_jupyter_sandbox(user_id="alice") + + +def test_native_endpoint_resolution_keeps_authenticated_server_proxy(monkeypatch): + host = connection._OpenSandboxConnectionMixin() + host._domain, host._api_key, host._request_timeout_s = "http://sandbox:8080", "key", 30 + monkeypatch.setenv("DEPLOY_PROFILE", "kubernetes") + assert host._make_config().use_server_proxy is True + monkeypatch.delenv("DEPLOY_PROFILE") + assert host._make_config().use_server_proxy is True + + +@pytest.fixture +def shared_state(monkeypatch): + from core.infra import ephemeral + state = ephemeral.LocalEphemeralState() + monkeypatch.setattr(ephemeral, "get_ephemeral_state", lambda: state) + return state + + +def _sandbox(sandbox_id="sbx-1", write_file=None): + return SimpleNamespace(id=sandbox_id, files=SimpleNamespace(write_file=write_file or AsyncMock()), + commands=SimpleNamespace(run=AsyncMock(return_value=SimpleNamespace(exit_code=0)))) + + +@pytest.fixture +def publication(tmp_path, monkeypatch, shared_state): + from core.capabilities.skills import publication + from core.capabilities.catalog import api + view = tmp_path / "alice" + view.mkdir() + monkeypatch.setattr(skills, "_database_visible", lambda names, user: set(names)) + monkeypatch.setattr(publication, "publication_lock", nullcontext) + monkeypatch.setattr(publication, "prepare_skill_view", lambda user: view if user == "alice" else tmp_path / "other") + monkeypatch.setattr(api, "get_catalog", lambda **kw: {"skills": [{"id": "disabled", "enabled": False}]}) + package = view / "selected" + package.mkdir() + (package / "SKILL.md").write_text("selected instructions") + (package / "run.sh").write_text("#!/bin/sh\necho selected") + (package / "run.sh").chmod(0o755) + private = tmp_path / "other" / "secret" + private.mkdir(parents=True) + (private / "SKILL.md").write_text("private") + return view + + +def test_view_contains_only_current_authorized_enabled_skill_bytes(publication): + disabled = publication / "disabled" + disabled.mkdir() + (disabled / "SKILL.md").write_text("disabled") + result = skills._view_archive(skills._view_files("alice")) + with tarfile.open(fileobj=io.BytesIO(result)) as archive: + assert sorted(archive.getnames()) == ["selected/SKILL.md", "selected/run.sh"] + assert archive.getmember("selected/run.sh").mode == 0o755 + (publication / "selected" / "run.sh").write_text("new revision") + assert skills._view_archive(skills._view_files("alice")) != result + + +def test_skill_symlinks_are_rejected_before_upload(publication): + (publication / "selected" / "leak").symlink_to(publication.parent / "other/secret/SKILL.md") + with pytest.raises(ValueError, match="软链接"): + skills._view_archive(skills._view_files("alice")) + + +@pytest.mark.asyncio +async def test_staging_runs_as_user_and_propagates_failure(publication): + sandbox = _sandbox() + req = ProcessRequest(script_content="/workspace/skills/selected/run.sh", script_name="test", user_id="alice") + await skills.stage_skill_view(sandbox, req) + sandbox.files.write_file.assert_awaited_once() + call = sandbox.commands.run.call_args_list[0] + assert str(call.kwargs["opts"].uid) == "1000" + assert "python3 -c" in call.args[0] + sandbox.commands.run.side_effect = [SimpleNamespace(exit_code=1), SimpleNamespace(exit_code=0)] + (publication / "selected" / "SKILL.md").write_text("changed instructions") + with pytest.raises(SandboxError, match="synchronize authorized"): + await skills.stage_skill_view(sandbox, req) + # A failed transfer is never remembered as installed. + sandbox.commands.run.side_effect = None + await skills.stage_skill_view(sandbox, req) + assert sandbox.files.write_file.await_count == 3 + + +@pytest.mark.asyncio +async def test_unchanged_view_is_not_retransferred_but_changes_and_resets_are(publication): + sandbox = _sandbox() + req = ProcessRequest(script_content="echo test", script_name="test", user_id="alice") + await skills.stage_skill_view(sandbox, req) + await skills.stage_skill_view(sandbox, req) + assert sandbox.files.write_file.await_count == 1 + assert sandbox.commands.run.await_count == 2 # one sync plus its cleanup + # Revocation is still evaluated for every command. + (publication / "selected" / "run.sh").write_text("#!/bin/sh\necho revised") + await skills.stage_skill_view(sandbox, req) + assert sandbox.files.write_file.await_count == 2 + # A sandbox reset for another session receives the view again. + await skills.forget_skill_view(sandbox) + await skills.stage_skill_view(sandbox, req) + assert sandbox.files.write_file.await_count == 3 + + +@pytest.mark.asyncio +async def test_revocation_applies_when_another_worker_installed_the_newer_view(publication): + req = ProcessRequest(script_content="echo test", script_name="test", user_id="alice") + worker_a, worker_b = _sandbox(), _sandbox() # two processes attached to one sandbox + await skills.stage_skill_view(worker_a, req) + extra = publication / "granted" + extra.mkdir() + (extra / "SKILL.md").write_text("granted later") + await skills.stage_skill_view(worker_b, req) + # Revoke it again; worker A's last view equals the current one, but the + # sandbox holds worker B's newer view, so A must transfer. + import shutil + shutil.rmtree(extra) + await skills.stage_skill_view(worker_a, req) + assert worker_a.files.write_file.await_count == 2 + + +@pytest.mark.asyncio +async def test_failed_transfer_leaves_no_installed_record(publication, shared_state): + sandbox = _sandbox() + sandbox.commands.run.side_effect = [SimpleNamespace(exit_code=1), SimpleNamespace(exit_code=0)] + req = ProcessRequest(script_content="echo test", script_name="test", user_id="alice") + with pytest.raises(SandboxError): + await skills.stage_skill_view(sandbox, req) + assert await shared_state.get(skills._installed_key(sandbox)) is None + + +@pytest.mark.asyncio +async def test_concurrent_staging_is_serialized_per_sandbox(publication): + import asyncio + active, peak = 0, 0 + + async def write_file(path, content): + nonlocal active, peak + active += 1 + peak = max(peak, active) + await asyncio.sleep(0.01) + active -= 1 + + sandbox = _sandbox(write_file=write_file) + req = ProcessRequest(script_content="echo test", script_name="test", user_id="alice") + await asyncio.gather(*(skills.stage_skill_view(sandbox, req) for _ in range(3))) + assert peak == 1 + + +@pytest.mark.asyncio +async def test_command_staging_keeps_api_packages_and_docker_mounts(monkeypatch): + from core.sandbox.providers.opensandbox.execution import _OpenSandboxExecMixin + stage = AsyncMock() + monkeypatch.setattr(skills, "stage_skill_view", stage) + host = _OpenSandboxExecMixin() + host._persistent_setup = AsyncMock() + host._sync_inputs_to_sandbox = AsyncMock() + session = SimpleNamespace(sandbox=object()) + request = ProcessRequest(script_content="echo test", script_name="test", user_id="alice") + monkeypatch.setenv("DEPLOY_PROFILE", "kubernetes") + await host._sync_inputs(session, request) + stage.assert_awaited_once_with(session.sandbox, request) + request.user_id = "api_ak_123" + await host._sync_inputs(session, request) + assert stage.await_count == 1 + monkeypatch.delenv("DEPLOY_PROFILE") + request.user_id = "alice" + await host._sync_inputs(session, request) + assert stage.await_count == 1 + assert host._sync_inputs_to_sandbox.await_count == 3 + + +def test_relative_execution_and_revocation_preserve_unchanged_cwd(publication, tmp_path, monkeypatch): + import subprocess + import sys + from core.capabilities.catalog import api + root = tmp_path / "sandbox" / "skills" + root.parent.mkdir() + script = skills._SYNC_VIEW.replace("Path('/workspace/skills')", "Path(" + repr(str(root)) + ")") + archive = tmp_path / "view.tgz" + + def synchronize(): + archive.write_bytes(skills._view_archive(skills._view_files("alice"))) + subprocess.run([sys.executable, "-c", script, str(archive)], check=True) + + synchronize() + assert subprocess.check_output(["sh", "selected/run.sh"], cwd=root, text=True).strip() == "selected" + inode = (root / "selected").stat().st_ino + synchronize() + assert (root / "selected").stat().st_ino == inode + # Relative references see the current revision, not the first copy. + (publication / "selected" / "run.sh").write_text("#!/bin/sh\necho updated") + synchronize() + assert subprocess.check_output(["sh", "selected/run.sh"], cwd=root, text=True).strip() == "updated" + # Even a command with no skill path now synchronizes an empty/revoked view. + monkeypatch.setattr(api, "get_catalog", lambda **kw: {"skills": [{"id": "selected", "enabled": False}]}) + synchronize() + assert not (root / "selected").exists() + assert subprocess.run(["sh", "selected/run.sh"], cwd=root, capture_output=True).returncode != 0 + + +def test_deleted_package_removed_and_sandbox_symlink_not_followed(publication, tmp_path): + import subprocess + import sys + import shutil + root = tmp_path / "sandbox-skills" + root.mkdir() + outside = tmp_path / "unrelated" + outside.mkdir() + (outside / "keep").write_text("keep") + (root / "stale-link").symlink_to(outside) + (root / "deleted").mkdir() + (root / "deleted" / "private").write_text("removed") + shutil.rmtree(publication / "selected") + archive = tmp_path / "empty.tgz" + archive.write_bytes(skills._view_archive(skills._view_files("alice"))) + script = skills._SYNC_VIEW.replace("Path('/workspace/skills')", "Path(" + repr(str(root)) + ")") + subprocess.run([sys.executable, "-c", script, str(archive)], check=True) + assert list(root.iterdir()) == [] + assert (outside / "keep").read_text() == "keep" + + +def test_database_enablement_owner_and_dependency_are_checked_without_cache(db_session, monkeypatch): + from sqlalchemy.orm import sessionmaker + from core.db import engine + from core.db.models import AdminSkill + monkeypatch.setattr(engine, "SessionLocal", sessionmaker(bind=db_session.get_bind())) + row = AdminSkill(skill_id="view-native-test", display_name="view", description="test", skill_content="test", + is_enabled=True, owner_user_id="alice", dep_status="ready") + db_session.add(row) + db_session.commit() + assert skills._database_visible([row.skill_id], "alice") == {row.skill_id} + assert skills._database_visible([row.skill_id], "bob") == set() + row.is_enabled = False + db_session.commit() + assert skills._database_visible([row.skill_id], "alice") == set() + row.is_enabled, row.dep_status = True, "pending" + db_session.commit() + assert skills._database_visible([row.skill_id], "alice") == set() + + +def test_kubernetes_rejects_desktop_cached_publication_instead_of_reusing_it(monkeypatch): + from core.harness.registry import run_capabilities + monkeypatch.setattr(run_capabilities, "has_run_capabilities", lambda: True) + with pytest.raises(SandboxError, match="desktop capability publication"): + skills._view_files("alice") diff --git a/src/backend/tests/test_ce_runtime_contracts.py b/src/backend/tests/test_ce_runtime_contracts.py index 11bce425..22e46dec 100644 --- a/src/backend/tests/test_ce_runtime_contracts.py +++ b/src/backend/tests/test_ce_runtime_contracts.py @@ -308,19 +308,23 @@ def fake_create_provider(_db, **kwargs): assert db.closed is True -def test_ce_schema_keeps_ontology_control_plane_tables(): +def test_ce_schema_has_no_ontology_tables(): import core.db.models # noqa: F401 register all model metadata from core.db.engine import Base + from core.db.ontology_retirement import ONTOLOGY_TABLES - ontology_tables = { - "ontology_packs", - "ontology_pack_versions", - "ontology_enforcement_events", - "ontology_review_runs", - "ontology_drafts", - } + assert not set(ONTOLOGY_TABLES) & set(Base.metadata.tables) + + +def test_ce_ontology_runtime_is_always_off(): + from core.ontology import output_review_state, runtime_pack, runtime_policy - assert ontology_tables <= set(Base.metadata.tables) + enabled, runtime = runtime_policy.build_user_ontology_runtime(user_id="u", task="t") + assert enabled is False + assert runtime == runtime_policy.disabled_ontology_runtime() + assert runtime_pack.activate_runtime_for_asset(runtime, kind="skill", asset_id="s") == [] + assert runtime_pack.render_runtime_prompt(runtime) == "" + assert output_review_state.claim_output_review(runtime, owner="o") is False def test_ce_schema_can_create_without_enterprise_foreign_keys(db_session): diff --git a/src/backend/tests/test_explicit_runtime_environment.py b/src/backend/tests/test_explicit_runtime_environment.py new file mode 100644 index 00000000..7b7a572f --- /dev/null +++ b/src/backend/tests/test_explicit_runtime_environment.py @@ -0,0 +1,10 @@ +"""An injected container environment must not read repository dotenv files.""" +from core.config import environment + + +def test_disabled_dotenv_does_not_read_any_file(monkeypatch): + monkeypatch.setenv("PYTHON_DOTENV_DISABLED", "1") + def unexpected_read(*args, **kwargs): + raise AssertionError("dotenv must not be read") + monkeypatch.setattr(environment, "dotenv_values", unexpected_read) + environment._load_env_files() diff --git a/src/backend/tests/test_kubernetes_space_mounts.py b/src/backend/tests/test_kubernetes_space_mounts.py new file mode 100644 index 00000000..f65b2a98 --- /dev/null +++ b/src/backend/tests/test_kubernetes_space_mounts.py @@ -0,0 +1,97 @@ +"""Controlled mount workload boundaries and upstream patch checks.""" +import importlib.util +from pathlib import Path +from types import SimpleNamespace +import pytest + +ROOT = Path(__file__).resolve().parents[3] +spec = importlib.util.spec_from_file_location("space_mounts", ROOT / "deploy_kit/kubernetes/space_mounts.py") +mounts = importlib.util.module_from_spec(spec) +spec.loader.exec_module(mounts) + +@pytest.fixture +def config(monkeypatch): + values = {"OPENSANDBOX_SPACE_MOUNT_IMAGE": "test/helper:1", + "HUGAGENT_SPACE_SECRET_NAME": "space-storage", + "HUGAGENT_SPACE_ENDPOINT": "http://minio:9000", + "HUGAGENT_SPACE_BUCKET": "spaces", "HUGAGENT_SPACE_PREFIX": "prod"} + for key, value in values.items(): + monkeypatch.setenv(key, value) + return values + +def volume(path="/workspace/myspace/alice", prefix="prod/spaces/users/alice/files", readonly=False): + return SimpleNamespace(ossfs=SimpleNamespace(bucket="spaces", endpoint="http://minio:9000", + access_key_id="server-managed", access_key_secret="server-managed", options=None), + mount_path=path, sub_path=prefix, read_only=readonly) + +def test_mount_isolated_credentials_and_kernel_propagation(config): + pod = {"containers": [{"name": "main", "securityContext": {"privileged": False}}]} + assert mounts.apply_object_volumes(pod, [volume(readonly=True)]) == [] + main = pod["containers"][0] + helper = pod["initContainers"][0] + assert helper["restartPolicy"] == "Always" + assert main["securityContext"] == {"privileged": False} + assert main["volumeMounts"][0]["mountPropagation"] == "HostToContainer" + assert helper["volumeMounts"][0]["mountPropagation"] == "Bidirectional" + assert all(v["name"] != "hugagent-space-credentials" for v in main["volumeMounts"]) + assert helper["securityContext"]["privileged"] + assert not pod["shareProcessNamespace"] + assert not pod["automountServiceAccountToken"] + assert all("hostPath" not in v and "persistentVolumeClaim" not in v for v in pod["volumes"]) + assert '"read_only": true' in helper["env"][0]["value"] + +@pytest.mark.parametrize("prefix", ["prod/spaces/users/alice/../bob", "other/spaces/users/alice/files", + "prod/spaces//files", "prod/spaces/users/alice/files/"]) +def test_invalid_prefix_rejected(config, prefix): + with pytest.raises(ValueError): + mounts.apply_object_volumes({"containers": [{"name": "main"}]}, [volume(prefix=prefix)]) + +def test_shared_process_namespace_rejected(config): + with pytest.raises(ValueError, match="private process"): + mounts.apply_object_volumes({"shareProcessNamespace": True, "containers": [{}]}, [volume()]) + +def test_missing_config_fails_closed(config, monkeypatch): + monkeypatch.delenv("OPENSANDBOX_SPACE_MOUNT_IMAGE") + with pytest.raises(ValueError, match="incomplete"): + mounts.apply_object_volumes({"containers": [{}]}, [volume()]) + +def test_endpoint_cannot_be_overridden(config): + v = volume() + v.ossfs.endpoint = "http://attacker" + with pytest.raises(ValueError, match="fixed server"): + mounts.apply_object_volumes({"containers": [{}]}, [v]) + + +def test_disconnected_mount_is_detached_before_path_access(monkeypatch): + helper_spec = importlib.util.spec_from_file_location("space_helper", ROOT / "docker/space-mount.py") + helper = importlib.util.module_from_spec(helper_spec) + helper_spec.loader.exec_module(helper) + state = {"mounted": True} + class Parent: + def mkdir(self, **kwargs): pass + def chmod(self, mode): pass + class Mount: + parent = Parent() + def __str__(self): return "/workspace/myspace/alice" + def mkdir(self, **kwargs): + if state["mounted"]: + raise OSError(107, "Transport endpoint is not connected") + def chmod(self, mode): + assert mode == 0 + monkeypatch.setattr(helper, "mounted", lambda path: state["mounted"]) + def detach(command, check): + assert command == ["fusermount", "-u", "/workspace/myspace/alice"] and check + state["mounted"] = False + monkeypatch.setattr(helper.subprocess, "run", detach) + helper.prepare_mount(Mount()) + assert not state["mounted"] + + +def test_mount_revision_changes_when_kubernetes_mode_changes(monkeypatch): + from core.files.spaces.mounts import mount_revision + monkeypatch.setenv("STORAGE_TYPE", "s3") + monkeypatch.setenv("DEPLOY_PROFILE", "kubernetes") + monkeypatch.setenv("HUGAGENT_SPACE_MOUNT_MODE", "csi") + old = mount_revision([]) + monkeypatch.setenv("HUGAGENT_SPACE_MOUNT_MODE", "sidecar") + assert mount_revision([]) != old diff --git a/src/backend/tests/test_local_onboarding.py b/src/backend/tests/test_local_onboarding.py index 932b93a0..7606c921 100644 --- a/src/backend/tests/test_local_onboarding.py +++ b/src/backend/tests/test_local_onboarding.py @@ -32,7 +32,6 @@ def test_ce_default_admin_is_idempotent_and_requires_password_change(db_session, assert shadow.extra_data["can_create_public_kb"] is False assert shadow.extra_data["can_create_channel_bot"] is True assert shadow.extra_data["can_switch_model"] is True - assert shadow.extra_data["can_use_ontology_validation"] is True assert shadow.extra_data["must_change_password"] is True assert shadow.extra_data["onboarding_required"] is True diff --git a/src/backend/tests/test_local_profile.py b/src/backend/tests/test_local_profile.py index e180121d..be3be656 100644 --- a/src/backend/tests/test_local_profile.py +++ b/src/backend/tests/test_local_profile.py @@ -518,6 +518,8 @@ def test_apply_local_env_seeds_the_profile_before_settings_is_frozen(tmp_path): "cli.apply_local_env(port=18999)\n" "from core.config.settings import settings\n" "print(json.dumps({'is_local': settings.deploy.is_local,\n" + " 'session_persist': settings.session.persist_path,\n" + " 'session_ttl_hours': settings.session.ttl_hours,\n" " 'site_home': os.environ.get('SITE_TEMPLATE_HOME', '')}))\n" ) out = subprocess.run( @@ -531,3 +533,6 @@ def test_apply_local_env_seeds_the_profile_before_settings_is_frozen(tmp_path): payload = json.loads(out.stdout.strip().splitlines()[-1]) assert payload["is_local"] is True assert payload["site_home"] + # The desktop restarts this process on every launch; logins must survive it. + assert payload["session_persist"] == str(tmp_path / "home" / "sessions.json") + assert payload["session_ttl_hours"] == 720 diff --git a/src/backend/tests/test_personal_agent_computer_recovery.py b/src/backend/tests/test_personal_agent_computer_recovery.py new file mode 100644 index 00000000..47ad99a2 --- /dev/null +++ b/src/backend/tests/test_personal_agent_computer_recovery.py @@ -0,0 +1,152 @@ +"""A permanent computer whose instance is lost or retired must be replaced, never left degraded.""" +import asyncio +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock + +import opensandbox +import pytest + +import personal_agent.computer.sandbox as computer +from personal_agent.computer.sandbox import ComputerGone, PersonalSandboxProvider + + +class Missing(Exception): + status_code = 404 + + +class Unreachable(Exception): + status_code = 500 + + +def provider_with(binding): + provider = object.__new__(PersonalSandboxProvider) + provider._sessions = {} + provider._get_session_lock = AsyncMock(return_value=asyncio.Lock()) + provider._binding = Mock(side_effect=binding) + provider._computer_config = Mock(return_value=(SimpleNamespace(), "token")) + provider._check_mount_revision = AsyncMock() + provider._retire = AsyncMock() + provider._repoint_execd_direct = AsyncMock() + return provider + + +@pytest.mark.parametrize("failure", [Missing(), ComputerGone("mounts changed")]) +def test_lost_or_retired_instance_is_replaced(monkeypatch, failure): + old = SimpleNamespace(id="old", close=AsyncMock()) + fresh = SimpleNamespace(id="fresh") + provider = provider_with([("pa", "old", 1), ("pa", None, 2)]) + provider._provision = AsyncMock(return_value=fresh) + if isinstance(failure, ComputerGone): + monkeypatch.setattr(opensandbox.Sandbox, "connect", AsyncMock(return_value=old)) + provider._check_mount_revision.side_effect = failure + else: + monkeypatch.setattr(opensandbox.Sandbox, "connect", AsyncMock(side_effect=failure)) + + session = asyncio.run(provider._get_or_create_session("box", "owner")) + + assert session.sandbox is fresh + provider._retire.assert_awaited_once_with("pa", "old") + provider._provision.assert_awaited_once_with("pa", "box", "owner", 2) + assert provider._sessions["box"] is session + if isinstance(failure, ComputerGone): + old.close.assert_awaited_once() + + +def test_network_error_keeps_the_binding(monkeypatch): + provider = provider_with([("pa", "old", 1)]) + provider._provision = AsyncMock() + monkeypatch.setattr(opensandbox.Sandbox, "connect", AsyncMock(side_effect=Unreachable())) + + with pytest.raises(Unreachable): + asyncio.run(provider._get_or_create_session("box", "owner")) + + provider._retire.assert_not_awaited() + provider._provision.assert_not_awaited() + + +def test_cached_instance_with_changed_mounts_is_replaced(monkeypatch): + cached = SimpleNamespace(sandbox=SimpleNamespace(id="old", close=AsyncMock())) + fresh = SimpleNamespace(id="fresh") + provider = provider_with([("pa", "old", 1), ("pa", None, 2)]) + provider._sessions["box"] = cached + provider._check_mount_revision.side_effect = ComputerGone("mounts changed") + provider._provision = AsyncMock(return_value=fresh) + monkeypatch.setattr(opensandbox.Sandbox, "connect", AsyncMock(side_effect=Missing())) + + session = asyncio.run(provider._get_or_create_session("box", "owner")) + + assert session.sandbox is fresh + cached.sandbox.close.assert_awaited_once() + provider._retire.assert_awaited_once_with("pa", "old") + + +class State: + async def hold(self, *args, **kwargs): return True + async def compare_exchange(self, *args, **kwargs): return True + + +def provisioning(monkeypatch, tmp_path, found_state, generations): + """A provider whose controller already lists one instance of this generation.""" + found = SimpleNamespace(id="orphan", status=SimpleNamespace(state=found_state)) + manager = SimpleNamespace( + list_sandbox_infos=AsyncMock(return_value=SimpleNamespace(sandbox_infos=[found])), + kill_sandbox=AsyncMock(), close=AsyncMock()) + monkeypatch.setattr(opensandbox.SandboxManager, "create", AsyncMock(return_value=manager)) + adopted = SimpleNamespace(id="orphan", close=AsyncMock()) + monkeypatch.setattr(opensandbox.Sandbox, "connect", AsyncMock(return_value=adopted)) + monkeypatch.setattr("core.infra.ephemeral.get_ephemeral_state", lambda: State()) + monkeypatch.setattr("personal_agent.workspace.persona.ensure_seeded", lambda ident: None) + monkeypatch.setattr("core.files.spaces.mounts.provision_volumes", lambda user: ([], {})) + monkeypatch.setenv("PERSONAL_AGENT_VOLUME_ROOT", str(tmp_path)) + monkeypatch.setenv("PERSONAL_AGENT_LOCAL_VOLUME_ROOT", str(tmp_path)) + row = SimpleNamespace(sandbox_id=None, computer_status="not_provisioned") + monkeypatch.setattr(computer, "run_in_session", AsyncMock(side_effect=lambda fn: fn(SimpleNamespace(get=lambda *a: row, commit=lambda: None)))) + + provider = provider_with([("pa", None, g) for g in generations]) + created = SimpleNamespace(id="fresh") + admission = SimpleNamespace(create=AsyncMock(return_value=created), release=AsyncMock()) + provider._sandbox_admission = Mock(return_value=admission) + provider._make_config = Mock(return_value=SimpleNamespace()) + return provider, manager, admission, adopted, row + + +def test_running_orphan_of_this_generation_is_adopted(monkeypatch, tmp_path): + provider, manager, admission, adopted, row = provisioning(monkeypatch, tmp_path, "Running", [1]) + + sandbox = asyncio.run(provider._provision_locked("pa", "box", "owner", 1)) + + assert sandbox is adopted + assert (row.sandbox_id, row.computer_status) == ("orphan", "ready") + admission.create.assert_not_awaited() + provider._retire.assert_not_awaited() + + +@pytest.mark.parametrize("state", ["Terminated", "Failed"]) +def test_stopped_orphan_starts_the_next_generation(monkeypatch, tmp_path, state): + provider, manager, admission, _, row = provisioning(monkeypatch, tmp_path, state, [1, 2]) + + sandbox = asyncio.run(provider._provision_locked("pa", "box", "owner", 1)) + + assert sandbox.id == "fresh" + manager.kill_sandbox.assert_awaited_once_with("orphan") + provider._retire.assert_awaited_once_with("pa", None) + assert admission.create.await_args.kwargs["metadata"]["generation"] == "2" + assert (row.sandbox_id, row.computer_status) == ("fresh", "ready") + + +@pytest.mark.parametrize("failure, status", [ + (computer.ProvisionInProgress("busy"), "not_provisioned"), + (Unreachable(), "degraded"), +]) +def test_maintenance_marks_degraded_only_on_real_failures(monkeypatch, failure, status): + import personal_agent.background as background + row = SimpleNamespace(computer_status="not_provisioned") + session = SimpleNamespace(execute=lambda *a: SimpleNamespace(all=lambda: [("pa", "box", "owner")]), + get=lambda *a: row, commit=lambda: None) + monkeypatch.setattr(background, "run_in_session", AsyncMock(side_effect=lambda fn: fn(session))) + provider = SimpleNamespace(ensure_user_workspace=AsyncMock(side_effect=failure), touch_session=AsyncMock()) + monkeypatch.setattr(computer, "get_personal_sandbox_provider", lambda: provider) + + asyncio.run(background.BackgroundServices(None).maintain_computers()) + + assert row.computer_status == status diff --git a/src/backend/tests/test_plugin_system.py b/src/backend/tests/test_plugin_system.py index ccc384ef..65de3c10 100644 --- a/src/backend/tests/test_plugin_system.py +++ b/src/backend/tests/test_plugin_system.py @@ -21,7 +21,6 @@ from core.capabilities.plugins.management import components as plugin_components -from core.ontology.policy import set_plugin_import_build_validation_forced OWNER = "test_user_123" @@ -131,58 +130,6 @@ def test_import_cc_plugin_into_db(tmp_path, db_session): assert len(row.component_ids["skills"]) == 2 -@pytest.mark.parametrize( - ("user_enabled", "forced", "expected_calls"), - [ - (False, False, 0), - (True, False, 4), - (False, True, 4), - ], -) -def test_plugin_import_ontology_validation_follows_user_and_force_policy( - tmp_path, - db_session, - monkeypatch, - user_enabled, - forced, - expected_calls, -): - """Personal opt-out skips checks unless the independent admin gate is on.""" - owner = f"ontology_import_{int(user_enabled)}_{int(forced)}" - db_session.add( - UserShadow( - user_id=owner, - username=owner, - extra_data={ - "ontology_enabled": user_enabled, - "can_use_ontology_validation": user_enabled, - }, - ) - ) - db_session.commit() - if forced: - set_plugin_import_build_validation_forced(db_session, True, updated_by="test") - - calls = [] - monkeypatch.setattr( - plugin_components, - "ensure_ontology_build_valid", - lambda *_args, **kwargs: calls.append(kwargs["asset_type"]), - ) - - ps.import_plugin( - db_session, - _make_cc_plugin(tmp_path), - owner_user_id=owner, - secrets={"api_token": "test"}, - ) - - assert len(calls) == expected_calls - if expected_calls: - assert calls.count("skill") == 2 - assert calls.count("tool") == 2 - - def test_owned_skill_enters_runtime_set(tmp_path, db_session): """Imported private skill with is_enabled=True → selected by the owned merge of resolve_all_runtime_enabled.""" from core.capabilities.catalog.resolver import _owned_enabled_ids diff --git a/src/backend/tests/test_session_persistence.py b/src/backend/tests/test_session_persistence.py new file mode 100644 index 00000000..88f818bc --- /dev/null +++ b/src/backend/tests/test_session_persistence.py @@ -0,0 +1,86 @@ +"""The in-process session table survives a restart when a persist file is set.""" + +import asyncio +import json +import os +import stat +from datetime import datetime, timedelta, timezone + +import pytest +from core.auth import session + + +@pytest.fixture() +def persisted(tmp_path, monkeypatch): + path = tmp_path / "sessions.json" + monkeypatch.setattr(session, "_use_memory_store", lambda: True) + monkeypatch.setattr(session, "_persist_path", lambda: path) + monkeypatch.setattr(session, "_STATE", session.SessionState()) + return path + + +def _restart(monkeypatch): + monkeypatch.setattr(session, "_STATE", session.SessionState()) + + +def test_session_survives_restart(persisted, monkeypatch): + token = asyncio.run(session.create_session({"user_id": "u1", "username": "admin"})) + + _restart(monkeypatch) + user = asyncio.run(session.validate_session(token)) + + assert user is not None and user["user_id"] == "u1" + + +def test_persist_file_holds_digests_only_and_is_private(persisted): + token = asyncio.run(session.create_session({"user_id": "u1"})) + + text = persisted.read_text(encoding="utf-8") + assert token not in text + assert session._hash_token(token) in json.loads(text) + if os.name == "posix": + assert stat.S_IMODE(persisted.stat().st_mode) == 0o600 + + +def test_revoked_session_stays_revoked_after_restart(persisted, monkeypatch): + token = asyncio.run(session.create_session({"user_id": "u1"})) + assert asyncio.run(session.revoke_session(token)) is True + + _restart(monkeypatch) + + assert asyncio.run(session.validate_session(token)) is None + + +def test_expired_and_malformed_entries_are_dropped_on_load(persisted, monkeypatch): + now = datetime.now(timezone.utc) + persisted.write_text( + json.dumps( + { + "expired": {"payload": {"user_id": "old"}, "expires_at": (now - timedelta(minutes=1)).isoformat()}, + "broken": {"payload": {"user_id": "x"}}, + "live": {"payload": {"user_id": "live"}, "expires_at": (now + timedelta(hours=1)).isoformat()}, + } + ), + encoding="utf-8", + ) + + assert set(session._memory_sessions()) == {"live"} + + +def test_unreadable_file_starts_empty(persisted): + persisted.write_text("{not json", encoding="utf-8") + + assert session._memory_sessions() == {} + token = asyncio.run(session.create_session({"user_id": "u1"})) + assert session._hash_token(token) in json.loads(persisted.read_text(encoding="utf-8")) + + +def test_without_persist_path_nothing_is_written(tmp_path, monkeypatch): + monkeypatch.setattr(session, "_use_memory_store", lambda: True) + monkeypatch.setattr(session, "_persist_path", lambda: None) + monkeypatch.setattr(session, "_STATE", session.SessionState()) + monkeypatch.chdir(tmp_path) + + asyncio.run(session.create_session({"user_id": "u1"})) + + assert list(tmp_path.iterdir()) == [] diff --git a/src/frontend/package.json b/src/frontend/package.json index ae2d4d8e..33325dd0 100755 --- a/src/frontend/package.json +++ b/src/frontend/package.json @@ -5,10 +5,11 @@ "type": "module", "scripts": { "dev": "vite", - "build": "node scripts/check-i18n.mjs && node scripts/check-dark-mode.mjs && node scripts/check-loading-motion.mjs && tsc -b && vite build && node scripts/precompress-dist.mjs", + "build": "node scripts/check-i18n.mjs && node scripts/check-dark-mode.mjs && node scripts/check-loading-motion.mjs && node scripts/check-native-controls.mjs && tsc -b && vite build && node scripts/precompress-dist.mjs", "check:i18n": "node scripts/check-i18n.mjs", "check:dark": "node scripts/check-dark-mode.mjs", "check:motion": "node scripts/check-loading-motion.mjs", + "check:controls": "node scripts/check-native-controls.mjs", "test:file-open-browser": "node scripts/test-file-open-browser.mjs", "test:canvas-quote-browser": "node scripts/test-canvas-quote-browser.mjs", "test:tool-capability-icons": "esbuild scripts/test-tool-capability-icons.ts --bundle --platform=node --format=esm --outfile=node_modules/.tmp/test-tool-capability-icons.mjs && node node_modules/.tmp/test-tool-capability-icons.mjs", diff --git a/src/frontend/public/loader-done.png b/src/frontend/public/loader-done.png deleted file mode 100644 index 433caf5a..00000000 Binary files a/src/frontend/public/loader-done.png and /dev/null differ diff --git a/src/frontend/public/loader.gif b/src/frontend/public/loader.gif deleted file mode 100644 index 73ffe489..00000000 Binary files a/src/frontend/public/loader.gif and /dev/null differ diff --git a/src/frontend/scripts/check-native-controls.mjs b/src/frontend/scripts/check-native-controls.mjs new file mode 100644 index 00000000..7d9dde81 --- /dev/null +++ b/src/frontend/scripts/check-native-controls.mjs @@ -0,0 +1,139 @@ +#!/usr/bin/env node +/** + * 原生控件门禁(构建前执行,与 check-dark-mode 同级)。 + * + * 背景:工作台(src/workbench)整块是「先接逻辑、UI 以后再说」写出来的——裸 + {open &&
{children}
} + + ); +} + +function SubagentRow({ item, chatId, active }: { item: MonitorSubagent; chatId: string; active: boolean }) { + const { tool, messageUid, isStreaming } = item; + const agent = useSubagentIdentity(tool); + const status = subagentStatus(tool, isStreaming); + return ( + + ); +} + +function FileRow({ file, active }: { file: CanvasArtifact; active: boolean }) { + const openCanvas = useCanvasStore((s) => s.openCanvas); + const extension = file.name.includes('.') ? file.name.split('.').pop()?.toUpperCase() : ''; + return ( + + ); +} + +function StopButton({ label, onStop }: { label: string; onStop: () => Promise }) { + const [stopping, setStopping] = useState(false); + const stop = async () => { + setStopping(true); + try { + await onStop(); + } catch (error) { + message.error(t('停止失败:{reason}', { reason: error instanceof Error ? error.message : String(error) })); + } finally { + setStopping(false); + } + }; + return ( + + + + ); +} + +function CommandRow({ task, onStop }: { task: CommandTaskBrief; onStop: () => Promise }) { + const started = parseServerTime(task.created_at); + return ( +
+ + {task.command} + {Number.isFinite(started) && } + +
+ ); +} + +function JobRow({ job, onStop }: { job: JobBrief; onStop: () => Promise }) { + const { settled, total } = job.stats; + return ( +
+ + {job.name} + {total > 0 && {settled}/{total}} + +
+ ); +} + +function BackgroundTasksCard({ chatId }: { chatId: string }) { + const { commands, jobs, cancelCommand, cancelJob } = useBackgroundTasks(chatId); + // Only what is still running: the card leaves once every task has ended. + const liveCommands = commands.filter(isLiveCommand); + const liveJobs = jobs.filter(isLiveJob); + const running = liveCommands.length + liveJobs.length; + if (!running) return null; + return ( + + {liveJobs.map((job) => cancelJob(job.job_id)} />)} + {liveCommands.map((task) => cancelCommand(task.task_id)} />)} + + ); +} + +function ContextCard({ toolCalls, toolFailed }: { toolCalls: number; toolFailed: number }) { + const view = useContextUsageView(); + if (!view) return null; + const { breakdown, window, pct, level } = view; + return ( + + {pct}% + + + + } + > +
+ {CONTEXT_CATEGORIES.map((c) => { + const value = breakdown[c.key] as number; + if (!(value > 0)) return null; + return ( +
+ + {t(c.label)} + {formatTokens(value)} +
+ ); + })} +
+ {t('合计')} + {formatTokens(breakdown.total)} / {formatTokens(window)} +
+
+ {t('工具调用')} + + {toolCalls} + {toolFailed > 0 && · {t('失败 {n}', { n: toolFailed })}} + +
+
+
+ ); +} + +const pathParts = (path: string) => path.split(/[/\\]/).filter(Boolean); + +/** + * The chat's project: its environment binding and its folder. A local project's + * folder opens in Finder / Explorer; a cloud project's folder opens in My Space. + */ +function EnvironmentCard({ projectId }: { projectId: string }) { + const info = useChatProject(projectId); + const [editing, setEditing] = useState(false); + const [opening, setOpening] = useState(false); + if (!info) return null; + const { project, environment, reloadEnvironment } = info; + const local = project.kind === 'local'; + const canConfigure = project.permission === 'admin'; + const localPath = local ? project.local_path || '' : ''; + const localParts = pathParts(localPath); + const folderName = local ? localParts[localParts.length - 1] || project.name : project.folder_name || project.name; + const teamName = 'team_name' in project && typeof project.team_name === 'string' ? project.team_name : ''; + const folderPlace = local ? localParts[localParts.length - 2] || '' : teamName || t('我的空间'); + const environmentLabel = environment === 'custom' ? t('自定义项目环境') : environment === 'all' ? t('全部可用环境') : t('项目环境'); + const EnvironmentIcon = local ? LaptopOutlined : CloudOutlined; + + const openFolder = async () => { + if (local) { + if (localPath) window.location.href = '/__desktop/open-path?path=' + encodeURIComponent(localPath); + return; + } + setOpening(true); + try { + const { space, path } = await projectSpaceLocation(project); + useMySpaceStore.getState().openSpaceFolder(space, path); + } catch (error) { + message.error(t('无法打开项目文件夹:{reason}', { reason: error instanceof Error ? error.message : String(error) })); + } finally { + setOpening(false); + } + }; + + return ( + + {/* Only the gear opens the settings; the row itself just reports the binding. */} +
+ + {environmentLabel} + {canConfigure && ( + + + + )} +
+ {(!local || localPath) && ( + + )} + {editing && ( + { setEditing(false); reloadEnvironment(); }} /> + )} +
+ ); +} + +/** + * Run monitor inside the conversation pane, right of the messages: one card per + * concern (project environment, context, sub-agents, output and input files, + * background tasks). + * Cards with nothing to show stay hidden. Rows open their sub-agent session or + * file in the canvas; the rail hides when the pane is too narrow for both. + */ +export function RunMonitorRail() { + const chatId = useChatStore((s) => s.currentChatId); + const projectId = useChatStore((s) => (s.currentChatId ? s.store.chats[s.currentChatId]?.projectId : undefined)); + const messages = useChatStore((s) => (s.currentChatId ? s.store.chats[s.currentChatId]?.messages : undefined)); + const activeTabId = useCanvasStore((s) => (s.isOpen ? s.activeTabId : null)); + + const subagents = useMemo(() => collectSubagents(messages), [messages]); + const files = useMemo(() => collectFiles(messages, chatId || undefined), [messages, chatId]); + const toolStats = useMemo(() => countToolCalls(messages), [messages]); + + if (!chatId || !messages?.length) return null; + + return ( + + ); +} diff --git a/src/frontend/src/components/monitor/index.ts b/src/frontend/src/components/monitor/index.ts new file mode 100644 index 00000000..bce7bcfa --- /dev/null +++ b/src/frontend/src/components/monitor/index.ts @@ -0,0 +1 @@ +export { RunMonitorRail } from './RunMonitorRail'; diff --git a/src/frontend/src/components/monitor/runMonitorData.ts b/src/frontend/src/components/monitor/runMonitorData.ts new file mode 100644 index 00000000..8aebe8bf --- /dev/null +++ b/src/frontend/src/components/monitor/runMonitorData.ts @@ -0,0 +1,104 @@ +import type { ChatMessage, ToolCall } from '../../types'; +import type { CanvasArtifact } from '../../stores/canvasStore'; +import { artifactOrigin } from '../../utils/artifactAccess'; +import { extractArtifactOutputs } from '../../utils/fileParser'; + +export interface MonitorSubagent { + tool: ToolCall; + messageUid: string; + isStreaming: boolean; +} + +export interface MonitorFiles { + input: CanvasArtifact[]; + output: CanvasArtifact[]; +} + +export interface MonitorToolStats { + calls: number; + failed: number; +} + +/** Every call_subagent in the chat, in the order the agent made them. */ +export function collectSubagents(messages: ChatMessage[] | undefined): MonitorSubagent[] { + const result: MonitorSubagent[] = []; + for (const m of messages || []) { + if (m.role !== 'assistant') continue; + for (const tool of m.toolCalls || []) { + if (tool.name === 'call_subagent' && tool.id) { + result.push({ tool, messageUid: m.uid, isStreaming: !!m.isStreaming }); + } + } + } + return result; +} + +function toArtifact(raw: Record, chatId?: string): CanvasArtifact { + const name = String(raw.name || raw.filename || '').split(/[\\/]/).pop() || String(raw.file_id); + const artifact = { + ...raw, + file_id: String(raw.file_id), + name, + url: typeof raw.url === 'string' ? raw.url : '', + chat_id: typeof raw.chat_id === 'string' ? raw.chat_id : chatId, + } as CanvasArtifact; + return { ...artifact, origin: artifactOrigin(artifact) }; +} + +/** + * Files that went into and came out of the chat. + * + * Input is what the user attached. Output is every file a tool or sub-agent + * wrote, including intermediates the agent did not pin to the conversation: + * the monitor shows the whole run, not only the delivered cards. + */ +export function collectFiles(messages: ChatMessage[] | undefined, chatId?: string): MonitorFiles { + const input = new Map(); + const output = new Map(); + const collect = (value: unknown) => { + for (const raw of extractArtifactOutputs(value)) { + const artifact = toArtifact(raw, chatId); + output.delete(artifact.file_id); + output.set(artifact.file_id, artifact); + } + }; + for (const m of messages || []) { + if (m.role === 'user') { + for (const att of m.attachments || []) { + if (!att.file_id) continue; + input.set(att.file_id, toArtifact({ + file_id: att.file_id, origin: att.origin, name: att.name, + url: att.download_url || `/files/${att.file_id}`, mime_type: att.mime_type, + }, chatId)); + } + continue; + } + for (const tool of m.toolCalls || []) { + if (tool.status != null && tool.status !== 'success') continue; + // pin_to_workspace echoes a file that another tool already produced. + if (tool.name !== 'pin_to_workspace') collect(tool.output); + for (const step of tool.subSteps || []) { + if (step.kind === 'tool' && step.status === 'success') collect(step.output); + } + } + } + return { input: [...input.values()], output: [...output.values()] }; +} + +/** Tool calls made in the chat, sub-agent steps included. */ +export function countToolCalls(messages: ChatMessage[] | undefined): MonitorToolStats { + let calls = 0; + let failed = 0; + for (const m of messages || []) { + for (const tool of m.toolCalls || []) { + calls += 1; + if (tool.status === 'error') failed += 1; + for (const step of tool.subSteps || []) { + if (step.kind !== 'tool') continue; + calls += 1; + if (step.status === 'error') failed += 1; + } + } + } + return { calls, failed }; +} diff --git a/src/frontend/src/components/monitor/useBackgroundTasks.ts b/src/frontend/src/components/monitor/useBackgroundTasks.ts new file mode 100644 index 00000000..d38532ab --- /dev/null +++ b/src/frontend/src/components/monitor/useBackgroundTasks.ts @@ -0,0 +1,74 @@ +import { useCallback, useEffect, useState } from 'react'; +import { cancelCommandApi, cancelJobApi, listChatCommands, listChatJobs } from '../../api'; +import { useChatStore } from '../../stores/chatStore'; +import type { CommandTaskBrief, JobBrief } from '../../types'; + +const LIVE_COMMAND = new Set(['starting', 'running']); +const LIVE_JOB = new Set(['pending', 'running']); +/** Live tasks change by the second; poll only while something can still move. */ +const POLL_MS = 3000; + +export interface BackgroundTasks { + commands: CommandTaskBrief[]; + jobs: JobBrief[]; + cancelCommand: (taskId: string) => Promise; + cancelJob: (jobId: string) => Promise; +} + +export const isLiveCommand = (task: CommandTaskBrief) => LIVE_COMMAND.has(task.status); +export const isLiveJob = (job: JobBrief) => LIVE_JOB.has(job.status); + +/** + * Background commands and batch jobs of a chat. + * + * Polls while a task is live or the chat has a run in flight (a run can hand a + * command to the background at any moment); otherwise it fetches once per chat + * and again when a run ends. + */ +export function useBackgroundTasks(chatId: string | null): BackgroundTasks { + // Keyed by chat so switching chats never shows the previous chat's tasks. + const [state, setState] = useState<{ chatId: string | null; commands: CommandTaskBrief[]; jobs: JobBrief[] }>( + { chatId: null, commands: [], jobs: [] }, + ); + const [refresh, setRefresh] = useState(0); + const runActive = useChatStore((s) => !!chatId && (s.sendingChatIds.has(chatId) || !!s.activeRuns[chatId])); + const commands = state.chatId === chatId ? state.commands : []; + const jobs = state.chatId === chatId ? state.jobs : []; + const live = commands.some(isLiveCommand) || jobs.some(isLiveJob); + + useEffect(() => { + if (!chatId) return; + let active = true; + let timer: ReturnType | undefined; + const load = async () => { + const [nextCommands, nextJobs] = await Promise.all([ + listChatCommands(chatId).catch(() => null), + listChatJobs(chatId, false).catch(() => null), + ]); + if (!active) return; + setState((prev) => { + const base = prev.chatId === chatId ? prev : { chatId, commands: [], jobs: [] }; + return { + chatId, + // Older backends do not send `background`; their live rows are the background ones. + commands: nextCommands ? nextCommands.filter((task) => task.background ?? isLiveCommand(task)) : base.commands, + jobs: nextJobs ?? base.jobs, + }; + }); + if (runActive || live) timer = setTimeout(load, POLL_MS); + }; + void load(); + return () => { active = false; clearTimeout(timer); }; + }, [chatId, runActive, live, refresh]); + + const cancelCommand = useCallback(async (taskId: string) => { + await cancelCommandApi(taskId, chatId || undefined); + setRefresh((n) => n + 1); + }, [chatId]); + const cancelJob = useCallback(async (jobId: string) => { + await cancelJobApi(jobId); + setRefresh((n) => n + 1); + }, []); + + return { commands, jobs, cancelCommand, cancelJob }; +} diff --git a/src/frontend/src/components/monitor/useChatProject.ts b/src/frontend/src/components/monitor/useChatProject.ts new file mode 100644 index 00000000..f2725f5a --- /dev/null +++ b/src/frontend/src/components/monitor/useChatProject.ts @@ -0,0 +1,36 @@ +import { useCallback, useEffect, useState } from 'react'; +import { getProject } from '../../api'; +import { getProjectEnvironment, type ProjectEnvironment } from '../../api/projectEnvironment'; +import type { ProjectDetail } from '../../types'; + +type EnvironmentMode = ProjectEnvironment['mode']; + +export interface ChatProject { + project: ProjectDetail; + /** null when the environment is not readable for this user. */ + environment: EnvironmentMode | null; + reloadEnvironment: () => void; +} + +/** The project a chat belongs to and its environment mode, for the run monitor. */ +export function useChatProject(projectId: string | undefined): ChatProject | null { + // Keyed by project so switching chats never shows the previous project. + const [state, setState] = useState<{ id: string; project: ProjectDetail | null; environment: EnvironmentMode | null } | null>(null); + const [revision, setRevision] = useState(0); + + useEffect(() => { + if (!projectId) return; + let active = true; + void Promise.all([ + getProject(projectId).catch(() => null), + getProjectEnvironment(projectId).then((detail) => detail.config.mode).catch(() => null), + ]).then(([project, environment]) => { + if (active) setState({ id: projectId, project, environment }); + }); + return () => { active = false; }; + }, [projectId, revision]); + + const reloadEnvironment = useCallback(() => setRevision((n) => n + 1), []); + if (!projectId || state?.id !== projectId || !state.project) return null; + return { project: state.project, environment: state.environment, reloadEnvironment }; +} diff --git a/src/frontend/src/components/myspace/MySpacePanel.tsx b/src/frontend/src/components/myspace/MySpacePanel.tsx index d16f104b..0e111bbc 100644 --- a/src/frontend/src/components/myspace/MySpacePanel.tsx +++ b/src/frontend/src/components/myspace/MySpacePanel.tsx @@ -1,8 +1,9 @@ -import { Suspense, useEffect } from 'react'; +import { Suspense, useEffect, useRef } from 'react'; import { useKbTab, useMySpaceTab } from '../../routing/subPages'; import { useMySpaceStore } from '../../stores/mySpaceStore'; import { useEditionStore } from '../../stores'; import { AgentWorkspaceView, usePersonalAgentName } from '../../personalAgentEdition'; +import { AgentWorkspaceSkeleton } from '../personal-agent/workspace/AgentWorkspaceSkeleton'; import { CatalogPanel } from '../catalog'; import { ShareRecordsPage } from '../share'; import { NotificationList } from './NotificationList'; @@ -28,10 +29,21 @@ export function MySpacePanel() { const tabs = TABS.filter(item => (item.scope !== 'team' || teams) && (item.kb !== 'public' || edition !== 'ce') && (!item.requiresPersonalAgent || !!agentName)); const active = tabs.find(item => item.tab === tab && (!item.scope || item.scope === kind) && (!item.kb || item.kb === kb)); + // 窄屏的 Tab 是横向滚动的胶囊条:切到靠后的 Tab(如从别处跳到「消息通知」)时把它滚进可视区。 + const tabsRef = useRef(null); + useEffect(() => { + const strip = tabsRef.current; + const current = strip?.querySelector('.jx-mySpace-tab.active'); + if (!strip || !current || strip.scrollWidth <= strip.clientWidth) return; + const left = current.getBoundingClientRect().left - strip.getBoundingClientRect().left + strip.scrollLeft; + if (left < strip.scrollLeft || left + current.offsetWidth > strip.scrollLeft + strip.clientWidth) { + strip.scrollTo({ left: Math.max(0, left - 16) }); + } + }, [active?.key]); return

{tab === 'agent' ? agentName : active?.label}

-
+
{tabs.map(item =>
{ useChatStore.getState().setCurrentChatId(id); useCatalogStore.getState().setPanel('chat'); }} onPreviewFile={openFile} /> diff --git a/src/frontend/src/components/myspace/SpaceFileBrowser.tsx b/src/frontend/src/components/myspace/SpaceFileBrowser.tsx index 4fa58016..6ddad65f 100644 --- a/src/frontend/src/components/myspace/SpaceFileBrowser.tsx +++ b/src/frontend/src/components/myspace/SpaceFileBrowser.tsx @@ -1,6 +1,6 @@ import { useEffect, useRef, useState } from 'react'; import { Alert, Button, Dropdown, Empty, Input, Modal } from 'antd'; -import { ArrowLeftOutlined, DownOutlined, FolderAddOutlined, ReloadOutlined, SafetyOutlined, SearchOutlined, UploadOutlined } from '@ant-design/icons'; +import { ArrowLeftOutlined, DownOutlined, FolderAddOutlined, FolderOpenOutlined, PlusOutlined, ReloadOutlined, SafetyOutlined, SearchOutlined, UploadOutlined } from '@ant-design/icons'; import { SidebarToggleIcon } from './SidebarToggleIcon'; import type { PersonalFolderNode, ResourceItem } from '../../types'; import { t } from '../../i18n'; @@ -11,8 +11,10 @@ import { MySpaceSkeleton } from './MySpaceSkeleton'; import { SpacePermissions } from './SpacePermissions'; import { useSpaceCanvas } from './useSpaceCanvas'; import { useDelayedFlag } from '../../hooks'; +import { useIsMobileViewport } from '../../hooks/useIsMobileViewport'; import { SpaceDirectoryTree } from './SpaceDirectoryTree'; import { useSpaceDirectory } from './useSpaceDirectory'; +import { useMySpaceStore } from '../../stores/mySpaceStore'; import { createSpaceDirectory, deleteSpaceEntry, downloadSpaceFile, renameSpaceFile, uploadSpaceFile, type SpaceEntry } from './spaceFilesApi'; const joinPath = (parent: string, name: string) => parent ? parent + '/' + name : name; @@ -21,6 +23,14 @@ const imageName = /\.(png|jpe?g|gif|webp|bmp|svg|avif|ico)$/i; export function SpaceFileBrowser({ kind }: { kind: 'personal' | 'team' }) { const [query, setQuery] = useState(''); const s = useSpaceDirectory(kind, query); + // Another view (the run monitor) asked for a folder: open it once its space is listed. + const pendingFolder = useMySpaceStore((state) => state.pendingFolder); + const { open: openFolder, roots: spaceRoots } = s; + useEffect(() => { + if (!pendingFolder || !spaceRoots.some((item) => item.space_id === pendingFolder.space)) return; + openFolder(pendingFolder.space, pendingFolder.path); + useMySpaceStore.getState().clearPendingFolder(); + }, [pendingFolder, spaceRoots, openFolder]); const { roots, selected, path, setPath, page, loading, initializing, busy, error, root, writable, reload, perform, loadMore, dialog, setDialog, name, setName, permissions, setPermissions } = s; const [filter, setFilter] = useState<'document' | 'image'>('document'); @@ -34,6 +44,8 @@ export function SpaceFileBrowser({ kind }: { kind: 'personal' | 'team' }) { const [drawerOpen, setDrawerOpen] = useState(false); const [pagedAt, setPagedAt] = useState(''); const fileInput = useRef(null); + // 手机上「新建文件夹 / 上传」收进一个圆形「+」菜单,像 iOS 导航栏的添加按钮,不再并排两个大按钮。 + const isMobile = useIsMobileViewport(); useEffect(() => { if (!drawerOpen) return; const close = (event: KeyboardEvent) => { if (event.key === 'Escape') setDrawerOpen(false); }; @@ -154,8 +166,18 @@ export function SpaceFileBrowser({ kind }: { kind: 'personal' | 'team' }) {
- {kind === 'personal' && path && } - {kind === 'personal' && writable && <> + {kind === 'personal' && path && } + {kind === 'personal' && writable && isMobile && , label: t('新建文件夹'), disabled: busy, onClick: () => { setName(''); setDialog('directory'); } }, + { key: 'file', icon: , label: t(filter === 'image' ? '上传图片' : '上传文件'), disabled: busy, onClick: () => fileInput.current?.click() }, + { key: 'folder', icon: , label: t('上传文件夹'), disabled: busy, onClick: () => folderInput.current?.click() }, + ] }}> + + } + {kind === 'personal' && writable && !isMobile && <> fileInput.current?.click() }, @@ -166,7 +188,7 @@ export function SpaceFileBrowser({ kind }: { kind: 'personal' | 'team' }) { } - {kind === 'personal' &&
diff --git a/src/frontend/src/components/onboarding/FirstRunSetup.tsx b/src/frontend/src/components/onboarding/FirstRunSetup.tsx index 02eadf24..3b78d5bc 100644 --- a/src/frontend/src/components/onboarding/FirstRunSetup.tsx +++ b/src/frontend/src/components/onboarding/FirstRunSetup.tsx @@ -3,7 +3,6 @@ import { CheckCircleFilled, ExclamationCircleFilled, RobotOutlined, - SafetyCertificateOutlined, ThunderboltOutlined, } from '@ant-design/icons'; @@ -29,7 +28,6 @@ import { getMemorySettings, getModelProviderSchemas, getMyServiceConfigs, - getOntologySettings, getUserPreferences, listModelProviders, listModelRoles, @@ -38,7 +36,6 @@ import { updateMemorySettings, updateMemoryWriteSettings, updateMyServiceConfigs, - updateOntologySettings, updateUserPreferences, type AuthUser, type ModelProviderItem, @@ -112,7 +109,7 @@ const STEP_META = [ { title: '模型配置', hint: '连接推理与检索服务' }, { title: '联网搜索', hint: '可选服务' }, { title: '文件解析', hint: '可选服务' }, - { title: '智能增强', hint: '记忆与本体' }, + { title: '智能增强', hint: '记忆与进化' }, { title: '准备就绪', hint: '检查并进入' }, ] as const; @@ -182,9 +179,6 @@ export function FirstRunSetup({ user, onComplete }: FirstRunSetupProps) { // below still decides whether that choice can take effect, so this is a // recommendation rather than a decision made on the user's behalf. const [evolutionEnabled, setEvolutionEnabled] = useState(true); - const [ontologyEnabled, setOntologyEnabled] = useState(false); - const [ontologyAvailable, setOntologyAvailable] = useState(false); - const [activeOntologyCount, setActiveOntologyCount] = useState(0); // Evolution cannot do its job without the embedding model (intent clustering // and the memory scan both fall back to something measurably blind) and it @@ -209,7 +203,7 @@ export function FirstRunSetup({ user, onComplete }: FirstRunSetupProps) { setLoading(true); setLoadError(''); try { - const [prefs, modelProviders, modelRoles, providerSchemas, groups, memory, ontology] = + const [prefs, modelProviders, modelRoles, providerSchemas, groups, memory] = await Promise.all([ getUserPreferences(user.user_id), listModelProviders(), @@ -217,7 +211,6 @@ export function FirstRunSetup({ user, onComplete }: FirstRunSetupProps) { getModelProviderSchemas(), getMyServiceConfigs(), getMemorySettings(), - getOntologySettings(), ]); setPreferences(prefs); @@ -272,9 +265,6 @@ export function FirstRunSetup({ user, onComplete }: FirstRunSetupProps) { setMemoryWriteEnabled(memory.memory_write_enabled); setMemoryAvailable(memory.mem0_available); setEmbeddingAvailable(memory.embedding_available); - setOntologyEnabled(ontology.ontology_enabled); - setOntologyAvailable(ontology.available); - setActiveOntologyCount(ontology.active_packs?.length || 0); } catch (error) { setLoadError((error as Error).message || t('初始化信息加载失败')); } finally { @@ -479,7 +469,6 @@ export function FirstRunSetup({ user, onComplete }: FirstRunSetupProps) { // Written on every pass, including when it resolves to false, so the // wizard is what decides this rather than the stored default. updateEvolutionPrefs({ enabled: evolutionActive }), - updateOntologySettings(ontologyAvailable && ontologyEnabled), ]); } else { await completeFirstRunSetup(); @@ -490,7 +479,6 @@ export function FirstRunSetup({ user, onComplete }: FirstRunSetupProps) { } await Promise.all([ useSettingsStore.getState().loadMemorySettings(), - useSettingsStore.getState().loadOntologySettings(), useModelCapabilitiesStore.getState().fetchCapabilities(), ]); message.success(t('初始化完成,欢迎使用 {name}', { name: brandName })); @@ -938,23 +926,6 @@ export function FirstRunSetup({ user, onComplete }: FirstRunSetupProps) { )}
-
-
-
-
- {t('本体核验')} - -
- {t('工具调用和高风险结果会经过领域规则检查,减少不符合业务约束的输出。')} - {ontologyAvailable - ? {t('已发布 {n} 个领域包', { n: activeOntologyCount })} - : {t('尚无已发布的领域本体包')}} -
-
); @@ -970,7 +941,6 @@ export function FirstRunSetup({ user, onComplete }: FirstRunSetupProps) {
{t('重排模型')}{selectedAuxProviderIds.reranker ? t('已配置') : t('未配置')}
{t('永久记忆')}{memoryReady ? t('开启') : t('关闭')}
{t('启动进化')}{evolutionActive ? t('开启') : t('关闭')}
-
{t('本体核验')}{ontologyAvailable && ontologyEnabled ? t('开启') : t('关闭')}
); @@ -988,7 +958,7 @@ export function FirstRunSetup({ user, onComplete }: FirstRunSetupProps) { ['连接你的 AI 模型', '主对话模型为必填项;索引模型和重排模型可按需配置。'], ['让智能体访问实时信息', '配置搜索服务后,天气、新闻和公开资料查询会更加完整。'], ['决定如何读取复杂文档', '外部解析服务适合 PDF、扫描件和包含公式的文档。'], - ['选择需要的智能增强', '你可以控制是否跨对话记忆、是否让系统从你的用法中自我进化,以及是否执行领域本体核验。'], + ['选择需要的智能增强', '你可以控制是否跨对话记忆,以及是否让系统从你的用法中自我进化。'], ['一切就绪', '完成初始化后,你将进入 HugAgentOS 工作台。'], ]; diff --git a/src/frontend/src/components/ontology/OntologyDetailDrawer.tsx b/src/frontend/src/components/ontology/OntologyDetailDrawer.tsx deleted file mode 100644 index 02a25658..00000000 --- a/src/frontend/src/components/ontology/OntologyDetailDrawer.tsx +++ /dev/null @@ -1,617 +0,0 @@ -import { useCallback, useEffect, useMemo, useState } from 'react'; -import { - Alert, - Button, - Card, - Descriptions, - Drawer, - Empty, - Select, - Space, - Spin, - Tabs, - Tag, - Typography, -} from 'antd'; -import { - CheckCircleOutlined, - ClockCircleOutlined, - CloseCircleOutlined, - DeploymentUnitOutlined, - EditOutlined, - SafetyCertificateOutlined, - ShareAltOutlined, - TagsOutlined, -} from '@ant-design/icons'; -import { adminFetch } from '../../utils/adminApi'; -import { formatDateTime } from '../../utils/date'; -import { t } from '../../i18n'; -import { OntologyListPagination } from './OntologyListPagination'; -import { ONTOLOGY_LIST_DEFAULT_PAGE_SIZE, paginateOntologyItems } from './ontologyPagination'; -import { OntologyModuleEditor } from './OntologyModuleEditor'; -import type { - OntologyDocument, - OntologyEditableModule, - OntologyPackSummary, - OntologyPackVersion, - ValidationIssue, -} from './ontologyTypes'; - -const { Paragraph, Text, Title } = Typography; - -interface OntologyDetailDrawerProps { - token: string; - apiPrefix: string; - pack: OntologyPackSummary | null; - initialVersionId?: string; - onClose: () => void; - onChanged: () => Promise | void; -} - -interface EditableModulePanelProps { - module: OntologyEditableModule; - description: string; - onEdit: (module: OntologyEditableModule) => void; - children: React.ReactNode; -} - -type PaginatedOntologyModule = Exclude; - -interface OntologyModulePaginationState { - current: number; - pageSize: number; -} - -interface OntologyPaginationStore { - requestKey: string; - modules: Record; -} - -function createModulePagination(): Record { - return { - concepts: { current: 1, pageSize: ONTOLOGY_LIST_DEFAULT_PAGE_SIZE }, - relations: { current: 1, pageSize: ONTOLOGY_LIST_DEFAULT_PAGE_SIZE }, - constraints: { current: 1, pageSize: ONTOLOGY_LIST_DEFAULT_PAGE_SIZE }, - workflows: { current: 1, pageSize: ONTOLOGY_LIST_DEFAULT_PAGE_SIZE }, - }; -} - -const MODULE_TITLES: Record = { - overview: t('基本信息与运行配置'), - concepts: t('概念词表'), - relations: t('概念关系'), - constraints: t('执行约束'), - workflows: t('领域工作流'), -}; - -function EditableModulePanel({ module, description, onEdit, children }: EditableModulePanelProps) { - return ( -
-
-
- {MODULE_TITLES[module]} - {description} -
- -
- {children} -
- ); -} - -function statusColor(status: OntologyPackVersion['status']): string { - if (status === 'active') return 'green'; - if (status === 'retired') return 'default'; - return 'blue'; -} - -function statusLabel(status: OntologyPackVersion['status']): string { - if (status === 'active') return t('已激活'); - if (status === 'retired') return t('已归档'); - return t('工作草稿'); -} - -function riskColor(risk?: string): string { - if (risk === 'high') return 'red'; - if (risk === 'medium') return 'orange'; - return 'green'; -} - -function yesNo(value?: boolean): string { - return value ? t('是') : t('否'); -} - -function ValueTags({ values, color }: { values?: string[]; color?: string }) { - if (!values?.length) return {t('无')}; - return ( - - {values.map((value) => {value})} - - ); -} - -function ValidationList({ items, kind }: { items?: ValidationIssue[]; kind: 'error' | 'warning' }) { - if (!items?.length) { - return ; - } - const Icon = kind === 'error' ? CloseCircleOutlined : ClockCircleOutlined; - return ( -
- {items.map((item, index) => ( -
- -
- {item.path && {item.path}} - {item.message || t('未知校验问题')} -
-
- ))} -
- ); -} - -export function OntologyDetailDrawer({ - token, - apiPrefix, - pack, - initialVersionId, - onClose, - onChanged, -}: OntologyDetailDrawerProps) { - const [selectedVersionId, setSelectedVersionId] = useState(initialVersionId); - const [editingModule, setEditingModule] = useState(null); - const [loadState, setLoadState] = useState<{ - key: string; - document: OntologyDocument | null; - error: string; - }>({ key: '', document: null, error: '' }); - const initialVersion = pack?.versions.find((version) => version.version_id === pack.active_version_id) - ?? pack?.versions[0]; - const effectiveVersionId = pack?.versions.some((version) => version.version_id === selectedVersionId) - ? selectedVersionId - : initialVersion?.version_id; - const packId = pack?.pack_id; - const selectedVersion = pack?.versions.find((version) => version.version_id === effectiveVersionId); - const workingDraft = pack?.versions.find( - (version) => version.version_id === pack.working_draft_version_id, - ) ?? pack?.versions.find((version) => version.status === 'draft'); - const existingVersions = useMemo( - () => pack?.versions.map((version) => version.version) ?? [], - [pack?.versions], - ); - const requestKey = packId && effectiveVersionId - ? `${packId}:${effectiveVersionId}:${selectedVersion?.checksum ?? ''}` - : ''; - const [paginationStore, setPaginationStore] = useState(() => ({ - requestKey: '', - modules: createModulePagination(), - })); - const modulePagination = paginationStore.requestKey === requestKey - ? paginationStore.modules - : createModulePagination(); - - const handleEditModule = (module: OntologyEditableModule) => { - if (workingDraft && effectiveVersionId !== workingDraft.version_id) { - setSelectedVersionId(workingDraft.version_id); - } - setEditingModule(module); - }; - - useEffect(() => { - if (!packId || !effectiveVersionId || !requestKey) return; - let cancelled = false; - void adminFetch( - token, - `${apiPrefix}/${encodeURIComponent(packId)}/versions/${encodeURIComponent(effectiveVersionId)}/export`, - ) - .then((response) => { - if (!cancelled) { - setLoadState({ key: requestKey, document: (response?.data ?? response) as OntologyDocument, error: '' }); - } - }) - .catch((error: unknown) => { - if (!cancelled) { - setLoadState({ - key: requestKey, - document: null, - error: error instanceof Error ? error.message : t('完整本体内容加载失败'), - }); - } - }); - return () => { cancelled = true; }; - }, [apiPrefix, effectiveVersionId, packId, requestKey, token]); - - const handleModulePageChange = useCallback(( - module: PaginatedOntologyModule, - current: number, - pageSize: number, - ) => { - setPaginationStore((previous) => { - const modules = previous.requestKey === requestKey - ? previous.modules - : createModulePagination(); - return { - requestKey, - modules: { - ...modules, - [module]: { current, pageSize }, - }, - }; - }); - }, [requestKey]); - - const isCurrentRequest = Boolean(requestKey) && loadState.key === requestKey; - const loading = Boolean(requestKey) && !isCurrentRequest; - const document = isCurrentRequest ? loadState.document : null; - const loadError = isCurrentRequest ? loadState.error : ''; - const concepts = useMemo(() => document?.concepts ?? [], [document]); - const relations = document?.relations ?? []; - const constraints = document?.constraints ?? []; - const workflows = document?.workflows ?? []; - const pagedConcepts = paginateOntologyItems( - concepts, - modulePagination.concepts.current, - modulePagination.concepts.pageSize, - ); - const pagedRelations = paginateOntologyItems( - relations, - modulePagination.relations.current, - modulePagination.relations.pageSize, - ); - const pagedConstraints = paginateOntologyItems( - constraints, - modulePagination.constraints.current, - modulePagination.constraints.pageSize, - ); - const pagedWorkflows = paginateOntologyItems( - workflows, - modulePagination.workflows.current, - modulePagination.workflows.pageSize, - ); - const conceptNames = useMemo( - () => new Map(concepts.map((concept) => [concept.id, concept.name])), - [concepts], - ); - - const overview = document && selectedVersion ? ( - -
- - - {document.pack_id} - {document.domain} - {document.schema_version || '-'} - v{document.version} - {formatDateTime(selectedVersion.created_at)} - {formatDateTime(selectedVersion.activated_at)} - {selectedVersion.checksum} - - - - - - {yesNo(document.config?.injection_enabled)} - {document.config?.max_concepts ?? '-'} - {document.config?.token_budget ?? '-'} - {document.config?.committee_size ?? '-'} - {document.config?.repeated_denial_threshold ?? '-'} - {document.config?.circuit_breaker_threshold ?? '-'} - {yesNo(document.config?.allow_unresolved_tools)} - - -
-
- ) : null; - - const conceptPanel = concepts.length ? ( - -
- {pagedConcepts.map((concept) => ( - -
-
- {concept.name} - {concept.id} -
- {t('{risk} 风险', { risk: concept.risk ?? 'low' })} -
- {concept.definition} - {concept.parent_id && ( -
- {t('上位概念')} - {conceptNames.get(concept.parent_id) || concept.parent_id} {concept.parent_id} -
- )} -
- {t('别名')} - -
-
- {t('标签')} - -
- {Boolean(concept.closed_values?.length) && ( -
- {t('受控取值')} - -
- )} -
- ))} -
- handleModulePageChange('concepts', current, pageSize)} - /> -
- ) : ( - - - - ); - - const relationPanel = relations.length ? ( - -
- {pagedRelations.map((relation) => ( - -
-
{conceptNames.get(relation.subject) || relation.subject}{relation.subject}
-
{relation.predicate}
-
{conceptNames.get(relation.object) || relation.object}{relation.object}
-
- {relation.description && {relation.description}} - - {relation.id} - {t('最少 {n} 个', { n: relation.min_cardinality ?? 0 })} - {relation.max_cardinality == null ? t('数量不设上限') : t('最多 {n} 个', { n: relation.max_cardinality })} - {relation.forbidden && {t('禁止关系')}} - -
- ))} -
- handleModulePageChange('relations', current, pageSize)} - /> -
- ) : ( - - - - ); - - const constraintPanel = constraints.length ? ( - -
- {pagedConstraints.map((constraint) => { - const target = constraint.target; - const targetValue = target.kind === 'output' - ? target.output_tag - : [target.tool, target.parameter].filter(Boolean).join(' · '); - return ( - -
-
- {constraint.name} - {constraint.id} -
- - {constraint.mode} - {t('{risk} 风险', { risk: constraint.risk ?? 'low' })} - {constraint.enabled === false ? t('已停用') : t('已启用')} - -
- - - {target.kind} {targetValue || '-'} - - {constraint.concept_id || '-'} - {yesNo(constraint.requires_citations)} - - - -
- {t('校验 Schema')} -
{JSON.stringify(constraint.schema ?? {}, null, 2)}
-
-
- ); - })} -
- handleModulePageChange('constraints', current, pageSize)} - /> -
- ) : ( - - - - ); - - const workflowPanel = workflows.length ? ( - -
- {pagedWorkflows.map((workflow, index) => ( - -
- {String((modulePagination.workflows.current - 1) * modulePagination.workflows.pageSize + index + 1).padStart(2, '0')} -
-
-
-
{workflow.name}{workflow.id}
- - {t('评审:{level}', { level: workflow.review_level ?? 'none' })} - {t('{risk} 风险', { risk: workflow.risk ?? 'low' })} - -
-
{t('命中词')}
-
- {t('资产触发')} - [ - ...(trigger.ids || []).map((id) => `${trigger.kind}:${id}`), - ...(trigger.tags_any || []).map((tag) => `${trigger.kind}:#${tag}`), - ])} - color="cyan" - /> -
-
{t('必需工具')}
-
{t('禁止工具')}
-
{t('输出标签')}
-
-
- ))} -
- handleModulePageChange('workflows', current, pageSize)} - /> -
- ) : ( - - - - ); - - const validationPanel = selectedVersion ? ( -
- - - -
- ) : null; - - const rawPanel = document ? ( -
-
- {t('原始 JSON 保留全部字段,可用于核对或迁移。')} - {t('复制完整 JSON')} -
-
{JSON.stringify(document, null, 2)}
-
- ) : null; - - return ( - <> - { - setEditingModule(null); - onClose(); - }} - width="min(1120px, calc(100vw - 24px))" - destroyOnHidden - className="jx-ontologyDetailDrawer" - extra={pack && ( -