diff --git a/src/services/api-handlers.ts b/src/services/api-handlers.ts index 730d8353..bd26f155 100644 --- a/src/services/api-handlers.ts +++ b/src/services/api-handlers.ts @@ -1427,6 +1427,8 @@ export async function handleDetectTagMigration(): Promise< interface MigrationProgress { processed: number; + /** Next index in the memory list to attempt; advances even when tagging fails. */ + cursor: number; total: number; currentBatch: number; totalBatches: number; @@ -1436,6 +1438,7 @@ interface MigrationProgress { const migrationProgress: MigrationProgress = { processed: 0, + cursor: 0, total: 0, currentBatch: 0, totalBatches: 0, @@ -1449,7 +1452,7 @@ export async function handleGetTagMigrationProgress(): Promise> { +): Promise> { try { await ensureTursoReady(); const { AIProviderFactory } = await import("./ai/ai-provider-factory.js"); @@ -1471,13 +1474,19 @@ export async function handleRunTagMigrationBatch( } } - if (migrationProgress.total === 0) { + // Fresh run (or retry after a completed pass): reset counters so soft + // failures from a prior attempt can be retried instead of being skipped. + if (migrationProgress.total === 0 || migrationProgress.isComplete) { + migrationProgress.processed = 0; + migrationProgress.cursor = 0; migrationProgress.total = allMemories.length; migrationProgress.totalBatches = Math.ceil(allMemories.length / batchSize); + migrationProgress.currentBatch = 0; migrationProgress.isComplete = false; + migrationProgress.errors = []; } - const startIdx = migrationProgress.processed; + const startIdx = migrationProgress.cursor; const endIdx = Math.min(startIdx + batchSize, allMemories.length); for (let i = startIdx; i < endIdx; i++) { @@ -1519,6 +1528,15 @@ export async function handleRunTagMigrationBatch( currentTags.join(","), m.id, ]); + } else { + // Soft failure (e.g. empty tool-call args): do not mark processed. + // Cursor still advances below so the batch window moves forward. + const errorMsg = `Tag generation failed for memory ${m.id}: ${ + result.error ?? "no tags returned" + }`; + migrationProgress.errors.push(errorMsg); + log("Migration error for memory", { id: m.id, error: errorMsg }); + continue; } } @@ -1538,8 +1556,9 @@ export async function handleRunTagMigrationBatch( } } + migrationProgress.cursor = endIdx; migrationProgress.currentBatch++; - const hasMore = migrationProgress.processed < migrationProgress.total; + const hasMore = migrationProgress.cursor < allMemories.length; if (!hasMore) { migrationProgress.isComplete = true; @@ -1547,7 +1566,12 @@ export async function handleRunTagMigrationBatch( return { success: true, - data: { processed: migrationProgress.processed, total: migrationProgress.total, hasMore }, + data: { + processed: migrationProgress.processed, + total: migrationProgress.total, + hasMore, + errors: migrationProgress.errors.length, + }, }; } catch (error) { return { success: false, error: String(error) }; diff --git a/tests/tag-migration-soft-failure.test.ts b/tests/tag-migration-soft-failure.test.ts new file mode 100644 index 00000000..e0bc0e8f --- /dev/null +++ b/tests/tag-migration-soft-failure.test.ts @@ -0,0 +1,182 @@ +import { afterEach, describe, expect, it } from "bun:test"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const tempDirs: string[] = []; + +afterEach(() => { + while (tempDirs.length > 0) { + const dir = tempDirs.pop(); + if (dir) rmSync(dir, { recursive: true, force: true }); + } +}); + +const apiHandlersUrl = new URL("../src/services/api-handlers.js", import.meta.url).href; +const embeddingUrl = new URL("../src/services/embedding.js", import.meta.url).href; +const vectorUtilsUrl = new URL("../src/services/turso/vector-utils.js", import.meta.url).href; +const connectionManagerUrl = new URL("../src/services/turso/connection-manager.js", import.meta.url) + .href; +const shardManagerUrl = new URL("../src/services/turso/shard-manager.js", import.meta.url).href; +const vectorSearchUrl = new URL("../src/services/turso/vector-search.js", import.meta.url).href; +const readyUrl = new URL("../src/services/turso/ready.js", import.meta.url).href; +const userPromptManagerUrl = new URL( + "../src/services/user-prompt/user-prompt-manager.js", + import.meta.url +).href; +const loggerUrl = new URL("../src/services/logger.js", import.meta.url).href; +const factoryUrl = new URL("../src/services/ai/ai-provider-factory.js", import.meta.url).href; +const providerConfigUrl = new URL("../src/services/ai/provider-config.js", import.meta.url).href; +const configUrl = new URL("../src/config.js", import.meta.url).href; + +function runScenario(scriptBody: string) { + const dir = mkdtempSync(join(tmpdir(), "opencode-mem-tag-mig-")); + tempDirs.push(dir); + const scriptPath = join(dir, "scenario.mjs"); + const script = ` +import { mock } from "bun:test"; + +const updateVectorCalls = []; +const updateTagCalls = []; +const memories = [ + { id: "m1", content: "untagged one", tags: null }, + { id: "m2", content: "untagged two", tags: "" }, +]; + +mock.module(${JSON.stringify(embeddingUrl)}, () => ({ + embeddingService: { + isWarmedUp: true, + warmup: async () => {}, + embedWithTimeout: async () => new Float32Array([1, 2, 3]), + }, +})); + +mock.module(${JSON.stringify(vectorUtilsUrl)}, () => ({ + formatTagsForEmbedding: (tags) => tags.join(","), +})); + +mock.module(${JSON.stringify(connectionManagerUrl)}, () => ({ + tursoConnectionManager: { + getConnection: async () => ({ + all: async () => memories, + run: async (sql, params) => { + updateTagCalls.push({ sql, params }); + }, + get: async () => ({ count: memories.length }), + }), + closeAll: async () => {}, + }, +})); + +mock.module(${JSON.stringify(shardManagerUrl)}, () => ({ + tursoShardManager: { + async getAllShards() { + return [{ id: 1, scope: "project", scopeHash: "h", shardIndex: 0, dbPath: "/tmp/shard.db" }]; + }, + async withScopeWriteLock(_scope, _hash, operation) { + return operation(); + }, + async getWriteShard() { + return { id: 1, scope: "project", scopeHash: "h", shardIndex: 0, dbPath: "/tmp/shard.db" }; + }, + async incrementVectorCount() {}, + }, +})); + +mock.module(${JSON.stringify(vectorSearchUrl)}, () => ({ + tursoVectorSearch: { + updateVector: async (_db, id) => { + updateVectorCalls.push(id); + }, + insertVector: async () => {}, + listMemories: async () => [], + }, +})); + +mock.module(${JSON.stringify(readyUrl)}, () => ({ + ensureTursoReady: async () => {}, +})); + +mock.module(${JSON.stringify(userPromptManagerUrl)}, () => ({ + userPromptManager: {}, +})); + +mock.module(${JSON.stringify(loggerUrl)}, () => ({ + log: () => {}, +})); + +mock.module(${JSON.stringify(configUrl)}, () => ({ + CONFIG: { memoryProvider: "openai-chat" }, +})); + +mock.module(${JSON.stringify(providerConfigUrl)}, () => ({ + buildMemoryProviderConfig: () => ({}), +})); + +mock.module(${JSON.stringify(factoryUrl)}, () => ({ + AIProviderFactory: { + createProvider() { + return { + async executeToolCall() { + return { success: false, error: "empty tool-call arguments" }; + }, + }; + }, + }, +})); + +const { + handleRunTagMigrationBatch, + handleGetTagMigrationProgress, +} = await import(${JSON.stringify(apiHandlersUrl)}); + +${scriptBody} +`; + writeFileSync(scriptPath, script, "utf-8"); + const result = Bun.spawnSync({ + cmd: [process.execPath, scriptPath], + stdout: "pipe", + stderr: "pipe", + }); + const stdout = Buffer.from(result.stdout).toString("utf8").trim(); + const stderr = Buffer.from(result.stderr).toString("utf8").trim(); + const jsonLine = stdout + .split("\n") + .reverse() + .find((line) => line.trim().startsWith("{")); + + return { + exitCode: result.exitCode, + stdout, + stderr, + parsed: jsonLine ? JSON.parse(jsonLine) : null, + }; +} + +describe("tag migration soft failure", () => { + it("does not count untagged memories as processed when tag generation soft-fails", () => { + const result = runScenario(` +const batch = await handleRunTagMigrationBatch(5); +const progress = await handleGetTagMigrationProgress(); +console.log(JSON.stringify({ + batch, + progress, + updateVectorCalls, + updateTagCalls, +})); +`); + + expect(result.exitCode).toBe(0); + expect(result.parsed).not.toBeNull(); + expect(result.parsed.batch.success).toBe(true); + expect(result.parsed.batch.data.processed).toBe(0); + expect(result.parsed.batch.data.hasMore).toBe(false); + expect(result.parsed.batch.data.errors).toBe(2); + expect(result.parsed.progress.data.processed).toBe(0); + expect(result.parsed.progress.data.isComplete).toBe(true); + expect(result.parsed.progress.data.errors.length).toBe(2); + expect(result.parsed.progress.data.errors[0]).toContain("m1"); + expect(result.parsed.updateVectorCalls).toEqual([]); + expect(result.parsed.updateTagCalls).toEqual([]); + }); +}); diff --git a/web/src/lib/components/explorer/TagMigrationDialog.tsx b/web/src/lib/components/explorer/TagMigrationDialog.tsx index 25fc4bc3..02909c21 100644 --- a/web/src/lib/components/explorer/TagMigrationDialog.tsx +++ b/web/src/lib/components/explorer/TagMigrationDialog.tsx @@ -44,6 +44,9 @@ export function TagMigrationDialog({ open = false, count = 0, onOpenChange, onCo let hasMore = true; let attempts = 0; + let totalErrors = 0; + let totalItems = 0; + let totalProcessed = 0; const maxAttempts = 1000; while (hasMore && attempts < maxAttempts) { @@ -52,6 +55,7 @@ export function TagMigrationDialog({ open = false, count = 0, onOpenChange, onCo processed: number; hasMore: boolean; total: number; + errors?: number; }>("/api/migration/tags/run-batch", { method: "POST", headers: { "Content-Type": "application/json" }, @@ -64,11 +68,12 @@ export function TagMigrationDialog({ open = false, count = 0, onOpenChange, onCo return; } - const processed = result.data.processed; + totalProcessed = result.data.processed; hasMore = result.data.hasMore; - const total = result.data.total; - setProgress(total > 0 ? Math.round((processed / total) * 100) : 0); - setStatus(t("status-migration-progress", { current: processed, total })); + totalItems = result.data.total; + totalErrors = result.data.errors ?? 0; + setProgress(totalItems > 0 ? Math.round((totalProcessed / totalItems) * 100) : 0); + setStatus(t("status-migration-progress", { current: totalProcessed, total: totalItems })); if (hasMore) { await new Promise((resolve) => setTimeout(resolve, 100)); } @@ -80,6 +85,16 @@ export function TagMigrationDialog({ open = false, count = 0, onOpenChange, onCo return; } + if (totalErrors > 0) { + const finalProgress = totalItems > 0 ? Math.round((totalProcessed / totalItems) * 100) : 0; + setProgress(finalProgress); + const failedMsg = t("toast-migration-tag-failures", { count: totalErrors }); + setStatus(failedMsg); + toast.error(failedMsg); + setRunning(false); + return; + } + setProgress(100); setStatus(t("toast-migration-success")); toast.success(t("toast-migration-success")); diff --git a/web/src/lib/i18n/translations.ts b/web/src/lib/i18n/translations.ts index 5d7127dc..4a6d4326 100644 --- a/web/src/lib/i18n/translations.ts +++ b/web/src/lib/i18n/translations.ts @@ -69,6 +69,7 @@ export const translations = { "toast-bulk-delete-failed": "Failed to delete selected memories", "toast-migration-success": "Migration completed successfully", "toast-migration-failed": "Migration failed", + "toast-migration-tag-failures": "Migration failed: {count} memory tag generation failure(s)", "toast-fresh-start-success": "Fresh start completed successfully", "toast-fresh-start-failed": "Fresh start failed", "confirm-delete": "Delete this memory?", @@ -216,6 +217,7 @@ export const translations = { "toast-bulk-delete-failed": "删除选中的记忆失败", "toast-migration-success": "迁移完成", "toast-migration-failed": "迁移失败", + "toast-migration-tag-failures": "迁移失败：{count} 条记忆标签生成失败", "toast-fresh-start-success": "重新开始完成", "toast-fresh-start-failed": "重新开始失败", "confirm-delete": "删除这条记忆？", @@ -384,6 +386,7 @@ export const translations = { "toast-migration-success": "اكتملت عملية الترحيل بنجاح", "toast-migration-failed": "فشلت عملية الترحيل", + "toast-migration-tag-failures": "فشلت عملية الترحيل: {count} من حالات فشل توليد الوسوم", "toast-fresh-start-success": "تمت البداية الجديدة بنجاح", "toast-fresh-start-failed": "فشلت البداية الجديدة",