From ad239a0821a75a4b8cb7f6fdfa7349e752528b26 Mon Sep 17 00:00:00 2001 From: Qiiks Date: Thu, 17 Sep 2026 20:06:56 +0530 Subject: [PATCH 1/2] fix(pi): resolve native embedding routing before registration --- .../pi-plugin/src/embedding-bootstrap.test.ts | 46 ++++++++++++++++++ packages/pi-plugin/src/embedding-bootstrap.ts | 48 +++++++++++++++---- 2 files changed, 85 insertions(+), 9 deletions(-) diff --git a/packages/pi-plugin/src/embedding-bootstrap.test.ts b/packages/pi-plugin/src/embedding-bootstrap.test.ts index 9430b4c66..e6b9997a2 100644 --- a/packages/pi-plugin/src/embedding-bootstrap.test.ts +++ b/packages/pi-plugin/src/embedding-bootstrap.test.ts @@ -1,4 +1,6 @@ import { describe, expect, it } from "bun:test"; +import { promises as fs } from "node:fs"; +import path from "node:path"; import { getProjectEmbeddingSnapshot } from "@magic-context/core/features/magic-context/memory/embedding"; import { getProjectEmbeddings, @@ -16,9 +18,11 @@ describe("ensureProjectRegisteredFromPiDirectory", () => { it("preserves the embedding cache across consecutive identical registrations", async () => { const db = createTestDb(); const oldHome = process.env.HOME; + const oldConfigHome = process.env.XDG_CONFIG_HOME; const directory = createTestTempDir("pi-embedding-bootstrap-").dir; const fakeHome = createTestTempDir("pi-embedding-home-").dir; process.env.HOME = fakeHome; + process.env.XDG_CONFIG_HOME = path.join(fakeHome, ".config"); resetEmbeddingCacheForTests(); try { const projectIdentity = resolveProjectIdentity(directory); @@ -43,6 +47,48 @@ describe("ensureProjectRegisteredFromPiDirectory", () => { } else { process.env.HOME = oldHome; } + if (oldConfigHome === undefined) delete process.env.XDG_CONFIG_HOME; + else process.env.XDG_CONFIG_HOME = oldConfigHome; + closeQuietly(db); + } + }); + + it("registers the fallback identity when native discovery is unavailable", async () => { + const db = createTestDb(); + const directory = createTestTempDir("pi-embedding-synapse-").dir; + const fakeHome = createTestTempDir("pi-embedding-synapse-home-").dir; + const previous = { + HOME: process.env.HOME, + XDG_CONFIG_HOME: process.env.XDG_CONFIG_HOME, + }; + process.env.HOME = fakeHome; + process.env.XDG_CONFIG_HOME = path.join(fakeHome, ".config"); + resetEmbeddingCacheForTests(); + try { + // Provider and SubC settings are user-tier only. + const configDir = path.join(fakeHome, ".config", "cortexkit"); + await fs.mkdir(configDir, { recursive: true }); + await fs.writeFile( + path.join(configDir, "magic-context.json"), + JSON.stringify({ + embedding: { provider: "synapse", fallback_provider: "off" }, + subc: { connection_file: path.join(fakeHome, "absent-subc.json") }, + }), + ); + const projectIdentity = resolveProjectIdentity(directory); + await ensureProjectRegisteredFromPiDirectory(directory, db); + expect(getProjectEmbeddingSnapshot(projectIdentity)?.provider).toBe( + "off", + ); + expect(getProjectEmbeddingSnapshot(projectIdentity)?.modelId).not.toMatch( + /synapse/u, + ); + } finally { + resetEmbeddingCacheForTests(); + for (const [key, value] of Object.entries(previous)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } closeQuietly(db); } }); diff --git a/packages/pi-plugin/src/embedding-bootstrap.ts b/packages/pi-plugin/src/embedding-bootstrap.ts index 221202bd7..8d1bf88f2 100644 --- a/packages/pi-plugin/src/embedding-bootstrap.ts +++ b/packages/pi-plugin/src/embedding-bootstrap.ts @@ -6,6 +6,7 @@ import { import { type EmbeddingFeatures, registerProjectEmbedding, + registerProjectShadowEmbedding, } from "@magic-context/core/features/magic-context/memory/embedding"; import { resolveProjectIdentityForSession } from "@magic-context/core/features/magic-context/memory/project-identity"; import type { ContextDatabase } from "@magic-context/core/features/magic-context/storage"; @@ -13,6 +14,8 @@ import { handleUntrustedLoad, isConfigLoadUntrusted, } from "@magic-context/core/plugin/embedding-bootstrap-helpers"; +import { resolveEmbeddingRouting } from "@magic-context/core/plugin/embedding-routing"; +import { log } from "@magic-context/core/shared/logger"; import { loadPiConfigDetailed } from "./config"; interface RegistrationFingerprint { @@ -76,6 +79,15 @@ export async function ensureProjectRegisteredFromPiDirectory( return; } + const routing = await resolveEmbeddingRouting({ + config: detailed.config, + projectRoot: directory, + session: `bootstrap:${projectIdentity}`, + }); + for (const warning of routing.warnings) { + log(`[magic-context] ${warning}`); + } + const features: EmbeddingFeatures = { memoryEnabled: detailed.config.memory.enabled, gitCommitEnabled: detailed.config.memory.git_commit_indexing.enabled, @@ -83,16 +95,34 @@ export async function ensureProjectRegisteredFromPiDirectory( registerProjectEmbedding( db, projectIdentity, - detailed.config.embedding, + routing.primary, features, directory, ); - const fingerprintPaths = configCandidatePaths( - directory, - detailed.loadedFromPaths, - ); - registrationFingerprints.set(projectIdentity, { - paths: fingerprintPaths, - fingerprint: configFingerprint(fingerprintPaths), - }); + if (routing.shadow) { + registerProjectShadowEmbedding( + db, + projectIdentity, + routing.shadow, + directory, + ); + } + // Retry discovery on the next registration if a configured lane was unavailable. + const configuredProvider = detailed.config.embedding.provider; + const shadowEnabled = detailed.config.shadow_embedding?.enabled === true; + if ( + routing.primary.provider === configuredProvider && + (configuredProvider === "synapse" || + !shadowEnabled || + routing.shadow !== null) + ) { + const fingerprintPaths = configCandidatePaths( + directory, + detailed.loadedFromPaths, + ); + registrationFingerprints.set(projectIdentity, { + paths: fingerprintPaths, + fingerprint: configFingerprint(fingerprintPaths), + }); + } } From c972f50291937f5c800bf990c70d49534d2dc624 Mon Sep 17 00:00:00 2001 From: Qiiks Date: Thu, 17 Sep 2026 21:07:41 +0530 Subject: [PATCH 2/2] fix(pi): retire stale shadow routing and memoize static fallback --- .../pi-plugin/src/embedding-bootstrap.test.ts | 112 +++++++++++++++++- packages/pi-plugin/src/embedding-bootstrap.ts | 26 ++-- .../magic-context/memory/embedding.ts | 1 + .../project-embedding-registry.ts | 17 +++ 4 files changed, 146 insertions(+), 10 deletions(-) diff --git a/packages/pi-plugin/src/embedding-bootstrap.test.ts b/packages/pi-plugin/src/embedding-bootstrap.test.ts index e6b9997a2..49677ba8a 100644 --- a/packages/pi-plugin/src/embedding-bootstrap.test.ts +++ b/packages/pi-plugin/src/embedding-bootstrap.test.ts @@ -1,13 +1,21 @@ -import { describe, expect, it } from "bun:test"; +import { describe, expect, it, spyOn } from "bun:test"; import { promises as fs } from "node:fs"; import path from "node:path"; -import { getProjectEmbeddingSnapshot } from "@magic-context/core/features/magic-context/memory/embedding"; +import type { EmbeddingConfig } from "@magic-context/core/config/schema/magic-context"; +import { + _resetProjectEmbeddingRegistryForTests, + _setTestProviderFactoryForProject, + getProjectEmbeddingSnapshot, + getShadowEmbeddingMeasurementCohort, + registerProjectShadowEmbedding, +} from "@magic-context/core/features/magic-context/memory/embedding"; import { getProjectEmbeddings, peekProjectEmbeddings, resetEmbeddingCacheForTests, } from "@magic-context/core/features/magic-context/memory/embedding-cache"; import { resolveProjectIdentity } from "@magic-context/core/features/magic-context/memory/project-identity"; +import * as logger from "@magic-context/core/shared/logger"; import { closeQuietly } from "@magic-context/core/shared/sqlite-helpers"; import { createTestTempDir } from "@magic-context/core/shared/test-temp-dir"; @@ -92,4 +100,104 @@ describe("ensureProjectRegisteredFromPiDirectory", () => { closeQuietly(db); } }); + it("retires a disabled shadow without removing the primary lane", async () => { + const db = createTestDb(); + const directory = createTestTempDir("pi-shadow-retirement-").dir; + const configHome = createTestTempDir("pi-shadow-config-").dir; + const previous = process.env.XDG_CONFIG_HOME; + process.env.XDG_CONFIG_HOME = configHome; + let disposed = false; + _setTestProviderFactoryForProject(() => ({ + modelId: "shadow", + initialize: async () => true, + embed: async () => new Float32Array([1, 0]), + embedBatch: async (texts: string[]) => + texts.map(() => new Float32Array([1, 0])), + dispose: async () => { + disposed = true; + }, + isLoaded: () => true, + })); + try { + await fs.mkdir(path.join(configHome, "cortexkit"), { recursive: true }); + await fs.writeFile( + path.join(configHome, "cortexkit", "magic-context.json"), + JSON.stringify({ + embedding: { provider: "off" }, + shadow_embedding: { enabled: false }, + }), + ); + const identity = resolveProjectIdentity(directory); + registerProjectShadowEmbedding( + db, + identity, + { + provider: "synapse", + model: "shadow", + synapse_fingerprint: "fixture", + } as unknown as EmbeddingConfig, + directory, + ); + expect(getShadowEmbeddingMeasurementCohort(identity)?.fingerprint).toBe( + "fixture", + ); + await ensureProjectRegisteredFromPiDirectory(directory, db); + expect(getShadowEmbeddingMeasurementCohort(identity)).toBeNull(); + expect(getProjectEmbeddingSnapshot(identity)?.provider).toBe("off"); + expect(disposed).toBe(true); + } finally { + _resetProjectEmbeddingRegistryForTests(); + if (previous === undefined) delete process.env.XDG_CONFIG_HOME; + else process.env.XDG_CONFIG_HOME = previous; + closeQuietly(db); + } + }); + it("does not repeat a missing-SubC warning until configuration changes", async () => { + const db = createTestDb(); + const directory = createTestTempDir("pi-routing-memo-").dir; + const configHome = createTestTempDir("pi-routing-memo-config-").dir; + const previous = process.env.XDG_CONFIG_HOME; + process.env.XDG_CONFIG_HOME = configHome; + const messages: string[] = []; + const logging = spyOn(logger, "log").mockImplementation((message) => { + messages.push(String(message)); + }); + try { + const configDir = path.join(configHome, "cortexkit"); + await fs.mkdir(configDir, { recursive: true }); + const configFile = path.join(configDir, "magic-context.json"); + await fs.writeFile( + configFile, + JSON.stringify({ + embedding: { provider: "synapse", fallback_provider: "off" }, + }), + ); + await ensureProjectRegisteredFromPiDirectory(directory, db); + await ensureProjectRegisteredFromPiDirectory(directory, db); + expect( + messages.filter((message) => message.includes("requires a subc block")), + ).toHaveLength(1); + await fs.writeFile( + configFile, + JSON.stringify({ + embedding: { provider: "synapse", fallback_provider: "off" }, + subc: { connection_file: path.join(configHome, "missing.json") }, + }), + ); + await ensureProjectRegisteredFromPiDirectory(directory, db); + await ensureProjectRegisteredFromPiDirectory(directory, db); + // A configured but unavailable daemon is retryable, unlike missing configuration. + expect( + messages.filter((message) => + message.startsWith("[magic-context] Synapse is not ready;"), + ), + ).toHaveLength(2); + } finally { + logging.mockRestore(); + _resetProjectEmbeddingRegistryForTests(); + if (previous === undefined) delete process.env.XDG_CONFIG_HOME; + else process.env.XDG_CONFIG_HOME = previous; + closeQuietly(db); + } + }); }); diff --git a/packages/pi-plugin/src/embedding-bootstrap.ts b/packages/pi-plugin/src/embedding-bootstrap.ts index 8d1bf88f2..cdca02734 100644 --- a/packages/pi-plugin/src/embedding-bootstrap.ts +++ b/packages/pi-plugin/src/embedding-bootstrap.ts @@ -7,6 +7,7 @@ import { type EmbeddingFeatures, registerProjectEmbedding, registerProjectShadowEmbedding, + unregisterProjectShadowEmbedding, } from "@magic-context/core/features/magic-context/memory/embedding"; import { resolveProjectIdentityForSession } from "@magic-context/core/features/magic-context/memory/project-identity"; import type { ContextDatabase } from "@magic-context/core/features/magic-context/storage"; @@ -106,16 +107,23 @@ export async function ensureProjectRegisteredFromPiDirectory( routing.shadow, directory, ); + } else { + unregisterProjectShadowEmbedding(projectIdentity); } - // Retry discovery on the next registration if a configured lane was unavailable. + // Only failed daemon discovery can recover without a configuration change. const configuredProvider = detailed.config.embedding.provider; - const shadowEnabled = detailed.config.shadow_embedding?.enabled === true; - if ( - routing.primary.provider === configuredProvider && - (configuredProvider === "synapse" || - !shadowEnabled || - routing.shadow !== null) - ) { + const canDiscover = + Boolean(detailed.config.subc) && + (configuredProvider === "synapse" + ? Boolean(detailed.config.embedding.fallback_provider) + : configuredProvider !== "off" && + detailed.config.shadow_embedding?.enabled === true); + const discoveryFailed = + canDiscover && + (configuredProvider === "synapse" + ? routing.primary.provider !== "synapse" + : routing.shadow === null); + if (!discoveryFailed) { const fingerprintPaths = configCandidatePaths( directory, detailed.loadedFromPaths, @@ -124,5 +132,7 @@ export async function ensureProjectRegisteredFromPiDirectory( paths: fingerprintPaths, fingerprint: configFingerprint(fingerprintPaths), }); + } else { + registrationFingerprints.delete(projectIdentity); } } diff --git a/packages/plugin/src/features/magic-context/memory/embedding.ts b/packages/plugin/src/features/magic-context/memory/embedding.ts index b21463607..533f8a7de 100644 --- a/packages/plugin/src/features/magic-context/memory/embedding.ts +++ b/packages/plugin/src/features/magic-context/memory/embedding.ts @@ -43,6 +43,7 @@ export { type ShadowEmbeddingMeasurementCohort, sweepAllRegisteredProjects, unregisterProjectEmbedding, + unregisterProjectShadowEmbedding, } from "../project-embedding-registry"; const DEFAULT_EMBEDDING_CONFIG: EmbeddingConfig = { diff --git a/packages/plugin/src/features/magic-context/project-embedding-registry.ts b/packages/plugin/src/features/magic-context/project-embedding-registry.ts index ec400dc32..9df6ac756 100644 --- a/packages/plugin/src/features/magic-context/project-embedding-registry.ts +++ b/packages/plugin/src/features/magic-context/project-embedding-registry.ts @@ -2232,6 +2232,23 @@ export function registerProjectInObservationMode( return snapshotFor(registration); } +export function unregisterProjectShadowEmbedding(projectIdentity: string): void { + const shadow = shadowRegistrations.get(projectIdentity); + shadowRegistrations.delete(projectIdentity); + dbForShadowQueue.delete(projectIdentity); + pendingShadowBackfills.delete(projectIdentity); + for (let index = shadowQueue.length - 1; index >= 0; index -= 1) { + if (shadowQueue[index].projectIdentity === projectIdentity) shadowQueue.splice(index, 1); + } + for (const scope of ["memory", "commit", "chunk"] as const) { + const key = `${projectIdentity}:${scope}`; + shadowBackfillLastIds.delete(key); + shadowBackfillStopReasons.delete(key); + shadowBackfillLastWriteOutcomes.delete(key); + } + disposeProvider(shadow?.provider ?? null); +} + export function unregisterProjectEmbedding(projectIdentity: string): void { const prior = projectRegistrations.get(projectIdentity); const shadow = shadowRegistrations.get(projectIdentity);