From 0dc67bb2872d5e5f00cc6355ad07523e1c01a7d2 Mon Sep 17 00:00:00 2001 From: Alex Weininger Date: Thu, 29 Oct 2026 14:47:09 -0700 Subject: [PATCH] Fix production release verification Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- README.md | 6 ++-- scripts/verify-plugin-marketplace.mjs | 52 ++++++++++++++++++--------- test/plugin-marketplace.test.mjs | 33 ++++++++++++++--- test/production-links.test.mjs | 6 ++-- test/storage-release.test.mjs | 7 ++-- 5 files changed, 76 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 9853a1e..2b3a9cb 100644 --- a/README.md +++ b/README.md @@ -36,10 +36,10 @@ the reproducible install points: | --- | --- | --- | | [Azure Functions Hosted Skills](canvases/azure-functions-hosted-skills/) | Build and run local Hosted Skills or invoke supported functions in an existing Function App. | 0.5.5 is tagged; `azure-functions-hosted-skills-latest` points to that release. | | [Azure Resources Query](canvases/azure-resources-query/) | Find and inspect Azure resources with read-only Resource Graph queries. | 0.1.6 candidate narrows inventory activation; `azure-resources-query-latest` remains on 0.1.5 until reviewed promotion. | -| [Canvas Authoring](plugins/canvas-authoring/) | Add toolkit setup and starter apps to the native `create-canvas` workflow. | 0.1.1 is tagged; `canvas-authoring-latest` points to that release. Skill-only plugin, with no canvas panel. | +| [Canvas Authoring](plugins/canvas-authoring/) | Add toolkit setup and starter apps to the native `create-canvas` workflow. | 0.1.2 is tagged; `canvas-authoring-latest` points to that release. Skill-only plugin, with no canvas panel. | | [Azure Cost Health Check](canvases/azure-cost-health-check/) | Review spend, forecasts, budgets, alerts, recommendations, and AI billing. | 0.4.5 is tagged; `azure-cost-health-check-latest` points to that release. | -| [Azure SRE Agent](canvases/azure-sre-agent/) | Connect to your SRE Agent, diagnose failing apps, and investigate incident threads. | 0.2.7 is tagged; `azure-sre-agent-latest` points to that release. | -| [Azure Storage canvas](canvases/azure-storage-canvas/) | Browse blobs, inspect statistics, download files, and plan storage transfers with Copilot. | 0.1.1; immutable and `azure-storage-canvas-latest` install links become usable after the reviewed merge and tag verification. | +| [Azure SRE Agent](canvases/azure-sre-agent/) | Connect to your SRE Agent, diagnose failing apps, and investigate incident threads. | 0.4.2 is tagged; `azure-sre-agent-latest` points to that release. | +| [Azure Storage canvas](canvases/azure-storage/) | Browse blobs, inspect statistics, download files, and plan storage transfers with Copilot. | 0.3.4 is tagged; `azure-storage-latest` points to that release. | | [Containers](canvases/containers/) | Browse and operate local Docker or Podman containers and images. | 0.1.1 candidate; immutable and `containers-latest` install links become usable after the reviewed merge and tag verification. | Native App marketplace installation has not been verified for every product; diff --git a/scripts/verify-plugin-marketplace.mjs b/scripts/verify-plugin-marketplace.mjs index 73ccfb8..7cccbb0 100644 --- a/scripts/verify-plugin-marketplace.mjs +++ b/scripts/verify-plugin-marketplace.mjs @@ -30,6 +30,7 @@ const packages = { manifest: "plugin.json", skills: ["./skills/create-canvas-app/"], receipt: "docs/canvas-authoring/{version}/SHA256SUMS", + protectReferencedDocumentation: true, }, "azure-cost-health-check": { path: "canvases/azure-cost-health-check", @@ -144,7 +145,8 @@ function isMutableDocument(file) { return (inDocDirectory || isReadme) && !noticeName.test(file); } -function referencesMutableDocument(file, content) { +function mutableDocumentReferences(file, content) { + const references = new Set(); const patterns = [ /\b(?:import|export)\s+(?:[^;\n"'`]*?\bfrom\s*)?["'`]([^"'`]+)["'`]/g, /\b(?:import|require|readFileSync|readFile|createReadStream|fetch|new\s+URL)\s*\(\s*["'`]([^"'`]+)["'`]/g, @@ -154,11 +156,12 @@ function referencesMutableDocument(file, content) { for (const match of content.matchAll(pattern)) { const target = match[1].replaceAll("\\", "/"); if (/^[a-z][a-z0-9+.-]*:/i.test(target)) continue; - if (isMutableDocument(posix.normalize(posix.join(posix.dirname(file), target))) || - isMutableDocument(target)) return true; + const resolved = posix.normalize(posix.join(posix.dirname(file), target)); + if (isMutableDocument(resolved)) references.add(resolved); + else if (isMutableDocument(target)) references.add(target); } } - return false; + return [...references]; } function verifyInertDocument(file, content) { @@ -185,7 +188,8 @@ function verifyInertDocument(file, content) { } } -export function verifyMutableDocumentationTrees(current, tagged, readCurrent, readTagged) { +export function verifyMutableDocumentationTrees(current, tagged, readCurrent, readTagged, + { protectReferencedDocumentation = false } = {}) { const pinned = (tree, readFile) => { const files = new Map(); for (const entry of tree) { @@ -194,13 +198,14 @@ export function verifyMutableDocumentationTrees(current, tagged, readCurrent, re throw new Error(`unsafe package file or mode: ${entry.file}`); } files.set(entry.file, entry); + const mutable = isMutableDocument(entry.file); const docPath = /^docs?\//i.test(entry.file) || /(^|\/)README/i.test(entry.file); const protectedReadme = !/^docs?\//i.test(entry.file) && isInertReadmeName(entry.file.split("/").at(-1)); - if (docPath && !isMutableDocument(entry.file) && !protectedReadme) { + if (docPath && !mutable && !protectedReadme) { throw new Error(`executable or legal content cannot hide in mutable documentation: ${entry.file}`); } - if (isMutableDocument(entry.file)) { + if (mutable) { if (entry.mode !== "100644" || !documentExtensions.test(entry.file) && !/^README$/i.test(entry.file)) { throw new Error(`unsafe mutable documentation file: ${entry.file}`); @@ -208,22 +213,36 @@ export function verifyMutableDocumentationTrees(current, tagged, readCurrent, re verifyInertDocument(entry.file, readFile(entry.file)); } } + const protectedDocuments = new Set(); for (const entry of tree) { if (!isMutableDocument(entry.file) && (executablePath.test(entry.file) || /^skills\/.*\/SKILL\.md$/.test(entry.file)) && - /^(?:(?:com\.github\.copilot\/)?extensions|skills)\//.test(entry.file) && - referencesMutableDocument(entry.file, readFile(entry.file).toString("utf8"))) { - throw new Error(`runtime or skill references mutable documentation: ${entry.file}`); + /^(?:(?:com\.github\.copilot\/)?extensions|skills)\//.test(entry.file)) { + const references = mutableDocumentReferences( + entry.file, readFile(entry.file).toString("utf8")); + if (references.length && !protectReferencedDocumentation) { + throw new Error(`runtime or skill references mutable documentation: ${entry.file}`); + } + for (const reference of references) { + if (!files.has(reference)) { + throw new Error(`runtime or skill references missing documentation: ${reference}`); + } + protectedDocuments.add(reference); + } } } - return files; + return { files, protectedDocuments }; }; - const currentFiles = pinned(current, readCurrent); - const taggedFiles = pinned(tagged, readTagged); + const currentPinned = pinned(current, readCurrent); + const taggedPinned = pinned(tagged, readTagged); + const currentFiles = currentPinned.files; + const taggedFiles = taggedPinned.files; const immutableFiles = [...currentFiles.keys()].filter((file) => - file !== "SHA256SUMS" && !isMutableDocument(file)); + file !== "SHA256SUMS" && + (!isMutableDocument(file) || currentPinned.protectedDocuments.has(file))); const taggedImmutableFiles = [...taggedFiles.keys()].filter((file) => - file !== "SHA256SUMS" && !isMutableDocument(file)); + file !== "SHA256SUMS" && + (!isMutableDocument(file) || taggedPinned.protectedDocuments.has(file))); if (immutableFiles.length !== taggedImmutableFiles.length || immutableFiles.some((file) => { const now = currentFiles.get(file); @@ -443,6 +462,7 @@ export function verifyPlugin({ source, name, version }, { candidate = false } = packageTree(releaseTag ?? revision, path), (file) => fileAt(revision, `${path}/${file}`), (file) => fileAt(releaseTag ?? revision, `${path}/${file}`), + { protectReferencedDocumentation: product.protectReferencedDocumentation === true }, ); if (files.includes("release.json")) { verifyRuntimeInventory(JSON.parse(fileAt(releaseTag ?? revision, `${path}/release.json`))); @@ -544,7 +564,7 @@ export function verifyPlugin({ source, name, version }, { candidate = false } = throw new Error("schema 2 checksum receipt must contain only protected plugin files"); } for (const { hash, file } of entries) { - if (isMutableDocument(file)) continue; + if (isMutableDocument(file) && !expectedFiles.includes(file)) continue; const fileRevision = mutableDocumentation ? revision : releaseTag ?? revision; if (createHash("sha256").update(fileAt(fileRevision, `${path}/${file}`)).digest("hex") !== hash) { throw new Error(`plugin file differs from checksum receipt: ${file}`); diff --git a/test/plugin-marketplace.test.mjs b/test/plugin-marketplace.test.mjs index abfe19a..a5a1bb2 100644 --- a/test/plugin-marketplace.test.mjs +++ b/test/plugin-marketplace.test.mjs @@ -199,6 +199,7 @@ test("canvas-authoring uses versioned receipts without changing historical recei }; try { execFileSync("git", ["clone", "--quiet", "--local", "--no-hardlinks", root, clone]); + copyFileSync(join(root, verifier), join(clone, verifier)); git("config", "user.name", "Release policy test"); git("config", "user.email", "release-policy@example.invalid"); const historical = "docs/canvas-authoring/SHA256SUMS"; @@ -231,15 +232,37 @@ test("canvas-authoring uses versioned receipts without changing historical recei manifest.version = product.version; const bytes = `${JSON.stringify(manifest, null, 2)}\n`; writeFileSync(join(clone, manifestPath), bytes); + const skillPath = `${product.source}/skills/create-canvas-app/SKILL.md`; + const skill = `${readFileSync(join(clone, skillPath), "utf8")}\n` + + "[Product-usage telemetry](references/toolkit/README.md#product-usage-telemetry).\n"; + writeFileSync(join(clone, skillPath), skill); const nextReceipt = `docs/canvas-authoring/${product.version}/SHA256SUMS`; mkdirSync(join(clone, `docs/canvas-authoring/${product.version}`), { recursive: true }); - writeFileSync(join(clone, nextReceipt), original.replace( - /^[0-9a-f]{64}(?= plugin.json$)/m, - createHash("sha256").update(bytes).digest("hex"))); - git("add", "--", manifestPath, "docs/canvas-authoring"); - git("commit", "--quiet", "-m", "Prepare next authoring receipt"); + const linkedReadme = "skills/create-canvas-app/references/toolkit/README.md"; + const nextReceiptBytes = original + .replace(/^[0-9a-f]{64}(?= plugin.json$)/m, + createHash("sha256").update(bytes).digest("hex")) + .replace(/^[0-9a-f]{64}(?= skills\/create-canvas-app\/SKILL\.md$)/m, + createHash("sha256").update(skill).digest("hex")); + writeFileSync(join(clone, nextReceipt), nextReceiptBytes); + git("add", "--", manifestPath, skillPath, "docs/canvas-authoring"); + git("commit", "--quiet", "-m", "Prepare linked full-receipt authoring candidate"); assert.doesNotThrow(verify); assert.throws(() => verify(false), /expected exactly one reviewed immutable release tag/); + + const linkedReadmePath = join(clone, product.source, linkedReadme); + const linkedReadmeBytes = readFileSync(linkedReadmePath); + writeFileSync(linkedReadmePath, Buffer.concat([linkedReadmeBytes, Buffer.from("\nChanged after receipt\n")])); + git("add", "--", linkedReadmePath); + git("commit", "--quiet", "-m", "Tamper with linked authoring guide"); + assert.throws(verify, new RegExp(`plugin file differs from checksum receipt: ${linkedReadme}`)); + + writeFileSync(linkedReadmePath, linkedReadmeBytes); + writeFileSync(join(clone, nextReceipt), nextReceiptBytes.split("\n") + .filter((line) => !line.endsWith(` ${linkedReadme}`)).join("\n")); + git("add", "--", linkedReadmePath, nextReceipt); + git("commit", "--quiet", "-m", "Remove linked authoring guide receipt"); + assert.throws(verify, /checksum receipt must cover every protected plugin file exactly once/); } finally { rmSync(checkout, { recursive: true, force: true }); } diff --git a/test/production-links.test.mjs b/test/production-links.test.mjs index ceb2bc4..9084639 100644 --- a/test/production-links.test.mjs +++ b/test/production-links.test.mjs @@ -37,6 +37,7 @@ test("production catalog matches all installable plugins in order", () => { "canvas-authoring": ["Canvas Authoring", "plugins/canvas-authoring/"], "azure-cost-health-check": ["Azure Cost Health Check", "canvases/azure-cost-health-check/"], "azure-sre-agent": ["Azure SRE Agent", "canvases/azure-sre-agent/"], + "azure-storage": ["Azure Storage canvas", "canvases/azure-storage/"], "containers": ["Containers", "canvases/containers/"], }; assert.equal(rows.length, manifest.plugins.length); @@ -67,8 +68,9 @@ test("SRE package has its catalog identity, official preview logo and customer c assert.equal(release.publicationRepository, "microsoft/azure-dev-tools"); assert.equal(release.plugin.extension.entry, "com.github.copilot/extensions/azure-sre-agent/extension.mjs"); - assert.match(readme, /Open an agent by URL or resource ID/); - assert.match(readme, /Save connected agent/); + assert.match(readme, /\*\*External URL or Resource ID\*\*/); + assert.match(readme, /\*\*Favorites\*\*/); + assert.match(readme, /\*\*Connect This Thread\*\*/); assert.match(readme, /My app is failing/); assert.match(readme, /microsoft\/azure-dev-tools/); }); diff --git a/test/storage-release.test.mjs b/test/storage-release.test.mjs index cb22b87..b9bcb1d 100644 --- a/test/storage-release.test.mjs +++ b/test/storage-release.test.mjs @@ -19,13 +19,16 @@ const catalog = JSON.parse(readFileSync( const root = fileURLToPath(new URL("../", import.meta.url)); test("Storage enforces the latest immutable version without pinning future versions", () => { - for (const version of ["0.3.2", "0.3.3", "0.3.4"]) { + const current = catalog.plugins.find((entry) => entry.name === name).version; + const parts = current.split(".").map(Number); + const nextPatch = `${parts[0]}.${parts[1]}.${parts[2] + 1}`; + for (const version of [current, nextPatch]) { assert.doesNotThrow(() => verifyCurrentVersion(name, version)); assert.doesNotThrow(() => verifyTagSource( name, version, `${name}-v${version.replaceAll(".", "-")}-1234567`, )); } - for (const version of ["0.1.0", "0.1.1", "0.2.0", "0.3.1"]) { + for (const version of ["0.1.0", "0.1.1", "0.2.0", "0.3.2"]) { assert.throws(() => verifyCurrentVersion(name, version), /must not precede the latest immutable release/); }