diff --git a/README.md b/README.md index fd53f3c..f987c4d 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,8 @@ JSON request bodies for both search endpoints are limited to 100 KiB (102,400 by Successful `/search` responses use an API-owned response structure rather than exposing search-backend response objects. The top-level fields are `apiVersion`, `status`, `warnings`, `pagination`, and `records`, in that order. `records` contains complete CVE Records directly, without search-backend hit wrappers or infrastructure metadata. `pagination` contains `page`, `pageSize`, `resultsReturned`, `totalResults`, `totalPages`, and `totalIsExact`. Non-fatal conditions are reported in `warnings` before the potentially long `records` array. +The API omits the top-level `opensearch_ops` ingestion metadata from each `/search` record, including ingestion timestamps and signatures, while preserving all other record fields and containers. This targeted removal does not validate or repair records against the CVE JSON schema and does not modify the indexed data. + CVE Records do not exist for years before 1999. A syntactically valid pre-1999 value in `cveIds` is therefore a valid search that contributes no matches rather than a request error. The response includes a `CVE_IDS_NOT_RETURNED` warning listing those IDs. When pre-1999 and supported-year IDs are requested together, matching supported-year records are still returned. Use the optional `scope` field to control which parts of each record participate in matching. Values are case-insensitive: `FULL` (the default) searches base record fields, CNA data, and every ADP provider; `CNA_ONLY` searches base record fields and CNA data; and `ADP_ONLY` searches base record fields and every ADP provider. Scope changes matching only, so successful responses always contain the complete CVE record. CPE applicability filters use CNA data and return HTTP 400 when combined with `ADP_ONLY`. diff --git a/api-docs/openapi.json b/api-docs/openapi.json index 43f0bec..dee24e4 100644 --- a/api-docs/openapi.json +++ b/api-docs/openapi.json @@ -542,7 +542,7 @@ "CveRecord": { "type": "object", "additionalProperties": true, - "description": "A complete CVE Record without search-backend transport metadata." + "description": "A complete CVE Record without search-backend transport metadata. The top-level opensearch_ops ingestion metadata is omitted; all other record fields and containers are preserved. This targeted removal does not validate or repair the record against the CVE JSON schema." }, "SearchResponse": { "type": "object", diff --git a/controllers/searchController.test.js b/controllers/searchController.test.js index 8319ec4..1ca9710 100644 --- a/controllers/searchController.test.js +++ b/controllers/searchController.test.js @@ -660,12 +660,14 @@ describe('Search controller SEARCH handler', () => { } } }; + const publicRecord = { + cveMetadata: { cveId: 'CVE-2024-0001' }, + containers: { cna: { title: 'complete record' }, adp: [{ title: 'enrichment' }] } + }; + const metadata = { ingest_timestamp: '2026-06-30T17:40:18.109339170Z' }; const fullRecord = { _id: 'CVE-2024-0001', - _source: { - cveMetadata: { cveId: 'CVE-2024-0001' }, - containers: { cna: { title: 'complete record' }, adp: [{ title: 'enrichment' }] } - } + _source: Object.freeze({ ...publicRecord, opensearch_ops: metadata }) }; mockSearchRaw.mockResolvedValue(rawResponse(2, [matching, splitAcrossEntries])); mockApiSearch.mockResolvedValue(advancedResponse(1, [fullRecord])); @@ -691,7 +693,8 @@ describe('Search controller SEARCH handler', () => { ]); expect(res.json.mock.calls[0][0].status).toBe('ok'); expect(res.json.mock.calls[0][0].pagination.totalResults).toBe(1); - expect(res.json.mock.calls[0][0].records).toStrictEqual([fullRecord._source]); + expect(res.json.mock.calls[0][0].records).toStrictEqual([publicRecord]); + expect(fullRecord._source.opensearch_ops).toStrictEqual(metadata); }); test('SEARCH semantically filters date-bounded KEV candidates within the selected scope', async () => { diff --git a/package-lock.json b/package-lock.json index 70c8592..045fcd6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,7 @@ "dotenv": "^6.1.0", "express": "^4.22.2", "express-validator": "^7.2.1", - "morgan": "~1.12.0", + "morgan": "~1.12.1", "swagger-ui-express": "^5.0.1" }, "devDependencies": { @@ -1510,9 +1510,9 @@ } }, "node_modules/adm-zip": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.6.0.tgz", - "integrity": "sha512-XleryMhbuksdKtofnWZ9Sk+4CUTbms4Mb/EU32SZwToAyZ5RgVos/ki8n+yr0LWHOGKuakbXTuuYNHLQjhddgg==", + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.6.1.tgz", + "integrity": "sha512-Xwrja8nx9e5o2N1my4DsKCeKpdrnACyr1wtbPxBDgGzKzKyE9kRtBFA8mWldI+RVlD7CBZNWY/wQ2+ydwOR6kQ==", "license": "MIT", "engines": { "node": ">=14.0" @@ -1617,13 +1617,13 @@ "license": "MIT" }, "node_modules/axios": { - "version": "1.18.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.18.0.tgz", - "integrity": "sha512-E32NzpYKp++W7XRe52rHiXV2ehxmh3wbdgO7MHeFM+vqxLBYHzt0ElkiImtOBxtOmyp0yoC8C6uESVV84Y2/hw==", + "version": "1.20.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.20.0.tgz", + "integrity": "sha512-r8aOh8j9cGKpgQAqpzrUHnSIc6a59Y3Xf/cv8sy1DrHCkZHzQGEuoq1tARk6qSyDdtQGSDgpb9kFlruzPvrgwg==", "license": "MIT", "dependencies": { "follow-redirects": "^1.16.0", - "form-data": "^4.0.5", + "form-data": "^4.0.6", "https-proxy-agent": "^5.0.1", "proxy-from-env": "^2.1.0" } @@ -1852,9 +1852,9 @@ "license": "MIT" }, "node_modules/brace-expansion": { - "version": "5.0.9", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", - "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", + "version": "5.0.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.12.tgz", + "integrity": "sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ==", "license": "MIT", "dependencies": { "balanced-match": "^4.0.2" @@ -4809,9 +4809,9 @@ } }, "node_modules/morgan": { - "version": "1.12.0", - "resolved": "https://registry.npmjs.org/morgan/-/morgan-1.12.0.tgz", - "integrity": "sha512-OHpTRQwn2ezasILW8iKe+Yww1XsfWsZIpUOLF7RDb2g5GwO3trPaRwi7+8BDiJ7HFx2Kg2mfUdCBcVhwYlOz2g==", + "version": "1.12.1", + "resolved": "https://registry.npmjs.org/morgan/-/morgan-1.12.1.tgz", + "integrity": "sha512-tljKC0ex20AjO58Ob/eZ53JloycbVswbVNCHx6V6VLGzqt/w8dIynVGL0G8qVjNKwiA7sYSogCrN6QtJ82IV+g==", "license": "MIT", "dependencies": { "basic-auth": "~2.0.1", @@ -5330,9 +5330,9 @@ } }, "node_modules/proxy-addr": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", - "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.8.tgz", + "integrity": "sha512-5nnx0yGyVUcY6t9RnWcARWtwT9F1D8O9rt08htPvnd49W1IgZtmLkhu9WfMzQj1cFxjHIO6connUNVW5k7AVyQ==", "license": "MIT", "dependencies": { "forwarded": "0.2.0", @@ -5340,6 +5340,10 @@ }, "engines": { "node": ">= 0.10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/proxy-from-env": { diff --git a/package.json b/package.json index abbe9f6..8507465 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "dotenv": "^6.1.0", "express": "^4.22.2", "express-validator": "^7.2.1", - "morgan": "~1.12.0", + "morgan": "~1.12.1", "swagger-ui-express": "^5.0.1" }, "devDependencies": { @@ -32,9 +32,9 @@ "swagger-autogen": "^2.23.7" }, "overrides": { - "adm-zip": "0.6.0", - "axios": "1.18.0", - "brace-expansion": "5.0.9", + "adm-zip": "0.6.1", + "axios": "1.20.0", + "brace-expansion": "5.0.12", "js-yaml": "4.3.2", "lodash": "4.18.1", "lodash.unset": "4.18.0", diff --git a/routes/search.integration.test.js b/routes/search.integration.test.js index 63c3958..8d7ad92 100644 --- a/routes/search.integration.test.js +++ b/routes/search.integration.test.js @@ -55,13 +55,15 @@ describe('search route integration', () => { cveId: 'CVE-2024-0001' } }; + const metadata = { _source_version: '0.1', ingest_timestamp: '2026-06-30T17:40:18.109339170Z' }; + const source = Object.freeze({ ...record, opensearch_ops: metadata }); mockSearchOpenSearch.mockResolvedValue( advancedResponse(1, [ { _index: 'private-index-name', _id: 'CVE-2024-0001', _score: null, - _source: record + _source: source } ]).data ); @@ -122,6 +124,7 @@ describe('search route integration', () => { }, records: [record] }); + expect(source.opensearch_ops).toStrictEqual(metadata); }); test('POST /search builds a scope-aware literal reference URL substring filter', async () => { diff --git a/routes/swagger.js b/routes/swagger.js index 2dcc8c8..f7ab7e3 100644 --- a/routes/swagger.js +++ b/routes/swagger.js @@ -447,7 +447,8 @@ const doc = { CveRecord: { type: 'object', additionalProperties: true, - description: 'A complete CVE Record without search-backend transport metadata.' + description: + 'A complete CVE Record without search-backend transport metadata. The top-level opensearch_ops ingestion metadata is omitted; all other record fields and containers are preserved. This targeted removal does not validate or repair the record against the CVE JSON schema.' }, SearchResponse: { type: 'object', diff --git a/utils/searchResponseBuilder.js b/utils/searchResponseBuilder.js index 04920e6..dd08d36 100644 --- a/utils/searchResponseBuilder.js +++ b/utils/searchResponseBuilder.js @@ -75,7 +75,10 @@ function getRequestWarnings(params = {}) { function buildSearchResponse({ totalResults, hits, pagingOptions, totalIsExact = true, warnings = [] }) { const pageSize = pagingOptions.resultsPerPage; - const records = hits.map((hit) => hit._source); + const records = hits.map((hit) => { + const { opensearch_ops: _ingestionMetadata, ...record } = hit._source; + return record; + }); const page = totalResults === 0 ? 0 : Math.floor(pagingOptions.from / pageSize) + 1; const totalPages = totalResults === 0 ? 0 : Math.ceil(totalResults / pageSize); diff --git a/utils/searchResponseBuilder.test.js b/utils/searchResponseBuilder.test.js index 8e51f4e..89c9a09 100644 --- a/utils/searchResponseBuilder.test.js +++ b/utils/searchResponseBuilder.test.js @@ -43,6 +43,35 @@ describe('search response builder', () => { }); }); + test.each(['PUBLISHED', 'REJECTED'])('omits ingestion metadata without mutating a %s record', (state) => { + const record = { + dataType: 'CVE_RECORD', + dataVersion: '5.1', + cveMetadata: { cveId: 'CVE-2024-0001', state }, + containers: { + cna: { descriptions: [{ lang: 'en', value: 'CNA content' }] }, + adp: [{ descriptions: [{ lang: 'en', value: 'ADP content' }] }] + }, + futureRecordField: { value: 'preserved without a schema allowlist' } + }; + const metadata = { + _source_version: '0.1', + ingest_timestamp: '2026-06-30T17:40:18.109339170Z', + ingested_cve_signature: 'CVE-2024-0001|2026-06-30T17:40:18.109Z|signature' + }; + const source = Object.freeze({ ...record, opensearch_ops: metadata }); + const response = searchResponseBuilder.buildSearchResponse({ + totalResults: 1, + hits: [{ _source: source }], + pagingOptions: { from: 0, resultsPerPage: 20 } + }); + + expect(response.records).toStrictEqual([record]); + expect(response.records[0]).not.toHaveProperty('opensearch_ops'); + expect(response.records[0]).not.toBe(source); + expect(source).toStrictEqual({ ...record, opensearch_ops: metadata }); + }); + test('preserves warnings before records and represents an empty result page', () => { const warnings = [searchResponseBuilder.semanticCandidateLimitWarning()]; const response = searchResponseBuilder.buildSearchResponse({