From 436f7f71296e667c3a8abb6f2227667868eb29d0 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Thu, 22 Oct 2026 09:11:17 -0500 Subject: [PATCH 1/2] fix(deps): update document parsers and TLS dependencies --- CHANGELOG.md | 2 + Cargo.lock | 93 ++++++++++++------- Cargo.toml | 12 ++- crates/fetchkit/src/documents/office.rs | 6 +- .../fetchkit/src/documents/pdf_inspector.rs | 43 +++++++++ knowledge/foundations/fetchers.md | 8 ++ knowledge/log.md | 4 + knowledge/security/threat-model.md | 5 +- 8 files changed, 132 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 485ff5d..f04bfbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,6 +60,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Use the patched PDF parser and TLS dependency graph for untrusted document fetching. + - Redact Cloudflare API tokens in `DocumentConversionSettings` Debug output, including pretty formatting, cloned settings and settings retained after build, independently of enabled document backends. diff --git a/Cargo.lock b/Cargo.lock index 235e0f2..7ac7e70 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -165,9 +165,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-lc-rs" -version = "1.17.1" +version = "1.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad" +checksum = "faac5829c2b74c28f830747e7818ccfb684261b5f48a1118b1e2a13d36dfab13" dependencies = [ "aws-lc-sys", "zeroize", @@ -175,9 +175,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.42.0" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444" +checksum = "f1622d8446a2d4b2ce0c7eefc73dd43a99779028d5ee5c2dd8073a658ba8a2bc" dependencies = [ "cc", "cmake", @@ -278,9 +278,9 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "calamine" -version = "0.36.1" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fa68281b1a76b54a62156474adb06bb380a67e07dd60656e3217152b42183f3" +checksum = "c9ca341b94bc4b29e08970b1a03914fad2dfe177e2ecb67fcd6de9046004aafb" dependencies = [ "atoi_simd", "byteorder", @@ -288,7 +288,7 @@ dependencies = [ "encoding_rs", "fast-float2", "log", - "quick-xml", + "quick-xml 0.42.0", "serde", "zip", ] @@ -896,7 +896,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -926,7 +926,7 @@ dependencies = [ "mira-eval", "pdf-inspector", "percent-encoding", - "quick-xml", + "quick-xml 0.41.0", "rakers", "rand 0.10.2", "reqwest", @@ -1158,9 +1158,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.15" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +checksum = "7d29020232d6aa3fb1daca64c1127cf662cf97f254ae16c18c05b8ab635fc118" dependencies = [ "atomic-waker", "bytes", @@ -1696,9 +1696,9 @@ checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "lopdf" -version = "0.41.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67513274c50a2b51e5f75d9e682fcf4ab064a8a9c9ae2c3c59309084882bb24d" +checksum = "25aab26d99567469098e64a02f42679f8965c6401263eefa31d8f2dcc37a221c" dependencies = [ "aes", "bitflags 2.13.0", @@ -1740,8 +1740,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" [[package]] name = "markkit" version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8995a77024f1cd878ba5d07ee850796f4782ad00961aa3f312e79123eaa2b901" +source = "git+https://github.com/everruns/markkit?rev=1405ff748f15ebb2c602e35591f7ed4cc85ec125#1405ff748f15ebb2c602e35591f7ed4cc85ec125" dependencies = [ "anyhow", "async-trait", @@ -1751,7 +1750,7 @@ dependencies = [ "encoding_rs", "futures", "mime", - "quick-xml", + "quick-xml 0.41.0", "regex", "serde", "serde_json", @@ -1883,6 +1882,18 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags 2.13.0", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nom" version = "8.0.0" @@ -1969,9 +1980,9 @@ dependencies = [ [[package]] name = "pdf-inspector" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7475018de0880b394b7cc50f871fac0c010aa411dcd14c64074a3e640a4c05c" +checksum = "fd2f755e49ad38eafbc82ba2bec1c59d57b5bf829b13a8f35e4263bc8913df3a" dependencies = [ "env_logger", "include_dir", @@ -2230,6 +2241,15 @@ name = "quick-xml" version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", +] + +[[package]] +name = "quick-xml" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b" dependencies = [ "encoding_rs", "memchr", @@ -2257,9 +2277,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.16" +version = "0.11.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" dependencies = [ "aws-lc-rs", "bytes", @@ -2289,7 +2309,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2604,14 +2624,14 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] name = "rustls" -version = "0.23.42" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "aws-lc-rs", "log", @@ -2663,7 +2683,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2674,9 +2694,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" dependencies = [ "aws-lc-rs", "ring", @@ -3166,7 +3186,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3744,9 +3764,9 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.8" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267" +checksum = "58ec80e73fd8baf57bf0d9615554c75348bda990d6d5bbeab964dd068319fecb" dependencies = [ "rustls-pki-types", ] @@ -3757,14 +3777,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.8", + "webpki-roots 1.0.10", ] [[package]] name = "webpki-roots" -version = "1.0.8" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" +checksum = "1c1c8e20c8a38fc7d65abfe74eb7280e4aa667e8eec8bc84996d2874810748d9" dependencies = [ "rustls-pki-types", ] @@ -3797,7 +3817,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4092,14 +4112,15 @@ dependencies = [ [[package]] name = "zip" -version = "8.6.0" +version = "9.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" +checksum = "6e04976da629d3b04204ba8105b396a193af8ad2583b2c79bfb3ab8af580a11e" dependencies = [ "crc32fast", "flate2", "indexmap", "memchr", + "nix", "typed-path", "zopfli", ] diff --git a/Cargo.toml b/Cargo.toml index 9295d0b..96363b6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,10 +54,10 @@ async-trait = "0.1" tower = { version = "0.5", features = ["util"] } # Binary document extraction -pdf-inspector = "0.1.7" +pdf-inspector = "0.1.8" markkit = { version = "0.1", default-features = false, features = ["docx", "spreadsheet"] } -calamine = { version = "0.36", default-features = false } -zip = { version = "8", default-features = false, features = ["deflate"] } +calamine = { version = "0.37", default-features = false } +zip = { version = "9", default-features = false, features = ["deflate"] } quick-xml = "0.41" # Platform filesystem flags @@ -71,3 +71,9 @@ ed25519-dalek = { version = "2", features = ["rand_core"] } base64 = "0.22" sha2 = "0.11" rand = "0.10" + +# Temporary source override until the maintainer publishes Markkit's security update. +# Before publishing Fetchkit, require that patched registry version and remove this +# override: dependency patches do not propagate to downstream Cargo consumers. +[patch.crates-io] +markkit = { git = "https://github.com/everruns/markkit", rev = "1405ff748f15ebb2c602e35591f7ed4cc85ec125" } diff --git a/crates/fetchkit/src/documents/office.rs b/crates/fetchkit/src/documents/office.rs index 53bc50c..c3b4b67 100644 --- a/crates/fetchkit/src/documents/office.rs +++ b/crates/fetchkit/src/documents/office.rs @@ -520,8 +520,12 @@ fn slide_parts(archive: &mut Archive<'_>) -> Result, ContentProcesso if let Some(slides) = ordered { return Ok(slides); } - let mut numbered: Vec<(u32, String)> = archive + let names = archive .file_names() + .collect::, _>>() + .map_err(|error| ContentProcessorError(format!("invalid Office archive: {error}")))?; + let mut numbered: Vec<(u32, String)> = names + .into_iter() .filter_map(|name| { let number = name .strip_prefix("ppt/slides/slide")? diff --git a/crates/fetchkit/src/documents/pdf_inspector.rs b/crates/fetchkit/src/documents/pdf_inspector.rs index 2cacc54..dd3cce6 100644 --- a/crates/fetchkit/src/documents/pdf_inspector.rs +++ b/crates/fetchkit/src/documents/pdf_inspector.rs @@ -52,3 +52,46 @@ impl DocumentConverter for PdfInspectorConverter { }) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn deeply_nested_pdf_returns_without_aborting() { + let nested = format!("{}0{}", "[".repeat(10_000), "]".repeat(10_000)); + let objects = [ + format!("<< /Type /Catalog /Pages 2 0 R /Nested {nested} >>"), + "<< /Type /Pages /Kids [] /Count 0 >>".to_owned(), + ]; + let mut body = b"%PDF-1.4\n".to_vec(); + let mut offsets = Vec::new(); + for (index, object) in objects.iter().enumerate() { + offsets.push(body.len()); + body.extend_from_slice(format!("{} 0 obj\n{object}\nendobj\n", index + 1).as_bytes()); + } + let xref = body.len(); + body.extend_from_slice(b"xref\n0 3\n0000000000 65535 f \n"); + for offset in offsets { + body.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes()); + } + body.extend_from_slice( + format!("trailer\n<< /Size 3 /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n").as_bytes(), + ); + + let result = PdfInspectorConverter::new() + .convert(DocumentInput { + url: "https://example.test/nested.pdf".parse().unwrap(), + format: DocumentFormat::Pdf, + content_type: Some("application/pdf".to_owned()), + file_name: "nested.pdf".to_owned(), + body: body.into(), + }) + .await; + // The bounded parser may reject the document or discard its invalid + // nested object. Either outcome must return control rather than abort. + if let Ok(document) = result { + assert!(document.markdown.is_empty()); + } + } +} diff --git a/knowledge/foundations/fetchers.md b/knowledge/foundations/fetchers.md index ec96b20..8400c21 100644 --- a/knowledge/foundations/fetchers.md +++ b/knowledge/foundations/fetchers.md @@ -124,6 +124,10 @@ Built-in converters (Cargo features): - `pdf-inspector`: classifies and extracts Markdown from bounded in-memory bytes; no OCR. OCR-required pages produce `pdf_requires_ocr` and `suggested_next_action: "use_ocr"` unless a later converter returns usable text. + The backend uses `pdf-inspector` 0.1.8 or newer within the 0.1 line, whose `lopdf` + parser bounds nested PDF objects. Deeply nested input must return control to + the caller without aborting; the converter regression exercises that boundary + in memory, allowing rejection or removal of the invalid nested object. - Default registration order: `pdf-inspector`, `markkit`, then `office` when the deprecated `docs-office` feature is enabled, so markkit converts Office formats first and `office` is the fallback and the only PPTX/XLSB backend. `docs-all` @@ -575,3 +579,7 @@ Tests real URLs: - [Fetchkit Tool Contract](tool-contract.md) — shared request, response, and policy behavior - [Threat Model](../security/threat-model.md) — network and SSRF requirements applied to every fetcher + +The temporary root `markkit` patch pins its security-reviewed main revision while a registry release +is pending. Cargo dependency patches apply only to this workspace: before Fetchkit publication, +replace it with the patched registry minimum so downstream users receive the same parser fixes. diff --git a/knowledge/log.md b/knowledge/log.md index 3042f65..07bafc4 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -1,5 +1,9 @@ # Fetchkit Knowledge Update Log +## 2026-10-22 + +* **Parser security**: Require PDF inspector 0.1.8 with bounded nested-object parsing and preserve ZIP 9 Office lookup compatibility. The exact patched Markkit source revision is temporary until its registry release; Fetchkit publication must first replace the root-only override with a patched registry minimum. + ## 2026-10-13 * **Credential redaction**: `DocumentConversionSettings` now redacts present Cloudflare tokens in compact/pretty `Debug`, including clones and after build, independently of backend features. Public string storage remains unchanged; explicit field logging and downstream diagnostic access remain caller responsibilities (TM-LEAK-007). diff --git a/knowledge/security/threat-model.md b/knowledge/security/threat-model.md index 889b3c1..5365c3c 100644 --- a/knowledge/security/threat-model.md +++ b/knowledge/security/threat-model.md @@ -324,7 +324,10 @@ Document converters receive only bodies bounded by `max_document_bytes` (25 MB d are rejected before parsing, extracted output is capped again, and synchronous parsing runs off the async runtime with at most two documents active per process. The blocking task owns its concurrency permit, preventing cancelled callers from starting replacement -parses while non-cancellable work continues. A crafted PDF may still cause expensive +parses while non-cancellable work continues. The default backend requires `pdf-inspector` +0.1.8 / `lopdf` 0.42 or newer to bound nested-object recursion (RUSTSEC-2026-0187); +`deeply_nested_pdf_returns_without_aborting` covers normal return for a 10,000-level +array. A crafted PDF may still cause expensive parser-internal object-stream expansion or prolonged CPU work. Operators handling hostile, high-volume PDFs should isolate the process and apply outer request CPU/memory limits. Accepted because the in-process Rust parser has no reliable From e5916f9665267e0173165370d9c0dcd6725ebff3 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Fri, 23 Oct 2026 16:45:08 -0500 Subject: [PATCH 2/2] chore(release): prepare v0.6.0 --- .github/workflows/publish.yml | 4 +-- .github/workflows/release.yml | 5 +-- CHANGELOG.md | 46 +++++++++++++++++++++++-- Cargo.lock | 27 ++++++++------- Cargo.toml | 12 ++----- crates/fetchkit-cli/Cargo.toml | 2 +- docs/security.md | 19 ++++++++++ knowledge/foundations/fetchers.md | 8 ++--- knowledge/log.md | 4 +++ knowledge/operations/release-process.md | 3 +- 10 files changed, 96 insertions(+), 34 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 510b98b..e26a02d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -53,7 +53,7 @@ jobs: echo "Version verified: $TAG_VERSION" - name: Publish fetchkit to crates.io - run: cargo publish -p fetchkit + run: cargo publish -p fetchkit --locked env: CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} @@ -74,6 +74,6 @@ jobs: run: sleep 30 - name: Publish fetchkit-cli to crates.io - run: cargo publish -p fetchkit-cli + run: cargo publish -p fetchkit-cli --locked env: CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5264812..9545c2c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -36,8 +36,8 @@ jobs: echo "Error: Could not determine version" exit 1 fi - echo "version=$VERSION" >> $GITHUB_OUTPUT - echo "tag=v$VERSION" >> $GITHUB_OUTPUT + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "tag=v$VERSION" >> "$GITHUB_OUTPUT" echo "Detected version: $VERSION" - name: Verify version matches Cargo.toml @@ -60,6 +60,7 @@ jobs: - name: Create GitHub Release uses: softprops/action-gh-release@v2 with: + target_commitish: ${{ github.sha }} tag_name: ${{ steps.version.outputs.tag }} name: Release ${{ steps.version.outputs.tag }} body_path: release_notes.md diff --git a/CHANGELOG.md b/CHANGELOG.md index f04bfbf..8cda56a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.6.0] - 2026-10-23 + +### Highlights + +- Fetch and convert PDF and Office documents with bounded, pluggable local converters. +- Discover agent-facing resources and preserve richer HTML semantics through replaceable content processors. +- Update PDF/TLS dependencies, require published Markkit 0.1.1 and Tokio 1.54.1, and publish from the tested lockfile. + +### Breaking Changes + +- `FetchOptions` gained `document_conversion` and `content_processors`. Add the fields or use `..Default::default()` in struct literals. +- `DefaultFetcher` now stores a processor registry. Replace the unit value `DefaultFetcher` with `DefaultFetcher::new()` or `DefaultFetcher::default()`. +- Default Office conversion now uses Markkit. Enable deprecated `docs-office` explicitly if PPTX/XLSB conversion is required; those formats are not converted by the default backend. +- HTML conversion now uses the content processor registry and may produce richer Markdown. Custom processors constructing `ContentProcessorInput` must populate its output-format and content-focus fields. + ### Added - Replace the specification directory with an indexed OKF v0.2 knowledge @@ -60,7 +75,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- Use the patched PDF parser and TLS dependency graph for untrusted document fetching. +- Use the patched PDF parser and TLS dependency graph for untrusted document fetching; require registry Markkit 0.1.1 and Tokio 1.54.1 (RUSTSEC-2026-0386). - Redact Cloudflare API tokens in `DocumentConversionSettings` Debug output, including pretty formatting, cloned settings and settings retained after build, @@ -70,6 +85,32 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 including the default markkit backend. The deprecated Office renderer admits Markdown bytes before escaping or padding into one capped buffer. +### What's Changed + +* chore(release): prepare v0.6.0 ([#186](https://github.com/everruns/fetchkit/pull/186)) +* fix(documents): redact credentials in settings debug ([#185](https://github.com/everruns/fetchkit/pull/185)) +* fix(documents): bound DOCX table rendering ([#184](https://github.com/everruns/fetchkit/pull/184)) +* feat(documents): markkit as default Office converter, deprecate office ([#183](https://github.com/everruns/fetchkit/pull/183)) +* feat(documents): pluggable document conversion with converter chain ([#182](https://github.com/everruns/fetchkit/pull/182)) +* feat(content): convert Office documents to Markdown locally ([#181](https://github.com/everruns/fetchkit/pull/181)) +* fix(fetch): retain PDF permits until parsing completes ([#180](https://github.com/everruns/fetchkit/pull/180)) +* fix(fetch): enforce URL policy on agent probes ([#179](https://github.com/everruns/fetchkit/pull/179)) +* feat(convert)!: add pluggable HTML content processing ([#178](https://github.com/everruns/fetchkit/pull/178)) +* chore(knowledge): migrate specs to OKF ([#177](https://github.com/everruns/fetchkit/pull/177)) +* feat(fetch): process PDF content as markdown ([#176](https://github.com/everruns/fetchkit/pull/176)) +* chore(ship): prompt for PR evidence in the ship skill ([#175](https://github.com/everruns/fetchkit/pull/175)) +* feat(fetch): discover agent resources ([#174](https://github.com/everruns/fetchkit/pull/174)) +* fix(fetchers): enforce release API URL policy ([#173](https://github.com/everruns/fetchkit/pull/173)) +* fix(fetchers): enforce GitLab API URL policy ([#172](https://github.com/everruns/fetchkit/pull/172)) +* fix(fetchers): validate notebook raw URLs ([#171](https://github.com/everruns/fetchkit/pull/171)) +* fix(fetchers): enforce policy on PubMed API requests ([#170](https://github.com/everruns/fetchkit/pull/170)) +* fix(fetchers): validate Crossref API policy ([#169](https://github.com/everruns/fetchkit/pull/169)) +* fix(fetchers): enforce policy for RFC source URLs ([#168](https://github.com/everruns/fetchkit/pull/168)) +* fix(fetchers): enforce policy for API subrequests ([#167](https://github.com/everruns/fetchkit/pull/167)) +* fix(fetchers): enforce GitHub commit API URL policy ([#166](https://github.com/everruns/fetchkit/pull/166)) + +**Full Changelog**: https://github.com/everruns/fetchkit/compare/v0.5.0...v0.6.0 + ## [0.5.0] - 2026-07-14 ### Highlights @@ -310,7 +351,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 **Full Changelog**: https://github.com/everruns/fetchkit/commits/v0.1.0 -[Unreleased]: https://github.com/everruns/fetchkit/compare/v0.5.0...HEAD +[Unreleased]: https://github.com/everruns/fetchkit/compare/v0.6.0...HEAD +[0.6.0]: https://github.com/everruns/fetchkit/compare/v0.5.0...v0.6.0 [0.5.0]: https://github.com/everruns/fetchkit/compare/v0.4.1...v0.5.0 [0.4.1]: https://github.com/everruns/fetchkit/compare/v0.4.0...v0.4.1 [0.4.0]: https://github.com/everruns/fetchkit/compare/v0.3.0...v0.4.0 diff --git a/Cargo.lock b/Cargo.lock index 7ac7e70..12eddcd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -896,7 +896,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -913,7 +913,7 @@ checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" [[package]] name = "fetchkit" -version = "0.5.0" +version = "0.6.0" dependencies = [ "async-trait", "base64", @@ -948,7 +948,7 @@ dependencies = [ [[package]] name = "fetchkit-cli" -version = "0.5.0" +version = "0.6.0" dependencies = [ "clap", "fetchkit", @@ -959,7 +959,7 @@ dependencies = [ [[package]] name = "fetchkit-python" -version = "0.5.0" +version = "0.6.0" dependencies = [ "fetchkit", "pyo3", @@ -1739,8 +1739,9 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" [[package]] name = "markkit" -version = "0.1.0" -source = "git+https://github.com/everruns/markkit?rev=1405ff748f15ebb2c602e35591f7ed4cc85ec125#1405ff748f15ebb2c602e35591f7ed4cc85ec125" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04c6e21845c0a20a0001b0688c5d3207201ad7daa4208ab3eb25b556533e22cd" dependencies = [ "anyhow", "async-trait", @@ -2309,7 +2310,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2624,7 +2625,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2683,7 +2684,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3186,7 +3187,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3295,9 +3296,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.52.3" +version = "1.54.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +checksum = "b52fd3255258d26fac7264ab5a76b53c13367136dc011d3d6ad4f8ed070396a3" dependencies = [ "bytes", "libc", @@ -3817,7 +3818,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 96363b6..feec92e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ resolver = "2" members = ["crates/*"] [workspace.package] -version = "0.5.0" +version = "0.6.0" edition = "2021" license = "MIT" authors = ["Everruns"] @@ -14,7 +14,7 @@ categories = ["web-programming", "text-processing"] [workspace.dependencies] # Async runtime -tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "sync"] } +tokio = { version = "1.54.1", features = ["rt-multi-thread", "macros", "time", "sync"] } # HTTP client reqwest = { version = "0.13", features = ["gzip", "brotli", "deflate", "stream", "json"] } @@ -55,7 +55,7 @@ tower = { version = "0.5", features = ["util"] } # Binary document extraction pdf-inspector = "0.1.8" -markkit = { version = "0.1", default-features = false, features = ["docx", "spreadsheet"] } +markkit = { version = "0.1.1", default-features = false, features = ["docx", "spreadsheet"] } calamine = { version = "0.37", default-features = false } zip = { version = "9", default-features = false, features = ["deflate"] } quick-xml = "0.41" @@ -71,9 +71,3 @@ ed25519-dalek = { version = "2", features = ["rand_core"] } base64 = "0.22" sha2 = "0.11" rand = "0.10" - -# Temporary source override until the maintainer publishes Markkit's security update. -# Before publishing Fetchkit, require that patched registry version and remove this -# override: dependency patches do not propagate to downstream Cargo consumers. -[patch.crates-io] -markkit = { git = "https://github.com/everruns/markkit", rev = "1405ff748f15ebb2c602e35591f7ed4cc85ec125" } diff --git a/crates/fetchkit-cli/Cargo.toml b/crates/fetchkit-cli/Cargo.toml index 3334aff..647ec30 100644 --- a/crates/fetchkit-cli/Cargo.toml +++ b/crates/fetchkit-cli/Cargo.toml @@ -27,7 +27,7 @@ docs-http = ["fetchkit/docs-http"] docs-all = ["fetchkit/docs-all"] [dependencies] -fetchkit = { path = "../fetchkit", version = "0.5.0" } +fetchkit = { path = "../fetchkit", version = "0.6.0" } tokio = { workspace = true } clap = { workspace = true } serde = { workspace = true } diff --git a/docs/security.md b/docs/security.md index e638dc5..343f362 100644 --- a/docs/security.md +++ b/docs/security.md @@ -3,6 +3,25 @@ Fetchkit is intended to run in agent, server, and cluster environments where URL input may be user-controlled. +## Updating to 0.6.0 + +Fetchkit 0.6.0 requires published Markkit 0.1.1 and Tokio 1.54.1 or later in +their compatible version lines. Its release lockfile also updates the PDF parser +and Rust TLS certificate-validation/root dependencies. Applications with existing +lockfiles should run `cargo update` and audit their resolved dependencies; a +library upgrade alone does not replace every transitive locked version. The default +HTTP client uses the platform certificate store via rustls-platform-verifier; +operators must also keep their OS CA packages current. Cargo root-bundle updates +do not replace system trust stores. Rebuild +installed CLI binaries with `cargo install fetchkit-cli --version 0.6.0 --locked`. + +`FetchOptions` has new `document_conversion` and `content_processors` fields. +Use `..Default::default()` in struct literals when default behavior is desired. +Construct the former unit struct `DefaultFetcher` with `DefaultFetcher::new()`. +Default Office conversion now uses Markkit for DOCX/XLSX/XLS/ODS; opt into the +deprecated `docs-office` feature for PPTX/XLSB support. See the +[release notes](../CHANGELOG.md#060---2026-10-23) for the other accumulated changes. + ## Safe Defaults - Private and reserved IP ranges are blocked by default via resolve-then-check DNS validation. diff --git a/knowledge/foundations/fetchers.md b/knowledge/foundations/fetchers.md index 8400c21..fafe519 100644 --- a/knowledge/foundations/fetchers.md +++ b/knowledge/foundations/fetchers.md @@ -128,6 +128,10 @@ Built-in converters (Cargo features): parser bounds nested PDF objects. Deeply nested input must return control to the caller without aborting; the converter regression exercises that boundary in memory, allowing rejection or removal of the invalid nested object. +Markkit is a registry dependency with a patched minimum of 0.1.1. There is no root-only +Cargo patch: packaged downstream consumers resolve the same published parser source. +Existing downstream lockfiles still need dependency updates for transitive TLS fixes. + - Default registration order: `pdf-inspector`, `markkit`, then `office` when the deprecated `docs-office` feature is enabled, so markkit converts Office formats first and `office` is the fallback and the only PPTX/XLSB backend. `docs-all` @@ -579,7 +583,3 @@ Tests real URLs: - [Fetchkit Tool Contract](tool-contract.md) — shared request, response, and policy behavior - [Threat Model](../security/threat-model.md) — network and SSRF requirements applied to every fetcher - -The temporary root `markkit` patch pins its security-reviewed main revision while a registry release -is pending. Cargo dependency patches apply only to this workspace: before Fetchkit publication, -replace it with the patched registry minimum so downstream users receive the same parser fixes. diff --git a/knowledge/log.md b/knowledge/log.md index 07bafc4..1a240e1 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -1,5 +1,9 @@ # Fetchkit Knowledge Update Log +## 2026-10-23 + +* **Release 0.6.0**: Replace the temporary Markkit git override with registry minimum 0.1.1; require Tokio 1.54.1 for RUSTSEC-2026-0386. Publish both Rust crates with `--locked`. The minor bump reflects public `FetchOptions` changes already accumulated since 0.5.0; see the changelog migration notes. + ## 2026-10-22 * **Parser security**: Require PDF inspector 0.1.8 with bounded nested-object parsing and preserve ZIP 9 Office lookup compatibility. The exact patched Markkit source revision is temporary until its registry release; Fetchkit publication must first replace the root-only override with a patched registry minimum. diff --git a/knowledge/operations/release-process.md b/knowledge/operations/release-process.md index 6ec5e3d..e9ee9e6 100644 --- a/knowledge/operations/release-process.md +++ b/knowledge/operations/release-process.md @@ -104,7 +104,7 @@ cargo build --workspace --exclude fetchkit-python --release - Root `Cargo.toml` on manual dispatch - Verify the extracted version matches root `Cargo.toml` - Extract release notes for that version from `CHANGELOG.md` -- Create a GitHub Release tagged `vX.Y.Z` +- Create a GitHub Release tagged `vX.Y.Z` at the exact triggering commit (`github.sha`) - Start the publish workflow after creating the release. GitHub does not fire the `release.published` event for releases created via the default `GITHUB_TOKEN`, so release.yml dispatches publish.yml explicitly via @@ -119,6 +119,7 @@ cargo build --workspace --exclude fetchkit-python --release - Trigger on GitHub Release `published` - Also support `workflow_dispatch` for manual retries - Verify the release tag version matches root `Cargo.toml` when a release tag is present +- Publish with `--locked` so the verified dependency graph is retained. - Publish crates in dependency order: 1. `fetchkit` 2. `fetchkit-cli`