diff --git a/dashboard/lib/milestones.json b/dashboard/lib/milestones.json index 831900f..932f5f8 100644 --- a/dashboard/lib/milestones.json +++ b/dashboard/lib/milestones.json @@ -202,5 +202,17 @@ "summary": "An experiment first: the same engine as one long-lived server on local NVMe, with the bucket as the durable copy and Postgres wire and Flight SQL protocols, measured against TimescaleDB before anything is built. It also lets TSBS run with its own loader and query runner, for numbers next to published results.", "specs": [43], "issues": [] + }, + { + "number": 18, + "slug": "formats", + "title": "Any table format, any file format", + "status": "planned", + "versions": "", + "updated": "2026-10-21", + "description": "The engine reaches tables and data files through lakekit's boundaries, Iceberg and Parquet by default, Vortex once Iceberg lists it.", + "summary": "The engine stops assuming Iceberg and Parquet: tables go through lakekit's common table view, and data files through its `FileFormat` trait, shared with lakeglass. Iceberg and Parquet stay the defaults, so every engine that reads Timelake tables today still does. Vortex data files come once Iceberg's spec lists the format.", + "specs": [47], + "issues": [] } ] diff --git a/dashboard/lib/spec-pages.ts b/dashboard/lib/spec-pages.ts index a371b32..2fdfb7e 100644 --- a/dashboard/lib/spec-pages.ts +++ b/dashboard/lib/spec-pages.ts @@ -49,6 +49,7 @@ export const SPEC_PAGES: Record Promise<{ default: MDXContent }>> "0044-timebench": () => import("@spec/0044-timebench.md"), "0045-iiot-benchmark": () => import("@spec/0045-iiot-benchmark.md"), "0046-agent-benchmark": () => import("@spec/0046-agent-benchmark.md"), + "0047-format-boundaries": () => import("@spec/0047-format-boundaries.md"), architecture: () => import("@spec/architecture.md"), "deployment-and-cost": () => import("@spec/deployment-and-cost.md"), engine: () => import("@spec/engine.md"), @@ -77,4 +78,5 @@ export const MILESTONE_PAGES: Record Promise<{ default: MDXContent apps: () => import("@spec/milestones/apps.md"), edge: () => import("@spec/milestones/edge.md"), "server-mode": () => import("@spec/milestones/server-mode.md"), + formats: () => import("@spec/milestones/formats.md"), } diff --git a/dashboard/lib/specs.json b/dashboard/lib/specs.json index 1a3c510..6261d10 100644 --- a/dashboard/lib/specs.json +++ b/dashboard/lib/specs.json @@ -596,5 +596,18 @@ "related": [40, 45], "issues": [], "milestones": ["agents"] + }, + { + "number": 47, + "slug": "0047-format-boundaries", + "title": "Table and file formats behind boundaries", + "area": "Tables and storage", + "status": "planned", + "updated": "2026-10-21", + "versions": [], + "summary": "The engine, the WASM reader and writer, and maintenance reach tables and data files through two boundaries from [lakekit](https://github.com/polythings/lakekit): a common table view (Iceberg today) and a data file format (Parquet today). What makes Timelake a time-series engine stays here: the head, the last-value index, dedup on read, newest-first scans, rollups, and the SQL functions. Iceberg and Parquet stay the defaults; another format is a choice a table makes, Vortex first once Iceberg lists it.", + "related": [1, 2, 7, 9, 12, 13, 21, 24], + "issues": [189, 190, 191, 193], + "milestones": ["formats"] } ] diff --git a/spec/0047-format-boundaries.md b/spec/0047-format-boundaries.md new file mode 100644 index 0000000..efdf89c --- /dev/null +++ b/spec/0047-format-boundaries.md @@ -0,0 +1,113 @@ +--- +number: 47 +title: Table and file formats behind boundaries +area: Tables and storage +status: planned +updated: 2026-10-21 +versions: [] +related: [1, 2, 7, 9, 12, 13, 21, 24] +issues: [189, 190, 191, 193] +--- + +# 47. Table and file formats behind boundaries + +**Planned** · updated 2026-10-21 · milestone: [Any table format, any file format](milestones/formats.md) +· related: [spec 1](0001-iceberg-tables.md), [spec 2](0002-commits.md), [spec 7](0007-table-head.md), [spec 9](0009-settings.md), [spec 12](0012-late-data-and-dedup.md), [spec 13](0013-wasm-writes.md), [spec 21](0021-reader.md), [spec 24](0024-in-list-pruning.md) + +## Summary + +The engine, the WASM reader and writer, and maintenance reach tables and data files through two boundaries from +[lakekit](https://github.com/polythings/lakekit): a common table view (Iceberg today) and a data file format (Parquet +today). What makes Timelake a time-series engine stays here: the head, the last-value index, dedup on read, +newest-first scans, rollups, and the SQL functions. Iceberg and Parquet stay the defaults; another format is a choice +a table makes, Vortex first once Iceberg lists it. + +## Design + +**The two boundaries** are lakekit's: + +- **Table formats:** the common table view ([lakekit spec 5](https://github.com/polythings/lakekit/blob/main/spec/0005-table-view.md)). + - It holds a table's schema, properties and snapshots, and its live files with their statistics, partition values, + and commit rank. + - It loads, creates, commits (append, replace, delete) and expires. +- **Data file formats:** `FileFormat` ([lakekit spec 6](https://github.com/polythings/lakekit/blob/main/spec/0006-file-formats.md)). + - Writers return bytes, statistics and a footer. + - Readers take a footer and a selection inside the file, ask for byte ranges, and decode them. + +**What moves out of Timelake:** + +- **The engine's use of iceberg-rust:** + - the adapter for iceberg-rust's file I/O in `storage.rs`; + - loading and creating tables; + - the metadata builder in `commit.rs`; + - snapshot expiry, branches and tags. + + iceberg-rust stays only as the reference implementation in interop tests ([#189](https://github.com/polythings/timelake/issues/189)). +- **`timelake-format`'s Parquet code:** the file writer (`file.rs`), footers (`footer.rs`), and statistics move to + `lakekit-parquet` ([#190](https://github.com/polythings/timelake/issues/190)). + +**What changes in what stays:** + +- **The head** (spec 7) stores each file's footer as opaque bytes with its format's name. This changes the head's + format, so it needs a head version bump (spec 8). +- **Scans** (spec 21): + - The provider picks each file's scan by the format its manifest entry records. + - Parquet's page access plans and the long `IN`-list selections (spec 24) become Parquet's own selection; another + format brings its own pruning. + - The scans come from lakekit's DataFusion providers ([lakekit#11](https://github.com/polythings/lakekit/issues/11)) + once they exist, and from Timelake's provider until then. +- **Dedup on read** (spec 12) orders copies of a key by the view's commit rank. In Iceberg that is the data sequence + number, as today. +- **The block cache** (spec 22) tells which files never change from the table's file list, not from the `.parquet` + extension. +- **The last-value index** (spec 6) stays a Parquet file whatever the data format, so DuckDB and others can read it. + +**Settings** (spec 9): + +- `data-format` chooses the format of new files: `parquet` (the default), and later `vortex`. + - Changing it doesn't rewrite anything at once; compaction rewrites older files as it would anyway. + - A table may hold files of several formats meanwhile. +- The Parquet settings keep their names and become Parquet's options: `compression`, `row-group-rows`, + `float-encoding`, and `data-page-rows`. +- A table's table format is fixed when the table is created: Iceberg, the only one for now. + +**Other engines.** A table of Iceberg and Parquet files stays readable by DuckDB, Spark, Trino and every other +Iceberg reader. +- **Vortex:** until Iceberg's spec lists it ([apache/iceberg#15915](https://github.com/apache/iceberg/pull/15915), + [apache/iceberg-rust#2835](https://github.com/apache/iceberg-rust/pull/2835)), only engines that know Vortex can + read Vortex files. So `vortex` comes off by default, with that said in the docs + ([#191](https://github.com/polythings/timelake/issues/191)). +- **Table formats other than Iceberg** (Delta Lake, DuckLake) follow lakekit's specs for them; each gets a spec here + before Timelake writes it. + +**Order:** + +1. lakekit 0.1 is published and [#186](https://github.com/polythings/timelake/pull/186) is merged. +2. [loko#2](https://github.com/polythings/loko/issues/2) gives `Send` futures, for the engine's multi-threaded executor. +3. Arrow moves to the newest DataFusion's major, as lakekit's D17 has it ([#193](https://github.com/polythings/timelake/issues/193)). DataFusion 56 also brings a native + `ASOF JOIN`, after which Timelake's own (spec 28) can go. +4. The table view: lakekit#13, here #189. +5. File formats: lakekit#14, here #190. +6. Vortex: lakekit#15, here #191. + +## Decisions + +| ID | Decision | Status | +|---|---|---| +| D26 | The engine, the WASM reader and writer, and maintenance reach tables and data files only through lakekit's common table view and `FileFormat`; iceberg-rust remains only as the reference in interop tests (supersedes D13 once built) | Accepted | +| D27 | Iceberg and Parquet stay the defaults, and the last-value index stays Parquet whatever the data format, so other engines keep reading every table that doesn't opt in (D1 holds) | Accepted | +| D28 | A table's data format is a setting (`data-format`) that applies to new files; existing files keep theirs until compaction rewrites them, so a table may hold files of several formats | Proposed | + +## Open questions and issues + +- [#189](https://github.com/polythings/timelake/issues/189) Engine: tables through lakekit's common table view + instead of iceberg-rust +- [#190](https://github.com/polythings/timelake/issues/190) Data files through a `FileFormat` trait, Parquet first +- [#191](https://github.com/polythings/timelake/issues/191) Vortex data files, once Iceberg lists the format +- [#193](https://github.com/polythings/timelake/issues/193) Follow the newest DataFusion's Arrow; on DataFusion 56, + replace our ASOF JOIN with DataFusion's + +## History + +- 2026-10-21: proposed, so that the engine assumes neither Iceberg nor Parquet, and the format code is shared with + lakeglass through lakekit. diff --git a/spec/README.md b/spec/README.md index 25580eb..fc509e8 100644 --- a/spec/README.md +++ b/spec/README.md @@ -50,7 +50,7 @@ Milestones are done, in progress, planned, or exploring (an idea being looked in -46 specs: 19 done, 9 in progress, 18 planned. Milestones group them on the [roadmap](milestones/README.md). +47 specs: 19 done, 9 in progress, 19 planned. Milestones group them on the [roadmap](milestones/README.md). ### Tables and storage @@ -66,6 +66,7 @@ Milestones are done, in progress, planned, or exploring (an idea being looked in | 8 | [Table format versions and upgrades](0008-format-versions.md) | Done | 0.3.0, 0.4.0, 0.5.0 | 2026-10-14 | | 9 | [Settings](0009-settings.md) | Done | 0.5.0, unreleased | 2026-10-20 | | 10 | [Supplemental timestamps](0010-supplemental-timestamps.md) | Planned | – | 2026-10-16 | +| 47 | [Table and file formats behind boundaries](0047-format-boundaries.md) | Planned | – | 2026-10-21 | ### Writing @@ -180,6 +181,9 @@ Every decision has an ID and lives in the spec or page it shapes. | D22 | Descriptive function names with pandas aliases; continuous aggregates are materialized rollups | [28. Function library](0028-function-library.md) | | D23 | Every benchmarked system runs default (as installed) and tuned (its docs' recommendations, precomputation and tuned queries included) | [44. Benchmark harness: timebench](0044-timebench.md) | | D24 | The reader has its own DataFusion table provider, not `iceberg-datafusion` | [Embedded engine](engine.md) | +| D26 | Tables and data files are reached only through lakekit's common table view and `FileFormat`; iceberg-rust only in interop tests | [47. Table and file formats behind boundaries](0047-format-boundaries.md) | +| D27 | Iceberg and Parquet stay the defaults, and the last-value index stays Parquet | [47. Table and file formats behind boundaries](0047-format-boundaries.md) | +| D28 | A table's data format is a setting for new files; a table may hold files of several formats | [47. Table and file formats behind boundaries](0047-format-boundaries.md) | Until 2026-10-19 all of this was one file, `SPEC.md`, with the original plan's phases; on 2026-10-19 the specs were regrouped one per feature, with milestones in place of the hand-written roadmap. Both are in git history. diff --git a/spec/milestones/README.md b/spec/milestones/README.md index bef5909..5bd48e8 100644 --- a/spec/milestones/README.md +++ b/spec/milestones/README.md @@ -60,6 +60,12 @@ Specs: [Hot tier and seconds-fresh reads](../0014-hot-tier.md) (planned), [Inges Specs: [Commits](../0002-commits.md) (done), [Appends](../0011-appends.md) (done), [Maintenance runs, retention and expiry](../0020-maintenance.md) (in progress), [REST catalog](../0032-rest-catalog.md) (planned), [DuckDB extension](../0033-duckdb-extension.md) (planned), [Snowflake connection](../0034-snowflake.md) (planned), [Multitenancy](../0035-multitenancy.md) (planned), [Time-series table contract](../0004-table-contract.md) (in progress), [Supplemental timestamps](../0010-supplemental-timestamps.md) (planned), [Backfill](../0015-backfill.md) (planned), [Import and migration](../0016-import.md) (in progress), [Deletes](../0017-deletes.md) (planned), [GDPR erasure](../0018-gdpr-erasure.md) (planned), [Hot tier and seconds-fresh reads](../0014-hot-tier.md) (planned), [Ingestion and rules](../0036-ingest-and-rules.md) (planned), [Data to events](../0037-data-to-events.md) (planned). +### [Any table format, any file format](formats.md) + +**Planned**. The engine reaches tables and data files through lakekit's boundaries, Iceberg and Parquet by default, Vortex once Iceberg lists it. + +Specs: [Table and file formats behind boundaries](../0047-format-boundaries.md) (planned). + ## Exploring ### [AI agents](agents.md) diff --git a/spec/milestones/formats.md b/spec/milestones/formats.md new file mode 100644 index 0000000..d837eb2 --- /dev/null +++ b/spec/milestones/formats.md @@ -0,0 +1,32 @@ +--- +number: 18 +title: Any table format, any file format +status: planned +updated: 2026-10-21 +description: The engine reaches tables and data files through lakekit's boundaries, Iceberg and Parquet by default, Vortex once Iceberg lists it. +specs: [47] +--- + +# Any table format, any file format + +**Planned** · updated 2026-10-21 + +## Summary + +The engine stops assuming Iceberg and Parquet: tables go through lakekit's common table view, and data files through +its `FileFormat` trait, shared with lakeglass. Iceberg and Parquet stay the defaults, so every engine that reads +Timelake tables today still does. Vortex data files come once Iceberg's spec lists the format. + + + +## Specs + +| # | Spec | Status | Versions | +|---:|---|---|---| +| 47 | [Table and file formats behind boundaries](../0047-format-boundaries.md) | Planned | – | + + + +## History + +- 2026-10-21: milestone proposed.