diff --git a/app/crowdb-access-server/Cargo.toml b/app/crowdb-access-server/Cargo.toml index 54273ec8e..46618583e 100644 --- a/app/crowdb-access-server/Cargo.toml +++ b/app/crowdb-access-server/Cargo.toml @@ -20,6 +20,7 @@ s3 = [ ] [dependencies] +arc-swap = "1.9" hex = "0.4" hmac = "0.12" async-trait = "0.1" @@ -59,7 +60,6 @@ bincode = "1" crowdb-access-server = { path = ".", default-features = false, features = ["test-util"] } futures = "0.3" crowdb-console-shared = { path = "../../lib/crowdb-console-shared" } -arc-swap = "1.9" async-trait = "0.1" crowdb-chunkdb-client = { path = "../../lib/crowdb-chunkdb-client" } crowdb-diskdb-client = { path = "../../lib/crowdb-diskdb-client" } diff --git a/app/crowdb-access-server/src/config.rs b/app/crowdb-access-server/src/config.rs index 779cacbbe..b87eae877 100644 --- a/app/crowdb-access-server/src/config.rs +++ b/app/crowdb-access-server/src/config.rs @@ -206,6 +206,7 @@ pub struct S3Config { #[derive(Clone, Debug, Default, Deserialize, Serialize)] #[serde(default)] pub struct IcebergConfig { + pub locator_signing: Option, /// Overrides the legacy shared small-write policy for Iceberg only. pub small_write: Option, pub listen: Option, @@ -223,6 +224,22 @@ pub struct IcebergConfig { pub gc: IcebergGcConfig, } +#[derive(Clone, Deserialize, Serialize)] +pub struct IcebergLocatorSigningConfig { + pub active_version: u32, + pub keys: std::collections::BTreeMap, +} + +impl std::fmt::Debug for IcebergLocatorSigningConfig { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("IcebergLocatorSigningConfig") + .field("active_version", &self.active_version) + .field("key_versions", &self.keys.keys().collect::>()) + .finish() + } +} + /// Dataset's listener and read-admission limits. Dataset is optional so an /// existing S3/Iceberg deployment keeps its current startup behavior. #[derive(Clone, Debug, Deserialize, Serialize)] diff --git a/app/crowdb-access-server/src/iceberg.rs b/app/crowdb-access-server/src/iceberg.rs index cac8ad8d8..1a33f11cd 100644 --- a/app/crowdb-access-server/src/iceberg.rs +++ b/app/crowdb-access-server/src/iceberg.rs @@ -5,13 +5,17 @@ mod connection; mod file_admission; mod file_auth; mod file_body; +mod file_bucket_request; mod file_complete; mod file_http; mod file_list_request; +pub use file_bucket_request::{parse_file_bucket, FileBucketRequest}; mod file_list_response; +mod file_read_conditions; mod file_recovery; mod file_request; pub use file_list_request::parse_file_list; +pub use file_read_conditions::{file_read_condition, FileReadCondition}; mod file_response; mod file_selection; mod file_upload; diff --git a/app/crowdb-access-server/src/iceberg/file_body.rs b/app/crowdb-access-server/src/iceberg/file_body.rs index 913cdcaa7..c94897627 100644 --- a/app/crowdb-access-server/src/iceberg/file_body.rs +++ b/app/crowdb-access-server/src/iceberg/file_body.rs @@ -108,6 +108,13 @@ pub struct FileReadBody { } impl FileReadBody { + /// # Errors + /// Rejects preflight bytes for a stream or a different file owner/root. + pub fn with_root_directory(mut self, bytes: Vec) -> Result { + let reader = self.reader.take().ok_or(FileIoError::Bounds)?; + self.reader = Some(reader.with_root_directory(bytes)?); + Ok(self) + } fn finish(&mut self) { self.reader = None; self.pending = None; diff --git a/app/crowdb-access-server/src/iceberg/file_bucket_request.rs b/app/crowdb-access-server/src/iceberg/file_bucket_request.rs new file mode 100644 index 000000000..d36a61abf --- /dev/null +++ b/app/crowdb-access-server/src/iceberg/file_bucket_request.rs @@ -0,0 +1,43 @@ +use hyper::{Method, Uri}; + +use super::file_request::{decode, query, FileRequestError}; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum FileBucketRequest { + Head(String), + Location(String), +} + +/// # Errors +/// Rejects unsupported bucket selectors and malformed transport encodings. +pub fn parse_file_bucket(method: &Method, uri: &Uri) -> Result, FileRequestError> { + if uri + .path_and_query() + .map_or(true, |value| value.as_str().len() > 8192) + { + return Err(FileRequestError::Invalid); + } + if uri.path().trim_end_matches('/').split('/').count() != 2 { + return Ok(None); + } + let path = decode(uri.path())?; + let bucket = path.strip_prefix('/').ok_or(FileRequestError::Invalid)?; + let bucket = bucket.strip_suffix('/').unwrap_or(bucket); + if bucket.is_empty() || bucket.contains('/') { + return Ok(None); + } + let mut fields = query(uri.query())?; + if method == Method::HEAD { + if !fields.is_empty() { + return Err(FileRequestError::Unsupported); + } + return Ok(Some(FileBucketRequest::Head(bucket.to_owned()))); + } + if method == Method::GET && fields.remove("location").as_deref() == Some("") { + if !fields.is_empty() { + return Err(FileRequestError::Unsupported); + } + return Ok(Some(FileBucketRequest::Location(bucket.to_owned()))); + } + Ok(None) +} diff --git a/app/crowdb-access-server/src/iceberg/file_http.rs b/app/crowdb-access-server/src/iceberg/file_http.rs index 8b0384118..d43c524e1 100644 --- a/app/crowdb-access-server/src/iceberg/file_http.rs +++ b/app/crowdb-access-server/src/iceberg/file_http.rs @@ -6,7 +6,7 @@ use crowdb_access_iceberg::catalog::{ CatalogError, CatalogLifecycle, CatalogRepository, CatalogStore, RootState, }; use crowdb_access_iceberg::file::{ - resolve_range, FileBlockStore, FileGrantError, FileGrantIssuer, FileOperation, FileRecord, + DirectFileLocations, FileBlockStore, FileGrantError, FileGrantIssuer, FileOperation, FileRecord, FileRepository, FileSealer, MultipartLister, MultipartPartStore, MultipartRepository, RangeError, }; use crowdb_access_iceberg::gc::{GcRepository, GcStore}; @@ -16,8 +16,8 @@ use crowdb_access_s3::auth::{RawAuthRequest, StreamingPayloadVerifier}; use crowdb_access_s3::native_buffer::{NativeBodyAllocator, NativeBodyReceiver}; use crowdb_chunk_client::LargeWritePolicy; use hyper::body::Incoming; -use hyper::http::header::{ACCEPT_RANGES, CONTENT_LENGTH, CONTENT_RANGE, ETAG, RANGE}; -use hyper::{Method, Request, Response, StatusCode}; +use hyper::http::header::{CONTENT_LENGTH, ETAG}; +use hyper::{Method, Request, Response}; use super::body::IcebergBody; use super::file_admission::{FileAdmissionError, FileServiceLimits, FileTransferAdmission}; @@ -28,15 +28,24 @@ use super::file_response::{FileS3ErrorCode, MultipartResponses}; use super::file_upload::FileUploadBudget; use crate::upload_flow::body_encoding::UploadBody; +mod bucket; mod delete; +mod injection; mod listing; mod metrics; mod multipart; +mod read; mod stream; use metrics::UploadFlowMetrics; pub use metrics::UploadFlowSnapshot; +struct CatalogLocators { + catalog: crowdb_access_iceberg::key::CatalogId, + key: Option, + locations: DirectFileLocations, +} + pub(super) struct FileHttp { repository: FileRepository, store: Arc, @@ -45,6 +54,9 @@ pub(super) struct FileHttp { lister: MultipartLister, blocks: Arc, issuer: FileGrantIssuer, + direct_locations: DirectFileLocations, + locator_cache: arc_swap::ArcSwapOption, + locator_keys_configured: bool, responses: FileResponseBudget, uploads: FileUploadBudget, small_threshold_exclusive: usize, @@ -57,6 +69,36 @@ pub(super) struct FileHttp { } impl FileHttp { + pub(super) fn set_direct_locations(&mut self, locations: DirectFileLocations) { + self.direct_locations = locations; + self.locator_keys_configured = true; + self.locator_cache.store(None); + } + + fn catalog_locations( + &self, + authority: &crowdb_access_iceberg::catalog::CatalogAuthority, + ) -> Result, FileS3ErrorCode> { + if let Some(cached) = self.locator_cache.load_full() { + if cached.catalog == authority.catalog && cached.key == authority.locator_key { + return Ok(cached); + } + } + let locations = if let Some(key) = authority.locator_key { + self.direct_locations + .for_catalog(key, self.locator_keys_configured) + .map_err(|_| FileS3ErrorCode::InternalError)? + } else { + self.direct_locations.clone() + }; + let locators = Arc::new(CatalogLocators { + catalog: authority.catalog, + key: authority.locator_key, + locations, + }); + self.locator_cache.store(Some(locators.clone())); + Ok(locators) + } pub(super) fn native_receive_metrics( &self, ) -> Option { @@ -94,6 +136,10 @@ impl FileHttp { return Err(FileGrantError::Invalid); } Ok(Self { + locator_cache: arc_swap::ArcSwapOption::empty(), + locator_keys_configured: false, + direct_locations: DirectFileLocations::new(1, std::collections::BTreeMap::from([(1, secret)])) + .map_err(|_| FileGrantError::Invalid)?, list_tokens: crowdb_access_iceberg::file::FileListTokens::new(secret) .map_err(|_| FileGrantError::Invalid)?, repository: FileRepository::new(store.clone()), @@ -139,6 +185,43 @@ impl FileHttp { Ok(()) } + fn receive_provider( + &self, + request: &mut Request, + operation: FileOperation, + streaming: bool, + ) -> Option> { + let declared_receive = request + .headers() + .get("x-amz-decoded-content-length") + .or_else(|| request.headers().get(CONTENT_LENGTH)) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()); + if matches!(operation, FileOperation::Put | FileOperation::UploadPart) + && !streaming + && !request + .headers() + .get("content-encoding") + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.split(',').any(|part| part.trim() == "aws-chunked")) + { + self.native_allocator.as_ref().and_then(|allocator| { + if let Some(length) = declared_receive + .and_then(|length| usize::try_from(length).ok()) + .filter(|length| *length > 0 && *length < self.small_threshold_exclusive) + { + crate::http_receive::install_small_body_receive_provider(request, allocator, length) + } else { + Some(crate::http_receive::install_native_body_receive_provider( + request, allocator, + )) + } + }) + } else { + None + } + } + pub(super) async fn dispatch( self: &Arc, catalog: &CatalogRepository, @@ -161,6 +244,13 @@ impl FileHttp { mut request: Request, request_timeout: Duration, ) -> Result, FileS3ErrorCode> { + if let Some(probe) = super::file_bucket_request::parse_file_bucket(request.method(), request.uri()) + .map_err(request_error)? + { + return self + .bucket_probe(catalog, &request, &probe, request_timeout) + .await; + } if request.method() == Method::POST && matches!(request.uri().query(), Some("delete" | "delete=")) { return self.delete_objects(catalog, request, request_timeout).await; } @@ -194,43 +284,29 @@ impl FileHttp { let admission = FileTransferAdmission::authorize(&grant, &file_request, self.limits, session.as_ref(), now_ms) .map_err(admission_error)?; - let declared_receive = request - .headers() - .get("x-amz-decoded-content-length") - .or_else(|| request.headers().get(CONTENT_LENGTH)) - .and_then(|value| value.to_str().ok()) - .and_then(|value| value.parse::().ok()); - let native_receiver = if matches!( - file_request.operation, - FileOperation::Put | FileOperation::UploadPart - ) && streaming.is_none() - && !request - .headers() - .get("content-encoding") - .and_then(|value| value.to_str().ok()) - .is_some_and(|value| value.split(',').any(|part| part.trim() == "aws-chunked")) - { - self.native_allocator.as_ref().and_then(|allocator| { - if let Some(length) = declared_receive - .and_then(|length| usize::try_from(length).ok()) - .filter(|length| *length > 0 && *length < self.small_threshold_exclusive) - { - crate::http_receive::install_small_body_receive_provider(&mut request, allocator, length) - } else { - Some(crate::http_receive::install_native_body_receive_provider( - &mut request, - allocator, - )) - } - }) - } else { - None - }; + let native_receiver = + self.receive_provider(&mut request, file_request.operation, streaming.is_some()); + let locators = self.catalog_locations(&authority)?; match file_request.operation { FileOperation::Head | FileOperation::Get => { - self.read(&file_request, &request, root.context, &admission).await + self.read( + &file_request, + &request, + root.context, + &admission, + &locators.locations, + ) + .await } FileOperation::Put => { + if file_request.injection.is_some() { + if streaming.is_some() { + return Err(FileS3ErrorCode::InvalidRequest); + } + return self + .inject(&file_request, request, &admission, now_ms, &locators.locations) + .await; + } self.put( &file_request, request, @@ -341,63 +417,6 @@ impl FileHttp { set_header(&mut response, ETAG, &etag(&published))?; Ok(response) } - - async fn read( - &self, - file_request: &FileRequest, - request: &Request, - context: crowdb_access_iceberg::catalog::CatalogContext, - admission: &FileTransferAdmission, - ) -> Result, FileS3ErrorCode> { - let record = self - .repository - .load(context, &file_request.location) - .await - .map_err(catalog_error)? - .ok_or(FileS3ErrorCode::NoSuchKey)?; - let range = match request - .headers() - .get_all(RANGE) - .iter() - .collect::>() - .as_slice() - { - [] => None, - [value] => Some(value.to_str().map_err(|_| FileS3ErrorCode::InvalidRange)?), - _ => return Err(FileS3ErrorCode::InvalidRange), - }; - let range = resolve_range(range, record.length).map_err(range_error)?; - let bytes = range.map_or(record.length, |range| range.end - range.start); - let body = if request.method() == Method::HEAD { - admission.check_bytes(0, record.length).map_err(admission_error)?; - IcebergBody::new(Vec::new()) - } else { - IcebergBody::file( - admission - .read_body(&self.responses, self.blocks.clone(), record.clone(), range) - .map_err(admission_error)?, - ) - }; - let mut response = Response::new(body); - *response.status_mut() = if range.is_some() { - StatusCode::PARTIAL_CONTENT - } else { - StatusCode::OK - }; - set_header(&mut response, CONTENT_LENGTH, &bytes.to_string())?; - set_header(&mut response, ETAG, &etag(&record))?; - response - .headers_mut() - .insert(ACCEPT_RANGES, hyper::header::HeaderValue::from_static("bytes")); - if let Some(range) = range { - set_header( - &mut response, - CONTENT_RANGE, - &format!("bytes {}-{}/{}", range.start, range.end - 1, record.length), - )?; - } - Ok(response) - } } fn set_header( diff --git a/app/crowdb-access-server/src/iceberg/file_http/bucket.rs b/app/crowdb-access-server/src/iceberg/file_http/bucket.rs new file mode 100644 index 000000000..c6a6ec627 --- /dev/null +++ b/app/crowdb-access-server/src/iceberg/file_http/bucket.rs @@ -0,0 +1,67 @@ +use std::time::Duration; + +use crowdb_access_iceberg::catalog::{CatalogLifecycle, CatalogRepository, RootState}; +use crowdb_access_iceberg::file::{FileOperation, TableLocation}; +use crowdb_access_s3::auth::RawAuthRequest; +use hyper::header::CONTENT_LENGTH; +use hyper::{body::Incoming, Request, Response}; + +use super::{ + authenticate_file_transfer, catalog_error, now_ms, set_header, FileHttp, FileS3ErrorCode, IcebergBody, +}; +use crate::iceberg::FileBucketRequest; + +impl FileHttp { + pub(super) async fn bucket_probe( + &self, + catalog: &CatalogRepository, + request: &Request, + probe: &FileBucketRequest, + request_timeout: Duration, + ) -> Result, FileS3ErrorCode> { + let (root, authority) = catalog.status().await.map_err(catalog_error)?; + if root.state != RootState::Ready + || authority.lifecycle != CatalogLifecycle::Ready + || request_timeout.is_zero() + || request_timeout > Duration::from_millis(authority.admission_bounds.request_ms) + { + return Err(FileS3ErrorCode::SlowDown); + } + let (grant, streaming) = authenticate_file_transfer( + &self.issuer, + root.context, + RawAuthRequest::from_parts(request.method(), request.uri(), request.headers()), + &self.region, + now_ms()?, + ) + .map_err(|_| FileS3ErrorCode::AccessDenied)?; + if streaming.is_some() || !grant.operations.allows(FileOperation::Head) { + return Err(FileS3ErrorCode::AccessDenied); + } + let table = TableLocation { + catalog: grant.context.catalog, + table: grant.table, + storage_namespace: grant.storage_namespace, + }; + let (bucket, location) = match probe { + FileBucketRequest::Head(bucket) => (bucket, false), + FileBucketRequest::Location(bucket) => (bucket, true), + }; + if *bucket != table.bucket() { + return Err(FileS3ErrorCode::AccessDenied); + } + let bytes = if location { + format!("{}", self.region).into_bytes() + } else { + Vec::new() + }; + let length = bytes.len(); + let mut response = Response::new(IcebergBody::new(bytes)); + set_header(&mut response, CONTENT_LENGTH, &length.to_string())?; + response.headers_mut().insert( + "x-amz-bucket-region", + hyper::header::HeaderValue::from_str(&self.region).map_err(|_| FileS3ErrorCode::InternalError)?, + ); + Ok(response) + } +} diff --git a/app/crowdb-access-server/src/iceberg/file_http/delete.rs b/app/crowdb-access-server/src/iceberg/file_http/delete.rs index 425f072b0..26bca9b71 100644 --- a/app/crowdb-access-server/src/iceberg/file_http/delete.rs +++ b/app/crowdb-access-server/src/iceberg/file_http/delete.rs @@ -9,7 +9,7 @@ use crowdb_access_iceberg::gc::{ avro_links, metadata_links, AvroMarkCursor, AvroMarkLimits, GcLimits, ReachableFile, ReachableKind, }; use crowdb_access_iceberg::record::StorageRecord; -use crowdb_access_iceberg::table::head_key; +use crowdb_access_iceberg::table::storage_head_key; use crowdb_access_s3::auth::RawAuthRequest; use http_body_util::BodyExt; use hyper::body::Incoming; @@ -69,6 +69,7 @@ impl FileHttp { return Err(FileS3ErrorCode::AccessDenied); } let bucket = TableLocation { + storage_namespace: grant.storage_namespace, catalog: root.context.catalog, table: grant.table, } @@ -111,6 +112,7 @@ impl FileHttp { return Err(FileS3ErrorCode::InvalidRequest); } let table = TableLocation { + storage_namespace: grant.storage_namespace, catalog: root.context.catalog, table: grant.table, }; @@ -223,7 +225,7 @@ impl FileHttp { context: CatalogContext, table: TableLocation, ) -> Result, FileS3ErrorCode> { - let key = head_key(context.catalog, table.table); + let key = storage_head_key(table); let Some(value) = self .store .get(&key.encode().map_err(|_| FileS3ErrorCode::InternalError)?) diff --git a/app/crowdb-access-server/src/iceberg/file_http/injection.rs b/app/crowdb-access-server/src/iceberg/file_http/injection.rs new file mode 100644 index 000000000..18b81f2d7 --- /dev/null +++ b/app/crowdb-access-server/src/iceberg/file_http/injection.rs @@ -0,0 +1,94 @@ +use crowdb_access_iceberg::{ + file::{FileIdentity, NativeInjectionRepository, NativeInjectionRequest}, + key::FileId, +}; +use hyper::{ + body::Incoming, + header::{CONTENT_LENGTH, CONTENT_TYPE, ETAG}, + Request, Response, +}; + +use super::{ + admission_error, catalog_error, etag, multipart, now_ms, set_header, FileHttp, FileRequest, + FileS3ErrorCode, FileTransferAdmission, IcebergBody, UploadBody, +}; + +impl FileHttp { + pub(super) async fn inject( + &self, + file_request: &FileRequest, + request: Request, + admission: &FileTransferAdmission, + current_ms: u64, + locations: &crowdb_access_iceberg::file::DirectFileLocations, + ) -> Result, FileS3ErrorCode> { + let digest = multipart::signed_digest(request.headers().get("x-amz-content-sha256"))? + .ok_or(FileS3ErrorCode::InvalidRequest)?; + let stem = file_request + .location + .relative_key() + .rsplit('/') + .next() + .and_then(|filename| filename.strip_suffix(".parquet")) + .ok_or(FileS3ErrorCode::InvalidRequest)? + .to_owned(); + let injection = NativeInjectionRequest { + context: admission.context(), + table: file_request.location.table(), + identity: file_request.injection.ok_or(FileS3ErrorCode::InvalidRequest)?, + principal: admission.principal(), + stem, + digest, + }; + let repository = NativeInjectionRepository::new(self.store.clone()); + crowdb_access_iceberg::file::DirectFileLocations::validate_stem(&injection.stem) + .map_err(|_| FileS3ErrorCode::InvalidRequest)?; + let file = if let Some(file) = repository + .replay(&injection, current_ms) + .await + .map_err(catalog_error)? + { + file + } else { + let (parts, incoming) = request.into_parts(); + let mut body = UploadBody::new(incoming, &parts.headers, None, admission.request_byte_limit()) + .map_err(multipart::encoding_error)?; + let owner = FileIdentity { + table: injection.table, + file: FileId::random(), + }; + let length = body.decoded_length(); + let tree = admission + .receive( + &self.uploads, + &mut body, + self.blocks.clone(), + owner, + length, + Some(digest), + ) + .await + .map_err(|error| { + body.failure() + .map_or_else(|| admission_error(error), multipart::encoding_error) + })?; + crowdb_access_iceberg::file::FileSealer::new(self.blocks.clone(), self.limits.max_file_bytes) + .map_err(|_| FileS3ErrorCode::InternalError)? + .seal_uploaded(owner, file_request.location.clone(), tree.clone()) + .await + .map_err(multipart::seal_error)?; + repository + .finish(&injection, owner, tree, locations, now_ms()?) + .await + .map_err(catalog_error)? + }; + let bytes = serde_json::to_vec(&serde_json::json!({"location": file.location.to_string()})) + .map_err(|_| FileS3ErrorCode::InternalError)?; + let length = bytes.len(); + let mut response = Response::new(IcebergBody::new(bytes)); + set_header(&mut response, CONTENT_LENGTH, &length.to_string())?; + set_header(&mut response, CONTENT_TYPE, "application/json")?; + set_header(&mut response, ETAG, &etag(&file))?; + Ok(response) + } +} diff --git a/app/crowdb-access-server/src/iceberg/file_http/read.rs b/app/crowdb-access-server/src/iceberg/file_http/read.rs new file mode 100644 index 000000000..71cce0e6f --- /dev/null +++ b/app/crowdb-access-server/src/iceberg/file_http/read.rs @@ -0,0 +1,110 @@ +use crowdb_access_iceberg::file::{resolve_range, ByteRange}; +use hyper::{ + body::Incoming, + header::{ACCEPT_RANGES, CONTENT_LENGTH, CONTENT_RANGE, ETAG, RANGE}, + Method, Request, Response, StatusCode, +}; + +use super::{ + admission_error, catalog_error, etag, range_error, set_header, FileHttp, FileRequest, FileS3ErrorCode, + FileTransferAdmission, IcebergBody, +}; +use crate::iceberg::file_read_conditions::{file_read_condition, FileReadCondition}; + +impl FileHttp { + pub(super) async fn read( + &self, + file_request: &FileRequest, + request: &Request, + context: crowdb_access_iceberg::catalog::CatalogContext, + admission: &FileTransferAdmission, + locations: &crowdb_access_iceberg::file::DirectFileLocations, + ) -> Result, FileS3ErrorCode> { + let (record, direct) = match locations + .resolve(&file_request.location) + .map_err(|_| FileS3ErrorCode::AccessDenied)? + { + Some(record) => (record, true), + None => ( + self.repository + .load(context, &file_request.location) + .await + .map_err(catalog_error)? + .ok_or(FileS3ErrorCode::NoSuchKey)?, + false, + ), + }; + admission.check_bytes(0, record.length).map_err(admission_error)?; + let root_directory = if direct { + Some( + crowdb_access_iceberg::file::FileReader::probe_root(self.blocks.as_ref(), &record) + .await + .map_err(|_| FileS3ErrorCode::NoSuchKey)?, + ) + } else { + None + }; + let condition = file_read_condition(request.headers(), &etag(&record))?; + let FileReadCondition::Read { use_range } = condition else { + let mut response = Response::new(IcebergBody::new(Vec::new())); + *response.status_mut() = StatusCode::NOT_MODIFIED; + set_header(&mut response, ETAG, &etag(&record))?; + return Ok(response); + }; + let range = requested_range(request, record.length, use_range)?; + let bytes = range.map_or(record.length, |range| range.end - range.start); + let body = if request.method() == Method::HEAD { + admission.check_bytes(0, record.length).map_err(admission_error)?; + IcebergBody::new(Vec::new()) + } else { + let body = admission + .read_body(&self.responses, self.blocks.clone(), record.clone(), range) + .map_err(admission_error)?; + let body = if let Some(bytes) = root_directory { + body.with_root_directory(bytes) + .map_err(|_| FileS3ErrorCode::NoSuchKey)? + } else { + body + }; + IcebergBody::file(body) + }; + let mut response = Response::new(body); + *response.status_mut() = if range.is_some() { + StatusCode::PARTIAL_CONTENT + } else { + StatusCode::OK + }; + set_header(&mut response, CONTENT_LENGTH, &bytes.to_string())?; + set_header(&mut response, ETAG, &etag(&record))?; + response + .headers_mut() + .insert(ACCEPT_RANGES, hyper::header::HeaderValue::from_static("bytes")); + if let Some(range) = range { + set_header( + &mut response, + CONTENT_RANGE, + &format!("bytes {}-{}/{}", range.start, range.end - 1, record.length), + )?; + } + Ok(response) + } +} + +fn requested_range( + request: &Request, + length: u64, + use_range: bool, +) -> Result, FileS3ErrorCode> { + let range = match request + .headers() + .get_all(RANGE) + .iter() + .collect::>() + .as_slice() + { + [] => None, + [value] => Some(value.to_str().map_err(|_| FileS3ErrorCode::InvalidRange)?), + _ => return Err(FileS3ErrorCode::InvalidRange), + }; + resolve_range(if use_range { range } else { None }, length).map_err(range_error) +} diff --git a/app/crowdb-access-server/src/iceberg/file_list_request.rs b/app/crowdb-access-server/src/iceberg/file_list_request.rs index d7c59c7d7..cfec9a149 100644 --- a/app/crowdb-access-server/src/iceberg/file_list_request.rs +++ b/app/crowdb-access-server/src/iceberg/file_list_request.rs @@ -2,6 +2,7 @@ use super::file_request::{decode, query, FileRequestError}; use crowdb_access_iceberg::file::{FileListRequest, TableLocation}; +use crowdb_access_iceberg::key::StorageTablePath; use hyper::{Method, Uri}; /// Parses `ListObjectsV2` independently of exact-object `FileIO`. @@ -28,7 +29,12 @@ pub fn parse_file_list(method: &Method, uri: &Uri) -> Result Result { + if let Some(matches) = condition(headers, IF_MATCH, etag, false)? { + if !matches { + return Err(FileS3ErrorCode::PreconditionFailed); + } + } + if condition(headers, IF_NONE_MATCH, etag, true)? == Some(true) { + return Ok(FileReadCondition::NotModified); + } + let use_range = match headers.get_all(IF_RANGE).iter().collect::>().as_slice() { + [] => true, + [value] => value.to_str().map_err(|_| FileS3ErrorCode::InvalidRequest)? == etag, + _ => return Err(FileS3ErrorCode::InvalidRequest), + }; + Ok(FileReadCondition::Read { use_range }) +} + +fn condition( + headers: &HeaderMap, + name: hyper::header::HeaderName, + etag: &str, + weak: bool, +) -> Result, FileS3ErrorCode> { + let mut result = None; + let mut count = 0; + let mut bytes = 0; + for value in headers.get_all(name) { + let value = value.to_str().map_err(|_| FileS3ErrorCode::InvalidRequest)?; + bytes += value.len(); + if bytes > 4096 { + return Err(FileS3ErrorCode::InvalidRequest); + } + for tag in value.split(',') { + count += 1; + let tag = tag.trim(); + if count > 64 { + return Err(FileS3ErrorCode::InvalidRequest); + } + let comparison = if weak { + tag.strip_prefix("W/").unwrap_or(tag) + } else { + tag + }; + let syntax = tag.strip_prefix("W/").unwrap_or(tag); + if syntax != "*" + && (syntax.len() < 2 + || !syntax.starts_with('"') + || !syntax.ends_with('"') + || syntax[1..syntax.len() - 1] + .bytes() + .any(|byte| byte == b'"' || byte <= 0x20 || byte == 0x7f)) + { + return Err(FileS3ErrorCode::InvalidRequest); + } + result = Some(result.unwrap_or(false) || comparison == "*" || comparison == etag); + } + } + Ok(result) +} diff --git a/app/crowdb-access-server/src/iceberg/file_request.rs b/app/crowdb-access-server/src/iceberg/file_request.rs index b3507620a..283432ceb 100644 --- a/app/crowdb-access-server/src/iceberg/file_request.rs +++ b/app/crowdb-access-server/src/iceberg/file_request.rs @@ -25,6 +25,7 @@ pub enum MultipartRequest { #[derive(Clone, Debug, Eq, PartialEq)] pub struct FileRequest { + pub injection: Option, pub location: FileLocation, pub operation: FileOperation, pub multipart: Option, @@ -50,11 +51,27 @@ impl FileRequest { } let path = decode(uri.path())?; let path = path.strip_prefix('/').ok_or(FileRequestError::Invalid)?; - let location = format!("s3://{path}") + let location: FileLocation = format!("s3://{path}") .parse() .map_err(|_| FileRequestError::Invalid)?; let mut query = query(uri.query())?; + let injection = query + .remove("crowdb-native-upload") + .map(|identity| { + if method != Method::PUT + || location.table().storage_namespace.is_none() + || !location.relative_key().ends_with(".parquet") + { + return Err(FileRequestError::Invalid); + } + crowdb_access_iceberg::operation::RequestIdentity::parse(&identity, u64::MAX) + .map_err(|_| FileRequestError::Invalid) + }) + .transpose()?; let multipart = multipart(method, &mut query)?; + if injection.is_some() && multipart.is_some() { + return Err(FileRequestError::Invalid); + } if !query.is_empty() { return Err(FileRequestError::Unsupported); } @@ -70,7 +87,14 @@ impl FileRequest { None if method == Method::DELETE => FileOperation::DeleteObject, None => return Err(FileRequestError::Unsupported), }; + if location.table().storage_namespace.is_some() + && matches!(operation, FileOperation::Put | FileOperation::CreateMultipart) + && (location.relative_key() == "f" || location.relative_key().starts_with("f/")) + { + return Err(FileRequestError::Invalid); + } Ok(Self { + injection, location, operation, multipart, diff --git a/app/crowdb-access-server/src/iceberg/file_response.rs b/app/crowdb-access-server/src/iceberg/file_response.rs index b4d858990..77d7d344f 100644 --- a/app/crowdb-access-server/src/iceberg/file_response.rs +++ b/app/crowdb-access-server/src/iceberg/file_response.rs @@ -33,6 +33,7 @@ pub enum FileS3ErrorCode { Conflict, BadDigest, EntityTooSmall, + PreconditionFailed, } impl FileS3ErrorCode { @@ -48,6 +49,7 @@ impl FileS3ErrorCode { Self::InvalidRange => StatusCode::RANGE_NOT_SATISFIABLE, Self::SlowDown => StatusCode::SERVICE_UNAVAILABLE, Self::Conflict => StatusCode::CONFLICT, + Self::PreconditionFailed => StatusCode::PRECONDITION_FAILED, } } @@ -65,6 +67,7 @@ impl FileS3ErrorCode { Self::Conflict => "The immutable object already exists with different content", Self::BadDigest => "The supplied digest does not match the uploaded content", Self::EntityTooSmall => "A nonfinal upload part is smaller than 5 MiB", + Self::PreconditionFailed => "An object precondition was not satisfied", } } @@ -82,6 +85,7 @@ impl FileS3ErrorCode { Self::Conflict => "OperationAborted", Self::BadDigest => "BadDigest", Self::EntityTooSmall => "EntityTooSmall", + Self::PreconditionFailed => "PreconditionFailed", } } } diff --git a/app/crowdb-access-server/src/iceberg/gc_control.rs b/app/crowdb-access-server/src/iceberg/gc_control.rs index bc22a3e82..5f0cc2582 100644 --- a/app/crowdb-access-server/src/iceberg/gc_control.rs +++ b/app/crowdb-access-server/src/iceberg/gc_control.rs @@ -5,10 +5,10 @@ use crowdb_access_iceberg::{ CatalogContext, CatalogRepository, CatalogStore, ManagementPrivilege, RootState, RoutedCatalogStore, }, gc::{GcLimits, GcRepository, GcTask}, - key::{CatalogId, OperationId, TableId}, + key::{CatalogId, OperationId, StorageTablePath, TableId}, operation::{ManagementAction, ManagementPhase}, record::StorageRecord, - table::{head_key, TableHead, TableLifecycle}, + table::{storage_head_key, TableHead, TableLifecycle}, wire::BearerAuthenticator, }; @@ -88,13 +88,28 @@ async fn start_table( table: &str, limits: GcLimits, ) -> Result<(), BoxError> { - let (root, _) = catalog.status().await?; + let (root, authority) = catalog.status().await?; if root.state != RootState::Ready { return Err("catalog is not ready".into()); } - let table: TableId = table.parse()?; + let table = match authority.storage_layout { + crowdb_access_iceberg::catalog::CatalogStorageLayout::Native => { + let path = StorageTablePath::from_prefix(&format!("{}/", table.trim_end_matches('/')))?; + if path.catalog != root.context.catalog { + return Err("table storage path belongs to another catalog".into()); + } + path.into() + } + crowdb_access_iceberg::catalog::CatalogStorageLayout::Legacy => { + crowdb_access_iceberg::file::TableLocation { + catalog: root.context.catalog, + table: table.parse::()?, + storage_namespace: None, + } + } + }; let identity: OperationId = identity.parse()?; - let head = load_head(store, root.context.catalog, table).await?; + let head = load_head(store, table).await?; if head.lifecycle != TableLifecycle::Tombstone { return Err("live-table GC is disabled; only tombstoned tables can be reclaimed".into()); } @@ -151,10 +166,9 @@ async fn start_retired( async fn load_head( store: &RoutedCatalogStore, - catalog: CatalogId, - table: TableId, + table: crowdb_access_iceberg::file::TableLocation, ) -> Result { - let key = head_key(catalog, table); + let key = storage_head_key(table); let value = store.get(&key.encode()?).await?.ok_or("table head is missing")?; let StorageRecord::TableHead(head) = StorageRecord::decode(&key, &value.bytes)? else { return Err("table head has an invalid record".into()); diff --git a/app/crowdb-access-server/src/iceberg/gc_runtime.rs b/app/crowdb-access-server/src/iceberg/gc_runtime.rs index dc00ee18c..1c32bb617 100644 --- a/app/crowdb-access-server/src/iceberg/gc_runtime.rs +++ b/app/crowdb-access-server/src/iceberg/gc_runtime.rs @@ -286,6 +286,7 @@ async fn scan_and_advance( }); } let scan = GcScan { + native: false, catalog, scope: Some(CatalogScope::GcTask), prefix: Vec::new(), @@ -388,6 +389,7 @@ async fn scan_purge( limits: GcLimits, ) -> Result, Box> { let scan = GcScan { + native: false, catalog: context.catalog, scope: Some(CatalogScope::Reclamation), prefix: Vec::new(), diff --git a/app/crowdb-access-server/src/iceberg/http.rs b/app/crowdb-access-server/src/iceberg/http.rs index c2a869048..d855dcd58 100644 --- a/app/crowdb-access-server/src/iceberg/http.rs +++ b/app/crowdb-access-server/src/iceberg/http.rs @@ -107,6 +107,23 @@ impl IcebergHttpService { Ok(self) } + /// Installs persistent locator keys, including retained older key versions. + /// # Errors + /// Rejects configuration after the file service has been shared. + pub fn with_direct_file_locations( + mut self, + locations: crowdb_access_iceberg::file::DirectFileLocations, + ) -> Result { + Arc::get_mut( + self.files + .as_mut() + .ok_or(crowdb_access_iceberg::file::FileGrantError::Invalid)?, + ) + .ok_or(crowdb_access_iceberg::file::FileGrantError::Invalid)? + .set_direct_locations(locations); + Ok(self) + } + /// Applies the configured shared-chunk routing threshold to native file uploads. /// # Errors /// Rejects an unavailable file service or an invalid threshold. @@ -250,7 +267,10 @@ impl IcebergHttpService { &self, request: Request, ) -> Result, IcebergErrorResponse> { - if request.uri().path().starts_with("/iceberg-") { + if request.uri().path().starts_with("/iceberg-") + || request.uri().path() == "/iceberg" + || request.uri().path().starts_with("/iceberg/") + { return Ok(match &self.files { Some(files) => { Box::pin(files.dispatch(&self.repository, request, self.request_timeout)).await diff --git a/app/crowdb-access-server/src/iceberg/runtime.rs b/app/crowdb-access-server/src/iceberg/runtime.rs index 38a285dad..60195ebdb 100644 --- a/app/crowdb-access-server/src/iceberg/runtime.rs +++ b/app/crowdb-access-server/src/iceberg/runtime.rs @@ -179,6 +179,13 @@ async fn start_listener( )? .with_small_object_threshold(access_config.iceberg_small_write().threshold_exclusive())? .with_large_write_policy(large_write)?; + if let Some(signing) = &access_config.iceberg.locator_signing { + service = + service.with_direct_file_locations(crowdb_access_iceberg::file::DirectFileLocations::new( + signing.active_version, + signing.keys.clone(), + )?)?; + } if authority.admission_bounds.delegated_access_ms >= 900_000 { let endpoint = std::env::var("CROWDB_ICEBERG_PUBLIC_URI").unwrap_or_else(|_| format!("http://{address}")); diff --git a/app/crowdb-access-server/src/iceberg/table_credentials.rs b/app/crowdb-access-server/src/iceberg/table_credentials.rs index c566420ea..11b3c434b 100644 --- a/app/crowdb-access-server/src/iceberg/table_credentials.rs +++ b/app/crowdb-access-server/src/iceberg/table_credentials.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeMap, sync::Arc}; +use std::{collections::BTreeMap, fmt::Write, sync::Arc}; use crowdb_access_iceberg::{ catalog::{ @@ -7,9 +7,9 @@ use crowdb_access_iceberg::{ }, commit::{TableCreateJournal, TableCreatePhase}, file::FileGrantIssuer, - key::{OperationId, TableId}, + key::{NamespaceId, OperationId, TableId}, namespace::{NamespaceIdentifier, NamespaceRepository, NamespaceStore}, - table::TableRepository, + table::{storage_head_key, TableLifecycle, TableRepository}, wire::{ FileDelegationLimits, FileDelegationTarget, IcebergErrorResponse, LoadCredentialsResponse, Principal, }, @@ -38,6 +38,7 @@ pub(super) struct TableFileConfig { #[derive(Clone, Copy)] pub(super) struct TableFileTarget<'a> { + pub(super) storage_namespace: Option, pub(super) namespace: &'a NamespaceIdentifier, pub(super) name: &'a str, pub(super) table: TableId, @@ -63,6 +64,19 @@ impl TableFileConfig { let endpoint = request_host.and_then(|host| self.endpoint_for_host(host)); let mut properties = self.properties(target.namespace, target.name, target.table, endpoint.as_deref()); + if target.storage_namespace.is_some() { + properties.insert("crowdb.native-injection", "true".into()); + properties.insert( + "crowdb.native-injection-max-bytes", + (1024_u64 * 1024 * 1024).to_string(), + ); + if let Some(namespace) = target.storage_namespace { + let endpoint = properties + .get_mut("client.refresh-credentials-endpoint") + .ok_or_else(service_unavailable)?; + write!(endpoint, "&storage-namespace-id={namespace}").expect("string writes do not fail"); + } + } let credentials = FileDelegationLimits { ttl_ms: 900_000, max_request_bytes: 1024 * 1024 * 1024, @@ -74,6 +88,7 @@ impl TableFileConfig { access.context, access.authority, FileDelegationTarget { + storage_namespace: target.storage_namespace, table: target.table, format_version: target.version, staged: target.staged, @@ -181,21 +196,19 @@ impl TableCredentials { principal: Principal, request: &Request, ) -> Result, IcebergErrorResponse> { - if request.method() != hyper::Method::GET { - return Err(super::table_read::unsupported()); - } - let path = request - .uri() - .path() - .strip_suffix("/credentials") - .ok_or_else(bad_request)?; - let target = super::table_write::request::parse(&path.parse().map_err(|_| bad_request())?)?; - let name = target.name.ok_or_else(bad_request)?; - let (selector, cleanup) = selector(request.uri().query())?; + let (namespace, name) = credential_target(request)?; + let (selector, storage_namespace, cleanup) = selector(request.uri().query())?; let now = now_ms()?; + if let (Some(table), Some(namespace)) = (selector, storage_namespace) { + if let Some(target) = self.native_target(context, table, namespace, cleanup).await? { + return self + .issue_response(repository, context, principal, target, 900_000, now, cleanup) + .await; + } + } let parent = self .namespaces - .load(context, &target.namespace) + .load(context, &namespace) .await .map_err(|_| service_unavailable())? .ok_or_else(missing_table)?; @@ -205,14 +218,19 @@ impl TableCredentials { .await .map_err(|_| service_unavailable())?; let mut ttl_ms = 900_000; - let (table, version, staged) = if let Some(selected) = + let (table, version, staged, storage_namespace) = if let Some(selected) = selected.filter(|selected| selector.map_or(true, |table| table == selected.head.table)) { self.tables .ensure_current(context, &selected) .await .map_err(|_| service_unavailable())?; - (selected.head.table, selected.head.format_version, false) + ( + selected.head.table, + selected.head.format_version, + false, + selected.head.metadata_location.table().storage_namespace, + ) } else { let table = selector.ok_or_else(missing_table)?; let identity = OperationId::from_bytes(table.as_bytes()).map_err(|_| bad_request())?; @@ -225,11 +243,14 @@ impl TableCredentials { let stage = operation.stage.as_ref().ok_or_else(missing_table)?; if !principal.namespace_write || operation.principal != principal.name - || operation.namespace != target.namespace + || operation.namespace != namespace || operation.candidate.namespace != parent.namespace || operation.candidate.name != name || operation.candidate.table != table || operation.phase != TableCreatePhase::Staged + || storage_namespace.is_some_and(|namespace| { + operation.candidate.storage_location().storage_namespace != Some(namespace) + }) { return Err(missing_table()); } @@ -249,13 +270,19 @@ impl TableCredentials { { return Err(service_unavailable()); } - (table, operation.candidate.format_version, true) + ( + table, + operation.candidate.format_version, + true, + operation.candidate.metadata_location.table().storage_namespace, + ) }; self.issue_response( repository, context, principal, FileDelegationTarget { + storage_namespace, table, format_version: version, staged, @@ -267,6 +294,45 @@ impl TableCredentials { .await } + async fn native_target( + &self, + context: CatalogContext, + table: TableId, + namespace: NamespaceId, + cleanup: bool, + ) -> Result, IcebergErrorResponse> { + let key = storage_head_key(crowdb_access_iceberg::file::TableLocation { + catalog: context.catalog, + storage_namespace: Some(namespace), + table, + }); + let Some(value) = self + .store + .get(&key.encode().map_err(|_| bad_request())?) + .await + .map_err(|_| service_unavailable())? + else { + return Ok(None); + }; + let crowdb_access_iceberg::record::StorageRecord::TableHead(head) = + crowdb_access_iceberg::record::StorageRecord::decode(&key, &value.bytes) + .map_err(|_| service_unavailable())? + else { + return Err(service_unavailable()); + }; + if head.lifecycle != TableLifecycle::Ready + && !(cleanup && head.lifecycle == TableLifecycle::Tombstone) + { + return Err(missing_table()); + } + Ok(Some(FileDelegationTarget { + table, + storage_namespace: Some(namespace), + format_version: head.format_version, + staged: false, + })) + } + #[allow(clippy::too_many_arguments)] async fn issue_response( &self, @@ -310,8 +376,11 @@ impl TableCredentials { } } -fn selector(query: Option<&str>) -> Result<(Option, bool), IcebergErrorResponse> { +fn selector( + query: Option<&str>, +) -> Result<(Option, Option, bool), IcebergErrorResponse> { let mut selector = None; + let mut namespace = None; let mut cleanup = false; for pair in query .unwrap_or_default() @@ -323,9 +392,31 @@ fn selector(query: Option<&str>) -> Result<(Option, bool), IcebergError "table-id" if selector.is_none() => { selector = Some(decode_query(value)?.parse().map_err(|_| bad_request())?); } + "storage-namespace-id" if namespace.is_none() => { + namespace = Some(decode_query(value)?.parse().map_err(|_| bad_request())?); + } "cleanup" if !cleanup && decode_query(value)? == "true" => cleanup = true, _ => return Err(bad_request()), } } - Ok((selector, cleanup)) + if namespace.is_some() && selector.is_none() { + return Err(bad_request()); + } + Ok((selector, namespace, cleanup)) +} + +fn credential_target( + request: &Request, +) -> Result<(NamespaceIdentifier, String), IcebergErrorResponse> { + if request.method() != hyper::Method::GET { + return Err(super::table_read::unsupported()); + } + let path = request + .uri() + .path() + .strip_suffix("/credentials") + .ok_or_else(bad_request)?; + let target = super::table_write::request::parse(&path.parse().map_err(|_| bad_request())?)?; + let name = target.name.ok_or_else(bad_request)?; + Ok((target.namespace, name)) } diff --git a/app/crowdb-access-server/src/iceberg/table_read.rs b/app/crowdb-access-server/src/iceberg/table_read.rs index 5a55ecaf2..1300f38ec 100644 --- a/app/crowdb-access-server/src/iceberg/table_read.rs +++ b/app/crowdb-access-server/src/iceberg/table_read.rs @@ -207,6 +207,7 @@ impl TableHttp { config.append( &mut bytes, super::table_credentials::TableFileTarget { + storage_namespace: head.metadata_location.table().storage_namespace, namespace, name, table: head.table, diff --git a/app/crowdb-access-server/src/iceberg/table_write.rs b/app/crowdb-access-server/src/iceberg/table_write.rs index 6c607fe09..717c50cc4 100644 --- a/app/crowdb-access-server/src/iceberg/table_write.rs +++ b/app/crowdb-access-server/src/iceberg/table_write.rs @@ -258,6 +258,7 @@ impl TableWrites { config.append( &mut body, super::table_credentials::TableFileTarget { + storage_namespace: table.storage_namespace, namespace: &namespace, name: &name, table: table.table, diff --git a/app/crowdb-access-server/tests/common/iceberg_commit_fault.rs b/app/crowdb-access-server/tests/common/iceberg_commit_fault.rs index 229e812b6..603c8b6db 100644 --- a/app/crowdb-access-server/tests/common/iceberg_commit_fault.rs +++ b/app/crowdb-access-server/tests/common/iceberg_commit_fault.rs @@ -9,7 +9,7 @@ use crowdb_access_iceberg::{ catalog::{CasOutcome, CatalogStore, StoreError, StoredValue}, file::{ChunkRoot, FileBlockStore, FileIdentity, FileIoError, MultipartPartScan, MultipartPartStore}, gc::{GcScan, GcStore, GcSystemScan}, - key::{CatalogScope, IcebergKey}, + key::{CatalogScope, IcebergKey, StorageKey, TableStorageKey, TableStorageScope}, namespace::{ChildScan, NamespaceStore}, record::StorageRecord, }; @@ -114,7 +114,7 @@ impl CatalogStore for Test value: &[u8], identity: ClientRequestId, ) -> Result { - let record_key = IcebergKey::decode(key)?; + let record_key = StorageKey::decode(key)?; let label = match StorageRecord::decode(&record_key, value)? { StorageRecord::TableCreateOperation(operation) => format!("create-{:?}", operation.phase), StorageRecord::TableCommitOperation(operation) => format!("commit-{:?}", operation.phase), @@ -122,10 +122,13 @@ impl CatalogStore for Test StorageRecord::File(_) if matches!( record_key, - IcebergKey::Catalog { + StorageKey::Legacy(IcebergKey::Catalog { scope: CatalogScope::FileLocation, .. - } + }) | StorageKey::NativeTable(TableStorageKey { + scope: TableStorageScope::File, + .. + }) ) => { "file-location-authority".into() diff --git a/app/crowdb-access-server/tests/common/iceberg_commit_recovery/blocks.rs b/app/crowdb-access-server/tests/common/iceberg_commit_recovery/blocks.rs index 1ec2a5b03..db8182e92 100644 --- a/app/crowdb-access-server/tests/common/iceberg_commit_recovery/blocks.rs +++ b/app/crowdb-access-server/tests/common/iceberg_commit_recovery/blocks.rs @@ -34,6 +34,15 @@ impl TestBlocks { values: self.values.clone(), } } + + #[allow(dead_code)] + pub fn remove_root(&self, root: &ChunkRoot) { + self.values.rcu(|values| { + let mut next = (**values).clone(); + next.remove(&root.chunk.low); + next + }); + } } #[async_trait] diff --git a/app/crowdb-access-server/tests/common/iceberg_file_blocks.rs b/app/crowdb-access-server/tests/common/iceberg_file_blocks.rs index c9f893c16..b59d26388 100644 --- a/app/crowdb-access-server/tests/common/iceberg_file_blocks.rs +++ b/app/crowdb-access-server/tests/common/iceberg_file_blocks.rs @@ -25,6 +25,7 @@ impl TestFileBlocks { FileRecord { file: FileId::random(), location: TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), } diff --git a/app/crowdb-access-server/tests/common/iceberg_store.rs b/app/crowdb-access-server/tests/common/iceberg_store.rs index 00e5d4043..1d69a0691 100644 --- a/app/crowdb-access-server/tests/common/iceberg_store.rs +++ b/app/crowdb-access-server/tests/common/iceberg_store.rs @@ -53,6 +53,8 @@ pub struct TestStore { pub read_delay_ms: AtomicU64, pub scan_delay_ms: AtomicU64, pub scans: AtomicU64, + pub native_file_reads: AtomicU64, + pub deny_native_file_reads: AtomicBool, pub lose_reply_kind: AtomicU8, pub pause_file_read: AtomicBool, pub file_read_entered: tokio::sync::Notify, @@ -174,14 +176,30 @@ impl CatalogStore for TestStore { }) } async fn get(&self, key: &[u8]) -> Result, StoreError> { + if matches!(crowdb_access_iceberg::key::StorageKey::decode(key), + Ok(crowdb_access_iceberg::key::StorageKey::NativeTable(key)) + if key.scope == crowdb_access_iceberg::key::TableStorageScope::File) + { + self.native_file_reads.fetch_add(1, Ordering::SeqCst); + if self.deny_native_file_reads.load(Ordering::SeqCst) { + return Err(StoreError::Response); + } + } let value = self.values.load().get(key).cloned(); if matches!( - crowdb_access_iceberg::key::IcebergKey::decode(key), - Ok(crowdb_access_iceberg::key::IcebergKey::Catalog { - scope: crowdb_access_iceberg::key::CatalogScope::File - | crowdb_access_iceberg::key::CatalogScope::FileLocation, - .. - }) + crowdb_access_iceberg::key::StorageKey::decode(key), + Ok(crowdb_access_iceberg::key::StorageKey::Legacy( + crowdb_access_iceberg::key::IcebergKey::Catalog { + scope: crowdb_access_iceberg::key::CatalogScope::File + | crowdb_access_iceberg::key::CatalogScope::FileLocation, + .. + } + ) | crowdb_access_iceberg::key::StorageKey::NativeTable( + crowdb_access_iceberg::key::TableStorageKey { + scope: crowdb_access_iceberg::key::TableStorageScope::File, + .. + } + )) ) && self.pause_file_read.swap(false, Ordering::SeqCst) { self.file_read_entered.notify_one(); @@ -202,11 +220,18 @@ impl CatalogStore for TestStore { ) -> Result { identity.validate().unwrap(); if matches!( - crowdb_access_iceberg::key::IcebergKey::decode(key), - Ok(crowdb_access_iceberg::key::IcebergKey::Catalog { - scope: crowdb_access_iceberg::key::CatalogScope::TableHead, - .. - }) + crowdb_access_iceberg::key::StorageKey::decode(key), + Ok(crowdb_access_iceberg::key::StorageKey::Legacy( + crowdb_access_iceberg::key::IcebergKey::Catalog { + scope: crowdb_access_iceberg::key::CatalogScope::TableHead, + .. + } + ) | crowdb_access_iceberg::key::StorageKey::NativeTable( + crowdb_access_iceberg::key::TableStorageKey { + scope: crowdb_access_iceberg::key::TableStorageScope::Head, + .. + } + )) ) && self.pause_head_cas.swap(false, Ordering::SeqCst) { self.head_cas_entered.notify_one(); @@ -230,8 +255,10 @@ impl CatalogStore for TestStore { if Arc::ptr_eq(¤t, &self.values.compare_and_swap(¤t, Arc::new(next))) { let mode = self.lose_reply_kind.load(Ordering::SeqCst); let lose = mode == 1 - || match crowdb_access_iceberg::key::IcebergKey::decode(key) { - Ok(crowdb_access_iceberg::key::IcebergKey::Catalog { scope, .. }) => { + || match crowdb_access_iceberg::key::StorageKey::decode(key) { + Ok(crowdb_access_iceberg::key::StorageKey::Legacy( + crowdb_access_iceberg::key::IcebergKey::Catalog { scope, .. }, + )) => { (mode == 2 && scope == crowdb_access_iceberg::key::CatalogScope::NamespaceAuthority) || (mode == 3 && scope == crowdb_access_iceberg::key::CatalogScope::Operation) @@ -244,6 +271,9 @@ impl CatalogStore for TestStore { Ok(crowdb_access_iceberg::record::StorageRecord::TableCommitOperation(operation)) if operation.phase == crowdb_access_iceberg::commit::TableCommitPhase::Rejected)) } + Ok(crowdb_access_iceberg::key::StorageKey::NativeTable(key)) => { + mode == 5 && key.scope == crowdb_access_iceberg::key::TableStorageScope::Head + } _ => false, }; if lose && self.lose_reply_kind.swap(0, Ordering::SeqCst) != 0 { diff --git a/app/crowdb-access-server/tests/common/iceberg_table_http.rs b/app/crowdb-access-server/tests/common/iceberg_table_http.rs index 1d112e560..65c4205c0 100644 --- a/app/crowdb-access-server/tests/common/iceberg_table_http.rs +++ b/app/crowdb-access-server/tests/common/iceberg_table_http.rs @@ -158,6 +158,7 @@ impl TestTableHttp { pub async fn install_with_location(&self, name: &str, trailing_slash: bool) -> (TableHead, Vec) { let location = TableLocation { + storage_namespace: None, catalog: self.context.catalog, table: TableId::random(), }; @@ -218,6 +219,7 @@ impl TestTableHttp { self.put( &name_key(head.catalog, head.namespace, name).unwrap(), &StorageRecord::TableMapping(TableMapping { + storage_namespace: None, catalog: head.catalog, namespace: head.namespace, name: name.into(), diff --git a/app/crowdb-access-server/tests/common/iceberg_upload.rs b/app/crowdb-access-server/tests/common/iceberg_upload.rs index f61d6f917..3debf5043 100644 --- a/app/crowdb-access-server/tests/common/iceberg_upload.rs +++ b/app/crowdb-access-server/tests/common/iceberg_upload.rs @@ -17,6 +17,7 @@ use sha2::{Digest, Sha256}; pub fn owner() -> FileIdentity { FileIdentity { table: TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }, diff --git a/app/crowdb-access-server/tests/iceberg_commit_recovery_test.rs b/app/crowdb-access-server/tests/iceberg_commit_recovery_test.rs index b22d631e3..393319e89 100644 --- a/app/crowdb-access-server/tests/iceberg_commit_recovery_test.rs +++ b/app/crowdb-access-server/tests/iceberg_commit_recovery_test.rs @@ -102,7 +102,8 @@ async fn table_publication_recovers_before_and_after_every_durable_write() { ]), "direct updates must publish immutable metadata through one head CAS" ); - assert_eq!(count, 4); + // The native tree persists data and its owner-bound directory before publication. + assert_eq!(count, 6); } else { assert!(labels.iter().any(|label| label.starts_with("create-"))); } diff --git a/app/crowdb-access-server/tests/iceberg_commit_sdk_test.rs b/app/crowdb-access-server/tests/iceberg_commit_sdk_test.rs index a38e188f3..7a61285f4 100644 --- a/app/crowdb-access-server/tests/iceberg_commit_sdk_test.rs +++ b/app/crowdb-access-server/tests/iceberg_commit_sdk_test.rs @@ -9,7 +9,7 @@ mod common; #[allow(dead_code)] mod fixture; -use crowdb_access_iceberg::key::IcebergKey; +use crowdb_access_iceberg::key::{IcebergKey, StorageKey}; use serde_json::{json, Value}; use std::{sync::atomic::Ordering, time::Duration}; @@ -66,11 +66,11 @@ async fn official_commit_loses_real_head_cas_without_rebase_and_keeps_hidden_ret assert!( fixture.store.values.load().keys().all(|key| { !matches!( - IcebergKey::decode(key).unwrap(), - IcebergKey::Catalog { + StorageKey::decode(key).unwrap(), + StorageKey::Legacy(IcebergKey::Catalog { scope: crowdb_access_iceberg::key::CatalogScope::TableCommitOperation, .. - } + }) ) }), "ordinary commits must not write a phase journal" diff --git a/app/crowdb-access-server/tests/iceberg_file_admission_test.rs b/app/crowdb-access-server/tests/iceberg_file_admission_test.rs index 2195ea810..1f8bad7c3 100644 --- a/app/crowdb-access-server/tests/iceberg_file_admission_test.rs +++ b/app/crowdb-access-server/tests/iceberg_file_admission_test.rs @@ -20,6 +20,7 @@ use hyper::Method; fn grant(operations: &[FileOperation]) -> FileGrant { FileGrant { + storage_namespace: None, context: CatalogContext { catalog: CatalogId::random(), activation_epoch: 1, @@ -37,6 +38,7 @@ fn grant(operations: &[FileOperation]) -> FileGrant { fn location(grant: &FileGrant) -> crowdb_access_iceberg::file::FileLocation { TableLocation { + storage_namespace: None, catalog: grant.context.catalog, table: grant.table, } @@ -112,6 +114,7 @@ fn authorization_intersects_grant_operation_scope_and_service_limits() { )); let mut wrong = get_request.clone(); wrong.location = TableLocation { + storage_namespace: None, catalog: grant.context.catalog, table: TableId::random(), } diff --git a/app/crowdb-access-server/tests/iceberg_file_auth_test.rs b/app/crowdb-access-server/tests/iceberg_file_auth_test.rs index 9e7138530..b56f94a90 100644 --- a/app/crowdb-access-server/tests/iceberg_file_auth_test.rs +++ b/app/crowdb-access-server/tests/iceberg_file_auth_test.rs @@ -18,6 +18,7 @@ fn credentials() -> (FileGrantIssuer, FileCredentials) { let issuer = FileGrantIssuer::new([42; 32], 60_000).unwrap(); let credentials = issuer .issue(FileGrant { + storage_namespace: None, context: CatalogContext { catalog: CatalogId::random(), activation_epoch: 1, diff --git a/app/crowdb-access-server/tests/iceberg_file_recovery_test.rs b/app/crowdb-access-server/tests/iceberg_file_recovery_test.rs index f05b9620a..d9d978873 100644 --- a/app/crowdb-access-server/tests/iceberg_file_recovery_test.rs +++ b/app/crowdb-access-server/tests/iceberg_file_recovery_test.rs @@ -112,6 +112,7 @@ async fn setup_with_bounds_and_file_limit( let started = now_ms(); let credentials = issuer .issue(FileGrant { + storage_namespace: table.storage_namespace, context, table: table.table, principal: [7; 32], diff --git a/app/crowdb-access-server/tests/iceberg_file_request_test.rs b/app/crowdb-access-server/tests/iceberg_file_request_test.rs index f333dd98b..739d7940d 100644 --- a/app/crowdb-access-server/tests/iceberg_file_request_test.rs +++ b/app/crowdb-access-server/tests/iceberg_file_request_test.rs @@ -6,11 +6,105 @@ use hyper::Method; fn table() -> TableLocation { TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), } } +#[test] +fn native_injection_accepts_only_new_layout_parquet_put_and_uuidv7_identity() { + let mut table = table(); + table.storage_namespace = Some(crowdb_access_iceberg::key::NamespaceId::random()); + let identity = "0199aabc-0000-7000-8000-000000000001"; + let uri = path( + table, + &format!("data/part.parquet?crowdb-native-upload={identity}"), + ); + assert!(FileRequest::parse(&Method::PUT, &uri) + .unwrap() + .injection + .is_some()); + assert!(FileRequest::parse(&Method::GET, &uri).is_err()); + assert!(FileRequest::parse( + &Method::PUT, + &path(table, "data/part.parquet?crowdb-native-upload=bad") + ) + .is_err()); + assert!(FileRequest::parse( + &Method::PUT, + &path( + table, + &format!("metadata/v1.json?crowdb-native-upload={identity}") + ) + ) + .is_err()); + table.storage_namespace = None; + assert!(FileRequest::parse( + &Method::PUT, + &path( + table, + &format!("data/part.parquet?crowdb-native-upload={identity}") + ) + ) + .is_err()); +} + +#[test] +fn bucket_probes_are_distinct_from_unscoped_object_lists() { + use crowdb_access_server::iceberg::{parse_file_bucket, FileBucketRequest}; + assert_eq!( + parse_file_bucket(&Method::HEAD, &"/iceberg".parse().unwrap()).unwrap(), + Some(FileBucketRequest::Head("iceberg".into())) + ); + assert_eq!( + parse_file_bucket(&Method::GET, &"/iceberg?location".parse().unwrap()).unwrap(), + Some(FileBucketRequest::Location("iceberg".into())) + ); + assert!( + parse_file_bucket(&Method::GET, &"/iceberg?list-type=2".parse().unwrap()) + .unwrap() + .is_none() + ); + assert!(parse_file_bucket(&Method::GET, &"/".parse().unwrap()) + .unwrap() + .is_none()); + assert!(parse_file_bucket(&Method::GET, &"/iceberg?location&prefix=".parse().unwrap()).is_err()); +} + +#[test] +fn three_id_http_paths_and_list_prefixes_use_the_fixed_bucket() { + let table = TableLocation { + storage_namespace: Some(crowdb_access_iceberg::key::NamespaceId::random()), + ..table() + }; + let object = format!("/iceberg/{}metadata/v1.json", table.object_prefix()); + let request = FileRequest::parse(&Method::GET, &object.parse().unwrap()).unwrap(); + assert_eq!(request.location.table(), table); + let list = format!("/iceberg?list-type=2&prefix={}metadata%2F", table.object_prefix()); + assert_eq!( + parse_file_list(&Method::GET, &list.parse().unwrap()) + .unwrap() + .unwrap() + .table, + table + ); + for prefix in [ + String::new(), + format!("{}/", table.catalog), + format!("{}/{}/", table.catalog, table.storage_namespace.unwrap()), + ] { + assert!(parse_file_list( + &Method::GET, + &format!("/iceberg?list-type=2&prefix={prefix}").parse().unwrap() + ) + .is_err()); + } + let reserved = format!("/iceberg/{}f/v1/name.token.parquet", table.object_prefix()); + assert!(FileRequest::parse(&Method::PUT, &reserved.parse().unwrap()).is_err()); + assert!(FileRequest::parse(&Method::GET, &reserved.parse().unwrap()).is_ok()); +} + #[test] fn native_list_requests_preserve_table_prefix_and_validate_all_selectors() { let table = table(); diff --git a/app/crowdb-access-server/tests/iceberg_file_response_test.rs b/app/crowdb-access-server/tests/iceberg_file_response_test.rs index 59d36f71e..b8338b147 100644 --- a/app/crowdb-access-server/tests/iceberg_file_response_test.rs +++ b/app/crowdb-access-server/tests/iceberg_file_response_test.rs @@ -15,6 +15,7 @@ use sha2::{Digest, Sha256}; fn session() -> MultipartSession { let table = TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }; @@ -207,3 +208,37 @@ fn complete_uses_only_a_published_matching_record() { session.published = None; assert!(MultipartResponses::complete(&session, &record, "https://storage.example/a").is_err()); } +#[test] +fn read_conditions_preserve_strong_preconditions_and_conditional_range_semantics() { + use crowdb_access_server::iceberg::{file_read_condition, FileReadCondition}; + use hyper::header::{HeaderMap, HeaderValue, IF_MATCH, IF_NONE_MATCH, IF_RANGE}; + let tag = "\"current\""; + let mut headers = HeaderMap::new(); + headers.insert(IF_MATCH, HeaderValue::from_static("\"old\", \"current\"")); + headers.insert(IF_RANGE, HeaderValue::from_static("\"old\"")); + assert_eq!( + file_read_condition(&headers, tag).unwrap(), + FileReadCondition::Read { use_range: false } + ); + headers.insert(IF_RANGE, HeaderValue::from_static("\"current\"")); + assert_eq!( + file_read_condition(&headers, tag).unwrap(), + FileReadCondition::Read { use_range: true } + ); + headers.insert(IF_MATCH, HeaderValue::from_static("W/\"current\"")); + assert_eq!( + file_read_condition(&headers, tag), + Err(FileS3ErrorCode::PreconditionFailed) + ); + headers.remove(IF_MATCH); + headers.insert(IF_NONE_MATCH, HeaderValue::from_static("W/\"current\"")); + assert_eq!( + file_read_condition(&headers, tag).unwrap(), + FileReadCondition::NotModified + ); + headers.insert(IF_NONE_MATCH, HeaderValue::from_static("\"old\", invalid")); + assert_eq!( + file_read_condition(&headers, tag), + Err(FileS3ErrorCode::InvalidRequest) + ); +} diff --git a/app/crowdb-access-server/tests/iceberg_native_injection_http_test.rs b/app/crowdb-access-server/tests/iceberg_native_injection_http_test.rs new file mode 100644 index 000000000..966dcf8cc --- /dev/null +++ b/app/crowdb-access-server/tests/iceberg_native_injection_http_test.rs @@ -0,0 +1,397 @@ +#[path = "common/iceberg_commit_recovery/blocks.rs"] +#[allow(dead_code)] +mod blocks; +#[path = "common/iceberg_commit_recovery/fixture.rs"] +#[allow(dead_code, unused_imports)] +mod common; +#[path = "common/iceberg_store.rs"] +mod memory; +#[path = "common/iceberg_signed_file.rs"] +#[allow(dead_code)] +mod signed; +#[path = "common/iceberg_commit_recovery/store.rs"] +mod store; + +use crowdb_access_iceberg::{ + catalog::{CatalogRepository, ClearBounds, ManagementPrivilege}, + file::{ + DirectFileLocations, FileGrant, FileGrantIssuer, FileOperation, FileOperations, FileRepository, + TableLocation, + }, + key::{NamespaceId, OperationId, TableId}, + operation::{ManagementAction, ManagementRequest, RequestIdentity}, + wire::BearerAuthenticator, +}; +use crowdb_access_server::iceberg::{serve, IcebergHttpService}; +use reqwest::Method; +use std::{ + collections::BTreeMap, + sync::{atomic::Ordering, Arc}, + time::Duration, +}; + +fn authentication() -> BearerAuthenticator { + BearerAuthenticator::new(&"r".repeat(32), &"w".repeat(32), &"m".repeat(32), &"c".repeat(32)).unwrap() +} + +fn rotated_authentication() -> BearerAuthenticator { + BearerAuthenticator::new(&"a".repeat(32), &"b".repeat(32), &"d".repeat(32), &"e".repeat(32)).unwrap() +} + +async fn rest( + address: std::net::SocketAddr, + method: Method, + path: &str, + body: serde_json::Value, + serial: u64, +) -> serde_json::Value { + let now = common::now_ms(); + let identity = format!("{:08x}-{:04x}-7000-8000-{serial:012x}", now >> 16, now & 0xffff); + let request = reqwest::Client::new() + .request(method, format!("http://{address}{path}")) + .bearer_auth("w".repeat(32)) + .header("idempotency-key", identity); + let request = if body.is_null() { + request + } else { + request.json(&body) + }; + let response = request.send().await.unwrap(); + let status = response.status(); + let text = response.text().await.unwrap(); + assert!(status.is_success(), "{path}: {status}: {text}"); + if text.is_empty() { + serde_json::Value::Null + } else { + serde_json::from_str(&text).unwrap() + } +} + +#[tokio::test] +async fn full_storage_identity_refresh_survives_move_and_default_namespace_recreation() { + use serde_json::json; + let (_stack, _repository, _table, client, server) = setup().await; + let create = json!({"name":"events","schema":{"type":"struct","schema-id":0, + "fields":[{"id":1,"name":"id","type":"long","required":true}]}}); + let initial = rest( + client.address, + Method::POST, + "/v1/namespaces/default/tables", + create.clone(), + 2, + ) + .await; + let original = initial["metadata"]["location"].as_str().unwrap(); + let refresh = initial["config"]["client.refresh-credentials-endpoint"] + .as_str() + .unwrap(); + assert!(refresh.contains("storage-namespace-id=")); + rest( + client.address, + Method::POST, + "/v1/namespaces", + json!({"namespace":["dest"]}), + 3, + ) + .await; + rest( + client.address, + Method::POST, + "/v1/tables/rename", + json!({ + "source":{"namespace":["default"],"name":"events"}, + "destination":{"namespace":["dest"],"name":"renamed"}}), + 4, + ) + .await; + rest( + client.address, + Method::DELETE, + "/v1/namespaces/default", + json!(null), + 5, + ) + .await; + let recreated = rest( + client.address, + Method::POST, + "/v1/namespaces/default/tables", + create, + 6, + ) + .await; + assert_ne!(recreated["metadata"]["location"].as_str().unwrap(), original); + let response = reqwest::Client::new() + .get(format!("http://{}{refresh}", client.address)) + .bearer_auth("r".repeat(32)) + .send() + .await + .unwrap(); + let status = response.status(); + let text = response.text().await.unwrap(); + assert_eq!(status, 200, "{text}"); + let refreshed: serde_json::Value = serde_json::from_str(&text).unwrap(); + assert_eq!( + refreshed["storage-credentials"][0]["prefix"] + .as_str() + .unwrap() + .trim_end_matches('/'), + original.trim_end_matches('/') + ); + let moved = reqwest::Client::new() + .get(format!( + "http://{}/v1/namespaces/dest/tables/renamed", + client.address + )) + .bearer_auth("r".repeat(32)) + .send() + .await + .unwrap() + .json::() + .await + .unwrap(); + assert_eq!(moved["metadata"]["location"].as_str().unwrap(), original); + server.abort(); + let _ = server.await; +} + +async fn listen( + stack: &common::TestCommitFixture, + repository: Arc, + rotate: bool, +) -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) { + let auth = if rotate { + rotated_authentication() + } else { + authentication() + }; + let mut service = IcebergHttpService::new(repository, auth, Duration::from_secs(300)) + .with_namespaces(stack.store()) + .unwrap() + .with_fileio(stack.store(), stack.blocks.clone(), "us-east-1".into()) + .unwrap() + .with_tables(stack.store(), stack.blocks.clone()) + .unwrap(); + if rotate { + service = service + .with_direct_file_locations(DirectFileLocations::new(2, BTreeMap::from([(2, [4; 32])])).unwrap()) + .unwrap(); + } + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + service = service + .with_table_credentials(stack.store(), format!("http://{address}")) + .unwrap(); + let task = tokio::spawn(async move { + serve(listener, Arc::new(service), std::future::pending()) + .await + .unwrap(); + }); + (address, task) +} + +async fn setup() -> ( + common::TestCommitFixture, + Arc, + TableLocation, + signed::TestFileClient, + tokio::task::JoinHandle<()>, +) { + let stack = common::TestCommitFixture::new(); + let repository = Arc::new( + CatalogRepository::new( + stack.store(), + ClearBounds { + request_ms: 300_000, + delegated_access_ms: 900_000, + ..ClearBounds::default() + }, + ) + .unwrap(), + ); + let now = common::now_ms(); + repository + .execute( + ManagementRequest { + identity: RequestIdentity { + operation: OperationId::random(), + issued_ms: now, + }, + principal: "manager".into(), + action: ManagementAction::Initialize, + expected_epoch: 0, + display_name: "native".into(), + confirmation: None, + capabilities: None, + }, + ManagementPrivilege::Manage, + now, + ) + .await + .unwrap(); + common::activate(&repository).await; + assert_eq!( + repository.status().await.unwrap().1.storage_layout, + crowdb_access_iceberg::catalog::CatalogStorageLayout::Native + ); + let context = repository.status().await.unwrap().0.context; + let table = TableLocation { + catalog: context.catalog, + table: TableId::random(), + storage_namespace: Some(NamespaceId::random()), + }; + let credentials = FileGrantIssuer::new(authentication().namespace_token_key(), 900_000) + .unwrap() + .issue(FileGrant { + context, + storage_namespace: table.storage_namespace, + table: table.table, + principal: [3; 32], + nonce: OperationId::random(), + issued_ms: now, + expires_ms: now + 900_000, + max_request_bytes: 1024 * 1024, + max_file_bytes: 1024 * 1024, + operations: FileOperations::new(&[ + FileOperation::Put, + FileOperation::Get, + FileOperation::Head, + FileOperation::ListObjects, + ]) + .unwrap(), + }) + .unwrap(); + let (address, server) = listen(&stack, repository.clone(), false).await; + let client = signed::TestFileClient { + client: reqwest::Client::new(), + credentials, + address, + }; + (stack, repository, table, client, server) +} + +#[tokio::test] +async fn direct_http_reads_skip_file_kv_after_upload_restart_rotation_and_logical_delete() { + let (stack, repository, table, mut client, server) = setup().await; + let context = repository.status().await.unwrap().0.context; + let now = common::now_ms(); + let identity = format!("{:08x}-{:04x}-7000-8000-000000000001", now >> 16, now & 0xffff); + let path = format!("/iceberg/{}data/part.parquet", table.storage_path().unwrap()); + let query = format!("crowdb-native-upload={identity}"); + let bytes = b"PAR1datafoot\x04\0\0\0PAR1"; + let response = client.send(Method::PUT, &path, &query, bytes, false).await; + let status = response.status(); + let body = response.text().await.unwrap(); + assert_eq!(status, 200, "{body}"); + let uri = serde_json::from_str::(&body).unwrap()["location"] + .as_str() + .unwrap() + .to_owned(); + let location = uri.parse::().unwrap(); + let direct = format!("/iceberg/{}", location.object_key()); + assert!(location.relative_key().starts_with("f/v1/part.")); + let replay = client.send(Method::PUT, &path, &query, bytes, false).await; + assert_eq!(replay.status(), 200); + assert_eq!(replay.text().await.unwrap(), body); + let file = FileRepository::new(stack.store()) + .load(context, &location) + .await + .unwrap() + .unwrap(); + server.abort(); + let _ = server.await; + let (address, server) = listen(&stack, repository, true).await; + client.address = address; + let mut grant = client.credentials.grant().clone(); + grant.nonce = OperationId::random(); + client.credentials = FileGrantIssuer::new(rotated_authentication().namespace_token_key(), 900_000) + .unwrap() + .issue(grant) + .unwrap(); + let kv = stack.store(); + kv.native_file_reads.store(0, Ordering::SeqCst); + kv.deny_native_file_reads.store(true, Ordering::SeqCst); + assert_eq!( + client + .send(Method::HEAD, "/iceberg", "", b"", false) + .await + .status(), + 200 + ); + let head = client.send(Method::HEAD, &direct, "", b"", false).await; + assert_eq!(head.status(), 200); + assert_eq!(head.headers()["content-length"], bytes.len().to_string()); + let tag = head.headers()["etag"].to_str().unwrap().to_owned(); + assert_conditional_ranges(&client, &direct, &tag).await; + let response = client + .send( + Method::GET, + "/iceberg", + &format!("list-type=2&prefix={}", table.storage_path().unwrap()), + b"", + false, + ) + .await; + let list = response.text().await.unwrap(); + assert!(list.contains("f/v1/part."), "{list}"); + assert_eq!(kv.native_file_reads.load(Ordering::SeqCst), 0); + kv.deny_native_file_reads.store(false, Ordering::SeqCst); + FileRepository::new(kv.clone()) + .mark_deleted(context, &file, common::now_ms()) + .await + .unwrap(); + kv.native_file_reads.store(0, Ordering::SeqCst); + kv.deny_native_file_reads.store(true, Ordering::SeqCst); + let response = client.send(Method::GET, &direct, "", b"", false).await; + assert_eq!(response.status(), 200); + assert_eq!(response.bytes().await.unwrap(), bytes.as_slice()); + let response = client + .send( + Method::GET, + "/iceberg", + &format!("list-type=2&prefix={}", table.storage_path().unwrap()), + b"", + false, + ) + .await; + assert!(!response.text().await.unwrap().contains("f/v1/part.")); + assert_eq!(kv.native_file_reads.load(Ordering::SeqCst), 0); + let crowdb_access_iceberg::file::FileContent::Chunks { root: Some(root) } = &file.content else { + panic!() + }; + stack.blocks.remove_root(root); + for method in [Method::HEAD, Method::GET] { + let response = client + .request(method, &direct, "", b"", false, None) + .header("If-None-Match", &tag) + .send() + .await + .unwrap(); + assert_eq!(response.status(), 404); + } + assert_eq!(kv.native_file_reads.load(Ordering::SeqCst), 0); + server.abort(); + let _ = server.await; +} + +async fn assert_conditional_ranges(client: &signed::TestFileClient, direct: &str, tag: &str) { + let response = client + .send_range(Method::GET, direct, "", b"", false, Some("bytes=-4")) + .await; + assert_eq!(response.status(), 206); + assert_eq!(response.bytes().await.unwrap(), b"PAR1".as_slice()); + let response = client + .request(Method::GET, direct, "", b"", false, None) + .header("If-None-Match", tag) + .send() + .await + .unwrap(); + assert_eq!(response.status(), 304); + let response = client + .request(Method::GET, direct, "", b"", false, None) + .header("If-Match", "\"wrong\"") + .send() + .await + .unwrap(); + assert_eq!(response.status(), 412); +} diff --git a/app/crowdb-access-server/tests/iceberg_table_acceptance_test.rs b/app/crowdb-access-server/tests/iceberg_table_acceptance_test.rs index 919215108..17aeb501b 100644 --- a/app/crowdb-access-server/tests/iceberg_table_acceptance_test.rs +++ b/app/crowdb-access-server/tests/iceberg_table_acceptance_test.rs @@ -11,7 +11,7 @@ use std::{collections::BTreeMap, sync::atomic::Ordering, time::Duration}; use crowdb_access_iceberg::{ catalog::StoredValue, - key::{CatalogScope, IcebergKey, OperationId, TableId}, + key::{CatalogScope, IcebergKey, OperationId, StorageKey, TableId, TableStorageKey, TableStorageScope}, record::StorageRecord, table::{head_key, name_key, TableHead, TableLifecycle, TableMapping, TableMappingState}, }; @@ -95,6 +95,7 @@ fn mapping(test: &TestTableHttp, head: &TableHead, name: &str, state: TableMappi test.put( &name_key(head.catalog, head.namespace, name).unwrap(), &StorageRecord::TableMapping(TableMapping { + storage_namespace: None, catalog: head.catalog, namespace: head.namespace, name: name.into(), @@ -163,15 +164,18 @@ fn authority(test: &TestTableHttp) -> BTreeMap, StoredValue> { .iter() .filter(|(key, _)| { matches!( - IcebergKey::decode(key), - Ok(IcebergKey::Catalog { + StorageKey::decode(key), + Ok(StorageKey::Legacy(IcebergKey::Catalog { scope: CatalogScope::TableHead | CatalogScope::TableName | CatalogScope::File | CatalogScope::FileLocation | CatalogScope::Reclamation, .. - }) + }) | StorageKey::NativeTable(TableStorageKey { + scope: TableStorageScope::Head | TableStorageScope::File, + .. + })) ) }) .map(|(key, value)| (key.clone(), value.clone())) @@ -231,11 +235,14 @@ async fn lost_drop_publication_reply_replays_once_and_preserves_files_for_both_p let after = authority(&test); for (key, value) in before { if matches!( - IcebergKey::decode(&key).unwrap(), - IcebergKey::Catalog { + StorageKey::decode(&key).unwrap(), + StorageKey::Legacy(IcebergKey::Catalog { scope: CatalogScope::File | CatalogScope::FileLocation, .. - } + }) | StorageKey::NativeTable(TableStorageKey { + scope: TableStorageScope::File, + .. + }) ) { assert_eq!(after.get(&key), Some(&value)); } @@ -244,11 +251,11 @@ async fn lost_drop_publication_reply_replays_once_and_preserves_files_for_both_p .keys() .filter(|key| { matches!( - IcebergKey::decode(key).unwrap(), - IcebergKey::Catalog { + StorageKey::decode(key).unwrap(), + StorageKey::Legacy(IcebergKey::Catalog { scope: CatalogScope::Reclamation, .. - } + }) ) }) .count(); diff --git a/app/crowdb-access-server/tests/iceberg_table_lifecycle_test.rs b/app/crowdb-access-server/tests/iceberg_table_lifecycle_test.rs index 06da2ac96..1fd30c9d6 100644 --- a/app/crowdb-access-server/tests/iceberg_table_lifecycle_test.rs +++ b/app/crowdb-access-server/tests/iceberg_table_lifecycle_test.rs @@ -232,7 +232,17 @@ async fn credential_refresh_follows_exact_renamed_identity_and_stops_after_drop( .await, ) .await; - value(test.request(Method::GET, old, "w", None).await, 404).await; + let refreshed = value(test.request(Method::GET, old, "w", None).await, 200).await; + assert_eq!( + refreshed["storage-credentials"][0]["prefix"] + .as_str() + .unwrap() + .trim_end_matches('/'), + created["metadata"]["location"] + .as_str() + .unwrap() + .trim_end_matches('/') + ); let renamed = "/v1/namespaces/analytics/tables/renamed"; let moved = value(test.request(Method::GET, renamed, "w", None).await, 200).await; let current = moved["config"]["client.refresh-credentials-endpoint"] diff --git a/app/crowdb-web/ui/src/access/Workbench.test.tsx b/app/crowdb-web/ui/src/access/Workbench.test.tsx new file mode 100644 index 000000000..f911c2ab9 --- /dev/null +++ b/app/crowdb-web/ui/src/access/Workbench.test.tsx @@ -0,0 +1,23 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. +import { fireEvent, render, screen, waitFor } from '@testing-library/react'; +import { beforeEach, expect, it, vi } from 'vitest'; +import { DocsHelp } from './Workbench'; + +beforeEach(() => { + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response(JSON.stringify({ + s3: { endpoint: 'http://localhost:9091', region: 'us-east-1', access_key_id: 'access', secret_access_key: 'secret' }, + iceberg: { uri: 'http://localhost:9092', token: 'token' }, + })))); + Object.defineProperty(navigator, 'clipboard', { configurable: true, value: { writeText: vi.fn().mockResolvedValue(undefined) } }); +}); + +it('reveals credentials and reports a successful copy', async () => { + render(); + fireEvent.click(screen.getByText('Connection credentials')); + fireEvent.click(screen.getByRole('button', { name: 'Show local credentials' })); + await waitFor(() => expect(screen.getByText(/ICEBERG_URI=http:\/\/localhost:9092/)).toBeVisible()); + fireEvent.click(screen.getByRole('button', { name: 'Copy env' })); + await waitFor(() => expect(screen.getByRole('status')).toHaveTextContent('Copied environment variables')); + expect(navigator.clipboard.writeText).toHaveBeenCalledWith(expect.stringContaining('AWS_ACCESS_KEY_ID=access')); +}); diff --git a/app/crowdb-web/ui/src/access/Workbench.tsx b/app/crowdb-web/ui/src/access/Workbench.tsx index 4bd7caeeb..0d5d70139 100644 --- a/app/crowdb-web/ui/src/access/Workbench.tsx +++ b/app/crowdb-web/ui/src/access/Workbench.tsx @@ -13,16 +13,29 @@ export const buttonClass = 'tw-rounded tw-border tw-border-border tw-px-3 tw-py- export function DocsHelp({ href, title, auth }: { href: string; title: string; auth?: string }) { const [credentials, setCredentials] = useState(null); const [error, setError] = useState(''); + const [copyStatus, setCopyStatus] = useState(''); + const endpointForBrowser = (value: string | null) => { + if (!value) return ''; + try { + const url = new URL(value); + const browserHost = window.location.hostname; + if (browserHost && ['localhost', '127.0.0.1', '[::1]', '::1'].includes(url.hostname)) url.hostname = browserHost; + return url.toString().replace(/\/$/, ''); + } catch { + return value; + } + }; const reveal = async () => { setError(''); + setCopyStatus(''); try { const response = await fetch(`${getApiBase()}/access/credentials`, { cache: 'no-store' }); if (!response.ok) throw new Error(`HTTP ${response.status}`); const value = await response.json() as { s3: Record; iceberg: Record }; const lines = [ - `ICEBERG_URI=${value.iceberg.uri ?? ''}`, + `ICEBERG_URI=${endpointForBrowser(value.iceberg.uri)}`, `ICEBERG_TOKEN=${value.iceberg.token ?? ''}`, - `AWS_ENDPOINT_URL=${value.s3.endpoint ?? ''}`, + `AWS_ENDPOINT_URL=${endpointForBrowser(value.s3.endpoint)}`, `AWS_DEFAULT_REGION=${value.s3.region ?? ''}`, `AWS_ACCESS_KEY_ID=${value.s3.access_key_id ?? ''}`, `AWS_SECRET_ACCESS_KEY=${value.s3.secret_access_key ?? ''}`, @@ -32,13 +45,29 @@ export function DocsHelp({ href, title, auth }: { href: string; title: string; a setError(`Credentials are unavailable: ${String(reason)}`); } }; - const copy = () => { if (credentials) void navigator.clipboard?.writeText(credentials); }; + const copy = async () => { + if (!credentials) return; + setCopyStatus(''); + try { + if (navigator.clipboard?.writeText) await navigator.clipboard.writeText(credentials); + else { + const input = document.createElement('textarea'); + input.value = credentials; input.setAttribute('readonly', ''); input.style.position = 'fixed'; input.style.opacity = '0'; + document.body.appendChild(input); input.select(); + if (!document.execCommand('copy')) throw new Error('Clipboard is unavailable'); + input.remove(); + } + setCopyStatus('Copied environment variables'); + } catch (reason) { + setCopyStatus(`Copy failed: ${String(reason)}`); + } + }; return
{title}
Help
- {auth &&
Connection credentials

{auth}

{credentials && }
{error &&

{error}

}{credentials && 
{credentials}
}
} + {auth &&
Connection credentials

{auth}

{credentials && }
{error &&

{error}

}{copyStatus &&

{copyStatus}

}{credentials && 
{credentials}
}
}
; } export function Workbench({ sidebar, children, detail, showActivity = true, resizableSidebar = true, help }: { resizableSidebar?: boolean; showActivity?: boolean; sidebar: ReactNode; children: ReactNode; detail?: ReactNode; help?: ReactNode }) { diff --git a/app/crowdb-web/ui/src/iceberg/Fields.test.tsx b/app/crowdb-web/ui/src/iceberg/Fields.test.tsx new file mode 100644 index 000000000..748fdfcea --- /dev/null +++ b/app/crowdb-web/ui/src/iceberg/Fields.test.tsx @@ -0,0 +1,31 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. +import { fireEvent, render, screen, waitFor } from '@testing-library/react'; +import { expect, it } from 'vitest'; +import { Fields, SchemaTable } from './Fields'; + +it('renders file metrics records as a table', () => { + render(); + const table = screen.getByRole('table', { name: 'File metrics' }); + expect(table).toHaveTextContent('null count'); + expect(table).toHaveTextContent('id'); + expect(table).toHaveTextContent('name'); + expect(table.querySelectorAll('tbody tr')).toHaveLength(2); +}); + +it('renders nested writer fields as an expandable table', () => { + render(); + const table = screen.getByRole('table', { name: 'Writer schema' }); + expect(table).toHaveTextContent('status'); + expect(table).toHaveTextContent('data_file'); + expect(table).not.toHaveTextContent('file_path'); + const button = screen.getByRole('button', { name: 'Expand field details' }); + fireEvent.click(button); + return waitFor(() => expect(table).toHaveTextContent('file_path')); +}); diff --git a/app/crowdb-web/ui/src/iceberg/Fields.tsx b/app/crowdb-web/ui/src/iceberg/Fields.tsx index 5e7eac3a2..d9700f643 100644 --- a/app/crowdb-web/ui/src/iceberg/Fields.tsx +++ b/app/crowdb-web/ui/src/iceberg/Fields.tsx @@ -18,15 +18,71 @@ export function byteSize(value: string | number | undefined): string { } export function Fields({ values, stacked = false }: { values: Record; stacked?: boolean }) { if (Object.keys(values).length > 50) return ; - return
{Object.entries(values).slice(0, 50).map(([key, value]) =>
{key.replaceAll('_', ' ').replaceAll('-', ' ')}
{typeof value === 'object' && value !== null ? stacked ? : : scalar(value)}
)}
; + return
{Object.entries(values).slice(0, 50).map(([key, value]) =>
{key.replaceAll('_', ' ').replaceAll('-', ' ')}
{key === 'metrics' && Array.isArray(value) && value.every(item => asObject(item)) ? asObject(item)!)} /> : typeof value === 'object' && value !== null ? stacked ? : : scalar(value)}
)}
; +} +function ObjectArrayTable({ label, values }: { label: string; values: SchemaNode[] }) { + const headings = [...new Set(values.flatMap(value => Object.keys(value)))]; + return
{headings.map(heading => )}{values.map((value, index) => {headings.map(heading => )})}
#{heading.replaceAll('_', ' ')}
{index}{typeof value[heading] === 'object' && value[heading] !== null ? JSON.stringify(value[heading]) : scalar(value[heading])}
; } export function Structured({ value }: { value: unknown }) { const [page, setPage] = useState(0); if (value == null || typeof value !== 'object') return <>{scalar(value)}; + if (Array.isArray(value) && value.length > 0 && value.every(item => asObject(item))) return asObject(item)!)} />; const entries = Object.entries(value); const start = Math.min(page * 30, Math.max(0, Math.floor((entries.length - 1) / 30) * 30)); return <>
{entries.slice(start, start + 30).map(([key, item]) =>
{key}
{item && typeof item === 'object' ? : scalar(item)}
)}
{entries.length > 30 &&
{start + 1}–{Math.min(start + 30, entries.length)} of {entries.length}
}; } +type SchemaNode = Record; +const asObject = (value: unknown): SchemaNode | null => value && typeof value === 'object' && !Array.isArray(value) ? value as SchemaNode : null; +const schemaFields = (value: unknown): SchemaNode[] => { + if (Array.isArray(value)) return value.flatMap(item => schemaFields(item)); + const object = asObject(value); + if (!object || !Array.isArray(object.fields)) return []; + return object.fields.filter((field): field is SchemaNode => !!asObject(field)).map(field => asObject(field)!); +}; +const schemaType = (value: unknown): string => { + if (Array.isArray(value)) return value.map(schemaType).join(' | '); + if (typeof value === 'string') return value; + const object = asObject(value); + if (!object) return scalar(value); + if (typeof object.type === 'string') { + if (object.logicalType) return `${object.type} (${String(object.logicalType)})`; + if (object.type === 'array' && object.items) return `array<${schemaType(object.items)}>`; + if (object.type === 'map' && object.values) return `map`; + return object.name ? `${object.type} ${String(object.name)}` : object.type; + } + if (object.fields) return object.name ? `record ${String(object.name)}` : 'record'; + return JSON.stringify(value); +}; +const schemaChildren = (field: SchemaNode): SchemaNode[] => { + const direct = schemaFields(field); + if (direct.length) return direct; + return schemaFields(field.type); +}; +function SchemaFieldRows({ fields, depth = 0 }: { fields: SchemaNode[]; depth?: number }) { + return <>{fields.map((field, index) => )}; +} +function SchemaFieldRow({ field, depth }: { field: SchemaNode; depth: number }) { + const [open, setOpen] = useState(false); + const children = schemaChildren(field); + const fieldType = schemaType(field.type ?? field); + const required = field.required === true || (Array.isArray(field.type) && !field.type.some(item => item === 'null')); + return <> + + {children.length > 0 ? : String(field.name ?? 'field')} + {fieldType} + {required ? 'Yes' : 'No'} + {scalar(field.id ?? field['field-id'])} + {field.doc ? String(field.doc) : children.length ? `${children.length} nested fields` : '—'} + + {open && } + ; +} +export function SchemaTable({ value }: { value: unknown }) { + const fields = schemaFields(value); + if (!fields.length) return ; + return
{['Field', 'Type', 'Required', 'ID', 'Details'].map(heading => )}
{heading}
; +} function Expandable({ value }: { value: object }) { const [open, setOpen] = useState(false); return
setOpen(event.currentTarget.open)}>{Object.keys(value).length} {Array.isArray(value) ? 'items' : 'fields'}{open && }
; diff --git a/app/crowdb-web/ui/src/iceberg/ReferenceExplorer.tsx b/app/crowdb-web/ui/src/iceberg/ReferenceExplorer.tsx index a41299e6c..ae876d9cd 100644 --- a/app/crowdb-web/ui/src/iceberg/ReferenceExplorer.tsx +++ b/app/crowdb-web/ui/src/iceberg/ReferenceExplorer.tsx @@ -1,7 +1,7 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. import { buttonClass } from '../access/Workbench'; -import { Fields, Records, Structured, byteSize } from './Fields'; +import { Fields, Records, SchemaTable, byteSize } from './Fields'; import { ManifestTable } from './ManifestTable'; import { ParquetInspector } from './ParquetInspector'; import type { Inspector } from './useInspection'; @@ -18,7 +18,7 @@ export function InspectionView({ inspector, propertyHost }: { inspector: Inspect {data?.kind === 'unsupported' && <>} />

{data.reason}

} {data?.kind === 'parquet' && } {data?.kind === 'manifest-list' && } - {data?.kind === 'manifest' && <> [, f.status, f.content, f.format, f.records, byteSize(f.size)])} />
Writer schema
} + {data?.kind === 'manifest' && <> [, f.status, f.content, f.format, f.records, byteSize(f.size)])} />
Writer schema
} {data && data.kind !== 'parquet' && Number(data.offset) > 0 && } {data?.next != null && } {data?.kind === 'parquet' && data.groups?.[0]?.index !== '0' && } diff --git a/app/crowdb-web/ui/src/menus/useClusterMenus.tsx b/app/crowdb-web/ui/src/menus/useClusterMenus.tsx index aa217811a..e0bd3e815 100644 --- a/app/crowdb-web/ui/src/menus/useClusterMenus.tsx +++ b/app/crowdb-web/ui/src/menus/useClusterMenus.tsx @@ -14,7 +14,12 @@ export function useClusterMenus({ readonly, managed, managementAuthorized, domai /** Build per-layer context menu items for a normalized target. */ return useCallback( (t: MenuTarget): MenuItemOrSeparator[] => { - if (readonly || managed || !managementAuthorized) return []; + // Existing service lifecycle actions must remain available while the + // authority probe is down. Stopping a KV service makes that probe + // unavailable, but restart/stop/delete are exactly the recovery actions + // needed to bring the authority back. + const isServiceTarget = t.type === 'Server' && t.serviceType != null; + if (readonly || managed || (!managementAuthorized && !isServiceTarget)) return []; const items: MenuItemOrSeparator[] = []; const p = t.parentIds || {}; if (physicalActive && t.type === 'Server' && isAuxiliaryKind(t.serviceType)) { diff --git a/app/crowdb-web/ui/src/shell/Sidebar.test.tsx b/app/crowdb-web/ui/src/shell/Sidebar.test.tsx index afdc59632..fe1987463 100644 --- a/app/crowdb-web/ui/src/shell/Sidebar.test.tsx +++ b/app/crowdb-web/ui/src/shell/Sidebar.test.tsx @@ -96,6 +96,19 @@ describe('Sidebar · Cluster tree projection', () => { expect(queryByText('PKV-11')).toBeNull(); }); + it('keeps a stopped registered PKV service visible for restart actions', () => { + const stoppedRack = { + ...rack, + nodes: rack.nodes.map(node => node.id === 10 ? { ...node, kv_server: undefined } : node), + }; + const { getByText } = renderSidebar(Domain.Cluster, { + racks: [stoppedRack], + servers: [], + allServers: [{ id: 'kv-10', node_id: 10, service_type: 'paxos-kv', health: 'down' }], + }); + expect(getByText('PKV-10', { exact: true })).toBeTruthy(); + }); + it('renders assigned disk groups and disks under the owning DiskDB service', () => { const { getByTestId, getByText } = renderSidebar(Domain.Cluster); const diskdbSubtree = getByTestId('tree-node-DDB-10'); diff --git a/app/crowdb-web/ui/src/shell/Sidebar.tsx b/app/crowdb-web/ui/src/shell/Sidebar.tsx index db23c312c..9ba14f0bf 100644 --- a/app/crowdb-web/ui/src/shell/Sidebar.tsx +++ b/app/crowdb-web/ui/src/shell/Sidebar.tsx @@ -140,7 +140,13 @@ export function Sidebar({ const kinds: ServiceKind[] = plan ? [...serviceOrder] : []; if (!plan) { for (const service of nodeServices) if (isAuxiliaryKind(service.service_type)) kinds.push(service.service_type); - if (serverByNodeId.has(nodeId)) kinds.push('paxos-kv'); + // Keep a registered KV service visible after it is stopped. The + // physical node projection may omit `kv_server` while the process + // is down, but `/api/servers` still owns the deployment record and + // the restart action must remain available for that record. + if (serverByNodeId.has(nodeId) || nodeServices.some(service => service.service_type === 'paxos-kv')) { + kinds.push('paxos-kv'); + } if (diskdbNodeIds?.has(nodeId)) kinds.push('diskdb'); } for (const kind of [...new Set(kinds)]) { diff --git a/app/crowdb-web/ui/src/views/S3View.tsx b/app/crowdb-web/ui/src/views/S3View.tsx index eb6c38056..cf0c0250a 100644 --- a/app/crowdb-web/ui/src/views/S3View.tsx +++ b/app/crowdb-web/ui/src/views/S3View.tsx @@ -122,7 +122,7 @@ export function S3View({ active, readonly, onChunk }: { active: boolean; readonl const response = await request('HEAD', objectPath(bucket, row.key)); setDetail(Object.fromEntries(response.headers.entries())); }, `Inspected ${row.key}`); }; - useEffect(() => { if (origin) void run(listBuckets, 'Buckets refreshed'); }, [origin]); + useEffect(() => { if (active && origin) void run(listBuckets, 'Buckets refreshed'); }, [active, origin]); useNavigationSnapshot(Domain.S3, 'object-query', () => { const state = { bucket, bucketStart, bucketFilter, prefix, listedPrefix, cursor, previous: [...previous], selected: selected ? { ...selected } : null, @@ -197,7 +197,7 @@ export function S3View({ active, readonly, onChunk }: { active: boolean; readonl const bytes = await previewBytes(response); setPreview(new TextDecoder('utf-8', { fatal: false }).decode(bytes)); }, 'Loaded first 4 KiB')}>Preview first 4 KiB - {!readonly && } + {!readonly && } } {error &&

{error}

} diff --git a/container/single-node-container/README.md b/container/single-node-container/README.md index c5fff2339..963514844 100644 --- a/container/single-node-container/README.md +++ b/container/single-node-container/README.md @@ -88,8 +88,11 @@ pixi run clean-container `start-container` builds the image, starts `crowdb-single-node`, exposes ports 9090–9094 on the container network, waits for readiness, and prints client credentials. Docker creates an anonymous data volume unless configured -otherwise. `inject-container` loads TPC-H SF1 into Iceberg and copies its -Parquet files to the container's S3 endpoint. The loader retains its data and +otherwise. `inject-container` loads TPC-H SF1 into native Iceberg and copies its +Parquet files to the container's S3 endpoint as an independent benchmark copy. +The S3 report's bucket and keys are not Iceberg locations or backing storage; +Iceberg manifests use the native listener's returned table-scoped URLs. +The loader retains its data and reports under `.crowdb-runtime/artifacts/tpc-loader/`. `clean-container` removes the named container, other exited containers created from the selected image, all untagged images that are not referenced by any container, and all diff --git a/doc/backlog/backlog.md b/doc/backlog/backlog.md index ebfe9e860..1a607f253 100644 --- a/doc/backlog/backlog.md +++ b/doc/backlog/backlog.md @@ -9,7 +9,7 @@ requirements and their temporary execution plans are removed during the final live in the linked document; implementation follows [`implement-requirement`](../../.agents/skills/implement-requirement/SKILL.md). -**Next R number: R231** — R221 is reserved by another workstream. +**Next R number: R235** — R231 through R233 are reserved by other workstreams. ## Dataset and access diff --git a/doc/design/access-server/iceberge/design-crowdb-iceberg-upload-flow.md b/doc/design/access-server/iceberge/design-crowdb-iceberg-upload-flow.md index db86e40f3..4dfbf222f 100644 --- a/doc/design/access-server/iceberge/design-crowdb-iceberg-upload-flow.md +++ b/doc/design/access-server/iceberge/design-crowdb-iceberg-upload-flow.md @@ -3,7 +3,7 @@ # CROWDB - Design: Iceberg File Upload Flow -This document defines the FileIO upload path used by Iceberg clients and records +This document defines native injection and the FileIO upload path used by Iceberg clients and records measurements from the single-node profile. It covers file publication, not table snapshot publication. @@ -19,7 +19,57 @@ Depends on: [Native Iceberg Storage](design-crowdb-iceberg.md) and ## 1. Client and server flow -The TPC loader hashes each local Parquet file through bounded reads, checks +### Native injection + +Native injection owns both byte ingestion and the subsequent construction of +Iceberg file references: + +1. Finish durable IcebergTable chunk writes and seal an owner-bound directory + root. Even a one-leaf native tree includes its file identity in that root. +2. Form `f/v1/..parquet` under the immutable + `///` prefix. Bind the locator's + unique file identity, length, descriptor and exact filename by signature. +3. Durably select the descriptor in the UUIDv7 injection operation before + publishing one `file/` record with create-only CAS. Return the one + `s3://iceberg//` URL after publication. A retry uses + the same operation/file identity and returns the selected URL; conflicts + never overwrite published content. +4. Construct standard manifests using that URL, write manifest lists and + metadata JSON, then publish the table head by its normal atomic commit. + Failed table commits leave unreferenced durable files for the existing + safe cleanup flow; they do not change a previously returned URL. + +Direct HEAD/GET/range requests validate cached permissions and the locator, +then read native storage without Iceberg file-location or file-state KV lookup. +The chunk layer still performs its normal replica/EC layout resolution. +HEAD and conditional reads probe the physical directory root; GET reuses the +verified directory in its pull reader. Reclaimed or reused roots fail closed. +List and cleanup still use the single file record. Logical deletion removes +the file from list; its known direct URL may read retained bytes until physical +GC. Reclaimed identities cannot alias newly allocated bytes. + +The native injection entry is a signed PUT with +`crowdb-native-upload=` on a table-scoped parquet upload URI. It requires +an explicit SHA-256 payload hash and returns one JSON `location` after durable +selection and publication. The operation binds its principal, full table path, +readable stem, payload digest and retry deadline. A missing publication is +completed from the selected receipt after response loss; a changed or deleted +publication cannot be replayed. The listener advertises the single-request byte +bound. Native loader parts stay within it; normal SDK multipart retains its +chosen logical URL. + +The loader journals the stable upload URI before transfer and the returned +location before adding standard manifest references. Its report preserves the +upload URI for recovery; it does not add a file-location field to Iceberg metadata. + +### Standard SDK uploads + +SDK location providers choose logical filenames before PUT. These uploads +retain that filename and resolve the single file record for reads. They do +not require a pre-upload reservation API, a second SDK location field, or +post-upload rewriting of manifests and path-based delete references. + +For a catalog without native injection, the TPC loader hashes each local Parquet file through bounded reads, checks that its unique target is absent, then sends a known-size S3 request using the vended native credentials. Below 256 MiB it uses a single PUT; larger files use bounded multipart parts aligned to complete frame payloads. The upload @@ -121,6 +171,13 @@ also counts repair-driven rotations, replayed bytes, and rotation time. ## 3. Integrity and durable publication +Large metadata JSON, manifest lists and manifests follow the same native +object durability rules. Each keeps one bounded file descriptor; complete +snapshot arrays and file bytes are not copied into TableHead or an unbounded +KV value. Public `metadata/` and `statistics/` prefixes remain available for +bounded listing without opening object payloads. Small inline metadata and +large chunk-backed metadata return identical canonical FileIO bytes. + The per-object OpenSSL digest state computes MD5 over ordered logical payload views. Small objects update it synchronously in the receive task; large objects use a checksum worker. It also computes SHA-256 when a signed payload requires it. It never diff --git a/doc/design/access-server/iceberge/design-crowdb-iceberg.md b/doc/design/access-server/iceberge/design-crowdb-iceberg.md index 945a47feb..e39bc5a66 100644 --- a/doc/design/access-server/iceberge/design-crowdb-iceberg.md +++ b/doc/design/access-server/iceberge/design-crowdb-iceberg.md @@ -62,6 +62,107 @@ head publication. No cross-generation cache or deduplication is implied. Iceberg metadata stores bounded logical records and opaque data references. Physical chunk placement and storage topology remain below the access boundary. +### Identity directory and file authority + +The Iceberg key codec carries its storage family and layout version separately +from the directory payload. A table-owned payload begins with +`///`; it has no redundant +`iceberg`, `namespace` or `table` labels before those IDs. The storage namespace +is frozen at table creation, including staged creation. The default namespace +has a durable ID under the same rules as named namespaces. + +```text +/ +├── head +├── namespace/name/ -> namespace-id +└── / + ├── head + ├── name/ -> (storage-namespace-id, table-id) + └── / + ├── head + ├── file/ + ├── projection// + └── operation//injection +``` + +Catalog and namespace control records have their own typed scopes. Namespace +names use the existing unambiguous multipart-name encoding. Table-name lookup +returns the complete storage identity; internal tasks carry it as well, so no +table-ID reverse index is needed. A move updates live name mappings and the +head's live identity/epoch while retaining the storage path. Deleting the old +namespace cannot cascade into moved tables, and IDs are never reassigned. + +Each file has one `file/` record. The suffix is also +its public object-key suffix. Its value contains FileId, canonical length, +format, integrity metadata, native descriptor/root and lifecycle state. FileId +is a value/locator binding, not another file index. One create-only CAS publishes +the complete record after durable bytes; exact-key conflicts cannot overwrite +selected content. Retries preserve the selected file identity and URL. Workflow +records exist only where recovery or multipart/GC progress requires them, not +as three mandatory records for every file. + +Service recovery journals are separate from that directory. Their existing +catalog control scopes remain explicitly versioned binary keys, even in a +native catalog. They are not file lookup indexes and cannot produce an object +listing. The service must discover abandoned work before a table head exists +and after a table has disappeared, without a table-ID reverse index: + +- Create, commit and lifecycle journals and payload pages are keyed by operation + identity; creation can precede name/head publication and rename spans two live + namespaces. Their selected heads carry the frozen storage identity. +- Multipart sessions and parts are keyed by upload identity/part number. Catalog + admission accounts for all sessions, including unpublished and expired uploads. + Every session and part carries and validates the full storage identity. +- Write intents/fences track allocated but potentially unpublished native + extents, including failed creation. Catalog maintenance scans their bounded + control scopes; the value binds the full table/file owner and exact chunk root. +- GC tasks, candidates, claims, proof nodes and pages belong to the recovery + worker. A retirement task can cover several tables and adopts interrupted + purge work. Claims coordinate physical deletion across tasks/generations; + candidate values retain the canonical filename and full storage identity. + +These records are retained only for retry, fencing and reclamation. They do not +add another published file authority or participate in direct GET/HEAD. GC data +discovery chooses v1 or v2 from the persisted catalog layout; independent +service-control scans never merge the two file namespaces. Head, file and +projection records use the native table directory. The native injection receipt +already knows its selected table and is stored under that directory. + +### Standard metadata and bounded scale + +TableHead selects an immutable standard metadata JSON file. Its `snapshots` +array contains snapshot records and its `refs` map selects branches/tags; +snapshot-log and metadata-log retain their standard meanings. Snapshot records +reference manifest-list files, which reference manifest files, which describe +data/delete files. Reading v1 also supports its direct `manifests` alternative. +The backed-up v1/v2/v3 specification determines fields, sequence inheritance, +schema/partition evolution, delete matching and row lineage. No authoritative +snapshot or manifest KV index substitutes for those standard file contents. + +All of these file kinds use the same single file-record model. Large metadata +JSON, manifests and manifest lists are native chunk-backed objects. KV values +and TableHead never embed an unbounded metadata document or snapshot array. +Small eligible metadata may use the bounded inline representation; larger +objects and large extent sets use chunk payloads and bounded directory roots. +Changing storage representation preserves the canonical bytes returned by +FileIO. Table publication selects a completely durable metadata generation by +CAS; a losing commit cannot expose a partially written generation. + +Metadata files use useful public relative prefixes such as +`metadata/.metadata.json`, `metadata/.avro` and +`statistics/.puffin`. These are defaults, not restrictions on safe +client-chosen paths. The optional generation-bound projections described above +accelerate metadata reads; they are not duplicate file authorities. Parser, +collection and response budgets remain explicit even for chunk-backed JSON. +An unsupported oversize request fails clearly rather than silently truncating +snapshots. ALL and REFS keep their standard response semantics; internal +projection paging does not introduce private REST pagination fields. + +New catalogs use the identity-directory layout version. Existing legacy +catalogs retain their own decoder, keys, URLs and read/write behavior. Layout +selection is catalog-bound; relabeling or mixing key versions in one scan is +rejected. In-place migration is not implicit in decoding. + ### Catalog foundation One active root selects a random stable CatalogId and activation epoch. Display @@ -254,20 +355,33 @@ Iceberg FileIO uses reserved S3-shaped locations so existing Iceberg clients can address immutable metadata and data files. The shape is a compatibility contract, not delegation to the general S3 authority. File publication, immutability, authorization, and deletion remain under Iceberg control. -Typed locations use lower-case unpadded base32 catalog IDs and lower-case hex -table IDs. Relative UTF-8 object keys preserve case, literal percent signs, plus +Public URIs use the fixed virtual bucket `iceberg`: +`s3://iceberg////`. +Native typed locations use lower-case fixed-width hexadecimal for all three +128-bit IDs. Legacy compatibility buckets retain their base32 catalog component. +Relative UTF-8 object keys preserve case, literal percent signs, plus signs and repeated internal slashes; the whole object key is bounded to 1,024 bytes. Dot traversal, leading slash, backslash, controls, query and fragment delimiters are rejected rather than normalized. HTTP percent decoding belongs only at the transport boundary, not in stored S3-shaped locations. Native `ListObjectsV2` retains this address model. A list request must specify -an explicit `t//` prefix and a table-bound delegated list capability; +the complete `///` prefix or a +descendant prefix and a table-bound delegated list capability; catalog-wide discovery and general S3 credentials are rejected. Reader and writer FileIO credentials include listing; cleanup-only credentials do not. Exact-object HEAD/GET/PUT remain independently usable. PyArrow missing-file probes may issue listing to distinguish a missing object from a directory. +Empty, catalog-only, namespace-only and foreign-table prefixes fail before a +scan. HeadBucket/GetBucketLocation expose only the configured virtual bucket +and region to a valid grant; ListBuckets returns AccessDenied, and bucket +creation/deletion is unsupported. The URI host determines the bucket. Arrow +FileSelector adds a trailing slash to its key prefix and uses `/` as delimiter +for nonrecursive listing; an exact HEAD miss may probe `/` without +widening scope. These behaviors follow the +[Arrow S3 implementation](https://raw.githubusercontent.com/apache/arrow/apache-arrow-25.0.0/cpp/src/arrow/filesystem/s3fs.cc). + Listing enumerates selected, fully published immutable file-location records, including files awaiting a table commit or later reclamation. It excludes incomplete uploads, unselected draft/losing candidates and logical deletion @@ -290,6 +404,15 @@ catalog epoch, table, prefix, delimiter and encoding. Their fixed expiry is no later than the originating grant, and invalid or foreign tokens fail before storage access. A credential refresh requires starting a new traversal. +List operates directly on `file/` records in public-key order. For example, +the public prefix `/metadata/` maps to the KV prefix +`/file/metadata/`; filename prefixes narrow that same range. +Pagination uses encoded prefix bounds and the last examined key, including +filtered tombstones and sparse pages. It never materializes or sorts the whole +table, opens metadata objects, scans unrelated file prefixes or enumerates +snapshots inside JSON. Returned keys omit the internal `file/` component. +The same limits apply whether a table has ten or millions of metadata files. + Native file records bind FileId to exact location, kind, format and canonical length. Legacy tree and inline records also bind a whole-file SHA-256 digest; streamed records bind a bounded array of complete Chunk locations and an HTTP ETag, and @@ -315,6 +438,67 @@ The isolated native HTTP surface exposes signed immutable object reads/writes, multipart operations and separately authorized single/batch cleanup, without a general S3 bucket authority. +### Native injection and direct reads + +Native injection finishes chunk writes and its bounded descriptor, publishes +one file record, then returns one canonical URL before constructing manifests. +For parquet, the public filename is +`f/v1/..parquet`. The URL-safe, dot-free locator carries +a unique FileId, length, integrity information and complete stable native +location or chunk-resident directory root. Its versioned signature binds the +full table identity, readable stem and unsigned locator fields; the signature +bytes are excluded from that signing input. Repeated readable stems are harmless; +an identical upload retry returns the original selected filename. Locator and +stem together must fit the object-key limit. + +Ordinary SDK writers choose logical filenames before S3 PUT and keep those +URLs. A PUT-time allocation does not change a client's chosen path. These +files resolve their one file record; no response field or manifest rewrite +converts them into direct locators. Metadata/Avro files may retain useful +logical prefixes regardless of their native object size. Position-delete and +deletion-vector path references always use the original canonical URL. + +Direct HEAD/GET/Range GET validate request permissions and locator authenticity +and read the native descriptor without Iceberg file-location or file-tombstone +KV queries. Chunk-layer layout resolution remains available for replica/EC +routing, cold reads, repair and migration. A KV-resolved opaque file token is not a direct +locator. Large files use bounded chunk-resident directory traversal, not an +unbounded URL. Native chunk routing preserves repair/failover without embedding +transient replica addresses in URLs. Reclaimed identities and reused extents +must never make an old locator return a different file's bytes. + +Native trees retain an owner-bound directory root even when they contain one +leaf. Its digest and encoded full file identity distinguish an identical-content +replacement at a reused extent. Direct request preflight checks that root before +HEAD or conditional success, and GET reuses its verified bytes in the pull reader. + +Authorization uses bounded validated grant/context caches and signature, +scope and expiry checks. An authorization cache miss may load authority; it +must not become a per-file location/state lookup. Expired entries are not +accepted during refresh failure. Cache entries bind principal, catalog epoch, +full storage identity and operation rights, with validity bounded by the grant +and applicable authority lease. Fresh credential issuance still checks live +authority. Persistent locators have no credential expiry; retain signing-key +verification material for their files' lifetime. A locator is not a bearer +permission. Cold/warm authorization costs and storage resolution counts are +measured separately. + +Native catalog authority persists the default locator verification/signing key, +independent of access tokens. Key version 1 is reserved for that catalog key; +renaming and capability activation preserve it. Additional signing versions are +configured through `iceberg.locator_signing`, with an active version and retained +verification keys. Configured version 1 must match the persisted catalog key. +Every serving node uses the same configured versions and retains them until the +corresponding files are physically reclaimed. A lock-free cache selects the +current catalog's key ring; clearing to a new catalog selects its new default key. +Native authority uses record envelope 2 so a legacy server cannot silently ignore +the storage layout. Legacy authorities retain envelope 1 and omitted layout +fields decode as the legacy layout. + +Credential refresh URLs carry the table ID and frozen storage namespace ID. +The listener resolves that immutable head directly, so refreshing an old endpoint +after a move or namespace recreation does not require a table-ID reverse index. + Legacy chunk-backed files use bounded leaf blocks and immutable chunk-resident directory pages, with at most 256 children per page and eight directory levels. Each page binds its catalog, table and file identity, child heights and covered byte count. @@ -503,7 +687,9 @@ Delegation tokens carry catalog activation epoch, table, principal fingerprint, nonce, exact operation set, issue/expiry times and independent request/file byte limits. Domain-separated HMAC authenticates bounded claims and derives per-grant S3 credential material without a mutable credential registry. Verification requires -a freshly checked Ready context. Ordinary FileIO grants exclude DELETE; +the operation's authorized context: writes and credential issuance check Ready +authority, while direct reads may reuse the bounded validated permission cache +described above. Ordinary FileIO grants exclude DELETE; explicit cleanup grants include the separate delete operation. These token primitives feed native request-signature verification through a request-local credential provider. Only the shared SigV4 algorithm is reused; general S3 @@ -513,7 +699,7 @@ expiry remains exact even when signature timestamps allow clock skew. Table credential issuance requires a matching Ready catalog authority and rejects lifetimes above its persisted delegated-access bound, independently of the signer's configured maximum. A zero persisted delegation bound disables issuance. -Callers still must freshly authorize the root and exact live table or draft; +Credential-issuance callers must freshly authorize the root and exact live table or draft; the serialization primitive does not perform those reads. The credential endpoint checks the current namespace and published table or exact unbound draft. Draft vending requires its original writer principal and a table-ID query selector; @@ -653,20 +839,24 @@ authorizes deletion. Committed files remain readable when new chunk allocation fails. Progress resumes after capacity is restored through the normal storage flow. Shared-chunk ranges remain pending while range deletion is unsupported. -Request entry checks catalog and table or draft authority without request-level -pins. Selected context and records flow through normal publication and reads; -PUT and GET do not perform deletion-fence or GC-claim GETs. Physical reclamation observes a -minimum retention interval that covers admitted request lifetimes. Once a file's canonical -logical deletion has started, its location-key tombstone rejects resolution -until cleanup state permits reuse, even if physical range reclamation is deferred. Legacy live tasks are retired without +Publication and logical-URL resolution check catalog and table or draft +authority without request-level pins. Direct reads instead use the validated +permission cache and authenticated locator. Physical reclamation observes the +configured retention interval. Logical deletion hides the file from listing; +its tombstone rejects logical-URL resolution but is not consulted by direct +HEAD/GET/Range GET. With a valid grant, an already known direct URL can still +read retained bytes until physical reclamation. Logical deletion is not an +instantaneous direct-URL revocation guarantee. Legacy live tasks are retired without further deletion, releasing an owned table fence. Retained and deferred candidates remain durable work for later inactive passes. An already authorized FileIO GET or HEAD can continue within the configured -retention interval after a logical deletion. A concurrent transition to -`Reclaiming` rejects new admission. Uploads and new table credentials require -a Ready table. Logical drop preserves admitted file reads until their bounded -response lifetime ends. +retention interval after logical deletion. Native storage arbitrates direct +read/reclamation races, returning the original bytes or a storage error, never +bytes from a reused allocation. Logical-URL admission rejects a reclaiming +record. Uploads and new credentials require a Ready table. After logical drop, +direct URLs remain readable under still-valid grants while bytes are retained; +physical purge and grant expiry bound that access. Retired catalog recovery scans system retry and management ledgers before file deletion and after the final file rescan. Pending or retained bindings stop the @@ -788,9 +978,11 @@ The native file endpoint exposes separately authorized S3 `DeleteObject` and credentials endpoint with `cleanup=true`; ordinary FileIO credentials cannot delete objects. The server scans retained table metadata and refuses a path that is referenced or whose reference status is uncertain. Successful deletion -records a logical tombstone, then queues bounded physical reclamation after -20 minutes. A new PUT may reuse the path only after the old file's GC candidate -is durable. Batch deletion returns individual errors inside HTTP 200, so callers +records a logical tombstone and removes list visibility, then queues bounded +physical reclamation after 20 minutes. Direct URLs remain readable until their +bytes are reclaimed, subject to grant validity. Their FileIds/locators are never +reused. A new logical-path PUT may reuse its path only after the old file's GC +candidate is durable. Batch deletion returns individual errors inside HTTP 200, so callers must inspect every item. > **Warning:** Before deleting, the client must stop and resolve every writer @@ -821,3 +1013,14 @@ must inspect every item. reclaimed solely because no retained snapshot refers to it; active cleanup handles expired multipart remnants and explicitly deleted files with durable owner checks. +- **ICE-I10 — Stable identity directory:** rename and cross-namespace move + preserve storage namespace, table path and every published file URL. +- **ICE-I11 — Single file authority:** one filename-keyed record selects each + published file; list and lifecycle use that record without secondary file indexes. +- **ICE-I12 — Direct native read:** a direct URL supplies its immutable native + descriptor without Iceberg file-location or file-state KV queries; native + chunk layout resolution and repair retain their normal behavior. + Logical deletion removes list visibility; physical GC ends byte availability. +- **ICE-I13 — Bounded metadata scale:** large metadata bytes live in native + objects, descriptors/head remain bounded, prefix listing reads bounded KV + pages, and optional projections cannot replace standard metadata authority. diff --git a/doc/doc_index.md b/doc/doc_index.md index a99bfec07..23029f563 100644 --- a/doc/doc_index.md +++ b/doc/doc_index.md @@ -42,7 +42,7 @@ Temporary plans live under `doc/working/`; flow analyses live under | `doc/design/kv/kv-write-flow-analysis.md` | KV write flow and optimization evidence. | | `doc/design/chunkio/chunkio-write-flow-analysis.md` | Chunk I/O large-write flow and benchmarks. | | `doc/design/chunkio/chunkio-small-io-flow-analysis.md` | Chunk I/O small-I/O flow and benchmarks. | -| `doc/design/access-server/iceberge/design-crowdb-iceberg-upload-flow.md` | Iceberg FileIO upload path and container measurements. | +| `doc/design/access-server/iceberge/design-crowdb-iceberg-upload-flow.md` | Native injection, SDK upload and container measurements. | | `doc/design/rpc/rpc-flow-analysis.md` | RPC flow, benchmarks, and performance history. | ## Dev Environment (`doc/dev/`) @@ -119,7 +119,7 @@ Temporary plans live under `doc/working/`; flow analyses live under | `doc/design/rpc/design-crowdb-rpc-diskdb-migration.md` | Diskdb RPC migration, rollout, connection lifetime. | | `doc/design/access-server/s3/design-crowdb-access-s3.md` | S3 namespace, publication, integrity, list, delete. | | `doc/design/access-server/s3/design-crowdb-access-s3-rdma.md` | Optional RDMA and direct-to-GPU data planes. | -| `doc/design/access-server/iceberge/design-crowdb-iceberg.md` | Native Iceberg authority, commits, files, and recovery. | +| `doc/design/access-server/iceberge/design-crowdb-iceberg.md` | Iceberg ID paths, direct files and metadata scale. | | `doc/design/access-server/iceberge/iceberg-table-spec-1.11.0.md` | Backed-up normative Iceberg table format specification. | | `doc/design/access-server/iceberge/iceberg-rest-catalog-open-api-1.11.0.yaml` | Backed-up normative Iceberg REST Catalog OpenAPI. | | `doc/design/access-server/dataset/design-crowdb-access-dataset.md` | Dataset HTTP, native topology access, and GPU delivery. | diff --git a/e2e-test/clients/console-ui/e2e/fixtures/defaultDeployments.ts b/e2e-test/clients/console-ui/e2e/fixtures/defaultDeployments.ts index 9c3859da1..930e6cc4c 100644 --- a/e2e-test/clients/console-ui/e2e/fixtures/defaultDeployments.ts +++ b/e2e-test/clients/console-ui/e2e/fixtures/defaultDeployments.ts @@ -28,7 +28,7 @@ export function observeDefaultDeployments(page: Page) { if (rejected) expect(rejected.status(), await rejected.text()).toBe(201); const completed = nodes.find(node => replies.get(`${node}:${kind}`)?.some(response => response.status() === 201)); if (completed !== undefined) return completed; - const timeout = process.platform === 'darwin' ? 30_000 : 3_000; + const timeout = process.platform === 'darwin' ? 30_000 : 10_000; const response = await page.waitForResponse(response => pending.has(key(response) ?? '') && (response.status() === 201 || !expectedWait(response)), { timeout }); expect(response.status(), await response.text()).toBe(201); @@ -36,7 +36,7 @@ export function observeDefaultDeployments(page: Page) { }, async verify(node: number, kind: string) { const identity = `${node}:${kind}`; - const timeout = process.platform === 'darwin' ? 30_000 : 3_000; + const timeout = process.platform === 'darwin' ? 30_000 : 10_000; const response = replies.get(identity)?.find(response => response.status() === 201) ?? await page.waitForResponse(response => key(response) === identity && response.status() === 201, { timeout }); for (const rejected of replies.get(identity) ?? []) { diff --git a/e2e-test/clients/console-ui/e2e/flows/71-s3-native.spec.ts b/e2e-test/clients/console-ui/e2e/flows/71-s3-native.spec.ts index 0a093cab8..945e21898 100644 --- a/e2e-test/clients/console-ui/e2e/flows/71-s3-native.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/71-s3-native.spec.ts @@ -24,7 +24,9 @@ test('S3 native multipart upload, HEAD, bounded preview and full round trip', as await step('native S3 DOM setup', async () => { await page.goto('/?domain=S3'); await expect(page.getByLabel('Access key', { exact: true })).toHaveCount(0); - await page.getByRole('navigation', { name: 'S3 buckets' }).getByRole('button', { name: bucket, exact: true }).click(); + const bucketButton = page.getByRole('navigation', { name: 'S3 buckets' }).getByRole('button', { name: bucket, exact: true }); + await expect(bucketButton).toBeVisible(); + await bucketButton.click(); await page.getByText('Bucket actions', { exact: true }).click(); await page.getByLabel('Object key', { exact: true }).fill(key); await page.getByLabel('Object file').setInputFiles({ name: 'multipart.txt', mimeType: 'text/plain', buffer: bytes }); diff --git a/e2e-test/clients/iceberg-java/src/main/java/TestIcebergFileOperations.java b/e2e-test/clients/iceberg-java/src/main/java/TestIcebergFileOperations.java index c0fed1ba6..7b82de346 100644 --- a/e2e-test/clients/iceberg-java/src/main/java/TestIcebergFileOperations.java +++ b/e2e-test/clients/iceberg-java/src/main/java/TestIcebergFileOperations.java @@ -31,7 +31,9 @@ public static void run(S3Client client, String prefix) { reject(403, "AccessDenied", () -> client.deleteObject(request -> request.bucket(bucket).key(key))); reject(400, "InvalidRequest", () -> client.putObject(request -> request.bucket(bucket) .key(root + "../escape.json"), RequestBody.fromBytes(bytes))); - String foreign = "t/" + (root.charAt(2) == '0' ? '1' : '0') + root.substring(3) + "metadata/foreign.json"; + int tableOffset = root.startsWith("t/") ? 2 : root.lastIndexOf('/', root.length() - 2) + 1; + String foreign = root.substring(0, tableOffset) + (root.charAt(tableOffset) == '0' ? '1' : '0') + + root.substring(tableOffset + 1) + "metadata/foreign.json"; reject(403, "AccessDenied", () -> client.putObject(request -> request.bucket(bucket).key(foreign), RequestBody.fromBytes(bytes))); if (!etag.equals(client.headObject(request -> request.bucket(bucket).key(key)).eTag()) diff --git a/e2e-test/clients/iceberg-python/injection.py b/e2e-test/clients/iceberg-python/injection.py new file mode 100644 index 000000000..90957da0c --- /dev/null +++ b/e2e-test/clients/iceberg-python/injection.py @@ -0,0 +1,90 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Native injection followed by unmodified PyIceberg and Arrow reads.""" + +import hashlib +import os +from pathlib import Path +import secrets +import tempfile +import time +import uuid + +from botocore.auth import S3SigV4Auth +from botocore.awsrequest import AWSRequest +from botocore.credentials import Credentials +import pyarrow as pa +import pyarrow.parquet as pq +from pyiceberg.catalog import load_catalog +from pyiceberg.schema import Schema +from pyiceberg.types import LongType, NestedField +import requests + + +def upload(table, data): + if os.environ.get("CROWDB_E2E_NATIVE_LOADER") == "1": + from crowdb_tpc_loader.native_upload import new_upload_id, upload as native_upload + + uri = table.location() + f"/data/selected-{new_upload_id()}.parquet" + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "selected.parquet" + path.write_bytes(data) + first = native_upload(table.io.properties, uri, path, len(data), 4096, lambda _: None) + second = native_upload(table.io.properties, uri, path, len(data), 4096, lambda _: None) + assert first[0] == second[0] + return first[0] + properties = table.io.properties + random = secrets.randbits(74) + identity = uuid.UUID(int=(int(time.time() * 1000) << 80) | (7 << 76) + | ((random >> 62) << 64) | (2 << 62) | (random & ((1 << 62) - 1))) + uri = table.location() + f"/data/selected-{identity}.parquet" + url = properties["s3.endpoint"].rstrip("/") + "/" + uri.removeprefix("s3://") + url += f"?crowdb-native-upload={identity}" + credentials = Credentials(properties["s3.access-key-id"], properties["s3.secret-access-key"], + properties["s3.session-token"]) + selected = None + for _ in range(2): + request = AWSRequest(method="PUT", url=url, data=data, + headers={"x-amz-content-sha256": hashlib.sha256(data).hexdigest()}) + S3SigV4Auth(credentials, "s3", "us-east-1").add_auth(request) + response = requests.put(url, data=data, headers=dict(request.headers), timeout=60) + response.raise_for_status() + location = response.json()["location"] + assert location.startswith(table.location() + "/f/v1/"), location + assert selected is None or selected == location + selected = location + return selected + + +def verify(config): + catalog = load_catalog("native", type="rest", uri=config["endpoint"], token="w" * 32) + identifier = ("direct_moved", "selected") + if config.get("verify_restart"): + table = catalog.load_table(identifier) + else: + table = catalog.create_table(("default", "selected"), Schema(NestedField(1, "id", LongType()))) + sink = pa.BufferOutputStream() + expected = pa.table({"id": pa.array([1, 2, 3], type=pa.int64())}) + pq.write_table(expected, sink) + location = upload(table, sink.getvalue().to_pybytes()) + table.add_files([location]) + original = table.scan().plan_files()[0].file.file_path + assert original == location + catalog.create_namespace("direct_moved") + catalog.rename_table(("default", "selected"), identifier) + catalog.drop_namespace("default") + catalog.create_namespace("default") + table = catalog.load_table(identifier) + assert table.scan().plan_files()[0].file.file_path == original + selected = table.scan().to_arrow() + assert selected.column("id").to_pylist() == [1, 2, 3] + if os.environ.get("CROWDB_E2E_NATIVE_LOADER") == "1": + import duckdb + + with duckdb.connect() as connection: + connection.register("selected", selected) + assert connection.execute("SELECT count(*), sum(id) FROM selected").fetchone() == (3, 6) + assert "/f/v1/" in table.scan().plan_files()[0].file.file_path + # PyIceberg's Arrow FileIO opens the returned URL and performs Parquet footer + # range reads. No additional location field is supplied to either SDK. + print("Native injection: retry, manifest registration, Arrow query, move and retained URL passed") diff --git a/e2e-test/clients/iceberg-python/listing.py b/e2e-test/clients/iceberg-python/listing.py index 9318ca6e4..0ca7624c0 100644 --- a/e2e-test/clients/iceberg-python/listing.py +++ b/e2e-test/clients/iceberg-python/listing.py @@ -2,6 +2,7 @@ # Licensed under the Apache License, Version 2.0. import json +import os import sys from urllib.parse import urlsplit @@ -9,10 +10,17 @@ import pyarrow.fs as fs import pyiceberg from pyiceberg.io.pyarrow import PyArrowFile +from injection import verify def main(): config = json.load(sys.stdin) + if config.get("verify_restart"): + verify(config) + if os.environ.get("CROWDB_E2E_NATIVE_LOADER_FLOW") == "1": + from native_loader import verify as verify_loader + verify_loader(config) + return endpoint = urlsplit(config["endpoint"]) filesystem = fs.S3FileSystem( access_key=config["access_key"], secret_key=config["secret_key"], @@ -33,6 +41,10 @@ def main(): names = sorted(file.path for file in selected if file.type == fs.FileType.File) assert names == sorted(prefix + suffix for suffix in ["a.json", "nested/b.json", "雪&<>+%.json"]), names print(f"PyIceberg {pyiceberg.__version__} / PyArrow {pyarrow.__version__}: exact creation, HEAD/GET and prefix discovery passed") + verify(config) + if os.environ.get("CROWDB_E2E_NATIVE_LOADER_FLOW") == "1": + from native_loader import verify as verify_loader + verify_loader(config) if __name__ == "__main__": diff --git a/e2e-test/clients/iceberg-python/native_loader.py b/e2e-test/clients/iceberg-python/native_loader.py new file mode 100644 index 000000000..6c0aadad3 --- /dev/null +++ b/e2e-test/clients/iceberg-python/native_loader.py @@ -0,0 +1,55 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Exercise the production loader's upload, report and commit dataflow.""" + +from decimal import Decimal +import json +from pathlib import Path +import tempfile + +import duckdb +import pyarrow as pa +import pyarrow.parquet as pq + +from crowdb_tpc_loader.backend import IcebergBackend +from crowdb_tpc_loader.loader import load_tables +from crowdb_tpc_loader.models import Options, ParquetPart, TableData +from crowdb_tpc_loader.report import RunReport +from crowdb_tpc_loader.security import Redactor + + +def verify(config): + report_path = Path(config["report_dir"]) / "native-load-report.json" + options = Options("load", "tpch", Decimal("0.01"), catalog_uri=config["endpoint"], + token="w" * 32, namespace=("native_loader",), report_file=report_path, + catalog_properties={"py-io-impl": "pyiceberg.io.pyarrow.PyArrowFileIO"}) + backend = IcebergBackend(options, Redactor([options.token])) + backend.connect() + if not config.get("verify_restart"): + backend.ensure_namespace() + report = RunReport(options, "native-container-fixture", backend.redactor) + report.paths.append(report_path) + report.data["generator"] = {"implementation": "acceptance-fixture", "version": "1"} + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + backend.set_staging(root) + schema = pa.schema([pa.field("id", pa.int64())]) + path = root / "part.parquet" + pq.write_table(pa.table({"id": [1, 2, 3]}, schema=schema), path) + part = ParquetPart(path, 3, path.stat().st_size, schema) + data = TableData("region", (part,), schema) + report.generated_table(data, root) + load_tables(backend, {data.name: data}, report, 4096, print, upload_workers=1) + assert report.table("region")["status"] == "succeeded", report.data + saved = json.loads(report_path.read_text()) + item = saved["tables"]["region"]["files"][0] + assert item["state"] == "registered", item + assert "/f/v1/" in item["remote_uri"] and "/data/" in item["upload_uri"] + assert item["remote_uri"] != item["upload_uri"] + table = backend.catalog.load_table((*options.namespace, "region")) + assert [task.file.file_path for task in table.scan().plan_files()] == [item["remote_uri"]] + selected = table.scan().to_arrow() + with duckdb.connect() as connection: + connection.register("selected", selected) + assert connection.execute("SELECT count(*), sum(id) FROM selected").fetchone() == (3, 6) + print("Production loader: selected URL report, standard commit and native SQL read passed") diff --git a/e2e-test/tests/access_iceberg/file_http/capacity.rs b/e2e-test/tests/access_iceberg/file_http/capacity.rs index 6a9a55012..7c269bf58 100644 --- a/e2e-test/tests/access_iceberg/file_http/capacity.rs +++ b/e2e-test/tests/access_iceberg/file_http/capacity.rs @@ -9,14 +9,14 @@ mod workspace; use crowdb_access_iceberg::{ catalog::{CatalogContext, CatalogRepository, CatalogStore, ClearBounds, ManagementPrivilege}, file::{ - location_key, ContentFormat, FileContent, FileIdentity, FileKind, FileLocation, FileReader, + canonical_file_key, ContentFormat, FileContent, FileIdentity, FileKind, FileLocation, FileReader, FileRecord, FileRepository, FileTreeWriter, NativeFileBlocks, TableLocation, }, gc::{GcLimits, GcPhase, GcRepository, GcStalledReason, GcTask, GcWorker}, key::{FileId, OperationId, TableId}, operation::{mutation_identity, ManagementAction, ManagementRequest, RequestIdentity}, record::StorageRecord, - table::{head_key, TableHead, TableLifecycle, TablePurgeTask}, + table::{storage_head_key, TableHead, TableLifecycle, TablePurgeTask}, }; use crowdb_chunk_client::{ChunkIoClient, ChunkIoClientConfig, SmallWritePolicy}; use crowdb_diskdb_client::{DiskdbClient, DiskdbClientError, DiskdbRpcTransport}; @@ -40,6 +40,7 @@ async fn full_simulated_disk_preserves_file_authority_then_recovers_after_compac let context = seed_catalog(&stack).await; let owner = FileIdentity { table: TableLocation { + storage_namespace: Some(crowdb_access_iceberg::key::NamespaceId::random()), catalog: context.catalog, table: TableId::random(), }, diff --git a/e2e-test/tests/access_iceberg/file_http/capacity/gc.rs b/e2e-test/tests/access_iceberg/file_http/capacity/gc.rs index a43eb0a59..2642dffeb 100644 --- a/e2e-test/tests/access_iceberg/file_http/capacity/gc.rs +++ b/e2e-test/tests/access_iceberg/file_http/capacity/gc.rs @@ -14,6 +14,7 @@ pub(super) async fn seed_gc_workspace_task( ) { let store = stack.store().await; let table = TableLocation { + storage_namespace: Some(crowdb_access_iceberg::key::NamespaceId::random()), catalog: context.catalog, table: TableId::random(), }; @@ -49,7 +50,7 @@ pub(super) async fn seed_gc_workspace_task( operation_fence: 2, pending_operation: Some(OperationId::random()), }; - let key = head_key(context.catalog, table.table).encode().unwrap(); + let key = storage_head_key(table).encode().unwrap(); let bytes = StorageRecord::TableHead(Box::new(head.clone())).encode().unwrap(); store .compare_exchange(&key, None, &bytes, mutation_identity(&key, None, &bytes)) @@ -127,7 +128,7 @@ pub(super) async fn assert_gc_workspace_stall( assert!(stack .store() .await - .get(&location_key(file).encode().unwrap()) + .get(&canonical_file_key(file).encode().unwrap()) .await .unwrap() .is_some()); @@ -158,7 +159,7 @@ pub(super) async fn assert_gc_workspace_recovered( assert!(stack .store() .await - .get(&location_key(file).encode().unwrap()) + .get(&canonical_file_key(file).encode().unwrap()) .await .unwrap() .is_none()); diff --git a/e2e-test/tests/access_iceberg/file_http/listing.rs b/e2e-test/tests/access_iceberg/file_http/listing.rs index 8c5400ef1..6455331c8 100644 --- a/e2e-test/tests/access_iceberg/file_http/listing.rs +++ b/e2e-test/tests/access_iceberg/file_http/listing.rs @@ -51,6 +51,7 @@ async fn native_list_objects_v2_serves_selected_files_and_rejects_foreign_scope( assert!(xml.contains(""), "{xml}"); assert!(xml.contains("data%2Fnested%2F"), "{xml}"); let foreign = crowdb_access_iceberg::file::TableLocation { + storage_namespace: table.storage_namespace, catalog: table.catalog, table: crowdb_access_iceberg::key::TableId::random(), }; @@ -85,13 +86,25 @@ async fn native_list_objects_v2_serves_selected_files_and_rejects_foreign_scope( #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[ignore = "requires the pinned PyIceberg/PyArrow environment and native storage"] async fn official_pyarrow_native_fileio_creates_exact_files_and_lists_table_prefixes() { - use std::io::Write as _; - use std::process::{Command, Stdio}; - let (_stack, _process, client, table) = setup().await; + let (stack, process, client, table) = setup().await; let config = serde_json::json!({ "endpoint": format!("http://{}", client.address), "bucket": table.bucket(), "prefix": table.object_prefix(), "access_key": client.credentials.access_key_id(), "secret_key": client.credentials.secret_access_key(), - "session_token": client.credentials.session_token() }); + "session_token": client.credentials.session_token(), + "report_dir": std::env::var("CROWDB_E2E_ARTIFACTS").unwrap() }); + run_python(config.clone()).await; + process.stop().await; + stack.restart_catalog().await; + let restarted = super::super::frontend::TestFrontend::start(13).await; + let mut config = config; + config["endpoint"] = format!("http://{}", restarted.address()).into(); + config["verify_restart"] = true.into(); + run_python(config).await; +} + +async fn run_python(config: serde_json::Value) { + use std::io::Write as _; + use std::process::{Command, Stdio}; let status = tokio::task::spawn_blocking(move || { let mut child = Command::new("timeout") .arg("60") diff --git a/e2e-test/tests/access_iceberg/file_http/setup.rs b/e2e-test/tests/access_iceberg/file_http/setup.rs index d19dcdcf2..a358402e3 100644 --- a/e2e-test/tests/access_iceberg/file_http/setup.rs +++ b/e2e-test/tests/access_iceberg/file_http/setup.rs @@ -89,6 +89,7 @@ pub(super) async fn setup_with_bounds_and_file_limit( let started = now_ms(); let credentials = issuer .issue(FileGrant { + storage_namespace: table.storage_namespace, context, table: table.table, principal: [7; 32], @@ -149,6 +150,7 @@ pub(super) async fn cleanup_client( client: client.client.clone(), credentials: issuer .issue(FileGrant { + storage_namespace: table.storage_namespace, context, table: table.table, principal: [9; 32], diff --git a/e2e-test/tests/access_iceberg/gc.rs b/e2e-test/tests/access_iceberg/gc.rs index 56bd3ef83..8ccf0e4bd 100644 --- a/e2e-test/tests/access_iceberg/gc.rs +++ b/e2e-test/tests/access_iceberg/gc.rs @@ -41,7 +41,12 @@ async fn authenticated_gc_controls_survive_separate_processes() { let stack = TestNativeStorage::new(); let (store, context, table) = seed_table(&stack).await; let identity = OperationId::random().to_string(); - let table_id = table.to_string(); + let table_id = crowdb_access_iceberg::key::StorageTablePath { + catalog: context.catalog, + storage_namespace: crowdb_access_iceberg::key::NamespaceId::from_bytes(table.as_bytes()).unwrap(), + table, + } + .to_string(); let catalog_id = context.catalog.to_string(); let denied = command(&stack, 'w', &["start-table", &identity, &table_id]); assert!(!denied.status.success()); diff --git a/e2e-test/tests/access_iceberg/gc/seed.rs b/e2e-test/tests/access_iceberg/gc/seed.rs index 983f95aa5..a5a863862 100644 --- a/e2e-test/tests/access_iceberg/gc/seed.rs +++ b/e2e-test/tests/access_iceberg/gc/seed.rs @@ -9,11 +9,11 @@ use crowdb_access_iceberg::{ catalog::{ CatalogContext, CatalogRepository, CatalogStore, ClearBounds, ManagementPrivilege, RoutedCatalogStore, }, - file::{file_key, ContentFormat, FileContent, FileKind, FileRecord, TableLocation}, + file::{canonical_file_key, ContentFormat, FileContent, FileKind, FileRecord, TableLocation}, key::{FileId, NamespaceId, OperationId, TableId}, operation::{mutation_identity, ManagementAction, ManagementRequest, RequestIdentity}, record::StorageRecord, - table::{head_key, TableHead, TableLifecycle, TablePurgeTask}, + table::{storage_head_key, TableHead, TableLifecycle, TablePurgeTask}, }; use sha2::{Digest, Sha256}; use std::sync::Arc; @@ -60,6 +60,7 @@ pub(super) async fn seed_table( pub(super) async fn seed_head(store: &RoutedCatalogStore, context: CatalogContext) -> TableId { let table = TableId::random(); let location = TableLocation { + storage_namespace: Some(NamespaceId::from_bytes(table.as_bytes()).unwrap()), catalog: context.catalog, table, }; @@ -80,7 +81,13 @@ pub(super) async fn seed_head(store: &RoutedCatalogStore, context: CatalogContex operation_fence: 1, pending_operation: None, }; - let key = head_key(context.catalog, table).encode().unwrap(); + let key = storage_head_key(TableLocation { + catalog: context.catalog, + storage_namespace: Some(NamespaceId::from_bytes(table.as_bytes()).unwrap()), + table, + }) + .encode() + .unwrap(); let bytes = StorageRecord::TableHead(Box::new(head)).encode().unwrap(); store .compare_exchange(&key, None, &bytes, mutation_identity(&key, None, &bytes)) @@ -90,7 +97,11 @@ pub(super) async fn seed_head(store: &RoutedCatalogStore, context: CatalogContex } pub(super) async fn tombstone_head(store: &RoutedCatalogStore, context: CatalogContext, table: TableId) { - let key = head_key(context.catalog, table); + let key = storage_head_key(TableLocation { + catalog: context.catalog, + storage_namespace: Some(NamespaceId::from_bytes(table.as_bytes()).unwrap()), + table, + }); let previous = store.get(&key.encode().unwrap()).await.unwrap().unwrap(); let StorageRecord::TableHead(mut head) = StorageRecord::decode(&key, &previous.bytes).unwrap() else { panic!("table head"); @@ -122,6 +133,7 @@ pub(super) async fn seed_files( let file = FileRecord { file: FileId::random(), location: TableLocation { + storage_namespace: Some(NamespaceId::from_bytes(table.as_bytes()).unwrap()), catalog: context.catalog, table, } @@ -134,7 +146,7 @@ pub(super) async fn seed_files( content: FileContent::select_inline(FileKind::Metadata, payload_json.as_bytes()).unwrap(), hint: None, }; - let key = file_key(context.catalog, file.file).encode().unwrap(); + let key = canonical_file_key(&file.location).encode().unwrap(); let bytes = StorageRecord::File(Box::new(file)).encode().unwrap(); store .compare_exchange(&key, None, &bytes, mutation_identity(&key, None, &bytes)) @@ -145,7 +157,11 @@ pub(super) async fn seed_files( pub(super) async fn seed_purge_marker(store: &RoutedCatalogStore, context: CatalogContext, table: TableId) { tombstone_head(store, context, table).await; - let key = head_key(context.catalog, table); + let key = storage_head_key(TableLocation { + catalog: context.catalog, + storage_namespace: Some(NamespaceId::from_bytes(table.as_bytes()).unwrap()), + table, + }); let head = store.get(&key.encode().unwrap()).await.unwrap().unwrap(); let StorageRecord::TableHead(head) = StorageRecord::decode(&key, &head.bytes).unwrap() else { panic!("table head"); diff --git a/e2e-test/tests/access_iceberg/native_files.rs b/e2e-test/tests/access_iceberg/native_files.rs index 9449e03cd..5b2051cad 100644 --- a/e2e-test/tests/access_iceberg/native_files.rs +++ b/e2e-test/tests/access_iceberg/native_files.rs @@ -42,6 +42,7 @@ async fn assert_checkpoint_intent( let intents = crowdb_access_iceberg::gc::GcStore::scan_gc( stack.store().await.as_ref(), crowdb_access_iceberg::gc::GcScan { + native: false, catalog: owner.table.catalog, scope: Some(crowdb_access_iceberg::key::CatalogScope::FileWriteIntent), prefix: owner.table.table.as_bytes().to_vec(), @@ -106,6 +107,7 @@ async fn native_file_tree_publication_and_ranges_survive_catalog_storage_restart let context = seed_root(&stack).await; let owner = FileIdentity { table: TableLocation { + storage_namespace: Some(crowdb_access_iceberg::key::NamespaceId::random()), catalog: context.catalog, table: TableId::random(), }, diff --git a/e2e-test/tests/access_s3/node_failure/recovery.rs b/e2e-test/tests/access_s3/node_failure/recovery.rs index 81b4eed16..d623ebd48 100644 --- a/e2e-test/tests/access_s3/node_failure/recovery.rs +++ b/e2e-test/tests/access_s3/node_failure/recovery.rs @@ -5,6 +5,7 @@ use crowdb_chunkdb_client::{ChunkdbClient, ChunkdbRpcTransport}; use crowdb_kv_client::{HardwareClient, RangeBindingClient, ServiceRegistryClient}; use crowdb_protocol::chunkdb::rpc::{ChunkType, ListChunksRequest, QueryChunkRequest, Strip, StripType}; use crowdb_protocol::common::{ChunkId, HwStatus}; +use std::collections::HashSet; use std::sync::Arc; use std::time::Duration; @@ -106,26 +107,59 @@ async fn ready(http: &reqwest::Client, endpoint: &str) { } pub(super) async fn storage_ready() { + let registry = ServiceRegistryClient::new(super::kv_client()); + let hardware = HardwareClient::new(super::kv_client()); let deadline = tokio::time::Instant::now() + Duration::from_secs(15); loop { let result = crowdb_diskio_client::DiskioClient::connect_with_clients( - ServiceRegistryClient::new(super::kv_client()), - HardwareClient::new(super::kv_client()), + registry.clone(), + hardware.clone(), crowdb_diskio_client::DiskioClientConfig::default(), ) .await; - match result { - Ok(client) if client.status().disks == 3 => return, - result => assert!( - tokio::time::Instant::now() < deadline, - "restarted DiskIO owners must publish all three disks: {:?}", - result.as_ref().map(|client| client.status().disks) - ), + let diskdb = diskdb_missing(®istry, &hardware).await; + if matches!(&result, Ok(client) if client.status().disks == 3) + && matches!(&diskdb, Ok(missing) if missing.is_empty()) + { + return; } + assert!( + tokio::time::Instant::now() < deadline, + "restarted storage topology is not ready: DiskIO={:?}, DiskDB={diskdb:?}", + result.as_ref().map(|client| client.status().disks) + ); tokio::time::sleep(Duration::from_millis(100)).await; } } +async fn diskdb_missing( + registry: &ServiceRegistryClient, + hardware: &HardwareClient, +) -> Result, String> { + let instances = registry + .read_all_diskdb_instances() + .await + .map_err(|error| format!("registry: {error}"))?; + let groups = hardware + .list_disk_groups() + .await + .map_err(|error| format!("hardware: {error}"))?; + let mut owned = HashSet::new(); + for (_, instance) in instances { + if instance.rpc_endpoint.is_empty() { + continue; + } + if let Some(diskdb) = instance.extra.and_then(|extra| extra.diskdb) { + owned.extend(diskdb.owned_dg_ids); + } + } + Ok(groups + .iter() + .map(|group| group.dg_id) + .filter(|group| !owned.contains(group)) + .collect()) +} + pub(super) async fn repaired(chunks: &[ChunkId]) { let kv = Arc::new(super::kv_client()); let client = ChunkdbClient::new( diff --git a/e2e-test/tests/access_s3/protocol_http.rs b/e2e-test/tests/access_s3/protocol_http.rs index 41d94ff81..0a93fd258 100644 --- a/e2e-test/tests/access_s3/protocol_http.rs +++ b/e2e-test/tests/access_s3/protocol_http.rs @@ -99,6 +99,7 @@ async fn write_iceberg(iceberg_addr: SocketAddr, seeds: &[String]) { let context = repository.status().await.unwrap().0.context; let credentials = issuer .issue(FileGrant { + storage_namespace: table.storage_namespace, context, table: table.table, principal: [7; 32], @@ -127,7 +128,14 @@ async fn write_iceberg(iceberg_addr: SocketAddr, seeds: &[String]) { table.file(&format!("data/{name}.bin")).unwrap().object_key() ); let put = file_client.send(Method::PUT, &path, "", &bytes, true).await; - assert_eq!(put.status(), 200, "{}", put.text().await.unwrap()); + if put.status() != 200 { + let status = put.status(); + let body = put + .text() + .await + .unwrap_or_else(|error| format!("")); + panic!("PUT {path} returned {status}: {body}"); + } let get = file_client.send(Method::GET, &path, "", b"", false).await; assert_eq!(get.status(), 200); assert_eq!(get.bytes().await.unwrap().as_ref(), bytes); diff --git a/lib/crowdb-access-iceberg/Cargo.toml b/lib/crowdb-access-iceberg/Cargo.toml index a0aa10ef5..0807448f2 100644 --- a/lib/crowdb-access-iceberg/Cargo.toml +++ b/lib/crowdb-access-iceberg/Cargo.toml @@ -14,6 +14,7 @@ workspace = true test-util = [] [dependencies] +arc-swap = "1.9" async-trait = "0.1" base64 = "0.22" bincode = "1.3" diff --git a/lib/crowdb-access-iceberg/src/catalog.rs b/lib/crowdb-access-iceberg/src/catalog.rs index 0fcaacdb5..030e9556d 100644 --- a/lib/crowdb-access-iceberg/src/catalog.rs +++ b/lib/crowdb-access-iceberg/src/catalog.rs @@ -14,7 +14,10 @@ pub(crate) use context::check_context; pub use deadline::ClearBounds; pub use repository::{CatalogError, CatalogRepository, ManagementPrivilege}; pub use root::{ActiveCatalogRecord, RootState}; -pub use state::{CatalogAuthority, CatalogContext, CatalogLifecycle, ClearTransition}; +pub use state::{ + CatalogAuthority, CatalogContext, CatalogLifecycle, CatalogLocatorKey, CatalogStorageLayout, + ClearTransition, +}; pub use storage::{ CasOutcome, CatalogStore, CatalogStoreOperationCounts, CatalogStoreOperationMeter, RoutedCatalogStore, StoreError, StoredValue, diff --git a/lib/crowdb-access-iceberg/src/catalog/repository.rs b/lib/crowdb-access-iceberg/src/catalog/repository.rs index 58f177c80..04b9371f8 100644 --- a/lib/crowdb-access-iceberg/src/catalog/repository.rs +++ b/lib/crowdb-access-iceberg/src/catalog/repository.rs @@ -272,7 +272,7 @@ impl CatalogRepository { .as_ref() .ok_or(ValidationError::Record)? .activated(request.capabilities.ok_or(ValidationError::Record)?)?, - _ => CatalogAuthority::new(candidate, request.display_name.clone())?, + _ => CatalogAuthority::new_native(candidate, request.display_name.clone())?, }; result.admission_bounds = bounds; Ok(ManagementOperation { diff --git a/lib/crowdb-access-iceberg/src/catalog/state.rs b/lib/crowdb-access-iceberg/src/catalog/state.rs index 27bb186d9..84f530fb1 100644 --- a/lib/crowdb-access-iceberg/src/catalog/state.rs +++ b/lib/crowdb-access-iceberg/src/catalog/state.rs @@ -42,8 +42,63 @@ pub enum CatalogLifecycle { Retired, } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[repr(u8)] +pub enum CatalogStorageLayout { + Legacy = 1, + Native = 2, +} + +#[derive(Clone, Copy, Eq, PartialEq)] +pub struct CatalogLocatorKey([u8; 32]); + +impl CatalogLocatorKey { + /// # Errors + /// Rejects an uninitialized persistent locator secret. + pub fn new(bytes: [u8; 32]) -> Result { + if bytes == [0; 32] { + return Err(ValidationError::Record); + } + Ok(Self(bytes)) + } + + #[must_use] + pub const fn bytes(self) -> [u8; 32] { + self.0 + } + + fn generate() -> Self { + use sha2::{Digest, Sha256}; + let mut digest = Sha256::new(); + digest.update(b"crowdb-native-locator-key-v1"); + digest.update(uuid::Uuid::new_v4().as_bytes()); + digest.update(uuid::Uuid::new_v4().as_bytes()); + Self(digest.finalize().into()) + } +} + +impl std::fmt::Debug for CatalogLocatorKey { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("CatalogLocatorKey([redacted])") + } +} + +impl CatalogStorageLayout { + /// # Errors + /// Rejects unrecognized persisted layouts without attempting a fallback. + pub fn from_version(version: u8) -> Result { + match version { + 1 => Ok(Self::Legacy), + 2 => Ok(Self::Native), + _ => Err(ValidationError::KeyVersion(version)), + } + } +} + #[derive(Clone, Debug, Eq, PartialEq)] pub struct CatalogAuthority { + pub storage_layout: CatalogStorageLayout, + pub locator_key: Option, pub catalog: CatalogId, pub display_name: String, pub name_generation: u64, @@ -58,6 +113,8 @@ impl CatalogAuthority { /// Rejects invalid display names; no table format support is enabled initially. pub fn new(catalog: CatalogId, display_name: String) -> Result { let authority = Self { + storage_layout: CatalogStorageLayout::Legacy, + locator_key: None, catalog, display_name, name_generation: 1, @@ -70,6 +127,17 @@ impl CatalogAuthority { Ok(authority) } + /// Creates an authority for a new catalog without relabeling existing data. + /// # Errors + /// Rejects invalid display names. + pub fn new_native(catalog: CatalogId, display_name: String) -> Result { + Ok(Self { + storage_layout: CatalogStorageLayout::Native, + locator_key: Some(CatalogLocatorKey::generate()), + ..Self::new(catalog, display_name)? + }) + } + /// # Errors /// Rejects malformed names, zero generations and inconsistent capabilities. pub fn validate(&self) -> Result<(), ValidationError> { @@ -77,6 +145,9 @@ impl CatalogAuthority { if self.name_generation == 0 || self.config_generation == 0 { return Err(ValidationError::Record); } + if (self.storage_layout == CatalogStorageLayout::Native) != self.locator_key.is_some() { + return Err(ValidationError::Record); + } self.admission_bounds.completion_deadline(0)?; self.capabilities.validate() } @@ -94,6 +165,8 @@ impl CatalogAuthority { .checked_add(1) .ok_or(ValidationError::GenerationExhausted)?; Ok(Self { + storage_layout: self.storage_layout, + locator_key: self.locator_key, catalog: self.catalog, display_name, name_generation, diff --git a/lib/crowdb-access-iceberg/src/catalog/storage.rs b/lib/crowdb-access-iceberg/src/catalog/storage.rs index 83711846c..96ac6a54a 100644 --- a/lib/crowdb-access-iceberg/src/catalog/storage.rs +++ b/lib/crowdb-access-iceberg/src/catalog/storage.rs @@ -11,7 +11,7 @@ use crowdb_protocol::chunk_kv::{ }; use crate::error::ValidationError; -use crate::key::{CatalogScope, IcebergKey, MAX_KEY_BYTES}; +use crate::key::{CatalogScope, IcebergKey, StorageKey}; use crate::record::MAX_RECORD_BYTES; #[derive(Debug, thiserror::Error)] @@ -155,11 +155,11 @@ impl RoutedCatalogStore { identity: ClientRequestId, ) -> Result { if matches!( - IcebergKey::decode(key)?, - IcebergKey::System { + StorageKey::decode(key)?, + StorageKey::Legacy(IcebergKey::System { scope: crate::key::SystemScope::ActiveRoot, .. - } + }) ) { return Err(ValidationError::Key.into()); } @@ -200,14 +200,8 @@ impl RoutedCatalogStore { let (Some(start), Some(end)) = (&request.start, &request.end) else { return Err(ValidationError::Key.into()); }; - if start.as_slice() < b"ICE\0\x01".as_slice() - || end.as_slice() > b"ICE\0\x02".as_slice() - || start >= end - || start.len() > MAX_KEY_BYTES - || end.len() > MAX_KEY_BYTES - { - return Err(ValidationError::Key.into()); - } + let range = start.clone()..end.clone(); + StorageKey::validate_range(&range)?; if request.max_items == 0 || request.max_items > 256 || request.max_bytes == 0 @@ -226,7 +220,10 @@ impl RoutedCatalogStore { if value.revision == 0 { return Err(StoreError::Response); } - IcebergKey::decode(&value.key)?; + StorageKey::decode(&value.key)?; + if !range.contains(&value.key) { + return Err(StoreError::Response); + } validate_value(&value.value)?; } Ok(page) @@ -246,7 +243,7 @@ impl CatalogStore for RoutedCatalogStore { } async fn get(&self, key: &[u8]) -> Result, StoreError> { - IcebergKey::decode(key)?; + StorageKey::decode(key)?; self.counters.get.fetch_add(1, Ordering::Relaxed); count_request(|counters| &counters.get); let response = self.client.get(key.to_vec(), None).await?; @@ -263,7 +260,7 @@ impl CatalogStore for RoutedCatalogStore { value: &[u8], identity: ClientRequestId, ) -> Result { - IcebergKey::decode(key)?; + StorageKey::decode(key)?; validate_value(value)?; let operation = match expected { Some(expected) => { diff --git a/lib/crowdb-access-iceberg/src/commit/create/operation.rs b/lib/crowdb-access-iceberg/src/commit/create/operation.rs index db9ad40e9..1a564a280 100644 --- a/lib/crowdb-access-iceberg/src/commit/create/operation.rs +++ b/lib/crowdb-access-iceberg/src/commit/create/operation.rs @@ -80,6 +80,7 @@ impl TableCreateOperation { #[must_use] pub fn mapping(&self, state: TableMappingState) -> TableMapping { TableMapping { + storage_namespace: self.candidate.metadata_location.table().storage_namespace, catalog: self.context.catalog, namespace: self.candidate.namespace, name: self.candidate.name.clone(), diff --git a/lib/crowdb-access-iceberg/src/commit/create/publisher/completion.rs b/lib/crowdb-access-iceberg/src/commit/create/publisher/completion.rs index 2e5caa74e..463242f7a 100644 --- a/lib/crowdb-access-iceberg/src/commit/create/publisher/completion.rs +++ b/lib/crowdb-access-iceberg/src/commit/create/publisher/completion.rs @@ -6,7 +6,7 @@ use crate::{ namespace::authority_key, operation::mutation_identity, record::StorageRecord, - table::{head_key, name_key, TableMappingState}, + table::{name_key, storage_head_key, TableMappingState}, }; impl TableCreator { @@ -16,7 +16,7 @@ impl TableCreator { self.check_admission(operation).await?; } let head = &operation.candidate; - let key = head_key(head.catalog, head.table).encode()?; + let key = storage_head_key(head.storage_location()).encode()?; let bytes = StorageRecord::TableHead(Box::new(head.clone())).encode()?; let result = self .store @@ -98,7 +98,7 @@ impl TableCreator { .map_err(CatalogError::from)?; } else if operation.phase == Phase::Complete { let head = &operation.candidate; - let key = head_key(head.catalog, head.table).encode()?; + let key = storage_head_key(head.storage_location()).encode()?; let before = StorageRecord::TableHead(Box::new(head.clone())).encode()?; let mut settled = head.clone(); settled.pending_operation = None; diff --git a/lib/crowdb-access-iceberg/src/commit/create/publisher/preparation.rs b/lib/crowdb-access-iceberg/src/commit/create/publisher/preparation.rs index ec182c9d8..7d29d67ea 100644 --- a/lib/crowdb-access-iceberg/src/commit/create/publisher/preparation.rs +++ b/lib/crowdb-access-iceberg/src/commit/create/publisher/preparation.rs @@ -5,6 +5,29 @@ use super::{ }; impl TableCreator { + async fn storage_layout( + &self, + context: crate::catalog::CatalogContext, + ) -> Result { + let authority_key = crate::key::IcebergKey::Catalog { + catalog: context.catalog, + scope: crate::key::CatalogScope::Authority, + suffix: Vec::new(), + }; + let value = self + .store + .get(&authority_key.encode()?) + .await + .map_err(CatalogError::from)? + .ok_or(CatalogError::Uninitialized)?; + let crate::record::StorageRecord::Authority(authority) = + crate::record::StorageRecord::decode(&authority_key, &value.bytes)? + else { + return Err(ValidationError::Record.into()); + }; + Ok(authority.storage_layout) + } + pub(super) async fn prepare( &self, request: &TableCreationRequest, @@ -14,10 +37,20 @@ impl TableCreator { if decoded.stage_create() != expires_ms.is_some() { return Err(Error::Unsupported("staged table creation")); } - let namespace = NamespaceRepository::from_parts(self.store.clone(), self.names.clone()) - .load(request.context, &request.namespace) - .await? - .ok_or(Error::NamespaceMissing)?; + let layout = self.storage_layout(request.context).await?; + let repository = NamespaceRepository::from_parts(self.store.clone(), self.names.clone()); + let namespace = if layout == crate::catalog::CatalogStorageLayout::Native + && request.namespace.components() == ["default"] + { + repository + .ensure_default(request.context, request.identity.issued_ms) + .await? + } else { + repository + .load(request.context, &request.namespace) + .await? + .ok_or(Error::NamespaceMissing)? + }; if let Some(expiry) = expires_ms { if request.timestamp_ms < 0 || expiry <= request.timestamp_ms { return Err(ValidationError::Record.into()); @@ -41,7 +74,13 @@ impl TableCreator { .entry("created-at") .or_insert(serde_json::Value::String(created)); } - let target = target(request, namespace.namespace, decoded.name(), expires_ms.is_some())?; + let target = target( + request, + namespace.namespace, + decoded.name(), + expires_ms.is_some(), + layout, + )?; let initial = evaluate_table_creation(&decoded, target, request.timestamp_ms, limits())?; let response = if expires_ms.is_some() { super::staged::stage_response(initial.document.canonical())? @@ -96,8 +135,13 @@ fn target( namespace: crate::key::NamespaceId, name: &str, staged: bool, + layout: crate::catalog::CatalogStorageLayout, ) -> Result { let table = TableLocation { + storage_namespace: match layout { + crate::catalog::CatalogStorageLayout::Legacy => None, + crate::catalog::CatalogStorageLayout::Native => Some(namespace), + }, catalog: request.context.catalog, table: if staged { TableId::from_bytes(request.identity.operation.as_bytes())? diff --git a/lib/crowdb-access-iceberg/src/commit/create/publisher/reservation.rs b/lib/crowdb-access-iceberg/src/commit/create/publisher/reservation.rs index b3508b1aa..f7df7c271 100644 --- a/lib/crowdb-access-iceberg/src/commit/create/publisher/reservation.rs +++ b/lib/crowdb-access-iceberg/src/commit/create/publisher/reservation.rs @@ -5,7 +5,7 @@ use crate::{ error::ValidationError, operation::mutation_identity, record::StorageRecord, - table::{head_key, name_key, TableMappingState}, + table::{name_key, storage_head_key, TableMappingState}, }; impl TableCreator { @@ -59,7 +59,7 @@ impl TableCreator { Box::pin(self.resume_with_budget(operation.context, existing.operation, budget)).await?; return Ok(()); } - let head_key = head_key(existing.catalog, existing.table); + let head_key = storage_head_key(existing.storage_location()); let value_head = self .store .get(&head_key.encode()?) diff --git a/lib/crowdb-access-iceberg/src/commit/direct.rs b/lib/crowdb-access-iceberg/src/commit/direct.rs index aa2a13cf7..3d58f9c4b 100644 --- a/lib/crowdb-access-iceberg/src/commit/direct.rs +++ b/lib/crowdb-access-iceberg/src/commit/direct.rs @@ -8,7 +8,8 @@ use crate::{ operation::{mutation_identity, RequestIdentity}, record::StorageRecord, table::{ - head_key, read_table_metadata_document, SelectedTable, TableHead, TableLifecycle, TableMetadataError, + read_table_metadata_document, storage_head_key, SelectedTable, TableHead, TableLifecycle, + TableMetadataError, }, }; @@ -115,7 +116,7 @@ pub async fn publish_direct_commit( } Err(error) => return Err(error.into()), } - let key = head_key(before.catalog, before.table).encode()?; + let key = storage_head_key(before.storage_location()).encode()?; let expected = StorageRecord::TableHead(Box::new(before.clone())).encode()?; let value = StorageRecord::TableHead(Box::new(candidate.clone())).encode()?; let outcome = store @@ -132,7 +133,7 @@ pub async fn publish_direct_commit( CasOutcome::Conflict(Some(observed)) if observed.bytes == value => Ok(body), CasOutcome::Conflict(Some(observed)) => { let StorageRecord::TableHead(current) = - StorageRecord::decode(&head_key(before.catalog, before.table), &observed.bytes)? + StorageRecord::decode(&storage_head_key(before.storage_location()), &observed.bytes)? else { return Err(ValidationError::Record.into()); }; diff --git a/lib/crowdb-access-iceberg/src/commit/files/fence.rs b/lib/crowdb-access-iceberg/src/commit/files/fence.rs index 4d800e678..b555a8421 100644 --- a/lib/crowdb-access-iceberg/src/commit/files/fence.rs +++ b/lib/crowdb-access-iceberg/src/commit/files/fence.rs @@ -4,7 +4,7 @@ use crate::{ catalog::{CatalogContext, CatalogError, CatalogStore}, commit::{PriorManifestSource, TableCreateJournal, TableCreateOperation}, record::StorageRecord, - table::{head_key, name_key, TableMappingState, TableRepository}, + table::{name_key, storage_head_key, TableMappingState, TableRepository}, }; pub(super) enum CandidateFence { @@ -47,7 +47,7 @@ impl CandidateFence { if StorageRecord::decode(&key, &value.bytes)? != StorageRecord::TableMapping(mapping) { return Err(CatalogError::Conflict); } - let head = head_key(context.catalog, operation.candidate.table).encode()?; + let head = storage_head_key(operation.candidate.storage_location()).encode()?; if store.get(&head).await?.is_some() || journal .load(context, operation.identity.operation) diff --git a/lib/crowdb-access-iceberg/src/commit/files/snapshots.rs b/lib/crowdb-access-iceberg/src/commit/files/snapshots.rs index 9024509e0..7504ab16b 100644 --- a/lib/crowdb-access-iceberg/src/commit/files/snapshots.rs +++ b/lib/crowdb-access-iceberg/src/commit/files/snapshots.rs @@ -48,7 +48,7 @@ impl CandidateFileSource { { return Err(Error::Binding); } - self.validate_snapshot_set(Some(prior), &limits).await + Box::pin(self.validate_snapshot_set(Some(prior), &limits)).await } /// Validates all initial snapshots, including parent-child DV checks within a staged transaction. @@ -61,7 +61,7 @@ impl CandidateFileSource { if self.fence.prior().is_some() { return Err(Error::Binding); } - self.validate_snapshot_set(None, &limits).await + Box::pin(self.validate_snapshot_set(None, &limits)).await } async fn validate_snapshot_set( diff --git a/lib/crowdb-access-iceberg/src/commit/journal.rs b/lib/crowdb-access-iceberg/src/commit/journal.rs index 8fa0450c5..0a737073e 100644 --- a/lib/crowdb-access-iceberg/src/commit/journal.rs +++ b/lib/crowdb-access-iceberg/src/commit/journal.rs @@ -7,7 +7,7 @@ use crate::{ key::{CatalogScope, IcebergKey, OperationId}, operation::{mutation_identity, PayloadStore}, record::StorageRecord, - table::head_key, + table::storage_head_key, }; /// CAS journal for update intents; it neither validates files nor mutates table heads. @@ -147,7 +147,7 @@ impl TableCommitJournal { operation: &TableCommitOperation, next: TableCommitPhase, ) -> Result<(), CatalogError> { - let key = head_key(operation.before.catalog, operation.before.table); + let key = storage_head_key(operation.before.storage_location()); let value = self.store.get(&key.encode()?).await?.ok_or(CatalogError::Busy)?; let StorageRecord::TableHead(head) = StorageRecord::decode(&key, &value.bytes)? else { return Err(ValidationError::Record.into()); diff --git a/lib/crowdb-access-iceberg/src/commit/publication/completion.rs b/lib/crowdb-access-iceberg/src/commit/publication/completion.rs index b0f4c890c..d342a3c60 100644 --- a/lib/crowdb-access-iceberg/src/commit/publication/completion.rs +++ b/lib/crowdb-access-iceberg/src/commit/publication/completion.rs @@ -6,7 +6,7 @@ use crate::{ file::FileRepository, operation::{mutation_identity, PayloadStore, MAX_PAYLOAD_BYTES}, record::StorageRecord, - table::{head_key, read_table_metadata_document, SelectedTable, TableMetadataLimits}, + table::{read_table_metadata_document, storage_head_key, SelectedTable, TableMetadataLimits}, }; impl Publisher { @@ -14,7 +14,7 @@ impl Publisher { &self, operation: &TableCommitOperation, ) -> Result, Error> { - let key = head_key(operation.before.catalog, operation.before.table); + let key = storage_head_key(operation.before.storage_location()); let value = self .store .get(&key.encode()?) @@ -85,7 +85,7 @@ impl Publisher { let candidate = operation.candidate.as_ref().ok_or(ValidationError::Record)?; let prepared_body = self.success_body(operation).await?; self.current(operation).await?; - let key = head_key(candidate.catalog, candidate.table).encode()?; + let key = storage_head_key(candidate.storage_location()).encode()?; let before = StorageRecord::TableHead(Box::new(operation.before.clone())).encode()?; let after = StorageRecord::TableHead(Box::new(candidate.clone())).encode()?; let result = self @@ -155,7 +155,7 @@ impl Publisher { let candidate = operation.candidate.as_ref().ok_or(ValidationError::Record)?; let mut settled = candidate.clone(); settled.pending_operation = None; - let key = head_key(candidate.catalog, candidate.table).encode()?; + let key = storage_head_key(candidate.storage_location()).encode()?; let before = StorageRecord::TableHead(Box::new(candidate.clone())).encode()?; let after = StorageRecord::TableHead(Box::new(settled)).encode()?; self.store diff --git a/lib/crowdb-access-iceberg/src/file.rs b/lib/crowdb-access-iceberg/src/file.rs index 4c56eb0e3..d2b45ad19 100644 --- a/lib/crowdb-access-iceberg/src/file.rs +++ b/lib/crowdb-access-iceberg/src/file.rs @@ -7,6 +7,9 @@ mod content; mod credentials; mod deletion_vector; mod digest; +mod direct_location; +mod injection; +pub use injection::{NativeInjection, NativeInjectionRepository, NativeInjectionRequest}; mod directory; mod format; mod json; @@ -21,6 +24,9 @@ mod multipart_list; mod multipart_recovery; mod multipart_repository; mod multipart_selection; +mod native_location; +pub(crate) use native_location::validate_native_relative_key; +pub use native_location::{NativeFileLocation, NATIVE_ICEBERG_BUCKET}; mod parquet; pub use parquet::ParquetPageLimits; pub(crate) use parquet::{ParquetColumnReader, ParquetColumnValue}; @@ -53,12 +59,13 @@ pub use deletion_vector::{ DeletionVectorPositions, DeletionVectorReference, DeletionVectorStats, }; pub use digest::FileDigest; +pub use direct_location::DirectFileLocations; pub use directory::{ChunkDirectory, ChunkEntry, FileIdentity, MAX_DIRECTORY_ENTRIES}; pub use format::{ probe_orc_footer, probe_parquet_footer, probe_puffin_footer, FormatProbeError, OrcFooter, PuffinFooter, }; pub use json::{JsonSealError, JsonSealer}; -pub use key::{file_key, location_key}; +pub use key::{canonical_file_key, file_key, location_key}; pub(crate) use location::validate_relative_key; pub use location::{FileLocation, TableLocation, MAX_OBJECT_KEY_BYTES}; pub use multipart::{ diff --git a/lib/crowdb-access-iceberg/src/file/blocks.rs b/lib/crowdb-access-iceberg/src/file/blocks.rs index 880878bb8..fbf9e420d 100644 --- a/lib/crowdb-access-iceberg/src/file/blocks.rs +++ b/lib/crowdb-access-iceberg/src/file/blocks.rs @@ -187,6 +187,9 @@ impl FileBlockStore for NativeFileBlocks { } let mut key = b"iceberg-file-block-v1".to_vec(); key.extend_from_slice(owner.table.catalog.as_bytes()); + if let Some(namespace) = owner.table.storage_namespace { + key.extend_from_slice(namespace.as_bytes()); + } key.extend_from_slice(owner.table.table.as_bytes()); key.extend_from_slice(owner.file.as_bytes()); let mut writer = self.client.prepare_small_write_for_key(bytes.len(), &key).await?; diff --git a/lib/crowdb-access-iceberg/src/file/credentials.rs b/lib/crowdb-access-iceberg/src/file/credentials.rs index 75ffe08ea..42bcbc040 100644 --- a/lib/crowdb-access-iceberg/src/file/credentials.rs +++ b/lib/crowdb-access-iceberg/src/file/credentials.rs @@ -1,5 +1,5 @@ use crate::catalog::CatalogContext; -use crate::key::{OperationId, TableId}; +use crate::key::{NamespaceId, OperationId, TableId}; use super::FileLocation; @@ -53,6 +53,7 @@ impl FileOperations { #[derive(Clone, Debug, Eq, PartialEq)] pub struct FileGrant { + pub storage_namespace: Option, pub context: CatalogContext, pub table: TableId, pub principal: [u8; 32], @@ -76,6 +77,7 @@ impl FileGrant { ) -> Result<(), FileGrantError> { if location.table().catalog != self.context.catalog || location.table().table != self.table + || location.table().storage_namespace != self.storage_namespace || !self.operations.allows(operation) { return Err(FileGrantError::Forbidden); @@ -117,6 +119,7 @@ pub enum FileGrantError { Bounds, } +#[derive(Clone)] pub struct FileCredentials { grant: FileGrant, access_key_id: String, diff --git a/lib/crowdb-access-iceberg/src/file/credentials/token.rs b/lib/crowdb-access-iceberg/src/file/credentials/token.rs index de71c69e5..1932900d1 100644 --- a/lib/crowdb-access-iceberg/src/file/credentials/token.rs +++ b/lib/crowdb-access-iceberg/src/file/credentials/token.rs @@ -1,11 +1,13 @@ +use arc_swap::ArcSwap; use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; use data_encoding::BASE32_NOPAD; use hmac::{Hmac, Mac}; use sha2::Sha256; +use std::{collections::BTreeMap, sync::Arc}; use subtle::ConstantTimeEq; use crate::catalog::CatalogContext; -use crate::key::{CatalogId, OperationId, TableId}; +use crate::key::{CatalogId, NamespaceId, OperationId, TableId}; use super::{FileCredentials, FileGrant, FileGrantError, FileOperations}; @@ -17,6 +19,7 @@ type Signer = Hmac; pub struct FileGrantIssuer { key: [u8; 32], max_ttl_ms: u64, + validated: ArcSwap>, } impl FileGrantIssuer { @@ -26,7 +29,11 @@ impl FileGrantIssuer { if key == [0; 32] || max_ttl_ms == 0 { return Err(FileGrantError::Invalid); } - Ok(Self { key, max_ttl_ms }) + Ok(Self { + key, + max_ttl_ms, + validated: ArcSwap::from_pointee(BTreeMap::new()), + }) } /// # Errors @@ -77,27 +84,41 @@ impl FileGrantIssuer { context: CatalogContext, now_ms: u64, ) -> Result { - if session_token.len() != TOKEN_CHARACTERS { + if ![TOKEN_CHARACTERS, 228].contains(&session_token.len()) { return Err(FileGrantError::Invalid); } + if let Some(cached) = self.validated.load().get(session_token) { + validate_request(&cached.grant, context, now_ms)?; + return Ok(cached.clone()); + } let bytes = URL_SAFE_NO_PAD .decode(session_token) .map_err(|_| FileGrantError::Invalid)?; - if bytes.len() != TOKEN_BYTES { + if ![TOKEN_BYTES, TOKEN_BYTES + 16].contains(&bytes.len()) { return Err(FileGrantError::Invalid); } - self.signer(b"iceberg-file-grant-v1", &bytes[..CLAIM_BYTES]) - .verify_slice(&bytes[CLAIM_BYTES..]) + let claim_bytes = bytes.len() - 32; + self.signer(b"iceberg-file-grant-v1", &bytes[..claim_bytes]) + .verify_slice(&bytes[claim_bytes..]) .map_err(|_| FileGrantError::Invalid)?; - let grant = decode(&bytes[..CLAIM_BYTES])?; + let grant = decode(&bytes[..claim_bytes])?; grant.validate(self.max_ttl_ms)?; - if grant.context != context { - return Err(FileGrantError::Forbidden); - } - if now_ms < grant.issued_ms || now_ms >= grant.expires_ms { - return Err(FileGrantError::Expired); - } - Ok(self.credentials(grant, &bytes)) + validate_request(&grant, context, now_ms)?; + let credentials = self.credentials(grant, &bytes); + self.remember(&credentials, now_ms); + Ok(credentials) + } + + fn remember(&self, credentials: &FileCredentials, now_ms: u64) { + self.validated.rcu(|current| { + let mut next = (**current).clone(); + next.retain(|_, value| value.grant.expires_ms > now_ms); + while next.len() >= 64 { + next.pop_first(); + } + next.insert(credentials.session_token.clone(), credentials.clone()); + Arc::new(next) + }); } fn signer(&self, domain: &[u8], bytes: &[u8]) -> Signer { @@ -125,9 +146,19 @@ impl FileGrantIssuer { } } +fn validate_request(grant: &FileGrant, context: CatalogContext, now_ms: u64) -> Result<(), FileGrantError> { + if grant.context != context { + return Err(FileGrantError::Forbidden); + } + if now_ms < grant.issued_ms || now_ms >= grant.expires_ms { + return Err(FileGrantError::Expired); + } + Ok(()) +} + fn encode(grant: &FileGrant) -> Vec { let mut bytes = Vec::with_capacity(TOKEN_BYTES); - bytes.push(1); + bytes.push(if grant.storage_namespace.is_some() { 2 } else { 1 }); bytes.extend_from_slice(grant.context.catalog.as_bytes()); bytes.extend_from_slice(&grant.context.activation_epoch.to_be_bytes()); bytes.extend_from_slice(grant.table.as_bytes()); @@ -138,11 +169,15 @@ fn encode(grant: &FileGrant) -> Vec { bytes.extend_from_slice(&grant.operations.0.to_be_bytes()); bytes.extend_from_slice(&grant.max_request_bytes.to_be_bytes()); bytes.extend_from_slice(&grant.max_file_bytes.to_be_bytes()); + if let Some(namespace) = grant.storage_namespace { + bytes.extend_from_slice(namespace.as_bytes()); + } bytes } fn decode(bytes: &[u8]) -> Result { - if bytes.len() != CLAIM_BYTES || bytes[0] != 1 { + if !((bytes.len() == CLAIM_BYTES && bytes[0] == 1) || (bytes.len() == CLAIM_BYTES + 16 && bytes[0] == 2)) + { return Err(FileGrantError::Invalid); } let array = |start, end| bytes.get(start..end).ok_or(FileGrantError::Invalid); @@ -154,6 +189,11 @@ fn decode(bytes: &[u8]) -> Result { )) }; Ok(FileGrant { + storage_namespace: if bytes[0] == 2 { + Some(NamespaceId::from_bytes(&bytes[CLAIM_BYTES..]).map_err(|_| FileGrantError::Invalid)?) + } else { + None + }, context: CatalogContext { catalog: CatalogId::from_bytes(array(1, 17)?).map_err(|_| FileGrantError::Invalid)?, activation_epoch: number(17)?, diff --git a/lib/crowdb-access-iceberg/src/file/digest.rs b/lib/crowdb-access-iceberg/src/file/digest.rs index 15e4a5817..2ce897c88 100644 --- a/lib/crowdb-access-iceberg/src/file/digest.rs +++ b/lib/crowdb-access-iceberg/src/file/digest.rs @@ -89,7 +89,11 @@ impl FileDigest { #[must_use] pub fn checkpoint(&self) -> Vec { let mut bytes = Vec::with_capacity(CHECKPOINT_BYTES); - bytes.extend_from_slice(b"ICHS\x01"); + bytes.extend_from_slice(if self.owner.table.storage_namespace.is_some() { + b"ICHS\x02" + } else { + b"ICHS\x01" + }); bytes.extend_from_slice(self.owner.table.catalog.as_bytes()); bytes.extend_from_slice(self.owner.table.table.as_bytes()); bytes.extend_from_slice(self.owner.file.as_bytes()); @@ -98,6 +102,9 @@ impl FileDigest { bytes.extend_from_slice(&word.to_be_bytes()); } bytes.extend_from_slice(&self.tail); + if let Some(namespace) = self.owner.table.storage_namespace { + bytes.extend_from_slice(namespace.as_bytes()); + } let checksum = Sha256::digest(&bytes); bytes.extend_from_slice(&checksum); bytes @@ -108,12 +115,23 @@ impl FileDigest { /// # Errors /// Rejects wrong versions, identities, lengths, checksums and noncanonical tails. pub fn restore(owner: FileIdentity, bytes: &[u8]) -> Result { - if bytes.len() != CHECKPOINT_BYTES - || &bytes[..5] != b"ICHS\x01" + let extra = if owner.table.storage_namespace.is_some() { + 16 + } else { + 0 + }; + let version = if extra == 0 { b"ICHS\x01" } else { b"ICHS\x02" }; + let checksum_offset = CHECKSUM_OFFSET + extra; + if bytes.len() != CHECKPOINT_BYTES + extra + || &bytes[..5] != version || &bytes[5..21] != owner.table.catalog.as_bytes() || &bytes[21..37] != owner.table.table.as_bytes() || &bytes[37..53] != owner.file.as_bytes() - || Sha256::digest(&bytes[..CHECKSUM_OFFSET]).as_slice() != &bytes[CHECKSUM_OFFSET..] + || owner + .table + .storage_namespace + .is_some_and(|id| &bytes[CHECKSUM_OFFSET..checksum_offset] != id.as_bytes()) + || Sha256::digest(&bytes[..checksum_offset]).as_slice() != &bytes[checksum_offset..] { return Err(FileIoError::Bounds); } diff --git a/lib/crowdb-access-iceberg/src/file/direct_location.rs b/lib/crowdb-access-iceberg/src/file/direct_location.rs new file mode 100644 index 000000000..cc45d6e4d --- /dev/null +++ b/lib/crowdb-access-iceberg/src/file/direct_location.rs @@ -0,0 +1,223 @@ +use std::collections::BTreeMap; + +use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; +use hmac::{Hmac, Mac}; +use sha2::Sha256; + +use crate::{error::ValidationError, key::FileId}; + +use super::{ChunkRoot, ContentFormat, FileContent, FileKind, FileLocation, FileRecord, TableLocation}; + +const PAYLOAD_BYTES: usize = 142; +const TOKEN_BYTES: usize = PAYLOAD_BYTES + 32; + +/// Persistent file locators use a key ring independent of expiring HTTP grants. +#[derive(Clone)] +pub struct DirectFileLocations { + active_version: u32, + keys: BTreeMap, +} + +impl DirectFileLocations { + /// # Errors + /// Rejects a configured replacement of the catalog's reserved key version. + pub fn for_catalog( + &self, + key: crate::catalog::CatalogLocatorKey, + strict: bool, + ) -> Result { + let key = key.bytes(); + if strict && self.keys.get(&1).is_some_and(|configured| *configured != key) { + return Err(ValidationError::IdentityMismatch); + } + let mut keys = self.keys.clone(); + keys.insert(1, key); + Self::new(self.active_version, keys) + } + /// # Errors + /// Rejects stems which cannot fit the final authenticated object key. + pub fn validate_stem(stem: &str) -> Result<(), ValidationError> { + const LOCATOR_KEY_BYTES: usize = 99 + 5 + 1 + 232 + 8; + super::validate_relative_key(stem)?; + if stem.contains('/') || stem.len() > super::MAX_OBJECT_KEY_BYTES - LOCATOR_KEY_BYTES { + return Err(ValidationError::Key); + } + Ok(()) + } + /// # Errors + /// Rejects missing active keys, reserved versions and uninitialized keys. + pub fn new(active_version: u32, keys: BTreeMap) -> Result { + if !keys.contains_key(&active_version) + || keys.iter().any(|(version, key)| *version == 0 || *key == [0; 32]) + { + return Err(ValidationError::Record); + } + Ok(Self { active_version, keys }) + } + + /// Called only after the supplied native tree is durable. + /// + /// # Errors + /// Rejects legacy tables, invalid stems and inconsistent native file descriptors. + pub fn parquet( + &self, + table: TableLocation, + stem: &str, + file: FileId, + length: u64, + digest: [u8; 32], + root: ChunkRoot, + ) -> Result { + if table.storage_namespace.is_none() || root.height == 0 { + return Err(ValidationError::Key); + } + Self::validate_stem(stem)?; + let mut payload = Vec::with_capacity(TOKEN_BYTES); + payload.push(1); + payload.extend_from_slice(&self.active_version.to_be_bytes()); + payload.extend_from_slice(file.as_bytes()); + payload.extend_from_slice(&length.to_be_bytes()); + payload.extend_from_slice(&digest); + encode_root(&mut payload, &root); + let signature = self + .signer(self.active_version, table, stem, &payload)? + .finalize() + .into_bytes(); + payload.extend_from_slice(&signature); + let location = table.file(&format!( + "f/v1/{stem}.{}.parquet", + URL_SAFE_NO_PAD.encode(payload) + ))?; + let record = FileRecord { + file, + location, + kind: FileKind::Unbound, + format: ContentFormat::Parquet, + length, + digest, + content: FileContent::Chunks { root: Some(root) }, + hint: None, + }; + record.validate()?; + Ok(record) + } + + /// Reconstructs a read descriptor without consulting any file record or state. + /// HTTP permission validation remains the caller's separate responsibility. + /// + /// # Errors + /// Rejects reserved-path corruption, altered identities and unknown signing keys. + pub fn resolve(&self, location: &FileLocation) -> Result, ValidationError> { + if location.table().storage_namespace.is_some() && location.relative_key() == "f" { + return Err(ValidationError::Key); + } + if location.table().storage_namespace.is_none() || !location.relative_key().starts_with("f/") { + return Ok(None); + } + let name = location + .relative_key() + .strip_prefix("f/v1/") + .ok_or(ValidationError::Key)?; + let name = name.strip_suffix(".parquet").ok_or(ValidationError::Key)?; + let (stem, token) = name.rsplit_once('.').ok_or(ValidationError::Key)?; + if stem.contains('/') || token.len() != TOKEN_BYTES.div_ceil(3) * 4 { + return Err(ValidationError::Key); + } + super::validate_relative_key(stem)?; + let payload = URL_SAFE_NO_PAD.decode(token).map_err(|_| ValidationError::Key)?; + if payload.len() != TOKEN_BYTES || payload[0] != 1 { + return Err(ValidationError::Key); + } + let version = u32::from_be_bytes(payload[1..5].try_into().map_err(|_| ValidationError::Key)?); + self.signer(version, location.table(), stem, &payload[..PAYLOAD_BYTES])? + .verify_slice(&payload[PAYLOAD_BYTES..]) + .map_err(|_| ValidationError::IdentityMismatch)?; + let mut fields = Cursor(&payload[5..PAYLOAD_BYTES]); + let file = FileId::from_bytes(&fields.take::<16>()?)?; + let length = fields.number()?; + let digest = fields.take()?; + let root = decode_root(&mut fields)?; + if root.height == 0 { + return Err(ValidationError::Record); + } + let record = FileRecord { + file, + location: location.clone(), + kind: FileKind::Unbound, + format: ContentFormat::Parquet, + length, + digest, + content: FileContent::Chunks { root: Some(root) }, + hint: None, + }; + record.validate()?; + Ok(Some(record)) + } + + fn signer( + &self, + version: u32, + table: TableLocation, + stem: &str, + payload: &[u8], + ) -> Result, ValidationError> { + let key = self.keys.get(&version).ok_or(ValidationError::IdentityMismatch)?; + let path = table.storage_path().ok_or(ValidationError::Key)?; + let mut signer = Hmac::::new_from_slice(key).map_err(|_| ValidationError::Record)?; + signer.update(b"iceberg-native-parquet-locator-v1"); + signer.update(path.to_string().as_bytes()); + signer.update(&(stem.len() as u64).to_be_bytes()); + signer.update(stem.as_bytes()); + signer.update(payload); + Ok(signer) + } +} + +fn encode_root(bytes: &mut Vec, root: &ChunkRoot) { + for field in [ + root.chunk.high, + root.chunk.low, + root.offset, + root.physical_length, + root.logical_offset, + root.logical_length, + ] { + bytes.extend_from_slice(&field.to_be_bytes()); + } + bytes.push(root.height); + bytes.extend_from_slice(&root.digest); +} + +fn decode_root(fields: &mut Cursor<'_>) -> Result { + Ok(ChunkRoot { + chunk: crowdb_protocol::common::ChunkId { + high: fields.number()?, + low: fields.number()?, + }, + offset: fields.number()?, + physical_length: fields.number()?, + logical_offset: fields.number()?, + logical_length: fields.number()?, + height: fields.take::<1>()?[0], + digest: fields.take()?, + }) +} + +struct Cursor<'a>(&'a [u8]); + +impl Cursor<'_> { + fn take(&mut self) -> Result<[u8; N], ValidationError> { + let value = self + .0 + .get(..N) + .ok_or(ValidationError::Record)? + .try_into() + .map_err(|_| ValidationError::Record)?; + self.0 = &self.0[N..]; + Ok(value) + } + + fn number(&mut self) -> Result { + Ok(u64::from_be_bytes(self.take()?)) + } +} diff --git a/lib/crowdb-access-iceberg/src/file/directory.rs b/lib/crowdb-access-iceberg/src/file/directory.rs index 98a1298f4..61ba0d41c 100644 --- a/lib/crowdb-access-iceberg/src/file/directory.rs +++ b/lib/crowdb-access-iceberg/src/file/directory.rs @@ -1,7 +1,7 @@ use crowdb_protocol::common::ChunkId; use crate::error::ValidationError; -use crate::key::{CatalogId, FileId, TableId}; +use crate::key::{CatalogId, FileId, NamespaceId, TableId}; use super::content::{MAX_CHUNK_DIRECTORY_BYTES, MAX_CHUNK_TREE_HEIGHT}; use super::{ChunkRoot, TableLocation}; @@ -72,8 +72,15 @@ impl ChunkDirectory { pub(crate) fn encode_frontier(&self) -> Result, ValidationError> { self.frontier_length()?; let mut bytes = Vec::with_capacity(HEADER_BYTES + ENTRY_BYTES * self.entries.len()); - bytes.extend_from_slice(b"ICEN\x01"); + bytes.extend_from_slice(if self.owner.table.storage_namespace.is_some() { + b"ICEN\x02" + } else { + b"ICEN\x01" + }); bytes.extend_from_slice(self.owner.table.catalog.as_bytes()); + if let Some(namespace) = self.owner.table.storage_namespace { + bytes.extend_from_slice(namespace.as_bytes()); + } bytes.extend_from_slice(self.owner.table.table.as_bytes()); bytes.extend_from_slice(self.owner.file.as_bytes()); bytes.push(self.height); @@ -121,12 +128,20 @@ impl ChunkDirectory { return Err(ValidationError::RecordTooLarge); } let mut cursor = Cursor(bytes); - if cursor.take::<5>()? != *b"ICEN\x01" { + let version = cursor.take::<5>()?; + if version != *b"ICEN\x01" && version != *b"ICEN\x02" { return Err(ValidationError::Record); } + let catalog = CatalogId::from_bytes(&cursor.take::<16>()?)?; + let storage_namespace = if version == *b"ICEN\x02" { + Some(NamespaceId::from_bytes(&cursor.take::<16>()?)?) + } else { + None + }; let actual_owner = FileIdentity { table: TableLocation { - catalog: CatalogId::from_bytes(&cursor.take::<16>()?)?, + storage_namespace, + catalog, table: TableId::from_bytes(&cursor.take::<16>()?)?, }, file: FileId::from_bytes(&cursor.take::<16>()?)?, diff --git a/lib/crowdb-access-iceberg/src/file/injection.rs b/lib/crowdb-access-iceberg/src/file/injection.rs new file mode 100644 index 000000000..8d5b63b48 --- /dev/null +++ b/lib/crowdb-access-iceberg/src/file/injection.rs @@ -0,0 +1,202 @@ +use std::sync::Arc; + +use crate::{ + catalog::{check_context, CasOutcome, CatalogContext, CatalogError, CatalogStore}, + error::ValidationError, + key::{StorageKey, TableStorageKey, TableStorageScope}, + operation::{mutation_identity, RequestIdentity, RETRY_WINDOW_MS}, + record::StorageRecord, +}; + +use super::{DirectFileLocations, FileIdentity, FileRecord, FileRepository, FileTree, TableLocation}; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct NativeInjection { + pub identity: RequestIdentity, + pub activation_epoch: u64, + pub principal: [u8; 32], + pub stem: String, + pub file: FileRecord, +} + +impl NativeInjection { + /// # Errors + /// Rejects legacy ownership, unsupported content and unbounded retention. + pub fn validate(&self) -> Result<(), ValidationError> { + self.file.validate()?; + DirectFileLocations::validate_stem(&self.stem)?; + if self.activation_epoch == 0 + || self.file.location.table().storage_namespace.is_none() + || self.stem.contains('/') + || !self + .file + .location + .relative_key() + .starts_with(&format!("f/v1/{}.", self.stem)) + { + return Err(ValidationError::Record); + } + self.identity + .issued_ms + .checked_add(RETRY_WINDOW_MS) + .ok_or(ValidationError::Deadline)?; + Ok(()) + } + + /// # Errors + /// Rejects legacy storage identities. + pub fn key(&self) -> Result { + injection_key(self.file.location.table(), self.identity) + } + + #[must_use] + pub fn retained(&self, now_ms: u64) -> bool { + now_ms <= self.identity.issued_ms.saturating_add(RETRY_WINDOW_MS) + } +} + +#[derive(Clone, Debug)] +pub struct NativeInjectionRequest { + pub context: CatalogContext, + pub table: TableLocation, + pub identity: RequestIdentity, + pub principal: [u8; 32], + pub stem: String, + pub digest: [u8; 32], +} + +impl NativeInjectionRequest { + fn validate(&self, now_ms: u64) -> Result<(), ValidationError> { + self.context.validate()?; + self.identity.validate(now_ms)?; + DirectFileLocations::validate_stem(&self.stem)?; + if self.table.catalog != self.context.catalog + || self.table.storage_namespace.is_none() + || self.stem.contains('/') + { + return Err(ValidationError::IdentityMismatch); + } + Ok(()) + } + + fn matches(&self, receipt: &NativeInjection) -> bool { + receipt.identity == self.identity + && receipt.activation_epoch == self.context.activation_epoch + && receipt.file.location.table() == self.table + && receipt.principal == self.principal + && receipt.stem == self.stem + && receipt.file.digest == self.digest + } +} + +pub struct NativeInjectionRepository { + store: Arc, +} + +impl NativeInjectionRepository { + #[must_use] + pub fn new(store: Arc) -> Self { + Self { store } + } + + /// Replays a selected upload without allocating a new tree or changing its URL. + /// # Errors + /// Rejects expired, foreign, changed or logically deleted upload retries. + pub async fn replay( + &self, + request: &NativeInjectionRequest, + now_ms: u64, + ) -> Result, CatalogError> { + request.validate(now_ms)?; + check_context(self.store.as_ref(), request.context).await?; + let key = injection_key(request.table, request.identity)?; + let Some(value) = self.store.get(&key.encode()?).await? else { + return Ok(None); + }; + let StorageRecord::NativeInjection(receipt) = StorageRecord::decode(&key, &value.bytes)? else { + return Err(ValidationError::Record.into()); + }; + if !request.matches(&receipt) { + return Err(CatalogError::Conflict); + } + self.publish(request.context, &receipt.file).await.map(Some) + } + + /// Selects one durable descriptor before publishing its sole filename authority. + /// # Errors + /// Rejects changed payloads and preserves the durable winner after uncertain replies. + pub async fn finish( + &self, + request: &NativeInjectionRequest, + owner: FileIdentity, + tree: FileTree, + locations: &DirectFileLocations, + now_ms: u64, + ) -> Result { + request.validate(now_ms)?; + if owner.table != request.table || tree.digest != request.digest { + return Err(CatalogError::Conflict); + } + check_context(self.store.as_ref(), request.context).await?; + let file = locations.parquet( + owner.table, + &request.stem, + owner.file, + tree.length, + tree.digest, + tree.root.ok_or(ValidationError::Record)?, + )?; + let receipt = NativeInjection { + identity: request.identity, + activation_epoch: request.context.activation_epoch, + principal: request.principal, + stem: request.stem.clone(), + file, + }; + let key = receipt.key()?; + let bytes = StorageRecord::NativeInjection(Box::new(receipt.clone())).encode()?; + let encoded = key.encode()?; + let selected = match self + .store + .compare_exchange(&encoded, None, &bytes, mutation_identity(&encoded, None, &bytes)) + .await + { + Ok(CasOutcome::Applied(_)) => receipt, + outcome => { + let value = match outcome { + Ok(CasOutcome::Conflict(Some(value))) => value, + Ok(CasOutcome::Conflict(None)) => return Err(CatalogError::Conflict), + Err(error) => self + .store + .get(&encoded) + .await? + .ok_or(CatalogError::Store(error))?, + Ok(CasOutcome::Applied(_)) => return Err(ValidationError::Record.into()), + }; + let StorageRecord::NativeInjection(receipt) = StorageRecord::decode(&key, &value.bytes)? + else { + return Err(ValidationError::Record.into()); + }; + if !request.matches(&receipt) { + return Err(CatalogError::Conflict); + } + *receipt + } + }; + self.publish(request.context, &selected.file).await + } + + async fn publish(&self, context: CatalogContext, file: &FileRecord) -> Result { + FileRepository::new(self.store.clone()) + .publish(context, file) + .await + } +} + +fn injection_key(table: TableLocation, identity: RequestIdentity) -> Result { + Ok(StorageKey::NativeTable(TableStorageKey { + table: table.storage_path().ok_or(ValidationError::Key)?, + scope: TableStorageScope::Operation, + suffix: format!("{}/injection", identity.operation), + })) +} diff --git a/lib/crowdb-access-iceberg/src/file/key.rs b/lib/crowdb-access-iceberg/src/file/key.rs index 633f30f17..9e0321192 100644 --- a/lib/crowdb-access-iceberg/src/file/key.rs +++ b/lib/crowdb-access-iceberg/src/file/key.rs @@ -1,7 +1,21 @@ -use crate::key::{CatalogId, CatalogScope, FileId, IcebergKey}; +use crate::key::{ + CatalogId, CatalogScope, FileId, IcebergKey, StorageKey, TableStorageKey, TableStorageScope, +}; use super::FileLocation; +#[must_use] +pub fn canonical_file_key(location: &FileLocation) -> StorageKey { + match location.table().storage_path() { + Some(table) => StorageKey::NativeTable(TableStorageKey { + table, + scope: TableStorageScope::File, + suffix: location.relative_key().to_owned(), + }), + None => StorageKey::Legacy(location_key(location)), + } +} + #[must_use] pub fn file_key(catalog: CatalogId, file: FileId) -> IcebergKey { IcebergKey::Catalog { diff --git a/lib/crowdb-access-iceberg/src/file/listing.rs b/lib/crowdb-access-iceberg/src/file/listing.rs index 1726bce69..fdc0445df 100644 --- a/lib/crowdb-access-iceberg/src/file/listing.rs +++ b/lib/crowdb-access-iceberg/src/file/listing.rs @@ -4,7 +4,7 @@ use crowdb_chunk_kv_client::MultiScanRequest; use crowdb_protocol::chunk_kv::ScanDirection; use crate::error::ValidationError; -use crate::key::{CatalogScope, IcebergKey}; +use crate::key::{CatalogScope, IcebergKey, TableStorageKey}; use super::{TableLocation, MAX_OBJECT_KEY_BYTES}; @@ -66,6 +66,10 @@ impl FileListRequest { .prefix .strip_prefix(&self.table.object_prefix()) .ok_or(ValidationError::Key)?; + if let Some(table) = self.table.storage_path() { + let range = TableStorageKey::file_range(table, relative)?; + return Ok((range.start, range.end)); + } let mut start = IcebergKey::catalog_range(self.table.catalog).start; start.push(CatalogScope::FileLocation as u8); start.extend_from_slice(self.table.table.as_bytes()); @@ -78,6 +82,9 @@ impl FileListRequest { let relative = object_key .strip_prefix(&self.table.object_prefix()) .ok_or(ValidationError::Key)?; + if let Some(table) = self.table.storage_path() { + return Ok(TableStorageKey::file_range(table, relative)?.start); + } let mut key = IcebergKey::catalog_range(self.table.catalog).start; key.push(CatalogScope::FileLocation as u8); key.extend_from_slice(self.table.table.as_bytes()); diff --git a/lib/crowdb-access-iceberg/src/file/listing/repository.rs b/lib/crowdb-access-iceberg/src/file/listing/repository.rs index cf2cd47f4..b2aa374e2 100644 --- a/lib/crowdb-access-iceberg/src/file/listing/repository.rs +++ b/lib/crowdb-access-iceberg/src/file/listing/repository.rs @@ -1,6 +1,6 @@ use crate::catalog::{check_context, CatalogError, StoreError}; use crate::error::ValidationError; -use crate::key::IcebergKey; +use crate::key::{IcebergKey, StorageKey}; use crate::record::StorageRecord; use super::{successor, FileListPage, FileListRequest, FileListTokens, FileLocationScan, ListedFile}; @@ -23,6 +23,7 @@ impl FileRepository { request.validate()?; if grant.context.catalog != request.table.catalog || grant.table != request.table.table + || grant.storage_namespace != request.table.storage_namespace || !grant.operations.allows(FileOperation::ListObjects) || now_ms < grant.issued_ms || now_ms >= grant.expires_ms @@ -87,18 +88,21 @@ impl FileRepository { consumed += 1; start.clone_from(&item.key); start.push(0); - let key = IcebergKey::decode(&item.key)?; + let key = StorageKey::decode(&item.key)?; if matches!( - StorageRecord::decode(&key, &item.value)?, + StorageRecord::decode_storage(&key, &item.value)?, StorageRecord::DeletedFile(_) ) { continue; } - let IcebergKey::Catalog { suffix, .. } = key else { - return Err(ValidationError::Key.into()); + let relative = match &key { + StorageKey::Legacy(IcebergKey::Catalog { suffix, .. }) => { + std::str::from_utf8(suffix.get(16..).ok_or(ValidationError::Key)?) + .map_err(|_| ValidationError::Key)? + } + StorageKey::NativeTable(key) => key.suffix.as_str(), + StorageKey::Legacy(_) => return Err(ValidationError::Key.into()), }; - let relative = std::str::from_utf8(suffix.get(16..).ok_or(ValidationError::Key)?) - .map_err(|_| ValidationError::Key)?; let location = request.table.file(relative)?; let record = self.resolve_value(&location, &item.value).await?; let object_key = location.object_key(); diff --git a/lib/crowdb-access-iceberg/src/file/location.rs b/lib/crowdb-access-iceberg/src/file/location.rs index 7667ed6a2..cbb79be35 100644 --- a/lib/crowdb-access-iceberg/src/file/location.rs +++ b/lib/crowdb-access-iceberg/src/file/location.rs @@ -4,7 +4,7 @@ use std::str::FromStr; use data_encoding::BASE32_NOPAD; use crate::error::ValidationError; -use crate::key::{CatalogId, TableId}; +use crate::key::{CatalogId, NamespaceId, StorageTablePath, TableId}; pub const MAX_OBJECT_KEY_BYTES: usize = 1024; const TABLE_PREFIX_BYTES: usize = 35; @@ -12,6 +12,7 @@ const BUCKET_BYTES: usize = 34; #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] pub struct TableLocation { + pub storage_namespace: Option, pub catalog: CatalogId, pub table: TableId, } @@ -19,6 +20,9 @@ pub struct TableLocation { impl TableLocation { #[must_use] pub fn bucket(self) -> String { + if self.storage_namespace.is_some() { + return super::NATIVE_ICEBERG_BUCKET.to_owned(); + } format!( "iceberg-{}", BASE32_NOPAD.encode(self.catalog.as_bytes()).to_ascii_lowercase() @@ -27,13 +31,31 @@ impl TableLocation { #[must_use] pub fn object_prefix(self) -> String { + if let Some(path) = self.storage_path() { + return path.to_string(); + } format!("t/{}/", self.table) } + #[must_use] + pub const fn storage_path(self) -> Option { + match self.storage_namespace { + Some(storage_namespace) => Some(StorageTablePath { + catalog: self.catalog, + storage_namespace, + table: self.table, + }), + None => None, + } + } + /// # Errors /// Rejects keys outside this table or keys that cannot be represented exactly. pub fn file(self, relative_key: &str) -> Result { validate_relative_key(relative_key)?; + if self.storage_namespace.is_some() { + super::validate_native_relative_key(relative_key)?; + } Ok(FileLocation { table: self, relative_key: relative_key.to_owned(), @@ -53,6 +75,9 @@ impl FromStr for TableLocation { fn from_str(value: &str) -> Result { let body = value.strip_prefix("s3://").ok_or(ValidationError::Key)?; let (bucket, prefix) = body.split_once('/').ok_or(ValidationError::Key)?; + if bucket == super::NATIVE_ICEBERG_BUCKET { + return StorageTablePath::from_prefix(prefix).map(Self::from); + } if prefix.len() != TABLE_PREFIX_BYTES || !prefix.ends_with('/') { return Err(ValidationError::Key); } @@ -73,6 +98,10 @@ impl FileLocation { if key.len() > MAX_OBJECT_KEY_BYTES { return Err(ValidationError::KeyTooLarge); } + if bucket == super::NATIVE_ICEBERG_BUCKET { + let (table, relative) = StorageTablePath::split_object_key(key)?; + return TableLocation::from(table).file(relative); + } let rest = key.strip_prefix("t/").ok_or(ValidationError::Key)?; let (table, relative) = rest.split_once('/').ok_or(ValidationError::Key)?; let table = parse_table(bucket, &format!("t/{table}"))?; @@ -95,6 +124,16 @@ impl FileLocation { } } +impl From for TableLocation { + fn from(path: StorageTablePath) -> Self { + Self { + catalog: path.catalog, + storage_namespace: Some(path.storage_namespace), + table: path.table, + } + } +} + impl fmt::Display for FileLocation { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { write!(formatter, "{}{}", self.table, self.relative_key) @@ -124,7 +163,11 @@ fn parse_table(bucket: &str, prefix: &str) -> Result String { + format!("s3://{NATIVE_ICEBERG_BUCKET}/{self}") + } + + /// # Errors + /// Rejects unsafe relative keys and object keys exceeding the SDK byte limit. + pub fn file(self, relative_key: &str) -> Result { + validate_native_relative_key(relative_key)?; + Ok(NativeFileLocation { + table: self, + relative_key: relative_key.to_owned(), + }) + } + + /// Admission for names chosen by ordinary SDK writers. + /// + /// # Errors + /// Rejects invalid keys and the server-owned direct locator directory. + pub fn logical_file(self, relative_key: &str) -> Result { + if relative_key == "f" || relative_key.starts_with("f/") { + return Err(ValidationError::Key); + } + self.file(relative_key) + } +} + +impl NativeFileLocation { + /// # Errors + /// Rejects foreign buckets, malformed table prefixes and unsafe filenames. + pub fn from_object_key(bucket: &str, key: &str) -> Result { + if bucket != NATIVE_ICEBERG_BUCKET { + return Err(ValidationError::Key); + } + if key.len() > MAX_OBJECT_KEY_BYTES { + return Err(ValidationError::KeyTooLarge); + } + let (table, relative) = StorageTablePath::split_object_key(key)?; + table.file(relative) + } + + #[must_use] + pub const fn table(&self) -> StorageTablePath { + self.table + } + + #[must_use] + pub fn relative_key(&self) -> &str { + &self.relative_key + } + + #[must_use] + pub fn object_key(&self) -> String { + format!("{}{}", self.table, self.relative_key) + } +} + +impl fmt::Display for NativeFileLocation { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + formatter, + "s3://{NATIVE_ICEBERG_BUCKET}/{}{}", + self.table, self.relative_key + ) + } +} + +impl FromStr for NativeFileLocation { + type Err = ValidationError; + + fn from_str(value: &str) -> Result { + let body = value.strip_prefix("s3://").ok_or(ValidationError::Key)?; + let (bucket, key) = body.split_once('/').ok_or(ValidationError::Key)?; + Self::from_object_key(bucket, key) + } +} + +pub(crate) fn validate_native_relative_key(key: &str) -> Result<(), ValidationError> { + if key.len() > MAX_OBJECT_KEY_BYTES - StorageTablePath::PREFIX_BYTES { + return Err(ValidationError::KeyTooLarge); + } + validate_relative_key(key) +} diff --git a/lib/crowdb-access-iceberg/src/file/reader.rs b/lib/crowdb-access-iceberg/src/file/reader.rs index 1c17e717f..8aa53a507 100644 --- a/lib/crowdb-access-iceberg/src/file/reader.rs +++ b/lib/crowdb-access-iceberg/src/file/reader.rs @@ -33,6 +33,43 @@ struct ReadSource { } impl FileReader { + /// Checks physical directory ownership without any file authority lookup. + /// # Errors + /// Rejects absent, reclaimed, reused or corrupt roots and non-directory descriptors. + pub async fn probe_root(store: &dyn FileBlockStore, record: &FileRecord) -> Result, FileIoError> { + record.validate()?; + let FileContent::Chunks { root: Some(root) } = &record.content else { + return Err(FileIoError::Bounds); + }; + if root.height == 0 { + return Err(FileIoError::Bounds); + } + let bytes = store.read(root).await?; + verify_block(root, &bytes)?; + ChunkDirectory::decode( + &bytes, + FileIdentity { + table: record.location.table(), + file: record.file, + }, + root.height, + record.length, + )?; + Ok(bytes) + } + + /// Reuses a preflight directory without keeping more than one cached directory. + /// # Errors + /// Rejects bytes which do not belong to this reader's root or owner. + pub fn with_root_directory(mut self, bytes: Vec) -> Result { + let FileContent::Chunks { root: Some(root) } = &self.record.content else { + return Err(FileIoError::Bounds); + }; + verify_block(root, &bytes)?; + ChunkDirectory::decode(&bytes, self.record.owner, root.height, self.record.length)?; + self.leaf_directory = Some((root.clone(), bytes)); + Ok(self) + } /// # Errors /// Rejects invalid records, out-of-file intervals and unbounded frame sizes. pub fn new( diff --git a/lib/crowdb-access-iceberg/src/file/repository.rs b/lib/crowdb-access-iceberg/src/file/repository.rs index aaeb627a6..034e975ec 100644 --- a/lib/crowdb-access-iceberg/src/file/repository.rs +++ b/lib/crowdb-access-iceberg/src/file/repository.rs @@ -6,7 +6,7 @@ use crate::gc::GcCandidate; use crate::operation::mutation_identity; use crate::record::StorageRecord; -use super::{file_key, location_key, DeletedFile, FileLocation, FileRecord}; +use super::{canonical_file_key, file_key, DeletedFile, FileLocation, FileRecord}; #[derive(Clone)] pub struct FileRepository { @@ -19,6 +19,27 @@ impl FileRepository { Self { store } } + /// Finalizes an injection using its durably selected upload tree. + /// The caller retains that same owner/tree in its upload recovery state. + /// + /// # Errors + /// Rejects legacy scope, corrupt trees and conflicting filename publication. + pub async fn finalize_native_parquet( + &self, + context: CatalogContext, + owner: super::FileIdentity, + stem: &str, + tree: super::FileTree, + locations: &super::DirectFileLocations, + ) -> Result { + if context.catalog != owner.table.catalog { + return Err(ValidationError::IdentityMismatch.into()); + } + let root = tree.root.ok_or(ValidationError::Record)?; + let candidate = locations.parquet(owner.table, stem, owner.file, tree.length, tree.digest, root)?; + self.publish(context, &candidate).await + } + /// # Errors /// Rejects retired contexts, corrupt bindings and missing published authorities. pub async fn load( @@ -53,12 +74,12 @@ impl FileRepository { ) -> Result { let location = &expected.location; self.check_context(context, location).await?; - let key = location_key(location); + let key = canonical_file_key(location); let encoded = key.encode()?; let Some(current) = self.store.get(&encoded).await? else { return Err(CatalogError::Conflict); }; - if let StorageRecord::DeletedFile(deleted) = StorageRecord::decode(&key, ¤t.bytes)? { + if let StorageRecord::DeletedFile(deleted) = StorageRecord::decode_storage(&key, ¤t.bytes)? { return if deleted.file == *expected { Ok(*deleted) } else { @@ -87,11 +108,12 @@ impl FileRepository { CasOutcome::Applied(_) => Ok(deleted), CasOutcome::Conflict(Some(value)) if matches!( - StorageRecord::decode(&key, &value.bytes)?, + StorageRecord::decode_storage(&key, &value.bytes)?, StorageRecord::DeletedFile(_) ) => { - let StorageRecord::DeletedFile(current) = StorageRecord::decode(&key, &value.bytes)? else { + let StorageRecord::DeletedFile(current) = StorageRecord::decode_storage(&key, &value.bytes)? + else { unreachable!() }; if current.file == *expected { @@ -113,11 +135,11 @@ impl FileRepository { location: &FileLocation, ) -> Result, CatalogError> { self.check_context(context, location).await?; - let key = location_key(location); + let key = canonical_file_key(location); let Some(value) = self.store.get(&key.encode()?).await? else { return Ok(None); }; - match StorageRecord::decode(&key, &value.bytes)? { + match StorageRecord::decode_storage(&key, &value.bytes)? { StorageRecord::DeletedFile(deleted) => Ok(Some(*deleted)), _ => Ok(None), } @@ -130,7 +152,7 @@ impl FileRepository { ) -> Result { candidate.validate()?; self.check_context(context, &candidate.location).await?; - let key = location_key(&candidate.location).encode()?; + let key = canonical_file_key(&candidate.location).encode()?; let bytes = StorageRecord::File(Box::new(candidate.clone())).encode()?; let result = self .store @@ -139,8 +161,9 @@ impl FileRepository { let published = match result { CasOutcome::Applied(_) => candidate.clone(), CasOutcome::Conflict(Some(value)) => { - let record_key = location_key(&candidate.location); - if let StorageRecord::DeletedFile(deleted) = StorageRecord::decode(&record_key, &value.bytes)? + let record_key = canonical_file_key(&candidate.location); + if let StorageRecord::DeletedFile(deleted) = + StorageRecord::decode_storage(&record_key, &value.bytes)? { if candidate.file == deleted.file.file { return Err(CatalogError::Conflict); @@ -179,12 +202,12 @@ impl FileRepository { } async fn resolve(&self, location: &FileLocation) -> Result, CatalogError> { - let key = location_key(location); + let key = canonical_file_key(location); let Some(value) = self.store.get(&key.encode()?).await? else { return Ok(None); }; if matches!( - StorageRecord::decode(&key, &value.bytes)?, + StorageRecord::decode_storage(&key, &value.bytes)?, StorageRecord::DeletedFile(_) ) { return Ok(None); @@ -197,8 +220,8 @@ impl FileRepository { location: &FileLocation, bytes: &[u8], ) -> Result { - let key = location_key(location); - let record = match StorageRecord::decode(&key, bytes)? { + let key = canonical_file_key(location); + let record = match StorageRecord::decode_storage(&key, bytes)? { StorageRecord::File(record) => record, StorageRecord::FileMapping(mapping) => { let key = file_key(location.table().catalog, mapping.file); diff --git a/lib/crowdb-access-iceberg/src/file/write_intent.rs b/lib/crowdb-access-iceberg/src/file/write_intent.rs index 914d900ed..bd3ae76e8 100644 --- a/lib/crowdb-access-iceberg/src/file/write_intent.rs +++ b/lib/crowdb-access-iceberg/src/file/write_intent.rs @@ -120,7 +120,7 @@ async fn check_writer(store: &dyn CatalogStore, owner: FileIdentity) -> Result<( if root.state != RootState::Ready || root.context.catalog != owner.table.catalog { return Err(CatalogError::Busy); } - let key = crate::table::head_key(owner.table.catalog, owner.table.table); + let key = crate::table::storage_head_key(owner.table); if let Some(value) = store.get(&key.encode()?).await? { let StorageRecord::TableHead(head) = StorageRecord::decode(&key, &value.bytes)? else { return Err(ValidationError::Record.into()); diff --git a/lib/crowdb-access-iceberg/src/file/writer.rs b/lib/crowdb-access-iceberg/src/file/writer.rs index 7afd0f61f..b3b2b5ce9 100644 --- a/lib/crowdb-access-iceberg/src/file/writer.rs +++ b/lib/crowdb-access-iceberg/src/file/writer.rs @@ -104,6 +104,11 @@ impl FileTreeWriter { continue; } if self.levels[level].len() == 1 && self.levels[level + 1..].iter().all(Vec::is_empty) { + if level == 0 && self.owner.table.storage_namespace.is_some() { + let entry = self.flush_directory(level).await?; + self.append(entry).await?; + continue; + } let entry = self.levels[level].pop().ok_or(FileIoError::Bounds)?; if entry.length != self.length { return Err(FileIoError::Bounds); diff --git a/lib/crowdb-access-iceberg/src/gc/admission.rs b/lib/crowdb-access-iceberg/src/gc/admission.rs index bb0c50751..329d00059 100644 --- a/lib/crowdb-access-iceberg/src/gc/admission.rs +++ b/lib/crowdb-access-iceberg/src/gc/admission.rs @@ -5,7 +5,7 @@ use crate::{ key::{CatalogScope, IcebergKey, OperationId, SystemScope}, operation::{ledger_locate, LedgerLocation, ManagementAction, ManagementOperation, ManagementPhase}, record::StorageRecord, - table::{head_key, TablePurgeTask}, + table::{storage_head_key, TablePurgeTask}, }; use super::{CandidatePhase, GcCandidate, GcLimits, GcRepository, GcTask, GcTaskKind, TreeReclaimCursor}; @@ -198,7 +198,7 @@ impl GcRepository { } return Ok(existing); } - let head_key = head_key(context.catalog, marker.head.table); + let head_key = storage_head_key(marker.head.storage_location()); let head = self .store .get(&head_key.encode()?) diff --git a/lib/crowdb-access-iceberg/src/gc/claim.rs b/lib/crowdb-access-iceberg/src/gc/claim.rs index 54dbd7b6a..4fa52d2b6 100644 --- a/lib/crowdb-access-iceberg/src/gc/claim.rs +++ b/lib/crowdb-access-iceberg/src/gc/claim.rs @@ -95,7 +95,7 @@ impl GcRepository { || previous .head .as_ref() - .is_some_and(|head| head.table != candidate.file.location.table().table) + .is_some_and(|head| head.storage_location() != candidate.file.location.table()) || !matches!( (task.kind, previous.kind), ( diff --git a/lib/crowdb-access-iceberg/src/gc/discovery.rs b/lib/crowdb-access-iceberg/src/gc/discovery.rs index 082ea454a..4298042e3 100644 --- a/lib/crowdb-access-iceberg/src/gc/discovery.rs +++ b/lib/crowdb-access-iceberg/src/gc/discovery.rs @@ -1,8 +1,8 @@ use crate::{ - catalog::{CatalogContext, CatalogError}, + catalog::{CatalogContext, CatalogError, CatalogStorageLayout}, error::ValidationError, file::MultipartPhase, - key::{CatalogScope, IcebergKey, OperationId}, + key::{CatalogScope, IcebergKey, OperationId, StorageKey, TableStorageScope}, record::StorageRecord, table::{TableHead, TableLifecycle}, }; @@ -83,32 +83,53 @@ impl GcRepository { if !matches!(task.phase, GcPhase::Discover | GcPhase::Rescan) || task.paused { return Err(CatalogError::Busy); } + let native = task.discovery_scope == task.last_discovery_scope(); let scan = GcScan { + native, catalog: task.context.catalog, - scope: Some(if task.kind == GcTaskKind::MultipartCleanup { - match task.discovery_scope { - 0 => CatalogScope::FileLocation, - 1 => CatalogScope::MultipartPart, - _ => CatalogScope::MultipartSession, - } + scope: if native { + None } else { - match task.discovery_scope { - 0 => CatalogScope::File, - 1 => CatalogScope::FileLocation, - 2 => CatalogScope::MultipartPart, - _ => CatalogScope::MultipartSession, - } - }), - prefix: Vec::new(), + Some(if task.kind == GcTaskKind::MultipartCleanup { + match task.discovery_scope { + 0 => CatalogScope::FileLocation, + 1 => CatalogScope::MultipartPart, + _ => CatalogScope::MultipartSession, + } + } else { + match task.discovery_scope { + 0 => CatalogScope::File, + 1 => CatalogScope::FileLocation, + 2 => CatalogScope::MultipartPart, + _ => CatalogScope::MultipartSession, + } + }) + }, + prefix: if native { + task.head + .as_ref() + .and_then(|head| head.storage_location().storage_path()) + .map_or_else(Vec::new, |table| table.to_string().into_bytes()) + } else { + Vec::new() + }, after: task.scan_after.clone(), items: usize::from(limits.page_items), bytes: (crate::record::MAX_RECORD_BYTES + crate::key::MAX_KEY_BYTES) .min(limits.step_bytes as usize), }; - let page = self.store.scan_gc(scan.clone()).await?; + let page = if self.discovery_layout_allows(task, native).await? { + self.store.scan_gc(scan.clone()).await? + } else { + crowdb_chunk_kv_client::MultiScanPage { + items: Vec::new(), + continuation: None, + terminal_failure: None, + } + }; scan.validate_page(&page)?; for item in &page.items { - let key = IcebergKey::decode(&item.key)?; + let key = StorageKey::decode(&item.key)?; if let Some(candidate) = self .discovery_candidate(task, &key, &item.value, limits, now_ms) .await? @@ -121,13 +142,7 @@ impl GcRepository { next.scan_after.clone_from(&last.key); } else { next.scan_after.clear(); - if task.discovery_scope - < if task.kind == GcTaskKind::MultipartCleanup { - 2 - } else { - 3 - } - { + if task.discovery_scope < task.last_discovery_scope() { next.discovery_scope += 1; } else if task.phase == GcPhase::Rescan { next.discovery_scope = 0; @@ -154,29 +169,47 @@ impl GcRepository { Ok(next) } + async fn discovery_layout_allows(&self, task: &GcTask, native: bool) -> Result { + let layout = self.catalog_layout(task.context.catalog).await?; + if task.head.as_ref().is_some_and(|head| { + head.storage_location().storage_namespace.is_some() != (layout == CatalogStorageLayout::Native) + }) { + return Err(ValidationError::IdentityMismatch.into()); + } + let data_scope = task.discovery_scope == 0 + || (task.kind != GcTaskKind::MultipartCleanup && task.discovery_scope == 1); + Ok(if native { + layout == CatalogStorageLayout::Native + } else { + !data_scope || layout == CatalogStorageLayout::Legacy + }) + } + async fn discovery_candidate( &self, task: &GcTask, - key: &IcebergKey, + key: &StorageKey, bytes: &[u8], limits: GcLimits, now_ms: u64, ) -> Result, CatalogError> { let mut deleted_ms = None; let (file, part, assembly) = match key { - IcebergKey::Catalog { + StorageKey::NativeTable(key) if key.scope != TableStorageScope::File => return Ok(None), + StorageKey::Legacy(IcebergKey::Catalog { scope: CatalogScope::File, .. - } => { + }) => { let StorageRecord::File(file) = StorageRecord::decode(key, bytes)? else { return Err(ValidationError::Record.into()); }; (*file, None, None) } - IcebergKey::Catalog { + StorageKey::NativeTable(_) + | StorageKey::Legacy(IcebergKey::Catalog { scope: CatalogScope::FileLocation, .. - } => match StorageRecord::decode(key, bytes)? { + }) => match StorageRecord::decode(key, bytes)? { StorageRecord::File(file) if task.kind != GcTaskKind::MultipartCleanup => (*file, None, None), StorageRecord::DeletedFile(record) => { deleted_ms = Some(record.deleted_ms); @@ -185,10 +218,10 @@ impl GcRepository { StorageRecord::File(_) | StorageRecord::FileMapping(_) => return Ok(None), _ => return Err(ValidationError::Record.into()), }, - IcebergKey::Catalog { + StorageKey::Legacy(IcebergKey::Catalog { scope: CatalogScope::MultipartPart, .. - } => { + }) => { let StorageRecord::MultipartPart(part) = StorageRecord::decode(key, bytes)? else { return Err(ValidationError::Record.into()); }; @@ -197,10 +230,10 @@ impl GcRepository { } (GcCandidate::part_file(&part)?, Some(*part), None) } - IcebergKey::Catalog { + StorageKey::Legacy(IcebergKey::Catalog { scope: CatalogScope::MultipartSession, .. - } => { + }) => { let StorageRecord::MultipartSession(session) = StorageRecord::decode(key, bytes)? else { return Err(ValidationError::Record.into()); }; @@ -215,12 +248,12 @@ impl GcRepository { } (GcCandidate::assembly_file(&session)?, None, Some(session)) } - _ => return Err(ValidationError::Record.into()), + StorageKey::Legacy(_) => return Err(ValidationError::Record.into()), }; if task .head .as_ref() - .is_some_and(|head| head.table != file.location.table().table) + .is_some_and(|head| head.storage_location() != file.location.table()) { return Ok(None); } diff --git a/lib/crowdb-access-iceberg/src/gc/fence.rs b/lib/crowdb-access-iceberg/src/gc/fence.rs index 5dce34d93..2da92ed51 100644 --- a/lib/crowdb-access-iceberg/src/gc/fence.rs +++ b/lib/crowdb-access-iceberg/src/gc/fence.rs @@ -2,7 +2,7 @@ use crate::{ catalog::CatalogError, error::ValidationError, record::StorageRecord, - table::{head_key, TableHead, TableLifecycle}, + table::{storage_head_key, TableHead, TableLifecycle}, }; use super::{GcPhase, GcRepository, GcStalledReason, GcTask, GcTaskKind}; @@ -23,7 +23,7 @@ impl GcRepository { return Ok(task.clone()); } let expected = fenced_head(task)?; - let key = head_key(expected.catalog, expected.table); + let key = storage_head_key(expected.storage_location()); if let Some(value) = self.store.get(&key.encode()?).await? { let StorageRecord::TableHead(current) = StorageRecord::decode(&key, &value.bytes)? else { return Err(ValidationError::Record.into()); @@ -55,7 +55,7 @@ impl GcRepository { .operation_fence .checked_add(2) .ok_or(ValidationError::GenerationExhausted)?; - let key = head_key(released.catalog, released.table); + let key = storage_head_key(released.storage_location()); let Some(value) = self.store.get(&key.encode()?).await? else { return Ok(false); }; @@ -76,8 +76,8 @@ impl GcRepository { return Err(CatalogError::Busy); } let fenced = fenced_head(task)?; - self.change( - &head_key(before.catalog, before.table), + self.change_storage( + &storage_head_key(before.storage_location()), Some(&StorageRecord::TableHead(Box::new(before.clone()))), &StorageRecord::TableHead(Box::new(fenced.clone())), ) @@ -89,7 +89,7 @@ impl GcRepository { /// Rejects lost ownership; no deletion may follow a failed verification. pub async fn verify_table_fence(&self, task: &GcTask) -> Result<(), CatalogError> { let expected = fenced_head(task)?; - let key = head_key(expected.catalog, expected.table); + let key = storage_head_key(expected.storage_location()); let value = self .store .get(&key.encode()?) @@ -110,8 +110,8 @@ impl GcRepository { .operation_fence .checked_add(1) .ok_or(ValidationError::GenerationExhausted)?; - self.change( - &head_key(before.catalog, before.table), + self.change_storage( + &storage_head_key(before.storage_location()), Some(&StorageRecord::TableHead(Box::new(before))), &StorageRecord::TableHead(Box::new(released)), ) diff --git a/lib/crowdb-access-iceberg/src/gc/mark.rs b/lib/crowdb-access-iceberg/src/gc/mark.rs index a7355a93c..e926289b0 100644 --- a/lib/crowdb-access-iceberg/src/gc/mark.rs +++ b/lib/crowdb-access-iceberg/src/gc/mark.rs @@ -1,7 +1,7 @@ use crate::{ catalog::CatalogError, error::ValidationError, - file::{file_key, location_key, FileIoError, FileRecord}, + file::{canonical_file_key, file_key, FileIoError, FileRecord}, record::StorageRecord, table::TableMetadataError, }; @@ -27,7 +27,7 @@ impl GcRepository { &self, location: &crate::file::FileLocation, ) -> Result { - let key = location_key(location); + let key = canonical_file_key(location); let value = self .store .get(&key.encode()?) diff --git a/lib/crowdb-access-iceberg/src/gc/proof/traversal.rs b/lib/crowdb-access-iceberg/src/gc/proof/traversal.rs index 2b4216c1f..581546086 100644 --- a/lib/crowdb-access-iceberg/src/gc/proof/traversal.rs +++ b/lib/crowdb-access-iceberg/src/gc/proof/traversal.rs @@ -6,7 +6,7 @@ use crate::{ file::{AvroDatumLimits, AvroLimits, FileBlockStore, FileReader}, key::{CatalogScope, IcebergKey}, record::StorageRecord, - table::{head_key, TableHead, TableMetadataDocument, TableMetadataLimits}, + table::{storage_head_key, TableHead, TableMetadataDocument, TableMetadataLimits}, }; use super::super::{ @@ -28,7 +28,7 @@ impl GcRepository { } check_context(self.store.as_ref(), task.context).await?; let head = task.head.as_ref().ok_or(ValidationError::Record)?; - let key = head_key(head.catalog, head.table); + let key = storage_head_key(head.storage_location()); let value = self .store .get(&key.encode()?) @@ -49,7 +49,10 @@ impl GcRepository { head: &TableHead, ) -> Result<(), CatalogError> { if head.catalog != task.context.catalog - || task.head.as_ref().map_or(true, |root| root.table != head.table) + || task + .head + .as_ref() + .map_or(true, |root| root.storage_location() != head.storage_location()) { return Err(ValidationError::IdentityMismatch.into()); } diff --git a/lib/crowdb-access-iceberg/src/gc/repository.rs b/lib/crowdb-access-iceberg/src/gc/repository.rs index 348587ac4..67380d2b4 100644 --- a/lib/crowdb-access-iceberg/src/gc/repository.rs +++ b/lib/crowdb-access-iceberg/src/gc/repository.rs @@ -16,6 +16,25 @@ pub struct GcRepository { } impl GcRepository { + pub(super) async fn catalog_layout( + &self, + catalog: CatalogId, + ) -> Result { + let key = IcebergKey::Catalog { + catalog, + scope: CatalogScope::Authority, + suffix: Vec::new(), + }; + let value = self + .store + .get(&key.encode()?) + .await? + .ok_or(ValidationError::Record)?; + let StorageRecord::Authority(authority) = StorageRecord::decode(&key, &value.bytes)? else { + return Err(ValidationError::Record.into()); + }; + Ok(authority.storage_layout) + } #[must_use] pub fn new(store: Arc) -> Self { Self { store } @@ -173,7 +192,24 @@ impl GcRepository { before: Option<&StorageRecord>, after: &StorageRecord, ) -> Result<(), CatalogError> { - self.check_retirement_mutation(key, after).await?; + self.change_storage(&crate::key::StorageKey::Legacy(key.clone()), before, after) + .await + } + + pub(super) async fn change_storage( + &self, + key: &crate::key::StorageKey, + before: Option<&StorageRecord>, + after: &StorageRecord, + ) -> Result<(), CatalogError> { + match key { + crate::key::StorageKey::Legacy(key) => self.check_retirement_mutation(key, after).await?, + crate::key::StorageKey::NativeTable(key) => { + if self.retirement(key.table.catalog).await?.is_some() { + return Err(CatalogError::Busy); + } + } + } let key = key.encode()?; let after = after.encode()?; let before = before.map(StorageRecord::encode).transpose()?; diff --git a/lib/crowdb-access-iceberg/src/gc/storage.rs b/lib/crowdb-access-iceberg/src/gc/storage.rs index 4fe3f603e..20f63c6ab 100644 --- a/lib/crowdb-access-iceberg/src/gc/storage.rs +++ b/lib/crowdb-access-iceberg/src/gc/storage.rs @@ -5,11 +5,12 @@ use crowdb_protocol::chunk_kv::{ClientRequestId, ScanDirection}; use crate::{ catalog::{CasOutcome, CatalogStore, RoutedCatalogStore, StoreError}, error::ValidationError, - key::{CatalogId, CatalogScope, IcebergKey, SystemScope}, + key::{CatalogId, CatalogScope, IcebergKey, StorageKey, StorageTablePath, SystemScope}, }; #[derive(Clone, Debug)] pub struct GcScan { + pub native: bool, pub catalog: CatalogId, pub scope: Option, pub prefix: Vec, @@ -107,6 +108,9 @@ impl GcScan { if self.items == 0 || self.items > 256 || self.bytes == 0 || self.bytes > 16 * 1024 * 1024 { return Err(ValidationError::RecordTooLarge); } + if self.native { + return self.native_request(); + } if self.prefix.len() > 40 || (!self.prefix.is_empty() && self.scope.is_none()) { return Err(ValidationError::Key); } @@ -151,6 +155,41 @@ impl GcScan { }) } + fn native_request(&self) -> Result { + if self.scope.is_some() { + return Err(ValidationError::Key); + } + let mut start = b"ICE\0\x02".to_vec(); + if self.prefix.is_empty() { + start.extend_from_slice(format!("{}/", self.catalog).as_bytes()); + } else { + let prefix = std::str::from_utf8(&self.prefix).map_err(|_| ValidationError::Key)?; + let table = StorageTablePath::from_prefix(prefix)?; + if table.catalog != self.catalog { + return Err(ValidationError::IdentityMismatch); + } + start.extend_from_slice(&self.prefix); + } + let mut end = start.clone(); + *end.last_mut().ok_or(ValidationError::Key)? += 1; + if !self.after.is_empty() { + if self.after < start || self.after >= end { + return Err(ValidationError::Key); + } + StorageKey::decode(&self.after)?; + start.clone_from(&self.after); + start.push(0); + } + Ok(MultiScanRequest { + start: Some(start), + end: Some(end), + direction: ScanDirection::Forward, + max_items: self.items, + max_bytes: self.bytes, + continuation: None, + }) + } + /// # Errors /// Rejects unordered, foreign, oversized or failed scan pages. pub fn validate_page(&self, page: &MultiScanPage) -> Result<(), StoreError> { diff --git a/lib/crowdb-access-iceberg/src/gc/task.rs b/lib/crowdb-access-iceberg/src/gc/task.rs index 75f85e4ae..f7fe28f48 100644 --- a/lib/crowdb-access-iceberg/src/gc/task.rs +++ b/lib/crowdb-access-iceberg/src/gc/task.rs @@ -1,7 +1,7 @@ use crate::{ catalog::CatalogContext, error::ValidationError, - key::{CatalogScope, IcebergKey, OperationId}, + key::{CatalogScope, IcebergKey, OperationId, StorageKey}, table::TableHead, }; @@ -33,6 +33,8 @@ pub enum GcPhase { SweepWrites, VerifyCleanup, CleanupGc, + CleanupNative, + RootsNative, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -78,6 +80,14 @@ pub struct GcTask { } impl GcTask { + pub(super) const fn last_discovery_scope(&self) -> u8 { + if matches!(self.kind, GcTaskKind::MultipartCleanup) { + 3 + } else { + 4 + } + } + pub(crate) fn retirement_key(catalog: crate::key::CatalogId) -> IcebergKey { IcebergKey::Catalog { catalog, @@ -101,12 +111,7 @@ impl GcTask { self.context.validate()?; self.proof.validate(self)?; if self.revision == 0 - || self.discovery_scope - > if self.kind == GcTaskKind::MultipartCleanup { - 2 - } else { - 3 - } + || self.discovery_scope > self.last_discovery_scope() || (self.discovery_scope != 0 && !matches!(self.phase, GcPhase::Discover | GcPhase::Rescan)) || self.created_ms == 0 || self.not_before_ms < self.created_ms @@ -121,6 +126,7 @@ impl GcTask { | GcPhase::PreSweepSystem | GcPhase::VerifyCleanup | GcPhase::CleanupGc + | GcPhase::CleanupNative ) && self.kind != GcTaskKind::RetiredCatalog) || (matches!( self.kind, @@ -139,8 +145,17 @@ impl GcTask { GcPhase::CleanupSystem | GcPhase::RootsSystem | GcPhase::PreSweepSystem ) { super::GcSystemScan::validate_cursor(&self.scan_after)?; - } else if !IcebergKey::catalog_range(self.context.catalog).contains(&self.scan_after) { - return Err(ValidationError::Key); + } else { + match StorageKey::decode(&self.scan_after)? { + StorageKey::Legacy(IcebergKey::Catalog { catalog, .. }) + if catalog == self.context.catalog => {} + StorageKey::NativeTable(key) + if key.table.catalog == self.context.catalog + && self.head.as_ref().map_or(true, |head| { + head.storage_location().storage_path() == Some(key.table) + }) => {} + _ => return Err(ValidationError::Key), + } } } if let Some(head) = &self.head { diff --git a/lib/crowdb-access-iceberg/src/gc/worker.rs b/lib/crowdb-access-iceberg/src/gc/worker.rs index 9c95c47f6..cdf64a0b4 100644 --- a/lib/crowdb-access-iceberg/src/gc/worker.rs +++ b/lib/crowdb-access-iceberg/src/gc/worker.rs @@ -5,7 +5,7 @@ use crowdb_chunk_client::ReclaimOutcome; use crate::{ catalog::{CatalogError, CatalogLifecycle, RootState}, error::ValidationError, - file::{file_key, location_key, FileBlockStore, FileIoError}, + file::{file_key, FileBlockStore, FileIoError}, key::{CatalogScope, IcebergKey, SystemScope}, record::StorageRecord, }; @@ -21,6 +21,7 @@ mod cleanup; mod inactive; mod live; mod multipart; +mod native; mod sweep; mod system; mod terminal; @@ -107,6 +108,7 @@ impl GcWorker { GcPhase::Roots if task.kind == GcTaskKind::LiveTable => self.live_roots(task, now_ms).await, GcPhase::Mark if task.kind == GcTaskKind::LiveTable => self.live_mark(task).await, GcPhase::Roots => self.inactive_roots(task, now_ms).await, + GcPhase::RootsNative => self.native_roots(task, now_ms).await, GcPhase::Fence if task.kind == GcTaskKind::LiveTable => self.live_fence(task, now_ms).await, GcPhase::Fence => self.fence(task, now_ms).await, GcPhase::Sweep => self.sweep(task, now_ms).await, @@ -116,6 +118,7 @@ impl GcWorker { GcPhase::SweepWrites => self.sweep_writes(task, now_ms).await, GcPhase::CleanupSystem => self.cleanup_system(task, now_ms).await, GcPhase::CleanupCatalog => self.cleanup_catalog(task, now_ms).await, + GcPhase::CleanupNative => self.cleanup_native(task, now_ms).await, GcPhase::VerifyCleanup => self.verify_cleanup(task).await, GcPhase::CleanupGc => self.cleanup_gc(task).await, GcPhase::RootsSystem | GcPhase::PreSweepSystem => self.scan_system_protection(task, now_ms).await, @@ -223,6 +226,7 @@ impl GcWorker { fn candidates(&self, task: &GcTask) -> GcScan { GcScan { + native: false, catalog: task.context.catalog, scope: Some(CatalogScope::GcCandidate), prefix: task diff --git a/lib/crowdb-access-iceberg/src/gc/worker/cleanup.rs b/lib/crowdb-access-iceberg/src/gc/worker/cleanup.rs index e47f3b009..5aab0c8c0 100644 --- a/lib/crowdb-access-iceberg/src/gc/worker/cleanup.rs +++ b/lib/crowdb-access-iceberg/src/gc/worker/cleanup.rs @@ -90,6 +90,7 @@ impl GcWorker { return Err(CatalogError::Busy.into()); } let scan = GcScan { + native: false, catalog: task.context.catalog, scope: None, prefix: Vec::new(), @@ -115,6 +116,63 @@ impl GcWorker { } } let mut next = task.progress()?; + if let Some(last) = page.items.last() { + next.scan_after.clone_from(&last.key); + } else { + next.scan_after.clear(); + next.phase = GcPhase::CleanupNative; + } + self.repository.update(task, &next).await?; + Ok(next) + } + + pub(super) async fn cleanup_native(&self, task: &GcTask, now_ms: u64) -> Result { + self.verify_inactive(task).await?; + if task.kind != GcTaskKind::RetiredCatalog || now_ms < task.not_before_ms { + return Err(CatalogError::Busy.into()); + } + let scan = GcScan { + native: true, + catalog: task.context.catalog, + scope: None, + prefix: Vec::new(), + after: task.scan_after.clone(), + items: usize::from(self.limits.page_items), + bytes: self.limits.step_bytes.min(16 * 1024 * 1024) as usize, + }; + let page = self + .repository + .store + .scan_gc(scan.clone()) + .await + .map_err(CatalogError::from)?; + scan.validate_page(&page).map_err(CatalogError::from)?; + for item in &page.items { + let crate::key::StorageKey::NativeTable(key) = crate::key::StorageKey::decode(&item.key)? else { + return Err(ValidationError::Key.into()); + }; + match key.scope { + crate::key::TableStorageScope::Projection => {} + crate::key::TableStorageScope::Head => { + StorageRecord::decode_storage(&crate::key::StorageKey::NativeTable(key), &item.value)?; + } + crate::key::TableStorageScope::Operation => { + let StorageRecord::NativeInjection(receipt) = StorageRecord::decode_storage( + &crate::key::StorageKey::NativeTable(key), + &item.value, + )? + else { + return Err(ValidationError::Record.into()); + }; + if receipt.retained(now_ms) { + return Err(CatalogError::Busy.into()); + } + } + _ => return Err(CatalogError::Busy.into()), + } + self.delete_exact(&item.key, &item.value).await?; + } + let mut next = task.progress()?; if let Some(last) = page.items.last() { next.scan_after.clone_from(&last.key); } else { @@ -182,6 +240,7 @@ impl GcWorker { return Err(CatalogError::Busy.into()); } let parts = GcScan { + native: false, catalog: task.context.catalog, scope: Some(CatalogScope::MultipartPart), prefix: session.upload.as_bytes().to_vec(), diff --git a/lib/crowdb-access-iceberg/src/gc/worker/inactive.rs b/lib/crowdb-access-iceberg/src/gc/worker/inactive.rs index 7d2b98e60..01cd93257 100644 --- a/lib/crowdb-access-iceberg/src/gc/worker/inactive.rs +++ b/lib/crowdb-access-iceberg/src/gc/worker/inactive.rs @@ -47,6 +47,7 @@ impl GcWorker { self.repository.verify_table_fence(task).await?; } let scan = GcScan { + native: false, catalog: task.context.catalog, scope: None, prefix: Vec::new(), @@ -97,7 +98,7 @@ impl GcWorker { next.scan_after.clone_from(&last.key); } else { next.scan_after.clear(); - next.phase = GcPhase::Rescan; + next.phase = GcPhase::RootsNative; } self.repository.update(task, &next).await?; Ok(next) @@ -121,7 +122,11 @@ impl GcWorker { } pub(super) fn protects(task: &GcTask, record: &StorageRecord, now_ms: u64, grace_ms: u64) -> bool { - let owns = |table| task.head.as_ref().map_or(true, |head| head.table == table); + let owns = |table| { + task.head + .as_ref() + .map_or(true, |head| head.storage_location() == table) + }; let retained = |issued: u64| { now_ms < issued @@ -130,7 +135,7 @@ pub(super) fn protects(task: &GcTask, record: &StorageRecord, now_ms: u64, grace }; match record { StorageRecord::TableCommitOperation(operation) => { - owns(operation.before.table) + owns(operation.before.storage_location()) && ((task.kind == GcTaskKind::RetiredCatalog && retained(operation.identity.issued_ms)) || !matches!( operation.phase, @@ -138,7 +143,7 @@ pub(super) fn protects(task: &GcTask, record: &StorageRecord, now_ms: u64, grace )) } StorageRecord::TableCreateOperation(operation) => { - owns(operation.candidate.table) + owns(operation.candidate.storage_location()) && ((task.kind == GcTaskKind::RetiredCatalog && retained(operation.identity.issued_ms)) || !matches!( operation.phase, @@ -146,7 +151,7 @@ pub(super) fn protects(task: &GcTask, record: &StorageRecord, now_ms: u64, grace )) } StorageRecord::TableLifecycleOperation(operation) => { - owns(operation.before.table) + owns(operation.before.storage_location()) && ((task.kind == GcTaskKind::RetiredCatalog && retained(operation.identity.issued_ms)) || !matches!( operation.phase, @@ -161,7 +166,7 @@ pub(super) fn protects(task: &GcTask, record: &StorageRecord, now_ms: u64, grace | crate::file::MultipartPhase::Aborted | crate::file::MultipartPhase::Conflicted ); - owns(session.owner.table.table) + owns(session.owner.table) && (!terminal || (task.kind == GcTaskKind::RetiredCatalog && now_ms < session.expires_ms.saturating_add(grace_ms))) diff --git a/lib/crowdb-access-iceberg/src/gc/worker/live.rs b/lib/crowdb-access-iceberg/src/gc/worker/live.rs index 33372a6f1..44b8de8e2 100644 --- a/lib/crowdb-access-iceberg/src/gc/worker/live.rs +++ b/lib/crowdb-access-iceberg/src/gc/worker/live.rs @@ -1,7 +1,7 @@ use crate::{ catalog::check_context, file::{AvroDatumLimits, AvroLimits}, - table::{head_key, TableMetadataLimits}, + table::{storage_head_key, TableMetadataLimits}, }; use super::{ @@ -21,7 +21,7 @@ impl GcWorker { }; if !changed_context { let head = task.head.as_ref().ok_or(ValidationError::Record)?; - let key = head_key(head.catalog, head.table); + let key = storage_head_key(head.storage_location()); if let Some(value) = self.repository.store.get(&key.encode()?).await? { let StorageRecord::TableHead(current) = StorageRecord::decode(&key, &value.bytes)? else { return Err(ValidationError::Record.into()); @@ -77,6 +77,7 @@ impl GcWorker { return Ok(self.repository.start_proof(task).await?); } let scan = GcScan { + native: false, catalog: task.context.catalog, scope: None, prefix: Vec::new(), @@ -126,8 +127,7 @@ impl GcWorker { next.scan_after.clone_from(&last.key); } else { next.scan_after.clear(); - next.proof.complete = false; - next.phase = GcPhase::Mark; + next.phase = GcPhase::RootsNative; } self.repository.update(task, &next).await?; Ok(next) @@ -168,7 +168,7 @@ impl GcWorker { return Ok(self.repository.verify_table_fence(task).await?); } let head = task.head.as_ref().ok_or(ValidationError::Record)?; - let key = head_key(head.catalog, head.table); + let key = storage_head_key(head.storage_location()); let value = self .repository .store diff --git a/lib/crowdb-access-iceberg/src/gc/worker/multipart.rs b/lib/crowdb-access-iceberg/src/gc/worker/multipart.rs index 8dd618223..d3f49419f 100644 --- a/lib/crowdb-access-iceberg/src/gc/worker/multipart.rs +++ b/lib/crowdb-access-iceberg/src/gc/worker/multipart.rs @@ -11,6 +11,7 @@ impl GcWorker { ) -> Result { self.verify_inactive(task).await?; let scan = GcScan { + native: false, catalog: task.context.catalog, scope: Some(CatalogScope::MultipartSession), prefix: Vec::new(), @@ -40,6 +41,7 @@ impl GcWorker { .await? { let parts = GcScan { + native: false, catalog: task.context.catalog, scope: Some(CatalogScope::MultipartPart), prefix: session.upload.as_bytes().to_vec(), diff --git a/lib/crowdb-access-iceberg/src/gc/worker/native.rs b/lib/crowdb-access-iceberg/src/gc/worker/native.rs new file mode 100644 index 000000000..3a75baa3a --- /dev/null +++ b/lib/crowdb-access-iceberg/src/gc/worker/native.rs @@ -0,0 +1,82 @@ +use crate::key::{StorageKey, TableStorageScope}; + +use super::{ + CatalogError, GcPhase, GcScan, GcStalledReason, GcTask, GcTaskKind, GcWorkError, GcWorker, StorageRecord, + ValidationError, +}; + +impl GcWorker { + pub(super) async fn native_roots(&self, task: &GcTask, now_ms: u64) -> Result { + if task.kind == GcTaskKind::LiveTable { + self.verify_live_head(task).await?; + } else { + self.verify_inactive(task).await?; + } + let scan = GcScan { + native: true, + catalog: task.context.catalog, + scope: None, + prefix: task + .head + .as_ref() + .and_then(|head| head.storage_location().storage_path()) + .map_or_else(Vec::new, |table| table.to_string().into_bytes()), + after: task.scan_after.clone(), + items: usize::from(self.limits.page_items), + bytes: self.limits.step_bytes.min(16 * 1024 * 1024) as usize, + }; + let page = self + .repository + .store + .scan_gc(scan.clone()) + .await + .map_err(CatalogError::from)?; + scan.validate_page(&page).map_err(CatalogError::from)?; + let mut next = task.progress()?; + for item in &page.items { + let StorageKey::NativeTable(key) = StorageKey::decode(&item.key)? else { + return Err(ValidationError::Key.into()); + }; + if key.scope != TableStorageScope::Operation { + continue; + } + let StorageRecord::NativeInjection(receipt) = + StorageRecord::decode_storage(&StorageKey::NativeTable(key), &item.value)? + else { + return Err(ValidationError::Record.into()); + }; + if receipt.retained(now_ms) { + next.stalled = GcStalledReason::Protected; + next.scan_after.clear(); + next.retry_at_ms = receipt + .identity + .issued_ms + .saturating_add(crate::operation::RETRY_WINDOW_MS) + .saturating_add(1); + if task.kind == GcTaskKind::LiveTable && task.fenced { + self.repository.release_table_fence(task).await?; + next.fenced = false; + next.phase = GcPhase::Complete; + } else { + next.phase = GcPhase::Waiting; + } + self.repository.update(task, &next).await?; + return Ok(next); + } + self.delete_exact(&item.key, &item.value).await?; + } + if let Some(last) = page.items.last() { + next.scan_after.clone_from(&last.key); + } else { + next.scan_after.clear(); + next.phase = if task.kind == GcTaskKind::LiveTable { + next.proof.complete = false; + GcPhase::Mark + } else { + GcPhase::Rescan + }; + } + self.repository.update(task, &next).await?; + Ok(next) + } +} diff --git a/lib/crowdb-access-iceberg/src/gc/worker/sweep.rs b/lib/crowdb-access-iceberg/src/gc/worker/sweep.rs index 6ae23e2d0..0175db71b 100644 --- a/lib/crowdb-access-iceberg/src/gc/worker/sweep.rs +++ b/lib/crowdb-access-iceberg/src/gc/worker/sweep.rs @@ -1,10 +1,10 @@ use crate::{catalog::CasOutcome, operation::mutation_identity}; +use crate::{file::canonical_file_key, record::StorageRecordKey}; use crowdb_protocol::chunkdb::rpc::Location; use super::{ - file_key, location_key, CandidatePhase, CatalogError, GcCandidate, GcPhase, GcStalledReason, GcTask, - GcTaskKind, GcWorkError, GcWorker, IcebergKey, ReclaimOutcome, ReclaimStep, StorageRecord, - ValidationError, + file_key, CandidatePhase, CatalogError, GcCandidate, GcPhase, GcStalledReason, GcTask, GcTaskKind, + GcWorkError, GcWorker, IcebergKey, ReclaimOutcome, ReclaimStep, StorageRecord, ValidationError, }; impl GcWorker { @@ -249,7 +249,7 @@ impl GcWorker { { return Ok(()); } - let key = location_key(&candidate.file.location); + let key = canonical_file_key(&candidate.file.location); let value = self .repository .store @@ -311,7 +311,7 @@ impl GcWorker { file: candidate.file.file, location: candidate.file.location.clone(), }; - let location = location_key(&mapping.location); + let location = canonical_file_key(&mapping.location); if let Some(value) = self .repository .store @@ -337,6 +337,9 @@ impl GcWorker { _ => return Err(CatalogError::Conflict.into()), } } + if mapping.location.table().storage_namespace.is_some() { + return Ok(()); + } self.remove_record( &file_key(mapping.location.table().catalog, mapping.file), &StorageRecord::File(Box::new(candidate.file.clone())), @@ -344,8 +347,12 @@ impl GcWorker { .await } - async fn remove_record(&self, key: &IcebergKey, record: &StorageRecord) -> Result<(), GcWorkError> { - let key = key.encode()?; + async fn remove_record( + &self, + key: &Key, + record: &StorageRecord, + ) -> Result<(), GcWorkError> { + let key = key.storage_bytes()?; let bytes = record.encode()?; match self .repository diff --git a/lib/crowdb-access-iceberg/src/gc/worker/terminal.rs b/lib/crowdb-access-iceberg/src/gc/worker/terminal.rs index a14829573..f85534754 100644 --- a/lib/crowdb-access-iceberg/src/gc/worker/terminal.rs +++ b/lib/crowdb-access-iceberg/src/gc/worker/terminal.rs @@ -86,6 +86,7 @@ impl GcWorker { fn terminal_scan(&self, task: &GcTask) -> GcScan { GcScan { + native: false, catalog: task.context.catalog, scope: None, prefix: Vec::new(), diff --git a/lib/crowdb-access-iceberg/src/gc/worker/writes.rs b/lib/crowdb-access-iceberg/src/gc/worker/writes.rs index 019248a5d..cd411de68 100644 --- a/lib/crowdb-access-iceberg/src/gc/worker/writes.rs +++ b/lib/crowdb-access-iceberg/src/gc/worker/writes.rs @@ -18,6 +18,7 @@ impl GcWorker { return Err(CatalogError::Busy.into()); } let scan = GcScan { + native: false, catalog: task.context.catalog, scope: Some(CatalogScope::FileWriteIntent), prefix: task diff --git a/lib/crowdb-access-iceberg/src/key.rs b/lib/crowdb-access-iceberg/src/key.rs index ada9471ee..9af6443b2 100644 --- a/lib/crowdb-access-iceberg/src/key.rs +++ b/lib/crowdb-access-iceberg/src/key.rs @@ -3,7 +3,13 @@ mod codec; mod identity; mod name; +mod storage; +mod storage_path; +mod table_storage; pub use codec::{CatalogScope, IcebergKey, SystemScope, MAX_KEY_BYTES}; pub use identity::{CatalogId, FileId, NamespaceId, OperationId, TableId}; pub use name::NameSuffix; +pub use storage::StorageKey; +pub use storage_path::StorageTablePath; +pub use table_storage::{TableStorageKey, TableStorageScope}; diff --git a/lib/crowdb-access-iceberg/src/key/storage.rs b/lib/crowdb-access-iceberg/src/key/storage.rs new file mode 100644 index 000000000..4871ca648 --- /dev/null +++ b/lib/crowdb-access-iceberg/src/key/storage.rs @@ -0,0 +1,55 @@ +use std::ops::Range; + +use crate::error::ValidationError; + +use super::{IcebergKey, TableStorageKey, MAX_KEY_BYTES}; + +/// Dispatches storage envelopes without trying another layout on decode failure. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum StorageKey { + Legacy(IcebergKey), + NativeTable(TableStorageKey), +} + +impl StorageKey { + /// # Errors + /// Rejects invalid keys in the explicitly selected layout. + pub fn encode(&self) -> Result, ValidationError> { + match self { + Self::Legacy(key) => key.encode(), + Self::NativeTable(key) => key.encode(), + } + } + /// # Errors + /// Rejects unsupported versions and malformed keys in their selected layout. + pub fn decode(bytes: &[u8]) -> Result { + if bytes.len() > MAX_KEY_BYTES { + return Err(ValidationError::KeyTooLarge); + } + if bytes.get(..4) != Some(b"ICE\0") { + return Err(ValidationError::Key); + } + match bytes.get(4).copied().ok_or(ValidationError::Key)? { + 1 => IcebergKey::decode(bytes).map(Self::Legacy), + 2 => TableStorageKey::decode(bytes).map(Self::NativeTable), + version => Err(ValidationError::KeyVersion(version)), + } + } + + /// # Errors + /// Rejects unbounded, reversed or oversized scans and scans crossing layouts. + pub fn validate_range(range: &Range>) -> Result<(), ValidationError> { + if range.start.len() > MAX_KEY_BYTES || range.end.len() > MAX_KEY_BYTES { + return Err(ValidationError::KeyTooLarge); + } + let (lower, upper): (&[u8], &[u8]) = match range.start.get(..5) { + Some(b"ICE\0\x01") => (b"ICE\0\x01", b"ICE\0\x02"), + Some(b"ICE\0\x02") => (b"ICE\0\x02", b"ICE\0\x03"), + _ => return Err(ValidationError::Key), + }; + if range.start.as_slice() < lower || range.end.as_slice() > upper || range.start >= range.end { + return Err(ValidationError::Key); + } + Ok(()) + } +} diff --git a/lib/crowdb-access-iceberg/src/key/storage_path.rs b/lib/crowdb-access-iceberg/src/key/storage_path.rs new file mode 100644 index 000000000..06190b784 --- /dev/null +++ b/lib/crowdb-access-iceberg/src/key/storage_path.rs @@ -0,0 +1,74 @@ +use std::{fmt, str::FromStr}; + +use crate::error::ValidationError; + +use super::{CatalogId, NamespaceId, TableId}; + +/// Immutable storage ownership, independent of a table's current name or namespace. +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub struct StorageTablePath { + pub catalog: CatalogId, + pub storage_namespace: NamespaceId, + pub table: TableId, +} + +impl StorageTablePath { + pub const PREFIX_BYTES: usize = 99; + + /// # Errors + /// Rejects aliases, noncanonical IDs and incomplete table prefixes. + pub fn from_prefix(prefix: &str) -> Result { + let (path, relative) = Self::split_object_key(prefix)?; + if !relative.is_empty() { + return Err(ValidationError::Key); + } + Ok(path) + } + + /// Separates a complete immutable table prefix from its undecoded suffix. + /// + /// # Errors + /// Rejects missing components, zero IDs and noncanonical identity encodings. + pub fn split_object_key(key: &str) -> Result<(Self, &str), ValidationError> { + let mut fields = key.splitn(4, '/'); + let catalog = fields.next().ok_or(ValidationError::Key)?; + let namespace = fields.next().ok_or(ValidationError::Key)?; + let table = fields.next().ok_or(ValidationError::Key)?; + let relative = fields.next().ok_or(ValidationError::Key)?; + let path = Self { + catalog: canonical_id(catalog)?, + storage_namespace: canonical_id(namespace)?, + table: canonical_id(table)?, + }; + Ok((path, relative)) + } +} + +impl fmt::Display for StorageTablePath { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + formatter, + "{}/{}/{}/", + self.catalog, self.storage_namespace, self.table + ) + } +} + +impl FromStr for StorageTablePath { + type Err = ValidationError; + + fn from_str(value: &str) -> Result { + Self::from_prefix(value) + } +} + +pub(super) fn canonical_id>(value: &str) -> Result { + if value.len() != 32 + || !value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err(ValidationError::Key); + } + value.parse() +} diff --git a/lib/crowdb-access-iceberg/src/key/table_storage.rs b/lib/crowdb-access-iceberg/src/key/table_storage.rs new file mode 100644 index 000000000..335fcc29b --- /dev/null +++ b/lib/crowdb-access-iceberg/src/key/table_storage.rs @@ -0,0 +1,153 @@ +use std::ops::Range; + +use crate::{error::ValidationError, file::validate_native_relative_key}; + +use super::{storage_path::canonical_id, OperationId, StorageTablePath, MAX_KEY_BYTES}; + +const HEADER: &[u8] = b"ICE\0\x02"; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum TableStorageScope { + Head, + File, + Projection, + Operation, + Multipart, + Gc, +} + +impl TableStorageScope { + const fn component(self) -> &'static str { + match self { + Self::Head => "head", + Self::File => "file", + Self::Projection => "projection", + Self::Operation => "operation", + Self::Multipart => "multipart", + Self::Gc => "gc", + } + } + + fn parse(value: &str) -> Result { + match value { + "head" => Ok(Self::Head), + "file" => Ok(Self::File), + "projection" => Ok(Self::Projection), + "operation" => Ok(Self::Operation), + "multipart" => Ok(Self::Multipart), + "gc" => Ok(Self::Gc), + _ => Err(ValidationError::Key), + } + } +} + +/// Version-isolated table keys; filenames are their only file authority suffix. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct TableStorageKey { + pub table: StorageTablePath, + pub scope: TableStorageScope, + pub suffix: String, +} + +impl TableStorageKey { + /// # Errors + /// Rejects unsafe suffixes, invalid head keys and oversized keys. + pub fn encode(&self) -> Result, ValidationError> { + self.validate()?; + let mut bytes = scope_prefix(self.table, self.scope); + bytes.extend_from_slice(self.suffix.as_bytes()); + if bytes.len() > MAX_KEY_BYTES { + return Err(ValidationError::KeyTooLarge); + } + Ok(bytes) + } + + /// # Errors + /// Rejects legacy keys, unknown scopes, aliases and invalid suffixes. + pub fn decode(bytes: &[u8]) -> Result { + if bytes.len() > MAX_KEY_BYTES { + return Err(ValidationError::KeyTooLarge); + } + let payload = bytes.strip_prefix(HEADER).ok_or(ValidationError::Key)?; + let payload = std::str::from_utf8(payload).map_err(|_| ValidationError::Key)?; + let (table, relative) = StorageTablePath::split_object_key(payload)?; + let (component, suffix) = relative.split_once('/').unwrap_or((relative, "")); + let scope = TableStorageScope::parse(component)?; + if (scope == TableStorageScope::Head) == relative.contains('/') { + return Err(ValidationError::Key); + } + let key = Self { + table, + scope, + suffix: suffix.to_owned(), + }; + key.validate()?; + Ok(key) + } + + #[must_use] + pub fn table_range(table: StorageTablePath) -> Range> { + let mut start = HEADER.to_vec(); + start.extend_from_slice(table.to_string().as_bytes()); + prefix_range(start) + } + + /// A bounded ordered file scan, including arbitrary public filename prefixes. + /// + /// # Errors + /// Rejects escaping prefixes and prefixes exceeding the object-key byte limit. + pub fn file_range( + table: StorageTablePath, + filename_prefix: &str, + ) -> Result>, ValidationError> { + if !filename_prefix.is_empty() { + validate_native_relative_key(filename_prefix)?; + } + let mut start = scope_prefix(table, TableStorageScope::File); + start.extend_from_slice(filename_prefix.as_bytes()); + Ok(prefix_range(start)) + } + + fn validate(&self) -> Result<(), ValidationError> { + match self.scope { + TableStorageScope::Head if self.suffix.is_empty() => Ok(()), + TableStorageScope::Head => Err(ValidationError::Key), + TableStorageScope::File => validate_native_relative_key(&self.suffix), + TableStorageScope::Projection => { + crate::file::validate_relative_key(&self.suffix)?; + let (generation, child) = self.suffix.split_once('/').ok_or(ValidationError::Key)?; + let number = generation.parse::().map_err(|_| ValidationError::Key)?; + if number == 0 || number.to_string() != generation || child.is_empty() { + return Err(ValidationError::Key); + } + Ok(()) + } + TableStorageScope::Operation | TableStorageScope::Multipart | TableStorageScope::Gc => { + crate::file::validate_relative_key(&self.suffix)?; + let identity = self.suffix.split('/').next().ok_or(ValidationError::Key)?; + canonical_id::(identity).map(|_| ()) + } + } + } +} + +fn scope_prefix(table: StorageTablePath, scope: TableStorageScope) -> Vec { + let mut bytes = HEADER.to_vec(); + bytes.extend_from_slice(table.to_string().as_bytes()); + bytes.extend_from_slice(scope.component().as_bytes()); + if scope != TableStorageScope::Head { + bytes.push(b'/'); + } + bytes +} + +fn prefix_range(start: Vec) -> Range> { + let mut end = start.clone(); + while let Some(byte) = end.pop() { + if byte != u8::MAX { + end.push(byte + 1); + break; + } + } + start..end +} diff --git a/lib/crowdb-access-iceberg/src/metadata_projection/model.rs b/lib/crowdb-access-iceberg/src/metadata_projection/model.rs index 16b75841e..6535a05d7 100644 --- a/lib/crowdb-access-iceberg/src/metadata_projection/model.rs +++ b/lib/crowdb-access-iceberg/src/metadata_projection/model.rs @@ -1,11 +1,12 @@ use std::collections::BTreeMap; +use std::fmt::Write; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; use sha2::{Digest, Sha256}; use crate::file::{FileRecord, TableLocation}; -use crate::key::{CatalogScope, IcebergKey}; +use crate::key::{CatalogScope, IcebergKey, TableStorageKey, TableStorageScope}; pub const PROJECTION_VERSION: u16 = 2; pub const PROJECTION_PAGE_BYTES: usize = 32 * 1024; @@ -32,6 +33,19 @@ impl ProjectionIdentity { } pub(super) fn key(self, child: u16, page: u16) -> Option> { + if let Some(table) = self.table.storage_path() { + let mut digest = String::with_capacity(64); + for byte in self.digest { + write!(digest, "{byte:02x}").expect("string writes do not fail"); + } + return TableStorageKey { + table, + scope: TableStorageScope::Projection, + suffix: format!("{}/{digest}/{}/{child}/{page}", self.generation, self.version), + } + .encode() + .ok(); + } let mut suffix = self.table.table.as_bytes().to_vec(); suffix.extend_from_slice(&self.generation.to_be_bytes()); suffix.extend_from_slice(&self.digest); diff --git a/lib/crowdb-access-iceberg/src/namespace/create.rs b/lib/crowdb-access-iceberg/src/namespace/create.rs index 8465b522f..6d1af80e2 100644 --- a/lib/crowdb-access-iceberg/src/namespace/create.rs +++ b/lib/crowdb-access-iceberg/src/namespace/create.rs @@ -29,6 +29,15 @@ pub struct NamespaceCreator { } impl NamespaceCreator { + pub(crate) fn from_parts( + store: Arc, + names: Arc, + ) -> Self { + Self { + repository: NamespaceRepository::from_parts(store, names.clone()), + names, + } + } #[must_use] pub fn new(store: Arc) -> Self { Self { diff --git a/lib/crowdb-access-iceberg/src/namespace/drop_probe.rs b/lib/crowdb-access-iceberg/src/namespace/drop_probe.rs index 83c6af4cb..51d2d7214 100644 --- a/lib/crowdb-access-iceberg/src/namespace/drop_probe.rs +++ b/lib/crowdb-access-iceberg/src/namespace/drop_probe.rs @@ -165,7 +165,7 @@ impl NamespaceDropper { .await?; return Ok(true); } - let key = crate::table::head_key(mapping.catalog, mapping.table); + let key = crate::table::storage_head_key(mapping.storage_location()); let value = self .creator .names diff --git a/lib/crowdb-access-iceberg/src/namespace/repository.rs b/lib/crowdb-access-iceberg/src/namespace/repository.rs index 8014953a7..b1a1ab5c7 100644 --- a/lib/crowdb-access-iceberg/src/namespace/repository.rs +++ b/lib/crowdb-access-iceberg/src/namespace/repository.rs @@ -13,6 +13,34 @@ pub struct NamespaceRepository { } impl NamespaceRepository { + pub(crate) async fn ensure_default( + &self, + context: CatalogContext, + issued_ms: u64, + ) -> Result { + let identifier = NamespaceIdentifier::new(vec!["default".into()])?; + if let Some(authority) = self.load(context, &identifier).await? { + return Ok(authority); + } + let outcome = super::NamespaceCreator::from_parts(self.store.clone(), self.names.clone()) + .create(&super::NamespaceCreateRequest { + context, + identity: crate::operation::RequestIdentity { + operation: crate::key::OperationId::random(), + issued_ms, + }, + principal: "native-default-namespace".into(), + identifier: identifier.clone(), + properties: super::NamespaceProperties::default(), + }) + .await; + if let Some(authority) = self.load(context, &identifier).await? { + Ok(authority) + } else { + outcome?; + Err(CatalogError::Busy) + } + } pub(crate) fn from_parts(store: Arc, names: Arc) -> Self { Self { store, names } } diff --git a/lib/crowdb-access-iceberg/src/record.rs b/lib/crowdb-access-iceberg/src/record.rs index d62d952ca..205f4925c 100644 --- a/lib/crowdb-access-iceberg/src/record.rs +++ b/lib/crowdb-access-iceberg/src/record.rs @@ -6,6 +6,7 @@ mod file; mod gc; mod gc_key; mod gc_node; +mod injection; mod management; mod multipart; mod multipart_admission; @@ -14,6 +15,7 @@ mod namespace_operation; mod payload; mod retry; mod root; +mod storage_key; mod table; mod table_commit; mod table_create; @@ -21,3 +23,4 @@ mod table_lifecycle; mod write_intent; pub use envelope::{StorageRecord, MAX_RECORD_BYTES}; +pub use storage_key::StorageRecordKey; diff --git a/lib/crowdb-access-iceberg/src/record/authority.rs b/lib/crowdb-access-iceberg/src/record/authority.rs index ebe5989f0..337cf2fdb 100644 --- a/lib/crowdb-access-iceberg/src/record/authority.rs +++ b/lib/crowdb-access-iceberg/src/record/authority.rs @@ -1,7 +1,9 @@ use crowdb_protocol::iceberg_fb::{FBCatalogAuthority, FBCatalogAuthorityArgs, FBCatalogLifecycle}; use flatbuffers::{FlatBufferBuilder, WIPOffset}; -use crate::catalog::{Capabilities, CatalogAuthority, CatalogLifecycle, ClearBounds}; +use crate::catalog::{ + Capabilities, CatalogAuthority, CatalogLifecycle, CatalogLocatorKey, CatalogStorageLayout, ClearBounds, +}; use crate::error::ValidationError; use crate::key::CatalogId; @@ -12,9 +14,14 @@ pub(super) fn encode<'buffer>( authority.validate()?; let catalog = builder.create_vector(authority.catalog.as_bytes()); let display_name = builder.create_string(&authority.display_name); + let locator_key = authority + .locator_key + .map(|key| builder.create_vector(&key.bytes())); Ok(FBCatalogAuthority::create( builder, &FBCatalogAuthorityArgs { + locator_key, + storage_layout: authority.storage_layout as u8, catalog: Some(catalog), display_name: Some(display_name), name_generation: authority.name_generation, @@ -42,6 +49,11 @@ pub(super) fn decode(value: FBCatalogAuthority<'_>) -> Result), FileWriteIntent(Box), GcNode(Box), GcTask(Box), @@ -53,7 +54,13 @@ impl StorageRecord { let envelope = FBIcebergRecord::create( &mut builder, &FBIcebergRecordArgs { - schema_version: SCHEMA_VERSION, + schema_version: if matches!(self, Self::Authority(authority) + if authority.storage_layout == crate::catalog::CatalogStorageLayout::Native) + { + 2 + } else { + SCHEMA_VERSION + }, value_type, value: Some(value), }, @@ -74,6 +81,10 @@ impl StorageRecord { return Ok(value); } Ok(match self { + Self::NativeInjection(receipt) => ( + FBRecordValue::FBNativeInjection, + super::injection::encode(builder, receipt)?.as_union_value(), + ), Self::FileWriteIntent(intent) => ( FBRecordValue::FBFileWriteIntent, super::write_intent::encode(builder, intent)?.as_union_value(), @@ -196,7 +207,36 @@ impl StorageRecord { /// # Errors /// Rejects malformed envelopes, unknown schemas, invalid fields or mismatched keys. - pub fn decode(key: &IcebergKey, bytes: &[u8]) -> Result { + pub fn decode(key: &Key, bytes: &[u8]) -> Result { + let record = Self::decode_bytes(bytes)?; + key.validate_record(&record)?; + Ok(record) + } + + /// # Errors + /// Rejects foreign storage identities, record kinds and key layouts. + pub fn decode_storage(key: &StorageKey, bytes: &[u8]) -> Result { + Self::decode(key, bytes) + } + + pub(super) fn validate_storage_key(&self, key: &StorageKey) -> Result<(), ValidationError> { + if let StorageKey::Legacy(key) = key { + return self.validate_key(key); + } + let expected = match self { + Self::NativeInjection(receipt) => receipt.key()?, + Self::File(file) => crate::file::canonical_file_key(&file.location), + Self::DeletedFile(file) => crate::file::canonical_file_key(&file.file.location), + Self::TableHead(head) => crate::table::storage_head_key(head.storage_location()), + _ => return Err(ValidationError::Record), + }; + if expected != *key { + return Err(ValidationError::IdentityMismatch); + } + Ok(()) + } + + fn decode_bytes(bytes: &[u8]) -> Result { if bytes.len() > MAX_RECORD_BYTES { return Err(ValidationError::RecordTooLarge); } @@ -204,16 +244,29 @@ impl StorageRecord { return Err(ValidationError::Record); } let envelope = fb::root_as_fbiceberg_record(bytes).map_err(|_| ValidationError::Record)?; - if envelope.schema_version() != SCHEMA_VERSION { + if envelope.schema_version() != SCHEMA_VERSION + && !(envelope.schema_version() == 2 && envelope.value_type() == FBRecordValue::FBCatalogAuthority) + { return Err(ValidationError::RecordVersion(envelope.schema_version())); } let record = Self::decode_value(envelope)?; - record.validate_key(key)?; + let native_authority = matches!(&record, Self::Authority(authority) + if authority.storage_layout == crate::catalog::CatalogStorageLayout::Native); + if (envelope.schema_version() == 2) != native_authority { + return Err(ValidationError::RecordVersion(envelope.schema_version())); + } Ok(record) } fn decode_value(envelope: FBIcebergRecord<'_>) -> Result { match envelope.value_type() { + FBRecordValue::FBNativeInjection => { + return Ok(Self::NativeInjection(Box::new(super::injection::decode( + envelope + .value_as_fbnative_injection() + .ok_or(ValidationError::Record)?, + )?))); + } FBRecordValue::FBFileWriteIntent => { return Ok(Self::FileWriteIntent(Box::new(super::write_intent::decode( envelope @@ -390,7 +443,7 @@ impl StorageRecord { Ok(record) } - fn validate_key(&self, key: &IcebergKey) -> Result<(), ValidationError> { + pub(super) fn validate_key(&self, key: &IcebergKey) -> Result<(), ValidationError> { if let Some(result) = self.validate_gc_key(key) { return result; } diff --git a/lib/crowdb-access-iceberg/src/record/gc.rs b/lib/crowdb-access-iceberg/src/record/gc.rs index 425ca38ad..4ec4a5871 100644 --- a/lib/crowdb-access-iceberg/src/record/gc.rs +++ b/lib/crowdb-access-iceberg/src/record/gc.rs @@ -154,6 +154,8 @@ fn decode_phase(value: u8) -> Result { 13 => GcPhase::SweepWrites, 14 => GcPhase::VerifyCleanup, 15 => GcPhase::CleanupGc, + 16 => GcPhase::CleanupNative, + 17 => GcPhase::RootsNative, _ => return Err(ValidationError::Record), }) } diff --git a/lib/crowdb-access-iceberg/src/record/injection.rs b/lib/crowdb-access-iceberg/src/record/injection.rs new file mode 100644 index 000000000..fc19d8cd0 --- /dev/null +++ b/lib/crowdb-access-iceberg/src/record/injection.rs @@ -0,0 +1,45 @@ +use crowdb_protocol::iceberg_fb::{FBNativeInjection, FBNativeInjectionArgs}; +use flatbuffers::{FlatBufferBuilder, WIPOffset}; + +use crate::{error::ValidationError, file::NativeInjection, key::OperationId, operation::RequestIdentity}; + +pub(super) fn encode<'buffer>( + builder: &mut FlatBufferBuilder<'buffer>, + value: &NativeInjection, +) -> Result>, ValidationError> { + value.validate()?; + let operation = builder.create_vector(value.identity.operation.as_bytes()); + let principal = builder.create_vector(&value.principal); + let stem = builder.create_string(&value.stem); + let file = super::file::encode(builder, &value.file)?; + Ok(FBNativeInjection::create( + builder, + &FBNativeInjectionArgs { + operation: Some(operation), + issued_ms: value.identity.issued_ms, + activation_epoch: value.activation_epoch, + principal: Some(principal), + stem: Some(stem), + file: Some(file), + }, + )) +} + +pub(super) fn decode(value: FBNativeInjection<'_>) -> Result { + let receipt = NativeInjection { + identity: RequestIdentity { + operation: OperationId::from_bytes(value.operation().bytes())?, + issued_ms: value.issued_ms(), + }, + activation_epoch: value.activation_epoch(), + principal: value + .principal() + .bytes() + .try_into() + .map_err(|_| ValidationError::Record)?, + stem: value.stem().into(), + file: super::file::decode(value.file())?, + }; + receipt.validate()?; + Ok(receipt) +} diff --git a/lib/crowdb-access-iceberg/src/record/multipart/completion.rs b/lib/crowdb-access-iceberg/src/record/multipart/completion.rs index d4a63b3ee..c70d608af 100644 --- a/lib/crowdb-access-iceberg/src/record/multipart/completion.rs +++ b/lib/crowdb-access-iceberg/src/record/multipart/completion.rs @@ -65,7 +65,7 @@ pub(super) fn decode(value: FBMultipartCompletion<'_>) -> Result( @@ -13,12 +13,17 @@ pub(super) fn encode_owner<'buffer>( let catalog = builder.create_vector(owner.table.catalog.as_bytes()); let table_id = builder.create_vector(owner.table.table.as_bytes()); let file_id = builder.create_vector(owner.file.as_bytes()); + let storage_namespace = owner + .table + .storage_namespace + .map(|id| builder.create_vector(id.as_bytes())); FBFileIdentity::create( builder, &FBFileIdentityArgs { catalog: Some(catalog), table_id: Some(table_id), file_id: Some(file_id), + storage_namespace, }, ) } @@ -26,6 +31,10 @@ pub(super) fn encode_owner<'buffer>( pub(super) fn decode_owner(value: FBFileIdentity<'_>) -> Result { Ok(FileIdentity { table: TableLocation { + storage_namespace: value + .storage_namespace() + .map(|id| NamespaceId::from_bytes(id.bytes())) + .transpose()?, catalog: CatalogId::from_bytes(value.catalog().bytes())?, table: TableId::from_bytes(value.table_id().bytes())?, }, diff --git a/lib/crowdb-access-iceberg/src/record/storage_key.rs b/lib/crowdb-access-iceberg/src/record/storage_key.rs new file mode 100644 index 000000000..5b9891b42 --- /dev/null +++ b/lib/crowdb-access-iceberg/src/record/storage_key.rs @@ -0,0 +1,40 @@ +use crate::{ + error::ValidationError, + key::{IcebergKey, StorageKey}, +}; + +use super::StorageRecord; + +/// Only versioned Iceberg storage keys can bind a decoded authority record. +pub trait StorageRecordKey: sealed::Sealed { + /// # Errors + /// Rejects malformed or oversized keys in their selected layout. + fn storage_bytes(&self) -> Result, ValidationError>; + /// # Errors + /// Rejects a record belonging to another key, storage owner or layout. + fn validate_record(&self, record: &StorageRecord) -> Result<(), ValidationError>; +} + +impl StorageRecordKey for IcebergKey { + fn storage_bytes(&self) -> Result, ValidationError> { + self.encode() + } + fn validate_record(&self, record: &StorageRecord) -> Result<(), ValidationError> { + record.validate_key(self) + } +} + +impl StorageRecordKey for StorageKey { + fn storage_bytes(&self) -> Result, ValidationError> { + self.encode() + } + fn validate_record(&self, record: &StorageRecord) -> Result<(), ValidationError> { + record.validate_storage_key(self) + } +} + +mod sealed { + pub trait Sealed {} + impl Sealed for crate::key::IcebergKey {} + impl Sealed for crate::key::StorageKey {} +} diff --git a/lib/crowdb-access-iceberg/src/record/table.rs b/lib/crowdb-access-iceberg/src/record/table.rs index b4220889b..0b3ac3c2e 100644 --- a/lib/crowdb-access-iceberg/src/record/table.rs +++ b/lib/crowdb-access-iceberg/src/record/table.rs @@ -102,6 +102,9 @@ pub(super) fn encode_mapping<'buffer>( let namespace_id = builder.create_vector(mapping.namespace.as_bytes()); let name = builder.create_string(&mapping.name); let operation = builder.create_vector(mapping.operation.as_bytes()); + let storage_namespace = mapping + .storage_namespace + .map(|id| builder.create_vector(id.as_bytes())); Ok(FBTableMapping::create( builder, &FBTableMappingArgs { @@ -115,12 +118,17 @@ pub(super) fn encode_mapping<'buffer>( TableMappingState::Reserved => 0, TableMappingState::Published => 1, }, + storage_namespace, }, )) } pub(super) fn decode_mapping(value: FBTableMapping<'_>) -> Result { let mapping = TableMapping { + storage_namespace: value + .storage_namespace() + .map(|id| NamespaceId::from_bytes(id.bytes())) + .transpose()?, catalog: CatalogId::from_bytes(value.catalog().bytes())?, table: TableId::from_bytes(value.table_id().bytes())?, namespace: NamespaceId::from_bytes(value.namespace_id().bytes())?, diff --git a/lib/crowdb-access-iceberg/src/record/write_intent.rs b/lib/crowdb-access-iceberg/src/record/write_intent.rs index f9cea357b..db10170a5 100644 --- a/lib/crowdb-access-iceberg/src/record/write_intent.rs +++ b/lib/crowdb-access-iceberg/src/record/write_intent.rs @@ -4,7 +4,7 @@ use flatbuffers::{FlatBufferBuilder, WIPOffset}; use crate::{ error::ValidationError, file::{FileIdentity, FileWriteIntent, TableLocation}, - key::{CatalogId, FileId, OperationId, TableId}, + key::{CatalogId, FileId, NamespaceId, OperationId, TableId}, }; pub(super) fn encode<'buffer>( @@ -16,6 +16,11 @@ pub(super) fn encode<'buffer>( let table_id = builder.create_vector(intent.owner.table.table.as_bytes()); let file_id = builder.create_vector(intent.owner.file.as_bytes()); let identity = builder.create_vector(intent.identity.as_bytes()); + let storage_namespace = intent + .owner + .table + .storage_namespace + .map(|id| builder.create_vector(id.as_bytes())); let root = super::file::encode_root(builder, &intent.root); Ok(FBFileWriteIntent::create( builder, @@ -28,6 +33,7 @@ pub(super) fn encode<'buffer>( created_ms: intent.created_ms, not_before_ms: intent.not_before_ms, deleting: intent.deleting, + storage_namespace, }, )) } @@ -37,6 +43,10 @@ pub(super) fn decode(value: FBFileWriteIntent<'_>) -> Result crate::key::StorageKey { + match location.storage_path() { + Some(table) => crate::key::StorageKey::NativeTable(crate::key::TableStorageKey { + table, + scope: crate::key::TableStorageScope::Head, + suffix: String::new(), + }), + None => crate::key::StorageKey::Legacy(head_key(location.catalog, location.table)), + } +} + #[must_use] pub fn head_key(catalog: CatalogId, table: TableId) -> IcebergKey { IcebergKey::Catalog { diff --git a/lib/crowdb-access-iceberg/src/table/lifecycle/completion.rs b/lib/crowdb-access-iceberg/src/table/lifecycle/completion.rs index 37302166f..4f5cfc1db 100644 --- a/lib/crowdb-access-iceberg/src/table/lifecycle/completion.rs +++ b/lib/crowdb-access-iceberg/src/table/lifecycle/completion.rs @@ -5,7 +5,7 @@ use crate::{ error::ValidationError, operation::mutation_identity, record::StorageRecord, - table::{head_key, name_key, TableMapping, TableMappingState}, + table::{name_key, storage_head_key, TableMapping, TableMappingState}, }; use std::time::{SystemTime, UNIX_EPOCH}; @@ -15,7 +15,7 @@ impl TableLifecycles { if operation.is_rename() { self.check_admission(operation).await?; } - let key = head_key(operation.context.catalog, operation.before.table); + let key = storage_head_key(operation.before.storage_location()); let before = StorageRecord::TableHead(Box::new(operation.before.clone())).encode()?; let after = StorageRecord::TableHead(Box::new(operation.candidate.clone())).encode()?; let encoded = key.encode()?; @@ -49,7 +49,7 @@ impl TableLifecycles { pub(super) async fn complete(&self, operation: &TableLifecycleOperation) -> Result<(), CatalogError> { self.current(operation).await?; - let head_key = head_key(operation.context.catalog, operation.before.table); + let head_key = storage_head_key(operation.before.storage_location()); let value = self .store .get(&head_key.encode()?) @@ -139,7 +139,7 @@ impl TableLifecycles { } else if operation.phase == Phase::Complete { self.remove_mapping(&operation.source).await?; if operation.is_rename() { - let key = head_key(operation.context.catalog, operation.before.table).encode()?; + let key = storage_head_key(operation.before.storage_location()).encode()?; let before = StorageRecord::TableHead(Box::new(operation.candidate.clone())).encode()?; let mut settled = operation.candidate.clone(); settled.pending_operation = None; diff --git a/lib/crowdb-access-iceberg/src/table/lifecycle/operation.rs b/lib/crowdb-access-iceberg/src/table/lifecycle/operation.rs index 361466ee0..0748d5c69 100644 --- a/lib/crowdb-access-iceberg/src/table/lifecycle/operation.rs +++ b/lib/crowdb-access-iceberg/src/table/lifecycle/operation.rs @@ -126,6 +126,7 @@ impl TableLifecycleOperation { pub(super) fn destination(&self, state: TableMappingState) -> TableMapping { TableMapping { + storage_namespace: self.candidate.metadata_location.table().storage_namespace, catalog: self.context.catalog, namespace: self.candidate.namespace, name: self.candidate.name.clone(), diff --git a/lib/crowdb-access-iceberg/src/table/lifecycle/reservation.rs b/lib/crowdb-access-iceberg/src/table/lifecycle/reservation.rs index 8a583c1e0..8baca14e5 100644 --- a/lib/crowdb-access-iceberg/src/table/lifecycle/reservation.rs +++ b/lib/crowdb-access-iceberg/src/table/lifecycle/reservation.rs @@ -5,7 +5,7 @@ use crate::{ key::{NamespaceId, OperationId}, operation::mutation_identity, record::StorageRecord, - table::{head_key, name_key, TableMapping, TableMappingState}, + table::{name_key, storage_head_key, TableMapping, TableMappingState}, }; impl TableLifecycles { @@ -50,7 +50,7 @@ impl TableLifecycles { .await?; return Ok(()); } - let head_key = head_key(existing.catalog, existing.table); + let head_key = storage_head_key(existing.storage_location()); let head = self .store .get(&head_key.encode()?) diff --git a/lib/crowdb-access-iceberg/src/table/list.rs b/lib/crowdb-access-iceberg/src/table/list.rs index 3f7dc57dc..aea847221 100644 --- a/lib/crowdb-access-iceberg/src/table/list.rs +++ b/lib/crowdb-access-iceberg/src/table/list.rs @@ -178,7 +178,7 @@ impl TableLister { if mapping.state != super::TableMappingState::Published { continue; } - let key = super::head_key(context.catalog, mapping.table); + let key = super::storage_head_key(mapping.storage_location()); let Some(stored) = self.store.get(&key.encode()?).await? else { continue; }; diff --git a/lib/crowdb-access-iceberg/src/table/record.rs b/lib/crowdb-access-iceberg/src/table/record.rs index f1abce04c..9b68f134c 100644 --- a/lib/crowdb-access-iceberg/src/table/record.rs +++ b/lib/crowdb-access-iceberg/src/table/record.rs @@ -31,6 +31,10 @@ pub struct TableHead { } impl TableHead { + #[must_use] + pub fn storage_location(&self) -> crate::file::TableLocation { + self.metadata_location.table() + } /// # Errors /// Rejects invalid revisions, identity, version and unfenced tombstones. pub fn validate(&self) -> Result<(), ValidationError> { @@ -58,6 +62,7 @@ pub enum TableMappingState { #[derive(Clone, Debug, Eq, PartialEq)] pub struct TableMapping { + pub storage_namespace: Option, pub catalog: CatalogId, pub namespace: NamespaceId, pub name: String, @@ -68,6 +73,14 @@ pub struct TableMapping { } impl TableMapping { + #[must_use] + pub const fn storage_location(&self) -> crate::file::TableLocation { + crate::file::TableLocation { + catalog: self.catalog, + table: self.table, + storage_namespace: self.storage_namespace, + } + } /// # Errors /// Rejects invalid names and zero name epochs. pub fn validate(&self) -> Result<(), ValidationError> { @@ -85,6 +98,7 @@ impl TableMapping { && self.catalog == head.catalog && self.namespace == head.namespace && self.table == head.table + && self.storage_namespace == head.metadata_location.table().storage_namespace && self.name == head.name && self.name_epoch == head.name_epoch } diff --git a/lib/crowdb-access-iceberg/src/table/repository.rs b/lib/crowdb-access-iceberg/src/table/repository.rs index a1f7b83c2..16264d966 100644 --- a/lib/crowdb-access-iceberg/src/table/repository.rs +++ b/lib/crowdb-access-iceberg/src/table/repository.rs @@ -59,7 +59,7 @@ impl TableRepository { if selected.head.catalog != context.catalog { return Err(ValidationError::IdentityMismatch.into()); } - let key = super::head_key(context.catalog, selected.head.table); + let key = super::storage_head_key(selected.head.storage_location()); let value = self .store .get(&key.encode()?) @@ -90,7 +90,7 @@ impl TableRepository { if mapping.state != TableMappingState::Published { return Ok(None); } - let key = super::head_key(context.catalog, mapping.table); + let key = super::storage_head_key(mapping.storage_location()); let Some(value) = self.store.get(&key.encode()?).await? else { return Ok(None); }; diff --git a/lib/crowdb-access-iceberg/src/wire/credentials.rs b/lib/crowdb-access-iceberg/src/wire/credentials.rs index fa520e68d..121e3986b 100644 --- a/lib/crowdb-access-iceberg/src/wire/credentials.rs +++ b/lib/crowdb-access-iceberg/src/wire/credentials.rs @@ -22,6 +22,7 @@ impl From for StorageCredential { let grant = credentials.grant(); Self { prefix: TableLocation { + storage_namespace: grant.storage_namespace, catalog: grant.context.catalog, table: grant.table, } @@ -59,6 +60,7 @@ pub struct FileDelegationLimits { #[derive(Clone, Copy, Debug)] pub struct FileDelegationTarget { + pub storage_namespace: Option, pub table: TableId, pub format_version: u8, pub staged: bool, @@ -159,6 +161,7 @@ impl FileDelegationLimits { digest.update(b"crowdb-iceberg-file-principal-v1"); digest.update(principal.name.as_bytes()); issuer.issue(FileGrant { + storage_namespace: target.storage_namespace, context, table: target.table, principal: digest.finalize().into(), diff --git a/lib/crowdb-access-iceberg/tests/commit_manifest_provenance_test.rs b/lib/crowdb-access-iceberg/tests/commit_manifest_provenance_test.rs index bb4e1ff5a..737c675b5 100644 --- a/lib/crowdb-access-iceberg/tests/commit_manifest_provenance_test.rs +++ b/lib/crowdb-access-iceberg/tests/commit_manifest_provenance_test.rs @@ -133,6 +133,7 @@ async fn candidate_sources_separate_reachable_history_from_new_definition_bound_ upload ); let foreign = crowdb_access_iceberg::file::TableLocation { + storage_namespace: None, catalog: fixture.namespace.context.catalog, table: crowdb_access_iceberg::key::TableId::random(), } diff --git a/lib/crowdb-access-iceberg/tests/common/deletion_vector.rs b/lib/crowdb-access-iceberg/tests/common/deletion_vector.rs index 8c7900d98..51b5c441a 100644 --- a/lib/crowdb-access-iceberg/tests/common/deletion_vector.rs +++ b/lib/crowdb-access-iceberg/tests/common/deletion_vector.rs @@ -69,6 +69,7 @@ pub async fn record( cardinality: u64, ) -> (FileRecord, DeletionVectorReference) { let table = TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }; diff --git a/lib/crowdb-access-iceberg/tests/common/file.rs b/lib/crowdb-access-iceberg/tests/common/file.rs index b35b610a5..1444a7cea 100644 --- a/lib/crowdb-access-iceberg/tests/common/file.rs +++ b/lib/crowdb-access-iceberg/tests/common/file.rs @@ -25,6 +25,7 @@ impl TestFile { activation_epoch: 1, }, table: TableLocation { + storage_namespace: None, catalog, table: TableId::random(), }, diff --git a/lib/crowdb-access-iceberg/tests/common/manifest_entry.rs b/lib/crowdb-access-iceberg/tests/common/manifest_entry.rs index 7042ed129..69519949e 100644 --- a/lib/crowdb-access-iceberg/tests/common/manifest_entry.rs +++ b/lib/crowdb-access-iceberg/tests/common/manifest_entry.rs @@ -13,6 +13,7 @@ pub struct TestManifestEntry { pub fn table() -> TableLocation { TableLocation { + storage_namespace: None, catalog: CatalogId::from_bytes(&[1; 16]).unwrap(), table: TableId::from_bytes(&[2; 16]).unwrap(), } diff --git a/lib/crowdb-access-iceberg/tests/common/manifest_list.rs b/lib/crowdb-access-iceberg/tests/common/manifest_list.rs index a9778153c..d623395d3 100644 --- a/lib/crowdb-access-iceberg/tests/common/manifest_list.rs +++ b/lib/crowdb-access-iceberg/tests/common/manifest_list.rs @@ -10,6 +10,7 @@ pub struct TestManifestList { pub fn table() -> TableLocation { TableLocation { + storage_namespace: None, catalog: CatalogId::from_bytes(&[1; 16]).unwrap(), table: TableId::from_bytes(&[2; 16]).unwrap(), } diff --git a/lib/crowdb-access-iceberg/tests/common/multipart.rs b/lib/crowdb-access-iceberg/tests/common/multipart.rs index acd515cac..92f76e857 100644 --- a/lib/crowdb-access-iceberg/tests/common/multipart.rs +++ b/lib/crowdb-access-iceberg/tests/common/multipart.rs @@ -8,6 +8,7 @@ use crowdb_access_iceberg::operation::PayloadReference; pub fn session() -> MultipartSession { let table = TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }; diff --git a/lib/crowdb-access-iceberg/tests/common/parquet_integers.rs b/lib/crowdb-access-iceberg/tests/common/parquet_integers.rs index 444afc9e5..4a0bf9cc6 100644 --- a/lib/crowdb-access-iceberg/tests/common/parquet_integers.rs +++ b/lib/crowdb-access-iceberg/tests/common/parquet_integers.rs @@ -70,6 +70,7 @@ pub async fn file( stored_content( &bytes, TableLocation { + storage_namespace: None, catalog: CatalogId::from_bytes(&[1; 16]).unwrap(), table: TableId::from_bytes(&[2; 16]).unwrap(), }, diff --git a/lib/crowdb-access-iceberg/tests/common/parquet_metadata.rs b/lib/crowdb-access-iceberg/tests/common/parquet_metadata.rs index e56d6cf7c..b43dacefc 100644 --- a/lib/crowdb-access-iceberg/tests/common/parquet_metadata.rs +++ b/lib/crowdb-access-iceberg/tests/common/parquet_metadata.rs @@ -121,6 +121,7 @@ pub async fn stored(footer: &[u8], body_bytes: usize) -> (Arc, FileR stored_content( &bytes, TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }, diff --git a/lib/crowdb-access-iceberg/tests/common/parquet_nullable.rs b/lib/crowdb-access-iceberg/tests/common/parquet_nullable.rs index 717f7c2bd..1bffe5466 100644 --- a/lib/crowdb-access-iceberg/tests/common/parquet_nullable.rs +++ b/lib/crowdb-access-iceberg/tests/common/parquet_nullable.rs @@ -105,6 +105,7 @@ pub async fn file_with_type_length( stored_content( &bytes, TableLocation { + storage_namespace: None, catalog: CatalogId::from_bytes(&[1; 16]).unwrap(), table: TableId::from_bytes(&[2; 16]).unwrap(), }, diff --git a/lib/crowdb-access-iceberg/tests/common/projection.rs b/lib/crowdb-access-iceberg/tests/common/projection.rs index 33273610d..698aca829 100644 --- a/lib/crowdb-access-iceberg/tests/common/projection.rs +++ b/lib/crowdb-access-iceberg/tests/common/projection.rs @@ -44,6 +44,7 @@ impl TestProjection { let blocks = Arc::new(TestBlocks::default()); let owner = FileIdentity { table: TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }, diff --git a/lib/crowdb-access-iceberg/tests/common/puffin.rs b/lib/crowdb-access-iceberg/tests/common/puffin.rs index eff1d2e50..9dd672a84 100644 --- a/lib/crowdb-access-iceberg/tests/common/puffin.rs +++ b/lib/crowdb-access-iceberg/tests/common/puffin.rs @@ -9,6 +9,7 @@ use std::sync::Arc; pub fn referenced() -> crowdb_access_iceberg::file::FileLocation { TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), } diff --git a/lib/crowdb-access-iceberg/tests/common/selected_parquet.rs b/lib/crowdb-access-iceberg/tests/common/selected_parquet.rs index 4c3fc9879..47b7b3ec1 100644 --- a/lib/crowdb-access-iceberg/tests/common/selected_parquet.rs +++ b/lib/crowdb-access-iceberg/tests/common/selected_parquet.rs @@ -68,6 +68,7 @@ pub fn entry(content: FileContentKind) -> ManifestScalarEntry { }, file: ManifestFileFields { location: TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), } diff --git a/lib/crowdb-access-iceberg/tests/common/store.rs b/lib/crowdb-access-iceberg/tests/common/store.rs index 35269da41..bbed28412 100644 --- a/lib/crowdb-access-iceberg/tests/common/store.rs +++ b/lib/crowdb-access-iceberg/tests/common/store.rs @@ -7,6 +7,7 @@ use std::sync::{ use arc_swap::ArcSwap; use async_trait::async_trait; use crowdb_access_iceberg::catalog::{CasOutcome, CatalogStore, StoreError, StoredValue}; +use crowdb_access_iceberg::key::{CatalogScope, IcebergKey, StorageKey, TableStorageKey, TableStorageScope}; use crowdb_protocol::chunk_kv::ClientRequestId; #[derive(Default)] @@ -14,6 +15,7 @@ pub struct TestStore { pub values: ArcSwap, StoredValue>>, pub fail_after: AtomicUsize, pub file_record_reply_loss: AtomicBool, + pub native_injection_reply_loss: AtomicBool, pub gc_workspace_denied: AtomicBool, #[allow(dead_code)] pub gc_delete_reply_loss: AtomicBool, @@ -64,12 +66,25 @@ impl TestStore { } fn lose_file_reply(&self, key: &[u8]) -> bool { + if matches!( + StorageKey::decode(key), + Ok(StorageKey::NativeTable(TableStorageKey { + scope: TableStorageScope::Operation, + .. + })) + ) && self.native_injection_reply_loss.swap(false, Ordering::SeqCst) + { + return true; + } matches!( - crowdb_access_iceberg::key::IcebergKey::decode(key), - Ok(crowdb_access_iceberg::key::IcebergKey::Catalog { - scope: crowdb_access_iceberg::key::CatalogScope::FileLocation, + StorageKey::decode(key), + Ok(StorageKey::Legacy(IcebergKey::Catalog { + scope: CatalogScope::FileLocation, .. - }) + }) | StorageKey::NativeTable(TableStorageKey { + scope: TableStorageScope::File, + .. + })) ) && self.file_record_reply_loss.swap(false, Ordering::SeqCst) } diff --git a/lib/crowdb-access-iceberg/tests/common/table_metadata.rs b/lib/crowdb-access-iceberg/tests/common/table_metadata.rs index 55679204c..ec25120e6 100644 --- a/lib/crowdb-access-iceberg/tests/common/table_metadata.rs +++ b/lib/crowdb-access-iceberg/tests/common/table_metadata.rs @@ -8,6 +8,7 @@ use sha2::{Digest, Sha256}; pub fn table() -> TableLocation { TableLocation { + storage_namespace: None, catalog: CatalogId::from_bytes(&[1; 16]).unwrap(), table: TableId::from_bytes(&[2; 16]).unwrap(), } diff --git a/lib/crowdb-access-iceberg/tests/common/table_read.rs b/lib/crowdb-access-iceberg/tests/common/table_read.rs index b41961b35..6de91dd34 100644 --- a/lib/crowdb-access-iceberg/tests/common/table_read.rs +++ b/lib/crowdb-access-iceberg/tests/common/table_read.rs @@ -37,6 +37,7 @@ impl TestTable { head.catalog = fixture.context.catalog; head.namespace = parent.namespace; let location = TableLocation { + storage_namespace: None, catalog: head.catalog, table: head.table, }; @@ -85,6 +86,7 @@ impl TestTable { .await .unwrap(); let mapping = TableMapping { + storage_namespace: None, catalog: head.catalog, namespace: head.namespace, name: head.name.clone(), diff --git a/lib/crowdb-access-iceberg/tests/direct_file_location_test.rs b/lib/crowdb-access-iceberg/tests/direct_file_location_test.rs new file mode 100644 index 000000000..8febc80ae --- /dev/null +++ b/lib/crowdb-access-iceberg/tests/direct_file_location_test.rs @@ -0,0 +1,176 @@ +#[path = "common/file_blocks.rs"] +mod blocks; +#[path = "common/store.rs"] +mod common; +#[path = "common/file.rs"] +mod fixture; + +use std::{collections::BTreeMap, sync::Arc}; + +use crowdb_access_iceberg::{ + file::{ + ByteRange, DirectFileLocations, FileIdentity, FileReader, FileRepository, FileTreeWriter, + TableLocation, + }, + key::{CatalogId, FileId, NamespaceId, TableId}, +}; + +fn table() -> TableLocation { + TableLocation { + storage_namespace: Some(NamespaceId::random()), + catalog: CatalogId::random(), + table: TableId::random(), + } +} + +#[tokio::test] +async fn identical_replacement_at_the_same_root_extent_cannot_revive_a_reclaimed_locator() { + let blocks = Arc::new(blocks::TestBlocks::default()); + let table = table(); + let signer = DirectFileLocations::new(1, BTreeMap::from([(1, [1; 32])])).unwrap(); + let bytes = b"identical parquet bytes"; + let mut files = Vec::new(); + for _ in 0..2 { + let file = FileId::random(); + let mut writer = FileTreeWriter::new(blocks.clone(), FileIdentity { table, file }, 1024).unwrap(); + writer.push(bytes).await.unwrap(); + let tree = writer.finish().await.unwrap(); + let root = tree.root.unwrap(); + assert!(root.height > 0); + files.push(( + signer + .parquet(table, "part", file, tree.length, tree.digest, root.clone()) + .unwrap(), + root, + )); + } + assert_ne!(files[0].0.location, files[1].0.location); + let mut reused = (*blocks.values.load_full()).clone(); + reused.insert(files[0].1.chunk.low, reused[&files[1].1.chunk.low].clone()); + blocks.values.store(Arc::new(reused)); + let old = signer.resolve(&files[0].0.location).unwrap().unwrap(); + let mut reader = + FileReader::new(blocks.clone(), old, Some(ByteRange { start: 0, end: 3 }), 1024).unwrap(); + assert!(reader.next().await.is_err()); + let mut reader = FileReader::new(blocks, files[1].0.clone(), None, 1024).unwrap(); + assert_eq!(reader.next().await.unwrap().unwrap(), bytes); +} + +#[tokio::test] +async fn injection_replay_and_logical_delete_do_not_add_direct_read_metadata_queries() { + use std::sync::atomic::Ordering; + let mut fixture = fixture::TestFile::new(common::TestStore::default()).await; + fixture.table.storage_namespace = Some(NamespaceId::random()); + let owner = FileIdentity { + table: fixture.table, + file: FileId::random(), + }; + let blocks = Arc::new(blocks::TestBlocks::default()); + let mut writer = FileTreeWriter::new(blocks.clone(), owner, 1024).unwrap(); + writer.push(b"retained parquet bytes").await.unwrap(); + let tree = writer.finish().await.unwrap(); + let signer = DirectFileLocations::new(1, BTreeMap::from([(1, [1; 32])])).unwrap(); + let repository = FileRepository::new(fixture.store.clone()); + repository + .publish(fixture.context, &fixture.record("metadata/v1.json", b"{}")) + .await + .unwrap(); + fixture.store.file_record_reply_loss.store(true, Ordering::SeqCst); + assert!(repository + .finalize_native_parquet(fixture.context, owner, "part", tree.clone(), &signer) + .await + .is_err()); + let published = repository + .finalize_native_parquet(fixture.context, owner, "part", tree, &signer) + .await + .unwrap(); + repository + .mark_deleted(fixture.context, &published, 100) + .await + .unwrap(); + assert!(repository + .load(fixture.context, &published.location) + .await + .unwrap() + .is_none()); + let before = fixture.store.reads.load(Ordering::SeqCst); + let record = signer.resolve(&published.location).unwrap().unwrap(); + let mut reader = FileReader::new(blocks.clone(), record, None, 1024).unwrap(); + assert_eq!(reader.next().await.unwrap().unwrap(), b"retained parquet bytes"); + assert_eq!(fixture.store.reads.load(Ordering::SeqCst), before); + blocks.values.store(Arc::new(BTreeMap::new())); + let mut reclaimed = FileReader::new( + blocks, + signer.resolve(&published.location).unwrap().unwrap(), + None, + 1024, + ) + .unwrap(); + assert!(reclaimed.next().await.is_err()); +} + +#[tokio::test] +async fn direct_urls_read_multi_extent_ranges_without_any_file_repository() { + let blocks = Arc::new(blocks::TestBlocks::default()); + let table = table(); + let file = FileId::random(); + let owner = FileIdentity { table, file }; + let mut writer = FileTreeWriter::new(blocks.clone(), owner, 3).unwrap(); + writer.push(b"abcdefghij").await.unwrap(); + let tree = writer.finish().await.unwrap(); + let signer = DirectFileLocations::new(1, BTreeMap::from([(1, [1; 32])])).unwrap(); + let record = signer + .parquet( + table, + "part.0001", + file, + tree.length, + tree.digest, + tree.root.unwrap(), + ) + .unwrap(); + let location = record.location.to_string().parse().unwrap(); + let resolved = signer.resolve(&location).unwrap().unwrap(); + assert_eq!(resolved, record); + assert!(location.object_key().len() <= 1024); + let mut reader = FileReader::new(blocks, resolved, Some(ByteRange { start: 2, end: 9 }), 3).unwrap(); + let mut bytes = Vec::new(); + while let Some(chunk) = reader.next().await.unwrap() { + bytes.extend_from_slice(&chunk); + } + assert_eq!(bytes, b"cdefghi"); +} + +#[tokio::test] +async fn locator_rotation_preserves_old_urls_and_tampering_never_resolves() { + let blocks = Arc::new(blocks::TestBlocks::default()); + let table = table(); + let file = FileId::random(); + let mut writer = FileTreeWriter::new(blocks, FileIdentity { table, file }, 1024).unwrap(); + writer.push(b"retained bytes").await.unwrap(); + let tree = writer.finish().await.unwrap(); + let signer = DirectFileLocations::new(1, BTreeMap::from([(1, [1; 32])])).unwrap(); + let record = signer + .parquet(table, "part", file, tree.length, tree.digest, tree.root.unwrap()) + .unwrap(); + let rotated = DirectFileLocations::new(2, BTreeMap::from([(1, [1; 32]), (2, [2; 32])])).unwrap(); + assert_eq!(rotated.resolve(&record.location).unwrap(), Some(record.clone())); + let mut foreign = table; + foreign.storage_namespace = Some(NamespaceId::random()); + assert!(rotated + .resolve(&foreign.file(record.location.relative_key()).unwrap()) + .is_err()); + let changed_name = record.location.relative_key().replacen("part.", "other.", 1); + assert!(rotated.resolve(&table.file(&changed_name).unwrap()).is_err()); + let uri = record.location.to_string(); + let marker = uri.find("part.").unwrap() + 5; + let mut changed = uri.into_bytes(); + changed[marker + 20] = if changed[marker + 20] == b'A' { b'B' } else { b'A' }; + assert!(rotated + .resolve(&String::from_utf8(changed).unwrap().parse().unwrap()) + .is_err()); + assert!(rotated + .resolve(&table.file("metadata/v1.json").unwrap()) + .unwrap() + .is_none()); +} diff --git a/lib/crowdb-access-iceberg/tests/file_assembly_pipeline_test.rs b/lib/crowdb-access-iceberg/tests/file_assembly_pipeline_test.rs index cefb85cf0..8be491210 100644 --- a/lib/crowdb-access-iceberg/tests/file_assembly_pipeline_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_assembly_pipeline_test.rs @@ -57,6 +57,7 @@ async fn setup() -> (Arc, FileAssembly, AssemblyPart, FileId let store = Arc::new(TestPipelineBlocks::default()); let owner = FileIdentity { table: TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }, diff --git a/lib/crowdb-access-iceberg/tests/file_assembly_test.rs b/lib/crowdb-access-iceberg/tests/file_assembly_test.rs index f281088d2..df9129395 100644 --- a/lib/crowdb-access-iceberg/tests/file_assembly_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_assembly_test.rs @@ -12,6 +12,7 @@ use std::sync::{atomic::Ordering, Arc}; fn owner() -> FileIdentity { FileIdentity { table: TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }, diff --git a/lib/crowdb-access-iceberg/tests/file_avro_test.rs b/lib/crowdb-access-iceberg/tests/file_avro_test.rs index c33b4c7bc..842e86cff 100644 --- a/lib/crowdb-access-iceberg/tests/file_avro_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_avro_test.rs @@ -124,6 +124,7 @@ fn block(bytes: &mut Vec, count: i64, encoded: &[u8]) { async fn record(store: Arc, bytes: &[u8]) -> FileRecord { let owner = FileIdentity { table: TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }, diff --git a/lib/crowdb-access-iceberg/tests/file_checkpoint_test.rs b/lib/crowdb-access-iceberg/tests/file_checkpoint_test.rs index df1ed95cc..7e3384634 100644 --- a/lib/crowdb-access-iceberg/tests/file_checkpoint_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_checkpoint_test.rs @@ -13,6 +13,7 @@ use std::sync::{atomic::Ordering, Arc}; fn owner() -> FileIdentity { FileIdentity { table: TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }, diff --git a/lib/crowdb-access-iceberg/tests/file_credentials_test.rs b/lib/crowdb-access-iceberg/tests/file_credentials_test.rs index 6857043c4..b3c251801 100644 --- a/lib/crowdb-access-iceberg/tests/file_credentials_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_credentials_test.rs @@ -6,6 +6,7 @@ use crowdb_access_iceberg::key::{CatalogId, OperationId, TableId}; fn grant() -> FileGrant { FileGrant { + storage_namespace: None, context: CatalogContext { catalog: CatalogId::random(), activation_epoch: 1, @@ -48,10 +49,39 @@ fn delegated_credentials_verify_across_servers_without_storing_secret_material() assert_ne!(issued.access_key_id(), another.access_key_id()); } +#[test] +fn cached_validation_still_checks_expiry_and_catalog_epoch() { + let issuer = FileGrantIssuer::new([2; 32], 10_000).unwrap(); + let grant = grant(); + let credentials = issuer.issue(grant.clone()).unwrap(); + for _ in 0..2 { + assert_eq!( + issuer + .verify_token(credentials.session_token(), grant.context, grant.issued_ms) + .unwrap() + .grant(), + &grant + ); + } + assert!(matches!( + issuer.verify_token(credentials.session_token(), grant.context, grant.expires_ms), + Err(FileGrantError::Expired) + )); + let changed = CatalogContext { + activation_epoch: grant.context.activation_epoch + 1, + ..grant.context + }; + assert!(matches!( + issuer.verify_token(credentials.session_token(), changed, grant.issued_ms), + Err(FileGrantError::Forbidden) + )); +} + #[test] fn grants_bind_exact_table_operations_and_independent_byte_limits() { let grant = grant(); let location = TableLocation { + storage_namespace: None, catalog: grant.context.catalog, table: grant.table, } @@ -72,10 +102,12 @@ fn grants_bind_exact_table_operations_and_independent_byte_limits() { ); for table in [ TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: grant.table, }, TableLocation { + storage_namespace: None, catalog: grant.context.catalog, table: TableId::random(), }, diff --git a/lib/crowdb-access-iceberg/tests/file_digest_test.rs b/lib/crowdb-access-iceberg/tests/file_digest_test.rs index 7b963a05b..a87d08857 100644 --- a/lib/crowdb-access-iceberg/tests/file_digest_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_digest_test.rs @@ -5,6 +5,7 @@ use sha2::{Digest, Sha256}; fn owner() -> FileIdentity { FileIdentity { table: TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }, diff --git a/lib/crowdb-access-iceberg/tests/file_directory_test.rs b/lib/crowdb-access-iceberg/tests/file_directory_test.rs index c45a57346..a11d6b2b0 100644 --- a/lib/crowdb-access-iceberg/tests/file_directory_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_directory_test.rs @@ -8,6 +8,7 @@ fn directory(count: usize) -> ChunkDirectory { ChunkDirectory { owner: FileIdentity { table: TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }, @@ -34,6 +35,23 @@ fn directory(count: usize) -> ChunkDirectory { } } +#[test] +fn native_directories_preserve_storage_namespace_and_reject_foreign_owners() { + let mut node = directory(2); + node.owner.table.storage_namespace = Some(crowdb_access_iceberg::key::NamespaceId::random()); + let bytes = node.encode().unwrap(); + assert_eq!(&bytes[..5], b"ICEN\x02"); + assert_eq!( + ChunkDirectory::decode(&bytes, node.owner, node.height, node.length().unwrap()).unwrap(), + node + ); + let mut foreign = node.owner; + foreign.table.storage_namespace = Some(crowdb_access_iceberg::key::NamespaceId::random()); + assert!(ChunkDirectory::decode(&bytes, foreign, node.height, node.length().unwrap()).is_err()); + foreign.table.storage_namespace = None; + assert!(ChunkDirectory::decode(&bytes, foreign, node.height, node.length().unwrap()).is_err()); +} + #[test] fn directories_bind_every_child_to_a_bounded_height_owner_and_span() { for count in [1, 2, MAX_DIRECTORY_ENTRIES] { diff --git a/lib/crowdb-access-iceberg/tests/file_format_test.rs b/lib/crowdb-access-iceberg/tests/file_format_test.rs index cf0b3f71b..cbab4348d 100644 --- a/lib/crowdb-access-iceberg/tests/file_format_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_format_test.rs @@ -13,6 +13,7 @@ use crowdb_access_iceberg::key::{CatalogId, FileId, TableId}; async fn record(store: Arc, bytes: &[u8], block_size: usize) -> FileRecord { let owner = FileIdentity { table: TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }, diff --git a/lib/crowdb-access-iceberg/tests/file_json_test.rs b/lib/crowdb-access-iceberg/tests/file_json_test.rs index 12b453aa8..6e1cbbbf8 100644 --- a/lib/crowdb-access-iceberg/tests/file_json_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_json_test.rs @@ -14,6 +14,7 @@ use crowdb_access_iceberg::key::{CatalogId, FileId, TableId}; async fn record(store: Arc, bytes: &[u8], block_bytes: usize) -> FileRecord { let owner = FileIdentity { table: TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }, diff --git a/lib/crowdb-access-iceberg/tests/file_listing_test.rs b/lib/crowdb-access-iceberg/tests/file_listing_test.rs index 3539c78d4..3e587c034 100644 --- a/lib/crowdb-access-iceberg/tests/file_listing_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_listing_test.rs @@ -14,6 +14,7 @@ use fixture::TestFile; fn grant(fixture: &TestFile) -> FileGrant { FileGrant { + storage_namespace: None, context: fixture.context, table: fixture.table.table, principal: [7; 32], @@ -80,6 +81,7 @@ async fn two_table_principals_cannot_discover_each_others_published_names() { let selected = fixture.record("data/own.json", b"{}"); repository.publish(fixture.context, &selected).await.unwrap(); let other_table = crowdb_access_iceberg::file::TableLocation { + storage_namespace: None, catalog: fixture.context.catalog, table: TableId::random(), }; diff --git a/lib/crowdb-access-iceberg/tests/file_location_test.rs b/lib/crowdb-access-iceberg/tests/file_location_test.rs index d7f780e10..a0a702588 100644 --- a/lib/crowdb-access-iceberg/tests/file_location_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_location_test.rs @@ -3,6 +3,7 @@ use crowdb_access_iceberg::key::{CatalogId, TableId}; fn table() -> TableLocation { TableLocation { + storage_namespace: None, catalog: CatalogId::from_bytes(&[0xff; 16]).unwrap(), table: TableId::from_bytes(&[0xab; 16]).unwrap(), } diff --git a/lib/crowdb-access-iceberg/tests/file_record_test.rs b/lib/crowdb-access-iceberg/tests/file_record_test.rs index ed251e8a8..ac83159e2 100644 --- a/lib/crowdb-access-iceberg/tests/file_record_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_record_test.rs @@ -12,6 +12,7 @@ fn record(input: &[u8]) -> FileRecord { FileRecord { file: FileId::random(), location: TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), } diff --git a/lib/crowdb-access-iceberg/tests/file_scale_test.rs b/lib/crowdb-access-iceberg/tests/file_scale_test.rs index dacadbb6d..8af3b5521 100644 --- a/lib/crowdb-access-iceberg/tests/file_scale_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_scale_test.rs @@ -16,6 +16,7 @@ use crowdb_access_iceberg::{ async fn logical_tib(store: &blocks::TestBlocks) -> FileRecord { let owner = FileIdentity { table: TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }, diff --git a/lib/crowdb-access-iceberg/tests/file_seal_test.rs b/lib/crowdb-access-iceberg/tests/file_seal_test.rs index 426dd3208..c056a4a39 100644 --- a/lib/crowdb-access-iceberg/tests/file_seal_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_seal_test.rs @@ -14,6 +14,7 @@ use crowdb_access_iceberg::record::StorageRecord; fn owner() -> FileIdentity { FileIdentity { table: TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }, diff --git a/lib/crowdb-access-iceberg/tests/file_staging_test.rs b/lib/crowdb-access-iceberg/tests/file_staging_test.rs index e237da1dd..253d8de0e 100644 --- a/lib/crowdb-access-iceberg/tests/file_staging_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_staging_test.rs @@ -12,6 +12,7 @@ use std::sync::Arc; fn owner() -> FileIdentity { FileIdentity { table: TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }, diff --git a/lib/crowdb-access-iceberg/tests/file_stream_test.rs b/lib/crowdb-access-iceberg/tests/file_stream_test.rs index 5157ef6ac..8db08a7c3 100644 --- a/lib/crowdb-access-iceberg/tests/file_stream_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_stream_test.rs @@ -13,6 +13,7 @@ use crowdb_access_iceberg::key::{CatalogId, FileId, TableId}; fn owner() -> FileIdentity { FileIdentity { table: TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }, diff --git a/lib/crowdb-access-iceberg/tests/gc_proof_test.rs b/lib/crowdb-access-iceberg/tests/gc_proof_test.rs index 715c2e842..9a1c21b37 100644 --- a/lib/crowdb-access-iceberg/tests/gc_proof_test.rs +++ b/lib/crowdb-access-iceberg/tests/gc_proof_test.rs @@ -473,6 +473,7 @@ async fn live_sweep_recovers_a_lost_final_fence_release_response() { if task.phase == GcPhase::SweepWrites { let page = store .scan_gc(GcScan { + native: false, catalog: task.context.catalog, scope: Some(CatalogScope::FileWriteIntent), prefix: task.head.as_ref().unwrap().table.as_bytes().to_vec(), diff --git a/lib/crowdb-access-iceberg/tests/gc_reachability_test.rs b/lib/crowdb-access-iceberg/tests/gc_reachability_test.rs index 73f078328..2969b5490 100644 --- a/lib/crowdb-access-iceberg/tests/gc_reachability_test.rs +++ b/lib/crowdb-access-iceberg/tests/gc_reachability_test.rs @@ -16,6 +16,7 @@ mod blocks; fn table() -> TableLocation { TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), } diff --git a/lib/crowdb-access-iceberg/tests/gc_record_test.rs b/lib/crowdb-access-iceberg/tests/gc_record_test.rs index 177f9bd3f..89534d65a 100644 --- a/lib/crowdb-access-iceberg/tests/gc_record_test.rs +++ b/lib/crowdb-access-iceberg/tests/gc_record_test.rs @@ -53,6 +53,8 @@ fn task_codec_rejects_foreign_keys_and_invalid_progress() { assert!(task.validate().is_err()); task.scan_after.clear(); task.discovery_scope = 4; + task.validate().unwrap(); + task.discovery_scope = 5; assert!(task.validate().is_err()); task.discovery_scope = 1; task.validate().unwrap(); diff --git a/lib/crowdb-access-iceberg/tests/gc_tree_test.rs b/lib/crowdb-access-iceberg/tests/gc_tree_test.rs index 5e2def6aa..a26237edc 100644 --- a/lib/crowdb-access-iceberg/tests/gc_tree_test.rs +++ b/lib/crowdb-access-iceberg/tests/gc_tree_test.rs @@ -18,6 +18,7 @@ async fn directory_deletion_resumes_without_rereading_deleted_children() { let blocks = Arc::new(blocks::TestBlocks::default()); let owner = FileIdentity { table: TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }, @@ -97,6 +98,7 @@ async fn corrupt_directory_never_authorizes_a_child_deletion() { let blocks = blocks::TestBlocks::default(); let owner = FileIdentity { table: TableLocation { + storage_namespace: None, catalog: CatalogId::random(), table: TableId::random(), }, diff --git a/lib/crowdb-access-iceberg/tests/gc_worker_test.rs b/lib/crowdb-access-iceberg/tests/gc_worker_test.rs index 0da02d896..1f8e869ed 100644 --- a/lib/crowdb-access-iceberg/tests/gc_worker_test.rs +++ b/lib/crowdb-access-iceberg/tests/gc_worker_test.rs @@ -39,7 +39,23 @@ async fn fixture( GcLimits, FileId, ) { - let fixture = common::file::TestFile::new(common::TestStore::default()).await; + fixture_at(retire, false).await +} + +async fn fixture_at( + retire: bool, + native: bool, +) -> ( + common::file::TestFile, + Arc, + GcTask, + GcLimits, + FileId, +) { + let mut fixture = common::file::TestFile::new(common::TestStore::default()).await; + if native { + fixture.table.storage_namespace = Some(crowdb_access_iceberg::key::NamespaceId::random()); + } let blocks = Arc::new(gc_blocks::TestReclaimBlocks::default()); let owner = FileIdentity { table: fixture.table, @@ -62,7 +78,11 @@ async fn fixture( .publish(fixture.context, &file) .await .unwrap(); - let mut retired = CatalogAuthority::new(fixture.context.catalog, "retired".into()).unwrap(); + let mut retired = if native { + CatalogAuthority::new_native(fixture.context.catalog, "retired".into()).unwrap() + } else { + CatalogAuthority::new(fixture.context.catalog, "retired".into()).unwrap() + }; if retire { retired.lifecycle = CatalogLifecycle::Retired; } @@ -231,6 +251,58 @@ async fn retired_file_reclamation_survives_worker_restart_at_every_step() { assert!(!published_file_exists(&fixture, file).await); } +#[tokio::test] +async fn native_retirement_discovers_and_reclaims_single_filename_records_after_restart() { + use crowdb_access_iceberg::file::canonical_file_key; + use crowdb_access_iceberg::key::{StorageKey, TableStorageKey, TableStorageScope}; + let (fixture, blocks, mut task, limits, _) = fixture_at(true, true).await; + let file = canonical_file_key(&fixture.table.file("data/object.parquet").unwrap()) + .encode() + .unwrap(); + let projection = TableStorageKey { + table: fixture.table.storage_path().unwrap(), + scope: TableStorageScope::Projection, + suffix: "1/derived/1/0".into(), + } + .encode() + .unwrap(); + fixture + .store + .compare_exchange( + &projection, + None, + b"derived", + mutation_identity(&projection, None, b"derived"), + ) + .await + .unwrap(); + for _ in 0..350 { + let repository = GcRepository::new(fixture.store.clone()); + let worker = GcWorker::new(repository.clone(), blocks.clone(), limits).unwrap(); + task = worker.step(&task, 2000_u64.max(task.retry_at_ms)).await.unwrap(); + task = repository + .task(task.context.catalog, task.identity) + .await + .unwrap() + .unwrap(); + if task.phase == GcPhase::Complete { + break; + } + } + assert_eq!(task.phase, GcPhase::Complete, "{task:?}"); + assert_eq!(task.deleted, 1); + assert_eq!(task.reclaimed_bytes, 4096); + assert!(blocks.blocks.values.load().is_empty()); + assert!(fixture.store.get(&file).await.unwrap().is_none()); + assert!(fixture.store.get(&projection).await.unwrap().is_none()); + assert!(!fixture + .store + .values + .load() + .keys() + .any(|key| matches!(StorageKey::decode(key), Ok(StorageKey::NativeTable(_))))); +} + #[tokio::test] async fn retired_streamed_file_reclaims_location_after_worker_restart() { let (fixture, blocks, mut task, limits, _) = fixture(true).await; @@ -896,6 +968,7 @@ async fn missing_file_claim_stops_sweep_before_any_physical_deletion() { let page = fixture .store .scan_gc(GcScan { + native: false, catalog: task.context.catalog, scope: Some(CatalogScope::GcClaim), prefix: Vec::new(), diff --git a/lib/crowdb-access-iceberg/tests/manifest_metadata_test.rs b/lib/crowdb-access-iceberg/tests/manifest_metadata_test.rs index ef17f0281..7170e051b 100644 --- a/lib/crowdb-access-iceberg/tests/manifest_metadata_test.rs +++ b/lib/crowdb-access-iceberg/tests/manifest_metadata_test.rs @@ -8,6 +8,7 @@ use crowdb_access_iceberg::manifest::{ fn table() -> TableLocation { TableLocation { + storage_namespace: None, catalog: CatalogId::from_bytes(&[1; 16]).unwrap(), table: TableId::from_bytes(&[2; 16]).unwrap(), } diff --git a/lib/crowdb-access-iceberg/tests/metadata_projection_test.rs b/lib/crowdb-access-iceberg/tests/metadata_projection_test.rs index 5f8111316..04d3a578b 100644 --- a/lib/crowdb-access-iceberg/tests/metadata_projection_test.rs +++ b/lib/crowdb-access-iceberg/tests/metadata_projection_test.rs @@ -14,6 +14,40 @@ use sha2::{Digest, Sha256}; const JSON: &[u8] = b"{ \n \"refs\" : {\"main\": { \"snapshot-id\": 123 }}, \"unknown\" : [1, 2] }\n"; +#[tokio::test] +async fn native_projection_pages_are_bound_to_storage_namespace_and_generation() { + use crowdb_access_iceberg::key::{NamespaceId, StorageKey, TableStorageScope}; + let mut fixture = TestProjection::new(JSON).await; + let mut table = fixture.record.location.table(); + table.storage_namespace = Some(NamespaceId::random()); + fixture.record.location = table.file("metadata/v7.json").unwrap(); + assert!(fixture.projection.put(&fixture.record, 7, JSON).await); + for key in fixture.store.values.load().keys() { + let StorageKey::NativeTable(key) = StorageKey::decode(key).unwrap() else { + panic!() + }; + assert_eq!(key.table, table.storage_path().unwrap()); + assert_eq!(key.scope, TableStorageScope::Projection); + assert!(key.suffix.starts_with("7/")); + } + assert!(matches!( + fixture + .projection + .select(fixture.record.clone(), 7, &["refs"]) + .await + .unwrap(), + MetadataRead::Selected(_) + )); + assert_eq!(fixture.blocks.reads.load(Ordering::SeqCst), 0); + table.storage_namespace = Some(NamespaceId::random()); + let mut foreign = fixture.record.clone(); + foreign.location = table.file("metadata/v7.json").unwrap(); + assert!(!matches!( + fixture.projection.select(foreign, 7, &["refs"]).await, + Ok(MetadataRead::Selected(_)) + )); +} + #[tokio::test] async fn selected_children_preserve_bytes_without_reading_canonical_blocks() { let fixture = TestProjection::new(JSON).await; diff --git a/lib/crowdb-access-iceberg/tests/multipart_record_test.rs b/lib/crowdb-access-iceberg/tests/multipart_record_test.rs index 22a1507ba..80762cb3b 100644 --- a/lib/crowdb-access-iceberg/tests/multipart_record_test.rs +++ b/lib/crowdb-access-iceberg/tests/multipart_record_test.rs @@ -11,6 +11,62 @@ use crowdb_access_iceberg::key::{CatalogId, CatalogScope, FileId, IcebergKey}; use crowdb_access_iceberg::record::StorageRecord; use std::sync::Arc; +#[tokio::test] +async fn native_multipart_identity_round_trips_the_full_storage_path() { + let mut session = fixtures::session(); + session.owner.table.storage_namespace = Some(crowdb_access_iceberg::key::NamespaceId::random()); + session.location = session.owner.table.file(session.location.relative_key()).unwrap(); + let record = StorageRecord::MultipartSession(Box::new(session.clone())); + assert_eq!( + StorageRecord::decode(&session.key(), &record.encode().unwrap()).unwrap(), + record + ); + session.owner.table.storage_namespace = Some(crowdb_access_iceberg::key::NamespaceId::random()); + assert!(session.validate().is_err()); +} + +#[tokio::test] +async fn native_partial_multipart_checkpoint_survives_record_reload() { + let store = Arc::new(TestBlocks::default()); + let mut session = fixtures::session(); + session.owner.table.storage_namespace = Some(crowdb_access_iceberg::key::NamespaceId::random()); + session.location = session.owner.table.file("data/native.parquet").unwrap(); + let owner = FileIdentity { + file: FileId::random(), + ..session.owner + }; + let mut writer = FileTreeWriter::new(store.clone(), owner, 3).unwrap(); + writer.push(b"bytes").await.unwrap(); + let part = AssemblyPart { + ordinal: 0, + owner, + tree: writer.finish().await.unwrap(), + }; + session.part_count = 1; + session.staged_bytes = 5; + session.phase = MultipartPhase::Completing; + session.completion = Some(fixtures::completion(&session)); + let assembly = FileAssembly::new(store, session.owner, [3; 32], 1, 1000, 2, 3).unwrap(); + let completion = session.completion.as_mut().unwrap(); + completion.progress = assembly.advance(&completion.progress, &part).await.unwrap(); + assert_eq!(completion.progress.part_digest.as_ref().unwrap().len(), 205); + let record = StorageRecord::MultipartSession(Box::new(session.clone())); + assert_eq!( + StorageRecord::decode(&session.key(), &record.encode().unwrap()).unwrap(), + record + ); + let completion = session.completion.as_mut().unwrap(); + completion + .progress + .active + .as_mut() + .unwrap() + .owner + .table + .storage_namespace = Some(crowdb_access_iceberg::key::NamespaceId::random()); + assert!(session.validate().is_err()); +} + #[tokio::test] async fn multipart_records_round_trip_open_partial_publishing_published_and_aborted_states() { let store = Arc::new(TestBlocks::default()); diff --git a/lib/crowdb-access-iceberg/tests/native_file_runtime_test.rs b/lib/crowdb-access-iceberg/tests/native_file_runtime_test.rs new file mode 100644 index 000000000..e8aef610b --- /dev/null +++ b/lib/crowdb-access-iceberg/tests/native_file_runtime_test.rs @@ -0,0 +1,211 @@ +#[path = "common/store.rs"] +mod common; +#[path = "common/file.rs"] +mod fixture; + +use std::sync::atomic::Ordering; + +use crowdb_access_iceberg::{ + file::{ + canonical_file_key, FileDigest, FileGrant, FileGrantIssuer, FileIdentity, FileListRequest, + FileListTokens, FileOperation, FileOperations, FileRepository, TableLocation, + }, + key::{NamespaceId, OperationId, StorageKey, StorageTablePath, TableStorageScope}, + record::StorageRecord, +}; + +async fn native_fixture() -> fixture::TestFile { + let mut fixture = fixture::TestFile::new(common::TestStore::default()).await; + fixture.table = StorageTablePath { + catalog: fixture.context.catalog, + storage_namespace: NamespaceId::random(), + table: fixture.table.table, + } + .into(); + fixture +} + +fn grant(fixture: &fixture::TestFile) -> FileGrant { + FileGrant { + storage_namespace: fixture.table.storage_namespace, + context: fixture.context, + table: fixture.table.table, + principal: [1; 32], + nonce: OperationId::random(), + issued_ms: 100, + expires_ms: 10_000, + operations: FileOperations::new(&[FileOperation::Get, FileOperation::ListObjects]).unwrap(), + max_request_bytes: 1024, + max_file_bytes: 1024, + } +} + +#[tokio::test] +async fn native_publication_has_one_filename_record_and_conflicts_preserve_bytes() { + let fixture = native_fixture().await; + let repository = FileRepository::new(fixture.store.clone()); + let candidate = fixture.record("metadata/v1.json", b"{}"); + assert_eq!( + repository.publish(fixture.context, &candidate).await.unwrap(), + candidate + ); + let equal = fixture.record("metadata/v1.json", b"{}"); + assert_eq!( + repository.publish(fixture.context, &equal).await.unwrap(), + candidate + ); + assert!(repository + .publish(fixture.context, &fixture.record("metadata/v1.json", b"[]")) + .await + .is_err()); + assert_eq!( + repository + .load(fixture.context, &candidate.location) + .await + .unwrap(), + Some(candidate.clone()) + ); + let key = canonical_file_key(&candidate.location); + let snapshot = fixture.store.values.load_full(); + let files: Vec<_> = snapshot.keys().filter(|key| matches!(StorageKey::decode(key), Ok(StorageKey::NativeTable(key)) if key.scope == TableStorageScope::File)).collect(); + assert_eq!(files, [&key.encode().unwrap()]); + assert_eq!( + StorageRecord::decode_storage(&key, &snapshot[&key.encode().unwrap()].bytes).unwrap(), + StorageRecord::File(Box::new(candidate.clone())) + ); + let mut foreign = candidate.location.table(); + foreign.storage_namespace = Some(NamespaceId::random()); + let foreign = canonical_file_key(&foreign.file(candidate.location.relative_key()).unwrap()); + assert!(StorageRecord::decode_storage(&foreign, &snapshot[&key.encode().unwrap()].bytes).is_err()); +} + +#[tokio::test] +async fn native_publication_lost_reply_replays_the_same_single_record() { + let fixture = native_fixture().await; + let repository = FileRepository::new(fixture.store.clone()); + let candidate = fixture.record("metadata/v1.json", b"{}"); + fixture.store.file_record_reply_loss.store(true, Ordering::SeqCst); + assert!(repository.publish(fixture.context, &candidate).await.is_err()); + let restarted = FileRepository::new(fixture.store.clone()); + assert_eq!( + restarted.publish(fixture.context, &candidate).await.unwrap(), + candidate + ); + assert_eq!( + restarted + .load(fixture.context, &candidate.location) + .await + .unwrap(), + Some(candidate) + ); +} + +#[tokio::test] +async fn many_metadata_files_page_by_public_prefix_without_file_resolution_gets() { + let fixture = native_fixture().await; + let repository = FileRepository::new(fixture.store.clone()); + for index in 0..600 { + for prefix in ["metadata", "other"] { + let record = fixture.record(&format!("{prefix}/v{index:04}.json"), b"{}"); + repository.publish(fixture.context, &record).await.unwrap(); + } + } + let tokens = FileListTokens::new([1; 32]).unwrap(); + let grant = grant(&fixture); + let mut request = FileListRequest { + table: fixture.table, + prefix: format!("{}metadata/", fixture.table.object_prefix()), + delimiter: None, + encoding_url: false, + max_keys: 73, + continuation_token: None, + start_after: None, + }; + let mut keys = Vec::new(); + loop { + let before = fixture.store.reads.load(Ordering::SeqCst); + let page = repository.list(&grant, &request, &tokens, 200).await.unwrap(); + // The two Ready root checks are independent of file-record resolution. + assert_eq!(fixture.store.reads.load(Ordering::SeqCst) - before, 2); + keys.extend(page.files.into_iter().map(|file| file.key)); + request.continuation_token = page.next_continuation_token; + if request.continuation_token.is_none() { + break; + } + } + let expected: Vec<_> = (0..600) + .map(|index| format!("{}metadata/v{index:04}.json", fixture.table.object_prefix())) + .collect(); + assert_eq!(keys, expected); + request.prefix = format!("{}metadata/v000", fixture.table.object_prefix()); + let page = repository.list(&grant, &request, &tokens, 200).await.unwrap(); + assert_eq!(page.files.len(), 10); + let mut foreign_grant = grant.clone(); + foreign_grant.storage_namespace = Some(NamespaceId::random()); + assert!(repository + .list(&foreign_grant, &request, &tokens, 200) + .await + .is_err()); +} + +#[tokio::test] +async fn native_grants_and_digest_checkpoints_bind_the_storage_namespace() { + let fixture = native_fixture().await; + let issuer = FileGrantIssuer::new([2; 32], 10_000).unwrap(); + let grant = grant(&fixture); + let credentials = issuer.issue(grant.clone()).unwrap(); + let verified = issuer + .verify( + credentials.access_key_id(), + credentials.session_token(), + fixture.context, + 200, + ) + .unwrap(); + assert_eq!(verified.grant(), &grant); + let location = fixture.table.file("metadata/v1.json").unwrap(); + verified + .grant() + .authorize(FileOperation::Get, &location, 0, 2) + .unwrap(); + let mut foreign = fixture.table; + foreign.storage_namespace = Some(NamespaceId::random()); + assert!(verified + .grant() + .authorize( + FileOperation::Get, + &foreign.file("metadata/v1.json").unwrap(), + 0, + 2 + ) + .is_err()); + let owner = FileIdentity { + table: fixture.table, + file: crowdb_access_iceberg::key::FileId::random(), + }; + let mut digest = FileDigest::new(owner); + digest.update(b"hello").unwrap(); + let checkpoint = digest.checkpoint(); + let restored = FileDigest::restore(owner, &checkpoint).unwrap(); + assert_eq!(restored.finish(), digest.finish()); + assert!(FileDigest::restore( + FileIdentity { + table: foreign, + ..owner + }, + &checkpoint + ) + .is_err()); + let legacy: TableLocation = TableLocation { + storage_namespace: None, + ..fixture.table + }; + assert!(FileDigest::restore( + FileIdentity { + table: legacy, + ..owner + }, + &checkpoint + ) + .is_err()); +} diff --git a/lib/crowdb-access-iceberg/tests/native_injection_test.rs b/lib/crowdb-access-iceberg/tests/native_injection_test.rs new file mode 100644 index 000000000..1007a88d1 --- /dev/null +++ b/lib/crowdb-access-iceberg/tests/native_injection_test.rs @@ -0,0 +1,90 @@ +#[path = "common/file_blocks.rs"] +mod blocks; +#[path = "common/store.rs"] +mod common; +#[path = "common/file.rs"] +#[allow(dead_code)] +mod fixture; + +use crowdb_access_iceberg::{ + file::{ + DirectFileLocations, FileIdentity, FileRepository, FileTreeWriter, NativeInjectionRepository, + NativeInjectionRequest, + }, + key::{FileId, NamespaceId, OperationId, StorageKey, TableStorageScope}, + operation::{RequestIdentity, RETRY_WINDOW_MS}, +}; +use std::{ + collections::BTreeMap, + sync::{atomic::Ordering, Arc}, +}; + +#[tokio::test] +async fn durable_injection_selection_survives_both_lost_replies_and_key_rotation() { + for selection_loss in [true, false] { + let mut fixture = fixture::TestFile::new(common::TestStore::default()).await; + fixture.table.storage_namespace = Some(NamespaceId::random()); + let blocks = Arc::new(blocks::TestBlocks::default()); + let owner = FileIdentity { + table: fixture.table, + file: FileId::random(), + }; + let mut writer = FileTreeWriter::new(blocks, owner, 128).unwrap(); + writer.push(&[7; 1024]).await.unwrap(); + let tree = writer.finish().await.unwrap(); + let request = NativeInjectionRequest { + context: fixture.context, + table: fixture.table, + identity: RequestIdentity { + operation: OperationId::random(), + issued_ms: 100, + }, + principal: [3; 32], + stem: "part-0001".into(), + digest: tree.digest, + }; + let signer = DirectFileLocations::new(1, BTreeMap::from([(1, [1; 32])])).unwrap(); + let repository = NativeInjectionRepository::new(fixture.store.clone()); + assert!(repository.replay(&request, 101).await.unwrap().is_none()); + if selection_loss { + fixture + .store + .native_injection_reply_loss + .store(true, Ordering::SeqCst); + repository + .finish(&request, owner, tree, &signer, 101) + .await + .unwrap(); + } else { + fixture.store.file_record_reply_loss.store(true, Ordering::SeqCst); + assert!(repository + .finish(&request, owner, tree, &signer, 101) + .await + .is_err()); + } + let restarted = NativeInjectionRepository::new(fixture.store.clone()); + let file = restarted.replay(&request, 102).await.unwrap().unwrap(); + assert_eq!(file.file, owner.file); + let rotated = DirectFileLocations::new(2, BTreeMap::from([(1, [1; 32]), (2, [2; 32])])).unwrap(); + assert_eq!(rotated.resolve(&file.location).unwrap(), Some(file.clone())); + assert_eq!(restarted.replay(&request, 103).await.unwrap(), Some(file.clone())); + let records = fixture.store.values.load_full(); + assert_eq!(records.keys().filter(|key| matches!(StorageKey::decode(key), Ok(StorageKey::NativeTable(key)) if key.scope == TableStorageScope::File)).count(), 1); + assert_eq!(records.keys().filter(|key| matches!(StorageKey::decode(key), Ok(StorageKey::NativeTable(key)) if key.scope == TableStorageScope::Operation)).count(), 1); + let mut changed = request.clone(); + changed.digest[0] ^= 1; + assert!(restarted.replay(&changed, 104).await.is_err()); + changed = request.clone(); + changed.principal[0] ^= 1; + assert!(restarted.replay(&changed, 104).await.is_err()); + changed = request.clone(); + changed.table.storage_namespace = Some(NamespaceId::random()); + assert!(restarted.replay(&changed, 104).await.unwrap().is_none()); + assert!(restarted.replay(&request, 100 + RETRY_WINDOW_MS).await.is_err()); + FileRepository::new(fixture.store.clone()) + .mark_deleted(fixture.context, &file, 105) + .await + .unwrap(); + assert!(restarted.replay(&request, 106).await.is_err()); + } +} diff --git a/lib/crowdb-access-iceberg/tests/native_storage_path_test.rs b/lib/crowdb-access-iceberg/tests/native_storage_path_test.rs new file mode 100644 index 000000000..e6e2edb6c --- /dev/null +++ b/lib/crowdb-access-iceberg/tests/native_storage_path_test.rs @@ -0,0 +1,225 @@ +use crowdb_access_iceberg::{ + file::{NativeFileLocation, MAX_OBJECT_KEY_BYTES}, + key::{ + CatalogId, CatalogScope, IcebergKey, NamespaceId, StorageKey, StorageTablePath, TableId, + TableStorageKey, TableStorageScope, + }, +}; + +fn table() -> StorageTablePath { + StorageTablePath { + catalog: CatalogId::from_bytes(&[0xab; 16]).unwrap(), + storage_namespace: NamespaceId::from_bytes(&[0xcd; 16]).unwrap(), + table: TableId::from_bytes(&[0xef; 16]).unwrap(), + } +} + +fn key(table: StorageTablePath, scope: TableStorageScope, suffix: &str) -> TableStorageKey { + TableStorageKey { + table, + scope, + suffix: suffix.to_owned(), + } +} + +#[test] +fn native_uri_preserves_immutable_identity_and_exact_filename() { + let table = table(); + assert_eq!(table.to_string().len(), StorageTablePath::PREFIX_BYTES); + assert_eq!(table.to_string().parse::().unwrap(), table); + assert_eq!(table.uri(), format!("s3://iceberg/{table}")); + for filename in ["metadata/v1.json", "数据/a+b", "a%2Fb", "a//b", "a/", "a b"] { + let location = table.logical_file(filename).unwrap(); + assert_eq!(location.table(), table); + assert_eq!(location.relative_key(), filename); + assert_eq!( + location.to_string().parse::().unwrap(), + location + ); + assert_eq!( + NativeFileLocation::from_object_key("iceberg", &location.object_key()).unwrap(), + location + ); + } + assert_ne!(table.file("a%2Fb").unwrap(), table.file("a/b").unwrap()); + assert!(table.logical_file("f/v1/name.token.parquet").is_err()); + assert!(table.file("f/v1/name.token.parquet").is_ok()); +} + +#[test] +fn native_uri_rejects_aliases_incomplete_scopes_and_escaping_names() { + let table = table(); + let uri = table.file("metadata/v1.json").unwrap().to_string(); + for invalid in [ + uri.replacen("abab", "ABAB", 1), + uri.replacen("abababab", "abab-abab", 1), + uri.replacen("iceberg/", "iceberg:9000/", 1), + uri.replacen("iceberg/", "user@iceberg/", 1), + uri.replacen("iceberg/", "iceberg-old/", 1), + uri.replacen("s3:", "s3a:", 1), + uri.replacen(&table.catalog.to_string(), &"0".repeat(32), 1), + ] { + assert!(invalid.parse::().is_err(), "{invalid}"); + } + for filename in ["", "/a", "../a", "a/../b", "a/./b", "a\\b", "a?b", "a#b", "a\0b"] { + assert!(table.file(filename).is_err(), "{filename:?}"); + } + for prefix in [ + format!("{}/", table.catalog), + format!("{}/{}/", table.catalog, table.storage_namespace), + table.to_string().trim_end_matches('/').to_owned(), + format!("{table}metadata/"), + ] { + assert!(prefix.parse::().is_err(), "{prefix}"); + } + assert!(table.uri().parse::().is_err()); +} + +#[test] +fn filename_limit_counts_three_ids_and_utf8_bytes() { + let table = table(); + let available = MAX_OBJECT_KEY_BYTES - StorageTablePath::PREFIX_BYTES; + let filename = "x".repeat(available); + let location = table.file(&filename).unwrap(); + assert_eq!(location.object_key().len(), MAX_OBJECT_KEY_BYTES); + assert_eq!( + location.to_string().parse::().unwrap(), + location + ); + assert!(table.file(&(filename + "x")).is_err()); + assert!(table.file(&("x".repeat(available - 1) + "冰")).is_err()); +} + +#[test] +fn table_keys_are_version_isolated_and_round_trip() { + let table = table(); + let legacy = IcebergKey::Catalog { + catalog: table.catalog, + scope: CatalogScope::File, + suffix: vec![1; 16], + } + .encode() + .unwrap(); + assert!(TableStorageKey::decode(&legacy).is_err()); + assert!(matches!( + StorageKey::decode(&legacy).unwrap(), + StorageKey::Legacy(_) + )); + for (scope, suffix, component) in [ + (TableStorageScope::Head, "", "head"), + (TableStorageScope::File, "metadata/v1.json", "file/"), + (TableStorageScope::Projection, "1/receipt", "projection/"), + ( + TableStorageScope::Operation, + "01010101010101010101010101010101/state", + "operation/", + ), + ( + TableStorageScope::Multipart, + "01010101010101010101010101010101/part/1", + "multipart/", + ), + ( + TableStorageScope::Gc, + "01010101010101010101010101010101/state", + "gc/", + ), + ] { + let key = key(table, scope, suffix); + let bytes = key.encode().unwrap(); + assert_eq!(TableStorageKey::decode(&bytes).unwrap(), key); + assert_eq!( + StorageKey::decode(&bytes).unwrap(), + StorageKey::NativeTable(key.clone()) + ); + assert!(IcebergKey::decode(&bytes).is_err()); + assert!(TableStorageKey::table_range(table).contains(&bytes)); + assert_eq!(&bytes[5..], format!("{table}{component}{suffix}").as_bytes()); + } +} + +#[test] +fn scan_validation_never_combines_storage_versions() { + let table = table(); + let legacy = IcebergKey::catalog_range(table.catalog); + let native = TableStorageKey::table_range(table); + assert!(StorageKey::validate_range(&legacy).is_ok()); + assert!(StorageKey::validate_range(&native).is_ok()); + assert!(StorageKey::validate_range(&TableStorageKey::file_range(table, "metadata/").unwrap()).is_ok()); + assert!(StorageKey::validate_range(&(legacy.start..native.end.clone())).is_err()); + assert!(StorageKey::validate_range(&(native.end.clone()..native.start.clone())).is_err()); + assert!(StorageKey::validate_range(&(native.start.clone()..native.start)).is_err()); + assert!(StorageKey::decode(b"ICE\0\x03").is_err()); + assert!(StorageKey::decode(b"ICE\0\x02not-a-path").is_err()); +} + +#[test] +fn prefix_scans_exclude_other_kinds_namespaces_and_tables() { + let table = table(); + let prefix = TableStorageKey::file_range(table, "metadata/v1").unwrap(); + for filename in [ + "metadata/v1.json", + "metadata/v10.json", + "metadata/v1/manifest.avro", + ] { + assert!(prefix.contains(&key(table, TableStorageScope::File, filename).encode().unwrap())); + } + for filename in ["data/v1.parquet", "metadata/v2.json", "metadata/v0.json"] { + assert!(!prefix.contains(&key(table, TableStorageScope::File, filename).encode().unwrap())); + } + for foreign in [ + StorageTablePath { + table: TableId::random(), + ..table + }, + StorageTablePath { + storage_namespace: NamespaceId::random(), + ..table + }, + StorageTablePath { + catalog: CatalogId::random(), + ..table + }, + ] { + let bytes = key(foreign, TableStorageScope::File, "metadata/v1.json") + .encode() + .unwrap(); + assert!(!prefix.contains(&bytes)); + assert!(!TableStorageKey::table_range(table).contains(&bytes)); + } + let all_files = TableStorageKey::file_range(table, "").unwrap(); + assert!(!all_files.contains(&key(table, TableStorageScope::Head, "").encode().unwrap())); + assert!(!all_files.contains( + &key(table, TableStorageScope::Projection, "1/receipt") + .encode() + .unwrap() + )); + assert!(TableStorageKey::file_range(table, "../").is_err()); +} + +#[test] +fn malformed_keys_cannot_alias_head_or_filename() { + let table = table(); + for (scope, suffix) in [ + (TableStorageScope::Head, "file"), + (TableStorageScope::File, ""), + (TableStorageScope::File, "../file"), + (TableStorageScope::Operation, "../operation"), + (TableStorageScope::Operation, "upload/state"), + (TableStorageScope::Gc, "00000000000000000000000000000000/state"), + ( + TableStorageScope::Multipart, + "ABABABABABABABABABABABABABABABAB/part", + ), + (TableStorageScope::Projection, "0/receipt"), + (TableStorageScope::Projection, "01/receipt"), + (TableStorageScope::Projection, "1/"), + ] { + assert!(key(table, scope, suffix).encode().is_err()); + } + for suffix in ["head/", "head/file", "file/", "unknown/x", "file/../x"] { + let mut bytes = b"ICE\0\x02".to_vec(); + bytes.extend_from_slice(format!("{table}{suffix}").as_bytes()); + assert!(TableStorageKey::decode(&bytes).is_err(), "{suffix}"); + } +} diff --git a/lib/crowdb-access-iceberg/tests/parquet_delete_encoding_test.rs b/lib/crowdb-access-iceberg/tests/parquet_delete_encoding_test.rs index 7bd262f9a..b157ab622 100644 --- a/lib/crowdb-access-iceberg/tests/parquet_delete_encoding_test.rs +++ b/lib/crowdb-access-iceberg/tests/parquet_delete_encoding_test.rs @@ -30,6 +30,7 @@ impl PositionDeleteTargets for TestTargets { fn table() -> TableLocation { TableLocation { + storage_namespace: None, catalog: CatalogId::from_bytes(&[1; 16]).unwrap(), table: TableId::from_bytes(&[2; 16]).unwrap(), } diff --git a/lib/crowdb-access-iceberg/tests/parquet_nullable_test.rs b/lib/crowdb-access-iceberg/tests/parquet_nullable_test.rs index 49703497f..43e66fc00 100644 --- a/lib/crowdb-access-iceberg/tests/parquet_nullable_test.rs +++ b/lib/crowdb-access-iceberg/tests/parquet_nullable_test.rs @@ -29,6 +29,7 @@ async fn official_parquet_java_v1_v2_nested_and_all_null_files_decode() { let (store, record) = fixture::stored_content( &bytes, TableLocation { + storage_namespace: None, catalog: CatalogId::from_bytes(&[1; 16]).unwrap(), table: TableId::from_bytes(&[2; 16]).unwrap(), }, diff --git a/lib/crowdb-access-iceberg/tests/parquet_position_delete_test.rs b/lib/crowdb-access-iceberg/tests/parquet_position_delete_test.rs index b1eac8cf0..68d47e501 100644 --- a/lib/crowdb-access-iceberg/tests/parquet_position_delete_test.rs +++ b/lib/crowdb-access-iceberg/tests/parquet_position_delete_test.rs @@ -34,6 +34,7 @@ async fn official_iceberg_java_zstd_v1_and_v2_delete_files_decode() { use crowdb_access_iceberg::file::TableLocation; use crowdb_access_iceberg::key::{CatalogId, TableId}; let table = TableLocation { + storage_namespace: None, catalog: CatalogId::from_bytes(&[1; 16]).unwrap(), table: TableId::from_bytes(&[2; 16]).unwrap(), }; @@ -56,6 +57,7 @@ async fn canonical_storage_does_not_make_corrupted_sdk_pages_valid() { use crowdb_access_iceberg::file::TableLocation; use crowdb_access_iceberg::key::{CatalogId, TableId}; let table = TableLocation { + storage_namespace: None, catalog: CatalogId::from_bytes(&[1; 16]).unwrap(), table: TableId::from_bytes(&[2; 16]).unwrap(), }; diff --git a/lib/crowdb-access-iceberg/tests/parquet_scalar_test.rs b/lib/crowdb-access-iceberg/tests/parquet_scalar_test.rs index 5b824336f..61901df9a 100644 --- a/lib/crowdb-access-iceberg/tests/parquet_scalar_test.rs +++ b/lib/crowdb-access-iceberg/tests/parquet_scalar_test.rs @@ -34,6 +34,7 @@ async fn official_java_v1_v2_nullable_scalar_files_decode_with_and_without_dicti let (store, record) = fixture::stored_content( &bytes, crowdb_access_iceberg::file::TableLocation { + storage_namespace: None, catalog: crowdb_access_iceberg::key::CatalogId::from_bytes(&[1; 16]).unwrap(), table: crowdb_access_iceberg::key::TableId::from_bytes(&[2; 16]).unwrap(), }, diff --git a/lib/crowdb-access-iceberg/tests/record_test.rs b/lib/crowdb-access-iceberg/tests/record_test.rs index afc19f107..8760fd981 100644 --- a/lib/crowdb-access-iceberg/tests/record_test.rs +++ b/lib/crowdb-access-iceberg/tests/record_test.rs @@ -17,6 +17,46 @@ fn root_key() -> IcebergKey { } } +#[test] +fn native_catalog_authority_uses_a_version_legacy_servers_cannot_silently_relabel() { + use crowdb_access_iceberg::catalog::{Capabilities, CatalogStorageLayout}; + let catalog = CatalogId::random(); + let key = IcebergKey::Catalog { + catalog, + scope: CatalogScope::Authority, + suffix: Vec::new(), + }; + for (authority, version) in [ + (CatalogAuthority::new(catalog, "legacy".into()).unwrap(), 1), + (CatalogAuthority::new_native(catalog, "native".into()).unwrap(), 2), + ] { + let record = StorageRecord::Authority(authority.clone()); + let bytes = record.encode().unwrap(); + assert_eq!( + fb::root_as_fbiceberg_record(&bytes).unwrap().schema_version(), + version + ); + assert_eq!(StorageRecord::decode(&key, &bytes).unwrap(), record); + let activated = authority + .activated(Capabilities::from_bits(0x3fff).unwrap()) + .unwrap(); + assert_eq!(activated.storage_layout, authority.storage_layout); + assert_eq!(activated.locator_key, authority.locator_key); + assert_eq!( + activated.renamed("renamed".into()).unwrap().storage_layout, + authority.storage_layout + ); + assert_eq!( + activated.renamed("renamed".into()).unwrap().locator_key, + authority.locator_key + ); + assert_eq!( + authority.storage_layout == CatalogStorageLayout::Native, + version == 2 + ); + } +} + #[test] fn every_root_phase_round_trips_with_its_exact_catalog_and_deadline() { let previous = CatalogContext { @@ -130,6 +170,8 @@ fn authority_codec_rejects_unknown_lifecycle_capabilities_and_invalid_generation let authority = FBCatalogAuthority::create( &mut builder, &FBCatalogAuthorityArgs { + locator_key: None, + storage_layout: 1, catalog: Some(identity), display_name: Some(name), lifecycle, diff --git a/lib/crowdb-access-iceberg/tests/table_commit_publication_test.rs b/lib/crowdb-access-iceberg/tests/table_commit_publication_test.rs index d289c9561..dda2fd5b2 100644 --- a/lib/crowdb-access-iceberg/tests/table_commit_publication_test.rs +++ b/lib/crowdb-access-iceberg/tests/table_commit_publication_test.rs @@ -168,6 +168,7 @@ async fn lifecycle_request( .put( name_key(head.catalog, head.namespace, &head.name).unwrap(), StorageRecord::TableMapping(TableMapping { + storage_namespace: None, catalog: head.catalog, namespace: head.namespace, name: head.name.clone(), diff --git a/lib/crowdb-access-iceberg/tests/table_create_publication_test.rs b/lib/crowdb-access-iceberg/tests/table_create_publication_test.rs index 7cac625d3..80b1cac15 100644 --- a/lib/crowdb-access-iceberg/tests/table_create_publication_test.rs +++ b/lib/crowdb-access-iceberg/tests/table_create_publication_test.rs @@ -22,6 +22,88 @@ use crowdb_access_iceberg::{ table::{read_table_metadata_document, TableMetadataLimits, TableRepository}, }; +#[tokio::test] +async fn native_catalog_creation_freezes_namespace_in_storage_identity_and_replays() { + use crowdb_access_iceberg::catalog::{CatalogAuthority, CatalogStorageLayout}; + use crowdb_access_iceberg::key::{CatalogScope, IcebergKey, StorageKey, TableStorageScope}; + let test = TestCreation::new().await; + let authority = CatalogAuthority::new_native(test.fixture.context.catalog, "native".into()).unwrap(); + test.fixture + .put( + IcebergKey::Catalog { + catalog: authority.catalog, + scope: CatalogScope::Authority, + suffix: Vec::new(), + }, + StorageRecord::Authority(authority.clone()), + ) + .await; + let creator = test.creator(); + let result = creator.create(&test.request).await.unwrap(); + assert_eq!(creator.create(&test.request).await.unwrap(), result); + let selected = TableRepository::new(test.fixture.store.clone()) + .select(test.fixture.context, test.parent.namespace, "events") + .await + .unwrap() + .unwrap(); + let path = selected.head.storage_location().storage_path().unwrap(); + assert_eq!(path.storage_namespace, test.parent.namespace); + assert_eq!(path.catalog, test.fixture.context.catalog); + assert!(selected + .head + .metadata_location + .to_string() + .starts_with(&format!("s3://iceberg/{path}"))); + let keys = test.fixture.store.values.load_full(); + let native: Vec<_> = keys + .keys() + .filter_map(|key| match StorageKey::decode(key) { + Ok(StorageKey::NativeTable(key)) => Some(key), + _ => None, + }) + .collect(); + assert_eq!( + native + .iter() + .filter(|key| key.scope == TableStorageScope::Head) + .count(), + 1 + ); + assert_eq!( + native + .iter() + .filter(|key| key.scope == TableStorageScope::File) + .count(), + 1 + ); + assert!(native.iter().all(|key| key.table == path)); + assert_eq!( + authority.renamed("renamed".into()).unwrap().storage_layout, + CatalogStorageLayout::Native + ); + let mut default = test.request.clone(); + default.namespace = + crowdb_access_iceberg::namespace::NamespaceIdentifier::new(vec!["default".into()]).unwrap(); + default.identity.operation = OperationId::random(); + let created = creator.create(&default).await.unwrap(); + assert_eq!(creator.create(&default).await.unwrap(), created); + let namespace = NamespaceRepository::new(test.fixture.store.clone()) + .load(test.fixture.context, &default.namespace) + .await + .unwrap() + .unwrap(); + let selected = TableRepository::new(test.fixture.store.clone()) + .select(test.fixture.context, namespace.namespace, "events") + .await + .unwrap() + .unwrap(); + assert_eq!( + selected.head.storage_location().storage_namespace, + Some(namespace.namespace) + ); + assert_ne!(namespace.namespace, test.parent.namespace); +} + #[tokio::test] async fn response_reserve_rejects_before_durable_creation() { let test = TestCreation::new().await; diff --git a/lib/crowdb-access-iceberg/tests/table_repository_test.rs b/lib/crowdb-access-iceberg/tests/table_repository_test.rs index 53cc66910..521e1f8da 100644 --- a/lib/crowdb-access-iceberg/tests/table_repository_test.rs +++ b/lib/crowdb-access-iceberg/tests/table_repository_test.rs @@ -36,6 +36,7 @@ fn head(record: &FileRecord) -> TableHead { fn mapping(head: &TableHead) -> TableMapping { TableMapping { + storage_namespace: None, catalog: head.catalog, namespace: head.namespace, name: head.name.clone(), @@ -75,6 +76,101 @@ async fn new_fixture() -> fixture::TestFile { fixture } +#[tokio::test] +async fn name_mappings_preserve_storage_namespace_across_live_namespace_moves() { + let fixture = new_fixture().await; + let mut file = fixture.record("metadata/one.json", b"{}"); + let mut location = file.location.table(); + let storage_namespace = NamespaceId::random(); + location.storage_namespace = Some(storage_namespace); + file.location = location.file("metadata/one.json").unwrap(); + let mut head = head(&file); + let mut mapping = mapping(&head); + mapping.storage_namespace = Some(storage_namespace); + assert!(mapping.resolves(&head)); + FileRepository::new(fixture.store.clone()) + .publish(fixture.context, &file) + .await + .unwrap(); + let canonical = crowdb_access_iceberg::table::storage_head_key(head.storage_location()); + let encoded = canonical.encode().unwrap(); + let initial = StorageRecord::TableHead(Box::new(head.clone())).encode().unwrap(); + fixture + .store + .compare_exchange( + &encoded, + None, + &initial, + mutation_identity(&encoded, None, &initial), + ) + .await + .unwrap(); + put( + &fixture.store, + name_key(head.catalog, head.namespace, &head.name).unwrap(), + StorageRecord::TableMapping(mapping.clone()), + ) + .await; + let repository = TableRepository::new(fixture.store.clone()); + assert_eq!( + repository + .select(fixture.context, head.namespace, &head.name) + .await + .unwrap() + .unwrap() + .head, + head + ); + let original = head.metadata_location.clone(); + head.namespace = NamespaceId::random(); + head.name = "moved".into(); + head.name_epoch += 1; + mapping.namespace = head.namespace; + mapping.name.clone_from(&head.name); + mapping.name_epoch = head.name_epoch; + assert!(mapping.resolves(&head)); + assert_eq!(head.metadata_location, original); + assert_eq!( + crowdb_access_iceberg::table::storage_head_key(head.storage_location()), + canonical + ); + let moved = StorageRecord::TableHead(Box::new(head.clone())).encode().unwrap(); + fixture + .store + .compare_exchange( + &encoded, + Some(&initial), + &moved, + mutation_identity(&encoded, Some(&initial), &moved), + ) + .await + .unwrap(); + put( + &fixture.store, + name_key(head.catalog, head.namespace, &head.name).unwrap(), + StorageRecord::TableMapping(mapping.clone()), + ) + .await; + let selected = repository + .select(fixture.context, head.namespace, &head.name) + .await + .unwrap() + .unwrap(); + repository + .ensure_current(fixture.context, &selected) + .await + .unwrap(); + assert_eq!(selected.metadata.location, original); + let key = name_key(head.catalog, head.namespace, &head.name).unwrap(); + let record = StorageRecord::TableMapping(mapping.clone()); + assert_eq!( + StorageRecord::decode(&key, &record.encode().unwrap()).unwrap(), + record + ); + mapping.storage_namespace = Some(NamespaceId::random()); + assert!(!mapping.resolves(&head)); +} + #[tokio::test] async fn selection_pins_one_head_and_immutable_metadata_generation() { let fixture = new_fixture().await; diff --git a/lib/crowdb-access-iceberg/tests/wire_credentials_test.rs b/lib/crowdb-access-iceberg/tests/wire_credentials_test.rs index 7a1f04bb6..e01346403 100644 --- a/lib/crowdb-access-iceberg/tests/wire_credentials_test.rs +++ b/lib/crowdb-access-iceberg/tests/wire_credentials_test.rs @@ -33,6 +33,7 @@ fn authority(context: CatalogContext) -> CatalogAuthority { fn target(table: TableId) -> FileDelegationTarget { FileDelegationTarget { + storage_namespace: None, table, format_version: 2, staged: false, @@ -46,6 +47,7 @@ fn only_independent_writer_receives_file_mutations() { let context = context(); let table = TableId::random(); let location = TableLocation { + storage_namespace: None, catalog: context.catalog, table, } @@ -99,6 +101,7 @@ fn only_independent_writer_receives_file_mutations() { Err(FileGrantError::Bounds) ); let other = TableLocation { + storage_namespace: None, catalog: context.catalog, table: TableId::random(), } @@ -136,6 +139,7 @@ fn delegated_file_operations_follow_selected_version_and_draft_stage() { context, &authority, FileDelegationTarget { + storage_namespace: None, table, format_version: 2, staged: true, @@ -151,6 +155,7 @@ fn delegated_file_operations_follow_selected_version_and_draft_stage() { context, &authority, FileDelegationTarget { + storage_namespace: None, table, format_version: 3, staged: false, @@ -201,6 +206,7 @@ fn refresh_rotates_credentials_and_serializes_official_sdk_properties() { assert_eq!( credentials[0]["prefix"], TableLocation { + storage_namespace: None, catalog: context.catalog, table } diff --git a/lib/crowdb-protocol/src/fbs/iceberg.fbs b/lib/crowdb-protocol/src/fbs/iceberg.fbs index 9fbe50fab..f5a4b2dc9 100644 --- a/lib/crowdb-protocol/src/fbs/iceberg.fbs +++ b/lib/crowdb-protocol/src/fbs/iceberg.fbs @@ -35,6 +35,8 @@ table FBCatalogAuthority { root_lease_ms:ulong; delegated_access_ms:ulong; clock_skew_ms:ulong; + storage_layout:ubyte = 1; + locator_key:[ubyte]; } table FBManagementOperation { @@ -189,6 +191,7 @@ table FBFileIdentity { catalog:[ubyte] (required); table_id:[ubyte] (required); file_id:[ubyte] (required); + storage_namespace:[ubyte]; } table FBFileTree { @@ -312,6 +315,7 @@ table FBTableMapping { name_epoch:ulong; operation:[ubyte] (required); state:ubyte = 255; + storage_namespace:[ubyte]; } table FBTableCommitOperation { @@ -450,6 +454,7 @@ table FBFileWriteIntent { created_ms:ulong; not_before_ms:ulong; deleting:bool; + storage_namespace:[ubyte]; } table FBGcEntry { @@ -477,7 +482,16 @@ table FBGcNode { complete:bool; } -union FBRecordValue { FBActiveCatalog, FBCatalogAuthority, FBManagementOperation, FBRetryRecord, FBNamespaceAuthority, FBNamespaceMapping, FBPayloadPage, FBRetryResult, FBNamespaceOperation, FBFileRecord, FBFileMapping, FBMultipartSession, FBMultipartPart, FBMultipartAdmission, FBTableHead, FBTableMapping, FBTableCommitOperation, FBTableCreateOperation, FBTableLifecycleOperation, FBTablePurgeTask, FBGcTask, FBGcCandidate, FBGcPage, FBGcNode, FBFileWriteIntent, FBDeletedFile } +table FBNativeInjection { + operation:[ubyte] (required); + issued_ms:ulong; + activation_epoch:ulong; + principal:[ubyte] (required); + stem:string (required); + file:FBFileRecord (required); +} + +union FBRecordValue { FBActiveCatalog, FBCatalogAuthority, FBManagementOperation, FBRetryRecord, FBNamespaceAuthority, FBNamespaceMapping, FBPayloadPage, FBRetryResult, FBNamespaceOperation, FBFileRecord, FBFileMapping, FBMultipartSession, FBMultipartPart, FBMultipartAdmission, FBTableHead, FBTableMapping, FBTableCommitOperation, FBTableCreateOperation, FBTableLifecycleOperation, FBTablePurgeTask, FBGcTask, FBGcCandidate, FBGcPage, FBGcNode, FBFileWriteIntent, FBDeletedFile, FBNativeInjection } table FBIcebergRecord { schema_version:ushort; diff --git a/tools/load-tpch.py b/tools/load-tpch.py index 90e7f251e..90d49d213 100644 --- a/tools/load-tpch.py +++ b/tools/load-tpch.py @@ -5,6 +5,8 @@ Run: pixi run -e iceberg-e2e python tools/load-tpch.py Local files and both upload reports remain under .crowdb-runtime/artifacts/. +The S3 upload is an independent benchmark copy. Its bucket and keys do not +describe or back the native Iceberg table's file locations. """ import argparse @@ -144,7 +146,7 @@ def copy_objects(args, report_path, work, properties, client): except ClientError as error: if error.response["Error"]["Code"] != "BucketAlreadyOwnedByYou": raise - copied = {"iceberg_report": str(report_path), "bucket": bucket, "status": "running", "objects": []} + copied = {"iceberg_report": str(report_path), "purpose": "independent-s3-benchmark-copy", "bucket": bucket, "status": "running", "objects": []} output = work / "s3-report.json" save_report(output, copied) for table in tables.values():