From d0a9409d49a6588633a7bd44fcaf7b1b37a6f331 Mon Sep 17 00:00:00 2001 From: Casey Rodarmor Date: Thu, 24 Sep 2026 19:40:07 -0700 Subject: [PATCH] Save expected server state on package upload --- src/error.rs | 16 ++++ src/functions.rs | 4 + src/lib.rs | 8 +- src/loader.rs | 4 + src/relative_path.rs | 8 +- src/server_state.rs | 8 ++ src/state.rs | 95 +++++++++++++++++++++ src/subcommand/create.rs | 6 +- src/subcommand/upload.rs | 71 ++++++++++++++-- src/subcommand/verify.rs | 2 +- tests/lib.rs | 3 +- tests/lint.rs | 7 +- tests/upload.rs | 173 ++++++++++++++++++++++++++++++++++++++- 13 files changed, 384 insertions(+), 21 deletions(-) create mode 100644 src/server_state.rs create mode 100644 src/state.rs diff --git a/src/error.rs b/src/error.rs index 22836317..2213dd2d 100644 --- a/src/error.rs +++ b/src/error.rs @@ -128,6 +128,12 @@ pub enum Error { path: DisplayPath, source: serde_yaml::Error, }, + #[snafu(display("failed to deserialize state at `{path}`"))] + DeserializeState { + backtrace: Option, + path: DisplayPath, + source: serde_json::Error, + }, #[snafu(display("directory `{hash}` totals error"))] DirectoryTotals { backtrace: Option, @@ -476,6 +482,16 @@ pub enum Error { status: StatusCode, url: Url, }, + #[snafu(display( + "package number {number} is at revision {server} on the server, but the last uploaded revision \ + is {local}" + ))] + RevisionStale { + backtrace: Option, + local: Revision, + number: u64, + server: Revision, + }, #[snafu(display("failed to install rustls ring crypto provider"))] RustlsProvider { backtrace: Option }, #[snafu(display("failed to serialize JSON to stdout"))] diff --git a/src/functions.rs b/src/functions.rs index ee8f9e08..7eb42f0b 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -43,6 +43,10 @@ pub fn gradient_alpha(width: u32, height: u32, alpha: u8) -> DynamicImage { .into() } +pub(crate) fn ignore(path: &RelativePath, ignore: &[RelativePath]) -> bool { + path.starts_with(State::DIR) || ignore.iter().any(|ignore| path.starts_with(ignore)) +} + pub fn install_default_crypto_provider() -> Result { static INSTALLED: LazyLock = LazyLock::new(|| { rustls::crypto::ring::default_provider() diff --git a/src/lib.rs b/src/lib.rs index 206b842a..cd0957f6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -80,7 +80,7 @@ use { float_ext::FloatExt, format::Format, functions::{ - current_dir, decode_path, default, format_size, now, parse_number, transfer_tempfile, + current_dir, decode_path, default, format_size, ignore, now, parse_number, transfer_tempfile, }, hashing_writer::HashingWriter, head::Head, @@ -165,7 +165,6 @@ use { url_error::UrlError, utf8_path_ext::Utf8PathExt, validate::Validate, - version::Version, video::Video, video_error::VideoError, video_metadata::VideoMetadata, @@ -280,7 +279,10 @@ pub use self::{ public_key::PublicKey, revision::Revision, revision_object::RevisionObject, + server_state::ServerState, + state::State, totals::Totals, + version::Version, }; #[cfg(test)] @@ -441,6 +443,7 @@ mod revision_object; mod rotation; mod server; mod server_error; +mod server_state; mod server_url; mod sign_options; mod signature; @@ -448,6 +451,7 @@ mod signature_error; mod sort; mod sort_key; mod sorted_set; +mod state; mod statement; mod static_asset; mod style; diff --git a/src/loader.rs b/src/loader.rs index 947b745c..52734166 100644 --- a/src/loader.rs +++ b/src/loader.rs @@ -57,6 +57,10 @@ impl Loader { &self.path } + pub(crate) fn root(&self) -> &Utf8Path { + self.path.parent().unwrap() + } + pub(crate) fn unpack(&self) -> Result { self .archive diff --git a/src/relative_path.rs b/src/relative_path.rs index 10e5f891..7139fdc5 100644 --- a/src/relative_path.rs +++ b/src/relative_path.rs @@ -16,13 +16,13 @@ use super::*; pub struct RelativePath(String); impl RelativePath { - const JUNK_NAMES: [&'static str; 2] = [".DS_Store", ".localized"]; + const JUNK_NAMES: &'static [&'static str] = &[".DS_Store", ".filepack", ".localized"]; const MAX_LENGTH: usize = 4096; - const WINDOWS_RESERVED_CHARACTERS: [char; 7] = ['"', '*', ':', '<', '>', '?', '|']; + const WINDOWS_RESERVED_CHARACTERS: &'static [char] = &['"', '*', ':', '<', '>', '?', '|']; - const WINDOWS_RESERVED_NAMES: [&'static str; 28] = [ + const WINDOWS_RESERVED_NAMES: &'static [&'static str] = &[ "AUX", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", "COM¹", "COM²", "COM³", "CON", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9", "LPT¹", "LPT²", "LPT³", "NUL", "PRN", @@ -62,7 +62,7 @@ impl RelativePath { if lints.contains(&Lint::WindowsReservedFilename) { let uppercase = component.to_uppercase(); - for name in Self::WINDOWS_RESERVED_NAMES { + for &name in Self::WINDOWS_RESERVED_NAMES { if uppercase == name { return Some(LintError::WindowsReservedFilename { name: component.into(), diff --git a/src/server_state.rs b/src/server_state.rs new file mode 100644 index 00000000..ce8a88df --- /dev/null +++ b/src/server_state.rs @@ -0,0 +1,8 @@ +use super::*; + +#[derive(Debug, Deserialize, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ServerState { + pub number: u64, + pub revision: Revision, +} diff --git a/src/state.rs b/src/state.rs new file mode 100644 index 00000000..31717311 --- /dev/null +++ b/src/state.rs @@ -0,0 +1,95 @@ +use super::*; + +#[derive(Debug, Default, Deserialize, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct State { + pub servers: BTreeMap, +} + +impl State { + pub(crate) const DIR: &'static str = ".filepack"; + const FILENAME: &'static str = "state.json"; + + pub fn load(root: &Utf8Path) -> Result { + let path = Self::path(root); + + let Some(json) = filesystem::read_to_string_opt(&path)? else { + return Ok(Self::default()); + }; + + serde_json::from_str(&json).context(error::DeserializeState { path }) + } + + fn path(root: &Utf8Path) -> Utf8PathBuf { + root.join(Self::DIR).join(Self::FILENAME) + } + + pub(crate) fn save(&self, root: &Utf8Path) -> Result { + let dir = root.join(Self::DIR); + + filesystem::create_dir_all(&dir)?; + + let path = Self::path(root); + + let mut json = serde_json::to_string_pretty(self).unwrap(); + + json.push('\n'); + + let mut tempfile = tempfile::Builder::new() + .tempfile_in(&dir) + .context(error::FilesystemIo { path: &dir })?; + + tempfile + .write_all(json.as_bytes()) + .context(error::FilesystemIo { path: &dir })?; + + tempfile + .persist(&path) + .map_err(|error| error.error) + .context(error::FilesystemIo { path: &path })?; + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn round_trip() { + let (_tempdir, root) = tempdir(); + + assert_eq!(State::load(&root).unwrap(), State::default()); + + let state = State { + servers: BTreeMap::from([( + "http://example.com/".into(), + ServerState { + number: 1, + revision: test::REVISION.parse().unwrap(), + }, + )]), + }; + + state.save(&root).unwrap(); + + assert_eq!( + fs::read_to_string(root.join(".filepack/state.json")).unwrap(), + format!( + r#"{{ + "servers": {{ + "http://example.com/": {{ + "number": 1, + "revision": "{}" + }} + }} +}} +"#, + test::REVISION, + ), + ); + + assert_eq!(State::load(&root).unwrap(), state); + } +} diff --git a/src/subcommand/create.rs b/src/subcommand/create.rs index 86f8e7b0..95faaa3c 100644 --- a/src/subcommand/create.rs +++ b/src/subcommand/create.rs @@ -126,11 +126,7 @@ impl Create { let relative = RelativePath::try_from(relative).context(error::Path { path: relative })?; - if self - .ignore - .iter() - .any(|ignore| relative.starts_with(ignore)) - { + if ignore(&relative, &self.ignore) { continue; } diff --git a/src/subcommand/upload.rs b/src/subcommand/upload.rs index 08c60de8..bbfe8666 100644 --- a/src/subcommand/upload.rs +++ b/src/subcommand/upload.rs @@ -111,17 +111,58 @@ impl Upload { fn upload_package(&self, options: Options, client: Client) -> Result { let loader = Loader::load(self.input.as_deref())?; + let root = loader.root().to_owned(); + + let mut state = State::load(&root)?; + + let server = self.server.to_string(); + let package = loader.package()?; let fingerprint = Fingerprint(package.hash()); - let previous = if let Some(number) = self.update { + let update = match (self.replace, self.update) { + (Some(_), Some(_)) => unreachable!(), + (Some(_), None) => None, + (None, Some(number)) => Some(number), + (None, None) => state + .servers + .get(&server) + .map(|server_state| server_state.number), + }; + + let previous = if let Some(number) = update { let head = client.number(number)?; + if let Some(server_state) = state.servers.get(&server) + && server_state.number == number + && server_state.revision != head.revision + { + return Err( + error::RevisionStale { + local: server_state.revision, + number, + server: head.revision, + } + .build(), + ); + } + if head.package == fingerprint { + state.servers.insert( + server, + ServerState { + number, + revision: head.revision, + }, + ); + + state.save(&root)?; + if !options.quiet { eprintln!("package number {number} is up to date"); } + return Ok(()); } @@ -138,10 +179,24 @@ impl Upload { let revision = revision_object.hash(); - if self.replace.is_none() && self.update.is_none() && client.is_head(revision)? { + if self.replace.is_none() && update.is_none() && client.is_head(revision)? { + let number = client.verify_revision( + revision, + api::revision::Request { + mode: api::revision::Mode::New, + }, + )?; + + state + .servers + .insert(server, ServerState { number, revision }); + + state.save(&root)?; + if !options.quiet { - eprintln!("server already has package"); + eprintln!("server already has package number {number}"); } + return Ok(()); } @@ -181,8 +236,6 @@ impl Upload { progress_bar: ProgressBar::items(&options, bytes, files, "files"), }; - let root = context.loader.path().parent().unwrap().to_owned(); - Self::upload_directory(&mut context, &root, package.hash(), package.size())?; context.client.verify_package(fingerprint)?; @@ -191,7 +244,7 @@ impl Upload { .client .put_file(revision.into(), revision_object.encode_to_vec().into())?; - let (mode, verb) = match (self.replace, self.update) { + let (mode, verb) = match (self.replace, update) { (Some(number), None) => (api::revision::Mode::Replace { number }, "replaced"), (None, Some(number)) => (api::revision::Mode::Update { number }, "updated"), (None, None) => (api::revision::Mode::New, "created"), @@ -202,6 +255,12 @@ impl Upload { .client .verify_revision(revision, api::revision::Request { mode })?; + state + .servers + .insert(server, ServerState { number, revision }); + + state.save(&root)?; + if !options.quiet { eprintln!("{verb} package number {number}"); } diff --git a/src/subcommand/verify.rs b/src/subcommand/verify.rs index 44bdae5b..c43bef00 100644 --- a/src/subcommand/verify.rs +++ b/src/subcommand/verify.rs @@ -167,7 +167,7 @@ fingerprint mismatch: `{}` let path = RelativePath::try_from(path).context(error::Path { path })?; - if self.ignore.iter().any(|ignore| path.starts_with(ignore)) { + if ignore(&path, &self.ignore) { continue; } diff --git a/tests/lib.rs b/tests/lib.rs index 8ddb5a73..1d4411f7 100644 --- a/tests/lib.rs +++ b/tests/lib.rs @@ -4,7 +4,8 @@ use { filepack::{ Decode, Decoder, Directory, DirectoryExt, Encode, Encoder, Entry, Fingerprint, FlacBuilder, Hash, Loader, MagicType, Manifest, Metadata, Mp3Builder, Mp4Builder, PackageIdentifier, Page, - PngBuilder, PrivateKey, PublicKey, Totals, gradient, gradient_alpha, hex, + PngBuilder, PrivateKey, PublicKey, RevisionObject, ServerState, State, Totals, Version, + gradient, gradient_alpha, hex, templates::{DirectoryHtml, PackageHtml}, }, image::{DynamicImage, ImageFormat}, diff --git a/tests/lint.rs b/tests/lint.rs index 28d6252b..399658db 100644 --- a/tests/lint.rs +++ b/tests/lint.rs @@ -335,12 +335,17 @@ fn deny_junk_ignores_compatibility() { Test::new() .touch("aux") .touch(".DS_Store") + .touch("foo/.filepack/bar") .args(["create", "--deny", "junk"]) .stderr( " error: path failed lint: `.DS_Store` └─ possible junk file - error: 1 lint error + error: path failed lint: `foo/.filepack` + └─ possible junk file + error: path failed lint: `foo/.filepack/bar` + └─ possible junk file + error: 3 lint errors ", ) .failure(); diff --git a/tests/upload.rs b/tests/upload.rs index f9e71c83..1aef617d 100644 --- a/tests/upload.rs +++ b/tests/upload.rs @@ -64,7 +64,7 @@ fn reupload_package_succeeds() { ) .success() .args(["upload", "--server", &server.address(), "manifest.filepack"]) - .stderr("server already has package\n") + .stderr("package number 1 is up to date\n") .success(); server.terminate().success(); @@ -596,6 +596,114 @@ fn upload_package_uploads_files() { server.terminate().success(); } +#[test] +fn upload_records_state() { + let server = Test::new().serve().spawn(); + + let test = Test::new() + .write("foo", "bar") + .args(["create", "."]) + .success(); + + let revision = RevisionObject { + version: Version::Zero, + package: fingerprint(&test.path().join("manifest.filepack")), + previous: None, + } + .hash(); + + let test = test + .args(["upload", "--server", &server.address(), "manifest.filepack"]) + .stderr( + " + uploading 1 of 1 file + created package number 1 + ", + ) + .success() + .args(["verify", "."]) + .stderr("successfully verified 1 file totaling 3 bytes\n") + .success(); + + assert_eq!( + State::load(&test.path()).unwrap(), + State { + servers: BTreeMap::from([( + format!("{}/", server.address()), + ServerState { + number: 1, + revision, + }, + )]), + }, + ); + + server.terminate().success(); +} + +#[test] +fn upload_rejects_stale_state() { + let server = Test::new().serve().spawn(); + + let a = Test::new() + .write("foo", "bar") + .args(["create", "."]) + .success() + .args(["upload", "--server", &server.address(), "manifest.filepack"]) + .stderr( + " + uploading 1 of 1 file + created package number 1 + ", + ) + .success(); + + let root = RevisionObject { + version: Version::Zero, + package: fingerprint(&a.path().join("manifest.filepack")), + previous: None, + } + .hash(); + + let b = Test::new() + .write("foo", "bar") + .args(["create", "."]) + .success() + .args(["upload", "--server", &server.address(), "manifest.filepack"]) + .stderr("server already has package number 1\n") + .success() + .write("foo", "baz") + .args(["create", "--force", "."]) + .success() + .args(["upload", "--server", &server.address(), "manifest.filepack"]) + .stderr( + " + uploading 1 of 1 file + updated package number 1 + ", + ) + .success(); + + let head = RevisionObject { + version: Version::Zero, + package: fingerprint(&b.path().join("manifest.filepack")), + previous: Some(root), + } + .hash(); + + a.write("foo", "qux") + .args(["create", "--force", "."]) + .success() + .args(["upload", "--server", &server.address(), "manifest.filepack"]) + .stderr(&format!( + "error: package number 1 is at revision {head} on the server, but the last uploaded \ + revision is {root}\n" + )) + .failure(); + + server.terminate().success(); +} + #[test] fn upload_replaces_package() { let server = Test::new().serve().spawn(); @@ -670,6 +778,69 @@ fn upload_replaces_package() { server.terminate().success(); } +#[test] +fn upload_updates_implicitly() { + let server = Test::new().serve().spawn(); + + let test = Test::new() + .write("foo", "bar") + .args(["create", "."]) + .success(); + + let root = RevisionObject { + version: Version::Zero, + package: fingerprint(&test.path().join("manifest.filepack")), + previous: None, + } + .hash(); + + let test = test + .args(["upload", "--server", &server.address(), "manifest.filepack"]) + .stderr( + " + uploading 1 of 1 file + created package number 1 + ", + ) + .success() + .write("foo", "baz") + .args(["create", "--force", "."]) + .success() + .args(["upload", "--server", &server.address(), "manifest.filepack"]) + .stderr( + " + uploading 1 of 1 file + updated package number 1 + ", + ) + .success() + .args(["upload", "--server", &server.address(), "manifest.filepack"]) + .stderr("package number 1 is up to date\n") + .success(); + + let head = RevisionObject { + version: Version::Zero, + package: fingerprint(&test.path().join("manifest.filepack")), + previous: Some(root), + } + .hash(); + + assert_eq!( + State::load(&test.path()).unwrap(), + State { + servers: BTreeMap::from([( + format!("{}/", server.address()), + ServerState { + number: 1, + revision: head, + }, + )]), + }, + ); + + server.terminate().success(); +} + #[test] fn upload_updates_package() { let server = Test::new().serve().spawn();