Merge pull request #22 from technocreatives/feature/proper-patches

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "artefacta"
-version = "0.0.7"
+version = "0.0.8"
 authors = ["Pascal Hertleif <[email protected]>"]
 edition = "2018"
 license = "MIT OR Apache-2.0"

src/apply_patch.rs

@@ -34,10 +34,10 @@ mod tests {
         let dir = tempdir()?;
 
         let file1 = dir.path().join("1.tar.zst");
-        let content1 = random_file(&file1)?;
+        let content1 = random_zstd_file(&file1)?;
 
         let file2 = dir.path().join("2.tar.zst");
-        let content2 = random_file(&file2)?;
+        let content2 = random_zstd_file(&file2)?;
 
         let patch_1_2 = dir.path().join("1-2.patch.zst");
 

src/compression.rs

@@ -1,15 +1,22 @@
 use erreur::{Context, Result};
-use std::{env, io::Write};
-use zstd::stream::write::Encoder as ZstdEncoder;
+use std::{
+    env,
+    io::{Read, Write},
+};
+use zstd::stream::{decode_all, write::Encoder as ZstdEncoder};
 
 pub fn compress<W: Write>(w: W) -> Result<ZstdEncoder<W>> {
     ZstdEncoder::new(w, compression_level()).context("Can't instantiate ZSTD encoder")
 }
 
+pub fn decompress<R: Read>(r: R) -> Result<Vec<u8>> {
+    decode_all(r).context("Can't read zstd compressed file")
+}
+
 const LEVEL_VAR: &str = "ARTEFACTA_COMPRESSION_LEVEL";
 
 #[cfg(test)]
-const DEFAULT_LEVEL: i32 = 10;
+const DEFAULT_LEVEL: i32 = 14;
 
 #[cfg(not(test))]
 const DEFAULT_LEVEL: i32 = 1;

src/git.rs

@@ -29,6 +29,13 @@ pub fn get_tags(repo: &git2::Repository) -> Result<Vec<Tag>> {
         .collect()
 }
 
+pub fn tag_to_slice(tag: &str) -> Vec<SmolStr> {
+    tag.to_lowercase()
+        .split(|c| c == '.' || c == '-')
+        .map(SmolStr::from)
+        .collect()
+}
+
 /// assume versions are in format `….c.b.a` (or `…-c-b-a`)
 pub fn find_tags_to_patch(current: &str, tags: &[String]) -> Result<Vec<String>> {
     fn dec(x: &SmolStr) -> Option<SmolStr> {
@@ -37,12 +44,6 @@ pub fn find_tags_to_patch(current: &str, tags: &[String]) -> Result<Vec<String>>
         Some(SmolStr::from(prev.to_string()))
     }
 
-    fn tag_to_slice(tag: &str) -> Vec<SmolStr> {
-        tag.split(|c| c == '.' || c == '-')
-            .map(SmolStr::from)
-            .collect()
-    }
-
     let parsed_tags = tags.iter().map(|tag| tag_to_slice(tag)).collect::<Vec<_>>();
     let current = tag_to_slice(current);
 
@@ -156,3 +157,20 @@ fn tags_to_patch_from_4() {
         vec!["IL40.2.18".to_string(), "IL40.1.0".to_string()]
     );
 }
+
+#[test]
+fn tags_to_patch_from_fuzzy() {
+    let tags = vec![
+        "IL40.0.1".to_string(),
+        "IL40.1.0".to_string(),
+        "IL40.2.17".to_string(),
+        "IL40.2.18".to_string(),
+        "IL40.x.0".to_string(),
+    ];
+    let current_tag = "il40-2-19";
+    let patch_these = find_tags_to_patch(current_tag, &tags).unwrap();
+    assert_eq!(
+        patch_these,
+        vec!["IL40.2.18".to_string(), "IL40.1.0".to_string()]
+    );
+}

src/index.rs

@@ -6,7 +6,7 @@ use erreur::{bail, ensure, Context, Help, Result};
 use std::{
     convert::TryFrom,
     fs::{self, File},
-    io::{self, BufReader, Read},
+    io::{self, BufReader, Cursor, Read},
     path::Path,
 };
 
@@ -79,6 +79,11 @@ impl Index {
             Ok(bytes)
         }
 
+        fn file_size(size: u64) -> String {
+            use humansize::{file_size_opts as options, FileSize};
+            size.file_size(options::BINARY).expect("never negative")
+        }
+
         if self.get_patch(from.clone(), to.clone()).await.is_ok() {
             log::warn!(
                 "asked to calculate patch from `{:?}` to `{:?}` but it's already present",
@@ -100,25 +105,35 @@ impl Index {
             .await
             .context("get old build")?;
         let old_build = read_file(old_build).context("read old build")?;
+        let old_build = crate::decompress(Cursor::new(old_build))?;
+
         let new_build = self.get_build(to.clone()).await.context("get new build")?;
+        let new_build_size = new_build.size;
         let new_build = read_file(new_build).context("read new build")?;
+        let new_build = crate::decompress(Cursor::new(new_build))?;
 
         let path_name = Patch::new(from.clone(), to.clone());
         // TODO: Fix that arbitrary "+ zst" here and everywhere else
         let patch_path = local.join(path_name.to_string() + ".zst");
-        log::info!("write patch {:?} to `{:?}`", path_name, patch_path);
-
-        let mut patch = crate::compress(File::create(&patch_path)?)?;
-        bidiff::simple_diff_with_params(
-            &old_build,
-            &new_build,
-            &mut patch,
-            &bidiff::DiffParams {
-                sort_partitions: 4,
-                scan_chunk_size: Some(10_000_000),
-            },
-        )?;
-        patch.finish()?;
+        log::debug!("write patch {:?} to `{:?}`", path_name, patch_path);
+
+        let mut patch =
+            crate::compress(File::create(&patch_path).context("creating file to write patch to")?)?;
+        bidiff::simple_diff_with_params(&old_build, &new_build, &mut patch, &{
+            const MB: u64 = 1_000_000;
+            bidiff::DiffParams {
+                sort_partitions: {
+                    if new_build_size > (100 * MB) {
+                        4
+                    } else {
+                        1
+                    }
+                },
+                scan_chunk_size: Some(100 * MB as usize),
+            }
+        })
+        .context("calculating binary diff between builds")?;
+        patch.finish().context("finishing zstd file")?;
 
         let patch_size = patch_path
             .metadata()
@@ -136,6 +151,15 @@ impl Index {
             size: patch_size,
         };
 
+        log::info!(
+            "Calculated new patch from {} to {} of size {} -- that's {:.1}% of the new build's {}",
+            from,
+            to,
+            file_size(patch_size),
+            (patch_size as f64) / (new_build_size as f64) * 100_f64,
+            file_size(new_build_size),
+        );
+
         self.patch_graph
             .add_patch(&from, &to, entry, Location::Local)?;
 
@@ -326,6 +350,16 @@ impl Index {
             .context("fetch newly added local build")
     }
 
+    pub fn get_build_for_tag(&self, tag: &str) -> Result<Version> {
+        let parsed_tag = crate::git::tag_to_slice(tag);
+        self.patch_graph
+            .builds
+            .keys()
+            .find(|build| crate::git::tag_to_slice(build.as_str()) == parsed_tag)
+            .cloned()
+            .with_context(|| format!("no build found matching tag `{}`", tag))
+    }
+
     pub async fn add_local_build(&mut self, path: impl AsRef<Path>) -> Result<Entry> {
         let entry = Entry::from_path(path.as_ref(), self.local.clone())
             .context("local build file as entry")?;
@@ -496,9 +530,9 @@ mod tests {
         let remote_dir = tempdir()?;
 
         // Add some builds
-        let _build1 = random_file(local_dir.path().join("build1.tar.zst"))?;
-        let _build2 = random_file(local_dir.path().join("build2.tar.zst"))?;
-        let _build3 = random_file(local_dir.path().join("build3.tar.zst"))?;
+        let _build1 = random_zstd_file(local_dir.path().join("build1.tar.zst"))?;
+        let _build2 = random_zstd_file(local_dir.path().join("build2.tar.zst"))?;
+        let _build3 = random_zstd_file(local_dir.path().join("build3.tar.zst"))?;
 
         let mut index = Index::new(local_dir.path(), remote_dir.path().try_into()?).await?;
 

src/index/graph.rs

@@ -14,7 +14,7 @@ use std::{collections::HashMap, convert::TryFrom, fs::ReadDir, io::Error as IoEr
 pub struct PatchGraph {
     graph: Graph<Build, Patch>,
     /// helper for looking up nodes in the graph
-    builds: HashMap<Version, NodeIndex<DefaultIx>>,
+    pub(crate) builds: HashMap<Version, NodeIndex<DefaultIx>>,
     /// helper for looking up edges in the graph
     patches: HashMap<(Version, Version), EdgeIndex<DefaultIx>>,
 }

src/lib.rs

@@ -13,7 +13,9 @@ mod storage;
 pub use storage::Storage;
 
 mod compression;
-pub use compression::compress;
+pub use compression::{compress, decompress};
+
+pub mod git;
 
 #[cfg(test)]
 pub(crate) mod test_helpers;