From d58e764c3f956ffee9d0963aab9dff020037fa58 Mon Sep 17 00:00:00 2001 From: Aaron Franke Date: Sat, 7 Oct 2023 17:10:46 -0500 Subject: [PATCH] Allow caching a history of optimized files to avoid reoptimizing them --- Cargo.lock | 111 +++++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 1 + src/cache.rs | 59 +++++++++++++++++++++++++++ src/lib.rs | 19 +++++++++ src/main.rs | 9 +++++ 5 files changed, 199 insertions(+) create mode 100644 src/cache.rs diff --git a/Cargo.lock b/Cargo.lock index 422f295d..8c1ce4db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -197,6 +197,27 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "dirs" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys", +] + [[package]] name = "either" version = "1.8.1" @@ -278,6 +299,17 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" +[[package]] +name = "getrandom" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "glob" version = "0.3.1" @@ -448,6 +480,12 @@ dependencies = [ "libc", ] +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "oxipng" version = "8.0.0" @@ -455,6 +493,7 @@ dependencies = [ "bitvec", "clap", "crossbeam-channel", + "dirs", "env_logger", "filetime", "glob", @@ -482,6 +521,24 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "proc-macro2" +version = "1.0.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b1106fec09662ec6dd98ccac0f81cef56984d0b49f75c92d8cbad76e20c005c" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +dependencies = [ + "proc-macro2", +] + [[package]] name = "radium" version = "0.7.0" @@ -519,6 +576,17 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_users" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +dependencies = [ + "getrandom", + "redox_syscall", + "thiserror", +] + [[package]] name = "rgb" version = "0.8.36" @@ -581,6 +649,17 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "syn" +version = "2.0.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "tap" version = "1.0.1" @@ -596,18 +675,50 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "thiserror" +version = "1.0.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1177e8c6d7ede7afde3585fd2513e611227efd6481bd78d2e82ba1ce16557ed4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "typed-arena" version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + [[package]] name = "utf8parse" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index 77e8dd79..5550000a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,6 +34,7 @@ libdeflater = "1.19.0" log = "0.4.19" bitvec = "1.0.1" rustc-hash = "1.1.0" +dirs = "5.0" [dependencies.env_logger] optional = true diff --git a/src/cache.rs b/src/cache.rs new file mode 100644 index 00000000..2fffb7e1 --- /dev/null +++ b/src/cache.rs @@ -0,0 +1,59 @@ +use std::hash::Hash; +use std::hash::Hasher; +use std::io::Write; + +extern crate dirs; + +fn hash_data_to_hex_string(data: &[u8]) -> String { + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + data.hash(&mut hasher); + let in_data_hash: u64 = hasher.finish(); + let in_data_has_str: String = format!("{:x}", in_data_hash); + return in_data_has_str; +} + +fn get_oxipng_cache_path() -> std::path::PathBuf { + let mut cache_path: std::path::PathBuf = dirs::cache_dir().unwrap(); + cache_path.push("oxipng"); + return cache_path; +} + +fn read_cache_file_and_check_for_hash(hash_string: &str) -> bool { + let mut cache_path = get_oxipng_cache_path(); + std::fs::create_dir_all(&cache_path).unwrap(); + cache_path.push("optimized_file_hashes"); + let file = match std::fs::File::open(cache_path) { + Ok(file) => file, + Err(_) => return false, + }; + use std::io::BufRead; + let buf_reader = std::io::BufReader::new(file); + for line in buf_reader.lines() { + let line = line.unwrap(); + if line == hash_string { + return true; + } + } + return false; +} + +pub fn check_cache_for_data_hash(data: &[u8]) -> bool { + let in_data_hash_str = hash_data_to_hex_string(data); + return read_cache_file_and_check_for_hash(&in_data_hash_str); +} + +pub fn write_data_hash_to_cache(data: &[u8]) { + let data_hash_str: String = hash_data_to_hex_string(data); + if read_cache_file_and_check_for_hash(&data_hash_str) { + return; + } + let mut cache_path = get_oxipng_cache_path(); + cache_path.push("optimized_file_hashes"); + let file = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(cache_path) + .unwrap(); + let mut buf_writer = std::io::BufWriter::new(file); + writeln!(buf_writer, "{}", data_hash_str).unwrap(); +} diff --git a/src/lib.rs b/src/lib.rs index 2530c310..e6c4d1c6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -51,6 +51,7 @@ pub use indexmap::{indexset, IndexSet}; pub use rgb::{RGB16, RGBA8}; mod atomicmin; +mod cache; mod colors; mod deflate; mod error; @@ -155,6 +156,10 @@ pub struct Options { /// /// Default: `false` pub force: bool, + /// Cache the history of optimized files and avoid reoptimizing optimal files. + /// + /// Default: `false` + pub cache: bool, /// Which RowFilters to try on the file /// /// Default: `None,Sub,Entropy,Bigrams` @@ -301,6 +306,7 @@ impl Default for Options { Options { fix_errors: false, force: false, + cache: false, filter: indexset! {RowFilter::None, RowFilter::Sub, RowFilter::Entropy, RowFilter::Bigrams}, interlace: Some(Interlacing::None), optimize_alpha: false, @@ -449,12 +455,25 @@ pub fn optimize(input: &InFile, output: &OutFile, opts: &Options) -> PngResult<( data } }; + // When cache is enabled, check if the file has already been optimized. + if opts.cache { + let is_data_in_cache: bool = crate::cache::check_cache_for_data_hash(&in_data); + if is_data_in_cache { + info!("{}: File was previously optimized, skipping.", input); + return Ok(()); + } + } let mut png = PngData::from_slice(&in_data, opts)?; // Run the optimizer on the decoded PNG. let mut optimized_output = optimize_png(&mut png, &in_data, opts, deadline)?; + // When cache is enabled, write the hash of the optimized file to the cache. + if opts.cache { + crate::cache::write_data_hash_to_cache(&optimized_output); + } + let in_length = in_data.len(); if is_fully_optimized(in_data.len(), optimized_output.len(), opts) { diff --git a/src/main.rs b/src/main.rs index 94c0cfc6..554429bf 100644 --- a/src/main.rs +++ b/src/main.rs @@ -242,6 +242,13 @@ fn main() { .long("force") .action(ArgAction::SetTrue), ) + .arg( + Arg::new("cache") + .help("Cache the history of optimized files and avoid reoptimizing optimal files") + .short('c') + .long("cache") + .action(ArgAction::SetTrue), + ) .arg( Arg::new("zopfli") .help("Use the slow but stronger Zopfli compressor (recommended use is with all filters and `--fast` enabled)") @@ -493,6 +500,8 @@ fn parse_opts_into_struct( opts.force = matches.get_flag("force"); + opts.cache = matches.get_flag("cache"); + opts.fix_errors = matches.get_flag("fix"); opts.bit_depth_reduction = !matches.get_flag("no-bit-reduction");