Allow caching a history of optimized files to avoid reoptimizing them
This commit is contained in:
Aaron Franke
parent
c16519b38b
commit
d58e764c3f
5 changed files with 199 additions and 0 deletions
111
Cargo.lock
generated
111
Cargo.lock
generated
|
|
@ -197,6 +197,27 @@ dependencies = [
|
|||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dirs"
|
||||
version = "5.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225"
|
||||
dependencies = [
|
||||
"dirs-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dirs-sys"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"option-ext",
|
||||
"redox_users",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "either"
|
||||
version = "1.8.1"
|
||||
|
|
@ -278,6 +299,17 @@ version = "2.0.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c"
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.2.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"wasi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "glob"
|
||||
version = "0.3.1"
|
||||
|
|
@ -448,6 +480,12 @@ dependencies = [
|
|||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "option-ext"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
|
||||
|
||||
[[package]]
|
||||
name = "oxipng"
|
||||
version = "8.0.0"
|
||||
|
|
@ -455,6 +493,7 @@ dependencies = [
|
|||
"bitvec",
|
||||
"clap",
|
||||
"crossbeam-channel",
|
||||
"dirs",
|
||||
"env_logger",
|
||||
"filetime",
|
||||
"glob",
|
||||
|
|
@ -482,6 +521,24 @@ dependencies = [
|
|||
"miniz_oxide",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.68"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5b1106fec09662ec6dd98ccac0f81cef56984d0b49f75c92d8cbad76e20c005c"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.33"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "radium"
|
||||
version = "0.7.0"
|
||||
|
|
@ -519,6 +576,17 @@ dependencies = [
|
|||
"bitflags",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_users"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"
|
||||
dependencies = [
|
||||
"getrandom",
|
||||
"redox_syscall",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rgb"
|
||||
version = "0.8.36"
|
||||
|
|
@ -581,6 +649,17 @@ version = "0.10.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.38"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tap"
|
||||
version = "1.0.1"
|
||||
|
|
@ -596,18 +675,50 @@ dependencies = [
|
|||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "1.0.49"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1177e8c6d7ede7afde3585fd2513e611227efd6481bd78d2e82ba1ce16557ed4"
|
||||
dependencies = [
|
||||
"thiserror-impl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror-impl"
|
||||
version = "1.0.49"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "typed-arena"
|
||||
version = "2.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
|
||||
|
||||
[[package]]
|
||||
name = "utf8parse"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.11.0+wasi-snapshot-preview1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
version = "0.3.9"
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ libdeflater = "1.19.0"
|
|||
log = "0.4.19"
|
||||
bitvec = "1.0.1"
|
||||
rustc-hash = "1.1.0"
|
||||
dirs = "5.0"
|
||||
|
||||
[dependencies.env_logger]
|
||||
optional = true
|
||||
|
|
|
|||
59
src/cache.rs
Normal file
59
src/cache.rs
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
use std::hash::Hash;
|
||||
use std::hash::Hasher;
|
||||
use std::io::Write;
|
||||
|
||||
extern crate dirs;
|
||||
|
||||
fn hash_data_to_hex_string(data: &[u8]) -> String {
|
||||
let mut hasher = std::collections::hash_map::DefaultHasher::new();
|
||||
data.hash(&mut hasher);
|
||||
let in_data_hash: u64 = hasher.finish();
|
||||
let in_data_has_str: String = format!("{:x}", in_data_hash);
|
||||
return in_data_has_str;
|
||||
}
|
||||
|
||||
fn get_oxipng_cache_path() -> std::path::PathBuf {
|
||||
let mut cache_path: std::path::PathBuf = dirs::cache_dir().unwrap();
|
||||
cache_path.push("oxipng");
|
||||
return cache_path;
|
||||
}
|
||||
|
||||
fn read_cache_file_and_check_for_hash(hash_string: &str) -> bool {
|
||||
let mut cache_path = get_oxipng_cache_path();
|
||||
std::fs::create_dir_all(&cache_path).unwrap();
|
||||
cache_path.push("optimized_file_hashes");
|
||||
let file = match std::fs::File::open(cache_path) {
|
||||
Ok(file) => file,
|
||||
Err(_) => return false,
|
||||
};
|
||||
use std::io::BufRead;
|
||||
let buf_reader = std::io::BufReader::new(file);
|
||||
for line in buf_reader.lines() {
|
||||
let line = line.unwrap();
|
||||
if line == hash_string {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
pub fn check_cache_for_data_hash(data: &[u8]) -> bool {
|
||||
let in_data_hash_str = hash_data_to_hex_string(data);
|
||||
return read_cache_file_and_check_for_hash(&in_data_hash_str);
|
||||
}
|
||||
|
||||
pub fn write_data_hash_to_cache(data: &[u8]) {
|
||||
let data_hash_str: String = hash_data_to_hex_string(data);
|
||||
if read_cache_file_and_check_for_hash(&data_hash_str) {
|
||||
return;
|
||||
}
|
||||
let mut cache_path = get_oxipng_cache_path();
|
||||
cache_path.push("optimized_file_hashes");
|
||||
let file = std::fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(cache_path)
|
||||
.unwrap();
|
||||
let mut buf_writer = std::io::BufWriter::new(file);
|
||||
writeln!(buf_writer, "{}", data_hash_str).unwrap();
|
||||
}
|
||||
19
src/lib.rs
19
src/lib.rs
|
|
@ -51,6 +51,7 @@ pub use indexmap::{indexset, IndexSet};
|
|||
pub use rgb::{RGB16, RGBA8};
|
||||
|
||||
mod atomicmin;
|
||||
mod cache;
|
||||
mod colors;
|
||||
mod deflate;
|
||||
mod error;
|
||||
|
|
@ -155,6 +156,10 @@ pub struct Options {
|
|||
///
|
||||
/// Default: `false`
|
||||
pub force: bool,
|
||||
/// Cache the history of optimized files and avoid reoptimizing optimal files.
|
||||
///
|
||||
/// Default: `false`
|
||||
pub cache: bool,
|
||||
/// Which RowFilters to try on the file
|
||||
///
|
||||
/// Default: `None,Sub,Entropy,Bigrams`
|
||||
|
|
@ -301,6 +306,7 @@ impl Default for Options {
|
|||
Options {
|
||||
fix_errors: false,
|
||||
force: false,
|
||||
cache: false,
|
||||
filter: indexset! {RowFilter::None, RowFilter::Sub, RowFilter::Entropy, RowFilter::Bigrams},
|
||||
interlace: Some(Interlacing::None),
|
||||
optimize_alpha: false,
|
||||
|
|
@ -449,12 +455,25 @@ pub fn optimize(input: &InFile, output: &OutFile, opts: &Options) -> PngResult<(
|
|||
data
|
||||
}
|
||||
};
|
||||
// When cache is enabled, check if the file has already been optimized.
|
||||
if opts.cache {
|
||||
let is_data_in_cache: bool = crate::cache::check_cache_for_data_hash(&in_data);
|
||||
if is_data_in_cache {
|
||||
info!("{}: File was previously optimized, skipping.", input);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
|
||||
let mut png = PngData::from_slice(&in_data, opts)?;
|
||||
|
||||
// Run the optimizer on the decoded PNG.
|
||||
let mut optimized_output = optimize_png(&mut png, &in_data, opts, deadline)?;
|
||||
|
||||
// When cache is enabled, write the hash of the optimized file to the cache.
|
||||
if opts.cache {
|
||||
crate::cache::write_data_hash_to_cache(&optimized_output);
|
||||
}
|
||||
|
||||
let in_length = in_data.len();
|
||||
|
||||
if is_fully_optimized(in_data.len(), optimized_output.len(), opts) {
|
||||
|
|
|
|||
|
|
@ -242,6 +242,13 @@ fn main() {
|
|||
.long("force")
|
||||
.action(ArgAction::SetTrue),
|
||||
)
|
||||
.arg(
|
||||
Arg::new("cache")
|
||||
.help("Cache the history of optimized files and avoid reoptimizing optimal files")
|
||||
.short('c')
|
||||
.long("cache")
|
||||
.action(ArgAction::SetTrue),
|
||||
)
|
||||
.arg(
|
||||
Arg::new("zopfli")
|
||||
.help("Use the slow but stronger Zopfli compressor (recommended use is with all filters and `--fast` enabled)")
|
||||
|
|
@ -493,6 +500,8 @@ fn parse_opts_into_struct(
|
|||
|
||||
opts.force = matches.get_flag("force");
|
||||
|
||||
opts.cache = matches.get_flag("cache");
|
||||
|
||||
opts.fix_errors = matches.get_flag("fix");
|
||||
|
||||
opts.bit_depth_reduction = !matches.get_flag("no-bit-reduction");
|
||||
|
|
|
|||
Loading…
Reference in a new issue