diff --git a/src/atomicmin.rs b/src/atomicmin.rs index 73c3d997..69a379f2 100644 --- a/src/atomicmin.rs +++ b/src/atomicmin.rs @@ -27,7 +27,8 @@ impl AtomicMin { &self.val } - pub fn set_min(&self, new_val: usize) { - self.val.fetch_min(new_val, SeqCst); + /// Try a new value, returning true if it is the new minimum + pub fn set_min(&self, new_val: usize) -> bool { + new_val < self.val.fetch_min(new_val, SeqCst) } } diff --git a/src/deflate/mod.rs b/src/deflate/mod.rs index ac17577c..9606a852 100644 --- a/src/deflate/mod.rs +++ b/src/deflate/mod.rs @@ -1,5 +1,3 @@ -use indexmap::IndexSet; - mod deflater; pub use deflater::crc32; pub use deflater::deflate; @@ -12,13 +10,13 @@ mod zopfli_oxipng; #[cfg(feature = "zopfli")] pub use zopfli_oxipng::deflate as zopfli_deflate; -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] /// DEFLATE algorithms supported by oxipng pub enum Deflaters { /// Use libdeflater. Libdeflater { - /// Which compression levels to try on the file (1-12) - compression: IndexSet, + /// Which compression level to use on the file (1-12) + compression: u8, }, #[cfg(feature = "zopfli")] /// Use the better but slower Zopfli implementation diff --git a/src/evaluate.rs b/src/evaluate.rs index b4d31919..15222f7a 100644 --- a/src/evaluate.rs +++ b/src/evaluate.rs @@ -11,6 +11,7 @@ use crate::rayon; use crate::Deadline; #[cfg(feature = "parallel")] use crossbeam_channel::{unbounded, Receiver, Sender}; +use indexmap::IndexSet; use rayon::prelude::*; #[cfg(not(feature = "parallel"))] use std::cell::RefCell; @@ -18,13 +19,10 @@ use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering::SeqCst; use std::sync::Arc; -/// Must use normal (lazy) compression, as faster ones (greedy) are not representative -const STD_COMPRESSION: u8 = 5; -const STD_FILTERS: [RowFilter; 2] = [RowFilter::None, RowFilter::MinSum]; - -struct Candidate { - image: PngData, - filter: RowFilter, +pub struct Candidate { + pub image: PngData, + pub filter: RowFilter, + pub is_reduction: bool, // first wins tie-breaker nth: usize, } @@ -44,6 +42,8 @@ impl Candidate { /// Collect image versions and pick one that compresses best pub(crate) struct Evaluator { deadline: Arc, + filters: IndexSet, + compression: u8, nth: AtomicUsize, best_candidate_size: Arc, /// images are sent to the caller thread for evaluation @@ -55,11 +55,13 @@ pub(crate) struct Evaluator { } impl Evaluator { - pub fn new(deadline: Arc) -> Self { + pub fn new(deadline: Arc, filters: IndexSet, compression: u8) -> Self { #[cfg(feature = "parallel")] let eval_channel = unbounded(); Self { deadline, + filters, + compression, best_candidate_size: Arc::new(AtomicMin::new(None)), nth: AtomicUsize::new(0), #[cfg(feature = "parallel")] @@ -72,26 +74,27 @@ impl Evaluator { /// Wait for all evaluations to finish and return smallest reduction /// Or `None` if all reductions were worse than baseline. #[cfg(feature = "parallel")] - fn get_best_candidate(self) -> Option { + pub fn get_best_candidate(self) -> Option { let (eval_send, eval_recv) = self.eval_channel; drop(eval_send); // disconnect the sender, breaking the loop in the thread eval_recv.into_iter().min_by_key(Candidate::cmp_key) } #[cfg(not(feature = "parallel"))] - fn get_best_candidate(self) -> Option { + pub fn get_best_candidate(self) -> Option { self.eval_best_candidate.into_inner() } - pub fn get_result(self) -> Option { - self.get_best_candidate().map(|candidate| candidate.image) - } - /// Set baseline image. It will be used only to measure minimum compression level required pub fn set_baseline(&self, image: Arc) { self.try_image_inner(image, false) } + /// Set best size, if known in advance + pub fn set_best_size(&self, size: usize) { + self.best_candidate_size.set_min(size); + } + /// Check if the image is smaller than others pub fn try_image(&self, image: Arc) { self.try_image_inner(image, true) @@ -101,13 +104,15 @@ impl Evaluator { let nth = self.nth.fetch_add(1, SeqCst); // These clones are only cheap refcounts let deadline = self.deadline.clone(); + let filters = self.filters.clone(); + let compression = self.compression; let best_candidate_size = self.best_candidate_size.clone(); // sends it off asynchronously for compression, // but results will be collected via the message queue #[cfg(feature = "parallel")] let eval_send = self.eval_channel.0.clone(); rayon::spawn(move || { - let filters_iter = STD_FILTERS.par_iter().with_max_len(1); + let filters_iter = filters.par_iter().with_max_len(1); // Updating of best result inside the parallel loop would require locks, // which are dangerous to do in side Rayon's loop. @@ -119,21 +124,17 @@ impl Evaluator { } if let Ok(idat_data) = deflate::deflate( &image.filter_image(filter), - STD_COMPRESSION, + compression, &best_candidate_size, ) { best_candidate_size.set_min(idat_data.len()); - // ignore baseline images after this point - if !is_reduction { - return; - } - // the rest is shipped to the evavluation/collection thread let new = Candidate { image: PngData { idat_data, raw: Arc::clone(&image), }, filter, + is_reduction, nth, }; diff --git a/src/lib.rs b/src/lib.rs index 899640da..f4be1979 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -47,7 +47,7 @@ pub use crate::deflate::Deflaters; pub use crate::error::PngError; pub use crate::filters::RowFilter; pub use crate::headers::Headers; -pub use indexmap::{IndexMap, IndexSet}; +pub use indexmap::{indexset, IndexMap, IndexSet}; mod atomicmin; mod colors; @@ -192,12 +192,12 @@ pub struct Options { /// /// Default: `Libdeflater` pub deflate: Deflaters, - /// Whether to use heuristics to pick the best filter and compression + /// Whether to use fast evaluation to pick the best filter /// /// Intended for use with `-o 1` from the CLI interface /// /// Default: `false` - pub use_heuristics: bool, + pub fast_evaluation: bool, /// Maximum amount of time to spend on optimizations. /// Further potential optimizations are skipped if the timeout is exceeded. @@ -232,20 +232,18 @@ impl Options { self.idat_recoding = false; self.filter.clear(); if let Deflaters::Libdeflater { compression } = &mut self.deflate { - compression.clear(); - compression.insert(5); + *compression = 5; } - self.use_heuristics = true; + self.fast_evaluation = true; self } fn apply_preset_1(mut self) -> Self { self.filter.clear(); if let Deflaters::Libdeflater { compression } = &mut self.deflate { - compression.clear(); - compression.insert(10); + *compression = 10; } - self.use_heuristics = true; + self.fast_evaluation = true; self } @@ -263,45 +261,23 @@ impl Options { fn apply_preset_4(mut self) -> Self { if let Deflaters::Libdeflater { compression } = &mut self.deflate { - compression.clear(); - compression.insert(12); + *compression = 12; } self.apply_preset_3() } - fn apply_preset_5(mut self) -> Self { - if let Deflaters::Libdeflater { compression } = &mut self.deflate { - compression.clear(); - for i in 9..=12 { - compression.insert(i); - } - } - self.apply_preset_3() + fn apply_preset_5(self) -> Self { + self.apply_preset_4() } - fn apply_preset_6(mut self) -> Self { - if let Deflaters::Libdeflater { compression } = &mut self.deflate { - compression.clear(); - for i in 1..=12 { - compression.insert(i); - } - } - self.apply_preset_3() + fn apply_preset_6(self) -> Self { + self.apply_preset_4() } } impl Default for Options { fn default() -> Options { // Default settings based on -o 2 from the CLI interface - let mut filter = IndexSet::new(); - filter.insert(RowFilter::None); - filter.insert(RowFilter::MinSum); - let mut compression = IndexSet::new(); - compression.insert(11); - // We always need NoOp to be present - let mut alphas = IndexSet::new(); - alphas.insert(AlphaOptim::NoOp); - Options { backup: false, check: false, @@ -309,17 +285,17 @@ impl Default for Options { fix_errors: false, force: false, preserve_attrs: false, - filter, + filter: indexset! {RowFilter::None, RowFilter::MinSum}, interlace: None, - alphas, + alphas: IndexSet::new(), bit_depth_reduction: true, color_type_reduction: true, palette_reduction: true, grayscale_reduction: true, idat_recoding: true, strip: Headers::None, - deflate: Deflaters::Libdeflater { compression }, - use_heuristics: false, + deflate: Deflaters::Libdeflater { compression: 11 }, + fast_evaluation: false, timeout: None, } } @@ -470,6 +446,7 @@ struct TrialOptions { pub filter: RowFilter, pub compression: u8, } +type TrialWithData = (TrialOptions, Vec); /// Perform optimization on the input PNG object using the options provided fn optimize_png( @@ -478,8 +455,6 @@ fn optimize_png( opts: &Options, deadline: Arc, ) -> PngResult> { - type TrialWithData = (TrialOptions, Vec); - let original_png = png.clone(); // Print png info @@ -506,135 +481,106 @@ fn optimize_png( info!(" IDAT size = {} bytes", idat_original_size); info!(" File size = {} bytes", file_original_size); - let mut filter = opts.filter.clone(); - - if opts.use_heuristics { - // Heuristically determine which set of options to use - let use_filter = if png.raw.ihdr.bit_depth.as_u8() >= 8 - && png.raw.ihdr.color_type != colors::ColorType::Indexed - { - RowFilter::MinSum - } else { - RowFilter::None - }; - if filter.is_empty() { - filter.insert(use_filter); - } - } - + // Must use normal (lazy) compression, as faster ones (greedy) are not representative + let eval_compression = 5; + let eval_filters = indexset! {RowFilter::None, RowFilter::MinSum}; // This will collect all versions of images and pick one that compresses best - let eval = Evaluator::new(deadline.clone()); - // Usually we want transformations that are smaller than the unmodified original, - // but if we're interlacing, we have to accept a possible file size increase. - if opts.interlace.is_none() { - eval.set_baseline(png.raw.clone()); - } + let eval = Evaluator::new(deadline.clone(), eval_filters.clone(), eval_compression); perform_reductions(png.raw.clone(), opts, &deadline, &eval); - let reduction_occurred = if let Some(result) = eval.get_result() { - *png = result; - true + let (reduction_occurred, mut eval_filter) = if let Some(result) = eval.get_best_candidate() { + *png = result.image; + (result.is_reduction, Some(result.filter)) } else { - false + (false, None) }; if opts.idat_recoding || reduction_occurred { - // Go through selected permutations and determine the best - let combinations = if let Deflaters::Libdeflater { compression } = &opts.deflate { - filter.len() * compression.len() - } else { - filter.len() - }; - let mut results: Vec = Vec::with_capacity(combinations); + let mut filters = opts.filter.clone(); + let fast_eval = opts.fast_evaluation && (filters.len() > 1 || eval_filter.is_some()); + let best: Option = if fast_eval { + // Perform a fast evaluation of selected filters followed by a single main compression trial + if eval_filter.is_some() { + // Some filters have already been evaluated, we don't need to try them again + filters = filters.difference(&eval_filters).cloned().collect(); + } - for f in &filter { - if let Deflaters::Libdeflater { compression } = &opts.deflate { - for zc in compression { - results.push(TrialOptions { - filter: *f, - compression: *zc, - }); - if deadline.passed() { - break; - } + if !filters.is_empty() { + debug!("Evaluating: {} filters", filters.len()); + let eval = Evaluator::new(deadline, filters, eval_compression); + if eval_filter.is_some() { + eval.set_best_size(png.idat_data.len()); + } + eval.try_image(png.raw.clone()); + if let Some(result) = eval.get_best_candidate() { + *png = result.image; + eval_filter = Some(result.filter); + } + } + + let trial = TrialOptions { + filter: eval_filter.unwrap(), + compression: match opts.deflate { + Deflaters::Libdeflater { compression } => compression, + _ => 0, + }, + }; + if trial.compression <= eval_compression { + // No further compression required + if png.idat_data.len() < idat_original_size || opts.force { + Some((trial, png.idat_data.clone())) + } else { + None } } else { - // Zopfli has no additional options. + info!("Trying: {}", trial.filter); + let original_len = idat_original_size; + let best_size = AtomicMin::new(if opts.force { None } else { Some(original_len) }); + perform_trial(&png.raw, opts, trial, &best_size) + } + } else { + // Perform full compression trials of selected filters and determine the best + if filters.is_empty() { + // Heuristically determine which filter to use + if png.raw.ihdr.bit_depth.as_u8() >= 8 + && png.raw.ihdr.color_type != colors::ColorType::Indexed + { + filters.insert(RowFilter::MinSum); + } else { + filters.insert(RowFilter::None); + } + } + + let mut results: Vec = Vec::with_capacity(filters.len()); + + for f in &filters { results.push(TrialOptions { filter: *f, - compression: 0, + compression: match opts.deflate { + Deflaters::Libdeflater { compression } => compression, + _ => 0, + }, }); } - if deadline.passed() { - break; - } - } + info!("Trying: {} filters", results.len()); - info!("Trying: {} combinations", results.len()); - - let filters: IndexMap> = filter - .par_iter() - .with_max_len(1) - .map(|f| { - let png = png.clone(); - (*f, png.raw.filter_image(*f)) - }) - .collect(); - - let original_len = original_png.idat_data.len(); - let added_interlacing = opts.interlace == Some(1) && original_png.raw.ihdr.interlaced == 0; - - let best_size = AtomicMin::new(if opts.force { None } else { Some(original_len) }); - let results_iter = results.into_par_iter().with_max_len(1); - let best = results_iter.filter_map(|trial| { - if deadline.passed() { - return None; - } - let filtered = &filters[&trial.filter]; - let new_idat = match opts.deflate { - Deflaters::Libdeflater { .. } => { - deflate::deflate(filtered, trial.compression, &best_size) - } - #[cfg(feature = "zopfli")] - Deflaters::Zopfli { iterations } => deflate::zopfli_deflate(filtered, iterations), - }; - - let new_idat = match new_idat { - Ok(n) => n, - Err(PngError::DeflatedDataTooLong(max)) => { - debug!( - " zc = {} f = {} >{} bytes", - trial.compression, trial.filter, max, - ); + let original_len = idat_original_size; + let best_size = AtomicMin::new(if opts.force { None } else { Some(original_len) }); + let results_iter = results.into_par_iter().with_max_len(1); + let best = results_iter.filter_map(|trial| { + if deadline.passed() { return None; } - Err(_) => return None, - }; - - // update best size across all threads - let new_size = new_idat.len(); - best_size.set_min(new_size); - - debug!( - " zc = {} f = {} {} bytes", - trial.compression, - trial.filter, - new_idat.len() - ); - - if new_size < original_len || added_interlacing || opts.force { - Some((trial, new_idat)) - } else { - None - } - }); - let best: Option = best.reduce_with(|i, j| { - if i.1.len() < j.1.len() || (i.1.len() == j.1.len() && i.0 < j.0) { - i - } else { - j - } - }); + perform_trial(&png.raw, opts, trial, &best_size) + }); + best.reduce_with(|i, j| { + if i.1.len() < j.1.len() || (i.1.len() == j.1.len() && i.0 < j.0) { + i + } else { + j + } + }) + }; if let Some((opts, idat_data)) = best { png.idat_data = idat_data; @@ -645,9 +591,11 @@ fn optimize_png( opts.filter, png.idat_data.len() ); - } else if reduction_occurred { + } else if eval_filter.is_some() { *png = original_png; } + } else if png.idat_data.len() >= idat_original_size { + *png = original_png; } perform_strip(png, opts); @@ -713,11 +661,17 @@ fn perform_reductions( deadline: &Deadline, eval: &Evaluator, ) { + // The eval baseline will be set from the original png only if we attempt any reductions + let mut baseline = Some(png.clone()); + let mut reduction_occurred = false; + // must be done first to evaluate rest with the correct interlacing if let Some(interlacing) = opts.interlace { if let Some(reduced) = png.change_interlacing(interlacing) { png = Arc::new(reduced); eval.try_image(png.clone()); + // If we're interlacing, we have to accept a possible file size increase + baseline = None; } if deadline.passed() { return; @@ -729,6 +683,7 @@ fn perform_reductions( png = Arc::new(reduced); eval.try_image(png.clone()); report_reduction(&png); + reduction_occurred = true; } if deadline.passed() { return; @@ -750,6 +705,7 @@ fn perform_reductions( } } report_reduction(&png); + reduction_occurred = true; } if deadline.passed() { return; @@ -761,13 +717,62 @@ fn perform_reductions( png = Arc::new(reduced); eval.try_image(png.clone()); report_reduction(&png); + reduction_occurred = true; } if deadline.passed() { return; } } - try_alpha_reductions(png, &opts.alphas, eval); + if try_alpha_reductions(png, &opts.alphas, eval) { + reduction_occurred = true; + } + + if let Some(baseline) = baseline { + if reduction_occurred { + eval.set_baseline(baseline); + } + } +} + +/// Execute a compression trial +fn perform_trial( + png: &PngImage, + opts: &Options, + trial: TrialOptions, + best_size: &AtomicMin, +) -> Option { + let filtered = &png.filter_image(trial.filter); + let new_idat = match opts.deflate { + Deflaters::Libdeflater { .. } => deflate::deflate(filtered, trial.compression, best_size), + #[cfg(feature = "zopfli")] + Deflaters::Zopfli { iterations } => deflate::zopfli_deflate(filtered, iterations), + }; + + // update best size or convert to error if not smaller + let new_idat = match new_idat { + Ok(n) if !best_size.set_min(n.len()) => Err(PngError::DeflatedDataTooLong(n.len())), + _ => new_idat, + }; + + match new_idat { + Ok(n) => { + let bytes = n.len(); + debug!( + " zc = {} f = {} {} bytes", + trial.compression, trial.filter, bytes + ); + Some((trial, n)) + } + Err(PngError::DeflatedDataTooLong(bytes)) => { + debug!( + " zc = {} f = {} >{} bytes", + trial.compression, trial.filter, bytes, + ); + None + } + Err(_) => None, + } } #[derive(Debug)] diff --git a/src/main.rs b/src/main.rs index 969b856d..5b9f41d4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -184,14 +184,11 @@ fn main() { ) .arg( Arg::new("compression") - .help("zlib compression levels (1-12) - Default: 12") + .help("zlib compression level (1-12) - Default: 11") .long("zc") .takes_value(true) - .value_name("levels") - .validator(|x| match parse_numeric_range_opts(x, 1, 12) { - Ok(_) => Ok(()), - Err(_) => Err("Invalid option for compression".to_owned()), - }) + .value_name("level") + .value_parser(1..=12) .conflicts_with("zopfli"), ) .arg( @@ -541,8 +538,8 @@ fn parse_opts_into_struct( opts.deflate = Deflaters::Zopfli { iterations }; } } else if let Deflaters::Libdeflater { compression } = &mut opts.deflate { - if let Some(x) = matches.value_of("compression") { - *compression = parse_numeric_range_opts(x, 1, 12).unwrap(); + if let Some(x) = matches.get_one::("compression") { + *compression = *x as u8; } } diff --git a/src/reduction/alpha.rs b/src/reduction/alpha.rs index 06df0676..b3c5c9a1 100644 --- a/src/reduction/alpha.rs +++ b/src/reduction/alpha.rs @@ -14,16 +14,18 @@ pub(crate) fn try_alpha_reductions( png: Arc, alphas: &IndexSet, eval: &Evaluator, -) { - if alphas.is_empty() { - return; +) -> bool { + match png.ihdr.color_type { + ColorType::RGBA | ColorType::GrayscaleAlpha if !alphas.is_empty() => { + alphas + .par_iter() + .with_max_len(1) + .filter_map(|&alpha| filtered_alpha_channel(&png, alpha)) + .for_each(|image| eval.try_image(Arc::new(image))); + true + } + _ => false, } - - alphas - .par_iter() - .with_max_len(1) - .filter_map(|&alpha| filtered_alpha_channel(&png, alpha)) - .for_each(|image| eval.try_image(Arc::new(image))); } pub fn filtered_alpha_channel(png: &PngImage, optim: AlphaOptim) -> Option {