diff --git a/src/evaluate.rs b/src/evaluate.rs index 3b78b10c..b0096c0e 100644 --- a/src/evaluate.rs +++ b/src/evaluate.rs @@ -35,7 +35,7 @@ pub(crate) struct Candidate { } impl Candidate { - fn cmp_key(&self) -> impl Ord + use<> { + pub fn cmp_key(&self) -> impl Ord + use<> { ( self.estimated_output_size, self.image.data.len(), @@ -56,6 +56,7 @@ pub(crate) struct Evaluator { nth: AtomicUsize, executed: Arc, best_candidate_size: Arc, + threshold: f64, /// images are sent to the caller thread for evaluation #[cfg(feature = "parallel")] eval_channel: (Sender, Receiver), @@ -70,6 +71,7 @@ impl Evaluator { filters: IndexSet, deflater: Deflater, optimize_alpha: bool, + threshold: f64, final_round: bool, ) -> Self { #[cfg(feature = "parallel")] @@ -83,6 +85,7 @@ impl Evaluator { nth: AtomicUsize::new(0), executed: Arc::new(AtomicUsize::new(0)), best_candidate_size: Arc::new(AtomicMin::new(None)), + threshold, #[cfg(feature = "parallel")] eval_channel, #[cfg(not(feature = "parallel"))] @@ -93,7 +96,7 @@ impl Evaluator { /// Wait for all evaluations to finish and return smallest reduction /// Or `None` if the queue is empty. #[cfg(feature = "parallel")] - pub fn get_best_candidate(self) -> Option { + pub fn get_best_candidates(self, limit: usize) -> Vec { let (eval_send, eval_recv) = self.eval_channel; // Disconnect the sender, breaking the loop in the thread drop(eval_send); @@ -103,17 +106,28 @@ impl Evaluator { while self.executed.load(Relaxed) < nth { rayon::yield_local(); } - eval_recv.into_iter().min_by_key(Candidate::cmp_key) + let mut candidates: Vec<_> = eval_recv.iter().collect(); + if candidates.is_empty() { + return candidates; + } + candidates.sort_by_key(Candidate::cmp_key); + let best_size = self.best_candidate_size.get().unwrap(); + candidates + .into_iter() + .take(limit) + .filter(|c| c.estimated_output_size <= best_size) + .collect() } #[cfg(not(feature = "parallel"))] - pub fn get_best_candidate(self) -> Option { - self.eval_best_candidate.into_inner() + pub fn get_best_candidates(self, _limit: usize) -> Vec { + vec![] } /// Set best size, if known in advance pub fn set_best_size(&self, size: usize) { - self.best_candidate_size.set_min(size); + self.best_candidate_size + .set_min(size + (self.threshold * size as f64) as usize); } /// Check if the image is smaller than others @@ -133,6 +147,7 @@ impl Evaluator { let final_round = self.final_round; let executed = self.executed.clone(); let best_candidate_size = self.best_candidate_size.clone(); + let threshold = self.threshold; let description = description.to_string(); // sends it off asynchronously for compression, // but results will be collected via the message queue @@ -153,30 +168,22 @@ impl Evaluator { let (filtered, filter_used) = image.filter_image(filter.clone(), optimize_alpha); let idat_data = deflater.deflate(&filtered, best_candidate_size.get()); if let Ok(idat_data) = idat_data { - let estimated_output_size = image.estimated_output_size(&idat_data); + let size = image.estimated_output_size(&idat_data); trace!( "Eval: {}-bit {:23} {:8} {} bytes", - image.ihdr.bit_depth, description, filter, estimated_output_size + image.ihdr.bit_depth, description, filter, size ); - // Skip if it exceeds best known size. (This is important to ensure - // the evaluator returns no result when all candidates are too large.) - if let Some(max) = best_candidate_size.get() { - if estimated_output_size > max { - return; - } - } - // We only need to retain the IDAT data in the final round let new = Candidate { image: image.clone(), idat_data: if final_round { Some(idat_data) } else { None }, - estimated_output_size, + estimated_output_size: size, filter: filter.clone(), filter_used, nth, }; - best_candidate_size.set_min(estimated_output_size); + best_candidate_size.set_min(size + (threshold * size as f64) as usize); #[cfg(feature = "parallel")] { @@ -199,4 +206,73 @@ impl Evaluator { }); }); } + + pub fn try_candidates(&self, candidates: Vec) { + let nth = self.nth.fetch_add(1, SeqCst); + // These clones are only cheap refcounts + let deadline = self.deadline.clone(); + let deflater = self.deflater; + let optimize_alpha = self.optimize_alpha; + let final_round = self.final_round; + let executed = self.executed.clone(); + let best_candidate_size = self.best_candidate_size.clone(); + let threshold = self.threshold; + // sends it off asynchronously for compression, + // but results will be collected via the message queue + #[cfg(feature = "parallel")] + let eval_send = self.eval_channel.0.clone(); + rayon::spawn(move || { + executed.fetch_add(1, Relaxed); + + // Updating of best result inside the parallel loop would require locks, + // which are dangerous to do in side Rayon's loop. + // Instead, only update (atomic) best size in real time, + // and the best result later without need for locks. + candidates.par_iter().with_max_len(1).for_each(|candidate| { + if deadline.passed() { + return; + } + let description = format!("{}", candidate.image.ihdr.color_type); + let (filtered, filter_used) = candidate + .image + .filter_image(candidate.filter_used.clone(), optimize_alpha); + let idat_data = deflater.deflate(&filtered, best_candidate_size.get()); + if let Ok(idat_data) = idat_data { + let size = candidate.image.estimated_output_size(&idat_data); + // For the final round we need the IDAT data, otherwise the filtered data + let new = Candidate { + image: candidate.image.clone(), + idat_data: if final_round { Some(idat_data) } else { None }, + estimated_output_size: size, + filter: candidate.filter.clone(), + filter_used, + nth, + }; + best_candidate_size.set_min(size + (threshold * size as f64) as usize); + trace!( + "Eval: {}-bit {:23} {:8} {} bytes", + new.image.ihdr.bit_depth, description, new.filter, size + ); + + #[cfg(feature = "parallel")] + { + eval_send.send(new).expect("send"); + } + + #[cfg(not(feature = "parallel"))] + { + match &mut *self.eval_best_candidate.borrow_mut() { + Some(prev) if prev.cmp_key() < new.cmp_key() => {} + best => *best = Some(new), + } + } + } else if let Err(PngError::DeflatedDataTooLong(size)) = idat_data { + trace!( + "Eval: {}-bit {:23} {:8} >{} bytes", + candidate.image.ihdr.bit_depth, description, candidate.filter, size + ); + } + }); + }); + } } diff --git a/src/lib.rs b/src/lib.rs index f517e745..49f440f6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -395,10 +395,8 @@ fn optimize_raw( // 9 is not appreciably better than 8 // 10 and higher are quite slow - good for filters but only good for reductions if matching the main zc level let compression = match opts.deflater { - Deflater::Libdeflater { compression } => { - if opts.fast_evaluation { 7 } else { 8 }.min(compression) - } - _ => 8, + Deflater::Libdeflater { compression } => 7.min(compression), + _ => 7, }; let eval_deflater = Deflater::Libdeflater { compression }; // If only one filter is selected, use this for evaluations @@ -414,16 +412,15 @@ fn optimize_raw( eval_filters.clone(), eval_deflater, false, + opts.candidate_threshold, opts.deflater == eval_deflater, ); let mut new_image = perform_reductions(image.clone(), opts, &deadline, &eval); - let eval_result = eval.get_best_candidate(); - if let Some(ref result) = eval_result { + let eval_results = eval.get_best_candidates(opts.max_candidates); + if let Some(result) = eval_results.first() { new_image = result.image.clone(); } - let reduction_occurred = new_image.ihdr.color_type != image.ihdr.color_type - || new_image.ihdr.bit_depth != image.ihdr.bit_depth - || new_image.ihdr.interlaced != image.ihdr.interlaced; + let reduction_occurred = !new_image.is_same_format(&image); if reduction_occurred { report_format("Transformed image to ", &new_image); @@ -435,7 +432,7 @@ fn optimize_raw( opts, deadline, max_size, - eval_result, + eval_results, eval_filters, eval_deflater, ); @@ -443,7 +440,8 @@ fn optimize_raw( } else { // If idat_recoding is off and reductions were attempted but ended up choosing the baseline, // we should still check if the evaluator compressed the baseline smaller than the original. - (eval_result?, eval_deflater) + let result = eval_results.into_iter().next(); + (result?, eval_deflater) }; if result.idat_data.is_some() @@ -462,58 +460,52 @@ fn perform_trials( opts: &Options, deadline: Arc, max_size: Option, - mut eval_result: Option, + mut eval_results: Vec, eval_filters: IndexSet, eval_deflater: Deflater, ) -> Option { + // First perform a fast evaluation of selected filters let mut filters = opts.filters.clone(); - let fast_eval = opts.fast_evaluation && (filters.len() > 1 || eval_result.is_some()); - if fast_eval { - // Perform a fast evaluation of selected filters followed by a single main compression trial + if !eval_results.is_empty() { + // Some filters have already been evaluated, we don't need to try them again + filters = filters.difference(&eval_filters).cloned().collect(); + } - if eval_result.is_some() { - // Some filters have already been evaluated, we don't need to try them again - filters = filters.difference(&eval_filters).cloned().collect(); - } - - if !filters.is_empty() { - trace!("Evaluating {} filters", filters.len()); - let eval = Evaluator::new( - deadline, - filters, - eval_deflater, - opts.optimize_alpha, - opts.deflater == eval_deflater, - ); - if let Some(result) = &eval_result { - eval.set_best_size(result.estimated_output_size); + if !filters.is_empty() { + trace!("Evaluating {} filters", filters.len()); + let eval = Evaluator::new( + deadline.clone(), + filters.clone(), + eval_deflater, + opts.optimize_alpha, + opts.candidate_threshold, + opts.deflater == eval_deflater, + ); + if let Some(best) = eval_results.first() { + eval.set_best_size(best.estimated_output_size); + for result in &eval_results { + eval.try_image(result.image.clone()); } + } else { eval.try_image(image.clone()); - if let Some(result) = eval.get_best_candidate() { - eval_result = Some(result); - } } - - // We should have a result here - fail if not (e.g. deadline passed) - let mut result = eval_result?; - - if result.idat_data.is_none() { - // Compress with the main deflater - debug!("Trying filter {} with {}", result.filter, opts.deflater); - let (data, _) = image.filter_image(result.filter_used.clone(), opts.optimize_alpha); - match opts.deflater.deflate(&data, max_size) { - Ok(idat_data) => { - result.estimated_output_size = result.image.estimated_output_size(&idat_data); - result.idat_data = Some(idat_data); - trace!("{} bytes", result.estimated_output_size); - } - Err(PngError::DeflatedDataTooLong(bytes)) => { - trace!(">{bytes} bytes"); - } - Err(_) => (), - } + let mut results = eval.get_best_candidates(opts.max_candidates); + if !results.is_empty() { + eval_results.append(&mut results); + eval_results.sort_by_key(Candidate::cmp_key); + let mut best_size = eval_results.first().unwrap().estimated_output_size; + best_size += (opts.candidate_threshold * best_size as f64) as usize; + eval_results = eval_results + .into_iter() + .take(opts.max_candidates) + .filter(|c| c.estimated_output_size <= best_size) + .collect(); } - return Some(result); + } + + // Return best result if this is the final round + if opts.deflater == eval_deflater { + return eval_results.into_iter().next(); } // Perform full compression trials of selected filters and determine the best @@ -530,12 +522,23 @@ fn perform_trials( } debug!("Trying {} filters with {}", filters.len(), opts.deflater); - let eval = Evaluator::new(deadline, filters, opts.deflater, opts.optimize_alpha, true); + let eval = Evaluator::new( + deadline, + filters, + opts.deflater, + opts.optimize_alpha, + opts.candidate_threshold, + true, + ); if let Some(max_size) = max_size { eval.set_best_size(max_size); } - eval.try_image(image); - eval.get_best_candidate() + if eval_results.is_empty() { + eval.try_image(image); + } else { + eval.try_candidates(eval_results); + } + eval.get_best_candidates(1).into_iter().next() } #[derive(Debug)] diff --git a/src/options.rs b/src/options.rs index 4e3e8d4f..a040a6a4 100644 --- a/src/options.rs +++ b/src/options.rs @@ -149,6 +149,8 @@ pub struct Options { /// /// Default: `true` pub fast_evaluation: bool, + pub max_candidates: usize, + pub candidate_threshold: f64, /// Maximum amount of time to spend on optimizations. /// Further potential optimizations are skipped if the timeout is exceeded. /// @@ -204,7 +206,6 @@ impl Options { } fn apply_preset_3(mut self) -> Self { - self.fast_evaluation = false; self.filters = indexset! { FilterStrategy::NONE, FilterStrategy::Bigrams, @@ -214,11 +215,12 @@ impl Options { level: 1, }, }; + self.max_candidates = 2; + self.candidate_threshold = 0.05; self } fn apply_preset_4(mut self) -> Self { - self.fast_evaluation = false; self.filters = indexset! { FilterStrategy::NONE, FilterStrategy::Bigrams, @@ -229,11 +231,12 @@ impl Options { }, }; self.deflater = Deflater::Libdeflater { compression: 12 }; + self.max_candidates = 4; + self.candidate_threshold = 0.05; self } fn apply_preset_5(mut self) -> Self { - self.fast_evaluation = false; self.filters = indexset! { FilterStrategy::NONE, FilterStrategy::SUB, @@ -248,11 +251,12 @@ impl Options { }, }; self.deflater = Deflater::Libdeflater { compression: 12 }; + self.max_candidates = 8; + self.candidate_threshold = 0.1; self } fn apply_preset_6(mut self) -> Self { - self.fast_evaluation = false; self.filters = indexset! { FilterStrategy::NONE, FilterStrategy::SUB, @@ -269,6 +273,8 @@ impl Options { }, }; self.deflater = Deflater::Libdeflater { compression: 12 }; + self.max_candidates = 16; + self.candidate_threshold = 0.2; self } } @@ -296,6 +302,8 @@ impl Default for Options { strip: StripChunks::None, deflater: Deflater::Libdeflater { compression: 11 }, fast_evaluation: true, + max_candidates: 1, + candidate_threshold: 0.0, timeout: None, max_decompressed_size: None, } diff --git a/src/png/mod.rs b/src/png/mod.rs index a2961f13..8f91778e 100644 --- a/src/png/mod.rs +++ b/src/png/mod.rs @@ -262,6 +262,13 @@ impl PngImage { Ok(image) } + #[must_use] + pub fn is_same_format(&self, other: &Self) -> bool { + self.ihdr.color_type != other.ihdr.color_type + || self.ihdr.bit_depth != other.ihdr.bit_depth + || self.ihdr.interlaced != other.ihdr.interlaced + } + /// Enable or disable interlacing /// Returns the new image if the interlacing was changed, None otherwise /// Assumes that the data has already been de-filtered diff --git a/src/reduction/mod.rs b/src/reduction/mod.rs index 5881a917..8cb24a4d 100644 --- a/src/reduction/mod.rs +++ b/src/reduction/mod.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use crate::{ColorType, Deadline, Deflater, Options, evaluate::Evaluator, png::PngImage}; +use crate::{ColorType, Deadline, Options, evaluate::Evaluator, png::PngImage}; pub mod alpha; use crate::alpha::*; @@ -21,10 +21,7 @@ pub(crate) fn perform_reductions( // At low compression levels, skip some transformations which are less likely to be effective // This currently affects optimization presets 0-2 - let cheap = match opts.deflater { - Deflater::Libdeflater { compression } => compression < 12 && opts.fast_evaluation, - _ => false, - }; + let cheap = opts.max_candidates == 1; // Interlacing must be processed first in order to evaluate the rest correctly if let Some(interlacing) = opts.interlace {