diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a69780c..9e1b2539 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +**Version 0.11.1 (unreleased)** + - Performance optimizations + **Version 0.11.0** - [SEMVER_MAJOR] Bump minimum rustc version to 1.9.0, required by dependencies - [SEMVER_MINOR] Allow calling optimization presets via crate using `Options::from_preset` diff --git a/Cargo.lock b/Cargo.lock index 93469384..e47ebec8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13,8 +13,8 @@ dependencies = [ "libz-sys 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "miniz-sys 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rayon 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "regex 0.1.77 (registry+https://github.com/rust-lang/crates.io-index)", - "scoped-pool 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -91,9 +91,12 @@ dependencies = [ ] [[package]] -name = "crossbeam" -version = "0.2.10" +name = "deque" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", +] [[package]] name = "enum_primitive" @@ -256,6 +259,14 @@ name = "num-traits" version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "num_cpus" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "num_cpus" version = "1.0.0" @@ -293,6 +304,16 @@ dependencies = [ "libc 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "rayon" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "deque 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "regex" version = "0.1.77" @@ -316,21 +337,6 @@ name = "rustc-serialize" version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "scoped-pool" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "crossbeam 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", - "scopeguard 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "variance 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "scopeguard" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "semver" version = "0.2.3" @@ -402,11 +408,6 @@ name = "utf8-ranges" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "variance" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "vec_map" version = "0.6.0" @@ -432,7 +433,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum clippy 0.0.90 (registry+https://github.com/rust-lang/crates.io-index)" = "d19bda68c3db98e3a780342f6101b44312fef20a5f13ce756d1202a35922b01b" "checksum clippy_lints 0.0.90 (registry+https://github.com/rust-lang/crates.io-index)" = "3d4ed67c69b9bb35169be2538691d290a3aa0cbfd4b9f0bfb7c221fc1d399a96" "checksum crc 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2a3f9159e74024e2cdb6f574e9117cdc2e91523a890930a022823d17abedc90a" -"checksum crossbeam 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "0c5ea215664ca264da8a9d9c3be80d2eaf30923c259d03e870388eb927508f97" +"checksum deque 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1614659040e711785ed8ea24219140654da1729f3ec8a47a9719d041112fe7bf" "checksum enum_primitive 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f79eff5be92a4d7d5bddf7daa7d650717ea71628634efe6ca7bcda85b2183c23" "checksum flate2 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "3eeb481e957304178d2e782f2da1257f1434dfecbae883bafb61ada2a9fea3bb" "checksum gcc 0.3.35 (registry+https://github.com/rust-lang/crates.io-index)" = "91ecd03771effb0c968fd6950b37e89476a578aaf1c70297d8e92b6516ec3312" @@ -454,16 +455,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum num-iter 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)" = "287a1c9969a847055e1122ec0ea7a5c5d6f72aad97934e131c83d5c08ab4e45c" "checksum num-rational 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)" = "54ff603b8334a72fbb27fe66948aac0abaaa40231b3cecd189e76162f6f38aaf" "checksum num-traits 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)" = "8359ea48994f253fa958b5b90b013728b06f54872e5a58bce39540fcdd0f2527" +"checksum num_cpus 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)" = "cee7e88156f3f9e19bdd598f8d6c9db7bf4078f99f8381f43a55b09648d1a6e3" "checksum num_cpus 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a859041cbf7a70ea1ece4b87d1a2c6ef364dcb68749c88db1f97304b9ec09d5f" "checksum pkg-config 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8cee804ecc7eaf201a4a207241472cc870e825206f6c031e3ee2a72fa425f2fa" "checksum png 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "06208e2ee243e3118a55dda9318f821f206d8563fb8d4df258767f8e62bb0997" "checksum quine-mc_cluskey 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "07589615d719a60c8dd8a4622e7946465dfef20d1a428f969e3443e7386d5f45" "checksum rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)" = "2791d88c6defac799c3f20d74f094ca33b9332612d9aef9078519c82e4fe04a5" +"checksum rayon 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "655df67c314c30fa3055a365eae276eb88aa4f3413a352a1ab32c1320eda41ea" "checksum regex 0.1.77 (registry+https://github.com/rust-lang/crates.io-index)" = "64b03446c466d35b42f2a8b203c8e03ed8b91c0f17b56e1f84f7210a257aa665" "checksum regex-syntax 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "279401017ae31cf4e15344aa3f085d0e2e5c1e70067289ef906906fdbe92c8fd" "checksum rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)" = "6159e4e6e559c81bd706afe9c8fd68f547d3e851ce12e76b1de7914bab61691b" -"checksum scoped-pool 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "817a3a15e704545ce59ed2b5c60a5d32bda4d7869befb8b36667b658a6c00b43" -"checksum scopeguard 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "59a076157c1e2dc561d8de585151ee6965d910dd4dcb5dabb7ae3e83981a6c57" "checksum semver 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2d5b7638a1f03815d94e88cb3b3c08e87f0db4d683ef499d1836aaf70a45623f" "checksum simd 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "63b5847c2d766ca7ce7227672850955802fabd779ba616aeabead4c2c3877023" "checksum strsim 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "50c069df92e4b01425a8bf3576d5d417943a6a7272fbabaf5bd80b1aaa76442e" @@ -475,7 +476,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum unicode-segmentation 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b905d0fc2a1f0befd86b0e72e31d1787944efef9d38b9358a9e92a69757f7e3b" "checksum unicode-width 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2d6722facc10989f63ee0e20a83cd4e1714a9ae11529403ac7e0afd069abc39e" "checksum utf8-ranges 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a1ca13c08c41c9c3e04224ed9ff80461d97e121589ff27c753a16cb10830ae0f" -"checksum variance 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3abfc2be1fb59663871379ea884fd81de80c496f2274e021c01d6fe56cd77b05" "checksum vec_map 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "cac5efe5cb0fa14ec2f84f83c701c562ee63f6dcc680861b21d65c682adfb05f" "checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" "checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" diff --git a/Cargo.toml b/Cargo.toml index 18bf7d83..e8879b04 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,7 @@ libc = "^0.2.4" libz-sys = "^1.0.0" miniz-sys = "^0.1.7" num_cpus = "^1.0.0" -scoped-pool = "^1.0.0" +rayon = "^0.4.2" [dependencies.clap] optional = true @@ -52,6 +52,6 @@ features = ["png_codec"] [features] default = ["binary"] binary = ["clap", "regex"] -nightly = [] +nightly = ["rayon/nightly"] nightly-binary = ["binary", "nightly", "regex/simd-accel"] dev = ["nightly-binary", "clippy"] diff --git a/src/lib.rs b/src/lib.rs index 5f2214cd..d7acfec5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,16 +10,15 @@ extern crate libc; extern crate libz_sys; extern crate miniz_sys; extern crate num_cpus; -extern crate scoped_pool; +extern crate rayon; use error::PngError; use headers::Headers; -use scoped_pool::Pool; +use rayon::prelude::*; use std::collections::{HashMap, HashSet}; use std::fs::{File, copy}; use std::io::{BufWriter, Write, stderr, stdout}; use std::path::{Path, PathBuf}; -use std::sync::Mutex; pub mod colors; pub mod deflate { @@ -277,6 +276,10 @@ impl Default for Options { /// Perform optimization on the input file using the options provided pub fn optimize(filepath: &Path, opts: &Options) -> Result<(), PngError> { + // Initialize the thread pool with correct number of threads + let thread_count = opts.threads; + rayon::initialize(rayon::Configuration::new().set_num_threads(thread_count)).ok(); + // Read in the file and try to decode as PNG. if opts.verbosity.is_some() { writeln!(&mut stderr(), "Processing: {}", filepath.to_str().unwrap()).ok(); @@ -396,6 +399,10 @@ pub fn optimize(filepath: &Path, opts: &Options) -> Result<(), PngError> { /// Perform optimization on the input file using the options provided, where the file is already /// loaded in-memory pub fn optimize_from_memory(data: &[u8], opts: &Options) -> Result, PngError> { + // Initialize the thread pool with correct number of threads + let thread_count = opts.threads; + rayon::initialize(rayon::Configuration::new().set_num_threads(thread_count)).ok(); + // Read in the file and try to decode as PNG. if opts.verbosity.is_some() { writeln!(&mut stderr(), "Processing from memory").ok(); @@ -481,74 +488,63 @@ fn optimize_png(mut png: &mut png::PngData, file_original_size: usize, opts: &Op let something_changed = perform_reductions(&mut png, opts); if opts.idat_recoding || something_changed { - let thread_count = opts.threads; - let pool = Pool::new(thread_count); // Go through selected permutations and determine the best - let best: Mutex> = Mutex::new(None); let combinations = filter.len() * compression.len() * memory.len() * strategies.len(); let mut results: Vec<(u8, u8, u8, u8)> = Vec::with_capacity(combinations); - let filters: Mutex>> = Mutex::new(HashMap::with_capacity(filter.len())); if opts.verbosity.is_some() { writeln!(&mut stderr(), "Trying: {} combinations", combinations).ok(); } - pool.scoped(|scope| { - for f in &filter { - let png = &png; - let filters = &filters; - for zc in compression { - for zm in memory { - for zs in &strategies { - results.push((*f, *zc, *zm, *zs)); - } + for f in &filter { + for zc in compression { + for zm in memory { + for zs in &strategies { + results.push((*f, *zc, *zm, *zs)); } } - scope.execute(move || { - let filtered = png.filter_image(*f); - let mut filters = filters.lock().unwrap(); - filters.insert(*f, filtered); - }); } - }); + } - let filters = filters.lock().unwrap(); + let mut filters_tmp: Vec<(u8, Vec)> = Vec::with_capacity(filter.len()); + filter.par_iter() + .weight_max() + .map(|f| (*f, png.filter_image(*f))) + .collect_into(&mut filters_tmp); - pool.scoped(|scope| { - let original_len = png.idat_data.len(); - let interlacing_changed = opts.interlace.is_some() && - opts.interlace != Some(png.ihdr_data.interlaced); - for trial in &results { + let filters: HashMap> = filters_tmp.into_iter().collect(); + + let original_len = png.idat_data.len(); + let interlacing_changed = opts.interlace.is_some() && + opts.interlace != Some(png.ihdr_data.interlaced); + + let best: Option = results.into_par_iter() + .weight_max() + .filter_map(|trial| { let filtered = filters.get(&trial.0).unwrap(); - let best = &best; - scope.execute(move || { - let new_idat = - deflate::deflate::deflate(filtered, trial.1, trial.2, trial.3, opts.window) - .unwrap(); + let new_idat = + deflate::deflate::deflate(filtered, trial.1, trial.2, trial.3, opts.window) + .unwrap(); - if opts.verbosity == Some(1) { - writeln!(&mut stderr(), - " zc = {} zm = {} zs = {} f = {} {} bytes", - trial.1, - trial.2, - trial.3, - trial.0, - new_idat.len()) - .ok(); - } + if opts.verbosity == Some(1) { + writeln!(&mut stderr(), + " zc = {} zm = {} zs = {} f = {} {} bytes", + trial.1, + trial.2, + trial.3, + trial.0, + new_idat.len()) + .ok(); + } - let mut best = best.lock().unwrap(); - if (best.is_some() && - new_idat.len() < best.as_ref().map(|x| x.4.len()).unwrap()) || - (best.is_none() && - (new_idat.len() < original_len || interlacing_changed || opts.force)) { - *best = Some((trial.0, trial.1, trial.2, trial.3, new_idat)); - } - }); - } - }); + if new_idat.len() < original_len || interlacing_changed || opts.force { + Some((trial.0, trial.1, trial.2, trial.3, new_idat)) + } else { + None + } + }) + .reduce_with(|i, j| if i.4.len() <= j.4.len() { i } else { j }); - let mut final_best = best.lock().unwrap(); - if let Some(better) = final_best.take() { + if let Some(better) = best { png.idat_data = better.4; if opts.verbosity.is_some() { writeln!(&mut stderr(), "Found better combination:").ok();