Replace current Mutex-based threading with Rayon
This commit is contained in:
Josh Holmer
parent
ce971cc665
commit
4977dd0948
4 changed files with 83 additions and 84 deletions
|
|
@ -1,3 +1,6 @@
|
|||
**Version 0.11.1 (unreleased)**
|
||||
- Performance optimizations
|
||||
|
||||
**Version 0.11.0**
|
||||
- [SEMVER_MAJOR] Bump minimum rustc version to 1.9.0, required by dependencies
|
||||
- [SEMVER_MINOR] Allow calling optimization presets via crate using `Options::from_preset`
|
||||
|
|
|
|||
54
Cargo.lock
generated
54
Cargo.lock
generated
|
|
@ -13,8 +13,8 @@ dependencies = [
|
|||
"libz-sys 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"miniz-sys 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"num_cpus 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rayon 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex 0.1.77 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"scoped-pool 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -91,9 +91,12 @@ dependencies = [
|
|||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam"
|
||||
version = "0.2.10"
|
||||
name = "deque"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "enum_primitive"
|
||||
|
|
@ -256,6 +259,14 @@ name = "num-traits"
|
|||
version = "0.1.35"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "num_cpus"
|
||||
version = "0.2.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"libc 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num_cpus"
|
||||
version = "1.0.0"
|
||||
|
|
@ -293,6 +304,16 @@ dependencies = [
|
|||
"libc 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon"
|
||||
version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"deque 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"num_cpus 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "0.1.77"
|
||||
|
|
@ -316,21 +337,6 @@ name = "rustc-serialize"
|
|||
version = "0.3.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "scoped-pool"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"crossbeam 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"scopeguard 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"variance 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "scopeguard"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "semver"
|
||||
version = "0.2.3"
|
||||
|
|
@ -402,11 +408,6 @@ name = "utf8-ranges"
|
|||
version = "0.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "variance"
|
||||
version = "0.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "vec_map"
|
||||
version = "0.6.0"
|
||||
|
|
@ -432,7 +433,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
"checksum clippy 0.0.90 (registry+https://github.com/rust-lang/crates.io-index)" = "d19bda68c3db98e3a780342f6101b44312fef20a5f13ce756d1202a35922b01b"
|
||||
"checksum clippy_lints 0.0.90 (registry+https://github.com/rust-lang/crates.io-index)" = "3d4ed67c69b9bb35169be2538691d290a3aa0cbfd4b9f0bfb7c221fc1d399a96"
|
||||
"checksum crc 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2a3f9159e74024e2cdb6f574e9117cdc2e91523a890930a022823d17abedc90a"
|
||||
"checksum crossbeam 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "0c5ea215664ca264da8a9d9c3be80d2eaf30923c259d03e870388eb927508f97"
|
||||
"checksum deque 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1614659040e711785ed8ea24219140654da1729f3ec8a47a9719d041112fe7bf"
|
||||
"checksum enum_primitive 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f79eff5be92a4d7d5bddf7daa7d650717ea71628634efe6ca7bcda85b2183c23"
|
||||
"checksum flate2 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "3eeb481e957304178d2e782f2da1257f1434dfecbae883bafb61ada2a9fea3bb"
|
||||
"checksum gcc 0.3.35 (registry+https://github.com/rust-lang/crates.io-index)" = "91ecd03771effb0c968fd6950b37e89476a578aaf1c70297d8e92b6516ec3312"
|
||||
|
|
@ -454,16 +455,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
"checksum num-iter 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)" = "287a1c9969a847055e1122ec0ea7a5c5d6f72aad97934e131c83d5c08ab4e45c"
|
||||
"checksum num-rational 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)" = "54ff603b8334a72fbb27fe66948aac0abaaa40231b3cecd189e76162f6f38aaf"
|
||||
"checksum num-traits 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)" = "8359ea48994f253fa958b5b90b013728b06f54872e5a58bce39540fcdd0f2527"
|
||||
"checksum num_cpus 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)" = "cee7e88156f3f9e19bdd598f8d6c9db7bf4078f99f8381f43a55b09648d1a6e3"
|
||||
"checksum num_cpus 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a859041cbf7a70ea1ece4b87d1a2c6ef364dcb68749c88db1f97304b9ec09d5f"
|
||||
"checksum pkg-config 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8cee804ecc7eaf201a4a207241472cc870e825206f6c031e3ee2a72fa425f2fa"
|
||||
"checksum png 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "06208e2ee243e3118a55dda9318f821f206d8563fb8d4df258767f8e62bb0997"
|
||||
"checksum quine-mc_cluskey 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "07589615d719a60c8dd8a4622e7946465dfef20d1a428f969e3443e7386d5f45"
|
||||
"checksum rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)" = "2791d88c6defac799c3f20d74f094ca33b9332612d9aef9078519c82e4fe04a5"
|
||||
"checksum rayon 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "655df67c314c30fa3055a365eae276eb88aa4f3413a352a1ab32c1320eda41ea"
|
||||
"checksum regex 0.1.77 (registry+https://github.com/rust-lang/crates.io-index)" = "64b03446c466d35b42f2a8b203c8e03ed8b91c0f17b56e1f84f7210a257aa665"
|
||||
"checksum regex-syntax 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "279401017ae31cf4e15344aa3f085d0e2e5c1e70067289ef906906fdbe92c8fd"
|
||||
"checksum rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)" = "6159e4e6e559c81bd706afe9c8fd68f547d3e851ce12e76b1de7914bab61691b"
|
||||
"checksum scoped-pool 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "817a3a15e704545ce59ed2b5c60a5d32bda4d7869befb8b36667b658a6c00b43"
|
||||
"checksum scopeguard 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "59a076157c1e2dc561d8de585151ee6965d910dd4dcb5dabb7ae3e83981a6c57"
|
||||
"checksum semver 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2d5b7638a1f03815d94e88cb3b3c08e87f0db4d683ef499d1836aaf70a45623f"
|
||||
"checksum simd 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "63b5847c2d766ca7ce7227672850955802fabd779ba616aeabead4c2c3877023"
|
||||
"checksum strsim 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "50c069df92e4b01425a8bf3576d5d417943a6a7272fbabaf5bd80b1aaa76442e"
|
||||
|
|
@ -475,7 +476,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
"checksum unicode-segmentation 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b905d0fc2a1f0befd86b0e72e31d1787944efef9d38b9358a9e92a69757f7e3b"
|
||||
"checksum unicode-width 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2d6722facc10989f63ee0e20a83cd4e1714a9ae11529403ac7e0afd069abc39e"
|
||||
"checksum utf8-ranges 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a1ca13c08c41c9c3e04224ed9ff80461d97e121589ff27c753a16cb10830ae0f"
|
||||
"checksum variance 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3abfc2be1fb59663871379ea884fd81de80c496f2274e021c01d6fe56cd77b05"
|
||||
"checksum vec_map 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "cac5efe5cb0fa14ec2f84f83c701c562ee63f6dcc680861b21d65c682adfb05f"
|
||||
"checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a"
|
||||
"checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc"
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ libc = "^0.2.4"
|
|||
libz-sys = "^1.0.0"
|
||||
miniz-sys = "^0.1.7"
|
||||
num_cpus = "^1.0.0"
|
||||
scoped-pool = "^1.0.0"
|
||||
rayon = "^0.4.2"
|
||||
|
||||
[dependencies.clap]
|
||||
optional = true
|
||||
|
|
@ -52,6 +52,6 @@ features = ["png_codec"]
|
|||
[features]
|
||||
default = ["binary"]
|
||||
binary = ["clap", "regex"]
|
||||
nightly = []
|
||||
nightly = ["rayon/nightly"]
|
||||
nightly-binary = ["binary", "nightly", "regex/simd-accel"]
|
||||
dev = ["nightly-binary", "clippy"]
|
||||
|
|
|
|||
106
src/lib.rs
106
src/lib.rs
|
|
@ -10,16 +10,15 @@ extern crate libc;
|
|||
extern crate libz_sys;
|
||||
extern crate miniz_sys;
|
||||
extern crate num_cpus;
|
||||
extern crate scoped_pool;
|
||||
extern crate rayon;
|
||||
|
||||
use error::PngError;
|
||||
use headers::Headers;
|
||||
use scoped_pool::Pool;
|
||||
use rayon::prelude::*;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fs::{File, copy};
|
||||
use std::io::{BufWriter, Write, stderr, stdout};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Mutex;
|
||||
|
||||
pub mod colors;
|
||||
pub mod deflate {
|
||||
|
|
@ -277,6 +276,10 @@ impl Default for Options {
|
|||
|
||||
/// Perform optimization on the input file using the options provided
|
||||
pub fn optimize(filepath: &Path, opts: &Options) -> Result<(), PngError> {
|
||||
// Initialize the thread pool with correct number of threads
|
||||
let thread_count = opts.threads;
|
||||
rayon::initialize(rayon::Configuration::new().set_num_threads(thread_count)).ok();
|
||||
|
||||
// Read in the file and try to decode as PNG.
|
||||
if opts.verbosity.is_some() {
|
||||
writeln!(&mut stderr(), "Processing: {}", filepath.to_str().unwrap()).ok();
|
||||
|
|
@ -396,6 +399,10 @@ pub fn optimize(filepath: &Path, opts: &Options) -> Result<(), PngError> {
|
|||
/// Perform optimization on the input file using the options provided, where the file is already
|
||||
/// loaded in-memory
|
||||
pub fn optimize_from_memory(data: &[u8], opts: &Options) -> Result<Vec<u8>, PngError> {
|
||||
// Initialize the thread pool with correct number of threads
|
||||
let thread_count = opts.threads;
|
||||
rayon::initialize(rayon::Configuration::new().set_num_threads(thread_count)).ok();
|
||||
|
||||
// Read in the file and try to decode as PNG.
|
||||
if opts.verbosity.is_some() {
|
||||
writeln!(&mut stderr(), "Processing from memory").ok();
|
||||
|
|
@ -481,74 +488,63 @@ fn optimize_png(mut png: &mut png::PngData, file_original_size: usize, opts: &Op
|
|||
let something_changed = perform_reductions(&mut png, opts);
|
||||
|
||||
if opts.idat_recoding || something_changed {
|
||||
let thread_count = opts.threads;
|
||||
let pool = Pool::new(thread_count);
|
||||
// Go through selected permutations and determine the best
|
||||
let best: Mutex<Option<TrialWithData>> = Mutex::new(None);
|
||||
let combinations = filter.len() * compression.len() * memory.len() * strategies.len();
|
||||
let mut results: Vec<(u8, u8, u8, u8)> = Vec::with_capacity(combinations);
|
||||
let filters: Mutex<HashMap<u8, Vec<u8>>> = Mutex::new(HashMap::with_capacity(filter.len()));
|
||||
if opts.verbosity.is_some() {
|
||||
writeln!(&mut stderr(), "Trying: {} combinations", combinations).ok();
|
||||
}
|
||||
|
||||
pool.scoped(|scope| {
|
||||
for f in &filter {
|
||||
let png = &png;
|
||||
let filters = &filters;
|
||||
for zc in compression {
|
||||
for zm in memory {
|
||||
for zs in &strategies {
|
||||
results.push((*f, *zc, *zm, *zs));
|
||||
}
|
||||
for f in &filter {
|
||||
for zc in compression {
|
||||
for zm in memory {
|
||||
for zs in &strategies {
|
||||
results.push((*f, *zc, *zm, *zs));
|
||||
}
|
||||
}
|
||||
scope.execute(move || {
|
||||
let filtered = png.filter_image(*f);
|
||||
let mut filters = filters.lock().unwrap();
|
||||
filters.insert(*f, filtered);
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let filters = filters.lock().unwrap();
|
||||
let mut filters_tmp: Vec<(u8, Vec<u8>)> = Vec::with_capacity(filter.len());
|
||||
filter.par_iter()
|
||||
.weight_max()
|
||||
.map(|f| (*f, png.filter_image(*f)))
|
||||
.collect_into(&mut filters_tmp);
|
||||
|
||||
pool.scoped(|scope| {
|
||||
let original_len = png.idat_data.len();
|
||||
let interlacing_changed = opts.interlace.is_some() &&
|
||||
opts.interlace != Some(png.ihdr_data.interlaced);
|
||||
for trial in &results {
|
||||
let filters: HashMap<u8, Vec<u8>> = filters_tmp.into_iter().collect();
|
||||
|
||||
let original_len = png.idat_data.len();
|
||||
let interlacing_changed = opts.interlace.is_some() &&
|
||||
opts.interlace != Some(png.ihdr_data.interlaced);
|
||||
|
||||
let best: Option<TrialWithData> = results.into_par_iter()
|
||||
.weight_max()
|
||||
.filter_map(|trial| {
|
||||
let filtered = filters.get(&trial.0).unwrap();
|
||||
let best = &best;
|
||||
scope.execute(move || {
|
||||
let new_idat =
|
||||
deflate::deflate::deflate(filtered, trial.1, trial.2, trial.3, opts.window)
|
||||
.unwrap();
|
||||
let new_idat =
|
||||
deflate::deflate::deflate(filtered, trial.1, trial.2, trial.3, opts.window)
|
||||
.unwrap();
|
||||
|
||||
if opts.verbosity == Some(1) {
|
||||
writeln!(&mut stderr(),
|
||||
" zc = {} zm = {} zs = {} f = {} {} bytes",
|
||||
trial.1,
|
||||
trial.2,
|
||||
trial.3,
|
||||
trial.0,
|
||||
new_idat.len())
|
||||
.ok();
|
||||
}
|
||||
if opts.verbosity == Some(1) {
|
||||
writeln!(&mut stderr(),
|
||||
" zc = {} zm = {} zs = {} f = {} {} bytes",
|
||||
trial.1,
|
||||
trial.2,
|
||||
trial.3,
|
||||
trial.0,
|
||||
new_idat.len())
|
||||
.ok();
|
||||
}
|
||||
|
||||
let mut best = best.lock().unwrap();
|
||||
if (best.is_some() &&
|
||||
new_idat.len() < best.as_ref().map(|x| x.4.len()).unwrap()) ||
|
||||
(best.is_none() &&
|
||||
(new_idat.len() < original_len || interlacing_changed || opts.force)) {
|
||||
*best = Some((trial.0, trial.1, trial.2, trial.3, new_idat));
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
if new_idat.len() < original_len || interlacing_changed || opts.force {
|
||||
Some((trial.0, trial.1, trial.2, trial.3, new_idat))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.reduce_with(|i, j| if i.4.len() <= j.4.len() { i } else { j });
|
||||
|
||||
let mut final_best = best.lock().unwrap();
|
||||
if let Some(better) = final_best.take() {
|
||||
if let Some(better) = best {
|
||||
png.idat_data = better.4;
|
||||
if opts.verbosity.is_some() {
|
||||
writeln!(&mut stderr(), "Found better combination:").ok();
|
||||
|
|
|
|||
Loading…
Reference in a new issue