From 87e21c20209e18c55f0df6329f3a29c559fdddbd Mon Sep 17 00:00:00 2001 From: Joshua Holmer Date: Sat, 19 Mar 2016 02:07:31 -0400 Subject: [PATCH 1/3] Limit number of simultaneous threads to help memory usage at high opt levels Still uses a lot of memory. Would like to remove the storage of all compression data attempts and calculate the best immediately during each thread. --- Cargo.toml | 3 +- src/lib.rs | 90 +++++++++++++++++++++++++++++------------------------- 2 files changed, 51 insertions(+), 42 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index bd9b7d70..05f0e55c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,10 +25,11 @@ bit-vec = "^0.4.2" byteorder = "^0.4.0" clap = "^1.5.4" crc = "^1.0.0" -crossbeam = "^0.2.1" libc = "^0.2.4" libz-sys = "^1.0.0" +num_cpus = "^0.2.11" regex = "^0.1.8" +scoped-pool = "^0.1.8" [dev-dependencies] image = "^0.6.1" diff --git a/src/lib.rs b/src/lib.rs index 560abd85..64f6a51a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,10 +1,12 @@ extern crate bit_vec; extern crate byteorder; extern crate crc; -extern crate crossbeam; extern crate libc; extern crate libz_sys; +extern crate num_cpus; +extern crate scoped_pool; +use scoped_pool::Pool; use std::collections::{HashMap, HashSet}; use std::fs::{File, copy}; use std::io::{BufWriter, Write, stderr, stdout}; @@ -189,58 +191,64 @@ pub fn optimize(filepath: &Path, opts: &Options) -> Result<(), String> { } if opts.idat_recoding || something_changed { + // Use 1 thread on single-core, otherwise use threads = 1.5x CPU cores + let num_cpus = num_cpus::get(); + let thread_count = num_cpus + (num_cpus >> 1); + let pool = Pool::new(thread_count); // Go through selected permutations and determine the best let mut best: Option<(u8, u8, u8, u8, Vec)> = None; let combinations = filter.len() * compression.len() * memory.len() * strategies.len(); - let mut results = Vec::with_capacity(combinations); + let mut results: Vec<(u8, u8, u8, u8, Vec)> = Vec::with_capacity(combinations); + let mut filters: HashMap> = HashMap::with_capacity(filter.len()); if opts.verbosity.is_some() { writeln!(&mut stderr(), "Trying: {} combinations", combinations).ok(); } - crossbeam::scope(|scope| { - for f in &filter { - let filtered = png.filter_image(*f); - for zc in &compression { - for zm in &memory { - for zs in &strategies { - let moved_filtered = filtered.clone(); - results.push(scope.spawn(move || { - let new_idat = match deflate::deflate::deflate(&moved_filtered, - *zc, - *zm, - *zs, - opts.window) { - Ok(x) => x, - Err(x) => return Err(x), - }; - if opts.verbosity == Some(1) { - writeln!(&mut stderr(), " zc = {} zm = {} zs = {} f = {} {} bytes", - *zc, - *zm, - *zs, - *f, - new_idat.len()).ok(); - } - - Ok((*f, *zc, *zm, *zs, new_idat.clone())) - })); - } + for f in &filter { + let filtered = png.filter_image(*f); + filters.insert(*f, filtered.clone()); + for zc in &compression { + for zm in &memory { + for zs in &strategies { + results.push((*f, *zc, *zm, *zs, Vec::new())); } } } + } + pool.scoped(|scope| { + for trial in &mut results { + let filtered = filters.get(&trial.0).unwrap(); + scope.execute(move || { + let new_idat = deflate::deflate::deflate(filtered, + trial.1, + trial.2, + trial.3, + opts.window) + .unwrap(); + + if opts.verbosity == Some(1) { + writeln!(&mut stderr(), + " zc = {} zm = {} zs = {} f = {} {} bytes", + trial.1, + trial.2, + trial.3, + trial.0, + new_idat.len()) + .ok(); + } + + trial.4 = new_idat; + }); + } }); - for result in results { - if let Ok(ok_result) = result.join() { - if (best.is_some() && - ok_result.4.len() < best.as_ref().map(|x| x.4.len()).unwrap()) || - (best.is_none() && - (ok_result.4.len() < png.idat_data.len() || - (opts.interlace.is_some() && - opts.interlace != Some(png.ihdr_data.interlaced)) || - opts.force)) { - best = Some(ok_result); - } + for result in &results { + if (best.is_some() && result.4.len() < best.as_ref().map(|x| x.4.len()).unwrap()) || + (best.is_none() && + (result.4.len() < png.idat_data.len() || + (opts.interlace.is_some() && opts.interlace != Some(png.ihdr_data.interlaced)) || + opts.force)) { + best = Some(result.clone()); } } From 132a15403aff5bf58f8e5ed3a912f22af6ec70e5 Mon Sep 17 00:00:00 2001 From: Joshua Holmer Date: Sat, 19 Mar 2016 10:42:03 -0400 Subject: [PATCH 2/3] Don't store intermediate copies of compressed image data unless it is current best. Reduces peak memory usage significantly. --- src/lib.rs | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 64f6a51a..3f054a51 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,6 +11,7 @@ use std::collections::{HashMap, HashSet}; use std::fs::{File, copy}; use std::io::{BufWriter, Write, stderr, stdout}; use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; pub mod deflate { pub mod deflate; @@ -80,6 +81,8 @@ pub struct Options { /// Perform optimization on the input file using the options provided pub fn optimize(filepath: &Path, opts: &Options) -> Result<(), String> { + type TrialWithData = (u8, u8, u8, u8, Vec); + // Decode PNG from file if opts.verbosity.is_some() { writeln!(&mut stderr(), "Processing: {}", filepath.to_str().unwrap()).ok(); @@ -196,9 +199,9 @@ pub fn optimize(filepath: &Path, opts: &Options) -> Result<(), String> { let thread_count = num_cpus + (num_cpus >> 1); let pool = Pool::new(thread_count); // Go through selected permutations and determine the best - let mut best: Option<(u8, u8, u8, u8, Vec)> = None; + let best: Arc>> = Arc::new(Mutex::new(None)); let combinations = filter.len() * compression.len() * memory.len() * strategies.len(); - let mut results: Vec<(u8, u8, u8, u8, Vec)> = Vec::with_capacity(combinations); + let mut results: Vec<(u8, u8, u8, u8)> = Vec::with_capacity(combinations); let mut filters: HashMap> = HashMap::with_capacity(filter.len()); if opts.verbosity.is_some() { writeln!(&mut stderr(), "Trying: {} combinations", combinations).ok(); @@ -210,14 +213,18 @@ pub fn optimize(filepath: &Path, opts: &Options) -> Result<(), String> { for zc in &compression { for zm in &memory { for zs in &strategies { - results.push((*f, *zc, *zm, *zs, Vec::new())); + results.push((*f, *zc, *zm, *zs)); } } } } pool.scoped(|scope| { - for trial in &mut results { + let original_len = png.idat_data.len(); + let interlacing_changed = opts.interlace.is_some() && + opts.interlace != Some(png.ihdr_data.interlaced); + for trial in &results { let filtered = filters.get(&trial.0).unwrap(); + let best = best.clone(); scope.execute(move || { let new_idat = deflate::deflate::deflate(filtered, trial.1, @@ -237,22 +244,19 @@ pub fn optimize(filepath: &Path, opts: &Options) -> Result<(), String> { .ok(); } - trial.4 = new_idat; + let mut best = best.lock().unwrap(); + if (best.is_some() && + new_idat.len() < best.as_ref().map(|x| x.4.len()).unwrap()) || + (best.is_none() && + (new_idat.len() < original_len || interlacing_changed || opts.force)) { + *best = Some((trial.0, trial.1, trial.2, trial.3, new_idat)); + } }); } }); - for result in &results { - if (best.is_some() && result.4.len() < best.as_ref().map(|x| x.4.len()).unwrap()) || - (best.is_none() && - (result.4.len() < png.idat_data.len() || - (opts.interlace.is_some() && opts.interlace != Some(png.ihdr_data.interlaced)) || - opts.force)) { - best = Some(result.clone()); - } - } - - if let Some(better) = best { + let mut final_best = best.lock().unwrap(); + if let Some(better) = final_best.take() { png.idat_data = better.4.clone(); if opts.verbosity.is_some() { writeln!(&mut stderr(), "Found better combination:").ok(); From 622e2c7ca259ad5eed9b6b652b23bba48a1b9849 Mon Sep 17 00:00:00 2001 From: Joshua Holmer Date: Sat, 19 Mar 2016 10:47:59 -0400 Subject: [PATCH 3/3] Update changelog [ci skip] --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad8b4a61..79d79fab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +**Version 0.2.2 (unreleased)** + - Limit number of threads to 1.5x number of cores + - Significantly improve memory usage, especially with high optimization levels. ([#32](https://github.com/shssoichiro/oxipng/issues/32)) + **Version 0.2.1** - Add rustdoc for public methods and structs - Improve filter mode 5 heuristic ([#16](https://github.com/shssoichiro/oxipng/issues/16))