Limit number of simultaneous threads to help memory usage at high opt levels

Still uses a lot of memory. Would like to remove the storage of all compression
data attempts and calculate the best immediately during each thread.

Cargo.toml

@@ -25,10 +25,11 @@ bit-vec = "^0.4.2"
 byteorder = "^0.4.0"
 clap = "^1.5.4"
 crc = "^1.0.0"
-crossbeam = "^0.2.1"
 libc = "^0.2.4"
 libz-sys = "^1.0.0"
+num_cpus = "^0.2.11"
 regex = "^0.1.8"
+scoped-pool = "^0.1.8"
 
 [dev-dependencies]
 image = "^0.6.1"

src/lib.rs

@@ -1,10 +1,12 @@
 extern crate bit_vec;
 extern crate byteorder;
 extern crate crc;
-extern crate crossbeam;
 extern crate libc;
 extern crate libz_sys;
+extern crate num_cpus;
+extern crate scoped_pool;
 
+use scoped_pool::Pool;
 use std::collections::{HashMap, HashSet};
 use std::fs::{File, copy};
 use std::io::{BufWriter, Write, stderr, stdout};
@@ -189,58 +191,64 @@ pub fn optimize(filepath: &Path, opts: &Options) -> Result<(), String> {
     }
 
     if opts.idat_recoding || something_changed {
+        // Use 1 thread on single-core, otherwise use threads = 1.5x CPU cores
+        let num_cpus = num_cpus::get();
+        let thread_count = num_cpus + (num_cpus >> 1);
+        let pool = Pool::new(thread_count);
         // Go through selected permutations and determine the best
         let mut best: Option<(u8, u8, u8, u8, Vec<u8>)> = None;
         let combinations = filter.len() * compression.len() * memory.len() * strategies.len();
-        let mut results = Vec::with_capacity(combinations);
+        let mut results: Vec<(u8, u8, u8, u8, Vec<u8>)> = Vec::with_capacity(combinations);
+        let mut filters: HashMap<u8, Vec<u8>> = HashMap::with_capacity(filter.len());
         if opts.verbosity.is_some() {
             writeln!(&mut stderr(), "Trying: {} combinations", combinations).ok();
         }
-        crossbeam::scope(|scope| {
-            for f in &filter {
-                let filtered = png.filter_image(*f);
-                for zc in &compression {
-                    for zm in &memory {
-                        for zs in &strategies {
-                            let moved_filtered = filtered.clone();
-                            results.push(scope.spawn(move || {
-                                let new_idat = match deflate::deflate::deflate(&moved_filtered,
-                                                                               *zc,
-                                                                               *zm,
-                                                                               *zs,
-                                                                               opts.window) {
-                                    Ok(x) => x,
-                                    Err(x) => return Err(x),
-                                };
 
-                                if opts.verbosity == Some(1) {
+        for f in &filter {
-                                    writeln!(&mut stderr(), "    zc = {}  zm = {}  zs = {}  f = {}        {} bytes",
+            let filtered = png.filter_image(*f);
-                                             *zc,
+            filters.insert(*f, filtered.clone());
-                                             *zm,
+            for zc in &compression {
-                                             *zs,
+                for zm in &memory {
-                                             *f,
+                    for zs in &strategies {
-                                             new_idat.len()).ok();
+                        results.push((*f, *zc, *zm, *zs, Vec::new()));
-                                }
-
-                                Ok((*f, *zc, *zm, *zs, new_idat.clone()))
-                            }));
-                        }
                     }
                 }
             }
+        }
+        pool.scoped(|scope| {
+            for trial in &mut results {
+                let filtered = filters.get(&trial.0).unwrap();
+                scope.execute(move || {
+                    let new_idat = deflate::deflate::deflate(filtered,
+                                                             trial.1,
+                                                             trial.2,
+                                                             trial.3,
+                                                             opts.window)
+                                       .unwrap();
+
+                    if opts.verbosity == Some(1) {
+                        writeln!(&mut stderr(),
+                                 "    zc = {}  zm = {}  zs = {}  f = {}        {} bytes",
+                                 trial.1,
+                                 trial.2,
+                                 trial.3,
+                                 trial.0,
+                                 new_idat.len())
+                            .ok();
+                    }
+
+                    trial.4 = new_idat;
+                });
+            }
         });
 
-        for result in results {
+        for result in &results {
-            if let Ok(ok_result) = result.join() {
+            if (best.is_some() && result.4.len() < best.as_ref().map(|x| x.4.len()).unwrap()) ||
-                if (best.is_some() &&
+               (best.is_none() &&
-                    ok_result.4.len() < best.as_ref().map(|x| x.4.len()).unwrap()) ||
+                (result.4.len() < png.idat_data.len() ||
-                   (best.is_none() &&
+                 (opts.interlace.is_some() && opts.interlace != Some(png.ihdr_data.interlaced)) ||
-                    (ok_result.4.len() < png.idat_data.len() ||
+                 opts.force)) {
-                     (opts.interlace.is_some() &&
+                best = Some(result.clone());
-                      opts.interlace != Some(png.ihdr_data.interlaced)) ||
-                     opts.force)) {
-                    best = Some(ok_result);
-                }
             }
         }