danvics/oxipng
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Limit number of simultaneous threads to help memory usage at high opt levels

Browse source 
Still uses a lot of memory. Would like to remove the storage of all compression
data attempts and calculate the best immediately during each thread.
This commit is contained in:
Joshua Holmer 2016-03-19 02:07:31 -04:00
parent aec8e412a3
commit 87e21c2020
2 changed files with 51 additions and 42 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
Cargo.toml
View file

@ -25,10 +25,11 @@ bit-vec = "^0.4.2"
byteorder = "^0.4.0"
clap = "^1.5.4"
crc = "^1.0.0"
crossbeam = "^0.2.1"
libc = "^0.2.4"
libz-sys = "^1.0.0"
num_cpus = "^0.2.11"
regex = "^0.1.8"
scoped-pool = "^0.1.8"
[dev-dependencies]
image = "^0.6.1"

90
src/lib.rs
View file

@ -1,10 +1,12 @@
extern crate bit_vec;
extern crate byteorder;
extern crate crc;
extern crate crossbeam;
extern crate libc;
extern crate libz_sys;
extern crate num_cpus;
extern crate scoped_pool;
use scoped_pool::Pool;
use std::collections::{HashMap, HashSet};
use std::fs::{File, copy};
use std::io::{BufWriter, Write, stderr, stdout};
@ -189,58 +191,64 @@ pub fn optimize(filepath: &Path, opts: &Options) -> Result<(), String> {
}
if opts.idat_recoding || something_changed {
// Use 1 thread on single-core, otherwise use threads = 1.5x CPU cores
let num_cpus = num_cpus::get();
let thread_count = num_cpus + (num_cpus >> 1);
let pool = Pool::new(thread_count);
// Go through selected permutations and determine the best
let mut best: Option<(u8, u8, u8, u8, Vec<u8>)> = None;
let combinations = filter.len() * compression.len() * memory.len() * strategies.len();
let mut results = Vec::with_capacity(combinations);
let mut results: Vec<(u8, u8, u8, u8, Vec<u8>)> = Vec::with_capacity(combinations);
let mut filters: HashMap<u8, Vec<u8>> = HashMap::with_capacity(filter.len());
if opts.verbosity.is_some() {
writeln!(&mut stderr(), "Trying: {} combinations", combinations).ok();
}
crossbeam::scope(|scope| {
for f in &filter {
let filtered = png.filter_image(*f);
for zc in &compression {
for zm in &memory {
for zs in &strategies {
let moved_filtered = filtered.clone();
results.push(scope.spawn(move || {
let new_idat = match deflate::deflate::deflate(&moved_filtered,
*zc,
*zm,
*zs,
opts.window) {
Ok(x) => x,
Err(x) => return Err(x),
};
if opts.verbosity == Some(1) {
writeln!(&mut stderr(), " zc = {} zm = {} zs = {} f = {} {} bytes",
*zc,
*zm,
*zs,
*f,
new_idat.len()).ok();
}
Ok((*f, *zc, *zm, *zs, new_idat.clone()))
}));
}
for f in &filter {
let filtered = png.filter_image(*f);
filters.insert(*f, filtered.clone());
for zc in &compression {
for zm in &memory {
for zs in &strategies {
results.push((*f, *zc, *zm, *zs, Vec::new()));
}
}
}
}
pool.scoped(|scope| {
for trial in &mut results {
let filtered = filters.get(&trial.0).unwrap();
scope.execute(move || {
let new_idat = deflate::deflate::deflate(filtered,
trial.1,
trial.2,
trial.3,
opts.window)
.unwrap();
if opts.verbosity == Some(1) {
writeln!(&mut stderr(),
" zc = {} zm = {} zs = {} f = {} {} bytes",
trial.1,
trial.2,
trial.3,
trial.0,
new_idat.len())
.ok();
}
trial.4 = new_idat;
});
}
});
for result in results {
if let Ok(ok_result) = result.join() {
if (best.is_some() &&
ok_result.4.len() < best.as_ref().map(|x| x.4.len()).unwrap()) ||
(best.is_none() &&
(ok_result.4.len() < png.idat_data.len() ||
(opts.interlace.is_some() &&
opts.interlace != Some(png.ihdr_data.interlaced)) ||
opts.force)) {
best = Some(ok_result);
}
for result in &results {
if (best.is_some() && result.4.len() < best.as_ref().map(|x| x.4.len()).unwrap()) ||
(best.is_none() &&
(result.4.len() < png.idat_data.len() ||
(opts.interlace.is_some() && opts.interlace != Some(png.ihdr_data.interlaced)) ||
opts.force)) {
best = Some(result.clone());
}
}