From 1089754436391d1cd4e7e58ffe88aa3bdac07cac Mon Sep 17 00:00:00 2001 From: Joshua Holmer Date: Wed, 20 Apr 2016 22:49:47 -0400 Subject: [PATCH 1/2] Add inflate bench for testing different implementations [ci skip] --- benches/deflate.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/benches/deflate.rs b/benches/deflate.rs index 41499f30..ee033246 100644 --- a/benches/deflate.rs +++ b/benches/deflate.rs @@ -207,3 +207,13 @@ fn bench_deflate_1_bits_strategy_3(b: &mut Bencher) { deflate::deflate(png.raw_data.as_ref(), 9, 9, 3, 15).ok(); }); } + +#[bench] +fn bench_inflate_generic(b: &mut Bencher) { + let input = test::black_box(PathBuf::from("tests/files/rgb_16_should_be_rgb_16.png")); + let png = png::PngData::new(&input).unwrap(); + + b.iter(|| { + deflate::inflate(png.idat_data.as_ref()).ok(); + }); +} From f3b8665278f112543f36b63528875376faf49bfc Mon Sep 17 00:00:00 2001 From: Joshua Holmer Date: Wed, 20 Apr 2016 23:14:41 -0400 Subject: [PATCH 2/2] Use miniz for compression strategies 0 and 1 Worst case (RGB/A images) it performs similarly to zlib Best case (Indexed images) it performs 1-2 orders of magnitude better Zlib is kept for strategies 2 and 3, where it performs an order of magnitude better than miniz --- CHANGELOG.md | 3 + Cargo.toml | 1 + src/deflate/deflate.rs | 38 +++++-- src/deflate/{stream.rs => libz_stream.rs} | 0 src/deflate/miniz_stream.rs | 125 ++++++++++++++++++++++ src/lib.rs | 4 +- 6 files changed, 160 insertions(+), 11 deletions(-) rename src/deflate/{stream.rs => libz_stream.rs} (100%) create mode 100644 src/deflate/miniz_stream.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index c85e817a..158d0ed1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +**Version 0.5.1 (unreleased)** + - Use miniz for compression strategies where it outperforms zlib + **Version 0.5.0** - [SEMVER_MINOR] Palette entries can now reduced, on by default ([#11](https://github.com/shssoichiro/oxipng/issues/11)) - Don't report that we are in pretend mode if verbosity is set to none diff --git a/Cargo.toml b/Cargo.toml index 69360a49..5791540b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,7 @@ clap = "^2.2.5" crc = "^1.2.0" libc = "^0.2.4" libz-sys = "^1.0.0" +miniz-sys = "^0.1.7" num_cpus = "^0.2.11" regex = "^0.1.63" scoped-pool = "^0.1.8" diff --git a/src/deflate/deflate.rs b/src/deflate/deflate.rs index 312b3a33..7452d7e8 100644 --- a/src/deflate/deflate.rs +++ b/src/deflate/deflate.rs @@ -1,11 +1,12 @@ use libz_sys; +use miniz_sys; use libc::c_int; use std::cmp::max; /// Decompress a data stream using the DEFLATE algorithm pub fn inflate(data: &[u8]) -> Result, String> { let mut input = data.to_owned(); - let mut stream = super::stream::Stream::new_decompress(); + let mut stream = super::libz_stream::Stream::new_decompress(); let mut output = Vec::with_capacity(data.len()); loop { match stream.decompress_vec(input.as_mut(), output.as_mut()) { @@ -22,20 +23,37 @@ pub fn inflate(data: &[u8]) -> Result, String> { /// Compress a data stream using the zlib implementation of the DEFLATE algorithm pub fn deflate(data: &[u8], zc: u8, zm: u8, zs: u8, zw: u8) -> Result, String> { let mut input = data.to_owned(); - let mut stream = super::stream::Stream::new_compress(zc as c_int, - zw as c_int, - zm as c_int, - zs as c_int); // Compressed input should be smaller than decompressed, so allocate less than data.len() // However, it needs a minimum capacity in order to handle very small images let mut output = Vec::with_capacity(max(1024, data.len() / 20)); - loop { - match stream.compress_vec(input.as_mut(), output.as_mut()) { - libz_sys::Z_OK => output.reserve(max(1024, data.len() / 20)), - libz_sys::Z_STREAM_END => break, - c => return Err(format!("Error code on compress: {}", c)), + if zs == 0 || zs == 1 { + // Miniz performs 1-2 orders of magnitude better for strategies 0 and 1 + let mut stream = super::miniz_stream::Stream::new_compress(zc as c_int, + zw as c_int, + zm as c_int, + zs as c_int); + loop { + match stream.compress_vec(input.as_mut(), output.as_mut()) { + miniz_sys::MZ_OK => output.reserve(max(1024, data.len() / 20)), + miniz_sys::MZ_STREAM_END => break, + c => return Err(format!("Error code on compress: {}", c)), + } + } + } else { + // libz performs an order of magnitude better for strategies 2 and 3 + let mut stream = super::libz_stream::Stream::new_compress(zc as c_int, + zw as c_int, + zm as c_int, + zs as c_int); + loop { + match stream.compress_vec(input.as_mut(), output.as_mut()) { + libz_sys::Z_OK => output.reserve(max(1024, data.len() / 20)), + libz_sys::Z_STREAM_END => break, + c => return Err(format!("Error code on compress: {}", c)), + } } } + output.shrink_to_fit(); Ok(output) diff --git a/src/deflate/stream.rs b/src/deflate/libz_stream.rs similarity index 100% rename from src/deflate/stream.rs rename to src/deflate/libz_stream.rs diff --git a/src/deflate/miniz_stream.rs b/src/deflate/miniz_stream.rs new file mode 100644 index 00000000..5ccc359c --- /dev/null +++ b/src/deflate/miniz_stream.rs @@ -0,0 +1,125 @@ +// Raw un-exported bindings to miniz for encoding/decoding +// Copyright (c) 2014 Alex Crichton, MIT & Apache licenses +// Originally from flate2 crate +// Modified for use in oxipng + +use std::marker; +use std::mem; +use libc::{c_int, c_uint}; +use miniz_sys; + +pub struct Stream { + raw: miniz_sys::mz_stream, + _marker: marker::PhantomData, +} + +pub enum Compress {} +pub enum Decompress {} + +#[doc(hidden)] +pub trait Direction { + unsafe fn destroy(stream: *mut miniz_sys::mz_stream) -> c_int; +} + +impl Stream { + pub fn new_compress(lvl: c_int, + window_bits: c_int, + mem_size: c_int, + strategy: c_int) + -> Stream { + unsafe { + let mut state: miniz_sys::mz_stream = mem::zeroed(); + let ret = miniz_sys::mz_deflateInit2(&mut state, + lvl, + miniz_sys::MZ_DEFLATED, + window_bits, + mem_size, + strategy); + debug_assert_eq!(ret, 0); + Stream { + raw: state, + _marker: marker::PhantomData, + } + } + } + + pub fn new_decompress() -> Stream { + unsafe { + let mut state: miniz_sys::mz_stream = mem::zeroed(); + let ret = miniz_sys::mz_inflateInit2(&mut state, 15); + debug_assert_eq!(ret, 0); + Stream { + raw: state, + _marker: marker::PhantomData, + } + } + } +} + +impl Stream { + pub fn total_in(&self) -> usize { + self.raw.total_in as usize + } + + pub fn total_out(&self) -> usize { + self.raw.total_out as usize + } +} + +impl Stream { + pub fn decompress_vec(&mut self, input: &mut [u8], output: &mut Vec) -> c_int { + self.raw.avail_in = (input.len() - self.total_in()) as c_uint; + self.raw.avail_out = (output.capacity() - self.total_out()) as c_uint; + + unsafe { + self.raw.next_in = input.as_mut_ptr().offset(self.total_in() as isize); + self.raw.next_out = output.as_mut_ptr().offset(self.total_out() as isize); + let rc = miniz_sys::mz_inflate(&mut self.raw, miniz_sys::MZ_NO_FLUSH); + output.set_len(self.total_out() as usize); + rc + } + } +} + +impl Stream { + pub fn compress_vec(&mut self, input: &mut [u8], output: &mut Vec) -> c_int { + self.raw.avail_in = (input.len() - self.total_in() as usize) as c_uint; + self.raw.avail_out = (output.capacity() - self.total_out() as usize) as c_uint; + + unsafe { + self.raw.next_in = input.as_mut_ptr().offset(self.total_in() as isize); + self.raw.next_out = output.as_mut_ptr().offset(self.total_out() as isize); + let rc = miniz_sys::mz_deflate(&mut self.raw, + if self.raw.avail_in > 0 { + miniz_sys::MZ_NO_FLUSH + } else { + miniz_sys::MZ_FINISH + }); + output.set_len(self.total_out() as usize); + rc + } + } + + pub fn reset(&mut self) -> c_int { + unsafe { miniz_sys::mz_deflateReset(&mut self.raw) } + } +} + +impl Direction for Compress { + unsafe fn destroy(stream: *mut miniz_sys::mz_stream) -> c_int { + miniz_sys::mz_deflateEnd(stream) + } +} +impl Direction for Decompress { + unsafe fn destroy(stream: *mut miniz_sys::mz_stream) -> c_int { + miniz_sys::mz_inflateEnd(stream) + } +} + +impl Drop for Stream { + fn drop(&mut self) { + unsafe { + let _ = ::destroy(&mut self.raw); + } + } +} diff --git a/src/lib.rs b/src/lib.rs index 7b1c0625..4b690586 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,6 +3,7 @@ extern crate byteorder; extern crate crc; extern crate libc; extern crate libz_sys; +extern crate miniz_sys; extern crate num_cpus; extern crate scoped_pool; @@ -15,7 +16,8 @@ use std::sync::{Arc, Mutex}; pub mod deflate { pub mod deflate; - pub mod stream; + pub mod libz_stream; + pub mod miniz_stream; } pub mod png;