From e472c8287616795119e7b1b06282ddf72940c215 Mon Sep 17 00:00:00 2001 From: Kornel Date: Mon, 4 Jun 2018 20:39:51 +0100 Subject: [PATCH] Option to use Cloudflare's zlib for compression (#103) Behind the nightly feature flag for now--stable ASM is coming in Rust 1.27 --- Cargo.lock | 16 +++++++++++ Cargo.toml | 7 ++++- appveyor.yml | 5 +++- src/deflate/mod.rs | 66 ++++++++++++++++++++++++++++++++++++++++++++-- src/lib.rs | 7 ++++- src/png/mod.rs | 19 ++++++------- 6 files changed, 106 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b9260efe..a33b2938 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -72,6 +72,11 @@ name = "byteorder" version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "cc" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "cfg-if" version = "0.1.2" @@ -121,6 +126,14 @@ dependencies = [ "url 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "cloudflare-zlib-sys" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cc 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "crc" version = "1.8.1" @@ -348,6 +361,7 @@ dependencies = [ "byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)", "clippy 0.0.195 (registry+https://github.com/rust-lang/crates.io-index)", + "cloudflare-zlib-sys 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "crc 1.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "image 0.19.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -717,10 +731,12 @@ dependencies = [ "checksum build_const 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "39092a32794787acd8525ee150305ff051b0aa6cc2abaf193924f5ab05425f39" "checksum byteorder 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0fc10e8cc6b2580fda3f36eb6dc5316657f812a3df879a44a66fc9f0fdbc4855" "checksum byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "73b5bdfe7ee3ad0b99c9801d58807a9dbc9e09196365b0203853b99889ab3c87" +"checksum cc 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)" = "49ec142f5768efb5b7622aebc3fdbdbb8950a4b9ba996393cb76ef7466e8747d" "checksum cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d4c819a1287eb618df47cc647173c5c4c66ba19d888a6e50d605672aed3140de" "checksum clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f0f16b89cbb9ee36d87483dc939fe9f1e13c05898d56d7b230a0d4dff033a536" "checksum clippy 0.0.195 (registry+https://github.com/rust-lang/crates.io-index)" = "1d48c23f9c2f4cb3ac08fb972e9a892778f7829c7ae611106887b42cf1749ad2" "checksum clippy_lints 0.0.195 (registry+https://github.com/rust-lang/crates.io-index)" = "f490c98750106b5034c6b625b83b69539e42537783cb802220153f53f9a83ffa" +"checksum cloudflare-zlib-sys 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1a27574ebbf993003163c445f5f70d3fa9017f09013e5007889991f46aa85ea2" "checksum crc 1.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d663548de7f5cca343f1e0a48d14dcfb0e9eb4e079ec58883b7251539fa10aeb" "checksum crossbeam-deque 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f739f8c5363aca78cfb059edf753d8f0d36908c348f3d8d1503f03d8b75d9cf3" "checksum crossbeam-epoch 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "927121f5407de9956180ff5e936fe3cf4324279280001cd56b669d28ee7e9150" diff --git a/Cargo.toml b/Cargo.toml index 6055f8c4..e64cd7a2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,6 +45,10 @@ version = "^2.10.0" optional = true version = ">=0.0.85" +[dependencies.cloudflare-zlib-sys] +optional = true +version = "^0.1.2" + [dependencies.image] default-features = false features = ["png_codec"] @@ -60,11 +64,12 @@ binary = [ "regex", ] default = ["binary"] +cfzlib = ["cloudflare-zlib-sys"] dev = [ "nightly-binary", "clippy", ] -nightly = [] +nightly = ["cfzlib"] nightly-binary = [ "binary", "nightly", diff --git a/appveyor.yml b/appveyor.yml index 5e842a11..55c2ddae 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -8,6 +8,9 @@ environment: CHANNEL: stable - TARGET: x86_64-pc-windows-msvc CHANNEL: stable + - TARGET: x86_64-pc-windows-msvc + CHANNEL: nightly + TESTFLAGS: --features=cfzlib # Install Rust and Cargo # (Based on from https://github.com/rust-lang/libc/blob/master/appveyor.yml) @@ -26,7 +29,7 @@ build: false # Equivalent to Travis' `script` phase test_script: - cargo build - - cargo test + - cargo test %TESTFLAGS% before_deploy: # Generate artifacts for release diff --git a/src/deflate/mod.rs b/src/deflate/mod.rs index b42c9a17..4d85090b 100644 --- a/src/deflate/mod.rs +++ b/src/deflate/mod.rs @@ -13,8 +13,70 @@ pub fn inflate(data: &[u8]) -> Result, PngError> { } /// Compress a data stream using the DEFLATE algorithm -pub fn deflate(data: &[u8], zc: u8, zs: u8, zw: u8) -> Vec { - miniz_stream::compress_to_vec_oxipng(data, zc, zw.into(), zs.into()) +pub fn deflate(data: &[u8], zc: u8, zs: u8, zw: u8) -> Result, PngError> { + #[cfg(feature = "cfzlib")] + { + if is_cfzlib_supported() { + return cfzlib_deflate(data, zc, zs, zw) + } + } + + Ok(miniz_stream::compress_to_vec_oxipng(data, zc, zw.into(), zs.into())) +} + +#[cfg(feature = "cfzlib")] +fn is_cfzlib_supported() -> bool { + #[cfg(target_arch = "x86_64")] { + if is_x86_feature_detected!("sse4.2") && is_x86_feature_detected!("pclmulqdq") { + return true; + } + } + #[cfg(target_arch = "aarch64")] { + if is_arm_feature_detected!("neon") && is_arm_feature_detected!("crc") { + return true; + } + } + false +} + +#[cfg(feature = "cfzlib")] +pub fn cfzlib_deflate(data: &[u8], level: u8, strategy: u8, window_bits: u8) -> Result, PngError> { + use std::mem; + use cloudflare_zlib_sys::*; + + assert!(data.len() < u32::max_value() as usize); + unsafe { + let mut stream = mem::zeroed(); + if Z_OK != deflateInit2( + &mut stream, + level.into(), + Z_DEFLATED, + window_bits.into(), + MAX_MEM_LEVEL, + strategy.into()) { + return Err(PngError::new("deflateInit2")); + } + + let max_size = deflateBound(&mut stream, data.len() as uLong) as usize; + // it's important to have the capacity pre-allocated, + // as unsafe set_len is called later + let mut out = Vec::with_capacity(max_size); + + stream.next_in = data.as_ptr() as *mut _; + stream.total_in = data.len() as uLong; + stream.avail_in = data.len() as uInt; + stream.next_out = out.as_mut_ptr(); + stream.avail_out = out.capacity() as uInt; + if Z_STREAM_END != deflate(&mut stream, Z_FINISH) { + return Err(PngError::new("deflate")); + } + if Z_OK != deflateEnd(&mut stream) { + return Err(PngError::new("deflateEnd")); + } + debug_assert!(stream.total_out as usize <= out.capacity()); + out.set_len(stream.total_out as usize); + return Ok(out); + } } pub fn zopfli_deflate(data: &[u8]) -> Result, PngError> { diff --git a/src/lib.rs b/src/lib.rs index 439dffe9..a49eaf2a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,6 +10,8 @@ extern crate miniz_oxide; extern crate num_cpus; extern crate rayon; extern crate zopfli; +#[cfg(feature = "cfzlib")] +extern crate cloudflare_zlib_sys; use image::{DynamicImage, GenericImage, ImageFormat, Pixel}; use png::PngData; @@ -499,7 +501,10 @@ fn optimize_png( let new_idat = if opts.deflate == Deflaters::Zlib { deflate::deflate(filtered, trial.compression, trial.strategy, opts.window) } else { - deflate::zopfli_deflate(filtered).unwrap() + deflate::zopfli_deflate(filtered) + }; + let new_idat = if let Ok(n) = new_idat {n} else { + return None; }; if opts.verbosity == Some(1) { diff --git a/src/png/mod.rs b/src/png/mod.rs index 0cf6d7af..12fdb3e2 100644 --- a/src/png/mod.rs +++ b/src/png/mod.rs @@ -603,27 +603,28 @@ impl PngData { assert!(!alphas.is_empty()); let best = alphas .iter() - .map(|alpha| { + .filter_map(|alpha| { let mut image = self.clone(); image.reduce_alpha_channel(*alpha); - let size = STD_FILTERS + STD_FILTERS .iter() - .map(|f| { + .filter_map(|f| { deflate::deflate( &image.filter_image(*f), STD_COMPRESSION, STD_STRATEGY, STD_WINDOW, - ).len() + ).ok() + .as_ref().map(|l| l.len()) }) .min() - .unwrap(); - (size, image) + .map(|size| (size, image)) }) - .min_by_key(|&(size, _)| size) - .unwrap(); + .min_by_key(|&(size, _)| size); - self.raw_data = best.1.raw_data; + if let Some(best) = best { + self.raw_data = best.1.raw_data; + } } pub fn reduce_alpha_channel(&mut self, optim: AlphaOptim) -> bool {