Add libdeflater as an option (#203)

libdeflater is a Rust wrapper around
[libdeflate](https://github.com/ebiggers/libdeflate) - an alternative
heavily optimised library for deflate/zlib/gzip compression and
decompression that is intended for situations where upper bounds of the
output are well-known.

In my benchmarks on test files in the repo it has shown to be usually
both slightly faster and providing better compressed output than
cloudflare-zlib, but in some cases showing the opposite, so rather
than swapping defaults, it's currently provided as another option,
similarly to zopfli.

Since it's not strictly better in all cases, I'm not providing median
numbers, but you can check distribution histograms for time and size
differences here (all using `oxipng -o 6 -t 6 -P`):
https://docs.google.com/spreadsheets/d/1WOKgeYZBhLkQvMGAC36snN4azilElzOFhx63RJu0EZY/edit?usp=sharing

Cargo.lock

@@ -218,6 +218,14 @@ name = "libc"
 version = "0.2.67"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "libdeflater"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cc 1.0.50 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "maybe-uninit"
 version = "2.0.0"
@@ -296,6 +304,7 @@ dependencies = [
  "crc 1.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "image 0.23.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "itertools 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libdeflater 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "miniz_oxide 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "num_cpus 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "rayon 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -464,6 +473,7 @@ dependencies = [
 "checksum itertools 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b"
 "checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 "checksum libc 0.2.67 (registry+https://github.com/rust-lang/crates.io-index)" = "eb147597cdf94ed43ab7a9038716637d2d1bf2bc571da995d0028dec06bd3018"
+"checksum libdeflater 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "66dca08b13369865b2f6dca1dd05f833985cbe6c12a676b04d55f78b85e80246"
 "checksum maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00"
 "checksum memoffset 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "75189eb85871ea5c2e2c15abbdd541185f63b408415e5051f5cac122d8c774b9"
 "checksum miniz_oxide 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "aa679ff6578b1cddee93d7e82e263b94a575e0bfced07284eb0c037c1d2416a5"

Cargo.toml

@@ -36,6 +36,7 @@ num_cpus = "^1.0.0"
 zopfli = "^0.4.0"
 miniz_oxide = "0.3"
 rgb = "0.8.11"
+libdeflater = "0.2.0"
 
 [dependencies.rayon]
 optional = true

benches/libdeflater.rs

@@ -0,0 +1,64 @@
+#![feature(test)]
+
+extern crate oxipng;
+extern crate test;
+
+use oxipng::internal_tests::*;
+use std::path::PathBuf;
+use test::Bencher;
+
+#[bench]
+fn libdeflater_16_bits_strategy_0(b: &mut Bencher) {
+    let input = test::black_box(PathBuf::from("tests/files/rgb_16_should_be_rgb_16.png"));
+    let png = PngData::new(&input, false).unwrap();
+
+    b.iter(|| {
+        libdeflater_deflate(png.raw.data.as_ref()).ok();
+    });
+}
+
+#[bench]
+fn libdeflater_8_bits_strategy_0(b: &mut Bencher) {
+    let input = test::black_box(PathBuf::from("tests/files/rgb_8_should_be_rgb_8.png"));
+    let png = PngData::new(&input, false).unwrap();
+
+    b.iter(|| {
+        libdeflater_deflate(png.raw.data.as_ref()).ok();
+    });
+}
+
+#[bench]
+fn libdeflater_4_bits_strategy_0(b: &mut Bencher) {
+    let input = test::black_box(PathBuf::from(
+        "tests/files/palette_4_should_be_palette_4.png",
+    ));
+    let png = PngData::new(&input, false).unwrap();
+
+    b.iter(|| {
+        libdeflater_deflate(png.raw.data.as_ref()).ok();
+    });
+}
+
+#[bench]
+fn libdeflater_2_bits_strategy_0(b: &mut Bencher) {
+    let input = test::black_box(PathBuf::from(
+        "tests/files/palette_2_should_be_palette_2.png",
+    ));
+    let png = PngData::new(&input, false).unwrap();
+
+    b.iter(|| {
+        libdeflater_deflate(png.raw.data.as_ref()).ok();
+    });
+}
+
+#[bench]
+fn libdeflater_1_bits_strategy_0(b: &mut Bencher) {
+    let input = test::black_box(PathBuf::from(
+        "tests/files/palette_1_should_be_palette_1.png",
+    ));
+    let png = PngData::new(&input, false).unwrap();
+
+    b.iter(|| {
+        libdeflater_deflate(png.raw.data.as_ref()).ok();
+    });
+}

src/deflate/deflater.rs

@@ -0,0 +1,31 @@
+use crate::{PngError, PngResult};
+use crate::atomicmin::AtomicMin;
+use libdeflater::{CompressionError, CompressionLvl, Compressor};
+
+pub fn deflate(data: &[u8], max_size: &AtomicMin) -> PngResult<Vec<u8>> {
+    let mut compressor = Compressor::new(CompressionLvl::best());
+    let capacity = max_size.get().unwrap_or(data.len() / 2);
+    let mut dest = Vec::with_capacity(capacity);
+    unsafe {
+        // This is ok because the Vec contains Copy-able data (u8)
+        // and because libdeflater wrapper doesn't try to read
+        // the bytes from the target.
+        //
+        // That said, it should be able to accept MaybeUninit instead,
+        // so I raised an upstream issue that should make this safer:
+        // https://github.com/adamkewley/libdeflater/issues/1
+        dest.set_len(capacity);
+    }
+    let len = compressor
+        .zlib_compress(data, &mut dest)
+        .map_err(|err| match err {
+            CompressionError::InsufficientSpace => PngError::DeflatedDataTooLong(capacity),
+        })?;
+    if let Some(max) = max_size.get() {
+        if len > max {
+            return Err(PngError::DeflatedDataTooLong(max));
+        }
+    }
+    dest.truncate(len);
+    Ok(dest)
+}

src/deflate/mod.rs

@@ -9,6 +9,9 @@ use zopfli;
 #[doc(hidden)]
 pub mod miniz_stream;
 
+mod deflater;
+pub use deflater::deflate as libdeflater_deflate;
+
 #[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))]
 pub mod cfzlib;
 
@@ -62,4 +65,6 @@ pub enum Deflaters {
     Zlib,
     /// Use the better but slower Zopfli implementation
     Zopfli,
+    /// Use libdeflater.
+    Libdeflater,
 }

src/lib.rs

@@ -353,6 +353,7 @@ pub fn optimize(input: &InFile, output: &OutFile, opts: &Options) -> PngResult<(
             data
         }
     };
+
     let mut png = PngData::from_slice(&in_data, opts.fix_errors)?;
 
     // Run the optimizer on the decoded PNG.
@@ -565,7 +566,7 @@ fn optimize_png(
                     }
                 }
             } else {
-                // Zopfli compression has no additional options
+                // Zopfli and Libdeflater compression have no additional options.
                 results.push(TrialOptions {
                     filter: *f,
                     compression: 0,
@@ -600,17 +601,17 @@ fn optimize_png(
                 return None;
             }
             let filtered = &filters[&trial.filter];
-            let new_idat = if opts.deflate == Deflaters::Zlib {
-                deflate::deflate(
+            let new_idat = match opts.deflate {
+                Deflaters::Zlib => deflate::deflate(
                     filtered,
                     trial.compression,
                     trial.strategy,
                     opts.window,
                     &best_size,
                     &deadline,
-                )
-            } else {
-                deflate::zopfli_deflate(filtered)
+                ),
+                Deflaters::Zopfli => deflate::zopfli_deflate(filtered),
+                Deflaters::Libdeflater => deflate::libdeflater_deflate(filtered, &best_size),
             };
 
             let new_idat = match new_idat {

src/main.rs

@@ -193,6 +193,10 @@ fn main() {
             .help("Use the slower but better compressing Zopfli algorithm, overrides zlib-specific options")
             .short("Z")
             .long("zopfli"))
+        .arg(Arg::with_name("libdeflater")
+            .help("Use an alternative Libdeflater algorithm, overrides zlib-specific options")
+            .short("D")
+            .long("libdeflater"))
         .arg(Arg::with_name("timeout")
             .help("Maximum amount of time, in seconds, to spend on optimizations")
             .takes_value(true)
@@ -481,6 +485,10 @@ fn parse_opts_into_struct(
         opts.deflate = Deflaters::Zopfli;
     }
 
+    if matches.is_present("libdeflater") {
+        opts.deflate = Deflaters::Libdeflater;
+    }
+
     if let Some(x) = matches.value_of("threads") {
         opts.threads = x.parse::<usize>().unwrap();
     }

tests/flags.rs

@@ -438,3 +438,21 @@ fn zopfli_mode() {
         BitDepth::Eight,
     );
 }
+
+
+#[test]
+fn libdeflater_mode() {
+    let input = PathBuf::from("tests/files/zopfli_mode.png");
+    let (output, mut opts) = get_opts(&input);
+    opts.deflate = Deflaters::Libdeflater;
+
+    test_it_converts(
+        input,
+        &output,
+        &opts,
+        ColorType::RGB,
+        BitDepth::Eight,
+        ColorType::RGB,
+        BitDepth::Eight,
+    );
+}