Merge pull request #51 from shssoichiro/feature/18-Zopfli-Compression

Implement zopfli compression, available as a CLI flag (-Z)
This commit is contained in:
Josh Holmer 2016-12-01 13:18:48 -05:00 committed by GitHub
commit 5958efe958
11 changed files with 177 additions and 40 deletions

View file

@ -6,6 +6,8 @@
- [SEMVER_MAJOR] Bump minimum rustc version to 1.9.0, required by dependencies
- [SEMVER_MINOR] Allow calling optimization presets via crate using `Options::from_preset`
- [SEMVER_MAJOR] Return proper `PngError` type which implements `std::error::Error` from `Result`s
- [SEMVER_MAJOR] Rename module `deflate::deflate` to `deflate`
- [SEMVER_MINOR] Add zopfli option (`-Z`)
- Performance optimizations
**Version 0.10.0**

30
Cargo.lock generated
View file

@ -5,7 +5,7 @@ dependencies = [
"bit-vec 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
"byteorder 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
"clap 2.10.4 (registry+https://github.com/rust-lang/crates.io-index)",
"clippy 0.0.90 (registry+https://github.com/rust-lang/crates.io-index)",
"clippy 0.0.103 (registry+https://github.com/rust-lang/crates.io-index)",
"crc 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"image 0.10.3 (registry+https://github.com/rust-lang/crates.io-index)",
"itertools 0.4.18 (registry+https://github.com/rust-lang/crates.io-index)",
@ -15,8 +15,14 @@ dependencies = [
"num_cpus 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rayon 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
"regex 0.1.77 (registry+https://github.com/rust-lang/crates.io-index)",
"zopfli 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "adler32"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "aho-corasick"
version = "0.5.3"
@ -62,15 +68,15 @@ dependencies = [
[[package]]
name = "clippy"
version = "0.0.90"
version = "0.0.103"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"clippy_lints 0.0.90 (registry+https://github.com/rust-lang/crates.io-index)",
"clippy_lints 0.0.103 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "clippy_lints"
version = "0.0.90"
version = "0.0.103"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"matches 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
@ -423,15 +429,26 @@ name = "winapi-build"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "zopfli"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"adler32 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"byteorder 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
"crc 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[metadata]
"checksum adler32 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e928aa58f6dbd754bda26eca562a242549cb606e27a2240fc305fc75a7f12af9"
"checksum aho-corasick 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ca972c2ea5f742bfce5687b9aef75506a764f61d37f8f649047846a9686ddb66"
"checksum ansi_term 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c877397e09fec7a240af5fa74ad0124054b8066149d6544cd1ace93f8de3be68"
"checksum bit-vec 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "5b97c2c8e8bbb4251754f559df8af22fb264853c7d009084a576cdf12565089d"
"checksum bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "aad18937a628ec6abcd26d1489012cc0e18c21798210f491af69ded9b881106d"
"checksum byteorder 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0fc10e8cc6b2580fda3f36eb6dc5316657f812a3df879a44a66fc9f0fdbc4855"
"checksum clap 2.10.4 (registry+https://github.com/rust-lang/crates.io-index)" = "3df6dcb3122b085b96399062f4fa59d69f4d0af50519944f2d76b7a7686629e3"
"checksum clippy 0.0.90 (registry+https://github.com/rust-lang/crates.io-index)" = "d19bda68c3db98e3a780342f6101b44312fef20a5f13ce756d1202a35922b01b"
"checksum clippy_lints 0.0.90 (registry+https://github.com/rust-lang/crates.io-index)" = "3d4ed67c69b9bb35169be2538691d290a3aa0cbfd4b9f0bfb7c221fc1d399a96"
"checksum clippy 0.0.103 (registry+https://github.com/rust-lang/crates.io-index)" = "5b4fabf979ddf6419a313c1c0ada4a5b95cfd2049c56e8418d622d27b4b6ff32"
"checksum clippy_lints 0.0.103 (registry+https://github.com/rust-lang/crates.io-index)" = "ce96ec05bfe018a0d5d43da115e54850ea2217981ff0f2e462780ab9d594651a"
"checksum crc 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2a3f9159e74024e2cdb6f574e9117cdc2e91523a890930a022823d17abedc90a"
"checksum deque 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1614659040e711785ed8ea24219140654da1729f3ec8a47a9719d041112fe7bf"
"checksum enum_primitive 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f79eff5be92a4d7d5bddf7daa7d650717ea71628634efe6ca7bcda85b2183c23"
@ -479,3 +496,4 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum vec_map 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "cac5efe5cb0fa14ec2f84f83c701c562ee63f6dcc680861b21d65c682adfb05f"
"checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a"
"checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc"
"checksum zopfli 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "ff93d211a7a08b2925db981e297c0885d19436782409ee6c69c0bfae4b36b17b"

View file

@ -1,25 +1,22 @@
[package]
name = "oxipng"
version = "0.11.0"
authors = ["Joshua Holmer <jholmer.in@gmail.com>"]
description = "A lossless PNG compression optimizer"
license = "MIT"
documentation = "https://docs.rs/oxipng"
exclude = ["tests/*", "bench/*"]
homepage = "https://github.com/shssoichiro/oxipng"
license = "MIT"
name = "oxipng"
repository = "https://github.com/shssoichiro/oxipng"
exclude = [
"tests/*",
"bench/*",
]
version = "0.11.0"
[lib]
name = "oxipng"
path = "src/lib.rs"
[[bin]]
doc = false
name = "oxipng"
path = "src/main.rs"
doc = false
[dependencies]
bit-vec = "^0.4.2"
@ -31,6 +28,7 @@ libz-sys = "^1.0.0"
miniz-sys = "^0.1.7"
num_cpus = "^1.0.0"
rayon = "^0.4.2"
zopfli = "^0.3.4"
[dependencies.clap]
optional = true
@ -45,9 +43,9 @@ optional = true
version = "^0.1.63"
[dev-dependencies.image]
version = "^0.10.0"
default-features = false
features = ["png_codec"]
version = "^0.10.0"
[features]
default = ["binary"]

View file

@ -4,7 +4,7 @@ extern crate test;
extern crate oxipng;
use oxipng::png;
use oxipng::deflate::deflate;
use oxipng::deflate;
use test::Bencher;
use std::path::PathBuf;

59
benches/zopfli.rs Normal file
View file

@ -0,0 +1,59 @@
#![feature(test)]
extern crate test;
extern crate oxipng;
use oxipng::png;
use oxipng::deflate;
use test::Bencher;
use std::path::PathBuf;
#[bench]
fn bench_zopfli_16_bits_strategy_0(b: &mut Bencher) {
let input = test::black_box(PathBuf::from("tests/files/rgb_16_should_be_rgb_16.png"));
let png = png::PngData::new(&input, false).unwrap();
b.iter(|| {
deflate::zopfli_deflate(png.raw_data.as_ref()).ok();
});
}
#[bench]
fn bench_zopfli_8_bits_strategy_0(b: &mut Bencher) {
let input = test::black_box(PathBuf::from("tests/files/rgb_8_should_be_rgb_8.png"));
let png = png::PngData::new(&input, false).unwrap();
b.iter(|| {
deflate::zopfli_deflate(png.raw_data.as_ref()).ok();
});
}
#[bench]
fn bench_zopfli_4_bits_strategy_0(b: &mut Bencher) {
let input = test::black_box(PathBuf::from("tests/files/palette_4_should_be_palette_4.png"));
let png = png::PngData::new(&input, false).unwrap();
b.iter(|| {
deflate::zopfli_deflate(png.raw_data.as_ref()).ok();
});
}
#[bench]
fn bench_zopfli_2_bits_strategy_0(b: &mut Bencher) {
let input = test::black_box(PathBuf::from("tests/files/palette_2_should_be_palette_2.png"));
let png = png::PngData::new(&input, false).unwrap();
b.iter(|| {
deflate::zopfli_deflate(png.raw_data.as_ref()).ok();
});
}
#[bench]
fn bench_zopfli_1_bits_strategy_0(b: &mut Bencher) {
let input = test::black_box(PathBuf::from("tests/files/palette_1_should_be_palette_1.png"));
let png = png::PngData::new(&input, false).unwrap();
b.iter(|| {
deflate::zopfli_deflate(png.raw_data.as_ref()).ok();
});
}

View file

@ -3,11 +3,15 @@ use libz_sys;
use miniz_sys;
use libc::c_int;
use std::cmp::max;
use zopfli;
pub mod libz_stream;
pub mod miniz_stream;
/// Decompress a data stream using the DEFLATE algorithm
pub fn inflate(data: &[u8]) -> Result<Vec<u8>, PngError> {
let mut input = data.to_owned();
let mut stream = super::libz_stream::Stream::new_decompress();
let mut stream = libz_stream::Stream::new_decompress();
let mut output = Vec::with_capacity(data.len());
loop {
match stream.decompress_vec(input.as_mut(), output.as_mut()) {
@ -29,10 +33,8 @@ pub fn deflate(data: &[u8], zc: u8, zm: u8, zs: u8, zw: u8) -> Result<Vec<u8>, P
let mut output = Vec::with_capacity(max(1024, data.len() / 20));
if zs == 0 || zs == 1 {
// Miniz performs 1-2 orders of magnitude better for strategies 0 and 1
let mut stream = super::miniz_stream::Stream::new_compress(zc as c_int,
zw as c_int,
zm as c_int,
zs as c_int);
let mut stream =
miniz_stream::Stream::new_compress(zc as c_int, zw as c_int, zm as c_int, zs as c_int);
loop {
match stream.compress_vec(input.as_mut(), output.as_mut()) {
miniz_sys::MZ_OK => output.reserve(max(1024, data.len() / 20)),
@ -42,10 +44,8 @@ pub fn deflate(data: &[u8], zc: u8, zm: u8, zs: u8, zw: u8) -> Result<Vec<u8>, P
}
} else {
// libz performs an order of magnitude better for strategies 2 and 3
let mut stream = super::libz_stream::Stream::new_compress(zc as c_int,
zw as c_int,
zm as c_int,
zs as c_int);
let mut stream =
libz_stream::Stream::new_compress(zc as c_int, zw as c_int, zm as c_int, zs as c_int);
loop {
match stream.compress_vec(input.as_mut(), output.as_mut()) {
libz_sys::Z_OK => output.reserve(max(1024, data.len() / 20)),
@ -59,3 +59,23 @@ pub fn deflate(data: &[u8], zc: u8, zm: u8, zs: u8, zw: u8) -> Result<Vec<u8>, P
Ok(output)
}
pub fn zopfli_deflate(data: &[u8]) -> Result<Vec<u8>, PngError> {
let mut output = Vec::with_capacity(max(1024, data.len() / 20));
let options = zopfli::Options::default();
match zopfli::compress(&options, &zopfli::Format::Zlib, data, &mut output) {
Ok(_) => (),
Err(_) => return Err(PngError::new("Failed to compress in zopfli")),
};
output.shrink_to_fit();
Ok(output)
}
#[derive(Clone, Copy, Debug, PartialEq)]
/// DEFLATE algorithms supported by oxipng
pub enum Deflaters {
/// Use the Zlib DEFLATE implementation
Zlib,
/// Use the better but slower Zopfli implementation
Zopfli,
}

View file

@ -1,6 +1,5 @@
#![cfg_attr(feature="dev", feature(plugin))]
#![cfg_attr(feature="dev", plugin(clippy))]
#![cfg_attr(feature="dev", allow(module_inception))]
extern crate bit_vec;
extern crate byteorder;
@ -11,7 +10,9 @@ extern crate libz_sys;
extern crate miniz_sys;
extern crate num_cpus;
extern crate rayon;
extern crate zopfli;
use deflate::Deflaters;
use error::PngError;
use headers::Headers;
use rayon::prelude::*;
@ -21,11 +22,7 @@ use std::io::{BufWriter, Write, stderr, stdout};
use std::path::{Path, PathBuf};
pub mod colors;
pub mod deflate {
pub mod deflate;
pub mod libz_stream;
pub mod miniz_stream;
}
pub mod deflate;
mod error;
mod filters;
pub mod headers;
@ -108,6 +105,9 @@ pub struct Options {
/// Which headers to strip from the PNG file, if any
/// Default: `None`
pub strip: Headers,
/// Which DEFLATE algorithm to use
/// Default: `Zlib`
pub deflate: Deflaters,
/// Whether to use heuristics to pick the best filter and compression
/// Intended for use with `-o 1` from the CLI interface
/// Default: `false`
@ -268,6 +268,7 @@ impl Default for Options {
palette_reduction: true,
idat_recoding: true,
strip: Headers::None,
deflate: Deflaters::Zlib,
use_heuristics: false,
threads: thread_count,
}
@ -489,19 +490,28 @@ fn optimize_png(mut png: &mut png::PngData, file_original_size: usize, opts: &Op
if opts.idat_recoding || something_changed {
// Go through selected permutations and determine the best
let combinations = filter.len() * compression.len() * memory.len() * strategies.len();
let combinations = if opts.deflate == Deflaters::Zlib {
filter.len() * compression.len() * memory.len() * strategies.len()
} else {
filter.len()
};
let mut results: Vec<(u8, u8, u8, u8)> = Vec::with_capacity(combinations);
if opts.verbosity.is_some() {
writeln!(&mut stderr(), "Trying: {} combinations", combinations).ok();
}
for f in &filter {
for zc in compression {
for zm in memory {
for zs in &strategies {
results.push((*f, *zc, *zm, *zs));
if opts.deflate == Deflaters::Zlib {
for zc in compression {
for zm in memory {
for zs in &strategies {
results.push((*f, *zc, *zm, *zs));
}
}
}
} else {
// Zopfli compression has no additional options
results.push((*f, 0, 0, 0));
}
}
@ -521,9 +531,12 @@ fn optimize_png(mut png: &mut png::PngData, file_original_size: usize, opts: &Op
.weight_max()
.filter_map(|trial| {
let filtered = filters.get(&trial.0).unwrap();
let new_idat =
deflate::deflate::deflate(filtered, trial.1, trial.2, trial.3, opts.window)
.unwrap();
let new_idat = if opts.deflate == Deflaters::Zlib {
deflate::deflate(filtered, trial.1, trial.2, trial.3, opts.window)
} else {
deflate::zopfli_deflate(filtered)
}
.unwrap();
if opts.verbosity == Some(1) {
writeln!(&mut stderr(),

View file

@ -12,6 +12,7 @@ extern crate clap;
extern crate regex;
use clap::{App, Arg, ArgMatches};
use oxipng::deflate::Deflaters;
use oxipng::headers::Headers;
use oxipng::Options;
use regex::Regex;
@ -180,6 +181,10 @@ fn main() {
.help("Strip safely-removable metadata objects")
.short("s")
.conflicts_with("strip"))
.arg(Arg::with_name("zopfli")
.help("Use the slower but better compressing Zopfli algorithm, overrides zlib-specific options")
.short("Z")
.long("zopfli"))
.arg(Arg::with_name("threads")
.help("Set number of threads to use - default 1.5x CPU cores")
.long("threads")
@ -410,6 +415,10 @@ fn parse_opts_into_struct(matches: &ArgMatches) -> Result<Options, String> {
opts.strip = Headers::Safe;
}
if matches.is_present("zopfli") {
opts.deflate = Deflaters::Zopfli;
}
if let Some(x) = matches.value_of("threads") {
opts.threads = x.parse::<usize>().unwrap();
}

View file

@ -2,6 +2,7 @@ use bit_vec::BitVec;
use byteorder::{BigEndian, WriteBytesExt};
use colors::{BitDepth, ColorType};
use crc::crc32;
use deflate;
use error::PngError;
use filters::*;
use headers::*;
@ -236,7 +237,7 @@ impl PngData {
Ok(x) => x,
Err(x) => return Err(x),
};
let raw_data = match super::deflate::deflate::inflate(idat_headers.as_ref()) {
let raw_data = match deflate::inflate(idat_headers.as_ref()) {
Ok(x) => x,
Err(x) => return Err(x),
};

BIN
tests/files/zopfli_mode.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 112 KiB

View file

@ -4,6 +4,7 @@ extern crate oxipng;
use image::GenericImage;
use image::Pixel;
use oxipng::colors::{BitDepth, ColorType};
use oxipng::deflate::Deflaters;
use oxipng::headers::Headers;
use oxipng::png;
use std::collections::HashSet;
@ -530,3 +531,19 @@ fn issue_42() {
remove_file(output).ok();
}
#[test]
fn zopfli_mode() {
let input = PathBuf::from("tests/files/zopfli_mode.png");
let mut opts = get_opts(&input);
opts.deflate = Deflaters::Zopfli;
let output = opts.out_file.clone();
test_it_converts(&input,
&output,
&opts,
ColorType::RGB,
BitDepth::Eight,
ColorType::RGB,
BitDepth::Eight);
}