danvics/oxipng
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Tidy up

Browse source
This commit is contained in:
Andrew 2022-10-23 09:58:14 +13:00
parent 460b9883cf
commit 3e34111ebf
11 changed files with 39 additions and 319 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

34
Cargo.lock generated
View file

@ -91,24 +91,6 @@ dependencies = [
"os_str_bytes",
]
[[package]]
name = "cloudflare-zlib"
version = "0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2cfcefb5df07f146eb15756342a135eb7d76b8bb609eff9c111f7539d060f94d"
dependencies = [
"cloudflare-zlib-sys",
]
[[package]]
name = "cloudflare-zlib-sys"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2040b6d1edfee6d75f172d81e2d2a7807534f3f294ce18184c70e7bb0105cd6f"
dependencies = [
"cc",
]
[[package]]
name = "color_quant"
version = "1.1.0"
@ -209,7 +191,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6"
dependencies = [
"crc32fast",
"miniz_oxide 0.5.4",
"miniz_oxide",
]
[[package]]
@ -324,15 +306,6 @@ dependencies = [
"adler",
]
[[package]]
name = "miniz_oxide"
version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa"
dependencies = [
"adler",
]
[[package]]
name = "num-integer"
version = "0.1.45"
@ -391,8 +364,6 @@ version = "6.0.1"
dependencies = [
"bit-vec",
"clap",
"cloudflare-zlib",
"crc",
"crossbeam-channel",
"filetime",
"image",
@ -400,7 +371,6 @@ dependencies = [
"itertools",
"libdeflater",
"log",
"miniz_oxide 0.6.2",
"rayon",
"rgb",
"rustc_version",
@ -418,7 +388,7 @@ dependencies = [
"bitflags",
"crc32fast",
"flate2",
"miniz_oxide 0.5.4",
"miniz_oxide",
]
[[package]]

10
Cargo.toml
View file

@ -24,13 +24,11 @@ required-features = ["binary"]
[dependencies]
bit-vec = "0.6.3"
crc = "3.0.0"
itertools = "0.10.3"
zopfli = { version = "0.7.1", optional = true }
miniz_oxide = "0.6.2"
rgb = "0.8.33"
indexmap = "1.9.1"
libdeflater = { version = "0.11.0", optional = true }
libdeflater = "0.11.0"
log = "0.4.17"
stderrlog = { version = "0.5.3", optional = true, default-features = false }
crossbeam-channel = "0.5.6"
@ -56,16 +54,12 @@ default-features = false
features = ["png"]
version = "0.24.3"
[target.'cfg(any(target_arch = "x86_64", target_arch = "aarch64"))'.dependencies.cloudflare-zlib]
features = ["arm-always"]
version = "0.2.9"
[build-dependencies]
rustc_version = "0.4.0"
[features]
binary = ["clap", "wild", "stderrlog"]
default = ["binary", "filetime", "parallel", "libdeflater", "zopfli"]
default = ["binary", "filetime", "parallel", "zopfli"]
parallel = ["rayon", "indexmap/rayon"]
[lib]

54
src/deflate/cfzlib.rs
View file

@ -1,54 +0,0 @@
use crate::atomicmin::AtomicMin;
use crate::Deadline;
use crate::PngError;
use crate::PngResult;
pub use cloudflare_zlib::is_supported;
use cloudflare_zlib::*;
impl From<ZError> for PngError {
fn from(err: ZError) -> Self {
match err {
ZError::DeflatedDataTooLarge(n) => PngError::DeflatedDataTooLong(n),
other => PngError::Other(other.to_string().into()),
}
}
}
pub(crate) fn cfzlib_deflate(
data: &[u8],
level: u8,
strategy: u8,
window_bits: u8,
max_size: &AtomicMin,
deadline: &Deadline,
) -> PngResult<Vec<u8>> {
let mut stream = Deflate::new(level.into(), strategy.into(), window_bits.into())?;
stream.reserve(max_size.get().unwrap_or(data.len() / 2));
let max_size = max_size.as_atomic_usize();
// max size is generally checked after each split,
// so splitting the buffer into pieces gives more checks
// = better chance of hitting it sooner.
let chunk_size = (data.len() / 4).max(1 << 15).min(1 << 18); // 32-256KB
for chunk in data.chunks(chunk_size) {
stream.compress_with_limit(chunk, max_size)?;
if deadline.passed() {
return Err(PngError::TimedOut);
}
}
Ok(stream.finish()?)
}
#[test]
fn compress_test() {
let vec = cfzlib_deflate(
b"azxcvbnm",
Z_BEST_COMPRESSION as u8,
Z_DEFAULT_STRATEGY as u8,
15,
&AtomicMin::new(None),
&Deadline::new(None),
)
.unwrap();
let res = crate::deflate::inflate(&vec).unwrap();
assert_eq!(&res, b"azxcvbnm");
}

71
src/deflate/miniz_stream.rs
View file

@ -1,71 +0,0 @@
use crate::atomicmin::AtomicMin;
use crate::error::PngError;
use crate::PngResult;
use miniz_oxide::deflate::core::*;
pub(crate) fn compress_to_vec_oxipng(
input: &[u8],
level: u8,
window_bits: i32,
strategy: i32,
max_size: &AtomicMin,
deadline: &crate::Deadline,
) -> PngResult<Vec<u8>> {
// The comp flags function sets the zlib flag if the window_bits parameter is > 0.
let flags = create_comp_flags_from_zip_params(level.into(), window_bits, strategy);
let mut compressor = CompressorOxide::new(flags);
// if max size is known, then expect that much data (but no more than input.len())
let mut output = Vec::with_capacity(max_size.get().unwrap_or(input.len() / 2).min(input.len()));
// # Unsafe
// We trust compress to not read the uninitialized bytes.
unsafe {
let cap = output.capacity();
output.set_len(cap);
}
let mut in_pos = 0;
let mut out_pos = 0;
loop {
let (status, bytes_in, bytes_out) = compress(
&mut compressor,
&input[in_pos..],
&mut output[out_pos..],
TDEFLFlush::Finish,
);
out_pos += bytes_out;
in_pos += bytes_in;
match status {
TDEFLStatus::Done => {
output.truncate(out_pos);
break;
}
TDEFLStatus::Okay => {
if let Some(max) = max_size.get() {
if output.len() > max {
return Err(PngError::DeflatedDataTooLong(output.len()));
}
}
if deadline.passed() {
return Err(PngError::TimedOut);
}
// We need more space, so extend the vector.
if output.len().saturating_sub(out_pos) < 30 {
let current_len = output.len();
output.reserve(current_len);
// # Unsafe
// We trust compress to not read the uninitialized bytes.
unsafe {
let cap = output.capacity();
output.set_len(cap);
}
}
}
// Not supposed to happen unless there is a bug.
_ => panic!("Bug! Unexpectedly failed to compress!"),
}
}
Ok(output)
}

79
src/deflate/mod.rs
View file

@ -1,64 +1,14 @@
use crate::atomicmin::AtomicMin;
use crate::error::PngError;
use crate::Deadline;
use crate::PngResult;
use indexmap::IndexSet;
#[cfg(feature = "zopfli")]
use std::num::NonZeroU8;
#[doc(hidden)]
pub mod miniz_stream;
#[cfg(feature = "libdeflater")]
mod deflater;
#[cfg(feature = "libdeflater")]
pub use deflater::crc32;
#[cfg(feature = "libdeflater")]
pub use deflater::deflate as libdeflater_deflate;
#[cfg(feature = "libdeflater")]
pub use deflater::inflate as libdeflater_inflate;
#[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))]
pub mod cfzlib;
#[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))]
pub mod cfzlib {
pub fn is_supported() -> bool {
return false;
}
}
/// Decompress a data stream using the DEFLATE algorithm
pub fn inflate(data: &[u8]) -> PngResult<Vec<u8>> {
miniz_oxide::inflate::decompress_to_vec_zlib(data).map_err(|e| {
PngError::new(&format!(
"Error on decompress: {:?} (after {:?} decompressed bytes)",
e.status,
e.output.len()
))
})
}
/// Compress a data stream using the DEFLATE algorithm
#[doc(hidden)]
pub fn deflate(
data: &[u8],
zc: u8,
zs: u8,
zw: u8,
max_size: &AtomicMin,
deadline: &Deadline,
) -> PngResult<Vec<u8>> {
#[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))]
{
if cfzlib::is_supported() {
return cfzlib::cfzlib_deflate(data, zc, zs, zw, max_size, deadline);
}
}
miniz_stream::compress_to_vec_oxipng(data, zc, zw.into(), zs.into(), max_size, deadline)
}
pub use deflater::deflate;
pub use deflater::inflate;
#[cfg(feature = "zopfli")]
pub fn zopfli_deflate(data: &[u8], iterations: NonZeroU8) -> PngResult<Vec<u8>> {
@ -80,23 +30,10 @@ pub fn zopfli_deflate(data: &[u8], iterations: NonZeroU8) -> PngResult<Vec<u8>>
#[derive(Clone, Debug, PartialEq, Eq)]
/// DEFLATE algorithms supported by oxipng
pub enum Deflaters {
/// Use the Zlib/Miniz DEFLATE implementation
Zlib {
/// Which zlib compression levels to try on the file (1-9)
///
/// Default: `9`
/// Use libdeflater.
Libdeflater {
/// Which compression levels to try on the file (1-12)
compression: IndexSet<u8>,
/// Which zlib compression strategies to try on the file (0-3)
///
/// Default: `0-3`
strategies: IndexSet<u8>,
/// Window size to use when compressing the file, as `2^window` bytes.
///
/// Doesn't affect compression but may affect speed and memory usage.
/// 8-15 are valid values.
///
/// Default: `15`
window: u8,
},
#[cfg(feature = "zopfli")]
/// Use the better but slower Zopfli implementation
@ -106,10 +43,4 @@ pub enum Deflaters {
/// less iterations, or else they will be too slow.
iterations: NonZeroU8,
},
#[cfg(feature = "libdeflater")]
/// Use libdeflater.
Libdeflater {
/// Which compression levels to try on the file (1-12)
compression: IndexSet<u8>,
},
}

2
src/evaluate.rs
View file

@ -114,7 +114,7 @@ impl Evaluator {
if deadline.passed() {
return;
}
if let Ok(idat_data) = deflate::libdeflater_deflate(
if let Ok(idat_data) = deflate::deflate(
&image.filter_image(filter),
STD_COMPRESSION,
&best_candidate_size,

94
src/lib.rs
View file

@ -14,7 +14,7 @@
#![allow(clippy::cognitive_complexity)]
#![allow(clippy::upper_case_acronyms)]
#![cfg_attr(
not(any(feature = "libdeflater", feature = "zopfli")),
not(feature = "zopfli"),
allow(irrefutable_let_patterns),
allow(unreachable_patterns)
)]
@ -26,7 +26,7 @@ mod rayon;
use crate::atomicmin::AtomicMin;
use crate::colors::BitDepth;
use crate::deflate::{crc32, libdeflater_inflate};
use crate::deflate::{crc32, inflate};
use crate::evaluate::Evaluator;
use crate::png::PngData;
use crate::png::PngImage;
@ -467,7 +467,6 @@ pub fn optimize_from_memory(data: &[u8], opts: &Options) -> PngResult<Vec<u8>> {
struct TrialOptions {
pub filter: u8,
pub compression: u8,
pub strategy: u8,
}
/// Perform optimization on the input PNG object using the options provided
@ -506,28 +505,19 @@ fn optimize_png(
info!(" File size = {} bytes", file_original_size);
let mut filter = opts.filter.clone();
let mut strategies = match &opts.deflate {
Deflaters::Zlib { strategies, .. } => Some(strategies.clone()),
_ => None,
};
if opts.use_heuristics {
// Heuristically determine which set of options to use
let (use_filter, use_strategy) = if png.raw.ihdr.bit_depth.as_u8() >= 8
let use_filter = if png.raw.ihdr.bit_depth.as_u8() >= 8
&& png.raw.ihdr.color_type != colors::ColorType::Indexed
{
(5, 1)
5
} else {
(0, 0)
0
};
if filter.is_empty() {
filter.insert(use_filter);
}
if let Some(strategies) = &mut strategies {
if strategies.is_empty() {
strategies.insert(use_strategy);
}
}
}
// This will collect all versions of images and pick one that compresses best
@ -547,51 +537,30 @@ fn optimize_png(
if opts.idat_recoding || reduction_occurred {
// Go through selected permutations and determine the best
let combinations = if let Deflaters::Zlib { compression, .. } = &opts.deflate {
filter.len() * compression.len() * strategies.as_ref().unwrap().len()
let combinations = if let Deflaters::Libdeflater { compression } = &opts.deflate {
filter.len() * compression.len()
} else {
filter.len()
};
let mut results: Vec<TrialOptions> = Vec::with_capacity(combinations);
for f in &filter {
match &opts.deflate {
Deflaters::Zlib { compression, .. } => {
for zc in compression {
for zs in strategies.as_ref().unwrap() {
results.push(TrialOptions {
filter: *f,
compression: *zc,
strategy: *zs,
});
}
if deadline.passed() {
break;
}
}
}
#[cfg(feature = "zopfli")]
Deflaters::Zopfli { .. } => {
// Zopfli has no additional options.
if let Deflaters::Libdeflater { compression } = &opts.deflate {
for zc in compression {
results.push(TrialOptions {
filter: *f,
compression: 0,
strategy: 0,
compression: *zc,
});
}
#[cfg(feature = "libdeflater")]
Deflaters::Libdeflater { compression } => {
for zc in compression {
results.push(TrialOptions {
filter: *f,
compression: *zc,
strategy: 0,
});
if deadline.passed() {
break;
}
if deadline.passed() {
break;
}
}
} else {
// Zopfli has no additional options.
results.push(TrialOptions {
filter: *f,
compression: 0,
});
}
if deadline.passed() {
@ -621,28 +590,19 @@ fn optimize_png(
}
let filtered = &filters[&trial.filter];
let new_idat = match opts.deflate {
Deflaters::Zlib { window, .. } => deflate::deflate(
filtered,
trial.compression,
trial.strategy,
window,
&best_size,
&deadline,
),
Deflaters::Libdeflater { .. } => {
deflate::deflate(filtered, trial.compression, &best_size)
}
#[cfg(feature = "zopfli")]
Deflaters::Zopfli { iterations } => deflate::zopfli_deflate(filtered, iterations),
#[cfg(feature = "libdeflater")]
Deflaters::Libdeflater { .. } => {
deflate::libdeflater_deflate(filtered, trial.compression, &best_size)
}
};
let new_idat = match new_idat {
Ok(n) => n,
Err(PngError::DeflatedDataTooLong(max)) => {
debug!(
" zc = {} zs = {} f = {} >{} bytes",
trial.compression, trial.strategy, trial.filter, max,
" zc = {} f = {} >{} bytes",
trial.compression, trial.filter, max,
);
return None;
}
@ -654,9 +614,8 @@ fn optimize_png(
best_size.set_min(new_size);
debug!(
" zc = {} zs = {} f = {} {} bytes",
" zc = {} f = {} {} bytes",
trial.compression,
trial.strategy,
trial.filter,
new_idat.len()
);
@ -679,9 +638,8 @@ fn optimize_png(
png.idat_data = idat_data;
info!("Found better combination:");
info!(
" zc = {} zs = {} f = {} {} bytes",
" zc = {} f = {} {} bytes",
opts.compression,
opts.strategy,
opts.filter,
png.idat_data.len()
);
@ -949,7 +907,7 @@ fn srgb_rendering_intent(mut iccp: &[u8]) -> Option<u8> {
}
// The decompressed size is unknown so we have to guess the required buffer size
let max_size = (compressed_data.len() * 2).max(1000);
let icc_data = libdeflater_inflate(compressed_data, max_size).ok()?;
let icc_data = inflate(compressed_data, max_size).ok()?;
let rendering_intent = *icc_data.get(67)?;

6
src/main.rs
View file

@ -14,7 +14,7 @@
#![allow(clippy::cognitive_complexity)]
use clap::{AppSettings, Arg, ArgMatches, Command};
use indexmap::{indexset, IndexSet};
use indexmap::IndexSet;
use log::{error, warn};
use oxipng::AlphaOptim;
use oxipng::Deflaters;
@ -502,10 +502,6 @@ fn parse_opts_into_struct(
opts.deflate = Deflaters::Zopfli {
iterations: NonZeroU8::new(15).unwrap(),
};
} else if matches.is_present("libdeflater") {
opts.deflate = Deflaters::Libdeflater {
compression: indexset! { 12 },
};
} else if let Deflaters::Libdeflater { compression } = &mut opts.deflate {
if let Some(x) = matches.value_of("compression") {
*compression = parse_numeric_range_opts(x, 1, 12).unwrap();

2
src/png/mod.rs
View file

@ -111,7 +111,7 @@ impl PngData {
None => return Err(PngError::ChunkMissing("IHDR")),
};
let ihdr_header = parse_ihdr_header(&ihdr)?;
let raw_data = deflate::libdeflater_inflate(idat_headers.as_ref(), ihdr_header.raw_data_size())?;
let raw_data = deflate::inflate(idat_headers.as_ref(), ihdr_header.raw_data_size())?;
// Reject files with incorrect width/height or truncated data
if raw_data.len() != ihdr_header.raw_data_size() {

2
tests/flags.rs
View file

@ -173,7 +173,7 @@ fn verbose_mode() {
assert_eq!(logs.len(), 1);
logs.sort();
for (i, log) in logs.into_iter().enumerate() {
let expected_prefix = format!(" zc = 11 zs = 0 f = 0 ");
let expected_prefix = format!(" zc = 11 f = 0 ");
assert!(
log.starts_with(&expected_prefix),
"logs[{}] = {:?} doesn't start with {:?}",

4
tests/interlaced.rs
View file

@ -443,10 +443,6 @@ fn interlaced_palette_8_should_be_palette_8() {
#[test]
fn interlaced_palette_8_should_be_palette_4() {
// miniz doesn't estimate compression that well
if !oxipng::internal_tests::cfzlib::is_supported() {
return;
}
test_it_converts(
"tests/files/interlaced_palette_8_should_be_palette_4.png",
ColorType::Indexed,