From e8a8dede55e35e05e84c6d90446a2ef44dee7259 Mon Sep 17 00:00:00 2001 From: Chris Hennick Date: Wed, 21 Jun 2023 10:08:30 -0700 Subject: [PATCH] Bug fix: need to implement Zlib format --- Cargo.lock | 1 + Cargo.toml | 2 ++ src/deflate/mod.rs | 45 ++++++++++++++++++++----------- src/deflate/zopfli_oxipng.rs | 51 ++++++++++++++++++++++++++++++++++++ 4 files changed, 83 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6dc42816..fa85192f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -367,6 +367,7 @@ dependencies = [ "rgb", "rustc-hash", "rustc_version", + "simd-adler32", "stderrlog", "wild", "zopfli", diff --git a/Cargo.toml b/Cargo.toml index cf3ab8dc..b97ee78a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,7 @@ required-features = ["zopfli"] [dependencies] zopfli = { version = "0.7.4", optional = true, default-features = false, features = ["std", "zlib"] } +simd-adler32 = { version = "0.3.5", optional = true, default-features = false } rgb = "0.8.36" indexmap = "1.9.3" libdeflater = "0.11.0" @@ -66,6 +67,7 @@ version = "0.24.6" rustc_version = "0.4.0" [features] +zopfli = ["zopfli/std", "zopfli/zlib", "simd-adler32"] binary = ["clap", "wild", "stderrlog"] default = ["binary", "filetime", "parallel", "zopfli"] parallel = ["rayon", "indexmap/rayon", "crossbeam-channel"] diff --git a/src/deflate/mod.rs b/src/deflate/mod.rs index ba606689..7d323cc7 100644 --- a/src/deflate/mod.rs +++ b/src/deflate/mod.rs @@ -4,18 +4,19 @@ use crate::{PngError, PngResult}; pub use deflater::crc32; pub use deflater::deflate; pub use deflater::inflate; -use std::io::{BufWriter, Cursor, Write}; +use std::io::{BufWriter, copy, Cursor, Write}; use std::{fmt, fmt::Display, io}; #[cfg(feature = "zopfli")] use std::num::NonZeroU8; #[cfg(feature = "zopfli")] use zopfli::{DeflateEncoder, Options}; - #[cfg(feature = "zopfli")] mod zopfli_oxipng; #[cfg(feature = "zopfli")] pub use zopfli_oxipng::deflate as zopfli_deflate; +#[cfg(feature = "zopfli")] +use simd_adler32::Adler32; #[derive(Clone, Copy, Debug, PartialEq, Eq)] /// DEFLATE algorithms supported by oxipng @@ -100,6 +101,8 @@ impl Default for BufferedZopfliDeflater { #[cfg(feature = "zopfli")] impl Deflater for BufferedZopfliDeflater { + + /// Fork of the zlib_compress function in Zopfli. fn deflate(&self, data: &[u8], max_size: &AtomicMin) -> PngResult> { #[allow(clippy::needless_update)] let options = Options { @@ -107,22 +110,32 @@ impl Deflater for BufferedZopfliDeflater { maximum_block_splits: self.max_block_splits, ..Default::default() // for forward compatibility }; - let mut out = Vec::with_capacity(self.output_buffer_size); - let mut buffer = BufWriter::with_capacity( - self.input_buffer_size, - DeflateEncoder::new( - options, - Default::default(), - &mut out, - ), - ); - let result = (|| -> io::Result<()> { - buffer.write_all(data)?; + let mut out = Cursor::new(Vec::with_capacity(self.output_buffer_size)); + let cmf = 120; /* CM 8, CINFO 7. See zlib spec.*/ + let flevel = 3; + let fdict = 0; + let mut cmfflg: u16 = 256 * cmf + fdict * 32 + flevel * 64; + let fcheck = 31 - cmfflg % 31; + cmfflg += fcheck; + + let out = (|| -> io::Result> { + let mut rolling_adler = Adler32::new(); + let mut in_data = zopfli_oxipng::HashingAndCountingRead::new(data, &mut rolling_adler, None); + out.write_all(&cmfflg.to_be_bytes())?; + let mut buffer = BufWriter::with_capacity( + self.input_buffer_size, + DeflateEncoder::new( + options, + Default::default(), + &mut out, + ), + ); + copy(&mut in_data, &mut buffer)?; buffer.into_inner()?.finish()?; - Ok(()) + out.write_all(&rolling_adler.finish().to_be_bytes())?; + Ok(out.into_inner()) })(); - result.map_err(|e| PngError::new(&e.to_string()))?; - println!("Compressed {} -> {} bytes", data.len(), out.len()); + let out = out.map_err(|e| PngError::new(&e.to_string()))?; if max_size.get().is_some_and(|max| max < out.len()) { Err(PngError::DeflatedDataTooLong(out.len())) } else { diff --git a/src/deflate/zopfli_oxipng.rs b/src/deflate/zopfli_oxipng.rs index b59b93df..f287dfe8 100644 --- a/src/deflate/zopfli_oxipng.rs +++ b/src/deflate/zopfli_oxipng.rs @@ -1,5 +1,7 @@ +use std::io::{Error, ErrorKind, Read}; use crate::{PngError, PngResult}; use std::num::NonZeroU8; +use simd_adler32::Adler32; pub fn deflate(data: &[u8], iterations: NonZeroU8) -> PngResult> { use std::cmp::max; @@ -16,3 +18,52 @@ pub fn deflate(data: &[u8], iterations: NonZeroU8) -> PngResult> { output.shrink_to_fit(); Ok(output) } + +/// Forked from zopfli crate +pub trait Hasher { + fn update(&mut self, data: &[u8]); +} + +impl Hasher for &mut Adler32 { + fn update(&mut self, data: &[u8]) { + Adler32::write(self, data) + } +} + +/// A reader that wraps another reader, a hasher and an optional counter, +/// updating the hasher state and incrementing a counter of bytes read so +/// far for each block of data read. +pub struct HashingAndCountingRead<'counter, R: Read, H: Hasher> { + inner: R, + hasher: H, + bytes_read: Option<&'counter mut u32>, +} + +impl<'counter, R: Read, H: Hasher> HashingAndCountingRead<'counter, R, H> { + pub fn new(inner: R, hasher: H, bytes_read: Option<&'counter mut u32>) -> Self { + Self { + inner, + hasher, + bytes_read, + } + } +} + +impl Read for HashingAndCountingRead<'_, R, H> { + fn read(&mut self, buf: &mut [u8]) -> Result { + match self.inner.read(buf) { + Ok(bytes_read) => { + self.hasher.update(&buf[..bytes_read]); + + if let Some(total_bytes_read) = &mut self.bytes_read { + **total_bytes_read = total_bytes_read + .checked_add(bytes_read.try_into().map_err(|_| ErrorKind::Other)?) + .ok_or(ErrorKind::Other)?; + } + + Ok(bytes_read) + } + Err(err) => Err(err), + } + } +}