From 052b047222e7f0e040f3aa45f94de654d49821ea Mon Sep 17 00:00:00 2001 From: Joshua Holmer Date: Fri, 1 Jan 2016 01:00:57 -0500 Subject: [PATCH] Get zlib compression to work --- .gitignore | 1 + Cargo.toml | 4 +- src/deflate/deflate.rs | 37 ++++++++++++ src/deflate/stream.rs | 130 +++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 110 ++++++++++++++++++++++------------ src/main.rs | 7 ++- src/png.rs | 97 +++++++++++++++++------------- 7 files changed, 304 insertions(+), 82 deletions(-) create mode 100644 src/deflate/deflate.rs create mode 100644 src/deflate/stream.rs diff --git a/.gitignore b/.gitignore index a9d37c56..afc99012 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ target Cargo.lock +*.bk diff --git a/Cargo.toml b/Cargo.toml index e98effd1..c06162e3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,5 +4,7 @@ version = "2.0.0" authors = ["Joshua Holmer "] [dependencies] -byteorder = "0.4." +byteorder = "~0.4.0" crc = "^1.0.0" +libz-sys = "^1.0.0" +libc = "~0.2.4" diff --git a/src/deflate/deflate.rs b/src/deflate/deflate.rs new file mode 100644 index 00000000..38fa9314 --- /dev/null +++ b/src/deflate/deflate.rs @@ -0,0 +1,37 @@ +use libz_sys; +use libc; + +pub fn inflate(data: &[u8]) -> Result, String> { + let mut input = data.to_owned(); + let mut stream = super::stream::Stream::new_decompress(); + let mut output = Vec::with_capacity(data.len()); + loop { + match stream.decompress_vec(input.as_mut(), output.as_mut()) { + libz_sys::Z_OK => output.reserve(data.len()), + libz_sys::Z_STREAM_END => break, + c => return Err(format!("Error code on decompress: {}", c)), + } + } + output.shrink_to_fit(); + + Ok(output) +} + +pub fn deflate(data: &[u8], zc: u8, zm: u8, zs: u8, zw: u16) -> Result, String> { + let mut input = data.to_owned(); + let mut stream = super::stream::Stream::new_compress(zc as libc::c_int, + zw as libc::c_int, + zm as libc::c_int, + zs as libc::c_int); + let mut output = Vec::with_capacity(data.len() / 20); + loop { + match stream.compress_vec(input.as_mut(), output.as_mut()) { + libz_sys::Z_OK => output.reserve(data.len() / 20), + libz_sys::Z_STREAM_END => break, + c => return Err(format!("Error code on compress: {}", c)), + } + } + output.shrink_to_fit(); + + Ok(output) +} diff --git a/src/deflate/stream.rs b/src/deflate/stream.rs new file mode 100644 index 00000000..715722eb --- /dev/null +++ b/src/deflate/stream.rs @@ -0,0 +1,130 @@ +// Raw un-exported bindings to libz for encoding/decoding +// Copyright (c) 2014 Alex Crichton, MIT & Apache licenses +// Originally from flate2 crate for miniz +// Modified for use in Optipng + +use std::marker; +use std::mem; +use libc::{c_int, c_uint}; +use libz_sys; + +pub struct Stream { + raw: libz_sys::z_stream, + _marker: marker::PhantomData, +} + +pub enum Compress {} +pub enum Decompress {} + +#[doc(hidden)] +pub trait Direction { + unsafe fn destroy(stream: *mut libz_sys::z_stream) -> c_int; +} + +impl Stream { + pub fn new_compress(lvl: c_int, + window_bits: c_int, + mem_size: c_int, + strategy: c_int) + -> Stream { + unsafe { + let mut state: libz_sys::z_stream = mem::zeroed(); + let ret = libz_sys::deflateInit2_(&mut state, + lvl, + libz_sys::Z_DEFLATED, + window_bits, + mem_size, + strategy, + libz_sys::zlibVersion(), + mem::size_of::() as i32); + debug_assert_eq!(ret, 0); + Stream { + raw: state, + _marker: marker::PhantomData, + } + } + } + + pub fn new_decompress() -> Stream { + unsafe { + let mut state: libz_sys::z_stream = mem::zeroed(); + let ret = libz_sys::inflateInit2_(&mut state, + 15, + libz_sys::zlibVersion(), + mem::size_of::() as i32); + debug_assert_eq!(ret, 0); + Stream { + raw: state, + _marker: marker::PhantomData, + } + } + } +} + +impl Stream { + pub fn total_in(&self) -> u64 { + self.raw.total_in as u64 + } + + pub fn total_out(&self) -> u64 { + self.raw.total_out as u64 + } +} + +impl Stream { + pub fn decompress_vec(&mut self, input: &mut [u8], output: &mut Vec) -> c_int { + self.raw.avail_in = (input.len() - self.total_in() as usize) as c_uint; + self.raw.avail_out = (output.capacity() - self.total_out() as usize) as c_uint; + + unsafe { + self.raw.next_in = input.as_mut_ptr().offset(self.total_in() as isize); + self.raw.next_out = output.as_mut_ptr().offset(self.total_out() as isize); + let rc = libz_sys::inflate(&mut self.raw, libz_sys::Z_NO_FLUSH); + output.set_len(self.total_out() as usize); + rc + } + } +} + +impl Stream { + pub fn compress_vec(&mut self, input: &mut [u8], output: &mut Vec) -> c_int { + self.raw.avail_in = (input.len() - self.total_in() as usize) as c_uint; + self.raw.avail_out = (output.capacity() - self.total_out() as usize) as c_uint; + + unsafe { + self.raw.next_in = input.as_mut_ptr().offset(self.total_in() as isize); + self.raw.next_out = output.as_mut_ptr().offset(self.total_out() as isize); + let rc = libz_sys::deflate(&mut self.raw, + if self.raw.avail_in > 0 { + libz_sys::Z_NO_FLUSH + } else { + libz_sys::Z_FINISH + }); + output.set_len(self.total_out() as usize); + rc + } + } + + pub fn reset(&mut self) -> c_int { + unsafe { libz_sys::deflateReset(&mut self.raw) } + } +} + +impl Direction for Compress { + unsafe fn destroy(stream: *mut libz_sys::z_stream) -> c_int { + libz_sys::deflateEnd(stream) + } +} +impl Direction for Decompress { + unsafe fn destroy(stream: *mut libz_sys::z_stream) -> c_int { + libz_sys::inflateEnd(stream) + } +} + +impl Drop for Stream { + fn drop(&mut self) { + unsafe { + let _ = ::destroy(&mut self.raw); + } + } +} diff --git a/src/lib.rs b/src/lib.rs index a2e8eaed..3f8726f6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,10 +1,16 @@ extern crate byteorder; extern crate crc; +extern crate libz_sys; +extern crate libc; use std::path::Path; use std::collections::HashSet; -mod png; +pub mod png; +pub mod deflate { + pub mod deflate; + pub mod stream; +} pub struct Options<'a> { pub backup: bool, @@ -20,7 +26,7 @@ pub struct Options<'a> { pub compression: HashSet, pub zm: HashSet, pub zs: HashSet, - pub zw: u32, + pub zw: u16, pub bit_depth_reduction: bool, pub color_type_reduction: bool, pub palette_reduction: bool, @@ -28,61 +34,89 @@ pub struct Options<'a> { pub idat_paranoia: bool, } -// Processing: /Users/holmerj/Downloads/renpy-6.99.7-sdk/doc/_images/frame_example.png -// 579x354 pixels, PNG format -// 3x8 bits/pixel, RGB -// IDAT size = 45711 bytes -// file size = 45881 bytes -// Trying: 8 combinations -// Output: -// IDAT size = 45711 bytes (no change) -// file size = 45821 bytes (60 bytes = 0.13% decrease) pub fn optimize(filepath: &Path, opts: Options) -> Result<(), String> { // Decode PNG from file println!("Processing: {}", filepath.to_str().unwrap()); let in_file = Path::new(filepath); - let png = match png::PngData::new(&in_file) { + let mut png = match png::PngData::new(&in_file) { Ok(x) => x, - Err(x) => return Err(x) + Err(x) => return Err(x), }; // Print png info let idat_current_size = png.idat_data.len(); let file_current_size = filepath.metadata().unwrap().len(); if opts.verbosity.is_some() { - println!(" {}x{} pixels, PNG format", png.ihdr_data.width, png.ihdr_data.height); + println!(" {}x{} pixels, PNG format", + png.ihdr_data.width, + png.ihdr_data.height); if let Some(palette) = png.palette.clone() { - println!(" {} bits/pixel, {} colors in palette", png.ihdr_data.bit_depth, palette.len() / 3); + println!(" {} bits/pixel, {} colors in palette", + png.ihdr_data.bit_depth, + palette.len() / 3); } else { - println!(" {}x{} bits/pixel, {:?}", png.bits_per_pixel(), png.ihdr_data.bit_depth, png.ihdr_data.color_type); + println!(" {}x{} bits/pixel, {:?}", + png.bits_per_pixel(), + png.ihdr_data.bit_depth, + png.ihdr_data.color_type); } println!(" IDAT size = {} bytes", idat_current_size); println!(" File size = {} bytes", file_current_size); } + // TODO: Bit depth/palette reduction // - // // TODO: Bit depth/palette reduction + // TODO: Apply interlacing changes + // TODO: Force reencoding if interlacing was changed // - // // Go through selected permutations and determine the best - // let mut best: (Option<(u8, u8, u8, u8)>, usize) = (None, idat_current_size.clone()); - // if opts.idat_recoding { - // let combinations = opts.f.len() * opts.zc.len() * opts.zm.len() * opts.zs.len(); - // println!("Trying: {} combinations", combinations); - // // TODO: Multithreading - // for f in &opts.f { - // for zc in &opts.zc { - // for zm in &opts.zm { - // for zs in &opts.zs { - // // TODO: Test compressions - // let new_idat = compress::compress(img_buf.clone(), *f, opts.i, *zc, *zm, *zs, opts.zw); - // // TODO: Force reencoding if interlacing was changed - // if new_idat.len() < best.1 { - // best = (Some((*f, *zc, *zm, *zs)), new_idat.len()); - // } - // } - // } - // } - // } - // } + // Go through selected permutations and determine the best + let mut best: Option<(u8, u8, u8, u8)> = None; + if opts.idat_recoding { + let combinations = opts.filter.len() * opts.compression.len() * opts.zm.len() * + opts.zs.len(); + println!("Trying: {} combinations", combinations); + // TODO: Multithreading + for f in &opts.filter { + for zc in &opts.compression { + for zm in &opts.zm { + for zs in &opts.zs { + let new_idat = match deflate::deflate::deflate(png.raw_data.as_ref(), + *zc, + *zm, + *zs, + opts.zw) { + Ok(x) => x, + Err(x) => return Err(x), + }; + // TODO: Apply filtering + if new_idat.len() < png.idat_data.len() { + best = Some((*f, *zc, *zm, *zs)); + png.idat_data = new_idat.clone(); + } + if opts.verbosity == Some(1) { + println!(" zc = {} zm = {} zs = {} f = {} {} bytes", + *zc, + *zm, + *zs, + *f, + new_idat.len()); + } + } + } + } + } + + if let Some(better) = best { + println!("Found better combination:"); + println!(" zc = {} zm = {} zs = {} f = {} {} bytes", + better.1, + better.2, + better.3, + better.0, + png.idat_data.len()); + } else { + println!("IDAT already optimized"); + } + } // TODO: Backup before writing? diff --git a/src/main.rs b/src/main.rs index f90d0848..4b32ac2b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -34,7 +34,7 @@ fn main() { compression: compression, zm: zm, zs: zs, - zw: 4096, + zw: 15, bit_depth_reduction: true, color_type_reduction: true, palette_reduction: true, @@ -43,5 +43,8 @@ fn main() { }; // TODO: Handle command line args // TODO: Handle optimization presets - optipng::optimize(infile, default_opts); + match optipng::optimize(infile, default_opts) { + Ok(_) => (), + Err(x) => panic!(x), + }; } diff --git a/src/png.rs b/src/png.rs index b69b1b91..e82c6745 100644 --- a/src/png.rs +++ b/src/png.rs @@ -18,13 +18,15 @@ pub enum ColorType { impl fmt::Display for ColorType { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{}", match *self { - ColorType::Grayscale => "Grayscale", - ColorType::RGB => "RGB", - ColorType::Indexed => "Indexed", - ColorType::GrayscaleAlpha => "Grayscale + Alpha", - ColorType::RGBA => "RGB + Alpha", - }) + write!(f, + "{}", + match *self { + ColorType::Grayscale => "Grayscale", + ColorType::RGB => "RGB", + ColorType::Indexed => "Indexed", + ColorType::GrayscaleAlpha => "Grayscale + Alpha", + ColorType::RGBA => "RGB + Alpha", + }) } } @@ -39,13 +41,15 @@ pub enum BitDepth { impl fmt::Display for BitDepth { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{}", match *self { - BitDepth::One => "1", - BitDepth::Two => "2", - BitDepth::Four => "4", - BitDepth::Eight => "8", - BitDepth::Sixteen => "16", - }) + write!(f, + "{}", + match *self { + BitDepth::One => "1", + BitDepth::Two => "2", + BitDepth::Four => "4", + BitDepth::Eight => "8", + BitDepth::Sixteen => "16", + }) } } @@ -53,6 +57,7 @@ impl fmt::Display for BitDepth { pub struct PngData { pub idat_data: Vec, pub ihdr_data: IhdrData, + pub raw_data: Vec, pub palette: Option>, } @@ -71,13 +76,13 @@ impl PngData { pub fn new(filepath: &Path) -> Result { let mut file = match File::open(filepath) { Ok(f) => f, - Err(_) => return Err("Failed to open file for reading".to_owned()) + Err(_) => return Err("Failed to open file for reading".to_owned()), }; let mut byte_data: Vec = Vec::new(); // Read raw png data into memory match file.read_to_end(&mut byte_data) { Ok(_) => (), - Err(_) => return Err("Failed to read from file".to_owned()) + Err(_) => return Err("Failed to read from file".to_owned()), } let mut byte_offset: usize = 0; // Test that png header is valid @@ -93,11 +98,11 @@ impl PngData { let header = parse_next_header(byte_data.as_ref(), &mut byte_offset); let header = match header { Ok(x) => x, - Err(x) => return Err(x) + Err(x) => return Err(x), }; let header = match header { Some(x) => x, - None => break + None => break, }; if header.0 == "IDAT" { idat_headers.extend(header.1); @@ -106,33 +111,29 @@ impl PngData { } } // Parse the headers into our PngData - if idat_headers.len() == 0 { + if idat_headers.is_empty() { return Err("Image data was empty, skipping".to_owned()); } if aux_headers.get("IHDR").is_none() { return Err("Image header data was missing, skipping".to_owned()); } let ihdr_header = parse_ihdr_header(aux_headers.get("IHDR").unwrap().as_ref()); + let raw_data = match super::deflate::deflate::inflate(idat_headers.as_ref()) { + Ok(x) => x, + Err(x) => return Err(x), + }; + // TODO: Reverse filtering? // Return the PngData Ok(PngData { - idat_data: idat_headers, + idat_data: idat_headers.clone(), ihdr_data: match ihdr_header { Ok(x) => x, - Err(x) => return Err(x) + Err(x) => return Err(x), }, + raw_data: raw_data, palette: aux_headers.get("PLTE").cloned(), }) } - pub fn buffer_size(&self) -> usize { - // Return the size needed to hold a decoded frame - self.ihdr_data.width * self.ihdr_data.height * match self.ihdr_data.bit_depth { - BitDepth::One => 1, - BitDepth::Two => 2, - BitDepth::Four => 4, - BitDepth::Eight => 8, - BitDepth::Sixteen => 16 - } as usize - } pub fn bits_per_pixel(&self) -> u8 { match self.ihdr_data.color_type { ColorType::Grayscale => 1, @@ -150,31 +151,45 @@ fn file_header_is_valid(bytes: &[u8]) -> bool { bytes.iter().zip(expected_header.iter()).all(|x| x.0 == x.1) } -fn parse_next_header(byte_data: &[u8], byte_offset: &mut usize) -> Result)>, String> { - let mut rdr = Cursor::new(byte_data.iter().skip(*byte_offset).take(4).cloned().collect::>()); +fn parse_next_header(byte_data: &[u8], + byte_offset: &mut usize) + -> Result)>, String> { + let mut rdr = Cursor::new(byte_data.iter() + .skip(*byte_offset) + .take(4) + .cloned() + .collect::>()); let length: u32 = match rdr.read_u32::() { Ok(x) => x, - Err(_) => return Err("Invalid data found--unable to read PNG file".to_owned()) + Err(_) => return Err("Invalid data found--unable to read PNG file".to_owned()), }; *byte_offset += 4; let mut header_bytes: Vec = byte_data.iter().skip(*byte_offset).take(4).cloned().collect(); let header = match String::from_utf8(header_bytes.clone()) { Ok(x) => x, - Err(_) => return Err("Invalid data found--unable to read PNG file".to_owned()) + Err(_) => return Err("Invalid data found--unable to read PNG file".to_owned()), }; - if header == "IEND".to_owned() { + if header == "IEND" { // End of data return Ok(None); } *byte_offset += 4; - let data: Vec = byte_data.iter().skip(*byte_offset).take(length as usize).cloned().collect(); + let data: Vec = byte_data.iter() + .skip(*byte_offset) + .take(length as usize) + .cloned() + .collect(); *byte_offset += length as usize; - let mut rdr = Cursor::new(byte_data.iter().skip(*byte_offset).take(4).cloned().collect::>()); + let mut rdr = Cursor::new(byte_data.iter() + .skip(*byte_offset) + .take(4) + .cloned() + .collect::>()); let crc: u32 = match rdr.read_u32::() { Ok(x) => x, - Err(_) => return Err("Invalid data found--unable to read PNG file".to_owned()) + Err(_) => return Err("Invalid data found--unable to read PNG file".to_owned()), }; *byte_offset += 4; header_bytes.extend(data.clone()); @@ -194,7 +209,7 @@ fn parse_ihdr_header(byte_data: &[u8]) -> Result { 3 => ColorType::Indexed, 4 => ColorType::GrayscaleAlpha, 6 => ColorType::RGBA, - _ => return Err("Unexpected color type in header".to_owned()) + _ => return Err("Unexpected color type in header".to_owned()), }, bit_depth: match byte_data[8] { 1 => BitDepth::One, @@ -202,7 +217,7 @@ fn parse_ihdr_header(byte_data: &[u8]) -> Result { 4 => BitDepth::Four, 8 => BitDepth::Eight, 16 => BitDepth::Sixteen, - _ => return Err("Unexpected bit depth in header".to_owned()) + _ => return Err("Unexpected bit depth in header".to_owned()), }, width: rdr.read_u32::().unwrap(), height: rdr.read_u32::().unwrap(),