diff --git a/Cargo.lock b/Cargo.lock index c8232081..7c76a27a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -264,6 +264,7 @@ dependencies = [ "miniz_oxide 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "rayon 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "rgb 0.8.11 (registry+https://github.com/rust-lang/crates.io-index)", "wild 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "zopfli 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -313,6 +314,11 @@ dependencies = [ "redox_syscall 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "rgb" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "scopeguard" version = "0.3.3" @@ -435,6 +441,7 @@ dependencies = [ "checksum rayon-core 1.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b055d1e92aba6877574d8fe604a63c8b5df60f60e5982bf7ccbb1338ea527356" "checksum redox_syscall 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "cf8fb82a4d1c9b28f1c26c574a5b541f5ffb4315f6c9a791fa47b6a04438fe93" "checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" +"checksum rgb 0.8.11 (registry+https://github.com/rust-lang/crates.io-index)" = "002bebda58b24482d6911a59512e8a17fa1defecf5a2162521113b7cc5422dd1" "checksum scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27" "checksum strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550" "checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096" diff --git a/Cargo.toml b/Cargo.toml index e0a48f82..dc8f6342 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,6 +34,7 @@ itertools = "^0.7.7" num_cpus = "^1.0.0" zopfli = "^0.3.4" miniz_oxide = "0.2.0" +rgb = "0.8.11" [dependencies.rayon] optional = true diff --git a/src/lib.rs b/src/lib.rs index f4b49097..54f55e4e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,6 +4,7 @@ extern crate byteorder; extern crate cloudflare_zlib_sys; extern crate crc; extern crate image; +extern crate rgb; extern crate itertools; extern crate miniz_oxide; extern crate num_cpus; diff --git a/src/png/mod.rs b/src/png/mod.rs index f907cd49..cd5af878 100644 --- a/src/png/mod.rs +++ b/src/png/mod.rs @@ -1,3 +1,5 @@ +use rgb::RGBA8; +use rgb::ComponentSlice; use atomicmin::AtomicMin; use bit_vec::BitVec; use byteorder::{BigEndian, WriteBytesExt}; @@ -16,7 +18,7 @@ use reduction::color::*; use std::collections::{HashMap, HashSet}; use std::fs::File; use std::io::{Read, Seek, SeekFrom}; -use std::iter::{Iterator, repeat}; +use std::iter::Iterator; use std::path::Path; const STD_COMPRESSION: u8 = 8; @@ -40,11 +42,9 @@ pub struct PngData { pub raw_data: Vec, /// The palette containing colors used in an Indexed image /// Contains 3 bytes per color (R+G+B), up to 768 - pub palette: Option>, + pub palette: Option>, /// The pixel value that should be rendered as transparent pub transparency_pixel: Option>, - /// A map of how transparent each color in the palette should be - pub transparency_palette: Option>, /// All non-critical headers from the PNG are stored here pub aux_headers: HashMap<[u8; 4], Vec>, } @@ -52,10 +52,10 @@ pub struct PngData { impl PngData { /// Create a new `PngData` struct by opening a file #[inline] - pub fn new(filepath: &Path, fix_errors: bool) -> Result { - let byte_data = PngData::read_file(filepath)?; + pub fn new(filepath: &Path, fix_errors: bool) -> Result { + let byte_data = Self::read_file(filepath)?; - PngData::from_slice(&byte_data, fix_errors) + Self::from_slice(&byte_data, fix_errors) } pub fn read_file(filepath: &Path) -> Result, PngError> { @@ -85,7 +85,7 @@ impl PngData { } /// Create a new `PngData` struct by reading a slice - pub fn from_slice(byte_data: &[u8], fix_errors: bool) -> Result { + pub fn from_slice(byte_data: &[u8], fix_errors: bool) -> Result { let mut byte_offset: usize = 0; // Test that png header is valid let header = byte_data.get(0..8).ok_or(PngError::TruncatedData)?; @@ -115,31 +115,15 @@ impl PngData { }; let ihdr_header = parse_ihdr_header(&ihdr)?; let raw_data = deflate::inflate(idat_headers.as_ref())?; - // Handle transparency header - let mut has_transparency_pixel = false; - let mut has_transparency_palette = false; - if aux_headers.contains_key(b"tRNS") { - if ihdr_header.color_type == ColorType::Indexed { - has_transparency_palette = true; - } else { - has_transparency_pixel = true; - } - } - let mut png_data = PngData { + + let (palette, transparency_pixel) = Self::palette_to_rgba(ihdr_header.color_type, aux_headers.remove(b"PLTE"), aux_headers.remove(b"tRNS"))?; + + let mut png_data = Self { idat_data: idat_headers, ihdr_data: ihdr_header, raw_data, - palette: aux_headers.remove(b"PLTE"), - transparency_pixel: if has_transparency_pixel { - aux_headers.remove(b"tRNS") - } else { - None - }, - transparency_palette: if has_transparency_palette { - aux_headers.remove(b"tRNS") - } else { - None - }, + palette, + transparency_pixel, aux_headers, }; png_data.raw_data = png_data.unfilter_image(); @@ -147,13 +131,31 @@ impl PngData { Ok(png_data) } - pub(crate) fn reset_from_original(&mut self, original: &PngData) { + /// Handle transparency header + fn palette_to_rgba(color_type: ColorType, palette_data: Option>, trns_data: Option>) -> Result<(Option>, Option>), PngError> { + if color_type == ColorType::Indexed { + let palette_data = palette_data.ok_or_else(|| PngError::new("no palette in indexed image"))?; + let mut palette: Vec<_> = palette_data.chunks(3) + .map(|color| RGBA8::new(color[0], color[1], color[2], 255)) + .collect(); + + if let Some(trns_data) = trns_data { + for (color, trns) in palette.iter_mut().zip(trns_data) { + color.a = trns; + } + } + Ok((Some(palette), None)) + } else { + Ok((None, trns_data)) + } + } + + pub(crate) fn reset_from_original(&mut self, original: &Self) { self.idat_data = original.idat_data.clone(); self.ihdr_data = original.ihdr_data; self.raw_data = original.raw_data.clone(); self.palette = original.palette.clone(); self.transparency_pixel = original.transparency_pixel.clone(); - self.transparency_palette = original.transparency_palette.clone(); self.aux_headers = original.aux_headers.clone(); } @@ -187,10 +189,17 @@ impl PngData { } // Palette if let Some(ref palette) = self.palette { - write_png_block(b"PLTE", palette, &mut output); - if let Some(ref transparency_palette) = self.transparency_palette { - // Transparency pixel - write_png_block(b"tRNS", transparency_palette, &mut output); + let mut palette_data = Vec::with_capacity(palette.len()*3); + for px in palette { + palette_data.extend_from_slice(px.rgb().as_slice()); + } + write_png_block(b"PLTE", &palette_data, &mut output); + let num_transparent = palette.iter().enumerate().fold(0, |prev, (index, px)| { + if px.a != 255 {index+1} else {prev} + }); + if num_transparent > 0 { + let trns_data: Vec<_> = palette[0..num_transparent].iter().map(|px| px.a).collect(); + write_png_block(b"tRNS", &trns_data, &mut output); } } else if let Some(ref transparency_pixel) = self.transparency_pixel { // Transparency pixel @@ -349,32 +358,22 @@ impl PngData { return false; } - // A palette with RGB or RGBA slices - let mut palette_tmp; - let mut indexed_palette: Vec<_> = if let Some(ref trns) = self.transparency_palette { - palette_tmp = Vec::with_capacity(1024); - for (pixel, trns) in self.palette.as_ref().unwrap().chunks(3) - .zip(trns.iter().cloned().chain(repeat(255))) { - palette_tmp.extend_from_slice(pixel); - palette_tmp.push(trns); - } - palette_tmp.chunks(4).collect() - } else { - palette_tmp = self.palette.clone().unwrap(); - palette_tmp.chunks(3).collect() - }; - // A map of old indexes to new ones, for any moved let mut index_map: HashMap = HashMap::new(); + let mut palette = match self.palette { + Some(ref p) => p.clone(), + None => return false, + }; + // A list of (original) indices that are duplicates and no longer needed let mut duplicates: Vec = Vec::new(); { // Find duplicate entries in the palette - let mut seen: HashMap<&[u8], u8> = HashMap::with_capacity(indexed_palette.len()); - for (i, color) in indexed_palette.iter().cloned().enumerate() { - if seen.contains_key(color) { - let index = seen[color]; + let mut seen: HashMap = HashMap::with_capacity(palette.len()); + for (i, color) in palette.iter().cloned().enumerate() { + if seen.contains_key(&color) { + let index = seen[&color]; duplicates.push(i as u8); index_map.insert(i as u8, index); } else { @@ -385,11 +384,11 @@ impl PngData { // Remove duplicates from the data if !duplicates.is_empty() { - self.do_palette_reduction(&duplicates, &mut index_map, &mut indexed_palette); + self.do_palette_reduction(&duplicates, &mut index_map, &mut palette); } // Find palette entries that are never used - let mut seen = HashSet::with_capacity(indexed_palette.len()); + let mut seen = HashSet::with_capacity(palette.len()); for line in self.scan_lines() { match self.ihdr_data.bit_depth { BitDepth::Eight => for &byte in line.data { @@ -426,7 +425,7 @@ impl PngData { _ => unreachable!(), } - if seen.len() == indexed_palette.len() { + if seen.len() == palette.len() { // Exit early if no further possible optimizations // Check at the end of each line // Checking after every pixel would be overly expensive @@ -434,12 +433,12 @@ impl PngData { } } - let unused: Vec = (0..indexed_palette.len() as u8) + let unused: Vec = (0..palette.len() as u8) .filter(|i| !seen.contains(i)) .collect(); // Remove unused palette indices - self.do_palette_reduction(&unused, &mut index_map, &mut indexed_palette); + self.do_palette_reduction(&unused, &mut index_map, &mut palette); true } @@ -448,29 +447,20 @@ impl PngData { &mut self, indices_to_remove: &[u8], index_map: &mut HashMap, - indexed_palette: &mut Vec<&[u8]>, + palette: &mut Vec, ) { let mut new_data = Vec::with_capacity(self.raw_data.len()); - let original_len = indexed_palette.len(); - for idx in indices_to_remove.iter().sorted_by(|a, b| b.cmp(a)) { - for i in (*idx as usize + 1)..original_len { + let original_len = palette.len(); + for idx in indices_to_remove.iter().cloned().sorted_by(|a, b| b.cmp(a)) { + for i in (idx as usize + 1)..original_len { let existing = index_map.entry(i as u8).or_insert(i as u8); - if *existing >= *idx { + if *existing >= idx { *existing -= 1; } } - indexed_palette.remove(*idx as usize); - if let Some(ref mut alpha) = self.transparency_palette { - if (*idx as usize) < alpha.len() { - alpha.remove(*idx as usize); - } - } - } - if let Some(ref mut alpha) = self.transparency_palette { - while let Some(255) = alpha.last().cloned() { - alpha.pop(); - } + palette.remove(idx as usize); } + // Reassign data bytes to new indices for line in self.scan_lines() { new_data.push(line.filter); @@ -531,12 +521,8 @@ impl PngData { } index_map.clear(); self.raw_data = new_data; - let new_palette = flatten(indexed_palette.iter().cloned()) - .enumerate() - .filter(|&(i, _)| !(self.transparency_palette.is_some() && i % 4 == 3)) - .map(|(_, x)| *x) - .collect::>(); - self.palette = Some(new_palette); + self.transparency_pixel = None; + self.palette = Some(palette.clone()); } /// Attempt to reduce the color type of the image @@ -550,7 +536,7 @@ impl PngData { if self.ihdr_data.color_type == ColorType::RGBA { if reduce_rgba_to_grayscale_alpha(self) || reduce_rgba_to_rgb(self) { changed = true; - } else if reduce_rgba_to_palette(self) { + } else if reduce_color_to_palette(self) { changed = true; should_reduce_bit_depth = true; } @@ -564,7 +550,7 @@ impl PngData { } if self.ihdr_data.color_type == ColorType::RGB - && (reduce_rgb_to_grayscale(self) || reduce_rgb_to_palette(self)) + && (reduce_rgb_to_grayscale(self) || reduce_color_to_palette(self)) { changed = true; should_reduce_bit_depth = true; @@ -652,7 +638,6 @@ impl PngData { ihdr_data: self.ihdr_data, palette: self.palette.clone(), transparency_pixel: self.transparency_pixel.clone(), - transparency_palette: self.transparency_palette.clone(), aux_headers: self.aux_headers.clone(), }) } diff --git a/src/reduction/color.rs b/src/reduction/color.rs index 02712baa..67adb03c 100644 --- a/src/reduction/color.rs +++ b/src/reduction/color.rs @@ -1,6 +1,9 @@ +use std::hash::Hash; +use rgb::{RGBA8, RGB8, FromSlice}; use colors::{BitDepth, ColorType}; use itertools::Itertools; use png::PngData; +use std::collections::HashMap; use super::alpha::reduce_alpha_channel; @@ -71,173 +74,88 @@ pub fn reduce_rgba_to_grayscale_alpha(png: &mut PngData) -> bool { true } -pub fn reduce_rgba_to_palette(png: &mut PngData) -> bool { +fn reduce_scanline_to_palette(iter: impl IntoIterator, palette: &mut HashMap, reduced: &mut Vec) -> bool + where T: Eq + Hash { + for pixel in iter { + let idx = if let Some(&idx) = palette.get(&pixel) { + idx + } else { + let len = palette.len(); + if len == 256 { + return false; + } + let idx = len as u8; + palette.insert(pixel, idx); + idx + }; + reduced.push(idx); + } + true +} + +pub fn reduce_color_to_palette(png: &mut PngData) -> bool { if png.ihdr_data.bit_depth != BitDepth::Eight { return false; } let mut reduced = Vec::with_capacity(png.raw_data.len()); - let mut palette = Vec::with_capacity(256); - let bpp: usize = (4 * png.ihdr_data.bit_depth.as_u8() as usize) >> 3; + let mut palette = HashMap::with_capacity(257); + let transparency_pixel = png.transparency_pixel.as_ref().map(|t| RGB8::new(t[1], t[3], t[5])); for line in png.scan_lines() { reduced.push(line.filter); - let mut cur_pixel = Vec::with_capacity(bpp); - for (i, byte) in line.data.iter().enumerate() { - cur_pixel.push(*byte); - if i % bpp == bpp - 1 { - if let Some(idx) = palette.iter().position(|x| x == &cur_pixel) { - reduced.push(idx as u8); - } else { - let len = palette.len(); - if len == 256 { - return false; - } - palette.push(cur_pixel); - reduced.push(len as u8); - } - cur_pixel = Vec::with_capacity(bpp); - } - } - } - - let mut color_palette = Vec::with_capacity( - palette.len() * 3 + if png.aux_headers.contains_key(b"bKGD") { - 6 + let ok = if png.ihdr_data.color_type == ColorType::RGB { + reduce_scanline_to_palette(line.data.as_rgb().iter().cloned().map(|px| { + px.alpha(if Some(px) != transparency_pixel {255} else {0}) + }), &mut palette, &mut reduced) } else { - 0 - }, - ); - let mut trans_palette = Vec::with_capacity(palette.len()); - for color in &palette { - for (i, byte) in color.iter().enumerate() { - if i < 3 { - color_palette.push(*byte); - } else { - trans_palette.push(*byte); - } + debug_assert_eq!(png.ihdr_data.color_type, ColorType::RGBA); + reduce_scanline_to_palette(line.data.as_rgba().iter().cloned(), &mut palette, &mut reduced) + }; + if !ok { + return false; } } - let headers_size = color_palette.len() + trans_palette.len() + 8; - if reduced.len() + headers_size > png.raw_data.len() * 4 { + let num_transparent = palette.iter() + .filter_map(|(px, &idx)| if px.a != 255 {Some(idx as usize +1)} else {None}) + .max(); + let trns_size = num_transparent.map(|n| n+8).unwrap_or(0); + + let headers_size = palette.len()*3 + 8 + trns_size; + if reduced.len() + headers_size > png.raw_data.len() { // Reduction would result in a larger image return false; } if let Some(bkgd_header) = png.aux_headers.get_mut(b"bKGD") { assert_eq!(bkgd_header.len(), 6); - let header_pixels = bkgd_header - .iter() - .skip(1) - .step(2) - .cloned() - .collect::>(); - if let Some(entry) = color_palette - .chunks(3) - .position(|x| x == header_pixels.as_slice()) - { - *bkgd_header = vec![entry as u8]; - } else if color_palette.len() / 3 == 256 { - return false; + // In bKGD 16-bit values are used even for 8-bit images + let bg = RGBA8::new(bkgd_header[1], bkgd_header[3], bkgd_header[5], 255); + let entry = if let Some(&entry) = palette.get(&bg) { + entry + } else if palette.len() < 256 { + let entry = palette.len() as u8; + palette.insert(bg, entry); + entry } else { - let entry = color_palette.len() / 3; - color_palette.extend_from_slice(&header_pixels); - *bkgd_header = vec![entry as u8]; - } + return false; + }; + *bkgd_header = vec![entry]; } + if let Some(sbit_header) = png.aux_headers.get_mut(b"sBIT") { assert_eq!(sbit_header.len(), 4); sbit_header.pop(); } - png.raw_data = reduced; - png.palette = Some(color_palette); - if trans_palette.iter().any(|x| *x != 255) { - while let Some(255) = trans_palette.last().cloned() { - trans_palette.pop(); - } - png.transparency_palette = Some(trans_palette); - } else { - png.transparency_palette = None; - } - png.ihdr_data.color_type = ColorType::Indexed; - true -} - -pub fn reduce_rgb_to_palette(png: &mut PngData) -> bool { - if png.ihdr_data.bit_depth != BitDepth::Eight { - return false; - } - let mut reduced = Vec::with_capacity(png.raw_data.len()); - let mut palette = Vec::with_capacity(256); - if let Some(ref trns) = png.transparency_pixel { - assert_eq!(trns.len(), 6); - if trns[0] != trns[1] || trns[2] != trns[3] || trns[4] != trns[5] { - return false; - } - palette.push(vec![trns[0], trns[2], trns[4]]); - } - let bpp: usize = (3 * png.ihdr_data.bit_depth.as_u8() as usize) >> 3; - for line in png.scan_lines() { - reduced.push(line.filter); - let mut cur_pixel = Vec::with_capacity(bpp); - for (i, byte) in line.data.iter().enumerate() { - cur_pixel.push(*byte); - if i % bpp == bpp - 1 { - if let Some(idx) = palette.iter().position(|x| x == &cur_pixel) { - reduced.push(idx as u8); - } else { - let len = palette.len(); - if len == 256 { - return false; - } - palette.push(cur_pixel); - reduced.push(len as u8); - } - cur_pixel = Vec::with_capacity(bpp); - } - } - } - - let mut color_palette = Vec::with_capacity(palette.len() * 3); - for color in &palette { - color_palette.extend_from_slice(color); - } - - let headers_size = color_palette.len() + 4; - if reduced.len() + headers_size > png.raw_data.len() * 3 { - // Reduction would result in a larger image - return false; - } - - if let Some(bkgd_header) = png.aux_headers.get_mut(b"bKGD") { - assert_eq!(bkgd_header.len(), 6); - let header_pixels = bkgd_header - .iter() - .skip(1) - .step(2) - .cloned() - .collect::>(); - if let Some(entry) = color_palette - .chunks(3) - .position(|x| x == header_pixels.as_slice()) - { - *bkgd_header = vec![entry as u8]; - } else if color_palette.len() == 255 { - return false; - } else { - let entry = color_palette.len() / 3; - color_palette.extend_from_slice(&header_pixels); - *bkgd_header = vec![entry as u8]; - } + let mut palette_vec = vec![RGBA8::new(0,0,0,0); palette.len()]; + for (color, idx) in palette { + palette_vec[idx as usize] = color; } png.raw_data = reduced; - png.palette = Some(color_palette); + png.transparency_pixel = None; + png.palette = Some(palette_vec); png.ihdr_data.color_type = ColorType::Indexed; - if png.transparency_pixel.is_some() { - png.transparency_pixel = None; - png.transparency_palette = Some(vec![0]); - }; true } diff --git a/tests/reduction.rs b/tests/reduction.rs index afe23dcc..23d95382 100644 --- a/tests/reduction.rs +++ b/tests/reduction.rs @@ -718,7 +718,7 @@ fn palette_should_be_reduced_with_dupes() { assert_eq!(png.ihdr_data.color_type, ColorType::Indexed); assert_eq!(png.ihdr_data.bit_depth, BitDepth::Eight); - assert_eq!(png.palette.unwrap().len(), 43 * 3); + assert_eq!(png.palette.unwrap().len(), 43); match oxipng::optimize(&InFile::Path(input), &output, &opts) { Ok(_) => (), @@ -737,7 +737,7 @@ fn palette_should_be_reduced_with_dupes() { assert_eq!(png.ihdr_data.color_type, ColorType::Indexed); assert_eq!(png.ihdr_data.bit_depth, BitDepth::Eight); - assert_eq!(png.palette.unwrap().len(), 35 * 3); + assert_eq!(png.palette.unwrap().len(), 35); remove_file(output).ok(); } @@ -751,7 +751,7 @@ fn palette_should_be_reduced_with_unused() { assert_eq!(png.ihdr_data.color_type, ColorType::Indexed); assert_eq!(png.ihdr_data.bit_depth, BitDepth::Eight); - assert_eq!(png.palette.unwrap().len(), 35 * 3); + assert_eq!(png.palette.unwrap().len(), 35); match oxipng::optimize(&InFile::Path(input), &output, &opts) { Ok(_) => (), @@ -770,7 +770,7 @@ fn palette_should_be_reduced_with_unused() { assert_eq!(png.ihdr_data.color_type, ColorType::Indexed); assert_eq!(png.ihdr_data.bit_depth, BitDepth::Eight); - assert_eq!(png.palette.unwrap().len(), 33 * 3); + assert_eq!(png.palette.unwrap().len(), 33); remove_file(output).ok(); } @@ -784,7 +784,7 @@ fn palette_should_be_reduced_with_both() { assert_eq!(png.ihdr_data.color_type, ColorType::Indexed); assert_eq!(png.ihdr_data.bit_depth, BitDepth::Eight); - assert_eq!(png.palette.unwrap().len(), 43 * 3); + assert_eq!(png.palette.unwrap().len(), 43); match oxipng::optimize(&InFile::Path(input), &output, &opts) { Ok(_) => (), @@ -803,7 +803,7 @@ fn palette_should_be_reduced_with_both() { assert_eq!(png.ihdr_data.color_type, ColorType::Indexed); assert_eq!(png.ihdr_data.bit_depth, BitDepth::Eight); - assert_eq!(png.palette.unwrap().len(), 33 * 3); + assert_eq!(png.palette.unwrap().len(), 33); remove_file(output).ok(); }