Merge branch 'master' into issue-373

Cargo.lock

@@ -57,17 +57,6 @@ dependencies = [
  "windows-sys",
 ]
 
-[[package]]
-name = "atty"
-version = "0.2.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
-dependencies = [
- "hermit-abi 0.1.19",
- "libc",
- "winapi",
-]
-
 [[package]]
 name = "autocfg"
 version = "1.1.0"
@@ -214,6 +203,17 @@ version = "1.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91"
 
+[[package]]
+name = "env_logger"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0"
+dependencies = [
+ "is-terminal",
+ "log",
+ "termcolor",
+]
+
 [[package]]
 name = "equivalent"
 version = "1.0.0"
@@ -290,15 +290,6 @@ version = "0.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a"
 
-[[package]]
-name = "hermit-abi"
-version = "0.1.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
-dependencies = [
- "libc",
-]
-
 [[package]]
 name = "hermit-abi"
 version = "0.2.6"
@@ -457,12 +448,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "once_cell"
-version = "1.18.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
-
 [[package]]
 name = "oxipng"
 version = "8.0.0"
@@ -470,6 +455,7 @@ dependencies = [
  "bitvec",
  "clap",
  "crossbeam-channel",
+ "env_logger",
  "filetime",
  "glob",
  "image",
@@ -480,7 +466,6 @@ dependencies = [
  "rgb",
  "rustc-hash",
  "rustc_version",
- "stderrlog",
  "zopfli",
 ]
 
@@ -590,18 +575,6 @@ version = "0.3.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "238abfbb77c1915110ad968465608b68e869e0772622c9656714e73e5a1a522f"
 
-[[package]]
-name = "stderrlog"
-version = "0.5.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "69a26bbf6de627d389164afa9783739b56746c6c72c4ed16539f4ff54170327b"
-dependencies = [
- "atty",
- "log",
- "termcolor",
- "thread_local",
-]
-
 [[package]]
 name = "strsim"
 version = "0.10.0"
@@ -623,16 +596,6 @@ dependencies = [
  "winapi-util",
 ]
 
-[[package]]
-name = "thread_local"
-version = "1.1.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152"
-dependencies = [
- "cfg-if",
- "once_cell",
-]
-
 [[package]]
 name = "typed-arena"
 version = "2.0.2"

Cargo.toml

@@ -32,10 +32,15 @@ rgb = "0.8.36"
 indexmap = "2.0.0"
 libdeflater = "0.14.0"
 log = "0.4.19"
-stderrlog = { version = "0.5.4", optional = true, default-features = false }
 bitvec = "1.0.1"
 rustc-hash = "1.1.0"
 
+[dependencies.env_logger]
+optional = true
+default-features = false
+features = ["auto-color"]
+version = "0.10.0"
+
 [dependencies.crossbeam-channel]
 optional = true
 version = "0.5.8"
@@ -66,7 +71,7 @@ version = "0.24.6"
 rustc_version = "0.4.0"
 
 [features]
-binary = ["clap", "glob", "stderrlog"]
+binary = ["clap", "glob", "env_logger"]
 default = ["binary", "filetime", "parallel", "zopfli"]
 parallel = ["rayon", "indexmap/rayon", "crossbeam-channel"]
 freestanding = ["libdeflater/freestanding"]

benches/reductions.rs

@@ -300,6 +300,16 @@ fn reductions_palette_sort(b: &mut Bencher) {
     b.iter(|| palette::sorted_palette(&png.raw));
 }
 
+#[bench]
+fn reductions_palette_sort_battiato(b: &mut Bencher) {
+    let input = test::black_box(PathBuf::from(
+        "tests/files/palette_8_should_be_palette_8.png",
+    ));
+    let png = PngData::new(&input, &Options::default()).unwrap();
+
+    b.iter(|| palette::sorted_palette_battiato(&png.raw));
+}
+
 #[bench]
 fn reductions_alpha(b: &mut Bencher) {
     let input = test::black_box(PathBuf::from("tests/files/rgba_8_reduce_alpha.png"));

src/lib.rs

@@ -160,7 +160,7 @@ pub struct Options {
     ///
     /// `Some(x)` will change the file to interlacing mode `x`.
     ///
-    /// Default: `Some(None)`
+    /// Default: `Some(Interlacing::None)`
     pub interlace: Option<Interlacing>,
     /// Whether to allow transparent pixels to be altered to improve compression.
     pub optimize_alpha: bool,

src/main.rs

@@ -18,7 +18,7 @@ mod rayon;
 
 use clap::{value_parser, Arg, ArgAction, ArgMatches, Command};
 use indexmap::IndexSet;
-use log::{error, warn};
+use log::{error, warn, Level, LevelFilter};
 use oxipng::Deflaters;
 use oxipng::Options;
 use oxipng::RowFilter;
@@ -26,6 +26,7 @@ use oxipng::StripChunks;
 use oxipng::{InFile, OutFile};
 use rayon::prelude::*;
 use std::fs::DirBuilder;
+use std::io::Write;
 #[cfg(feature = "zopfli")]
 use std::num::NonZeroU8;
 use std::path::PathBuf;
@@ -407,13 +408,22 @@ fn apply_glob_pattern(path: PathBuf) -> Vec<PathBuf> {
 fn parse_opts_into_struct(
     matches: &ArgMatches,
 ) -> Result<(OutFile, Option<PathBuf>, Options), String> {
-    stderrlog::new()
+    let log_level = match matches.get_count("verbose") {
-        .module(module_path!())
+        _ if matches.get_flag("quiet") => LevelFilter::Off,
-        .quiet(matches.get_flag("quiet"))
+        0 => LevelFilter::Info,
-        .verbosity(matches.get_count("verbose") as usize + 2)
+        1 => LevelFilter::Debug,
-        .show_level(false)
+        _ => LevelFilter::Trace,
-        .init()
+    };
-        .unwrap();
+    env_logger::builder()
+        .filter_module(module_path!(), log_level)
+        .format(|buf, record| {
+            let style = match record.level() {
+                Level::Info => buf.style(),
+                _ => buf.default_level_style(record.level()),
+            };
+            writeln!(buf, "{}", style.value(record.args()))
+        })
+        .init();
 
     let mut opts = match matches.get_one::<String>("optimization") {
         None => Options::default(),
@@ -467,7 +477,10 @@ fn parse_opts_into_struct(
 
     opts.scale_16 = matches.get_flag("scale16");
 
-    opts.fast_evaluation = matches.get_flag("fast");
+    // The default value for fast depends on the preset - make sure we don't change when not provided
+    if matches.get_flag("fast") {
+        opts.fast_evaluation = matches.get_flag("fast");
+    }
 
     opts.backup = matches.get_flag("backup");
 

src/png/mod.rs

@@ -279,9 +279,8 @@ impl PngImage {
         match &self.ihdr.color_type {
             ColorType::Indexed { palette } => {
                 let plte = 12 + palette.len() * 3;
-                let trns = palette.iter().filter(|p| p.a != 255).count();
+                if let Some(trns) = palette.iter().rposition(|p| p.a != 255) {
-                if trns != 0 {
+                    plte + 12 + trns + 1
-                    plte + 12 + trns
                 } else {
                     plte
                 }

src/reduction/mod.rs

@@ -1,6 +1,7 @@
 use crate::evaluate::Evaluator;
 use crate::png::PngImage;
 use crate::Deadline;
+use crate::Deflaters;
 use crate::Options;
 use std::sync::Arc;
 
@@ -21,6 +22,13 @@ pub(crate) fn perform_reductions(
 ) -> Arc<PngImage> {
     let mut evaluation_added = false;
 
+    // At low compression levels, skip some transformations which are less likely to be effective
+    // This currently affects optimization presets 0-2
+    let cheap = match opts.deflate {
+        Deflaters::Libdeflater { compression } => compression < 12 && opts.fast_evaluation,
+        _ => false,
+    };
+
     // Interlacing must be processed first in order to evaluate the rest correctly
     if let Some(interlacing) = opts.interlace {
         if let Some(reduced) = png.change_interlacing(interlacing) {
@@ -98,7 +106,7 @@ pub(crate) fn perform_reductions(
 
     // Attempt to convert from indexed to channels
     // This may give a better result due to dropping the PLTE chunk
-    if opts.color_type_reduction && !deadline.passed() {
+    if !cheap && opts.color_type_reduction && !deadline.passed() {
         if let Some(reduced) = indexed_to_channels(&png, opts.grayscale_reduction) {
             // This result should not be passed on to subsequent reductions
             eval.try_image(Arc::new(reduced));
@@ -124,6 +132,14 @@ pub(crate) fn perform_reductions(
         }
     }
 
+    // Attempt to sort the palette using an alternative method
+    if !cheap && opts.palette_reduction && !deadline.passed() {
+        if let Some(reduced) = sorted_palette_battiato(&png) {
+            eval.try_image(Arc::new(reduced));
+            evaluation_added = true;
+        }
+    }
+
     // Attempt to reduce to a lower bit depth
     if opts.bit_depth_reduction && !deadline.passed() {
         // Try reducing the previous png, falling back to the indexed one if it exists

src/reduction/palette.rs

@@ -1,6 +1,8 @@
 use crate::colors::{BitDepth, ColorType};
 use crate::headers::IhdrData;
+use crate::png::scan_lines::ScanLine;
 use crate::png::PngImage;
+use crate::Interlacing;
 use indexmap::IndexSet;
 use rgb::RGBA8;
 
@@ -69,24 +71,30 @@ fn add_color_to_set(mut color: RGBA8, set: &mut IndexSet<RGBA8>, optimize_alpha:
     idx as u8
 }
 
-/// Attempt to sort the colors in the palette, returning the sorted image if successful
+/// Attempt to sort the colors in the palette by luma, returning the sorted image if successful
 #[must_use]
 pub fn sorted_palette(png: &PngImage) -> Option<PngImage> {
     if png.ihdr.bit_depth != BitDepth::Eight {
         return None;
     }
     let palette = match &png.ihdr.color_type {
-        ColorType::Indexed { palette } => palette,
+        ColorType::Indexed { palette } if palette.len() > 1 => palette,
         _ => return None,
     };
 
     let mut enumerated: Vec<_> = palette.iter().enumerate().collect();
+    // Put the most popular edge color first, which can help slightly if the filter bytes are 0
+    let keep_first = most_popular_edge_color(palette.len(), png);
+    let first = enumerated.remove(keep_first);
 
     // Sort the palette
     enumerated.sort_by(|a, b| {
         // Sort by ascending alpha and descending luma
         let color_val = |color: &RGBA8| {
-            ((color.a as i32) << 18)
+            let a = i32::from(color.a);
+            // Put 7 high bits of alpha first, then luma, then low bit of alpha
+            // This provides notable improvement in images with a lot of alpha
+            ((a & 0xFE) << 18) + (a & 0x01)
             // These are coefficients for standard sRGB to luma conversion
             - i32::from(color.r) * 299
             - i32::from(color.g) * 587
@@ -94,6 +102,7 @@ pub fn sorted_palette(png: &PngImage) -> Option<PngImage> {
         };
         color_val(a.1).cmp(&color_val(b.1))
     });
+    enumerated.insert(0, first);
 
     // Extract the new palette and determine if anything changed
     let (old_map, palette): (Vec<_>, Vec<RGBA8>) = enumerated.into_iter().unzip();
@@ -116,3 +125,178 @@ pub fn sorted_palette(png: &PngImage) -> Option<PngImage> {
         data,
     })
 }
+
+/// Sort the colors in the palette by minimizing entropy, returning the sorted image if successful
+#[must_use]
+pub fn sorted_palette_battiato(png: &PngImage) -> Option<PngImage> {
+    // Interlacing not currently supported
+    if png.ihdr.bit_depth != BitDepth::Eight || png.ihdr.interlaced != Interlacing::None {
+        return None;
+    }
+    let palette = match &png.ihdr.color_type {
+        // Images with only two colors will remain unchanged from previous luma sort
+        ColorType::Indexed { palette } if palette.len() > 2 => palette,
+        _ => return None,
+    };
+
+    let matrix = co_occurrence_matrix(palette.len(), png);
+    let edges = weighted_edges(&matrix);
+    let mut old_map = battiato_tsp(palette.len(), edges);
+
+    // Put the most popular edge color first, which can help slightly if the filter bytes are 0
+    let keep_first = most_popular_edge_color(palette.len(), png);
+    let first_idx = old_map.iter().position(|&i| i == keep_first).unwrap();
+    // If the index is past halfway, reverse the order so as to minimize the change
+    if first_idx >= old_map.len() / 2 {
+        old_map.reverse();
+        old_map.rotate_right(first_idx + 1);
+    } else {
+        old_map.rotate_left(first_idx);
+    }
+
+    // Check if anything changed
+    if old_map.iter().enumerate().all(|(a, b)| a == *b) {
+        return None;
+    }
+
+    // Construct the palette and byte maps and convert the data
+    let mut new_palette = Vec::new();
+    let mut byte_map = [0; 256];
+    for (i, &v) in old_map.iter().enumerate() {
+        new_palette.push(palette[v]);
+        byte_map[v] = i as u8;
+    }
+    let data = png.data.iter().map(|&b| byte_map[b as usize]).collect();
+
+    Some(PngImage {
+        ihdr: IhdrData {
+            color_type: ColorType::Indexed {
+                palette: new_palette,
+            },
+            ..png.ihdr
+        },
+        data,
+    })
+}
+
+// Find the most popular color on the image edges (the pixels neighboring the filter bytes)
+fn most_popular_edge_color(num_colors: usize, png: &PngImage) -> usize {
+    let mut counts = vec![0; num_colors];
+    for line in png.scan_lines(false) {
+        counts[line.data[0] as usize] += 1;
+        counts[line.data[line.data.len() - 1] as usize] += 1;
+    }
+    counts.iter().enumerate().max_by_key(|(_, &v)| v).unwrap().0
+}
+
+// Calculate co-occurences matrix
+fn co_occurrence_matrix(num_colors: usize, png: &PngImage) -> Vec<Vec<usize>> {
+    let mut matrix = vec![vec![0; num_colors]; num_colors];
+    let mut prev: Option<ScanLine> = None;
+    for line in png.scan_lines(false) {
+        for i in 0..line.data.len() {
+            let val = line.data[i] as usize;
+            if i > 0 {
+                matrix[line.data[i - 1] as usize][val] += 1;
+            }
+            if let Some(prev) = &prev {
+                matrix[prev.data[i] as usize][val] += 1;
+            }
+        }
+        prev = Some(line)
+    }
+    matrix
+}
+
+// Calculate edge list sorted by weight
+fn weighted_edges(matrix: &[Vec<usize>]) -> Vec<(usize, usize)> {
+    let mut edges = Vec::new();
+    for i in 0..matrix.len() {
+        for j in 0..i {
+            edges.push(((j, i), matrix[i][j] + matrix[j][i]));
+        }
+    }
+    edges.sort_by(|(_, w1), (_, w2)| w2.cmp(w1));
+    edges.into_iter().map(|(e, _)| e).collect()
+}
+
+// Calculate an approximate solution of the Traveling Salesman Problem using the algorithm
+// from "An efficient Re-indexing algorithm for color-mapped images" by Battiato et al
+// https://ieeexplore.ieee.org/document/1344033
+fn battiato_tsp(num_colors: usize, edges: Vec<(usize, usize)>) -> Vec<usize> {
+    let mut chains = Vec::new();
+    // Keep track of the state of each vertex (.0) and it's chain number (.1)
+    // 0 = an unvisited vertex (White)
+    // 1 = an endpoint of a chain (Red)
+    // 2 = part of the middle of a chain (Black)
+    let mut vx = vec![(0, 0); num_colors];
+
+    // Iterate the edges and assemble them into a chain
+    for (i, j) in edges {
+        let vi = vx[i];
+        let vj = vx[j];
+        if vi.0 == 0 && vj.0 == 0 {
+            // Two unvisited vertices - create a new chain
+            vx[i].0 = 1;
+            vx[i].1 = chains.len();
+            vx[j].0 = 1;
+            vx[j].1 = chains.len();
+            chains.push(vec![i, j]);
+        } else if vi.0 == 0 && vj.0 == 1 {
+            // An unvisited vertex connects with an endpoint of an existing chain
+            vx[i].0 = 1;
+            vx[i].1 = vj.1;
+            vx[j].0 = 2;
+            let chain = &mut chains[vj.1];
+            if chain[0] == j {
+                chain.insert(0, i);
+            } else {
+                chain.push(i);
+            }
+        } else if vi.0 == 1 && vj.0 == 0 {
+            // An unvisited vertex connects with an endpoint of an existing chain
+            vx[j].0 = 1;
+            vx[j].1 = vi.1;
+            vx[i].0 = 2;
+            let chain = &mut chains[vi.1];
+            if chain[0] == i {
+                chain.insert(0, j);
+            } else {
+                chain.push(j);
+            }
+        } else if vi.0 == 1 && vj.0 == 1 && vi.1 != vj.1 {
+            // Two endpoints of different chains are connected together
+            vx[i].0 = 2;
+            vx[j].0 = 2;
+            let (a, b) = if vi.1 < vj.1 { (i, j) } else { (j, i) };
+            let ca = vx[a].1;
+            let cb = vx[b].1;
+            let chainb = std::mem::take(&mut chains[cb]);
+            for &v in &chainb {
+                vx[v].1 = ca;
+            }
+            let chaina = &mut chains[ca];
+            if chaina[0] == a && chainb[0] == b {
+                for v in chainb {
+                    chaina.insert(0, v);
+                }
+            } else if chaina[0] == a {
+                chaina.splice(0..0, chainb);
+            } else if chainb[0] == b {
+                chaina.extend(chainb);
+            } else {
+                let pos = chaina.len();
+                for v in chainb {
+                    chaina.insert(pos, v);
+                }
+            }
+        }
+
+        if chains[0].len() == num_colors {
+            break;
+        }
+    }
+
+    // Return the completed chain
+    chains.swap_remove(0)
+}

tests/flags.rs

@@ -11,6 +11,7 @@ use std::ops::Deref;
 use std::path::Path;
 use std::path::PathBuf;
 
+const GRAYSCALE: u8 = 0;
 const RGB: u8 = 2;
 const INDEXED: u8 = 3;
 const RGBA: u8 = 6;
@@ -18,6 +19,7 @@ const RGBA: u8 = 6;
 fn get_opts(input: &Path) -> (OutFile, oxipng::Options) {
     let mut options = oxipng::Options {
         force: true,
+        fast_evaluation: false,
         ..Default::default()
     };
     let mut filter = IndexSet::new();
@@ -597,7 +599,7 @@ fn fix_errors() {
         }
     };
 
-    assert_eq!(png.raw.ihdr.color_type.png_header_code(), INDEXED);
+    assert_eq!(png.raw.ihdr.color_type.png_header_code(), GRAYSCALE);
     assert_eq!(png.raw.ihdr.bit_depth, BitDepth::Eight);
 
     // Cannot check if pixels are equal because image crate cannot read corrupt (input) PNGs

tests/interlaced.rs

@@ -14,6 +14,8 @@ const RGBA: u8 = 6;
 fn get_opts(input: &Path) -> (OutFile, oxipng::Options) {
     let mut options = oxipng::Options {
         force: true,
+        fast_evaluation: false,
+        interlace: None,
         ..Default::default()
     };
     let mut filter = IndexSet::new();

tests/reduction.rs

@@ -14,6 +14,7 @@ const RGBA: u8 = 6;
 fn get_opts(input: &Path) -> (OutFile, oxipng::Options) {
     let mut options = oxipng::Options {
         force: true,
+        fast_evaluation: false,
         ..Default::default()
     };
     let mut filter = IndexSet::new();