Add bigram entropy filter

Cargo.lock

@@ -392,6 +392,7 @@ dependencies = [
  "log",
  "rayon",
  "rgb",
+ "rustc-hash",
  "rustc_version",
  "stderrlog",
  "wild",
@@ -458,6 +459,12 @@ dependencies = [
  "bytemuck",
 ]
 
+[[package]]
+name = "rustc-hash"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
+
 [[package]]
 name = "rustc_version"
 version = "0.4.0"

Cargo.toml

@@ -33,6 +33,7 @@ log = "0.4.17"
 stderrlog = { version = "0.5.3", optional = true, default-features = false }
 crossbeam-channel = "0.5.6"
 bitvec = "1.0.1"
+rustc-hash = "1.1.0"
 
 [dependencies.filetime]
 optional = true

src/filters.rs

@@ -15,6 +15,7 @@ pub enum RowFilter {
     MinSum,
     Entropy,
     Bigrams,
+    BigEnt,
 }
 
 impl TryFrom<u8> for RowFilter {
@@ -42,13 +43,14 @@ impl Display for RowFilter {
                 Self::MinSum => "MinSum",
                 Self::Entropy => "Entropy",
                 Self::Bigrams => "Bigrams",
+                Self::BigEnt => "BigEnt",
             }
         )
     }
 }
 
 impl RowFilter {
-    pub const LAST: u8 = Self::Bigrams as u8;
+    pub const LAST: u8 = Self::BigEnt as u8;
     pub const STANDARD: [Self; 5] = [Self::None, Self::Sub, Self::Up, Self::Average, Self::Paeth];
     pub const SINGLE_LINE: [Self; 2] = [Self::None, Self::Sub];
 

src/png/mod.rs

@@ -8,6 +8,7 @@ use bitvec::bitarr;
 use indexmap::IndexMap;
 use rgb::ComponentSlice;
 use rgb::RGBA8;
+use rustc_hash::FxHashMap;
 use std::fs::File;
 use std::io::{BufReader, Read, Write};
 use std::iter::Iterator;
@@ -386,6 +387,25 @@ impl PngImage {
                             }
                         }
                     }
+                    RowFilter::BigEnt => {
+                        // Bigram entropy, combined from Entropy and Bigrams filters
+                        let mut best_size = i32::MIN;
+                        // FxHasher is the fastest rust hasher currently available for this purpose
+                        let mut counts = FxHashMap::<u16, u32>::default();
+                        for try_filter in try_filters {
+                            try_filter.filter_line(bpp, line.data, last_line, &mut f_buf);
+                            counts.clear();
+                            for i in 1..f_buf.len() {
+                                let bigram = (f_buf[i - 1] as u16) << 8 | f_buf[i] as u16;
+                                counts.entry(bigram).and_modify(|e| *e += 1).or_insert(1);
+                            }
+                            let size = counts.values().fold(0, |acc, &x| acc + ilog2i(x)) as i32;
+                            if size > best_size {
+                                best_size = size;
+                                std::mem::swap(&mut best_line, &mut f_buf);
+                            }
+                        }
+                    }
                     _ => unreachable!(),
                 }
                 filtered.extend_from_slice(&best_line);