src/Cargo.lock

@@ -1682,6 +1682,13 @@ dependencies = [
  "xxhash-rust",
 ]
 
+[[package]]
+name = "obikentropy"
+version = "0.1.0"
+dependencies = [
+ "obikseq",
+]
+
 [[package]]
 name = "obikindex"
 version = "0.1.0"
@@ -1742,6 +1749,7 @@ dependencies = [
  "niffler 3.0.0",
  "obicompactvec",
  "obidebruinj",
+ "obikentropy",
  "obikrope",
  "obikseq",
  "obilayeredmap",
@@ -1824,7 +1832,9 @@ dependencies = [
 name = "obiskbuilder"
 version = "0.1.0"
 dependencies = [
+ "criterion2",
  "lazy_static",
+ "obikentropy",
  "obikrope",
  "obikseq",
  "obiread",

src/Cargo.toml

@@ -1,5 +1,5 @@
 [workspace]
 resolver = "3"
-members = ["obikseq", "obiread", "obiskbuilder", "obifastwrite", "obikmer","obikrope","obipipeline", "obikpartitionner","obiskio","obidebruinj","obilayeredmap", "obicompactvec", "obisys", "obikindex", "obitaxonomy"]
+members = ["obikseq", "obiread", "obiskbuilder", "obifastwrite", "obikmer","obikrope","obipipeline", "obikpartitionner","obiskio","obidebruinj","obilayeredmap", "obicompactvec", "obisys", "obikindex", "obitaxonomy", "obikentropy"]
 [profile.release]
 debug = 1

src/obikentropy/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "obikentropy"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+obikseq = { path = "../obikseq" }
+
+[dev-dependencies]
+obikseq = { path = "../obikseq", features = ["test-utils"] }

src/obikentropy/src/kmer_entropy.rs

@@ -0,0 +1,41 @@
+//! Normalized entropy of an isolated, already-built k-mer (e.g. one
+//! reconstructed from an index's `unitigs.bin`, with no surrounding
+//! sequence) — drives the window through [`EntropyTracker`] one base at a
+//! time, exactly like the streaming path, so a `theta` threshold means the
+//! same thing whether applied during index construction or after the fact
+//! (e.g. `obikmer filter`).
+
+use obikseq::CanonicalKmer;
+
+use crate::tracker::{EntropyTracker, sub_word_canon};
+
+/// Extension trait: compute the normalized entropy of a single canonical
+/// k-mer, independent of any surrounding sequence.
+pub trait KmerEntropy {
+    /// Normalized entropy across sub-word orders `1..=level_max` (the
+    /// minimum is taken across orders). Lower means less complex; `theta`
+    /// in `index`/`filter` rejects k-mers with a score `< theta`.
+    fn entropy(&self, level_max: usize) -> f64;
+}
+
+impl KmerEntropy for CanonicalKmer {
+    fn entropy(&self, level_max: usize) -> f64 {
+        let raw = self.raw(); // left-aligned, 2 bits/base, MSB-first
+        let k = obikseq::params::k();
+        let mask = (!0u64) >> (64 - k * 2);
+
+        let mut tracker = EntropyTracker::new(k);
+        let mut rolling: u64 = 0;
+        for i in 0..k {
+            let shift = 64 - 2 * (i + 1);
+            let base = (raw >> shift) & 3;
+            rolling = ((rolling << 2) | base) & mask;
+            tracker.push(i + 1, sub_word_canon(rolling));
+        }
+        tracker.normalized_entropy(level_max)
+    }
+}
+
+#[cfg(test)]
+#[path = "tests/kmer_entropy.rs"]
+mod tests;

src/obikentropy/src/lib.rs

@@ -0,0 +1,17 @@
+//! Normalized k-mer entropy: formulas, tables, and a streaming tracker.
+//!
+//! This crate holds every piece of the entropy computation described in
+//! `docmd/theory/entropy.md`: the compile-time tables ([`table`], private),
+//! the incremental accumulator ([`EntropyTracker`]) that callers compose
+//! into their own streaming state, and the [`KmerEntropy`] convenience trait
+//! for scoring a single, already-built k-mer.
+
+#![deny(missing_docs)]
+
+mod kmer_entropy;
+mod ring;
+mod table;
+mod tracker;
+
+pub use kmer_entropy::KmerEntropy;
+pub use tracker::{EntropyTracker, SubWordCanon, sub_word_canon};

src/obikentropy/src/ring.rs

@@ -0,0 +1,40 @@
+//! Stack-allocated ring buffer backing the sliding sub-word windows.
+
+/// Fixed-capacity ring buffer backed by a stack array.
+/// N must be a power of two; operations are branchless via `% N`.
+pub(crate) struct Ring<T: Copy + Default, const N: usize> {
+    buf: [T; N],
+    head: usize,
+    len: usize,
+}
+
+impl<T: Copy + Default, const N: usize> Ring<T, N> {
+    #[inline]
+    pub(crate) fn new() -> Self {
+        Self {
+            buf: [T::default(); N],
+            head: 0,
+            len: 0,
+        }
+    }
+
+    #[inline]
+    pub(crate) fn clear(&mut self) {
+        self.len = 0;
+        self.head = 0;
+    }
+
+    #[inline]
+    pub(crate) fn push_back(&mut self, val: T) {
+        self.buf[(self.head + self.len) % N] = val;
+        self.len += 1;
+    }
+
+    #[inline]
+    pub(crate) fn pop_front(&mut self) -> T {
+        let val = self.buf[self.head];
+        self.head = (self.head + 1) % N;
+        self.len -= 1;
+        val
+    }
+}

src/obikentropy/src/table.rs

@@ -1,3 +1,7 @@
+//! Compile-time tables backing the normalized k-mer entropy formula:
+//! circular-canonical sub-word classes, their log-sizes, and the max-entropy
+//! correction for small samples. See `docmd/theory/entropy.md`.
+
 pub(crate) const NORMK1: [u64; 4] = build_normalized_kmer::<4>();
 pub(crate) const NORMK2: [u64; 16] = build_normalized_kmer::<16>();
 pub(crate) const NORMK3: [u64; 64] = build_normalized_kmer::<64>();

src/obikentropy/src/tests/kmer_entropy.rs

@@ -3,12 +3,10 @@ use obikseq::Sequence;
 use obikseq::kmer::Kmer;
 
 const K: usize = 21;
-const M: usize = 9; // RollingStat also tracks the minimizer window internally
 const LEVEL_MAX: usize = 6;
 
 fn kmer_from_ascii(seq: &[u8]) -> CanonicalKmer {
     obikseq::set_k(K);
-    obikseq::set_m(M);
     Kmer::from_ascii(seq).expect("valid k-mer sequence").canonical()
 }
 

src/obikentropy/src/tracker.rs

@@ -0,0 +1,280 @@
+//! Incremental (streaming) normalized k-mer entropy.
+//!
+//! [`EntropyTracker`] maintains, over a sliding window of the last `k` bases,
+//! the per-sub-word-size frequency statistics needed to evaluate the
+//! corrected Shannon entropy described in `docmd/theory/entropy.md`, updated
+//! in O(1) per base rather than recomputed from scratch.
+//!
+//! It carries no notion of minimizers or superkmer segmentation — callers
+//! that need both (e.g. `obiskbuilder::RollingStat`) compose an
+//! `EntropyTracker` as a plain field alongside their own state, so the two
+//! concerns update in the same streaming pass without being conflated in one
+//! struct.
+
+use crate::ring::Ring;
+use crate::table::{WS_MAX, emax, entropy_norm_kmer, ln_class_size, log_nwords, n_log_n};
+
+/// Canonical sub-word values for word sizes 1..=6, computed by the caller
+/// from its own rolling k-mer bits (see [`EntropyTracker::push`]).
+pub type SubWordCanon = [u64; WS_MAX];
+
+/// Compute the six canonical sub-word values (word sizes 1..=6) for the
+/// current rolling right-aligned k-mer window.
+#[inline]
+pub fn sub_word_canon(rolling_kmer: u64) -> SubWordCanon {
+    [
+        entropy_norm_kmer::<false, 1>(rolling_kmer & 3),
+        entropy_norm_kmer::<false, 2>(rolling_kmer & 15),
+        entropy_norm_kmer::<false, 3>(rolling_kmer & 63),
+        entropy_norm_kmer::<false, 4>(rolling_kmer & 255),
+        entropy_norm_kmer::<false, 5>(rolling_kmer & 1023),
+        entropy_norm_kmer::<false, 6>(rolling_kmer & 4095),
+    ]
+}
+
+/// Incremental normalized-entropy accumulator over a sliding window of `k`
+/// bases. Composed as a plain field by callers that also need other
+/// per-base state (e.g. minimizer selection) in the same streaming pass.
+pub struct EntropyTracker {
+    k: usize,
+    steady: bool,
+
+    // Sliding-window queues over the last `k` canonical sub-words, one per
+    // word size — stack-allocated, capacity ≤ k ≤ 31.
+    k1q: Ring<u64, 32>,
+    k2q: Ring<u64, 32>,
+    k3q: Ring<u64, 32>,
+    k4q: Ring<u64, 32>,
+    k5q: Ring<u64, 32>,
+    k6q: Ring<u64, 32>,
+
+    // Frequency count arrays. Max count per cell ≤ k ≤ 31 → u8 is sufficient.
+    k1c: [u8; 4],
+    k2c: [u8; 16],
+    k3c: [u8; 64],
+    k4c: [u8; 256],
+    k5c: [u8; 1024],
+    k6c: [u8; 4096],
+
+    sum_f_log_f: [f64; WS_MAX + 1],
+    sum_f_log_s: [f64; WS_MAX + 1],
+}
+
+impl EntropyTracker {
+    /// New tracker for a window of `k` bases (1..=31).
+    pub fn new(k: usize) -> Self {
+        Self {
+            k,
+            steady: false,
+            k1q: Ring::new(),
+            k2q: Ring::new(),
+            k3q: Ring::new(),
+            k4q: Ring::new(),
+            k5q: Ring::new(),
+            k6q: Ring::new(),
+            k1c: [0; 4],
+            k2c: [0; 16],
+            k3c: [0; 64],
+            k4c: [0; 256],
+            k5c: [0; 1024],
+            k6c: [0; 4096],
+            sum_f_log_f: [0.0; WS_MAX + 1],
+            sum_f_log_s: [0.0; WS_MAX + 1],
+        }
+    }
+
+    /// Clear all accumulated state, ready to track a new window from
+    /// scratch (`k` is unchanged).
+    pub fn reset(&mut self) {
+        self.steady = false;
+
+        self.k1c.fill(0);
+        self.k2c.fill(0);
+        self.k3c.fill(0);
+        self.k4c.fill(0);
+        self.k5c.fill(0);
+        self.k6c.fill(0);
+
+        self.k1q.clear();
+        self.k2q.clear();
+        self.k3q.clear();
+        self.k4q.clear();
+        self.k5q.clear();
+        self.k6q.clear();
+
+        self.sum_f_log_f = [0.0; WS_MAX + 1];
+        self.sum_f_log_s = [0.0; WS_MAX + 1];
+    }
+
+    #[inline]
+    fn update_sums_decrement<const K: usize>(
+        sum_f_log_f: &mut [f64; WS_MAX + 1],
+        sum_f_log_s: &mut [f64; WS_MAX + 1],
+        canonical: u64,
+        f: usize,
+    ) {
+        sum_f_log_f[K] += n_log_n(f - 1) - n_log_n(f);
+        sum_f_log_s[K] -= ln_class_size::<false, K>(canonical);
+    }
+
+    #[inline]
+    fn update_sums_increment<const K: usize>(
+        sum_f_log_f: &mut [f64; WS_MAX + 1],
+        sum_f_log_s: &mut [f64; WS_MAX + 1],
+        canonical: u64,
+        g: usize,
+    ) {
+        sum_f_log_f[K] += n_log_n(g + 1) - n_log_n(g);
+        sum_f_log_s[K] += ln_class_size::<false, K>(canonical);
+    }
+
+    /// Advance the window by one base. `received` is the caller's running
+    /// count of bases pushed so far (1-based, i.e. after this base);
+    /// `canon` are the six canonical sub-word values for the current
+    /// rolling k-mer, from [`sub_word_canon`].
+    pub fn push(&mut self, received: usize, canon: SubWordCanon) {
+        let [canonical_k1, canonical_k2, canonical_k3, canonical_k4, canonical_k5, canonical_k6] =
+            canon;
+
+        if received > self.k {
+            let old1 = self.k1q.pop_front();
+            let f1 = self.k1c[old1 as usize] as usize;
+            Self::update_sums_decrement::<1>(&mut self.sum_f_log_f, &mut self.sum_f_log_s, old1, f1);
+            self.k1c[old1 as usize] -= 1;
+
+            let old2 = self.k2q.pop_front();
+            let f2 = self.k2c[old2 as usize] as usize;
+            Self::update_sums_decrement::<2>(&mut self.sum_f_log_f, &mut self.sum_f_log_s, old2, f2);
+            self.k2c[old2 as usize] -= 1;
+
+            let old3 = self.k3q.pop_front();
+            let f3 = self.k3c[old3 as usize] as usize;
+            Self::update_sums_decrement::<3>(&mut self.sum_f_log_f, &mut self.sum_f_log_s, old3, f3);
+            self.k3c[old3 as usize] -= 1;
+
+            let old4 = self.k4q.pop_front();
+            let f4 = self.k4c[old4 as usize] as usize;
+            Self::update_sums_decrement::<4>(&mut self.sum_f_log_f, &mut self.sum_f_log_s, old4, f4);
+            self.k4c[old4 as usize] -= 1;
+
+            let old5 = self.k5q.pop_front();
+            let f5 = self.k5c[old5 as usize] as usize;
+            Self::update_sums_decrement::<5>(&mut self.sum_f_log_f, &mut self.sum_f_log_s, old5, f5);
+            self.k5c[old5 as usize] -= 1;
+
+            let old6 = self.k6q.pop_front();
+            let f6 = self.k6c[old6 as usize] as usize;
+            Self::update_sums_decrement::<6>(&mut self.sum_f_log_f, &mut self.sum_f_log_s, old6, f6);
+            self.k6c[old6 as usize] -= 1;
+        }
+
+        if self.steady {
+            let g1 = self.k1c[canonical_k1 as usize] as usize;
+            Self::update_sums_increment::<1>(&mut self.sum_f_log_f, &mut self.sum_f_log_s, canonical_k1, g1);
+            self.k1c[canonical_k1 as usize] += 1;
+            self.k1q.push_back(canonical_k1);
+
+            let g2 = self.k2c[canonical_k2 as usize] as usize;
+            Self::update_sums_increment::<2>(&mut self.sum_f_log_f, &mut self.sum_f_log_s, canonical_k2, g2);
+            self.k2c[canonical_k2 as usize] += 1;
+            self.k2q.push_back(canonical_k2);
+
+            let g3 = self.k3c[canonical_k3 as usize] as usize;
+            Self::update_sums_increment::<3>(&mut self.sum_f_log_f, &mut self.sum_f_log_s, canonical_k3, g3);
+            self.k3c[canonical_k3 as usize] += 1;
+            self.k3q.push_back(canonical_k3);
+
+            let g4 = self.k4c[canonical_k4 as usize] as usize;
+            Self::update_sums_increment::<4>(&mut self.sum_f_log_f, &mut self.sum_f_log_s, canonical_k4, g4);
+            self.k4c[canonical_k4 as usize] += 1;
+            self.k4q.push_back(canonical_k4);
+
+            let g5 = self.k5c[canonical_k5 as usize] as usize;
+            Self::update_sums_increment::<5>(&mut self.sum_f_log_f, &mut self.sum_f_log_s, canonical_k5, g5);
+            self.k5c[canonical_k5 as usize] += 1;
+            self.k5q.push_back(canonical_k5);
+
+            let g6 = self.k6c[canonical_k6 as usize] as usize;
+            Self::update_sums_increment::<6>(&mut self.sum_f_log_f, &mut self.sum_f_log_s, canonical_k6, g6);
+            self.k6c[canonical_k6 as usize] += 1;
+            self.k6q.push_back(canonical_k6);
+        } else {
+            self.push_warmup_increments(received, canonical_k1, canonical_k2, canonical_k3, canonical_k4, canonical_k5, canonical_k6);
+        }
+    }
+
+    #[cold]
+    #[inline(never)]
+    fn push_warmup_increments(
+        &mut self,
+        received: usize,
+        canonical_k1: u64, canonical_k2: u64, canonical_k3: u64,
+        canonical_k4: u64, canonical_k5: u64, canonical_k6: u64,
+    ) {
+        let g1 = self.k1c[canonical_k1 as usize] as usize;
+        Self::update_sums_increment::<1>(&mut self.sum_f_log_f, &mut self.sum_f_log_s, canonical_k1, g1);
+        self.k1c[canonical_k1 as usize] += 1;
+        self.k1q.push_back(canonical_k1);
+
+        if received >= 2 {
+            let g2 = self.k2c[canonical_k2 as usize] as usize;
+            Self::update_sums_increment::<2>(&mut self.sum_f_log_f, &mut self.sum_f_log_s, canonical_k2, g2);
+            self.k2c[canonical_k2 as usize] += 1;
+            self.k2q.push_back(canonical_k2);
+
+            if received >= 3 {
+                let g3 = self.k3c[canonical_k3 as usize] as usize;
+                Self::update_sums_increment::<3>(&mut self.sum_f_log_f, &mut self.sum_f_log_s, canonical_k3, g3);
+                self.k3c[canonical_k3 as usize] += 1;
+                self.k3q.push_back(canonical_k3);
+
+                if received >= 4 {
+                    let g4 = self.k4c[canonical_k4 as usize] as usize;
+                    Self::update_sums_increment::<4>(&mut self.sum_f_log_f, &mut self.sum_f_log_s, canonical_k4, g4);
+                    self.k4c[canonical_k4 as usize] += 1;
+                    self.k4q.push_back(canonical_k4);
+
+                    if received >= 5 {
+                        let g5 = self.k5c[canonical_k5 as usize] as usize;
+                        Self::update_sums_increment::<5>(&mut self.sum_f_log_f, &mut self.sum_f_log_s, canonical_k5, g5);
+                        self.k5c[canonical_k5 as usize] += 1;
+                        self.k5q.push_back(canonical_k5);
+
+                        if received >= 6 {
+                            let g6 = self.k6c[canonical_k6 as usize] as usize;
+                            Self::update_sums_increment::<6>(&mut self.sum_f_log_f, &mut self.sum_f_log_s, canonical_k6, g6);
+                            self.k6c[canonical_k6 as usize] += 1;
+                            self.k6q.push_back(canonical_k6);
+                            self.steady = true;
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    /// Normalized entropy at sub-word size `order` (1..=6). The caller is
+    /// responsible for not calling this before the window is full (`k`
+    /// bases pushed) — an empty/partial window yields a meaningless value.
+    pub fn entropy(&self, order: usize) -> f64 {
+        let k = self.k;
+        let em = emax(k, order);
+        if em <= 0.0 {
+            return 1.0;
+        }
+        let nwords = k - order + 1;
+        let log_nw = log_nwords(k, order);
+        let nw_f = nwords as f64;
+        let h_corr = log_nw + (self.sum_f_log_s[order] - self.sum_f_log_f[order]) / nw_f;
+        (h_corr / em).max(0.0)
+    }
+
+    /// Minimum of [`Self::entropy`] over sub-word sizes `1..=order_max`, same
+    /// caller responsibility re: window readiness as `entropy`.
+    pub fn normalized_entropy(&self, order_max: usize) -> f64 {
+        let min_e = (1..=order_max)
+            .map(|ws| self.entropy(ws))
+            .fold(f64::MAX, f64::min);
+        if min_e == f64::MAX { 1.0 } else { min_e }
+    }
+}

src/obikpartitionner/Cargo.toml

@@ -13,6 +13,7 @@ obikrope  = { path = "../obikrope" }
 niffler = "3.0.0"
 remove_dir_all = "0.8"
 obikseq      = { path = "../obikseq" }
+obikentropy  = { path = "../obikentropy" }
 obiskbuilder = { path = "../obiskbuilder" }
 obiskio      = { path = "../obiskio" }
 obidebruinj  = { path = "../obidebruinj" }

src/obikpartitionner/src/filter.rs

@@ -277,7 +277,7 @@ impl KmerFilter for GroupQuorumFilter {
 
 /// Reject k-mers with normalized entropy below `theta` — the same complexity
 /// metric `obikmer index`'s `--theta`/`--level-max` apply *during* superkmer
-/// construction (see [`obiskbuilder::KmerEntropy`]), applied here after the
+/// construction (see [`obikentropy::KmerEntropy`]), applied here after the
 /// fact, to k-mers already committed to a built index.
 ///
 /// Unlike every other filter in this module, this one needs the k-mer's own
@@ -292,7 +292,7 @@ pub struct MinComplexity {
 
 impl KmerFilter for MinComplexity {
     fn passes(&self, kmer: CanonicalKmer, _row: &[u32], _n_genomes: usize) -> bool {
-        use obiskbuilder::KmerEntropy;
+        use obikentropy::KmerEntropy;
         kmer.entropy(self.level_max) >= self.theta
     }
 }

src/obiskbuilder/Cargo.toml

@@ -7,7 +7,13 @@ edition = "2024"
 obikseq = { path = "../obikseq" }
 obikrope = { path = "../obikrope" }
 obiread = { path = "../obiread" }
+obikentropy = { path = "../obikentropy" }
 lazy_static = "1.5.0"
 
 [dev-dependencies]
 obikseq = { path = "../obikseq", features = ["test-utils"] }
+criterion2 = { version = "3", features = ["cargo_bench_support"] }
+
+[[bench]]
+name = "superkmer_stream"
+harness = false

src/obiskbuilder/benches/superkmer_stream.rs

@@ -0,0 +1,58 @@
+//! Throughput of the streaming superkmer pipeline (`RollingStat`'s hot path:
+//! minimizer selection + entropy tracking fused in a single pass).
+//!
+//! Reference point for the `obikentropy` extraction: the entropy bookkeeping
+//! that used to live inline in `RollingStat` was pulled out into a composed
+//! `EntropyTracker`. This benchmark is run before and after that change to
+//! confirm no regression.
+
+use criterion::{Criterion, Throughput, criterion_group, criterion_main};
+use obikrope::Rope;
+use obiskbuilder::SuperKmerIter;
+
+const K: usize = 21;
+const M: usize = 9;
+const LEVEL_MAX: usize = 6;
+const THETA: f64 = 0.7;
+const SEQ_LEN: usize = 200_000;
+
+/// Deterministic pseudo-random ACGT sequence — high enough complexity that
+/// the entropy filter rarely rejects, so the bench stays on the steady-state
+/// path rather than repeatedly resetting.
+fn make_sequence(len: usize) -> Vec<u8> {
+    let mut state: u64 = 0x9E3779B97F4A7C15;
+    (0..len)
+        .map(|_| {
+            state ^= state << 13;
+            state ^= state >> 7;
+            state ^= state << 17;
+            b"ACGT"[(state % 4) as usize]
+        })
+        .collect()
+}
+
+fn make_rope(seq: &[u8]) -> Rope {
+    let mut rope = Rope::new(None);
+    rope.push(seq.to_vec());
+    rope
+}
+
+fn bench_build_superkmers(c: &mut Criterion) {
+    obikseq::set_k(K);
+    obikseq::set_m(M);
+
+    let seq = make_sequence(SEQ_LEN);
+    let rope = make_rope(&seq);
+
+    let mut group = c.benchmark_group("build_superkmers");
+    group.throughput(Throughput::Bytes(SEQ_LEN as u64));
+    group.bench_function("stream", |b| {
+        b.iter(|| {
+            SuperKmerIter::new(std::hint::black_box(&rope), K, LEVEL_MAX, THETA).count()
+        });
+    });
+    group.finish();
+}
+
+criterion_group!(benches, bench_build_superkmers);
+criterion_main!(benches);

src/obiskbuilder/src/kmer_entropy.rs

@@ -1,49 +0,0 @@
-//! Normalized entropy of an isolated, already-built k-mer.
-//!
-//! [`SuperKmerIter`](crate::SuperKmerIter) uses [`RollingStat`] to reject
-//! low-complexity k-mers *during* superkmer construction, incrementally, over
-//! a streaming window. [`KmerEntropy`] exposes the same metric for a single
-//! k-mer taken in isolation (e.g. one already reconstructed from an index's
-//! `unitigs.bin`, with no surrounding sequence) — built on the identical
-//! `RollingStat` code path, not a re-derived formula, so a `theta` threshold
-//! chosen for index-build-time filtering means the same thing when applied
-//! after the fact (e.g. `obikmer filter`).
-
-use obikseq::CanonicalKmer;
-
-use crate::rolling_stat::RollingStat;
-
-/// Extension trait: compute the normalized entropy of a single canonical
-/// k-mer, independent of any surrounding sequence.
-pub trait KmerEntropy {
-    /// Normalized entropy across sub-word orders `1..=level_max` (the
-    /// minimum is taken across orders, same as
-    /// [`RollingStat::normalized_entropy`]). Lower means less complex;
-    /// `theta` in `index`/`filter` rejects k-mers with a score `< theta`.
-    ///
-    /// # Panics
-    ///
-    /// Requires both `obikseq::params::k()` *and* `params::m()` to already be
-    /// set (via `set_k`/`set_m`), even though the minimizer length plays no
-    /// conceptual role in an entropy score: `RollingStat` is a shared,
-    /// general-purpose struct (also used for minimizer selection during
-    /// superkmer decomposition) and unconditionally sizes an `m`-dependent
-    /// mask in its constructor. Callers must call `set_m` with a valid value
-    /// (e.g. the index's own stored `minimizer_size`) even when only the
-    /// entropy score is needed.
-    fn entropy(&self, level_max: usize) -> f64;
-}
-
-impl KmerEntropy for CanonicalKmer {
-    fn entropy(&self, level_max: usize) -> f64 {
-        let mut stat = RollingStat::new(level_max);
-        for &base in self.to_ascii().iter() {
-            stat.push(base);
-        }
-        stat.normalized_entropy().unwrap_or(1.0)
-    }
-}
-
-#[cfg(test)]
-#[path = "tests/kmer_entropy.rs"]
-mod tests;

src/obiskbuilder/src/lib.rs

@@ -6,16 +6,13 @@
 #![deny(missing_docs)]
 
 pub mod iter;
-pub mod kmer_entropy;
 pub mod stream_iter;
 mod scratch;
 
 pub(crate) mod encoding;
-pub(crate) mod entropy_table;
 pub(crate) mod rolling_stat;
 
 pub use iter::SuperKmerIter;
-pub use kmer_entropy::KmerEntropy;
 pub use scratch::SuperKmerScratch;
 pub use stream_iter::SuperKmerStreamIter;
 

src/obiskbuilder/src/rolling_stat.rs

@@ -1,8 +1,8 @@
+use obikentropy::{EntropyTracker, sub_word_canon};
 use obikseq::kmer::{Minimizer, hash_kmer};
 use obikseq::params;
 
 use crate::encoding::encode_nuc;
-use crate::entropy_table::{WS_MAX, emax, entropy_norm_kmer, ln_class_size, log_nwords, n_log_n};
 
 // ── Stack-allocated ring buffer ───────────────────────────────────────────────
 
@@ -83,33 +83,19 @@ pub struct RollingStat {
     entropy_max_k: usize,
     k: usize,
     m: usize,
-    steady: bool,
     rolling_k: u64,
     rolling_rck: u64,
     k_mask: u64,
     m_mask: u64,
     received: usize,
 
-    // Sliding-window queues — stack-allocated, capacity ≤ k ≤ 31.
+    // Minimizer selection state.
-    k1q: Ring<u64, 32>,
-    k2q: Ring<u64, 32>,
-    k3q: Ring<u64, 32>,
-    k4q: Ring<u64, 32>,
-    k5q: Ring<u64, 32>,
-    k6q: Ring<u64, 32>,
     minimier: Ring<MmerItem, 32>,
 
-    // Frequency count arrays.
+    // Entropy tracking, composed as a plain inline field so both concerns
-    // Max count per cell ≤ k ≤ 31 → u8 is sufficient.
+    // update in the same streaming pass without being conflated in one
-    k1c: [u8; 4],
+    // struct — see `obikentropy::EntropyTracker`.
-    k2c: [u8; 16],
+    entropy: EntropyTracker,
-    k3c: [u8; 64],
-    k4c: [u8; 256],
-    k5c: [u8; 1024],
-    k6c: [u8; 4096],
-
-    sum_f_log_f: [f64; WS_MAX + 1],
-    sum_f_log_s: [f64; WS_MAX + 1],
 }
 
 impl RollingStat {
@@ -120,27 +106,13 @@ impl RollingStat {
             entropy_max_k,
             k,
             m,
-            steady: false,
             rolling_k: 0,
             rolling_rck: 0,
             k_mask: (!0u64) >> (64 - k * 2),
             m_mask: (!0u64) >> (64 - m * 2),
             received: 0,
-            k1q: Ring::new(),
-            k2q: Ring::new(),
-            k3q: Ring::new(),
-            k4q: Ring::new(),
-            k5q: Ring::new(),
-            k6q: Ring::new(),
             minimier: Ring::new(),
-            k1c: [0; 4],
+            entropy: EntropyTracker::new(k),
-            k2c: [0; 16],
-            k3c: [0; 64],
-            k4c: [0; 256],
-            k5c: [0; 1024],
-            k6c: [0; 4096],
-            sum_f_log_f: [0.0; WS_MAX + 1],
-            sum_f_log_s: [0.0; WS_MAX + 1],
         }
     }
 
@@ -148,54 +120,9 @@ impl RollingStat {
         self.rolling_k = 0;
         self.rolling_rck = 0;
         self.received = 0;
-        self.steady = false;
 
-        //        for i in self.k1q.iter() { self.k1c[i as usize] = 0; }
-        //        for i in self.k2q.iter() { self.k2c[i as usize] = 0; }
-        //        for i in self.k3q.iter() { self.k3c[i as usize] = 0; }
-        //        for i in self.k4q.iter() { self.k4c[i as usize] = 0; }
-        //        for i in self.k5q.iter() { self.k5c[i as usize] = 0; }
-        //        for i in self.k6q.iter() { self.k6c[i as usize] = 0; }
-
-        self.k1c.fill(0);
-        self.k2c.fill(0);
-        self.k3c.fill(0);
-        self.k4c.fill(0);
-        self.k5c.fill(0);
-        self.k6c.fill(0);
-
-        self.k1q.clear();
-        self.k2q.clear();
-        self.k3q.clear();
-        self.k4q.clear();
-        self.k5q.clear();
-        self.k6q.clear();
         self.minimier.clear();
-
+        self.entropy.reset();
-        self.sum_f_log_f = [0.0; WS_MAX + 1];
-        self.sum_f_log_s = [0.0; WS_MAX + 1];
-    }
-
-    #[inline]
-    fn update_sums_decrement<const K: usize>(
-        sum_f_log_f: &mut [f64; WS_MAX + 1],
-        sum_f_log_s: &mut [f64; WS_MAX + 1],
-        canonical: u64,
-        f: usize,
-    ) {
-        sum_f_log_f[K] += n_log_n(f - 1) - n_log_n(f);
-        sum_f_log_s[K] -= ln_class_size::<false, K>(canonical);
-    }
-
-    #[inline]
-    fn update_sums_increment<const K: usize>(
-        sum_f_log_f: &mut [f64; WS_MAX + 1],
-        sum_f_log_s: &mut [f64; WS_MAX + 1],
-        canonical: u64,
-        g: usize,
-    ) {
-        sum_f_log_f[K] += n_log_n(g + 1) - n_log_n(g);
-        sum_f_log_s[K] += ln_class_size::<false, K>(canonical);
     }
 
     pub fn push(&mut self, nuc: u8) {
@@ -209,12 +136,7 @@ impl RollingStat {
         self.rolling_rck =
             ((self.rolling_rck >> 2) | ((cnuc as u64) << ((k - 1) * 2))) & self.k_mask;
 
-        let canonical_k1 = entropy_norm_kmer::<false, 1>(self.rolling_k & 3);
+        let canon = sub_word_canon(self.rolling_k);
-        let canonical_k2 = entropy_norm_kmer::<false, 2>(self.rolling_k & 15);
-        let canonical_k3 = entropy_norm_kmer::<false, 3>(self.rolling_k & 63);
-        let canonical_k4 = entropy_norm_kmer::<false, 4>(self.rolling_k & 255);
-        let canonical_k5 = entropy_norm_kmer::<false, 5>(self.rolling_k & 1023);
-        let canonical_k6 = entropy_norm_kmer::<false, 6>(self.rolling_k & 4095);
 
         self.received += 1;
 
@@ -248,153 +170,7 @@ impl RollingStat {
             }
         }
 
-        if self.received > k {
+        self.entropy.push(self.received, canon);
-            let old1 = self.k1q.pop_front();
-            let f1 = self.k1c[old1 as usize] as usize;
-            Self::update_sums_decrement::<1>(
-                &mut self.sum_f_log_f,
-                &mut self.sum_f_log_s,
-                old1,
-                f1,
-            );
-            self.k1c[old1 as usize] -= 1;
-
-            let old2 = self.k2q.pop_front();
-            let f2 = self.k2c[old2 as usize] as usize;
-            Self::update_sums_decrement::<2>(
-                &mut self.sum_f_log_f,
-                &mut self.sum_f_log_s,
-                old2,
-                f2,
-            );
-            self.k2c[old2 as usize] -= 1;
-
-            let old3 = self.k3q.pop_front();
-            let f3 = self.k3c[old3 as usize] as usize;
-            Self::update_sums_decrement::<3>(
-                &mut self.sum_f_log_f,
-                &mut self.sum_f_log_s,
-                old3,
-                f3,
-            );
-            self.k3c[old3 as usize] -= 1;
-
-            let old4 = self.k4q.pop_front();
-            let f4 = self.k4c[old4 as usize] as usize;
-            Self::update_sums_decrement::<4>(
-                &mut self.sum_f_log_f,
-                &mut self.sum_f_log_s,
-                old4,
-                f4,
-            );
-            self.k4c[old4 as usize] -= 1;
-
-            let old5 = self.k5q.pop_front();
-            let f5 = self.k5c[old5 as usize] as usize;
-            Self::update_sums_decrement::<5>(
-                &mut self.sum_f_log_f,
-                &mut self.sum_f_log_s,
-                old5,
-                f5,
-            );
-            self.k5c[old5 as usize] -= 1;
-
-            let old6 = self.k6q.pop_front();
-            let f6 = self.k6c[old6 as usize] as usize;
-            Self::update_sums_decrement::<6>(
-                &mut self.sum_f_log_f,
-                &mut self.sum_f_log_s,
-                old6,
-                f6,
-            );
-            self.k6c[old6 as usize] -= 1;
-        }
-
-        if self.steady {
-            let g1 = self.k1c[canonical_k1 as usize] as usize;
-            Self::update_sums_increment::<1>(&mut self.sum_f_log_f, &mut self.sum_f_log_s, canonical_k1, g1);
-            self.k1c[canonical_k1 as usize] += 1;
-            self.k1q.push_back(canonical_k1);
-
-            let g2 = self.k2c[canonical_k2 as usize] as usize;
-            Self::update_sums_increment::<2>(&mut self.sum_f_log_f, &mut self.sum_f_log_s, canonical_k2, g2);
-            self.k2c[canonical_k2 as usize] += 1;
-            self.k2q.push_back(canonical_k2);
-
-            let g3 = self.k3c[canonical_k3 as usize] as usize;
-            Self::update_sums_increment::<3>(&mut self.sum_f_log_f, &mut self.sum_f_log_s, canonical_k3, g3);
-            self.k3c[canonical_k3 as usize] += 1;
-            self.k3q.push_back(canonical_k3);
-
-            let g4 = self.k4c[canonical_k4 as usize] as usize;
-            Self::update_sums_increment::<4>(&mut self.sum_f_log_f, &mut self.sum_f_log_s, canonical_k4, g4);
-            self.k4c[canonical_k4 as usize] += 1;
-            self.k4q.push_back(canonical_k4);
-
-            let g5 = self.k5c[canonical_k5 as usize] as usize;
-            Self::update_sums_increment::<5>(&mut self.sum_f_log_f, &mut self.sum_f_log_s, canonical_k5, g5);
-            self.k5c[canonical_k5 as usize] += 1;
-            self.k5q.push_back(canonical_k5);
-
-            let g6 = self.k6c[canonical_k6 as usize] as usize;
-            Self::update_sums_increment::<6>(&mut self.sum_f_log_f, &mut self.sum_f_log_s, canonical_k6, g6);
-            self.k6c[canonical_k6 as usize] += 1;
-            self.k6q.push_back(canonical_k6);
-        } else {
-            self.push_warmup_increments(
-                canonical_k1, canonical_k2, canonical_k3,
-                canonical_k4, canonical_k5, canonical_k6,
-            );
-        }
-    }
-
-    #[cold]
-    #[inline(never)]
-    fn push_warmup_increments(
-        &mut self,
-        canonical_k1: u64, canonical_k2: u64, canonical_k3: u64,
-        canonical_k4: u64, canonical_k5: u64, canonical_k6: u64,
-    ) {
-        let g1 = self.k1c[canonical_k1 as usize] as usize;
-        Self::update_sums_increment::<1>(&mut self.sum_f_log_f, &mut self.sum_f_log_s, canonical_k1, g1);
-        self.k1c[canonical_k1 as usize] += 1;
-        self.k1q.push_back(canonical_k1);
-
-        if self.received >= 2 {
-            let g2 = self.k2c[canonical_k2 as usize] as usize;
-            Self::update_sums_increment::<2>(&mut self.sum_f_log_f, &mut self.sum_f_log_s, canonical_k2, g2);
-            self.k2c[canonical_k2 as usize] += 1;
-            self.k2q.push_back(canonical_k2);
-
-            if self.received >= 3 {
-                let g3 = self.k3c[canonical_k3 as usize] as usize;
-                Self::update_sums_increment::<3>(&mut self.sum_f_log_f, &mut self.sum_f_log_s, canonical_k3, g3);
-                self.k3c[canonical_k3 as usize] += 1;
-                self.k3q.push_back(canonical_k3);
-
-                if self.received >= 4 {
-                    let g4 = self.k4c[canonical_k4 as usize] as usize;
-                    Self::update_sums_increment::<4>(&mut self.sum_f_log_f, &mut self.sum_f_log_s, canonical_k4, g4);
-                    self.k4c[canonical_k4 as usize] += 1;
-                    self.k4q.push_back(canonical_k4);
-
-                    if self.received >= 5 {
-                        let g5 = self.k5c[canonical_k5 as usize] as usize;
-                        Self::update_sums_increment::<5>(&mut self.sum_f_log_f, &mut self.sum_f_log_s, canonical_k5, g5);
-                        self.k5c[canonical_k5 as usize] += 1;
-                        self.k5q.push_back(canonical_k5);
-
-                        if self.received >= 6 {
-                            let g6 = self.k6c[canonical_k6 as usize] as usize;
-                            Self::update_sums_increment::<6>(&mut self.sum_f_log_f, &mut self.sum_f_log_s, canonical_k6, g6);
-                            self.k6c[canonical_k6 as usize] += 1;
-                            self.k6q.push_back(canonical_k6);
-                            self.steady = true;
-                        }
-                    }
-                }
-            }
-        }
     }
 
     pub fn ready(&self) -> bool {
@@ -426,25 +202,13 @@ impl RollingStat {
         if !self.ready() {
             return None;
         }
-        let k = self.k;
+        Some(self.entropy.entropy(order))
-        let em = emax(k, order);
-        if em <= 0.0 {
-            return Some(1.0);
-        }
-        let nwords = k - order + 1;
-        let log_nw = log_nwords(k, order);
-        let nw_f = nwords as f64;
-        let h_corr = log_nw + (self.sum_f_log_s[order] - self.sum_f_log_f[order]) / nw_f;
-        Some((h_corr / em).max(0.0))
     }
 
     pub fn normalized_entropy(&self) -> Option<f64> {
         if !self.ready() {
             return None;
         }
-        let min_e = (1..=self.entropy_max_k)
+        Some(self.entropy.normalized_entropy(self.entropy_max_k))
-            .filter_map(|ws| self.entropy(ws))
-            .fold(f64::MAX, f64::min);
-        Some(if min_e == f64::MAX { 1.0 } else { min_e })
     }
 }