Extract index modules into specialized workspace subcrates

This commit partitions the obikindex crate into multiple focused subcrates (obikfilter, obikmerge, obikquery, obikrebuild, obikselect, obikstats, obikdump, and obikidxcache) to reduce coupling and clarify module boundaries. It standardizes error handling across the workspace using OKIError and OKIResult, updates index APIs to support lazy, disk-backed partition access, and migrates NUMA system utilities to a new obisys crate. All modifications are structural, focusing on dependency graph expansion, import path updates, and API surface reorganization without altering core runtime behavior.

src/obikstats/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+name = "obikstats"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+obikindex     = { path = "../obikindex" }
+obicompactvec = { path = "../obicompactvec" }
+rayon         = "1"

src/obikstats/src/lib.rs

@@ -0,0 +1,10 @@
+//! Read-only aggregation/statistics over an already-built
+//! `obikindex::KmerIndex`: bits-per-kmer breakdown by index component
+//! (`bits_per_kmer`), per-genome k-mer counts (`genome_kmer_counts`).
+//! Not part of the `Index { Partition { Layer } }` data model itself —
+//! kept out of `obikindex` like every other read/write extension
+//! (`obikquery`, `obikdump`, `obikselect`, `obikrebuild`, `obikmerge`).
+
+mod stats;
+
+pub use stats::IndexBitsPerKmer;

src/obikstats/src/stats.rs

@@ -0,0 +1,188 @@
+use std::fs;
+use std::path::Path;
+
+use obicompactvec::{LayerMeta, PersistentBitMatrix, PersistentCompactIntMatrix};
+use obicompactvec::traits::ColumnWeights;
+use rayon::prelude::*;
+
+use obikindex::{OKIError, OKIResult};
+use obikindex::KmerIndex;
+
+/// Bits per kmer broken down by index component.
+pub struct IndexBitsPerKmer {
+    /// Total distinct k-mers across all partitions and layers.
+    pub n_kmers:          usize,
+    /// Number of genomes in the index.
+    pub n_genomes:        usize,
+    /// Bits used by the minimal perfect hash function (`mphf.bin`).
+    pub mphf:             f64,
+    /// Bits used by the evidence files (`evidence.bin`, `unitigs.bin*`,
+    /// `fingerprint.bin`).
+    pub evidence:         f64,
+    /// Bits used by the count/presence matrices (`counts/` and `presence/`),
+    /// normalised by k-mers only.
+    pub matrix:           f64,
+    /// `matrix` divided by the number of genomes — intrinsic encoding
+    /// efficiency, independent of index size.
+    pub matrix_per_genome: f64,
+    /// Sum of mphf + evidence + matrix.
+    pub total:            f64,
+}
+
+// ── File-size helpers ─────────────────────────────────────────────────────────
+
+fn file_bytes(path: &Path) -> u64 {
+    fs::metadata(path).map(|m| m.len()).unwrap_or(0)
+}
+
+fn dir_bytes(dir: &Path) -> u64 {
+    if !dir.exists() { return 0; }
+    fs::read_dir(dir)
+        .map(|entries| {
+            entries
+                .filter_map(|e| e.ok())
+                .filter_map(|e| e.metadata().ok())
+                .filter(|m| m.is_file())
+                .map(|m| m.len())
+                .sum()
+        })
+        .unwrap_or(0)
+}
+
+// ── Per-layer accounting ──────────────────────────────────────────────────────
+
+struct LayerBytes {
+    n_kmers:  usize,
+    mphf:     u64,
+    evidence: u64,
+    matrix:   u64,
+}
+
+fn layer_bytes(layer_dir: &Path) -> LayerBytes {
+    let n_kmers = LayerMeta::load(layer_dir).map(|m| m.n).unwrap_or(0);
+
+    let mphf = file_bytes(&layer_dir.join("mphf.bin"));
+
+    let evidence = file_bytes(&layer_dir.join("unitigs.bin"))
+        + file_bytes(&layer_dir.join("unitigs.bin.idx"))
+        + file_bytes(&layer_dir.join("evidence.bin"))
+        + file_bytes(&layer_dir.join("fingerprint.bin"));
+
+    let matrix = dir_bytes(&layer_dir.join("counts"))
+        + dir_bytes(&layer_dir.join("presence"));
+
+    LayerBytes { n_kmers, mphf, evidence, matrix }
+}
+
+// ── KmerIndex::bits_per_kmer ──────────────────────────────────────────────────
+
+impl KmerIndex {
+    /// Compute bits-per-kmer statistics for the built index.
+    ///
+    /// File sizes are read directly from disk; kmer counts come from
+    /// `layer_meta.json` (no need to scan the MPHF or unitig files).
+    /// Computation is parallelised across partitions.
+    pub fn bits_per_kmer(&self) -> OKIResult<IndexBitsPerKmer> {
+        let n          = self.n_partitions();
+        let n_genomes  = self.meta().genomes().map_err(OKIError::Io)?.len().max(1);
+
+        let (n_kmers, mphf_b, evidence_b, matrix_b) = (0..n)
+            .into_par_iter()
+            .map(|i| {
+                let index_dir = self.index_dir(i);
+                if !index_dir.exists() { return (0usize, 0u64, 0u64, 0u64); }
+
+                let n_layers = self.n_layers(i).unwrap_or(0);
+
+                (0..n_layers).fold((0usize, 0u64, 0u64, 0u64), |acc, l| {
+                    let Ok(layer_dir) = self.layer_dir(i, l) else { return acc };
+                    let lb = layer_bytes(&layer_dir);
+                    (acc.0 + lb.n_kmers, acc.1 + lb.mphf, acc.2 + lb.evidence, acc.3 + lb.matrix)
+                })
+            })
+            .reduce(|| (0, 0, 0, 0), |a, b| (a.0 + b.0, a.1 + b.1, a.2 + b.2, a.3 + b.3));
+
+        if n_kmers == 0 {
+            return Ok(IndexBitsPerKmer {
+                n_kmers: 0, n_genomes,
+                mphf: 0.0, evidence: 0.0,
+                matrix: 0.0, matrix_per_genome: 0.0, total: 0.0,
+            });
+        }
+
+        let bpk    = |bytes: u64| bytes as f64 * 8.0 / n_kmers as f64;
+        let matrix = bpk(matrix_b);
+
+        Ok(IndexBitsPerKmer {
+            n_kmers,
+            n_genomes,
+            mphf:              bpk(mphf_b),
+            evidence:          bpk(evidence_b),
+            matrix,
+            matrix_per_genome: matrix / n_genomes as f64,
+            total:             bpk(mphf_b + evidence_b + matrix_b),
+        })
+    }
+
+    /// Return `(total_distinct_kmers, per_genome_kmer_counts)`.
+    ///
+    /// For each genome, the count is the number of distinct k-mers for which
+    /// that genome has a non-zero value (presence = 1, count > 0).
+    /// Partitions are scanned in parallel; results are summed across partitions.
+    pub fn genome_kmer_counts(&self) -> OKIResult<(usize, Vec<u64>)> {
+        let n          = self.n_partitions();
+        let n_genomes  = self.meta.genomes().map_err(OKIError::Io)?.len();
+
+        let partials: Vec<(usize, Vec<u64>)> = (0..n)
+            .into_par_iter()
+            .map(|i| {
+                let mut counts  = vec![0u64; n_genomes];
+                let mut n_kmers = 0usize;
+
+                let index_dir = self.index_dir(i);
+                if !index_dir.exists() { return (0, counts); }
+
+                let n_layers = self.n_layers(i).unwrap_or(0);
+
+                for l in 0..n_layers {
+                    let Ok(this_layer_dir) = self.layer_dir(i, l) else { continue };
+                    if !this_layer_dir.exists() { continue; }
+
+                    n_kmers += LayerMeta::load(&this_layer_dir).map(|m| m.n).unwrap_or(0);
+
+                    let mat: Box<dyn ColumnWeights> =
+                        if this_layer_dir.join("counts").exists()
+                            && !this_layer_dir.join("presence").exists()
+                        {
+                            match obikindex::layer::open_data::<PersistentCompactIntMatrix>(&index_dir, l) {
+                                Ok(m)  => Box::new(m),
+                                Err(_) => continue,
+                            }
+                        } else {
+                            match obikindex::layer::open_data::<PersistentBitMatrix>(&index_dir, l) {
+                                Ok(m)  => Box::new(m),
+                                Err(_) => continue,
+                            }
+                        };
+                    let col_counts = mat.partial_kmer_counts();
+
+                    for (c, &v) in col_counts.iter().enumerate() {
+                        if c < n_genomes { counts[c] += v; }
+                    }
+                }
+
+                (n_kmers, counts)
+            })
+            .collect();
+
+        let total_kmers: usize = partials.iter().map(|(n, _)| n).sum();
+        let mut total_counts   = vec![0u64; n_genomes];
+        for (_, counts) in partials {
+            for (i, v) in counts.into_iter().enumerate() {
+                total_counts[i] += v;
+            }
+        }
+
+        Ok((total_kmers, total_counts))
+    }
+}