Add unified matrix builder abstraction and k-mer filtering CLI

Introduces a unified MatrixBuilder abstraction for persistent bit and integer matrices, replacing custom dispatch enums and boolean flags across consumers. Refactors index, merge, and select layers to adopt explicit merge modes and centralized lifecycle management. Adds a complete k-mer filtering implementation with partition-level processing, progress tracking, and a new CLI subcommand supporting configurable predicates and thresholds.
This commit is contained in:
Eric Coissac committed 2026-08-26 14:19:15 +02:00
1 parent b7a8b5e6cf
commit 354e6f9bf1
17 files changed
+588 -125

No files matched your search

+5
View File
@@ -11,3 +11,8 @@ obiskio = { path = "../obiskio" }
obitaxonomy = { path = "../obitaxonomy" }
obikentropy = { path = "../obikentropy" }
obikidxcache = { path = "../obikidxcache" }
obikindexer = { path = "../obikindexer" }
obidebruinj = { path = "../obidebruinj" }
obikalgorithm = { path = "../obikalgorithm" }
obisys = { path = "../obisys" }
tracing = "0.1.44"
+141
View File
@@ -0,0 +1,141 @@
//! Removing kmers from a source `obikindex::KmerIndex`, according to a list
//! of [`crate::KmerFilter`]s. Structured exactly like `obikselect::Select`/
//! `obikmerge::Merge`: two-phase construction (`new` + setters), then
//! `obikalgorithm::Algorithm::run`, via `obikindex::IndexBuilder`
//! (`create_skeleton` + `finalize_indexed`) — no in-place variant, same as
//! `Select`/`Merge`. Unlike `Select`, every surviving layer's identity
//! (unitigs/MPHF/evidence) is rebuilt from scratch, not copied — see
//! [`crate::filter_partition`].
use std::io;
use std::path::PathBuf;
use obikalgorithm::Algorithm;
use obikindex::{IndexBuilder, IndexState, KmerIndex};
use obisys::{PartitionRunner, Progress, Reporter, Stage};
use tracing::info;
use crate::filter::KmerFilter;
use crate::filter_partition::filter_partition;
pub struct Filter<'a> {
src: &'a KmerIndex,
output: PathBuf,
filters: &'a [Box<dyn KmerFilter>],
presence: bool,
force: bool,
sparse: bool,
reporter: Reporter,
on_progress: Option<Box<dyn FnMut(Progress) + Send + 'a>>,
}
impl<'a> Filter<'a> {
pub fn new(src: &'a KmerIndex, output: impl Into<PathBuf>, filters: &'a [Box<dyn KmerFilter>]) -> Self {
Self {
src,
output: output.into(),
filters,
presence: false,
force: false,
sparse: true,
reporter: Reporter::new(),
on_progress: None,
}
}
/// Output presence/absence instead of counts, even if the source has
/// count data (default: `false` — keep the source's own content kind).
pub fn presence(mut self, v: bool) -> Self {
self.presence = v;
self
}
/// Remove a pre-existing index at `output` before creating the
/// filtered one (default: `false`, fails if one already exists).
pub fn force(mut self, v: bool) -> Self {
self.force = v;
self
}
/// Pack the output's presence matrices in the compact sparse format
/// rather than dense (default: `true`).
pub fn sparse(mut self, v: bool) -> Self {
self.sparse = v;
self
}
/// Progress callback, called once per completed partition.
pub fn on_progress(mut self, cb: impl FnMut(Progress) + Send + 'a) -> Self {
self.on_progress = Some(Box::new(cb));
self
}
/// Per-stage timing report accumulated during `run` — call after `run` returns.
pub fn reporter(&self) -> &Reporter {
&self.reporter
}
}
impl Algorithm for Filter<'_> {
type Output = KmerIndex;
fn run(&mut self) -> obikalgorithm::Result<KmerIndex> {
let src = self.src;
let output = self.output.clone();
if src.state()? != IndexState::Indexed {
return Err(format!("{}: source index is not fully built", src.dir().display()).into());
}
let genomes = src.meta().genomes()?;
let n_genomes = genomes.len();
let use_counts = !self.presence && src.meta().config.with_counts;
info!(
"filter: {} partition(s), {} genome(s), output={}",
src.n_partitions(),
n_genomes,
if use_counts { "count" } else { "presence" },
);
KmerIndex::clear_output_for_create(&output, self.force)?;
let mut config = src.meta().config.clone();
config.with_counts = use_counts;
let block_bits = src.block_bits();
let mode = src.evidence_mode().clone();
let t = Stage::start("filter");
let dst = KmerIndex::create_skeleton(&output, config, genomes)?;
let n_partitions = src.n_partitions();
let order: Vec<usize> = (0..n_partitions).collect();
let runner = PartitionRunner::new();
let on_progress = &mut self.on_progress;
let mut done: u64 = 0;
let filters = self.filters;
runner
.run(
&order,
|i| filter_partition(&dst, src, i, filters, use_counts, !use_counts, block_bits, &mode),
|_, _, _| {
done += 1;
if let Some(cb) = on_progress.as_mut() {
cb(Progress {
position: done,
total: Some(n_partitions as u64),
});
}
},
)
.map_err(|e| io::Error::other(e.to_string()))?;
self.reporter.push(t.stop());
let t = Stage::start("finalize");
let dst = KmerIndex::finalize_indexed(&output, &mut self.reporter, self.sparse)?;
self.reporter.push(t.stop());
Ok(dst)
}
}
+77
View File
@@ -0,0 +1,77 @@
//! Per-partition rebuild — the mechanics [`crate::filter_algo::Filter`]
//! runs once per partition. Unlike `obikselect::select_partition` (columns
//! only, kmer identity copied as-is), this removes kmers, so every layer's
//! identity (unitigs + MPHF + evidence) has to be rebuilt from scratch, not
//! copied — removing kmers changes MPHF slot assignment layer-wide.
use std::collections::HashMap;
use obidebruinj::GraphDeBruijn;
use obikidxcache::index_cache::IndexCache;
use obikindex::layer::{IndexMode, KmerLayer, TypedLayer};
use obikindex::{KmerIndex, OKIError, OKIResult};
use obikindexer::write_graph_as_unitigs;
use obikseq::CanonicalKmer;
use crate::filter::KmerFilter;
use crate::partition_iter::FilteredPartitionIter;
/// Rewrite partition `i` from `src` into `dst`, keeping only kmers that pass
/// every filter in `filters`.
///
/// `use_counts` selects which values are read from `src` (count vs
/// presence-collapsed); `output_presence` selects the destination's own
/// matrix kind — independent, since `--presence` can force a presence
/// output from a count source.
pub(crate) fn filter_partition(
dst: &KmerIndex,
src: &KmerIndex,
i: usize,
filters: &[Box<dyn KmerFilter>],
use_counts: bool,
output_presence: bool,
block_bits: u8,
mode: &IndexMode,
) -> OKIResult<()> {
let n_genomes = src.meta().genomes().map_err(OKIError::Io)?.len();
let cache = IndexCache::new(src, Some(vec![i]));
let dst_partition = dst.partition(i)?;
let n_layers = src.partition(i)?.n_layers();
// Bucket survivors by layer — bounded to one partition's worth of data,
// the same order of magnitude `IndexCache` already holds open for this
// partition.
let mut by_layer: HashMap<usize, HashMap<CanonicalKmer, Box<[u32]>>> = HashMap::new();
cache.iter_partition_kmers_located(i, use_counts, n_genomes, filters, |_part, layer, kmer, row| {
by_layer.entry(layer).or_default().insert(kmer, row);
true
})?;
for l in 0..n_layers {
let survivors = by_layer.remove(&l).unwrap_or_default();
let dst_layer_dir = KmerLayer::new(&dst_partition, l)
.create()
.map_err(OKIError::Io)?
.dir()
.to_path_buf();
let mut g = GraphDeBruijn::new();
for &kmer in survivors.keys() {
g.push(kmer);
}
write_graph_as_unitigs(g, &dst_layer_dir)?;
TypedLayer::<()>::build_with_matrix(
&dst_layer_dir,
block_bits,
mode,
output_presence,
n_genomes,
|kmer| survivors.get(&kmer).cloned().expect("kmer pushed into the graph must be a survivor"),
)?;
}
Ok(())
}
+3
View File
@@ -11,6 +11,8 @@
//! `Filter` algorithm (index-to-index rebuild).
mod filter;
mod filter_algo;
mod filter_partition;
mod partition_iter;
mod predicate;
@@ -18,5 +20,6 @@ pub use filter::{
GroupQuorumFilter, KmerFilter, MaxGenomeCount, MaxGenomeFraction, MaxTotalCount,
MinComplexity, MinGenomeCount, MinGenomeFraction, MinTotalCount, passes_all,
};
pub use filter_algo::Filter;
pub use partition_iter::FilteredPartitionIter;
pub use predicate::{GenomeSelector, GroupFilterParams, MetaPred, Selection};