refactor: relocate partition iterator and expose graph builder API

Move the partition iterator implementation from obikdump to obikfilter, updating imports and exposing the type publicly. Add obikidxcache as a local dependency for obikfilter. In obikindexer, expose the new build_layer_from_kmers function to enable shared graph-building logic across pipelines without code duplication.
This commit is contained in:
Eric Coissac committed 2026-08-26 09:39:02 +02:00
1 parent 16ade823d6
commit 294f132a0a
8 files changed
+35 -14

No files matched your search

+1
View File
@@ -10,3 +10,4 @@ obikseq = { path = "../obikseq" }
obiskio = { path = "../obiskio" }
obitaxonomy = { path = "../obitaxonomy" }
obikentropy = { path = "../obikentropy" }
obikidxcache = { path = "../obikidxcache" }
+8 -7
View File
@@ -3,19 +3,20 @@
//! Orthogonal to genome-column selection/aggregation (`obikselect`), which
//! operates on already-retained k-mers.
//!
//! [`filter`] (the `KmerFilter` trait + its implementations) depends only
//! on `obicompactvec`/`obikseq`, not on `obikindex`. The partition/layer
//! iteration that actually runs these filters over an index
//! (`iter_partition_kmers`, `iter_partition_kmers_located`) lives in
//! `obikdump` instead (`FilteredPartitionIter`) — it needs `obikidxcache`'s
//! `IndexCache` to read a *complete* source index, which this crate has no
//! reason to depend on.
//! [`filter`] (the `KmerFilter` trait + its implementations) is the pure
//! judging logic. [`partition_iter`] (`FilteredPartitionIter`) is the read
//! side that actually runs those filters over a *complete* source index's
//! partitions/layers, batched through `obikidxcache::IndexCache` for
//! locality — used both by `obikdump` (CSV export) and by this crate's own
//! `Filter` algorithm (index-to-index rebuild).
mod filter;
mod partition_iter;
mod predicate;
pub use filter::{
GroupQuorumFilter, KmerFilter, MaxGenomeCount, MaxGenomeFraction, MaxTotalCount,
MinComplexity, MinGenomeCount, MinGenomeFraction, MinTotalCount, passes_all,
};
pub use partition_iter::FilteredPartitionIter;
pub use predicate::{GenomeSelector, GroupFilterParams, MetaPred, Selection};
+126
View File
@@ -0,0 +1,126 @@
//! Filtered, batch-oriented iteration over an already-cached index's
//! partitions/layers — the read side of [`crate::KmerFilter`]s.
//! `IndexCache` is a foreign type (`obikidxcache`), so this is an extension
//! trait rather than an inherent `impl`.
//!
//! Only meaningful on a *complete* source index: `IndexCache` panics if a
//! layer is missing, which a finished index never has. Never use this on a
//! destination index still being built (see `obikmerge::partition_merge`,
//! which follows the same source-only-cache rule).
use obikindex::OKIResult;
use obikindex::layer::{KmerLayer, LayerContent};
use obikidxcache::index_cache::IndexCache;
use obikseq::CanonicalKmer;
use crate::filter::{KmerFilter, passes_all};
/// Kmers pulled per batch from a layer before filtering — keeps matrix reads
/// grouped by (partition, layer) for locality instead of hopping row to row
/// across the index. Same convention as `obikphylo::siblings::build`.
const BATCH_SIZE: usize = 32768;
pub trait FilteredPartitionIter {
/// Iterate all indexed kmers in partition `part`, calling `cb(kmer, row)` for each
/// kmer that passes every filter in `filters`.
///
/// `use_counts = true` → reads count columns (u32 values per genome), only
/// meaningful for `Count` layers. `use_counts = false` → reads presence
/// columns, converted to 0/1 u32 (works for both `Count` and `Presence`
/// layers — counts collapse to presence).
///
/// Returns `Ok(true)` if all kmers were visited, `Ok(false)` if the callback halted.
fn iter_partition_kmers(
&self,
part: usize,
use_counts: bool,
n_genomes: usize,
filters: &[Box<dyn KmerFilter>],
cb: impl FnMut(CanonicalKmer, Box<[u32]>) -> bool,
) -> OKIResult<bool>;
/// Like [`iter_partition_kmers`](Self::iter_partition_kmers) but the callback
/// also receives `(partition, layer)` indices, enabling debug output that
/// identifies where each kmer was stored.
fn iter_partition_kmers_located(
&self,
part: usize,
use_counts: bool,
n_genomes: usize,
filters: &[Box<dyn KmerFilter>],
cb: impl FnMut(usize, usize, CanonicalKmer, Box<[u32]>) -> bool,
) -> OKIResult<bool>;
}
impl FilteredPartitionIter for IndexCache<'_> {
fn iter_partition_kmers(
&self,
part: usize,
use_counts: bool,
n_genomes: usize,
filters: &[Box<dyn KmerFilter>],
mut cb: impl FnMut(CanonicalKmer, Box<[u32]>) -> bool,
) -> OKIResult<bool> {
for l in 0..self.n_layer(part).unwrap_or(0) {
let layer = self.get_layer(part, l).expect("layer within n_layer(part)");
if !iter_layer_kmers(layer, use_counts, n_genomes, filters, &mut |kmer, row| cb(kmer, row))? {
return Ok(false);
}
}
Ok(true)
}
fn iter_partition_kmers_located(
&self,
part: usize,
use_counts: bool,
n_genomes: usize,
filters: &[Box<dyn KmerFilter>],
mut cb: impl FnMut(usize, usize, CanonicalKmer, Box<[u32]>) -> bool,
) -> OKIResult<bool> {
for l in 0..self.n_layer(part).unwrap_or(0) {
let layer = self.get_layer(part, l).expect("layer within n_layer(part)");
if !iter_layer_kmers(layer, use_counts, n_genomes, filters, &mut |kmer, row| cb(part, l, kmer, row))? {
return Ok(false);
}
}
Ok(true)
}
}
/// Batch-and-transpose one layer's kmers into per-kmer filtered rows.
/// Returns `Ok(false)` if `cb` asked to stop early.
fn iter_layer_kmers(
layer: &KmerLayer,
use_counts: bool,
n_genomes: usize,
filters: &[Box<dyn KmerFilter>],
cb: &mut dyn FnMut(CanonicalKmer, Box<[u32]>) -> bool,
) -> OKIResult<bool> {
let read_counts = use_counts && matches!(layer.content(), LayerContent::Count);
for kmers in layer.iter_kmers_batch(BATCH_SIZE) {
// Kmers come straight from this layer's own iterator, so every one
// is a guaranteed member — a raw hash is enough, no membership
// recheck needed (see `KmerLayer::hash_batch`'s own doc).
let slots = layer.hash_batch(&kmers);
let cols: Vec<Vec<u32>> = if read_counts {
let mut cols: Vec<Vec<u32>> = vec![Vec::new(); n_genomes];
layer.fill_sub_matrix(&slots, &mut cols);
cols
} else {
let mut bool_cols: Vec<Vec<bool>> = vec![Vec::new(); n_genomes];
layer.fill_sub_matrix_carries(&slots, &mut bool_cols);
bool_cols.iter().map(|c| c.iter().map(|&b| b as u32).collect()).collect()
};
for (i, kmer) in kmers.into_iter().enumerate() {
let row: Box<[u32]> = cols.iter().map(|c| c[i]).collect();
if passes_all(filters, kmer, &row, n_genomes) && !cb(kmer, row) {
return Ok(false);
}
}
}
Ok(true)
}