refactor: relocate partition iterator and expose graph builder API
Move the partition iterator implementation from obikdump to obikfilter, updating imports and exposing the type publicly. Add obikidxcache as a local dependency for obikfilter. In obikindexer, expose the new build_layer_from_kmers function to enable shared graph-building logic across pipelines without code duplication.
This commit is contained in:
1 parent
16ade823d6
commit
294f132a0a
8 files changed
+35
-14
No files matched your search
@@ -10,3 +10,4 @@ obikseq = { path = "../obikseq" }
|
||||
obiskio = { path = "../obiskio" }
|
||||
obitaxonomy = { path = "../obitaxonomy" }
|
||||
obikentropy = { path = "../obikentropy" }
|
||||
obikidxcache = { path = "../obikidxcache" }
|
||||
@@ -3,19 +3,20 @@
|
||||
//! Orthogonal to genome-column selection/aggregation (`obikselect`), which
|
||||
//! operates on already-retained k-mers.
|
||||
//!
|
||||
//! [`filter`] (the `KmerFilter` trait + its implementations) depends only
|
||||
//! on `obicompactvec`/`obikseq`, not on `obikindex`. The partition/layer
|
||||
//! iteration that actually runs these filters over an index
|
||||
//! (`iter_partition_kmers`, `iter_partition_kmers_located`) lives in
|
||||
//! `obikdump` instead (`FilteredPartitionIter`) — it needs `obikidxcache`'s
|
||||
//! `IndexCache` to read a *complete* source index, which this crate has no
|
||||
//! reason to depend on.
|
||||
//! [`filter`] (the `KmerFilter` trait + its implementations) is the pure
|
||||
//! judging logic. [`partition_iter`] (`FilteredPartitionIter`) is the read
|
||||
//! side that actually runs those filters over a *complete* source index's
|
||||
//! partitions/layers, batched through `obikidxcache::IndexCache` for
|
||||
//! locality — used both by `obikdump` (CSV export) and by this crate's own
|
||||
//! `Filter` algorithm (index-to-index rebuild).
|
||||
|
||||
mod filter;
|
||||
mod partition_iter;
|
||||
mod predicate;
|
||||
|
||||
pub use filter::{
|
||||
GroupQuorumFilter, KmerFilter, MaxGenomeCount, MaxGenomeFraction, MaxTotalCount,
|
||||
MinComplexity, MinGenomeCount, MinGenomeFraction, MinTotalCount, passes_all,
|
||||
};
|
||||
pub use partition_iter::FilteredPartitionIter;
|
||||
pub use predicate::{GenomeSelector, GroupFilterParams, MetaPred, Selection};
|
||||
@@ -0,0 +1,126 @@
|
||||
//! Filtered, batch-oriented iteration over an already-cached index's
|
||||
//! partitions/layers — the read side of [`crate::KmerFilter`]s.
|
||||
//! `IndexCache` is a foreign type (`obikidxcache`), so this is an extension
|
||||
//! trait rather than an inherent `impl`.
|
||||
//!
|
||||
//! Only meaningful on a *complete* source index: `IndexCache` panics if a
|
||||
//! layer is missing, which a finished index never has. Never use this on a
|
||||
//! destination index still being built (see `obikmerge::partition_merge`,
|
||||
//! which follows the same source-only-cache rule).
|
||||
|
||||
use obikindex::OKIResult;
|
||||
use obikindex::layer::{KmerLayer, LayerContent};
|
||||
use obikidxcache::index_cache::IndexCache;
|
||||
use obikseq::CanonicalKmer;
|
||||
|
||||
use crate::filter::{KmerFilter, passes_all};
|
||||
|
||||
/// Kmers pulled per batch from a layer before filtering — keeps matrix reads
|
||||
/// grouped by (partition, layer) for locality instead of hopping row to row
|
||||
/// across the index. Same convention as `obikphylo::siblings::build`.
|
||||
const BATCH_SIZE: usize = 32768;
|
||||
|
||||
pub trait FilteredPartitionIter {
|
||||
/// Iterate all indexed kmers in partition `part`, calling `cb(kmer, row)` for each
|
||||
/// kmer that passes every filter in `filters`.
|
||||
///
|
||||
/// `use_counts = true` → reads count columns (u32 values per genome), only
|
||||
/// meaningful for `Count` layers. `use_counts = false` → reads presence
|
||||
/// columns, converted to 0/1 u32 (works for both `Count` and `Presence`
|
||||
/// layers — counts collapse to presence).
|
||||
///
|
||||
/// Returns `Ok(true)` if all kmers were visited, `Ok(false)` if the callback halted.
|
||||
fn iter_partition_kmers(
|
||||
&self,
|
||||
part: usize,
|
||||
use_counts: bool,
|
||||
n_genomes: usize,
|
||||
filters: &[Box<dyn KmerFilter>],
|
||||
cb: impl FnMut(CanonicalKmer, Box<[u32]>) -> bool,
|
||||
) -> OKIResult<bool>;
|
||||
|
||||
/// Like [`iter_partition_kmers`](Self::iter_partition_kmers) but the callback
|
||||
/// also receives `(partition, layer)` indices, enabling debug output that
|
||||
/// identifies where each kmer was stored.
|
||||
fn iter_partition_kmers_located(
|
||||
&self,
|
||||
part: usize,
|
||||
use_counts: bool,
|
||||
n_genomes: usize,
|
||||
filters: &[Box<dyn KmerFilter>],
|
||||
cb: impl FnMut(usize, usize, CanonicalKmer, Box<[u32]>) -> bool,
|
||||
) -> OKIResult<bool>;
|
||||
}
|
||||
|
||||
impl FilteredPartitionIter for IndexCache<'_> {
|
||||
fn iter_partition_kmers(
|
||||
&self,
|
||||
part: usize,
|
||||
use_counts: bool,
|
||||
n_genomes: usize,
|
||||
filters: &[Box<dyn KmerFilter>],
|
||||
mut cb: impl FnMut(CanonicalKmer, Box<[u32]>) -> bool,
|
||||
) -> OKIResult<bool> {
|
||||
for l in 0..self.n_layer(part).unwrap_or(0) {
|
||||
let layer = self.get_layer(part, l).expect("layer within n_layer(part)");
|
||||
if !iter_layer_kmers(layer, use_counts, n_genomes, filters, &mut |kmer, row| cb(kmer, row))? {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
fn iter_partition_kmers_located(
|
||||
&self,
|
||||
part: usize,
|
||||
use_counts: bool,
|
||||
n_genomes: usize,
|
||||
filters: &[Box<dyn KmerFilter>],
|
||||
mut cb: impl FnMut(usize, usize, CanonicalKmer, Box<[u32]>) -> bool,
|
||||
) -> OKIResult<bool> {
|
||||
for l in 0..self.n_layer(part).unwrap_or(0) {
|
||||
let layer = self.get_layer(part, l).expect("layer within n_layer(part)");
|
||||
if !iter_layer_kmers(layer, use_counts, n_genomes, filters, &mut |kmer, row| cb(part, l, kmer, row))? {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
}
|
||||
|
||||
/// Batch-and-transpose one layer's kmers into per-kmer filtered rows.
|
||||
/// Returns `Ok(false)` if `cb` asked to stop early.
|
||||
fn iter_layer_kmers(
|
||||
layer: &KmerLayer,
|
||||
use_counts: bool,
|
||||
n_genomes: usize,
|
||||
filters: &[Box<dyn KmerFilter>],
|
||||
cb: &mut dyn FnMut(CanonicalKmer, Box<[u32]>) -> bool,
|
||||
) -> OKIResult<bool> {
|
||||
let read_counts = use_counts && matches!(layer.content(), LayerContent::Count);
|
||||
|
||||
for kmers in layer.iter_kmers_batch(BATCH_SIZE) {
|
||||
// Kmers come straight from this layer's own iterator, so every one
|
||||
// is a guaranteed member — a raw hash is enough, no membership
|
||||
// recheck needed (see `KmerLayer::hash_batch`'s own doc).
|
||||
let slots = layer.hash_batch(&kmers);
|
||||
|
||||
let cols: Vec<Vec<u32>> = if read_counts {
|
||||
let mut cols: Vec<Vec<u32>> = vec![Vec::new(); n_genomes];
|
||||
layer.fill_sub_matrix(&slots, &mut cols);
|
||||
cols
|
||||
} else {
|
||||
let mut bool_cols: Vec<Vec<bool>> = vec![Vec::new(); n_genomes];
|
||||
layer.fill_sub_matrix_carries(&slots, &mut bool_cols);
|
||||
bool_cols.iter().map(|c| c.iter().map(|&b| b as u32).collect()).collect()
|
||||
};
|
||||
|
||||
for (i, kmer) in kmers.into_iter().enumerate() {
|
||||
let row: Box<[u32]> = cols.iter().map(|c| c[i]).collect();
|
||||
if passes_all(filters, kmer, &row, n_genomes) && !cb(kmer, row) {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
Reference in new issue
Block a user