refactor: extract partition iteration and unify group selection
Move partition iteration logic to obikdump, introducing a FilteredPartitionIter trait over IndexCache for batch-oriented scanning with configurable data retrieval and early termination. Consolidate ingroup and outgroup index storage in GroupQuorumFilter into a unified Selection struct driven by predicate matching. Update dependency manifests to include obikidxcache, rayon, and obikentropy, and remove the deprecated dump_layer module while adjusting public API re-exports.
This commit is contained in:
1 parent
881b1532b5
commit
16ade823d6
10 files changed
+256
-291
No files matched your search
@@ -9,3 +9,4 @@ obicompactvec = { path = "../obicompactvec" }
|
||||
obikseq = { path = "../obikseq" }
|
||||
obiskio = { path = "../obiskio" }
|
||||
obitaxonomy = { path = "../obitaxonomy" }
|
||||
obikentropy = { path = "../obikentropy" }
|
||||
@@ -1,192 +0,0 @@
|
||||
use obikindex::layer::MphfLayer;
|
||||
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
|
||||
use obikseq::CanonicalKmer;
|
||||
use obiskio::UnitigFileReader;
|
||||
use obikindex::{OKIError, OKIResult};
|
||||
|
||||
use crate::filter::{KmerFilter, passes_all};
|
||||
use obikindex::KmerIndex;
|
||||
|
||||
impl KmerIndex {
|
||||
/// Iterate all indexed kmers in partition `part`, calling `cb(kmer, row)` for each
|
||||
/// kmer that passes every filter in `filters`.
|
||||
///
|
||||
/// `use_counts = true` → reads count columns (u32 values per genome).
|
||||
/// `use_counts = false` → reads presence columns, converted to 0/1 u32.
|
||||
///
|
||||
/// If no data matrix exists for a layer (pure set-membership, single genome),
|
||||
/// a row of `n_genomes` ones is emitted for every kmer in that layer — unless
|
||||
/// the filter rejects it, in which case the whole layer is skipped.
|
||||
/// Like [`iter_partition_kmers`] but the callback returns `false` to stop early.
|
||||
/// Returns `Ok(true)` if all kmers were visited, `Ok(false)` if the callback halted.
|
||||
pub fn iter_partition_kmers(
|
||||
&self,
|
||||
part: usize,
|
||||
use_counts: bool,
|
||||
n_genomes: usize,
|
||||
filters: &[Box<dyn KmerFilter>],
|
||||
mut cb: impl FnMut(CanonicalKmer, Box<[u32]>) -> bool,
|
||||
) -> OKIResult<bool> {
|
||||
let index_dir = self.index_dir(part);
|
||||
if !index_dir.exists() {
|
||||
return Ok(true);
|
||||
}
|
||||
|
||||
let mut l = 0;
|
||||
loop {
|
||||
let layer_dir = self.layer_dir(part, l)?;
|
||||
if !layer_dir.exists() {
|
||||
break;
|
||||
}
|
||||
l += 1;
|
||||
let mphf = MphfLayer::open(&layer_dir)?;
|
||||
let reader = UnitigFileReader::open_sequential(&layer_dir.join("unitigs.bin"))?;
|
||||
|
||||
let counts_dir = layer_dir.join("counts");
|
||||
let presence_dir = layer_dir.join("presence");
|
||||
|
||||
let cont = if use_counts && counts_dir.exists() {
|
||||
let mat = PersistentCompactIntMatrix::open(&layer_dir).map_err(OKIError::Io)?;
|
||||
let mut cont = true;
|
||||
for (kmer, _, _) in reader.iter_indexed_canonical_kmers() {
|
||||
if let Some(slot) = mphf.find(kmer) {
|
||||
let row = mat.row(slot);
|
||||
if passes_all(filters, kmer, &row, n_genomes) {
|
||||
cont = cb(kmer, row);
|
||||
if !cont {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
cont
|
||||
} else if !use_counts && presence_dir.exists() {
|
||||
let mat = PersistentBitMatrix::open(&layer_dir).map_err(OKIError::Io)?;
|
||||
let mut cont = true;
|
||||
for (kmer, _, _) in reader.iter_indexed_canonical_kmers() {
|
||||
if let Some(slot) = mphf.find(kmer) {
|
||||
let row: Box<[u32]> = mat.row(slot).iter().map(|&b| b as u32).collect();
|
||||
if passes_all(filters, kmer, &row, n_genomes) {
|
||||
cont = cb(kmer, row);
|
||||
if !cont {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
cont
|
||||
} else {
|
||||
// No data matrix: implicit presence — all values are 1. `row`
|
||||
// is identical for every kmer, but a filter can still depend
|
||||
// on the kmer's own sequence (e.g. MinComplexity), so this
|
||||
// cannot be evaluated once for the whole layer — filters must
|
||||
// still be tested per kmer.
|
||||
let all_present: Box<[u32]> = vec![1u32; n_genomes].into();
|
||||
let mut cont = true;
|
||||
for (kmer, _, _) in reader.iter_indexed_canonical_kmers() {
|
||||
if mphf.find(kmer).is_some()
|
||||
&& passes_all(filters, kmer, &all_present, n_genomes)
|
||||
{
|
||||
cont = cb(kmer, all_present.clone());
|
||||
if !cont {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
cont
|
||||
};
|
||||
|
||||
if !cont {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Like [`iter_partition_kmers`] but the callback also receives `(partition, layer)`
|
||||
/// indices, enabling debug output that identifies where each kmer was stored.
|
||||
/// Returns `Ok(true)` if all kmers were visited, `Ok(false)` if the callback halted.
|
||||
pub fn iter_partition_kmers_located(
|
||||
&self,
|
||||
part: usize,
|
||||
use_counts: bool,
|
||||
n_genomes: usize,
|
||||
filters: &[Box<dyn KmerFilter>],
|
||||
mut cb: impl FnMut(usize, usize, CanonicalKmer, Box<[u32]>) -> bool,
|
||||
) -> OKIResult<bool> {
|
||||
let index_dir = self.index_dir(part);
|
||||
if !index_dir.exists() {
|
||||
return Ok(true);
|
||||
}
|
||||
|
||||
let mut layer = 0;
|
||||
loop {
|
||||
let layer_dir = self.layer_dir(part, layer);
|
||||
if !layer_dir.exists() {
|
||||
break;
|
||||
}
|
||||
let mphf = MphfLayer::open(&layer_dir)?;
|
||||
let reader = UnitigFileReader::open_sequential(&layer_dir.join("unitigs.bin"))?;
|
||||
|
||||
let counts_dir = layer_dir.join("counts");
|
||||
let presence_dir = layer_dir.join("presence");
|
||||
|
||||
let cont = if use_counts && counts_dir.exists() {
|
||||
let mat = PersistentCompactIntMatrix::open(&layer_dir).map_err(OKIError::Io)?;
|
||||
let mut cont = true;
|
||||
for (kmer, _, _) in reader.iter_indexed_canonical_kmers() {
|
||||
if let Some(slot) = mphf.find(kmer) {
|
||||
let row = mat.row(slot);
|
||||
if passes_all(filters, kmer, &row, n_genomes) {
|
||||
cont = cb(part, layer, kmer, row);
|
||||
if !cont {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
cont
|
||||
} else if !use_counts && presence_dir.exists() {
|
||||
let mat = PersistentBitMatrix::open(&layer_dir).map_err(OKIError::Io)?;
|
||||
let mut cont = true;
|
||||
for (kmer, _, _) in reader.iter_indexed_canonical_kmers() {
|
||||
if let Some(slot) = mphf.find(kmer) {
|
||||
let row: Box<[u32]> = mat.row(slot).iter().map(|&b| b as u32).collect();
|
||||
if passes_all(filters, kmer, &row, n_genomes) {
|
||||
cont = cb(part, layer, kmer, row);
|
||||
if !cont {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
cont
|
||||
} else {
|
||||
// Same as iter_partition_kmers: row is constant but a filter
|
||||
// may still depend on the kmer's own sequence, so this must
|
||||
// be tested per kmer, not once for the whole layer.
|
||||
let all_present: Box<[u32]> = vec![1u32; n_genomes].into();
|
||||
let mut cont = true;
|
||||
for (kmer, _, _) in reader.iter_indexed_canonical_kmers() {
|
||||
if mphf.find(kmer).is_some()
|
||||
&& passes_all(filters, kmer, &all_present, n_genomes)
|
||||
{
|
||||
cont = cb(part, layer, kmer, all_present.clone());
|
||||
if !cont {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
cont
|
||||
};
|
||||
|
||||
if !cont {
|
||||
return Ok(false);
|
||||
}
|
||||
layer += 1;
|
||||
}
|
||||
|
||||
Ok(true)
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
use obicompactvec::FilterMask;
|
||||
use obikseq::CanonicalKmer;
|
||||
|
||||
use crate::predicate::Selection;
|
||||
|
||||
/// Trait for kmer filters.
|
||||
///
|
||||
/// `kmer` is the k-mer's own canonical sequence, reconstructed from the
|
||||
@@ -173,14 +175,12 @@ impl KmerFilter for MaxTotalCount {
|
||||
|
||||
// ── Group-based quorum filter ─────────────────────────────────────────────────
|
||||
|
||||
/// Quorum filter operating on pre-classified genome groups.
|
||||
/// Quorum filter operating on a pre-classified genome [`Selection`].
|
||||
///
|
||||
/// `ingroup_idx` / `outgroup_idx` are column indices into the per-genome row.
|
||||
/// When `ingroup_idx` is empty, no ingroup quorum is checked.
|
||||
/// When `outgroup_idx` is empty, no outgroup quorum is checked.
|
||||
/// `selection.ingroup_idx` / `selection.outgroup_idx` are column indices into
|
||||
/// the per-genome row. When empty, the corresponding quorum is not checked.
|
||||
pub struct GroupQuorumFilter {
|
||||
pub ingroup_idx: Vec<usize>,
|
||||
pub outgroup_idx: Vec<usize>,
|
||||
pub selection: Selection,
|
||||
pub threshold: u32,
|
||||
pub min_count: usize,
|
||||
pub max_count: usize,
|
||||
@@ -225,22 +225,22 @@ impl GroupQuorumFilter {
|
||||
|
||||
impl KmerFilter for GroupQuorumFilter {
|
||||
fn passes(&self, _kmer: CanonicalKmer, row: &[u32], _n_genomes: usize) -> bool {
|
||||
if !self.ingroup_idx.is_empty() {
|
||||
let n = self.ingroup_idx.iter()
|
||||
if !self.selection.ingroup_idx.is_empty() {
|
||||
let n = self.selection.ingroup_idx.iter()
|
||||
.filter(|&&i| row.get(i).copied().unwrap_or(0) > self.threshold)
|
||||
.count();
|
||||
let denom = self.ingroup_idx.len();
|
||||
let denom = self.selection.ingroup_idx.len();
|
||||
if n < self.min_count { return false; }
|
||||
if n > self.max_count { return false; }
|
||||
let frac = n as f64 / denom as f64;
|
||||
if frac < self.min_frac { return false; }
|
||||
if frac > self.max_frac { return false; }
|
||||
}
|
||||
if !self.outgroup_idx.is_empty() {
|
||||
let n = self.outgroup_idx.iter()
|
||||
if !self.selection.outgroup_idx.is_empty() {
|
||||
let n = self.selection.outgroup_idx.iter()
|
||||
.filter(|&&i| row.get(i).copied().unwrap_or(0) > self.threshold)
|
||||
.count();
|
||||
let denom = self.outgroup_idx.len();
|
||||
let denom = self.selection.outgroup_idx.len();
|
||||
if n < self.min_outgroup_count { return false; }
|
||||
if n > self.max_outgroup_count { return false; }
|
||||
let frac = n as f64 / denom as f64;
|
||||
@@ -253,17 +253,17 @@ impl KmerFilter for GroupQuorumFilter {
|
||||
fn column_mask_expr(&self, _n_genomes: usize) -> Option<FilterMask> {
|
||||
let t = self.threshold.checked_add(1)?;
|
||||
let mut parts: Vec<FilterMask> = Vec::new();
|
||||
if !self.ingroup_idx.is_empty() {
|
||||
if !self.selection.ingroup_idx.is_empty() {
|
||||
Self::group_mask_parts(
|
||||
&self.ingroup_idx, t,
|
||||
&self.selection.ingroup_idx, t,
|
||||
self.min_count, self.max_count,
|
||||
self.min_frac, self.max_frac,
|
||||
&mut parts,
|
||||
);
|
||||
}
|
||||
if !self.outgroup_idx.is_empty() {
|
||||
if !self.selection.outgroup_idx.is_empty() {
|
||||
Self::group_mask_parts(
|
||||
&self.outgroup_idx, t,
|
||||
&self.selection.outgroup_idx, t,
|
||||
self.min_outgroup_count, self.max_outgroup_count,
|
||||
self.min_outgroup_frac, self.max_outgroup_frac,
|
||||
&mut parts,
|
||||
|
||||
@@ -4,19 +4,18 @@
|
||||
//! operates on already-retained k-mers.
|
||||
//!
|
||||
//! [`filter`] (the `KmerFilter` trait + its implementations) depends only
|
||||
//! on `obicompactvec`/`obikseq`, not on `obikindex`. [`dump_layer`] is the
|
||||
//! extension over `obikindex::KmerIndex` that actually iterates a
|
||||
//! partition's k-mers through those filters (`iter_partition_kmers`,
|
||||
//! `iter_partition_kmers_located`) — every k-mer, filtered or not, goes
|
||||
//! through this same path (`passes_all` on an empty filter list is always
|
||||
//! `true`), so this crate depends one-way on `obikindex`, not the reverse.
|
||||
//! on `obicompactvec`/`obikseq`, not on `obikindex`. The partition/layer
|
||||
//! iteration that actually runs these filters over an index
|
||||
//! (`iter_partition_kmers`, `iter_partition_kmers_located`) lives in
|
||||
//! `obikdump` instead (`FilteredPartitionIter`) — it needs `obikidxcache`'s
|
||||
//! `IndexCache` to read a *complete* source index, which this crate has no
|
||||
//! reason to depend on.
|
||||
|
||||
mod filter;
|
||||
mod dump_layer;
|
||||
mod predicate;
|
||||
|
||||
pub use filter::{
|
||||
GroupQuorumFilter, KmerFilter, MaxGenomeCount, MaxGenomeFraction, MaxTotalCount,
|
||||
MinComplexity, MinGenomeCount, MinGenomeFraction, MinTotalCount, passes_all,
|
||||
};
|
||||
pub use predicate::{GroupFilterParams, MetaPred};
|
||||
pub use predicate::{GenomeSelector, GroupFilterParams, MetaPred, Selection};
|
||||
@@ -68,14 +68,6 @@ impl MetaPred {
|
||||
}
|
||||
}
|
||||
|
||||
impl GenomeInfo {
|
||||
/// Evaluate a single metadata predicate against this genome.
|
||||
/// Returns `None` when the predicate's key is absent (NA propagation).
|
||||
pub fn matches(&self, pred: &MetaPred) -> Option<bool> {
|
||||
pred.eval(&self.meta)
|
||||
}
|
||||
}
|
||||
|
||||
// ── Path matching ─────────────────────────────────────────────────────────────
|
||||
|
||||
/// True if the stored taxonomy `value` matches `pattern`.
|
||||
@@ -153,19 +145,49 @@ pub struct GroupFilterParams {
|
||||
pub max_outgroup_frac: Option<f64>,
|
||||
}
|
||||
|
||||
impl IndexMeta {
|
||||
/// Returns indices of genomes matching `pred_str` (single predicate).
|
||||
pub fn matching_genome_indices(&self, pred_str: &str) -> Result<Vec<usize>, String> {
|
||||
let pred = MetaPred::parse(pred_str)?;
|
||||
let genomes = self.genomes().map_err(|e| e.to_string())?;
|
||||
Ok(genomes.iter().enumerate()
|
||||
.filter_map(|(i, g)| {
|
||||
if g.matches(&pred) == Some(true) { Some(i) } else { std::option::Option::None }
|
||||
})
|
||||
.collect())
|
||||
// ── Genome selector ──────────────────────────────────────────────────────────
|
||||
|
||||
/// Result of running a [`GenomeSelector`] against an index's metadata.
|
||||
pub struct Selection {
|
||||
pub ingroup_idx: Vec<usize>,
|
||||
pub outgroup_idx: Vec<usize>,
|
||||
}
|
||||
|
||||
pub struct GenomeSelector {
|
||||
pub(crate) ingroup: Vec<MetaPred>,
|
||||
pub(crate) outgroup: Vec<MetaPred>,
|
||||
}
|
||||
|
||||
impl GenomeSelector {
|
||||
/// Parse ingroup (AND'd) and outgroup (OR'd) predicate strings.
|
||||
pub fn parse(ingroup: &[String], outgroup: &[String]) -> Result<Self, String> {
|
||||
let ingroup = ingroup.iter().map(|s| MetaPred::parse(s)).collect::<Result<Vec<_>, _>>()?;
|
||||
let outgroup = outgroup.iter().map(|s| MetaPred::parse(s)).collect::<Result<Vec<_>, _>>()?;
|
||||
Ok(Self { ingroup, outgroup })
|
||||
}
|
||||
|
||||
/// Build a `GroupQuorumFilter` from parsed predicates, evaluated against `self.genomes`.
|
||||
/// Classify `meta`'s genomes into ingroup/outgroup indices.
|
||||
///
|
||||
/// - No predicates at all: every genome is (implicitly) ingroup.
|
||||
/// - Otherwise: ingroup wins on overlap; uncategorized genomes are dropped.
|
||||
pub fn run(&self, meta: &IndexMeta) -> Result<Selection, String> {
|
||||
let genomes = meta.genomes().map_err(|e| e.to_string())?;
|
||||
|
||||
if self.ingroup.is_empty() && self.outgroup.is_empty() {
|
||||
return Ok(Selection { ingroup_idx: (0..genomes.len()).collect(), outgroup_idx: vec![] });
|
||||
}
|
||||
|
||||
let members = classify(&genomes, &self.ingroup, &self.outgroup);
|
||||
let ingroup_idx: Vec<usize> = members.iter().enumerate()
|
||||
.filter(|(_, m)| matches!(m, Membership::Ingroup))
|
||||
.map(|(i, _)| i).collect();
|
||||
let outgroup_idx: Vec<usize> = members.iter().enumerate()
|
||||
.filter(|(_, m)| matches!(m, Membership::Outgroup))
|
||||
.map(|(i, _)| i).collect();
|
||||
Ok(Selection { ingroup_idx, outgroup_idx })
|
||||
}
|
||||
|
||||
/// Build a `GroupQuorumFilter` from this selector's classification of `meta`.
|
||||
///
|
||||
/// - No groups defined: `ingroup_idx` = all genomes (implicit ingroup).
|
||||
/// - `ingroup` predicates only: outgroup indices are empty.
|
||||
@@ -173,34 +195,20 @@ impl IndexMeta {
|
||||
/// - Both defined: ingroup wins on overlap; uncategorized genomes are ignored.
|
||||
pub fn build_group_filter(
|
||||
&self,
|
||||
ingroup_preds: &[MetaPred],
|
||||
outgroup_preds: &[MetaPred],
|
||||
p: GroupFilterParams,
|
||||
meta: &IndexMeta,
|
||||
p: GroupFilterParams,
|
||||
) -> Result<GroupQuorumFilter, String> {
|
||||
let genomes = self.genomes().map_err(|e| e.to_string())?;
|
||||
let (ingroup_idx, outgroup_idx) = if ingroup_preds.is_empty() && outgroup_preds.is_empty() {
|
||||
((0..genomes.len()).collect(), vec![])
|
||||
} else {
|
||||
let members = classify(&genomes, ingroup_preds, outgroup_preds);
|
||||
let in_idx: Vec<usize> = members.iter().enumerate()
|
||||
.filter(|(_, m)| matches!(m, Membership::Ingroup))
|
||||
.map(|(i, _)| i).collect();
|
||||
let out_idx: Vec<usize> = members.iter().enumerate()
|
||||
.filter(|(_, m)| matches!(m, Membership::Outgroup))
|
||||
.map(|(i, _)| i).collect();
|
||||
(in_idx, out_idx)
|
||||
};
|
||||
|
||||
let in_size = ingroup_idx.len();
|
||||
let out_size = outgroup_idx.len();
|
||||
let selection = self.run(meta)?;
|
||||
let in_size = selection.ingroup_idx.len();
|
||||
let out_size = selection.outgroup_idx.len();
|
||||
|
||||
let ingroup_quorum_explicit = p.min_count.is_some() || p.max_count.is_some()
|
||||
|| p.min_frac.is_some() || p.max_frac.is_some();
|
||||
let outgroup_quorum_explicit = p.min_outgroup_count.is_some() || p.max_outgroup_count.is_some()
|
||||
|| p.min_outgroup_frac.is_some() || p.max_outgroup_frac.is_some();
|
||||
|
||||
let default_min_frac = if !ingroup_preds.is_empty() && !ingroup_quorum_explicit { 1.0 } else { 0.0 };
|
||||
let default_max_outgroup_count = if !outgroup_preds.is_empty() && !outgroup_quorum_explicit { 0 } else { out_size };
|
||||
let default_min_frac = if !self.ingroup.is_empty() && !ingroup_quorum_explicit { 1.0 } else { 0.0 };
|
||||
let default_max_outgroup_count = if !self.outgroup.is_empty() && !outgroup_quorum_explicit { 0 } else { out_size };
|
||||
|
||||
// Resolve a signed count: negative means an offset from the group size
|
||||
// (e.g. -1 = all but one), floored at 1 so the negative form always keeps
|
||||
@@ -238,8 +246,7 @@ impl IndexMeta {
|
||||
}
|
||||
|
||||
Ok(GroupQuorumFilter {
|
||||
ingroup_idx,
|
||||
outgroup_idx,
|
||||
selection,
|
||||
threshold: p.threshold,
|
||||
min_count,
|
||||
max_count,
|
||||
@@ -252,3 +259,4 @@ impl IndexMeta {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user