diff --git a/src/Cargo.lock b/src/Cargo.lock index 0c7fa600..5a5915cd 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -1519,6 +1519,14 @@ dependencies = [ name = "obikalgorithm" version = "0.1.0" +[[package]] +name = "obikdump" +version = "0.1.0" +dependencies = [ + "obikfilter", + "obikindex", +] + [[package]] name = "obikentropy" version = "0.1.0" @@ -1526,6 +1534,27 @@ dependencies = [ "obikseq", ] +[[package]] +name = "obikfilter" +version = "0.1.0" +dependencies = [ + "obicompactvec", + "obikindex", + "obikseq", + "obiskio", + "obitaxonomy", +] + +[[package]] +name = "obikidxcache" +version = "0.1.0" +dependencies = [ + "ndarray", + "obicompactvec", + "rayon", + "tempfile", +] + [[package]] name = "obikindex" version = "0.1.0" @@ -1533,9 +1562,7 @@ dependencies = [ "anyhow", "bitvec", "cacheline-ef", - "crossbeam-channel", "epserde", - "hwlocality", "indicatif", "memmap2", "ndarray", @@ -1597,11 +1624,13 @@ dependencies = [ "obidebruinj", "obifastwrite", "obikalgorithm", + "obikfilter", "obikindex", "obikindexer", "obikphylo", "obikrope", "obikseq", + "obikstats", "obipipeline", "obiread", "obiskbuilder", @@ -1618,6 +1647,37 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "obikmer2" +version = "1.2.2" +dependencies = [ + "clap", + "obikalgorithm", + "obikindex", + "obikindexer", + "obikseq", + "obipipeline", + "obiread", + "obisys", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "obikmerge" +version = "0.1.0" +dependencies = [ + "obicompactvec", + "obikfilter", + "obikindex", + "obikindexer", + "obikseq", + "obipipeline", + "obiskio", + "obisys", + "tracing", +] + [[package]] name = "obikphylo" version = "0.1.0" @@ -1626,6 +1686,7 @@ dependencies = [ "ndarray", "obicompactvec", "obikalgorithm", + "obikidxcache", "obikindex", "obikindexer", "obikseq", @@ -1641,6 +1702,29 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "obikquery" +version = "0.1.0" +dependencies = [ + "obikindex", +] + +[[package]] +name = "obikrebuild" +version = "0.1.0" +dependencies = [ + "obicompactvec", + "obidebruinj", + "obikfilter", + "obikindex", + "obikindexer", + "obikmerge", + "obikseq", + "obiskio", + "obisys", + "tracing", +] + [[package]] name = "obikrope" version = "0.1.0" @@ -1649,6 +1733,16 @@ dependencies = [ "criterion2", ] +[[package]] +name = "obikselect" +version = "0.1.0" +dependencies = [ + "obicompactvec", + "obikindex", + "obisys", + "tracing", +] + [[package]] name = "obikseq" version = "0.1.0" @@ -1660,6 +1754,15 @@ dependencies = [ "xxhash-rust", ] +[[package]] +name = "obikstats" +version = "0.1.0" +dependencies = [ + "obicompactvec", + "obikindex", + "rayon", +] + [[package]] name = "obipipeline" version = "0.1.0" @@ -1712,8 +1815,11 @@ dependencies = [ name = "obisys" version = "0.1.0" dependencies = [ + "crossbeam-channel", + "hwlocality", "indicatif", "libc", + "rayon", "sysinfo", "tracing", ] diff --git a/src/Cargo.toml b/src/Cargo.toml index 296753ad..cf2b32bc 100644 --- a/src/Cargo.toml +++ b/src/Cargo.toml @@ -1,5 +1,5 @@ [workspace] resolver = "3" -members = ["obikseq", "obiread", "obiskbuilder", "obifastwrite", "obikmer","obikrope","obipipeline", "obiskio","obidebruinj", "obicompactvec", "obisys", "obikindex", "obikindexer", "obitaxonomy", "obikentropy", "obikphylo", "obikalgorithm"] +members = ["obikseq", "obiread", "obiskbuilder", "obifastwrite", "obikmer","obikmer2","obikrope","obipipeline", "obiskio","obidebruinj", "obicompactvec", "obisys", "obikindex", "obikindexer", "obikquery", "obikdump", "obikfilter", "obikselect", "obikrebuild", "obikmerge", "obikstats", "obikidxcache", "obitaxonomy", "obikentropy", "obikphylo", "obikalgorithm"] [profile.release] debug = 1 diff --git a/src/obikdump/Cargo.toml b/src/obikdump/Cargo.toml new file mode 100644 index 00000000..b802741d --- /dev/null +++ b/src/obikdump/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "obikdump" +version = "0.1.0" +edition = "2024" + +[dependencies] +obikindex = { path = "../obikindex" } +obikfilter = { path = "../obikfilter" } diff --git a/src/obikindex/src/index/dump.rs b/src/obikdump/src/dump.rs similarity index 97% rename from src/obikindex/src/index/dump.rs rename to src/obikdump/src/dump.rs index 45ff962c..560ef95f 100644 --- a/src/obikindex/src/index/dump.rs +++ b/src/obikdump/src/dump.rs @@ -3,9 +3,9 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use rayon::prelude::*; -use crate::index::error::{OKIError, OKIResult}; -use crate::index::kmer_index::KmerIndex; -use crate::index::KmerFilter; +use obikindex::{OKIError, OKIResult}; +use obikindex::KmerIndex; +use obikfilter::KmerFilter; impl KmerIndex { /// Write a CSV table of all indexed kmers to `out`. diff --git a/src/obikdump/src/lib.rs b/src/obikdump/src/lib.rs new file mode 100644 index 00000000..f953444f --- /dev/null +++ b/src/obikdump/src/lib.rs @@ -0,0 +1,8 @@ +//! Raw content export of an `obikindex::KmerIndex`: staging ground for +//! code currently living in `obikindex::index`'s dump path (`dump.rs`, +//! `dump_layer.rs`), to be migrated here to lighten that crate. Kept as +//! a separate crate — not a module of `obikindex` — so the dependency +//! runs one way only (dump code depends on the data model, never the +//! reverse), same pattern as `obikindexer`/`obikquery`. + +mod dump; diff --git a/src/obikfilter/Cargo.toml b/src/obikfilter/Cargo.toml new file mode 100644 index 00000000..f1bde2f8 --- /dev/null +++ b/src/obikfilter/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "obikfilter" +version = "0.1.0" +edition = "2024" + +[dependencies] +obikindex = { path = "../obikindex" } +obicompactvec = { path = "../obicompactvec" } +obikseq = { path = "../obikseq" } +obiskio = { path = "../obiskio" } +obitaxonomy = { path = "../obitaxonomy" } diff --git a/src/obikindex/src/index/dump_layer.rs b/src/obikfilter/src/dump_layer.rs similarity index 90% rename from src/obikindex/src/index/dump_layer.rs rename to src/obikfilter/src/dump_layer.rs index 69fca0d5..47507806 100644 --- a/src/obikindex/src/index/dump_layer.rs +++ b/src/obikfilter/src/dump_layer.rs @@ -1,20 +1,11 @@ +use obikindex::layer::MphfLayer; use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix}; use obikseq::CanonicalKmer; -use crate::layer::{MphfLayer, OLMError}; -use obiskio::{SKError, SKResult, UnitigFileReader}; +use obiskio::UnitigFileReader; +use obikindex::{OKIError, OKIResult}; -use crate::index::filter::{KmerFilter, passes_all}; -use crate::index::kmer_index::KmerIndex; - -fn olm_to_sk(e: OLMError) -> SKError { - match e { - OLMError::Io(e) => SKError::Io(e), - other => SKError::InvalidData { - context: "dump", - detail: other.to_string(), - }, - } -} +use crate::filter::{KmerFilter, passes_all}; +use obikindex::KmerIndex; impl KmerIndex { /// Iterate all indexed kmers in partition `part`, calling `cb(kmer, row)` for each @@ -35,7 +26,7 @@ impl KmerIndex { n_genomes: usize, filters: &[Box], mut cb: impl FnMut(CanonicalKmer, Box<[u32]>) -> bool, - ) -> SKResult { + ) -> OKIResult { let index_dir = self.index_dir(part); if !index_dir.exists() { return Ok(true); @@ -43,19 +34,19 @@ impl KmerIndex { let mut l = 0; loop { - let layer_dir = self.layer_dir(part, l); + let layer_dir = self.layer_dir(part, l)?; if !layer_dir.exists() { break; } l += 1; - let mphf = MphfLayer::open(&layer_dir).map_err(olm_to_sk)?; + let mphf = MphfLayer::open(&layer_dir)?; let reader = UnitigFileReader::open_sequential(&layer_dir.join("unitigs.bin"))?; let counts_dir = layer_dir.join("counts"); let presence_dir = layer_dir.join("presence"); let cont = if use_counts && counts_dir.exists() { - let mat = PersistentCompactIntMatrix::open(&layer_dir).map_err(SKError::Io)?; + let mat = PersistentCompactIntMatrix::open(&layer_dir).map_err(OKIError::Io)?; let mut cont = true; for (kmer, _, _) in reader.iter_indexed_canonical_kmers() { if let Some(slot) = mphf.find(kmer) { @@ -70,7 +61,7 @@ impl KmerIndex { } cont } else if !use_counts && presence_dir.exists() { - let mat = PersistentBitMatrix::open(&layer_dir).map_err(SKError::Io)?; + let mat = PersistentBitMatrix::open(&layer_dir).map_err(OKIError::Io)?; let mut cont = true; for (kmer, _, _) in reader.iter_indexed_canonical_kmers() { if let Some(slot) = mphf.find(kmer) { @@ -123,7 +114,7 @@ impl KmerIndex { n_genomes: usize, filters: &[Box], mut cb: impl FnMut(usize, usize, CanonicalKmer, Box<[u32]>) -> bool, - ) -> SKResult { + ) -> OKIResult { let index_dir = self.index_dir(part); if !index_dir.exists() { return Ok(true); @@ -135,14 +126,14 @@ impl KmerIndex { if !layer_dir.exists() { break; } - let mphf = MphfLayer::open(&layer_dir).map_err(olm_to_sk)?; + let mphf = MphfLayer::open(&layer_dir)?; let reader = UnitigFileReader::open_sequential(&layer_dir.join("unitigs.bin"))?; let counts_dir = layer_dir.join("counts"); let presence_dir = layer_dir.join("presence"); let cont = if use_counts && counts_dir.exists() { - let mat = PersistentCompactIntMatrix::open(&layer_dir).map_err(SKError::Io)?; + let mat = PersistentCompactIntMatrix::open(&layer_dir).map_err(OKIError::Io)?; let mut cont = true; for (kmer, _, _) in reader.iter_indexed_canonical_kmers() { if let Some(slot) = mphf.find(kmer) { @@ -157,7 +148,7 @@ impl KmerIndex { } cont } else if !use_counts && presence_dir.exists() { - let mat = PersistentBitMatrix::open(&layer_dir).map_err(SKError::Io)?; + let mat = PersistentBitMatrix::open(&layer_dir).map_err(OKIError::Io)?; let mut cont = true; for (kmer, _, _) in reader.iter_indexed_canonical_kmers() { if let Some(slot) = mphf.find(kmer) { diff --git a/src/obikindex/src/index/filter.rs b/src/obikfilter/src/filter.rs similarity index 100% rename from src/obikindex/src/index/filter.rs rename to src/obikfilter/src/filter.rs diff --git a/src/obikfilter/src/lib.rs b/src/obikfilter/src/lib.rs new file mode 100644 index 00000000..26a95c23 --- /dev/null +++ b/src/obikfilter/src/lib.rs @@ -0,0 +1,22 @@ +//! Per-kmer filtering: `KmerFilter` judges one k-mer (its canonical +//! sequence + its per-genome row) at a time, deciding whether it passes. +//! Orthogonal to genome-column selection/aggregation (`obikselect`), which +//! operates on already-retained k-mers. +//! +//! [`filter`] (the `KmerFilter` trait + its implementations) depends only +//! on `obicompactvec`/`obikseq`, not on `obikindex`. [`dump_layer`] is the +//! extension over `obikindex::KmerIndex` that actually iterates a +//! partition's k-mers through those filters (`iter_partition_kmers`, +//! `iter_partition_kmers_located`) — every k-mer, filtered or not, goes +//! through this same path (`passes_all` on an empty filter list is always +//! `true`), so this crate depends one-way on `obikindex`, not the reverse. + +mod filter; +mod dump_layer; +mod predicate; + +pub use filter::{ + GroupQuorumFilter, KmerFilter, MaxGenomeCount, MaxGenomeFraction, MaxTotalCount, + MinComplexity, MinGenomeCount, MinGenomeFraction, MinTotalCount, passes_all, +}; +pub use predicate::{GroupFilterParams, MetaPred}; diff --git a/src/obikindex/src/index/predicate.rs b/src/obikfilter/src/predicate.rs similarity index 99% rename from src/obikindex/src/index/predicate.rs rename to src/obikfilter/src/predicate.rs index 7791748c..db1be2d8 100644 --- a/src/obikindex/src/index/predicate.rs +++ b/src/obikfilter/src/predicate.rs @@ -1,9 +1,9 @@ use std::collections::HashMap; -use crate::index::GroupQuorumFilter; +use crate::filter::GroupQuorumFilter; use obitaxonomy::{TaxPath, TaxPattern}; -use crate::index::meta::{GenomeInfo, IndexMeta}; +use obikindex::{GenomeInfo, IndexMeta}; // ── Operator ────────────────────────────────────────────────────────────────── diff --git a/src/obikidxcache/Cargo.toml b/src/obikidxcache/Cargo.toml new file mode 100644 index 00000000..884e8c88 --- /dev/null +++ b/src/obikidxcache/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "obikidxcache" +version = "0.1.0" +edition = "2024" + +[dependencies] +obicompactvec = { path = "../obicompactvec" } +ndarray = "0.17" +rayon = "1" + +[dev-dependencies] +tempfile = "3" diff --git a/src/obikindex/src/layer/layered_store.rs b/src/obikidxcache/src/layered_store.rs similarity index 100% rename from src/obikindex/src/layer/layered_store.rs rename to src/obikidxcache/src/layered_store.rs diff --git a/src/obikidxcache/src/lib.rs b/src/obikidxcache/src/lib.rs new file mode 100644 index 00000000..15919976 --- /dev/null +++ b/src/obikidxcache/src/lib.rs @@ -0,0 +1,11 @@ +//! Caching/aggregation over already-opened `obikindex` layer/partition +//! objects — a runtime cache shape, not part of the stateless +//! `Index { Partition { Layer } }` directory-structure model itself. +//! [`LayeredStore`] wraps one entry per layer (or per partition, via +//! `LayeredStore>`) and propagates `ColumnWeights`/ +//! `CountPartials`/`BitPartials` aggregation across them. No dependency on +//! `obikindex` — generic over any `S` implementing those traits. + +mod layered_store; + +pub use layered_store::LayeredStore; diff --git a/src/obikindex/src/layer/tests/layered_store.rs b/src/obikidxcache/src/tests/layered_store.rs similarity index 100% rename from src/obikindex/src/layer/tests/layered_store.rs rename to src/obikidxcache/src/tests/layered_store.rs diff --git a/src/obikindex/Cargo.toml b/src/obikindex/Cargo.toml index ce698577..e58cdf8e 100644 --- a/src/obikindex/Cargo.toml +++ b/src/obikindex/Cargo.toml @@ -19,13 +19,11 @@ niffler = "3.0.0" memmap2 = "0.9.10" ndarray = "0.17" rayon = "1" -crossbeam-channel = "0.5" serde = { version = "1", features = ["derive"] } serde_json = "1" indicatif = "0.18" tracing = "0.1.44" bitvec = "1" -hwlocality = { version = "1.0.0-alpha.11", features = ["vendored"], optional = true } [dev-dependencies] obiread = { path = "../obiread" } @@ -33,7 +31,3 @@ obikseq = { path = "../obikseq", features = ["test-utils"] } tempfile = "3" tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] } anyhow = "1" - -[features] -default = ["numa"] -numa = ["hwlocality"] diff --git a/src/obikindex/src/index/builder.rs b/src/obikindex/src/index/builder.rs index 28f7f2a1..958a94ca 100644 --- a/src/obikindex/src/index/builder.rs +++ b/src/obikindex/src/index/builder.rs @@ -44,7 +44,11 @@ pub trait IndexBuilder: Sized { /// For construction paths that build partitions from scratch (`select`, /// `rebuild`). `merge` bootstraps by copying a source index instead, so /// it does not use this. - fn create_skeleton>(output: P, config: IndexConfig, genomes: Vec) -> OKIResult; + fn create_skeleton>( + output: P, + config: IndexConfig, + genomes: Vec, + ) -> OKIResult; /// Mark `output` as fully indexed, pack its column matrices, and reopen it. /// @@ -61,7 +65,7 @@ pub trait IndexBuilder: Sized { impl IndexBuilder for KmerIndex { fn clear_output_for_create>(output: P, force: bool) -> OKIResult<()> { let output = output.as_ref(); - if Self::exists(output) { + if Self::is_an_index(output) { if force { fs::remove_dir_all(output).map_err(OKIError::Io)?; } else { @@ -74,11 +78,18 @@ impl IndexBuilder for KmerIndex { Ok(()) } - fn create_skeleton>(output: P, config: IndexConfig, genomes: Vec) -> OKIResult { + fn create_skeleton>( + output: P, + config: IndexConfig, + genomes: Vec, + ) -> OKIResult { let output = output.as_ref(); fs::create_dir_all(output).map_err(OKIError::Io)?; let meta = IndexMeta::create_at(output, config, genomes).map_err(OKIError::Io)?; - Ok(KmerIndex { root_path: output.to_owned(), meta: Arc::new(meta) }) + Ok(KmerIndex { + root_path: output.to_owned(), + meta: Arc::new(meta), + }) } fn finalize_indexed>(output: P, rep: &mut Reporter) -> OKIResult { diff --git a/src/obikindex/src/index/common.rs b/src/obikindex/src/index/common.rs index cfa20609..f22fdc12 100644 --- a/src/obikindex/src/index/common.rs +++ b/src/obikindex/src/index/common.rs @@ -1,65 +1,25 @@ -use std::path::Path; - +use crate::index::error::{OKIError, OKIResult}; use obicompactvec::{PersistentBitVecBuilder, PersistentCompactIntVecBuilder}; -use crate::layer::meta::PartitionMeta; -use crate::layer::{layer_dir, IndexMode, OLMError}; -use obiskio::{SKError, SKResult}; - -// ── olm_to_sk ──────────────────────────────────────────────────────────────── - -pub fn olm_to_sk(e: OLMError, context: &'static str) -> SKError { - match e { - OLMError::Io(e) => SKError::Io(e), - other => SKError::InvalidData { - context, - detail: other.to_string(), - }, - } -} - -// ── load_meta ──────────────────────────────────────────────────────────────── - -/// Load PartitionMeta, or recover it by probing layer directories. -/// Indexes built before meta.json was introduced lack the file. -pub(crate) fn load_meta(dir: &Path, context: &'static str) -> SKResult { - match PartitionMeta::load(dir) { - Ok(m) => Ok(m), - Err(e) if matches!(e, OLMError::Io(ref io_e) if io_e.kind() == std::io::ErrorKind::NotFound) => - { - let mut n = 0usize; - while layer_dir(dir, n).exists() { - n += 1; - } - let m = PartitionMeta { - n_layers: n, - mode: IndexMode::default(), - }; - m.save(dir).map_err(|e| olm_to_sk(e, context))?; - Ok(m) - } - Err(e) => Err(olm_to_sk(e, context)), - } -} // ── ColBuilder ──────────────────────────────────────────────────────────────── -pub(crate) enum ColBuilder { +pub enum ColBuilder { Bit(PersistentBitVecBuilder), Int(PersistentCompactIntVecBuilder), } impl ColBuilder { - pub(crate) fn set_val(&mut self, slot: usize, value: u32) { + pub fn set_val(&mut self, slot: usize, value: u32) { match self { ColBuilder::Bit(b) => b.set(slot, value > 0), ColBuilder::Int(b) => b.set(slot, value), } } - pub(crate) fn close(self) -> SKResult<()> { + pub fn close(self) -> OKIResult<()> { match self { - ColBuilder::Bit(b) => b.close().map_err(SKError::Io), - ColBuilder::Int(b) => b.close().map_err(SKError::Io), + ColBuilder::Bit(b) => b.close().map_err(OKIError::Io), + ColBuilder::Int(b) => b.close().map_err(OKIError::Io), } } } diff --git a/src/obikindex/src/index/error.rs b/src/obikindex/src/index/error.rs index f9ddf7e4..6f95f6ee 100644 --- a/src/obikindex/src/index/error.rs +++ b/src/obikindex/src/index/error.rs @@ -8,6 +8,7 @@ pub enum OKIError { Io(io::Error), Json(serde_json::Error), Partition(SKError), + Layer(SKError), /// Source index is not in `Indexed` state. NotIndexed(std::path::PathBuf), /// Source indexes have incompatible configurations (k, m, n_bits). @@ -20,6 +21,9 @@ pub enum OKIError { InvalidInput(String), /// Sources mix exact and approximate evidence, or use incompatible approx parameters. IncompatibleEvidence(String), + /// Malformed data encountered while reading/writing a layer file, tagged + /// with the operation that hit it. + InvalidData { context: &'static str, detail: String }, } pub type OKIResult = Result; @@ -27,15 +31,24 @@ pub type OKIResult = Result; impl fmt::Display for OKIError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - OKIError::Io(e) => write!(f, "I/O error: {e}"), - OKIError::Json(e) => write!(f, "JSON error: {e}"), - OKIError::Partition(e) => write!(f, "partition error: {e}"), - OKIError::NotIndexed(p) => write!(f, "index not fully built: {}", p.display()), + OKIError::Io(e) => write!(f, "I/O error: {e}"), + OKIError::Json(e) => write!(f, "JSON error: {e}"), + OKIError::Partition(e) => write!(f, "partition error: {e}"), + OKIError::Layer(e) => write!(f, "layer error: {e}"), + OKIError::NotIndexed(p) => write!(f, "index not fully built: {}", p.display()), OKIError::IncompatibleConfig => write!(f, "incompatible index configurations"), - OKIError::MismatchedMode => write!(f, "count mode requires all sources to have with_counts=true"), - OKIError::DuplicateGenomeLabel(l) => write!(f, "duplicate genome label across sources: {l}"), - OKIError::InvalidInput(m) => write!(f, "invalid input: {m}"), - OKIError::IncompatibleEvidence(m) => write!(f, "incompatible evidence: {m}"), + OKIError::MismatchedMode => write!( + f, + "count mode requires all sources to have with_counts=true" + ), + OKIError::DuplicateGenomeLabel(l) => { + write!(f, "duplicate genome label across sources: {l}") + } + OKIError::InvalidInput(m) => write!(f, "invalid input: {m}"), + OKIError::IncompatibleEvidence(m) => write!(f, "incompatible evidence: {m}"), + OKIError::InvalidData { context, detail } => { + write!(f, "invalid data ({context}): {detail}") + } } } } @@ -43,22 +56,28 @@ impl fmt::Display for OKIError { impl std::error::Error for OKIError { fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { match self { - OKIError::Io(e) => Some(e), - OKIError::Json(e) => Some(e), + OKIError::Io(e) => Some(e), + OKIError::Json(e) => Some(e), OKIError::Partition(e) => Some(e), - _ => None, // IncompatibleConfig, MismatchedMode, DuplicateGenomeLabel + _ => None, // IncompatibleConfig, MismatchedMode, DuplicateGenomeLabel } } } impl From for OKIError { - fn from(e: io::Error) -> Self { OKIError::Io(e) } + fn from(e: io::Error) -> Self { + OKIError::Io(e) + } } impl From for OKIError { - fn from(e: serde_json::Error) -> Self { OKIError::Json(e) } + fn from(e: serde_json::Error) -> Self { + OKIError::Json(e) + } } impl From for OKIError { - fn from(e: SKError) -> Self { OKIError::Partition(e) } + fn from(e: SKError) -> Self { + OKIError::Partition(e) + } } diff --git a/src/obikindex/src/index/kmer_index.rs b/src/obikindex/src/index/kmer_index.rs index 5f4efbd7..3aa0846b 100644 --- a/src/obikindex/src/index/kmer_index.rs +++ b/src/obikindex/src/index/kmer_index.rs @@ -2,13 +2,13 @@ use std::fs; use std::path::{Path, PathBuf}; use std::sync::Arc; -use crate::layer::meta::PartitionMeta; +use crate::layer::KmerLayer; +use crate::partition::KmerPartition; use obisys::progress_bar; use rayon::prelude::*; use obikseq::{set_k, set_m}; -use crate::index::common::load_meta; use crate::index::error::{OKIError, OKIResult}; use crate::index::meta::{GenomeInfo, IndexConfig, IndexMeta}; @@ -28,12 +28,15 @@ impl KmerIndex { genome_info: Option, ) -> OKIResult { let root_path = path.as_ref().to_owned(); - fs::create_dir_all(&root_path).map_err(OKIError::Io)?; + fs::create_dir(&root_path).map_err(OKIError::Io)?; set_k(config.kmer_size); set_m(config.minimizer_size); let genomes = genome_info.into_iter().collect(); let meta = IndexMeta::create_at(&root_path, config, genomes).map_err(OKIError::Io)?; - Ok(Self { root_path, meta: Arc::new(meta) }) + Ok(Self { + root_path, + meta: Arc::new(meta), + }) } pub fn open>(path: P) -> OKIResult { @@ -41,16 +44,30 @@ impl KmerIndex { let meta = IndexMeta::open_at(&root_path).map_err(OKIError::Io)?; set_k(meta.config.kmer_size); set_m(meta.config.minimizer_size); - Ok(Self { root_path, meta: Arc::new(meta) }) + Ok(Self { + root_path, + meta: Arc::new(meta), + }) } - /// Return `true` if `path` contains an `index.meta` file. - pub fn exists>(path: P) -> bool { + /// Return `true` if `path` points to a valid obikmer index directory. + /// + /// A directory is considered an index only when it contains an `index.meta` + /// file. The absence of that file means the directory is not an index, even + /// if it otherwise exists. + /// + /// This is a static method (associated function), not an instance method. + /// Call it as `KmerIndex::is_an_index(path)`. + /// + /// `path` is generic over any type implementing [`AsRef`]. This allows + /// callers to pass `&str`, `String`, `PathBuf`, `&Path`, etc. without an + /// explicit conversion; the value is coerced via `.as_ref()` inside. + pub fn is_an_index>(path: P) -> bool { IndexMeta::exists(path.as_ref()) } /// The index's root directory. - pub fn root_path(&self) -> &Path { + pub fn dir(&self) -> &Path { &self.root_path } pub fn meta(&self) -> Arc { @@ -102,42 +119,48 @@ impl KmerIndex { /// same directory only indirectly, through this method — `obikindexer` /// depends on `KmerIndex`, not the other way around — see /// `DevDocMD/implementation/partition_layer_cache.md`. - pub fn partition_dir(&self, i: usize) -> PathBuf { - crate::partition::partition_dir(&self.root_path, i) + pub(crate) fn partition_dir(&self, i: usize) -> OKIResult { + Ok(self.partition(i)?.dir().to_path_buf()) + } + + pub fn partition(&self, i: usize) -> OKIResult { + let n = self.n_partitions(); + if i >= n { + return Err(OKIError::InvalidData { + context: "New KmerPartition", + detail: format!("Partition {i} requested, Index has only {n} partitions"), + }); + } + Ok(KmerPartition::new(self, i)) + } + + pub(crate) fn layer_dir(&self, i: usize, j: usize) -> OKIResult { + Ok(self.layer(i, j)?.dir().to_path_buf()) + } + + pub fn layer(&self, i: usize, j: usize) -> OKIResult { + self.partition(i)?.layer(j) } /// Path of partition `i`'s layered-index directory (`/index`). - pub fn index_dir(&self, i: usize) -> PathBuf { + pub(crate) fn index_dir(&self, i: usize) -> PathBuf { crate::partition::index_dir(&self.root_path, i) } - /// Path of layer `l` within partition `i`'s layered index. - pub fn layer_dir(&self, i: usize, l: usize) -> PathBuf { - crate::layer::layer_dir(&self.index_dir(i), l) + /// Number of layers in partition `i` — a disk scan, not a metadata read + /// (see [`KmerPartition::n_layers`](crate::partition::KmerPartition::n_layers)). + pub fn n_layers(&self, i: usize) -> OKIResult { + Ok(self.partition(i)?.n_layers()) } - /// Partition `i`'s metadata (layer count, evidence mode). Returns - /// `obiskio::SKResult`, not `OKIResult` — matches the error convention - /// of the partition/layer-construction code below (moved here from - /// the former `obikpartitionner` crate, which predates `OKIError`); - /// `?` still converts it to `OKIResult` at any call site that needs - /// one (`OKIError: From`). - pub fn partition_meta(&self, i: usize) -> obiskio::SKResult { - load_meta(&self.index_dir(i), "partition_meta") - } - - /// Number of layers in partition `i` — see [`partition_meta`](Self::partition_meta). - pub fn n_layers(&self, i: usize) -> obiskio::SKResult { - Ok(self.partition_meta(i)?.n_layers) - } - - /// Evidence mode partition `i` was actually built with — see - /// [`partition_meta`](Self::partition_meta). Distinct from - /// [`evidence_mode`](Self::evidence_mode): that one is the index-level - /// config, this one is per-partition ground truth (the two agree in - /// practice, but this is what `Layer::open` needs). - pub fn partition_mode(&self, i: usize) -> obiskio::SKResult { - Ok(self.partition_meta(i)?.mode) + /// Evidence mode partition `i` was actually built with — detected from + /// layer 0's evidence files on disk (see + /// [`KmerPartition::mode`](crate::partition::KmerPartition::mode)). + /// Distinct from [`evidence_mode`](Self::evidence_mode): that one is the + /// index-level config, this one is per-partition ground truth (the two + /// agree in practice, but this is what `Layer::open` needs). + pub fn partition_mode(&self, i: usize) -> OKIResult { + self.partition(i)?.mode() } /// Number of layers per partition. @@ -150,8 +173,8 @@ impl KmerIndex { } /// Path to the unitigs file for partition `part`, layer `layer`. - pub fn layer_unitigs_path(&self, part: usize, layer: usize) -> PathBuf { - self.layer_dir(part, layer).join("unitigs.bin") + pub fn layer_unitigs_path(&self, part: usize, layer: usize) -> OKIResult { + Ok(self.layer_dir(part, layer)?.join("unitigs.bin")) } /// Pack all partition matrices into single-file format (presence → .pbmx, counts → .pcmx). @@ -164,13 +187,14 @@ impl KmerIndex { /// on-disk format (see `DevDocMD/architecture/siblings.md`) — count /// matrices are unaffected, sparse count matrices aren't implemented /// (see the sparse-matrix design plan's "Explicitly deferred"). - pub fn pack_matrices(&self, sparse: bool) -> OKIResult<()> { + /// TODO: To be moved somewhere else + pub(crate) fn pack_matrices(&self, sparse: bool) -> OKIResult<()> { use obicompactvec::{pack_bit_matrix, pack_compact_int_matrix, pack_sparse_bit_matrix}; let n = self.n_partitions(); let order: Vec = (0..n).collect(); let pb = progress_bar("pack", n as u64, "partitions"); - crate::index::numa::PartitionRunner::new().run( + obisys::PartitionRunner::new().run( &order, |i| -> OKIResult<()> { let index_dir = self.index_dir(i); @@ -179,7 +203,7 @@ impl KmerIndex { } let n_layers = self.n_layers(i)?; for l in 0..n_layers { - let layer_dir = self.layer_dir(i, l); + let layer_dir = self.layer_dir(i, l)?; let presence_dir = layer_dir.join("presence"); let counts_dir = layer_dir.join("counts"); if presence_dir.exists() { @@ -207,7 +231,8 @@ impl KmerIndex { /// /// Old indexes were built before this file was required. The number of /// kmers is recovered from `unitigs.bin`, which is always present. - pub fn upgrade_layer_meta(&self) -> OKIResult<()> { + /// TODO: Should not exist anymore + pub(crate) fn upgrade_layer_meta(&self) -> OKIResult<()> { use obicompactvec::LayerMeta; use obiskio::UnitigFileReader; @@ -229,7 +254,15 @@ impl KmerIndex { } }; for l in 0..n_layers { - let layer_dir = self.layer_dir(i, l); + let layer_dir = match self.layer_dir(i, l) { + Ok(d) => d, + Err(e) => { + return Some(OKIError::Io(std::io::Error::new( + std::io::ErrorKind::Other, + e.to_string(), + ))); + } + }; let meta_path = layer_dir.join(LayerMeta::FILENAME); if meta_path.exists() { continue; diff --git a/src/obikindex/src/index/meta.rs b/src/obikindex/src/index/meta.rs index 7e272e20..f4a2a4d3 100644 --- a/src/obikindex/src/index/meta.rs +++ b/src/obikindex/src/index/meta.rs @@ -21,8 +21,44 @@ pub struct GenomeInfo { } impl GenomeInfo { + /// Panics if `label` fails [`validate_label`](Self::validate_label) — + /// an invalid genome label is a programmer error to surface loudly, + /// not a recoverable condition for this constructor to paper over. + /// Callers that need a graceful error (e.g. a CLI parsing user input) + /// should call [`validate_label`](Self::validate_label) themselves + /// first and report it their own way. pub fn new(label: impl Into) -> Self { - Self { label: label.into(), meta: HashMap::new() } + let label = label.into(); + Self::validate_label(&label).expect("invalid genome label"); + Self { + label, + meta: HashMap::new(), + } + } + + /// Validate a user-supplied genome label. + /// + /// Forbidden: `/` (filesystem separator), `=` (--new-label parser separator), + /// `\0` (null byte), `\n`, `\r`, `\t` (break CSV output). + /// Empty labels are also rejected. + pub fn validate_label(label: &str) -> Result<(), String> { + if label.is_empty() { + return Err("genome label must not be empty".into()); + } + const FORBIDDEN: &[char] = &['/', '=', '\0', '\n', '\r', '\t']; + if let Some(c) = label.chars().find(|c| FORBIDDEN.contains(c)) { + let display = match c { + '\0' => "\\0 (null)".to_string(), + '\n' => "\\n (newline)".to_string(), + '\r' => "\\r (carriage return)".to_string(), + '\t' => "\\t (tab)".to_string(), + c => format!("'{c}'"), + }; + return Err(format!( + "genome label contains forbidden character {display}" + )); + } + Ok(()) } } @@ -47,7 +83,7 @@ pub struct IndexConfig { /// On-disk shape of `index.meta` — the whole file, read/written atomically /// as one JSON blob by every `IndexMeta` operation below. #[derive(Debug, Clone, Serialize, Deserialize)] -struct OnDiskMeta { +struct IndexMetadata { version: u32, config: IndexConfig, #[serde(default)] @@ -81,21 +117,29 @@ impl IndexMeta { /// Create a brand-new `index.meta` for `index`, from fixed config and /// an initial genome list (often empty — genomes are usually added /// later via [`push_genome`](Self::push_genome)). - pub fn create(index: &KmerIndex, config: IndexConfig, genomes: Vec) -> io::Result { - Self::create_at(index.root_path(), config, genomes) + pub fn create( + index: &KmerIndex, + config: IndexConfig, + genomes: Vec, + ) -> io::Result { + Self::create_at(index.dir(), config, genomes) } /// Same as [`create`](Self::create), taking a raw path — for /// `KmerIndex::create` itself, which doesn't have a complete /// `KmerIndex` yet to hand in (it's what's being built). - pub(crate) fn create_at(root_path: &Path, config: IndexConfig, genomes: Vec) -> io::Result { + pub(crate) fn create_at( + root_path: &Path, + config: IndexConfig, + genomes: Vec, + ) -> io::Result { let meta = Self { root_path: root_path.to_owned(), version: META_VERSION, config, lock: RwLock::new(()), }; - meta.write_full(&OnDiskMeta { + meta.write_full(&IndexMetadata { version: meta.version, config: meta.config.clone(), genomes, @@ -106,7 +150,7 @@ impl IndexMeta { /// Reopen the `index.meta` already on disk for `index`. pub fn open(index: &KmerIndex) -> io::Result { - Self::open_at(index.root_path()) + Self::open_at(index.dir()) } pub(crate) fn open_at(root_path: &Path) -> io::Result { @@ -119,7 +163,12 @@ impl IndexMeta { }) } - pub fn exists(root: &Path) -> bool { + /// Returns whether the metadata file exists in `root`. + /// + /// This is a static method (associated function), not an instance method. + /// It only checks for the presence of the `index.meta` file inside the + /// given directory; it does not validate its contents. + pub(crate) fn exists(root: &Path) -> bool { root.join(META_FILENAME).exists() } @@ -177,10 +226,20 @@ impl IndexMeta { /// (e.g. `self.meta = Arc::new(IndexMeta::open(self)?)`) — this method /// only updates the file, it has no way to reach back into whatever /// `KmerIndex` holds it. - pub fn rewrite_config(&self, config: IndexConfig, genomes: Vec) -> io::Result<()> { + /// TODO: This methode is strange + pub(crate) fn rewrite_config( + &self, + config: IndexConfig, + genomes: Vec, + ) -> io::Result<()> { let _guard = self.lock.write().unwrap(); let state = Self::read_full(&self.root_path)?.state; - self.write_full(&OnDiskMeta { version: self.version, config, genomes, state }) + self.write_full(&IndexMetadata { + version: self.version, + config, + genomes, + state, + }) } pub fn set_state(&self, state: IndexState) -> io::Result<()> { @@ -221,12 +280,12 @@ impl IndexMeta { // ── private ────────────────────────────────────────────────────────────── - fn read_full(root: &Path) -> io::Result { + fn read_full(root: &Path) -> io::Result { let file = fs::File::open(root.join(META_FILENAME))?; serde_json::from_reader(file).map_err(io::Error::other) } - fn write_full(&self, on_disk: &OnDiskMeta) -> io::Result<()> { + fn write_full(&self, on_disk: &IndexMetadata) -> io::Result<()> { let file = fs::File::create(self.root_path.join(META_FILENAME))?; serde_json::to_writer_pretty(file, on_disk).map_err(io::Error::other) } @@ -248,26 +307,3 @@ fn label_from_path(path: &Path) -> String { s } } - -/// Validate a user-supplied genome label. -/// -/// Forbidden: `/` (filesystem separator), `=` (--new-label parser separator), -/// `\0` (null byte), `\n`, `\r`, `\t` (break CSV output). -/// Empty labels are also rejected. -pub fn validate_label(label: &str) -> Result<(), String> { - if label.is_empty() { - return Err("genome label must not be empty".into()); - } - const FORBIDDEN: &[char] = &['/', '=', '\0', '\n', '\r', '\t']; - if let Some(c) = label.chars().find(|c| FORBIDDEN.contains(c)) { - let display = match c { - '\0' => "\\0 (null)".to_string(), - '\n' => "\\n (newline)".to_string(), - '\r' => "\\r (carriage return)".to_string(), - '\t' => "\\t (tab)".to_string(), - c => format!("'{c}'"), - }; - return Err(format!("genome label contains forbidden character {display}")); - } - Ok(()) -} diff --git a/src/obikindex/src/index/mod.rs b/src/obikindex/src/index/mod.rs index 975676f0..8a60f488 100644 --- a/src/obikindex/src/index/mod.rs +++ b/src/obikindex/src/index/mod.rs @@ -1,39 +1,13 @@ pub mod error; pub mod meta; -pub mod predicate; pub mod state; mod builder; mod common; -mod distance; -mod dump; -mod dump_layer; -pub mod filter; -mod graph_pipeline; mod kmer_index; -mod matrix_store; -mod merge; -mod merge_layer; -mod numa; -mod query_layer; -mod rebuild; -mod rebuild_layer; -mod reindex; -mod select; -mod select_layer; -mod stats; pub use error::{OKIError, OKIResult}; pub use builder::IndexBuilder; -pub use common::olm_to_sk; -pub use graph_pipeline::{materialize_layer, write_graph_as_unitigs}; -pub use distance::{DistanceMetric, DistanceOutput}; -pub use filter::{GroupQuorumFilter, KmerFilter, passes_all}; +pub use common::ColBuilder; pub use kmer_index::KmerIndex; -pub use merge_layer::MergeMode; -pub use meta::{validate_label, GenomeInfo, IndexConfig, IndexMeta, META_FILENAME}; -pub use predicate::{GroupFilterParams, MetaPred}; -pub use query_layer::{KmerDesc, QueryHit, QueryStats}; -pub use select_layer::{AggOp, OutputCol}; +pub use meta::{GenomeInfo, IndexConfig, IndexMeta, META_FILENAME}; pub use state::IndexState; -pub use stats::IndexBitsPerKmer; -pub use numa::PartitionRunner; diff --git a/src/obikindex/src/index/progress.rs b/src/obikindex/src/index/progress.rs deleted file mode 100644 index 1c3f948c..00000000 --- a/src/obikindex/src/index/progress.rs +++ /dev/null @@ -1,21 +0,0 @@ -use std::sync::OnceLock; - -use indicatif::MultiProgress; - -static MULTI: OnceLock = OnceLock::new(); - -/// Initialise the shared progress display. Call once from the binary before -/// any index operation. Subsequent calls are silently ignored. -pub fn init(multi: MultiProgress) { - let _ = MULTI.set(multi); -} - -/// Return the shared `MultiProgress`, creating a plain default one if the -/// binary never called [`init`]. -pub fn get() -> &'static MultiProgress { - MULTI.get_or_init(MultiProgress::new) -} - -pub(crate) fn multi() -> &'static MultiProgress { - get() -} diff --git a/src/obikindex/src/layer/content_layer.rs b/src/obikindex/src/layer/content_layer.rs index 0222368d..c318ab0a 100644 --- a/src/obikindex/src/layer/content_layer.rs +++ b/src/obikindex/src/layer/content_layer.rs @@ -24,77 +24,103 @@ use std::path::{Path, PathBuf}; +use crate::layer::utils::LAYERNAME_SUFFIX; + use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix}; use obikseq::CanonicalKmer; -use crate::layer::error::OLMResult; -use crate::layer::meta::IndexMode; +use crate::index::error::OKIResult; use crate::layer::mphf_layer::{ EVIDENCE_FILE, EvidenceKind, FINGERPRINT_FILE, MPHF_FILE, UNITIGS_FILE, }; use crate::layer::typed_layer::{COUNTS_DIR, LayerContent, PRESENCE_DIR, TypedLayer}; +use crate::partition::KmerPartition; + +/// Superkmer-file extension used by every pre-layer-construction artifact +/// below (`raw`/`dereplicated`) — an implementation detail of the SK file +/// format, never meant to leak as a literal string past this module. +const SK_EXT: &str = "skmer.zst"; /// One layer, at any point in its life — see the module docs. Only /// [`Empty`](Layer::Empty) and the two ready-to-read states /// (`Count`/`Presence`) exist so far; states in between (unitigs written, /// MPHF built, evidence built, matrix not yet built) are not represented /// yet. +/// +/// Every variant carries `dir` + `id` — a layer's identity never depends on +/// its state. `id` is its own layer number within the partition (not yet +/// consumed by anything — `KmerPartition` doesn't hand it in yet, that +/// lands with `KmerLayer::at`). `dir`/`id` live here, on `KmerLayer` itself, +/// not inside `TypedLayer`/`MphfLayer`: those only ever need `dir` +/// transiently, while opening — `KmerLayer` is what has to answer `.dir()` +/// after the fact, so it's the one that keeps it. pub enum KmerLayer { /// Just a directory — nothing built yet. Every method below other than /// the path accessors panics on this variant: calling them means the /// caller assumed a layer was ready when it wasn't, an implementation /// error to surface loudly, not paper over with a default value. - Empty { + Empty { dir: PathBuf, id: usize }, + Count { dir: PathBuf, + id: usize, + layer: TypedLayer, + }, + Presence { + dir: PathBuf, + id: usize, + layer: TypedLayer, }, - Count(TypedLayer), - Presence(TypedLayer), } impl KmerLayer { + pub fn new(partition: &KmerPartition, i: usize) -> Self { + let dir = partition.dir().join(format!("{LAYERNAME_SUFFIX}{i}")); + KmerLayer::Empty { dir, id: i } + } /// An empty shell at `dir` — creates the directory (if it doesn't /// already exist) but nothing inside it. The starting state for /// building a new layer. - pub fn create(dir: &Path) -> std::io::Result { - std::fs::create_dir_all(dir)?; - Ok(KmerLayer::Empty { - dir: dir.to_owned(), - }) + pub fn create(self) -> std::io::Result { + std::fs::create_dir_all(self.dir())?; + Ok(self) } - /// Open one layer, auto-detecting count vs. presence from what is - /// actually on disk (`counts/` present and wanted, else presence) — the - /// single source of truth every caller that opens a layer's own matrix - /// should share, so two callers can never disagree about which format a - /// given layer is. Dense vs. sparse presence storage is - /// `PersistentBitMatrix::open`'s own concern (it's a 4-way - /// `Columnar`/`Packed`/`Sparse`/`Implicit` enum internally), not decided - /// here. - pub fn open(layer_dir: &Path, mode: &IndexMode, with_counts: bool) -> OLMResult { - if with_counts && layer_dir.join(COUNTS_DIR).exists() { - return TypedLayer::::open(layer_dir, mode) - .map(KmerLayer::Count); + /// Open this layer, auto-detecting count vs. presence from what is + /// actually on disk (`counts/` present, else presence) — the single + /// source of truth every caller that opens a layer's own matrix should + /// share, so two callers can never disagree about which format a given + /// layer is. Dense vs. sparse presence storage is `PersistentBitMatrix:: + /// open`'s own concern (a 4-way `Columnar`/`Packed`/`Sparse`/`Implicit` + /// enum internally), not decided here. A no-op (returns `self` + /// unchanged) if already `Count`/`Presence`. + pub fn open(self) -> OKIResult { + let (dir, id) = match self { + KmerLayer::Empty { dir, id } => (dir, id), + already_open => return Ok(already_open), + }; + if dir.join(COUNTS_DIR).exists() { + let layer = TypedLayer::::open(&dir)?; + Ok(KmerLayer::Count { dir, id, layer }) + } else { + let layer = TypedLayer::::open(&dir)?; + Ok(KmerLayer::Presence { dir, id, layer }) } - TypedLayer::::open(layer_dir, mode).map(KmerLayer::Presence) } - // ── Paths — only meaningful before anything is built ─────────────── + // ── Paths — valid in every state, `dir` never varies with content ──── // - // Once a layer is `Count`/`Presence`, its caller already knows the - // directory (it had to pass it to `open`) — these exist for builder - // code holding an `Empty` layer, so the `mphf.bin`/`unitigs.bin`/… - // naming stays defined once, here, rather than re-declared as - // string literals at every call site that writes into a layer - // directory (the same duplication `layer_dir`/`index_dir` fixed one - // level up — see `DevDocMD/implementation/partition_layer_cache.md`). + // The `mphf.bin`/`unitigs.bin`/… naming stays defined once, here, + // rather than re-declared as string literals at every call site that + // writes into a layer directory (the same duplication `layer_dir`/ + // `index_dir` fixed one level up — see `DevDocMD/implementation/ + // partition_layer_cache.md`). - /// This layer's own directory. Panics on `Count`/`Presence` — by that - /// point the caller already has the directory it opened with; asking - /// again here would mean it lost track of its own state. + /// This layer's own directory. pub fn dir(&self) -> &Path { match self { - KmerLayer::Empty { dir } => dir, - _ => panic!("Layer::dir() only available on Empty — caller already has this path"), + KmerLayer::Empty { dir, .. } => dir, + KmerLayer::Count { dir, .. } => dir, + KmerLayer::Presence { dir, .. } => dir, } } @@ -117,36 +143,57 @@ impl KmerLayer { self.dir().join(PRESENCE_DIR) } + /// Path of this layer's raw, not-yet-dereplicated superkmer file — + /// written by whichever algorithm routes superkmers into this layer + /// (today: `obikindexer::algorithms::partitionner::PartitionRouter`), + /// read by whichever algorithm dereplicates it (today: + /// `obikindexer::algorithms::dereplicator::Dereplicator`). Naming this + /// once here, rather than in either algorithm submodule, is what lets + /// the two agree on the filename without depending on each other + /// directly — see `DevDocMD/implementation/partition_layer_cache.md`. + pub fn raw_superkmers_path(&self) -> PathBuf { + self.dir().join(format!("raw.{SK_EXT}")) + } + + /// Path of this layer's dereplicated superkmer file — written by + /// `obikindexer::algorithms::dereplicator::Dereplicator`, read by + /// whichever algorithm counts kmer abundances from it (today: + /// `obikindexer::algorithms::partitionner::PartitionRouter::count_kmer`) + /// and, later, by `obikindex::build_index_layer` to build the real layer. + pub fn dereplicated_superkmers_path(&self) -> PathBuf { + self.dir().join(format!("dereplicated.{SK_EXT}")) + } + // ── Ready-only surface ─────────────────────────────────────────────── pub fn content(&self) -> LayerContent { match self { - KmerLayer::Count(_) => LayerContent::Count, - KmerLayer::Presence(_) => LayerContent::Presence, + KmerLayer::Count { .. } => LayerContent::Count, + KmerLayer::Presence { .. } => LayerContent::Presence, KmerLayer::Empty { .. } => panic!("Layer::content() called on an Empty layer"), } } pub fn evidence_kind(&self) -> EvidenceKind { match self { - KmerLayer::Count(l) => l.evidence_kind(), - KmerLayer::Presence(l) => l.evidence_kind(), + KmerLayer::Count { layer, .. } => layer.evidence_kind(), + KmerLayer::Presence { layer, .. } => layer.evidence_kind(), KmerLayer::Empty { .. } => panic!("Layer::evidence_kind() called on an Empty layer"), } } pub fn n(&self) -> usize { match self { - KmerLayer::Count(l) => l.n(), - KmerLayer::Presence(l) => l.n(), + KmerLayer::Count { layer, .. } => layer.n(), + KmerLayer::Presence { layer, .. } => layer.n(), KmerLayer::Empty { .. } => panic!("Layer::n() called on an Empty layer"), } } pub fn find_slot(&self, kmer: CanonicalKmer) -> Option { match self { - KmerLayer::Count(l) => l.find_slot(kmer), - KmerLayer::Presence(l) => l.find_slot(kmer), + KmerLayer::Count { layer, .. } => layer.find_slot(kmer), + KmerLayer::Presence { layer, .. } => layer.find_slot(kmer), KmerLayer::Empty { .. } => panic!("Layer::find_slot() called on an Empty layer"), } } @@ -156,16 +203,16 @@ impl KmerLayer { /// the evidence check `find_slot`/`find` would perform is redundant. pub fn hash_batch(&self, kmers: &[CanonicalKmer]) -> Vec { match self { - KmerLayer::Count(l) => l.hash_batch(kmers), - KmerLayer::Presence(l) => l.hash_batch(kmers), + KmerLayer::Count { layer, .. } => layer.hash_batch(kmers), + KmerLayer::Presence { layer, .. } => layer.hash_batch(kmers), KmerLayer::Empty { .. } => panic!("Layer::hash_batch() called on an Empty layer"), } } pub fn n_cols(&self) -> usize { match self { - KmerLayer::Count(l) => l.n_cols(), - KmerLayer::Presence(l) => l.n_cols(), + KmerLayer::Count { layer, .. } => layer.n_cols(), + KmerLayer::Presence { layer, .. } => layer.n_cols(), KmerLayer::Empty { .. } => panic!("Layer::n_cols() called on an Empty layer"), } } @@ -178,10 +225,10 @@ impl KmerLayer { /// presence (`!= 0`) in place. pub fn fill_sub_matrix_carries(&self, slots: &[usize], out: &mut [Vec]) { match self { - KmerLayer::Presence(l) => l.fill_sub_matrix(slots, out), - KmerLayer::Count(l) => { + KmerLayer::Presence { layer, .. } => layer.fill_sub_matrix(slots, out), + KmerLayer::Count { layer, .. } => { let mut counts: Vec> = out.iter().map(|_| Vec::new()).collect(); - l.fill_sub_matrix(slots, &mut counts); + layer.fill_sub_matrix(slots, &mut counts); for (o, c) in out.iter_mut().zip(counts.iter()) { o.clear(); o.extend(c.iter().map(|&v| v != 0)); @@ -196,8 +243,8 @@ impl KmerLayer { /// Raw MPHF lookup: kmer → slot, no membership check. pub fn hash(&self, kmer: CanonicalKmer) -> usize { match self { - KmerLayer::Count(l) => l.hash(kmer), - KmerLayer::Presence(l) => l.hash(kmer), + KmerLayer::Count { layer, .. } => layer.hash(kmer), + KmerLayer::Presence { layer, .. } => layer.hash(kmer), KmerLayer::Empty { .. } => panic!("Layer::hash() called on an Empty layer"), } } @@ -205,8 +252,8 @@ impl KmerLayer { /// Iterate over all canonical kmers in the layer, in deterministic order. pub fn iter_kmers(&self) -> crate::layer::mphf_layer::KmerIter { match self { - KmerLayer::Count(l) => l.iter_kmers(), - KmerLayer::Presence(l) => l.iter_kmers(), + KmerLayer::Count { layer, .. } => layer.iter_kmers(), + KmerLayer::Presence { layer, .. } => layer.iter_kmers(), KmerLayer::Empty { .. } => panic!("Layer::iter_kmers() called on an Empty layer"), } } @@ -215,8 +262,8 @@ impl KmerLayer { /// sequence index in `unitigs.bin`. pub fn enumerate_kmers(&self) -> std::iter::Enumerate { match self { - KmerLayer::Count(l) => l.enumerate_kmers(), - KmerLayer::Presence(l) => l.enumerate_kmers(), + KmerLayer::Count { layer, .. } => layer.enumerate_kmers(), + KmerLayer::Presence { layer, .. } => layer.enumerate_kmers(), KmerLayer::Empty { .. } => panic!("Layer::enumerate_kmers() called on an Empty layer"), } } @@ -224,8 +271,8 @@ impl KmerLayer { /// Iterate over the layer's canonical kmers in batches of `n`. pub fn iter_kmers_batch(&self, n: usize) -> crate::layer::mphf_layer::KmerBatchIter { match self { - KmerLayer::Count(l) => l.iter_kmers_batch(n), - KmerLayer::Presence(l) => l.iter_kmers_batch(n), + KmerLayer::Count { layer, .. } => layer.iter_kmers_batch(n), + KmerLayer::Presence { layer, .. } => layer.iter_kmers_batch(n), KmerLayer::Empty { .. } => panic!("Layer::iter_kmers_batch() called on an Empty layer"), } } @@ -237,8 +284,8 @@ impl KmerLayer { n: usize, ) -> Box)> + Send + 'static> { match self { - KmerLayer::Count(l) => Box::new(l.enumerate_kmers_batch(n)), - KmerLayer::Presence(l) => Box::new(l.enumerate_kmers_batch(n)), + KmerLayer::Count { layer, .. } => Box::new(layer.enumerate_kmers_batch(n)), + KmerLayer::Presence { layer, .. } => Box::new(layer.enumerate_kmers_batch(n)), KmerLayer::Empty { .. } => { panic!("Layer::enumerate_kmers_batch() called on an Empty layer") } diff --git a/src/obikindex/src/layer/error.rs b/src/obikindex/src/layer/error.rs deleted file mode 100644 index c6b4e7f6..00000000 --- a/src/obikindex/src/layer/error.rs +++ /dev/null @@ -1,50 +0,0 @@ -use std::fmt; -use std::io; - -#[derive(Debug)] -pub enum OLMError { - Io(io::Error), - Json(serde_json::Error), - Mphf(String), - InvalidLayer(String), -} - -pub type OLMResult = Result; - -impl fmt::Display for OLMError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - OLMError::Io(e) => write!(f, "I/O error: {e}"), - OLMError::Json(e) => write!(f, "JSON error: {e}"), - OLMError::Mphf(s) => write!(f, "MPHF error: {s}"), - OLMError::InvalidLayer(s) => write!(f, "invalid layer: {s}"), - } - } -} - -impl std::error::Error for OLMError { - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - match self { - OLMError::Io(e) => Some(e), - OLMError::Json(e) => Some(e), - _ => None, - } - } -} - -impl From for OLMError { - fn from(e: io::Error) -> Self { OLMError::Io(e) } -} - -impl From for OLMError { - fn from(e: serde_json::Error) -> Self { OLMError::Json(e) } -} - -impl From for OLMError { - fn from(e: obiskio::SKError) -> Self { - match e { - obiskio::SKError::Io(io_err) => OLMError::Io(io_err), - other => OLMError::InvalidLayer(other.to_string()), - } - } -} diff --git a/src/obikindex/src/layer/evidence.rs b/src/obikindex/src/layer/evidence.rs index 48736e42..15cd126c 100644 --- a/src/obikindex/src/layer/evidence.rs +++ b/src/obikindex/src/layer/evidence.rs @@ -6,14 +6,14 @@ use std::path::Path; use memmap2::Mmap; -use crate::layer::error::{OLMError, OLMResult}; +use crate::index::error::{OKIError, OKIResult}; pub struct Evidence { mmap: Mmap, } impl Evidence { - pub fn open(path: &Path) -> OLMResult { + pub fn open(path: &Path) -> OKIResult { let f = File::open(path)?; let mmap = unsafe { Mmap::map(&f)? }; Ok(Self { mmap }) @@ -26,16 +26,16 @@ impl Evidence { (raw >> 7, (raw & 0x7F) as u8) } + #[inline] + pub fn encode(chunk_id: u32, rank: u8) -> u32 { + (chunk_id << 7) | (rank as u32 & 0x7F) + } + pub fn len(&self) -> usize { self.mmap.len() / 4 } } -#[inline] -pub fn encode(chunk_id: u32, rank: u8) -> u32 { - (chunk_id << 7) | (rank as u32 & 0x7F) -} - pub struct EvidenceWriter { buf: Vec, } @@ -47,13 +47,13 @@ impl EvidenceWriter { #[inline] pub fn set(&mut self, slot: usize, chunk_id: u32, rank: u8) { - self.buf[slot] = encode(chunk_id, rank); + self.buf[slot] = Evidence::encode(chunk_id, rank); } - pub fn write(self, path: &Path) -> OLMResult<()> { + pub fn write(self, path: &Path) -> OKIResult<()> { let mut f = BufWriter::new(File::create(path)?); for v in self.buf { - f.write_all(&v.to_le_bytes()).map_err(OLMError::Io)?; + f.write_all(&v.to_le_bytes()).map_err(OKIError::Io)?; } Ok(()) } diff --git a/src/obikindex/src/layer/fingerprint.rs b/src/obikindex/src/layer/fingerprint.rs index 2297fc15..c5c199d8 100644 --- a/src/obikindex/src/layer/fingerprint.rs +++ b/src/obikindex/src/layer/fingerprint.rs @@ -14,7 +14,7 @@ use std::path::Path; use bitvec::prelude::*; use memmap2::Mmap; -use crate::layer::error::{OLMError, OLMResult}; +use crate::index::error::{OKIError, OKIResult}; const MAGIC: &[u8; 4] = b"FPVF"; const HEADER: usize = 16; @@ -30,15 +30,15 @@ pub struct FingerprintVec { } impl FingerprintVec { - pub fn open(path: &Path) -> OLMResult { + pub fn open(path: &Path) -> OKIResult { let f = File::open(path)?; let mmap = unsafe { Mmap::map(&f)? }; if mmap.len() < HEADER || &mmap[..4] != MAGIC { - return Err(OLMError::InvalidLayer("bad fingerprint magic".into())); + return Err(OKIError::InvalidData { context: "layer", detail: "bad fingerprint magic".into() }); } let b = mmap[4]; if b == 0 || b > 64 { - return Err(OLMError::InvalidLayer("invalid fingerprint width".into())); + return Err(OKIError::InvalidData { context: "layer", detail: "invalid fingerprint width".into() }); } let n = u64::from_le_bytes(mmap[8..16].try_into().unwrap()) as usize; let mask: u64 = if b == 64 { u64::MAX } else { (1u64 << b) - 1 }; @@ -89,12 +89,12 @@ impl FingerprintVecWriter { bits[lo .. lo + self.b as usize].store_le(fingerprint); } - pub fn write(self, path: &Path) -> OLMResult<()> { - let mut f = BufWriter::new(File::create(path).map_err(OLMError::Io)?); - f.write_all(MAGIC).map_err(OLMError::Io)?; - f.write_all(&[self.b, 0, 0, 0]).map_err(OLMError::Io)?; - f.write_all(&(self.n as u64).to_le_bytes()).map_err(OLMError::Io)?; - f.write_all(&self.buf).map_err(OLMError::Io)?; + pub fn write(self, path: &Path) -> OKIResult<()> { + let mut f = BufWriter::new(File::create(path).map_err(OKIError::Io)?); + f.write_all(MAGIC).map_err(OKIError::Io)?; + f.write_all(&[self.b, 0, 0, 0]).map_err(OKIError::Io)?; + f.write_all(&(self.n as u64).to_le_bytes()).map_err(OKIError::Io)?; + f.write_all(&self.buf).map_err(OKIError::Io)?; Ok(()) } } diff --git a/src/obikindex/src/layer/map.rs b/src/obikindex/src/layer/map.rs deleted file mode 100644 index 5e77c1cc..00000000 --- a/src/obikindex/src/layer/map.rs +++ /dev/null @@ -1,128 +0,0 @@ -use std::collections::HashMap; -use std::fs; -use std::path::{Path, PathBuf}; - -use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix, StorageKind}; -use obikseq::CanonicalKmer; -use obiskio::{UnitigFileWriter, DEFAULT_BLOCK_BITS}; - -use crate::layer::error::{OLMError, OLMResult}; -use crate::layer::typed_layer::{layer_dir, Hit, TypedLayer, LayerContent, LayerData}; -use crate::layer::meta::{IndexMode, PartitionMeta}; -use crate::layer::mphf_layer::EvidenceKind; - -/// Layered kmer index for a single partition. -/// -/// Each layer covers a disjoint kmer set. Queries probe layers in order; -/// the first match wins. Adding a dataset appends a new layer without -/// rebuilding existing ones. -pub struct LayeredMap { - root: PathBuf, - meta: PartitionMeta, - layers: Vec>, -} - -// ── Common methods ──────────────────────────────────────────────────────────── - -impl LayeredMap { - /// Open an existing layered index at `root`. - /// The mode is read once from `PartitionMeta` and applied to all layers. - pub fn open(root: &Path) -> OLMResult { - let meta = PartitionMeta::load(root)?; - let layers = (0..meta.n_layers) - .map(|i| TypedLayer::::open(&layer_dir(root, i), &meta.mode)) - .collect::>>()?; - Ok(Self { root: root.to_owned(), meta, layers }) - } - - /// Create a new, empty layered index at `root` with the given mode. - pub fn create(root: &Path, mode: IndexMode) -> OLMResult { - fs::create_dir_all(root)?; - let meta = PartitionMeta::new(mode); - meta.save(root)?; - Ok(Self { root: root.to_owned(), meta, layers: Vec::new() }) - } - - pub fn n_layers(&self) -> usize { self.layers.len() } - pub fn layer(&self, i: usize) -> &TypedLayer { &self.layers[i] } - pub fn mode(&self) -> &IndexMode { &self.meta.mode } - - /// TypedLayer `i`'s content (`Count`/`Presence`) — a lightweight disk probe, - /// no MPHF/matrix opened. For picking a `D` before committing to - /// [`TypedLayer::open`], or for reporting/diagnostics over an already-open - /// `LayeredMap` without re-opening every layer under a different `D`. - pub fn detect_layer_content(&self, i: usize) -> LayerContent { - LayerContent::detect(&layer_dir(&self.root, i)) - } - - /// TypedLayer `i`'s storage format — a lightweight disk probe, no - /// MPHF/matrix opened. See [`detect_layer_content`](Self::detect_layer_content). - pub fn detect_layer_storage(&self, i: usize) -> OLMResult { - let dir = layer_dir(&self.root, i); - match LayerContent::detect(&dir) { - LayerContent::Count => PersistentCompactIntMatrix::detect_storage(&dir).map_err(OLMError::Io), - LayerContent::Presence => PersistentBitMatrix::detect_storage(&dir).map_err(OLMError::Io), - } - } - - /// TypedLayer `i`'s evidence mode — a lightweight disk probe, no - /// MPHF/matrix opened. See [`detect_layer_content`](Self::detect_layer_content). - pub fn detect_layer_evidence(&self, i: usize) -> OLMResult { - EvidenceKind::detect(&layer_dir(&self.root, i)) - } - - /// Query `kmer` across all layers. Returns `(layer_index, Hit)` on match. - pub fn query(&self, kmer: CanonicalKmer) -> Option<(usize, Hit)> { - self.layers - .iter() - .enumerate() - .find_map(|(i, layer)| layer.query(kmer).map(|hit| (i, hit))) - } - - pub fn next_layer_writer(&self) -> OLMResult { - let dir = layer_dir(&self.root, self.layers.len()); - TypedLayer::::unitig_writer(&dir) - } - - fn append_layer(&mut self) -> OLMResult<()> { - let i = self.layers.len(); - let dir = layer_dir(&self.root, i); - self.layers.push(TypedLayer::::open(&dir, &self.meta.mode)?); - self.meta.n_layers = self.layers.len(); - self.meta.save(&self.root)?; - Ok(()) - } -} - -// ── Mode 1 — set membership ─────────────────────────────────────────────────── - -impl LayeredMap<()> { - pub fn push_layer(&mut self) -> OLMResult { - let i = self.layers.len(); - let dir = layer_dir(&self.root, i); - TypedLayer::<()>::build(&dir, DEFAULT_BLOCK_BITS, &self.meta.mode)?; - self.append_layer()?; - Ok(i) - } -} - -// ── Mode 2 — count matrix ───────────────────────────────────────────────────── - -impl LayeredMap { - pub fn push_layer(&mut self, count_of: impl Fn(CanonicalKmer) -> u32) -> OLMResult { - let i = self.layers.len(); - let dir = layer_dir(&self.root, i); - TypedLayer::::build(&dir, DEFAULT_BLOCK_BITS, &self.meta.mode, count_of)?; - self.append_layer()?; - Ok(i) - } - - pub fn push_layer_from_map(&mut self, counts: &HashMap) -> OLMResult { - self.push_layer(|kmer| counts.get(&kmer).copied().unwrap_or(0)) - } -} - -#[cfg(test)] -#[path = "tests/map.rs"] -mod tests; - diff --git a/src/obikindex/src/layer/meta.rs b/src/obikindex/src/layer/meta.rs index 8c4de76b..935ae45f 100644 --- a/src/obikindex/src/layer/meta.rs +++ b/src/obikindex/src/layer/meta.rs @@ -1,18 +1,20 @@ -use std::fs::File; use std::path::Path; use serde::{Deserialize, Serialize}; -use crate::layer::error::OLMResult; - -const META_FILE: &str = "meta.json"; +use crate::index::error::OKIResult; +use crate::layer::fingerprint::FingerprintVec; +use crate::layer::mphf_layer::{EvidenceKind, FINGERPRINT_FILE}; +use crate::layer::utils::layer_dir; // ── IndexMode ───────────────────────────────────────────────────────────────── /// Evidence mode for an entire partitioned index — homogeneous across all layers. /// -/// Determined once at build time; stored in `PartitionMeta` (`meta.json`). -/// All layers within an index share the same mode. +/// Determined once at build time. All layers within an index share the same +/// mode, so it is never persisted separately — [`IndexMode::detect`] reads it +/// straight back from whichever evidence file(s) layer 0 actually has on +/// disk, the same probe [`EvidenceKind::detect`] uses. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "type", rename_all = "snake_case")] pub enum IndexMode { @@ -20,7 +22,9 @@ pub enum IndexMode { Exact, /// Approximate evidence: `fingerprint.bin` only. /// `b` — fingerprint bits per slot; false-positive rate ≈ 1/2^b per query. - /// `z` — Findere consecutive-kmer parameter (build-time only; not used at query time). + /// `z` — Findere consecutive-kmer parameter (build-time only; not used at + /// query time, not stored on disk, always `0` once recovered via + /// [`detect`](Self::detect)). Approx { b: u8, z: u8 }, /// Hybrid: both `fingerprint.bin` and `evidence.bin` + `unitigs.bin.idx`. /// `find()` uses the fingerprint (O(1), approx); `find_strict()` uses exact evidence. @@ -31,33 +35,24 @@ impl Default for IndexMode { fn default() -> Self { Self::Exact } } -// ── PartitionMeta ───────────────────────────────────────────────────────────── - -/// Index-level metadata stored in `meta.json` at the root of a partition index. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PartitionMeta { - pub n_layers: usize, - #[serde(default)] - pub mode: IndexMode, -} - -impl PartitionMeta { - pub fn new(mode: IndexMode) -> Self { - Self { n_layers: 0, mode } - } - - pub fn load(dir: &Path) -> OLMResult { - let f = File::open(dir.join(META_FILE))?; - Ok(serde_json::from_reader(f)?) - } - - pub fn save(&self, dir: &Path) -> OLMResult<()> { - let f = File::create(dir.join(META_FILE))?; - serde_json::to_writer_pretty(f, self)?; - Ok(()) +impl IndexMode { + /// Detect the evidence mode actually on disk, from partition index + /// directory `root`'s layer 0 — the single source of truth every layer + /// in the same index shares. `z` cannot be recovered (not stored in + /// `fingerprint.bin`, and irrelevant past build time), so it comes back + /// as `0`. + pub fn detect(root: &Path) -> OKIResult { + let layer0 = layer_dir(root, 0); + Ok(match EvidenceKind::detect(&layer0)? { + EvidenceKind::Exact => IndexMode::Exact, + EvidenceKind::Approx => IndexMode::Approx { + b: FingerprintVec::open(&layer0.join(FINGERPRINT_FILE))?.b(), + z: 0, + }, + EvidenceKind::Hybrid => IndexMode::Hybrid { + b: FingerprintVec::open(&layer0.join(FINGERPRINT_FILE))?.b(), + z: 0, + }, + }) } } - -impl Default for PartitionMeta { - fn default() -> Self { Self::new(IndexMode::Exact) } -} diff --git a/src/obikindex/src/layer/mod.rs b/src/obikindex/src/layer/mod.rs index a3021aee..fb0b1334 100644 --- a/src/obikindex/src/layer/mod.rs +++ b/src/obikindex/src/layer/mod.rs @@ -1,20 +1,13 @@ pub mod content_layer; -pub mod error; pub mod evidence; pub mod fingerprint; -pub mod layered_store; -pub mod map; pub mod meta; pub(crate) mod mphf_layer; pub mod typed_layer; +pub(crate) mod utils; +pub use crate::index::error::{OKIError, OKIResult}; pub use content_layer::KmerLayer; -pub use error::{OLMError, OLMResult}; -pub use layered_store::LayeredStore; -pub use map::LayeredMap; -pub use meta::{IndexMode, PartitionMeta}; +pub use meta::IndexMode; pub use mphf_layer::{EvidenceKind, KmerBatchIter, KmerIter, MphfLayer, MphfOnly}; -pub use typed_layer::{ - HasLayerContent, HasStorageKind, Hit, LayerContent, LayerData, TypedLayer, - dereplicated_superkmers_path, layer_dir, open_data, raw_superkmers_path, -}; +pub use typed_layer::{HasLayerContent, HasStorageKind, Hit, LayerContent, LayerData, TypedLayer}; diff --git a/src/obikindex/src/layer/mphf_layer.rs b/src/obikindex/src/layer/mphf_layer.rs index a75e9340..ce16c2b7 100644 --- a/src/obikindex/src/layer/mphf_layer.rs +++ b/src/obikindex/src/layer/mphf_layer.rs @@ -10,7 +10,7 @@ use obikseq::CanonicalKmer; use obiskio::{CanonicalKmerIter, UnitigFileReader, UnitigFileWriter, build_unitig_idx}; use ptr_hash::{PtrHash, PtrHashParams, bucket_fn::CubicEps, hash::Xx64}; -use crate::layer::error::{OLMError, OLMResult}; +use crate::index::error::{OKIError, OKIResult}; use crate::layer::evidence::{Evidence, EvidenceWriter}; use crate::layer::fingerprint::{FingerprintVec, FingerprintVecWriter}; use crate::layer::meta::IndexMode; @@ -76,7 +76,7 @@ impl LayerEvidence { /// Identify which evidence files exist at `layer_dir`, without opening /// any of them — a lightweight disk probe (same signal `EvidenceKind:: /// detect` uses), not a load. - fn at(layer_dir: &Path) -> OLMResult { + fn at(layer_dir: &Path) -> OKIResult { Ok(match EvidenceKind::detect(layer_dir)? { EvidenceKind::Exact => LayerEvidence::Exact { evidence_path: layer_dir.join(EVIDENCE_FILE), @@ -94,7 +94,7 @@ impl LayerEvidence { } /// mmap the evidence file(s) this identity points to. - fn open(self) -> OLMResult { + fn open(self) -> OKIResult { Ok(match self { LayerEvidence::Exact { evidence_path, unitig_path } => OpenLayerEvidence::Exact { evidence: Evidence::open(&evidence_path)?, @@ -127,17 +127,17 @@ impl EvidenceKind { /// Lightweight disk probe, no MPHF/evidence opened: presence of /// `evidence.bin`/`fingerprint.bin` alone determines the mode — same /// signal `MphfLayer::open` uses, just without opening either file. - pub fn detect(layer_dir: &Path) -> OLMResult { + pub fn detect(layer_dir: &Path) -> OKIResult { let has_evidence = layer_dir.join(EVIDENCE_FILE).exists(); let has_fingerprint = layer_dir.join(FINGERPRINT_FILE).exists(); match (has_evidence, has_fingerprint) { (true, false) => Ok(EvidenceKind::Exact), (false, true) => Ok(EvidenceKind::Approx), (true, true) => Ok(EvidenceKind::Hybrid), - (false, false) => Err(OLMError::InvalidLayer(format!( + (false, false) => Err(OKIError::InvalidData { context: "layer", detail: format!( "no evidence.bin or fingerprint.bin in {}", layer_dir.display() - ))), + ) }), } } } @@ -151,7 +151,6 @@ impl EvidenceKind { /// - [`find_strict`](Self::find_strict) — always exact; O(1) on Exact/Hybrid layers, /// O(n) sequential scan on Approx layers. pub struct MphfLayer { - layer_dir: PathBuf, /// Mode-independent copy — used by `iter_kmers` (all modes) and /// `find_strict`'s `Approx` fallback. `Exact`/`Hybrid` additionally /// carry their own inside `OpenLayerEvidence` (see its doc comment for @@ -165,18 +164,15 @@ pub struct MphfLayer { } impl MphfLayer { - /// This layer's own directory. - pub(crate) fn dir(&self) -> &Path { - &self.layer_dir - } - /// Open a layer — evidence mode is auto-detected from which files exist /// on disk (see [`LayerEvidence::at`]), never passed in: the caller /// can't tell this type anything about its own mode that isn't already - /// derivable from `layer_dir` itself. - pub fn open(dir: &Path) -> OLMResult { + /// derivable from `dir` itself. `dir` is only used transiently here — + /// not stored: `KmerLayer`, one level up, is what owns a layer's + /// directory as a persistent field, not this type. + pub fn open(dir: &Path) -> OKIResult { let mphf: MemCase = Mphf::mmap(&dir.join(MPHF_FILE), Flags::empty()) - .map_err(|e| OLMError::InvalidLayer(e.to_string()))?; + .map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?; let unitigs = Arc::new(UnitigFileReader::open(&dir.join(UNITIGS_FILE))?); let ev = LayerEvidence::at(dir)?.open()?; let n = match &ev { @@ -184,7 +180,7 @@ impl MphfLayer { OpenLayerEvidence::Hybrid { evidence, .. } => evidence.len(), OpenLayerEvidence::Approx { fingerprint } => fingerprint.n(), }; - Ok(Self { layer_dir: dir.to_owned(), unitigs, mphf, ev, n }) + Ok(Self { unitigs, mphf, ev, n }) } // ── Query API ───────────────────────────────────────────────────────────── @@ -405,9 +401,9 @@ impl Iterator for KmerBatchIter { pub struct MphfOnly(MemCase); impl MphfOnly { - pub fn open(dir: &Path) -> OLMResult { + pub fn open(dir: &Path) -> OKIResult { let mphf: MemCase = Mphf::mmap(&dir.join(MPHF_FILE), Flags::empty()) - .map_err(|e| OLMError::InvalidLayer(e.to_string()))?; + .map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?; Ok(Self(mphf)) } @@ -421,13 +417,13 @@ impl MphfOnly { impl MphfLayer { // ── Build helpers ───────────────────────────────────────────────────────── - pub fn unitig_writer(dir: &Path) -> OLMResult { + pub fn unitig_writer(dir: &Path) -> OKIResult { fs::create_dir_all(dir)?; Ok(UnitigFileWriter::create(&dir.join(UNITIGS_FILE))?) } /// Build `evidence.bin` + `unitigs.bin.idx` from `unitigs.bin` + `mphf.bin`. - pub fn build_exact_evidence(dir: &Path, block_bits: u8) -> OLMResult { + pub fn build_exact_evidence(dir: &Path, block_bits: u8) -> OKIResult { let unitig_path = dir.join(UNITIGS_FILE); let unitigs = UnitigFileReader::open_sequential(&unitig_path)?; let n = unitigs.n_kmers(); @@ -439,7 +435,7 @@ impl MphfLayer { } let mphf: Mphf = Mphf::load_full(&dir.join(MPHF_FILE)) - .map_err(|e| OLMError::InvalidLayer(e.to_string()))?; + .map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?; let mut ev = EvidenceWriter::new(n); let mut seen = vec![0u8; (n + 7) / 8]; @@ -447,12 +443,12 @@ impl MphfLayer { for (kmer, chunk_id, rank) in unitigs.iter_indexed_canonical_kmers() { let slot = mphf.index(&kmer.raw()); if slot >= n { - return Err(OLMError::Mphf("slot out of bounds".into())); + return Err(OKIError::InvalidData { context: "mphf", detail: "slot out of bounds".into() }); } let byte = slot / 8; let bit = 1u8 << (slot % 8); if seen[byte] & bit != 0 { - return Err(OLMError::Mphf("duplicate slot".into())); + return Err(OKIError::InvalidData { context: "mphf", detail: "duplicate slot".into() }); } seen[byte] |= bit; ev.set(slot, chunk_id as u32, rank as u8); @@ -464,14 +460,14 @@ impl MphfLayer { } /// Build `fingerprint.bin` from `unitigs.bin` + `mphf.bin`. - pub fn build_approx_evidence(dir: &Path, b: u8, z: u8) -> OLMResult { + pub fn build_approx_evidence(dir: &Path, b: u8, z: u8) -> OKIResult { if b == 0 || b > 64 { - return Err(OLMError::InvalidLayer( + return Err(OKIError::InvalidData { context: "layer", detail: "fingerprint width must be 1..=64".into(), - )); + }); } if z == 0 { - return Err(OLMError::InvalidLayer("z must be ≥ 1".into())); + return Err(OKIError::InvalidData { context: "layer", detail: "z must be ≥ 1".into() }); } let unitig_path = dir.join(UNITIGS_FILE); let unitigs = UnitigFileReader::open_sequential(&unitig_path)?; @@ -483,14 +479,14 @@ impl MphfLayer { } let mphf: Mphf = Mphf::load_full(&dir.join(MPHF_FILE)) - .map_err(|e| OLMError::InvalidLayer(e.to_string()))?; + .map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?; let mut fw = FingerprintVecWriter::new(n, b); for (kmer, _, _) in unitigs.iter_indexed_canonical_kmers() { let slot = mphf.index(&kmer.raw()); if slot >= n { - return Err(OLMError::Mphf("slot out of bounds".into())); + return Err(OKIError::InvalidData { context: "mphf", detail: "slot out of bounds".into() }); } fw.set(slot, kmer.seq_hash()); } @@ -508,24 +504,24 @@ impl MphfLayer { dir: &Path, block_bits: u8, mode: &IndexMode, - fill_slot: &mut impl FnMut(usize, CanonicalKmer) -> OLMResult<()>, - ) -> OLMResult { + fill_slot: &mut impl FnMut(usize, CanonicalKmer) -> OKIResult<()>, + ) -> OKIResult { use rayon::prelude::*; let unitig_path = dir.join(UNITIGS_FILE); let n = UnitigFileReader::open_sequential(&unitig_path)?.n_kmers(); let sk_to_olm = |e: obiskio::SKError| match e { - obiskio::SKError::Io(io) => OLMError::Io(io), - e => OLMError::InvalidLayer(e.to_string()), + obiskio::SKError::Io(io) => OKIError::Io(io), + e => OKIError::InvalidData { context: "layer", detail: e.to_string() }, }; // ── Empty layer ─────────────────────────────────────────────────────── if n == 0 { let mphf: Mphf = Mphf::try_new(&[] as &[u64], PtrHashParams::::default()) - .ok_or_else(|| OLMError::Mphf("construction failed".into()))?; + .ok_or_else(|| OKIError::InvalidData { context: "mphf", detail: "construction failed".into() })?; mphf.store(&dir.join(MPHF_FILE)) - .map_err(|e| OLMError::InvalidLayer(e.to_string()))?; + .map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?; match mode { IndexMode::Exact | IndexMode::Hybrid { .. } => { fs::File::create(dir.join(EVIDENCE_FILE))?; @@ -549,7 +545,7 @@ impl MphfLayer { PtrHashParams::::default(), ); mphf.store(&dir.join(MPHF_FILE)) - .map_err(|e| OLMError::InvalidLayer(e.to_string()))?; + .map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?; // ── Pass 2: fill evidence files + callback ──────────────────────────── let unitigs2 = UnitigFileReader::open_sequential(&unitig_path)?; @@ -561,12 +557,12 @@ impl MphfLayer { for (kmer, chunk_id, rank) in unitigs2.iter_indexed_canonical_kmers() { let slot = mphf.index(&kmer.raw()); if slot >= n { - return Err(OLMError::Mphf("slot out of bounds".into())); + return Err(OKIError::InvalidData { context: "mphf", detail: "slot out of bounds".into() }); } let byte = slot / 8; let bit = 1u8 << (slot % 8); if seen[byte] & bit != 0 { - return Err(OLMError::Mphf("duplicate slot".into())); + return Err(OKIError::InvalidData { context: "mphf", detail: "duplicate slot".into() }); } seen[byte] |= bit; ev.set(slot, chunk_id as u32, rank as u8); @@ -581,12 +577,12 @@ impl MphfLayer { for (kmer, _, _) in unitigs2.iter_indexed_canonical_kmers() { let slot = mphf.index(&kmer.raw()); if slot >= n { - return Err(OLMError::Mphf("slot out of bounds".into())); + return Err(OKIError::InvalidData { context: "mphf", detail: "slot out of bounds".into() }); } let byte = slot / 8; let bit = 1u8 << (slot % 8); if seen[byte] & bit != 0 { - return Err(OLMError::Mphf("duplicate slot".into())); + return Err(OKIError::InvalidData { context: "mphf", detail: "duplicate slot".into() }); } seen[byte] |= bit; fw.set(slot, kmer.seq_hash()); @@ -601,12 +597,12 @@ impl MphfLayer { for (kmer, chunk_id, rank) in unitigs2.iter_indexed_canonical_kmers() { let slot = mphf.index(&kmer.raw()); if slot >= n { - return Err(OLMError::Mphf("slot out of bounds".into())); + return Err(OKIError::InvalidData { context: "mphf", detail: "slot out of bounds".into() }); } let byte = slot / 8; let bit = 1u8 << (slot % 8); if seen[byte] & bit != 0 { - return Err(OLMError::Mphf("duplicate slot".into())); + return Err(OKIError::InvalidData { context: "mphf", detail: "duplicate slot".into() }); } seen[byte] |= bit; ev.set(slot, chunk_id as u32, rank as u8); diff --git a/src/obikindex/src/layer/tests/map.rs b/src/obikindex/src/layer/tests/map.rs deleted file mode 100644 index 38bc6edd..00000000 --- a/src/obikindex/src/layer/tests/map.rs +++ /dev/null @@ -1,219 +0,0 @@ -use super::*; -use obicompactvec::{pack_bit_matrix, pack_sparse_bit_matrix, PersistentBitMatrix, PersistentCompactIntMatrix, StorageKind}; -use obikseq::{set_k, Sequence as _, Unitig}; -use obiskio::DEFAULT_BLOCK_BITS; -use crate::layer::typed_layer::LayerContent; -use crate::layer::meta::IndexMode; -use crate::layer::mphf_layer::EvidenceKind; -use tempfile::tempdir; - -fn push_unitigs_and_layer( - map: &mut LayeredMap, - seqs: &[&[u8]], - count: u32, -) { - let mut w = map.next_layer_writer().unwrap(); - for s in seqs { - w.write(&Unitig::from_ascii(s)).unwrap(); - } - w.close().unwrap(); - map.push_layer(|_| count).unwrap(); -} - -fn canonical(ascii: &[u8]) -> CanonicalKmer { - obikseq::Kmer::from_ascii(ascii).unwrap().canonical() -} - -#[test] -fn create_empty_map() { - set_k(4); - let dir = tempdir().unwrap(); - let map = LayeredMap::<()>::create(dir.path(), IndexMode::Exact).unwrap(); - assert_eq!(map.n_layers(), 0); -} - -#[test] -fn open_reloads_layer_count() { - set_k(4); - let dir = tempdir().unwrap(); - { - let mut map = LayeredMap::::create(dir.path(), IndexMode::Exact).unwrap(); - push_unitigs_and_layer(&mut map, &[b"AAAACGT"], 1); - } - let map = LayeredMap::::open(dir.path()).unwrap(); - assert_eq!(map.n_layers(), 1); -} - -#[test] -fn query_finds_kmer_in_layer_zero() { - set_k(4); - let dir = tempdir().unwrap(); - let mut map = LayeredMap::::create(dir.path(), IndexMode::Exact).unwrap(); - push_unitigs_and_layer(&mut map, &[b"AAAACGT"], 3); - let kmer = canonical(b"AAAC"); - let (layer_idx, hit) = map.query(kmer).expect("kmer must be found"); - assert_eq!(layer_idx, 0); - assert_eq!(hit.data[0], 3); -} - -#[test] -fn query_finds_kmer_in_correct_layer() { - set_k(4); - let dir = tempdir().unwrap(); - let mut map = LayeredMap::::create(dir.path(), IndexMode::Exact).unwrap(); - push_unitigs_and_layer(&mut map, &[b"AAAACGT"], 1); - push_unitigs_and_layer(&mut map, &[b"GGGACGT"], 2); - assert_eq!(map.n_layers(), 2); - - let (li, hit) = map.query(canonical(b"AAAA")).expect("AAAA must be found"); - assert_eq!(li, 0); - assert_eq!(hit.data[0], 1); - - let (li, hit) = map.query(canonical(b"GGGA")).expect("GGGA must be found"); - assert_eq!(li, 1); - assert_eq!(hit.data[0], 2); -} - -#[test] -fn query_absent_returns_none() { - set_k(4); - let dir = tempdir().unwrap(); - let mut map = LayeredMap::::create(dir.path(), IndexMode::Exact).unwrap(); - push_unitigs_and_layer(&mut map, &[b"AAAACGT"], 1); - let absent = canonical(b"CCCC"); - assert!(map.query(absent).is_none()); -} - -#[test] -fn push_layer_from_map_convenience() { - set_k(4); - let dir = tempdir().unwrap(); - let mut map = LayeredMap::::create(dir.path(), IndexMode::Exact).unwrap(); - let mut w = map.next_layer_writer().unwrap(); - w.write(&Unitig::from_ascii(b"AAAACGT")).unwrap(); - w.close().unwrap(); - let counts: HashMap = vec![ - (canonical(b"AAAA"), 10u32), - ].into_iter().collect(); - map.push_layer_from_map(&counts).unwrap(); - let (_, hit) = map.query(canonical(b"AAAA")).unwrap(); - assert_eq!(hit.data[0], 10); -} - -// ── detect_layer_content / detect_layer_storage / detect_layer_evidence ──── -// -// All built directly on disk (bypassing `LayeredMap::push_layer`, which -// only exists for modes 1 and 2 — mode 3/presence has no `push_layer`), -// then reopened as `LayeredMap<()>` — `()`'s `LayerData::open` never reads -// the matrix, so it's the right handle for exercising the disk-probing -// `detect_layer_*` methods regardless of what content/storage the layer -// underneath actually is. - -fn write_unitigs_at(dir: &Path, seqs: &[&[u8]]) { - fs::create_dir_all(dir).unwrap(); - let mut w = UnitigFileWriter::create(&dir.join(crate::layer::typed_layer::UNITIGS_FILE)).unwrap(); - for s in seqs { - w.write(&Unitig::from_ascii(s)).unwrap(); - } - w.close().unwrap(); -} - -/// Build a one-layer partition root with a presence layer at `layer_0`, -/// save `meta.json`, and reopen it as `LayeredMap<()>`. -fn presence_partition(seqs: &[&[u8]], n_genomes: usize) -> (tempfile::TempDir, LayeredMap<()>) { - let dir = tempdir().unwrap(); - let mode = IndexMode::Exact; - let layer0 = layer_dir(dir.path(), 0); - write_unitigs_at(&layer0, seqs); - TypedLayer::::build_presence(&layer0, DEFAULT_BLOCK_BITS, &mode, n_genomes, |kmer, g| { - (kmer.raw().wrapping_add(g as u64)) % 2 == 0 - }).unwrap(); - PartitionMeta { n_layers: 1, mode }.save(dir.path()).unwrap(); - let map = LayeredMap::<()>::open(dir.path()).unwrap(); - (dir, map) -} - -#[test] -fn detect_layer_content_count() { - set_k(4); - let dir = tempdir().unwrap(); - let mut map = LayeredMap::::create(dir.path(), IndexMode::Exact).unwrap(); - push_unitigs_and_layer(&mut map, &[b"AAAACGT"], 3); - assert_eq!(map.detect_layer_content(0), LayerContent::Count); -} - -#[test] -fn detect_layer_content_presence() { - set_k(4); - let (_dir, map) = presence_partition(&[b"AAATCTA", b"CTTCGCC", b"TGATACG"], 3); - assert_eq!(map.detect_layer_content(0), LayerContent::Presence); -} - -#[test] -fn detect_layer_storage_columnar_then_packed_then_sparse() { - set_k(4); - let (dir, map) = presence_partition(&[b"AAATCTA", b"CTTCGCC", b"TGATACG"], 3); - let presence_dir = layer_dir(dir.path(), 0).join("presence"); - - assert_eq!(map.detect_layer_storage(0).unwrap(), StorageKind::Columnar); - - pack_bit_matrix(&presence_dir).unwrap(); - assert_eq!(map.detect_layer_storage(0).unwrap(), StorageKind::Packed); - - pack_sparse_bit_matrix(&presence_dir).unwrap(); - assert_eq!(map.detect_layer_storage(0).unwrap(), StorageKind::Sparse); -} - -#[test] -fn detect_layer_storage_implicit() { - set_k(4); - let dir = tempdir().unwrap(); - let mode = IndexMode::Exact; - let layer0 = layer_dir(dir.path(), 0); - write_unitigs_at(&layer0, &[b"AAAACGT"]); - // Mode-1 build: MPHF + evidence + `layer_meta.json`, no matrix at all — - // must read back as `Presence`/`Implicit`, never a third "empty" state. - TypedLayer::<()>::build(&layer0, DEFAULT_BLOCK_BITS, &mode).unwrap(); - PartitionMeta { n_layers: 1, mode }.save(dir.path()).unwrap(); - let map = LayeredMap::<()>::open(dir.path()).unwrap(); - - assert_eq!(map.detect_layer_content(0), LayerContent::Presence); - assert_eq!(map.detect_layer_storage(0).unwrap(), StorageKind::Implicit); -} - -#[test] -fn detect_layer_storage_count_columnar_then_packed() { - set_k(4); - let dir = tempdir().unwrap(); - let mut map = LayeredMap::::create(dir.path(), IndexMode::Exact).unwrap(); - push_unitigs_and_layer(&mut map, &[b"AAAACGT"], 3); - drop(map); - - let map = LayeredMap::<()>::open(dir.path()).unwrap(); - assert_eq!(map.detect_layer_storage(0).unwrap(), StorageKind::Columnar); - - obicompactvec::pack_compact_int_matrix(&layer_dir(dir.path(), 0).join("counts")).unwrap(); - assert_eq!(map.detect_layer_storage(0).unwrap(), StorageKind::Packed); -} - -#[test] -fn detect_layer_evidence_exact_vs_approx() { - set_k(4); - - let dir = tempdir().unwrap(); - let layer0 = layer_dir(dir.path(), 0); - write_unitigs_at(&layer0, &[b"AAAACGT"]); - TypedLayer::<()>::build(&layer0, DEFAULT_BLOCK_BITS, &IndexMode::Exact).unwrap(); - PartitionMeta { n_layers: 1, mode: IndexMode::Exact }.save(dir.path()).unwrap(); - let map = LayeredMap::<()>::open(dir.path()).unwrap(); - assert_eq!(map.detect_layer_evidence(0).unwrap(), EvidenceKind::Exact); - - let approx = IndexMode::Approx { b: 8, z: 1 }; - let dir2 = tempdir().unwrap(); - let layer0b = layer_dir(dir2.path(), 0); - write_unitigs_at(&layer0b, &[b"AAAACGT"]); - TypedLayer::<()>::build(&layer0b, DEFAULT_BLOCK_BITS, &approx).unwrap(); - PartitionMeta { n_layers: 1, mode: approx }.save(dir2.path()).unwrap(); - let map2 = LayeredMap::<()>::open(dir2.path()).unwrap(); - assert_eq!(map2.detect_layer_evidence(0).unwrap(), EvidenceKind::Approx); -} diff --git a/src/obikindex/src/layer/typed_layer.rs b/src/obikindex/src/layer/typed_layer.rs index 11809725..ec1d27d0 100644 --- a/src/obikindex/src/layer/typed_layer.rs +++ b/src/obikindex/src/layer/typed_layer.rs @@ -1,19 +1,18 @@ -use std::collections::HashMap; -use std::fs; -use std::path::{Path, PathBuf}; - +use crate::layer::utils::layer_dir; use obicompactvec::{ BinaryMatrix, PersistentBitMatrix, PersistentBitMatrixBuilder, PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder, PersistentSparseBitMatrix, }; use obikseq::CanonicalKmer; use obiskio::{UnitigFileReader, UnitigFileWriter}; +use std::collections::HashMap; +use std::fs; +use std::path::Path; -use crate::layer::error::{OLMError, OLMResult}; +use crate::index::error::{OKIError, OKIResult}; use crate::layer::meta::IndexMode; use crate::layer::mphf_layer::MphfLayer; pub(crate) use crate::layer::mphf_layer::UNITIGS_FILE; - pub(crate) const COUNTS_DIR: &str = "counts"; pub(crate) const PRESENCE_DIR: &str = "presence"; @@ -21,48 +20,10 @@ pub(crate) const PRESENCE_DIR: &str = "presence"; pub trait LayerData: Sized { type Item; - fn open(layer_dir: &Path) -> OLMResult; + fn open(layer_dir: &Path) -> OKIResult; fn read(&self, slot: usize) -> Self::Item; } -/// TypedLayer directory path for layer `i` within a partition's index root — the -/// single source of truth for the on-disk `layer_N` naming convention -/// (mirrors what `LayeredMap::open`/`push_layer` already use internally). -/// -/// `obikindex::layer` operates within a single partition's index root; it has -/// no notion of "partition" at all. Turning a partition number into that -/// root is `obikindex::KmerIndex::index_dir`'s job, one layer up — callers -/// here only ever name a layer *number*, never build the path themselves. -pub fn layer_dir(root: &Path, i: usize) -> PathBuf { - root.join(format!("layer_{i}")) -} - -/// Superkmer-file extension used by every pre-layer-construction artifact -/// below (`raw`/`dereplicated`) — an implementation detail of the SK file -/// format, never meant to leak as a literal string past this module. -const SK_EXT: &str = "skmer.zst"; - -/// Path of a layer's raw, not-yet-dereplicated superkmer file — written by -/// whichever algorithm routes superkmers into this layer (today: -/// `obikindexer::algorithms::partitionner::PartitionRouter`), read by whichever -/// algorithm dereplicates it (today: -/// `obikindexer::algorithms::dereplicator::Dereplicator`). Naming this once -/// here, rather than in either algorithm submodule, is what lets the two -/// agree on the filename without depending on each other directly — see -/// `DevDocMD/implementation/partition_layer_cache.md`. -pub fn raw_superkmers_path(layer_dir: &Path) -> PathBuf { - layer_dir.join(format!("raw.{SK_EXT}")) -} - -/// Path of a layer's dereplicated superkmer file — written by -/// `obikindexer::algorithms::dereplicator::Dereplicator`, read by whichever -/// algorithm counts kmer abundances from it (today: -/// `obikindexer::algorithms::partitionner::PartitionRouter::count_kmer`) and, -/// later, by `obikindex::build_index_layer` to build the real layer. -pub fn dereplicated_superkmers_path(layer_dir: &Path) -> PathBuf { - layer_dir.join(format!("dereplicated.{SK_EXT}")) -} - /// Opens layer `i`'s data only, skipping the MPHF — for callers that only /// need matrix-level operations (distance traits, column weights, group /// filters, sub-matrix extraction) and never look up a kmer for this layer. @@ -74,13 +35,13 @@ pub fn dereplicated_superkmers_path(layer_dir: &Path) -> PathBuf { /// *number* within a partition it already knows the root of, the same /// vocabulary as [`layer_dir`] and `LayeredMap` — never the `layer_N` /// naming convention itself, which stays private to this crate. -pub fn open_data(root: &Path, i: usize) -> OLMResult { +pub(crate) fn open_data(root: &Path, i: usize) -> OKIResult { D::open(&layer_dir(root, i)) } impl LayerData for () { type Item = (); - fn open(_layer_dir: &Path) -> OLMResult { + fn open(_layer_dir: &Path) -> OKIResult { Ok(()) } fn read(&self, _slot: usize) {} @@ -88,8 +49,8 @@ impl LayerData for () { impl LayerData for PersistentCompactIntMatrix { type Item = Box<[u32]>; - fn open(layer_dir: &Path) -> OLMResult { - PersistentCompactIntMatrix::open(layer_dir).map_err(OLMError::Io) + fn open(layer_dir: &Path) -> OKIResult { + PersistentCompactIntMatrix::open(layer_dir).map_err(OKIError::Io) } fn read(&self, slot: usize) -> Box<[u32]> { self.row(slot) @@ -98,8 +59,8 @@ impl LayerData for PersistentCompactIntMatrix { impl LayerData for PersistentBitMatrix { type Item = Box<[bool]>; - fn open(layer_dir: &Path) -> OLMResult { - PersistentBitMatrix::open(layer_dir).map_err(OLMError::Io) + fn open(layer_dir: &Path) -> OKIResult { + PersistentBitMatrix::open(layer_dir).map_err(OKIError::Io) } fn read(&self, slot: usize) -> Box<[bool]> { self.row(slot) @@ -108,8 +69,8 @@ impl LayerData for PersistentBitMatrix { impl LayerData for PersistentSparseBitMatrix { type Item = Box<[bool]>; - fn open(layer_dir: &Path) -> OLMResult { - PersistentSparseBitMatrix::open(&layer_dir.join(PRESENCE_DIR)).map_err(OLMError::Io) + fn open(layer_dir: &Path) -> OKIResult { + PersistentSparseBitMatrix::open(&layer_dir.join(PRESENCE_DIR)).map_err(OKIError::Io) } fn read(&self, slot: usize) -> Box<[bool]> { self.row(slot) @@ -201,11 +162,7 @@ pub struct Hit { // ── Common read path ────────────────────────────────────────────────────────── impl TypedLayer { - // `mode` no longer forwarded to `MphfLayer::open` (auto-detected from - // disk now, see mphf_layer.rs) — kept as a parameter for the moment so - // this signature (and `KmerLayer::open`'s call to it) doesn't have to - // change in the same step; dropped when content_layer.rs is done. - pub fn open(path: &Path, _mode: &IndexMode) -> OLMResult { + pub fn open(path: &Path) -> OKIResult { let mphf = MphfLayer::open(path)?; let data = D::open(path)?; Ok(Self { mphf, data }) @@ -266,21 +223,21 @@ impl TypedLayer { self.mphf.enumerate_kmers_batch(n) } - pub fn unitig_writer(out_dir: &Path) -> OLMResult { + pub fn unitig_writer(out_dir: &Path) -> OKIResult { MphfLayer::unitig_writer(out_dir) } /// Build `unitigs.bin.idx` and `evidence.bin` from `unitigs.bin` and /// `mphf.bin` already present in `layer_dir`. /// `block_bits` controls the `.idx` block size (2^block_bits chunks/block). - pub fn build_exact_evidence(layer_dir: &Path, block_bits: u8) -> OLMResult { + pub fn build_exact_evidence(layer_dir: &Path, block_bits: u8) -> OKIResult { MphfLayer::build_exact_evidence(layer_dir, block_bits) } /// Build `fingerprint.bin` from `unitigs.bin` and `mphf.bin` already /// present in `layer_dir`. `b` — fingerprint bits (1..=64); `z` — Findere /// consecutive k-mer parameter (≥1). - pub fn build_approx_evidence(layer_dir: &Path, b: u8, z: u8) -> OLMResult { + pub fn build_approx_evidence(layer_dir: &Path, b: u8, z: u8) -> OKIResult { MphfLayer::build_approx_evidence(layer_dir, b, z) } @@ -311,21 +268,21 @@ impl TypedLayer { // ── Mode 1 — set membership ─────────────────────────────────────────────────── impl TypedLayer<()> { - pub fn build(out_dir: &Path, block_bits: u8, mode: &IndexMode) -> OLMResult { + pub fn build(out_dir: &Path, block_bits: u8, mode: &IndexMode) -> OKIResult { MphfLayer::build(out_dir, block_bits, mode, &mut |_, _| Ok(())) } /// Create a presence matrix for a set-membership layer (first merge). - pub fn init_presence_matrix(layer_dir: &Path, n_kmers: usize) -> OLMResult<()> { + pub fn init_presence_matrix(layer_dir: &Path, n_kmers: usize) -> OKIResult<()> { let presence_dir = layer_dir.join(PRESENCE_DIR); - fs::create_dir_all(&presence_dir).map_err(OLMError::Io)?; + fs::create_dir_all(&presence_dir).map_err(OKIError::Io)?; let mut mb = - PersistentBitMatrixBuilder::new(n_kmers, &presence_dir).map_err(OLMError::Io)?; + PersistentBitMatrixBuilder::new(n_kmers, &presence_dir).map_err(OKIError::Io)?; mb.add_col_ones() - .map_err(OLMError::Io)? + .map_err(OKIError::Io)? .close() - .map_err(OLMError::Io)?; - mb.close().map_err(OLMError::Io) + .map_err(OKIError::Io)?; + mb.close().map_err(OKIError::Io) } } @@ -337,18 +294,18 @@ impl TypedLayer { block_bits: u8, mode: &IndexMode, count_of: impl Fn(CanonicalKmer) -> u32, - ) -> OLMResult { + ) -> OKIResult { let n = UnitigFileReader::open_sequential(&out_dir.join(UNITIGS_FILE))?.n_kmers(); let counts_dir = out_dir.join(COUNTS_DIR); let mut mb = - PersistentCompactIntMatrixBuilder::new(n, &counts_dir).map_err(OLMError::Io)?; - let mut col = mb.add_col().map_err(OLMError::Io)?; + PersistentCompactIntMatrixBuilder::new(n, &counts_dir).map_err(OKIError::Io)?; + let mut col = mb.add_col().map_err(OKIError::Io)?; let n_built = MphfLayer::build(out_dir, block_bits, mode, &mut |slot, kmer| { col.set(slot, count_of(kmer)); Ok(()) })?; - col.close().map_err(OLMError::Io)?; - mb.close().map_err(OLMError::Io)?; + col.close().map_err(OKIError::Io)?; + mb.close().map_err(OKIError::Io)?; Ok(n_built) } @@ -357,7 +314,7 @@ impl TypedLayer { block_bits: u8, mode: &IndexMode, counts: &HashMap, - ) -> OLMResult { + ) -> OKIResult { Self::build(out_dir, block_bits, mode, |kmer| { counts.get(&kmer).copied().unwrap_or(0) }) @@ -370,9 +327,9 @@ impl TypedLayer { pub fn append_genome_column( layer_dir: &Path, value_of: impl Fn(usize) -> u32, - ) -> OLMResult<()> { + ) -> OKIResult<()> { PersistentCompactIntMatrix::append_column(&layer_dir.join(COUNTS_DIR), value_of) - .map_err(OLMError::Io) + .map_err(OKIError::Io) } /// Number of genome columns in this layer's count matrix. @@ -448,9 +405,9 @@ impl TypedLayer { pub fn append_genome_column( layer_dir: &Path, value_of: impl Fn(usize) -> bool, - ) -> OLMResult<()> { + ) -> OKIResult<()> { PersistentBitMatrix::append_column(&layer_dir.join(PRESENCE_DIR), value_of) - .map_err(OLMError::Io) + .map_err(OKIError::Io) } pub fn build_presence( @@ -459,13 +416,13 @@ impl TypedLayer { mode: &IndexMode, n_genomes: usize, present_in: impl Fn(CanonicalKmer, usize) -> bool, - ) -> OLMResult { + ) -> OKIResult { let n = UnitigFileReader::open_sequential(&out_dir.join(UNITIGS_FILE))?.n_kmers(); let presence_dir = out_dir.join(PRESENCE_DIR); - let mut mb = PersistentBitMatrixBuilder::new(n, &presence_dir).map_err(OLMError::Io)?; + let mut mb = PersistentBitMatrixBuilder::new(n, &presence_dir).map_err(OKIError::Io)?; let mut cols: Vec<_> = (0..n_genomes) - .map(|_| mb.add_col().map_err(OLMError::Io)) - .collect::>()?; + .map(|_| mb.add_col().map_err(OKIError::Io)) + .collect::>()?; let n_built = MphfLayer::build(out_dir, block_bits, mode, &mut |slot, kmer| { for (g, col) in cols.iter_mut().enumerate() { col.set(slot, present_in(kmer, g)); @@ -473,9 +430,9 @@ impl TypedLayer { Ok(()) })?; for col in cols { - col.close().map_err(OLMError::Io)?; + col.close().map_err(OKIError::Io)?; } - mb.close().map_err(OLMError::Io)?; + mb.close().map_err(OKIError::Io)?; Ok(n_built) } } diff --git a/src/obikindex/src/layer/utils.rs b/src/obikindex/src/layer/utils.rs new file mode 100644 index 00000000..2a17313b --- /dev/null +++ b/src/obikindex/src/layer/utils.rs @@ -0,0 +1,14 @@ +pub(crate) const LAYERNAME_SUFFIX: &str = "layer_"; +use std::path::{Path, PathBuf}; + +/// TypedLayer directory path for layer `i` within a partition's index root — the +/// single source of truth for the on-disk `layer_N` naming convention +/// (mirrors what `LayeredMap::open`/`push_layer` already use internally). +/// +/// `obikindex::layer` operates within a single partition's index root; it has +/// no notion of "partition" at all. Turning a partition number into that +/// root is `obikindex::KmerIndex::index_dir`'s job, one layer up — callers +/// here only ever name a layer *number*, never build the path themselves. +pub(crate) fn layer_dir(root: &Path, i: usize) -> PathBuf { + root.join(format!("{LAYERNAME_SUFFIX}{i}")) +} diff --git a/src/obikindex/src/lib.rs b/src/obikindex/src/lib.rs index 093ed46b..2ab9c8fe 100644 --- a/src/obikindex/src/lib.rs +++ b/src/obikindex/src/lib.rs @@ -13,10 +13,10 @@ pub mod layer; pub mod partition; pub use index::{ - validate_label, AggOp, DistanceMetric, DistanceOutput, GenomeInfo, GroupFilterParams, - GroupQuorumFilter, IndexBitsPerKmer, IndexBuilder, IndexConfig, IndexMeta, IndexState, - KmerDesc, KmerFilter, KmerIndex, MergeMode, MetaPred, OKIError, OKIResult, OutputCol, - PartitionRunner, QueryHit, QueryStats, META_FILENAME, - materialize_layer, olm_to_sk, write_graph_as_unitigs, passes_all, + GenomeInfo, + IndexBuilder, IndexConfig, IndexMeta, IndexState, + KmerIndex, OKIError, OKIResult, + META_FILENAME, + ColBuilder, }; -pub use index::{filter, meta}; +pub use index::meta; diff --git a/src/obikindex/src/partition/mod.rs b/src/obikindex/src/partition/mod.rs index 34da67f0..87b31152 100644 --- a/src/obikindex/src/partition/mod.rs +++ b/src/obikindex/src/partition/mod.rs @@ -19,10 +19,13 @@ //! entirely) both still reinvent a fragment of this. Migrating them is a //! separate, deferred step. +use crate::OKIError; use std::path::{Path, PathBuf}; -use crate::layer::{IndexMode, KmerLayer, OLMResult, layer_dir}; -use obikseq::CanonicalKmer; +use crate::{ + KmerIndex, OKIResult, + layer::{KmerLayer, utils::layer_dir}, +}; /// Partition subdirectory name, under an index's root — the single source /// of truth for the on-disk `partitions/part_NNNNN` naming convention. @@ -34,59 +37,77 @@ pub const PARTITIONS_SUBDIR: &str = "partitions"; /// Path of partition `i`'s directory under `root` — `/partitions/part_NNNNN`, /// zero-padded to 5 digits. -pub fn partition_dir(root: &Path, i: usize) -> PathBuf { +pub(crate) fn partition_dir(root: &Path, i: usize) -> PathBuf { root.join(PARTITIONS_SUBDIR).join(format!("part_{i:05}")) } /// Path of partition `i`'s layered-index directory — `/index`, /// the root every [`crate::layer::layer_dir`] call for this partition is /// relative to. -pub fn index_dir(root: &Path, i: usize) -> PathBuf { +pub(crate) fn index_dir(root: &Path, i: usize) -> PathBuf { partition_dir(root, i).join("index") } -/// One partition's open layers, in layer order (layer 0 first). +/// One partition — identified by its own `index_dir`, nothing eagerly +/// opened. `index_dir` is the minimal field `KmerLayer::at` needs to +/// compute a layer's path; added here only because `layer()` below can't +/// be written without it now that there's no `layers: Vec` to +/// index into. pub struct KmerPartition { - layers: Vec, + partition_dir: PathBuf, + id: usize, } impl KmerPartition { - /// Open every layer under `index_dir` (`index_dir/layer_0`, - /// `index_dir/layer_1`, ... up to `n_layers`), eagerly — not lazily on - /// first access, so the caller pays the mmap cost once, up front, - /// rather than at an unpredictable point during later lookups. - pub fn open( - index_dir: &Path, - mode: &IndexMode, - n_layers: usize, - with_counts: bool, - ) -> OLMResult { - let layers = (0..n_layers) - .map(|l| KmerLayer::open(&layer_dir(index_dir, l), mode, with_counts)) - .collect::>>()?; - Ok(Self { layers }) + pub fn new(index: &KmerIndex, i: usize) -> Self { + KmerPartition { + partition_dir: index.dir().join(format!("part_{i:05}")), + id: i, + } } + pub fn id(&self) -> usize { + self.id + } + + /// Number of layers under `index_dir`, found by probing `layer_0`, + /// `layer_1`, ... sequentially until one is missing — a disk scan, not + /// a metadata read (`meta.json`'s own `n_layers` is redundant with this + /// and no longer consulted here). Intrinsically slow: one `exists()` + /// syscall per layer, called fresh every time — not for a hot path. pub fn n_layers(&self) -> usize { - self.layers.len() + let mut n = 0; + while layer_dir(&self.partition_dir, n).exists() { + n += 1; + } + n } - pub fn layer(&self, i: usize) -> &KmerLayer { - &self.layers[i] + /// Evidence mode for this partition, detected from layer 0's evidence + /// files on disk (see [`crate::layer::IndexMode::detect`]) — never + /// persisted as metadata, since every layer within a partition shares + /// the same mode by construction. + pub fn mode(&self) -> OKIResult { + crate::layer::IndexMode::detect(&self.partition_dir) } - pub fn layers(&self) -> &[KmerLayer] { - &self.layers + pub fn layer(&self, i: usize) -> OKIResult { + let n = self.n_layers(); + if i >= n { + return Err(OKIError::Layer(obiskio::SKError::InvalidData { + context: "New KmerLayer", + detail: format!("Layer {i} requested, Partition has only {n} layer"), + })); + } + Ok(KmerLayer::new(self, i)) } - /// Existence lookup of `kmer` across this partition's layers: tries - /// each in turn, stopping at the first hit and reporting which layer it - /// was — `find_slot`, not a data read, for a plain existence check. - pub fn find(&self, kmer: CanonicalKmer) -> Option { - self.layers - .iter() - .enumerate() - .find_map(|(li, layer)| layer.find_slot(kmer).map(|_| li)) + pub fn layer_dir(&self, i: usize) -> OKIResult { + Ok(self.layer(i)?.dir().to_path_buf()) + } + + pub fn dir(&self) -> &Path { + &self.partition_dir } } diff --git a/src/obikindexer/src/algorithms/layer_builder/mod.rs b/src/obikindexer/src/algorithms/layer_builder/mod.rs index eb4225ef..1522e975 100644 --- a/src/obikindexer/src/algorithms/layer_builder/mod.rs +++ b/src/obikindexer/src/algorithms/layer_builder/mod.rs @@ -18,7 +18,7 @@ use crate::extensions::PrivateBuilder; /// The actual per-partition construction (De Bruijn graph, unitigs, MPHF, /// matrix) stays a `KmerIndex` method (`build_index_layer`) — unlike /// `Dereplicator`/`Counter`, it depends on several `obikindex`-internal -/// helpers (`graph_pipeline`, `common::olm_to_sk`) already shared with +/// helpers (`graph_pipeline`) already shared with /// `merge`/`select`/`rebuild`'s own layer-construction paths, so it /// belongs there, not duplicated or newly exported just for this /// algorithm. `LayerBuilder`'s own job is the orchestration around it: diff --git a/src/obikindexer/src/extensions/mod.rs b/src/obikindexer/src/extensions/mod.rs index 3ac6b003..4fce656e 100644 --- a/src/obikindexer/src/extensions/mod.rs +++ b/src/obikindexer/src/extensions/mod.rs @@ -36,7 +36,7 @@ use obidebruinj::GraphDeBruijn; use obikindex::layer::IndexMode; use obikindex::layer::{TypedLayer, meta::PartitionMeta}; use obikindex::{KmerIndex, OKIError, OKIResult}; -use obikindex::{materialize_layer, olm_to_sk, write_graph_as_unitigs}; +use crate::graph_pipeline::{materialize_layer, write_graph_as_unitigs}; use obiskio::{SKError, SKFileMeta, SKFileReader}; use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xx64}; @@ -109,7 +109,7 @@ impl PrivateBuilder for KmerIndex { .map(|g| g.label.as_str()) .unwrap_or("unknown") .to_owned(); - let spectrums_dir = self.root_path().join("spectrums"); + let spectrums_dir = self.dir().join("spectrums"); fs::create_dir_all(&spectrums_dir)?; let path = spectrums_dir.join(format!("{label}.json")); let spectrum_map: BTreeMap = counts @@ -194,8 +194,7 @@ impl PrivateBuilder for KmerIndex { (Some(mphf), Some(counts)) => counts.get(mphf.index(&kmer.raw())), _ => 1, }, - ) - .map_err(|e| olm_to_sk(e, "layer build"))?; + )?; n } else { materialize_layer(g, &layer0_dir, block_bits, mode)? @@ -206,8 +205,7 @@ impl PrivateBuilder for KmerIndex { n_layers: 1, mode: mode.clone(), } - .save(index_dir) - .map_err(|e| olm_to_sk(e, "layer build"))?; + .save(index_dir)?; Ok(n_kmers) } diff --git a/src/obikindex/src/index/graph_pipeline.rs b/src/obikindexer/src/graph_pipeline.rs similarity index 89% rename from src/obikindex/src/index/graph_pipeline.rs rename to src/obikindexer/src/graph_pipeline.rs index 307436d8..5fd9ae9b 100644 --- a/src/obikindex/src/index/graph_pipeline.rs +++ b/src/obikindexer/src/graph_pipeline.rs @@ -10,10 +10,9 @@ use obipipeline::{ use obidebruinj::GraphDeBruijn; use obikseq::CanonicalKmer; -use crate::layer::{IndexMode, TypedLayer}; -use obiskio::{SKError, SKResult}; +use obikindex::layer::{IndexMode, TypedLayer}; +use obikindex::{OKIError, OKIResult}; -use crate::index::common::olm_to_sk; // ── KmerGraphData ───────────────────────────────────────────────────────────── @@ -29,16 +28,16 @@ enum KmerGraphData { /// /// `flat_fn(path, emit)`: opens path, iterates kmers, calls `emit(batch)` for each batch. /// `filter(kmer) -> bool`: secondary filter applied in the Transform stage. -pub(crate) fn build_graph( +pub fn build_graph( file_source: I, flat_fn: F, filter: G, n_workers: usize, max_open: usize, -) -> SKResult +) -> OKIResult where I: Iterator + Send + 'static, - F: Fn(PathBuf, &mut dyn FnMut(Vec)) -> SKResult<()> + Send + Sync + 'static, + F: Fn(PathBuf, &mut dyn FnMut(Vec)) -> OKIResult<()> + Send + Sync + 'static, G: Fn(CanonicalKmer) -> bool + Send + Sync + 'static, { let capacity = 2; @@ -47,7 +46,7 @@ where let filter = Arc::new(filter); let g_shared = Arc::new(Mutex::new(GraphDeBruijn::new())); let g_sink = Arc::clone(&g_shared); - let err_cap: Arc>> = Arc::new(Mutex::new(None)); + let err_cap: Arc>> = Arc::new(Mutex::new(None)); let err_flat = Arc::clone(&err_cap); let throttled = throttle(file_source, max_open); @@ -121,11 +120,11 @@ where /// Phase 2 (write unitigs only): compute degrees, write unitigs to `layer_dir`, drop graph. /// /// Returns n_kmers. Does NOT build the MPHF — caller does it. -pub fn write_graph_as_unitigs(g: GraphDeBruijn, layer_dir: &Path) -> SKResult { +pub fn write_graph_as_unitigs(g: GraphDeBruijn, layer_dir: &Path) -> OKIResult { let n_kmers = g.len(); g.compute_degrees_and_mark_starts(); std::fs::create_dir_all(layer_dir)?; - let mut uw = TypedLayer::<()>::unitig_writer(layer_dir).map_err(|e| olm_to_sk(e, "graph pipeline"))?; + let mut uw = TypedLayer::<()>::unitig_writer(layer_dir)?; g.try_for_each_unitig(|unitig| uw.write(unitig))?; uw.close()?; drop(g); @@ -142,11 +141,10 @@ pub fn materialize_layer( layer_dir: &Path, block_bits: u8, evidence: &IndexMode, -) -> SKResult { +) -> OKIResult { let n = write_graph_as_unitigs(g, layer_dir)?; debug!("materialize_layer: unitigs written ({n} kmers), building MPHF"); - TypedLayer::<()>::build(layer_dir, block_bits, evidence) - .map_err(|e| olm_to_sk(e, "graph pipeline"))?; + TypedLayer::<()>::build(layer_dir, block_bits, evidence)?; debug!("materialize_layer: MPHF build done"); Ok(n) } diff --git a/src/obikindexer/src/lib.rs b/src/obikindexer/src/lib.rs index 0bda5d6d..2b6bc442 100644 --- a/src/obikindexer/src/lib.rs +++ b/src/obikindexer/src/lib.rs @@ -14,3 +14,6 @@ pub mod algorithms; pub(crate) mod extensions; +pub mod graph_pipeline; + +pub use graph_pipeline::{build_graph, materialize_layer, write_graph_as_unitigs}; diff --git a/src/obikmer/Cargo.toml b/src/obikmer/Cargo.toml index 3aaedb34..0ff9dcb1 100644 --- a/src/obikmer/Cargo.toml +++ b/src/obikmer/Cargo.toml @@ -19,6 +19,8 @@ obisys = { path = "../obisys" } obiskio = { path = "../obiskio" } obikindex = { path = "../obikindex", default-features = false } obikindexer = { path = "../obikindexer" } +obikfilter = { path = "../obikfilter" } +obikstats = { path = "../obikstats" } obikalgorithm = { path = "../obikalgorithm" } obikphylo = { path = "../obikphylo" } obitaxonomy = { path = "../obitaxonomy" } @@ -37,5 +39,5 @@ pprof = { version = "0.15", features = ["prost-codec"], optional = true } [features] default = ["numa"] -numa = ["obikindex/numa"] +numa = ["obisys/numa"] profiling = ["dep:pprof"] diff --git a/src/obikmer/src/cmd/index/mod.rs b/src/obikmer/src/cmd/index/mod.rs index 3159b250..642dc0b8 100644 --- a/src/obikmer/src/cmd/index/mod.rs +++ b/src/obikmer/src/cmd/index/mod.rs @@ -2,13 +2,13 @@ use std::path::PathBuf; use std::time::Instant; use clap::Args; +use obikalgorithm::Algorithm; +use obikindex::layer::IndexMode; +use obikindex::{GenomeInfo, IndexBuilder, IndexConfig, IndexState, KmerIndex}; use obikindexer::algorithms::counter::Counter; use obikindexer::algorithms::dereplicator::Dereplicator; use obikindexer::algorithms::layer_builder::LayerBuilder; use obikindexer::algorithms::partitionner::PartitionRouter; -use obikindex::{validate_label, GenomeInfo, IndexBuilder, IndexConfig, IndexState, KmerIndex}; -use obikindex::layer::IndexMode; -use obikalgorithm::Algorithm; fn current_state(idx: &KmerIndex) -> IndexState { idx.state().unwrap_or_else(|e| { @@ -18,10 +18,12 @@ fn current_state(idx: &KmerIndex) -> IndexState { } fn parse_key_value(s: &str) -> Result<(String, String), String> { - let pos = s.find('=').ok_or_else(|| format!("invalid key=value: no '=' in '{s}'"))?; + let pos = s + .find('=') + .ok_or_else(|| format!("invalid key=value: no '=' in '{s}'"))?; Ok((s[..pos].to_string(), s[pos + 1..].to_string())) } -use obisys::{progress_bar, spinner, Progress, Reporter, Stage}; +use obisys::{Progress, Reporter, Stage, progress_bar, spinner}; use tracing::info; use crate::cli::{CommonArgs, block_size_to_bits, partitions_to_bits}; @@ -108,9 +110,7 @@ pub(crate) fn resolve_approx_params( const DEFAULT_B: u8 = 8; const DEFAULT_Z: u8 = 1; - let bits_needed = |fp: f64| -> u8 { - (-fp.log2()).ceil() as u8 - }; + let bits_needed = |fp: f64| -> u8 { (-fp.log2()).ceil() as u8 }; match (z_opt, b_opt, fp_opt) { // All three given: use b and z, recompute fp conservatively. @@ -198,7 +198,7 @@ pub fn run(args: IndexArgs) { }; // ── Open or create the index ───────────────────────────────────────────── - if KmerIndex::exists(&output) { + if KmerIndex::is_an_index(&output) { if !args.force { eprintln!( "error: an index already exists at {} (use --force to overwrite it)", @@ -211,23 +211,32 @@ pub fn run(args: IndexArgs) { eprintln!("error removing existing index: {e}"); std::process::exit(1); }); + } else if output.exists() { + eprintln!( + "error: {} exists but is not an obikmer index, it cannot be deleted", + output.display() + ); + std::process::exit(1); } let n_bits = partitions_to_bits(args.common.partitions); let effective = 1usize << n_bits; if effective != args.common.partitions { - info!("partitions: {} → {} (next power of 2)", args.common.partitions, effective); + info!( + "partitions: {} → {} (next power of 2)", + args.common.partitions, effective + ); } let block_bits = block_size_to_bits(args.block_size); let config = IndexConfig { - kmer_size: effective_kmer_size, + kmer_size: effective_kmer_size, minimizer_size: args.common.minimizer_size, n_bits, - with_counts: args.with_counts, - evidence: evidence.clone(), + with_counts: args.with_counts, + evidence: evidence.clone(), block_bits, }; let genome_info = args.label.as_ref().map(|label| { - validate_label(label).unwrap_or_else(|e| { + GenomeInfo::validate_label(label).unwrap_or_else(|e| { eprintln!("error: --label: {e}"); std::process::exit(1); }); @@ -242,7 +251,6 @@ pub fn run(args: IndexArgs) { std::process::exit(1); }); - // ── Stage 1: scatter ───────────────────────────────────────────────────── if current_state(&idx) < IndexState::Scattered { let n_workers = args.common.threads.max(1); @@ -272,19 +280,23 @@ pub fn run(args: IndexArgs) { last_bases = p.position; let bp = p.position as f64; let (count_str, rate_str) = if bp >= 1e9 { - (format!("{:.2} Gbp", bp / 1e9), format!("{:.0} Mbp/s", ema_rate / 1e6)) + ( + format!("{:.2} Gbp", bp / 1e9), + format!("{:.0} Mbp/s", ema_rate / 1e6), + ) } else { - (format!("{:.0} Mbp", bp / 1e6), format!("{:.0} Mbp/s", ema_rate / 1e6)) + ( + format!("{:.0} Mbp", bp / 1e6), + format!("{:.0} Mbp/s", ema_rate / 1e6), + ) }; pb.set_message(format!("{count_str} {rate_str}")); }); - router - .run() - .unwrap_or_else(|e| { - eprintln!("error: {e}"); - std::process::exit(1); - }); + router.run().unwrap_or_else(|e| { + eprintln!("error: {e}"); + std::process::exit(1); + }); pb.finish_and_clear(); rep.push(t.stop()); drop(router); // ends the borrow of `idx` early — `PartitionRouter`'s `Drop` impl would otherwise extend it to the end of scope (`run()` already called `close()`, which marks scatter done, internally) diff --git a/src/obikmer/src/cmd/predicate.rs b/src/obikmer/src/cmd/predicate.rs index 87e0e653..0633cacc 100644 --- a/src/obikmer/src/cmd/predicate.rs +++ b/src/obikmer/src/cmd/predicate.rs @@ -1,6 +1,6 @@ use clap::Args; -use obikindex::{GroupFilterParams, IndexMeta, MetaPred}; -use obikindex::KmerFilter; +use obikindex::IndexMeta; +use obikfilter::{GroupFilterParams, KmerFilter, MetaPred}; /// CLI args for ingroup/outgroup filtering — embeddable in any command via `#[command(flatten)]`. #[derive(Args)] diff --git a/src/obikmer/src/cmd/utils/maintenance.rs b/src/obikmer/src/cmd/utils/maintenance.rs index fd805759..fa464b2b 100644 --- a/src/obikmer/src/cmd/utils/maintenance.rs +++ b/src/obikmer/src/cmd/utils/maintenance.rs @@ -1,6 +1,7 @@ use std::path::PathBuf; -use obikindex::{validate_label, IndexBitsPerKmer, KmerIndex}; +use obikindex::{GenomeInfo, KmerIndex}; +use obikstats::IndexBitsPerKmer; use tracing::info; pub(super) fn run_stats(index_path: &PathBuf) { @@ -76,7 +77,7 @@ pub(super) fn run_rename(index_path: &PathBuf, spec: &str) { std::process::exit(1); }); - validate_label(&new_label).unwrap_or_else(|e| { + GenomeInfo::validate_label(&new_label).unwrap_or_else(|e| { eprintln!("error: --new-label: {e}"); std::process::exit(1); }); diff --git a/src/obikmer2/Cargo.toml b/src/obikmer2/Cargo.toml new file mode 100644 index 00000000..f332c200 --- /dev/null +++ b/src/obikmer2/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "obikmer2" +version = "1.2.2" +edition = "2024" + +[[bin]] +name = "obikmer2" +path = "src/main.rs" + +[dependencies] +obikseq = { path = "../obikseq" } +obiread = { path = "../obiread" } +obipipeline = { path = "../obipipeline" } +obisys = { path = "../obisys" } +obikindex = { path = "../obikindex", default-features = false } +obikindexer = { path = "../obikindexer" } +obikalgorithm = { path = "../obikalgorithm" } +clap = { version = "4", features = ["derive"] } +tracing = "0.1.44" +tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] } + +[features] +default = ["numa"] +numa = ["obisys/numa"] diff --git a/src/obikmer2/src/cli.rs b/src/obikmer2/src/cli.rs new file mode 100644 index 00000000..6018ad30 --- /dev/null +++ b/src/obikmer2/src/cli.rs @@ -0,0 +1,114 @@ +use std::path::PathBuf; + +use clap::Args; +use obiread::NucPage; +use obikseq::RoutableSuperKmer; +use obipipeline::Throttled; + +// ── Shared arguments ────────────────────────────────────────────────────────── + +#[derive(Args)] +pub struct CommonArgs { + /// Input files or directories (FASTA/FASTQ, optionally gzip-compressed). + /// If omitted, reads from stdin. + #[arg(num_args = 0..)] + pub inputs: Vec, + + /// k-mer size + #[arg(short, long, default_value_t = 31)] + pub kmer_size: usize, + + /// Minimizer size + #[arg(short, long, default_value_t = 11)] + pub minimizer_size: usize, + + /// Entropy threshold (k-mers with score ≤ theta are rejected) + #[arg(long, default_value_t = 0.7)] + pub theta: f64, + + /// Maximum sub-word size for entropy computation + #[arg(long, default_value_t = 6)] + pub level_max: usize, + + /// Number of partitions (rounded up to the next power of 2) + #[arg(short, long, default_value_t = 256)] + pub partitions: usize, + + /// Number of worker threads + #[arg( + short = 'T', + long, + default_value_t = obisys::effective_parallelism() + )] + pub threads: usize, + + /// Maximum number of input files open simultaneously. + /// Defaults to threads/4 (minimum 1). Keep below the number of workers + /// to ensure CPU workers are always available for the transform stage. + #[arg(long)] + pub max_open_files: Option, +} + +/// Smallest `b` such that `2^b >= n` (i.e. `n.next_power_of_two().ilog2()`). +/// Minimum 1 (degenerate n=0 or n=1 → 1 partition). +pub fn partitions_to_bits(n: usize) -> usize { + n.max(1).next_power_of_two().trailing_zeros() as usize +} + +/// Convert a block size (number of unitigs per block) to its `block_bits` exponent. +/// `block_size=1` → `block_bits=0` (one entry per unitig, O(1) random access). +pub fn block_size_to_bits(n: usize) -> u8 { + n.max(1).next_power_of_two().trailing_zeros() as u8 +} + +impl CommonArgs { + /// Validate k and m constraints. Exits on error. + pub fn validate(&self) { + let k = self.kmer_size; + let m = self.minimizer_size; + + if k < 11 || k > 31 { + eprintln!("error: --kmer-size must be in [11, 31] (got {k})"); + std::process::exit(1); + } + if k % 2 == 0 { + eprintln!("error: --kmer-size must be odd (got {k}); even k allows palindromic k-mers"); + std::process::exit(1); + } + if m < 3 || m >= k { + eprintln!("error: --minimizer-size must be in [3, k−1] = [3, {}] (got {m})", k - 1); + std::process::exit(1); + } + if m % 2 == 0 { + eprintln!("error: --minimizer-size must be odd (got {m})"); + std::process::exit(1); + } + } + + pub fn effective_max_open(&self) -> usize { + self.max_open_files + .unwrap_or_else(|| (self.threads / 4).max(1)) + .max(1) + } + + pub fn seqfile_paths(&self) -> obiread::PathIter { + let paths: Vec = if self.inputs.is_empty() { + vec![PathBuf::from("-")] + } else { + self.inputs.iter().map(PathBuf::from).collect() + }; + obiread::PathIter::new(paths) + } +} + +// ── Pipeline data carrier ───────────────────────────────────────────────────── + +pub enum PipelineData { + Path(Throttled), + NucPage(NucPage), + Batch(Vec), +} + +unsafe impl Send for PipelineData {} +unsafe impl Sync for PipelineData {} + diff --git a/src/obikmer2/src/cmd/index/mod.rs b/src/obikmer2/src/cmd/index/mod.rs new file mode 100644 index 00000000..642dc0b8 --- /dev/null +++ b/src/obikmer2/src/cmd/index/mod.rs @@ -0,0 +1,363 @@ +use std::path::PathBuf; +use std::time::Instant; + +use clap::Args; +use obikalgorithm::Algorithm; +use obikindex::layer::IndexMode; +use obikindex::{GenomeInfo, IndexBuilder, IndexConfig, IndexState, KmerIndex}; +use obikindexer::algorithms::counter::Counter; +use obikindexer::algorithms::dereplicator::Dereplicator; +use obikindexer::algorithms::layer_builder::LayerBuilder; +use obikindexer::algorithms::partitionner::PartitionRouter; + +fn current_state(idx: &KmerIndex) -> IndexState { + idx.state().unwrap_or_else(|e| { + eprintln!("error reading index metadata: {e}"); + std::process::exit(1); + }) +} + +fn parse_key_value(s: &str) -> Result<(String, String), String> { + let pos = s + .find('=') + .ok_or_else(|| format!("invalid key=value: no '=' in '{s}'"))?; + Ok((s[..pos].to_string(), s[pos + 1..].to_string())) +} +use obisys::{Progress, Reporter, Stage, progress_bar, spinner}; +use tracing::info; + +use crate::cli::{CommonArgs, block_size_to_bits, partitions_to_bits}; + +#[derive(Args)] +pub struct IndexArgs { + /// Output index directory + #[arg(short, long)] + pub output: PathBuf, + + /// Overwrite output directory if it already exists + #[arg(long, default_value_t = false)] + pub force: bool, + + /// Genome label (default: input filename without path/extension) + #[arg(long)] + pub label: Option, + + /// Genome categorical metadata as key=value pairs (repeatable) + #[arg(long = "meta", value_parser = parse_key_value)] + pub meta: Vec<(String, String)>, + + /// Minimum kmer abundance (inclusive) + #[arg(long, default_value_t = 1)] + pub min_abundance: u32, + + /// Maximum kmer abundance (inclusive) + #[arg(long)] + pub max_abundance: Option, + + /// Store kmer counts in the index (default: set membership only) + #[arg(long, default_value_t = false)] + pub with_counts: bool, + + /// Keep intermediate build files (dereplicated superkmers, mphf1, counts1) + #[arg(long, default_value_t = false)] + pub keep_intermediate: bool, + + /// Use approximate (fingerprint-based) evidence instead of exact evidence. + /// False-positive rate per z-window: 1/2^(b·z). + #[arg(long, default_value_t = false)] + pub approx: bool, + + /// Findere z parameter: number of consecutive k-mers that must all match. + /// Effective indexed k-mer size is kmer_size - z + 1. + #[arg(short = 'z', long, default_value = None)] + pub findere_z: Option, + + /// Fingerprint bits per slot (b). FP per z-window = 1/2^(b·z). + #[arg(long, default_value = None)] + pub evidence_bits: Option, + + /// Target false-positive rate per z-window (e.g. 0.01). + /// Used to derive missing b or z. + #[arg(long, default_value = None)] + pub fp: Option, + + /// Block size for exact evidence `.idx` (number of unitigs per block). + /// Must be a power of two; rounded up if not. Default 1 = O(1) random access. + #[arg(long, default_value_t = 1)] + pub block_size: usize, + + #[command(flatten)] + pub common: CommonArgs, +} + +/// Resolve the (z, b, fp) triplet from the user-supplied subset. +/// +/// Model: FP = 1/2^(b·z) ⟹ b·z = ⌈-log₂(fp)⌉ +/// +/// Rules when one value is missing (conservative = ceiling): +/// given z, b → fp = 1/2^(b·z) +/// given z, fp → b = ⌈-log₂(fp) / z⌉ +/// given b, fp → z = ⌈-log₂(fp) / b⌉ +/// given z only → b = 8 (default), fp derived +/// given b only → z = 1 (default), fp derived +/// given fp only → b = 8 (default), z derived +/// none given → z = 1, b = 8, fp = 1/256 +pub(crate) fn resolve_approx_params( + z_opt: Option, + b_opt: Option, + fp_opt: Option, +) -> (u8, u8, f64) { + const DEFAULT_B: u8 = 8; + const DEFAULT_Z: u8 = 1; + + let bits_needed = |fp: f64| -> u8 { (-fp.log2()).ceil() as u8 }; + + match (z_opt, b_opt, fp_opt) { + // All three given: use b and z, recompute fp conservatively. + (Some(z), Some(b), Some(_fp)) => { + let fp = 1.0_f64 / (1u64 << (b as u32 * z as u32)) as f64; + (z, b, fp) + } + // Two given, derive third. + (Some(z), Some(b), None) => { + let fp = 1.0_f64 / (1u64 << (b as u32 * z as u32)) as f64; + (z, b, fp) + } + (Some(z), None, Some(fp)) => { + let bz = (-fp.log2()).ceil() as u32; + let b = ((bz + z as u32 - 1) / z as u32).max(1) as u8; + let actual_fp = 1.0_f64 / (1u64 << (b as u32 * z as u32)) as f64; + (z, b, actual_fp) + } + (None, Some(b), Some(fp)) => { + let bz = (-fp.log2()).ceil() as u32; + let z = ((bz + b as u32 - 1) / b as u32).max(1) as u8; + let actual_fp = 1.0_f64 / (1u64 << (b as u32 * z as u32)) as f64; + (z, b, actual_fp) + } + // One given, apply defaults for the other. + (Some(z), None, None) => { + let b = DEFAULT_B; + let fp = 1.0_f64 / (1u64 << (b as u32 * z as u32)) as f64; + (z, b, fp) + } + (None, Some(b), None) => { + let z = DEFAULT_Z; + let fp = 1.0_f64 / (1u64 << (b as u32 * z as u32)) as f64; + (z, b, fp) + } + (None, None, Some(fp)) => { + let b = DEFAULT_B; + let z = ((bits_needed(fp) as u32 + b as u32 - 1) / b as u32).max(1) as u8; + let actual_fp = 1.0_f64 / (1u64 << (b as u32 * z as u32)) as f64; + (z, b, actual_fp) + } + // None given: defaults. + (None, None, None) => { + let b = DEFAULT_B; + let z = DEFAULT_Z; + let fp = 1.0_f64 / (1u64 << (b as u32 * z as u32)) as f64; + (z, b, fp) + } + } +} + +pub fn run(args: IndexArgs) { + args.common.validate(); + + let output = args.output.clone(); + let mut rep = Reporter::new(); + + // Locked for the whole build (including a possible --force removal + + // recreation below): a second `index` run resuming/overwriting the same + // output directory concurrently would otherwise corrupt it. Unlinking + // the lock file via --force's remove_dir_all is safe — the held file + // descriptor keeps the lock regardless of the directory entry. + let _lock = obisys::DirLock::acquire(&output).unwrap_or_else(|e| { + eprintln!("error locking output directory {}: {e}", output.display()); + std::process::exit(1); + }); + + // ── Resolve evidence kind ──────────────────────────────────────────────── + let (evidence, effective_kmer_size) = if args.approx { + let (z, b, fp) = resolve_approx_params(args.findere_z, args.evidence_bits, args.fp); + let k = args.common.kmer_size; + if z as usize >= k { + eprintln!( + "error: Findere z={z} must be < kmer-size={k} \ + (effective kmer size k−z+1 = {} ≤ 0)", + k as isize - z as isize + 1 + ); + std::process::exit(1); + } + let s = k - z as usize + 1; + info!("approximate evidence: b={b}, z={z}, fp={fp:.2e}, indexed kmer size={s}"); + (IndexMode::Approx { b, z }, s) + } else { + (IndexMode::Exact, args.common.kmer_size) + }; + + // ── Open or create the index ───────────────────────────────────────────── + if KmerIndex::is_an_index(&output) { + if !args.force { + eprintln!( + "error: an index already exists at {} (use --force to overwrite it)", + output.display() + ); + std::process::exit(1); + } + info!("--force: removing existing index at {}", output.display()); + std::fs::remove_dir_all(&output).unwrap_or_else(|e| { + eprintln!("error removing existing index: {e}"); + std::process::exit(1); + }); + } else if output.exists() { + eprintln!( + "error: {} exists but is not an obikmer index, it cannot be deleted", + output.display() + ); + std::process::exit(1); + } + let n_bits = partitions_to_bits(args.common.partitions); + let effective = 1usize << n_bits; + if effective != args.common.partitions { + info!( + "partitions: {} → {} (next power of 2)", + args.common.partitions, effective + ); + } + let block_bits = block_size_to_bits(args.block_size); + let config = IndexConfig { + kmer_size: effective_kmer_size, + minimizer_size: args.common.minimizer_size, + n_bits, + with_counts: args.with_counts, + evidence: evidence.clone(), + block_bits, + }; + let genome_info = args.label.as_ref().map(|label| { + GenomeInfo::validate_label(label).unwrap_or_else(|e| { + eprintln!("error: --label: {e}"); + std::process::exit(1); + }); + let mut info = GenomeInfo::new(label.clone()); + for (k, v) in &args.meta { + info.meta.insert(k.clone(), v.clone()); + } + info + }); + let idx = KmerIndex::create(&output, config, genome_info).unwrap_or_else(|e| { + eprintln!("error creating index: {e}"); + std::process::exit(1); + }); + + // ── Stage 1: scatter ───────────────────────────────────────────────────── + if current_state(&idx) < IndexState::Scattered { + let n_workers = args.common.threads.max(1); + let max_open = args.common.effective_max_open(); + + let t = Stage::start("scatter"); + let pb = spinner("scatter"); + let mut ema_rate: f64 = 0.0; + let mut last_t = Instant::now(); + let mut last_bases: u64 = 0; + const ALPHA: f64 = 0.15; + + let mut router = PartitionRouter::new(&idx) + .level_max(args.common.level_max) + .theta(args.common.theta) + .workers(n_workers) + .max_open(max_open) + .files(args.common.seqfile_paths()) + .on_progress(|p: Progress| { + let now = Instant::now(); + let dt = now.duration_since(last_t).as_secs_f64(); + if dt > 0.0 { + let instant = (p.position - last_bases) as f64 / dt; + ema_rate = ALPHA * instant + (1.0 - ALPHA) * ema_rate; + } + last_t = now; + last_bases = p.position; + let bp = p.position as f64; + let (count_str, rate_str) = if bp >= 1e9 { + ( + format!("{:.2} Gbp", bp / 1e9), + format!("{:.0} Mbp/s", ema_rate / 1e6), + ) + } else { + ( + format!("{:.0} Mbp", bp / 1e6), + format!("{:.0} Mbp/s", ema_rate / 1e6), + ) + }; + pb.set_message(format!("{count_str} {rate_str}")); + }); + + router.run().unwrap_or_else(|e| { + eprintln!("error: {e}"); + std::process::exit(1); + }); + pb.finish_and_clear(); + rep.push(t.stop()); + drop(router); // ends the borrow of `idx` early — `PartitionRouter`'s `Drop` impl would otherwise extend it to the end of scope (`run()` already called `close()`, which marks scatter done, internally) + } else { + info!("scatter already done, skipping"); + } + + // ── Stage 2: dereplicate + count ───────────────────────────────────────── + if current_state(&idx) < IndexState::Counted { + let t = Stage::start("dereplicate"); + let pb = progress_bar("dereplication", idx.n_partitions() as u64, "partitions"); + Dereplicator::new(&idx) + .on_progress(|_: Progress| pb.inc(1)) + .run() + .unwrap_or_else(|e| { + eprintln!("error: {e}"); + std::process::exit(1); + }); + pb.finish_and_clear(); + rep.push(t.stop()); + + let t = Stage::start("count_kmer"); + let pb = progress_bar("counting", idx.n_partitions() as u64, "partitions"); + // `Counter::run` writes `spectrums/{label}.json` and marks count + // done (`count.done`) internally once every partition succeeds. + Counter::new(&idx) + .keep_partial(args.keep_intermediate) + .on_progress(|_: Progress| pb.inc(1)) + .run() + .unwrap_or_else(|e| { + eprintln!("error: {e}"); + std::process::exit(1); + }); + pb.finish_and_clear(); + rep.push(t.stop()); + } else { + info!("dereplicate+count already done, skipping"); + } + + // ── Stage 3: build layered index ───────────────────────────────────────── + if current_state(&idx) < IndexState::Indexed { + let t = Stage::start("index"); + let pb = progress_bar("index", idx.n_partitions() as u64, "partitions"); + let total_kmers = LayerBuilder::new(&idx) + .min_abundance(args.min_abundance) + .max_abundance(args.max_abundance) + .keep_intermediate(args.keep_intermediate) + .on_progress(|_: Progress| pb.inc(1)) + .run() + .unwrap_or_else(|e| { + eprintln!("error: {e}"); + std::process::exit(1); + }); + pb.finish_and_clear(); + info!("done — {total_kmers} total kmers indexed"); + rep.push(t.stop()); + // `LayerBuilder::run` marks the index done (`index.done`) internally + // once every partition succeeds. + } else { + info!("index already built, skipping"); + } + + rep.print(); +} diff --git a/src/obikmer2/src/cmd/mod.rs b/src/obikmer2/src/cmd/mod.rs new file mode 100644 index 00000000..33edc959 --- /dev/null +++ b/src/obikmer2/src/cmd/mod.rs @@ -0,0 +1 @@ +pub mod index; diff --git a/src/obikmer2/src/main.rs b/src/obikmer2/src/main.rs new file mode 100644 index 00000000..a490edd1 --- /dev/null +++ b/src/obikmer2/src/main.rs @@ -0,0 +1,32 @@ +mod cli; +mod cmd; + +use clap::{Parser, Subcommand}; +use tracing_subscriber::{EnvFilter, fmt}; + +#[derive(Parser)] +#[command(name = "obikmer2", about = "DNA k-mer tools", version)] +struct Cli { + #[command(subcommand)] + command: Commands, +} + +#[derive(Subcommand)] +enum Commands { + /// Build the complete genome index (scatter → dereplicate → count → layered MPHF) + Index(cmd::index::IndexArgs), +} + +fn main() { + fmt() + .with_env_filter( + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")), + ) + .with_writer(std::io::stderr) + .init(); + + let cli = Cli::parse(); + match cli.command { + Commands::Index(args) => cmd::index::run(args), + } +} diff --git a/src/obikmerge/Cargo.toml b/src/obikmerge/Cargo.toml new file mode 100644 index 00000000..1620c3c9 --- /dev/null +++ b/src/obikmerge/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "obikmerge" +version = "0.1.0" +edition = "2024" + +[dependencies] +obikindex = { path = "../obikindex" } +obikindexer = { path = "../obikindexer" } +obikfilter = { path = "../obikfilter" } +obicompactvec = { path = "../obicompactvec" } +obiskio = { path = "../obiskio" } +obikseq = { path = "../obikseq" } +obipipeline = { path = "../obipipeline" } +obisys = { path = "../obisys" } +tracing = "0.1.44" diff --git a/src/obikmerge/src/lib.rs b/src/obikmerge/src/lib.rs new file mode 100644 index 00000000..89d51a79 --- /dev/null +++ b/src/obikmerge/src/lib.rs @@ -0,0 +1,17 @@ +//! Merging multiple `obikindex::KmerIndex` sources into one: bootstrapping +//! from the first source, then merging each remaining source partition by +//! partition (de Bruijn graph union + column fill). A maintenance/ +//! transformation operation on already-built indexes, not part of the +//! `Index { Partition { Layer } }` data model itself — kept out of +//! `obikindex` so that crate doesn't grow this algorithm's own +//! dependencies (`obipipeline`, `obidebruinj` via `obikindexer`). +//! +//! [`merge_layer`] holds the per-partition merge primitive +//! (`merge_partition`, `SrcLayerData`) — the latter is also reused by +//! `obikrebuild::rebuild_layer`, hence its `pub` visibility here. + +mod merge; +mod merge_layer; + +pub use merge::*; +pub use merge_layer::{MergeMode, SrcLayerData}; diff --git a/src/obikindex/src/index/merge.rs b/src/obikmerge/src/merge.rs similarity index 97% rename from src/obikindex/src/index/merge.rs rename to src/obikmerge/src/merge.rs index 4ddad6eb..c428a94b 100644 --- a/src/obikindex/src/index/merge.rs +++ b/src/obikmerge/src/merge.rs @@ -3,19 +3,20 @@ use std::fs; use std::io; use std::path::Path; -use crate::index::builder::IndexBuilder; +use obikindex::IndexBuilder; use obisys::{Reporter, Stage, progress_bar, spinner}; use tracing::{debug, info}; -use crate::layer::IndexMode; +use obikindex::layer::IndexMode; -use crate::index::error::{OKIError, OKIResult}; -use crate::index::kmer_index::KmerIndex; -use crate::index::meta::{GenomeInfo, IndexMeta}; -use crate::index::state::IndexState; +use obikindex::{OKIError, OKIResult}; +use obikindex::KmerIndex; +use obikindex::{GenomeInfo, IndexMeta}; +use obikindex::IndexState; +use obisys::PartitionRunner; -pub use crate::index::merge_layer::MergeMode; +pub use crate::merge_layer::MergeMode; // ── per-partition diagnostic record ────────────────────────────────────────── @@ -229,7 +230,7 @@ impl KmerIndex { let srcs = &srcs; let evidence = &evidence; - let runner = crate::index::numa::PartitionRunner::new(); + let runner = PartitionRunner::new(); let mut part_stats: Vec = Vec::with_capacity(n_partitions); runner diff --git a/src/obikindex/src/index/merge_layer/mod.rs b/src/obikmerge/src/merge_layer/mod.rs similarity index 94% rename from src/obikindex/src/index/merge_layer/mod.rs rename to src/obikmerge/src/merge_layer/mod.rs index f8a35849..f659327c 100644 --- a/src/obikindex/src/index/merge_layer/mod.rs +++ b/src/obikmerge/src/merge_layer/mod.rs @@ -20,16 +20,18 @@ use tracing::debug; use obicompactvec::{PersistentBitMatrixBuilder, PersistentCompactIntMatrixBuilder}; use obikseq::CanonicalKmer; -use crate::layer::{IndexMode, TypedLayer, LayeredMap, MphfOnly, layer_dir}; -use obiskio::{SKError, SKResult, UnitigFileReader}; +use obikindex::layer::{IndexMode, TypedLayer, LayeredMap, MphfOnly}; +use obikindex::layer::utils::layer_dir; +use obiskio::UnitigFileReader; +use obikindex::{OKIError, OKIResult}; -use crate::index::common::{ColBuilder, load_meta, olm_to_sk}; -use crate::index::graph_pipeline::{build_graph, materialize_layer}; -use crate::index::kmer_index::KmerIndex; +use obikindex::{ColBuilder, load_meta}; +use obikindexer::{build_graph, materialize_layer}; +use obikindex::KmerIndex; mod src_layer; -pub(crate) use src_layer::SrcLayerData; +pub use src_layer::SrcLayerData; // ── MergeMode ───────────────────────────────────────────────────────────────── @@ -216,15 +218,14 @@ impl KmerIndex { n_dst_genomes: usize, block_bits: u8, evidence: &IndexMode, - ) -> SKResult { + ) -> OKIResult { let dst_index_dir = self.index_dir(i); if !dst_index_dir.exists() { return Ok(0); } - load_meta(&dst_index_dir, "merge")?; // ensure meta.json exists before LayeredMap::open - let dst_map = - Arc::new(LayeredMap::<()>::open(&dst_index_dir).map_err(|e| olm_to_sk(e, "merge"))?); + load_meta(&dst_index_dir)?; // ensure meta.json exists before LayeredMap::open + let dst_map = Arc::new(LayeredMap::<()>::open(&dst_index_dir)?); let n_dst_layers = dst_map.n_layers(); let n_src_total: usize = sources.iter().map(|(_, n)| *n).sum(); @@ -235,8 +236,7 @@ impl KmerIndex { TypedLayer::<()>::init_presence_matrix( &layer_dir(&dst_index_dir, l), dst_map.layer(l).n(), - ) - .map_err(|e| olm_to_sk(e, "merge"))?; + )?; } } @@ -254,7 +254,7 @@ impl KmerIndex { if !src_index_dir.exists() { continue; } - let src_meta = load_meta(&src_index_dir, "merge")?; + let src_meta = load_meta(&src_index_dir)?; for l in 0..src_meta.n_layers { let p = layer_dir(&src_index_dir, l).join("unitigs.bin"); if p.exists() { @@ -277,7 +277,7 @@ impl KmerIndex { let g = build_graph( unitig_paths.into_iter(), - move |path: PathBuf, emit: &mut dyn FnMut(Vec)| -> SKResult<()> { + move |path: PathBuf, emit: &mut dyn FnMut(Vec)| -> OKIResult<()> { let reader = UnitigFileReader::open_sequential(&path)?; let mut batch: Vec = Vec::with_capacity(BATCH); for (kmer, _, _) in reader.iter_indexed_canonical_kmers() { @@ -318,9 +318,7 @@ impl KmerIndex { let t_open = std::time::Instant::now(); let new_mphf: Option> = if any_new { - Some(Arc::new( - MphfOnly::open(&new_layer_dir).map_err(|e| olm_to_sk(e, "merge"))?, - )) + Some(Arc::new(MphfOnly::open(&new_layer_dir)?)) } else { None }; @@ -340,13 +338,13 @@ impl KmerIndex { MergeMode::Count => new_layer_dir.join("counts"), }; fs::create_dir_all(&data_dir)?; - let mut mb = MatrixBuilder::new(mode, n_new, &data_dir).map_err(SKError::Io)?; + let mut mb = MatrixBuilder::new(mode, n_new, &data_dir).map_err(OKIError::Io)?; for _ in 0..n_dst_genomes { - mb.add_absent_col().map_err(SKError::Io)?; + mb.add_absent_col().map_err(OKIError::Io)?; } let cols = (0..n_src_total) - .map(|_| mb.add_col().map_err(SKError::Io)) - .collect::>>()?; + .map(|_| mb.add_col().map_err(OKIError::Io)) + .collect::>>()?; (cols, Some(mb)) } else { (vec![], None) @@ -365,10 +363,10 @@ impl KmerIndex { MergeMode::Presence => layer_dir.join("presence"), MergeMode::Count => layer_dir.join("counts"), }; - let mut mb = MatrixBuilder::resume(mode, &data_dir).map_err(SKError::Io)?; + let mut mb = MatrixBuilder::resume(mode, &data_dir).map_err(OKIError::Io)?; let cols = (0..n_src_total) - .map(|_| mb.add_col().map_err(SKError::Io)) - .collect::>>()?; + .map(|_| mb.add_col().map_err(OKIError::Io)) + .collect::>>()?; exist_mbs.push(mb); exist_builders.push(cols); } @@ -391,7 +389,7 @@ impl KmerIndex { col_offset += src_n; continue; } - let src_meta = load_meta(&src_index_dir, "merge")?; + let src_meta = load_meta(&src_index_dir)?; for l in 0..src_meta.n_layers { let src_layer_dir = layer_dir(&src_index_dir, l); if src_layer_dir.join("unitigs.bin").exists() { @@ -552,7 +550,7 @@ impl KmerIndex { .into_inner() .unwrap_or_else(|e| e.into_inner()) { - return Err(SKError::InvalidData { + return Err(OKIError::InvalidData { context: "merge pass2", detail: msg, }); @@ -568,7 +566,7 @@ impl KmerIndex { .unwrap_or_else(|e| e.into_inner()) .close()?; } - mb.close().map_err(SKError::Io)?; + mb.close().map_err(OKIError::Io)?; } for b in new_locked { @@ -579,13 +577,12 @@ impl KmerIndex { .close()?; } if let Some(mb) = new_mb { - mb.close().map_err(SKError::Io)?; + mb.close().map_err(OKIError::Io)?; let mut part_meta = self.partition_meta(i)?; part_meta.n_layers = new_layer_idx + 1; part_meta - .save(&dst_index_dir) - .map_err(|e| olm_to_sk(e, "merge"))?; + .save(&dst_index_dir)?; } debug!( diff --git a/src/obikindex/src/index/merge_layer/src_layer.rs b/src/obikmerge/src/merge_layer/src_layer.rs similarity index 76% rename from src/obikindex/src/index/merge_layer/src_layer.rs rename to src/obikmerge/src/merge_layer/src_layer.rs index 4f104b6f..5078f735 100644 --- a/src/obikindex/src/index/merge_layer/src_layer.rs +++ b/src/obikmerge/src/merge_layer/src_layer.rs @@ -2,44 +2,43 @@ use std::path::Path; use obicompactvec::{MatrixGroupOps, PersistentBitMatrix, PersistentCompactIntMatrix}; use obikseq::CanonicalKmer; -use crate::layer::MphfOnly; -use obiskio::{SKError, SKResult}; +use obikindex::layer::MphfOnly; +use obikindex::{OKIError, OKIResult}; -use crate::index::common::olm_to_sk; use super::MergeMode; // ── SrcLayerData — opened source matrix for pass-2 lookup ───────────────────── -pub(crate) enum SrcLayerData { +pub enum SrcLayerData { Presence(MphfOnly, PersistentBitMatrix), Count(MphfOnly, PersistentCompactIntMatrix), } impl SrcLayerData { - pub(crate) fn open(layer_dir: &Path, merge_mode: MergeMode) -> SKResult { + pub fn open(layer_dir: &Path, merge_mode: MergeMode) -> OKIResult { let counts_dir = layer_dir.join("counts"); match merge_mode { MergeMode::Presence => { if counts_dir.exists() && !layer_dir.join("presence").exists() { - let mphf = MphfOnly::open(layer_dir).map_err(|e| olm_to_sk(e, "merge"))?; - let mat = PersistentCompactIntMatrix::open(layer_dir).map_err(SKError::Io)?; + let mphf = MphfOnly::open(layer_dir)?; + let mat = PersistentCompactIntMatrix::open(layer_dir).map_err(OKIError::Io)?; Ok(SrcLayerData::Count(mphf, mat)) } else { // presence dir exists, or neither exists → Implicit handled by open() - let mphf = MphfOnly::open(layer_dir).map_err(|e| olm_to_sk(e, "merge"))?; - let mat = PersistentBitMatrix::open(layer_dir).map_err(SKError::Io)?; + let mphf = MphfOnly::open(layer_dir)?; + let mat = PersistentBitMatrix::open(layer_dir).map_err(OKIError::Io)?; Ok(SrcLayerData::Presence(mphf, mat)) } } MergeMode::Count => { - let mphf = MphfOnly::open(layer_dir).map_err(|e| olm_to_sk(e, "merge"))?; + let mphf = MphfOnly::open(layer_dir)?; if counts_dir.exists() { - let mat = PersistentCompactIntMatrix::open(layer_dir).map_err(SKError::Io)?; + let mat = PersistentCompactIntMatrix::open(layer_dir).map_err(OKIError::Io)?; Ok(SrcLayerData::Count(mphf, mat)) } else { // No counts → treat as implicit presence (all 1s) - let mat = PersistentBitMatrix::open(layer_dir).map_err(SKError::Io)?; + let mat = PersistentBitMatrix::open(layer_dir).map_err(OKIError::Io)?; Ok(SrcLayerData::Presence(mphf, mat)) } } @@ -58,7 +57,7 @@ impl SrcLayerData { buf } - pub(crate) fn n_slots(&self) -> usize { + pub fn n_slots(&self) -> usize { match self { SrcLayerData::Presence(_, mat) => mat.n(), SrcLayerData::Count(_, mat) => mat.n(), @@ -67,7 +66,7 @@ impl SrcLayerData { /// MPHF lookup: returns the slot index for `kmer` (kmer must be in the domain). #[inline] - pub(crate) fn slot(&self, kmer: CanonicalKmer) -> usize { + pub fn slot(&self, kmer: CanonicalKmer) -> usize { match self { SrcLayerData::Presence(mphf, _) => mphf.index(kmer), SrcLayerData::Count(mphf, _) => mphf.index(kmer), @@ -76,7 +75,7 @@ impl SrcLayerData { /// Row lookup by slot index, bypassing the MPHF. #[inline] - pub(crate) fn fill_row_by_slot(&self, slot: usize, n_genomes: usize) -> Vec { + pub fn fill_row_by_slot(&self, slot: usize, n_genomes: usize) -> Vec { let mut buf = vec![0u32; n_genomes]; match self { SrcLayerData::Presence(_, mat) => mat.fill_row(slot, &mut buf), @@ -86,7 +85,7 @@ impl SrcLayerData { } /// Call `f` with a reference to the underlying matrix as `&dyn MatrixGroupOps`. - pub(crate) fn with_matrix(&self, f: impl FnOnce(&dyn MatrixGroupOps) -> R) -> R { + pub fn with_matrix(&self, f: impl FnOnce(&dyn MatrixGroupOps) -> R) -> R { match self { SrcLayerData::Presence(_, mat) => f(mat), SrcLayerData::Count(_, mat) => f(mat), diff --git a/src/obikphylo/Cargo.toml b/src/obikphylo/Cargo.toml index df689c2c..ee7a5593 100644 --- a/src/obikphylo/Cargo.toml +++ b/src/obikphylo/Cargo.toml @@ -9,6 +9,7 @@ obikseq = { path = "../obikseq" } obiskio = { path = "../obiskio" } obisys = { path = "../obisys" } obicompactvec = { path = "../obicompactvec" } +obikidxcache = { path = "../obikidxcache" } obiskbuilder = { path = "../obiskbuilder" } obipipeline = { path = "../obipipeline" } memmap2 = "0.9" diff --git a/src/obikindex/src/index/distance.rs b/src/obikphylo/src/distance.rs similarity index 97% rename from src/obikindex/src/index/distance.rs rename to src/obikphylo/src/distance.rs index 957ed055..00e0f3a2 100644 --- a/src/obikindex/src/index/distance.rs +++ b/src/obikphylo/src/distance.rs @@ -1,10 +1,10 @@ use ndarray::Array2; use obicompactvec::traits::{BitPartials, CountPartials}; -use crate::layer::LayeredStore; +use obikidxcache::LayeredStore; use rayon::prelude::*; -use crate::index::error::{OKIError, OKIResult}; -use crate::index::kmer_index::KmerIndex; +use obikindex::{OKIError, OKIResult}; +use obikindex::KmerIndex; // ── Public API ──────────────────────────────────────────────────────────────── diff --git a/src/obikphylo/src/lib.rs b/src/obikphylo/src/lib.rs index 53a34bab..653c2cb3 100644 --- a/src/obikphylo/src/lib.rs +++ b/src/obikphylo/src/lib.rs @@ -9,6 +9,9 @@ //! incrementally. mod cardcomp; +mod distance; +mod matrix_store; pub mod siblings; pub use cardcomp::{cardinality_transition_probs, composition_transition_probs, pairwise_cost_matrix}; +pub use distance::{DistanceMetric, DistanceOutput}; diff --git a/src/obikindex/src/index/matrix_store.rs b/src/obikphylo/src/matrix_store.rs similarity index 58% rename from src/obikindex/src/index/matrix_store.rs rename to src/obikphylo/src/matrix_store.rs index 0d92a5c1..74ddf02f 100644 --- a/src/obikindex/src/index/matrix_store.rs +++ b/src/obikphylo/src/matrix_store.rs @@ -1,46 +1,47 @@ +use obikindex::OKIResult; +use obikindex::layer::open_data; +use obikidxcache::LayeredStore; use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix}; -use crate::layer::{LayeredStore, open_data}; -use obiskio::SKResult; -use crate::index::common::{load_meta, olm_to_sk}; -use crate::index::kmer_index::KmerIndex; +use obikindex::load_meta; +use obikindex::KmerIndex; impl KmerIndex { /// Open all count matrices for partition `part`, one per layer. /// Layers without a `counts/` directory are skipped. - pub fn count_store(&self, part: usize) -> SKResult> { + pub fn count_store(&self, part: usize) -> OKIResult> { let index_dir = self.index_dir(part); if !index_dir.exists() { return Ok(LayeredStore::new(vec![])); } - let n_layers = load_meta(&index_dir, "distance")?.n_layers; + let n_layers = self.n_layers(); let matrices = (0..n_layers) .filter_map(|l| { self.layer_dir(part, l) .join("counts") .exists() - .then(|| open_data(&index_dir, l).map_err(|e| olm_to_sk(e, "distance"))) + .then(|| open_data(&index_dir, l)) }) - .collect::>>()?; + .collect::>>()?; Ok(LayeredStore::new(matrices)) } /// Open all presence matrices for partition `part`, one per layer. /// Layers without a `presence/` directory are skipped. - pub fn presence_store(&self, part: usize) -> SKResult> { + pub fn presence_store(&self, part: usize) -> OKIResult> { let index_dir = self.index_dir(part); if !index_dir.exists() { return Ok(LayeredStore::new(vec![])); } - let n_layers = load_meta(&index_dir, "distance")?.n_layers; + let n_layers = load_meta(&index_dir)?.n_layers; let matrices = (0..n_layers) .filter_map(|l| { self.layer_dir(part, l) .join("presence") .exists() - .then(|| open_data(&index_dir, l).map_err(|e| olm_to_sk(e, "distance"))) + .then(|| open_data(&index_dir, l)) }) - .collect::>>()?; + .collect::>>()?; Ok(LayeredStore::new(matrices)) } } diff --git a/src/obikphylo/src/siblings/build.rs b/src/obikphylo/src/siblings/build.rs index 090ea03b..4ed4a703 100644 --- a/src/obikphylo/src/siblings/build.rs +++ b/src/obikphylo/src/siblings/build.rs @@ -14,7 +14,7 @@ use obikindex::OKIResult; use super::cache::PartitionCache; use super::helpers::{central_base, is_minorant}; -use super::{ANNEX_FILE_NAME, FamilyMask, SiblingAnnexBuilder, olm_to_ok}; +use super::{ANNEX_FILE_NAME, FamilyMask, SiblingAnnexBuilder}; // ── obipipeline data types ───────────────────────────────────────────────── @@ -125,7 +125,7 @@ fn build_layer_sibling_annex( l: usize, cache: &Arc, ) -> OKIResult { - let mphf = MphfLayer::open(layer_dir).map_err(olm_to_ok)?; + let mphf = MphfLayer::open(layer_dir)?; let k = index.kmer_size(); let n = mphf.n(); diff --git a/src/obikphylo/src/siblings/family_scan.rs b/src/obikphylo/src/siblings/family_scan.rs index 572ade99..59d4d368 100644 --- a/src/obikphylo/src/siblings/family_scan.rs +++ b/src/obikphylo/src/siblings/family_scan.rs @@ -65,7 +65,7 @@ use obikindex::layer::KmerLayer; use super::cache::PartitionCache; use super::helpers::central_base; use super::iter::{SiblingEntry, SiblingLayerExt}; -use super::{ANNEX_FILE_NAME, FamilyMask, SiblingAnnex, olm_to_ok}; +use super::{ANNEX_FILE_NAME, FamilyMask, SiblingAnnex}; /// Families per batch — see the module docs for the memory-vs-per-partition- /// density trade-off this picks a point on. At ~90 genomes and a few @@ -194,10 +194,10 @@ pub(super) fn scan_layer_families( let index_dir = layer_dir .parent() .expect("layer_dir has a parent index dir"); - let meta = PartitionMeta::load(index_dir).map_err(olm_to_ok)?; + let meta = PartitionMeta::load(index_dir)?; let annex = Arc::new(SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME))?); - let mat = KmerLayer::open(layer_dir, &meta.mode, with_counts).map_err(olm_to_ok)?; + let mat = KmerLayer::open(layer_dir, &meta.mode, with_counts)?; let n_cols = mat.n_cols().min(n_genomes); let ctx = Arc::new(LayerCtx { diff --git a/src/obikphylo/src/siblings/mod.rs b/src/obikphylo/src/siblings/mod.rs index 6556bedd..bbd03b59 100644 --- a/src/obikphylo/src/siblings/mod.rs +++ b/src/obikphylo/src/siblings/mod.rs @@ -71,15 +71,4 @@ pub(crate) use siblingannex::{FamilyMask, SiblingAnnex, SiblingAnnexBuilder}; pub use stats::{SiblingAnnexStats, SiblingStatsExt}; pub use subsample::EntropyBias; -use obikindex::layer::OLMError; - -use obikindex::OKIError; - pub(super) const ANNEX_FILE_NAME: &str = "siblings.psib"; - -pub(super) fn olm_to_ok(e: OLMError) -> OKIError { - match e { - OLMError::Io(e) => OKIError::Io(e), - other => OKIError::InvalidInput(format!("layered-map error: {other}")), - } -} diff --git a/src/obikquery/Cargo.toml b/src/obikquery/Cargo.toml new file mode 100644 index 00000000..df532dc6 --- /dev/null +++ b/src/obikquery/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "obikquery" +version = "0.1.0" +edition = "2024" + +[dependencies] +obikindex = { path = "../obikindex" } diff --git a/src/obikquery/src/lib.rs b/src/obikquery/src/lib.rs new file mode 100644 index 00000000..4201bd0a --- /dev/null +++ b/src/obikquery/src/lib.rs @@ -0,0 +1,10 @@ +//! Query-side operations on an `obikindex::KmerIndex`: staging ground for +//! code currently living in `obikindex::index`'s query path, to be migrated +//! here to lighten that crate — see `DevDocMD/` for the rationale. Kept as +//! a separate crate — not a module of `obikindex` — so the dependency runs +//! one way only (query code depends on the data model, never the reverse), +//! same pattern as `obikindexer` for the build side. + +mod query_layer; + +pub use query_layer::{KmerDesc, QueryHit, QueryStats}; diff --git a/src/obikindex/src/index/query_layer.rs b/src/obikquery/src/query_layer.rs similarity index 93% rename from src/obikindex/src/index/query_layer.rs rename to src/obikquery/src/query_layer.rs index 7cfd3084..2684408e 100644 --- a/src/obikindex/src/index/query_layer.rs +++ b/src/obikquery/src/query_layer.rs @@ -3,20 +3,10 @@ use std::path::Path; use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix}; use obikseq::CanonicalKmer; -use crate::layer::{MphfLayer, OLMError}; -use obiskio::{SKError, SKResult}; +use obikindex::layer::MphfLayer; +use obikindex::{OKIError, OKIResult}; -use crate::index::kmer_index::KmerIndex; - -fn olm_to_sk(e: OLMError) -> SKError { - match e { - OLMError::Io(io_err) => SKError::Io(io_err), - other => SKError::InvalidData { - context: "query", - detail: other.to_string(), - }, - } -} +use obikindex::KmerIndex; // ── per-layer query handle ──────────────────────────────────────────────────── @@ -26,21 +16,21 @@ enum QueryLayer { } impl QueryLayer { - fn open(layer_dir: &Path, with_counts: bool) -> SKResult { - let mphf = MphfLayer::open(layer_dir).map_err(olm_to_sk)?; + fn open(layer_dir: &Path, with_counts: bool) -> OKIResult { + let mphf = MphfLayer::open(layer_dir)?; let counts_dir = layer_dir.join("counts"); let presence_dir = layer_dir.join("presence"); if with_counts && counts_dir.exists() { - let mat = PersistentCompactIntMatrix::open(layer_dir).map_err(SKError::Io)?; + let mat = PersistentCompactIntMatrix::open(layer_dir).map_err(OKIError::Io)?; Ok(QueryLayer::Count(mphf, mat)) } else if presence_dir.exists() || !counts_dir.exists() { // presence mode, or no matrix at all → Implicit handled inside open() - let mat = PersistentBitMatrix::open(layer_dir).map_err(SKError::Io)?; + let mat = PersistentBitMatrix::open(layer_dir).map_err(OKIError::Io)?; Ok(QueryLayer::Presence(mphf, mat)) } else { // counts exist but not presence — count layer, no presence requested - let mat = PersistentCompactIntMatrix::open(layer_dir).map_err(SKError::Io)?; + let mat = PersistentCompactIntMatrix::open(layer_dir).map_err(OKIError::Io)?; Ok(QueryLayer::Count(mphf, mat)) } } @@ -168,7 +158,7 @@ impl KmerIndex { n_genomes: usize, with_counts: bool, mut on_event: F, - ) -> SKResult + ) -> OKIResult where F: FnMut(QueryHit), { @@ -186,7 +176,7 @@ impl KmerIndex { let meta = self.partition_meta(part_idx)?; let layers: Vec = (0..meta.n_layers) .map(|i| QueryLayer::open(&self.layer_dir(part_idx, i), with_counts)) - .collect::>()?; + .collect::>()?; // ── Stage 1: MPHF-only pass, bucket hits by (layer_idx, slot) ──────── let mut by_layer: Vec>> = diff --git a/src/obikindex/src/index/tests/query_layer.rs b/src/obikquery/src/tests/query_layer.rs similarity index 95% rename from src/obikindex/src/index/tests/query_layer.rs rename to src/obikquery/src/tests/query_layer.rs index 702da3af..5dca0266 100644 --- a/src/obikindex/src/index/tests/query_layer.rs +++ b/src/obikquery/src/tests/query_layer.rs @@ -1,5 +1,5 @@ use super::*; -use crate::index::meta::IndexConfig; +use obikindex::IndexConfig; // ── QueryStats::AddAssign ─────────────────────────────────────────────────── @@ -50,7 +50,7 @@ fn query_partition_with_missing_index_dir_returns_default_stats() { minimizer_size: 9, n_bits: 2, with_counts: false, - evidence: crate::layer::IndexMode::Exact, + evidence: obikindex::layer::IndexMode::Exact, block_bits: 0, }; let index = KmerIndex::create(tmp.path().join("idx"), config, None).expect("create index"); @@ -78,7 +78,7 @@ fn query_partition_with_empty_kmers_is_a_noop() { minimizer_size: 9, n_bits: 2, with_counts: false, - evidence: crate::layer::IndexMode::Exact, + evidence: obikindex::layer::IndexMode::Exact, block_bits: 0, }; let index = KmerIndex::create(tmp.path().join("idx"), config, None).expect("create index"); diff --git a/src/obikrebuild/Cargo.toml b/src/obikrebuild/Cargo.toml new file mode 100644 index 00000000..50c39c6c --- /dev/null +++ b/src/obikrebuild/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "obikrebuild" +version = "0.1.0" +edition = "2024" + +[dependencies] +obikindex = { path = "../obikindex" } +obikindexer = { path = "../obikindexer" } +obikfilter = { path = "../obikfilter" } +obikmerge = { path = "../obikmerge" } +obicompactvec = { path = "../obicompactvec" } +obiskio = { path = "../obiskio" } +obikseq = { path = "../obikseq" } +obidebruinj = { path = "../obidebruinj" } +obisys = { path = "../obisys" } +tracing = "0.1.44" diff --git a/src/obikrebuild/src/lib.rs b/src/obikrebuild/src/lib.rs new file mode 100644 index 00000000..71c8a41f --- /dev/null +++ b/src/obikrebuild/src/lib.rs @@ -0,0 +1,12 @@ +//! Rebuilding an `obikindex::KmerIndex`: compacting a source index into a +//! new single-layer index while applying k-mer filters (`rebuild`), and +//! reindexing an existing index's evidence representation in place +//! (`reindex`, `Exact`/`Approx`/`Hybrid`). Both are maintenance/ +//! transformation operations on an already-built index, not part of the +//! `Index { Partition { Layer } }` data model itself — kept out of +//! `obikindex` so that crate doesn't grow every algorithm's own +//! dependencies (`obikfilter`, `obikmerge`, `obikindexer`). + +mod rebuild; +mod rebuild_layer; +mod reindex; diff --git a/src/obikindex/src/index/rebuild.rs b/src/obikrebuild/src/rebuild.rs similarity index 90% rename from src/obikindex/src/index/rebuild.rs rename to src/obikrebuild/src/rebuild.rs index 63240725..67746572 100644 --- a/src/obikindex/src/index/rebuild.rs +++ b/src/obikrebuild/src/rebuild.rs @@ -1,13 +1,15 @@ use std::path::Path; -use crate::index::builder::IndexBuilder; -use crate::index::{KmerFilter, MergeMode}; +use obikindex::IndexBuilder; +use obikfilter::KmerFilter; +use obikmerge::MergeMode; use obisys::{Reporter, Stage, progress_bar}; use tracing::info; -use crate::index::error::{OKIError, OKIResult}; -use crate::index::kmer_index::KmerIndex; -use crate::index::state::IndexState; +use obikindex::{OKIError, OKIResult}; +use obikindex::KmerIndex; +use obikindex::IndexState; +use obisys::PartitionRunner; impl KmerIndex { /// Rebuild `src` into a new compact single-layer index at `output`. @@ -61,7 +63,7 @@ impl KmerIndex { let pb = progress_bar("rebuild", n_partitions as u64, "partitions"); let order: Vec = (0..n_partitions).collect(); - let runner = crate::index::numa::PartitionRunner::new(); + let runner = PartitionRunner::new(); runner.run( &order, |i| dst_partition.rebuild_partition(src, i, filters, mode, n_genomes, block_bits), diff --git a/src/obikindex/src/index/rebuild_layer.rs b/src/obikrebuild/src/rebuild_layer.rs similarity index 86% rename from src/obikindex/src/index/rebuild_layer.rs rename to src/obikrebuild/src/rebuild_layer.rs index 530df658..a3e61d7b 100644 --- a/src/obikindex/src/index/rebuild_layer.rs +++ b/src/obikrebuild/src/rebuild_layer.rs @@ -6,15 +6,17 @@ use obicompactvec::{ }; use obidebruinj::GraphDeBruijn; use obikseq::CanonicalKmer; -use crate::layer::meta::PartitionMeta; -use crate::layer::{IndexMode, MphfLayer, layer_dir}; -use obiskio::{SKError, SKResult, UnitigFileReader}; +use obikindex::layer::meta::PartitionMeta; +use obikindex::layer::{IndexMode, MphfLayer}; +use obikindex::layer::utils::layer_dir; +use obiskio::UnitigFileReader; +use obikindex::{OKIError, OKIResult}; -use crate::index::common::{load_meta, olm_to_sk}; -use crate::index::filter::KmerFilter; -use crate::index::graph_pipeline::materialize_layer; -use crate::index::merge_layer::{MergeMode, SrcLayerData}; -use crate::index::kmer_index::KmerIndex; +use obikindex::load_meta; +use obikfilter::KmerFilter; +use obikindexer::materialize_layer; +use obikmerge::{MergeMode, SrcLayerData}; +use obikindex::KmerIndex; // ── Builders — pair matrix builder + column builders for one mode ───────────── @@ -27,22 +29,22 @@ enum Builders { } impl Builders { - fn new(mode: MergeMode, n: usize, dir: &Path, n_genomes: usize) -> SKResult { + fn new(mode: MergeMode, n: usize, dir: &Path, n_genomes: usize) -> OKIResult { match mode { MergeMode::Presence => { - let mut mat = PersistentBitMatrixBuilder::new(n, dir).map_err(SKError::Io)?; + let mut mat = PersistentBitMatrixBuilder::new(n, dir).map_err(OKIError::Io)?; let mut cols = Vec::with_capacity(n_genomes); for _ in 0..n_genomes { - cols.push(mat.add_col().map_err(SKError::Io)?); + cols.push(mat.add_col().map_err(OKIError::Io)?); } Ok(Builders::Presence(mat, cols)) } MergeMode::Count => { let mut mat = - PersistentCompactIntMatrixBuilder::new(n, dir).map_err(SKError::Io)?; + PersistentCompactIntMatrixBuilder::new(n, dir).map_err(OKIError::Io)?; let mut cols = Vec::with_capacity(n_genomes); for _ in 0..n_genomes { - cols.push(mat.add_col().map_err(SKError::Io)?); + cols.push(mat.add_col().map_err(OKIError::Io)?); } Ok(Builders::Count(mat, cols)) } @@ -56,19 +58,19 @@ impl Builders { } } - fn close(self) -> SKResult<()> { + fn close(self) -> OKIResult<()> { match self { Builders::Presence(mat, cols) => { for b in cols { - b.close().map_err(SKError::Io)?; + b.close().map_err(OKIError::Io)?; } - mat.close().map_err(SKError::Io) + mat.close().map_err(OKIError::Io) } Builders::Count(mat, cols) => { for b in cols { - b.close().map_err(SKError::Io)?; + b.close().map_err(OKIError::Io)?; } - mat.close().map_err(SKError::Io) + mat.close().map_err(OKIError::Io) } } } @@ -86,7 +88,7 @@ fn try_compute_combined_mask( filters: &[Box], src_data: &SrcLayerData, n_genomes: usize, -) -> SKResult> { +) -> OKIResult> { if filters.is_empty() { return Ok(None); } @@ -101,7 +103,7 @@ fn try_compute_combined_mask( let n = src_data.n_slots(); let mask = src_data .with_matrix(|mat| eval_filter_mask(&combined, mat, n)) - .map_err(SKError::Io)?; + .map_err(OKIError::Io)?; Ok(Some(mask)) } @@ -118,8 +120,8 @@ fn iter_src_kmers_masked( n_genomes: usize, filters: &[Box], mut cb: impl FnMut(CanonicalKmer), -) -> SKResult<()> { - let src_meta = load_meta(src_index_dir, "rebuild")?; +) -> OKIResult<()> { + let src_meta = load_meta(src_index_dir)?; for l in 0..src_meta.n_layers { let src_layer_dir = layer_dir(src_index_dir, l); let unitigs_path = src_layer_dir.join("unitigs.bin"); @@ -159,8 +161,8 @@ fn iter_src_layers( n_genomes: usize, filters: &[Box], mut cb: impl FnMut(CanonicalKmer, Box<[u32]>), -) -> SKResult<()> { - let src_meta = load_meta(src_index_dir, "rebuild")?; +) -> OKIResult<()> { + let src_meta = load_meta(src_index_dir)?; for l in 0..src_meta.n_layers { let src_layer_dir = layer_dir(src_index_dir, l); let unitigs_path = src_layer_dir.join("unitigs.bin"); @@ -209,13 +211,13 @@ impl KmerIndex { mode: MergeMode, n_genomes: usize, block_bits: u8, - ) -> SKResult<()> { + ) -> OKIResult<()> { let src_index_dir = src.index_dir(i); if !src_index_dir.exists() { return Ok(()); } - let src_meta = load_meta(&src_index_dir, "rebuild")?; + let src_meta = load_meta(&src_index_dir)?; if src_meta.n_layers == 0 { return Ok(()); } @@ -235,8 +237,7 @@ impl KmerIndex { let dst_layer_dir = self.layer_dir(i, 0); let n_new = materialize_layer(g, &dst_layer_dir, block_bits, &IndexMode::Exact)?; - let dst_mphf = MphfLayer::open(&dst_layer_dir) - .map_err(|e| olm_to_sk(e, "rebuild"))?; + let dst_mphf = MphfLayer::open(&dst_layer_dir)?; // ── Prepare matrix builders (one column per genome) ─────────────────── let data_dir = match mode { @@ -262,8 +263,7 @@ impl KmerIndex { n_layers: 1, mode: IndexMode::Exact, } - .save(&dst_index_dir) - .map_err(|e| olm_to_sk(e, "rebuild"))?; + .save(&dst_index_dir)?; Ok(()) } diff --git a/src/obikindex/src/index/reindex.rs b/src/obikrebuild/src/reindex.rs similarity index 88% rename from src/obikindex/src/index/reindex.rs rename to src/obikrebuild/src/reindex.rs index 03d084d4..0ede2cf2 100644 --- a/src/obikindex/src/index/reindex.rs +++ b/src/obikrebuild/src/reindex.rs @@ -1,24 +1,21 @@ -use crate::index::builder::IndexBuilder; -use crate::index::meta::IndexMeta; -use crate::layer::{IndexMode, TypedLayer}; +use obikindex::IndexBuilder; +use obikindex::IndexMeta; +use obikindex::layer::{IndexMode, TypedLayer}; use obisys::{Reporter, Stage, progress_bar}; use std::fs; use std::path::Path; use std::sync::Arc; use tracing::info; -use crate::index::error::{OKIError, OKIResult}; -use crate::index::kmer_index::KmerIndex; -use crate::index::state::IndexState; +use obikindex::{OKIError, OKIResult}; +use obikindex::KmerIndex; +use obikindex::IndexState; +use obisys::PartitionRunner; const EVIDENCE_FILE: &str = "evidence.bin"; const FINGERPRINT_FILE: &str = "fingerprint.bin"; const UNITIG_IDX_FILE: &str = "unitigs.bin.idx"; -fn olm_to_oki(e: crate::layer::OLMError) -> OKIError { - OKIError::InvalidInput(e.to_string()) -} - impl KmerIndex { /// Convert every layer's evidence bundle to `target` in-place. /// @@ -47,7 +44,7 @@ impl KmerIndex { let pb = progress_bar("reindex", n as u64, "partitions"); let order: Vec = (0..n).collect(); - let runner = crate::index::numa::PartitionRunner::new(); + let runner = PartitionRunner::new(); runner.run( &order, |i| { @@ -96,10 +93,10 @@ fn reindex_partition( fn reindex_layer(layer_dir: &Path, target: &IndexMode, block_bits: u8) -> OKIResult<()> { match target { IndexMode::Exact => { - TypedLayer::<()>::build_exact_evidence(layer_dir, block_bits).map_err(olm_to_oki)?; + TypedLayer::<()>::build_exact_evidence(layer_dir, block_bits)?; } IndexMode::Approx { b, z } | IndexMode::Hybrid { b, z } => { - TypedLayer::<()>::build_approx_evidence(layer_dir, *b, *z).map_err(olm_to_oki)?; + TypedLayer::<()>::build_approx_evidence(layer_dir, *b, *z)?; } } remove_stale_evidence(layer_dir, target) diff --git a/src/obikselect/Cargo.toml b/src/obikselect/Cargo.toml new file mode 100644 index 00000000..4efb38ec --- /dev/null +++ b/src/obikselect/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "obikselect" +version = "0.1.0" +edition = "2024" + +[dependencies] +obikindex = { path = "../obikindex" } +obicompactvec = { path = "../obicompactvec" } +obisys = { path = "../obisys" } +tracing = "0.1.44" diff --git a/src/obikselect/src/lib.rs b/src/obikselect/src/lib.rs new file mode 100644 index 00000000..30b7192b --- /dev/null +++ b/src/obikselect/src/lib.rs @@ -0,0 +1,11 @@ +//! Genome-column selection/aggregation on an `obikindex::KmerIndex`: +//! projecting/aggregating genome columns (`AggOp`) into new output columns +//! (`OutputCol`), either into a new index (`select`) or in place +//! (`select_in_place`). Orthogonal to per-kmer filtering (`obikfilter`), +//! which operates on rows, not columns. Kept as a separate crate — not a +//! module of `obikindex` — so the dependency runs one way only. + +mod select; +mod select_layer; + +pub use select_layer::{AggOp, OutputCol}; diff --git a/src/obikindex/src/index/select.rs b/src/obikselect/src/select.rs similarity index 91% rename from src/obikindex/src/index/select.rs rename to src/obikselect/src/select.rs index 01492b9a..ca00620b 100644 --- a/src/obikindex/src/index/select.rs +++ b/src/obikselect/src/select.rs @@ -1,15 +1,15 @@ use std::path::Path; use std::sync::Arc; -use crate::index::builder::IndexBuilder; -use crate::index::OutputCol; -use obisys::{Reporter, Stage, progress_bar}; +use obikindex::IndexBuilder; +use crate::OutputCol; +use obisys::{Reporter, Stage, progress_bar, PartitionRunner}; use tracing::info; -use crate::index::error::{OKIError, OKIResult}; -use crate::index::kmer_index::KmerIndex; -use crate::index::meta::{GenomeInfo, IndexMeta}; -use crate::index::state::IndexState; +use obikindex::{OKIError, OKIResult}; +use obikindex::KmerIndex; +use obikindex::{GenomeInfo, IndexMeta}; +use obikindex::IndexState; impl KmerIndex { /// Create a new index at `output` by projecting/aggregating the genome columns @@ -58,7 +58,7 @@ impl KmerIndex { let pb = progress_bar("select", n_partitions as u64, "partitions"); let order: Vec = (0..n_partitions).collect(); - let runner = crate::index::numa::PartitionRunner::new(); + let runner = PartitionRunner::new(); runner .run( &order, @@ -113,7 +113,7 @@ impl KmerIndex { let pb = progress_bar("select", n_partitions as u64, "partitions"); let order: Vec = (0..n_partitions).collect(); - let runner = crate::index::numa::PartitionRunner::new(); + let runner = PartitionRunner::new(); runner .run( &order, diff --git a/src/obikindex/src/index/select_layer.rs b/src/obikselect/src/select_layer.rs similarity index 85% rename from src/obikindex/src/index/select_layer.rs rename to src/obikselect/src/select_layer.rs index 3087538c..51c768da 100644 --- a/src/obikindex/src/index/select_layer.rs +++ b/src/obikselect/src/select_layer.rs @@ -6,10 +6,9 @@ use obicompactvec::{ ColGroup, MatrixGroupOps, PersistentBitMatrix, PersistentBitMatrixBuilder, PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder, }; -use crate::layer::OLMError; -use obiskio::{SKError, SKResult}; +use obikindex::{OKIError, OKIResult}; -use crate::index::kmer_index::KmerIndex; +use obikindex::KmerIndex; // ── AggOp ───────────────────────────────────────────────────────────────────── @@ -39,16 +38,6 @@ pub struct OutputCol { // ── Helpers ─────────────────────────────────────────────────────────────────── -fn olm_to_sk(e: OLMError) -> SKError { - match e { - OLMError::Io(e) => SKError::Io(e), - other => SKError::InvalidData { - context: "select", - detail: other.to_string(), - }, - } -} - /// Copy all plain files (not subdirectories) from `src_dir` to `dst_dir`. fn copy_layer_files(src_dir: &Path, dst_dir: &Path) -> io::Result<()> { for entry in fs::read_dir(src_dir)? { @@ -71,97 +60,97 @@ fn fill_builders( output_presence: bool, mut dst_bit: Option<&mut PersistentBitMatrixBuilder>, mut dst_int: Option<&mut PersistentCompactIntMatrixBuilder>, -) -> SKResult<()> { +) -> OKIResult<()> { if src_is_count { - let mat = PersistentCompactIntMatrix::open(src_layer_dir).map_err(SKError::Io)?; + let mat = PersistentCompactIntMatrix::open(src_layer_dir).map_err(OKIError::Io)?; for spec in specs { let g = ColGroup::new(&spec.label, spec.indices.clone()); if output_presence { let b = dst_bit.as_deref_mut().unwrap(); match spec.op { AggOp::Any => { - b.add_col_from(&mat.partial_group_any(&g, threshold).map_err(SKError::Io)?) + b.add_col_from(&mat.partial_group_any(&g, threshold).map_err(OKIError::Io)?) } AggOp::All => { - b.add_col_from(&mat.partial_group_all(&g, threshold).map_err(SKError::Io)?) + b.add_col_from(&mat.partial_group_all(&g, threshold).map_err(OKIError::Io)?) } AggOp::None => { - b.add_col_from(&mat.partial_group_none(&g, threshold).map_err(SKError::Io)?) + b.add_col_from(&mat.partial_group_none(&g, threshold).map_err(OKIError::Io)?) } AggOp::Sum => { - b.add_col_from_int(&mat.partial_group_sum(&g).map_err(SKError::Io)?) + b.add_col_from_int(&mat.partial_group_sum(&g).map_err(OKIError::Io)?) } AggOp::Min => { - b.add_col_from_int(&mat.partial_group_min(&g).map_err(SKError::Io)?) + b.add_col_from_int(&mat.partial_group_min(&g).map_err(OKIError::Io)?) } AggOp::Max => { - b.add_col_from_int(&mat.partial_group_max(&g).map_err(SKError::Io)?) + b.add_col_from_int(&mat.partial_group_max(&g).map_err(OKIError::Io)?) } } - .map_err(SKError::Io)?; + .map_err(OKIError::Io)?; } else { let b = dst_int.as_deref_mut().unwrap(); match spec.op { - AggOp::Sum => b.add_col_from(&mat.partial_group_sum(&g).map_err(SKError::Io)?), - AggOp::Min => b.add_col_from(&mat.partial_group_min(&g).map_err(SKError::Io)?), - AggOp::Max => b.add_col_from(&mat.partial_group_max(&g).map_err(SKError::Io)?), + AggOp::Sum => b.add_col_from(&mat.partial_group_sum(&g).map_err(OKIError::Io)?), + AggOp::Min => b.add_col_from(&mat.partial_group_min(&g).map_err(OKIError::Io)?), + AggOp::Max => b.add_col_from(&mat.partial_group_max(&g).map_err(OKIError::Io)?), AggOp::Any => b.add_col_from_bit( - &mat.partial_group_any(&g, threshold).map_err(SKError::Io)?, + &mat.partial_group_any(&g, threshold).map_err(OKIError::Io)?, ), AggOp::All => b.add_col_from_bit( - &mat.partial_group_all(&g, threshold).map_err(SKError::Io)?, + &mat.partial_group_all(&g, threshold).map_err(OKIError::Io)?, ), AggOp::None => b.add_col_from_bit( - &mat.partial_group_none(&g, threshold).map_err(SKError::Io)?, + &mat.partial_group_none(&g, threshold).map_err(OKIError::Io)?, ), } - .map_err(SKError::Io)?; + .map_err(OKIError::Io)?; } } } else { - let mat = PersistentBitMatrix::open(src_layer_dir).map_err(SKError::Io)?; + let mat = PersistentBitMatrix::open(src_layer_dir).map_err(OKIError::Io)?; for spec in specs { let g = ColGroup::new(&spec.label, spec.indices.clone()); if output_presence { let b = dst_bit.as_deref_mut().unwrap(); match spec.op { AggOp::Any => { - b.add_col_from(&mat.partial_group_any(&g, 1).map_err(SKError::Io)?) + b.add_col_from(&mat.partial_group_any(&g, 1).map_err(OKIError::Io)?) } AggOp::All => { - b.add_col_from(&mat.partial_group_all(&g, 1).map_err(SKError::Io)?) + b.add_col_from(&mat.partial_group_all(&g, 1).map_err(OKIError::Io)?) } AggOp::None => { - b.add_col_from(&mat.partial_group_none(&g, 1).map_err(SKError::Io)?) + b.add_col_from(&mat.partial_group_none(&g, 1).map_err(OKIError::Io)?) } AggOp::Sum => { - b.add_col_from_int(&mat.partial_group_sum(&g).map_err(SKError::Io)?) + b.add_col_from_int(&mat.partial_group_sum(&g).map_err(OKIError::Io)?) } AggOp::Min => { - b.add_col_from_int(&mat.partial_group_min(&g).map_err(SKError::Io)?) + b.add_col_from_int(&mat.partial_group_min(&g).map_err(OKIError::Io)?) } AggOp::Max => { - b.add_col_from_int(&mat.partial_group_max(&g).map_err(SKError::Io)?) + b.add_col_from_int(&mat.partial_group_max(&g).map_err(OKIError::Io)?) } } - .map_err(SKError::Io)?; + .map_err(OKIError::Io)?; } else { let b = dst_int.as_deref_mut().unwrap(); match spec.op { - AggOp::Sum => b.add_col_from(&mat.partial_group_sum(&g).map_err(SKError::Io)?), - AggOp::Min => b.add_col_from(&mat.partial_group_min(&g).map_err(SKError::Io)?), - AggOp::Max => b.add_col_from(&mat.partial_group_max(&g).map_err(SKError::Io)?), + AggOp::Sum => b.add_col_from(&mat.partial_group_sum(&g).map_err(OKIError::Io)?), + AggOp::Min => b.add_col_from(&mat.partial_group_min(&g).map_err(OKIError::Io)?), + AggOp::Max => b.add_col_from(&mat.partial_group_max(&g).map_err(OKIError::Io)?), AggOp::Any => { - b.add_col_from_bit(&mat.partial_group_any(&g, 1).map_err(SKError::Io)?) + b.add_col_from_bit(&mat.partial_group_any(&g, 1).map_err(OKIError::Io)?) } AggOp::All => { - b.add_col_from_bit(&mat.partial_group_all(&g, 1).map_err(SKError::Io)?) + b.add_col_from_bit(&mat.partial_group_all(&g, 1).map_err(OKIError::Io)?) } AggOp::None => { - b.add_col_from_bit(&mat.partial_group_none(&g, 1).map_err(SKError::Io)?) + b.add_col_from_bit(&mat.partial_group_none(&g, 1).map_err(OKIError::Io)?) } } - .map_err(SKError::Io)?; + .map_err(OKIError::Io)?; } } } @@ -185,7 +174,7 @@ impl KmerIndex { threshold: u32, output_presence: bool, in_place: bool, - ) -> SKResult<()> { + ) -> OKIResult<()> { let src_index_dir = src.index_dir(i); if !src_index_dir.exists() { return Ok(()); @@ -222,11 +211,11 @@ impl KmerIndex { // Determine number of slots and detect implicit layers. let n = if counts_dir.exists() { PersistentCompactIntMatrix::open(&src_layer_dir) - .map_err(SKError::Io)? + .map_err(OKIError::Io)? .n() } else if presence_dir.exists() { PersistentBitMatrix::open(&src_layer_dir) - .map_err(SKError::Io)? + .map_err(OKIError::Io)? .n() } else { // Implicit single-genome layer: no data matrix needed in output either. @@ -255,7 +244,7 @@ impl KmerIndex { let (mut dst_bit, mut dst_int) = if output_presence { ( - Some(PersistentBitMatrixBuilder::new(n, &dst_data_dir).map_err(SKError::Io)?), + Some(PersistentBitMatrixBuilder::new(n, &dst_data_dir).map_err(OKIError::Io)?), None, ) } else { @@ -263,7 +252,7 @@ impl KmerIndex { None, Some( PersistentCompactIntMatrixBuilder::new(n, &dst_data_dir) - .map_err(SKError::Io)?, + .map_err(OKIError::Io)?, ), ) }; @@ -279,9 +268,9 @@ impl KmerIndex { )?; if output_presence { - dst_bit.unwrap().close().map_err(SKError::Io)?; + dst_bit.unwrap().close().map_err(OKIError::Io)?; } else { - dst_int.unwrap().close().map_err(SKError::Io)?; + dst_int.unwrap().close().map_err(OKIError::Io)?; } // In-place: swap old data dir for new. @@ -300,8 +289,7 @@ impl KmerIndex { if !in_place { src.partition_meta(i)? - .save(&dst_index_dir) - .map_err(olm_to_sk)?; + .save(&dst_index_dir)?; } Ok(()) diff --git a/src/obikstats/Cargo.toml b/src/obikstats/Cargo.toml new file mode 100644 index 00000000..d302450a --- /dev/null +++ b/src/obikstats/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "obikstats" +version = "0.1.0" +edition = "2024" + +[dependencies] +obikindex = { path = "../obikindex" } +obicompactvec = { path = "../obicompactvec" } +rayon = "1" diff --git a/src/obikstats/src/lib.rs b/src/obikstats/src/lib.rs new file mode 100644 index 00000000..66baa672 --- /dev/null +++ b/src/obikstats/src/lib.rs @@ -0,0 +1,10 @@ +//! Read-only aggregation/statistics over an already-built +//! `obikindex::KmerIndex`: bits-per-kmer breakdown by index component +//! (`bits_per_kmer`), per-genome k-mer counts (`genome_kmer_counts`). +//! Not part of the `Index { Partition { Layer } }` data model itself — +//! kept out of `obikindex` like every other read/write extension +//! (`obikquery`, `obikdump`, `obikselect`, `obikrebuild`, `obikmerge`). + +mod stats; + +pub use stats::IndexBitsPerKmer; diff --git a/src/obikindex/src/index/stats.rs b/src/obikstats/src/stats.rs similarity index 93% rename from src/obikindex/src/index/stats.rs rename to src/obikstats/src/stats.rs index c506f213..d32fb25e 100644 --- a/src/obikindex/src/index/stats.rs +++ b/src/obikstats/src/stats.rs @@ -5,8 +5,8 @@ use obicompactvec::{LayerMeta, PersistentBitMatrix, PersistentCompactIntMatrix}; use obicompactvec::traits::ColumnWeights; use rayon::prelude::*; -use crate::index::error::{OKIError, OKIResult}; -use crate::index::kmer_index::KmerIndex; +use obikindex::{OKIError, OKIResult}; +use obikindex::KmerIndex; /// Bits per kmer broken down by index component. pub struct IndexBitsPerKmer { @@ -95,7 +95,8 @@ impl KmerIndex { let n_layers = self.n_layers(i).unwrap_or(0); (0..n_layers).fold((0usize, 0u64, 0u64, 0u64), |acc, l| { - let lb = layer_bytes(&self.layer_dir(i, l)); + let Ok(layer_dir) = self.layer_dir(i, l) else { return acc }; + let lb = layer_bytes(&layer_dir); (acc.0 + lb.n_kmers, acc.1 + lb.mphf, acc.2 + lb.evidence, acc.3 + lb.matrix) }) }) @@ -144,7 +145,7 @@ impl KmerIndex { let n_layers = self.n_layers(i).unwrap_or(0); for l in 0..n_layers { - let this_layer_dir = self.layer_dir(i, l); + let Ok(this_layer_dir) = self.layer_dir(i, l) else { continue }; if !this_layer_dir.exists() { continue; } n_kmers += LayerMeta::load(&this_layer_dir).map(|m| m.n).unwrap_or(0); @@ -153,12 +154,12 @@ impl KmerIndex { if this_layer_dir.join("counts").exists() && !this_layer_dir.join("presence").exists() { - match crate::layer::open_data::(&index_dir, l) { + match obikindex::layer::open_data::(&index_dir, l) { Ok(m) => Box::new(m), Err(_) => continue, } } else { - match crate::layer::open_data::(&index_dir, l) { + match obikindex::layer::open_data::(&index_dir, l) { Ok(m) => Box::new(m), Err(_) => continue, } diff --git a/src/obisys/Cargo.toml b/src/obisys/Cargo.toml index 8c41ad77..8952e948 100644 --- a/src/obisys/Cargo.toml +++ b/src/obisys/Cargo.toml @@ -4,7 +4,14 @@ version = "0.1.0" edition = "2024" [dependencies] -libc = "0.2" -sysinfo = "0.39" -indicatif = "0.18" -tracing = "0.1" +libc = "0.2" +sysinfo = "0.39" +indicatif = "0.18" +tracing = "0.1" +crossbeam-channel = "0.5" +rayon = "1" +hwlocality = { version = "1.0.0-alpha.11", features = ["vendored"], optional = true } + +[features] +default = ["numa"] +numa = ["hwlocality"] diff --git a/src/obisys/src/lib.rs b/src/obisys/src/lib.rs index e387cba5..a37cceff 100644 --- a/src/obisys/src/lib.rs +++ b/src/obisys/src/lib.rs @@ -3,12 +3,14 @@ mod budget; mod lock; +mod numa; mod progress; mod resources; mod stage; pub use budget::MemoryBudget; pub use lock::DirLock; +pub use numa::PartitionRunner; pub use progress::{Progress, TracedBar, progress_bar, spinner}; pub use resources::{CpuSample, IoSample, available_memory_bytes, effective_parallelism, peak_rss_bytes}; pub use stage::{Reporter, Stage, StageStats}; diff --git a/src/obikindex/src/index/numa/mod.rs b/src/obisys/src/numa/mod.rs similarity index 100% rename from src/obikindex/src/index/numa/mod.rs rename to src/obisys/src/numa/mod.rs diff --git a/src/obikindex/src/index/numa/runner.rs b/src/obisys/src/numa/runner.rs similarity index 99% rename from src/obikindex/src/index/numa/runner.rs rename to src/obisys/src/numa/runner.rs index 483fe675..ffe2db7e 100644 --- a/src/obikindex/src/index/numa/runner.rs +++ b/src/obisys/src/numa/runner.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use crossbeam_channel::unbounded; -use obisys::{CpuSample, IoSample}; +use crate::{CpuSample, IoSample}; use tracing::debug; use super::topology::{build, pin_current_thread}; diff --git a/src/obikindex/src/index/numa/topology.rs b/src/obisys/src/numa/topology.rs similarity index 97% rename from src/obikindex/src/index/numa/topology.rs rename to src/obisys/src/numa/topology.rs index 31c266cf..0263e90c 100644 --- a/src/obikindex/src/index/numa/topology.rs +++ b/src/obisys/src/numa/topology.rs @@ -66,7 +66,7 @@ pub fn build() -> NumaSetup { } // UMA fallback: single synthetic node, all cores, no pool, no pinning. - let n_cores = obisys::effective_parallelism(); + let n_cores = crate::effective_parallelism(); debug!("UMA: single synthetic node, {} core(s)", n_cores); NumaSetup { pools: vec![None], @@ -76,7 +76,7 @@ pub fn build() -> NumaSetup { #[cfg(not(feature = "numa"))] pub fn build() -> NumaSetup { - let n_cores = obisys::effective_parallelism(); + let n_cores = crate::effective_parallelism(); debug!("UMA: single synthetic node, {} core(s)", n_cores); NumaSetup { pools: vec![None],