OBIKmers/obikmer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 34 Wiki Activity

Extract index modules into specialized workspace subcrates

Browse Source 
This commit partitions the obikindex crate into multiple focused subcrates (obikfilter, obikmerge, obikquery, obikrebuild, obikselect, obikstats, obikdump, and obikidxcache) to reduce coupling and clarify module boundaries. It standardizes error handling across the workspace using OKIError and OKIResult, updates index APIs to support lazy, disk-backed partition access, and migrates NUMA system utilities to a new obisys crate. All modifications are structural, focusing on dependency graph expansion, import path updates, and API surface reorganization without altering core runtime behavior.
This commit is contained in:
Eric Coissac
2026-08-22 06:25:28 +02:00
parent c9d10d55c7
commit fc4464a0ef
81 changed files with 1640 additions and 1196 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Cargo.lock
Generated
+108 -2
View File
@@ -1519,6 +1519,14 @@ dependencies = [
name = "obikalgorithm"
version = "0.1.0"
[[package]]
name = "obikdump"
version = "0.1.0"
dependencies = [
"obikfilter",
"obikindex",
]
[[package]]
name = "obikentropy"
version = "0.1.0"
@@ -1526,6 +1534,27 @@ dependencies = [
"obikseq",
]
[[package]]
name = "obikfilter"
version = "0.1.0"
dependencies = [
"obicompactvec",
"obikindex",
"obikseq",
"obiskio",
"obitaxonomy",
]
[[package]]
name = "obikidxcache"
version = "0.1.0"
dependencies = [
"ndarray",
"obicompactvec",
"rayon",
"tempfile",
]
[[package]]
name = "obikindex"
version = "0.1.0"
@@ -1533,9 +1562,7 @@ dependencies = [
"anyhow",
"bitvec",
"cacheline-ef",
"crossbeam-channel",
"epserde",
"hwlocality",
"indicatif",
"memmap2",
"ndarray",
@@ -1597,11 +1624,13 @@ dependencies = [
"obidebruinj",
"obifastwrite",
"obikalgorithm",
"obikfilter",
"obikindex",
"obikindexer",
"obikphylo",
"obikrope",
"obikseq",
"obikstats",
"obipipeline",
"obiread",
"obiskbuilder",
@@ -1618,6 +1647,37 @@ dependencies = [
"tracing-subscriber",
]
[[package]]
name = "obikmer2"
version = "1.2.2"
dependencies = [
"clap",
"obikalgorithm",
"obikindex",
"obikindexer",
"obikseq",
"obipipeline",
"obiread",
"obisys",
"tracing",
"tracing-subscriber",
]
[[package]]
name = "obikmerge"
version = "0.1.0"
dependencies = [
"obicompactvec",
"obikfilter",
"obikindex",
"obikindexer",
"obikseq",
"obipipeline",
"obiskio",
"obisys",
"tracing",
]
[[package]]
name = "obikphylo"
version = "0.1.0"
@@ -1626,6 +1686,7 @@ dependencies = [
"ndarray",
"obicompactvec",
"obikalgorithm",
"obikidxcache",
"obikindex",
"obikindexer",
"obikseq",
@@ -1641,6 +1702,29 @@ dependencies = [
"tracing-subscriber",
]
[[package]]
name = "obikquery"
version = "0.1.0"
dependencies = [
"obikindex",
]
[[package]]
name = "obikrebuild"
version = "0.1.0"
dependencies = [
"obicompactvec",
"obidebruinj",
"obikfilter",
"obikindex",
"obikindexer",
"obikmerge",
"obikseq",
"obiskio",
"obisys",
"tracing",
]
[[package]]
name = "obikrope"
version = "0.1.0"
@@ -1649,6 +1733,16 @@ dependencies = [
"criterion2",
]
[[package]]
name = "obikselect"
version = "0.1.0"
dependencies = [
"obicompactvec",
"obikindex",
"obisys",
"tracing",
]
[[package]]
name = "obikseq"
version = "0.1.0"
@@ -1660,6 +1754,15 @@ dependencies = [
"xxhash-rust",
]
[[package]]
name = "obikstats"
version = "0.1.0"
dependencies = [
"obicompactvec",
"obikindex",
"rayon",
]
[[package]]
name = "obipipeline"
version = "0.1.0"
@@ -1712,8 +1815,11 @@ dependencies = [
name = "obisys"
version = "0.1.0"
dependencies = [
"crossbeam-channel",
"hwlocality",
"indicatif",
"libc",
"rayon",
"sysinfo",
"tracing",
]
src/Cargo.toml
+1 -1
View File
@@ -1,5 +1,5 @@
[workspace]
resolver = "3"
members = ["obikseq", "obiread", "obiskbuilder", "obifastwrite", "obikmer","obikrope","obipipeline", "obiskio","obidebruinj", "obicompactvec", "obisys", "obikindex", "obikindexer", "obitaxonomy", "obikentropy", "obikphylo", "obikalgorithm"]
members = ["obikseq", "obiread", "obiskbuilder", "obifastwrite", "obikmer","obikmer2","obikrope","obipipeline", "obiskio","obidebruinj", "obicompactvec", "obisys", "obikindex", "obikindexer", "obikquery", "obikdump", "obikfilter", "obikselect", "obikrebuild", "obikmerge", "obikstats", "obikidxcache", "obitaxonomy", "obikentropy", "obikphylo", "obikalgorithm"]
[profile.release]
debug = 1
src/obikdump/Cargo.toml
+8
View File
@@ -0,0 +1,8 @@
[package]
name = "obikdump"
version = "0.1.0"
edition = "2024"
[dependencies]
obikindex = { path = "../obikindex" }
obikfilter = { path = "../obikfilter" }
src/obikindex/src/index/dump.rs → src/obikdump/src/dump.rs
+3 -3
View File
@@ -3,9 +3,9 @@ use std::sync::atomic::{AtomicUsize, Ordering};
use rayon::prelude::*;
use crate::index::error::{OKIError, OKIResult};
use crate::index::kmer_index::KmerIndex;
use crate::index::KmerFilter;
use obikindex::{OKIError, OKIResult};
use obikindex::KmerIndex;
use obikfilter::KmerFilter;
impl KmerIndex {
/// Write a CSV table of all indexed kmers to `out`.
src/obikdump/src/lib.rs
+8
View File
@@ -0,0 +1,8 @@
//! Raw content export of an `obikindex::KmerIndex`: staging ground for
//! code currently living in `obikindex::index`'s dump path (`dump.rs`,
//! `dump_layer.rs`), to be migrated here to lighten that crate. Kept as
//! a separate crate — not a module of `obikindex` — so the dependency
//! runs one way only (dump code depends on the data model, never the
//! reverse), same pattern as `obikindexer`/`obikquery`.
mod dump;
src/obikfilter/Cargo.toml
+11
View File
@@ -0,0 +1,11 @@
[package]
name = "obikfilter"
version = "0.1.0"
edition = "2024"
[dependencies]
obikindex = { path = "../obikindex" }
obicompactvec = { path = "../obicompactvec" }
obikseq = { path = "../obikseq" }
obiskio = { path = "../obiskio" }
obitaxonomy = { path = "../obitaxonomy" }
src/obikindex/src/index/dump_layer.rs → src/obikfilter/src/dump_layer.rs
+14 -23
View File
@@ -1,20 +1,11 @@
use obikindex::layer::MphfLayer;
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
use obikseq::CanonicalKmer;
use crate::layer::{MphfLayer, OLMError};
use obiskio::{SKError, SKResult, UnitigFileReader};
use obiskio::UnitigFileReader;
use obikindex::{OKIError, OKIResult};
use crate::index::filter::{KmerFilter, passes_all};
use crate::index::kmer_index::KmerIndex;
fn olm_to_sk(e: OLMError) -> SKError {
match e {
OLMError::Io(e) => SKError::Io(e),
other => SKError::InvalidData {
context: "dump",
detail: other.to_string(),
},
}
}
use crate::filter::{KmerFilter, passes_all};
use obikindex::KmerIndex;
impl KmerIndex {
/// Iterate all indexed kmers in partition `part`, calling `cb(kmer, row)` for each
@@ -35,7 +26,7 @@ impl KmerIndex {
n_genomes: usize,
filters: &[Box<dyn KmerFilter>],
mut cb: impl FnMut(CanonicalKmer, Box<[u32]>) -> bool,
) -> SKResult<bool> {
) -> OKIResult<bool> {
let index_dir = self.index_dir(part);
if !index_dir.exists() {
return Ok(true);
@@ -43,19 +34,19 @@ impl KmerIndex {
let mut l = 0;
loop {
let layer_dir = self.layer_dir(part, l);
let layer_dir = self.layer_dir(part, l)?;
if !layer_dir.exists() {
break;
}
l += 1;
let mphf = MphfLayer::open(&layer_dir).map_err(olm_to_sk)?;
let mphf = MphfLayer::open(&layer_dir)?;
let reader = UnitigFileReader::open_sequential(&layer_dir.join("unitigs.bin"))?;
let counts_dir = layer_dir.join("counts");
let presence_dir = layer_dir.join("presence");
let cont = if use_counts && counts_dir.exists() {
let mat = PersistentCompactIntMatrix::open(&layer_dir).map_err(SKError::Io)?;
let mat = PersistentCompactIntMatrix::open(&layer_dir).map_err(OKIError::Io)?;
let mut cont = true;
for (kmer, _, _) in reader.iter_indexed_canonical_kmers() {
if let Some(slot) = mphf.find(kmer) {
@@ -70,7 +61,7 @@ impl KmerIndex {
}
cont
} else if !use_counts && presence_dir.exists() {
let mat = PersistentBitMatrix::open(&layer_dir).map_err(SKError::Io)?;
let mat = PersistentBitMatrix::open(&layer_dir).map_err(OKIError::Io)?;
let mut cont = true;
for (kmer, _, _) in reader.iter_indexed_canonical_kmers() {
if let Some(slot) = mphf.find(kmer) {
@@ -123,7 +114,7 @@ impl KmerIndex {
n_genomes: usize,
filters: &[Box<dyn KmerFilter>],
mut cb: impl FnMut(usize, usize, CanonicalKmer, Box<[u32]>) -> bool,
) -> SKResult<bool> {
) -> OKIResult<bool> {
let index_dir = self.index_dir(part);
if !index_dir.exists() {
return Ok(true);
@@ -135,14 +126,14 @@ impl KmerIndex {
if !layer_dir.exists() {
break;
}
let mphf = MphfLayer::open(&layer_dir).map_err(olm_to_sk)?;
let mphf = MphfLayer::open(&layer_dir)?;
let reader = UnitigFileReader::open_sequential(&layer_dir.join("unitigs.bin"))?;
let counts_dir = layer_dir.join("counts");
let presence_dir = layer_dir.join("presence");
let cont = if use_counts && counts_dir.exists() {
let mat = PersistentCompactIntMatrix::open(&layer_dir).map_err(SKError::Io)?;
let mat = PersistentCompactIntMatrix::open(&layer_dir).map_err(OKIError::Io)?;
let mut cont = true;
for (kmer, _, _) in reader.iter_indexed_canonical_kmers() {
if let Some(slot) = mphf.find(kmer) {
@@ -157,7 +148,7 @@ impl KmerIndex {
}
cont
} else if !use_counts && presence_dir.exists() {
let mat = PersistentBitMatrix::open(&layer_dir).map_err(SKError::Io)?;
let mat = PersistentBitMatrix::open(&layer_dir).map_err(OKIError::Io)?;
let mut cont = true;
for (kmer, _, _) in reader.iter_indexed_canonical_kmers() {
if let Some(slot) = mphf.find(kmer) {
src/obikindex/src/index/filter.rs → src/obikfilter/src/filter.rs
View File
src/obikfilter/src/lib.rs
+22
View File
@@ -0,0 +1,22 @@
//! Per-kmer filtering: `KmerFilter` judges one k-mer (its canonical
//! sequence + its per-genome row) at a time, deciding whether it passes.
//! Orthogonal to genome-column selection/aggregation (`obikselect`), which
//! operates on already-retained k-mers.
//!
//! [`filter`] (the `KmerFilter` trait + its implementations) depends only
//! on `obicompactvec`/`obikseq`, not on `obikindex`. [`dump_layer`] is the
//! extension over `obikindex::KmerIndex` that actually iterates a
//! partition's k-mers through those filters (`iter_partition_kmers`,
//! `iter_partition_kmers_located`) — every k-mer, filtered or not, goes
//! through this same path (`passes_all` on an empty filter list is always
//! `true`), so this crate depends one-way on `obikindex`, not the reverse.
mod filter;
mod dump_layer;
mod predicate;
pub use filter::{
GroupQuorumFilter, KmerFilter, MaxGenomeCount, MaxGenomeFraction, MaxTotalCount,
MinComplexity, MinGenomeCount, MinGenomeFraction, MinTotalCount, passes_all,
};
pub use predicate::{GroupFilterParams, MetaPred};
src/obikindex/src/index/predicate.rs → src/obikfilter/src/predicate.rs
+2 -2
View File
@@ -1,9 +1,9 @@
use std::collections::HashMap;
use crate::index::GroupQuorumFilter;
use crate::filter::GroupQuorumFilter;
use obitaxonomy::{TaxPath, TaxPattern};
use crate::index::meta::{GenomeInfo, IndexMeta};
use obikindex::{GenomeInfo, IndexMeta};
// ── Operator ──────────────────────────────────────────────────────────────────
src/obikidxcache/Cargo.toml
+12
View File
@@ -0,0 +1,12 @@
[package]
name = "obikidxcache"
version = "0.1.0"
edition = "2024"
[dependencies]
obicompactvec = { path = "../obicompactvec" }
ndarray = "0.17"
rayon = "1"
[dev-dependencies]
tempfile = "3"
src/obikindex/src/layer/layered_store.rs → src/obikidxcache/src/layered_store.rs
View File
src/obikidxcache/src/lib.rs
+11
View File
@@ -0,0 +1,11 @@
//! Caching/aggregation over already-opened `obikindex` layer/partition
//! objects — a runtime cache shape, not part of the stateless
//! `Index { Partition { Layer } }` directory-structure model itself.
//! [`LayeredStore`] wraps one entry per layer (or per partition, via
//! `LayeredStore<LayeredStore<S>>`) and propagates `ColumnWeights`/
//! `CountPartials`/`BitPartials` aggregation across them. No dependency on
//! `obikindex` — generic over any `S` implementing those traits.
mod layered_store;
pub use layered_store::LayeredStore;
src/obikindex/src/layer/tests/layered_store.rs → src/obikidxcache/src/tests/layered_store.rs
View File
src/obikindex/Cargo.toml
-6
View File
@@ -19,13 +19,11 @@ niffler = "3.0.0"
memmap2 = "0.9.10"
ndarray = "0.17"
rayon = "1"
crossbeam-channel = "0.5"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
indicatif = "0.18"
tracing = "0.1.44"
bitvec = "1"
hwlocality = { version = "1.0.0-alpha.11", features = ["vendored"], optional = true }
[dev-dependencies]
obiread = { path = "../obiread" }
@@ -33,7 +31,3 @@ obikseq = { path = "../obikseq", features = ["test-utils"] }
tempfile = "3"
tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] }
anyhow = "1"
[features]
default = ["numa"]
numa = ["hwlocality"]
src/obikindex/src/index/builder.rs
+15 -4
View File
@@ -44,7 +44,11 @@ pub trait IndexBuilder: Sized {
/// For construction paths that build partitions from scratch (`select`,
/// `rebuild`). `merge` bootstraps by copying a source index instead, so
/// it does not use this.
fn create_skeleton<P: AsRef<Path>>(output: P, config: IndexConfig, genomes: Vec<GenomeInfo>) -> OKIResult<Self>;
fn create_skeleton<P: AsRef<Path>>(
output: P,
config: IndexConfig,
genomes: Vec<GenomeInfo>,
) -> OKIResult<Self>;
/// Mark `output` as fully indexed, pack its column matrices, and reopen it.
///
@@ -61,7 +65,7 @@ pub trait IndexBuilder: Sized {
impl IndexBuilder for KmerIndex {
fn clear_output_for_create<P: AsRef<Path>>(output: P, force: bool) -> OKIResult<()> {
let output = output.as_ref();
if Self::exists(output) {
if Self::is_an_index(output) {
if force {
fs::remove_dir_all(output).map_err(OKIError::Io)?;
} else {
@@ -74,11 +78,18 @@ impl IndexBuilder for KmerIndex {
Ok(())
}
fn create_skeleton<P: AsRef<Path>>(output: P, config: IndexConfig, genomes: Vec<GenomeInfo>) -> OKIResult<Self> {
fn create_skeleton<P: AsRef<Path>>(
output: P,
config: IndexConfig,
genomes: Vec<GenomeInfo>,
) -> OKIResult<Self> {
let output = output.as_ref();
fs::create_dir_all(output).map_err(OKIError::Io)?;
let meta = IndexMeta::create_at(output, config, genomes).map_err(OKIError::Io)?;
Ok(KmerIndex { root_path: output.to_owned(), meta: Arc::new(meta) })
Ok(KmerIndex {
root_path: output.to_owned(),
meta: Arc::new(meta),
})
}
fn finalize_indexed<P: AsRef<Path>>(output: P, rep: &mut Reporter) -> OKIResult<Self> {
src/obikindex/src/index/common.rs
+6 -46
View File
@@ -1,65 +1,25 @@
use std::path::Path;
use crate::index::error::{OKIError, OKIResult};
use obicompactvec::{PersistentBitVecBuilder, PersistentCompactIntVecBuilder};
use crate::layer::meta::PartitionMeta;
use crate::layer::{layer_dir, IndexMode, OLMError};
use obiskio::{SKError, SKResult};
// ── olm_to_sk ────────────────────────────────────────────────────────────────
pub fn olm_to_sk(e: OLMError, context: &'static str) -> SKError {
match e {
OLMError::Io(e) => SKError::Io(e),
other => SKError::InvalidData {
context,
detail: other.to_string(),
},
}
}
// ── load_meta ────────────────────────────────────────────────────────────────
/// Load PartitionMeta, or recover it by probing layer directories.
/// Indexes built before meta.json was introduced lack the file.
pub(crate) fn load_meta(dir: &Path, context: &'static str) -> SKResult<PartitionMeta> {
match PartitionMeta::load(dir) {
Ok(m) => Ok(m),
Err(e) if matches!(e, OLMError::Io(ref io_e) if io_e.kind() == std::io::ErrorKind::NotFound) =>
{
let mut n = 0usize;
while layer_dir(dir, n).exists() {
n += 1;
}
let m = PartitionMeta {
n_layers: n,
mode: IndexMode::default(),
};
m.save(dir).map_err(|e| olm_to_sk(e, context))?;
Ok(m)
}
Err(e) => Err(olm_to_sk(e, context)),
}
}
// ── ColBuilder ────────────────────────────────────────────────────────────────
pub(crate) enum ColBuilder {
pub enum ColBuilder {
Bit(PersistentBitVecBuilder),
Int(PersistentCompactIntVecBuilder),
}
impl ColBuilder {
pub(crate) fn set_val(&mut self, slot: usize, value: u32) {
pub fn set_val(&mut self, slot: usize, value: u32) {
match self {
ColBuilder::Bit(b) => b.set(slot, value > 0),
ColBuilder::Int(b) => b.set(slot, value),
}
}
pub(crate) fn close(self) -> SKResult<()> {
pub fn close(self) -> OKIResult<()> {
match self {
ColBuilder::Bit(b) => b.close().map_err(SKError::Io),
ColBuilder::Int(b) => b.close().map_err(SKError::Io),
ColBuilder::Bit(b) => b.close().map_err(OKIError::Io),
ColBuilder::Int(b) => b.close().map_err(OKIError::Io),
}
}
}
src/obikindex/src/index/error.rs
+33 -14
View File
@@ -8,6 +8,7 @@ pub enum OKIError {
Io(io::Error),
Json(serde_json::Error),
Partition(SKError),
Layer(SKError),
/// Source index is not in `Indexed` state.
NotIndexed(std::path::PathBuf),
/// Source indexes have incompatible configurations (k, m, n_bits).
@@ -20,6 +21,9 @@ pub enum OKIError {
InvalidInput(String),
/// Sources mix exact and approximate evidence, or use incompatible approx parameters.
IncompatibleEvidence(String),
/// Malformed data encountered while reading/writing a layer file, tagged
/// with the operation that hit it.
InvalidData { context: &'static str, detail: String },
}
pub type OKIResult<T> = Result<T, OKIError>;
@@ -27,15 +31,24 @@ pub type OKIResult<T> = Result<T, OKIError>;
impl fmt::Display for OKIError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
OKIError::Io(e) => write!(f, "I/O error: {e}"),
OKIError::Json(e) => write!(f, "JSON error: {e}"),
OKIError::Partition(e) => write!(f, "partition error: {e}"),
OKIError::NotIndexed(p) => write!(f, "index not fully built: {}", p.display()),
OKIError::Io(e) => write!(f, "I/O error: {e}"),
OKIError::Json(e) => write!(f, "JSON error: {e}"),
OKIError::Partition(e) => write!(f, "partition error: {e}"),
OKIError::Layer(e) => write!(f, "layer error: {e}"),
OKIError::NotIndexed(p) => write!(f, "index not fully built: {}", p.display()),
OKIError::IncompatibleConfig => write!(f, "incompatible index configurations"),
OKIError::MismatchedMode => write!(f, "count mode requires all sources to have with_counts=true"),
OKIError::DuplicateGenomeLabel(l) => write!(f, "duplicate genome label across sources: {l}"),
OKIError::InvalidInput(m) => write!(f, "invalid input: {m}"),
OKIError::IncompatibleEvidence(m) => write!(f, "incompatible evidence: {m}"),
OKIError::MismatchedMode => write!(
f,
"count mode requires all sources to have with_counts=true"
),
OKIError::DuplicateGenomeLabel(l) => {
write!(f, "duplicate genome label across sources: {l}")
}
OKIError::InvalidInput(m) => write!(f, "invalid input: {m}"),
OKIError::IncompatibleEvidence(m) => write!(f, "incompatible evidence: {m}"),
OKIError::InvalidData { context, detail } => {
write!(f, "invalid data ({context}): {detail}")
}
}
}
}
@@ -43,22 +56,28 @@ impl fmt::Display for OKIError {
impl std::error::Error for OKIError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
match self {
OKIError::Io(e) => Some(e),
OKIError::Json(e) => Some(e),
OKIError::Io(e) => Some(e),
OKIError::Json(e) => Some(e),
OKIError::Partition(e) => Some(e),
_ => None, // IncompatibleConfig, MismatchedMode, DuplicateGenomeLabel
_ => None, // IncompatibleConfig, MismatchedMode, DuplicateGenomeLabel
}
}
}
impl From<io::Error> for OKIError {
fn from(e: io::Error) -> Self { OKIError::Io(e) }
fn from(e: io::Error) -> Self {
OKIError::Io(e)
}
}
impl From<serde_json::Error> for OKIError {
fn from(e: serde_json::Error) -> Self { OKIError::Json(e) }
fn from(e: serde_json::Error) -> Self {
OKIError::Json(e)
}
}
impl From<SKError> for OKIError {
fn from(e: SKError) -> Self { OKIError::Partition(e) }
fn from(e: SKError) -> Self {
OKIError::Partition(e)
}
}
src/obikindex/src/index/kmer_index.rs
+76 -43
View File
@@ -2,13 +2,13 @@ use std::fs;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use crate::layer::meta::PartitionMeta;
use crate::layer::KmerLayer;
use crate::partition::KmerPartition;
use obisys::progress_bar;
use rayon::prelude::*;
use obikseq::{set_k, set_m};
use crate::index::common::load_meta;
use crate::index::error::{OKIError, OKIResult};
use crate::index::meta::{GenomeInfo, IndexConfig, IndexMeta};
@@ -28,12 +28,15 @@ impl KmerIndex {
genome_info: Option<GenomeInfo>,
) -> OKIResult<Self> {
let root_path = path.as_ref().to_owned();
fs::create_dir_all(&root_path).map_err(OKIError::Io)?;
fs::create_dir(&root_path).map_err(OKIError::Io)?;
set_k(config.kmer_size);
set_m(config.minimizer_size);
let genomes = genome_info.into_iter().collect();
let meta = IndexMeta::create_at(&root_path, config, genomes).map_err(OKIError::Io)?;
Ok(Self { root_path, meta: Arc::new(meta) })
Ok(Self {
root_path,
meta: Arc::new(meta),
})
}
pub fn open<P: AsRef<Path>>(path: P) -> OKIResult<Self> {
@@ -41,16 +44,30 @@ impl KmerIndex {
let meta = IndexMeta::open_at(&root_path).map_err(OKIError::Io)?;
set_k(meta.config.kmer_size);
set_m(meta.config.minimizer_size);
Ok(Self { root_path, meta: Arc::new(meta) })
Ok(Self {
root_path,
meta: Arc::new(meta),
})
}
/// Return `true` if `path` contains an `index.meta` file.
pub fn exists<P: AsRef<Path>>(path: P) -> bool {
/// Return `true` if `path` points to a valid obikmer index directory.
///
/// A directory is considered an index only when it contains an `index.meta`
/// file. The absence of that file means the directory is not an index, even
/// if it otherwise exists.
///
/// This is a static method (associated function), not an instance method.
/// Call it as `KmerIndex::is_an_index(path)`.
///
/// `path` is generic over any type implementing [`AsRef<Path>`]. This allows
/// callers to pass `&str`, `String`, `PathBuf`, `&Path`, etc. without an
/// explicit conversion; the value is coerced via `.as_ref()` inside.
pub fn is_an_index<P: AsRef<Path>>(path: P) -> bool {
IndexMeta::exists(path.as_ref())
}
/// The index's root directory.
pub fn root_path(&self) -> &Path {
pub fn dir(&self) -> &Path {
&self.root_path
}
pub fn meta(&self) -> Arc<IndexMeta> {
@@ -102,42 +119,48 @@ impl KmerIndex {
/// same directory only indirectly, through this method — `obikindexer`
/// depends on `KmerIndex`, not the other way around — see
/// `DevDocMD/implementation/partition_layer_cache.md`.
pub fn partition_dir(&self, i: usize) -> PathBuf {
crate::partition::partition_dir(&self.root_path, i)
pub(crate) fn partition_dir(&self, i: usize) -> OKIResult<PathBuf> {
Ok(self.partition(i)?.dir().to_path_buf())
}
pub fn partition(&self, i: usize) -> OKIResult<KmerPartition> {
let n = self.n_partitions();
if i >= n {
return Err(OKIError::InvalidData {
context: "New KmerPartition",
detail: format!("Partition {i} requested, Index has only {n} partitions"),
});
}
Ok(KmerPartition::new(self, i))
}
pub(crate) fn layer_dir(&self, i: usize, j: usize) -> OKIResult<PathBuf> {
Ok(self.layer(i, j)?.dir().to_path_buf())
}
pub fn layer(&self, i: usize, j: usize) -> OKIResult<KmerLayer> {
self.partition(i)?.layer(j)
}
/// Path of partition `i`'s layered-index directory (`<partition>/index`).
pub fn index_dir(&self, i: usize) -> PathBuf {
pub(crate) fn index_dir(&self, i: usize) -> PathBuf {
crate::partition::index_dir(&self.root_path, i)
}
/// Path of layer `l` within partition `i`'s layered index.
pub fn layer_dir(&self, i: usize, l: usize) -> PathBuf {
crate::layer::layer_dir(&self.index_dir(i), l)
/// Number of layers in partition `i` — a disk scan, not a metadata read
/// (see [`KmerPartition::n_layers`](crate::partition::KmerPartition::n_layers)).
pub fn n_layers(&self, i: usize) -> OKIResult<usize> {
Ok(self.partition(i)?.n_layers())
}
/// Partition `i`'s metadata (layer count, evidence mode). Returns
/// `obiskio::SKResult`, not `OKIResult` — matches the error convention
/// of the partition/layer-construction code below (moved here from
/// the former `obikpartitionner` crate, which predates `OKIError`);
/// `?` still converts it to `OKIResult` at any call site that needs
/// one (`OKIError: From<SKError>`).
pub fn partition_meta(&self, i: usize) -> obiskio::SKResult<PartitionMeta> {
load_meta(&self.index_dir(i), "partition_meta")
}
/// Number of layers in partition `i` — see [`partition_meta`](Self::partition_meta).
pub fn n_layers(&self, i: usize) -> obiskio::SKResult<usize> {
Ok(self.partition_meta(i)?.n_layers)
}
/// Evidence mode partition `i` was actually built with — see
/// [`partition_meta`](Self::partition_meta). Distinct from
/// [`evidence_mode`](Self::evidence_mode): that one is the index-level
/// config, this one is per-partition ground truth (the two agree in
/// practice, but this is what `Layer::open` needs).
pub fn partition_mode(&self, i: usize) -> obiskio::SKResult<crate::layer::IndexMode> {
Ok(self.partition_meta(i)?.mode)
/// Evidence mode partition `i` was actually built with — detected from
/// layer 0's evidence files on disk (see
/// [`KmerPartition::mode`](crate::partition::KmerPartition::mode)).
/// Distinct from [`evidence_mode`](Self::evidence_mode): that one is the
/// index-level config, this one is per-partition ground truth (the two
/// agree in practice, but this is what `Layer::open` needs).
pub fn partition_mode(&self, i: usize) -> OKIResult<crate::layer::IndexMode> {
self.partition(i)?.mode()
}
/// Number of layers per partition.
@@ -150,8 +173,8 @@ impl KmerIndex {
}
/// Path to the unitigs file for partition `part`, layer `layer`.
pub fn layer_unitigs_path(&self, part: usize, layer: usize) -> PathBuf {
self.layer_dir(part, layer).join("unitigs.bin")
pub fn layer_unitigs_path(&self, part: usize, layer: usize) -> OKIResult<PathBuf> {
Ok(self.layer_dir(part, layer)?.join("unitigs.bin"))
}
/// Pack all partition matrices into single-file format (presence → .pbmx, counts → .pcmx).
@@ -164,13 +187,14 @@ impl KmerIndex {
/// on-disk format (see `DevDocMD/architecture/siblings.md`) — count
/// matrices are unaffected, sparse count matrices aren't implemented
/// (see the sparse-matrix design plan's "Explicitly deferred").
pub fn pack_matrices(&self, sparse: bool) -> OKIResult<()> {
/// TODO: To be moved somewhere else
pub(crate) fn pack_matrices(&self, sparse: bool) -> OKIResult<()> {
use obicompactvec::{pack_bit_matrix, pack_compact_int_matrix, pack_sparse_bit_matrix};
let n = self.n_partitions();
let order: Vec<usize> = (0..n).collect();
let pb = progress_bar("pack", n as u64, "partitions");
crate::index::numa::PartitionRunner::new().run(
obisys::PartitionRunner::new().run(
&order,
|i| -> OKIResult<()> {
let index_dir = self.index_dir(i);
@@ -179,7 +203,7 @@ impl KmerIndex {
}
let n_layers = self.n_layers(i)?;
for l in 0..n_layers {
let layer_dir = self.layer_dir(i, l);
let layer_dir = self.layer_dir(i, l)?;
let presence_dir = layer_dir.join("presence");
let counts_dir = layer_dir.join("counts");
if presence_dir.exists() {
@@ -207,7 +231,8 @@ impl KmerIndex {
///
/// Old indexes were built before this file was required. The number of
/// kmers is recovered from `unitigs.bin`, which is always present.
pub fn upgrade_layer_meta(&self) -> OKIResult<()> {
/// TODO: Should not exist anymore
pub(crate) fn upgrade_layer_meta(&self) -> OKIResult<()> {
use obicompactvec::LayerMeta;
use obiskio::UnitigFileReader;
@@ -229,7 +254,15 @@ impl KmerIndex {
}
};
for l in 0..n_layers {
let layer_dir = self.layer_dir(i, l);
let layer_dir = match self.layer_dir(i, l) {
Ok(d) => d,
Err(e) => {
return Some(OKIError::Io(std::io::Error::new(
std::io::ErrorKind::Other,
e.to_string(),
)));
}
};
let meta_path = layer_dir.join(LayerMeta::FILENAME);
if meta_path.exists() {
continue;
src/obikindex/src/index/meta.rs
+71 -35
View File
@@ -21,8 +21,44 @@ pub struct GenomeInfo {
}
impl GenomeInfo {
/// Panics if `label` fails [`validate_label`](Self::validate_label) —
/// an invalid genome label is a programmer error to surface loudly,
/// not a recoverable condition for this constructor to paper over.
/// Callers that need a graceful error (e.g. a CLI parsing user input)
/// should call [`validate_label`](Self::validate_label) themselves
/// first and report it their own way.
pub fn new(label: impl Into<String>) -> Self {
Self { label: label.into(), meta: HashMap::new() }
let label = label.into();
Self::validate_label(&label).expect("invalid genome label");
Self {
label,
meta: HashMap::new(),
}
}
/// Validate a user-supplied genome label.
///
/// Forbidden: `/` (filesystem separator), `=` (--new-label parser separator),
/// `\0` (null byte), `\n`, `\r`, `\t` (break CSV output).
/// Empty labels are also rejected.
pub fn validate_label(label: &str) -> Result<(), String> {
if label.is_empty() {
return Err("genome label must not be empty".into());
}
const FORBIDDEN: &[char] = &['/', '=', '\0', '\n', '\r', '\t'];
if let Some(c) = label.chars().find(|c| FORBIDDEN.contains(c)) {
let display = match c {
'\0' => "\\0 (null)".to_string(),
'\n' => "\\n (newline)".to_string(),
'\r' => "\\r (carriage return)".to_string(),
'\t' => "\\t (tab)".to_string(),
c => format!("'{c}'"),
};
return Err(format!(
"genome label contains forbidden character {display}"
));
}
Ok(())
}
}
@@ -47,7 +83,7 @@ pub struct IndexConfig {
/// On-disk shape of `index.meta` — the whole file, read/written atomically
/// as one JSON blob by every `IndexMeta` operation below.
#[derive(Debug, Clone, Serialize, Deserialize)]
struct OnDiskMeta {
struct IndexMetadata {
version: u32,
config: IndexConfig,
#[serde(default)]
@@ -81,21 +117,29 @@ impl IndexMeta {
/// Create a brand-new `index.meta` for `index`, from fixed config and
/// an initial genome list (often empty — genomes are usually added
/// later via [`push_genome`](Self::push_genome)).
pub fn create(index: &KmerIndex, config: IndexConfig, genomes: Vec<GenomeInfo>) -> io::Result<Self> {
Self::create_at(index.root_path(), config, genomes)
pub fn create(
index: &KmerIndex,
config: IndexConfig,
genomes: Vec<GenomeInfo>,
) -> io::Result<Self> {
Self::create_at(index.dir(), config, genomes)
}
/// Same as [`create`](Self::create), taking a raw path — for
/// `KmerIndex::create` itself, which doesn't have a complete
/// `KmerIndex` yet to hand in (it's what's being built).
pub(crate) fn create_at(root_path: &Path, config: IndexConfig, genomes: Vec<GenomeInfo>) -> io::Result<Self> {
pub(crate) fn create_at(
root_path: &Path,
config: IndexConfig,
genomes: Vec<GenomeInfo>,
) -> io::Result<Self> {
let meta = Self {
root_path: root_path.to_owned(),
version: META_VERSION,
config,
lock: RwLock::new(()),
};
meta.write_full(&OnDiskMeta {
meta.write_full(&IndexMetadata {
version: meta.version,
config: meta.config.clone(),
genomes,
@@ -106,7 +150,7 @@ impl IndexMeta {
/// Reopen the `index.meta` already on disk for `index`.
pub fn open(index: &KmerIndex) -> io::Result<Self> {
Self::open_at(index.root_path())
Self::open_at(index.dir())
}
pub(crate) fn open_at(root_path: &Path) -> io::Result<Self> {
@@ -119,7 +163,12 @@ impl IndexMeta {
})
}
pub fn exists(root: &Path) -> bool {
/// Returns whether the metadata file exists in `root`.
///
/// This is a static method (associated function), not an instance method.
/// It only checks for the presence of the `index.meta` file inside the
/// given directory; it does not validate its contents.
pub(crate) fn exists(root: &Path) -> bool {
root.join(META_FILENAME).exists()
}
@@ -177,10 +226,20 @@ impl IndexMeta {
/// (e.g. `self.meta = Arc::new(IndexMeta::open(self)?)`) — this method
/// only updates the file, it has no way to reach back into whatever
/// `KmerIndex` holds it.
pub fn rewrite_config(&self, config: IndexConfig, genomes: Vec<GenomeInfo>) -> io::Result<()> {
/// TODO: This methode is strange
pub(crate) fn rewrite_config(
&self,
config: IndexConfig,
genomes: Vec<GenomeInfo>,
) -> io::Result<()> {
let _guard = self.lock.write().unwrap();
let state = Self::read_full(&self.root_path)?.state;
self.write_full(&OnDiskMeta { version: self.version, config, genomes, state })
self.write_full(&IndexMetadata {
version: self.version,
config,
genomes,
state,
})
}
pub fn set_state(&self, state: IndexState) -> io::Result<()> {
@@ -221,12 +280,12 @@ impl IndexMeta {
// ── private ──────────────────────────────────────────────────────────────
fn read_full(root: &Path) -> io::Result<OnDiskMeta> {
fn read_full(root: &Path) -> io::Result<IndexMetadata> {
let file = fs::File::open(root.join(META_FILENAME))?;
serde_json::from_reader(file).map_err(io::Error::other)
}
fn write_full(&self, on_disk: &OnDiskMeta) -> io::Result<()> {
fn write_full(&self, on_disk: &IndexMetadata) -> io::Result<()> {
let file = fs::File::create(self.root_path.join(META_FILENAME))?;
serde_json::to_writer_pretty(file, on_disk).map_err(io::Error::other)
}
@@ -248,26 +307,3 @@ fn label_from_path(path: &Path) -> String {
s
}
}
/// Validate a user-supplied genome label.
///
/// Forbidden: `/` (filesystem separator), `=` (--new-label parser separator),
/// `\0` (null byte), `\n`, `\r`, `\t` (break CSV output).
/// Empty labels are also rejected.
pub fn validate_label(label: &str) -> Result<(), String> {
if label.is_empty() {
return Err("genome label must not be empty".into());
}
const FORBIDDEN: &[char] = &['/', '=', '\0', '\n', '\r', '\t'];
if let Some(c) = label.chars().find(|c| FORBIDDEN.contains(c)) {
let display = match c {
'\0' => "\\0 (null)".to_string(),
'\n' => "\\n (newline)".to_string(),
'\r' => "\\r (carriage return)".to_string(),
'\t' => "\\t (tab)".to_string(),
c => format!("'{c}'"),
};
return Err(format!("genome label contains forbidden character {display}"));
}
Ok(())
}
src/obikindex/src/index/mod.rs
+2 -28
View File
@@ -1,39 +1,13 @@
pub mod error;
pub mod meta;
pub mod predicate;
pub mod state;
mod builder;
mod common;
mod distance;
mod dump;
mod dump_layer;
pub mod filter;
mod graph_pipeline;
mod kmer_index;
mod matrix_store;
mod merge;
mod merge_layer;
mod numa;
mod query_layer;
mod rebuild;
mod rebuild_layer;
mod reindex;
mod select;
mod select_layer;
mod stats;
pub use error::{OKIError, OKIResult};
pub use builder::IndexBuilder;
pub use common::olm_to_sk;
pub use graph_pipeline::{materialize_layer, write_graph_as_unitigs};
pub use distance::{DistanceMetric, DistanceOutput};
pub use filter::{GroupQuorumFilter, KmerFilter, passes_all};
pub use common::ColBuilder;
pub use kmer_index::KmerIndex;
pub use merge_layer::MergeMode;
pub use meta::{validate_label, GenomeInfo, IndexConfig, IndexMeta, META_FILENAME};
pub use predicate::{GroupFilterParams, MetaPred};
pub use query_layer::{KmerDesc, QueryHit, QueryStats};
pub use select_layer::{AggOp, OutputCol};
pub use meta::{GenomeInfo, IndexConfig, IndexMeta, META_FILENAME};
pub use state::IndexState;
pub use stats::IndexBitsPerKmer;
pub use numa::PartitionRunner;
src/obikindex/src/index/progress.rs
-21
View File
@@ -1,21 +0,0 @@
use std::sync::OnceLock;
use indicatif::MultiProgress;
static MULTI: OnceLock<MultiProgress> = OnceLock::new();
/// Initialise the shared progress display. Call once from the binary before
/// any index operation. Subsequent calls are silently ignored.
pub fn init(multi: MultiProgress) {
let _ = MULTI.set(multi);
}
/// Return the shared `MultiProgress`, creating a plain default one if the
/// binary never called [`init`].
pub fn get() -> &'static MultiProgress {
MULTI.get_or_init(MultiProgress::new)
}
pub(crate) fn multi() -> &'static MultiProgress {
get()
}
src/obikindex/src/layer/content_layer.rs
+108 -61
View File
@@ -24,77 +24,103 @@
use std::path::{Path, PathBuf};
use crate::layer::utils::LAYERNAME_SUFFIX;
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
use obikseq::CanonicalKmer;
use crate::layer::error::OLMResult;
use crate::layer::meta::IndexMode;
use crate::index::error::OKIResult;
use crate::layer::mphf_layer::{
EVIDENCE_FILE, EvidenceKind, FINGERPRINT_FILE, MPHF_FILE, UNITIGS_FILE,
};
use crate::layer::typed_layer::{COUNTS_DIR, LayerContent, PRESENCE_DIR, TypedLayer};
use crate::partition::KmerPartition;
/// Superkmer-file extension used by every pre-layer-construction artifact
/// below (`raw`/`dereplicated`) — an implementation detail of the SK file
/// format, never meant to leak as a literal string past this module.
const SK_EXT: &str = "skmer.zst";
/// One layer, at any point in its life — see the module docs. Only
/// [`Empty`](Layer::Empty) and the two ready-to-read states
/// (`Count`/`Presence`) exist so far; states in between (unitigs written,
/// MPHF built, evidence built, matrix not yet built) are not represented
/// yet.
///
/// Every variant carries `dir` + `id` — a layer's identity never depends on
/// its state. `id` is its own layer number within the partition (not yet
/// consumed by anything — `KmerPartition` doesn't hand it in yet, that
/// lands with `KmerLayer::at`). `dir`/`id` live here, on `KmerLayer` itself,
/// not inside `TypedLayer`/`MphfLayer`: those only ever need `dir`
/// transiently, while opening — `KmerLayer` is what has to answer `.dir()`
/// after the fact, so it's the one that keeps it.
pub enum KmerLayer {
/// Just a directory — nothing built yet. Every method below other than
/// the path accessors panics on this variant: calling them means the
/// caller assumed a layer was ready when it wasn't, an implementation
/// error to surface loudly, not paper over with a default value.
Empty {
Empty { dir: PathBuf, id: usize },
Count {
dir: PathBuf,
id: usize,
layer: TypedLayer<PersistentCompactIntMatrix>,
},
Presence {
dir: PathBuf,
id: usize,
layer: TypedLayer<PersistentBitMatrix>,
},
Count(TypedLayer<PersistentCompactIntMatrix>),
Presence(TypedLayer<PersistentBitMatrix>),
}
impl KmerLayer {
pub fn new(partition: &KmerPartition, i: usize) -> Self {
let dir = partition.dir().join(format!("{LAYERNAME_SUFFIX}{i}"));
KmerLayer::Empty { dir, id: i }
}
/// An empty shell at `dir` — creates the directory (if it doesn't
/// already exist) but nothing inside it. The starting state for
/// building a new layer.
pub fn create(dir: &Path) -> std::io::Result<Self> {
std::fs::create_dir_all(dir)?;
Ok(KmerLayer::Empty {
dir: dir.to_owned(),
})
pub fn create(self) -> std::io::Result<Self> {
std::fs::create_dir_all(self.dir())?;
Ok(self)
}
/// Open one layer, auto-detecting count vs. presence from what is
/// actually on disk (`counts/` present and wanted, else presence) — the
/// single source of truth every caller that opens a layer's own matrix
/// should share, so two callers can never disagree about which format a
/// given layer is. Dense vs. sparse presence storage is
/// `PersistentBitMatrix::open`'s own concern (it's a 4-way
/// `Columnar`/`Packed`/`Sparse`/`Implicit` enum internally), not decided
/// here.
pub fn open(layer_dir: &Path, mode: &IndexMode, with_counts: bool) -> OLMResult<Self> {
if with_counts && layer_dir.join(COUNTS_DIR).exists() {
return TypedLayer::<PersistentCompactIntMatrix>::open(layer_dir, mode)
.map(KmerLayer::Count);
/// Open this layer, auto-detecting count vs. presence from what is
/// actually on disk (`counts/` present, else presence) — the single
/// source of truth every caller that opens a layer's own matrix should
/// share, so two callers can never disagree about which format a given
/// layer is. Dense vs. sparse presence storage is `PersistentBitMatrix::
/// open`'s own concern (a 4-way `Columnar`/`Packed`/`Sparse`/`Implicit`
/// enum internally), not decided here. A no-op (returns `self`
/// unchanged) if already `Count`/`Presence`.
pub fn open(self) -> OKIResult<Self> {
let (dir, id) = match self {
KmerLayer::Empty { dir, id } => (dir, id),
already_open => return Ok(already_open),
};
if dir.join(COUNTS_DIR).exists() {
let layer = TypedLayer::<PersistentCompactIntMatrix>::open(&dir)?;
Ok(KmerLayer::Count { dir, id, layer })
} else {
let layer = TypedLayer::<PersistentBitMatrix>::open(&dir)?;
Ok(KmerLayer::Presence { dir, id, layer })
}
TypedLayer::<PersistentBitMatrix>::open(layer_dir, mode).map(KmerLayer::Presence)
}
// ── Paths — only meaningful before anything is built ───────────────
// ── Paths — valid in every state, `dir` never varies with content ────
//
// Once a layer is `Count`/`Presence`, its caller already knows the
// directory (it had to pass it to `open`) — these exist for builder
// code holding an `Empty` layer, so the `mphf.bin`/`unitigs.bin`/…
// naming stays defined once, here, rather than re-declared as
// string literals at every call site that writes into a layer
// directory (the same duplication `layer_dir`/`index_dir` fixed one
// level up — see `DevDocMD/implementation/partition_layer_cache.md`).
// The `mphf.bin`/`unitigs.bin`/… naming stays defined once, here,
// rather than re-declared as string literals at every call site that
// writes into a layer directory (the same duplication `layer_dir`/
// `index_dir` fixed one level up — see `DevDocMD/implementation/
// partition_layer_cache.md`).
/// This layer's own directory. Panics on `Count`/`Presence` — by that
/// point the caller already has the directory it opened with; asking
/// again here would mean it lost track of its own state.
/// This layer's own directory.
pub fn dir(&self) -> &Path {
match self {
KmerLayer::Empty { dir } => dir,
_ => panic!("Layer::dir() only available on Empty — caller already has this path"),
KmerLayer::Empty { dir, .. } => dir,
KmerLayer::Count { dir, .. } => dir,
KmerLayer::Presence { dir, .. } => dir,
}
}
@@ -117,36 +143,57 @@ impl KmerLayer {
self.dir().join(PRESENCE_DIR)
}
/// Path of this layer's raw, not-yet-dereplicated superkmer file —
/// written by whichever algorithm routes superkmers into this layer
/// (today: `obikindexer::algorithms::partitionner::PartitionRouter`),
/// read by whichever algorithm dereplicates it (today:
/// `obikindexer::algorithms::dereplicator::Dereplicator`). Naming this
/// once here, rather than in either algorithm submodule, is what lets
/// the two agree on the filename without depending on each other
/// directly — see `DevDocMD/implementation/partition_layer_cache.md`.
pub fn raw_superkmers_path(&self) -> PathBuf {
self.dir().join(format!("raw.{SK_EXT}"))
}
/// Path of this layer's dereplicated superkmer file — written by
/// `obikindexer::algorithms::dereplicator::Dereplicator`, read by
/// whichever algorithm counts kmer abundances from it (today:
/// `obikindexer::algorithms::partitionner::PartitionRouter::count_kmer`)
/// and, later, by `obikindex::build_index_layer` to build the real layer.
pub fn dereplicated_superkmers_path(&self) -> PathBuf {
self.dir().join(format!("dereplicated.{SK_EXT}"))
}
// ── Ready-only surface ───────────────────────────────────────────────
pub fn content(&self) -> LayerContent {
match self {
KmerLayer::Count(_) => LayerContent::Count,
KmerLayer::Presence(_) => LayerContent::Presence,
KmerLayer::Count { .. } => LayerContent::Count,
KmerLayer::Presence { .. } => LayerContent::Presence,
KmerLayer::Empty { .. } => panic!("Layer::content() called on an Empty layer"),
}
}
pub fn evidence_kind(&self) -> EvidenceKind {
match self {
KmerLayer::Count(l) => l.evidence_kind(),
KmerLayer::Presence(l) => l.evidence_kind(),
KmerLayer::Count { layer, .. } => layer.evidence_kind(),
KmerLayer::Presence { layer, .. } => layer.evidence_kind(),
KmerLayer::Empty { .. } => panic!("Layer::evidence_kind() called on an Empty layer"),
}
}
pub fn n(&self) -> usize {
match self {
KmerLayer::Count(l) => l.n(),
KmerLayer::Presence(l) => l.n(),
KmerLayer::Count { layer, .. } => layer.n(),
KmerLayer::Presence { layer, .. } => layer.n(),
KmerLayer::Empty { .. } => panic!("Layer::n() called on an Empty layer"),
}
}
pub fn find_slot(&self, kmer: CanonicalKmer) -> Option<usize> {
match self {
KmerLayer::Count(l) => l.find_slot(kmer),
KmerLayer::Presence(l) => l.find_slot(kmer),
KmerLayer::Count { layer, .. } => layer.find_slot(kmer),
KmerLayer::Presence { layer, .. } => layer.find_slot(kmer),
KmerLayer::Empty { .. } => panic!("Layer::find_slot() called on an Empty layer"),
}
}
@@ -156,16 +203,16 @@ impl KmerLayer {
/// the evidence check `find_slot`/`find` would perform is redundant.
pub fn hash_batch(&self, kmers: &[CanonicalKmer]) -> Vec<usize> {
match self {
KmerLayer::Count(l) => l.hash_batch(kmers),
KmerLayer::Presence(l) => l.hash_batch(kmers),
KmerLayer::Count { layer, .. } => layer.hash_batch(kmers),
KmerLayer::Presence { layer, .. } => layer.hash_batch(kmers),
KmerLayer::Empty { .. } => panic!("Layer::hash_batch() called on an Empty layer"),
}
}
pub fn n_cols(&self) -> usize {
match self {
KmerLayer::Count(l) => l.n_cols(),
KmerLayer::Presence(l) => l.n_cols(),
KmerLayer::Count { layer, .. } => layer.n_cols(),
KmerLayer::Presence { layer, .. } => layer.n_cols(),
KmerLayer::Empty { .. } => panic!("Layer::n_cols() called on an Empty layer"),
}
}
@@ -178,10 +225,10 @@ impl KmerLayer {
/// presence (`!= 0`) in place.
pub fn fill_sub_matrix_carries(&self, slots: &[usize], out: &mut [Vec<bool>]) {
match self {
KmerLayer::Presence(l) => l.fill_sub_matrix(slots, out),
KmerLayer::Count(l) => {
KmerLayer::Presence { layer, .. } => layer.fill_sub_matrix(slots, out),
KmerLayer::Count { layer, .. } => {
let mut counts: Vec<Vec<u32>> = out.iter().map(|_| Vec::new()).collect();
l.fill_sub_matrix(slots, &mut counts);
layer.fill_sub_matrix(slots, &mut counts);
for (o, c) in out.iter_mut().zip(counts.iter()) {
o.clear();
o.extend(c.iter().map(|&v| v != 0));
@@ -196,8 +243,8 @@ impl KmerLayer {
/// Raw MPHF lookup: kmer → slot, no membership check.
pub fn hash(&self, kmer: CanonicalKmer) -> usize {
match self {
KmerLayer::Count(l) => l.hash(kmer),
KmerLayer::Presence(l) => l.hash(kmer),
KmerLayer::Count { layer, .. } => layer.hash(kmer),
KmerLayer::Presence { layer, .. } => layer.hash(kmer),
KmerLayer::Empty { .. } => panic!("Layer::hash() called on an Empty layer"),
}
}
@@ -205,8 +252,8 @@ impl KmerLayer {
/// Iterate over all canonical kmers in the layer, in deterministic order.
pub fn iter_kmers(&self) -> crate::layer::mphf_layer::KmerIter {
match self {
KmerLayer::Count(l) => l.iter_kmers(),
KmerLayer::Presence(l) => l.iter_kmers(),
KmerLayer::Count { layer, .. } => layer.iter_kmers(),
KmerLayer::Presence { layer, .. } => layer.iter_kmers(),
KmerLayer::Empty { .. } => panic!("Layer::iter_kmers() called on an Empty layer"),
}
}
@@ -215,8 +262,8 @@ impl KmerLayer {
/// sequence index in `unitigs.bin`.
pub fn enumerate_kmers(&self) -> std::iter::Enumerate<crate::layer::mphf_layer::KmerIter> {
match self {
KmerLayer::Count(l) => l.enumerate_kmers(),
KmerLayer::Presence(l) => l.enumerate_kmers(),
KmerLayer::Count { layer, .. } => layer.enumerate_kmers(),
KmerLayer::Presence { layer, .. } => layer.enumerate_kmers(),
KmerLayer::Empty { .. } => panic!("Layer::enumerate_kmers() called on an Empty layer"),
}
}
@@ -224,8 +271,8 @@ impl KmerLayer {
/// Iterate over the layer's canonical kmers in batches of `n`.
pub fn iter_kmers_batch(&self, n: usize) -> crate::layer::mphf_layer::KmerBatchIter {
match self {
KmerLayer::Count(l) => l.iter_kmers_batch(n),
KmerLayer::Presence(l) => l.iter_kmers_batch(n),
KmerLayer::Count { layer, .. } => layer.iter_kmers_batch(n),
KmerLayer::Presence { layer, .. } => layer.iter_kmers_batch(n),
KmerLayer::Empty { .. } => panic!("Layer::iter_kmers_batch() called on an Empty layer"),
}
}
@@ -237,8 +284,8 @@ impl KmerLayer {
n: usize,
) -> Box<dyn Iterator<Item = (usize, Vec<CanonicalKmer>)> + Send + 'static> {
match self {
KmerLayer::Count(l) => Box::new(l.enumerate_kmers_batch(n)),
KmerLayer::Presence(l) => Box::new(l.enumerate_kmers_batch(n)),
KmerLayer::Count { layer, .. } => Box::new(layer.enumerate_kmers_batch(n)),
KmerLayer::Presence { layer, .. } => Box::new(layer.enumerate_kmers_batch(n)),
KmerLayer::Empty { .. } => {
panic!("Layer::enumerate_kmers_batch() called on an Empty layer")
}
src/obikindex/src/layer/error.rs
-50
View File
@@ -1,50 +0,0 @@
use std::fmt;
use std::io;
#[derive(Debug)]
pub enum OLMError {
Io(io::Error),
Json(serde_json::Error),
Mphf(String),
InvalidLayer(String),
}
pub type OLMResult<T> = Result<T, OLMError>;
impl fmt::Display for OLMError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
OLMError::Io(e) => write!(f, "I/O error: {e}"),
OLMError::Json(e) => write!(f, "JSON error: {e}"),
OLMError::Mphf(s) => write!(f, "MPHF error: {s}"),
OLMError::InvalidLayer(s) => write!(f, "invalid layer: {s}"),
}
}
}
impl std::error::Error for OLMError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
match self {
OLMError::Io(e) => Some(e),
OLMError::Json(e) => Some(e),
_ => None,
}
}
}
impl From<io::Error> for OLMError {
fn from(e: io::Error) -> Self { OLMError::Io(e) }
}
impl From<serde_json::Error> for OLMError {
fn from(e: serde_json::Error) -> Self { OLMError::Json(e) }
}
impl From<obiskio::SKError> for OLMError {
fn from(e: obiskio::SKError) -> Self {
match e {
obiskio::SKError::Io(io_err) => OLMError::Io(io_err),
other => OLMError::InvalidLayer(other.to_string()),
}
}
}
src/obikindex/src/layer/evidence.rs
+10 -10
View File
@@ -6,14 +6,14 @@ use std::path::Path;
use memmap2::Mmap;
use crate::layer::error::{OLMError, OLMResult};
use crate::index::error::{OKIError, OKIResult};
pub struct Evidence {
mmap: Mmap,
}
impl Evidence {
pub fn open(path: &Path) -> OLMResult<Self> {
pub fn open(path: &Path) -> OKIResult<Self> {
let f = File::open(path)?;
let mmap = unsafe { Mmap::map(&f)? };
Ok(Self { mmap })
@@ -26,16 +26,16 @@ impl Evidence {
(raw >> 7, (raw & 0x7F) as u8)
}
#[inline]
pub fn encode(chunk_id: u32, rank: u8) -> u32 {
(chunk_id << 7) | (rank as u32 & 0x7F)
}
pub fn len(&self) -> usize {
self.mmap.len() / 4
}
}
#[inline]
pub fn encode(chunk_id: u32, rank: u8) -> u32 {
(chunk_id << 7) | (rank as u32 & 0x7F)
}
pub struct EvidenceWriter {
buf: Vec<u32>,
}
@@ -47,13 +47,13 @@ impl EvidenceWriter {
#[inline]
pub fn set(&mut self, slot: usize, chunk_id: u32, rank: u8) {
self.buf[slot] = encode(chunk_id, rank);
self.buf[slot] = Evidence::encode(chunk_id, rank);
}
pub fn write(self, path: &Path) -> OLMResult<()> {
pub fn write(self, path: &Path) -> OKIResult<()> {
let mut f = BufWriter::new(File::create(path)?);
for v in self.buf {
f.write_all(&v.to_le_bytes()).map_err(OLMError::Io)?;
f.write_all(&v.to_le_bytes()).map_err(OKIError::Io)?;
}
Ok(())
}
src/obikindex/src/layer/fingerprint.rs
+10 -10
View File
@@ -14,7 +14,7 @@ use std::path::Path;
use bitvec::prelude::*;
use memmap2::Mmap;
use crate::layer::error::{OLMError, OLMResult};
use crate::index::error::{OKIError, OKIResult};
const MAGIC: &[u8; 4] = b"FPVF";
const HEADER: usize = 16;
@@ -30,15 +30,15 @@ pub struct FingerprintVec {
}
impl FingerprintVec {
pub fn open(path: &Path) -> OLMResult<Self> {
pub fn open(path: &Path) -> OKIResult<Self> {
let f = File::open(path)?;
let mmap = unsafe { Mmap::map(&f)? };
if mmap.len() < HEADER || &mmap[..4] != MAGIC {
return Err(OLMError::InvalidLayer("bad fingerprint magic".into()));
return Err(OKIError::InvalidData { context: "layer", detail: "bad fingerprint magic".into() });
}
let b = mmap[4];
if b == 0 || b > 64 {
return Err(OLMError::InvalidLayer("invalid fingerprint width".into()));
return Err(OKIError::InvalidData { context: "layer", detail: "invalid fingerprint width".into() });
}
let n = u64::from_le_bytes(mmap[8..16].try_into().unwrap()) as usize;
let mask: u64 = if b == 64 { u64::MAX } else { (1u64 << b) - 1 };
@@ -89,12 +89,12 @@ impl FingerprintVecWriter {
bits[lo .. lo + self.b as usize].store_le(fingerprint);
}
pub fn write(self, path: &Path) -> OLMResult<()> {
let mut f = BufWriter::new(File::create(path).map_err(OLMError::Io)?);
f.write_all(MAGIC).map_err(OLMError::Io)?;
f.write_all(&[self.b, 0, 0, 0]).map_err(OLMError::Io)?;
f.write_all(&(self.n as u64).to_le_bytes()).map_err(OLMError::Io)?;
f.write_all(&self.buf).map_err(OLMError::Io)?;
pub fn write(self, path: &Path) -> OKIResult<()> {
let mut f = BufWriter::new(File::create(path).map_err(OKIError::Io)?);
f.write_all(MAGIC).map_err(OKIError::Io)?;
f.write_all(&[self.b, 0, 0, 0]).map_err(OKIError::Io)?;
f.write_all(&(self.n as u64).to_le_bytes()).map_err(OKIError::Io)?;
f.write_all(&self.buf).map_err(OKIError::Io)?;
Ok(())
}
}
src/obikindex/src/layer/map.rs
-128
View File
@@ -1,128 +0,0 @@
use std::collections::HashMap;
use std::fs;
use std::path::{Path, PathBuf};
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix, StorageKind};
use obikseq::CanonicalKmer;
use obiskio::{UnitigFileWriter, DEFAULT_BLOCK_BITS};
use crate::layer::error::{OLMError, OLMResult};
use crate::layer::typed_layer::{layer_dir, Hit, TypedLayer, LayerContent, LayerData};
use crate::layer::meta::{IndexMode, PartitionMeta};
use crate::layer::mphf_layer::EvidenceKind;
/// Layered kmer index for a single partition.
///
/// Each layer covers a disjoint kmer set. Queries probe layers in order;
/// the first match wins. Adding a dataset appends a new layer without
/// rebuilding existing ones.
pub struct LayeredMap<D: LayerData = ()> {
root: PathBuf,
meta: PartitionMeta,
layers: Vec<TypedLayer<D>>,
}
// ── Common methods ────────────────────────────────────────────────────────────
impl<D: LayerData> LayeredMap<D> {
/// Open an existing layered index at `root`.
/// The mode is read once from `PartitionMeta` and applied to all layers.
pub fn open(root: &Path) -> OLMResult<Self> {
let meta = PartitionMeta::load(root)?;
let layers = (0..meta.n_layers)
.map(|i| TypedLayer::<D>::open(&layer_dir(root, i), &meta.mode))
.collect::<OLMResult<Vec<_>>>()?;
Ok(Self { root: root.to_owned(), meta, layers })
}
/// Create a new, empty layered index at `root` with the given mode.
pub fn create(root: &Path, mode: IndexMode) -> OLMResult<Self> {
fs::create_dir_all(root)?;
let meta = PartitionMeta::new(mode);
meta.save(root)?;
Ok(Self { root: root.to_owned(), meta, layers: Vec::new() })
}
pub fn n_layers(&self) -> usize { self.layers.len() }
pub fn layer(&self, i: usize) -> &TypedLayer<D> { &self.layers[i] }
pub fn mode(&self) -> &IndexMode { &self.meta.mode }
/// TypedLayer `i`'s content (`Count`/`Presence`) — a lightweight disk probe,
/// no MPHF/matrix opened. For picking a `D` before committing to
/// [`TypedLayer::open`], or for reporting/diagnostics over an already-open
/// `LayeredMap` without re-opening every layer under a different `D`.
pub fn detect_layer_content(&self, i: usize) -> LayerContent {
LayerContent::detect(&layer_dir(&self.root, i))
}
/// TypedLayer `i`'s storage format — a lightweight disk probe, no
/// MPHF/matrix opened. See [`detect_layer_content`](Self::detect_layer_content).
pub fn detect_layer_storage(&self, i: usize) -> OLMResult<StorageKind> {
let dir = layer_dir(&self.root, i);
match LayerContent::detect(&dir) {
LayerContent::Count => PersistentCompactIntMatrix::detect_storage(&dir).map_err(OLMError::Io),
LayerContent::Presence => PersistentBitMatrix::detect_storage(&dir).map_err(OLMError::Io),
}
}
/// TypedLayer `i`'s evidence mode — a lightweight disk probe, no
/// MPHF/matrix opened. See [`detect_layer_content`](Self::detect_layer_content).
pub fn detect_layer_evidence(&self, i: usize) -> OLMResult<EvidenceKind> {
EvidenceKind::detect(&layer_dir(&self.root, i))
}
/// Query `kmer` across all layers. Returns `(layer_index, Hit)` on match.
pub fn query(&self, kmer: CanonicalKmer) -> Option<(usize, Hit<D::Item>)> {
self.layers
.iter()
.enumerate()
.find_map(|(i, layer)| layer.query(kmer).map(|hit| (i, hit)))
}
pub fn next_layer_writer(&self) -> OLMResult<UnitigFileWriter> {
let dir = layer_dir(&self.root, self.layers.len());
TypedLayer::<D>::unitig_writer(&dir)
}
fn append_layer(&mut self) -> OLMResult<()> {
let i = self.layers.len();
let dir = layer_dir(&self.root, i);
self.layers.push(TypedLayer::<D>::open(&dir, &self.meta.mode)?);
self.meta.n_layers = self.layers.len();
self.meta.save(&self.root)?;
Ok(())
}
}
// ── Mode 1 — set membership ───────────────────────────────────────────────────
impl LayeredMap<()> {
pub fn push_layer(&mut self) -> OLMResult<usize> {
let i = self.layers.len();
let dir = layer_dir(&self.root, i);
TypedLayer::<()>::build(&dir, DEFAULT_BLOCK_BITS, &self.meta.mode)?;
self.append_layer()?;
Ok(i)
}
}
// ── Mode 2 — count matrix ─────────────────────────────────────────────────────
impl LayeredMap<PersistentCompactIntMatrix> {
pub fn push_layer(&mut self, count_of: impl Fn(CanonicalKmer) -> u32) -> OLMResult<usize> {
let i = self.layers.len();
let dir = layer_dir(&self.root, i);
TypedLayer::<PersistentCompactIntMatrix>::build(&dir, DEFAULT_BLOCK_BITS, &self.meta.mode, count_of)?;
self.append_layer()?;
Ok(i)
}
pub fn push_layer_from_map(&mut self, counts: &HashMap<CanonicalKmer, u32>) -> OLMResult<usize> {
self.push_layer(|kmer| counts.get(&kmer).copied().unwrap_or(0))
}
}
#[cfg(test)]
#[path = "tests/map.rs"]
mod tests;
src/obikindex/src/layer/meta.rs
+30 -35
View File
@@ -1,18 +1,20 @@
use std::fs::File;
use std::path::Path;
use serde::{Deserialize, Serialize};
use crate::layer::error::OLMResult;
const META_FILE: &str = "meta.json";
use crate::index::error::OKIResult;
use crate::layer::fingerprint::FingerprintVec;
use crate::layer::mphf_layer::{EvidenceKind, FINGERPRINT_FILE};
use crate::layer::utils::layer_dir;
// ── IndexMode ─────────────────────────────────────────────────────────────────
/// Evidence mode for an entire partitioned index — homogeneous across all layers.
///
/// Determined once at build time; stored in `PartitionMeta` (`meta.json`).
/// All layers within an index share the same mode.
/// Determined once at build time. All layers within an index share the same
/// mode, so it is never persisted separately — [`IndexMode::detect`] reads it
/// straight back from whichever evidence file(s) layer 0 actually has on
/// disk, the same probe [`EvidenceKind::detect`] uses.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum IndexMode {
@@ -20,7 +22,9 @@ pub enum IndexMode {
Exact,
/// Approximate evidence: `fingerprint.bin` only.
/// `b` — fingerprint bits per slot; false-positive rate ≈ 1/2^b per query.
/// `z` — Findere consecutive-kmer parameter (build-time only; not used at query time).
/// `z` — Findere consecutive-kmer parameter (build-time only; not used at
/// query time, not stored on disk, always `0` once recovered via
/// [`detect`](Self::detect)).
Approx { b: u8, z: u8 },
/// Hybrid: both `fingerprint.bin` and `evidence.bin` + `unitigs.bin.idx`.
/// `find()` uses the fingerprint (O(1), approx); `find_strict()` uses exact evidence.
@@ -31,33 +35,24 @@ impl Default for IndexMode {
fn default() -> Self { Self::Exact }
}
// ── PartitionMeta ─────────────────────────────────────────────────────────────
/// Index-level metadata stored in `meta.json` at the root of a partition index.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PartitionMeta {
pub n_layers: usize,
#[serde(default)]
pub mode: IndexMode,
}
impl PartitionMeta {
pub fn new(mode: IndexMode) -> Self {
Self { n_layers: 0, mode }
}
pub fn load(dir: &Path) -> OLMResult<Self> {
let f = File::open(dir.join(META_FILE))?;
Ok(serde_json::from_reader(f)?)
}
pub fn save(&self, dir: &Path) -> OLMResult<()> {
let f = File::create(dir.join(META_FILE))?;
serde_json::to_writer_pretty(f, self)?;
Ok(())
impl IndexMode {
/// Detect the evidence mode actually on disk, from partition index
/// directory `root`'s layer 0 — the single source of truth every layer
/// in the same index shares. `z` cannot be recovered (not stored in
/// `fingerprint.bin`, and irrelevant past build time), so it comes back
/// as `0`.
pub fn detect(root: &Path) -> OKIResult<Self> {
let layer0 = layer_dir(root, 0);
Ok(match EvidenceKind::detect(&layer0)? {
EvidenceKind::Exact => IndexMode::Exact,
EvidenceKind::Approx => IndexMode::Approx {
b: FingerprintVec::open(&layer0.join(FINGERPRINT_FILE))?.b(),
z: 0,
},
EvidenceKind::Hybrid => IndexMode::Hybrid {
b: FingerprintVec::open(&layer0.join(FINGERPRINT_FILE))?.b(),
z: 0,
},
})
}
}
impl Default for PartitionMeta {
fn default() -> Self { Self::new(IndexMode::Exact) }
}
src/obikindex/src/layer/mod.rs
+4 -11
View File
@@ -1,20 +1,13 @@
pub mod content_layer;
pub mod error;
pub mod evidence;
pub mod fingerprint;
pub mod layered_store;
pub mod map;
pub mod meta;
pub(crate) mod mphf_layer;
pub mod typed_layer;
pub(crate) mod utils;
pub use crate::index::error::{OKIError, OKIResult};
pub use content_layer::KmerLayer;
pub use error::{OLMError, OLMResult};
pub use layered_store::LayeredStore;
pub use map::LayeredMap;
pub use meta::{IndexMode, PartitionMeta};
pub use meta::IndexMode;
pub use mphf_layer::{EvidenceKind, KmerBatchIter, KmerIter, MphfLayer, MphfOnly};
pub use typed_layer::{
HasLayerContent, HasStorageKind, Hit, LayerContent, LayerData, TypedLayer,
dereplicated_superkmers_path, layer_dir, open_data, raw_superkmers_path,
};
pub use typed_layer::{HasLayerContent, HasStorageKind, Hit, LayerContent, LayerData, TypedLayer};
src/obikindex/src/layer/mphf_layer.rs
+38 -42
View File
@@ -10,7 +10,7 @@ use obikseq::CanonicalKmer;
use obiskio::{CanonicalKmerIter, UnitigFileReader, UnitigFileWriter, build_unitig_idx};
use ptr_hash::{PtrHash, PtrHashParams, bucket_fn::CubicEps, hash::Xx64};
use crate::layer::error::{OLMError, OLMResult};
use crate::index::error::{OKIError, OKIResult};
use crate::layer::evidence::{Evidence, EvidenceWriter};
use crate::layer::fingerprint::{FingerprintVec, FingerprintVecWriter};
use crate::layer::meta::IndexMode;
@@ -76,7 +76,7 @@ impl LayerEvidence {
/// Identify which evidence files exist at `layer_dir`, without opening
/// any of them — a lightweight disk probe (same signal `EvidenceKind::
/// detect` uses), not a load.
fn at(layer_dir: &Path) -> OLMResult<Self> {
fn at(layer_dir: &Path) -> OKIResult<Self> {
Ok(match EvidenceKind::detect(layer_dir)? {
EvidenceKind::Exact => LayerEvidence::Exact {
evidence_path: layer_dir.join(EVIDENCE_FILE),
@@ -94,7 +94,7 @@ impl LayerEvidence {
}
/// mmap the evidence file(s) this identity points to.
fn open(self) -> OLMResult<OpenLayerEvidence> {
fn open(self) -> OKIResult<OpenLayerEvidence> {
Ok(match self {
LayerEvidence::Exact { evidence_path, unitig_path } => OpenLayerEvidence::Exact {
evidence: Evidence::open(&evidence_path)?,
@@ -127,17 +127,17 @@ impl EvidenceKind {
/// Lightweight disk probe, no MPHF/evidence opened: presence of
/// `evidence.bin`/`fingerprint.bin` alone determines the mode — same
/// signal `MphfLayer::open` uses, just without opening either file.
pub fn detect(layer_dir: &Path) -> OLMResult<EvidenceKind> {
pub fn detect(layer_dir: &Path) -> OKIResult<EvidenceKind> {
let has_evidence = layer_dir.join(EVIDENCE_FILE).exists();
let has_fingerprint = layer_dir.join(FINGERPRINT_FILE).exists();
match (has_evidence, has_fingerprint) {
(true, false) => Ok(EvidenceKind::Exact),
(false, true) => Ok(EvidenceKind::Approx),
(true, true) => Ok(EvidenceKind::Hybrid),
(false, false) => Err(OLMError::InvalidLayer(format!(
(false, false) => Err(OKIError::InvalidData { context: "layer", detail: format!(
"no evidence.bin or fingerprint.bin in {}",
layer_dir.display()
))),
) }),
}
}
}
@@ -151,7 +151,6 @@ impl EvidenceKind {
/// - [`find_strict`](Self::find_strict) — always exact; O(1) on Exact/Hybrid layers,
/// O(n) sequential scan on Approx layers.
pub struct MphfLayer {
layer_dir: PathBuf,
/// Mode-independent copy — used by `iter_kmers` (all modes) and
/// `find_strict`'s `Approx` fallback. `Exact`/`Hybrid` additionally
/// carry their own inside `OpenLayerEvidence` (see its doc comment for
@@ -165,18 +164,15 @@ pub struct MphfLayer {
}
impl MphfLayer {
/// This layer's own directory.
pub(crate) fn dir(&self) -> &Path {
&self.layer_dir
}
/// Open a layer — evidence mode is auto-detected from which files exist
/// on disk (see [`LayerEvidence::at`]), never passed in: the caller
/// can't tell this type anything about its own mode that isn't already
/// derivable from `layer_dir` itself.
pub fn open(dir: &Path) -> OLMResult<Self> {
/// derivable from `dir` itself. `dir` is only used transiently here —
/// not stored: `KmerLayer`, one level up, is what owns a layer's
/// directory as a persistent field, not this type.
pub fn open(dir: &Path) -> OKIResult<Self> {
let mphf: MemCase<MphfEps> = Mphf::mmap(&dir.join(MPHF_FILE), Flags::empty())
.map_err(|e| OLMError::InvalidLayer(e.to_string()))?;
.map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?;
let unitigs = Arc::new(UnitigFileReader::open(&dir.join(UNITIGS_FILE))?);
let ev = LayerEvidence::at(dir)?.open()?;
let n = match &ev {
@@ -184,7 +180,7 @@ impl MphfLayer {
OpenLayerEvidence::Hybrid { evidence, .. } => evidence.len(),
OpenLayerEvidence::Approx { fingerprint } => fingerprint.n(),
};
Ok(Self { layer_dir: dir.to_owned(), unitigs, mphf, ev, n })
Ok(Self { unitigs, mphf, ev, n })
}
// ── Query API ─────────────────────────────────────────────────────────────
@@ -405,9 +401,9 @@ impl Iterator for KmerBatchIter {
pub struct MphfOnly(MemCase<MphfEps>);
impl MphfOnly {
pub fn open(dir: &Path) -> OLMResult<Self> {
pub fn open(dir: &Path) -> OKIResult<Self> {
let mphf: MemCase<MphfEps> = Mphf::mmap(&dir.join(MPHF_FILE), Flags::empty())
.map_err(|e| OLMError::InvalidLayer(e.to_string()))?;
.map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?;
Ok(Self(mphf))
}
@@ -421,13 +417,13 @@ impl MphfOnly {
impl MphfLayer {
// ── Build helpers ─────────────────────────────────────────────────────────
pub fn unitig_writer(dir: &Path) -> OLMResult<UnitigFileWriter> {
pub fn unitig_writer(dir: &Path) -> OKIResult<UnitigFileWriter> {
fs::create_dir_all(dir)?;
Ok(UnitigFileWriter::create(&dir.join(UNITIGS_FILE))?)
}
/// Build `evidence.bin` + `unitigs.bin.idx` from `unitigs.bin` + `mphf.bin`.
pub fn build_exact_evidence(dir: &Path, block_bits: u8) -> OLMResult<usize> {
pub fn build_exact_evidence(dir: &Path, block_bits: u8) -> OKIResult<usize> {
let unitig_path = dir.join(UNITIGS_FILE);
let unitigs = UnitigFileReader::open_sequential(&unitig_path)?;
let n = unitigs.n_kmers();
@@ -439,7 +435,7 @@ impl MphfLayer {
}
let mphf: Mphf = Mphf::load_full(&dir.join(MPHF_FILE))
.map_err(|e| OLMError::InvalidLayer(e.to_string()))?;
.map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?;
let mut ev = EvidenceWriter::new(n);
let mut seen = vec![0u8; (n + 7) / 8];
@@ -447,12 +443,12 @@ impl MphfLayer {
for (kmer, chunk_id, rank) in unitigs.iter_indexed_canonical_kmers() {
let slot = mphf.index(&kmer.raw());
if slot >= n {
return Err(OLMError::Mphf("slot out of bounds".into()));
return Err(OKIError::InvalidData { context: "mphf", detail: "slot out of bounds".into() });
}
let byte = slot / 8;
let bit = 1u8 << (slot % 8);
if seen[byte] & bit != 0 {
return Err(OLMError::Mphf("duplicate slot".into()));
return Err(OKIError::InvalidData { context: "mphf", detail: "duplicate slot".into() });
}
seen[byte] |= bit;
ev.set(slot, chunk_id as u32, rank as u8);
@@ -464,14 +460,14 @@ impl MphfLayer {
}
/// Build `fingerprint.bin` from `unitigs.bin` + `mphf.bin`.
pub fn build_approx_evidence(dir: &Path, b: u8, z: u8) -> OLMResult<usize> {
pub fn build_approx_evidence(dir: &Path, b: u8, z: u8) -> OKIResult<usize> {
if b == 0 || b > 64 {
return Err(OLMError::InvalidLayer(
return Err(OKIError::InvalidData { context: "layer", detail:
"fingerprint width must be 1..=64".into(),
));
});
}
if z == 0 {
return Err(OLMError::InvalidLayer("z must be ≥ 1".into()));
return Err(OKIError::InvalidData { context: "layer", detail: "z must be ≥ 1".into() });
}
let unitig_path = dir.join(UNITIGS_FILE);
let unitigs = UnitigFileReader::open_sequential(&unitig_path)?;
@@ -483,14 +479,14 @@ impl MphfLayer {
}
let mphf: Mphf = Mphf::load_full(&dir.join(MPHF_FILE))
.map_err(|e| OLMError::InvalidLayer(e.to_string()))?;
.map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?;
let mut fw = FingerprintVecWriter::new(n, b);
for (kmer, _, _) in unitigs.iter_indexed_canonical_kmers() {
let slot = mphf.index(&kmer.raw());
if slot >= n {
return Err(OLMError::Mphf("slot out of bounds".into()));
return Err(OKIError::InvalidData { context: "mphf", detail: "slot out of bounds".into() });
}
fw.set(slot, kmer.seq_hash());
}
@@ -508,24 +504,24 @@ impl MphfLayer {
dir: &Path,
block_bits: u8,
mode: &IndexMode,
fill_slot: &mut impl FnMut(usize, CanonicalKmer) -> OLMResult<()>,
) -> OLMResult<usize> {
fill_slot: &mut impl FnMut(usize, CanonicalKmer) -> OKIResult<()>,
) -> OKIResult<usize> {
use rayon::prelude::*;
let unitig_path = dir.join(UNITIGS_FILE);
let n = UnitigFileReader::open_sequential(&unitig_path)?.n_kmers();
let sk_to_olm = |e: obiskio::SKError| match e {
obiskio::SKError::Io(io) => OLMError::Io(io),
e => OLMError::InvalidLayer(e.to_string()),
obiskio::SKError::Io(io) => OKIError::Io(io),
e => OKIError::InvalidData { context: "layer", detail: e.to_string() },
};
// ── Empty layer ───────────────────────────────────────────────────────
if n == 0 {
let mphf: Mphf = Mphf::try_new(&[] as &[u64], PtrHashParams::<CubicEps>::default())
.ok_or_else(|| OLMError::Mphf("construction failed".into()))?;
.ok_or_else(|| OKIError::InvalidData { context: "mphf", detail: "construction failed".into() })?;
mphf.store(&dir.join(MPHF_FILE))
.map_err(|e| OLMError::InvalidLayer(e.to_string()))?;
.map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?;
match mode {
IndexMode::Exact | IndexMode::Hybrid { .. } => {
fs::File::create(dir.join(EVIDENCE_FILE))?;
@@ -549,7 +545,7 @@ impl MphfLayer {
PtrHashParams::<CubicEps>::default(),
);
mphf.store(&dir.join(MPHF_FILE))
.map_err(|e| OLMError::InvalidLayer(e.to_string()))?;
.map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?;
// ── Pass 2: fill evidence files + callback ────────────────────────────
let unitigs2 = UnitigFileReader::open_sequential(&unitig_path)?;
@@ -561,12 +557,12 @@ impl MphfLayer {
for (kmer, chunk_id, rank) in unitigs2.iter_indexed_canonical_kmers() {
let slot = mphf.index(&kmer.raw());
if slot >= n {
return Err(OLMError::Mphf("slot out of bounds".into()));
return Err(OKIError::InvalidData { context: "mphf", detail: "slot out of bounds".into() });
}
let byte = slot / 8;
let bit = 1u8 << (slot % 8);
if seen[byte] & bit != 0 {
return Err(OLMError::Mphf("duplicate slot".into()));
return Err(OKIError::InvalidData { context: "mphf", detail: "duplicate slot".into() });
}
seen[byte] |= bit;
ev.set(slot, chunk_id as u32, rank as u8);
@@ -581,12 +577,12 @@ impl MphfLayer {
for (kmer, _, _) in unitigs2.iter_indexed_canonical_kmers() {
let slot = mphf.index(&kmer.raw());
if slot >= n {
return Err(OLMError::Mphf("slot out of bounds".into()));
return Err(OKIError::InvalidData { context: "mphf", detail: "slot out of bounds".into() });
}
let byte = slot / 8;
let bit = 1u8 << (slot % 8);
if seen[byte] & bit != 0 {
return Err(OLMError::Mphf("duplicate slot".into()));
return Err(OKIError::InvalidData { context: "mphf", detail: "duplicate slot".into() });
}
seen[byte] |= bit;
fw.set(slot, kmer.seq_hash());
@@ -601,12 +597,12 @@ impl MphfLayer {
for (kmer, chunk_id, rank) in unitigs2.iter_indexed_canonical_kmers() {
let slot = mphf.index(&kmer.raw());
if slot >= n {
return Err(OLMError::Mphf("slot out of bounds".into()));
return Err(OKIError::InvalidData { context: "mphf", detail: "slot out of bounds".into() });
}
let byte = slot / 8;
let bit = 1u8 << (slot % 8);
if seen[byte] & bit != 0 {
return Err(OLMError::Mphf("duplicate slot".into()));
return Err(OKIError::InvalidData { context: "mphf", detail: "duplicate slot".into() });
}
seen[byte] |= bit;
ev.set(slot, chunk_id as u32, rank as u8);
src/obikindex/src/layer/tests/map.rs
-219
View File
@@ -1,219 +0,0 @@
use super::*;
use obicompactvec::{pack_bit_matrix, pack_sparse_bit_matrix, PersistentBitMatrix, PersistentCompactIntMatrix, StorageKind};
use obikseq::{set_k, Sequence as _, Unitig};
use obiskio::DEFAULT_BLOCK_BITS;
use crate::layer::typed_layer::LayerContent;
use crate::layer::meta::IndexMode;
use crate::layer::mphf_layer::EvidenceKind;
use tempfile::tempdir;
fn push_unitigs_and_layer(
map: &mut LayeredMap<PersistentCompactIntMatrix>,
seqs: &[&[u8]],
count: u32,
) {
let mut w = map.next_layer_writer().unwrap();
for s in seqs {
w.write(&Unitig::from_ascii(s)).unwrap();
}
w.close().unwrap();
map.push_layer(|_| count).unwrap();
}
fn canonical(ascii: &[u8]) -> CanonicalKmer {
obikseq::Kmer::from_ascii(ascii).unwrap().canonical()
}
#[test]
fn create_empty_map() {
set_k(4);
let dir = tempdir().unwrap();
let map = LayeredMap::<()>::create(dir.path(), IndexMode::Exact).unwrap();
assert_eq!(map.n_layers(), 0);
}
#[test]
fn open_reloads_layer_count() {
set_k(4);
let dir = tempdir().unwrap();
{
let mut map = LayeredMap::<PersistentCompactIntMatrix>::create(dir.path(), IndexMode::Exact).unwrap();
push_unitigs_and_layer(&mut map, &[b"AAAACGT"], 1);
}
let map = LayeredMap::<PersistentCompactIntMatrix>::open(dir.path()).unwrap();
assert_eq!(map.n_layers(), 1);
}
#[test]
fn query_finds_kmer_in_layer_zero() {
set_k(4);
let dir = tempdir().unwrap();
let mut map = LayeredMap::<PersistentCompactIntMatrix>::create(dir.path(), IndexMode::Exact).unwrap();
push_unitigs_and_layer(&mut map, &[b"AAAACGT"], 3);
let kmer = canonical(b"AAAC");
let (layer_idx, hit) = map.query(kmer).expect("kmer must be found");
assert_eq!(layer_idx, 0);
assert_eq!(hit.data[0], 3);
}
#[test]
fn query_finds_kmer_in_correct_layer() {
set_k(4);
let dir = tempdir().unwrap();
let mut map = LayeredMap::<PersistentCompactIntMatrix>::create(dir.path(), IndexMode::Exact).unwrap();
push_unitigs_and_layer(&mut map, &[b"AAAACGT"], 1);
push_unitigs_and_layer(&mut map, &[b"GGGACGT"], 2);
assert_eq!(map.n_layers(), 2);
let (li, hit) = map.query(canonical(b"AAAA")).expect("AAAA must be found");
assert_eq!(li, 0);
assert_eq!(hit.data[0], 1);
let (li, hit) = map.query(canonical(b"GGGA")).expect("GGGA must be found");
assert_eq!(li, 1);
assert_eq!(hit.data[0], 2);
}
#[test]
fn query_absent_returns_none() {
set_k(4);
let dir = tempdir().unwrap();
let mut map = LayeredMap::<PersistentCompactIntMatrix>::create(dir.path(), IndexMode::Exact).unwrap();
push_unitigs_and_layer(&mut map, &[b"AAAACGT"], 1);
let absent = canonical(b"CCCC");
assert!(map.query(absent).is_none());
}
#[test]
fn push_layer_from_map_convenience() {
set_k(4);
let dir = tempdir().unwrap();
let mut map = LayeredMap::<PersistentCompactIntMatrix>::create(dir.path(), IndexMode::Exact).unwrap();
let mut w = map.next_layer_writer().unwrap();
w.write(&Unitig::from_ascii(b"AAAACGT")).unwrap();
w.close().unwrap();
let counts: HashMap<CanonicalKmer, u32> = vec![
(canonical(b"AAAA"), 10u32),
].into_iter().collect();
map.push_layer_from_map(&counts).unwrap();
let (_, hit) = map.query(canonical(b"AAAA")).unwrap();
assert_eq!(hit.data[0], 10);
}
// ── detect_layer_content / detect_layer_storage / detect_layer_evidence ────
//
// All built directly on disk (bypassing `LayeredMap<D>::push_layer`, which
// only exists for modes 1 and 2 — mode 3/presence has no `push_layer`),
// then reopened as `LayeredMap<()>` — `()`'s `LayerData::open` never reads
// the matrix, so it's the right handle for exercising the disk-probing
// `detect_layer_*` methods regardless of what content/storage the layer
// underneath actually is.
fn write_unitigs_at(dir: &Path, seqs: &[&[u8]]) {
fs::create_dir_all(dir).unwrap();
let mut w = UnitigFileWriter::create(&dir.join(crate::layer::typed_layer::UNITIGS_FILE)).unwrap();
for s in seqs {
w.write(&Unitig::from_ascii(s)).unwrap();
}
w.close().unwrap();
}
/// Build a one-layer partition root with a presence layer at `layer_0`,
/// save `meta.json`, and reopen it as `LayeredMap<()>`.
fn presence_partition(seqs: &[&[u8]], n_genomes: usize) -> (tempfile::TempDir, LayeredMap<()>) {
let dir = tempdir().unwrap();
let mode = IndexMode::Exact;
let layer0 = layer_dir(dir.path(), 0);
write_unitigs_at(&layer0, seqs);
TypedLayer::<PersistentBitMatrix>::build_presence(&layer0, DEFAULT_BLOCK_BITS, &mode, n_genomes, |kmer, g| {
(kmer.raw().wrapping_add(g as u64)) % 2 == 0
}).unwrap();
PartitionMeta { n_layers: 1, mode }.save(dir.path()).unwrap();
let map = LayeredMap::<()>::open(dir.path()).unwrap();
(dir, map)
}
#[test]
fn detect_layer_content_count() {
set_k(4);
let dir = tempdir().unwrap();
let mut map = LayeredMap::<PersistentCompactIntMatrix>::create(dir.path(), IndexMode::Exact).unwrap();
push_unitigs_and_layer(&mut map, &[b"AAAACGT"], 3);
assert_eq!(map.detect_layer_content(0), LayerContent::Count);
}
#[test]
fn detect_layer_content_presence() {
set_k(4);
let (_dir, map) = presence_partition(&[b"AAATCTA", b"CTTCGCC", b"TGATACG"], 3);
assert_eq!(map.detect_layer_content(0), LayerContent::Presence);
}
#[test]
fn detect_layer_storage_columnar_then_packed_then_sparse() {
set_k(4);
let (dir, map) = presence_partition(&[b"AAATCTA", b"CTTCGCC", b"TGATACG"], 3);
let presence_dir = layer_dir(dir.path(), 0).join("presence");
assert_eq!(map.detect_layer_storage(0).unwrap(), StorageKind::Columnar);
pack_bit_matrix(&presence_dir).unwrap();
assert_eq!(map.detect_layer_storage(0).unwrap(), StorageKind::Packed);
pack_sparse_bit_matrix(&presence_dir).unwrap();
assert_eq!(map.detect_layer_storage(0).unwrap(), StorageKind::Sparse);
}
#[test]
fn detect_layer_storage_implicit() {
set_k(4);
let dir = tempdir().unwrap();
let mode = IndexMode::Exact;
let layer0 = layer_dir(dir.path(), 0);
write_unitigs_at(&layer0, &[b"AAAACGT"]);
// Mode-1 build: MPHF + evidence + `layer_meta.json`, no matrix at all —
// must read back as `Presence`/`Implicit`, never a third "empty" state.
TypedLayer::<()>::build(&layer0, DEFAULT_BLOCK_BITS, &mode).unwrap();
PartitionMeta { n_layers: 1, mode }.save(dir.path()).unwrap();
let map = LayeredMap::<()>::open(dir.path()).unwrap();
assert_eq!(map.detect_layer_content(0), LayerContent::Presence);
assert_eq!(map.detect_layer_storage(0).unwrap(), StorageKind::Implicit);
}
#[test]
fn detect_layer_storage_count_columnar_then_packed() {
set_k(4);
let dir = tempdir().unwrap();
let mut map = LayeredMap::<PersistentCompactIntMatrix>::create(dir.path(), IndexMode::Exact).unwrap();
push_unitigs_and_layer(&mut map, &[b"AAAACGT"], 3);
drop(map);
let map = LayeredMap::<()>::open(dir.path()).unwrap();
assert_eq!(map.detect_layer_storage(0).unwrap(), StorageKind::Columnar);
obicompactvec::pack_compact_int_matrix(&layer_dir(dir.path(), 0).join("counts")).unwrap();
assert_eq!(map.detect_layer_storage(0).unwrap(), StorageKind::Packed);
}
#[test]
fn detect_layer_evidence_exact_vs_approx() {
set_k(4);
let dir = tempdir().unwrap();
let layer0 = layer_dir(dir.path(), 0);
write_unitigs_at(&layer0, &[b"AAAACGT"]);
TypedLayer::<()>::build(&layer0, DEFAULT_BLOCK_BITS, &IndexMode::Exact).unwrap();
PartitionMeta { n_layers: 1, mode: IndexMode::Exact }.save(dir.path()).unwrap();
let map = LayeredMap::<()>::open(dir.path()).unwrap();
assert_eq!(map.detect_layer_evidence(0).unwrap(), EvidenceKind::Exact);
let approx = IndexMode::Approx { b: 8, z: 1 };
let dir2 = tempdir().unwrap();
let layer0b = layer_dir(dir2.path(), 0);
write_unitigs_at(&layer0b, &[b"AAAACGT"]);
TypedLayer::<()>::build(&layer0b, DEFAULT_BLOCK_BITS, &approx).unwrap();
PartitionMeta { n_layers: 1, mode: approx }.save(dir2.path()).unwrap();
let map2 = LayeredMap::<()>::open(dir2.path()).unwrap();
assert_eq!(map2.detect_layer_evidence(0).unwrap(), EvidenceKind::Approx);
}
src/obikindex/src/layer/typed_layer.rs
+41 -84
View File
@@ -1,19 +1,18 @@
use std::collections::HashMap;
use std::fs;
use std::path::{Path, PathBuf};
use crate::layer::utils::layer_dir;
use obicompactvec::{
BinaryMatrix, PersistentBitMatrix, PersistentBitMatrixBuilder, PersistentCompactIntMatrix,
PersistentCompactIntMatrixBuilder, PersistentSparseBitMatrix,
};
use obikseq::CanonicalKmer;
use obiskio::{UnitigFileReader, UnitigFileWriter};
use std::collections::HashMap;
use std::fs;
use std::path::Path;
use crate::layer::error::{OLMError, OLMResult};
use crate::index::error::{OKIError, OKIResult};
use crate::layer::meta::IndexMode;
use crate::layer::mphf_layer::MphfLayer;
pub(crate) use crate::layer::mphf_layer::UNITIGS_FILE;
pub(crate) const COUNTS_DIR: &str = "counts";
pub(crate) const PRESENCE_DIR: &str = "presence";
@@ -21,48 +20,10 @@ pub(crate) const PRESENCE_DIR: &str = "presence";
pub trait LayerData: Sized {
type Item;
fn open(layer_dir: &Path) -> OLMResult<Self>;
fn open(layer_dir: &Path) -> OKIResult<Self>;
fn read(&self, slot: usize) -> Self::Item;
}
/// TypedLayer directory path for layer `i` within a partition's index root — the
/// single source of truth for the on-disk `layer_N` naming convention
/// (mirrors what `LayeredMap::open`/`push_layer` already use internally).
///
/// `obikindex::layer` operates within a single partition's index root; it has
/// no notion of "partition" at all. Turning a partition number into that
/// root is `obikindex::KmerIndex::index_dir`'s job, one layer up — callers
/// here only ever name a layer *number*, never build the path themselves.
pub fn layer_dir(root: &Path, i: usize) -> PathBuf {
root.join(format!("layer_{i}"))
}
/// Superkmer-file extension used by every pre-layer-construction artifact
/// below (`raw`/`dereplicated`) — an implementation detail of the SK file
/// format, never meant to leak as a literal string past this module.
const SK_EXT: &str = "skmer.zst";
/// Path of a layer's raw, not-yet-dereplicated superkmer file — written by
/// whichever algorithm routes superkmers into this layer (today:
/// `obikindexer::algorithms::partitionner::PartitionRouter`), read by whichever
/// algorithm dereplicates it (today:
/// `obikindexer::algorithms::dereplicator::Dereplicator`). Naming this once
/// here, rather than in either algorithm submodule, is what lets the two
/// agree on the filename without depending on each other directly — see
/// `DevDocMD/implementation/partition_layer_cache.md`.
pub fn raw_superkmers_path(layer_dir: &Path) -> PathBuf {
layer_dir.join(format!("raw.{SK_EXT}"))
}
/// Path of a layer's dereplicated superkmer file — written by
/// `obikindexer::algorithms::dereplicator::Dereplicator`, read by whichever
/// algorithm counts kmer abundances from it (today:
/// `obikindexer::algorithms::partitionner::PartitionRouter::count_kmer`) and,
/// later, by `obikindex::build_index_layer` to build the real layer.
pub fn dereplicated_superkmers_path(layer_dir: &Path) -> PathBuf {
layer_dir.join(format!("dereplicated.{SK_EXT}"))
}
/// Opens layer `i`'s data only, skipping the MPHF — for callers that only
/// need matrix-level operations (distance traits, column weights, group
/// filters, sub-matrix extraction) and never look up a kmer for this layer.
@@ -74,13 +35,13 @@ pub fn dereplicated_superkmers_path(layer_dir: &Path) -> PathBuf {
/// *number* within a partition it already knows the root of, the same
/// vocabulary as [`layer_dir`] and `LayeredMap` — never the `layer_N`
/// naming convention itself, which stays private to this crate.
pub fn open_data<D: LayerData>(root: &Path, i: usize) -> OLMResult<D> {
pub(crate) fn open_data<D: LayerData>(root: &Path, i: usize) -> OKIResult<D> {
D::open(&layer_dir(root, i))
}
impl LayerData for () {
type Item = ();
fn open(_layer_dir: &Path) -> OLMResult<Self> {
fn open(_layer_dir: &Path) -> OKIResult<Self> {
Ok(())
}
fn read(&self, _slot: usize) {}
@@ -88,8 +49,8 @@ impl LayerData for () {
impl LayerData for PersistentCompactIntMatrix {
type Item = Box<[u32]>;
fn open(layer_dir: &Path) -> OLMResult<Self> {
PersistentCompactIntMatrix::open(layer_dir).map_err(OLMError::Io)
fn open(layer_dir: &Path) -> OKIResult<Self> {
PersistentCompactIntMatrix::open(layer_dir).map_err(OKIError::Io)
}
fn read(&self, slot: usize) -> Box<[u32]> {
self.row(slot)
@@ -98,8 +59,8 @@ impl LayerData for PersistentCompactIntMatrix {
impl LayerData for PersistentBitMatrix {
type Item = Box<[bool]>;
fn open(layer_dir: &Path) -> OLMResult<Self> {
PersistentBitMatrix::open(layer_dir).map_err(OLMError::Io)
fn open(layer_dir: &Path) -> OKIResult<Self> {
PersistentBitMatrix::open(layer_dir).map_err(OKIError::Io)
}
fn read(&self, slot: usize) -> Box<[bool]> {
self.row(slot)
@@ -108,8 +69,8 @@ impl LayerData for PersistentBitMatrix {
impl LayerData for PersistentSparseBitMatrix {
type Item = Box<[bool]>;
fn open(layer_dir: &Path) -> OLMResult<Self> {
PersistentSparseBitMatrix::open(&layer_dir.join(PRESENCE_DIR)).map_err(OLMError::Io)
fn open(layer_dir: &Path) -> OKIResult<Self> {
PersistentSparseBitMatrix::open(&layer_dir.join(PRESENCE_DIR)).map_err(OKIError::Io)
}
fn read(&self, slot: usize) -> Box<[bool]> {
self.row(slot)
@@ -201,11 +162,7 @@ pub struct Hit<T = ()> {
// ── Common read path ──────────────────────────────────────────────────────────
impl<D: LayerData> TypedLayer<D> {
// `mode` no longer forwarded to `MphfLayer::open` (auto-detected from
// disk now, see mphf_layer.rs) — kept as a parameter for the moment so
// this signature (and `KmerLayer::open`'s call to it) doesn't have to
// change in the same step; dropped when content_layer.rs is done.
pub fn open(path: &Path, _mode: &IndexMode) -> OLMResult<Self> {
pub fn open(path: &Path) -> OKIResult<Self> {
let mphf = MphfLayer::open(path)?;
let data = D::open(path)?;
Ok(Self { mphf, data })
@@ -266,21 +223,21 @@ impl<D: LayerData> TypedLayer<D> {
self.mphf.enumerate_kmers_batch(n)
}
pub fn unitig_writer(out_dir: &Path) -> OLMResult<UnitigFileWriter> {
pub fn unitig_writer(out_dir: &Path) -> OKIResult<UnitigFileWriter> {
MphfLayer::unitig_writer(out_dir)
}
/// Build `unitigs.bin.idx` and `evidence.bin` from `unitigs.bin` and
/// `mphf.bin` already present in `layer_dir`.
/// `block_bits` controls the `.idx` block size (2^block_bits chunks/block).
pub fn build_exact_evidence(layer_dir: &Path, block_bits: u8) -> OLMResult<usize> {
pub fn build_exact_evidence(layer_dir: &Path, block_bits: u8) -> OKIResult<usize> {
MphfLayer::build_exact_evidence(layer_dir, block_bits)
}
/// Build `fingerprint.bin` from `unitigs.bin` and `mphf.bin` already
/// present in `layer_dir`. `b` — fingerprint bits (1..=64); `z` — Findere
/// consecutive k-mer parameter (≥1).
pub fn build_approx_evidence(layer_dir: &Path, b: u8, z: u8) -> OLMResult<usize> {
pub fn build_approx_evidence(layer_dir: &Path, b: u8, z: u8) -> OKIResult<usize> {
MphfLayer::build_approx_evidence(layer_dir, b, z)
}
@@ -311,21 +268,21 @@ impl<D: LayerData + HasStorageKind> TypedLayer<D> {
// ── Mode 1 — set membership ───────────────────────────────────────────────────
impl TypedLayer<()> {
pub fn build(out_dir: &Path, block_bits: u8, mode: &IndexMode) -> OLMResult<usize> {
pub fn build(out_dir: &Path, block_bits: u8, mode: &IndexMode) -> OKIResult<usize> {
MphfLayer::build(out_dir, block_bits, mode, &mut |_, _| Ok(()))
}
/// Create a presence matrix for a set-membership layer (first merge).
pub fn init_presence_matrix(layer_dir: &Path, n_kmers: usize) -> OLMResult<()> {
pub fn init_presence_matrix(layer_dir: &Path, n_kmers: usize) -> OKIResult<()> {
let presence_dir = layer_dir.join(PRESENCE_DIR);
fs::create_dir_all(&presence_dir).map_err(OLMError::Io)?;
fs::create_dir_all(&presence_dir).map_err(OKIError::Io)?;
let mut mb =
PersistentBitMatrixBuilder::new(n_kmers, &presence_dir).map_err(OLMError::Io)?;
PersistentBitMatrixBuilder::new(n_kmers, &presence_dir).map_err(OKIError::Io)?;
mb.add_col_ones()
.map_err(OLMError::Io)?
.map_err(OKIError::Io)?
.close()
.map_err(OLMError::Io)?;
mb.close().map_err(OLMError::Io)
.map_err(OKIError::Io)?;
mb.close().map_err(OKIError::Io)
}
}
@@ -337,18 +294,18 @@ impl TypedLayer<PersistentCompactIntMatrix> {
block_bits: u8,
mode: &IndexMode,
count_of: impl Fn(CanonicalKmer) -> u32,
) -> OLMResult<usize> {
) -> OKIResult<usize> {
let n = UnitigFileReader::open_sequential(&out_dir.join(UNITIGS_FILE))?.n_kmers();
let counts_dir = out_dir.join(COUNTS_DIR);
let mut mb =
PersistentCompactIntMatrixBuilder::new(n, &counts_dir).map_err(OLMError::Io)?;
let mut col = mb.add_col().map_err(OLMError::Io)?;
PersistentCompactIntMatrixBuilder::new(n, &counts_dir).map_err(OKIError::Io)?;
let mut col = mb.add_col().map_err(OKIError::Io)?;
let n_built = MphfLayer::build(out_dir, block_bits, mode, &mut |slot, kmer| {
col.set(slot, count_of(kmer));
Ok(())
})?;
col.close().map_err(OLMError::Io)?;
mb.close().map_err(OLMError::Io)?;
col.close().map_err(OKIError::Io)?;
mb.close().map_err(OKIError::Io)?;
Ok(n_built)
}
@@ -357,7 +314,7 @@ impl TypedLayer<PersistentCompactIntMatrix> {
block_bits: u8,
mode: &IndexMode,
counts: &HashMap<CanonicalKmer, u32>,
) -> OLMResult<usize> {
) -> OKIResult<usize> {
Self::build(out_dir, block_bits, mode, |kmer| {
counts.get(&kmer).copied().unwrap_or(0)
})
@@ -370,9 +327,9 @@ impl TypedLayer<PersistentCompactIntMatrix> {
pub fn append_genome_column(
layer_dir: &Path,
value_of: impl Fn(usize) -> u32,
) -> OLMResult<()> {
) -> OKIResult<()> {
PersistentCompactIntMatrix::append_column(&layer_dir.join(COUNTS_DIR), value_of)
.map_err(OLMError::Io)
.map_err(OKIError::Io)
}
/// Number of genome columns in this layer's count matrix.
@@ -448,9 +405,9 @@ impl TypedLayer<PersistentBitMatrix> {
pub fn append_genome_column(
layer_dir: &Path,
value_of: impl Fn(usize) -> bool,
) -> OLMResult<()> {
) -> OKIResult<()> {
PersistentBitMatrix::append_column(&layer_dir.join(PRESENCE_DIR), value_of)
.map_err(OLMError::Io)
.map_err(OKIError::Io)
}
pub fn build_presence(
@@ -459,13 +416,13 @@ impl TypedLayer<PersistentBitMatrix> {
mode: &IndexMode,
n_genomes: usize,
present_in: impl Fn(CanonicalKmer, usize) -> bool,
) -> OLMResult<usize> {
) -> OKIResult<usize> {
let n = UnitigFileReader::open_sequential(&out_dir.join(UNITIGS_FILE))?.n_kmers();
let presence_dir = out_dir.join(PRESENCE_DIR);
let mut mb = PersistentBitMatrixBuilder::new(n, &presence_dir).map_err(OLMError::Io)?;
let mut mb = PersistentBitMatrixBuilder::new(n, &presence_dir).map_err(OKIError::Io)?;
let mut cols: Vec<_> = (0..n_genomes)
.map(|_| mb.add_col().map_err(OLMError::Io))
.collect::<OLMResult<_>>()?;
.map(|_| mb.add_col().map_err(OKIError::Io))
.collect::<OKIResult<_>>()?;
let n_built = MphfLayer::build(out_dir, block_bits, mode, &mut |slot, kmer| {
for (g, col) in cols.iter_mut().enumerate() {
col.set(slot, present_in(kmer, g));
@@ -473,9 +430,9 @@ impl TypedLayer<PersistentBitMatrix> {
Ok(())
})?;
for col in cols {
col.close().map_err(OLMError::Io)?;
col.close().map_err(OKIError::Io)?;
}
mb.close().map_err(OLMError::Io)?;
mb.close().map_err(OKIError::Io)?;
Ok(n_built)
}
}
src/obikindex/src/layer/utils.rs
+14
View File
@@ -0,0 +1,14 @@
pub(crate) const LAYERNAME_SUFFIX: &str = "layer_";
use std::path::{Path, PathBuf};
/// TypedLayer directory path for layer `i` within a partition's index root — the
/// single source of truth for the on-disk `layer_N` naming convention
/// (mirrors what `LayeredMap::open`/`push_layer` already use internally).
///
/// `obikindex::layer` operates within a single partition's index root; it has
/// no notion of "partition" at all. Turning a partition number into that
/// root is `obikindex::KmerIndex::index_dir`'s job, one layer up — callers
/// here only ever name a layer *number*, never build the path themselves.
pub(crate) fn layer_dir(root: &Path, i: usize) -> PathBuf {
root.join(format!("{LAYERNAME_SUFFIX}{i}"))
}
src/obikindex/src/lib.rs
+6 -6
View File
@@ -13,10 +13,10 @@ pub mod layer;
pub mod partition;
pub use index::{
validate_label, AggOp, DistanceMetric, DistanceOutput, GenomeInfo, GroupFilterParams,
GroupQuorumFilter, IndexBitsPerKmer, IndexBuilder, IndexConfig, IndexMeta, IndexState,
KmerDesc, KmerFilter, KmerIndex, MergeMode, MetaPred, OKIError, OKIResult, OutputCol,
PartitionRunner, QueryHit, QueryStats, META_FILENAME,
materialize_layer, olm_to_sk, write_graph_as_unitigs, passes_all,
GenomeInfo,
IndexBuilder, IndexConfig, IndexMeta, IndexState,
KmerIndex, OKIError, OKIResult,
META_FILENAME,
ColBuilder,
};
pub use index::{filter, meta};
pub use index::meta;
src/obikindex/src/partition/mod.rs
+54 -33
View File
@@ -19,10 +19,13 @@
//! entirely) both still reinvent a fragment of this. Migrating them is a
//! separate, deferred step.
use crate::OKIError;
use std::path::{Path, PathBuf};
use crate::layer::{IndexMode, KmerLayer, OLMResult, layer_dir};
use obikseq::CanonicalKmer;
use crate::{
KmerIndex, OKIResult,
layer::{KmerLayer, utils::layer_dir},
};
/// Partition subdirectory name, under an index's root — the single source
/// of truth for the on-disk `partitions/part_NNNNN` naming convention.
@@ -34,59 +37,77 @@ pub const PARTITIONS_SUBDIR: &str = "partitions";
/// Path of partition `i`'s directory under `root` — `<root>/partitions/part_NNNNN`,
/// zero-padded to 5 digits.
pub fn partition_dir(root: &Path, i: usize) -> PathBuf {
pub(crate) fn partition_dir(root: &Path, i: usize) -> PathBuf {
root.join(PARTITIONS_SUBDIR).join(format!("part_{i:05}"))
}
/// Path of partition `i`'s layered-index directory — `<partition_dir>/index`,
/// the root every [`crate::layer::layer_dir`] call for this partition is
/// relative to.
pub fn index_dir(root: &Path, i: usize) -> PathBuf {
pub(crate) fn index_dir(root: &Path, i: usize) -> PathBuf {
partition_dir(root, i).join("index")
}
/// One partition's open layers, in layer order (layer 0 first).
/// One partition — identified by its own `index_dir`, nothing eagerly
/// opened. `index_dir` is the minimal field `KmerLayer::at` needs to
/// compute a layer's path; added here only because `layer()` below can't
/// be written without it now that there's no `layers: Vec<KmerLayer>` to
/// index into.
pub struct KmerPartition {
layers: Vec<KmerLayer>,
partition_dir: PathBuf,
id: usize,
}
impl KmerPartition {
/// Open every layer under `index_dir` (`index_dir/layer_0`,
/// `index_dir/layer_1`, ... up to `n_layers`), eagerly — not lazily on
/// first access, so the caller pays the mmap cost once, up front,
/// rather than at an unpredictable point during later lookups.
pub fn open(
index_dir: &Path,
mode: &IndexMode,
n_layers: usize,
with_counts: bool,
) -> OLMResult<Self> {
let layers = (0..n_layers)
.map(|l| KmerLayer::open(&layer_dir(index_dir, l), mode, with_counts))
.collect::<OLMResult<Vec<_>>>()?;
Ok(Self { layers })
pub fn new(index: &KmerIndex, i: usize) -> Self {
KmerPartition {
partition_dir: index.dir().join(format!("part_{i:05}")),
id: i,
}
}
pub fn id(&self) -> usize {
self.id
}
/// Number of layers under `index_dir`, found by probing `layer_0`,
/// `layer_1`, ... sequentially until one is missing — a disk scan, not
/// a metadata read (`meta.json`'s own `n_layers` is redundant with this
/// and no longer consulted here). Intrinsically slow: one `exists()`
/// syscall per layer, called fresh every time — not for a hot path.
pub fn n_layers(&self) -> usize {
self.layers.len()
let mut n = 0;
while layer_dir(&self.partition_dir, n).exists() {
n += 1;
}
n
}
pub fn layer(&self, i: usize) -> &KmerLayer {
&self.layers[i]
/// Evidence mode for this partition, detected from layer 0's evidence
/// files on disk (see [`crate::layer::IndexMode::detect`]) — never
/// persisted as metadata, since every layer within a partition shares
/// the same mode by construction.
pub fn mode(&self) -> OKIResult<crate::layer::IndexMode> {
crate::layer::IndexMode::detect(&self.partition_dir)
}
pub fn layers(&self) -> &[KmerLayer] {
&self.layers
pub fn layer(&self, i: usize) -> OKIResult<KmerLayer> {
let n = self.n_layers();
if i >= n {
return Err(OKIError::Layer(obiskio::SKError::InvalidData {
context: "New KmerLayer",
detail: format!("Layer {i} requested, Partition has only {n} layer"),
}));
}
Ok(KmerLayer::new(self, i))
}
/// Existence lookup of `kmer` across this partition's layers: tries
/// each in turn, stopping at the first hit and reporting which layer it
/// was — `find_slot`, not a data read, for a plain existence check.
pub fn find(&self, kmer: CanonicalKmer) -> Option<usize> {
self.layers
.iter()
.enumerate()
.find_map(|(li, layer)| layer.find_slot(kmer).map(|_| li))
pub fn layer_dir(&self, i: usize) -> OKIResult<PathBuf> {
Ok(self.layer(i)?.dir().to_path_buf())
}
pub fn dir(&self) -> &Path {
&self.partition_dir
}
}
src/obikindexer/src/algorithms/layer_builder/mod.rs
+1 -1
View File
@@ -18,7 +18,7 @@ use crate::extensions::PrivateBuilder;
/// The actual per-partition construction (De Bruijn graph, unitigs, MPHF,
/// matrix) stays a `KmerIndex` method (`build_index_layer`) — unlike
/// `Dereplicator`/`Counter`, it depends on several `obikindex`-internal
/// helpers (`graph_pipeline`, `common::olm_to_sk`) already shared with
/// helpers (`graph_pipeline`) already shared with
/// `merge`/`select`/`rebuild`'s own layer-construction paths, so it
/// belongs there, not duplicated or newly exported just for this
/// algorithm. `LayerBuilder`'s own job is the orchestration around it:
src/obikindexer/src/extensions/mod.rs
+4 -6
View File
@@ -36,7 +36,7 @@ use obidebruinj::GraphDeBruijn;
use obikindex::layer::IndexMode;
use obikindex::layer::{TypedLayer, meta::PartitionMeta};
use obikindex::{KmerIndex, OKIError, OKIResult};
use obikindex::{materialize_layer, olm_to_sk, write_graph_as_unitigs};
use crate::graph_pipeline::{materialize_layer, write_graph_as_unitigs};
use obiskio::{SKError, SKFileMeta, SKFileReader};
use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xx64};
@@ -109,7 +109,7 @@ impl PrivateBuilder for KmerIndex {
.map(|g| g.label.as_str())
.unwrap_or("unknown")
.to_owned();
let spectrums_dir = self.root_path().join("spectrums");
let spectrums_dir = self.dir().join("spectrums");
fs::create_dir_all(&spectrums_dir)?;
let path = spectrums_dir.join(format!("{label}.json"));
let spectrum_map: BTreeMap<String, u64> = counts
@@ -194,8 +194,7 @@ impl PrivateBuilder for KmerIndex {
(Some(mphf), Some(counts)) => counts.get(mphf.index(&kmer.raw())),
_ => 1,
},
)
.map_err(|e| olm_to_sk(e, "layer build"))?;
)?;
n
} else {
materialize_layer(g, &layer0_dir, block_bits, mode)?
@@ -206,8 +205,7 @@ impl PrivateBuilder for KmerIndex {
n_layers: 1,
mode: mode.clone(),
}
.save(index_dir)
.map_err(|e| olm_to_sk(e, "layer build"))?;
.save(index_dir)?;
Ok(n_kmers)
}
src/obikindex/src/index/graph_pipeline.rs → src/obikindexer/src/graph_pipeline.rs
+10 -12
View File
@@ -10,10 +10,9 @@ use obipipeline::{
use obidebruinj::GraphDeBruijn;
use obikseq::CanonicalKmer;
use crate::layer::{IndexMode, TypedLayer};
use obiskio::{SKError, SKResult};
use obikindex::layer::{IndexMode, TypedLayer};
use obikindex::{OKIError, OKIResult};
use crate::index::common::olm_to_sk;
// ── KmerGraphData ─────────────────────────────────────────────────────────────
@@ -29,16 +28,16 @@ enum KmerGraphData {
///
/// `flat_fn(path, emit)`: opens path, iterates kmers, calls `emit(batch)` for each batch.
/// `filter(kmer) -> bool`: secondary filter applied in the Transform stage.
pub(crate) fn build_graph<I, F, G>(
pub fn build_graph<I, F, G>(
file_source: I,
flat_fn: F,
filter: G,
n_workers: usize,
max_open: usize,
) -> SKResult<GraphDeBruijn>
) -> OKIResult<GraphDeBruijn>
where
I: Iterator<Item = PathBuf> + Send + 'static,
F: Fn(PathBuf, &mut dyn FnMut(Vec<CanonicalKmer>)) -> SKResult<()> + Send + Sync + 'static,
F: Fn(PathBuf, &mut dyn FnMut(Vec<CanonicalKmer>)) -> OKIResult<()> + Send + Sync + 'static,
G: Fn(CanonicalKmer) -> bool + Send + Sync + 'static,
{
let capacity = 2;
@@ -47,7 +46,7 @@ where
let filter = Arc::new(filter);
let g_shared = Arc::new(Mutex::new(GraphDeBruijn::new()));
let g_sink = Arc::clone(&g_shared);
let err_cap: Arc<Mutex<Option<SKError>>> = Arc::new(Mutex::new(None));
let err_cap: Arc<Mutex<Option<OKIError>>> = Arc::new(Mutex::new(None));
let err_flat = Arc::clone(&err_cap);
let throttled = throttle(file_source, max_open);
@@ -121,11 +120,11 @@ where
/// Phase 2 (write unitigs only): compute degrees, write unitigs to `layer_dir`, drop graph.
///
/// Returns n_kmers. Does NOT build the MPHF — caller does it.
pub fn write_graph_as_unitigs(g: GraphDeBruijn, layer_dir: &Path) -> SKResult<usize> {
pub fn write_graph_as_unitigs(g: GraphDeBruijn, layer_dir: &Path) -> OKIResult<usize> {
let n_kmers = g.len();
g.compute_degrees_and_mark_starts();
std::fs::create_dir_all(layer_dir)?;
let mut uw = TypedLayer::<()>::unitig_writer(layer_dir).map_err(|e| olm_to_sk(e, "graph pipeline"))?;
let mut uw = TypedLayer::<()>::unitig_writer(layer_dir)?;
g.try_for_each_unitig(|unitig| uw.write(unitig))?;
uw.close()?;
drop(g);
@@ -142,11 +141,10 @@ pub fn materialize_layer(
layer_dir: &Path,
block_bits: u8,
evidence: &IndexMode,
) -> SKResult<usize> {
) -> OKIResult<usize> {
let n = write_graph_as_unitigs(g, layer_dir)?;
debug!("materialize_layer: unitigs written ({n} kmers), building MPHF");
TypedLayer::<()>::build(layer_dir, block_bits, evidence)
.map_err(|e| olm_to_sk(e, "graph pipeline"))?;
TypedLayer::<()>::build(layer_dir, block_bits, evidence)?;
debug!("materialize_layer: MPHF build done");
Ok(n)
}
src/obikindexer/src/lib.rs
+3
View File
@@ -14,3 +14,6 @@
pub mod algorithms;
pub(crate) mod extensions;
pub mod graph_pipeline;
pub use graph_pipeline::{build_graph, materialize_layer, write_graph_as_unitigs};
src/obikmer/Cargo.toml
+3 -1
View File
@@ -19,6 +19,8 @@ obisys = { path = "../obisys" }
obiskio = { path = "../obiskio" }
obikindex = { path = "../obikindex", default-features = false }
obikindexer = { path = "../obikindexer" }
obikfilter = { path = "../obikfilter" }
obikstats = { path = "../obikstats" }
obikalgorithm = { path = "../obikalgorithm" }
obikphylo = { path = "../obikphylo" }
obitaxonomy = { path = "../obitaxonomy" }
@@ -37,5 +39,5 @@ pprof = { version = "0.15", features = ["prost-codec"], optional = true }
[features]
default = ["numa"]
numa = ["obikindex/numa"]
numa = ["obisys/numa"]
profiling = ["dep:pprof"]
src/obikmer/src/cmd/index/mod.rs
+35 -23
View File
@@ -2,13 +2,13 @@ use std::path::PathBuf;
use std::time::Instant;
use clap::Args;
use obikalgorithm::Algorithm;
use obikindex::layer::IndexMode;
use obikindex::{GenomeInfo, IndexBuilder, IndexConfig, IndexState, KmerIndex};
use obikindexer::algorithms::counter::Counter;
use obikindexer::algorithms::dereplicator::Dereplicator;
use obikindexer::algorithms::layer_builder::LayerBuilder;
use obikindexer::algorithms::partitionner::PartitionRouter;
use obikindex::{validate_label, GenomeInfo, IndexBuilder, IndexConfig, IndexState, KmerIndex};
use obikindex::layer::IndexMode;
use obikalgorithm::Algorithm;
fn current_state(idx: &KmerIndex) -> IndexState {
idx.state().unwrap_or_else(|e| {
@@ -18,10 +18,12 @@ fn current_state(idx: &KmerIndex) -> IndexState {
}
fn parse_key_value(s: &str) -> Result<(String, String), String> {
let pos = s.find('=').ok_or_else(|| format!("invalid key=value: no '=' in '{s}'"))?;
let pos = s
.find('=')
.ok_or_else(|| format!("invalid key=value: no '=' in '{s}'"))?;
Ok((s[..pos].to_string(), s[pos + 1..].to_string()))
}
use obisys::{progress_bar, spinner, Progress, Reporter, Stage};
use obisys::{Progress, Reporter, Stage, progress_bar, spinner};
use tracing::info;
use crate::cli::{CommonArgs, block_size_to_bits, partitions_to_bits};
@@ -108,9 +110,7 @@ pub(crate) fn resolve_approx_params(
const DEFAULT_B: u8 = 8;
const DEFAULT_Z: u8 = 1;
let bits_needed = |fp: f64| -> u8 {
(-fp.log2()).ceil() as u8
};
let bits_needed = |fp: f64| -> u8 { (-fp.log2()).ceil() as u8 };
match (z_opt, b_opt, fp_opt) {
// All three given: use b and z, recompute fp conservatively.
@@ -198,7 +198,7 @@ pub fn run(args: IndexArgs) {
};
// ── Open or create the index ─────────────────────────────────────────────
if KmerIndex::exists(&output) {
if KmerIndex::is_an_index(&output) {
if !args.force {
eprintln!(
"error: an index already exists at {} (use --force to overwrite it)",
@@ -211,23 +211,32 @@ pub fn run(args: IndexArgs) {
eprintln!("error removing existing index: {e}");
std::process::exit(1);
});
} else if output.exists() {
eprintln!(
"error: {} exists but is not an obikmer index, it cannot be deleted",
output.display()
);
std::process::exit(1);
}
let n_bits = partitions_to_bits(args.common.partitions);
let effective = 1usize << n_bits;
if effective != args.common.partitions {
info!("partitions: {} → {} (next power of 2)", args.common.partitions, effective);
info!(
"partitions: {} → {} (next power of 2)",
args.common.partitions, effective
);
}
let block_bits = block_size_to_bits(args.block_size);
let config = IndexConfig {
kmer_size: effective_kmer_size,
kmer_size: effective_kmer_size,
minimizer_size: args.common.minimizer_size,
n_bits,
with_counts: args.with_counts,
evidence: evidence.clone(),
with_counts: args.with_counts,
evidence: evidence.clone(),
block_bits,
};
let genome_info = args.label.as_ref().map(|label| {
validate_label(label).unwrap_or_else(|e| {
GenomeInfo::validate_label(label).unwrap_or_else(|e| {
eprintln!("error: --label: {e}");
std::process::exit(1);
});
@@ -242,7 +251,6 @@ pub fn run(args: IndexArgs) {
std::process::exit(1);
});
// ── Stage 1: scatter ─────────────────────────────────────────────────────
if current_state(&idx) < IndexState::Scattered {
let n_workers = args.common.threads.max(1);
@@ -272,19 +280,23 @@ pub fn run(args: IndexArgs) {
last_bases = p.position;
let bp = p.position as f64;
let (count_str, rate_str) = if bp >= 1e9 {
(format!("{:.2} Gbp", bp / 1e9), format!("{:.0} Mbp/s", ema_rate / 1e6))
(
format!("{:.2} Gbp", bp / 1e9),
format!("{:.0} Mbp/s", ema_rate / 1e6),
)
} else {
(format!("{:.0} Mbp", bp / 1e6), format!("{:.0} Mbp/s", ema_rate / 1e6))
(
format!("{:.0} Mbp", bp / 1e6),
format!("{:.0} Mbp/s", ema_rate / 1e6),
)
};
pb.set_message(format!("{count_str} {rate_str}"));
});
router
.run()
.unwrap_or_else(|e| {
eprintln!("error: {e}");
std::process::exit(1);
});
router.run().unwrap_or_else(|e| {
eprintln!("error: {e}");
std::process::exit(1);
});
pb.finish_and_clear();
rep.push(t.stop());
drop(router); // ends the borrow of `idx` early — `PartitionRouter`'s `Drop` impl would otherwise extend it to the end of scope (`run()` already called `close()`, which marks scatter done, internally)
src/obikmer/src/cmd/predicate.rs
+2 -2
View File
@@ -1,6 +1,6 @@
use clap::Args;
use obikindex::{GroupFilterParams, IndexMeta, MetaPred};
use obikindex::KmerFilter;
use obikindex::IndexMeta;
use obikfilter::{GroupFilterParams, KmerFilter, MetaPred};
/// CLI args for ingroup/outgroup filtering — embeddable in any command via `#[command(flatten)]`.
#[derive(Args)]
src/obikmer/src/cmd/utils/maintenance.rs
+3 -2
View File
@@ -1,6 +1,7 @@
use std::path::PathBuf;
use obikindex::{validate_label, IndexBitsPerKmer, KmerIndex};
use obikindex::{GenomeInfo, KmerIndex};
use obikstats::IndexBitsPerKmer;
use tracing::info;
pub(super) fn run_stats(index_path: &PathBuf) {
@@ -76,7 +77,7 @@ pub(super) fn run_rename(index_path: &PathBuf, spec: &str) {
std::process::exit(1);
});
validate_label(&new_label).unwrap_or_else(|e| {
GenomeInfo::validate_label(&new_label).unwrap_or_else(|e| {
eprintln!("error: --new-label: {e}");
std::process::exit(1);
});
src/obikmer2/Cargo.toml
+24
View File
@@ -0,0 +1,24 @@
[package]
name = "obikmer2"
version = "1.2.2"
edition = "2024"
[[bin]]
name = "obikmer2"
path = "src/main.rs"
[dependencies]
obikseq = { path = "../obikseq" }
obiread = { path = "../obiread" }
obipipeline = { path = "../obipipeline" }
obisys = { path = "../obisys" }
obikindex = { path = "../obikindex", default-features = false }
obikindexer = { path = "../obikindexer" }
obikalgorithm = { path = "../obikalgorithm" }
clap = { version = "4", features = ["derive"] }
tracing = "0.1.44"
tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] }
[features]
default = ["numa"]
numa = ["obisys/numa"]
src/obikmer2/src/cli.rs
+114
View File
@@ -0,0 +1,114 @@
use std::path::PathBuf;
use clap::Args;
use obiread::NucPage;
use obikseq::RoutableSuperKmer;
use obipipeline::Throttled;
// ── Shared arguments ──────────────────────────────────────────────────────────
#[derive(Args)]
pub struct CommonArgs {
/// Input files or directories (FASTA/FASTQ, optionally gzip-compressed).
/// If omitted, reads from stdin.
#[arg(num_args = 0..)]
pub inputs: Vec<String>,
/// k-mer size
#[arg(short, long, default_value_t = 31)]
pub kmer_size: usize,
/// Minimizer size
#[arg(short, long, default_value_t = 11)]
pub minimizer_size: usize,
/// Entropy threshold (k-mers with score ≤ theta are rejected)
#[arg(long, default_value_t = 0.7)]
pub theta: f64,
/// Maximum sub-word size for entropy computation
#[arg(long, default_value_t = 6)]
pub level_max: usize,
/// Number of partitions (rounded up to the next power of 2)
#[arg(short, long, default_value_t = 256)]
pub partitions: usize,
/// Number of worker threads
#[arg(
short = 'T',
long,
default_value_t = obisys::effective_parallelism()
)]
pub threads: usize,
/// Maximum number of input files open simultaneously.
/// Defaults to threads/4 (minimum 1). Keep below the number of workers
/// to ensure CPU workers are always available for the transform stage.
#[arg(long)]
pub max_open_files: Option<usize>,
}
/// Smallest `b` such that `2^b >= n` (i.e. `n.next_power_of_two().ilog2()`).
/// Minimum 1 (degenerate n=0 or n=1 → 1 partition).
pub fn partitions_to_bits(n: usize) -> usize {
n.max(1).next_power_of_two().trailing_zeros() as usize
}
/// Convert a block size (number of unitigs per block) to its `block_bits` exponent.
/// `block_size=1` → `block_bits=0` (one entry per unitig, O(1) random access).
pub fn block_size_to_bits(n: usize) -> u8 {
n.max(1).next_power_of_two().trailing_zeros() as u8
}
impl CommonArgs {
/// Validate k and m constraints. Exits on error.
pub fn validate(&self) {
let k = self.kmer_size;
let m = self.minimizer_size;
if k < 11 || k > 31 {
eprintln!("error: --kmer-size must be in [11, 31] (got {k})");
std::process::exit(1);
}
if k % 2 == 0 {
eprintln!("error: --kmer-size must be odd (got {k}); even k allows palindromic k-mers");
std::process::exit(1);
}
if m < 3 || m >= k {
eprintln!("error: --minimizer-size must be in [3, k−1] = [3, {}] (got {m})", k - 1);
std::process::exit(1);
}
if m % 2 == 0 {
eprintln!("error: --minimizer-size must be odd (got {m})");
std::process::exit(1);
}
}
pub fn effective_max_open(&self) -> usize {
self.max_open_files
.unwrap_or_else(|| (self.threads / 4).max(1))
.max(1)
}
pub fn seqfile_paths(&self) -> obiread::PathIter {
let paths: Vec<PathBuf> = if self.inputs.is_empty() {
vec![PathBuf::from("-")]
} else {
self.inputs.iter().map(PathBuf::from).collect()
};
obiread::PathIter::new(paths)
}
}
// ── Pipeline data carrier ─────────────────────────────────────────────────────
pub enum PipelineData {
Path(Throttled<PathBuf>),
NucPage(NucPage),
Batch(Vec<RoutableSuperKmer>),
}
unsafe impl Send for PipelineData {}
unsafe impl Sync for PipelineData {}
src/obikmer2/src/cmd/index/mod.rs
+363
View File
@@ -0,0 +1,363 @@
use std::path::PathBuf;
use std::time::Instant;
use clap::Args;
use obikalgorithm::Algorithm;
use obikindex::layer::IndexMode;
use obikindex::{GenomeInfo, IndexBuilder, IndexConfig, IndexState, KmerIndex};
use obikindexer::algorithms::counter::Counter;
use obikindexer::algorithms::dereplicator::Dereplicator;
use obikindexer::algorithms::layer_builder::LayerBuilder;
use obikindexer::algorithms::partitionner::PartitionRouter;
fn current_state(idx: &KmerIndex) -> IndexState {
idx.state().unwrap_or_else(|e| {
eprintln!("error reading index metadata: {e}");
std::process::exit(1);
})
}
fn parse_key_value(s: &str) -> Result<(String, String), String> {
let pos = s
.find('=')
.ok_or_else(|| format!("invalid key=value: no '=' in '{s}'"))?;
Ok((s[..pos].to_string(), s[pos + 1..].to_string()))
}
use obisys::{Progress, Reporter, Stage, progress_bar, spinner};
use tracing::info;
use crate::cli::{CommonArgs, block_size_to_bits, partitions_to_bits};
#[derive(Args)]
pub struct IndexArgs {
/// Output index directory
#[arg(short, long)]
pub output: PathBuf,
/// Overwrite output directory if it already exists
#[arg(long, default_value_t = false)]
pub force: bool,
/// Genome label (default: input filename without path/extension)
#[arg(long)]
pub label: Option<String>,
/// Genome categorical metadata as key=value pairs (repeatable)
#[arg(long = "meta", value_parser = parse_key_value)]
pub meta: Vec<(String, String)>,
/// Minimum kmer abundance (inclusive)
#[arg(long, default_value_t = 1)]
pub min_abundance: u32,
/// Maximum kmer abundance (inclusive)
#[arg(long)]
pub max_abundance: Option<u32>,
/// Store kmer counts in the index (default: set membership only)
#[arg(long, default_value_t = false)]
pub with_counts: bool,
/// Keep intermediate build files (dereplicated superkmers, mphf1, counts1)
#[arg(long, default_value_t = false)]
pub keep_intermediate: bool,
/// Use approximate (fingerprint-based) evidence instead of exact evidence.
/// False-positive rate per z-window: 1/2^(b·z).
#[arg(long, default_value_t = false)]
pub approx: bool,
/// Findere z parameter: number of consecutive k-mers that must all match.
/// Effective indexed k-mer size is kmer_size - z + 1.
#[arg(short = 'z', long, default_value = None)]
pub findere_z: Option<u8>,
/// Fingerprint bits per slot (b). FP per z-window = 1/2^(b·z).
#[arg(long, default_value = None)]
pub evidence_bits: Option<u8>,
/// Target false-positive rate per z-window (e.g. 0.01).
/// Used to derive missing b or z.
#[arg(long, default_value = None)]
pub fp: Option<f64>,
/// Block size for exact evidence `.idx` (number of unitigs per block).
/// Must be a power of two; rounded up if not. Default 1 = O(1) random access.
#[arg(long, default_value_t = 1)]
pub block_size: usize,
#[command(flatten)]
pub common: CommonArgs,
}
/// Resolve the (z, b, fp) triplet from the user-supplied subset.
///
/// Model: FP = 1/2^(b·z) ⟹ b·z = ⌈-log₂(fp)⌉
///
/// Rules when one value is missing (conservative = ceiling):
/// given z, b → fp = 1/2^(b·z)
/// given z, fp → b = ⌈-log₂(fp) / z⌉
/// given b, fp → z = ⌈-log₂(fp) / b⌉
/// given z only → b = 8 (default), fp derived
/// given b only → z = 1 (default), fp derived
/// given fp only → b = 8 (default), z derived
/// none given → z = 1, b = 8, fp = 1/256
pub(crate) fn resolve_approx_params(
z_opt: Option<u8>,
b_opt: Option<u8>,
fp_opt: Option<f64>,
) -> (u8, u8, f64) {
const DEFAULT_B: u8 = 8;
const DEFAULT_Z: u8 = 1;
let bits_needed = |fp: f64| -> u8 { (-fp.log2()).ceil() as u8 };
match (z_opt, b_opt, fp_opt) {
// All three given: use b and z, recompute fp conservatively.
(Some(z), Some(b), Some(_fp)) => {
let fp = 1.0_f64 / (1u64 << (b as u32 * z as u32)) as f64;
(z, b, fp)
}
// Two given, derive third.
(Some(z), Some(b), None) => {
let fp = 1.0_f64 / (1u64 << (b as u32 * z as u32)) as f64;
(z, b, fp)
}
(Some(z), None, Some(fp)) => {
let bz = (-fp.log2()).ceil() as u32;
let b = ((bz + z as u32 - 1) / z as u32).max(1) as u8;
let actual_fp = 1.0_f64 / (1u64 << (b as u32 * z as u32)) as f64;
(z, b, actual_fp)
}
(None, Some(b), Some(fp)) => {
let bz = (-fp.log2()).ceil() as u32;
let z = ((bz + b as u32 - 1) / b as u32).max(1) as u8;
let actual_fp = 1.0_f64 / (1u64 << (b as u32 * z as u32)) as f64;
(z, b, actual_fp)
}
// One given, apply defaults for the other.
(Some(z), None, None) => {
let b = DEFAULT_B;
let fp = 1.0_f64 / (1u64 << (b as u32 * z as u32)) as f64;
(z, b, fp)
}
(None, Some(b), None) => {
let z = DEFAULT_Z;
let fp = 1.0_f64 / (1u64 << (b as u32 * z as u32)) as f64;
(z, b, fp)
}
(None, None, Some(fp)) => {
let b = DEFAULT_B;
let z = ((bits_needed(fp) as u32 + b as u32 - 1) / b as u32).max(1) as u8;
let actual_fp = 1.0_f64 / (1u64 << (b as u32 * z as u32)) as f64;
(z, b, actual_fp)
}
// None given: defaults.
(None, None, None) => {
let b = DEFAULT_B;
let z = DEFAULT_Z;
let fp = 1.0_f64 / (1u64 << (b as u32 * z as u32)) as f64;
(z, b, fp)
}
}
}
pub fn run(args: IndexArgs) {
args.common.validate();
let output = args.output.clone();
let mut rep = Reporter::new();
// Locked for the whole build (including a possible --force removal +
// recreation below): a second `index` run resuming/overwriting the same
// output directory concurrently would otherwise corrupt it. Unlinking
// the lock file via --force's remove_dir_all is safe — the held file
// descriptor keeps the lock regardless of the directory entry.
let _lock = obisys::DirLock::acquire(&output).unwrap_or_else(|e| {
eprintln!("error locking output directory {}: {e}", output.display());
std::process::exit(1);
});
// ── Resolve evidence kind ────────────────────────────────────────────────
let (evidence, effective_kmer_size) = if args.approx {
let (z, b, fp) = resolve_approx_params(args.findere_z, args.evidence_bits, args.fp);
let k = args.common.kmer_size;
if z as usize >= k {
eprintln!(
"error: Findere z={z} must be < kmer-size={k} \
(effective kmer size k−z+1 = {} ≤ 0)",
k as isize - z as isize + 1
);
std::process::exit(1);
}
let s = k - z as usize + 1;
info!("approximate evidence: b={b}, z={z}, fp={fp:.2e}, indexed kmer size={s}");
(IndexMode::Approx { b, z }, s)
} else {
(IndexMode::Exact, args.common.kmer_size)
};
// ── Open or create the index ─────────────────────────────────────────────
if KmerIndex::is_an_index(&output) {
if !args.force {
eprintln!(
"error: an index already exists at {} (use --force to overwrite it)",
output.display()
);
std::process::exit(1);
}
info!("--force: removing existing index at {}", output.display());
std::fs::remove_dir_all(&output).unwrap_or_else(|e| {
eprintln!("error removing existing index: {e}");
std::process::exit(1);
});
} else if output.exists() {
eprintln!(
"error: {} exists but is not an obikmer index, it cannot be deleted",
output.display()
);
std::process::exit(1);
}
let n_bits = partitions_to_bits(args.common.partitions);
let effective = 1usize << n_bits;
if effective != args.common.partitions {
info!(
"partitions: {} → {} (next power of 2)",
args.common.partitions, effective
);
}
let block_bits = block_size_to_bits(args.block_size);
let config = IndexConfig {
kmer_size: effective_kmer_size,
minimizer_size: args.common.minimizer_size,
n_bits,
with_counts: args.with_counts,
evidence: evidence.clone(),
block_bits,
};
let genome_info = args.label.as_ref().map(|label| {
GenomeInfo::validate_label(label).unwrap_or_else(|e| {
eprintln!("error: --label: {e}");
std::process::exit(1);
});
let mut info = GenomeInfo::new(label.clone());
for (k, v) in &args.meta {
info.meta.insert(k.clone(), v.clone());
}
info
});
let idx = KmerIndex::create(&output, config, genome_info).unwrap_or_else(|e| {
eprintln!("error creating index: {e}");
std::process::exit(1);
});
// ── Stage 1: scatter ─────────────────────────────────────────────────────
if current_state(&idx) < IndexState::Scattered {
let n_workers = args.common.threads.max(1);
let max_open = args.common.effective_max_open();
let t = Stage::start("scatter");
let pb = spinner("scatter");
let mut ema_rate: f64 = 0.0;
let mut last_t = Instant::now();
let mut last_bases: u64 = 0;
const ALPHA: f64 = 0.15;
let mut router = PartitionRouter::new(&idx)
.level_max(args.common.level_max)
.theta(args.common.theta)
.workers(n_workers)
.max_open(max_open)
.files(args.common.seqfile_paths())
.on_progress(|p: Progress| {
let now = Instant::now();
let dt = now.duration_since(last_t).as_secs_f64();
if dt > 0.0 {
let instant = (p.position - last_bases) as f64 / dt;
ema_rate = ALPHA * instant + (1.0 - ALPHA) * ema_rate;
}
last_t = now;
last_bases = p.position;
let bp = p.position as f64;
let (count_str, rate_str) = if bp >= 1e9 {
(
format!("{:.2} Gbp", bp / 1e9),
format!("{:.0} Mbp/s", ema_rate / 1e6),
)
} else {
(
format!("{:.0} Mbp", bp / 1e6),
format!("{:.0} Mbp/s", ema_rate / 1e6),
)
};
pb.set_message(format!("{count_str} {rate_str}"));
});
router.run().unwrap_or_else(|e| {
eprintln!("error: {e}");
std::process::exit(1);
});
pb.finish_and_clear();
rep.push(t.stop());
drop(router); // ends the borrow of `idx` early — `PartitionRouter`'s `Drop` impl would otherwise extend it to the end of scope (`run()` already called `close()`, which marks scatter done, internally)
} else {
info!("scatter already done, skipping");
}
// ── Stage 2: dereplicate + count ─────────────────────────────────────────
if current_state(&idx) < IndexState::Counted {
let t = Stage::start("dereplicate");
let pb = progress_bar("dereplication", idx.n_partitions() as u64, "partitions");
Dereplicator::new(&idx)
.on_progress(|_: Progress| pb.inc(1))
.run()
.unwrap_or_else(|e| {
eprintln!("error: {e}");
std::process::exit(1);
});
pb.finish_and_clear();
rep.push(t.stop());
let t = Stage::start("count_kmer");
let pb = progress_bar("counting", idx.n_partitions() as u64, "partitions");
// `Counter::run` writes `spectrums/{label}.json` and marks count
// done (`count.done`) internally once every partition succeeds.
Counter::new(&idx)
.keep_partial(args.keep_intermediate)
.on_progress(|_: Progress| pb.inc(1))
.run()
.unwrap_or_else(|e| {
eprintln!("error: {e}");
std::process::exit(1);
});
pb.finish_and_clear();
rep.push(t.stop());
} else {
info!("dereplicate+count already done, skipping");
}
// ── Stage 3: build layered index ─────────────────────────────────────────
if current_state(&idx) < IndexState::Indexed {
let t = Stage::start("index");
let pb = progress_bar("index", idx.n_partitions() as u64, "partitions");
let total_kmers = LayerBuilder::new(&idx)
.min_abundance(args.min_abundance)
.max_abundance(args.max_abundance)
.keep_intermediate(args.keep_intermediate)
.on_progress(|_: Progress| pb.inc(1))
.run()
.unwrap_or_else(|e| {
eprintln!("error: {e}");
std::process::exit(1);
});
pb.finish_and_clear();
info!("done — {total_kmers} total kmers indexed");
rep.push(t.stop());
// `LayerBuilder::run` marks the index done (`index.done`) internally
// once every partition succeeds.
} else {
info!("index already built, skipping");
}
rep.print();
}
src/obikmer2/src/cmd/mod.rs
+1
View File
@@ -0,0 +1 @@
pub mod index;
src/obikmer2/src/main.rs
+32
View File
@@ -0,0 +1,32 @@
mod cli;
mod cmd;
use clap::{Parser, Subcommand};
use tracing_subscriber::{EnvFilter, fmt};
#[derive(Parser)]
#[command(name = "obikmer2", about = "DNA k-mer tools", version)]
struct Cli {
#[command(subcommand)]
command: Commands,
}
#[derive(Subcommand)]
enum Commands {
/// Build the complete genome index (scatter → dereplicate → count → layered MPHF)
Index(cmd::index::IndexArgs),
}
fn main() {
fmt()
.with_env_filter(
EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")),
)
.with_writer(std::io::stderr)
.init();
let cli = Cli::parse();
match cli.command {
Commands::Index(args) => cmd::index::run(args),
}
}
src/obikmerge/Cargo.toml
+15
View File
@@ -0,0 +1,15 @@
[package]
name = "obikmerge"
version = "0.1.0"
edition = "2024"
[dependencies]
obikindex = { path = "../obikindex" }
obikindexer = { path = "../obikindexer" }
obikfilter = { path = "../obikfilter" }
obicompactvec = { path = "../obicompactvec" }
obiskio = { path = "../obiskio" }
obikseq = { path = "../obikseq" }
obipipeline = { path = "../obipipeline" }
obisys = { path = "../obisys" }
tracing = "0.1.44"
src/obikmerge/src/lib.rs
+17
View File
@@ -0,0 +1,17 @@
//! Merging multiple `obikindex::KmerIndex` sources into one: bootstrapping
//! from the first source, then merging each remaining source partition by
//! partition (de Bruijn graph union + column fill). A maintenance/
//! transformation operation on already-built indexes, not part of the
//! `Index { Partition { Layer } }` data model itself — kept out of
//! `obikindex` so that crate doesn't grow this algorithm's own
//! dependencies (`obipipeline`, `obidebruinj` via `obikindexer`).
//!
//! [`merge_layer`] holds the per-partition merge primitive
//! (`merge_partition`, `SrcLayerData`) — the latter is also reused by
//! `obikrebuild::rebuild_layer`, hence its `pub` visibility here.
mod merge;
mod merge_layer;
pub use merge::*;
pub use merge_layer::{MergeMode, SrcLayerData};
src/obikindex/src/index/merge.rs → src/obikmerge/src/merge.rs
+9 -8
View File
@@ -3,19 +3,20 @@ use std::fs;
use std::io;
use std::path::Path;
use crate::index::builder::IndexBuilder;
use obikindex::IndexBuilder;
use obisys::{Reporter, Stage, progress_bar, spinner};
use tracing::{debug, info};
use crate::layer::IndexMode;
use obikindex::layer::IndexMode;
use crate::index::error::{OKIError, OKIResult};
use crate::index::kmer_index::KmerIndex;
use crate::index::meta::{GenomeInfo, IndexMeta};
use crate::index::state::IndexState;
use obikindex::{OKIError, OKIResult};
use obikindex::KmerIndex;
use obikindex::{GenomeInfo, IndexMeta};
use obikindex::IndexState;
use obisys::PartitionRunner;
pub use crate::index::merge_layer::MergeMode;
pub use crate::merge_layer::MergeMode;
// ── per-partition diagnostic record ──────────────────────────────────────────
@@ -229,7 +230,7 @@ impl KmerIndex {
let srcs = &srcs;
let evidence = &evidence;
let runner = crate::index::numa::PartitionRunner::new();
let runner = PartitionRunner::new();
let mut part_stats: Vec<PartStat> = Vec::with_capacity(n_partitions);
runner
src/obikindex/src/index/merge_layer/mod.rs → src/obikmerge/src/merge_layer/mod.rs
+27 -30
View File
@@ -20,16 +20,18 @@ use tracing::debug;
use obicompactvec::{PersistentBitMatrixBuilder, PersistentCompactIntMatrixBuilder};
use obikseq::CanonicalKmer;
use crate::layer::{IndexMode, TypedLayer, LayeredMap, MphfOnly, layer_dir};
use obiskio::{SKError, SKResult, UnitigFileReader};
use obikindex::layer::{IndexMode, TypedLayer, LayeredMap, MphfOnly};
use obikindex::layer::utils::layer_dir;
use obiskio::UnitigFileReader;
use obikindex::{OKIError, OKIResult};
use crate::index::common::{ColBuilder, load_meta, olm_to_sk};
use crate::index::graph_pipeline::{build_graph, materialize_layer};
use crate::index::kmer_index::KmerIndex;
use obikindex::{ColBuilder, load_meta};
use obikindexer::{build_graph, materialize_layer};
use obikindex::KmerIndex;
mod src_layer;
pub(crate) use src_layer::SrcLayerData;
pub use src_layer::SrcLayerData;
// ── MergeMode ─────────────────────────────────────────────────────────────────
@@ -216,15 +218,14 @@ impl KmerIndex {
n_dst_genomes: usize,
block_bits: u8,
evidence: &IndexMode,
) -> SKResult<usize> {
) -> OKIResult<usize> {
let dst_index_dir = self.index_dir(i);
if !dst_index_dir.exists() {
return Ok(0);
}
load_meta(&dst_index_dir, "merge")?; // ensure meta.json exists before LayeredMap::open
let dst_map =
Arc::new(LayeredMap::<()>::open(&dst_index_dir).map_err(|e| olm_to_sk(e, "merge"))?);
load_meta(&dst_index_dir)?; // ensure meta.json exists before LayeredMap::open
let dst_map = Arc::new(LayeredMap::<()>::open(&dst_index_dir)?);
let n_dst_layers = dst_map.n_layers();
let n_src_total: usize = sources.iter().map(|(_, n)| *n).sum();
@@ -235,8 +236,7 @@ impl KmerIndex {
TypedLayer::<()>::init_presence_matrix(
&layer_dir(&dst_index_dir, l),
dst_map.layer(l).n(),
)
.map_err(|e| olm_to_sk(e, "merge"))?;
)?;
}
}
@@ -254,7 +254,7 @@ impl KmerIndex {
if !src_index_dir.exists() {
continue;
}
let src_meta = load_meta(&src_index_dir, "merge")?;
let src_meta = load_meta(&src_index_dir)?;
for l in 0..src_meta.n_layers {
let p = layer_dir(&src_index_dir, l).join("unitigs.bin");
if p.exists() {
@@ -277,7 +277,7 @@ impl KmerIndex {
let g = build_graph(
unitig_paths.into_iter(),
move |path: PathBuf, emit: &mut dyn FnMut(Vec<CanonicalKmer>)| -> SKResult<()> {
move |path: PathBuf, emit: &mut dyn FnMut(Vec<CanonicalKmer>)| -> OKIResult<()> {
let reader = UnitigFileReader::open_sequential(&path)?;
let mut batch: Vec<CanonicalKmer> = Vec::with_capacity(BATCH);
for (kmer, _, _) in reader.iter_indexed_canonical_kmers() {
@@ -318,9 +318,7 @@ impl KmerIndex {
let t_open = std::time::Instant::now();
let new_mphf: Option<Arc<MphfOnly>> = if any_new {
Some(Arc::new(
MphfOnly::open(&new_layer_dir).map_err(|e| olm_to_sk(e, "merge"))?,
))
Some(Arc::new(MphfOnly::open(&new_layer_dir)?))
} else {
None
};
@@ -340,13 +338,13 @@ impl KmerIndex {
MergeMode::Count => new_layer_dir.join("counts"),
};
fs::create_dir_all(&data_dir)?;
let mut mb = MatrixBuilder::new(mode, n_new, &data_dir).map_err(SKError::Io)?;
let mut mb = MatrixBuilder::new(mode, n_new, &data_dir).map_err(OKIError::Io)?;
for _ in 0..n_dst_genomes {
mb.add_absent_col().map_err(SKError::Io)?;
mb.add_absent_col().map_err(OKIError::Io)?;
}
let cols = (0..n_src_total)
.map(|_| mb.add_col().map_err(SKError::Io))
.collect::<SKResult<Vec<_>>>()?;
.map(|_| mb.add_col().map_err(OKIError::Io))
.collect::<OKIResult<Vec<_>>>()?;
(cols, Some(mb))
} else {
(vec![], None)
@@ -365,10 +363,10 @@ impl KmerIndex {
MergeMode::Presence => layer_dir.join("presence"),
MergeMode::Count => layer_dir.join("counts"),
};
let mut mb = MatrixBuilder::resume(mode, &data_dir).map_err(SKError::Io)?;
let mut mb = MatrixBuilder::resume(mode, &data_dir).map_err(OKIError::Io)?;
let cols = (0..n_src_total)
.map(|_| mb.add_col().map_err(SKError::Io))
.collect::<SKResult<Vec<_>>>()?;
.map(|_| mb.add_col().map_err(OKIError::Io))
.collect::<OKIResult<Vec<_>>>()?;
exist_mbs.push(mb);
exist_builders.push(cols);
}
@@ -391,7 +389,7 @@ impl KmerIndex {
col_offset += src_n;
continue;
}
let src_meta = load_meta(&src_index_dir, "merge")?;
let src_meta = load_meta(&src_index_dir)?;
for l in 0..src_meta.n_layers {
let src_layer_dir = layer_dir(&src_index_dir, l);
if src_layer_dir.join("unitigs.bin").exists() {
@@ -552,7 +550,7 @@ impl KmerIndex {
.into_inner()
.unwrap_or_else(|e| e.into_inner())
{
return Err(SKError::InvalidData {
return Err(OKIError::InvalidData {
context: "merge pass2",
detail: msg,
});
@@ -568,7 +566,7 @@ impl KmerIndex {
.unwrap_or_else(|e| e.into_inner())
.close()?;
}
mb.close().map_err(SKError::Io)?;
mb.close().map_err(OKIError::Io)?;
}
for b in new_locked {
@@ -579,13 +577,12 @@ impl KmerIndex {
.close()?;
}
if let Some(mb) = new_mb {
mb.close().map_err(SKError::Io)?;
mb.close().map_err(OKIError::Io)?;
let mut part_meta = self.partition_meta(i)?;
part_meta.n_layers = new_layer_idx + 1;
part_meta
.save(&dst_index_dir)
.map_err(|e| olm_to_sk(e, "merge"))?;
.save(&dst_index_dir)?;
}
debug!(
src/obikindex/src/index/merge_layer/src_layer.rs → src/obikmerge/src/merge_layer/src_layer.rs
+15 -16
View File
@@ -2,44 +2,43 @@ use std::path::Path;
use obicompactvec::{MatrixGroupOps, PersistentBitMatrix, PersistentCompactIntMatrix};
use obikseq::CanonicalKmer;
use crate::layer::MphfOnly;
use obiskio::{SKError, SKResult};
use obikindex::layer::MphfOnly;
use obikindex::{OKIError, OKIResult};
use crate::index::common::olm_to_sk;
use super::MergeMode;
// ── SrcLayerData — opened source matrix for pass-2 lookup ─────────────────────
pub(crate) enum SrcLayerData {
pub enum SrcLayerData {
Presence(MphfOnly, PersistentBitMatrix),
Count(MphfOnly, PersistentCompactIntMatrix),
}
impl SrcLayerData {
pub(crate) fn open(layer_dir: &Path, merge_mode: MergeMode) -> SKResult<Self> {
pub fn open(layer_dir: &Path, merge_mode: MergeMode) -> OKIResult<Self> {
let counts_dir = layer_dir.join("counts");
match merge_mode {
MergeMode::Presence => {
if counts_dir.exists() && !layer_dir.join("presence").exists() {
let mphf = MphfOnly::open(layer_dir).map_err(|e| olm_to_sk(e, "merge"))?;
let mat = PersistentCompactIntMatrix::open(layer_dir).map_err(SKError::Io)?;
let mphf = MphfOnly::open(layer_dir)?;
let mat = PersistentCompactIntMatrix::open(layer_dir).map_err(OKIError::Io)?;
Ok(SrcLayerData::Count(mphf, mat))
} else {
// presence dir exists, or neither exists → Implicit handled by open()
let mphf = MphfOnly::open(layer_dir).map_err(|e| olm_to_sk(e, "merge"))?;
let mat = PersistentBitMatrix::open(layer_dir).map_err(SKError::Io)?;
let mphf = MphfOnly::open(layer_dir)?;
let mat = PersistentBitMatrix::open(layer_dir).map_err(OKIError::Io)?;
Ok(SrcLayerData::Presence(mphf, mat))
}
}
MergeMode::Count => {
let mphf = MphfOnly::open(layer_dir).map_err(|e| olm_to_sk(e, "merge"))?;
let mphf = MphfOnly::open(layer_dir)?;
if counts_dir.exists() {
let mat = PersistentCompactIntMatrix::open(layer_dir).map_err(SKError::Io)?;
let mat = PersistentCompactIntMatrix::open(layer_dir).map_err(OKIError::Io)?;
Ok(SrcLayerData::Count(mphf, mat))
} else {
// No counts → treat as implicit presence (all 1s)
let mat = PersistentBitMatrix::open(layer_dir).map_err(SKError::Io)?;
let mat = PersistentBitMatrix::open(layer_dir).map_err(OKIError::Io)?;
Ok(SrcLayerData::Presence(mphf, mat))
}
}
@@ -58,7 +57,7 @@ impl SrcLayerData {
buf
}
pub(crate) fn n_slots(&self) -> usize {
pub fn n_slots(&self) -> usize {
match self {
SrcLayerData::Presence(_, mat) => mat.n(),
SrcLayerData::Count(_, mat) => mat.n(),
@@ -67,7 +66,7 @@ impl SrcLayerData {
/// MPHF lookup: returns the slot index for `kmer` (kmer must be in the domain).
#[inline]
pub(crate) fn slot(&self, kmer: CanonicalKmer) -> usize {
pub fn slot(&self, kmer: CanonicalKmer) -> usize {
match self {
SrcLayerData::Presence(mphf, _) => mphf.index(kmer),
SrcLayerData::Count(mphf, _) => mphf.index(kmer),
@@ -76,7 +75,7 @@ impl SrcLayerData {
/// Row lookup by slot index, bypassing the MPHF.
#[inline]
pub(crate) fn fill_row_by_slot(&self, slot: usize, n_genomes: usize) -> Vec<u32> {
pub fn fill_row_by_slot(&self, slot: usize, n_genomes: usize) -> Vec<u32> {
let mut buf = vec![0u32; n_genomes];
match self {
SrcLayerData::Presence(_, mat) => mat.fill_row(slot, &mut buf),
@@ -86,7 +85,7 @@ impl SrcLayerData {
}
/// Call `f` with a reference to the underlying matrix as `&dyn MatrixGroupOps`.
pub(crate) fn with_matrix<R>(&self, f: impl FnOnce(&dyn MatrixGroupOps) -> R) -> R {
pub fn with_matrix<R>(&self, f: impl FnOnce(&dyn MatrixGroupOps) -> R) -> R {
match self {
SrcLayerData::Presence(_, mat) => f(mat),
SrcLayerData::Count(_, mat) => f(mat),
src/obikphylo/Cargo.toml
+1
View File
@@ -9,6 +9,7 @@ obikseq = { path = "../obikseq" }
obiskio = { path = "../obiskio" }
obisys = { path = "../obisys" }
obicompactvec = { path = "../obicompactvec" }
obikidxcache = { path = "../obikidxcache" }
obiskbuilder = { path = "../obiskbuilder" }
obipipeline = { path = "../obipipeline" }
memmap2 = "0.9"
src/obikindex/src/index/distance.rs → src/obikphylo/src/distance.rs
+3 -3
View File
@@ -1,10 +1,10 @@
use ndarray::Array2;
use obicompactvec::traits::{BitPartials, CountPartials};
use crate::layer::LayeredStore;
use obikidxcache::LayeredStore;
use rayon::prelude::*;
use crate::index::error::{OKIError, OKIResult};
use crate::index::kmer_index::KmerIndex;
use obikindex::{OKIError, OKIResult};
use obikindex::KmerIndex;
// ── Public API ────────────────────────────────────────────────────────────────
src/obikphylo/src/lib.rs
+3
View File
@@ -9,6 +9,9 @@
//! incrementally.
mod cardcomp;
mod distance;
mod matrix_store;
pub mod siblings;
pub use cardcomp::{cardinality_transition_probs, composition_transition_probs, pairwise_cost_matrix};
pub use distance::{DistanceMetric, DistanceOutput};
src/obikindex/src/index/matrix_store.rs → src/obikphylo/src/matrix_store.rs
+13 -12
View File
@@ -1,46 +1,47 @@
use obikindex::OKIResult;
use obikindex::layer::open_data;
use obikidxcache::LayeredStore;
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
use crate::layer::{LayeredStore, open_data};
use obiskio::SKResult;
use crate::index::common::{load_meta, olm_to_sk};
use crate::index::kmer_index::KmerIndex;
use obikindex::load_meta;
use obikindex::KmerIndex;
impl KmerIndex {
/// Open all count matrices for partition `part`, one per layer.
/// Layers without a `counts/` directory are skipped.
pub fn count_store(&self, part: usize) -> SKResult<LayeredStore<PersistentCompactIntMatrix>> {
pub fn count_store(&self, part: usize) -> OKIResult<LayeredStore<PersistentCompactIntMatrix>> {
let index_dir = self.index_dir(part);
if !index_dir.exists() {
return Ok(LayeredStore::new(vec![]));
}
let n_layers = load_meta(&index_dir, "distance")?.n_layers;
let n_layers = self.n_layers();
let matrices = (0..n_layers)
.filter_map(|l| {
self.layer_dir(part, l)
.join("counts")
.exists()
.then(|| open_data(&index_dir, l).map_err(|e| olm_to_sk(e, "distance")))
.then(|| open_data(&index_dir, l))
})
.collect::<SKResult<Vec<_>>>()?;
.collect::<OKIResult<Vec<_>>>()?;
Ok(LayeredStore::new(matrices))
}
/// Open all presence matrices for partition `part`, one per layer.
/// Layers without a `presence/` directory are skipped.
pub fn presence_store(&self, part: usize) -> SKResult<LayeredStore<PersistentBitMatrix>> {
pub fn presence_store(&self, part: usize) -> OKIResult<LayeredStore<PersistentBitMatrix>> {
let index_dir = self.index_dir(part);
if !index_dir.exists() {
return Ok(LayeredStore::new(vec![]));
}
let n_layers = load_meta(&index_dir, "distance")?.n_layers;
let n_layers = load_meta(&index_dir)?.n_layers;
let matrices = (0..n_layers)
.filter_map(|l| {
self.layer_dir(part, l)
.join("presence")
.exists()
.then(|| open_data(&index_dir, l).map_err(|e| olm_to_sk(e, "distance")))
.then(|| open_data(&index_dir, l))
})
.collect::<SKResult<Vec<_>>>()?;
.collect::<OKIResult<Vec<_>>>()?;
Ok(LayeredStore::new(matrices))
}
}
src/obikphylo/src/siblings/build.rs
+2 -2
View File
@@ -14,7 +14,7 @@ use obikindex::OKIResult;
use super::cache::PartitionCache;
use super::helpers::{central_base, is_minorant};
use super::{ANNEX_FILE_NAME, FamilyMask, SiblingAnnexBuilder, olm_to_ok};
use super::{ANNEX_FILE_NAME, FamilyMask, SiblingAnnexBuilder};
// ── obipipeline data types ─────────────────────────────────────────────────
@@ -125,7 +125,7 @@ fn build_layer_sibling_annex(
l: usize,
cache: &Arc<PartitionCache>,
) -> OKIResult<u64> {
let mphf = MphfLayer::open(layer_dir).map_err(olm_to_ok)?;
let mphf = MphfLayer::open(layer_dir)?;
let k = index.kmer_size();
let n = mphf.n();
src/obikphylo/src/siblings/family_scan.rs
+3 -3
View File
@@ -65,7 +65,7 @@ use obikindex::layer::KmerLayer;
use super::cache::PartitionCache;
use super::helpers::central_base;
use super::iter::{SiblingEntry, SiblingLayerExt};
use super::{ANNEX_FILE_NAME, FamilyMask, SiblingAnnex, olm_to_ok};
use super::{ANNEX_FILE_NAME, FamilyMask, SiblingAnnex};
/// Families per batch — see the module docs for the memory-vs-per-partition-
/// density trade-off this picks a point on. At ~90 genomes and a few
@@ -194,10 +194,10 @@ pub(super) fn scan_layer_families(
let index_dir = layer_dir
.parent()
.expect("layer_dir has a parent index dir");
let meta = PartitionMeta::load(index_dir).map_err(olm_to_ok)?;
let meta = PartitionMeta::load(index_dir)?;
let annex = Arc::new(SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME))?);
let mat = KmerLayer::open(layer_dir, &meta.mode, with_counts).map_err(olm_to_ok)?;
let mat = KmerLayer::open(layer_dir, &meta.mode, with_counts)?;
let n_cols = mat.n_cols().min(n_genomes);
let ctx = Arc::new(LayerCtx {
src/obikphylo/src/siblings/mod.rs
-11
View File
@@ -71,15 +71,4 @@ pub(crate) use siblingannex::{FamilyMask, SiblingAnnex, SiblingAnnexBuilder};
pub use stats::{SiblingAnnexStats, SiblingStatsExt};
pub use subsample::EntropyBias;
use obikindex::layer::OLMError;
use obikindex::OKIError;
pub(super) const ANNEX_FILE_NAME: &str = "siblings.psib";
pub(super) fn olm_to_ok(e: OLMError) -> OKIError {
match e {
OLMError::Io(e) => OKIError::Io(e),
other => OKIError::InvalidInput(format!("layered-map error: {other}")),
}
}
src/obikquery/Cargo.toml
+7
View File
@@ -0,0 +1,7 @@
[package]
name = "obikquery"
version = "0.1.0"
edition = "2024"
[dependencies]
obikindex = { path = "../obikindex" }
src/obikquery/src/lib.rs
+10
View File
@@ -0,0 +1,10 @@
//! Query-side operations on an `obikindex::KmerIndex`: staging ground for
//! code currently living in `obikindex::index`'s query path, to be migrated
//! here to lighten that crate — see `DevDocMD/` for the rationale. Kept as
//! a separate crate — not a module of `obikindex` — so the dependency runs
//! one way only (query code depends on the data model, never the reverse),
//! same pattern as `obikindexer` for the build side.
mod query_layer;
pub use query_layer::{KmerDesc, QueryHit, QueryStats};
src/obikindex/src/index/query_layer.rs → src/obikquery/src/query_layer.rs
+10 -20
View File
@@ -3,20 +3,10 @@ use std::path::Path;
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
use obikseq::CanonicalKmer;
use crate::layer::{MphfLayer, OLMError};
use obiskio::{SKError, SKResult};
use obikindex::layer::MphfLayer;
use obikindex::{OKIError, OKIResult};
use crate::index::kmer_index::KmerIndex;
fn olm_to_sk(e: OLMError) -> SKError {
match e {
OLMError::Io(io_err) => SKError::Io(io_err),
other => SKError::InvalidData {
context: "query",
detail: other.to_string(),
},
}
}
use obikindex::KmerIndex;
// ── per-layer query handle ────────────────────────────────────────────────────
@@ -26,21 +16,21 @@ enum QueryLayer {
}
impl QueryLayer {
fn open(layer_dir: &Path, with_counts: bool) -> SKResult<Self> {
let mphf = MphfLayer::open(layer_dir).map_err(olm_to_sk)?;
fn open(layer_dir: &Path, with_counts: bool) -> OKIResult<Self> {
let mphf = MphfLayer::open(layer_dir)?;
let counts_dir = layer_dir.join("counts");
let presence_dir = layer_dir.join("presence");
if with_counts && counts_dir.exists() {
let mat = PersistentCompactIntMatrix::open(layer_dir).map_err(SKError::Io)?;
let mat = PersistentCompactIntMatrix::open(layer_dir).map_err(OKIError::Io)?;
Ok(QueryLayer::Count(mphf, mat))
} else if presence_dir.exists() || !counts_dir.exists() {
// presence mode, or no matrix at all → Implicit handled inside open()
let mat = PersistentBitMatrix::open(layer_dir).map_err(SKError::Io)?;
let mat = PersistentBitMatrix::open(layer_dir).map_err(OKIError::Io)?;
Ok(QueryLayer::Presence(mphf, mat))
} else {
// counts exist but not presence — count layer, no presence requested
let mat = PersistentCompactIntMatrix::open(layer_dir).map_err(SKError::Io)?;
let mat = PersistentCompactIntMatrix::open(layer_dir).map_err(OKIError::Io)?;
Ok(QueryLayer::Count(mphf, mat))
}
}
@@ -168,7 +158,7 @@ impl KmerIndex {
n_genomes: usize,
with_counts: bool,
mut on_event: F,
) -> SKResult<QueryStats>
) -> OKIResult<QueryStats>
where
F: FnMut(QueryHit),
{
@@ -186,7 +176,7 @@ impl KmerIndex {
let meta = self.partition_meta(part_idx)?;
let layers: Vec<QueryLayer> = (0..meta.n_layers)
.map(|i| QueryLayer::open(&self.layer_dir(part_idx, i), with_counts))
.collect::<SKResult<_>>()?;
.collect::<OKIResult<_>>()?;
// ── Stage 1: MPHF-only pass, bucket hits by (layer_idx, slot) ────────
let mut by_layer: Vec<HashMap<usize, &Vec<KmerDesc>>> =
src/obikindex/src/index/tests/query_layer.rs → src/obikquery/src/tests/query_layer.rs
+3 -3
View File
@@ -1,5 +1,5 @@
use super::*;
use crate::index::meta::IndexConfig;
use obikindex::IndexConfig;
// ── QueryStats::AddAssign ───────────────────────────────────────────────────
@@ -50,7 +50,7 @@ fn query_partition_with_missing_index_dir_returns_default_stats() {
minimizer_size: 9,
n_bits: 2,
with_counts: false,
evidence: crate::layer::IndexMode::Exact,
evidence: obikindex::layer::IndexMode::Exact,
block_bits: 0,
};
let index = KmerIndex::create(tmp.path().join("idx"), config, None).expect("create index");
@@ -78,7 +78,7 @@ fn query_partition_with_empty_kmers_is_a_noop() {
minimizer_size: 9,
n_bits: 2,
with_counts: false,
evidence: crate::layer::IndexMode::Exact,
evidence: obikindex::layer::IndexMode::Exact,
block_bits: 0,
};
let index = KmerIndex::create(tmp.path().join("idx"), config, None).expect("create index");
src/obikrebuild/Cargo.toml
+16
View File
@@ -0,0 +1,16 @@
[package]
name = "obikrebuild"
version = "0.1.0"
edition = "2024"
[dependencies]
obikindex = { path = "../obikindex" }
obikindexer = { path = "../obikindexer" }
obikfilter = { path = "../obikfilter" }
obikmerge = { path = "../obikmerge" }
obicompactvec = { path = "../obicompactvec" }
obiskio = { path = "../obiskio" }
obikseq = { path = "../obikseq" }
obidebruinj = { path = "../obidebruinj" }
obisys = { path = "../obisys" }
tracing = "0.1.44"
src/obikrebuild/src/lib.rs
+12
View File
@@ -0,0 +1,12 @@
//! Rebuilding an `obikindex::KmerIndex`: compacting a source index into a
//! new single-layer index while applying k-mer filters (`rebuild`), and
//! reindexing an existing index's evidence representation in place
//! (`reindex`, `Exact`/`Approx`/`Hybrid`). Both are maintenance/
//! transformation operations on an already-built index, not part of the
//! `Index { Partition { Layer } }` data model itself — kept out of
//! `obikindex` so that crate doesn't grow every algorithm's own
//! dependencies (`obikfilter`, `obikmerge`, `obikindexer`).
mod rebuild;
mod rebuild_layer;
mod reindex;
src/obikindex/src/index/rebuild.rs → src/obikrebuild/src/rebuild.rs
+8 -6
View File
@@ -1,13 +1,15 @@
use std::path::Path;
use crate::index::builder::IndexBuilder;
use crate::index::{KmerFilter, MergeMode};
use obikindex::IndexBuilder;
use obikfilter::KmerFilter;
use obikmerge::MergeMode;
use obisys::{Reporter, Stage, progress_bar};
use tracing::info;
use crate::index::error::{OKIError, OKIResult};
use crate::index::kmer_index::KmerIndex;
use crate::index::state::IndexState;
use obikindex::{OKIError, OKIResult};
use obikindex::KmerIndex;
use obikindex::IndexState;
use obisys::PartitionRunner;
impl KmerIndex {
/// Rebuild `src` into a new compact single-layer index at `output`.
@@ -61,7 +63,7 @@ impl KmerIndex {
let pb = progress_bar("rebuild", n_partitions as u64, "partitions");
let order: Vec<usize> = (0..n_partitions).collect();
let runner = crate::index::numa::PartitionRunner::new();
let runner = PartitionRunner::new();
runner.run(
&order,
|i| dst_partition.rebuild_partition(src, i, filters, mode, n_genomes, block_bits),
src/obikindex/src/index/rebuild_layer.rs → src/obikrebuild/src/rebuild_layer.rs
+30 -30
View File
@@ -6,15 +6,17 @@ use obicompactvec::{
};
use obidebruinj::GraphDeBruijn;
use obikseq::CanonicalKmer;
use crate::layer::meta::PartitionMeta;
use crate::layer::{IndexMode, MphfLayer, layer_dir};
use obiskio::{SKError, SKResult, UnitigFileReader};
use obikindex::layer::meta::PartitionMeta;
use obikindex::layer::{IndexMode, MphfLayer};
use obikindex::layer::utils::layer_dir;
use obiskio::UnitigFileReader;
use obikindex::{OKIError, OKIResult};
use crate::index::common::{load_meta, olm_to_sk};
use crate::index::filter::KmerFilter;
use crate::index::graph_pipeline::materialize_layer;
use crate::index::merge_layer::{MergeMode, SrcLayerData};
use crate::index::kmer_index::KmerIndex;
use obikindex::load_meta;
use obikfilter::KmerFilter;
use obikindexer::materialize_layer;
use obikmerge::{MergeMode, SrcLayerData};
use obikindex::KmerIndex;
// ── Builders — pair matrix builder + column builders for one mode ─────────────
@@ -27,22 +29,22 @@ enum Builders {
}
impl Builders {
fn new(mode: MergeMode, n: usize, dir: &Path, n_genomes: usize) -> SKResult<Self> {
fn new(mode: MergeMode, n: usize, dir: &Path, n_genomes: usize) -> OKIResult<Self> {
match mode {
MergeMode::Presence => {
let mut mat = PersistentBitMatrixBuilder::new(n, dir).map_err(SKError::Io)?;
let mut mat = PersistentBitMatrixBuilder::new(n, dir).map_err(OKIError::Io)?;
let mut cols = Vec::with_capacity(n_genomes);
for _ in 0..n_genomes {
cols.push(mat.add_col().map_err(SKError::Io)?);
cols.push(mat.add_col().map_err(OKIError::Io)?);
}
Ok(Builders::Presence(mat, cols))
}
MergeMode::Count => {
let mut mat =
PersistentCompactIntMatrixBuilder::new(n, dir).map_err(SKError::Io)?;
PersistentCompactIntMatrixBuilder::new(n, dir).map_err(OKIError::Io)?;
let mut cols = Vec::with_capacity(n_genomes);
for _ in 0..n_genomes {
cols.push(mat.add_col().map_err(SKError::Io)?);
cols.push(mat.add_col().map_err(OKIError::Io)?);
}
Ok(Builders::Count(mat, cols))
}
@@ -56,19 +58,19 @@ impl Builders {
}
}
fn close(self) -> SKResult<()> {
fn close(self) -> OKIResult<()> {
match self {
Builders::Presence(mat, cols) => {
for b in cols {
b.close().map_err(SKError::Io)?;
b.close().map_err(OKIError::Io)?;
}
mat.close().map_err(SKError::Io)
mat.close().map_err(OKIError::Io)
}
Builders::Count(mat, cols) => {
for b in cols {
b.close().map_err(SKError::Io)?;
b.close().map_err(OKIError::Io)?;
}
mat.close().map_err(SKError::Io)
mat.close().map_err(OKIError::Io)
}
}
}
@@ -86,7 +88,7 @@ fn try_compute_combined_mask(
filters: &[Box<dyn KmerFilter>],
src_data: &SrcLayerData,
n_genomes: usize,
) -> SKResult<Option<obicompactvec::TempBitVec>> {
) -> OKIResult<Option<obicompactvec::TempBitVec>> {
if filters.is_empty() {
return Ok(None);
}
@@ -101,7 +103,7 @@ fn try_compute_combined_mask(
let n = src_data.n_slots();
let mask = src_data
.with_matrix(|mat| eval_filter_mask(&combined, mat, n))
.map_err(SKError::Io)?;
.map_err(OKIError::Io)?;
Ok(Some(mask))
}
@@ -118,8 +120,8 @@ fn iter_src_kmers_masked(
n_genomes: usize,
filters: &[Box<dyn KmerFilter>],
mut cb: impl FnMut(CanonicalKmer),
) -> SKResult<()> {
let src_meta = load_meta(src_index_dir, "rebuild")?;
) -> OKIResult<()> {
let src_meta = load_meta(src_index_dir)?;
for l in 0..src_meta.n_layers {
let src_layer_dir = layer_dir(src_index_dir, l);
let unitigs_path = src_layer_dir.join("unitigs.bin");
@@ -159,8 +161,8 @@ fn iter_src_layers(
n_genomes: usize,
filters: &[Box<dyn KmerFilter>],
mut cb: impl FnMut(CanonicalKmer, Box<[u32]>),
) -> SKResult<()> {
let src_meta = load_meta(src_index_dir, "rebuild")?;
) -> OKIResult<()> {
let src_meta = load_meta(src_index_dir)?;
for l in 0..src_meta.n_layers {
let src_layer_dir = layer_dir(src_index_dir, l);
let unitigs_path = src_layer_dir.join("unitigs.bin");
@@ -209,13 +211,13 @@ impl KmerIndex {
mode: MergeMode,
n_genomes: usize,
block_bits: u8,
) -> SKResult<()> {
) -> OKIResult<()> {
let src_index_dir = src.index_dir(i);
if !src_index_dir.exists() {
return Ok(());
}
let src_meta = load_meta(&src_index_dir, "rebuild")?;
let src_meta = load_meta(&src_index_dir)?;
if src_meta.n_layers == 0 {
return Ok(());
}
@@ -235,8 +237,7 @@ impl KmerIndex {
let dst_layer_dir = self.layer_dir(i, 0);
let n_new = materialize_layer(g, &dst_layer_dir, block_bits, &IndexMode::Exact)?;
let dst_mphf = MphfLayer::open(&dst_layer_dir)
.map_err(|e| olm_to_sk(e, "rebuild"))?;
let dst_mphf = MphfLayer::open(&dst_layer_dir)?;
// ── Prepare matrix builders (one column per genome) ───────────────────
let data_dir = match mode {
@@ -262,8 +263,7 @@ impl KmerIndex {
n_layers: 1,
mode: IndexMode::Exact,
}
.save(&dst_index_dir)
.map_err(|e| olm_to_sk(e, "rebuild"))?;
.save(&dst_index_dir)?;
Ok(())
}
src/obikindex/src/index/reindex.rs → src/obikrebuild/src/reindex.rs
+10 -13
View File
@@ -1,24 +1,21 @@
use crate::index::builder::IndexBuilder;
use crate::index::meta::IndexMeta;
use crate::layer::{IndexMode, TypedLayer};
use obikindex::IndexBuilder;
use obikindex::IndexMeta;
use obikindex::layer::{IndexMode, TypedLayer};
use obisys::{Reporter, Stage, progress_bar};
use std::fs;
use std::path::Path;
use std::sync::Arc;
use tracing::info;
use crate::index::error::{OKIError, OKIResult};
use crate::index::kmer_index::KmerIndex;
use crate::index::state::IndexState;
use obikindex::{OKIError, OKIResult};
use obikindex::KmerIndex;
use obikindex::IndexState;
use obisys::PartitionRunner;
const EVIDENCE_FILE: &str = "evidence.bin";
const FINGERPRINT_FILE: &str = "fingerprint.bin";
const UNITIG_IDX_FILE: &str = "unitigs.bin.idx";
fn olm_to_oki(e: crate::layer::OLMError) -> OKIError {
OKIError::InvalidInput(e.to_string())
}
impl KmerIndex {
/// Convert every layer's evidence bundle to `target` in-place.
///
@@ -47,7 +44,7 @@ impl KmerIndex {
let pb = progress_bar("reindex", n as u64, "partitions");
let order: Vec<usize> = (0..n).collect();
let runner = crate::index::numa::PartitionRunner::new();
let runner = PartitionRunner::new();
runner.run(
&order,
|i| {
@@ -96,10 +93,10 @@ fn reindex_partition(
fn reindex_layer(layer_dir: &Path, target: &IndexMode, block_bits: u8) -> OKIResult<()> {
match target {
IndexMode::Exact => {
TypedLayer::<()>::build_exact_evidence(layer_dir, block_bits).map_err(olm_to_oki)?;
TypedLayer::<()>::build_exact_evidence(layer_dir, block_bits)?;
}
IndexMode::Approx { b, z } | IndexMode::Hybrid { b, z } => {
TypedLayer::<()>::build_approx_evidence(layer_dir, *b, *z).map_err(olm_to_oki)?;
TypedLayer::<()>::build_approx_evidence(layer_dir, *b, *z)?;
}
}
remove_stale_evidence(layer_dir, target)
src/obikselect/Cargo.toml
+10
View File
@@ -0,0 +1,10 @@
[package]
name = "obikselect"
version = "0.1.0"
edition = "2024"
[dependencies]
obikindex = { path = "../obikindex" }
obicompactvec = { path = "../obicompactvec" }
obisys = { path = "../obisys" }
tracing = "0.1.44"
src/obikselect/src/lib.rs
+11
View File
@@ -0,0 +1,11 @@
//! Genome-column selection/aggregation on an `obikindex::KmerIndex`:
//! projecting/aggregating genome columns (`AggOp`) into new output columns
//! (`OutputCol`), either into a new index (`select`) or in place
//! (`select_in_place`). Orthogonal to per-kmer filtering (`obikfilter`),
//! which operates on rows, not columns. Kept as a separate crate — not a
//! module of `obikindex` — so the dependency runs one way only.
mod select;
mod select_layer;
pub use select_layer::{AggOp, OutputCol};
src/obikindex/src/index/select.rs → src/obikselect/src/select.rs
+9 -9
View File
@@ -1,15 +1,15 @@
use std::path::Path;
use std::sync::Arc;
use crate::index::builder::IndexBuilder;
use crate::index::OutputCol;
use obisys::{Reporter, Stage, progress_bar};
use obikindex::IndexBuilder;
use crate::OutputCol;
use obisys::{Reporter, Stage, progress_bar, PartitionRunner};
use tracing::info;
use crate::index::error::{OKIError, OKIResult};
use crate::index::kmer_index::KmerIndex;
use crate::index::meta::{GenomeInfo, IndexMeta};
use crate::index::state::IndexState;
use obikindex::{OKIError, OKIResult};
use obikindex::KmerIndex;
use obikindex::{GenomeInfo, IndexMeta};
use obikindex::IndexState;
impl KmerIndex {
/// Create a new index at `output` by projecting/aggregating the genome columns
@@ -58,7 +58,7 @@ impl KmerIndex {
let pb = progress_bar("select", n_partitions as u64, "partitions");
let order: Vec<usize> = (0..n_partitions).collect();
let runner = crate::index::numa::PartitionRunner::new();
let runner = PartitionRunner::new();
runner
.run(
&order,
@@ -113,7 +113,7 @@ impl KmerIndex {
let pb = progress_bar("select", n_partitions as u64, "partitions");
let order: Vec<usize> = (0..n_partitions).collect();
let runner = crate::index::numa::PartitionRunner::new();
let runner = PartitionRunner::new();
runner
.run(
&order,
src/obikindex/src/index/select_layer.rs → src/obikselect/src/select_layer.rs
+41 -53
View File
@@ -6,10 +6,9 @@ use obicompactvec::{
ColGroup, MatrixGroupOps, PersistentBitMatrix, PersistentBitMatrixBuilder,
PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder,
};
use crate::layer::OLMError;
use obiskio::{SKError, SKResult};
use obikindex::{OKIError, OKIResult};
use crate::index::kmer_index::KmerIndex;
use obikindex::KmerIndex;
// ── AggOp ─────────────────────────────────────────────────────────────────────
@@ -39,16 +38,6 @@ pub struct OutputCol {
// ── Helpers ───────────────────────────────────────────────────────────────────
fn olm_to_sk(e: OLMError) -> SKError {
match e {
OLMError::Io(e) => SKError::Io(e),
other => SKError::InvalidData {
context: "select",
detail: other.to_string(),
},
}
}
/// Copy all plain files (not subdirectories) from `src_dir` to `dst_dir`.
fn copy_layer_files(src_dir: &Path, dst_dir: &Path) -> io::Result<()> {
for entry in fs::read_dir(src_dir)? {
@@ -71,97 +60,97 @@ fn fill_builders(
output_presence: bool,
mut dst_bit: Option<&mut PersistentBitMatrixBuilder>,
mut dst_int: Option<&mut PersistentCompactIntMatrixBuilder>,
) -> SKResult<()> {
) -> OKIResult<()> {
if src_is_count {
let mat = PersistentCompactIntMatrix::open(src_layer_dir).map_err(SKError::Io)?;
let mat = PersistentCompactIntMatrix::open(src_layer_dir).map_err(OKIError::Io)?;
for spec in specs {
let g = ColGroup::new(&spec.label, spec.indices.clone());
if output_presence {
let b = dst_bit.as_deref_mut().unwrap();
match spec.op {
AggOp::Any => {
b.add_col_from(&mat.partial_group_any(&g, threshold).map_err(SKError::Io)?)
b.add_col_from(&mat.partial_group_any(&g, threshold).map_err(OKIError::Io)?)
}
AggOp::All => {
b.add_col_from(&mat.partial_group_all(&g, threshold).map_err(SKError::Io)?)
b.add_col_from(&mat.partial_group_all(&g, threshold).map_err(OKIError::Io)?)
}
AggOp::None => {
b.add_col_from(&mat.partial_group_none(&g, threshold).map_err(SKError::Io)?)
b.add_col_from(&mat.partial_group_none(&g, threshold).map_err(OKIError::Io)?)
}
AggOp::Sum => {
b.add_col_from_int(&mat.partial_group_sum(&g).map_err(SKError::Io)?)
b.add_col_from_int(&mat.partial_group_sum(&g).map_err(OKIError::Io)?)
}
AggOp::Min => {
b.add_col_from_int(&mat.partial_group_min(&g).map_err(SKError::Io)?)
b.add_col_from_int(&mat.partial_group_min(&g).map_err(OKIError::Io)?)
}
AggOp::Max => {
b.add_col_from_int(&mat.partial_group_max(&g).map_err(SKError::Io)?)
b.add_col_from_int(&mat.partial_group_max(&g).map_err(OKIError::Io)?)
}
}
.map_err(SKError::Io)?;
.map_err(OKIError::Io)?;
} else {
let b = dst_int.as_deref_mut().unwrap();
match spec.op {
AggOp::Sum => b.add_col_from(&mat.partial_group_sum(&g).map_err(SKError::Io)?),
AggOp::Min => b.add_col_from(&mat.partial_group_min(&g).map_err(SKError::Io)?),
AggOp::Max => b.add_col_from(&mat.partial_group_max(&g).map_err(SKError::Io)?),
AggOp::Sum => b.add_col_from(&mat.partial_group_sum(&g).map_err(OKIError::Io)?),
AggOp::Min => b.add_col_from(&mat.partial_group_min(&g).map_err(OKIError::Io)?),
AggOp::Max => b.add_col_from(&mat.partial_group_max(&g).map_err(OKIError::Io)?),
AggOp::Any => b.add_col_from_bit(
&mat.partial_group_any(&g, threshold).map_err(SKError::Io)?,
&mat.partial_group_any(&g, threshold).map_err(OKIError::Io)?,
),
AggOp::All => b.add_col_from_bit(
&mat.partial_group_all(&g, threshold).map_err(SKError::Io)?,
&mat.partial_group_all(&g, threshold).map_err(OKIError::Io)?,
),
AggOp::None => b.add_col_from_bit(
&mat.partial_group_none(&g, threshold).map_err(SKError::Io)?,
&mat.partial_group_none(&g, threshold).map_err(OKIError::Io)?,
),
}
.map_err(SKError::Io)?;
.map_err(OKIError::Io)?;
}
}
} else {
let mat = PersistentBitMatrix::open(src_layer_dir).map_err(SKError::Io)?;
let mat = PersistentBitMatrix::open(src_layer_dir).map_err(OKIError::Io)?;
for spec in specs {
let g = ColGroup::new(&spec.label, spec.indices.clone());
if output_presence {
let b = dst_bit.as_deref_mut().unwrap();
match spec.op {
AggOp::Any => {
b.add_col_from(&mat.partial_group_any(&g, 1).map_err(SKError::Io)?)
b.add_col_from(&mat.partial_group_any(&g, 1).map_err(OKIError::Io)?)
}
AggOp::All => {
b.add_col_from(&mat.partial_group_all(&g, 1).map_err(SKError::Io)?)
b.add_col_from(&mat.partial_group_all(&g, 1).map_err(OKIError::Io)?)
}
AggOp::None => {
b.add_col_from(&mat.partial_group_none(&g, 1).map_err(SKError::Io)?)
b.add_col_from(&mat.partial_group_none(&g, 1).map_err(OKIError::Io)?)
}
AggOp::Sum => {
b.add_col_from_int(&mat.partial_group_sum(&g).map_err(SKError::Io)?)
b.add_col_from_int(&mat.partial_group_sum(&g).map_err(OKIError::Io)?)
}
AggOp::Min => {
b.add_col_from_int(&mat.partial_group_min(&g).map_err(SKError::Io)?)
b.add_col_from_int(&mat.partial_group_min(&g).map_err(OKIError::Io)?)
}
AggOp::Max => {
b.add_col_from_int(&mat.partial_group_max(&g).map_err(SKError::Io)?)
b.add_col_from_int(&mat.partial_group_max(&g).map_err(OKIError::Io)?)
}
}
.map_err(SKError::Io)?;
.map_err(OKIError::Io)?;
} else {
let b = dst_int.as_deref_mut().unwrap();
match spec.op {
AggOp::Sum => b.add_col_from(&mat.partial_group_sum(&g).map_err(SKError::Io)?),
AggOp::Min => b.add_col_from(&mat.partial_group_min(&g).map_err(SKError::Io)?),
AggOp::Max => b.add_col_from(&mat.partial_group_max(&g).map_err(SKError::Io)?),
AggOp::Sum => b.add_col_from(&mat.partial_group_sum(&g).map_err(OKIError::Io)?),
AggOp::Min => b.add_col_from(&mat.partial_group_min(&g).map_err(OKIError::Io)?),
AggOp::Max => b.add_col_from(&mat.partial_group_max(&g).map_err(OKIError::Io)?),
AggOp::Any => {
b.add_col_from_bit(&mat.partial_group_any(&g, 1).map_err(SKError::Io)?)
b.add_col_from_bit(&mat.partial_group_any(&g, 1).map_err(OKIError::Io)?)
}
AggOp::All => {
b.add_col_from_bit(&mat.partial_group_all(&g, 1).map_err(SKError::Io)?)
b.add_col_from_bit(&mat.partial_group_all(&g, 1).map_err(OKIError::Io)?)
}
AggOp::None => {
b.add_col_from_bit(&mat.partial_group_none(&g, 1).map_err(SKError::Io)?)
b.add_col_from_bit(&mat.partial_group_none(&g, 1).map_err(OKIError::Io)?)
}
}
.map_err(SKError::Io)?;
.map_err(OKIError::Io)?;
}
}
}
@@ -185,7 +174,7 @@ impl KmerIndex {
threshold: u32,
output_presence: bool,
in_place: bool,
) -> SKResult<()> {
) -> OKIResult<()> {
let src_index_dir = src.index_dir(i);
if !src_index_dir.exists() {
return Ok(());
@@ -222,11 +211,11 @@ impl KmerIndex {
// Determine number of slots and detect implicit layers.
let n = if counts_dir.exists() {
PersistentCompactIntMatrix::open(&src_layer_dir)
.map_err(SKError::Io)?
.map_err(OKIError::Io)?
.n()
} else if presence_dir.exists() {
PersistentBitMatrix::open(&src_layer_dir)
.map_err(SKError::Io)?
.map_err(OKIError::Io)?
.n()
} else {
// Implicit single-genome layer: no data matrix needed in output either.
@@ -255,7 +244,7 @@ impl KmerIndex {
let (mut dst_bit, mut dst_int) = if output_presence {
(
Some(PersistentBitMatrixBuilder::new(n, &dst_data_dir).map_err(SKError::Io)?),
Some(PersistentBitMatrixBuilder::new(n, &dst_data_dir).map_err(OKIError::Io)?),
None,
)
} else {
@@ -263,7 +252,7 @@ impl KmerIndex {
None,
Some(
PersistentCompactIntMatrixBuilder::new(n, &dst_data_dir)
.map_err(SKError::Io)?,
.map_err(OKIError::Io)?,
),
)
};
@@ -279,9 +268,9 @@ impl KmerIndex {
)?;
if output_presence {
dst_bit.unwrap().close().map_err(SKError::Io)?;
dst_bit.unwrap().close().map_err(OKIError::Io)?;
} else {
dst_int.unwrap().close().map_err(SKError::Io)?;
dst_int.unwrap().close().map_err(OKIError::Io)?;
}
// In-place: swap old data dir for new.
@@ -300,8 +289,7 @@ impl KmerIndex {
if !in_place {
src.partition_meta(i)?
.save(&dst_index_dir)
.map_err(olm_to_sk)?;
.save(&dst_index_dir)?;
}
Ok(())
src/obikstats/Cargo.toml
+9
View File
@@ -0,0 +1,9 @@
[package]
name = "obikstats"
version = "0.1.0"
edition = "2024"
[dependencies]
obikindex = { path = "../obikindex" }
obicompactvec = { path = "../obicompactvec" }
rayon = "1"
src/obikstats/src/lib.rs
+10
View File
@@ -0,0 +1,10 @@
//! Read-only aggregation/statistics over an already-built
//! `obikindex::KmerIndex`: bits-per-kmer breakdown by index component
//! (`bits_per_kmer`), per-genome k-mer counts (`genome_kmer_counts`).
//! Not part of the `Index { Partition { Layer } }` data model itself —
//! kept out of `obikindex` like every other read/write extension
//! (`obikquery`, `obikdump`, `obikselect`, `obikrebuild`, `obikmerge`).
mod stats;
pub use stats::IndexBitsPerKmer;
src/obikindex/src/index/stats.rs → src/obikstats/src/stats.rs
+7 -6
View File
@@ -5,8 +5,8 @@ use obicompactvec::{LayerMeta, PersistentBitMatrix, PersistentCompactIntMatrix};
use obicompactvec::traits::ColumnWeights;
use rayon::prelude::*;
use crate::index::error::{OKIError, OKIResult};
use crate::index::kmer_index::KmerIndex;
use obikindex::{OKIError, OKIResult};
use obikindex::KmerIndex;
/// Bits per kmer broken down by index component.
pub struct IndexBitsPerKmer {
@@ -95,7 +95,8 @@ impl KmerIndex {
let n_layers = self.n_layers(i).unwrap_or(0);
(0..n_layers).fold((0usize, 0u64, 0u64, 0u64), |acc, l| {
let lb = layer_bytes(&self.layer_dir(i, l));
let Ok(layer_dir) = self.layer_dir(i, l) else { return acc };
let lb = layer_bytes(&layer_dir);
(acc.0 + lb.n_kmers, acc.1 + lb.mphf, acc.2 + lb.evidence, acc.3 + lb.matrix)
})
})
@@ -144,7 +145,7 @@ impl KmerIndex {
let n_layers = self.n_layers(i).unwrap_or(0);
for l in 0..n_layers {
let this_layer_dir = self.layer_dir(i, l);
let Ok(this_layer_dir) = self.layer_dir(i, l) else { continue };
if !this_layer_dir.exists() { continue; }
n_kmers += LayerMeta::load(&this_layer_dir).map(|m| m.n).unwrap_or(0);
@@ -153,12 +154,12 @@ impl KmerIndex {
if this_layer_dir.join("counts").exists()
&& !this_layer_dir.join("presence").exists()
{
match crate::layer::open_data::<PersistentCompactIntMatrix>(&index_dir, l) {
match obikindex::layer::open_data::<PersistentCompactIntMatrix>(&index_dir, l) {
Ok(m) => Box::new(m),
Err(_) => continue,
}
} else {
match crate::layer::open_data::<PersistentBitMatrix>(&index_dir, l) {
match obikindex::layer::open_data::<PersistentBitMatrix>(&index_dir, l) {
Ok(m) => Box::new(m),
Err(_) => continue,
}
src/obisys/Cargo.toml
+11 -4
View File
@@ -4,7 +4,14 @@ version = "0.1.0"
edition = "2024"
[dependencies]
libc = "0.2"
sysinfo = "0.39"
indicatif = "0.18"
tracing = "0.1"
libc = "0.2"
sysinfo = "0.39"
indicatif = "0.18"
tracing = "0.1"
crossbeam-channel = "0.5"
rayon = "1"
hwlocality = { version = "1.0.0-alpha.11", features = ["vendored"], optional = true }
[features]
default = ["numa"]
numa = ["hwlocality"]
src/obisys/src/lib.rs
+2
View File
@@ -3,12 +3,14 @@
mod budget;
mod lock;
mod numa;
mod progress;
mod resources;
mod stage;
pub use budget::MemoryBudget;
pub use lock::DirLock;
pub use numa::PartitionRunner;
pub use progress::{Progress, TracedBar, progress_bar, spinner};
pub use resources::{CpuSample, IoSample, available_memory_bytes, effective_parallelism, peak_rss_bytes};
pub use stage::{Reporter, Stage, StageStats};
src/obikindex/src/index/numa/mod.rs → src/obisys/src/numa/mod.rs
View File
src/obikindex/src/index/numa/runner.rs → src/obisys/src/numa/runner.rs
+1 -1
View File
@@ -2,7 +2,7 @@ use std::sync::Arc;
use std::time::{Duration, Instant};
use crossbeam_channel::unbounded;
use obisys::{CpuSample, IoSample};
use crate::{CpuSample, IoSample};
use tracing::debug;
use super::topology::{build, pin_current_thread};
src/obikindex/src/index/numa/topology.rs → src/obisys/src/numa/topology.rs
+2 -2
View File
@@ -66,7 +66,7 @@ pub fn build() -> NumaSetup {
}
// UMA fallback: single synthetic node, all cores, no pool, no pinning.
let n_cores = obisys::effective_parallelism();
let n_cores = crate::effective_parallelism();
debug!("UMA: single synthetic node, {} core(s)", n_cores);
NumaSetup {
pools: vec![None],
@@ -76,7 +76,7 @@ pub fn build() -> NumaSetup {
#[cfg(not(feature = "numa"))]
pub fn build() -> NumaSetup {
let n_cores = obisys::effective_parallelism();
let n_cores = crate::effective_parallelism();
debug!("UMA: single synthetic node, {} core(s)", n_cores);
NumaSetup {
pools: vec![None],