OBIKmers/obikmer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 34 Wiki Activity

Push zunrplorkwkt #70

Merged
coissac merged 93 commits from push-zunrplorkwkt into main 2026-08-28 23:15:38 +00:00
Conversation 0 Commits 93 Files Changed 428
+1640 -1196
81 changed files with 1640 additions and 1196 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit fc4464a0ef - Show all commits
src/Cargo.lock
Generated
+108 -2
View File
@@ -1519,6 +1519,14 @@ dependencies = [
name = "obikalgorithm"
version = "0.1.0"
[[package]]
name = "obikdump"
version = "0.1.0"
dependencies = [
"obikfilter",
"obikindex",
]
[[package]]
name = "obikentropy"
version = "0.1.0"
@@ -1526,6 +1534,27 @@ dependencies = [
"obikseq",
]
[[package]]
name = "obikfilter"
version = "0.1.0"
dependencies = [
"obicompactvec",
"obikindex",
"obikseq",
"obiskio",
"obitaxonomy",
]
[[package]]
name = "obikidxcache"
version = "0.1.0"
dependencies = [
"ndarray",
"obicompactvec",
"rayon",
"tempfile",
]
[[package]]
name = "obikindex"
version = "0.1.0"
@@ -1533,9 +1562,7 @@ dependencies = [
"anyhow",
"bitvec",
"cacheline-ef",
"crossbeam-channel",
"epserde",
"hwlocality",
"indicatif",
"memmap2",
"ndarray",
@@ -1597,11 +1624,13 @@ dependencies = [
"obidebruinj",
"obifastwrite",
"obikalgorithm",
"obikfilter",
"obikindex",
"obikindexer",
"obikphylo",
"obikrope",
"obikseq",
"obikstats",
"obipipeline",
"obiread",
"obiskbuilder",
@@ -1618,6 +1647,37 @@ dependencies = [
"tracing-subscriber",
]
[[package]]
name = "obikmer2"
version = "1.2.2"
dependencies = [
"clap",
"obikalgorithm",
"obikindex",
"obikindexer",
"obikseq",
"obipipeline",
"obiread",
"obisys",
"tracing",
"tracing-subscriber",
]
[[package]]
name = "obikmerge"
version = "0.1.0"
dependencies = [
"obicompactvec",
"obikfilter",
"obikindex",
"obikindexer",
"obikseq",
"obipipeline",
"obiskio",
"obisys",
"tracing",
]
[[package]]
name = "obikphylo"
version = "0.1.0"
@@ -1626,6 +1686,7 @@ dependencies = [
"ndarray",
"obicompactvec",
"obikalgorithm",
"obikidxcache",
"obikindex",
"obikindexer",
"obikseq",
@@ -1641,6 +1702,29 @@ dependencies = [
"tracing-subscriber",
]
[[package]]
name = "obikquery"
version = "0.1.0"
dependencies = [
"obikindex",
]
[[package]]
name = "obikrebuild"
version = "0.1.0"
dependencies = [
"obicompactvec",
"obidebruinj",
"obikfilter",
"obikindex",
"obikindexer",
"obikmerge",
"obikseq",
"obiskio",
"obisys",
"tracing",
]
[[package]]
name = "obikrope"
version = "0.1.0"
@@ -1649,6 +1733,16 @@ dependencies = [
"criterion2",
]
[[package]]
name = "obikselect"
version = "0.1.0"
dependencies = [
"obicompactvec",
"obikindex",
"obisys",
"tracing",
]
[[package]]
name = "obikseq"
version = "0.1.0"
@@ -1660,6 +1754,15 @@ dependencies = [
"xxhash-rust",
]
[[package]]
name = "obikstats"
version = "0.1.0"
dependencies = [
"obicompactvec",
"obikindex",
"rayon",
]
[[package]]
name = "obipipeline"
version = "0.1.0"
@@ -1712,8 +1815,11 @@ dependencies = [
name = "obisys"
version = "0.1.0"
dependencies = [
"crossbeam-channel",
"hwlocality",
"indicatif",
"libc",
"rayon",
"sysinfo",
"tracing",
]
src/Cargo.toml
+1 -1
View File
@@ -1,5 +1,5 @@
[workspace]
resolver = "3"
members = ["obikseq", "obiread", "obiskbuilder", "obifastwrite", "obikmer","obikrope","obipipeline", "obiskio","obidebruinj", "obicompactvec", "obisys", "obikindex", "obikindexer", "obitaxonomy", "obikentropy", "obikphylo", "obikalgorithm"]
members = ["obikseq", "obiread", "obiskbuilder", "obifastwrite", "obikmer","obikmer2","obikrope","obipipeline", "obiskio","obidebruinj", "obicompactvec", "obisys", "obikindex", "obikindexer", "obikquery", "obikdump", "obikfilter", "obikselect", "obikrebuild", "obikmerge", "obikstats", "obikidxcache", "obitaxonomy", "obikentropy", "obikphylo", "obikalgorithm"]
[profile.release]
debug = 1
src/obikdump/Cargo.toml
+8
View File
@@ -0,0 +1,8 @@
[package]
name = "obikdump"
version = "0.1.0"
edition = "2024"
[dependencies]
obikindex = { path = "../obikindex" }
obikfilter = { path = "../obikfilter" }
src/obikindex/src/index/dump.rs → src/obikdump/src/dump.rs
+3 -3
View File
@@ -3,9 +3,9 @@ use std::sync::atomic::{AtomicUsize, Ordering};
use rayon::prelude::*;
use crate::index::error::{OKIError, OKIResult};
use crate::index::kmer_index::KmerIndex;
use crate::index::KmerFilter;
use obikindex::{OKIError, OKIResult};
use obikindex::KmerIndex;
use obikfilter::KmerFilter;
impl KmerIndex {
/// Write a CSV table of all indexed kmers to `out`.
src/obikdump/src/lib.rs
+8
View File
@@ -0,0 +1,8 @@
//! Raw content export of an `obikindex::KmerIndex`: staging ground for
//! code currently living in `obikindex::index`'s dump path (`dump.rs`,
//! `dump_layer.rs`), to be migrated here to lighten that crate. Kept as
//! a separate crate — not a module of `obikindex` — so the dependency
//! runs one way only (dump code depends on the data model, never the
//! reverse), same pattern as `obikindexer`/`obikquery`.
mod dump;
src/obikfilter/Cargo.toml
+11
View File
@@ -0,0 +1,11 @@
[package]
name = "obikfilter"
version = "0.1.0"
edition = "2024"
[dependencies]
obikindex = { path = "../obikindex" }
obicompactvec = { path = "../obicompactvec" }
obikseq = { path = "../obikseq" }
obiskio = { path = "../obiskio" }
obitaxonomy = { path = "../obitaxonomy" }
src/obikindex/src/index/dump_layer.rs → src/obikfilter/src/dump_layer.rs
+14 -23
View File
@@ -1,20 +1,11 @@
use obikindex::layer::MphfLayer;
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
use obikseq::CanonicalKmer;
use crate::layer::{MphfLayer, OLMError};
use obiskio::{SKError, SKResult, UnitigFileReader};
use obiskio::UnitigFileReader;
use obikindex::{OKIError, OKIResult};
use crate::index::filter::{KmerFilter, passes_all};
use crate::index::kmer_index::KmerIndex;
fn olm_to_sk(e: OLMError) -> SKError {
match e {
OLMError::Io(e) => SKError::Io(e),
other => SKError::InvalidData {
context: "dump",
detail: other.to_string(),
},
}
}
use crate::filter::{KmerFilter, passes_all};
use obikindex::KmerIndex;
impl KmerIndex {
/// Iterate all indexed kmers in partition `part`, calling `cb(kmer, row)` for each
@@ -35,7 +26,7 @@ impl KmerIndex {
n_genomes: usize,
filters: &[Box<dyn KmerFilter>],
mut cb: impl FnMut(CanonicalKmer, Box<[u32]>) -> bool,
) -> SKResult<bool> {
) -> OKIResult<bool> {
let index_dir = self.index_dir(part);
if !index_dir.exists() {
return Ok(true);
@@ -43,19 +34,19 @@ impl KmerIndex {
let mut l = 0;
loop {
let layer_dir = self.layer_dir(part, l);
let layer_dir = self.layer_dir(part, l)?;
if !layer_dir.exists() {
break;
}
l += 1;
let mphf = MphfLayer::open(&layer_dir).map_err(olm_to_sk)?;
let mphf = MphfLayer::open(&layer_dir)?;
let reader = UnitigFileReader::open_sequential(&layer_dir.join("unitigs.bin"))?;
let counts_dir = layer_dir.join("counts");
let presence_dir = layer_dir.join("presence");
let cont = if use_counts && counts_dir.exists() {
let mat = PersistentCompactIntMatrix::open(&layer_dir).map_err(SKError::Io)?;
let mat = PersistentCompactIntMatrix::open(&layer_dir).map_err(OKIError::Io)?;
let mut cont = true;
for (kmer, _, _) in reader.iter_indexed_canonical_kmers() {
if let Some(slot) = mphf.find(kmer) {
@@ -70,7 +61,7 @@ impl KmerIndex {
}
cont
} else if !use_counts && presence_dir.exists() {
let mat = PersistentBitMatrix::open(&layer_dir).map_err(SKError::Io)?;
let mat = PersistentBitMatrix::open(&layer_dir).map_err(OKIError::Io)?;
let mut cont = true;
for (kmer, _, _) in reader.iter_indexed_canonical_kmers() {
if let Some(slot) = mphf.find(kmer) {
@@ -123,7 +114,7 @@ impl KmerIndex {
n_genomes: usize,
filters: &[Box<dyn KmerFilter>],
mut cb: impl FnMut(usize, usize, CanonicalKmer, Box<[u32]>) -> bool,
) -> SKResult<bool> {
) -> OKIResult<bool> {
let index_dir = self.index_dir(part);
if !index_dir.exists() {
return Ok(true);
@@ -135,14 +126,14 @@ impl KmerIndex {
if !layer_dir.exists() {
break;
}
let mphf = MphfLayer::open(&layer_dir).map_err(olm_to_sk)?;
let mphf = MphfLayer::open(&layer_dir)?;
let reader = UnitigFileReader::open_sequential(&layer_dir.join("unitigs.bin"))?;
let counts_dir = layer_dir.join("counts");
let presence_dir = layer_dir.join("presence");
let cont = if use_counts && counts_dir.exists() {
let mat = PersistentCompactIntMatrix::open(&layer_dir).map_err(SKError::Io)?;
let mat = PersistentCompactIntMatrix::open(&layer_dir).map_err(OKIError::Io)?;
let mut cont = true;
for (kmer, _, _) in reader.iter_indexed_canonical_kmers() {
if let Some(slot) = mphf.find(kmer) {
@@ -157,7 +148,7 @@ impl KmerIndex {
}
cont
} else if !use_counts && presence_dir.exists() {
let mat = PersistentBitMatrix::open(&layer_dir).map_err(SKError::Io)?;
let mat = PersistentBitMatrix::open(&layer_dir).map_err(OKIError::Io)?;
let mut cont = true;
for (kmer, _, _) in reader.iter_indexed_canonical_kmers() {
if let Some(slot) = mphf.find(kmer) {
src/obikindex/src/index/filter.rs → src/obikfilter/src/filter.rs
View File
src/obikfilter/src/lib.rs
+22
View File
@@ -0,0 +1,22 @@
//! Per-kmer filtering: `KmerFilter` judges one k-mer (its canonical
//! sequence + its per-genome row) at a time, deciding whether it passes.
//! Orthogonal to genome-column selection/aggregation (`obikselect`), which
//! operates on already-retained k-mers.
//!
//! [`filter`] (the `KmerFilter` trait + its implementations) depends only
//! on `obicompactvec`/`obikseq`, not on `obikindex`. [`dump_layer`] is the
//! extension over `obikindex::KmerIndex` that actually iterates a
//! partition's k-mers through those filters (`iter_partition_kmers`,
//! `iter_partition_kmers_located`) — every k-mer, filtered or not, goes
//! through this same path (`passes_all` on an empty filter list is always
//! `true`), so this crate depends one-way on `obikindex`, not the reverse.
mod filter;
mod dump_layer;
mod predicate;
pub use filter::{
GroupQuorumFilter, KmerFilter, MaxGenomeCount, MaxGenomeFraction, MaxTotalCount,
MinComplexity, MinGenomeCount, MinGenomeFraction, MinTotalCount, passes_all,
};
pub use predicate::{GroupFilterParams, MetaPred};
src/obikindex/src/index/predicate.rs → src/obikfilter/src/predicate.rs
+2 -2
View File
@@ -1,9 +1,9 @@
use std::collections::HashMap;
use crate::index::GroupQuorumFilter;
use crate::filter::GroupQuorumFilter;
use obitaxonomy::{TaxPath, TaxPattern};
use crate::index::meta::{GenomeInfo, IndexMeta};
use obikindex::{GenomeInfo, IndexMeta};
// ── Operator ──────────────────────────────────────────────────────────────────
src/obikidxcache/Cargo.toml
+12
View File
@@ -0,0 +1,12 @@
[package]
name = "obikidxcache"
version = "0.1.0"
edition = "2024"
[dependencies]
obicompactvec = { path = "../obicompactvec" }
ndarray = "0.17"
rayon = "1"
[dev-dependencies]
tempfile = "3"
src/obikindex/src/layer/layered_store.rs → src/obikidxcache/src/layered_store.rs
View File
src/obikidxcache/src/lib.rs
+11
View File
@@ -0,0 +1,11 @@
//! Caching/aggregation over already-opened `obikindex` layer/partition
//! objects — a runtime cache shape, not part of the stateless
//! `Index { Partition { Layer } }` directory-structure model itself.
//! [`LayeredStore`] wraps one entry per layer (or per partition, via
//! `LayeredStore<LayeredStore<S>>`) and propagates `ColumnWeights`/
//! `CountPartials`/`BitPartials` aggregation across them. No dependency on
//! `obikindex` — generic over any `S` implementing those traits.
mod layered_store;
pub use layered_store::LayeredStore;
src/obikindex/src/layer/tests/layered_store.rs → src/obikidxcache/src/tests/layered_store.rs
View File
src/obikindex/Cargo.toml
-6
View File
@@ -19,13 +19,11 @@ niffler = "3.0.0"
memmap2 = "0.9.10"
ndarray = "0.17"
rayon = "1"
crossbeam-channel = "0.5"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
indicatif = "0.18"
tracing = "0.1.44"
bitvec = "1"
hwlocality = { version = "1.0.0-alpha.11", features = ["vendored"], optional = true }
[dev-dependencies]
obiread = { path = "../obiread" }
@@ -33,7 +31,3 @@ obikseq = { path = "../obikseq", features = ["test-utils"] }
tempfile = "3"
tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] }
anyhow = "1"
[features]
default = ["numa"]
numa = ["hwlocality"]
src/obikindex/src/index/builder.rs
+15 -4
View File
@@ -44,7 +44,11 @@ pub trait IndexBuilder: Sized {
/// For construction paths that build partitions from scratch (`select`,
/// `rebuild`). `merge` bootstraps by copying a source index instead, so
/// it does not use this.
fn create_skeleton<P: AsRef<Path>>(output: P, config: IndexConfig, genomes: Vec<GenomeInfo>) -> OKIResult<Self>;
fn create_skeleton<P: AsRef<Path>>(
output: P,
config: IndexConfig,
genomes: Vec<GenomeInfo>,
) -> OKIResult<Self>;
/// Mark `output` as fully indexed, pack its column matrices, and reopen it.
///
@@ -61,7 +65,7 @@ pub trait IndexBuilder: Sized {
impl IndexBuilder for KmerIndex {
fn clear_output_for_create<P: AsRef<Path>>(output: P, force: bool) -> OKIResult<()> {
let output = output.as_ref();
if Self::exists(output) {
if Self::is_an_index(output) {
if force {
fs::remove_dir_all(output).map_err(OKIError::Io)?;
} else {
@@ -74,11 +78,18 @@ impl IndexBuilder for KmerIndex {
Ok(())
}
fn create_skeleton<P: AsRef<Path>>(output: P, config: IndexConfig, genomes: Vec<GenomeInfo>) -> OKIResult<Self> {
fn create_skeleton<P: AsRef<Path>>(
output: P,
config: IndexConfig,
genomes: Vec<GenomeInfo>,
) -> OKIResult<Self> {
let output = output.as_ref();
fs::create_dir_all(output).map_err(OKIError::Io)?;
let meta = IndexMeta::create_at(output, config, genomes).map_err(OKIError::Io)?;
Ok(KmerIndex { root_path: output.to_owned(), meta: Arc::new(meta) })
Ok(KmerIndex {
root_path: output.to_owned(),
meta: Arc::new(meta),
})
}
fn finalize_indexed<P: AsRef<Path>>(output: P, rep: &mut Reporter) -> OKIResult<Self> {
src/obikindex/src/index/common.rs
+6 -46
View File
@@ -1,65 +1,25 @@
use std::path::Path;
use crate::index::error::{OKIError, OKIResult};
use obicompactvec::{PersistentBitVecBuilder, PersistentCompactIntVecBuilder};
use crate::layer::meta::PartitionMeta;
use crate::layer::{layer_dir, IndexMode, OLMError};
use obiskio::{SKError, SKResult};
// ── olm_to_sk ────────────────────────────────────────────────────────────────
pub fn olm_to_sk(e: OLMError, context: &'static str) -> SKError {
match e {
OLMError::Io(e) => SKError::Io(e),
other => SKError::InvalidData {
context,
detail: other.to_string(),
},
}
}
// ── load_meta ────────────────────────────────────────────────────────────────
/// Load PartitionMeta, or recover it by probing layer directories.
/// Indexes built before meta.json was introduced lack the file.
pub(crate) fn load_meta(dir: &Path, context: &'static str) -> SKResult<PartitionMeta> {
match PartitionMeta::load(dir) {
Ok(m) => Ok(m),
Err(e) if matches!(e, OLMError::Io(ref io_e) if io_e.kind() == std::io::ErrorKind::NotFound) =>
{
let mut n = 0usize;
while layer_dir(dir, n).exists() {
n += 1;
}
let m = PartitionMeta {
n_layers: n,
mode: IndexMode::default(),
};
m.save(dir).map_err(|e| olm_to_sk(e, context))?;
Ok(m)
}
Err(e) => Err(olm_to_sk(e, context)),
}
}
// ── ColBuilder ────────────────────────────────────────────────────────────────
pub(crate) enum ColBuilder {
pub enum ColBuilder {
Bit(PersistentBitVecBuilder),
Int(PersistentCompactIntVecBuilder),
}
impl ColBuilder {
pub(crate) fn set_val(&mut self, slot: usize, value: u32) {
pub fn set_val(&mut self, slot: usize, value: u32) {
match self {
ColBuilder::Bit(b) => b.set(slot, value > 0),
ColBuilder::Int(b) => b.set(slot, value),
}
}
pub(crate) fn close(self) -> SKResult<()> {
pub fn close(self) -> OKIResult<()> {
match self {
ColBuilder::Bit(b) => b.close().map_err(SKError::Io),
ColBuilder::Int(b) => b.close().map_err(SKError::Io),
ColBuilder::Bit(b) => b.close().map_err(OKIError::Io),
ColBuilder::Int(b) => b.close().map_err(OKIError::Io),
}
}
}
src/obikindex/src/index/error.rs
+33 -14
View File
@@ -8,6 +8,7 @@ pub enum OKIError {
Io(io::Error),
Json(serde_json::Error),
Partition(SKError),
Layer(SKError),
/// Source index is not in `Indexed` state.
NotIndexed(std::path::PathBuf),
/// Source indexes have incompatible configurations (k, m, n_bits).
@@ -20,6 +21,9 @@ pub enum OKIError {
InvalidInput(String),
/// Sources mix exact and approximate evidence, or use incompatible approx parameters.
IncompatibleEvidence(String),
/// Malformed data encountered while reading/writing a layer file, tagged
/// with the operation that hit it.
InvalidData { context: &'static str, detail: String },
}
pub type OKIResult<T> = Result<T, OKIError>;
@@ -27,15 +31,24 @@ pub type OKIResult<T> = Result<T, OKIError>;
impl fmt::Display for OKIError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
OKIError::Io(e) => write!(f, "I/O error: {e}"),
OKIError::Json(e) => write!(f, "JSON error: {e}"),
OKIError::Partition(e) => write!(f, "partition error: {e}"),
OKIError::NotIndexed(p) => write!(f, "index not fully built: {}", p.display()),
OKIError::Io(e) => write!(f, "I/O error: {e}"),
OKIError::Json(e) => write!(f, "JSON error: {e}"),
OKIError::Partition(e) => write!(f, "partition error: {e}"),
OKIError::Layer(e) => write!(f, "layer error: {e}"),
OKIError::NotIndexed(p) => write!(f, "index not fully built: {}", p.display()),
OKIError::IncompatibleConfig => write!(f, "incompatible index configurations"),
OKIError::MismatchedMode => write!(f, "count mode requires all sources to have with_counts=true"),
OKIError::DuplicateGenomeLabel(l) => write!(f, "duplicate genome label across sources: {l}"),
OKIError::InvalidInput(m) => write!(f, "invalid input: {m}"),
OKIError::IncompatibleEvidence(m) => write!(f, "incompatible evidence: {m}"),
OKIError::MismatchedMode => write!(
f,
"count mode requires all sources to have with_counts=true"
),
OKIError::DuplicateGenomeLabel(l) => {
write!(f, "duplicate genome label across sources: {l}")
}
OKIError::InvalidInput(m) => write!(f, "invalid input: {m}"),
OKIError::IncompatibleEvidence(m) => write!(f, "incompatible evidence: {m}"),
OKIError::InvalidData { context, detail } => {
write!(f, "invalid data ({context}): {detail}")
}
}
}
}
@@ -43,22 +56,28 @@ impl fmt::Display for OKIError {
impl std::error::Error for OKIError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
match self {
OKIError::Io(e) => Some(e),
OKIError::Json(e) => Some(e),
OKIError::Io(e) => Some(e),
OKIError::Json(e) => Some(e),
OKIError::Partition(e) => Some(e),
_ => None, // IncompatibleConfig, MismatchedMode, DuplicateGenomeLabel
_ => None, // IncompatibleConfig, MismatchedMode, DuplicateGenomeLabel
}
}
}
impl From<io::Error> for OKIError {
fn from(e: io::Error) -> Self { OKIError::Io(e) }
fn from(e: io::Error) -> Self {
OKIError::Io(e)
}
}
impl From<serde_json::Error> for OKIError {
fn from(e: serde_json::Error) -> Self { OKIError::Json(e) }
fn from(e: serde_json::Error) -> Self {
OKIError::Json(e)
}
}
impl From<SKError> for OKIError {
fn from(e: SKError) -> Self { OKIError::Partition(e) }
fn from(e: SKError) -> Self {
OKIError::Partition(e)
}
}
src/obikindex/src/index/kmer_index.rs
+76 -43
View File
@@ -2,13 +2,13 @@ use std::fs;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use crate::layer::meta::PartitionMeta;
use crate::layer::KmerLayer;
use crate::partition::KmerPartition;
use obisys::progress_bar;
use rayon::prelude::*;
use obikseq::{set_k, set_m};
use crate::index::common::load_meta;
use crate::index::error::{OKIError, OKIResult};
use crate::index::meta::{GenomeInfo, IndexConfig, IndexMeta};
@@ -28,12 +28,15 @@ impl KmerIndex {
genome_info: Option<GenomeInfo>,
) -> OKIResult<Self> {
let root_path = path.as_ref().to_owned();
fs::create_dir_all(&root_path).map_err(OKIError::Io)?;
fs::create_dir(&root_path).map_err(OKIError::Io)?;
set_k(config.kmer_size);
set_m(config.minimizer_size);
let genomes = genome_info.into_iter().collect();
let meta = IndexMeta::create_at(&root_path, config, genomes).map_err(OKIError::Io)?;
Ok(Self { root_path, meta: Arc::new(meta) })
Ok(Self {
root_path,
meta: Arc::new(meta),
})
}
pub fn open<P: AsRef<Path>>(path: P) -> OKIResult<Self> {
@@ -41,16 +44,30 @@ impl KmerIndex {
let meta = IndexMeta::open_at(&root_path).map_err(OKIError::Io)?;
set_k(meta.config.kmer_size);
set_m(meta.config.minimizer_size);
Ok(Self { root_path, meta: Arc::new(meta) })
Ok(Self {
root_path,
meta: Arc::new(meta),
})
}
/// Return `true` if `path` contains an `index.meta` file.
pub fn exists<P: AsRef<Path>>(path: P) -> bool {
/// Return `true` if `path` points to a valid obikmer index directory.
///
/// A directory is considered an index only when it contains an `index.meta`
/// file. The absence of that file means the directory is not an index, even
/// if it otherwise exists.
///
/// This is a static method (associated function), not an instance method.
/// Call it as `KmerIndex::is_an_index(path)`.
///
/// `path` is generic over any type implementing [`AsRef<Path>`]. This allows
/// callers to pass `&str`, `String`, `PathBuf`, `&Path`, etc. without an
/// explicit conversion; the value is coerced via `.as_ref()` inside.
pub fn is_an_index<P: AsRef<Path>>(path: P) -> bool {
IndexMeta::exists(path.as_ref())
}
/// The index's root directory.
pub fn root_path(&self) -> &Path {
pub fn dir(&self) -> &Path {
&self.root_path
}
pub fn meta(&self) -> Arc<IndexMeta> {
@@ -102,42 +119,48 @@ impl KmerIndex {
/// same directory only indirectly, through this method — `obikindexer`
/// depends on `KmerIndex`, not the other way around — see
/// `DevDocMD/implementation/partition_layer_cache.md`.
pub fn partition_dir(&self, i: usize) -> PathBuf {
crate::partition::partition_dir(&self.root_path, i)
pub(crate) fn partition_dir(&self, i: usize) -> OKIResult<PathBuf> {
Ok(self.partition(i)?.dir().to_path_buf())
}
pub fn partition(&self, i: usize) -> OKIResult<KmerPartition> {
let n = self.n_partitions();
if i >= n {
return Err(OKIError::InvalidData {
context: "New KmerPartition",
detail: format!("Partition {i} requested, Index has only {n} partitions"),
});
}
Ok(KmerPartition::new(self, i))
}
pub(crate) fn layer_dir(&self, i: usize, j: usize) -> OKIResult<PathBuf> {
Ok(self.layer(i, j)?.dir().to_path_buf())
}
pub fn layer(&self, i: usize, j: usize) -> OKIResult<KmerLayer> {
self.partition(i)?.layer(j)
}
/// Path of partition `i`'s layered-index directory (`<partition>/index`).
pub fn index_dir(&self, i: usize) -> PathBuf {
pub(crate) fn index_dir(&self, i: usize) -> PathBuf {
crate::partition::index_dir(&self.root_path, i)
}
/// Path of layer `l` within partition `i`'s layered index.
pub fn layer_dir(&self, i: usize, l: usize) -> PathBuf {
crate::layer::layer_dir(&self.index_dir(i), l)
/// Number of layers in partition `i` — a disk scan, not a metadata read
/// (see [`KmerPartition::n_layers`](crate::partition::KmerPartition::n_layers)).
pub fn n_layers(&self, i: usize) -> OKIResult<usize> {
Ok(self.partition(i)?.n_layers())
}
/// Partition `i`'s metadata (layer count, evidence mode). Returns
/// `obiskio::SKResult`, not `OKIResult` — matches the error convention
/// of the partition/layer-construction code below (moved here from
/// the former `obikpartitionner` crate, which predates `OKIError`);
/// `?` still converts it to `OKIResult` at any call site that needs
/// one (`OKIError: From<SKError>`).
pub fn partition_meta(&self, i: usize) -> obiskio::SKResult<PartitionMeta> {
load_meta(&self.index_dir(i), "partition_meta")
}
/// Number of layers in partition `i` — see [`partition_meta`](Self::partition_meta).
pub fn n_layers(&self, i: usize) -> obiskio::SKResult<usize> {
Ok(self.partition_meta(i)?.n_layers)
}
/// Evidence mode partition `i` was actually built with — see
/// [`partition_meta`](Self::partition_meta). Distinct from
/// [`evidence_mode`](Self::evidence_mode): that one is the index-level
/// config, this one is per-partition ground truth (the two agree in
/// practice, but this is what `Layer::open` needs).
pub fn partition_mode(&self, i: usize) -> obiskio::SKResult<crate::layer::IndexMode> {
Ok(self.partition_meta(i)?.mode)
/// Evidence mode partition `i` was actually built with — detected from
/// layer 0's evidence files on disk (see
/// [`KmerPartition::mode`](crate::partition::KmerPartition::mode)).
/// Distinct from [`evidence_mode`](Self::evidence_mode): that one is the
/// index-level config, this one is per-partition ground truth (the two
/// agree in practice, but this is what `Layer::open` needs).
pub fn partition_mode(&self, i: usize) -> OKIResult<crate::layer::IndexMode> {
self.partition(i)?.mode()
}
/// Number of layers per partition.
@@ -150,8 +173,8 @@ impl KmerIndex {
}
/// Path to the unitigs file for partition `part`, layer `layer`.
pub fn layer_unitigs_path(&self, part: usize, layer: usize) -> PathBuf {
self.layer_dir(part, layer).join("unitigs.bin")
pub fn layer_unitigs_path(&self, part: usize, layer: usize) -> OKIResult<PathBuf> {
Ok(self.layer_dir(part, layer)?.join("unitigs.bin"))
}
/// Pack all partition matrices into single-file format (presence → .pbmx, counts → .pcmx).
@@ -164,13 +187,14 @@ impl KmerIndex {
/// on-disk format (see `DevDocMD/architecture/siblings.md`) — count
/// matrices are unaffected, sparse count matrices aren't implemented
/// (see the sparse-matrix design plan's "Explicitly deferred").
pub fn pack_matrices(&self, sparse: bool) -> OKIResult<()> {
/// TODO: To be moved somewhere else
pub(crate) fn pack_matrices(&self, sparse: bool) -> OKIResult<()> {
use obicompactvec::{pack_bit_matrix, pack_compact_int_matrix, pack_sparse_bit_matrix};
let n = self.n_partitions();
let order: Vec<usize> = (0..n).collect();
let pb = progress_bar("pack", n as u64, "partitions");
crate::index::numa::PartitionRunner::new().run(
obisys::PartitionRunner::new().run(
&order,
|i| -> OKIResult<()> {
let index_dir = self.index_dir(i);
@@ -179,7 +203,7 @@ impl KmerIndex {
}
let n_layers = self.n_layers(i)?;
for l in 0..n_layers {
let layer_dir = self.layer_dir(i, l);
let layer_dir = self.layer_dir(i, l)?;
let presence_dir = layer_dir.join("presence");
let counts_dir = layer_dir.join("counts");
if presence_dir.exists() {
@@ -207,7 +231,8 @@ impl KmerIndex {
///
/// Old indexes were built before this file was required. The number of
/// kmers is recovered from `unitigs.bin`, which is always present.
pub fn upgrade_layer_meta(&self) -> OKIResult<()> {
/// TODO: Should not exist anymore
pub(crate) fn upgrade_layer_meta(&self) -> OKIResult<()> {
use obicompactvec::LayerMeta;
use obiskio::UnitigFileReader;
@@ -229,7 +254,15 @@ impl KmerIndex {
}
};
for l in 0..n_layers {
let layer_dir = self.layer_dir(i, l);
let layer_dir = match self.layer_dir(i, l) {
Ok(d) => d,
Err(e) => {
return Some(OKIError::Io(std::io::Error::new(
std::io::ErrorKind::Other,
e.to_string(),
)));
}
};
let meta_path = layer_dir.join(LayerMeta::FILENAME);
if meta_path.exists() {
continue;
src/obikindex/src/index/meta.rs
+71 -35
View File
@@ -21,8 +21,44 @@ pub struct GenomeInfo {
}
impl GenomeInfo {
/// Panics if `label` fails [`validate_label`](Self::validate_label) —
/// an invalid genome label is a programmer error to surface loudly,
/// not a recoverable condition for this constructor to paper over.
/// Callers that need a graceful error (e.g. a CLI parsing user input)
/// should call [`validate_label`](Self::validate_label) themselves
/// first and report it their own way.
pub fn new(label: impl Into<String>) -> Self {
Self { label: label.into(), meta: HashMap::new() }
let label = label.into();
Self::validate_label(&label).expect("invalid genome label");
Self {
label,
meta: HashMap::new(),
}
}
/// Validate a user-supplied genome label.
///
/// Forbidden: `/` (filesystem separator), `=` (--new-label parser separator),
/// `\0` (null byte), `\n`, `\r`, `\t` (break CSV output).
/// Empty labels are also rejected.
pub fn validate_label(label: &str) -> Result<(), String> {
if label.is_empty() {
return Err("genome label must not be empty".into());
}
const FORBIDDEN: &[char] = &['/', '=', '\0', '\n', '\r', '\t'];
if let Some(c) = label.chars().find(|c| FORBIDDEN.contains(c)) {
let display = match c {
'\0' => "\\0 (null)".to_string(),
'\n' => "\\n (newline)".to_string(),
'\r' => "\\r (carriage return)".to_string(),
'\t' => "\\t (tab)".to_string(),
c => format!("'{c}'"),
};
return Err(format!(
"genome label contains forbidden character {display}"
));
}
Ok(())
}
}
@@ -47,7 +83,7 @@ pub struct IndexConfig {
/// On-disk shape of `index.meta` — the whole file, read/written atomically
/// as one JSON blob by every `IndexMeta` operation below.
#[derive(Debug, Clone, Serialize, Deserialize)]
struct OnDiskMeta {
struct IndexMetadata {
version: u32,
config: IndexConfig,
#[serde(default)]
@@ -81,21 +117,29 @@ impl IndexMeta {
/// Create a brand-new `index.meta` for `index`, from fixed config and
/// an initial genome list (often empty — genomes are usually added
/// later via [`push_genome`](Self::push_genome)).
pub fn create(index: &KmerIndex, config: IndexConfig, genomes: Vec<GenomeInfo>) -> io::Result<Self> {
Self::create_at(index.root_path(), config, genomes)
pub fn create(
index: &KmerIndex,
config: IndexConfig,
genomes: Vec<GenomeInfo>,
) -> io::Result<Self> {
Self::create_at(index.dir(), config, genomes)
}
/// Same as [`create`](Self::create), taking a raw path — for
/// `KmerIndex::create` itself, which doesn't have a complete
/// `KmerIndex` yet to hand in (it's what's being built).
pub(crate) fn create_at(root_path: &Path, config: IndexConfig, genomes: Vec<GenomeInfo>) -> io::Result<Self> {
pub(crate) fn create_at(
root_path: &Path,
config: IndexConfig,
genomes: Vec<GenomeInfo>,
) -> io::Result<Self> {
let meta = Self {
root_path: root_path.to_owned(),
version: META_VERSION,
config,
lock: RwLock::new(()),
};
meta.write_full(&OnDiskMeta {
meta.write_full(&IndexMetadata {
version: meta.version,
config: meta.config.clone(),
genomes,
@@ -106,7 +150,7 @@ impl IndexMeta {
/// Reopen the `index.meta` already on disk for `index`.
pub fn open(index: &KmerIndex) -> io::Result<Self> {
Self::open_at(index.root_path())
Self::open_at(index.dir())
}
pub(crate) fn open_at(root_path: &Path) -> io::Result<Self> {
@@ -119,7 +163,12 @@ impl IndexMeta {
})
}
pub fn exists(root: &Path) -> bool {
/// Returns whether the metadata file exists in `root`.
///
/// This is a static method (associated function), not an instance method.
/// It only checks for the presence of the `index.meta` file inside the
/// given directory; it does not validate its contents.
pub(crate) fn exists(root: &Path) -> bool {
root.join(META_FILENAME).exists()
}
@@ -177,10 +226,20 @@ impl IndexMeta {
/// (e.g. `self.meta = Arc::new(IndexMeta::open(self)?)`) — this method
/// only updates the file, it has no way to reach back into whatever
/// `KmerIndex` holds it.
pub fn rewrite_config(&self, config: IndexConfig, genomes: Vec<GenomeInfo>) -> io::Result<()> {
/// TODO: This methode is strange
pub(crate) fn rewrite_config(
&self,
config: IndexConfig,
genomes: Vec<GenomeInfo>,
) -> io::Result<()> {
let _guard = self.lock.write().unwrap();
let state = Self::read_full(&self.root_path)?.state;
self.write_full(&OnDiskMeta { version: self.version, config, genomes, state })
self.write_full(&IndexMetadata {
version: self.version,
config,
genomes,
state,
})
}
pub fn set_state(&self, state: IndexState) -> io::Result<()> {
@@ -221,12 +280,12 @@ impl IndexMeta {
// ── private ──────────────────────────────────────────────────────────────
fn read_full(root: &Path) -> io::Result<OnDiskMeta> {
fn read_full(root: &Path) -> io::Result<IndexMetadata> {
let file = fs::File::open(root.join(META_FILENAME))?;
serde_json::from_reader(file).map_err(io::Error::other)
}
fn write_full(&self, on_disk: &OnDiskMeta) -> io::Result<()> {
fn write_full(&self, on_disk: &IndexMetadata) -> io::Result<()> {
let file = fs::File::create(self.root_path.join(META_FILENAME))?;
serde_json::to_writer_pretty(file, on_disk).map_err(io::Error::other)
}
@@ -248,26 +307,3 @@ fn label_from_path(path: &Path) -> String {
s
}
}
/// Validate a user-supplied genome label.
///
/// Forbidden: `/` (filesystem separator), `=` (--new-label parser separator),
/// `\0` (null byte), `\n`, `\r`, `\t` (break CSV output).
/// Empty labels are also rejected.
pub fn validate_label(label: &str) -> Result<(), String> {
if label.is_empty() {
return Err("genome label must not be empty".into());
}
const FORBIDDEN: &[char] = &['/', '=', '\0', '\n', '\r', '\t'];
if let Some(c) = label.chars().find(|c| FORBIDDEN.contains(c)) {
let display = match c {
'\0' => "\\0 (null)".to_string(),
'\n' => "\\n (newline)".to_string(),
'\r' => "\\r (carriage return)".to_string(),
'\t' => "\\t (tab)".to_string(),
c => format!("'{c}'"),
};
return Err(format!("genome label contains forbidden character {display}"));
}
Ok(())
}
src/obikindex/src/index/mod.rs
+2 -28
View File
@@ -1,39 +1,13 @@
pub mod error;
pub mod meta;
pub mod predicate;
pub mod state;
mod builder;
mod common;
mod distance;
mod dump;
mod dump_layer;
pub mod filter;
mod graph_pipeline;
mod kmer_index;
mod matrix_store;
mod merge;
mod merge_layer;
mod numa;
mod query_layer;
mod rebuild;
mod rebuild_layer;
mod reindex;
mod select;
mod select_layer;
mod stats;
pub use error::{OKIError, OKIResult};
pub use builder::IndexBuilder;
pub use common::olm_to_sk;
pub use graph_pipeline::{materialize_layer, write_graph_as_unitigs};
pub use distance::{DistanceMetric, DistanceOutput};
pub use filter::{GroupQuorumFilter, KmerFilter, passes_all};
pub use common::ColBuilder;
pub use kmer_index::KmerIndex;
pub use merge_layer::MergeMode;
pub use meta::{validate_label, GenomeInfo, IndexConfig, IndexMeta, META_FILENAME};
pub use predicate::{GroupFilterParams, MetaPred};
pub use query_layer::{KmerDesc, QueryHit, QueryStats};
pub use select_layer::{AggOp, OutputCol};
pub use meta::{GenomeInfo, IndexConfig, IndexMeta, META_FILENAME};
pub use state::IndexState;
pub use stats::IndexBitsPerKmer;
pub use numa::PartitionRunner;
src/obikindex/src/index/progress.rs
-21
View File
@@ -1,21 +0,0 @@
use std::sync::OnceLock;
use indicatif::MultiProgress;
static MULTI: OnceLock<MultiProgress> = OnceLock::new();
/// Initialise the shared progress display. Call once from the binary before
/// any index operation. Subsequent calls are silently ignored.
pub fn init(multi: MultiProgress) {
let _ = MULTI.set(multi);
}
/// Return the shared `MultiProgress`, creating a plain default one if the
/// binary never called [`init`].
pub fn get() -> &'static MultiProgress {
MULTI.get_or_init(MultiProgress::new)
}
pub(crate) fn multi() -> &'static MultiProgress {
get()
}
src/obikindex/src/layer/content_layer.rs
+108 -61
View File
@@ -24,77 +24,103 @@
use std::path::{Path, PathBuf};
use crate::layer::utils::LAYERNAME_SUFFIX;
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
use obikseq::CanonicalKmer;
use crate::layer::error::OLMResult;
use crate::layer::meta::IndexMode;
use crate::index::error::OKIResult;
use crate::layer::mphf_layer::{
EVIDENCE_FILE, EvidenceKind, FINGERPRINT_FILE, MPHF_FILE, UNITIGS_FILE,
};
use crate::layer::typed_layer::{COUNTS_DIR, LayerContent, PRESENCE_DIR, TypedLayer};
use crate::partition::KmerPartition;
/// Superkmer-file extension used by every pre-layer-construction artifact
/// below (`raw`/`dereplicated`) — an implementation detail of the SK file
/// format, never meant to leak as a literal string past this module.
const SK_EXT: &str = "skmer.zst";
/// One layer, at any point in its life — see the module docs. Only
/// [`Empty`](Layer::Empty) and the two ready-to-read states
/// (`Count`/`Presence`) exist so far; states in between (unitigs written,
/// MPHF built, evidence built, matrix not yet built) are not represented
/// yet.
///
/// Every variant carries `dir` + `id` — a layer's identity never depends on
/// its state. `id` is its own layer number within the partition (not yet
/// consumed by anything — `KmerPartition` doesn't hand it in yet, that
/// lands with `KmerLayer::at`). `dir`/`id` live here, on `KmerLayer` itself,
/// not inside `TypedLayer`/`MphfLayer`: those only ever need `dir`
/// transiently, while opening — `KmerLayer` is what has to answer `.dir()`
/// after the fact, so it's the one that keeps it.
pub enum KmerLayer {
/// Just a directory — nothing built yet. Every method below other than
/// the path accessors panics on this variant: calling them means the
/// caller assumed a layer was ready when it wasn't, an implementation
/// error to surface loudly, not paper over with a default value.
Empty {
Empty { dir: PathBuf, id: usize },
Count {
dir: PathBuf,
id: usize,
layer: TypedLayer<PersistentCompactIntMatrix>,
},
Presence {
dir: PathBuf,
id: usize,
layer: TypedLayer<PersistentBitMatrix>,
},
Count(TypedLayer<PersistentCompactIntMatrix>),
Presence(TypedLayer<PersistentBitMatrix>),
}
impl KmerLayer {
pub fn new(partition: &KmerPartition, i: usize) -> Self {
let dir = partition.dir().join(format!("{LAYERNAME_SUFFIX}{i}"));
KmerLayer::Empty { dir, id: i }
}
/// An empty shell at `dir` — creates the directory (if it doesn't
/// already exist) but nothing inside it. The starting state for
/// building a new layer.
pub fn create(dir: &Path) -> std::io::Result<Self> {
std::fs::create_dir_all(dir)?;
Ok(KmerLayer::Empty {
dir: dir.to_owned(),
})
pub fn create(self) -> std::io::Result<Self> {
std::fs::create_dir_all(self.dir())?;
Ok(self)
}
/// Open one layer, auto-detecting count vs. presence from what is
/// actually on disk (`counts/` present and wanted, else presence) — the
/// single source of truth every caller that opens a layer's own matrix
/// should share, so two callers can never disagree about which format a
/// given layer is. Dense vs. sparse presence storage is
/// `PersistentBitMatrix::open`'s own concern (it's a 4-way
/// `Columnar`/`Packed`/`Sparse`/`Implicit` enum internally), not decided
/// here.
pub fn open(layer_dir: &Path, mode: &IndexMode, with_counts: bool) -> OLMResult<Self> {
if with_counts && layer_dir.join(COUNTS_DIR).exists() {
return TypedLayer::<PersistentCompactIntMatrix>::open(layer_dir, mode)
.map(KmerLayer::Count);
/// Open this layer, auto-detecting count vs. presence from what is
/// actually on disk (`counts/` present, else presence) — the single
/// source of truth every caller that opens a layer's own matrix should
/// share, so two callers can never disagree about which format a given
/// layer is. Dense vs. sparse presence storage is `PersistentBitMatrix::
/// open`'s own concern (a 4-way `Columnar`/`Packed`/`Sparse`/`Implicit`
/// enum internally), not decided here. A no-op (returns `self`
/// unchanged) if already `Count`/`Presence`.
pub fn open(self) -> OKIResult<Self> {
let (dir, id) = match self {
KmerLayer::Empty { dir, id } => (dir, id),
already_open => return Ok(already_open),
};
if dir.join(COUNTS_DIR).exists() {
let layer = TypedLayer::<PersistentCompactIntMatrix>::open(&dir)?;
Ok(KmerLayer::Count { dir, id, layer })
} else {
let layer = TypedLayer::<PersistentBitMatrix>::open(&dir)?;
Ok(KmerLayer::Presence { dir, id, layer })
}
TypedLayer::<PersistentBitMatrix>::open(layer_dir, mode).map(KmerLayer::Presence)
}
// ── Paths — only meaningful before anything is built ───────────────
// ── Paths — valid in every state, `dir` never varies with content ────
//
// Once a layer is `Count`/`Presence`, its caller already knows the
// directory (it had to pass it to `open`) — these exist for builder
// code holding an `Empty` layer, so the `mphf.bin`/`unitigs.bin`/…
// naming stays defined once, here, rather than re-declared as
// string literals at every call site that writes into a layer
// directory (the same duplication `layer_dir`/`index_dir` fixed one
// level up — see `DevDocMD/implementation/partition_layer_cache.md`).
// The `mphf.bin`/`unitigs.bin`/… naming stays defined once, here,
// rather than re-declared as string literals at every call site that
// writes into a layer directory (the same duplication `layer_dir`/
// `index_dir` fixed one level up — see `DevDocMD/implementation/
// partition_layer_cache.md`).
/// This layer's own directory. Panics on `Count`/`Presence` — by that
/// point the caller already has the directory it opened with; asking
/// again here would mean it lost track of its own state.
/// This layer's own directory.
pub fn dir(&self) -> &Path {
match self {
KmerLayer::Empty { dir } => dir,
_ => panic!("Layer::dir() only available on Empty — caller already has this path"),
KmerLayer::Empty { dir, .. } => dir,
KmerLayer::Count { dir, .. } => dir,
KmerLayer::Presence { dir, .. } => dir,
}
}
@@ -117,36 +143,57 @@ impl KmerLayer {
self.dir().join(PRESENCE_DIR)
}
/// Path of this layer's raw, not-yet-dereplicated superkmer file —
/// written by whichever algorithm routes superkmers into this layer
/// (today: `obikindexer::algorithms::partitionner::PartitionRouter`),
/// read by whichever algorithm dereplicates it (today:
/// `obikindexer::algorithms::dereplicator::Dereplicator`). Naming this
/// once here, rather than in either algorithm submodule, is what lets
/// the two agree on the filename without depending on each other
/// directly — see `DevDocMD/implementation/partition_layer_cache.md`.
pub fn raw_superkmers_path(&self) -> PathBuf {
self.dir().join(format!("raw.{SK_EXT}"))
}
/// Path of this layer's dereplicated superkmer file — written by
/// `obikindexer::algorithms::dereplicator::Dereplicator`, read by
/// whichever algorithm counts kmer abundances from it (today:
/// `obikindexer::algorithms::partitionner::PartitionRouter::count_kmer`)
/// and, later, by `obikindex::build_index_layer` to build the real layer.
pub fn dereplicated_superkmers_path(&self) -> PathBuf {
self.dir().join(format!("dereplicated.{SK_EXT}"))
}
// ── Ready-only surface ───────────────────────────────────────────────
pub fn content(&self) -> LayerContent {
match self {
KmerLayer::Count(_) => LayerContent::Count,
KmerLayer::Presence(_) => LayerContent::Presence,
KmerLayer::Count { .. } => LayerContent::Count,
KmerLayer::Presence { .. } => LayerContent::Presence,
KmerLayer::Empty { .. } => panic!("Layer::content() called on an Empty layer"),
}
}
pub fn evidence_kind(&self) -> EvidenceKind {
match self {
KmerLayer::Count(l) => l.evidence_kind(),
KmerLayer::Presence(l) => l.evidence_kind(),
KmerLayer::Count { layer, .. } => layer.evidence_kind(),
KmerLayer::Presence { layer, .. } => layer.evidence_kind(),
KmerLayer::Empty { .. } => panic!("Layer::evidence_kind() called on an Empty layer"),
}
}
pub fn n(&self) -> usize {
match self {
KmerLayer::Count(l) => l.n(),
KmerLayer::Presence(l) => l.n(),
KmerLayer::Count { layer, .. } => layer.n(),
KmerLayer::Presence { layer, .. } => layer.n(),
KmerLayer::Empty { .. } => panic!("Layer::n() called on an Empty layer"),
}
}
pub fn find_slot(&self, kmer: CanonicalKmer) -> Option<usize> {
match self {
KmerLayer::Count(l) => l.find_slot(kmer),
KmerLayer::Presence(l) => l.find_slot(kmer),
KmerLayer::Count { layer, .. } => layer.find_slot(kmer),
KmerLayer::Presence { layer, .. } => layer.find_slot(kmer),
KmerLayer::Empty { .. } => panic!("Layer::find_slot() called on an Empty layer"),
}
}
@@ -156,16 +203,16 @@ impl KmerLayer {
/// the evidence check `find_slot`/`find` would perform is redundant.
pub fn hash_batch(&self, kmers: &[CanonicalKmer]) -> Vec<usize> {
match self {
KmerLayer::Count(l) => l.hash_batch(kmers),
KmerLayer::Presence(l) => l.hash_batch(kmers),
KmerLayer::Count { layer, .. } => layer.hash_batch(kmers),
KmerLayer::Presence { layer, .. } => layer.hash_batch(kmers),
KmerLayer::Empty { .. } => panic!("Layer::hash_batch() called on an Empty layer"),
}
}
pub fn n_cols(&self) -> usize {
match self {
KmerLayer::Count(l) => l.n_cols(),
KmerLayer::Presence(l) => l.n_cols(),
KmerLayer::Count { layer, .. } => layer.n_cols(),
KmerLayer::Presence { layer, .. } => layer.n_cols(),
KmerLayer::Empty { .. } => panic!("Layer::n_cols() called on an Empty layer"),
}
}
@@ -178,10 +225,10 @@ impl KmerLayer {
/// presence (`!= 0`) in place.
pub fn fill_sub_matrix_carries(&self, slots: &[usize], out: &mut [Vec<bool>]) {
match self {
KmerLayer::Presence(l) => l.fill_sub_matrix(slots, out),
KmerLayer::Count(l) => {
KmerLayer::Presence { layer, .. } => layer.fill_sub_matrix(slots, out),
KmerLayer::Count { layer, .. } => {
let mut counts: Vec<Vec<u32>> = out.iter().map(|_| Vec::new()).collect();
l.fill_sub_matrix(slots, &mut counts);
layer.fill_sub_matrix(slots, &mut counts);
for (o, c) in out.iter_mut().zip(counts.iter()) {
o.clear();
o.extend(c.iter().map(|&v| v != 0));
@@ -196,8 +243,8 @@ impl KmerLayer {
/// Raw MPHF lookup: kmer → slot, no membership check.
pub fn hash(&self, kmer: CanonicalKmer) -> usize {
match self {
KmerLayer::Count(l) => l.hash(kmer),
KmerLayer::Presence(l) => l.hash(kmer),
KmerLayer::Count { layer, .. } => layer.hash(kmer),
KmerLayer::Presence { layer, .. } => layer.hash(kmer),
KmerLayer::Empty { .. } => panic!("Layer::hash() called on an Empty layer"),
}
}
@@ -205,8 +252,8 @@ impl KmerLayer {
/// Iterate over all canonical kmers in the layer, in deterministic order.
pub fn iter_kmers(&self) -> crate::layer::mphf_layer::KmerIter {
match self {
KmerLayer::Count(l) => l.iter_kmers(),
KmerLayer::Presence(l) => l.iter_kmers(),
KmerLayer::Count { layer, .. } => layer.iter_kmers(),
KmerLayer::Presence { layer, .. } => layer.iter_kmers(),
KmerLayer::Empty { .. } => panic!("Layer::iter_kmers() called on an Empty layer"),
}
}
@@ -215,8 +262,8 @@ impl KmerLayer {
/// sequence index in `unitigs.bin`.
pub fn enumerate_kmers(&self) -> std::iter::Enumerate<crate::layer::mphf_layer::KmerIter> {
match self {
KmerLayer::Count(l) => l.enumerate_kmers(),
KmerLayer::Presence(l) => l.enumerate_kmers(),
KmerLayer::Count { layer, .. } => layer.enumerate_kmers(),
KmerLayer::Presence { layer, .. } => layer.enumerate_kmers(),
KmerLayer::Empty { .. } => panic!("Layer::enumerate_kmers() called on an Empty layer"),
}
}
@@ -224,8 +271,8 @@ impl KmerLayer {
/// Iterate over the layer's canonical kmers in batches of `n`.
pub fn iter_kmers_batch(&self, n: usize) -> crate::layer::mphf_layer::KmerBatchIter {
match self {
KmerLayer::Count(l) => l.iter_kmers_batch(n),
KmerLayer::Presence(l) => l.iter_kmers_batch(n),
KmerLayer::Count { layer, .. } => layer.iter_kmers_batch(n),
KmerLayer::Presence { layer, .. } => layer.iter_kmers_batch(n),
KmerLayer::Empty { .. } => panic!("Layer::iter_kmers_batch() called on an Empty layer"),
}
}
@@ -237,8 +284,8 @@ impl KmerLayer {
n: usize,
) -> Box<dyn Iterator<Item = (usize, Vec<CanonicalKmer>)> + Send + 'static> {
match self {
KmerLayer::Count(l) => Box::new(l.enumerate_kmers_batch(n)),
KmerLayer::Presence(l) => Box::new(l.enumerate_kmers_batch(n)),
KmerLayer::Count { layer, .. } => Box::new(layer.enumerate_kmers_batch(n)),
KmerLayer::Presence { layer, .. } => Box::new(layer.enumerate_kmers_batch(n)),
KmerLayer::Empty { .. } => {
panic!("Layer::enumerate_kmers_batch() called on an Empty layer")
}
src/obikindex/src/layer/error.rs
-50
View File
@@ -1,50 +0,0 @@
use std::fmt;
use std::io;
#[derive(Debug)]
pub enum OLMError {
Io(io::Error),
Json(serde_json::Error),
Mphf(String),
InvalidLayer(String),
}
pub type OLMResult<T> = Result<T, OLMError>;
impl fmt::Display for OLMError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
OLMError::Io(e) => write!(f, "I/O error: {e}"),
OLMError::Json(e) => write!(f, "JSON error: {e}"),
OLMError::Mphf(s) => write!(f, "MPHF error: {s}"),
OLMError::InvalidLayer(s) => write!(f, "invalid layer: {s}"),
}
}
}
impl std::error::Error for OLMError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
match self {
OLMError::Io(e) => Some(e),
OLMError::Json(e) => Some(e),
_ => None,
}
}
}
impl From<io::Error> for OLMError {
fn from(e: io::Error) -> Self { OLMError::Io(e) }
}
impl From<serde_json::Error> for OLMError {
fn from(e: serde_json::Error) -> Self { OLMError::Json(e) }
}
impl From<obiskio::SKError> for OLMError {
fn from(e: obiskio::SKError) -> Self {
match e {
obiskio::SKError::Io(io_err) => OLMError::Io(io_err),
other => OLMError::InvalidLayer(other.to_string()),
}
}
}
src/obikindex/src/layer/evidence.rs
+10 -10
View File
@@ -6,14 +6,14 @@ use std::path::Path;
use memmap2::Mmap;
use crate::layer::error::{OLMError, OLMResult};
use crate::index::error::{OKIError, OKIResult};
pub struct Evidence {
mmap: Mmap,
}
impl Evidence {
pub fn open(path: &Path) -> OLMResult<Self> {
pub fn open(path: &Path) -> OKIResult<Self> {
let f = File::open(path)?;
let mmap = unsafe { Mmap::map(&f)? };
Ok(Self { mmap })
@@ -26,16 +26,16 @@ impl Evidence {
(raw >> 7, (raw & 0x7F) as u8)
}
#[inline]
pub fn encode(chunk_id: u32, rank: u8) -> u32 {
(chunk_id << 7) | (rank as u32 & 0x7F)
}
pub fn len(&self) -> usize {
self.mmap.len() / 4
}
}
#[inline]
pub fn encode(chunk_id: u32, rank: u8) -> u32 {
(chunk_id << 7) | (rank as u32 & 0x7F)
}
pub struct EvidenceWriter {
buf: Vec<u32>,
}
@@ -47,13 +47,13 @@ impl EvidenceWriter {
#[inline]
pub fn set(&mut self, slot: usize, chunk_id: u32, rank: u8) {
self.buf[slot] = encode(chunk_id, rank);
self.buf[slot] = Evidence::encode(chunk_id, rank);
}
pub fn write(self, path: &Path) -> OLMResult<()> {
pub fn write(self, path: &Path) -> OKIResult<()> {
let mut f = BufWriter::new(File::create(path)?);
for v in self.buf {
f.write_all(&v.to_le_bytes()).map_err(OLMError::Io)?;
f.write_all(&v.to_le_bytes()).map_err(OKIError::Io)?;
}
Ok(())
}
src/obikindex/src/layer/fingerprint.rs
+10 -10
View File
@@ -14,7 +14,7 @@ use std::path::Path;
use bitvec::prelude::*;
use memmap2::Mmap;
use crate::layer::error::{OLMError, OLMResult};
use crate::index::error::{OKIError, OKIResult};
const MAGIC: &[u8; 4] = b"FPVF";
const HEADER: usize = 16;
@@ -30,15 +30,15 @@ pub struct FingerprintVec {
}
impl FingerprintVec {
pub fn open(path: &Path) -> OLMResult<Self> {
pub fn open(path: &Path) -> OKIResult<Self> {
let f = File::open(path)?;
let mmap = unsafe { Mmap::map(&f)? };
if mmap.len() < HEADER || &mmap[..4] != MAGIC {
return Err(OLMError::InvalidLayer("bad fingerprint magic".into()));
return Err(OKIError::InvalidData { context: "layer", detail: "bad fingerprint magic".into() });
}
let b = mmap[4];
if b == 0 || b > 64 {
return Err(OLMError::InvalidLayer("invalid fingerprint width".into()));
return Err(OKIError::InvalidData { context: "layer", detail: "invalid fingerprint width".into() });
}
let n = u64::from_le_bytes(mmap[8..16].try_into().unwrap()) as usize;
let mask: u64 = if b == 64 { u64::MAX } else { (1u64 << b) - 1 };
@@ -89,12 +89,12 @@ impl FingerprintVecWriter {
bits[lo .. lo + self.b as usize].store_le(fingerprint);
}
pub fn write(self, path: &Path) -> OLMResult<()> {
let mut f = BufWriter::new(File::create(path).map_err(OLMError::Io)?);
f.write_all(MAGIC).map_err(OLMError::Io)?;
f.write_all(&[self.b, 0, 0, 0]).map_err(OLMError::Io)?;
f.write_all(&(self.n as u64).to_le_bytes()).map_err(OLMError::Io)?;
f.write_all(&self.buf).map_err(OLMError::Io)?;
pub fn write(self, path: &Path) -> OKIResult<()> {
let mut f = BufWriter::new(File::create(path).map_err(OKIError::Io)?);
f.write_all(MAGIC).map_err(OKIError::Io)?;
f.write_all(&[self.b, 0, 0, 0]).map_err(OKIError::Io)?;
f.write_all(&(self.n as u64).to_le_bytes()).map_err(OKIError::Io)?;
f.write_all(&self.buf).map_err(OKIError::Io)?;
Ok(())
}
}
src/obikindex/src/layer/map.rs
-128
View File
@@ -1,128 +0,0 @@
use std::collections::HashMap;
use std::fs;
use std::path::{Path, PathBuf};
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix, StorageKind};
use obikseq::CanonicalKmer;
use obiskio::{UnitigFileWriter, DEFAULT_BLOCK_BITS};
use crate::layer::error::{OLMError, OLMResult};
use crate::layer::typed_layer::{layer_dir, Hit, TypedLayer, LayerContent, LayerData};
use crate::layer::meta::{IndexMode, PartitionMeta};
use crate::layer::mphf_layer::EvidenceKind;
/// Layered kmer index for a single partition.
///
/// Each layer covers a disjoint kmer set. Queries probe layers in order;
/// the first match wins. Adding a dataset appends a new layer without
/// rebuilding existing ones.
pub struct LayeredMap<D: LayerData = ()> {
root: PathBuf,
meta: PartitionMeta,
layers: Vec<TypedLayer<D>>,
}
// ── Common methods ────────────────────────────────────────────────────────────
impl<D: LayerData> LayeredMap<D> {
/// Open an existing layered index at `root`.
/// The mode is read once from `PartitionMeta` and applied to all layers.
pub fn open(root: &Path) -> OLMResult<Self> {
let meta = PartitionMeta::load(root)?;
let layers = (0..meta.n_layers)
.map(|i| TypedLayer::<D>::open(&layer_dir(root, i), &meta.mode))
.collect::<OLMResult<Vec<_>>>()?;
Ok(Self { root: root.to_owned(), meta, layers })
}
/// Create a new, empty layered index at `root` with the given mode.
pub fn create(root: &Path, mode: IndexMode) -> OLMResult<Self> {
fs::create_dir_all(root)?;
let meta = PartitionMeta::new(mode);
meta.save(root)?;
Ok(Self { root: root.to_owned(), meta, layers: Vec::new() })
}
pub fn n_layers(&self) -> usize { self.layers.len() }
pub fn layer(&self, i: usize) -> &TypedLayer<D> { &self.layers[i] }
pub fn mode(&self) -> &IndexMode { &self.meta.mode }
/// TypedLayer `i`'s content (`Count`/`Presence`) — a lightweight disk probe,
/// no MPHF/matrix opened. For picking a `D` before committing to
/// [`TypedLayer::open`], or for reporting/diagnostics over an already-open
/// `LayeredMap` without re-opening every layer under a different `D`.
pub fn detect_layer_content(&self, i: usize) -> LayerContent {
LayerContent::detect(&layer_dir(&self.root, i))
}
/// TypedLayer `i`'s storage format — a lightweight disk probe, no
/// MPHF/matrix opened. See [`detect_layer_content`](Self::detect_layer_content).
pub fn detect_layer_storage(&self, i: usize) -> OLMResult<StorageKind> {
let dir = layer_dir(&self.root, i);
match LayerContent::detect(&dir) {
LayerContent::Count => PersistentCompactIntMatrix::detect_storage(&dir).map_err(OLMError::Io),
LayerContent::Presence => PersistentBitMatrix::detect_storage(&dir).map_err(OLMError::Io),
}
}
/// TypedLayer `i`'s evidence mode — a lightweight disk probe, no
/// MPHF/matrix opened. See [`detect_layer_content`](Self::detect_layer_content).
pub fn detect_layer_evidence(&self, i: usize) -> OLMResult<EvidenceKind> {
EvidenceKind::detect(&layer_dir(&self.root, i))
}
/// Query `kmer` across all layers. Returns `(layer_index, Hit)` on match.
pub fn query(&self, kmer: CanonicalKmer) -> Option<(usize, Hit<D::Item>)> {
self.layers
.iter()
.enumerate()
.find_map(|(i, layer)| layer.query(kmer).map(|hit| (i, hit)))
}
pub fn next_layer_writer(&self) -> OLMResult<UnitigFileWriter> {
let dir = layer_dir(&self.root, self.layers.len());
TypedLayer::<D>::unitig_writer(&dir)
}
fn append_layer(&mut self) -> OLMResult<()> {
let i = self.layers.len();
let dir = layer_dir(&self.root, i);
self.layers.push(TypedLayer::<D>::open(&dir, &self.meta.mode)?);
self.meta.n_layers = self.layers.len();
self.meta.save(&self.root)?;
Ok(())
}
}
// ── Mode 1 — set membership ───────────────────────────────────────────────────
impl LayeredMap<()> {
pub fn push_layer(&mut self) -> OLMResult<usize> {
let i = self.layers.len();
let dir = layer_dir(&self.root, i);
TypedLayer::<()>::build(&dir, DEFAULT_BLOCK_BITS, &self.meta.mode)?;
self.append_layer()?;
Ok(i)
}
}
// ── Mode 2 — count matrix ─────────────────────────────────────────────────────
impl LayeredMap<PersistentCompactIntMatrix> {
pub fn push_layer(&mut self, count_of: impl Fn(CanonicalKmer) -> u32) -> OLMResult<usize> {
let i = self.layers.len();
let dir = layer_dir(&self.root, i);
TypedLayer::<PersistentCompactIntMatrix>::build(&dir, DEFAULT_BLOCK_BITS, &self.meta.mode, count_of)?;
self.append_layer()?;
Ok(i)
}
pub fn push_layer_from_map(&mut self, counts: &HashMap<CanonicalKmer, u32>) -> OLMResult<usize> {
self.push_layer(|kmer| counts.get(&kmer).copied().unwrap_or(0))
}
}
#[cfg(test)]
#[path = "tests/map.rs"]
mod tests;
src/obikindex/src/layer/meta.rs
+30 -35
View File
@@ -1,18 +1,20 @@
use std::fs::File;
use std::path::Path;
use serde::{Deserialize, Serialize};
use crate::layer::error::OLMResult;
const META_FILE: &str = "meta.json";
use crate::index::error::OKIResult;
use crate::layer::fingerprint::FingerprintVec;
use crate::layer::mphf_layer::{EvidenceKind, FINGERPRINT_FILE};
use crate::layer::utils::layer_dir;
// ── IndexMode ─────────────────────────────────────────────────────────────────
/// Evidence mode for an entire partitioned index — homogeneous across all layers.
///
/// Determined once at build time; stored in `PartitionMeta` (`meta.json`).
/// All layers within an index share the same mode.
/// Determined once at build time. All layers within an index share the same
/// mode, so it is never persisted separately — [`IndexMode::detect`] reads it
/// straight back from whichever evidence file(s) layer 0 actually has on
/// disk, the same probe [`EvidenceKind::detect`] uses.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum IndexMode {
@@ -20,7 +22,9 @@ pub enum IndexMode {
Exact,
/// Approximate evidence: `fingerprint.bin` only.
/// `b` — fingerprint bits per slot; false-positive rate ≈ 1/2^b per query.
/// `z` — Findere consecutive-kmer parameter (build-time only; not used at query time).
/// `z` — Findere consecutive-kmer parameter (build-time only; not used at
/// query time, not stored on disk, always `0` once recovered via
/// [`detect`](Self::detect)).
Approx { b: u8, z: u8 },
/// Hybrid: both `fingerprint.bin` and `evidence.bin` + `unitigs.bin.idx`.
/// `find()` uses the fingerprint (O(1), approx); `find_strict()` uses exact evidence.
@@ -31,33 +35,24 @@ impl Default for IndexMode {
fn default() -> Self { Self::Exact }
}
// ── PartitionMeta ─────────────────────────────────────────────────────────────
/// Index-level metadata stored in `meta.json` at the root of a partition index.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PartitionMeta {
pub n_layers: usize,
#[serde(default)]
pub mode: IndexMode,
}
impl PartitionMeta {
pub fn new(mode: IndexMode) -> Self {
Self { n_layers: 0, mode }
}
pub fn load(dir: &Path) -> OLMResult<Self> {
let f = File::open(dir.join(META_FILE))?;
Ok(serde_json::from_reader(f)?)
}
pub fn save(&self, dir: &Path) -> OLMResult<()> {
let f = File::create(dir.join(META_FILE))?;
serde_json::to_writer_pretty(f, self)?;
Ok(())
impl IndexMode {
/// Detect the evidence mode actually on disk, from partition index
/// directory `root`'s layer 0 — the single source of truth every layer
/// in the same index shares. `z` cannot be recovered (not stored in
/// `fingerprint.bin`, and irrelevant past build time), so it comes back
/// as `0`.
pub fn detect(root: &Path) -> OKIResult<Self> {
let layer0 = layer_dir(root, 0);
Ok(match EvidenceKind::detect(&layer0)? {
EvidenceKind::Exact => IndexMode::Exact,
EvidenceKind::Approx => IndexMode::Approx {
b: FingerprintVec::open(&layer0.join(FINGERPRINT_FILE))?.b(),
z: 0,
},
EvidenceKind::Hybrid => IndexMode::Hybrid {
b: FingerprintVec::open(&layer0.join(FINGERPRINT_FILE))?.b(),
z: 0,
},
})
}
}
impl Default for PartitionMeta {
fn default() -> Self { Self::new(IndexMode::Exact) }
}
src/obikindex/src/layer/mod.rs
+4 -11
View File
@@ -1,20 +1,13 @@
pub mod content_layer;
pub mod error;
pub mod evidence;
pub mod fingerprint;
pub mod layered_store;
pub mod map;
pub mod meta;
pub(crate) mod mphf_layer;
pub mod typed_layer;
pub(crate) mod utils;
pub use crate::index::error::{OKIError, OKIResult};
pub use content_layer::KmerLayer;
pub use error::{OLMError, OLMResult};
pub use layered_store::LayeredStore;
pub use map::LayeredMap;
pub use meta::{IndexMode, PartitionMeta};
pub use meta::IndexMode;
pub use mphf_layer::{EvidenceKind, KmerBatchIter, KmerIter, MphfLayer, MphfOnly};
pub use typed_layer::{
HasLayerContent, HasStorageKind, Hit, LayerContent, LayerData, TypedLayer,
dereplicated_superkmers_path, layer_dir, open_data, raw_superkmers_path,
};
pub use typed_layer::{HasLayerContent, HasStorageKind, Hit, LayerContent, LayerData, TypedLayer};
src/obikindex/src/layer/mphf_layer.rs
+38 -42
View File
@@ -10,7 +10,7 @@ use obikseq::CanonicalKmer;
use obiskio::{CanonicalKmerIter, UnitigFileReader, UnitigFileWriter, build_unitig_idx};
use ptr_hash::{PtrHash, PtrHashParams, bucket_fn::CubicEps, hash::Xx64};
use crate::layer::error::{OLMError, OLMResult};
use crate::index::error::{OKIError, OKIResult};
use crate::layer::evidence::{Evidence, EvidenceWriter};
use crate::layer::fingerprint::{FingerprintVec, FingerprintVecWriter};
use crate::layer::meta::IndexMode;
@@ -76,7 +76,7 @@ impl LayerEvidence {
/// Identify which evidence files exist at `layer_dir`, without opening
/// any of them — a lightweight disk probe (same signal `EvidenceKind::
/// detect` uses), not a load.
fn at(layer_dir: &Path) -> OLMResult<Self> {
fn at(layer_dir: &Path) -> OKIResult<Self> {
Ok(match EvidenceKind::detect(layer_dir)? {
EvidenceKind::Exact => LayerEvidence::Exact {
evidence_path: layer_dir.join(EVIDENCE_FILE),
@@ -94,7 +94,7 @@ impl LayerEvidence {
}
/// mmap the evidence file(s) this identity points to.
fn open(self) -> OLMResult<OpenLayerEvidence> {
fn open(self) -> OKIResult<OpenLayerEvidence> {
Ok(match self {
LayerEvidence::Exact { evidence_path, unitig_path } => OpenLayerEvidence::Exact {
evidence: Evidence::open(&evidence_path)?,
@@ -127,17 +127,17 @@ impl EvidenceKind {
/// Lightweight disk probe, no MPHF/evidence opened: presence of
/// `evidence.bin`/`fingerprint.bin` alone determines the mode — same
/// signal `MphfLayer::open` uses, just without opening either file.
pub fn detect(layer_dir: &Path) -> OLMResult<EvidenceKind> {
pub fn detect(layer_dir: &Path) -> OKIResult<EvidenceKind> {
let has_evidence = layer_dir.join(EVIDENCE_FILE).exists();
let has_fingerprint = layer_dir.join(FINGERPRINT_FILE).exists();
match (has_evidence, has_fingerprint) {
(true, false) => Ok(EvidenceKind::Exact),
(false, true) => Ok(EvidenceKind::Approx),
(true, true) => Ok(EvidenceKind::Hybrid),
(false, false) => Err(OLMError::InvalidLayer(format!(
(false, false) => Err(OKIError::InvalidData { context: "layer", detail: format!(
"no evidence.bin or fingerprint.bin in {}",
layer_dir.display()
))),
) }),
}
}
}
@@ -151,7 +151,6 @@ impl EvidenceKind {
/// - [`find_strict`](Self::find_strict) — always exact; O(1) on Exact/Hybrid layers,
/// O(n) sequential scan on Approx layers.
pub struct MphfLayer {
layer_dir: PathBuf,
/// Mode-independent copy — used by `iter_kmers` (all modes) and
/// `find_strict`'s `Approx` fallback. `Exact`/`Hybrid` additionally
/// carry their own inside `OpenLayerEvidence` (see its doc comment for
@@ -165,18 +164,15 @@ pub struct MphfLayer {
}
impl MphfLayer {
/// This layer's own directory.
pub(crate) fn dir(&self) -> &Path {
&self.layer_dir
}
/// Open a layer — evidence mode is auto-detected from which files exist
/// on disk (see [`LayerEvidence::at`]), never passed in: the caller
/// can't tell this type anything about its own mode that isn't already
/// derivable from `layer_dir` itself.
pub fn open(dir: &Path) -> OLMResult<Self> {
/// derivable from `dir` itself. `dir` is only used transiently here —
/// not stored: `KmerLayer`, one level up, is what owns a layer's
/// directory as a persistent field, not this type.
pub fn open(dir: &Path) -> OKIResult<Self> {
let mphf: MemCase<MphfEps> = Mphf::mmap(&dir.join(MPHF_FILE), Flags::empty())
.map_err(|e| OLMError::InvalidLayer(e.to_string()))?;
.map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?;
let unitigs = Arc::new(UnitigFileReader::open(&dir.join(UNITIGS_FILE))?);
let ev = LayerEvidence::at(dir)?.open()?;
let n = match &ev {
@@ -184,7 +180,7 @@ impl MphfLayer {
OpenLayerEvidence::Hybrid { evidence, .. } => evidence.len(),
OpenLayerEvidence::Approx { fingerprint } => fingerprint.n(),
};
Ok(Self { layer_dir: dir.to_owned(), unitigs, mphf, ev, n })
Ok(Self { unitigs, mphf, ev, n })
}
// ── Query API ─────────────────────────────────────────────────────────────
@@ -405,9 +401,9 @@ impl Iterator for KmerBatchIter {
pub struct MphfOnly(MemCase<MphfEps>);
impl MphfOnly {
pub fn open(dir: &Path) -> OLMResult<Self> {
pub fn open(dir: &Path) -> OKIResult<Self> {
let mphf: MemCase<MphfEps> = Mphf::mmap(&dir.join(MPHF_FILE), Flags::empty())
.map_err(|e| OLMError::InvalidLayer(e.to_string()))?;
.map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?;
Ok(Self(mphf))
}
@@ -421,13 +417,13 @@ impl MphfOnly {
impl MphfLayer {
// ── Build helpers ─────────────────────────────────────────────────────────
pub fn unitig_writer(dir: &Path) -> OLMResult<UnitigFileWriter> {
pub fn unitig_writer(dir: &Path) -> OKIResult<UnitigFileWriter> {
fs::create_dir_all(dir)?;
Ok(UnitigFileWriter::create(&dir.join(UNITIGS_FILE))?)
}
/// Build `evidence.bin` + `unitigs.bin.idx` from `unitigs.bin` + `mphf.bin`.
pub fn build_exact_evidence(dir: &Path, block_bits: u8) -> OLMResult<usize> {
pub fn build_exact_evidence(dir: &Path, block_bits: u8) -> OKIResult<usize> {
let unitig_path = dir.join(UNITIGS_FILE);
let unitigs = UnitigFileReader::open_sequential(&unitig_path)?;
let n = unitigs.n_kmers();
@@ -439,7 +435,7 @@ impl MphfLayer {
}
let mphf: Mphf = Mphf::load_full(&dir.join(MPHF_FILE))
.map_err(|e| OLMError::InvalidLayer(e.to_string()))?;
.map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?;
let mut ev = EvidenceWriter::new(n);
let mut seen = vec![0u8; (n + 7) / 8];
@@ -447,12 +443,12 @@ impl MphfLayer {
for (kmer, chunk_id, rank) in unitigs.iter_indexed_canonical_kmers() {
let slot = mphf.index(&kmer.raw());
if slot >= n {
return Err(OLMError::Mphf("slot out of bounds".into()));
return Err(OKIError::InvalidData { context: "mphf", detail: "slot out of bounds".into() });
}
let byte = slot / 8;
let bit = 1u8 << (slot % 8);
if seen[byte] & bit != 0 {
return Err(OLMError::Mphf("duplicate slot".into()));
return Err(OKIError::InvalidData { context: "mphf", detail: "duplicate slot".into() });
}
seen[byte] |= bit;
ev.set(slot, chunk_id as u32, rank as u8);
@@ -464,14 +460,14 @@ impl MphfLayer {
}
/// Build `fingerprint.bin` from `unitigs.bin` + `mphf.bin`.
pub fn build_approx_evidence(dir: &Path, b: u8, z: u8) -> OLMResult<usize> {
pub fn build_approx_evidence(dir: &Path, b: u8, z: u8) -> OKIResult<usize> {
if b == 0 || b > 64 {
return Err(OLMError::InvalidLayer(
return Err(OKIError::InvalidData { context: "layer", detail:
"fingerprint width must be 1..=64".into(),
));
});
}
if z == 0 {
return Err(OLMError::InvalidLayer("z must be ≥ 1".into()));
return Err(OKIError::InvalidData { context: "layer", detail: "z must be ≥ 1".into() });
}
let unitig_path = dir.join(UNITIGS_FILE);
let unitigs = UnitigFileReader::open_sequential(&unitig_path)?;
@@ -483,14 +479,14 @@ impl MphfLayer {
}
let mphf: Mphf = Mphf::load_full(&dir.join(MPHF_FILE))
.map_err(|e| OLMError::InvalidLayer(e.to_string()))?;
.map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?;
let mut fw = FingerprintVecWriter::new(n, b);
for (kmer, _, _) in unitigs.iter_indexed_canonical_kmers() {
let slot = mphf.index(&kmer.raw());
if slot >= n {
return Err(OLMError::Mphf("slot out of bounds".into()));
return Err(OKIError::InvalidData { context: "mphf", detail: "slot out of bounds".into() });
}
fw.set(slot, kmer.seq_hash());
}
@@ -508,24 +504,24 @@ impl MphfLayer {
dir: &Path,
block_bits: u8,
mode: &IndexMode,
fill_slot: &mut impl FnMut(usize, CanonicalKmer) -> OLMResult<()>,
) -> OLMResult<usize> {
fill_slot: &mut impl FnMut(usize, CanonicalKmer) -> OKIResult<()>,
) -> OKIResult<usize> {
use rayon::prelude::*;
let unitig_path = dir.join(UNITIGS_FILE);
let n = UnitigFileReader::open_sequential(&unitig_path)?.n_kmers();
let sk_to_olm = |e: obiskio::SKError| match e {
obiskio::SKError::Io(io) => OLMError::Io(io),
e => OLMError::InvalidLayer(e.to_string()),
obiskio::SKError::Io(io) => OKIError::Io(io),
e => OKIError::InvalidData { context: "layer", detail: e.to_string() },
};
// ── Empty layer ───────────────────────────────────────────────────────
if n == 0 {
let mphf: Mphf = Mphf::try_new(&[] as &[u64], PtrHashParams::<CubicEps>::default())
.ok_or_else(|| OLMError::Mphf("construction failed".into()))?;
.ok_or_else(|| OKIError::InvalidData { context: "mphf", detail: "construction failed".into() })?;
mphf.store(&dir.join(MPHF_FILE))
.map_err(|e| OLMError::InvalidLayer(e.to_string()))?;
.map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?;
match mode {
IndexMode::Exact | IndexMode::Hybrid { .. } => {
fs::File::create(dir.join(EVIDENCE_FILE))?;
@@ -549,7 +545,7 @@ impl MphfLayer {
PtrHashParams::<CubicEps>::default(),
);
mphf.store(&dir.join(MPHF_FILE))
.map_err(|e| OLMError::InvalidLayer(e.to_string()))?;
.map_err(|e| OKIError::InvalidData { context: "layer", detail: e.to_string() })?;
// ── Pass 2: fill evidence files + callback ────────────────────────────
let unitigs2 = UnitigFileReader::open_sequential(&unitig_path)?;
@@ -561,12 +557,12 @@ impl MphfLayer {
for (kmer, chunk_id, rank) in unitigs2.iter_indexed_canonical_kmers() {
let slot = mphf.index(&kmer.raw());
if slot >= n {
return Err(OLMError::Mphf("slot out of bounds".into()));
return Err(OKIError::InvalidData { context: "mphf", detail: "slot out of bounds".into() });
}
let byte = slot / 8;
let bit = 1u8 << (slot % 8);
if seen[byte] & bit != 0 {
return Err(OLMError::Mphf("duplicate slot".into()));
return Err(OKIError::InvalidData { context: "mphf", detail: "duplicate slot".into() });
}
seen[byte] |= bit;
ev.set(slot, chunk_id as u32, rank as u8);
@@ -581,12 +577,12 @@ impl MphfLayer {
for (kmer, _, _) in unitigs2.iter_indexed_canonical_kmers() {
let slot = mphf.index(&kmer.raw());
if slot >= n {
return Err(OLMError::Mphf("slot out of bounds".into()));
return Err(OKIError::InvalidData { context: "mphf", detail: "slot out of bounds".into() });
}
let byte = slot / 8;
let bit = 1u8 << (slot % 8);
if seen[byte] & bit != 0 {
return Err(OLMError::Mphf("duplicate slot".into()));
return Err(OKIError::InvalidData { context: "mphf", detail: "duplicate slot".into() });
}
seen[byte] |= bit;
fw.set(slot, kmer.seq_hash());
@@ -601,12 +597,12 @@ impl MphfLayer {
for (kmer, chunk_id, rank) in unitigs2.iter_indexed_canonical_kmers() {
let slot = mphf.index(&kmer.raw());
if slot >= n {
return Err(OLMError::Mphf("slot out of bounds".into()));
return Err(OKIError::InvalidData { context: "mphf", detail: "slot out of bounds".into() });
}
let byte = slot / 8;
let bit = 1u8 << (slot % 8);
if seen[byte] & bit != 0 {
return Err(OLMError::Mphf("duplicate slot".into()));
return Err(OKIError::InvalidData { context: "mphf", detail: "duplicate slot".into() });
}
seen[byte] |= bit;
ev.set(slot, chunk_id as u32, rank as u8);
src/obikindex/src/layer/tests/map.rs
-219
View File
@@ -1,219 +0,0 @@
use super::*;
use obicompactvec::{pack_bit_matrix, pack_sparse_bit_matrix, PersistentBitMatrix, PersistentCompactIntMatrix, StorageKind};
use obikseq::{set_k, Sequence as _, Unitig};
use obiskio::DEFAULT_BLOCK_BITS;
use crate::layer::typed_layer::LayerContent;
use crate::layer::meta::IndexMode;
use crate::layer::mphf_layer::EvidenceKind;
use tempfile::tempdir;
fn push_unitigs_and_layer(
map: &mut LayeredMap<PersistentCompactIntMatrix>,
seqs: &[&[u8]],
count: u32,
) {
let mut w = map.next_layer_writer().unwrap();
for s in seqs {
w.write(&Unitig::from_ascii(s)).unwrap();
}
w.close().unwrap();
map.push_layer(|_| count).unwrap();
}
fn canonical(ascii: &[u8]) -> CanonicalKmer {
obikseq::Kmer::from_ascii(ascii).unwrap().canonical()
}
#[test]
fn create_empty_map() {
set_k(4);
let dir = tempdir().unwrap();
let map = LayeredMap::<()>::create(dir.path(), IndexMode::Exact).unwrap();
assert_eq!(map.n_layers(), 0);
}
#[test]
fn open_reloads_layer_count() {
set_k(4);
let dir = tempdir().unwrap();
{
let mut map = LayeredMap::<PersistentCompactIntMatrix>::create(dir.path(), IndexMode::Exact).unwrap();
push_unitigs_and_layer(&mut map, &[b"AAAACGT"], 1);
}
let map = LayeredMap::<PersistentCompactIntMatrix>::open(dir.path()).unwrap();
assert_eq!(map.n_layers(), 1);
}
#[test]
fn query_finds_kmer_in_layer_zero() {
set_k(4);
let dir = tempdir().unwrap();
let mut map = LayeredMap::<PersistentCompactIntMatrix>::create(dir.path(), IndexMode::Exact).unwrap();
push_unitigs_and_layer(&mut map, &[b"AAAACGT"], 3);
let kmer = canonical(b"AAAC");
let (layer_idx, hit) = map.query(kmer).expect("kmer must be found");
assert_eq!(layer_idx, 0);
assert_eq!(hit.data[0], 3);
}
#[test]
fn query_finds_kmer_in_correct_layer() {
set_k(4);
let dir = tempdir().unwrap();
let mut map = LayeredMap::<PersistentCompactIntMatrix>::create(dir.path(), IndexMode::Exact).unwrap();
push_unitigs_and_layer(&mut map, &[b"AAAACGT"], 1);
push_unitigs_and_layer(&mut map, &[b"GGGACGT"], 2);
assert_eq!(map.n_layers(), 2);
let (li, hit) = map.query(canonical(b"AAAA")).expect("AAAA must be found");
assert_eq!(li, 0);
assert_eq!(hit.data[0], 1);
let (li, hit) = map.query(canonical(b"GGGA")).expect("GGGA must be found");
assert_eq!(li, 1);
assert_eq!(hit.data[0], 2);
}
#[test]
fn query_absent_returns_none() {
set_k(4);
let dir = tempdir().unwrap();
let mut map = LayeredMap::<PersistentCompactIntMatrix>::create(dir.path(), IndexMode::Exact).unwrap();
push_unitigs_and_layer(&mut map, &[b"AAAACGT"], 1);
let absent = canonical(b"CCCC");
assert!(map.query(absent).is_none());
}
#[test]
fn push_layer_from_map_convenience() {
set_k(4);
let dir = tempdir().unwrap();
let mut map = LayeredMap::<PersistentCompactIntMatrix>::create(dir.path(), IndexMode::Exact).unwrap();
let mut w = map.next_layer_writer().unwrap();
w.write(&Unitig::from_ascii(b"AAAACGT")).unwrap();
w.close().unwrap();
let counts: HashMap<CanonicalKmer, u32> = vec![
(canonical(b"AAAA"), 10u32),
].into_iter().collect();
map.push_layer_from_map(&counts).unwrap();
let (_, hit) = map.query(canonical(b"AAAA")).unwrap();
assert_eq!(hit.data[0], 10);
}
// ── detect_layer_content / detect_layer_storage / detect_layer_evidence ────
//
// All built directly on disk (bypassing `LayeredMap<D>::push_layer`, which
// only exists for modes 1 and 2 — mode 3/presence has no `push_layer`),
// then reopened as `LayeredMap<()>` — `()`'s `LayerData::open` never reads
// the matrix, so it's the right handle for exercising the disk-probing
// `detect_layer_*` methods regardless of what content/storage the layer
// underneath actually is.
fn write_unitigs_at(dir: &Path, seqs: &[&[u8]]) {
fs::create_dir_all(dir).unwrap();
let mut w = UnitigFileWriter::create(&dir.join(crate::layer::typed_layer::UNITIGS_FILE)).unwrap();
for s in seqs {
w.write(&Unitig::from_ascii(s)).unwrap();
}
w.close().unwrap();
}
/// Build a one-layer partition root with a presence layer at `layer_0`,
/// save `meta.json`, and reopen it as `LayeredMap<()>`.
fn presence_partition(seqs: &[&[u8]], n_genomes: usize) -> (tempfile::TempDir, LayeredMap<()>) {
let dir = tempdir().unwrap();
let mode = IndexMode::Exact;
let layer0 = layer_dir(dir.path(), 0);
write_unitigs_at(&layer0, seqs);
TypedLayer::<PersistentBitMatrix>::build_presence(&layer0, DEFAULT_BLOCK_BITS, &mode, n_genomes, |kmer, g| {
(kmer.raw().wrapping_add(g as u64)) % 2 == 0
}).unwrap();
PartitionMeta { n_layers: 1, mode }.save(dir.path()).unwrap();
let map = LayeredMap::<()>::open(dir.path()).unwrap();
(dir, map)
}
#[test]
fn detect_layer_content_count() {
set_k(4);
let dir = tempdir().unwrap();
let mut map = LayeredMap::<PersistentCompactIntMatrix>::create(dir.path(), IndexMode::Exact).unwrap();
push_unitigs_and_layer(&mut map, &[b"AAAACGT"], 3);
assert_eq!(map.detect_layer_content(0), LayerContent::Count);
}
#[test]
fn detect_layer_content_presence() {
set_k(4);
let (_dir, map) = presence_partition(&[b"AAATCTA", b"CTTCGCC", b"TGATACG"], 3);
assert_eq!(map.detect_layer_content(0), LayerContent::Presence);
}
#[test]
fn detect_layer_storage_columnar_then_packed_then_sparse() {
set_k(4);
let (dir, map) = presence_partition(&[b"AAATCTA", b"CTTCGCC", b"TGATACG"], 3);
let presence_dir = layer_dir(dir.path(), 0).join("presence");
assert_eq!(map.detect_layer_storage(0).unwrap(), StorageKind::Columnar);
pack_bit_matrix(&presence_dir).unwrap();
assert_eq!(map.detect_layer_storage(0).unwrap(), StorageKind::Packed);
pack_sparse_bit_matrix(&presence_dir).unwrap();
assert_eq!(map.detect_layer_storage(0).unwrap(), StorageKind::Sparse);
}
#[test]
fn detect_layer_storage_implicit() {
set_k(4);
let dir = tempdir().unwrap();
let mode = IndexMode::Exact;
let layer0 = layer_dir(dir.path(), 0);
write_unitigs_at(&layer0, &[b"AAAACGT"]);
// Mode-1 build: MPHF + evidence + `layer_meta.json`, no matrix at all —
// must read back as `Presence`/`Implicit`, never a third "empty" state.
TypedLayer::<()>::build(&layer0, DEFAULT_BLOCK_BITS, &mode).unwrap();
PartitionMeta { n_layers: 1, mode }.save(dir.path()).unwrap();
let map = LayeredMap::<()>::open(dir.path()).unwrap();
assert_eq!(map.detect_layer_content(0), LayerContent::Presence);
assert_eq!(map.detect_layer_storage(0).unwrap(), StorageKind::Implicit);
}
#[test]
fn detect_layer_storage_count_columnar_then_packed() {
set_k(4);
let dir = tempdir().unwrap();
let mut map = LayeredMap::<PersistentCompactIntMatrix>::create(dir.path(), IndexMode::Exact).unwrap();
push_unitigs_and_layer(&mut map, &[b"AAAACGT"], 3);
drop(map);
let map = LayeredMap::<()>::open(dir.path()).unwrap();
assert_eq!(map.detect_layer_storage(0).unwrap(), StorageKind::Columnar);
obicompactvec::pack_compact_int_matrix(&layer_dir(dir.path(), 0).join("counts")).unwrap();
assert_eq!(map.detect_layer_storage(0).unwrap(), StorageKind::Packed);
}
#[test]
fn detect_layer_evidence_exact_vs_approx() {
set_k(4);
let dir = tempdir().unwrap();
let layer0 = layer_dir(dir.path(), 0);
write_unitigs_at(&layer0, &[b"AAAACGT"]);
TypedLayer::<()>::build(&layer0, DEFAULT_BLOCK_BITS, &IndexMode::Exact).unwrap();
PartitionMeta { n_layers: 1, mode: IndexMode::Exact }.save(dir.path()).unwrap();
let map = LayeredMap::<()>::open(dir.path()).unwrap();
assert_eq!(map.detect_layer_evidence(0).unwrap(), EvidenceKind::Exact);
let approx = IndexMode::Approx { b: 8, z: 1 };
let dir2 = tempdir().unwrap();
let layer0b = layer_dir(dir2.path(), 0);
write_unitigs_at(&layer0b, &[b"AAAACGT"]);
TypedLayer::<()>::build(&layer0b, DEFAULT_BLOCK_BITS, &approx).unwrap();
PartitionMeta { n_layers: 1, mode: approx }.save(dir2.path()).unwrap();
let map2 = LayeredMap::<()>::open(dir2.path()).unwrap();
assert_eq!(map2.detect_layer_evidence(0).unwrap(), EvidenceKind::Approx);
}
src/obikindex/src/layer/typed_layer.rs
+41 -84
View File
@@ -1,19 +1,18 @@
use std::collections::HashMap;
use std::fs;
use std::path::{Path, PathBuf};
use crate::layer::utils::layer_dir;
use obicompactvec::{
BinaryMatrix, PersistentBitMatrix, PersistentBitMatrixBuilder, PersistentCompactIntMatrix,
PersistentCompactIntMatrixBuilder, PersistentSparseBitMatrix,
};
use obikseq::CanonicalKmer;
use obiskio::{UnitigFileReader, UnitigFileWriter};
use std::collections::HashMap;
use std::fs;
use std::path::Path;
use crate::layer::error::{OLMError, OLMResult};
use crate::index::error::{OKIError, OKIResult};
use crate::layer::meta::IndexMode;
use crate::layer::mphf_layer::MphfLayer;
pub(crate) use crate::layer::mphf_layer::UNITIGS_FILE;
pub(crate) const COUNTS_DIR: &str = "counts";
pub(crate) const PRESENCE_DIR: &str = "presence";
@@ -21,48 +20,10 @@ pub(crate) const PRESENCE_DIR: &str = "presence";
pub trait LayerData: Sized {
type Item;
fn open(layer_dir: &Path) -> OLMResult<Self>;
fn open(layer_dir: &Path) -> OKIResult<Self>;
fn read(&self, slot: usize) -> Self::Item;
}
/// TypedLayer directory path for layer `i` within a partition's index root — the
/// single source of truth for the on-disk `layer_N` naming convention
/// (mirrors what `LayeredMap::open`/`push_layer` already use internally).
///
/// `obikindex::layer` operates within a single partition's index root; it has
/// no notion of "partition" at all. Turning a partition number into that
/// root is `obikindex::KmerIndex::index_dir`'s job, one layer up — callers
/// here only ever name a layer *number*, never build the path themselves.
pub fn layer_dir(root: &Path, i: usize) -> PathBuf {
root.join(format!("layer_{i}"))
}
/// Superkmer-file extension used by every pre-layer-construction artifact
/// below (`raw`/`dereplicated`) — an implementation detail of the SK file
/// format, never meant to leak as a literal string past this module.
const SK_EXT: &str = "skmer.zst";
/// Path of a layer's raw, not-yet-dereplicated superkmer file — written by
/// whichever algorithm routes superkmers into this layer (today:
/// `obikindexer::algorithms::partitionner::PartitionRouter`), read by whichever
/// algorithm dereplicates it (today:
/// `obikindexer::algorithms::dereplicator::Dereplicator`). Naming this once
/// here, rather than in either algorithm submodule, is what lets the two
/// agree on the filename without depending on each other directly — see
/// `DevDocMD/implementation/partition_layer_cache.md`.
pub fn raw_superkmers_path(layer_dir: &Path) -> PathBuf {
layer_dir.join(format!("raw.{SK_EXT}"))
}
/// Path of a layer's dereplicated superkmer file — written by
/// `obikindexer::algorithms::dereplicator::Dereplicator`, read by whichever
/// algorithm counts kmer abundances from it (today:
/// `obikindexer::algorithms::partitionner::PartitionRouter::count_kmer`) and,
/// later, by `obikindex::build_index_layer` to build the real layer.
pub fn dereplicated_superkmers_path(layer_dir: &Path) -> PathBuf {
layer_dir.join(format!("dereplicated.{SK_EXT}"))
}
/// Opens layer `i`'s data only, skipping the MPHF — for callers that only
/// need matrix-level operations (distance traits, column weights, group
/// filters, sub-matrix extraction) and never look up a kmer for this layer.
@@ -74,13 +35,13 @@ pub fn dereplicated_superkmers_path(layer_dir: &Path) -> PathBuf {
/// *number* within a partition it already knows the root of, the same
/// vocabulary as [`layer_dir`] and `LayeredMap` — never the `layer_N`
/// naming convention itself, which stays private to this crate.
pub fn open_data<D: LayerData>(root: &Path, i: usize) -> OLMResult<D> {
pub(crate) fn open_data<D: LayerData>(root: &Path, i: usize) -> OKIResult<D> {
D::open(&layer_dir(root, i))
}
impl LayerData for () {
type Item = ();
fn open(_layer_dir: &Path) -> OLMResult<Self> {
fn open(_layer_dir: &Path) -> OKIResult<Self> {
Ok(())
}
fn read(&self, _slot: usize) {}
@@ -88,8 +49,8 @@ impl LayerData for () {
impl LayerData for PersistentCompactIntMatrix {
type Item = Box<[u32]>;
fn open(layer_dir: &Path) -> OLMResult<Self> {
PersistentCompactIntMatrix::open(layer_dir).map_err(OLMError::Io)
fn open(layer_dir: &Path) -> OKIResult<Self> {
PersistentCompactIntMatrix::open(layer_dir).map_err(OKIError::Io)
}
fn read(&self, slot: usize) -> Box<[u32]> {
self.row(slot)
@@ -98,8 +59,8 @@ impl LayerData for PersistentCompactIntMatrix {
impl LayerData for PersistentBitMatrix {
type Item = Box<[bool]>;
fn open(layer_dir: &Path) -> OLMResult<Self> {
PersistentBitMatrix::open(layer_dir).map_err(OLMError::Io)
fn open(layer_dir: &Path) -> OKIResult<Self> {
PersistentBitMatrix::open(layer_dir).map_err(OKIError::Io)
}
fn read(&self, slot: usize) -> Box<[bool]> {
self.row(slot)
@@ -108,8 +69,8 @@ impl LayerData for PersistentBitMatrix {
impl LayerData for PersistentSparseBitMatrix {
type Item = Box<[bool]>;
fn open(layer_dir: &Path) -> OLMResult<Self> {
PersistentSparseBitMatrix::open(&layer_dir.join(PRESENCE_DIR)).map_err(OLMError::Io)
fn open(layer_dir: &Path) -> OKIResult<Self> {
PersistentSparseBitMatrix::open(&layer_dir.join(PRESENCE_DIR)).map_err(OKIError::Io)
}
fn read(&self, slot: usize) -> Box<[bool]> {
self.row(slot)
@@ -201,11 +162,7 @@ pub struct Hit<T = ()> {
// ── Common read path ──────────────────────────────────────────────────────────
impl<D: LayerData> TypedLayer<D> {
// `mode` no longer forwarded to `MphfLayer::open` (auto-detected from
// disk now, see mphf_layer.rs) — kept as a parameter for the moment so
// this signature (and `KmerLayer::open`'s call to it) doesn't have to
// change in the same step; dropped when content_layer.rs is done.
pub fn open(path: &Path, _mode: &IndexMode) -> OLMResult<Self> {
pub fn open(path: &Path) -> OKIResult<Self> {
let mphf = MphfLayer::open(path)?;
let data = D::open(path)?;
Ok(Self { mphf, data })
@@ -266,21 +223,21 @@ impl<D: LayerData> TypedLayer<D> {
self.mphf.enumerate_kmers_batch(n)
}
pub fn unitig_writer(out_dir: &Path) -> OLMResult<UnitigFileWriter> {
pub fn unitig_writer(out_dir: &Path) -> OKIResult<UnitigFileWriter> {
MphfLayer::unitig_writer(out_dir)
}
/// Build `unitigs.bin.idx` and `evidence.bin` from `unitigs.bin` and
/// `mphf.bin` already present in `layer_dir`.
/// `block_bits` controls the `.idx` block size (2^block_bits chunks/block).
pub fn build_exact_evidence(layer_dir: &Path, block_bits: u8) -> OLMResult<usize> {
pub fn build_exact_evidence(layer_dir: &Path, block_bits: u8) -> OKIResult<usize> {
MphfLayer::build_exact_evidence(layer_dir, block_bits)
}
/// Build `fingerprint.bin` from `unitigs.bin` and `mphf.bin` already
/// present in `layer_dir`. `b` — fingerprint bits (1..=64); `z` — Findere
/// consecutive k-mer parameter (≥1).
pub fn build_approx_evidence(layer_dir: &Path, b: u8, z: u8) -> OLMResult<usize> {
pub fn build_approx_evidence(layer_dir: &Path, b: u8, z: u8) -> OKIResult<usize> {
MphfLayer::build_approx_evidence(layer_dir, b, z)
}
@@ -311,21 +268,21 @@ impl<D: LayerData + HasStorageKind> TypedLayer<D> {
// ── Mode 1 — set membership ───────────────────────────────────────────────────
impl TypedLayer<()> {
pub fn build(out_dir: &Path, block_bits: u8, mode: &IndexMode) -> OLMResult<usize> {
pub fn build(out_dir: &Path, block_bits: u8, mode: &IndexMode) -> OKIResult<usize> {
MphfLayer::build(out_dir, block_bits, mode, &mut |_, _| Ok(()))
}
/// Create a presence matrix for a set-membership layer (first merge).
pub fn init_presence_matrix(layer_dir: &Path, n_kmers: usize) -> OLMResult<()> {
pub fn init_presence_matrix(layer_dir: &Path, n_kmers: usize) -> OKIResult<()> {
let presence_dir = layer_dir.join(PRESENCE_DIR);
fs::create_dir_all(&presence_dir).map_err(OLMError::Io)?;
fs::create_dir_all(&presence_dir).map_err(OKIError::Io)?;
let mut mb =
PersistentBitMatrixBuilder::new(n_kmers, &presence_dir).map_err(OLMError::Io)?;
PersistentBitMatrixBuilder::new(n_kmers, &presence_dir).map_err(OKIError::Io)?;
mb.add_col_ones()
.map_err(OLMError::Io)?
.map_err(OKIError::Io)?
.close()
.map_err(OLMError::Io)?;
mb.close().map_err(OLMError::Io)
.map_err(OKIError::Io)?;
mb.close().map_err(OKIError::Io)
}
}
@@ -337,18 +294,18 @@ impl TypedLayer<PersistentCompactIntMatrix> {
block_bits: u8,
mode: &IndexMode,
count_of: impl Fn(CanonicalKmer) -> u32,
) -> OLMResult<usize> {
) -> OKIResult<usize> {
let n = UnitigFileReader::open_sequential(&out_dir.join(UNITIGS_FILE))?.n_kmers();
let counts_dir = out_dir.join(COUNTS_DIR);
let mut mb =
PersistentCompactIntMatrixBuilder::new(n, &counts_dir).map_err(OLMError::Io)?;
let mut col = mb.add_col().map_err(OLMError::Io)?;
PersistentCompactIntMatrixBuilder::new(n, &counts_dir).map_err(OKIError::Io)?;
let mut col = mb.add_col().map_err(OKIError::Io)?;
let n_built = MphfLayer::build(out_dir, block_bits, mode, &mut |slot, kmer| {
col.set(slot, count_of(kmer));
Ok(())
})?;
col.close().map_err(OLMError::Io)?;
mb.close().map_err(OLMError::Io)?;
col.close().map_err(OKIError::Io)?;
mb.close().map_err(OKIError::Io)?;
Ok(n_built)
}
@@ -357,7 +314,7 @@ impl TypedLayer<PersistentCompactIntMatrix> {
block_bits: u8,
mode: &IndexMode,
counts: &HashMap<CanonicalKmer, u32>,
) -> OLMResult<usize> {
) -> OKIResult<usize> {
Self::build(out_dir, block_bits, mode, |kmer| {
counts.get(&kmer).copied().unwrap_or(0)
})
@@ -370,9 +327,9 @@ impl TypedLayer<PersistentCompactIntMatrix> {
pub fn append_genome_column(
layer_dir: &Path,
value_of: impl Fn(usize) -> u32,
) -> OLMResult<()> {
) -> OKIResult<()> {
PersistentCompactIntMatrix::append_column(&layer_dir.join(COUNTS_DIR), value_of)
.map_err(OLMError::Io)
.map_err(OKIError::Io)
}
/// Number of genome columns in this layer's count matrix.
@@ -448,9 +405,9 @@ impl TypedLayer<PersistentBitMatrix> {
pub fn append_genome_column(
layer_dir: &Path,
value_of: impl Fn(usize) -> bool,
) -> OLMResult<()> {
) -> OKIResult<()> {
PersistentBitMatrix::append_column(&layer_dir.join(PRESENCE_DIR), value_of)
.map_err(OLMError::Io)
.map_err(OKIError::Io)
}
pub fn build_presence(
@@ -459,13 +416,13 @@ impl TypedLayer<PersistentBitMatrix> {
mode: &IndexMode,
n_genomes: usize,
present_in: impl Fn(CanonicalKmer, usize) -> bool,
) -> OLMResult<usize> {
) -> OKIResult<usize> {
let n = UnitigFileReader::open_sequential(&out_dir.join(UNITIGS_FILE))?.n_kmers();
let presence_dir = out_dir.join(PRESENCE_DIR);
let mut mb = PersistentBitMatrixBuilder::new(n, &presence_dir).map_err(OLMError::Io)?;
let mut mb = PersistentBitMatrixBuilder::new(n, &presence_dir).map_err(OKIError::Io)?;
let mut cols: Vec<_> = (0..n_genomes)
.map(|_| mb.add_col().map_err(OLMError::Io))
.collect::<OLMResult<_>>()?;
.map(|_| mb.add_col().map_err(OKIError::Io))
.collect::<OKIResult<_>>()?;
let n_built = MphfLayer::build(out_dir, block_bits, mode, &mut |slot, kmer| {
for (g, col) in cols.iter_mut().enumerate() {
col.set(slot, present_in(kmer, g));
@@ -473,9 +430,9 @@ impl TypedLayer<PersistentBitMatrix> {
Ok(())
})?;
for col in cols {
col.close().map_err(OLMError::Io)?;
col.close().map_err(OKIError::Io)?;
}
mb.close().map_err(OLMError::Io)?;
mb.close().map_err(OKIError::Io)?;
Ok(n_built)
}
}
src/obikindex/src/layer/utils.rs
+14
View File
@@ -0,0 +1,14 @@
pub(crate) const LAYERNAME_SUFFIX: &str = "layer_";
use std::path::{Path, PathBuf};
/// TypedLayer directory path for layer `i` within a partition's index root — the
/// single source of truth for the on-disk `layer_N` naming convention
/// (mirrors what `LayeredMap::open`/`push_layer` already use internally).
///
/// `obikindex::layer` operates within a single partition's index root; it has
/// no notion of "partition" at all. Turning a partition number into that
/// root is `obikindex::KmerIndex::index_dir`'s job, one layer up — callers
/// here only ever name a layer *number*, never build the path themselves.
pub(crate) fn layer_dir(root: &Path, i: usize) -> PathBuf {
root.join(format!("{LAYERNAME_SUFFIX}{i}"))
}
src/obikindex/src/lib.rs
+6 -6
View File
@@ -13,10 +13,10 @@ pub mod layer;
pub mod partition;
pub use index::{
validate_label, AggOp, DistanceMetric, DistanceOutput, GenomeInfo, GroupFilterParams,
GroupQuorumFilter, IndexBitsPerKmer, IndexBuilder, IndexConfig, IndexMeta, IndexState,
KmerDesc, KmerFilter, KmerIndex, MergeMode, MetaPred, OKIError, OKIResult, OutputCol,
PartitionRunner, QueryHit, QueryStats, META_FILENAME,
materialize_layer, olm_to_sk, write_graph_as_unitigs, passes_all,
GenomeInfo,
IndexBuilder, IndexConfig, IndexMeta, IndexState,
KmerIndex, OKIError, OKIResult,
META_FILENAME,
ColBuilder,
};
pub use index::{filter, meta};
pub use index::meta;
src/obikindex/src/partition/mod.rs
+54 -33
View File
@@ -19,10 +19,13 @@
//! entirely) both still reinvent a fragment of this. Migrating them is a
//! separate, deferred step.
use crate::OKIError;
use std::path::{Path, PathBuf};
use crate::layer::{IndexMode, KmerLayer, OLMResult, layer_dir};
use obikseq::CanonicalKmer;
use crate::{
KmerIndex, OKIResult,
layer::{KmerLayer, utils::layer_dir},
};
/// Partition subdirectory name, under an index's root — the single source
/// of truth for the on-disk `partitions/part_NNNNN` naming convention.
@@ -34,59 +37,77 @@ pub const PARTITIONS_SUBDIR: &str = "partitions";
/// Path of partition `i`'s directory under `root` — `<root>/partitions/part_NNNNN`,
/// zero-padded to 5 digits.
pub fn partition_dir(root: &Path, i: usize) -> PathBuf {
pub(crate) fn partition_dir(root: &Path, i: usize) -> PathBuf {
root.join(PARTITIONS_SUBDIR).join(format!("part_{i:05}"))
}
/// Path of partition `i`'s layered-index directory — `<partition_dir>/index`,
/// the root every [`crate::layer::layer_dir`] call for this partition is
/// relative to.
pub fn index_dir(root: &Path, i: usize) -> PathBuf {
pub(crate) fn index_dir(root: &Path, i: usize) -> PathBuf {
partition_dir(root, i).join("index")
}
/// One partition's open layers, in layer order (layer 0 first).
/// One partition — identified by its own `index_dir`, nothing eagerly
/// opened. `index_dir` is the minimal field `KmerLayer::at` needs to
/// compute a layer's path; added here only because `layer()` below can't
/// be written without it now that there's no `layers: Vec<KmerLayer>` to
/// index into.
pub struct KmerPartition {
layers: Vec<KmerLayer>,
partition_dir: PathBuf,
id: usize,
}
impl KmerPartition {
/// Open every layer under `index_dir` (`index_dir/layer_0`,
/// `index_dir/layer_1`, ... up to `n_layers`), eagerly — not lazily on
/// first access, so the caller pays the mmap cost once, up front,
/// rather than at an unpredictable point during later lookups.
pub fn open(
index_dir: &Path,
mode: &IndexMode,
n_layers: usize,
with_counts: bool,
) -> OLMResult<Self> {
let layers = (0..n_layers)
.map(|l| KmerLayer::open(&layer_dir(index_dir, l), mode, with_counts))
.collect::<OLMResult<Vec<_>>>()?;
Ok(Self { layers })
pub fn new(index: &KmerIndex, i: usize) -> Self {
KmerPartition {
partition_dir: index.dir().join(format!("part_{i:05}")),
id: i,
}
}
pub fn id(&self) -> usize {
self.id
}
/// Number of layers under `index_dir`, found by probing `layer_0`,
/// `layer_1`, ... sequentially until one is missing — a disk scan, not
/// a metadata read (`meta.json`'s own `n_layers` is redundant with this
/// and no longer consulted here). Intrinsically slow: one `exists()`
/// syscall per layer, called fresh every time — not for a hot path.
pub fn n_layers(&self) -> usize {
self.layers.len()
let mut n = 0;
while layer_dir(&self.partition_dir, n).exists() {
n += 1;
}
n
}
pub fn layer(&self, i: usize) -> &KmerLayer {
&self.layers[i]
/// Evidence mode for this partition, detected from layer 0's evidence
/// files on disk (see [`crate::layer::IndexMode::detect`]) — never
/// persisted as metadata, since every layer within a partition shares
/// the same mode by construction.
pub fn mode(&self) -> OKIResult<crate::layer::IndexMode> {
crate::layer::IndexMode::detect(&self.partition_dir)
}
pub fn layers(&self) -> &[KmerLayer] {
&self.layers
pub fn layer(&self, i: usize) -> OKIResult<KmerLayer> {
let n = self.n_layers();
if i >= n {
return Err(OKIError::Layer(obiskio::SKError::InvalidData {
context: "New KmerLayer",
detail: format!("Layer {i} requested, Partition has only {n} layer"),
}));
}
Ok(KmerLayer::new(self, i))
}
/// Existence lookup of `kmer` across this partition's layers: tries
/// each in turn, stopping at the first hit and reporting which layer it
/// was — `find_slot`, not a data read, for a plain existence check.
pub fn find(&self, kmer: CanonicalKmer) -> Option<usize> {
self.layers
.iter()
.enumerate()
.find_map(|(li, layer)| layer.find_slot(kmer).map(|_| li))
pub fn layer_dir(&self, i: usize) -> OKIResult<PathBuf> {
Ok(self.layer(i)?.dir().to_path_buf())
}
pub fn dir(&self) -> &Path {
&self.partition_dir
}
}
src/obikindexer/src/algorithms/layer_builder/mod.rs
+1 -1
View File
@@ -18,7 +18,7 @@ use crate::extensions::PrivateBuilder;
/// The actual per-partition construction (De Bruijn graph, unitigs, MPHF,
/// matrix) stays a `KmerIndex` method (`build_index_layer`) — unlike
/// `Dereplicator`/`Counter`, it depends on several `obikindex`-internal
/// helpers (`graph_pipeline`, `common::olm_to_sk`) already shared with
/// helpers (`graph_pipeline`) already shared with
/// `merge`/`select`/`rebuild`'s own layer-construction paths, so it
/// belongs there, not duplicated or newly exported just for this
/// algorithm. `LayerBuilder`'s own job is the orchestration around it:
src/obikindexer/src/extensions/mod.rs
+4 -6
View File
@@ -36,7 +36,7 @@ use obidebruinj::GraphDeBruijn;
use obikindex::layer::IndexMode;
use obikindex::layer::{TypedLayer, meta::PartitionMeta};
use obikindex::{KmerIndex, OKIError, OKIResult};
use obikindex::{materialize_layer, olm_to_sk, write_graph_as_unitigs};
use crate::graph_pipeline::{materialize_layer, write_graph_as_unitigs};
use obiskio::{SKError, SKFileMeta, SKFileReader};
use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xx64};
@@ -109,7 +109,7 @@ impl PrivateBuilder for KmerIndex {
.map(|g| g.label.as_str())
.unwrap_or("unknown")
.to_owned();
let spectrums_dir = self.root_path().join("spectrums");
let spectrums_dir = self.dir().join("spectrums");
fs::create_dir_all(&spectrums_dir)?;
let path = spectrums_dir.join(format!("{label}.json"));
let spectrum_map: BTreeMap<String, u64> = counts
@@ -194,8 +194,7 @@ impl PrivateBuilder for KmerIndex {
(Some(mphf), Some(counts)) => counts.get(mphf.index(&kmer.raw())),
_ => 1,
},
)
.map_err(|e| olm_to_sk(e, "layer build"))?;
)?;
n
} else {
materialize_layer(g, &layer0_dir, block_bits, mode)?
@@ -206,8 +205,7 @@ impl PrivateBuilder for KmerIndex {
n_layers: 1,
mode: mode.clone(),
}
.save(index_dir)
.map_err(|e| olm_to_sk(e, "layer build"))?;
.save(index_dir)?;
Ok(n_kmers)
}
src/obikindex/src/index/graph_pipeline.rs → src/obikindexer/src/graph_pipeline.rs
+10 -12
View File
@@ -10,10 +10,9 @@ use obipipeline::{
use obidebruinj::GraphDeBruijn;
use obikseq::CanonicalKmer;
use crate::layer::{IndexMode, TypedLayer};
use obiskio::{SKError, SKResult};
use obikindex::layer::{IndexMode, TypedLayer};
use obikindex::{OKIError, OKIResult};
use crate::index::common::olm_to_sk;
// ── KmerGraphData ─────────────────────────────────────────────────────────────
@@ -29,16 +28,16 @@ enum KmerGraphData {
///
/// `flat_fn(path, emit)`: opens path, iterates kmers, calls `emit(batch)` for each batch.
/// `filter(kmer) -> bool`: secondary filter applied in the Transform stage.
pub(crate) fn build_graph<I, F, G>(
pub fn build_graph<I, F, G>(
file_source: I,
flat_fn: F,
filter: G,
n_workers: usize,
max_open: usize,
) -> SKResult<GraphDeBruijn>
) -> OKIResult<GraphDeBruijn>
where
I: Iterator<Item = PathBuf> + Send + 'static,
F: Fn(PathBuf, &mut dyn FnMut(Vec<CanonicalKmer>)) -> SKResult<()> + Send + Sync + 'static,
F: Fn(PathBuf, &mut dyn FnMut(Vec<CanonicalKmer>)) -> OKIResult<()> + Send + Sync + 'static,
G: Fn(CanonicalKmer) -> bool + Send + Sync + 'static,
{
let capacity = 2;
@@ -47,7 +46,7 @@ where
let filter = Arc::new(filter);
let g_shared = Arc::new(Mutex::new(GraphDeBruijn::new()));
let g_sink = Arc::clone(&g_shared);
let err_cap: Arc<Mutex<Option<SKError>>> = Arc::new(Mutex::new(None));
let err_cap: Arc<Mutex<Option<OKIError>>> = Arc::new(Mutex::new(None));
let err_flat = Arc::clone(&err_cap);
let throttled = throttle(file_source, max_open);
@@ -121,11 +120,11 @@ where
/// Phase 2 (write unitigs only): compute degrees, write unitigs to `layer_dir`, drop graph.
///
/// Returns n_kmers. Does NOT build the MPHF — caller does it.
pub fn write_graph_as_unitigs(g: GraphDeBruijn, layer_dir: &Path) -> SKResult<usize> {
pub fn write_graph_as_unitigs(g: GraphDeBruijn, layer_dir: &Path) -> OKIResult<usize> {
let n_kmers = g.len();
g.compute_degrees_and_mark_starts();
std::fs::create_dir_all(layer_dir)?;
let mut uw = TypedLayer::<()>::unitig_writer(layer_dir).map_err(|e| olm_to_sk(e, "graph pipeline"))?;
let mut uw = TypedLayer::<()>::unitig_writer(layer_dir)?;
g.try_for_each_unitig(|unitig| uw.write(unitig))?;
uw.close()?;
drop(g);
@@ -142,11 +141,10 @@ pub fn materialize_layer(
layer_dir: &Path,
block_bits: u8,
evidence: &IndexMode,
) -> SKResult<usize> {
) -> OKIResult<usize> {
let n = write_graph_as_unitigs(g, layer_dir)?;
debug!("materialize_layer: unitigs written ({n} kmers), building MPHF");
TypedLayer::<()>::build(layer_dir, block_bits, evidence)
.map_err(|e| olm_to_sk(e, "graph pipeline"))?;
TypedLayer::<()>::build(layer_dir, block_bits, evidence)?;
debug!("materialize_layer: MPHF build done");
Ok(n)
}
src/obikindexer/src/lib.rs
+3
View File
@@ -14,3 +14,6 @@
pub mod algorithms;
pub(crate) mod extensions;
pub mod graph_pipeline;
pub use graph_pipeline::{build_graph, materialize_layer, write_graph_as_unitigs};
src/obikmer/Cargo.toml
+3 -1
View File
@@ -19,6 +19,8 @@ obisys = { path = "../obisys" }
obiskio = { path = "../obiskio" }
obikindex = { path = "../obikindex", default-features = false }
obikindexer = { path = "../obikindexer" }
obikfilter = { path = "../obikfilter" }
obikstats = { path = "../obikstats" }
obikalgorithm = { path = "../obikalgorithm" }
obikphylo = { path = "../obikphylo" }
obitaxonomy = { path = "../obitaxonomy" }
@@ -37,5 +39,5 @@ pprof = { version = "0.15", features = ["prost-codec"], optional = true }
[features]
default = ["numa"]
numa = ["obikindex/numa"]
numa = ["obisys/numa"]
profiling = ["dep:pprof"]
src/obikmer/src/cmd/index/mod.rs
+35 -23
View File
@@ -2,13 +2,13 @@ use std::path::PathBuf;
use std::time::Instant;
use clap::Args;
use obikalgorithm::Algorithm;
use obikindex::layer::IndexMode;
use obikindex::{GenomeInfo, IndexBuilder, IndexConfig, IndexState, KmerIndex};
use obikindexer::algorithms::counter::Counter;
use obikindexer::algorithms::dereplicator::Dereplicator;
use obikindexer::algorithms::layer_builder::LayerBuilder;
use obikindexer::algorithms::partitionner::PartitionRouter;
use obikindex::{validate_label, GenomeInfo, IndexBuilder, IndexConfig, IndexState, KmerIndex};
use obikindex::layer::IndexMode;
use obikalgorithm::Algorithm;
fn current_state(idx: &KmerIndex) -> IndexState {
idx.state().unwrap_or_else(|e| {
@@ -18,10 +18,12 @@ fn current_state(idx: &KmerIndex) -> IndexState {
}
fn parse_key_value(s: &str) -> Result<(String, String), String> {
let pos = s.find('=').ok_or_else(|| format!("invalid key=value: no '=' in '{s}'"))?;
let pos = s
.find('=')
.ok_or_else(|| format!("invalid key=value: no '=' in '{s}'"))?;
Ok((s[..pos].to_string(), s[pos + 1..].to_string()))
}
use obisys::{progress_bar, spinner, Progress, Reporter, Stage};
use obisys::{Progress, Reporter, Stage, progress_bar, spinner};
use tracing::info;
use crate::cli::{CommonArgs, block_size_to_bits, partitions_to_bits};
@@ -108,9 +110,7 @@ pub(crate) fn resolve_approx_params(
const DEFAULT_B: u8 = 8;
const DEFAULT_Z: u8 = 1;
let bits_needed = |fp: f64| -> u8 {
(-fp.log2()).ceil() as u8
};
let bits_needed = |fp: f64| -> u8 { (-fp.log2()).ceil() as u8 };
match (z_opt, b_opt, fp_opt) {
// All three given: use b and z, recompute fp conservatively.
@@ -198,7 +198,7 @@ pub fn run(args: IndexArgs) {
};
// ── Open or create the index ─────────────────────────────────────────────
if KmerIndex::exists(&output) {
if KmerIndex::is_an_index(&output) {
if !args.force {
eprintln!(
"error: an index already exists at {} (use --force to overwrite it)",
@@ -211,23 +211,32 @@ pub fn run(args: IndexArgs) {
eprintln!("error removing existing index: {e}");
std::process::exit(1);
});
} else if output.exists() {
eprintln!(
"error: {} exists but is not an obikmer index, it cannot be deleted",
output.display()
);
std::process::exit(1);
}
let n_bits = partitions_to_bits(args.common.partitions);
let effective = 1usize << n_bits;
if effective != args.common.partitions {
info!("partitions: {} → {} (next power of 2)", args.common.partitions, effective);
info!(
"partitions: {} → {} (next power of 2)",
args.common.partitions, effective
);
}
let block_bits = block_size_to_bits(args.block_size);
let config = IndexConfig {
kmer_size: effective_kmer_size,
kmer_size: effective_kmer_size,
minimizer_size: args.common.minimizer_size,
n_bits,
with_counts: args.with_counts,
evidence: evidence.clone(),
with_counts: args.with_counts,
evidence: evidence.clone(),
block_bits,
};
let genome_info = args.label.as_ref().map(|label| {
validate_label(label).unwrap_or_else(|e| {
GenomeInfo::validate_label(label).unwrap_or_else(|e| {
eprintln!("error: --label: {e}");
std::process::exit(1);
});
@@ -242,7 +251,6 @@ pub fn run(args: IndexArgs) {
std::process::exit(1);
});
// ── Stage 1: scatter ─────────────────────────────────────────────────────
if current_state(&idx) < IndexState::Scattered {
let n_workers = args.common.threads.max(1);
@@ -272,19 +280,23 @@ pub fn run(args: IndexArgs) {
last_bases = p.position;
let bp = p.position as f64;
let (count_str, rate_str) = if bp >= 1e9 {
(format!("{:.2} Gbp", bp / 1e9), format!("{:.0} Mbp/s", ema_rate / 1e6))
(
format!("{:.2} Gbp", bp / 1e9),
format!("{:.0} Mbp/s", ema_rate / 1e6),
)
} else {
(format!("{:.0} Mbp", bp / 1e6), format!("{:.0} Mbp/s", ema_rate / 1e6))
(
format!("{:.0} Mbp", bp / 1e6),
format!("{:.0} Mbp/s", ema_rate / 1e6),
)
};
pb.set_message(format!("{count_str} {rate_str}"));
});
router
.run()
.unwrap_or_else(|e| {
eprintln!("error: {e}");
std::process::exit(1);
});
router.run().unwrap_or_else(|e| {
eprintln!("error: {e}");
std::process::exit(1);
});
pb.finish_and_clear();
rep.push(t.stop());
drop(router); // ends the borrow of `idx` early — `PartitionRouter`'s `Drop` impl would otherwise extend it to the end of scope (`run()` already called `close()`, which marks scatter done, internally)
src/obikmer/src/cmd/predicate.rs
+2 -2
View File
@@ -1,6 +1,6 @@
use clap::Args;
use obikindex::{GroupFilterParams, IndexMeta, MetaPred};
use obikindex::KmerFilter;
use obikindex::IndexMeta;
use obikfilter::{GroupFilterParams, KmerFilter, MetaPred};
/// CLI args for ingroup/outgroup filtering — embeddable in any command via `#[command(flatten)]`.
#[derive(Args)]
src/obikmer/src/cmd/utils/maintenance.rs
+3 -2
View File
@@ -1,6 +1,7 @@
use std::path::PathBuf;
use obikindex::{validate_label, IndexBitsPerKmer, KmerIndex};
use obikindex::{GenomeInfo, KmerIndex};
use obikstats::IndexBitsPerKmer;
use tracing::info;
pub(super) fn run_stats(index_path: &PathBuf) {
@@ -76,7 +77,7 @@ pub(super) fn run_rename(index_path: &PathBuf, spec: &str) {
std::process::exit(1);
});
validate_label(&new_label).unwrap_or_else(|e| {
GenomeInfo::validate_label(&new_label).unwrap_or_else(|e| {
eprintln!("error: --new-label: {e}");
std::process::exit(1);
});
src/obikmer2/Cargo.toml
+24
View File
@@ -0,0 +1,24 @@
[package]
name = "obikmer2"
version = "1.2.2"
edition = "2024"
[[bin]]
name = "obikmer2"
path = "src/main.rs"
[dependencies]
obikseq = { path = "../obikseq" }
obiread = { path = "../obiread" }
obipipeline = { path = "../obipipeline" }
obisys = { path = "../obisys" }
obikindex = { path = "../obikindex", default-features = false }
obikindexer = { path = "../obikindexer" }
obikalgorithm = { path = "../obikalgorithm" }
clap = { version = "4", features = ["derive"] }
tracing = "0.1.44"
tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] }
[features]
default = ["numa"]
numa = ["obisys/numa"]
src/obikmer2/src/cli.rs
+114
View File
@@ -0,0 +1,114 @@
use std::path::PathBuf;
use clap::Args;
use obiread::NucPage;
use obikseq::RoutableSuperKmer;
use obipipeline::Throttled;
// ── Shared arguments ──────────────────────────────────────────────────────────
#[derive(Args)]
pub struct CommonArgs {
/// Input files or directories (FASTA/FASTQ, optionally gzip-compressed).
/// If omitted, reads from stdin.
#[arg(num_args = 0..)]
pub inputs: Vec<String>,
/// k-mer size
#[arg(short, long, default_value_t = 31)]
pub kmer_size: usize,
/// Minimizer size
#[arg(short, long, default_value_t = 11)]
pub minimizer_size: usize,
/// Entropy threshold (k-mers with score ≤ theta are rejected)
#[arg(long, default_value_t = 0.7)]
pub theta: f64,
/// Maximum sub-word size for entropy computation
#[arg(long, default_value_t = 6)]
pub level_max: usize,
/// Number of partitions (rounded up to the next power of 2)
#[arg(short, long, default_value_t = 256)]
pub partitions: usize,
/// Number of worker threads
#[arg(
short = 'T',
long,
default_value_t = obisys::effective_parallelism()
)]
pub threads: usize,
/// Maximum number of input files open simultaneously.
/// Defaults to threads/4 (minimum 1). Keep below the number of workers
/// to ensure CPU workers are always available for the transform stage.
#[arg(long)]
pub max_open_files: Option<usize>,
}
/// Smallest `b` such that `2^b >= n` (i.e. `n.next_power_of_two().ilog2()`).
/// Minimum 1 (degenerate n=0 or n=1 → 1 partition).
pub fn partitions_to_bits(n: usize) -> usize {
n.max(1).next_power_of_two().trailing_zeros() as usize
}
/// Convert a block size (number of unitigs per block) to its `block_bits` exponent.
/// `block_size=1` → `block_bits=0` (one entry per unitig, O(1) random access).
pub fn block_size_to_bits(n: usize) -> u8 {
n.max(1).next_power_of_two().trailing_zeros() as u8
}
impl CommonArgs {
/// Validate k and m constraints. Exits on error.
pub fn validate(&self) {
let k = self.kmer_size;
let m = self.minimizer_size;
if k < 11 || k > 31 {
eprintln!("error: --kmer-size must be in [11, 31] (got {k})");
std::process::exit(1);
}
if k % 2 == 0 {
eprintln!("error: --kmer-size must be odd (got {k}); even k allows palindromic k-mers");
std::process::exit(1);
}
if m < 3 || m >= k {
eprintln!("error: --minimizer-size must be in [3, k−1] = [3, {}] (got {m})", k - 1);
std::process::exit(1);
}
if m % 2 == 0 {
eprintln!("error: --minimizer-size must be odd (got {m})");
std::process::exit(1);
}
}
pub fn effective_max_open(&self) -> usize {
self.max_open_files
.unwrap_or_else(|| (self.threads / 4).max(1))
.max(1)
}
pub fn seqfile_paths(&self) -> obiread::PathIter {
let paths: Vec<PathBuf> = if self.inputs.is_empty() {
vec![PathBuf::from("-")]
} else {
self.inputs.iter().map(PathBuf::from).collect()
};
obiread::PathIter::new(paths)
}
}
// ── Pipeline data carrier ─────────────────────────────────────────────────────
pub enum PipelineData {
Path(Throttled<PathBuf>),
NucPage(NucPage),
Batch(Vec<RoutableSuperKmer>),
}
unsafe impl Send for PipelineData {}
unsafe impl Sync for PipelineData {}
src/obikmer2/src/cmd/index/mod.rs
+363
View File
@@ -0,0 +1,363 @@
use std::path::PathBuf;
use std::time::Instant;
use clap::Args;
use obikalgorithm::Algorithm;
use obikindex::layer::IndexMode;
use obikindex::{GenomeInfo, IndexBuilder, IndexConfig, IndexState, KmerIndex};
use obikindexer::algorithms::counter::Counter;
use obikindexer::algorithms::dereplicator::Dereplicator;
use obikindexer::algorithms::layer_builder::LayerBuilder;
use obikindexer::algorithms::partitionner::PartitionRouter;
fn current_state(idx: &KmerIndex) -> IndexState {
idx.state().unwrap_or_else(|e| {
eprintln!("error reading index metadata: {e}");
std::process::exit(1);
})
}
fn parse_key_value(s: &str) -> Result<(String, String), String> {
let pos = s
.find('=')
.ok_or_else(|| format!("invalid key=value: no '=' in '{s}'"))?;
Ok((s[..pos].to_string(), s[pos + 1..].to_string()))
}
use obisys::{Progress, Reporter, Stage, progress_bar, spinner};
use tracing::info;
use crate::cli::{CommonArgs, block_size_to_bits, partitions_to_bits};
#[derive(Args)]
pub struct IndexArgs {
/// Output index directory
#[arg(short, long)]
pub output: PathBuf,
/// Overwrite output directory if it already exists
#[arg(long, default_value_t = false)]
pub force: bool,
/// Genome label (default: input filename without path/extension)
#[arg(long)]
pub label: Option<String>,
/// Genome categorical metadata as key=value pairs (repeatable)
#[arg(long = "meta", value_parser = parse_key_value)]
pub meta: Vec<(String, String)>,
/// Minimum kmer abundance (inclusive)
#[arg(long, default_value_t = 1)]
pub min_abundance: u32,
/// Maximum kmer abundance (inclusive)
#[arg(long)]
pub max_abundance: Option<u32>,
/// Store kmer counts in the index (default: set membership only)
#[arg(long, default_value_t = false)]
pub with_counts: bool,
/// Keep intermediate build files (dereplicated superkmers, mphf1, counts1)
#[arg(long, default_value_t = false)]
pub keep_intermediate: bool,
/// Use approximate (fingerprint-based) evidence instead of exact evidence.
/// False-positive rate per z-window: 1/2^(b·z).
#[arg(long, default_value_t = false)]
pub approx: bool,
/// Findere z parameter: number of consecutive k-mers that must all match.
/// Effective indexed k-mer size is kmer_size - z + 1.
#[arg(short = 'z', long, default_value = None)]
pub findere_z: Option<u8>,
/// Fingerprint bits per slot (b). FP per z-window = 1/2^(b·z).
#[arg(long, default_value = None)]
pub evidence_bits: Option<u8>,
/// Target false-positive rate per z-window (e.g. 0.01).
/// Used to derive missing b or z.
#[arg(long, default_value = None)]
pub fp: Option<f64>,
/// Block size for exact evidence `.idx` (number of unitigs per block).
/// Must be a power of two; rounded up if not. Default 1 = O(1) random access.
#[arg(long, default_value_t = 1)]
pub block_size: usize,
#[command(flatten)]
pub common: CommonArgs,
}
/// Resolve the (z, b, fp) triplet from the user-supplied subset.
///
/// Model: FP = 1/2^(b·z) ⟹ b·z = ⌈-log₂(fp)⌉
///
/// Rules when one value is missing (conservative = ceiling):
/// given z, b → fp = 1/2^(b·z)
/// given z, fp → b = ⌈-log₂(fp) / z⌉
/// given b, fp → z = ⌈-log₂(fp) / b⌉
/// given z only → b = 8 (default), fp derived
/// given b only → z = 1 (default), fp derived
/// given fp only → b = 8 (default), z derived
/// none given → z = 1, b = 8, fp = 1/256
pub(crate) fn resolve_approx_params(
z_opt: Option<u8>,
b_opt: Option<u8>,
fp_opt: Option<f64>,
) -> (u8, u8, f64) {
const DEFAULT_B: u8 = 8;
const DEFAULT_Z: u8 = 1;
let bits_needed = |fp: f64| -> u8 { (-fp.log2()).ceil() as u8 };
match (z_opt, b_opt, fp_opt) {
// All three given: use b and z, recompute fp conservatively.
(Some(z), Some(b), Some(_fp)) => {
let fp = 1.0_f64 / (1u64 << (b as u32 * z as u32)) as f64;
(z, b, fp)
}
// Two given, derive third.
(Some(z), Some(b), None) => {
let fp = 1.0_f64 / (1u64 << (b as u32 * z as u32)) as f64;
(z, b, fp)
}
(Some(z), None, Some(fp)) => {
let bz = (-fp.log2()).ceil() as u32;
let b = ((bz + z as u32 - 1) / z as u32).max(1) as u8;
let actual_fp = 1.0_f64 / (1u64 << (b as u32 * z as u32)) as f64;
(z, b, actual_fp)
}
(None, Some(b), Some(fp)) => {
let bz = (-fp.log2()).ceil() as u32;
let z = ((bz + b as u32 - 1) / b as u32).max(1) as u8;
let actual_fp = 1.0_f64 / (1u64 << (b as u32 * z as u32)) as f64;
(z, b, actual_fp)
}
// One given, apply defaults for the other.
(Some(z), None, None) => {
let b = DEFAULT_B;
let fp = 1.0_f64 / (1u64 << (b as u32 * z as u32)) as f64;
(z, b, fp)
}
(None, Some(b), None) => {
let z = DEFAULT_Z;
let fp = 1.0_f64 / (1u64 << (b as u32 * z as u32)) as f64;
(z, b, fp)
}
(None, None, Some(fp)) => {
let b = DEFAULT_B;
let z = ((bits_needed(fp) as u32 + b as u32 - 1) / b as u32).max(1) as u8;
let actual_fp = 1.0_f64 / (1u64 << (b as u32 * z as u32)) as f64;
(z, b, actual_fp)
}
// None given: defaults.
(None, None, None) => {
let b = DEFAULT_B;
let z = DEFAULT_Z;
let fp = 1.0_f64 / (1u64 << (b as u32 * z as u32)) as f64;
(z, b, fp)
}
}
}
pub fn run(args: IndexArgs) {
args.common.validate();
let output = args.output.clone();
let mut rep = Reporter::new();
// Locked for the whole build (including a possible --force removal +
// recreation below): a second `index` run resuming/overwriting the same
// output directory concurrently would otherwise corrupt it. Unlinking
// the lock file via --force's remove_dir_all is safe — the held file
// descriptor keeps the lock regardless of the directory entry.
let _lock = obisys::DirLock::acquire(&output).unwrap_or_else(|e| {
eprintln!("error locking output directory {}: {e}", output.display());
std::process::exit(1);
});
// ── Resolve evidence kind ────────────────────────────────────────────────
let (evidence, effective_kmer_size) = if args.approx {
let (z, b, fp) = resolve_approx_params(args.findere_z, args.evidence_bits, args.fp);
let k = args.common.kmer_size;
if z as usize >= k {
eprintln!(
"error: Findere z={z} must be < kmer-size={k} \
(effective kmer size k−z+1 = {} ≤ 0)",
k as isize - z as isize + 1
);
std::process::exit(1);
}
let s = k - z as usize + 1;
info!("approximate evidence: b={b}, z={z}, fp={fp:.2e}, indexed kmer size={s}");
(IndexMode::Approx { b, z }, s)
} else {
(IndexMode::Exact, args.common.kmer_size)
};
// ── Open or create the index ─────────────────────────────────────────────
if KmerIndex::is_an_index(&output) {
if !args.force {
eprintln!(
"error: an index already exists at {} (use --force to overwrite it)",
output.display()
);
std::process::exit(1);
}
info!("--force: removing existing index at {}", output.display());
std::fs::remove_dir_all(&output).unwrap_or_else(|e| {
eprintln!("error removing existing index: {e}");
std::process::exit(1);
});
} else if output.exists() {
eprintln!(
"error: {} exists but is not an obikmer index, it cannot be deleted",
output.display()
);
std::process::exit(1);
}
let n_bits = partitions_to_bits(args.common.partitions);
let effective = 1usize << n_bits;
if effective != args.common.partitions {
info!(
"partitions: {} → {} (next power of 2)",
args.common.partitions, effective
);
}
let block_bits = block_size_to_bits(args.block_size);
let config = IndexConfig {
kmer_size: effective_kmer_size,
minimizer_size: args.common.minimizer_size,
n_bits,
with_counts: args.with_counts,
evidence: evidence.clone(),
block_bits,
};
let genome_info = args.label.as_ref().map(|label| {
GenomeInfo::validate_label(label).unwrap_or_else(|e| {
eprintln!("error: --label: {e}");
std::process::exit(1);
});
let mut info = GenomeInfo::new(label.clone());
for (k, v) in &args.meta {
info.meta.insert(k.clone(), v.clone());
}
info
});
let idx = KmerIndex::create(&output, config, genome_info).unwrap_or_else(|e| {
eprintln!("error creating index: {e}");
std::process::exit(1);
});
// ── Stage 1: scatter ─────────────────────────────────────────────────────
if current_state(&idx) < IndexState::Scattered {
let n_workers = args.common.threads.max(1);
let max_open = args.common.effective_max_open();
let t = Stage::start("scatter");
let pb = spinner("scatter");
let mut ema_rate: f64 = 0.0;
let mut last_t = Instant::now();
let mut last_bases: u64 = 0;
const ALPHA: f64 = 0.15;
let mut router = PartitionRouter::new(&idx)
.level_max(args.common.level_max)
.theta(args.common.theta)
.workers(n_workers)
.max_open(max_open)
.files(args.common.seqfile_paths())
.on_progress(|p: Progress| {
let now = Instant::now();
let dt = now.duration_since(last_t).as_secs_f64();
if dt > 0.0 {
let instant = (p.position - last_bases) as f64 / dt;
ema_rate = ALPHA * instant + (1.0 - ALPHA) * ema_rate;
}
last_t = now;
last_bases = p.position;
let bp = p.position as f64;
let (count_str, rate_str) = if bp >= 1e9 {
(
format!("{:.2} Gbp", bp / 1e9),
format!("{:.0} Mbp/s", ema_rate / 1e6),
)
} else {
(
format!("{:.0} Mbp", bp / 1e6),
format!("{:.0} Mbp/s", ema_rate / 1e6),
)
};
pb.set_message(format!("{count_str} {rate_str}"));
});
router.run().unwrap_or_else(|e| {
eprintln!("error: {e}");
std::process::exit(1);
});
pb.finish_and_clear();
rep.push(t.stop());
drop(router); // ends the borrow of `idx` early — `PartitionRouter`'s `Drop` impl would otherwise extend it to the end of scope (`run()` already called `close()`, which marks scatter done, internally)
} else {
info!("scatter already done, skipping");
}
// ── Stage 2: dereplicate + count ─────────────────────────────────────────
if current_state(&idx) < IndexState::Counted {
let t = Stage::start("dereplicate");
let pb = progress_bar("dereplication", idx.n_partitions() as u64, "partitions");
Dereplicator::new(&idx)
.on_progress(|_: Progress| pb.inc(1))
.run()
.unwrap_or_else(|e| {
eprintln!("error: {e}");
std::process::exit(1);
});
pb.finish_and_clear();
rep.push(t.stop());
let t = Stage::start("count_kmer");
let pb = progress_bar("counting", idx.n_partitions() as u64, "partitions");
// `Counter::run` writes `spectrums/{label}.json` and marks count
// done (`count.done`) internally once every partition succeeds.
Counter::new(&idx)
.keep_partial(args.keep_intermediate)
.on_progress(|_: Progress| pb.inc(1))
.run()
.unwrap_or_else(|e| {
eprintln!("error: {e}");
std::process::exit(1);
});
pb.finish_and_clear();
rep.push(t.stop());
} else {
info!("dereplicate+count already done, skipping");
}
// ── Stage 3: build layered index ─────────────────────────────────────────
if current_state(&idx) < IndexState::Indexed {
let t = Stage::start("index");
let pb = progress_bar("index", idx.n_partitions() as u64, "partitions");
let total_kmers = LayerBuilder::new(&idx)
.min_abundance(args.min_abundance)
.max_abundance(args.max_abundance)
.keep_intermediate(args.keep_intermediate)
.on_progress(|_: Progress| pb.inc(1))
.run()
.unwrap_or_else(|e| {
eprintln!("error: {e}");
std::process::exit(1);
});
pb.finish_and_clear();
info!("done — {total_kmers} total kmers indexed");
rep.push(t.stop());
// `LayerBuilder::run` marks the index done (`index.done`) internally
// once every partition succeeds.
} else {
info!("index already built, skipping");
}
rep.print();
}
src/obikmer2/src/cmd/mod.rs
+1
View File
@@ -0,0 +1 @@
pub mod index;
src/obikmer2/src/main.rs
+32
View File
@@ -0,0 +1,32 @@
mod cli;
mod cmd;
use clap::{Parser, Subcommand};
use tracing_subscriber::{EnvFilter, fmt};
#[derive(Parser)]
#[command(name = "obikmer2", about = "DNA k-mer tools", version)]
struct Cli {
#[command(subcommand)]
command: Commands,
}
#[derive(Subcommand)]
enum Commands {
/// Build the complete genome index (scatter → dereplicate → count → layered MPHF)
Index(cmd::index::IndexArgs),
}
fn main() {
fmt()
.with_env_filter(
EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")),
)
.with_writer(std::io::stderr)
.init();
let cli = Cli::parse();
match cli.command {
Commands::Index(args) => cmd::index::run(args),
}
}
src/obikmerge/Cargo.toml
+15
View File
@@ -0,0 +1,15 @@
[package]
name = "obikmerge"
version = "0.1.0"
edition = "2024"
[dependencies]
obikindex = { path = "../obikindex" }
obikindexer = { path = "../obikindexer" }
obikfilter = { path = "../obikfilter" }
obicompactvec = { path = "../obicompactvec" }
obiskio = { path = "../obiskio" }
obikseq = { path = "../obikseq" }
obipipeline = { path = "../obipipeline" }
obisys = { path = "../obisys" }
tracing = "0.1.44"
src/obikmerge/src/lib.rs
+17
View File
@@ -0,0 +1,17 @@
//! Merging multiple `obikindex::KmerIndex` sources into one: bootstrapping
//! from the first source, then merging each remaining source partition by
//! partition (de Bruijn graph union + column fill). A maintenance/
//! transformation operation on already-built indexes, not part of the
//! `Index { Partition { Layer } }` data model itself — kept out of
//! `obikindex` so that crate doesn't grow this algorithm's own
//! dependencies (`obipipeline`, `obidebruinj` via `obikindexer`).
//!
//! [`merge_layer`] holds the per-partition merge primitive
//! (`merge_partition`, `SrcLayerData`) — the latter is also reused by
//! `obikrebuild::rebuild_layer`, hence its `pub` visibility here.
mod merge;
mod merge_layer;
pub use merge::*;
pub use merge_layer::{MergeMode, SrcLayerData};
src/obikindex/src/index/merge.rs → src/obikmerge/src/merge.rs
+9 -8
View File
@@ -3,19 +3,20 @@ use std::fs;
use std::io;
use std::path::Path;
use crate::index::builder::IndexBuilder;
use obikindex::IndexBuilder;
use obisys::{Reporter, Stage, progress_bar, spinner};
use tracing::{debug, info};
use crate::layer::IndexMode;
use obikindex::layer::IndexMode;
use crate::index::error::{OKIError, OKIResult};
use crate::index::kmer_index::KmerIndex;
use crate::index::meta::{GenomeInfo, IndexMeta};
use crate::index::state::IndexState;
use obikindex::{OKIError, OKIResult};
use obikindex::KmerIndex;
use obikindex::{GenomeInfo, IndexMeta};
use obikindex::IndexState;
use obisys::PartitionRunner;
pub use crate::index::merge_layer::MergeMode;
pub use crate::merge_layer::MergeMode;
// ── per-partition diagnostic record ──────────────────────────────────────────
@@ -229,7 +230,7 @@ impl KmerIndex {
let srcs = &srcs;
let evidence = &evidence;
let runner = crate::index::numa::PartitionRunner::new();
let runner = PartitionRunner::new();
let mut part_stats: Vec<PartStat> = Vec::with_capacity(n_partitions);
runner
src/obikindex/src/index/merge_layer/mod.rs → src/obikmerge/src/merge_layer/mod.rs
+27 -30
View File
@@ -20,16 +20,18 @@ use tracing::debug;
use obicompactvec::{PersistentBitMatrixBuilder, PersistentCompactIntMatrixBuilder};
use obikseq::CanonicalKmer;
use crate::layer::{IndexMode, TypedLayer, LayeredMap, MphfOnly, layer_dir};
use obiskio::{SKError, SKResult, UnitigFileReader};
use obikindex::layer::{IndexMode, TypedLayer, LayeredMap, MphfOnly};
use obikindex::layer::utils::layer_dir;
use obiskio::UnitigFileReader;
use obikindex::{OKIError, OKIResult};
use crate::index::common::{ColBuilder, load_meta, olm_to_sk};
use crate::index::graph_pipeline::{build_graph, materialize_layer};
use crate::index::kmer_index::KmerIndex;
use obikindex::{ColBuilder, load_meta};
use obikindexer::{build_graph, materialize_layer};
use obikindex::KmerIndex;
mod src_layer;
pub(crate) use src_layer::SrcLayerData;
pub use src_layer::SrcLayerData;
// ── MergeMode ─────────────────────────────────────────────────────────────────
@@ -216,15 +218,14 @@ impl KmerIndex {
n_dst_genomes: usize,
block_bits: u8,
evidence: &IndexMode,
) -> SKResult<usize> {
) -> OKIResult<usize> {
let dst_index_dir = self.index_dir(i);
if !dst_index_dir.exists() {
return Ok(0);
}
load_meta(&dst_index_dir, "merge")?; // ensure meta.json exists before LayeredMap::open
let dst_map =
Arc::new(LayeredMap::<()>::open(&dst_index_dir).map_err(|e| olm_to_sk(e, "merge"))?);
load_meta(&dst_index_dir)?; // ensure meta.json exists before LayeredMap::open
let dst_map = Arc::new(LayeredMap::<()>::open(&dst_index_dir)?);
let n_dst_layers = dst_map.n_layers();
let n_src_total: usize = sources.iter().map(|(_, n)| *n).sum();
@@ -235,8 +236,7 @@ impl KmerIndex {
TypedLayer::<()>::init_presence_matrix(
&layer_dir(&dst_index_dir, l),
dst_map.layer(l).n(),
)
.map_err(|e| olm_to_sk(e, "merge"))?;
)?;
}
}
@@ -254,7 +254,7 @@ impl KmerIndex {
if !src_index_dir.exists() {
continue;
}
let src_meta = load_meta(&src_index_dir, "merge")?;
let src_meta = load_meta(&src_index_dir)?;
for l in 0..src_meta.n_layers {
let p = layer_dir(&src_index_dir, l).join("unitigs.bin");
if p.exists() {
@@ -277,7 +277,7 @@ impl KmerIndex {
let g = build_graph(
unitig_paths.into_iter(),
move |path: PathBuf, emit: &mut dyn FnMut(Vec<CanonicalKmer>)| -> SKResult<()> {
move |path: PathBuf, emit: &mut dyn FnMut(Vec<CanonicalKmer>)| -> OKIResult<()> {
let reader = UnitigFileReader::open_sequential(&path)?;
let mut batch: Vec<CanonicalKmer> = Vec::with_capacity(BATCH);
for (kmer, _, _) in reader.iter_indexed_canonical_kmers() {
@@ -318,9 +318,7 @@ impl KmerIndex {
let t_open = std::time::Instant::now();
let new_mphf: Option<Arc<MphfOnly>> = if any_new {
Some(Arc::new(
MphfOnly::open(&new_layer_dir).map_err(|e| olm_to_sk(e, "merge"))?,
))
Some(Arc::new(MphfOnly::open(&new_layer_dir)?))
} else {
None
};
@@ -340,13 +338,13 @@ impl KmerIndex {
MergeMode::Count => new_layer_dir.join("counts"),
};
fs::create_dir_all(&data_dir)?;
let mut mb = MatrixBuilder::new(mode, n_new, &data_dir).map_err(SKError::Io)?;
let mut mb = MatrixBuilder::new(mode, n_new, &data_dir).map_err(OKIError::Io)?;
for _ in 0..n_dst_genomes {
mb.add_absent_col().map_err(SKError::Io)?;
mb.add_absent_col().map_err(OKIError::Io)?;
}
let cols = (0..n_src_total)
.map(|_| mb.add_col().map_err(SKError::Io))
.collect::<SKResult<Vec<_>>>()?;
.map(|_| mb.add_col().map_err(OKIError::Io))
.collect::<OKIResult<Vec<_>>>()?;
(cols, Some(mb))
} else {
(vec![], None)
@@ -365,10 +363,10 @@ impl KmerIndex {
MergeMode::Presence => layer_dir.join("presence"),
MergeMode::Count => layer_dir.join("counts"),
};
let mut mb = MatrixBuilder::resume(mode, &data_dir).map_err(SKError::Io)?;
let mut mb = MatrixBuilder::resume(mode, &data_dir).map_err(OKIError::Io)?;
let cols = (0..n_src_total)
.map(|_| mb.add_col().map_err(SKError::Io))
.collect::<SKResult<Vec<_>>>()?;
.map(|_| mb.add_col().map_err(OKIError::Io))
.collect::<OKIResult<Vec<_>>>()?;
exist_mbs.push(mb);
exist_builders.push(cols);
}
@@ -391,7 +389,7 @@ impl KmerIndex {
col_offset += src_n;
continue;
}
let src_meta = load_meta(&src_index_dir, "merge")?;
let src_meta = load_meta(&src_index_dir)?;
for l in 0..src_meta.n_layers {
let src_layer_dir = layer_dir(&src_index_dir, l);
if src_layer_dir.join("unitigs.bin").exists() {
@@ -552,7 +550,7 @@ impl KmerIndex {
.into_inner()
.unwrap_or_else(|e| e.into_inner())
{
return Err(SKError::InvalidData {
return Err(OKIError::InvalidData {
context: "merge pass2",
detail: msg,
});
@@ -568,7 +566,7 @@ impl KmerIndex {
.unwrap_or_else(|e| e.into_inner())
.close()?;
}
mb.close().map_err(SKError::Io)?;
mb.close().map_err(OKIError::Io)?;
}
for b in new_locked {
@@ -579,13 +577,12 @@ impl KmerIndex {
.close()?;
}
if let Some(mb) = new_mb {
mb.close().map_err(SKError::Io)?;
mb.close().map_err(OKIError::Io)?;
let mut part_meta = self.partition_meta(i)?;
part_meta.n_layers = new_layer_idx + 1;
part_meta
.save(&dst_index_dir)
.map_err(|e| olm_to_sk(e, "merge"))?;
.save(&dst_index_dir)?;
}
debug!(
src/obikindex/src/index/merge_layer/src_layer.rs → src/obikmerge/src/merge_layer/src_layer.rs
+15 -16
View File
@@ -2,44 +2,43 @@ use std::path::Path;
use obicompactvec::{MatrixGroupOps, PersistentBitMatrix, PersistentCompactIntMatrix};
use obikseq::CanonicalKmer;
use crate::layer::MphfOnly;
use obiskio::{SKError, SKResult};
use obikindex::layer::MphfOnly;
use obikindex::{OKIError, OKIResult};
use crate::index::common::olm_to_sk;
use super::MergeMode;
// ── SrcLayerData — opened source matrix for pass-2 lookup ─────────────────────
pub(crate) enum SrcLayerData {
pub enum SrcLayerData {
Presence(MphfOnly, PersistentBitMatrix),
Count(MphfOnly, PersistentCompactIntMatrix),
}
impl SrcLayerData {
pub(crate) fn open(layer_dir: &Path, merge_mode: MergeMode) -> SKResult<Self> {
pub fn open(layer_dir: &Path, merge_mode: MergeMode) -> OKIResult<Self> {
let counts_dir = layer_dir.join("counts");
match merge_mode {
MergeMode::Presence => {
if counts_dir.exists() && !layer_dir.join("presence").exists() {
let mphf = MphfOnly::open(layer_dir).map_err(|e| olm_to_sk(e, "merge"))?;
let mat = PersistentCompactIntMatrix::open(layer_dir).map_err(SKError::Io)?;
let mphf = MphfOnly::open(layer_dir)?;
let mat = PersistentCompactIntMatrix::open(layer_dir).map_err(OKIError::Io)?;
Ok(SrcLayerData::Count(mphf, mat))
} else {
// presence dir exists, or neither exists → Implicit handled by open()
let mphf = MphfOnly::open(layer_dir).map_err(|e| olm_to_sk(e, "merge"))?;
let mat = PersistentBitMatrix::open(layer_dir).map_err(SKError::Io)?;
let mphf = MphfOnly::open(layer_dir)?;
let mat = PersistentBitMatrix::open(layer_dir).map_err(OKIError::Io)?;
Ok(SrcLayerData::Presence(mphf, mat))
}
}
MergeMode::Count => {
let mphf = MphfOnly::open(layer_dir).map_err(|e| olm_to_sk(e, "merge"))?;
let mphf = MphfOnly::open(layer_dir)?;
if counts_dir.exists() {
let mat = PersistentCompactIntMatrix::open(layer_dir).map_err(SKError::Io)?;
let mat = PersistentCompactIntMatrix::open(layer_dir).map_err(OKIError::Io)?;
Ok(SrcLayerData::Count(mphf, mat))
} else {
// No counts → treat as implicit presence (all 1s)
let mat = PersistentBitMatrix::open(layer_dir).map_err(SKError::Io)?;
let mat = PersistentBitMatrix::open(layer_dir).map_err(OKIError::Io)?;
Ok(SrcLayerData::Presence(mphf, mat))
}
}
@@ -58,7 +57,7 @@ impl SrcLayerData {
buf
}
pub(crate) fn n_slots(&self) -> usize {
pub fn n_slots(&self) -> usize {
match self {
SrcLayerData::Presence(_, mat) => mat.n(),
SrcLayerData::Count(_, mat) => mat.n(),
@@ -67,7 +66,7 @@ impl SrcLayerData {
/// MPHF lookup: returns the slot index for `kmer` (kmer must be in the domain).
#[inline]
pub(crate) fn slot(&self, kmer: CanonicalKmer) -> usize {
pub fn slot(&self, kmer: CanonicalKmer) -> usize {
match self {
SrcLayerData::Presence(mphf, _) => mphf.index(kmer),
SrcLayerData::Count(mphf, _) => mphf.index(kmer),
@@ -76,7 +75,7 @@ impl SrcLayerData {
/// Row lookup by slot index, bypassing the MPHF.
#[inline]
pub(crate) fn fill_row_by_slot(&self, slot: usize, n_genomes: usize) -> Vec<u32> {
pub fn fill_row_by_slot(&self, slot: usize, n_genomes: usize) -> Vec<u32> {
let mut buf = vec![0u32; n_genomes];
match self {
SrcLayerData::Presence(_, mat) => mat.fill_row(slot, &mut buf),
@@ -86,7 +85,7 @@ impl SrcLayerData {
}
/// Call `f` with a reference to the underlying matrix as `&dyn MatrixGroupOps`.
pub(crate) fn with_matrix<R>(&self, f: impl FnOnce(&dyn MatrixGroupOps) -> R) -> R {
pub fn with_matrix<R>(&self, f: impl FnOnce(&dyn MatrixGroupOps) -> R) -> R {
match self {
SrcLayerData::Presence(_, mat) => f(mat),
SrcLayerData::Count(_, mat) => f(mat),
src/obikphylo/Cargo.toml
+1
View File
@@ -9,6 +9,7 @@ obikseq = { path = "../obikseq" }
obiskio = { path = "../obiskio" }
obisys = { path = "../obisys" }
obicompactvec = { path = "../obicompactvec" }
obikidxcache = { path = "../obikidxcache" }
obiskbuilder = { path = "../obiskbuilder" }
obipipeline = { path = "../obipipeline" }
memmap2 = "0.9"
src/obikindex/src/index/distance.rs → src/obikphylo/src/distance.rs
+3 -3
View File
@@ -1,10 +1,10 @@
use ndarray::Array2;
use obicompactvec::traits::{BitPartials, CountPartials};
use crate::layer::LayeredStore;
use obikidxcache::LayeredStore;
use rayon::prelude::*;
use crate::index::error::{OKIError, OKIResult};
use crate::index::kmer_index::KmerIndex;
use obikindex::{OKIError, OKIResult};
use obikindex::KmerIndex;
// ── Public API ────────────────────────────────────────────────────────────────
src/obikphylo/src/lib.rs
+3
View File
@@ -9,6 +9,9 @@
//! incrementally.
mod cardcomp;
mod distance;
mod matrix_store;
pub mod siblings;
pub use cardcomp::{cardinality_transition_probs, composition_transition_probs, pairwise_cost_matrix};
pub use distance::{DistanceMetric, DistanceOutput};
src/obikindex/src/index/matrix_store.rs → src/obikphylo/src/matrix_store.rs
+13 -12
View File
@@ -1,46 +1,47 @@
use obikindex::OKIResult;
use obikindex::layer::open_data;
use obikidxcache::LayeredStore;
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
use crate::layer::{LayeredStore, open_data};
use obiskio::SKResult;
use crate::index::common::{load_meta, olm_to_sk};
use crate::index::kmer_index::KmerIndex;
use obikindex::load_meta;
use obikindex::KmerIndex;
impl KmerIndex {
/// Open all count matrices for partition `part`, one per layer.
/// Layers without a `counts/` directory are skipped.
pub fn count_store(&self, part: usize) -> SKResult<LayeredStore<PersistentCompactIntMatrix>> {
pub fn count_store(&self, part: usize) -> OKIResult<LayeredStore<PersistentCompactIntMatrix>> {
let index_dir = self.index_dir(part);
if !index_dir.exists() {
return Ok(LayeredStore::new(vec![]));
}
let n_layers = load_meta(&index_dir, "distance")?.n_layers;
let n_layers = self.n_layers();
let matrices = (0..n_layers)
.filter_map(|l| {
self.layer_dir(part, l)
.join("counts")
.exists()
.then(|| open_data(&index_dir, l).map_err(|e| olm_to_sk(e, "distance")))
.then(|| open_data(&index_dir, l))
})
.collect::<SKResult<Vec<_>>>()?;
.collect::<OKIResult<Vec<_>>>()?;
Ok(LayeredStore::new(matrices))
}
/// Open all presence matrices for partition `part`, one per layer.
/// Layers without a `presence/` directory are skipped.
pub fn presence_store(&self, part: usize) -> SKResult<LayeredStore<PersistentBitMatrix>> {
pub fn presence_store(&self, part: usize) -> OKIResult<LayeredStore<PersistentBitMatrix>> {
let index_dir = self.index_dir(part);
if !index_dir.exists() {
return Ok(LayeredStore::new(vec![]));
}
let n_layers = load_meta(&index_dir, "distance")?.n_layers;
let n_layers = load_meta(&index_dir)?.n_layers;
let matrices = (0..n_layers)
.filter_map(|l| {
self.layer_dir(part, l)
.join("presence")
.exists()
.then(|| open_data(&index_dir, l).map_err(|e| olm_to_sk(e, "distance")))
.then(|| open_data(&index_dir, l))
})
.collect::<SKResult<Vec<_>>>()?;
.collect::<OKIResult<Vec<_>>>()?;
Ok(LayeredStore::new(matrices))
}
}
src/obikphylo/src/siblings/build.rs
+2 -2
View File
@@ -14,7 +14,7 @@ use obikindex::OKIResult;
use super::cache::PartitionCache;
use super::helpers::{central_base, is_minorant};
use super::{ANNEX_FILE_NAME, FamilyMask, SiblingAnnexBuilder, olm_to_ok};
use super::{ANNEX_FILE_NAME, FamilyMask, SiblingAnnexBuilder};
// ── obipipeline data types ─────────────────────────────────────────────────
@@ -125,7 +125,7 @@ fn build_layer_sibling_annex(
l: usize,
cache: &Arc<PartitionCache>,
) -> OKIResult<u64> {
let mphf = MphfLayer::open(layer_dir).map_err(olm_to_ok)?;
let mphf = MphfLayer::open(layer_dir)?;
let k = index.kmer_size();
let n = mphf.n();
src/obikphylo/src/siblings/family_scan.rs
+3 -3
View File
@@ -65,7 +65,7 @@ use obikindex::layer::KmerLayer;
use super::cache::PartitionCache;
use super::helpers::central_base;
use super::iter::{SiblingEntry, SiblingLayerExt};
use super::{ANNEX_FILE_NAME, FamilyMask, SiblingAnnex, olm_to_ok};
use super::{ANNEX_FILE_NAME, FamilyMask, SiblingAnnex};
/// Families per batch — see the module docs for the memory-vs-per-partition-
/// density trade-off this picks a point on. At ~90 genomes and a few
@@ -194,10 +194,10 @@ pub(super) fn scan_layer_families(
let index_dir = layer_dir
.parent()
.expect("layer_dir has a parent index dir");
let meta = PartitionMeta::load(index_dir).map_err(olm_to_ok)?;
let meta = PartitionMeta::load(index_dir)?;
let annex = Arc::new(SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME))?);
let mat = KmerLayer::open(layer_dir, &meta.mode, with_counts).map_err(olm_to_ok)?;
let mat = KmerLayer::open(layer_dir, &meta.mode, with_counts)?;
let n_cols = mat.n_cols().min(n_genomes);
let ctx = Arc::new(LayerCtx {
src/obikphylo/src/siblings/mod.rs
-11
View File
@@ -71,15 +71,4 @@ pub(crate) use siblingannex::{FamilyMask, SiblingAnnex, SiblingAnnexBuilder};
pub use stats::{SiblingAnnexStats, SiblingStatsExt};
pub use subsample::EntropyBias;
use obikindex::layer::OLMError;
use obikindex::OKIError;
pub(super) const ANNEX_FILE_NAME: &str = "siblings.psib";
pub(super) fn olm_to_ok(e: OLMError) -> OKIError {
match e {
OLMError::Io(e) => OKIError::Io(e),
other => OKIError::InvalidInput(format!("layered-map error: {other}")),
}
}
src/obikquery/Cargo.toml
+7
View File
@@ -0,0 +1,7 @@
[package]
name = "obikquery"
version = "0.1.0"
edition = "2024"
[dependencies]
obikindex = { path = "../obikindex" }
src/obikquery/src/lib.rs
+10
View File
@@ -0,0 +1,10 @@
//! Query-side operations on an `obikindex::KmerIndex`: staging ground for
//! code currently living in `obikindex::index`'s query path, to be migrated
//! here to lighten that crate — see `DevDocMD/` for the rationale. Kept as
//! a separate crate — not a module of `obikindex` — so the dependency runs
//! one way only (query code depends on the data model, never the reverse),
//! same pattern as `obikindexer` for the build side.
mod query_layer;
pub use query_layer::{KmerDesc, QueryHit, QueryStats};
src/obikindex/src/index/query_layer.rs → src/obikquery/src/query_layer.rs
+10 -20
View File
@@ -3,20 +3,10 @@ use std::path::Path;
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
use obikseq::CanonicalKmer;
use crate::layer::{MphfLayer, OLMError};
use obiskio::{SKError, SKResult};
use obikindex::layer::MphfLayer;
use obikindex::{OKIError, OKIResult};
use crate::index::kmer_index::KmerIndex;
fn olm_to_sk(e: OLMError) -> SKError {
match e {
OLMError::Io(io_err) => SKError::Io(io_err),
other => SKError::InvalidData {
context: "query",
detail: other.to_string(),
},
}
}
use obikindex::KmerIndex;
// ── per-layer query handle ────────────────────────────────────────────────────
@@ -26,21 +16,21 @@ enum QueryLayer {
}
impl QueryLayer {
fn open(layer_dir: &Path, with_counts: bool) -> SKResult<Self> {
let mphf = MphfLayer::open(layer_dir).map_err(olm_to_sk)?;
fn open(layer_dir: &Path, with_counts: bool) -> OKIResult<Self> {
let mphf = MphfLayer::open(layer_dir)?;
let counts_dir = layer_dir.join("counts");
let presence_dir = layer_dir.join("presence");
if with_counts && counts_dir.exists() {
let mat = PersistentCompactIntMatrix::open(layer_dir).map_err(SKError::Io)?;
let mat = PersistentCompactIntMatrix::open(layer_dir).map_err(OKIError::Io)?;
Ok(QueryLayer::Count(mphf, mat))
} else if presence_dir.exists() || !counts_dir.exists() {
// presence mode, or no matrix at all → Implicit handled inside open()
let mat = PersistentBitMatrix::open(layer_dir).map_err(SKError::Io)?;
let mat = PersistentBitMatrix::open(layer_dir).map_err(OKIError::Io)?;
Ok(QueryLayer::Presence(mphf, mat))
} else {
// counts exist but not presence — count layer, no presence requested
let mat = PersistentCompactIntMatrix::open(layer_dir).map_err(SKError::Io)?;
let mat = PersistentCompactIntMatrix::open(layer_dir).map_err(OKIError::Io)?;
Ok(QueryLayer::Count(mphf, mat))
}
}
@@ -168,7 +158,7 @@ impl KmerIndex {
n_genomes: usize,
with_counts: bool,
mut on_event: F,
) -> SKResult<QueryStats>
) -> OKIResult<QueryStats>
where
F: FnMut(QueryHit),
{
@@ -186,7 +176,7 @@ impl KmerIndex {
let meta = self.partition_meta(part_idx)?;
let layers: Vec<QueryLayer> = (0..meta.n_layers)
.map(|i| QueryLayer::open(&self.layer_dir(part_idx, i), with_counts))
.collect::<SKResult<_>>()?;
.collect::<OKIResult<_>>()?;
// ── Stage 1: MPHF-only pass, bucket hits by (layer_idx, slot) ────────
let mut by_layer: Vec<HashMap<usize, &Vec<KmerDesc>>> =
src/obikindex/src/index/tests/query_layer.rs → src/obikquery/src/tests/query_layer.rs
+3 -3
View File
@@ -1,5 +1,5 @@
use super::*;
use crate::index::meta::IndexConfig;
use obikindex::IndexConfig;
// ── QueryStats::AddAssign ───────────────────────────────────────────────────
@@ -50,7 +50,7 @@ fn query_partition_with_missing_index_dir_returns_default_stats() {
minimizer_size: 9,
n_bits: 2,
with_counts: false,
evidence: crate::layer::IndexMode::Exact,
evidence: obikindex::layer::IndexMode::Exact,
block_bits: 0,
};
let index = KmerIndex::create(tmp.path().join("idx"), config, None).expect("create index");
@@ -78,7 +78,7 @@ fn query_partition_with_empty_kmers_is_a_noop() {
minimizer_size: 9,
n_bits: 2,
with_counts: false,
evidence: crate::layer::IndexMode::Exact,
evidence: obikindex::layer::IndexMode::Exact,
block_bits: 0,
};
let index = KmerIndex::create(tmp.path().join("idx"), config, None).expect("create index");
src/obikrebuild/Cargo.toml
+16
View File
@@ -0,0 +1,16 @@
[package]
name = "obikrebuild"
version = "0.1.0"
edition = "2024"
[dependencies]
obikindex = { path = "../obikindex" }
obikindexer = { path = "../obikindexer" }
obikfilter = { path = "../obikfilter" }
obikmerge = { path = "../obikmerge" }
obicompactvec = { path = "../obicompactvec" }
obiskio = { path = "../obiskio" }
obikseq = { path = "../obikseq" }
obidebruinj = { path = "../obidebruinj" }
obisys = { path = "../obisys" }
tracing = "0.1.44"
src/obikrebuild/src/lib.rs
+12
View File
@@ -0,0 +1,12 @@
//! Rebuilding an `obikindex::KmerIndex`: compacting a source index into a
//! new single-layer index while applying k-mer filters (`rebuild`), and
//! reindexing an existing index's evidence representation in place
//! (`reindex`, `Exact`/`Approx`/`Hybrid`). Both are maintenance/
//! transformation operations on an already-built index, not part of the
//! `Index { Partition { Layer } }` data model itself — kept out of
//! `obikindex` so that crate doesn't grow every algorithm's own
//! dependencies (`obikfilter`, `obikmerge`, `obikindexer`).
mod rebuild;
mod rebuild_layer;
mod reindex;
src/obikindex/src/index/rebuild.rs → src/obikrebuild/src/rebuild.rs
+8 -6
View File
@@ -1,13 +1,15 @@
use std::path::Path;
use crate::index::builder::IndexBuilder;
use crate::index::{KmerFilter, MergeMode};
use obikindex::IndexBuilder;
use obikfilter::KmerFilter;
use obikmerge::MergeMode;
use obisys::{Reporter, Stage, progress_bar};
use tracing::info;
use crate::index::error::{OKIError, OKIResult};
use crate::index::kmer_index::KmerIndex;
use crate::index::state::IndexState;
use obikindex::{OKIError, OKIResult};
use obikindex::KmerIndex;
use obikindex::IndexState;
use obisys::PartitionRunner;
impl KmerIndex {
/// Rebuild `src` into a new compact single-layer index at `output`.
@@ -61,7 +63,7 @@ impl KmerIndex {
let pb = progress_bar("rebuild", n_partitions as u64, "partitions");
let order: Vec<usize> = (0..n_partitions).collect();
let runner = crate::index::numa::PartitionRunner::new();
let runner = PartitionRunner::new();
runner.run(
&order,
|i| dst_partition.rebuild_partition(src, i, filters, mode, n_genomes, block_bits),
src/obikindex/src/index/rebuild_layer.rs → src/obikrebuild/src/rebuild_layer.rs
+30 -30
View File
@@ -6,15 +6,17 @@ use obicompactvec::{
};
use obidebruinj::GraphDeBruijn;
use obikseq::CanonicalKmer;
use crate::layer::meta::PartitionMeta;
use crate::layer::{IndexMode, MphfLayer, layer_dir};
use obiskio::{SKError, SKResult, UnitigFileReader};
use obikindex::layer::meta::PartitionMeta;
use obikindex::layer::{IndexMode, MphfLayer};
use obikindex::layer::utils::layer_dir;
use obiskio::UnitigFileReader;
use obikindex::{OKIError, OKIResult};
use crate::index::common::{load_meta, olm_to_sk};
use crate::index::filter::KmerFilter;
use crate::index::graph_pipeline::materialize_layer;
use crate::index::merge_layer::{MergeMode, SrcLayerData};
use crate::index::kmer_index::KmerIndex;
use obikindex::load_meta;
use obikfilter::KmerFilter;
use obikindexer::materialize_layer;
use obikmerge::{MergeMode, SrcLayerData};
use obikindex::KmerIndex;
// ── Builders — pair matrix builder + column builders for one mode ─────────────
@@ -27,22 +29,22 @@ enum Builders {
}
impl Builders {
fn new(mode: MergeMode, n: usize, dir: &Path, n_genomes: usize) -> SKResult<Self> {
fn new(mode: MergeMode, n: usize, dir: &Path, n_genomes: usize) -> OKIResult<Self> {
match mode {
MergeMode::Presence => {
let mut mat = PersistentBitMatrixBuilder::new(n, dir).map_err(SKError::Io)?;
let mut mat = PersistentBitMatrixBuilder::new(n, dir).map_err(OKIError::Io)?;
let mut cols = Vec::with_capacity(n_genomes);
for _ in 0..n_genomes {
cols.push(mat.add_col().map_err(SKError::Io)?);
cols.push(mat.add_col().map_err(OKIError::Io)?);
}
Ok(Builders::Presence(mat, cols))
}
MergeMode::Count => {
let mut mat =
PersistentCompactIntMatrixBuilder::new(n, dir).map_err(SKError::Io)?;
PersistentCompactIntMatrixBuilder::new(n, dir).map_err(OKIError::Io)?;
let mut cols = Vec::with_capacity(n_genomes);
for _ in 0..n_genomes {
cols.push(mat.add_col().map_err(SKError::Io)?);
cols.push(mat.add_col().map_err(OKIError::Io)?);
}
Ok(Builders::Count(mat, cols))
}
@@ -56,19 +58,19 @@ impl Builders {
}
}
fn close(self) -> SKResult<()> {
fn close(self) -> OKIResult<()> {
match self {
Builders::Presence(mat, cols) => {
for b in cols {
b.close().map_err(SKError::Io)?;
b.close().map_err(OKIError::Io)?;
}
mat.close().map_err(SKError::Io)
mat.close().map_err(OKIError::Io)
}
Builders::Count(mat, cols) => {
for b in cols {
b.close().map_err(SKError::Io)?;
b.close().map_err(OKIError::Io)?;
}
mat.close().map_err(SKError::Io)
mat.close().map_err(OKIError::Io)
}
}
}
@@ -86,7 +88,7 @@ fn try_compute_combined_mask(
filters: &[Box<dyn KmerFilter>],
src_data: &SrcLayerData,
n_genomes: usize,
) -> SKResult<Option<obicompactvec::TempBitVec>> {
) -> OKIResult<Option<obicompactvec::TempBitVec>> {
if filters.is_empty() {
return Ok(None);
}
@@ -101,7 +103,7 @@ fn try_compute_combined_mask(
let n = src_data.n_slots();
let mask = src_data
.with_matrix(|mat| eval_filter_mask(&combined, mat, n))
.map_err(SKError::Io)?;
.map_err(OKIError::Io)?;
Ok(Some(mask))
}
@@ -118,8 +120,8 @@ fn iter_src_kmers_masked(
n_genomes: usize,
filters: &[Box<dyn KmerFilter>],
mut cb: impl FnMut(CanonicalKmer),
) -> SKResult<()> {
let src_meta = load_meta(src_index_dir, "rebuild")?;
) -> OKIResult<()> {
let src_meta = load_meta(src_index_dir)?;
for l in 0..src_meta.n_layers {
let src_layer_dir = layer_dir(src_index_dir, l);
let unitigs_path = src_layer_dir.join("unitigs.bin");
@@ -159,8 +161,8 @@ fn iter_src_layers(
n_genomes: usize,
filters: &[Box<dyn KmerFilter>],
mut cb: impl FnMut(CanonicalKmer, Box<[u32]>),
) -> SKResult<()> {
let src_meta = load_meta(src_index_dir, "rebuild")?;
) -> OKIResult<()> {
let src_meta = load_meta(src_index_dir)?;
for l in 0..src_meta.n_layers {
let src_layer_dir = layer_dir(src_index_dir, l);
let unitigs_path = src_layer_dir.join("unitigs.bin");
@@ -209,13 +211,13 @@ impl KmerIndex {
mode: MergeMode,
n_genomes: usize,
block_bits: u8,
) -> SKResult<()> {
) -> OKIResult<()> {
let src_index_dir = src.index_dir(i);
if !src_index_dir.exists() {
return Ok(());
}
let src_meta = load_meta(&src_index_dir, "rebuild")?;
let src_meta = load_meta(&src_index_dir)?;
if src_meta.n_layers == 0 {
return Ok(());
}
@@ -235,8 +237,7 @@ impl KmerIndex {
let dst_layer_dir = self.layer_dir(i, 0);
let n_new = materialize_layer(g, &dst_layer_dir, block_bits, &IndexMode::Exact)?;
let dst_mphf = MphfLayer::open(&dst_layer_dir)
.map_err(|e| olm_to_sk(e, "rebuild"))?;
let dst_mphf = MphfLayer::open(&dst_layer_dir)?;
// ── Prepare matrix builders (one column per genome) ───────────────────
let data_dir = match mode {
@@ -262,8 +263,7 @@ impl KmerIndex {
n_layers: 1,
mode: IndexMode::Exact,
}
.save(&dst_index_dir)
.map_err(|e| olm_to_sk(e, "rebuild"))?;
.save(&dst_index_dir)?;
Ok(())
}
src/obikindex/src/index/reindex.rs → src/obikrebuild/src/reindex.rs
+10 -13
View File
@@ -1,24 +1,21 @@
use crate::index::builder::IndexBuilder;
use crate::index::meta::IndexMeta;
use crate::layer::{IndexMode, TypedLayer};
use obikindex::IndexBuilder;
use obikindex::IndexMeta;
use obikindex::layer::{IndexMode, TypedLayer};
use obisys::{Reporter, Stage, progress_bar};
use std::fs;
use std::path::Path;
use std::sync::Arc;
use tracing::info;
use crate::index::error::{OKIError, OKIResult};
use crate::index::kmer_index::KmerIndex;
use crate::index::state::IndexState;
use obikindex::{OKIError, OKIResult};
use obikindex::KmerIndex;
use obikindex::IndexState;
use obisys::PartitionRunner;
const EVIDENCE_FILE: &str = "evidence.bin";
const FINGERPRINT_FILE: &str = "fingerprint.bin";
const UNITIG_IDX_FILE: &str = "unitigs.bin.idx";
fn olm_to_oki(e: crate::layer::OLMError) -> OKIError {
OKIError::InvalidInput(e.to_string())
}
impl KmerIndex {
/// Convert every layer's evidence bundle to `target` in-place.
///
@@ -47,7 +44,7 @@ impl KmerIndex {
let pb = progress_bar("reindex", n as u64, "partitions");
let order: Vec<usize> = (0..n).collect();
let runner = crate::index::numa::PartitionRunner::new();
let runner = PartitionRunner::new();
runner.run(
&order,
|i| {
@@ -96,10 +93,10 @@ fn reindex_partition(
fn reindex_layer(layer_dir: &Path, target: &IndexMode, block_bits: u8) -> OKIResult<()> {
match target {
IndexMode::Exact => {
TypedLayer::<()>::build_exact_evidence(layer_dir, block_bits).map_err(olm_to_oki)?;
TypedLayer::<()>::build_exact_evidence(layer_dir, block_bits)?;
}
IndexMode::Approx { b, z } | IndexMode::Hybrid { b, z } => {
TypedLayer::<()>::build_approx_evidence(layer_dir, *b, *z).map_err(olm_to_oki)?;
TypedLayer::<()>::build_approx_evidence(layer_dir, *b, *z)?;
}
}
remove_stale_evidence(layer_dir, target)
src/obikselect/Cargo.toml
+10
View File
@@ -0,0 +1,10 @@
[package]
name = "obikselect"
version = "0.1.0"
edition = "2024"
[dependencies]
obikindex = { path = "../obikindex" }
obicompactvec = { path = "../obicompactvec" }
obisys = { path = "../obisys" }
tracing = "0.1.44"
src/obikselect/src/lib.rs
+11
View File
@@ -0,0 +1,11 @@
//! Genome-column selection/aggregation on an `obikindex::KmerIndex`:
//! projecting/aggregating genome columns (`AggOp`) into new output columns
//! (`OutputCol`), either into a new index (`select`) or in place
//! (`select_in_place`). Orthogonal to per-kmer filtering (`obikfilter`),
//! which operates on rows, not columns. Kept as a separate crate — not a
//! module of `obikindex` — so the dependency runs one way only.
mod select;
mod select_layer;
pub use select_layer::{AggOp, OutputCol};
src/obikindex/src/index/select.rs → src/obikselect/src/select.rs
+9 -9
View File
@@ -1,15 +1,15 @@
use std::path::Path;
use std::sync::Arc;
use crate::index::builder::IndexBuilder;
use crate::index::OutputCol;
use obisys::{Reporter, Stage, progress_bar};
use obikindex::IndexBuilder;
use crate::OutputCol;
use obisys::{Reporter, Stage, progress_bar, PartitionRunner};
use tracing::info;
use crate::index::error::{OKIError, OKIResult};
use crate::index::kmer_index::KmerIndex;
use crate::index::meta::{GenomeInfo, IndexMeta};
use crate::index::state::IndexState;
use obikindex::{OKIError, OKIResult};
use obikindex::KmerIndex;
use obikindex::{GenomeInfo, IndexMeta};
use obikindex::IndexState;
impl KmerIndex {
/// Create a new index at `output` by projecting/aggregating the genome columns
@@ -58,7 +58,7 @@ impl KmerIndex {
let pb = progress_bar("select", n_partitions as u64, "partitions");
let order: Vec<usize> = (0..n_partitions).collect();
let runner = crate::index::numa::PartitionRunner::new();
let runner = PartitionRunner::new();
runner
.run(
&order,
@@ -113,7 +113,7 @@ impl KmerIndex {
let pb = progress_bar("select", n_partitions as u64, "partitions");
let order: Vec<usize> = (0..n_partitions).collect();
let runner = crate::index::numa::PartitionRunner::new();
let runner = PartitionRunner::new();
runner
.run(
&order,
src/obikindex/src/index/select_layer.rs → src/obikselect/src/select_layer.rs
+41 -53
View File
@@ -6,10 +6,9 @@ use obicompactvec::{
ColGroup, MatrixGroupOps, PersistentBitMatrix, PersistentBitMatrixBuilder,
PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder,
};
use crate::layer::OLMError;
use obiskio::{SKError, SKResult};
use obikindex::{OKIError, OKIResult};
use crate::index::kmer_index::KmerIndex;
use obikindex::KmerIndex;
// ── AggOp ─────────────────────────────────────────────────────────────────────
@@ -39,16 +38,6 @@ pub struct OutputCol {
// ── Helpers ───────────────────────────────────────────────────────────────────
fn olm_to_sk(e: OLMError) -> SKError {
match e {
OLMError::Io(e) => SKError::Io(e),
other => SKError::InvalidData {
context: "select",
detail: other.to_string(),
},
}
}
/// Copy all plain files (not subdirectories) from `src_dir` to `dst_dir`.
fn copy_layer_files(src_dir: &Path, dst_dir: &Path) -> io::Result<()> {
for entry in fs::read_dir(src_dir)? {
@@ -71,97 +60,97 @@ fn fill_builders(
output_presence: bool,
mut dst_bit: Option<&mut PersistentBitMatrixBuilder>,
mut dst_int: Option<&mut PersistentCompactIntMatrixBuilder>,
) -> SKResult<()> {
) -> OKIResult<()> {
if src_is_count {
let mat = PersistentCompactIntMatrix::open(src_layer_dir).map_err(SKError::Io)?;
let mat = PersistentCompactIntMatrix::open(src_layer_dir).map_err(OKIError::Io)?;
for spec in specs {
let g = ColGroup::new(&spec.label, spec.indices.clone());
if output_presence {
let b = dst_bit.as_deref_mut().unwrap();
match spec.op {
AggOp::Any => {
b.add_col_from(&mat.partial_group_any(&g, threshold).map_err(SKError::Io)?)
b.add_col_from(&mat.partial_group_any(&g, threshold).map_err(OKIError::Io)?)
}
AggOp::All => {
b.add_col_from(&mat.partial_group_all(&g, threshold).map_err(SKError::Io)?)
b.add_col_from(&mat.partial_group_all(&g, threshold).map_err(OKIError::Io)?)
}
AggOp::None => {
b.add_col_from(&mat.partial_group_none(&g, threshold).map_err(SKError::Io)?)
b.add_col_from(&mat.partial_group_none(&g, threshold).map_err(OKIError::Io)?)
}
AggOp::Sum => {
b.add_col_from_int(&mat.partial_group_sum(&g).map_err(SKError::Io)?)
b.add_col_from_int(&mat.partial_group_sum(&g).map_err(OKIError::Io)?)
}
AggOp::Min => {
b.add_col_from_int(&mat.partial_group_min(&g).map_err(SKError::Io)?)
b.add_col_from_int(&mat.partial_group_min(&g).map_err(OKIError::Io)?)
}
AggOp::Max => {
b.add_col_from_int(&mat.partial_group_max(&g).map_err(SKError::Io)?)
b.add_col_from_int(&mat.partial_group_max(&g).map_err(OKIError::Io)?)
}
}
.map_err(SKError::Io)?;
.map_err(OKIError::Io)?;
} else {
let b = dst_int.as_deref_mut().unwrap();
match spec.op {
AggOp::Sum => b.add_col_from(&mat.partial_group_sum(&g).map_err(SKError::Io)?),
AggOp::Min => b.add_col_from(&mat.partial_group_min(&g).map_err(SKError::Io)?),
AggOp::Max => b.add_col_from(&mat.partial_group_max(&g).map_err(SKError::Io)?),
AggOp::Sum => b.add_col_from(&mat.partial_group_sum(&g).map_err(OKIError::Io)?),
AggOp::Min => b.add_col_from(&mat.partial_group_min(&g).map_err(OKIError::Io)?),
AggOp::Max => b.add_col_from(&mat.partial_group_max(&g).map_err(OKIError::Io)?),
AggOp::Any => b.add_col_from_bit(
&mat.partial_group_any(&g, threshold).map_err(SKError::Io)?,
&mat.partial_group_any(&g, threshold).map_err(OKIError::Io)?,
),
AggOp::All => b.add_col_from_bit(
&mat.partial_group_all(&g, threshold).map_err(SKError::Io)?,
&mat.partial_group_all(&g, threshold).map_err(OKIError::Io)?,
),
AggOp::None => b.add_col_from_bit(
&mat.partial_group_none(&g, threshold).map_err(SKError::Io)?,
&mat.partial_group_none(&g, threshold).map_err(OKIError::Io)?,
),
}
.map_err(SKError::Io)?;
.map_err(OKIError::Io)?;
}
}
} else {
let mat = PersistentBitMatrix::open(src_layer_dir).map_err(SKError::Io)?;
let mat = PersistentBitMatrix::open(src_layer_dir).map_err(OKIError::Io)?;
for spec in specs {
let g = ColGroup::new(&spec.label, spec.indices.clone());
if output_presence {
let b = dst_bit.as_deref_mut().unwrap();
match spec.op {
AggOp::Any => {
b.add_col_from(&mat.partial_group_any(&g, 1).map_err(SKError::Io)?)
b.add_col_from(&mat.partial_group_any(&g, 1).map_err(OKIError::Io)?)
}
AggOp::All => {
b.add_col_from(&mat.partial_group_all(&g, 1).map_err(SKError::Io)?)
b.add_col_from(&mat.partial_group_all(&g, 1).map_err(OKIError::Io)?)
}
AggOp::None => {
b.add_col_from(&mat.partial_group_none(&g, 1).map_err(SKError::Io)?)
b.add_col_from(&mat.partial_group_none(&g, 1).map_err(OKIError::Io)?)
}
AggOp::Sum => {
b.add_col_from_int(&mat.partial_group_sum(&g).map_err(SKError::Io)?)
b.add_col_from_int(&mat.partial_group_sum(&g).map_err(OKIError::Io)?)
}
AggOp::Min => {
b.add_col_from_int(&mat.partial_group_min(&g).map_err(SKError::Io)?)
b.add_col_from_int(&mat.partial_group_min(&g).map_err(OKIError::Io)?)
}
AggOp::Max => {
b.add_col_from_int(&mat.partial_group_max(&g).map_err(SKError::Io)?)
b.add_col_from_int(&mat.partial_group_max(&g).map_err(OKIError::Io)?)
}
}
.map_err(SKError::Io)?;
.map_err(OKIError::Io)?;
} else {
let b = dst_int.as_deref_mut().unwrap();
match spec.op {
AggOp::Sum => b.add_col_from(&mat.partial_group_sum(&g).map_err(SKError::Io)?),
AggOp::Min => b.add_col_from(&mat.partial_group_min(&g).map_err(SKError::Io)?),
AggOp::Max => b.add_col_from(&mat.partial_group_max(&g).map_err(SKError::Io)?),
AggOp::Sum => b.add_col_from(&mat.partial_group_sum(&g).map_err(OKIError::Io)?),
AggOp::Min => b.add_col_from(&mat.partial_group_min(&g).map_err(OKIError::Io)?),
AggOp::Max => b.add_col_from(&mat.partial_group_max(&g).map_err(OKIError::Io)?),
AggOp::Any => {
b.add_col_from_bit(&mat.partial_group_any(&g, 1).map_err(SKError::Io)?)
b.add_col_from_bit(&mat.partial_group_any(&g, 1).map_err(OKIError::Io)?)
}
AggOp::All => {
b.add_col_from_bit(&mat.partial_group_all(&g, 1).map_err(SKError::Io)?)
b.add_col_from_bit(&mat.partial_group_all(&g, 1).map_err(OKIError::Io)?)
}
AggOp::None => {
b.add_col_from_bit(&mat.partial_group_none(&g, 1).map_err(SKError::Io)?)
b.add_col_from_bit(&mat.partial_group_none(&g, 1).map_err(OKIError::Io)?)
}
}
.map_err(SKError::Io)?;
.map_err(OKIError::Io)?;
}
}
}
@@ -185,7 +174,7 @@ impl KmerIndex {
threshold: u32,
output_presence: bool,
in_place: bool,
) -> SKResult<()> {
) -> OKIResult<()> {
let src_index_dir = src.index_dir(i);
if !src_index_dir.exists() {
return Ok(());
@@ -222,11 +211,11 @@ impl KmerIndex {
// Determine number of slots and detect implicit layers.
let n = if counts_dir.exists() {
PersistentCompactIntMatrix::open(&src_layer_dir)
.map_err(SKError::Io)?
.map_err(OKIError::Io)?
.n()
} else if presence_dir.exists() {
PersistentBitMatrix::open(&src_layer_dir)
.map_err(SKError::Io)?
.map_err(OKIError::Io)?
.n()
} else {
// Implicit single-genome layer: no data matrix needed in output either.
@@ -255,7 +244,7 @@ impl KmerIndex {
let (mut dst_bit, mut dst_int) = if output_presence {
(
Some(PersistentBitMatrixBuilder::new(n, &dst_data_dir).map_err(SKError::Io)?),
Some(PersistentBitMatrixBuilder::new(n, &dst_data_dir).map_err(OKIError::Io)?),
None,
)
} else {
@@ -263,7 +252,7 @@ impl KmerIndex {
None,
Some(
PersistentCompactIntMatrixBuilder::new(n, &dst_data_dir)
.map_err(SKError::Io)?,
.map_err(OKIError::Io)?,
),
)
};
@@ -279,9 +268,9 @@ impl KmerIndex {
)?;
if output_presence {
dst_bit.unwrap().close().map_err(SKError::Io)?;
dst_bit.unwrap().close().map_err(OKIError::Io)?;
} else {
dst_int.unwrap().close().map_err(SKError::Io)?;
dst_int.unwrap().close().map_err(OKIError::Io)?;
}
// In-place: swap old data dir for new.
@@ -300,8 +289,7 @@ impl KmerIndex {
if !in_place {
src.partition_meta(i)?
.save(&dst_index_dir)
.map_err(olm_to_sk)?;
.save(&dst_index_dir)?;
}
Ok(())
src/obikstats/Cargo.toml
+9
View File
@@ -0,0 +1,9 @@
[package]
name = "obikstats"
version = "0.1.0"
edition = "2024"
[dependencies]
obikindex = { path = "../obikindex" }
obicompactvec = { path = "../obicompactvec" }
rayon = "1"
src/obikstats/src/lib.rs
+10
View File
@@ -0,0 +1,10 @@
//! Read-only aggregation/statistics over an already-built
//! `obikindex::KmerIndex`: bits-per-kmer breakdown by index component
//! (`bits_per_kmer`), per-genome k-mer counts (`genome_kmer_counts`).
//! Not part of the `Index { Partition { Layer } }` data model itself —
//! kept out of `obikindex` like every other read/write extension
//! (`obikquery`, `obikdump`, `obikselect`, `obikrebuild`, `obikmerge`).
mod stats;
pub use stats::IndexBitsPerKmer;
src/obikindex/src/index/stats.rs → src/obikstats/src/stats.rs
+7 -6
View File
@@ -5,8 +5,8 @@ use obicompactvec::{LayerMeta, PersistentBitMatrix, PersistentCompactIntMatrix};
use obicompactvec::traits::ColumnWeights;
use rayon::prelude::*;
use crate::index::error::{OKIError, OKIResult};
use crate::index::kmer_index::KmerIndex;
use obikindex::{OKIError, OKIResult};
use obikindex::KmerIndex;
/// Bits per kmer broken down by index component.
pub struct IndexBitsPerKmer {
@@ -95,7 +95,8 @@ impl KmerIndex {
let n_layers = self.n_layers(i).unwrap_or(0);
(0..n_layers).fold((0usize, 0u64, 0u64, 0u64), |acc, l| {
let lb = layer_bytes(&self.layer_dir(i, l));
let Ok(layer_dir) = self.layer_dir(i, l) else { return acc };
let lb = layer_bytes(&layer_dir);
(acc.0 + lb.n_kmers, acc.1 + lb.mphf, acc.2 + lb.evidence, acc.3 + lb.matrix)
})
})
@@ -144,7 +145,7 @@ impl KmerIndex {
let n_layers = self.n_layers(i).unwrap_or(0);
for l in 0..n_layers {
let this_layer_dir = self.layer_dir(i, l);
let Ok(this_layer_dir) = self.layer_dir(i, l) else { continue };
if !this_layer_dir.exists() { continue; }
n_kmers += LayerMeta::load(&this_layer_dir).map(|m| m.n).unwrap_or(0);
@@ -153,12 +154,12 @@ impl KmerIndex {
if this_layer_dir.join("counts").exists()
&& !this_layer_dir.join("presence").exists()
{
match crate::layer::open_data::<PersistentCompactIntMatrix>(&index_dir, l) {
match obikindex::layer::open_data::<PersistentCompactIntMatrix>(&index_dir, l) {
Ok(m) => Box::new(m),
Err(_) => continue,
}
} else {
match crate::layer::open_data::<PersistentBitMatrix>(&index_dir, l) {
match obikindex::layer::open_data::<PersistentBitMatrix>(&index_dir, l) {
Ok(m) => Box::new(m),
Err(_) => continue,
}
src/obisys/Cargo.toml
+11 -4
View File
@@ -4,7 +4,14 @@ version = "0.1.0"
edition = "2024"
[dependencies]
libc = "0.2"
sysinfo = "0.39"
indicatif = "0.18"
tracing = "0.1"
libc = "0.2"
sysinfo = "0.39"
indicatif = "0.18"
tracing = "0.1"
crossbeam-channel = "0.5"
rayon = "1"
hwlocality = { version = "1.0.0-alpha.11", features = ["vendored"], optional = true }
[features]
default = ["numa"]
numa = ["hwlocality"]
src/obisys/src/lib.rs
+2
View File
@@ -3,12 +3,14 @@
mod budget;
mod lock;
mod numa;
mod progress;
mod resources;
mod stage;
pub use budget::MemoryBudget;
pub use lock::DirLock;
pub use numa::PartitionRunner;
pub use progress::{Progress, TracedBar, progress_bar, spinner};
pub use resources::{CpuSample, IoSample, available_memory_bytes, effective_parallelism, peak_rss_bytes};
pub use stage::{Reporter, Stage, StageStats};
src/obikindex/src/index/numa/mod.rs → src/obisys/src/numa/mod.rs
View File
src/obikindex/src/index/numa/runner.rs → src/obisys/src/numa/runner.rs
+1 -1
View File
@@ -2,7 +2,7 @@ use std::sync::Arc;
use std::time::{Duration, Instant};
use crossbeam_channel::unbounded;
use obisys::{CpuSample, IoSample};
use crate::{CpuSample, IoSample};
use tracing::debug;
use super::topology::{build, pin_current_thread};
src/obikindex/src/index/numa/topology.rs → src/obisys/src/numa/topology.rs
+2 -2
View File
@@ -66,7 +66,7 @@ pub fn build() -> NumaSetup {
}
// UMA fallback: single synthetic node, all cores, no pool, no pinning.
let n_cores = obisys::effective_parallelism();
let n_cores = crate::effective_parallelism();
debug!("UMA: single synthetic node, {} core(s)", n_cores);
NumaSetup {
pools: vec![None],
@@ -76,7 +76,7 @@ pub fn build() -> NumaSetup {
#[cfg(not(feature = "numa"))]
pub fn build() -> NumaSetup {
let n_cores = obisys::effective_parallelism();
let n_cores = crate::effective_parallelism();
debug!("UMA: single synthetic node, {} core(s)", n_cores);
NumaSetup {
pools: vec![None],