OBIKmers/obikmer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 34 Wiki Activity

feat: enable index resumption and enforce directory creation

Browse Source 
The command now supports reopening existing indexes instead of failing when the output file exists. Control flow branches between opening an existing index and constructing a new one, moving configuration setup exclusively to the creation path. Directory existence is enforced upfront with proper I/O error propagation. The --force flag retains its original semantics by removing the target directory before proceeding with a fresh build.
This commit is contained in:
Eric Coissac
2026-08-21 05:06:38 +02:00
parent abc51c2add
commit 5c1584967f
161 changed files with 5274 additions and 845 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Cargo.lock
Generated
+4 -84
View File
@@ -1515,24 +1515,6 @@ dependencies = [
"xxhash-rust",
]
[[package]]
name = "obikderep"
version = "0.1.0"
dependencies = [
"niffler",
"obikindex",
"obikrope",
"obikseq",
"obilayeredmap",
"obiskbuilder",
"obiskio",
"obisys",
"rayon",
"sysinfo",
"tempfile",
"tracing",
]
[[package]]
name = "obikentropy"
version = "0.1.0"
@@ -1545,6 +1527,7 @@ name = "obikindex"
version = "0.1.0"
dependencies = [
"anyhow",
"bitvec",
"cacheline-ef",
"crossbeam-channel",
"epserde",
@@ -1556,11 +1539,11 @@ dependencies = [
"obicompactvec",
"obidebruinj",
"obikentropy",
"obikpartition",
"obikrope",
"obikseq",
"obilayeredmap",
"obipipeline",
"obiread",
"obiskbuilder",
"obiskio",
"obisys",
"obitaxonomy",
@@ -1568,6 +1551,7 @@ dependencies = [
"rayon",
"serde",
"serde_json",
"sysinfo",
"tempfile",
"tracing",
"tracing-subscriber",
@@ -1583,13 +1567,10 @@ dependencies = [
"kodama",
"obidebruinj",
"obifastwrite",
"obikderep",
"obikindex",
"obikpartitionner",
"obikphylo",
"obikrope",
"obikseq",
"obilayeredmap",
"obipipeline",
"obiread",
"obiskbuilder",
@@ -1606,45 +1587,6 @@ dependencies = [
"tracing-subscriber",
]
[[package]]
name = "obikpartition"
version = "0.1.0"
dependencies = [
"obicompactvec",
"obikseq",
"obilayeredmap",
"obiskio",
"tempfile",
]
[[package]]
name = "obikpartitionner"
version = "0.1.0"
dependencies = [
"cacheline-ef",
"epserde",
"memmap2",
"niffler",
"obicompactvec",
"obikderep",
"obikindex",
"obikrope",
"obikseq",
"obilayeredmap",
"obipipeline",
"obiread",
"obiskbuilder",
"obiskio",
"obisys",
"ptr_hash",
"rayon",
"serde",
"serde_json",
"sysinfo",
"tempfile",
"tracing",
]
[[package]]
name = "obikphylo"
version = "0.1.0"
@@ -1652,11 +1594,8 @@ dependencies = [
"memmap2",
"ndarray",
"obicompactvec",
"obikderep",
"obikindex",
"obikpartitionner",
"obikseq",
"obilayeredmap",
"obipipeline",
"obiread",
"obiskbuilder",
@@ -1688,25 +1627,6 @@ dependencies = [
"xxhash-rust",
]
[[package]]
name = "obilayeredmap"
version = "0.1.0"
dependencies = [
"bitvec",
"cacheline-ef",
"epserde",
"memmap2",
"ndarray",
"obicompactvec",
"obikseq",
"obiskio",
"ptr_hash",
"rayon",
"serde",
"serde_json",
"tempfile",
]
[[package]]
name = "obipipeline"
version = "0.1.0"
src/Cargo.toml
+1 -1
View File
@@ -1,5 +1,5 @@
[workspace]
resolver = "3"
members = ["obikseq", "obiread", "obiskbuilder", "obifastwrite", "obikmer","obikrope","obipipeline", "obikpartitionner","obiskio","obidebruinj","obilayeredmap", "obicompactvec", "obisys", "obikindex", "obitaxonomy", "obikentropy", "obikphylo", "obikpartition", "obikderep"]
members = ["obikseq", "obiread", "obiskbuilder", "obifastwrite", "obikmer","obikrope","obipipeline", "obiskio","obidebruinj", "obicompactvec", "obisys", "obikindex", "obitaxonomy", "obikentropy", "obikphylo"]
[profile.release]
debug = 1
src/obicompactvec/src/bitmatrix/packed.rs
+1 -1
View File
@@ -129,7 +129,7 @@ pub fn pack_bit_matrix(dir: &Path) -> io::Result<()> {
// A `matrix.pbmx` can already exist here even though columnar data is
// still pending — e.g. copied verbatim from a merge's base source
// before this layer was widened with more genome columns (see
// `obikpartitionner::merge_partition`). Only skip (re-)packing if the
// `obikindex::index::merge_layer::merge_partition`). Only skip (re-)packing if the
// existing file already reflects the current column count; otherwise
// the columnar files are newer and must be (re-)packed, overwriting the
// stale one — never silently discarded as "leftover cleanup".
src/obicompactvec/src/intmatrix.rs
+1 -1
View File
@@ -232,7 +232,7 @@ pub fn pack_compact_int_matrix(dir: &Path) -> io::Result<()> {
// A `matrix.pcmx` can already exist here even though columnar data is
// still pending — e.g. copied verbatim from a merge's base source
// before this layer was widened with more genome columns (see
// `obikpartitionner::merge_partition`). Only skip (re-)packing if the
// `obikindex::index::merge_layer::merge_partition`). Only skip (re-)packing if the
// existing file already reflects the current column count; otherwise
// the columnar files are newer and must be (re-)packed, overwriting the
// stale one — never silently discarded as "leftover cleanup".
src/obicompactvec/src/layer_meta.rs
+1 -1
View File
@@ -2,7 +2,7 @@ use std::{fs, io, path::Path};
/// Lightweight metadata stored at the layer level (`layer_meta.json`).
///
/// Written by `obilayeredmap::MphfLayer::build` alongside `mphf.bin`.
/// Written by `obikindex::layer::MphfLayer::build` alongside `mphf.bin`.
/// Read by `PersistentBitMatrix::open` to determine `n_rows` for the
/// implicit (mono-genome presence/absence) case.
pub struct LayerMeta {
src/obicompactvec/src/storage_kind.rs
+1 -1
View File
@@ -1,5 +1,5 @@
//! How a persistent matrix's data is physically laid out on disk —
//! orthogonal to *what* it stores (count vs. presence, `obilayeredmap`'s
//! orthogonal to *what* it stores (count vs. presence, `obikindex::layer`'s
//! `LayerContent` concern, one crate up). `PersistentCompactIntMatrix` only
//! ever reports `Columnar`/`Packed`; `PersistentBitMatrix` is the only type
//! that can also report `Sparse`/`Implicit`.
src/obikderep/Cargo.toml
-21
View File
@@ -1,21 +0,0 @@
[package]
name = "obikderep"
version = "0.1.0"
edition = "2024"
[dependencies]
niffler = "3.0.0"
obikseq = { path = "../obikseq" }
obikindex = { path = "../obikindex" }
obilayeredmap = { path = "../obilayeredmap" }
obiskio = { path = "../obiskio" }
obisys = { path = "../obisys" }
rayon = "1"
sysinfo = "0.39"
tracing = "0.1.44"
[dev-dependencies]
tempfile = "3"
obikseq = { path = "../obikseq", features = ["test-utils"] }
obikrope = { path = "../obikrope" }
obiskbuilder = { path = "../obiskbuilder" }
src/obikindex/Cargo.toml
+6 -3
View File
@@ -5,15 +5,15 @@ edition = "2024"
[dependencies]
obikseq = { path = "../obikseq" }
obikpartition = { path = "../obikpartition" }
obitaxonomy = { path = "../obitaxonomy" }
obiskio = { path = "../obiskio" }
obisys = { path = "../obisys" }
obicompactvec = { path = "../obicompactvec" }
obilayeredmap = { path = "../obilayeredmap" }
obidebruinj = { path = "../obidebruinj" }
obipipeline = { path = "../obipipeline" }
obikentropy = { path = "../obikentropy" }
obiread = { path = "../obiread" }
obiskbuilder = { path = "../obiskbuilder" }
cacheline-ef = "1.1"
epserde = "0.8"
ptr_hash = "1.1"
@@ -26,10 +26,13 @@ serde = { version = "1", features = ["derive"] }
serde_json = "1"
indicatif = "0.18"
tracing = "0.1.44"
bitvec = "1"
sysinfo = "0.39"
hwlocality = { version = "1.0.0-alpha.11", features = ["vendored"], optional = true }
[dev-dependencies]
obiread = { path = "../obiread" }
obikseq = { path = "../obikseq", features = ["test-utils"] }
obikrope = { path = "../obikrope" }
tempfile = "3"
tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] }
anyhow = "1"
src/obikderep/src/dereplicate.rs → src/obikindex/src/algorithms/dereplicator/dereplicate.rs
+7 -6
View File
@@ -1,6 +1,7 @@
//! Per-partition dereplication mechanics — private to this crate.
//! [`crate::Dereplicator`] is the public entry point; this module is the
//! two-phase split+merge algorithm it runs once per partition.
//! Per-partition dereplication mechanics — private to this module.
//! [`crate::algorithms::dereplicator::Dereplicator`] is the public entry
//! point; this module is the two-phase split+merge algorithm it runs once
//! per partition.
use std::collections::HashMap;
use std::fs;
@@ -13,13 +14,13 @@ use niffler::send::compression::Format;
use niffler::Level;
use obikseq::superkmer::SuperKmer;
use obikseq::Sequence;
use obilayeredmap::{dereplicated_superkmers_path, raw_superkmers_path};
use crate::layer::{dereplicated_superkmers_path, raw_superkmers_path};
use obiskio::{SKFileMeta, SKFileReader, SKFileWriter, SKResult};
/// Scratch-file extension for this algorithm's own intermediate split
/// buckets — never read by anything outside [`dereplicate_partition`],
/// unlike `raw`/`dereplicated` (see `obilayeredmap::{raw_superkmers_path,
/// dereplicated_superkmers_path}`, the actual cross-crate contract).
/// unlike `raw`/`dereplicated` (see `crate::layer::{raw_superkmers_path,
/// dereplicated_superkmers_path}`, the actual cross-module contract).
const TEMP_EXT: &str = "skmer.zst";
/// Estimate the number of in-memory buckets needed to deduplicate the
src/obikderep/src/lib.rs → src/obikindex/src/algorithms/dereplicator/mod.rs
+11 -15
View File
@@ -1,26 +1,22 @@
//! Superkmer dereplication — the second stage of the indexing pipeline,
//! after `obikpartitionner::PartitionRouter::run` (scatter) has written
//! each partition's raw superkmer file, before
//! `obikpartitionner::PartitionRouter::count_kmer` (counting) reads the
//! result. One algorithm, one crate — see
//! `DevDocMD/implementation/partition_layer_cache.md`'s "on avance pas à
//! pas" note: `obikpartitionner` used to also own dereplication and
//! counting; this crate is step one of splitting that bundle apart,
//! deliberately one algorithm at a time rather than all at once, so a
//! shared `Algorithm` pattern can be factored out later from real
//! examples instead of guessed at up front.
//! Superkmer dereplication — the sibling algorithm to
//! [`crate::algorithms::partitionner`]: runs after
//! `partitionner::PartitionRouter::run` (scatter) has written each
//! partition's raw superkmer file, before
//! `partitionner::PartitionRouter::count_kmer` (counting) reads the
//! result — see `DevDocMD/implementation/partition_layer_cache.md`.
mod dereplicate;
use std::sync::atomic::{AtomicU64, Ordering};
use niffler::Level;
use obikindex::KmerIndex;
use obiskio::SKResult;
use obisys::Progress;
use rayon::prelude::*;
use sysinfo::System;
use crate::index::KmerIndex;
use dereplicate::{dereplicate_partition, optimal_buckets};
/// Deduplicates every partition's raw superkmer file in place, replacing
@@ -68,8 +64,8 @@ impl<'a> Dereplicator<'a> {
/// `obisys::TracedBar`'s own methods take `&self`, not `&mut self`).
/// `total: Some(n_partitions)` — known up front here, unlike
/// `PartitionRouter::run`'s bases-processed count, so the caller can
/// render an actual progress bar rather than a spinner. This crate
/// never renders anything itself — see the module docs.
/// render an actual progress bar rather than a spinner. This algorithm
/// never renders anything itself.
pub fn run(&self, on_progress: Option<impl Fn(Progress) + Sync>) -> SKResult<()> {
let level = self.level;
let sys = System::new_all();
@@ -88,7 +84,7 @@ impl<'a> Dereplicator<'a> {
.map(|i| {
let dir = self.index.layer_dir(i, 0);
let result = if dir.exists() {
let raw_path = obilayeredmap::raw_superkmers_path(&dir);
let raw_path = crate::layer::raw_superkmers_path(&dir);
let n_buckets = optimal_buckets(&raw_path, available_per_thread);
dereplicate_partition(&dir, level, n_buckets)
} else {
src/obikindex/src/algorithms/mod.rs
+9
View File
@@ -0,0 +1,9 @@
//! Indexing-pipeline algorithms: code that operates on a
//! [`crate::index::KmerIndex`] to build or transform its content, as
//! opposed to the `index`/`partition`/`layer` modules, which are the data
//! model itself. Each algorithm is its own submodule: [`partitionner`]
//! (routing raw super-kmers into partitions, then counting), [`dereplicator`]
//! (deduplicating a partition's raw super-kmers before counting).
pub mod dereplicator;
pub mod partitionner;
src/obikpartitionner/src/partition/count.rs → src/obikindex/src/algorithms/partitionner/count.rs
+1 -1
View File
@@ -11,7 +11,7 @@ use obiskio::{SKFileReader, SKResult};
use ptr_hash::{PtrHash, PtrHashParams, bucket_fn::CubicEps, hash::Xx64};
use tracing::debug;
use crate::kmer_sort::sort_unique_kmers;
use super::kmer_sort::sort_unique_kmers;
pub(super) type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xx64, Vec<u8>>;
src/obikpartitionner/src/kmer_sort.rs → src/obikindex/src/algorithms/partitionner/kmer_sort.rs
View File
src/obikindex/src/algorithms/partitionner/mod.rs
+20
View File
@@ -0,0 +1,20 @@
//! K-mer partitioning: routing super-kmers into per-partition, layer-0
//! files, and counting unique canonical k-mers. Dereplication of the raw
//! super-kmers in between is [`crate::algorithms::dereplicator`], a
//! sibling algorithm, not a step of this one — see
//! `DevDocMD/implementation/partition_layer_cache.md`.
//!
//! Submodules: [`router`] (`PartitionRouter`, `KmerSpectrum`, the
//! routing/counting lifecycle API — partition/layer path naming itself
//! lives on `crate::index::KmerIndex`, not here), [`count`] (unique-kmer
//! enumeration, MPHF, abundance counting), `kmer_sort` (external sort
//! support for `count`).
mod count;
mod kmer_sort;
mod router;
#[cfg(test)]
mod tests;
pub use router::{KmerSpectrum, PartitionRouter};
src/obikpartitionner/src/partition/router.rs → src/obikindex/src/algorithms/partitionner/router.rs
+12 -12
View File
@@ -6,9 +6,9 @@ use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
use std::sync::Arc;
use std::time::Instant;
use obikindex::KmerIndex;
use crate::index::KmerIndex;
use obikseq::RoutableSuperKmer;
use obilayeredmap::Layer;
use crate::layer::Layer;
use obiskio::SKResult;
use obisys::{progress_bar, Progress};
use rayon::prelude::*;
@@ -22,7 +22,7 @@ use obiskio::SKFileWriter;
use obipipeline::{throttle, ThrottleGuard, Throttled};
use obiread::NucPage;
use crate::kmer_sort::chunk_size_from_ram;
use super::kmer_sort::chunk_size_from_ram;
use super::count::count_partition;
@@ -73,14 +73,14 @@ impl Drop for GuardedIter {
// ── PartitionRouter ──────────────────────────────────────────────────────────
/// Routes raw super-kmers into per-partition, layer-0 files, then
/// dereplicates and counts them — this crate's entire job now that
/// layer/query/merge/select/rebuild/dump/distance concerns have moved to
/// `obikindex` (they operate on built layers, which don't exist yet at this
/// stage — see `DevDocMD/implementation/partition_layer_cache.md`).
/// dereplicates (via the sibling [`crate::algorithms::dereplicator`]
/// algorithm) and counts them — the entry point of the indexing pipeline,
/// operating on layer-0 content that doesn't exist yet at this stage (see
/// `DevDocMD/implementation/partition_layer_cache.md`).
///
/// Holds `&mut KmerIndex` — this is an algorithm operating on an index, not
/// a data structure of its own; it owns no path-naming knowledge (every
/// path comes from `index.index_dir`/`obilayeredmap::layer_dir`/
/// path comes from `index.index_dir`/`crate::layer::layer_dir`/
/// `Layer::create`), only the transient routing/dereplication/counting
/// state a run needs.
///
@@ -291,7 +291,7 @@ impl<'a> PartitionRouter<'a> {
.into_par_iter()
.map(|i| {
let dir = self.layer0_dir(i);
let dedup_path = obilayeredmap::dereplicated_superkmers_path(&dir);
let dedup_path = crate::layer::dereplicated_superkmers_path(&dir);
if !dedup_path.exists() {
pb.inc(1);
return Ok(());
@@ -343,9 +343,9 @@ impl<'a> PartitionRouter<'a> {
/// Directory of partition `i`'s layer 0 — every raw/dereplicated
/// superkmer file and provisional `mphf1.bin`/`counts1.bin` this router
/// produces lives here, alongside where `build_index_layer`
/// (`obikindex`) will later turn it into the real layer 0.
/// (`crate::index`) will later turn it into the real layer 0.
fn layer0_dir(&self, i: usize) -> PathBuf {
obilayeredmap::layer_dir(&self.index.index_dir(i), 0)
crate::layer::layer_dir(&self.index.index_dir(i), 0)
}
fn check_not_closed(&self) -> SKResult<()> {
@@ -360,7 +360,7 @@ impl<'a> PartitionRouter<'a> {
if self.writers[partition].is_none() {
let dir = self.layer0_dir(partition);
Layer::create(&dir).map_err(|e| io::Error::other(e.to_string()))?;
let file_path = obilayeredmap::raw_superkmers_path(&dir);
let file_path = crate::layer::raw_superkmers_path(&dir);
let writer = SKFileWriter::create_with(file_path, Format::Zstd, self.level)?;
self.writers[partition] = Some(writer);
}
src/obikpartitionner/src/partition/tests.rs → src/obikindex/src/algorithms/partitionner/tests.rs
+4 -4
View File
@@ -1,11 +1,11 @@
use std::collections::HashMap;
use std::fs;
use obikderep::Dereplicator;
use obikindex::{IndexConfig, KmerIndex};
use crate::algorithms::dereplicator::Dereplicator;
use crate::{IndexConfig, KmerIndex};
use crate::layer::IndexMode;
use obikrope::Rope;
use obikseq::SuperKmer;
use obilayeredmap::IndexMode;
use obiskbuilder::build_superkmers;
use super::count::count_partition;
@@ -69,7 +69,7 @@ fn pipeline_counts(seqs: &[&[u8]]) -> (u64, u64) {
Dereplicator::new(&index).run(None::<fn(obisys::Progress)>).unwrap();
let part_dir = index.layer_dir(0, 0);
let dedup_path = obilayeredmap::dereplicated_superkmers_path(&part_dir);
let dedup_path = crate::layer::dereplicated_superkmers_path(&part_dir);
if !dedup_path.exists() {
return (0, 0);
}
src/obikindex/src/common.rs → src/obikindex/src/index/common.rs
+2 -2
View File
@@ -1,8 +1,8 @@
use std::path::Path;
use obicompactvec::{PersistentBitVecBuilder, PersistentCompactIntVecBuilder};
use obilayeredmap::meta::PartitionMeta;
use obilayeredmap::{layer_dir, IndexMode, OLMError};
use crate::layer::meta::PartitionMeta;
use crate::layer::{layer_dir, IndexMode, OLMError};
use obiskio::{SKError, SKResult};
// ── olm_to_sk ────────────────────────────────────────────────────────────────
src/obikindex/src/distance.rs → src/obikindex/src/index/distance.rs
+3 -3
View File
@@ -1,10 +1,10 @@
use ndarray::Array2;
use obicompactvec::traits::{BitPartials, CountPartials};
use obilayeredmap::LayeredStore;
use crate::layer::LayeredStore;
use rayon::prelude::*;
use crate::error::{OKIError, OKIResult};
use crate::index::KmerIndex;
use crate::index::error::{OKIError, OKIResult};
use crate::index::kmer_index::KmerIndex;
// ── Public API ────────────────────────────────────────────────────────────────
src/obikindex/src/dump.rs → src/obikindex/src/index/dump.rs
+3 -3
View File
@@ -3,9 +3,9 @@ use std::sync::atomic::{AtomicUsize, Ordering};
use rayon::prelude::*;
use crate::error::{OKIError, OKIResult};
use crate::index::KmerIndex;
use crate::KmerFilter;
use crate::index::error::{OKIError, OKIResult};
use crate::index::kmer_index::KmerIndex;
use crate::index::KmerFilter;
impl KmerIndex {
/// Write a CSV table of all indexed kmers to `out`.
src/obikindex/src/dump_layer.rs → src/obikindex/src/index/dump_layer.rs
+3 -3
View File
@@ -1,10 +1,10 @@
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
use obikseq::CanonicalKmer;
use obilayeredmap::{IndexMode, MphfLayer, OLMError};
use crate::layer::{IndexMode, MphfLayer, OLMError};
use obiskio::{SKError, SKResult, UnitigFileReader};
use crate::filter::{KmerFilter, passes_all};
use crate::index::KmerIndex;
use crate::index::filter::{KmerFilter, passes_all};
use crate::index::kmer_index::KmerIndex;
fn olm_to_sk(e: OLMError) -> SKError {
match e {
src/obikindex/src/error.rs → src/obikindex/src/index/error.rs
View File
src/obikindex/src/filter.rs → src/obikindex/src/index/filter.rs
View File
src/obikindex/src/graph_pipeline.rs → src/obikindex/src/index/graph_pipeline.rs
+2 -2
View File
@@ -10,10 +10,10 @@ use obipipeline::{
use obidebruinj::GraphDeBruijn;
use obikseq::CanonicalKmer;
use obilayeredmap::{IndexMode, TypedLayer};
use crate::layer::{IndexMode, TypedLayer};
use obiskio::{SKError, SKResult};
use crate::common::olm_to_sk;
use crate::index::common::olm_to_sk;
// ── KmerGraphData ─────────────────────────────────────────────────────────────
src/obikindex/src/index_layer.rs → src/obikindex/src/index/index_layer.rs
+7 -7
View File
@@ -5,14 +5,14 @@ use cacheline_ef::{CachelineEf, CachelineEfVec};
use epserde::prelude::*;
use obicompactvec::{PersistentCompactIntMatrix, PersistentCompactIntVec};
use obidebruinj::GraphDeBruijn;
use obilayeredmap::meta::PartitionMeta;
use obilayeredmap::{IndexMode, layer::TypedLayer};
use crate::layer::meta::PartitionMeta;
use crate::layer::{IndexMode, TypedLayer};
use obiskio::{SKError, SKFileMeta, SKFileReader};
use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xx64};
use crate::common::olm_to_sk;
use crate::graph_pipeline::{materialize_layer, write_graph_as_unitigs};
use crate::index::KmerIndex;
use crate::index::common::olm_to_sk;
use crate::index::graph_pipeline::{materialize_layer, write_graph_as_unitigs};
use crate::index::kmer_index::KmerIndex;
type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xx64, Vec<u8>>;
@@ -43,7 +43,7 @@ impl KmerIndex {
block_bits: u8,
) -> Result<usize, SKError> {
let layer0_dir = self.layer_dir(i, 0);
let dedup_path = obilayeredmap::dereplicated_superkmers_path(&layer0_dir);
let dedup_path = crate::layer::dereplicated_superkmers_path(&layer0_dir);
if !dedup_path.exists() {
return Ok(0);
}
@@ -123,7 +123,7 @@ impl KmerIndex {
/// Deletes `dereplicated.skmer.zst` (+ sidecar), `mphf1.bin`, `counts1.bin`.
pub fn remove_build_artifacts(&self, i: usize) {
let layer0_dir = self.layer_dir(i, 0);
let dedup = obilayeredmap::dereplicated_superkmers_path(&layer0_dir);
let dedup = crate::layer::dereplicated_superkmers_path(&layer0_dir);
remove_if_exists(&SKFileMeta::sidecar_path(&dedup));
remove_if_exists(&dedup);
remove_if_exists(&layer0_dir.join("mphf1.bin"));
src/obikindex/src/index.rs → src/obikindex/src/index/kmer_index.rs
+31 -28
View File
@@ -2,17 +2,17 @@ use std::collections::BTreeMap;
use std::fs;
use std::path::{Path, PathBuf};
use obilayeredmap::meta::PartitionMeta;
use crate::layer::meta::PartitionMeta;
use obisys::{Reporter, Stage, progress_bar};
use rayon::prelude::*;
use tracing::info;
use obikseq::{set_k, set_m};
use crate::common::load_meta;
use crate::error::{OKIError, OKIResult};
use crate::meta::{GenomeInfo, IndexConfig, IndexMeta};
use crate::state::{IndexState, SENTINEL_COUNTED, SENTINEL_INDEXED, SENTINEL_SCATTERED};
use crate::index::common::load_meta;
use crate::index::error::{OKIError, OKIResult};
use crate::index::meta::{GenomeInfo, IndexConfig, IndexMeta};
use crate::index::state::{IndexState, SENTINEL_COUNTED, SENTINEL_INDEXED, SENTINEL_SCATTERED};
pub struct KmerIndex {
pub(crate) root_path: PathBuf,
@@ -30,6 +30,7 @@ impl KmerIndex {
genome_info: Option<GenomeInfo>,
) -> OKIResult<Self> {
let root_path = path.as_ref().to_owned();
fs::create_dir_all(&root_path).map_err(OKIError::Io)?;
set_k(config.kmer_size);
set_m(config.minimizer_size);
let mut meta = IndexMeta::new(config);
@@ -130,7 +131,7 @@ impl KmerIndex {
pub fn with_counts(&self) -> bool {
self.meta.config.with_counts
}
pub fn evidence_mode(&self) -> &obilayeredmap::IndexMode {
pub fn evidence_mode(&self) -> &crate::layer::IndexMode {
&self.meta.config.evidence
}
/// Coarse evidence mode (`Exact`/`Approx`/`Hybrid`, no `b`/`z`
@@ -138,11 +139,11 @@ impl KmerIndex {
/// full `IndexMode`. Same discriminant as `Layer::evidence_kind`, so a
/// layer's own evidence kind can be compared directly against the
/// index's.
pub fn evidence_kind(&self) -> obilayeredmap::EvidenceKind {
pub fn evidence_kind(&self) -> crate::layer::EvidenceKind {
match self.meta.config.evidence {
obilayeredmap::IndexMode::Exact => obilayeredmap::EvidenceKind::Exact,
obilayeredmap::IndexMode::Approx { .. } => obilayeredmap::EvidenceKind::Approx,
obilayeredmap::IndexMode::Hybrid { .. } => obilayeredmap::EvidenceKind::Hybrid,
crate::layer::IndexMode::Exact => crate::layer::EvidenceKind::Exact,
crate::layer::IndexMode::Approx { .. } => crate::layer::EvidenceKind::Approx,
crate::layer::IndexMode::Hybrid { .. } => crate::layer::EvidenceKind::Hybrid,
}
}
pub fn block_bits(&self) -> u8 {
@@ -156,31 +157,32 @@ impl KmerIndex {
}
/// Path of partition `i`'s raw directory (`partitions/part_{i:05}`) —
/// delegates to `obikpartition`, the Partition tier's own naming
/// primitive (mirrors `layer_dir` delegating to `obilayeredmap`).
/// `obikpartitionner::PartitionRouter` reaches this same directory
/// only indirectly, through this method (it depends on `KmerIndex`,
/// not the other way around — see
/// `DevDocMD/implementation/partition_layer_cache.md`).
/// delegates to `crate::partition`, the Partition tier's own naming
/// primitive (mirrors `layer_dir` delegating to `crate::layer`).
/// `crate::algorithms::partitionner::PartitionRouter` reaches this same
/// directory only indirectly, through this method — it depends on
/// `KmerIndex`, not the other way around, even though both now live in
/// this crate — see `DevDocMD/implementation/partition_layer_cache.md`.
pub fn partition_dir(&self, i: usize) -> PathBuf {
obikpartition::partition_dir(&self.root_path, i)
crate::partition::partition_dir(&self.root_path, i)
}
/// Path of partition `i`'s layered-index directory (`<partition>/index`).
pub fn index_dir(&self, i: usize) -> PathBuf {
obikpartition::index_dir(&self.root_path, i)
crate::partition::index_dir(&self.root_path, i)
}
/// Path of layer `l` within partition `i`'s layered index.
pub fn layer_dir(&self, i: usize, l: usize) -> PathBuf {
obilayeredmap::layer_dir(&self.index_dir(i), l)
crate::layer::layer_dir(&self.index_dir(i), l)
}
/// Partition `i`'s metadata (layer count, evidence mode). Returns
/// `obiskio::SKResult`, not `OKIResult` — matches the error convention
/// of the partition/layer-construction code below (moved here from
/// `obikpartitionner`, which predates `OKIError`); `?` still converts
/// it to `OKIResult` at any call site that needs one (`OKIError: From<SKError>`).
/// the former `obikpartitionner` crate, which predates `OKIError`);
/// `?` still converts it to `OKIResult` at any call site that needs
/// one (`OKIError: From<SKError>`).
pub fn partition_meta(&self, i: usize) -> obiskio::SKResult<PartitionMeta> {
load_meta(&self.index_dir(i), "partition_meta")
}
@@ -195,7 +197,7 @@ impl KmerIndex {
/// [`evidence_mode`](Self::evidence_mode): that one is the index-level
/// config, this one is per-partition ground truth (the two agree in
/// practice, but this is what `Layer::open` needs).
pub fn partition_mode(&self, i: usize) -> obiskio::SKResult<obilayeredmap::IndexMode> {
pub fn partition_mode(&self, i: usize) -> obiskio::SKResult<crate::layer::IndexMode> {
Ok(self.partition_meta(i)?.mode)
}
@@ -230,10 +232,11 @@ impl KmerIndex {
/// Write `spectrums/{label}.json` from an already-computed kmer
/// spectrum (`f0`/`f1`/abundance histogram). Takes plain values rather
/// than `obikpartitionner::KmerSpectrum` — `KmerIndex` cannot depend on
/// `obikpartitionner` (that dependency runs the other way, see
/// `DevDocMD/implementation/partition_layer_cache.md`), and doesn't
/// need to: this is the only field of that type it actually uses.
/// than `crate::algorithms::partitionner::KmerSpectrum` — `KmerIndex`
/// is the data model, `PartitionRouter` the algorithm that depends on
/// it (see `DevDocMD/implementation/partition_layer_cache.md`), not
/// the other way around, and this is the only field of that type it
/// actually uses.
pub fn write_spectrum(&self, f0: u64, f1: u64, counts: &BTreeMap<u32, u64>) -> OKIResult<()> {
let label = self
.meta
@@ -276,7 +279,7 @@ impl KmerIndex {
let pb = progress_bar("index", n as u64, "partitions");
let order: Vec<usize> = (0..n).collect();
let runner = crate::numa::PartitionRunner::new();
let runner = crate::index::numa::PartitionRunner::new();
runner
.run(
&order,
@@ -335,7 +338,7 @@ impl KmerIndex {
let n = self.n_partitions();
let order: Vec<usize> = (0..n).collect();
let pb = progress_bar("pack", n as u64, "partitions");
crate::numa::PartitionRunner::new().run(
crate::index::numa::PartitionRunner::new().run(
&order,
|i| -> OKIResult<()> {
let index_dir = self.index_dir(i);
src/obikindex/src/matrix_store.rs → src/obikindex/src/index/matrix_store.rs
+3 -3
View File
@@ -1,9 +1,9 @@
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
use obilayeredmap::{LayeredStore, open_data};
use crate::layer::{LayeredStore, open_data};
use obiskio::SKResult;
use crate::common::{load_meta, olm_to_sk};
use crate::index::KmerIndex;
use crate::index::common::{load_meta, olm_to_sk};
use crate::index::kmer_index::KmerIndex;
impl KmerIndex {
/// Open all count matrices for partition `part`, one per layer.
src/obikindex/src/merge.rs → src/obikindex/src/index/merge.rs
+7 -7
View File
@@ -6,14 +6,14 @@ use std::path::Path;
use obisys::{Reporter, Stage, progress_bar, spinner};
use tracing::{debug, info};
use obilayeredmap::IndexMode;
use crate::layer::IndexMode;
use crate::error::{OKIError, OKIResult};
use crate::index::KmerIndex;
use crate::meta::{GenomeInfo, IndexMeta};
use crate::state::{IndexState, SENTINEL_INDEXED};
use crate::index::error::{OKIError, OKIResult};
use crate::index::kmer_index::KmerIndex;
use crate::index::meta::{GenomeInfo, IndexMeta};
use crate::index::state::{IndexState, SENTINEL_INDEXED};
pub use crate::merge_layer::MergeMode;
pub use crate::index::merge_layer::MergeMode;
// ── per-partition diagnostic record ──────────────────────────────────────────
@@ -217,7 +217,7 @@ impl KmerIndex {
let srcs = &srcs;
let evidence = &evidence;
let runner = crate::numa::PartitionRunner::new();
let runner = crate::index::numa::PartitionRunner::new();
let mut part_stats: Vec<PartStat> = Vec::with_capacity(n_partitions);
runner
src/obikindex/src/merge_layer/mod.rs → src/obikindex/src/index/merge_layer/mod.rs
+4 -4
View File
@@ -20,12 +20,12 @@ use tracing::debug;
use obicompactvec::{PersistentBitMatrixBuilder, PersistentCompactIntMatrixBuilder};
use obikseq::CanonicalKmer;
use obilayeredmap::{IndexMode, TypedLayer, LayeredMap, MphfOnly, layer_dir};
use crate::layer::{IndexMode, TypedLayer, LayeredMap, MphfOnly, layer_dir};
use obiskio::{SKError, SKResult, UnitigFileReader};
use crate::common::{ColBuilder, load_meta, olm_to_sk};
use crate::graph_pipeline::{build_graph, materialize_layer};
use crate::index::KmerIndex;
use crate::index::common::{ColBuilder, load_meta, olm_to_sk};
use crate::index::graph_pipeline::{build_graph, materialize_layer};
use crate::index::kmer_index::KmerIndex;
mod src_layer;
src/obikindex/src/merge_layer/src_layer.rs → src/obikindex/src/index/merge_layer/src_layer.rs
+2 -2
View File
@@ -2,10 +2,10 @@ use std::path::Path;
use obicompactvec::{MatrixGroupOps, PersistentBitMatrix, PersistentCompactIntMatrix};
use obikseq::CanonicalKmer;
use obilayeredmap::MphfOnly;
use crate::layer::MphfOnly;
use obiskio::{SKError, SKResult};
use crate::common::olm_to_sk;
use crate::index::common::olm_to_sk;
use super::MergeMode;
src/obikindex/src/meta.rs → src/obikindex/src/index/meta.rs
+1 -1
View File
@@ -3,7 +3,7 @@ use std::fs;
use std::io;
use std::path::Path;
use obilayeredmap::IndexMode;
use crate::layer::IndexMode;
use serde::{Deserialize, Serialize};
pub const META_FILENAME: &str = "index.meta";
src/obikindex/src/index/mod.rs
+36
View File
@@ -0,0 +1,36 @@
pub mod error;
pub mod meta;
pub mod predicate;
pub mod state;
mod common;
mod distance;
mod dump;
mod dump_layer;
pub mod filter;
mod graph_pipeline;
mod kmer_index;
mod index_layer;
mod matrix_store;
mod merge;
mod merge_layer;
mod numa;
mod query_layer;
mod rebuild;
mod rebuild_layer;
mod reindex;
mod select;
mod select_layer;
mod stats;
pub use error::{OKIError, OKIResult};
pub use distance::{DistanceMetric, DistanceOutput};
pub use filter::{GroupQuorumFilter, KmerFilter, passes_all};
pub use kmer_index::KmerIndex;
pub use merge_layer::MergeMode;
pub use meta::{validate_label, GenomeInfo, IndexConfig, IndexMeta, META_FILENAME};
pub use predicate::{GroupFilterParams, MetaPred};
pub use query_layer::{KmerDesc, QueryHit, QueryStats};
pub use select_layer::{AggOp, OutputCol};
pub use state::{IndexState, SENTINEL_COUNTED, SENTINEL_INDEXED, SENTINEL_SCATTERED};
pub use stats::IndexBitsPerKmer;
pub use numa::PartitionRunner;
src/obikindex/src/numa/mod.rs → src/obikindex/src/index/numa/mod.rs
View File
src/obikindex/src/numa/runner.rs → src/obikindex/src/index/numa/runner.rs
View File
src/obikindex/src/numa/topology.rs → src/obikindex/src/index/numa/topology.rs
View File
src/obikindex/src/predicate.rs → src/obikindex/src/index/predicate.rs
+2 -2
View File
@@ -1,9 +1,9 @@
use std::collections::HashMap;
use crate::GroupQuorumFilter;
use crate::index::GroupQuorumFilter;
use obitaxonomy::{TaxPath, TaxPattern};
use crate::meta::{GenomeInfo, IndexMeta};
use crate::index::meta::{GenomeInfo, IndexMeta};
// ── Operator ──────────────────────────────────────────────────────────────────
src/obikindex/src/progress.rs → src/obikindex/src/index/progress.rs
View File
src/obikindex/src/query_layer.rs → src/obikindex/src/index/query_layer.rs
+2 -2
View File
@@ -3,10 +3,10 @@ use std::path::Path;
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
use obikseq::CanonicalKmer;
use obilayeredmap::{IndexMode, MphfLayer, OLMError};
use crate::layer::{IndexMode, MphfLayer, OLMError};
use obiskio::{SKError, SKResult};
use crate::index::KmerIndex;
use crate::index::kmer_index::KmerIndex;
fn olm_to_sk(e: OLMError) -> SKError {
match e {
src/obikindex/src/rebuild.rs → src/obikindex/src/index/rebuild.rs
+6 -6
View File
@@ -1,13 +1,13 @@
use std::path::Path;
use crate::{KmerFilter, MergeMode};
use crate::index::{KmerFilter, MergeMode};
use obisys::{Reporter, Stage, progress_bar};
use tracing::info;
use crate::error::{OKIError, OKIResult};
use crate::index::KmerIndex;
use crate::meta::IndexMeta;
use crate::state::IndexState;
use crate::index::error::{OKIError, OKIResult};
use crate::index::kmer_index::KmerIndex;
use crate::index::meta::IndexMeta;
use crate::index::state::IndexState;
impl KmerIndex {
/// Rebuild `src` into a new compact single-layer index at `output`.
@@ -62,7 +62,7 @@ impl KmerIndex {
let block_bits = meta.config.block_bits;
let order: Vec<usize> = (0..n_partitions).collect();
let runner = crate::numa::PartitionRunner::new();
let runner = crate::index::numa::PartitionRunner::new();
runner.run(
&order,
|i| dst_partition.rebuild_partition(src, i, filters, mode, n_genomes, block_bits),
src/obikindex/src/rebuild_layer.rs → src/obikindex/src/index/rebuild_layer.rs
+7 -7
View File
@@ -6,15 +6,15 @@ use obicompactvec::{
};
use obidebruinj::GraphDeBruijn;
use obikseq::CanonicalKmer;
use obilayeredmap::meta::PartitionMeta;
use obilayeredmap::{IndexMode, MphfLayer, layer_dir};
use crate::layer::meta::PartitionMeta;
use crate::layer::{IndexMode, MphfLayer, layer_dir};
use obiskio::{SKError, SKResult, UnitigFileReader};
use crate::common::{load_meta, olm_to_sk};
use crate::filter::KmerFilter;
use crate::graph_pipeline::materialize_layer;
use crate::merge_layer::{MergeMode, SrcLayerData};
use crate::index::KmerIndex;
use crate::index::common::{load_meta, olm_to_sk};
use crate::index::filter::KmerFilter;
use crate::index::graph_pipeline::materialize_layer;
use crate::index::merge_layer::{MergeMode, SrcLayerData};
use crate::index::kmer_index::KmerIndex;
// ── Builders — pair matrix builder + column builders for one mode ─────────────
src/obikindex/src/reindex.rs → src/obikindex/src/index/reindex.rs
+6 -6
View File
@@ -1,18 +1,18 @@
use obilayeredmap::{IndexMode, layer::TypedLayer};
use crate::layer::{IndexMode, TypedLayer};
use obisys::{Reporter, Stage, progress_bar};
use std::fs;
use std::path::Path;
use tracing::info;
use crate::error::{OKIError, OKIResult};
use crate::index::KmerIndex;
use crate::state::IndexState;
use crate::index::error::{OKIError, OKIResult};
use crate::index::kmer_index::KmerIndex;
use crate::index::state::IndexState;
const EVIDENCE_FILE: &str = "evidence.bin";
const FINGERPRINT_FILE: &str = "fingerprint.bin";
const UNITIG_IDX_FILE: &str = "unitigs.bin.idx";
fn olm_to_oki(e: obilayeredmap::OLMError) -> OKIError {
fn olm_to_oki(e: crate::layer::OLMError) -> OKIError {
OKIError::InvalidInput(e.to_string())
}
@@ -44,7 +44,7 @@ impl KmerIndex {
let pb = progress_bar("reindex", n as u64, "partitions");
let order: Vec<usize> = (0..n).collect();
let runner = crate::numa::PartitionRunner::new();
let runner = crate::index::numa::PartitionRunner::new();
runner.run(
&order,
|i| {
src/obikindex/src/select.rs → src/obikindex/src/index/select.rs
+7 -7
View File
@@ -1,13 +1,13 @@
use std::path::Path;
use crate::OutputCol;
use crate::index::OutputCol;
use obisys::{Reporter, Stage, progress_bar};
use tracing::info;
use crate::error::{OKIError, OKIResult};
use crate::index::KmerIndex;
use crate::meta::{GenomeInfo, IndexMeta};
use crate::state::IndexState;
use crate::index::error::{OKIError, OKIResult};
use crate::index::kmer_index::KmerIndex;
use crate::index::meta::{GenomeInfo, IndexMeta};
use crate::index::state::IndexState;
impl KmerIndex {
/// Create a new index at `output` by projecting/aggregating the genome columns
@@ -56,7 +56,7 @@ impl KmerIndex {
let pb = progress_bar("select", n_partitions as u64, "partitions");
let order: Vec<usize> = (0..n_partitions).collect();
let runner = crate::numa::PartitionRunner::new();
let runner = crate::index::numa::PartitionRunner::new();
runner
.run(
&order,
@@ -111,7 +111,7 @@ impl KmerIndex {
let pb = progress_bar("select", n_partitions as u64, "partitions");
let order: Vec<usize> = (0..n_partitions).collect();
let runner = crate::numa::PartitionRunner::new();
let runner = crate::index::numa::PartitionRunner::new();
runner
.run(
&order,
src/obikindex/src/select_layer.rs → src/obikindex/src/index/select_layer.rs
+2 -2
View File
@@ -6,10 +6,10 @@ use obicompactvec::{
ColGroup, MatrixGroupOps, PersistentBitMatrix, PersistentBitMatrixBuilder,
PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder,
};
use obilayeredmap::OLMError;
use crate::layer::OLMError;
use obiskio::{SKError, SKResult};
use crate::index::KmerIndex;
use crate::index::kmer_index::KmerIndex;
// ── AggOp ─────────────────────────────────────────────────────────────────────
src/obikindex/src/state.rs → src/obikindex/src/index/state.rs
+1 -1
View File
@@ -1,6 +1,6 @@
use std::path::Path;
use crate::meta::META_FILENAME;
use crate::index::meta::META_FILENAME;
pub const SENTINEL_SCATTERED: &str = "scatter.done";
pub const SENTINEL_COUNTED: &str = "count.done";
src/obikindex/src/stats.rs → src/obikindex/src/index/stats.rs
+4 -4
View File
@@ -5,8 +5,8 @@ use obicompactvec::{LayerMeta, PersistentBitMatrix, PersistentCompactIntMatrix};
use obicompactvec::traits::ColumnWeights;
use rayon::prelude::*;
use crate::error::OKIResult;
use crate::index::KmerIndex;
use crate::index::error::OKIResult;
use crate::index::kmer_index::KmerIndex;
/// Bits per kmer broken down by index component.
pub struct IndexBitsPerKmer {
@@ -153,12 +153,12 @@ impl KmerIndex {
if this_layer_dir.join("counts").exists()
&& !this_layer_dir.join("presence").exists()
{
match obilayeredmap::open_data::<PersistentCompactIntMatrix>(&index_dir, l) {
match crate::layer::open_data::<PersistentCompactIntMatrix>(&index_dir, l) {
Ok(m) => Box::new(m),
Err(_) => continue,
}
} else {
match obilayeredmap::open_data::<PersistentBitMatrix>(&index_dir, l) {
match crate::layer::open_data::<PersistentBitMatrix>(&index_dir, l) {
Ok(m) => Box::new(m),
Err(_) => continue,
}
src/obikindex/src/tests/query_layer.rs → src/obikindex/src/index/tests/query_layer.rs
+3 -3
View File
@@ -1,5 +1,5 @@
use super::*;
use crate::meta::IndexConfig;
use crate::index::meta::IndexConfig;
// ── QueryStats::AddAssign ───────────────────────────────────────────────────
@@ -50,7 +50,7 @@ fn query_partition_with_missing_index_dir_returns_default_stats() {
minimizer_size: 9,
n_bits: 2,
with_counts: false,
evidence: obilayeredmap::IndexMode::Exact,
evidence: crate::layer::IndexMode::Exact,
block_bits: 0,
};
let index = KmerIndex::create(tmp.path().join("idx"), config, None).expect("create index");
@@ -78,7 +78,7 @@ fn query_partition_with_empty_kmers_is_a_noop() {
minimizer_size: 9,
n_bits: 2,
with_counts: false,
evidence: obilayeredmap::IndexMode::Exact,
evidence: crate::layer::IndexMode::Exact,
block_bits: 0,
};
let index = KmerIndex::create(tmp.path().join("idx"), config, None).expect("create index");
src/obilayeredmap/src/content_layer.rs → src/obikindex/src/layer/content_layer.rs
+5 -5
View File
@@ -11,7 +11,7 @@
//! This is exactly the shape `obikphylo::siblings::cache::Mat` used to
//! reimplement locally, minus its one sibling-specific method
//! (`iter_minorants_batch`, which stays an extension trait over there —
//! `obilayeredmap` has no business knowing about sibling annexes).
//! `obikindex::layer` has no business knowing about sibling annexes).
//!
//! `Layer` is meant to eventually represent a layer's *whole* life —
//! empty shell, under construction, ready to read — not just the
@@ -27,10 +27,10 @@ use std::path::{Path, PathBuf};
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
use obikseq::CanonicalKmer;
use crate::error::OLMResult;
use crate::layer::{LayerContent, TypedLayer, COUNTS_DIR, PRESENCE_DIR};
use crate::meta::IndexMode;
use crate::mphf_layer::{EvidenceKind, EVIDENCE_FILE, FINGERPRINT_FILE, MPHF_FILE, UNITIGS_FILE};
use crate::layer::error::OLMResult;
use crate::layer::typed_layer::{LayerContent, TypedLayer, COUNTS_DIR, PRESENCE_DIR};
use crate::layer::meta::IndexMode;
use crate::layer::mphf_layer::{EvidenceKind, EVIDENCE_FILE, FINGERPRINT_FILE, MPHF_FILE, UNITIGS_FILE};
/// One layer, at any point in its life — see the module docs. Only
/// [`Empty`](Layer::Empty) and the two ready-to-read states
src/obilayeredmap/src/error.rs → src/obikindex/src/layer/error.rs
View File
src/obilayeredmap/src/evidence.rs → src/obikindex/src/layer/evidence.rs
+1 -1
View File
@@ -6,7 +6,7 @@ use std::path::Path;
use memmap2::Mmap;
use crate::error::{OLMError, OLMResult};
use crate::layer::error::{OLMError, OLMResult};
pub struct Evidence {
mmap: Mmap,
src/obilayeredmap/src/fingerprint.rs → src/obikindex/src/layer/fingerprint.rs
+1 -1
View File
@@ -14,7 +14,7 @@ use std::path::Path;
use bitvec::prelude::*;
use memmap2::Mmap;
use crate::error::{OLMError, OLMResult};
use crate::layer::error::{OLMError, OLMResult};
const MAGIC: &[u8; 4] = b"FPVF";
const HEADER: usize = 16;
src/obilayeredmap/src/layered_store.rs → src/obikindex/src/layer/layered_store.rs
View File
src/obilayeredmap/src/map.rs → src/obikindex/src/layer/map.rs
+4 -4
View File
@@ -6,10 +6,10 @@ use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix, StorageKind
use obikseq::CanonicalKmer;
use obiskio::{UnitigFileWriter, DEFAULT_BLOCK_BITS};
use crate::error::{OLMError, OLMResult};
use crate::layer::{layer_dir, Hit, TypedLayer, LayerContent, LayerData};
use crate::meta::{IndexMode, PartitionMeta};
use crate::mphf_layer::EvidenceKind;
use crate::layer::error::{OLMError, OLMResult};
use crate::layer::typed_layer::{layer_dir, Hit, TypedLayer, LayerContent, LayerData};
use crate::layer::meta::{IndexMode, PartitionMeta};
use crate::layer::mphf_layer::EvidenceKind;
/// Layered kmer index for a single partition.
///
src/obilayeredmap/src/meta.rs → src/obikindex/src/layer/meta.rs
+1 -1
View File
@@ -3,7 +3,7 @@ use std::path::Path;
use serde::{Deserialize, Serialize};
use crate::error::OLMResult;
use crate::layer::error::OLMResult;
const META_FILE: &str = "meta.json";
src/obilayeredmap/src/lib.rs → src/obikindex/src/layer/mod.rs
+2 -2
View File
@@ -2,7 +2,7 @@ pub mod content_layer;
pub mod error;
pub mod evidence;
pub mod fingerprint;
pub mod layer;
pub mod typed_layer;
pub mod layered_store;
pub mod map;
pub mod meta;
@@ -10,7 +10,7 @@ pub(crate) mod mphf_layer;
pub use content_layer::Layer;
pub use error::{OLMError, OLMResult};
pub use layer::{
pub use typed_layer::{
dereplicated_superkmers_path, layer_dir, open_data, raw_superkmers_path, HasLayerContent,
HasStorageKind, Hit, LayerContent, LayerData, TypedLayer,
};
src/obilayeredmap/src/mphf_layer.rs → src/obikindex/src/layer/mphf_layer.rs
+4 -4
View File
@@ -10,10 +10,10 @@ use obikseq::CanonicalKmer;
use obiskio::{CanonicalKmerIter, UnitigFileReader, UnitigFileWriter, build_unitig_idx};
use ptr_hash::{PtrHash, PtrHashParams, bucket_fn::CubicEps, hash::Xx64};
use crate::error::{OLMError, OLMResult};
use crate::evidence::{Evidence, EvidenceWriter};
use crate::fingerprint::{FingerprintVec, FingerprintVecWriter};
use crate::meta::IndexMode;
use crate::layer::error::{OLMError, OLMResult};
use crate::layer::evidence::{Evidence, EvidenceWriter};
use crate::layer::fingerprint::{FingerprintVec, FingerprintVecWriter};
use crate::layer::meta::IndexMode;
pub(crate) const MPHF_FILE: &str = "mphf.bin";
pub(crate) const UNITIGS_FILE: &str = "unitigs.bin";
src/obilayeredmap/src/tests/layered_store.rs → src/obikindex/src/layer/tests/layered_store.rs
View File
src/obilayeredmap/src/tests/map.rs → src/obikindex/src/layer/tests/map.rs
+4 -4
View File
@@ -2,9 +2,9 @@ use super::*;
use obicompactvec::{pack_bit_matrix, pack_sparse_bit_matrix, PersistentBitMatrix, PersistentCompactIntMatrix, StorageKind};
use obikseq::{set_k, Sequence as _, Unitig};
use obiskio::DEFAULT_BLOCK_BITS;
use crate::layer::LayerContent;
use crate::meta::IndexMode;
use crate::mphf_layer::EvidenceKind;
use crate::layer::typed_layer::LayerContent;
use crate::layer::meta::IndexMode;
use crate::layer::mphf_layer::EvidenceKind;
use tempfile::tempdir;
fn push_unitigs_and_layer(
@@ -111,7 +111,7 @@ fn push_layer_from_map_convenience() {
fn write_unitigs_at(dir: &Path, seqs: &[&[u8]]) {
fs::create_dir_all(dir).unwrap();
let mut w = UnitigFileWriter::create(&dir.join(crate::layer::UNITIGS_FILE)).unwrap();
let mut w = UnitigFileWriter::create(&dir.join(crate::layer::typed_layer::UNITIGS_FILE)).unwrap();
for s in seqs {
w.write(&Unitig::from_ascii(s)).unwrap();
}
src/obilayeredmap/src/tests/layer.rs → src/obikindex/src/layer/tests/typed_layer.rs
+1 -1
View File
@@ -111,7 +111,7 @@ fn count_layer_reports_count_content_and_columnar_storage() {
assert_eq!(layer.content(), LayerContent::Count);
assert_eq!(layer.storage_kind(), obicompactvec::StorageKind::Columnar);
assert_eq!(layer.evidence_kind(), crate::mphf_layer::EvidenceKind::Exact);
assert_eq!(layer.evidence_kind(), crate::layer::mphf_layer::EvidenceKind::Exact);
}
#[test]
src/obilayeredmap/src/layer.rs → src/obikindex/src/layer/typed_layer.rs
+20 -19
View File
@@ -11,10 +11,10 @@ use obicompactvec::{
use obikseq::CanonicalKmer;
use obiskio::{UnitigFileReader, UnitigFileWriter};
use crate::error::{OLMError, OLMResult};
use crate::meta::IndexMode;
use crate::mphf_layer::MphfLayer;
pub(crate) use crate::mphf_layer::UNITIGS_FILE;
use crate::layer::error::{OLMError, OLMResult};
use crate::layer::meta::IndexMode;
use crate::layer::mphf_layer::MphfLayer;
pub(crate) use crate::layer::mphf_layer::UNITIGS_FILE;
pub(crate) const COUNTS_DIR: &str = "counts";
pub(crate) const PRESENCE_DIR: &str = "presence";
@@ -31,7 +31,7 @@ pub trait LayerData: Sized {
/// single source of truth for the on-disk `layer_N` naming convention
/// (mirrors what `LayeredMap::open`/`push_layer` already use internally).
///
/// `obilayeredmap` operates within a single partition's index root; it has
/// `obikindex::layer` operates within a single partition's index root; it has
/// no notion of "partition" at all. Turning a partition number into that
/// root is `obikindex::KmerIndex::index_dir`'s job, one layer up — callers
/// here only ever name a layer *number*, never build the path themselves.
@@ -46,20 +46,21 @@ const SK_EXT: &str = "skmer.zst";
/// Path of a layer's raw, not-yet-dereplicated superkmer file — written by
/// whichever algorithm routes superkmers into this layer (today:
/// `obikpartitionner::PartitionRouter`), read by whichever algorithm
/// dereplicates it (today: `obikderep::Dereplicator`). Naming this once
/// here, rather than in either algorithm crate, is what lets two
/// independent crates agree on the filename without depending on each
/// other — see `DevDocMD/implementation/partition_layer_cache.md`.
/// `crate::algorithms::partitionner::PartitionRouter`), read by whichever
/// algorithm dereplicates it (today:
/// `crate::algorithms::dereplicator::Dereplicator`). Naming this once
/// here, rather than in either algorithm submodule, is what lets the two
/// agree on the filename without depending on each other directly — see
/// `DevDocMD/implementation/partition_layer_cache.md`.
pub fn raw_superkmers_path(layer_dir: &Path) -> PathBuf {
layer_dir.join(format!("raw.{SK_EXT}"))
}
/// Path of a layer's dereplicated superkmer file — written by
/// `obikderep::Dereplicator`, read by whichever algorithm counts kmer
/// abundances from it (today: `obikpartitionner::PartitionRouter::
/// count_kmer`) and, later, by `obikindex::build_index_layer` to build the
/// real layer.
/// `crate::algorithms::dereplicator::Dereplicator`, read by whichever
/// algorithm counts kmer abundances from it (today:
/// `crate::algorithms::partitionner::PartitionRouter::count_kmer`) and,
/// later, by `obikindex::build_index_layer` to build the real layer.
pub fn dereplicated_superkmers_path(layer_dir: &Path) -> PathBuf {
layer_dir.join(format!("dereplicated.{SK_EXT}"))
}
@@ -226,18 +227,18 @@ impl<D: LayerData> TypedLayer<D> {
}
/// Iterate over all canonical kmers in the layer, in deterministic order.
pub fn iter_kmers(&self) -> crate::mphf_layer::KmerIter {
pub fn iter_kmers(&self) -> crate::layer::mphf_layer::KmerIter {
self.mphf.iter_kmers()
}
/// Iterate over all canonical kmers, each paired with its zero-based
/// sequence index in `unitigs.bin`.
pub fn enumerate_kmers(&self) -> std::iter::Enumerate<crate::mphf_layer::KmerIter> {
pub fn enumerate_kmers(&self) -> std::iter::Enumerate<crate::layer::mphf_layer::KmerIter> {
self.mphf.enumerate_kmers()
}
/// Iterate over the layer's canonical kmers in batches of `n`.
pub fn iter_kmers_batch(&self, n: usize) -> crate::mphf_layer::KmerBatchIter {
pub fn iter_kmers_batch(&self, n: usize) -> crate::layer::mphf_layer::KmerBatchIter {
self.mphf.iter_kmers_batch(n)
}
@@ -268,7 +269,7 @@ impl<D: LayerData> TypedLayer<D> {
/// This already-open layer's evidence mode — reads the discriminant
/// already in memory (`MphfLayer`'s own `LayerEvidence`), no disk
/// access. Available regardless of `D`, unlike `content`/`storage_kind`.
pub fn evidence_kind(&self) -> crate::mphf_layer::EvidenceKind {
pub fn evidence_kind(&self) -> crate::layer::mphf_layer::EvidenceKind {
self.mphf.evidence_kind()
}
}
@@ -456,5 +457,5 @@ impl TypedLayer<PersistentBitMatrix> {
}
#[cfg(test)]
#[path = "tests/layer.rs"]
#[path = "tests/typed_layer.rs"]
mod tests;
src/obikindex/src/lib.rs
+21 -35
View File
@@ -1,36 +1,22 @@
pub mod error;
pub mod meta;
pub mod predicate;
pub mod state;
mod common;
mod distance;
mod dump;
mod dump_layer;
pub mod filter;
mod graph_pipeline;
mod index;
mod index_layer;
mod matrix_store;
mod merge;
mod merge_layer;
mod numa;
mod query_layer;
mod rebuild;
mod rebuild_layer;
mod reindex;
mod select;
mod select_layer;
mod stats;
//! The `Index { Partition { Layer } }` model — see
//! `DevDocMD/implementation/partition_layer_cache.md`'s "Definitions"
//! section for the authoritative naming/scope discussion. Each tier is a
//! submodule: [`layer`] (was the `obilayeredmap` crate), [`partition`]
//! (was the `obikpartition` crate), [`index`] (this crate's original
//! content). [`algorithms`] (was the `obikpartitionner` and `obikderep`
//! crates) sits alongside the model: code that builds or transforms an
//! index's content rather than defining its shape.
pub use error::{OKIError, OKIResult};
pub use distance::{DistanceMetric, DistanceOutput};
pub use filter::{GroupQuorumFilter, KmerFilter, passes_all};
pub use index::KmerIndex;
pub use merge_layer::MergeMode;
pub use meta::{validate_label, GenomeInfo, IndexConfig, IndexMeta, META_FILENAME};
pub use predicate::{GroupFilterParams, MetaPred};
pub use query_layer::{KmerDesc, QueryHit, QueryStats};
pub use select_layer::{AggOp, OutputCol};
pub use state::{IndexState, SENTINEL_COUNTED, SENTINEL_INDEXED, SENTINEL_SCATTERED};
pub use stats::IndexBitsPerKmer;
pub use numa::PartitionRunner;
pub mod algorithms;
pub mod index;
pub mod layer;
pub mod partition;
pub use index::{
validate_label, AggOp, DistanceMetric, DistanceOutput, GenomeInfo, GroupFilterParams,
GroupQuorumFilter, IndexBitsPerKmer, IndexConfig, IndexMeta, IndexState, KmerDesc, KmerFilter,
KmerIndex, MergeMode, MetaPred, OKIError, OKIResult, OutputCol, PartitionRunner, QueryHit,
QueryStats, META_FILENAME, SENTINEL_COUNTED, SENTINEL_INDEXED, SENTINEL_SCATTERED,
passes_all,
};
pub use index::{filter, meta};
src/obikpartition/src/lib.rs → src/obikindex/src/partition/mod.rs
+12 -13
View File
@@ -1,29 +1,28 @@
//! The **Partition** tier of the `Index { Partition { Layer } }` model —
//! see `DevDocMD/implementation/partition_layer_cache.md`'s "Definitions"
//! section for the authoritative naming/scope discussion this crate
//! implements. `obilayeredmap` already holds the **Layer** tier as its own
//! crate rather than living inside `obikindex`; this crate holds the tier
//! above it the same way.
//! section for the authoritative naming/scope discussion this module
//! implements. [`crate::layer`] holds the **Layer** tier the same way;
//! this module holds the tier above it.
//!
//! [`KmerPartition`] represents one partition's already-open layers — a
//! read cache, opened once and held for the run, not rebuilt per lookup.
//! It does no path computation of its own beyond the shared
//! `obilayeredmap::layer_dir` naming primitive: which partition, which
//! `index_dir`, how many layers, and `IndexMode` are `KmerIndex`'s
//! (`obikindex`) job to resolve and hand in as plain arguments — this
//! crate depends only on `obilayeredmap` and below, never on `obikindex`,
//! so it cannot reach back for them itself.
//! `crate::layer::layer_dir` naming primitive: which partition, which
//! `index_dir`, how many layers, and `IndexMode` are `crate::index::
//! KmerIndex`'s job to resolve and hand in as plain arguments — this
//! module depends only on `crate::layer` and below, never on
//! `crate::index`, so it cannot reach back for them itself.
//!
//! Not yet wired into any caller: `obikphylo::siblings::cache::
//! PartitionCache` (`Vec<Vec<obilayeredmap::Layer>>`) and
//! `obikindex::query_layer::QueryLayer` (uncached, bypasses `Layer`
//! PartitionCache` (`Vec<Vec<crate::layer::Layer>>`) and
//! `crate::index::query_layer::QueryLayer` (uncached, bypasses `Layer`
//! entirely) both still reinvent a fragment of this. Migrating them is a
//! separate, deferred step.
use std::path::{Path, PathBuf};
use obikseq::CanonicalKmer;
use obilayeredmap::{layer_dir, IndexMode, Layer, OLMResult};
use crate::layer::{layer_dir, IndexMode, Layer, OLMResult};
/// Partition subdirectory name, under an index's root — the single source
/// of truth for the on-disk `partitions/part_NNNNN` naming convention.
@@ -40,7 +39,7 @@ pub fn partition_dir(root: &Path, i: usize) -> PathBuf {
}
/// Path of partition `i`'s layered-index directory — `<partition_dir>/index`,
/// the root every [`obilayeredmap::layer_dir`] call for this partition is
/// the root every [`crate::layer::layer_dir`] call for this partition is
/// relative to.
pub fn index_dir(root: &Path, i: usize) -> PathBuf {
partition_dir(root, i).join("index")
src/obikpartition/src/tests.rs → src/obikindex/src/partition/tests.rs
+2 -2
View File
@@ -11,7 +11,7 @@ fn write_layer(root: &Path, l: usize, seqs: &[&[u8]], n_genomes: usize, mode: &I
w.write(&Unitig::from_ascii(s)).unwrap();
}
w.close().unwrap();
obilayeredmap::TypedLayer::<obicompactvec::PersistentBitMatrix>::build_presence(
crate::layer::TypedLayer::<obicompactvec::PersistentBitMatrix>::build_presence(
&dir,
DEFAULT_BLOCK_BITS,
mode,
@@ -23,7 +23,7 @@ fn write_layer(root: &Path, l: usize, seqs: &[&[u8]], n_genomes: usize, mode: &I
#[test]
fn open_reads_every_layer_in_order() {
// k=4, process-wide in test builds — see obilayeredmap's own tests for
// k=4, process-wide in test builds — see obikindex::layer's own tests for
// why a fixed, unshared k matters here.
set_k(4);
let dir = tempdir().unwrap();
src/obikmer/Cargo.toml
-3
View File
@@ -15,14 +15,11 @@ obifastwrite = { path = "../obifastwrite" }
obidebruinj = { path = "../obidebruinj" }
obipipeline = { path = "../obipipeline" }
obikrope = { path = "../obikrope" }
obikpartitionner = { path = "../obikpartitionner" }
obikderep = { path = "../obikderep" }
obisys = { path = "../obisys" }
obiskio = { path = "../obiskio" }
obikindex = { path = "../obikindex", default-features = false }
obikphylo = { path = "../obikphylo" }
obitaxonomy = { path = "../obitaxonomy" }
obilayeredmap = { path = "../obilayeredmap" }
clap = { version = "4", features = ["derive"] }
serde = { version = "1", features = ["derive"] }
serde_json = "1"
src/obikmer/src/cmd/index/mod.rs
+42 -43
View File
@@ -2,10 +2,10 @@ use std::path::PathBuf;
use std::time::Instant;
use clap::Args;
use obikderep::Dereplicator;
use obikindex::algorithms::dereplicator::Dereplicator;
use obikindex::algorithms::partitionner::PartitionRouter;
use obikindex::{validate_label, GenomeInfo, IndexConfig, IndexState, KmerIndex};
use obikpartitionner::PartitionRouter;
use obilayeredmap::IndexMode;
use obikindex::layer::IndexMode;
fn parse_key_value(s: &str) -> Result<(String, String), String> {
let pos = s.find('=').ok_or_else(|| format!("invalid key=value: no '=' in '{s}'"))?;
@@ -188,50 +188,49 @@ pub fn run(args: IndexArgs) {
};
// ── Open or create the index ─────────────────────────────────────────────
let mut idx = if KmerIndex::exists(&output) && !args.force {
info!("resuming from existing index at {}", output.display());
KmerIndex::open(&output).unwrap_or_else(|e| {
eprintln!("error opening index: {e}");
if KmerIndex::exists(&output) {
if !args.force {
eprintln!(
"error: an index already exists at {} (use --force to overwrite it)",
output.display()
);
std::process::exit(1);
})
} else {
if args.force && KmerIndex::exists(&output) {
info!("--force: removing existing index at {}", output.display());
std::fs::remove_dir_all(&output).unwrap_or_else(|e| {
eprintln!("error removing existing index: {e}");
std::process::exit(1);
});
}
let n_bits = partitions_to_bits(args.common.partitions);
let effective = 1usize << n_bits;
if effective != args.common.partitions {
info!("partitions: {} → {} (next power of 2)", args.common.partitions, effective);
}
let block_bits = block_size_to_bits(args.block_size);
let config = IndexConfig {
kmer_size: effective_kmer_size,
minimizer_size: args.common.minimizer_size,
n_bits,
with_counts: args.with_counts,
evidence: evidence.clone(),
block_bits,
};
let genome_info = args.label.as_ref().map(|label| {
validate_label(label).unwrap_or_else(|e| {
eprintln!("error: --label: {e}");
std::process::exit(1);
});
let mut info = GenomeInfo::new(label.clone());
for (k, v) in &args.meta {
info.meta.insert(k.clone(), v.clone());
}
info
info!("--force: removing existing index at {}", output.display());
std::fs::remove_dir_all(&output).unwrap_or_else(|e| {
eprintln!("error removing existing index: {e}");
std::process::exit(1);
});
KmerIndex::create(&output, config, genome_info).unwrap_or_else(|e| {
eprintln!("error creating index: {e}");
std::process::exit(1);
})
}
let n_bits = partitions_to_bits(args.common.partitions);
let effective = 1usize << n_bits;
if effective != args.common.partitions {
info!("partitions: {} → {} (next power of 2)", args.common.partitions, effective);
}
let block_bits = block_size_to_bits(args.block_size);
let config = IndexConfig {
kmer_size: effective_kmer_size,
minimizer_size: args.common.minimizer_size,
n_bits,
with_counts: args.with_counts,
evidence: evidence.clone(),
block_bits,
};
let genome_info = args.label.as_ref().map(|label| {
validate_label(label).unwrap_or_else(|e| {
eprintln!("error: --label: {e}");
std::process::exit(1);
});
let mut info = GenomeInfo::new(label.clone());
for (k, v) in &args.meta {
info.meta.insert(k.clone(), v.clone());
}
info
});
let mut idx = KmerIndex::create(&output, config, genome_info).unwrap_or_else(|e| {
eprintln!("error creating index: {e}");
std::process::exit(1);
});
// ── Stage 1: scatter ─────────────────────────────────────────────────────
src/obikmer/src/cmd/query/mod.rs
+1 -1
View File
@@ -13,7 +13,7 @@ use std::time::Instant;
use clap::Args;
use obikindex::KmerIndex;
use obikrope::Rope;
use obilayeredmap::IndexMode;
use obikindex::layer::IndexMode;
use obipipeline::{Throttled, ThrottleGuard, throttle};
use obiread::chunk::read_sequence_chunks_sized;
use obisys::{Reporter, Stage, available_memory_bytes, spinner};
src/obikmer/src/cmd/reindex/mod.rs
+1 -1
View File
@@ -2,7 +2,7 @@ use std::path::PathBuf;
use clap::Args;
use obikindex::KmerIndex;
use obilayeredmap::IndexMode;
use obikindex::layer::IndexMode;
use obisys::Reporter;
use tracing::info;
src/obikpartition/Cargo.toml
-14
View File
@@ -1,14 +0,0 @@
[package]
name = "obikpartition"
version = "0.1.0"
edition = "2024"
[dependencies]
obikseq = { path = "../obikseq" }
obilayeredmap = { path = "../obilayeredmap" }
[dev-dependencies]
tempfile = "3"
obikseq = { path = "../obikseq", features = ["test-utils"] }
obiskio = { path = "../obiskio" }
obicompactvec = { path = "../obicompactvec" }
src/obikpartitionner/Cargo.toml
-31
View File
@@ -1,31 +0,0 @@
[package]
name = "obikpartitionner"
version = "0.1.0"
edition = "2024"
[dev-dependencies]
tempfile = "3"
obikseq = { path = "../obikseq", features = ["test-utils"] }
obikrope = { path = "../obikrope" }
obikderep = { path = "../obikderep" }
[dependencies]
niffler = "3.0.0"
obikseq = { path = "../obikseq" }
obikindex = { path = "../obikindex" }
obilayeredmap = { path = "../obilayeredmap" }
obipipeline = { path = "../obipipeline" }
obiread = { path = "../obiread" }
obiskbuilder = { path = "../obiskbuilder" }
obiskio = { path = "../obiskio" }
rayon = "1"
sysinfo = "0.39"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
tracing = "0.1.44"
cacheline-ef = "1.1"
epserde = "0.8"
memmap2 = "0.9.10"
obicompactvec = { path = "../obicompactvec" }
ptr_hash = "1.1"
obisys = { path = "../obisys" }
src/obikpartitionner/src/lib.rs
-4
View File
@@ -1,4 +0,0 @@
mod kmer_sort;
mod partition;
pub use partition::{KmerSpectrum, PartitionRouter};
src/obikpartitionner/src/partition/mod.rs
-18
View File
@@ -1,18 +0,0 @@
//! K-mer partitioning: routing super-kmers into per-partition, layer-0
//! files, and counting unique canonical k-mers. Dereplication itself moved
//! to `obikderep` (2026-08-20) — see
//! `DevDocMD/implementation/partition_layer_cache.md`: one algorithm, one
//! crate, split off one at a time rather than all at once.
//!
//! Submodules: [`router`] (`PartitionRouter`, `KmerSpectrum`, the
//! routing/counting lifecycle API — partition/layer path naming itself
//! lives on `obikindex::KmerIndex`, not here), [`count`] (unique-kmer
//! enumeration, MPHF, abundance counting).
mod count;
mod router;
#[cfg(test)]
mod tests;
pub use router::{KmerSpectrum, PartitionRouter};
src/obikphylo/Cargo.toml
-3
View File
@@ -6,11 +6,9 @@ edition = "2024"
[dependencies]
obikindex = { path = "../obikindex", default-features = false }
obikseq = { path = "../obikseq" }
obikpartitionner = { path = "../obikpartitionner" }
obiskio = { path = "../obiskio" }
obisys = { path = "../obisys" }
obicompactvec = { path = "../obicompactvec" }
obilayeredmap = { path = "../obilayeredmap" }
obiskbuilder = { path = "../obiskbuilder" }
obipipeline = { path = "../obipipeline" }
memmap2 = "0.9"
@@ -21,6 +19,5 @@ tracing = "0.1.44"
[dev-dependencies]
obiread = { path = "../obiread" }
obikderep = { path = "../obikderep" }
tempfile = "3"
tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] }
src/obikphylo/src/lib.rs
+1 -1
View File
@@ -1,5 +1,5 @@
//! Library-level phylogenetic functionality for `obikmer`, built as
//! extension traits over `obikindex::KmerIndex` and `obilayeredmap`'s
//! extension traits over `obikindex::KmerIndex` and `obikindex::layer`'s
//! generic layer types — the `phylo` CLI command is a consumer of this
//! crate, not the owner of this logic (see `DevDocMD/architecture/siblings.md`).
//!
src/obikphylo/src/siblings/build.rs
+2 -2
View File
@@ -5,8 +5,8 @@ use std::sync::atomic::Ordering;
use rayon::prelude::*;
use obikseq::CanonicalKmer;
use obilayeredmap::MphfLayer;
use obilayeredmap::meta::IndexMode;
use obikindex::layer::MphfLayer;
use obikindex::layer::meta::IndexMode;
use obipipeline::ThrottleGuard;
use obisys::progress_bar;
src/obikphylo/src/siblings/cache.rs
+2 -2
View File
@@ -1,7 +1,7 @@
use rayon::prelude::*;
use obikseq::CanonicalKmer;
use obilayeredmap::Layer;
use obikindex::layer::Layer;
use obisys::progress_bar;
use obikindex::{KmerIndex, OKIResult};
@@ -24,7 +24,7 @@ use obikindex::{KmerIndex, OKIResult};
/// partition for the entire run, regardless of how many source layers or
/// lookups follow.
///
/// `mats[partition][layer]` holds `obilayeredmap::Layer` — the
/// `mats[partition][layer]` holds `obikindex::layer::Layer` — the
/// format-erased `Count`/`Presence` handle, not a sibling-specific type:
/// this module used to bundle its own `Mat` enum here, duplicating exactly
/// what `Layer` now does one crate down (see
src/obikphylo/src/siblings/family_scan.rs
+2 -2
View File
@@ -54,13 +54,13 @@ use std::sync::atomic::{AtomicU8, Ordering};
use rayon::prelude::*;
use obikseq::CanonicalKmer;
use obilayeredmap::meta::PartitionMeta;
use obikindex::layer::meta::PartitionMeta;
use obipipeline::{ThrottleGuard, throttle};
use obikindex::{OKIError, OKIResult};
use obikindex::KmerIndex;
use obilayeredmap::Layer;
use obikindex::layer::Layer;
use super::cache::PartitionCache;
use super::helpers::central_base;
src/obikphylo/src/siblings/iter.rs
+18 -18
View File
@@ -1,6 +1,6 @@
//! Phylo/sibling-domain iteration over a layer — an extension trait, not a
//! new field on `MphfLayer`/`TypedLayer<D>`: "family"/"minorant" are phylo
//! concepts, `obilayeredmap` stays kmer/slot-mapping only (see
//! concepts, `obikindex::layer` stays kmer/slot-mapping only (see
//! `DevDocMD/architecture/siblings.md`).
//!
//! The sibling annex is persisted in the same order as `iter_kmers()`
@@ -12,7 +12,7 @@
//! batch — never collected whole into memory (see the project's "no full
//! collect" rule).
//!
//! Four iterator types, deliberately mirroring `obilayeredmap`'s own
//! Four iterator types, deliberately mirroring `obikindex::layer`'s own
//! `KmerIter`/`KmerBatchIter` pair (single item vs. `Vec` batch) — plus the
//! minorant-filtered variant of each, since "all siblings" and "one row per
//! family" are both common cases:
@@ -29,7 +29,7 @@
use std::sync::Arc;
use obikseq::CanonicalKmer;
use obilayeredmap::{KmerIter, TypedLayer, LayerData};
use obikindex::layer::{KmerIter, TypedLayer, LayerData};
use super::{FamilyMask, SiblingAnnex};
@@ -172,42 +172,42 @@ impl<D: LayerData> SiblingLayerExt for TypedLayer<D> {
}
}
/// Same extension, over `obilayeredmap::Layer` (the format-erased
/// Same extension, over `obikindex::layer::Layer` (the format-erased
/// `Count`/`Presence` handle `PartitionCache` actually holds) — dispatch
/// only, both arms return the same concrete iterator types (they don't
/// depend on which `D` is inside), so no boxing is needed. `obilayeredmap`
/// depend on which `D` is inside), so no boxing is needed. `obikindex::layer`
/// itself can't implement this: "family"/"minorant" are phylo concepts, it
/// stays kmer/slot-mapping only (see the module docs above).
impl SiblingLayerExt for obilayeredmap::Layer {
impl SiblingLayerExt for obikindex::layer::Layer {
fn iter_siblings(&self, annex: Arc<SiblingAnnex>) -> SiblingIter {
match self {
obilayeredmap::Layer::Count(l) => l.iter_siblings(annex),
obilayeredmap::Layer::Presence(l) => l.iter_siblings(annex),
obilayeredmap::Layer::Empty { .. } => panic!("iter_siblings() called on an Empty layer"),
obikindex::layer::Layer::Count(l) => l.iter_siblings(annex),
obikindex::layer::Layer::Presence(l) => l.iter_siblings(annex),
obikindex::layer::Layer::Empty { .. } => panic!("iter_siblings() called on an Empty layer"),
}
}
fn iter_siblings_batch(&self, annex: Arc<SiblingAnnex>, batch_size: usize) -> SiblingBatchIter {
match self {
obilayeredmap::Layer::Count(l) => l.iter_siblings_batch(annex, batch_size),
obilayeredmap::Layer::Presence(l) => l.iter_siblings_batch(annex, batch_size),
obilayeredmap::Layer::Empty { .. } => panic!("iter_siblings_batch() called on an Empty layer"),
obikindex::layer::Layer::Count(l) => l.iter_siblings_batch(annex, batch_size),
obikindex::layer::Layer::Presence(l) => l.iter_siblings_batch(annex, batch_size),
obikindex::layer::Layer::Empty { .. } => panic!("iter_siblings_batch() called on an Empty layer"),
}
}
fn iter_minorants(&self, annex: Arc<SiblingAnnex>) -> MinorantIter {
match self {
obilayeredmap::Layer::Count(l) => l.iter_minorants(annex),
obilayeredmap::Layer::Presence(l) => l.iter_minorants(annex),
obilayeredmap::Layer::Empty { .. } => panic!("iter_minorants() called on an Empty layer"),
obikindex::layer::Layer::Count(l) => l.iter_minorants(annex),
obikindex::layer::Layer::Presence(l) => l.iter_minorants(annex),
obikindex::layer::Layer::Empty { .. } => panic!("iter_minorants() called on an Empty layer"),
}
}
fn iter_minorants_batch(&self, annex: Arc<SiblingAnnex>, batch_size: usize) -> MinorantBatchIter {
match self {
obilayeredmap::Layer::Count(l) => l.iter_minorants_batch(annex, batch_size),
obilayeredmap::Layer::Presence(l) => l.iter_minorants_batch(annex, batch_size),
obilayeredmap::Layer::Empty { .. } => panic!("iter_minorants_batch() called on an Empty layer"),
obikindex::layer::Layer::Count(l) => l.iter_minorants_batch(annex, batch_size),
obikindex::layer::Layer::Presence(l) => l.iter_minorants_batch(annex, batch_size),
obikindex::layer::Layer::Empty { .. } => panic!("iter_minorants_batch() called on an Empty layer"),
}
}
}
src/obikphylo/src/siblings/mod.rs
+1 -1
View File
@@ -71,7 +71,7 @@ pub(crate) use siblingannex::{FamilyMask, SiblingAnnex, SiblingAnnexBuilder};
pub use stats::{SiblingAnnexStats, SiblingStatsExt};
pub use subsample::EntropyBias;
use obilayeredmap::OLMError;
use obikindex::layer::OLMError;
use obikindex::OKIError;
src/obikphylo/src/siblings/tests.rs
+5 -5
View File
@@ -2,13 +2,13 @@ use std::io::Write;
use std::path::Path;
use obikseq::{CanonicalKmer, Kmer, Sequence};
use obilayeredmap::MphfLayer;
use obikindex::layer::MphfLayer;
use obisys::Reporter;
use tempfile::tempdir;
use obikderep::Dereplicator;
use obikindex::algorithms::dereplicator::Dereplicator;
use obikindex::algorithms::partitionner::PartitionRouter;
use obikindex::{GenomeInfo, IndexConfig, KmerIndex, MergeMode};
use obikpartitionner::PartitionRouter;
use super::alignment::SnpAlignmentExt;
use super::build::SiblingAnnexBuildExt;
@@ -63,7 +63,7 @@ fn build_single_genome_index(dir: &Path, label: &str, seq: &[u8]) -> KmerIndex {
minimizer_size: M,
n_bits: 0, // 1 partition — keeps the test deterministic and simple
with_counts: false,
evidence: obilayeredmap::IndexMode::Exact,
evidence: obikindex::layer::IndexMode::Exact,
block_bits: 0,
};
let mut idx = KmerIndex::create(&index_path, config, Some(GenomeInfo::new(label)))
@@ -169,7 +169,7 @@ fn sibling_annex_works_after_pack_sparse() {
// is_multi.prsb`) and the generic `TypedLayer<D>` methods it relies on
// actually round-trip through the real build pipeline, not just the
// unit-level `TypedLayer<PersistentSparseBitMatrix>` tests in
// `obilayeredmap`.
// `obikindex::layer`.
let dir = tempdir().unwrap();
let g1 = build_single_genome_index(dir.path(), "g1", b"AACCGCTTAAG");
let g2 = build_single_genome_index(dir.path(), "g2", b"AACCGGTTAAG");
src/obilayeredmap/Cargo.toml
-22
View File
@@ -1,22 +0,0 @@
[package]
name = "obilayeredmap"
version = "0.1.0"
edition = "2024"
[dependencies]
obikseq = { path = "../obikseq" }
obiskio = { path = "../obiskio" }
obicompactvec = { path = "../obicompactvec" }
ptr_hash = "1.1"
cacheline-ef = "1.1"
epserde = "0.8"
rayon = "1"
ndarray = "0.17"
bitvec = "1"
memmap2 = "0.9"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
[dev-dependencies]
tempfile = "3"
obikseq = { path = "../obikseq", features = ["test-utils"] }