feat: enable index resumption and enforce directory creation

The command now supports reopening existing indexes instead of failing when the output file exists. Control flow branches between opening an existing index and constructing a new one, moving configuration setup exclusively to the creation path. Directory existence is enforced upfront with proper I/O error propagation. The --force flag retains its original semantics by removing the target directory before proceeding with a fresh build.
This commit is contained in:
Eric Coissac
2026-08-21 05:06:38 +02:00
parent abc51c2add
commit 5c1584967f
161 changed files with 5274 additions and 845 deletions
-3
View File
@@ -6,11 +6,9 @@ edition = "2024"
[dependencies]
obikindex = { path = "../obikindex", default-features = false }
obikseq = { path = "../obikseq" }
obikpartitionner = { path = "../obikpartitionner" }
obiskio = { path = "../obiskio" }
obisys = { path = "../obisys" }
obicompactvec = { path = "../obicompactvec" }
obilayeredmap = { path = "../obilayeredmap" }
obiskbuilder = { path = "../obiskbuilder" }
obipipeline = { path = "../obipipeline" }
memmap2 = "0.9"
@@ -21,6 +19,5 @@ tracing = "0.1.44"
[dev-dependencies]
obiread = { path = "../obiread" }
obikderep = { path = "../obikderep" }
tempfile = "3"
tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] }
+1 -1
View File
@@ -1,5 +1,5 @@
//! Library-level phylogenetic functionality for `obikmer`, built as
//! extension traits over `obikindex::KmerIndex` and `obilayeredmap`'s
//! extension traits over `obikindex::KmerIndex` and `obikindex::layer`'s
//! generic layer types — the `phylo` CLI command is a consumer of this
//! crate, not the owner of this logic (see `DevDocMD/architecture/siblings.md`).
//!
+2 -2
View File
@@ -5,8 +5,8 @@ use std::sync::atomic::Ordering;
use rayon::prelude::*;
use obikseq::CanonicalKmer;
use obilayeredmap::MphfLayer;
use obilayeredmap::meta::IndexMode;
use obikindex::layer::MphfLayer;
use obikindex::layer::meta::IndexMode;
use obipipeline::ThrottleGuard;
use obisys::progress_bar;
+2 -2
View File
@@ -1,7 +1,7 @@
use rayon::prelude::*;
use obikseq::CanonicalKmer;
use obilayeredmap::Layer;
use obikindex::layer::Layer;
use obisys::progress_bar;
use obikindex::{KmerIndex, OKIResult};
@@ -24,7 +24,7 @@ use obikindex::{KmerIndex, OKIResult};
/// partition for the entire run, regardless of how many source layers or
/// lookups follow.
///
/// `mats[partition][layer]` holds `obilayeredmap::Layer` — the
/// `mats[partition][layer]` holds `obikindex::layer::Layer` — the
/// format-erased `Count`/`Presence` handle, not a sibling-specific type:
/// this module used to bundle its own `Mat` enum here, duplicating exactly
/// what `Layer` now does one crate down (see
+2 -2
View File
@@ -54,13 +54,13 @@ use std::sync::atomic::{AtomicU8, Ordering};
use rayon::prelude::*;
use obikseq::CanonicalKmer;
use obilayeredmap::meta::PartitionMeta;
use obikindex::layer::meta::PartitionMeta;
use obipipeline::{ThrottleGuard, throttle};
use obikindex::{OKIError, OKIResult};
use obikindex::KmerIndex;
use obilayeredmap::Layer;
use obikindex::layer::Layer;
use super::cache::PartitionCache;
use super::helpers::central_base;
+18 -18
View File
@@ -1,6 +1,6 @@
//! Phylo/sibling-domain iteration over a layer — an extension trait, not a
//! new field on `MphfLayer`/`TypedLayer<D>`: "family"/"minorant" are phylo
//! concepts, `obilayeredmap` stays kmer/slot-mapping only (see
//! concepts, `obikindex::layer` stays kmer/slot-mapping only (see
//! `DevDocMD/architecture/siblings.md`).
//!
//! The sibling annex is persisted in the same order as `iter_kmers()`
@@ -12,7 +12,7 @@
//! batch — never collected whole into memory (see the project's "no full
//! collect" rule).
//!
//! Four iterator types, deliberately mirroring `obilayeredmap`'s own
//! Four iterator types, deliberately mirroring `obikindex::layer`'s own
//! `KmerIter`/`KmerBatchIter` pair (single item vs. `Vec` batch) — plus the
//! minorant-filtered variant of each, since "all siblings" and "one row per
//! family" are both common cases:
@@ -29,7 +29,7 @@
use std::sync::Arc;
use obikseq::CanonicalKmer;
use obilayeredmap::{KmerIter, TypedLayer, LayerData};
use obikindex::layer::{KmerIter, TypedLayer, LayerData};
use super::{FamilyMask, SiblingAnnex};
@@ -172,42 +172,42 @@ impl<D: LayerData> SiblingLayerExt for TypedLayer<D> {
}
}
/// Same extension, over `obilayeredmap::Layer` (the format-erased
/// Same extension, over `obikindex::layer::Layer` (the format-erased
/// `Count`/`Presence` handle `PartitionCache` actually holds) — dispatch
/// only, both arms return the same concrete iterator types (they don't
/// depend on which `D` is inside), so no boxing is needed. `obilayeredmap`
/// depend on which `D` is inside), so no boxing is needed. `obikindex::layer`
/// itself can't implement this: "family"/"minorant" are phylo concepts, it
/// stays kmer/slot-mapping only (see the module docs above).
impl SiblingLayerExt for obilayeredmap::Layer {
impl SiblingLayerExt for obikindex::layer::Layer {
fn iter_siblings(&self, annex: Arc<SiblingAnnex>) -> SiblingIter {
match self {
obilayeredmap::Layer::Count(l) => l.iter_siblings(annex),
obilayeredmap::Layer::Presence(l) => l.iter_siblings(annex),
obilayeredmap::Layer::Empty { .. } => panic!("iter_siblings() called on an Empty layer"),
obikindex::layer::Layer::Count(l) => l.iter_siblings(annex),
obikindex::layer::Layer::Presence(l) => l.iter_siblings(annex),
obikindex::layer::Layer::Empty { .. } => panic!("iter_siblings() called on an Empty layer"),
}
}
fn iter_siblings_batch(&self, annex: Arc<SiblingAnnex>, batch_size: usize) -> SiblingBatchIter {
match self {
obilayeredmap::Layer::Count(l) => l.iter_siblings_batch(annex, batch_size),
obilayeredmap::Layer::Presence(l) => l.iter_siblings_batch(annex, batch_size),
obilayeredmap::Layer::Empty { .. } => panic!("iter_siblings_batch() called on an Empty layer"),
obikindex::layer::Layer::Count(l) => l.iter_siblings_batch(annex, batch_size),
obikindex::layer::Layer::Presence(l) => l.iter_siblings_batch(annex, batch_size),
obikindex::layer::Layer::Empty { .. } => panic!("iter_siblings_batch() called on an Empty layer"),
}
}
fn iter_minorants(&self, annex: Arc<SiblingAnnex>) -> MinorantIter {
match self {
obilayeredmap::Layer::Count(l) => l.iter_minorants(annex),
obilayeredmap::Layer::Presence(l) => l.iter_minorants(annex),
obilayeredmap::Layer::Empty { .. } => panic!("iter_minorants() called on an Empty layer"),
obikindex::layer::Layer::Count(l) => l.iter_minorants(annex),
obikindex::layer::Layer::Presence(l) => l.iter_minorants(annex),
obikindex::layer::Layer::Empty { .. } => panic!("iter_minorants() called on an Empty layer"),
}
}
fn iter_minorants_batch(&self, annex: Arc<SiblingAnnex>, batch_size: usize) -> MinorantBatchIter {
match self {
obilayeredmap::Layer::Count(l) => l.iter_minorants_batch(annex, batch_size),
obilayeredmap::Layer::Presence(l) => l.iter_minorants_batch(annex, batch_size),
obilayeredmap::Layer::Empty { .. } => panic!("iter_minorants_batch() called on an Empty layer"),
obikindex::layer::Layer::Count(l) => l.iter_minorants_batch(annex, batch_size),
obikindex::layer::Layer::Presence(l) => l.iter_minorants_batch(annex, batch_size),
obikindex::layer::Layer::Empty { .. } => panic!("iter_minorants_batch() called on an Empty layer"),
}
}
}
+1 -1
View File
@@ -71,7 +71,7 @@ pub(crate) use siblingannex::{FamilyMask, SiblingAnnex, SiblingAnnexBuilder};
pub use stats::{SiblingAnnexStats, SiblingStatsExt};
pub use subsample::EntropyBias;
use obilayeredmap::OLMError;
use obikindex::layer::OLMError;
use obikindex::OKIError;
+5 -5
View File
@@ -2,13 +2,13 @@ use std::io::Write;
use std::path::Path;
use obikseq::{CanonicalKmer, Kmer, Sequence};
use obilayeredmap::MphfLayer;
use obikindex::layer::MphfLayer;
use obisys::Reporter;
use tempfile::tempdir;
use obikderep::Dereplicator;
use obikindex::algorithms::dereplicator::Dereplicator;
use obikindex::algorithms::partitionner::PartitionRouter;
use obikindex::{GenomeInfo, IndexConfig, KmerIndex, MergeMode};
use obikpartitionner::PartitionRouter;
use super::alignment::SnpAlignmentExt;
use super::build::SiblingAnnexBuildExt;
@@ -63,7 +63,7 @@ fn build_single_genome_index(dir: &Path, label: &str, seq: &[u8]) -> KmerIndex {
minimizer_size: M,
n_bits: 0, // 1 partition — keeps the test deterministic and simple
with_counts: false,
evidence: obilayeredmap::IndexMode::Exact,
evidence: obikindex::layer::IndexMode::Exact,
block_bits: 0,
};
let mut idx = KmerIndex::create(&index_path, config, Some(GenomeInfo::new(label)))
@@ -169,7 +169,7 @@ fn sibling_annex_works_after_pack_sparse() {
// is_multi.prsb`) and the generic `TypedLayer<D>` methods it relies on
// actually round-trip through the real build pipeline, not just the
// unit-level `TypedLayer<PersistentSparseBitMatrix>` tests in
// `obilayeredmap`.
// `obikindex::layer`.
let dir = tempdir().unwrap();
let g1 = build_single_genome_index(dir.path(), "g1", b"AACCGCTTAAG");
let g2 = build_single_genome_index(dir.path(), "g2", b"AACCGGTTAAG");