feat: enable index resumption and enforce directory creation
The command now supports reopening existing indexes instead of failing when the output file exists. Control flow branches between opening an existing index and constructing a new one, moving configuration setup exclusively to the creation path. Directory existence is enforced upfront with proper I/O error propagation. The --force flag retains its original semantics by removing the target directory before proceeding with a fresh build.
This commit is contained in:
@@ -6,11 +6,9 @@ edition = "2024"
|
||||
[dependencies]
|
||||
obikindex = { path = "../obikindex", default-features = false }
|
||||
obikseq = { path = "../obikseq" }
|
||||
obikpartitionner = { path = "../obikpartitionner" }
|
||||
obiskio = { path = "../obiskio" }
|
||||
obisys = { path = "../obisys" }
|
||||
obicompactvec = { path = "../obicompactvec" }
|
||||
obilayeredmap = { path = "../obilayeredmap" }
|
||||
obiskbuilder = { path = "../obiskbuilder" }
|
||||
obipipeline = { path = "../obipipeline" }
|
||||
memmap2 = "0.9"
|
||||
@@ -21,6 +19,5 @@ tracing = "0.1.44"
|
||||
|
||||
[dev-dependencies]
|
||||
obiread = { path = "../obiread" }
|
||||
obikderep = { path = "../obikderep" }
|
||||
tempfile = "3"
|
||||
tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] }
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
//! Library-level phylogenetic functionality for `obikmer`, built as
|
||||
//! extension traits over `obikindex::KmerIndex` and `obilayeredmap`'s
|
||||
//! extension traits over `obikindex::KmerIndex` and `obikindex::layer`'s
|
||||
//! generic layer types — the `phylo` CLI command is a consumer of this
|
||||
//! crate, not the owner of this logic (see `DevDocMD/architecture/siblings.md`).
|
||||
//!
|
||||
|
||||
@@ -5,8 +5,8 @@ use std::sync::atomic::Ordering;
|
||||
use rayon::prelude::*;
|
||||
|
||||
use obikseq::CanonicalKmer;
|
||||
use obilayeredmap::MphfLayer;
|
||||
use obilayeredmap::meta::IndexMode;
|
||||
use obikindex::layer::MphfLayer;
|
||||
use obikindex::layer::meta::IndexMode;
|
||||
use obipipeline::ThrottleGuard;
|
||||
use obisys::progress_bar;
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use rayon::prelude::*;
|
||||
|
||||
use obikseq::CanonicalKmer;
|
||||
use obilayeredmap::Layer;
|
||||
use obikindex::layer::Layer;
|
||||
use obisys::progress_bar;
|
||||
|
||||
use obikindex::{KmerIndex, OKIResult};
|
||||
@@ -24,7 +24,7 @@ use obikindex::{KmerIndex, OKIResult};
|
||||
/// partition for the entire run, regardless of how many source layers or
|
||||
/// lookups follow.
|
||||
///
|
||||
/// `mats[partition][layer]` holds `obilayeredmap::Layer` — the
|
||||
/// `mats[partition][layer]` holds `obikindex::layer::Layer` — the
|
||||
/// format-erased `Count`/`Presence` handle, not a sibling-specific type:
|
||||
/// this module used to bundle its own `Mat` enum here, duplicating exactly
|
||||
/// what `Layer` now does one crate down (see
|
||||
|
||||
@@ -54,13 +54,13 @@ use std::sync::atomic::{AtomicU8, Ordering};
|
||||
use rayon::prelude::*;
|
||||
|
||||
use obikseq::CanonicalKmer;
|
||||
use obilayeredmap::meta::PartitionMeta;
|
||||
use obikindex::layer::meta::PartitionMeta;
|
||||
use obipipeline::{ThrottleGuard, throttle};
|
||||
|
||||
use obikindex::{OKIError, OKIResult};
|
||||
use obikindex::KmerIndex;
|
||||
|
||||
use obilayeredmap::Layer;
|
||||
use obikindex::layer::Layer;
|
||||
|
||||
use super::cache::PartitionCache;
|
||||
use super::helpers::central_base;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! Phylo/sibling-domain iteration over a layer — an extension trait, not a
|
||||
//! new field on `MphfLayer`/`TypedLayer<D>`: "family"/"minorant" are phylo
|
||||
//! concepts, `obilayeredmap` stays kmer/slot-mapping only (see
|
||||
//! concepts, `obikindex::layer` stays kmer/slot-mapping only (see
|
||||
//! `DevDocMD/architecture/siblings.md`).
|
||||
//!
|
||||
//! The sibling annex is persisted in the same order as `iter_kmers()`
|
||||
@@ -12,7 +12,7 @@
|
||||
//! batch — never collected whole into memory (see the project's "no full
|
||||
//! collect" rule).
|
||||
//!
|
||||
//! Four iterator types, deliberately mirroring `obilayeredmap`'s own
|
||||
//! Four iterator types, deliberately mirroring `obikindex::layer`'s own
|
||||
//! `KmerIter`/`KmerBatchIter` pair (single item vs. `Vec` batch) — plus the
|
||||
//! minorant-filtered variant of each, since "all siblings" and "one row per
|
||||
//! family" are both common cases:
|
||||
@@ -29,7 +29,7 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use obikseq::CanonicalKmer;
|
||||
use obilayeredmap::{KmerIter, TypedLayer, LayerData};
|
||||
use obikindex::layer::{KmerIter, TypedLayer, LayerData};
|
||||
|
||||
use super::{FamilyMask, SiblingAnnex};
|
||||
|
||||
@@ -172,42 +172,42 @@ impl<D: LayerData> SiblingLayerExt for TypedLayer<D> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Same extension, over `obilayeredmap::Layer` (the format-erased
|
||||
/// Same extension, over `obikindex::layer::Layer` (the format-erased
|
||||
/// `Count`/`Presence` handle `PartitionCache` actually holds) — dispatch
|
||||
/// only, both arms return the same concrete iterator types (they don't
|
||||
/// depend on which `D` is inside), so no boxing is needed. `obilayeredmap`
|
||||
/// depend on which `D` is inside), so no boxing is needed. `obikindex::layer`
|
||||
/// itself can't implement this: "family"/"minorant" are phylo concepts, it
|
||||
/// stays kmer/slot-mapping only (see the module docs above).
|
||||
impl SiblingLayerExt for obilayeredmap::Layer {
|
||||
impl SiblingLayerExt for obikindex::layer::Layer {
|
||||
fn iter_siblings(&self, annex: Arc<SiblingAnnex>) -> SiblingIter {
|
||||
match self {
|
||||
obilayeredmap::Layer::Count(l) => l.iter_siblings(annex),
|
||||
obilayeredmap::Layer::Presence(l) => l.iter_siblings(annex),
|
||||
obilayeredmap::Layer::Empty { .. } => panic!("iter_siblings() called on an Empty layer"),
|
||||
obikindex::layer::Layer::Count(l) => l.iter_siblings(annex),
|
||||
obikindex::layer::Layer::Presence(l) => l.iter_siblings(annex),
|
||||
obikindex::layer::Layer::Empty { .. } => panic!("iter_siblings() called on an Empty layer"),
|
||||
}
|
||||
}
|
||||
|
||||
fn iter_siblings_batch(&self, annex: Arc<SiblingAnnex>, batch_size: usize) -> SiblingBatchIter {
|
||||
match self {
|
||||
obilayeredmap::Layer::Count(l) => l.iter_siblings_batch(annex, batch_size),
|
||||
obilayeredmap::Layer::Presence(l) => l.iter_siblings_batch(annex, batch_size),
|
||||
obilayeredmap::Layer::Empty { .. } => panic!("iter_siblings_batch() called on an Empty layer"),
|
||||
obikindex::layer::Layer::Count(l) => l.iter_siblings_batch(annex, batch_size),
|
||||
obikindex::layer::Layer::Presence(l) => l.iter_siblings_batch(annex, batch_size),
|
||||
obikindex::layer::Layer::Empty { .. } => panic!("iter_siblings_batch() called on an Empty layer"),
|
||||
}
|
||||
}
|
||||
|
||||
fn iter_minorants(&self, annex: Arc<SiblingAnnex>) -> MinorantIter {
|
||||
match self {
|
||||
obilayeredmap::Layer::Count(l) => l.iter_minorants(annex),
|
||||
obilayeredmap::Layer::Presence(l) => l.iter_minorants(annex),
|
||||
obilayeredmap::Layer::Empty { .. } => panic!("iter_minorants() called on an Empty layer"),
|
||||
obikindex::layer::Layer::Count(l) => l.iter_minorants(annex),
|
||||
obikindex::layer::Layer::Presence(l) => l.iter_minorants(annex),
|
||||
obikindex::layer::Layer::Empty { .. } => panic!("iter_minorants() called on an Empty layer"),
|
||||
}
|
||||
}
|
||||
|
||||
fn iter_minorants_batch(&self, annex: Arc<SiblingAnnex>, batch_size: usize) -> MinorantBatchIter {
|
||||
match self {
|
||||
obilayeredmap::Layer::Count(l) => l.iter_minorants_batch(annex, batch_size),
|
||||
obilayeredmap::Layer::Presence(l) => l.iter_minorants_batch(annex, batch_size),
|
||||
obilayeredmap::Layer::Empty { .. } => panic!("iter_minorants_batch() called on an Empty layer"),
|
||||
obikindex::layer::Layer::Count(l) => l.iter_minorants_batch(annex, batch_size),
|
||||
obikindex::layer::Layer::Presence(l) => l.iter_minorants_batch(annex, batch_size),
|
||||
obikindex::layer::Layer::Empty { .. } => panic!("iter_minorants_batch() called on an Empty layer"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -71,7 +71,7 @@ pub(crate) use siblingannex::{FamilyMask, SiblingAnnex, SiblingAnnexBuilder};
|
||||
pub use stats::{SiblingAnnexStats, SiblingStatsExt};
|
||||
pub use subsample::EntropyBias;
|
||||
|
||||
use obilayeredmap::OLMError;
|
||||
use obikindex::layer::OLMError;
|
||||
|
||||
use obikindex::OKIError;
|
||||
|
||||
|
||||
@@ -2,13 +2,13 @@ use std::io::Write;
|
||||
use std::path::Path;
|
||||
|
||||
use obikseq::{CanonicalKmer, Kmer, Sequence};
|
||||
use obilayeredmap::MphfLayer;
|
||||
use obikindex::layer::MphfLayer;
|
||||
use obisys::Reporter;
|
||||
use tempfile::tempdir;
|
||||
|
||||
use obikderep::Dereplicator;
|
||||
use obikindex::algorithms::dereplicator::Dereplicator;
|
||||
use obikindex::algorithms::partitionner::PartitionRouter;
|
||||
use obikindex::{GenomeInfo, IndexConfig, KmerIndex, MergeMode};
|
||||
use obikpartitionner::PartitionRouter;
|
||||
|
||||
use super::alignment::SnpAlignmentExt;
|
||||
use super::build::SiblingAnnexBuildExt;
|
||||
@@ -63,7 +63,7 @@ fn build_single_genome_index(dir: &Path, label: &str, seq: &[u8]) -> KmerIndex {
|
||||
minimizer_size: M,
|
||||
n_bits: 0, // 1 partition — keeps the test deterministic and simple
|
||||
with_counts: false,
|
||||
evidence: obilayeredmap::IndexMode::Exact,
|
||||
evidence: obikindex::layer::IndexMode::Exact,
|
||||
block_bits: 0,
|
||||
};
|
||||
let mut idx = KmerIndex::create(&index_path, config, Some(GenomeInfo::new(label)))
|
||||
@@ -169,7 +169,7 @@ fn sibling_annex_works_after_pack_sparse() {
|
||||
// is_multi.prsb`) and the generic `TypedLayer<D>` methods it relies on
|
||||
// actually round-trip through the real build pipeline, not just the
|
||||
// unit-level `TypedLayer<PersistentSparseBitMatrix>` tests in
|
||||
// `obilayeredmap`.
|
||||
// `obikindex::layer`.
|
||||
let dir = tempdir().unwrap();
|
||||
let g1 = build_single_genome_index(dir.path(), "g1", b"AACCGCTTAAG");
|
||||
let g2 = build_single_genome_index(dir.path(), "g2", b"AACCGGTTAAG");
|
||||
|
||||
Reference in New Issue
Block a user