OBIKmers/obikmer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 34 Wiki Activity

feat: add convert command for in-place index evidence modification

Browse Source 
Introduce a new CLI command that modifies existing indices to support exact, approximate, or hybrid evidence modes. This change enables the reindex module in obikrebuild, exposing an IndexReindex trait for evidence persistence and layer processing. It also refactors obikindexer to publicly expose an IndexBuilder trait for post-build evidence construction, while adding a set_evidence API in obikindex to safely update configuration fields without altering core parameters.
This commit is contained in:
Eric Coissac
2026-08-28 19:43:14 +02:00
parent bd7729b095
commit 4ea3cd32ba
10 changed files with 562 additions and 340 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/obikindex/src/index/meta.rs
+22 -23
View File
@@ -218,29 +218,28 @@ impl IndexMeta {
self.write_full(&on_disk)
}
// /// Overwrite `config` and the whole `genomes` list at once, preserving
// /// `state`. `config` otherwise never changes once an index exists —
// /// this is the deliberate, rare exception for construction paths that
// /// legitimately rewrite it in place (`select_in_place`, `reindex`).
// /// The caller must refresh its own cached `Arc<IndexMeta>` afterward
// /// (e.g. `self.meta = Arc::new(IndexMeta::open(self)?)`) — this method
// /// only updates the file, it has no way to reach back into whatever
// /// `KmerIndex` holds it.
// /// TODO: This methode is strange
// pub(crate) fn rewrite_config(
// &self,
// config: IndexConfig,
// genomes: Vec<GenomeInfo>,
// ) -> io::Result<()> {
// let _guard = self.lock.write().unwrap();
// let state = Self::read_full(&self.root_path)?.state;
// self.write_full(&IndexMetadata {
// version: self.version,
// config,
// genomes,
// state,
// })
// }
/// Update the evidence representation in place — `evidence`, and (for
/// `Exact`) its block size. The only `IndexConfig` fields a reindex is
/// ever allowed to touch: `kmer_size`/`minimizer_size`/`n_bits`/
/// `with_counts` are baked into the already-built MPHF/unitigs/matrices
/// and can never change post-build, unlike the evidence bundle, which
/// is derived from `unitigs.bin` + `mphf.bin` alone and can be
/// rebuilt in a different form at any time (see
/// `obikindexer::IndexBuilder`).
///
/// The caller must refresh its own cached `Arc<IndexMeta>` afterward
/// (e.g. `self.meta = Arc::new(IndexMeta::open(self)?)`) — this method
/// only updates the file, it has no way to reach back into whatever
/// `KmerIndex` holds it.
pub fn set_evidence(&self, evidence: IndexMode, block_bits: u8) -> io::Result<()> {
let _guard = self.lock.write().unwrap();
let mut on_disk = Self::read_full(&self.root_path)?;
on_disk.config.evidence = evidence;
if matches!(on_disk.config.evidence, IndexMode::Exact) {
on_disk.config.block_bits = block_bits;
}
self.write_full(&on_disk)
}
pub fn set_state(&self, state: IndexState) -> io::Result<()> {
let _guard = self.lock.write().unwrap();
src/obikindexer/src/extensions/index_builder.rs
+68
View File
@@ -0,0 +1,68 @@
//! `IndexBuilder` — public, idempotent evidence-construction primitives
//! for an already-built `KmerLayer` (`unitigs.bin` + `mphf.bin` already on
//! disk; only the evidence bundle — `evidence.bin`/`unitigs.bin.idx` and/or
//! `fingerprint.bin` — is touched). Unlike `PrivateBuilder` (this module's
//! private sibling), these are meant for callers outside this crate that
//! need to add or convert a layer's evidence after the fact — today,
//! `obikrebuild::reindex`.
//!
//! `KmerLayer` itself (`obikindex`) stays a pure data-model type: it
//! exposes paths and reads content, it never builds anything. Construction
//! lives here instead, in the crate whose whole purpose is building index
//! content — same reasoning as `PrivateBuilder`, just public.
use obikindex::OKIResult;
use obikindex::layer::fingerprint::FingerprintVec;
use obikindex::layer::{KmerLayer, MphfLayer};
pub trait IndexBuilder {
/// Build this layer's exact evidence bundle (`evidence.bin` +
/// `unitigs.bin.idx`) if not already present.
///
/// No-op (`Ok(0)`) if exact evidence already exists — both `Exact` and
/// `Hybrid` layers already carry it.
fn build_exact_evidence(&self, block_bits: u8) -> OKIResult<usize>;
/// Build this layer's approximate (fingerprint) evidence with `b` bits
/// per slot and Findere parameter `z`, if not already present with a
/// matching `b`.
///
/// No-op (`Ok(0)`) if `Approx`/`Hybrid` evidence with the same `b`
/// already exists — rebuilds it (same file, same `b`, whatever `z`)
/// otherwise. `z` is never persisted (see `IndexMode::Approx`'s own
/// doc): it only governs how many consecutive stored k-mers a query
/// must match, so changing it alone is not something this method has
/// any file-level evidence to detect — the caller decides.
fn build_approx_evidence(&self, b: u8, z: u8) -> OKIResult<usize>;
/// Build whichever of the exact/approximate bundles this layer is
/// still missing, so it ends up carrying both (`Hybrid`).
///
/// Just the composition of `build_exact_evidence` and
/// `build_approx_evidence` — each is already independently idempotent,
/// so there is no extra "what's already there" decision to make here.
fn build_hybrid_evidence(&self, b: u8, z: u8, block_bits: u8) -> OKIResult<usize>;
}
impl IndexBuilder for KmerLayer {
fn build_exact_evidence(&self, block_bits: u8) -> OKIResult<usize> {
if self.evidence_path().exists() {
return Ok(0);
}
MphfLayer::build_exact_evidence(self.dir(), block_bits)
}
fn build_approx_evidence(&self, b: u8, z: u8) -> OKIResult<usize> {
let fingerprint_path = self.fingerprint_path();
if fingerprint_path.exists() && FingerprintVec::open(&fingerprint_path)?.b() == b {
return Ok(0);
}
MphfLayer::build_approx_evidence(self.dir(), b, z)
}
fn build_hybrid_evidence(&self, b: u8, z: u8, block_bits: u8) -> OKIResult<usize> {
let n_exact = self.build_exact_evidence(block_bits)?;
let n_approx = self.build_approx_evidence(b, z)?;
Ok(n_exact.max(n_approx))
}
}
src/obikindexer/src/extensions/mod.rs
+17 -242
View File
@@ -1,245 +1,20 @@
//! Construction-only `KmerIndex` operations — semantically meaningless
//! once an index is built and being used for reading (query, dump, phylo,
//! or as a `merge`/`select`/`rebuild` *source*): sentinel marking, the
//! per-partition layer-build primitive, and cleanup of build-time scratch
//! files. Private to this crate — no consumer outside the
//! indexing-pipeline algorithms in `obikindexer::algorithms` should ever
//! need them. See `DevDocMD/implementation/partition_layer_cache.md`,
//! "(8)".
//! Extension traits over `obikindex`'s foreign types (`KmerIndex`,
//! `KmerLayer`) — Rust's orphan rule requires such traits to be defined in
//! the crate that implements them for a foreign type, not in `obikindex`
//! itself; `obikindex` has no way to make a trait "visible to only some
//! crates" from its own side.
//!
//! Defined here, not in `obikindex`, so it can be genuinely `pub(crate)`
//! while extending a foreign type (`KmerIndex`) — Rust's orphan rule
//! requires the trait itself to be local to the crate that implements it
//! for a foreign type; `obikindex` has no way to make a trait "private
//! except to `obikindexer`" from its own side.
//!
//! Scoped to the six methods exclusive to this crate's own pipeline
//! (`partitionner`/`dereplicator`/`counter`/`layer_builder`).
//! `clear_output_for_create`/`create_skeleton`/`finalize_indexed`/`state`
//! stay inherent on `KmerIndex`, deliberately not moved here: they're also
//! called from `obikindex`'s own `merge`/`select`/`rebuild`/`reindex`
//! modules, which — living inside `obikindex` itself — could never reach a
//! trait defined in `obikindexer` (the dependency only runs
//! `obikindexer → obikindex`, never the other way). Moving those four
//! would require extracting `merge`/`select`/`rebuild`/`reindex` into
//! algorithms first — separate, larger future work, not this one.
//! Two traits, two visibilities, one per submodule:
//! - [`private_builder::PrivateBuilder`] (`pub(crate)`) — construction-only
//! `KmerIndex` operations exclusive to this crate's own indexing
//! pipeline (`algorithms::{partitionner,dereplicator,counter,layer_builder}`).
//! - [`index_builder::IndexBuilder`] (`pub`, re-exported at the crate
//! root) — idempotent per-layer evidence-construction primitives, for
//! any external caller that needs to add/convert a layer's evidence
//! bundle after the fact (`obikrebuild::reindex`).
use std::collections::BTreeMap;
use std::fs;
use std::io;
use std::path::Path;
mod index_builder;
mod private_builder;
use cacheline_ef::{CachelineEf, CachelineEfVec};
use epserde::prelude::*;
use obicompactvec::{PersistentCompactIntMatrix, PersistentCompactIntVec};
use obidebruinj::GraphDeBruijn;
use obikindex::layer::IndexMode;
use obikindex::layer::{KmerLayer, TypedLayer};
use obikindex::{KmerIndex, OKIError, OKIResult};
use crate::graph_pipeline::{materialize_layer, write_graph_as_unitigs};
use obiskio::{SKError, SKFileMeta, SKFileReader};
use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xx64};
type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xx64, Vec<u8>>;
pub(crate) trait PrivateBuilder {
/// Mark scatter as complete (`IndexState::Scattered`).
///
/// If no genome label was set at creation time, one is derived from
/// the index root directory name (stripped of all extensions). `&self`
/// — not `&mut self` anymore: nothing is cached in memory to mutate,
/// see `DevDocMD/implementation/partition_layer_cache.md`, "(11)".
fn mark_scattered(&self) -> OKIResult<()>;
/// Mark dereplicate+count as complete (`IndexState::Counted`).
fn mark_counted(&self) -> OKIResult<()>;
/// Partition `i`'s layer 0 — the one and only layer every pipeline
/// algorithm in this crate (`partitionner`/`dereplicator`/`counter`/
/// `layer_builder`) reads from or writes to; later layers only exist
/// after a `merge`, which lives in `obikindex`, not here.
fn layer0(&self, i: usize) -> OKIResult<KmerLayer>;
/// Mark layer construction as complete (`IndexState::Indexed`).
fn mark_indexed(&self) -> OKIResult<()>;
/// Write `spectrums/{label}.json` from an already-computed kmer
/// spectrum (`f0`/`f1`/abundance histogram). Takes plain values rather
/// than `algorithms::counter::KmerSpectrum` — `KmerIndex` is the data
/// model, `Counter` the algorithm that depends on it, not the other
/// way around, and this is the only field of that type it actually
/// uses.
fn write_spectrum(&self, f0: u64, f1: u64, counts: &BTreeMap<u32, u64>) -> OKIResult<()>;
/// Build the layered MPHF index for partition `i`.
///
/// Returns the number of canonical k-mers indexed, or 0 if the
/// partition has no data or its layer was already built
/// (resume-safe).
///
/// Abundance filtering is applied when `min_ab > 1` or
/// `max_ab.is_some()`, using `mphf1.bin` + `counts1.bin` if they
/// exist. Count payload is stored iff `with_counts` is true.
fn build_index_layer(
&self,
i: usize,
min_ab: u32,
max_ab: Option<u32>,
with_counts: bool,
mode: &IndexMode,
block_bits: u8,
) -> Result<usize, SKError>;
/// Remove intermediate build artifacts for partition `i`: dereplicated
/// superkmers (+ sidecar), `mphf1.bin`, `counts1.bin`.
fn remove_build_artifacts(&self, i: usize);
}
impl PrivateBuilder for KmerIndex {
fn mark_scattered(&self) -> OKIResult<()> {
self.meta().mark_scattered().map_err(OKIError::Io)
}
fn mark_counted(&self) -> OKIResult<()> {
self.meta().mark_counted().map_err(OKIError::Io)
}
fn layer0(&self, i: usize) -> OKIResult<KmerLayer> {
self.partition(i)?.layer(0)
}
fn mark_indexed(&self) -> OKIResult<()> {
self.meta().mark_indexed().map_err(OKIError::Io)
}
fn write_spectrum(&self, f0: u64, f1: u64, counts: &BTreeMap<u32, u64>) -> OKIResult<()> {
let genomes = self.meta().genomes().map_err(OKIError::Io)?;
let label = genomes
.first()
.map(|g| g.label.as_str())
.unwrap_or("unknown")
.to_owned();
let spectrums_dir = self.dir().join("spectrums");
fs::create_dir_all(&spectrums_dir)?;
let path = spectrums_dir.join(format!("{label}.json"));
let spectrum_map: BTreeMap<String, u64> = counts
.iter()
.map(|(&c, &f)| (format!("{c:010}"), f))
.collect();
let f = fs::File::create(&path)?;
serde_json::to_writer_pretty(
f,
&serde_json::json!({ "f0": f0, "f1": f1, "spectrum": spectrum_map }),
)
.map_err(OKIError::Json)?;
Ok(())
}
fn build_index_layer(
&self,
i: usize,
min_ab: u32,
max_ab: Option<u32>,
with_counts: bool,
mode: &IndexMode,
block_bits: u8,
) -> Result<usize, SKError> {
let layer0 = self.layer0(i).map_err(|e| io::Error::other(e.to_string()))?;
let layer0_dir = layer0.dir();
let dedup_path = layer0.dereplicated_superkmers_path();
if !dedup_path.exists() {
return Ok(0);
}
if layer0_dir.join("mphf.bin").exists() {
return Ok(0);
}
let filter_active = min_ab > 1 || max_ab.is_some();
let need_counts = filter_active || with_counts;
let mphf1_opt: Option<Mphf> = if need_counts {
let p = layer0_dir.join("mphf1.bin");
p.exists().then(|| Mphf::load_full(&p).ok()).flatten()
} else {
None
};
let counts1_opt: Option<PersistentCompactIntVec> = if need_counts {
let p = layer0_dir.join("counts1.bin");
p.exists()
.then(|| PersistentCompactIntVec::open(&p).ok())
.flatten()
} else {
None
};
let mut g = GraphDeBruijn::new();
let mut reader = SKFileReader::open(&dedup_path)?;
for sk in reader.iter() {
for kmer in sk.iter_canonical_kmers() {
let accept = if filter_active {
match (&mphf1_opt, &counts1_opt) {
(Some(mphf), Some(counts)) => {
let ab = counts.get(mphf.index(&kmer.raw()));
ab >= min_ab && max_ab.map_or(true, |max| ab <= max)
}
_ => true,
}
} else {
true
};
if accept {
g.push(kmer);
}
}
}
let n_kmers = if with_counts {
let n = write_graph_as_unitigs(g, layer0_dir)
.map_err(|e| io::Error::other(e.to_string()))?;
TypedLayer::<PersistentCompactIntMatrix>::build(
layer0_dir,
block_bits,
mode,
|kmer| match (&mphf1_opt, &counts1_opt) {
(Some(mphf), Some(counts)) => counts.get(mphf.index(&kmer.raw())),
_ => 1,
},
)
.map_err(|e| io::Error::other(e.to_string()))?;
n
} else {
materialize_layer(g, layer0_dir, block_bits, mode)
.map_err(|e| io::Error::other(e.to_string()))?
};
Ok(n_kmers)
}
fn remove_build_artifacts(&self, i: usize) {
let layer0 = match self.layer0(i) {
Ok(layer) => layer,
Err(e) => {
eprintln!("warning: could not locate layer 0 of partition {i}: {e}");
return;
}
};
let layer0_dir = layer0.dir();
let dedup = layer0.dereplicated_superkmers_path();
remove_if_exists(&SKFileMeta::sidecar_path(&dedup));
remove_if_exists(&dedup);
remove_if_exists(&layer0_dir.join("mphf1.bin"));
remove_if_exists(&layer0_dir.join("counts1.bin"));
}
}
// ── private helpers ──────────────────────────────────────────────────────
fn remove_if_exists(path: &Path) {
if let Err(e) = fs::remove_file(path) {
if e.kind() != io::ErrorKind::NotFound {
eprintln!("warning: could not remove {}: {e}", path.display());
}
}
}
pub use index_builder::IndexBuilder;
pub(crate) use private_builder::PrivateBuilder;
src/obikindexer/src/extensions/private_builder.rs
+240
View File
@@ -0,0 +1,240 @@
//! `PrivateBuilder` — construction-only `KmerIndex` operations:
//! sentinel marking, the per-partition layer-build primitive, and cleanup
//! of build-time scratch files. Semantically meaningless once an index is
//! built and being used for reading (query, dump, phylo, or as a
//! `merge`/`select`/`rebuild` *source*).
//!
//! Scoped to the six methods exclusive to this crate's own pipeline
//! (`partitionner`/`dereplicator`/`counter`/`layer_builder`) — no
//! consumer outside those algorithms should ever need them, hence
//! `pub(crate)` (see the module-level orphan-rule note in `extensions`'s
//! `mod.rs`). See `DevDocMD/implementation/partition_layer_cache.md`, "(8)".
//!
//! `clear_output_for_create`/`create_skeleton`/`finalize_indexed`/`state`
//! stay inherent on `KmerIndex`, deliberately not moved here: they're also
//! called from `obikindex`'s own `merge`/`select`/`rebuild`/`reindex`
//! modules, which — living inside `obikindex` itself — could never reach a
//! trait defined in `obikindexer` (the dependency only runs
//! `obikindexer → obikindex`, never the other way). Moving those four
//! would require extracting `merge`/`select`/`rebuild`/`reindex` into
//! algorithms first — separate, larger future work, not this one.
use std::collections::BTreeMap;
use std::fs;
use std::io;
use std::path::Path;
use cacheline_ef::{CachelineEf, CachelineEfVec};
use epserde::prelude::*;
use obicompactvec::{PersistentCompactIntMatrix, PersistentCompactIntVec};
use obidebruinj::GraphDeBruijn;
use obikindex::layer::IndexMode;
use obikindex::layer::{KmerLayer, TypedLayer};
use obikindex::{KmerIndex, OKIError, OKIResult};
use crate::graph_pipeline::{materialize_layer, write_graph_as_unitigs};
use obiskio::{SKError, SKFileMeta, SKFileReader};
use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xx64};
type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xx64, Vec<u8>>;
pub(crate) trait PrivateBuilder {
/// Mark scatter as complete (`IndexState::Scattered`).
///
/// If no genome label was set at creation time, one is derived from
/// the index root directory name (stripped of all extensions). `&self`
/// — not `&mut self` anymore: nothing is cached in memory to mutate,
/// see `DevDocMD/implementation/partition_layer_cache.md`, "(11)".
fn mark_scattered(&self) -> OKIResult<()>;
/// Mark dereplicate+count as complete (`IndexState::Counted`).
fn mark_counted(&self) -> OKIResult<()>;
/// Partition `i`'s layer 0 — the one and only layer every pipeline
/// algorithm in this crate (`partitionner`/`dereplicator`/`counter`/
/// `layer_builder`) reads from or writes to; later layers only exist
/// after a `merge`, which lives in `obikindex`, not here.
fn layer0(&self, i: usize) -> OKIResult<KmerLayer>;
/// Mark layer construction as complete (`IndexState::Indexed`).
fn mark_indexed(&self) -> OKIResult<()>;
/// Write `spectrums/{label}.json` from an already-computed kmer
/// spectrum (`f0`/`f1`/abundance histogram). Takes plain values rather
/// than `algorithms::counter::KmerSpectrum` — `KmerIndex` is the data
/// model, `Counter` the algorithm that depends on it, not the other
/// way around, and this is the only field of that type it actually
/// uses.
fn write_spectrum(&self, f0: u64, f1: u64, counts: &BTreeMap<u32, u64>) -> OKIResult<()>;
/// Build the layered MPHF index for partition `i`.
///
/// Returns the number of canonical k-mers indexed, or 0 if the
/// partition has no data or its layer was already built
/// (resume-safe).
///
/// Abundance filtering is applied when `min_ab > 1` or
/// `max_ab.is_some()`, using `mphf1.bin` + `counts1.bin` if they
/// exist. Count payload is stored iff `with_counts` is true.
fn build_index_layer(
&self,
i: usize,
min_ab: u32,
max_ab: Option<u32>,
with_counts: bool,
mode: &IndexMode,
block_bits: u8,
) -> Result<usize, SKError>;
/// Remove intermediate build artifacts for partition `i`: dereplicated
/// superkmers (+ sidecar), `mphf1.bin`, `counts1.bin`.
fn remove_build_artifacts(&self, i: usize);
}
impl PrivateBuilder for KmerIndex {
fn mark_scattered(&self) -> OKIResult<()> {
self.meta().mark_scattered().map_err(OKIError::Io)
}
fn mark_counted(&self) -> OKIResult<()> {
self.meta().mark_counted().map_err(OKIError::Io)
}
fn layer0(&self, i: usize) -> OKIResult<KmerLayer> {
self.partition(i)?.layer(0)
}
fn mark_indexed(&self) -> OKIResult<()> {
self.meta().mark_indexed().map_err(OKIError::Io)
}
fn write_spectrum(&self, f0: u64, f1: u64, counts: &BTreeMap<u32, u64>) -> OKIResult<()> {
let genomes = self.meta().genomes().map_err(OKIError::Io)?;
let label = genomes
.first()
.map(|g| g.label.as_str())
.unwrap_or("unknown")
.to_owned();
let spectrums_dir = self.dir().join("spectrums");
fs::create_dir_all(&spectrums_dir)?;
let path = spectrums_dir.join(format!("{label}.json"));
let spectrum_map: BTreeMap<String, u64> = counts
.iter()
.map(|(&c, &f)| (format!("{c:010}"), f))
.collect();
let f = fs::File::create(&path)?;
serde_json::to_writer_pretty(
f,
&serde_json::json!({ "f0": f0, "f1": f1, "spectrum": spectrum_map }),
)
.map_err(OKIError::Json)?;
Ok(())
}
fn build_index_layer(
&self,
i: usize,
min_ab: u32,
max_ab: Option<u32>,
with_counts: bool,
mode: &IndexMode,
block_bits: u8,
) -> Result<usize, SKError> {
let layer0 = self.layer0(i).map_err(|e| io::Error::other(e.to_string()))?;
let layer0_dir = layer0.dir();
let dedup_path = layer0.dereplicated_superkmers_path();
if !dedup_path.exists() {
return Ok(0);
}
if layer0_dir.join("mphf.bin").exists() {
return Ok(0);
}
let filter_active = min_ab > 1 || max_ab.is_some();
let need_counts = filter_active || with_counts;
let mphf1_opt: Option<Mphf> = if need_counts {
let p = layer0_dir.join("mphf1.bin");
p.exists().then(|| Mphf::load_full(&p).ok()).flatten()
} else {
None
};
let counts1_opt: Option<PersistentCompactIntVec> = if need_counts {
let p = layer0_dir.join("counts1.bin");
p.exists()
.then(|| PersistentCompactIntVec::open(&p).ok())
.flatten()
} else {
None
};
let mut g = GraphDeBruijn::new();
let mut reader = SKFileReader::open(&dedup_path)?;
for sk in reader.iter() {
for kmer in sk.iter_canonical_kmers() {
let accept = if filter_active {
match (&mphf1_opt, &counts1_opt) {
(Some(mphf), Some(counts)) => {
let ab = counts.get(mphf.index(&kmer.raw()));
ab >= min_ab && max_ab.map_or(true, |max| ab <= max)
}
_ => true,
}
} else {
true
};
if accept {
g.push(kmer);
}
}
}
let n_kmers = if with_counts {
let n = write_graph_as_unitigs(g, layer0_dir)
.map_err(|e| io::Error::other(e.to_string()))?;
TypedLayer::<PersistentCompactIntMatrix>::build(
layer0_dir,
block_bits,
mode,
|kmer| match (&mphf1_opt, &counts1_opt) {
(Some(mphf), Some(counts)) => counts.get(mphf.index(&kmer.raw())),
_ => 1,
},
)
.map_err(|e| io::Error::other(e.to_string()))?;
n
} else {
materialize_layer(g, layer0_dir, block_bits, mode)
.map_err(|e| io::Error::other(e.to_string()))?
};
Ok(n_kmers)
}
fn remove_build_artifacts(&self, i: usize) {
let layer0 = match self.layer0(i) {
Ok(layer) => layer,
Err(e) => {
eprintln!("warning: could not locate layer 0 of partition {i}: {e}");
return;
}
};
let layer0_dir = layer0.dir();
let dedup = layer0.dereplicated_superkmers_path();
remove_if_exists(&SKFileMeta::sidecar_path(&dedup));
remove_if_exists(&dedup);
remove_if_exists(&layer0_dir.join("mphf1.bin"));
remove_if_exists(&layer0_dir.join("counts1.bin"));
}
}
// ── private helpers ──────────────────────────────────────────────────────
fn remove_if_exists(path: &Path) {
if let Err(e) = fs::remove_file(path) {
if e.kind() != io::ErrorKind::NotFound {
eprintln!("warning: could not remove {}: {e}", path.display());
}
}
}
src/obikindexer/src/lib.rs
+6 -3
View File
@@ -9,11 +9,14 @@
//! (routing raw super-kmers into partitions), `dereplicator` (deduplicating
//! a partition's raw super-kmers), `counter` (counting unique canonical
//! kmers), `layer_builder` (turning dereplicated super-kmers + counts into
//! the real layer 0). [`extensions`] (private — see its own module docs)
//! holds construction-only `KmerIndex` operations these algorithms share.
//! the real layer 0). `extensions` (module itself private — see its own
//! docs) holds `KmerIndex`/`KmerLayer` extension traits: some exclusive to
//! this crate's own pipeline, one ([`IndexBuilder`]) public and re-exported
//! here for external callers.
pub mod algorithms;
pub(crate) mod extensions;
mod extensions;
pub mod graph_pipeline;
pub use extensions::IndexBuilder;
pub use graph_pipeline::{build_graph, build_layer_from_kmers, materialize_layer, write_graph_as_unitigs};
src/obikmer2/src/cmd/convert/mod.rs
+138
View File
@@ -0,0 +1,138 @@
use std::path::PathBuf;
use clap::Args;
use obikindex::KmerIndex;
use obikindex::layer::IndexMode;
use obikrebuild::IndexReindex;
use obisys::Reporter;
use tracing::info;
use crate::cli::block_size_to_bits;
use super::index::resolve_approx_params;
#[derive(Args)]
pub struct ConvertArgs {
/// Index directory to convert (modified in-place)
pub index: PathBuf,
#[command(flatten)]
mode: ModeArgs,
/// Fingerprint bits per slot (b) — required with --hybrid-evidence
/// when the source index is currently exact (there is no existing
/// fingerprint to inherit `b` from); rejected otherwise (the source
/// already fixes `b`, either from its own --approx-evidence or from
/// an already-approximate/hybrid index being converted to hybrid).
#[arg(long, value_name = "BITS")]
pub evidence_bits: Option<u8>,
/// Findere z parameter: number of consecutive *stored* k-mers (length
/// = the index's own k-mer size, fixed forever at `index` build time)
/// that must all match to confirm a hit.
///
/// WARNING: this does NOT shorten the indexed k-mer length the way
/// `-z` does at `index` build time — that length can never change
/// after the fact. Instead it *extends* the effective match window:
/// requiring z consecutive stored k-mers of length s to all match is
/// equivalent to requiring (s + z − 1) consecutive identical bases in
/// the query. E.g. on a k=31 index, z=2 requires 32 consecutive
/// matching bases, not 30 — going from z=1 to z=2 makes matching
/// *stricter*, it does not "index shorter k-mers".
#[arg(short = 'z', long)]
pub findere_z: Option<u8>,
/// Target false-positive rate per z-window (e.g. 0.01) — derives `b`
/// or `z` when one of them isn't given directly (see `index --fp` for
/// the same resolution rules).
#[arg(long)]
pub fp: Option<f64>,
/// Block size for exact evidence `.idx` (number of unitigs per block).
/// Ignored when converting to pure approximate evidence.
#[arg(long, default_value_t = 1)]
pub block_size: usize,
}
#[derive(Args)]
#[group(required = true, multiple = false)]
struct ModeArgs {
/// Convert to exact evidence (zero false positives).
#[arg(long)]
exact_evidence: bool,
/// Convert to approximate (fingerprint-only) evidence. Value = fingerprint bits per slot (b).
#[arg(long, value_name = "BITS")]
approx_evidence: Option<u8>,
/// Convert to hybrid evidence (both exact and approximate bundles kept).
#[arg(long)]
hybrid_evidence: bool,
}
pub fn run(args: ConvertArgs) {
// Modifies the index in place; acquired before opening so a concurrent
// writer can't slip in between the open and the convert below.
let _lock = obisys::DirLock::acquire(&args.index).unwrap_or_else(|e| {
eprintln!("error locking index directory {}: {e}", args.index.display());
std::process::exit(1);
});
let idx = KmerIndex::open(&args.index).unwrap_or_else(|e| {
eprintln!("error opening index: {e}");
std::process::exit(1);
});
let current = idx.meta().config().evidence.clone();
let target = if args.mode.exact_evidence {
warn_if_evidence_params_given(&args, "--exact-evidence");
info!("target: exact evidence");
IndexMode::Exact
} else if let Some(bits) = args.mode.approx_evidence {
let (z, b, fp) = resolve_approx_params(args.findere_z, Some(bits), args.fp);
info!("target: approximate evidence — b={b}, z={z}, fp={fp:.2e}");
IndexMode::Approx { b, z }
} else {
debug_assert!(args.mode.hybrid_evidence);
match current {
IndexMode::Exact => {
let bits = args.evidence_bits.unwrap_or_else(|| {
eprintln!(
"error: --hybrid-evidence requires --evidence-bits when starting from an exact index"
);
std::process::exit(1);
});
let (z, b, fp) = resolve_approx_params(args.findere_z, Some(bits), args.fp);
info!("target: hybrid evidence — b={b}, z={z}, fp={fp:.2e}");
IndexMode::Hybrid { b, z }
}
IndexMode::Approx { b, z } | IndexMode::Hybrid { b, z } => {
if args.evidence_bits.is_some() || args.findere_z.is_some() || args.fp.is_some() {
eprintln!(
"error: --evidence-bits/-z/--fp are not allowed with --hybrid-evidence \
when the source index is already approximate/hybrid — its existing \
b={b}, z={z} are reused as-is"
);
std::process::exit(1);
}
info!("target: hybrid evidence — reusing existing b={b}, z={z}");
IndexMode::Hybrid { b, z }
}
}
};
let block_bits = block_size_to_bits(args.block_size);
let mut rep = Reporter::new();
idx.reindex(target, block_bits, &mut rep).unwrap_or_else(|e| {
eprintln!("convert error: {e}");
std::process::exit(1);
});
rep.print();
}
fn warn_if_evidence_params_given(args: &ConvertArgs, mode: &str) {
if args.evidence_bits.is_some() || args.findere_z.is_some() || args.fp.is_some() {
eprintln!("warning: --evidence-bits/-z/--fp ignored with {mode}");
}
}
src/obikmer2/src/cmd/mod.rs
+1
View File
@@ -1,4 +1,5 @@
pub mod annotate;
pub mod convert;
pub mod dump;
pub mod estimate;
pub mod filter;
src/obikmer2/src/main.rs
+3
View File
@@ -37,6 +37,8 @@ enum Commands {
Annotate(cmd::annotate::AnnotateArgs),
/// Maintenance/inspection operations on already-built indexes
Utils(cmd::utils::UtilsArgs),
/// Convert an index's evidence representation (exact/approximate/hybrid), in place
Convert(cmd::convert::ConvertArgs),
}
fn main() {
@@ -61,5 +63,6 @@ fn main() {
Commands::Estimate(args) => cmd::estimate::run(args),
Commands::Annotate(args) => cmd::annotate::run(args),
Commands::Utils(args) => cmd::utils::run(args),
Commands::Convert(args) => cmd::convert::run(args),
}
}
src/obikrebuild/src/lib.rs
+5 -7
View File
@@ -6,12 +6,9 @@
//!
//! [`compact_layer`] collapses a partition's accumulated layers (the
//! stopgap `obikmerge` uses to add genomes without a full rebuild) back
//! into one, in place. `reindex` (reindexing an existing index's evidence
//! representation in place — `Exact`/`Approx`/`Hybrid`) is temporarily
//! disabled: it predates this session's refactor and still reaches
//! `obikindex` internals (`root_path`/`meta` fields, `index_dir`/
//! `layer_dir` methods) that are no longer accessible from outside that
//! crate. Fixing it is separate, future work.
//! into one, in place. [`reindex`] converts an existing index's evidence
//! representation in place (`Exact`/`Approx`/`Hybrid`), delegating the
//! actual per-layer construction to `obikindexer::IndexBuilder`.
//!
//! Removing k-mers (filtering) and repacking matrix files are *not* here:
//! the former lives in `obikfilter::Filter` (a fresh index at a new
@@ -21,6 +18,7 @@
//! cannot depend back on this crate.
mod compact_layer;
// mod reindex; // disabled — see module doc above.
mod reindex;
pub use compact_layer::IndexCompact;
pub use reindex::IndexReindex;
src/obikrebuild/src/reindex.rs
+62 -65
View File
@@ -1,43 +1,52 @@
use obikindex::IndexBuilder;
use obikindex::IndexMeta;
use obikindex::layer::{IndexMode, TypedLayer};
use obisys::{Reporter, Stage, progress_bar};
use std::fs;
use std::path::Path;
use std::sync::Arc;
//! Convert an existing index's evidence representation in place —
//! `Exact`/`Approx`/`Hybrid`. `unitigs.bin`/`mphf.bin` are never touched;
//! only the evidence bundle (`evidence.bin`/`unitigs.bin.idx` and/or
//! `fingerprint.bin`) is — the actual per-layer construction is
//! `obikindexer::IndexBuilder`'s job, this module only orchestrates it
//! across every partition/layer and updates `index.meta` on success.
//!
//! `KmerIndex` is foreign to this crate, so this is an extension trait,
//! same reasoning as `IndexCompact`/`IndexDump`/`IndexUnitigs`.
use obikindex::layer::{IndexMode, KmerLayer};
use obikindex::{IndexState, KmerIndex, OKIError, OKIResult};
use obikindexer::IndexBuilder;
use obisys::{PartitionRunner, Reporter, Stage, progress_bar};
use tracing::info;
use obikindex::{OKIError, OKIResult};
use obikindex::KmerIndex;
use obikindex::IndexState;
use obisys::PartitionRunner;
const EVIDENCE_FILE: &str = "evidence.bin";
const FINGERPRINT_FILE: &str = "fingerprint.bin";
/// `unitigs.bin.idx`'s filename isn't exposed by `obikindex`/`obiskio` as a
/// path accessor (it's a private implementation detail of exact-evidence
/// construction) — mirrored here as a literal, same as before this module
/// was disabled.
const UNITIG_IDX_FILE: &str = "unitigs.bin.idx";
impl KmerIndex {
/// Convert every layer's evidence bundle to `target` in-place.
pub trait IndexReindex {
/// Convert every layer's evidence bundle to `target`, in place.
///
/// - `Exact` → builds `evidence.bin` + `unitigs.bin.idx`, removes `fingerprint.bin`
/// - `Approx` → builds `fingerprint.bin`, removes `evidence.bin` + `unitigs.bin.idx`
/// - `Exact` → builds `evidence.bin` + `unitigs.bin.idx` where missing, removes `fingerprint.bin`.
/// - `Approx` → builds `fingerprint.bin` where missing or built with a different `b`, removes `evidence.bin` + `unitigs.bin.idx`.
/// - `Hybrid` → builds whichever bundle a layer is still missing, keeps both.
///
/// The MPHF (`mphf.bin`) and unitigs (`unitigs.bin`) are never touched.
/// `index.meta` is updated with the new evidence kind on success.
pub fn reindex(
&mut self,
target: IndexMode,
block_bits: u8,
rep: &mut Reporter,
) -> OKIResult<()> {
if self.state()? != IndexState::Indexed {
return Err(OKIError::NotIndexed(self.root_path.clone()));
/// Each layer's own build step is idempotent (see `IndexBuilder`), so
/// re-running this with the same `target` on an already-converted
/// index is a cheap no-op per layer, not a full rebuild.
fn reindex(&self, target: IndexMode, block_bits: u8, rep: &mut Reporter) -> OKIResult<()>;
}
impl IndexReindex for KmerIndex {
fn reindex(&self, target: IndexMode, block_bits: u8, rep: &mut Reporter) -> OKIResult<()> {
if self.meta().state().map_err(OKIError::Io)? != IndexState::Indexed {
return Err(OKIError::InvalidInput(format!(
"{}: index is not fully built",
self.dir().display()
)));
}
let n = self.n_partitions();
info!(
"reindex {} partition(s): {:?} → {:?}",
n, self.meta.config.evidence, target,
"reindex {} partition(s): {:?} → {target:?}",
n,
self.meta().config().evidence,
);
let t = Stage::start("reindex");
@@ -58,68 +67,56 @@ impl KmerIndex {
pb.finish_and_clear();
let mut config = self.meta.config.clone();
config.evidence = target;
if matches!(config.evidence, IndexMode::Exact) {
config.block_bits = block_bits;
}
let genomes = self.meta.genomes().map_err(OKIError::Io)?;
self.meta.rewrite_config(config, genomes).map_err(OKIError::Io)?;
self.meta = Arc::new(IndexMeta::open(self).map_err(OKIError::Io)?);
self.meta().set_evidence(target, block_bits).map_err(OKIError::Io)?;
rep.push(t.stop());
Ok(())
}
}
/// Process all layers of one partition's index directory.
fn reindex_partition(
index: &KmerIndex,
i: usize,
target: &IndexMode,
block_bits: u8,
) -> OKIResult<()> {
if !index.index_dir(i).exists() {
return Ok(());
}
let n_layers = index
.n_layers(i)
.map_err(|e| OKIError::InvalidInput(e.to_string()))?;
for layer_idx in 0..n_layers {
reindex_layer(&index.layer_dir(i, layer_idx), target, block_bits)?;
/// Process every layer of partition `i`.
fn reindex_partition(index: &KmerIndex, i: usize, target: &IndexMode, block_bits: u8) -> OKIResult<()> {
let partition = index.partition(i)?;
let n_layers = partition.n_layers();
for l in 0..n_layers {
let layer = partition.layer(l)?.open()?;
reindex_layer(&layer, target, block_bits)?;
remove_stale_evidence(&layer, target);
}
Ok(())
}
fn reindex_layer(layer_dir: &Path, target: &IndexMode, block_bits: u8) -> OKIResult<()> {
fn reindex_layer(layer: &KmerLayer, target: &IndexMode, block_bits: u8) -> OKIResult<()> {
match target {
IndexMode::Exact => {
TypedLayer::<()>::build_exact_evidence(layer_dir, block_bits)?;
layer.build_exact_evidence(block_bits)?;
}
IndexMode::Approx { b, z } | IndexMode::Hybrid { b, z } => {
TypedLayer::<()>::build_approx_evidence(layer_dir, *b, *z)?;
IndexMode::Approx { b, z } => {
layer.build_approx_evidence(*b, *z)?;
}
IndexMode::Hybrid { b, z } => {
layer.build_hybrid_evidence(*b, *z, block_bits)?;
}
}
remove_stale_evidence(layer_dir, target)
Ok(())
}
fn remove_stale_evidence(layer_dir: &Path, target: &IndexMode) -> OKIResult<()> {
fn remove_stale_evidence(layer: &KmerLayer, target: &IndexMode) {
match target {
IndexMode::Exact => {
remove_if_exists(&layer_dir.join(FINGERPRINT_FILE));
remove_if_exists(&layer.fingerprint_path());
}
IndexMode::Approx { .. } => {
remove_if_exists(&layer_dir.join(EVIDENCE_FILE));
remove_if_exists(&layer_dir.join(UNITIG_IDX_FILE));
remove_if_exists(&layer.evidence_path());
remove_if_exists(&layer.dir().join(UNITIG_IDX_FILE));
}
IndexMode::Hybrid { .. } => {
// both bundles kept — nothing to remove
}
}
Ok(())
}
fn remove_if_exists(path: &Path) {
if let Err(e) = fs::remove_file(path) {
fn remove_if_exists(path: &std::path::Path) {
if let Err(e) = std::fs::remove_file(path) {
if e.kind() != std::io::ErrorKind::NotFound {
eprintln!("warning: could not remove {}: {e}", path.display());
}