OBIKmers/obikmer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 34 Wiki Activity

Push zunrplorkwkt #70

Merged
coissac merged 93 commits from push-zunrplorkwkt into main 2026-08-28 23:15:38 +00:00
Conversation 0 Commits 93 Files Changed 428
+562 -340
10 changed files with 562 additions and 340 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 4ea3cd32ba - Show all commits
src/obikindex/src/index/meta.rs
+22 -23
View File
@@ -218,29 +218,28 @@ impl IndexMeta {
self.write_full(&on_disk)
}
// /// Overwrite `config` and the whole `genomes` list at once, preserving
// /// `state`. `config` otherwise never changes once an index exists —
// /// this is the deliberate, rare exception for construction paths that
// /// legitimately rewrite it in place (`select_in_place`, `reindex`).
// /// The caller must refresh its own cached `Arc<IndexMeta>` afterward
// /// (e.g. `self.meta = Arc::new(IndexMeta::open(self)?)`) — this method
// /// only updates the file, it has no way to reach back into whatever
// /// `KmerIndex` holds it.
// /// TODO: This methode is strange
// pub(crate) fn rewrite_config(
// &self,
// config: IndexConfig,
// genomes: Vec<GenomeInfo>,
// ) -> io::Result<()> {
// let _guard = self.lock.write().unwrap();
// let state = Self::read_full(&self.root_path)?.state;
// self.write_full(&IndexMetadata {
// version: self.version,
// config,
// genomes,
// state,
// })
// }
/// Update the evidence representation in place — `evidence`, and (for
/// `Exact`) its block size. The only `IndexConfig` fields a reindex is
/// ever allowed to touch: `kmer_size`/`minimizer_size`/`n_bits`/
/// `with_counts` are baked into the already-built MPHF/unitigs/matrices
/// and can never change post-build, unlike the evidence bundle, which
/// is derived from `unitigs.bin` + `mphf.bin` alone and can be
/// rebuilt in a different form at any time (see
/// `obikindexer::IndexBuilder`).
///
/// The caller must refresh its own cached `Arc<IndexMeta>` afterward
/// (e.g. `self.meta = Arc::new(IndexMeta::open(self)?)`) — this method
/// only updates the file, it has no way to reach back into whatever
/// `KmerIndex` holds it.
pub fn set_evidence(&self, evidence: IndexMode, block_bits: u8) -> io::Result<()> {
let _guard = self.lock.write().unwrap();
let mut on_disk = Self::read_full(&self.root_path)?;
on_disk.config.evidence = evidence;
if matches!(on_disk.config.evidence, IndexMode::Exact) {
on_disk.config.block_bits = block_bits;
}
self.write_full(&on_disk)
}
pub fn set_state(&self, state: IndexState) -> io::Result<()> {
let _guard = self.lock.write().unwrap();
src/obikindexer/src/extensions/index_builder.rs
+68
View File
@@ -0,0 +1,68 @@
//! `IndexBuilder` — public, idempotent evidence-construction primitives
//! for an already-built `KmerLayer` (`unitigs.bin` + `mphf.bin` already on
//! disk; only the evidence bundle — `evidence.bin`/`unitigs.bin.idx` and/or
//! `fingerprint.bin` — is touched). Unlike `PrivateBuilder` (this module's
//! private sibling), these are meant for callers outside this crate that
//! need to add or convert a layer's evidence after the fact — today,
//! `obikrebuild::reindex`.
//!
//! `KmerLayer` itself (`obikindex`) stays a pure data-model type: it
//! exposes paths and reads content, it never builds anything. Construction
//! lives here instead, in the crate whose whole purpose is building index
//! content — same reasoning as `PrivateBuilder`, just public.
use obikindex::OKIResult;
use obikindex::layer::fingerprint::FingerprintVec;
use obikindex::layer::{KmerLayer, MphfLayer};
pub trait IndexBuilder {
/// Build this layer's exact evidence bundle (`evidence.bin` +
/// `unitigs.bin.idx`) if not already present.
///
/// No-op (`Ok(0)`) if exact evidence already exists — both `Exact` and
/// `Hybrid` layers already carry it.
fn build_exact_evidence(&self, block_bits: u8) -> OKIResult<usize>;
/// Build this layer's approximate (fingerprint) evidence with `b` bits
/// per slot and Findere parameter `z`, if not already present with a
/// matching `b`.
///
/// No-op (`Ok(0)`) if `Approx`/`Hybrid` evidence with the same `b`
/// already exists — rebuilds it (same file, same `b`, whatever `z`)
/// otherwise. `z` is never persisted (see `IndexMode::Approx`'s own
/// doc): it only governs how many consecutive stored k-mers a query
/// must match, so changing it alone is not something this method has
/// any file-level evidence to detect — the caller decides.
fn build_approx_evidence(&self, b: u8, z: u8) -> OKIResult<usize>;
/// Build whichever of the exact/approximate bundles this layer is
/// still missing, so it ends up carrying both (`Hybrid`).
///
/// Just the composition of `build_exact_evidence` and
/// `build_approx_evidence` — each is already independently idempotent,
/// so there is no extra "what's already there" decision to make here.
fn build_hybrid_evidence(&self, b: u8, z: u8, block_bits: u8) -> OKIResult<usize>;
}
impl IndexBuilder for KmerLayer {
fn build_exact_evidence(&self, block_bits: u8) -> OKIResult<usize> {
if self.evidence_path().exists() {
return Ok(0);
}
MphfLayer::build_exact_evidence(self.dir(), block_bits)
}
fn build_approx_evidence(&self, b: u8, z: u8) -> OKIResult<usize> {
let fingerprint_path = self.fingerprint_path();
if fingerprint_path.exists() && FingerprintVec::open(&fingerprint_path)?.b() == b {
return Ok(0);
}
MphfLayer::build_approx_evidence(self.dir(), b, z)
}
fn build_hybrid_evidence(&self, b: u8, z: u8, block_bits: u8) -> OKIResult<usize> {
let n_exact = self.build_exact_evidence(block_bits)?;
let n_approx = self.build_approx_evidence(b, z)?;
Ok(n_exact.max(n_approx))
}
}
src/obikindexer/src/extensions/mod.rs
+17 -242
View File
@@ -1,245 +1,20 @@
//! Construction-only `KmerIndex` operations — semantically meaningless
//! once an index is built and being used for reading (query, dump, phylo,
//! or as a `merge`/`select`/`rebuild` *source*): sentinel marking, the
//! per-partition layer-build primitive, and cleanup of build-time scratch
//! files. Private to this crate — no consumer outside the
//! indexing-pipeline algorithms in `obikindexer::algorithms` should ever
//! need them. See `DevDocMD/implementation/partition_layer_cache.md`,
//! "(8)".
//! Extension traits over `obikindex`'s foreign types (`KmerIndex`,
//! `KmerLayer`) — Rust's orphan rule requires such traits to be defined in
//! the crate that implements them for a foreign type, not in `obikindex`
//! itself; `obikindex` has no way to make a trait "visible to only some
//! crates" from its own side.
//!
//! Defined here, not in `obikindex`, so it can be genuinely `pub(crate)`
//! while extending a foreign type (`KmerIndex`) — Rust's orphan rule
//! requires the trait itself to be local to the crate that implements it
//! for a foreign type; `obikindex` has no way to make a trait "private
//! except to `obikindexer`" from its own side.
//!
//! Scoped to the six methods exclusive to this crate's own pipeline
//! (`partitionner`/`dereplicator`/`counter`/`layer_builder`).
//! `clear_output_for_create`/`create_skeleton`/`finalize_indexed`/`state`
//! stay inherent on `KmerIndex`, deliberately not moved here: they're also
//! called from `obikindex`'s own `merge`/`select`/`rebuild`/`reindex`
//! modules, which — living inside `obikindex` itself — could never reach a
//! trait defined in `obikindexer` (the dependency only runs
//! `obikindexer → obikindex`, never the other way). Moving those four
//! would require extracting `merge`/`select`/`rebuild`/`reindex` into
//! algorithms first — separate, larger future work, not this one.
//! Two traits, two visibilities, one per submodule:
//! - [`private_builder::PrivateBuilder`] (`pub(crate)`) — construction-only
//! `KmerIndex` operations exclusive to this crate's own indexing
//! pipeline (`algorithms::{partitionner,dereplicator,counter,layer_builder}`).
//! - [`index_builder::IndexBuilder`] (`pub`, re-exported at the crate
//! root) — idempotent per-layer evidence-construction primitives, for
//! any external caller that needs to add/convert a layer's evidence
//! bundle after the fact (`obikrebuild::reindex`).
use std::collections::BTreeMap;
use std::fs;
use std::io;
use std::path::Path;
mod index_builder;
mod private_builder;
use cacheline_ef::{CachelineEf, CachelineEfVec};
use epserde::prelude::*;
use obicompactvec::{PersistentCompactIntMatrix, PersistentCompactIntVec};
use obidebruinj::GraphDeBruijn;
use obikindex::layer::IndexMode;
use obikindex::layer::{KmerLayer, TypedLayer};
use obikindex::{KmerIndex, OKIError, OKIResult};
use crate::graph_pipeline::{materialize_layer, write_graph_as_unitigs};
use obiskio::{SKError, SKFileMeta, SKFileReader};
use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xx64};
type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xx64, Vec<u8>>;
pub(crate) trait PrivateBuilder {
/// Mark scatter as complete (`IndexState::Scattered`).
///
/// If no genome label was set at creation time, one is derived from
/// the index root directory name (stripped of all extensions). `&self`
/// — not `&mut self` anymore: nothing is cached in memory to mutate,
/// see `DevDocMD/implementation/partition_layer_cache.md`, "(11)".
fn mark_scattered(&self) -> OKIResult<()>;
/// Mark dereplicate+count as complete (`IndexState::Counted`).
fn mark_counted(&self) -> OKIResult<()>;
/// Partition `i`'s layer 0 — the one and only layer every pipeline
/// algorithm in this crate (`partitionner`/`dereplicator`/`counter`/
/// `layer_builder`) reads from or writes to; later layers only exist
/// after a `merge`, which lives in `obikindex`, not here.
fn layer0(&self, i: usize) -> OKIResult<KmerLayer>;
/// Mark layer construction as complete (`IndexState::Indexed`).
fn mark_indexed(&self) -> OKIResult<()>;
/// Write `spectrums/{label}.json` from an already-computed kmer
/// spectrum (`f0`/`f1`/abundance histogram). Takes plain values rather
/// than `algorithms::counter::KmerSpectrum` — `KmerIndex` is the data
/// model, `Counter` the algorithm that depends on it, not the other
/// way around, and this is the only field of that type it actually
/// uses.
fn write_spectrum(&self, f0: u64, f1: u64, counts: &BTreeMap<u32, u64>) -> OKIResult<()>;
/// Build the layered MPHF index for partition `i`.
///
/// Returns the number of canonical k-mers indexed, or 0 if the
/// partition has no data or its layer was already built
/// (resume-safe).
///
/// Abundance filtering is applied when `min_ab > 1` or
/// `max_ab.is_some()`, using `mphf1.bin` + `counts1.bin` if they
/// exist. Count payload is stored iff `with_counts` is true.
fn build_index_layer(
&self,
i: usize,
min_ab: u32,
max_ab: Option<u32>,
with_counts: bool,
mode: &IndexMode,
block_bits: u8,
) -> Result<usize, SKError>;
/// Remove intermediate build artifacts for partition `i`: dereplicated
/// superkmers (+ sidecar), `mphf1.bin`, `counts1.bin`.
fn remove_build_artifacts(&self, i: usize);
}
impl PrivateBuilder for KmerIndex {
fn mark_scattered(&self) -> OKIResult<()> {
self.meta().mark_scattered().map_err(OKIError::Io)
}
fn mark_counted(&self) -> OKIResult<()> {
self.meta().mark_counted().map_err(OKIError::Io)
}
fn layer0(&self, i: usize) -> OKIResult<KmerLayer> {
self.partition(i)?.layer(0)
}
fn mark_indexed(&self) -> OKIResult<()> {
self.meta().mark_indexed().map_err(OKIError::Io)
}
fn write_spectrum(&self, f0: u64, f1: u64, counts: &BTreeMap<u32, u64>) -> OKIResult<()> {
let genomes = self.meta().genomes().map_err(OKIError::Io)?;
let label = genomes
.first()
.map(|g| g.label.as_str())
.unwrap_or("unknown")
.to_owned();
let spectrums_dir = self.dir().join("spectrums");
fs::create_dir_all(&spectrums_dir)?;
let path = spectrums_dir.join(format!("{label}.json"));
let spectrum_map: BTreeMap<String, u64> = counts
.iter()
.map(|(&c, &f)| (format!("{c:010}"), f))
.collect();
let f = fs::File::create(&path)?;
serde_json::to_writer_pretty(
f,
&serde_json::json!({ "f0": f0, "f1": f1, "spectrum": spectrum_map }),
)
.map_err(OKIError::Json)?;
Ok(())
}
fn build_index_layer(
&self,
i: usize,
min_ab: u32,
max_ab: Option<u32>,
with_counts: bool,
mode: &IndexMode,
block_bits: u8,
) -> Result<usize, SKError> {
let layer0 = self.layer0(i).map_err(|e| io::Error::other(e.to_string()))?;
let layer0_dir = layer0.dir();
let dedup_path = layer0.dereplicated_superkmers_path();
if !dedup_path.exists() {
return Ok(0);
}
if layer0_dir.join("mphf.bin").exists() {
return Ok(0);
}
let filter_active = min_ab > 1 || max_ab.is_some();
let need_counts = filter_active || with_counts;
let mphf1_opt: Option<Mphf> = if need_counts {
let p = layer0_dir.join("mphf1.bin");
p.exists().then(|| Mphf::load_full(&p).ok()).flatten()
} else {
None
};
let counts1_opt: Option<PersistentCompactIntVec> = if need_counts {
let p = layer0_dir.join("counts1.bin");
p.exists()
.then(|| PersistentCompactIntVec::open(&p).ok())
.flatten()
} else {
None
};
let mut g = GraphDeBruijn::new();
let mut reader = SKFileReader::open(&dedup_path)?;
for sk in reader.iter() {
for kmer in sk.iter_canonical_kmers() {
let accept = if filter_active {
match (&mphf1_opt, &counts1_opt) {
(Some(mphf), Some(counts)) => {
let ab = counts.get(mphf.index(&kmer.raw()));
ab >= min_ab && max_ab.map_or(true, |max| ab <= max)
}
_ => true,
}
} else {
true
};
if accept {
g.push(kmer);
}
}
}
let n_kmers = if with_counts {
let n = write_graph_as_unitigs(g, layer0_dir)
.map_err(|e| io::Error::other(e.to_string()))?;
TypedLayer::<PersistentCompactIntMatrix>::build(
layer0_dir,
block_bits,
mode,
|kmer| match (&mphf1_opt, &counts1_opt) {
(Some(mphf), Some(counts)) => counts.get(mphf.index(&kmer.raw())),
_ => 1,
},
)
.map_err(|e| io::Error::other(e.to_string()))?;
n
} else {
materialize_layer(g, layer0_dir, block_bits, mode)
.map_err(|e| io::Error::other(e.to_string()))?
};
Ok(n_kmers)
}
fn remove_build_artifacts(&self, i: usize) {
let layer0 = match self.layer0(i) {
Ok(layer) => layer,
Err(e) => {
eprintln!("warning: could not locate layer 0 of partition {i}: {e}");
return;
}
};
let layer0_dir = layer0.dir();
let dedup = layer0.dereplicated_superkmers_path();
remove_if_exists(&SKFileMeta::sidecar_path(&dedup));
remove_if_exists(&dedup);
remove_if_exists(&layer0_dir.join("mphf1.bin"));
remove_if_exists(&layer0_dir.join("counts1.bin"));
}
}
// ── private helpers ──────────────────────────────────────────────────────
fn remove_if_exists(path: &Path) {
if let Err(e) = fs::remove_file(path) {
if e.kind() != io::ErrorKind::NotFound {
eprintln!("warning: could not remove {}: {e}", path.display());
}
}
}
pub use index_builder::IndexBuilder;
pub(crate) use private_builder::PrivateBuilder;
src/obikindexer/src/extensions/private_builder.rs
+240
View File
@@ -0,0 +1,240 @@
//! `PrivateBuilder` — construction-only `KmerIndex` operations:
//! sentinel marking, the per-partition layer-build primitive, and cleanup
//! of build-time scratch files. Semantically meaningless once an index is
//! built and being used for reading (query, dump, phylo, or as a
//! `merge`/`select`/`rebuild` *source*).
//!
//! Scoped to the six methods exclusive to this crate's own pipeline
//! (`partitionner`/`dereplicator`/`counter`/`layer_builder`) — no
//! consumer outside those algorithms should ever need them, hence
//! `pub(crate)` (see the module-level orphan-rule note in `extensions`'s
//! `mod.rs`). See `DevDocMD/implementation/partition_layer_cache.md`, "(8)".
//!
//! `clear_output_for_create`/`create_skeleton`/`finalize_indexed`/`state`
//! stay inherent on `KmerIndex`, deliberately not moved here: they're also
//! called from `obikindex`'s own `merge`/`select`/`rebuild`/`reindex`
//! modules, which — living inside `obikindex` itself — could never reach a
//! trait defined in `obikindexer` (the dependency only runs
//! `obikindexer → obikindex`, never the other way). Moving those four
//! would require extracting `merge`/`select`/`rebuild`/`reindex` into
//! algorithms first — separate, larger future work, not this one.
use std::collections::BTreeMap;
use std::fs;
use std::io;
use std::path::Path;
use cacheline_ef::{CachelineEf, CachelineEfVec};
use epserde::prelude::*;
use obicompactvec::{PersistentCompactIntMatrix, PersistentCompactIntVec};
use obidebruinj::GraphDeBruijn;
use obikindex::layer::IndexMode;
use obikindex::layer::{KmerLayer, TypedLayer};
use obikindex::{KmerIndex, OKIError, OKIResult};
use crate::graph_pipeline::{materialize_layer, write_graph_as_unitigs};
use obiskio::{SKError, SKFileMeta, SKFileReader};
use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xx64};
type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xx64, Vec<u8>>;
pub(crate) trait PrivateBuilder {
/// Mark scatter as complete (`IndexState::Scattered`).
///
/// If no genome label was set at creation time, one is derived from
/// the index root directory name (stripped of all extensions). `&self`
/// — not `&mut self` anymore: nothing is cached in memory to mutate,
/// see `DevDocMD/implementation/partition_layer_cache.md`, "(11)".
fn mark_scattered(&self) -> OKIResult<()>;
/// Mark dereplicate+count as complete (`IndexState::Counted`).
fn mark_counted(&self) -> OKIResult<()>;
/// Partition `i`'s layer 0 — the one and only layer every pipeline
/// algorithm in this crate (`partitionner`/`dereplicator`/`counter`/
/// `layer_builder`) reads from or writes to; later layers only exist
/// after a `merge`, which lives in `obikindex`, not here.
fn layer0(&self, i: usize) -> OKIResult<KmerLayer>;
/// Mark layer construction as complete (`IndexState::Indexed`).
fn mark_indexed(&self) -> OKIResult<()>;
/// Write `spectrums/{label}.json` from an already-computed kmer
/// spectrum (`f0`/`f1`/abundance histogram). Takes plain values rather
/// than `algorithms::counter::KmerSpectrum` — `KmerIndex` is the data
/// model, `Counter` the algorithm that depends on it, not the other
/// way around, and this is the only field of that type it actually
/// uses.
fn write_spectrum(&self, f0: u64, f1: u64, counts: &BTreeMap<u32, u64>) -> OKIResult<()>;
/// Build the layered MPHF index for partition `i`.
///
/// Returns the number of canonical k-mers indexed, or 0 if the
/// partition has no data or its layer was already built
/// (resume-safe).
///
/// Abundance filtering is applied when `min_ab > 1` or
/// `max_ab.is_some()`, using `mphf1.bin` + `counts1.bin` if they
/// exist. Count payload is stored iff `with_counts` is true.
fn build_index_layer(
&self,
i: usize,
min_ab: u32,
max_ab: Option<u32>,
with_counts: bool,
mode: &IndexMode,
block_bits: u8,
) -> Result<usize, SKError>;
/// Remove intermediate build artifacts for partition `i`: dereplicated
/// superkmers (+ sidecar), `mphf1.bin`, `counts1.bin`.
fn remove_build_artifacts(&self, i: usize);
}
impl PrivateBuilder for KmerIndex {
fn mark_scattered(&self) -> OKIResult<()> {
self.meta().mark_scattered().map_err(OKIError::Io)
}
fn mark_counted(&self) -> OKIResult<()> {
self.meta().mark_counted().map_err(OKIError::Io)
}
fn layer0(&self, i: usize) -> OKIResult<KmerLayer> {
self.partition(i)?.layer(0)
}
fn mark_indexed(&self) -> OKIResult<()> {
self.meta().mark_indexed().map_err(OKIError::Io)
}
fn write_spectrum(&self, f0: u64, f1: u64, counts: &BTreeMap<u32, u64>) -> OKIResult<()> {
let genomes = self.meta().genomes().map_err(OKIError::Io)?;
let label = genomes
.first()
.map(|g| g.label.as_str())
.unwrap_or("unknown")
.to_owned();
let spectrums_dir = self.dir().join("spectrums");
fs::create_dir_all(&spectrums_dir)?;
let path = spectrums_dir.join(format!("{label}.json"));
let spectrum_map: BTreeMap<String, u64> = counts
.iter()
.map(|(&c, &f)| (format!("{c:010}"), f))
.collect();
let f = fs::File::create(&path)?;
serde_json::to_writer_pretty(
f,
&serde_json::json!({ "f0": f0, "f1": f1, "spectrum": spectrum_map }),
)
.map_err(OKIError::Json)?;
Ok(())
}
fn build_index_layer(
&self,
i: usize,
min_ab: u32,
max_ab: Option<u32>,
with_counts: bool,
mode: &IndexMode,
block_bits: u8,
) -> Result<usize, SKError> {
let layer0 = self.layer0(i).map_err(|e| io::Error::other(e.to_string()))?;
let layer0_dir = layer0.dir();
let dedup_path = layer0.dereplicated_superkmers_path();
if !dedup_path.exists() {
return Ok(0);
}
if layer0_dir.join("mphf.bin").exists() {
return Ok(0);
}
let filter_active = min_ab > 1 || max_ab.is_some();
let need_counts = filter_active || with_counts;
let mphf1_opt: Option<Mphf> = if need_counts {
let p = layer0_dir.join("mphf1.bin");
p.exists().then(|| Mphf::load_full(&p).ok()).flatten()
} else {
None
};
let counts1_opt: Option<PersistentCompactIntVec> = if need_counts {
let p = layer0_dir.join("counts1.bin");
p.exists()
.then(|| PersistentCompactIntVec::open(&p).ok())
.flatten()
} else {
None
};
let mut g = GraphDeBruijn::new();
let mut reader = SKFileReader::open(&dedup_path)?;
for sk in reader.iter() {
for kmer in sk.iter_canonical_kmers() {
let accept = if filter_active {
match (&mphf1_opt, &counts1_opt) {
(Some(mphf), Some(counts)) => {
let ab = counts.get(mphf.index(&kmer.raw()));
ab >= min_ab && max_ab.map_or(true, |max| ab <= max)
}
_ => true,
}
} else {
true
};
if accept {
g.push(kmer);
}
}
}
let n_kmers = if with_counts {
let n = write_graph_as_unitigs(g, layer0_dir)
.map_err(|e| io::Error::other(e.to_string()))?;
TypedLayer::<PersistentCompactIntMatrix>::build(
layer0_dir,
block_bits,
mode,
|kmer| match (&mphf1_opt, &counts1_opt) {
(Some(mphf), Some(counts)) => counts.get(mphf.index(&kmer.raw())),
_ => 1,
},
)
.map_err(|e| io::Error::other(e.to_string()))?;
n
} else {
materialize_layer(g, layer0_dir, block_bits, mode)
.map_err(|e| io::Error::other(e.to_string()))?
};
Ok(n_kmers)
}
fn remove_build_artifacts(&self, i: usize) {
let layer0 = match self.layer0(i) {
Ok(layer) => layer,
Err(e) => {
eprintln!("warning: could not locate layer 0 of partition {i}: {e}");
return;
}
};
let layer0_dir = layer0.dir();
let dedup = layer0.dereplicated_superkmers_path();
remove_if_exists(&SKFileMeta::sidecar_path(&dedup));
remove_if_exists(&dedup);
remove_if_exists(&layer0_dir.join("mphf1.bin"));
remove_if_exists(&layer0_dir.join("counts1.bin"));
}
}
// ── private helpers ──────────────────────────────────────────────────────
fn remove_if_exists(path: &Path) {
if let Err(e) = fs::remove_file(path) {
if e.kind() != io::ErrorKind::NotFound {
eprintln!("warning: could not remove {}: {e}", path.display());
}
}
}
src/obikindexer/src/lib.rs
+6 -3
View File
@@ -9,11 +9,14 @@
//! (routing raw super-kmers into partitions), `dereplicator` (deduplicating
//! a partition's raw super-kmers), `counter` (counting unique canonical
//! kmers), `layer_builder` (turning dereplicated super-kmers + counts into
//! the real layer 0). [`extensions`] (private — see its own module docs)
//! holds construction-only `KmerIndex` operations these algorithms share.
//! the real layer 0). `extensions` (module itself private — see its own
//! docs) holds `KmerIndex`/`KmerLayer` extension traits: some exclusive to
//! this crate's own pipeline, one ([`IndexBuilder`]) public and re-exported
//! here for external callers.
pub mod algorithms;
pub(crate) mod extensions;
mod extensions;
pub mod graph_pipeline;
pub use extensions::IndexBuilder;
pub use graph_pipeline::{build_graph, build_layer_from_kmers, materialize_layer, write_graph_as_unitigs};
src/obikmer2/src/cmd/convert/mod.rs
+138
View File
@@ -0,0 +1,138 @@
use std::path::PathBuf;
use clap::Args;
use obikindex::KmerIndex;
use obikindex::layer::IndexMode;
use obikrebuild::IndexReindex;
use obisys::Reporter;
use tracing::info;
use crate::cli::block_size_to_bits;
use super::index::resolve_approx_params;
#[derive(Args)]
pub struct ConvertArgs {
/// Index directory to convert (modified in-place)
pub index: PathBuf,
#[command(flatten)]
mode: ModeArgs,
/// Fingerprint bits per slot (b) — required with --hybrid-evidence
/// when the source index is currently exact (there is no existing
/// fingerprint to inherit `b` from); rejected otherwise (the source
/// already fixes `b`, either from its own --approx-evidence or from
/// an already-approximate/hybrid index being converted to hybrid).
#[arg(long, value_name = "BITS")]
pub evidence_bits: Option<u8>,
/// Findere z parameter: number of consecutive *stored* k-mers (length
/// = the index's own k-mer size, fixed forever at `index` build time)
/// that must all match to confirm a hit.
///
/// WARNING: this does NOT shorten the indexed k-mer length the way
/// `-z` does at `index` build time — that length can never change
/// after the fact. Instead it *extends* the effective match window:
/// requiring z consecutive stored k-mers of length s to all match is
/// equivalent to requiring (s + z − 1) consecutive identical bases in
/// the query. E.g. on a k=31 index, z=2 requires 32 consecutive
/// matching bases, not 30 — going from z=1 to z=2 makes matching
/// *stricter*, it does not "index shorter k-mers".
#[arg(short = 'z', long)]
pub findere_z: Option<u8>,
/// Target false-positive rate per z-window (e.g. 0.01) — derives `b`
/// or `z` when one of them isn't given directly (see `index --fp` for
/// the same resolution rules).
#[arg(long)]
pub fp: Option<f64>,
/// Block size for exact evidence `.idx` (number of unitigs per block).
/// Ignored when converting to pure approximate evidence.
#[arg(long, default_value_t = 1)]
pub block_size: usize,
}
#[derive(Args)]
#[group(required = true, multiple = false)]
struct ModeArgs {
/// Convert to exact evidence (zero false positives).
#[arg(long)]
exact_evidence: bool,
/// Convert to approximate (fingerprint-only) evidence. Value = fingerprint bits per slot (b).
#[arg(long, value_name = "BITS")]
approx_evidence: Option<u8>,
/// Convert to hybrid evidence (both exact and approximate bundles kept).
#[arg(long)]
hybrid_evidence: bool,
}
pub fn run(args: ConvertArgs) {
// Modifies the index in place; acquired before opening so a concurrent
// writer can't slip in between the open and the convert below.
let _lock = obisys::DirLock::acquire(&args.index).unwrap_or_else(|e| {
eprintln!("error locking index directory {}: {e}", args.index.display());
std::process::exit(1);
});
let idx = KmerIndex::open(&args.index).unwrap_or_else(|e| {
eprintln!("error opening index: {e}");
std::process::exit(1);
});
let current = idx.meta().config().evidence.clone();
let target = if args.mode.exact_evidence {
warn_if_evidence_params_given(&args, "--exact-evidence");
info!("target: exact evidence");
IndexMode::Exact
} else if let Some(bits) = args.mode.approx_evidence {
let (z, b, fp) = resolve_approx_params(args.findere_z, Some(bits), args.fp);
info!("target: approximate evidence — b={b}, z={z}, fp={fp:.2e}");
IndexMode::Approx { b, z }
} else {
debug_assert!(args.mode.hybrid_evidence);
match current {
IndexMode::Exact => {
let bits = args.evidence_bits.unwrap_or_else(|| {
eprintln!(
"error: --hybrid-evidence requires --evidence-bits when starting from an exact index"
);
std::process::exit(1);
});
let (z, b, fp) = resolve_approx_params(args.findere_z, Some(bits), args.fp);
info!("target: hybrid evidence — b={b}, z={z}, fp={fp:.2e}");
IndexMode::Hybrid { b, z }
}
IndexMode::Approx { b, z } | IndexMode::Hybrid { b, z } => {
if args.evidence_bits.is_some() || args.findere_z.is_some() || args.fp.is_some() {
eprintln!(
"error: --evidence-bits/-z/--fp are not allowed with --hybrid-evidence \
when the source index is already approximate/hybrid — its existing \
b={b}, z={z} are reused as-is"
);
std::process::exit(1);
}
info!("target: hybrid evidence — reusing existing b={b}, z={z}");
IndexMode::Hybrid { b, z }
}
}
};
let block_bits = block_size_to_bits(args.block_size);
let mut rep = Reporter::new();
idx.reindex(target, block_bits, &mut rep).unwrap_or_else(|e| {
eprintln!("convert error: {e}");
std::process::exit(1);
});
rep.print();
}
fn warn_if_evidence_params_given(args: &ConvertArgs, mode: &str) {
if args.evidence_bits.is_some() || args.findere_z.is_some() || args.fp.is_some() {
eprintln!("warning: --evidence-bits/-z/--fp ignored with {mode}");
}
}
src/obikmer2/src/cmd/mod.rs
+1
View File
@@ -1,4 +1,5 @@
pub mod annotate;
pub mod convert;
pub mod dump;
pub mod estimate;
pub mod filter;
src/obikmer2/src/main.rs
+3
View File
@@ -37,6 +37,8 @@ enum Commands {
Annotate(cmd::annotate::AnnotateArgs),
/// Maintenance/inspection operations on already-built indexes
Utils(cmd::utils::UtilsArgs),
/// Convert an index's evidence representation (exact/approximate/hybrid), in place
Convert(cmd::convert::ConvertArgs),
}
fn main() {
@@ -61,5 +63,6 @@ fn main() {
Commands::Estimate(args) => cmd::estimate::run(args),
Commands::Annotate(args) => cmd::annotate::run(args),
Commands::Utils(args) => cmd::utils::run(args),
Commands::Convert(args) => cmd::convert::run(args),
}
}
src/obikrebuild/src/lib.rs
+5 -7
View File
@@ -6,12 +6,9 @@
//!
//! [`compact_layer`] collapses a partition's accumulated layers (the
//! stopgap `obikmerge` uses to add genomes without a full rebuild) back
//! into one, in place. `reindex` (reindexing an existing index's evidence
//! representation in place — `Exact`/`Approx`/`Hybrid`) is temporarily
//! disabled: it predates this session's refactor and still reaches
//! `obikindex` internals (`root_path`/`meta` fields, `index_dir`/
//! `layer_dir` methods) that are no longer accessible from outside that
//! crate. Fixing it is separate, future work.
//! into one, in place. [`reindex`] converts an existing index's evidence
//! representation in place (`Exact`/`Approx`/`Hybrid`), delegating the
//! actual per-layer construction to `obikindexer::IndexBuilder`.
//!
//! Removing k-mers (filtering) and repacking matrix files are *not* here:
//! the former lives in `obikfilter::Filter` (a fresh index at a new
@@ -21,6 +18,7 @@
//! cannot depend back on this crate.
mod compact_layer;
// mod reindex; // disabled — see module doc above.
mod reindex;
pub use compact_layer::IndexCompact;
pub use reindex::IndexReindex;
src/obikrebuild/src/reindex.rs
+62 -65
View File
@@ -1,43 +1,52 @@
use obikindex::IndexBuilder;
use obikindex::IndexMeta;
use obikindex::layer::{IndexMode, TypedLayer};
use obisys::{Reporter, Stage, progress_bar};
use std::fs;
use std::path::Path;
use std::sync::Arc;
//! Convert an existing index's evidence representation in place —
//! `Exact`/`Approx`/`Hybrid`. `unitigs.bin`/`mphf.bin` are never touched;
//! only the evidence bundle (`evidence.bin`/`unitigs.bin.idx` and/or
//! `fingerprint.bin`) is — the actual per-layer construction is
//! `obikindexer::IndexBuilder`'s job, this module only orchestrates it
//! across every partition/layer and updates `index.meta` on success.
//!
//! `KmerIndex` is foreign to this crate, so this is an extension trait,
//! same reasoning as `IndexCompact`/`IndexDump`/`IndexUnitigs`.
use obikindex::layer::{IndexMode, KmerLayer};
use obikindex::{IndexState, KmerIndex, OKIError, OKIResult};
use obikindexer::IndexBuilder;
use obisys::{PartitionRunner, Reporter, Stage, progress_bar};
use tracing::info;
use obikindex::{OKIError, OKIResult};
use obikindex::KmerIndex;
use obikindex::IndexState;
use obisys::PartitionRunner;
const EVIDENCE_FILE: &str = "evidence.bin";
const FINGERPRINT_FILE: &str = "fingerprint.bin";
/// `unitigs.bin.idx`'s filename isn't exposed by `obikindex`/`obiskio` as a
/// path accessor (it's a private implementation detail of exact-evidence
/// construction) — mirrored here as a literal, same as before this module
/// was disabled.
const UNITIG_IDX_FILE: &str = "unitigs.bin.idx";
impl KmerIndex {
/// Convert every layer's evidence bundle to `target` in-place.
pub trait IndexReindex {
/// Convert every layer's evidence bundle to `target`, in place.
///
/// - `Exact` → builds `evidence.bin` + `unitigs.bin.idx`, removes `fingerprint.bin`
/// - `Approx` → builds `fingerprint.bin`, removes `evidence.bin` + `unitigs.bin.idx`
/// - `Exact` → builds `evidence.bin` + `unitigs.bin.idx` where missing, removes `fingerprint.bin`.
/// - `Approx` → builds `fingerprint.bin` where missing or built with a different `b`, removes `evidence.bin` + `unitigs.bin.idx`.
/// - `Hybrid` → builds whichever bundle a layer is still missing, keeps both.
///
/// The MPHF (`mphf.bin`) and unitigs (`unitigs.bin`) are never touched.
/// `index.meta` is updated with the new evidence kind on success.
pub fn reindex(
&mut self,
target: IndexMode,
block_bits: u8,
rep: &mut Reporter,
) -> OKIResult<()> {
if self.state()? != IndexState::Indexed {
return Err(OKIError::NotIndexed(self.root_path.clone()));
/// Each layer's own build step is idempotent (see `IndexBuilder`), so
/// re-running this with the same `target` on an already-converted
/// index is a cheap no-op per layer, not a full rebuild.
fn reindex(&self, target: IndexMode, block_bits: u8, rep: &mut Reporter) -> OKIResult<()>;
}
impl IndexReindex for KmerIndex {
fn reindex(&self, target: IndexMode, block_bits: u8, rep: &mut Reporter) -> OKIResult<()> {
if self.meta().state().map_err(OKIError::Io)? != IndexState::Indexed {
return Err(OKIError::InvalidInput(format!(
"{}: index is not fully built",
self.dir().display()
)));
}
let n = self.n_partitions();
info!(
"reindex {} partition(s): {:?} → {:?}",
n, self.meta.config.evidence, target,
"reindex {} partition(s): {:?} → {target:?}",
n,
self.meta().config().evidence,
);
let t = Stage::start("reindex");
@@ -58,68 +67,56 @@ impl KmerIndex {
pb.finish_and_clear();
let mut config = self.meta.config.clone();
config.evidence = target;
if matches!(config.evidence, IndexMode::Exact) {
config.block_bits = block_bits;
}
let genomes = self.meta.genomes().map_err(OKIError::Io)?;
self.meta.rewrite_config(config, genomes).map_err(OKIError::Io)?;
self.meta = Arc::new(IndexMeta::open(self).map_err(OKIError::Io)?);
self.meta().set_evidence(target, block_bits).map_err(OKIError::Io)?;
rep.push(t.stop());
Ok(())
}
}
/// Process all layers of one partition's index directory.
fn reindex_partition(
index: &KmerIndex,
i: usize,
target: &IndexMode,
block_bits: u8,
) -> OKIResult<()> {
if !index.index_dir(i).exists() {
return Ok(());
}
let n_layers = index
.n_layers(i)
.map_err(|e| OKIError::InvalidInput(e.to_string()))?;
for layer_idx in 0..n_layers {
reindex_layer(&index.layer_dir(i, layer_idx), target, block_bits)?;
/// Process every layer of partition `i`.
fn reindex_partition(index: &KmerIndex, i: usize, target: &IndexMode, block_bits: u8) -> OKIResult<()> {
let partition = index.partition(i)?;
let n_layers = partition.n_layers();
for l in 0..n_layers {
let layer = partition.layer(l)?.open()?;
reindex_layer(&layer, target, block_bits)?;
remove_stale_evidence(&layer, target);
}
Ok(())
}
fn reindex_layer(layer_dir: &Path, target: &IndexMode, block_bits: u8) -> OKIResult<()> {
fn reindex_layer(layer: &KmerLayer, target: &IndexMode, block_bits: u8) -> OKIResult<()> {
match target {
IndexMode::Exact => {
TypedLayer::<()>::build_exact_evidence(layer_dir, block_bits)?;
layer.build_exact_evidence(block_bits)?;
}
IndexMode::Approx { b, z } | IndexMode::Hybrid { b, z } => {
TypedLayer::<()>::build_approx_evidence(layer_dir, *b, *z)?;
IndexMode::Approx { b, z } => {
layer.build_approx_evidence(*b, *z)?;
}
IndexMode::Hybrid { b, z } => {
layer.build_hybrid_evidence(*b, *z, block_bits)?;
}
}
remove_stale_evidence(layer_dir, target)
Ok(())
}
fn remove_stale_evidence(layer_dir: &Path, target: &IndexMode) -> OKIResult<()> {
fn remove_stale_evidence(layer: &KmerLayer, target: &IndexMode) {
match target {
IndexMode::Exact => {
remove_if_exists(&layer_dir.join(FINGERPRINT_FILE));
remove_if_exists(&layer.fingerprint_path());
}
IndexMode::Approx { .. } => {
remove_if_exists(&layer_dir.join(EVIDENCE_FILE));
remove_if_exists(&layer_dir.join(UNITIG_IDX_FILE));
remove_if_exists(&layer.evidence_path());
remove_if_exists(&layer.dir().join(UNITIG_IDX_FILE));
}
IndexMode::Hybrid { .. } => {
// both bundles kept — nothing to remove
}
}
Ok(())
}
fn remove_if_exists(path: &Path) {
if let Err(e) = fs::remove_file(path) {
fn remove_if_exists(path: &std::path::Path) {
if let Err(e) = std::fs::remove_file(path) {
if e.kind() != std::io::ErrorKind::NotFound {
eprintln!("warning: could not remove {}: {e}", path.display());
}