diff --git a/src/obikindex/src/index/meta.rs b/src/obikindex/src/index/meta.rs index fb82753f..83da9cad 100644 --- a/src/obikindex/src/index/meta.rs +++ b/src/obikindex/src/index/meta.rs @@ -218,29 +218,28 @@ impl IndexMeta { self.write_full(&on_disk) } - // /// Overwrite `config` and the whole `genomes` list at once, preserving - // /// `state`. `config` otherwise never changes once an index exists — - // /// this is the deliberate, rare exception for construction paths that - // /// legitimately rewrite it in place (`select_in_place`, `reindex`). - // /// The caller must refresh its own cached `Arc` afterward - // /// (e.g. `self.meta = Arc::new(IndexMeta::open(self)?)`) — this method - // /// only updates the file, it has no way to reach back into whatever - // /// `KmerIndex` holds it. - // /// TODO: This methode is strange - // pub(crate) fn rewrite_config( - // &self, - // config: IndexConfig, - // genomes: Vec, - // ) -> io::Result<()> { - // let _guard = self.lock.write().unwrap(); - // let state = Self::read_full(&self.root_path)?.state; - // self.write_full(&IndexMetadata { - // version: self.version, - // config, - // genomes, - // state, - // }) - // } + /// Update the evidence representation in place — `evidence`, and (for + /// `Exact`) its block size. The only `IndexConfig` fields a reindex is + /// ever allowed to touch: `kmer_size`/`minimizer_size`/`n_bits`/ + /// `with_counts` are baked into the already-built MPHF/unitigs/matrices + /// and can never change post-build, unlike the evidence bundle, which + /// is derived from `unitigs.bin` + `mphf.bin` alone and can be + /// rebuilt in a different form at any time (see + /// `obikindexer::IndexBuilder`). + /// + /// The caller must refresh its own cached `Arc` afterward + /// (e.g. `self.meta = Arc::new(IndexMeta::open(self)?)`) — this method + /// only updates the file, it has no way to reach back into whatever + /// `KmerIndex` holds it. + pub fn set_evidence(&self, evidence: IndexMode, block_bits: u8) -> io::Result<()> { + let _guard = self.lock.write().unwrap(); + let mut on_disk = Self::read_full(&self.root_path)?; + on_disk.config.evidence = evidence; + if matches!(on_disk.config.evidence, IndexMode::Exact) { + on_disk.config.block_bits = block_bits; + } + self.write_full(&on_disk) + } pub fn set_state(&self, state: IndexState) -> io::Result<()> { let _guard = self.lock.write().unwrap(); diff --git a/src/obikindexer/src/extensions/index_builder.rs b/src/obikindexer/src/extensions/index_builder.rs new file mode 100644 index 00000000..2cd2b46f --- /dev/null +++ b/src/obikindexer/src/extensions/index_builder.rs @@ -0,0 +1,68 @@ +//! `IndexBuilder` — public, idempotent evidence-construction primitives +//! for an already-built `KmerLayer` (`unitigs.bin` + `mphf.bin` already on +//! disk; only the evidence bundle — `evidence.bin`/`unitigs.bin.idx` and/or +//! `fingerprint.bin` — is touched). Unlike `PrivateBuilder` (this module's +//! private sibling), these are meant for callers outside this crate that +//! need to add or convert a layer's evidence after the fact — today, +//! `obikrebuild::reindex`. +//! +//! `KmerLayer` itself (`obikindex`) stays a pure data-model type: it +//! exposes paths and reads content, it never builds anything. Construction +//! lives here instead, in the crate whose whole purpose is building index +//! content — same reasoning as `PrivateBuilder`, just public. + +use obikindex::OKIResult; +use obikindex::layer::fingerprint::FingerprintVec; +use obikindex::layer::{KmerLayer, MphfLayer}; + +pub trait IndexBuilder { + /// Build this layer's exact evidence bundle (`evidence.bin` + + /// `unitigs.bin.idx`) if not already present. + /// + /// No-op (`Ok(0)`) if exact evidence already exists — both `Exact` and + /// `Hybrid` layers already carry it. + fn build_exact_evidence(&self, block_bits: u8) -> OKIResult; + + /// Build this layer's approximate (fingerprint) evidence with `b` bits + /// per slot and Findere parameter `z`, if not already present with a + /// matching `b`. + /// + /// No-op (`Ok(0)`) if `Approx`/`Hybrid` evidence with the same `b` + /// already exists — rebuilds it (same file, same `b`, whatever `z`) + /// otherwise. `z` is never persisted (see `IndexMode::Approx`'s own + /// doc): it only governs how many consecutive stored k-mers a query + /// must match, so changing it alone is not something this method has + /// any file-level evidence to detect — the caller decides. + fn build_approx_evidence(&self, b: u8, z: u8) -> OKIResult; + + /// Build whichever of the exact/approximate bundles this layer is + /// still missing, so it ends up carrying both (`Hybrid`). + /// + /// Just the composition of `build_exact_evidence` and + /// `build_approx_evidence` — each is already independently idempotent, + /// so there is no extra "what's already there" decision to make here. + fn build_hybrid_evidence(&self, b: u8, z: u8, block_bits: u8) -> OKIResult; +} + +impl IndexBuilder for KmerLayer { + fn build_exact_evidence(&self, block_bits: u8) -> OKIResult { + if self.evidence_path().exists() { + return Ok(0); + } + MphfLayer::build_exact_evidence(self.dir(), block_bits) + } + + fn build_approx_evidence(&self, b: u8, z: u8) -> OKIResult { + let fingerprint_path = self.fingerprint_path(); + if fingerprint_path.exists() && FingerprintVec::open(&fingerprint_path)?.b() == b { + return Ok(0); + } + MphfLayer::build_approx_evidence(self.dir(), b, z) + } + + fn build_hybrid_evidence(&self, b: u8, z: u8, block_bits: u8) -> OKIResult { + let n_exact = self.build_exact_evidence(block_bits)?; + let n_approx = self.build_approx_evidence(b, z)?; + Ok(n_exact.max(n_approx)) + } +} diff --git a/src/obikindexer/src/extensions/mod.rs b/src/obikindexer/src/extensions/mod.rs index 04d79000..f3f6e0c0 100644 --- a/src/obikindexer/src/extensions/mod.rs +++ b/src/obikindexer/src/extensions/mod.rs @@ -1,245 +1,20 @@ -//! Construction-only `KmerIndex` operations — semantically meaningless -//! once an index is built and being used for reading (query, dump, phylo, -//! or as a `merge`/`select`/`rebuild` *source*): sentinel marking, the -//! per-partition layer-build primitive, and cleanup of build-time scratch -//! files. Private to this crate — no consumer outside the -//! indexing-pipeline algorithms in `obikindexer::algorithms` should ever -//! need them. See `DevDocMD/implementation/partition_layer_cache.md`, -//! "(8)". +//! Extension traits over `obikindex`'s foreign types (`KmerIndex`, +//! `KmerLayer`) — Rust's orphan rule requires such traits to be defined in +//! the crate that implements them for a foreign type, not in `obikindex` +//! itself; `obikindex` has no way to make a trait "visible to only some +//! crates" from its own side. //! -//! Defined here, not in `obikindex`, so it can be genuinely `pub(crate)` -//! while extending a foreign type (`KmerIndex`) — Rust's orphan rule -//! requires the trait itself to be local to the crate that implements it -//! for a foreign type; `obikindex` has no way to make a trait "private -//! except to `obikindexer`" from its own side. -//! -//! Scoped to the six methods exclusive to this crate's own pipeline -//! (`partitionner`/`dereplicator`/`counter`/`layer_builder`). -//! `clear_output_for_create`/`create_skeleton`/`finalize_indexed`/`state` -//! stay inherent on `KmerIndex`, deliberately not moved here: they're also -//! called from `obikindex`'s own `merge`/`select`/`rebuild`/`reindex` -//! modules, which — living inside `obikindex` itself — could never reach a -//! trait defined in `obikindexer` (the dependency only runs -//! `obikindexer → obikindex`, never the other way). Moving those four -//! would require extracting `merge`/`select`/`rebuild`/`reindex` into -//! algorithms first — separate, larger future work, not this one. +//! Two traits, two visibilities, one per submodule: +//! - [`private_builder::PrivateBuilder`] (`pub(crate)`) — construction-only +//! `KmerIndex` operations exclusive to this crate's own indexing +//! pipeline (`algorithms::{partitionner,dereplicator,counter,layer_builder}`). +//! - [`index_builder::IndexBuilder`] (`pub`, re-exported at the crate +//! root) — idempotent per-layer evidence-construction primitives, for +//! any external caller that needs to add/convert a layer's evidence +//! bundle after the fact (`obikrebuild::reindex`). -use std::collections::BTreeMap; -use std::fs; -use std::io; -use std::path::Path; +mod index_builder; +mod private_builder; -use cacheline_ef::{CachelineEf, CachelineEfVec}; -use epserde::prelude::*; -use obicompactvec::{PersistentCompactIntMatrix, PersistentCompactIntVec}; -use obidebruinj::GraphDeBruijn; -use obikindex::layer::IndexMode; -use obikindex::layer::{KmerLayer, TypedLayer}; -use obikindex::{KmerIndex, OKIError, OKIResult}; -use crate::graph_pipeline::{materialize_layer, write_graph_as_unitigs}; -use obiskio::{SKError, SKFileMeta, SKFileReader}; -use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xx64}; - -type Mphf = PtrHash>, Xx64, Vec>; - -pub(crate) trait PrivateBuilder { - /// Mark scatter as complete (`IndexState::Scattered`). - /// - /// If no genome label was set at creation time, one is derived from - /// the index root directory name (stripped of all extensions). `&self` - /// — not `&mut self` anymore: nothing is cached in memory to mutate, - /// see `DevDocMD/implementation/partition_layer_cache.md`, "(11)". - fn mark_scattered(&self) -> OKIResult<()>; - - /// Mark dereplicate+count as complete (`IndexState::Counted`). - fn mark_counted(&self) -> OKIResult<()>; - - /// Partition `i`'s layer 0 — the one and only layer every pipeline - /// algorithm in this crate (`partitionner`/`dereplicator`/`counter`/ - /// `layer_builder`) reads from or writes to; later layers only exist - /// after a `merge`, which lives in `obikindex`, not here. - fn layer0(&self, i: usize) -> OKIResult; - - /// Mark layer construction as complete (`IndexState::Indexed`). - fn mark_indexed(&self) -> OKIResult<()>; - - /// Write `spectrums/{label}.json` from an already-computed kmer - /// spectrum (`f0`/`f1`/abundance histogram). Takes plain values rather - /// than `algorithms::counter::KmerSpectrum` — `KmerIndex` is the data - /// model, `Counter` the algorithm that depends on it, not the other - /// way around, and this is the only field of that type it actually - /// uses. - fn write_spectrum(&self, f0: u64, f1: u64, counts: &BTreeMap) -> OKIResult<()>; - - /// Build the layered MPHF index for partition `i`. - /// - /// Returns the number of canonical k-mers indexed, or 0 if the - /// partition has no data or its layer was already built - /// (resume-safe). - /// - /// Abundance filtering is applied when `min_ab > 1` or - /// `max_ab.is_some()`, using `mphf1.bin` + `counts1.bin` if they - /// exist. Count payload is stored iff `with_counts` is true. - fn build_index_layer( - &self, - i: usize, - min_ab: u32, - max_ab: Option, - with_counts: bool, - mode: &IndexMode, - block_bits: u8, - ) -> Result; - - /// Remove intermediate build artifacts for partition `i`: dereplicated - /// superkmers (+ sidecar), `mphf1.bin`, `counts1.bin`. - fn remove_build_artifacts(&self, i: usize); -} - -impl PrivateBuilder for KmerIndex { - fn mark_scattered(&self) -> OKIResult<()> { - self.meta().mark_scattered().map_err(OKIError::Io) - } - - fn mark_counted(&self) -> OKIResult<()> { - self.meta().mark_counted().map_err(OKIError::Io) - } - - fn layer0(&self, i: usize) -> OKIResult { - self.partition(i)?.layer(0) - } - - fn mark_indexed(&self) -> OKIResult<()> { - self.meta().mark_indexed().map_err(OKIError::Io) - } - - fn write_spectrum(&self, f0: u64, f1: u64, counts: &BTreeMap) -> OKIResult<()> { - let genomes = self.meta().genomes().map_err(OKIError::Io)?; - let label = genomes - .first() - .map(|g| g.label.as_str()) - .unwrap_or("unknown") - .to_owned(); - let spectrums_dir = self.dir().join("spectrums"); - fs::create_dir_all(&spectrums_dir)?; - let path = spectrums_dir.join(format!("{label}.json")); - let spectrum_map: BTreeMap = counts - .iter() - .map(|(&c, &f)| (format!("{c:010}"), f)) - .collect(); - let f = fs::File::create(&path)?; - serde_json::to_writer_pretty( - f, - &serde_json::json!({ "f0": f0, "f1": f1, "spectrum": spectrum_map }), - ) - .map_err(OKIError::Json)?; - Ok(()) - } - - fn build_index_layer( - &self, - i: usize, - min_ab: u32, - max_ab: Option, - with_counts: bool, - mode: &IndexMode, - block_bits: u8, - ) -> Result { - let layer0 = self.layer0(i).map_err(|e| io::Error::other(e.to_string()))?; - let layer0_dir = layer0.dir(); - let dedup_path = layer0.dereplicated_superkmers_path(); - if !dedup_path.exists() { - return Ok(0); - } - - if layer0_dir.join("mphf.bin").exists() { - return Ok(0); - } - - let filter_active = min_ab > 1 || max_ab.is_some(); - let need_counts = filter_active || with_counts; - - let mphf1_opt: Option = if need_counts { - let p = layer0_dir.join("mphf1.bin"); - p.exists().then(|| Mphf::load_full(&p).ok()).flatten() - } else { - None - }; - - let counts1_opt: Option = if need_counts { - let p = layer0_dir.join("counts1.bin"); - p.exists() - .then(|| PersistentCompactIntVec::open(&p).ok()) - .flatten() - } else { - None - }; - - let mut g = GraphDeBruijn::new(); - let mut reader = SKFileReader::open(&dedup_path)?; - for sk in reader.iter() { - for kmer in sk.iter_canonical_kmers() { - let accept = if filter_active { - match (&mphf1_opt, &counts1_opt) { - (Some(mphf), Some(counts)) => { - let ab = counts.get(mphf.index(&kmer.raw())); - ab >= min_ab && max_ab.map_or(true, |max| ab <= max) - } - _ => true, - } - } else { - true - }; - if accept { - g.push(kmer); - } - } - } - - let n_kmers = if with_counts { - let n = write_graph_as_unitigs(g, layer0_dir) - .map_err(|e| io::Error::other(e.to_string()))?; - TypedLayer::::build( - layer0_dir, - block_bits, - mode, - |kmer| match (&mphf1_opt, &counts1_opt) { - (Some(mphf), Some(counts)) => counts.get(mphf.index(&kmer.raw())), - _ => 1, - }, - ) - .map_err(|e| io::Error::other(e.to_string()))?; - n - } else { - materialize_layer(g, layer0_dir, block_bits, mode) - .map_err(|e| io::Error::other(e.to_string()))? - }; - - Ok(n_kmers) - } - - fn remove_build_artifacts(&self, i: usize) { - let layer0 = match self.layer0(i) { - Ok(layer) => layer, - Err(e) => { - eprintln!("warning: could not locate layer 0 of partition {i}: {e}"); - return; - } - }; - let layer0_dir = layer0.dir(); - let dedup = layer0.dereplicated_superkmers_path(); - remove_if_exists(&SKFileMeta::sidecar_path(&dedup)); - remove_if_exists(&dedup); - remove_if_exists(&layer0_dir.join("mphf1.bin")); - remove_if_exists(&layer0_dir.join("counts1.bin")); - } -} - -// ── private helpers ────────────────────────────────────────────────────── - -fn remove_if_exists(path: &Path) { - if let Err(e) = fs::remove_file(path) { - if e.kind() != io::ErrorKind::NotFound { - eprintln!("warning: could not remove {}: {e}", path.display()); - } - } -} +pub use index_builder::IndexBuilder; +pub(crate) use private_builder::PrivateBuilder; diff --git a/src/obikindexer/src/extensions/private_builder.rs b/src/obikindexer/src/extensions/private_builder.rs new file mode 100644 index 00000000..8134b99f --- /dev/null +++ b/src/obikindexer/src/extensions/private_builder.rs @@ -0,0 +1,240 @@ +//! `PrivateBuilder` — construction-only `KmerIndex` operations: +//! sentinel marking, the per-partition layer-build primitive, and cleanup +//! of build-time scratch files. Semantically meaningless once an index is +//! built and being used for reading (query, dump, phylo, or as a +//! `merge`/`select`/`rebuild` *source*). +//! +//! Scoped to the six methods exclusive to this crate's own pipeline +//! (`partitionner`/`dereplicator`/`counter`/`layer_builder`) — no +//! consumer outside those algorithms should ever need them, hence +//! `pub(crate)` (see the module-level orphan-rule note in `extensions`'s +//! `mod.rs`). See `DevDocMD/implementation/partition_layer_cache.md`, "(8)". +//! +//! `clear_output_for_create`/`create_skeleton`/`finalize_indexed`/`state` +//! stay inherent on `KmerIndex`, deliberately not moved here: they're also +//! called from `obikindex`'s own `merge`/`select`/`rebuild`/`reindex` +//! modules, which — living inside `obikindex` itself — could never reach a +//! trait defined in `obikindexer` (the dependency only runs +//! `obikindexer → obikindex`, never the other way). Moving those four +//! would require extracting `merge`/`select`/`rebuild`/`reindex` into +//! algorithms first — separate, larger future work, not this one. + +use std::collections::BTreeMap; +use std::fs; +use std::io; +use std::path::Path; + +use cacheline_ef::{CachelineEf, CachelineEfVec}; +use epserde::prelude::*; +use obicompactvec::{PersistentCompactIntMatrix, PersistentCompactIntVec}; +use obidebruinj::GraphDeBruijn; +use obikindex::layer::IndexMode; +use obikindex::layer::{KmerLayer, TypedLayer}; +use obikindex::{KmerIndex, OKIError, OKIResult}; +use crate::graph_pipeline::{materialize_layer, write_graph_as_unitigs}; +use obiskio::{SKError, SKFileMeta, SKFileReader}; +use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xx64}; + +type Mphf = PtrHash>, Xx64, Vec>; + +pub(crate) trait PrivateBuilder { + /// Mark scatter as complete (`IndexState::Scattered`). + /// + /// If no genome label was set at creation time, one is derived from + /// the index root directory name (stripped of all extensions). `&self` + /// — not `&mut self` anymore: nothing is cached in memory to mutate, + /// see `DevDocMD/implementation/partition_layer_cache.md`, "(11)". + fn mark_scattered(&self) -> OKIResult<()>; + + /// Mark dereplicate+count as complete (`IndexState::Counted`). + fn mark_counted(&self) -> OKIResult<()>; + + /// Partition `i`'s layer 0 — the one and only layer every pipeline + /// algorithm in this crate (`partitionner`/`dereplicator`/`counter`/ + /// `layer_builder`) reads from or writes to; later layers only exist + /// after a `merge`, which lives in `obikindex`, not here. + fn layer0(&self, i: usize) -> OKIResult; + + /// Mark layer construction as complete (`IndexState::Indexed`). + fn mark_indexed(&self) -> OKIResult<()>; + + /// Write `spectrums/{label}.json` from an already-computed kmer + /// spectrum (`f0`/`f1`/abundance histogram). Takes plain values rather + /// than `algorithms::counter::KmerSpectrum` — `KmerIndex` is the data + /// model, `Counter` the algorithm that depends on it, not the other + /// way around, and this is the only field of that type it actually + /// uses. + fn write_spectrum(&self, f0: u64, f1: u64, counts: &BTreeMap) -> OKIResult<()>; + + /// Build the layered MPHF index for partition `i`. + /// + /// Returns the number of canonical k-mers indexed, or 0 if the + /// partition has no data or its layer was already built + /// (resume-safe). + /// + /// Abundance filtering is applied when `min_ab > 1` or + /// `max_ab.is_some()`, using `mphf1.bin` + `counts1.bin` if they + /// exist. Count payload is stored iff `with_counts` is true. + fn build_index_layer( + &self, + i: usize, + min_ab: u32, + max_ab: Option, + with_counts: bool, + mode: &IndexMode, + block_bits: u8, + ) -> Result; + + /// Remove intermediate build artifacts for partition `i`: dereplicated + /// superkmers (+ sidecar), `mphf1.bin`, `counts1.bin`. + fn remove_build_artifacts(&self, i: usize); +} + +impl PrivateBuilder for KmerIndex { + fn mark_scattered(&self) -> OKIResult<()> { + self.meta().mark_scattered().map_err(OKIError::Io) + } + + fn mark_counted(&self) -> OKIResult<()> { + self.meta().mark_counted().map_err(OKIError::Io) + } + + fn layer0(&self, i: usize) -> OKIResult { + self.partition(i)?.layer(0) + } + + fn mark_indexed(&self) -> OKIResult<()> { + self.meta().mark_indexed().map_err(OKIError::Io) + } + + fn write_spectrum(&self, f0: u64, f1: u64, counts: &BTreeMap) -> OKIResult<()> { + let genomes = self.meta().genomes().map_err(OKIError::Io)?; + let label = genomes + .first() + .map(|g| g.label.as_str()) + .unwrap_or("unknown") + .to_owned(); + let spectrums_dir = self.dir().join("spectrums"); + fs::create_dir_all(&spectrums_dir)?; + let path = spectrums_dir.join(format!("{label}.json")); + let spectrum_map: BTreeMap = counts + .iter() + .map(|(&c, &f)| (format!("{c:010}"), f)) + .collect(); + let f = fs::File::create(&path)?; + serde_json::to_writer_pretty( + f, + &serde_json::json!({ "f0": f0, "f1": f1, "spectrum": spectrum_map }), + ) + .map_err(OKIError::Json)?; + Ok(()) + } + + fn build_index_layer( + &self, + i: usize, + min_ab: u32, + max_ab: Option, + with_counts: bool, + mode: &IndexMode, + block_bits: u8, + ) -> Result { + let layer0 = self.layer0(i).map_err(|e| io::Error::other(e.to_string()))?; + let layer0_dir = layer0.dir(); + let dedup_path = layer0.dereplicated_superkmers_path(); + if !dedup_path.exists() { + return Ok(0); + } + + if layer0_dir.join("mphf.bin").exists() { + return Ok(0); + } + + let filter_active = min_ab > 1 || max_ab.is_some(); + let need_counts = filter_active || with_counts; + + let mphf1_opt: Option = if need_counts { + let p = layer0_dir.join("mphf1.bin"); + p.exists().then(|| Mphf::load_full(&p).ok()).flatten() + } else { + None + }; + + let counts1_opt: Option = if need_counts { + let p = layer0_dir.join("counts1.bin"); + p.exists() + .then(|| PersistentCompactIntVec::open(&p).ok()) + .flatten() + } else { + None + }; + + let mut g = GraphDeBruijn::new(); + let mut reader = SKFileReader::open(&dedup_path)?; + for sk in reader.iter() { + for kmer in sk.iter_canonical_kmers() { + let accept = if filter_active { + match (&mphf1_opt, &counts1_opt) { + (Some(mphf), Some(counts)) => { + let ab = counts.get(mphf.index(&kmer.raw())); + ab >= min_ab && max_ab.map_or(true, |max| ab <= max) + } + _ => true, + } + } else { + true + }; + if accept { + g.push(kmer); + } + } + } + + let n_kmers = if with_counts { + let n = write_graph_as_unitigs(g, layer0_dir) + .map_err(|e| io::Error::other(e.to_string()))?; + TypedLayer::::build( + layer0_dir, + block_bits, + mode, + |kmer| match (&mphf1_opt, &counts1_opt) { + (Some(mphf), Some(counts)) => counts.get(mphf.index(&kmer.raw())), + _ => 1, + }, + ) + .map_err(|e| io::Error::other(e.to_string()))?; + n + } else { + materialize_layer(g, layer0_dir, block_bits, mode) + .map_err(|e| io::Error::other(e.to_string()))? + }; + + Ok(n_kmers) + } + + fn remove_build_artifacts(&self, i: usize) { + let layer0 = match self.layer0(i) { + Ok(layer) => layer, + Err(e) => { + eprintln!("warning: could not locate layer 0 of partition {i}: {e}"); + return; + } + }; + let layer0_dir = layer0.dir(); + let dedup = layer0.dereplicated_superkmers_path(); + remove_if_exists(&SKFileMeta::sidecar_path(&dedup)); + remove_if_exists(&dedup); + remove_if_exists(&layer0_dir.join("mphf1.bin")); + remove_if_exists(&layer0_dir.join("counts1.bin")); + } +} + +// ── private helpers ────────────────────────────────────────────────────── + +fn remove_if_exists(path: &Path) { + if let Err(e) = fs::remove_file(path) { + if e.kind() != io::ErrorKind::NotFound { + eprintln!("warning: could not remove {}: {e}", path.display()); + } + } +} diff --git a/src/obikindexer/src/lib.rs b/src/obikindexer/src/lib.rs index a6f0f5bc..514f69a2 100644 --- a/src/obikindexer/src/lib.rs +++ b/src/obikindexer/src/lib.rs @@ -9,11 +9,14 @@ //! (routing raw super-kmers into partitions), `dereplicator` (deduplicating //! a partition's raw super-kmers), `counter` (counting unique canonical //! kmers), `layer_builder` (turning dereplicated super-kmers + counts into -//! the real layer 0). [`extensions`] (private — see its own module docs) -//! holds construction-only `KmerIndex` operations these algorithms share. +//! the real layer 0). `extensions` (module itself private — see its own +//! docs) holds `KmerIndex`/`KmerLayer` extension traits: some exclusive to +//! this crate's own pipeline, one ([`IndexBuilder`]) public and re-exported +//! here for external callers. pub mod algorithms; -pub(crate) mod extensions; +mod extensions; pub mod graph_pipeline; +pub use extensions::IndexBuilder; pub use graph_pipeline::{build_graph, build_layer_from_kmers, materialize_layer, write_graph_as_unitigs}; diff --git a/src/obikmer2/src/cmd/convert/mod.rs b/src/obikmer2/src/cmd/convert/mod.rs new file mode 100644 index 00000000..2e5da7c6 --- /dev/null +++ b/src/obikmer2/src/cmd/convert/mod.rs @@ -0,0 +1,138 @@ +use std::path::PathBuf; + +use clap::Args; +use obikindex::KmerIndex; +use obikindex::layer::IndexMode; +use obikrebuild::IndexReindex; +use obisys::Reporter; +use tracing::info; + +use crate::cli::block_size_to_bits; +use super::index::resolve_approx_params; + +#[derive(Args)] +pub struct ConvertArgs { + /// Index directory to convert (modified in-place) + pub index: PathBuf, + + #[command(flatten)] + mode: ModeArgs, + + /// Fingerprint bits per slot (b) — required with --hybrid-evidence + /// when the source index is currently exact (there is no existing + /// fingerprint to inherit `b` from); rejected otherwise (the source + /// already fixes `b`, either from its own --approx-evidence or from + /// an already-approximate/hybrid index being converted to hybrid). + #[arg(long, value_name = "BITS")] + pub evidence_bits: Option, + + /// Findere z parameter: number of consecutive *stored* k-mers (length + /// = the index's own k-mer size, fixed forever at `index` build time) + /// that must all match to confirm a hit. + /// + /// WARNING: this does NOT shorten the indexed k-mer length the way + /// `-z` does at `index` build time — that length can never change + /// after the fact. Instead it *extends* the effective match window: + /// requiring z consecutive stored k-mers of length s to all match is + /// equivalent to requiring (s + z − 1) consecutive identical bases in + /// the query. E.g. on a k=31 index, z=2 requires 32 consecutive + /// matching bases, not 30 — going from z=1 to z=2 makes matching + /// *stricter*, it does not "index shorter k-mers". + #[arg(short = 'z', long)] + pub findere_z: Option, + + /// Target false-positive rate per z-window (e.g. 0.01) — derives `b` + /// or `z` when one of them isn't given directly (see `index --fp` for + /// the same resolution rules). + #[arg(long)] + pub fp: Option, + + /// Block size for exact evidence `.idx` (number of unitigs per block). + /// Ignored when converting to pure approximate evidence. + #[arg(long, default_value_t = 1)] + pub block_size: usize, +} + +#[derive(Args)] +#[group(required = true, multiple = false)] +struct ModeArgs { + /// Convert to exact evidence (zero false positives). + #[arg(long)] + exact_evidence: bool, + + /// Convert to approximate (fingerprint-only) evidence. Value = fingerprint bits per slot (b). + #[arg(long, value_name = "BITS")] + approx_evidence: Option, + + /// Convert to hybrid evidence (both exact and approximate bundles kept). + #[arg(long)] + hybrid_evidence: bool, +} + +pub fn run(args: ConvertArgs) { + // Modifies the index in place; acquired before opening so a concurrent + // writer can't slip in between the open and the convert below. + let _lock = obisys::DirLock::acquire(&args.index).unwrap_or_else(|e| { + eprintln!("error locking index directory {}: {e}", args.index.display()); + std::process::exit(1); + }); + + let idx = KmerIndex::open(&args.index).unwrap_or_else(|e| { + eprintln!("error opening index: {e}"); + std::process::exit(1); + }); + + let current = idx.meta().config().evidence.clone(); + + let target = if args.mode.exact_evidence { + warn_if_evidence_params_given(&args, "--exact-evidence"); + info!("target: exact evidence"); + IndexMode::Exact + } else if let Some(bits) = args.mode.approx_evidence { + let (z, b, fp) = resolve_approx_params(args.findere_z, Some(bits), args.fp); + info!("target: approximate evidence — b={b}, z={z}, fp={fp:.2e}"); + IndexMode::Approx { b, z } + } else { + debug_assert!(args.mode.hybrid_evidence); + match current { + IndexMode::Exact => { + let bits = args.evidence_bits.unwrap_or_else(|| { + eprintln!( + "error: --hybrid-evidence requires --evidence-bits when starting from an exact index" + ); + std::process::exit(1); + }); + let (z, b, fp) = resolve_approx_params(args.findere_z, Some(bits), args.fp); + info!("target: hybrid evidence — b={b}, z={z}, fp={fp:.2e}"); + IndexMode::Hybrid { b, z } + } + IndexMode::Approx { b, z } | IndexMode::Hybrid { b, z } => { + if args.evidence_bits.is_some() || args.findere_z.is_some() || args.fp.is_some() { + eprintln!( + "error: --evidence-bits/-z/--fp are not allowed with --hybrid-evidence \ + when the source index is already approximate/hybrid — its existing \ + b={b}, z={z} are reused as-is" + ); + std::process::exit(1); + } + info!("target: hybrid evidence — reusing existing b={b}, z={z}"); + IndexMode::Hybrid { b, z } + } + } + }; + + let block_bits = block_size_to_bits(args.block_size); + let mut rep = Reporter::new(); + idx.reindex(target, block_bits, &mut rep).unwrap_or_else(|e| { + eprintln!("convert error: {e}"); + std::process::exit(1); + }); + + rep.print(); +} + +fn warn_if_evidence_params_given(args: &ConvertArgs, mode: &str) { + if args.evidence_bits.is_some() || args.findere_z.is_some() || args.fp.is_some() { + eprintln!("warning: --evidence-bits/-z/--fp ignored with {mode}"); + } +} diff --git a/src/obikmer2/src/cmd/mod.rs b/src/obikmer2/src/cmd/mod.rs index e4b61300..f448b81c 100644 --- a/src/obikmer2/src/cmd/mod.rs +++ b/src/obikmer2/src/cmd/mod.rs @@ -1,4 +1,5 @@ pub mod annotate; +pub mod convert; pub mod dump; pub mod estimate; pub mod filter; diff --git a/src/obikmer2/src/main.rs b/src/obikmer2/src/main.rs index a1fec405..f7680a5e 100644 --- a/src/obikmer2/src/main.rs +++ b/src/obikmer2/src/main.rs @@ -37,6 +37,8 @@ enum Commands { Annotate(cmd::annotate::AnnotateArgs), /// Maintenance/inspection operations on already-built indexes Utils(cmd::utils::UtilsArgs), + /// Convert an index's evidence representation (exact/approximate/hybrid), in place + Convert(cmd::convert::ConvertArgs), } fn main() { @@ -61,5 +63,6 @@ fn main() { Commands::Estimate(args) => cmd::estimate::run(args), Commands::Annotate(args) => cmd::annotate::run(args), Commands::Utils(args) => cmd::utils::run(args), + Commands::Convert(args) => cmd::convert::run(args), } } diff --git a/src/obikrebuild/src/lib.rs b/src/obikrebuild/src/lib.rs index 3e6658e8..67fe5a01 100644 --- a/src/obikrebuild/src/lib.rs +++ b/src/obikrebuild/src/lib.rs @@ -6,12 +6,9 @@ //! //! [`compact_layer`] collapses a partition's accumulated layers (the //! stopgap `obikmerge` uses to add genomes without a full rebuild) back -//! into one, in place. `reindex` (reindexing an existing index's evidence -//! representation in place — `Exact`/`Approx`/`Hybrid`) is temporarily -//! disabled: it predates this session's refactor and still reaches -//! `obikindex` internals (`root_path`/`meta` fields, `index_dir`/ -//! `layer_dir` methods) that are no longer accessible from outside that -//! crate. Fixing it is separate, future work. +//! into one, in place. [`reindex`] converts an existing index's evidence +//! representation in place (`Exact`/`Approx`/`Hybrid`), delegating the +//! actual per-layer construction to `obikindexer::IndexBuilder`. //! //! Removing k-mers (filtering) and repacking matrix files are *not* here: //! the former lives in `obikfilter::Filter` (a fresh index at a new @@ -21,6 +18,7 @@ //! cannot depend back on this crate. mod compact_layer; -// mod reindex; // disabled — see module doc above. +mod reindex; pub use compact_layer::IndexCompact; +pub use reindex::IndexReindex; diff --git a/src/obikrebuild/src/reindex.rs b/src/obikrebuild/src/reindex.rs index 0ede2cf2..053eb502 100644 --- a/src/obikrebuild/src/reindex.rs +++ b/src/obikrebuild/src/reindex.rs @@ -1,43 +1,52 @@ -use obikindex::IndexBuilder; -use obikindex::IndexMeta; -use obikindex::layer::{IndexMode, TypedLayer}; -use obisys::{Reporter, Stage, progress_bar}; -use std::fs; -use std::path::Path; -use std::sync::Arc; +//! Convert an existing index's evidence representation in place — +//! `Exact`/`Approx`/`Hybrid`. `unitigs.bin`/`mphf.bin` are never touched; +//! only the evidence bundle (`evidence.bin`/`unitigs.bin.idx` and/or +//! `fingerprint.bin`) is — the actual per-layer construction is +//! `obikindexer::IndexBuilder`'s job, this module only orchestrates it +//! across every partition/layer and updates `index.meta` on success. +//! +//! `KmerIndex` is foreign to this crate, so this is an extension trait, +//! same reasoning as `IndexCompact`/`IndexDump`/`IndexUnitigs`. + +use obikindex::layer::{IndexMode, KmerLayer}; +use obikindex::{IndexState, KmerIndex, OKIError, OKIResult}; +use obikindexer::IndexBuilder; +use obisys::{PartitionRunner, Reporter, Stage, progress_bar}; use tracing::info; -use obikindex::{OKIError, OKIResult}; -use obikindex::KmerIndex; -use obikindex::IndexState; -use obisys::PartitionRunner; - -const EVIDENCE_FILE: &str = "evidence.bin"; -const FINGERPRINT_FILE: &str = "fingerprint.bin"; +/// `unitigs.bin.idx`'s filename isn't exposed by `obikindex`/`obiskio` as a +/// path accessor (it's a private implementation detail of exact-evidence +/// construction) — mirrored here as a literal, same as before this module +/// was disabled. const UNITIG_IDX_FILE: &str = "unitigs.bin.idx"; -impl KmerIndex { - /// Convert every layer's evidence bundle to `target` in-place. +pub trait IndexReindex { + /// Convert every layer's evidence bundle to `target`, in place. /// - /// - `Exact` → builds `evidence.bin` + `unitigs.bin.idx`, removes `fingerprint.bin` - /// - `Approx` → builds `fingerprint.bin`, removes `evidence.bin` + `unitigs.bin.idx` + /// - `Exact` → builds `evidence.bin` + `unitigs.bin.idx` where missing, removes `fingerprint.bin`. + /// - `Approx` → builds `fingerprint.bin` where missing or built with a different `b`, removes `evidence.bin` + `unitigs.bin.idx`. + /// - `Hybrid` → builds whichever bundle a layer is still missing, keeps both. /// - /// The MPHF (`mphf.bin`) and unitigs (`unitigs.bin`) are never touched. - /// `index.meta` is updated with the new evidence kind on success. - pub fn reindex( - &mut self, - target: IndexMode, - block_bits: u8, - rep: &mut Reporter, - ) -> OKIResult<()> { - if self.state()? != IndexState::Indexed { - return Err(OKIError::NotIndexed(self.root_path.clone())); + /// Each layer's own build step is idempotent (see `IndexBuilder`), so + /// re-running this with the same `target` on an already-converted + /// index is a cheap no-op per layer, not a full rebuild. + fn reindex(&self, target: IndexMode, block_bits: u8, rep: &mut Reporter) -> OKIResult<()>; +} + +impl IndexReindex for KmerIndex { + fn reindex(&self, target: IndexMode, block_bits: u8, rep: &mut Reporter) -> OKIResult<()> { + if self.meta().state().map_err(OKIError::Io)? != IndexState::Indexed { + return Err(OKIError::InvalidInput(format!( + "{}: index is not fully built", + self.dir().display() + ))); } let n = self.n_partitions(); info!( - "reindex {} partition(s): {:?} → {:?}", - n, self.meta.config.evidence, target, + "reindex {} partition(s): {:?} → {target:?}", + n, + self.meta().config().evidence, ); let t = Stage::start("reindex"); @@ -58,68 +67,56 @@ impl KmerIndex { pb.finish_and_clear(); - let mut config = self.meta.config.clone(); - config.evidence = target; - if matches!(config.evidence, IndexMode::Exact) { - config.block_bits = block_bits; - } - let genomes = self.meta.genomes().map_err(OKIError::Io)?; - self.meta.rewrite_config(config, genomes).map_err(OKIError::Io)?; - self.meta = Arc::new(IndexMeta::open(self).map_err(OKIError::Io)?); + self.meta().set_evidence(target, block_bits).map_err(OKIError::Io)?; rep.push(t.stop()); Ok(()) } } -/// Process all layers of one partition's index directory. -fn reindex_partition( - index: &KmerIndex, - i: usize, - target: &IndexMode, - block_bits: u8, -) -> OKIResult<()> { - if !index.index_dir(i).exists() { - return Ok(()); - } - let n_layers = index - .n_layers(i) - .map_err(|e| OKIError::InvalidInput(e.to_string()))?; - for layer_idx in 0..n_layers { - reindex_layer(&index.layer_dir(i, layer_idx), target, block_bits)?; +/// Process every layer of partition `i`. +fn reindex_partition(index: &KmerIndex, i: usize, target: &IndexMode, block_bits: u8) -> OKIResult<()> { + let partition = index.partition(i)?; + let n_layers = partition.n_layers(); + for l in 0..n_layers { + let layer = partition.layer(l)?.open()?; + reindex_layer(&layer, target, block_bits)?; + remove_stale_evidence(&layer, target); } Ok(()) } -fn reindex_layer(layer_dir: &Path, target: &IndexMode, block_bits: u8) -> OKIResult<()> { +fn reindex_layer(layer: &KmerLayer, target: &IndexMode, block_bits: u8) -> OKIResult<()> { match target { IndexMode::Exact => { - TypedLayer::<()>::build_exact_evidence(layer_dir, block_bits)?; + layer.build_exact_evidence(block_bits)?; } - IndexMode::Approx { b, z } | IndexMode::Hybrid { b, z } => { - TypedLayer::<()>::build_approx_evidence(layer_dir, *b, *z)?; + IndexMode::Approx { b, z } => { + layer.build_approx_evidence(*b, *z)?; + } + IndexMode::Hybrid { b, z } => { + layer.build_hybrid_evidence(*b, *z, block_bits)?; } } - remove_stale_evidence(layer_dir, target) + Ok(()) } -fn remove_stale_evidence(layer_dir: &Path, target: &IndexMode) -> OKIResult<()> { +fn remove_stale_evidence(layer: &KmerLayer, target: &IndexMode) { match target { IndexMode::Exact => { - remove_if_exists(&layer_dir.join(FINGERPRINT_FILE)); + remove_if_exists(&layer.fingerprint_path()); } IndexMode::Approx { .. } => { - remove_if_exists(&layer_dir.join(EVIDENCE_FILE)); - remove_if_exists(&layer_dir.join(UNITIG_IDX_FILE)); + remove_if_exists(&layer.evidence_path()); + remove_if_exists(&layer.dir().join(UNITIG_IDX_FILE)); } IndexMode::Hybrid { .. } => { // both bundles kept — nothing to remove } } - Ok(()) } -fn remove_if_exists(path: &Path) { - if let Err(e) = fs::remove_file(path) { +fn remove_if_exists(path: &std::path::Path) { + if let Err(e) = std::fs::remove_file(path) { if e.kind() != std::io::ErrorKind::NotFound { eprintln!("warning: could not remove {}: {e}", path.display()); }