OBIKmers/obikmer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 34 Wiki Activity

refactor: shift indexing algorithms to layer-centric partition APIs

Browse Source 
Replaces manual filesystem path handling and parallel iteration with dedicated internal utilities. Introduces `par_over_layer0` and `layer0()` to abstract partition layer access, enabling partition-driven initialization and centralized progress tracking. Removes disabled or internal methods (`rewrite_config`, `open_data`) no longer part of the active interface. Updates error propagation and metadata persistence to align with the new layer-centric workflow.
This commit is contained in:
Eric Coissac
2026-08-22 13:51:42 +02:00
parent fc4464a0ef
commit 2419a6c21d
10 changed files with 234 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/obikindex/src/index/kmer_index.rs
+67 -67
View File
@@ -5,7 +5,7 @@ use std::sync::Arc;
use crate::layer::KmerLayer;
use crate::partition::KmerPartition;
use obisys::progress_bar;
use rayon::prelude::*;
// use rayon::prelude::*;
use obikseq::{set_k, set_m};
@@ -112,16 +112,16 @@ impl KmerIndex {
1usize << self.meta.config.n_bits
}
/// Path of partition `i`'s raw directory (`partitions/part_{i:05}`) —
/// delegates to `crate::partition`, the Partition tier's own naming
/// primitive (mirrors `layer_dir` delegating to `crate::layer`).
/// `obikindexer::algorithms::partitionner::PartitionRouter` reaches this
/// same directory only indirectly, through this method — `obikindexer`
/// depends on `KmerIndex`, not the other way around — see
/// `DevDocMD/implementation/partition_layer_cache.md`.
pub(crate) fn partition_dir(&self, i: usize) -> OKIResult<PathBuf> {
Ok(self.partition(i)?.dir().to_path_buf())
}
// /// Path of partition `i`'s raw directory (`partitions/part_{i:05}`) —
// /// delegates to `crate::partition`, the Partition tier's own naming
// /// primitive (mirrors `layer_dir` delegating to `crate::layer`).
// /// `obikindexer::algorithms::partitionner::PartitionRouter` reaches this
// /// same directory only indirectly, through this method — `obikindexer`
// /// depends on `KmerIndex`, not the other way around — see
// /// `DevDocMD/implementation/partition_layer_cache.md`.
// pub(crate) fn partition_dir(&self, i: usize) -> OKIResult<PathBuf> {
// Ok(self.partition(i)?.dir().to_path_buf())
// }
pub fn partition(&self, i: usize) -> OKIResult<KmerPartition> {
let n = self.n_partitions();
@@ -227,62 +227,62 @@ impl KmerIndex {
Ok(())
}
/// Write a `layer_meta.json` in any layer directory that is missing one.
///
/// Old indexes were built before this file was required. The number of
/// kmers is recovered from `unitigs.bin`, which is always present.
/// TODO: Should not exist anymore
pub(crate) fn upgrade_layer_meta(&self) -> OKIResult<()> {
use obicompactvec::LayerMeta;
use obiskio::UnitigFileReader;
// /// Write a `layer_meta.json` in any layer directory that is missing one.
// ///
// /// Old indexes were built before this file was required. The number of
// /// kmers is recovered from `unitigs.bin`, which is always present.
// /// TODO: Should not exist anymore
// pub(crate) fn upgrade_layer_meta(&self) -> OKIResult<()> {
// use obicompactvec::LayerMeta;
// use obiskio::UnitigFileReader;
let n = self.n_partitions();
let errors: Vec<_> = (0..n)
.into_par_iter()
.filter_map(|i| {
let index_dir = self.index_dir(i);
if !index_dir.exists() {
return None;
}
let n_layers = match self.n_layers(i) {
Ok(n) => n,
Err(e) => {
return Some(OKIError::Io(std::io::Error::new(
std::io::ErrorKind::Other,
e.to_string(),
)));
}
};
for l in 0..n_layers {
let layer_dir = match self.layer_dir(i, l) {
Ok(d) => d,
Err(e) => {
return Some(OKIError::Io(std::io::Error::new(
std::io::ErrorKind::Other,
e.to_string(),
)));
}
};
let meta_path = layer_dir.join(LayerMeta::FILENAME);
if meta_path.exists() {
continue;
}
let unitigs_path = layer_dir.join("unitigs.bin");
let n_kmers = match UnitigFileReader::open_sequential(&unitigs_path) {
Ok(r) => r.n_kmers(),
Err(e) => return Some(OKIError::Partition(e)),
};
if let Err(e) = LayerMeta::save(&layer_dir, n_kmers) {
return Some(OKIError::Io(e));
}
}
None
})
.collect();
// let n = self.n_partitions();
// let errors: Vec<_> = (0..n)
// .into_par_iter()
// .filter_map(|i| {
// let index_dir = self.index_dir(i);
// if !index_dir.exists() {
// return None;
// }
// let n_layers = match self.n_layers(i) {
// Ok(n) => n,
// Err(e) => {
// return Some(OKIError::Io(std::io::Error::new(
// std::io::ErrorKind::Other,
// e.to_string(),
// )));
// }
// };
// for l in 0..n_layers {
// let layer_dir = match self.layer_dir(i, l) {
// Ok(d) => d,
// Err(e) => {
// return Some(OKIError::Io(std::io::Error::new(
// std::io::ErrorKind::Other,
// e.to_string(),
// )));
// }
// };
// let meta_path = layer_dir.join(LayerMeta::FILENAME);
// if meta_path.exists() {
// continue;
// }
// let unitigs_path = layer_dir.join("unitigs.bin");
// let n_kmers = match UnitigFileReader::open_sequential(&unitigs_path) {
// Ok(r) => r.n_kmers(),
// Err(e) => return Some(OKIError::Partition(e)),
// };
// if let Err(e) = LayerMeta::save(&layer_dir, n_kmers) {
// return Some(OKIError::Io(e));
// }
// }
// None
// })
// .collect();
if let Some(e) = errors.into_iter().next() {
return Err(e);
}
Ok(())
}
// if let Some(e) = errors.into_iter().next() {
// return Err(e);
// }
// Ok(())
// }
}
src/obikindex/src/index/meta.rs
+23 -23
View File
@@ -218,29 +218,29 @@ impl IndexMeta {
self.write_full(&on_disk)
}
/// Overwrite `config` and the whole `genomes` list at once, preserving
/// `state`. `config` otherwise never changes once an index exists —
/// this is the deliberate, rare exception for construction paths that
/// legitimately rewrite it in place (`select_in_place`, `reindex`).
/// The caller must refresh its own cached `Arc<IndexMeta>` afterward
/// (e.g. `self.meta = Arc::new(IndexMeta::open(self)?)`) — this method
/// only updates the file, it has no way to reach back into whatever
/// `KmerIndex` holds it.
/// TODO: This methode is strange
pub(crate) fn rewrite_config(
&self,
config: IndexConfig,
genomes: Vec<GenomeInfo>,
) -> io::Result<()> {
let _guard = self.lock.write().unwrap();
let state = Self::read_full(&self.root_path)?.state;
self.write_full(&IndexMetadata {
version: self.version,
config,
genomes,
state,
})
}
// /// Overwrite `config` and the whole `genomes` list at once, preserving
// /// `state`. `config` otherwise never changes once an index exists —
// /// this is the deliberate, rare exception for construction paths that
// /// legitimately rewrite it in place (`select_in_place`, `reindex`).
// /// The caller must refresh its own cached `Arc<IndexMeta>` afterward
// /// (e.g. `self.meta = Arc::new(IndexMeta::open(self)?)`) — this method
// /// only updates the file, it has no way to reach back into whatever
// /// `KmerIndex` holds it.
// /// TODO: This methode is strange
// pub(crate) fn rewrite_config(
// &self,
// config: IndexConfig,
// genomes: Vec<GenomeInfo>,
// ) -> io::Result<()> {
// let _guard = self.lock.write().unwrap();
// let state = Self::read_full(&self.root_path)?.state;
// self.write_full(&IndexMetadata {
// version: self.version,
// config,
// genomes,
// state,
// })
// }
pub fn set_state(&self, state: IndexState) -> io::Result<()> {
let _guard = self.lock.write().unwrap();
src/obikindex/src/layer/typed_layer.rs
+15 -15
View File
@@ -1,4 +1,4 @@
use crate::layer::utils::layer_dir;
// use crate::layer::utils::layer_dir;
use obicompactvec::{
BinaryMatrix, PersistentBitMatrix, PersistentBitMatrixBuilder, PersistentCompactIntMatrix,
PersistentCompactIntMatrixBuilder, PersistentSparseBitMatrix,
@@ -24,20 +24,20 @@ pub trait LayerData: Sized {
fn read(&self, slot: usize) -> Self::Item;
}
/// Opens layer `i`'s data only, skipping the MPHF — for callers that only
/// need matrix-level operations (distance traits, column weights, group
/// filters, sub-matrix extraction) and never look up a kmer for this layer.
/// `TypedLayer<D>::open` always pays for the MPHF too (and doesn't expose `data`
/// once open), so it's the wrong tool for these; this is the other half of
/// the same `D::open(layer_dir)` call, without the MPHF alongside it.
///
/// Takes `(root, i)`, not a pre-built path: the caller names a layer
/// *number* within a partition it already knows the root of, the same
/// vocabulary as [`layer_dir`] and `LayeredMap` — never the `layer_N`
/// naming convention itself, which stays private to this crate.
pub(crate) fn open_data<D: LayerData>(root: &Path, i: usize) -> OKIResult<D> {
D::open(&layer_dir(root, i))
}
// /// Opens layer `i`'s data only, skipping the MPHF — for callers that only
// /// need matrix-level operations (distance traits, column weights, group
// /// filters, sub-matrix extraction) and never look up a kmer for this layer.
// /// `TypedLayer<D>::open` always pays for the MPHF too (and doesn't expose `data`
// /// once open), so it's the wrong tool for these; this is the other half of
// /// the same `D::open(layer_dir)` call, without the MPHF alongside it.
// ///
// /// Takes `(root, i)`, not a pre-built path: the caller names a layer
// /// *number* within a partition it already knows the root of, the same
// /// vocabulary as [`layer_dir`] and `LayeredMap` — never the `layer_N`
// /// naming convention itself, which stays private to this crate.
// pub(crate) fn open_data<D: LayerData>(root: &Path, i: usize) -> OKIResult<D> {
// D::open(&layer_dir(root, i))
// }
impl LayerData for () {
type Item = ();