centralize layer path construction and data loading

Replace manual directory formatting and inline matrix opening with the newly introduced `layer_dir` and `open_data` helpers from the `obilayeredmap` crate. This standardizes filesystem access, encapsulates layer naming conventions, and simplifies error handling without altering computational behavior or test suites.
This commit is contained in:
Eric Coissac
2026-08-20 14:43:10 +02:00
parent 640f29725b
commit 5a91817488
5 changed files with 49 additions and 30 deletions
+7 -7
View File
@@ -149,21 +149,21 @@ impl KmerIndex {
.unwrap_or(0); .unwrap_or(0);
for l in 0..n_layers { for l in 0..n_layers {
let layer_dir = index_dir.join(format!("layer_{l}")); let this_layer_dir = obilayeredmap::layer_dir(&index_dir, l);
if !layer_dir.exists() { continue; } if !this_layer_dir.exists() { continue; }
n_kmers += LayerMeta::load(&layer_dir).map(|m| m.n).unwrap_or(0); n_kmers += LayerMeta::load(&this_layer_dir).map(|m| m.n).unwrap_or(0);
let mat: Box<dyn ColumnWeights> = let mat: Box<dyn ColumnWeights> =
if layer_dir.join("counts").exists() if this_layer_dir.join("counts").exists()
&& !layer_dir.join("presence").exists() && !this_layer_dir.join("presence").exists()
{ {
match PersistentCompactIntMatrix::open(&layer_dir) { match obilayeredmap::open_data::<PersistentCompactIntMatrix>(&index_dir, l) {
Ok(m) => Box::new(m), Ok(m) => Box::new(m),
Err(_) => continue, Err(_) => continue,
} }
} else { } else {
match PersistentBitMatrix::open(&layer_dir) { match obilayeredmap::open_data::<PersistentBitMatrix>(&index_dir, l) {
Ok(m) => Box::new(m), Ok(m) => Box::new(m),
Err(_) => continue, Err(_) => continue,
} }
+11 -16
View File
@@ -1,17 +1,12 @@
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix}; use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
use obilayeredmap::LayeredStore; use obilayeredmap::{layer_dir, open_data, LayeredStore};
use obiskio::{SKError, SKResult}; use obiskio::SKResult;
use crate::common::{load_meta, olm_to_sk};
use crate::partition::KmerPartition; use crate::partition::KmerPartition;
const INDEX_SUBDIR: &str = "index"; const INDEX_SUBDIR: &str = "index";
fn probe_n_layers(index_dir: &std::path::Path) -> usize {
let mut n = 0;
while index_dir.join(format!("layer_{n}")).exists() { n += 1; }
n
}
impl KmerPartition { impl KmerPartition {
/// Open all count matrices for partition `part`, one per layer. /// Open all count matrices for partition `part`, one per layer.
/// Layers without a `counts/` directory are skipped. /// Layers without a `counts/` directory are skipped.
@@ -20,11 +15,11 @@ impl KmerPartition {
if !index_dir.exists() { if !index_dir.exists() {
return Ok(LayeredStore::new(vec![])); return Ok(LayeredStore::new(vec![]));
} }
let matrices = (0..probe_n_layers(&index_dir)) let n_layers = load_meta(&index_dir, "distance")?.n_layers;
let matrices = (0..n_layers)
.filter_map(|l| { .filter_map(|l| {
let layer_dir = index_dir.join(format!("layer_{l}")); layer_dir(&index_dir, l).join("counts").exists()
layer_dir.join("counts").exists() .then(|| open_data(&index_dir, l).map_err(|e| olm_to_sk(e, "distance")))
.then(|| PersistentCompactIntMatrix::open(&layer_dir).map_err(SKError::Io))
}) })
.collect::<SKResult<Vec<_>>>()?; .collect::<SKResult<Vec<_>>>()?;
Ok(LayeredStore::new(matrices)) Ok(LayeredStore::new(matrices))
@@ -37,11 +32,11 @@ impl KmerPartition {
if !index_dir.exists() { if !index_dir.exists() {
return Ok(LayeredStore::new(vec![])); return Ok(LayeredStore::new(vec![]));
} }
let matrices = (0..probe_n_layers(&index_dir)) let n_layers = load_meta(&index_dir, "distance")?.n_layers;
let matrices = (0..n_layers)
.filter_map(|l| { .filter_map(|l| {
let layer_dir = index_dir.join(format!("layer_{l}")); layer_dir(&index_dir, l).join("presence").exists()
layer_dir.join("presence").exists() .then(|| open_data(&index_dir, l).map_err(|e| olm_to_sk(e, "distance")))
.then(|| PersistentBitMatrix::open(&layer_dir).map_err(SKError::Io))
}) })
.collect::<SKResult<Vec<_>>>()?; .collect::<SKResult<Vec<_>>>()?;
Ok(LayeredStore::new(matrices)) Ok(LayeredStore::new(matrices))
+29 -1
View File
@@ -1,6 +1,6 @@
use std::collections::HashMap; use std::collections::HashMap;
use std::fs; use std::fs;
use std::path::Path; use std::path::{Path, PathBuf};
use obicompactvec::{ use obicompactvec::{
BinaryMatrix, BinaryMatrix,
@@ -27,6 +27,34 @@ pub trait LayerData: Sized {
fn read(&self, slot: usize) -> Self::Item; fn read(&self, slot: usize) -> Self::Item;
} }
/// Layer directory path for layer `i` within a partition's index root — the
/// single source of truth for the on-disk `layer_N` naming convention
/// (mirrors what `LayeredMap::open`/`push_layer` already use internally).
///
/// `obilayeredmap` operates within a single partition's index root; it has
/// no notion of "partition" at all. Turning a partition number into that
/// root is `obikpartitionner::KmerPartition::part_dir`'s job, one layer up
/// — callers here only ever name a layer *number*, never build the path
/// themselves.
pub fn layer_dir(root: &Path, i: usize) -> PathBuf {
root.join(format!("layer_{i}"))
}
/// Opens layer `i`'s data only, skipping the MPHF — for callers that only
/// need matrix-level operations (distance traits, column weights, group
/// filters, sub-matrix extraction) and never look up a kmer for this layer.
/// `Layer<D>::open` always pays for the MPHF too (and doesn't expose `data`
/// once open), so it's the wrong tool for these; this is the other half of
/// the same `D::open(layer_dir)` call, without the MPHF alongside it.
///
/// Takes `(root, i)`, not a pre-built path: the caller names a layer
/// *number* within a partition it already knows the root of, the same
/// vocabulary as [`layer_dir`] and `LayeredMap` — never the `layer_N`
/// naming convention itself, which stays private to this crate.
pub fn open_data<D: LayerData>(root: &Path, i: usize) -> OLMResult<D> {
D::open(&layer_dir(root, i))
}
impl LayerData for () { impl LayerData for () {
type Item = (); type Item = ();
fn open(_layer_dir: &Path) -> OLMResult<Self> { Ok(()) } fn open(_layer_dir: &Path) -> OLMResult<Self> { Ok(()) }
+1 -1
View File
@@ -8,7 +8,7 @@ pub mod meta;
pub(crate) mod mphf_layer; pub(crate) mod mphf_layer;
pub use error::{OLMError, OLMResult}; pub use error::{OLMError, OLMResult};
pub use layer::{Hit, Layer, LayerData}; pub use layer::{layer_dir, open_data, Hit, Layer, LayerData};
pub use layered_store::LayeredStore; pub use layered_store::LayeredStore;
pub use map::LayeredMap; pub use map::LayeredMap;
pub use meta::{IndexMode, PartitionMeta}; pub use meta::{IndexMode, PartitionMeta};
+1 -5
View File
@@ -7,7 +7,7 @@ use obikseq::CanonicalKmer;
use obiskio::{UnitigFileWriter, DEFAULT_BLOCK_BITS}; use obiskio::{UnitigFileWriter, DEFAULT_BLOCK_BITS};
use crate::error::OLMResult; use crate::error::OLMResult;
use crate::layer::{Hit, Layer, LayerData}; use crate::layer::{layer_dir, Hit, Layer, LayerData};
use crate::meta::{IndexMode, PartitionMeta}; use crate::meta::{IndexMode, PartitionMeta};
/// Layered kmer index for a single partition. /// Layered kmer index for a single partition.
@@ -97,10 +97,6 @@ impl LayeredMap<PersistentCompactIntMatrix> {
} }
} }
fn layer_dir(root: &Path, i: usize) -> PathBuf {
root.join(format!("layer_{i}"))
}
#[cfg(test)] #[cfg(test)]
#[path = "tests/map.rs"] #[path = "tests/map.rs"]
mod tests; mod tests;