centralize layer path construction and data loading
Replace manual directory formatting and inline matrix opening with the newly introduced `layer_dir` and `open_data` helpers from the `obilayeredmap` crate. This standardizes filesystem access, encapsulates layer naming conventions, and simplifies error handling without altering computational behavior or test suites.
This commit is contained in:
@@ -149,21 +149,21 @@ impl KmerIndex {
|
|||||||
.unwrap_or(0);
|
.unwrap_or(0);
|
||||||
|
|
||||||
for l in 0..n_layers {
|
for l in 0..n_layers {
|
||||||
let layer_dir = index_dir.join(format!("layer_{l}"));
|
let this_layer_dir = obilayeredmap::layer_dir(&index_dir, l);
|
||||||
if !layer_dir.exists() { continue; }
|
if !this_layer_dir.exists() { continue; }
|
||||||
|
|
||||||
n_kmers += LayerMeta::load(&layer_dir).map(|m| m.n).unwrap_or(0);
|
n_kmers += LayerMeta::load(&this_layer_dir).map(|m| m.n).unwrap_or(0);
|
||||||
|
|
||||||
let mat: Box<dyn ColumnWeights> =
|
let mat: Box<dyn ColumnWeights> =
|
||||||
if layer_dir.join("counts").exists()
|
if this_layer_dir.join("counts").exists()
|
||||||
&& !layer_dir.join("presence").exists()
|
&& !this_layer_dir.join("presence").exists()
|
||||||
{
|
{
|
||||||
match PersistentCompactIntMatrix::open(&layer_dir) {
|
match obilayeredmap::open_data::<PersistentCompactIntMatrix>(&index_dir, l) {
|
||||||
Ok(m) => Box::new(m),
|
Ok(m) => Box::new(m),
|
||||||
Err(_) => continue,
|
Err(_) => continue,
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
match PersistentBitMatrix::open(&layer_dir) {
|
match obilayeredmap::open_data::<PersistentBitMatrix>(&index_dir, l) {
|
||||||
Ok(m) => Box::new(m),
|
Ok(m) => Box::new(m),
|
||||||
Err(_) => continue,
|
Err(_) => continue,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,17 +1,12 @@
|
|||||||
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
|
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
|
||||||
use obilayeredmap::LayeredStore;
|
use obilayeredmap::{layer_dir, open_data, LayeredStore};
|
||||||
use obiskio::{SKError, SKResult};
|
use obiskio::SKResult;
|
||||||
|
|
||||||
|
use crate::common::{load_meta, olm_to_sk};
|
||||||
use crate::partition::KmerPartition;
|
use crate::partition::KmerPartition;
|
||||||
|
|
||||||
const INDEX_SUBDIR: &str = "index";
|
const INDEX_SUBDIR: &str = "index";
|
||||||
|
|
||||||
fn probe_n_layers(index_dir: &std::path::Path) -> usize {
|
|
||||||
let mut n = 0;
|
|
||||||
while index_dir.join(format!("layer_{n}")).exists() { n += 1; }
|
|
||||||
n
|
|
||||||
}
|
|
||||||
|
|
||||||
impl KmerPartition {
|
impl KmerPartition {
|
||||||
/// Open all count matrices for partition `part`, one per layer.
|
/// Open all count matrices for partition `part`, one per layer.
|
||||||
/// Layers without a `counts/` directory are skipped.
|
/// Layers without a `counts/` directory are skipped.
|
||||||
@@ -20,11 +15,11 @@ impl KmerPartition {
|
|||||||
if !index_dir.exists() {
|
if !index_dir.exists() {
|
||||||
return Ok(LayeredStore::new(vec![]));
|
return Ok(LayeredStore::new(vec![]));
|
||||||
}
|
}
|
||||||
let matrices = (0..probe_n_layers(&index_dir))
|
let n_layers = load_meta(&index_dir, "distance")?.n_layers;
|
||||||
|
let matrices = (0..n_layers)
|
||||||
.filter_map(|l| {
|
.filter_map(|l| {
|
||||||
let layer_dir = index_dir.join(format!("layer_{l}"));
|
layer_dir(&index_dir, l).join("counts").exists()
|
||||||
layer_dir.join("counts").exists()
|
.then(|| open_data(&index_dir, l).map_err(|e| olm_to_sk(e, "distance")))
|
||||||
.then(|| PersistentCompactIntMatrix::open(&layer_dir).map_err(SKError::Io))
|
|
||||||
})
|
})
|
||||||
.collect::<SKResult<Vec<_>>>()?;
|
.collect::<SKResult<Vec<_>>>()?;
|
||||||
Ok(LayeredStore::new(matrices))
|
Ok(LayeredStore::new(matrices))
|
||||||
@@ -37,11 +32,11 @@ impl KmerPartition {
|
|||||||
if !index_dir.exists() {
|
if !index_dir.exists() {
|
||||||
return Ok(LayeredStore::new(vec![]));
|
return Ok(LayeredStore::new(vec![]));
|
||||||
}
|
}
|
||||||
let matrices = (0..probe_n_layers(&index_dir))
|
let n_layers = load_meta(&index_dir, "distance")?.n_layers;
|
||||||
|
let matrices = (0..n_layers)
|
||||||
.filter_map(|l| {
|
.filter_map(|l| {
|
||||||
let layer_dir = index_dir.join(format!("layer_{l}"));
|
layer_dir(&index_dir, l).join("presence").exists()
|
||||||
layer_dir.join("presence").exists()
|
.then(|| open_data(&index_dir, l).map_err(|e| olm_to_sk(e, "distance")))
|
||||||
.then(|| PersistentBitMatrix::open(&layer_dir).map_err(SKError::Io))
|
|
||||||
})
|
})
|
||||||
.collect::<SKResult<Vec<_>>>()?;
|
.collect::<SKResult<Vec<_>>>()?;
|
||||||
Ok(LayeredStore::new(matrices))
|
Ok(LayeredStore::new(matrices))
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::fs;
|
use std::fs;
|
||||||
use std::path::Path;
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
use obicompactvec::{
|
use obicompactvec::{
|
||||||
BinaryMatrix,
|
BinaryMatrix,
|
||||||
@@ -27,6 +27,34 @@ pub trait LayerData: Sized {
|
|||||||
fn read(&self, slot: usize) -> Self::Item;
|
fn read(&self, slot: usize) -> Self::Item;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Layer directory path for layer `i` within a partition's index root — the
|
||||||
|
/// single source of truth for the on-disk `layer_N` naming convention
|
||||||
|
/// (mirrors what `LayeredMap::open`/`push_layer` already use internally).
|
||||||
|
///
|
||||||
|
/// `obilayeredmap` operates within a single partition's index root; it has
|
||||||
|
/// no notion of "partition" at all. Turning a partition number into that
|
||||||
|
/// root is `obikpartitionner::KmerPartition::part_dir`'s job, one layer up
|
||||||
|
/// — callers here only ever name a layer *number*, never build the path
|
||||||
|
/// themselves.
|
||||||
|
pub fn layer_dir(root: &Path, i: usize) -> PathBuf {
|
||||||
|
root.join(format!("layer_{i}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Opens layer `i`'s data only, skipping the MPHF — for callers that only
|
||||||
|
/// need matrix-level operations (distance traits, column weights, group
|
||||||
|
/// filters, sub-matrix extraction) and never look up a kmer for this layer.
|
||||||
|
/// `Layer<D>::open` always pays for the MPHF too (and doesn't expose `data`
|
||||||
|
/// once open), so it's the wrong tool for these; this is the other half of
|
||||||
|
/// the same `D::open(layer_dir)` call, without the MPHF alongside it.
|
||||||
|
///
|
||||||
|
/// Takes `(root, i)`, not a pre-built path: the caller names a layer
|
||||||
|
/// *number* within a partition it already knows the root of, the same
|
||||||
|
/// vocabulary as [`layer_dir`] and `LayeredMap` — never the `layer_N`
|
||||||
|
/// naming convention itself, which stays private to this crate.
|
||||||
|
pub fn open_data<D: LayerData>(root: &Path, i: usize) -> OLMResult<D> {
|
||||||
|
D::open(&layer_dir(root, i))
|
||||||
|
}
|
||||||
|
|
||||||
impl LayerData for () {
|
impl LayerData for () {
|
||||||
type Item = ();
|
type Item = ();
|
||||||
fn open(_layer_dir: &Path) -> OLMResult<Self> { Ok(()) }
|
fn open(_layer_dir: &Path) -> OLMResult<Self> { Ok(()) }
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ pub mod meta;
|
|||||||
pub(crate) mod mphf_layer;
|
pub(crate) mod mphf_layer;
|
||||||
|
|
||||||
pub use error::{OLMError, OLMResult};
|
pub use error::{OLMError, OLMResult};
|
||||||
pub use layer::{Hit, Layer, LayerData};
|
pub use layer::{layer_dir, open_data, Hit, Layer, LayerData};
|
||||||
pub use layered_store::LayeredStore;
|
pub use layered_store::LayeredStore;
|
||||||
pub use map::LayeredMap;
|
pub use map::LayeredMap;
|
||||||
pub use meta::{IndexMode, PartitionMeta};
|
pub use meta::{IndexMode, PartitionMeta};
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ use obikseq::CanonicalKmer;
|
|||||||
use obiskio::{UnitigFileWriter, DEFAULT_BLOCK_BITS};
|
use obiskio::{UnitigFileWriter, DEFAULT_BLOCK_BITS};
|
||||||
|
|
||||||
use crate::error::OLMResult;
|
use crate::error::OLMResult;
|
||||||
use crate::layer::{Hit, Layer, LayerData};
|
use crate::layer::{layer_dir, Hit, Layer, LayerData};
|
||||||
use crate::meta::{IndexMode, PartitionMeta};
|
use crate::meta::{IndexMode, PartitionMeta};
|
||||||
|
|
||||||
/// Layered kmer index for a single partition.
|
/// Layered kmer index for a single partition.
|
||||||
@@ -97,10 +97,6 @@ impl LayeredMap<PersistentCompactIntMatrix> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn layer_dir(root: &Path, i: usize) -> PathBuf {
|
|
||||||
root.join(format!("layer_{i}"))
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
#[path = "tests/map.rs"]
|
#[path = "tests/map.rs"]
|
||||||
mod tests;
|
mod tests;
|
||||||
|
|||||||
Reference in New Issue
Block a user