OBIKmers/obikmer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 34 Wiki Activity

Centralize partition directory resolution and add caching design spec

Browse Source 
Introduce a design specification outlining performance bottlenecks in layer data access and an agreed-upon implementation direction for caching. Refactor the codebase to centralize index and layer directory path resolution within the partition object, replacing manual string joining and external helper functions with dedicated accessor methods.
This commit is contained in:
Eric Coissac
2026-08-20 15:10:57 +02:00
parent 7eaa8c2016
commit f7ebc7a1ab
19 changed files with 221 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/obikindex/examples/compare_sparse.rs
+5 -8
View File
@@ -5,7 +5,6 @@
use obicompactvec::{PersistentBitMatrix, PersistentSparseBitMatrix};
use obikindex::KmerIndex;
use obilayeredmap::layer_dir;
fn main() -> anyhow::Result<()> {
let sparse_root = std::env::args().nth(1).expect("usage: compare_sparse <sparse_root> <dense_root>");
@@ -26,18 +25,16 @@ fn main() -> anyhow::Result<()> {
let mut first_mismatch = None;
for part in 0..n_parts {
let part_dir_sparse = sparse.partition().part_dir(part);
let part_dir_dense = dense.partition().part_dir(part);
let index_dir_sparse = sparse.partition().index_dir(part);
let index_dir_dense = dense.partition().index_dir(part);
if !part_dir_sparse.join("index").exists() || !part_dir_dense.join("index").exists() {
if !index_dir_sparse.exists() || !index_dir_dense.exists() {
continue;
}
for layer in 0..n_layers {
let index_dir_sparse = part_dir_sparse.join("index");
let index_dir_dense = part_dir_dense.join("index");
let layer_dir_sparse = layer_dir(&index_dir_sparse, layer);
let layer_dir_dense = layer_dir(&index_dir_dense, layer);
let layer_dir_sparse = sparse.partition().layer_dir(part, layer);
let layer_dir_dense = dense.partition().layer_dir(part, layer);
if !layer_dir_sparse.exists() || !layer_dir_dense.exists() {
continue;
src/obikindex/src/index.rs
+6 -7
View File
@@ -149,7 +149,7 @@ impl KmerIndex {
/// is enough, no need to scan every partition.
pub fn n_layers_per_partition(&self) -> OKIResult<usize> {
use obilayeredmap::meta::PartitionMeta;
let index_dir = self.partition.part_dir(0).join("index");
let index_dir = self.partition.index_dir(0);
let meta = PartitionMeta::load(&index_dir)
.map_err(|e| OKIError::Io(std::io::Error::new(std::io::ErrorKind::Other, e.to_string())))?;
Ok(meta.n_layers)
@@ -264,8 +264,7 @@ impl KmerIndex {
/// Path to the unitigs file for partition `part`, layer `layer`.
pub fn layer_unitigs_path(&self, part: usize, layer: usize) -> PathBuf {
let index_dir = self.partition.part_dir(part).join("index");
obilayeredmap::layer_dir(&index_dir, layer).join("unitigs.bin")
self.partition.layer_dir(part, layer).join("unitigs.bin")
}
/// Pack all partition matrices into single-file format (presence → .pbmx, counts → .pcmx).
@@ -288,12 +287,12 @@ impl KmerIndex {
crate::numa::PartitionRunner::new().run(
&order,
|i| -> OKIResult<()> {
let index_dir = self.partition.part_dir(i).join("index");
let index_dir = self.partition.index_dir(i);
if !index_dir.exists() { return Ok(()); }
let meta = PartitionMeta::load(&index_dir)
.map_err(|e| OKIError::Io(std::io::Error::new(std::io::ErrorKind::Other, e.to_string())))?;
for l in 0..meta.n_layers {
let layer_dir = obilayeredmap::layer_dir(&index_dir, l);
let layer_dir = self.partition.layer_dir(i, l);
let presence_dir = layer_dir.join("presence");
let counts_dir = layer_dir.join("counts");
if presence_dir.exists() {
@@ -326,14 +325,14 @@ impl KmerIndex {
let errors: Vec<_> = (0..n)
.into_par_iter()
.filter_map(|i| {
let index_dir = self.partition.part_dir(i).join("index");
let index_dir = self.partition.index_dir(i);
if !index_dir.exists() { return None; }
let meta = match PartitionMeta::load(&index_dir) {
Ok(m) => m,
Err(e) => return Some(OKIError::Io(std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))),
};
for l in 0..meta.n_layers {
let layer_dir = obilayeredmap::layer_dir(&index_dir, l);
let layer_dir = self.partition.layer_dir(i, l);
let meta_path = layer_dir.join(LayerMeta::FILENAME);
if meta_path.exists() { continue; }
let unitigs_path = layer_dir.join("unitigs.bin");
src/obikindex/src/reindex.rs
+1 -1
View File
@@ -48,7 +48,7 @@ impl KmerIndex {
let runner = crate::numa::PartitionRunner::new();
runner.run(
&order,
|i| reindex_partition(&self.partition.part_dir(i).join("index"), &target, block_bits)
|i| reindex_partition(&self.partition.index_dir(i), &target, block_bits)
.map_err(|e| OKIError::InvalidInput(format!("partition {i}: {e}"))),
|_, _, _| { pb.inc(1); },
)?;
src/obikindex/src/stats.rs
+4 -5
View File
@@ -3,7 +3,6 @@ use std::path::Path;
use obicompactvec::{LayerMeta, PersistentBitMatrix, PersistentCompactIntMatrix};
use obicompactvec::traits::ColumnWeights;
use obilayeredmap::layer_dir;
use obilayeredmap::meta::PartitionMeta;
use rayon::prelude::*;
@@ -91,7 +90,7 @@ impl KmerIndex {
let (n_kmers, mphf_b, evidence_b, matrix_b) = (0..n)
.into_par_iter()
.map(|i| {
let index_dir = self.partition.part_dir(i).join("index");
let index_dir = self.partition.index_dir(i);
if !index_dir.exists() { return (0usize, 0u64, 0u64, 0u64); }
let n_layers = PartitionMeta::load(&index_dir)
@@ -99,7 +98,7 @@ impl KmerIndex {
.unwrap_or(0);
(0..n_layers).fold((0usize, 0u64, 0u64, 0u64), |acc, l| {
let lb = layer_bytes(&layer_dir(&index_dir, l));
let lb = layer_bytes(&self.partition.layer_dir(i, l));
(acc.0 + lb.n_kmers, acc.1 + lb.mphf, acc.2 + lb.evidence, acc.3 + lb.matrix)
})
})
@@ -142,7 +141,7 @@ impl KmerIndex {
let mut counts = vec![0u64; n_genomes];
let mut n_kmers = 0usize;
let index_dir = self.partition.part_dir(i).join("index");
let index_dir = self.partition.index_dir(i);
if !index_dir.exists() { return (0, counts); }
let n_layers = PartitionMeta::load(&index_dir)
@@ -150,7 +149,7 @@ impl KmerIndex {
.unwrap_or(0);
for l in 0..n_layers {
let this_layer_dir = obilayeredmap::layer_dir(&index_dir, l);
let this_layer_dir = self.partition.layer_dir(i, l);
if !this_layer_dir.exists() { continue; }
n_kmers += LayerMeta::load(&this_layer_dir).map(|m| m.n).unwrap_or(0);