Centralize partition directory resolution and add caching design spec

Introduce a design specification outlining performance bottlenecks in layer data access and an agreed-upon implementation direction for caching. Refactor the codebase to centralize index and layer directory path resolution within the partition object, replacing manual string joining and external helper functions with dedicated accessor methods.
This commit is contained in:
Eric Coissac
2026-08-20 15:10:57 +02:00
parent 7eaa8c2016
commit f7ebc7a1ab
19 changed files with 221 additions and 94 deletions
+3 -4
View File
@@ -54,7 +54,6 @@ use std::sync::atomic::{AtomicU8, Ordering};
use rayon::prelude::*;
use obikseq::CanonicalKmer;
use obilayeredmap::layer_dir;
use obilayeredmap::meta::PartitionMeta;
use obipipeline::{ThrottleGuard, throttle};
@@ -64,7 +63,7 @@ use obikindex::KmerIndex;
use super::cache::{Mat, PartitionCache};
use super::helpers::central_base;
use super::iter::SiblingEntry;
use super::{olm_to_ok, FamilyMask, SiblingAnnex, ANNEX_FILE_NAME, INDEX_SUBDIR};
use super::{olm_to_ok, FamilyMask, SiblingAnnex, ANNEX_FILE_NAME};
/// Families per batch — see the module docs for the memory-vs-per-partition-
/// density trade-off this picks a point on. At ~90 genomes and a few
@@ -106,13 +105,13 @@ pub(crate) fn sibling_layer_dirs(index: &KmerIndex) -> OKIResult<Vec<PathBuf>> {
let n_parts = index.n_partitions();
let mut layer_dirs = Vec::new();
for part in 0..n_parts {
let index_dir = index.partition().part_dir(part).join(INDEX_SUBDIR);
let index_dir = index.partition().index_dir(part);
if !index_dir.exists() {
continue;
}
let meta = PartitionMeta::load(&index_dir).map_err(olm_to_ok)?;
for l in 0..meta.n_layers {
let this_layer_dir = layer_dir(&index_dir, l);
let this_layer_dir = index.partition().layer_dir(part, l);
let annex_path = this_layer_dir.join(ANNEX_FILE_NAME);
if !annex_path.exists() {
return Err(OKIError::InvalidInput(format!(