Centralize partition directory resolution and add caching design spec

Introduce a design specification outlining performance bottlenecks in layer data access and an agreed-upon implementation direction for caching. Refactor the codebase to centralize index and layer directory path resolution within the partition object, replacing manual string joining and external helper functions with dedicated accessor methods.
This commit is contained in:
Eric Coissac
2026-08-20 15:10:57 +02:00
parent 7eaa8c2016
commit f7ebc7a1ab
19 changed files with 221 additions and 94 deletions
+4 -4
View File
@@ -7,7 +7,7 @@ use rayon::prelude::*;
use obikpartitionner::KmerPartition;
use obipipeline::ThrottleGuard;
use obikseq::CanonicalKmer;
use obilayeredmap::{layer_dir, MphfLayer};
use obilayeredmap::MphfLayer;
use obilayeredmap::meta::PartitionMeta;
use obisys::progress_bar;
@@ -16,7 +16,7 @@ use obikindex::KmerIndex;
use super::cache::PartitionCache;
use super::helpers::{central_base, is_minorant};
use super::{olm_to_ok, FamilyMask, SiblingAnnexBuilder, ANNEX_FILE_NAME, INDEX_SUBDIR};
use super::{olm_to_ok, FamilyMask, SiblingAnnexBuilder, ANNEX_FILE_NAME};
// ── obipipeline data types ─────────────────────────────────────────────────
@@ -92,7 +92,7 @@ impl SiblingAnnexBuildExt for KmerIndex {
let pb = progress_bar("sibling_annex", n_parts as u64, "partitions");
let mut total_slots: u64 = 0;
for part in 0..n_parts {
let index_dir = self.partition().part_dir(part).join(INDEX_SUBDIR);
let index_dir = self.partition().index_dir(part);
if !index_dir.exists() {
pb.inc(1);
continue;
@@ -101,7 +101,7 @@ impl SiblingAnnexBuildExt for KmerIndex {
let mut part_slots: u64 = 0;
for l in 0..meta.n_layers {
part_slots += build_layer_sibling_annex(self, &layer_dir(&index_dir, l), n_parts, l, &cache)?;
part_slots += build_layer_sibling_annex(self, &self.partition().layer_dir(part, l), n_parts, l, &cache)?;
}
total_slots += part_slots;
pb.inc(1);