OBIKmers/obikmer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 34 Wiki Activity

Centralize partition directory resolution and add caching design spec

Browse Source 
Introduce a design specification outlining performance bottlenecks in layer data access and an agreed-upon implementation direction for caching. Refactor the codebase to centralize index and layer directory path resolution within the partition object, replacing manual string joining and external helper functions with dedicated accessor methods.
This commit is contained in:
Eric Coissac
2026-08-20 15:10:57 +02:00
parent 7eaa8c2016
commit f7ebc7a1ab
19 changed files with 221 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/obikphylo/src/siblings/build.rs
+4 -4
View File
@@ -7,7 +7,7 @@ use rayon::prelude::*;
use obikpartitionner::KmerPartition;
use obipipeline::ThrottleGuard;
use obikseq::CanonicalKmer;
use obilayeredmap::{layer_dir, MphfLayer};
use obilayeredmap::MphfLayer;
use obilayeredmap::meta::PartitionMeta;
use obisys::progress_bar;
@@ -16,7 +16,7 @@ use obikindex::KmerIndex;
use super::cache::PartitionCache;
use super::helpers::{central_base, is_minorant};
use super::{olm_to_ok, FamilyMask, SiblingAnnexBuilder, ANNEX_FILE_NAME, INDEX_SUBDIR};
use super::{olm_to_ok, FamilyMask, SiblingAnnexBuilder, ANNEX_FILE_NAME};
// ── obipipeline data types ─────────────────────────────────────────────────
@@ -92,7 +92,7 @@ impl SiblingAnnexBuildExt for KmerIndex {
let pb = progress_bar("sibling_annex", n_parts as u64, "partitions");
let mut total_slots: u64 = 0;
for part in 0..n_parts {
let index_dir = self.partition().part_dir(part).join(INDEX_SUBDIR);
let index_dir = self.partition().index_dir(part);
if !index_dir.exists() {
pb.inc(1);
continue;
@@ -101,7 +101,7 @@ impl SiblingAnnexBuildExt for KmerIndex {
let mut part_slots: u64 = 0;
for l in 0..meta.n_layers {
part_slots += build_layer_sibling_annex(self, &layer_dir(&index_dir, l), n_parts, l, &cache)?;
part_slots += build_layer_sibling_annex(self, &self.partition().layer_dir(part, l), n_parts, l, &cache)?;
}
total_slots += part_slots;
pb.inc(1);
src/obikphylo/src/siblings/cache.rs
+4 -4
View File
@@ -5,14 +5,14 @@ use std::path::Path;
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix, PersistentSparseBitMatrix};
use obikpartitionner::KmerPartition;
use obikseq::CanonicalKmer;
use obilayeredmap::{layer_dir, Layer, OLMResult};
use obilayeredmap::{Layer, OLMResult};
use obilayeredmap::meta::{IndexMode, PartitionMeta};
use obisys::progress_bar;
use obikindex::OKIResult;
use super::iter::SiblingLayerExt;
use super::{olm_to_ok, SiblingAnnex, INDEX_SUBDIR};
use super::{olm_to_ok, SiblingAnnex};
/// Every partition's already-open layers, built **once** for the whole
/// `build_sibling_annex` run and shared (read-only) across every lookup, in
@@ -174,7 +174,7 @@ impl PartitionCache {
let built: Vec<(Vec<Mat>, usize)> = (0..n_parts)
.into_par_iter()
.map(|part| -> OKIResult<(Vec<Mat>, usize)> {
let index_dir = partition.part_dir(part).join(INDEX_SUBDIR);
let index_dir = partition.index_dir(part);
if !index_dir.exists() {
pb.inc(1);
return Ok((Vec::new(), 0));
@@ -182,7 +182,7 @@ impl PartitionCache {
let meta = PartitionMeta::load(&index_dir).map_err(olm_to_ok)?;
let mut mats = Vec::with_capacity(meta.n_layers);
for l in 0..meta.n_layers {
let Ok(mat) = Mat::open(&layer_dir(&index_dir, l), &meta.mode, with_counts) else { continue };
let Ok(mat) = Mat::open(&partition.layer_dir(part, l), &meta.mode, with_counts) else { continue };
mats.push(mat);
}
pb.inc(1);
src/obikphylo/src/siblings/family_scan.rs
+3 -4
View File
@@ -54,7 +54,6 @@ use std::sync::atomic::{AtomicU8, Ordering};
use rayon::prelude::*;
use obikseq::CanonicalKmer;
use obilayeredmap::layer_dir;
use obilayeredmap::meta::PartitionMeta;
use obipipeline::{ThrottleGuard, throttle};
@@ -64,7 +63,7 @@ use obikindex::KmerIndex;
use super::cache::{Mat, PartitionCache};
use super::helpers::central_base;
use super::iter::SiblingEntry;
use super::{olm_to_ok, FamilyMask, SiblingAnnex, ANNEX_FILE_NAME, INDEX_SUBDIR};
use super::{olm_to_ok, FamilyMask, SiblingAnnex, ANNEX_FILE_NAME};
/// Families per batch — see the module docs for the memory-vs-per-partition-
/// density trade-off this picks a point on. At ~90 genomes and a few
@@ -106,13 +105,13 @@ pub(crate) fn sibling_layer_dirs(index: &KmerIndex) -> OKIResult<Vec<PathBuf>> {
let n_parts = index.n_partitions();
let mut layer_dirs = Vec::new();
for part in 0..n_parts {
let index_dir = index.partition().part_dir(part).join(INDEX_SUBDIR);
let index_dir = index.partition().index_dir(part);
if !index_dir.exists() {
continue;
}
let meta = PartitionMeta::load(&index_dir).map_err(olm_to_ok)?;
for l in 0..meta.n_layers {
let this_layer_dir = layer_dir(&index_dir, l);
let this_layer_dir = index.partition().layer_dir(part, l);
let annex_path = this_layer_dir.join(ANNEX_FILE_NAME);
if !annex_path.exists() {
return Err(OKIError::InvalidInput(format!(
src/obikphylo/src/siblings/mod.rs
-1
View File
@@ -75,7 +75,6 @@ use obilayeredmap::OLMError;
use obikindex::OKIError;
pub(super) const INDEX_SUBDIR: &str = "index";
pub(super) const ANNEX_FILE_NAME: &str = "siblings.psib";
pub(super) fn olm_to_ok(e: OLMError) -> OKIError {
src/obikphylo/src/siblings/tests.rs
+7 -8
View File
@@ -3,7 +3,6 @@ use std::path::Path;
use obikseq::{CanonicalKmer, Kmer, Sequence};
use obilayeredmap::MphfLayer;
use obilayeredmap::layer_dir;
use obilayeredmap::meta::PartitionMeta;
use obisys::Reporter;
use tempfile::tempdir;
@@ -20,7 +19,7 @@ use super::helpers::is_minorant;
use super::sankoff_bundle::SankoffBundleExt;
use super::stats::SiblingStatsExt;
use super::subsample::EntropyBias;
use super::{FamilyMask, SiblingAnnex, ANNEX_FILE_NAME, INDEX_SUBDIR};
use super::{FamilyMask, SiblingAnnex, ANNEX_FILE_NAME};
// k must be >= 11 (project constraint, "k ∈ [11,31]"); k=11, level_max=1,
// theta=0.0 mirror `obiskbuilder`'s own tests (smaller k/level_max
@@ -89,10 +88,10 @@ fn canonical(ascii: &[u8]) -> CanonicalKmer {
/// Read back the annex entry for a given canonical k-mer from the merged
/// index's (single) partition/layer, asserting it was found at all.
fn annex_info_for(idx: &KmerIndex, kmer: CanonicalKmer) -> FamilyMask {
let index_dir = idx.partition().part_dir(0).join(INDEX_SUBDIR);
let index_dir = idx.partition().index_dir(0);
let meta = PartitionMeta::load(&index_dir).unwrap();
for l in 0..meta.n_layers {
let layer_dir = layer_dir(&index_dir, l);
let layer_dir = idx.partition().layer_dir(0, l);
let mphf = MphfLayer::open(&layer_dir, &meta.mode).unwrap();
if let Some(slot) = mphf.find(kmer) {
let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME)).unwrap();
@@ -168,7 +167,7 @@ fn sibling_annex_works_after_pack_sparse() {
let merged = merge_two(dir.path(), &g1, &g2);
merged.pack_matrices(true).expect("pack_matrices(sparse)");
let index_dir = merged.partition().part_dir(0).join(INDEX_SUBDIR);
let index_dir = merged.partition().index_dir(0);
assert!(
index_dir.join("layer_0").join("presence").join("is_multi.prsb").exists(),
"pack_matrices(true) must leave the sparse marker file behind"
@@ -311,10 +310,10 @@ fn sibling_annex_no_empty_masks_after_build() {
let g1 = build_single_genome_index(dir.path(), "g1", &seq);
g1.build_sibling_annex().expect("build_sibling_annex");
let index_dir = g1.partition().part_dir(0).join(INDEX_SUBDIR);
let index_dir = g1.partition().index_dir(0);
let meta = PartitionMeta::load(&index_dir).expect("partition meta");
for l in 0..meta.n_layers {
let layer_dir = layer_dir(&index_dir, l);
let layer_dir = g1.partition().layer_dir(0, l);
let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME)).expect("annex open");
for slot in 0..annex.len() {
let mask = annex.get(slot).expect("slot must have an entry");
@@ -362,7 +361,7 @@ fn sibling_annex_records_the_real_layer_of_each_family_member() {
let merged = merge_two(dir.path(), &g1, &g2);
merged.build_sibling_annex().expect("build_sibling_annex");
let index_dir = merged.partition().part_dir(0).join(INDEX_SUBDIR);
let index_dir = merged.partition().index_dir(0);
let meta = PartitionMeta::load(&index_dir).unwrap();
assert_eq!(meta.n_layers, 2, "fixture assumption: one merge, one new layer");