Centralize partition metadata access and add layer introspection APIs

Replaced scattered direct metadata loading with centralized instance methods on `KmerPartition` to guarantee consistent error mapping and legacy recovery. Introduced `StorageKind`, `LayerContent`, and `EvidenceKind` enums alongside lightweight disk-probe methods that inspect file presence without opening heavy data structures. Updated callers across the index, partitioner, and phylo modules to use the new partition API, and added unit tests validating the introspection behavior.
This commit is contained in:
Eric Coissac
2026-08-20 15:29:53 +02:00
parent f7ebc7a1ab
commit 76cbd3a886
23 changed files with 580 additions and 74 deletions
+7 -6
View File
@@ -8,7 +8,7 @@ use obikpartitionner::KmerPartition;
use obipipeline::ThrottleGuard;
use obikseq::CanonicalKmer;
use obilayeredmap::MphfLayer;
use obilayeredmap::meta::PartitionMeta;
use obilayeredmap::meta::IndexMode;
use obisys::progress_bar;
use obikindex::{OKIError, OKIResult};
@@ -97,11 +97,13 @@ impl SiblingAnnexBuildExt for KmerIndex {
pb.inc(1);
continue;
}
let meta = PartitionMeta::load(&index_dir).map_err(olm_to_ok)?;
let meta = self.partition().partition_meta(part)?;
let mut part_slots: u64 = 0;
for l in 0..meta.n_layers {
part_slots += build_layer_sibling_annex(self, &self.partition().layer_dir(part, l), n_parts, l, &cache)?;
part_slots += build_layer_sibling_annex(
self, &self.partition().layer_dir(part, l), &meta.mode, n_parts, l, &cache,
)?;
}
total_slots += part_slots;
pb.inc(1);
@@ -120,13 +122,12 @@ impl SiblingAnnexBuildExt for KmerIndex {
fn build_layer_sibling_annex(
index: &KmerIndex,
layer_dir: &Path,
mode: &IndexMode,
n_parts: usize,
l: usize,
cache: &Arc<PartitionCache>,
) -> OKIResult<u64> {
let index_dir = layer_dir.parent().expect("layer_dir has a parent index dir");
let meta = PartitionMeta::load(index_dir).map_err(olm_to_ok)?;
let mphf = MphfLayer::open(layer_dir, &meta.mode).map_err(olm_to_ok)?;
let mphf = MphfLayer::open(layer_dir, mode).map_err(olm_to_ok)?;
let k = index.kmer_size();
let n = mphf.n();
+3 -3
View File
@@ -6,13 +6,13 @@ use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix, PersistentS
use obikpartitionner::KmerPartition;
use obikseq::CanonicalKmer;
use obilayeredmap::{Layer, OLMResult};
use obilayeredmap::meta::{IndexMode, PartitionMeta};
use obilayeredmap::meta::IndexMode;
use obisys::progress_bar;
use obikindex::OKIResult;
use super::iter::SiblingLayerExt;
use super::{olm_to_ok, SiblingAnnex};
use super::SiblingAnnex;
/// Every partition's already-open layers, built **once** for the whole
/// `build_sibling_annex` run and shared (read-only) across every lookup, in
@@ -179,7 +179,7 @@ impl PartitionCache {
pb.inc(1);
return Ok((Vec::new(), 0));
}
let meta = PartitionMeta::load(&index_dir).map_err(olm_to_ok)?;
let meta = partition.partition_meta(part)?;
let mut mats = Vec::with_capacity(meta.n_layers);
for l in 0..meta.n_layers {
let Ok(mat) = Mat::open(&partition.layer_dir(part, l), &meta.mode, with_counts) else { continue };
+2 -2
View File
@@ -109,8 +109,8 @@ pub(crate) fn sibling_layer_dirs(index: &KmerIndex) -> OKIResult<Vec<PathBuf>> {
if !index_dir.exists() {
continue;
}
let meta = PartitionMeta::load(&index_dir).map_err(olm_to_ok)?;
for l in 0..meta.n_layers {
let n_layers = index.partition().n_layers(part)?;
for l in 0..n_layers {
let this_layer_dir = index.partition().layer_dir(part, l);
let annex_path = this_layer_dir.join(ANNEX_FILE_NAME);
if !annex_path.exists() {
+5 -9
View File
@@ -3,7 +3,6 @@ use std::path::Path;
use obikseq::{CanonicalKmer, Kmer, Sequence};
use obilayeredmap::MphfLayer;
use obilayeredmap::meta::PartitionMeta;
use obisys::Reporter;
use tempfile::tempdir;
@@ -88,8 +87,7 @@ fn canonical(ascii: &[u8]) -> CanonicalKmer {
/// Read back the annex entry for a given canonical k-mer from the merged
/// index's (single) partition/layer, asserting it was found at all.
fn annex_info_for(idx: &KmerIndex, kmer: CanonicalKmer) -> FamilyMask {
let index_dir = idx.partition().index_dir(0);
let meta = PartitionMeta::load(&index_dir).unwrap();
let meta = idx.partition().partition_meta(0).unwrap();
for l in 0..meta.n_layers {
let layer_dir = idx.partition().layer_dir(0, l);
let mphf = MphfLayer::open(&layer_dir, &meta.mode).unwrap();
@@ -310,9 +308,8 @@ fn sibling_annex_no_empty_masks_after_build() {
let g1 = build_single_genome_index(dir.path(), "g1", &seq);
g1.build_sibling_annex().expect("build_sibling_annex");
let index_dir = g1.partition().index_dir(0);
let meta = PartitionMeta::load(&index_dir).expect("partition meta");
for l in 0..meta.n_layers {
let n_layers = g1.partition().n_layers(0).expect("partition meta");
for l in 0..n_layers {
let layer_dir = g1.partition().layer_dir(0, l);
let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME)).expect("annex open");
for slot in 0..annex.len() {
@@ -361,9 +358,8 @@ fn sibling_annex_records_the_real_layer_of_each_family_member() {
let merged = merge_two(dir.path(), &g1, &g2);
merged.build_sibling_annex().expect("build_sibling_annex");
let index_dir = merged.partition().index_dir(0);
let meta = PartitionMeta::load(&index_dir).unwrap();
assert_eq!(meta.n_layers, 2, "fixture assumption: one merge, one new layer");
let n_layers = merged.partition().n_layers(0).unwrap();
assert_eq!(n_layers, 2, "fixture assumption: one merge, one new layer");
let g1_kmer = canonical(b"AACCGCTTAAG"); // own base C (1), sibling base G (2) — lives in layer 1
let g2_kmer = canonical(b"AACCGGTTAAG"); // own base G (2), sibling base C (1) — lives in layer 0