Centralize partition metadata access and add layer introspection APIs

Replaced scattered direct metadata loading with centralized instance methods on `KmerPartition` to guarantee consistent error mapping and legacy recovery. Introduced `StorageKind`, `LayerContent`, and `EvidenceKind` enums alongside lightweight disk-probe methods that inspect file presence without opening heavy data structures. Updated callers across the index, partitioner, and phylo modules to use the new partition API, and added unit tests validating the introspection behavior.
This commit is contained in:
Eric Coissac
2026-08-20 15:29:53 +02:00
parent f7ebc7a1ab
commit 76cbd3a886
23 changed files with 580 additions and 74 deletions
@@ -6,6 +6,7 @@ use ndarray::{Array1, Array2};
use crate::bitvec::{PersistentBitVec, PersistentBitVecBuilder};
use crate::layer_meta::LayerMeta;
use crate::meta::MatrixMeta;
use crate::storage_kind::StorageKind;
use crate::traits::{BitPartials, ColumnWeights};
use crate::views::{nonzero_triples, BitSliceView};
@@ -63,6 +64,42 @@ impl PersistentBitMatrix {
Ok(Self::Implicit { n_rows: meta.n, n_cols: 1 })
}
/// This already-open matrix's storage format — reads the discriminant
/// already in memory, no disk access.
#[inline]
pub fn storage_kind(&self) -> StorageKind {
match self {
Self::Columnar(_) => StorageKind::Columnar,
Self::Packed(_) => StorageKind::Packed,
Self::Sparse(_) => StorageKind::Sparse,
Self::Implicit { .. } => StorageKind::Implicit,
}
}
/// Lightweight disk probe: which format `open` would pick, without
/// opening (mmap'ing) anything. Mirrors `open`'s own priority order by
/// hand — keep the two in sync if that order ever changes.
pub fn detect_storage(layer_dir: &Path) -> io::Result<StorageKind> {
let presence_dir = layer_dir.join("presence");
if presence_dir.join("matrix.pbmx").exists() {
return Ok(StorageKind::Packed);
}
if MatrixMeta::load(&presence_dir).is_ok() {
return Ok(StorageKind::Columnar);
}
if presence_dir.join("sparse_meta.json").exists() {
return Ok(StorageKind::Sparse);
}
if LayerMeta::load(layer_dir).is_ok() {
return Ok(StorageKind::Implicit);
}
Err(io::Error::new(
io::ErrorKind::NotFound,
format!("cannot detect presence storage format in {}", layer_dir.display()),
))
}
#[inline]
pub fn n(&self) -> usize {
match self {
+28
View File
@@ -11,6 +11,7 @@ use crate::colgroup::{chunked_presence_count, ColGroup, MatrixGroupOps};
use crate::format::{HEADER_SIZE, OVERFLOW_ENTRY_SIZE};
use crate::meta::MatrixMeta;
use crate::reader::PersistentCompactIntVec;
use crate::storage_kind::StorageKind;
use crate::tempbitvec::{TempBitVec, TempBitVecBuilder};
use crate::tempintvec::{TempCompactIntVec, TempCompactIntVecBuilder};
use crate::views::{nonzero_triples, IntSliceView};
@@ -287,6 +288,33 @@ impl PersistentCompactIntMatrix {
))
}
/// This already-open matrix's storage format — reads the discriminant
/// already in memory, no disk access.
#[inline]
pub fn storage_kind(&self) -> StorageKind {
match self {
Self::Columnar(_) => StorageKind::Columnar,
Self::Packed(_) => StorageKind::Packed,
}
}
/// Lightweight disk probe: which format `open` would pick, without
/// opening (mmap'ing) anything. Mirrors `open`'s own priority order by
/// hand — keep the two in sync if that order ever changes.
pub fn detect_storage(layer_dir: &Path) -> io::Result<StorageKind> {
let counts_dir = layer_dir.join("counts");
if counts_dir.join("matrix.pcmx").exists() {
return Ok(StorageKind::Packed);
}
if MatrixMeta::load(&counts_dir).is_ok() {
return Ok(StorageKind::Columnar);
}
Err(io::Error::new(
io::ErrorKind::NotFound,
format!("no count matrix found in {}", layer_dir.display()),
))
}
#[inline]
pub fn n(&self) -> usize {
match self { Self::Columnar(m) => m.n(), Self::Packed(m) => m.n_rows }
+2
View File
@@ -10,6 +10,7 @@ mod intmatrix;
mod layer_meta;
mod meta;
mod reader;
mod storage_kind;
mod tempbitvec;
mod tempintvec;
mod views;
@@ -25,6 +26,7 @@ pub use colgroup::{ColGroup, FilterMask, MatrixGroupOps, eval_filter_mask};
pub use intmatrix::{PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder, pack_compact_int_matrix};
pub use layer_meta::LayerMeta;
pub use reader::{PersistentCompactIntVec, Iter as CompactIntVecIter};
pub use storage_kind::StorageKind;
pub use tempbitvec::{TempBitVec, TempBitVecBuilder};
pub use tempintvec::{TempCompactIntVec, TempCompactIntVecBuilder};
pub use traits::{BinaryMatrix, BitPartials, ColumnWeights, CountPartials};
+19
View File
@@ -0,0 +1,19 @@
//! How a persistent matrix's data is physically laid out on disk —
//! orthogonal to *what* it stores (count vs. presence, `obilayeredmap`'s
//! `LayerContent` concern, one crate up). `PersistentCompactIntMatrix` only
//! ever reports `Columnar`/`Packed`; `PersistentBitMatrix` is the only type
//! that can also report `Sparse`/`Implicit`.
/// On-disk storage format of a persistent matrix.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum StorageKind {
/// Per-column files (`.pbiv`/`.pciv`), the build-time format.
Columnar,
/// Single mmap'd file (`matrix.pbmx`/`matrix.pcmx`), query-optimised.
Packed,
/// Row-major deduplicated sparse format (`PersistentBitMatrix` only).
Sparse,
/// No file at all — mono-genome, all values present
/// (`PersistentBitMatrix` only).
Implicit,
}
+6 -14
View File
@@ -3,7 +3,6 @@ use std::fs;
use std::path::{Path, PathBuf};
use obikpartitionner::{KmerPartition, KmerSpectrum, PARTITIONS_SUBDIR};
use obilayeredmap;
use obisys::{Reporter, Stage, progress_bar};
use rayon::prelude::*;
use tracing::info;
@@ -148,11 +147,7 @@ impl KmerIndex {
/// homogeneous across all partitions — reading it off partition 0
/// is enough, no need to scan every partition.
pub fn n_layers_per_partition(&self) -> OKIResult<usize> {
use obilayeredmap::meta::PartitionMeta;
let index_dir = self.partition.index_dir(0);
let meta = PartitionMeta::load(&index_dir)
.map_err(|e| OKIError::Io(std::io::Error::new(std::io::ErrorKind::Other, e.to_string())))?;
Ok(meta.n_layers)
Ok(self.partition.n_layers(0)?)
}
/// Expose the inner partition so the caller can run scatter into it.
@@ -279,7 +274,6 @@ impl KmerIndex {
/// (see the sparse-matrix design plan's "Explicitly deferred").
pub fn pack_matrices(&self, sparse: bool) -> OKIResult<()> {
use obicompactvec::{pack_bit_matrix, pack_compact_int_matrix, pack_sparse_bit_matrix};
use obilayeredmap::meta::PartitionMeta;
let n = self.n_partitions();
let order: Vec<usize> = (0..n).collect();
@@ -289,9 +283,8 @@ impl KmerIndex {
|i| -> OKIResult<()> {
let index_dir = self.partition.index_dir(i);
if !index_dir.exists() { return Ok(()); }
let meta = PartitionMeta::load(&index_dir)
.map_err(|e| OKIError::Io(std::io::Error::new(std::io::ErrorKind::Other, e.to_string())))?;
for l in 0..meta.n_layers {
let n_layers = self.partition.n_layers(i)?;
for l in 0..n_layers {
let layer_dir = self.partition.layer_dir(i, l);
let presence_dir = layer_dir.join("presence");
let counts_dir = layer_dir.join("counts");
@@ -319,7 +312,6 @@ impl KmerIndex {
pub fn upgrade_layer_meta(&self) -> OKIResult<()> {
use obicompactvec::LayerMeta;
use obiskio::UnitigFileReader;
use obilayeredmap::meta::PartitionMeta;
let n = self.n_partitions();
let errors: Vec<_> = (0..n)
@@ -327,11 +319,11 @@ impl KmerIndex {
.filter_map(|i| {
let index_dir = self.partition.index_dir(i);
if !index_dir.exists() { return None; }
let meta = match PartitionMeta::load(&index_dir) {
Ok(m) => m,
let n_layers = match self.partition.n_layers(i) {
Ok(n) => n,
Err(e) => return Some(OKIError::Io(std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))),
};
for l in 0..meta.n_layers {
for l in 0..n_layers {
let layer_dir = self.partition.layer_dir(i, l);
let meta_path = layer_dir.join(LayerMeta::FILENAME);
if meta_path.exists() { continue; }
+8 -8
View File
@@ -1,7 +1,7 @@
use std::fs;
use std::path::Path;
use obilayeredmap::{layer_dir, IndexMode, layer::Layer};
use obilayeredmap::meta::PartitionMeta;
use obikpartitionner::KmerPartition;
use obilayeredmap::{IndexMode, layer::Layer};
use obisys::{Reporter, Stage, progress_bar};
use tracing::info;
@@ -48,7 +48,7 @@ impl KmerIndex {
let runner = crate::numa::PartitionRunner::new();
runner.run(
&order,
|i| reindex_partition(&self.partition.index_dir(i), &target, block_bits)
|i| reindex_partition(&self.partition, i, &target, block_bits)
.map_err(|e| OKIError::InvalidInput(format!("partition {i}: {e}"))),
|_, _, _| { pb.inc(1); },
)?;
@@ -66,13 +66,13 @@ impl KmerIndex {
}
/// Process all layers of one partition's index directory.
fn reindex_partition(index_dir: &Path, target: &IndexMode, block_bits: u8) -> OKIResult<()> {
if !index_dir.exists() {
fn reindex_partition(partition: &KmerPartition, i: usize, target: &IndexMode, block_bits: u8) -> OKIResult<()> {
if !partition.index_dir(i).exists() {
return Ok(());
}
let pm = PartitionMeta::load(index_dir).map_err(olm_to_oki)?;
for layer_idx in 0..pm.n_layers {
reindex_layer(&layer_dir(index_dir, layer_idx), target, block_bits)?;
let n_layers = partition.n_layers(i).map_err(|e| OKIError::InvalidInput(e.to_string()))?;
for layer_idx in 0..n_layers {
reindex_layer(&partition.layer_dir(i, layer_idx), target, block_bits)?;
}
Ok(())
}
+2 -7
View File
@@ -3,7 +3,6 @@ use std::path::Path;
use obicompactvec::{LayerMeta, PersistentBitMatrix, PersistentCompactIntMatrix};
use obicompactvec::traits::ColumnWeights;
use obilayeredmap::meta::PartitionMeta;
use rayon::prelude::*;
use crate::error::OKIResult;
@@ -93,9 +92,7 @@ impl KmerIndex {
let index_dir = self.partition.index_dir(i);
if !index_dir.exists() { return (0usize, 0u64, 0u64, 0u64); }
let n_layers = PartitionMeta::load(&index_dir)
.map(|m| m.n_layers)
.unwrap_or(0);
let n_layers = self.partition.n_layers(i).unwrap_or(0);
(0..n_layers).fold((0usize, 0u64, 0u64, 0u64), |acc, l| {
let lb = layer_bytes(&self.partition.layer_dir(i, l));
@@ -144,9 +141,7 @@ impl KmerIndex {
let index_dir = self.partition.index_dir(i);
if !index_dir.exists() { return (0, counts); }
let n_layers = PartitionMeta::load(&index_dir)
.map(|m| m.n_layers)
.unwrap_or(0);
let n_layers = self.partition.n_layers(i).unwrap_or(0);
for l in 0..n_layers {
let this_layer_dir = self.partition.layer_dir(i, l);
+2 -7
View File
@@ -2,7 +2,6 @@ use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
use obikseq::CanonicalKmer;
use obiskio::{SKError, SKResult, UnitigFileReader};
use obilayeredmap::{IndexMode, MphfLayer, OLMError};
use obilayeredmap::meta::PartitionMeta;
use crate::filter::{KmerFilter, passes_all};
use crate::partition::KmerPartition;
@@ -39,9 +38,7 @@ impl KmerPartition {
return Ok(true);
}
let index_mode = PartitionMeta::load(&index_dir)
.map(|m| m.mode)
.unwrap_or(IndexMode::Exact);
let index_mode = self.index_mode(part).unwrap_or(IndexMode::Exact);
let mut l = 0;
loop {
@@ -119,9 +116,7 @@ impl KmerPartition {
return Ok(true);
}
let index_mode = PartitionMeta::load(&index_dir)
.map(|m| m.mode)
.unwrap_or(IndexMode::Exact);
let index_mode = self.index_mode(part).unwrap_or(IndexMode::Exact);
let mut layer = 0;
loop {
+1 -2
View File
@@ -21,7 +21,6 @@ use obipipeline::{
use obicompactvec::{PersistentBitMatrixBuilder, PersistentCompactIntMatrixBuilder};
use obikseq::CanonicalKmer;
use obilayeredmap::meta::PartitionMeta;
use obilayeredmap::{layer_dir, IndexMode, Layer, LayeredMap, MphfOnly};
use obiskio::{SKError, SKResult, UnitigFileReader};
@@ -526,7 +525,7 @@ impl KmerPartition {
if let Some(mb) = new_mb {
mb.close().map_err(SKError::Io)?;
let mut part_meta = PartitionMeta::load(&dst_index_dir).map_err(|e| olm_to_sk(e, "merge"))?;
let mut part_meta = self.partition_meta(i)?;
part_meta.n_layers = new_layer_idx + 1;
part_meta.save(&dst_index_dir).map_err(|e| olm_to_sk(e, "merge"))?;
}
@@ -8,6 +8,8 @@ use obisys::progress_bar;
use obikseq::RoutableSuperKmer;
use obiskio::SKResult;
use obilayeredmap::IndexMode;
use obilayeredmap::meta::PartitionMeta;
use rayon::prelude::*;
use remove_dir_all::remove_dir_all;
use sysinfo::System;
@@ -16,6 +18,7 @@ use niffler::Level;
use niffler::send::compression::Format;
use obiskio::SKFileWriter;
use crate::common::load_meta;
use crate::kmer_sort::chunk_size_from_ram;
use super::count::count_partition;
@@ -191,6 +194,26 @@ impl KmerPartition {
obilayeredmap::layer_dir(&self.index_dir(i), l)
}
/// Partition `i`'s metadata (layer count, evidence mode) — the single
/// entry point for this, so that callers outside `obikpartitionner`
/// never need to know it's a `meta.json` loaded via
/// `obilayeredmap::meta::PartitionMeta`, nor handle its own recovery
/// path for indexes built before that file existed (see
/// [`crate::common::load_meta`]).
pub fn partition_meta(&self, i: usize) -> SKResult<PartitionMeta> {
load_meta(&self.index_dir(i), "partition_meta")
}
/// Number of layers in partition `i` — see [`partition_meta`](Self::partition_meta).
pub fn n_layers(&self, i: usize) -> SKResult<usize> {
self.partition_meta(i).map(|m| m.n_layers)
}
/// Evidence mode of partition `i` — see [`partition_meta`](Self::partition_meta).
pub fn index_mode(&self, i: usize) -> SKResult<IndexMode> {
self.partition_meta(i).map(|m| m.mode)
}
pub fn kmer_size(&self) -> usize {
self.kmer_size
}
+1 -2
View File
@@ -5,7 +5,6 @@ use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
use obikseq::CanonicalKmer;
use obiskio::{SKError, SKResult};
use obilayeredmap::{IndexMode, MphfLayer, OLMError};
use obilayeredmap::meta::PartitionMeta;
use crate::partition::KmerPartition;
@@ -178,7 +177,7 @@ impl KmerPartition {
return Ok(stats);
}
let meta = PartitionMeta::load(&index_dir).map_err(olm_to_sk)?;
let meta = self.partition_meta(part_idx)?;
let layers: Vec<QueryLayer> = (0..meta.n_layers)
.map(|i| QueryLayer::open(&self.layer_dir(part_idx, i), with_counts, &meta.mode))
.collect::<SKResult<_>>()?;
+4 -6
View File
@@ -7,7 +7,6 @@ use obicompactvec::{
PersistentBitMatrix, PersistentBitMatrixBuilder,
PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder,
};
use obilayeredmap::meta::PartitionMeta;
use obilayeredmap::OLMError;
use obiskio::{SKError, SKResult};
@@ -150,8 +149,8 @@ impl KmerPartition {
return Ok(());
}
let src_meta = PartitionMeta::load(&src_index_dir).map_err(olm_to_sk)?;
if src_meta.n_layers == 0 {
let n_src_layers = src.n_layers(i)?;
if n_src_layers == 0 {
return Ok(());
}
@@ -162,7 +161,7 @@ impl KmerPartition {
let data_subdir = if output_presence { "presence" } else { "counts" };
for l in 0..src_meta.n_layers {
for l in 0..n_src_layers {
let src_layer_dir = src.layer_dir(i, l);
if !src_layer_dir.exists() { continue; }
@@ -234,8 +233,7 @@ impl KmerPartition {
}
if !in_place {
PartitionMeta::load(&src_index_dir).map_err(olm_to_sk)?
.save(&dst_index_dir).map_err(olm_to_sk)?;
src.partition_meta(i)?.save(&dst_index_dir).map_err(olm_to_sk)?;
}
Ok(())
+7 -6
View File
@@ -8,7 +8,7 @@ use obikpartitionner::KmerPartition;
use obipipeline::ThrottleGuard;
use obikseq::CanonicalKmer;
use obilayeredmap::MphfLayer;
use obilayeredmap::meta::PartitionMeta;
use obilayeredmap::meta::IndexMode;
use obisys::progress_bar;
use obikindex::{OKIError, OKIResult};
@@ -97,11 +97,13 @@ impl SiblingAnnexBuildExt for KmerIndex {
pb.inc(1);
continue;
}
let meta = PartitionMeta::load(&index_dir).map_err(olm_to_ok)?;
let meta = self.partition().partition_meta(part)?;
let mut part_slots: u64 = 0;
for l in 0..meta.n_layers {
part_slots += build_layer_sibling_annex(self, &self.partition().layer_dir(part, l), n_parts, l, &cache)?;
part_slots += build_layer_sibling_annex(
self, &self.partition().layer_dir(part, l), &meta.mode, n_parts, l, &cache,
)?;
}
total_slots += part_slots;
pb.inc(1);
@@ -120,13 +122,12 @@ impl SiblingAnnexBuildExt for KmerIndex {
fn build_layer_sibling_annex(
index: &KmerIndex,
layer_dir: &Path,
mode: &IndexMode,
n_parts: usize,
l: usize,
cache: &Arc<PartitionCache>,
) -> OKIResult<u64> {
let index_dir = layer_dir.parent().expect("layer_dir has a parent index dir");
let meta = PartitionMeta::load(index_dir).map_err(olm_to_ok)?;
let mphf = MphfLayer::open(layer_dir, &meta.mode).map_err(olm_to_ok)?;
let mphf = MphfLayer::open(layer_dir, mode).map_err(olm_to_ok)?;
let k = index.kmer_size();
let n = mphf.n();
+3 -3
View File
@@ -6,13 +6,13 @@ use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix, PersistentS
use obikpartitionner::KmerPartition;
use obikseq::CanonicalKmer;
use obilayeredmap::{Layer, OLMResult};
use obilayeredmap::meta::{IndexMode, PartitionMeta};
use obilayeredmap::meta::IndexMode;
use obisys::progress_bar;
use obikindex::OKIResult;
use super::iter::SiblingLayerExt;
use super::{olm_to_ok, SiblingAnnex};
use super::SiblingAnnex;
/// Every partition's already-open layers, built **once** for the whole
/// `build_sibling_annex` run and shared (read-only) across every lookup, in
@@ -179,7 +179,7 @@ impl PartitionCache {
pb.inc(1);
return Ok((Vec::new(), 0));
}
let meta = PartitionMeta::load(&index_dir).map_err(olm_to_ok)?;
let meta = partition.partition_meta(part)?;
let mut mats = Vec::with_capacity(meta.n_layers);
for l in 0..meta.n_layers {
let Ok(mat) = Mat::open(&partition.layer_dir(part, l), &meta.mode, with_counts) else { continue };
+2 -2
View File
@@ -109,8 +109,8 @@ pub(crate) fn sibling_layer_dirs(index: &KmerIndex) -> OKIResult<Vec<PathBuf>> {
if !index_dir.exists() {
continue;
}
let meta = PartitionMeta::load(&index_dir).map_err(olm_to_ok)?;
for l in 0..meta.n_layers {
let n_layers = index.partition().n_layers(part)?;
for l in 0..n_layers {
let this_layer_dir = index.partition().layer_dir(part, l);
let annex_path = this_layer_dir.join(ANNEX_FILE_NAME);
if !annex_path.exists() {
+5 -9
View File
@@ -3,7 +3,6 @@ use std::path::Path;
use obikseq::{CanonicalKmer, Kmer, Sequence};
use obilayeredmap::MphfLayer;
use obilayeredmap::meta::PartitionMeta;
use obisys::Reporter;
use tempfile::tempdir;
@@ -88,8 +87,7 @@ fn canonical(ascii: &[u8]) -> CanonicalKmer {
/// Read back the annex entry for a given canonical k-mer from the merged
/// index's (single) partition/layer, asserting it was found at all.
fn annex_info_for(idx: &KmerIndex, kmer: CanonicalKmer) -> FamilyMask {
let index_dir = idx.partition().index_dir(0);
let meta = PartitionMeta::load(&index_dir).unwrap();
let meta = idx.partition().partition_meta(0).unwrap();
for l in 0..meta.n_layers {
let layer_dir = idx.partition().layer_dir(0, l);
let mphf = MphfLayer::open(&layer_dir, &meta.mode).unwrap();
@@ -310,9 +308,8 @@ fn sibling_annex_no_empty_masks_after_build() {
let g1 = build_single_genome_index(dir.path(), "g1", &seq);
g1.build_sibling_annex().expect("build_sibling_annex");
let index_dir = g1.partition().index_dir(0);
let meta = PartitionMeta::load(&index_dir).expect("partition meta");
for l in 0..meta.n_layers {
let n_layers = g1.partition().n_layers(0).expect("partition meta");
for l in 0..n_layers {
let layer_dir = g1.partition().layer_dir(0, l);
let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME)).expect("annex open");
for slot in 0..annex.len() {
@@ -361,9 +358,8 @@ fn sibling_annex_records_the_real_layer_of_each_family_member() {
let merged = merge_two(dir.path(), &g1, &g2);
merged.build_sibling_annex().expect("build_sibling_annex");
let index_dir = merged.partition().index_dir(0);
let meta = PartitionMeta::load(&index_dir).unwrap();
assert_eq!(meta.n_layers, 2, "fixture assumption: one merge, one new layer");
let n_layers = merged.partition().n_layers(0).unwrap();
assert_eq!(n_layers, 2, "fixture assumption: one merge, one new layer");
let g1_kmer = canonical(b"AACCGCTTAAG"); // own base C (1), sibling base G (2) — lives in layer 1
let g2_kmer = canonical(b"AACCGGTTAAG"); // own base G (2), sibling base C (1) — lives in layer 0
+92
View File
@@ -85,6 +85,76 @@ impl LayerData for PersistentSparseBitMatrix {
fn read(&self, slot: usize) -> Box<[bool]> { self.row(slot) }
}
// ── LayerContent ─────────────────────────────────────────────────────────────
/// What a layer's data matrix represents — orthogonal to *how* it's stored
/// on disk (`obicompactvec::StorageKind`'s concern). Only meaningful for
/// `D` that actually carry a matrix: `Layer<()>` (mode 1, set membership,
/// no matrix at all) has neither a `LayerContent` nor a `content()` method
/// — it's a write-time-only state, never a queryable content. Once a layer
/// is closed, "no matrix file" reads back as `Presence` via
/// `PersistentBitMatrix::open`'s own `Implicit` fallback, not as some third
/// "empty" content — see `DevDocMD/implementation/partition_layer_cache.md`.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum LayerContent {
Count,
Presence,
}
impl LayerContent {
/// Lightweight disk probe, no MPHF/matrix opened: `counts/` present →
/// `Count`, otherwise `Presence` — an absent/`Implicit` presence
/// matrix still reads as `Presence`, never a separate content.
pub fn detect(layer_dir: &Path) -> LayerContent {
if layer_dir.join(COUNTS_DIR).exists() {
LayerContent::Count
} else {
LayerContent::Presence
}
}
}
/// Implemented only by `D` that carry a real matrix — gates
/// [`Layer::content`](Layer::content) so `Layer<()>` doesn't get it.
pub trait HasLayerContent {
const CONTENT: LayerContent;
}
impl HasLayerContent for PersistentCompactIntMatrix {
const CONTENT: LayerContent = LayerContent::Count;
}
impl HasLayerContent for PersistentBitMatrix {
const CONTENT: LayerContent = LayerContent::Presence;
}
impl HasLayerContent for PersistentSparseBitMatrix {
const CONTENT: LayerContent = LayerContent::Presence;
}
// ── StorageKind passthrough ────────────────────────────────────────────────────
/// Implemented only by `D` that expose their own `storage_kind()` — gates
/// [`Layer::storage_kind`]. `PersistentSparseBitMatrix` has a single,
/// fixed on-disk format (no `Columnar`/`Packed`/`Implicit` variants of its
/// own), so it's deliberately not given an impl: there's nothing to report
/// beyond what `content()` already says.
pub trait HasStorageKind {
fn storage_kind(&self) -> obicompactvec::StorageKind;
}
impl HasStorageKind for PersistentCompactIntMatrix {
fn storage_kind(&self) -> obicompactvec::StorageKind {
PersistentCompactIntMatrix::storage_kind(self)
}
}
impl HasStorageKind for PersistentBitMatrix {
fn storage_kind(&self) -> obicompactvec::StorageKind {
PersistentBitMatrix::storage_kind(self)
}
}
// ── Structures ────────────────────────────────────────────────────────────────
pub struct Layer<D: LayerData = ()> {
@@ -171,6 +241,28 @@ impl<D: LayerData> Layer<D> {
MphfLayer::build_approx_evidence(layer_dir, b, z)
}
/// This already-open layer's evidence mode — reads the discriminant
/// already in memory (`MphfLayer`'s own `LayerEvidence`), no disk
/// access. Available regardless of `D`, unlike `content`/`storage_kind`.
pub fn evidence_kind(&self) -> crate::mphf_layer::EvidenceKind {
self.mphf.evidence_kind()
}
}
impl<D: LayerData + HasLayerContent> Layer<D> {
/// This already-open layer's content (`Count`/`Presence`) — a
/// compile-time fact about `D`, not a runtime read.
pub const fn content(&self) -> LayerContent {
D::CONTENT
}
}
impl<D: LayerData + HasStorageKind> Layer<D> {
/// This already-open layer's storage format — delegates to `D`'s own
/// `storage_kind()`, reading a discriminant already in memory.
pub fn storage_kind(&self) -> obicompactvec::StorageKind {
self.data.storage_kind()
}
}
// ── Mode 1 — set membership ───────────────────────────────────────────────────
+2 -2
View File
@@ -8,8 +8,8 @@ pub mod meta;
pub(crate) mod mphf_layer;
pub use error::{OLMError, OLMResult};
pub use layer::{layer_dir, open_data, Hit, Layer, LayerData};
pub use layer::{layer_dir, open_data, HasLayerContent, HasStorageKind, Hit, Layer, LayerContent, LayerData};
pub use layered_store::LayeredStore;
pub use map::LayeredMap;
pub use meta::{IndexMode, PartitionMeta};
pub use mphf_layer::{KmerBatchIter, KmerIter, MphfLayer, MphfOnly};
pub use mphf_layer::{EvidenceKind, KmerBatchIter, KmerIter, MphfLayer, MphfOnly};
+28 -3
View File
@@ -2,13 +2,14 @@ use std::collections::HashMap;
use std::fs;
use std::path::{Path, PathBuf};
use obicompactvec::PersistentCompactIntMatrix;
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix, StorageKind};
use obikseq::CanonicalKmer;
use obiskio::{UnitigFileWriter, DEFAULT_BLOCK_BITS};
use crate::error::OLMResult;
use crate::layer::{layer_dir, Hit, Layer, LayerData};
use crate::error::{OLMError, OLMResult};
use crate::layer::{layer_dir, Hit, Layer, LayerContent, LayerData};
use crate::meta::{IndexMode, PartitionMeta};
use crate::mphf_layer::EvidenceKind;
/// Layered kmer index for a single partition.
///
@@ -46,6 +47,30 @@ impl<D: LayerData> LayeredMap<D> {
pub fn layer(&self, i: usize) -> &Layer<D> { &self.layers[i] }
pub fn mode(&self) -> &IndexMode { &self.meta.mode }
/// Layer `i`'s content (`Count`/`Presence`) — a lightweight disk probe,
/// no MPHF/matrix opened. For picking a `D` before committing to
/// [`Layer::open`], or for reporting/diagnostics over an already-open
/// `LayeredMap` without re-opening every layer under a different `D`.
pub fn detect_layer_content(&self, i: usize) -> LayerContent {
LayerContent::detect(&layer_dir(&self.root, i))
}
/// Layer `i`'s storage format — a lightweight disk probe, no
/// MPHF/matrix opened. See [`detect_layer_content`](Self::detect_layer_content).
pub fn detect_layer_storage(&self, i: usize) -> OLMResult<StorageKind> {
let dir = layer_dir(&self.root, i);
match LayerContent::detect(&dir) {
LayerContent::Count => PersistentCompactIntMatrix::detect_storage(&dir).map_err(OLMError::Io),
LayerContent::Presence => PersistentBitMatrix::detect_storage(&dir).map_err(OLMError::Io),
}
}
/// Layer `i`'s evidence mode — a lightweight disk probe, no
/// MPHF/matrix opened. See [`detect_layer_content`](Self::detect_layer_content).
pub fn detect_layer_evidence(&self, i: usize) -> OLMResult<EvidenceKind> {
EvidenceKind::detect(&layer_dir(&self.root, i))
}
/// Query `kmer` across all layers. Returns `(layer_index, Hit)` on match.
pub fn query(&self, kmer: CanonicalKmer) -> Option<(usize, Hit<D::Item>)> {
self.layers
+39
View File
@@ -35,6 +35,35 @@ enum LayerEvidence {
Hybrid { evidence: Evidence, unitigs: Arc<UnitigFileReader>, fingerprint: FingerprintVec },
}
// ── EvidenceKind ──────────────────────────────────────────────────────────────
/// Coarse evidence mode of a layer — the `IndexMode` a layer was opened
/// with, without the `Approx`/`Hybrid` `b`/`z` parameters.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum EvidenceKind {
Exact,
Approx,
Hybrid,
}
impl EvidenceKind {
/// Lightweight disk probe, no MPHF/evidence opened: presence of
/// `evidence.bin`/`fingerprint.bin` alone determines the mode — same
/// signal `MphfLayer::open` uses, just without opening either file.
pub fn detect(layer_dir: &Path) -> OLMResult<EvidenceKind> {
let has_evidence = layer_dir.join(EVIDENCE_FILE).exists();
let has_fingerprint = layer_dir.join(FINGERPRINT_FILE).exists();
match (has_evidence, has_fingerprint) {
(true, false) => Ok(EvidenceKind::Exact),
(false, true) => Ok(EvidenceKind::Approx),
(true, true) => Ok(EvidenceKind::Hybrid),
(false, false) => Err(OLMError::InvalidLayer(format!(
"no evidence.bin or fingerprint.bin in {}", layer_dir.display()
))),
}
}
}
// ── MphfLayer ─────────────────────────────────────────────────────────────────
/// Autonomous kmer → slot mapping for one layer.
@@ -163,6 +192,16 @@ impl MphfLayer {
pub fn n(&self) -> usize { self.n }
/// This already-open layer's evidence mode — reads the discriminant
/// already in memory, no disk access.
pub fn evidence_kind(&self) -> EvidenceKind {
match &self.ev {
LayerEvidence::Exact { .. } => EvidenceKind::Exact,
LayerEvidence::Approx { .. } => EvidenceKind::Approx,
LayerEvidence::Hybrid { .. } => EvidenceKind::Hybrid,
}
}
/// Raw MPHF lookup: kmer → slot.
///
/// Returns the slot assigned by the MPHF without performing any membership
+30
View File
@@ -98,6 +98,36 @@ fn presence_layer_generic_over_sparse_matches_dense() {
}
}
// ── content / storage_kind / evidence_kind (runtime introspection) ─────────
#[test]
fn count_layer_reports_count_content_and_columnar_storage() {
set_k(4);
let dir = tempdir().unwrap();
write_unitigs(dir.path(), &[b"AAAACGT"]);
Layer::<PersistentCompactIntMatrix>::build(dir.path(), DEFAULT_BLOCK_BITS, &IndexMode::Exact, |_| 1)
.unwrap();
let layer = Layer::<PersistentCompactIntMatrix>::open(dir.path(), &IndexMode::Exact).unwrap();
assert_eq!(layer.content(), LayerContent::Count);
assert_eq!(layer.storage_kind(), obicompactvec::StorageKind::Columnar);
assert_eq!(layer.evidence_kind(), crate::mphf_layer::EvidenceKind::Exact);
}
#[test]
fn presence_layer_reports_presence_content_and_columnar_storage() {
set_k(4);
let dir = tempdir().unwrap();
write_unitigs(dir.path(), &[b"AAATCTA", b"CTTCGCC", b"TGATACG"]);
Layer::<PersistentBitMatrix>::build_presence(dir.path(), DEFAULT_BLOCK_BITS, &IndexMode::Exact, 2, |kmer, g| {
(kmer.raw().wrapping_add(g as u64)) % 2 == 0
}).unwrap();
let layer = Layer::<PersistentBitMatrix>::open(dir.path(), &IndexMode::Exact).unwrap();
assert_eq!(layer.content(), LayerContent::Presence);
assert_eq!(layer.storage_kind(), obicompactvec::StorageKind::Columnar);
}
#[test]
fn enumerate_kmers_is_stable_across_calls() {
set_k(4);
+122 -1
View File
@@ -1,7 +1,10 @@
use super::*;
use obicompactvec::PersistentCompactIntMatrix;
use obicompactvec::{pack_bit_matrix, pack_sparse_bit_matrix, PersistentBitMatrix, PersistentCompactIntMatrix, StorageKind};
use obikseq::{set_k, Sequence as _, Unitig};
use obiskio::DEFAULT_BLOCK_BITS;
use crate::layer::LayerContent;
use crate::meta::IndexMode;
use crate::mphf_layer::EvidenceKind;
use tempfile::tempdir;
fn push_unitigs_and_layer(
@@ -96,3 +99,121 @@ fn push_layer_from_map_convenience() {
let (_, hit) = map.query(canonical(b"AAAA")).unwrap();
assert_eq!(hit.data[0], 10);
}
// ── detect_layer_content / detect_layer_storage / detect_layer_evidence ────
//
// All built directly on disk (bypassing `LayeredMap<D>::push_layer`, which
// only exists for modes 1 and 2 — mode 3/presence has no `push_layer`),
// then reopened as `LayeredMap<()>` — `()`'s `LayerData::open` never reads
// the matrix, so it's the right handle for exercising the disk-probing
// `detect_layer_*` methods regardless of what content/storage the layer
// underneath actually is.
fn write_unitigs_at(dir: &Path, seqs: &[&[u8]]) {
fs::create_dir_all(dir).unwrap();
let mut w = UnitigFileWriter::create(&dir.join(crate::layer::UNITIGS_FILE)).unwrap();
for s in seqs {
w.write(&Unitig::from_ascii(s)).unwrap();
}
w.close().unwrap();
}
/// Build a one-layer partition root with a presence layer at `layer_0`,
/// save `meta.json`, and reopen it as `LayeredMap<()>`.
fn presence_partition(seqs: &[&[u8]], n_genomes: usize) -> (tempfile::TempDir, LayeredMap<()>) {
let dir = tempdir().unwrap();
let mode = IndexMode::Exact;
let layer0 = layer_dir(dir.path(), 0);
write_unitigs_at(&layer0, seqs);
Layer::<PersistentBitMatrix>::build_presence(&layer0, DEFAULT_BLOCK_BITS, &mode, n_genomes, |kmer, g| {
(kmer.raw().wrapping_add(g as u64)) % 2 == 0
}).unwrap();
PartitionMeta { n_layers: 1, mode }.save(dir.path()).unwrap();
let map = LayeredMap::<()>::open(dir.path()).unwrap();
(dir, map)
}
#[test]
fn detect_layer_content_count() {
set_k(4);
let dir = tempdir().unwrap();
let mut map = LayeredMap::<PersistentCompactIntMatrix>::create(dir.path(), IndexMode::Exact).unwrap();
push_unitigs_and_layer(&mut map, &[b"AAAACGT"], 3);
assert_eq!(map.detect_layer_content(0), LayerContent::Count);
}
#[test]
fn detect_layer_content_presence() {
set_k(4);
let (_dir, map) = presence_partition(&[b"AAATCTA", b"CTTCGCC", b"TGATACG"], 3);
assert_eq!(map.detect_layer_content(0), LayerContent::Presence);
}
#[test]
fn detect_layer_storage_columnar_then_packed_then_sparse() {
set_k(4);
let (dir, map) = presence_partition(&[b"AAATCTA", b"CTTCGCC", b"TGATACG"], 3);
let presence_dir = layer_dir(dir.path(), 0).join("presence");
assert_eq!(map.detect_layer_storage(0).unwrap(), StorageKind::Columnar);
pack_bit_matrix(&presence_dir).unwrap();
assert_eq!(map.detect_layer_storage(0).unwrap(), StorageKind::Packed);
pack_sparse_bit_matrix(&presence_dir).unwrap();
assert_eq!(map.detect_layer_storage(0).unwrap(), StorageKind::Sparse);
}
#[test]
fn detect_layer_storage_implicit() {
set_k(4);
let dir = tempdir().unwrap();
let mode = IndexMode::Exact;
let layer0 = layer_dir(dir.path(), 0);
write_unitigs_at(&layer0, &[b"AAAACGT"]);
// Mode-1 build: MPHF + evidence + `layer_meta.json`, no matrix at all —
// must read back as `Presence`/`Implicit`, never a third "empty" state.
Layer::<()>::build(&layer0, DEFAULT_BLOCK_BITS, &mode).unwrap();
PartitionMeta { n_layers: 1, mode }.save(dir.path()).unwrap();
let map = LayeredMap::<()>::open(dir.path()).unwrap();
assert_eq!(map.detect_layer_content(0), LayerContent::Presence);
assert_eq!(map.detect_layer_storage(0).unwrap(), StorageKind::Implicit);
}
#[test]
fn detect_layer_storage_count_columnar_then_packed() {
set_k(4);
let dir = tempdir().unwrap();
let mut map = LayeredMap::<PersistentCompactIntMatrix>::create(dir.path(), IndexMode::Exact).unwrap();
push_unitigs_and_layer(&mut map, &[b"AAAACGT"], 3);
drop(map);
let map = LayeredMap::<()>::open(dir.path()).unwrap();
assert_eq!(map.detect_layer_storage(0).unwrap(), StorageKind::Columnar);
obicompactvec::pack_compact_int_matrix(&layer_dir(dir.path(), 0).join("counts")).unwrap();
assert_eq!(map.detect_layer_storage(0).unwrap(), StorageKind::Packed);
}
#[test]
fn detect_layer_evidence_exact_vs_approx() {
set_k(4);
let dir = tempdir().unwrap();
let layer0 = layer_dir(dir.path(), 0);
write_unitigs_at(&layer0, &[b"AAAACGT"]);
Layer::<()>::build(&layer0, DEFAULT_BLOCK_BITS, &IndexMode::Exact).unwrap();
PartitionMeta { n_layers: 1, mode: IndexMode::Exact }.save(dir.path()).unwrap();
let map = LayeredMap::<()>::open(dir.path()).unwrap();
assert_eq!(map.detect_layer_evidence(0).unwrap(), EvidenceKind::Exact);
let approx = IndexMode::Approx { b: 8, z: 1 };
let dir2 = tempdir().unwrap();
let layer0b = layer_dir(dir2.path(), 0);
write_unitigs_at(&layer0b, &[b"AAAACGT"]);
Layer::<()>::build(&layer0b, DEFAULT_BLOCK_BITS, &approx).unwrap();
PartitionMeta { n_layers: 1, mode: approx }.save(dir2.path()).unwrap();
let map2 = LayeredMap::<()>::open(dir2.path()).unwrap();
assert_eq!(map2.detect_layer_evidence(0).unwrap(), EvidenceKind::Approx);
}