OBIKmers/obikmer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 34 Wiki Activity

Centralize partition metadata access and add layer introspection APIs

Browse Source 
Replaced scattered direct metadata loading with centralized instance methods on `KmerPartition` to guarantee consistent error mapping and legacy recovery. Introduced `StorageKind`, `LayerContent`, and `EvidenceKind` enums alongside lightweight disk-probe methods that inspect file presence without opening heavy data structures. Updated callers across the index, partitioner, and phylo modules to use the new partition API, and added unit tests validating the introspection behavior.
This commit is contained in:
Eric Coissac
2026-08-20 15:29:53 +02:00
parent f7ebc7a1ab
commit 76cbd3a886
23 changed files with 580 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/obikindex/src/index.rs
+6 -14
View File
@@ -3,7 +3,6 @@ use std::fs;
use std::path::{Path, PathBuf};
use obikpartitionner::{KmerPartition, KmerSpectrum, PARTITIONS_SUBDIR};
use obilayeredmap;
use obisys::{Reporter, Stage, progress_bar};
use rayon::prelude::*;
use tracing::info;
@@ -148,11 +147,7 @@ impl KmerIndex {
/// homogeneous across all partitions — reading it off partition 0
/// is enough, no need to scan every partition.
pub fn n_layers_per_partition(&self) -> OKIResult<usize> {
use obilayeredmap::meta::PartitionMeta;
let index_dir = self.partition.index_dir(0);
let meta = PartitionMeta::load(&index_dir)
.map_err(|e| OKIError::Io(std::io::Error::new(std::io::ErrorKind::Other, e.to_string())))?;
Ok(meta.n_layers)
Ok(self.partition.n_layers(0)?)
}
/// Expose the inner partition so the caller can run scatter into it.
@@ -279,7 +274,6 @@ impl KmerIndex {
/// (see the sparse-matrix design plan's "Explicitly deferred").
pub fn pack_matrices(&self, sparse: bool) -> OKIResult<()> {
use obicompactvec::{pack_bit_matrix, pack_compact_int_matrix, pack_sparse_bit_matrix};
use obilayeredmap::meta::PartitionMeta;
let n = self.n_partitions();
let order: Vec<usize> = (0..n).collect();
@@ -289,9 +283,8 @@ impl KmerIndex {
|i| -> OKIResult<()> {
let index_dir = self.partition.index_dir(i);
if !index_dir.exists() { return Ok(()); }
let meta = PartitionMeta::load(&index_dir)
.map_err(|e| OKIError::Io(std::io::Error::new(std::io::ErrorKind::Other, e.to_string())))?;
for l in 0..meta.n_layers {
let n_layers = self.partition.n_layers(i)?;
for l in 0..n_layers {
let layer_dir = self.partition.layer_dir(i, l);
let presence_dir = layer_dir.join("presence");
let counts_dir = layer_dir.join("counts");
@@ -319,7 +312,6 @@ impl KmerIndex {
pub fn upgrade_layer_meta(&self) -> OKIResult<()> {
use obicompactvec::LayerMeta;
use obiskio::UnitigFileReader;
use obilayeredmap::meta::PartitionMeta;
let n = self.n_partitions();
let errors: Vec<_> = (0..n)
@@ -327,11 +319,11 @@ impl KmerIndex {
.filter_map(|i| {
let index_dir = self.partition.index_dir(i);
if !index_dir.exists() { return None; }
let meta = match PartitionMeta::load(&index_dir) {
Ok(m) => m,
let n_layers = match self.partition.n_layers(i) {
Ok(n) => n,
Err(e) => return Some(OKIError::Io(std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))),
};
for l in 0..meta.n_layers {
for l in 0..n_layers {
let layer_dir = self.partition.layer_dir(i, l);
let meta_path = layer_dir.join(LayerMeta::FILENAME);
if meta_path.exists() { continue; }
src/obikindex/src/reindex.rs
+8 -8
View File
@@ -1,7 +1,7 @@
use std::fs;
use std::path::Path;
use obilayeredmap::{layer_dir, IndexMode, layer::Layer};
use obilayeredmap::meta::PartitionMeta;
use obikpartitionner::KmerPartition;
use obilayeredmap::{IndexMode, layer::Layer};
use obisys::{Reporter, Stage, progress_bar};
use tracing::info;
@@ -48,7 +48,7 @@ impl KmerIndex {
let runner = crate::numa::PartitionRunner::new();
runner.run(
&order,
|i| reindex_partition(&self.partition.index_dir(i), &target, block_bits)
|i| reindex_partition(&self.partition, i, &target, block_bits)
.map_err(|e| OKIError::InvalidInput(format!("partition {i}: {e}"))),
|_, _, _| { pb.inc(1); },
)?;
@@ -66,13 +66,13 @@ impl KmerIndex {
}
/// Process all layers of one partition's index directory.
fn reindex_partition(index_dir: &Path, target: &IndexMode, block_bits: u8) -> OKIResult<()> {
if !index_dir.exists() {
fn reindex_partition(partition: &KmerPartition, i: usize, target: &IndexMode, block_bits: u8) -> OKIResult<()> {
if !partition.index_dir(i).exists() {
return Ok(());
}
let pm = PartitionMeta::load(index_dir).map_err(olm_to_oki)?;
for layer_idx in 0..pm.n_layers {
reindex_layer(&layer_dir(index_dir, layer_idx), target, block_bits)?;
let n_layers = partition.n_layers(i).map_err(|e| OKIError::InvalidInput(e.to_string()))?;
for layer_idx in 0..n_layers {
reindex_layer(&partition.layer_dir(i, layer_idx), target, block_bits)?;
}
Ok(())
}
src/obikindex/src/stats.rs
+2 -7
View File
@@ -3,7 +3,6 @@ use std::path::Path;
use obicompactvec::{LayerMeta, PersistentBitMatrix, PersistentCompactIntMatrix};
use obicompactvec::traits::ColumnWeights;
use obilayeredmap::meta::PartitionMeta;
use rayon::prelude::*;
use crate::error::OKIResult;
@@ -93,9 +92,7 @@ impl KmerIndex {
let index_dir = self.partition.index_dir(i);
if !index_dir.exists() { return (0usize, 0u64, 0u64, 0u64); }
let n_layers = PartitionMeta::load(&index_dir)
.map(|m| m.n_layers)
.unwrap_or(0);
let n_layers = self.partition.n_layers(i).unwrap_or(0);
(0..n_layers).fold((0usize, 0u64, 0u64, 0u64), |acc, l| {
let lb = layer_bytes(&self.partition.layer_dir(i, l));
@@ -144,9 +141,7 @@ impl KmerIndex {
let index_dir = self.partition.index_dir(i);
if !index_dir.exists() { return (0, counts); }
let n_layers = PartitionMeta::load(&index_dir)
.map(|m| m.n_layers)
.unwrap_or(0);
let n_layers = self.partition.n_layers(i).unwrap_or(0);
for l in 0..n_layers {
let this_layer_dir = self.partition.layer_dir(i, l);