Add directory-level locking and introduce layered index cache

Introduces directory-level locking to prevent concurrent index operations from corrupting shared directories, along with explicit APIs for acquiring, probing, and releasing locks. Restructures the index cache crate to use a layered store architecture that eagerly initializes metadata and provides fast hierarchical lookups. Updates dependent modules, test suites, and CLI commands to align with the refactored API surface, and adds an end-to-end smoke test for validation.
This commit is contained in:
Eric Coissac committed 2026-08-22 14:04:46 +02:00
1 parent 2419a6c21d
commit 7183e3adb4
15 files changed
+509 -77

No files matched your search

+2
View File
@@ -4,6 +4,8 @@ version = "0.1.0"
edition = "2024"
[dependencies]
obikindex = { path = "../obikindex"}
obikseq = { path = "../obikseq"}
obicompactvec = { path = "../obicompactvec" }
ndarray = "0.17"
rayon = "1"
+132
View File
@@ -0,0 +1,132 @@
use obikindex::{KmerIndex, layer::KmerLayer};
use obikseq::CanonicalKmer;
use crate::meta_cache::MetaCache;
pub struct IndexCache<'a> {
raw_index: &'a KmerIndex,
meta: MetaCache,
n_partition: usize,
n_layer: usize,
layer_cache: Vec<KmerLayer>,
}
impl<'a> IndexCache<'a> {
/// Opens every layer of every partition once, up front, and keeps them
/// alive for the cache's lifetime — `layer(p, l)` stored flat at
/// `p * n_layer + l` (`n_layer` uniform across partitions, guaranteed by
/// construction).
///
/// Panics if the index isn't fully built: a cache can only be built on a
/// functional index, so any failure here (missing/incomplete layer) is a
/// caller error, not a case to design around with a `Result`.
pub fn new(index: &'a KmerIndex) -> Self {
let n_partition = index.n_partitions();
let n_layer = index.n_layers_per_partition().unwrap_or_else(|e| {
panic!("IndexCache::new: index is not fully built (n_layers_per_partition failed): {e}")
});
let mut layer_cache = Vec::with_capacity(n_partition * n_layer);
for p in 0..n_partition {
for l in 0..n_layer {
let layer = index
.layer(p, l)
.and_then(KmerLayer::open)
.unwrap_or_else(|e| {
panic!(
"IndexCache::new: failed to open layer (partition {p}, layer {l}): {e} — index is not fully built"
)
});
layer_cache.push(layer);
}
}
IndexCache {
raw_index: index,
meta: MetaCache::from_meta(&index.meta()),
n_partition,
n_layer,
layer_cache,
}
}
/// The index's metadata, snapshotted once at construction — see
/// [`MetaCache`]. Read-only
#[inline]
pub fn meta(&self) -> &MetaCache {
&self.meta
}
#[inline]
pub fn n_layer(&self) -> usize {
self.n_layer
}
#[inline]
pub fn n_partition(&self) -> usize {
self.n_partition
}
#[inline]
pub fn get_layer(&self, partition: usize, layer: usize) -> Option<&KmerLayer> {
self.layer_cache.get(partition * self.n_layer + layer)
}
pub fn iter(&self) -> impl Iterator<Item = &KmerLayer> {
self.layer_cache.iter()
}
pub fn iter_indexed(&self) -> impl Iterator<Item = (usize, usize, &KmerLayer)> {
let n_layer = self.n_layer;
self.layer_cache
.iter()
.enumerate()
.map(move |(i, layer)| (i / n_layer, i % n_layer, layer))
}
#[inline]
pub fn hash(&self, partition: usize, layer: usize, kmer: CanonicalKmer) -> Option<usize> {
Some(self.get_layer(partition, layer)?.hash(kmer))
}
pub fn n_kmer(&self, partition: usize, layer: usize) -> Option<usize> {
Some(self.get_layer(partition, layer)?.n())
}
/// Membership-checked lookup in one `(partition, layer)` — `None` if
/// that layer doesn't carry `kmer`. Unlike [`hash`](Self::hash), this
/// goes through `KmerLayer::find_slot`'s evidence check, not a blind
/// MPHF lookup.
#[inline]
pub fn find_in_layer(&self, partition: usize, layer: usize, kmer: CanonicalKmer) -> Option<usize> {
self.get_layer(partition, layer)?.find_slot(kmer)
}
/// Membership-checked lookup across every layer of one partition —
/// first layer that carries `kmer` wins. `None` if `partition` is out
/// of range or no layer of it carries `kmer`.
pub fn find_in_partition(
&self,
partition: usize,
kmer: CanonicalKmer,
) -> Option<(usize, usize, usize)> {
for l in 0..self.n_layer {
if let Some(slot) = self.find_in_layer(partition, l, kmer) {
return Some((partition, l, slot));
}
}
None
}
/// Membership-checked lookup across every partition and layer — first
/// one that carries `kmer` wins.
pub fn find(&self, kmer: CanonicalKmer) -> Option<(usize, usize, usize)> {
for p in 0..self.n_partition {
if let Some(hit) = self.find_in_partition(p, kmer) {
return Some(hit);
}
}
None
}
}
+2 -3
View File
@@ -6,6 +6,5 @@
//! `CountPartials`/`BitPartials` aggregation across them. No dependency on
//! `obikindex` — generic over any `S` implementing those traits.
mod layered_store;
pub use layered_store::LayeredStore;
pub mod index_cache;
pub mod meta_cache;
+49
View File
@@ -0,0 +1,49 @@
use obikindex::{GenomeInfo, IndexConfig, IndexMeta, IndexState};
/// A read-only, fully in-memory snapshot of an index's `IndexMeta`, taken
/// once at construction. `IndexMeta` itself only caches `config` — `genomes`
/// and `state` are re-read from disk on every call, by design, since they
/// can legitimately change during a run. This type is the opposite trade:
/// everything frozen at snapshot time, no disk access ever, at the cost of
/// going stale if the index is mutated after the cache is built — matching
/// the "cache can only be built on a functional index, read-only" contract
/// the rest of `IndexCache` follows.
pub struct MetaCache {
config: IndexConfig,
genomes: Vec<GenomeInfo>,
state: IndexState,
}
impl MetaCache {
/// Panics if reading `genomes`/`state` off `meta` fails — same
/// discipline as `IndexCache::new`: a cache can only be built on a
/// functional index, so a failure here means the index isn't one.
pub(crate) fn from_meta(meta: &IndexMeta) -> Self {
let genomes = meta
.genomes()
.unwrap_or_else(|e| panic!("MetaCache::from_meta: failed to read genomes: {e}"));
let state = meta
.state()
.unwrap_or_else(|e| panic!("MetaCache::from_meta: failed to read state: {e}"));
MetaCache {
config: meta.config().clone(),
genomes,
state,
}
}
#[inline]
pub fn config(&self) -> &IndexConfig {
&self.config
}
#[inline]
pub fn genomes(&self) -> &[GenomeInfo] {
&self.genomes
}
#[inline]
pub fn state(&self) -> IndexState {
self.state
}
}