Add directory-level locking and introduce layered index cache

Introduces directory-level locking to prevent concurrent index operations from corrupting shared directories, along with explicit APIs for acquiring, probing, and releasing locks. Restructures the index cache crate to use a layered store architecture that eagerly initializes metadata and provides fast hierarchical lookups. Updates dependent modules, test suites, and CLI commands to align with the refactored API surface, and adds an end-to-end smoke test for validation.
This commit is contained in:
Eric Coissac committed 2026-08-22 14:04:46 +02:00
1 parent 2419a6c21d
commit 7183e3adb4
15 files changed
+509 -77

No files matched your search

@@ -10,6 +10,7 @@ use obikseq::SuperKmer;
use obiskbuilder::build_superkmers;
use super::PartitionRouter;
use crate::extensions::PrivateBuilder;
const K: usize = 11;
const M: usize = 5;
@@ -68,8 +69,8 @@ fn pipeline_counts(seqs: &[&[u8]]) -> (u64, u64) {
drop(kp); // ends the borrow of `index` early — `PartitionRouter`'s `Drop` impl would otherwise extend it to the end of scope
Dereplicator::new(&index).run().unwrap();
let part_dir = index.layer_dir(0, 0);
let dedup_path = obikindex::layer::dereplicated_superkmers_path(&part_dir);
let layer0 = index.layer0(0).unwrap();
let dedup_path = layer0.dereplicated_superkmers_path();
if !dedup_path.exists() {
return (0, 0);
}