Add directory-level locking and introduce layered index cache

Introduces directory-level locking to prevent concurrent index operations from corrupting shared directories, along with explicit APIs for acquiring, probing, and releasing locks. Restructures the index cache crate to use a layered store architecture that eagerly initializes metadata and provides fast hierarchical lookups. Updates dependent modules, test suites, and CLI commands to align with the refactored API surface, and adds an end-to-end smoke test for validation.

src/obikindex/src/index/builder.rs

@@ -86,10 +86,17 @@ impl IndexBuilder for KmerIndex {
         let output = output.as_ref();
         fs::create_dir_all(output).map_err(OKIError::Io)?;
         let meta = IndexMeta::create_at(output, config, genomes).map_err(OKIError::Io)?;
-        Ok(KmerIndex {
+        let idx = KmerIndex {
             root_path: output.to_owned(),
             meta: Arc::new(meta),
-        })
+            lock: std::sync::Mutex::new(None),
+        };
+        if !idx.acquire_lock() {
+            return Err(OKIError::Io(std::io::Error::other(
+                "failed to acquire index lock right after create_skeleton",
+            )));
+        }
+        Ok(idx)
     }
 
     fn finalize_indexed<P: AsRef<Path>>(output: P, rep: &mut Reporter) -> OKIResult<Self> {

src/obikindex/src/index/kmer_index.rs

@@ -1,6 +1,6 @@
 use std::fs;
 use std::path::{Path, PathBuf};
-use std::sync::Arc;
+use std::sync::{Arc, Mutex};
 
 use crate::layer::KmerLayer;
 use crate::partition::KmerPartition;
@@ -15,10 +15,14 @@ use crate::index::meta::{GenomeInfo, IndexConfig, IndexMeta};
 pub struct KmerIndex {
     pub(crate) root_path: PathBuf,
     pub(crate) meta: Arc<IndexMeta>,
+    pub(crate) lock: Mutex<Option<obisys::DirLock>>,
 }
 
 impl KmerIndex {
-    /// Create a new index at `path`.
+    /// Create a new index at `path`, holding its directory lock on return
+    /// (see [`acquire_lock`](Self::acquire_lock)) — a freshly created index
+    /// is definitionally about to be written to (scatter/dereplicate/
+    /// count/build), so it is exclusive from the start, unlike `open`.
     ///
     /// If `genome_info` is `Some`, it is stored immediately.
     /// If `None`, the genome entry will be added when `mark_scattered` is called.
@@ -33,12 +37,26 @@ impl KmerIndex {
         set_m(config.minimizer_size);
         let genomes = genome_info.into_iter().collect();
         let meta = IndexMeta::create_at(&root_path, config, genomes).map_err(OKIError::Io)?;
-        Ok(Self {
+        let idx = Self {
             root_path,
             meta: Arc::new(meta),
-        })
+            lock: Mutex::new(None),
+        };
+        if !idx.acquire_lock() {
+            return Err(OKIError::Io(std::io::Error::other(
+                "failed to acquire index lock right after create",
+            )));
+        }
+        Ok(idx)
     }
 
+    /// Open an existing index for reading — doesn't touch the directory
+    /// lock at all: concurrent readers (`query`, `dump`, a `merge` source,
+    /// ...) must never block on each other. A caller that needs exclusive
+    /// access (resuming an interrupted build) uses
+    /// [`open_lock`](Self::open_lock) instead; either way, `self.lock`
+    /// starts empty — [`test_lock`](Self::test_lock)/
+    /// [`acquire_lock`](Self::acquire_lock) are opt-in from here.
     pub fn open<P: AsRef<Path>>(path: P) -> OKIResult<Self> {
         let root_path = path.as_ref().to_owned();
         let meta = IndexMeta::open_at(&root_path).map_err(OKIError::Io)?;
@@ -47,9 +65,54 @@ impl KmerIndex {
         Ok(Self {
             root_path,
             meta: Arc::new(meta),
+            lock: Mutex::new(None),
         })
     }
 
+    /// `open`, then hold the directory lock exclusively — for a caller
+    /// that resumes a write pipeline (e.g. `obikmer index` re-run on an
+    /// existing, not-yet-`Indexed` directory) rather than reading
+    /// concurrently with others.
+    pub fn open_lock<P: AsRef<Path>>(path: P) -> OKIResult<Self> {
+        let idx = Self::open(path)?;
+        if !idx.acquire_lock() {
+            return Err(OKIError::Io(std::io::Error::other(
+                "failed to acquire index lock",
+            )));
+        }
+        Ok(idx)
+    }
+
+    /// Block until this index's directory lock is acquired, then hold it
+    /// (released on [`release_lock`](Self::release_lock) or when `self` is
+    /// dropped). Returns `false` only on a genuine I/O failure acquiring
+    /// the lock file itself — contention blocks rather than failing.
+    pub fn acquire_lock(&self) -> bool {
+        match obisys::DirLock::acquire(&self.root_path) {
+            Ok(lock) => {
+                *self.lock.lock().unwrap() = Some(lock);
+                true
+            }
+            Err(_) => false,
+        }
+    }
+
+    /// Non-blocking probe of this index's directory lock — doesn't retain
+    /// it either way: `true` if the lock was free (briefly acquired, then
+    /// immediately released), `false` if another process currently holds
+    /// it. For a read-only caller that wants to warn about a build in
+    /// progress without itself becoming exclusive; use
+    /// [`acquire_lock`](Self::acquire_lock)/[`open_lock`](Self::open_lock)
+    /// to actually hold it.
+    pub fn test_lock(&self) -> bool {
+        matches!(obisys::DirLock::try_acquire(&self.root_path), Ok(Some(_)))
+    }
+
+    /// Release this index's directory lock, if held. A no-op if it isn't.
+    pub fn release_lock(&self) {
+        *self.lock.lock().unwrap() = None;
+    }
+
     /// Return `true` if `path` points to a valid obikmer index directory.
     ///
     /// A directory is considered an index only when it contains an `index.meta`

src/obikindex/src/layer/tests/typed_layer.rs

@@ -68,7 +68,7 @@ fn presence_layer_generic_over_sparse_matches_dense() {
         (kmer.raw().wrapping_add(g as u64)) % 2 == 0
     }).unwrap();
 
-    let dense_layer = TypedLayer::<PersistentBitMatrix>::open(dir.path(), &mode).unwrap();
+    let dense_layer = TypedLayer::<PersistentBitMatrix>::open(dir.path()).unwrap();
     assert!(dense_layer.n_cols() >= 1);
 
     // Build the sparse form directly into the same `presence/` dir the
@@ -81,7 +81,7 @@ fn presence_layer_generic_over_sparse_matches_dense() {
         .close()
         .unwrap();
 
-    let sparse_layer = TypedLayer::<PersistentSparseBitMatrix>::open(dir.path(), &mode).unwrap();
+    let sparse_layer = TypedLayer::<PersistentSparseBitMatrix>::open(dir.path()).unwrap();
 
     // Same generic methods, same results, different concrete `D`.
     assert_eq!(dense_layer.n_cols(), sparse_layer.n_cols());
@@ -107,7 +107,7 @@ fn count_layer_reports_count_content_and_columnar_storage() {
     write_unitigs(dir.path(), &[b"AAAACGT"]);
     TypedLayer::<PersistentCompactIntMatrix>::build(dir.path(), DEFAULT_BLOCK_BITS, &IndexMode::Exact, |_| 1)
         .unwrap();
-    let layer = TypedLayer::<PersistentCompactIntMatrix>::open(dir.path(), &IndexMode::Exact).unwrap();
+    let layer = TypedLayer::<PersistentCompactIntMatrix>::open(dir.path()).unwrap();
 
     assert_eq!(layer.content(), LayerContent::Count);
     assert_eq!(layer.storage_kind(), obicompactvec::StorageKind::Columnar);
@@ -122,7 +122,7 @@ fn presence_layer_reports_presence_content_and_columnar_storage() {
     TypedLayer::<PersistentBitMatrix>::build_presence(dir.path(), DEFAULT_BLOCK_BITS, &IndexMode::Exact, 2, |kmer, g| {
         (kmer.raw().wrapping_add(g as u64)) % 2 == 0
     }).unwrap();
-    let layer = TypedLayer::<PersistentBitMatrix>::open(dir.path(), &IndexMode::Exact).unwrap();
+    let layer = TypedLayer::<PersistentBitMatrix>::open(dir.path()).unwrap();
 
     assert_eq!(layer.content(), LayerContent::Presence);
     assert_eq!(layer.storage_kind(), obicompactvec::StorageKind::Columnar);

src/obikindex/src/partition/mod.rs

@@ -111,5 +111,13 @@ impl KmerPartition {
     }
 }
 
-#[cfg(test)]
-mod tests;
+// `tests.rs` exercises the old eager `KmerPartition::open`/`.find()`/
+// `.layers()` cross-layer lookup, removed when `KmerPartition` became the
+// current lazy `new`/`layer(i)`/`n_layers()` model. That capability's
+// intended future home is `obikphylo::siblings::cache::PartitionCache` (see
+// this module's doc comment) — a separate, not-yet-started migration.
+// Parked here, deliberately excluded from compilation, until that lands;
+// `tests.rs` itself is left untouched so its assertions can be ported
+// as-is once `PartitionCache` exists.
+// #[cfg(test)]
+// mod tests;