Implement explicit directory locking for index operations

Introduces explicit directory locking mechanisms (`DirLock`) to ensure exclusive access during index operations across command stages. This includes modifying index opening logic to acquire locks before proceeding, handling lock acquisition failures by exiting, and implementing safe conditional building of sibling annexes using locking to prevent concurrent corruption.
This commit is contained in:
Eric Coissac committed 2026-10-16 16:00:49 +02:00
1 parent 7c7d7615cb
commit 9ac3a37529
11 files changed
+242 -72

No files matched your search

+58 -17
View File
@@ -56,7 +56,7 @@ impl KmerIndex {
/// access (resuming an interrupted build) uses
/// [`open_lock`](Self::open_lock) instead; either way, `self.lock`
/// starts empty — [`test_lock`](Self::test_lock)/
/// [`acquire_lock`](Self::acquire_lock) are opt-in from here.
/// [`with_lock`](Self::with_lock) are opt-in from here.
pub fn open<P: AsRef<Path>>(path: P) -> OKIResult<Self> {
let root_path = path.as_ref().to_owned();
let meta = IndexMeta::open_at(&root_path).map_err(OKIError::Io)?;
@@ -69,47 +69,88 @@ impl KmerIndex {
})
}
/// `open`, then hold the directory lock exclusively — for a caller
/// that resumes a write pipeline (e.g. `obikmer index` re-run on an
/// existing, not-yet-`Indexed` directory) rather than reading
/// concurrently with others.
/// `open`, holding the directory lock exclusively for as long as the
/// returned index lives — **the** way for a command that modifies an
/// existing index in place (`convert`, `pack`, `annotate`, `utils
/// rename`, resuming an interrupted build, ...) to claim it. The lock is
/// taken *before* the metadata is read, so what the caller reads is what
/// it will modify. A directory that is not an index is reported by
/// `open`, without a lock file being created in it.
///
/// A command that only reads, or that writes a *new* index elsewhere
/// (`merge`, `filter`, `select`: they lock their destination, not their
/// sources), uses [`open`](Self::open). Library code that has to write
/// into an index it was handed (annexes built on demand) wraps that
/// write in [`with_lock`](Self::with_lock), which does nothing more when
/// the lock is already held.
pub fn open_lock<P: AsRef<Path>>(path: P) -> OKIResult<Self> {
let idx = Self::open(path)?;
if !idx.acquire_lock() {
return Err(OKIError::Io(std::io::Error::other(
"failed to acquire index lock",
)));
let path = path.as_ref();
if !Self::is_an_index(path) {
return Self::open(path);
}
let lock = obisys::DirLock::acquire(path).map_err(OKIError::Io)?;
let idx = Self::open(path)?;
*idx.lock.lock().unwrap() = Some(lock);
Ok(idx)
}
/// Block until this index's directory lock is acquired, then hold it
/// (released on [`release_lock`](Self::release_lock) or when `self` is
/// dropped). Returns `false` only on a genuine I/O failure acquiring
/// the lock file itself — contention blocks rather than failing.
pub fn acquire_lock(&self) -> bool {
/// dropped). Idempotent: a no-op, returning `true`, if this index
/// already holds it (a second `DirLock` on the same directory would
/// block on the first). Returns `false` only on a genuine I/O failure
/// acquiring the lock file itself — contention blocks rather than
/// failing.
pub(crate) fn acquire_lock(&self) -> bool {
let mut held = self.lock.lock().unwrap();
if held.is_some() {
return true;
}
match obisys::DirLock::acquire(&self.root_path) {
Ok(lock) => {
*self.lock.lock().unwrap() = Some(lock);
*held = Some(lock);
true
}
Err(_) => false,
}
}
/// Whether this index object currently holds its directory lock.
pub fn holds_lock(&self) -> bool {
self.lock.lock().unwrap().is_some()
}
/// Run `f` with the directory lock held: as is if this index already
/// holds it (opened with [`open_lock`](Self::open_lock), or an enclosing
/// `with_lock`), otherwise taking it for the duration of `f` only — also
/// released if `f` fails. Not for concurrent use from several threads of
/// one process: the first one to finish would release the lock under
/// the others.
pub fn with_lock<T>(&self, f: impl FnOnce() -> OKIResult<T>) -> OKIResult<T> {
let was_held = self.holds_lock();
if !was_held && !self.acquire_lock() {
return Err(OKIError::Io(std::io::Error::other("failed to acquire index lock")));
}
let out = f();
if !was_held {
self.release_lock();
}
out
}
/// Non-blocking probe of this index's directory lock — doesn't retain
/// it either way: `true` if the lock was free (briefly acquired, then
/// immediately released), `false` if another process currently holds
/// it. For a read-only caller that wants to warn about a build in
/// progress without itself becoming exclusive; use
/// [`acquire_lock`](Self::acquire_lock)/[`open_lock`](Self::open_lock)
/// to actually hold it.
/// [`open_lock`](Self::open_lock)/[`with_lock`](Self::with_lock) to
/// actually hold it.
pub fn test_lock(&self) -> bool {
matches!(obisys::DirLock::try_acquire(&self.root_path), Ok(Some(_)))
}
/// Release this index's directory lock, if held. A no-op if it isn't.
pub fn release_lock(&self) {
pub(crate) fn release_lock(&self) {
*self.lock.lock().unwrap() = None;
}
+101
View File
@@ -0,0 +1,101 @@
//! The directory-lock API of [`KmerIndex`]: `open` never locks, `open_lock`
//! holds for the object's lifetime, `with_lock` holds for a section and is a
//! no-op when the lock is already held. A second handle on the directory
//! (`test_lock`) is what tells held from free.
use std::path::PathBuf;
use crate::index::error::OKIError;
use crate::index::kmer_index::KmerIndex;
use crate::index::meta::IndexConfig;
use crate::layer::IndexMode;
/// Same k/m pair as the other tests of this crate (`obikseq`'s k/m are
/// process-wide in test builds).
fn index_dir() -> (tempfile::TempDir, PathBuf) {
let tmp = tempfile::tempdir().expect("tempdir");
let path = tmp.path().join("idx");
let config = IndexConfig {
kmer_size: 11,
minimizer_size: 5,
n_bits: 2,
with_counts: false,
evidence: IndexMode::Exact,
block_bits: 0,
};
drop(KmerIndex::create(&path, config, None).expect("create index")); // lock released on drop
(tmp, path)
}
#[test]
fn open_does_not_lock() {
let (_tmp, path) = index_dir();
let idx = KmerIndex::open(&path).unwrap();
assert!(!idx.holds_lock());
assert!(idx.test_lock(), "the directory must be free");
}
#[test]
fn open_lock_holds_until_dropped() {
let (_tmp, path) = index_dir();
let idx = KmerIndex::open_lock(&path).unwrap();
assert!(idx.holds_lock());
assert!(!idx.test_lock(), "another handle must see the directory locked");
drop(idx);
assert!(KmerIndex::open(&path).unwrap().test_lock(), "dropping releases the lock");
}
#[test]
fn open_lock_on_a_non_index_reports_it_and_leaves_no_lock_file() {
let tmp = tempfile::tempdir().unwrap();
assert!(KmerIndex::open_lock(tmp.path()).is_err());
assert!(!tmp.path().join(".obikmer.lock").exists());
}
#[test]
fn with_lock_holds_for_the_section_only() {
let (_tmp, path) = index_dir();
let idx = KmerIndex::open(&path).unwrap();
let out = idx
.with_lock(|| {
assert!(idx.holds_lock());
assert!(!idx.test_lock());
Ok(7)
})
.unwrap();
assert_eq!(out, 7);
assert!(!idx.holds_lock());
assert!(idx.test_lock());
}
#[test]
fn with_lock_releases_on_error() {
let (_tmp, path) = index_dir();
let idx = KmerIndex::open(&path).unwrap();
let r: Result<(), _> = idx.with_lock(|| Err(OKIError::Io(std::io::Error::other("boom"))));
assert!(r.is_err());
assert!(!idx.holds_lock());
assert!(idx.test_lock());
}
#[test]
fn with_lock_is_a_no_op_when_already_held() {
let (_tmp, path) = index_dir();
{
// Opened with the lock: the section must neither block on itself nor release it.
let idx = KmerIndex::open_lock(&path).unwrap();
idx.with_lock(|| Ok(())).unwrap();
assert!(idx.holds_lock());
assert!(!idx.test_lock());
}
// Nested sections: only the outermost one releases.
let idx = KmerIndex::open(&path).unwrap();
idx.with_lock(|| {
idx.with_lock(|| Ok(()))?;
assert!(idx.holds_lock(), "the inner section must not release the outer lock");
Ok(())
})
.unwrap();
assert!(!idx.holds_lock());
}
+2
View File
@@ -3,6 +3,8 @@ pub mod meta;
pub mod state;
mod builder;
mod kmer_index;
#[cfg(test)]
mod lock_tests;
pub use error::{OKIError, OKIResult};
pub use builder::IndexBuilder;
+5 -1
View File
@@ -93,7 +93,11 @@ fn run_annotate(args: &AnnotateArgs) {
}
};
let idx = open_index(&args.index);
// Rewrites the index metadata in place: exclusive for the whole command.
let idx = KmerIndex::open_lock(&args.index).unwrap_or_else(|e| {
eprintln!("error opening index: {e}");
std::process::exit(1);
});
let mut genomes = idx.meta().genomes().unwrap_or_else(|e| {
eprintln!("error reading index metadata: {e}");
+2 -8
View File
@@ -70,14 +70,8 @@ struct ModeArgs {
}
pub fn run(args: ConvertArgs) {
// Modifies the index in place; acquired before opening so a concurrent
// writer can't slip in between the open and the convert below.
let _lock = obisys::DirLock::acquire(&args.index).unwrap_or_else(|e| {
eprintln!("error locking index directory {}: {e}", args.index.display());
std::process::exit(1);
});
let idx = KmerIndex::open(&args.index).unwrap_or_else(|e| {
// Modifies the index in place: exclusive for the whole command.
let idx = KmerIndex::open_lock(&args.index).unwrap_or_else(|e| {
eprintln!("error opening index: {e}");
std::process::exit(1);
});
+2 -8
View File
@@ -26,14 +26,8 @@ pub struct PackArgs {
}
pub fn run(args: PackArgs) {
// Modifies the index in place; acquired before opening so a concurrent
// writer can't slip in between the open and the pack below.
let _lock = obisys::DirLock::acquire(&args.index).unwrap_or_else(|e| {
eprintln!("error locking index directory {}: {e}", args.index.display());
std::process::exit(1);
});
let idx = Arc::new(KmerIndex::open(&args.index).unwrap_or_else(|e| {
// Modifies the index in place: exclusive for the whole command.
let idx = Arc::new(KmerIndex::open_lock(&args.index).unwrap_or_else(|e| {
eprintln!("error opening index: {e}");
std::process::exit(1);
}));
+3 -7
View File
@@ -78,13 +78,9 @@ pub fn run(args: PhyloArgs) {
// Meant to be (re)built routinely, ahead of any SNP-family distance
// computation that will later consume it.
if args.sibling_annex {
// Writes into the index directory — hold an exclusive lock for the
// duration so a second, concurrent `--sibling-annex` run on the
// same index can't corrupt these writes (see obisys::DirLock).
let _lock = obisys::DirLock::acquire(&args.index).unwrap_or_else(|e| {
eprintln!("error locking index directory {}: {e}", args.index.display());
std::process::exit(1);
});
// Writes into the index directory; `build_sibling_annex` holds the
// index's exclusive lock for the duration (`KmerIndex::with_lock`),
// so a concurrent run on the same index can't corrupt these writes.
info!("building sibling-count/minorant annex");
let t = Stage::start("sibling_annex");
cache.build_sibling_annex().unwrap_or_else(|e| {
+3 -1
View File
@@ -53,7 +53,9 @@ pub(super) fn run_bits_per_kmer(index_path: &PathBuf) {
pub(super) fn run_rename(index_path: &PathBuf, spec: &str) {
let (old_label, new_label) = parse_rename_spec(spec);
let idx = KmerIndex::open(index_path).unwrap_or_else(|e| {
// Rewrites the index metadata and renames a spectrum file in place:
// exclusive for the whole command.
let idx = KmerIndex::open_lock(index_path).unwrap_or_else(|e| {
eprintln!("error opening index: {e}");
std::process::exit(1);
});
@@ -22,7 +22,9 @@ use super::masking::is_quartet_informative;
use super::minorant_selection::non_monomorphic_reindex_layer;
use super::pair_weight::pair_balance_weight;
use crate::siblings::quartet_annex::quartet_annex_file_name;
use crate::siblings::{FamilyOverlapAccumulator, FamilyScalarAnnexBuilder, ScalarKind};
use crate::siblings::{
FamilyOverlapAccumulator, FamilyScalarAnnexBuilder, ScalarKind, family_overlap_file_name,
};
/// Which annexes a caller needs, each listed once per `--no-ambiguity`
/// counting convention (`false` = genomes carrying several members count as a
@@ -50,6 +52,24 @@ impl AnnexRequest {
dedup(self.overlap.iter().chain(&self.pair_weight).copied())
}
/// Whether every annex this request asks for already exists — the same
/// per-file existence test the builders apply, so `true` means they would
/// find nothing to build and no lock is needed. Cheap (no file is opened).
pub(crate) fn is_satisfied(&self, cache: &IndexCache) -> bool {
let overlap = self
.overlap_conventions()
.into_iter()
.all(|nb| cache.index().dir().join(family_overlap_file_name(nb)).exists());
let pair_weight = self.pair_weight_conventions();
overlap
&& cache.iter_indexed().all(|(_, _, layer)| {
let has = |name: &str| layer.dir().join(name).exists();
self.entropy.iter().all(|&nb| has(ScalarKind::Entropy.file_name(nb)))
&& self.quartet.iter().all(|&nb| has(quartet_annex_file_name(nb)))
&& pair_weight.iter().all(|&nb| has(ScalarKind::PairWeight.file_name(nb)))
})
}
/// The pair-weight conventions requested, each once.
pub(crate) fn pair_weight_conventions(&self) -> Vec<bool> {
dedup(self.pair_weight.iter().copied())
@@ -9,7 +9,7 @@
use obikidxcache::index_cache::IndexCache;
use obikindex::{OKIError, OKIResult};
use obisys::{DirLock, progress_bar};
use obisys::progress_bar;
use crate::siblings::algorithms::{
AnnexRequest, build_layer_sibling_annex, ensure_layer_family_annexes,
@@ -29,26 +29,34 @@ pub(crate) trait SiblingBuilder {
/// a missing entropy, quartet or overlap annex triggers one scan that
/// feeds every missing one, per `ensure_layer_family_annexes`'s own docs,
/// then a second scan builds the missing pair weights, which need the
/// finished overlap matrix. A no-op, cheap existence-check pass, once
/// everything requested already exists.
/// finished overlap matrix. Makes sure the sibling annex exists first
/// ([`ensure_sibling_annex`](Self::ensure_sibling_annex)). A no-op, cheap
/// existence-check pass that takes no lock, once everything requested
/// already exists; otherwise the build runs under the index's exclusive
/// lock ([`KmerIndex::with_lock`](obikindex::KmerIndex::with_lock)).
fn ensure_family_annexes(&self, request: &AnnexRequest) -> OKIResult<()>;
/// [`ensure_family_annexes`](Self::ensure_family_annexes)'s build step,
/// without the existence pre-check and without the lock: the caller
/// holds it.
fn build_family_annexes(&self, request: &AnnexRequest) -> OKIResult<()>;
/// Build the sibling annex (`siblings.psib`, every annex above is derived
/// from it) if any cached layer lacks it — the state of an index right
/// after `migrate-mphf`, which sets every annex aside. The build writes
/// into every layer, so it runs under the index's exclusive
/// [`DirLock`]; a process that waited for it finds the annex built by the
/// one that held it, and does nothing. A no-op, cheap existence-check
/// pass, once every layer has its annex. Must not be called while this
/// process already holds the lock *and* an annex is missing (the lock is
/// not reentrant); `--sibling-annex` holds it but builds the annex first.
/// into every layer, so it runs under the index's exclusive lock
/// ([`KmerIndex::with_lock`](obikindex::KmerIndex::with_lock), a no-op
/// when the caller already holds it); a process that waited for it finds
/// the annex built by the one that held it, and does nothing. A no-op,
/// cheap existence-check pass that takes no lock, once every layer has
/// its annex.
fn ensure_sibling_annex(&self) -> OKIResult<()>;
/// (Re)build the sibling annex of every cached layer, unconditionally,
/// removing the annexes derived from it (entropy, quartet, pair weight,
/// family overlap), which its renumbering makes stale. Takes no lock:
/// the caller holds it (`--sibling-annex`) or goes through
/// [`ensure_sibling_annex`](Self::ensure_sibling_annex).
/// the caller holds it ([`SiblingExt::build_sibling_annex`], or
/// [`ensure_sibling_annex`](Self::ensure_sibling_annex)).
fn build_base_sibling_annex(&self) -> OKIResult<()>;
}
@@ -58,12 +66,13 @@ impl SiblingBuilder for IndexCache {
if !missing() {
return Ok(());
}
let _lock = DirLock::acquire(self.index().dir()).map_err(OKIError::Io)?;
if !missing() {
return Ok(()); // built by the process we waited for
}
tracing::info!("sibling annex missing — building it");
self.build_base_sibling_annex()
self.index().with_lock(|| {
if !missing() {
return Ok(()); // built by the process we waited for
}
tracing::info!("sibling annex missing — building it");
self.build_base_sibling_annex()
})
}
fn build_base_sibling_annex(&self) -> OKIResult<()> {
@@ -136,6 +145,13 @@ impl SiblingBuilder for IndexCache {
fn ensure_family_annexes(&self, request: &AnnexRequest) -> OKIResult<()> {
self.ensure_sibling_annex()?;
if request.is_satisfied(self) {
return Ok(());
}
self.index().with_lock(|| self.build_family_annexes(request))
}
fn build_family_annexes(&self, request: &AnnexRequest) -> OKIResult<()> {
let n_genomes = self.meta().genomes().len();
let fast_mode = is_fast_mode(self);
let overlap_path = |no_ambiguity: bool| self.index().dir().join(family_overlap_file_name(no_ambiguity));
@@ -106,18 +106,18 @@ pub trait SiblingExt {
impl SiblingExt for IndexCache {
fn build_sibling_annex(&self) -> OKIResult<()> {
self.build_base_sibling_annex()?;
// Precompute the entropy and family-overlap annexes too, now that
// every layer's fresh sibling annex is on disk
// (`non_monomorphic_reindex_layer`, which both depend on, reads
// it) — same routine-rebuild footing as the sibling annex itself,
// rather than leaving them to be built lazily on first
// `--shannon`/`--entropy-bias`/`--min-shared-family`/
// `--family-overlap` use.
tracing::info!("building entropy and quartet (both --no-ambiguity conventions) and family-overlap annexes");
self.ensure_family_annexes(&AnnexRequest::everything())?;
Ok(())
self.index().with_lock(|| {
self.build_base_sibling_annex()?;
// Precompute the entropy, quartet, pair-weight and family-overlap
// annexes too, now that every layer's fresh sibling annex is on
// disk (`non_monomorphic_reindex_layer`, which they depend on,
// reads it) — same routine-rebuild footing as the sibling annex
// itself, rather than leaving them to be built lazily on first
// `--shannon`/`--entropy`/`--quartet`/`--pair-balance`/
// `--min-shared-family`/`--family-overlap` use.
tracing::info!("building entropy, quartet, pair-weight (both --no-ambiguity conventions) and family-overlap annexes");
self.ensure_family_annexes(&AnnexRequest::everything())
})
}
fn shannon_entropy_csv(&self, path: &Path) -> OKIResult<()> {