Implement directory locking with shared and exclusive modes
Introduces a robust, OS-level directory locking mechanism using LockSlot for managing shared and exclusive access, supporting lock transitions and non-blocking probes. Also sets up the index structure with new builder and index modules.
This commit is contained in:
1 parent
9ac3a37529
commit
85296a2ef8
10 files changed
+569
-231
No files matched your search
Generated
+1
-1
@@ -1537,7 +1537,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "obikmer"
|
||||
version = "1.4.0"
|
||||
version = "1.4.1"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"csv",
|
||||
|
||||
@@ -90,7 +90,7 @@ impl IndexBuilder for KmerIndex {
|
||||
let idx = KmerIndex {
|
||||
root_path: output.to_owned(),
|
||||
meta: Arc::new(meta),
|
||||
lock: std::sync::Mutex::new(None),
|
||||
lock: obisys::LockSlot::empty(output),
|
||||
};
|
||||
if !idx.acquire_lock() {
|
||||
return Err(OKIError::Io(std::io::Error::other(
|
||||
@@ -102,7 +102,9 @@ impl IndexBuilder for KmerIndex {
|
||||
|
||||
fn finalize_indexed<P: AsRef<Path>>(output: P, rep: &mut Reporter, sparse: bool) -> OKIResult<Self> {
|
||||
let output = output.as_ref();
|
||||
let idx = KmerIndex::open(output)?;
|
||||
// `output` is held exclusively by the builder that wrote it: reopen
|
||||
// without a lock, a locking `open` would wait for that builder for ever.
|
||||
let idx = KmerIndex::open_unlocked(output)?;
|
||||
idx.meta.mark_indexed().map_err(OKIError::Io)?;
|
||||
let t_pack = Stage::start("pack");
|
||||
idx.pack_matrices(sparse)?;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::layer::KmerLayer;
|
||||
use crate::partition::KmerPartition;
|
||||
@@ -15,7 +15,7 @@ use crate::index::meta::{GenomeInfo, IndexConfig, IndexMeta};
|
||||
pub struct KmerIndex {
|
||||
pub(crate) root_path: PathBuf,
|
||||
pub(crate) meta: Arc<IndexMeta>,
|
||||
pub(crate) lock: Mutex<Option<obisys::DirLock>>,
|
||||
pub(crate) lock: obisys::LockSlot,
|
||||
}
|
||||
|
||||
impl KmerIndex {
|
||||
@@ -37,11 +37,8 @@ impl KmerIndex {
|
||||
set_m(config.minimizer_size);
|
||||
let genomes = genome_info.into_iter().collect();
|
||||
let meta = IndexMeta::create_at(&root_path, config, genomes).map_err(OKIError::Io)?;
|
||||
let idx = Self {
|
||||
root_path,
|
||||
meta: Arc::new(meta),
|
||||
lock: Mutex::new(None),
|
||||
};
|
||||
let lock = obisys::LockSlot::empty(&root_path);
|
||||
let idx = Self { root_path, meta: Arc::new(meta), lock };
|
||||
if !idx.acquire_lock() {
|
||||
return Err(OKIError::Io(std::io::Error::other(
|
||||
"failed to acquire index lock right after create",
|
||||
@@ -50,108 +47,92 @@ impl KmerIndex {
|
||||
Ok(idx)
|
||||
}
|
||||
|
||||
/// Open an existing index for reading — doesn't touch the directory
|
||||
/// lock at all: concurrent readers (`query`, `dump`, a `merge` source,
|
||||
/// ...) must never block on each other. A caller that needs exclusive
|
||||
/// access (resuming an interrupted build) uses
|
||||
/// [`open_lock`](Self::open_lock) instead; either way, `self.lock`
|
||||
/// starts empty — [`test_lock`](Self::test_lock)/
|
||||
/// [`with_lock`](Self::with_lock) are opt-in from here.
|
||||
/// Open an existing index for reading, holding a **shared** directory
|
||||
/// lock for as long as the returned index lives: any number of readers
|
||||
/// coexist, but a writer ([`open_lock`](Self::open_lock),
|
||||
/// [`with_lock`](Self::with_lock)) waits for them all, and readers wait
|
||||
/// for a writer at work — a reader never sees files change under it.
|
||||
/// The lock is taken before the metadata is read.
|
||||
///
|
||||
/// Never open, from a process that holds the write lock on a directory
|
||||
/// through another handle, that same directory again with `open`: the
|
||||
/// new handle would wait for the first for ever.
|
||||
pub fn open<P: AsRef<Path>>(path: P) -> OKIResult<Self> {
|
||||
let root_path = path.as_ref().to_owned();
|
||||
let meta = IndexMeta::open_at(&root_path).map_err(OKIError::Io)?;
|
||||
set_k(meta.config.kmer_size);
|
||||
set_m(meta.config.minimizer_size);
|
||||
Ok(Self {
|
||||
root_path,
|
||||
meta: Arc::new(meta),
|
||||
lock: Mutex::new(None),
|
||||
})
|
||||
Self::open_with(path.as_ref(), obisys::LockMode::Shared)
|
||||
}
|
||||
|
||||
/// `open`, holding the directory lock exclusively for as long as the
|
||||
/// `open`, holding the directory lock **exclusively** for as long as the
|
||||
/// returned index lives — **the** way for a command that modifies an
|
||||
/// existing index in place (`convert`, `pack`, `annotate`, `utils
|
||||
/// rename`, resuming an interrupted build, ...) to claim it. The lock is
|
||||
/// taken *before* the metadata is read, so what the caller reads is what
|
||||
/// it will modify. A directory that is not an index is reported by
|
||||
/// `open`, without a lock file being created in it.
|
||||
/// it will modify.
|
||||
///
|
||||
/// A command that only reads, or that writes a *new* index elsewhere
|
||||
/// (`merge`, `filter`, `select`: they lock their destination, not their
|
||||
/// sources), uses [`open`](Self::open). Library code that has to write
|
||||
/// into an index it was handed (annexes built on demand) wraps that
|
||||
/// write in [`with_lock`](Self::with_lock), which does nothing more when
|
||||
/// the lock is already held.
|
||||
/// (`merge`, `filter`, `select`: exclusive on their destination, shared
|
||||
/// on their sources), uses [`open`](Self::open). Library code that has to
|
||||
/// write into an index it was handed (annexes built on demand) wraps that
|
||||
/// write in [`with_lock`](Self::with_lock).
|
||||
pub fn open_lock<P: AsRef<Path>>(path: P) -> OKIResult<Self> {
|
||||
let path = path.as_ref();
|
||||
if !Self::is_an_index(path) {
|
||||
return Self::open(path);
|
||||
Self::open_with(path.as_ref(), obisys::LockMode::Exclusive)
|
||||
}
|
||||
let lock = obisys::DirLock::acquire(path).map_err(OKIError::Io)?;
|
||||
let idx = Self::open(path)?;
|
||||
*idx.lock.lock().unwrap() = Some(lock);
|
||||
|
||||
fn open_with(path: &Path, mode: obisys::LockMode) -> OKIResult<Self> {
|
||||
// A directory that is not an index is reported by `open_unlocked`,
|
||||
// without a lock file being created in it.
|
||||
if !Self::is_an_index(path) {
|
||||
return Self::open_unlocked(path);
|
||||
}
|
||||
let lock = match mode {
|
||||
obisys::LockMode::Shared => obisys::LockSlot::shared(path),
|
||||
obisys::LockMode::Exclusive => obisys::LockSlot::exclusive(path),
|
||||
}
|
||||
.map_err(OKIError::Io)?;
|
||||
let mut idx = Self::open_unlocked(path)?;
|
||||
idx.lock = lock;
|
||||
Ok(idx)
|
||||
}
|
||||
|
||||
/// Block until this index's directory lock is acquired, then hold it
|
||||
/// (released on [`release_lock`](Self::release_lock) or when `self` is
|
||||
/// dropped). Idempotent: a no-op, returning `true`, if this index
|
||||
/// already holds it (a second `DirLock` on the same directory would
|
||||
/// block on the first). Returns `false` only on a genuine I/O failure
|
||||
/// acquiring the lock file itself — contention blocks rather than
|
||||
/// failing.
|
||||
/// `open` without taking any lock — for a caller that already holds the
|
||||
/// directory through another handle (`finalize_indexed`, reopening the
|
||||
/// index its own builder holds exclusively).
|
||||
pub(crate) fn open_unlocked(path: &Path) -> OKIResult<Self> {
|
||||
let root_path = path.to_owned();
|
||||
let meta = IndexMeta::open_at(&root_path).map_err(OKIError::Io)?;
|
||||
set_k(meta.config.kmer_size);
|
||||
set_m(meta.config.minimizer_size);
|
||||
let lock = obisys::LockSlot::empty(&root_path);
|
||||
Ok(Self { root_path, meta: Arc::new(meta), lock })
|
||||
}
|
||||
|
||||
/// Take the exclusive directory lock and hold it until `self` is dropped
|
||||
/// — for an index just created, which is about to be written to.
|
||||
/// Idempotent. Returns `false` only on a genuine I/O failure.
|
||||
pub(crate) fn acquire_lock(&self) -> bool {
|
||||
let mut held = self.lock.lock().unwrap();
|
||||
if held.is_some() {
|
||||
return true;
|
||||
}
|
||||
match obisys::DirLock::acquire(&self.root_path) {
|
||||
Ok(lock) => {
|
||||
*held = Some(lock);
|
||||
true
|
||||
}
|
||||
Err(_) => false,
|
||||
}
|
||||
self.lock.acquire_exclusive().is_ok()
|
||||
}
|
||||
|
||||
/// Whether this index object currently holds its directory lock.
|
||||
pub fn holds_lock(&self) -> bool {
|
||||
self.lock.lock().unwrap().is_some()
|
||||
/// Whether this index object currently holds its directory lock
|
||||
/// exclusively (a shared lock is not "write access").
|
||||
pub fn holds_write_lock(&self) -> bool {
|
||||
self.lock.holds_write_lock()
|
||||
}
|
||||
|
||||
/// Run `f` with the directory lock held: as is if this index already
|
||||
/// holds it (opened with [`open_lock`](Self::open_lock), or an enclosing
|
||||
/// `with_lock`), otherwise taking it for the duration of `f` only — also
|
||||
/// released if `f` fails. Not for concurrent use from several threads of
|
||||
/// one process: the first one to finish would release the lock under
|
||||
/// the others.
|
||||
/// Run `f` with the directory lock held **exclusively** — promoting the
|
||||
/// shared lock a handle from [`open`](Self::open) holds, and restoring it
|
||||
/// afterwards; see [`obisys::LockSlot::with_exclusive`], including what
|
||||
/// `f` must re-check. Library code that writes into an index it was
|
||||
/// handed (annexes built on demand) wraps that write in this.
|
||||
pub fn with_lock<T>(&self, f: impl FnOnce() -> OKIResult<T>) -> OKIResult<T> {
|
||||
let was_held = self.holds_lock();
|
||||
if !was_held && !self.acquire_lock() {
|
||||
return Err(OKIError::Io(std::io::Error::other("failed to acquire index lock")));
|
||||
}
|
||||
let out = f();
|
||||
if !was_held {
|
||||
self.release_lock();
|
||||
}
|
||||
out
|
||||
self.lock.with_exclusive(f)
|
||||
}
|
||||
|
||||
/// Non-blocking probe of this index's directory lock — doesn't retain
|
||||
/// it either way: `true` if the lock was free (briefly acquired, then
|
||||
/// immediately released), `false` if another process currently holds
|
||||
/// it. For a read-only caller that wants to warn about a build in
|
||||
/// progress without itself becoming exclusive; use
|
||||
/// [`open_lock`](Self::open_lock)/[`with_lock`](Self::with_lock) to
|
||||
/// actually hold it.
|
||||
/// anything: `true` if no process currently holds it exclusively, `false`
|
||||
/// if one does (this process included). For a caller that wants to warn
|
||||
/// about a write in progress without waiting for it.
|
||||
pub fn test_lock(&self) -> bool {
|
||||
matches!(obisys::DirLock::try_acquire(&self.root_path), Ok(Some(_)))
|
||||
}
|
||||
|
||||
/// Release this index's directory lock, if held. A no-op if it isn't.
|
||||
pub(crate) fn release_lock(&self) {
|
||||
*self.lock.lock().unwrap() = None;
|
||||
self.lock.is_free_of_writer()
|
||||
}
|
||||
|
||||
/// Return `true` if `path` points to a valid obikmer index directory.
|
||||
|
||||
@@ -1,101 +0,0 @@
|
||||
//! The directory-lock API of [`KmerIndex`]: `open` never locks, `open_lock`
|
||||
//! holds for the object's lifetime, `with_lock` holds for a section and is a
|
||||
//! no-op when the lock is already held. A second handle on the directory
|
||||
//! (`test_lock`) is what tells held from free.
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use crate::index::error::OKIError;
|
||||
use crate::index::kmer_index::KmerIndex;
|
||||
use crate::index::meta::IndexConfig;
|
||||
use crate::layer::IndexMode;
|
||||
|
||||
/// Same k/m pair as the other tests of this crate (`obikseq`'s k/m are
|
||||
/// process-wide in test builds).
|
||||
fn index_dir() -> (tempfile::TempDir, PathBuf) {
|
||||
let tmp = tempfile::tempdir().expect("tempdir");
|
||||
let path = tmp.path().join("idx");
|
||||
let config = IndexConfig {
|
||||
kmer_size: 11,
|
||||
minimizer_size: 5,
|
||||
n_bits: 2,
|
||||
with_counts: false,
|
||||
evidence: IndexMode::Exact,
|
||||
block_bits: 0,
|
||||
};
|
||||
drop(KmerIndex::create(&path, config, None).expect("create index")); // lock released on drop
|
||||
(tmp, path)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn open_does_not_lock() {
|
||||
let (_tmp, path) = index_dir();
|
||||
let idx = KmerIndex::open(&path).unwrap();
|
||||
assert!(!idx.holds_lock());
|
||||
assert!(idx.test_lock(), "the directory must be free");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn open_lock_holds_until_dropped() {
|
||||
let (_tmp, path) = index_dir();
|
||||
let idx = KmerIndex::open_lock(&path).unwrap();
|
||||
assert!(idx.holds_lock());
|
||||
assert!(!idx.test_lock(), "another handle must see the directory locked");
|
||||
drop(idx);
|
||||
assert!(KmerIndex::open(&path).unwrap().test_lock(), "dropping releases the lock");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn open_lock_on_a_non_index_reports_it_and_leaves_no_lock_file() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
assert!(KmerIndex::open_lock(tmp.path()).is_err());
|
||||
assert!(!tmp.path().join(".obikmer.lock").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn with_lock_holds_for_the_section_only() {
|
||||
let (_tmp, path) = index_dir();
|
||||
let idx = KmerIndex::open(&path).unwrap();
|
||||
let out = idx
|
||||
.with_lock(|| {
|
||||
assert!(idx.holds_lock());
|
||||
assert!(!idx.test_lock());
|
||||
Ok(7)
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(out, 7);
|
||||
assert!(!idx.holds_lock());
|
||||
assert!(idx.test_lock());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn with_lock_releases_on_error() {
|
||||
let (_tmp, path) = index_dir();
|
||||
let idx = KmerIndex::open(&path).unwrap();
|
||||
let r: Result<(), _> = idx.with_lock(|| Err(OKIError::Io(std::io::Error::other("boom"))));
|
||||
assert!(r.is_err());
|
||||
assert!(!idx.holds_lock());
|
||||
assert!(idx.test_lock());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn with_lock_is_a_no_op_when_already_held() {
|
||||
let (_tmp, path) = index_dir();
|
||||
{
|
||||
// Opened with the lock: the section must neither block on itself nor release it.
|
||||
let idx = KmerIndex::open_lock(&path).unwrap();
|
||||
idx.with_lock(|| Ok(())).unwrap();
|
||||
assert!(idx.holds_lock());
|
||||
assert!(!idx.test_lock());
|
||||
}
|
||||
|
||||
// Nested sections: only the outermost one releases.
|
||||
let idx = KmerIndex::open(&path).unwrap();
|
||||
idx.with_lock(|| {
|
||||
idx.with_lock(|| Ok(()))?;
|
||||
assert!(idx.holds_lock(), "the inner section must not release the outer lock");
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
assert!(!idx.holds_lock());
|
||||
}
|
||||
@@ -3,8 +3,6 @@ pub mod meta;
|
||||
pub mod state;
|
||||
mod builder;
|
||||
mod kmer_index;
|
||||
#[cfg(test)]
|
||||
mod lock_tests;
|
||||
|
||||
pub use error::{OKIError, OKIResult};
|
||||
pub use builder::IndexBuilder;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "obikmer"
|
||||
version = "1.4.0"
|
||||
version = "1.4.1"
|
||||
edition = "2024"
|
||||
|
||||
[[bin]]
|
||||
|
||||
@@ -3,6 +3,9 @@
|
||||
|
||||
mod budget;
|
||||
mod lock;
|
||||
mod lock_slot;
|
||||
#[cfg(test)]
|
||||
mod lock_tests;
|
||||
mod numa;
|
||||
mod profiler;
|
||||
mod progress;
|
||||
@@ -10,7 +13,8 @@ mod resources;
|
||||
mod stage;
|
||||
|
||||
pub use budget::MemoryBudget;
|
||||
pub use lock::DirLock;
|
||||
pub use lock::{DirLock, LockMode};
|
||||
pub use lock_slot::LockSlot;
|
||||
pub use numa::PartitionRunner;
|
||||
pub use profiler::start as start_profiler;
|
||||
pub use progress::{Progress, TracedBar, progress_bar, spinner};
|
||||
|
||||
+188
-38
@@ -1,56 +1,206 @@
|
||||
use std::fs::File;
|
||||
use std::io;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
|
||||
use tracing::info;
|
||||
|
||||
/// Exclusive advisory lock on an index directory, held for the duration of
|
||||
/// any command that writes into an already-existing index (building the
|
||||
/// sibling annex, merging into a destination, filtering/selecting in place,
|
||||
/// ...). Two such commands racing on the same directory can otherwise
|
||||
/// corrupt each other's writes with no error from either side.
|
||||
/// Advisory readers/writer lock on an index directory, shared between
|
||||
/// processes (and between independent handles of one process).
|
||||
///
|
||||
/// Only the directory actually being *written to* needs a lock — a command
|
||||
/// like `merge` that reads several source indexes to build one destination
|
||||
/// only needs to lock the destination.
|
||||
/// - **Shared** ([`acquire_shared`](DirLock::acquire_shared)): held by
|
||||
/// anything that reads the index (`query`, `dump`, a `merge` source, a
|
||||
/// `phylo` run, ...). Any number of shared holders coexist.
|
||||
/// - **Exclusive** ([`acquire`](DirLock::acquire)): held by anything that
|
||||
/// writes into an already-existing index (building annexes, `convert`,
|
||||
/// `pack`, `annotate`, ...). It waits for every shared holder to leave and
|
||||
/// keeps others out while held. Two writers racing on one directory can
|
||||
/// otherwise corrupt each other's files with no error from either side,
|
||||
/// and a reader would see files vanish under it.
|
||||
///
|
||||
/// Uses the OS's advisory file lock (`flock` on Unix, `LockFileEx` on
|
||||
/// Windows) via `std::fs::File::lock`/`try_lock`, not a hand-rolled PID
|
||||
/// file: the OS releases it automatically on process exit, including a
|
||||
/// crash — no stale-lock cleanup logic needed.
|
||||
/// Only the directory actually being *written to* needs an exclusive lock —
|
||||
/// `merge` holds shared locks on its sources and an exclusive one on its
|
||||
/// destination.
|
||||
///
|
||||
/// # How it is built
|
||||
///
|
||||
/// Each mode is one OS advisory file lock (`flock` on Unix, `LockFileEx` on
|
||||
/// Windows) on `.obikmer.lock`, so a crash releases it with no stale-lock
|
||||
/// cleanup. On top of that, **every transition** (acquire, release into
|
||||
/// another mode, promote, demote) happens while holding a *gate* — a second
|
||||
/// lock file, `.obikmer.gate`, taken for the length of one non-blocking
|
||||
/// attempt and never while waiting. A blocked request therefore releases the
|
||||
/// gate, sleeps, and tries again (polling, 20 ms growing to 500 ms), instead
|
||||
/// of sleeping in the kernel while holding it. Because nobody else can touch
|
||||
/// the lock state while the gate is held, a shared → exclusive
|
||||
/// [promotion](DirLock::promote) cannot be overtaken by another writer, which
|
||||
/// the OS's own `flock` conversion does not guarantee.
|
||||
///
|
||||
/// There is no writer preference: a steady stream of readers can delay an
|
||||
/// exclusive request.
|
||||
#[derive(Debug)]
|
||||
pub struct DirLock {
|
||||
_file: std::fs::File,
|
||||
file: File,
|
||||
dir: PathBuf,
|
||||
mode: LockMode,
|
||||
}
|
||||
|
||||
impl DirLock {
|
||||
/// Block until the exclusive lock on `dir` is acquired (creating `dir`
|
||||
/// and the lock file within it if needed). Logs once if the wait is
|
||||
/// non-trivial, so a blocked command doesn't look silently hung.
|
||||
pub fn acquire(dir: &std::path::Path) -> std::io::Result<Self> {
|
||||
let file = Self::open_lock_file(dir)?;
|
||||
if file.try_lock().is_err() {
|
||||
info!(dir = %dir.display(), "waiting for another obikmer process to release this index");
|
||||
file.lock()?;
|
||||
}
|
||||
Ok(Self { _file: file })
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum LockMode {
|
||||
Shared,
|
||||
Exclusive,
|
||||
}
|
||||
|
||||
/// Non-blocking probe: `Ok(Some(lock))` if `dir`'s lock was free and is
|
||||
/// now held by the returned `DirLock`, `Ok(None)` if another process
|
||||
/// currently holds it (not an error — contention is the expected,
|
||||
/// common case this exists to distinguish from a genuine I/O failure).
|
||||
pub fn try_acquire(dir: &std::path::Path) -> std::io::Result<Option<Self>> {
|
||||
let file = Self::open_lock_file(dir)?;
|
||||
match file.try_lock() {
|
||||
Ok(()) => Ok(Some(Self { _file: file })),
|
||||
Err(_) => Ok(None),
|
||||
}
|
||||
}
|
||||
const LOCK_FILE: &str = ".obikmer.lock";
|
||||
const GATE_FILE: &str = ".obikmer.gate";
|
||||
const CLAIM_FILE: &str = ".obikmer.promote";
|
||||
const POLL_MIN: Duration = Duration::from_millis(20);
|
||||
const POLL_MAX: Duration = Duration::from_millis(500);
|
||||
|
||||
fn open_lock_file(dir: &std::path::Path) -> std::io::Result<std::fs::File> {
|
||||
fn open_file(dir: &Path, name: &str) -> io::Result<File> {
|
||||
std::fs::create_dir_all(dir)?;
|
||||
let lock_path = dir.join(".obikmer.lock");
|
||||
std::fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.truncate(false)
|
||||
.write(true)
|
||||
.open(&lock_path)
|
||||
.open(dir.join(name))
|
||||
}
|
||||
|
||||
/// The gate: blocks, but its holders only ever run instantaneous,
|
||||
/// non-blocking operations, so the wait is negligible. Released on drop.
|
||||
fn gate(dir: &Path) -> io::Result<File> {
|
||||
let f = open_file(dir, GATE_FILE)?;
|
||||
f.lock()?;
|
||||
Ok(f)
|
||||
}
|
||||
|
||||
/// Non-blocking attempt in `mode`; `Ok(false)` means "held by someone else".
|
||||
fn try_lock(file: &File, mode: LockMode) -> io::Result<bool> {
|
||||
let r = match mode {
|
||||
LockMode::Exclusive => file.try_lock(),
|
||||
LockMode::Shared => file.try_lock_shared(),
|
||||
};
|
||||
match r {
|
||||
Ok(()) => Ok(true),
|
||||
Err(std::fs::TryLockError::WouldBlock) => Ok(false),
|
||||
Err(std::fs::TryLockError::Error(e)) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
/// Sleeps between two attempts and logs once, at the first one.
|
||||
struct Backoff {
|
||||
delay: Duration,
|
||||
logged: bool,
|
||||
}
|
||||
|
||||
impl Backoff {
|
||||
fn new() -> Self {
|
||||
Self { delay: POLL_MIN, logged: false }
|
||||
}
|
||||
|
||||
fn wait(&mut self, dir: &Path, why: &str) {
|
||||
if !self.logged {
|
||||
info!(dir = %dir.display(), "{why}");
|
||||
self.logged = true;
|
||||
}
|
||||
std::thread::sleep(self.delay);
|
||||
self.delay = (self.delay * 2).min(POLL_MAX);
|
||||
}
|
||||
}
|
||||
|
||||
impl DirLock {
|
||||
/// Block until the exclusive lock on `dir` is acquired (creating `dir`
|
||||
/// and the lock files within it if needed). Logs once if the wait is
|
||||
/// non-trivial, so a blocked command doesn't look silently hung.
|
||||
pub fn acquire(dir: &Path) -> io::Result<Self> {
|
||||
Self::acquire_mode(dir, LockMode::Exclusive, "waiting for other obikmer processes to release this index")
|
||||
}
|
||||
|
||||
/// Block until a shared lock on `dir` is acquired — until no exclusive
|
||||
/// holder remains. Logs once if the wait is non-trivial.
|
||||
pub fn acquire_shared(dir: &Path) -> io::Result<Self> {
|
||||
Self::acquire_mode(dir, LockMode::Shared, "waiting for an obikmer process writing this index")
|
||||
}
|
||||
|
||||
fn acquire_mode(dir: &Path, mode: LockMode, why: &str) -> io::Result<Self> {
|
||||
let file = open_file(dir, LOCK_FILE)?;
|
||||
let mut backoff = Backoff::new();
|
||||
loop {
|
||||
{
|
||||
let _gate = gate(dir)?;
|
||||
if try_lock(&file, mode)? {
|
||||
return Ok(Self { file, dir: dir.to_owned(), mode });
|
||||
}
|
||||
}
|
||||
backoff.wait(dir, why);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn mode(&self) -> LockMode {
|
||||
self.mode
|
||||
}
|
||||
|
||||
/// Non-blocking probe: `Ok(true)` if no exclusive holder exists right
|
||||
/// now (a shared lock could be taken), `Ok(false)` if one does. Retains
|
||||
/// nothing either way.
|
||||
pub fn is_free_of_writer(dir: &Path) -> io::Result<bool> {
|
||||
let file = open_file(dir, LOCK_FILE)?;
|
||||
let _gate = gate(dir)?;
|
||||
try_lock(&file, LockMode::Shared)
|
||||
}
|
||||
|
||||
/// Shared → exclusive. Returns the exclusive lock and whether the
|
||||
/// promotion was **gapless**: `true` means the lock was never released,
|
||||
/// so nothing in the directory can have changed since the shared lock
|
||||
/// was held and whatever the caller read stays valid; `false` means the
|
||||
/// shared lock had to be given up while waiting, so a writer may have
|
||||
/// run in between and the caller must re-check what it relied on.
|
||||
///
|
||||
/// Gapless promotion is available to one promoter at a time (it holds
|
||||
/// `.obikmer.promote` while it waits for the other readers to leave).
|
||||
/// A second concurrent promoter releases its shared lock instead — two
|
||||
/// promoters each keeping theirs while waiting for the other's would
|
||||
/// wait for ever. An exclusive lock is returned unchanged.
|
||||
pub fn promote(self) -> io::Result<(Self, bool)> {
|
||||
if self.mode == LockMode::Exclusive {
|
||||
return Ok((self, true));
|
||||
}
|
||||
let claim = open_file(&self.dir, CLAIM_FILE)?;
|
||||
if !try_lock(&claim, LockMode::Exclusive)? {
|
||||
let dir = self.dir.clone();
|
||||
drop(self); // gives the shared lock up
|
||||
return Ok((Self::acquire(&dir)?, false));
|
||||
}
|
||||
let mut backoff = Backoff::new();
|
||||
loop {
|
||||
{
|
||||
let _gate = gate(&self.dir)?;
|
||||
self.file.unlock()?;
|
||||
if try_lock(&self.file, LockMode::Exclusive)? {
|
||||
return Ok((Self { mode: LockMode::Exclusive, ..self }, true));
|
||||
}
|
||||
// Other readers remain. Nobody can hold the lock exclusively
|
||||
// without the gate, so taking our shared lock back succeeds.
|
||||
if !try_lock(&self.file, LockMode::Shared)? {
|
||||
return Err(io::Error::other("lost the shared index lock during promotion"));
|
||||
}
|
||||
}
|
||||
backoff.wait(&self.dir, "waiting for other readers of this index before writing to it");
|
||||
}
|
||||
}
|
||||
|
||||
/// Exclusive → shared, without ever leaving the lock free in between.
|
||||
/// A shared lock is returned unchanged.
|
||||
pub fn demote(self) -> io::Result<Self> {
|
||||
if self.mode == LockMode::Shared {
|
||||
return Ok(self);
|
||||
}
|
||||
let held_gate = gate(&self.dir)?;
|
||||
self.file.unlock()?;
|
||||
if !try_lock(&self.file, LockMode::Shared)? {
|
||||
return Err(io::Error::other("could not take the shared index lock back after writing"));
|
||||
}
|
||||
drop(held_gate);
|
||||
Ok(Self { mode: LockMode::Shared, ..self })
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
use std::io;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Mutex;
|
||||
|
||||
use crate::lock::{DirLock, LockMode};
|
||||
|
||||
/// The directory lock of one handle on an index: nothing, shared or
|
||||
/// exclusive, and the transitions between them that a handle needs. What
|
||||
/// `obikindex::KmerIndex` delegates its locking to, kept apart so that it can
|
||||
/// be tested without building an index.
|
||||
///
|
||||
/// Interior mutability (`&self` methods): a handle is shared behind `Arc`.
|
||||
/// Not for concurrent use from several threads of one handle — the first
|
||||
/// section to finish would restore the lock under the others.
|
||||
#[derive(Debug)]
|
||||
pub struct LockSlot {
|
||||
dir: PathBuf,
|
||||
held: Mutex<Option<DirLock>>,
|
||||
}
|
||||
|
||||
impl LockSlot {
|
||||
/// A slot holding nothing (a handle that took no lock, or one that
|
||||
/// reopens a directory its own builder already holds).
|
||||
pub fn empty(dir: &Path) -> Self {
|
||||
Self { dir: dir.to_owned(), held: Mutex::new(None) }
|
||||
}
|
||||
|
||||
/// A slot holding a shared lock on `dir`; blocks while a writer is at work.
|
||||
pub fn shared(dir: &Path) -> io::Result<Self> {
|
||||
Ok(Self { dir: dir.to_owned(), held: Mutex::new(Some(DirLock::acquire_shared(dir)?)) })
|
||||
}
|
||||
|
||||
/// A slot holding the exclusive lock on `dir`; blocks while anyone else holds it.
|
||||
pub fn exclusive(dir: &Path) -> io::Result<Self> {
|
||||
Ok(Self { dir: dir.to_owned(), held: Mutex::new(Some(DirLock::acquire(dir)?)) })
|
||||
}
|
||||
|
||||
/// Whether the slot holds the exclusive lock (a shared one is not write access).
|
||||
pub fn holds_write_lock(&self) -> bool {
|
||||
matches!(&*self.held.lock().unwrap(), Some(l) if l.mode() == LockMode::Exclusive)
|
||||
}
|
||||
|
||||
/// Hold the exclusive lock from now on: a no-op if already held that way,
|
||||
/// a promotion of a shared lock, a plain acquisition of an empty slot.
|
||||
pub fn acquire_exclusive(&self) -> io::Result<()> {
|
||||
let held = self.held.lock().unwrap().take();
|
||||
let lock = match held {
|
||||
Some(l) => l.promote()?.0,
|
||||
None => DirLock::acquire(&self.dir)?,
|
||||
};
|
||||
*self.held.lock().unwrap() = Some(lock);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Run `f` with the exclusive lock held:
|
||||
/// - already exclusive: `f` runs as is (so sections nest);
|
||||
/// - shared: promoted for the duration of `f`, then demoted back, so the
|
||||
/// slot keeps the shared lock it came with. The promotion is gapless
|
||||
/// when no other process promotes at the same time; otherwise other
|
||||
/// writers may run between the shared lock being given up and the
|
||||
/// exclusive one being taken. Either way **`f` must re-check, before
|
||||
/// building, what it is about to build**, and the caller must not hold
|
||||
/// anything read earlier that a writer could have replaced;
|
||||
/// - empty: the lock is taken for `f` only.
|
||||
///
|
||||
/// Restored also if `f` fails; an I/O error taking or restoring the lock
|
||||
/// is converted to `E`.
|
||||
pub fn with_exclusive<T, E: From<io::Error>>(&self, f: impl FnOnce() -> Result<T, E>) -> Result<T, E> {
|
||||
enum Before {
|
||||
Exclusive,
|
||||
Shared,
|
||||
Nothing,
|
||||
}
|
||||
let held = self.held.lock().unwrap().take();
|
||||
let (lock, before) = match held {
|
||||
Some(l) if l.mode() == LockMode::Exclusive => (l, Before::Exclusive),
|
||||
Some(l) => (l.promote()?.0, Before::Shared),
|
||||
None => (DirLock::acquire(&self.dir)?, Before::Nothing),
|
||||
};
|
||||
*self.held.lock().unwrap() = Some(lock);
|
||||
let out = f();
|
||||
let lock = self.held.lock().unwrap().take();
|
||||
match (before, lock) {
|
||||
(Before::Exclusive, lock) => *self.held.lock().unwrap() = lock,
|
||||
(Before::Shared, Some(l)) => *self.held.lock().unwrap() = Some(l.demote()?),
|
||||
_ => {} // taken for `f` only: released here
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Non-blocking probe: `true` if no process holds the directory
|
||||
/// exclusively right now (this slot included), `false` if one does.
|
||||
pub fn is_free_of_writer(&self) -> bool {
|
||||
DirLock::is_free_of_writer(&self.dir).unwrap_or(false)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
//! The directory lock is a readers/writer lock: `shared` slots coexist, an
|
||||
//! `exclusive` one waits for them all and keeps new ones out, and a shared
|
||||
//! slot can be promoted for a section (`with_exclusive`) and demoted back.
|
||||
//! No index is involved, and therefore none of the process-wide k-mer
|
||||
//! parameters other crates' tests share.
|
||||
|
||||
use std::io;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::mpsc;
|
||||
use std::sync::{Arc, Barrier};
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::lock::{DirLock, LockMode};
|
||||
use crate::lock_slot::LockSlot;
|
||||
|
||||
const SETTLE: Duration = Duration::from_millis(400);
|
||||
const PATIENCE: Duration = Duration::from_secs(10);
|
||||
|
||||
/// A scratch directory removed on drop (no `tempfile` dependency here).
|
||||
struct Scratch(PathBuf);
|
||||
|
||||
impl Scratch {
|
||||
fn new() -> Self {
|
||||
static N: AtomicUsize = AtomicUsize::new(0);
|
||||
let dir = std::env::temp_dir().join(format!(
|
||||
"obisys-lock-{}-{}",
|
||||
std::process::id(),
|
||||
N.fetch_add(1, Ordering::Relaxed)
|
||||
));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
Self(dir)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Scratch {
|
||||
fn drop(&mut self) {
|
||||
let _ = std::fs::remove_dir_all(&self.0);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn readers_coexist() {
|
||||
let d = Scratch::new();
|
||||
let a = LockSlot::shared(&d.0).unwrap();
|
||||
let b = LockSlot::shared(&d.0).unwrap();
|
||||
assert!(!a.holds_write_lock() && !b.holds_write_lock());
|
||||
assert!(a.is_free_of_writer(), "readers are not writers");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exclusive_is_exclusive_and_released_on_drop() {
|
||||
let d = Scratch::new();
|
||||
let w = LockSlot::exclusive(&d.0).unwrap();
|
||||
assert!(w.holds_write_lock());
|
||||
assert!(!w.is_free_of_writer());
|
||||
drop(w);
|
||||
assert!(LockSlot::empty(&d.0).is_free_of_writer());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_writer_waits_for_the_readers() {
|
||||
let d = Scratch::new();
|
||||
let reader = LockSlot::shared(&d.0).unwrap();
|
||||
let (tx, rx) = mpsc::channel();
|
||||
let dir = d.0.clone();
|
||||
let writer = thread::spawn(move || {
|
||||
drop(LockSlot::exclusive(&dir).unwrap());
|
||||
tx.send(()).unwrap();
|
||||
});
|
||||
assert!(rx.recv_timeout(SETTLE).is_err(), "the writer must wait for the reader");
|
||||
drop(reader);
|
||||
rx.recv_timeout(PATIENCE).expect("the writer proceeds once the reader is gone");
|
||||
writer.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_reader_waits_for_the_writer() {
|
||||
let d = Scratch::new();
|
||||
let writer = LockSlot::exclusive(&d.0).unwrap();
|
||||
let (tx, rx) = mpsc::channel();
|
||||
let dir = d.0.clone();
|
||||
let reader = thread::spawn(move || {
|
||||
drop(LockSlot::shared(&dir).unwrap());
|
||||
tx.send(()).unwrap();
|
||||
});
|
||||
assert!(rx.recv_timeout(SETTLE).is_err(), "the reader must wait for the writer");
|
||||
drop(writer);
|
||||
rx.recv_timeout(PATIENCE).expect("the reader proceeds once the writer is gone");
|
||||
reader.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn with_exclusive_on_a_reader_promotes_then_restores_the_shared_lock() {
|
||||
let d = Scratch::new();
|
||||
let slot = LockSlot::shared(&d.0).unwrap();
|
||||
let out: io::Result<i32> = slot.with_exclusive(|| {
|
||||
assert!(slot.holds_write_lock());
|
||||
assert!(!slot.is_free_of_writer());
|
||||
Ok(7)
|
||||
});
|
||||
assert_eq!(out.unwrap(), 7);
|
||||
assert!(!slot.holds_write_lock());
|
||||
assert!(slot.is_free_of_writer());
|
||||
|
||||
// Still a reader: a writer elsewhere waits for it.
|
||||
let (tx, rx) = mpsc::channel();
|
||||
let dir = d.0.clone();
|
||||
let writer = thread::spawn(move || {
|
||||
drop(LockSlot::exclusive(&dir).unwrap());
|
||||
tx.send(()).unwrap();
|
||||
});
|
||||
assert!(rx.recv_timeout(SETTLE).is_err(), "the slot must be a reader again");
|
||||
drop(slot);
|
||||
rx.recv_timeout(PATIENCE).unwrap();
|
||||
writer.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn with_exclusive_on_an_empty_slot_holds_the_lock_for_the_section_only() {
|
||||
let d = Scratch::new();
|
||||
let slot = LockSlot::empty(&d.0);
|
||||
slot.with_exclusive::<(), io::Error>(|| {
|
||||
assert!(slot.holds_write_lock());
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
assert!(!slot.holds_write_lock());
|
||||
assert!(slot.is_free_of_writer());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn with_exclusive_restores_on_error() {
|
||||
let d = Scratch::new();
|
||||
let slot = LockSlot::shared(&d.0).unwrap();
|
||||
let r: io::Result<()> = slot.with_exclusive(|| Err(io::Error::other("boom")));
|
||||
assert!(r.is_err());
|
||||
assert!(!slot.holds_write_lock());
|
||||
assert!(slot.is_free_of_writer());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn with_exclusive_nests_and_keeps_a_lock_taken_at_open() {
|
||||
let d = Scratch::new();
|
||||
let slot = LockSlot::exclusive(&d.0).unwrap();
|
||||
slot.with_exclusive::<(), io::Error>(|| {
|
||||
slot.with_exclusive::<(), io::Error>(|| Ok(()))?;
|
||||
assert!(slot.holds_write_lock(), "an inner section must not release the outer lock");
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
assert!(slot.holds_write_lock(), "a slot opened exclusive keeps it");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn acquire_exclusive_promotes_a_shared_slot_and_is_idempotent() {
|
||||
let d = Scratch::new();
|
||||
let slot = LockSlot::shared(&d.0).unwrap();
|
||||
slot.acquire_exclusive().unwrap();
|
||||
slot.acquire_exclusive().unwrap();
|
||||
assert!(slot.holds_write_lock());
|
||||
}
|
||||
|
||||
/// The case a naive upgrade deadlocks on: two readers, each holding a shared
|
||||
/// lock, both ask for the exclusive one at the same time.
|
||||
#[test]
|
||||
fn two_readers_promoting_at_once_both_finish() {
|
||||
let d = Scratch::new();
|
||||
let barrier = Arc::new(Barrier::new(2));
|
||||
let (tx, rx) = mpsc::channel();
|
||||
let handles: Vec<_> = (0..2)
|
||||
.map(|_| {
|
||||
let (dir, b, tx) = (d.0.clone(), Arc::clone(&barrier), tx.clone());
|
||||
thread::spawn(move || {
|
||||
let slot = LockSlot::shared(&dir).unwrap();
|
||||
b.wait(); // both hold a shared lock
|
||||
slot.with_exclusive::<(), io::Error>(|| {
|
||||
assert!(slot.holds_write_lock());
|
||||
thread::sleep(Duration::from_millis(50));
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
tx.send(()).unwrap();
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
for _ in 0..2 {
|
||||
rx.recv_timeout(PATIENCE).expect("a promotion deadlocked");
|
||||
}
|
||||
for h in handles {
|
||||
h.join().unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_lone_promotion_is_gapless_and_a_demotion_keeps_the_lock() {
|
||||
let d = Scratch::new();
|
||||
let shared = DirLock::acquire_shared(&d.0).unwrap();
|
||||
let (excl, gapless) = shared.promote().unwrap();
|
||||
assert_eq!(excl.mode(), LockMode::Exclusive);
|
||||
assert!(gapless, "nobody else was promoting: the lock was never released");
|
||||
let shared = excl.demote().unwrap();
|
||||
assert_eq!(shared.mode(), LockMode::Shared);
|
||||
// Shared again: other readers get in, a writer does not.
|
||||
let _other = DirLock::acquire_shared(&d.0).unwrap();
|
||||
assert!(DirLock::is_free_of_writer(&d.0).unwrap());
|
||||
}
|
||||
Reference in new issue
Block a user