Introduces a robust, OS-level directory locking mechanism using LockSlot for managing shared and exclusive access, supporting lock transitions and non-blocking probes. Also sets up the index structure with new builder and index modules.
207 lines
7.8 KiB
Rust
207 lines
7.8 KiB
Rust
use std::fs::File;
|
|
use std::io;
|
|
use std::path::{Path, PathBuf};
|
|
use std::time::Duration;
|
|
|
|
use tracing::info;
|
|
|
|
/// Advisory readers/writer lock on an index directory, shared between
|
|
/// processes (and between independent handles of one process).
|
|
///
|
|
/// - **Shared** ([`acquire_shared`](DirLock::acquire_shared)): held by
|
|
/// anything that reads the index (`query`, `dump`, a `merge` source, a
|
|
/// `phylo` run, ...). Any number of shared holders coexist.
|
|
/// - **Exclusive** ([`acquire`](DirLock::acquire)): held by anything that
|
|
/// writes into an already-existing index (building annexes, `convert`,
|
|
/// `pack`, `annotate`, ...). It waits for every shared holder to leave and
|
|
/// keeps others out while held. Two writers racing on one directory can
|
|
/// otherwise corrupt each other's files with no error from either side,
|
|
/// and a reader would see files vanish under it.
|
|
///
|
|
/// Only the directory actually being *written to* needs an exclusive lock —
|
|
/// `merge` holds shared locks on its sources and an exclusive one on its
|
|
/// destination.
|
|
///
|
|
/// # How it is built
|
|
///
|
|
/// Each mode is one OS advisory file lock (`flock` on Unix, `LockFileEx` on
|
|
/// Windows) on `.obikmer.lock`, so a crash releases it with no stale-lock
|
|
/// cleanup. On top of that, **every transition** (acquire, release into
|
|
/// another mode, promote, demote) happens while holding a *gate* — a second
|
|
/// lock file, `.obikmer.gate`, taken for the length of one non-blocking
|
|
/// attempt and never while waiting. A blocked request therefore releases the
|
|
/// gate, sleeps, and tries again (polling, 20 ms growing to 500 ms), instead
|
|
/// of sleeping in the kernel while holding it. Because nobody else can touch
|
|
/// the lock state while the gate is held, a shared → exclusive
|
|
/// [promotion](DirLock::promote) cannot be overtaken by another writer, which
|
|
/// the OS's own `flock` conversion does not guarantee.
|
|
///
|
|
/// There is no writer preference: a steady stream of readers can delay an
|
|
/// exclusive request.
|
|
#[derive(Debug)]
|
|
pub struct DirLock {
|
|
file: File,
|
|
dir: PathBuf,
|
|
mode: LockMode,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub enum LockMode {
|
|
Shared,
|
|
Exclusive,
|
|
}
|
|
|
|
const LOCK_FILE: &str = ".obikmer.lock";
|
|
const GATE_FILE: &str = ".obikmer.gate";
|
|
const CLAIM_FILE: &str = ".obikmer.promote";
|
|
const POLL_MIN: Duration = Duration::from_millis(20);
|
|
const POLL_MAX: Duration = Duration::from_millis(500);
|
|
|
|
fn open_file(dir: &Path, name: &str) -> io::Result<File> {
|
|
std::fs::create_dir_all(dir)?;
|
|
std::fs::OpenOptions::new()
|
|
.create(true)
|
|
.truncate(false)
|
|
.write(true)
|
|
.open(dir.join(name))
|
|
}
|
|
|
|
/// The gate: blocks, but its holders only ever run instantaneous,
|
|
/// non-blocking operations, so the wait is negligible. Released on drop.
|
|
fn gate(dir: &Path) -> io::Result<File> {
|
|
let f = open_file(dir, GATE_FILE)?;
|
|
f.lock()?;
|
|
Ok(f)
|
|
}
|
|
|
|
/// Non-blocking attempt in `mode`; `Ok(false)` means "held by someone else".
|
|
fn try_lock(file: &File, mode: LockMode) -> io::Result<bool> {
|
|
let r = match mode {
|
|
LockMode::Exclusive => file.try_lock(),
|
|
LockMode::Shared => file.try_lock_shared(),
|
|
};
|
|
match r {
|
|
Ok(()) => Ok(true),
|
|
Err(std::fs::TryLockError::WouldBlock) => Ok(false),
|
|
Err(std::fs::TryLockError::Error(e)) => Err(e),
|
|
}
|
|
}
|
|
|
|
/// Sleeps between two attempts and logs once, at the first one.
|
|
struct Backoff {
|
|
delay: Duration,
|
|
logged: bool,
|
|
}
|
|
|
|
impl Backoff {
|
|
fn new() -> Self {
|
|
Self { delay: POLL_MIN, logged: false }
|
|
}
|
|
|
|
fn wait(&mut self, dir: &Path, why: &str) {
|
|
if !self.logged {
|
|
info!(dir = %dir.display(), "{why}");
|
|
self.logged = true;
|
|
}
|
|
std::thread::sleep(self.delay);
|
|
self.delay = (self.delay * 2).min(POLL_MAX);
|
|
}
|
|
}
|
|
|
|
impl DirLock {
|
|
/// Block until the exclusive lock on `dir` is acquired (creating `dir`
|
|
/// and the lock files within it if needed). Logs once if the wait is
|
|
/// non-trivial, so a blocked command doesn't look silently hung.
|
|
pub fn acquire(dir: &Path) -> io::Result<Self> {
|
|
Self::acquire_mode(dir, LockMode::Exclusive, "waiting for other obikmer processes to release this index")
|
|
}
|
|
|
|
/// Block until a shared lock on `dir` is acquired — until no exclusive
|
|
/// holder remains. Logs once if the wait is non-trivial.
|
|
pub fn acquire_shared(dir: &Path) -> io::Result<Self> {
|
|
Self::acquire_mode(dir, LockMode::Shared, "waiting for an obikmer process writing this index")
|
|
}
|
|
|
|
fn acquire_mode(dir: &Path, mode: LockMode, why: &str) -> io::Result<Self> {
|
|
let file = open_file(dir, LOCK_FILE)?;
|
|
let mut backoff = Backoff::new();
|
|
loop {
|
|
{
|
|
let _gate = gate(dir)?;
|
|
if try_lock(&file, mode)? {
|
|
return Ok(Self { file, dir: dir.to_owned(), mode });
|
|
}
|
|
}
|
|
backoff.wait(dir, why);
|
|
}
|
|
}
|
|
|
|
pub fn mode(&self) -> LockMode {
|
|
self.mode
|
|
}
|
|
|
|
/// Non-blocking probe: `Ok(true)` if no exclusive holder exists right
|
|
/// now (a shared lock could be taken), `Ok(false)` if one does. Retains
|
|
/// nothing either way.
|
|
pub fn is_free_of_writer(dir: &Path) -> io::Result<bool> {
|
|
let file = open_file(dir, LOCK_FILE)?;
|
|
let _gate = gate(dir)?;
|
|
try_lock(&file, LockMode::Shared)
|
|
}
|
|
|
|
/// Shared → exclusive. Returns the exclusive lock and whether the
|
|
/// promotion was **gapless**: `true` means the lock was never released,
|
|
/// so nothing in the directory can have changed since the shared lock
|
|
/// was held and whatever the caller read stays valid; `false` means the
|
|
/// shared lock had to be given up while waiting, so a writer may have
|
|
/// run in between and the caller must re-check what it relied on.
|
|
///
|
|
/// Gapless promotion is available to one promoter at a time (it holds
|
|
/// `.obikmer.promote` while it waits for the other readers to leave).
|
|
/// A second concurrent promoter releases its shared lock instead — two
|
|
/// promoters each keeping theirs while waiting for the other's would
|
|
/// wait for ever. An exclusive lock is returned unchanged.
|
|
pub fn promote(self) -> io::Result<(Self, bool)> {
|
|
if self.mode == LockMode::Exclusive {
|
|
return Ok((self, true));
|
|
}
|
|
let claim = open_file(&self.dir, CLAIM_FILE)?;
|
|
if !try_lock(&claim, LockMode::Exclusive)? {
|
|
let dir = self.dir.clone();
|
|
drop(self); // gives the shared lock up
|
|
return Ok((Self::acquire(&dir)?, false));
|
|
}
|
|
let mut backoff = Backoff::new();
|
|
loop {
|
|
{
|
|
let _gate = gate(&self.dir)?;
|
|
self.file.unlock()?;
|
|
if try_lock(&self.file, LockMode::Exclusive)? {
|
|
return Ok((Self { mode: LockMode::Exclusive, ..self }, true));
|
|
}
|
|
// Other readers remain. Nobody can hold the lock exclusively
|
|
// without the gate, so taking our shared lock back succeeds.
|
|
if !try_lock(&self.file, LockMode::Shared)? {
|
|
return Err(io::Error::other("lost the shared index lock during promotion"));
|
|
}
|
|
}
|
|
backoff.wait(&self.dir, "waiting for other readers of this index before writing to it");
|
|
}
|
|
}
|
|
|
|
/// Exclusive → shared, without ever leaving the lock free in between.
|
|
/// A shared lock is returned unchanged.
|
|
pub fn demote(self) -> io::Result<Self> {
|
|
if self.mode == LockMode::Shared {
|
|
return Ok(self);
|
|
}
|
|
let held_gate = gate(&self.dir)?;
|
|
self.file.unlock()?;
|
|
if !try_lock(&self.file, LockMode::Shared)? {
|
|
return Err(io::Error::other("could not take the shared index lock back after writing"));
|
|
}
|
|
drop(held_gate);
|
|
Ok(Self { mode: LockMode::Shared, ..self })
|
|
}
|
|
}
|