Push ltpzonnnkpqs #79

Merged
coissac merged 3 commits from push-ltpzonnnkpqs into main 2026-10-16 14:43:51 +00:00
11 changed files with 242 additions and 72 deletions
Showing only changes of commit 9ac3a37529 - Show all commits

No files matched your search

+58 -17
View File
@@ -56,7 +56,7 @@ impl KmerIndex {
/// access (resuming an interrupted build) uses
/// [`open_lock`](Self::open_lock) instead; either way, `self.lock`
/// starts empty — [`test_lock`](Self::test_lock)/
/// [`acquire_lock`](Self::acquire_lock) are opt-in from here.
/// [`with_lock`](Self::with_lock) are opt-in from here.
pub fn open<P: AsRef<Path>>(path: P) -> OKIResult<Self> {
let root_path = path.as_ref().to_owned();
let meta = IndexMeta::open_at(&root_path).map_err(OKIError::Io)?;
@@ -69,47 +69,88 @@ impl KmerIndex {
})
}
/// `open`, then hold the directory lock exclusively — for a caller
/// that resumes a write pipeline (e.g. `obikmer index` re-run on an
/// existing, not-yet-`Indexed` directory) rather than reading
/// concurrently with others.
/// `open`, holding the directory lock exclusively for as long as the
/// returned index lives — **the** way for a command that modifies an
/// existing index in place (`convert`, `pack`, `annotate`, `utils
/// rename`, resuming an interrupted build, ...) to claim it. The lock is
/// taken *before* the metadata is read, so what the caller reads is what
/// it will modify. A directory that is not an index is reported by
/// `open`, without a lock file being created in it.
///
/// A command that only reads, or that writes a *new* index elsewhere
/// (`merge`, `filter`, `select`: they lock their destination, not their
/// sources), uses [`open`](Self::open). Library code that has to write
/// into an index it was handed (annexes built on demand) wraps that
/// write in [`with_lock`](Self::with_lock), which does nothing more when
/// the lock is already held.
pub fn open_lock<P: AsRef<Path>>(path: P) -> OKIResult<Self> {
let idx = Self::open(path)?;
if !idx.acquire_lock() {
return Err(OKIError::Io(std::io::Error::other(
"failed to acquire index lock",
)));
let path = path.as_ref();
if !Self::is_an_index(path) {
return Self::open(path);
}
let lock = obisys::DirLock::acquire(path).map_err(OKIError::Io)?;
let idx = Self::open(path)?;
*idx.lock.lock().unwrap() = Some(lock);
Ok(idx)
}
/// Block until this index's directory lock is acquired, then hold it
/// (released on [`release_lock`](Self::release_lock) or when `self` is
/// dropped). Returns `false` only on a genuine I/O failure acquiring
/// the lock file itself — contention blocks rather than failing.
pub fn acquire_lock(&self) -> bool {
/// dropped). Idempotent: a no-op, returning `true`, if this index
/// already holds it (a second `DirLock` on the same directory would
/// block on the first). Returns `false` only on a genuine I/O failure
/// acquiring the lock file itself — contention blocks rather than
/// failing.
pub(crate) fn acquire_lock(&self) -> bool {
let mut held = self.lock.lock().unwrap();
if held.is_some() {
return true;
}
match obisys::DirLock::acquire(&self.root_path) {
Ok(lock) => {
*self.lock.lock().unwrap() = Some(lock);
*held = Some(lock);
true
}
Err(_) => false,
}
}
/// Whether this index object currently holds its directory lock.
pub fn holds_lock(&self) -> bool {
self.lock.lock().unwrap().is_some()
}
/// Run `f` with the directory lock held: as is if this index already
/// holds it (opened with [`open_lock`](Self::open_lock), or an enclosing
/// `with_lock`), otherwise taking it for the duration of `f` only — also
/// released if `f` fails. Not for concurrent use from several threads of
/// one process: the first one to finish would release the lock under
/// the others.
pub fn with_lock<T>(&self, f: impl FnOnce() -> OKIResult<T>) -> OKIResult<T> {
let was_held = self.holds_lock();
if !was_held && !self.acquire_lock() {
return Err(OKIError::Io(std::io::Error::other("failed to acquire index lock")));
}
let out = f();
if !was_held {
self.release_lock();
}
out
}
/// Non-blocking probe of this index's directory lock — doesn't retain
/// it either way: `true` if the lock was free (briefly acquired, then
/// immediately released), `false` if another process currently holds
/// it. For a read-only caller that wants to warn about a build in
/// progress without itself becoming exclusive; use
/// [`acquire_lock`](Self::acquire_lock)/[`open_lock`](Self::open_lock)
/// to actually hold it.
/// [`open_lock`](Self::open_lock)/[`with_lock`](Self::with_lock) to
/// actually hold it.
pub fn test_lock(&self) -> bool {
matches!(obisys::DirLock::try_acquire(&self.root_path), Ok(Some(_)))
}
/// Release this index's directory lock, if held. A no-op if it isn't.
pub fn release_lock(&self) {
pub(crate) fn release_lock(&self) {
*self.lock.lock().unwrap() = None;
}
+101
View File
@@ -0,0 +1,101 @@
//! The directory-lock API of [`KmerIndex`]: `open` never locks, `open_lock`
//! holds for the object's lifetime, `with_lock` holds for a section and is a
//! no-op when the lock is already held. A second handle on the directory
//! (`test_lock`) is what tells held from free.
use std::path::PathBuf;
use crate::index::error::OKIError;
use crate::index::kmer_index::KmerIndex;
use crate::index::meta::IndexConfig;
use crate::layer::IndexMode;
/// Same k/m pair as the other tests of this crate (`obikseq`'s k/m are
/// process-wide in test builds).
fn index_dir() -> (tempfile::TempDir, PathBuf) {
let tmp = tempfile::tempdir().expect("tempdir");
let path = tmp.path().join("idx");
let config = IndexConfig {
kmer_size: 11,
minimizer_size: 5,
n_bits: 2,
with_counts: false,
evidence: IndexMode::Exact,
block_bits: 0,
};
drop(KmerIndex::create(&path, config, None).expect("create index")); // lock released on drop
(tmp, path)
}
#[test]
fn open_does_not_lock() {
let (_tmp, path) = index_dir();
let idx = KmerIndex::open(&path).unwrap();
assert!(!idx.holds_lock());
assert!(idx.test_lock(), "the directory must be free");
}
#[test]
fn open_lock_holds_until_dropped() {
let (_tmp, path) = index_dir();
let idx = KmerIndex::open_lock(&path).unwrap();
assert!(idx.holds_lock());
assert!(!idx.test_lock(), "another handle must see the directory locked");
drop(idx);
assert!(KmerIndex::open(&path).unwrap().test_lock(), "dropping releases the lock");
}
#[test]
fn open_lock_on_a_non_index_reports_it_and_leaves_no_lock_file() {
let tmp = tempfile::tempdir().unwrap();
assert!(KmerIndex::open_lock(tmp.path()).is_err());
assert!(!tmp.path().join(".obikmer.lock").exists());
}
#[test]
fn with_lock_holds_for_the_section_only() {
let (_tmp, path) = index_dir();
let idx = KmerIndex::open(&path).unwrap();
let out = idx
.with_lock(|| {
assert!(idx.holds_lock());
assert!(!idx.test_lock());
Ok(7)
})
.unwrap();
assert_eq!(out, 7);
assert!(!idx.holds_lock());
assert!(idx.test_lock());
}
#[test]
fn with_lock_releases_on_error() {
let (_tmp, path) = index_dir();
let idx = KmerIndex::open(&path).unwrap();
let r: Result<(), _> = idx.with_lock(|| Err(OKIError::Io(std::io::Error::other("boom"))));
assert!(r.is_err());
assert!(!idx.holds_lock());
assert!(idx.test_lock());
}
#[test]
fn with_lock_is_a_no_op_when_already_held() {
let (_tmp, path) = index_dir();
{
// Opened with the lock: the section must neither block on itself nor release it.
let idx = KmerIndex::open_lock(&path).unwrap();
idx.with_lock(|| Ok(())).unwrap();
assert!(idx.holds_lock());
assert!(!idx.test_lock());
}
// Nested sections: only the outermost one releases.
let idx = KmerIndex::open(&path).unwrap();
idx.with_lock(|| {
idx.with_lock(|| Ok(()))?;
assert!(idx.holds_lock(), "the inner section must not release the outer lock");
Ok(())
})
.unwrap();
assert!(!idx.holds_lock());
}
+2
View File
@@ -3,6 +3,8 @@ pub mod meta;
pub mod state;
mod builder;
mod kmer_index;
#[cfg(test)]
mod lock_tests;
pub use error::{OKIError, OKIResult};
pub use builder::IndexBuilder;
+5 -1
View File
@@ -93,7 +93,11 @@ fn run_annotate(args: &AnnotateArgs) {
}
};
let idx = open_index(&args.index);
// Rewrites the index metadata in place: exclusive for the whole command.
let idx = KmerIndex::open_lock(&args.index).unwrap_or_else(|e| {
eprintln!("error opening index: {e}");
std::process::exit(1);
});
let mut genomes = idx.meta().genomes().unwrap_or_else(|e| {
eprintln!("error reading index metadata: {e}");
+2 -8
View File
@@ -70,14 +70,8 @@ struct ModeArgs {
}
pub fn run(args: ConvertArgs) {
// Modifies the index in place; acquired before opening so a concurrent
// writer can't slip in between the open and the convert below.
let _lock = obisys::DirLock::acquire(&args.index).unwrap_or_else(|e| {
eprintln!("error locking index directory {}: {e}", args.index.display());
std::process::exit(1);
});
let idx = KmerIndex::open(&args.index).unwrap_or_else(|e| {
// Modifies the index in place: exclusive for the whole command.
let idx = KmerIndex::open_lock(&args.index).unwrap_or_else(|e| {
eprintln!("error opening index: {e}");
std::process::exit(1);
});
+2 -8
View File
@@ -26,14 +26,8 @@ pub struct PackArgs {
}
pub fn run(args: PackArgs) {
// Modifies the index in place; acquired before opening so a concurrent
// writer can't slip in between the open and the pack below.
let _lock = obisys::DirLock::acquire(&args.index).unwrap_or_else(|e| {
eprintln!("error locking index directory {}: {e}", args.index.display());
std::process::exit(1);
});
let idx = Arc::new(KmerIndex::open(&args.index).unwrap_or_else(|e| {
// Modifies the index in place: exclusive for the whole command.
let idx = Arc::new(KmerIndex::open_lock(&args.index).unwrap_or_else(|e| {
eprintln!("error opening index: {e}");
std::process::exit(1);
}));
+3 -7
View File
@@ -78,13 +78,9 @@ pub fn run(args: PhyloArgs) {
// Meant to be (re)built routinely, ahead of any SNP-family distance
// computation that will later consume it.
if args.sibling_annex {
// Writes into the index directory — hold an exclusive lock for the
// duration so a second, concurrent `--sibling-annex` run on the
// same index can't corrupt these writes (see obisys::DirLock).
let _lock = obisys::DirLock::acquire(&args.index).unwrap_or_else(|e| {
eprintln!("error locking index directory {}: {e}", args.index.display());
std::process::exit(1);
});
// Writes into the index directory; `build_sibling_annex` holds the
// index's exclusive lock for the duration (`KmerIndex::with_lock`),
// so a concurrent run on the same index can't corrupt these writes.
info!("building sibling-count/minorant annex");
let t = Stage::start("sibling_annex");
cache.build_sibling_annex().unwrap_or_else(|e| {
+3 -1
View File
@@ -53,7 +53,9 @@ pub(super) fn run_bits_per_kmer(index_path: &PathBuf) {
pub(super) fn run_rename(index_path: &PathBuf, spec: &str) {
let (old_label, new_label) = parse_rename_spec(spec);
let idx = KmerIndex::open(index_path).unwrap_or_else(|e| {
// Rewrites the index metadata and renames a spectrum file in place:
// exclusive for the whole command.
let idx = KmerIndex::open_lock(index_path).unwrap_or_else(|e| {
eprintln!("error opening index: {e}");
std::process::exit(1);
});
@@ -22,7 +22,9 @@ use super::masking::is_quartet_informative;
use super::minorant_selection::non_monomorphic_reindex_layer;
use super::pair_weight::pair_balance_weight;
use crate::siblings::quartet_annex::quartet_annex_file_name;
use crate::siblings::{FamilyOverlapAccumulator, FamilyScalarAnnexBuilder, ScalarKind};
use crate::siblings::{
FamilyOverlapAccumulator, FamilyScalarAnnexBuilder, ScalarKind, family_overlap_file_name,
};
/// Which annexes a caller needs, each listed once per `--no-ambiguity`
/// counting convention (`false` = genomes carrying several members count as a
@@ -50,6 +52,24 @@ impl AnnexRequest {
dedup(self.overlap.iter().chain(&self.pair_weight).copied())
}
/// Whether every annex this request asks for already exists — the same
/// per-file existence test the builders apply, so `true` means they would
/// find nothing to build and no lock is needed. Cheap (no file is opened).
pub(crate) fn is_satisfied(&self, cache: &IndexCache) -> bool {
let overlap = self
.overlap_conventions()
.into_iter()
.all(|nb| cache.index().dir().join(family_overlap_file_name(nb)).exists());
let pair_weight = self.pair_weight_conventions();
overlap
&& cache.iter_indexed().all(|(_, _, layer)| {
let has = |name: &str| layer.dir().join(name).exists();
self.entropy.iter().all(|&nb| has(ScalarKind::Entropy.file_name(nb)))
&& self.quartet.iter().all(|&nb| has(quartet_annex_file_name(nb)))
&& pair_weight.iter().all(|&nb| has(ScalarKind::PairWeight.file_name(nb)))
})
}
/// The pair-weight conventions requested, each once.
pub(crate) fn pair_weight_conventions(&self) -> Vec<bool> {
dedup(self.pair_weight.iter().copied())
@@ -9,7 +9,7 @@
use obikidxcache::index_cache::IndexCache;
use obikindex::{OKIError, OKIResult};
use obisys::{DirLock, progress_bar};
use obisys::progress_bar;
use crate::siblings::algorithms::{
AnnexRequest, build_layer_sibling_annex, ensure_layer_family_annexes,
@@ -29,26 +29,34 @@ pub(crate) trait SiblingBuilder {
/// a missing entropy, quartet or overlap annex triggers one scan that
/// feeds every missing one, per `ensure_layer_family_annexes`'s own docs,
/// then a second scan builds the missing pair weights, which need the
/// finished overlap matrix. A no-op, cheap existence-check pass, once
/// everything requested already exists.
/// finished overlap matrix. Makes sure the sibling annex exists first
/// ([`ensure_sibling_annex`](Self::ensure_sibling_annex)). A no-op, cheap
/// existence-check pass that takes no lock, once everything requested
/// already exists; otherwise the build runs under the index's exclusive
/// lock ([`KmerIndex::with_lock`](obikindex::KmerIndex::with_lock)).
fn ensure_family_annexes(&self, request: &AnnexRequest) -> OKIResult<()>;
/// [`ensure_family_annexes`](Self::ensure_family_annexes)'s build step,
/// without the existence pre-check and without the lock: the caller
/// holds it.
fn build_family_annexes(&self, request: &AnnexRequest) -> OKIResult<()>;
/// Build the sibling annex (`siblings.psib`, every annex above is derived
/// from it) if any cached layer lacks it — the state of an index right
/// after `migrate-mphf`, which sets every annex aside. The build writes
/// into every layer, so it runs under the index's exclusive
/// [`DirLock`]; a process that waited for it finds the annex built by the
/// one that held it, and does nothing. A no-op, cheap existence-check
/// pass, once every layer has its annex. Must not be called while this
/// process already holds the lock *and* an annex is missing (the lock is
/// not reentrant); `--sibling-annex` holds it but builds the annex first.
/// into every layer, so it runs under the index's exclusive lock
/// ([`KmerIndex::with_lock`](obikindex::KmerIndex::with_lock), a no-op
/// when the caller already holds it); a process that waited for it finds
/// the annex built by the one that held it, and does nothing. A no-op,
/// cheap existence-check pass that takes no lock, once every layer has
/// its annex.
fn ensure_sibling_annex(&self) -> OKIResult<()>;
/// (Re)build the sibling annex of every cached layer, unconditionally,
/// removing the annexes derived from it (entropy, quartet, pair weight,
/// family overlap), which its renumbering makes stale. Takes no lock:
/// the caller holds it (`--sibling-annex`) or goes through
/// [`ensure_sibling_annex`](Self::ensure_sibling_annex).
/// the caller holds it ([`SiblingExt::build_sibling_annex`], or
/// [`ensure_sibling_annex`](Self::ensure_sibling_annex)).
fn build_base_sibling_annex(&self) -> OKIResult<()>;
}
@@ -58,12 +66,13 @@ impl SiblingBuilder for IndexCache {
if !missing() {
return Ok(());
}
let _lock = DirLock::acquire(self.index().dir()).map_err(OKIError::Io)?;
if !missing() {
return Ok(()); // built by the process we waited for
}
tracing::info!("sibling annex missing — building it");
self.build_base_sibling_annex()
self.index().with_lock(|| {
if !missing() {
return Ok(()); // built by the process we waited for
}
tracing::info!("sibling annex missing — building it");
self.build_base_sibling_annex()
})
}
fn build_base_sibling_annex(&self) -> OKIResult<()> {
@@ -136,6 +145,13 @@ impl SiblingBuilder for IndexCache {
fn ensure_family_annexes(&self, request: &AnnexRequest) -> OKIResult<()> {
self.ensure_sibling_annex()?;
if request.is_satisfied(self) {
return Ok(());
}
self.index().with_lock(|| self.build_family_annexes(request))
}
fn build_family_annexes(&self, request: &AnnexRequest) -> OKIResult<()> {
let n_genomes = self.meta().genomes().len();
let fast_mode = is_fast_mode(self);
let overlap_path = |no_ambiguity: bool| self.index().dir().join(family_overlap_file_name(no_ambiguity));
@@ -106,18 +106,18 @@ pub trait SiblingExt {
impl SiblingExt for IndexCache {
fn build_sibling_annex(&self) -> OKIResult<()> {
self.build_base_sibling_annex()?;
// Precompute the entropy and family-overlap annexes too, now that
// every layer's fresh sibling annex is on disk
// (`non_monomorphic_reindex_layer`, which both depend on, reads
// it) — same routine-rebuild footing as the sibling annex itself,
// rather than leaving them to be built lazily on first
// `--shannon`/`--entropy-bias`/`--min-shared-family`/
// `--family-overlap` use.
tracing::info!("building entropy and quartet (both --no-ambiguity conventions) and family-overlap annexes");
self.ensure_family_annexes(&AnnexRequest::everything())?;
Ok(())
self.index().with_lock(|| {
self.build_base_sibling_annex()?;
// Precompute the entropy, quartet, pair-weight and family-overlap
// annexes too, now that every layer's fresh sibling annex is on
// disk (`non_monomorphic_reindex_layer`, which they depend on,
// reads it) — same routine-rebuild footing as the sibling annex
// itself, rather than leaving them to be built lazily on first
// `--shannon`/`--entropy`/`--quartet`/`--pair-balance`/
// `--min-shared-family`/`--family-overlap` use.
tracing::info!("building entropy, quartet, pair-weight (both --no-ambiguity conventions) and family-overlap annexes");
self.ensure_family_annexes(&AnnexRequest::everything())
})
}
fn shannon_entropy_csv(&self, path: &Path) -> OKIResult<()> {