diff --git a/src/obikindex/src/index/kmer_index.rs b/src/obikindex/src/index/kmer_index.rs index 3cf4fcb5..d9152fba 100644 --- a/src/obikindex/src/index/kmer_index.rs +++ b/src/obikindex/src/index/kmer_index.rs @@ -56,7 +56,7 @@ impl KmerIndex { /// access (resuming an interrupted build) uses /// [`open_lock`](Self::open_lock) instead; either way, `self.lock` /// starts empty — [`test_lock`](Self::test_lock)/ - /// [`acquire_lock`](Self::acquire_lock) are opt-in from here. + /// [`with_lock`](Self::with_lock) are opt-in from here. pub fn open>(path: P) -> OKIResult { let root_path = path.as_ref().to_owned(); let meta = IndexMeta::open_at(&root_path).map_err(OKIError::Io)?; @@ -69,47 +69,88 @@ impl KmerIndex { }) } - /// `open`, then hold the directory lock exclusively — for a caller - /// that resumes a write pipeline (e.g. `obikmer index` re-run on an - /// existing, not-yet-`Indexed` directory) rather than reading - /// concurrently with others. + /// `open`, holding the directory lock exclusively for as long as the + /// returned index lives — **the** way for a command that modifies an + /// existing index in place (`convert`, `pack`, `annotate`, `utils + /// rename`, resuming an interrupted build, ...) to claim it. The lock is + /// taken *before* the metadata is read, so what the caller reads is what + /// it will modify. A directory that is not an index is reported by + /// `open`, without a lock file being created in it. + /// + /// A command that only reads, or that writes a *new* index elsewhere + /// (`merge`, `filter`, `select`: they lock their destination, not their + /// sources), uses [`open`](Self::open). Library code that has to write + /// into an index it was handed (annexes built on demand) wraps that + /// write in [`with_lock`](Self::with_lock), which does nothing more when + /// the lock is already held. pub fn open_lock>(path: P) -> OKIResult { - let idx = Self::open(path)?; - if !idx.acquire_lock() { - return Err(OKIError::Io(std::io::Error::other( - "failed to acquire index lock", - ))); + let path = path.as_ref(); + if !Self::is_an_index(path) { + return Self::open(path); } + let lock = obisys::DirLock::acquire(path).map_err(OKIError::Io)?; + let idx = Self::open(path)?; + *idx.lock.lock().unwrap() = Some(lock); Ok(idx) } /// Block until this index's directory lock is acquired, then hold it /// (released on [`release_lock`](Self::release_lock) or when `self` is - /// dropped). Returns `false` only on a genuine I/O failure acquiring - /// the lock file itself — contention blocks rather than failing. - pub fn acquire_lock(&self) -> bool { + /// dropped). Idempotent: a no-op, returning `true`, if this index + /// already holds it (a second `DirLock` on the same directory would + /// block on the first). Returns `false` only on a genuine I/O failure + /// acquiring the lock file itself — contention blocks rather than + /// failing. + pub(crate) fn acquire_lock(&self) -> bool { + let mut held = self.lock.lock().unwrap(); + if held.is_some() { + return true; + } match obisys::DirLock::acquire(&self.root_path) { Ok(lock) => { - *self.lock.lock().unwrap() = Some(lock); + *held = Some(lock); true } Err(_) => false, } } + /// Whether this index object currently holds its directory lock. + pub fn holds_lock(&self) -> bool { + self.lock.lock().unwrap().is_some() + } + + /// Run `f` with the directory lock held: as is if this index already + /// holds it (opened with [`open_lock`](Self::open_lock), or an enclosing + /// `with_lock`), otherwise taking it for the duration of `f` only — also + /// released if `f` fails. Not for concurrent use from several threads of + /// one process: the first one to finish would release the lock under + /// the others. + pub fn with_lock(&self, f: impl FnOnce() -> OKIResult) -> OKIResult { + let was_held = self.holds_lock(); + if !was_held && !self.acquire_lock() { + return Err(OKIError::Io(std::io::Error::other("failed to acquire index lock"))); + } + let out = f(); + if !was_held { + self.release_lock(); + } + out + } + /// Non-blocking probe of this index's directory lock — doesn't retain /// it either way: `true` if the lock was free (briefly acquired, then /// immediately released), `false` if another process currently holds /// it. For a read-only caller that wants to warn about a build in /// progress without itself becoming exclusive; use - /// [`acquire_lock`](Self::acquire_lock)/[`open_lock`](Self::open_lock) - /// to actually hold it. + /// [`open_lock`](Self::open_lock)/[`with_lock`](Self::with_lock) to + /// actually hold it. pub fn test_lock(&self) -> bool { matches!(obisys::DirLock::try_acquire(&self.root_path), Ok(Some(_))) } /// Release this index's directory lock, if held. A no-op if it isn't. - pub fn release_lock(&self) { + pub(crate) fn release_lock(&self) { *self.lock.lock().unwrap() = None; } diff --git a/src/obikindex/src/index/lock_tests.rs b/src/obikindex/src/index/lock_tests.rs new file mode 100644 index 00000000..a4aa8cc5 --- /dev/null +++ b/src/obikindex/src/index/lock_tests.rs @@ -0,0 +1,101 @@ +//! The directory-lock API of [`KmerIndex`]: `open` never locks, `open_lock` +//! holds for the object's lifetime, `with_lock` holds for a section and is a +//! no-op when the lock is already held. A second handle on the directory +//! (`test_lock`) is what tells held from free. + +use std::path::PathBuf; + +use crate::index::error::OKIError; +use crate::index::kmer_index::KmerIndex; +use crate::index::meta::IndexConfig; +use crate::layer::IndexMode; + +/// Same k/m pair as the other tests of this crate (`obikseq`'s k/m are +/// process-wide in test builds). +fn index_dir() -> (tempfile::TempDir, PathBuf) { + let tmp = tempfile::tempdir().expect("tempdir"); + let path = tmp.path().join("idx"); + let config = IndexConfig { + kmer_size: 11, + minimizer_size: 5, + n_bits: 2, + with_counts: false, + evidence: IndexMode::Exact, + block_bits: 0, + }; + drop(KmerIndex::create(&path, config, None).expect("create index")); // lock released on drop + (tmp, path) +} + +#[test] +fn open_does_not_lock() { + let (_tmp, path) = index_dir(); + let idx = KmerIndex::open(&path).unwrap(); + assert!(!idx.holds_lock()); + assert!(idx.test_lock(), "the directory must be free"); +} + +#[test] +fn open_lock_holds_until_dropped() { + let (_tmp, path) = index_dir(); + let idx = KmerIndex::open_lock(&path).unwrap(); + assert!(idx.holds_lock()); + assert!(!idx.test_lock(), "another handle must see the directory locked"); + drop(idx); + assert!(KmerIndex::open(&path).unwrap().test_lock(), "dropping releases the lock"); +} + +#[test] +fn open_lock_on_a_non_index_reports_it_and_leaves_no_lock_file() { + let tmp = tempfile::tempdir().unwrap(); + assert!(KmerIndex::open_lock(tmp.path()).is_err()); + assert!(!tmp.path().join(".obikmer.lock").exists()); +} + +#[test] +fn with_lock_holds_for_the_section_only() { + let (_tmp, path) = index_dir(); + let idx = KmerIndex::open(&path).unwrap(); + let out = idx + .with_lock(|| { + assert!(idx.holds_lock()); + assert!(!idx.test_lock()); + Ok(7) + }) + .unwrap(); + assert_eq!(out, 7); + assert!(!idx.holds_lock()); + assert!(idx.test_lock()); +} + +#[test] +fn with_lock_releases_on_error() { + let (_tmp, path) = index_dir(); + let idx = KmerIndex::open(&path).unwrap(); + let r: Result<(), _> = idx.with_lock(|| Err(OKIError::Io(std::io::Error::other("boom")))); + assert!(r.is_err()); + assert!(!idx.holds_lock()); + assert!(idx.test_lock()); +} + +#[test] +fn with_lock_is_a_no_op_when_already_held() { + let (_tmp, path) = index_dir(); + { + // Opened with the lock: the section must neither block on itself nor release it. + let idx = KmerIndex::open_lock(&path).unwrap(); + idx.with_lock(|| Ok(())).unwrap(); + assert!(idx.holds_lock()); + assert!(!idx.test_lock()); + } + + // Nested sections: only the outermost one releases. + let idx = KmerIndex::open(&path).unwrap(); + idx.with_lock(|| { + idx.with_lock(|| Ok(()))?; + assert!(idx.holds_lock(), "the inner section must not release the outer lock"); + Ok(()) + }) + .unwrap(); + assert!(!idx.holds_lock()); +} diff --git a/src/obikindex/src/index/mod.rs b/src/obikindex/src/index/mod.rs index becde665..648d06bb 100644 --- a/src/obikindex/src/index/mod.rs +++ b/src/obikindex/src/index/mod.rs @@ -3,6 +3,8 @@ pub mod meta; pub mod state; mod builder; mod kmer_index; +#[cfg(test)] +mod lock_tests; pub use error::{OKIError, OKIResult}; pub use builder::IndexBuilder; diff --git a/src/obikmer/src/cmd/annotate/mod.rs b/src/obikmer/src/cmd/annotate/mod.rs index a4018dde..ec4ffe09 100644 --- a/src/obikmer/src/cmd/annotate/mod.rs +++ b/src/obikmer/src/cmd/annotate/mod.rs @@ -93,7 +93,11 @@ fn run_annotate(args: &AnnotateArgs) { } }; - let idx = open_index(&args.index); + // Rewrites the index metadata in place: exclusive for the whole command. + let idx = KmerIndex::open_lock(&args.index).unwrap_or_else(|e| { + eprintln!("error opening index: {e}"); + std::process::exit(1); + }); let mut genomes = idx.meta().genomes().unwrap_or_else(|e| { eprintln!("error reading index metadata: {e}"); diff --git a/src/obikmer/src/cmd/convert/mod.rs b/src/obikmer/src/cmd/convert/mod.rs index 2e5da7c6..9f05c8e2 100644 --- a/src/obikmer/src/cmd/convert/mod.rs +++ b/src/obikmer/src/cmd/convert/mod.rs @@ -70,14 +70,8 @@ struct ModeArgs { } pub fn run(args: ConvertArgs) { - // Modifies the index in place; acquired before opening so a concurrent - // writer can't slip in between the open and the convert below. - let _lock = obisys::DirLock::acquire(&args.index).unwrap_or_else(|e| { - eprintln!("error locking index directory {}: {e}", args.index.display()); - std::process::exit(1); - }); - - let idx = KmerIndex::open(&args.index).unwrap_or_else(|e| { + // Modifies the index in place: exclusive for the whole command. + let idx = KmerIndex::open_lock(&args.index).unwrap_or_else(|e| { eprintln!("error opening index: {e}"); std::process::exit(1); }); diff --git a/src/obikmer/src/cmd/pack/mod.rs b/src/obikmer/src/cmd/pack/mod.rs index cdf85ba0..c3073acc 100644 --- a/src/obikmer/src/cmd/pack/mod.rs +++ b/src/obikmer/src/cmd/pack/mod.rs @@ -26,14 +26,8 @@ pub struct PackArgs { } pub fn run(args: PackArgs) { - // Modifies the index in place; acquired before opening so a concurrent - // writer can't slip in between the open and the pack below. - let _lock = obisys::DirLock::acquire(&args.index).unwrap_or_else(|e| { - eprintln!("error locking index directory {}: {e}", args.index.display()); - std::process::exit(1); - }); - - let idx = Arc::new(KmerIndex::open(&args.index).unwrap_or_else(|e| { + // Modifies the index in place: exclusive for the whole command. + let idx = Arc::new(KmerIndex::open_lock(&args.index).unwrap_or_else(|e| { eprintln!("error opening index: {e}"); std::process::exit(1); })); diff --git a/src/obikmer/src/cmd/phylo/mod.rs b/src/obikmer/src/cmd/phylo/mod.rs index 4b4158aa..995077a3 100644 --- a/src/obikmer/src/cmd/phylo/mod.rs +++ b/src/obikmer/src/cmd/phylo/mod.rs @@ -78,13 +78,9 @@ pub fn run(args: PhyloArgs) { // Meant to be (re)built routinely, ahead of any SNP-family distance // computation that will later consume it. if args.sibling_annex { - // Writes into the index directory — hold an exclusive lock for the - // duration so a second, concurrent `--sibling-annex` run on the - // same index can't corrupt these writes (see obisys::DirLock). - let _lock = obisys::DirLock::acquire(&args.index).unwrap_or_else(|e| { - eprintln!("error locking index directory {}: {e}", args.index.display()); - std::process::exit(1); - }); + // Writes into the index directory; `build_sibling_annex` holds the + // index's exclusive lock for the duration (`KmerIndex::with_lock`), + // so a concurrent run on the same index can't corrupt these writes. info!("building sibling-count/minorant annex"); let t = Stage::start("sibling_annex"); cache.build_sibling_annex().unwrap_or_else(|e| { diff --git a/src/obikmer/src/cmd/utils/maintenance.rs b/src/obikmer/src/cmd/utils/maintenance.rs index d360131f..81aec0f3 100644 --- a/src/obikmer/src/cmd/utils/maintenance.rs +++ b/src/obikmer/src/cmd/utils/maintenance.rs @@ -53,7 +53,9 @@ pub(super) fn run_bits_per_kmer(index_path: &PathBuf) { pub(super) fn run_rename(index_path: &PathBuf, spec: &str) { let (old_label, new_label) = parse_rename_spec(spec); - let idx = KmerIndex::open(index_path).unwrap_or_else(|e| { + // Rewrites the index metadata and renames a spectrum file in place: + // exclusive for the whole command. + let idx = KmerIndex::open_lock(index_path).unwrap_or_else(|e| { eprintln!("error opening index: {e}"); std::process::exit(1); }); diff --git a/src/obikphylo/src/siblings/algorithms/family_annexes.rs b/src/obikphylo/src/siblings/algorithms/family_annexes.rs index ffcec3c5..b034734e 100644 --- a/src/obikphylo/src/siblings/algorithms/family_annexes.rs +++ b/src/obikphylo/src/siblings/algorithms/family_annexes.rs @@ -22,7 +22,9 @@ use super::masking::is_quartet_informative; use super::minorant_selection::non_monomorphic_reindex_layer; use super::pair_weight::pair_balance_weight; use crate::siblings::quartet_annex::quartet_annex_file_name; -use crate::siblings::{FamilyOverlapAccumulator, FamilyScalarAnnexBuilder, ScalarKind}; +use crate::siblings::{ + FamilyOverlapAccumulator, FamilyScalarAnnexBuilder, ScalarKind, family_overlap_file_name, +}; /// Which annexes a caller needs, each listed once per `--no-ambiguity` /// counting convention (`false` = genomes carrying several members count as a @@ -50,6 +52,24 @@ impl AnnexRequest { dedup(self.overlap.iter().chain(&self.pair_weight).copied()) } + /// Whether every annex this request asks for already exists — the same + /// per-file existence test the builders apply, so `true` means they would + /// find nothing to build and no lock is needed. Cheap (no file is opened). + pub(crate) fn is_satisfied(&self, cache: &IndexCache) -> bool { + let overlap = self + .overlap_conventions() + .into_iter() + .all(|nb| cache.index().dir().join(family_overlap_file_name(nb)).exists()); + let pair_weight = self.pair_weight_conventions(); + overlap + && cache.iter_indexed().all(|(_, _, layer)| { + let has = |name: &str| layer.dir().join(name).exists(); + self.entropy.iter().all(|&nb| has(ScalarKind::Entropy.file_name(nb))) + && self.quartet.iter().all(|&nb| has(quartet_annex_file_name(nb))) + && pair_weight.iter().all(|&nb| has(ScalarKind::PairWeight.file_name(nb))) + }) + } + /// The pair-weight conventions requested, each once. pub(crate) fn pair_weight_conventions(&self) -> Vec { dedup(self.pair_weight.iter().copied()) diff --git a/src/obikphylo/src/siblings/extensions/sibling_builder.rs b/src/obikphylo/src/siblings/extensions/sibling_builder.rs index 12df9414..cf57eeae 100644 --- a/src/obikphylo/src/siblings/extensions/sibling_builder.rs +++ b/src/obikphylo/src/siblings/extensions/sibling_builder.rs @@ -9,7 +9,7 @@ use obikidxcache::index_cache::IndexCache; use obikindex::{OKIError, OKIResult}; -use obisys::{DirLock, progress_bar}; +use obisys::progress_bar; use crate::siblings::algorithms::{ AnnexRequest, build_layer_sibling_annex, ensure_layer_family_annexes, @@ -29,26 +29,34 @@ pub(crate) trait SiblingBuilder { /// a missing entropy, quartet or overlap annex triggers one scan that /// feeds every missing one, per `ensure_layer_family_annexes`'s own docs, /// then a second scan builds the missing pair weights, which need the - /// finished overlap matrix. A no-op, cheap existence-check pass, once - /// everything requested already exists. + /// finished overlap matrix. Makes sure the sibling annex exists first + /// ([`ensure_sibling_annex`](Self::ensure_sibling_annex)). A no-op, cheap + /// existence-check pass that takes no lock, once everything requested + /// already exists; otherwise the build runs under the index's exclusive + /// lock ([`KmerIndex::with_lock`](obikindex::KmerIndex::with_lock)). fn ensure_family_annexes(&self, request: &AnnexRequest) -> OKIResult<()>; + /// [`ensure_family_annexes`](Self::ensure_family_annexes)'s build step, + /// without the existence pre-check and without the lock: the caller + /// holds it. + fn build_family_annexes(&self, request: &AnnexRequest) -> OKIResult<()>; + /// Build the sibling annex (`siblings.psib`, every annex above is derived /// from it) if any cached layer lacks it — the state of an index right /// after `migrate-mphf`, which sets every annex aside. The build writes - /// into every layer, so it runs under the index's exclusive - /// [`DirLock`]; a process that waited for it finds the annex built by the - /// one that held it, and does nothing. A no-op, cheap existence-check - /// pass, once every layer has its annex. Must not be called while this - /// process already holds the lock *and* an annex is missing (the lock is - /// not reentrant); `--sibling-annex` holds it but builds the annex first. + /// into every layer, so it runs under the index's exclusive lock + /// ([`KmerIndex::with_lock`](obikindex::KmerIndex::with_lock), a no-op + /// when the caller already holds it); a process that waited for it finds + /// the annex built by the one that held it, and does nothing. A no-op, + /// cheap existence-check pass that takes no lock, once every layer has + /// its annex. fn ensure_sibling_annex(&self) -> OKIResult<()>; /// (Re)build the sibling annex of every cached layer, unconditionally, /// removing the annexes derived from it (entropy, quartet, pair weight, /// family overlap), which its renumbering makes stale. Takes no lock: - /// the caller holds it (`--sibling-annex`) or goes through - /// [`ensure_sibling_annex`](Self::ensure_sibling_annex). + /// the caller holds it ([`SiblingExt::build_sibling_annex`], or + /// [`ensure_sibling_annex`](Self::ensure_sibling_annex)). fn build_base_sibling_annex(&self) -> OKIResult<()>; } @@ -58,12 +66,13 @@ impl SiblingBuilder for IndexCache { if !missing() { return Ok(()); } - let _lock = DirLock::acquire(self.index().dir()).map_err(OKIError::Io)?; - if !missing() { - return Ok(()); // built by the process we waited for - } - tracing::info!("sibling annex missing — building it"); - self.build_base_sibling_annex() + self.index().with_lock(|| { + if !missing() { + return Ok(()); // built by the process we waited for + } + tracing::info!("sibling annex missing — building it"); + self.build_base_sibling_annex() + }) } fn build_base_sibling_annex(&self) -> OKIResult<()> { @@ -136,6 +145,13 @@ impl SiblingBuilder for IndexCache { fn ensure_family_annexes(&self, request: &AnnexRequest) -> OKIResult<()> { self.ensure_sibling_annex()?; + if request.is_satisfied(self) { + return Ok(()); + } + self.index().with_lock(|| self.build_family_annexes(request)) + } + + fn build_family_annexes(&self, request: &AnnexRequest) -> OKIResult<()> { let n_genomes = self.meta().genomes().len(); let fast_mode = is_fast_mode(self); let overlap_path = |no_ambiguity: bool| self.index().dir().join(family_overlap_file_name(no_ambiguity)); diff --git a/src/obikphylo/src/siblings/extensions/sibling_ext.rs b/src/obikphylo/src/siblings/extensions/sibling_ext.rs index c5b6cc1c..e38a467b 100644 --- a/src/obikphylo/src/siblings/extensions/sibling_ext.rs +++ b/src/obikphylo/src/siblings/extensions/sibling_ext.rs @@ -106,18 +106,18 @@ pub trait SiblingExt { impl SiblingExt for IndexCache { fn build_sibling_annex(&self) -> OKIResult<()> { - self.build_base_sibling_annex()?; - // Precompute the entropy and family-overlap annexes too, now that - // every layer's fresh sibling annex is on disk - // (`non_monomorphic_reindex_layer`, which both depend on, reads - // it) — same routine-rebuild footing as the sibling annex itself, - // rather than leaving them to be built lazily on first - // `--shannon`/`--entropy-bias`/`--min-shared-family`/ - // `--family-overlap` use. - tracing::info!("building entropy and quartet (both --no-ambiguity conventions) and family-overlap annexes"); - self.ensure_family_annexes(&AnnexRequest::everything())?; - - Ok(()) + self.index().with_lock(|| { + self.build_base_sibling_annex()?; + // Precompute the entropy, quartet, pair-weight and family-overlap + // annexes too, now that every layer's fresh sibling annex is on + // disk (`non_monomorphic_reindex_layer`, which they depend on, + // reads it) — same routine-rebuild footing as the sibling annex + // itself, rather than leaving them to be built lazily on first + // `--shannon`/`--entropy`/`--quartet`/`--pair-balance`/ + // `--min-shared-family`/`--family-overlap` use. + tracing::info!("building entropy, quartet, pair-weight (both --no-ambiguity conventions) and family-overlap annexes"); + self.ensure_family_annexes(&AnnexRequest::everything()) + }) } fn shannon_entropy_csv(&self, path: &Path) -> OKIResult<()> {