fix: resolve test race conditions, add logging, and fix CI deadlock
Re-enables the `numa` feature in CI workflows to prevent container/cgroup deadlocks while preserving validation correctness. Fixes concurrent test race conditions by replacing thread-local parameter storage with process-wide atomics and mutex locks. Integrates `tracing-subscriber` for structured logging and adds thread-ID tracking to debug worker lifecycles. Additionally bumps the crate version, updates `.gitignore`, documents experimental evolutionary distance pipelines, and refactors hardcoded test constants.
This commit is contained in:
@@ -24,6 +24,7 @@ hwlocality = { version = "1.0.0-alpha.11", features = ["vendored"], option
|
||||
[dev-dependencies]
|
||||
obiread = { path = "../obiread" }
|
||||
tempfile = "3"
|
||||
tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] }
|
||||
|
||||
[features]
|
||||
default = ["numa"]
|
||||
|
||||
@@ -295,20 +295,27 @@ impl PartitionRunner {
|
||||
let pool = node.pool.clone();
|
||||
|
||||
s.spawn(move || {
|
||||
let tid = std::thread::current().id();
|
||||
debug!(?tid, "PartitionRunner worker: waiting on activation");
|
||||
if arx.recv().is_err() {
|
||||
debug!(?tid, "PartitionRunner worker: activation channel closed, exiting");
|
||||
return;
|
||||
}
|
||||
debug!(?tid, "PartitionRunner worker: activated");
|
||||
if !cpu_ids.is_empty() {
|
||||
pin_current_thread(cpu_ids);
|
||||
}
|
||||
for i in &prx {
|
||||
debug!(?tid, partition = i, "PartitionRunner worker: picked partition");
|
||||
let t = Instant::now();
|
||||
let r = match &pool {
|
||||
Some(p) => p.install(|| f(i)),
|
||||
None => f(i),
|
||||
};
|
||||
debug!(?tid, partition = i, "PartitionRunner worker: partition done");
|
||||
etx.send(WorkerEvent::Completed(i, r, t.elapsed())).ok();
|
||||
}
|
||||
debug!(?tid, "PartitionRunner worker: no more partitions, exiting");
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -319,13 +326,18 @@ impl PartitionRunner {
|
||||
// ── Controller ────────────────────────────────────────────────────
|
||||
let mut activation = NodeActivation::new(&activate_txs, &node_caps, max_workers);
|
||||
activation.activate_initial(INITIAL_DIVISOR, n_total);
|
||||
debug!(n_total, activated = activation.total(), "PartitionRunner controller: initial activation");
|
||||
|
||||
let mut cpu_sample = CpuSample::now();
|
||||
let mut io_sample = IoSample::now();
|
||||
let mut completed = 0usize;
|
||||
|
||||
while completed < n_total {
|
||||
let Ok(event) = event_rx.recv() else { break };
|
||||
debug!(completed, n_total, "PartitionRunner controller: waiting for an event");
|
||||
let Ok(event) = event_rx.recv() else {
|
||||
debug!("PartitionRunner controller: event channel closed, stopping");
|
||||
break;
|
||||
};
|
||||
match event {
|
||||
WorkerEvent::Completed(i, r, dur) => {
|
||||
match r {
|
||||
|
||||
@@ -976,7 +976,23 @@ mod tests {
|
||||
/// the same primitives `obikmer`'s `scatter` step uses (minus the
|
||||
/// multi-file `obipipeline` wrapper — a single sequence needs none of
|
||||
/// that): normalise -> build superkmers -> route -> write.
|
||||
/// `cargo test` doesn't install a `tracing` subscriber the way `obikmer`'s
|
||||
/// CLI does, so `debug!`/etc. are silent no-ops by default — including the
|
||||
/// `PartitionRunner` instrumentation that would matter most for
|
||||
/// re-diagnosing a hang here. `try_init` is idempotent across concurrently
|
||||
/// running tests (later calls just find a subscriber already installed).
|
||||
fn init_tracing() {
|
||||
let _ = tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
tracing_subscriber::EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")),
|
||||
)
|
||||
.with_writer(std::io::stderr)
|
||||
.try_init();
|
||||
}
|
||||
|
||||
fn build_single_genome_index(dir: &Path, label: &str, seq: &[u8]) -> KmerIndex {
|
||||
init_tracing();
|
||||
let fasta_path = dir.join(format!("{label}.fasta"));
|
||||
let mut f = std::fs::File::create(&fasta_path).unwrap();
|
||||
writeln!(f, ">{label}").unwrap();
|
||||
|
||||
Reference in New Issue
Block a user