Extract indexing stage into LayerBuilder with NUMA-aware scheduling

Introduce a dedicated LayerBuilder struct to orchestrate parallel layer 0 construction across partitions. The fluent API supports configurable abundance thresholds and intermediate artifact retention. Parallel execution is delegated to a NUMA-aware PartitionRunner scheduler, while progress reporting and stage timing are shifted to the command layer. A new mark_indexed method cleanly separates state tracking from orchestration by generating a completion sentinel.
This commit is contained in:
Eric Coissac
2026-08-21 10:20:03 +02:00
parent 346095b9eb
commit 4b7b3c3c1a
6 changed files with 245 additions and 70 deletions
+6 -58
View File
@@ -5,7 +5,6 @@ use std::path::{Path, PathBuf};
use crate::layer::meta::PartitionMeta;
use obisys::{Reporter, Stage, progress_bar};
use rayon::prelude::*;
use tracing::info;
use obikseq::{set_k, set_m};
@@ -230,6 +229,12 @@ impl KmerIndex {
Ok(())
}
/// Mark layer construction as complete and write `index.done`.
pub fn mark_indexed(&self) -> OKIResult<()> {
touch(&self.root_path.join(SENTINEL_INDEXED))?;
Ok(())
}
/// Write `spectrums/{label}.json` from an already-computed kmer
/// spectrum (`f0`/`f1`/abundance histogram). Takes plain values rather
/// than `obikindexer::algorithms::partitionner::KmerSpectrum` — `KmerIndex`
@@ -260,63 +265,6 @@ impl KmerIndex {
Ok(())
}
/// Build the layered MPHF index for all partitions in parallel.
///
/// Writes `index.done` upon completion.
pub fn build_layers(
&self,
min_ab: u32,
max_ab: Option<u32>,
keep_intermediate: bool,
rep: &mut Reporter,
) -> OKIResult<()> {
let n = self.n_partitions();
let t = Stage::start("index");
let with_counts = self.meta.config.with_counts;
let evidence = self.meta.config.evidence.clone();
let block_bits = self.meta.config.block_bits;
let mut total_kmers: usize = 0;
let pb = progress_bar("index", n as u64, "partitions");
let order: Vec<usize> = (0..n).collect();
let runner = crate::index::numa::PartitionRunner::new();
runner
.run(
&order,
|i| {
self.build_index_layer(
i,
min_ab,
max_ab,
with_counts,
&evidence,
block_bits,
)
},
|i, n_kmers, _| {
if n_kmers > 0 {
total_kmers += n_kmers;
pb.inc(1);
pb.set_message(format!("{i}: {n_kmers} kmers"));
}
},
)
.map_err(OKIError::Partition)?;
pb.finish_and_clear();
info!("done — {} total kmers indexed", total_kmers);
if !keep_intermediate {
for i in 0..n {
self.remove_build_artifacts(i);
}
}
touch(&self.root_path.join(SENTINEL_INDEXED))?;
rep.push(t.stop());
Ok(())
}
/// Path to the unitigs file for partition `part`, layer `layer`.
pub fn layer_unitigs_path(&self, part: usize, layer: usize) -> PathBuf {
self.layer_dir(part, layer).join("unitigs.bin")