Extract indexing stage into LayerBuilder with NUMA-aware scheduling
Introduce a dedicated LayerBuilder struct to orchestrate parallel layer 0 construction across partitions. The fluent API supports configurable abundance thresholds and intermediate artifact retention. Parallel execution is delegated to a NUMA-aware PartitionRunner scheduler, while progress reporting and stage timing are shifted to the command layer. A new mark_indexed method cleanly separates state tracking from orchestration by generating a completion sentinel.
This commit is contained in:
@@ -5,7 +5,6 @@ use std::path::{Path, PathBuf};
|
||||
use crate::layer::meta::PartitionMeta;
|
||||
use obisys::{Reporter, Stage, progress_bar};
|
||||
use rayon::prelude::*;
|
||||
use tracing::info;
|
||||
|
||||
use obikseq::{set_k, set_m};
|
||||
|
||||
@@ -230,6 +229,12 @@ impl KmerIndex {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Mark layer construction as complete and write `index.done`.
|
||||
pub fn mark_indexed(&self) -> OKIResult<()> {
|
||||
touch(&self.root_path.join(SENTINEL_INDEXED))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Write `spectrums/{label}.json` from an already-computed kmer
|
||||
/// spectrum (`f0`/`f1`/abundance histogram). Takes plain values rather
|
||||
/// than `obikindexer::algorithms::partitionner::KmerSpectrum` — `KmerIndex`
|
||||
@@ -260,63 +265,6 @@ impl KmerIndex {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Build the layered MPHF index for all partitions in parallel.
|
||||
///
|
||||
/// Writes `index.done` upon completion.
|
||||
pub fn build_layers(
|
||||
&self,
|
||||
min_ab: u32,
|
||||
max_ab: Option<u32>,
|
||||
keep_intermediate: bool,
|
||||
rep: &mut Reporter,
|
||||
) -> OKIResult<()> {
|
||||
let n = self.n_partitions();
|
||||
let t = Stage::start("index");
|
||||
let with_counts = self.meta.config.with_counts;
|
||||
let evidence = self.meta.config.evidence.clone();
|
||||
let block_bits = self.meta.config.block_bits;
|
||||
let mut total_kmers: usize = 0;
|
||||
let pb = progress_bar("index", n as u64, "partitions");
|
||||
|
||||
let order: Vec<usize> = (0..n).collect();
|
||||
let runner = crate::index::numa::PartitionRunner::new();
|
||||
runner
|
||||
.run(
|
||||
&order,
|
||||
|i| {
|
||||
self.build_index_layer(
|
||||
i,
|
||||
min_ab,
|
||||
max_ab,
|
||||
with_counts,
|
||||
&evidence,
|
||||
block_bits,
|
||||
)
|
||||
},
|
||||
|i, n_kmers, _| {
|
||||
if n_kmers > 0 {
|
||||
total_kmers += n_kmers;
|
||||
pb.inc(1);
|
||||
pb.set_message(format!("{i}: {n_kmers} kmers"));
|
||||
}
|
||||
},
|
||||
)
|
||||
.map_err(OKIError::Partition)?;
|
||||
|
||||
pb.finish_and_clear();
|
||||
info!("done — {} total kmers indexed", total_kmers);
|
||||
|
||||
if !keep_intermediate {
|
||||
for i in 0..n {
|
||||
self.remove_build_artifacts(i);
|
||||
}
|
||||
}
|
||||
|
||||
touch(&self.root_path.join(SENTINEL_INDEXED))?;
|
||||
rep.push(t.stop());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Path to the unitigs file for partition `part`, layer `layer`.
|
||||
pub fn layer_unitigs_path(&self, part: usize, layer: usize) -> PathBuf {
|
||||
self.layer_dir(part, layer).join("unitigs.bin")
|
||||
|
||||
Reference in New Issue
Block a user