Extract indexing stage into LayerBuilder with NUMA-aware scheduling

Introduce a dedicated LayerBuilder struct to orchestrate parallel layer 0 construction across partitions. The fluent API supports configurable abundance thresholds and intermediate artifact retention. Parallel execution is delegated to a NUMA-aware PartitionRunner scheduler, while progress reporting and stage timing are shifted to the command layer. A new mark_indexed method cleanly separates state tracking from orchestration by generating a completion sentinel.
This commit is contained in:
Eric Coissac
2026-08-21 10:20:03 +02:00
parent 346095b9eb
commit 4b7b3c3c1a
6 changed files with 245 additions and 70 deletions
+5 -2
View File
@@ -8,6 +8,7 @@ use tempfile::tempdir;
use obikindexer::algorithms::counter::Counter;
use obikindexer::algorithms::dereplicator::Dereplicator;
use obikindexer::algorithms::layer_builder::LayerBuilder;
use obikindexer::algorithms::partitionner::PartitionRouter;
use obikindex::{GenomeInfo, IndexConfig, KmerIndex, MergeMode};
@@ -70,7 +71,6 @@ fn build_single_genome_index(dir: &Path, label: &str, seq: &[u8]) -> KmerIndex {
let mut idx = KmerIndex::create(&index_path, config, Some(GenomeInfo::new(label)))
.expect("create");
let mut rep = Reporter::new();
let stream = obiread::open_nuc_stream(fasta_path.to_str().unwrap(), K).expect("open fasta");
let mut router = PartitionRouter::new(&mut idx);
for page in stream {
@@ -86,7 +86,10 @@ fn build_single_genome_index(dir: &Path, label: &str, seq: &[u8]) -> KmerIndex {
idx.mark_scattered().expect("mark_scattered");
idx.write_spectrum(spectrum.f0, spectrum.f1, &spectrum.counts).expect("write_spectrum");
idx.mark_counted().expect("mark_counted");
idx.build_layers(1, None, false, &mut rep).expect("build_layers");
LayerBuilder::new(&idx)
.run(None::<fn(obisys::Progress)>)
.expect("build_layers");
idx.mark_indexed().expect("mark_indexed");
idx
}