Extract index modules into specialized workspace subcrates

This commit partitions the obikindex crate into multiple focused subcrates (obikfilter, obikmerge, obikquery, obikrebuild, obikselect, obikstats, obikdump, and obikidxcache) to reduce coupling and clarify module boundaries. It standardizes error handling across the workspace using OKIError and OKIResult, updates index APIs to support lazy, disk-backed partition access, and migrates NUMA system utilities to a new obisys crate. All modifications are structural, focusing on dependency graph expansion, import path updates, and API surface reorganization without altering core runtime behavior.
This commit is contained in:
Eric Coissac committed 2026-08-22 06:25:28 +02:00
1 parent c9d10d55c7
commit fc4464a0ef
81 files changed
+1640 -1196

No files matched your search

@@ -18,7 +18,7 @@ use crate::extensions::PrivateBuilder;
/// The actual per-partition construction (De Bruijn graph, unitigs, MPHF,
/// matrix) stays a `KmerIndex` method (`build_index_layer`) — unlike
/// `Dereplicator`/`Counter`, it depends on several `obikindex`-internal
/// helpers (`graph_pipeline`, `common::olm_to_sk`) already shared with
/// helpers (`graph_pipeline`) already shared with
/// `merge`/`select`/`rebuild`'s own layer-construction paths, so it
/// belongs there, not duplicated or newly exported just for this
/// algorithm. `LayerBuilder`'s own job is the orchestration around it:
+4 -6
View File
@@ -36,7 +36,7 @@ use obidebruinj::GraphDeBruijn;
use obikindex::layer::IndexMode;
use obikindex::layer::{TypedLayer, meta::PartitionMeta};
use obikindex::{KmerIndex, OKIError, OKIResult};
use obikindex::{materialize_layer, olm_to_sk, write_graph_as_unitigs};
use crate::graph_pipeline::{materialize_layer, write_graph_as_unitigs};
use obiskio::{SKError, SKFileMeta, SKFileReader};
use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xx64};
@@ -109,7 +109,7 @@ impl PrivateBuilder for KmerIndex {
.map(|g| g.label.as_str())
.unwrap_or("unknown")
.to_owned();
let spectrums_dir = self.root_path().join("spectrums");
let spectrums_dir = self.dir().join("spectrums");
fs::create_dir_all(&spectrums_dir)?;
let path = spectrums_dir.join(format!("{label}.json"));
let spectrum_map: BTreeMap<String, u64> = counts
@@ -194,8 +194,7 @@ impl PrivateBuilder for KmerIndex {
(Some(mphf), Some(counts)) => counts.get(mphf.index(&kmer.raw())),
_ => 1,
},
)
.map_err(|e| olm_to_sk(e, "layer build"))?;
)?;
n
} else {
materialize_layer(g, &layer0_dir, block_bits, mode)?
@@ -206,8 +205,7 @@ impl PrivateBuilder for KmerIndex {
n_layers: 1,
mode: mode.clone(),
}
.save(index_dir)
.map_err(|e| olm_to_sk(e, "layer build"))?;
.save(index_dir)?;
Ok(n_kmers)
}
+150
View File
@@ -0,0 +1,150 @@
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
use tracing::debug;
use obipipeline::{
Pipeline, PipelineError, PipelineSender, SharedFlatFn, Stage, WorkerPool,
make_sink, make_source, make_transform,
throttle,
};
use obidebruinj::GraphDeBruijn;
use obikseq::CanonicalKmer;
use obikindex::layer::{IndexMode, TypedLayer};
use obikindex::{OKIError, OKIResult};
// ── KmerGraphData ─────────────────────────────────────────────────────────────
enum KmerGraphData {
File(obipipeline::Throttled<PathBuf>),
RawBatch(Vec<CanonicalKmer>),
FilteredBatch(Vec<CanonicalKmer>),
}
// ── build_graph ───────────────────────────────────────────────────────────────
/// Phase 1: pipeline that reads files, filters kmers, pushes into a GraphDeBruijn.
///
/// `flat_fn(path, emit)`: opens path, iterates kmers, calls `emit(batch)` for each batch.
/// `filter(kmer) -> bool`: secondary filter applied in the Transform stage.
pub fn build_graph<I, F, G>(
file_source: I,
flat_fn: F,
filter: G,
n_workers: usize,
max_open: usize,
) -> OKIResult<GraphDeBruijn>
where
I: Iterator<Item = PathBuf> + Send + 'static,
F: Fn(PathBuf, &mut dyn FnMut(Vec<CanonicalKmer>)) -> OKIResult<()> + Send + Sync + 'static,
G: Fn(CanonicalKmer) -> bool + Send + Sync + 'static,
{
let capacity = 2;
let flat_fn = Arc::new(flat_fn);
let filter = Arc::new(filter);
let g_shared = Arc::new(Mutex::new(GraphDeBruijn::new()));
let g_sink = Arc::clone(&g_shared);
let err_cap: Arc<Mutex<Option<OKIError>>> = Arc::new(Mutex::new(None));
let err_flat = Arc::clone(&err_cap);
let throttled = throttle(file_source, max_open);
let pipeline = Pipeline::new(
make_source!(KmerGraphData, throttled, File),
vec![
Stage::Flat(Arc::new(
move |data: KmerGraphData,
push: &PipelineSender<Result<KmerGraphData, PipelineError>>,
delta: &PipelineSender<isize>|
{
if let KmerGraphData::File(t) = data {
let path = t.item;
let _guard = t.guard; // released at end of block
let mut count: isize = 0;
let push_clone = push.clone();
let result = flat_fn(path, &mut |batch: Vec<CanonicalKmer>| {
push_clone.send(Ok(KmerGraphData::RawBatch(batch))).ok();
count += 1;
});
match result {
Ok(()) => {
delta.send(count - 1).ok();
}
Err(e) => {
*err_flat.lock().unwrap() = Some(e);
delta.send(-1).ok();
}
}
}
},
) as SharedFlatFn<KmerGraphData>),
make_transform!(KmerGraphData, {
let filter = Arc::clone(&filter);
move |batch: Vec<CanonicalKmer>| -> Vec<CanonicalKmer> {
batch.into_iter().filter(|k| filter(*k)).collect()
}
}, RawBatch, FilteredBatch),
],
make_sink!(KmerGraphData, {
move |batch: Vec<CanonicalKmer>| {
let mut g = g_sink.lock().unwrap();
for kmer in batch {
g.push(kmer);
}
}
}, FilteredBatch),
);
WorkerPool::new(pipeline, n_workers, capacity).run();
if let Some(e) = Arc::try_unwrap(err_cap)
.unwrap_or_else(|_| panic!("build_graph: err_cap not uniquely owned after pipeline"))
.into_inner()
.unwrap_or_else(|e| e.into_inner())
{
return Err(e);
}
let g = Arc::try_unwrap(g_shared)
.unwrap_or_else(|_| panic!("build_graph: g_shared not uniquely owned after pipeline"))
.into_inner()
.unwrap_or_else(|e| e.into_inner());
Ok(g)
}
// ── write_graph_as_unitigs ────────────────────────────────────────────────────
/// Phase 2 (write unitigs only): compute degrees, write unitigs to `layer_dir`, drop graph.
///
/// Returns n_kmers. Does NOT build the MPHF — caller does it.
pub fn write_graph_as_unitigs(g: GraphDeBruijn, layer_dir: &Path) -> OKIResult<usize> {
let n_kmers = g.len();
g.compute_degrees_and_mark_starts();
std::fs::create_dir_all(layer_dir)?;
let mut uw = TypedLayer::<()>::unitig_writer(layer_dir)?;
g.try_for_each_unitig(|unitig| uw.write(unitig))?;
uw.close()?;
drop(g);
Ok(n_kmers)
}
// ── materialize_layer ─────────────────────────────────────────────────────────
/// Phase 2 (full): write_graph_as_unitigs + `TypedLayer::<()>::build`.
///
/// Returns n_kmers.
pub fn materialize_layer(
g: GraphDeBruijn,
layer_dir: &Path,
block_bits: u8,
evidence: &IndexMode,
) -> OKIResult<usize> {
let n = write_graph_as_unitigs(g, layer_dir)?;
debug!("materialize_layer: unitigs written ({n} kmers), building MPHF");
TypedLayer::<()>::build(layer_dir, block_bits, evidence)?;
debug!("materialize_layer: MPHF build done");
Ok(n)
}
+3
View File
@@ -14,3 +14,6 @@
pub mod algorithms;
pub(crate) mod extensions;
pub mod graph_pipeline;
pub use graph_pipeline::{build_graph, materialize_layer, write_graph_as_unitigs};