From cbf6893f3819c4e23db25a2039fda6a6398d9a84 Mon Sep 17 00:00:00 2001 From: Eric Coissac Date: Sat, 22 Aug 2026 15:25:00 +0200 Subject: [PATCH] Add configurable sparsity flag to index finalization and merge CLI Extends `KmerIndex::finalize_indexed` to accept a `sparse` boolean parameter, enabling control over whether output presence matrices are packed in a compact sparse format or stored densely. The merge command now exposes a `--dense` flag to invert this setting, defaulting to the existing sparse representation. Call sites in select and rebuild modules pass `false` to preserve current behavior. A benchmark lock file is also added for dependency tracking. --- benchmark/tmp/bacteria_sparse/.obikmer.lock | 0 src/obikindex/src/index/builder.rs | 11 ++++++----- src/obikmer2/src/cmd/merge/mod.rs | 5 +++++ src/obikmerge/src/merge.rs | 11 ++++++++++- src/obikrebuild/src/rebuild.rs | 2 +- src/obikselect/src/select.rs | 2 +- 6 files changed, 23 insertions(+), 8 deletions(-) create mode 100644 benchmark/tmp/bacteria_sparse/.obikmer.lock diff --git a/benchmark/tmp/bacteria_sparse/.obikmer.lock b/benchmark/tmp/bacteria_sparse/.obikmer.lock new file mode 100644 index 00000000..e69de29b diff --git a/src/obikindex/src/index/builder.rs b/src/obikindex/src/index/builder.rs index 2a24f7f7..0ce5144e 100644 --- a/src/obikindex/src/index/builder.rs +++ b/src/obikindex/src/index/builder.rs @@ -52,9 +52,10 @@ pub trait IndexBuilder: Sized { /// Mark `output` as fully indexed, pack its column matrices, and reopen it. /// - /// Shared tail of the `select`/`rebuild` construction paths, once their - /// partitions have been written. - fn finalize_indexed>(output: P, rep: &mut Reporter) -> OKIResult; + /// Shared tail of the `select`/`rebuild`/`merge` construction paths, + /// once their partitions have been written. `sparse` selects the + /// on-disk presence-matrix format (see `pack_matrices`). + fn finalize_indexed>(output: P, rep: &mut Reporter, sparse: bool) -> OKIResult; /// Current construction state, read fresh from `index.meta` (see /// `DevDocMD/implementation/partition_layer_cache.md`, "(11)" — no @@ -99,12 +100,12 @@ impl IndexBuilder for KmerIndex { Ok(idx) } - fn finalize_indexed>(output: P, rep: &mut Reporter) -> OKIResult { + fn finalize_indexed>(output: P, rep: &mut Reporter, sparse: bool) -> OKIResult { let output = output.as_ref(); let idx = KmerIndex::open(output)?; idx.meta.mark_indexed().map_err(OKIError::Io)?; let t_pack = Stage::start("pack"); - idx.pack_matrices(false)?; + idx.pack_matrices(sparse)?; rep.push(t_pack.stop()); Ok(idx) } diff --git a/src/obikmer2/src/cmd/merge/mod.rs b/src/obikmer2/src/cmd/merge/mod.rs index d1545e7d..04607618 100644 --- a/src/obikmer2/src/cmd/merge/mod.rs +++ b/src/obikmer2/src/cmd/merge/mod.rs @@ -28,6 +28,10 @@ pub struct MergeArgs { /// Disambiguate duplicate genome labels by appending .1, .2, … instead of erroring #[arg(long, default_value_t = false)] pub rename_duplicates: bool, + + /// Pack the output's presence matrices in the dense format instead of the default sparse one + #[arg(long, default_value_t = false)] + pub dense: bool, } pub fn run(args: MergeArgs) { @@ -89,6 +93,7 @@ pub fn run(args: MergeArgs) { let mut merge = Merge::new(&source_refs, &args.output, mode) .force(args.force) .rename_duplicates(args.rename_duplicates) + .sparse(!args.dense) .on_progress(|_: Progress| pb.inc(1)); let dst = merge.run().unwrap_or_else(|e| { diff --git a/src/obikmerge/src/merge.rs b/src/obikmerge/src/merge.rs index e4c5a120..b204759b 100644 --- a/src/obikmerge/src/merge.rs +++ b/src/obikmerge/src/merge.rs @@ -45,6 +45,7 @@ pub struct Merge<'a> { mode: MergeMode, force: bool, rename_duplicates: bool, + sparse: bool, reporter: Reporter, on_progress: Option>, } @@ -57,6 +58,7 @@ impl<'a> Merge<'a> { mode, force: false, rename_duplicates: false, + sparse: true, reporter: Reporter::new(), on_progress: None, } @@ -76,6 +78,13 @@ impl<'a> Merge<'a> { self } + /// Pack the merged output's presence matrices in the compact sparse + /// format rather than dense (default: `true`). + pub fn sparse(mut self, v: bool) -> Self { + self.sparse = v; + self + } + /// Progress callback, called once per completed partition. pub fn on_progress(mut self, cb: impl FnMut(Progress) + Send + 'a) -> Self { self.on_progress = Some(Box::new(cb)); @@ -322,7 +331,7 @@ impl Algorithm for Merge<'_> { // ── Finalize: pack matrices, mark indexed, reopen ────────────────────── let t = Stage::start("finalize"); - let dst = KmerIndex::finalize_indexed(&output, &mut self.reporter)?; + let dst = KmerIndex::finalize_indexed(&output, &mut self.reporter, self.sparse)?; self.reporter.push(t.stop()); Ok(dst) diff --git a/src/obikrebuild/src/rebuild.rs b/src/obikrebuild/src/rebuild.rs index 67746572..c7945328 100644 --- a/src/obikrebuild/src/rebuild.rs +++ b/src/obikrebuild/src/rebuild.rs @@ -74,6 +74,6 @@ impl KmerIndex { rep.push(t.stop()); - KmerIndex::finalize_indexed(output, rep) + KmerIndex::finalize_indexed(output, rep, false) } } diff --git a/src/obikselect/src/select.rs b/src/obikselect/src/select.rs index ca00620b..7b05aeeb 100644 --- a/src/obikselect/src/select.rs +++ b/src/obikselect/src/select.rs @@ -82,7 +82,7 @@ impl KmerIndex { pb.finish_and_clear(); rep.push(t.stop()); - KmerIndex::finalize_indexed(output, rep) + KmerIndex::finalize_indexed(output, rep, false) } /// Rewrite the genome columns of this index in-place according to `specs`.