feat: add configurable block sizes and in-place reindex command

Propagate configurable block size (`block_bits`) through index and layer construction to control unitig chunking and optimize memory/performance trade-offs. Introduce an in-place `reindex` command and library method to convert indices between exact and approximate evidence formats. Add validation to reject merging indexes with mismatched evidence types, and update parallel kmer counting to use `AtomicUsize` for thread-safe aggregation. Includes CLI argument parsing, metadata persistence, and updated tests.
This commit is contained in:
Eric Coissac
2026-05-23 12:50:03 +02:00
parent 876bc0127f
commit bc51cd9861
21 changed files with 318 additions and 51 deletions
+4 -1
View File
@@ -18,6 +18,8 @@ pub enum OKIError {
DuplicateGenomeLabel(String),
/// Operation not valid for this index configuration.
InvalidInput(String),
/// Sources mix exact and approximate evidence, or use incompatible approx parameters.
IncompatibleEvidence(String),
}
pub type OKIResult<T> = Result<T, OKIError>;
@@ -32,7 +34,8 @@ impl fmt::Display for OKIError {
OKIError::IncompatibleConfig => write!(f, "incompatible index configurations"),
OKIError::MismatchedMode => write!(f, "count mode requires all sources to have with_counts=true"),
OKIError::DuplicateGenomeLabel(l) => write!(f, "duplicate genome label across sources: {l}"),
OKIError::InvalidInput(m) => write!(f, "invalid input: {m}"),
OKIError::InvalidInput(m) => write!(f, "invalid input: {m}"),
OKIError::IncompatibleEvidence(m) => write!(f, "incompatible evidence: {m}"),
}
}
}
+2 -1
View File
@@ -145,6 +145,7 @@ impl KmerIndex {
let t = Stage::start("index");
let with_counts = self.meta.config.with_counts;
let evidence = self.meta.config.evidence.clone();
let block_bits = self.meta.config.block_bits;
let total_kmers = AtomicUsize::new(0);
let pb = Arc::new(Mutex::new(
@@ -154,7 +155,7 @@ impl KmerIndex {
));
(0..n).into_par_iter().for_each(|i| {
match self.partition.build_index_layer(i, min_ab, max_ab, with_counts, &evidence) {
match self.partition.build_index_layer(i, min_ab, max_ab, with_counts, &evidence, block_bits) {
Ok(0) => {}
Ok(n_kmers) => {
total_kmers.fetch_add(n_kmers, Ordering::Relaxed);
+1
View File
@@ -6,6 +6,7 @@ mod dump;
mod index;
mod merge;
mod rebuild;
mod reindex;
pub use error::{OKIError, OKIResult};
pub use distance::{DistanceMetric, DistanceOutput};
+36 -1
View File
@@ -9,6 +9,8 @@ use obisys::{Reporter, Stage};
use rayon::prelude::*;
use tracing::info;
use obilayeredmap::EvidenceKind;
use crate::error::{OKIError, OKIResult};
use crate::index::KmerIndex;
use crate::meta::{GenomeInfo, IndexMeta};
@@ -61,6 +63,9 @@ impl KmerIndex {
}
}
// ── Validate evidence compatibility ───────────────────────────────────
let evidence = validate_evidence_compat(sources)?;
// ── Compute final genome labels (rename duplicates if requested) ───────
let (source_labels, all_genomes) = compute_labels(sources, rename_duplicates)?;
@@ -91,6 +96,7 @@ impl KmerIndex {
let mut meta = IndexMeta::read(output).map_err(OKIError::Io)?;
meta.genomes = all_genomes;
meta.config.with_counts = mode == MergeMode::Count;
meta.config.evidence = evidence;
meta.write(output)?;
// In presence/absence mode, purge counts/ directories inherited from
@@ -134,13 +140,14 @@ impl KmerIndex {
let pb = partition_bar(n_partitions as u64);
let dst_partition = &dst.partition;
let block_bits = dst.meta.config.block_bits;
let errors: Vec<obiskio::SKError> = (0..n_partitions)
.into_par_iter()
.filter_map(|i| {
let srcs: Vec<(&obikpartitionner::KmerPartition, usize)> =
remaining_sources.iter().map(|s| (&s.partition, s.meta.genomes.len())).collect();
let result = dst_partition.merge_partition(i, &srcs, mode, n_dst_genomes).err();
let result = dst_partition.merge_partition(i, &srcs, mode, n_dst_genomes, block_bits).err();
pb.inc(1);
result
})
@@ -258,6 +265,34 @@ fn partition_bar(n: u64) -> ProgressBar {
pb
}
/// Check that all sources share the same evidence kind.
///
/// Rules:
/// - all `Exact` → OK, returns `Exact`
/// - all `Approx { b, z }` same params → OK, returns `Approx { b, z }`
/// - mixed exact/approx or different approx params → `IncompatibleEvidence`
fn validate_evidence_compat(sources: &[&KmerIndex]) -> OKIResult<EvidenceKind> {
let ref_ev = &sources[0].meta.config.evidence;
for src in &sources[1..] {
let ev = &src.meta.config.evidence;
let compat = match (ref_ev, ev) {
(EvidenceKind::Exact, EvidenceKind::Exact) => true,
(EvidenceKind::Approx { b: b1, z: z1 },
EvidenceKind::Approx { b: b2, z: z2 }) => b1 == b2 && z1 == z2,
_ => false,
};
if !compat {
return Err(OKIError::IncompatibleEvidence(format!(
"source {:?} has evidence {:?}, expected {:?}\
convert all sources to the same evidence kind first \
(use the `reindex` command)",
src.root_path.display(), ev, ref_ev,
)));
}
}
Ok(ref_ev.clone())
}
fn copy_dir_all(src: &Path, dst: &Path) -> io::Result<()> {
fs::create_dir_all(dst)?;
for entry in fs::read_dir(src)? {
+5
View File
@@ -31,6 +31,11 @@ pub struct IndexConfig {
pub with_counts: bool,
#[serde(default)]
pub evidence: EvidenceKind,
/// Block size for the unitig index as a power-of-two exponent.
/// The `.idx` block covers 2^block_bits consecutive unitigs.
/// 0 = one entry per unitig (O(1) access, largest `.idx`).
#[serde(default)]
pub block_bits: u8,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
+2 -1
View File
@@ -88,12 +88,13 @@ impl KmerIndex {
pb.enable_steady_tick(Duration::from_millis(100));
let src_partition = &src.partition;
let block_bits = meta.config.block_bits;
let errors: Vec<obiskio::SKError> = (0..n_partitions)
.into_par_iter()
.filter_map(|i| {
let result = dst_partition
.rebuild_partition(src_partition, i, filters, mode, n_genomes)
.rebuild_partition(src_partition, i, filters, mode, n_genomes, block_bits)
.err();
pb.inc(1);
result
+126
View File
@@ -0,0 +1,126 @@
use std::fs;
use std::path::Path;
use std::time::Duration;
use indicatif::{ProgressBar, ProgressStyle};
use obilayeredmap::{EvidenceKind, layer::Layer};
use obilayeredmap::meta::PartitionMeta;
use obisys::{Reporter, Stage};
use rayon::prelude::*;
use tracing::info;
use crate::error::{OKIError, OKIResult};
use crate::index::KmerIndex;
use crate::state::IndexState;
const EVIDENCE_FILE: &str = "evidence.bin";
const FINGERPRINT_FILE: &str = "fingerprint.bin";
const UNITIG_IDX_FILE: &str = "unitigs.bin.idx";
fn olm_to_oki(e: obilayeredmap::OLMError) -> OKIError {
OKIError::InvalidInput(e.to_string())
}
impl KmerIndex {
/// Convert every layer's evidence bundle to `target` in-place.
///
/// - `Exact` → builds `evidence.bin` + `unitigs.bin.idx`, removes `fingerprint.bin`
/// - `Approx` → builds `fingerprint.bin`, removes `evidence.bin` + `unitigs.bin.idx`
///
/// The MPHF (`mphf.bin`) and unitigs (`unitigs.bin`) are never touched.
/// `index.meta` is updated with the new evidence kind on success.
pub fn reindex(
&mut self,
target: EvidenceKind,
block_bits: u8,
rep: &mut Reporter,
) -> OKIResult<()> {
if self.state() != IndexState::Indexed {
return Err(OKIError::NotIndexed(self.root_path.clone()));
}
let n = self.partition.n_partitions();
info!(
"reindex {} partition(s): {:?} → {:?}",
n, self.meta.config.evidence, target,
);
let t = Stage::start("reindex");
let pb = ProgressBar::new(n as u64).with_style(
ProgressStyle::with_template(
"reindex — [{bar:20}] {pos}/{len} | {msg}",
)
.unwrap()
.tick_strings(&["","","","","","","","","",""]),
);
pb.enable_steady_tick(Duration::from_millis(80));
let errors: Vec<String> = (0..n)
.into_par_iter()
.filter_map(|i| {
let res = reindex_partition(
&self.partition.part_dir(i).join("index"),
&target,
block_bits,
);
pb.inc(1);
res.err().map(|e| format!("partition {i}: {e}"))
})
.collect();
pb.finish_and_clear();
if let Some(e) = errors.into_iter().next() {
return Err(OKIError::InvalidInput(e));
}
self.meta.config.evidence = target;
if matches!(self.meta.config.evidence, EvidenceKind::Exact) {
self.meta.config.block_bits = block_bits;
}
self.meta.write(&self.root_path)?;
rep.push(t.stop());
Ok(())
}
}
/// Process all layers of one partition's index directory.
fn reindex_partition(index_dir: &Path, target: &EvidenceKind, block_bits: u8) -> OKIResult<()> {
if !index_dir.exists() {
return Ok(());
}
let pm = PartitionMeta::load(index_dir).map_err(olm_to_oki)?;
for layer_idx in 0..pm.n_layers {
let layer_dir = index_dir.join(format!("layer_{layer_idx}"));
reindex_layer(&layer_dir, target, block_bits)?;
}
Ok(())
}
fn reindex_layer(layer_dir: &Path, target: &EvidenceKind, block_bits: u8) -> OKIResult<()> {
Layer::<()>::build_evidence(layer_dir, target, block_bits).map_err(olm_to_oki)?;
remove_stale_evidence(layer_dir, target)
}
fn remove_stale_evidence(layer_dir: &Path, target: &EvidenceKind) -> OKIResult<()> {
match target {
EvidenceKind::Exact => {
// fingerprint.bin is no longer valid
remove_if_exists(&layer_dir.join(FINGERPRINT_FILE));
}
EvidenceKind::Approx { .. } => {
// exact bundle is no longer valid
remove_if_exists(&layer_dir.join(EVIDENCE_FILE));
remove_if_exists(&layer_dir.join(UNITIG_IDX_FILE));
}
}
Ok(())
}
fn remove_if_exists(path: &Path) {
if let Err(e) = fs::remove_file(path) {
if e.kind() != std::io::ErrorKind::NotFound {
eprintln!("warning: could not remove {}: {e}", path.display());
}
}
}
+6
View File
@@ -55,6 +55,12 @@ pub fn partitions_to_bits(n: usize) -> usize {
n.max(1).next_power_of_two().trailing_zeros() as usize
}
/// Convert a block size (number of unitigs per block) to its `block_bits` exponent.
/// `block_size=1` → `block_bits=0` (one entry per unitig, O(1) random access).
pub fn block_size_to_bits(n: usize) -> u8 {
n.max(1).next_power_of_two().trailing_zeros() as u8
}
impl CommonArgs {
pub fn seqfile_paths(&self) -> obiread::PathIter {
let paths = self.inputs.iter().map(PathBuf::from).collect();
+8 -1
View File
@@ -12,7 +12,7 @@ use obikseq::{set_k, set_m};
use obisys::Reporter;
use tracing::info;
use crate::cli::{CommonArgs, partitions_to_bits};
use crate::cli::{CommonArgs, block_size_to_bits, partitions_to_bits};
use crate::steps::scatter;
#[derive(Args)]
@@ -68,6 +68,11 @@ pub struct IndexArgs {
#[arg(long, default_value = None)]
pub fp: Option<f64>,
/// Block size for exact evidence `.idx` (number of unitigs per block).
/// Must be a power of two; rounded up if not. Default 1 = O(1) random access.
#[arg(long, default_value_t = 1)]
pub block_size: usize,
#[command(flatten)]
pub common: CommonArgs,
}
@@ -179,12 +184,14 @@ pub fn run(args: IndexArgs) {
if effective != args.common.partitions {
info!("partitions: {} → {} (next power of 2)", args.common.partitions, effective);
}
let block_bits = block_size_to_bits(args.block_size);
let config = IndexConfig {
kmer_size: args.common.kmer_size,
minimizer_size: args.common.minimizer_size,
n_bits,
with_counts: args.with_counts,
evidence: evidence.clone(),
block_bits,
};
let genome_info = args.label.as_ref().map(|label| {
let mut info = GenomeInfo::new(label.clone());
+1
View File
@@ -6,5 +6,6 @@ pub mod index;
pub mod merge;
pub mod query;
pub mod rebuild;
pub mod reindex;
pub mod superkmer;
pub mod unitig;
+68
View File
@@ -0,0 +1,68 @@
use std::path::PathBuf;
use clap::Args;
use obikindex::KmerIndex;
use obilayeredmap::EvidenceKind;
use obisys::Reporter;
use tracing::info;
use crate::cli::block_size_to_bits;
use super::index::resolve_approx_params;
#[derive(Args)]
pub struct ReindexArgs {
/// Index directory to convert (modified in-place)
pub index: PathBuf,
/// Convert to approximate evidence (default: convert to exact).
/// Requires --evidence-bits and/or -z and/or --fp.
#[arg(long, default_value_t = false)]
pub approx: bool,
/// Findere z parameter (≥1).
#[arg(short = 'z', long, default_value = None)]
pub findere_z: Option<u8>,
/// Fingerprint bits per slot (b).
#[arg(long, default_value = None)]
pub evidence_bits: Option<u8>,
/// Target false-positive rate per z-window.
#[arg(long, default_value = None)]
pub fp: Option<f64>,
/// Block size for exact evidence `.idx` (number of unitigs per block).
/// Ignored when converting to approximate evidence.
#[arg(long, default_value_t = 1)]
pub block_size: usize,
}
pub fn run(args: ReindexArgs) {
let target = if args.approx {
let (z, b, fp) = resolve_approx_params(args.findere_z, args.evidence_bits, args.fp);
info!("target: approximate evidence — b={b}, z={z}, fp={fp:.2e}");
EvidenceKind::Approx { b, z }
} else {
info!("target: exact evidence");
EvidenceKind::Exact
};
let mut idx = KmerIndex::open(&args.index).unwrap_or_else(|e| {
eprintln!("error opening index: {e}");
std::process::exit(1);
});
info!(
"current evidence: {:?}",
idx.meta().config.evidence,
);
let block_bits = block_size_to_bits(args.block_size);
let mut rep = Reporter::new();
idx.reindex(target, block_bits, &mut rep).unwrap_or_else(|e| {
eprintln!("reindex error: {e}");
std::process::exit(1);
});
rep.print();
}
+3 -3
View File
@@ -35,13 +35,13 @@ pub fn run(args: UnitigArgs) {
return;
}
let reader = UnitigFileReader::open(&path).unwrap_or_else(|e| {
// open_sequential: works with and without .idx (approx or exact index)
let reader = UnitigFileReader::open_sequential(&path).unwrap_or_else(|e| {
eprintln!("error opening unitigs (partition {i}): {e}");
std::process::exit(1)
});
for j in 0..reader.len() {
let unitig = reader.unitig(j);
for (j, unitig) in reader.iter_unitigs() {
let mut out = stdout.lock().unwrap();
write_unitig(&unitig, k, i, j, &mut *out).unwrap_or_else(|e| {
eprintln!("write error: {e}");
+3
View File
@@ -34,6 +34,8 @@ enum Commands {
Unitig(cmd::unitig::UnitigArgs),
/// Estimate approximate-index parameters (z, evidence bits, FP rates) before indexing
Estimate(cmd::estimate::EstimateArgs),
/// Convert an index's evidence in-place: exact ↔ approx
Reindex(cmd::reindex::ReindexArgs),
}
fn main() {
@@ -65,6 +67,7 @@ fn main() {
Commands::Distance(args) => cmd::distance::run(args),
Commands::Unitig(args) => cmd::unitig::run(args),
Commands::Estimate(args) => cmd::estimate::run(args),
Commands::Reindex(args) => cmd::reindex::run(args),
}
#[cfg(feature = "profiling")]
+3 -2
View File
@@ -45,6 +45,7 @@ impl KmerPartition {
max_ab: Option<u32>,
with_counts: bool,
evidence: &EvidenceKind,
block_bits: u8,
) -> Result<usize, SKError> {
let part_dir = self.part_dir(i);
let dedup_path = part_dir.join("dereplicated.skmer.zst");
@@ -109,7 +110,7 @@ impl KmerPartition {
uw.close()?;
if with_counts {
Layer::<PersistentCompactIntMatrix>::build(&layer_dir, |kmer| {
Layer::<PersistentCompactIntMatrix>::build(&layer_dir, block_bits, |kmer| {
match (&mphf1_opt, &counts1_opt) {
(Some(mphf), Some(counts)) => counts.get(mphf.index(&kmer.raw())),
_ => 1,
@@ -117,7 +118,7 @@ impl KmerPartition {
})
.map_err(olm_to_sk)?;
} else {
Layer::<()>::build(&layer_dir).map_err(olm_to_sk)?;
Layer::<()>::build(&layer_dir, block_bits).map_err(olm_to_sk)?;
}
// For approximate evidence: replace the exact evidence bundle with a
+2 -1
View File
@@ -161,6 +161,7 @@ impl KmerPartition {
sources: &[(&KmerPartition, usize)],
mode: MergeMode,
n_dst_genomes: usize,
block_bits: u8,
) -> SKResult<()> {
let dst_index_dir = self.part_dir(i).join(INDEX_SUBDIR);
if !dst_index_dir.exists() {
@@ -216,7 +217,7 @@ impl KmerPartition {
uw.write(&unitig)?;
}
uw.close()?;
Layer::<()>::build(&new_layer_dir).map_err(olm_to_sk)?;
Layer::<()>::build(&new_layer_dir, block_bits).map_err(olm_to_sk)?;
}
drop(g);
+2 -1
View File
@@ -96,6 +96,7 @@ impl KmerPartition {
filters: &[Box<dyn KmerFilter>],
mode: MergeMode,
n_genomes: usize,
block_bits: u8,
) -> SKResult<()> {
let src_index_dir = src.part_dir(i).join(INDEX_SUBDIR);
if !src_index_dir.exists() {
@@ -145,7 +146,7 @@ impl KmerPartition {
uw.close()?;
drop(g);
Layer::<()>::build(&dst_layer_dir).map_err(olm_to_sk)?;
Layer::<()>::build(&dst_layer_dir, block_bits).map_err(olm_to_sk)?;
let dst_mphf = MphfLayer::open(&dst_layer_dir).map_err(olm_to_sk)?;
// ── Prepare matrix builders (one column per genome) ───────────────────
+21 -23
View File
@@ -80,39 +80,33 @@ impl<D: LayerData> Layer<D> {
/// Build `unitigs.bin.idx` and `evidence.bin` from `unitigs.bin` and
/// `mphf.bin` already present in `layer_dir`.
///
/// See [`MphfLayer::build_exact_evidence`] for the full contract.
pub fn build_exact_evidence(layer_dir: &Path) -> OLMResult<usize> {
MphfLayer::build_exact_evidence(layer_dir)
/// `block_bits` controls the `.idx` block size (2^block_bits chunks/block).
pub fn build_exact_evidence(layer_dir: &Path, block_bits: u8) -> OLMResult<usize> {
MphfLayer::build_exact_evidence(layer_dir, block_bits)
}
/// Build `fingerprint.bin` from `unitigs.bin` and `mphf.bin` already
/// present in `layer_dir`. `b` — fingerprint bits (1..=64); `z` — Findere
/// consecutive k-mer parameter (≥1).
///
/// See [`MphfLayer::build_approx_evidence`] for the full contract.
pub fn build_approx_evidence(layer_dir: &Path, b: u8, z: u8) -> OLMResult<usize> {
MphfLayer::build_approx_evidence(layer_dir, b, z)
}
/// Dispatch to `build_exact_evidence` or `build_approx_evidence` based on
/// `kind`.
pub fn build_evidence(layer_dir: &Path, kind: &EvidenceKind) -> OLMResult<usize> {
MphfLayer::build_evidence(layer_dir, kind)
/// Dispatch to `build_exact_evidence` or `build_approx_evidence`.
/// `block_bits` is forwarded to exact evidence only.
pub fn build_evidence(layer_dir: &Path, kind: &EvidenceKind, block_bits: u8) -> OLMResult<usize> {
MphfLayer::build_evidence(layer_dir, kind, block_bits)
}
}
// ── Mode 1 — set membership ───────────────────────────────────────────────────
impl Layer<()> {
pub fn build(out_dir: &Path) -> OLMResult<usize> {
MphfLayer::build(out_dir, &mut |_, _| Ok(()))
pub fn build(out_dir: &Path, block_bits: u8) -> OLMResult<usize> {
MphfLayer::build(out_dir, block_bits, &mut |_, _| Ok(()))
}
/// Create a presence matrix for a set-membership layer (first merge).
///
/// All `n_kmers` slots are set to `true`: every kmer in this layer belongs
/// to genome_0, so genome_0 is present at every slot.
pub fn init_presence_matrix(layer_dir: &Path, n_kmers: usize) -> OLMResult<()> {
let presence_dir = layer_dir.join(PRESENCE_DIR);
fs::create_dir_all(&presence_dir).map_err(OLMError::Io)?;
@@ -126,16 +120,20 @@ impl Layer<()> {
}
}
// ── Mode 2 — count matrix (1 column per layer) ────────────────────────────────
// ── Mode 2 — count matrix ─────────────────────────────────────────────────────
impl Layer<PersistentCompactIntMatrix> {
pub fn build(out_dir: &Path, count_of: impl Fn(CanonicalKmer) -> u32) -> OLMResult<usize> {
pub fn build(
out_dir: &Path,
block_bits: u8,
count_of: impl Fn(CanonicalKmer) -> u32,
) -> OLMResult<usize> {
let n = UnitigFileReader::open_sequential(&out_dir.join(UNITIGS_FILE))?.n_kmers();
let counts_dir = out_dir.join(COUNTS_DIR);
let mut mb = PersistentCompactIntMatrixBuilder::new(n, &counts_dir)
.map_err(OLMError::Io)?;
let mut col = mb.add_col().map_err(OLMError::Io)?;
let n_built = MphfLayer::build(out_dir, &mut |slot, kmer| {
let n_built = MphfLayer::build(out_dir, block_bits, &mut |slot, kmer| {
col.set(slot, count_of(kmer));
Ok(())
})?;
@@ -146,16 +144,16 @@ impl Layer<PersistentCompactIntMatrix> {
pub fn build_from_map(
out_dir: &Path,
block_bits: u8,
counts: &HashMap<CanonicalKmer, u32>,
) -> OLMResult<usize> {
Self::build(out_dir, |kmer| counts.get(&kmer).copied().unwrap_or(0))
Self::build(out_dir, block_bits, |kmer| counts.get(&kmer).copied().unwrap_or(0))
}
}
// ── Mode 2 — count matrix column append ──────────────────────────────────────
impl Layer<PersistentCompactIntMatrix> {
/// Append a genome column to an existing count matrix.
pub fn append_genome_column(
layer_dir: &Path,
value_of: impl Fn(usize) -> u32,
@@ -165,10 +163,9 @@ impl Layer<PersistentCompactIntMatrix> {
}
}
// ── Mode 3 — presence/absence matrix (1 column per genome) ───────────────────
// ── Mode 3 — presence/absence matrix ─────────────────────────────────────────
impl Layer<PersistentBitMatrix> {
/// Append a genome column to an existing presence matrix.
pub fn append_genome_column(
layer_dir: &Path,
value_of: impl Fn(usize) -> bool,
@@ -179,6 +176,7 @@ impl Layer<PersistentBitMatrix> {
pub fn build_presence(
out_dir: &Path,
block_bits: u8,
n_genomes: usize,
present_in: impl Fn(CanonicalKmer, usize) -> bool,
) -> OLMResult<usize> {
@@ -188,7 +186,7 @@ impl Layer<PersistentBitMatrix> {
let mut cols: Vec<_> = (0..n_genomes)
.map(|_| mb.add_col().map_err(OLMError::Io))
.collect::<OLMResult<_>>()?;
let n_built = MphfLayer::build(out_dir, &mut |slot, kmer| {
let n_built = MphfLayer::build(out_dir, block_bits, &mut |slot, kmer| {
for (g, col) in cols.iter_mut().enumerate() {
col.set(slot, present_in(kmer, g));
}
+3 -3
View File
@@ -4,7 +4,7 @@ use std::path::{Path, PathBuf};
use obicompactvec::PersistentCompactIntMatrix;
use obikseq::CanonicalKmer;
use obiskio::UnitigFileWriter;
use obiskio::{UnitigFileWriter, DEFAULT_BLOCK_BITS};
use crate::error::OLMResult;
use crate::layer::{Hit, Layer, LayerData};
@@ -90,7 +90,7 @@ impl LayeredMap<()> {
pub fn push_layer(&mut self) -> OLMResult<usize> {
let i = self.layers.len();
let dir = layer_dir(&self.root, i);
Layer::<()>::build(&dir)?;
Layer::<()>::build(&dir, DEFAULT_BLOCK_BITS)?;
self.append_layer()?;
Ok(i)
}
@@ -102,7 +102,7 @@ impl LayeredMap<PersistentCompactIntMatrix> {
pub fn push_layer(&mut self, count_of: impl Fn(CanonicalKmer) -> u32) -> OLMResult<usize> {
let i = self.layers.len();
let dir = layer_dir(&self.root, i);
Layer::<PersistentCompactIntMatrix>::build(&dir, count_of)?;
Layer::<PersistentCompactIntMatrix>::build(&dir, DEFAULT_BLOCK_BITS, count_of)?;
self.append_layer()?;
Ok(i)
}
+11 -9
View File
@@ -4,7 +4,7 @@ use std::path::Path;
use cacheline_ef::{CachelineEf, CachelineEfVec};
use epserde::prelude::*;
use obikseq::CanonicalKmer;
use obiskio::{UnitigFileReader, UnitigFileWriter, build_unitig_idx, DEFAULT_BLOCK_BITS};
use obiskio::{UnitigFileReader, UnitigFileWriter, build_unitig_idx};
use ptr_hash::{PtrHash, PtrHashParams, bucket_fn::CubicEps, hash::Xx64};
use crate::error::{OLMError, OLMResult};
@@ -110,25 +110,26 @@ impl MphfLayer {
}
/// Dispatch to `build_exact_evidence` or `build_approx_evidence` based on
/// `kind`.
pub fn build_evidence(dir: &Path, kind: &EvidenceKind) -> OLMResult<usize> {
/// `kind`. `block_bits` is forwarded to exact evidence only.
pub fn build_evidence(dir: &Path, kind: &EvidenceKind, block_bits: u8) -> OLMResult<usize> {
match kind {
EvidenceKind::Exact => Self::build_exact_evidence(dir),
EvidenceKind::Approx { b, z } => Self::build_approx_evidence(dir, *b, *z),
EvidenceKind::Exact => Self::build_exact_evidence(dir, block_bits),
EvidenceKind::Approx { b, z } => Self::build_approx_evidence(dir, *b, *z),
}
}
/// Build `evidence.bin` + `unitigs.bin.idx` from `unitigs.bin` + `mphf.bin`.
///
/// `block_bits` controls the `.idx` block size (2^block_bits chunks per block).
/// Uses sequential iteration — no `.idx` required on entry.
pub fn build_exact_evidence(dir: &Path) -> OLMResult<usize> {
pub fn build_exact_evidence(dir: &Path, block_bits: u8) -> OLMResult<usize> {
let unitig_path = dir.join(UNITIGS_FILE);
let unitigs = UnitigFileReader::open_sequential(&unitig_path)?;
let n = unitigs.n_kmers();
if n == 0 {
fs::File::create(dir.join(EVIDENCE_FILE))?;
build_unitig_idx(&unitig_path, DEFAULT_BLOCK_BITS)?;
build_unitig_idx(&unitig_path, block_bits)?;
LayerMeta::exact().save(dir)?;
return Ok(0);
}
@@ -154,7 +155,7 @@ impl MphfLayer {
}
ev.write(&dir.join(EVIDENCE_FILE))?;
build_unitig_idx(&unitig_path, DEFAULT_BLOCK_BITS)?;
build_unitig_idx(&unitig_path, block_bits)?;
LayerMeta::exact().save(dir)?;
Ok(n)
}
@@ -202,13 +203,14 @@ impl MphfLayer {
/// population. Returns the number of kmers indexed.
pub(crate) fn build(
dir: &Path,
block_bits: u8,
fill_slot: &mut impl FnMut(usize, CanonicalKmer) -> OLMResult<()>,
) -> OLMResult<usize> {
use rayon::prelude::*;
let unitig_path = dir.join(UNITIGS_FILE);
build_unitig_idx(&unitig_path, DEFAULT_BLOCK_BITS)?;
build_unitig_idx(&unitig_path, block_bits)?;
let unitigs = UnitigFileReader::open(&unitig_path)?;
let n = unitigs.n_kmers();
+5 -3
View File
@@ -1,6 +1,7 @@
use super::*;
use obicompactvec::PersistentCompactIntMatrix;
use obikseq::{set_k, Kmer, Sequence as _, Unitig};
use obiskio::DEFAULT_BLOCK_BITS;
use tempfile::tempdir;
fn write_unitigs(dir: &Path, seqs: &[&[u8]]) {
@@ -23,7 +24,7 @@ fn build_and_query_all_kmers_found() {
let dir = tempdir().unwrap();
write_unitigs(dir.path(), &[b"AAAACGT"]);
let kmers = all_canonical_kmers(dir.path());
Layer::<()>::build(dir.path()).unwrap();
Layer::<()>::build(dir.path(), DEFAULT_BLOCK_BITS).unwrap();
let layer = Layer::<()>::open(dir.path()).unwrap();
for kmer in kmers {
assert!(layer.query(kmer).is_some(), "kmer should be present");
@@ -40,6 +41,7 @@ fn counts_are_stored_and_retrieved() {
kmers.iter().enumerate().map(|(i, &k)| (k, i as u32 + 1)).collect();
Layer::<PersistentCompactIntMatrix>::build(
dir.path(),
DEFAULT_BLOCK_BITS,
|kmer| count_map.get(&kmer).copied().unwrap_or(0),
).unwrap();
let layer = Layer::<PersistentCompactIntMatrix>::open(dir.path()).unwrap();
@@ -54,7 +56,7 @@ fn query_absent_returns_none() {
set_k(4);
let dir = tempdir().unwrap();
write_unitigs(dir.path(), &[b"AAAACGT"]);
Layer::<()>::build(dir.path()).unwrap();
Layer::<()>::build(dir.path(), DEFAULT_BLOCK_BITS).unwrap();
let layer = Layer::<()>::open(dir.path()).unwrap();
let absent = Kmer::from_ascii(b"CCCC").unwrap().canonical();
assert!(layer.query(absent).is_none());
@@ -65,7 +67,7 @@ fn open_after_build_is_consistent() {
set_k(4);
let dir = tempdir().unwrap();
write_unitigs(dir.path(), &[b"AAAACGT"]);
let n = Layer::<PersistentCompactIntMatrix>::build(dir.path(), |_| 7).unwrap();
let n = Layer::<PersistentCompactIntMatrix>::build(dir.path(), DEFAULT_BLOCK_BITS, |_| 7).unwrap();
assert_eq!(n, 4);
let layer = Layer::<PersistentCompactIntMatrix>::open(dir.path()).unwrap();
let kmer = Kmer::from_ascii(b"AAAA").unwrap().canonical();
+6 -1
View File
@@ -21,7 +21,7 @@ use crate::error::{SKError, SKResult};
const MAGIC: [u8; 4] = *b"UIX3";
/// Default block granularity used by [`UnitigFileWriter::create`].
pub const DEFAULT_BLOCK_BITS: u8 = 6;
pub const DEFAULT_BLOCK_BITS: u8 = 0;
fn idx_path(path: &Path) -> PathBuf {
crate::append_path_suffix(path, ".idx")
@@ -325,6 +325,11 @@ impl UnitigFileReader {
})
}
/// Iterate all unitigs sequentially. Works without `.idx` (sequential open).
pub fn iter_unitigs(&self) -> impl Iterator<Item = (usize, Unitig)> + '_ {
self.iter_chunks_sequential()
}
pub fn iter_kmers(&self) -> impl Iterator<Item = Kmer> + '_ {
self.iter_chunks_sequential()
.flat_map(|(_, u)| u.into_kmers())