Extract index modules into specialized workspace subcrates

This commit partitions the obikindex crate into multiple focused subcrates (obikfilter, obikmerge, obikquery, obikrebuild, obikselect, obikstats, obikdump, and obikidxcache) to reduce coupling and clarify module boundaries. It standardizes error handling across the workspace using OKIError and OKIResult, updates index APIs to support lazy, disk-backed partition access, and migrates NUMA system utilities to a new obisys crate. All modifications are structural, focusing on dependency graph expansion, import path updates, and API surface reorganization without altering core runtime behavior.
This commit is contained in:
Eric Coissac
2026-08-22 06:25:28 +02:00
parent c9d10d55c7
commit fc4464a0ef
81 changed files with 1640 additions and 1196 deletions
+24
View File
@@ -0,0 +1,24 @@
[package]
name = "obikmer2"
version = "1.2.2"
edition = "2024"
[[bin]]
name = "obikmer2"
path = "src/main.rs"
[dependencies]
obikseq = { path = "../obikseq" }
obiread = { path = "../obiread" }
obipipeline = { path = "../obipipeline" }
obisys = { path = "../obisys" }
obikindex = { path = "../obikindex", default-features = false }
obikindexer = { path = "../obikindexer" }
obikalgorithm = { path = "../obikalgorithm" }
clap = { version = "4", features = ["derive"] }
tracing = "0.1.44"
tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] }
[features]
default = ["numa"]
numa = ["obisys/numa"]
+114
View File
@@ -0,0 +1,114 @@
use std::path::PathBuf;
use clap::Args;
use obiread::NucPage;
use obikseq::RoutableSuperKmer;
use obipipeline::Throttled;
// ── Shared arguments ──────────────────────────────────────────────────────────
#[derive(Args)]
pub struct CommonArgs {
/// Input files or directories (FASTA/FASTQ, optionally gzip-compressed).
/// If omitted, reads from stdin.
#[arg(num_args = 0..)]
pub inputs: Vec<String>,
/// k-mer size
#[arg(short, long, default_value_t = 31)]
pub kmer_size: usize,
/// Minimizer size
#[arg(short, long, default_value_t = 11)]
pub minimizer_size: usize,
/// Entropy threshold (k-mers with score ≤ theta are rejected)
#[arg(long, default_value_t = 0.7)]
pub theta: f64,
/// Maximum sub-word size for entropy computation
#[arg(long, default_value_t = 6)]
pub level_max: usize,
/// Number of partitions (rounded up to the next power of 2)
#[arg(short, long, default_value_t = 256)]
pub partitions: usize,
/// Number of worker threads
#[arg(
short = 'T',
long,
default_value_t = obisys::effective_parallelism()
)]
pub threads: usize,
/// Maximum number of input files open simultaneously.
/// Defaults to threads/4 (minimum 1). Keep below the number of workers
/// to ensure CPU workers are always available for the transform stage.
#[arg(long)]
pub max_open_files: Option<usize>,
}
/// Smallest `b` such that `2^b >= n` (i.e. `n.next_power_of_two().ilog2()`).
/// Minimum 1 (degenerate n=0 or n=1 → 1 partition).
pub fn partitions_to_bits(n: usize) -> usize {
n.max(1).next_power_of_two().trailing_zeros() as usize
}
/// Convert a block size (number of unitigs per block) to its `block_bits` exponent.
/// `block_size=1` → `block_bits=0` (one entry per unitig, O(1) random access).
pub fn block_size_to_bits(n: usize) -> u8 {
n.max(1).next_power_of_two().trailing_zeros() as u8
}
impl CommonArgs {
/// Validate k and m constraints. Exits on error.
pub fn validate(&self) {
let k = self.kmer_size;
let m = self.minimizer_size;
if k < 11 || k > 31 {
eprintln!("error: --kmer-size must be in [11, 31] (got {k})");
std::process::exit(1);
}
if k % 2 == 0 {
eprintln!("error: --kmer-size must be odd (got {k}); even k allows palindromic k-mers");
std::process::exit(1);
}
if m < 3 || m >= k {
eprintln!("error: --minimizer-size must be in [3, k−1] = [3, {}] (got {m})", k - 1);
std::process::exit(1);
}
if m % 2 == 0 {
eprintln!("error: --minimizer-size must be odd (got {m})");
std::process::exit(1);
}
}
pub fn effective_max_open(&self) -> usize {
self.max_open_files
.unwrap_or_else(|| (self.threads / 4).max(1))
.max(1)
}
pub fn seqfile_paths(&self) -> obiread::PathIter {
let paths: Vec<PathBuf> = if self.inputs.is_empty() {
vec![PathBuf::from("-")]
} else {
self.inputs.iter().map(PathBuf::from).collect()
};
obiread::PathIter::new(paths)
}
}
// ── Pipeline data carrier ─────────────────────────────────────────────────────
pub enum PipelineData {
Path(Throttled<PathBuf>),
NucPage(NucPage),
Batch(Vec<RoutableSuperKmer>),
}
unsafe impl Send for PipelineData {}
unsafe impl Sync for PipelineData {}
+363
View File
@@ -0,0 +1,363 @@
use std::path::PathBuf;
use std::time::Instant;
use clap::Args;
use obikalgorithm::Algorithm;
use obikindex::layer::IndexMode;
use obikindex::{GenomeInfo, IndexBuilder, IndexConfig, IndexState, KmerIndex};
use obikindexer::algorithms::counter::Counter;
use obikindexer::algorithms::dereplicator::Dereplicator;
use obikindexer::algorithms::layer_builder::LayerBuilder;
use obikindexer::algorithms::partitionner::PartitionRouter;
fn current_state(idx: &KmerIndex) -> IndexState {
idx.state().unwrap_or_else(|e| {
eprintln!("error reading index metadata: {e}");
std::process::exit(1);
})
}
fn parse_key_value(s: &str) -> Result<(String, String), String> {
let pos = s
.find('=')
.ok_or_else(|| format!("invalid key=value: no '=' in '{s}'"))?;
Ok((s[..pos].to_string(), s[pos + 1..].to_string()))
}
use obisys::{Progress, Reporter, Stage, progress_bar, spinner};
use tracing::info;
use crate::cli::{CommonArgs, block_size_to_bits, partitions_to_bits};
#[derive(Args)]
pub struct IndexArgs {
/// Output index directory
#[arg(short, long)]
pub output: PathBuf,
/// Overwrite output directory if it already exists
#[arg(long, default_value_t = false)]
pub force: bool,
/// Genome label (default: input filename without path/extension)
#[arg(long)]
pub label: Option<String>,
/// Genome categorical metadata as key=value pairs (repeatable)
#[arg(long = "meta", value_parser = parse_key_value)]
pub meta: Vec<(String, String)>,
/// Minimum kmer abundance (inclusive)
#[arg(long, default_value_t = 1)]
pub min_abundance: u32,
/// Maximum kmer abundance (inclusive)
#[arg(long)]
pub max_abundance: Option<u32>,
/// Store kmer counts in the index (default: set membership only)
#[arg(long, default_value_t = false)]
pub with_counts: bool,
/// Keep intermediate build files (dereplicated superkmers, mphf1, counts1)
#[arg(long, default_value_t = false)]
pub keep_intermediate: bool,
/// Use approximate (fingerprint-based) evidence instead of exact evidence.
/// False-positive rate per z-window: 1/2^(b·z).
#[arg(long, default_value_t = false)]
pub approx: bool,
/// Findere z parameter: number of consecutive k-mers that must all match.
/// Effective indexed k-mer size is kmer_size - z + 1.
#[arg(short = 'z', long, default_value = None)]
pub findere_z: Option<u8>,
/// Fingerprint bits per slot (b). FP per z-window = 1/2^(b·z).
#[arg(long, default_value = None)]
pub evidence_bits: Option<u8>,
/// Target false-positive rate per z-window (e.g. 0.01).
/// Used to derive missing b or z.
#[arg(long, default_value = None)]
pub fp: Option<f64>,
/// Block size for exact evidence `.idx` (number of unitigs per block).
/// Must be a power of two; rounded up if not. Default 1 = O(1) random access.
#[arg(long, default_value_t = 1)]
pub block_size: usize,
#[command(flatten)]
pub common: CommonArgs,
}
/// Resolve the (z, b, fp) triplet from the user-supplied subset.
///
/// Model: FP = 1/2^(b·z) ⟹ b·z = ⌈-log₂(fp)⌉
///
/// Rules when one value is missing (conservative = ceiling):
/// given z, b → fp = 1/2^(b·z)
/// given z, fp → b = ⌈-log₂(fp) / z⌉
/// given b, fp → z = ⌈-log₂(fp) / b⌉
/// given z only → b = 8 (default), fp derived
/// given b only → z = 1 (default), fp derived
/// given fp only → b = 8 (default), z derived
/// none given → z = 1, b = 8, fp = 1/256
pub(crate) fn resolve_approx_params(
z_opt: Option<u8>,
b_opt: Option<u8>,
fp_opt: Option<f64>,
) -> (u8, u8, f64) {
const DEFAULT_B: u8 = 8;
const DEFAULT_Z: u8 = 1;
let bits_needed = |fp: f64| -> u8 { (-fp.log2()).ceil() as u8 };
match (z_opt, b_opt, fp_opt) {
// All three given: use b and z, recompute fp conservatively.
(Some(z), Some(b), Some(_fp)) => {
let fp = 1.0_f64 / (1u64 << (b as u32 * z as u32)) as f64;
(z, b, fp)
}
// Two given, derive third.
(Some(z), Some(b), None) => {
let fp = 1.0_f64 / (1u64 << (b as u32 * z as u32)) as f64;
(z, b, fp)
}
(Some(z), None, Some(fp)) => {
let bz = (-fp.log2()).ceil() as u32;
let b = ((bz + z as u32 - 1) / z as u32).max(1) as u8;
let actual_fp = 1.0_f64 / (1u64 << (b as u32 * z as u32)) as f64;
(z, b, actual_fp)
}
(None, Some(b), Some(fp)) => {
let bz = (-fp.log2()).ceil() as u32;
let z = ((bz + b as u32 - 1) / b as u32).max(1) as u8;
let actual_fp = 1.0_f64 / (1u64 << (b as u32 * z as u32)) as f64;
(z, b, actual_fp)
}
// One given, apply defaults for the other.
(Some(z), None, None) => {
let b = DEFAULT_B;
let fp = 1.0_f64 / (1u64 << (b as u32 * z as u32)) as f64;
(z, b, fp)
}
(None, Some(b), None) => {
let z = DEFAULT_Z;
let fp = 1.0_f64 / (1u64 << (b as u32 * z as u32)) as f64;
(z, b, fp)
}
(None, None, Some(fp)) => {
let b = DEFAULT_B;
let z = ((bits_needed(fp) as u32 + b as u32 - 1) / b as u32).max(1) as u8;
let actual_fp = 1.0_f64 / (1u64 << (b as u32 * z as u32)) as f64;
(z, b, actual_fp)
}
// None given: defaults.
(None, None, None) => {
let b = DEFAULT_B;
let z = DEFAULT_Z;
let fp = 1.0_f64 / (1u64 << (b as u32 * z as u32)) as f64;
(z, b, fp)
}
}
}
pub fn run(args: IndexArgs) {
args.common.validate();
let output = args.output.clone();
let mut rep = Reporter::new();
// Locked for the whole build (including a possible --force removal +
// recreation below): a second `index` run resuming/overwriting the same
// output directory concurrently would otherwise corrupt it. Unlinking
// the lock file via --force's remove_dir_all is safe — the held file
// descriptor keeps the lock regardless of the directory entry.
let _lock = obisys::DirLock::acquire(&output).unwrap_or_else(|e| {
eprintln!("error locking output directory {}: {e}", output.display());
std::process::exit(1);
});
// ── Resolve evidence kind ────────────────────────────────────────────────
let (evidence, effective_kmer_size) = if args.approx {
let (z, b, fp) = resolve_approx_params(args.findere_z, args.evidence_bits, args.fp);
let k = args.common.kmer_size;
if z as usize >= k {
eprintln!(
"error: Findere z={z} must be < kmer-size={k} \
(effective kmer size k−z+1 = {} ≤ 0)",
k as isize - z as isize + 1
);
std::process::exit(1);
}
let s = k - z as usize + 1;
info!("approximate evidence: b={b}, z={z}, fp={fp:.2e}, indexed kmer size={s}");
(IndexMode::Approx { b, z }, s)
} else {
(IndexMode::Exact, args.common.kmer_size)
};
// ── Open or create the index ─────────────────────────────────────────────
if KmerIndex::is_an_index(&output) {
if !args.force {
eprintln!(
"error: an index already exists at {} (use --force to overwrite it)",
output.display()
);
std::process::exit(1);
}
info!("--force: removing existing index at {}", output.display());
std::fs::remove_dir_all(&output).unwrap_or_else(|e| {
eprintln!("error removing existing index: {e}");
std::process::exit(1);
});
} else if output.exists() {
eprintln!(
"error: {} exists but is not an obikmer index, it cannot be deleted",
output.display()
);
std::process::exit(1);
}
let n_bits = partitions_to_bits(args.common.partitions);
let effective = 1usize << n_bits;
if effective != args.common.partitions {
info!(
"partitions: {} → {} (next power of 2)",
args.common.partitions, effective
);
}
let block_bits = block_size_to_bits(args.block_size);
let config = IndexConfig {
kmer_size: effective_kmer_size,
minimizer_size: args.common.minimizer_size,
n_bits,
with_counts: args.with_counts,
evidence: evidence.clone(),
block_bits,
};
let genome_info = args.label.as_ref().map(|label| {
GenomeInfo::validate_label(label).unwrap_or_else(|e| {
eprintln!("error: --label: {e}");
std::process::exit(1);
});
let mut info = GenomeInfo::new(label.clone());
for (k, v) in &args.meta {
info.meta.insert(k.clone(), v.clone());
}
info
});
let idx = KmerIndex::create(&output, config, genome_info).unwrap_or_else(|e| {
eprintln!("error creating index: {e}");
std::process::exit(1);
});
// ── Stage 1: scatter ─────────────────────────────────────────────────────
if current_state(&idx) < IndexState::Scattered {
let n_workers = args.common.threads.max(1);
let max_open = args.common.effective_max_open();
let t = Stage::start("scatter");
let pb = spinner("scatter");
let mut ema_rate: f64 = 0.0;
let mut last_t = Instant::now();
let mut last_bases: u64 = 0;
const ALPHA: f64 = 0.15;
let mut router = PartitionRouter::new(&idx)
.level_max(args.common.level_max)
.theta(args.common.theta)
.workers(n_workers)
.max_open(max_open)
.files(args.common.seqfile_paths())
.on_progress(|p: Progress| {
let now = Instant::now();
let dt = now.duration_since(last_t).as_secs_f64();
if dt > 0.0 {
let instant = (p.position - last_bases) as f64 / dt;
ema_rate = ALPHA * instant + (1.0 - ALPHA) * ema_rate;
}
last_t = now;
last_bases = p.position;
let bp = p.position as f64;
let (count_str, rate_str) = if bp >= 1e9 {
(
format!("{:.2} Gbp", bp / 1e9),
format!("{:.0} Mbp/s", ema_rate / 1e6),
)
} else {
(
format!("{:.0} Mbp", bp / 1e6),
format!("{:.0} Mbp/s", ema_rate / 1e6),
)
};
pb.set_message(format!("{count_str} {rate_str}"));
});
router.run().unwrap_or_else(|e| {
eprintln!("error: {e}");
std::process::exit(1);
});
pb.finish_and_clear();
rep.push(t.stop());
drop(router); // ends the borrow of `idx` early — `PartitionRouter`'s `Drop` impl would otherwise extend it to the end of scope (`run()` already called `close()`, which marks scatter done, internally)
} else {
info!("scatter already done, skipping");
}
// ── Stage 2: dereplicate + count ─────────────────────────────────────────
if current_state(&idx) < IndexState::Counted {
let t = Stage::start("dereplicate");
let pb = progress_bar("dereplication", idx.n_partitions() as u64, "partitions");
Dereplicator::new(&idx)
.on_progress(|_: Progress| pb.inc(1))
.run()
.unwrap_or_else(|e| {
eprintln!("error: {e}");
std::process::exit(1);
});
pb.finish_and_clear();
rep.push(t.stop());
let t = Stage::start("count_kmer");
let pb = progress_bar("counting", idx.n_partitions() as u64, "partitions");
// `Counter::run` writes `spectrums/{label}.json` and marks count
// done (`count.done`) internally once every partition succeeds.
Counter::new(&idx)
.keep_partial(args.keep_intermediate)
.on_progress(|_: Progress| pb.inc(1))
.run()
.unwrap_or_else(|e| {
eprintln!("error: {e}");
std::process::exit(1);
});
pb.finish_and_clear();
rep.push(t.stop());
} else {
info!("dereplicate+count already done, skipping");
}
// ── Stage 3: build layered index ─────────────────────────────────────────
if current_state(&idx) < IndexState::Indexed {
let t = Stage::start("index");
let pb = progress_bar("index", idx.n_partitions() as u64, "partitions");
let total_kmers = LayerBuilder::new(&idx)
.min_abundance(args.min_abundance)
.max_abundance(args.max_abundance)
.keep_intermediate(args.keep_intermediate)
.on_progress(|_: Progress| pb.inc(1))
.run()
.unwrap_or_else(|e| {
eprintln!("error: {e}");
std::process::exit(1);
});
pb.finish_and_clear();
info!("done — {total_kmers} total kmers indexed");
rep.push(t.stop());
// `LayerBuilder::run` marks the index done (`index.done`) internally
// once every partition succeeds.
} else {
info!("index already built, skipping");
}
rep.print();
}
+1
View File
@@ -0,0 +1 @@
pub mod index;
+32
View File
@@ -0,0 +1,32 @@
mod cli;
mod cmd;
use clap::{Parser, Subcommand};
use tracing_subscriber::{EnvFilter, fmt};
#[derive(Parser)]
#[command(name = "obikmer2", about = "DNA k-mer tools", version)]
struct Cli {
#[command(subcommand)]
command: Commands,
}
#[derive(Subcommand)]
enum Commands {
/// Build the complete genome index (scatter → dereplicate → count → layered MPHF)
Index(cmd::index::IndexArgs),
}
fn main() {
fmt()
.with_env_filter(
EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")),
)
.with_writer(std::io::stderr)
.init();
let cli = Cli::parse();
match cli.command {
Commands::Index(args) => cmd::index::run(args),
}
}