From 768fa0183da5776246b166e60d0f439a57e6da63 Mon Sep 17 00:00:00 2001 From: Eric Coissac Date: Wed, 26 Aug 2026 08:22:49 +0200 Subject: [PATCH] Add dump subcommand and introduce reusable group filtering arguments Introduces a new `dump` CLI command to export index k-mers as a CSV table. Adds a reusable `GroupFilterArgs` struct for ingroup/outgroup metadata-predicate quorum filtering with configurable count and fraction thresholds. Refactors the filter command to use explicit flags and a `GenomeSelector`-based pipeline, improving error handling and predicate application order. Exposes `IndexDump` from the `obikdump` crate and updates local dependencies accordingly. --- src/Cargo.lock | 1 + src/obikdump/src/lib.rs | 2 + src/obikmer2/Cargo.toml | 1 + src/obikmer2/src/cmd/dump/mod.rs | 62 +++++++++++++++++++++ src/obikmer2/src/cmd/filter/mod.rs | 85 +++-------------------------- src/obikmer2/src/cmd/mod.rs | 2 + src/obikmer2/src/cmd/predicate.rs | 87 ++++++++++++++++++++++++++++++ src/obikmer2/src/main.rs | 3 ++ 8 files changed, 165 insertions(+), 78 deletions(-) create mode 100644 src/obikmer2/src/cmd/dump/mod.rs create mode 100644 src/obikmer2/src/cmd/predicate.rs diff --git a/src/Cargo.lock b/src/Cargo.lock index a1cad46c..b0a15fcd 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -1667,6 +1667,7 @@ dependencies = [ "csv", "obifastwrite", "obikalgorithm", + "obikdump", "obikfilter", "obikindex", "obikindexer", diff --git a/src/obikdump/src/lib.rs b/src/obikdump/src/lib.rs index f953444f..577fd412 100644 --- a/src/obikdump/src/lib.rs +++ b/src/obikdump/src/lib.rs @@ -6,3 +6,5 @@ //! reverse), same pattern as `obikindexer`/`obikquery`. mod dump; + +pub use dump::IndexDump; diff --git a/src/obikmer2/Cargo.toml b/src/obikmer2/Cargo.toml index 0d849d15..a8445e7d 100644 --- a/src/obikmer2/Cargo.toml +++ b/src/obikmer2/Cargo.toml @@ -18,6 +18,7 @@ obikalgorithm = { path = "../obikalgorithm" } obikmerge = { path = "../obikmerge" } obikfilter = { path = "../obikfilter" } obikselect = { path = "../obikselect" } +obikdump = { path = "../obikdump" } obifastwrite = { path = "../obifastwrite" } obiskbuilder = { path = "../obiskbuilder" } clap = { version = "4", features = ["derive"] } diff --git a/src/obikmer2/src/cmd/dump/mod.rs b/src/obikmer2/src/cmd/dump/mod.rs new file mode 100644 index 00000000..abe28a2b --- /dev/null +++ b/src/obikmer2/src/cmd/dump/mod.rs @@ -0,0 +1,62 @@ +use std::io::{self, BufWriter}; +use std::path::PathBuf; + +use clap::Args; +use obikdump::IndexDump; +use obikfilter::KmerFilter; +use obikindex::KmerIndex; +use obisys::progress_bar; +use tracing::info; + +use super::predicate::GroupFilterArgs; + +#[derive(Args)] +pub struct DumpArgs { + /// Index directory to dump + pub index: PathBuf, + + /// Output presence/absence (0/1) even if the index stores counts + #[arg(long, default_value_t = false)] + pub force_presence: bool, + + /// Prepend partition and layer columns to each row + #[arg(long, default_value_t = false)] + pub debug: bool, + + /// Only output the first N kmers + #[arg(long)] + pub head: Option, + + #[command(flatten)] + pub group_filter: GroupFilterArgs, +} + +pub fn run(args: DumpArgs) { + let idx = KmerIndex::open(&args.index).unwrap_or_else(|e| { + eprintln!("error opening index: {e}"); + std::process::exit(1); + }); + + let n_genomes = idx.meta().genomes().unwrap_or_else(|e| { + eprintln!("error reading index metadata: {e}"); + std::process::exit(1); + }).len(); + info!( + "dumping {} partition(s), {} genome(s)", + idx.n_partitions(), + n_genomes + ); + + let filters: Vec> = vec![Box::new(args.group_filter.build_filter(&idx.meta()))]; + let pb = progress_bar("dump", idx.n_partitions() as u64, "partitions"); + + let stdout = io::stdout(); + let mut out = BufWriter::new(stdout.lock()); + + idx.dump(&mut out, args.force_presence, args.debug, args.head, &filters, || pb.inc(1)) + .unwrap_or_else(|e| { + eprintln!("dump error: {e}"); + std::process::exit(1); + }); + pb.finish_and_clear(); +} diff --git a/src/obikmer2/src/cmd/filter/mod.rs b/src/obikmer2/src/cmd/filter/mod.rs index 4da8cd6e..b54d6c70 100644 --- a/src/obikmer2/src/cmd/filter/mod.rs +++ b/src/obikmer2/src/cmd/filter/mod.rs @@ -2,14 +2,13 @@ use std::path::PathBuf; use clap::Args; use obikalgorithm::Algorithm; -use obikfilter::{ - Filter, GenomeSelector, GroupFilterParams, KmerFilter, MaxTotalCount, MinComplexity, - MinTotalCount, -}; +use obikfilter::{Filter, KmerFilter, MaxTotalCount, MinComplexity, MinTotalCount}; use obikindex::KmerIndex; use obisys::{Progress, Reporter, Stage, progress_bar}; use tracing::info; +use super::predicate::GroupFilterArgs; + #[derive(Args)] pub struct FilterArgs { /// Source index directory @@ -19,55 +18,8 @@ pub struct FilterArgs { #[arg(short, long)] pub output: PathBuf, - /// Ingroup predicate (repeatable; AND). Forms: `key=v1|v2`, `key!=v`, `key~path`, `key!~path`, `*`/`all` - #[arg(long, value_name = "PRED")] - pub ingroup: Vec, - - /// Outgroup predicate (repeatable; OR). Forms: `key=v1|v2`, `key!=v`, `key~path`, `key!~path`, `*`/`all` - #[arg(long, value_name = "PRED")] - pub outgroup: Vec, - - /// Minimum number of ingroup genomes containing the k-mer - /// (negative: offset from group size, e.g. -1 = all but one) - #[arg(long, allow_hyphen_values = true)] - pub min_count: Option, - - /// Maximum number of ingroup genomes containing the k-mer - /// (negative: offset from group size, e.g. -1 = all but one) - #[arg(long, allow_hyphen_values = true)] - pub max_count: Option, - - /// Minimum fraction of ingroup genomes containing the k-mer [0.0-1.0] - /// (default 1.0 when --ingroup is set, 0.0 otherwise) - #[arg(long)] - pub min_frac: Option, - - /// Maximum fraction of ingroup genomes containing the k-mer [0.0-1.0] - #[arg(long)] - pub max_frac: Option, - - /// Minimum number of outgroup genomes containing the k-mer - /// (negative: offset from outgroup size, e.g. -1 = all but one) - #[arg(long, allow_hyphen_values = true)] - pub min_outgroup_count: Option, - - /// Maximum number of outgroup genomes containing the k-mer - /// (default 0 when --outgroup is set, no constraint otherwise; - /// negative: offset from outgroup size, e.g. -1 = all but one) - #[arg(long, allow_hyphen_values = true)] - pub max_outgroup_count: Option, - - /// Minimum fraction of outgroup genomes containing the k-mer [0.0-1.0] - #[arg(long)] - pub min_outgroup_frac: Option, - - /// Maximum fraction of outgroup genomes containing the k-mer [0.0-1.0] - #[arg(long)] - pub max_outgroup_frac: Option, - - /// Per-genome count threshold to consider a genome as "containing" the k-mer (default 0) - #[arg(long, default_value = "0")] - pub presence_threshold: u32, + #[command(flatten)] + pub group_filter: GroupFilterArgs, /// Minimum total count across all genomes (count index only) #[arg(long)] @@ -104,31 +56,8 @@ pub fn run(args: FilterArgs) { std::process::exit(1); }); - let selector = GenomeSelector::parse(&args.ingroup, &args.outgroup).unwrap_or_else(|e| { - eprintln!("error in --ingroup/--outgroup: {e}"); - std::process::exit(1); - }); - let group_filter = selector - .build_group_filter( - &src.meta(), - GroupFilterParams { - threshold: args.presence_threshold, - min_count: args.min_count, - max_count: args.max_count, - min_frac: args.min_frac, - max_frac: args.max_frac, - min_outgroup_count: args.min_outgroup_count, - max_outgroup_count: args.max_outgroup_count, - min_outgroup_frac: args.min_outgroup_frac, - max_outgroup_frac: args.max_outgroup_frac, - }, - ) - .unwrap_or_else(|e| { - eprintln!("error in filter parameters: {e}"); - std::process::exit(1); - }); - - let mut filters: Vec> = vec![Box::new(group_filter)]; + let mut filters: Vec> = + vec![Box::new(args.group_filter.build_filter(&src.meta()))]; if let Some(v) = args.min_total_count { filters.push(Box::new(MinTotalCount { total: v })); } diff --git a/src/obikmer2/src/cmd/mod.rs b/src/obikmer2/src/cmd/mod.rs index 9b94a285..732ba742 100644 --- a/src/obikmer2/src/cmd/mod.rs +++ b/src/obikmer2/src/cmd/mod.rs @@ -1,7 +1,9 @@ pub mod annotate; +pub mod dump; pub mod estimate; pub mod filter; pub mod index; pub mod merge; +mod predicate; pub mod select; pub mod superkmer; diff --git a/src/obikmer2/src/cmd/predicate.rs b/src/obikmer2/src/cmd/predicate.rs new file mode 100644 index 00000000..62ab29d4 --- /dev/null +++ b/src/obikmer2/src/cmd/predicate.rs @@ -0,0 +1,87 @@ +use clap::Args; +use obikfilter::{GenomeSelector, GroupFilterParams, GroupQuorumFilter}; +use obikindex::IndexMeta; + +/// Ingroup/outgroup metadata-predicate quorum filtering — embeddable in any +/// command via `#[command(flatten)]` (`filter`, `dump`). +#[derive(Args)] +pub struct GroupFilterArgs { + /// Ingroup predicate (repeatable; AND). Forms: `key=v1|v2`, `key!=v`, `key~path`, `key!~path`, `*`/`all` + #[arg(long, value_name = "PRED")] + pub ingroup: Vec, + + /// Outgroup predicate (repeatable; OR). Forms: `key=v1|v2`, `key!=v`, `key~path`, `key!~path`, `*`/`all` + #[arg(long, value_name = "PRED")] + pub outgroup: Vec, + + /// Minimum number of ingroup genomes containing the k-mer + /// (negative: offset from group size, e.g. -1 = all but one) + #[arg(long, allow_hyphen_values = true)] + pub min_count: Option, + + /// Maximum number of ingroup genomes containing the k-mer + /// (negative: offset from group size, e.g. -1 = all but one) + #[arg(long, allow_hyphen_values = true)] + pub max_count: Option, + + /// Minimum fraction of ingroup genomes containing the k-mer [0.0-1.0] + /// (default 1.0 when --ingroup is set, 0.0 otherwise) + #[arg(long)] + pub min_frac: Option, + + /// Maximum fraction of ingroup genomes containing the k-mer [0.0-1.0] + #[arg(long)] + pub max_frac: Option, + + /// Minimum number of outgroup genomes containing the k-mer + /// (negative: offset from outgroup size, e.g. -1 = all but one) + #[arg(long, allow_hyphen_values = true)] + pub min_outgroup_count: Option, + + /// Maximum number of outgroup genomes containing the k-mer + /// (default 0 when --outgroup is set, no constraint otherwise; + /// negative: offset from outgroup size, e.g. -1 = all but one) + #[arg(long, allow_hyphen_values = true)] + pub max_outgroup_count: Option, + + /// Minimum fraction of outgroup genomes containing the k-mer [0.0-1.0] + #[arg(long)] + pub min_outgroup_frac: Option, + + /// Maximum fraction of outgroup genomes containing the k-mer [0.0-1.0] + #[arg(long)] + pub max_outgroup_frac: Option, + + /// Per-genome count threshold to consider a genome as "containing" the k-mer (default 0) + #[arg(long, default_value = "0")] + pub presence_threshold: u32, +} + +impl GroupFilterArgs { + /// Parse `--ingroup`/`--outgroup` and build the quorum filter. Exits on error. + pub fn build_filter(&self, meta: &IndexMeta) -> GroupQuorumFilter { + let selector = GenomeSelector::parse(&self.ingroup, &self.outgroup).unwrap_or_else(|e| { + eprintln!("error in --ingroup/--outgroup: {e}"); + std::process::exit(1); + }); + selector + .build_group_filter( + meta, + GroupFilterParams { + threshold: self.presence_threshold, + min_count: self.min_count, + max_count: self.max_count, + min_frac: self.min_frac, + max_frac: self.max_frac, + min_outgroup_count: self.min_outgroup_count, + max_outgroup_count: self.max_outgroup_count, + min_outgroup_frac: self.min_outgroup_frac, + max_outgroup_frac: self.max_outgroup_frac, + }, + ) + .unwrap_or_else(|e| { + eprintln!("error in filter parameters: {e}"); + std::process::exit(1); + }) + } +} diff --git a/src/obikmer2/src/main.rs b/src/obikmer2/src/main.rs index 27f96d53..743ff416 100644 --- a/src/obikmer2/src/main.rs +++ b/src/obikmer2/src/main.rs @@ -23,6 +23,8 @@ enum Commands { Filter(cmd::filter::FilterArgs), /// Project/aggregate genome columns into a new index Select(cmd::select::SelectArgs), + /// Dump an index's kmers as a CSV table + Dump(cmd::dump::DumpArgs), /// Estimate approximate-evidence false-positive rates for given parameters Estimate(cmd::estimate::EstimateArgs), /// Read/write genome metadata (CSV) on an already-built index @@ -44,6 +46,7 @@ fn main() { Commands::Merge(args) => cmd::merge::run(args), Commands::Filter(args) => cmd::filter::run(args), Commands::Select(args) => cmd::select::run(args), + Commands::Dump(args) => cmd::dump::run(args), Commands::Estimate(args) => cmd::estimate::run(args), Commands::Annotate(args) => cmd::annotate::run(args), }