feat: add CLI command to export indexed k-mers to CSV

This change introduces a new `dump` subcommand that exports all indexed k-mers to a CSV stream. The implementation spans multiple crates, adding core export logic to `obikindex` and partition iteration to `obikpartitionner`. The command supports a `--force-presence` flag to output binary presence/absence data instead of stored counts, and includes necessary module registrations and structural updates across the codebase.
This commit is contained in:
Eric Coissac
2026-05-21 06:02:27 +02:00
parent e1d59fde54
commit 1a1f95e59d
7 changed files with 172 additions and 0 deletions
+50
View File
@@ -0,0 +1,50 @@
use std::io::Write;
use crate::error::{OKIError, OKIResult};
use crate::index::KmerIndex;
impl KmerIndex {
/// Write a CSV table of all indexed kmers to `out`.
///
/// Columns: `kmer`, then one column per genome (in index order).
/// Values are counts (u32) when `use_counts = true`, otherwise 0/1.
///
/// `force_presence` overrides `with_counts`: even if the index stores counts,
/// the output uses 0/1 presence columns.
///
/// The caller must have set the global kmer length (`obikseq::set_k`) before
/// calling this method.
pub fn dump<W: Write>(&self, out: &mut W, force_presence: bool) -> OKIResult<()> {
let genomes = &self.meta.genomes;
let use_counts = self.meta.config.with_counts && !force_presence;
let n_genomes = genomes.len().max(1);
// ── Header ────────────────────────────────────────────────────────────
write!(out, "kmer")?;
for g in genomes {
write!(out, ",{g}")?;
}
writeln!(out)?;
// ── Rows ──────────────────────────────────────────────────────────────
let n = self.n_partitions();
for i in 0..n {
self.partition
.iter_partition_kmers(i, use_counts, n_genomes, |kmer, row| {
let seq = String::from_utf8(kmer.to_ascii())
.unwrap_or_else(|_| "?".repeat(self.kmer_size()));
// write is infallible inside a closure — propagate via a flag if needed
let _ = write!(out, "{seq}");
for &v in row.iter() {
let _ = write!(out, ",{v}");
}
let _ = writeln!(out);
})
.map_err(OKIError::Partition)?;
}
out.flush()?;
Ok(())
}
}
+1
View File
@@ -1,6 +1,7 @@
pub mod error;
pub mod meta;
pub mod state;
mod dump;
mod index;
mod merge;
+39
View File
@@ -0,0 +1,39 @@
use std::io::{self, BufWriter};
use std::path::PathBuf;
use clap::Args;
use obikindex::KmerIndex;
use obikseq::set_k;
use tracing::info;
#[derive(Args)]
pub struct DumpArgs {
/// Index directory to dump
pub index: PathBuf,
/// Output presence/absence (0/1) even if the index stores counts
#[arg(long, default_value_t = false)]
pub force_presence: bool,
}
pub fn run(args: DumpArgs) {
let idx = KmerIndex::open(&args.index).unwrap_or_else(|e| {
eprintln!("error opening index: {e}");
std::process::exit(1);
});
set_k(idx.kmer_size());
info!(
"dumping {} partitions, {} genome(s)",
idx.n_partitions(),
idx.meta().genomes.len()
);
let stdout = io::stdout();
let mut out = BufWriter::new(stdout.lock());
idx.dump(&mut out, args.force_presence).unwrap_or_else(|e| {
eprintln!("dump error: {e}");
std::process::exit(1);
});
}
+1
View File
@@ -1,3 +1,4 @@
pub mod dump;
pub mod index;
pub mod merge;
pub mod superkmer;
+3
View File
@@ -20,6 +20,8 @@ enum Commands {
Index(cmd::index::IndexArgs),
/// Merge multiple built indexes into one
Merge(cmd::merge::MergeArgs),
/// Dump all indexed kmers as CSV (kmer + per-genome counts or presence)
Dump(cmd::dump::DumpArgs),
/// Dump unitigs from a built index to stdout (debug)
Unitig(cmd::unitig::UnitigArgs),
}
@@ -46,6 +48,7 @@ fn main() {
Commands::Superkmer(args) => cmd::superkmer::run(args),
Commands::Index(args) => cmd::index::run(args),
Commands::Merge(args) => cmd::merge::run(args),
Commands::Dump(args) => cmd::dump::run(args),
Commands::Unitig(args) => cmd::unitig::run(args),
}
+77
View File
@@ -0,0 +1,77 @@
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
use obikseq::CanonicalKmer;
use obiskio::{SKError, SKResult, UnitigFileReader};
use obilayeredmap::OLMError;
use obilayeredmap::meta::PartitionMeta;
use obilayeredmap::MphfLayer;
use crate::partition::KmerPartition;
const INDEX_SUBDIR: &str = "index";
fn olm_to_sk(e: OLMError) -> SKError {
match e {
OLMError::Io(e) => SKError::Io(e),
other => SKError::InvalidData { context: "dump", detail: other.to_string() },
}
}
impl KmerPartition {
/// Iterate all indexed kmers in partition `part`, calling `cb(kmer, row)` for each.
///
/// `use_counts = true` → reads count columns (u32 values per genome).
/// `use_counts = false` → reads presence columns, converted to 0/1 u32.
///
/// If no data matrix exists for a layer (pure set-membership, single genome),
/// a row of `n_genomes` ones is emitted for every kmer in that layer.
pub fn iter_partition_kmers(
&self,
part: usize,
use_counts: bool,
n_genomes: usize,
mut cb: impl FnMut(CanonicalKmer, Box<[u32]>),
) -> SKResult<()> {
let index_dir = self.part_dir(part).join(INDEX_SUBDIR);
if !index_dir.exists() {
return Ok(());
}
let meta = PartitionMeta::load(&index_dir).map_err(olm_to_sk)?;
for l in 0..meta.n_layers {
let layer_dir = index_dir.join(format!("layer_{l}"));
let mphf = MphfLayer::open(&layer_dir).map_err(olm_to_sk)?;
let reader = UnitigFileReader::open(&layer_dir.join("unitigs.bin"))?;
let counts_dir = layer_dir.join("counts");
let presence_dir = layer_dir.join("presence");
if use_counts && counts_dir.exists() {
let mat = PersistentCompactIntMatrix::open(&counts_dir).map_err(SKError::Io)?;
for (kmer, _, _) in reader.iter_indexed_canonical_kmers() {
if let Some(slot) = mphf.find(kmer) {
cb(kmer, mat.row(slot));
}
}
} else if !use_counts && presence_dir.exists() {
let mat = PersistentBitMatrix::open(&presence_dir).map_err(SKError::Io)?;
for (kmer, _, _) in reader.iter_indexed_canonical_kmers() {
if let Some(slot) = mphf.find(kmer) {
let row: Box<[u32]> = mat.row(slot).iter().map(|&b| b as u32).collect();
cb(kmer, row);
}
}
} else {
// No data matrix: pure set-membership layer, all kmers belong to every genome.
let all_present: Box<[u32]> = vec![1u32; n_genomes].into();
for (kmer, _, _) in reader.iter_indexed_canonical_kmers() {
if mphf.find(kmer).is_some() {
cb(kmer, all_present.clone());
}
}
}
}
Ok(())
}
}
+1
View File
@@ -1,3 +1,4 @@
mod dump_layer;
mod index_layer;
mod kmer_sort;
mod merge_layer;