This change replaces the HashMap-based k-mer storage mechanism with a compact, flat array structure for `PartitionKmers`. This significantly reduces memory usage per partition and improves query execution time. It also updates the batch processing logic to handle the new data structure, introduces detailed memory footprint calculations, and updates relevant query layer APIs. Test coverage has been expanded to validate correct partitioning and k-mer ordering.
142 lines
5.6 KiB
Rust
142 lines
5.6 KiB
Rust
use super::*;
|
|
use obikrope::Rope;
|
|
use obikseq::CanonicalKmer;
|
|
use obiread::record::parse_chunk;
|
|
|
|
const K: usize = 11;
|
|
const M: usize = 5;
|
|
|
|
/// Build a `QueryBatch` from raw FASTA text, going through the same
|
|
/// `Rope` + `parse_chunk` path `process_chunk` uses — avoids hand-building a
|
|
/// `normalized` `Rope`, which is an implementation detail of `obiread`.
|
|
///
|
|
/// `obikseq`'s global K/M params are thread-local under `test-utils` (see
|
|
/// `obikseq::params`), so setting them here is per-test-thread and does not
|
|
/// need coordination with other tests.
|
|
fn batch_from_fasta(fasta: &str, k: usize, n_partitions: usize) -> QueryBatch {
|
|
obikseq::set_k(k);
|
|
obikseq::set_m(M);
|
|
let mut rope = Rope::new(Some("text/fasta"));
|
|
rope.push(fasta.as_bytes().to_vec());
|
|
let records = parse_chunk(&rope, k);
|
|
QueryBatch::from_records(records, k, 6, 0.7, n_partitions)
|
|
}
|
|
|
|
fn total_occurrences(batch: &QueryBatch) -> u64 {
|
|
batch.n_kmers.iter().map(|&n| n as u64).sum()
|
|
}
|
|
|
|
fn total_unique_kmers(batch: &QueryBatch) -> u64 {
|
|
batch.by_partition.iter().map(|m| m.len() as u64).sum()
|
|
}
|
|
|
|
// A 60 bp sequence, arbitrary but fixed — no attempt is made to prove it is
|
|
// free of internal k=11 repeats; the tests below only rely on inequalities
|
|
// that hold regardless (see each test's comment).
|
|
const SEQ: &str = "CATTAGCGTACCTGATCAGGTTACAGCTTAGGCATCCAGTTGACCATGACTGGACTTAGC";
|
|
|
|
#[test]
|
|
fn single_sequence_yields_plausible_kmer_counts() {
|
|
// A single record can still contain internal repeats (SEQ isn't
|
|
// guaranteed repeat-free at k=11) — this only checks the batch is
|
|
// internally consistent, not a specific dedup ratio. The cross-record
|
|
// tests below make the actual, unconditional dedup claims.
|
|
let fasta = format!(">r1\n{SEQ}\n");
|
|
let batch = batch_from_fasta(&fasta, K, 1);
|
|
|
|
assert_eq!(batch.ids, vec!["r1".to_string()]);
|
|
let occurrences = total_occurrences(&batch);
|
|
let unique = total_unique_kmers(&batch);
|
|
assert!(occurrences > 0, "sequence should yield at least one k-mer");
|
|
assert!(unique > 0 && unique <= occurrences);
|
|
}
|
|
|
|
#[test]
|
|
fn duplicated_sequence_across_records_deduplicates() {
|
|
// Two records with byte-identical sequences: every k-mer in record 1
|
|
// exactly duplicates one in record 0, so unique kmers <= n_kmers[0],
|
|
// strictly less than the summed occurrences (2 * n_kmers[0]) as long as
|
|
// the sequence yields at least one k-mer. This holds regardless of
|
|
// whether SEQ has internal repeats.
|
|
let fasta = format!(">r1\n{SEQ}\n>r2\n{SEQ}\n");
|
|
let batch = batch_from_fasta(&fasta, K, 1);
|
|
|
|
assert_eq!(batch.ids.len(), 2);
|
|
let occurrences = total_occurrences(&batch);
|
|
let unique = total_unique_kmers(&batch);
|
|
|
|
assert!(batch.n_kmers[0] > 0);
|
|
assert_eq!(occurrences, batch.n_kmers[0] as u64 + batch.n_kmers[1] as u64);
|
|
assert!(
|
|
unique <= batch.n_kmers[0] as u64,
|
|
"identical sequences must not produce more unique k-mers than one copy has"
|
|
);
|
|
assert!(
|
|
unique < occurrences,
|
|
"k-mer-level dedup must collapse at least the cross-record duplication"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn duplicated_sequence_broadcasts_to_both_seq_indices() {
|
|
// Stronger than the ratio check above: pick any k-mer that hit in both
|
|
// records and confirm its occurrence list actually references both
|
|
// seq_idx 0 and seq_idx 1 — this is the specific new capability (dedup
|
|
// reaching across records/superkmers), not just a smaller unique count.
|
|
let fasta = format!(">r1\n{SEQ}\n>r2\n{SEQ}\n");
|
|
let batch = batch_from_fasta(&fasta, K, 1);
|
|
|
|
let shared = batch.by_partition[0]
|
|
.iter()
|
|
.find(|(_, descs)| descs.iter().any(|d| d.seq_idx == 0) && descs.iter().any(|d| d.seq_idx == 1));
|
|
|
|
assert!(
|
|
shared.is_some(),
|
|
"expected at least one k-mer shared between the two identical records"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn empty_records_yield_empty_batch() {
|
|
let batch = batch_from_fasta("", K, 1);
|
|
assert!(batch.ids.is_empty());
|
|
assert_eq!(total_occurrences(&batch), 0);
|
|
assert_eq!(total_unique_kmers(&batch), 0);
|
|
}
|
|
|
|
#[test]
|
|
fn partition_routing_is_a_pure_function_of_the_kmer() {
|
|
// With n_partitions=4, every occurrence of a given k-mer must land in
|
|
// the same partition bucket as every other occurrence of that k-mer
|
|
// (partition routing is derived from the minimizer, shared by
|
|
// definition among instances of the same k-mer's containing superkmer
|
|
// in this test's single-sequence-pair setup).
|
|
let fasta = format!(">r1\n{SEQ}\n>r2\n{SEQ}\n");
|
|
let batch = batch_from_fasta(&fasta, K, 4);
|
|
|
|
let total_unique: u64 = batch.by_partition.iter().map(|m| m.len() as u64).sum();
|
|
assert!(total_unique > 0);
|
|
|
|
// No k-mer key appears in more than one partition's map.
|
|
let mut seen: std::collections::HashSet<CanonicalKmer> = std::collections::HashSet::new();
|
|
for part in &batch.by_partition {
|
|
for (kmer, _) in part.iter() {
|
|
assert!(seen.insert(kmer), "k-mer routed to more than one partition");
|
|
}
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn occurrences_of_a_kmer_come_out_in_query_order() {
|
|
// Grouping sorts occurrences; within a k-mer they must stay ordered by
|
|
// (seq_idx, pos) — the order the former insertion-ordered `Vec` had, which
|
|
// downstream output ordering relied on.
|
|
let fasta = format!(">r1\n{SEQ}\n>r2\n{SEQ}\n");
|
|
let batch = batch_from_fasta(&fasta, K, 1);
|
|
for (_, descs) in batch.by_partition[0].iter() {
|
|
assert!(descs.windows(2).all(|w| (w[0].seq_idx, w[0].pos) < (w[1].seq_idx, w[1].pos)));
|
|
}
|
|
let occ: usize = batch.by_partition[0].iter().map(|(_, d)| d.len()).sum();
|
|
assert_eq!(occ as u64, total_occurrences(&batch));
|
|
}
|