Replace slot-based indexing with iteration order and stream k-mers

Transitions the index from MPHF slot-based to physical iteration-order indexing, aligning with the unitig layout. Introduces a streaming-only pipeline for k-mer iteration that adheres to memory constraints by avoiding full in-memory collections. Updates layer and sibling iterators to own an Arc clone of the file reader, making them Send + 'static and safe for concurrent use without borrowing the parent. Exposes batch and k-mer iterator types publicly while simplifying signature syntax with modern lifetime elision.
This commit is contained in:
Eric Coissac
2026-08-16 14:07:22 +02:00
parent b1f54b7d2f
commit 519195d4a1
7 changed files with 219 additions and 84 deletions
+29
View File
@@ -1,5 +1,6 @@
use std::fs::File;
use std::path::Path;
use std::sync::Arc;
use memmap2::Mmap;
use obikseq::{CanonicalKmer, Kmer, Unitig};
@@ -203,6 +204,34 @@ impl UnitigFileReader {
.map(move |(rank, kmer)| (kmer, chunk_id, rank))
})
}
/// Same streamed sequence as [`iter_indexed_canonical_kmers`](Self::iter_indexed_canonical_kmers),
/// but owning a clone of `self` instead of borrowing it — `Send + 'static`,
/// so it can be handed to a threaded consumer (e.g. `obipipeline`) without
/// first collecting the layer's k-mers into memory. Reads `mmap` fresh
/// through the `Arc` on every step; no data is duplicated up front.
pub fn iter_indexed_canonical_kmers_owned(
self: &Arc<Self>,
) -> impl Iterator<Item = (CanonicalKmer, usize, usize)> + Send + 'static {
let this = Arc::clone(self);
let k = this.k;
let n = this.n_unitigs;
let mut offset = 0usize;
(0..n)
.map(move |chunk_id| {
let mmap = &*this.mmap;
let seql = mmap[offset] as usize + k;
let byte_len = (seql + 3) / 4;
let bytes = mmap[offset + 1..offset + 1 + byte_len].to_vec().into_boxed_slice();
offset += 1 + byte_len;
(chunk_id, Unitig::new((seql % 4) as u8, bytes))
})
.flat_map(|(chunk_id, u)| {
u.into_canonical_kmers()
.enumerate()
.map(move |(rank, kmer)| (kmer, chunk_id, rank))
})
}
}
fn read_idx(path: &Path) -> SKResult<(usize, usize, u8, Vec<u32>)> {