Replace slot-based indexing with iteration order and stream k-mers
Transitions the index from MPHF slot-based to physical iteration-order indexing, aligning with the unitig layout. Introduces a streaming-only pipeline for k-mer iteration that adheres to memory constraints by avoiding full in-memory collections. Updates layer and sibling iterators to own an Arc clone of the file reader, making them Send + 'static and safe for concurrent use without borrowing the parent. Exposes batch and k-mer iterator types publicly while simplifying signature syntax with modern lifetime elision.
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
use std::fs::File;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use memmap2::Mmap;
|
||||
use obikseq::{CanonicalKmer, Kmer, Unitig};
|
||||
@@ -203,6 +204,34 @@ impl UnitigFileReader {
|
||||
.map(move |(rank, kmer)| (kmer, chunk_id, rank))
|
||||
})
|
||||
}
|
||||
|
||||
/// Same streamed sequence as [`iter_indexed_canonical_kmers`](Self::iter_indexed_canonical_kmers),
|
||||
/// but owning a clone of `self` instead of borrowing it — `Send + 'static`,
|
||||
/// so it can be handed to a threaded consumer (e.g. `obipipeline`) without
|
||||
/// first collecting the layer's k-mers into memory. Reads `mmap` fresh
|
||||
/// through the `Arc` on every step; no data is duplicated up front.
|
||||
pub fn iter_indexed_canonical_kmers_owned(
|
||||
self: &Arc<Self>,
|
||||
) -> impl Iterator<Item = (CanonicalKmer, usize, usize)> + Send + 'static {
|
||||
let this = Arc::clone(self);
|
||||
let k = this.k;
|
||||
let n = this.n_unitigs;
|
||||
let mut offset = 0usize;
|
||||
(0..n)
|
||||
.map(move |chunk_id| {
|
||||
let mmap = &*this.mmap;
|
||||
let seql = mmap[offset] as usize + k;
|
||||
let byte_len = (seql + 3) / 4;
|
||||
let bytes = mmap[offset + 1..offset + 1 + byte_len].to_vec().into_boxed_slice();
|
||||
offset += 1 + byte_len;
|
||||
(chunk_id, Unitig::new((seql % 4) as u8, bytes))
|
||||
})
|
||||
.flat_map(|(chunk_id, u)| {
|
||||
u.into_canonical_kmers()
|
||||
.enumerate()
|
||||
.map(move |(rank, kmer)| (kmer, chunk_id, rank))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn read_idx(path: &Path) -> SKResult<(usize, usize, u8, Vec<u32>)> {
|
||||
|
||||
Reference in New Issue
Block a user