Add raw lookup and iteration APIs to Layer struct

Introduces `index` and `index_batch` methods for direct MPHF slot mapping without membership validation, alongside four public iterator methods for deterministic traversal of canonical kmers. These are backed by dedicated `KmerIter` and `KmerBatchIter` structs that wrap the underlying unitig file reader. Updates `LayerEvidence::Approx` to eagerly open the unitig reader during initialization, enforcing a clear separation between raw mapping and verified lookup workflows.
This commit is contained in:
Eric Coissac
2026-08-16 13:54:09 +02:00
parent 0de078fdf1
commit dae543fdfc
3 changed files with 184 additions and 6 deletions
+32
View File
@@ -74,6 +74,38 @@ impl<D: LayerData> Layer<D> {
pub fn n(&self) -> usize { self.mphf.n() }
/// Raw MPHF lookup: kmer → slot, no membership check.
pub fn index(&self, kmer: CanonicalKmer) -> usize {
self.mphf.index(kmer)
}
/// Batch raw MPHF lookup: kmers → slots, no membership check.
pub fn index_batch(&self, kmers: &[CanonicalKmer]) -> Vec<usize> {
self.mphf.index_batch(kmers)
}
/// Iterate over all canonical kmers in the layer, in deterministic order.
pub fn iter_kmers(&self) -> crate::mphf_layer::KmerIter<'_> {
self.mphf.iter_kmers()
}
/// Iterate over all canonical kmers, each paired with its zero-based
/// sequence index in `unitigs.bin`.
pub fn enumerate_kmers(&self) -> std::iter::Enumerate<crate::mphf_layer::KmerIter<'_>> {
self.mphf.enumerate_kmers()
}
/// Iterate over the layer's canonical kmers in batches of `n`.
pub fn iter_kmers_batch(&self, n: usize) -> crate::mphf_layer::KmerBatchIter<'_> {
self.mphf.iter_kmers_batch(n)
}
/// Iterate over batches, each paired with the zero-based index of the
/// first kmer in the batch.
pub fn enumerate_kmers_batch(&self, n: usize) -> std::iter::Enumerate<crate::mphf_layer::KmerBatchIter<'_>> {
self.mphf.enumerate_kmers_batch(n)
}
pub fn unitig_writer(out_dir: &Path) -> OLMResult<UnitigFileWriter> {
MphfLayer::unitig_writer(out_dir)
}
+120 -6
View File
@@ -1,4 +1,5 @@
use std::fs;
use std::iter::Enumerate;
use std::path::{Path, PathBuf};
use cacheline_ef::{CachelineEf, CachelineEfVec};
@@ -29,7 +30,7 @@ type MphfEps = PtrHash<u64, CubicEps, CachelineEfVec<&'static [CachelineEf]>, Xx
enum LayerEvidence {
Exact { evidence: Evidence, unitigs: UnitigFileReader },
Approx { fingerprint: FingerprintVec, unitigs_path: PathBuf },
Approx { fingerprint: FingerprintVec, unitigs: UnitigFileReader, unitigs_path: PathBuf },
Hybrid { evidence: Evidence, unitigs: UnitigFileReader, fingerprint: FingerprintVec },
}
@@ -57,15 +58,15 @@ impl MphfLayer {
IndexMode::Exact => {
let evidence = Evidence::open(&dir.join(EVIDENCE_FILE))?;
let n = evidence.len();
// open() auto-detects: uses direct access since exact layers always have .idx
let unitigs = UnitigFileReader::open(&dir.join(UNITIGS_FILE))?;
(LayerEvidence::Exact { evidence, unitigs }, n)
}
IndexMode::Approx { .. } => {
let fingerprint = FingerprintVec::open(&dir.join(FINGERPRINT_FILE))?;
let n = fingerprint.n();
let unitigs = UnitigFileReader::open(&dir.join(UNITIGS_FILE))?;
let unitigs_path = dir.join(UNITIGS_FILE);
(LayerEvidence::Approx { fingerprint, unitigs_path }, n)
(LayerEvidence::Approx { fingerprint, unitigs, unitigs_path }, n)
}
IndexMode::Hybrid { .. } => {
let evidence = Evidence::open(&dir.join(EVIDENCE_FILE))?;
@@ -89,7 +90,7 @@ impl MphfLayer {
let slot = self.mphf.index(&kmer.raw());
if slot >= self.n { return None; }
match &self.ev {
LayerEvidence::Exact { evidence, unitigs } => {
LayerEvidence::Exact { evidence, unitigs, .. } => {
let (chunk_id, rank) = evidence.decode(slot);
if unitigs.verify_canonical_kmer(chunk_id as usize, rank as usize, kmer) {
Some(slot)
@@ -113,7 +114,7 @@ impl MphfLayer {
let slot = self.mphf.index(&kmer.raw());
if slot >= self.n { return None; }
match &self.ev {
LayerEvidence::Exact { evidence, unitigs } |
LayerEvidence::Exact { evidence, unitigs, .. } |
LayerEvidence::Hybrid { evidence, unitigs, .. } => {
let (chunk_id, rank) = evidence.decode(slot);
if unitigs.verify_canonical_kmer(chunk_id as usize, rank as usize, kmer) {
@@ -149,7 +150,7 @@ impl MphfLayer {
return None;
}
match &self.ev {
LayerEvidence::Exact { evidence, unitigs } |
LayerEvidence::Exact { evidence, unitigs, .. } |
LayerEvidence::Hybrid { evidence, unitigs, .. } => {
let (chunk_id, rank) = evidence.decode(slot);
let raw = unitigs.canonical_raw_kmer(chunk_id as usize, rank as usize);
@@ -160,6 +161,119 @@ impl MphfLayer {
}
pub fn n(&self) -> usize { self.n }
/// Raw MPHF lookup: kmer → slot.
///
/// Returns the slot assigned by the MPHF without performing any membership
/// check against the layer's evidence. The returned slot is only meaningful
/// when the kmer is actually present in the layer; callers that need an
/// existence test should use [`find`](Self::find) instead.
pub fn index(&self, kmer: CanonicalKmer) -> usize {
self.mphf.index(&kmer.raw())
}
/// Batch raw MPHF lookup: kmers → slots.
///
/// Returns a [`Vec`] of slots, one per input kmer, in the same order as the
/// input slice. Like [`index`](Self::index), no membership check is
/// performed.
pub fn index_batch(&self, kmers: &[CanonicalKmer]) -> Vec<usize> {
kmers.iter().map(|k| self.mphf.index(&k.raw())).collect()
}
/// Iterate over all canonical kmers in the layer, in deterministic order.
///
/// The iteration order follows the physical layout of `unitigs.bin` and is
/// **not** correlated with MPHF slot numbers. Multiple `KmerIter` instances
/// can be held concurrently for as long as the layer lives, because each
/// carries its own cursor.
pub fn iter_kmers(&self) -> KmerIter<'_> {
let reader = match &self.ev {
LayerEvidence::Exact { unitigs, .. } => unitigs,
LayerEvidence::Approx { unitigs, .. } => unitigs,
LayerEvidence::Hybrid { unitigs, .. } => unitigs,
};
KmerIter { inner: Box::new(reader.iter_indexed_canonical_kmers()) }
}
/// Iterate over all canonical kmers, each paired with its zero-based
/// sequence index in `unitigs.bin`.
///
/// Yields `(index, kmer)` where `index` starts at `0` for the first kmer
/// stored in the layer. This is the standard Rust `enumerate` adapter
/// applied to [`iter_kmers`](Self::iter_kmers).
pub fn enumerate_kmers(&self) -> Enumerate<KmerIter<'_>> {
self.iter_kmers().enumerate()
}
/// Iterate over the layer's canonical kmers in batches of `n`.
///
/// Each call to [`next`](Iterator::next) returns a [`Vec`] of up to `n`
/// kmers. The final batch may be shorter when the layer is exhausted.
pub fn iter_kmers_batch(&self, n: usize) -> KmerBatchIter<'_> {
KmerBatchIter { inner: self.iter_kmers(), batch_size: n }
}
/// Iterate over batches, each paired with the zero-based index of the
/// first kmer in the batch.
///
/// Yields `(batch_start_index, Vec<CanonicalKmer>)` where
/// `batch_start_index` is a multiple of `n`. This is the standard Rust
/// `enumerate` adapter applied to [`iter_kmers_batch`](Self::iter_kmers_batch).
pub fn enumerate_kmers_batch(&self, n: usize) -> Enumerate<KmerBatchIter<'_>> {
self.iter_kmers_batch(n).enumerate()
}
}
// ── Iterator types ────────────────────────────────────────────────────────────
/// Iterator over the canonical kmers stored in a layer.
///
/// Produced by [`MphfLayer::iter_kmers`]. Multiple `KmerIter` instances can
/// coexist concurrently for as long as the parent layer lives, because each
/// holds its own cursor over the underlying [`UnitigFileReader`].
pub struct KmerIter<'a> {
inner: Box<dyn Iterator<Item = (CanonicalKmer, usize, usize)> + 'a>,
}
impl<'a> Iterator for KmerIter<'a> {
type Item = CanonicalKmer;
/// Return the next canonical kmer in iteration order.
fn next(&mut self) -> Option<Self::Item> {
self.inner.next().map(|(kmer, _, _)| kmer)
}
}
/// Iterator over batches of canonical kmers stored in a layer.
///
/// Produced by [`MphfLayer::iter_kmers_batch`]. Each call to [`next`](Self::next)
/// returns a [`Vec`] of up to `batch_size` kmers. The last batch may be shorter
/// than `batch_size` when the layer is exhausted.
pub struct KmerBatchIter<'a> {
inner: KmerIter<'a>,
batch_size: usize,
}
impl<'a> Iterator for KmerBatchIter<'a> {
type Item = Vec<CanonicalKmer>;
/// Return the next batch of kmers, or `None` if the layer is exhausted.
fn next(&mut self) -> Option<Self::Item> {
let mut batch = Vec::with_capacity(self.batch_size);
for _ in 0..self.batch_size {
if let Some(kmer) = self.inner.next() {
batch.push(kmer);
} else {
break;
}
}
if batch.is_empty() {
None
} else {
Some(batch)
}
}
}
// ── MphfOnly ──────────────────────────────────────────────────────────────────