Add raw lookup and iteration APIs to Layer struct
Introduces `index` and `index_batch` methods for direct MPHF slot mapping without membership validation, alongside four public iterator methods for deterministic traversal of canonical kmers. These are backed by dedicated `KmerIter` and `KmerBatchIter` structs that wrap the underlying unitig file reader. Updates `LayerEvidence::Approx` to eagerly open the unitig reader during initialization, enforcing a clear separation between raw mapping and verified lookup workflows.
This commit is contained in:
@@ -74,6 +74,38 @@ impl<D: LayerData> Layer<D> {
|
||||
|
||||
pub fn n(&self) -> usize { self.mphf.n() }
|
||||
|
||||
/// Raw MPHF lookup: kmer → slot, no membership check.
|
||||
pub fn index(&self, kmer: CanonicalKmer) -> usize {
|
||||
self.mphf.index(kmer)
|
||||
}
|
||||
|
||||
/// Batch raw MPHF lookup: kmers → slots, no membership check.
|
||||
pub fn index_batch(&self, kmers: &[CanonicalKmer]) -> Vec<usize> {
|
||||
self.mphf.index_batch(kmers)
|
||||
}
|
||||
|
||||
/// Iterate over all canonical kmers in the layer, in deterministic order.
|
||||
pub fn iter_kmers(&self) -> crate::mphf_layer::KmerIter<'_> {
|
||||
self.mphf.iter_kmers()
|
||||
}
|
||||
|
||||
/// Iterate over all canonical kmers, each paired with its zero-based
|
||||
/// sequence index in `unitigs.bin`.
|
||||
pub fn enumerate_kmers(&self) -> std::iter::Enumerate<crate::mphf_layer::KmerIter<'_>> {
|
||||
self.mphf.enumerate_kmers()
|
||||
}
|
||||
|
||||
/// Iterate over the layer's canonical kmers in batches of `n`.
|
||||
pub fn iter_kmers_batch(&self, n: usize) -> crate::mphf_layer::KmerBatchIter<'_> {
|
||||
self.mphf.iter_kmers_batch(n)
|
||||
}
|
||||
|
||||
/// Iterate over batches, each paired with the zero-based index of the
|
||||
/// first kmer in the batch.
|
||||
pub fn enumerate_kmers_batch(&self, n: usize) -> std::iter::Enumerate<crate::mphf_layer::KmerBatchIter<'_>> {
|
||||
self.mphf.enumerate_kmers_batch(n)
|
||||
}
|
||||
|
||||
pub fn unitig_writer(out_dir: &Path) -> OLMResult<UnitigFileWriter> {
|
||||
MphfLayer::unitig_writer(out_dir)
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use std::fs;
|
||||
use std::iter::Enumerate;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use cacheline_ef::{CachelineEf, CachelineEfVec};
|
||||
@@ -29,7 +30,7 @@ type MphfEps = PtrHash<u64, CubicEps, CachelineEfVec<&'static [CachelineEf]>, Xx
|
||||
|
||||
enum LayerEvidence {
|
||||
Exact { evidence: Evidence, unitigs: UnitigFileReader },
|
||||
Approx { fingerprint: FingerprintVec, unitigs_path: PathBuf },
|
||||
Approx { fingerprint: FingerprintVec, unitigs: UnitigFileReader, unitigs_path: PathBuf },
|
||||
Hybrid { evidence: Evidence, unitigs: UnitigFileReader, fingerprint: FingerprintVec },
|
||||
}
|
||||
|
||||
@@ -57,15 +58,15 @@ impl MphfLayer {
|
||||
IndexMode::Exact => {
|
||||
let evidence = Evidence::open(&dir.join(EVIDENCE_FILE))?;
|
||||
let n = evidence.len();
|
||||
// open() auto-detects: uses direct access since exact layers always have .idx
|
||||
let unitigs = UnitigFileReader::open(&dir.join(UNITIGS_FILE))?;
|
||||
(LayerEvidence::Exact { evidence, unitigs }, n)
|
||||
}
|
||||
IndexMode::Approx { .. } => {
|
||||
let fingerprint = FingerprintVec::open(&dir.join(FINGERPRINT_FILE))?;
|
||||
let n = fingerprint.n();
|
||||
let unitigs = UnitigFileReader::open(&dir.join(UNITIGS_FILE))?;
|
||||
let unitigs_path = dir.join(UNITIGS_FILE);
|
||||
(LayerEvidence::Approx { fingerprint, unitigs_path }, n)
|
||||
(LayerEvidence::Approx { fingerprint, unitigs, unitigs_path }, n)
|
||||
}
|
||||
IndexMode::Hybrid { .. } => {
|
||||
let evidence = Evidence::open(&dir.join(EVIDENCE_FILE))?;
|
||||
@@ -89,7 +90,7 @@ impl MphfLayer {
|
||||
let slot = self.mphf.index(&kmer.raw());
|
||||
if slot >= self.n { return None; }
|
||||
match &self.ev {
|
||||
LayerEvidence::Exact { evidence, unitigs } => {
|
||||
LayerEvidence::Exact { evidence, unitigs, .. } => {
|
||||
let (chunk_id, rank) = evidence.decode(slot);
|
||||
if unitigs.verify_canonical_kmer(chunk_id as usize, rank as usize, kmer) {
|
||||
Some(slot)
|
||||
@@ -113,7 +114,7 @@ impl MphfLayer {
|
||||
let slot = self.mphf.index(&kmer.raw());
|
||||
if slot >= self.n { return None; }
|
||||
match &self.ev {
|
||||
LayerEvidence::Exact { evidence, unitigs } |
|
||||
LayerEvidence::Exact { evidence, unitigs, .. } |
|
||||
LayerEvidence::Hybrid { evidence, unitigs, .. } => {
|
||||
let (chunk_id, rank) = evidence.decode(slot);
|
||||
if unitigs.verify_canonical_kmer(chunk_id as usize, rank as usize, kmer) {
|
||||
@@ -149,7 +150,7 @@ impl MphfLayer {
|
||||
return None;
|
||||
}
|
||||
match &self.ev {
|
||||
LayerEvidence::Exact { evidence, unitigs } |
|
||||
LayerEvidence::Exact { evidence, unitigs, .. } |
|
||||
LayerEvidence::Hybrid { evidence, unitigs, .. } => {
|
||||
let (chunk_id, rank) = evidence.decode(slot);
|
||||
let raw = unitigs.canonical_raw_kmer(chunk_id as usize, rank as usize);
|
||||
@@ -160,6 +161,119 @@ impl MphfLayer {
|
||||
}
|
||||
|
||||
pub fn n(&self) -> usize { self.n }
|
||||
|
||||
/// Raw MPHF lookup: kmer → slot.
|
||||
///
|
||||
/// Returns the slot assigned by the MPHF without performing any membership
|
||||
/// check against the layer's evidence. The returned slot is only meaningful
|
||||
/// when the kmer is actually present in the layer; callers that need an
|
||||
/// existence test should use [`find`](Self::find) instead.
|
||||
pub fn index(&self, kmer: CanonicalKmer) -> usize {
|
||||
self.mphf.index(&kmer.raw())
|
||||
}
|
||||
|
||||
/// Batch raw MPHF lookup: kmers → slots.
|
||||
///
|
||||
/// Returns a [`Vec`] of slots, one per input kmer, in the same order as the
|
||||
/// input slice. Like [`index`](Self::index), no membership check is
|
||||
/// performed.
|
||||
pub fn index_batch(&self, kmers: &[CanonicalKmer]) -> Vec<usize> {
|
||||
kmers.iter().map(|k| self.mphf.index(&k.raw())).collect()
|
||||
}
|
||||
|
||||
/// Iterate over all canonical kmers in the layer, in deterministic order.
|
||||
///
|
||||
/// The iteration order follows the physical layout of `unitigs.bin` and is
|
||||
/// **not** correlated with MPHF slot numbers. Multiple `KmerIter` instances
|
||||
/// can be held concurrently for as long as the layer lives, because each
|
||||
/// carries its own cursor.
|
||||
pub fn iter_kmers(&self) -> KmerIter<'_> {
|
||||
let reader = match &self.ev {
|
||||
LayerEvidence::Exact { unitigs, .. } => unitigs,
|
||||
LayerEvidence::Approx { unitigs, .. } => unitigs,
|
||||
LayerEvidence::Hybrid { unitigs, .. } => unitigs,
|
||||
};
|
||||
KmerIter { inner: Box::new(reader.iter_indexed_canonical_kmers()) }
|
||||
}
|
||||
|
||||
/// Iterate over all canonical kmers, each paired with its zero-based
|
||||
/// sequence index in `unitigs.bin`.
|
||||
///
|
||||
/// Yields `(index, kmer)` where `index` starts at `0` for the first kmer
|
||||
/// stored in the layer. This is the standard Rust `enumerate` adapter
|
||||
/// applied to [`iter_kmers`](Self::iter_kmers).
|
||||
pub fn enumerate_kmers(&self) -> Enumerate<KmerIter<'_>> {
|
||||
self.iter_kmers().enumerate()
|
||||
}
|
||||
|
||||
/// Iterate over the layer's canonical kmers in batches of `n`.
|
||||
///
|
||||
/// Each call to [`next`](Iterator::next) returns a [`Vec`] of up to `n`
|
||||
/// kmers. The final batch may be shorter when the layer is exhausted.
|
||||
pub fn iter_kmers_batch(&self, n: usize) -> KmerBatchIter<'_> {
|
||||
KmerBatchIter { inner: self.iter_kmers(), batch_size: n }
|
||||
}
|
||||
|
||||
/// Iterate over batches, each paired with the zero-based index of the
|
||||
/// first kmer in the batch.
|
||||
///
|
||||
/// Yields `(batch_start_index, Vec<CanonicalKmer>)` where
|
||||
/// `batch_start_index` is a multiple of `n`. This is the standard Rust
|
||||
/// `enumerate` adapter applied to [`iter_kmers_batch`](Self::iter_kmers_batch).
|
||||
pub fn enumerate_kmers_batch(&self, n: usize) -> Enumerate<KmerBatchIter<'_>> {
|
||||
self.iter_kmers_batch(n).enumerate()
|
||||
}
|
||||
}
|
||||
|
||||
// ── Iterator types ────────────────────────────────────────────────────────────
|
||||
|
||||
/// Iterator over the canonical kmers stored in a layer.
|
||||
///
|
||||
/// Produced by [`MphfLayer::iter_kmers`]. Multiple `KmerIter` instances can
|
||||
/// coexist concurrently for as long as the parent layer lives, because each
|
||||
/// holds its own cursor over the underlying [`UnitigFileReader`].
|
||||
pub struct KmerIter<'a> {
|
||||
inner: Box<dyn Iterator<Item = (CanonicalKmer, usize, usize)> + 'a>,
|
||||
}
|
||||
|
||||
impl<'a> Iterator for KmerIter<'a> {
|
||||
type Item = CanonicalKmer;
|
||||
|
||||
/// Return the next canonical kmer in iteration order.
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
self.inner.next().map(|(kmer, _, _)| kmer)
|
||||
}
|
||||
}
|
||||
|
||||
/// Iterator over batches of canonical kmers stored in a layer.
|
||||
///
|
||||
/// Produced by [`MphfLayer::iter_kmers_batch`]. Each call to [`next`](Self::next)
|
||||
/// returns a [`Vec`] of up to `batch_size` kmers. The last batch may be shorter
|
||||
/// than `batch_size` when the layer is exhausted.
|
||||
pub struct KmerBatchIter<'a> {
|
||||
inner: KmerIter<'a>,
|
||||
batch_size: usize,
|
||||
}
|
||||
|
||||
impl<'a> Iterator for KmerBatchIter<'a> {
|
||||
type Item = Vec<CanonicalKmer>;
|
||||
|
||||
/// Return the next batch of kmers, or `None` if the layer is exhausted.
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
let mut batch = Vec::with_capacity(self.batch_size);
|
||||
for _ in 0..self.batch_size {
|
||||
if let Some(kmer) = self.inner.next() {
|
||||
batch.push(kmer);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if batch.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(batch)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── MphfOnly ──────────────────────────────────────────────────────────────────
|
||||
|
||||
Reference in New Issue
Block a user