feat: implement per-k-mer sibling counts and central neighbor generation
Introduce the siblingannex module in obicompactvec to store per-slot minorant flags and sibling counts in a memory-mapped annex file. Add a scatter-gather pipeline in obikindex to compute these values across index layers and write them to .psib files. Implement central_canonical_neighbors in obikseq for generating strand-aware k-mer variants around the middle base. Expose rolling statistics in obiskbuilder and update dependency graphs accordingly.
This commit is contained in:
1 parent
8bc6d533e5
commit
ea914bb536
9 files changed
+609
-1
No files matched your search
@@ -7,6 +7,7 @@ mod intmatrix;
|
||||
mod layer_meta;
|
||||
mod meta;
|
||||
mod reader;
|
||||
mod siblingannex;
|
||||
mod tempbitvec;
|
||||
mod tempintvec;
|
||||
mod views;
|
||||
@@ -18,6 +19,7 @@ pub use builder::PersistentCompactIntVecBuilder;
|
||||
pub use colgroup::{ColGroup, FilterMask, MatrixGroupOps, eval_filter_mask};
|
||||
pub use intmatrix::{PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder, pack_compact_int_matrix};
|
||||
pub use layer_meta::LayerMeta;
|
||||
pub use siblingannex::{SiblingAnnex, SiblingAnnexBuilder, SiblingInfo};
|
||||
pub use reader::{PersistentCompactIntVec, Iter as CompactIntVecIter};
|
||||
pub use tempbitvec::{TempBitVec, TempBitVecBuilder};
|
||||
pub use tempintvec::{TempCompactIntVec, TempCompactIntVecBuilder};
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
//! Sibling-count / minorant annex: a compact, read-only-after-build, per-slot
|
||||
//! derived value used by the central-position SNP distance estimator (see
|
||||
//! `docmd/theory/evolutionary_distances.md`, "Step 2b").
|
||||
//!
|
||||
//! One byte is stored per MPHF slot of a partition/layer, encoding two
|
||||
//! independent facts about the slot's k-mer's "family" (the up-to-4 k-mers
|
||||
//! sharing the same flanks, differing only at the central base), both
|
||||
//! properties of the whole current multi-genome index, not of any one
|
||||
//! genome:
|
||||
//!
|
||||
//! - bit 0: `minorant` — is this k-mer's canonical encoding the smallest
|
||||
//! among the family members actually observed in the index?
|
||||
//! - bits 1-2: `siblings` — how many *other* family members (0-3) are
|
||||
//! observed anywhere in the index.
|
||||
//!
|
||||
//! Byte value 0 (`minorant = false`, `siblings = 0`) is logically
|
||||
//! unreachable as a real result (0 siblings always implies minorant — see
|
||||
//! the design doc) and is reused as the "not yet computed" sentinel: annex
|
||||
//! files are pre-initialised to all-zero, and a real value is only ever
|
||||
//! written once, by the computation pass.
|
||||
//!
|
||||
//! Deliberately simpler than a true 3-bit pack (1 byte/slot instead of 3
|
||||
//! bits/slot): correctness and simplicity first: for a first implementation.
|
||||
//! Packing to 3 bits/slot is a pure storage-density follow-up, not a
|
||||
//! behavioural change, left for later.
|
||||
|
||||
use std::fs::{File, OpenOptions};
|
||||
use std::io;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use memmap2::{Mmap, MmapMut};
|
||||
|
||||
const MAGIC: [u8; 4] = *b"PSIB";
|
||||
|
||||
// Header: magic(4) + _pad(4) + n(8) = 16 bytes. Data (1 byte/slot) follows.
|
||||
const HEADER_SIZE: usize = 16;
|
||||
|
||||
/// Decoded value of one slot's annex entry.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct SiblingInfo {
|
||||
pub minorant: bool,
|
||||
pub siblings: u8, // 0..=3
|
||||
}
|
||||
|
||||
impl SiblingInfo {
|
||||
#[inline]
|
||||
fn encode(self) -> u8 {
|
||||
(self.siblings << 1) | (self.minorant as u8)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn decode(byte: u8) -> Option<Self> {
|
||||
if byte == 0 {
|
||||
// The unreachable "not minorant + 0 siblings" combination —
|
||||
// reserved as the "not yet computed" sentinel.
|
||||
return None;
|
||||
}
|
||||
Some(SiblingInfo {
|
||||
minorant: byte & 1 != 0,
|
||||
siblings: (byte >> 1) & 0b11,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ── SiblingAnnex (reader) ───────────────────────────────────────────────────
|
||||
|
||||
pub struct SiblingAnnex {
|
||||
mmap: Mmap,
|
||||
n: usize,
|
||||
path: PathBuf,
|
||||
}
|
||||
|
||||
impl SiblingAnnex {
|
||||
pub fn open(path: &Path) -> io::Result<Self> {
|
||||
let mmap = unsafe { Mmap::map(&File::open(path)?)? };
|
||||
if mmap.len() < HEADER_SIZE {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "PSIB file too short"));
|
||||
}
|
||||
if mmap[0..4] != MAGIC {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "bad PSIB magic"));
|
||||
}
|
||||
let n = u64::from_le_bytes(mmap[8..16].try_into().unwrap()) as usize;
|
||||
if mmap.len() < HEADER_SIZE + n {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "PSIB file truncated"));
|
||||
}
|
||||
Ok(Self { mmap, n, path: path.to_path_buf() })
|
||||
}
|
||||
|
||||
pub fn path(&self) -> &Path { &self.path }
|
||||
pub fn len(&self) -> usize { self.n }
|
||||
pub fn is_empty(&self) -> bool { self.n == 0 }
|
||||
|
||||
/// `None` means the slot has not (yet) been computed — see module docs.
|
||||
pub fn get(&self, slot: usize) -> Option<SiblingInfo> {
|
||||
SiblingInfo::decode(self.mmap[HEADER_SIZE + slot])
|
||||
}
|
||||
}
|
||||
|
||||
// ── SiblingAnnexBuilder (writer) ────────────────────────────────────────────
|
||||
|
||||
pub struct SiblingAnnexBuilder {
|
||||
mmap: MmapMut,
|
||||
n: usize,
|
||||
path: PathBuf,
|
||||
}
|
||||
|
||||
impl SiblingAnnexBuilder {
|
||||
/// Create a new annex of `n` slots at `path`, pre-initialised to the
|
||||
/// "not yet computed" sentinel (all-zero).
|
||||
pub fn new(n: usize, path: &Path) -> io::Result<Self> {
|
||||
let file_size = HEADER_SIZE + n;
|
||||
let file = OpenOptions::new()
|
||||
.read(true).write(true).create(true).truncate(true)
|
||||
.open(path)?;
|
||||
file.set_len(file_size as u64)?;
|
||||
let mut mmap = unsafe { MmapMut::map_mut(&file)? };
|
||||
mmap[0..4].copy_from_slice(&MAGIC);
|
||||
mmap[4..8].copy_from_slice(&[0u8; 4]);
|
||||
mmap[8..16].copy_from_slice(&(n as u64).to_le_bytes());
|
||||
// Data region left at 0 by `set_len`/mmap — the sentinel value.
|
||||
Ok(Self { mmap, n, path: path.to_path_buf() })
|
||||
}
|
||||
|
||||
pub fn len(&self) -> usize { self.n }
|
||||
pub fn is_empty(&self) -> bool { self.n == 0 }
|
||||
|
||||
pub fn get(&self, slot: usize) -> Option<SiblingInfo> {
|
||||
SiblingInfo::decode(self.mmap[HEADER_SIZE + slot])
|
||||
}
|
||||
|
||||
pub fn set(&mut self, slot: usize, info: SiblingInfo) {
|
||||
// Redundant concurrent writes from independent recomputation paths
|
||||
// converge to the same encoded byte for a given slot, so a plain
|
||||
// store here is safe even without external synchronisation, as long
|
||||
// as the byte write itself is atomic (true for a single aligned
|
||||
// byte on every platform this project targets).
|
||||
self.mmap[HEADER_SIZE + slot] = info.encode();
|
||||
}
|
||||
|
||||
pub fn close(self) -> io::Result<()> { self.mmap.flush() }
|
||||
|
||||
pub fn finish(self) -> io::Result<SiblingAnnex> {
|
||||
let path = self.path.clone();
|
||||
self.close()?;
|
||||
SiblingAnnex::open(&path)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::tempdir;
|
||||
|
||||
#[test]
|
||||
fn sentinel_is_zero_and_unset_slots_read_as_uncomputed() {
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("test.psib");
|
||||
let builder = SiblingAnnexBuilder::new(4, &path).unwrap();
|
||||
for slot in 0..4 {
|
||||
assert_eq!(builder.get(slot), None);
|
||||
}
|
||||
builder.close().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn roundtrip_all_valid_states() {
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("test.psib");
|
||||
let mut builder = SiblingAnnexBuilder::new(5, &path).unwrap();
|
||||
|
||||
let cases = [
|
||||
SiblingInfo { minorant: true, siblings: 0 },
|
||||
SiblingInfo { minorant: true, siblings: 1 },
|
||||
SiblingInfo { minorant: true, siblings: 2 },
|
||||
SiblingInfo { minorant: true, siblings: 3 },
|
||||
SiblingInfo { minorant: false, siblings: 2 },
|
||||
];
|
||||
for (slot, info) in cases.iter().enumerate() {
|
||||
builder.set(slot, *info);
|
||||
}
|
||||
let annex = builder.finish().unwrap();
|
||||
for (slot, info) in cases.iter().enumerate() {
|
||||
assert_eq!(annex.get(slot), Some(*info));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn not_minorant_zero_siblings_is_unreachable_via_set_and_decodes_as_sentinel() {
|
||||
// Documented invariant, not enforced by the type: callers must never
|
||||
// construct this combination. If they do, it is indistinguishable
|
||||
// from "not computed" — exercised here to pin the behaviour down.
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("test.psib");
|
||||
let mut builder = SiblingAnnexBuilder::new(1, &path).unwrap();
|
||||
builder.set(0, SiblingInfo { minorant: false, siblings: 0 });
|
||||
assert_eq!(builder.get(0), None);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user