feat: implement per-k-mer sibling counts and central neighbor generation

Introduce the siblingannex module in obicompactvec to store per-slot minorant flags and sibling counts in a memory-mapped annex file. Add a scatter-gather pipeline in obikindex to compute these values across index layers and write them to .psib files. Implement central_canonical_neighbors in obikseq for generating strand-aware k-mer variants around the middle base. Expose rolling statistics in obiskbuilder and update dependency graphs accordingly.
This commit is contained in:
Eric Coissac committed 2026-08-10 15:01:59 +02:00
1 parent 8bc6d533e5
commit ea914bb536
9 files changed
+609 -1

No files matched your search

+2
View File
@@ -7,6 +7,7 @@ mod intmatrix;
mod layer_meta;
mod meta;
mod reader;
mod siblingannex;
mod tempbitvec;
mod tempintvec;
mod views;
@@ -18,6 +19,7 @@ pub use builder::PersistentCompactIntVecBuilder;
pub use colgroup::{ColGroup, FilterMask, MatrixGroupOps, eval_filter_mask};
pub use intmatrix::{PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder, pack_compact_int_matrix};
pub use layer_meta::LayerMeta;
pub use siblingannex::{SiblingAnnex, SiblingAnnexBuilder, SiblingInfo};
pub use reader::{PersistentCompactIntVec, Iter as CompactIntVecIter};
pub use tempbitvec::{TempBitVec, TempBitVecBuilder};
pub use tempintvec::{TempCompactIntVec, TempCompactIntVecBuilder};
+198
View File
@@ -0,0 +1,198 @@
//! Sibling-count / minorant annex: a compact, read-only-after-build, per-slot
//! derived value used by the central-position SNP distance estimator (see
//! `docmd/theory/evolutionary_distances.md`, "Step 2b").
//!
//! One byte is stored per MPHF slot of a partition/layer, encoding two
//! independent facts about the slot's k-mer's "family" (the up-to-4 k-mers
//! sharing the same flanks, differing only at the central base), both
//! properties of the whole current multi-genome index, not of any one
//! genome:
//!
//! - bit 0: `minorant` — is this k-mer's canonical encoding the smallest
//! among the family members actually observed in the index?
//! - bits 1-2: `siblings` — how many *other* family members (0-3) are
//! observed anywhere in the index.
//!
//! Byte value 0 (`minorant = false`, `siblings = 0`) is logically
//! unreachable as a real result (0 siblings always implies minorant — see
//! the design doc) and is reused as the "not yet computed" sentinel: annex
//! files are pre-initialised to all-zero, and a real value is only ever
//! written once, by the computation pass.
//!
//! Deliberately simpler than a true 3-bit pack (1 byte/slot instead of 3
//! bits/slot): correctness and simplicity first: for a first implementation.
//! Packing to 3 bits/slot is a pure storage-density follow-up, not a
//! behavioural change, left for later.
use std::fs::{File, OpenOptions};
use std::io;
use std::path::{Path, PathBuf};
use memmap2::{Mmap, MmapMut};
const MAGIC: [u8; 4] = *b"PSIB";
// Header: magic(4) + _pad(4) + n(8) = 16 bytes. Data (1 byte/slot) follows.
const HEADER_SIZE: usize = 16;
/// Decoded value of one slot's annex entry.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct SiblingInfo {
pub minorant: bool,
pub siblings: u8, // 0..=3
}
impl SiblingInfo {
#[inline]
fn encode(self) -> u8 {
(self.siblings << 1) | (self.minorant as u8)
}
#[inline]
fn decode(byte: u8) -> Option<Self> {
if byte == 0 {
// The unreachable "not minorant + 0 siblings" combination —
// reserved as the "not yet computed" sentinel.
return None;
}
Some(SiblingInfo {
minorant: byte & 1 != 0,
siblings: (byte >> 1) & 0b11,
})
}
}
// ── SiblingAnnex (reader) ───────────────────────────────────────────────────
pub struct SiblingAnnex {
mmap: Mmap,
n: usize,
path: PathBuf,
}
impl SiblingAnnex {
pub fn open(path: &Path) -> io::Result<Self> {
let mmap = unsafe { Mmap::map(&File::open(path)?)? };
if mmap.len() < HEADER_SIZE {
return Err(io::Error::new(io::ErrorKind::InvalidData, "PSIB file too short"));
}
if mmap[0..4] != MAGIC {
return Err(io::Error::new(io::ErrorKind::InvalidData, "bad PSIB magic"));
}
let n = u64::from_le_bytes(mmap[8..16].try_into().unwrap()) as usize;
if mmap.len() < HEADER_SIZE + n {
return Err(io::Error::new(io::ErrorKind::InvalidData, "PSIB file truncated"));
}
Ok(Self { mmap, n, path: path.to_path_buf() })
}
pub fn path(&self) -> &Path { &self.path }
pub fn len(&self) -> usize { self.n }
pub fn is_empty(&self) -> bool { self.n == 0 }
/// `None` means the slot has not (yet) been computed — see module docs.
pub fn get(&self, slot: usize) -> Option<SiblingInfo> {
SiblingInfo::decode(self.mmap[HEADER_SIZE + slot])
}
}
// ── SiblingAnnexBuilder (writer) ────────────────────────────────────────────
pub struct SiblingAnnexBuilder {
mmap: MmapMut,
n: usize,
path: PathBuf,
}
impl SiblingAnnexBuilder {
/// Create a new annex of `n` slots at `path`, pre-initialised to the
/// "not yet computed" sentinel (all-zero).
pub fn new(n: usize, path: &Path) -> io::Result<Self> {
let file_size = HEADER_SIZE + n;
let file = OpenOptions::new()
.read(true).write(true).create(true).truncate(true)
.open(path)?;
file.set_len(file_size as u64)?;
let mut mmap = unsafe { MmapMut::map_mut(&file)? };
mmap[0..4].copy_from_slice(&MAGIC);
mmap[4..8].copy_from_slice(&[0u8; 4]);
mmap[8..16].copy_from_slice(&(n as u64).to_le_bytes());
// Data region left at 0 by `set_len`/mmap — the sentinel value.
Ok(Self { mmap, n, path: path.to_path_buf() })
}
pub fn len(&self) -> usize { self.n }
pub fn is_empty(&self) -> bool { self.n == 0 }
pub fn get(&self, slot: usize) -> Option<SiblingInfo> {
SiblingInfo::decode(self.mmap[HEADER_SIZE + slot])
}
pub fn set(&mut self, slot: usize, info: SiblingInfo) {
// Redundant concurrent writes from independent recomputation paths
// converge to the same encoded byte for a given slot, so a plain
// store here is safe even without external synchronisation, as long
// as the byte write itself is atomic (true for a single aligned
// byte on every platform this project targets).
self.mmap[HEADER_SIZE + slot] = info.encode();
}
pub fn close(self) -> io::Result<()> { self.mmap.flush() }
pub fn finish(self) -> io::Result<SiblingAnnex> {
let path = self.path.clone();
self.close()?;
SiblingAnnex::open(&path)
}
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::tempdir;
#[test]
fn sentinel_is_zero_and_unset_slots_read_as_uncomputed() {
let dir = tempdir().unwrap();
let path = dir.path().join("test.psib");
let builder = SiblingAnnexBuilder::new(4, &path).unwrap();
for slot in 0..4 {
assert_eq!(builder.get(slot), None);
}
builder.close().unwrap();
}
#[test]
fn roundtrip_all_valid_states() {
let dir = tempdir().unwrap();
let path = dir.path().join("test.psib");
let mut builder = SiblingAnnexBuilder::new(5, &path).unwrap();
let cases = [
SiblingInfo { minorant: true, siblings: 0 },
SiblingInfo { minorant: true, siblings: 1 },
SiblingInfo { minorant: true, siblings: 2 },
SiblingInfo { minorant: true, siblings: 3 },
SiblingInfo { minorant: false, siblings: 2 },
];
for (slot, info) in cases.iter().enumerate() {
builder.set(slot, *info);
}
let annex = builder.finish().unwrap();
for (slot, info) in cases.iter().enumerate() {
assert_eq!(annex.get(slot), Some(*info));
}
}
#[test]
fn not_minorant_zero_siblings_is_unreachable_via_set_and_decodes_as_sentinel() {
// Documented invariant, not enforced by the type: callers must never
// construct this combination. If they do, it is indistinguishable
// from "not computed" — exercised here to pin the behaviour down.
let dir = tempdir().unwrap();
let path = dir.path().join("test.psib");
let mut builder = SiblingAnnexBuilder::new(1, &path).unwrap();
builder.set(0, SiblingInfo { minorant: false, siblings: 0 });
assert_eq!(builder.get(0), None);
}
}