Refactor k-mer sibling tracking to compact bitmask and on-demand counts

Replaces the explicit `SiblingInfo` struct and 3-bit minorant flags with a derived 4-bit presence mask (`FamilyMask`) that tracks observed bases per family. This eliminates redundant file I/O overhead by introducing a `PartitionCache` for batch lookups, simplifies serialization, and updates all downstream builders, stats computation, and tests to operate on the new bitmask representation. Adjusts CLI output to report deduplicated family sizes instead of histograms, ignores generated CSV files, and updates documentation to reflect the fixed canonical reference and new theory.
This commit is contained in:
Eric Coissac committed 2026-08-10 17:53:52 +02:00
1 parent ba990a48a0
commit 1a470eab9e
6 files changed
+671 -277

No files matched your search

+1 -1
View File
@@ -19,7 +19,7 @@ pub use builder::PersistentCompactIntVecBuilder;
pub use colgroup::{ColGroup, FilterMask, MatrixGroupOps, eval_filter_mask};
pub use intmatrix::{PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder, pack_compact_int_matrix};
pub use layer_meta::LayerMeta;
pub use siblingannex::{SiblingAnnex, SiblingAnnexBuilder, SiblingInfo};
pub use siblingannex::{FamilyMask, SiblingAnnex, SiblingAnnexBuilder};
pub use reader::{PersistentCompactIntVec, Iter as CompactIntVecIter};
pub use tempbitvec::{TempBitVec, TempBitVecBuilder};
pub use tempintvec::{TempCompactIntVec, TempCompactIntVecBuilder};
+106 -59
View File
@@ -1,27 +1,33 @@
//! Sibling-count / minorant annex: a compact, read-only-after-build, per-slot
//! Family presence-mask annex: a compact, read-only-after-build, per-slot
//! derived value used by the central-position SNP distance estimator (see
//! `docmd/theory/evolutionary_distances.md`, "Step 2b").
//! `docmd/theory/evolutionary_distances.md`, "Step 2b" and "Definitions:
//! family, and the canonical form of a family").
//!
//! One byte is stored per MPHF slot of a partition/layer, encoding two
//! independent facts about the slot's k-mer's "family" (the up-to-4 k-mers
//! sharing the same flanks, differing only at the central base), both
//! properties of the whole current multi-genome index, not of any one
//! genome:
//! One byte is stored per MPHF slot of a partition/layer, its low 4 bits
//! encoding a **presence mask** for the slot's k-mer's "family" (the up to 4
//! k-mers sharing the same flanks, differing only at the central base):
//! bit `b` (`b` = 0..3, in the fixed A/C/G/T = 0/1/2/3 encoding already used
//! for a single nucleotide) is set iff the family member whose *own* central
//! base — in its own canonical orientation — is `b`, is observed anywhere in
//! the current multi-genome index. This is a property of the whole index,
//! not of any one genome.
//!
//! - bit 0: `minorant` — is this k-mer's canonical encoding the smallest
//! among the family members actually observed in the index?
//! - bits 1-2: `siblings` — how many *other* family members (0-3) are
//! observed anywhere in the index.
//! Both facts the earlier (superseded) 3-bit design stored explicitly are
//! derived from the mask instead, not stored:
//! - sibling count = `popcount(mask) - 1`;
//! - minorant = regenerate the family's 4 canonical forms from the slot's
//! own k-mer (`CanonicalKmerOf::central_canonical_neighbors`, cheap, no
//! lookup), compare the raw encodings of whichever are set in the mask,
//! take the smallest — see `obikindex::siblings`.
//!
//! Byte value 0 (`minorant = false`, `siblings = 0`) is logically
//! unreachable as a real result (0 siblings always implies minorant — see
//! the design doc) and is reused as the "not yet computed" sentinel: annex
//! files are pre-initialised to all-zero, and a real value is only ever
//! written once, by the computation pass.
//! Mask value 0 is logically unreachable as a real result (a slot's own base
//! is always present in its own family) and is reused as the "not yet
//! computed" sentinel: annex files are pre-initialised to all-zero, and a
//! real value is only ever written once, by the computation pass.
//!
//! Deliberately simpler than a true 3-bit pack (1 byte/slot instead of 3
//! bits/slot): correctness and simplicity first: for a first implementation.
//! Packing to 3 bits/slot is a pure storage-density follow-up, not a
//! Deliberately simpler than a true 4-bit pack (1 byte/slot instead of 4
//! bits/slot): correctness and simplicity first, for a first implementation.
//! Packing to 4 bits/slot is a pure storage-density follow-up, not a
//! behavioural change, left for later.
use std::fs::{File, OpenOptions};
@@ -35,30 +41,70 @@ const MAGIC: [u8; 4] = *b"PSIB";
// Header: magic(4) + _pad(4) + n(8) = 16 bytes. Data (1 byte/slot) follows.
const HEADER_SIZE: usize = 16;
/// Decoded value of one slot's annex entry.
/// A family presence mask: bit `b` set iff the member whose own canonical
/// central base is `b` (0=A, 1=C, 2=G, 3=T) is observed in the index.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct SiblingInfo {
pub minorant: bool,
pub siblings: u8, // 0..=3
}
pub struct FamilyMask(u8);
impl FamilyMask {
/// The empty mask — never a valid *computed* result (a slot's own base
/// is always present in its own family) — used only to build up a mask
/// via repeated [`with`](Self::with) calls before storing it.
pub const EMPTY: FamilyMask = FamilyMask(0);
/// Set bit `base` (0=A, 1=C, 2=G, 3=T).
#[inline]
pub fn with(self, base: u8) -> Self {
debug_assert!(base < 4, "base out of range: {base}");
FamilyMask(self.0 | (1 << base))
}
/// Is the member with central base `base` (0..3) present?
#[inline]
pub fn has(self, base: u8) -> bool {
debug_assert!(base < 4, "base out of range: {base}");
self.0 & (1 << base) != 0
}
/// Number of family members observed anywhere in the index (1..=4).
#[inline]
pub fn family_size(self) -> u32 {
self.0.count_ones()
}
/// Number of *other* members observed (0..=3) — `family_size() - 1`.
#[inline]
pub fn siblings(self) -> u32 {
self.family_size() - 1
}
/// Raw bitmask (bit `b` = base `b` present) — for callers that build up
/// a mask via their own bit operations (e.g. concurrently, via an
/// `AtomicU8`) and only need the `FamilyMask` wrapper at the end.
#[inline]
pub fn bits(self) -> u8 {
self.0
}
/// Construct from a raw bitmask (only the low 4 bits are kept).
#[inline]
pub fn from_bits(bits: u8) -> Self {
FamilyMask(bits & 0b1111)
}
impl SiblingInfo {
#[inline]
fn encode(self) -> u8 {
(self.siblings << 1) | (self.minorant as u8)
self.0
}
#[inline]
fn decode(byte: u8) -> Option<Self> {
if byte == 0 {
// The unreachable "not minorant + 0 siblings" combination —
// reserved as the "not yet computed" sentinel.
// Unreachable for a real result — reserved as the "not yet
// computed" sentinel.
return None;
}
Some(SiblingInfo {
minorant: byte & 1 != 0,
siblings: (byte >> 1) & 0b11,
})
Some(FamilyMask(byte & 0b1111))
}
}
@@ -91,8 +137,8 @@ impl SiblingAnnex {
pub fn is_empty(&self) -> bool { self.n == 0 }
/// `None` means the slot has not (yet) been computed — see module docs.
pub fn get(&self, slot: usize) -> Option<SiblingInfo> {
SiblingInfo::decode(self.mmap[HEADER_SIZE + slot])
pub fn get(&self, slot: usize) -> Option<FamilyMask> {
FamilyMask::decode(self.mmap[HEADER_SIZE + slot])
}
}
@@ -124,17 +170,17 @@ impl SiblingAnnexBuilder {
pub fn len(&self) -> usize { self.n }
pub fn is_empty(&self) -> bool { self.n == 0 }
pub fn get(&self, slot: usize) -> Option<SiblingInfo> {
SiblingInfo::decode(self.mmap[HEADER_SIZE + slot])
pub fn get(&self, slot: usize) -> Option<FamilyMask> {
FamilyMask::decode(self.mmap[HEADER_SIZE + slot])
}
pub fn set(&mut self, slot: usize, info: SiblingInfo) {
pub fn set(&mut self, slot: usize, mask: FamilyMask) {
// Redundant concurrent writes from independent recomputation paths
// converge to the same encoded byte for a given slot, so a plain
// store here is safe even without external synchronisation, as long
// as the byte write itself is atomic (true for a single aligned
// byte on every platform this project targets).
self.mmap[HEADER_SIZE + slot] = info.encode();
self.mmap[HEADER_SIZE + slot] = mask.encode();
}
pub fn close(self) -> io::Result<()> { self.mmap.flush() }
@@ -163,36 +209,37 @@ mod tests {
}
#[test]
fn roundtrip_all_valid_states() {
fn roundtrip_all_valid_masks() {
let dir = tempdir().unwrap();
let path = dir.path().join("test.psib");
let mut builder = SiblingAnnexBuilder::new(5, &path).unwrap();
let mut builder = SiblingAnnexBuilder::new(4, &path).unwrap();
let cases = [
SiblingInfo { minorant: true, siblings: 0 },
SiblingInfo { minorant: true, siblings: 1 },
SiblingInfo { minorant: true, siblings: 2 },
SiblingInfo { minorant: true, siblings: 3 },
SiblingInfo { minorant: false, siblings: 2 },
let masks = [
FamilyMask::EMPTY.with(0), // just A: family size 1
FamilyMask::EMPTY.with(0).with(3), // A + T: size 2
FamilyMask::EMPTY.with(1).with(2).with(3), // C+G+T: size 3
FamilyMask::EMPTY.with(0).with(1).with(2).with(3), // all 4
];
for (slot, info) in cases.iter().enumerate() {
builder.set(slot, *info);
for (slot, mask) in masks.iter().enumerate() {
builder.set(slot, *mask);
}
let annex = builder.finish().unwrap();
for (slot, info) in cases.iter().enumerate() {
assert_eq!(annex.get(slot), Some(*info));
for (slot, mask) in masks.iter().enumerate() {
assert_eq!(annex.get(slot), Some(*mask));
}
assert_eq!(annex.get(0).unwrap().siblings(), 0);
assert_eq!(annex.get(1).unwrap().siblings(), 1);
assert_eq!(annex.get(2).unwrap().siblings(), 2);
assert_eq!(annex.get(3).unwrap().siblings(), 3);
assert_eq!(annex.get(3).unwrap().family_size(), 4);
}
#[test]
fn not_minorant_zero_siblings_is_unreachable_via_set_and_decodes_as_sentinel() {
// Documented invariant, not enforced by the type: callers must never
// construct this combination. If they do, it is indistinguishable
// from "not computed" — exercised here to pin the behaviour down.
let dir = tempdir().unwrap();
let path = dir.path().join("test.psib");
let mut builder = SiblingAnnexBuilder::new(1, &path).unwrap();
builder.set(0, SiblingInfo { minorant: false, siblings: 0 });
assert_eq!(builder.get(0), None);
fn has_reflects_individual_bits() {
let mask = FamilyMask::EMPTY.with(0).with(2);
assert!(mask.has(0));
assert!(!mask.has(1));
assert!(mask.has(2));
assert!(!mask.has(3));
}
}