Move sibling annex implementation to obikphylo siblings module
Relocates `SiblingAnnex`, `FamilyMask`, and `SiblingAnnexBuilder` from `obicompactvec` to the local `obikphylo::siblings` module. Replaces the previous implementation with a memory-mapped version using `memmap2`, featuring a 2-byte-per-slot layout, explicit bitfield manipulation, and support for concurrent atomic writes. Updates all sibling module imports to local paths and adds the `memmap2` dependency to `obikphylo`.
This commit is contained in:
1 parent
c990087ef3
commit
fecfe84ea6
13 files changed
+404
-293
No files matched your search
@@ -7,7 +7,6 @@ mod intmatrix;
|
||||
mod layer_meta;
|
||||
mod meta;
|
||||
mod reader;
|
||||
mod siblingannex;
|
||||
mod tempbitvec;
|
||||
mod tempintvec;
|
||||
mod views;
|
||||
@@ -19,7 +18,6 @@ pub use builder::PersistentCompactIntVecBuilder;
|
||||
pub use colgroup::{ColGroup, FilterMask, MatrixGroupOps, eval_filter_mask};
|
||||
pub use intmatrix::{PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder, pack_compact_int_matrix};
|
||||
pub use layer_meta::LayerMeta;
|
||||
pub use siblingannex::{FamilyMask, SiblingAnnex, SiblingAnnexBuilder};
|
||||
pub use reader::{PersistentCompactIntVec, Iter as CompactIntVecIter};
|
||||
pub use tempbitvec::{TempBitVec, TempBitVecBuilder};
|
||||
pub use tempintvec::{TempCompactIntVec, TempCompactIntVecBuilder};
|
||||
|
||||
@@ -1,280 +0,0 @@
|
||||
//! Family presence-mask annex: a compact, read-only-after-build, per-slot
|
||||
//! derived value used by the central-position SNP distance estimator (see
|
||||
//! `docmd/theory/evolutionary_distances.md`, "Step 2b" and "Definitions:
|
||||
//! family, and the canonical form of a family").
|
||||
//!
|
||||
//! One byte is stored per MPHF slot of a partition/layer, its low 4 bits
|
||||
//! encoding a **presence mask** for the slot's k-mer's "family" (the up to 4
|
||||
//! k-mers sharing the same flanks, differing only at the central base):
|
||||
//! bit `b` (`b` = 0..3, in the fixed A/C/G/T = 0/1/2/3 encoding already used
|
||||
//! for a single nucleotide) is set iff the family member whose *own* central
|
||||
//! base — in its own canonical orientation — is `b`, is observed anywhere in
|
||||
//! the current multi-genome index. This is a property of the whole index,
|
||||
//! not of any one genome.
|
||||
//!
|
||||
//! Both facts the earlier (superseded) 3-bit design stored explicitly are
|
||||
//! derived from the mask instead, not stored:
|
||||
//! - sibling count = `popcount(mask) - 1`;
|
||||
//! - minorant = regenerate the family's 4 canonical forms from the slot's
|
||||
//! own k-mer (`CanonicalKmerOf::central_canonical_neighbors`, cheap, no
|
||||
//! lookup), compare the raw encodings of whichever are set in the mask,
|
||||
//! take the smallest — see `obikindex::siblings`.
|
||||
//!
|
||||
//! Mask value 0 is logically unreachable as a real result (a slot's own base
|
||||
//! is always present in its own family) and is reused as the "not yet
|
||||
//! computed" sentinel: annex files are pre-initialised to all-zero, and a
|
||||
//! real value is only ever written once, by the computation pass.
|
||||
//!
|
||||
//! Deliberately simpler than a true 4-bit pack (1 byte/slot instead of 4
|
||||
//! bits/slot): correctness and simplicity first, for a first implementation.
|
||||
//! Packing to 4 bits/slot is a pure storage-density follow-up, not a
|
||||
//! behavioural change, left for later.
|
||||
|
||||
use std::fs::{File, OpenOptions};
|
||||
use std::io;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use memmap2::{Mmap, MmapMut};
|
||||
|
||||
const MAGIC: [u8; 4] = *b"PSIB";
|
||||
|
||||
// Header: magic(4) + _pad(4) + n(8) = 16 bytes. Data (1 byte/slot) follows.
|
||||
const HEADER_SIZE: usize = 16;
|
||||
|
||||
/// A family presence mask: bit `b` (0..3) set iff the member whose own
|
||||
/// canonical central base is `b` (0=A, 1=C, 2=G, 3=T) is observed in the
|
||||
/// index. Bit 4 is a second, independent fact piggy-backed onto the same
|
||||
/// byte: whether *this slot's own k-mer* is its family's minorant (the
|
||||
/// smallest raw encoding among the family's observed members) — computed
|
||||
/// once, when the whole family's mask is already final (see
|
||||
/// `obikindex::siblings::build_sibling_annex`), and read back by every
|
||||
/// consumer that would otherwise have to reconstruct this slot's k-mer from
|
||||
/// `unitigs.bin` and hash it through the MPHF again just to ask the same
|
||||
/// question. Bits 5-7 unused.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct FamilyMask(u8);
|
||||
|
||||
const PRESENCE_BITS: u8 = 0b0000_1111;
|
||||
const MINORANT_BIT: u8 = 0b0001_0000;
|
||||
|
||||
impl FamilyMask {
|
||||
/// The empty mask — never a valid *computed* result (a slot's own base
|
||||
/// is always present in its own family) — used only to build up a mask
|
||||
/// via repeated [`with`](Self::with) calls before storing it.
|
||||
pub const EMPTY: FamilyMask = FamilyMask(0);
|
||||
|
||||
/// Set bit `base` (0=A, 1=C, 2=G, 3=T).
|
||||
#[inline]
|
||||
pub fn with(self, base: u8) -> Self {
|
||||
debug_assert!(base < 4, "base out of range: {base}");
|
||||
FamilyMask(self.0 | (1 << base))
|
||||
}
|
||||
|
||||
/// Is the member with central base `base` (0..3) present?
|
||||
#[inline]
|
||||
pub fn has(self, base: u8) -> bool {
|
||||
debug_assert!(base < 4, "base out of range: {base}");
|
||||
self.0 & (1 << base) != 0
|
||||
}
|
||||
|
||||
/// Number of family members observed anywhere in the index (1..=4).
|
||||
#[inline]
|
||||
pub fn family_size(self) -> u32 {
|
||||
(self.0 & PRESENCE_BITS).count_ones()
|
||||
}
|
||||
|
||||
/// Number of *other* members observed (0..=3) — `family_size() - 1`.
|
||||
#[inline]
|
||||
pub fn siblings(self) -> u32 {
|
||||
self.family_size() - 1
|
||||
}
|
||||
|
||||
/// Set or clear the minorant flag (see the struct docs).
|
||||
#[inline]
|
||||
pub fn with_minorant(self, is_minorant: bool) -> Self {
|
||||
if is_minorant {
|
||||
FamilyMask(self.0 | MINORANT_BIT)
|
||||
} else {
|
||||
FamilyMask(self.0 & !MINORANT_BIT)
|
||||
}
|
||||
}
|
||||
|
||||
/// Is this slot's own k-mer its family's minorant? Only meaningful once
|
||||
/// `with_minorant` has been called with the family's *final* mask (i.e.
|
||||
/// after construction) — see the struct docs.
|
||||
#[inline]
|
||||
pub fn is_minorant(self) -> bool {
|
||||
self.0 & MINORANT_BIT != 0
|
||||
}
|
||||
|
||||
/// Raw presence bitmask (bit `b` = base `b` present, low 4 bits only —
|
||||
/// never includes the minorant flag) — for callers that build up a mask
|
||||
/// via their own bit operations (e.g. concurrently, via an `AtomicU8`)
|
||||
/// and only need the `FamilyMask` wrapper at the end.
|
||||
#[inline]
|
||||
pub fn bits(self) -> u8 {
|
||||
self.0 & PRESENCE_BITS
|
||||
}
|
||||
|
||||
/// Construct from a raw presence bitmask (only the low 4 bits are kept —
|
||||
/// the minorant flag is not part of this, use [`with_minorant`](Self::with_minorant)
|
||||
/// separately).
|
||||
#[inline]
|
||||
pub fn from_bits(bits: u8) -> Self {
|
||||
FamilyMask(bits & PRESENCE_BITS)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn encode(self) -> u8 {
|
||||
self.0
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn decode(byte: u8) -> Option<Self> {
|
||||
if byte == 0 {
|
||||
// Unreachable for a real result — reserved as the "not yet
|
||||
// computed" sentinel. Still safe as a sentinel with the
|
||||
// minorant bit added: a real entry always has at least one
|
||||
// presence bit set (bits 0-3), so byte == 0 still means
|
||||
// "nothing written yet" and never a genuine minorant-only value.
|
||||
return None;
|
||||
}
|
||||
Some(FamilyMask(byte))
|
||||
}
|
||||
}
|
||||
|
||||
// ── SiblingAnnex (reader) ───────────────────────────────────────────────────
|
||||
|
||||
pub struct SiblingAnnex {
|
||||
mmap: Mmap,
|
||||
n: usize,
|
||||
path: PathBuf,
|
||||
}
|
||||
|
||||
impl SiblingAnnex {
|
||||
pub fn open(path: &Path) -> io::Result<Self> {
|
||||
let mmap = unsafe { Mmap::map(&File::open(path)?)? };
|
||||
if mmap.len() < HEADER_SIZE {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "PSIB file too short"));
|
||||
}
|
||||
if mmap[0..4] != MAGIC {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "bad PSIB magic"));
|
||||
}
|
||||
let n = u64::from_le_bytes(mmap[8..16].try_into().unwrap()) as usize;
|
||||
if mmap.len() < HEADER_SIZE + n {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "PSIB file truncated"));
|
||||
}
|
||||
Ok(Self { mmap, n, path: path.to_path_buf() })
|
||||
}
|
||||
|
||||
pub fn path(&self) -> &Path { &self.path }
|
||||
pub fn len(&self) -> usize { self.n }
|
||||
pub fn is_empty(&self) -> bool { self.n == 0 }
|
||||
|
||||
/// `None` means the slot has not (yet) been computed — see module docs.
|
||||
pub fn get(&self, slot: usize) -> Option<FamilyMask> {
|
||||
FamilyMask::decode(self.mmap[HEADER_SIZE + slot])
|
||||
}
|
||||
}
|
||||
|
||||
// ── SiblingAnnexBuilder (writer) ────────────────────────────────────────────
|
||||
|
||||
pub struct SiblingAnnexBuilder {
|
||||
mmap: MmapMut,
|
||||
n: usize,
|
||||
path: PathBuf,
|
||||
}
|
||||
|
||||
impl SiblingAnnexBuilder {
|
||||
/// Create a new annex of `n` slots at `path`, pre-initialised to the
|
||||
/// "not yet computed" sentinel (all-zero).
|
||||
pub fn new(n: usize, path: &Path) -> io::Result<Self> {
|
||||
let file_size = HEADER_SIZE + n;
|
||||
let file = OpenOptions::new()
|
||||
.read(true).write(true).create(true).truncate(true)
|
||||
.open(path)?;
|
||||
file.set_len(file_size as u64)?;
|
||||
let mut mmap = unsafe { MmapMut::map_mut(&file)? };
|
||||
mmap[0..4].copy_from_slice(&MAGIC);
|
||||
mmap[4..8].copy_from_slice(&[0u8; 4]);
|
||||
mmap[8..16].copy_from_slice(&(n as u64).to_le_bytes());
|
||||
// Data region left at 0 by `set_len`/mmap — the sentinel value.
|
||||
Ok(Self { mmap, n, path: path.to_path_buf() })
|
||||
}
|
||||
|
||||
pub fn len(&self) -> usize { self.n }
|
||||
pub fn is_empty(&self) -> bool { self.n == 0 }
|
||||
|
||||
pub fn get(&self, slot: usize) -> Option<FamilyMask> {
|
||||
FamilyMask::decode(self.mmap[HEADER_SIZE + slot])
|
||||
}
|
||||
|
||||
pub fn set(&mut self, slot: usize, mask: FamilyMask) {
|
||||
// Redundant concurrent writes from independent recomputation paths
|
||||
// converge to the same encoded byte for a given slot, so a plain
|
||||
// store here is safe even without external synchronisation, as long
|
||||
// as the byte write itself is atomic (true for a single aligned
|
||||
// byte on every platform this project targets).
|
||||
self.mmap[HEADER_SIZE + slot] = mask.encode();
|
||||
}
|
||||
|
||||
pub fn close(self) -> io::Result<()> { self.mmap.flush() }
|
||||
|
||||
pub fn finish(self) -> io::Result<SiblingAnnex> {
|
||||
let path = self.path.clone();
|
||||
self.close()?;
|
||||
SiblingAnnex::open(&path)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::tempdir;
|
||||
|
||||
#[test]
|
||||
fn sentinel_is_zero_and_unset_slots_read_as_uncomputed() {
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("test.psib");
|
||||
let builder = SiblingAnnexBuilder::new(4, &path).unwrap();
|
||||
for slot in 0..4 {
|
||||
assert_eq!(builder.get(slot), None);
|
||||
}
|
||||
builder.close().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn roundtrip_all_valid_masks() {
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("test.psib");
|
||||
let mut builder = SiblingAnnexBuilder::new(4, &path).unwrap();
|
||||
|
||||
let masks = [
|
||||
FamilyMask::EMPTY.with(0), // just A: family size 1
|
||||
FamilyMask::EMPTY.with(0).with(3), // A + T: size 2
|
||||
FamilyMask::EMPTY.with(1).with(2).with(3), // C+G+T: size 3
|
||||
FamilyMask::EMPTY.with(0).with(1).with(2).with(3), // all 4
|
||||
];
|
||||
for (slot, mask) in masks.iter().enumerate() {
|
||||
builder.set(slot, *mask);
|
||||
}
|
||||
let annex = builder.finish().unwrap();
|
||||
for (slot, mask) in masks.iter().enumerate() {
|
||||
assert_eq!(annex.get(slot), Some(*mask));
|
||||
}
|
||||
assert_eq!(annex.get(0).unwrap().siblings(), 0);
|
||||
assert_eq!(annex.get(1).unwrap().siblings(), 1);
|
||||
assert_eq!(annex.get(2).unwrap().siblings(), 2);
|
||||
assert_eq!(annex.get(3).unwrap().siblings(), 3);
|
||||
assert_eq!(annex.get(3).unwrap().family_size(), 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn has_reflects_individual_bits() {
|
||||
let mask = FamilyMask::EMPTY.with(0).with(2);
|
||||
assert!(mask.has(0));
|
||||
assert!(!mask.has(1));
|
||||
assert!(mask.has(2));
|
||||
assert!(!mask.has(3));
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user