Implement memory mapping for file construction

Refactors the building process to utilize memory mapping (`memmap2`) for file creation and introduces the necessary infrastructure for publishing built files atomically. This includes expanding the public API surface and cleaning up deprecated architectural components.
This commit is contained in:
Eric Coissac committed 2026-10-15 12:02:33 +02:00
1 parent c226744405
commit 3bd7fd4f41
11 files changed
+135 -27

No files matched your search

+6 -5
View File
@@ -4,7 +4,7 @@ use std::path::{Path, PathBuf};
use memmap2::{Mmap, MmapMut};
use crate::mmap_file::{create_mmap_file, open_mmap_file};
use crate::mmap_file::{building_path, create_mmap_file, open_mmap_file, publish_built};
use crate::reader::PersistentCompactIntVec;
use crate::views::{BitSliceIter, BitSliceView, IntSliceView};
@@ -206,7 +206,7 @@ impl PersistentBitVecBuilder {
.write(true)
.create(true)
.truncate(true)
.open(path)?;
.open(building_path(path))?;
file.write_all(&MAGIC)?;
file.write_all(&[0u8; 4])?;
file.write_all(&(n as u64).to_le_bytes())?;
@@ -229,8 +229,8 @@ impl PersistentBitVecBuilder {
}
pub fn build_from(source: &PersistentBitVec, path: &Path) -> io::Result<Self> {
fs::copy(source.path(), path)?;
let file = OpenOptions::new().read(true).write(true).open(path)?;
fs::copy(source.path(), building_path(path))?;
let file = OpenOptions::new().read(true).write(true).open(building_path(path))?;
let mmap = unsafe { MmapMut::map_mut(&file)? };
let n = source.len();
Ok(Self {
@@ -441,7 +441,8 @@ impl PersistentBitVecBuilder {
}
pub fn close(self) -> io::Result<()> {
self.mmap.flush()
self.mmap.flush()?;
publish_built(&self.path)
}
pub fn finish(self) -> io::Result<PersistentBitVec> {
+3 -2
View File
@@ -15,7 +15,7 @@ use std::path::{Path, PathBuf};
use memmap2::{Mmap, MmapMut};
use crate::mmap_file::{create_mmap_file, open_mmap_file};
use crate::mmap_file::{create_mmap_file, open_mmap_file, publish_built};
const MAGIC: [u8; 4] = *b"PFIV";
@@ -178,7 +178,8 @@ impl PersistentFixedIntVecBuilder {
}
pub fn close(self) -> io::Result<()> {
self.mmap.flush()
self.mmap.flush()?;
publish_built(&self.path)
}
pub fn finish(self) -> io::Result<PersistentFixedIntVec> {
+1
View File
@@ -35,6 +35,7 @@ pub use intmatrix::{
};
pub use layer_meta::LayerMeta;
pub use mmap_file::{building_path, publish_built};
pub use matrix_builder::{ColBuilder, MatrixBuilder};
pub use rankselect::{PersistentRankSelectBitVec, PersistentRankSelectBitVecBuilder};
pub use reader::{Iter as CompactIntVecIter, PersistentCompactIntVec};
+26 -4
View File
@@ -10,13 +10,35 @@
//! `block_ranks`) and its own higher-level API — those genuinely differ
//! between formats and aren't forced into a shared shape.
use std::fs::{File, OpenOptions};
use std::fs::{self, File, OpenOptions};
use std::io::{self, Seek, SeekFrom, Write as _};
use std::path::Path;
use std::path::{Path, PathBuf};
use memmap2::{Mmap, MmapMut};
/// Creates (or truncates) `path`, writes the standard header — `magic` +
/// Suffix of the name a file lives under while it is being built.
const BUILDING_SUFFIX: &str = ".building";
/// The name `path` is built under: a file only appears at `path` itself once
/// complete ([`publish_built`]). A process killed mid-build leaves at most a
/// `.building` file, never a `path` with a valid header and unwritten data
/// that `path.exists()` would take for a finished one — what every
/// "skip it, it already exists" caller (per-layer annexes) relies on.
pub fn building_path(path: &Path) -> PathBuf {
let mut name = path.as_os_str().to_owned();
name.push(BUILDING_SUFFIX);
PathBuf::from(name)
}
/// Atomically moves the file built under [`building_path`]`(path)` to `path`
/// (same directory, so a plain `rename`), replacing any previous one. To be
/// called once the data is complete and flushed.
pub fn publish_built(path: &Path) -> io::Result<()> {
fs::rename(building_path(path), path)
}
/// Creates (or truncates) [`building_path`]`(path)` — the caller publishes it
/// with [`publish_built`] once complete — writes the standard header — `magic` +
/// `pad` (4 bytes right after `magic`; usually all-zero, but
/// `PersistentFixedIntVec` repurposes byte 4 to store its `width`) +
/// `n as u64`, followed by `extra_header_zeros` more zero bytes for a
@@ -37,7 +59,7 @@ pub(crate) fn create_mmap_file(
.write(true)
.create(true)
.truncate(true)
.open(path)?;
.open(building_path(path))?;
file.write_all(&magic)?;
file.write_all(&pad)?;
file.write_all(&(n as u64).to_le_bytes())?;
+3 -2
View File
@@ -21,7 +21,7 @@ use std::path::{Path, PathBuf};
use common_traits::SelectInWord;
use memmap2::{Mmap, MmapMut};
use crate::mmap_file::{create_mmap_file, open_mmap_file};
use crate::mmap_file::{create_mmap_file, open_mmap_file, publish_built};
const MAGIC: [u8; 4] = *b"PRSB";
@@ -268,7 +268,8 @@ impl PersistentRankSelectBitVecBuilder {
let block_off = HEADER_SIZE + bits_bytes(self.n);
let block_bytes = u64_slice_to_le_bytes(&block_ranks);
self.mmap[block_off..block_off + block_bytes.len()].copy_from_slice(&block_bytes);
self.mmap.flush()
self.mmap.flush()?;
publish_built(&self.path)
}
pub fn finish(self) -> io::Result<PersistentRankSelectBitVec> {
@@ -129,16 +129,19 @@ impl SiblingExt for IndexCache {
let layer = self
.get_layer(part, l)
.expect("cache-consistent: n_layer(part) already bounds l");
part_slots += build_layer_sibling_annex(self, &layer, l, n_parts, fast_mode)?;
// Invalidate this layer's entropy and quartet annexes, if
// any — they were built against the sibling annex's
// *previous* `family_idx` numbering
// (`non_monomorphic_reindex_layer` reads the sibling
// annex), now stale. `ensure_family_annexes` below only
// builds what's *missing*, so the old files must be removed
// here for them to be rebuilt rather than silently left
// stale.
// annex), and are stale once the new one exists.
// `ensure_family_annexes` below only builds what's
// *missing*, so the old files must be removed here for
// them to be rebuilt rather than silently left stale.
// Removed *before* the new annex is built and published
// (atomically, at its end): a build killed in between
// leaves the previous annex with no derived files — still
// consistent, rebuilt on demand — never a new annex next to
// derived files computed against the old numbering.
for name in ScalarKind::Entropy
.file_names()
.into_iter()
@@ -150,6 +153,7 @@ impl SiblingExt for IndexCache {
std::fs::remove_file(&entropy_path).map_err(OKIError::Io)?;
}
}
part_slots += build_layer_sibling_annex(self, &layer, l, n_parts, fast_mode)?;
}
total_slots += part_slots;
pb.inc(1);
+3 -1
View File
@@ -23,6 +23,7 @@ use std::io;
use std::path::Path;
use memmap2::Mmap;
use obicompactvec::{building_path, publish_built};
use crate::siblings::helpers::{is_carrier, triangle_index};
@@ -166,7 +167,8 @@ impl FamilyOverlapAccumulator {
for &c in &self.counts {
buf.extend_from_slice(&c.to_le_bytes());
}
std::fs::write(path, buf)
std::fs::write(building_path(path), buf)?;
publish_built(path)
}
}
@@ -20,9 +20,10 @@
use std::fs::{File, OpenOptions};
use std::io;
use std::path::Path;
use std::path::{Path, PathBuf};
use memmap2::{Mmap, MmapMut};
use obicompactvec::{building_path, publish_built};
// Header: magic(4) + flags(4) + n(8) = 16 bytes. Data (4 bytes/entry) follows.
// `flags` bit 0 = `no_ambiguity` (the counting convention the values were
@@ -125,6 +126,7 @@ impl FamilyScalarAnnex {
pub struct FamilyScalarAnnexBuilder {
mmap: MmapMut,
path: PathBuf,
}
impl FamilyScalarAnnexBuilder {
@@ -139,7 +141,7 @@ impl FamilyScalarAnnexBuilder {
.write(true)
.create(true)
.truncate(true)
.open(path)?;
.open(building_path(path))?;
file.set_len(file_size as u64)?;
let mut mmap = unsafe { MmapMut::map_mut(&file)? };
mmap[0..4].copy_from_slice(&kind.magic());
@@ -150,7 +152,7 @@ impl FamilyScalarAnnexBuilder {
let off = HEADER_SIZE + i * 4;
mmap[off..off + 4].copy_from_slice(&SENTINEL.to_le_bytes());
}
Ok(Self { mmap })
Ok(Self { mmap, path: path.to_path_buf() })
}
pub(crate) fn set(&mut self, idx: usize, value: f32) {
@@ -159,8 +161,12 @@ impl FamilyScalarAnnexBuilder {
self.mmap[off..off + 4].copy_from_slice(&value.to_le_bytes());
}
/// Flushes, then publishes the file under its final name: until then it
/// lives under `building_path`, so a killed build never leaves a file
/// `path.exists()` would take for a finished annex.
pub(crate) fn close(self) -> io::Result<()> {
self.mmap.flush()
self.mmap.flush()?;
publish_built(&self.path)
}
}
+11 -3
View File
@@ -53,6 +53,7 @@ use std::path::{Path, PathBuf};
use std::sync::atomic::AtomicU16;
use memmap2::{Mmap, MmapMut};
use obicompactvec::{building_path, publish_built};
use obikseq::CanonicalKmer;
@@ -283,6 +284,7 @@ impl SiblingAnnex {
pub struct SiblingAnnexBuilder {
mmap: MmapMut,
path: PathBuf,
}
impl SiblingAnnexBuilder {
@@ -292,14 +294,14 @@ impl SiblingAnnexBuilder {
let file_size = HEADER_SIZE + n * 2;
let file = OpenOptions::new()
.read(true).write(true).create(true).truncate(true)
.open(path)?;
.open(building_path(path))?;
file.set_len(file_size as u64)?;
let mut mmap = unsafe { MmapMut::map_mut(&file)? };
mmap[0..4].copy_from_slice(&MAGIC);
mmap[4..8].copy_from_slice(&[0u8; 4]);
mmap[8..16].copy_from_slice(&(n as u64).to_le_bytes());
// Data region left at 0 by `set_len`/mmap — the sentinel value.
Ok(Self { mmap })
Ok(Self { mmap, path: path.to_path_buf() })
}
/// `None` means the slot has not (yet) been computed — see module
@@ -336,7 +338,13 @@ impl SiblingAnnexBuilder {
self.mmap[off..off + 2].copy_from_slice(&mask.encode().to_le_bytes());
}
pub fn close(self) -> io::Result<()> { self.mmap.flush() }
/// Flushes, then publishes the annex under its final name — until then
/// it lives under `building_path`, so a killed build leaves no
/// `siblings.psib` for a reader to trust.
pub fn close(self) -> io::Result<()> {
self.mmap.flush()?;
publish_built(&self.path)
}
}
#[cfg(test)]