Implement memory mapping for file construction

Refactors the building process to utilize memory mapping (`memmap2`) for file creation and introduces the necessary infrastructure for publishing built files atomically. This includes expanding the public API surface and cleaning up deprecated architectural components.
This commit is contained in:
Eric Coissac committed 2026-10-15 12:02:33 +02:00
1 parent c226744405
commit 3bd7fd4f41
11 files changed
+135 -27

No files matched your search

+6 -5
View File
@@ -4,7 +4,7 @@ use std::path::{Path, PathBuf};
use memmap2::{Mmap, MmapMut};
use crate::mmap_file::{create_mmap_file, open_mmap_file};
use crate::mmap_file::{building_path, create_mmap_file, open_mmap_file, publish_built};
use crate::reader::PersistentCompactIntVec;
use crate::views::{BitSliceIter, BitSliceView, IntSliceView};
@@ -206,7 +206,7 @@ impl PersistentBitVecBuilder {
.write(true)
.create(true)
.truncate(true)
.open(path)?;
.open(building_path(path))?;
file.write_all(&MAGIC)?;
file.write_all(&[0u8; 4])?;
file.write_all(&(n as u64).to_le_bytes())?;
@@ -229,8 +229,8 @@ impl PersistentBitVecBuilder {
}
pub fn build_from(source: &PersistentBitVec, path: &Path) -> io::Result<Self> {
fs::copy(source.path(), path)?;
let file = OpenOptions::new().read(true).write(true).open(path)?;
fs::copy(source.path(), building_path(path))?;
let file = OpenOptions::new().read(true).write(true).open(building_path(path))?;
let mmap = unsafe { MmapMut::map_mut(&file)? };
let n = source.len();
Ok(Self {
@@ -441,7 +441,8 @@ impl PersistentBitVecBuilder {
}
pub fn close(self) -> io::Result<()> {
self.mmap.flush()
self.mmap.flush()?;
publish_built(&self.path)
}
pub fn finish(self) -> io::Result<PersistentBitVec> {
+3 -2
View File
@@ -15,7 +15,7 @@ use std::path::{Path, PathBuf};
use memmap2::{Mmap, MmapMut};
use crate::mmap_file::{create_mmap_file, open_mmap_file};
use crate::mmap_file::{create_mmap_file, open_mmap_file, publish_built};
const MAGIC: [u8; 4] = *b"PFIV";
@@ -178,7 +178,8 @@ impl PersistentFixedIntVecBuilder {
}
pub fn close(self) -> io::Result<()> {
self.mmap.flush()
self.mmap.flush()?;
publish_built(&self.path)
}
pub fn finish(self) -> io::Result<PersistentFixedIntVec> {
+1
View File
@@ -35,6 +35,7 @@ pub use intmatrix::{
};
pub use layer_meta::LayerMeta;
pub use mmap_file::{building_path, publish_built};
pub use matrix_builder::{ColBuilder, MatrixBuilder};
pub use rankselect::{PersistentRankSelectBitVec, PersistentRankSelectBitVecBuilder};
pub use reader::{Iter as CompactIntVecIter, PersistentCompactIntVec};
+26 -4
View File
@@ -10,13 +10,35 @@
//! `block_ranks`) and its own higher-level API — those genuinely differ
//! between formats and aren't forced into a shared shape.
use std::fs::{File, OpenOptions};
use std::fs::{self, File, OpenOptions};
use std::io::{self, Seek, SeekFrom, Write as _};
use std::path::Path;
use std::path::{Path, PathBuf};
use memmap2::{Mmap, MmapMut};
/// Creates (or truncates) `path`, writes the standard header — `magic` +
/// Suffix of the name a file lives under while it is being built.
const BUILDING_SUFFIX: &str = ".building";
/// The name `path` is built under: a file only appears at `path` itself once
/// complete ([`publish_built`]). A process killed mid-build leaves at most a
/// `.building` file, never a `path` with a valid header and unwritten data
/// that `path.exists()` would take for a finished one — what every
/// "skip it, it already exists" caller (per-layer annexes) relies on.
pub fn building_path(path: &Path) -> PathBuf {
let mut name = path.as_os_str().to_owned();
name.push(BUILDING_SUFFIX);
PathBuf::from(name)
}
/// Atomically moves the file built under [`building_path`]`(path)` to `path`
/// (same directory, so a plain `rename`), replacing any previous one. To be
/// called once the data is complete and flushed.
pub fn publish_built(path: &Path) -> io::Result<()> {
fs::rename(building_path(path), path)
}
/// Creates (or truncates) [`building_path`]`(path)` — the caller publishes it
/// with [`publish_built`] once complete — writes the standard header — `magic` +
/// `pad` (4 bytes right after `magic`; usually all-zero, but
/// `PersistentFixedIntVec` repurposes byte 4 to store its `width`) +
/// `n as u64`, followed by `extra_header_zeros` more zero bytes for a
@@ -37,7 +59,7 @@ pub(crate) fn create_mmap_file(
.write(true)
.create(true)
.truncate(true)
.open(path)?;
.open(building_path(path))?;
file.write_all(&magic)?;
file.write_all(&pad)?;
file.write_all(&(n as u64).to_le_bytes())?;
+3 -2
View File
@@ -21,7 +21,7 @@ use std::path::{Path, PathBuf};
use common_traits::SelectInWord;
use memmap2::{Mmap, MmapMut};
use crate::mmap_file::{create_mmap_file, open_mmap_file};
use crate::mmap_file::{create_mmap_file, open_mmap_file, publish_built};
const MAGIC: [u8; 4] = *b"PRSB";
@@ -268,7 +268,8 @@ impl PersistentRankSelectBitVecBuilder {
let block_off = HEADER_SIZE + bits_bytes(self.n);
let block_bytes = u64_slice_to_le_bytes(&block_ranks);
self.mmap[block_off..block_off + block_bytes.len()].copy_from_slice(&block_bytes);
self.mmap.flush()
self.mmap.flush()?;
publish_built(&self.path)
}
pub fn finish(self) -> io::Result<PersistentRankSelectBitVec> {