Add resume API to persistent matrix builders for incremental appending

Introduce a `resume` method for persistent bit and int matrix builders to restore dimensions from persisted metadata. This enables incremental column appending across separate build sessions without requiring manual dimension tracking. Consolidate builder lifecycle management in the merge layer using a unified `MatrixBuilder` enum, simplify closure logic, and add instrumentation. Include tests verifying state preservation and data integrity across multiple resume cycles.
This commit is contained in:
Eric Coissac committed 2026-08-20 12:36:47 +02:00
1 parent 308d2b9f92
commit dc3d82f8db
6 files changed
+305 -115

No files matched your search

@@ -23,6 +23,17 @@ impl PersistentBitMatrixBuilder {
Ok(Self { dir: dir.to_path_buf(), n, n_cols: 0 })
}
/// Resume appending columns to a Columnar matrix directory already
/// closed by a previous builder session — reads `n`/`n_cols` from
/// `meta.json` instead of starting a fresh matrix at `n_cols == 0`.
/// Lets callers that build columns incrementally over time (e.g. a
/// streamed merge pipeline) reach `add_col`/`close` without knowing the
/// on-disk column naming or `MatrixMeta` themselves.
pub fn resume(dir: &Path) -> io::Result<Self> {
let meta = MatrixMeta::load(dir)?;
Ok(Self { dir: dir.to_path_buf(), n: meta.n, n_cols: meta.n_cols })
}
pub fn n(&self) -> usize { self.n }
pub fn n_cols(&self) -> usize { self.n_cols }
+12
View File
@@ -481,6 +481,18 @@ impl PersistentCompactIntMatrixBuilder {
fs::create_dir_all(dir)?;
Ok(Self { dir: dir.to_path_buf(), n, n_cols: 0 })
}
/// Resume appending columns to a Columnar matrix directory already
/// closed by a previous builder session — reads `n`/`n_cols` from
/// `meta.json` instead of starting a fresh matrix at `n_cols == 0`.
/// Lets callers that build columns incrementally over time (e.g. a
/// streamed merge pipeline) reach `add_col`/`close` without knowing the
/// on-disk column naming or `MatrixMeta` themselves.
pub fn resume(dir: &Path) -> io::Result<Self> {
let meta = MatrixMeta::load(dir)?;
Ok(Self { dir: dir.to_path_buf(), n: meta.n, n_cols: meta.n_cols })
}
#[inline]
pub fn n(&self) -> usize { self.n }
#[inline]
+47
View File
@@ -62,6 +62,53 @@ fn two_cols_roundtrip() {
assert_eq!(&*m.row(2), &[true, false]);
}
#[test]
fn resume_continues_n_cols_and_appends_columns() {
let dir = tempdir().unwrap();
let presence = dir.path().join("presence");
let mut b = PersistentBitMatrixBuilder::new(3, &presence).unwrap();
let mut col0 = b.add_col().unwrap();
col0.set(0, true); col0.set(1, false); col0.set(2, true);
col0.close().unwrap();
b.close().unwrap();
let mut resumed = PersistentBitMatrixBuilder::resume(&presence).unwrap();
assert_eq!(resumed.n(), 3);
assert_eq!(resumed.n_cols(), 1);
let mut col1 = resumed.add_col().unwrap();
col1.set(0, false); col1.set(1, true); col1.set(2, false);
col1.close().unwrap();
resumed.close().unwrap();
let m = PersistentBitMatrix::open(dir.path()).unwrap();
assert_eq!(m.n_cols(), 2);
assert_eq!(&*m.row(0), &[true, false]);
assert_eq!(&*m.row(1), &[false, true]);
assert_eq!(&*m.row(2), &[true, false]);
}
#[test]
fn resume_twice_keeps_appending() {
let dir = tempdir().unwrap();
let presence = dir.path().join("presence");
PersistentBitMatrixBuilder::new(2, &presence).unwrap().close().unwrap();
for v in [true, false, true] {
let mut b = PersistentBitMatrixBuilder::resume(&presence).unwrap();
let mut col = b.add_col().unwrap();
col.set(0, v); col.set(1, !v);
col.close().unwrap();
b.close().unwrap();
}
let m = PersistentBitMatrix::open(dir.path()).unwrap();
assert_eq!(m.n_cols(), 3);
assert_eq!(&*m.row(0), &[true, false, true]);
assert_eq!(&*m.row(1), &[false, true, false]);
}
#[test]
fn col_accessor() {
let dir = tempdir().unwrap();
+47
View File
@@ -59,6 +59,53 @@ fn two_cols_roundtrip() {
assert_eq!(&*m.row(2), &[3u32, 30]);
}
#[test]
fn resume_continues_n_cols_and_appends_columns() {
let dir = tempdir().unwrap();
let counts = dir.path().join("counts");
let mut b = PersistentCompactIntMatrixBuilder::new(3, &counts).unwrap();
let mut col0 = b.add_col().unwrap();
col0.set(0, 1); col0.set(1, 2); col0.set(2, 3);
col0.close().unwrap();
b.close().unwrap();
let mut resumed = PersistentCompactIntMatrixBuilder::resume(&counts).unwrap();
assert_eq!(resumed.n(), 3);
assert_eq!(resumed.n_cols(), 1);
let mut col1 = resumed.add_col().unwrap();
col1.set(0, 10); col1.set(1, 20); col1.set(2, 30);
col1.close().unwrap();
resumed.close().unwrap();
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
assert_eq!(m.n_cols(), 2);
assert_eq!(&*m.row(0), &[1u32, 10]);
assert_eq!(&*m.row(1), &[2u32, 20]);
assert_eq!(&*m.row(2), &[3u32, 30]);
}
#[test]
fn resume_twice_keeps_appending() {
let dir = tempdir().unwrap();
let counts = dir.path().join("counts");
PersistentCompactIntMatrixBuilder::new(2, &counts).unwrap().close().unwrap();
for v in [1u32, 2, 3] {
let mut b = PersistentCompactIntMatrixBuilder::resume(&counts).unwrap();
let mut col = b.add_col().unwrap();
col.set(0, v); col.set(1, v * 10);
col.close().unwrap();
b.close().unwrap();
}
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
assert_eq!(m.n_cols(), 3);
assert_eq!(&*m.row(0), &[1u32, 2, 3]);
assert_eq!(&*m.row(1), &[10u32, 20, 30]);
}
#[test]
fn col_accessor() {
let dir = tempdir().unwrap();