Add resume API to persistent matrix builders for incremental appending
Introduce a `resume` method for persistent bit and int matrix builders to restore dimensions from persisted metadata. This enables incremental column appending across separate build sessions without requiring manual dimension tracking. Consolidate builder lifecycle management in the merge layer using a unified `MatrixBuilder` enum, simplify closure logic, and add instrumentation. Include tests verifying state preservation and data integrity across multiple resume cycles.
This commit is contained in:
1 parent
308d2b9f92
commit
dc3d82f8db
6 files changed
+305
-115
No files matched your search
@@ -23,6 +23,17 @@ impl PersistentBitMatrixBuilder {
|
||||
Ok(Self { dir: dir.to_path_buf(), n, n_cols: 0 })
|
||||
}
|
||||
|
||||
/// Resume appending columns to a Columnar matrix directory already
|
||||
/// closed by a previous builder session — reads `n`/`n_cols` from
|
||||
/// `meta.json` instead of starting a fresh matrix at `n_cols == 0`.
|
||||
/// Lets callers that build columns incrementally over time (e.g. a
|
||||
/// streamed merge pipeline) reach `add_col`/`close` without knowing the
|
||||
/// on-disk column naming or `MatrixMeta` themselves.
|
||||
pub fn resume(dir: &Path) -> io::Result<Self> {
|
||||
let meta = MatrixMeta::load(dir)?;
|
||||
Ok(Self { dir: dir.to_path_buf(), n: meta.n, n_cols: meta.n_cols })
|
||||
}
|
||||
|
||||
pub fn n(&self) -> usize { self.n }
|
||||
pub fn n_cols(&self) -> usize { self.n_cols }
|
||||
|
||||
|
||||
@@ -481,6 +481,18 @@ impl PersistentCompactIntMatrixBuilder {
|
||||
fs::create_dir_all(dir)?;
|
||||
Ok(Self { dir: dir.to_path_buf(), n, n_cols: 0 })
|
||||
}
|
||||
|
||||
/// Resume appending columns to a Columnar matrix directory already
|
||||
/// closed by a previous builder session — reads `n`/`n_cols` from
|
||||
/// `meta.json` instead of starting a fresh matrix at `n_cols == 0`.
|
||||
/// Lets callers that build columns incrementally over time (e.g. a
|
||||
/// streamed merge pipeline) reach `add_col`/`close` without knowing the
|
||||
/// on-disk column naming or `MatrixMeta` themselves.
|
||||
pub fn resume(dir: &Path) -> io::Result<Self> {
|
||||
let meta = MatrixMeta::load(dir)?;
|
||||
Ok(Self { dir: dir.to_path_buf(), n: meta.n, n_cols: meta.n_cols })
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn n(&self) -> usize { self.n }
|
||||
#[inline]
|
||||
|
||||
@@ -62,6 +62,53 @@ fn two_cols_roundtrip() {
|
||||
assert_eq!(&*m.row(2), &[true, false]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resume_continues_n_cols_and_appends_columns() {
|
||||
let dir = tempdir().unwrap();
|
||||
let presence = dir.path().join("presence");
|
||||
|
||||
let mut b = PersistentBitMatrixBuilder::new(3, &presence).unwrap();
|
||||
let mut col0 = b.add_col().unwrap();
|
||||
col0.set(0, true); col0.set(1, false); col0.set(2, true);
|
||||
col0.close().unwrap();
|
||||
b.close().unwrap();
|
||||
|
||||
let mut resumed = PersistentBitMatrixBuilder::resume(&presence).unwrap();
|
||||
assert_eq!(resumed.n(), 3);
|
||||
assert_eq!(resumed.n_cols(), 1);
|
||||
let mut col1 = resumed.add_col().unwrap();
|
||||
col1.set(0, false); col1.set(1, true); col1.set(2, false);
|
||||
col1.close().unwrap();
|
||||
resumed.close().unwrap();
|
||||
|
||||
let m = PersistentBitMatrix::open(dir.path()).unwrap();
|
||||
assert_eq!(m.n_cols(), 2);
|
||||
assert_eq!(&*m.row(0), &[true, false]);
|
||||
assert_eq!(&*m.row(1), &[false, true]);
|
||||
assert_eq!(&*m.row(2), &[true, false]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resume_twice_keeps_appending() {
|
||||
let dir = tempdir().unwrap();
|
||||
let presence = dir.path().join("presence");
|
||||
|
||||
PersistentBitMatrixBuilder::new(2, &presence).unwrap().close().unwrap();
|
||||
|
||||
for v in [true, false, true] {
|
||||
let mut b = PersistentBitMatrixBuilder::resume(&presence).unwrap();
|
||||
let mut col = b.add_col().unwrap();
|
||||
col.set(0, v); col.set(1, !v);
|
||||
col.close().unwrap();
|
||||
b.close().unwrap();
|
||||
}
|
||||
|
||||
let m = PersistentBitMatrix::open(dir.path()).unwrap();
|
||||
assert_eq!(m.n_cols(), 3);
|
||||
assert_eq!(&*m.row(0), &[true, false, true]);
|
||||
assert_eq!(&*m.row(1), &[false, true, false]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn col_accessor() {
|
||||
let dir = tempdir().unwrap();
|
||||
|
||||
@@ -59,6 +59,53 @@ fn two_cols_roundtrip() {
|
||||
assert_eq!(&*m.row(2), &[3u32, 30]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resume_continues_n_cols_and_appends_columns() {
|
||||
let dir = tempdir().unwrap();
|
||||
let counts = dir.path().join("counts");
|
||||
|
||||
let mut b = PersistentCompactIntMatrixBuilder::new(3, &counts).unwrap();
|
||||
let mut col0 = b.add_col().unwrap();
|
||||
col0.set(0, 1); col0.set(1, 2); col0.set(2, 3);
|
||||
col0.close().unwrap();
|
||||
b.close().unwrap();
|
||||
|
||||
let mut resumed = PersistentCompactIntMatrixBuilder::resume(&counts).unwrap();
|
||||
assert_eq!(resumed.n(), 3);
|
||||
assert_eq!(resumed.n_cols(), 1);
|
||||
let mut col1 = resumed.add_col().unwrap();
|
||||
col1.set(0, 10); col1.set(1, 20); col1.set(2, 30);
|
||||
col1.close().unwrap();
|
||||
resumed.close().unwrap();
|
||||
|
||||
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
|
||||
assert_eq!(m.n_cols(), 2);
|
||||
assert_eq!(&*m.row(0), &[1u32, 10]);
|
||||
assert_eq!(&*m.row(1), &[2u32, 20]);
|
||||
assert_eq!(&*m.row(2), &[3u32, 30]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resume_twice_keeps_appending() {
|
||||
let dir = tempdir().unwrap();
|
||||
let counts = dir.path().join("counts");
|
||||
|
||||
PersistentCompactIntMatrixBuilder::new(2, &counts).unwrap().close().unwrap();
|
||||
|
||||
for v in [1u32, 2, 3] {
|
||||
let mut b = PersistentCompactIntMatrixBuilder::resume(&counts).unwrap();
|
||||
let mut col = b.add_col().unwrap();
|
||||
col.set(0, v); col.set(1, v * 10);
|
||||
col.close().unwrap();
|
||||
b.close().unwrap();
|
||||
}
|
||||
|
||||
let m = PersistentCompactIntMatrix::open(dir.path()).unwrap();
|
||||
assert_eq!(m.n_cols(), 3);
|
||||
assert_eq!(&*m.row(0), &[1u32, 2, 3]);
|
||||
assert_eq!(&*m.row(1), &[10u32, 20, 30]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn col_accessor() {
|
||||
let dir = tempdir().unwrap();
|
||||
|
||||
Reference in new issue
Block a user