feat: introduce column-major matrix storage and migrate layered map

Introduces `PersistentBitMatrix` and `PersistentCompactIntMatrix` to replace single-file vector storage with a column-major, directory-based layout. Each column is persisted as an individual file alongside a lightweight `meta.json` for dimension tracking. Migrates `obilayeredmap` to use these multi-column structures, updating Rust APIs, query return types, and build signatures. Includes comprehensive documentation, unit and integration tests for persistence and accessors, and refactors distance calculation helpers.
This commit is contained in:
Eric Coissac
2026-05-14 09:31:11 +08:00
parent f48f7500cd
commit b218bf012b
15 changed files with 843 additions and 201 deletions
+58
View File
@@ -0,0 +1,58 @@
use std::{fs, io, path::{Path, PathBuf}};
use crate::builder::PersistentCompactIntVecBuilder;
use crate::meta::MatrixMeta;
use crate::reader::PersistentCompactIntVec;
fn col_path(dir: &Path, col: usize) -> PathBuf {
dir.join(format!("col_{col:06}.pciv"))
}
pub struct PersistentCompactIntMatrix {
cols: Vec<PersistentCompactIntVec>,
n: usize,
}
impl PersistentCompactIntMatrix {
pub fn open(dir: &Path) -> io::Result<Self> {
let meta = MatrixMeta::load(dir)?;
let cols = (0..meta.n_cols)
.map(|c| PersistentCompactIntVec::open(&col_path(dir, c)))
.collect::<io::Result<Vec<_>>>()?;
Ok(Self { cols, n: meta.n })
}
pub fn n(&self) -> usize { self.n }
pub fn n_cols(&self) -> usize { self.cols.len() }
pub fn col(&self, c: usize) -> &PersistentCompactIntVec { &self.cols[c] }
pub fn row(&self, slot: usize) -> Box<[u32]> {
self.cols.iter().map(|c| c.get(slot)).collect()
}
}
pub struct PersistentCompactIntMatrixBuilder {
dir: PathBuf,
n: usize,
n_cols: usize,
}
impl PersistentCompactIntMatrixBuilder {
pub fn new(n: usize, dir: &Path) -> io::Result<Self> {
fs::create_dir_all(dir)?;
Ok(Self { dir: dir.to_path_buf(), n, n_cols: 0 })
}
pub fn n(&self) -> usize { self.n }
pub fn n_cols(&self) -> usize { self.n_cols }
pub fn add_col(&mut self) -> io::Result<PersistentCompactIntVecBuilder> {
let path = col_path(&self.dir, self.n_cols);
self.n_cols += 1;
PersistentCompactIntVecBuilder::new(self.n, &path)
}
pub fn close(self) -> io::Result<()> {
MatrixMeta { n: self.n, n_cols: self.n_cols }.save(&self.dir)
}
}