fix: validate packed matrix columns before repacking
Add header parsing helpers to extract column counts without memory mapping. Update packing functions to verify existing files match current metadata, preventing stale or widened-column artifacts. Extract inline tests in obilayeredmap to an external module and add comprehensive aggregation tests. Bump obikmer to 1.1.35 and clean up repository configuration.
This commit is contained in:
1 parent
7844239a8e
commit
cd2f2f9417
8 files changed
+450
-174
No files matched your search
@@ -1,5 +1,5 @@
|
||||
use std::fs::{self, File};
|
||||
use std::io::{self, BufWriter, Write as _};
|
||||
use std::io::{self, BufWriter, Read as _, Write as _};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use memmap2::Mmap;
|
||||
@@ -171,19 +171,43 @@ impl PackedBitMatrix {
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads just the `n_cols` field from an existing packed matrix's header,
|
||||
/// without mapping the file. Used by `pack_bit_matrix` to tell a genuinely
|
||||
/// complete pack from a stale one that predates a later column-widening.
|
||||
fn packed_bit_matrix_n_cols(path: &Path) -> io::Result<usize> {
|
||||
let mut f = File::open(path)?;
|
||||
let mut header = [0u8; PBMX_HEADER];
|
||||
f.read_exact(&mut header)?;
|
||||
Ok(u64::from_le_bytes(header[16..24].try_into().unwrap()) as usize)
|
||||
}
|
||||
|
||||
/// Build `presence/matrix.pbmx` from existing `col_*.pbiv` files.
|
||||
pub fn pack_bit_matrix(dir: &Path) -> io::Result<()> {
|
||||
let packed_path = dir.join("matrix.pbmx");
|
||||
if packed_path.exists() {
|
||||
// Matrix complete; remove any leftover column files from a killed cleanup.
|
||||
if let Ok(meta) = MatrixMeta::load(dir) {
|
||||
for c in 0..meta.n_cols { let _ = fs::remove_file(col_path(dir, c)); }
|
||||
let _ = fs::remove_file(dir.join("meta.json"));
|
||||
|
||||
let meta = match MatrixMeta::load(dir) {
|
||||
Ok(meta) => meta,
|
||||
Err(e) => {
|
||||
// No columnar data pending: either this layer was already
|
||||
// packed and cleaned up (matrix.pbmx complete, nothing left to
|
||||
// do), or genuinely nothing was ever written here.
|
||||
return if packed_path.exists() { Ok(()) } else { Err(e) };
|
||||
}
|
||||
};
|
||||
|
||||
// A `matrix.pbmx` can already exist here even though columnar data is
|
||||
// still pending — e.g. copied verbatim from a merge's base source
|
||||
// before this layer was widened with more genome columns (see
|
||||
// `obikpartitionner::merge_partition`). Only skip (re-)packing if the
|
||||
// existing file already reflects the current column count; otherwise
|
||||
// the columnar files are newer and must be (re-)packed, overwriting the
|
||||
// stale one — never silently discarded as "leftover cleanup".
|
||||
if packed_bit_matrix_n_cols(&packed_path).ok() == Some(meta.n_cols) {
|
||||
for c in 0..meta.n_cols { let _ = fs::remove_file(col_path(dir, c)); }
|
||||
let _ = fs::remove_file(dir.join("meta.json"));
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let meta = MatrixMeta::load(dir)?;
|
||||
let n_cols = meta.n_cols;
|
||||
|
||||
// Compute offsets from file sizes — no column data loaded into RAM.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use std::fs::{self, File};
|
||||
use std::io::{self, BufWriter, Write as _};
|
||||
use std::io::{self, BufWriter, Read as _, Write as _};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use memmap2::Mmap;
|
||||
@@ -228,17 +228,44 @@ impl PackedCompactIntMatrix {
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads just the `n_cols` field from an existing packed matrix's header,
|
||||
/// without mapping the file. Used by `pack_compact_int_matrix` to tell a
|
||||
/// genuinely complete pack from a stale one that predates a later
|
||||
/// column-widening.
|
||||
fn packed_int_matrix_n_cols(path: &Path) -> io::Result<usize> {
|
||||
let mut f = File::open(path)?;
|
||||
let mut header = [0u8; PCMX_HEADER];
|
||||
f.read_exact(&mut header)?;
|
||||
Ok(u64::from_le_bytes(header[16..24].try_into().unwrap()) as usize)
|
||||
}
|
||||
|
||||
/// Build `counts/matrix.pcmx` from existing `col_*.pciv` files.
|
||||
pub fn pack_compact_int_matrix(dir: &Path) -> io::Result<()> {
|
||||
let packed_path = dir.join("matrix.pcmx");
|
||||
if packed_path.exists() {
|
||||
if let Ok(meta) = MatrixMeta::load(dir) {
|
||||
for c in 0..meta.n_cols { let _ = fs::remove_file(col_path(dir, c)); }
|
||||
let _ = fs::remove_file(dir.join("meta.json"));
|
||||
|
||||
let meta = match MatrixMeta::load(dir) {
|
||||
Ok(meta) => meta,
|
||||
Err(e) => {
|
||||
// No columnar data pending: either this layer was already
|
||||
// packed and cleaned up (matrix.pcmx complete, nothing left to
|
||||
// do), or genuinely nothing was ever written here.
|
||||
return if packed_path.exists() { Ok(()) } else { Err(e) };
|
||||
}
|
||||
};
|
||||
|
||||
// A `matrix.pcmx` can already exist here even though columnar data is
|
||||
// still pending — e.g. copied verbatim from a merge's base source
|
||||
// before this layer was widened with more genome columns (see
|
||||
// `obikpartitionner::merge_partition`). Only skip (re-)packing if the
|
||||
// existing file already reflects the current column count; otherwise
|
||||
// the columnar files are newer and must be (re-)packed, overwriting the
|
||||
// stale one — never silently discarded as "leftover cleanup".
|
||||
if packed_int_matrix_n_cols(&packed_path).ok() == Some(meta.n_cols) {
|
||||
for c in 0..meta.n_cols { let _ = fs::remove_file(col_path(dir, c)); }
|
||||
let _ = fs::remove_file(dir.join("meta.json"));
|
||||
return Ok(());
|
||||
}
|
||||
let meta = MatrixMeta::load(dir)?;
|
||||
|
||||
let n_cols = meta.n_cols;
|
||||
let col_sizes: Vec<u64> = (0..n_cols)
|
||||
.map(|c| fs::metadata(col_path(dir, c)).map(|m| m.len()))
|
||||
|
||||
Reference in new issue
Block a user