fix: validate packed matrix columns before repacking
Release / create-release (push) Successful in 2m27s
Release / build-linux-x86_64 (push) Successful in 8m15s
Release / build-macos-arm64 (push) Failing after 30s
CI / build (pull_request) Successful in 3m17s

Add header parsing helpers to extract column counts without memory mapping. Update packing functions to verify existing files match current metadata, preventing stale or widened-column artifacts. Extract inline tests in obilayeredmap to an external module and add comprehensive aggregation tests. Bump obikmer to 1.1.35 and clean up repository configuration.
This commit is contained in:
Eric Coissac committed 2026-07-03 17:20:22 +02:00
1 parent 7844239a8e
commit cd2f2f9417
8 files changed
+450 -174

No files matched your search

+31 -7
View File
@@ -1,5 +1,5 @@
use std::fs::{self, File};
use std::io::{self, BufWriter, Write as _};
use std::io::{self, BufWriter, Read as _, Write as _};
use std::path::{Path, PathBuf};
use memmap2::Mmap;
@@ -171,19 +171,43 @@ impl PackedBitMatrix {
}
}
/// Reads just the `n_cols` field from an existing packed matrix's header,
/// without mapping the file. Used by `pack_bit_matrix` to tell a genuinely
/// complete pack from a stale one that predates a later column-widening.
fn packed_bit_matrix_n_cols(path: &Path) -> io::Result<usize> {
let mut f = File::open(path)?;
let mut header = [0u8; PBMX_HEADER];
f.read_exact(&mut header)?;
Ok(u64::from_le_bytes(header[16..24].try_into().unwrap()) as usize)
}
/// Build `presence/matrix.pbmx` from existing `col_*.pbiv` files.
pub fn pack_bit_matrix(dir: &Path) -> io::Result<()> {
let packed_path = dir.join("matrix.pbmx");
if packed_path.exists() {
// Matrix complete; remove any leftover column files from a killed cleanup.
if let Ok(meta) = MatrixMeta::load(dir) {
for c in 0..meta.n_cols { let _ = fs::remove_file(col_path(dir, c)); }
let _ = fs::remove_file(dir.join("meta.json"));
let meta = match MatrixMeta::load(dir) {
Ok(meta) => meta,
Err(e) => {
// No columnar data pending: either this layer was already
// packed and cleaned up (matrix.pbmx complete, nothing left to
// do), or genuinely nothing was ever written here.
return if packed_path.exists() { Ok(()) } else { Err(e) };
}
};
// A `matrix.pbmx` can already exist here even though columnar data is
// still pending — e.g. copied verbatim from a merge's base source
// before this layer was widened with more genome columns (see
// `obikpartitionner::merge_partition`). Only skip (re-)packing if the
// existing file already reflects the current column count; otherwise
// the columnar files are newer and must be (re-)packed, overwriting the
// stale one — never silently discarded as "leftover cleanup".
if packed_bit_matrix_n_cols(&packed_path).ok() == Some(meta.n_cols) {
for c in 0..meta.n_cols { let _ = fs::remove_file(col_path(dir, c)); }
let _ = fs::remove_file(dir.join("meta.json"));
return Ok(());
}
let meta = MatrixMeta::load(dir)?;
let n_cols = meta.n_cols;
// Compute offsets from file sizes — no column data loaded into RAM.
+33 -6
View File
@@ -1,5 +1,5 @@
use std::fs::{self, File};
use std::io::{self, BufWriter, Write as _};
use std::io::{self, BufWriter, Read as _, Write as _};
use std::path::{Path, PathBuf};
use memmap2::Mmap;
@@ -228,17 +228,44 @@ impl PackedCompactIntMatrix {
}
}
/// Reads just the `n_cols` field from an existing packed matrix's header,
/// without mapping the file. Used by `pack_compact_int_matrix` to tell a
/// genuinely complete pack from a stale one that predates a later
/// column-widening.
fn packed_int_matrix_n_cols(path: &Path) -> io::Result<usize> {
let mut f = File::open(path)?;
let mut header = [0u8; PCMX_HEADER];
f.read_exact(&mut header)?;
Ok(u64::from_le_bytes(header[16..24].try_into().unwrap()) as usize)
}
/// Build `counts/matrix.pcmx` from existing `col_*.pciv` files.
pub fn pack_compact_int_matrix(dir: &Path) -> io::Result<()> {
let packed_path = dir.join("matrix.pcmx");
if packed_path.exists() {
if let Ok(meta) = MatrixMeta::load(dir) {
for c in 0..meta.n_cols { let _ = fs::remove_file(col_path(dir, c)); }
let _ = fs::remove_file(dir.join("meta.json"));
let meta = match MatrixMeta::load(dir) {
Ok(meta) => meta,
Err(e) => {
// No columnar data pending: either this layer was already
// packed and cleaned up (matrix.pcmx complete, nothing left to
// do), or genuinely nothing was ever written here.
return if packed_path.exists() { Ok(()) } else { Err(e) };
}
};
// A `matrix.pcmx` can already exist here even though columnar data is
// still pending — e.g. copied verbatim from a merge's base source
// before this layer was widened with more genome columns (see
// `obikpartitionner::merge_partition`). Only skip (re-)packing if the
// existing file already reflects the current column count; otherwise
// the columnar files are newer and must be (re-)packed, overwriting the
// stale one — never silently discarded as "leftover cleanup".
if packed_int_matrix_n_cols(&packed_path).ok() == Some(meta.n_cols) {
for c in 0..meta.n_cols { let _ = fs::remove_file(col_path(dir, c)); }
let _ = fs::remove_file(dir.join("meta.json"));
return Ok(());
}
let meta = MatrixMeta::load(dir)?;
let n_cols = meta.n_cols;
let col_sizes: Vec<u64> = (0..n_cols)
.map(|c| fs::metadata(col_path(dir, c)).map(|m| m.len()))