Fix batch enumeration offsets and refactor sibling annex construction

Shifts sibling annex construction from slot-indexed enumeration to iteration-order traversal by correcting cumulative k-mer offset tracking in batch enumeration. Replaces coarse per-partition parallelism with chunked work distribution to prevent thread starvation on skewed partitions. Decouples custom progress messages from ETA updates to eliminate display clobbering during high-frequency callbacks. Adds regression tests validating batch offset correctness, partial batch handling, and iterator-order consistency across layer builds.
This commit is contained in:
Eric Coissac committed 2026-08-16 20:51:19 +02:00
1 parent d2548e8c33
commit 32d6720f50
11 files changed
+413 -88

No files matched your search

+63
View File
@@ -214,3 +214,66 @@ fn hamming_dist_basic() {
let (_db, rb) = make_bv(&[true, false, true, true]);
assert_eq!(ra.hamming_dist(&rb), 2);
}
// ── get_batch tests ────────────────────────────────────────────────────────────
#[test]
fn bitvec_get_batch_in_order() {
let bits = vec![true, false, true, false, true];
let (_dir, r) = make_bv(&bits);
let got = r.get_batch(&[0, 1, 2, 3, 4]);
assert_eq!(got, bits);
}
#[test]
fn bitvec_get_batch_out_of_order() {
let bits = vec![true, false, true, false, true];
let (_dir, r) = make_bv(&bits);
let got = r.get_batch(&[3, 0, 4, 1]);
assert_eq!(got, vec![false, true, true, false]);
}
#[test]
fn bitvec_get_batch_with_duplicates() {
let bits = vec![true, false, true];
let (_dir, r) = make_bv(&bits);
let got = r.get_batch(&[0, 2, 0, 1]);
assert_eq!(got, vec![true, true, true, false]);
}
#[test]
fn bitvec_get_batch_empty() {
let (_dir, r) = make_bv(&[true, false]);
let got = r.get_batch(&[]);
assert!(got.is_empty());
}
#[test]
fn bitvec_get_batch_out_of_bounds_returns_false() {
// PersistentBitVec::get does NOT bounds-check — out-of-range slots read
// whatever bit happens to be in the mmap (zero-initialised, so false).
// This is a known gap; get_batch inherits it.
let (_dir, r) = make_bv(&[true, false]);
let got = r.get_batch(&[0, 2]);
assert_eq!(got, vec![true, false]);
}
// BitSliceView get_batch (same logic, exercised through the view)
#[test]
fn bitslice_view_get_batch() {
let bits = vec![true, false, true, false, true];
let (_dir, r) = make_bv(&bits);
let view = r.view();
assert_eq!(view.get_batch(&[0, 1, 2]), vec![true, false, true]);
assert_eq!(view.get_batch(&[4, 3]), vec![true, false]);
}
#[test]
fn bitslice_view_get_batch_out_of_bounds_returns_false() {
// BitSliceView::get does NOT bounds-check either.
let bits = vec![true, false];
let (_dir, r) = make_bv(&bits);
let view = r.view();
let got = view.get_batch(&[0, 2]);
assert_eq!(got, vec![true, false]);
}
+61 -1
View File
@@ -1,6 +1,6 @@
use tempfile::tempdir;
use crate::{pack_compact_int_matrix, PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder};
use crate::{pack_compact_int_matrix, PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder, PersistentCompactIntVec, PersistentCompactIntVecBuilder, IntSliceView};
use crate::traits::CountPartials;
fn make_matrix(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentCompactIntMatrix) {
@@ -320,3 +320,63 @@ fn partial_relfreq_bray_additive_across_split() {
}
}
}
// ── get_batch tests ────────────────────────────────────────────────────────────
fn make_pciv(counts: &[u32]) -> (tempfile::TempDir, PersistentCompactIntVec) {
let dir = tempdir().unwrap();
let path = dir.path().join("c.pciv");
let mut b = PersistentCompactIntVecBuilder::new(counts.len(), &path).unwrap();
for (i, &v) in counts.iter().enumerate() { b.set(i, v); }
b.close().unwrap();
let r = PersistentCompactIntVec::open(&path).unwrap();
(dir, r)
}
#[test]
fn pciv_get_batch_in_order() {
let counts = vec![10u32, 255, 300, 1000];
let (_dir, v) = make_pciv(&counts);
let got = v.get_batch(&[0, 1, 2, 3]);
assert_eq!(got, counts);
}
#[test]
fn pciv_get_batch_out_of_order() {
let counts = vec![10u32, 255, 300, 1000];
let (_dir, v) = make_pciv(&counts);
let got = v.get_batch(&[3, 0, 2, 1]);
assert_eq!(got, vec![1000, 10, 300, 255]);
}
#[test]
fn pciv_get_batch_with_duplicates() {
let counts = vec![10u32, 255, 300];
let (_dir, v) = make_pciv(&counts);
let got = v.get_batch(&[0, 2, 0, 1]);
assert_eq!(got, vec![10, 300, 10, 255]);
}
#[test]
fn pciv_get_batch_empty() {
let (_dir, v) = make_pciv(&[10u32, 20]);
let got: Vec<u32> = v.get_batch(&[]);
assert!(got.is_empty());
}
#[test]
fn pciv_get_batch_out_of_bounds_panics() {
let (_dir, v) = make_pciv(&[10u32, 20]);
let result = std::panic::catch_unwind(|| v.get_batch(&[0, 2]));
assert!(result.is_err(), "get_batch should panic on out-of-bounds slot");
}
// IntSliceView get_batch (same logic, exercised through the view)
#[test]
fn intslice_view_get_batch() {
let counts = vec![10u32, 255, 300, 1000];
let (_dir, v) = make_pciv(&counts);
let view = v.view();
assert_eq!(view.get_batch(&[0, 1, 2]), vec![10, 255, 300]);
assert_eq!(view.get_batch(&[3, 1]), vec![1000, 255]);
}