Refactor chunking logic to enforce 128 k-mer limit

This change reduces the maximum number of k-mers per chunk from 256 to 128. This modification requires updating the data model to handle sequences exceeding this limit by splitting them into overlapping chunks, ensuring no k-mers are lost. Extensive testing has been added to validate the new chunking logic and k-mer constraints.
This commit is contained in:
Eric Coissac committed 2026-10-15 12:20:36 +02:00
1 parent 8468319a63
commit 025c40c0db
8 files changed
+80 -25

No files matched your search

+3 -1
View File
@@ -104,7 +104,9 @@ fn roundtrip_single() {
fn roundtrip_all_lengths() {
setup();
let bases: Vec<u8> = (0..300).map(|i| b"ACGT"[i % 4]).collect();
for len in (TEST_K..=19).chain([255, 256, 257]) {
// single-chunk lengths only: seql ≤ MAX_KMERS_PER_CHUNK + k - 1
let max_len = obikseq::MAX_KMERS_PER_CHUNK + TEST_K - 1;
for len in (TEST_K..=19).chain([max_len - 2, max_len - 1, max_len]) {
let sk = make_sk(&bases[..len]);
let mut buf = Vec::new();
sk.write_to_binary(&mut buf).unwrap();
+27 -13
View File
@@ -222,31 +222,45 @@ fn iter_indexed_chunk_id_and_rank_two_chunks() {
#[test]
fn short_unitig_not_split() {
// seql=259 → n_kmers=256 = MAX_KMERS_PER_CHUNK → no split
// seql=131 → n_kmers=128 = MAX_KMERS_PER_CHUNK → no split
set_k(4);
let seq: Vec<u8> = (0..259_usize).map(|i| b"ACGT"[i % 4]).collect();
let seq: Vec<u8> = (0..131_usize).map(|i| b"ACGT"[i % 4]).collect();
let (_dir, r) = write_read(&[&seq]);
assert_eq!(r.len(), 1);
assert_eq!(r.seql(0), 259);
assert_eq!(r.seql(0), 131);
}
#[test]
fn long_unitig_split_no_kmer_lost() {
// seql=260 → n_kmers=257 > MAX_KMERS_PER_CHUNK(256) → 2 chunks
// chunk_nucl=259, stride=256
// Chunk 0: nucl 0..259 (259 nucl, 256 kmers)
// Chunk 1: nucl 256..260 (4 nucl, 1 kmer)
// seql=132 → n_kmers=129 > MAX_KMERS_PER_CHUNK(128) → 2 chunks
// chunk_nucl=131, stride=128
// Chunk 0: nucl 0..131 (131 nucl, 128 kmers)
// Chunk 1: nucl 128..132 (4 nucl, 1 kmer)
set_k(4);
let seq: Vec<u8> = (0..260_usize).map(|i| b"ACGT"[i % 4]).collect();
let seq: Vec<u8> = (0..132_usize).map(|i| b"ACGT"[i % 4]).collect();
let (_dir, r) = write_read(&[&seq]);
assert_eq!(r.len(), 2);
assert_eq!(r.seql(0), 259);
assert_eq!(r.seql(0), 131);
assert_eq!(r.seql(1), 4);
// k-1=3 nucleotide overlap → 0 kmers duplicated, 0 kmers lost.
// Last kmer of chunk 0 = original nucl 255..259.
assert!(r.verify_canonical_kmer(0, 255, canonical_of(&seq[255..259])));
// First kmer of chunk 1 = original nucl 256..260 — a different, adjacent kmer.
assert!(r.verify_canonical_kmer(1, 0, canonical_of(&seq[256..260])));
// Last kmer of chunk 0 = original nucl 127..131.
assert!(r.verify_canonical_kmer(0, 127, canonical_of(&seq[127..131])));
// First kmer of chunk 1 = original nucl 128..132 — a different, adjacent kmer.
assert!(r.verify_canonical_kmer(1, 0, canonical_of(&seq[128..132])));
}
#[test]
fn chunk_ranks_fit_the_7_bit_evidence_field() {
// The exact evidence of a layer stores a kmer's rank in its chunk on 7
// bits (obikindex::layer::evidence): a unitig long enough to need several
// chunks must never yield a rank above 127 — a larger one would be
// silently truncated there and the kmer would no longer verify.
set_k(4);
let seq: Vec<u8> = (0..1000_usize).map(|i| b"ACGT"[(i * i + i / 3) % 4]).collect();
let (_dir, r) = write_read(&[&seq]);
assert!(r.len() > 1, "a 997-kmer unitig must be split into several chunks");
let max_rank = r.iter_indexed_canonical_kmers().map(|(_, _, rank)| rank).max().unwrap();
assert!(max_rank <= 127, "rank {max_rank} does not fit 7 bits");
}
// ── block_bits parametrisation ────────────────────────────────────────────────