Refactor chunking logic to enforce 128 k-mer limit

This change reduces the maximum number of k-mers per chunk from 256 to 128. This modification requires updating the data model to handle sequences exceeding this limit by splitting them into overlapping chunks, ensuring no k-mers are lost. Extensive testing has been added to validate the new chunking logic and k-mer constraints.
This commit is contained in:
Eric Coissac committed 2026-10-15 12:20:36 +02:00
1 parent 8468319a63
commit 025c40c0db
8 files changed
+80 -25

No files matched your search

+7 -2
View File
@@ -22,8 +22,13 @@ use crate::kmer::{CanonicalKmer, Kmer, KmerError, KLen, KmerLength, KmerOf, MLen
use crate::params::k;
use crate::revcomp_lookup::REVCOMP4;
/// Maximum kmers per stored chunk. Enforces the u8 max-kmer-index field in the binary format.
pub const MAX_KMERS_PER_CHUNK: usize = 256;
/// Maximum kmers per stored chunk.
///
/// 128 = 2^7, not the 256 the u8 length field of the binary format could
/// carry: the exact evidence of a layer (`obikindex::layer::evidence`) stores a
/// kmer's rank inside its unitig chunk on 7 bits, so every rank must be ≤ 127.
/// Longer unitigs are split into consecutive chunks at write time.
pub const MAX_KMERS_PER_CHUNK: usize = 128;
// ── PackedSeq ─────────────────────────────────────────────────────────────────
+4 -4
View File
@@ -70,7 +70,7 @@ fn seql_roundtrip() {
fn binary_test_lengths(k: usize) -> Vec<usize> {
use crate::packed_seq::MAX_KMERS_PER_CHUNK;
// Only single-chunk lengths: seql in [k, MAX_KMERS_PER_CHUNK+k-1].
(k..=k + 5).chain([255, 256, 257, MAX_KMERS_PER_CHUNK + k - 1]).collect()
(k..=k + 5).chain([MAX_KMERS_PER_CHUNK + k - 3, MAX_KMERS_PER_CHUNK + k - 2, MAX_KMERS_PER_CHUNK + k - 1]).collect()
}
#[test]
@@ -94,7 +94,7 @@ fn binary_split_roundtrip() {
use crate::packed_seq::MAX_KMERS_PER_CHUNK;
set_k(4);
let k = crate::params::k();
// seql = MAX_KMERS_PER_CHUNK + k = 260 → n_kmers = 257 > 256 → 2 chunks
// seql = MAX_KMERS_PER_CHUNK + k → n_kmers = MAX + 1 > MAX → 2 chunks
let seql = MAX_KMERS_PER_CHUNK + k;
let mut sk = SuperKmer::from_ascii(&make_seq(seql));
sk.set_count(7);
@@ -108,8 +108,8 @@ fn binary_split_roundtrip() {
assert_eq!(chunk0.count(), 7);
assert_eq!(chunk1.count(), 7);
// Chunks cover the original sequence with k-1 overlap — no kmer lost.
assert_eq!(chunk0.seql(), MAX_KMERS_PER_CHUNK + k - 1); // 259
assert_eq!(chunk1.seql(), k); // 4 (1 kmer)
assert_eq!(chunk0.seql(), MAX_KMERS_PER_CHUNK + k - 1);
assert_eq!(chunk1.seql(), k); // 1 kmer
}
#[test]
+1 -1
View File
@@ -164,7 +164,7 @@ mod tests {
use crate::packed_seq::MAX_KMERS_PER_CHUNK;
set_k(4);
let k = crate::params::k();
let valid_lengths: Vec<usize> = (k..=9).chain([255, 256, 257, MAX_KMERS_PER_CHUNK + k - 1]).collect();
let valid_lengths: Vec<usize> = (k..=9).chain([MAX_KMERS_PER_CHUNK + k - 3, MAX_KMERS_PER_CHUNK + k - 2, MAX_KMERS_PER_CHUNK + k - 1]).collect();
for len in valid_lengths {
let u = Unitig::from_ascii(&make_seq(len));
let mut buf = Vec::new();
+3 -1
View File
@@ -104,7 +104,9 @@ fn roundtrip_single() {
fn roundtrip_all_lengths() {
setup();
let bases: Vec<u8> = (0..300).map(|i| b"ACGT"[i % 4]).collect();
for len in (TEST_K..=19).chain([255, 256, 257]) {
// single-chunk lengths only: seql ≤ MAX_KMERS_PER_CHUNK + k - 1
let max_len = obikseq::MAX_KMERS_PER_CHUNK + TEST_K - 1;
for len in (TEST_K..=19).chain([max_len - 2, max_len - 1, max_len]) {
let sk = make_sk(&bases[..len]);
let mut buf = Vec::new();
sk.write_to_binary(&mut buf).unwrap();
+27 -13
View File
@@ -222,31 +222,45 @@ fn iter_indexed_chunk_id_and_rank_two_chunks() {
#[test]
fn short_unitig_not_split() {
// seql=259 → n_kmers=256 = MAX_KMERS_PER_CHUNK → no split
// seql=131 → n_kmers=128 = MAX_KMERS_PER_CHUNK → no split
set_k(4);
let seq: Vec<u8> = (0..259_usize).map(|i| b"ACGT"[i % 4]).collect();
let seq: Vec<u8> = (0..131_usize).map(|i| b"ACGT"[i % 4]).collect();
let (_dir, r) = write_read(&[&seq]);
assert_eq!(r.len(), 1);
assert_eq!(r.seql(0), 259);
assert_eq!(r.seql(0), 131);
}
#[test]
fn long_unitig_split_no_kmer_lost() {
// seql=260 → n_kmers=257 > MAX_KMERS_PER_CHUNK(256) → 2 chunks
// chunk_nucl=259, stride=256
// Chunk 0: nucl 0..259 (259 nucl, 256 kmers)
// Chunk 1: nucl 256..260 (4 nucl, 1 kmer)
// seql=132 → n_kmers=129 > MAX_KMERS_PER_CHUNK(128) → 2 chunks
// chunk_nucl=131, stride=128
// Chunk 0: nucl 0..131 (131 nucl, 128 kmers)
// Chunk 1: nucl 128..132 (4 nucl, 1 kmer)
set_k(4);
let seq: Vec<u8> = (0..260_usize).map(|i| b"ACGT"[i % 4]).collect();
let seq: Vec<u8> = (0..132_usize).map(|i| b"ACGT"[i % 4]).collect();
let (_dir, r) = write_read(&[&seq]);
assert_eq!(r.len(), 2);
assert_eq!(r.seql(0), 259);
assert_eq!(r.seql(0), 131);
assert_eq!(r.seql(1), 4);
// k-1=3 nucleotide overlap → 0 kmers duplicated, 0 kmers lost.
// Last kmer of chunk 0 = original nucl 255..259.
assert!(r.verify_canonical_kmer(0, 255, canonical_of(&seq[255..259])));
// First kmer of chunk 1 = original nucl 256..260 — a different, adjacent kmer.
assert!(r.verify_canonical_kmer(1, 0, canonical_of(&seq[256..260])));
// Last kmer of chunk 0 = original nucl 127..131.
assert!(r.verify_canonical_kmer(0, 127, canonical_of(&seq[127..131])));
// First kmer of chunk 1 = original nucl 128..132 — a different, adjacent kmer.
assert!(r.verify_canonical_kmer(1, 0, canonical_of(&seq[128..132])));
}
#[test]
fn chunk_ranks_fit_the_7_bit_evidence_field() {
// The exact evidence of a layer stores a kmer's rank in its chunk on 7
// bits (obikindex::layer::evidence): a unitig long enough to need several
// chunks must never yield a rank above 127 — a larger one would be
// silently truncated there and the kmer would no longer verify.
set_k(4);
let seq: Vec<u8> = (0..1000_usize).map(|i| b"ACGT"[(i * i + i / 3) % 4]).collect();
let (_dir, r) = write_read(&[&seq]);
assert!(r.len() > 1, "a 997-kmer unitig must be split into several chunks");
let max_rank = r.iter_indexed_canonical_kmers().map(|(_, _, rank)| rank).max().unwrap();
assert!(max_rank <= 127, "rank {max_rank} does not fit 7 bits");
}
// ── block_bits parametrisation ────────────────────────────────────────────────