Refactor chunking logic to enforce 128 k-mer limit
This change reduces the maximum number of k-mers per chunk from 256 to 128. This modification requires updating the data model to handle sequences exceeding this limit by splitting them into overlapping chunks, ensuring no k-mers are lost. Extensive testing has been added to validate the new chunking logic and k-mer constraints.
This commit is contained in:
1 parent
8468319a63
commit
025c40c0db
8 files changed
+80
-25
No files matched your search
Executable
+24
@@ -0,0 +1,24 @@
|
||||
#!/usr/bin/env python3
|
||||
# check_unitig_chunks.py GLOB... — report unitigs.bin chunks holding more than 128 k-mers.
|
||||
# Indexes built before MAX_KMERS_PER_CHUNK went from 256 to 128 (2026-10-15) could hold chunks whose
|
||||
# rank does not fit the 7-bit evidence field; the k-mers beyond rank 127 of such a chunk then fail
|
||||
# exact verification. Example: scripts/check_unitig_chunks.py "<index>/partitions/part_*/index/layer_*/unitigs.bin"
|
||||
# (k is hard-coded below — edit it for an index built with another k.)
|
||||
import sys,glob,mmap
|
||||
k=31
|
||||
def scan(path):
|
||||
with open(path,'rb') as f:
|
||||
m=mmap.mmap(f.fileno(),0,access=mmap.ACCESS_READ)
|
||||
o=0; n=len(m); mx=0; over=0; chunks=0
|
||||
while o<n:
|
||||
c=m[o]+1; chunks+=1
|
||||
if c>mx: mx=c
|
||||
if c>128: over+=1
|
||||
o+=1+(m[o]+k+3)//4
|
||||
return mx,over,chunks
|
||||
tot_over=0
|
||||
for pat in sys.argv[1:]:
|
||||
for f in sorted(glob.glob(pat)):
|
||||
mx,over,ch=scan(f); tot_over+=over
|
||||
print(f"{f.split('partitions/')[1]}: chunks={ch} max_kmers={mx} chunks>128={over}")
|
||||
print("TOTAL chunks over 128 kmers:",tot_over)
|
||||
Reference in new issue
Block a user