2026-08-15 20:56:29 +02:00
<!DOCTYPE html>
< html class = "no-js" lang = "en" >
< head >
< meta charset = "utf-8" />
< meta content = "width=device-width,initial-scale=1" name = "viewport" />
< link href = "../../assets/images/favicon.png" rel = "icon" />
< meta content = "mkdocs-1.6.1, mkdocs-material-9.7.6" name = "generator" />
< title > obicompactvec — Complete Reference - obikmer</ title >
< link href = "../../assets/stylesheets/main.484c7ddc.min.css" rel = "stylesheet" />
< link crossorigin = "" href = "https://fonts.gstatic.com" rel = "preconnect" />
< link href = "https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback" rel = "stylesheet" />
< style >: root { --md-text-font : "Roboto" ; --md-code-font : "Roboto Mono" }</ style >
< script > __md_scope = new URL ( "../.." , location ), __md_hash = e =>[... e ]. reduce ((( e , _ )=>( e << 5 ) - e + _ . charCodeAt ( 0 )), 0 ), __md_get = ( e , _ = localStorage , t = __md_scope )=> JSON . parse ( _ . getItem ( t . pathname + "." + e )), __md_set = ( e , _ , t = localStorage , a = __md_scope )=>{ try { t . setItem ( a . pathname + "." + e , JSON . stringify ( _ ))} catch ( e ){}}</ script >
</ head >
< body dir = "ltr" >
< input autocomplete = "off" class = "md-toggle" data-md-toggle = "drawer" id = "__drawer" type = "checkbox" />
< input autocomplete = "off" class = "md-toggle" data-md-toggle = "search" id = "__search" type = "checkbox" />
< label class = "md-overlay" for = "__drawer" ></ label >
< div data-md-component = "skip" >
< a class = "md-skip" href = "#obicompactvec-complete-reference" >
Skip to content
</ a >
</ div >
< div data-md-component = "announce" >
</ div >
< header class = "md-header md-header--shadow" data-md-component = "header" >
< nav aria-label = "Header" class = "md-header__inner md-grid" >
< a aria-label = "obikmer" class = "md-header__button md-logo" data-md-component = "logo" href = "../.." title = "obikmer" >
< svg viewbox = "0 0 24 24" xmlns = "http://www.w3.org/2000/svg" >< path d = "M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54" ></ path ></ svg >
</ a >
< label class = "md-header__button md-icon" for = "__drawer" >
< svg viewbox = "0 0 24 24" xmlns = "http://www.w3.org/2000/svg" >< path d = "M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z" ></ path ></ svg >
</ label >
< div class = "md-header__title" data-md-component = "header-title" >
< div class = "md-header__ellipsis" >
< div class = "md-header__topic" >
< span class = "md-ellipsis" >
obikmer
</ span >
</ div >
< div class = "md-header__topic" data-md-component = "header-topic" >
< span class = "md-ellipsis" >
obicompactvec — Complete Reference
</ span >
</ div >
</ div >
</ div >
< script > var palette = __md_get ( "__palette" ); if ( palette && palette . color ){ if ( "(prefers-color-scheme)" === palette . color . media ){ var media = matchMedia ( "(prefers-color-scheme: light)" ), input = document . querySelector ( media . matches ? "[data-md-color-media='(prefers-color-scheme: light)']" : "[data-md-color-media='(prefers-color-scheme: dark)']" ); palette . color . media = input . getAttribute ( "data-md-color-media" ), palette . color . scheme = input . getAttribute ( "data-md-color-scheme" ), palette . color . primary = input . getAttribute ( "data-md-color-primary" ), palette . color . accent = input . getAttribute ( "data-md-color-accent" )} for ( var [ key , value ] of Object . entries ( palette . color )) document . body . setAttribute ( "data-md-color-" + key , value )}</ script >
</ nav >
</ header >
< div class = "md-container" data-md-component = "container" >
< main class = "md-main" data-md-component = "main" >
< div class = "md-main__inner md-grid" >
< div class = "md-sidebar md-sidebar--primary" data-md-component = "sidebar" data-md-type = "navigation" >
< div class = "md-sidebar__scrollwrap" >
< div class = "md-sidebar__inner" >
< nav aria-label = "Navigation" class = "md-nav md-nav--primary" data-md-level = "0" >
< label class = "md-nav__title" for = "__drawer" >
< a aria-label = "obikmer" class = "md-nav__button md-logo" data-md-component = "logo" href = "../.." title = "obikmer" >
< svg viewbox = "0 0 24 24" xmlns = "http://www.w3.org/2000/svg" >< path d = "M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54" ></ path ></ svg >
</ a >
obikmer
</ label >
< ul class = "md-nav__list" data-md-scrollfix = "" >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "../.." >
< span class = "md-ellipsis" >
Home
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "../../installation/" >
< span class = "md-ellipsis" >
Installation
</ span >
</ a >
</ li >
< li class = "md-nav__item md-nav__item--nested" >
< input class = "md-nav__toggle md-toggle" id = "__nav_3" type = "checkbox" />
< label class = "md-nav__link" for = "__nav_3" id = "__nav_3_label" tabindex = "0" >
< span class = "md-ellipsis" >
Theory
</ span >
< span class = "md-nav__icon md-icon" ></ span >
</ label >
< nav aria-expanded = "false" aria-labelledby = "__nav_3_label" class = "md-nav" data-md-level = "1" >
< label class = "md-nav__title" for = "__nav_3" >
< span class = "md-nav__icon md-icon" ></ span >
Theory
</ label >
< ul class = "md-nav__list" data-md-scrollfix = "" >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "../../kmers/" >
< span class = "md-ellipsis" >
Kmers and super-kmers
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "../../theory/encoding/" >
< span class = "md-ellipsis" >
DNA encoding
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "../../theory/entropy/" >
< span class = "md-ellipsis" >
Entropy filter
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "../../theory/minimizer/" >
< span class = "md-ellipsis" >
Minimizer selection
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "../../theory/indexing/" >
< span class = "md-ellipsis" >
Partitioning architecture
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "../../theory/evolutionary_distances/" >
< span class = "md-ellipsis" >
Central-position SNP distance (discussion)
</ span >
</ a >
</ li >
</ ul >
</ nav >
</ li >
< li class = "md-nav__item md-nav__item--nested" >
< input class = "md-nav__toggle md-toggle" id = "__nav_4" type = "checkbox" />
< label class = "md-nav__link" for = "__nav_4" id = "__nav_4_label" tabindex = "0" >
< span class = "md-ellipsis" >
Implementation
</ span >
< span class = "md-nav__icon md-icon" ></ span >
</ label >
< nav aria-expanded = "false" aria-labelledby = "__nav_4_label" class = "md-nav" data-md-level = "1" >
< label class = "md-nav__title" for = "__nav_4" >
< span class = "md-nav__icon md-icon" ></ span >
Implementation
</ label >
< ul class = "md-nav__list" data-md-scrollfix = "" >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "../superkmer/" >
< span class = "md-ellipsis" >
SuperKmer
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "../kmer/" >
< span class = "md-ellipsis" >
Kmer
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "../chunkreader/" >
< span class = "md-ellipsis" >
Chunk reader
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "../pipeline/" >
< span class = "md-ellipsis" >
Construction pipeline
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "../obipipeline/" >
< span class = "md-ellipsis" >
obipipeline library
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "../storage/" >
< span class = "md-ellipsis" >
On-disk storage
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "../mphf/" >
< span class = "md-ellipsis" >
MPHF selection
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "../unitig_evidence/" >
< span class = "md-ellipsis" >
Unitig evidence encoding
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "../evidence_elimination/" >
< span class = "md-ellipsis" >
Evidence elimination (discussion)
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "../obilayeredmap/" >
< span class = "md-ellipsis" >
obilayeredmap crate
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "../persistent_compact_int_vec/" >
< span class = "md-ellipsis" >
PersistentCompactIntVec
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "../persistent_bit_vec/" >
< span class = "md-ellipsis" >
PersistentBitVec
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "../merge/" >
< span class = "md-ellipsis" >
Merge command
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "../merge_parallelism/" >
< span class = "md-ellipsis" >
Merge parallelism & memory
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "../filtering/" >
< span class = "md-ellipsis" >
Kmer filtering
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "../select/" >
< span class = "md-ellipsis" >
Select command
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "../obitaxonomy/" >
< span class = "md-ellipsis" >
obitaxonomy crate
2026-08-20 03:32:37 +02:00
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "../benchmark_query_testing/" >
< span class = "md-ellipsis" >
Benchmark: query-path testing
2026-08-15 20:56:29 +02:00
</ span >
</ a >
</ li >
</ ul >
</ nav >
</ li >
< li class = "md-nav__item md-nav__item--nested" >
< input class = "md-nav__toggle md-toggle" id = "__nav_5" type = "checkbox" />
< label class = "md-nav__link" for = "__nav_5" id = "__nav_5_label" tabindex = "0" >
< span class = "md-ellipsis" >
Architecture
</ span >
< span class = "md-nav__icon md-icon" ></ span >
</ label >
< nav aria-expanded = "false" aria-labelledby = "__nav_5_label" class = "md-nav" data-md-level = "1" >
< label class = "md-nav__title" for = "__nav_5" >
< span class = "md-nav__icon md-icon" ></ span >
Architecture
</ label >
< ul class = "md-nav__list" data-md-scrollfix = "" >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "../../architecture/sequences/invariant/" >
< span class = "md-ellipsis" >
Sequences
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "../../architecture/index_architecture/" >
< span class = "md-ellipsis" >
Kmer index
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "../../architecture/siblings/" >
< span class = "md-ellipsis" >
Sibling annex (discussion)
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "../../architecture/numa_worker_pools/" >
< span class = "md-ellipsis" >
NUMA-aware worker pools
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "../../architecture/numa_partition_runner/" >
< span class = "md-ellipsis" >
NUMA-aware partition runner
</ span >
</ a >
</ li >
</ ul >
</ nav >
</ li >
</ ul >
</ nav >
</ div >
</ div >
</ div >
< div class = "md-sidebar md-sidebar--secondary" data-md-component = "sidebar" data-md-type = "toc" >
< div class = "md-sidebar__scrollwrap" >
< div class = "md-sidebar__inner" >
< nav aria-label = "Table of contents" class = "md-nav md-nav--secondary" >
< label class = "md-nav__title" for = "__toc" >
< span class = "md-nav__icon md-icon" ></ span >
Table of contents
</ label >
< ul class = "md-nav__list" data-md-component = "toc" data-md-scrollfix = "" >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "#module-structure" >
< span class = "md-ellipsis" >
Module structure
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "#compact-int-encoding" >
< span class = "md-ellipsis" >
Compact int encoding
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "#view-types" >
< span class = "md-ellipsis" >
View types
</ span >
</ a >
< nav aria-label = "View types" class = "md-nav" >
< ul class = "md-nav__list" >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "#bitsliceviewa" >
< span class = "md-ellipsis" >
BitSliceView< 'a>
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "#intsliceviewa" >
< span class = "md-ellipsis" >
IntSliceView< 'a>
</ span >
</ a >
</ li >
</ ul >
</ nav >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "#concrete-types" >
< span class = "md-ellipsis" >
Concrete types
</ span >
</ a >
< nav aria-label = "Concrete types" class = "md-nav" >
< ul class = "md-nav__list" >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "#persistentbitvec-persistentbitvecbuilder" >
< span class = "md-ellipsis" >
PersistentBitVec / PersistentBitVecBuilder
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "#persistentcompactintvec-persistentcompactintvecbuilder" >
< span class = "md-ellipsis" >
PersistentCompactIntVec / PersistentCompactIntVecBuilder
</ span >
</ a >
</ li >
</ ul >
</ nav >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "#matrix-types" >
< span class = "md-ellipsis" >
Matrix types
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "#aggregation-traits-matrix-level" >
< span class = "md-ellipsis" >
Aggregation traits (matrix level)
</ span >
</ a >
< nav aria-label = "Aggregation traits (matrix level)" class = "md-nav" >
< ul class = "md-nav__list" >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "#columnweights" >
< span class = "md-ellipsis" >
ColumnWeights
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "#countpartials" >
< span class = "md-ellipsis" >
CountPartials
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "#bitpartials" >
< span class = "md-ellipsis" >
BitPartials
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "#mash-distance" >
< span class = "md-ellipsis" >
Mash distance
</ span >
</ a >
</ li >
</ ul >
</ nav >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "#temp-file-backed-types" >
< span class = "md-ellipsis" >
Temp-file-backed types
</ span >
</ a >
< nav aria-label = "Temp-file-backed types" class = "md-nav" >
< ul class = "md-nav__list" >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "#lifecycle" >
< span class = "md-ellipsis" >
Lifecycle
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "#tempcompactintvec-tempcompactintvecbuilder" >
< span class = "md-ellipsis" >
TempCompactIntVec / TempCompactIntVecBuilder
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "#tempbitvec-tempbitvecbuilder" >
< span class = "md-ellipsis" >
TempBitVec / TempBitVecBuilder
</ span >
</ a >
</ li >
</ ul >
</ nav >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "#filter-select-api" >
< span class = "md-ellipsis" >
Filter / Select API
</ span >
</ a >
< nav aria-label = "Filter / Select API" class = "md-nav" >
< ul class = "md-nav__list" >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "#colgroup" >
< span class = "md-ellipsis" >
ColGroup
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "#composition-axis" >
< span class = "md-ellipsis" >
Composition axis
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "#matrixgroupops" >
< span class = "md-ellipsis" >
MatrixGroupOps
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "#add_col_from-matrix-builder-integration" >
< span class = "md-ellipsis" >
add_col_from — matrix builder integration
</ span >
</ a >
</ li >
< li class = "md-nav__item" >
< a class = "md-nav__link" href = "#mask_with" >
< span class = "md-ellipsis" >
mask_with
</ span >
</ a >
</ li >
</ ul >
</ nav >
</ li >
</ ul >
</ nav >
</ div >
</ div >
</ div >
< div class = "md-content" data-md-component = "content" >
< article class = "md-content__inner md-typeset" >
< h1 id = "obicompactvec-complete-reference" > obicompactvec — Complete Reference</ h1 >
< h2 id = "module-structure" > Module structure</ h2 >
< div class = "highlight" >< pre >< span ></ span >< code > src/obicompactvec/src/
lib.rs public re-exports
views.rs BitSliceView< 'a> , IntSliceView< 'a> — zero-copy read views
traits.rs ColumnWeights, CountPartials, BitPartials (matrix aggregation)
bitvec.rs PersistentBitVec, PersistentBitVecBuilder, BitIter
reader.rs PersistentCompactIntVec (read-only)
builder.rs PersistentCompactIntVecBuilder (read-write)
tempintvec.rs TempCompactIntVec, TempCompactIntVecBuilder (temp-file-backed)
tempbitvec.rs TempBitVec, TempBitVecBuilder (temp-file-backed)
bitmatrix.rs PersistentBitMatrix, PersistentBitMatrixBuilder
intmatrix.rs PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder
colgroup.rs ColGroup, MatrixGroupOps trait
format.rs file format constants, encode/decode helpers
layer_meta.rs LayerMeta (column metadata)
meta.rs matrix metadata
</ code ></ pre ></ div >
< pre class = "mermaid" >< code > graph TD
views --> bitvec
views --> builder
views --> tempbitvec
views --> tempintvec
views --> bitmatrix
views --> intmatrix
format --> reader
format --> builder
reader --> intmatrix
reader --> tempintvec
builder --> intmatrix
builder --> tempintvec
bitvec --> tempbitvec
bitvec --> bitmatrix
tempintvec --> intmatrix
tempintvec --> bitmatrix
tempbitvec --> intmatrix
tempbitvec --> bitmatrix
colgroup --> intmatrix
colgroup --> bitmatrix
layer_meta --> bitmatrix
layer_meta --> intmatrix
meta --> bitmatrix
meta --> intmatrix</ code ></ pre >
< hr />
< h2 id = "compact-int-encoding" > Compact int encoding</ h2 >
< p > All integer vectors use the same two-tier encoding regardless of storage backend.</ p >
< p >< strong > Primary array</ strong > — one < code > u8</ code > per slot:</ p >
< ul >
< li > Values < strong > 0–254</ strong > are stored directly. No overhead.</ li >
< li > Value < strong > 255 is a sentinel</ strong > : the slot's actual value is ≥ 255 and lives in the overflow store.</ li >
</ ul >
< p >< strong > Overflow store</ strong > — maps slot index to a < code > u32</ code > value ≥ 255:</ p >
< ul >
< li > In < code > PersistentCompactIntVecBuilder</ code > : a < code > HashMap< usize, u32> </ code > in RAM.</ li >
< li > In < code > PersistentCompactIntVec</ code > (reader): a sorted < code > [(slot: u64, value: u32)]</ code > array in the mmap, with a sparse L1-resident index for binary search.</ li >
</ ul >
< pre class = "mermaid" >< code > flowchart LR
slot --> P["primary[slot]: u8"]
P --> |"< 255"| V["value = byte (0–254)"]
P --> |"= 255 sentinel"| OV["overflow store"]
OV --> |"Builder"| HM["HashMap& lt;usize, u32& gt;\nin RAM"]
OV --> |"PersistentCompactIntVec"| SA["sorted [(slot,value)] in mmap\n+ sparse L1 index"]</ code ></ pre >
< p >< strong > Key property — sentinel 255 = +∞ on < code > u8</ code > :</ strong ></ p >
< ul >
< li >< code > min(a, 255) = a</ code > for all < code > a ≤ 254</ code > → correct when only one side is overflow</ li >
< li >< code > max(a, 255) = 255</ code > → correct sentinel when either side is overflow</ li >
< li > Only the < strong > both-overflow</ strong > case requires reading actual values from the overflow store.</ li >
</ ul >
< p > In practice, k (overflow count) ≪ n (total slots). Observed genomic data: ~0.07% of kmer slots are in overflow.</ p >
< hr />
< h2 id = "view-types" > View types</ h2 >
< p > The previous trait hierarchy (< code > BitSlice</ code > , < code > BitSliceMut</ code > , < code > IntSlice</ code > , < code > IntSliceMut</ code > ) has been replaced by two concrete zero-copy view structs with inherent methods. Views are < strong >< code > Copy</ code ></ strong > — passing them is free. All read operations live on these two types.</ p >
< h3 id = "bitsliceviewa" >< code > BitSliceView< 'a> </ code ></ h3 >
< div class = "highlight" >< pre >< span ></ span >< code >< span class = "cp" > #[derive(Clone, Copy)]</ span >
< span class = "k" > pub</ span >< span class = "w" > </ span >< span class = "k" > struct</ span >< span class = "w" > </ span >< span class = "nc" > BitSliceView</ span >< span class = "o" > < '</ span >< span class = "na" > a</ span >< span class = "o" > > </ span >< span class = "w" > </ span >< span class = "p" > {</ span >< span class = "w" > </ span >< span class = "k" > pub</ span >< span class = "p" > (</ span >< span class = "k" > crate</ span >< span class = "p" > )</ span >< span class = "w" > </ span >< span class = "n" > words</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "kp" > & </ span >< span class = "o" > '</ span >< span class = "na" > a</ span >< span class = "w" > </ span >< span class = "p" > [</ span >< span class = "kt" > u64</ span >< span class = "p" > ],</ span >< span class = "w" > </ span >< span class = "k" > pub</ span >< span class = "p" > (</ span >< span class = "k" > crate</ span >< span class = "p" > )</ span >< span class = "w" > </ span >< span class = "n" > n</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "kt" > usize</ span >< span class = "w" > </ span >< span class = "p" > }</ span >
</ code ></ pre ></ div >
< p > Bit < code > i</ code > is at < code > words[i >> 6]</ code > bit < code > i & 63</ code > (LSB-first). Padding bits in the last word are zero.</ p >
< table >
< thead >
< tr >
< th > Method</ th >
< th > Cost</ th >
</ tr >
</ thead >
< tbody >
< tr >
< td >< code > len()</ code > , < code > is_empty()</ code ></ td >
< td > O(1)</ td >
</ tr >
< tr >
< td >< code > get(slot)</ code ></ td >
< td > O(1)</ td >
</ tr >
< tr >
< td >< code > count_ones()</ code ></ td >
< td > POPCNT per word, O(n/64)</ td >
</ tr >
< tr >
< td >< code > count_zeros()</ code ></ td >
< td >< code > n − count_ones()</ code > , O(n/64)</ td >
</ tr >
< tr >
< td >< code > iter() -> BitSliceIter< 'a> </ code ></ td >
< td > O(1) setup, O(n) iteration</ td >
</ tr >
< tr >
< td >< code > partial_jaccard_dist(other: BitSliceView)</ code ></ td >
< td >< code > (a& b).popcount</ code > , < code > (a\|b).popcount</ code > per word, O(n/64)</ td >
</ tr >
< tr >
< td >< code > jaccard_dist(other: BitSliceView)</ code ></ td >
< td > from partial, O(n/64)</ td >
</ tr >
< tr >
< td >< code > hamming_dist(other: BitSliceView)</ code ></ td >
< td >< code > (a^b).popcount</ code > per word, O(n/64)</ td >
</ tr >
</ tbody >
</ table >
< p >< code > BitSliceIter< 'a> </ code > : word-level scan; one word per 64 iterations.</ p >
< h3 id = "intsliceviewa" >< code > IntSliceView< 'a> </ code ></ h3 >
< div class = "highlight" >< pre >< span ></ span >< code >< span class = "cp" > #[derive(Clone, Copy)]</ span >
< span class = "k" > pub</ span >< span class = "w" > </ span >< span class = "k" > struct</ span >< span class = "w" > </ span >< span class = "nc" > IntSliceView</ span >< span class = "o" > < '</ span >< span class = "na" > a</ span >< span class = "o" > > </ span >< span class = "w" > </ span >< span class = "p" > {</ span >
< span class = "w" > </ span >< span class = "k" > pub</ span >< span class = "p" > (</ span >< span class = "k" > crate</ span >< span class = "p" > )</ span >< span class = "w" > </ span >< span class = "n" > primary</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "kp" > & </ span >< span class = "o" > '</ span >< span class = "na" > a</ span >< span class = "w" > </ span >< span class = "p" > [</ span >< span class = "kt" > u8</ span >< span class = "p" > ],</ span >
< span class = "w" > </ span >< span class = "k" > pub</ span >< span class = "p" > (</ span >< span class = "k" > crate</ span >< span class = "p" > )</ span >< span class = "w" > </ span >< span class = "n" > overflow_raw</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "kp" > & </ span >< span class = "o" > '</ span >< span class = "na" > a</ span >< span class = "w" > </ span >< span class = "p" > [</ span >< span class = "kt" > u8</ span >< span class = "p" > ],</ span >< span class = "w" > </ span >< span class = "c1" > // sorted [(slot:u64, value:u32)] entries</ span >
< span class = "w" > </ span >< span class = "k" > pub</ span >< span class = "p" > (</ span >< span class = "k" > crate</ span >< span class = "p" > )</ span >< span class = "w" > </ span >< span class = "n" > n_overflow</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "kt" > usize</ span >< span class = "p" > ,</ span >
< span class = "w" > </ span >< span class = "k" > pub</ span >< span class = "p" > (</ span >< span class = "k" > crate</ span >< span class = "p" > )</ span >< span class = "w" > </ span >< span class = "n" > n</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "kt" > usize</ span >< span class = "p" > ,</ span >
< span class = "p" > }</ span >
</ code ></ pre ></ div >
< p >< code > overflow_raw</ code > contains < code > n_overflow</ code > entries of < code > OVERFLOW_ENTRY_SIZE</ code > bytes each, sorted by slot. The sort invariant is established at < code > close()</ code > /< code > freeze()</ code > time.</ p >
< table >
< thead >
< tr >
< th > Method</ th >
< th > Cost</ th >
</ tr >
</ thead >
< tbody >
< tr >
< td >< code > len()</ code > , < code > is_empty()</ code ></ td >
< td > O(1)</ td >
</ tr >
< tr >
< td >< code > primary_bytes()</ code ></ td >
< td > O(1)</ td >
</ tr >
< tr >
< td >< code > overflow_entries() -> impl Iterator< (usize,u32)> </ code ></ td >
< td > O(n_overflow) iteration</ td >
</ tr >
< tr >
< td >< code > get(slot)</ code ></ td >
< td > O(1) primary; binary search O(log k) for overflow slots</ td >
</ tr >
< tr >
< td >< code > iter() -> IntSliceViewIter< 'a> </ code ></ td >
< td > merge scan, O(n + k)</ td >
</ tr >
< tr >
< td >< code > sum()</ code ></ td >
< td > byte scan + overflow, O(n + k)</ td >
</ tr >
< tr >
< td >< code > count_nonzero()</ code ></ td >
< td > byte scan, O(n)</ td >
</ tr >
< tr >
< td > Distance methods (< code > bray_dist</ code > , < code > euclidean_dist</ code > , < code > jaccard_dist</ code > , …)</ td >
< td > O(n + k)</ td >
</ tr >
</ tbody >
</ table >
< p >< code > IntSliceViewIter< 'a> </ code > : merge scan using < code > overflow_pos</ code > index. Requires sorted overflow — guaranteed by the construction lifecycle.</ p >
< p >< strong > Builder < code > view()</ code > vs reader < code > view()</ code > :</ strong > < code > PersistentCompactIntVecBuilder</ code > stores overflow as an unsorted < code > HashMap</ code > , not raw bytes. Its < code > view()</ code > returns an < code > IntSliceView</ code > with < code > overflow_raw = & []</ code > and < code > n_overflow = 0</ code > . This is intentional — the view is primarily useful after < code > freeze()</ code > . During building, callers that need overflow use < code > overflow_entries()</ code > directly.</ p >
< hr />
< h2 id = "concrete-types" > Concrete types</ h2 >
< pre class = "mermaid" >< code > classDiagram
class BitSliceView {
+words: & [u64]
+n: usize
+get(slot) bool
+count_ones() u64
+iter() BitSliceIter
+jaccard_dist/hamming_dist(other: BitSliceView)
}
class IntSliceView {
+primary: & [u8]
+overflow_raw: & [u8]
+n_overflow: usize
+n: usize
+get(slot) u32
+iter() IntSliceViewIter
+overflow_entries() Iterator
+bray_dist/euclidean_dist/…(other: IntSliceView)
}
class PersistentBitVec {
-mmap: Mmap
-n: usize
+view() BitSliceView
+get(slot) bool
+count_ones/zeros() u64
+iter() BitIter
+partial_jaccard_dist(& Self) (u64,u64)
+jaccard_dist/hamming_dist(& Self) …
}
class PersistentBitVecBuilder {
-mmap: MmapMut
-n: usize
+view() BitSliceView
+set(slot, bool)
+or/and/xor/not(BitSliceView)
+copy_from(BitSliceView)
+close() / finish() → PersistentBitVec
}
class PersistentCompactIntVec {
-mmap: Mmap
-n: usize
-n_overflow: usize
-step: usize
-index: Vec~(usize,usize)~
+view() IntSliceView
+get(slot) u32
+iter() Iter
+sum/count_nonzero() u64
+bray_dist/euclidean_dist/… (& Self)
}
class PersistentCompactIntVecBuilder {
-mmap: MmapMut
-n: usize
-overflow: HashMap~usize,u32~
+view() IntSliceView
+set(slot, u32) / get(slot) u32
+inc / inc_present / inc_present_fast
+inc_predicate / inc_predicate_fast
+add/min/max/diff/mask_with(…View)
+primary_bytes/primary_bytes_mut()
+close() / finish() → PersistentCompactIntVec
}
PersistentBitVec --> BitSliceView : view()
PersistentBitVecBuilder --> BitSliceView : view()
PersistentCompactIntVec --> IntSliceView : view()
PersistentCompactIntVecBuilder --> IntSliceView : view() (primary only)
PersistentBitVecBuilder --> PersistentBitVec : close() then open()
PersistentCompactIntVecBuilder --> PersistentCompactIntVec : close() then open()</ code ></ pre >
< h3 id = "persistentbitvec-persistentbitvecbuilder" >< code > PersistentBitVec</ code > / < code > PersistentBitVecBuilder</ code ></ h3 >
< p >< code > PersistentBitVec</ code > is the read-only type. < code > view()</ code > returns a < code > BitSliceView< '_> </ code > over the mmap word array. Direct inherent methods delegate to the view: < code > count_ones()</ code > , < code > count_zeros()</ code > , < code > partial_jaccard_dist(& Self)</ code > , < code > jaccard_dist(& Self)</ code > , < code > hamming_dist(& Self)</ code > .</ p >
< p >< code > BitIter< 'a> </ code > — exported iterator for < code > PersistentBitVec::iter()</ code > :</ p >
< div class = "highlight" >< pre >< span ></ span >< code >< span class = "k" > pub</ span >< span class = "w" > </ span >< span class = "k" > struct</ span >< span class = "w" > </ span >< span class = "nc" > BitIter</ span >< span class = "o" > < '</ span >< span class = "na" > a</ span >< span class = "o" > > </ span >< span class = "w" > </ span >< span class = "p" > {</ span >< span class = "w" > </ span >< span class = "k" > pub</ span >< span class = "p" > (</ span >< span class = "k" > crate</ span >< span class = "p" > )</ span >< span class = "w" > </ span >< span class = "n" > words</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "kp" > & </ span >< span class = "o" > '</ span >< span class = "na" > a</ span >< span class = "w" > </ span >< span class = "p" > [</ span >< span class = "kt" > u64</ span >< span class = "p" > ],</ span >< span class = "w" > </ span >< span class = "k" > pub</ span >< span class = "p" > (</ span >< span class = "k" > crate</ span >< span class = "p" > )</ span >< span class = "w" > </ span >< span class = "n" > slot</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "kt" > usize</ span >< span class = "p" > ,</ span >< span class = "w" > </ span >< span class = "k" > pub</ span >< span class = "p" > (</ span >< span class = "k" > crate</ span >< span class = "p" > )</ span >< span class = "w" > </ span >< span class = "n" > n</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "kt" > usize</ span >< span class = "w" > </ span >< span class = "p" > }</ span >
</ code ></ pre ></ div >
< p >< code > PersistentBitVecBuilder</ code > is the read-write type. Mutation operations accept < code > BitSliceView< '_> </ code > :</ p >
< table >
< thead >
< tr >
< th > Method</ th >
< th > Cost</ th >
</ tr >
</ thead >
< tbody >
< tr >
< td >< code > set(slot, bool)</ code ></ td >
< td > O(1)</ td >
</ tr >
< tr >
< td >< code > view() -> BitSliceView< '_> </ code ></ td >
< td > O(1)</ td >
</ tr >
< tr >
< td >< code > or/and/xor(BitSliceView)</ code ></ td >
< td > word-level, O(n/64), SIMD-friendly</ td >
</ tr >
< tr >
< td >< code > not()</ code ></ td >
< td >< code > w ^= u64::MAX</ code > per word, re-masks last word</ td >
</ tr >
< tr >
< td >< code > copy_from(BitSliceView)</ code ></ td >
< td >< code > copy_from_slice</ code ></ td >
</ tr >
</ tbody >
</ table >
< h3 id = "persistentcompactintvec-persistentcompactintvecbuilder" >< code > PersistentCompactIntVec</ code > / < code > PersistentCompactIntVecBuilder</ code ></ h3 >
< p >< code > PersistentCompactIntVec</ code > is the read-only type. < code > view()</ code > returns an < code > IntSliceView< '_> </ code > over the mmap primary and overflow arrays. Inherent < code > iter()</ code > is a merge scan (< code > Iter</ code > struct). Inherent < code > sum()</ code > and < code > count_nonzero()</ code > use fast byte-scan helpers.</ p >
< p >< code > PersistentCompactIntVecBuilder</ code > is the read-write type. Mutation methods on the builder fall into two categories:</ p >
< p >< strong > Point mutations:</ strong ></ p >
< table >
< thead >
< tr >
< th > Method</ th >
< th > Note</ th >
</ tr >
</ thead >
< tbody >
< tr >
< td >< code > set(slot, u32)</ code ></ td >
< td > writes primary[slot] or 255+overflow</ td >
</ tr >
< tr >
< td >< code > get(slot) -> u32</ code ></ td >
< td > reads primary byte or HashMap</ td >
</ tr >
< tr >
< td >< code > inc(slot)</ code ></ td >
< td >< code > get</ code > + < code > set</ code > , O(1)</ td >
</ tr >
</ tbody >
</ table >
< p >< strong > Bulk computation methods</ strong > — accept view arguments:</ p >
< table >
< thead >
< tr >
< th > Method</ th >
< th > Semantics</ th >
< th > Overflow</ th >
</ tr >
</ thead >
< tbody >
< tr >
< td >< code > inc_present(BitSliceView)</ code ></ td >
< td >< code > += 1</ code > at each 1-bit</ td >
< td > via < code > inc</ code > , safe for any group size</ td >
</ tr >
< tr >
< td >< code > inc_present_fast(BitSliceView)</ code ></ td >
< td > same, raw u8 < code > += 1</ code ></ td >
< td >< code > debug_assert</ code > no 255 reached</ td >
</ tr >
< tr >
< td >< code > inc_predicate(IntSliceView, pred)</ code ></ td >
< td >< code > += 1</ code > where < code > pred(col[s])</ code ></ td >
< td > two-pass, safe</ td >
</ tr >
< tr >
< td >< code > inc_predicate_fast(IntSliceView, pred)</ code ></ td >
< td > same, raw u8</ td >
< td >< code > debug_assert</ code > no 255 reached</ td >
</ tr >
< tr >
< td >< code > add(IntSliceView)</ code ></ td >
< td >< code > self[s] += other[s]</ code ></ td >
< td > primary fast path + overflow fallback</ td >
</ tr >
< tr >
< td >< code > min(IntSliceView)</ code ></ td >
< td > byte min + both-overflow fixup</ td >
< td > see algorithm below</ td >
</ tr >
< tr >
< td >< code > max(IntSliceView)</ code ></ td >
< td > pre-pass + byte max</ td >
< td > see algorithm below</ td >
</ tr >
< tr >
< td >< code > diff(IntSliceView)</ code ></ td >
< td > saturating sub</ td >
< td > self< 255 hot path</ td >
</ tr >
< tr >
< td >< code > mask_with(BitSliceView)</ code ></ td >
< td > zeros slots where mask bit = 0</ td >
< td > O(n_zeros)</ td >
</ tr >
</ tbody >
</ table >
< p >< strong >< code > inc_present_fast</ code > / < code > inc_predicate_fast</ code > invariant:</ strong > caller guarantees no counter reaches 255 during the operation (group size < 255 for < code > inc_present_fast</ code > , or chunk size < 255 for < code > inc_predicate_fast</ code > ). Violation is caught by < code > debug_assert</ code > in dev builds.</ p >
< p >< strong >< code > min</ code > algorithm:</ strong ></ p >
< p > Exploits 255 = +∞: byte-level min is correct unless both sides are overflow.</ p >
< div class = "highlight" >< pre >< span ></ span >< code > snapshot self_ov: Vec< (slot,val)>
snapshot other_ov: HashMap< slot,val>
clear_overflow()
Pass 1 — byte min, SIMD-vectorizable, O(n)
Pass 2 — both-overflow fixup, O(k_self):
for (slot, self_val) in self_ov:
if slot ∈ other_ov: set(slot, min(self_val, other_ov[slot]))
</ code ></ pre ></ div >
< p >< strong >< code > max</ code > algorithm:</ strong ></ p >
< p > Cannot do byte max first — < code > max(255, b< 255)=255</ code > overwrites self's original overflow value. Pre-pass reads self's value at other's overflow slots before the byte pass.</ p >
< div class = "highlight" >< pre >< span ></ span >< code > Pre-pass O(k_other): for (slot, other_val) in other.overflow_entries():
set(slot, max(self.get(slot), other_val))
Pass 1 — byte max, SIMD-vectorizable, O(n)
</ code ></ pre ></ div >
< hr />
< h2 id = "matrix-types" > Matrix types</ h2 >
2026-08-20 03:32:37 +02:00
< p > Both matrix types are enums behind a transparent API — the caller never matches on the variant. < code > PersistentCompactIntMatrix</ code > has two variants (< code > Columnar</ code > , < code > Packed</ code > ). < code > PersistentBitMatrix</ code > has four:</ p >
2026-08-15 20:56:29 +02:00
< table >
< thead >
< tr >
2026-08-20 03:32:37 +02:00
< th > Variant</ th >
< th > Storage</ th >
< th > When</ th >
2026-08-15 20:56:29 +02:00
</ tr >
</ thead >
< tbody >
< tr >
2026-08-20 03:32:37 +02:00
< td >< code > Columnar</ code ></ td >
< td > one < code > .pbiv</ code > /< code > .pciv</ code > file per column + < code > meta.json</ code ></ td >
< td > build-time default (< code > *Builder::new</ code > )</ td >
2026-08-15 20:56:29 +02:00
</ tr >
< tr >
2026-08-20 03:32:37 +02:00
< td >< code > Packed</ code ></ td >
< td > single < code > matrix.pbmx</ code > mmap file</ td >
< td > query-optimised, produced by < code > pack_bit_matrix</ code > /< code > pack_compact_int_matrix</ code ></ td >
</ tr >
< tr >
< td >< code > Sparse</ code > (bit only)</ td >
< td >< code > sparse_meta.json</ code > + PFIV/Elias-Fano component files, row-major</ td >
< td >< code > pack --sparse</ code > ; see < a href = "../../architecture/siblings/" > siblings.md</ a > for the sparse-vs-dense access-pattern trade-off</ td >
</ tr >
< tr >
< td >< code > Implicit</ code > (bit only)</ td >
< td > no file at all</ td >
< td > mono-genome presence layers — < code > n_cols</ code > is always reported as < code > 1</ code > , every value is < code > true</ code ></ td >
2026-08-15 20:56:29 +02:00
</ tr >
</ tbody >
</ table >
2026-08-20 03:32:37 +02:00
< p >< code > PersistentBitMatrix::open(layer_dir)</ code > auto-detects the variant, in order: < code > matrix.pbmx</ code > → Packed, < code > presence/meta.json</ code > → Columnar, < code > presence/sparse_meta.json</ code > → Sparse, < code > layer_meta.json</ code > (no presence dir at all) → Implicit. < code > col_view</ code > /< code > col</ code > /< code > sub_matrix</ code > panic on < code > Sparse</ code > /< code > Implicit</ code > where the operation has no direct-slice equivalent (Sparse is k-mer-major, not column-major; Implicit has no backing storage) — callers needing per-column data on those variants go through < code > row</ code > /< code > fill_row</ code > .</ p >
< p >< code > col_view(c)</ code > returns the appropriate view directly:</ p >
2026-08-15 20:56:29 +02:00
< div class = "highlight" >< pre >< span ></ span >< code >< span class = "c1" > // PersistentBitMatrix</ span >
< span class = "k" > pub</ span >< span class = "w" > </ span >< span class = "k" > fn</ span >< span class = "w" > </ span >< span class = "nf" > col_view</ span >< span class = "p" > (</ span >< span class = "o" > & </ span >< span class = "bp" > self</ span >< span class = "p" > ,</ span >< span class = "w" > </ span >< span class = "n" > c</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "kt" > usize</ span >< span class = "p" > )</ span >< span class = "w" > </ span >< span class = "p" > -> </ span >< span class = "w" > </ span >< span class = "nc" > BitSliceView</ span >< span class = "o" > < '</ span >< span class = "nb" > _</ span >< span class = "o" > > </ span >
< span class = "c1" > // PersistentCompactIntMatrix</ span >
< span class = "k" > pub</ span >< span class = "w" > </ span >< span class = "k" > fn</ span >< span class = "w" > </ span >< span class = "nf" > col_view</ span >< span class = "p" > (</ span >< span class = "o" > & </ span >< span class = "bp" > self</ span >< span class = "p" > ,</ span >< span class = "w" > </ span >< span class = "n" > c</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "kt" > usize</ span >< span class = "p" > )</ span >< span class = "w" > </ span >< span class = "p" > -> </ span >< span class = "w" > </ span >< span class = "nc" > IntSliceView</ span >< span class = "o" > < '</ span >< span class = "nb" > _</ span >< span class = "o" > > </ span >
</ code ></ pre ></ div >
< p > No wrapper enums (< code > BitColView</ code > , < code > IntColView</ code > ): the caller receives a < code > Copy</ code > view struct immediately usable with any view method or bulk builder method.</ p >
< p >< code > pack_compact_int_matrix</ code > and < code > pack_bit_matrix</ code > convert columnar → packed format.</ p >
< hr />
< h2 id = "aggregation-traits-matrix-level" > Aggregation traits (matrix level)</ h2 >
< h3 id = "columnweights" > ColumnWeights</ h3 >
< div class = "highlight" >< pre >< span ></ span >< code >< span class = "k" > trait</ span >< span class = "w" > </ span >< span class = "n" > ColumnWeights</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "nb" > Send</ span >< span class = "w" > </ span >< span class = "o" > +</ span >< span class = "w" > </ span >< span class = "nb" > Sync</ span >< span class = "w" > </ span >< span class = "p" > {</ span >
< span class = "w" > </ span >< span class = "k" > fn</ span >< span class = "w" > </ span >< span class = "nf" > col_weights</ span >< span class = "p" > (</ span >< span class = "o" > & </ span >< span class = "bp" > self</ span >< span class = "p" > )</ span >< span class = "w" > </ span >< span class = "p" > -> </ span >< span class = "w" > </ span >< span class = "nc" > Array1</ span >< span class = "o" > < </ span >< span class = "kt" > u64</ span >< span class = "o" > > </ span >< span class = "p" > ;</ span >< span class = "w" > </ span >< span class = "c1" > // sum per column</ span >
< span class = "w" > </ span >< span class = "k" > fn</ span >< span class = "w" > </ span >< span class = "nf" > partial_kmer_counts</ span >< span class = "p" > (</ span >< span class = "o" > & </ span >< span class = "bp" > self</ span >< span class = "p" > )</ span >< span class = "w" > </ span >< span class = "p" > -> </ span >< span class = "w" > </ span >< span class = "nc" > Array1</ span >< span class = "o" > < </ span >< span class = "kt" > u64</ span >< span class = "o" > > </ span >< span class = "p" > ;</ span >< span class = "w" > </ span >< span class = "c1" > // default = col_weights()</ span >
< span class = "p" > }</ span >
</ code ></ pre ></ div >
< p >< code > partial_kmer_counts</ code > is overridden for count matrices to return < code > count_nonzero</ code > per column (distinct kmers) rather than total count.</ p >
< h3 id = "countpartials" > CountPartials</ h3 >
< p > Abstract required methods: < code > partial_bray</ code > , < code > partial_euclidean</ code > , < code > partial_threshold_jaccard</ code > , < code > partial_relfreq_bray</ code > , < code > partial_relfreq_euclidean</ code > , < code > partial_hellinger</ code > .</ p >
< p >< strong > Additivity rule:</ strong > self-contained partials (< code > partial_bray</ code > , < code > partial_euclidean</ code > , < code > partial_threshold_jaccard</ code > ) can be element-wise summed across all < code > (partition, layer)</ code > pairs. Normalised partials (< code > partial_relfreq_*</ code > , < code > partial_hellinger</ code > ) require the < strong > global</ strong > < code > col_weights</ code > (accumulated across all layers and all partitions) as parameter.</ p >
< p >< strong >< code > partial_threshold_jaccard</ code > returns < code > (inter, union)</ code ></ strong > because < code > union[i,j]</ code > depends on both columns simultaneously.</ p >
< p > Provided finalisations:</ p >
< table >
< thead >
< tr >
< th > Finalisation</ th >
< th > Formula</ th >
</ tr >
</ thead >
< tbody >
< tr >
< td >< code > bray_dist_matrix()</ code ></ td >
< td >< code > 1 − 2·partial_bray[i,j] / (w[i] + w[j])</ code ></ td >
</ tr >
< tr >
< td >< code > euclidean_dist_matrix()</ code ></ td >
< td >< code > √partial_euclidean[i,j]</ code ></ td >
</ tr >
< tr >
< td >< code > threshold_jaccard_dist_matrix(t)</ code ></ td >
< td >< code > 1 − inter[i,j] / union[i,j]</ code ></ td >
</ tr >
< tr >
< td >< code > relfreq_bray_dist_matrix()</ code ></ td >
< td >< code > 1 − partial_relfreq_bray[i,j]</ code ></ td >
</ tr >
< tr >
< td >< code > relfreq_euclidean_dist_matrix()</ code ></ td >
< td >< code > √partial_relfreq_euclidean[i,j]</ code ></ td >
</ tr >
< tr >
< td >< code > hellinger_dist_matrix()</ code ></ td >
< td >< code > √partial_hellinger[i,j] / √2</ code ></ td >
</ tr >
< tr >
< td >< code > hellinger_euclidean_dist_matrix()</ code ></ td >
< td >< code > √partial_hellinger[i,j]</ code ></ td >
</ tr >
< tr >
< td >< code > threshold_mash_dist_matrix(k, t)</ code ></ td >
< td > Mash distance, derived from < code > threshold_jaccard_dist_matrix(t)</ code > — no separate partial</ td >
</ tr >
</ tbody >
</ table >
< h3 id = "bitpartials" > BitPartials</ h3 >
< p > Required: < code > partial_jaccard() -> (Array2< u64> , Array2< u64> )</ code > , < code > partial_hamming() -> Array2< u64> </ code > . Both additive across layers and partitions.</ p >
< p > Provided finalisations also include < code > jaccard_dist_matrix()</ code > , < code > hamming_dist_matrix()</ code > , and < code > mash_dist_matrix(k)</ code > .</ p >
< h3 id = "mash-distance" > Mash distance</ h3 >
< p >< code > mash_dist_matrix</ code > /< code > threshold_mash_dist_matrix</ code > add no new additive primitive: both are a pointwise transform of the existing Jaccard distance matrix, per the Mash mutation-rate estimator (Fan < em > et al.</ em > 2015; Marbl Lab 2026)< sup id = "fnref:Mash-distances-doc" >< a class = "footnote-ref" href = "#fn:Mash-distances-doc" > 1</ a ></ sup > < sup id = "fnref:Fan2015-mash-formula" >< a class = "footnote-ref" href = "#fn:Fan2015-mash-formula" > 2</ a ></ sup > :</ p >
< div class = "highlight" >< pre >< span ></ span >< code > D = -1/k · ln(2J / (1+J)), J = 1 - d_jaccard
</ code ></ pre ></ div >
< p >< code > J ≤ 0</ code > (i.e. < code > d_jaccard ≥ 1</ code > , no shared k-mers) maps to < code > D = 1</ code > (maximal distance) rather than the < code > ln</ code > singularity at < code > J = 0</ code > .</ p >
< hr />
< h2 id = "temp-file-backed-types" > Temp-file-backed types</ h2 >
< p >< strong > All inter-function results use temp-file-backed types</ strong > so the OS can page them out under memory pressure. This matters in practice: processing dozens of layers × hundreds of partitions in parallel would otherwise accumulate gigabytes of live anonymous memory.</ p >
< h3 id = "lifecycle" > Lifecycle</ h3 >
< div class = "highlight" >< pre >< span ></ span >< code > TempCompactIntVecBuilder::new(n) → writable mmap in TempDir
↓ (inc_present_fast / inc_predicate_fast / add / mask_with / …)
.freeze() → TempCompactIntVec (read-only mmap + TempDir)
↓ (optional)
.make_persistent(path) → PersistentCompactIntVec (permanent file)
</ code ></ pre ></ div >
< p > Same pattern for < code > TempBitVecBuilder</ code > → < code > TempBitVec</ code > → < code > PersistentBitVec</ code > .</ p >
< p >< strong > Drop order</ strong > : < code > TempCompactIntVec { vec: PersistentCompactIntVec, _temp: TempDir }</ code > — Rust drops fields in declaration order. < code > vec</ code > (mmap) released before < code > _temp</ code > (directory deleted). No explicit < code > drop()</ code > needed.</ p >
< h3 id = "tempcompactintvec-tempcompactintvecbuilder" > TempCompactIntVec / TempCompactIntVecBuilder</ h3 >
< div class = "highlight" >< pre >< span ></ span >< code >< span class = "k" > pub</ span >< span class = "w" > </ span >< span class = "k" > struct</ span >< span class = "w" > </ span >< span class = "nc" > TempCompactIntVec</ span >< span class = "w" > </ span >< span class = "p" > {</ span >
< span class = "w" > </ span >< span class = "n" > vec</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "nc" > PersistentCompactIntVec</ span >< span class = "p" > ,</ span >
< span class = "w" > </ span >< span class = "n" > _temp</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "nc" > TempDir</ span >< span class = "p" > ,</ span >< span class = "w" > </ span >< span class = "c1" > // dropped after vec</ span >
< span class = "p" > }</ span >
< span class = "k" > pub</ span >< span class = "p" > (</ span >< span class = "k" > crate</ span >< span class = "p" > )</ span >< span class = "w" > </ span >< span class = "k" > struct</ span >< span class = "w" > </ span >< span class = "nc" > TempCompactIntVecBuilder</ span >< span class = "w" > </ span >< span class = "p" > {</ span >
< span class = "w" > </ span >< span class = "n" > builder</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "nc" > PersistentCompactIntVecBuilder</ span >< span class = "p" > ,</ span >
< span class = "w" > </ span >< span class = "n" > temp</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "nc" > TempDir</ span >< span class = "p" > ,</ span >
< span class = "p" > }</ span >
</ code ></ pre ></ div >
< p >< code > TempCompactIntVec</ code > : read access via < code > get(slot)</ code > , < code > sum()</ code > , < code > iter()</ code > , < code > view() -> IntSliceView< '_> </ code > .</ p >
< p >< code > TempCompactIntVecBuilder</ code > : full delegation to inner < code > PersistentCompactIntVecBuilder</ code > — all bulk computation methods (< code > inc_present_fast</ code > , < code > inc_predicate_fast</ code > , < code > add</ code > , < code > min</ code > , < code > max</ code > , < code > diff</ code > , < code > mask_with</ code > ) are exposed as < code > pub(crate)</ code > .</ p >
< h3 id = "tempbitvec-tempbitvecbuilder" > TempBitVec / TempBitVecBuilder</ h3 >
< div class = "highlight" >< pre >< span ></ span >< code >< span class = "k" > pub</ span >< span class = "w" > </ span >< span class = "k" > struct</ span >< span class = "w" > </ span >< span class = "nc" > TempBitVec</ span >< span class = "w" > </ span >< span class = "p" > {</ span >
< span class = "w" > </ span >< span class = "n" > vec</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "nc" > PersistentBitVec</ span >< span class = "p" > ,</ span >
< span class = "w" > </ span >< span class = "n" > _temp</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "nc" > TempDir</ span >< span class = "p" > ,</ span >
< span class = "p" > }</ span >
< span class = "k" > pub</ span >< span class = "p" > (</ span >< span class = "k" > crate</ span >< span class = "p" > )</ span >< span class = "w" > </ span >< span class = "k" > struct</ span >< span class = "w" > </ span >< span class = "nc" > TempBitVecBuilder</ span >< span class = "w" > </ span >< span class = "p" > {</ span >
< span class = "w" > </ span >< span class = "n" > builder</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "nc" > PersistentBitVecBuilder</ span >< span class = "p" > ,</ span >
< span class = "w" > </ span >< span class = "n" > temp</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "nc" > TempDir</ span >< span class = "p" > ,</ span >
< span class = "p" > }</ span >
</ code ></ pre ></ div >
< p >< code > TempBitVec</ code > : read access via < code > get(slot)</ code > , < code > count_ones()</ code > , < code > view() -> BitSliceView< '_> </ code > , < code > iter()</ code > .</ p >
< p >< code > TempBitVecBuilder</ code > : exposes < code > set(slot, bool)</ code > , < code > or(BitSliceView)</ code > , and:</ p >
< div class = "highlight" >< pre >< span ></ span >< code >< span class = "k" > pub</ span >< span class = "p" > (</ span >< span class = "k" > crate</ span >< span class = "p" > )</ span >< span class = "w" > </ span >< span class = "k" > fn</ span >< span class = "w" > </ span >< span class = "nf" > or_where</ span >< span class = "p" > (</ span >< span class = "o" > & </ span >< span class = "k" > mut</ span >< span class = "w" > </ span >< span class = "bp" > self</ span >< span class = "p" > ,</ span >< span class = "w" > </ span >< span class = "n" > col</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "nc" > IntSliceView</ span >< span class = "o" > < '</ span >< span class = "nb" > _</ span >< span class = "o" > > </ span >< span class = "p" > ,</ span >< span class = "w" > </ span >< span class = "n" > pred</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "nc" > impl</ span >< span class = "w" > </ span >< span class = "nb" > Fn</ span >< span class = "p" > (</ span >< span class = "kt" > u32</ span >< span class = "p" > )</ span >< span class = "w" > </ span >< span class = "p" > -> </ span >< span class = "w" > </ span >< span class = "kt" > bool</ span >< span class = "p" > )</ span >
</ code ></ pre ></ div >
< p >< code > or_where</ code > — two passes, no intermediate allocation:</ p >
< div class = "highlight" >< pre >< span ></ span >< code > Pass 1 — primary bytes, O(n):
for slot in 0..n:
b = col.primary_bytes()[slot]
if b < 255 AND pred(b as u32): self.set(slot, true)
Pass 2 — overflow, O(k):
for (slot, val) in col.overflow_entries():
if pred(val): self.set(slot, true)
</ code ></ pre ></ div >
< hr />
< h2 id = "filter-select-api" > Filter / Select API</ h2 >
< h3 id = "colgroup" > ColGroup</ h3 >
< div class = "highlight" >< pre >< span ></ span >< code >< span class = "k" > pub</ span >< span class = "w" > </ span >< span class = "k" > struct</ span >< span class = "w" > </ span >< span class = "nc" > ColGroup</ span >< span class = "w" > </ span >< span class = "p" > {</ span >< span class = "w" > </ span >< span class = "k" > pub</ span >< span class = "w" > </ span >< span class = "n" > name</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "nb" > String</ span >< span class = "p" > ,</ span >< span class = "w" > </ span >< span class = "k" > pub</ span >< span class = "w" > </ span >< span class = "n" > indices</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "nb" > Vec</ span >< span class = "o" > < </ span >< span class = "kt" > usize</ span >< span class = "o" > > </ span >< span class = "w" > </ span >< span class = "p" > }</ span >
</ code ></ pre ></ div >
< p > Defined < strong > once at the index level</ strong > from column metadata. Valid in all matrices of all layers and partitions — column structure is identical across the entire hierarchy; only rows (kmer slots) are partitioned.</ p >
< h3 id = "composition-axis" > Composition axis</ h3 >
< ul >
< li >< strong > Across partitions</ strong > : kmer space is partitioned → partial results < strong > concatenated</ strong > (disjoint kmer ranges).</ li >
< li >< strong > Across layers</ strong > : same kmer space, different counts → partial results < strong > aggregated</ strong > (add, OR, etc.).</ li >
</ ul >
< h3 id = "matrixgroupops" > MatrixGroupOps</ h3 >
< p > Five required primitives + two default methods derived from them. All return temp-file-backed types.</ p >
< div class = "highlight" >< pre >< span ></ span >< code >< span class = "k" > pub</ span >< span class = "w" > </ span >< span class = "k" > trait</ span >< span class = "w" > </ span >< span class = "n" > MatrixGroupOps</ span >< span class = "w" > </ span >< span class = "p" > {</ span >
< span class = "w" > </ span >< span class = "c1" > // required</ span >
< span class = "w" > </ span >< span class = "k" > fn</ span >< span class = "w" > </ span >< span class = "nf" > partial_group_presence_count</ span >< span class = "p" > (</ span >< span class = "o" > & </ span >< span class = "bp" > self</ span >< span class = "p" > ,</ span >< span class = "w" > </ span >< span class = "n" > g</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "kp" > & </ span >< span class = "nc" > ColGroup</ span >< span class = "p" > ,</ span >< span class = "w" > </ span >< span class = "n" > threshold</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "kt" > u32</ span >< span class = "p" > )</ span >
< span class = "w" > </ span >< span class = "p" > -> </ span >< span class = "w" > </ span >< span class = "nc" > io</ span >< span class = "p" > ::</ span >< span class = "nb" > Result</ span >< span class = "o" > < </ span >< span class = "n" > TempCompactIntVec</ span >< span class = "o" > > </ span >< span class = "p" > ;</ span >
< span class = "w" > </ span >< span class = "k" > fn</ span >< span class = "w" > </ span >< span class = "nf" > partial_group_sum</ span >< span class = "p" > (</ span >< span class = "o" > & </ span >< span class = "bp" > self</ span >< span class = "p" > ,</ span >< span class = "w" > </ span >< span class = "n" > g</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "kp" > & </ span >< span class = "nc" > ColGroup</ span >< span class = "p" > )</ span >
< span class = "w" > </ span >< span class = "p" > -> </ span >< span class = "w" > </ span >< span class = "nc" > io</ span >< span class = "p" > ::</ span >< span class = "nb" > Result</ span >< span class = "o" > < </ span >< span class = "n" > TempCompactIntVec</ span >< span class = "o" > > </ span >< span class = "p" > ;</ span >
< span class = "w" > </ span >< span class = "k" > fn</ span >< span class = "w" > </ span >< span class = "nf" > partial_group_any</ span >< span class = "p" > (</ span >< span class = "o" > & </ span >< span class = "bp" > self</ span >< span class = "p" > ,</ span >< span class = "w" > </ span >< span class = "n" > g</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "kp" > & </ span >< span class = "nc" > ColGroup</ span >< span class = "p" > ,</ span >< span class = "w" > </ span >< span class = "n" > threshold</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "kt" > u32</ span >< span class = "p" > )</ span >
< span class = "w" > </ span >< span class = "p" > -> </ span >< span class = "w" > </ span >< span class = "nc" > io</ span >< span class = "p" > ::</ span >< span class = "nb" > Result</ span >< span class = "o" > < </ span >< span class = "n" > TempBitVec</ span >< span class = "o" > > </ span >< span class = "p" > ;</ span >
< span class = "w" > </ span >< span class = "k" > fn</ span >< span class = "w" > </ span >< span class = "nf" > partial_group_min</ span >< span class = "p" > (</ span >< span class = "o" > & </ span >< span class = "bp" > self</ span >< span class = "p" > ,</ span >< span class = "w" > </ span >< span class = "n" > g</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "kp" > & </ span >< span class = "nc" > ColGroup</ span >< span class = "p" > )</ span >
< span class = "w" > </ span >< span class = "p" > -> </ span >< span class = "w" > </ span >< span class = "nc" > io</ span >< span class = "p" > ::</ span >< span class = "nb" > Result</ span >< span class = "o" > < </ span >< span class = "n" > TempCompactIntVec</ span >< span class = "o" > > </ span >< span class = "p" > ;</ span >
< span class = "w" > </ span >< span class = "k" > fn</ span >< span class = "w" > </ span >< span class = "nf" > partial_group_max</ span >< span class = "p" > (</ span >< span class = "o" > & </ span >< span class = "bp" > self</ span >< span class = "p" > ,</ span >< span class = "w" > </ span >< span class = "n" > g</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "kp" > & </ span >< span class = "nc" > ColGroup</ span >< span class = "p" > )</ span >
< span class = "w" > </ span >< span class = "p" > -> </ span >< span class = "w" > </ span >< span class = "nc" > io</ span >< span class = "p" > ::</ span >< span class = "nb" > Result</ span >< span class = "o" > < </ span >< span class = "n" > TempCompactIntVec</ span >< span class = "o" > > </ span >< span class = "p" > ;</ span >
< span class = "w" > </ span >< span class = "c1" > // defaults derived from partial_group_presence_count</ span >
< span class = "w" > </ span >< span class = "k" > fn</ span >< span class = "w" > </ span >< span class = "nf" > partial_group_all</ span >< span class = "p" > (</ span >< span class = "o" > & </ span >< span class = "bp" > self</ span >< span class = "p" > ,</ span >< span class = "w" > </ span >< span class = "n" > g</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "kp" > & </ span >< span class = "nc" > ColGroup</ span >< span class = "p" > ,</ span >< span class = "w" > </ span >< span class = "n" > threshold</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "kt" > u32</ span >< span class = "p" > )</ span >
< span class = "w" > </ span >< span class = "p" > -> </ span >< span class = "w" > </ span >< span class = "nc" > io</ span >< span class = "p" > ::</ span >< span class = "nb" > Result</ span >< span class = "o" > < </ span >< span class = "n" > TempBitVec</ span >< span class = "o" > > </ span >< span class = "p" > ;</ span >< span class = "w" > </ span >< span class = "c1" > // slot=1 iff count == g.indices.len()</ span >
< span class = "w" > </ span >< span class = "k" > fn</ span >< span class = "w" > </ span >< span class = "nf" > partial_group_none</ span >< span class = "p" > (</ span >< span class = "o" > & </ span >< span class = "bp" > self</ span >< span class = "p" > ,</ span >< span class = "w" > </ span >< span class = "n" > g</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "kp" > & </ span >< span class = "nc" > ColGroup</ span >< span class = "p" > ,</ span >< span class = "w" > </ span >< span class = "n" > threshold</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "kt" > u32</ span >< span class = "p" > )</ span >
< span class = "w" > </ span >< span class = "p" > -> </ span >< span class = "w" > </ span >< span class = "nc" > io</ span >< span class = "p" > ::</ span >< span class = "nb" > Result</ span >< span class = "o" > < </ span >< span class = "n" > TempBitVec</ span >< span class = "o" > > </ span >< span class = "p" > ;</ span >< span class = "w" > </ span >< span class = "c1" > // slot=1 iff count == 0</ span >
< span class = "p" > }</ span >
</ code ></ pre ></ div >
< p > Implemented for both < code > PersistentCompactIntMatrix</ code > and < code > PersistentBitMatrix</ code > .</ p >
< p > For < strong > bit matrices</ strong > : values are 0/1, so < code > partial_group_sum</ code > = < code > partial_group_presence_count(g, 1)</ code > ; < code > partial_group_min</ code > is AND (set first column then mask-with remaining); < code > partial_group_max</ code > is OR via < code > partial_group_any</ code > + < code > inc_present</ code > .</ p >
< p >< strong >< code > partial_group_presence_count</ code > — chunking for large groups:</ strong ></ p >
< p > When < code > g.indices.len() < 255</ code > : per-slot counts stay within < code > u8</ code > range. Use < code > inc_present_fast</ code > (bit) or < code > inc_predicate_fast(col_view(c), |v| v > = threshold)</ code > (int) — raw u8 increment, no overflow entry written.</ p >
< p > When < code > g.indices.len() ≥ 255</ code > : process in chunks of 254 columns, accumulate via < code > .add(chunk_frozen.view())</ code > .</ p >
< p >< strong >< code > partial_group_min</ code > (int matrix)</ strong > : copy first column via < code > .add(col_view(first))</ code > (start from 0 ⇒ copy), then < code > .min(col_view(c))</ code > for remaining.</ p >
< p >< strong >< code > partial_group_max</ code > (int matrix)</ strong > : < code > .max(col_view(c))</ code > for all columns (start from 0 ⇒ first column acts as copy).</ p >
< p >< strong >< code > partial_group_any</ code ></ strong > uses < code > or_where</ code > on < code > TempBitVecBuilder</ code > (two-pass: primary bytes then overflow entries).</ p >
< p >< strong >< code > partial_group_all</ code > / < code > partial_group_none</ code ></ strong > (default): call < code > partial_group_presence_count</ code > , then iterate slots to produce the bit result. O(n) extra pass, not chunked.</ p >
< h3 id = "add_col_from-matrix-builder-integration" > add_col_from — matrix builder integration</ h3 >
< p > Both matrix builders accept temp-file results directly:</ p >
< div class = "highlight" >< pre >< span ></ span >< code >< span class = "c1" > // PersistentBitMatrixBuilder</ span >
< span class = "k" > fn</ span >< span class = "w" > </ span >< span class = "nf" > add_col_from</ span >< span class = "p" > (</ span >< span class = "o" > & </ span >< span class = "k" > mut</ span >< span class = "w" > </ span >< span class = "bp" > self</ span >< span class = "p" > ,</ span >< span class = "w" > </ span >< span class = "n" > src</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "kp" > & </ span >< span class = "nc" > TempBitVec</ span >< span class = "p" > )</ span >< span class = "w" > </ span >< span class = "p" > -> </ span >< span class = "w" > </ span >< span class = "nc" > io</ span >< span class = "p" > ::</ span >< span class = "nb" > Result</ span >< span class = "o" > < </ span >< span class = "p" > ()</ span >< span class = "o" > > </ span >
< span class = "k" > fn</ span >< span class = "w" > </ span >< span class = "nf" > add_col_from_int</ span >< span class = "p" > (</ span >< span class = "o" > & </ span >< span class = "k" > mut</ span >< span class = "w" > </ span >< span class = "bp" > self</ span >< span class = "p" > ,</ span >< span class = "w" > </ span >< span class = "n" > src</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "kp" > & </ span >< span class = "nc" > TempCompactIntVec</ span >< span class = "p" > )</ span >< span class = "w" > </ span >< span class = "p" > -> </ span >< span class = "w" > </ span >< span class = "nc" > io</ span >< span class = "p" > ::</ span >< span class = "nb" > Result</ span >< span class = "o" > < </ span >< span class = "p" > ()</ span >< span class = "o" > > </ span >< span class = "w" > </ span >< span class = "c1" > // nonzero → 1</ span >
< span class = "c1" > // PersistentCompactIntMatrixBuilder</ span >
< span class = "k" > fn</ span >< span class = "w" > </ span >< span class = "nf" > add_col_from</ span >< span class = "p" > (</ span >< span class = "o" > & </ span >< span class = "k" > mut</ span >< span class = "w" > </ span >< span class = "bp" > self</ span >< span class = "p" > ,</ span >< span class = "w" > </ span >< span class = "n" > src</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "kp" > & </ span >< span class = "nc" > TempCompactIntVec</ span >< span class = "p" > )</ span >< span class = "w" > </ span >< span class = "p" > -> </ span >< span class = "w" > </ span >< span class = "nc" > io</ span >< span class = "p" > ::</ span >< span class = "nb" > Result</ span >< span class = "o" > < </ span >< span class = "p" > ()</ span >< span class = "o" > > </ span >
< span class = "k" > fn</ span >< span class = "w" > </ span >< span class = "nf" > add_col_from_bit</ span >< span class = "p" > (</ span >< span class = "o" > & </ span >< span class = "k" > mut</ span >< span class = "w" > </ span >< span class = "bp" > self</ span >< span class = "p" > ,</ span >< span class = "w" > </ span >< span class = "n" > src</ span >< span class = "p" > :</ span >< span class = "w" > </ span >< span class = "kp" > & </ span >< span class = "nc" > TempBitVec</ span >< span class = "p" > )</ span >< span class = "w" > </ span >< span class = "p" > -> </ span >< span class = "w" > </ span >< span class = "nc" > io</ span >< span class = "p" > ::</ span >< span class = "nb" > Result</ span >< span class = "o" > < </ span >< span class = "p" > ()</ span >< span class = "o" > > </ span >< span class = "w" > </ span >< span class = "c1" > // bit → 0/1 u32</ span >
</ code ></ pre ></ div >
< p >< code > add_col_from</ code > copies the temp file to the matrix directory and increments < code > n_cols</ code > ; < code > close()</ code > writes < code > meta.json</ code > with the final column count. No separate < code > write_meta</ code > step needed.</ p >
< h3 id = "mask_with" > mask_with</ h3 >
< p > Direct method on < code > PersistentCompactIntVecBuilder</ code > (and delegation via < code > TempCompactIntVecBuilder</ code > ). Zeros every slot where the corresponding mask bit is 0. Iterates only zero bits — O(n_zeros), O(1) when mask is all-ones.</ p >
< div class = "highlight" >< pre >< span ></ span >< code > for (w_idx, word) in mask.words():
if word == u64::MAX: continue // skip all-ones words
zeros = !word
while zeros != 0:
bit = trailing_zeros(zeros)
s = w_idx * 64 + bit
if primary[s] != 0: set(s, 0) // clears overflow entry too
zeros & = zeros − 1
</ code ></ pre ></ div >
< p > Terminal operation for Filter (retain only selected kmer slots in a count vector) and Select (positional selection without MPHF).</ p >
< div class = "footnote" >
< hr />
< ol >
< li id = "fn:Mash-distances-doc" >
< p > Marbl Lab. (2026). < a href = "https://mash.readthedocs.io/en/latest/distances.html" > Mash distance</ a > . < a class = "footnote-backref" href = "#fnref:Mash-distances-doc" title = "Jump back to footnote 1 in the text" > ↩</ a ></ p >
</ li >
< li id = "fn:Fan2015-mash-formula" >
< p > Fan, H., Ives, A.R., Surget-Groba, Y. & Cannon, C.H. (2015). < a href = "https://doi.org/10.1186/s12864-015-1647-5" > An assembly and alignment-free method of phylogeny reconstruction from next-generation sequencing data</ a > . < em > BMC Genomics</ em > , 16. < a class = "footnote-backref" href = "#fnref:Fan2015-mash-formula" title = "Jump back to footnote 2 in the text" > ↩</ a ></ p >
</ li >
</ ol >
</ div >
</ article >
</ div >
< script > var target = document . getElementById ( location . hash . slice ( 1 )); target && target . name && ( target . checked = target . name . startsWith ( "__tabbed_" ))</ script >
</ div >
</ main >
< footer class = "md-footer" >
< div class = "md-footer-meta md-typeset" >
< div class = "md-footer-meta__inner md-grid" >
< div class = "md-copyright" >
Made with
< a href = "https://squidfunk.github.io/mkdocs-material/" rel = "noopener" target = "_blank" >
Material for MkDocs
</ a >
</ div >
</ div >
</ div >
</ footer >
</ div >
< div class = "md-dialog" data-md-component = "dialog" >
< div class = "md-dialog__inner md-typeset" ></ div >
</ div >
< script id = "__config" type = "application/json" >{ "annotate" : null , "base" : "../.." , "features" : [], "search" : "../../assets/javascripts/workers/search.2c215733.min.js" , "tags" : null , "translations" : { "clipboard.copied" : "Copied to clipboard" , "clipboard.copy" : "Copy to clipboard" , "search.result.more.one" : "1 more on this page" , "search.result.more.other" : "# more on this page" , "search.result.none" : "No matching documents" , "search.result.one" : "1 matching document" , "search.result.other" : "# matching documents" , "search.result.placeholder" : "Type to start searching" , "search.result.term.missing" : "Missing" , "select.version" : "Select version" }, "version" : null }</ script >
< script src = "../../assets/javascripts/bundle.79ae519e.min.js" ></ script >
< script src = "https://unpkg.com/mathjax@3/es5/tex-mml-chtml.js" ></ script >
</ body >
</ html >
