2365 lines
81 KiB
HTML
2365 lines
81 KiB
HTML
|
||
<!doctype html>
|
||
<html lang="en" class="no-js">
|
||
<head>
|
||
|
||
<meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||
|
||
|
||
|
||
|
||
<link rel="prev" href="../index_architecture/">
|
||
|
||
|
||
<link rel="next" href="../numa_worker_pools/">
|
||
|
||
|
||
|
||
|
||
|
||
<link rel="icon" href="../../assets/images/favicon.png">
|
||
<meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.7.6">
|
||
|
||
|
||
|
||
<title>Sibling annex (discussion) - obikmer</title>
|
||
|
||
|
||
|
||
<link rel="stylesheet" href="../../assets/stylesheets/main.484c7ddc.min.css">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
||
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
|
||
<style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
|
||
|
||
|
||
|
||
<script>__md_scope=new URL("../..",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
|
||
|
||
|
||
|
||
|
||
|
||
</head>
|
||
|
||
|
||
<body dir="ltr">
|
||
|
||
|
||
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
||
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
||
<label class="md-overlay" for="__drawer"></label>
|
||
<div data-md-component="skip">
|
||
|
||
|
||
<a href="#sibling-annex-architecture-discussion" class="md-skip">
|
||
Skip to content
|
||
</a>
|
||
|
||
</div>
|
||
<div data-md-component="announce">
|
||
|
||
</div>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<header class="md-header md-header--shadow" data-md-component="header">
|
||
<nav class="md-header__inner md-grid" aria-label="Header">
|
||
<a href="../.." title="obikmer" class="md-header__button md-logo" aria-label="obikmer" data-md-component="logo">
|
||
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
|
||
|
||
</a>
|
||
<label class="md-header__button md-icon" for="__drawer">
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
|
||
</label>
|
||
<div class="md-header__title" data-md-component="header-title">
|
||
<div class="md-header__ellipsis">
|
||
<div class="md-header__topic">
|
||
<span class="md-ellipsis">
|
||
obikmer
|
||
</span>
|
||
</div>
|
||
<div class="md-header__topic" data-md-component="header-topic">
|
||
<span class="md-ellipsis">
|
||
|
||
Sibling annex (discussion)
|
||
|
||
</span>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
<script>var palette=__md_get("__palette");if(palette&&palette.color){if("(prefers-color-scheme)"===palette.color.media){var media=matchMedia("(prefers-color-scheme: light)"),input=document.querySelector(media.matches?"[data-md-color-media='(prefers-color-scheme: light)']":"[data-md-color-media='(prefers-color-scheme: dark)']");palette.color.media=input.getAttribute("data-md-color-media"),palette.color.scheme=input.getAttribute("data-md-color-scheme"),palette.color.primary=input.getAttribute("data-md-color-primary"),palette.color.accent=input.getAttribute("data-md-color-accent")}for(var[key,value]of Object.entries(palette.color))document.body.setAttribute("data-md-color-"+key,value)}</script>
|
||
|
||
|
||
|
||
|
||
</nav>
|
||
|
||
</header>
|
||
|
||
<div class="md-container" data-md-component="container">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<main class="md-main" data-md-component="main">
|
||
<div class="md-main__inner md-grid">
|
||
|
||
|
||
|
||
<div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
|
||
<div class="md-sidebar__scrollwrap">
|
||
<div class="md-sidebar__inner">
|
||
|
||
|
||
|
||
|
||
<nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
|
||
<label class="md-nav__title" for="__drawer">
|
||
<a href="../.." title="obikmer" class="md-nav__button md-logo" aria-label="obikmer" data-md-component="logo">
|
||
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
|
||
|
||
</a>
|
||
obikmer
|
||
</label>
|
||
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../.." class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Home
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../installation/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Installation
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_3" id="__nav_3_label" tabindex="0">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Theory
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_3_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_3">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Theory
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../kmers/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Kmers and super-kmers
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../theory/encoding/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
DNA encoding
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../theory/entropy/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Entropy filter
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../theory/minimizer/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Minimizer selection
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../theory/indexing/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Partitioning architecture
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../theory/evolutionary_distances/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Central-position SNP distance (discussion)
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_4" id="__nav_4_label" tabindex="0">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Implementation
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_4_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_4">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Implementation
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../implementation/superkmer/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
SuperKmer
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../implementation/kmer/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Kmer
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../implementation/chunkreader/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Chunk reader
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../implementation/pipeline/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Construction pipeline
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../implementation/obipipeline/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
obipipeline library
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../implementation/storage/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
On-disk storage
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../implementation/mphf/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
MPHF selection
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../implementation/unitig_evidence/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Unitig evidence encoding
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../implementation/evidence_elimination/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Evidence elimination (discussion)
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../implementation/obilayeredmap/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
obilayeredmap crate
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../implementation/persistent_compact_int_vec/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
PersistentCompactIntVec
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../implementation/persistent_bit_vec/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
PersistentBitVec
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../implementation/merge/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Merge command
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../implementation/merge_parallelism/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Merge parallelism & memory
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../implementation/filtering/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Kmer filtering
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../implementation/select/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Select command
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../implementation/obitaxonomy/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
obitaxonomy crate
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--active md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_5" checked>
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_5" id="__nav_5_label" tabindex="0">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Architecture
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_5_label" aria-expanded="true">
|
||
<label class="md-nav__title" for="__nav_5">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Architecture
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../sequences/invariant/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Sequences
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../index_architecture/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Kmer index
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--active">
|
||
|
||
<input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
|
||
|
||
|
||
|
||
|
||
|
||
<label class="md-nav__link md-nav__link--active" for="__toc">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Sibling annex (discussion)
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<a href="./" class="md-nav__link md-nav__link--active">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Sibling annex (discussion)
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
|
||
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<label class="md-nav__title" for="__toc">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
Table of contents
|
||
</label>
|
||
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#two-index-spaces-uncorrelated" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Two index spaces, uncorrelated
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#two-pipelines-never-mixed" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Two pipelines, never mixed
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#sibling-annex-an-iteration-pipeline-artifact-only" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Sibling annex: an iteration-pipeline artifact only
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#pending-work-done" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Pending work — done
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#performance-build_sibling_annex-parallelism-2026-08-14" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Performance: build_sibling_annex parallelism (2026-08-14)
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#cross-partition-batch-resolution-current-state-vs-the-batched-accumulator-design-discussion-2026-08-14" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Cross-partition batch resolution — current state vs. the batched-accumulator design (discussion, 2026-08-14)
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#pseudo-alignment-at-scale-pruning-is-unavoidable-discussion-2026-08-1415" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Pseudo-alignment at scale — pruning is unavoidable (discussion, 2026-08-14/15)
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#-subsample-shannon-sampling-strategy-decided-2026-08-15" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
--subsample / --shannon — sampling strategy (decided 2026-08-15)
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#cheap-entropy-pre-filtering-row-marginal-sums-idea-not-implemented-2026-08-15" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Cheap entropy pre-filtering — row-marginal sums (idea, not implemented, 2026-08-15)
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#-free-loss-tnt-pipeline-four-independent-scans-three-of-them-unsampled-found-2026-08-15-fixed-2026-08-15-see-implemented-below" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
--free-loss/--tnt pipeline: four independent scans, three of them unsampled (found 2026-08-15, fixed 2026-08-15 — see "Implemented" below)
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#entropy-biased-selection-soft-gaussian-weighting-not-a-hard-cutoff-decided-and-implemented-2026-08-15" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Entropy-biased selection: soft Gaussian weighting, not a hard cutoff (decided and implemented 2026-08-15)
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#two-entropy-definitions-kept-side-by-side-for-comparison-2026-08-15" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Two entropy definitions kept side by side, for comparison (2026-08-15)
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#persistentsparsebitmatrix-implemented-and-measured-2026-08-15" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
PersistentSparseBitMatrix — implemented and measured (2026-08-15)
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#wired-into-pack-and-the-sibling-annex-build-path-2026-08-15" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Wired into pack and the sibling-annex build path (2026-08-15)
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#sankoff-pipeline-fusion-entropy-biased-selection-implemented-2026-08-15" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Sankoff pipeline fusion + entropy-biased selection — implemented (2026-08-15)
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../numa_worker_pools/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
NUMA-aware worker pools
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../numa_partition_runner/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
NUMA-aware partition runner
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
|
||
<div class="md-sidebar__scrollwrap">
|
||
<div class="md-sidebar__inner">
|
||
|
||
|
||
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<label class="md-nav__title" for="__toc">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
Table of contents
|
||
</label>
|
||
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#two-index-spaces-uncorrelated" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Two index spaces, uncorrelated
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#two-pipelines-never-mixed" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Two pipelines, never mixed
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#sibling-annex-an-iteration-pipeline-artifact-only" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Sibling annex: an iteration-pipeline artifact only
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#pending-work-done" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Pending work — done
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#performance-build_sibling_annex-parallelism-2026-08-14" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Performance: build_sibling_annex parallelism (2026-08-14)
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#cross-partition-batch-resolution-current-state-vs-the-batched-accumulator-design-discussion-2026-08-14" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Cross-partition batch resolution — current state vs. the batched-accumulator design (discussion, 2026-08-14)
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#pseudo-alignment-at-scale-pruning-is-unavoidable-discussion-2026-08-1415" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Pseudo-alignment at scale — pruning is unavoidable (discussion, 2026-08-14/15)
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#-subsample-shannon-sampling-strategy-decided-2026-08-15" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
--subsample / --shannon — sampling strategy (decided 2026-08-15)
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#cheap-entropy-pre-filtering-row-marginal-sums-idea-not-implemented-2026-08-15" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Cheap entropy pre-filtering — row-marginal sums (idea, not implemented, 2026-08-15)
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#-free-loss-tnt-pipeline-four-independent-scans-three-of-them-unsampled-found-2026-08-15-fixed-2026-08-15-see-implemented-below" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
--free-loss/--tnt pipeline: four independent scans, three of them unsampled (found 2026-08-15, fixed 2026-08-15 — see "Implemented" below)
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#entropy-biased-selection-soft-gaussian-weighting-not-a-hard-cutoff-decided-and-implemented-2026-08-15" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Entropy-biased selection: soft Gaussian weighting, not a hard cutoff (decided and implemented 2026-08-15)
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#two-entropy-definitions-kept-side-by-side-for-comparison-2026-08-15" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Two entropy definitions kept side by side, for comparison (2026-08-15)
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#persistentsparsebitmatrix-implemented-and-measured-2026-08-15" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
PersistentSparseBitMatrix — implemented and measured (2026-08-15)
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#wired-into-pack-and-the-sibling-annex-build-path-2026-08-15" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Wired into pack and the sibling-annex build path (2026-08-15)
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#sankoff-pipeline-fusion-entropy-biased-selection-implemented-2026-08-15" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Sankoff pipeline fusion + entropy-biased selection — implemented (2026-08-15)
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div class="md-content" data-md-component="content">
|
||
|
||
<article class="md-content__inner md-typeset">
|
||
|
||
|
||
|
||
|
||
|
||
<h1 id="sibling-annex-architecture-discussion">Sibling annex — architecture (discussion)</h1>
|
||
<p>Status: architecture decided (2026-08-14). Implementation not yet mandated.</p>
|
||
<h2 id="two-index-spaces-uncorrelated">Two index spaces, uncorrelated</h2>
|
||
<p>Every kmer stored in a <code>Layer</code> lives in two independent index spaces:</p>
|
||
<ul>
|
||
<li><strong>Iteration order</strong>: its position when enumerating <code>unitigs.bin</code> (the
|
||
superkmer file), deterministic but arbitrary with respect to slot.</li>
|
||
<li><strong>MPHF slot</strong>: <code>MphfLayer::index(kmer)</code>, the number the MPHF assigns.</li>
|
||
</ul>
|
||
<p>The two are not correlated by any formula. Converting from one to the other
|
||
requires either recomputing the MPHF (kmer → slot) or scanning the iteration
|
||
stream (kmer → order). There is no <code>slot → kmer</code> operation: the MPHF is a
|
||
one-way function, not an invertible bijection with a stored inverse. Any
|
||
method that reconstructs a kmer from a bare slot number is wrong by
|
||
construction, regardless of the mechanism used (MPHF re-hash, or evidence
|
||
decode + direct unitig read). See <code>MphfLayer::kmer_at</code>
|
||
(<code>obilayeredmap/src/mphf_layer.rs</code>) — flagged for removal, currently called
|
||
from <code>obikphylo/siblings/build.rs</code> and <code>family_scan.rs</code> (since removed — see
|
||
"Pending work" status below).</p>
|
||
<h2 id="two-pipelines-never-mixed">Two pipelines, never mixed</h2>
|
||
<table>
|
||
<thead>
|
||
<tr>
|
||
<th></th>
|
||
<th>origin of the kmer</th>
|
||
<th>membership known?</th>
|
||
<th>correct mapping</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr>
|
||
<td><strong>query pipeline</strong></td>
|
||
<td>external (caller-supplied)</td>
|
||
<td>no</td>
|
||
<td><code>query</code>/<code>find</code>/<code>find_strict</code> — MPHF + evidence check</td>
|
||
</tr>
|
||
<tr>
|
||
<td><strong>iteration pipeline</strong></td>
|
||
<td>enumerated from this layer's own <code>unitigs.bin</code></td>
|
||
<td>yes, by construction</td>
|
||
<td><code>index</code>/<code>index_batch</code> — MPHF only, no evidence</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
<p>Evidence exists solely to answer "is this external kmer a member of the
|
||
layer" for the query pipeline. Using it (or the MPHF) to go the other way —
|
||
recover a kmer from a slot, or re-verify a kmer that was just produced by
|
||
iterating the layer — is a conceptual error: evidence can be probabilistic
|
||
(<code>Approx</code> mode), so any slot→kmer attempt is unsound in general, and
|
||
pointless even in <code>Exact</code>/<code>Hybrid</code> mode since the kmer was already known.</p>
|
||
<h2 id="sibling-annex-an-iteration-pipeline-artifact-only">Sibling annex: an iteration-pipeline artifact only</h2>
|
||
<p>The sibling annex (<code>FamilyMask</code>/<code>SiblingAnnex</code>, <code>.psib</code>,
|
||
<code>obicompactvec/src/siblingannex.rs</code>) records, per kmer, whether it is a
|
||
family minorant and which family members are present in the index. Its only
|
||
consumers (<code>obikphylo/siblings/stats.rs</code>, <code>family_scan.rs</code>) enumerate it
|
||
exhaustively (<code>0..annex.len()</code>); no query-pipeline code path touches it.</p>
|
||
<p><strong>Decision</strong>: the annex must be persisted in iteration order, not slot
|
||
order. This lets readers zip-iterate <code>Layer::iter_kmers()</code> and the annex
|
||
file directly — one linear, cache-friendly pass, no MPHF/slot indirection,
|
||
no <code>kmer_at</code>. It also enables specialized iterators building on this zip:
|
||
minorants-only iteration, batch-of-kmers → batch-of-family-members, etc.</p>
|
||
<p>Today the annex is built and stored in <strong>slot</strong> order
|
||
(<code>build_layer_sibling_annex</code>, <code>siblings/build.rs</code>): <code>slot_kmer</code> is populated
|
||
via <code>(0..n_slots).map(|slot| mphf.kmer_at(slot))</code>, and the origin <code>slot</code> is
|
||
threaded through the whole cross-partition reconciliation pipeline (variant
|
||
generation, <code>query_partition_with</code>, final <code>mask[slot].fetch_or(...)</code>). This
|
||
must change to iterating <code>iter_kmers()</code>/<code>enumerate_kmers()</code> and threading
|
||
the <strong>iteration index</strong> instead of the slot end to end — eliminating
|
||
<code>kmer_at</code> from the build path entirely, not just the read path. No
|
||
slot-indexed intermediate is needed even during construction; the
|
||
iteration-order id is sufficient throughout.</p>
|
||
<p>The cross-partition side of the same pipeline is unaffected: checking
|
||
whether a generated family-variant kmer exists in another partition is a
|
||
genuine query-pipeline operation (the variant's membership in the <em>target</em>
|
||
partition is unknown) and must keep going through
|
||
<code>KmerPartition::query_partition_with</code> (MPHF + evidence), never a raw
|
||
<code>index()</code>.</p>
|
||
<h2 id="pending-work-done">Pending work — done</h2>
|
||
<p>The plan above shipped: <code>obikphylo</code> (a new crate — phylo-domain extension
|
||
traits over <code>obikindex::KmerIndex</code>/<code>obilayeredmap::Layer<D></code>, replacing the
|
||
old <code>obikindex::siblings</code> module) builds and reads the annex purely in
|
||
iteration order (<code>SiblingLayerExt::iter_siblings</code>/<code>iter_minorants</code>, both with
|
||
batch variants, mirroring <code>Layer<D></code>'s own <code>KmerIter</code>/<code>KmerBatchIter</code>
|
||
shape). <code>MphfLayer::kmer_at</code> has no remaining callers.</p>
|
||
<p>A separate, unrelated bug surfaced during this work and was fixed
|
||
(2026-08-14): <code>MphfLayer::enumerate_kmers_batch</code> computed its
|
||
<code>batch_start_index</code> via the stdlib <code>.enumerate()</code> adapter, which counts
|
||
<em>batches</em> (0, 1, 2…), not the cumulative k-mer offset the annex is actually
|
||
keyed on — every batch past the first wrote its mask/annex entries at the
|
||
wrong iteration-order position. Fixed by tracking a running offset instead;
|
||
regression tests added (<code>sibling_annex_no_empty_masks_after_build</code>,
|
||
<code>sibling_histogram_does_not_panic_on_partial_last_batch</code>).</p>
|
||
<h2 id="performance-build_sibling_annex-parallelism-2026-08-14">Performance: <code>build_sibling_annex</code> parallelism (2026-08-14)</h2>
|
||
<p>Investigated on a real multi-genome run (<code>phyloskims_sal_vac</code>, k=31/m=11).
|
||
Baseline: mostly one active core, with short multi-core bursts — average
|
||
~3 cores.</p>
|
||
<p><strong>Fixes that helped, kept:</strong></p>
|
||
<ul>
|
||
<li><code>CanonicalKmerOf::minimizer()</code> (<code>obikseq/src/kmer.rs</code>) — a direct O(k)
|
||
bit-arithmetic minimiser for a single isolated k-mer, replacing a
|
||
<code>RollingStat</code> instance fed byte-by-byte through an ASCII round-trip (used
|
||
by <code>helpers::partition_of</code>, called for every generated family variant).
|
||
~3x wall-clock improvement on its own, confirmed by sampling
|
||
(<code>obiskbuilder::rolling_stat</code>/<code>obikentropy</code> frames disappeared from the
|
||
hot path). <code>CanonicalKmerOf::partition()</code> added alongside it (wraps
|
||
<code>minimizer().seq_hash() & mask</code>, the same routing rule
|
||
<code>KmerPartition</code>/<code>RoutableSuperKmer</code> use).</li>
|
||
<li>Cross-partition resolution (<code>outgoing.par_iter()</code> in
|
||
<code>build_layer_sibling_annex</code>) parallelised at the <em>partition</em> level — one
|
||
Rayon task per non-empty <code>outgoing[dest]</code> bucket. For k=31/m=11, a
|
||
central-base substitution changes the winning minimiser (and thus the
|
||
destination partition) only when that window overlaps the central base:
|
||
~11 of the 21 possible windows do, so ~10/21 (≈48%) of generated variants
|
||
route right back to the partition already being built. That self bucket
|
||
ends up far larger than any other, so the per-partition split pinned one
|
||
thread to it alone while the rest of the pool finished instantly —
|
||
confirmed by sampling: one thread solid in <code>MphfLayer::find</code>, everyone
|
||
else idle. Fixed by splitting each non-empty bucket into
|
||
<code>total_queries / n_workers</code> (capped 4096) chunks <em>before</em> <code>par_iter()</code>,
|
||
preserving per-partition mmap locality (each chunk stays contiguous
|
||
within one partition) while letting Rayon spread an oversized bucket
|
||
across several threads. Net effect of both fixes together: ~3 cores
|
||
average → ~10-13 cores average on the same run, and a projected total
|
||
build time of ~1h15 down to ~30min on the real <code>phyloskims_sal_vac</code> run
|
||
this was measured against.</li>
|
||
<li><code>TracedBar</code>'s ETA (<code>obisys/src/progress.rs</code>) was silently starved: the
|
||
custom progress message and the self-computed ETA text used to share one
|
||
<code>pb.set_message()</code> slot, with the ETA holding off for 2s after any custom
|
||
message — fine when custom messages are rare, broken once
|
||
<code>build_sibling_annex</code>'s per-partition callback fires more often than
|
||
that. Fixed by keeping the two texts in separate fields, composed
|
||
together on every render instead of one overwriting the other.</li>
|
||
</ul>
|
||
<p><strong>Tried and reverted — do not repeat blindly:</strong></p>
|
||
<ul>
|
||
<li>Parallelising the <em>outer</em> partition loop in <code>build_sibling_annex</code> with
|
||
<code>obikindex::PartitionRunner</code> (already used by <code>merge</code>/<code>build_layers</code>),
|
||
splitting a fixed core budget between outer (partition) and inner
|
||
(pipeline + resolution) concurrency so their product wouldn't exceed the
|
||
budget. Measured <em>worse</em>: throughput dropped over time (26
|
||
partitions/5min → 38/11-12min) and peak resolution concurrency fell from
|
||
~11-12 cores to ~7-8. Cause: this capped the resolution burst — which
|
||
scales very well on its own — to make room for outer concurrency, and
|
||
running several partitions' resolution at once scatters access across
|
||
multiple partitions' mmap regions at once, working against the
|
||
locality <code>outgoing</code>'s per-partition grouping exists for. <code>PartitionRunner</code>
|
||
stayed exported from <code>obikindex</code> (<code>new_capped</code> too) since it's
|
||
general-purpose, but nothing in <code>obikphylo</code> calls it.</li>
|
||
<li>Splitting resolution chunks even finer (<code>/(n_workers*8)</code>, cap 1024,
|
||
instead of <code>/n_workers</code>, cap 4096) to smooth the residual sawtooth.
|
||
Measured ~10% <em>slower</em>, wider dips, not narrower. Reverted to the
|
||
original chunk sizing.</li>
|
||
</ul>
|
||
<p><strong>Known remaining limitation, not yet worth fixing:</strong> within one layer, the
|
||
four stages (sequential <code>unitigs.bin</code> read → parallel generation →
|
||
parallel resolution → sequential annex write) never overlap — confirmed by
|
||
1s-interval sampling: generation alone occupies ~17 threads evenly, but the
|
||
next layer's read/generation never starts until the current layer's
|
||
resolution and write are both done. This produces a real, periodic (~layer
|
||
duration) alternation between "many cores" and "few cores" that neither of
|
||
the fixes above touches, since both operate <em>within</em> one layer's resolution
|
||
step. The only remaining lever is overlapping consecutive layers (e.g. a
|
||
depth-2 pipeline: start layer N+1's read/generation while layer N's
|
||
resolution/write is still running) — a real restructuring, not a parameter
|
||
tweak, and explicitly <em>not</em> to be combined with the reverted
|
||
budget-capping idea above (let each phase use however many cores it
|
||
naturally wants; only the <em>scheduling</em> needs to overlap). Deferred, not
|
||
started.</p>
|
||
<h2 id="cross-partition-batch-resolution-current-state-vs-the-batched-accumulator-design-discussion-2026-08-14">Cross-partition batch resolution — current state vs. the batched-accumulator design (discussion, 2026-08-14)</h2>
|
||
<p><code>family_scan.rs::scan_layer_families</code> (shared by <code>snp_pseudo_alignment</code>,
|
||
<code>sibling_annex_stats</code>, <code>cardinality_tally</code>, <code>scan_family_pairs</code>) already
|
||
implements most of a dispatch/accumulate/resolve pipeline: generation
|
||
(cheap, CPU-only — builds <code>outgoing[dest_partition]</code> from <code>FamilyMask</code> and
|
||
buckets cross-partition queries) runs on an <code>obipipeline::throttle</code> +
|
||
<code>make_pipe!</code> stage, decoupled from resolution (I/O-bound, <code>rayon::par_iter</code>
|
||
<em>across partitions</em>, one generated batch resolved at a time, never several
|
||
concurrently — this ordering is deliberate, see the module's own docs on a
|
||
reverted concurrent-batch-resolution attempt that scattered mmap access).
|
||
The fast/slow mode gate (<code>PartitionCache::fast_mode</code>, <code>cache.rs:162-163</code>)
|
||
already exists: <code>n_layers <= 7</code> (checked once from the first non-empty
|
||
partition's <code>PartitionMeta::n_layers</code>, documented as identical across every
|
||
partition of an index — a structural, build-time property, never a
|
||
per-partition state) decides whether <code>FamilyMask</code>'s recorded <code>layer_value</code>
|
||
can be trusted to skip straight to the right layer
|
||
(<code>find_presence_batch_fast</code>) or must fall back to scanning every layer of
|
||
the destination partition (<code>find_presence_batch</code>).</p>
|
||
<p><strong>Real gap, confirmed not implemented</strong>: resolution is triggered by the
|
||
<em>source</em> batch finishing (<code>FAMILY_BATCH = 65536</code> minorants read from the
|
||
scanned layer), not by an <em>output</em> accumulator filling up. Since most
|
||
central-base variants of a family route back to the same partition being
|
||
scanned (~48% per the k=31/m=11 measurement above), a <code>FAMILY_BATCH</code>'s
|
||
<code>outgoing[dest]</code> is large for the local/self partition and thin for the
|
||
other ~255 (or however many) destination partitions — each of those gets
|
||
resolved at low query density every batch instead of being accumulated
|
||
across several source batches until resolving it is worthwhile. This is
|
||
distinct from, and not fixed by, the fast/slow layer gate above.</p>
|
||
<p>Redesign sketched (not built): per-destination accumulators decoupled from
|
||
<code>FAMILY_BATCH</code>, flushed on reaching a size threshold instead of on source-batch
|
||
completion — a "hot" accumulator for the partition being scanned (sharded
|
||
one-per-generation-worker, no lock, since all <code>n_workers</code> pipeline workers
|
||
write to it concurrently — this differs from an earlier, simpler mental
|
||
model of "one thread owns one layer's local collector," which doesn't hold
|
||
here since <code>n_workers</code> threads cooperate on scanning <em>one</em> layer at a time,
|
||
not one thread per layer) and "cold" mutex-per-partition accumulators for
|
||
the rest, low contention expected since traffic to any single cold
|
||
destination is a small fraction of total.</p>
|
||
<p>This breaks the current strict-iteration-order delivery of <code>on_family</code>
|
||
(today: a reorder buffer keyed by batch, since a whole <code>FAMILY_BATCH</code>
|
||
resolves atomically). With cross-batch accumulation, a family only becomes
|
||
complete once <em>every</em> accumulator holding one of its outgoing queries has
|
||
flushed, at unpredictable, independent times — no longer streamable
|
||
strictly in order without a large, unbounded pending buffer. Resolution
|
||
sketched: replace order-dependent consumers with coordinate-addressed
|
||
writes instead of order-dependent appends (see <code>PseudoAlignment</code> idea
|
||
below) wherever possible, since <code>sibling_annex_stats</code>'s reduction (plain
|
||
counts) is already order-independent and needs nothing here.</p>
|
||
<p><strong>Superseded 2026-08-15</strong> by the <code>--subsample</code>/<code>--shannon</code> design below,
|
||
which sidesteps the accumulator redesign for now: bounding the number of
|
||
families actually resolved per layer (via sampling) keeps per-layer
|
||
resolution volume small enough that the batch-density problem above stops
|
||
mattering in practice for these two consumers. The accumulator redesign
|
||
remains relevant for a future <em>unsampled, full-index</em> run, but is not
|
||
required to ship <code>--subsample</code>/<code>--shannon</code>.</p>
|
||
<h2 id="pseudo-alignment-at-scale-pruning-is-unavoidable-discussion-2026-08-1415">Pseudo-alignment at scale — pruning is unavoidable (discussion, 2026-08-14/15)</h2>
|
||
<p>The reference run (<code>phyloskims_sal_vac</code>-scale bacterial test set,
|
||
<code>iqtree.fasta</code>) produced a dense alignment for 13 genomes × 383,965 sites
|
||
(4.8 MB) — trivially small. The in-progress plant index is expected to
|
||
carry on the order of 9 billion minorant families; a dense byte-per-cell
|
||
alignment at that column count is unbuildable regardless of genome count
|
||
(hundreds of GB even at a handful of genomes). Long-term ambition is 6,000–
|
||
8,000 genomes on a large machine, which makes the per-cell cost dominant in
|
||
the other dimension too. Pruning the retained family set before
|
||
materializing anything is mandatory, not an optimization.</p>
|
||
<p><strong>Already free</strong>: <code>family_size() < 2</code> (no sibling variant registered at
|
||
all) is a zero-cost structural filter, read directly off <code>FamilyMask</code> bits,
|
||
already applied in <code>snp_pseudo_alignment</code>. Insufficient alone — per the
|
||
~80-85% mono-family estimate from earlier discussion, this only brings 9
|
||
billion down to roughly 1.3-1.8 billion, still unusable.</p>
|
||
<p><strong>Entropy definition — settled 2026-08-15, correcting an earlier wrong
|
||
turn.</strong> The project does <strong>not</strong> encode families as IUPAC ambiguity codes
|
||
interpreted the classical way (Fitch-parsimony subset-compatibility, or
|
||
ML's "one true state, uncertain which"); see
|
||
<code>docmd/theory/evolutionary_distances.md</code> ("Why the IUPAC/DNA encoding used
|
||
for the first <code>--snp</code> test was wrong") and the Sankoff resolution that
|
||
followed it. The real model is a genuine 16-state alphabet (the powerset of
|
||
<code>{A,C,G,T}</code>, <code>∅</code> included as a real state) scored with a <em>calibrated
|
||
pairwise cost matrix</em> (<code>obikphylo::cardcomp::pairwise_cost_matrix</code>,
|
||
<code>cmd/phylo/sankoff.rs</code>), not a compatibility/subset relation between
|
||
states. Under that model, each of the 16 states — including multi-bit ones
|
||
like <code>AC</code> — is a first-class, independently-costed state, not an
|
||
uncertainty encoding of a single true base. So: <strong>entropy over the 15
|
||
non-empty states (<code>∅</code> excluded, matching the earlier decision to exclude
|
||
genomes where the family is absent) is the correct informativeness
|
||
measure</strong> for this project — not a 4-symbol reduction, which would discard
|
||
exactly the cardinality/composition information the calibrated cost matrix
|
||
is built to exploit.</p>
|
||
<h2 id="-subsample-shannon-sampling-strategy-decided-2026-08-15"><code>--subsample</code> / <code>--shannon</code> — sampling strategy (decided 2026-08-15)</h2>
|
||
<p>Goal: make both the pseudo-alignment (<code>--snp</code>) and a Shannon-entropy
|
||
diagnostic usable at any index scale, from the 13-genome bacterial
|
||
reference run up to the 9-billion-family plant index, without requiring the
|
||
batched-accumulator redesign above.</p>
|
||
<p><strong><code>--subsample N</code></strong> (integer, families to retain): bounds the pseudo-alignment
|
||
to <code>N</code> minorant families, sampled <strong>proportionally per layer</strong> among
|
||
non-monomorphic minorants (<code>family_size >= 2</code>) — this sidesteps the need
|
||
for a true global reservoir merge across layers while still approximating a
|
||
uniform sample over the whole index, and directly answers the earlier open
|
||
question of global-vs-per-layer selection scope.</p>
|
||
<p>Three passes, in order:</p>
|
||
<ol>
|
||
<li><strong>Global count</strong> (cheap, structural, parallel across layers — same shape
|
||
as the existing <code>sibling_family_size_histogram</code>, extended to report a
|
||
<strong>per-layer</strong> breakdown rather than one index-wide aggregate): for each
|
||
layer, <code>count_layer</code> = number of non-monomorphic minorants. Gives
|
||
<code>total_count = Σ count_layer</code>.</li>
|
||
<li><strong>Per-layer proportional reservoir sampling</strong> (cheap, structural, one
|
||
pass per layer, no cross-partition resolution): <code>N_layer = round(N ×
|
||
count_layer / total_count)</code>. Since <code>N_layer</code> is a proportion of
|
||
<code>count_layer</code>, it can never exceed it as long as <code>N <= total_count</code> — the
|
||
one edge case is <code>total_count <= N</code>, in which case sampling is skipped
|
||
entirely and <em>every</em> non-monomorphic minorant of every layer is kept
|
||
(no reservoir needed, <code>N</code> was never a real constraint). Otherwise:
|
||
Algorithm-R reservoir sampling over the layer's non-monomorphic minorant
|
||
indices, producing <code>N_layer</code> iteration-order indices directly, no
|
||
intermediate full list ever materialized.</li>
|
||
<li><strong>Filtered resolution</strong> (the expensive step, the existing
|
||
<code>scan_layer_families</code> engine, unchanged): re-scan the layer, generating
|
||
and resolving cross-partition queries <strong>only</strong> for the indices selected
|
||
in step 2 (cheap membership test against a small per-layer index set) —
|
||
this is what keeps <code>--subsample</code> cheap even on an unsampled-scale index,
|
||
since the cross-partition resolution volume is bounded by <code>N</code>, not by
|
||
the layer's true size.</li>
|
||
</ol>
|
||
<p>Steps 2 and 3 cannot be merged into one pass: true single-pass reservoir
|
||
sampling would waste step-3's expensive resolution work on candidates later
|
||
evicted by the reservoir. Step 1 must fully complete (every layer) before
|
||
step 2 can start for any layer, since <code>total_count</code> is a global quantity.</p>
|
||
<p><strong><code>--shannon</code></strong> (no argument): emits a CSV of per-family Shannon entropy
|
||
(15 non-empty states, <code>∅</code>/absent genomes excluded from the denominator, per
|
||
the settled definition above). Independent of <code>--subsample</code> — entropy is
|
||
computed and written per family as soon as its <code>genome_mask</code> resolves,
|
||
O(1) memory per family, so it streams fine even unsampled at full index
|
||
scale (a time cost, not a memory one). Combined with <code>--subsample N</code>, it
|
||
delivers the original exploratory diagnostic (e.g. <code>--subsample 1000000
|
||
--shannon</code>) directly from this general machinery, rather than a
|
||
purpose-built one-off script.</p>
|
||
<p>Validated end-to-end (2026-08-15) against real data: <code>--sibling-hist</code> on
|
||
<code>phyloskims_sal_vac</code> (91 real genomes, k=31/m=11, 256 partitions × 2
|
||
layers) confirms the ~9-billion-family estimate almost exactly (8,925,068,238
|
||
total, 97.9% monomorphic — a sharper mono fraction than the ~80-85% earlier
|
||
guess, corrected here). <code>--subsample</code>/<code>--shannon</code> on the smaller 20-genome
|
||
bacterial reference (<code>benchmark/global_index_presence</code>) produced a sample
|
||
size within rounding of the request (99,742/100,000) and a
|
||
[0.8,1.2)-bucket share (40.7%) matching the full unsampled population
|
||
(41.6%) — the two histograms only diverged wildly (5‰ vs 41.6%) under a
|
||
real bug in <code>reservoir_sample_layer</code> (see next section), now fixed.</p>
|
||
<p><strong>Bug found and fixed (2026-08-15): <code>family_idx</code> numbering mismatch.</strong>
|
||
<code>scan_layer_families</code>'s <code>family_idx</code> counts <em>every minorant</em> of a layer
|
||
(monomorphic ones included, since <code>iter_minorants_batch</code> filters only on
|
||
<code>is_minorant()</code>), not the raw annex slot (<code>SiblingAnnex::get(slot)</code> spans
|
||
every k-mer, minorant or not) and not a counter over non-monomorphic
|
||
minorants alone. <code>subsample.rs</code>'s <code>reservoir_sample_layer</code> originally
|
||
stored raw slot numbers in its <code>HashSet<usize></code> selection, which drifts
|
||
away from <code>family_idx</code> as soon as <em>any</em> monomorphic minorant is seen —
|
||
i.e. almost immediately, since ~98% of minorants are monomorphic. Fixed by
|
||
tracking two separate counters: <code>family_idx</code> (every minorant, matching
|
||
<code>scan_layer_families</code>) and <code>seen</code> (non-monomorphic minorants only, what
|
||
Algorithm R actually samples over) — only <code>family_idx</code> values are ever
|
||
stored in the selection set. The existing unit test never exercised this
|
||
(its fixture has exactly one non-monomorphic family, always hitting the
|
||
"keep everything" shortcut) — a stronger fixture with several interleaved
|
||
monomorphic/non-monomorphic families would be needed to catch a regression
|
||
here automatically; not yet written.</p>
|
||
<h2 id="cheap-entropy-pre-filtering-row-marginal-sums-idea-not-implemented-2026-08-15">Cheap entropy pre-filtering — row-marginal sums (idea, not implemented, 2026-08-15)</h2>
|
||
<p>Motivation: on real data (bacterial reference, full unsampled run), only
|
||
~5‰ of non-monomorphic minorants fall in the <code>[0.5, 1.5]</code> bit band judged
|
||
phylogenetically interesting (entropy too low = uninformative near-invariant
|
||
site; too high = saturated/noisy, see <code>family_entropy</code>'s 15-state
|
||
discussion) — roughly 1 in 10,000 minorants overall. Computing exact
|
||
entropy for every candidate just to discard 99.99% of them is wasteful at
|
||
the full 9-billion scale.</p>
|
||
<p><strong>The idea</strong>: <code>PersistentBitMatrix::col_view(c)</code> gives a genome's whole
|
||
presence column as a <code>BitSliceView</code> (sequential, no MPHF, no cross-partition
|
||
routing — purely local to one layer's own matrix). Accumulating
|
||
<code>TempCompactIntVecBuilder::inc_present(col)</code> (already exists in
|
||
<code>obicompactvec/src/builder.rs:121</code>, along with <code>add</code>/<code>min</code>/<code>max</code>/<code>diff</code> on
|
||
<code>IntSliceView</code> — no new low-level API needed) over every column of a layer
|
||
produces <code>coverage[slot]</code>: how many genomes carry each exact k-mer, in one
|
||
sequential per-layer pass, entirely decoupled from family/sibling
|
||
structure. Persisted once per layer, a family's members' coverage could
|
||
then be looked up via a plain local MPHF <code>index()</code> (cheap) instead of a
|
||
full cross-partition presence resolution (<code>find_presence_batch</code>) — the
|
||
expensive part today is specifically the cross-partition/cross-layer
|
||
routing to a sibling's own matrix, not the bit-reading itself, and
|
||
<code>coverage[slot]</code> sidesteps that routing entirely by moving the cost into a
|
||
one-time, purely local, embarrassingly-parallel build step.</p>
|
||
<p><strong>Why not implemented</strong>: <code>coverage[slot]</code> is a per-member marginal —
|
||
summing members' coverages to approximate a family's entropy silently
|
||
assumes no genome carries more than one member at once. It cannot
|
||
represent or detect joint co-occurrence (a genome carrying both <code>A</code> and
|
||
<code>C</code> at once, i.e. a combined 15-symbol state) at all, which is exactly the
|
||
phenomenon <code>family_entropy</code>'s 15-state definition exists to capture (see
|
||
<code>CardinalityTally</code>/<code>cardinality_transition_probs</code>, the project's own
|
||
existing machinery for this same co-occurrence structure, built for the
|
||
Sankoff matrix calibration). A family that is in truth uniformly <code>AC</code>
|
||
across every carrying genome would look like a well-balanced 2-state split
|
||
under the marginal approximation (entropy ≈ 1) while its true 15-state
|
||
entropy is 0 — i.e. the marginal proxy's failure mode lands on exactly the
|
||
"saturated, uninformative" tail this pre-filter would need to catch,
|
||
undermining the point. It stays plausible as a coarse filter for the
|
||
<em>low</em> tail only (a dominant single member's marginal share reliably
|
||
predicts low true entropy too), but not as a stand-in for the high tail —
|
||
not pursued further for now.</p>
|
||
<h2 id="-free-loss-tnt-pipeline-four-independent-scans-three-of-them-unsampled-found-2026-08-15-fixed-2026-08-15-see-implemented-below"><code>--free-loss</code>/<code>--tnt</code> pipeline: four independent scans, three of them unsampled (found 2026-08-15, fixed 2026-08-15 — see "Implemented" below)</h2>
|
||
<p>Measured on <code>phyloskims_sal_vac</code> (91 genomes): <code>obikmer phylo --subsample 500000
|
||
--free-loss --tnt</code> logs four sequential stages —
|
||
<code>raw_snp_distance</code> (1413s), <code>base_pair_tally</code> (1456s),
|
||
<code>cardinality_tally</code> (2078s), <code>snp_pseudo_alignment</code> (143s). Reading the
|
||
code (<code>obikmer/src/cmd/phylo/mod.rs:210-242</code>,
|
||
<code>obikphylo/src/siblings/distance.rs</code>, <code>cardinality.rs</code>, <code>alignment.rs</code>)
|
||
surfaced two compounding problems, not one:</p>
|
||
<ol>
|
||
<li><strong>Four separate full scans of the annex</strong>, each opening its own
|
||
<code>KmerPartition</code>/<code>PartitionCache</code> and calling <code>scan_family_pairs</code>/
|
||
<code>scan_layer_families</code> independently — nothing computed in one stage is
|
||
reused by another. <code>base_pair_tally</code> is explicitly documented
|
||
(<code>distance.rs:138-143</code>) as a second full pass over the same
|
||
traversal <code>raw_snp_distance</code> already did, needed only because
|
||
<code>raw_snp_distance</code> doesn't keep the resolved bases, only aggregate
|
||
counts. <code>cardinality_tally</code> and <code>snp_pseudo_alignment</code> are each a
|
||
third and fourth independent full pass. Per the module's own earlier
|
||
profiling note (<code>family_scan.rs:26-28</code>, cited already above), this
|
||
traversal is page-fault/mmap-bound, not compute-bound — the ~10-12%
|
||
CPU efficiency ("contention" status) observed on these three slow
|
||
stages is consistent with I/O stalls scaled by repeated full scans,
|
||
not lock contention (there are no <code>Mutex</code>/<code>RwLock</code> anywhere in
|
||
<code>siblings/*.rs</code>; shared writes use per-slot <code>AtomicU8::fetch_or</code>).</li>
|
||
<li><strong>Worse: <code>raw_snp_distance</code> and <code>cardinality_tally</code> don't honor
|
||
<code>--subsample</code> at all</strong> — both call <code>scan_layer_families</code> with
|
||
<code>Selection::All</code> hardcoded, and <code>args.subsample</code> isn't even threaded
|
||
into their function signatures (<code>mod.rs:212</code>, <code>mod.rs:226</code>). Only
|
||
<code>snp_pseudo_alignment(args.subsample)</code> builds a real reservoir-sampled
|
||
<code>Selection::Some(set)</code> (via <code>compute_selections</code>, <code>alignment.rs:89</code>).
|
||
So today, <code>--subsample 500000</code> only bounds the pseudo-alignment step —
|
||
the SNP-distance matrix, the Sankoff base-pair calibration, and the
|
||
cardinality histogram are always computed over the <strong>full, unsampled</strong>
|
||
index regardless of the flag. This is not merely "different subsamples
|
||
per stage" (which would already be a problem worth fixing) — it's that
|
||
three of the four stages never subsample, which explains most of the
|
||
~10x runtime gap against <code>snp_pseudo_alignment</code> on its own.</li>
|
||
</ol>
|
||
<p><strong>Decided requirement</strong>: all four stages consume <strong>one shared selection</strong>,
|
||
computed once, not each stage either scanning everything or drawing its
|
||
own independent sample. Per-genome-pair SNP counts, the Sankoff base-pair
|
||
calibration, the cardinality histogram, and the pseudo-alignment all
|
||
describe the same set of families — the calibration and the alignment it
|
||
calibrates are now guaranteed to agree on which sites exist.</p>
|
||
<p><strong>Correction to the "single pass" framing above</strong>: <code>base_pair_tally</code>/
|
||
<code>cardinality_tally</code> both need <code>raw_snp_distance</code>'s <em>complete</em> aggregate
|
||
SNP/shared counts before they can derive <code>included[i,j]</code> (the
|
||
<code>ratio_ceiling</code> filter) — a genuine sequential dependency (<code>included</code>
|
||
can't be known until every pair's aggregate count is final), not an
|
||
artifact of the old code's structure. So the fix is <strong>two</strong> passes over
|
||
the shared selection, not one: pass A computes the aggregate counts (and
|
||
derives <code>included</code>); pass B fuses <code>base_pair_tally</code> + <code>cardinality_tally</code>
|
||
+ the pseudo-alignment (mutually independent once <code>included</code> is known)
|
||
into a single scan. Still a 4→2 reduction, and — per the clarification
|
||
that settled this — pass A itself now runs over the <em>same shared
|
||
selection</em> pass B uses (not the full unsampled index): "les ratios, on
|
||
les fait sur les sites sélectionnés, c'est tout, les autres sites
|
||
n'existent pas" — once a selection is chosen, both passes are bounded by
|
||
it, so on a real <code>--subsample</code>/<code>--entropy</code> run pass A is cheap too, not
|
||
just pass B.</p>
|
||
<p><strong>Entropy-biased selection's own resolution to the "forward-looking
|
||
complication"</strong> (entropy must be known before selection, but selection
|
||
happens during the same scan that would resolve it): see "Entropy-biased
|
||
selection" below — resolved via a persisted per-layer entropy annex, not
|
||
by restructuring the scan into an inline pre-pass.</p>
|
||
<h2 id="entropy-biased-selection-soft-gaussian-weighting-not-a-hard-cutoff-decided-and-implemented-2026-08-15">Entropy-biased selection: soft Gaussian weighting, not a hard cutoff (decided and implemented 2026-08-15)</h2>
|
||
<p>Refines the "forward-looking complication" above with a concrete
|
||
mechanism. Instead of a hard <code>[low, high]</code> entropy band (or any other
|
||
exact cutoff) deciding which non-monomorphic minorant families are
|
||
eligible, selection is weighted by an <strong>unnormalized Gaussian kernel</strong>
|
||
centered on a target entropy: <code>w(entropy) = exp(-(entropy - μ)² / (2σ²))</code>
|
||
— deliberately not the normalized Gaussian density (which would peak
|
||
below 1 and complicate the "probability" reading) — this kernel form
|
||
equals 1 exactly at <code>entropy = μ</code> and decays smoothly to 0 away from it,
|
||
so it reads directly as an acceptance weight: the further a family's
|
||
entropy from the target, the less likely it is picked, with no hard
|
||
in/out boundary — a few "bad" sites can still get in, by design. <code>μ</code>
|
||
(default ~1.0) and <code>σ</code> (default ~0.5) are meant to be user-tunable.</p>
|
||
<p><strong>Mechanism: joint probability, not a weighted reservoir</strong>. Not
|
||
Efraimidis–Spirakis weighted reservoir sampling (an earlier, more complex
|
||
proposal, superseded before implementation) — instead, a single
|
||
independent accept/reject draw per qualifying candidate: draw
|
||
<code>u ~ Uniform(0,1)</code>, accept iff <code>u < p₀ · w(entropy)</code>. <code>p₀</code> is a single
|
||
<strong>index-wide</strong> rate, <code>N / total_count</code> (<code>total_count</code> = the sum of
|
||
<code>non_monomorphic_counts</code> across every layer, <code>N</code> = <code>--subsample</code>'s
|
||
target), applied identically at every layer — this alone already gives
|
||
each layer its proportional share (the same effect the old uniform
|
||
reservoir's explicit per-layer <code>n_layer = N · count_layer / total_count</code>
|
||
computation achieved, but without needing to compute it: applying one
|
||
rate uniformly is mathematically the same as proportioning per layer).
|
||
<code>p₀ = 1.0</code> when there is no <code>--subsample</code> at all — <code>--entropy</code> alone is a
|
||
pure soft entropy filter over the whole index, no size target. Properties:
|
||
(1) <strong>strictly generalizes the existing uniform sampler</strong> — with <code>σ</code> large
|
||
enough that <code>w ≈ 1</code> everywhere, this reduces to the old uniform <code>N/total_count</code>
|
||
draw; (2) yields "approximately N", not exactly N — expected accepted
|
||
count is <code>N · mean(w)</code>, always <code>≤ N</code> — an intentional relaxation, matching
|
||
"sous-échantillonnage à environ n" rather than the old reservoir's exact-N
|
||
guarantee; (3) one streaming pass, one random draw per candidate, no
|
||
reservoir state.</p>
|
||
<p><strong>Resolving "entropy must be known before selection, but selection
|
||
happens during the same resolving scan"</strong>: solved with a <strong>persisted,
|
||
per-layer entropy annex</strong> (<code>obikphylo/src/siblings/entropy_annex.rs</code>,
|
||
<code>EntropyAnnex</code>/<code>EntropyAnnexBuilder</code>), not by restructuring the scan.
|
||
Mirrors <code>SiblingAnnex</code>'s mmap-backed, read-only-after-build convention,
|
||
but indexed by <code>family_idx</code> (every minorant of the layer, monomorphic
|
||
included — the same numbering <code>Selection</code>/<code>scan_layer_families</code> already
|
||
use), one <code>f32</code> entropy15 value per entry, <code>-1.0</code> sentinel for monomorphic/
|
||
not-yet-computed. First use of <code>--entropy</code>/<code>--entropy-sd</code> on an index
|
||
pays a one-time cost (<code>ensure_entropy_annexes</code> in <code>entropy.rs</code>: a full,
|
||
unsampled <code>Selection::All</code> scan, resolving every non-monomorphic
|
||
minorant's <code>genome_mask</code> once to compute and persist its entropy) — every
|
||
later run (any <code>μ</code>/<code>σ</code>, any command) reads the file positionally, no
|
||
re-scan, restoring the usual <code>Selection::Some</code> "skip resolving excluded
|
||
families" speedup that a naive "weigh during the resolving scan" design
|
||
would have permanently forfeited.</p>
|
||
<p><strong>Resolved</strong>: the existing hard "non-monomorphic minorant" eligibility
|
||
filter stays a hard gate upstream of the Gaussian weighting — only
|
||
qualifying families ever get a stored entropy value or a weighted draw.</p>
|
||
<p><strong>CLI, implemented</strong>: two <code>phylo</code> options, <code>--entropy <μ></code> and
|
||
<code>--entropy-sd <σ></code> (<code>obikmer/src/cmd/phylo/args.rs</code>). The entropic filter
|
||
activates as soon as <em>either</em> is given (<code>mod.rs</code>, computed once into an
|
||
<code>Option<EntropyBias></code> threaded through <code>--snp</code>/<code>--family-overlap</code>/
|
||
<code>--shannon</code>/the fused sankoff pipeline below). If active but one or both
|
||
are unset, defaults are <code>μ = 1.0</code>, <code>σ = 0.5</code>.</p>
|
||
<h2 id="two-entropy-definitions-kept-side-by-side-for-comparison-2026-08-15">Two entropy definitions kept side by side, for comparison (2026-08-15)</h2>
|
||
<p><code>--shannon</code>'s CSV carries both <code>entropy15</code> (<code>family_entropy</code> — the settled
|
||
15-non-empty-state definition, see above) and <code>entropy4</code> (<code>family_entropy_4</code>
|
||
— plain nucleotide reduction), computed from the same already-resolved
|
||
<code>genome_mask</code>, not from the marginal approximation above. A genome carrying
|
||
several bases at once contributes to <em>each</em> base's count (counted once per
|
||
base present, not fractionally split, not folded into one combined state)
|
||
— a genome polymorphic for the family is present at more than one base by
|
||
construction, so it is expected to count more than once; the denominator is
|
||
the total base-occurrence count, not the genome count (the two coincide
|
||
only when no genome carries more than one base). Kept side by side
|
||
specifically to measure, on real data, how much the two diverge — not yet
|
||
analyzed.</p>
|
||
<h2 id="persistentsparsebitmatrix-implemented-and-measured-2026-08-15"><code>PersistentSparseBitMatrix</code> — implemented and measured (2026-08-15)</h2>
|
||
<p>A row-major (k-mer-major), deduplicated sparse alternative to
|
||
<code>obicompactvec::PersistentBitMatrix</code>, motivated by the same sparsity that
|
||
drove <code>--subsample</code>/<code>--shannon</code> above, but pursued as a foundational
|
||
storage-layer change rather than an index-level workaround. Full design
|
||
history, rationale, and rejected alternatives (external Elias-Fano crates,
|
||
<code>cacheline-ef</code>, a single unsplit <code>dict_id</code> array) are in the dedicated
|
||
implementation plan (<code>vivid-mapping-tiger.md</code> at the time of writing — the
|
||
content below is the durable summary, not a pointer to a session-scoped
|
||
file). Also directly informed by Alanko, Bille, Gørtz, Navarro, Puglisi,
|
||
"Compact Data Structures for Collections of Sets" (2025,
|
||
<code>biblio/Alanko et al. - Compact Data Structures for Collections of
|
||
Sets.pdf</code>) — this design implements only their exact-duplicate special
|
||
case (a plain dedup dictionary), not their full subset-containment
|
||
hierarchy.</p>
|
||
<p><strong>Design</strong>: four on-disk components, each mmap-backed, built once per
|
||
layer (matching how the rest of the build pipeline already works — never
|
||
the whole multi-billion-row index at once): an <code>is_multi</code> rank-capable
|
||
flag per row (singleton vs. multi-genome), a fixed-bit-width array for
|
||
singleton rows (genome index directly, <code>ceil(log2(n_cols))</code> bits), a
|
||
separate fixed-bit-width array for multi-genome rows (<code>dict_id</code>,
|
||
<code>ceil(log2(n_distinct_multi_sets))</code> bits — kept apart from the singleton
|
||
array specifically because <code>n_distinct_multi_sets</code> can be large in
|
||
absolute terms even when multi-genome rows are a small <em>fraction</em> of all
|
||
rows, and a single shared array would force every row, singletons
|
||
included, to pay the wider width), and a deduplicated dictionary of
|
||
distinct multi-genome sets (Elias-Fano-encoded byte offsets + a
|
||
varint-encoded values blob). New low-level primitives added to
|
||
<code>obicompactvec</code> to build this: <code>PersistentFixedIntVec</code> (arbitrary,
|
||
runtime-parameterized bit width, width 0 included — needed once a real
|
||
bug surfaced, see below), <code>PersistentRankSelectBitVec</code> (rank1/rank0/select1
|
||
on top of the crate's existing <code>count_ones</code>, using
|
||
<code>common_traits::SelectInWord</code>), <code>EliasFano</code> (composes the two). A new
|
||
<code>BinaryMatrix</code> trait (<code>n</code>, <code>n_cols</code>, <code>row</code>/<code>fill_row</code>, <code>fill_sub_matrix</code>,
|
||
<code>count_ones</code>) unifies dense and sparse at the one call site that needs
|
||
both interchangeably (<code>obikphylo::siblings::cache::Mat</code>) — column-oriented
|
||
methods (<code>col</code>, <code>col_view</code>, the <code>partial_*_dist_matrix</code> family) stay
|
||
dense-only.</p>
|
||
<p><strong>Two real bugs caught by tests, not by inspection</strong>: (1) <code>EliasFano::open</code>
|
||
re-derived its low-bits width from the persisted low-vector file's own
|
||
width byte; the zero-width case was built with a dummy 1-bit placeholder
|
||
(the builder rejected true width 0), so every reopened value silently
|
||
doubled. Fixed by making <code>PersistentFixedIntVec</code> genuinely support width 0
|
||
(no storage, <code>get</code> always 0) instead of working around the limitation in
|
||
<code>EliasFano</code>. (2) An empty row (cardinality 0 — not expected on a real
|
||
built index, but not guarded against either) was recorded as a singleton
|
||
at genome 0, indistinguishable on read-back from a <em>real</em> singleton at
|
||
genome 0. Fixed by routing cardinality-0 rows through the dictionary path
|
||
(a genuine empty entry) instead of the singleton shortcut. Both caught by
|
||
<code>obicompactvec</code>'s test suite (142 tests, including disk-reopen round-trips
|
||
that drop every builder/mmap before reopening fresh), not by manual
|
||
review — worth remembering next time a "this edge case can't happen in
|
||
practice" shortcut is tempting.</p>
|
||
<p><strong>Measured on real data</strong> (<code>layer_1</code> of <code>phyloskims_sal_vac</code>'s
|
||
<code>part_00018</code>, 30,246,774 rows, 91 genomes — <code>#[ignore]</code>d benchmarks in
|
||
<code>obikphylo/src/siblings/tests.rs</code>):</p>
|
||
<table>
|
||
<thead>
|
||
<tr>
|
||
<th></th>
|
||
<th>dense</th>
|
||
<th>sparse</th>
|
||
<th>ratio</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr>
|
||
<td>on-disk size</td>
|
||
<td>328.1MB</td>
|
||
<td>43.7MB</td>
|
||
<td><strong>7.5x</strong> smaller</td>
|
||
</tr>
|
||
<tr>
|
||
<td>build time / peak RSS</td>
|
||
<td>—</td>
|
||
<td>4.26s / 628MB</td>
|
||
<td>(per-layer, in-memory construction — comfortable)</td>
|
||
</tr>
|
||
<tr>
|
||
<td>row access, sequential (2M reads)</td>
|
||
<td>43ns/row</td>
|
||
<td>32ns/row</td>
|
||
<td>sparse <strong>faster</strong> (smaller structure, better cache fit)</td>
|
||
</tr>
|
||
<tr>
|
||
<td>row access, random (2M reads)</td>
|
||
<td>409ns/row</td>
|
||
<td>85ns/row</td>
|
||
<td>sparse <strong>~4.8x faster</strong> (the real <code>--shannon</code>/family-lookup shape)</td>
|
||
</tr>
|
||
<tr>
|
||
<td>column access, one full column (30.2M rows)</td>
|
||
<td>11.5ms</td>
|
||
<td>993ms</td>
|
||
<td>sparse <strong>86x slower</strong> (no native column method — every read decodes a full row to keep one bit)</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
<p>The row-access wins (both directions) weren't the design's stated goal —
|
||
compactness was — but turn out real: dense's genome-major layout scatters
|
||
a single row read across a much bigger file, which costs more than
|
||
sparse's rank/select/varint decode once the file is this much smaller.
|
||
The column-access cost is the flip side of the same layout choice, and is
|
||
exactly what the next item below exists to fix.</p>
|
||
<p><strong>Next, not yet planned</strong>: rewrite <code>partial_jaccard_dist_matrix</code>/
|
||
<code>partial_hamming_dist_matrix</code>/etc. (<code>obicompactvec/src/bitmatrix/pairwise.rs</code>)
|
||
as a row-major co-occurrence accumulation (<code>O(Σ_rows k²)</code>, per-row
|
||
increments into an <code>NxN</code> genome-pair counter — the known alternative to
|
||
today's column-fold, plausibly cheaper on data this sparse, not just a
|
||
fallback) so <code>obikindex</code>'s <code>--metric</code>/distance-matrix path can use the
|
||
sparse type without the measured 86x column-access penalty. Needs its own
|
||
design pass (in particular how it plugs into the <code>BitPartials</code>/
|
||
<code>ColumnWeights</code> traits so both matrix types keep serving <code>--metric</code>)
|
||
before implementation — not just "port the loop", a genuinely different
|
||
algorithm.</p>
|
||
<p>Also still deferred, unchanged from the implementation plan: full Alanko
|
||
et al. subset-hierarchy compression (only the exact-duplicate special case
|
||
is built), a sparse <code>PersistentCompactIntMatrix</code> (count matrices), and
|
||
BRWT-style column-correlation exploitation.</p>
|
||
<h2 id="wired-into-pack-and-the-sibling-annex-build-path-2026-08-15">Wired into <code>pack</code> and the sibling-annex build path (2026-08-15)</h2>
|
||
<p><code>PersistentSparseBitMatrix</code> went from a validated but unused type to a
|
||
real, selectable on-disk format:</p>
|
||
<ul>
|
||
<li><strong>Generic <code>Layer<D></code></strong>: <code>obilayeredmap::Layer<D></code>'s presence-only methods
|
||
(<code>n_cols</code>, <code>sub_matrix</code>, <code>fill_sub_matrix</code>) are generic over any
|
||
<code>D: LayerData<Item = Box<[bool]>> + BinaryMatrix</code>, not hardcoded to
|
||
<code>PersistentBitMatrix</code> — <code>PersistentSparseBitMatrix</code> implements
|
||
<code>LayerData</code> (<code>open</code>/<code>read</code>) the same way. <code>find_slot</code>/<code>index_batch</code> were
|
||
already generic over any <code>D: LayerData</code>, so they needed no change.
|
||
Verified by <code>obilayeredmap</code>'s
|
||
<code>presence_layer_generic_over_sparse_matches_dense</code> test: build a dense
|
||
presence layer, convert it to sparse via <code>build_from_dense</code>, open both
|
||
as <code>Layer<PersistentBitMatrix></code>/<code>Layer<PersistentSparseBitMatrix></code> on
|
||
the same directory, assert <code>n_cols</code>/<code>sub_matrix</code>/<code>find_slot</code> agree.
|
||
(This test must stay at <code>k=4</code> with mutually non-colliding canonical
|
||
4-mers across its input sequences — <code>K</code>/<code>M</code> are process-wide
|
||
<code>AtomicUsize</code>s in test builds, not thread-local, so a test using a
|
||
different <code>k</code> races every other test in the same crate binary; a k=11
|
||
version of this test passed alone but failed under the full
|
||
<code>obilayeredmap</code> suite for exactly that reason before being fixed.)</li>
|
||
<li><strong><code>obikphylo::siblings::cache::Mat</code></strong> gained a third variant,
|
||
<code>SparsePresence(Layer<PersistentSparseBitMatrix>)</code>, alongside <code>Count</code>
|
||
and <code>Presence</code> — every method (<code>find_slot</code>, <code>index_batch</code>,
|
||
<code>iter_minorants_batch</code>, <code>n_cols</code>, <code>fill_sub_matrix_carries</code>) dispatches
|
||
to it identically to <code>Presence</code>, since both go through the same generic
|
||
<code>Layer<D></code> code. <code>PartitionCache::build</code> picks the variant per layer by
|
||
checking for <code>presence/is_multi.prsb</code> (the sparse format's own marker
|
||
file, see the design section above) before falling back to the dense
|
||
open path.</li>
|
||
<li><strong><code>pack_sparse_bit_matrix</code></strong> (new, <code>obicompactvec::bitmatrix::sparse</code>):
|
||
<code>pack --sparse</code>'s entry point. Idempotent (checks <code>is_multi.prsb</code>
|
||
first); packs to dense <code>matrix.pbmx</code> first if that hasn't happened yet
|
||
(the dense→sparse transpose needs random row access, which only the
|
||
packed/columnar dense forms give), then <code>build_from_dense</code>s the sparse
|
||
form into the same directory and deletes <code>matrix.pbmx</code> — old-format
|
||
files are removed only after the new format is fully written, mirroring
|
||
<code>pack_bit_matrix</code>'s own crash-safety convention.</li>
|
||
<li><strong>CLI</strong>: <code>obikmer pack --sparse</code> threads a <code>sparse: bool</code> through
|
||
<code>KmerIndex::pack_matrices</code> (all other call sites — <code>select</code>, <code>merge</code>,
|
||
<code>finalize_indexed</code> — pass <code>false</code>, unchanged dense behaviour). Count
|
||
matrices are untouched by <code>--sparse</code> (no sparse <code>PersistentCompactIntMatrix</code>
|
||
— see "still deferred" above).</li>
|
||
<li><strong>End-to-end coverage</strong>: <code>obikphylo::siblings::tests::
|
||
sibling_annex_works_after_pack_sparse</code> builds a two-genome index, packs
|
||
it <code>--sparse</code>, asserts <code>is_multi.prsb</code> exists, then runs
|
||
<code>build_sibling_annex</code> and checks the resulting <code>FamilyMask</code>s match the
|
||
dense-path test (<code>sibling_annex_one_sibling_each</code>) exactly — proves the
|
||
sparse format round-trips through the real build pipeline
|
||
(<code>PartitionCache</code> sparse-detection included), not just the
|
||
<code>obicompactvec</code>/<code>obilayeredmap</code> unit layers below it.</li>
|
||
</ul>
|
||
<p>Full workspace <code>cargo test</code> (all crates, unit + doc tests) green after
|
||
this change.</p>
|
||
<h2 id="sankoff-pipeline-fusion-entropy-biased-selection-implemented-2026-08-15">Sankoff pipeline fusion + entropy-biased selection — implemented (2026-08-15)</h2>
|
||
<p>Replaces the "four independent scans" problem above and implements
|
||
"Entropy-biased selection" above, end to end:</p>
|
||
<ul>
|
||
<li><strong><code>SankoffBundleExt::sankoff_bundle</code></strong> (new,
|
||
<code>obikphylo/src/siblings/sankoff_bundle.rs</code>) — the <code>--sankoff</code>/<code>--tnt</code>/
|
||
<code>--phyg</code>/<code>--iqtree</code> block in <code>obikmer/src/cmd/phylo/mod.rs</code> now calls
|
||
this once instead of three separate <code>raw_snp_distance</code>/
|
||
<code>base_pair_tally</code>/<code>cardinality_tally</code> calls. One <code>PartitionCache</code>, one
|
||
shared (possibly subsampled/entropy-biased) selection computed once via
|
||
<code>compute_selections</code>, two scans over it: pass A (aggregate SNP/shared
|
||
counts, <code>--exclude-genome</code> zeroing, then <code>included[i,j]</code>), pass B
|
||
(<code>base_pair_tally</code> + <code>cardinality_tally</code> + the pseudo-alignment, fused
|
||
into one <code>scan_layer_families</code> call per layer, all three read off the
|
||
same resolved <code>genome_mask</code>). <code>snp_pseudo_alignment</code>/
|
||
<code>shannon_entropy_csv</code> (still used standalone by <code>--snp</code>/
|
||
<code>--family-overlap</code>/<code>--shannon</code>) both gained an <code>entropy_bias</code> parameter
|
||
too, so entropy-biased selection isn't sankoff-specific.</li>
|
||
<li><strong>Regression proof</strong>: <code>sankoff_bundle_matches_old_separate_calls</code>
|
||
(<code>obikphylo/src/siblings/tests.rs</code>) asserts <code>sankoff_bundle</code>'s four
|
||
outputs are bit-identical to calling the old, separate
|
||
<code>raw_snp_distance</code>/<code>base_pair_tally</code>/<code>cardinality_tally</code>/
|
||
<code>snp_pseudo_alignment</code> on the same fixture with no subsample — the
|
||
fusion is a performance change, not a behavior change.</li>
|
||
<li><strong><code>EntropyAnnex</code>/<code>EntropyAnnexBuilder</code></strong> (new,
|
||
<code>obikphylo/src/siblings/entropy_annex.rs</code>) and <strong><code>ensure_entropy_annexes</code></strong>
|
||
(<code>entropy.rs</code>) implement the persisted-entropy mechanism from
|
||
"Entropy-biased selection" above. <code>entropy_annex_builds_on_demand_and_biases_selection</code>
|
||
(<code>tests.rs</code>) proves, on a fixture with one known-entropy family: the
|
||
annex file doesn't exist before any entropy-biased call; <code>compute_selections</code>
|
||
builds it on first use; <code>μ</code> set to the family's exact entropy with
|
||
<code>p₀ = 1.0</code> selects it deterministically (<code>u < 1.0</code> always, for
|
||
<code>u ∈ [0,1)</code>); <code>μ</code> set far away with tiny <code>σ</code> deterministically excludes
|
||
it (<code>w</code> underflows to exactly <code>0.0</code>); the persisted value matches
|
||
<code>--shannon</code>'s own <code>family_entropy</code> computation to <code>1e-6</code>.</li>
|
||
<li><strong><code>EntropyBias</code></strong> (<code>pub</code>, <code>obikphylo::siblings::EntropyBias { mu, sigma }</code>)
|
||
is the one new public type threading <code>--entropy</code>/<code>--entropy-sd</code> through
|
||
every <code>Option<EntropyBias></code>-accepting method — resolved once in
|
||
<code>obikmer/src/cmd/phylo/mod.rs</code> from <code>args.entropy</code>/<code>args.entropy_sd</code>
|
||
(activation: either given; defaults <code>1.0</code>/<code>0.5</code> for whichever is unset).</li>
|
||
</ul>
|
||
<p>Full workspace <code>cargo test</code> green after this change (167 unit tests in
|
||
<code>obicompactvec</code>+<code>obilayeredmap</code>+<code>obikphylo</code> alone, plus every other
|
||
crate's suite, no regressions).</p>
|
||
<p><strong>Still open, not part of this change</strong> (per "Correction to the 'single
|
||
pass' framing" above): <code>--raw-snp-distance</code>/<code>--raw-snp-counts</code> (the
|
||
standalone diagnostic flags, not the <code>--sankoff</code> pipeline) still always
|
||
scan the full unsampled index — never threaded <code>--subsample</code>/<code>--entropy</code>,
|
||
out of scope here since the reported problem was specifically about the
|
||
<code>--sankoff</code>/<code>--tnt</code> pipeline's redundant/inconsistent scans, not these
|
||
two standalone flags.</p>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
</article>
|
||
</div>
|
||
|
||
|
||
<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
|
||
</div>
|
||
|
||
</main>
|
||
|
||
<footer class="md-footer">
|
||
|
||
<div class="md-footer-meta md-typeset">
|
||
<div class="md-footer-meta__inner md-grid">
|
||
<div class="md-copyright">
|
||
|
||
|
||
Made with
|
||
<a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
|
||
Material for MkDocs
|
||
</a>
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
</footer>
|
||
|
||
</div>
|
||
<div class="md-dialog" data-md-component="dialog">
|
||
<div class="md-dialog__inner md-typeset"></div>
|
||
</div>
|
||
|
||
|
||
|
||
|
||
|
||
<script id="__config" type="application/json">{"annotate": null, "base": "../..", "features": [], "search": "../../assets/javascripts/workers/search.2c215733.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
|
||
|
||
|
||
<script src="../../assets/javascripts/bundle.79ae519e.min.js"></script>
|
||
|
||
<script src="https://unpkg.com/mathjax@3/es5/tex-mml-chtml.js"></script>
|
||
|
||
|
||
</body>
|
||
</html> |