This commit introduces significant functional improvements related to sparse matrix handling, including new storage layers, performance fixes for sparse operations, and bug corrections in matrix logic. Additionally, architectural documentation is cleaned up by removing obsolete entries and modifying structural text related to the Sibling Annex discussion.
1702 lines
35 KiB
HTML
1702 lines
35 KiB
HTML
|
|
<!doctype html>
|
|
<html lang="en" class="no-js">
|
|
<head>
|
|
|
|
<meta charset="utf-8">
|
|
<meta name="viewport" content="width=device-width,initial-scale=1">
|
|
|
|
|
|
|
|
|
|
<link rel="prev" href="../index_architecture/">
|
|
|
|
|
|
<link rel="next" href="../numa_worker_pools/">
|
|
|
|
|
|
|
|
|
|
|
|
<link rel="icon" href="../../assets/images/favicon.png">
|
|
<meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.7.6">
|
|
|
|
|
|
|
|
<title>Sibling annex - obikmer</title>
|
|
|
|
|
|
|
|
<link rel="stylesheet" href="../../assets/stylesheets/main.484c7ddc.min.css">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
|
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
|
|
<style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
|
|
|
|
|
|
|
|
<script>__md_scope=new URL("../..",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
|
|
|
|
|
|
|
|
|
|
|
|
</head>
|
|
|
|
|
|
<body dir="ltr">
|
|
|
|
|
|
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
|
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
|
<label class="md-overlay" for="__drawer"></label>
|
|
<div data-md-component="skip">
|
|
|
|
|
|
<a href="#sibling-annex-snp-family-pipeline-architecture" class="md-skip">
|
|
Skip to content
|
|
</a>
|
|
|
|
</div>
|
|
<div data-md-component="announce">
|
|
|
|
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<header class="md-header md-header--shadow" data-md-component="header">
|
|
<nav class="md-header__inner md-grid" aria-label="Header">
|
|
<a href="../.." title="obikmer" class="md-header__button md-logo" aria-label="obikmer" data-md-component="logo">
|
|
|
|
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
|
|
|
|
</a>
|
|
<label class="md-header__button md-icon" for="__drawer">
|
|
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
|
|
</label>
|
|
<div class="md-header__title" data-md-component="header-title">
|
|
<div class="md-header__ellipsis">
|
|
<div class="md-header__topic">
|
|
<span class="md-ellipsis">
|
|
obikmer
|
|
</span>
|
|
</div>
|
|
<div class="md-header__topic" data-md-component="header-topic">
|
|
<span class="md-ellipsis">
|
|
|
|
Sibling annex
|
|
|
|
</span>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<script>var palette=__md_get("__palette");if(palette&&palette.color){if("(prefers-color-scheme)"===palette.color.media){var media=matchMedia("(prefers-color-scheme: light)"),input=document.querySelector(media.matches?"[data-md-color-media='(prefers-color-scheme: light)']":"[data-md-color-media='(prefers-color-scheme: dark)']");palette.color.media=input.getAttribute("data-md-color-media"),palette.color.scheme=input.getAttribute("data-md-color-scheme"),palette.color.primary=input.getAttribute("data-md-color-primary"),palette.color.accent=input.getAttribute("data-md-color-accent")}for(var[key,value]of Object.entries(palette.color))document.body.setAttribute("data-md-color-"+key,value)}</script>
|
|
|
|
|
|
|
|
|
|
</nav>
|
|
|
|
</header>
|
|
|
|
<div class="md-container" data-md-component="container">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<main class="md-main" data-md-component="main">
|
|
<div class="md-main__inner md-grid">
|
|
|
|
|
|
|
|
<div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
|
|
<div class="md-sidebar__scrollwrap">
|
|
<div class="md-sidebar__inner">
|
|
|
|
|
|
|
|
|
|
<nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
|
|
<label class="md-nav__title" for="__drawer">
|
|
<a href="../.." title="obikmer" class="md-nav__button md-logo" aria-label="obikmer" data-md-component="logo">
|
|
|
|
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
|
|
|
|
</a>
|
|
obikmer
|
|
</label>
|
|
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../.." class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Home
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../installation/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Installation
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item md-nav__item--nested">
|
|
|
|
|
|
|
|
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3" >
|
|
|
|
|
|
<label class="md-nav__link" for="__nav_3" id="__nav_3_label" tabindex="0">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Theory
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
<span class="md-nav__icon md-icon"></span>
|
|
</label>
|
|
|
|
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_3_label" aria-expanded="false">
|
|
<label class="md-nav__title" for="__nav_3">
|
|
<span class="md-nav__icon md-icon"></span>
|
|
|
|
|
|
Theory
|
|
|
|
|
|
</label>
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../kmers/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Kmers and super-kmers
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../theory/encoding/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
DNA encoding
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../theory/entropy/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Entropy filter
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../theory/minimizer/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Minimizer selection
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../theory/indexing/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Partitioning architecture
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../theory/evolutionary_distances/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Central-position SNP distance (discussion)
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
</ul>
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item md-nav__item--nested">
|
|
|
|
|
|
|
|
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4" >
|
|
|
|
|
|
<label class="md-nav__link" for="__nav_4" id="__nav_4_label" tabindex="0">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Implementation
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
<span class="md-nav__icon md-icon"></span>
|
|
</label>
|
|
|
|
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_4_label" aria-expanded="false">
|
|
<label class="md-nav__title" for="__nav_4">
|
|
<span class="md-nav__icon md-icon"></span>
|
|
|
|
|
|
Implementation
|
|
|
|
|
|
</label>
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../implementation/superkmer/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
SuperKmer
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../implementation/kmer/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Kmer
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../implementation/chunkreader/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Chunk reader
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../implementation/pipeline/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Construction pipeline
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../implementation/obipipeline/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
obipipeline library
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../implementation/storage/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
On-disk storage
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../implementation/mphf/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
MPHF selection
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../implementation/unitig_evidence/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Unitig evidence encoding
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../implementation/evidence_elimination/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Evidence elimination (discussion)
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../implementation/layer_tier/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
obikindex layer tier
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../implementation/persistent_compact_int_vec/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
PersistentCompactIntVec
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../implementation/persistent_bit_vec/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
PersistentBitVec
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../implementation/merge/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Merge command
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../implementation/merge_parallelism/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Merge parallelism & memory
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../implementation/filtering/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Kmer filtering
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../implementation/select/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Select command
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../implementation/obitaxonomy/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
obitaxonomy crate
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../implementation/benchmark_query_testing/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Benchmark: query-path testing
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../implementation/partition_layer_cache/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Partition and layer caching (discussion)
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
</ul>
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item md-nav__item--active md-nav__item--nested">
|
|
|
|
|
|
|
|
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_5" checked>
|
|
|
|
|
|
<label class="md-nav__link" for="__nav_5" id="__nav_5_label" tabindex="0">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Architecture
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
<span class="md-nav__icon md-icon"></span>
|
|
</label>
|
|
|
|
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_5_label" aria-expanded="true">
|
|
<label class="md-nav__title" for="__nav_5">
|
|
<span class="md-nav__icon md-icon"></span>
|
|
|
|
|
|
Architecture
|
|
|
|
|
|
</label>
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../sequences/invariant/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Sequences
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../index_architecture/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Kmer index
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item md-nav__item--active">
|
|
|
|
<input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
|
|
|
|
|
|
|
|
|
|
|
|
<label class="md-nav__link md-nav__link--active" for="__toc">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Sibling annex
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
<span class="md-nav__icon md-icon"></span>
|
|
</label>
|
|
|
|
<a href="./" class="md-nav__link md-nav__link--active">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Sibling annex
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
|
|
|
|
|
|
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<label class="md-nav__title" for="__toc">
|
|
<span class="md-nav__icon md-icon"></span>
|
|
Table of contents
|
|
</label>
|
|
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#data-model" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Data model
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#construction" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Construction
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#shared-traversal" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Shared traversal
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#masking-computed-once-shared-everywhere" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Masking — computed once, shared everywhere
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#precomputed-annexes-entropy-family-overlap" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Precomputed annexes (entropy, family overlap)
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#sampling-and-the-shared-scan-trunk" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Sampling and the shared scan trunk
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#from-a-tally-to-distances-and-calibration" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
From a tally to distances and calibration
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#public-api-surface" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Public API surface
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#known-gaps" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Known gaps
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../numa_worker_pools/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
NUMA-aware worker pools
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../numa_partition_runner/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
NUMA-aware partition runner
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
</ul>
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
|
|
|
|
</ul>
|
|
</nav>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
|
|
<div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
|
|
<div class="md-sidebar__scrollwrap">
|
|
<div class="md-sidebar__inner">
|
|
|
|
|
|
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<label class="md-nav__title" for="__toc">
|
|
<span class="md-nav__icon md-icon"></span>
|
|
Table of contents
|
|
</label>
|
|
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#data-model" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Data model
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#construction" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Construction
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#shared-traversal" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Shared traversal
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#masking-computed-once-shared-everywhere" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Masking — computed once, shared everywhere
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#precomputed-annexes-entropy-family-overlap" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Precomputed annexes (entropy, family overlap)
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#sampling-and-the-shared-scan-trunk" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Sampling and the shared scan trunk
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#from-a-tally-to-distances-and-calibration" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
From a tally to distances and calibration
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#public-api-surface" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Public API surface
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#known-gaps" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Known gaps
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
|
|
</nav>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
|
|
<div class="md-content" data-md-component="content">
|
|
|
|
<article class="md-content__inner md-typeset">
|
|
|
|
|
|
|
|
|
|
|
|
<h1 id="sibling-annex-snp-family-pipeline-architecture">Sibling annex & SNP-family pipeline — architecture</h1>
|
|
<p>Current state of <code>obikphylo::siblings</code> (crate <code>obikphylo</code>, module <code>siblings</code>).
|
|
Historical rationale, dated findings, and superseded designs live in <code>jj</code>
|
|
history, not here.</p>
|
|
<h2 id="data-model">Data model</h2>
|
|
<p>A <strong>family</strong> is the up-to-4 k-mers sharing all flanks and differing only at
|
|
the central base (well-defined for odd <code>k</code>). <code>FamilyMask</code> (<code>siblingannex.rs</code>)
|
|
is a 4-bit presence mask (bit <code>b</code> set iff the family member whose own
|
|
canonical central base is <code>b</code> is observed anywhere in the index) plus a
|
|
minorant flag; <code>family_size()</code>/<code>siblings()</code> derive member count from it.
|
|
<code>SiblingAnnex</code>/<code>SiblingAnnexBuilder</code> persist one <code>FamilyMask</code> per k-mer,
|
|
<strong>keyed by iteration order</strong> (the physical layout of <code>unitigs.bin</code>), never by
|
|
MPHF slot — this lets every reader zip-iterate a layer's own k-mer stream
|
|
against the annex file directly, no MPHF/slot indirection.</p>
|
|
<h2 id="construction">Construction</h2>
|
|
<p><code>algorithms::annex::build_layer_sibling_annex</code> builds one layer's annex:
|
|
seeds each k-mer's own central base, generates its up-to-3 central-base
|
|
variants, resolves each cross-partition (<code>IndexCache::find_in_partition</code>,
|
|
grouped and chunked per destination partition for mmap locality), writes the
|
|
final mask (with the minorant flag) in a second streamed pass.
|
|
<code>SiblingExt::build_sibling_annex</code> drives this for every layer of every
|
|
partition, then invalidates and rebuilds the entropy and family-overlap
|
|
annexes (their <code>family_idx</code> numbering depends on the sibling annex).</p>
|
|
<h2 id="shared-traversal">Shared traversal</h2>
|
|
<p><code>algorithms::family_scan::scan_layer_families</code> is the <strong>only</strong> place that
|
|
resolves a family's full per-genome presence (<code>genome_mask: &[u8]</code>,
|
|
cross-partition) — every other consumer in this module is built on it, never
|
|
reimplements the resolution. Two decoupled concurrency mechanisms: batch
|
|
generation (CPU-only) runs on an <code>obipipeline</code> throttle/pipe; per-batch
|
|
resolution (I/O-bound, mmap page faults) runs <code>rayon</code>-parallel across
|
|
destination partitions, one generated batch at a time (never several
|
|
concurrently, to keep one partition's pages hot instead of scattering
|
|
access). <code>fast_mode</code> (<code>is_fast_mode</code>, <code><= 7</code> layers per partition) picks
|
|
between trusting a family member's recorded layer index outright or probing
|
|
every layer of the destination partition.</p>
|
|
<p><code>Selection<'a></code> bounds which families of a layer are resolved:</p>
|
|
<ul>
|
|
<li><code>All</code> — every minorant, monomorphic included (<code>SiblingExt::shannon_entropy_csv</code>,
|
|
<code>sibling_annex_stats</code>).</li>
|
|
<li><code>Some(&HashSet<usize>)</code> — a caller-picked subset, skipping the expensive
|
|
cross-partition resolution entirely for excluded families
|
|
(<code>subsample::sample_layer</code>'s accepted-candidates batch;
|
|
<code>entropy::ensure_layer_entropy_annex</code>'s non-monomorphic-only restriction).</li>
|
|
</ul>
|
|
<p><code>minorant_selection::eligible_bitset_layer</code>/<code>non_monomorphic_reindex_layer</code>
|
|
provide the cheap, annex-bits-only (no cross-partition resolution) filters
|
|
these restricted scans are built from: monomorphism (<code>family_size() < 2</code>) is
|
|
knowable straight from the annex, so nothing pays <code>scan_layer_families</code>'s
|
|
resolution cost just to discard a monomorphic result.</p>
|
|
<h2 id="masking-computed-once-shared-everywhere">Masking — computed once, shared everywhere</h2>
|
|
<p><code>algorithms::masking::masked_state(m, free_loss, no_ambiguity)</code> is the single
|
|
function that turns a raw per-genome 4-bit state into the masked view every
|
|
consumer needs (<code>None</code> for absence under <code>--free-loss</code> or ambiguity under
|
|
<code>--no-ambiguity</code>). It is called <strong>exactly once per (family, genome)</strong>, in
|
|
<code>subsample::sample_layer</code>, immediately after <code>scan_layer_families</code> resolves
|
|
the raw state — never recomputed downstream. <code>survives_masking</code> (does the
|
|
family still carry ≥2 distinct masked states among included genomes?) and
|
|
<code>SurvivingFamily::masked: Vec<Option<u8>></code> both consume this same
|
|
precomputed array, which is also what <code>AlignmentReducer</code>/<code>PairwiseReducer</code>
|
|
read (see below) — this is what guarantees the alignment and the pairwise
|
|
tally can never disagree about a genome's state at a given site.</p>
|
|
<h2 id="precomputed-annexes-entropy-family-overlap">Precomputed annexes (entropy, family overlap)</h2>
|
|
<p><code>algorithms::entropy::ensure_layer_entropy_annex</code> builds <code>EntropyAnnex</code>
|
|
(<code>entropy_annex.rs</code>, one <code>f32</code> Shannon-entropy value per non-monomorphic
|
|
minorant, mmap-backed) and/or feeds <code>FamilyOverlapAccumulator</code>
|
|
(<code>family_overlap.rs</code>) from one shared <code>Selection::Some</code> scan — both need the
|
|
same <code>genome_mask</code> resolution, so a missing one never pays for it twice.
|
|
<code>family_entropy</code> (15 non-empty states of <code>{A,C,G,T}</code>'s powerset, <code>∅</code>
|
|
excluded from the denominator — the project's settled informativeness
|
|
definition, matching the calibrated 16-state Sankoff cost matrix) and
|
|
<code>family_entropy_4</code> (plain 4-symbol reduction, kept only for comparison) are
|
|
pure functions of <code>genome_mask</code>. <code>extensions::SiblingBuilder::ensure_entropy_and_overlap_annexes</code>
|
|
drives this for every cached layer, lazily (only what's missing).</p>
|
|
<h2 id="sampling-and-the-shared-scan-trunk">Sampling and the shared scan trunk</h2>
|
|
<p><code>algorithms::subsample::sample_index</code> draws up to <code>n</code> non-monomorphic
|
|
families index-wide, proportionally per layer: for each layer, a target
|
|
quota sized from that layer's share of the index-wide eligible count, filled
|
|
by a circular walk with a Bernoulli(<code>p</code>) draw per candidate — <code>p</code> optionally
|
|
weighted by an unnormalized Gaussian kernel over the candidate's entropy
|
|
(<code>EntropyBias { mu, sigma }</code>, read from the entropy annex). Accepted
|
|
candidates are resolved in batches (<code>scan_layer_families</code>) and checked
|
|
against <code>survives_masking</code>; rejected-after-masking candidates don't count
|
|
toward quota, triggering further "topup" rounds (bounded, <code>MAX_TOPUP_ROUNDS
|
|
= 5</code>) rather than silently under-filling. Every <code>SurvivingFamily</code> reaching a
|
|
caller has already survived both the draw and post-masking informativeness —
|
|
no second filter pass needed downstream.</p>
|
|
<p><code>algorithms::scan::sampled_scan</code> is the single entry point every sampled
|
|
computation goes through: one <code>sample_index</code> pass, fanned out to whichever
|
|
<code>LayerReducer</code> implementations the caller actually needs, each receiving the
|
|
exact same per-layer <code>SurvivingFamily</code> batch (an <code>Arc</code>, cloned not copied).
|
|
This is what guarantees several outputs requested together
|
|
(<code>--pseudo-alignment</code> and <code>--sankoff</code> and a <code>snp-*</code> <code>--distance</code> in one CLI
|
|
invocation) come from <strong>one</strong> random sample, not independently-drawn ones —
|
|
there is no other driver of <code>sample_index</code> anywhere in this crate.</p>
|
|
<p>Two <code>LayerReducer</code> implementations exist today:</p>
|
|
<ul>
|
|
<li><code>AlignmentReducer</code> (<code>alignment.rs</code>) — accumulates <code>SnpAlignment</code> (one
|
|
IUPAC-coded row per included genome), reading <code>family.masked</code> directly.</li>
|
|
<li><code>PairwiseReducer</code> (<code>pairwise.rs</code>) — accumulates <code>PairwiseTally</code>/
|
|
<code>PartitionDispersion</code>, also reading <code>family.masked</code> directly (substitution/
|
|
agreement counts and cardinality both derive from the same masked state
|
|
the alignment uses).</li>
|
|
</ul>
|
|
<h2 id="from-a-tally-to-distances-and-calibration">From a tally to distances and calibration</h2>
|
|
<p><code>PairwiseTally</code> stores per-genome-pair substitution/agreement/cardinality
|
|
counts, unfiltered, from which every downstream output is cheap <code>O(n²)</code>
|
|
post-processing (never a second scan):</p>
|
|
<ul>
|
|
<li><code>algorithms::snp_distance::distance_matrix</code> — closed-form <code>snp-*</code>
|
|
corrections (raw p-distance, JC69, K80, K81, F81, T92, TN93, transversions-
|
|
only), optionally gamma-corrected (<code>GammaShape</code>, <code>PartitionDispersion::estimate_alpha</code>
|
|
for <code>Auto</code>).</li>
|
|
<li><code>algorithms::sankoff::SankoffBundle::finish</code> — assembles the
|
|
<code>--sankoff</code>/<code>--tnt</code>/<code>--phyg</code>/<code>--iqtree</code> pipeline's inputs
|
|
(<code>RawSnpDistanceOutput</code>, <code>BasePairTally</code>, <code>CardinalityTally</code>) from an
|
|
already-scanned <code>AlignmentReducer</code>/<code>PairwiseReducer</code> pair, plus
|
|
<code>SankoffBundle::snp_distance</code> to reuse that exact sample for a <code>snp-*</code>
|
|
<code>--distance</code> in the same invocation without a second scan.</li>
|
|
<li><code>algorithms::cardcomp</code> — pure functions turning <code>CardinalityTally</code>/
|
|
<code>BasePairTally</code> into row-stochastic transition probability matrices, then
|
|
the calibrated 16-state Sankoff cost matrix (<code>pairwise_cost_matrix</code>).</li>
|
|
</ul>
|
|
<h2 id="public-api-surface">Public API surface</h2>
|
|
<p><code>extensions::SiblingExt</code> (CLI-facing trait on <code>IndexCache</code>) covers only the
|
|
operations with no cross-invocation sharing concern: <code>build_sibling_annex</code>,
|
|
<code>shannon_entropy_csv</code>, <code>sibling_family_size_histogram</code>/<code>sibling_annex_stats</code>,
|
|
<code>family_overlap</code>. The sampled SNP path (<code>sampled_scan</code>, <code>AlignmentReducer</code>,
|
|
<code>PairwiseReducer</code>, <code>distance_matrix</code>, <code>SankoffBundle</code>) is <strong>not</strong> wrapped in
|
|
trait methods — the CLI (<code>obikmer::cmd::phylo</code>) composes them directly,
|
|
because it alone knows the union of reducers a given invocation needs
|
|
(<code>--pseudo-alignment</code>/Sankoff group/<code>snp-*</code> <code>--distance</code> can combine in one
|
|
call). A per-output trait method would have to either duplicate that
|
|
composition logic or reintroduce the "two independent scans" problem this
|
|
design replaced.</p>
|
|
<h2 id="known-gaps">Known gaps</h2>
|
|
<ul>
|
|
<li><code>sample_index</code>'s <code>skip_layers</code> parameter (checkpointed-resume support) is
|
|
never called with a non-zero value by any current caller — <code>--session</code>
|
|
persistence was removed pending a redesign on top of <code>sampled_scan</code>
|
|
(single shared trunk, not per-driver checkpoint plumbing). The parameter
|
|
stays as the seam a future resume mechanism would use.</li>
|
|
<li>No cheap entropy pre-filter exists yet for the very high (saturated) tail —
|
|
a row-marginal coverage approximation was considered and rejected (cannot
|
|
represent joint multi-base co-occurrence, which is exactly what the
|
|
15-state entropy definition exists to capture).</li>
|
|
</ul>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
</article>
|
|
</div>
|
|
|
|
|
|
<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
|
|
</div>
|
|
|
|
</main>
|
|
|
|
<footer class="md-footer">
|
|
|
|
<div class="md-footer-meta md-typeset">
|
|
<div class="md-footer-meta__inner md-grid">
|
|
<div class="md-copyright">
|
|
|
|
|
|
Made with
|
|
<a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
|
|
Material for MkDocs
|
|
</a>
|
|
|
|
</div>
|
|
|
|
</div>
|
|
</div>
|
|
</footer>
|
|
|
|
</div>
|
|
<div class="md-dialog" data-md-component="dialog">
|
|
<div class="md-dialog__inner md-typeset"></div>
|
|
</div>
|
|
|
|
|
|
|
|
|
|
|
|
<script id="__config" type="application/json">{"annotate": null, "base": "../..", "features": [], "search": "../../assets/javascripts/workers/search.2c215733.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
|
|
|
|
|
|
<script src="../../assets/javascripts/bundle.79ae519e.min.js"></script>
|
|
|
|
<script src="https://unpkg.com/mathjax@3/es5/tex-mml-chtml.js"></script>
|
|
|
|
|
|
</body>
|
|
</html> |