This commit introduces significant functional improvements related to sparse matrix handling, including new storage layers, performance fixes for sparse operations, and bug corrections in matrix logic. Additionally, architectural documentation is cleaned up by removing obsolete entries and modifying structural text related to the Sibling Annex discussion.
1957 lines
62 KiB
HTML
1957 lines
62 KiB
HTML
|
|
<!doctype html>
|
|
<html lang="en" class="no-js">
|
|
<head>
|
|
|
|
<meta charset="utf-8">
|
|
<meta name="viewport" content="width=device-width,initial-scale=1">
|
|
|
|
|
|
|
|
|
|
<link rel="prev" href="../numa_worker_pools/">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<link rel="icon" href="../../assets/images/favicon.png">
|
|
<meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.7.6">
|
|
|
|
|
|
|
|
<title>NUMA-aware partition runner - obikmer</title>
|
|
|
|
|
|
|
|
<link rel="stylesheet" href="../../assets/stylesheets/main.484c7ddc.min.css">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
|
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
|
|
<style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
|
|
|
|
|
|
|
|
<script>__md_scope=new URL("../..",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
|
|
|
|
|
|
|
|
|
|
|
|
</head>
|
|
|
|
|
|
<body dir="ltr">
|
|
|
|
|
|
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
|
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
|
<label class="md-overlay" for="__drawer"></label>
|
|
<div data-md-component="skip">
|
|
|
|
|
|
<a href="#numa-aware-partition-runner" class="md-skip">
|
|
Skip to content
|
|
</a>
|
|
|
|
</div>
|
|
<div data-md-component="announce">
|
|
|
|
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<header class="md-header md-header--shadow" data-md-component="header">
|
|
<nav class="md-header__inner md-grid" aria-label="Header">
|
|
<a href="../.." title="obikmer" class="md-header__button md-logo" aria-label="obikmer" data-md-component="logo">
|
|
|
|
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
|
|
|
|
</a>
|
|
<label class="md-header__button md-icon" for="__drawer">
|
|
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
|
|
</label>
|
|
<div class="md-header__title" data-md-component="header-title">
|
|
<div class="md-header__ellipsis">
|
|
<div class="md-header__topic">
|
|
<span class="md-ellipsis">
|
|
obikmer
|
|
</span>
|
|
</div>
|
|
<div class="md-header__topic" data-md-component="header-topic">
|
|
<span class="md-ellipsis">
|
|
|
|
NUMA-aware partition runner
|
|
|
|
</span>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<script>var palette=__md_get("__palette");if(palette&&palette.color){if("(prefers-color-scheme)"===palette.color.media){var media=matchMedia("(prefers-color-scheme: light)"),input=document.querySelector(media.matches?"[data-md-color-media='(prefers-color-scheme: light)']":"[data-md-color-media='(prefers-color-scheme: dark)']");palette.color.media=input.getAttribute("data-md-color-media"),palette.color.scheme=input.getAttribute("data-md-color-scheme"),palette.color.primary=input.getAttribute("data-md-color-primary"),palette.color.accent=input.getAttribute("data-md-color-accent")}for(var[key,value]of Object.entries(palette.color))document.body.setAttribute("data-md-color-"+key,value)}</script>
|
|
|
|
|
|
|
|
|
|
</nav>
|
|
|
|
</header>
|
|
|
|
<div class="md-container" data-md-component="container">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<main class="md-main" data-md-component="main">
|
|
<div class="md-main__inner md-grid">
|
|
|
|
|
|
|
|
<div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
|
|
<div class="md-sidebar__scrollwrap">
|
|
<div class="md-sidebar__inner">
|
|
|
|
|
|
|
|
|
|
<nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
|
|
<label class="md-nav__title" for="__drawer">
|
|
<a href="../.." title="obikmer" class="md-nav__button md-logo" aria-label="obikmer" data-md-component="logo">
|
|
|
|
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
|
|
|
|
</a>
|
|
obikmer
|
|
</label>
|
|
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../.." class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Home
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../installation/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Installation
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item md-nav__item--nested">
|
|
|
|
|
|
|
|
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3" >
|
|
|
|
|
|
<label class="md-nav__link" for="__nav_3" id="__nav_3_label" tabindex="0">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Theory
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
<span class="md-nav__icon md-icon"></span>
|
|
</label>
|
|
|
|
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_3_label" aria-expanded="false">
|
|
<label class="md-nav__title" for="__nav_3">
|
|
<span class="md-nav__icon md-icon"></span>
|
|
|
|
|
|
Theory
|
|
|
|
|
|
</label>
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../kmers/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Kmers and super-kmers
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../theory/encoding/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
DNA encoding
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../theory/entropy/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Entropy filter
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../theory/minimizer/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Minimizer selection
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../theory/indexing/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Partitioning architecture
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../theory/evolutionary_distances/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Central-position SNP distance (discussion)
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
</ul>
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item md-nav__item--nested">
|
|
|
|
|
|
|
|
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4" >
|
|
|
|
|
|
<label class="md-nav__link" for="__nav_4" id="__nav_4_label" tabindex="0">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Implementation
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
<span class="md-nav__icon md-icon"></span>
|
|
</label>
|
|
|
|
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_4_label" aria-expanded="false">
|
|
<label class="md-nav__title" for="__nav_4">
|
|
<span class="md-nav__icon md-icon"></span>
|
|
|
|
|
|
Implementation
|
|
|
|
|
|
</label>
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../implementation/superkmer/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
SuperKmer
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../implementation/kmer/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Kmer
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../implementation/chunkreader/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Chunk reader
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../implementation/pipeline/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Construction pipeline
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../implementation/obipipeline/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
obipipeline library
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../implementation/storage/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
On-disk storage
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../implementation/mphf/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
MPHF selection
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../implementation/unitig_evidence/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Unitig evidence encoding
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../implementation/evidence_elimination/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Evidence elimination (discussion)
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../implementation/layer_tier/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
obikindex layer tier
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../implementation/persistent_compact_int_vec/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
PersistentCompactIntVec
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../implementation/persistent_bit_vec/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
PersistentBitVec
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../implementation/merge/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Merge command
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../implementation/merge_parallelism/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Merge parallelism & memory
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../implementation/filtering/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Kmer filtering
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../implementation/select/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Select command
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../implementation/obitaxonomy/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
obitaxonomy crate
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../implementation/benchmark_query_testing/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Benchmark: query-path testing
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../implementation/partition_layer_cache/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Partition and layer caching (discussion)
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
</ul>
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item md-nav__item--active md-nav__item--nested">
|
|
|
|
|
|
|
|
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_5" checked>
|
|
|
|
|
|
<label class="md-nav__link" for="__nav_5" id="__nav_5_label" tabindex="0">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Architecture
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
<span class="md-nav__icon md-icon"></span>
|
|
</label>
|
|
|
|
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_5_label" aria-expanded="true">
|
|
<label class="md-nav__title" for="__nav_5">
|
|
<span class="md-nav__icon md-icon"></span>
|
|
|
|
|
|
Architecture
|
|
|
|
|
|
</label>
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../sequences/invariant/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Sequences
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../index_architecture/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Kmer index
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../siblings/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Sibling annex
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../numa_worker_pools/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
NUMA-aware worker pools
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item md-nav__item--active">
|
|
|
|
<input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
|
|
|
|
|
|
|
|
|
|
|
|
<label class="md-nav__link md-nav__link--active" for="__toc">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
NUMA-aware partition runner
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
<span class="md-nav__icon md-icon"></span>
|
|
</label>
|
|
|
|
<a href="./" class="md-nav__link md-nav__link--active">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
NUMA-aware partition runner
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
|
|
|
|
|
|
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<label class="md-nav__title" for="__toc">
|
|
<span class="md-nav__icon md-icon"></span>
|
|
Table of contents
|
|
</label>
|
|
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#problem" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Problem
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#unified-model" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Unified model
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#adaptive-mechanism" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Adaptive mechanism
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#proposed-api" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Proposed API
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#migration-examples" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Migration examples
|
|
|
|
</span>
|
|
</a>
|
|
|
|
<nav class="md-nav" aria-label="Migration examples">
|
|
<ul class="md-nav__list">
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#mergers-before-180-lines-of-bespoke-machinery" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
merge.rs (before: ~180 lines of bespoke machinery)
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#indexrs-build_layers-before-naive-into_par_iter" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
index.rs build_layers (before: naive into_par_iter)
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#placement" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Placement
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#known-issue-cpu-only-activation-signal-stalls-on-io-bound-stages" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Known issue: CPU-only activation signal stalls on I/O-bound stages
|
|
|
|
</span>
|
|
</a>
|
|
|
|
<nav class="md-nav" aria-label="Known issue: CPU-only activation signal stalls on I/O-bound stages">
|
|
<ul class="md-nav__list">
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#implemented-io-signal-shared-debounce-guard" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Implemented: I/O signal + shared debounce guard
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#known-issue-ramp-up-too-slow-and-confused-with-node-count" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Known issue: ramp-up too slow, and confused with node count
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#panic-propagation-fixed-2026-08-28" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Panic propagation (fixed, 2026-08-28)
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#open-questions" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Open questions
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
</ul>
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
|
|
|
|
</ul>
|
|
</nav>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
|
|
<div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
|
|
<div class="md-sidebar__scrollwrap">
|
|
<div class="md-sidebar__inner">
|
|
|
|
|
|
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<label class="md-nav__title" for="__toc">
|
|
<span class="md-nav__icon md-icon"></span>
|
|
Table of contents
|
|
</label>
|
|
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#problem" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Problem
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#unified-model" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Unified model
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#adaptive-mechanism" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Adaptive mechanism
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#proposed-api" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Proposed API
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#migration-examples" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Migration examples
|
|
|
|
</span>
|
|
</a>
|
|
|
|
<nav class="md-nav" aria-label="Migration examples">
|
|
<ul class="md-nav__list">
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#mergers-before-180-lines-of-bespoke-machinery" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
merge.rs (before: ~180 lines of bespoke machinery)
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#indexrs-build_layers-before-naive-into_par_iter" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
index.rs build_layers (before: naive into_par_iter)
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#placement" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Placement
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#known-issue-cpu-only-activation-signal-stalls-on-io-bound-stages" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Known issue: CPU-only activation signal stalls on I/O-bound stages
|
|
|
|
</span>
|
|
</a>
|
|
|
|
<nav class="md-nav" aria-label="Known issue: CPU-only activation signal stalls on I/O-bound stages">
|
|
<ul class="md-nav__list">
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#implemented-io-signal-shared-debounce-guard" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Implemented: I/O signal + shared debounce guard
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#known-issue-ramp-up-too-slow-and-confused-with-node-count" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Known issue: ramp-up too slow, and confused with node count
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#panic-propagation-fixed-2026-08-28" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Panic propagation (fixed, 2026-08-28)
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#open-questions" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Open questions
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
|
|
</nav>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
|
|
<div class="md-content" data-md-component="content">
|
|
|
|
<article class="md-content__inner md-typeset">
|
|
|
|
|
|
|
|
|
|
|
|
<h1 id="numa-aware-partition-runner">NUMA-aware partition runner</h1>
|
|
<h2 id="problem">Problem</h2>
|
|
<p>All partition-level parallel loops in obikindex currently fall into two
|
|
categories:</p>
|
|
<p><strong>Naive Rayon</strong> — used in <code>build_layers</code>, <code>pack_matrices</code>, <code>dump</code>, <code>select</code>,
|
|
<code>stats</code>, <code>rebuild</code>, <code>reindex</code>:</p>
|
|
<div class="highlight"><pre><span></span><code><span class="p">(</span><span class="mi">0</span><span class="o">..</span><span class="n">n</span><span class="p">).</span><span class="n">into_par_iter</span><span class="p">().</span><span class="n">for_each</span><span class="p">(</span><span class="o">|</span><span class="n">i</span><span class="o">|</span><span class="w"> </span><span class="n">work</span><span class="p">(</span><span class="n">i</span><span class="p">));</span>
|
|
</code></pre></div>
|
|
<p>Threads come from the global Rayon pool with no NUMA awareness. On
|
|
multi-socket machines this produces cross-socket memory traffic and degrades
|
|
performance super-linearly (see <a href="../numa_worker_pools/">NUMA-aware worker pools</a>).</p>
|
|
<p><strong>Ad-hoc adaptive pool</strong> — used in <code>merge</code>:</p>
|
|
<p>A bespoke implementation with pre-spawned workers, channel-based dispatch, and
|
|
activation control. It handles NUMA correctly but is not reusable.</p>
|
|
<p>Both cases should be replaced by a single generic mechanism.</p>
|
|
<h2 id="unified-model">Unified model</h2>
|
|
<p>The key insight is that <strong>UMA is just the NUMA case with a single node</strong>. The
|
|
runner always works the same way: one controller thread per node, each
|
|
independently managing its own workers with the same adaptive logic. The only
|
|
difference between UMA and NUMA is the number of nodes and whether workers are
|
|
pinned.</p>
|
|
<div class="highlight"><pre><span></span><code>NUMA (k nodes) UMA (1 node)
|
|
|
|
controller-0 controller-1 … controller-0
|
|
│ │ │
|
|
workers[0] workers[1] workers[0]
|
|
(pinned) (pinned) (global pool)
|
|
└───────────────┴──────────────────┘
|
|
shared work queue
|
|
</code></pre></div>
|
|
<p>On each node, the Rayon <code>ThreadPool</code> is pinned to that node's CPUs.
|
|
<code>pool.install()</code> ensures all internal Rayon calls (inside the work function)
|
|
use the node-local pool. Linux first-touch then places heap allocations in
|
|
local DRAM automatically.</p>
|
|
<p>On UMA the global Rayon pool is used directly — no pinning, no overhead.</p>
|
|
<h2 id="adaptive-mechanism">Adaptive mechanism</h2>
|
|
<p>Each controller follows the same logic regardless of node count:</p>
|
|
<ol>
|
|
<li>Pre-spawn <code>workers_per_node</code> dormant worker threads (blocked on <code>activate_rx</code>).</li>
|
|
<li>Activate the first worker immediately.</li>
|
|
<li>Loop on result channel with a <code>SPAWN_POLL</code> timeout:</li>
|
|
<li>On result: call <code>on_done</code>; check whether to activate the next worker.</li>
|
|
<li>On timeout: same check.</li>
|
|
<li>Activation criterion: <code>should_spawn_worker(active, global_efficiency, prev_efficiency)</code>.</li>
|
|
<li>Drop <code>activate_tx</code> when done — dormant workers exit cleanly.</li>
|
|
</ol>
|
|
<p><strong>Global CPU efficiency</strong> (<code>CpuSample</code>, reads <code>/proc/stat</code> on Linux) is used by
|
|
all controllers — no per-node measurement needed. The signal is coarser than
|
|
per-node efficiency but correct in practice: if any node saturates memory
|
|
bandwidth, the global efficiency drops and all controllers stop activating
|
|
workers. Using a standard portable primitive avoids platform-specific CPU
|
|
accounting and keeps the implementation clean.</p>
|
|
<h2 id="proposed-api">Proposed API</h2>
|
|
<div class="highlight"><pre><span></span><code><span class="k">pub</span><span class="w"> </span><span class="k">struct</span><span class="w"> </span><span class="nc">PartitionRunner</span><span class="w"> </span><span class="p">{</span>
|
|
<span class="w"> </span><span class="c1">// One entry per NUMA node; one entry total on UMA.</span>
|
|
<span class="w"> </span><span class="n">nodes</span><span class="p">:</span><span class="w"> </span><span class="nb">Vec</span><span class="o"><</span><span class="n">NodeConfig</span><span class="o">></span><span class="p">,</span>
|
|
<span class="p">}</span>
|
|
|
|
<span class="k">struct</span><span class="w"> </span><span class="nc">NodeConfig</span><span class="w"> </span><span class="p">{</span>
|
|
<span class="w"> </span><span class="n">pool</span><span class="p">:</span><span class="w"> </span><span class="nb">Option</span><span class="o"><</span><span class="n">Arc</span><span class="o"><</span><span class="n">rayon</span><span class="p">::</span><span class="n">ThreadPool</span><span class="o">>></span><span class="p">,</span><span class="w"> </span><span class="c1">// None = global Rayon pool (UMA)</span>
|
|
<span class="w"> </span><span class="n">cpu_ids</span><span class="p">:</span><span class="w"> </span><span class="nb">Vec</span><span class="o"><</span><span class="kt">usize</span><span class="o">></span><span class="p">,</span><span class="w"> </span><span class="c1">// empty = no pinning (UMA)</span>
|
|
<span class="w"> </span><span class="n">max_workers</span><span class="p">:</span><span class="w"> </span><span class="kt">usize</span><span class="p">,</span>
|
|
<span class="p">}</span>
|
|
|
|
<span class="k">impl</span><span class="w"> </span><span class="n">PartitionRunner</span><span class="w"> </span><span class="p">{</span>
|
|
<span class="w"> </span><span class="sd">/// Detect topology and build the runner.</span>
|
|
<span class="w"> </span><span class="sd">/// Returns a single-node runner on UMA / macOS / hwloc failure.</span>
|
|
<span class="w"> </span><span class="k">pub</span><span class="w"> </span><span class="k">fn</span><span class="w"> </span><span class="nf">new</span><span class="p">()</span><span class="w"> </span><span class="p">-></span><span class="w"> </span><span class="nc">Self</span><span class="p">;</span>
|
|
|
|
<span class="w"> </span><span class="sd">/// Run `f(i)` for every index in `order`, collecting results.</span>
|
|
<span class="w"> </span><span class="sd">///</span>
|
|
<span class="w"> </span><span class="sd">/// `on_done(i, result, elapsed)` is called under an internal mutex as</span>
|
|
<span class="w"> </span><span class="sd">/// each partition completes — use it for progress bars and aggregation.</span>
|
|
<span class="w"> </span><span class="sd">/// The runner serialises all calls to `on_done` via an internal</span>
|
|
<span class="w"> </span><span class="sd">/// `Arc<Mutex<C>>`, so no `Sync` bound is required on the callback.</span>
|
|
<span class="w"> </span><span class="sd">/// `Send` is required because the Arc clone crosses thread boundaries.</span>
|
|
<span class="w"> </span><span class="sd">///</span>
|
|
<span class="w"> </span><span class="sd">/// Serialisation is free in practice: a partition takes seconds to</span>
|
|
<span class="w"> </span><span class="sd">/// minutes; the callback takes microseconds. Contention is negligible.</span>
|
|
<span class="w"> </span><span class="sd">///</span>
|
|
<span class="w"> </span><span class="sd">/// Returns the first error from `f`, if any.</span>
|
|
<span class="w"> </span><span class="k">pub</span><span class="w"> </span><span class="k">fn</span><span class="w"> </span><span class="nf">run</span><span class="o"><</span><span class="n">F</span><span class="p">,</span><span class="w"> </span><span class="n">R</span><span class="p">,</span><span class="w"> </span><span class="n">E</span><span class="p">,</span><span class="w"> </span><span class="n">C</span><span class="o">></span><span class="p">(</span>
|
|
<span class="w"> </span><span class="o">&</span><span class="bp">self</span><span class="p">,</span>
|
|
<span class="w"> </span><span class="n">order</span><span class="p">:</span><span class="w"> </span><span class="kp">&</span><span class="p">[</span><span class="kt">usize</span><span class="p">],</span>
|
|
<span class="w"> </span><span class="n">f</span><span class="p">:</span><span class="w"> </span><span class="nc">F</span><span class="p">,</span>
|
|
<span class="w"> </span><span class="n">on_done</span><span class="p">:</span><span class="w"> </span><span class="nc">C</span><span class="p">,</span>
|
|
<span class="w"> </span><span class="p">)</span><span class="w"> </span><span class="p">-></span><span class="w"> </span><span class="nb">Result</span><span class="o"><</span><span class="p">(),</span><span class="w"> </span><span class="n">E</span><span class="o">></span>
|
|
<span class="w"> </span><span class="k">where</span>
|
|
<span class="w"> </span><span class="n">F</span><span class="p">:</span><span class="w"> </span><span class="nb">Fn</span><span class="p">(</span><span class="kt">usize</span><span class="p">)</span><span class="w"> </span><span class="p">-></span><span class="w"> </span><span class="nb">Result</span><span class="o"><</span><span class="n">R</span><span class="p">,</span><span class="w"> </span><span class="n">E</span><span class="o">></span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="nb">Send</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="nb">Sync</span><span class="p">,</span>
|
|
<span class="w"> </span><span class="n">R</span><span class="p">:</span><span class="w"> </span><span class="nb">Send</span><span class="p">,</span>
|
|
<span class="w"> </span><span class="n">E</span><span class="p">:</span><span class="w"> </span><span class="nb">Send</span><span class="p">,</span>
|
|
<span class="w"> </span><span class="n">C</span><span class="p">:</span><span class="w"> </span><span class="nb">FnMut</span><span class="p">(</span><span class="kt">usize</span><span class="p">,</span><span class="w"> </span><span class="n">R</span><span class="p">,</span><span class="w"> </span><span class="n">Duration</span><span class="p">)</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="nb">Send</span><span class="p">;</span><span class="w"> </span><span class="c1">// Send required, Sync is not</span>
|
|
<span class="p">}</span>
|
|
</code></pre></div>
|
|
<p><code>order</code> is caller-supplied so each command chooses its scheduling strategy:
|
|
largest-first for <code>merge</code>, sequential for <code>build_layers</code>, etc.</p>
|
|
<h2 id="migration-examples">Migration examples</h2>
|
|
<h3 id="mergers-before-180-lines-of-bespoke-machinery">merge.rs (before: ~180 lines of bespoke machinery)</h3>
|
|
<div class="highlight"><pre><span></span><code><span class="kd">let</span><span class="w"> </span><span class="n">runner</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">PartitionRunner</span><span class="p">::</span><span class="n">new</span><span class="p">();</span>
|
|
<span class="n">runner</span><span class="p">.</span><span class="n">run</span><span class="p">(</span>
|
|
<span class="w"> </span><span class="o">&</span><span class="n">order</span><span class="p">,</span>
|
|
<span class="w"> </span><span class="o">|</span><span class="n">i</span><span class="o">|</span><span class="w"> </span><span class="n">dst_partition</span><span class="p">.</span><span class="n">merge_partition</span><span class="p">(</span><span class="n">i</span><span class="p">,</span><span class="w"> </span><span class="n">srcs</span><span class="p">,</span><span class="w"> </span><span class="n">mode</span><span class="p">,</span><span class="w"> </span><span class="n">n_dst_genomes</span><span class="p">,</span><span class="w"> </span><span class="n">block_bits</span><span class="p">,</span><span class="w"> </span><span class="n">evidence</span><span class="p">)</span>
|
|
<span class="w"> </span><span class="p">.</span><span class="n">map_err</span><span class="p">(</span><span class="n">OKIError</span><span class="p">::</span><span class="n">Partition</span><span class="p">),</span>
|
|
<span class="w"> </span><span class="o">|</span><span class="n">i</span><span class="p">,</span><span class="w"> </span><span class="n">g_len</span><span class="p">,</span><span class="w"> </span><span class="n">dur</span><span class="o">|</span><span class="w"> </span><span class="p">{</span>
|
|
<span class="w"> </span><span class="n">pb</span><span class="p">.</span><span class="n">inc</span><span class="p">(</span><span class="mi">1</span><span class="p">);</span>
|
|
<span class="w"> </span><span class="n">debug</span><span class="o">!</span><span class="p">(</span><span class="s">"partition {i}: done in {:.1}s — {g_len} new kmers"</span><span class="p">,</span><span class="w"> </span><span class="n">dur</span><span class="p">.</span><span class="n">as_secs_f64</span><span class="p">());</span>
|
|
<span class="w"> </span><span class="n">part_stats</span><span class="p">.</span><span class="n">push</span><span class="p">(</span><span class="n">PartStat</span><span class="w"> </span><span class="p">{</span><span class="w"> </span><span class="n">id</span><span class="p">:</span><span class="w"> </span><span class="nc">i</span><span class="p">,</span><span class="w"> </span><span class="n">unitig_bytes</span><span class="p">:</span><span class="w"> </span><span class="nc">partition_sizes</span><span class="p">[</span><span class="n">i</span><span class="p">],</span><span class="w"> </span><span class="n">g_len</span><span class="w"> </span><span class="p">});</span>
|
|
<span class="w"> </span><span class="p">},</span>
|
|
<span class="p">)</span><span class="o">?</span><span class="p">;</span>
|
|
</code></pre></div>
|
|
<h3 id="indexrs-build_layers-before-naive-into_par_iter">index.rs build_layers (before: naive into_par_iter)</h3>
|
|
<div class="highlight"><pre><span></span><code><span class="kd">let</span><span class="w"> </span><span class="n">order</span><span class="p">:</span><span class="w"> </span><span class="nb">Vec</span><span class="o"><</span><span class="kt">usize</span><span class="o">></span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">(</span><span class="mi">0</span><span class="o">..</span><span class="n">n</span><span class="p">).</span><span class="n">collect</span><span class="p">();</span>
|
|
<span class="kd">let</span><span class="w"> </span><span class="n">runner</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">PartitionRunner</span><span class="p">::</span><span class="n">new</span><span class="p">();</span>
|
|
<span class="n">runner</span><span class="p">.</span><span class="n">run</span><span class="p">(</span>
|
|
<span class="w"> </span><span class="o">&</span><span class="n">order</span><span class="p">,</span>
|
|
<span class="w"> </span><span class="o">|</span><span class="n">i</span><span class="o">|</span><span class="w"> </span><span class="bp">self</span><span class="p">.</span><span class="n">partition</span><span class="p">.</span><span class="n">build_index_layer</span><span class="p">(</span><span class="n">i</span><span class="p">,</span><span class="w"> </span><span class="n">min_ab</span><span class="p">,</span><span class="w"> </span><span class="n">max_ab</span><span class="p">,</span><span class="w"> </span><span class="n">with_counts</span><span class="p">,</span><span class="w"> </span><span class="o">&</span><span class="n">evidence</span><span class="p">,</span><span class="w"> </span><span class="n">block_bits</span><span class="p">)</span>
|
|
<span class="w"> </span><span class="p">.</span><span class="n">map_err</span><span class="p">(</span><span class="n">OKIError</span><span class="p">::</span><span class="n">Partition</span><span class="p">),</span>
|
|
<span class="w"> </span><span class="o">|</span><span class="n">_</span><span class="p">,</span><span class="w"> </span><span class="n">n_kmers</span><span class="p">,</span><span class="w"> </span><span class="n">_</span><span class="o">|</span><span class="w"> </span><span class="p">{</span>
|
|
<span class="w"> </span><span class="n">total_kmers</span><span class="p">.</span><span class="n">fetch_add</span><span class="p">(</span><span class="n">n_kmers</span><span class="p">,</span><span class="w"> </span><span class="n">Ordering</span><span class="p">::</span><span class="n">Relaxed</span><span class="p">);</span>
|
|
<span class="w"> </span><span class="n">pb</span><span class="p">.</span><span class="n">inc</span><span class="p">(</span><span class="mi">1</span><span class="p">);</span>
|
|
<span class="w"> </span><span class="p">},</span>
|
|
<span class="p">)</span><span class="o">?</span><span class="p">;</span>
|
|
</code></pre></div>
|
|
<p>All other sites (<code>pack_matrices</code>, <code>dump</code>, <code>select</code>, etc.) follow the same
|
|
pattern.</p>
|
|
<h2 id="placement">Placement</h2>
|
|
<p><code>PartitionRunner</code> lives in <code>obikindex/src/numa.rs</code> alongside <code>NumaSetup</code>.
|
|
It depends only on standard library primitives and Rayon — no new dependencies.</p>
|
|
<p>A single <code>PartitionRunner</code> instance can be built once per command invocation
|
|
and reused across multiple <code>run()</code> calls (e.g. <code>merge</code> runs
|
|
<code>merge_partitions</code> then <code>pack_matrices</code>).</p>
|
|
<h2 id="known-issue-cpu-only-activation-signal-stalls-on-io-bound-stages">Known issue: CPU-only activation signal stalls on I/O-bound stages</h2>
|
|
<p>Observed on a real <code>filter</code> run (109 genomes, 256 partitions, 8×24-core NUMA):
|
|
<code>rebuild</code> (CPU-bound — k-mer construction) scales cleanly from 9 to 43 active
|
|
workers as <code>CpuSample::do_i_activate</code> (<code>obisys::lib.rs</code>) sees efficiency climb.
|
|
<code>pack_matrices</code> (I/O-bound — reopens and recomposes per-genome column files
|
|
into <code>.pbmx</code>/<code>.pcmx</code>) activates one extra worker then flatlines at 10/192 for
|
|
the rest of the stage, even though 256 partitions keep completing over several
|
|
minutes. This matches the documented intent (§ Adaptive mechanism — "avoids
|
|
over-provisioning ... I/O-bound ... workloads") but conflates two different
|
|
things: <em>"CPU is not the bottleneck"</em> and <em>"more workers would not help"</em>. On
|
|
storage with real queue depth (NVMe, RAID, parallel FS) the second stage could
|
|
still benefit from more concurrent workers even with flat CPU usage — a signal
|
|
the current mechanism cannot see.</p>
|
|
<p>A one-off artefact was also found in the same log: right after a stage
|
|
transition, <code>do_i_activate</code> produced a physically impossible spike (efficiency
|
|
~94 cores on a 192-core box) because it has no minimum-window guard — unlike
|
|
its sibling <code>cpu_efficiency</code>, which returns <code>0.0</code> if <code>wall < 0.1s</code>
|
|
(<code>obisys::lib.rs:260</code>). <code>do_i_activate</code> unconditionally overwrites
|
|
<code>self.wall</code>/<code>self.user_secs</code>/<code>self.sys_secs</code> even when the elapsed window is
|
|
too short to be meaningful, so a burst of rapid completions right after
|
|
activating a worker can divide a real CPU delta by a near-zero wall delta.</p>
|
|
<h3 id="implemented-io-signal-shared-debounce-guard">Implemented: I/O signal + shared debounce guard</h3>
|
|
<p><code>IoSample</code> (<code>obisys::lib.rs</code>, alongside <code>CpuSample</code>) is fed by
|
|
<code>read_bytes</code>/<code>write_bytes</code> from <code>/proc/self/io</code> on Linux (actual bytes
|
|
submitted to the block layer — not <code>rchar</code>/<code>wchar</code>, which also count
|
|
page-cache hits, and not <code>ru_inblock</code>/<code>ru_oublock</code>, unreliable on macOS), with
|
|
a <code>proc_pid_rusage(RUSAGE_INFO_V4)</code> fallback on macOS
|
|
(<code>ri_diskio_bytesread</code>/<code>ri_diskio_byteswritten</code>, FFI only via <code>libc</code>, no new
|
|
dependency — same pattern as the existing <code>getrusage</code> bindings). Any other
|
|
target degrades gracefully to a signal that never triggers (falls back to
|
|
CPU-only activation), same pattern as <code>cgroup_v2_available</code>.</p>
|
|
<p><code>maybe_activate</code> (<code>numa.rs</code>) activates a worker if <em>either</em> signal still shows
|
|
headroom, making <code>PartitionRunner</code> adapt to whichever resource is actually the
|
|
bottleneck without per-call configuration. Both samplers are called
|
|
unconditionally — no <code>||</code> short-circuit — so neither window starves behind
|
|
whichever signal fires first:</p>
|
|
<div class="highlight"><pre><span></span><code><span class="kd">let</span><span class="w"> </span><span class="n">cpu_threshold</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">CPU_SPAWN_THRESHOLD</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="n">activation</span><span class="p">.</span><span class="n">last_step</span><span class="p">()</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="kt">f64</span><span class="p">;</span>
|
|
<span class="kd">let</span><span class="w"> </span><span class="n">cpu_wants_more</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">cpu_sample</span><span class="p">.</span><span class="n">do_i_activate</span><span class="p">(</span><span class="n">cpu_threshold</span><span class="p">);</span>
|
|
<span class="kd">let</span><span class="w"> </span><span class="n">io_wants_more</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">io_sample</span><span class="p">.</span><span class="n">do_i_activate</span><span class="p">(</span><span class="n">IO_SPAWN_THRESHOLD</span><span class="p">);</span>
|
|
<span class="k">if</span><span class="w"> </span><span class="n">cpu_wants_more</span><span class="w"> </span><span class="o">||</span><span class="w"> </span><span class="n">io_wants_more</span><span class="w"> </span><span class="p">{</span>
|
|
<span class="w"> </span><span class="n">activation</span><span class="p">.</span><span class="n">grow</span><span class="p">(</span><span class="n">GROWTH_DIVISOR</span><span class="p">,</span><span class="w"> </span><span class="n">n_total</span><span class="p">);</span>
|
|
<span class="p">}</span>
|
|
</code></pre></div>
|
|
<p>The CPU threshold is <em>not</em> the flat absolute delta it started as: it scales
|
|
with <code>activation.last_step()</code> — the number of workers activated in the last
|
|
growth step, tracked by <code>NodeActivation</code> (<code>numa.rs</code>) and updated every time
|
|
<code>grow()</code> actually grows something. Growing by 8 workers should add ~8 cores of
|
|
efficiency if the workload is truly CPU-bound; requiring only
|
|
<code>CPU_SPAWN_THRESHOLD</code> (20 %) of that expected gain confirms the growth was
|
|
useful without demanding perfect linear scaling. Scaling by the <em>last step's
|
|
size</em> rather than the cumulative total keeps the bar equally meaningful
|
|
whether it's the 2nd growth step or the 20th — a flat absolute threshold
|
|
(0.2 core) is a strong signal at 8 active workers but pure noise at 150; a
|
|
threshold scaled by the <em>cumulative</em> total instead (considered and rejected)
|
|
would have made the bar essentially impossible to clear late in the ramp,
|
|
strangling exactly the CPU-bound saturation the mechanism exists to allow.</p>
|
|
<p>Unlike the CPU signal (an absolute delta in cores — a bounded, portable unit),
|
|
raw I/O throughput has no natural scale across devices, so <code>IoSample</code> uses a
|
|
<strong>relative</strong> growth threshold instead of an absolute one:</p>
|
|
<div class="highlight"><pre><span></span><code><span class="k">pub</span><span class="w"> </span><span class="k">fn</span><span class="w"> </span><span class="nf">do_i_activate</span><span class="p">(</span><span class="o">&</span><span class="k">mut</span><span class="w"> </span><span class="bp">self</span><span class="p">,</span><span class="w"> </span><span class="n">threshold</span><span class="p">:</span><span class="w"> </span><span class="kt">f64</span><span class="p">)</span><span class="w"> </span><span class="p">-></span><span class="w"> </span><span class="kt">bool</span><span class="w"> </span><span class="p">{</span>
|
|
<span class="w"> </span><span class="kd">let</span><span class="w"> </span><span class="n">elapsed</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="bp">self</span><span class="p">.</span><span class="n">wall</span><span class="p">.</span><span class="n">elapsed</span><span class="p">().</span><span class="n">as_secs_f64</span><span class="p">();</span>
|
|
<span class="w"> </span><span class="k">if</span><span class="w"> </span><span class="n">elapsed</span><span class="w"> </span><span class="o"><</span><span class="w"> </span><span class="mf">0.1</span><span class="w"> </span><span class="p">{</span><span class="w"> </span><span class="k">return</span><span class="w"> </span><span class="kc">false</span><span class="p">;</span><span class="w"> </span><span class="p">}</span><span class="w"> </span><span class="c1">// state untouched — window keeps accumulating</span>
|
|
|
|
<span class="w"> </span><span class="kd">let</span><span class="w"> </span><span class="n">n</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="bp">Self</span><span class="p">::</span><span class="n">read_bytes</span><span class="p">();</span>
|
|
<span class="w"> </span><span class="kd">let</span><span class="w"> </span><span class="n">rate</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">n</span><span class="p">.</span><span class="n">saturating_sub</span><span class="p">(</span><span class="bp">self</span><span class="p">.</span><span class="n">bytes</span><span class="p">)</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="kt">f64</span><span class="w"> </span><span class="o">/</span><span class="w"> </span><span class="n">elapsed</span><span class="p">;</span>
|
|
<span class="w"> </span><span class="kd">let</span><span class="w"> </span><span class="n">activate</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="k">if</span><span class="w"> </span><span class="bp">self</span><span class="p">.</span><span class="n">previous_rate</span><span class="w"> </span><span class="o">==</span><span class="w"> </span><span class="mf">0.0</span><span class="w"> </span><span class="p">{</span>
|
|
<span class="w"> </span><span class="n">rate</span><span class="w"> </span><span class="o">></span><span class="w"> </span><span class="mf">0.0</span><span class="w"> </span><span class="c1">// bootstrap: any measured throughput is signal</span>
|
|
<span class="w"> </span><span class="p">}</span><span class="w"> </span><span class="k">else</span><span class="w"> </span><span class="p">{</span>
|
|
<span class="w"> </span><span class="p">(</span><span class="n">rate</span><span class="w"> </span><span class="o">-</span><span class="w"> </span><span class="bp">self</span><span class="p">.</span><span class="n">previous_rate</span><span class="p">)</span><span class="w"> </span><span class="o">/</span><span class="w"> </span><span class="bp">self</span><span class="p">.</span><span class="n">previous_rate</span><span class="w"> </span><span class="o">>=</span><span class="w"> </span><span class="n">threshold</span>
|
|
<span class="w"> </span><span class="p">};</span>
|
|
|
|
<span class="w"> </span><span class="bp">self</span><span class="p">.</span><span class="n">bytes</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">n</span><span class="p">;</span>
|
|
<span class="w"> </span><span class="bp">self</span><span class="p">.</span><span class="n">wall</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">Instant</span><span class="p">::</span><span class="n">now</span><span class="p">();</span><span class="w"> </span><span class="c1">// reset only on a real sample</span>
|
|
<span class="w"> </span><span class="n">activate</span>
|
|
<span class="p">}</span>
|
|
</code></pre></div>
|
|
<p>The <code>elapsed < 0.1s → return false without mutating state</code> guard was also
|
|
back-ported into <code>CpuSample::do_i_activate</code> (previously missing — source of
|
|
the ~94-core artefact above) — one fix for both problems, and it removes the
|
|
need for any arbitrary I/O-rate floor: a short/noisy window is rejected
|
|
outright rather than papered over with a hardware-dependent constant.</p>
|
|
<p>Both spawn thresholds (<code>CPU_SPAWN_THRESHOLD</code>, <code>IO_SPAWN_THRESHOLD</code>, module-level
|
|
<code>const</code> in <code>numa.rs</code>, both <code>0.2</code>) are a starting point, not a derived value:
|
|
<code>0.2</code> (20 % relative growth) for <code>IoSample</code> was chosen to match the CPU
|
|
threshold's <em>implicit</em> relative sensitivity (in the observed log, an 8→9
|
|
worker step raised efficiency by ~12 %) — but I/O throughput is lumpier than
|
|
CPU time (buffered writes flush in bursts), so it needs empirical validation
|
|
against a real <code>pack</code> run before being considered final.</p>
|
|
<h2 id="known-issue-ramp-up-too-slow-and-confused-with-node-count">Known issue: ramp-up too slow, and confused with node count</h2>
|
|
<p>The original design started <code>n_nodes</code> workers (one per node) and grew one
|
|
worker at a time. On a real <code>filter</code> run this took ~10 minutes to climb from
|
|
9 to ~40 active workers even on the CPU-bound <code>rebuild</code> stage — most of a
|
|
35-minute stage spent under-provisioned while waiting for evidence to
|
|
accumulate one worker at a time. There is no scale-down mechanism (<code>n_active</code>
|
|
only grows), so the original caution was deliberate — but a quarter of
|
|
available cores is still far from saturation, and the real risk zone (over-provisioning
|
|
a memory-bandwidth-bound stage) only shows up much later in the ramp, near
|
|
full occupancy — not at 25 %.</p>
|
|
<p>The fix decouples ramp speed from node <em>count</em>: both the initial size and the
|
|
growth step are a fraction of <code>workers_per_node</code> (node <em>size</em>), applied
|
|
identically on every node. A single-NUMA-node (UMA) machine ramps exactly as
|
|
fast as an 8-node one — growing by <code>n_nodes</code> per step, as first considered,
|
|
would have degenerated to "grow by 1" on UMA, reproducing the original
|
|
problem for exactly the machines that need the fix most.</p>
|
|
<div class="highlight"><pre><span></span><code><span class="c1">// NodeActivation::grow — called both at startup (activate_initial) and on</span>
|
|
<span class="c1">// every CPU/IO-triggered growth step, with a different divisor each time.</span>
|
|
<span class="kd">let</span><span class="w"> </span><span class="n">wanted</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">(</span><span class="bp">self</span><span class="p">.</span><span class="n">caps</span><span class="p">[</span><span class="n">idx</span><span class="p">]</span><span class="w"> </span><span class="o">/</span><span class="w"> </span><span class="n">divisor</span><span class="p">).</span><span class="n">max</span><span class="p">(</span><span class="mi">1</span><span class="p">);</span><span class="w"> </span><span class="c1">// INITIAL_DIVISOR=4 at startup, GROWTH_DIVISOR=8 per step</span>
|
|
<span class="kd">let</span><span class="w"> </span><span class="n">room</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="bp">self</span><span class="p">.</span><span class="n">caps</span><span class="p">[</span><span class="n">idx</span><span class="p">].</span><span class="n">saturating_sub</span><span class="p">(</span><span class="bp">self</span><span class="p">.</span><span class="n">active</span><span class="p">[</span><span class="n">idx</span><span class="p">]);</span>
|
|
<span class="kd">let</span><span class="w"> </span><span class="n">grow</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">wanted</span><span class="p">.</span><span class="n">min</span><span class="p">(</span><span class="n">room</span><span class="p">).</span><span class="n">min</span><span class="p">(</span><span class="n">n_total</span><span class="p">.</span><span class="n">saturating_sub</span><span class="p">(</span><span class="bp">self</span><span class="p">.</span><span class="n">total</span><span class="p">));</span>
|
|
</code></pre></div>
|
|
<p>This also fixed a latent correctness gap: the original single shared
|
|
<code>activate_tx</code>/<code>activate_rx</code> pair had <em>no</em> per-node addressing — sending one
|
|
activation signal woke up whichever dormant worker (from any node) happened
|
|
to win the race on that channel. <code>crossbeam_channel</code> gives no fairness
|
|
guarantee across competing receivers, so "round-robin across nodes" was an
|
|
assumption the code never actually enforced. <code>PartitionRunner::run</code> now opens
|
|
one activation channel per node (<code>activate_txs</code>/<code>activate_rxs</code>, one pair per
|
|
<code>NodeConfig</code>); <code>NodeActivation</code> (<code>numa.rs</code>) tracks how many of each node's
|
|
dormant workers have been woken and grows every node by the same amount per
|
|
step, capped by that node's remaining dormant workers and by the run's total
|
|
budget (<code>n_total</code>) — balance across nodes is now guaranteed by construction,
|
|
not incidental to channel implementation details.</p>
|
|
<h2 id="panic-propagation-fixed-2026-08-28">Panic propagation (fixed, 2026-08-28)</h2>
|
|
<p>A worker whose <code>f(i)</code> panicked used to never send its <code>WorkerEvent::Completed</code>
|
|
— the controller's <code>while completed < n_total</code> loop then waited forever for
|
|
an event that partition could no longer produce, since other live
|
|
workers/timer threads kept <code>event_rx</code> open. Discovered via <code>obikselect</code>
|
|
panicking on an unimplemented <code>Sparse</code> matrix case (see <code>implementation/select.md</code>):
|
|
the process hung indefinitely instead of erroring out.</p>
|
|
<p>Fixed: each worker wraps <code>f(i)</code> in <code>std::panic::catch_unwind</code> and sends a new
|
|
<code>WorkerEvent::Panicked(i, payload)</code> instead of silently dropping the
|
|
partition. The controller counts it toward <code>completed</code> (unblocking the loop)
|
|
and keeps the first payload seen; once <code>run</code> returns, <code>panic::resume_unwind</code>
|
|
re-raises it on the caller's thread — the original message/backtrace still
|
|
surfaces, from the right place, instead of a silent deadlock. Takes priority
|
|
over a plain <code>Err</code> (a panic means a bug, not a normal typed failure).</p>
|
|
<h2 id="open-questions">Open questions</h2>
|
|
<ul>
|
|
<li>
|
|
<p><strong>Error handling</strong>: <code>run</code> currently returns the first error; remaining errors
|
|
are dropped. A <code>Vec<E></code> return would give complete diagnostics. (Panics are
|
|
now caught and re-raised individually — see above — this only concerns
|
|
plain <code>Err</code> results.)</p>
|
|
</li>
|
|
<li>
|
|
<p><strong><code>INITIAL_DIVISOR</code> / <code>GROWTH_DIVISOR</code> tuning</strong>: currently <code>4</code> and <code>8</code>
|
|
(start at 1/4 of a node's cores, grow by 1/8 per step), chosen to fix an
|
|
observed too-slow ramp — not yet validated against a real <code>pack</code> (I/O-bound)
|
|
run, where over-provisioning risk is different from the CPU-bound <code>rebuild</code>
|
|
case this was tuned against.</p>
|
|
</li>
|
|
<li>
|
|
<p><strong><code>on_done</code> ordering</strong>: the runner serialises calls to <code>on_done</code> via an
|
|
internal <code>Arc<Mutex<C>></code>. <code>Send</code> is required (the Arc clone crosses thread
|
|
boundaries); <code>Sync</code> is not (only one thread holds the lock at a time).
|
|
Contention is negligible because a partition takes seconds while the callback
|
|
takes microseconds. The callback is therefore simple to write (plain
|
|
<code>Vec::push</code>, plain <code>FnMut</code>) with no measurable performance cost.</p>
|
|
</li>
|
|
</ul>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
</article>
|
|
</div>
|
|
|
|
|
|
<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
|
|
</div>
|
|
|
|
</main>
|
|
|
|
<footer class="md-footer">
|
|
|
|
<div class="md-footer-meta md-typeset">
|
|
<div class="md-footer-meta__inner md-grid">
|
|
<div class="md-copyright">
|
|
|
|
|
|
Made with
|
|
<a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
|
|
Material for MkDocs
|
|
</a>
|
|
|
|
</div>
|
|
|
|
</div>
|
|
</div>
|
|
</footer>
|
|
|
|
</div>
|
|
<div class="md-dialog" data-md-component="dialog">
|
|
<div class="md-dialog__inner md-typeset"></div>
|
|
</div>
|
|
|
|
|
|
|
|
|
|
|
|
<script id="__config" type="application/json">{"annotate": null, "base": "../..", "features": [], "search": "../../assets/javascripts/workers/search.2c215733.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
|
|
|
|
|
|
<script src="../../assets/javascripts/bundle.79ae519e.min.js"></script>
|
|
|
|
<script src="https://unpkg.com/mathjax@3/es5/tex-mml-chtml.js"></script>
|
|
|
|
|
|
</body>
|
|
</html> |