1829 lines
40 KiB
HTML
1829 lines
40 KiB
HTML
|
||
<!doctype html>
|
||
<html lang="en" class="no-js">
|
||
<head>
|
||
|
||
<meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||
|
||
|
||
|
||
|
||
<link rel="prev" href="../merge/">
|
||
|
||
|
||
<link rel="next" href="../filtering/">
|
||
|
||
|
||
|
||
|
||
|
||
<link rel="icon" href="../../assets/images/favicon.png">
|
||
<meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.7.6">
|
||
|
||
|
||
|
||
<title>Merge parallelism & memory - obikmer</title>
|
||
|
||
|
||
|
||
<link rel="stylesheet" href="../../assets/stylesheets/main.484c7ddc.min.css">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
||
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
|
||
<style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
|
||
|
||
|
||
|
||
<script>__md_scope=new URL("../..",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
|
||
|
||
|
||
|
||
|
||
|
||
</head>
|
||
|
||
|
||
<body dir="ltr">
|
||
|
||
|
||
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
||
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
||
<label class="md-overlay" for="__drawer"></label>
|
||
<div data-md-component="skip">
|
||
|
||
|
||
<a href="#merge-parallelism-and-memory-pressure" class="md-skip">
|
||
Skip to content
|
||
</a>
|
||
|
||
</div>
|
||
<div data-md-component="announce">
|
||
|
||
</div>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<header class="md-header md-header--shadow" data-md-component="header">
|
||
<nav class="md-header__inner md-grid" aria-label="Header">
|
||
<a href="../.." title="obikmer" class="md-header__button md-logo" aria-label="obikmer" data-md-component="logo">
|
||
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
|
||
|
||
</a>
|
||
<label class="md-header__button md-icon" for="__drawer">
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
|
||
</label>
|
||
<div class="md-header__title" data-md-component="header-title">
|
||
<div class="md-header__ellipsis">
|
||
<div class="md-header__topic">
|
||
<span class="md-ellipsis">
|
||
obikmer
|
||
</span>
|
||
</div>
|
||
<div class="md-header__topic" data-md-component="header-topic">
|
||
<span class="md-ellipsis">
|
||
|
||
Merge parallelism & memory
|
||
|
||
</span>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
<script>var palette=__md_get("__palette");if(palette&&palette.color){if("(prefers-color-scheme)"===palette.color.media){var media=matchMedia("(prefers-color-scheme: light)"),input=document.querySelector(media.matches?"[data-md-color-media='(prefers-color-scheme: light)']":"[data-md-color-media='(prefers-color-scheme: dark)']");palette.color.media=input.getAttribute("data-md-color-media"),palette.color.scheme=input.getAttribute("data-md-color-scheme"),palette.color.primary=input.getAttribute("data-md-color-primary"),palette.color.accent=input.getAttribute("data-md-color-accent")}for(var[key,value]of Object.entries(palette.color))document.body.setAttribute("data-md-color-"+key,value)}</script>
|
||
|
||
|
||
|
||
|
||
</nav>
|
||
|
||
</header>
|
||
|
||
<div class="md-container" data-md-component="container">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<main class="md-main" data-md-component="main">
|
||
<div class="md-main__inner md-grid">
|
||
|
||
|
||
|
||
<div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
|
||
<div class="md-sidebar__scrollwrap">
|
||
<div class="md-sidebar__inner">
|
||
|
||
|
||
|
||
|
||
<nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
|
||
<label class="md-nav__title" for="__drawer">
|
||
<a href="../.." title="obikmer" class="md-nav__button md-logo" aria-label="obikmer" data-md-component="logo">
|
||
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
|
||
|
||
</a>
|
||
obikmer
|
||
</label>
|
||
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../.." class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Home
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../installation/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Installation
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_3" id="__nav_3_label" tabindex="0">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Theory
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_3_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_3">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Theory
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../kmers/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Kmers and super-kmers
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../theory/encoding/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
DNA encoding
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../theory/entropy/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Entropy filter
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../theory/minimizer/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Minimizer selection
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../theory/indexing/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Partitioning architecture
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../theory/evolutionary_distances/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Central-position SNP distance (discussion)
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--active md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4" checked>
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_4" id="__nav_4_label" tabindex="0">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Implementation
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_4_label" aria-expanded="true">
|
||
<label class="md-nav__title" for="__nav_4">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Implementation
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../superkmer/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
SuperKmer
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../kmer/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Kmer
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../chunkreader/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Chunk reader
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../pipeline/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Construction pipeline
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../obipipeline/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
obipipeline library
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../storage/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
On-disk storage
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../mphf/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
MPHF selection
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../unitig_evidence/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Unitig evidence encoding
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../evidence_elimination/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Evidence elimination (discussion)
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../obilayeredmap/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
obilayeredmap crate
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../persistent_compact_int_vec/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
PersistentCompactIntVec
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../persistent_bit_vec/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
PersistentBitVec
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../merge/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Merge command
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--active">
|
||
|
||
<input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
|
||
|
||
|
||
|
||
|
||
|
||
<label class="md-nav__link md-nav__link--active" for="__toc">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Merge parallelism & memory
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<a href="./" class="md-nav__link md-nav__link--active">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Merge parallelism & memory
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
|
||
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<label class="md-nav__title" for="__toc">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
Table of contents
|
||
</label>
|
||
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#problem-observed" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Problem observed
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#root-cause" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Root cause
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Root cause">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#the-merge-pipeline-per-partition" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
The merge pipeline per partition
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#unbounded-rayon-parallelism" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Unbounded Rayon parallelism
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#the-85-gb-single-allocation" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
The 8.5 GB single allocation
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#partition-size-distribution" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Partition size distribution
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#solution-lfd-scheduling-memory-budget-semaphore" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Solution: LFD scheduling + memory budget semaphore
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Solution: LFD scheduling + memory budget semaphore">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#principle" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Principle
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#memorybudget-obisys" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
MemoryBudget (obisys)
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#adaptive-expansion-factor" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Adaptive expansion factor
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#ram-source" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
RAM source
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#diagnostic-report" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Diagnostic report
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#parameters" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Parameters
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../filtering/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Kmer filtering
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../select/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Select command
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../obitaxonomy/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
obitaxonomy crate
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_5" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_5" id="__nav_5_label" tabindex="0">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Architecture
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_5_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_5">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Architecture
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../architecture/sequences/invariant/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Sequences
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../architecture/index_architecture/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Kmer index
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../architecture/siblings/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Sibling annex (discussion)
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../architecture/numa_worker_pools/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
NUMA-aware worker pools
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../architecture/numa_partition_runner/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
NUMA-aware partition runner
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
|
||
<div class="md-sidebar__scrollwrap">
|
||
<div class="md-sidebar__inner">
|
||
|
||
|
||
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<label class="md-nav__title" for="__toc">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
Table of contents
|
||
</label>
|
||
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#problem-observed" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Problem observed
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#root-cause" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Root cause
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Root cause">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#the-merge-pipeline-per-partition" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
The merge pipeline per partition
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#unbounded-rayon-parallelism" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Unbounded Rayon parallelism
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#the-85-gb-single-allocation" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
The 8.5 GB single allocation
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#partition-size-distribution" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Partition size distribution
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#solution-lfd-scheduling-memory-budget-semaphore" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Solution: LFD scheduling + memory budget semaphore
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Solution: LFD scheduling + memory budget semaphore">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#principle" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Principle
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#memorybudget-obisys" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
MemoryBudget (obisys)
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#adaptive-expansion-factor" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Adaptive expansion factor
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#ram-source" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
RAM source
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#diagnostic-report" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Diagnostic report
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#parameters" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Parameters
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div class="md-content" data-md-component="content">
|
||
|
||
<article class="md-content__inner md-typeset">
|
||
|
||
|
||
|
||
|
||
|
||
<h1 id="merge-parallelism-and-memory-pressure">Merge parallelism and memory pressure</h1>
|
||
<h2 id="problem-observed">Problem observed</h2>
|
||
<p>Running <code>obikmer merge</code> over 109 indexes (108 sources + 1 bootstrap) on a 192-core machine
|
||
produces a fatal OOM during the <code>merge_partitions</code> stage:</p>
|
||
<div class="highlight"><pre><span></span><code>memory allocation of 9126805520 bytes failed
|
||
</code></pre></div>
|
||
<p>A single allocation of ~8.5 GB fails. This is not an aggregate; it is one <code>malloc</code> call
|
||
from hashbrown during a HashMap resize.</p>
|
||
<hr />
|
||
<h2 id="root-cause">Root cause</h2>
|
||
<h3 id="the-merge-pipeline-per-partition">The merge pipeline per partition</h3>
|
||
<div class="highlight"><pre><span></span><code>source unitigs.bin
|
||
→ iter_indexed_canonical_kmers()
|
||
→ GraphDeBruijn::push() ← HashSet<u64> + 1 byte flags, all in RAM
|
||
→ compute_degrees_and_mark_starts()
|
||
→ try_for_each_unitig()
|
||
→ unitigs.bin (new layer)
|
||
→ Layer::build() → MPHF + evidence
|
||
</code></pre></div>
|
||
<p><code>GraphDeBruijn</code> is a <code>FastHashMap<CanonicalKmer, AtomicU8></code> — a <code>HashSet<u64></code> with
|
||
one flag byte per node. Neighbor lookup is implicit: 4 probes into the same map.
|
||
No edges are stored. The full kmer set of one partition must reside in RAM
|
||
simultaneously to compute degrees and mark unitig starts.</p>
|
||
<p>The matrix builders that follow (pass 2) are mmapped files — they do <strong>not</strong> consume
|
||
significant RAM. The pressure is entirely in pass 1.</p>
|
||
<h3 id="unbounded-rayon-parallelism">Unbounded Rayon parallelism</h3>
|
||
<p>With 192 cores, Rayon ran up to 192 partitions concurrently. Each partition built its
|
||
own <code>GraphDeBruijn</code> accumulating all kmers absent from the destination. Peak memory =
|
||
192 × peak_partition_hashset.</p>
|
||
<h3 id="the-85-gb-single-allocation">The 8.5 GB single allocation</h3>
|
||
<p>hashbrown allocates the entire backing array in one call when rehashing.
|
||
At load factor 7/8: <code>capacity × (sizeof(K,V) + 1 control byte)</code>.
|
||
For <code>(u64, AtomicU8)</code> with alignment: ~16 bytes per slot.</p>
|
||
<div class="highlight"><pre><span></span><code>9 127 MB / 16 bytes ≈ 570 M slots → ~380 M new kmers in one partition
|
||
</code></pre></div>
|
||
<p>Plausible for the largest partition of 108 Salix/Betula sources (~450 Mbp each).</p>
|
||
<hr />
|
||
<h2 id="partition-size-distribution">Partition size distribution</h2>
|
||
<p><code>obikmer utils --partition-stats</code> measures the sum of <code>unitigs.bin</code> file sizes
|
||
per partition across all source indexes (pure <code>stat()</code> syscalls, negligible cost).</p>
|
||
<p>Observed on a 9-genome pilot (256 partitions):</p>
|
||
<table>
|
||
<thead>
|
||
<tr>
|
||
<th>Stat</th>
|
||
<th>Value</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr>
|
||
<td>min</td>
|
||
<td>30.5 MB</td>
|
||
</tr>
|
||
<tr>
|
||
<td>max</td>
|
||
<td>232.1 MB</td>
|
||
</tr>
|
||
<tr>
|
||
<td>mean</td>
|
||
<td>40.1 MB</td>
|
||
</tr>
|
||
<tr>
|
||
<td>median</td>
|
||
<td>37.2 MB</td>
|
||
</tr>
|
||
<tr>
|
||
<td>p95</td>
|
||
<td>47.1 MB</td>
|
||
</tr>
|
||
<tr>
|
||
<td>max/median ratio</td>
|
||
<td>6.23×</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
<p>The distribution is <strong>bimodal with a heavy tail</strong>:
|
||
- 238/256 partitions in a narrow 30–50 MB band
|
||
- 4 structurally extreme partitions (3–6× the median): 221, 233, 135, 191</p>
|
||
<p>These correspond to minimizers over-represented in repetitive regions shared across
|
||
all sources. They are extreme in every run on this dataset.</p>
|
||
<p>With 109 sources, outlier partitions do not scale linearly: only kmers <strong>absent from
|
||
the destination</strong> enter the GraphDeBruijn, and inter-source overlap is high for closely
|
||
related species. Partition 221 is the likely trigger for the 8.5 GB crash.</p>
|
||
<hr />
|
||
<h2 id="solution-lfd-scheduling-memory-budget-semaphore">Solution: LFD scheduling + memory budget semaphore</h2>
|
||
<h3 id="principle">Principle</h3>
|
||
<p>Pre-sort partitions by <strong>decreasing estimated size</strong> (First Fit Decreasing — FFD),
|
||
then schedule them through a <strong>continuous memory budget semaphore</strong>. Each worker
|
||
acquires an estimated cost before starting and releases it on completion.</p>
|
||
<p>Large partitions run first when the full budget is available; small partitions fill
|
||
the gaps. No hard outlier threshold is needed.</p>
|
||
<h3 id="memorybudget-obisys"><code>MemoryBudget</code> (<code>obisys</code>)</h3>
|
||
<div class="highlight"><pre><span></span><code><span class="k">pub</span><span class="w"> </span><span class="k">struct</span><span class="w"> </span><span class="nc">MemoryBudget</span><span class="w"> </span><span class="p">{</span><span class="w"> </span><span class="err">…</span><span class="w"> </span><span class="p">}</span>
|
||
|
||
<span class="k">impl</span><span class="w"> </span><span class="n">MemoryBudget</span><span class="w"> </span><span class="p">{</span>
|
||
<span class="w"> </span><span class="k">pub</span><span class="w"> </span><span class="k">fn</span><span class="w"> </span><span class="nf">new</span><span class="p">(</span><span class="n">total</span><span class="p">:</span><span class="w"> </span><span class="kt">u64</span><span class="p">)</span><span class="w"> </span><span class="p">-></span><span class="w"> </span><span class="nc">Self</span><span class="p">;</span>
|
||
<span class="w"> </span><span class="k">pub</span><span class="w"> </span><span class="k">fn</span><span class="w"> </span><span class="nf">acquire</span><span class="p">(</span><span class="o">&</span><span class="bp">self</span><span class="p">,</span><span class="w"> </span><span class="n">cost</span><span class="p">:</span><span class="w"> </span><span class="kt">u64</span><span class="p">);</span><span class="w"> </span><span class="c1">// blocks until budget available</span>
|
||
<span class="w"> </span><span class="k">pub</span><span class="w"> </span><span class="k">fn</span><span class="w"> </span><span class="nf">release</span><span class="p">(</span><span class="o">&</span><span class="bp">self</span><span class="p">,</span><span class="w"> </span><span class="n">cost</span><span class="p">:</span><span class="w"> </span><span class="kt">u64</span><span class="p">);</span>
|
||
<span class="w"> </span><span class="k">pub</span><span class="w"> </span><span class="k">fn</span><span class="w"> </span><span class="nf">peak_active</span><span class="p">(</span><span class="o">&</span><span class="bp">self</span><span class="p">)</span><span class="w"> </span><span class="p">-></span><span class="w"> </span><span class="kt">usize</span><span class="p">;</span>
|
||
<span class="p">}</span>
|
||
</code></pre></div>
|
||
<p>Non-deadlock guarantee: when <code>active == 0</code>, acquire always succeeds regardless of cost.
|
||
Without this, a partition whose estimated cost exceeds the total budget would block forever.</p>
|
||
<h3 id="adaptive-expansion-factor">Adaptive expansion factor</h3>
|
||
<p>The expansion factor converts raw <code>unitigs.bin</code> bytes into an estimated GraphDeBruijn
|
||
RAM footprint. hashbrown stores each kmer as <code>(u64, AtomicU8)</code> ≈ 16 bytes/kmer at 7/8
|
||
load factor; unitig files encode ≈ 2 bits/base. The ratio depends on average unitig
|
||
length (short unitigs: ~2×; long unitigs: up to ~50×).</p>
|
||
<p><strong>Phase 1 — sequential pilot (worst partition)</strong></p>
|
||
<p>The largest partition runs alone first. Its actual <code>g.len()</code> seeds the expansion factor
|
||
before any parallel job starts. <code>FALLBACK_EXPANSION = 4×</code> is used only for empty partitions.</p>
|
||
<div class="highlight"><pre><span></span><code><span class="kd">let</span><span class="w"> </span><span class="n">worst_g_len</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">dst_partition</span><span class="p">.</span><span class="n">merge_partition</span><span class="p">(</span><span class="n">worst_id</span><span class="p">,</span><span class="w"> </span><span class="err">…</span><span class="p">)</span><span class="o">?</span><span class="p">;</span>
|
||
<span class="c1">// ↑ now returns SKResult<usize> (was SKResult<()>)</span>
|
||
|
||
<span class="kd">let</span><span class="w"> </span><span class="n">seed_expansion</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">worst_g_len</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="kt">u64</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="mi">16</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="mi">1000</span><span class="w"> </span><span class="o">/</span><span class="w"> </span><span class="n">worst_bytes</span><span class="p">;</span>
|
||
<span class="kd">let</span><span class="w"> </span><span class="n">max_expansion</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">AtomicU64</span><span class="p">::</span><span class="n">new</span><span class="p">(</span><span class="n">seed_expansion</span><span class="p">);</span>
|
||
</code></pre></div>
|
||
<p><strong>Phase 2 — parallel with adaptive updates</strong></p>
|
||
<div class="highlight"><pre><span></span><code><span class="n">order</span><span class="p">[</span><span class="mi">1</span><span class="o">..</span><span class="p">].</span><span class="n">into_par_iter</span><span class="p">().</span><span class="n">for_each</span><span class="p">(</span><span class="o">|&</span><span class="n">i</span><span class="o">|</span><span class="w"> </span><span class="p">{</span>
|
||
<span class="w"> </span><span class="kd">let</span><span class="w"> </span><span class="n">cost</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">partition_sizes</span><span class="p">[</span><span class="n">i</span><span class="p">]</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="n">max_expansion</span><span class="p">.</span><span class="n">load</span><span class="p">(</span><span class="n">Relaxed</span><span class="p">)</span><span class="w"> </span><span class="o">/</span><span class="w"> </span><span class="mi">1000</span><span class="p">;</span>
|
||
<span class="w"> </span><span class="n">budget</span><span class="p">.</span><span class="n">acquire</span><span class="p">(</span><span class="n">cost</span><span class="p">);</span>
|
||
<span class="w"> </span><span class="kd">let</span><span class="w"> </span><span class="n">g_len</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">dst_partition</span><span class="p">.</span><span class="n">merge_partition</span><span class="p">(</span><span class="n">i</span><span class="p">,</span><span class="w"> </span><span class="err">…</span><span class="p">)</span><span class="o">?</span><span class="p">;</span>
|
||
<span class="w"> </span><span class="n">budget</span><span class="p">.</span><span class="n">release</span><span class="p">(</span><span class="n">cost</span><span class="p">);</span><span class="w"> </span><span class="c1">// releases estimated cost, not actual</span>
|
||
|
||
<span class="w"> </span><span class="kd">let</span><span class="w"> </span><span class="n">actual</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">g_len</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="kt">u64</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="mi">16</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="mi">1000</span><span class="w"> </span><span class="o">/</span><span class="w"> </span><span class="n">partition_sizes</span><span class="p">[</span><span class="n">i</span><span class="p">];</span>
|
||
<span class="w"> </span><span class="n">max_expansion</span><span class="p">.</span><span class="n">fetch_max</span><span class="p">(</span><span class="n">actual</span><span class="p">,</span><span class="w"> </span><span class="n">Relaxed</span><span class="p">);</span><span class="w"> </span><span class="c1">// always pessimistic (max)</span>
|
||
<span class="p">});</span>
|
||
</code></pre></div>
|
||
<p><code>budget.release(cost)</code> uses the estimated cost, not the actual one. The budget tracks
|
||
reservations, not physical RAM; each partition pays what it promised at acquisition.</p>
|
||
<p><strong>On the safety margin</strong></p>
|
||
<p>There is no separate multiplier <code>k</code>. It is redundant with <code>budget_fraction</code>: both
|
||
reduce effective concurrency by the same amount. A single parameter is easier to
|
||
calibrate. <code>budget_fraction = 0.5</code> (default) reserves half of available RAM for the
|
||
OS, MPHF build, pass 2, and estimation error.</p>
|
||
<p><code>--budget-fraction</code> is exposed as a CLI flag — the only escape hatch for pathological
|
||
cases (extreme repetitive content, unusually long unitigs) that still cause OOM.</p>
|
||
<h3 id="ram-source">RAM source</h3>
|
||
<p><code>obisys::available_memory_bytes()</code> — wraps <code>sysinfo::System::available_memory()</code>,
|
||
falls back to <code>total / 2</code> on macOS when the memory compressor returns 0.</p>
|
||
<hr />
|
||
<h2 id="diagnostic-report">Diagnostic report</h2>
|
||
<p>After the parallel phase, <code>merge_partition</code> emits a structured report via <code>tracing::info!</code>:</p>
|
||
<div class="highlight"><pre><span></span><code>─── merge_partitions memory report ───
|
||
available RAM : 512.0 GB budget 50% = 256.0 GB
|
||
expansion factor — seed: 4.2× final max: 6.1× (mean: 1.8× median: 1.6×)
|
||
peak concurrent workers: 42
|
||
expansion factor distribution (256 partitions with data):
|
||
0.50× – 1.25× │██████████████████████████████ 148
|
||
1.25× – 2.00× │████████████████████████ 82
|
||
…
|
||
5.50× – 6.25× │█ 2
|
||
top partitions by actual expansion factor:
|
||
partition 221 : 6.10× (232.1 MB unitigs → 48M kmers, reserved at 4.20×)
|
||
partition 135 : 5.82× (127.3 MB unitigs → 24M kmers, reserved at 4.20×)
|
||
…
|
||
──────────────────────────────────────
|
||
</code></pre></div>
|
||
<p>Fields useful for diagnosis:</p>
|
||
<table>
|
||
<thead>
|
||
<tr>
|
||
<th>Field</th>
|
||
<th>Interpretation</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr>
|
||
<td><code>seed</code> vs <code>final max</code> expansion</td>
|
||
<td>gap indicates partitions with higher expansion than the worst-by-size</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>reserved at X×</code></td>
|
||
<td>the factor used at acquisition; if much lower than actual, the budget was under-reserved for that partition</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>peak concurrent workers</code></td>
|
||
<td>effective parallelism achieved under the budget constraint</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>mean</code> / <code>median</code> expansion</td>
|
||
<td>typical dataset characteristic; stable across runs on the same data</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
<hr />
|
||
<h2 id="parameters">Parameters</h2>
|
||
<table>
|
||
<thead>
|
||
<tr>
|
||
<th>Parameter</th>
|
||
<th>Default</th>
|
||
<th>CLI flag</th>
|
||
<th>Notes</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr>
|
||
<td><code>fallback_expansion</code></td>
|
||
<td>4×</td>
|
||
<td>—</td>
|
||
<td>seed for empty partitions only</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>budget_fraction</code></td>
|
||
<td>0.5</td>
|
||
<td><code>--budget-fraction</code></td>
|
||
<td>reduce if OOM persists</td>
|
||
</tr>
|
||
<tr>
|
||
<td>RAM source</td>
|
||
<td><code>obisys::available_memory_bytes()</code></td>
|
||
<td>—</td>
|
||
<td>falls back to <code>total/2</code> on macOS</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
</article>
|
||
</div>
|
||
|
||
|
||
<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
|
||
</div>
|
||
|
||
</main>
|
||
|
||
<footer class="md-footer">
|
||
|
||
<div class="md-footer-meta md-typeset">
|
||
<div class="md-footer-meta__inner md-grid">
|
||
<div class="md-copyright">
|
||
|
||
|
||
Made with
|
||
<a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
|
||
Material for MkDocs
|
||
</a>
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
</footer>
|
||
|
||
</div>
|
||
<div class="md-dialog" data-md-component="dialog">
|
||
<div class="md-dialog__inner md-typeset"></div>
|
||
</div>
|
||
|
||
|
||
|
||
|
||
|
||
<script id="__config" type="application/json">{"annotate": null, "base": "../..", "features": [], "search": "../../assets/javascripts/workers/search.2c215733.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
|
||
|
||
|
||
<script src="../../assets/javascripts/bundle.79ae519e.min.js"></script>
|
||
|
||
<script src="https://unpkg.com/mathjax@3/es5/tex-mml-chtml.js"></script>
|
||
|
||
|
||
</body>
|
||
</html> |