DevDoc/implementation/merge_parallelism/index.html


<!doctype html>
<html lang="en" class="no-js">
  <head>
    
      <meta charset="utf-8">
      <meta name="viewport" content="width=device-width,initial-scale=1">
      
      
        <link rel="prev" href="../merge/">
      
      
        <link rel="next" href="../filtering/">
      
      
      <link rel="icon" href="../../assets/images/favicon.png">
      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.7.6">
    
    
        <title>Merge parallelism & memory - obikmer</title>
      
    
      <link rel="stylesheet" href="../../assets/stylesheets/main.484c7ddc.min.css">
      
      
        <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
        <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
        <style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
      
    
    <script>__md_scope=new URL("../..",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
    
      
  </head>
  
  
    <body dir="ltr">
  
    
    <input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
    <input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
    <label class="md-overlay" for="__drawer"></label>
    <div data-md-component="skip">
      
        
        <a href="#merge-parallelism-and-memory-pressure" class="md-skip">
          Skip to content
        </a>
      
    </div>
    <div data-md-component="announce">
      
    </div>
    
    
<header class="md-header md-header--shadow" data-md-component="header">
  <nav class="md-header__inner md-grid" aria-label="Header">
    <a href="../.." title="obikmer" class="md-header__button md-logo" aria-label="obikmer" data-md-component="logo">
      
  
  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>

    </a>
    <label class="md-header__button md-icon" for="__drawer">
      
      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
    </label>
    <div class="md-header__title" data-md-component="header-title">
      <div class="md-header__ellipsis">
        <div class="md-header__topic">
          <span class="md-ellipsis">
            obikmer
          </span>
        </div>
        <div class="md-header__topic" data-md-component="header-topic">
          <span class="md-ellipsis">
            
              Merge parallelism & memory
            
          </span>
        </div>
      </div>
    </div>
    
    
      <script>var palette=__md_get("__palette");if(palette&&palette.color){if("(prefers-color-scheme)"===palette.color.media){var media=matchMedia("(prefers-color-scheme: light)"),input=document.querySelector(media.matches?"[data-md-color-media='(prefers-color-scheme: light)']":"[data-md-color-media='(prefers-color-scheme: dark)']");palette.color.media=input.getAttribute("data-md-color-media"),palette.color.scheme=input.getAttribute("data-md-color-scheme"),palette.color.primary=input.getAttribute("data-md-color-primary"),palette.color.accent=input.getAttribute("data-md-color-accent")}for(var[key,value]of Object.entries(palette.color))document.body.setAttribute("data-md-color-"+key,value)}</script>
    
    
  </nav>
  
</header>
    
    <div class="md-container" data-md-component="container">
      
      
      <main class="md-main" data-md-component="main">
        <div class="md-main__inner md-grid">
          
            
              <div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
                <div class="md-sidebar__scrollwrap">
                  <div class="md-sidebar__inner">
                    

<nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
  <label class="md-nav__title" for="__drawer">
    <a href="../.." title="obikmer" class="md-nav__button md-logo" aria-label="obikmer" data-md-component="logo">
      
  
  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>

    </a>
    obikmer
  </label>
  
  <ul class="md-nav__list" data-md-scrollfix>
    
      
    <li class="md-nav__item">
      <a href="../.." class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    Home
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../../installation/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    Installation
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item md-nav__item--nested">
      
        
        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3" >
        
          
          <label class="md-nav__link" for="__nav_3" id="__nav_3_label" tabindex="0">
            
  
  <span class="md-ellipsis">
    
  
    Theory
  

  </span>
  
  
            <span class="md-nav__icon md-icon"></span>
          </label>
        
        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_3_label" aria-expanded="false">
          <label class="md-nav__title" for="__nav_3">
            <span class="md-nav__icon md-icon"></span>
            
  
    Theory
  

          </label>
          <ul class="md-nav__list" data-md-scrollfix>
            
              
    <li class="md-nav__item">
      <a href="../../kmers/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    Kmers and super-kmers
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../../theory/encoding/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    DNA encoding
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../../theory/entropy/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    Entropy filter
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../../theory/minimizer/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    Minimizer selection
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../../theory/indexing/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    Partitioning architecture
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../../theory/evolutionary_distances/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    Central-position SNP distance (discussion)
  

  </span>
  
  
      </a>
    </li>
  

          </ul>
        </nav>
      
    </li>
  

    <li class="md-nav__item md-nav__item--active md-nav__item--nested">
      
        
        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4" checked>
        
          
          <label class="md-nav__link" for="__nav_4" id="__nav_4_label" tabindex="0">
            
  
  <span class="md-ellipsis">
    
  
    Implementation
  

  </span>
  
  
            <span class="md-nav__icon md-icon"></span>
          </label>
        
        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_4_label" aria-expanded="true">
          <label class="md-nav__title" for="__nav_4">
            <span class="md-nav__icon md-icon"></span>
            
  
    Implementation
  

          </label>
          <ul class="md-nav__list" data-md-scrollfix>
            
              
    <li class="md-nav__item">
      <a href="../superkmer/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    SuperKmer
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../kmer/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    Kmer
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../chunkreader/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    Chunk reader
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../pipeline/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    Construction pipeline
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../obipipeline/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    obipipeline library
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../storage/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    On-disk storage
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../mphf/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    MPHF selection
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../unitig_evidence/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    Unitig evidence encoding
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../evidence_elimination/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    Evidence elimination (discussion)
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../obilayeredmap/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    obilayeredmap crate
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../persistent_compact_int_vec/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    PersistentCompactIntVec
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../persistent_bit_vec/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    PersistentBitVec
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../merge/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    Merge command
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item md-nav__item--active">
      
      <input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
      
      
        <label class="md-nav__link md-nav__link--active" for="__toc">
          
  
  <span class="md-ellipsis">
    
  
    Merge parallelism & memory
  

  </span>
  
  
          <span class="md-nav__icon md-icon"></span>
        </label>
      
      <a href="./" class="md-nav__link md-nav__link--active">
        
  
  <span class="md-ellipsis">
    
  
    Merge parallelism & memory
  

  </span>
  
  
      </a>
      
        
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
  
  
    <label class="md-nav__title" for="__toc">
      <span class="md-nav__icon md-icon"></span>
      Table of contents
    </label>
    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
      
        <li class="md-nav__item">
  <a href="#problem-observed" class="md-nav__link">
    <span class="md-ellipsis">
      
        Problem observed
      
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#root-cause" class="md-nav__link">
    <span class="md-ellipsis">
      
        Root cause
      
    </span>
  </a>
  
    <nav class="md-nav" aria-label="Root cause">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#the-merge-pipeline-per-partition" class="md-nav__link">
    <span class="md-ellipsis">
      
        The merge pipeline per partition
      
    </span>
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#unbounded-rayon-parallelism" class="md-nav__link">
    <span class="md-ellipsis">
      
        Unbounded Rayon parallelism
      
    </span>
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#the-85-gb-single-allocation" class="md-nav__link">
    <span class="md-ellipsis">
      
        The 8.5 GB single allocation
      
    </span>
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
      
        <li class="md-nav__item">
  <a href="#partition-size-distribution" class="md-nav__link">
    <span class="md-ellipsis">
      
        Partition size distribution
      
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#solution-lfd-scheduling-memory-budget-semaphore" class="md-nav__link">
    <span class="md-ellipsis">
      
        Solution: LFD scheduling + memory budget semaphore
      
    </span>
  </a>
  
    <nav class="md-nav" aria-label="Solution: LFD scheduling + memory budget semaphore">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#principle" class="md-nav__link">
    <span class="md-ellipsis">
      
        Principle
      
    </span>
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#memorybudget-obisys" class="md-nav__link">
    <span class="md-ellipsis">
      
        MemoryBudget (obisys)
      
    </span>
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#adaptive-expansion-factor" class="md-nav__link">
    <span class="md-ellipsis">
      
        Adaptive expansion factor
      
    </span>
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#ram-source" class="md-nav__link">
    <span class="md-ellipsis">
      
        RAM source
      
    </span>
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
      
        <li class="md-nav__item">
  <a href="#diagnostic-report" class="md-nav__link">
    <span class="md-ellipsis">
      
        Diagnostic report
      
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#parameters" class="md-nav__link">
    <span class="md-ellipsis">
      
        Parameters
      
    </span>
  </a>
  
</li>
      
    </ul>
  
</nav>
      
    </li>
  

    <li class="md-nav__item">
      <a href="../filtering/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    Kmer filtering
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../select/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    Select command
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../obitaxonomy/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    obitaxonomy crate
  

  </span>
  
  
      </a>
    </li>
  

          </ul>
        </nav>
      
    </li>
  

    <li class="md-nav__item md-nav__item--nested">
      
        
        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_5" >
        
          
          <label class="md-nav__link" for="__nav_5" id="__nav_5_label" tabindex="0">
            
  
  <span class="md-ellipsis">
    
  
    Architecture
  

  </span>
  
  
            <span class="md-nav__icon md-icon"></span>
          </label>
        
        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_5_label" aria-expanded="false">
          <label class="md-nav__title" for="__nav_5">
            <span class="md-nav__icon md-icon"></span>
            
  
    Architecture
  

          </label>
          <ul class="md-nav__list" data-md-scrollfix>
            
              
    <li class="md-nav__item">
      <a href="../../architecture/sequences/invariant/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    Sequences
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../../architecture/index_architecture/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    Kmer index
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../../architecture/siblings/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    Sibling annex (discussion)
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../../architecture/numa_worker_pools/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    NUMA-aware worker pools
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../../architecture/numa_partition_runner/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    NUMA-aware partition runner
  

  </span>
  
  
      </a>
    </li>
  

          </ul>
        </nav>
      
    </li>
  

  </ul>
</nav>
                  </div>
                </div>
              </div>
            
            
              <div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
                <div class="md-sidebar__scrollwrap">
                  <div class="md-sidebar__inner">
                    

<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
  
  
    <label class="md-nav__title" for="__toc">
      <span class="md-nav__icon md-icon"></span>
      Table of contents
    </label>
    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
      
        <li class="md-nav__item">
  <a href="#problem-observed" class="md-nav__link">
    <span class="md-ellipsis">
      
        Problem observed
      
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#root-cause" class="md-nav__link">
    <span class="md-ellipsis">
      
        Root cause
      
    </span>
  </a>
  
    <nav class="md-nav" aria-label="Root cause">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#the-merge-pipeline-per-partition" class="md-nav__link">
    <span class="md-ellipsis">
      
        The merge pipeline per partition
      
    </span>
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#unbounded-rayon-parallelism" class="md-nav__link">
    <span class="md-ellipsis">
      
        Unbounded Rayon parallelism
      
    </span>
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#the-85-gb-single-allocation" class="md-nav__link">
    <span class="md-ellipsis">
      
        The 8.5 GB single allocation
      
    </span>
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
      
        <li class="md-nav__item">
  <a href="#partition-size-distribution" class="md-nav__link">
    <span class="md-ellipsis">
      
        Partition size distribution
      
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#solution-lfd-scheduling-memory-budget-semaphore" class="md-nav__link">
    <span class="md-ellipsis">
      
        Solution: LFD scheduling + memory budget semaphore
      
    </span>
  </a>
  
    <nav class="md-nav" aria-label="Solution: LFD scheduling + memory budget semaphore">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#principle" class="md-nav__link">
    <span class="md-ellipsis">
      
        Principle
      
    </span>
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#memorybudget-obisys" class="md-nav__link">
    <span class="md-ellipsis">
      
        MemoryBudget (obisys)
      
    </span>
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#adaptive-expansion-factor" class="md-nav__link">
    <span class="md-ellipsis">
      
        Adaptive expansion factor
      
    </span>
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#ram-source" class="md-nav__link">
    <span class="md-ellipsis">
      
        RAM source
      
    </span>
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
      
        <li class="md-nav__item">
  <a href="#diagnostic-report" class="md-nav__link">
    <span class="md-ellipsis">
      
        Diagnostic report
      
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#parameters" class="md-nav__link">
    <span class="md-ellipsis">
      
        Parameters
      
    </span>
  </a>
  
</li>
      
    </ul>
  
</nav>
                  </div>
                </div>
              </div>
            
          
            <div class="md-content" data-md-component="content">
              
              <article class="md-content__inner md-typeset">
                
                  
<h1 id="merge-parallelism-and-memory-pressure">Merge parallelism and memory pressure</h1>
<h2 id="problem-observed">Problem observed</h2>
<p>Running <code>obikmer merge</code> over 109 indexes (108 sources + 1 bootstrap) on a 192-core machine
produces a fatal OOM during the <code>merge_partitions</code> stage:</p>
<div class="highlight"><pre><span></span><code>memory allocation of 9126805520 bytes failed
</code></pre></div>
<p>A single allocation of ~8.5 GB fails. This is not an aggregate; it is one <code>malloc</code> call
from hashbrown during a HashMap resize.</p>
<hr />
<h2 id="root-cause">Root cause</h2>
<h3 id="the-merge-pipeline-per-partition">The merge pipeline per partition</h3>
<div class="highlight"><pre><span></span><code>source unitigs.bin
  → iter_indexed_canonical_kmers()
  → GraphDeBruijn::push()       ← HashSet&lt;u64&gt; + 1 byte flags, all in RAM
  → compute_degrees_and_mark_starts()
  → try_for_each_unitig()
  → unitigs.bin (new layer)
  → Layer::build() → MPHF + evidence
</code></pre></div>
<p><code>GraphDeBruijn</code> is a <code>FastHashMap&lt;CanonicalKmer, AtomicU8&gt;</code> — a <code>HashSet&lt;u64&gt;</code> with
one flag byte per node. Neighbor lookup is implicit: 4 probes into the same map.
No edges are stored. The full kmer set of one partition must reside in RAM
simultaneously to compute degrees and mark unitig starts.</p>
<p>The matrix builders that follow (pass 2) are mmapped files — they do <strong>not</strong> consume
significant RAM. The pressure is entirely in pass 1.</p>
<h3 id="unbounded-rayon-parallelism">Unbounded Rayon parallelism</h3>
<p>With 192 cores, Rayon ran up to 192 partitions concurrently. Each partition built its
own <code>GraphDeBruijn</code> accumulating all kmers absent from the destination. Peak memory =
192 × peak_partition_hashset.</p>
<h3 id="the-85-gb-single-allocation">The 8.5 GB single allocation</h3>
<p>hashbrown allocates the entire backing array in one call when rehashing.
At load factor 7/8: <code>capacity × (sizeof(K,V) + 1 control byte)</code>.
For <code>(u64, AtomicU8)</code> with alignment: ~16 bytes per slot.</p>
<div class="highlight"><pre><span></span><code>9 127 MB / 16 bytes ≈ 570 M slots → ~380 M new kmers in one partition
</code></pre></div>
<p>Plausible for the largest partition of 108 Salix/Betula sources (~450 Mbp each).</p>
<hr />
<h2 id="partition-size-distribution">Partition size distribution</h2>
<p><code>obikmer utils --partition-stats</code> measures the sum of <code>unitigs.bin</code> file sizes
per partition across all source indexes (pure <code>stat()</code> syscalls, negligible cost).</p>
<p>Observed on a 9-genome pilot (256 partitions):</p>
<table>
<thead>
<tr>
<th>Stat</th>
<th>Value</th>
</tr>
</thead>
<tbody>
<tr>
<td>min</td>
<td>30.5 MB</td>
</tr>
<tr>
<td>max</td>
<td>232.1 MB</td>
</tr>
<tr>
<td>mean</td>
<td>40.1 MB</td>
</tr>
<tr>
<td>median</td>
<td>37.2 MB</td>
</tr>
<tr>
<td>p95</td>
<td>47.1 MB</td>
</tr>
<tr>
<td>max/median ratio</td>
<td>6.23×</td>
</tr>
</tbody>
</table>
<p>The distribution is <strong>bimodal with a heavy tail</strong>:
- 238/256 partitions in a narrow 30–50 MB band
- 4 structurally extreme partitions (3–6× the median): 221, 233, 135, 191</p>
<p>These correspond to minimizers over-represented in repetitive regions shared across
all sources. They are extreme in every run on this dataset.</p>
<p>With 109 sources, outlier partitions do not scale linearly: only kmers <strong>absent from
the destination</strong> enter the GraphDeBruijn, and inter-source overlap is high for closely
related species. Partition 221 is the likely trigger for the 8.5 GB crash.</p>
<hr />
<h2 id="solution-lfd-scheduling-memory-budget-semaphore">Solution: LFD scheduling + memory budget semaphore</h2>
<h3 id="principle">Principle</h3>
<p>Pre-sort partitions by <strong>decreasing estimated size</strong> (First Fit Decreasing — FFD),
then schedule them through a <strong>continuous memory budget semaphore</strong>. Each worker
acquires an estimated cost before starting and releases it on completion.</p>
<p>Large partitions run first when the full budget is available; small partitions fill
the gaps. No hard outlier threshold is needed.</p>
<h3 id="memorybudget-obisys"><code>MemoryBudget</code> (<code>obisys</code>)</h3>
<div class="highlight"><pre><span></span><code><span class="k">pub</span><span class="w"> </span><span class="k">struct</span><span class="w"> </span><span class="nc">MemoryBudget</span><span class="w"> </span><span class="p">{</span><span class="w"> </span><span class="err">…</span><span class="w"> </span><span class="p">}</span>

<span class="k">impl</span><span class="w"> </span><span class="n">MemoryBudget</span><span class="w"> </span><span class="p">{</span>
<span class="w">    </span><span class="k">pub</span><span class="w"> </span><span class="k">fn</span><span class="w"> </span><span class="nf">new</span><span class="p">(</span><span class="n">total</span><span class="p">:</span><span class="w"> </span><span class="kt">u64</span><span class="p">)</span><span class="w"> </span><span class="p">-&gt;</span><span class="w"> </span><span class="nc">Self</span><span class="p">;</span>
<span class="w">    </span><span class="k">pub</span><span class="w"> </span><span class="k">fn</span><span class="w"> </span><span class="nf">acquire</span><span class="p">(</span><span class="o">&amp;</span><span class="bp">self</span><span class="p">,</span><span class="w"> </span><span class="n">cost</span><span class="p">:</span><span class="w"> </span><span class="kt">u64</span><span class="p">);</span><span class="w">  </span><span class="c1">// blocks until budget available</span>
<span class="w">    </span><span class="k">pub</span><span class="w"> </span><span class="k">fn</span><span class="w"> </span><span class="nf">release</span><span class="p">(</span><span class="o">&amp;</span><span class="bp">self</span><span class="p">,</span><span class="w"> </span><span class="n">cost</span><span class="p">:</span><span class="w"> </span><span class="kt">u64</span><span class="p">);</span>
<span class="w">    </span><span class="k">pub</span><span class="w"> </span><span class="k">fn</span><span class="w"> </span><span class="nf">peak_active</span><span class="p">(</span><span class="o">&amp;</span><span class="bp">self</span><span class="p">)</span><span class="w"> </span><span class="p">-&gt;</span><span class="w"> </span><span class="kt">usize</span><span class="p">;</span>
<span class="p">}</span>
</code></pre></div>
<p>Non-deadlock guarantee: when <code>active == 0</code>, acquire always succeeds regardless of cost.
Without this, a partition whose estimated cost exceeds the total budget would block forever.</p>
<h3 id="adaptive-expansion-factor">Adaptive expansion factor</h3>
<p>The expansion factor converts raw <code>unitigs.bin</code> bytes into an estimated GraphDeBruijn
RAM footprint. hashbrown stores each kmer as <code>(u64, AtomicU8)</code> ≈ 16 bytes/kmer at 7/8
load factor; unitig files encode ≈ 2 bits/base. The ratio depends on average unitig
length (short unitigs: ~2×; long unitigs: up to ~50×).</p>
<p><strong>Phase 1 — sequential pilot (worst partition)</strong></p>
<p>The largest partition runs alone first. Its actual <code>g.len()</code> seeds the expansion factor
before any parallel job starts. <code>FALLBACK_EXPANSION = 4×</code> is used only for empty partitions.</p>
<div class="highlight"><pre><span></span><code><span class="kd">let</span><span class="w"> </span><span class="n">worst_g_len</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">dst_partition</span><span class="p">.</span><span class="n">merge_partition</span><span class="p">(</span><span class="n">worst_id</span><span class="p">,</span><span class="w"> </span><span class="err">…</span><span class="p">)</span><span class="o">?</span><span class="p">;</span>
<span class="c1">//                              ↑ now returns SKResult&lt;usize&gt; (was SKResult&lt;()&gt;)</span>

<span class="kd">let</span><span class="w"> </span><span class="n">seed_expansion</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">worst_g_len</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="kt">u64</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="mi">16</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="mi">1000</span><span class="w"> </span><span class="o">/</span><span class="w"> </span><span class="n">worst_bytes</span><span class="p">;</span>
<span class="kd">let</span><span class="w"> </span><span class="n">max_expansion</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">AtomicU64</span><span class="p">::</span><span class="n">new</span><span class="p">(</span><span class="n">seed_expansion</span><span class="p">);</span>
</code></pre></div>
<p><strong>Phase 2 — parallel with adaptive updates</strong></p>
<div class="highlight"><pre><span></span><code><span class="n">order</span><span class="p">[</span><span class="mi">1</span><span class="o">..</span><span class="p">].</span><span class="n">into_par_iter</span><span class="p">().</span><span class="n">for_each</span><span class="p">(</span><span class="o">|&amp;</span><span class="n">i</span><span class="o">|</span><span class="w"> </span><span class="p">{</span>
<span class="w">    </span><span class="kd">let</span><span class="w"> </span><span class="n">cost</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">partition_sizes</span><span class="p">[</span><span class="n">i</span><span class="p">]</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="n">max_expansion</span><span class="p">.</span><span class="n">load</span><span class="p">(</span><span class="n">Relaxed</span><span class="p">)</span><span class="w"> </span><span class="o">/</span><span class="w"> </span><span class="mi">1000</span><span class="p">;</span>
<span class="w">    </span><span class="n">budget</span><span class="p">.</span><span class="n">acquire</span><span class="p">(</span><span class="n">cost</span><span class="p">);</span>
<span class="w">    </span><span class="kd">let</span><span class="w"> </span><span class="n">g_len</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">dst_partition</span><span class="p">.</span><span class="n">merge_partition</span><span class="p">(</span><span class="n">i</span><span class="p">,</span><span class="w"> </span><span class="err">…</span><span class="p">)</span><span class="o">?</span><span class="p">;</span>
<span class="w">    </span><span class="n">budget</span><span class="p">.</span><span class="n">release</span><span class="p">(</span><span class="n">cost</span><span class="p">);</span><span class="w">          </span><span class="c1">// releases estimated cost, not actual</span>

<span class="w">    </span><span class="kd">let</span><span class="w"> </span><span class="n">actual</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">g_len</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="kt">u64</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="mi">16</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="mi">1000</span><span class="w"> </span><span class="o">/</span><span class="w"> </span><span class="n">partition_sizes</span><span class="p">[</span><span class="n">i</span><span class="p">];</span>
<span class="w">    </span><span class="n">max_expansion</span><span class="p">.</span><span class="n">fetch_max</span><span class="p">(</span><span class="n">actual</span><span class="p">,</span><span class="w"> </span><span class="n">Relaxed</span><span class="p">);</span><span class="w">  </span><span class="c1">// always pessimistic (max)</span>
<span class="p">});</span>
</code></pre></div>
<p><code>budget.release(cost)</code> uses the estimated cost, not the actual one. The budget tracks
reservations, not physical RAM; each partition pays what it promised at acquisition.</p>
<p><strong>On the safety margin</strong></p>
<p>There is no separate multiplier <code>k</code>. It is redundant with <code>budget_fraction</code>: both
reduce effective concurrency by the same amount. A single parameter is easier to
calibrate. <code>budget_fraction = 0.5</code> (default) reserves half of available RAM for the
OS, MPHF build, pass 2, and estimation error.</p>
<p><code>--budget-fraction</code> is exposed as a CLI flag — the only escape hatch for pathological
cases (extreme repetitive content, unusually long unitigs) that still cause OOM.</p>
<h3 id="ram-source">RAM source</h3>
<p><code>obisys::available_memory_bytes()</code> — wraps <code>sysinfo::System::available_memory()</code>,
falls back to <code>total / 2</code> on macOS when the memory compressor returns 0.</p>
<hr />
<h2 id="diagnostic-report">Diagnostic report</h2>
<p>After the parallel phase, <code>merge_partition</code> emits a structured report via <code>tracing::info!</code>:</p>
<div class="highlight"><pre><span></span><code>─── merge_partitions memory report ───
  available RAM : 512.0 GB   budget 50% = 256.0 GB
  expansion factor — seed: 4.2×  final max: 6.1×  (mean: 1.8×  median: 1.6×)
  peak concurrent workers: 42
  expansion factor distribution (256 partitions with data):
     0.50× –  1.25× │██████████████████████████████ 148
     1.25× –  2.00× │████████████████████████        82
     …
     5.50× –  6.25× │█                                2
  top partitions by actual expansion factor:
    partition  221 : 6.10×  (232.1 MB unitigs → 48M kmers, reserved at 4.20×)
    partition  135 : 5.82×  (127.3 MB unitigs → 24M kmers, reserved at 4.20×)
    …
──────────────────────────────────────
</code></pre></div>
<p>Fields useful for diagnosis:</p>
<table>
<thead>
<tr>
<th>Field</th>
<th>Interpretation</th>
</tr>
</thead>
<tbody>
<tr>
<td><code>seed</code> vs <code>final max</code> expansion</td>
<td>gap indicates partitions with higher expansion than the worst-by-size</td>
</tr>
<tr>
<td><code>reserved at X×</code></td>
<td>the factor used at acquisition; if much lower than actual, the budget was under-reserved for that partition</td>
</tr>
<tr>
<td><code>peak concurrent workers</code></td>
<td>effective parallelism achieved under the budget constraint</td>
</tr>
<tr>
<td><code>mean</code> / <code>median</code> expansion</td>
<td>typical dataset characteristic; stable across runs on the same data</td>
</tr>
</tbody>
</table>
<hr />
<h2 id="parameters">Parameters</h2>
<table>
<thead>
<tr>
<th>Parameter</th>
<th>Default</th>
<th>CLI flag</th>
<th>Notes</th>
</tr>
</thead>
<tbody>
<tr>
<td><code>fallback_expansion</code></td>
<td>4×</td>
<td>—</td>
<td>seed for empty partitions only</td>
</tr>
<tr>
<td><code>budget_fraction</code></td>
<td>0.5</td>
<td><code>--budget-fraction</code></td>
<td>reduce if OOM persists</td>
</tr>
<tr>
<td>RAM source</td>
<td><code>obisys::available_memory_bytes()</code></td>
<td>—</td>
<td>falls back to <code>total/2</code> on macOS</td>
</tr>
</tbody>
</table>


              </article>
            </div>
          
          
<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
        </div>
        
      </main>
      
        <footer class="md-footer">
  
  <div class="md-footer-meta md-typeset">
    <div class="md-footer-meta__inner md-grid">
      <div class="md-copyright">
  
  
    Made with
    <a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
      Material for MkDocs
    </a>
  
</div>
      
    </div>
  </div>
</footer>
      
    </div>
    <div class="md-dialog" data-md-component="dialog">
      <div class="md-dialog__inner md-typeset"></div>
    </div>
    
    
      <script id="__config" type="application/json">{"annotate": null, "base": "../..", "features": [], "search": "../../assets/javascripts/workers/search.2c215733.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
    
    
      <script src="../../assets/javascripts/bundle.79ae519e.min.js"></script>
      
        <script src="https://unpkg.com/mathjax@3/es5/tex-mml-chtml.js"></script>
      
    
  </body>
</html>
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								<!doctype html>
 								<html lang="en" class="no-js">
 								  <head>
 								      <meta charset="utf-8">
 								      <meta name="viewport" content="width=device-width,initial-scale=1">
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								        <link rel="prev" href="../merge/">
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								        <link rel="next" href="../filtering/">
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								      <link rel="icon" href="../../assets/images/favicon.png">
 								      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.7.6">
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								        <title>Merge parallelism & memory - obikmer</title>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								      <link rel="stylesheet" href="../../assets/stylesheets/main.484c7ddc.min.css">
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								        <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
 								        <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
 								        <style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
 								    <script>__md_scope=new URL("../..",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
 								  </head>
 								    <body dir="ltr">
 								    <input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
 								    <input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
 								    <label class="md-overlay" for="__drawer"></label>
 								    <div data-md-component="skip">
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								        <a href="#merge-parallelism-and-memory-pressure" class="md-skip">
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
+								          Skip to content
 								        </a>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								    </div>
 								    <div data-md-component="announce">
 								    </div>
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
+								<header class="md-header md-header--shadow" data-md-component="header">
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								  <nav class="md-header__inner md-grid" aria-label="Header">
 								    <a href="../.." title="obikmer" class="md-header__button md-logo" aria-label="obikmer" data-md-component="logo">
 								  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
 								    </a>
 								    <label class="md-header__button md-icon" for="__drawer">
 								      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
 								    </label>
 								    <div class="md-header__title" data-md-component="header-title">
 								      <div class="md-header__ellipsis">
 								        <div class="md-header__topic">
 								          <span class="md-ellipsis">
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
+								            obikmer
 								          </span>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								        </div>
 								        <div class="md-header__topic" data-md-component="header-topic">
 								          <span class="md-ellipsis">
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								              Merge parallelism & memory
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
 								          </span>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								        </div>
 								      </div>
 								    </div>
 								      <script>var palette=__md_get("__palette");if(palette&&palette.color){if("(prefers-color-scheme)"===palette.color.media){var media=matchMedia("(prefers-color-scheme: light)"),input=document.querySelector(media.matches?"[data-md-color-media='(prefers-color-scheme: light)']":"[data-md-color-media='(prefers-color-scheme: dark)']");palette.color.media=input.getAttribute("data-md-color-media"),palette.color.scheme=input.getAttribute("data-md-color-scheme"),palette.color.primary=input.getAttribute("data-md-color-primary"),palette.color.accent=input.getAttribute("data-md-color-accent")}for(var[key,value]of Object.entries(palette.color))document.body.setAttribute("data-md-color-"+key,value)}</script>
 								  </nav>
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
+								</header>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								    <div class="md-container" data-md-component="container">
 								      <main class="md-main" data-md-component="main">
 								        <div class="md-main__inner md-grid">
 								              <div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
 								                <div class="md-sidebar__scrollwrap">
 								                  <div class="md-sidebar__inner">
 								<nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
 								  <label class="md-nav__title" for="__drawer">
 								    <a href="../.." title="obikmer" class="md-nav__button md-logo" aria-label="obikmer" data-md-component="logo">
 								  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
 								    </a>
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
+								    obikmer
 								  </label>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								  <ul class="md-nav__list" data-md-scrollfix>
 								    <li class="md-nav__item">
 								      <a href="../.." class="md-nav__link">
 								  <span class="md-ellipsis">
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
 								    Home
 								  </span>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								      </a>
 								    </li>
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								    <li class="md-nav__item">
 								      <a href="../../installation/" class="md-nav__link">
 								  <span class="md-ellipsis">
 								    Installation
 								  </span>
 								      </a>
 								    </li>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								    <li class="md-nav__item md-nav__item--nested">
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3" >
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								          <label class="md-nav__link" for="__nav_3" id="__nav_3_label" tabindex="0">
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								  <span class="md-ellipsis">
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
 								    Theory
 								  </span>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								            <span class="md-nav__icon md-icon"></span>
 								          </label>
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_3_label" aria-expanded="false">
 								          <label class="md-nav__title" for="__nav_3">
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								            <span class="md-nav__icon md-icon"></span>
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
 								    Theory
 								          </label>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								          <ul class="md-nav__list" data-md-scrollfix>
 								    <li class="md-nav__item">
 								      <a href="../../kmers/" class="md-nav__link">
 								  <span class="md-ellipsis">
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
 								    Kmers and super-kmers
 								  </span>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								      </a>
 								    </li>
 								    <li class="md-nav__item">
 								      <a href="../../theory/encoding/" class="md-nav__link">
 								  <span class="md-ellipsis">
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
 								    DNA encoding
 								  </span>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								      </a>
 								    </li>
 								    <li class="md-nav__item">
 								      <a href="../../theory/entropy/" class="md-nav__link">
 								  <span class="md-ellipsis">
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
 								    Entropy filter
 								  </span>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								      </a>
 								    </li>
 								    <li class="md-nav__item">
 								      <a href="../../theory/minimizer/" class="md-nav__link">
 								  <span class="md-ellipsis">
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
 								    Minimizer selection
 								  </span>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								      </a>
 								    </li>
 								    <li class="md-nav__item">
 								      <a href="../../theory/indexing/" class="md-nav__link">
 								  <span class="md-ellipsis">
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
 								    Partitioning architecture
 								  </span>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								      </a>
 								    </li>
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
 								    <li class="md-nav__item">
 								      <a href="../../theory/evolutionary_distances/" class="md-nav__link">
 								  <span class="md-ellipsis">
 								    Central-position SNP distance (discussion)
 								  </span>
 								      </a>
 								    </li>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								          </ul>
 								        </nav>
 								    </li>
 								    <li class="md-nav__item md-nav__item--active md-nav__item--nested">
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4" checked>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								          <label class="md-nav__link" for="__nav_4" id="__nav_4_label" tabindex="0">
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								  <span class="md-ellipsis">
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
 								    Implementation
 								  </span>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								            <span class="md-nav__icon md-icon"></span>
 								          </label>
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_4_label" aria-expanded="true">
 								          <label class="md-nav__title" for="__nav_4">
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								            <span class="md-nav__icon md-icon"></span>
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
 								    Implementation
 								          </label>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								          <ul class="md-nav__list" data-md-scrollfix>
 								    <li class="md-nav__item">
 								      <a href="../superkmer/" class="md-nav__link">
 								  <span class="md-ellipsis">
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
 								    SuperKmer
 								  </span>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								      </a>
 								    </li>
 								    <li class="md-nav__item">
 								      <a href="../kmer/" class="md-nav__link">
 								  <span class="md-ellipsis">
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
 								    Kmer
 								  </span>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								      </a>
 								    </li>
 								    <li class="md-nav__item">
 								      <a href="../chunkreader/" class="md-nav__link">
 								  <span class="md-ellipsis">
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
 								    Chunk reader
 								  </span>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								      </a>
 								    </li>
 								    <li class="md-nav__item">
 								      <a href="../pipeline/" class="md-nav__link">
 								  <span class="md-ellipsis">
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
 								    Construction pipeline
 								  </span>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								      </a>
 								    </li>
 								    <li class="md-nav__item">
 								      <a href="../obipipeline/" class="md-nav__link">
 								  <span class="md-ellipsis">
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
 								    obipipeline library
 								  </span>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								      </a>
 								    </li>
 								    <li class="md-nav__item">
 								      <a href="../storage/" class="md-nav__link">
 								  <span class="md-ellipsis">
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
 								    On-disk storage
 								  </span>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								      </a>
 								    </li>
 								    <li class="md-nav__item">
 								      <a href="../mphf/" class="md-nav__link">
 								  <span class="md-ellipsis">
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
 								    MPHF selection
 								  </span>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								      </a>
 								    </li>
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								    <li class="md-nav__item">
 								      <a href="../unitig_evidence/" class="md-nav__link">
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								  <span class="md-ellipsis">
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
 								    Unitig evidence encoding
 								  </span>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								      </a>
 								    </li>
 								    <li class="md-nav__item">
 								      <a href="../evidence_elimination/" class="md-nav__link">
 								  <span class="md-ellipsis">
 								    Evidence elimination (discussion)
 								  </span>
 								      </a>
 								    </li>
 								    <li class="md-nav__item">
 								      <a href="../obilayeredmap/" class="md-nav__link">
 								  <span class="md-ellipsis">
-											docs: document k-mer index architecture and refactor distance metrics
										
										
											2026-05-15 21:07:23 +08:00
 								    obilayeredmap crate
 								  </span>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								      </a>
 								    </li>
 								    <li class="md-nav__item">
 								      <a href="../persistent_compact_int_vec/" class="md-nav__link">
 								  <span class="md-ellipsis">
-											docs: document k-mer index architecture and refactor distance metrics
										
										
											2026-05-15 21:07:23 +08:00
 								    PersistentCompactIntVec
 								  </span>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								      </a>
 								    </li>
 								    <li class="md-nav__item">
 								      <a href="../persistent_bit_vec/" class="md-nav__link">
 								  <span class="md-ellipsis">
-											docs: document k-mer index architecture and refactor distance metrics
										
										
											2026-05-15 21:07:23 +08:00
 								    PersistentBitVec
 								  </span>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								      </a>
 								    </li>
 								    <li class="md-nav__item">
 								      <a href="../merge/" class="md-nav__link">
 								  <span class="md-ellipsis">
 								    Merge command
 								  </span>
 								      </a>
 								    </li>
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
 								    <li class="md-nav__item md-nav__item--active">
 								      <input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
 								        <label class="md-nav__link md-nav__link--active" for="__toc">
 								  <span class="md-ellipsis">
 								    Merge parallelism & memory
 								  </span>
 								          <span class="md-nav__icon md-icon"></span>
 								        </label>
 								      <a href="./" class="md-nav__link md-nav__link--active">
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								  <span class="md-ellipsis">
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								    Merge parallelism & memory
 								  </span>
 								      </a>
 								<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
 								    <label class="md-nav__title" for="__toc">
 								      <span class="md-nav__icon md-icon"></span>
 								      Table of contents
 								    </label>
 								    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
 								        <li class="md-nav__item">
 								  <a href="#problem-observed" class="md-nav__link">
 								    <span class="md-ellipsis">
 								        Problem observed
 								    </span>
 								  </a>
 								</li>
 								        <li class="md-nav__item">
 								  <a href="#root-cause" class="md-nav__link">
 								    <span class="md-ellipsis">
 								        Root cause
 								    </span>
 								  </a>
 								    <nav class="md-nav" aria-label="Root cause">
 								      <ul class="md-nav__list">
 								          <li class="md-nav__item">
 								  <a href="#the-merge-pipeline-per-partition" class="md-nav__link">
 								    <span class="md-ellipsis">
 								        The merge pipeline per partition
 								    </span>
 								  </a>
 								</li>
 								          <li class="md-nav__item">
 								  <a href="#unbounded-rayon-parallelism" class="md-nav__link">
 								    <span class="md-ellipsis">
 								        Unbounded Rayon parallelism
 								    </span>
 								  </a>
 								</li>
 								          <li class="md-nav__item">
 								  <a href="#the-85-gb-single-allocation" class="md-nav__link">
 								    <span class="md-ellipsis">
 								        The 8.5 GB single allocation
 								    </span>
 								  </a>
 								</li>
 								      </ul>
 								    </nav>
 								</li>
 								        <li class="md-nav__item">
 								  <a href="#partition-size-distribution" class="md-nav__link">
 								    <span class="md-ellipsis">
 								        Partition size distribution
 								    </span>
 								  </a>
 								</li>
 								        <li class="md-nav__item">
 								  <a href="#solution-lfd-scheduling-memory-budget-semaphore" class="md-nav__link">
 								    <span class="md-ellipsis">
 								        Solution: LFD scheduling + memory budget semaphore
 								    </span>
 								  </a>
 								    <nav class="md-nav" aria-label="Solution: LFD scheduling + memory budget semaphore">
 								      <ul class="md-nav__list">
 								          <li class="md-nav__item">
 								  <a href="#principle" class="md-nav__link">
 								    <span class="md-ellipsis">
 								        Principle
 								    </span>
 								  </a>
 								</li>
 								          <li class="md-nav__item">
 								  <a href="#memorybudget-obisys" class="md-nav__link">
 								    <span class="md-ellipsis">
 								        MemoryBudget (obisys)
 								    </span>
 								  </a>
 								</li>
 								          <li class="md-nav__item">
 								  <a href="#adaptive-expansion-factor" class="md-nav__link">
 								    <span class="md-ellipsis">
 								        Adaptive expansion factor
 								    </span>
 								  </a>
 								</li>
 								          <li class="md-nav__item">
 								  <a href="#ram-source" class="md-nav__link">
 								    <span class="md-ellipsis">
 								        RAM source
 								    </span>
 								  </a>
 								</li>
 								      </ul>
 								    </nav>
 								</li>
 								        <li class="md-nav__item">
 								  <a href="#diagnostic-report" class="md-nav__link">
 								    <span class="md-ellipsis">
 								        Diagnostic report
 								    </span>
 								  </a>
 								</li>
 								        <li class="md-nav__item">
 								  <a href="#parameters" class="md-nav__link">
 								    <span class="md-ellipsis">
 								        Parameters
 								    </span>
 								  </a>
 								</li>
 								    </ul>
 								</nav>
 								    </li>
 								    <li class="md-nav__item">
 								      <a href="../filtering/" class="md-nav__link">
 								  <span class="md-ellipsis">
 								    Kmer filtering
 								  </span>
 								      </a>
 								    </li>
 								    <li class="md-nav__item">
 								      <a href="../select/" class="md-nav__link">
 								  <span class="md-ellipsis">
 								    Select command
 								  </span>
 								      </a>
 								    </li>
 								    <li class="md-nav__item">
 								      <a href="../obitaxonomy/" class="md-nav__link">
 								  <span class="md-ellipsis">
 								    obitaxonomy crate
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								  </span>
 								      </a>
 								    </li>
 								          </ul>
 								        </nav>
 								    </li>
 								    <li class="md-nav__item md-nav__item--nested">
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_5" >
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								          <label class="md-nav__link" for="__nav_5" id="__nav_5_label" tabindex="0">
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								  <span class="md-ellipsis">
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
 								    Architecture
 								  </span>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								            <span class="md-nav__icon md-icon"></span>
 								          </label>
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_5_label" aria-expanded="false">
 								          <label class="md-nav__title" for="__nav_5">
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								            <span class="md-nav__icon md-icon"></span>
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
 								    Architecture
 								          </label>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								          <ul class="md-nav__list" data-md-scrollfix>
 								    <li class="md-nav__item">
 								      <a href="../../architecture/sequences/invariant/" class="md-nav__link">
 								  <span class="md-ellipsis">
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
 								    Sequences
-											docs: document k-mer index architecture and refactor distance metrics
										
										
											2026-05-15 21:07:23 +08:00
+								  </span>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								      </a>
 								    </li>
 								    <li class="md-nav__item">
 								      <a href="../../architecture/index_architecture/" class="md-nav__link">
 								  <span class="md-ellipsis">
-											docs: document k-mer index architecture and refactor distance metrics
										
										
											2026-05-15 21:07:23 +08:00
 								    Kmer index
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
+								  </span>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								      </a>
 								    </li>
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
 								    <li class="md-nav__item">
 								      <a href="../../architecture/siblings/" class="md-nav__link">
 								  <span class="md-ellipsis">
 								    Sibling annex (discussion)
 								  </span>
 								      </a>
 								    </li>
 								    <li class="md-nav__item">
 								      <a href="../../architecture/numa_worker_pools/" class="md-nav__link">
 								  <span class="md-ellipsis">
 								    NUMA-aware worker pools
 								  </span>
 								      </a>
 								    </li>
 								    <li class="md-nav__item">
 								      <a href="../../architecture/numa_partition_runner/" class="md-nav__link">
 								  <span class="md-ellipsis">
 								    NUMA-aware partition runner
 								  </span>
 								      </a>
 								    </li>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								          </ul>
 								        </nav>
 								    </li>
 								  </ul>
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
+								</nav>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								                  </div>
 								                </div>
 								              </div>
 								              <div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
 								                <div class="md-sidebar__scrollwrap">
 								                  <div class="md-sidebar__inner">
 								<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
 								    <label class="md-nav__title" for="__toc">
 								      <span class="md-nav__icon md-icon"></span>
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
+								      Table of contents
 								    </label>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
 								        <li class="md-nav__item">
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								  <a href="#problem-observed" class="md-nav__link">
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								    <span class="md-ellipsis">
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								        Problem observed
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
 								    </span>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								  </a>
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
+								</li>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								        <li class="md-nav__item">
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								  <a href="#root-cause" class="md-nav__link">
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								    <span class="md-ellipsis">
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								        Root cause
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
 								    </span>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								  </a>
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								    <nav class="md-nav" aria-label="Root cause">
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								      <ul class="md-nav__list">
 								          <li class="md-nav__item">
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								  <a href="#the-merge-pipeline-per-partition" class="md-nav__link">
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								    <span class="md-ellipsis">
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								        The merge pipeline per partition
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
 								    </span>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								  </a>
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
+								</li>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								          <li class="md-nav__item">
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								  <a href="#unbounded-rayon-parallelism" class="md-nav__link">
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								    <span class="md-ellipsis">
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								        Unbounded Rayon parallelism
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
 								    </span>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								  </a>
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
+								</li>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								          <li class="md-nav__item">
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								  <a href="#the-85-gb-single-allocation" class="md-nav__link">
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								    <span class="md-ellipsis">
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								        The 8.5 GB single allocation
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
 								    </span>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								  </a>
 								</li>
 								      </ul>
 								    </nav>
 								</li>
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								        <li class="md-nav__item">
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								  <a href="#partition-size-distribution" class="md-nav__link">
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								    <span class="md-ellipsis">
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								        Partition size distribution
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
 								    </span>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								  </a>
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								</li>
 								        <li class="md-nav__item">
 								  <a href="#solution-lfd-scheduling-memory-budget-semaphore" class="md-nav__link">
 								    <span class="md-ellipsis">
 								        Solution: LFD scheduling + memory budget semaphore
 								    </span>
 								  </a>
 								    <nav class="md-nav" aria-label="Solution: LFD scheduling + memory budget semaphore">
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								      <ul class="md-nav__list">
 								          <li class="md-nav__item">
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								  <a href="#principle" class="md-nav__link">
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								    <span class="md-ellipsis">
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								        Principle
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
 								    </span>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								  </a>
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
+								</li>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								          <li class="md-nav__item">
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								  <a href="#memorybudget-obisys" class="md-nav__link">
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								    <span class="md-ellipsis">
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								        MemoryBudget (obisys)
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
 								    </span>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								  </a>
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
+								</li>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								          <li class="md-nav__item">
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								  <a href="#adaptive-expansion-factor" class="md-nav__link">
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								    <span class="md-ellipsis">
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								        Adaptive expansion factor
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
 								    </span>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								  </a>
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
+								</li>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								          <li class="md-nav__item">
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								  <a href="#ram-source" class="md-nav__link">
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								    <span class="md-ellipsis">
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								        RAM source
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
 								    </span>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								  </a>
 								</li>
 								      </ul>
 								    </nav>
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
+								</li>
-											docs: clarify MPHF indexing, storage layout, and distance traits
										
										
											2026-05-17 10:20:22 +08:00
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								        <li class="md-nav__item">
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								  <a href="#diagnostic-report" class="md-nav__link">
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								    <span class="md-ellipsis">
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								        Diagnostic report
-											docs: clarify MPHF indexing, storage layout, and distance traits
										
										
											2026-05-17 10:20:22 +08:00
 								    </span>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								  </a>
-											docs: clarify MPHF indexing, storage layout, and distance traits
										
										
											2026-05-17 10:20:22 +08:00
+								</li>
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								        <li class="md-nav__item">
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								  <a href="#parameters" class="md-nav__link">
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								    <span class="md-ellipsis">
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								        Parameters
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								    </span>
 								  </a>
 								</li>
 								    </ul>
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
+								</nav>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								                  </div>
 								                </div>
 								              </div>
 								            <div class="md-content" data-md-component="content">
 								              <article class="md-content__inner md-typeset">
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								<h1 id="merge-parallelism-and-memory-pressure">Merge parallelism and memory pressure</h1>
 								<h2 id="problem-observed">Problem observed</h2>
 								<p>Running <code>obikmer merge</code> over 109 indexes (108 sources + 1 bootstrap) on a 192-core machine
 								produces a fatal OOM during the <code>merge_partitions</code> stage:</p>
 								<div class="highlight"><pre><span></span><code>memory allocation of 9126805520 bytes failed
 								</code></pre></div>
 								<p>A single allocation of ~8.5 GB fails. This is not an aggregate; it is one <code>malloc</code> call
 								from hashbrown during a HashMap resize.</p>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								<hr />
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								<h2 id="root-cause">Root cause</h2>
 								<h3 id="the-merge-pipeline-per-partition">The merge pipeline per partition</h3>
 								<div class="highlight"><pre><span></span><code>source unitigs.bin
 								  → iter_indexed_canonical_kmers()
 								  → GraphDeBruijn::push()       ← HashSet&lt;u64&gt; + 1 byte flags, all in RAM
 								  → compute_degrees_and_mark_starts()
 								  → try_for_each_unitig()
 								  → unitigs.bin (new layer)
 								  → Layer::build() → MPHF + evidence
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
+								</code></pre></div>
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								<p><code>GraphDeBruijn</code> is a <code>FastHashMap&lt;CanonicalKmer, AtomicU8&gt;</code> — a <code>HashSet&lt;u64&gt;</code> with
 								one flag byte per node. Neighbor lookup is implicit: 4 probes into the same map.
 								No edges are stored. The full kmer set of one partition must reside in RAM
 								simultaneously to compute degrees and mark unitig starts.</p>
 								<p>The matrix builders that follow (pass 2) are mmapped files — they do <strong>not</strong> consume
 								significant RAM. The pressure is entirely in pass 1.</p>
 								<h3 id="unbounded-rayon-parallelism">Unbounded Rayon parallelism</h3>
 								<p>With 192 cores, Rayon ran up to 192 partitions concurrently. Each partition built its
 								own <code>GraphDeBruijn</code> accumulating all kmers absent from the destination. Peak memory =
 × peak_partition_hashset.</p>
 								<h3 id="the-85-gb-single-allocation">The 8.5 GB single allocation</h3>
 								<p>hashbrown allocates the entire backing array in one call when rehashing.
 								At load factor 7/8: <code>capacity × (sizeof(K,V) + 1 control byte)</code>.
 								For <code>(u64, AtomicU8)</code> with alignment: ~16 bytes per slot.</p>
 								<div class="highlight"><pre><span></span><code>9 127 MB / 16 bytes ≈ 570 M slots → ~380 M new kmers in one partition
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
+								</code></pre></div>
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								<p>Plausible for the largest partition of 108 Salix/Betula sources (~450 Mbp each).</p>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								<hr />
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								<h2 id="partition-size-distribution">Partition size distribution</h2>
 								<p><code>obikmer utils --partition-stats</code> measures the sum of <code>unitigs.bin</code> file sizes
 								per partition across all source indexes (pure <code>stat()</code> syscalls, negligible cost).</p>
 								<p>Observed on a 9-genome pilot (256 partitions):</p>
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
+								<table>
 								<thead>
 								<tr>
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								<th>Stat</th>
 								<th>Value</th>
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
+								</tr>
 								</thead>
 								<tbody>
 								<tr>
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								<td>min</td>
 								<td>30.5 MB</td>
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
+								</tr>
 								<tr>
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								<td>max</td>
 								<td>232.1 MB</td>
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
+								</tr>
 								<tr>
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								<td>mean</td>
 								<td>40.1 MB</td>
 								</tr>
 								<tr>
 								<td>median</td>
 								<td>37.2 MB</td>
 								</tr>
 								<tr>
 								<td>p95</td>
 								<td>47.1 MB</td>
 								</tr>
 								<tr>
 								<td>max/median ratio</td>
 								<td>6.23×</td>
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
+								</tr>
 								</tbody>
 								</table>
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								<p>The distribution is <strong>bimodal with a heavy tail</strong>:
 								- 238/256 partitions in a narrow 30–50 MB band
 								- 4 structurally extreme partitions (3–6× the median): 221, 233, 135, 191</p>
 								<p>These correspond to minimizers over-represented in repetitive regions shared across
 								all sources. They are extreme in every run on this dataset.</p>
 								<p>With 109 sources, outlier partitions do not scale linearly: only kmers <strong>absent from
 								the destination</strong> enter the GraphDeBruijn, and inter-source overlap is high for closely
 								related species. Partition 221 is the likely trigger for the 8.5 GB crash.</p>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								<hr />
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								<h2 id="solution-lfd-scheduling-memory-budget-semaphore">Solution: LFD scheduling + memory budget semaphore</h2>
 								<h3 id="principle">Principle</h3>
 								<p>Pre-sort partitions by <strong>decreasing estimated size</strong> (First Fit Decreasing — FFD),
 								then schedule them through a <strong>continuous memory budget semaphore</strong>. Each worker
 								acquires an estimated cost before starting and releases it on completion.</p>
 								<p>Large partitions run first when the full budget is available; small partitions fill
 								the gaps. No hard outlier threshold is needed.</p>
 								<h3 id="memorybudget-obisys"><code>MemoryBudget</code> (<code>obisys</code>)</h3>
 								<div class="highlight"><pre><span></span><code><span class="k">pub</span><span class="w"> </span><span class="k">struct</span><span class="w"> </span><span class="nc">MemoryBudget</span><span class="w"> </span><span class="p">{</span><span class="w"> </span><span class="err">…</span><span class="w"> </span><span class="p">}</span>
 								<span class="k">impl</span><span class="w"> </span><span class="n">MemoryBudget</span><span class="w"> </span><span class="p">{</span>
 								<span class="w">    </span><span class="k">pub</span><span class="w"> </span><span class="k">fn</span><span class="w"> </span><span class="nf">new</span><span class="p">(</span><span class="n">total</span><span class="p">:</span><span class="w"> </span><span class="kt">u64</span><span class="p">)</span><span class="w"> </span><span class="p">-&gt;</span><span class="w"> </span><span class="nc">Self</span><span class="p">;</span>
 								<span class="w">    </span><span class="k">pub</span><span class="w"> </span><span class="k">fn</span><span class="w"> </span><span class="nf">acquire</span><span class="p">(</span><span class="o">&amp;</span><span class="bp">self</span><span class="p">,</span><span class="w"> </span><span class="n">cost</span><span class="p">:</span><span class="w"> </span><span class="kt">u64</span><span class="p">);</span><span class="w">  </span><span class="c1">// blocks until budget available</span>
 								<span class="w">    </span><span class="k">pub</span><span class="w"> </span><span class="k">fn</span><span class="w"> </span><span class="nf">release</span><span class="p">(</span><span class="o">&amp;</span><span class="bp">self</span><span class="p">,</span><span class="w"> </span><span class="n">cost</span><span class="p">:</span><span class="w"> </span><span class="kt">u64</span><span class="p">);</span>
 								<span class="w">    </span><span class="k">pub</span><span class="w"> </span><span class="k">fn</span><span class="w"> </span><span class="nf">peak_active</span><span class="p">(</span><span class="o">&amp;</span><span class="bp">self</span><span class="p">)</span><span class="w"> </span><span class="p">-&gt;</span><span class="w"> </span><span class="kt">usize</span><span class="p">;</span>
 								<span class="p">}</span>
-											docs: document k-mer index architecture and refactor distance metrics
										
										
											2026-05-15 21:07:23 +08:00
+								</code></pre></div>
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								<p>Non-deadlock guarantee: when <code>active == 0</code>, acquire always succeeds regardless of cost.
 								Without this, a partition whose estimated cost exceeds the total budget would block forever.</p>
 								<h3 id="adaptive-expansion-factor">Adaptive expansion factor</h3>
 								<p>The expansion factor converts raw <code>unitigs.bin</code> bytes into an estimated GraphDeBruijn
 								RAM footprint. hashbrown stores each kmer as <code>(u64, AtomicU8)</code> ≈ 16 bytes/kmer at 7/8
 								load factor; unitig files encode ≈ 2 bits/base. The ratio depends on average unitig
 								length (short unitigs: ~2×; long unitigs: up to ~50×).</p>
 								<p><strong>Phase 1 — sequential pilot (worst partition)</strong></p>
 								<p>The largest partition runs alone first. Its actual <code>g.len()</code> seeds the expansion factor
 								before any parallel job starts. <code>FALLBACK_EXPANSION = 4×</code> is used only for empty partitions.</p>
 								<div class="highlight"><pre><span></span><code><span class="kd">let</span><span class="w"> </span><span class="n">worst_g_len</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">dst_partition</span><span class="p">.</span><span class="n">merge_partition</span><span class="p">(</span><span class="n">worst_id</span><span class="p">,</span><span class="w"> </span><span class="err">…</span><span class="p">)</span><span class="o">?</span><span class="p">;</span>
 								<span class="c1">//                              ↑ now returns SKResult&lt;usize&gt; (was SKResult&lt;()&gt;)</span>
 								<span class="kd">let</span><span class="w"> </span><span class="n">seed_expansion</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">worst_g_len</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="kt">u64</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="mi">16</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="mi">1000</span><span class="w"> </span><span class="o">/</span><span class="w"> </span><span class="n">worst_bytes</span><span class="p">;</span>
 								<span class="kd">let</span><span class="w"> </span><span class="n">max_expansion</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">AtomicU64</span><span class="p">::</span><span class="n">new</span><span class="p">(</span><span class="n">seed_expansion</span><span class="p">);</span>
 								</code></pre></div>
 								<p><strong>Phase 2 — parallel with adaptive updates</strong></p>
 								<div class="highlight"><pre><span></span><code><span class="n">order</span><span class="p">[</span><span class="mi">1</span><span class="o">..</span><span class="p">].</span><span class="n">into_par_iter</span><span class="p">().</span><span class="n">for_each</span><span class="p">(</span><span class="o">|&amp;</span><span class="n">i</span><span class="o">|</span><span class="w"> </span><span class="p">{</span>
 								<span class="w">    </span><span class="kd">let</span><span class="w"> </span><span class="n">cost</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">partition_sizes</span><span class="p">[</span><span class="n">i</span><span class="p">]</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="n">max_expansion</span><span class="p">.</span><span class="n">load</span><span class="p">(</span><span class="n">Relaxed</span><span class="p">)</span><span class="w"> </span><span class="o">/</span><span class="w"> </span><span class="mi">1000</span><span class="p">;</span>
 								<span class="w">    </span><span class="n">budget</span><span class="p">.</span><span class="n">acquire</span><span class="p">(</span><span class="n">cost</span><span class="p">);</span>
 								<span class="w">    </span><span class="kd">let</span><span class="w"> </span><span class="n">g_len</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">dst_partition</span><span class="p">.</span><span class="n">merge_partition</span><span class="p">(</span><span class="n">i</span><span class="p">,</span><span class="w"> </span><span class="err">…</span><span class="p">)</span><span class="o">?</span><span class="p">;</span>
 								<span class="w">    </span><span class="n">budget</span><span class="p">.</span><span class="n">release</span><span class="p">(</span><span class="n">cost</span><span class="p">);</span><span class="w">          </span><span class="c1">// releases estimated cost, not actual</span>
 								<span class="w">    </span><span class="kd">let</span><span class="w"> </span><span class="n">actual</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">g_len</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="kt">u64</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="mi">16</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="mi">1000</span><span class="w"> </span><span class="o">/</span><span class="w"> </span><span class="n">partition_sizes</span><span class="p">[</span><span class="n">i</span><span class="p">];</span>
 								<span class="w">    </span><span class="n">max_expansion</span><span class="p">.</span><span class="n">fetch_max</span><span class="p">(</span><span class="n">actual</span><span class="p">,</span><span class="w"> </span><span class="n">Relaxed</span><span class="p">);</span><span class="w">  </span><span class="c1">// always pessimistic (max)</span>
 								<span class="p">});</span>
 								</code></pre></div>
 								<p><code>budget.release(cost)</code> uses the estimated cost, not the actual one. The budget tracks
 								reservations, not physical RAM; each partition pays what it promised at acquisition.</p>
 								<p><strong>On the safety margin</strong></p>
 								<p>There is no separate multiplier <code>k</code>. It is redundant with <code>budget_fraction</code>: both
 								reduce effective concurrency by the same amount. A single parameter is easier to
 								calibrate. <code>budget_fraction = 0.5</code> (default) reserves half of available RAM for the
 								OS, MPHF build, pass 2, and estimation error.</p>
 								<p><code>--budget-fraction</code> is exposed as a CLI flag — the only escape hatch for pathological
 								cases (extreme repetitive content, unusually long unitigs) that still cause OOM.</p>
 								<h3 id="ram-source">RAM source</h3>
 								<p><code>obisys::available_memory_bytes()</code> — wraps <code>sysinfo::System::available_memory()</code>,
 								falls back to <code>total / 2</code> on macOS when the memory compressor returns 0.</p>
 								<hr />
 								<h2 id="diagnostic-report">Diagnostic report</h2>
 								<p>After the parallel phase, <code>merge_partition</code> emits a structured report via <code>tracing::info!</code>:</p>
 								<div class="highlight"><pre><span></span><code>─── merge_partitions memory report ───
 								  available RAM : 512.0 GB   budget 50% = 256.0 GB
 								  expansion factor — seed: 4.2×  final max: 6.1×  (mean: 1.8×  median: 1.6×)
 								  peak concurrent workers: 42
 								  expansion factor distribution (256 partitions with data):
 .50× –  1.25× │██████████████████████████████ 148
 .25× –  2.00× │████████████████████████        82
 								     …
 .50× –  6.25× │█                                2
 								  top partitions by actual expansion factor:
 								    partition  221 : 6.10×  (232.1 MB unitigs → 48M kmers, reserved at 4.20×)
 								    partition  135 : 5.82×  (127.3 MB unitigs → 24M kmers, reserved at 4.20×)
 								    …
 								──────────────────────────────────────
 								</code></pre></div>
 								<p>Fields useful for diagnosis:</p>
-											docs: document k-mer index architecture and refactor distance metrics
										
										
											2026-05-15 21:07:23 +08:00
+								<table>
 								<thead>
 								<tr>
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								<th>Field</th>
 								<th>Interpretation</th>
-											docs: document k-mer index architecture and refactor distance metrics
										
										
											2026-05-15 21:07:23 +08:00
+								</tr>
 								</thead>
 								<tbody>
 								<tr>
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								<td><code>seed</code> vs <code>final max</code> expansion</td>
 								<td>gap indicates partitions with higher expansion than the worst-by-size</td>
-											docs: document k-mer index architecture and refactor distance metrics
										
										
											2026-05-15 21:07:23 +08:00
+								</tr>
 								<tr>
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								<td><code>reserved at X×</code></td>
 								<td>the factor used at acquisition; if much lower than actual, the budget was under-reserved for that partition</td>
 								</tr>
 								<tr>
 								<td><code>peak concurrent workers</code></td>
 								<td>effective parallelism achieved under the budget constraint</td>
 								</tr>
 								<tr>
 								<td><code>mean</code> / <code>median</code> expansion</td>
 								<td>typical dataset characteristic; stable across runs on the same data</td>
-											docs: document k-mer index architecture and refactor distance metrics
										
										
											2026-05-15 21:07:23 +08:00
+								</tr>
 								</tbody>
 								</table>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								<hr />
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								<h2 id="parameters">Parameters</h2>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								<table>
 								<thead>
 								<tr>
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								<th>Parameter</th>
 								<th>Default</th>
 								<th>CLI flag</th>
 								<th>Notes</th>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								</tr>
 								</thead>
 								<tbody>
 								<tr>
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								<td><code>fallback_expansion</code></td>
 								<td>4×</td>
 								<td>—</td>
 								<td>seed for empty partitions only</td>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								</tr>
 								<tr>
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								<td><code>budget_fraction</code></td>
 								<td>0.5</td>
 								<td><code>--budget-fraction</code></td>
 								<td>reduce if OOM persists</td>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								</tr>
 								<tr>
-											large refactoring
										
										
											2026-08-15 20:56:29 +02:00
+								<td>RAM source</td>
 								<td><code>obisys::available_memory_bytes()</code></td>
 								<td>—</td>
 								<td>falls back to <code>total/2</code> on macOS</td>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								</tr>
 								</tbody>
 								</table>
 								              </article>
 								            </div>
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
+								<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								        </div>
 								      </main>
 								        <footer class="md-footer">
 								  <div class="md-footer-meta md-typeset">
 								    <div class="md-footer-meta__inner md-grid">
 								      <div class="md-copyright">
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
 								    Made with
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
+								    <a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
+								      Material for MkDocs
 								    </a>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
+								</div>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								    </div>
 								  </div>
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
+								</footer>
-											docs: expand kmer indexing, filtering, and merging documentation
										
										
											2026-06-04 21:27:01 +02:00
 								    </div>
 								    <div class="md-dialog" data-md-component="dialog">
 								      <div class="md-dialog__inner md-typeset"></div>
 								    </div>
 								      <script id="__config" type="application/json">{"annotate": null, "base": "../..", "features": [], "search": "../../assets/javascripts/workers/search.2c215733.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
 								      <script src="../../assets/javascripts/bundle.79ae519e.min.js"></script>
 								        <script src="https://unpkg.com/mathjax@3/es5/tex-mml-chtml.js"></script>
 								  </body>
-											refactor: implement RoutableSuperKmer and update k-mer indexing pipeline
										
										
											2026-04-29 22:52:42 +02:00
+								</html>