doc/implementation/storage/index.html


<!doctype html>
<html lang="en" class="no-js">
  <head>
    
      <meta charset="utf-8">
      <meta name="viewport" content="width=device-width,initial-scale=1">
      
      
        <link rel="prev" href="../obipipeline/">
      
      
        <link rel="next" href="../mphf/">
      
      
      <link rel="icon" href="../../assets/images/favicon.png">
      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.7.6">
    
    
        <title>On-disk storage - obikmer</title>
      
    
      <link rel="stylesheet" href="../../assets/stylesheets/main.484c7ddc.min.css">
      
      
        <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
        <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
        <style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
      
    
    <script>__md_scope=new URL("../..",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
    
      
  </head>
  
  
    <body dir="ltr">
  
    
    <input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
    <input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
    <label class="md-overlay" for="__drawer"></label>
    <div data-md-component="skip">
      
        
        <a href="#on-disk-collection-structure" class="md-skip">
          Skip to content
        </a>
      
    </div>
    <div data-md-component="announce">
      
    </div>
    
    
<header class="md-header md-header--shadow" data-md-component="header">
  <nav class="md-header__inner md-grid" aria-label="Header">
    <a href="../.." title="obikmer" class="md-header__button md-logo" aria-label="obikmer" data-md-component="logo">
      
  
  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>

    </a>
    <label class="md-header__button md-icon" for="__drawer">
      
      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
    </label>
    <div class="md-header__title" data-md-component="header-title">
      <div class="md-header__ellipsis">
        <div class="md-header__topic">
          <span class="md-ellipsis">
            obikmer
          </span>
        </div>
        <div class="md-header__topic" data-md-component="header-topic">
          <span class="md-ellipsis">
            
              On-disk storage
            
          </span>
        </div>
      </div>
    </div>
    
    
      <script>var palette=__md_get("__palette");if(palette&&palette.color){if("(prefers-color-scheme)"===palette.color.media){var media=matchMedia("(prefers-color-scheme: light)"),input=document.querySelector(media.matches?"[data-md-color-media='(prefers-color-scheme: light)']":"[data-md-color-media='(prefers-color-scheme: dark)']");palette.color.media=input.getAttribute("data-md-color-media"),palette.color.scheme=input.getAttribute("data-md-color-scheme"),palette.color.primary=input.getAttribute("data-md-color-primary"),palette.color.accent=input.getAttribute("data-md-color-accent")}for(var[key,value]of Object.entries(palette.color))document.body.setAttribute("data-md-color-"+key,value)}</script>
    
    
  </nav>
  
</header>
    
    <div class="md-container" data-md-component="container">
      
      
      <main class="md-main" data-md-component="main">
        <div class="md-main__inner md-grid">
          
            
              <div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
                <div class="md-sidebar__scrollwrap">
                  <div class="md-sidebar__inner">
                    

<nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
  <label class="md-nav__title" for="__drawer">
    <a href="../.." title="obikmer" class="md-nav__button md-logo" aria-label="obikmer" data-md-component="logo">
      
  
  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>

    </a>
    obikmer
  </label>
  
  <ul class="md-nav__list" data-md-scrollfix>
    
      
    <li class="md-nav__item">
      <a href="../.." class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    Home
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item md-nav__item--nested">
      
        
        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2" >
        
          
          <label class="md-nav__link" for="__nav_2" id="__nav_2_label" tabindex="0">
            
  
  <span class="md-ellipsis">
    
  
    Theory
  

  </span>
  
  
            <span class="md-nav__icon md-icon"></span>
          </label>
        
        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_2_label" aria-expanded="false">
          <label class="md-nav__title" for="__nav_2">
            <span class="md-nav__icon md-icon"></span>
            
  
    Theory
  

          </label>
          <ul class="md-nav__list" data-md-scrollfix>
            
              
    <li class="md-nav__item">
      <a href="../../kmers/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    Kmers and super-kmers
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../../theory/encoding/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    DNA encoding
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../../theory/entropy/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    Entropy filter
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../../theory/minimizer/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    Minimizer selection
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../../theory/indexing/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    Partitioning architecture
  

  </span>
  
  
      </a>
    </li>
  

          </ul>
        </nav>
      
    </li>
  

    <li class="md-nav__item md-nav__item--active md-nav__item--nested">
      
        
        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3" checked>
        
          
          <label class="md-nav__link" for="__nav_3" id="__nav_3_label" tabindex="0">
            
  
  <span class="md-ellipsis">
    
  
    Implementation
  

  </span>
  
  
            <span class="md-nav__icon md-icon"></span>
          </label>
        
        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_3_label" aria-expanded="true">
          <label class="md-nav__title" for="__nav_3">
            <span class="md-nav__icon md-icon"></span>
            
  
    Implementation
  

          </label>
          <ul class="md-nav__list" data-md-scrollfix>
            
              
    <li class="md-nav__item">
      <a href="../superkmer/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    SuperKmer
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../kmer/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    Kmer
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../chunkreader/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    Chunk reader
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../pipeline/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    Construction pipeline
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../obipipeline/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    obipipeline library
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item md-nav__item--active">
      
      <input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
      
      
        <label class="md-nav__link md-nav__link--active" for="__toc">
          
  
  <span class="md-ellipsis">
    
  
    On-disk storage
  

  </span>
  
  
          <span class="md-nav__icon md-icon"></span>
        </label>
      
      <a href="./" class="md-nav__link md-nav__link--active">
        
  
  <span class="md-ellipsis">
    
  
    On-disk storage
  

  </span>
  
  
      </a>
      
        
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
  
  
    <label class="md-nav__title" for="__toc">
      <span class="md-nav__icon md-icon"></span>
      Table of contents
    </label>
    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
      
        <li class="md-nav__item">
  <a href="#collection-parameters" class="md-nav__link">
    <span class="md-ellipsis">
      
        Collection parameters
      
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#count-storage" class="md-nav__link">
    <span class="md-ellipsis">
      
        Count storage
      
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#query-protocol" class="md-nav__link">
    <span class="md-ellipsis">
      
        Query protocol
      
    </span>
  </a>
  
</li>
      
    </ul>
  
</nav>
      
    </li>
  

    <li class="md-nav__item">
      <a href="../mphf/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    MPHF selection
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../unitig_evidence/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    Unitig evidence encoding
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../obilayeredmap/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    obilayeredmap crate
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../persistent_compact_int_vec/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    PersistentCompactIntVec
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../persistent_bit_vec/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    PersistentBitVec
  

  </span>
  
  
      </a>
    </li>
  

          </ul>
        </nav>
      
    </li>
  

    <li class="md-nav__item md-nav__item--nested">
      
        
        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4" >
        
          
          <label class="md-nav__link" for="__nav_4" id="__nav_4_label" tabindex="0">
            
  
  <span class="md-ellipsis">
    
  
    Architecture
  

  </span>
  
  
            <span class="md-nav__icon md-icon"></span>
          </label>
        
        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_4_label" aria-expanded="false">
          <label class="md-nav__title" for="__nav_4">
            <span class="md-nav__icon md-icon"></span>
            
  
    Architecture
  

          </label>
          <ul class="md-nav__list" data-md-scrollfix>
            
              
    <li class="md-nav__item">
      <a href="../../architecture/sequences/invariant/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    Sequences
  

  </span>
  
  
      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../../architecture/index_architecture/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    
  
    Kmer index
  

  </span>
  
  
      </a>
    </li>
  

          </ul>
        </nav>
      
    </li>
  

  </ul>
</nav>
                  </div>
                </div>
              </div>
            
            
              <div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
                <div class="md-sidebar__scrollwrap">
                  <div class="md-sidebar__inner">
                    

<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
  
  
    <label class="md-nav__title" for="__toc">
      <span class="md-nav__icon md-icon"></span>
      Table of contents
    </label>
    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
      
        <li class="md-nav__item">
  <a href="#collection-parameters" class="md-nav__link">
    <span class="md-ellipsis">
      
        Collection parameters
      
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#count-storage" class="md-nav__link">
    <span class="md-ellipsis">
      
        Count storage
      
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#query-protocol" class="md-nav__link">
    <span class="md-ellipsis">
      
        Query protocol
      
    </span>
  </a>
  
</li>
      
    </ul>
  
</nav>
                  </div>
                </div>
              </div>
            
          
            <div class="md-content" data-md-component="content">
              
              <article class="md-content__inner md-typeset">
                
                  
<h1 id="on-disk-collection-structure">On-disk collection structure</h1>
<p>Collections are too large to hold in RAM (hundreds of genomes, billions of kmers). The collection lives on disk as a directory of memory-mapped files:</p>
<div class="highlight"><pre><span></span><code>collection/
  metadata.toml          — collection parameters (see below)
  part_XXXX/
    superkmers.bin.gz    — dereplicated super-kmers for this partition (construction artifact)
    mphf.bin             — minimal perfect hash function for this partition
    counts.bin           — packed n-bit count array (or 1-bit presence array)
    refs.bin             — back-references u32 nucleotide offset into unitigs.bin per kmer
    unitigs.bin          — local de Bruijn unitigs (permanent evidence structure)
    overflow.bin         — counts exceeding the packed range (optional)
</code></pre></div>
<p><code>superkmers.bin.gz</code> is produced during phase 1 and consumed through phases 2–4. It can be deleted after phase 5 — it is not needed for querying. The permanent query structure is <code>mphf.bin + counts.bin + refs.bin + unitigs.bin</code>.</p>
<h2 id="collection-parameters">Collection parameters</h2>
<p>Stored in <code>metadata.toml</code>:</p>
<table>
<thead>
<tr>
<th>Parameter</th>
<th>Role</th>
</tr>
</thead>
<tbody>
<tr>
<td>k</td>
<td>kmer length</td>
</tr>
<tr>
<td>m</td>
<td>minimizer length (odd, &lt; k)</td>
</tr>
<tr>
<td>p</td>
<td>partition bits (0 ≤ p ≤ min(14, 2m−16))</td>
</tr>
<tr>
<td>mode</td>
<td><code>presence</code> (1 bit/kmer) or <code>count</code> (n bits/kmer)</td>
</tr>
<tr>
<td>n</td>
<td>bits per kmer in count mode (chosen at construction)</td>
</tr>
<tr>
<td>min_count</td>
<td>singleton filtering threshold (0 = keep all)</td>
</tr>
<tr>
<td>hash_fn</td>
<td>hash function identifier</td>
</tr>
<tr>
<td>hash_seed</td>
<td>seed for the hash function</td>
</tr>
</tbody>
</table>
<h2 id="count-storage">Count storage</h2>
<p><strong>refs.bin capacity:</strong> <code>unitigs.bin</code> is a flat 2-bit-packed nucleotide stream with no separators. Each entry in <code>refs.bin</code> is a u32 nucleotide offset pointing to the first base of the kmer. A u32 covers 4 billion nucleotide positions = 1 GB of sequence per partition. In the worst case (all unitigs of length 1 kmer, offsets spaced k apart), this supports 4 billion / k ≈ 130 million kmers per partition at k=31. In the typical case (long unitigs, consecutive kmers at offset +1), the limit approaches 4 billion kmers — well beyond any realistic partition size.</p>
<p><em>Presence mode</em> (coverage ≤ 1x, or when only presence/absence matters):</p>
<ul>
<li><code>counts.bin</code> is a packed 1-bit array — all bits set to 1 for indexed kmers</li>
<li>Singletons are the signal, not filtered</li>
</ul>
<p><em>Count mode</em> (coverage &gt; 1x):</p>
<ul>
<li><code>counts.bin</code> is a packed n-bit array; n chosen at construction from the observed distribution</li>
<li>Value 0: absent sentinel; values 1..2ⁿ−2: direct count; value 2ⁿ−1: overflow</li>
<li>Overflow counts stored in a separate <code>overflow.bin</code> as sorted <code>(index: u32, count: u32)</code> pairs</li>
<li>Empirically (k=31, 15x coverage): n=5 covers 97% of real kmers, n=6 covers 99%</li>
<li>min_count threshold filters low-frequency kmers (errors) before indexing; for ≤1x, min_count=0</li>
</ul>
<h2 id="query-protocol">Query protocol</h2>
<div class="highlight"><pre><span></span><code>query kmer q
  → canonical_minimizer(q) → hash → PART → part_XXXX/
  → MPHF(q) → index i
  → refs[i] = (unitig_id, kmer_offset)
  → read unitig from unitigs.bin → extract kmer at kmer_offset → compare with q
  → match   : return counts[i]
  → no match: kmer absent
</code></pre></div>


              </article>
            </div>
          
          
<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
        </div>
        
      </main>
      
        <footer class="md-footer">
  
  <div class="md-footer-meta md-typeset">
    <div class="md-footer-meta__inner md-grid">
      <div class="md-copyright">
  
  
    Made with
    <a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
      Material for MkDocs
    </a>
  
</div>
      
    </div>
  </div>
</footer>
      
    </div>
    <div class="md-dialog" data-md-component="dialog">
      <div class="md-dialog__inner md-typeset"></div>
    </div>
    
    
      <script id="__config" type="application/json">{"annotate": null, "base": "../..", "features": [], "search": "../../assets/javascripts/workers/search.2c215733.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
    
    
      <script src="../../assets/javascripts/bundle.79ae519e.min.js"></script>
      
        <script src="https://unpkg.com/mathjax@3/es5/tex-mml-chtml.js"></script>
      
    
  </body>
</html>