2843 lines
122 KiB
HTML
2843 lines
122 KiB
HTML
|
|
||
|
|
<!doctype html>
|
||
|
|
<html lang="en" class="no-js">
|
||
|
|
<head>
|
||
|
|
|
||
|
|
<meta charset="utf-8">
|
||
|
|
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<link rel="prev" href="../benchmark_query_testing/">
|
||
|
|
|
||
|
|
|
||
|
|
<link rel="next" href="../../architecture/sequences/invariant/">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<link rel="icon" href="../../assets/images/favicon.png">
|
||
|
|
<meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.7.6">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<title>Partition and layer caching (discussion) - obikmer</title>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<link rel="stylesheet" href="../../assets/stylesheets/main.484c7ddc.min.css">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
||
|
|
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
|
||
|
|
<style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<script>__md_scope=new URL("../..",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</head>
|
||
|
|
|
||
|
|
|
||
|
|
<body dir="ltr">
|
||
|
|
|
||
|
|
|
||
|
|
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
||
|
|
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
||
|
|
<label class="md-overlay" for="__drawer"></label>
|
||
|
|
<div data-md-component="skip">
|
||
|
|
|
||
|
|
|
||
|
|
<a href="#partition-and-layer-caching-discussion" class="md-skip">
|
||
|
|
Skip to content
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</div>
|
||
|
|
<div data-md-component="announce">
|
||
|
|
|
||
|
|
</div>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<header class="md-header md-header--shadow" data-md-component="header">
|
||
|
|
<nav class="md-header__inner md-grid" aria-label="Header">
|
||
|
|
<a href="../.." title="obikmer" class="md-header__button md-logo" aria-label="obikmer" data-md-component="logo">
|
||
|
|
|
||
|
|
|
||
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
|
||
|
|
|
||
|
|
</a>
|
||
|
|
<label class="md-header__button md-icon" for="__drawer">
|
||
|
|
|
||
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
|
||
|
|
</label>
|
||
|
|
<div class="md-header__title" data-md-component="header-title">
|
||
|
|
<div class="md-header__ellipsis">
|
||
|
|
<div class="md-header__topic">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
obikmer
|
||
|
|
</span>
|
||
|
|
</div>
|
||
|
|
<div class="md-header__topic" data-md-component="header-topic">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
Partition and layer caching (discussion)
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</div>
|
||
|
|
</div>
|
||
|
|
</div>
|
||
|
|
|
||
|
|
|
||
|
|
<script>var palette=__md_get("__palette");if(palette&&palette.color){if("(prefers-color-scheme)"===palette.color.media){var media=matchMedia("(prefers-color-scheme: light)"),input=document.querySelector(media.matches?"[data-md-color-media='(prefers-color-scheme: light)']":"[data-md-color-media='(prefers-color-scheme: dark)']");palette.color.media=input.getAttribute("data-md-color-media"),palette.color.scheme=input.getAttribute("data-md-color-scheme"),palette.color.primary=input.getAttribute("data-md-color-primary"),palette.color.accent=input.getAttribute("data-md-color-accent")}for(var[key,value]of Object.entries(palette.color))document.body.setAttribute("data-md-color-"+key,value)}</script>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</nav>
|
||
|
|
|
||
|
|
</header>
|
||
|
|
|
||
|
|
<div class="md-container" data-md-component="container">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<main class="md-main" data-md-component="main">
|
||
|
|
<div class="md-main__inner md-grid">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
|
||
|
|
<div class="md-sidebar__scrollwrap">
|
||
|
|
<div class="md-sidebar__inner">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
|
||
|
|
<label class="md-nav__title" for="__drawer">
|
||
|
|
<a href="../.." title="obikmer" class="md-nav__button md-logo" aria-label="obikmer" data-md-component="logo">
|
||
|
|
|
||
|
|
|
||
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
|
||
|
|
|
||
|
|
</a>
|
||
|
|
obikmer
|
||
|
|
</label>
|
||
|
|
|
||
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="../.." class="md-nav__link">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
|
||
|
|
Home
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</span>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</a>
|
||
|
|
</li>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="../../installation/" class="md-nav__link">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
|
||
|
|
Installation
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</span>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</a>
|
||
|
|
</li>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<li class="md-nav__item md-nav__item--nested">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3" >
|
||
|
|
|
||
|
|
|
||
|
|
<label class="md-nav__link" for="__nav_3" id="__nav_3_label" tabindex="0">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
|
||
|
|
Theory
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</span>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-nav__icon md-icon"></span>
|
||
|
|
</label>
|
||
|
|
|
||
|
|
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_3_label" aria-expanded="false">
|
||
|
|
<label class="md-nav__title" for="__nav_3">
|
||
|
|
<span class="md-nav__icon md-icon"></span>
|
||
|
|
|
||
|
|
|
||
|
|
Theory
|
||
|
|
|
||
|
|
|
||
|
|
</label>
|
||
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="../../kmers/" class="md-nav__link">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
|
||
|
|
Kmers and super-kmers
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</span>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</a>
|
||
|
|
</li>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="../../theory/encoding/" class="md-nav__link">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
|
||
|
|
DNA encoding
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</span>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</a>
|
||
|
|
</li>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="../../theory/entropy/" class="md-nav__link">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
|
||
|
|
Entropy filter
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</span>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</a>
|
||
|
|
</li>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="../../theory/minimizer/" class="md-nav__link">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
|
||
|
|
Minimizer selection
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</span>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</a>
|
||
|
|
</li>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="../../theory/indexing/" class="md-nav__link">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
|
||
|
|
Partitioning architecture
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</span>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</a>
|
||
|
|
</li>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="../../theory/evolutionary_distances/" class="md-nav__link">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
|
||
|
|
Central-position SNP distance (discussion)
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</span>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</a>
|
||
|
|
</li>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</ul>
|
||
|
|
</nav>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<li class="md-nav__item md-nav__item--active md-nav__item--nested">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4" checked>
|
||
|
|
|
||
|
|
|
||
|
|
<label class="md-nav__link" for="__nav_4" id="__nav_4_label" tabindex="0">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
|
||
|
|
Implementation
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</span>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-nav__icon md-icon"></span>
|
||
|
|
</label>
|
||
|
|
|
||
|
|
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_4_label" aria-expanded="true">
|
||
|
|
<label class="md-nav__title" for="__nav_4">
|
||
|
|
<span class="md-nav__icon md-icon"></span>
|
||
|
|
|
||
|
|
|
||
|
|
Implementation
|
||
|
|
|
||
|
|
|
||
|
|
</label>
|
||
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="../superkmer/" class="md-nav__link">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
|
||
|
|
SuperKmer
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</span>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</a>
|
||
|
|
</li>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="../kmer/" class="md-nav__link">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
|
||
|
|
Kmer
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</span>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</a>
|
||
|
|
</li>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="../chunkreader/" class="md-nav__link">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
|
||
|
|
Chunk reader
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</span>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</a>
|
||
|
|
</li>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="../pipeline/" class="md-nav__link">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
|
||
|
|
Construction pipeline
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</span>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</a>
|
||
|
|
</li>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="../obipipeline/" class="md-nav__link">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
|
||
|
|
obipipeline library
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</span>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</a>
|
||
|
|
</li>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="../storage/" class="md-nav__link">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
|
||
|
|
On-disk storage
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</span>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</a>
|
||
|
|
</li>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="../mphf/" class="md-nav__link">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
|
||
|
|
MPHF selection
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</span>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</a>
|
||
|
|
</li>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="../unitig_evidence/" class="md-nav__link">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
|
||
|
|
Unitig evidence encoding
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</span>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</a>
|
||
|
|
</li>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="../evidence_elimination/" class="md-nav__link">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
|
||
|
|
Evidence elimination (discussion)
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</span>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</a>
|
||
|
|
</li>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="../layer_tier/" class="md-nav__link">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
|
||
|
|
obikindex layer tier
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</span>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</a>
|
||
|
|
</li>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="../persistent_compact_int_vec/" class="md-nav__link">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
|
||
|
|
PersistentCompactIntVec
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</span>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</a>
|
||
|
|
</li>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="../persistent_bit_vec/" class="md-nav__link">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
|
||
|
|
PersistentBitVec
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</span>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</a>
|
||
|
|
</li>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="../merge/" class="md-nav__link">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
|
||
|
|
Merge command
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</span>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</a>
|
||
|
|
</li>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="../merge_parallelism/" class="md-nav__link">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
|
||
|
|
Merge parallelism & memory
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</span>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</a>
|
||
|
|
</li>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="../filtering/" class="md-nav__link">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
|
||
|
|
Kmer filtering
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</span>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</a>
|
||
|
|
</li>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="../select/" class="md-nav__link">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
|
||
|
|
Select command
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</span>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</a>
|
||
|
|
</li>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="../obitaxonomy/" class="md-nav__link">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
|
||
|
|
obitaxonomy crate
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</span>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</a>
|
||
|
|
</li>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="../benchmark_query_testing/" class="md-nav__link">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
|
||
|
|
Benchmark: query-path testing
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</span>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</a>
|
||
|
|
</li>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<li class="md-nav__item md-nav__item--active">
|
||
|
|
|
||
|
|
<input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<label class="md-nav__link md-nav__link--active" for="__toc">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
|
||
|
|
Partition and layer caching (discussion)
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</span>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-nav__icon md-icon"></span>
|
||
|
|
</label>
|
||
|
|
|
||
|
|
<a href="./" class="md-nav__link md-nav__link--active">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
|
||
|
|
Partition and layer caching (discussion)
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</span>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</a>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<label class="md-nav__title" for="__toc">
|
||
|
|
<span class="md-nav__icon md-icon"></span>
|
||
|
|
Table of contents
|
||
|
|
</label>
|
||
|
|
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#definitions-obikpartition-and-kmerpartition-not-yet-created" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
Definitions: obikpartition and KmerPartition (not yet created)
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#type-to-concept-mapping-index-partition-layer" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
Type-to-concept mapping: Index / Partition / Layer
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#major-restructuring-2026-08-20-kmerpartitions-merged-into-kmerindex" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
Major restructuring (2026-08-20): KmerPartitions merged into KmerIndex
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#1-done-2026-08-20-layer-is-now-the-heterogeneous-handle-mat-is-gone" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
(1) done (2026-08-20): Layer is now the heterogeneous handle, Mat is gone
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#1b-done-2026-08-20-layerempty-the-first-non-ready-to-read-state" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
(1b) done (2026-08-20): Layer::Empty — the first non-ready-to-read state
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#2a-done-2026-08-20-obikpartition-crate-kmerpartition" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
(2a) done (2026-08-20): obikpartition crate + KmerPartition
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#3-done-2026-08-20-obikindex-obikpartitionner-dependency-inverted-partitionrouter-now-fills-layerempty-shells" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
(3) done (2026-08-20): obikindex ↔ obikpartitionner dependency inverted, PartitionRouter now fills Layer::Empty shells
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#4-done-2026-08-20-obikderep-dereplication-split-out-of-obikpartitionner-one-algorithm-at-a-time" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
(4) done (2026-08-20): obikderep — dereplication split out of obikpartitionner, one algorithm at a time
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#5-design-agreed-not-yet-implemented-2026-08-20-kmerpartition-rewritten-layer-gains-self-naming-a-future-cache-crate-over-kmerindex" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
(5) design agreed, not yet implemented (2026-08-20): KmerPartition rewritten, Layer gains self-naming, a future cache crate over KmerIndex
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
<nav class="md-nav" aria-label="(5) design agreed, not yet implemented (2026-08-20): KmerPartition rewritten, Layer gains self-naming, a future cache crate over KmerIndex">
|
||
|
|
<ul class="md-nav__list">
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#how-this-was-found" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
How this was found
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#the-agreed-shape" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
The agreed shape
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#known-blast-radius-why-this-wasnt-done-in-the-same-session" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
Known blast radius (why this wasn't done in the same session)
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#also-agreed-separately-partitionrouternew-never-needed-mut-kmerindex" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
Also agreed, separately: PartitionRouter::new never needed &mut KmerIndex
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#also-discussed-a-future-cache-crate-not-part-of-5-not-obikalgorithm-either" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
Also discussed: a future cache crate, not part of (5), not obikalgorithm either
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#order-of-remaining-work-as-currently-understood" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
Order of remaining work, as currently understood
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
</ul>
|
||
|
|
</nav>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#the-problem" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
The problem
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#three-independent-implementations-of-the-same-bundle-historical-1-fixed-this" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
Three independent implementations of the same bundle (historical — (1) fixed this)
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#the-gap-in-obilayeredmaps-existing-cache-historical-1-fixed-this" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
The gap in obilayeredmap's existing cache (historical — (1) fixed this)
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#resource-cost-mmap-does-not-hold-a-file-descriptor" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
Resource cost: mmap does not hold a file descriptor
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#layering-who-owns-what-superseded-see-definitions-above" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
Layering: who owns what (superseded — see Definitions above)
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#direction-agreed-not-yet-implemented" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
Direction agreed, not yet implemented
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#preparatory-work-done-2026-08-20" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
Preparatory work done (2026-08-20)
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#one-bug-found-while-reading-around-this-signalled-not-fixed-one-earlier-claim-retracted" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
One bug found while reading around this (signalled, not fixed); one earlier claim retracted
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#remaining-instance-of-the-partitionmeta-encapsulation-problem" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
Remaining instance of the PartitionMeta-encapsulation problem
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
</ul>
|
||
|
|
|
||
|
|
</nav>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</ul>
|
||
|
|
</nav>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<li class="md-nav__item md-nav__item--nested">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_5" >
|
||
|
|
|
||
|
|
|
||
|
|
<label class="md-nav__link" for="__nav_5" id="__nav_5_label" tabindex="0">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
|
||
|
|
Architecture
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</span>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-nav__icon md-icon"></span>
|
||
|
|
</label>
|
||
|
|
|
||
|
|
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_5_label" aria-expanded="false">
|
||
|
|
<label class="md-nav__title" for="__nav_5">
|
||
|
|
<span class="md-nav__icon md-icon"></span>
|
||
|
|
|
||
|
|
|
||
|
|
Architecture
|
||
|
|
|
||
|
|
|
||
|
|
</label>
|
||
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="../../architecture/sequences/invariant/" class="md-nav__link">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
|
||
|
|
Sequences
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</span>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</a>
|
||
|
|
</li>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="../../architecture/index_architecture/" class="md-nav__link">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
|
||
|
|
Kmer index
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</span>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</a>
|
||
|
|
</li>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="../../architecture/siblings/" class="md-nav__link">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
|
||
|
|
Sibling annex (discussion)
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</span>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</a>
|
||
|
|
</li>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="../../architecture/numa_worker_pools/" class="md-nav__link">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
|
||
|
|
NUMA-aware worker pools
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</span>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</a>
|
||
|
|
</li>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="../../architecture/numa_partition_runner/" class="md-nav__link">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
|
||
|
|
NUMA-aware partition runner
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</span>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</a>
|
||
|
|
</li>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</ul>
|
||
|
|
</nav>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</ul>
|
||
|
|
</nav>
|
||
|
|
</div>
|
||
|
|
</div>
|
||
|
|
</div>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
|
||
|
|
<div class="md-sidebar__scrollwrap">
|
||
|
|
<div class="md-sidebar__inner">
|
||
|
|
|
||
|
|
|
||
|
|
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<label class="md-nav__title" for="__toc">
|
||
|
|
<span class="md-nav__icon md-icon"></span>
|
||
|
|
Table of contents
|
||
|
|
</label>
|
||
|
|
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#definitions-obikpartition-and-kmerpartition-not-yet-created" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
Definitions: obikpartition and KmerPartition (not yet created)
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#type-to-concept-mapping-index-partition-layer" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
Type-to-concept mapping: Index / Partition / Layer
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#major-restructuring-2026-08-20-kmerpartitions-merged-into-kmerindex" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
Major restructuring (2026-08-20): KmerPartitions merged into KmerIndex
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#1-done-2026-08-20-layer-is-now-the-heterogeneous-handle-mat-is-gone" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
(1) done (2026-08-20): Layer is now the heterogeneous handle, Mat is gone
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#1b-done-2026-08-20-layerempty-the-first-non-ready-to-read-state" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
(1b) done (2026-08-20): Layer::Empty — the first non-ready-to-read state
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#2a-done-2026-08-20-obikpartition-crate-kmerpartition" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
(2a) done (2026-08-20): obikpartition crate + KmerPartition
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#3-done-2026-08-20-obikindex-obikpartitionner-dependency-inverted-partitionrouter-now-fills-layerempty-shells" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
(3) done (2026-08-20): obikindex ↔ obikpartitionner dependency inverted, PartitionRouter now fills Layer::Empty shells
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#4-done-2026-08-20-obikderep-dereplication-split-out-of-obikpartitionner-one-algorithm-at-a-time" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
(4) done (2026-08-20): obikderep — dereplication split out of obikpartitionner, one algorithm at a time
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#5-design-agreed-not-yet-implemented-2026-08-20-kmerpartition-rewritten-layer-gains-self-naming-a-future-cache-crate-over-kmerindex" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
(5) design agreed, not yet implemented (2026-08-20): KmerPartition rewritten, Layer gains self-naming, a future cache crate over KmerIndex
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
<nav class="md-nav" aria-label="(5) design agreed, not yet implemented (2026-08-20): KmerPartition rewritten, Layer gains self-naming, a future cache crate over KmerIndex">
|
||
|
|
<ul class="md-nav__list">
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#how-this-was-found" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
How this was found
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#the-agreed-shape" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
The agreed shape
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#known-blast-radius-why-this-wasnt-done-in-the-same-session" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
Known blast radius (why this wasn't done in the same session)
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#also-agreed-separately-partitionrouternew-never-needed-mut-kmerindex" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
Also agreed, separately: PartitionRouter::new never needed &mut KmerIndex
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#also-discussed-a-future-cache-crate-not-part-of-5-not-obikalgorithm-either" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
Also discussed: a future cache crate, not part of (5), not obikalgorithm either
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#order-of-remaining-work-as-currently-understood" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
Order of remaining work, as currently understood
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
</ul>
|
||
|
|
</nav>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#the-problem" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
The problem
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#three-independent-implementations-of-the-same-bundle-historical-1-fixed-this" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
Three independent implementations of the same bundle (historical — (1) fixed this)
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#the-gap-in-obilayeredmaps-existing-cache-historical-1-fixed-this" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
The gap in obilayeredmap's existing cache (historical — (1) fixed this)
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#resource-cost-mmap-does-not-hold-a-file-descriptor" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
Resource cost: mmap does not hold a file descriptor
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#layering-who-owns-what-superseded-see-definitions-above" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
Layering: who owns what (superseded — see Definitions above)
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#direction-agreed-not-yet-implemented" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
Direction agreed, not yet implemented
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#preparatory-work-done-2026-08-20" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
Preparatory work done (2026-08-20)
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#one-bug-found-while-reading-around-this-signalled-not-fixed-one-earlier-claim-retracted" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
One bug found while reading around this (signalled, not fixed); one earlier claim retracted
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
<li class="md-nav__item">
|
||
|
|
<a href="#remaining-instance-of-the-partitionmeta-encapsulation-problem" class="md-nav__link">
|
||
|
|
<span class="md-ellipsis">
|
||
|
|
|
||
|
|
Remaining instance of the PartitionMeta-encapsulation problem
|
||
|
|
|
||
|
|
</span>
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</li>
|
||
|
|
|
||
|
|
</ul>
|
||
|
|
|
||
|
|
</nav>
|
||
|
|
</div>
|
||
|
|
</div>
|
||
|
|
</div>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<div class="md-content" data-md-component="content">
|
||
|
|
|
||
|
|
<article class="md-content__inner md-typeset">
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<h1 id="partition-and-layer-caching-discussion">Partition and layer caching (discussion)</h1>
|
||
|
|
<p><strong>Superseded (2026-08-21):</strong> <code>obikpartition</code> and <code>obilayeredmap</code> are no
|
||
|
|
longer separate workspace crates — both were folded back into <code>obikindex</code>
|
||
|
|
as submodules (<code>obikindex::partition</code>, <code>obikindex::layer</code>), alongside the
|
||
|
|
crate's original content as <code>obikindex::index</code>, purely to reduce the
|
||
|
|
crate count (no behavior change). Every mention of <code>obikpartition</code>/
|
||
|
|
<code>obilayeredmap</code> as a <em>crate</em> below, and every dependency-direction
|
||
|
|
argument phrased in terms of "which crate depends on which" (e.g. "this
|
||
|
|
crate depends only on <code>obilayeredmap</code> and below, never on <code>obikindex</code>"),
|
||
|
|
describes that now-superseded split-crate architecture and is kept as-is
|
||
|
|
for historical context — read <code>obikpartition::X</code> as <code>obikindex::
|
||
|
|
partition::X</code> and <code>obilayeredmap::X</code> as <code>obikindex::layer::X</code> throughout.
|
||
|
|
The underlying module boundary and its rationale (Layer tier / Partition
|
||
|
|
tier / Index tier, each depending only downward) are unchanged; only the
|
||
|
|
crate-vs-module packaging changed. See <a href="../layer_tier/">obikindex::layer</a>
|
||
|
|
for the current module doc.</p>
|
||
|
|
<p>Status (2026-08-20, latest pass): (1) done — <code>obilayeredmap::Layer</code>
|
||
|
|
exists, <code>Mat</code> is gone. (1b) done — <code>Layer::Empty</code>, the first non-ready
|
||
|
|
state, added (panics on every read method). (2a) done — the
|
||
|
|
<code>obikpartition</code> crate and <code>KmerPartition</code> itself exist (<code>open</code>/<code>n_layers</code>/
|
||
|
|
<code>layer</code>/<code>layers</code>/<code>find</code>). (2b) — migrating <code>PartitionCache</code>/<code>QueryLayer</code>
|
||
|
|
onto it — <strong>not started</strong>, deliberately deferred. (3) done — the
|
||
|
|
<code>obikindex ↔ obikpartitionner</code> dependency inverted: <code>PartitionRouter</code> now
|
||
|
|
takes <code>&mut KmerIndex</code> and produces <code>Layer::Empty</code> shells directly, closing
|
||
|
|
the gap <code>Layer::Empty</code> was built for in (1b) — see "(3) done" below. (4)
|
||
|
|
done — dereplication split out into its own crate, <code>obikderep</code>, first step
|
||
|
|
of an incremental "one algorithm at a time" split of <code>obikpartitionner</code>'s
|
||
|
|
remaining bundle (<code>count_kmer</code>/<code>build_layers</code> not yet moved) — see "(4)
|
||
|
|
done" below. <strong>(5) — full design agreed, not yet implemented</strong> —
|
||
|
|
<code>KmerPartition</code> was found to be wired into nothing (<code>KmerIndex</code> never
|
||
|
|
calls it; every path is still computed via free functions), and the fix
|
||
|
|
turned out to be bigger than <code>KmerPartition</code> alone: <code>Layer</code>'s own
|
||
|
|
constructors don't self-name either. Full redesign of both, agreed in
|
||
|
|
detail, session ended (budget) before implementation — see "(5) design
|
||
|
|
agreed" below; <strong>read it before touching <code>KmerPartition</code>/<code>Layer</code>
|
||
|
|
signatures</strong>, the shape is fully specified. Earlier mix-up, for
|
||
|
|
context: an earlier
|
||
|
|
version of this doc used the name <code>KmerPartition</code> (singular) for what was
|
||
|
|
actually the <em>collection</em> type (later renamed <code>KmerPartitions</code>, later
|
||
|
|
merged into <code>KmerIndex</code> — see "Major restructuring" below), and never
|
||
|
|
retracted that usage before this section was rewritten. An agent working
|
||
|
|
from that stale wording built the wrong thing. <strong>If you are about to
|
||
|
|
implement (2), read "Definitions: <code>obikpartition</code> and <code>KmerPartition</code>"
|
||
|
|
below — it is the current, authoritative naming — before touching any
|
||
|
|
other section of this file, some of which still describe superseded
|
||
|
|
states of the code and are kept only as dated history.</strong></p>
|
||
|
|
<h2 id="definitions-obikpartition-and-kmerpartition-not-yet-created">Definitions: <code>obikpartition</code> and <code>KmerPartition</code> (not yet created)</h2>
|
||
|
|
<p><strong><code>obikpartition</code></strong> — a new workspace crate, not created yet. Holds the
|
||
|
|
<strong>Partition</strong> tier of the <code>Index { Partition { Layer } }</code> model, the same
|
||
|
|
way <code>obilayeredmap</code> already holds the <strong>Layer</strong> tier as its own crate
|
||
|
|
rather than living inside <code>obikindex</code>. Depends only on <code>obilayeredmap</code>
|
||
|
|
(for <code>Layer</code>) and lower (<code>obikseq</code>, <code>obiskio</code>). Does <strong>not</strong> depend on
|
||
|
|
<code>obikindex</code>, <code>obikpartitionner</code>, or <code>obikphylo</code>. Dependency direction:
|
||
|
|
<code>obikindex → obikpartition → obilayeredmap</code>; <code>obikphylo → obikindex</code>
|
||
|
|
(and/or <code>obikpartition</code> directly if it ends up needing it without going
|
||
|
|
through <code>KmerIndex</code>).</p>
|
||
|
|
<p><strong><code>KmerPartition</code></strong> (singular) — the one type this crate exists for.
|
||
|
|
Represents <strong>one partition's already-open layers</strong> — a read cache, built
|
||
|
|
once per partition and held for the run, not rebuilt per lookup. Shape:</p>
|
||
|
|
<div class="highlight"><pre><span></span><code><span class="k">pub</span><span class="w"> </span><span class="k">struct</span><span class="w"> </span><span class="nc">KmerPartition</span><span class="w"> </span><span class="p">{</span>
|
||
|
|
<span class="w"> </span><span class="n">layers</span><span class="p">:</span><span class="w"> </span><span class="nb">Vec</span><span class="o"><</span><span class="n">obilayeredmap</span><span class="p">::</span><span class="n">Layer</span><span class="o">></span><span class="p">,</span>
|
||
|
|
<span class="p">}</span>
|
||
|
|
</code></pre></div>
|
||
|
|
<p>Nothing else. In particular:
|
||
|
|
- <strong>No path computation.</strong> <code>KmerPartition::open</code> takes an already-resolved
|
||
|
|
<code>index_dir: &Path</code> (plus <code>mode: &IndexMode</code>, <code>n_layers: usize</code>,
|
||
|
|
<code>with_counts: bool</code> — whatever it needs, as plain arguments), the same
|
||
|
|
discipline <code>obikpartitionner::PartitionRouter::open</code> already follows.
|
||
|
|
Computing <code>index_dir</code>/<code>layer_dir</code> from a partition number is
|
||
|
|
<code>KmerIndex</code>'s job (<code>obikindex</code>, which owns that already — see "Major
|
||
|
|
restructuring" below); <code>KmerPartition</code> never reaches back into
|
||
|
|
<code>KmerIndex</code> to get it (would require <code>obikpartition → obikindex</code>, the
|
||
|
|
wrong direction).
|
||
|
|
- <strong>No routing/write state.</strong> Writing raw superkmers, <code>dereplicate</code>,
|
||
|
|
<code>count_kmer</code> stay in <code>obikpartitionner::PartitionRouter</code> — a completely
|
||
|
|
different crate, a completely different phase (pre-layer, whereas
|
||
|
|
<code>KmerPartition</code> only makes sense once layers exist).
|
||
|
|
- <strong>No multi-partition collection baked in.</strong> <code>KmerPartition</code> is <em>one</em>
|
||
|
|
partition. Whatever ends up caching several of them (replacing
|
||
|
|
<code>obikphylo::siblings::cache::PartitionCache</code>'s <code>Vec<Vec<Layer>></code> and
|
||
|
|
<code>obikindex::query_layer</code>'s per-call reopen) holds <code>Vec<KmerPartition></code> —
|
||
|
|
that collection can live in <code>obikpartition</code> too, or in <code>obikindex</code>
|
||
|
|
alongside <code>KmerIndex</code>; not yet decided, secondary to getting
|
||
|
|
<code>KmerPartition</code> itself right first.</p>
|
||
|
|
<p><strong>Do not confuse with <code>KmerPartitions</code></strong> (plural — note the <code>s</code>): that
|
||
|
|
type is <strong>gone</strong>. It used to be <code>obikpartitionner</code>'s (nee <code>obikpartition</code>,
|
||
|
|
briefly — see the crate-rename history below, itself a separate rename
|
||
|
|
from this one) do-everything struct — routing, dereplication, <em>and</em> path
|
||
|
|
lookups all in one. It was deleted on 2026-08-20; its read-side (paths,
|
||
|
|
<code>n_layers</code>, <code>partition_meta</code>) was absorbed into <code>KmerIndex</code>, its
|
||
|
|
write-side became <code>PartitionRouter</code>. <code>KmerPartition</code> (this section,
|
||
|
|
singular, no final <code>s</code>) is a brand-new type with a different job, in a
|
||
|
|
crate that doesn't exist yet — not a revival, not a renaming, of
|
||
|
|
<code>KmerPartitions</code>.</p>
|
||
|
|
<h2 id="type-to-concept-mapping-index-partition-layer">Type-to-concept mapping: Index / Partition / Layer</h2>
|
||
|
|
<p>The conceptual nesting <code>Index { Partition { Layer { MPHF, Evidence, Matrix
|
||
|
|
} } } }</code>, current state:</p>
|
||
|
|
<ul>
|
||
|
|
<li><strong>Index</strong> = <code>obikindex::KmerIndex</code> — <code>{ root_path, meta: IndexMeta }</code>.
|
||
|
|
Also directly exposes the partition-path/metadata accessors
|
||
|
|
(<code>partition_dir(i)</code>, <code>index_dir(i)</code>, <code>layer_dir(i, l)</code>,
|
||
|
|
<code>partition_meta(i)</code>, <code>n_layers(i)</code>, <code>partition_mode(i)</code>,
|
||
|
|
<code>n_partitions()</code>) since <code>KmerPartitions</code> merged into it (see "Major
|
||
|
|
restructuring" below) — <code>KmerIndex</code> today <em>is</em> "index + collection of
|
||
|
|
partitions' paths & metadata," just without a <code>Vec</code> of open layers.</li>
|
||
|
|
<li><strong>Partition, the collection</strong> = no dedicated type today; the closest
|
||
|
|
thing is <code>KmerIndex</code> itself (previous bullet). Once <code>KmerPartition</code>
|
||
|
|
(singular, see Definitions above) exists, a <code>Vec<KmerPartition></code>
|
||
|
|
somewhere would be this — still open, see "Direction agreed" below.</li>
|
||
|
|
<li><strong>Partition, one of them</strong> = <code>obikpartition::KmerPartition</code> — <strong>to be
|
||
|
|
built</strong>, see Definitions above. Nothing plays this role today;
|
||
|
|
<code>obikphylo::siblings::cache::PartitionCache</code> and
|
||
|
|
<code>obikindex::query_layer::QueryLayer</code> each independently reinvent a
|
||
|
|
fragment of it.</li>
|
||
|
|
<li><strong>Layer</strong> = <code>obilayeredmap::Layer</code> (format-erased: <code>Count</code>/<code>Presence</code>,
|
||
|
|
each wrapping a <code>TypedLayer<D></code>) — see "(1) done" below for how this
|
||
|
|
came to be; <code>TypedLayer<D></code> (<code>{ mphf: MphfLayer, data: D }</code>, monomorphic)
|
||
|
|
is the lower-level, <code>D</code>-fixed building block <code>Layer</code> is built on, not
|
||
|
|
what other crates should reach for directly.</li>
|
||
|
|
<li><strong>MPHF</strong> = <code>MphfLayer.mphf: MemCase<MphfEps></code> — kmer → slot.</li>
|
||
|
|
<li><strong>Evidence</strong> = <code>MphfLayer.ev: LayerEvidence</code> (<code>Exact</code>/<code>Approx</code>/
|
||
|
|
<code>Hybrid</code> — <code>evidence.bin</code>/<code>fingerprint.bin</code>; see <code>EvidenceKind</code>).</li>
|
||
|
|
<li><strong>Matrix</strong> = <code>TypedLayer<D>.data: D</code> — <code>PersistentBitMatrix</code> /
|
||
|
|
<code>PersistentCompactIntMatrix</code>.</li>
|
||
|
|
</ul>
|
||
|
|
<p>Target nesting once <code>KmerPartition</code> exists:</p>
|
||
|
|
<div class="highlight"><pre><span></span><code>KmerIndex (obikindex)
|
||
|
|
└─ (opened on demand, per i) KmerPartition (obikpartition — not yet built)
|
||
|
|
└─ layers: Vec<Layer> (obilayeredmap)
|
||
|
|
└─ Layer::Count/Presence(TypedLayer<D>)
|
||
|
|
└─ TypedLayer<D> { mphf: MphfLayer, data: D }
|
||
|
|
├─ mphf.mphf → MPHF
|
||
|
|
├─ mphf.ev → Evidence
|
||
|
|
└─ data → Matrix
|
||
|
|
</code></pre></div>
|
||
|
|
<h2 id="major-restructuring-2026-08-20-kmerpartitions-merged-into-kmerindex">Major restructuring (2026-08-20): <code>KmerPartitions</code> merged into <code>KmerIndex</code></h2>
|
||
|
|
<p>Prompted by a direct question: why keep <code>KmerIndex</code>/<code>KmerPartitions</code> split
|
||
|
|
when, one level down, <code>KmerPartitions</code> is going to directly hold
|
||
|
|
<code>Vec<KmerPartition></code> rather than being split again into
|
||
|
|
"collection-holder" + "collection"? Investigating the actual justification
|
||
|
|
("<code>KmerPartitions</code> has an independent lifecycle, used before an index
|
||
|
|
exists") turned out to be <strong>false</strong> — <code>KmerPartitions::create</code> was called
|
||
|
|
in exactly one place, inside <code>KmerIndex::create</code>, and every
|
||
|
|
<code>open_with_config</code> reopen outside <code>KmerIndex</code>'s own constructors was a
|
||
|
|
redundant re-derivation of a <code>KmerPartitions</code> already reachable via
|
||
|
|
<code>index.partition()</code> (the exact kind of duplication this whole doc has been
|
||
|
|
tracking). Once that was gone, so was the reason to keep them separate.</p>
|
||
|
|
<p>Second correction, from the same conversation: <code>obikpartitionner</code> had
|
||
|
|
accumulated query/merge/select/rebuild/dump/distance logic that has
|
||
|
|
nothing to do with partitioning super-kmers — it operates on <em>layers</em>,
|
||
|
|
which don't exist yet at the phase <code>obikpartitionner</code> is actually
|
||
|
|
responsible for (scatter → dereplicate → count, all pre-layer). That
|
||
|
|
logic moved to <code>obikindex</code>, which already depends on <code>obilayeredmap</code> and
|
||
|
|
never needed <code>obikpartitionner</code> for it. No crate-dependency inversion was
|
||
|
|
needed — <code>obikindex → obikpartitionner</code> stays the same direction as before.</p>
|
||
|
|
<p><strong>Result:</strong>
|
||
|
|
- <code>obikpartitionner</code> (renamed back from <code>obikpartition</code>) now contains only
|
||
|
|
<code>PartitionRouter</code> (superkmer routing: <code>write</code>/<code>write_batch</code>/<code>flush</code>/
|
||
|
|
<code>close</code>, <code>dereplicate</code>, <code>count_kmer</code>, <code>KmerSpectrum</code>) and the
|
||
|
|
<code>partition_dir(root, i)</code> naming primitive both <code>PartitionRouter</code> and
|
||
|
|
<code>KmerIndex</code> build on. <code>KmerPartitions</code> no longer exists as a type.
|
||
|
|
- <code>KmerIndex</code> (<code>obikindex</code>) absorbed <code>KmerPartitions</code>'s read-side entirely:
|
||
|
|
<code>partition_dir</code>/<code>index_dir</code>/<code>layer_dir</code>/<code>partition_meta</code>/<code>n_layers</code>/
|
||
|
|
<code>partition_mode</code>/<code>n_partitions</code> (the last now derived from
|
||
|
|
<code>2^config.n_bits</code>, no longer a stored, independently-set duplicate field
|
||
|
|
— <code>kmer_size</code>/<code>minimizer_size</code> used to be double-stored, in both
|
||
|
|
<code>KmerPartitions</code> and <code>IndexMeta.config</code>, a latent-drift risk flagged
|
||
|
|
earlier in this doc; now single-sourced from <code>IndexMeta.config</code>). Seven
|
||
|
|
whole files moved from <code>obikpartitionner</code> into <code>obikindex</code> verbatim as
|
||
|
|
<code>impl KmerIndex</code> blocks, kept as separate files (not merged into
|
||
|
|
existing same-topic files): <code>index_layer.rs</code>, <code>query_layer.rs</code>,
|
||
|
|
<code>merge_layer/</code>, <code>select_layer.rs</code>, <code>rebuild_layer.rs</code>, <code>dump_layer.rs</code>,
|
||
|
|
plus <code>distance.rs</code>'s <code>count_store</code>/<code>presence_store</code> (renamed
|
||
|
|
<code>matrix_store.rs</code> to avoid colliding with <code>obikindex</code>'s own pre-existing
|
||
|
|
<code>distance.rs</code>), and their shared support (<code>common.rs</code>'s <code>load_meta</code>/
|
||
|
|
<code>olm_to_sk</code>, <code>filter.rs</code>, <code>graph_pipeline.rs</code>).
|
||
|
|
- <code>obikphylo::siblings::cache::PartitionCache::build</code> now takes <code>&KmerIndex</code>
|
||
|
|
directly instead of a separately-opened <code>&KmerPartitions</code> — this deleted
|
||
|
|
the redundant-reopen pattern at all 8 call sites
|
||
|
|
(<code>alignment</code>/<code>build</code>/<code>cardinality</code>/<code>distance</code>/<code>entropy</code>×2/
|
||
|
|
<code>sankoff_bundle</code>/<code>stats</code>), the same bug flagged earlier in this
|
||
|
|
conversation as a side effect of investigating the false "independent
|
||
|
|
lifecycle" claim.
|
||
|
|
- <code>KmerIndex::partition()</code>/<code>partition_mut()</code> are gone; <code>scatter()</code>
|
||
|
|
(<code>obikmer</code>) and any write-side code get a transient <code>PartitionRouter</code> via
|
||
|
|
<code>KmerIndex::partition_router()</code>.
|
||
|
|
- A real bug caught by the test suite during this move:
|
||
|
|
<code>PartitionRouter::open</code> initially defaulted to <code>closed: true</code> (inherited
|
||
|
|
from <code>KmerPartitions::open_with_config</code>'s old read-only-reopen
|
||
|
|
semantics), which broke every write through a router obtained via
|
||
|
|
<code>partition_router()</code>. Fixed — <code>PartitionRouter</code> is exclusively a
|
||
|
|
write/processing tool now, so <code>open</code> always starts open.</p>
|
||
|
|
<p>Full workspace test suite green (0 failed) after, including all 27
|
||
|
|
<code>obikphylo::siblings</code> tests.</p>
|
||
|
|
<h2 id="1-done-2026-08-20-layer-is-now-the-heterogeneous-handle-mat-is-gone">(1) done (2026-08-20): <code>Layer</code> is now the heterogeneous handle, <code>Mat</code> is gone</h2>
|
||
|
|
<p>Resolved the naming question left open above. <code>Layer<D></code> (the old
|
||
|
|
generic/monomorphic type) renamed to <code>TypedLayer<D></code> throughout
|
||
|
|
(<code>obilayeredmap</code>, <code>obikindex</code>, <code>obikphylo</code> — 12 files, mechanical) to free
|
||
|
|
<code>Layer</code> for the type that's actually meant to be everyone's default
|
||
|
|
handle. <code>obilayeredmap::content_layer::Layer</code> (re-exported at the crate
|
||
|
|
root) is that type — <code>Count(TypedLayer<PersistentCompactIntMatrix>)</code>/
|
||
|
|
<code>Presence(TypedLayer<PersistentBitMatrix>)</code>, <code>Layer::open</code> doing the same
|
||
|
|
disk probe <code>Mat::open</code> used to, <code>find_slot</code>/<code>index_batch</code>/<code>n_cols</code>/
|
||
|
|
<code>fill_sub_matrix_carries</code> dispatching per variant exactly as <code>Mat</code> did.</p>
|
||
|
|
<p><code>obikphylo::siblings::cache::Mat</code> deleted outright — <code>PartitionCache</code> now
|
||
|
|
holds <code>Vec<Vec<obilayeredmap::Layer>></code> directly. The one sibling-specific
|
||
|
|
method <code>Mat</code> carried (<code>iter_minorants_batch</code>) is not on <code>obilayeredmap::
|
||
|
|
Layer</code> (phylo concepts don't belong in <code>obilayeredmap</code>) — it's an
|
||
|
|
<code>impl SiblingLayerExt for obilayeredmap::Layer</code> in <code>iter.rs</code>, dispatching
|
||
|
|
to each variant's existing <code>impl<D: LayerData> SiblingLayerExt for
|
||
|
|
TypedLayer<D></code>.</p>
|
||
|
|
<p>Full workspace suite green (0 failed) after, including all 27
|
||
|
|
<code>obikphylo::siblings</code> tests.</p>
|
||
|
|
<p>Still not built: (2) — <code>KmerPartition</code> (singular, one partition's open
|
||
|
|
<code>Vec<Layer></code>) and a multi-partition cache in <code>obikpartitionner</code> to replace
|
||
|
|
<code>obikphylo::siblings::cache::PartitionCache</code> and <code>obikindex::query_layer</code>'s
|
||
|
|
still-separate <code>QueryLayer</code> (which still independently bundles MPHF+matrix,
|
||
|
|
2-way not using <code>Layer</code> at all). Both remaining consumers now sit one
|
||
|
|
<code>Layer::open</code> call away from unifying onto (2) once it exists.</p>
|
||
|
|
<h2 id="1b-done-2026-08-20-layerempty-the-first-non-ready-to-read-state">(1b) done (2026-08-20): <code>Layer::Empty</code> — the first non-ready-to-read state</h2>
|
||
|
|
<p>First step toward <code>Layer</code> representing a layer's whole life, not just the
|
||
|
|
open-for-reading end of it (see "Definitions" above: <code>KmerPartition</code> will
|
||
|
|
hold <code>Vec<Layer></code> regardless of each layer's state, states in between
|
||
|
|
included). Added one variant:</p>
|
||
|
|
<div class="highlight"><pre><span></span><code><span class="k">pub</span><span class="w"> </span><span class="k">enum</span><span class="w"> </span><span class="nc">Layer</span><span class="w"> </span><span class="p">{</span>
|
||
|
|
<span class="w"> </span><span class="n">Empty</span><span class="w"> </span><span class="p">{</span><span class="w"> </span><span class="n">dir</span><span class="p">:</span><span class="w"> </span><span class="nc">PathBuf</span><span class="w"> </span><span class="p">},</span>
|
||
|
|
<span class="w"> </span><span class="n">Count</span><span class="p">(</span><span class="n">TypedLayer</span><span class="o"><</span><span class="n">PersistentCompactIntMatrix</span><span class="o">></span><span class="p">),</span>
|
||
|
|
<span class="w"> </span><span class="n">Presence</span><span class="p">(</span><span class="n">TypedLayer</span><span class="o"><</span><span class="n">PersistentBitMatrix</span><span class="o">></span><span class="p">),</span>
|
||
|
|
<span class="p">}</span>
|
||
|
|
</code></pre></div>
|
||
|
|
<p><code>Layer::create(dir)</code> makes the directory and returns <code>Empty { dir }</code> —
|
||
|
|
nothing else; no MPHF/unitigs/evidence construction yet (that's the
|
||
|
|
deferred next step: <code>build_mphf()</code>/<code>build_unitigs()</code>/<code>build_evidence()</code>
|
||
|
|
methods to progress <code>Empty</code> → eventually <code>Count</code>/<code>Presence</code>). <code>Empty</code>
|
||
|
|
carries path accessors so builder code has one place to get
|
||
|
|
<code>mphf_path()</code>/<code>unitigs_path()</code>/<code>evidence_path()</code>/<code>fingerprint_path()</code>/
|
||
|
|
<code>counts_dir()</code>/<code>presence_dir()</code> from, instead of redeclaring the
|
||
|
|
<code>mphf.bin</code>/<code>unitigs.bin</code>/… filenames at each write site — reusing the
|
||
|
|
constants <code>layer.rs</code>/<code>mphf_layer.rs</code> already own (<code>COUNTS_DIR</code>/
|
||
|
|
<code>PRESENCE_DIR</code> widened from private to <code>pub(crate)</code>, file-name constants
|
||
|
|
already were).</p>
|
||
|
|
<p>Every read method (<code>content</code>/<code>evidence_kind</code>/<code>n</code>/<code>find_slot</code>/
|
||
|
|
<code>index_batch</code>/<code>n_cols</code>/<code>fill_sub_matrix_carries</code>) panics on <code>Empty</code> with a
|
||
|
|
one-line message naming the method — confirmed as the right behaviour:
|
||
|
|
calling any of them on an <code>Empty</code> layer means the caller assumed a layer
|
||
|
|
was ready when it wasn't, an implementation error to surface loudly, not
|
||
|
|
a case to design around (<code>Option</code>/<code>Result</code> would let it silently
|
||
|
|
propagate instead of failing at the actual mistake). Same panic added to
|
||
|
|
<code>obikphylo::siblings::iter.rs</code>'s <code>impl SiblingLayerExt for
|
||
|
|
obilayeredmap::Layer</code> (4 methods), the one other place that exhaustively
|
||
|
|
matched <code>Layer</code>'s variants.</p>
|
||
|
|
<p>Full workspace suite green (<code>cargo check --workspace --all-targets</code> then
|
||
|
|
<code>cargo test --workspace</code>, exit code 0) after.</p>
|
||
|
|
<p>Still deferred, per explicit instruction: <code>build_mphf()</code>/
|
||
|
|
<code>build_unitigs()</code>/<code>build_evidence()</code> to progress <code>Empty</code> further, and (2)
|
||
|
|
— <code>KmerPartition</code> itself — unchanged from above (see "(2a) done" below,
|
||
|
|
added next).</p>
|
||
|
|
<h2 id="2a-done-2026-08-20-obikpartition-crate-kmerpartition">(2a) done (2026-08-20): <code>obikpartition</code> crate + <code>KmerPartition</code></h2>
|
||
|
|
<p>Built exactly the shape "Definitions" (top of file) specifies, nothing
|
||
|
|
more — deliberately scoped down from the full "Direction agreed" plan
|
||
|
|
below: only steps 1–2 (<code>open</code>/<code>n_layers</code>/<code>layer</code>/<code>layers</code>/<code>find</code>), not 3–4
|
||
|
|
(migrating <code>PartitionCache</code>/<code>QueryLayer</code> onto it), per explicit
|
||
|
|
instruction to implement <code>KmerPartition</code> first and decide the wiring
|
||
|
|
("comment on branche tout ça dans la construction") separately, later.</p>
|
||
|
|
<div class="highlight"><pre><span></span><code><span class="k">pub</span><span class="w"> </span><span class="k">struct</span><span class="w"> </span><span class="nc">KmerPartition</span><span class="w"> </span><span class="p">{</span>
|
||
|
|
<span class="w"> </span><span class="n">layers</span><span class="p">:</span><span class="w"> </span><span class="nb">Vec</span><span class="o"><</span><span class="n">obilayeredmap</span><span class="p">::</span><span class="n">Layer</span><span class="o">></span><span class="p">,</span>
|
||
|
|
<span class="p">}</span>
|
||
|
|
|
||
|
|
<span class="k">impl</span><span class="w"> </span><span class="n">KmerPartition</span><span class="w"> </span><span class="p">{</span>
|
||
|
|
<span class="w"> </span><span class="k">pub</span><span class="w"> </span><span class="k">fn</span><span class="w"> </span><span class="nf">open</span><span class="p">(</span><span class="n">index_dir</span><span class="p">:</span><span class="w"> </span><span class="kp">&</span><span class="nc">Path</span><span class="p">,</span><span class="w"> </span><span class="n">mode</span><span class="p">:</span><span class="w"> </span><span class="kp">&</span><span class="nc">IndexMode</span><span class="p">,</span><span class="w"> </span><span class="n">n_layers</span><span class="p">:</span><span class="w"> </span><span class="kt">usize</span><span class="p">,</span><span class="w"> </span><span class="n">with_counts</span><span class="p">:</span><span class="w"> </span><span class="kt">bool</span><span class="p">)</span><span class="w"> </span><span class="p">-></span><span class="w"> </span><span class="nc">OLMResult</span><span class="o"><</span><span class="bp">Self</span><span class="o">></span><span class="p">;</span>
|
||
|
|
<span class="w"> </span><span class="k">pub</span><span class="w"> </span><span class="k">fn</span><span class="w"> </span><span class="nf">n_layers</span><span class="p">(</span><span class="o">&</span><span class="bp">self</span><span class="p">)</span><span class="w"> </span><span class="p">-></span><span class="w"> </span><span class="kt">usize</span><span class="p">;</span>
|
||
|
|
<span class="w"> </span><span class="k">pub</span><span class="w"> </span><span class="k">fn</span><span class="w"> </span><span class="nf">layer</span><span class="p">(</span><span class="o">&</span><span class="bp">self</span><span class="p">,</span><span class="w"> </span><span class="n">i</span><span class="p">:</span><span class="w"> </span><span class="kt">usize</span><span class="p">)</span><span class="w"> </span><span class="p">-></span><span class="w"> </span><span class="kp">&</span><span class="nc">Layer</span><span class="p">;</span>
|
||
|
|
<span class="w"> </span><span class="k">pub</span><span class="w"> </span><span class="k">fn</span><span class="w"> </span><span class="nf">layers</span><span class="p">(</span><span class="o">&</span><span class="bp">self</span><span class="p">)</span><span class="w"> </span><span class="p">-></span><span class="w"> </span><span class="kp">&</span><span class="p">[</span><span class="n">Layer</span><span class="p">];</span>
|
||
|
|
<span class="w"> </span><span class="k">pub</span><span class="w"> </span><span class="k">fn</span><span class="w"> </span><span class="nf">find</span><span class="p">(</span><span class="o">&</span><span class="bp">self</span><span class="p">,</span><span class="w"> </span><span class="n">kmer</span><span class="p">:</span><span class="w"> </span><span class="nc">CanonicalKmer</span><span class="p">)</span><span class="w"> </span><span class="p">-></span><span class="w"> </span><span class="nb">Option</span><span class="o"><</span><span class="kt">usize</span><span class="o">></span><span class="p">;</span>
|
||
|
|
<span class="p">}</span>
|
||
|
|
</code></pre></div>
|
||
|
|
<p><code>open</code> takes <code>index_dir</code>/<code>mode</code>/<code>n_layers</code>/<code>with_counts</code> as plain
|
||
|
|
arguments — no reach-back into <code>KmerIndex</code> (would need <code>obikpartition →
|
||
|
|
obikindex</code>, the wrong direction) — and builds each layer's path via
|
||
|
|
<code>obilayeredmap::layer_dir(index_dir, l)</code>, the same shared naming
|
||
|
|
primitive <code>KmerIndex::layer_dir</code> itself delegates to, not a second copy of
|
||
|
|
the <code>layer_N</code> convention. <code>find</code> mirrors <code>PartitionCache::find</code>'s
|
||
|
|
semantics (first layer that carries the kmer wins) but doesn't yet cover
|
||
|
|
<code>find_presence_batch</code>/<code>find_presence_batch_fast</code> — those exist only to
|
||
|
|
serve <code>PartitionCache</code>, so they're part of the (2b) migration, not this
|
||
|
|
step; building them now against the current sibling-specific tuple shape
|
||
|
|
<code>(CanonicalKmer, usize, u8, u8)</code> would either bake phylo vocabulary
|
||
|
|
(<code>family_idx</code>, <code>base</code>) into <code>obikpartition</code> or require deciding a generic
|
||
|
|
payload shape — a real design fork, deferred to when (2b) is actually
|
||
|
|
tackled rather than guessed at here.</p>
|
||
|
|
<p>Crate deps: <code>obikseq</code>, <code>obilayeredmap</code> only (dev-deps add <code>obiskio</code>,
|
||
|
|
<code>obicompactvec</code>, <code>tempfile</code> for tests) — matches the "Definitions"
|
||
|
|
constraint (<code>obikpartition</code> depends on <code>obilayeredmap</code> and below, never
|
||
|
|
<code>obikindex</code>/<code>obikpartitionner</code>/<code>obikphylo</code>). Registered as a new workspace
|
||
|
|
member (<code>src/Cargo.toml</code>). 3 new tests (<code>open_reads_every_layer_in_order</code>,
|
||
|
|
<code>find_reports_the_first_layer_that_carries_the_kmer</code>,
|
||
|
|
<code>find_returns_none_for_an_absent_kmer</code>). Full workspace suite green
|
||
|
|
(<code>cargo check --workspace --all-targets</code> then <code>cargo test --workspace</code>)
|
||
|
|
after.</p>
|
||
|
|
<p>Still not done: (2b) — migrating <code>obikphylo::siblings::cache::
|
||
|
|
PartitionCache</code> (currently <code>Vec<Vec<Layer>></code>) and
|
||
|
|
<code>obikindex::query_layer::QueryLayer</code> (currently uncached, bypasses <code>Layer</code>
|
||
|
|
entirely) onto <code>KmerPartition</code>/<code>Vec<KmerPartition></code>; deciding whether that
|
||
|
|
collection lives in <code>obikpartition</code> or <code>obikindex</code>; deciding the
|
||
|
|
batch-lookup surface's exact shape (generic payload vs. as-is sibling
|
||
|
|
tuple moved in wholesale); <code>scan_layer_families</code>'s still-independent
|
||
|
|
<code>PartitionMeta::load</code> (see "Remaining instance…" below) — all explicitly
|
||
|
|
deferred to whenever wiring is tackled next.</p>
|
||
|
|
<h2 id="3-done-2026-08-20-obikindex-obikpartitionner-dependency-inverted-partitionrouter-now-fills-layerempty-shells">(3) done (2026-08-20): <code>obikindex ↔ obikpartitionner</code> dependency inverted, <code>PartitionRouter</code> now fills <code>Layer::Empty</code> shells</h2>
|
||
|
|
<p>Resolved a question left implicit since "Major restructuring": that pass
|
||
|
|
set the direction <code>obikindex → obikpartitionner</code> (so <code>KmerIndex</code> could
|
||
|
|
delegate <code>partition_dir</code> to it) without questioning whether that was the
|
||
|
|
right direction at all. Challenged directly: <code>obikpartitionner</code> is an
|
||
|
|
<em>algorithm</em> (superkmer routing/dereplication/counting) operating on an
|
||
|
|
<em>index</em> (<code>KmerIndex</code>, the data structure) — algorithms depend on the data
|
||
|
|
types they need, not the other way around. [[feedback_no_precedent_defense]]
|
||
|
|
applied here: "that's the direction we already picked" was not treated as
|
||
|
|
a justification for keeping it.</p>
|
||
|
|
<p><strong>New direction</strong>: <code>obikpartitionner → obikindex</code> (+ <code>obilayeredmap</code>,
|
||
|
|
<code>obipipeline</code>, <code>obiread</code> directly, for what <code>run</code>'s pipeline itself needs).
|
||
|
|
<code>obikindex → obikpartitionner</code> is gone entirely — <code>KmerIndex</code> no longer
|
||
|
|
imports <code>PartitionRouter</code>/<code>KmerSpectrum</code> in any form. Two path-naming
|
||
|
|
primitives that used to make this edge necessary moved down a tier instead
|
||
|
|
of staying put:
|
||
|
|
- <code>partition_dir</code>/<code>PARTITIONS_SUBDIR</code> moved from <code>obikpartitionner</code> into
|
||
|
|
<code>obikpartition</code> (the Partition-tier crate <code>KmerPartition</code> already lives
|
||
|
|
in), alongside a new <code>index_dir(root, i)</code> — both free functions,
|
||
|
|
mirroring <code>obilayeredmap::layer_dir</code> one tier down. <code>KmerIndex::
|
||
|
|
partition_dir</code>/<code>index_dir</code> now delegate here instead of to
|
||
|
|
<code>obikpartitionner</code>/an inline <code>.join("index")</code>.
|
||
|
|
- <code>KmerIndex::create</code>/<code>create_skeleton</code> no longer call
|
||
|
|
<code>PartitionRouter::create</code> to lay out an empty <code>partitions/</code> skeleton
|
||
|
|
upfront — turned out to be dead weight once traced: <code>select_layer.rs</code>/
|
||
|
|
<code>rebuild_layer.rs</code> already <code>create_dir_all</code> their own partition/layer
|
||
|
|
directories on demand, and <code>Layer::create</code>'s directory-creation covers
|
||
|
|
the scatter path the same way. Partitions and their layer-0 shells now
|
||
|
|
come into existence lazily, on first write, with nothing to pre-create.
|
||
|
|
<code>KmerIndex::create</code>'s now-unused <code>force: bool</code> parameter was dropped
|
||
|
|
(4 call sites updated) rather than left as a dead parameter.</p>
|
||
|
|
<p><strong><code>PartitionRouter</code> reshaped</strong> (<code>obikpartitionner/src/partition/router.rs</code>)
|
||
|
|
around the "création, paramétrage, run()" shape agreed on: <code>new(index:
|
||
|
|
&mut KmerIndex) -> Self</code> (no disk access), chainable setters
|
||
|
|
(<code>level_max</code>/<code>theta</code>/<code>workers</code>/<code>max_open</code>, defaults matching the CLI's old
|
||
|
|
hardcoded values), then <code>run(path_source, on_progress)</code>. <code>write</code>/
|
||
|
|
<code>write_batch</code>/<code>flush</code>/<code>close</code>/<code>dereplicate</code>/<code>count_kmer</code> stay public,
|
||
|
|
unconsumed (<code>&self</code>/<code>&mut self</code>, not <code>self</code>) — callers needing fine-grained
|
||
|
|
control (tests, <code>obikphylo</code>'s test harness) still get it, <code>run</code> is a
|
||
|
|
convenience layered on top, not the only way in.</p>
|
||
|
|
<p><code>run</code> absorbs the entire body of what used to be the free function
|
||
|
|
<code>obikmer::steps::scatter</code> (now deleted, along with the <code>steps</code> module
|
||
|
|
entirely) — the <code>obipipeline::make_pipe!</code> two-stage pipeline
|
||
|
|
(file→pages→superkmers), throttling, per-file logging. What changed:
|
||
|
|
- Every <code>ensure_writer(partition)</code> call now does <code>Layer::create(&layer0_dir)</code>
|
||
|
|
(<code>layer0_dir = obilayeredmap::layer_dir(&index.index_dir(i), 0)</code>) before
|
||
|
|
opening <code>raw.{ext}</code> inside it — raw/dereplicated superkmer files and the
|
||
|
|
provisional <code>mphf1.bin</code>/<code>counts1.bin</code>/<code>kmer_spectrum_raw.json</code> now live
|
||
|
|
under <code><partition>/index/layer_0/</code>, not flat under <code><partition>/</code> as
|
||
|
|
before. This is <code>Layer::Empty</code> actually being used as the "builder code
|
||
|
|
holding an <code>Empty</code> layer" its own (1b) docs anticipated, not just a shell
|
||
|
|
with no consumer.
|
||
|
|
- <strong>Caught by an end-to-end smoke test, not by <code>cargo test</code></strong>: this path
|
||
|
|
move broke <code>obikindex::index_layer::build_index_layer</code> and
|
||
|
|
<code>remove_build_artifacts</code>, both of which still read/deleted
|
||
|
|
<code>dereplicated.skmer.zst</code>/<code>mphf1.bin</code>/<code>counts1.bin</code> from
|
||
|
|
<code>self.partition_dir(i)</code> (the old flat location) — no test in the
|
||
|
|
workspace suite exercises the real CLI's file-reading <code>scatter</code> path
|
||
|
|
end-to-end (<code>obikphylo</code>'s test harness and <code>obikpartitionner</code>'s own
|
||
|
|
tests both call <code>write_batch</code> directly, bypassing <code>run</code>/file discovery
|
||
|
|
entirely), so the whole suite stayed green while <code>obikmer index</code> on
|
||
|
|
real FASTA silently indexed 0 kmers. Found by running the actual CLI
|
||
|
|
against a small FASTA and noticing <code>count.json</code>'s <code>f0</code> (870, correct)
|
||
|
|
didn't match "0 total kmers indexed" at the final stage. Fixed by
|
||
|
|
retargeting both functions to <code>self.layer_dir(i, 0)</code>. <strong>Lesson,
|
||
|
|
consistent with the retracted-claim lesson above</strong>: a green test suite
|
||
|
|
is not proof a refactor is correct when no test in it exercises the
|
||
|
|
specific path that changed — for anything touching the CLI's own
|
||
|
|
file-driven entry point, running the CLI for real is not optional
|
||
|
|
verification.
|
||
|
|
- The internal <code>obisys::spinner("scatter")</code> + hand-rolled EMA-rate display
|
||
|
|
is gone from the library entirely, replaced by an <code>Option<impl
|
||
|
|
FnMut(obisys::Progress)></code> parameter — a new, deliberately generic
|
||
|
|
progress-reporting type (<code>obisys::Progress { position: u64, total:
|
||
|
|
Option<u64> }</code>, alongside the existing <code>TracedBar</code>/<code>spinner</code>/
|
||
|
|
<code>progress_bar</code>) added specifically so every future algo crate's <code>run()</code>
|
||
|
|
reports progress the same shape, once, rather than each inventing its
|
||
|
|
own. <code>total: None</code> here (bases processed isn't knowable without
|
||
|
|
pre-scanning every input file) — deliberately simpler than the old
|
||
|
|
in-library rate/file-count/thread-count message; the caller can
|
||
|
|
recompute a Mbp/s rate from consecutive <code>position</code> values +
|
||
|
|
wall-clock time itself, which is exactly what <code>cmd/index/mod.rs</code> now
|
||
|
|
does to reproduce the old spinner message. This is a real, intentional
|
||
|
|
restriction of the library's job: it reports raw ticks, the CLI decides
|
||
|
|
what a human sees — same "generic vs. domain-specific" split applied
|
||
|
|
again, this time to progress reporting rather than to Layer content.
|
||
|
|
Explicitly <strong>not</strong> the same mechanism as <code>Stage</code>/<code>Reporter</code> (per
|
||
|
|
[[feedback_stage_reporter_in_cmd_layer]]): <code>Stage</code>/<code>Reporter</code> measures a
|
||
|
|
whole call's wall time from outside it; a progress callback has to fire
|
||
|
|
<em>from inside</em> a loop mid-call, which wrapping from outside cannot
|
||
|
|
express — two different needs, not the same rule reapplied under a new
|
||
|
|
name. <code>Stage::start("scatter")</code>/<code>rep.push(...)</code> stayed in
|
||
|
|
<code>cmd/index/mod.rs</code>, wrapping the whole <code>run()</code> call, unchanged in kind.
|
||
|
|
- <code>dereplicate</code>/<code>count_kmer</code> keep their existing internal
|
||
|
|
<code>obisys::progress_bar(...)</code> calls as-is (unconverted to the callback) —
|
||
|
|
explicitly out of scope for this pass, by agreement.</p>
|
||
|
|
<p><strong>Forced, not optional, consequence of the dependency inversion</strong>:
|
||
|
|
<code>KmerIndex::dereplicate_and_count</code>/<code>partition_router</code>/<code>write_spectrum(&
|
||
|
|
KmerSpectrum)</code> could not stay on <code>KmerIndex</code> at all once <code>obikindex</code> can no
|
||
|
|
longer name <code>obikpartitionner::{PartitionRouter, KmerSpectrum}</code> in any
|
||
|
|
position — not a design choice, a mechanical requirement of severing the
|
||
|
|
edge. Replaced by: <code>KmerIndex::write_spectrum(f0: u64, f1: u64, counts:
|
||
|
|
&BTreeMap<u32, u64>)</code> (plain values, no <code>KmerSpectrum</code> dependency) and a
|
||
|
|
new <code>KmerIndex::mark_counted()</code> (symmetric to the already-existing
|
||
|
|
<code>mark_scattered</code>), with the orchestration itself (<code>router.dereplicate()</code> →
|
||
|
|
<code>router.count_kmer()</code> → <code>write_spectrum</code> → <code>mark_counted()</code>) now living in
|
||
|
|
<code>cmd/index/mod.rs</code>, not <code>obikindex</code>.</p>
|
||
|
|
<p>Every <code>PartitionRouter::new(&mut index)</code> call in this codebase runs into
|
||
|
|
the same NLL trap once: <code>PartitionRouter</code> has a <code>Drop</code> impl (auto-<code>close</code>
|
||
|
|
on scope exit), which extends its <code>&mut KmerIndex</code> borrow to the end of
|
||
|
|
the enclosing scope even after its last real use — <code>idx.mark_scattered()</code>
|
||
|
|
right after <code>router.run(...)</code> (or <code>idx.write_spectrum(...)</code> right after
|
||
|
|
<code>router.count_kmer(...)</code>) fails to borrow-check unless the router is
|
||
|
|
<code>drop()</code>-ed explicitly first. Hit and fixed identically at all three call
|
||
|
|
sites that needed it (<code>cmd/index/mod.rs</code> ×2, <code>obikphylo</code>'s test harness,
|
||
|
|
<code>obikpartitionner</code>'s own tests).</p>
|
||
|
|
<p>Full workspace suite green (<code>cargo check --workspace --all-targets</code> +
|
||
|
|
<code>cargo test --workspace</code>, exit code 0) both before and after the
|
||
|
|
<code>index_layer.rs</code> fix above — the smoke test is what actually caught the
|
||
|
|
regression the suite missed.</p>
|
||
|
|
<h2 id="4-done-2026-08-20-obikderep-dereplication-split-out-of-obikpartitionner-one-algorithm-at-a-time">(4) done (2026-08-20): <code>obikderep</code> — dereplication split out of <code>obikpartitionner</code>, one algorithm at a time</h2>
|
||
|
|
<p>Follow-on question after (3): the indexing pipeline has 4 stages (scatter,
|
||
|
|
dereplicate, count_kmer, index-build — see the CLI's own <code>Reporter</code> output,
|
||
|
|
one line per stage), but <code>obikpartitionner</code> — a name that says
|
||
|
|
<em>partitioning</em> — owned three of them (routing, dereplication, counting).
|
||
|
|
Challenged directly, same as (3)'s dependency-direction question: a crate
|
||
|
|
should hold what its name says, not accumulate unrelated stages just
|
||
|
|
because they happened to land there first. Two ways to fix it — one crate
|
||
|
|
renamed to hold all remaining stages, or one crate per stage — decided in
|
||
|
|
favour of the latter, explicitly <strong>incremental</strong>: build the <em>second</em> algo
|
||
|
|
crate first (<code>obikderep</code>, dereplication only), only then look at what it
|
||
|
|
and <code>PartitionRouter</code> actually have in common, and factor a shared
|
||
|
|
<code>Algorithm</code> trait (future <code>obikalgorithm</code> crate) from that real overlap —
|
||
|
|
not guessed at from a single example. <code>count_kmer</code> and <code>build_layers</code>
|
||
|
|
(currently <code>KmerIndex</code> inherent methods — itself flagged as inconsistent
|
||
|
|
with "<code>KmerIndex</code> is a data structure, not a compute structure") are left
|
||
|
|
alone this round, on purpose — one stage moves at a time.</p>
|
||
|
|
<p><strong><code>obikderep</code></strong> (new crate): <code>Dereplicator<'a> { index: &'a KmerIndex, n_partitions, level }</code>
|
||
|
|
— <code>new(index: &KmerIndex)</code> (shared borrow, not <code>&mut</code>: dereplication never
|
||
|
|
writes index metadata), no setters yet (nothing to configure), <code>run(on_progress)</code>
|
||
|
|
does the two-phase split+merge dereplication in parallel across partitions,
|
||
|
|
ported unchanged from <code>PartitionRouter::dereplicate</code> (moved wholesale:
|
||
|
|
<code>optimal_buckets</code>/<code>dereplicate_partition</code>/<code>load_bucket</code>/<code>flush_map</code>/
|
||
|
|
<code>remove_skmer_file</code>, now private to this crate in <code>dereplicate.rs</code>).
|
||
|
|
<code>obikpartitionner::PartitionRouter::dereplicate</code> is gone; <code>count_kmer</code>
|
||
|
|
stays.</p>
|
||
|
|
<p><strong>A real signature difference from <code>PartitionRouter::run</code>, not an
|
||
|
|
inconsistency</strong>: <code>Dereplicator::run</code> takes <code>Option<impl Fn(Progress) +
|
||
|
|
Sync></code>, not <code>FnMut</code>. <code>PartitionRouter::run</code>'s callback is invoked from one
|
||
|
|
sequential loop (<code>FnMut</code> is fine); <code>Dereplicator::run</code>'s work is
|
||
|
|
<code>rayon::par_iter</code>, so the callback can be invoked concurrently from
|
||
|
|
multiple worker threads — same reason <code>obisys::TracedBar</code>'s own methods
|
||
|
|
take <code>&self</code>, not <code>&mut self</code>. Progress position is tracked with an
|
||
|
|
<code>AtomicU64</code>, incremented from inside the parallel closure so each
|
||
|
|
completed partition reports immediately — collecting all results first and
|
||
|
|
reporting after (the first draft of this) would have delivered every tick
|
||
|
|
in one burst at the very end, defeating the point of a live progress bar.
|
||
|
|
<code>total: Some(n_partitions)</code> (known up front, unlike scatter's bases count)
|
||
|
|
— <code>cmd/index/mod.rs</code> renders a real <code>progress_bar</code>, not a spinner, driven
|
||
|
|
by the callback exactly like scatter's spinner is.</p>
|
||
|
|
<p><strong>A second, pre-existing instance of the exact bug (3) fixed, caught
|
||
|
|
before it shipped</strong>: <code>dereplicated.skmer.zst</code> was hand-built as a string
|
||
|
|
literal independently in three places — <code>obikpartitionner</code>'s
|
||
|
|
<code>dereplicate.rs</code>/<code>count.rs</code> <em>and</em> <code>obikindex</code>'s <code>index_layer.rs</code> (a literal
|
||
|
|
that already predated this session, never caught until now). Splitting
|
||
|
|
dereplication into its own crate turns this from "two places, still
|
||
|
|
matching by luck" into "three independent crates that must agree on a
|
||
|
|
filename with no shared dependency forcing them to" — no longer
|
||
|
|
deferrable. Fixed by adding <code>obilayeredmap::{raw_superkmers_path,
|
||
|
|
dereplicated_superkmers_path}</code> (free functions, <code>layer_dir: &Path ->
|
||
|
|
PathBuf</code>, mirroring <code>layer_dir</code> itself) — the filename lives in one place,
|
||
|
|
in the Layer-tier crate every consumer here already depends on
|
||
|
|
(<code>obikpartitionner</code>, <code>obikderep</code>, <code>obikindex</code> all reach it without a new
|
||
|
|
edge), and no external crate ever sees the literal <code>"skmer.zst"</code> again.
|
||
|
|
This reverses (3)'s own earlier call to keep <code>SK_EXT</code> private to
|
||
|
|
<code>obikpartitionner</code> — that call assumed a single owner; a second owner
|
||
|
|
appearing (<code>obikderep</code>) removed the assumption it rested on, so the
|
||
|
|
decision changed with it, not out of inconsistency.</p>
|
||
|
|
<p>Every <code>count_kmer</code> call site that used to run after <code>router.dereplicate()</code>
|
||
|
|
on the same <code>PartitionRouter</code> now runs after a separate
|
||
|
|
<code>Dereplicator::new(&idx).run(...)</code> call, on a freshly-constructed
|
||
|
|
<code>PartitionRouter</code> — <code>PartitionRouter</code> no longer offers a combined
|
||
|
|
"dereplicate then count" path. Updated at all three call sites that had
|
||
|
|
one: <code>cmd/index/mod.rs</code>, <code>obikphylo</code>'s test harness, <code>obikpartitionner</code>'s
|
||
|
|
own tests.</p>
|
||
|
|
<p>Full workspace suite green (<code>cargo check --workspace --all-targets</code> +
|
||
|
|
<code>cargo test --workspace</code>, exit code 0), plus an end-to-end CLI smoke test
|
||
|
|
against real FASTA data (scatter → dereplicate → count → index-build →
|
||
|
|
query, same numbers as (3)'s smoke test: 870 kmers) — required this time
|
||
|
|
too, per (3)'s own lesson: no test in the suite exercises <code>obikmer index</code>'s
|
||
|
|
real file-driven path.</p>
|
||
|
|
<p>Still not done: <code>count_kmer</code>/<code>build_layers</code> staying where they are, the
|
||
|
|
<code>obikalgorithm</code> shared-trait extraction (deliberately deferred until a
|
||
|
|
third data point exists), and everything already listed under (2b).</p>
|
||
|
|
<h2 id="5-design-agreed-not-yet-implemented-2026-08-20-kmerpartition-rewritten-layer-gains-self-naming-a-future-cache-crate-over-kmerindex">(5) design agreed, not yet implemented (2026-08-20): <code>KmerPartition</code> rewritten, <code>Layer</code> gains self-naming, a future cache crate over <code>KmerIndex</code></h2>
|
||
|
|
<p>Session ended (out of budget) before any of this was coded. Everything
|
||
|
|
below is a <strong>fully specified plan</strong>, agreed sentence by sentence with the
|
||
|
|
user — not a sketch to re-derive, not a proposal to re-litigate. Implement
|
||
|
|
it as written; if something here turns out to be wrong once coded, fix it
|
||
|
|
and update this section, don't restart the design conversation.</p>
|
||
|
|
<h3 id="how-this-was-found">How this was found</h3>
|
||
|
|
<p>Direct question from the user: "tu as bien créé une structure
|
||
|
|
<code>KmerPartition</code> ?" — yes (2a), but investigating exposed that it is
|
||
|
|
<strong>wired into nothing</strong>. <code>KmerIndex</code> has no <code>partition(i)</code> method at all;
|
||
|
|
<code>partition_dir</code>/<code>index_dir</code>/<code>layer_dir</code> still call <code>obikpartition::
|
||
|
|
partition_dir</code>/<code>index_dir</code> and <code>obilayeredmap::layer_dir</code> as bare free
|
||
|
|
functions directly, never touching a <code>KmerPartition</code>/<code>Layer</code> object to get
|
||
|
|
there. The end result on disk is identical (same paths), which is exactly
|
||
|
|
why no test caught it — but the <em>responsibility</em> is in the wrong place:
|
||
|
|
one function (on <code>KmerIndex</code>) knows the whole three-tier naming
|
||
|
|
convention, instead of each tier asking the one below it for its own
|
||
|
|
path. User's framing, verbatim, now saved as [[feedback_no_spaghetti_petits_pois]]:
|
||
|
|
"spaghetti" (logic untraceable, split across too many unrelated crates)
|
||
|
|
and "petits pois" (small bits of naming logic dispersed with no owning
|
||
|
|
object) are <strong>strictly forbidden</strong> — this was a live example of both.</p>
|
||
|
|
<p>Pushed further, twice:
|
||
|
|
1. First correction: <code>Layer::create(&obilayeredmap::layer_dir(&dir, 0))</code> —
|
||
|
|
still a free-function call from <em>outside</em> <code>Layer</code> to compute where it
|
||
|
|
should live. "Le layer n'est pas con, c'est lui qui dit où est-ce qu'il
|
||
|
|
doit être sauvé" (the layer isn't stupid, it says itself where it
|
||
|
|
should be saved).
|
||
|
|
2. Second correction, the general principle: <strong>"une partition est juste
|
||
|
|
identifiée par un numéro, tout se calcule à partir du numéro, et un
|
||
|
|
layer est identifié à partir d'un numéro et tout se calcule à partir de
|
||
|
|
ce numéro."</strong> Concretely: each object stores its own local identifying
|
||
|
|
number <em>plus</em> its immediate parent's path (captured once, at
|
||
|
|
construction) — never a path handed in again later by a caller, and
|
||
|
|
never a free function outside the object that can compute that path
|
||
|
|
independently. Explicitly rejected along the way: making users pass
|
||
|
|
"the partition's path that contains the layer" to open a layer — the
|
||
|
|
parent path is captured once, at the child's construction, not
|
||
|
|
re-supplied at every call.</p>
|
||
|
|
<h3 id="the-agreed-shape">The agreed shape</h3>
|
||
|
|
<p><strong><code>Layer</code></strong> (<code>obilayeredmap</code>) — identified by <code>l</code> + its parent partition's
|
||
|
|
directory, both captured at construction, never received again:</p>
|
||
|
|
<div class="highlight"><pre><span></span><code><span class="k">pub</span><span class="w"> </span><span class="k">enum</span><span class="w"> </span><span class="nc">Layer</span><span class="w"> </span><span class="p">{</span>
|
||
|
|
<span class="w"> </span><span class="n">Empty</span><span class="w"> </span><span class="p">{</span><span class="w"> </span><span class="n">partition_dir</span><span class="p">:</span><span class="w"> </span><span class="nc">PathBuf</span><span class="p">,</span><span class="w"> </span><span class="n">l</span><span class="p">:</span><span class="w"> </span><span class="kt">usize</span><span class="w"> </span><span class="p">},</span><span class="w"> </span><span class="c1">// pure identification, no disk I/O</span>
|
||
|
|
<span class="w"> </span><span class="n">Count</span><span class="p">(</span><span class="n">TypedLayer</span><span class="o"><</span><span class="n">PersistentCompactIntMatrix</span><span class="o">></span><span class="p">),</span>
|
||
|
|
<span class="w"> </span><span class="n">Presence</span><span class="p">(</span><span class="n">TypedLayer</span><span class="o"><</span><span class="n">PersistentBitMatrix</span><span class="o">></span><span class="p">),</span>
|
||
|
|
<span class="p">}</span>
|
||
|
|
|
||
|
|
<span class="k">impl</span><span class="w"> </span><span class="n">Layer</span><span class="w"> </span><span class="p">{</span>
|
||
|
|
<span class="w"> </span><span class="k">pub</span><span class="w"> </span><span class="k">fn</span><span class="w"> </span><span class="nf">at</span><span class="p">(</span><span class="n">partition_dir</span><span class="p">:</span><span class="w"> </span><span class="kp">&</span><span class="nc">Path</span><span class="p">,</span><span class="w"> </span><span class="n">l</span><span class="p">:</span><span class="w"> </span><span class="kt">usize</span><span class="p">)</span><span class="w"> </span><span class="p">-></span><span class="w"> </span><span class="nc">Self</span><span class="p">;</span><span class="w"> </span><span class="c1">// identify only</span>
|
||
|
|
<span class="w"> </span><span class="k">fn</span><span class="w"> </span><span class="nf">dir</span><span class="p">(</span><span class="o">&</span><span class="bp">self</span><span class="p">)</span><span class="w"> </span><span class="p">-></span><span class="w"> </span><span class="nc">PathBuf</span><span class="p">;</span><span class="w"> </span><span class="c1">// private — layer_dir() no longer a public free function, folded in here</span>
|
||
|
|
<span class="w"> </span><span class="k">pub</span><span class="w"> </span><span class="k">fn</span><span class="w"> </span><span class="nf">create</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span><span class="w"> </span><span class="p">-></span><span class="w"> </span><span class="nc">io</span><span class="p">::</span><span class="nb">Result</span><span class="o"><</span><span class="bp">Self</span><span class="o">></span><span class="p">;</span><span class="w"> </span><span class="c1">// creates the directory if needed; no path parameter anymore</span>
|
||
|
|
<span class="w"> </span><span class="k">pub</span><span class="w"> </span><span class="k">fn</span><span class="w"> </span><span class="nf">open</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span><span class="w"> </span><span class="n">mode</span><span class="p">:</span><span class="w"> </span><span class="kp">&</span><span class="nc">IndexMode</span><span class="p">,</span><span class="w"> </span><span class="n">with_counts</span><span class="p">:</span><span class="w"> </span><span class="kt">bool</span><span class="p">)</span><span class="w"> </span><span class="p">-></span><span class="w"> </span><span class="nc">OLMResult</span><span class="o"><</span><span class="bp">Self</span><span class="o">></span><span class="p">;</span><span class="w"> </span><span class="c1">// no path parameter anymore</span>
|
||
|
|
<span class="w"> </span><span class="c1">// mphf_path()/unitigs_path()/evidence_path()/fingerprint_path()/counts_dir()/presence_dir()</span>
|
||
|
|
<span class="w"> </span><span class="c1">// unchanged in spirit, implemented via self.dir() instead of a stored `dir` field read directly</span>
|
||
|
|
<span class="p">}</span>
|
||
|
|
</code></pre></div>
|
||
|
|
<p>Note this <strong>replaces</strong> <code>Layer::Empty { dir: PathBuf }</code> from (1b) — <code>dir</code>
|
||
|
|
becomes a computed value (<code>partition_dir.join(format!("layer_{l}"))</code>), not
|
||
|
|
a stored field. <code>obilayeredmap::layer_dir</code>/<code>raw_superkmers_path</code>/
|
||
|
|
<code>dereplicated_superkmers_path</code> (currently public free functions,
|
||
|
|
introduced in (3)/(4)) stop being called from outside <code>obilayeredmap</code>
|
||
|
|
entirely once this lands — they were the right fix for their moment (a
|
||
|
|
second crate, <code>obikderep</code>, needed to agree on a filename with no owner),
|
||
|
|
but the <em>real</em> fix, now visible with a third data point, is that <code>Layer</code>
|
||
|
|
itself should be the only thing anyone asks.</p>
|
||
|
|
<p><strong><code>KmerPartition</code></strong> (<code>obikpartition</code>) — same principle, one tier up:</p>
|
||
|
|
<div class="highlight"><pre><span></span><code><span class="k">pub</span><span class="w"> </span><span class="k">struct</span><span class="w"> </span><span class="nc">KmerPartition</span><span class="w"> </span><span class="p">{</span>
|
||
|
|
<span class="w"> </span><span class="n">index_root</span><span class="p">:</span><span class="w"> </span><span class="nc">PathBuf</span><span class="p">,</span><span class="w"> </span><span class="c1">// the parent KmerIndex's root, captured once</span>
|
||
|
|
<span class="w"> </span><span class="n">i</span><span class="p">:</span><span class="w"> </span><span class="kt">usize</span><span class="p">,</span>
|
||
|
|
<span class="p">}</span>
|
||
|
|
|
||
|
|
<span class="k">impl</span><span class="w"> </span><span class="n">KmerPartition</span><span class="w"> </span><span class="p">{</span>
|
||
|
|
<span class="w"> </span><span class="k">pub</span><span class="w"> </span><span class="k">fn</span><span class="w"> </span><span class="nf">new</span><span class="p">(</span><span class="n">index_root</span><span class="p">:</span><span class="w"> </span><span class="nc">PathBuf</span><span class="p">,</span><span class="w"> </span><span class="n">i</span><span class="p">:</span><span class="w"> </span><span class="kt">usize</span><span class="p">)</span><span class="w"> </span><span class="p">-></span><span class="w"> </span><span class="nc">Self</span><span class="p">;</span><span class="w"> </span><span class="c1">// identify only, no disk I/O</span>
|
||
|
|
<span class="w"> </span><span class="k">pub</span><span class="w"> </span><span class="k">fn</span><span class="w"> </span><span class="nf">create</span><span class="p">(</span><span class="n">index_root</span><span class="p">:</span><span class="w"> </span><span class="nc">PathBuf</span><span class="p">,</span><span class="w"> </span><span class="n">i</span><span class="p">:</span><span class="w"> </span><span class="kt">usize</span><span class="p">)</span><span class="w"> </span><span class="p">-></span><span class="w"> </span><span class="nc">io</span><span class="p">::</span><span class="nb">Result</span><span class="o"><</span><span class="bp">Self</span><span class="o">></span><span class="p">;</span><span class="w"> </span><span class="c1">// creates this partition's directory + an empty layer 0 (a partition is never born without one — that knowledge lives here, not in whoever calls create)</span>
|
||
|
|
<span class="w"> </span><span class="k">pub</span><span class="w"> </span><span class="k">fn</span><span class="w"> </span><span class="nf">partition_dir</span><span class="p">(</span><span class="o">&</span><span class="bp">self</span><span class="p">)</span><span class="w"> </span><span class="p">-></span><span class="w"> </span><span class="nc">PathBuf</span><span class="p">;</span><span class="w"> </span><span class="c1">// part_{i:05}</span>
|
||
|
|
<span class="w"> </span><span class="k">pub</span><span class="w"> </span><span class="k">fn</span><span class="w"> </span><span class="nf">index_dir</span><span class="p">(</span><span class="o">&</span><span class="bp">self</span><span class="p">)</span><span class="w"> </span><span class="p">-></span><span class="w"> </span><span class="nc">PathBuf</span><span class="p">;</span><span class="w"> </span><span class="c1">// part_{i:05}/index</span>
|
||
|
|
<span class="w"> </span><span class="k">pub</span><span class="w"> </span><span class="k">fn</span><span class="w"> </span><span class="nf">layer</span><span class="p">(</span><span class="o">&</span><span class="bp">self</span><span class="p">,</span><span class="w"> </span><span class="n">l</span><span class="p">:</span><span class="w"> </span><span class="kt">usize</span><span class="p">)</span><span class="w"> </span><span class="p">-></span><span class="w"> </span><span class="nc">Layer</span><span class="p">;</span><span class="w"> </span><span class="c1">// Layer::at(&self.index_dir(), l) — caller never touches a path</span>
|
||
|
|
<span class="w"> </span><span class="k">pub</span><span class="w"> </span><span class="k">fn</span><span class="w"> </span><span class="nf">meta</span><span class="p">(</span><span class="o">&</span><span class="bp">self</span><span class="p">)</span><span class="w"> </span><span class="p">-></span><span class="w"> </span><span class="nc">SKResult</span><span class="o"><</span><span class="n">PartitionMeta</span><span class="o">></span><span class="p">;</span><span class="w"> </span><span class="c1">// n_layers + mode; must absorb the recovery-on-missing-file logic</span>
|
||
|
|
<span class="w"> </span><span class="c1">// currently private in obikindex::common::load_meta (obikpartition</span>
|
||
|
|
<span class="w"> </span><span class="c1">// can't depend on obikindex to reuse it — this logic moves down)</span>
|
||
|
|
<span class="w"> </span><span class="k">pub</span><span class="w"> </span><span class="k">fn</span><span class="w"> </span><span class="nf">n_layers</span><span class="p">(</span><span class="o">&</span><span class="bp">self</span><span class="p">)</span><span class="w"> </span><span class="p">-></span><span class="w"> </span><span class="nc">SKResult</span><span class="o"><</span><span class="kt">usize</span><span class="o">></span><span class="p">;</span><span class="w"> </span><span class="c1">// meta()?.n_layers</span>
|
||
|
|
<span class="w"> </span><span class="k">pub</span><span class="w"> </span><span class="k">fn</span><span class="w"> </span><span class="nf">mode</span><span class="p">(</span><span class="o">&</span><span class="bp">self</span><span class="p">)</span><span class="w"> </span><span class="p">-></span><span class="w"> </span><span class="nc">SKResult</span><span class="o"><</span><span class="n">IndexMode</span><span class="o">></span><span class="p">;</span><span class="w"> </span><span class="c1">// meta()?.mode — "exact/approximatif"</span>
|
||
|
|
<span class="w"> </span><span class="k">pub</span><span class="w"> </span><span class="k">fn</span><span class="w"> </span><span class="nf">is_filled</span><span class="p">(</span><span class="o">&</span><span class="bp">self</span><span class="p">)</span><span class="w"> </span><span class="p">-></span><span class="w"> </span><span class="kt">bool</span><span class="p">;</span><span class="w"> </span><span class="c1">// does this partition's directory exist at all</span>
|
||
|
|
<span class="w"> </span><span class="k">pub</span><span class="w"> </span><span class="k">fn</span><span class="w"> </span><span class="nf">n_kmers</span><span class="p">(</span><span class="o">&</span><span class="bp">self</span><span class="p">)</span><span class="w"> </span><span class="p">-></span><span class="w"> </span><span class="nc">io</span><span class="p">::</span><span class="nb">Result</span><span class="o"><</span><span class="kt">usize</span><span class="o">></span><span class="p">;</span><span class="w"> </span><span class="c1">// LayerMeta::load(&self.layer(0).dir()).n — reads layer 0's count as</span>
|
||
|
|
<span class="w"> </span><span class="c1">// a representative figure, same "read the first one" trick</span>
|
||
|
|
<span class="w"> </span><span class="c1">// n_layers_per_partition() already uses at the KmerIndex level</span>
|
||
|
|
<span class="p">}</span>
|
||
|
|
</code></pre></div>
|
||
|
|
<p>This <strong>replaces</strong> (2a)'s <code>KmerPartition { layers: Vec<Layer> }</code> entirely
|
||
|
|
— no eagerly-opened <code>Vec<Layer></code>, no <code>find()</code> (both belong to the future
|
||
|
|
cache, see below, which is the thing that actually holds opened layers
|
||
|
|
alive across many lookups). (2a)'s version is safe to delete outright: it
|
||
|
|
was never wired into anything (confirmed above), so nothing depends on
|
||
|
|
its current shape. New dependencies needed: <code>obikpartition</code> gains
|
||
|
|
<code>obiskio</code> (for <code>SKResult</code>) and <code>obicompactvec</code> (for <code>LayerMeta</code>).</p>
|
||
|
|
<p>Deliberately <strong>not built this round</strong>: cross-level consistency checks
|
||
|
|
("verify everything below me is in the same state") — a real idea, raised
|
||
|
|
by the user, but nothing concrete needs it yet; building it speculatively
|
||
|
|
would be exactly the premature-abstraction pattern this project avoids.</p>
|
||
|
|
<p><strong><code>KmerIndex</code></strong> (<code>obikindex</code>) — becomes the sole entry point:</p>
|
||
|
|
<div class="highlight"><pre><span></span><code><span class="k">pub</span><span class="w"> </span><span class="k">fn</span><span class="w"> </span><span class="nf">partition</span><span class="p">(</span><span class="o">&</span><span class="bp">self</span><span class="p">,</span><span class="w"> </span><span class="n">i</span><span class="p">:</span><span class="w"> </span><span class="kt">usize</span><span class="p">)</span><span class="w"> </span><span class="p">-></span><span class="w"> </span><span class="nc">KmerPartition</span><span class="p">;</span><span class="w"> </span><span class="c1">// KmerPartition::new(self.root_path.clone(), i)</span>
|
||
|
|
</code></pre></div>
|
||
|
|
<p><code>partition_dir(i)</code>/<code>index_dir(i)</code>/<code>layer_dir(i, l)</code> <strong>stay</strong> as public
|
||
|
|
methods (≈30 existing call sites across <code>obikindex</code>/<code>obikphylo</code> — see (3)'s
|
||
|
|
option A, applied identically here) but become pure delegations:
|
||
|
|
<code>self.partition(i).partition_dir()</code>, <code>self.partition(i).index_dir()</code>,
|
||
|
|
<code>self.partition(i).layer(l).dir()</code> (needs <code>Layer::dir()</code> to be visible
|
||
|
|
enough for this — likely <code>pub(crate)</code> in <code>obilayeredmap</code> plus a thin
|
||
|
|
public wrapper, or a public accessor on <code>Layer</code> itself; not fully nailed
|
||
|
|
down, decide while implementing). No caller outside <code>obikindex</code> changes.</p>
|
||
|
|
<h3 id="known-blast-radius-why-this-wasnt-done-in-the-same-session">Known blast radius (why this wasn't done in the same session)</h3>
|
||
|
|
<ul>
|
||
|
|
<li><strong>14 files</strong> call <code>Layer::open</code>/<code>Layer::create</code> directly today
|
||
|
|
(<code>obikphylo/siblings/{cache,build,family_scan,tests}.rs</code>,
|
||
|
|
<code>obikpartitionner/partition/router.rs</code>, <code>obikpartition/src/lib.rs</code>,
|
||
|
|
<code>obikindex/{rebuild_layer,dump_layer,index,query_layer}.rs</code>,
|
||
|
|
<code>obilayeredmap/{mphf_layer,layer,map,content_layer}.rs</code>) — every one
|
||
|
|
loses its path parameter and gains a <code>(partition_dir, l)</code> or an
|
||
|
|
already-identified <code>Layer</code> to call <code>.create()</code>/<code>.open()</code> on instead.</li>
|
||
|
|
<li><strong>≈30 files</strong> call <code>KmerIndex::partition_dir</code>/<code>index_dir</code>/<code>layer_dir</code> —
|
||
|
|
unaffected in their own code (same public signatures), but worth
|
||
|
|
re-checking once (5) lands that none of them were relying on the old
|
||
|
|
free-function-based implementation in a way the new delegation breaks.</li>
|
||
|
|
<li><code>obikpartitionner::PartitionRouter::ensure_writer</code> and <code>obikderep</code>'s
|
||
|
|
<code>run</code> both currently call <code>obilayeredmap::{layer_dir, raw_superkmers_path,
|
||
|
|
dereplicated_superkmers_path}</code> directly (from (3)/(4)) — both need to
|
||
|
|
switch to going through <code>index.partition(i).layer(0)</code> instead.</li>
|
||
|
|
</ul>
|
||
|
|
<h3 id="also-agreed-separately-partitionrouternew-never-needed-mut-kmerindex">Also agreed, separately: <code>PartitionRouter::new</code> never needed <code>&mut KmerIndex</code></h3>
|
||
|
|
<p>Verified by reading the code: every call <code>PartitionRouter</code> makes on
|
||
|
|
<code>index</code> is <code>&self</code> (<code>index.kmer_size()</code>, <code>index.index_dir(i)</code>). The <code>&mut</code>
|
||
|
|
in its current signature (from (3)) was inherited from the original
|
||
|
|
"the router writes to the partitions" reasoning, never actually required
|
||
|
|
by any method call. This is <em>exactly</em> what caused every <code>drop(router)</code>
|
||
|
|
workaround needed throughout (3)/(4) (<code>cmd/index/mod.rs</code> ×2, <code>obikphylo</code>'s
|
||
|
|
test harness, <code>obikpartitionner</code>'s own tests) — <code>PartitionRouter</code> holds a
|
||
|
|
<code>Drop</code> impl, which extends a <code>&mut</code> borrow to the end of its scope even
|
||
|
|
past its last real use. <strong>Fix alongside (5)</strong>: change
|
||
|
|
<code>PartitionRouter::new(index: &'a mut KmerIndex)</code> to <code>&'a KmerIndex</code>, and
|
||
|
|
remove the now-unnecessary <code>drop(router)</code> calls at all four sites.</p>
|
||
|
|
<h3 id="also-discussed-a-future-cache-crate-not-part-of-5-not-obikalgorithm-either">Also discussed: a future cache crate, not part of (5), not <code>obikalgorithm</code> either</h3>
|
||
|
|
<p>Separate idea, explicitly <strong>not</strong> part of this design and <strong>not</strong> started:
|
||
|
|
a new crate whose only job is to cache open <code>KmerPartition</code>s (and their
|
||
|
|
opened <code>Layer</code>s) across one run — replacing both <code>obikphylo::siblings::
|
||
|
|
cache::PartitionCache</code> (today, sibling-specific, holds <code>Vec<Vec<Layer>></code>)
|
||
|
|
and <code>obikindex::query_layer::QueryLayer</code> (today, uncached, bypasses
|
||
|
|
<code>Layer</code> entirely) — the two consumers (2b) already identified as each
|
||
|
|
reinventing a fragment of the same thing.</p>
|
||
|
|
<p>User's framing: this is <strong>not</strong> a third <code>obikalgorithm</code> data point — an
|
||
|
|
algorithm has a <code>new → run → done</code> shape; a cache has a fundamentally
|
||
|
|
different one (open, stay alive for a whole run, serve lookups, maybe
|
||
|
|
evict) — "on crée un cache sur un index, ça consomme un index." Two
|
||
|
|
distinct crate <em>roles</em> in this ecosystem (data crates: <code>obikpartition</code>/
|
||
|
|
<code>obilayeredmap</code>; algorithm crates: <code>obikpartitionner</code>/<code>obikderep</code>/future
|
||
|
|
<code>obikalgorithm</code> implementors; and now a cache/service crate), not one
|
||
|
|
unified shape to force everything into.</p>
|
||
|
|
<p>Depends on (5) being done first: the cache crate's whole job is holding
|
||
|
|
<code>Vec<KmerPartition></code>/opened <code>Layer</code>s alive, built via <code>KmerIndex::
|
||
|
|
partition(i)</code> as its factory — nothing to build it on top of until (5)
|
||
|
|
lands. Still open once (5) is done: eviction policy vs. holding everything
|
||
|
|
open for the process lifetime (the never-measured mmap/VM-mapping-count
|
||
|
|
question from earlier in this doc), and whether it lives in <code>obikpartition</code>
|
||
|
|
itself or a new crate.</p>
|
||
|
|
<h3 id="order-of-remaining-work-as-currently-understood">Order of remaining work, as currently understood</h3>
|
||
|
|
<ol>
|
||
|
|
<li><strong>(5)</strong> — <code>Layer</code>/<code>KmerPartition</code>/<code>KmerIndex</code> rewrite described above,
|
||
|
|
plus the <code>PartitionRouter</code> <code>&mut</code> → <code>&</code> fix (same root cause, same
|
||
|
|
session, do together).</li>
|
||
|
|
<li>The future cache crate (name not chosen), consuming <code>KmerIndex::
|
||
|
|
partition(i)</code> — unblocks migrating <code>PartitionCache</code>/<code>QueryLayer</code> (2b).</li>
|
||
|
|
<li><code>obikalgorithm</code> — still deliberately waiting for a third <code>run()</code>-shaped
|
||
|
|
data point (<code>count_kmer</code> or <code>build_layers</code> migrating out of
|
||
|
|
<code>KmerIndex</code>/<code>PartitionRouter</code>) before extracting a shared trait; two
|
||
|
|
examples were judged not enough to be sure of the shape (<code>Fn+Sync</code> vs
|
||
|
|
<code>FnMut</code> callback bound already diverged between the two that exist).</li>
|
||
|
|
</ol>
|
||
|
|
<h2 id="the-problem">The problem</h2>
|
||
|
|
<p>Reading a layer's data (MPHF + matrix) is not free: <code>MphfLayer::open</code> mmaps
|
||
|
|
<code>mphf.bin</code> plus (<code>evidence.bin</code>/<code>fingerprint.bin</code> + <code>unitigs.bin</code>), and the
|
||
|
|
matrix side mmaps <code>matrix.pbmx</code>/<code>matrix.pcmx</code> (or one file per genome column
|
||
|
|
if not yet packed). Any code path that reopens a layer per lookup instead of
|
||
|
|
once per run pays this cost repeatedly.</p>
|
||
|
|
<p><code>obikphylo::siblings::cache::PartitionCache</code> was built to avoid exactly this
|
||
|
|
for <code>build_sibling_annex</code>/<code>sibling_annex_stats</code>: those commands probe many
|
||
|
|
partitions, once per source layer, over the whole run. Profiling a real run
|
||
|
|
showed wall-clock time dominated by repeated <code>open()</code>/mmap syscalls, not
|
||
|
|
computation — parallelising the naive per-lookup opens spread the cost
|
||
|
|
across cores without reducing it. <code>PartitionCache::build</code> opens every
|
||
|
|
partition's every layer once, up front, in parallel, and keeps the handles
|
||
|
|
alive for the run.</p>
|
||
|
|
<h2 id="three-independent-implementations-of-the-same-bundle-historical-1-fixed-this">Three independent implementations of the same bundle (historical — (1) fixed this)</h2>
|
||
|
|
<p><strong>As of 2026-08-20 this table describes the pre-(1) state.</strong> <code>Mat</code> no
|
||
|
|
longer exists (deleted when <code>obilayeredmap::Layer</code> replaced it — see "(1)
|
||
|
|
done" above); <code>Layer<D></code> in the table below is what's now called
|
||
|
|
<code>TypedLayer<D></code>. <code>QueryLayer</code> is unaffected and still stands as described —
|
||
|
|
still uncached, still not using <code>Layer</code> at all — which is exactly what (2)
|
||
|
|
needs to fix. Kept for the original motivation, not as current fact:</p>
|
||
|
|
<p>Searching the codebase for "who bundles MPHF + matrix, with per-layer format
|
||
|
|
auto-detection" turned up three unrelated implementations:</p>
|
||
|
|
<table>
|
||
|
|
<thead>
|
||
|
|
<tr>
|
||
|
|
<th></th>
|
||
|
|
<th>lives in</th>
|
||
|
|
<th>scope</th>
|
||
|
|
<th>cached?</th>
|
||
|
|
</tr>
|
||
|
|
</thead>
|
||
|
|
<tbody>
|
||
|
|
<tr>
|
||
|
|
<td><code>Layer<D></code> (now <code>TypedLayer<D></code>)</td>
|
||
|
|
<td><code>obilayeredmap</code></td>
|
||
|
|
<td>one layer, <code>D</code> fixed at compile time</td>
|
||
|
|
<td>held alive by whoever owns the <code>Layer</code>, no policy of its own</td>
|
||
|
|
</tr>
|
||
|
|
<tr>
|
||
|
|
<td><code>Mat</code> (now deleted; superseded by <code>obilayeredmap::Layer</code>)</td>
|
||
|
|
<td><code>obikphylo::siblings::cache</code></td>
|
||
|
|
<td>one layer, format resolved per instance from an enum of 3 <code>Layer<D></code> variants</td>
|
||
|
|
<td>yes, via <code>PartitionCache</code></td>
|
||
|
|
</tr>
|
||
|
|
<tr>
|
||
|
|
<td><code>QueryLayer</code> (unchanged, still current)</td>
|
||
|
|
<td><code>obikindex::query_layer</code> (moved crates since this was written — see "Major restructuring")</td>
|
||
|
|
<td>one layer, <code>(MphfLayer, PersistentBitMatrix\|PersistentCompactIntMatrix)</code> pair, bypasses <code>TypedLayer<D></code>/<code>Layer</code> entirely</td>
|
||
|
|
<td><strong>no</strong> — opened fresh inside <code>query_partition_with</code> on every call</td>
|
||
|
|
</tr>
|
||
|
|
</tbody>
|
||
|
|
</table>
|
||
|
|
<p><code>query_partition_with</code> is <code>obikmer query</code>'s normal query path — the one
|
||
|
|
most exposed to repeated cross-partition lookups — and it is still the one
|
||
|
|
with no cache at all. <code>obikphylo</code> built a cache first only because
|
||
|
|
sibling-annex construction hits the cost hardest, not because the need is
|
||
|
|
sibling-specific.</p>
|
||
|
|
<h2 id="the-gap-in-obilayeredmaps-existing-cache-historical-1-fixed-this">The gap in <code>obilayeredmap</code>'s existing cache (historical — (1) fixed this)</h2>
|
||
|
|
<p><code>obilayeredmap::LayeredMap<D></code> already caches correctly at the granularity
|
||
|
|
of one partition: <code>open(root)</code> opens every layer once, keeps
|
||
|
|
<code>Vec<TypedLayer<D>></code> alive for the <code>LayeredMap</code>'s lifetime. But it is
|
||
|
|
monomorphic — every layer in the <code>Vec</code> must share the same concrete <code>D</code>.
|
||
|
|
In practice this was false: layers in the same partition are packed
|
||
|
|
independently over time (<code>pack --sparse</code> converts one layer's presence
|
||
|
|
matrix at a time). This motivated (1) — <code>obilayeredmap::Layer</code>, done — but
|
||
|
|
note the specific <code>PersistentSparseBitMatrix</code>-mixing scenario described
|
||
|
|
here turned out to be moot: <code>PersistentBitMatrix</code> itself absorbed sparse
|
||
|
|
storage as a 4th internal variant before (1) was built (see "One bug found
|
||
|
|
… one earlier claim retracted" below), so the only heterogeneity <code>Layer</code>
|
||
|
|
actually needs to represent is <code>Count</code> vs. <code>Presence</code>, not dense-vs-sparse
|
||
|
|
presence. <code>LayeredMap<D></code> itself is unaffected by any of this — it's still
|
||
|
|
monomorphic, still not used by <code>Layer</code>/<code>KmerPartition</code> (which bypass it
|
||
|
|
entirely, opening each <code>TypedLayer<D></code> directly, the same way <code>Mat</code> did).</p>
|
||
|
|
<h2 id="resource-cost-mmap-does-not-hold-a-file-descriptor">Resource cost: mmap does not hold a file descriptor</h2>
|
||
|
|
<p>Before deciding how many layers/partitions a cache may hold open
|
||
|
|
simultaneously, the binding constraint needs to be identified correctly.</p>
|
||
|
|
<p>Confirmed against upstream documentation, not inferred from behaviour:</p>
|
||
|
|
<blockquote>
|
||
|
|
<p>"After the mmap() call has returned, the file descriptor, fd, can be
|
||
|
|
closed immediately without invalidating the mapping."
|
||
|
|
— <a href="https://man7.org/linux/man-pages/man2/mmap.2.html">mmap(2), man7.org</a></p>
|
||
|
|
<p>"The close(2) function does not unmap pages"
|
||
|
|
— <a href="https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man2/mmap.2.html">mmap(2), Apple Developer</a></p>
|
||
|
|
<p>"A file backed Mmap ... will remain valid even after the File is dropped.
|
||
|
|
... the Mmap handle is completely independent of the File used to create
|
||
|
|
it."
|
||
|
|
— <a href="https://docs.rs/memmap2/latest/memmap2/struct.Mmap.html">memmap2::Mmap, docs.rs</a></p>
|
||
|
|
</blockquote>
|
||
|
|
<p>Every read-only mmap in this codebase already follows this: <code>Mmap::map(&File::open(path)?)?</code>
|
||
|
|
— the <code>File</code> is a temporary, dropped (fd closed) immediately after the
|
||
|
|
mapping is established; every persistent struct (<code>PersistentBitVec</code>,
|
||
|
|
<code>PersistentCompactIntVec</code>, <code>PackedBitMatrix</code>, <code>Evidence</code>, <code>FingerprintVec</code>,
|
||
|
|
...) stores only the <code>Mmap</code>, never the <code>File</code>. So a cache built on these
|
||
|
|
types does <strong>not</strong> consume the process's open-file-descriptor budget
|
||
|
|
(<code>ulimit -n</code>, notoriously low by default on macOS) proportionally to how
|
||
|
|
many mmapped files it holds.</p>
|
||
|
|
<p>It does consume a different resource — the process's virtual-memory mapping
|
||
|
|
table (one entry per active <code>mmap()</code> region). Linux exposes this as
|
||
|
|
<code>vm.max_map_count</code> (default 65530). No documented macOS equivalent (fixed
|
||
|
|
numeric ceiling) was found; the constraint there appears to be virtual
|
||
|
|
address space rather than an explicit mapping counter, but this is not
|
||
|
|
sourced and should not be assumed. This is the resource actually worth
|
||
|
|
measuring before deciding on cache size, not fd count — and it is why
|
||
|
|
<em>packing</em> (<code>matrix.pbmx</code>/<code>matrix.pcmx</code>, one mmap for all columns) matters
|
||
|
|
independently of any caching decision: an unpacked <code>Columnar</code> matrix opens
|
||
|
|
one mmap <strong>per genome column</strong>, multiplying the mapping count a cache would
|
||
|
|
have to hold by <code>n_genomes</code>.</p>
|
||
|
|
<h2 id="layering-who-owns-what-superseded-see-definitions-above">Layering: who owns what (superseded — see Definitions above)</h2>
|
||
|
|
<p>This section used to argue nobody owned "the collection of partitions."
|
||
|
|
That's resolved: <code>KmerIndex</code> (<code>obikindex</code>) owns it now, directly (see
|
||
|
|
"Major restructuring" below). What's still genuinely unowned is <em>one
|
||
|
|
partition's open layers</em> — <code>KmerPartition</code>, in the not-yet-created
|
||
|
|
<code>obikpartition</code> — see "Definitions" at the top of this file for the
|
||
|
|
current, authoritative answer. Left here only so old links/references to
|
||
|
|
this heading don't 404; don't read this section for current facts.</p>
|
||
|
|
<h2 id="direction-agreed-not-yet-implemented">Direction agreed, not yet implemented</h2>
|
||
|
|
<p>Only (2) remains — (1) shipped as <code>obilayeredmap::Layer</code> (see "(1) done"
|
||
|
|
above). Concretely, in order:</p>
|
||
|
|
<ol>
|
||
|
|
<li>Create the <code>obikpartition</code> crate (<code>obikindex → obikpartition →
|
||
|
|
obilayeredmap</code>, no other edges — see "Definitions" above for the exact
|
||
|
|
constraint and why).</li>
|
||
|
|
<li><code>KmerPartition { layers: Vec<obilayeredmap::Layer> }</code> — <code>open</code>,
|
||
|
|
<code>n_layers</code>, <code>layer(i)</code>, <code>find</code>, plus whatever batch-lookup surface
|
||
|
|
<code>obikphylo::siblings::cache::PartitionCache</code> currently needs
|
||
|
|
(<code>find_presence_batch</code>/<code>find_presence_batch_fast</code>; <code>fast_mode</code> is
|
||
|
|
sibling-specific bookkeeping and should probably stay in <code>obikphylo</code>,
|
||
|
|
wrapping a <code>KmerPartition</code>/<code>Vec<KmerPartition></code> rather than living
|
||
|
|
inside it — same "generic vs. domain-specific" split <code>iter_minorants_batch</code>
|
||
|
|
already went through for <code>Layer</code> in (1)).</li>
|
||
|
|
<li>Migrate <code>obikphylo::siblings::cache::PartitionCache</code> to hold
|
||
|
|
<code>Vec<KmerPartition></code> instead of <code>Vec<Vec<Layer>></code>.</li>
|
||
|
|
<li>Migrate <code>obikindex::query_layer::QueryLayer</code>/<code>query_partition_with</code> to
|
||
|
|
use <code>KmerPartition</code> too, closing the "no cache at all" gap on
|
||
|
|
<code>obikmer</code>'s normal query path (see "Three independent implementations,"
|
||
|
|
historical, above).</li>
|
||
|
|
</ol>
|
||
|
|
<p>Open before implementing: exact API shape of <code>KmerPartition</code> (propose,
|
||
|
|
confirm before coding — non-trivial), and whether the multi-partition
|
||
|
|
<code>Vec<KmerPartition></code> needs an eviction policy or can simply hold every
|
||
|
|
partition open for the process lifetime (revisit once the
|
||
|
|
VM-mapping-count question above has a real number behind it for this
|
||
|
|
codebase's scale — still not measured).</p>
|
||
|
|
<h2 id="preparatory-work-done-2026-08-20">Preparatory work done (2026-08-20)</h2>
|
||
|
|
<p>Groundwork for (1)/(2), landed ahead of the design itself. <strong>Note</strong>: at
|
||
|
|
the time this was written, the collection type these bullets describe was
|
||
|
|
named <code>KmerPartition</code> (singular) in this doc; it was renamed
|
||
|
|
<code>KmerPartitions</code> (plural) shortly after, then deleted entirely and merged
|
||
|
|
into <code>KmerIndex</code> (see "Major restructuring" above). The bullets below are
|
||
|
|
edited to say <code>KmerPartitions</code> throughout, to not collide with the
|
||
|
|
unrelated, brand-new singular <code>KmerPartition</code> defined at the top of this
|
||
|
|
file — the accessors described here live on <code>KmerIndex</code> today, not on
|
||
|
|
any type called <code>KmerPartition</code>.</p>
|
||
|
|
<ul>
|
||
|
|
<li><code>KmerPartitions</code> (<code>obikpartitionner</code>, at the time) gained
|
||
|
|
<code>partition_dir</code>/<code>index_dir</code>/<code>layer_dir</code> as the single source of truth
|
||
|
|
for a partition's on-disk layout, replacing per-module duplicated
|
||
|
|
<code>const INDEX_SUBDIR: &str = "index"</code> (7 copies) and ad hoc path joins —
|
||
|
|
including one found duplicated <em>inside the struct itself</em>
|
||
|
|
(<code>ensure_writer</code> rebuilt <code>part_dir</code>'s own logic by hand).</li>
|
||
|
|
<li>Same struct gained <code>partition_meta</code>/<code>n_layers</code>/<code>index_mode</code>, wrapping
|
||
|
|
<code>obilayeredmap::meta::PartitionMeta::load</code> (via the existing
|
||
|
|
<code>common::load_meta</code>, which also recovers indexes built before
|
||
|
|
<code>meta.json</code> existed). Before this, <code>obikphylo</code> and <code>obikindex</code> imported
|
||
|
|
<code>obilayeredmap::meta::PartitionMeta</code> directly and called <code>::load()</code>
|
||
|
|
themselves at 21 call sites, each redoing its own error-mapping —
|
||
|
|
every one of those crates knew the on-disk metadata format instead of
|
||
|
|
going through an interface. Fixed everywhere except one remaining spot
|
||
|
|
(below). Caught as a side effect: <code>dump_layer.rs</code>/<code>query_layer.rs</code> had
|
||
|
|
been calling <code>PartitionMeta::load</code> directly, bypassing <code>load_meta</code>
|
||
|
|
entirely — they never got the missing-<code>meta.json</code> recovery the other
|
||
|
|
callers did.</li>
|
||
|
|
<li>Layer introspection API discussed but <strong>not yet implemented</strong> — three
|
||
|
|
axes, deliberately kept separate after an initial draft conflated them:</li>
|
||
|
|
<li><code>LayerContent { Count, Presence }</code> — what the layer stores; a <code>const</code>
|
||
|
|
on <code>LayerData</code> (compile-time, zero-cost), not a runtime field.</li>
|
||
|
|
<li><code>StorageKind { Implicit, Columnar, Packed, Sparse }</code> — how it's
|
||
|
|
stored; only meaningful for <code>D</code> that actually carry data (<code>Layer<()></code>
|
||
|
|
has neither this nor <code>LayerContent</code> — it's a write-time-only state,
|
||
|
|
never a queryable content: once a layer is closed, "no matrix file"
|
||
|
|
reads back as <code>Presence</code>/<code>Implicit</code> via <code>PersistentBitMatrix::open</code>'s
|
||
|
|
own fallback, not as some third "empty" content).</li>
|
||
|
|
<li><code>EvidenceKind { Exact, Approx, Hybrid }</code> — from <code>MphfLayer</code>'s own
|
||
|
|
already-in-memory <code>LayerEvidence</code> discriminant.</li>
|
||
|
|
<li>Not all <code>(LayerContent, StorageKind)</code> pairs are legal: <code>Count</code> never
|
||
|
|
has <code>Implicit</code> or <code>Sparse</code>.</li>
|
||
|
|
</ul>
|
||
|
|
<p><strong>Implemented (2026-08-20).</strong> <code>LayerContent</code>/<code>StorageKind</code>/<code>EvidenceKind</code>
|
||
|
|
now exist, each with two forms:
|
||
|
|
- A runtime accessor on an already-open value (<code>Layer<D>::content()</code>/
|
||
|
|
<code>storage_kind()</code>/<code>evidence_kind()</code>, <code>PersistentBitMatrix::storage_kind()</code>,
|
||
|
|
<code>PersistentCompactIntMatrix::storage_kind()</code>, <code>MphfLayer::evidence_kind()</code>)
|
||
|
|
— reads a discriminant already in memory, zero disk access.
|
||
|
|
- A lightweight <code>detect()</code>/<code>detect_storage()</code> disk probe that mirrors the
|
||
|
|
corresponding <code>open()</code>'s own priority order by hand (file-existence
|
||
|
|
checks only, no mmap) — usable <em>before</em> committing to a <code>D</code>, unlike the
|
||
|
|
runtime accessors. Exposed per-layer on <code>LayeredMap<D></code> as
|
||
|
|
<code>detect_layer_content</code>/<code>detect_layer_storage</code>/<code>detect_layer_evidence</code>
|
||
|
|
(work regardless of <code>D</code>, since they only use <code>self.root</code> + the layer
|
||
|
|
index).</p>
|
||
|
|
<p><code>StorageKind</code> lives in <code>obicompactvec</code> (owner of <code>PersistentBitMatrix</code>/
|
||
|
|
<code>PersistentCompactIntMatrix</code>); <code>LayerContent</code>/<code>EvidenceKind</code> live in
|
||
|
|
<code>obilayeredmap</code>. <code>HasLayerContent</code>/<code>HasStorageKind</code> gate <code>Layer<()></code> out of
|
||
|
|
<code>content()</code>/<code>storage_kind()</code> (no matrix, nothing to report), matching the
|
||
|
|
"empty is transitional" conclusion above. 42 new tests across
|
||
|
|
<code>obilayeredmap</code>'s <code>tests/layer.rs</code> and <code>tests/map.rs</code>; full workspace
|
||
|
|
suite green (0 failed) after.</p>
|
||
|
|
<p>Not done: these <code>detect()</code> probes don't yet replace <code>Mat::open</code>'s or
|
||
|
|
<code>QueryLayer::open</code>'s own hand-rolled equivalents (still duplicated content/
|
||
|
|
storage decisions, now a <em>third</em> copy of the same logic to keep in sync)
|
||
|
|
— that consolidation is (1)/(2)'s job, not this prep step's.</p>
|
||
|
|
<h2 id="one-bug-found-while-reading-around-this-signalled-not-fixed-one-earlier-claim-retracted">One bug found while reading around this (signalled, not fixed); one earlier claim retracted</h2>
|
||
|
|
<ul>
|
||
|
|
<li><code>obicompactvec::bitmatrix::sparse.rs</code>'s module doc says
|
||
|
|
"Not used by any production code path yet" — false since <code>obikmer pack
|
||
|
|
--sparse</code> (<code>cmd/pack/mod.rs</code>) is wired to <code>pack_sparse_bit_matrix</code> and
|
||
|
|
<code>Mat::open</code> already reads the result back in the sibling-annex path.
|
||
|
|
Stale comment, not corrected.</li>
|
||
|
|
<li><strong>Retracted (2026-08-20)</strong>: an earlier pass through this doc claimed
|
||
|
|
<code>obikpartition::query_layer::QueryLayer::open</code> had no sparse-format
|
||
|
|
detection and would silently corrupt reads on a <code>pack --sparse</code>d layer.
|
||
|
|
False — <code>PersistentBitMatrix</code> (<code>obicompactvec::bitmatrix::persistent</code>)
|
||
|
|
is a 4-way enum (<code>Columnar</code>/<code>Packed</code>/<code>Sparse</code>/<code>Implicit</code>), not 3-way as
|
||
|
|
first read; its <code>open()</code> already detects <code>Sparse</code> via
|
||
|
|
<code>presence/sparse_meta.json</code>, and every method on the type (<code>row</code>,
|
||
|
|
<code>fill_row</code>, <code>nonzero_iter</code>, …) already dispatches all 4 arms.
|
||
|
|
<code>QueryLayer::open</code>'s <code>PersistentBitMatrix::open(layer_dir)</code> call was
|
||
|
|
never the bug. Root cause of the false claim: a <code>grep -n
|
||
|
|
"Implicit\|Columnar\|Packed"</code> used to read the enum definition silently
|
||
|
|
skipped the <code>Sparse(...)</code> line because it matched none of those three
|
||
|
|
words — a self-inflicted blind spot from a filtered read, not a fact
|
||
|
|
about the code. Lesson: for a <code>pub enum</code> whose variant list matters,
|
||
|
|
read the definition unfiltered, don't grep for the variant names you
|
||
|
|
expect to find.</li>
|
||
|
|
<li>One real consequence of that same correction, <strong>fixed (2026-08-20)</strong>:
|
||
|
|
<code>obikphylo::siblings::cache::Mat::SparsePresence(Layer<
|
||
|
|
PersistentSparseBitMatrix>)</code> was redundant — <code>Mat::Presence(Layer<
|
||
|
|
PersistentBitMatrix>)</code> alone already handles sparse layers
|
||
|
|
transparently, since <code>PersistentBitMatrix</code> absorbs <code>Sparse</code> internally.
|
||
|
|
Removed the variant, the <code>presence/is_multi.prsb</code> probe in <code>Mat::open</code>
|
||
|
|
(now just opens <code>Layer::<PersistentBitMatrix></code> unconditionally for the
|
||
|
|
non-count case — sparse-vs-dense is <code>PersistentBitMatrix::open</code>'s own
|
||
|
|
concern), and every now-single-armed match in <code>find_slot</code>/<code>index_batch</code>/
|
||
|
|
<code>iter_minorants_batch</code>/<code>n_cols</code>/<code>fill_sub_matrix_carries</code>. Full
|
||
|
|
workspace test suite green after, including the 27 <code>obikphylo::siblings</code>
|
||
|
|
tests that exercise <code>pack_matrices(true)</code>/sparse through <code>Mat</code>.</li>
|
||
|
|
</ul>
|
||
|
|
<h2 id="remaining-instance-of-the-partitionmeta-encapsulation-problem">Remaining instance of the PartitionMeta-encapsulation problem</h2>
|
||
|
|
<p><code>obikphylo::siblings::family_scan::scan_layer_families</code> still re-derives
|
||
|
|
<code>index_dir</code> from <code>layer_dir.parent()</code> and calls <code>PartitionMeta::load</code>
|
||
|
|
itself, purely to get <code>.mode</code> for <code>obilayeredmap::Layer::open</code> (was
|
||
|
|
<code>Mat::open</code>, same gap, survived the <code>Mat</code> → <code>Layer</code> swap in (1) unchanged).
|
||
|
|
Fixing it the way the 21 other call sites were fixed needs more than a 1:1
|
||
|
|
swap: <code>scan_layer_families</code> only receives a bare <code>layer_dir: &Path</code>, not a
|
||
|
|
<code>(partition, part, layer)</code> triple, and its single upstream source of layer
|
||
|
|
paths, <code>sibling_layer_dirs</code>, returns a flat <code>Vec<PathBuf></code> with the
|
||
|
|
partition/layer indices already discarded. Fixing it properly means either
|
||
|
|
having <code>sibling_layer_dirs</code> return <code>(PathBuf, IndexMode)</code> (or <code>(part,
|
||
|
|
layer)</code>) pairs, or threading <code>&KmerIndex</code> + indices through instead of
|
||
|
|
paths (not <code>&KmerPartition</code> — that type doesn't exist yet, and once it
|
||
|
|
does it still won't know <code>IndexMode</code>, which lives on <code>KmerIndex</code>/
|
||
|
|
<code>PartitionMeta</code>) — and touching every one of <code>scan_layer_families</code>'s 8
|
||
|
|
callers (<code>distance.rs</code>, <code>alignment.rs</code>, <code>cardinality.rs</code>, <code>entropy.rs</code> ×2,
|
||
|
|
<code>sankoff_bundle.rs</code> ×2, <code>stats.rs</code>). Left alone this round; worth doing as
|
||
|
|
part of the same pass that builds <code>KmerPartition</code>, since those callers are
|
||
|
|
exactly the sibling-annex consumers it's meant to serve.</p>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
</article>
|
||
|
|
</div>
|
||
|
|
|
||
|
|
|
||
|
|
<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
|
||
|
|
</div>
|
||
|
|
|
||
|
|
</main>
|
||
|
|
|
||
|
|
<footer class="md-footer">
|
||
|
|
|
||
|
|
<div class="md-footer-meta md-typeset">
|
||
|
|
<div class="md-footer-meta__inner md-grid">
|
||
|
|
<div class="md-copyright">
|
||
|
|
|
||
|
|
|
||
|
|
Made with
|
||
|
|
<a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
|
||
|
|
Material for MkDocs
|
||
|
|
</a>
|
||
|
|
|
||
|
|
</div>
|
||
|
|
|
||
|
|
</div>
|
||
|
|
</div>
|
||
|
|
</footer>
|
||
|
|
|
||
|
|
</div>
|
||
|
|
<div class="md-dialog" data-md-component="dialog">
|
||
|
|
<div class="md-dialog__inner md-typeset"></div>
|
||
|
|
</div>
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
<script id="__config" type="application/json">{"annotate": null, "base": "../..", "features": [], "search": "../../assets/javascripts/workers/search.2c215733.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
|
||
|
|
|
||
|
|
|
||
|
|
<script src="../../assets/javascripts/bundle.79ae519e.min.js"></script>
|
||
|
|
|
||
|
|
<script src="https://unpkg.com/mathjax@3/es5/tex-mml-chtml.js"></script>
|
||
|
|
|
||
|
|
|
||
|
|
</body>
|
||
|
|
</html>
|