OBIKmers/obikmer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 27 Wiki Activity
Files
66ab1d0947136c1480e4d35c61d31ba62df7cd79
obikmer/doc/index.html
T
Eric Coissac 7bac0f3850 large refactoring
2026-08-17 09:28:53 +02:00

1449 lines
26 KiB
HTML
Raw Blame History

<!doctype html>
<html lang="en" class="no-js">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<link rel="next" href="installation/">
<link rel="icon" href="assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.7.6">
<title>obikmer — User Guide</title>
<link rel="stylesheet" href="assets/stylesheets/main.484c7ddc.min.css">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
<style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
<script>__md_scope=new URL(".",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
</head>
<body dir="ltr">
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
<label class="md-overlay" for="__drawer"></label>
<div data-md-component="skip">
<a href="#obikmer" class="md-skip">
Skip to content
</a>
</div>
<div data-md-component="announce">
</div>
<header class="md-header md-header--shadow" data-md-component="header">
<nav class="md-header__inner md-grid" aria-label="Header">
<a href="." title="obikmer — User Guide" class="md-header__button md-logo" aria-label="obikmer — User Guide" data-md-component="logo">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
</a>
<label class="md-header__button md-icon" for="__drawer">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
</label>
<div class="md-header__title" data-md-component="header-title">
<div class="md-header__ellipsis">
<div class="md-header__topic">
<span class="md-ellipsis">
obikmer — User Guide
</span>
</div>
<div class="md-header__topic" data-md-component="header-topic">
<span class="md-ellipsis">
Home
</span>
</div>
</div>
</div>
<script>var palette=__md_get("__palette");if(palette&&palette.color){if("(prefers-color-scheme)"===palette.color.media){var media=matchMedia("(prefers-color-scheme: light)"),input=document.querySelector(media.matches?"[data-md-color-media='(prefers-color-scheme: light)']":"[data-md-color-media='(prefers-color-scheme: dark)']");palette.color.media=input.getAttribute("data-md-color-media"),palette.color.scheme=input.getAttribute("data-md-color-scheme"),palette.color.primary=input.getAttribute("data-md-color-primary"),palette.color.accent=input.getAttribute("data-md-color-accent")}for(var[key,value]of Object.entries(palette.color))document.body.setAttribute("data-md-color-"+key,value)}</script>
</nav>
</header>
<div class="md-container" data-md-component="container">
<main class="md-main" data-md-component="main">
<div class="md-main__inner md-grid">
<div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
<label class="md-nav__title" for="__drawer">
<a href="." title="obikmer — User Guide" class="md-nav__button md-logo" aria-label="obikmer — User Guide" data-md-component="logo">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
</a>
obikmer — User Guide
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item md-nav__item--active">
<input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
<label class="md-nav__link md-nav__link--active" for="__toc">
<span class="md-ellipsis">
Home
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<a href="." class="md-nav__link md-nav__link--active">
<span class="md-ellipsis">
Home
</span>
</a>
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
Table of contents
</label>
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
<li class="md-nav__item">
<a href="#core-principles" class="md-nav__link">
<span class="md-ellipsis">
Core principles
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#commands" class="md-nav__link">
<span class="md-ellipsis">
Commands
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#further-reading" class="md-nav__link">
<span class="md-ellipsis">
Further reading
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#input-formats" class="md-nav__link">
<span class="md-ellipsis">
Input formats
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#parameter-constraints" class="md-nav__link">
<span class="md-ellipsis">
Parameter constraints
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#genome-label-constraints" class="md-nav__link">
<span class="md-ellipsis">
Genome label constraints
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="installation/" class="md-nav__link">
<span class="md-ellipsis">
Installation
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3" >
<label class="md-nav__link" for="__nav_3" id="__nav_3_label" tabindex="0">
<span class="md-ellipsis">
Theory
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_3_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_3">
<span class="md-nav__icon md-icon"></span>
Theory
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="theory/kmers_and_superkmers/" class="md-nav__link">
<span class="md-ellipsis">
Kmers and super-kmers
</span>
</a>
</li>
<li class="md-nav__item">
<a href="theory/encoding/" class="md-nav__link">
<span class="md-ellipsis">
DNA encoding
</span>
</a>
</li>
<li class="md-nav__item">
<a href="theory/entropy_filter/" class="md-nav__link">
<span class="md-ellipsis">
Low-complexity kmer filter
</span>
</a>
</li>
<li class="md-nav__item">
<a href="theory/minimizer_selection/" class="md-nav__link">
<span class="md-ellipsis">
Minimizer selection
</span>
</a>
</li>
<li class="md-nav__item">
<a href="theory/indexing_architecture/" class="md-nav__link">
<span class="md-ellipsis">
Partitioning and indexing architecture
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4" >
<label class="md-nav__link" for="__nav_4" id="__nav_4_label" tabindex="0">
<span class="md-ellipsis">
Usage
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_4_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_4">
<span class="md-nav__icon md-icon"></span>
Usage
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="usage/superkmer/" class="md-nav__link">
<span class="md-ellipsis">
superkmer
</span>
</a>
</li>
<li class="md-nav__item">
<a href="usage/index_command/" class="md-nav__link">
<span class="md-ellipsis">
index
</span>
</a>
</li>
<li class="md-nav__item">
<a href="usage/merge/" class="md-nav__link">
<span class="md-ellipsis">
merge
</span>
</a>
</li>
<li class="md-nav__item">
<a href="usage/filter/" class="md-nav__link">
<span class="md-ellipsis">
filter
</span>
</a>
</li>
<li class="md-nav__item">
<a href="usage/select/" class="md-nav__link">
<span class="md-ellipsis">
select
</span>
</a>
</li>
<li class="md-nav__item">
<a href="usage/query/" class="md-nav__link">
<span class="md-ellipsis">
query
</span>
</a>
</li>
<li class="md-nav__item">
<a href="usage/dump/" class="md-nav__link">
<span class="md-ellipsis">
dump
</span>
</a>
</li>
<li class="md-nav__item">
<a href="usage/annotate/" class="md-nav__link">
<span class="md-ellipsis">
annotate
</span>
</a>
</li>
<li class="md-nav__item">
<a href="usage/phylo/" class="md-nav__link">
<span class="md-ellipsis">
phylo
</span>
</a>
</li>
<li class="md-nav__item">
<a href="usage/name-tree/" class="md-nav__link">
<span class="md-ellipsis">
name-tree
</span>
</a>
</li>
<li class="md-nav__item">
<a href="usage/unitig/" class="md-nav__link">
<span class="md-ellipsis">
unitig
</span>
</a>
</li>
<li class="md-nav__item">
<a href="usage/estimate/" class="md-nav__link">
<span class="md-ellipsis">
estimate
</span>
</a>
</li>
<li class="md-nav__item">
<a href="usage/reindex/" class="md-nav__link">
<span class="md-ellipsis">
reindex
</span>
</a>
</li>
<li class="md-nav__item">
<a href="usage/utils/" class="md-nav__link">
<span class="md-ellipsis">
utils
</span>
</a>
</li>
<li class="md-nav__item">
<a href="usage/pack/" class="md-nav__link">
<span class="md-ellipsis">
pack
</span>
</a>
</li>
<li class="md-nav__item">
<a href="usage/predicates/" class="md-nav__link">
<span class="md-ellipsis">
Predicates and taxonomy paths
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_5" >
<label class="md-nav__link" for="__nav_5" id="__nav_5_label" tabindex="0">
<span class="md-ellipsis">
Formats
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_5_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_5">
<span class="md-nav__icon md-icon"></span>
Formats
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="formats/index_layout/" class="md-nav__link">
<span class="md-ellipsis">
Index construction and on-disk layout
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="architecture/" class="md-nav__link">
<span class="md-ellipsis">
Architecture notes
</span>
</a>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
Table of contents
</label>
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
<li class="md-nav__item">
<a href="#core-principles" class="md-nav__link">
<span class="md-ellipsis">
Core principles
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#commands" class="md-nav__link">
<span class="md-ellipsis">
Commands
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#further-reading" class="md-nav__link">
<span class="md-ellipsis">
Further reading
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#input-formats" class="md-nav__link">
<span class="md-ellipsis">
Input formats
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#parameter-constraints" class="md-nav__link">
<span class="md-ellipsis">
Parameter constraints
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#genome-label-constraints" class="md-nav__link">
<span class="md-ellipsis">
Genome label constraints
</span>
</a>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-content" data-md-component="content">
<article class="md-content__inner md-typeset">
<h1 id="obikmer">obikmer</h1>
<p><code>obikmer</code> is a command-line tool for counting, indexing, querying and comparing DNA sequences represented as kmer sets. It targets individual genome datasets of tens of gigabases, with an emphasis on computational, memory, and disk efficiency.</p>
<p>All functionality is exposed through a single binary, <code>obikmer</code>, organized as subcommands.</p>
<h2 id="core-principles">Core principles</h2>
<ul>
<li>Kmers are of fixed, odd length <span class="arithmatex">\(k\)</span>, chosen at index-construction time in the range <span class="arithmatex">\([11, 31]\)</span> (see <a href="theory/kmers_and_superkmers/">Kmers and super-kmers</a>).</li>
<li>Each kmer fits in a 64-bit word using a 2-bit-per-base encoding (see <a href="theory/encoding/">DNA encoding</a>).</li>
<li>Kmers are handled in <strong>canonical form</strong> (<span class="arithmatex">\(\text{canonical}(kmer) = \min(kmer, \text{revcomp}(kmer))\)</span>), making counting strand-independent.</li>
<li>Sequences are decomposed into <strong>super-kmers</strong> before storage, anchored on a hash-selected <strong>minimizer</strong> (see <a href="theory/minimizer_selection/">Minimizer selection</a>), then routed to one of several <strong>partitions</strong> for parallel, memory-bounded processing (see <a href="theory/indexing_architecture/">Partitioning and indexing architecture</a>).</li>
<li>Low-complexity kmers can be filtered out at index-construction time using an entropy-based score (see <a href="theory/entropy_filter/">Low-complexity kmer filter</a>).</li>
</ul>
<h2 id="commands">Commands</h2>
<table>
<thead>
<tr>
<th>Command</th>
<th>Purpose</th>
</tr>
</thead>
<tbody>
<tr>
<td><a href="usage/superkmer/"><code>superkmer</code></a></td>
<td>Extract super-kmers from a sequence file and write them to stdout</td>
</tr>
<tr>
<td><a href="usage/index_command/"><code>index</code></a></td>
<td>Build a genome index</td>
</tr>
<tr>
<td><a href="usage/merge/"><code>merge</code></a></td>
<td>Merge multiple indexes into one</td>
</tr>
<tr>
<td><a href="usage/filter/"><code>filter</code></a></td>
<td>Retain only kmers matching ingroup/outgroup predicates</td>
</tr>
<tr>
<td><a href="usage/select/"><code>select</code></a></td>
<td>Project and/or aggregate genome columns of an index</td>
</tr>
<tr>
<td><a href="usage/query/"><code>query</code></a></td>
<td>Query an index with sequences and annotate matches</td>
</tr>
<tr>
<td><a href="usage/dump/"><code>dump</code></a></td>
<td>Dump indexed kmers as CSV</td>
</tr>
<tr>
<td><a href="usage/annotate/"><code>annotate</code></a></td>
<td>Add, update, or dump genome metadata</td>
</tr>
<tr>
<td><a href="usage/phylo/"><code>phylo</code></a></td>
<td>Compute pairwise evolutionary-distance proxies, trees, and phylogenetic exports</td>
</tr>
<tr>
<td><a href="usage/name-tree/"><code>name-tree</code></a></td>
<td>Translate a TNT/PhyG numeric-label tree export back to real taxon names</td>
</tr>
<tr>
<td><a href="usage/unitig/"><code>unitig</code></a></td>
<td>Dump the unitigs of an index as FASTA</td>
</tr>
<tr>
<td><a href="usage/estimate/"><code>estimate</code></a></td>
<td>Estimate approximate-index parameters before indexing</td>
</tr>
<tr>
<td><a href="usage/reindex/"><code>reindex</code></a></td>
<td>Convert an index's evidence representation (exact ↔ approximate)</td>
</tr>
<tr>
<td><a href="usage/utils/"><code>utils</code></a></td>
<td>Miscellaneous index maintenance and inspection utilities</td>
</tr>
<tr>
<td><a href="usage/pack/"><code>pack</code></a></td>
<td>Pack per-column matrix files into a single-file format</td>
</tr>
</tbody>
</table>
<p>See <a href="usage/predicates/">Genome predicates and taxonomy paths</a> for the selection language shared by <code>filter</code>, <code>select</code>, <code>dump</code>, and <code>unitig</code>.</p>
<h2 id="further-reading">Further reading</h2>
<ul>
<li><a href="formats/index_layout/">Index construction and on-disk layout</a></li>
<li><a href="architecture/">Architecture notes for advanced use</a> — parallel execution, NUMA awareness, index dimensioning</li>
</ul>
<h2 id="input-formats">Input formats</h2>
<ul>
<li><code>superkmer</code> and <code>index</code>: FASTA (<code>.fa</code>, <code>.fasta</code>), FASTQ (<code>.fq</code>, <code>.fastq</code>), GenBank flat file (<code>.gb</code>, <code>.gbk</code>, <code>.gbff</code>), all optionally gzip-compressed; directories are expanded recursively; streaming stdin via <code>-</code> or when no input path is given.</li>
<li><code>query</code>: FASTA or FASTQ, optionally gzip-compressed; streaming stdin the same way.</li>
</ul>
<h2 id="parameter-constraints">Parameter constraints</h2>
<p>These constraints are checked at startup; an invalid value exits immediately with an error.</p>
<table>
<thead>
<tr>
<th>Parameter</th>
<th>Constraint</th>
<th>Reason</th>
</tr>
</thead>
<tbody>
<tr>
<td><span class="arithmatex">\(k\)</span> (<code>--kmer-size</code>)</td>
<td>odd, <span class="arithmatex">\(k \in [11, 31]\)</span></td>
<td>odd length guarantees the canonical form is always well defined; the range keeps a kmer within a 64-bit word while retaining specificity</td>
</tr>
<tr>
<td><span class="arithmatex">\(m\)</span> (<code>--minimizer-size</code>)</td>
<td>odd, <span class="arithmatex">\(3 \le m \le k-1\)</span></td>
<td>same palindrome argument as <span class="arithmatex">\(k\)</span>; must be strictly shorter than the kmer</td>
</tr>
<tr>
<td><span class="arithmatex">\(z\)</span> (<code>-z</code>, approximate evidence only)</td>
<td><span class="arithmatex">\(z \le k-1\)</span></td>
<td>the effective indexed kmer size is <span class="arithmatex">\(k-z+1\)</span></td>
</tr>
</tbody>
</table>
<h2 id="genome-label-constraints">Genome label constraints</h2>
<p>Genome labels are arbitrary Unicode strings, with the following restrictions:</p>
<table>
<thead>
<tr>
<th>Character</th>
<th>Forbidden</th>
<th>Reason</th>
</tr>
</thead>
<tbody>
<tr>
<td><code>/</code></td>
<td>yes</td>
<td>filesystem path separator</td>
</tr>
<tr>
<td><code>=</code></td>
<td>yes</td>
<td>separator used by <code>--new-label</code></td>
</tr>
<tr>
<td><code>\0</code></td>
<td>yes</td>
<td>null byte</td>
</tr>
<tr>
<td><code>\n</code>, <code>\r</code>, <code>\t</code></td>
<td>yes</td>
<td>would break CSV output</td>
</tr>
<tr>
<td>spaces</td>
<td>allowed</td>
<td>quote in the shell, e.g. <code>--new-label 'new label=old label'</code></td>
</tr>
</tbody>
</table>
<p>Empty labels are rejected. A label derived automatically from the input file name (when <code>--label</code> is omitted) is not validated, since it is already filesystem-safe.</p>
</article>
</div>
<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
</div>
</main>
<footer class="md-footer">
<div class="md-footer-meta md-typeset">
<div class="md-footer-meta__inner md-grid">
<div class="md-copyright">
Made with
<a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
Material for MkDocs
</a>
</div>
</div>
</div>
</footer>
</div>
<div class="md-dialog" data-md-component="dialog">
<div class="md-dialog__inner md-typeset"></div>
</div>
<script id="__config" type="application/json">{"annotate": null, "base": ".", "features": [], "search": "assets/javascripts/workers/search.2c215733.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
<script src="assets/javascripts/bundle.79ae519e.min.js"></script>
<script src="https://unpkg.com/mathjax@3/es5/tex-mml-chtml.js"></script>
</body>
</html>
Reference in New Issue View Git Blame Copy Permalink