Eric Coissac
579cfd8752
Update documentation to reflect architectural fixes in the NUMA partition runner and implementation changes in the merge and select modules. The partition runner now wraps task execution in `catch_unwind`, emitting panic events that count toward completion thresholds to prevent controller deadlocks, while re-raising the first payload on return. CLI documentation for select has been updated to mandate output directories, introduce default hard-linking with automatic fallback, and document presence aggregation optimizations. Known issues such as self-merge crashes and sparse-packed index gaps are now documented as failing fast due to these improvements.
2040 lines
43 KiB
HTML
2040 lines
43 KiB
HTML
|
|
<!doctype html>
|
|
<html lang="en" class="no-js">
|
|
<head>
|
|
|
|
<meta charset="utf-8">
|
|
<meta name="viewport" content="width=device-width,initial-scale=1">
|
|
|
|
|
|
|
|
|
|
<link rel="prev" href="../filtering/">
|
|
|
|
|
|
<link rel="next" href="../obitaxonomy/">
|
|
|
|
|
|
|
|
|
|
|
|
<link rel="icon" href="../../assets/images/favicon.png">
|
|
<meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.7.6">
|
|
|
|
|
|
|
|
<title>Select command - obikmer</title>
|
|
|
|
|
|
|
|
<link rel="stylesheet" href="../../assets/stylesheets/main.484c7ddc.min.css">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
|
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
|
|
<style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
|
|
|
|
|
|
|
|
<script>__md_scope=new URL("../..",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
|
|
|
|
|
|
|
|
|
|
|
|
</head>
|
|
|
|
|
|
<body dir="ltr">
|
|
|
|
|
|
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
|
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
|
<label class="md-overlay" for="__drawer"></label>
|
|
<div data-md-component="skip">
|
|
|
|
|
|
<a href="#select-column-projection-and-aggregation" class="md-skip">
|
|
Skip to content
|
|
</a>
|
|
|
|
</div>
|
|
<div data-md-component="announce">
|
|
|
|
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<header class="md-header md-header--shadow" data-md-component="header">
|
|
<nav class="md-header__inner md-grid" aria-label="Header">
|
|
<a href="../.." title="obikmer" class="md-header__button md-logo" aria-label="obikmer" data-md-component="logo">
|
|
|
|
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
|
|
|
|
</a>
|
|
<label class="md-header__button md-icon" for="__drawer">
|
|
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
|
|
</label>
|
|
<div class="md-header__title" data-md-component="header-title">
|
|
<div class="md-header__ellipsis">
|
|
<div class="md-header__topic">
|
|
<span class="md-ellipsis">
|
|
obikmer
|
|
</span>
|
|
</div>
|
|
<div class="md-header__topic" data-md-component="header-topic">
|
|
<span class="md-ellipsis">
|
|
|
|
Select command
|
|
|
|
</span>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<script>var palette=__md_get("__palette");if(palette&&palette.color){if("(prefers-color-scheme)"===palette.color.media){var media=matchMedia("(prefers-color-scheme: light)"),input=document.querySelector(media.matches?"[data-md-color-media='(prefers-color-scheme: light)']":"[data-md-color-media='(prefers-color-scheme: dark)']");palette.color.media=input.getAttribute("data-md-color-media"),palette.color.scheme=input.getAttribute("data-md-color-scheme"),palette.color.primary=input.getAttribute("data-md-color-primary"),palette.color.accent=input.getAttribute("data-md-color-accent")}for(var[key,value]of Object.entries(palette.color))document.body.setAttribute("data-md-color-"+key,value)}</script>
|
|
|
|
|
|
|
|
|
|
</nav>
|
|
|
|
</header>
|
|
|
|
<div class="md-container" data-md-component="container">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<main class="md-main" data-md-component="main">
|
|
<div class="md-main__inner md-grid">
|
|
|
|
|
|
|
|
<div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
|
|
<div class="md-sidebar__scrollwrap">
|
|
<div class="md-sidebar__inner">
|
|
|
|
|
|
|
|
|
|
<nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
|
|
<label class="md-nav__title" for="__drawer">
|
|
<a href="../.." title="obikmer" class="md-nav__button md-logo" aria-label="obikmer" data-md-component="logo">
|
|
|
|
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
|
|
|
|
</a>
|
|
obikmer
|
|
</label>
|
|
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../.." class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Home
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../installation/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Installation
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item md-nav__item--nested">
|
|
|
|
|
|
|
|
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3" >
|
|
|
|
|
|
<label class="md-nav__link" for="__nav_3" id="__nav_3_label" tabindex="0">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Theory
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
<span class="md-nav__icon md-icon"></span>
|
|
</label>
|
|
|
|
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_3_label" aria-expanded="false">
|
|
<label class="md-nav__title" for="__nav_3">
|
|
<span class="md-nav__icon md-icon"></span>
|
|
|
|
|
|
Theory
|
|
|
|
|
|
</label>
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../kmers/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Kmers and super-kmers
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../theory/encoding/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
DNA encoding
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../theory/entropy/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Entropy filter
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../theory/minimizer/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Minimizer selection
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../theory/indexing/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Partitioning architecture
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../theory/evolutionary_distances/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Central-position SNP distance (discussion)
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
</ul>
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item md-nav__item--active md-nav__item--nested">
|
|
|
|
|
|
|
|
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4" checked>
|
|
|
|
|
|
<label class="md-nav__link" for="__nav_4" id="__nav_4_label" tabindex="0">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Implementation
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
<span class="md-nav__icon md-icon"></span>
|
|
</label>
|
|
|
|
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_4_label" aria-expanded="true">
|
|
<label class="md-nav__title" for="__nav_4">
|
|
<span class="md-nav__icon md-icon"></span>
|
|
|
|
|
|
Implementation
|
|
|
|
|
|
</label>
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../superkmer/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
SuperKmer
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../kmer/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Kmer
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../chunkreader/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Chunk reader
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../pipeline/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Construction pipeline
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../obipipeline/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
obipipeline library
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../storage/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
On-disk storage
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../mphf/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
MPHF selection
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../unitig_evidence/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Unitig evidence encoding
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../evidence_elimination/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Evidence elimination (discussion)
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../layer_tier/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
obikindex layer tier
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../persistent_compact_int_vec/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
PersistentCompactIntVec
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../persistent_bit_vec/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
PersistentBitVec
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../merge/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Merge command
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../merge_parallelism/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Merge parallelism & memory
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../filtering/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Kmer filtering
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item md-nav__item--active">
|
|
|
|
<input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
|
|
|
|
|
|
|
|
|
|
|
|
<label class="md-nav__link md-nav__link--active" for="__toc">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Select command
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
<span class="md-nav__icon md-icon"></span>
|
|
</label>
|
|
|
|
<a href="./" class="md-nav__link md-nav__link--active">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Select command
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
|
|
|
|
|
|
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<label class="md-nav__title" for="__toc">
|
|
<span class="md-nav__icon md-icon"></span>
|
|
Table of contents
|
|
</label>
|
|
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#synopsis" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Synopsis
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#output-destination" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Output destination
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#defining-output-columns" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Defining output columns
|
|
|
|
</span>
|
|
</a>
|
|
|
|
<nav class="md-nav" aria-label="Defining output columns">
|
|
<ul class="md-nav__list">
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#named-groups-group" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Named groups — --group
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#per-group-operator-group-op" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Per-group operator — --group-op
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#shorthand-aggregate-by-aggregate-op" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Shorthand — --aggregate-by / --aggregate-op
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#column-selection-and-ordering-select" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Column selection and ordering — --select
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#aggregation-operators" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Aggregation operators
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#behaviour-for-edge-cases" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Behaviour for edge cases
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#examples" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Examples
|
|
|
|
</span>
|
|
</a>
|
|
|
|
<nav class="md-nav" aria-label="Examples">
|
|
<ul class="md-nav__list">
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#aggregate-by-metadata-group-default-operators" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Aggregate by metadata group, default operators
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#named-groups-with-different-operators" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Named groups with different operators
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#mix-aggregated-group-and-individual-genome" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Mix aggregated group and individual genome
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#pure-column-projection-no-aggregation" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Pure column projection (no aggregation)
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#compose-with-filter" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Compose with filter
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#implementation-notes" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Implementation notes
|
|
|
|
</span>
|
|
</a>
|
|
|
|
<nav class="md-nav" aria-label="Implementation notes">
|
|
<ul class="md-nav__list">
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#known-gap-not-yet-fixed-2026-08-28" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Known gap (not yet fixed, 2026-08-28)
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../obitaxonomy/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
obitaxonomy crate
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../benchmark_query_testing/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Benchmark: query-path testing
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../partition_layer_cache/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Partition and layer caching (discussion)
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
</ul>
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item md-nav__item--nested">
|
|
|
|
|
|
|
|
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_5" >
|
|
|
|
|
|
<label class="md-nav__link" for="__nav_5" id="__nav_5_label" tabindex="0">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Architecture
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
<span class="md-nav__icon md-icon"></span>
|
|
</label>
|
|
|
|
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_5_label" aria-expanded="false">
|
|
<label class="md-nav__title" for="__nav_5">
|
|
<span class="md-nav__icon md-icon"></span>
|
|
|
|
|
|
Architecture
|
|
|
|
|
|
</label>
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../architecture/sequences/invariant/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Sequences
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../architecture/index_architecture/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Kmer index
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../architecture/siblings/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
Sibling annex (discussion)
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../architecture/numa_worker_pools/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
NUMA-aware worker pools
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../architecture/numa_partition_runner/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
|
|
|
|
NUMA-aware partition runner
|
|
|
|
|
|
|
|
</span>
|
|
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
</ul>
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
|
|
|
|
</ul>
|
|
</nav>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
|
|
<div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
|
|
<div class="md-sidebar__scrollwrap">
|
|
<div class="md-sidebar__inner">
|
|
|
|
|
|
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<label class="md-nav__title" for="__toc">
|
|
<span class="md-nav__icon md-icon"></span>
|
|
Table of contents
|
|
</label>
|
|
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#synopsis" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Synopsis
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#output-destination" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Output destination
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#defining-output-columns" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Defining output columns
|
|
|
|
</span>
|
|
</a>
|
|
|
|
<nav class="md-nav" aria-label="Defining output columns">
|
|
<ul class="md-nav__list">
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#named-groups-group" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Named groups — --group
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#per-group-operator-group-op" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Per-group operator — --group-op
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#shorthand-aggregate-by-aggregate-op" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Shorthand — --aggregate-by / --aggregate-op
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#column-selection-and-ordering-select" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Column selection and ordering — --select
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#aggregation-operators" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Aggregation operators
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#behaviour-for-edge-cases" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Behaviour for edge cases
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#examples" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Examples
|
|
|
|
</span>
|
|
</a>
|
|
|
|
<nav class="md-nav" aria-label="Examples">
|
|
<ul class="md-nav__list">
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#aggregate-by-metadata-group-default-operators" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Aggregate by metadata group, default operators
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#named-groups-with-different-operators" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Named groups with different operators
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#mix-aggregated-group-and-individual-genome" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Mix aggregated group and individual genome
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#pure-column-projection-no-aggregation" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Pure column projection (no aggregation)
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#compose-with-filter" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Compose with filter
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#implementation-notes" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Implementation notes
|
|
|
|
</span>
|
|
</a>
|
|
|
|
<nav class="md-nav" aria-label="Implementation notes">
|
|
<ul class="md-nav__list">
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#known-gap-not-yet-fixed-2026-08-28" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
|
|
Known gap (not yet fixed, 2026-08-28)
|
|
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
|
|
</nav>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
|
|
<div class="md-content" data-md-component="content">
|
|
|
|
<article class="md-content__inner md-typeset">
|
|
|
|
|
|
|
|
|
|
|
|
<h1 id="select-column-projection-and-aggregation"><code>select</code> — column projection and aggregation</h1>
|
|
<p><code>select</code> transforms an index by operating on its <strong>genome columns</strong>: projecting a
|
|
subset of columns, aggregating groups of genomes into synthetic columns, or both.
|
|
It is the column-axis counterpart of <code>filter</code> (row-axis operations).</p>
|
|
<p>Following relational algebra conventions:</p>
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>Command</th>
|
|
<th>Relational operation</th>
|
|
<th>Axis</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
<tr>
|
|
<td><code>filter</code></td>
|
|
<td>σ — selection</td>
|
|
<td>rows (k-mers)</td>
|
|
</tr>
|
|
<tr>
|
|
<td><code>select</code></td>
|
|
<td>π — projection</td>
|
|
<td>columns (genomes)</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
<p>The two commands compose naturally: run <code>filter</code> first to restrict the kmer set,
|
|
then <code>select</code> to reshape the genome columns.</p>
|
|
<p><code>select</code> never changes the kmer set. The MPHF and <code>unitigs.bin</code> of each layer
|
|
are preserved unchanged; only the data matrices are rewritten.</p>
|
|
<hr />
|
|
<h2 id="synopsis">Synopsis</h2>
|
|
<div class="highlight"><pre><span></span><code>obikmer<span class="w"> </span><span class="k">select</span><span class="w"> </span><input-index>
|
|
<span class="w"> </span>--output<span class="w"> </span><dir>
|
|
<span class="w"> </span><span class="o">[</span>--group<span class="w"> </span><name>:<pred><span class="w"> </span>...<span class="o">]</span>
|
|
<span class="w"> </span><span class="o">[</span>--group-op<span class="w"> </span><name>:<op><span class="w"> </span>...<span class="o">]</span>
|
|
<span class="w"> </span><span class="o">[</span>--aggregate-by<span class="w"> </span><key><span class="w"> </span><span class="o">]</span>
|
|
<span class="w"> </span><span class="o">[</span>--aggregate-op<span class="w"> </span><op><span class="w"> </span><span class="o">]</span>
|
|
<span class="w"> </span><span class="o">[</span>--select<span class="w"> </span><col1,col2,...><span class="w"> </span><span class="o">]</span>
|
|
<span class="w"> </span><span class="o">[</span>--presence-threshold<span class="w"> </span><N><span class="w"> </span><span class="o">]</span>
|
|
<span class="w"> </span><span class="o">[</span>--force-copy<span class="w"> </span><span class="o">]</span>
|
|
</code></pre></div>
|
|
<hr />
|
|
<h2 id="output-destination">Output destination</h2>
|
|
<p><code>--output <dir></code> is required — <code>select</code> always writes a new index; there is no
|
|
<code>--in-place</code> mode (2026-08-28: never implemented, removed from the design). The
|
|
source index is unchanged.</p>
|
|
<p>Each layer's kmer-identity files (<code>mphf.bin</code>/<code>unitigs.bin</code>/<code>evidence.bin</code>/
|
|
<code>unitigs.bin.idx</code>/<code>fingerprint.bin</code>/<code>layer_meta.json</code>) are never rewritten by a
|
|
column projection/aggregation, so they are hard-linked into the output rather
|
|
than copied — no extra disk for them even on a large index. Falls back to a
|
|
real copy automatically if linking fails (different filesystems); <code>--force-copy</code>
|
|
forces a real copy always, for an output that must survive independently of the
|
|
source on disk (a hard link shares the same inode — rewriting one path outside
|
|
<code>select</code> itself would affect the other). Only the <code>presence</code>/<code>counts</code>
|
|
subdirectory is ever a genuinely new, independent file.</p>
|
|
<p>To replace an index with a selected version of itself, select to a temporary
|
|
directory and swap it in (<code>rm -rf INDEX && mv INDEX.tmp INDEX</code>) — the case
|
|
<code>--in-place</code> used to cover.</p>
|
|
<hr />
|
|
<h2 id="defining-output-columns">Defining output columns</h2>
|
|
<h3 id="named-groups-group">Named groups — <code>--group</code></h3>
|
|
<div class="highlight"><pre><span></span><code>--group <name>:<pred>
|
|
</code></pre></div>
|
|
<p>Defines a named group of genomes using the same predicate syntax as <code>filter</code>.
|
|
Repeatable; a genome can belong to several groups.</p>
|
|
<div class="highlight"><pre><span></span><code>--group<span class="w"> </span><span class="s2">"pub:species=Betula_pubescens"</span>
|
|
--group<span class="w"> </span><span class="s2">"nan:species=Betula_nana"</span>
|
|
</code></pre></div>
|
|
<h3 id="per-group-operator-group-op">Per-group operator — <code>--group-op</code></h3>
|
|
<div class="highlight"><pre><span></span><code>--group-op <name>:<op>
|
|
</code></pre></div>
|
|
<p>Assigns an aggregation operator to a named group. Optional; if absent, the
|
|
default operator applies (see below).</p>
|
|
<div class="highlight"><pre><span></span><code>--group-op<span class="w"> </span><span class="s2">"pub:any"</span>
|
|
--group-op<span class="w"> </span><span class="s2">"nan:all"</span>
|
|
</code></pre></div>
|
|
<h3 id="shorthand-aggregate-by-aggregate-op">Shorthand — <code>--aggregate-by</code> / <code>--aggregate-op</code></h3>
|
|
<p><code>--aggregate-by <key></code> automatically creates one group per unique value of the
|
|
metadata key <code><key></code>. Equivalent to one <code>--group <val>:<key>=<val></code> per distinct
|
|
value. <code>--aggregate-op <op></code> sets the operator for all auto-generated groups.</p>
|
|
<p><code>--aggregate-by</code> and <code>--group</code> are mutually exclusive.</p>
|
|
<h3 id="column-selection-and-ordering-select">Column selection and ordering — <code>--select</code></h3>
|
|
<div class="highlight"><pre><span></span><code>--select col1,col2,...
|
|
</code></pre></div>
|
|
<p>Lists the output columns in order. Each element is either a group name (defined
|
|
by <code>--group</code> or generated by <code>--aggregate-by</code>) or a genome label from the source
|
|
index (pass-through, no aggregation).</p>
|
|
<p><strong>Default when <code>--select</code> is absent:</strong>
|
|
all defined groups in declaration order (for <code>--group</code>), or all generated groups
|
|
in metadata-value order (for <code>--aggregate-by</code>). Individual genomes not in any
|
|
group are excluded unless named explicitly.</p>
|
|
<p><strong>When neither <code>--group</code> nor <code>--aggregate-by</code> is specified:</strong>
|
|
<code>--select</code> can still reference genome labels for pure column projection (no
|
|
aggregation). If <code>--select</code> is also absent, all genomes are output unchanged
|
|
(identity transform — useful combined with row filtering via a prior <code>filter</code>
|
|
run).</p>
|
|
<hr />
|
|
<h2 id="aggregation-operators">Aggregation operators</h2>
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>Operator</th>
|
|
<th>Input</th>
|
|
<th>Output</th>
|
|
<th>Semantics</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
<tr>
|
|
<td><code>any</code></td>
|
|
<td>pres / count</td>
|
|
<td>presence</td>
|
|
<td>1 if ≥ 1 genome in group carries the k-mer</td>
|
|
</tr>
|
|
<tr>
|
|
<td><code>all</code></td>
|
|
<td>pres / count</td>
|
|
<td>presence</td>
|
|
<td>1 if every genome in group carries the k-mer</td>
|
|
</tr>
|
|
<tr>
|
|
<td><code>none</code></td>
|
|
<td>pres / count</td>
|
|
<td>presence</td>
|
|
<td>1 if no genome in group carries the k-mer</td>
|
|
</tr>
|
|
<tr>
|
|
<td><code>sum</code></td>
|
|
<td>count</td>
|
|
<td>count</td>
|
|
<td>sum of counts across the group</td>
|
|
</tr>
|
|
<tr>
|
|
<td><code>min</code></td>
|
|
<td>count</td>
|
|
<td>count</td>
|
|
<td>minimum count</td>
|
|
</tr>
|
|
<tr>
|
|
<td><code>max</code></td>
|
|
<td>count</td>
|
|
<td>count</td>
|
|
<td>maximum count</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
<p><strong>Default operator:</strong>
|
|
- Presence index: <code>any</code>
|
|
- Count index: <code>sum</code></p>
|
|
<p>Logical operators (<code>any</code>/<code>all</code>/<code>none</code>) on a count index use
|
|
<code>--presence-threshold N</code> (default 0): a genome "carries" the k-mer if its count
|
|
is > N.</p>
|
|
<p><strong>Output index type:</strong>
|
|
- If the source is a presence index, the output is always a presence index.
|
|
- If the source is a count index and every output column uses a logical operator
|
|
or is a pass-through from a presence source, the output is a presence index.
|
|
- Otherwise (at least one arithmetic operator on a count source), the output is
|
|
a count index.</p>
|
|
<hr />
|
|
<h2 id="behaviour-for-edge-cases">Behaviour for edge cases</h2>
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>Situation</th>
|
|
<th>Behaviour</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
<tr>
|
|
<td>Genome missing the metadata key in <code>--aggregate-by</code></td>
|
|
<td>genome ignored (no <code>NA</code> group)</td>
|
|
</tr>
|
|
<tr>
|
|
<td>Genome in multiple groups</td>
|
|
<td>contributes independently to each</td>
|
|
</tr>
|
|
<tr>
|
|
<td><code>--group-op</code> references undefined group</td>
|
|
<td>error</td>
|
|
</tr>
|
|
<tr>
|
|
<td><code>--select</code> element is neither group name nor genome label</td>
|
|
<td>error</td>
|
|
</tr>
|
|
<tr>
|
|
<td><code>--output</code> and <code>--in-place</code> both specified</td>
|
|
<td>error</td>
|
|
</tr>
|
|
<tr>
|
|
<td>Neither <code>--output</code> nor <code>--in-place</code></td>
|
|
<td>error</td>
|
|
</tr>
|
|
<tr>
|
|
<td>Group with zero matching genomes</td>
|
|
<td>column is all-zeros (or all-ones for <code>none</code>)</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
<hr />
|
|
<h2 id="examples">Examples</h2>
|
|
<h3 id="aggregate-by-metadata-group-default-operators">Aggregate by metadata group, default operators</h3>
|
|
<div class="highlight"><pre><span></span><code>obikmer<span class="w"> </span><span class="k">select</span><span class="w"> </span>myindex<span class="w"> </span>--output<span class="w"> </span>out<span class="w"> </span>--aggregate-by<span class="w"> </span>group
|
|
<span class="c1"># one column per unique value of "group"; presence→any, count→sum</span>
|
|
</code></pre></div>
|
|
<h3 id="named-groups-with-different-operators">Named groups with different operators</h3>
|
|
<div class="highlight"><pre><span></span><code>obikmer<span class="w"> </span><span class="k">select</span><span class="w"> </span>myindex<span class="w"> </span>--output<span class="w"> </span>out<span class="w"> </span><span class="se">\</span>
|
|
<span class="w"> </span>--group<span class="w"> </span><span class="s2">"pub:species=Betula_pubescens"</span><span class="w"> </span><span class="se">\</span>
|
|
<span class="w"> </span>--group<span class="w"> </span><span class="s2">"nan:species=Betula_nana"</span><span class="w"> </span><span class="se">\</span>
|
|
<span class="w"> </span>--group-op<span class="w"> </span><span class="s2">"pub:any"</span><span class="w"> </span><span class="se">\</span>
|
|
<span class="w"> </span>--group-op<span class="w"> </span><span class="s2">"nan:all"</span><span class="w"> </span><span class="se">\</span>
|
|
<span class="w"> </span>--select<span class="w"> </span><span class="s2">"pub,nan"</span>
|
|
</code></pre></div>
|
|
<h3 id="mix-aggregated-group-and-individual-genome">Mix aggregated group and individual genome</h3>
|
|
<div class="highlight"><pre><span></span><code>obikmer<span class="w"> </span><span class="k">select</span><span class="w"> </span>myindex<span class="w"> </span>--output<span class="w"> </span>out<span class="w"> </span><span class="se">\</span>
|
|
<span class="w"> </span>--group<span class="w"> </span><span class="s2">"A:group=A"</span><span class="w"> </span><span class="se">\</span>
|
|
<span class="w"> </span>--select<span class="w"> </span><span class="s2">"A,Betula_nana--IGA-24-39"</span>
|
|
</code></pre></div>
|
|
<h3 id="pure-column-projection-no-aggregation">Pure column projection (no aggregation)</h3>
|
|
<div class="highlight"><pre><span></span><code>obikmer<span class="w"> </span><span class="k">select</span><span class="w"> </span>myindex<span class="w"> </span>--output<span class="w"> </span>out<span class="w"> </span><span class="se">\</span>
|
|
<span class="w"> </span>--select<span class="w"> </span><span class="s2">"Betula_nana--TROM-V-149986,Betula_nana--AG-P04-25-01"</span>
|
|
</code></pre></div>
|
|
<h3 id="compose-with-filter">Compose with filter</h3>
|
|
<div class="highlight"><pre><span></span><code><span class="c1"># Step 1: keep only B. nana-specific k-mers</span>
|
|
obikmer<span class="w"> </span>filter<span class="w"> </span>myindex<span class="w"> </span>--output<span class="w"> </span>filtered<span class="w"> </span><span class="se">\</span>
|
|
<span class="w"> </span>--ingroup<span class="w"> </span><span class="s2">"species=Betula_nana"</span><span class="w"> </span>--outgroup<span class="w"> </span><span class="s2">"*"</span>
|
|
|
|
<span class="c1"># Step 2: aggregate genome columns by collection site</span>
|
|
obikmer<span class="w"> </span><span class="k">select</span><span class="w"> </span>filtered<span class="w"> </span>--output<span class="w"> </span>final<span class="w"> </span>--aggregate-by<span class="w"> </span>site
|
|
</code></pre></div>
|
|
<hr />
|
|
<h2 id="implementation-notes">Implementation notes</h2>
|
|
<p><code>select</code> does not rebuild the MPHF. Every partition is processed independently
|
|
(<code>PartitionRunner</code>), each writing its own output layers; no cross-partition
|
|
synchronisation is needed.</p>
|
|
<p>For each layer in each partition (<code>obikselect::select_layer::select_partition</code>):</p>
|
|
<ol>
|
|
<li><code>copy_layer_files</code> hard-links the source layer's kmer-identity files
|
|
(<code>mphf.bin</code>/<code>unitigs.bin</code>/<code>evidence.bin</code>/<code>unitigs.bin.idx</code>/
|
|
<code>fingerprint.bin</code>/<code>layer_meta.json</code>) into the destination — never a real
|
|
copy unless linking fails or <code>--force-copy</code> is given.</li>
|
|
<li>A new data matrix is built with M columns (M = number of output columns),
|
|
under a fresh <code>presence/</code>/<code>counts/</code> subdirectory (never touching the
|
|
source's own).</li>
|
|
<li><strong>Presence source (2026-08-28: <code>batch_presence_counts</code>)</strong>: one shared pass
|
|
over the source bit matrix computes every output group's presence count at
|
|
once — row-major native for a <code>Sparse</code> source (<code>for_each_genome_in_row</code>,
|
|
which has no column representation to read a <code>col_view</code> from at all — the
|
|
reason this replaced the old per-group loop, not just an optimisation of
|
|
it), deduplicated column-major (one <code>col_view</code> per <em>distinct referenced
|
|
column</em>, not per group) for <code>Columnar</code>/<code>Packed</code>. Every <code>AggOp</code> for a bit
|
|
matrix is then a cheap derivation of that one count vector (<code>sum</code> = the
|
|
count itself, <code>any</code>/<code>max</code> = <code>count ≥ 1</code>, <code>all</code>/<code>min</code> = <code>count == group
|
|
size</code>, <code>none</code> = <code>count == 0</code>) — see
|
|
<code>obikselect::select_layer::agg_result_from_count</code>.</li>
|
|
<li><strong>Count source</strong>: unchanged, one <code>col_view</code>-driven pass per output column
|
|
via <code>MatrixGroupOps</code> — <code>sum</code>/<code>min</code>/<code>max</code> are genuine per-value reductions
|
|
for a count matrix, not derivable from a single presence count the way
|
|
they are for a bit matrix.</li>
|
|
<li><code>index.meta</code> is rewritten with the new genome list and updated <code>with_counts</code>.</li>
|
|
</ol>
|
|
<h3 id="known-gap-not-yet-fixed-2026-08-28">Known gap (not yet fixed, 2026-08-28)</h3>
|
|
<p>Step 4 above still panics (<code>col_view() not available on Sparse
|
|
PersistentCompactIntMatrix</code>) if the source is a <strong>count</strong> index packed
|
|
sparse — <code>batch_presence_counts</code>' row-major treatment was only ported to the
|
|
bit-matrix (<code>Presence</code>) case, since that was the one actually blocking a real
|
|
benchmark run. <code>select</code>/<code>filter</code> on a sparse-packed count index still hits
|
|
this; the fix would follow the same shape (a <code>PersistentSparseCompactIntMatrix</code>
|
|
row-major decode, analogous to <code>for_each_genome_in_row</code>), just not done. Since
|
|
<code>obisys::numa::runner::PartitionRunner</code>'s panic-propagation fix (see
|
|
<code>architecture/numa_partition_runner.md</code>), this at least fails fast (process
|
|
panic, exit 101) instead of hanging.</p>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
</article>
|
|
</div>
|
|
|
|
|
|
<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
|
|
</div>
|
|
|
|
</main>
|
|
|
|
<footer class="md-footer">
|
|
|
|
<div class="md-footer-meta md-typeset">
|
|
<div class="md-footer-meta__inner md-grid">
|
|
<div class="md-copyright">
|
|
|
|
|
|
Made with
|
|
<a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
|
|
Material for MkDocs
|
|
</a>
|
|
|
|
</div>
|
|
|
|
</div>
|
|
</div>
|
|
</footer>
|
|
|
|
</div>
|
|
<div class="md-dialog" data-md-component="dialog">
|
|
<div class="md-dialog__inner md-typeset"></div>
|
|
</div>
|
|
|
|
|
|
|
|
|
|
|
|
<script id="__config" type="application/json">{"annotate": null, "base": "../..", "features": [], "search": "../../assets/javascripts/workers/search.2c215733.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
|
|
|
|
|
|
<script src="../../assets/javascripts/bundle.79ae519e.min.js"></script>
|
|
|
|
<script src="https://unpkg.com/mathjax@3/es5/tex-mml-chtml.js"></script>
|
|
|
|
|
|
</body>
|
|
</html> |