OBIKmers/obikmer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 27 Wiki Activity
Files
66ab1d0947136c1480e4d35c61d31ba62df7cd79
obikmer/DevDoc/implementation/select/index.html
T

1952 lines
41 KiB
HTML
Raw Normal View History

large refactoring
2026-08-15 20:56:29 +02:00
<!doctype html>
<html lang="en" class="no-js">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<link rel="prev" href="../filtering/">
<link rel="next" href="../obitaxonomy/">
<link rel="icon" href="../../assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.7.6">
<title>Select command - obikmer</title>
<link rel="stylesheet" href="../../assets/stylesheets/main.484c7ddc.min.css">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
<style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
<script>__md_scope=new URL("../..",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
</head>
<body dir="ltr">
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
<label class="md-overlay" for="__drawer"></label>
<div data-md-component="skip">
<a href="#select-column-projection-and-aggregation" class="md-skip">
Skip to content
</a>
</div>
<div data-md-component="announce">
</div>
<header class="md-header md-header--shadow" data-md-component="header">
<nav class="md-header__inner md-grid" aria-label="Header">
<a href="../.." title="obikmer" class="md-header__button md-logo" aria-label="obikmer" data-md-component="logo">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
</a>
<label class="md-header__button md-icon" for="__drawer">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
</label>
<div class="md-header__title" data-md-component="header-title">
<div class="md-header__ellipsis">
<div class="md-header__topic">
<span class="md-ellipsis">
obikmer
</span>
</div>
<div class="md-header__topic" data-md-component="header-topic">
<span class="md-ellipsis">
Select command
</span>
</div>
</div>
</div>
<script>var palette=__md_get("__palette");if(palette&&palette.color){if("(prefers-color-scheme)"===palette.color.media){var media=matchMedia("(prefers-color-scheme: light)"),input=document.querySelector(media.matches?"[data-md-color-media='(prefers-color-scheme: light)']":"[data-md-color-media='(prefers-color-scheme: dark)']");palette.color.media=input.getAttribute("data-md-color-media"),palette.color.scheme=input.getAttribute("data-md-color-scheme"),palette.color.primary=input.getAttribute("data-md-color-primary"),palette.color.accent=input.getAttribute("data-md-color-accent")}for(var[key,value]of Object.entries(palette.color))document.body.setAttribute("data-md-color-"+key,value)}</script>
</nav>
</header>
<div class="md-container" data-md-component="container">
<main class="md-main" data-md-component="main">
<div class="md-main__inner md-grid">
<div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
<label class="md-nav__title" for="__drawer">
<a href="../.." title="obikmer" class="md-nav__button md-logo" aria-label="obikmer" data-md-component="logo">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
</a>
obikmer
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../.." class="md-nav__link">
<span class="md-ellipsis">
Home
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../installation/" class="md-nav__link">
<span class="md-ellipsis">
Installation
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3" >
<label class="md-nav__link" for="__nav_3" id="__nav_3_label" tabindex="0">
<span class="md-ellipsis">
Theory
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_3_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_3">
<span class="md-nav__icon md-icon"></span>
Theory
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../kmers/" class="md-nav__link">
<span class="md-ellipsis">
Kmers and super-kmers
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../theory/encoding/" class="md-nav__link">
<span class="md-ellipsis">
DNA encoding
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../theory/entropy/" class="md-nav__link">
<span class="md-ellipsis">
Entropy filter
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../theory/minimizer/" class="md-nav__link">
<span class="md-ellipsis">
Minimizer selection
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../theory/indexing/" class="md-nav__link">
<span class="md-ellipsis">
Partitioning architecture
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../theory/evolutionary_distances/" class="md-nav__link">
<span class="md-ellipsis">
Central-position SNP distance (discussion)
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--active md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4" checked>
<label class="md-nav__link" for="__nav_4" id="__nav_4_label" tabindex="0">
<span class="md-ellipsis">
Implementation
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_4_label" aria-expanded="true">
<label class="md-nav__title" for="__nav_4">
<span class="md-nav__icon md-icon"></span>
Implementation
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../superkmer/" class="md-nav__link">
<span class="md-ellipsis">
SuperKmer
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../kmer/" class="md-nav__link">
<span class="md-ellipsis">
Kmer
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../chunkreader/" class="md-nav__link">
<span class="md-ellipsis">
Chunk reader
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../pipeline/" class="md-nav__link">
<span class="md-ellipsis">
Construction pipeline
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../obipipeline/" class="md-nav__link">
<span class="md-ellipsis">
obipipeline library
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../storage/" class="md-nav__link">
<span class="md-ellipsis">
On-disk storage
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../mphf/" class="md-nav__link">
<span class="md-ellipsis">
MPHF selection
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../unitig_evidence/" class="md-nav__link">
<span class="md-ellipsis">
Unitig evidence encoding
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../evidence_elimination/" class="md-nav__link">
<span class="md-ellipsis">
Evidence elimination (discussion)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../obilayeredmap/" class="md-nav__link">
<span class="md-ellipsis">
obilayeredmap crate
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../persistent_compact_int_vec/" class="md-nav__link">
<span class="md-ellipsis">
PersistentCompactIntVec
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../persistent_bit_vec/" class="md-nav__link">
<span class="md-ellipsis">
PersistentBitVec
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../merge/" class="md-nav__link">
<span class="md-ellipsis">
Merge command
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../merge_parallelism/" class="md-nav__link">
<span class="md-ellipsis">
Merge parallelism & memory
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../filtering/" class="md-nav__link">
<span class="md-ellipsis">
Kmer filtering
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--active">
<input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
<label class="md-nav__link md-nav__link--active" for="__toc">
<span class="md-ellipsis">
Select command
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<a href="./" class="md-nav__link md-nav__link--active">
<span class="md-ellipsis">
Select command
</span>
</a>
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
Table of contents
</label>
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
<li class="md-nav__item">
<a href="#synopsis" class="md-nav__link">
<span class="md-ellipsis">
Synopsis
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#output-destination" class="md-nav__link">
<span class="md-ellipsis">
Output destination
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#defining-output-columns" class="md-nav__link">
<span class="md-ellipsis">
Defining output columns
</span>
</a>
<nav class="md-nav" aria-label="Defining output columns">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#named-groups-group" class="md-nav__link">
<span class="md-ellipsis">
Named groups — --group
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#per-group-operator-group-op" class="md-nav__link">
<span class="md-ellipsis">
Per-group operator — --group-op
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#shorthand-aggregate-by-aggregate-op" class="md-nav__link">
<span class="md-ellipsis">
Shorthand — --aggregate-by / --aggregate-op
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#column-selection-and-ordering-select" class="md-nav__link">
<span class="md-ellipsis">
Column selection and ordering — --select
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#aggregation-operators" class="md-nav__link">
<span class="md-ellipsis">
Aggregation operators
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#behaviour-for-edge-cases" class="md-nav__link">
<span class="md-ellipsis">
Behaviour for edge cases
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#examples" class="md-nav__link">
<span class="md-ellipsis">
Examples
</span>
</a>
<nav class="md-nav" aria-label="Examples">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#aggregate-by-metadata-group-default-operators" class="md-nav__link">
<span class="md-ellipsis">
Aggregate by metadata group, default operators
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#named-groups-with-different-operators" class="md-nav__link">
<span class="md-ellipsis">
Named groups with different operators
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#mix-aggregated-group-and-individual-genome" class="md-nav__link">
<span class="md-ellipsis">
Mix aggregated group and individual genome
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#pure-column-projection-no-aggregation" class="md-nav__link">
<span class="md-ellipsis">
Pure column projection (no aggregation)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#in-place-keep-only-group-a" class="md-nav__link">
<span class="md-ellipsis">
In-place: keep only group A
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#compose-with-filter" class="md-nav__link">
<span class="md-ellipsis">
Compose with filter
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#implementation-notes" class="md-nav__link">
<span class="md-ellipsis">
Implementation notes
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../obitaxonomy/" class="md-nav__link">
<span class="md-ellipsis">
obitaxonomy crate
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_5" >
<label class="md-nav__link" for="__nav_5" id="__nav_5_label" tabindex="0">
<span class="md-ellipsis">
Architecture
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_5_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_5">
<span class="md-nav__icon md-icon"></span>
Architecture
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../architecture/sequences/invariant/" class="md-nav__link">
<span class="md-ellipsis">
Sequences
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../architecture/index_architecture/" class="md-nav__link">
<span class="md-ellipsis">
Kmer index
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../architecture/siblings/" class="md-nav__link">
<span class="md-ellipsis">
Sibling annex (discussion)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../architecture/numa_worker_pools/" class="md-nav__link">
<span class="md-ellipsis">
NUMA-aware worker pools
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../architecture/numa_partition_runner/" class="md-nav__link">
<span class="md-ellipsis">
NUMA-aware partition runner
</span>
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
Table of contents
</label>
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
<li class="md-nav__item">
<a href="#synopsis" class="md-nav__link">
<span class="md-ellipsis">
Synopsis
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#output-destination" class="md-nav__link">
<span class="md-ellipsis">
Output destination
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#defining-output-columns" class="md-nav__link">
<span class="md-ellipsis">
Defining output columns
</span>
</a>
<nav class="md-nav" aria-label="Defining output columns">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#named-groups-group" class="md-nav__link">
<span class="md-ellipsis">
Named groups — --group
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#per-group-operator-group-op" class="md-nav__link">
<span class="md-ellipsis">
Per-group operator — --group-op
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#shorthand-aggregate-by-aggregate-op" class="md-nav__link">
<span class="md-ellipsis">
Shorthand — --aggregate-by / --aggregate-op
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#column-selection-and-ordering-select" class="md-nav__link">
<span class="md-ellipsis">
Column selection and ordering — --select
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#aggregation-operators" class="md-nav__link">
<span class="md-ellipsis">
Aggregation operators
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#behaviour-for-edge-cases" class="md-nav__link">
<span class="md-ellipsis">
Behaviour for edge cases
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#examples" class="md-nav__link">
<span class="md-ellipsis">
Examples
</span>
</a>
<nav class="md-nav" aria-label="Examples">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#aggregate-by-metadata-group-default-operators" class="md-nav__link">
<span class="md-ellipsis">
Aggregate by metadata group, default operators
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#named-groups-with-different-operators" class="md-nav__link">
<span class="md-ellipsis">
Named groups with different operators
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#mix-aggregated-group-and-individual-genome" class="md-nav__link">
<span class="md-ellipsis">
Mix aggregated group and individual genome
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#pure-column-projection-no-aggregation" class="md-nav__link">
<span class="md-ellipsis">
Pure column projection (no aggregation)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#in-place-keep-only-group-a" class="md-nav__link">
<span class="md-ellipsis">
In-place: keep only group A
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#compose-with-filter" class="md-nav__link">
<span class="md-ellipsis">
Compose with filter
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#implementation-notes" class="md-nav__link">
<span class="md-ellipsis">
Implementation notes
</span>
</a>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-content" data-md-component="content">
<article class="md-content__inner md-typeset">
<h1 id="select-column-projection-and-aggregation"><code>select</code> — column projection and aggregation</h1>
<p><code>select</code> transforms an index by operating on its <strong>genome columns</strong>: projecting a
subset of columns, aggregating groups of genomes into synthetic columns, or both.
It is the column-axis counterpart of <code>filter</code> (row-axis operations).</p>
<p>Following relational algebra conventions:</p>
<table>
<thead>
<tr>
<th>Command</th>
<th>Relational operation</th>
<th>Axis</th>
</tr>
</thead>
<tbody>
<tr>
<td><code>filter</code></td>
<td>σ — selection</td>
<td>rows (k-mers)</td>
</tr>
<tr>
<td><code>select</code></td>
<td>π — projection</td>
<td>columns (genomes)</td>
</tr>
</tbody>
</table>
<p>The two commands compose naturally: run <code>filter</code> first to restrict the kmer set,
then <code>select</code> to reshape the genome columns.</p>
<p><code>select</code> never changes the kmer set. The MPHF and <code>unitigs.bin</code> of each layer
are preserved unchanged; only the data matrices are rewritten.</p>
<hr />
<h2 id="synopsis">Synopsis</h2>
<div class="highlight"><pre><span></span><code>obikmer<span class="w"> </span><span class="k">select</span><span class="w"> </span>&lt;input-index&gt;
<span class="w"> </span><span class="o">{</span><span class="w"> </span>--output<span class="w"> </span>&lt;dir&gt;<span class="w"> </span><span class="p">|</span><span class="w"> </span>--in-place<span class="w"> </span><span class="o">}</span>
<span class="w"> </span><span class="o">[</span>--group<span class="w"> </span>&lt;name&gt;:&lt;pred&gt;<span class="w"> </span>...<span class="o">]</span>
<span class="w"> </span><span class="o">[</span>--group-op<span class="w"> </span>&lt;name&gt;:&lt;op&gt;<span class="w"> </span>...<span class="o">]</span>
<span class="w"> </span><span class="o">[</span>--aggregate-by<span class="w"> </span>&lt;key&gt;<span class="w"> </span><span class="o">]</span>
<span class="w"> </span><span class="o">[</span>--aggregate-op<span class="w"> </span>&lt;op&gt;<span class="w"> </span><span class="o">]</span>
<span class="w"> </span><span class="o">[</span>--select<span class="w"> </span>&lt;col1,col2,...&gt;<span class="w"> </span><span class="o">]</span>
<span class="w"> </span><span class="o">[</span>--presence-threshold<span class="w"> </span>&lt;N&gt;<span class="w"> </span><span class="o">]</span>
</code></pre></div>
<hr />
<h2 id="output-destination">Output destination</h2>
<p>Exactly one of <code>--output</code> or <code>--in-place</code> must be specified.</p>
<p><strong><code>--output &lt;dir&gt;</code></strong> — writes a new index to <code>&lt;dir&gt;</code>. The source index is
unchanged. The MPHF and unitig files are copied; only the data matrices are
rewritten with the new column layout.</p>
<p><strong><code>--in-place</code></strong> — rewrites the data matrices of the source index directly.
Removed or replaced columns are lost. The operation writes to temporary files
first, then renames atomically, so an interrupted run leaves the index intact.</p>
<hr />
<h2 id="defining-output-columns">Defining output columns</h2>
<h3 id="named-groups-group">Named groups — <code>--group</code></h3>
<div class="highlight"><pre><span></span><code>--group &lt;name&gt;:&lt;pred&gt;
</code></pre></div>
<p>Defines a named group of genomes using the same predicate syntax as <code>filter</code>.
Repeatable; a genome can belong to several groups.</p>
<div class="highlight"><pre><span></span><code>--group<span class="w"> </span><span class="s2">&quot;pub:species=Betula_pubescens&quot;</span>
--group<span class="w"> </span><span class="s2">&quot;nan:species=Betula_nana&quot;</span>
</code></pre></div>
<h3 id="per-group-operator-group-op">Per-group operator — <code>--group-op</code></h3>
<div class="highlight"><pre><span></span><code>--group-op &lt;name&gt;:&lt;op&gt;
</code></pre></div>
<p>Assigns an aggregation operator to a named group. Optional; if absent, the
default operator applies (see below).</p>
<div class="highlight"><pre><span></span><code>--group-op<span class="w"> </span><span class="s2">&quot;pub:any&quot;</span>
--group-op<span class="w"> </span><span class="s2">&quot;nan:all&quot;</span>
</code></pre></div>
<h3 id="shorthand-aggregate-by-aggregate-op">Shorthand — <code>--aggregate-by</code> / <code>--aggregate-op</code></h3>
<p><code>--aggregate-by &lt;key&gt;</code> automatically creates one group per unique value of the
metadata key <code>&lt;key&gt;</code>. Equivalent to one <code>--group &lt;val&gt;:&lt;key&gt;=&lt;val&gt;</code> per distinct
value. <code>--aggregate-op &lt;op&gt;</code> sets the operator for all auto-generated groups.</p>
<p><code>--aggregate-by</code> and <code>--group</code> are mutually exclusive.</p>
<h3 id="column-selection-and-ordering-select">Column selection and ordering — <code>--select</code></h3>
<div class="highlight"><pre><span></span><code>--select col1,col2,...
</code></pre></div>
<p>Lists the output columns in order. Each element is either a group name (defined
by <code>--group</code> or generated by <code>--aggregate-by</code>) or a genome label from the source
index (pass-through, no aggregation).</p>
<p><strong>Default when <code>--select</code> is absent:</strong>
all defined groups in declaration order (for <code>--group</code>), or all generated groups
in metadata-value order (for <code>--aggregate-by</code>). Individual genomes not in any
group are excluded unless named explicitly.</p>
<p><strong>When neither <code>--group</code> nor <code>--aggregate-by</code> is specified:</strong>
<code>--select</code> can still reference genome labels for pure column projection (no
aggregation). If <code>--select</code> is also absent, all genomes are output unchanged
(identity transform — useful combined with row filtering via a prior <code>filter</code>
run).</p>
<hr />
<h2 id="aggregation-operators">Aggregation operators</h2>
<table>
<thead>
<tr>
<th>Operator</th>
<th>Input</th>
<th>Output</th>
<th>Semantics</th>
</tr>
</thead>
<tbody>
<tr>
<td><code>any</code></td>
<td>pres / count</td>
<td>presence</td>
<td>1 if ≥ 1 genome in group carries the k-mer</td>
</tr>
<tr>
<td><code>all</code></td>
<td>pres / count</td>
<td>presence</td>
<td>1 if every genome in group carries the k-mer</td>
</tr>
<tr>
<td><code>none</code></td>
<td>pres / count</td>
<td>presence</td>
<td>1 if no genome in group carries the k-mer</td>
</tr>
<tr>
<td><code>sum</code></td>
<td>count</td>
<td>count</td>
<td>sum of counts across the group</td>
</tr>
<tr>
<td><code>min</code></td>
<td>count</td>
<td>count</td>
<td>minimum count</td>
</tr>
<tr>
<td><code>max</code></td>
<td>count</td>
<td>count</td>
<td>maximum count</td>
</tr>
</tbody>
</table>
<p><strong>Default operator:</strong>
- Presence index: <code>any</code>
- Count index: <code>sum</code></p>
<p>Logical operators (<code>any</code>/<code>all</code>/<code>none</code>) on a count index use
<code>--presence-threshold N</code> (default 0): a genome "carries" the k-mer if its count
is &gt; N.</p>
<p><strong>Output index type:</strong>
- If the source is a presence index, the output is always a presence index.
- If the source is a count index and every output column uses a logical operator
or is a pass-through from a presence source, the output is a presence index.
- Otherwise (at least one arithmetic operator on a count source), the output is
a count index.</p>
<hr />
<h2 id="behaviour-for-edge-cases">Behaviour for edge cases</h2>
<table>
<thead>
<tr>
<th>Situation</th>
<th>Behaviour</th>
</tr>
</thead>
<tbody>
<tr>
<td>Genome missing the metadata key in <code>--aggregate-by</code></td>
<td>genome ignored (no <code>NA</code> group)</td>
</tr>
<tr>
<td>Genome in multiple groups</td>
<td>contributes independently to each</td>
</tr>
<tr>
<td><code>--group-op</code> references undefined group</td>
<td>error</td>
</tr>
<tr>
<td><code>--select</code> element is neither group name nor genome label</td>
<td>error</td>
</tr>
<tr>
<td><code>--output</code> and <code>--in-place</code> both specified</td>
<td>error</td>
</tr>
<tr>
<td>Neither <code>--output</code> nor <code>--in-place</code></td>
<td>error</td>
</tr>
<tr>
<td>Group with zero matching genomes</td>
<td>column is all-zeros (or all-ones for <code>none</code>)</td>
</tr>
</tbody>
</table>
<hr />
<h2 id="examples">Examples</h2>
<h3 id="aggregate-by-metadata-group-default-operators">Aggregate by metadata group, default operators</h3>
<div class="highlight"><pre><span></span><code>obikmer<span class="w"> </span><span class="k">select</span><span class="w"> </span>myindex<span class="w"> </span>--output<span class="w"> </span>out<span class="w"> </span>--aggregate-by<span class="w"> </span>group
<span class="c1"># one column per unique value of &quot;group&quot;; presence→any, count→sum</span>
</code></pre></div>
<h3 id="named-groups-with-different-operators">Named groups with different operators</h3>
<div class="highlight"><pre><span></span><code>obikmer<span class="w"> </span><span class="k">select</span><span class="w"> </span>myindex<span class="w"> </span>--output<span class="w"> </span>out<span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--group<span class="w"> </span><span class="s2">&quot;pub:species=Betula_pubescens&quot;</span><span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--group<span class="w"> </span><span class="s2">&quot;nan:species=Betula_nana&quot;</span><span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--group-op<span class="w"> </span><span class="s2">&quot;pub:any&quot;</span><span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--group-op<span class="w"> </span><span class="s2">&quot;nan:all&quot;</span><span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--select<span class="w"> </span><span class="s2">&quot;pub,nan&quot;</span>
</code></pre></div>
<h3 id="mix-aggregated-group-and-individual-genome">Mix aggregated group and individual genome</h3>
<div class="highlight"><pre><span></span><code>obikmer<span class="w"> </span><span class="k">select</span><span class="w"> </span>myindex<span class="w"> </span>--output<span class="w"> </span>out<span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--group<span class="w"> </span><span class="s2">&quot;A:group=A&quot;</span><span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--select<span class="w"> </span><span class="s2">&quot;A,Betula_nana--IGA-24-39&quot;</span>
</code></pre></div>
<h3 id="pure-column-projection-no-aggregation">Pure column projection (no aggregation)</h3>
<div class="highlight"><pre><span></span><code>obikmer<span class="w"> </span><span class="k">select</span><span class="w"> </span>myindex<span class="w"> </span>--output<span class="w"> </span>out<span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--select<span class="w"> </span><span class="s2">&quot;Betula_nana--TROM-V-149986,Betula_nana--AG-P04-25-01&quot;</span>
</code></pre></div>
<h3 id="in-place-keep-only-group-a">In-place: keep only group A</h3>
<div class="highlight"><pre><span></span><code>obikmer<span class="w"> </span><span class="k">select</span><span class="w"> </span>myindex<span class="w"> </span>--in-place<span class="w"> </span>--group<span class="w"> </span><span class="s2">&quot;A:group=A&quot;</span><span class="w"> </span>--select<span class="w"> </span><span class="s2">&quot;A&quot;</span>
</code></pre></div>
<h3 id="compose-with-filter">Compose with filter</h3>
<div class="highlight"><pre><span></span><code><span class="c1"># Step 1: keep only B. nana-specific k-mers</span>
obikmer<span class="w"> </span>filter<span class="w"> </span>myindex<span class="w"> </span>--output<span class="w"> </span>filtered<span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--ingroup<span class="w"> </span><span class="s2">&quot;species=Betula_nana&quot;</span><span class="w"> </span>--outgroup<span class="w"> </span><span class="s2">&quot;*&quot;</span>
<span class="c1"># Step 2: aggregate genome columns by collection site</span>
obikmer<span class="w"> </span><span class="k">select</span><span class="w"> </span>filtered<span class="w"> </span>--output<span class="w"> </span>final<span class="w"> </span>--aggregate-by<span class="w"> </span>site
</code></pre></div>
<hr />
<h2 id="implementation-notes">Implementation notes</h2>
<p><code>select</code> does not rebuild the MPHF. The 256 partitions are processed in parallel
(rayon), each writing its output independently; results require no synchronisation
because every partition owns a distinct set of files.</p>
<p>For each layer in each partition:</p>
<ol>
<li>The slot count <code>n</code> is read by opening the source data matrix.</li>
<li>A new data matrix is built with M columns (M = number of output columns).</li>
<li>For each slot <code>s</code> in <code>0..n</code>:</li>
<li><code>old_row = matrix.fill_row(s)</code> — reads the original <code>N</code>-column row without allocating.</li>
<li>For each output column <code>j</code>:<ul>
<li><code>new_row[j] = aggregate(op, old_row[group_indices])</code>.</li>
<li>Pass-through columns are represented as single-element groups with the
default operator (<code>any</code> for presence, <code>sum</code> for count) — same code path.</li>
</ul>
</li>
<li>The new row is written slot by slot into each column builder.</li>
<li>All plain files in the source layer directory (<code>mphf.bin</code>, <code>unitigs.bin</code>,
evidence files, <code>layer_meta.json</code>) are copied verbatim; only the <code>presence/</code>
or <code>counts/</code> subdirectory is rewritten.</li>
<li><code>index.meta</code> is rewritten with the new genome list and updated <code>with_counts</code>.</li>
</ol>
<p><strong><code>--in-place</code> write strategy:</strong> new data is written to a temporary sibling
directory (<code>presence_new/</code> or <code>counts_new/</code>); on success the old directory is
removed and the temporary one is renamed into place. An interrupted run leaves
at most one stale <code>*_new/</code> directory; the original data is intact until the
rename step.</p>
</article>
</div>
<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
</div>
</main>
<footer class="md-footer">
<div class="md-footer-meta md-typeset">
<div class="md-footer-meta__inner md-grid">
<div class="md-copyright">
Made with
<a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
Material for MkDocs
</a>
</div>
</div>
</div>
</footer>
</div>
<div class="md-dialog" data-md-component="dialog">
<div class="md-dialog__inner md-typeset"></div>
</div>
<script id="__config" type="application/json">{"annotate": null, "base": "../..", "features": [], "search": "../../assets/javascripts/workers/search.2c215733.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
<script src="../../assets/javascripts/bundle.79ae519e.min.js"></script>
<script src="https://unpkg.com/mathjax@3/es5/tex-mml-chtml.js"></script>
</body>
</html>
Reference in New Issue Copy Permalink