Merge pull request #123 from metabarcoding/push-mowoswzluyqu

Release 4.5.2

.github/workflows/obitest.yml

@@ -12,7 +12,7 @@ jobs:
     - name: Setup Go
       uses: actions/setup-go@v5
       with:
-        go-version: '1.23'
+        go-version: '1.27'
     - name: Checkout obitools4 project
       uses: actions/checkout@v5
     - name: Run tests

.github/workflows/release.yml

@@ -16,7 +16,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: "1.26"
+          go-version: "1.27"
       - name: Checkout obitools4 project
         uses: actions/checkout@v5
       - name: Run tests
@@ -54,7 +54,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: "1.26"
+          go-version: "1.27"
 
       - name: Extract version from tag
         id: get_version
@@ -78,7 +78,8 @@ jobs:
             -v "$(pwd):/src" \
             -w /src \
             -e VERSION="${VERSION}" \
-            golang:1.26-alpine \
+            -e GOEXPERIMENT=simd \
+            golang:1.27-alpine \
             sh -c "apk add --no-cache gcc musl-dev zlib-dev zlib-static make && \
                    make LDFLAGS='-linkmode=external -extldflags=-static' obitools"
           mkdir -p artifacts

.serena/project.yml

@@ -1,39 +1,6 @@
-# the name by which the project can be referenced within Serena
+# the name by which the project can be referenced within Serena/when chatting with the LLM.
 project_name: "obitools4"
 
-
-# list of languages for which language servers are started; choose from:
-#   al                  angular             ansible             bash                clojure
-#   cpp                 cpp_ccls            crystal             csharp              csharp_omnisharp
-#   dart                elixir              elm                 erlang              fortran
-#   fsharp              go                  groovy              haskell             haxe
-#   hlsl                html                java                json                julia
-#   kotlin              lean4               lua                 luau                markdown
-#   matlab              msl                 nix                 ocaml               pascal
-#   perl                php                 php_phpactor        powershell          python
-#   python_jedi         python_ty           r                   rego                ruby
-#   ruby_solargraph     rust                scala               scss                solidity
-#   svelte              swift               systemverilog       terraform           toml
-#   typescript          typescript_vts      vue                 yaml                zig
-#   (This list may be outdated. For the current list, see values of Language enum here:
-#   https://github.com/oraios/serena/blob/main/src/solidlsp/ls_config.py
-#   For some languages, there are alternative language servers, e.g. csharp_omnisharp, ruby_solargraph.)
-# Note:
-#   - For C, use cpp
-#   - For JavaScript, use typescript
-#   - For Angular projects, use angular (subsumes typescript+html; requires `npm install` in the project root)
-#   - For Svelte projects, use svelte (subsumes typescript/javascript for .svelte projects; requires npm)
-#   - For SCSS / Sass / plain CSS, use scss (some-sass-language-server handles all three)
-#   - For Free Pascal/Lazarus, use pascal
-# Special requirements:
-#   Some languages require additional setup/installations.
-#   See here for details: https://oraios.github.io/serena/01-about/020_programming-languages.html#language-servers
-# When using multiple languages, the first language server that supports a given file will be used for that file.
-# The first language is the default language and the respective language server will be used as a fallback.
-# Note that when using the JetBrains backend, language servers are not used and this list is correspondingly ignored.
-languages:
-- go
-
 # the encoding used by text files in the project
 # For a list of possible encodings, see https://docs.python.org/3.11/library/codecs.html#standard-encodings
 encoding: "utf-8"
@@ -55,23 +22,19 @@ ignore_all_files_in_gitignore: true
 
 # advanced configuration option allowing to configure language server-specific options.
 # Maps the language key to the options.
-# Have a look at the docstring of the constructors of the LS implementations within solidlsp (e.g., for C# or PHP) to see which options are available.
+# The settings are considered only if the project is trusted (see global configuration to define trusted projects).
-# No documentation on options means no options are available.
+# See https://oraios.github.io/serena/02-usage/050_configuration.html#language-server-specific-settings
 ls_specific_settings: {}
 
-# list of additional workspace folder paths for cross-package reference support (e.g. in monorepos).
-# Paths can be absolute or relative to the project root.
-# Each folder is registered as an LSP workspace folder, enabling language servers to discover
-# symbols and references across package boundaries.
-# Currently supported for: TypeScript.
-# Example:
-#   additional_workspace_folders:
-#     - ../sibling-package
-#     - ../shared-lib
-additional_workspace_folders: []
-
 # list of additional paths to ignore in this project.
 # Same syntax as gitignore, so you can use * and **.
+# Important: quote patterns that start with `*`, otherwise YAML treats them as aliases.
+# Example:
+#   ignored_paths:
+#     - "examples/**"
+#     - ".worktrees/**"
+#     - "**/bin/**"
+#     - "**/obj/**"
 # Note: global ignored_paths from serena_config.yml are also applied additively.
 ignored_paths: []
 
@@ -131,3 +94,76 @@ read_only_memory_patterns: []
 # Extends the list from the global configuration, merging the two lists.
 # Example: ["_archive/.*", "_episodes/.*"]
 ignored_memory_patterns: []
+
+# list of additional workspace folder paths for cross-package reference support.
+# Paths can be absolute or relative to the project root.
+# Each folder is registered as an LSP workspace folder, enabling language servers to discover
+# symbols and references across package boundaries, but these folders are not indexed by Serena,
+# i.e. the respective symbols will not be found using Serena's symbol search tools.
+# Example:
+#   additional_workspace_folders:
+#     - ../sibling-package
+#     - ../shared-lib
+ls_additional_workspace_folders: []
+
+# list of language servers to start when using the LSP backend; choose from:
+#   ada                    al                     angular                ansible                bash
+#   bsl                    clojure                cpp                    cpp_ccls               crystal
+#   csharp                 csharp_omnisharp       cue                    dart                   deno
+#   elixir                 elm                    erlang                 fortran                fsharp
+#   gdscript               gleam                  go                     groovy                 haskell
+#   haxe                   hlsl                   html                   java                   json
+#   julia                  kotlin                 latex                  lean4                  lua
+#   luau                   markdown               matlab                 msl                    nextflow
+#   nix                    ocaml                  pascal                 perl                   php
+#   php_phpactor           php_phpantom           powershell             python                 python_basedpyright
+#   python_jedi            python_pyrefly         python_ty              qml                    r
+#   rego                   ruby                   ruby_solargraph        rust                   scala
+#   scss                   solidity               svelte                 swift                  systemverilog
+#   terraform              toml                   typescript             typescript_vts         vue
+#   wolfram                yaml                   zig
+#   (This list may be outdated; generated with scripts/print_language_list.py;
+#   For the current list, see values of the LanguageServerId enum here:
+#   https://github.com/oraios/serena/blob/main/src/solidlsp/ls_config.py)
+# For some languages, there are several alternative language servers, e.g. csharp_omnisharp, ruby_solargraph.)
+# Note:
+#   - For C, use cpp
+#   - For JavaScript, use typescript
+#   - For Angular projects, use angular (subsumes typescript+html; requires `npm install` in the project root)
+#   - For Svelte projects, use svelte (subsumes typescript/javascript for .svelte projects; requires npm)
+#   - For Deno projects, use deno (serves the same .ts/.js files as typescript; requires the deno CLI on PATH)
+#   - For SCSS / Sass / plain CSS, use scss (some-sass-language-server handles all three)
+#   - For Free Pascal/Lazarus, use pascal
+# Special requirements:
+#   Some language servers require additional setup/installations.
+#   See here for details: https://oraios.github.io/serena/01-about/020_programming-languages.html#language-servers
+# When using multiple language servers, the first language server that supports a given file will be used for that file.
+# The first language server is the default language and the respective language server will be used as a fallback.
+# Note that when using the JetBrains backend, language servers are not used and this list is correspondingly ignored.
+language_servers:
+- go
+
+# list of workspace folder paths (LSP backend only).
+# These folders will be used to build up Serena's symbol index.
+# Paths must be within the project root and should thus be relative to the project root.
+# Furthermore, the paths should not be filtered by ignore settings.
+# Default setting: The entire project root folder (".") is considered.
+# In (large) monorepos, this can be used to index only subfolders of the project root, e.g.
+#   ls_workspace_folders:
+#     - "./subproject1"
+#     - "./subproject2"
+ls_workspace_folders:
+- .
+
+# optional shell command to run before the language backend (LSP or JetBrains) is initialised.
+# the command runs in the project root directory and is only executed if the project is trusted
+# (see trusted_project_path_patterns in the global configuration).
+# serena waits for the command to exit: a non-zero exit code is logged as an error but does not
+# abort activation. a per-project timeout (activation_command_timeout, default 180s) is the safety
+# backstop for non-terminating commands; on expiry the process is killed and activation continues.
+# example: activation_command: "npx nx run-many -t build"
+activation_command:
+
+# maximum time in seconds to wait for activation_command to complete before killing it (default 180s).
+# must be a positive number.
+activation_command_timeout: 180.0

Makefile

@@ -11,6 +11,14 @@ NC     := \033[0m
 
 GOFLAGS=
 LDFLAGS=
+
+# The SIMD code (pkg/obikmer, simd package of Go >= 1.27) is only compiled with
+# GOEXPERIMENT=simd; without it, a portable scalar code is used instead.
+# The binaries select the vector width at run time (and fall back on an
+# emulation without SIMD hardware), so they run on any CPU.
+# Use `make GOEXPERIMENT= obitools` to build without SIMD.
+GOEXPERIMENT?=simd
+export GOEXPERIMENT
 GOCMD=go
 GOBUILD=$(GOCMD) build $(GOFLAGS) $(if $(LDFLAGS),-ldflags="$(LDFLAGS)")
 GOGENERATE=$(GOCMD) generate
@@ -36,13 +44,18 @@ OBITOOLS_DIRS:=$(sort $(patsubst %/,%,$(dir $(OBITOOLS_SRC))))
 OBITOOLS:=$(notdir $(OBITOOLS_DIRS))
 
 
+# Name of the target of a package: its path under pkg/, so that two packages of
+# the same name in different directories (pkg/obialign and
+# pkg/obitools/obialign) do not share a target.
+pkgname=$(subst /,-,$(patsubst pkg/%,%,$(1)))
+
 define MAKE_PKG_RULE
-pkg-$(notdir $(1)): $(1) pkg/obioptions/version.go
+pkg-$(call pkgname,$(1)): $(1) pkg/obioptions/version.go
-	@echo -n - Building package $(notdir $(1))...
+	@echo -n - Building package $(call pkgname,$(1))...
 	@$(GOBUILD) ./$(1) \
-	    2> pkg-$(notdir $(1)).log \
+	    2> pkg-$(call pkgname,$(1)).log \
-		|| cat pkg-$(notdir $(1)).log
+		|| cat pkg-$(call pkgname,$(1)).log
-	@rm -f pkg-$(notdir $(1)).log
+	@rm -f pkg-$(call pkgname,$(1)).log
 	@echo Done.
 endef
 

OBIPRIMERS_MEMORY_PROMPT.md

@@ -0,0 +1,149 @@
+# Prompt : optimisation de l'usage mémoire de `obiprimers`
+
+Tu travailles dans le dépôt Go `/Users/coissac/Sync/travail/__MOI__/GO/obitools4` (module
+`git.metabarcoding.org/obitools/obitools4/obitools4`, Go : `/usr/local/go/bin/go`, non présent dans le PATH).
+Ta mission : **réduire l'usage mémoire de la commande `obiprimers`** sans changer ses résultats, en procédant
+par la mesure.
+
+## 1. Contexte
+
+`obiprimers` est la ré-implémentation OBITools4 d'ecoPrimers (design de paires d'amorces à partir de génomes
+entiers, avec évaluation de la qualité du marqueur délimité). Le mode « voie A » est terminé et validé.
+Avant toute chose, **lis `OBIPRIMERS_NOTES.md`** (racine du dépôt) : la section « État courant », puis les
+sections 11 à 13 (appariement approché), 16 et 16 bis (taxonomie embarquée ; **une erreur de mesure à ne pas
+refaire**), 25 (définitions de Bc, Bs, Bd et élagage de la phase 2).
+
+Code concerné :
+- `cmd/obitools/obiprimers/main.go`, `pkg/obitools/obiprimers/` (options, lecture, sortie CSV) ;
+- `pkg/obiprimer/` : `corpus.go` (`LoadCorpus`), `taxa.go`, `words.go` (`FindStrictWords`), `match.go` et
+  `match_seed.go` (`MatchWords`), `pairs.go` (`selectPrimers`, `countPairs`), `pairs_eval.go` (`FindPairs`),
+  `amplicons.go` (`collectAmplicons`), `resolution.go`, `search.go` (`Search`) ;
+- `pkg/obikmer/seqquorum*.go` (`SeqKmerCounter`, comptage par partitions sur disque) ;
+- `pkg/obithermo/` (Tm), code partagé : `pkg/obiformats`, `pkg/obiiter`, `pkg/obiseq`, `pkg/obitax`.
+
+## 2. Décisions de l'utilisateur à respecter (non négociables)
+
+- **Tout est chargé en mémoire** : toutes les séquences de la base (cibles, contre-exemples, non-cibles) sont
+  gardées, parce que la lecture sur l'**entrée standard** doit continuer à marcher comme pour tout obitool.
+  Tu peux **compacter** ce qui est gardé ; tu ne peux pas passer à un schéma qui exige de relire le fichier.
+- **Un seul calcul par indice** : ne touche pas aux définitions de Bc, Bs et Bd (`resolution.go`). Pas de
+  colonne ou d'option alternative.
+- **Les résultats ne changent pas.** Critère : la sortie CSV doit rester **identique octet pour octet** sur les
+  cas de référence du §4. Si tu penses qu'un changement de résultat est justifié, **demande avant**.
+- Aucune commande git qui modifie quoi que ce soit (pas de commit, add, mv, rm, reset…). Lecture seule
+  (`status`, `diff`, `log`) autorisée. Déplace les fichiers avec `mv`/`Write`/`Edit`.
+- Fichiers source de moins d'environ 500 lignes. Pas de `.collect()` d'une grande structure quand un flux
+  suffit. Réutilise l'existant avant d'écrire du code ; enrichis une brique partagée plutôt que d'écrire du
+  code ad hoc, **mais** toute modification d'un paquet partagé (obiformats, obiiter, obiseq, obitax) doit
+  être signalée clairement et validée par les tests de ces paquets.
+- **Pas de spéculation présentée comme un fait** : ce qui n'est pas mesuré est une hypothèse, dis-le.
+  Une régression de justesse est pire qu'un gain de mémoire : avant d'échanger l'une contre l'autre, demande.
+
+## 3. Ce qui est mesuré aujourd'hui (macOS, M3 Max, 16 cœurs)
+
+Jeu : 7 822 génomes mitochondriaux de vertébrés (131 Mb de bases), `-r 7953` (Cyprinidae : 325 cibles, 7 497
+non-cibles). Commande :
+`obiprimers -r 7953 -e 3 --three-prime 2 -l 30 -L 150 -c <fichier annoté> -o out.csv` → 3 247 paires
+(3 248 lignes avec l'en-tête).
+
+| Mesure | Valeur |
+|---|---|
+| pic RSS, taxonomie embarquée, fichier | **1 217 à 1 372 Mo** (bruit d'environ ±100 Mo d'une exécution à l'autre), 4,7 à 5,3 s |
+| pic RSS, taxonomie embarquée, entrée standard | médiane **1 334 à 1 346 Mo** sur 8 exécutions entrelacées |
+| tas des 7 822 séquences chargées | **193,7 Mo** (dont `taxonomic_path` : 19,2 Mo, soit 2,46 Ko par séquence) |
+| même exécution **sans** l'élagage de phase 2 | 5 604 Mo (c'est ce qu'on a corrigé, voir §25 des notes) |
+| avec `-t` (taxonomie NCBI complète) | +1,07 Go pour le seul chargeur (mesuré avec `obitaxonomy`), pic ~2,8 Go (mesuré avant les nouvelles définitions) |
+| `GOGC=25` (ancienne mesure, `-t` NCBI) | pic 2,82 Go → 1,98 Go, mais temps CPU utilisateur ×2 |
+| ecoPrimers (C) sur les mêmes jeux | 1,86 Go (Cyprinidae), 2,11 Go (Teleostei) |
+
+Tailles d'enregistrements : `ampRecord` 24 octets, `Match` 12, `seqMatch` 12, `hit` (intermédiaire de
+`matchCorpus`) 16. Environ 6,85 millions d'occurrences approchées quand toutes les séquences sont chargées
+(cas `-i 1`).
+
+**Le constat qui motive la mission** : les séquences gardées pèsent ~194 Mo, le pic du processus est ~1,3 Go.
+Les 1,1 Go restants ne sont **pas expliqués** (hypothèses ci-dessous, aucune n'est vérifiée).
+
+## 4. Garde-fous à mettre en place AVANT de modifier quoi que ce soit
+
+1. **Binaire de référence** : `cd /Users/coissac/Sync/travail/__MOI__/GO/obitools4 &&
+   GOEXPERIMENT=simd /usr/local/go/bin/go build -o /tmp/obiprimers.baseline ./cmd/obitools/obiprimers`
+2. **Données** (les fichiers de `/tmp` peuvent avoir disparu, regénère-les) : dossier du tutoriel
+   `D="$HOME/Documents/Enseignement/DNA metabarcoding spring school/ecoPrimers-Fish"` : `mito.vert.fasta.gz`,
+   `ncbitaxo_20250211.tgz`, et pour le test d'échelle `mito.all.fasta.gz` / `mito.one.fasta.gz`. Fichier annoté
+   (taxonomie embarquée, évite le chargeur NCBI) :
+   `/Users/coissac/travail/__MOI__/GO/obitools4/build/obiannotate -t "$D/ncbitaxo_20250211.tgz"
+   --taxonomic-path -Z -o /tmp/mito.vert.path.fasta.gz "$D/mito.vert.fasta.gz"`
+   (les binaires déjà compilés sont dans `/Users/coissac/travail/__MOI__/GO/obitools4/build/`, sans `Sync`).
+3. **CSV de référence**, à produire avec le binaire de référence et à conserver : Cyprinidae `-r 7953`,
+   Teleostei `-r 32443`, un cas avec contre-exemples `-r 7953 -i 1`, un cas `-U`, un cas `-R NC_083213`, et la
+   lecture sur entrée standard (`gzcat fichier | obiprimers ...`). Après chaque modification : `cmp` octet pour
+   octet.
+4. **Tests** : `go test ./pkg/obiprimer/ ./pkg/obitools/obiprimers/ ./pkg/obithermo/ ./pkg/obikmer/ -race
+   -count=1`, et les tests opt-in sur les données réelles (durent ~75 s hors `-race`) :
+   `OBIPRIMERS_TUTORIAL="$D" go test ./pkg/obiprimer/ -count=1`.
+5. **Méthode de mesure** (le piège principal) : le pic RSS bouge de ±100 Mo entre deux exécutions identiques
+   (rythme du ramasse-miettes). Mesure **au moins 5 exécutions entrelacées avant/après et compare les
+   médianes** (`/usr/bin/time -l` : octets sur macOS). Pour attribuer la mémoire, ne te fie pas au RSS : utilise
+   des profils de tas (`--pprof` de la CLI sert `/debug/pprof/heap` sur `localhost:6060`, `net/http/pprof` est
+   importé ; ou `runtime/pprof.WriteHeapProfile` aux frontières d'étapes ; `go tool pprof -sample_index=inuse_space`
+   et `alloc_space` ; `GODEBUG=gctrace=1`). **Erreur déjà commise (voir la section 16 bis des notes)** : en
+   mesurant le tas libéré par la suppression d'un attribut, la tranche de séquences n'était plus référencée
+   après la boucle et le GC a libéré *tout* ; utilise `runtime.KeepAlive` sur ce que tu veux garder vivant.
+
+## 5. Pistes à évaluer (toutes des HYPOTHÈSES à confirmer par un profil, classées sans ordre)
+
+- **Retard du ramasse-miettes** (`GOGC=100` laisse le tas monter jusqu'à ~2× le vivant). Pistes : limite
+  mémoire douce (`debug.SetMemoryLimit`) ou `debug.SetGCPercent` ajustés par la commande après la lecture ;
+  mesurer le coût CPU (l'ancienne mesure avec `GOGC=25` doublait le temps utilisateur).
+- **Lecture** (code partagé `obiformats`/`obiiter`) : 131 Mb sont analysés, avec une table d'attributs et des
+  chaînes JSON par séquence. L'option `--batch-mem` (lots limités en mémoire) existe déjà : l'évaluer.
+- **Séquences gardées** : `BioSequence` garde 1 octet par base et une table d'attributs. Ne garder que
+  l'identifiant, la séquence et le `taxid` (retirer les autres attributs, comme déjà fait pour
+  `taxonomic_path`, dans `dropTaxonomicPath` de `pkg/obitools/obiprimers/obiprimers.go`) ; ou stocker la
+  séquence à 2 bits par base (~33 Mo pour 131 Mb) avec une liste d'exceptions pour les bases ambiguës. Ce
+  second point touche `forEachOccurrence`, `ampHash`, `windowAsIs`, `windowRevComp` et `MatchWords`, qui lisent
+  les octets : à ne tenter que si le profil le justifie.
+- **Occurrences stockées deux fois** : `[]WordMatches` (`matchCorpus`, intermédiaire `hit` de 16 octets par
+  occurrence puis recopie) et `primerSet.perSeq` (`selectPrimers`). Vérifier à quel moment chacune est
+  libérée, construire directement les structures finales, présizer les tranches (les `append` en doublement
+  créent un pic transitoire).
+- **Phase 2b** (`collectAmplicons` sur les seules paires survivantes) : `ampRecord` de 24 octets (16 possibles
+  en empaquetant le rôle), enregistrements par worker ; **traitement des paires survivantes par lots bornés**
+  si le nombre d'enregistrements estimé dépasse un budget (idée notée en 25.2). Cas dur : `-T 0` (aucune paire
+  élaguée), où les tests atteignent ~5,5 Go de tas.
+- **`SeqKmerCounter`** (`pkg/obikmer/seqquorum*.go`) : environ 16 octets par occurrence de k-mer pour les
+  partitions en cours de traitement ; le nombre de partitions est **fixé à 256**, alors que le total de bases
+  est connu une fois le corpus chargé : le choisir automatiquement.
+- **Lecture en deux passes** pour la taxonomie embarquée (`readSequencesAndTaxonomy`) : la première passe ne
+  garde que les chemins, mais analyse tout.
+- **Chargeur de taxonomie NCBI** : 1,07 Go à lui seul ; c'est du code partagé (`obitax`), hors périmètre sauf
+  si tu y trouves une économie sûre ; sinon, documente-le dans l'aide (la taxonomie embarquée évite ce coût).
+
+## 6. Test d'échelle (jamais fait)
+
+Tout est validé sur des génomes mitochondriaux (5 à 131 Mb). Mesure la croissance sur un fichier plus gros
+(`mito.all.fasta.gz`, 172 Mo compressés, avec `--restrict-to-taxon` sur un grand clade, p. ex. les Metazoa) :
+temps et pic RSS en fonction de la taille, pour savoir ce qui grossit vite (corpus, occurrences, amplicons,
+compteur de k-mers).
+
+## 7. Livrables
+
+1. Un **rapport de profil** : où va la mémoire à chaque étape (lecture, sélection, mots stricts, appariement,
+   paires, phase 2a/2b), avec les profils de tas et les chiffres, avant toute modification.
+2. Les optimisations **classées par gain mesuré et par risque**, puis implémentées une par une, chacune avec
+   mesure avant/après (médianes d'au moins 5 exécutions entrelacées), `cmp` des CSV de référence, tests verts
+   sous `-race`.
+3. Une **nouvelle section** à la fin de `OBIPRIMERS_NOTES.md` (ne réécris pas l'historique) : mesures,
+   décisions, pistes écartées et pourquoi, ce qui reste. Pas de commit.
+4. Si une optimisation exige de changer un résultat, une décision ou un paquet partagé : **arrête-toi et
+   demande**.
+
+## 8. Pièges connus de l'environnement
+
+- zsh ne découpe pas une variable en arguments : utilise des tableaux (`A=(-r 7953 ...)` puis `"${A[@]}"`)
+  ou des scripts `bash`.
+- macOS : `sed -i ''` (sans suffixe) ; `/usr/bin/time -l` donne le pic en **octets** (`maximum resident set size`).
+- `OBIPRIMERS_WORKERS` n'est lue que par les tests opt-in ; la CLI utilise `--max-cpu`.
+- Les avertissements C de `obiformats/fastseq_read.h` sont connus et sans gravité.
+- `go test -race` sur les tests opt-in est lent (~90 s) ; réserve-le aux tests unitaires.
+- Le test `obiutils/TestSetString` était instable : il est corrigé (tri des membres dans `Set.String`).

Release-notes.md

@@ -2,6 +2,23 @@
 
 ## New changes
 
+### New features
+
+- New `obioligotag` command, a re-implementation of the `oligotag` command of
+  OBITools 2. It builds a family of oligonucleotides (tags) of at least
+  `--family-size` members that are pairwise at a distance of at least
+  `--distance`. Candidates are either generated (`--oligo-size`, up to 12) or
+  read from a file (`--oligo-list`, sequence file or one oligonucleotide per
+  line). They can be restricted by alphabet (`--alphabet`, for instance `ACT`
+  to forbid G), GC content, homopolymers and IUPAC patterns. The distance is
+  either the Hamming distance or the LCS-mode alignment distance
+  (`--distance-type`).
+
+- New functions in `obikmer` to compare and enumerate k-mers encoded as
+  `uint64`: `HammingDistance`, `KmerLCS` and `KmerLCSBatch` (SIMD when built with
+  `GOEXPERIMENT=simd`), `IterConstrainedKmers`, and in `obigraph` a clique search
+  (`FindClique`) that does not need the adjacency matrix.
+
 ### Bug fixes
 
 - In `obipairing` correct the misspelling of the `obiparing_*` tags where the `i`

cmd/obitools/obialign/main.go

@@ -0,0 +1,26 @@
+package main
+
+import (
+	"os"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obialign"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+)
+
+func main() {
+	optionParser := obioptions.GenerateOptionParser(
+		"obialign",
+		"pairwise similarity of nucleotide sequences, as a CSV table (the principle of sumatra)",
+		obialign.OptionSet)
+
+	_, args := optionParser(os.Args)
+
+	fs, err := obiconvert.CLIReadBioSequences(args...)
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
+
+	obialign.CLIWriteTable(obialign.CLIOBIAlign(fs))
+
+	obiutils.WaitForLastPipe()
+}

cmd/obitools/obiclust/main.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"os"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiclust"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+)
+
+func main() {
+	optionParser := obioptions.GenerateOptionParser(
+		"obiclust",
+		"star clustering of sequences (the algorithm of sumaclust, UCLUST and CD-HIT)",
+		obiclust.OptionSet)
+
+	_, args := optionParser(os.Args)
+
+	fs, err := obiconvert.CLIReadBioSequences(args...)
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
+
+	clustered := obiclust.CLIOBIClust(fs)
+
+	obiconvert.CLIWriteBioSequences(clustered, true)
+
+	obiutils.WaitForLastPipe()
+}

cmd/obitools/obioligotag/main.go

@@ -0,0 +1,33 @@
+package main
+
+import (
+	"os"
+
+	log "github.com/sirupsen/logrus"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obioligotag"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+)
+
+func main() {
+	optionParser := obioptions.GenerateOptionParser(
+		"obioligotag",
+		"designs a family of oligonucleotides (tags) at a minimum distance from each other",
+		obioligotag.OptionSet)
+
+	_, args := optionParser(os.Args)
+	if len(args) > 0 {
+		log.Fatalf("unexpected arguments %v: use --oligo-list to provide candidate oligonucleotides", args)
+	}
+
+	oligos, err := obioligotag.CLIOligoFamily()
+	if err != nil {
+		log.Fatalf("%v", err)
+	}
+
+	obiconvert.CLIWriteBioSequences(obioligotag.CLIOligosToIterator(oligos), true)
+
+	obiutils.WaitForLastPipe()
+}

cmd/obitools/obiprimers/main.go

@@ -0,0 +1,38 @@
+package main
+
+import (
+	"os"
+
+	log "github.com/sirupsen/logrus"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obicsv"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiprimers"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+)
+
+func main() {
+	obiprimers.SetReadingDefaults()
+
+	optionParser := obioptions.GenerateOptionParser(
+		"obiprimers",
+		"designs the primer pairs amplifying a set of example sequences and evaluates the DNA barcode they delimit. "+
+			"The taxonomy comes from the -t option, or is read from the taxonomic paths embedded in the sequences (obiannotate --taxonomic-path)",
+		obiprimers.OptionSet)
+
+	_, args := optionParser(os.Args)
+
+	sequences, taxonomy, err := obiprimers.CLIReadSequencesAndTaxonomy(args)
+	if err != nil {
+		log.Fatalf("Cannot read the sequences: %v", err)
+	}
+
+	pairs, err := obiprimers.CLIPrimerPairs(sequences, taxonomy)
+	if err != nil {
+		log.Fatalf("%v", err)
+	}
+
+	obicsv.CLICSVWriter(obiprimers.PairsToCSVIterator(pairs, obiprimers.CLIReference() != ""), true)
+
+	obiutils.WaitForLastPipe()
+}

doc/book/installation.qmd

@@ -8,7 +8,7 @@ All the sources of the [*OBITools4*](http://metabarcoding.org/obitools4) can be
 
 ## Prerequisites
 
-The *OBITools4* are developped using the [GO programming language](https://go.dev/), we stick to the latest version of the language, today the $1.21.4$. If you want to download and compile the sources yourself, you first need to install the corresponding compiler on your system. Some parts of the soft are also written in C, therefore a recent C compiler is also requested, GCC on Linux or Windows, the Developer Tools on Mac.
+The *OBITools4* are developped using the [GO programming language](https://go.dev/), we stick to the latest version of the language, today the $1.27$. If you want to download and compile the sources yourself, you first need to install the corresponding compiler on your system. Some computations (distances between k-mers) use the experimental `simd` package of Go: the `Makefile` compiles them with `GOEXPERIMENT=simd`, and the installation script uses a Go 1.27 compiler for that reason. The resulting programs run on any processor, a portable code being used when no SIMD instruction is available. Some parts of the soft are also written in C, therefore a recent C compiler is also requested, GCC on Linux or Windows, the Developer Tools on Mac.
 
 Whatever the installation you decide for, you will have to ensure that a C compiler is available on your system.
 

go.mod

@@ -1,6 +1,6 @@
 module git.metabarcoding.org/obitools/obitools4/obitools4
 
-go 1.26.1
+go 1.27
 
 require (
 	github.com/DavidGamba/go-getoptions v0.33.0

go.work

@@ -1,3 +1,3 @@
-go 1.26.1
+go 1.27
 
 use .

install_obitools.sh

@@ -2,6 +2,11 @@
 
 # Default values
 URL="https://go.dev/dl/"
+# Series of the Go compiler used to build OBITools4. The SIMD code relies on the
+# experimental simd package of Go (GOEXPERIMENT=simd), whose API may change
+# between Go versions: the build is therefore tied to the series the code was
+# written for rather than to the latest release.
+GO_SERIES="1.27"
 GITHUB_REPO="https://github.com/metabarcoding/obitools4"
 INSTALL_DIR="/usr/local"
 OBITOOLS_PREFIX=""
@@ -174,10 +179,16 @@ GOFILE=$(curl -s "$URL" \
             | grep 'class="download"' \
             | grep "\.tar\.gz" \
             | sed -E 's@^.*/dl/(go[1-9].+\.tar\.gz)".*$@\1@' \
+            | awk -v prefix="go${GO_SERIES}." 'index($0, prefix) == 1' \
             | grep -i "$OS" \
             | grep -i "$ARCH" \
             | head -1)
 
+if [ -z "$GOFILE" ]; then
+  echo "Error: no Go ${GO_SERIES} archive found for ${OS}/${ARCH} at ${URL}" 1>&2
+  exit 1
+fi
+
 GOURL=$(curl -s "${URL}${GOFILE}" \
         | sed -E 's@^.*href="(.*\.tar\.gz)".*$@\1@')
 
@@ -191,8 +202,12 @@ export PATH
 export GOPATH="$(pwd)/gopath"
 export GOCACHE="$(pwd)/cache"
 export GOTOOLCHAIN=local
+# OBITools4 is built with the SIMD code enabled (it falls back on a portable
+# code at run time if the processor has no SIMD instructions).
+export GOEXPERIMENT=simd
 
 echo "GOROOT=$GOROOT" 1>&2
+echo "GOEXPERIMENT=$GOEXPERIMENT" 1>&2
 echo "GOCACHE=$GOCACHE" 1>&2
 mkdir -p "$GOPATH" "$GOCACHE"
 

obitests/obitools/obialign/test.sh

@@ -0,0 +1,206 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obialign
+CMD=obialign
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1
+then
+    log "$MCMD: printing help OK"
+    ((success++))
+else
+    log "$MCMD: printing help failed"
+    ((failed++))
+fi
+
+
+#
+# Dataset used by the following tests: two families of similar sequences, a
+# sequence identical to another one, and a sequence with an ambiguous
+# nucleotide that is discarded.
+#
+cat > "${TMPDIR}/data.fasta" <<DATA
+>a1 count=20;
+acgtacgtacgtaacctagcctagcctagggatcgatcgatcgatcgaatcg
+>a2 count=3;
+acgtacgtacgtaacctagcctagcctagggatcgatcgatcgttcgaatcg
+>a3 count=2;
+acgtacgtacgtaacctagcctagcctagggatcgatcgatcgatcgaatcg
+>b1 count=15;
+ttgcatgcatgctagctagctagcatgcatcgatcgatgctagctagtagct
+>b2 count=1;
+ttgcatgcatgctagctagctagcatgcatcgatcgatgctagctagtaact
+>x1 count=4;
+ttgcatgcatgctangctagctagcatgcatcgatcgatgctagctagtagc
+DATA
+
+cat > "${TMPDIR}/ref.fasta" <<DATA
+>r1
+acgtacgtacgtaacctagcctagcctagggatcgatcgatcgatcgaatcg
+>r2
+ttgcatgcatgctagctagctagcatgcatcgatcgatgctagctagtagct
+>r3
+ggggccccggggccccggggccccggggccccggggccccggggcccc
+DATA
+
+((ntest++))
+if $CMD --max-cpu 2 "${TMPDIR}/data.fasta" > "${TMPDIR}/all.csv" 2> /dev/null &&
+   [ "$(head -n 1 "${TMPDIR}/all.csv")" = "id1,id2,score" ] &&
+   [ "$(tail -n +2 "${TMPDIR}/all.csv" | wc -l | tr -d ' ')" = "10" ]
+then
+    log "$MCMD: without threshold all the pairs of the 5 usable sequences are written OK"
+    ((success++))
+else
+    log "$MCMD: the table without threshold is wrong"
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD --max-cpu 2 -t 0.95 "${TMPDIR}/data.fasta" > "${TMPDIR}/t95.csv" 2> /dev/null &&
+   [ "$(tail -n +2 "${TMPDIR}/t95.csv" | cut -d, -f1,2 | tr '\n' ' ')" = "a1,a2 a1,a3 a2,a3 b1,b2 " ] &&
+   grep -q '^a1,a3,1$' "${TMPDIR}/t95.csv"
+then
+    log "$MCMD: only the pairs reaching the threshold are written, in order OK"
+    ((success++))
+else
+    log "$MCMD: the table with a threshold is wrong"
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD --max-cpu 2 -t 0.95 -x "${TMPDIR}/data.fasta" 2> /dev/null | head -n 2 | tail -n 1 | grep -q '^a1,a2,[0-9.]*,20,3,52,52$'
+then
+    log "$MCMD: extra columns OK"
+    ((success++))
+else
+    log "$MCMD: extra columns failed"
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD --max-cpu 2 -g -t 0.9 "${TMPDIR}/data.fasta" 2> /dev/null | grep -q '^b1,x1,'
+then
+    log "$MCMD: a sequence with an IUPAC letter is kept with --accept-ambiguous OK"
+    ((success++))
+else
+    log "$MCMD: --accept-ambiguous failed"
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD --max-cpu 2 -R "${TMPDIR}/ref.fasta" -t 0.95 "${TMPDIR}/data.fasta" 2> /dev/null | tail -n +2 | cut -d, -f1,2 | tr '\n' ' ' > "${TMPDIR}/two.txt" &&
+   [ "$(cat "${TMPDIR}/two.txt")" = "a1,r1 a2,r1 a3,r1 b1,r2 b2,r2 " ]
+then
+    log "$MCMD: comparison with a reference file OK"
+    ((success++))
+else
+    log "$MCMD: comparison with a reference file failed"
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD --max-cpu 1 "${TMPDIR}/data.fasta" 2> /dev/null > "${TMPDIR}/cpu1.csv" &&
+   $CMD --max-cpu 8 "${TMPDIR}/data.fasta" 2> /dev/null > "${TMPDIR}/cpu8.csv" &&
+   cmp -s "${TMPDIR}/cpu1.csv" "${TMPDIR}/cpu8.csv"
+then
+    log "$MCMD: the table does not depend on the number of threads OK"
+    ((success++))
+else
+    log "$MCMD: the table depends on the number of threads"
+    ((failed++))
+fi
+
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obiclust/test.sh

@@ -0,0 +1,188 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obiclust
+CMD=obiclust
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1
+then
+    log "$MCMD: printing help OK"
+    ((success++))
+else
+    log "$MCMD: printing help failed"
+    ((failed++))
+fi
+
+
+#
+# Dataset used by the following tests: two families with a rare variant each,
+# a duplicate, and a sequence with an ambiguous nucleotide that is discarded.
+#
+cat > "${TMPDIR}/data.fasta" <<DATA
+>a1 count=20;
+acgtacgtacgtaacctagcctagcctagggatcgatcgatcgatcgaatcg
+>a2 count=3;
+acgtacgtacgtaacctagcctagcctagggatcgatcgatcgttcgaatcg
+>a3 count=2;
+acgtacgtacgtaacctagcctagcctagggatcgatcgatcgatcgaatcg
+>b1 count=15;
+ttgcatgcatgctagctagctagcatgcatcgatcgatgctagctagtagct
+>b2 count=1;
+ttgcatgcatgctagctagctagcatgcatcgatcgatgctagctagtaact
+>x1 count=4;
+ttgcatgcatgctangctagctagcatgcatcgatcgatgctagctagtagc
+DATA
+
+((ntest++))
+if $CMD --max-cpu 2 -O -t 0.95 "${TMPDIR}/data.fasta" > "${TMPDIR}/clusters.fasta" 2> "${TMPDIR}/clusters.log"
+then
+    log "$MCMD: clustering runs OK"
+    ((success++))
+else
+    log "$MCMD: clustering failed"
+    ((failed++))
+fi
+
+((ntest++))
+if [ "$(grep -c '^>' "${TMPDIR}/clusters.fasta")" = "5" ]
+then
+    log "$MCMD: the sequence with an ambiguous nucleotide is discarded OK"
+    ((success++))
+else
+    log "$MCMD: wrong number of sequences in the output"
+    ((failed++))
+fi
+
+((ntest++))
+if [ "$(grep '^>a[23] ' "${TMPDIR}/clusters.fasta" | grep -c 'cluster=a1;')" = "2" ] &&
+   [ "$(grep '^>b2 ' "${TMPDIR}/clusters.fasta" | grep -c 'cluster=b1;')" = "1" ] &&
+   grep '^>b1 ' "${TMPDIR}/clusters.fasta" | grep -q 'cluster_center=true;' &&
+   grep '^>a1 ' "${TMPDIR}/clusters.fasta" | grep -q 'cluster_weight=25;'
+then
+    log "$MCMD: sequences are attached to the right center OK"
+    ((success++))
+else
+    log "$MCMD: sequences are not attached to the right center"
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD --max-cpu 2 -t 0.95 --otu-map "${TMPDIR}/otu.csv" --no-sequence-output "${TMPDIR}/data.fasta" > "${TMPDIR}/none.fasta" 2> /dev/null &&
+   [ ! -s "${TMPDIR}/none.fasta" ] &&
+   [ "$(head -n 1 "${TMPDIR}/otu.csv")" = "id,cluster,cluster_center,cluster_score,count" ] &&
+   [ "$(sed -n 2,4p "${TMPDIR}/otu.csv" | cut -d, -f1,2 | tr '\n' ' ')" = "a1,a1 a3,a1 a2,a1 " ] &&
+   [ "$(grep -c ',true,' "${TMPDIR}/otu.csv")" = "2" ]
+then
+    log "$MCMD: CSV table of the clusters OK"
+    ((success++))
+else
+    log "$MCMD: CSV table of the clusters failed"
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD --max-cpu 1 -O -t 0.95 "${TMPDIR}/data.fasta" 2> /dev/null | grep '^>' | sed 's/ .*//' > "${TMPDIR}/order1.txt" &&
+   $CMD --max-cpu 8 -O -t 0.95 "${TMPDIR}/data.fasta" 2> /dev/null | grep '^>' | sed 's/ .*//' > "${TMPDIR}/order8.txt" &&
+   cmp -s "${TMPDIR}/order1.txt" "${TMPDIR}/order8.txt"
+then
+    log "$MCMD: the result does not depend on the number of threads OK"
+    ((success++))
+else
+    log "$MCMD: the result depends on the number of threads"
+    ((failed++))
+fi
+
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obioligotag/test.sh

@@ -0,0 +1,205 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obioligotag
+CMD=obioligotag
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+#
+# Every result is checked on its properties (size, alphabet, constraints),
+# not against a reference file, because several equivalent families exist.
+#
+
+printf 'acgtac\nacgtat\nttttgg\nggcaac\ncaccat\nacgtac\n' > "${TMPDIR}/oligos.txt"
+printf '>a\nACGTAC\n>b\nACGTAT\n>c\nTTTTGG\n>d\nGGCAAC\n>e\nCACCAT\n' > "${TMPDIR}/oligos.fasta"
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1 
+then
+    log "$MCMD: printing help OK" 
+    ((success++))
+else
+    log "$MCMD: printing help failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD -s 5 -f 24 -d 3 -g 3 -p 2 > "${TMPDIR}/tags.fasta" 2> /dev/null \
+   && [ "$(grep -vc '>' "${TMPDIR}/tags.fasta")" -ge 24 ] \
+   && ! grep -v '>' "${TMPDIR}/tags.fasta" | grep -qE '(a{3}|c{3}|g{3}|t{3})'
+then
+    log "$MCMD: generating a family with homopolymer constraint OK" 
+    ((success++))
+else
+    log "$MCMD: generating a family with homopolymer constraint failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD -s 7 -f 60 -d 3 --alphabet ACT > "${TMPDIR}/noG.fasta" 2> /dev/null \
+   && [ "$(grep -vc '>' "${TMPDIR}/noG.fasta")" -ge 60 ] \
+   && ! grep -v '>' "${TMPDIR}/noG.fasta" | grep -q 'g'
+then
+    log "$MCMD: restricted alphabet OK" 
+    ((success++))
+else
+    log "$MCMD: restricted alphabet failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD -s 6 -f 20 -d 3 --distance-type lcs > "${TMPDIR}/lcs.fasta" 2> /dev/null \
+   && [ "$(grep -vc '>' "${TMPDIR}/lcs.fasta")" -ge 20 ]
+then
+    log "$MCMD: LCS distance OK" 
+    ((success++))
+else
+    log "$MCMD: LCS distance failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD -s 7 -f 40 -d 3 -r cnnnnnn > "${TMPDIR}/rejected.fasta" 2> /dev/null \
+   && [ "$(grep -vc '>' "${TMPDIR}/rejected.fasta")" -ge 40 ] \
+   && ! grep -v '>' "${TMPDIR}/rejected.fasta" | grep -q '^c'
+then
+    log "$MCMD: rejected pattern OK" 
+    ((success++))
+else
+    log "$MCMD: rejected pattern failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD -L "${TMPDIR}/oligos.txt" -f 3 -d 3 > "${TMPDIR}/from_text.fasta" 2> /dev/null \
+   && [ "$(grep -vc '>' "${TMPDIR}/from_text.fasta")" -ge 3 ]
+then
+    log "$MCMD: selecting oligonucleotides from a text file OK" 
+    ((success++))
+else
+    log "$MCMD: selecting oligonucleotides from a text file failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD -L "${TMPDIR}/oligos.fasta" -f 3 -d 3 > "${TMPDIR}/from_fasta.fasta" 2> /dev/null \
+   && [ "$(grep -vc '>' "${TMPDIR}/from_fasta.fasta")" -ge 3 ]
+then
+    log "$MCMD: selecting oligonucleotides from a fasta file OK" 
+    ((success++))
+else
+    log "$MCMD: selecting oligonucleotides from a fasta file failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if ! $CMD -s 5 -f 100 -d 3 > /dev/null 2>&1
+then
+    log "$MCMD: family size beyond the Hamming bound refused OK" 
+    ((success++))
+else
+    log "$MCMD: family size beyond the Hamming bound not refused" 
+    ((failed++))
+fi
+
+((ntest++))
+if ! $CMD -s 5 -L "${TMPDIR}/oligos.txt" -f 3 > /dev/null 2>&1
+then
+    log "$MCMD: --oligo-size with --oligo-list refused OK" 
+    ((success++))
+else
+    log "$MCMD: --oligo-size with --oligo-list not refused" 
+    ((failed++))
+fi
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

pkg/obialign/fastlcs_bench_test.go

@@ -0,0 +1,111 @@
+package obialign
+
+import (
+	"fmt"
+	"math/rand"
+	"testing"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+// randomDNA returns a random ACGT sequence of length n.
+func randomDNA(rng *rand.Rand, n int) []byte {
+	const nuc = "ACGT"
+	s := make([]byte, n)
+	for i := range s {
+		s[i] = nuc[rng.Intn(4)]
+	}
+	return s
+}
+
+// mutate returns a copy of s in which each position is, with probability
+// rate, substituted, deleted or followed by an inserted base (equiprobable).
+func mutate(rng *rand.Rand, s []byte, rate float64) []byte {
+	const nuc = "ACGT"
+	out := make([]byte, 0, len(s)+len(s)/10)
+	for _, c := range s {
+		if rng.Float64() >= rate {
+			out = append(out, c)
+			continue
+		}
+		switch rng.Intn(3) {
+		case 0:
+			out = append(out, nuc[rng.Intn(4)])
+		case 1: // deletion
+		case 2:
+			out = append(out, c, nuc[rng.Intn(4)])
+		}
+	}
+	return out
+}
+
+// benchPairs builds a fixed set of pairs. "similar" pairs differ by about
+// 2% of mutated positions, "random" pairs are unrelated sequences.
+func benchPairs(n, length int, similar bool) [][2]*obiseq.BioSequence {
+	rng := rand.New(rand.NewSource(42))
+	pairs := make([][2]*obiseq.BioSequence, n)
+	for i := range pairs {
+		a := randomDNA(rng, length)
+		var b []byte
+		if similar {
+			b = mutate(rng, a, 0.02)
+		} else {
+			b = randomDNA(rng, length)
+		}
+		pairs[i] = [2]*obiseq.BioSequence{
+			obiseq.NewBioSequence("a", a, ""),
+			obiseq.NewBioSequence("b", b, ""),
+		}
+	}
+	return pairs
+}
+
+// BenchmarkLegacyLCS measures the historical implementation (the one
+// FastLCSScore used before BandedLCS), kept as the "before" reference.
+func BenchmarkLegacyLCS(b *testing.B) {
+	for _, similar := range []bool{true, false} {
+		kind := "random"
+		if similar {
+			kind = "similar"
+		}
+		for _, length := range []int{100, 500, 2000} {
+			pairs := benchPairs(64, length, similar)
+			for _, maxErr := range []int{-1, 5, length / 20, length / 5} {
+				name := fmt.Sprintf("%s/len=%d/maxErr=%d", kind, length, maxErr)
+				b.Run(name, func(b *testing.B) {
+					var buffer []uint64
+					b.ReportAllocs()
+					b.ResetTimer()
+					for i := 0; i < b.N; i++ {
+						p := pairs[i%len(pairs)]
+						FastLCSEGFScoreByte(p[0].Sequence(), p[1].Sequence(), maxErr, false, &buffer)
+					}
+				})
+			}
+		}
+	}
+}
+
+// BenchmarkBandedLCS runs on the same pairs as BenchmarkLegacyLCS.
+func BenchmarkBandedLCS(b *testing.B) {
+	for _, similar := range []bool{true, false} {
+		kind := "random"
+		if similar {
+			kind = "similar"
+		}
+		for _, length := range []int{100, 500, 2000} {
+			pairs := benchPairs(64, length, similar)
+			for _, maxErr := range []int{-1, 5, length / 20, length / 5} {
+				name := fmt.Sprintf("%s/len=%d/maxErr=%d", kind, length, maxErr)
+				b.Run(name, func(b *testing.B) {
+					b.ReportAllocs()
+					b.ResetTimer()
+					for i := 0; i < b.N; i++ {
+						p := pairs[i%len(pairs)]
+						BandedLCS(p[0].Sequence(), p[1].Sequence(), maxErr)
+					}
+				})
+			}
+		}
+	}
+}

pkg/obialign/fastlcsegf.go

@@ -2,6 +2,7 @@ package obialign
 
 import (
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+	log "github.com/sirupsen/logrus"
 )
 
 // FastLCSEGFScoreByte calculates the score of the Longest Common Subsequence (LCS) between two byte slices.
@@ -355,27 +356,35 @@ func FastLCSEGFScore(seqA, seqB *obiseq.BioSequence, maxError int, buffer *[]uin
 	return FastLCSEGFScoreByte(seqA.Sequence(), seqB.Sequence(), maxError, true, buffer)
 }
 
-// FastLCSScore calculates the score of the longest common subsequence between two bio sequences.
+// FastLCSScore calculates the score of the longest common subsequence between two bio sequences,
+// and the length of the shortest alignment achieving it (the distance is alilen - lcs).
 //
-// if maxError > 0, the maximum allowed error between the sequences is maxError.
+// If maxError >= 0, the result is exact when its distance is at most maxError,
-// Otherwise, no error checking is done.
+// and (-1, -1) is returned otherwise. If maxError < 0, no limit is applied.
-// If the actual number of errors is larger than maxError, -1 is returned for both values.
+// See BandedLCS, which does the work.
 //
 // The score matrix is:
-//   - Matching: 1
+//   - Matching: 1 (bases are compared with the IUPAC rules)
 //   - Mismatch or gap: 0
 //
+// The program stops with a fatal error if a sequence is longer than
+// MaxBandedLCSLength.
+//
 // Parameters:
 // - seqA: The first bio sequence.
 // - seqB: The second bio sequence.
 // - maxError: The maximum allowed error between the sequences. If set to -1, no limit is applied.
-// - buffer: A pointer to a uint64 slice to store intermediate results. If nil, a new slice is created.
+// - buffer: Unused, kept for compatibility; the working memory is pooled.
 //
 // Returns:
 // - The score of the longest common subsequence.
 // - The length of the shortest alignment corresponding to the LCS.
 func FastLCSScore(seqA, seqB *obiseq.BioSequence, maxError int, buffer *[]uint64) (int, int) {
-	score, alilen, _ := FastLCSEGFScoreByte(seqA.Sequence(), seqB.Sequence(), maxError, false, buffer)
+	score, alilen, err := BandedLCS(seqA.Sequence(), seqB.Sequence(), maxError)
+	if err != nil {
+		log.Fatalf("FastLCSScore on sequences %s (%d bp) and %s (%d bp): %v (maximum %d bp)",
+			seqA.Id(), seqA.Len(), seqB.Id(), seqB.Len(), err, MaxBandedLCSLength)
+	}
 
 	return score, alilen
 }

pkg/obialign/lcsband.go

@@ -0,0 +1,185 @@
+package obialign
+
+import (
+	"errors"
+	"sync"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+// MaxBandedLCSLength is the longest sequence BandedLCS accepts. The score and
+// the alignment length are packed on 16 bits each, and the alignment length
+// can reach the sum of both sequence lengths.
+const MaxBandedLCSLength = 1<<15 - 1
+
+// ErrSequenceTooLong is returned by BandedLCS when a sequence is longer than
+// MaxBandedLCSLength.
+var ErrSequenceTooLong = errors.New("obialign: sequence too long for BandedLCS")
+
+// A DP cell is packed on 32 bits as
+//
+//	score<<16 | (lcsLenBase - alignment length)
+//
+// so that the maximum of two cells ranks them by score first and, for equal
+// scores, by shortest alignment. The three moves then reduce to
+//
+//	cell(i,j) = max(diag + match<<16, up, left) - 1
+//
+// since every move lengthens the alignment by one and only a diagonal match
+// adds one to the score. The value 0 stands for an unavailable cell (outside
+// the band): it loses against any real cell. Every cell of the band has at
+// least one real neighbour (the band is connected and contains (0,0)), so
+// the subtraction never wraps for a cell that is used; the scalar reference
+// still clamps the maximum to 1 as a safety net.
+const (
+	lcsScoreUnit = uint32(1) << 16
+	lcsLenBase   = uint32(0xFFFF)
+)
+
+// bandedLCSWorkspace holds the scratch memory of one alignment.
+type bandedLCSWorkspace struct {
+	aRev, bCode []uint32    // encoded bases (aRev is reversed), SIMD kernel only
+	rows        [3][]uint32 // antidiagonal (SIMD) or row (scalar) buffers
+	aRev16      []uint16    // 16-bit encoded bases of the score-only kernel
+	bCode16     []uint16
+	rows16      [3][]uint16
+	widened     int // number of alignments that needed a second pass
+}
+
+var bandedLCSPool = sync.Pool{New: func() any { return new(bandedLCSWorkspace) }}
+
+func growUint32(s []uint32, n int) []uint32 {
+	if cap(s) < n {
+		return make([]uint32, n)
+	}
+	return s[:n]
+}
+
+// iupacMask gives for each byte its 4-bit nucleotide set (A=1, C=2, G=4,
+// T=8), or 0 when the byte is not an IUPAC letter. For two bytes of non-zero
+// mask, SameIUPACNuc(a, b) is exactly (mask[a] & mask[b]) != 0.
+var iupacMask = buildIUPACMask()
+
+func buildIUPACMask() (m [256]uint8) {
+	probes := [4]byte{'a', 'c', 'g', 't'}
+	for c := 0; c < 256; c++ {
+		if !(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z') {
+			continue
+		}
+		for bit, p := range probes {
+			if obiseq.SameIUPACNuc(byte(c), p) {
+				m[c] |= 1 << uint(bit)
+			}
+		}
+	}
+	return m
+}
+
+// allRegularIUPAC reports whether every base has a non-zero IUPAC mask, in
+// which case base compatibility is a plain AND of masks.
+func allRegularIUPAC(s []byte) bool {
+	for _, c := range s {
+		if iupacMask[c] == 0 {
+			return false
+		}
+	}
+	return true
+}
+
+// bandOffsets returns the diagonal offsets k = j - i covered by a band of
+// half-width g around the diagonals 0..delta, with sequence A (columns j) not
+// shorter than sequence B (rows i), clipped to the matrix.
+func bandOffsets(lA, lB, g int) (kmin, kmax int) {
+	return max(-g, -lB), min(lA-lB+g, lA)
+}
+
+// BandedLCS computes the score of the longest common subsequence of two
+// nucleotide sequences, and the length of the shortest alignment achieving
+// it (match: +1, mismatch and gap: 0; bases are compared with the IUPAC
+// rules of obiseq.SameIUPACNuc). The distance is alilen - lcs.
+//
+// If maxError >= 0, the result is the optimal (lcs, alilen) when its distance
+// is at most maxError, and (-1, -1) otherwise. This is exact, not a
+// heuristic: the DP is restricted to a band, and a bound on what a path
+// leaving the band can score proves when the band was wide enough. maxError < 0
+// means no limit.
+//
+// An error is returned when a sequence is longer than MaxBandedLCSLength.
+func BandedLCS(seqA, seqB []byte, maxError int) (lcs, alilen int, err error) {
+	if len(seqA) > MaxBandedLCSLength || len(seqB) > MaxBandedLCSLength {
+		return -1, -1, ErrSequenceTooLong
+	}
+	if len(seqA) < len(seqB) {
+		seqA, seqB = seqB, seqA
+	}
+	ws := bandedLCSPool.Get().(*bandedLCSWorkspace)
+	lcs, alilen = bandedLCS(seqA, seqB, maxError, ws, haveSIMD)
+	bandedLCSPool.Put(ws)
+	return lcs, alilen, nil
+}
+
+// bandedLCS expects len(a) >= len(b), both within MaxBandedLCSLength.
+//
+// With delta = lA - lB and a band of half-width g (offsets -g .. delta+g):
+//   - an alignment with at most E errors needs at least delta+2h gaps to reach
+//     the offset delta+h or -h, so it lies in the band of g0 = (E-delta)/2;
+//   - a path leaving a band of half-width g makes at least g+1 gaps on the B
+//     side, hence scores at most lB-g-1 matches, and has more than E errors
+//     as soon as g >= g0.
+//
+// Let Q be the best path of the band g0. If its distance exceeds E, the
+// global optimum cannot be within E either (it would be in the band and be
+// Q): the answer is -1. Otherwise, if score(Q) >= lB-g0-1 no outside path can
+// beat Q (a tie in score loses on length, the outside path having more errors
+// than Q): Q is the global optimum. Else the band is widened once to
+// g1 = min(E-delta, lB-score(Q)-1); since score(Q) >= lA-E for any alignment
+// within E errors, g1 always certifies the result.
+func bandedLCS(a, b []byte, maxError int, ws *bandedLCSWorkspace, useSIMD bool) (int, int) {
+	lA, lB := len(a), len(b)
+	delta := lA - lB
+	if maxError >= 0 && maxError < delta {
+		return -1, -1
+	}
+	if lB == 0 {
+		return 0, lA
+	}
+	simdOK := useSIMD && allRegularIUPAC(a) && allRegularIUPAC(b)
+	if simdOK {
+		encodeBases(ws, a, b)
+	}
+	run := func(g int) (int, int) {
+		kmin, kmax := bandOffsets(lA, lB, g)
+		return ws.align(a, b, kmin, kmax, simdOK)
+	}
+
+	if maxError < 0 {
+		return run(lA)
+	}
+	g0 := (maxError - delta) / 2
+	lcs, alilen := run(g0)
+	if alilen-lcs > maxError {
+		return -1, -1
+	}
+	if lcs >= lB-g0-1 {
+		return lcs, alilen
+	}
+	ws.widened++
+	lcs, alilen = run(min(maxError-delta, lB-lcs-1))
+	if alilen-lcs > maxError {
+		return -1, -1
+	}
+	return lcs, alilen
+}
+
+// align runs the banded DP on the offsets [kmin, kmax] (kmin <= 0 <= kmax) and
+// returns the best (lcs, alilen) of the band. The bases must have been encoded
+// with encodeBases when simdOK is set.
+func (ws *bandedLCSWorkspace) align(a, b []byte, kmin, kmax int, simdOK bool) (int, int) {
+	var key uint32
+	if simdOK {
+		key = bandedLCSSIMD(len(a), len(b), kmin, kmax, ws)
+	} else {
+		key = bandedLCSScalar(a, b, kmin, kmax, ws)
+	}
+	return int(key >> 16), int(lcsLenBase - key&0xFFFF)
+}

pkg/obialign/lcsband_nosimd.go

@@ -0,0 +1,17 @@
+//go:build !goexperiment.simd
+
+package obialign
+
+const haveSIMD = false
+
+// encodeBases and bandedLCSSIMD are never called without GOEXPERIMENT=simd
+// (haveSIMD is false); they exist to keep the dispatcher uniform.
+func encodeBases(ws *bandedLCSWorkspace, a, b []byte) {}
+
+func bandedLCSSIMD(lA, lB, kmin, kmax int, ws *bandedLCSWorkspace) uint32 {
+	return 0
+}
+
+func lcsScoreSIMD(a, b []byte, kmin, kmax, minLCS int, ws *bandedLCSWorkspace) int {
+	return -1
+}

pkg/obialign/lcsband_scalar.go

@@ -0,0 +1,53 @@
+package obialign
+
+import "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+
+// bandedLCSScalar is the portable reference of the banded DP, row by row. It
+// returns the packed cell (lB, lA). len(a) >= len(b) >= 1.
+//
+// Cells outside the band hold 0 (unavailable). Row i spans the columns
+// [i+kmin, i+kmax] clipped to [1, lA]; the cell just left of that span and
+// the one just right of it are reset so that the next row never reads a stale
+// value.
+func bandedLCSScalar(a, b []byte, kmin, kmax int, ws *bandedLCSWorkspace) uint32 {
+	lA, lB := len(a), len(b)
+
+	prev := growUint32(ws.rows[0], lA+2)
+	cur := growUint32(ws.rows[1], lA+2)
+	ws.rows[0], ws.rows[1] = prev, cur
+
+	for j := 0; j <= lA; j++ {
+		prev[j] = 0
+		if j <= kmax {
+			prev[j] = lcsLenBase - uint32(j)
+		}
+	}
+	prev[lA+1] = 0
+
+	for i := 1; i <= lB; i++ {
+		if -i >= kmin {
+			cur[0] = lcsLenBase - uint32(i)
+		} else {
+			cur[0] = 0
+		}
+		jlo := max(1, i+kmin)
+		jhi := min(lA, i+kmax)
+		if jlo > 1 {
+			cur[jlo-1] = 0
+		}
+		bi := b[i-1]
+		for j := jlo; j <= jhi; j++ {
+			d := prev[j-1]
+			if obiseq.SameIUPACNuc(a[j-1], bi) {
+				d += lcsScoreUnit
+			}
+			m := max(d, prev[j], cur[j-1], 1)
+			cur[j] = m - 1
+		}
+		if jhi < lA {
+			cur[jhi+1] = 0
+		}
+		prev, cur = cur, prev
+	}
+	return prev[lA]
+}

pkg/obialign/lcsband_simd.go

@@ -0,0 +1,107 @@
+//go:build goexperiment.simd
+
+package obialign
+
+import "simd"
+
+const haveSIMD = true
+
+func simdLanes() int {
+	var probe simd.Uint32s
+	return probe.Len()
+}
+
+// encodeIUPAC writes in dst the 4-bit IUPAC mask of each base of s, reversed
+// if rev is set, followed by zero padding up to len(dst).
+func encodeIUPAC(dst []uint32, s []byte, rev bool) {
+	n := len(s)
+	for x := range dst {
+		dst[x] = 0
+	}
+	if rev {
+		for x, c := range s {
+			dst[n-1-x] = uint32(iupacMask[c])
+		}
+		return
+	}
+	for x, c := range s {
+		dst[x] = uint32(iupacMask[c])
+	}
+}
+
+// encodeBases encodes both sequences once for all the passes of an alignment.
+func encodeBases(ws *bandedLCSWorkspace, a, b []byte) {
+	vl := simdLanes()
+	ws.aRev = growUint32(ws.aRev, len(a)+vl)
+	ws.bCode = growUint32(ws.bCode, len(b)+vl)
+	encodeIUPAC(ws.aRev, a, true)
+	encodeIUPAC(ws.bCode, b, false)
+}
+
+// bandedLCSSIMD runs the banded DP along antidiagonals t = i + j. Lane i of
+// the buffer of antidiagonal t holds the cell (i, t-i), so that
+//
+//	diag = buffer[t-2][i-1]   up = buffer[t-1][i-1]   left = buffer[t-1][i]
+//
+// are plain unaligned loads. Along an antidiagonal the offset j-i = t-2i
+// must stay in [kmin, kmax], which gives the window [lo, hi] of rows. The
+// vector loop may run past hi; the lanes beyond are then reset to 0
+// (unavailable), as is the lane just before the window, because the next
+// antidiagonals read them as neighbours. Lane 0 and lane t carry the first
+// row and the first column of the matrix.
+//
+// Requires encodeBases to have been called, lA >= lB >= 1 and kmin <= 0 <=
+// kmax.
+func bandedLCSSIMD(lA, lB, kmin, kmax int, ws *bandedLCSWorkspace) uint32 {
+	vl := simdLanes()
+	aRev, bCode := ws.aRev, ws.bCode
+
+	size := lB + 2*vl + 2
+	for r := range ws.rows {
+		ws.rows[r] = growUint32(ws.rows[r], size)
+		clear(ws.rows[r])
+	}
+	p2, p1, cur := ws.rows[0], ws.rows[1], ws.rows[2]
+
+	p2[0] = lcsLenBase // cell (0,0)
+	if kmax >= 1 {
+		p1[0] = lcsLenBase - 1 // cell (0,1)
+	}
+	if kmin <= -1 {
+		p1[1] = lcsLenBase - 1 // cell (1,0)
+	}
+
+	zero := simd.BroadcastUint32s(0)
+	one := simd.BroadcastUint32s(1)
+	unit := simd.BroadcastUint32s(lcsScoreUnit)
+
+	for t := 2; t <= lA+lB; t++ {
+		lo := max(1, t-lA, (t-kmax+1)>>1)
+		hi := min(lB, t-1, (t-kmin)>>1)
+
+		for i := lo; i <= hi; i += vl {
+			ca := simd.LoadUint32s(aRev[lA-t+i:])
+			cb := simd.LoadUint32s(bCode[i-1:])
+			inc := unit.Masked(ca.And(cb).NotEqual(zero))
+			d := simd.LoadUint32s(p2[i-1:]).Add(inc)
+			u := simd.LoadUint32s(p1[i-1:])
+			l := simd.LoadUint32s(p1[i:])
+			d.Max(u).Max(l).Sub(one).Store(cur[i:])
+		}
+
+		zero.Store(cur[hi+1:])
+		if lo > 1 {
+			cur[lo-1] = 0
+		}
+		cur[0] = 0
+		if t <= lA && t <= kmax {
+			cur[0] = lcsLenBase - uint32(t)
+		}
+		if t <= lB && -t >= kmin {
+			cur[t] = lcsLenBase - uint32(t)
+		}
+
+		p2, p1, cur = p1, cur, p2
+	}
+	return p1[lB]
+}

pkg/obialign/lcsband_test.go

@@ -0,0 +1,360 @@
+package obialign
+
+import (
+	"math/rand"
+	"testing"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+// bandDP is the oracle: a plain DP over the whole matrix, written
+// independently of the production code, in which only the cells with a
+// diagonal offset j-i in [kmin, kmax] exist (kmin=-lB, kmax=lA when
+// unbounded). It returns the optimal LCS score and the shortest alignment
+// length achieving it among the alignments staying in the band.
+func bandDP(a, b []byte, kmin, kmax int) (int, int) {
+	lA, lB := len(a), len(b)
+	type cell struct {
+		s, l int
+		ok   bool
+	}
+	better := func(x, y cell) bool {
+		return !y.ok || (x.ok && (x.s > y.s || (x.s == y.s && x.l < y.l)))
+	}
+	m := make([][]cell, lB+1)
+	for i := range m {
+		m[i] = make([]cell, lA+1)
+	}
+	for i := 0; i <= lB; i++ {
+		for j := 0; j <= lA; j++ {
+			if k := j - i; k < kmin || k > kmax {
+				continue
+			}
+			if i == 0 || j == 0 {
+				m[i][j] = cell{0, i + j, true}
+				continue
+			}
+			var best cell
+			if d := m[i-1][j-1]; d.ok {
+				best = cell{d.s, d.l + 1, true}
+				if obiseq.SameIUPACNuc(a[j-1], b[i-1]) {
+					best.s++
+				}
+			}
+			if u := m[i-1][j]; u.ok && better(cell{u.s, u.l + 1, true}, best) {
+				best = cell{u.s, u.l + 1, true}
+			}
+			if l := m[i][j-1]; l.ok && better(cell{l.s, l.l + 1, true}, best) {
+				best = cell{l.s, l.l + 1, true}
+			}
+			m[i][j] = best
+		}
+	}
+	return m[lB][lA].s, m[lB][lA].l
+}
+
+// fullDP is the unbanded optimum.
+func fullDP(a, b []byte) (int, int) { return bandDP(a, b, -len(b), len(a)) }
+
+// expected is the contract of BandedLCS: the unbounded optimum when it has at
+// most maxError errors, (-1,-1) otherwise.
+func expected(a, b []byte, maxError int) (int, int) {
+	s, l := fullDP(a, b)
+	if maxError >= 0 && l-s > maxError {
+		return -1, -1
+	}
+	return s, l
+}
+
+func randomFrom(rng *rand.Rand, alphabet string, n int) []byte {
+	s := make([]byte, n)
+	for i := range s {
+		s[i] = alphabet[rng.Intn(len(alphabet))]
+	}
+	return s
+}
+
+func mutateFrom(rng *rand.Rand, alphabet string, s []byte, rate float64) []byte {
+	out := make([]byte, 0, len(s)+8)
+	for _, c := range s {
+		if rng.Float64() >= rate {
+			out = append(out, c)
+			continue
+		}
+		switch rng.Intn(3) {
+		case 0:
+			out = append(out, alphabet[rng.Intn(len(alphabet))])
+		case 1:
+		case 2:
+			out = append(out, c, alphabet[rng.Intn(len(alphabet))])
+		}
+	}
+	return out
+}
+
+var testAlphabets = []struct {
+	name, letters string
+	regular       bool // only IUPAC letters of non-zero mask
+}{
+	{"acgt", "ACGT", true},
+	{"low-complexity", "AC", true},
+	{"mixed-case", "ACGTacgt", true},
+	{"iupac", "ACGTRYSWKMBDHVNUacgtrywsn", true},
+	{"exotic", "ACGT-.EXz*", false},
+}
+
+// TestBandedLCSDifferential checks the scalar reference against the oracle,
+// and the dispatcher (SIMD kernel when built with GOEXPERIMENT=simd) against
+// the scalar reference, for random pairs of unequal lengths and many bands.
+func TestBandedLCSDifferential(t *testing.T) {
+	rng := rand.New(rand.NewSource(7))
+	ws := new(bandedLCSWorkspace)
+	for _, alpha := range testAlphabets {
+		for it := 0; it < 3000; it++ {
+			lA := 1 + rng.Intn(90)
+			a := randomFrom(rng, alpha.letters, lA)
+			var b []byte
+			if rng.Intn(3) > 0 {
+				b = mutateFrom(rng, alpha.letters, a, 0.02+rng.Float64()*0.5)
+			} else {
+				b = randomFrom(rng, alpha.letters, 1+rng.Intn(90))
+			}
+			if len(b) == 0 {
+				continue
+			}
+			if len(a) < len(b) {
+				a, b = b, a
+			}
+			delta := len(a) - len(b)
+			for _, maxError := range []int{-1, 0, 1, 2, 3, 5, 8, 13, 21, 40, 200, delta, delta + 1, delta + 2, delta + 3} {
+				ws, wl := expected(a, b, maxError)
+				ss, sl := bandedLCS(a, b, maxError, new(bandedLCSWorkspace), false)
+				if ss != ws || sl != wl {
+					t.Fatalf("%s scalar maxError=%d a=%q b=%q: got (%d,%d), oracle (%d,%d)",
+						alpha.name, maxError, a, b, ss, sl, ws, wl)
+				}
+				ds, dl := bandedLCS(a, b, maxError, new(bandedLCSWorkspace), haveSIMD)
+				if ds != ss || dl != sl {
+					t.Fatalf("%s dispatch(simd=%v) maxError=%d a=%q b=%q: got (%d,%d), scalar (%d,%d)",
+						alpha.name, haveSIMD, maxError, a, b, ds, dl, ss, sl)
+				}
+			}
+		}
+	}
+	_ = ws
+}
+
+// TestBandedLCSLongPairs covers lengths well above the vector width, with
+// reused workspaces (stale buffers must never leak between calls).
+func TestBandedLCSLongPairs(t *testing.T) {
+	rng := rand.New(rand.NewSource(11))
+	ws := new(bandedLCSWorkspace)
+	for it := 0; it < 60; it++ {
+		a := randomFrom(rng, "ACGT", 150+rng.Intn(500))
+		b := mutateFrom(rng, "ACGT", a, 0.01+rng.Float64()*0.2)
+		if len(a) < len(b) {
+			a, b = b, a
+		}
+		for _, maxError := range []int{-1, 4, 20, 60, 150} {
+			wantS, wantL := expected(a, b, maxError)
+			gotS, gotL := bandedLCS(a, b, maxError, ws, haveSIMD)
+			if gotS != wantS || gotL != wantL {
+				t.Fatalf("maxError=%d lens=%d/%d: got (%d,%d), oracle (%d,%d)",
+					maxError, len(a), len(b), gotS, gotL, wantS, wantL)
+			}
+		}
+	}
+}
+
+// TestBandedLCSMatchesLegacy compares with the historical implementation
+// (FastLCSEGFScoreByte, not end-gap-free) wherever that one is exact: no
+// limit, or a result within the limit.
+func TestBandedLCSMatchesLegacy(t *testing.T) {
+	rng := rand.New(rand.NewSource(3))
+	for it := 0; it < 4000; it++ {
+		a := randomFrom(rng, "ACGTRYN", 1+rng.Intn(120))
+		b := mutateFrom(rng, "ACGTRYN", a, rng.Float64()*0.4)
+		if len(b) == 0 {
+			continue
+		}
+		for _, maxError := range []int{-1, 2, 7, 25} {
+			ls, ll, err := BandedLCS(a, b, maxError)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if ls < 0 {
+				continue
+			}
+			os, ol, _ := FastLCSEGFScoreByte(a, b, maxError, false, nil)
+			if os != ls || ol != ll {
+				t.Fatalf("maxError=%d a=%q b=%q: BandedLCS (%d,%d), legacy (%d,%d)",
+					maxError, a, b, ls, ll, os, ol)
+			}
+		}
+	}
+}
+
+func TestBandedLCSEdgeCases(t *testing.T) {
+	cases := []struct {
+		a, b     string
+		maxError int
+		s, l     int
+	}{
+		{"", "", -1, 0, 0},
+		{"ACGT", "", -1, 0, 4},
+		{"ACGT", "", 3, -1, -1},
+		{"ACGT", "", 4, 0, 4},
+		{"ACGT", "ACGT", 0, 4, 4},
+		{"ACGT", "ACGA", 0, -1, -1},
+		{"ACGT", "ACGA", 1, 3, 4},
+		{"ACGTT", "ACGT", 0, -1, -1}, // length difference above the limit
+		{"ACGTT", "ACGT", 1, 4, 5},
+	}
+	for _, c := range cases {
+		s, l, err := BandedLCS([]byte(c.a), []byte(c.b), c.maxError)
+		if err != nil || s != c.s || l != c.l {
+			t.Errorf("%q/%q maxError=%d: got (%d,%d,%v), want (%d,%d)", c.a, c.b, c.maxError, s, l, err, c.s, c.l)
+		}
+		// the arguments are symmetric
+		s, l, _ = BandedLCS([]byte(c.b), []byte(c.a), c.maxError)
+		if s != c.s || l != c.l {
+			t.Errorf("swapped %q/%q maxError=%d: got (%d,%d), want (%d,%d)", c.b, c.a, c.maxError, s, l, c.s, c.l)
+		}
+	}
+}
+
+func TestBandedLCSTooLong(t *testing.T) {
+	long := make([]byte, MaxBandedLCSLength+1)
+	for i := range long {
+		long[i] = 'A'
+	}
+	if _, _, err := BandedLCS(long, []byte("ACGT"), -1); err != ErrSequenceTooLong {
+		t.Errorf("A too long: err = %v", err)
+	}
+	if _, _, err := BandedLCS([]byte("ACGT"), long, 5); err != ErrSequenceTooLong {
+		t.Errorf("B too long: err = %v", err)
+	}
+	// the maximum length itself is accepted
+	ok := long[:MaxBandedLCSLength]
+	if s, l, err := BandedLCS(ok, ok, 0); err != nil || s != len(ok) || l != len(ok) {
+		t.Errorf("max length: got (%d,%d,%v)", s, l, err)
+	}
+}
+
+// TestBandedLCSSecondPass checks the exactness over many bands, also on
+// low-complexity sequences where the best LCS is not the closest alignment,
+// and reports how often the band had to be widened (never more than once).
+func TestBandedLCSSecondPass(t *testing.T) {
+	rng := rand.New(rand.NewSource(5))
+	for _, alpha := range testAlphabets[:4] {
+		for _, rate := range []float64{0.02, 0.1, 0.3} {
+			ws := new(bandedLCSWorkspace)
+			var total int
+			for it := 0; it < 1500; it++ {
+				a := randomFrom(rng, alpha.letters, 20+rng.Intn(120))
+				b := mutateFrom(rng, alpha.letters, a, rate)
+				if len(b) == 0 {
+					continue
+				}
+				if len(a) < len(b) {
+					a, b = b, a
+				}
+				for _, maxError := range []int{1, 3, 6, 10, 20, 40} {
+					want, wantL := expected(a, b, maxError)
+					got, gotL := bandedLCS(a, b, maxError, ws, haveSIMD)
+					if got != want || gotL != wantL {
+						t.Fatalf("%s maxError=%d a=%q b=%q: got (%d,%d), want (%d,%d)",
+							alpha.name, maxError, a, b, got, gotL, want, wantL)
+					}
+					total++
+				}
+			}
+			t.Logf("%-15s mutation %.2f: %d bounded calls, %d widened (%.1f%%)",
+				alpha.name, rate, total, ws.widened, 100*float64(ws.widened)/float64(total))
+		}
+	}
+}
+
+// TestFastLCSScoreWrapper checks the BioSequence entry point.
+func TestFastLCSScoreWrapper(t *testing.T) {
+	sa := obiseq.NewBioSequence("a", []byte("ACGTACGTAC"), "")
+	sb := obiseq.NewBioSequence("b", []byte("ACGTTCGTAC"), "")
+	if s, l := FastLCSScore(sa, sb, -1, nil); s != 9 || l != 10 {
+		t.Errorf("got (%d,%d), want (9,10)", s, l)
+	}
+	if s, l := FastLCSScore(sa, sb, 0, nil); s != -1 || l != -1 {
+		t.Errorf("got (%d,%d), want (-1,-1)", s, l)
+	}
+}
+
+// TestBandedLCSAtLeast checks the exactness of the bound expressed on the
+// score: the optimum when it reaches minLCS, (-1,-1) otherwise.
+func TestBandedLCSAtLeast(t *testing.T) {
+	rng := rand.New(rand.NewSource(13))
+	var buf BandedLCSBuffer
+	for _, alpha := range testAlphabets {
+		for it := 0; it < 1500; it++ {
+			a := randomFrom(rng, alpha.letters, 1+rng.Intn(100))
+			b := mutateFrom(rng, alpha.letters, a, rng.Float64()*0.5)
+			if len(b) == 0 {
+				continue
+			}
+			os, ol := fullDP(a, b)
+			for _, minLCS := range []int{-3, 0, 1, os - 1, os, os + 1, len(b) / 2, len(b), len(b) + 1} {
+				ws, wl := os, ol
+				if os < minLCS {
+					ws, wl = -1, -1
+				}
+				for _, got := range [][2]int{func() [2]int { s, l, _ := buf.AtLeast(a, b, minLCS); return [2]int{s, l} }(),
+					func() [2]int { s, l, _ := BandedLCSAtLeast(b, a, minLCS); return [2]int{s, l} }()} {
+					if got[0] != ws || got[1] != wl {
+						t.Fatalf("%s minLCS=%d a=%q b=%q: got (%d,%d), want (%d,%d)",
+							alpha.name, minLCS, a, b, got[0], got[1], ws, wl)
+					}
+				}
+			}
+		}
+	}
+	if _, _, err := buf.AtLeast(make([]byte, MaxBandedLCSLength+1), []byte("a"), 0); err != ErrSequenceTooLong {
+		t.Errorf("too long: err = %v", err)
+	}
+}
+
+// TestBandedLCSScoreAtLeast checks the score-only entry point, with sequences
+// long enough for the early stop of the kernel to be exercised.
+func TestBandedLCSScoreAtLeast(t *testing.T) {
+	rng := rand.New(rand.NewSource(17))
+	var buf BandedLCSBuffer
+	check := func(name string, a, b []byte) {
+		os, _ := fullDP(a, b)
+		for _, minLCS := range []int{-1, 0, os - 3, os - 1, os, os + 1, os + 5, len(b) - 1, len(b), (len(b) * 9) / 10, len(b) + 1} {
+			want := os
+			if os < minLCS {
+				want = -1
+			}
+			got, err := buf.ScoreAtLeast(a, b, minLCS)
+			if err != nil || got != want {
+				t.Fatalf("%s minLCS=%d a=%q b=%q: got %d (%v), want %d", name, minLCS, a, b, got, err, want)
+			}
+		}
+	}
+	for _, alpha := range testAlphabets {
+		for it := 0; it < 800; it++ {
+			a := randomFrom(rng, alpha.letters, 1+rng.Intn(60))
+			b := mutateFrom(rng, alpha.letters, a, rng.Float64()*0.6)
+			if len(b) > 0 {
+				check(alpha.name, a, b)
+			}
+		}
+	}
+	for it := 0; it < 150; it++ {
+		a := randomFrom(rng, "ACGT", 100+rng.Intn(400))
+		b := mutateFrom(rng, "ACGT", a, rng.Float64()*0.3)
+		if it%3 == 0 {
+			b = randomFrom(rng, "ACGT", 50+rng.Intn(300))
+		}
+		if len(b) > 0 {
+			check("long", a, b)
+		}
+	}
+}

pkg/obialign/lcsmin.go

@@ -0,0 +1,103 @@
+package obialign
+
+// BandedLCSBuffer is a working memory that can be reused from one alignment to
+// the next to avoid any allocation. It must not be shared between goroutines.
+type BandedLCSBuffer struct {
+	ws bandedLCSWorkspace
+}
+
+// AtLeast is BandedLCSAtLeast on the buffer.
+func (b *BandedLCSBuffer) AtLeast(seqA, seqB []byte, minLCS int) (lcs, alilen int, err error) {
+	return atLeast(seqA, seqB, minLCS, &b.ws)
+}
+
+func atLeast(seqA, seqB []byte, minLCS int, ws *bandedLCSWorkspace) (lcs, alilen int, err error) {
+	if len(seqA) > MaxBandedLCSLength || len(seqB) > MaxBandedLCSLength {
+		return -1, -1, ErrSequenceTooLong
+	}
+	if len(seqA) < len(seqB) {
+		seqA, seqB = seqB, seqA
+	}
+	lcs, alilen = bandedLCSAtLeast(seqA, seqB, minLCS, ws, haveSIMD)
+	return lcs, alilen, nil
+}
+
+// BandedLCSAtLeast computes the LCS score of two nucleotide sequences and the
+// length of the shortest alignment achieving it, as BandedLCS does, but with
+// the bound expressed on the score: the result is the exact optimum when its
+// score is at least minLCS, and (-1, -1) otherwise.
+//
+// The DP only visits the diagonals an alignment of score minLCS can use (one
+// whose path makes at most lB-minLCS gaps on one side and lA-minLCS on the
+// other), so the cost decreases as minLCS grows. This is exact: every alignment
+// of score >= minLCS lies in that band, hence so does the optimum when it
+// reaches minLCS, and when the best path of the band is below minLCS so is
+// the global optimum.
+func BandedLCSAtLeast(seqA, seqB []byte, minLCS int) (lcs, alilen int, err error) {
+	ws := bandedLCSPool.Get().(*bandedLCSWorkspace)
+	lcs, alilen, err = atLeast(seqA, seqB, minLCS, ws)
+	bandedLCSPool.Put(ws)
+	return lcs, alilen, err
+}
+
+// bandedLCSAtLeast expects len(a) >= len(b), both within MaxBandedLCSLength.
+func bandedLCSAtLeast(a, b []byte, minLCS int, ws *bandedLCSWorkspace, useSIMD bool) (int, int) {
+	lA, lB := len(a), len(b)
+	minLCS = max(minLCS, 0)
+	if minLCS > lB {
+		return -1, -1
+	}
+	if lB == 0 {
+		return 0, lA
+	}
+	simdOK := useSIMD && allRegularIUPAC(a) && allRegularIUPAC(b)
+	kmin, kmax := -(lB - minLCS), lA-minLCS
+
+	if !simdOK {
+		lcs, alilen := ws.align(a, b, kmin, kmax, false)
+		if lcs < minLCS {
+			return -1, -1
+		}
+		return lcs, alilen
+	}
+
+	// The score alone is much cheaper than the score with the length of the
+	// alignment: the latter is only computed when the pair reaches minLCS. With
+	// no minimum there is nothing to reject, and the pass would be wasted.
+	if minLCS > 0 && lcsScoreSIMD(a, b, kmin, kmax, minLCS, ws) < minLCS {
+		return -1, -1
+	}
+	encodeBases(ws, a, b)
+	return ws.align(a, b, kmin, kmax, true)
+}
+
+// ScoreAtLeast computes the LCS score of two nucleotide sequences when it is
+// at least minLCS, and returns -1 otherwise. It is much cheaper than AtLeast,
+// which also computes the length of the shortest alignment.
+func (b *BandedLCSBuffer) ScoreAtLeast(seqA, seqB []byte, minLCS int) (int, error) {
+	if len(seqA) > MaxBandedLCSLength || len(seqB) > MaxBandedLCSLength {
+		return -1, ErrSequenceTooLong
+	}
+	if len(seqA) < len(seqB) {
+		seqA, seqB = seqB, seqA
+	}
+	lA, lB := len(seqA), len(seqB)
+	minLCS = max(minLCS, 0)
+	if minLCS > lB {
+		return -1, nil
+	}
+	if lB == 0 {
+		return 0, nil
+	}
+	kmin, kmax := -(lB - minLCS), lA-minLCS
+	var lcs int
+	if haveSIMD && allRegularIUPAC(seqA) && allRegularIUPAC(seqB) {
+		lcs = lcsScoreSIMD(seqA, seqB, kmin, kmax, minLCS, &b.ws)
+	} else {
+		lcs, _ = b.ws.align(seqA, seqB, kmin, kmax, false)
+	}
+	if lcs < minLCS {
+		return -1, nil
+	}
+	return lcs, nil
+}

pkg/obialign/lcsscore_simd.go

@@ -0,0 +1,156 @@
+//go:build goexperiment.simd
+
+package obialign
+
+import "simd"
+
+func simdLanes16() int {
+	var probe simd.Uint16s
+	return probe.Len()
+}
+
+func growUint16(s []uint16, n int) []uint16 {
+	if cap(s) < n {
+		return make([]uint16, n)
+	}
+	return s[:n]
+}
+
+func encodeIUPAC16(dst []uint16, s []byte, rev bool) {
+	n := len(s)
+	clear(dst)
+	if rev {
+		for x, c := range s {
+			dst[n-1-x] = uint16(iupacMask[c])
+		}
+		return
+	}
+	for x, c := range s {
+		dst[x] = uint16(iupacMask[c])
+	}
+}
+
+// pruneEvery is the number of antidiagonals between two tests of the upper
+// bound that stops a hopeless alignment.
+const pruneEvery = 8
+
+// lcsScoreSIMD runs the same banded DP as bandedLCSSIMD on the score only,
+// with 16-bit lanes (twice the cells per instruction):
+//
+//	cell(i,j) = max(diag + match, up, left)
+//
+// A cell outside the band is 0, which never beats a real cell. The score of a
+// path ending at the cell (i, j) of an antidiagonal cannot exceed the score of
+// the cell plus min(lB-i, lA-j), the characters left to pair. Every path
+// crosses each antidiagonal inside the window, so when the best score of the
+// window plus the largest of these remainders is under minLCS the final score
+// is under minLCS: -1 is returned without computing the rest of the matrix.
+//
+// Requires lA >= lB >= 1, kmin <= 0 <= kmax, and bases with non-zero masks.
+// Returns the best score of the band, or -1 if it is proven to be under
+// minLCS.
+func lcsScoreSIMD(a, b []byte, kmin, kmax, minLCS int, ws *bandedLCSWorkspace) int {
+	lA, lB := len(a), len(b)
+	vl := simdLanes16()
+
+	ws.aRev16 = growUint16(ws.aRev16, lA+vl)
+	ws.bCode16 = growUint16(ws.bCode16, lB+vl)
+	encodeIUPAC16(ws.aRev16, a, true)
+	encodeIUPAC16(ws.bCode16, b, false)
+	aRev, bCode := ws.aRev16, ws.bCode16
+
+	size := lB + 2*vl + 2
+	for r := range ws.rows16 {
+		ws.rows16[r] = growUint16(ws.rows16[r], size)
+		clear(ws.rows16[r])
+	}
+	p2, p1, cur := ws.rows16[0], ws.rows16[1], ws.rows16[2]
+
+	// the first row and the first column hold a score of 0: nothing to set
+
+	zero := simd.BroadcastUint16s(0)
+	one := simd.BroadcastUint16s(1)
+
+	for t := 2; t <= lA+lB; t++ {
+		lo := max(1, t-lA, (t-kmax+1)>>1)
+		hi := min(lB, t-1, (t-kmin)>>1)
+
+		for i := lo; i <= hi; i += vl {
+			ca := simd.LoadUint16s(aRev[lA-t+i:])
+			cb := simd.LoadUint16s(bCode[i-1:])
+			inc := one.Masked(ca.And(cb).NotEqual(zero))
+			d := simd.LoadUint16s(p2[i-1:]).Add(inc)
+			u := simd.LoadUint16s(p1[i-1:])
+			l := simd.LoadUint16s(p1[i:])
+			d.Max(u).Max(l).Store(cur[i:])
+		}
+
+		zero.Store(cur[hi+1:])
+		if lo > 1 {
+			cur[lo-1] = 0
+		}
+
+		// A diagonal move skips an antidiagonal, but a path always visits
+		// one of two consecutive antidiagonals.
+		if t%pruneEvery == 0 && minLCS > 0 &&
+			scoreBound(p1, t-1, lA, lB, kmin, kmax) < minLCS &&
+			scoreBound(cur, t, lA, lB, kmin, kmax) < minLCS {
+			return -1
+		}
+
+		p2, p1, cur = p1, cur, p2
+	}
+	return int(p1[lB])
+}
+
+// iota16 is 0, 1, 2... to build the row index of the lanes.
+var iota16 = func() (t [64]uint16) {
+	for i := range t {
+		t[i] = uint16(i)
+	}
+	return t
+}()
+
+// scoreBound is an upper bound of the final score of the paths going through
+// the antidiagonal t, whose scores are in row. A path through the cell (i, j)
+// scores at most the score of the cell plus min(lB-i, lA-j), the characters
+// left to pair, so the bound is the maximum of this sum on the cells of the
+// window. The sum is computed by vectors on the full chunks of the window.
+func scoreBound(row []uint16, t, lA, lB, kmin, kmax int) int {
+	lo := max(1, t-lA, (t-kmax+1)>>1)
+	hi := min(lB, t-1, (t-kmin)>>1)
+
+	bound := 0
+	i := lo
+	if hi-lo >= simdLanes16() {
+		vl := simdLanes16()
+		idx := simd.LoadUint16s(iota16[:]).Add(simd.BroadcastUint16s(uint16(lo)))
+		step := simd.BroadcastUint16s(uint16(vl))
+		remB := simd.BroadcastUint16s(uint16(lB))
+		// lA-t+i is not negative on the window; the 16-bit arithmetic is modular
+		offA := simd.BroadcastUint16s(uint16(lA - t))
+		acc := simd.BroadcastUint16s(0)
+		for ; i+vl-1 <= hi; i += vl {
+			left := remB.Sub(idx).Min(idx.Add(offA))
+			acc = acc.Max(simd.LoadUint16s(row[i:]).Add(left))
+			idx = idx.Add(step)
+		}
+		var lanes [64]uint16
+		acc.Store(lanes[:])
+		for _, v := range lanes[:vl] {
+			bound = max(bound, int(v))
+		}
+	}
+	for ; i <= hi; i++ {
+		bound = max(bound, int(row[i])+min(lB-i, lA-t+i))
+	}
+
+	// the paths running along the first row or the first column have a score of 0
+	if t <= lA && t <= kmax {
+		bound = max(bound, min(lB, lA-t))
+	}
+	if t <= lB && -t >= kmin {
+		bound = max(bound, min(lB-t, lA))
+	}
+	return bound
+}

pkg/obidefault/progressbar.go

@@ -1,5 +1,7 @@
 package obidefault
 
+import "os"
+
 var __no_progress_bar__ = false
 
 func ProgressBar() bool {
@@ -17,3 +19,22 @@ func SetNoProgressBar(b bool) {
 func NoProgressBarPtr() *bool {
 	return &__no_progress_bar__
 }
+
+// ProgressBarVisible tells whether a progress bar can be displayed: it is not
+// disabled by --no-progressbar, the standard error output is a terminal and
+// the standard output is not piped to another command.
+func ProgressBarVisible() bool {
+	if !ProgressBar() {
+		return false
+	}
+
+	if e, err := os.Stderr.Stat(); err != nil || (e.Mode()&os.ModeCharDevice) != os.ModeCharDevice {
+		return false
+	}
+
+	if o, err := os.Stdout.Stat(); err == nil && (o.Mode()&os.ModeNamedPipe) == os.ModeNamedPipe {
+		return false
+	}
+
+	return true
+}

pkg/obiformats/batch_of_files_reader.go

@@ -48,6 +48,14 @@ func ReadSequencesBatchFromFiles(filenames []string,
 
 				log.Printf("Start reading of file : %s", filename)
 
+				// The batches of a file reach us in any order (parallel parsing
+				// of the headers). They are numbered here in arrival order, so
+				// the order of the file must be restored first, unless the
+				// order of the data is not wanted (several concurrent readers).
+				if concurrent_readers == 1 {
+					iter = iter.SortBatches()
+				}
+
 				for iter.Next() {
 					batch := iter.Get()
 					batchiter.Push(batch.Reorder(nextCounter()))

pkg/obiformats/batch_of_files_reader_test.go

@@ -0,0 +1,38 @@
+package obiformats
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// With a single reader, the sequences of several files must come out in the
+// order of the files, whatever the order in which the batches are parsed.
+func TestReadSequencesBatchFromFilesKeepsOrder(t *testing.T) {
+	dir := t.TempDir()
+	var names []string
+	for _, prefix := range []string{"a", "b"} {
+		var sb strings.Builder
+		for i := 0; i < 25000; i++ {
+			fmt.Fprintf(&sb, ">%s%07d {\"k\":%d}\n%s\n", prefix, i, i, strings.Repeat("acgt", 40))
+		}
+		name := filepath.Join(dir, prefix+".fasta")
+		if err := os.WriteFile(name, []byte(sb.String()), 0o600); err != nil {
+			t.Fatal(err)
+		}
+		names = append(names, name)
+	}
+
+	it := ReadSequencesBatchFromFiles(names, nil, 1)
+	_, db := it.Load()
+	if len(db) != 50000 {
+		t.Fatalf("expected 50000 sequences, got %d", len(db))
+	}
+	for i := 1; i < len(db); i++ {
+		if db[i-1].Id() >= db[i].Id() {
+			t.Fatalf("out of order at %d: %s then %s", i, db[i-1].Id(), db[i].Id())
+		}
+	}
+}

pkg/obigraph/clique.go

@@ -0,0 +1,343 @@
+package obigraph
+
+import (
+	"context"
+	"math/rand"
+	"sync"
+	"sync/atomic"
+)
+
+// NeighborFilter defines an implicit graph over the vertices 0..n-1.
+// It appends to out[:0] the vertices of candidates that are adjacent to v,
+// keeping their order, and returns the resulting slice. It is called
+// concurrently by several workers and must not modify candidates.
+//
+// Describing the adjacency by a filter, instead of a matrix, keeps the memory
+// in O(n) and lets the caller evaluate adjacency in bulk (SIMD, for instance).
+type NeighborFilter func(v int32, candidates, out []int32) []int32
+
+// CliqueOptions tunes FindClique.
+type CliqueOptions struct {
+	// Workers is the number of concurrent searches (default 1).
+	Workers int
+	// Seed seeds the random vertex orders of the restarts.
+	Seed int64
+	// Progress, if set, is called with the size of the best clique each time a
+	// larger one is found. Calls are serialized and made while a lock is held:
+	// it must return quickly and must not call FindClique.
+	Progress func(best int)
+
+	// ColorLimit enables the coloring bound for the candidate lists of at most
+	// ColorLimit vertices (0 disables it). The vertices of a list are split into
+	// classes of pairwise non-adjacent vertices: a clique takes at most one
+	// vertex per class, so the number of classes bounds its size. Computing the
+	// classes costs up to the square of the list length in calls to the filter,
+	// which is why only short lists are colored.
+	ColorLimit int
+
+	// mode forces the kind of search (tests only): 0 alternates, modeFocused
+	// and modeImproving restrict the search to one of them.
+	mode int
+}
+
+const (
+	modeFocused   = 1
+	modeImproving = 2
+)
+
+// CliqueResult is the outcome of FindClique.
+type CliqueResult struct {
+	// Clique is the largest clique found (a clique of size >= target if Reached).
+	Clique []int32
+	// Reached is true if len(Clique) >= target.
+	Reached bool
+	// Exhausted is true if the whole search space has been explored. If Reached
+	// is false, it proves that no clique of the target size exists.
+	Exhausted bool
+	// Maximum is true if Clique is proven to be a maximum clique of the graph:
+	// no larger clique exists.
+	Maximum bool
+}
+
+// FindClique looks for a clique of at least target vertices in the graph of n
+// vertices described by filter. It stops as soon as one is found, when the
+// search space is exhausted, or when ctx is done; in the last case, the largest
+// clique met so far is returned.
+//
+// The search is a depth first search with backtracking, which comes in two
+// flavours that the restarts alternate:
+//
+//   - the focused search prunes every branch that cannot reach target, which
+//     is the most efficient way to reach it but gives poor cliques when target
+//     is out of reach;
+//   - the improving search only prunes the branches that cannot beat the best
+//     clique found so far (by any worker), so it keeps improving that clique
+//     whatever target is. When it ends, the best clique is a maximum one.
+//
+// When a focused search proves that no clique of target vertices exists, every
+// search turns to the improving flavour, so that the largest clique is returned
+// (and proved to be so) instead of an arbitrary one.
+//
+// On short candidate lists (see CliqueOptions.ColorLimit), a coloring bound
+// prunes the branches that cannot be large enough, and orders the vertices so
+// that the most constrained ones are tried first.
+//
+// The first pass of the first worker follows the natural order of the vertices
+// (greedy "lexicographic" construction); every other pass restarts from a
+// random order, with a work budget that doubles at each restart, so that the
+// search is complete in the long run.
+func FindClique(ctx context.Context, n int, filter NeighborFilter, target int, opts CliqueOptions) CliqueResult {
+	if target <= 0 {
+		return CliqueResult{Reached: true, Exhausted: true}
+	}
+	if n <= 0 {
+		return CliqueResult{Exhausted: true}
+	}
+
+	workers := max(opts.Workers, 1)
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	var (
+		mutex     sync.Mutex
+		best      []int32
+		bestSize  atomic.Int32
+		exhausted bool
+		maximum   bool
+		wg        sync.WaitGroup
+		// set when a focused search has proved that target cannot be reached:
+		// every search then turns to finding the largest clique
+		unreachable atomic.Bool
+	)
+
+	publish := func(clique []int32) {
+		mutex.Lock()
+		if len(clique) > len(best) {
+			best = append(best[:0], clique...)
+			bestSize.Store(int32(len(best)))
+			if opts.Progress != nil {
+				opts.Progress(len(best))
+			}
+		}
+		mutex.Unlock()
+	}
+
+	for w := range workers {
+		wg.Go(func() {
+			s := &cliqueSearch{
+				ctx:         ctx,
+				filter:      filter,
+				target:      target,
+				bestSize:    &bestSize,
+				colorLimit:  opts.ColorLimit,
+				unreachable: &unreachable,
+				publish: func(c []int32) {
+					publish(c)
+					if len(c) >= target {
+						cancel()
+					}
+				},
+			}
+			rng := rand.New(rand.NewSource(opts.Seed + int64(w)))
+			// the search starts from the end of the list: lowest vertex first
+			candidates := make([]int32, n)
+			for i := range candidates {
+				candidates[i] = int32(n - 1 - i)
+			}
+
+			for restart := 0; ctx.Err() == nil; restart++ {
+				if w > 0 || restart > 0 {
+					rng.Shuffle(n, func(i, j int) { candidates[i], candidates[j] = candidates[j], candidates[i] })
+				}
+				switch opts.mode {
+				case modeFocused:
+					s.improving = false
+				case modeImproving:
+					s.improving = true
+				default:
+					s.improving = unreachable.Load() || (w+restart)%2 == 1
+				}
+				s.budget = int64(n) * int64(target) << min(restart, 20)
+				s.work = 0
+				s.aborted = false
+				s.search(0, candidates)
+
+				if !s.aborted && ctx.Err() == nil {
+					mutex.Lock()
+					exhausted = true
+					if s.improving {
+						maximum = true
+					}
+					mutex.Unlock()
+
+					if s.improving || opts.mode == modeFocused {
+						cancel()
+					} else {
+						// target is out of reach: look for the largest clique
+						unreachable.Store(true)
+					}
+				}
+			}
+		})
+	}
+	wg.Wait()
+
+	return CliqueResult{
+		Clique:    best,
+		Reached:   len(best) >= target,
+		Exhausted: exhausted,
+		Maximum:   maximum,
+	}
+}
+
+type cliqueSearch struct {
+	ctx        context.Context
+	filter     NeighborFilter
+	target     int
+	bestSize   *atomic.Int32 // size of the best clique found by any worker
+	publish    func([]int32)
+	colorLimit int
+
+	unreachable *atomic.Bool // target is proved out of reach
+
+	improving bool // see FindClique: the kind of the current pass
+
+	current []int32   // the clique under construction
+	buffers [][]int32 // candidate list of each depth
+	sorted  [][]int32 // candidate list of each depth, sorted by color
+	colors  [][]int32 // color (from 1) of each vertex of sorted
+	classes [][]int32 // scratch space of the coloring
+	probe   []int32   // scratch space of the coloring
+
+	budget  int64 // maximum work (number of candidates tested) for this restart
+	work    int64
+	aborted bool // budget or context exhausted before the end of the exploration
+}
+
+// threshold is the size a clique must reach to be worth exploring: the target
+// for the focused search, one more than the best clique for the improving one.
+func (s *cliqueSearch) threshold() int {
+	if s.improving {
+		return min(s.target, int(s.bestSize.Load())+1)
+	}
+	return s.target
+}
+
+// mustStop tells whether the current pass has to be interrupted: its work
+// budget is spent, or the pass must stop whatever its budget (see mustAbort).
+func (s *cliqueSearch) mustStop() bool {
+	return s.work > s.budget || s.mustAbort()
+}
+
+// mustAbort tells whether the pass has to be interrupted whatever its budget:
+// the context is done, or it is a focused pass and its target has been proved
+// out of reach.
+func (s *cliqueSearch) mustAbort() bool {
+	return s.ctx.Err() != nil || (!s.improving && s.unreachable.Load())
+}
+
+// colorSort splits candidates into classes of pairwise non-adjacent vertices
+// (greedy sequential coloring) and returns them class after class, with the
+// color of each vertex. Any clique takes at most one vertex of each class.
+//
+// The coloring can be long: it stops, and sets s.aborted, as soon as the search
+// has to be aborted. Its cost is not charged to the work budget of the pass,
+// which limits the exploration of the tree, not the bound computed at each node.
+func (s *cliqueSearch) colorSort(depth int, candidates []int32) (sorted, colors []int32) {
+	k := 0
+	for _, v := range candidates {
+		if s.mustAbort() {
+			s.aborted = true
+			return nil, nil
+		}
+		placed := false
+		for c := 0; c < k; c++ {
+			// v joins the class if it is adjacent to none of its members
+			s.probe = s.filter(v, s.classes[c], s.probe[:0])
+			if len(s.probe) == 0 {
+				s.classes[c] = append(s.classes[c], v)
+				placed = true
+				break
+			}
+		}
+		if !placed {
+			if k == len(s.classes) {
+				s.classes = append(s.classes, nil)
+			}
+			s.classes[k] = append(s.classes[k][:0], v)
+			k++
+		}
+	}
+
+	for len(s.sorted) <= depth {
+		s.sorted = append(s.sorted, nil)
+		s.colors = append(s.colors, nil)
+	}
+	sorted, colors = s.sorted[depth][:0], s.colors[depth][:0]
+	for c := 0; c < k; c++ {
+		for _, v := range s.classes[c] {
+			sorted = append(sorted, v)
+			colors = append(colors, int32(c+1))
+		}
+	}
+	s.sorted[depth], s.colors[depth] = sorted, colors
+	return sorted, colors
+}
+
+// search extends the current clique of depth vertices with vertices taken in
+// candidates, all adjacent to the vertices of the current clique. The list is
+// read from its end: the vertex at position i is combined with the vertices
+// before it, so that every clique is met once.
+func (s *cliqueSearch) search(depth int, candidates []int32) {
+	var colors []int32
+	if s.colorLimit > 0 && len(candidates) > 1 && len(candidates) <= s.colorLimit {
+		candidates, colors = s.colorSort(depth, candidates)
+		if s.aborted {
+			return
+		}
+	}
+
+	for i := len(candidates) - 1; i >= 0; i-- {
+		// largest clique the vertices 0..i can make
+		bound := i + 1
+		if colors != nil {
+			bound = int(colors[i])
+		}
+		if depth+bound < s.threshold() {
+			return
+		}
+		if s.mustStop() {
+			s.aborted = true
+			return
+		}
+
+		v := candidates[i]
+		s.current = append(s.current[:depth], v)
+		if len(s.current) > int(s.bestSize.Load()) {
+			s.publish(s.current)
+			if len(s.current) >= s.target {
+				s.aborted = true // the search is over: do not report an exhaustion
+				return
+			}
+		}
+
+		rest := candidates[:i]
+		if depth+1+len(rest) < s.threshold() {
+			continue
+		}
+		for len(s.buffers) <= depth+1 {
+			s.buffers = append(s.buffers, nil)
+		}
+		next := s.filter(v, rest, s.buffers[depth+1][:0])
+		s.buffers[depth+1] = next
+		s.work += int64(len(rest))
+
+		if depth+1+len(next) < s.threshold() {
+			continue
+		}
+		s.search(depth+1, next)
+		if s.aborted {
+			return
+		}
+	}
+}

pkg/obigraph/clique_test.go

@@ -0,0 +1,249 @@
+package obigraph
+
+import (
+	"context"
+	"math/bits"
+	"math/rand"
+	"testing"
+	"time"
+)
+
+// adjacency matrix filter for small graphs
+func matrixFilter(adj [][]bool) NeighborFilter {
+	return func(v int32, candidates, out []int32) []int32 {
+		for _, c := range candidates {
+			if adj[v][c] {
+				out = append(out, c)
+			}
+		}
+		return out
+	}
+}
+
+func randomGraph(rng *rand.Rand, n int, p float64) [][]bool {
+	adj := make([][]bool, n)
+	for i := range adj {
+		adj[i] = make([]bool, n)
+	}
+	for i := 0; i < n; i++ {
+		for j := i + 1; j < n; j++ {
+			if rng.Float64() < p {
+				adj[i][j], adj[j][i] = true, true
+			}
+		}
+	}
+	return adj
+}
+
+// maximum clique size by brute force over all the subsets (n <= 20)
+func bruteMaxClique(adj [][]bool) int {
+	n := len(adj)
+	best := 0
+	for mask := 1; mask < 1<<n; mask++ {
+		size := bits.OnesCount(uint(mask))
+		if size <= best {
+			continue
+		}
+		ok := true
+		for i := 0; i < n && ok; i++ {
+			if mask>>i&1 == 0 {
+				continue
+			}
+			for j := i + 1; j < n; j++ {
+				if mask>>j&1 == 1 && !adj[i][j] {
+					ok = false
+					break
+				}
+			}
+		}
+		if ok {
+			best = size
+		}
+	}
+	return best
+}
+
+func isClique(adj [][]bool, c []int32) bool {
+	seen := map[int32]bool{}
+	for i, a := range c {
+		if seen[a] {
+			return false
+		}
+		seen[a] = true
+		for _, b := range c[i+1:] {
+			if !adj[a][b] {
+				return false
+			}
+		}
+	}
+	return true
+}
+
+func TestFindCliqueAgainstBruteForce(t *testing.T) {
+	rng := rand.New(rand.NewSource(1))
+	for trial := 0; trial < 40; trial++ {
+		n := 8 + rng.Intn(10)
+		adj := randomGraph(rng, n, 0.3+0.5*rng.Float64())
+		maxSize := bruteMaxClique(adj)
+		filter := matrixFilter(adj)
+
+		for _, workers := range []int{1, 3} {
+			r := FindClique(context.Background(), n, filter, maxSize, CliqueOptions{Workers: workers})
+			if !r.Reached || len(r.Clique) < maxSize || !isClique(adj, r.Clique) {
+				t.Fatalf("trial %d: target %d not reached: %+v", trial, maxSize, r)
+			}
+
+			r = FindClique(context.Background(), n, filter, maxSize+1, CliqueOptions{Workers: workers})
+			if r.Reached || !r.Exhausted {
+				t.Fatalf("trial %d: size %d cannot exist, got %+v", trial, maxSize+1, r)
+			}
+			// best effort: a valid clique, not necessarily a maximum one
+			if len(r.Clique) < 1 || len(r.Clique) > maxSize || !isClique(adj, r.Clique) {
+				t.Fatalf("trial %d: invalid best clique %v (maximum is %d)", trial, r.Clique, maxSize)
+			}
+		}
+	}
+}
+
+func TestFindCliqueTrivialTargets(t *testing.T) {
+	adj := randomGraph(rand.New(rand.NewSource(2)), 5, 0.5)
+	if r := FindClique(context.Background(), 5, matrixFilter(adj), 0, CliqueOptions{}); !r.Reached {
+		t.Fatal("target 0 is always reached")
+	}
+	if r := FindClique(context.Background(), 0, matrixFilter(adj), 1, CliqueOptions{}); r.Reached {
+		t.Fatal("no clique in an empty graph")
+	}
+	if r := FindClique(context.Background(), 5, matrixFilter(adj), 1, CliqueOptions{}); !r.Reached || len(r.Clique) != 1 {
+		t.Fatalf("a single vertex is a clique: %+v", r)
+	}
+}
+
+// A hopeless search must stop with the deadline and return a valid clique.
+// (When the target is far above the clique number of a sparse graph, pruning
+// proves the absence at once: see TestFindCliqueAgainstBruteForce.)
+func TestFindCliqueTimeout(t *testing.T) {
+	// dense graph: its largest clique is far below the target, but pruning only
+	// bites after many levels, so the search cannot finish within the deadline
+	const n = 300
+	adj := randomGraph(rand.New(rand.NewSource(3)), n, 0.9)
+	ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond)
+	defer cancel()
+
+	start := time.Now()
+	r := FindClique(ctx, n, matrixFilter(adj), 150, CliqueOptions{Workers: 2})
+	if elapsed := time.Since(start); elapsed > 3*time.Second {
+		t.Fatalf("deadline not honoured: %v", elapsed)
+	}
+	if r.Reached || r.Exhausted {
+		t.Fatalf("size 150 cannot be reached nor proven absent in time: %+v", r)
+	}
+	if len(r.Clique) < 2 || !isClique(adj, r.Clique) {
+		t.Fatalf("invalid best clique %v", r.Clique)
+	}
+}
+
+func TestFindCliqueProgress(t *testing.T) {
+	adj := randomGraph(rand.New(rand.NewSource(4)), 40, 0.7)
+
+	var seen []int
+	r := FindClique(context.Background(), 40, matrixFilter(adj), 5, CliqueOptions{
+		Progress: func(best int) { seen = append(seen, best) },
+	})
+	if !r.Reached {
+		t.Fatalf("size 5 should be reached: %+v", r)
+	}
+	if len(seen) == 0 || seen[len(seen)-1] != len(r.Clique) {
+		t.Fatalf("progress %v does not end at the clique size %d", seen, len(r.Clique))
+	}
+	for i := 1; i < len(seen); i++ {
+		if seen[i] <= seen[i-1] {
+			t.Fatalf("progress must be strictly increasing: %v", seen)
+		}
+	}
+}
+
+// The improving search must end with a proven maximum clique, even when the
+// target is out of reach; the focused one only proves the absence of a clique
+// of the target size.
+func TestFindCliqueModes(t *testing.T) {
+	rng := rand.New(rand.NewSource(5))
+	for trial := 0; trial < 40; trial++ {
+		n := 8 + rng.Intn(10)
+		adj := randomGraph(rng, n, 0.3+0.5*rng.Float64())
+		maxSize := bruteMaxClique(adj)
+		filter := matrixFilter(adj)
+
+		for _, workers := range []int{1, 3} {
+			for _, limit := range []int{0, 5, 1 << 20} {
+				r := FindClique(context.Background(), n, filter, maxSize+1,
+					CliqueOptions{Workers: workers, ColorLimit: limit, mode: modeImproving})
+				if r.Reached || !r.Exhausted || !r.Maximum {
+					t.Fatalf("trial %d limit %d: improving search should prove the maximum: %+v", trial, limit, r)
+				}
+				if len(r.Clique) != maxSize || !isClique(adj, r.Clique) {
+					t.Fatalf("trial %d limit %d: expected a maximum clique of %d, got %v", trial, limit, maxSize, r.Clique)
+				}
+
+				r = FindClique(context.Background(), n, filter, maxSize+1,
+					CliqueOptions{Workers: workers, ColorLimit: limit, mode: modeFocused})
+				if r.Reached || !r.Exhausted || r.Maximum {
+					t.Fatalf("trial %d limit %d: focused search proves absence only: %+v", trial, limit, r)
+				}
+
+				r = FindClique(context.Background(), n, filter, maxSize,
+					CliqueOptions{Workers: workers, ColorLimit: limit, mode: modeImproving})
+				if !r.Reached || len(r.Clique) < maxSize || !isClique(adj, r.Clique) {
+					t.Fatalf("trial %d limit %d: target %d not reached by the improving search: %+v", trial, limit, maxSize, r)
+				}
+			}
+		}
+	}
+}
+
+// When the target is out of reach, the default search must end with a proven
+// maximum clique, even if the coloring bound proves the absence of the target
+// before any clique has been found.
+func TestFindCliqueUnreachableTargetGivesMaximum(t *testing.T) {
+	rng := rand.New(rand.NewSource(6))
+	for trial := 0; trial < 40; trial++ {
+		n := 8 + rng.Intn(10)
+		adj := randomGraph(rng, n, 0.3+0.5*rng.Float64())
+		maxSize := bruteMaxClique(adj)
+
+		for _, workers := range []int{1, 3} {
+			for _, limit := range []int{0, 5, 1 << 20} {
+				r := FindClique(context.Background(), n, matrixFilter(adj), maxSize+1,
+					CliqueOptions{Workers: workers, ColorLimit: limit})
+				if r.Reached || !r.Exhausted || !r.Maximum {
+					t.Fatalf("trial %d workers %d limit %d: expected a proven maximum: %+v", trial, workers, limit, r)
+				}
+				if len(r.Clique) != maxSize || !isClique(adj, r.Clique) {
+					t.Fatalf("trial %d workers %d limit %d: expected a maximum clique of %d, got %v",
+						trial, workers, limit, maxSize, r.Clique)
+				}
+			}
+		}
+	}
+}
+
+// A long coloring must not delay the deadline.
+func TestFindCliqueColoringHonoursDeadline(t *testing.T) {
+	const n = 3000
+	adj := randomGraph(rand.New(rand.NewSource(7)), n, 0.5)
+	base := matrixFilter(adj)
+	slow := func(v int32, candidates, out []int32) []int32 {
+		time.Sleep(200 * time.Microsecond)
+		return base(v, candidates, out)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond)
+	defer cancel()
+	start := time.Now()
+	r := FindClique(ctx, n, slow, 100, CliqueOptions{Workers: 2, ColorLimit: 1 << 20})
+	if elapsed := time.Since(start); elapsed > 2*time.Second {
+		t.Fatalf("deadline not honoured by the coloring: %v", elapsed)
+	}
+	if r.Reached || !isClique(adj, r.Clique) {
+		t.Fatalf("unexpected result %+v", r)
+	}
+}

pkg/obiiter/batchiterator.go

@@ -465,7 +465,9 @@ func (iterator IBioSequence) RebatchBySize(maxBytes int, maxCount int) IBioSeque
 
 	go func() {
 		order := 0
-		iterator = iterator.SortBatches()
+		// a local variable: reassigning the captured iterator would race with
+		// the IsPaired call below, made by the caller on the same variable
+		sorted := iterator.SortBatches()
 		buffer := obiseq.MakeBioSequenceSlice()
 		bufBytes := 0
 		source := ""
@@ -479,8 +481,8 @@ func (iterator IBioSequence) RebatchBySize(maxBytes int, maxCount int) IBioSeque
 			}
 		}
 
-		for iterator.Next() {
+		for sorted.Next() {
-			seqs := iterator.Get()
+			seqs := sorted.Get()
 			source = seqs.Source()
 			for _, s := range seqs.Slice() {
 				sz := s.MemorySize()
@@ -683,7 +685,8 @@ func (iterator IBioSequence) FilterOn(predicate obiseq.SequencePredicate,
 			}
 
 			seqs.slice = slice[:j]
-			trueIter.pointer.channel <- seqs
+			trueIter.Push(seqs)
+			// trueIter.pointer.channel <- seqs
 		}
 
 		trueIter.Done()

pkg/obikmer/debruijn.go

@@ -5,7 +5,6 @@ import (
 	"container/heap"
 	"fmt"
 	"math"
-	"math/bits"
 	"os"
 	"slices"
 
@@ -610,14 +609,6 @@ func (graph *DeBruijnGraph) WriteGml(filename string) error {
 	return err
 }
 
-// Calculating the hamming distance between two k-mers.
-func (g *DeBruijnGraph) HammingDistance(kmer1, kmer2 uint64) int {
-	ident := ^((kmer1 & kmer2) | (^kmer1 & ^kmer2))
-	ident |= (ident >> 1)
-	ident &= 0x5555555555555555 & g.kmermask
-	return bits.OnesCount64(ident)
-}
-
 type UInt64Heap []uint64
 
 func (h UInt64Heap) Len() int           { return len(h) }

pkg/obikmer/enumkmer.go

@@ -0,0 +1,129 @@
+package obikmer
+
+import "iter"
+
+// KmerConstraints describes lexicographic properties a k-mer must satisfy.
+//
+// MaxCount is indexed by the 2-bit base code (A=0, C=1, G=2, T=3):
+// a negative value means unlimited, 0 forbids the base altogether
+// (so MaxCount[2] = 0 restricts the alphabet to A, C, T).
+//
+// MaxGC is the maximum number of C or G bases; a negative value means
+// unlimited.
+//
+// MaxRun is the maximum number of consecutive identical bases;
+// a value <= 0 means unlimited. MinRun requires at least one run of that
+// length; a value <= 1 means no requirement.
+//
+// Accepted and Rejected are optional per-position sets of bases (4 bits masks,
+// bit b set when the base of code b is in the set, see ParseIUPACMasks).
+// When set, their length must be k. A k-mer is accepted only if each of its
+// bases belongs to the Accepted set of its position, and is refused if every
+// one of its bases belongs to the Rejected set of its position.
+//
+// The zero value forbids every base: use NoKmerConstraints as a starting point.
+type KmerConstraints struct {
+	MaxCount [4]int
+	MaxGC    int
+	MaxRun   int
+	MinRun   int
+	Accepted []uint8
+	Rejected []uint8
+}
+
+// NoKmerConstraints returns constraints accepting every k-mer.
+func NoKmerConstraints() KmerConstraints {
+	return KmerConstraints{MaxCount: [4]int{-1, -1, -1, -1}, MaxGC: -1}
+}
+
+// IterConstrainedKmers returns an iterator over all k-mers of size k
+// satisfying the constraints c, encoded as in EncodeKmer, in increasing
+// numerical order.
+//
+// K-mers are built base by base and a branch is abandoned as soon as a
+// constraint is violated or cannot be met any more (MaxCount, MaxGC, MaxRun
+// and Accepted are monotone: a violating prefix cannot be repaired by
+// extension; MinRun is checked against the positions left), so the cost is
+// proportional to the number of explored prefixes, not to 4^k. Rejected is
+// checked on complete k-mers.
+//
+// k must be between 1 and 31, otherwise the iterator is empty. It is also
+// empty if Accepted or Rejected is set with a length different from k.
+//
+// Example:
+//
+//	c := NoKmerConstraints()
+//	c.MaxCount[2] = 0 // no G
+//	c.MaxRun = 3
+//	for kmer := range IterConstrainedKmers(8, c) { ... }
+func IterConstrainedKmers(k int, c KmerConstraints) iter.Seq[uint64] {
+	return func(yield func(uint64) bool) {
+		if k < 1 || k > 31 || !c.validPositions(k) {
+			return
+		}
+
+		// Nothing to enumerate if the per-base caps cannot even sum to k.
+		capacity := 0
+		for _, m := range c.MaxCount {
+			if m < 0 || m >= k {
+				capacity += k
+			} else {
+				capacity += m
+			}
+		}
+		if capacity < k {
+			return
+		}
+
+		var counts [4]int
+		gc := 0
+
+		var walk func(depth int, kmer uint64, last, run, longest int) bool
+		walk = func(depth int, kmer uint64, last, run, longest int) bool {
+			if depth == k {
+				if c.Rejected != nil && matchesAllMasks(kmer, k, c.Rejected) {
+					return true
+				}
+				return yield(kmer)
+			}
+			for b := range 4 {
+				if c.MaxCount[b] >= 0 && counts[b] >= c.MaxCount[b] {
+					continue
+				}
+				if c.Accepted != nil && c.Accepted[depth]>>uint(b)&1 == 0 {
+					continue
+				}
+				isGC := b == 1 || b == 2
+				if isGC && c.MaxGC >= 0 && gc >= c.MaxGC {
+					continue
+				}
+				r := 1
+				if b == last {
+					r = run + 1
+				}
+				if c.MaxRun > 0 && r > c.MaxRun {
+					continue
+				}
+				l := max(longest, r)
+				if c.MinRun > 1 && l < c.MinRun && r+(k-depth-1) < c.MinRun {
+					continue
+				}
+				counts[b]++
+				if isGC {
+					gc++
+				}
+				ok := walk(depth+1, kmer<<2|uint64(b), b, r, l)
+				counts[b]--
+				if isGC {
+					gc--
+				}
+				if !ok {
+					return false
+				}
+			}
+			return true
+		}
+
+		walk(0, 0, -1, 0, 0)
+	}
+}

pkg/obikmer/enumkmer_test.go

@@ -0,0 +1,85 @@
+package obikmer
+
+import (
+	"strings"
+	"testing"
+)
+
+func collectConstrained(k int, c KmerConstraints) []uint64 {
+	var out []uint64
+	for km := range IterConstrainedKmers(k, c) {
+		out = append(out, km)
+	}
+	return out
+}
+
+func TestIterConstrainedKmersUnconstrained(t *testing.T) {
+	got := collectConstrained(4, NoKmerConstraints())
+	if len(got) != 256 {
+		t.Fatalf("expected 256 k-mers, got %d", len(got))
+	}
+	for i, km := range got {
+		if km != uint64(i) {
+			t.Fatalf("not in increasing order at %d: %d", i, km)
+		}
+	}
+}
+
+func TestIterConstrainedKmersNoG(t *testing.T) {
+	c := NoKmerConstraints()
+	c.MaxCount[2] = 0
+	got := collectConstrained(5, c)
+	if len(got) != 243 {
+		t.Fatalf("expected 3^5 = 243 k-mers, got %d", len(got))
+	}
+	for _, km := range got {
+		if strings.Contains(string(DecodeKmer(km, 5, nil)), "g") {
+			t.Fatalf("k-mer %d contains G", km)
+		}
+	}
+}
+
+func TestIterConstrainedKmersMaxRun(t *testing.T) {
+	c := NoKmerConstraints()
+	c.MaxRun = 2
+	// a(n) = 3*(a(n-1)+a(n-2)), a(1)=4, a(2)=16
+	want := map[int]int{3: 60, 4: 228}
+	for k, n := range want {
+		if got := len(collectConstrained(k, c)); got != n {
+			t.Fatalf("k=%d: expected %d, got %d", k, n, got)
+		}
+	}
+}
+
+func TestIterConstrainedKmersMaxCount(t *testing.T) {
+	c := NoKmerConstraints()
+	c.MaxCount[0] = 1 // at most one A
+	for _, km := range collectConstrained(6, c) {
+		if strings.Count(string(DecodeKmer(km, 6, nil)), "a") > 1 {
+			t.Fatalf("k-mer %d has more than one A", km)
+		}
+	}
+}
+
+func TestIterConstrainedKmersImpossible(t *testing.T) {
+	c := KmerConstraints{MaxCount: [4]int{1, 1, 1, 0}}
+	if got := collectConstrained(4, c); len(got) != 0 {
+		t.Fatalf("expected no k-mer, got %d", len(got))
+	}
+	if got := collectConstrained(0, NoKmerConstraints()); len(got) != 0 {
+		t.Fatalf("expected no k-mer for k=0")
+	}
+}
+
+func TestIterConstrainedKmersEarlyStop(t *testing.T) {
+	n := 0
+	for range IterConstrainedKmers(10, NoKmerConstraints()) {
+		n++
+		if n == 5 {
+			break
+		}
+	}
+	if n != 5 {
+		t.Fatalf("early stop failed: %d", n)
+	}
+}

pkg/obikmer/kmer_constraints.go

@@ -0,0 +1,141 @@
+package obikmer
+
+import (
+	"fmt"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+// ParseIUPACMasks converts an IUPAC nucleotide pattern into one 4 bits mask per
+// position (bit 0 is A, bit 1 is C, bit 2 is G, bit 3 is T), the format used by
+// KmerConstraints.Accepted and Rejected.
+func ParseIUPACMasks(pattern string) ([]uint8, error) {
+	masks := make([]uint8, len(pattern))
+	for i := 0; i < len(pattern); i++ {
+		m := obiseq.IUPACNucMask(pattern[i])
+		if m == 0 {
+			return nil, fmt.Errorf("invalid IUPAC nucleotide code %q at position %d of pattern %q",
+				pattern[i], i+1, pattern)
+		}
+		masks[i] = m
+	}
+	return masks, nil
+}
+
+// RestrictAlphabet forbids every base that is not denoted by one of the IUPAC
+// codes of alphabet. For instance "ACT" (or "H") forbids G.
+func (c *KmerConstraints) RestrictAlphabet(alphabet string) error {
+	masks, err := ParseIUPACMasks(alphabet)
+	if err != nil {
+		return err
+	}
+	var allowed uint8
+	for _, m := range masks {
+		allowed |= m
+	}
+	if allowed == 0 {
+		return fmt.Errorf("empty alphabet")
+	}
+	for b := range 4 {
+		if allowed>>uint(b)&1 == 0 {
+			c.MaxCount[b] = 0
+		}
+	}
+	return nil
+}
+
+// validPositions tells whether Accepted and Rejected, when set, have length k.
+func (c *KmerConstraints) validPositions(k int) bool {
+	return (c.Accepted == nil || len(c.Accepted) == k) &&
+		(c.Rejected == nil || len(c.Rejected) == k)
+}
+
+// matchesAllMasks tells whether every base of the k-mer belongs to the set
+// given for its position.
+func matchesAllMasks(kmer uint64, k int, masks []uint8) bool {
+	for i := k - 1; i >= 0; i-- {
+		if masks[i]>>(kmer&3)&1 == 0 {
+			return false
+		}
+		kmer >>= 2
+	}
+	return true
+}
+
+// Accepts tells whether a k-mer of size k satisfies every constraint.
+// It is the predicate whose solutions IterConstrainedKmers enumerates, to be
+// used to filter k-mers coming from elsewhere.
+func (c *KmerConstraints) Accepts(kmer uint64, k int) bool {
+	if k < 1 || k > 31 || !c.validPositions(k) {
+		return false
+	}
+
+	var counts [4]int
+	gc, run, longest, last := 0, 0, 0, -1
+	for i := k - 1; i >= 0; i-- {
+		b := int(kmer>>(2*uint(i))) & 3
+		if c.Accepted != nil && c.Accepted[k-1-i]>>uint(b)&1 == 0 {
+			return false
+		}
+		counts[b]++
+		if c.MaxCount[b] >= 0 && counts[b] > c.MaxCount[b] {
+			return false
+		}
+		if b == 1 || b == 2 {
+			gc++
+			if c.MaxGC >= 0 && gc > c.MaxGC {
+				return false
+			}
+		}
+		if b == last {
+			run++
+		} else {
+			run = 1
+		}
+		last = b
+		longest = max(longest, run)
+		if c.MaxRun > 0 && run > c.MaxRun {
+			return false
+		}
+	}
+
+	if c.MinRun > 1 && longest < c.MinRun {
+		return false
+	}
+	if c.Rejected != nil && matchesAllMasks(kmer, k, c.Rejected) {
+		return false
+	}
+	return true
+}
+
+// HammingBound returns the sphere-packing (Hamming) upper bound on the number
+// of words of length k over an alphabet of alphabetSize letters that can be
+// pairwise at Hamming distance >= d. No larger family exists; a family of
+// that size may not exist either.
+func HammingBound(alphabetSize, k, d int) float64 {
+	if d <= 1 {
+		total := 1.0
+		for i := 0; i < k; i++ {
+			total *= float64(alphabetSize)
+		}
+		return total
+	}
+	if d%2 == 0 { // A(k, d) <= A(k-1, d-1)
+		k--
+		d--
+	}
+	if k < 1 {
+		return 1
+	}
+	t := (d - 1) / 2
+	total, ball, binom, pow := 1.0, 1.0, 1.0, 1.0
+	for i := 0; i < k; i++ {
+		total *= float64(alphabetSize)
+	}
+	for i := 1; i <= t && i <= k; i++ {
+		binom = binom * float64(k-i+1) / float64(i)
+		pow *= float64(alphabetSize - 1)
+		ball += binom * pow
+	}
+	return total / ball
+}

pkg/obikmer/kmer_constraints_test.go

@@ -0,0 +1,123 @@
+package obikmer
+
+import (
+	"math"
+	"math/rand"
+	"testing"
+)
+
+// IterConstrainedKmers must enumerate exactly the k-mers accepted by Accepts.
+func TestIterConstrainedKmersMatchesAccepts(t *testing.T) {
+	rng := rand.New(rand.NewSource(10))
+	patterns := []string{"", "nnnnnn", "yrnnnn", "cnnnnn", "annnnt"}
+
+	for trial := 0; trial < 60; trial++ {
+		k := 6
+		c := NoKmerConstraints()
+		for b := range 4 {
+			if rng.Intn(3) == 0 {
+				c.MaxCount[b] = rng.Intn(4)
+			}
+		}
+		if rng.Intn(2) == 0 {
+			c.MaxGC = rng.Intn(5)
+		}
+		if rng.Intn(2) == 0 {
+			c.MaxRun = 1 + rng.Intn(3)
+		}
+		if rng.Intn(2) == 0 {
+			c.MinRun = 2 + rng.Intn(3)
+		}
+		if p := patterns[rng.Intn(len(patterns))]; p != "" {
+			c.Accepted, _ = ParseIUPACMasks(p)
+		}
+		if p := patterns[rng.Intn(len(patterns))]; p != "" && rng.Intn(2) == 0 {
+			c.Rejected, _ = ParseIUPACMasks(p)
+		}
+
+		var want []uint64
+		for km := uint64(0); km < 1<<(2*uint(k)); km++ {
+			if c.Accepts(km, k) {
+				want = append(want, km)
+			}
+		}
+		got := collectConstrained(k, c)
+		if len(got) != len(want) {
+			t.Fatalf("trial %d %+v: got %d k-mers, want %d", trial, c, len(got), len(want))
+		}
+		for i := range got {
+			if got[i] != want[i] {
+				t.Fatalf("trial %d %+v: mismatch at %d: %d != %d", trial, c, i, got[i], want[i])
+			}
+		}
+	}
+}
+
+func TestConstraintsGCAndRuns(t *testing.T) {
+	enc := func(s string) uint64 { return EncodeKmer([]byte(s), len(s)) }
+
+	c := NoKmerConstraints()
+	c.MaxGC = 2
+	if !c.Accepts(enc("ACGTAA"), 6) || c.Accepts(enc("ACGCAA"), 6) {
+		t.Fatal("MaxGC not enforced")
+	}
+
+	c = NoKmerConstraints()
+	c.MinRun = 3
+	if !c.Accepts(enc("ACCCGT"), 6) || c.Accepts(enc("ACCGGT"), 6) {
+		t.Fatal("MinRun not enforced")
+	}
+
+	c = NoKmerConstraints()
+	c.Rejected, _ = ParseIUPACMasks("cnnnnn") // no C in first position
+	if c.Accepts(enc("CAAAAA"), 6) || !c.Accepts(enc("AAAAAA"), 6) {
+		t.Fatal("Rejected not enforced")
+	}
+}
+
+func TestRestrictAlphabetAndPatterns(t *testing.T) {
+	c := NoKmerConstraints()
+	if err := c.RestrictAlphabet("ACT"); err != nil {
+		t.Fatal(err)
+	}
+	if got := len(collectConstrained(4, c)); got != 81 {
+		t.Fatalf("expected 3^4 = 81 k-mers, got %d", got)
+	}
+	c = NoKmerConstraints()
+	if err := c.RestrictAlphabet("H"); err != nil { // H = A, C or T
+		t.Fatal(err)
+	}
+	if c.MaxCount != [4]int{-1, -1, 0, -1} {
+		t.Fatalf("unexpected counts %v", c.MaxCount)
+	}
+	if err := c.RestrictAlphabet("AX"); err == nil {
+		t.Fatal("invalid IUPAC code must be refused")
+	}
+	if _, err := ParseIUPACMasks("acgz"); err == nil {
+		t.Fatal("invalid IUPAC code must be refused")
+	}
+	// wrong pattern length
+	c = NoKmerConstraints()
+	c.Accepted, _ = ParseIUPACMasks("nnn")
+	if got := collectConstrained(4, c); len(got) != 0 {
+		t.Fatalf("pattern of wrong length must give nothing, got %d", len(got))
+	}
+}
+
+func TestHammingBound(t *testing.T) {
+	cases := []struct {
+		q, k, d int
+		want    float64
+	}{
+		{4, 7, 3, 16384.0 / 22},
+		{4, 8, 4, 16384.0 / 22}, // d even: reduces to (7, 3)
+		{4, 9, 5, 262144.0 / 352},
+		{3, 9, 3, 19683.0 / 19},
+		{4, 5, 1, 1024},
+	}
+	for _, c := range cases {
+		if got := HammingBound(c.q, c.k, c.d); math.Abs(got-c.want) > 1e-6 {
+			t.Errorf("HammingBound(%d,%d,%d) = %v, want %v", c.q, c.k, c.d, got, c.want)
+		}
+	}
+}

pkg/obikmer/kmer_dissimilarity.go

@@ -0,0 +1,84 @@
+package obikmer
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+)
+
+// KmerMetric selects the distance used to compare two k-mers.
+type KmerMetric int
+
+const (
+	// HammingMetric counts the positions at which two k-mers differ.
+	HammingMetric KmerMetric = iota
+	// LCSMetric is the alignment length minus the LCS length (see KmerLCS).
+	LCSMetric
+)
+
+// ParseKmerMetric converts "hamming" or "lcs" (case insensitive) to a KmerMetric.
+func ParseKmerMetric(name string) (KmerMetric, error) {
+	switch strings.ToLower(name) {
+	case "hamming":
+		return HammingMetric, nil
+	case "lcs":
+		return LCSMetric, nil
+	}
+	return HammingMetric, fmt.Errorf("unknown distance type %q (expected hamming or lcs)", name)
+}
+
+func (m KmerMetric) String() string {
+	if m == LCSMetric {
+		return "lcs"
+	}
+	return "hamming"
+}
+
+type lcsScratch struct {
+	partners []uint64
+	results  []KmerLCSResult
+}
+
+// NewKmerDissimilarityFilter describes the graph whose vertices are kmers[i]
+// (k-mers of size k, 1 <= k <= 31) and where two vertices are linked when
+// their distance is at least dmin. The returned function has the signature of
+// obigraph.NeighborFilter: it appends to out[:0] the candidates (indices in
+// kmers) linked to v. It is safe for concurrent use.
+func NewKmerDissimilarityFilter(kmers []uint64, k int, metric KmerMetric, dmin int) func(v int32, candidates, out []int32) []int32 {
+	if metric == HammingMetric {
+		return func(v int32, candidates, out []int32) []int32 {
+			q := kmers[v]
+			out = out[:0]
+			for _, c := range candidates {
+				if HammingDistance(q, kmers[c], k) >= dmin {
+					out = append(out, c)
+				}
+			}
+			return out
+		}
+	}
+
+	pool := sync.Pool{New: func() any { return new(lcsScratch) }}
+	return func(v int32, candidates, out []int32) []int32 {
+		s := pool.Get().(*lcsScratch)
+		defer pool.Put(s)
+
+		s.partners = s.partners[:0]
+		for _, c := range candidates {
+			s.partners = append(s.partners, kmers[c])
+		}
+		if cap(s.results) < len(candidates) {
+			s.results = make([]KmerLCSResult, len(candidates))
+		}
+		s.results = s.results[:len(candidates)]
+		KmerLCSBatch(kmers[v], s.partners, k, s.results)
+
+		out = out[:0]
+		for i, c := range candidates {
+			if s.results[i].Distance() >= dmin {
+				out = append(out, c)
+			}
+		}
+		return out
+	}
+}

pkg/obikmer/kmer_dissimilarity_test.go

@@ -0,0 +1,139 @@
+package obikmer_test
+
+import (
+	"context"
+	"math/rand"
+	"testing"
+	"time"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obigraph"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obikmer"
+)
+
+func allKmers(k int) []uint64 {
+	kmers := make([]uint64, 0, 1<<(2*uint(k)))
+	for km := range obikmer.IterConstrainedKmers(k, obikmer.NoKmerConstraints()) {
+		kmers = append(kmers, km)
+	}
+	return kmers
+}
+
+func TestDissimilarityFilterMatchesDistances(t *testing.T) {
+	const k = 7
+	rng := rand.New(rand.NewSource(1))
+	kmers := make([]uint64, 300)
+	for i := range kmers {
+		kmers[i] = rng.Uint64() & (1<<(2*k) - 1)
+	}
+	candidates := make([]int32, 0, len(kmers))
+	for i := 1; i < len(kmers); i++ {
+		candidates = append(candidates, int32(i))
+	}
+
+	for _, metric := range []obikmer.KmerMetric{obikmer.HammingMetric, obikmer.LCSMetric} {
+		for _, dmin := range []int{1, 3, 5} {
+			filter := obikmer.NewKmerDissimilarityFilter(kmers, k, metric, dmin)
+			got := filter(0, candidates, nil)
+
+			var want []int32
+			for _, c := range candidates {
+				var d int
+				if metric == obikmer.HammingMetric {
+					d = obikmer.HammingDistance(kmers[0], kmers[c], k)
+				} else {
+					d = obikmer.KmerLCS(kmers[0], kmers[c], k).Distance()
+				}
+				if d >= dmin {
+					want = append(want, c)
+				}
+			}
+			if len(got) != len(want) {
+				t.Fatalf("%v d>=%d: got %d vertices, want %d", metric, dmin, len(got), len(want))
+			}
+			for i := range got {
+				if got[i] != want[i] {
+					t.Fatalf("%v d>=%d: mismatch at %d", metric, dmin, i)
+				}
+			}
+		}
+	}
+}
+
+// A family of 5-mers pairwise at Hamming distance >= 3, built with FindClique.
+func TestFindClique_HammingFamily(t *testing.T) {
+	const k, dmin, target = 5, 3, 32
+	kmers := allKmers(k)
+	filter := obikmer.NewKmerDissimilarityFilter(kmers, k, obikmer.HammingMetric, dmin)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	r := obigraph.FindClique(ctx, len(kmers), filter, target, obigraph.CliqueOptions{Workers: 2})
+	if !r.Reached {
+		t.Fatalf("family of %d not found (best %d, exhausted %v)", target, len(r.Clique), r.Exhausted)
+	}
+	for i, a := range r.Clique {
+		for _, b := range r.Clique[i+1:] {
+			if obikmer.HammingDistance(kmers[a], kmers[b], k) < dmin {
+				t.Fatalf("%d and %d are too close", kmers[a], kmers[b])
+			}
+		}
+	}
+}
+
+func TestFindClique_LCSFamily(t *testing.T) {
+	const k, dmin, target = 6, 3, 20
+	kmers := allKmers(k)
+	filter := obikmer.NewKmerDissimilarityFilter(kmers, k, obikmer.LCSMetric, dmin)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+	r := obigraph.FindClique(ctx, len(kmers), filter, target, obigraph.CliqueOptions{Workers: 2})
+	if !r.Reached {
+		t.Fatalf("family of %d not found (best %d, exhausted %v)", target, len(r.Clique), r.Exhausted)
+	}
+	for i, a := range r.Clique {
+		for _, b := range r.Clique[i+1:] {
+			if obikmer.KmerLCS(kmers[a], kmers[b], k).Distance() < dmin {
+				t.Fatalf("%d and %d are too close", kmers[a], kmers[b])
+			}
+		}
+	}
+}
+
+func TestParseKmerMetric(t *testing.T) {
+	if m, err := obikmer.ParseKmerMetric("LCS"); err != nil || m != obikmer.LCSMetric {
+		t.Fatal("lcs not parsed")
+	}
+	if m, err := obikmer.ParseKmerMetric("hamming"); err != nil || m != obikmer.HammingMetric {
+		t.Fatal("hamming not parsed")
+	}
+	if _, err := obikmer.ParseKmerMetric("levenshtein"); err == nil {
+		t.Fatal("unknown metric must be refused")
+	}
+}
+
+// With a target out of reach (17 4-mers at Hamming distance >= 3 do not exist,
+// 16 do), the search must return the best family it can build, not a poor one
+// stuck by the pruning on the target, and the coloring bound must prove that
+// the target is out of reach.
+func TestFindClique_BestWhenTargetOutOfReach(t *testing.T) {
+	const k, dmin = 4, 3
+	kmers := allKmers(k)
+	filter := obikmer.NewKmerDissimilarityFilter(kmers, k, obikmer.HammingMetric, dmin)
+
+	for _, limit := range []int{0, 1024} {
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+		r := obigraph.FindClique(ctx, len(kmers), filter, 17, obigraph.CliqueOptions{Workers: 2, ColorLimit: limit})
+		cancel()
+		if r.Reached {
+			t.Fatal("17 4-mers at Hamming distance >= 3 cannot exist (sphere-packing bound 19, true maximum 16)")
+		}
+		if len(r.Clique) < 16 {
+			t.Fatalf("limit %d: expected the family of 16 to be found, got %d (exhausted %v, maximum %v)",
+				limit, len(r.Clique), r.Exhausted, r.Maximum)
+		}
+		if limit > 0 && !r.Exhausted {
+			t.Fatalf("limit %d: the coloring bound should prove that 17 is out of reach", limit)
+		}
+	}
+}

pkg/obikmer/kmer_distance.go

@@ -0,0 +1,136 @@
+package obikmer
+
+import "math/bits"
+
+// KmerLCSResult is the result of a k-mer LCS alignment: the length of the
+// Longest Common Subsequence and the length of the shortest alignment
+// achieving it (same semantics as obialign.FastLCSScore).
+type KmerLCSResult struct {
+	LCS      int
+	AlignLen int
+}
+
+// Distance returns AlignLen - LCS, the LCS-mode edit distance.
+func (r KmerLCSResult) Distance() int {
+	return r.AlignLen - r.LCS
+}
+
+// HammingDistance returns the number of positions at which the two k-mers
+// differ. Both k-mers are encoded with 2 bits per base (see EncodeKmer);
+// bits above the 2*k lowest ones are ignored. k must be between 0 and 32.
+func HammingDistance(a, b uint64, k int) int {
+	diff := a ^ b
+	diff = (diff | diff>>1) & 0x5555555555555555
+	diff &= ^uint64(0) >> (64 - 2*k)
+	return bits.OnesCount64(diff)
+}
+
+// KmerLCS aligns two k-mers of the same size in LCS mode (match +1, mismatch
+// and indel 0) and returns the LCS length together with the length of the
+// shortest alignment among those reaching it, so that the distance is
+// AlignLen - LCS.
+//
+// It is the scalar reference implementation, straightforward on purpose:
+// it is the oracle against which the vectorised versions are tested.
+// It returns {-1, -1} if k is not between 1 and 31.
+func KmerLCS(a, b uint64, k int) KmerLCSResult {
+	if k < 1 || k > 31 {
+		return KmerLCSResult{-1, -1}
+	}
+
+	var ba, bb [32]uint8
+	for i := k - 1; i >= 0; i-- {
+		ba[i] = uint8(a & 3)
+		bb[i] = uint8(b & 3)
+		a >>= 2
+		b >>= 2
+	}
+
+	// One DP row: best (score, alignment length) per column.
+	var score, length [32]int
+	for j := 0; j <= k; j++ {
+		length[j] = j
+	}
+
+	better := func(s1, l1, s2, l2 int) bool {
+		return s1 > s2 || (s1 == s2 && l1 < l2)
+	}
+
+	for i := 1; i <= k; i++ {
+		diagS, diagL := score[0], length[0]
+		score[0], length[0] = 0, i
+
+		for j := 1; j <= k; j++ {
+			upS, upL := score[j], length[j]
+
+			bestS, bestL := diagS, diagL+1
+			if ba[j-1] == bb[i-1] {
+				bestS++
+			}
+			if better(upS, upL+1, bestS, bestL) {
+				bestS, bestL = upS, upL+1
+			}
+			if better(score[j-1], length[j-1]+1, bestS, bestL) {
+				bestS, bestL = score[j-1], length[j-1]+1
+			}
+
+			diagS, diagL = upS, upL
+			score[j], length[j] = bestS, bestL
+		}
+	}
+
+	return KmerLCSResult{score[k], length[k]}
+}
+
+// KmerLCSBatch computes KmerLCS(query, partners[i], k) for every partner and
+// stores it in out[i]. out must be at least as long as partners.
+//
+// When built with GOEXPERIMENT=simd the partners are aligned several at a
+// time, one per SIMD lane; otherwise the scalar reference is used.
+// It returns false, leaving out untouched, if k is not between 1 and 31 or
+// if out is too short.
+func KmerLCSBatch(query uint64, partners []uint64, k int, out []KmerLCSResult) bool {
+	if k < 1 || k > 31 || len(out) < len(partners) {
+		return false
+	}
+	kmerLCSBatch(query, partners, k, out)
+	return true
+}
+
+// KmerLCSPartners is a fixed set of k-mers prepared once to be aligned against
+// many queries with KmerLCSBatch-like speed, without redoing the per-call
+// reorganisation of the partners. It is read-only after creation and can be
+// shared between goroutines.
+type KmerLCSPartners struct {
+	prepared kmerLCSPrepared
+	n        int
+}
+
+// NewKmerLCSPartners prepares partners for repeated alignment.
+// It returns nil if k is not between 1 and 31. The slice is not copied when
+// built without GOEXPERIMENT=simd, so it must not be modified afterwards.
+func NewKmerLCSPartners(partners []uint64, k int) *KmerLCSPartners {
+	if k < 1 || k > 31 {
+		return nil
+	}
+	return &KmerLCSPartners{prepared: prepareKmerLCS(partners, k), n: len(partners)}
+}
+
+// Len returns the number of prepared partners.
+func (p *KmerLCSPartners) Len() int { return p.n }
+
+// Align computes the alignment of query against every prepared partner and
+// stores it in out. It returns false if out is shorter than Len().
+func (p *KmerLCSPartners) Align(query uint64, out []KmerLCSResult) bool {
+	if len(out) < p.n {
+		return false
+	}
+	p.prepared.align(query, out[:p.n])
+	return true
+}
+
+func kmerLCSBatchScalar(query uint64, partners []uint64, k int, out []KmerLCSResult) {
+	for i, p := range partners {
+		out[i] = KmerLCS(query, p, k)
+	}
+}

pkg/obikmer/kmer_distance_test.go

@@ -0,0 +1,171 @@
+package obikmer
+
+import (
+	"math/rand"
+	"testing"
+)
+
+func randomKmer(rng *rand.Rand, k int) uint64 {
+	return rng.Uint64() & (uint64(1)<<(2*uint(k)) - 1)
+}
+
+func TestHammingDistance(t *testing.T) {
+	rng := rand.New(rand.NewSource(1))
+	for _, k := range []int{1, 2, 7, 21, 31} {
+		for n := 0; n < 200; n++ {
+			a, b := randomKmer(rng, k), randomKmer(rng, k)
+			sa, sb := DecodeKmer(a, k, nil), DecodeKmer(b, k, nil)
+			want := 0
+			for i := range sa {
+				if sa[i] != sb[i] {
+					want++
+				}
+			}
+			if got := HammingDistance(a, b, k); got != want {
+				t.Fatalf("k=%d %s/%s: got %d, want %d", k, sa, sb, got, want)
+			}
+		}
+	}
+}
+
+func TestHammingDistanceIgnoresHighBits(t *testing.T) {
+	if got := HammingDistance(0xFF00, 0x0000, 4); got != 0 {
+		t.Fatalf("bits above 2k must be ignored, got %d", got)
+	}
+}
+
+func TestKmerLCSKnown(t *testing.T) {
+	enc := func(s string) uint64 { return EncodeKmer([]byte(s), len(s)) }
+	cases := []struct {
+		a, b string
+		want KmerLCSResult
+	}{
+		{"ACGT", "ACGT", KmerLCSResult{4, 4}},
+		{"AAAA", "CCCC", KmerLCSResult{0, 4}},
+		{"ACGT", "ACGA", KmerLCSResult{3, 4}},
+		{"ACGT", "CGTA", KmerLCSResult{3, 5}}, // one shift: 1 indel on each side
+	}
+	for _, c := range cases {
+		if got := KmerLCS(enc(c.a), enc(c.b), 4); got != c.want {
+			t.Errorf("%s/%s: got %+v, want %+v", c.a, c.b, got, c.want)
+		}
+	}
+	if got := KmerLCS(0, 0, 32); got.LCS != -1 {
+		t.Errorf("k=32 must be rejected, got %+v", got)
+	}
+}
+
+// The batch implementation (SIMD when built with GOEXPERIMENT=simd) must agree
+// with the scalar reference for all k and for batch sizes around the lane count.
+func TestKmerLCSBatchMatchesReference(t *testing.T) {
+	rng := rand.New(rand.NewSource(2))
+	for _, k := range []int{1, 2, 3, 5, 8, 16, 21, 30, 31} {
+		for _, n := range []int{0, 1, 3, 7, 8, 9, 16, 17, 100} {
+			query := randomKmer(rng, k)
+			partners := make([]uint64, n)
+			for i := range partners {
+				partners[i] = randomKmer(rng, k)
+			}
+			if n > 2 {
+				partners[0] = query // identical
+			}
+			out := make([]KmerLCSResult, n)
+			if !KmerLCSBatch(query, partners, k, out) {
+				t.Fatalf("k=%d n=%d: batch refused", k, n)
+			}
+			for i, p := range partners {
+				if want := KmerLCS(query, p, k); out[i] != want {
+					t.Fatalf("k=%d n=%d partner %d: got %+v, want %+v", k, n, i, out[i], want)
+				}
+			}
+		}
+	}
+}
+
+func TestKmerLCSBatchRejectsBadInput(t *testing.T) {
+	if KmerLCSBatch(0, []uint64{1}, 0, make([]KmerLCSResult, 1)) {
+		t.Fatal("k=0 must be rejected")
+	}
+	if KmerLCSBatch(0, []uint64{1, 2}, 4, make([]KmerLCSResult, 1)) {
+		t.Fatal("short output must be rejected")
+	}
+}
+
+func benchBatch(b *testing.B, k int, f func(uint64, []uint64, int, []KmerLCSResult)) {
+	rng := rand.New(rand.NewSource(3))
+	partners := make([]uint64, 1024)
+	for i := range partners {
+		partners[i] = randomKmer(rng, k)
+	}
+	out := make([]KmerLCSResult, len(partners))
+	query := randomKmer(rng, k)
+	b.ResetTimer()
+	for b.Loop() {
+		f(query, partners, k, out)
+	}
+	b.ReportMetric(float64(b.Elapsed().Nanoseconds())/float64(b.N*len(partners)), "ns/pair")
+}
+
+func BenchmarkKmerLCSScalar21(b *testing.B) { benchBatch(b, 21, kmerLCSBatchScalar) }
+func BenchmarkKmerLCSBatch21(b *testing.B)  { benchBatch(b, 21, kmerLCSBatch) }
+func BenchmarkKmerLCSScalar11(b *testing.B) { benchBatch(b, 11, kmerLCSBatchScalar) }
+func BenchmarkKmerLCSBatch11(b *testing.B)  { benchBatch(b, 11, kmerLCSBatch) }
+
+func TestKmerLCSPartnersMatchesReference(t *testing.T) {
+	rng := rand.New(rand.NewSource(6))
+	for _, k := range []int{1, 5, 21, 31} {
+		for _, n := range []int{0, 1, 7, 8, 9, 50} {
+			partners := make([]uint64, n)
+			for i := range partners {
+				partners[i] = randomKmer(rng, k)
+			}
+			prep := NewKmerLCSPartners(partners, k)
+			out := make([]KmerLCSResult, n)
+			for q := 0; q < 3; q++ { // reuse the same preparation for several queries
+				query := randomKmer(rng, k)
+				if !prep.Align(query, out) {
+					t.Fatalf("k=%d n=%d: Align refused", k, n)
+				}
+				for i, p := range partners {
+					if want := KmerLCS(query, p, k); out[i] != want {
+						t.Fatalf("k=%d n=%d partner %d: got %+v, want %+v", k, n, i, out[i], want)
+					}
+				}
+			}
+		}
+	}
+	if NewKmerLCSPartners(nil, 0) != nil || NewKmerLCSPartners(nil, 32) != nil {
+		t.Fatal("invalid k must give nil")
+	}
+}
+
+// All-pairs scenario: many queries against the same partners.
+func benchAllPairs(b *testing.B, k int, prepared bool) {
+	rng := rand.New(rand.NewSource(7))
+	partners := make([]uint64, 1024)
+	for i := range partners {
+		partners[i] = randomKmer(rng, k)
+	}
+	queries := make([]uint64, 64)
+	for i := range queries {
+		queries[i] = randomKmer(rng, k)
+	}
+	out := make([]KmerLCSResult, len(partners))
+	prep := NewKmerLCSPartners(partners, k)
+	b.ResetTimer()
+	for b.Loop() {
+		for _, q := range queries {
+			if prepared {
+				prep.Align(q, out)
+			} else {
+				KmerLCSBatch(q, partners, k, out)
+			}
+		}
+	}
+	b.ReportMetric(float64(b.Elapsed().Nanoseconds())/float64(b.N*len(partners)*len(queries)), "ns/pair")
+}
+
+func BenchmarkAllPairsBatch21(b *testing.B)    { benchAllPairs(b, 21, false) }
+func BenchmarkAllPairsPrepared21(b *testing.B) { benchAllPairs(b, 21, true) }
+func BenchmarkAllPairsBatch11(b *testing.B)    { benchAllPairs(b, 11, false) }
+func BenchmarkAllPairsPrepared11(b *testing.B) { benchAllPairs(b, 11, true) }

pkg/obikmer/kmer_lcs_obialign_test.go

@@ -0,0 +1,29 @@
+package obikmer_test
+
+import (
+	"math/rand"
+	"testing"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obialign"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obikmer"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+// KmerLCS must reproduce obialign.FastLCSScore on the decoded sequences.
+func TestKmerLCSMatchesFastLCSScore(t *testing.T) {
+	rng := rand.New(rand.NewSource(4))
+	for _, k := range []int{1, 2, 4, 9, 15, 21, 31} {
+		mask := uint64(1)<<(2*uint(k)) - 1
+		for n := 0; n < 300; n++ {
+			a, b := rng.Uint64()&mask, rng.Uint64()&mask
+			sa := obiseq.NewBioSequence("a", obikmer.DecodeKmer(a, k, nil), "")
+			sb := obiseq.NewBioSequence("b", obikmer.DecodeKmer(b, k, nil), "")
+			lcs, alen := obialign.FastLCSScore(sa, sb, -1, nil)
+			got := obikmer.KmerLCS(a, b, k)
+			if got.LCS != lcs || got.AlignLen != alen {
+				t.Fatalf("k=%d %s/%s: KmerLCS=%+v, FastLCSScore=(%d,%d)",
+					k, sa.String(), sb.String(), got, lcs, alen)
+			}
+		}
+	}
+}

pkg/obikmer/kmer_lcs_scalar.go

@@ -0,0 +1,21 @@
+//go:build !goexperiment.simd
+
+package obikmer
+
+func kmerLCSBatch(query uint64, partners []uint64, k int, out []KmerLCSResult) {
+	kmerLCSBatchScalar(query, partners, k, out)
+}
+
+// kmerLCSPrepared has nothing to precompute without SIMD.
+type kmerLCSPrepared struct {
+	k        int
+	partners []uint64
+}
+
+func prepareKmerLCS(partners []uint64, k int) kmerLCSPrepared {
+	return kmerLCSPrepared{k: k, partners: partners}
+}
+
+func (p *kmerLCSPrepared) align(query uint64, out []KmerLCSResult) {
+	kmerLCSBatchScalar(query, p.partners, p.k, out)
+}

pkg/obikmer/kmer_lcs_simd.go

@@ -0,0 +1,114 @@
+//go:build goexperiment.simd
+
+package obikmer
+
+import "simd"
+
+// Each lane of a vector holds the state of one (query, partner) alignment as
+//
+//	score<<6 | (63 - alignment length)
+//
+// so that taking the maximum of two cells ranks them by score first and, for
+// equal scores, by shortest alignment. With k <= 31 the score is <= 31 and the
+// alignment length <= 62, hence the value fits in 11 bits and the low field
+// is never exhausted. The three DP moves then all reduce to
+//
+//	cell(i,j) = max(diag + 64*match, up, left) - 1
+//
+// (every move lengthens the alignment by one; only a diagonal match adds 1
+// to the score).
+const (
+	lcsScoreUnit = 64
+	lcsLenBase   = 63
+)
+
+func simdLanes() int {
+	var probe simd.Uint16s
+	return probe.Len()
+}
+
+// transposeChunk writes the bases of up to `lanes` partners in dst, laid out
+// so that dst[j*lanes+lane] is base j of the partner in `lane`.
+func transposeChunk(chunk []uint64, k, lanes int, dst []uint16) {
+	for lane, p := range chunk {
+		for j := 0; j < k; j++ {
+			dst[j*lanes+lane] = uint16(p>>(2*uint(k-1-j))) & 3
+		}
+	}
+}
+
+// alignChunk aligns the query against the partners of one transposed chunk
+// and writes one result per lane in out (len(out) <= lanes).
+func alignChunk(query uint64, pbase, row []uint16, k, lanes int, out []KmerLCSResult) {
+	one := simd.BroadcastUint16s(1)
+	bonus := simd.BroadcastUint16s(lcsScoreUnit)
+
+	for j := 0; j <= k; j++ {
+		simd.BroadcastUint16s(uint16(lcsLenBase - j)).Store(row[j*lanes:])
+	}
+
+	for i := 1; i <= k; i++ {
+		qv := simd.BroadcastUint16s(uint16(query>>(2*uint(k-i))) & 3)
+		diag := simd.LoadUint16s(row)
+		left := simd.BroadcastUint16s(uint16(lcsLenBase - i))
+		left.Store(row)
+
+		for j := 1; j <= k; j++ {
+			up := simd.LoadUint16s(row[j*lanes:])
+			pv := simd.LoadUint16s(pbase[(j-1)*lanes:])
+			match := qv.Equal(pv)
+			best := diag.Add(bonus.Masked(match)).Max(up).Max(left).Sub(one)
+			best.Store(row[j*lanes:])
+			diag = up
+			left = best
+		}
+	}
+
+	last := row[k*lanes:]
+	for lane := range out {
+		v := int(last[lane])
+		out[lane] = KmerLCSResult{
+			LCS:      v / lcsScoreUnit,
+			AlignLen: lcsLenBase - v%lcsScoreUnit,
+		}
+	}
+}
+
+func kmerLCSBatch(query uint64, partners []uint64, k int, out []KmerLCSResult) {
+	lanes := simdLanes()
+	pbase := make([]uint16, k*lanes)
+	row := make([]uint16, (k+1)*lanes)
+
+	for start := 0; start < len(partners); start += lanes {
+		end := min(start+lanes, len(partners))
+		transposeChunk(partners[start:end], k, lanes, pbase)
+		alignChunk(query, pbase, row, k, lanes, out[start:end])
+	}
+}
+
+// kmerLCSPrepared holds partners already transposed, chunk after chunk.
+type kmerLCSPrepared struct {
+	k, lanes, n int
+	pbase       []uint16 // chunk c at [c*k*lanes : (c+1)*k*lanes]
+}
+
+func prepareKmerLCS(partners []uint64, k int) kmerLCSPrepared {
+	lanes := simdLanes()
+	chunks := (len(partners) + lanes - 1) / lanes
+	p := kmerLCSPrepared{k: k, lanes: lanes, n: len(partners),
+		pbase: make([]uint16, chunks*k*lanes)}
+	for c := 0; c < chunks; c++ {
+		end := min((c+1)*lanes, len(partners))
+		transposeChunk(partners[c*lanes:end], k, lanes, p.pbase[c*k*lanes:(c+1)*k*lanes])
+	}
+	return p
+}
+
+func (p *kmerLCSPrepared) align(query uint64, out []KmerLCSResult) {
+	row := make([]uint16, (p.k+1)*p.lanes)
+	size := p.k * p.lanes
+	for c, start := 0, 0; start < p.n; c, start = c+1, start+p.lanes {
+		end := min(start+p.lanes, p.n)
+		alignChunk(query, p.pbase[c*size:(c+1)*size], row, p.k, p.lanes, out[start:end])
+	}
+}

pkg/obikmer/seqquorum.go

@@ -0,0 +1,229 @@
+package obikmer
+
+import (
+	"bufio"
+	"encoding/binary"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
+)
+
+// SeqKmerCounter counts, for every canonical k-mer, in how many distinct input
+// sequences it occurs. It answers "which k-mers are present in at least q of
+// the N sequences" without one k-mer set per sequence: a KmerSetGroup needs
+// N*P partition files, which is out of reach for thousands of genomes, whereas
+// a SeqKmerCounter uses a single set of P files whatever N is.
+//
+// Construction is two-phase, like KmerSetGroupBuilder:
+//
+//  1. AddSequence extracts super-kmers (grouped by minimizer, so a k-mer and its
+//     reverse complement always land in the same partition) and appends them,
+//     tagged with the sequence id, to one of P temporary files. It is safe to
+//     call concurrently.
+//  2. CountAtLeast processes the partitions in parallel: each one is loaded,
+//     expanded to (canonical k-mer, sequence id) pairs, sorted, and reduced.
+//
+// Memory during phase 2 is 16 bytes per k-mer occurrence of the partitions
+// being processed at the same time. Their number is bounded by an occurrence
+// budget (see SetOccurrenceBudget), so that the memory does not depend on how
+// the k-mers happen to be spread over the partitions; P only has to be large
+// enough for the largest partition to be small against the budget (total bases
+// / P is the occurrence count of an average partition, see DefaultPartitions).
+//
+// Bases other than A, C, G, T, U break a sequence into independent runs: no
+// k-mer ever contains an ambiguous base.
+type SeqKmerCounter struct {
+	dir     string
+	k, m, p int
+
+	parts []*seqPartWriter
+
+	mu       sync.Mutex
+	firstErr error
+	sealed   bool
+
+	budget int // k-mer occurrences held at once in phase 2
+}
+
+type seqPartWriter struct {
+	mu    sync.Mutex
+	w     *bufio.Writer
+	file  *os.File
+	path  string
+	kmers int // k-mer occurrences written, so that the partition is loaded in an array of the exact size
+}
+
+// SeqKmerCount is the result for one k-mer.
+type SeqKmerCount struct {
+	Kmer      uint64 // canonical k-mer
+	Sequences uint32 // number of distinct sequences containing it
+	Repeated  bool   // occurs more than once (either strand) in at least one sequence
+}
+
+// NewSeqKmerCounter creates a counter whose temporary files live in a new
+// directory created under parentDir (use "" for the system temporary
+// directory). k is the k-mer size (2..31), m the minimizer size (<=0 for the
+// default) and partitions the number of partition files P.
+func NewSeqKmerCounter(parentDir string, k, m, partitions int) (*SeqKmerCounter, error) {
+	if k < 2 || k > 31 {
+		return nil, fmt.Errorf("obikmer: k must be between 2 and 31, got %d", k)
+	}
+	if partitions < 1 {
+		return nil, fmt.Errorf("obikmer: partitions must be >= 1, got %d", partitions)
+	}
+	if m <= 0 {
+		m = DefaultMinimizerSize(k)
+	}
+	if m >= k {
+		return nil, fmt.Errorf("obikmer: minimizer size %d must be smaller than k=%d", m, k)
+	}
+
+	dir, err := os.MkdirTemp(parentDir, "obikmer-seqcount-")
+	if err != nil {
+		return nil, err
+	}
+
+	c := &SeqKmerCounter{dir: dir, k: k, m: m, p: partitions, budget: DefaultOccurrenceBudget}
+	c.parts = make([]*seqPartWriter, partitions)
+	for i := range c.parts {
+		path := filepath.Join(dir, fmt.Sprintf("part_%04d.sqk", i))
+		f, err := os.Create(path)
+		if err != nil {
+			c.Close()
+			return nil, err
+		}
+		c.parts[i] = &seqPartWriter{w: bufio.NewWriterSize(f, 32*1024), file: f, path: path}
+	}
+	return c, nil
+}
+
+// DefaultOccurrenceBudget is the number of k-mer occurrences phase 2 holds at
+// once by default: 32 million, 512 MB. A partition larger than the budget is
+// processed alone. Measured on 16 cores, 526 million occurrences to count: with
+// 32M the count is as fast as without any limit; 16M make it 15% slower, 8M
+// 60% slower.
+const DefaultOccurrenceBudget = 32 << 20
+
+// SetOccurrenceBudget bounds the k-mer occurrences (16 bytes each) that
+// CountAtLeast holds in memory at once. It must be called before CountAtLeast.
+func (c *SeqKmerCounter) SetOccurrenceBudget(occurrences int) {
+	if occurrences > 0 {
+		c.budget = occurrences
+	}
+}
+
+// partitionOccurrences is the occurrence count aimed at for an average
+// partition by DefaultPartitions (a partition of 1M occurrences is 16 MB).
+const partitionOccurrences = 1 << 20
+
+// DefaultPartitions returns a partition count suited to a total of kmers
+// k-mer occurrences: 256 at least, and more, as a power of two up to 1024, when
+// the average partition would otherwise exceed about a million occurrences.
+// The largest partitions hold several times the average. The limit keeps the
+// number of simultaneously open files reasonable.
+func DefaultPartitions(kmers int) int {
+	p := 256
+	for p < 1024 && kmers/p > partitionOccurrences {
+		p *= 2
+	}
+	return p
+}
+
+// K returns the k-mer size.
+func (c *SeqKmerCounter) K() int { return c.k }
+
+var unambiguousBase = func() (t [256]bool) {
+	for _, b := range "ACGTUacgtu" {
+		t[b] = true
+	}
+	return
+}()
+
+// AddSequence registers sequence id. The same id must be used for every call
+// concerning the same sequence; ids need not be contiguous. If circular is
+// true the k-1 first bases are appended so that k-mers spanning the junction
+// are counted. AddSequence is safe for concurrent use and must not be called
+// after CountAtLeast.
+func (c *SeqKmerCounter) AddSequence(id uint32, seq []byte, circular bool) {
+	if len(seq) < c.k {
+		return
+	}
+	if circular {
+		ext := make([]byte, 0, len(seq)+c.k-1)
+		ext = append(ext, seq...)
+		seq = append(ext, seq[:c.k-1]...)
+	}
+
+	var scratch []byte
+	start := -1
+	for i, b := range seq {
+		switch {
+		case unambiguousBase[b]:
+			if start < 0 {
+				start = i
+			}
+		case start >= 0:
+			scratch = c.addRun(id, seq[start:i], scratch)
+			start = -1
+		}
+	}
+	if start >= 0 {
+		c.addRun(id, seq[start:], scratch)
+	}
+}
+
+// addRun writes the super-kmers of one ambiguity-free run.
+func (c *SeqKmerCounter) addRun(id uint32, run []byte, scratch []byte) []byte {
+	if len(run) < c.k {
+		return scratch
+	}
+	for sk := range IterSuperKmers(run, c.k, c.m) {
+		scratch = binary.AppendUvarint(scratch[:0], uint64(id))
+		scratch = binary.AppendUvarint(scratch, uint64(len(sk.Sequence)))
+		scratch = appendPacked(scratch, sk.Sequence)
+
+		pw := c.parts[int(sk.Minimizer%uint64(c.p))]
+		pw.mu.Lock()
+		_, err := pw.w.Write(scratch)
+		pw.kmers += len(sk.Sequence) - c.k + 1
+		pw.mu.Unlock()
+		if err != nil {
+			c.setErr(err)
+		}
+	}
+	return scratch
+}
+
+func (c *SeqKmerCounter) setErr(err error) {
+	c.mu.Lock()
+	if c.firstErr == nil {
+		c.firstErr = err
+	}
+	c.mu.Unlock()
+}
+
+// seal flushes and closes the partition files; no more sequences can be added.
+func (c *SeqKmerCounter) seal() error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.sealed {
+		return c.firstErr
+	}
+	c.sealed = true
+	for _, pw := range c.parts {
+		if err := pw.w.Flush(); err != nil && c.firstErr == nil {
+			c.firstErr = err
+		}
+		if err := pw.file.Close(); err != nil && c.firstErr == nil {
+			c.firstErr = err
+		}
+	}
+	return c.firstErr
+}
+
+// Close removes the temporary files. It is safe to call more than once.
+func (c *SeqKmerCounter) Close() error {
+	_ = c.seal()
+	return os.RemoveAll(c.dir)
+}

pkg/obikmer/seqquorum_budget_test.go

@@ -0,0 +1,118 @@
+package obikmer
+
+import (
+	"math/rand"
+	"reflect"
+	"sync"
+	"testing"
+	"time"
+)
+
+// The occurrence budget only bounds the memory: any budget, down to one
+// occurrence (every partition then goes alone), gives the same counts.
+func TestSeqKmerCounterBudgetDoesNotChangeTheCounts(t *testing.T) {
+	rng := rand.New(rand.NewSource(7))
+	seqs := randomSeqs(rng, 40, 20, 500, true)
+	build := func(budget int) []SeqKmerCount {
+		c, err := NewSeqKmerCounter(t.TempDir(), 9, 4, 11)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer c.Close()
+		if budget > 0 {
+			c.SetOccurrenceBudget(budget)
+		}
+		for i, s := range seqs {
+			c.AddSequence(uint32(i), s, i%2 == 0)
+		}
+		total := 0
+		for _, pw := range c.parts {
+			total += pw.kmers
+		}
+		if total == 0 {
+			t.Fatal("no occurrence counted")
+		}
+		got, err := c.CountAtLeast(2)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return got
+	}
+	want := build(0)
+	if len(want) == 0 {
+		t.Fatal("the test needs k-mers shared by several sequences")
+	}
+	for _, budget := range []int{1, 100, 5000} {
+		if got := build(budget); !reflect.DeepEqual(got, want) {
+			t.Errorf("budget %d changes the counts", budget)
+		}
+	}
+}
+
+// The occurrences announced for a partition are those that its file holds.
+func TestSeqKmerCounterPartitionOccurrenceCount(t *testing.T) {
+	rng := rand.New(rand.NewSource(3))
+	seqs := randomSeqs(rng, 25, 10, 300, true)
+	c, err := NewSeqKmerCounter(t.TempDir(), 8, 3, 5)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer c.Close()
+	for i, s := range seqs {
+		c.AddSequence(uint32(i), s, i%3 == 0)
+	}
+	if err := c.seal(); err != nil {
+		t.Fatal(err)
+	}
+	for p, pw := range c.parts {
+		occ, err := c.loadPartition(p, nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(occ) != pw.kmers {
+			t.Errorf("partition %d: %d occurrences loaded, %d counted when written", p, len(occ), pw.kmers)
+		}
+	}
+}
+
+func TestOccurrenceBudget(t *testing.T) {
+	b := newOccurrenceBudget(10)
+	if got := b.acquire(4); got != 4 {
+		t.Fatalf("acquire(4) = %d", got)
+	}
+	// 7 does not fit in the 6 left: it must wait for the release
+	done := make(chan int)
+	go func() { done <- b.acquire(7) }()
+	select {
+	case <-done:
+		t.Fatal("acquire(7) must wait")
+	case <-time.After(50 * time.Millisecond):
+	}
+	b.release(4)
+	if got := <-done; got != 7 {
+		t.Fatalf("acquire(7) = %d", got)
+	}
+	b.release(7)
+
+	// a request above the whole budget takes all of it, once everything is free
+	if got := b.acquire(1000); got != 10 {
+		t.Fatalf("an oversized request must take the whole budget, got %d", got)
+	}
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() { defer wg.Done(); b.release(b.acquire(3)) }()
+	time.Sleep(20 * time.Millisecond)
+	b.release(10)
+	wg.Wait()
+}
+
+func TestDefaultPartitions(t *testing.T) {
+	for _, tc := range []struct{ kmers, want int }{
+		{0, 256}, {5_000_000, 256}, {256 << 20, 256}, {257 << 20, 512}, {526_000_000, 512},
+		{600 << 20, 1024}, {1 << 40, 1024},
+	} {
+		if got := DefaultPartitions(tc.kmers); got != tc.want {
+			t.Errorf("DefaultPartitions(%d) = %d, want %d", tc.kmers, got, tc.want)
+		}
+	}
+}

pkg/obikmer/seqquorum_count.go

@@ -0,0 +1,207 @@
+package obikmer
+
+import (
+	"bufio"
+	"cmp"
+	"encoding/binary"
+	"io"
+	"os"
+	"slices"
+	"sync"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
+)
+
+// kmerOcc is one occurrence of a canonical k-mer in a sequence.
+type kmerOcc struct {
+	kmer uint64
+	seq  uint32
+}
+
+func cmpOcc(a, b kmerOcc) int {
+	if c := cmp.Compare(a.kmer, b.kmer); c != 0 {
+		return c
+	}
+	return cmp.Compare(a.seq, b.seq)
+}
+
+// CountAtLeast returns, sorted by k-mer, every canonical k-mer found in at
+// least minSeq distinct sequences, with its exact sequence count.
+//
+// The result is held in memory: it is meant for quorum-filtered sets (the
+// conserved k-mers), not for the complete k-mer spectrum. The first call
+// seals the counter; AddSequence must not be called afterwards. The count
+// phase uses obidefault.ParallelWorkers() goroutines over the partitions.
+func (c *SeqKmerCounter) CountAtLeast(minSeq int) ([]SeqKmerCount, error) {
+	if err := c.seal(); err != nil {
+		return nil, err
+	}
+	if minSeq < 1 {
+		minSeq = 1
+	}
+
+	workers := min(obidefault.ParallelWorkers(), c.p)
+
+	// the largest partitions first: they are the hardest to fit in the budget
+	// and the last ones to finish, if they come late
+	order := make([]int, c.p)
+	for i := range order {
+		order[i] = i
+	}
+	slices.SortStableFunc(order, func(a, b int) int { return cmp.Compare(c.parts[b].kmers, c.parts[a].kmers) })
+	budget := newOccurrenceBudget(c.budget)
+
+	jobs := make(chan int)
+	results := make([][]SeqKmerCount, c.p)
+
+	var wg sync.WaitGroup
+	var errMu sync.Mutex
+	var firstErr error
+
+	for range workers {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			var occ []kmerOcc // reused between partitions of similar sizes
+			for p := range jobs {
+				n := c.parts[p].kmers
+				held := budget.acquire(n)
+				if cap(occ) < n || cap(occ) > 2*n+1024 {
+					occ = make([]kmerOcc, 0, n)
+				}
+				var err error
+				occ, err = c.loadPartition(p, occ[:0])
+				if err == nil {
+					results[p] = reduceOccurrences(occ, minSeq)
+				}
+				budget.release(held)
+				if err != nil {
+					errMu.Lock()
+					if firstErr == nil {
+						firstErr = err
+					}
+					errMu.Unlock()
+				}
+			}
+		}()
+	}
+	for _, p := range order {
+		jobs <- p
+	}
+	close(jobs)
+	wg.Wait()
+	if firstErr != nil {
+		return nil, firstErr
+	}
+
+	total := 0
+	for _, r := range results {
+		total += len(r)
+	}
+	all := make([]SeqKmerCount, 0, total)
+	for _, r := range results {
+		all = append(all, r...)
+	}
+	slices.SortFunc(all, func(a, b SeqKmerCount) int { return cmp.Compare(a.Kmer, b.Kmer) })
+	return all, nil
+}
+
+// loadPartition appends to occ every (canonical k-mer, sequence id) of the
+// partition file.
+func (c *SeqKmerCounter) loadPartition(p int, occ []kmerOcc) ([]kmerOcc, error) {
+	f, err := os.Open(c.parts[p].path)
+	if err != nil {
+		return occ, err
+	}
+	defer f.Close()
+	r := bufio.NewReaderSize(f, 64*1024)
+
+	var packed, bases []byte
+	for {
+		id, err := binary.ReadUvarint(r)
+		if err == io.EOF {
+			return occ, nil
+		}
+		if err != nil {
+			return occ, err
+		}
+		n, err := binary.ReadUvarint(r)
+		if err != nil {
+			return occ, err
+		}
+
+		nb := (int(n) + 3) / 4
+		packed = slices.Grow(packed[:0], nb)[:nb]
+		if _, err := io.ReadFull(r, packed); err != nil {
+			return occ, err
+		}
+		bases = slices.Grow(bases[:0], int(n))[:n]
+		unpackBases(bases, packed)
+
+		for kmer := range IterCanonicalKmers(bases, c.k) {
+			occ = append(occ, kmerOcc{kmer, uint32(id)})
+		}
+	}
+}
+
+// reduceOccurrences sorts occ and returns the k-mers present in at least
+// minSeq distinct sequences. occ is reordered in place.
+func reduceOccurrences(occ []kmerOcc, minSeq int) []SeqKmerCount {
+	slices.SortFunc(occ, cmpOcc)
+
+	var out []SeqKmerCount
+	for i := 0; i < len(occ); {
+		kmer := occ[i].kmer
+		var seqs uint32
+		repeated := false
+		j := i
+		for ; j < len(occ) && occ[j].kmer == kmer; j++ {
+			if j == i || occ[j].seq != occ[j-1].seq {
+				seqs++
+			} else {
+				repeated = true
+			}
+		}
+		if int(seqs) >= minSeq {
+			out = append(out, SeqKmerCount{Kmer: kmer, Sequences: seqs, Repeated: repeated})
+		}
+		i = j
+	}
+	return out
+}
+
+// occurrenceBudget bounds the k-mer occurrences held in memory at once by the
+// goroutines that load the partitions.
+type occurrenceBudget struct {
+	mu    sync.Mutex
+	freed *sync.Cond
+	total int
+	free  int
+}
+
+func newOccurrenceBudget(total int) *occurrenceBudget {
+	b := &occurrenceBudget{total: total, free: total}
+	b.freed = sync.NewCond(&b.mu)
+	return b
+}
+
+// acquire waits until n occurrences fit in the budget and takes them; a request
+// above the whole budget waits for the budget to be free and takes all of it.
+// It returns what was taken, to give back with release.
+func (b *occurrenceBudget) acquire(n int) int {
+	n = min(n, b.total)
+	b.mu.Lock()
+	for b.free < n {
+		b.freed.Wait()
+	}
+	b.free -= n
+	b.mu.Unlock()
+	return n
+}
+
+func (b *occurrenceBudget) release(n int) {
+	b.mu.Lock()
+	b.free += n
+	b.mu.Unlock()
+	b.freed.Broadcast()
+}

pkg/obikmer/seqquorum_test.go

@@ -0,0 +1,272 @@
+package obikmer
+
+import (
+	"math/rand"
+	"slices"
+	"strings"
+	"sync"
+	"testing"
+)
+
+type bruteInfo struct {
+	seqs     map[uint32]int // sequence id -> occurrences
+	repeated bool
+}
+
+// bruteForce is the oracle: every window of k unambiguous bases, circular
+// junction included, encoded with EncodeCanonicalKmer.
+func bruteForce(seqs [][]byte, circular []bool, k int) map[uint64]*bruteInfo {
+	res := map[uint64]*bruteInfo{}
+	for id, s := range seqs {
+		if len(s) < k {
+			continue
+		}
+		if circular[id] {
+			s = append(slices.Clone(s), s[:k-1]...)
+		}
+	window:
+		for i := 0; i+k <= len(s); i++ {
+			w := s[i : i+k]
+			for _, b := range w {
+				if !unambiguousBase[b] {
+					continue window
+				}
+			}
+			kmer := EncodeCanonicalKmer(w, k)
+			info := res[kmer]
+			if info == nil {
+				info = &bruteInfo{seqs: map[uint32]int{}}
+				res[kmer] = info
+			}
+			info.seqs[uint32(id)]++
+			if info.seqs[uint32(id)] > 1 {
+				info.repeated = true
+			}
+		}
+	}
+	return res
+}
+
+func randomSeqs(rng *rand.Rand, n, minLen, maxLen int, withN bool) [][]byte {
+	alphabet := "acgtACGT"
+	if withN {
+		alphabet = "acgtACGTacgtACGTNnR"
+	}
+	// A shared motif and its reverse complement make k-mers common to several
+	// sequences on both strands.
+	motif := make([]byte, 40)
+	for i := range motif {
+		motif[i] = "ACGT"[rng.Intn(4)]
+	}
+	rc := make([]byte, len(motif))
+	for i, b := range motif {
+		rc[len(motif)-1-i] = map[byte]byte{'A': 'T', 'C': 'G', 'G': 'C', 'T': 'A'}[b]
+	}
+
+	out := make([][]byte, n)
+	for i := range out {
+		l := minLen + rng.Intn(maxLen-minLen+1)
+		s := make([]byte, l)
+		for j := range s {
+			s[j] = alphabet[rng.Intn(len(alphabet))]
+		}
+		mode := rng.Intn(4)
+		if l < 60 { // too short to host the motifs
+			mode = 3
+		}
+		switch mode {
+		case 0:
+			copy(s[l/4:], motif)
+		case 1:
+			copy(s[l/4:], rc)
+		case 2: // internal repeat, both strands
+			copy(s[2:], motif[:20])
+			copy(s[l-25:], rc[:20])
+		}
+		out[i] = s
+	}
+	return out
+}
+
+func TestSeqKmerCounterAgainstBruteForce(t *testing.T) {
+	cases := []struct {
+		name    string
+		k, m, p int
+		withN   bool
+		circ    bool
+	}{
+		{"k7-linear", 7, 3, 8, false, false},
+		{"k7-circular-N", 7, 3, 5, true, true},
+		{"k18-linear-N", 18, 8, 16, true, false},
+		{"k18-circular", 18, 8, 7, false, true},
+		{"k31-p1", 31, 10, 1, false, true},
+		{"k4-small-m", 4, 1, 3, true, false},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			rng := rand.New(rand.NewSource(int64(tc.k*100 + tc.p)))
+			seqs := randomSeqs(rng, 30, 5, 400, tc.withN)
+			circ := make([]bool, len(seqs))
+			for i := range circ {
+				circ[i] = tc.circ && i%3 != 0
+			}
+
+			c, err := NewSeqKmerCounter(t.TempDir(), tc.k, tc.m, tc.p)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer c.Close()
+
+			// Concurrent insertion, shuffled order, ids not contiguous.
+			var wg sync.WaitGroup
+			for i := range seqs {
+				wg.Add(1)
+				go func() {
+					defer wg.Done()
+					c.AddSequence(uint32(i), seqs[i], circ[i])
+				}()
+			}
+			wg.Wait()
+
+			want := bruteForce(seqs, circ, tc.k)
+			for _, q := range []int{1, 2, 3, 10} {
+				got, err := c.CountAtLeast(q)
+				if err != nil {
+					t.Fatal(err)
+				}
+				if !slices.IsSortedFunc(got, func(a, b SeqKmerCount) int {
+					switch {
+					case a.Kmer < b.Kmer:
+						return -1
+					case a.Kmer > b.Kmer:
+						return 1
+					}
+					return 0
+				}) {
+					t.Fatal("result is not sorted by k-mer")
+				}
+
+				n := 0
+				for kmer, info := range want {
+					if len(info.seqs) >= q {
+						n++
+						_ = kmer
+					}
+				}
+				if len(got) != n {
+					t.Fatalf("q=%d: %d k-mers, want %d", q, len(got), n)
+				}
+				for _, g := range got {
+					info := want[g.Kmer]
+					if info == nil {
+						t.Fatalf("q=%d: unexpected k-mer %d", q, g.Kmer)
+					}
+					if int(g.Sequences) != len(info.seqs) || g.Repeated != info.repeated {
+						t.Fatalf("q=%d kmer %d: got (%d,%v) want (%d,%v)",
+							q, g.Kmer, g.Sequences, g.Repeated, len(info.seqs), info.repeated)
+					}
+				}
+			}
+		})
+	}
+}
+
+// A k-mer appearing on both strands of one sequence counts once but is Repeated.
+func TestSeqKmerCounterBothStrandsOneSequence(t *testing.T) {
+	const k = 8
+	fwd := "ACGGTCAT"
+	rc := "ATGACCGT"
+	seq := []byte("TTTT" + fwd + "GGGGGGGG" + rc + "CCCC")
+
+	c, err := NewSeqKmerCounter(t.TempDir(), k, 3, 4)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer c.Close()
+	c.AddSequence(7, seq, false)
+
+	got, err := c.CountAtLeast(1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	target := EncodeCanonicalKmer([]byte(fwd), k)
+	found := false
+	for _, g := range got {
+		if g.Kmer == target {
+			found = true
+			if g.Sequences != 1 || !g.Repeated {
+				t.Errorf("got %+v, want 1 sequence and Repeated", g)
+			}
+		}
+	}
+	if !found {
+		t.Fatal("k-mer not found")
+	}
+}
+
+// Ambiguous bases must never be part of a k-mer, and the circular junction
+// must produce the wrap-around k-mers.
+func TestSeqKmerCounterAmbiguityAndCircular(t *testing.T) {
+	const k = 5
+	c, err := NewSeqKmerCounter(t.TempDir(), k, 2, 3)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer c.Close()
+
+	c.AddSequence(0, []byte("ACGTNACGT"), false) // runs ACGT (<k) and ACGT (<k): nothing
+	c.AddSequence(1, []byte("AACCGGTT"), true)   // 8 circular windows
+
+	got, err := c.CountAtLeast(1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) == 0 {
+		t.Fatal("expected k-mers from the circular sequence")
+	}
+	var circ []string
+	for i := 0; i < 8; i++ {
+		var sb strings.Builder
+		for j := 0; j < k; j++ {
+			sb.WriteByte("AACCGGTT"[(i+j)%8])
+		}
+		circ = append(circ, sb.String())
+	}
+	want := map[uint64]bool{}
+	for _, w := range circ {
+		want[EncodeCanonicalKmer([]byte(w), k)] = true
+	}
+	if len(got) != len(want) {
+		t.Fatalf("got %d k-mers, want %d", len(got), len(want))
+	}
+	for _, g := range got {
+		if !want[g.Kmer] || g.Sequences != 1 {
+			t.Errorf("unexpected %+v", g)
+		}
+	}
+}
+
+func TestSeqKmerCounterParameters(t *testing.T) {
+	if _, err := NewSeqKmerCounter(t.TempDir(), 1, 1, 4); err == nil {
+		t.Error("k=1 must fail")
+	}
+	if _, err := NewSeqKmerCounter(t.TempDir(), 32, 5, 4); err == nil {
+		t.Error("k=32 must fail")
+	}
+	if _, err := NewSeqKmerCounter(t.TempDir(), 10, 10, 4); err == nil {
+		t.Error("m>=k must fail")
+	}
+	if _, err := NewSeqKmerCounter(t.TempDir(), 10, 4, 0); err == nil {
+		t.Error("zero partitions must fail")
+	}
+	c, err := NewSeqKmerCounter(t.TempDir(), 10, 0, 4) // default minimizer
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := c.Close(); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.Close(); err != nil {
+		t.Fatalf("second Close must be harmless: %v", err)
+	}
+}

pkg/obikmer/skm_reader.go

@@ -10,6 +10,16 @@ import (
 // decode2bit maps 2-bit codes back to nucleotide bytes.
 var decode2bit = [4]byte{'a', 'c', 'g', 't'}
 
+// unpackBases decodes the 2-bit packed bases in packed into dst (one lowercase
+// nucleotide per byte, len(dst) bases) and returns dst.
+func unpackBases(dst, packed []byte) []byte {
+	for i := range dst {
+		code := (packed[i/4] >> uint(6-(i%4)*2)) & 0x03
+		dst[i] = decode2bit[code]
+	}
+	return dst
+}
+
 // SkmReader reads super-kmers from a binary .skm file.
 type SkmReader struct {
 	r    *bufio.Reader
@@ -46,13 +56,7 @@ func (sr *SkmReader) Next() (SuperKmer, bool) {
 	}
 
 	// Decode to nucleotide bytes
-	seq := make([]byte, seqLen)
+	seq := unpackBases(make([]byte, seqLen), packed)
-	for i := 0; i < seqLen; i++ {
-		byteIdx := i / 4
-		bitPos := uint(6 - (i%4)*2)
-		code := (packed[byteIdx] >> bitPos) & 0x03
-		seq[i] = decode2bit[code]
-	}
 
 	return SuperKmer{
 		Sequence: seq,

pkg/obikmer/skm_writer.go

@@ -16,8 +16,9 @@ import (
 // Nucleotide encoding: A=00, C=01, G=10, T=11.
 // The last byte is zero-padded on the low bits if len%4 != 0.
 type SkmWriter struct {
-	w    *bufio.Writer
+	w       *bufio.Writer
-	file *os.File
+	file    *os.File
+	scratch []byte
 }
 
 // NewSkmWriter creates a new SkmWriter writing to the given file path.
@@ -32,36 +33,36 @@ func NewSkmWriter(path string) (*SkmWriter, error) {
 	}, nil
 }
 
+// appendPacked appends seq to dst packed 2 bits per base (A=00, C=01, G=10,
+// T=11), four bases per byte, the last byte zero-padded on the low bits.
+func appendPacked(dst, seq []byte) []byte {
+	for i := 0; i < len(seq); i += 4 {
+		var packed byte
+		for j := 0; j < 4; j++ {
+			packed <<= 2
+			if i+j < len(seq) {
+				packed |= __single_base_code__[seq[i+j]&31]
+			}
+		}
+		dst = append(dst, packed)
+	}
+	return dst
+}
+
 // Write encodes a SuperKmer to the .skm file.
 // The sequence bytes are packed 2 bits per base.
 func (sw *SkmWriter) Write(sk SuperKmer) error {
-	seq := sk.Sequence
-	seqLen := uint16(len(seq))
-
 	// Write length
 	var lenbuf [2]byte
-	binary.LittleEndian.PutUint16(lenbuf[:], seqLen)
+	binary.LittleEndian.PutUint16(lenbuf[:], uint16(len(sk.Sequence)))
 	if _, err := sw.w.Write(lenbuf[:]); err != nil {
 		return err
 	}
 
 	// Encode and write packed sequence (2 bits/base)
-	nBytes := (int(seqLen) + 3) / 4
+	sw.scratch = appendPacked(sw.scratch[:0], sk.Sequence)
-	for i := 0; i < nBytes; i++ {
+	_, err := sw.w.Write(sw.scratch)
-		var packed byte
+	return err
-		for j := 0; j < 4; j++ {
-			pos := i*4 + j
-			packed <<= 2
-			if pos < int(seqLen) {
-				packed |= __single_base_code__[seq[pos]&31]
-			}
-		}
-		if err := sw.w.WriteByte(packed); err != nil {
-			return err
-		}
-	}
-
-	return nil
 }
 
 // Close flushes buffered data and closes the underlying file.

pkg/obioptions/version.go

@@ -3,7 +3,7 @@ package obioptions
 // Version is automatically updated by the Makefile from version.txt
 // The patch number (third digit) is incremented on each push to the repository
 
-var _Version = "Release 4.5.0"
+var _Version = "Release 4.5.2"
 
 // Version returns the version of the obitools package.
 //

pkg/obiprimer/amplicons.go

@@ -0,0 +1,293 @@
+package obiprimer
+
+import (
+	"math"
+	"sync"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obithermo"
+)
+
+// refHit is an amplicon of a retained pair on the reference sequence.
+type refHit struct {
+	up, down seqMatch
+	swapped  bool
+}
+
+// ampHash hashes the amplicon between the two primers read from the side of
+// the first primer of the pair (reverse complemented when the upstream match is
+// the second primer), so that identical amplicons give the same value
+// whatever their strand. Non ACGT bases hash as a fifth symbol.
+func ampHash(raw []byte, from, to int, reverse bool) uint64 {
+	const prime = 1099511628211
+	h := uint64(14695981039346656037) ^ uint64(to-from)
+	h *= prime
+	if !reverse {
+		for _, b := range raw[from:to] {
+			h = (h ^ uint64(baseCode[b])) * prime
+		}
+		return h
+	}
+	for i := to - 1; i >= from; i-- {
+		c := baseCode[raw[i]]
+		if c < 4 {
+			c = 3 - c
+		}
+		h = (h ^ uint64(c)) * prime
+	}
+	return h
+}
+
+// eachAmplicon runs forEachOccurrence over the sequences of the corpus (the
+// examples only, or all of them) in parallel, and calls visit for every
+// amplicon of a pair of index. Every worker gets its own visit function, from
+// newVisit, and done is called once the worker has no more sequence.
+//
+// Without a plan the sequences are handed to the workers as they get free, and
+// the assignment is recorded in record. With one, every worker scans the
+// sequences that the plan gave it.
+func (ps *primerSet) eachAmplicon(c *Corpus, p PairParams, index map[pairKey]int32, examplesOnly bool, plan *ampPlan, record []uint16,
+	newVisit func(worker int) (visit func(s int, raw []byte, o occurrence, pair int32), done func())) {
+
+	workers := obidefault.ParallelWorkers()
+
+	// only the matches of the primers of the pairs of index can make an amplicon
+	// of these pairs, and dropping the others spares most of the comparisons
+	wanted := make([]bool, len(ps.primers))
+	for k := range index {
+		wanted[k.p1], wanted[k.p2] = true, true
+	}
+	type scratch struct {
+		ms    []seqMatch
+		bases []byte
+	}
+	scan := func(sc *scratch, visit func(int, []byte, occurrence, int32), s int) {
+		if examplesOnly && c.Roles[s] != Example {
+			return
+		}
+		ms := sc.ms[:0]
+		for _, m := range ps.perSeq[s] {
+			if wanted[m.primer] {
+				ms = append(ms, m)
+			}
+		}
+		sc.ms = ms
+		if len(ms) < 2 {
+			return // an amplicon needs two primer sites
+		}
+		var raw []byte
+		raw, sc.bases = c.bases(s, sc.bases)
+		ps.forEachOccurrenceIn(raw, ms, p, func(o occurrence) {
+			if pi, ok := index[o.key]; ok {
+				visit(s, raw, o, pi)
+			}
+		})
+	}
+
+	var wg sync.WaitGroup
+	if plan != nil {
+		mine := make([][]int, workers)
+		for s, w := range plan.owner {
+			mine[w] = append(mine[w], s)
+		}
+		for w := range workers {
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				visit, done := newVisit(w)
+				var sc scratch
+				for _, s := range mine[w] {
+					scan(&sc, visit, s)
+				}
+				if done != nil {
+					done()
+				}
+			}()
+		}
+		wg.Wait()
+		return
+	}
+
+	jobs := make(chan int)
+	for w := range workers {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			visit, done := newVisit(w)
+			var sc scratch
+			for s := range jobs {
+				record[s] = uint16(w)
+				scan(&sc, visit, s)
+			}
+			if done != nil {
+				done()
+			}
+		}()
+	}
+	for s := range c.Roles {
+		jobs <- s
+	}
+	close(jobs)
+	wg.Wait()
+}
+
+// planAmplicons counts the amplicons of every pair of index in the sequences
+// of the corpus (all of them, or the examples only), by worker.
+func (ps *primerSet) planAmplicons(c *Corpus, p PairParams, index map[pairKey]int32, examplesOnly bool) *ampPlan {
+	workers := obidefault.ParallelWorkers()
+	plan := &ampPlan{owner: make([]uint16, c.Len()), count: make([][]uint32, workers)}
+	for w := range plan.count {
+		plan.count[w] = make([]uint32, len(index))
+	}
+	ps.eachAmplicon(c, p, index, examplesOnly, nil, plan.owner, func(w int) (func(int, []byte, occurrence, int32), func()) {
+		row := plan.count[w]
+		return func(_ int, _ []byte, _ occurrence, pi int32) { row[pi]++ }, nil
+	})
+	return plan
+}
+
+// ampSites is what the gathering of the amplicons of the targets learns besides
+// the records.
+type ampSites struct {
+	minTm1 []float32 // by pair: lowest Tm of the first primer over the example sites
+	minTm2 []float32
+	refs   map[int32]refHit // by pair: the amplicon on the reference sequence
+}
+
+// collectOptions selects what collectAmplicons gathers besides the records.
+type collectOptions struct {
+	examplesOnly bool             // skip the counterexamples and the other sequences
+	model        *obithermo.Model // melting temperatures of the example sites (nil: none)
+	reference    bool             // record the amplicons on the reference sequence
+}
+
+// collectAmplicons writes into store, in parallel, one record per amplicon of
+// the pairs of index (whose values are the pair numbers in the store) and
+// returns the lowest melting temperatures over the example sites. The store
+// was sized from the plan, which says which worker scans which sequence; the
+// batch only holds the pairs of index, the plan more.
+func (ps *primerSet) collectAmplicons(c *Corpus, taxa *Taxa, p PairParams, index map[pairKey]int32,
+	plan *ampPlan, store *ampStore, opt collectOptions) *ampSites {
+
+	model := opt.model
+	lp := p.PrimerLength
+	npairs := len(index)
+
+	res := &ampSites{
+		minTm1: make([]float32, npairs),
+		minTm2: make([]float32, npairs),
+		refs:   map[int32]refHit{}, // written by the single worker that reads the reference
+	}
+	inf := float32(math.Inf(1))
+	for i := range npairs {
+		res.minTm1[i], res.minTm2[i] = inf, inf
+	}
+	var tmMu sync.Mutex
+
+	// oriented[primer][0] is the reverse complement of the word, [1] the word itself.
+	oriented := make([][2][]byte, len(ps.primers))
+	for i, pr := range ps.primers {
+		oriented[i][0] = []byte(orientedWord(pr.Word, lp, false))
+		oriented[i][1] = []byte(orientedWord(pr.Word, lp, true))
+	}
+
+	ps.eachAmplicon(c, p, index, opt.examplesOnly, plan, nil, func(w int) (func(int, []byte, occurrence, int32), func()) {
+		site1, site2 := make([]byte, lp), make([]byte, lp)
+		lowTm1, lowTm2 := map[int32]float32{}, map[int32]float32{}
+
+		visit := func(s int, raw []byte, o occurrence, pi int32) {
+			n := len(raw)
+			role := c.Roles[s]
+			if opt.reference && p.HasReference && s == p.Reference {
+				if old, seen := res.refs[pi]; !seen || o.up.pos() < old.up.pos() {
+					res.refs[pi] = refHit{o.up, o.down, o.swapped}
+				}
+			}
+			store.add(w, pi, ampHash(raw, int(o.up.pos())+lp, int(o.down.pos()), o.swapped), packTaxon(taxa.Of[s], role))
+
+			if role != Example || model == nil {
+				return
+			}
+			fp := oriented[o.up.primer][b2i(!o.up.reverse())]
+			rp := oriented[o.down.primer][b2i(o.down.reverse())]
+			tmF, tmR := inf, inf
+			if windowAsIs(site1, raw, int(o.up.pos()), n, p.Circular) {
+				if t, err := model.Tm(fp, site1); err == nil && !math.IsNaN(t) {
+					tmF = float32(t)
+				}
+			}
+			if windowRevComp(site2, raw, int(o.down.pos()), n, p.Circular) {
+				if t, err := model.Tm(rp, site2); err == nil && !math.IsNaN(t) {
+					tmR = float32(t)
+				}
+			}
+			if o.swapped {
+				tmF, tmR = tmR, tmF // slot 1 is the downstream primer
+			}
+			lowTm1[pi] = min(lowOr(lowTm1, pi, inf), tmF)
+			lowTm2[pi] = min(lowOr(lowTm2, pi, inf), tmR)
+		}
+		done := func() {
+			tmMu.Lock()
+			for pi, t := range lowTm1 {
+				res.minTm1[pi] = min(res.minTm1[pi], t)
+			}
+			for pi, t := range lowTm2 {
+				res.minTm2[pi] = min(res.minTm2[pi], t)
+			}
+			tmMu.Unlock()
+		}
+		return visit, done
+	})
+	return res
+}
+
+func lowOr(m map[int32]float32, k int32, def float32) float32 {
+	if v, ok := m[k]; ok {
+		return v
+	}
+	return def
+}
+
+func b2i(b bool) int {
+	if b {
+		return 1
+	}
+	return 0
+}
+
+const upper = "ACGT"
+
+// windowAsIs copies the len(dst) bases at pos as upper-case ACGT; it reports
+// false if one of them is not A, C, G or T.
+func windowAsIs(dst, raw []byte, pos, n int, circular bool) bool {
+	for t := range dst {
+		i := pos + t
+		if circular {
+			i %= n
+		}
+		c := baseCode[raw[i]]
+		if c > 3 {
+			return false
+		}
+		dst[t] = upper[c]
+	}
+	return true
+}
+
+// windowRevComp writes the reverse complement of the len(dst) bases at pos.
+func windowRevComp(dst, raw []byte, pos, n int, circular bool) bool {
+	l := len(dst)
+	for t := range dst {
+		i := pos + l - 1 - t
+		if circular {
+			i %= n
+		}
+		c := baseCode[raw[i]]
+		if c > 3 {
+			return false
+		}
+		dst[t] = upper[3-c]
+	}
+	return true
+}

pkg/obiprimer/ampstore.go

@@ -0,0 +1,132 @@
+package obiprimer
+
+import "fmt"
+
+// ampRecord is the logical content of the record of one amplicon: its barcode
+// (see ampHash), the taxon of its sequence at the chosen rank (-1 when it has
+// none) and the role of the sequence. It is stored packed in an ampStore.
+type ampRecord struct {
+	hash  uint64
+	taxon int32
+	role  Role
+}
+
+const (
+	roleBits = 2
+	roleMask = 1<<roleBits - 1
+
+	// maxTaxa is the number of taxa a record can carry: the taxon, shifted by
+	// one so that "none" is 0, shares a 32 bits word with the role.
+	maxTaxa = 1<<(32-roleBits) - 2
+)
+
+func packTaxon(taxon int32, role Role) uint32 { return uint32(taxon+1)<<roleBits | uint32(role) }
+
+func unpackTaxon(t uint32) (int32, Role) { return int32(t>>roleBits) - 1, Role(t & roleMask) }
+
+// ampSlice is the amplicons of one pair: 12 bytes per amplicon, kept as two
+// parallel arrays (a struct holding a uint64 and a uint32 would take 16).
+type ampSlice struct {
+	hash []uint64
+	tax  []uint32 // packTaxon
+}
+
+// ampPlan says how many amplicons each worker finds for each pair, and which
+// worker scans which sequence: the write pass replays the counting pass with
+// the very same assignment, so that every worker knows where to write its
+// records without sharing any counter with the others.
+type ampPlan struct {
+	owner []uint16   // owner[s]: the worker that scans sequence s
+	count [][]uint32 // count[w][pair]: amplicons of the pair in the sequences of worker w
+}
+
+// total returns the number of amplicons of every pair.
+func (pl *ampPlan) total() []int {
+	t := make([]int, len(pl.count[0]))
+	for _, row := range pl.count {
+		for b, n := range row {
+			t[b] += int(n)
+		}
+	}
+	return t
+}
+
+// ampStore holds the amplicons of a batch of pairs, grouped by pair: for each
+// pair, two slices of the exact size, since their number is known before they
+// are written (see planAmplicons). Nothing grows by doubling, nothing is copied
+// nor sorted afterwards. The records of a pair are those of worker 0, then those
+// of worker 1, and so on.
+//
+// The pairs have slices of their own, rather than slices of two huge arrays: a
+// few kB each find room in the free memory, where blocks of tens of MB make the
+// process map new memory (and, on some systems, count the freed one still).
+type ampStore struct {
+	plan   *ampPlan
+	lo     int // first pair of the batch in the plan
+	pairs  []ampSlice
+	cursor [][]int // cursor[w][b]: next free slot of worker w for pair b
+}
+
+// newAmpStore allocates the store of the pairs lo..hi-1 of the plan.
+func newAmpStore(plan *ampPlan, lo, hi int) *ampStore {
+	n := hi - lo
+	s := &ampStore{plan: plan, lo: lo, pairs: make([]ampSlice, n), cursor: make([][]int, len(plan.count))}
+	for w := range s.cursor {
+		s.cursor[w] = make([]int, n)
+	}
+	for b := range n {
+		pos := 0
+		for w, row := range plan.count {
+			s.cursor[w][b] = pos
+			pos += int(row[lo+b])
+		}
+		if pos > 0 {
+			s.pairs[b] = ampSlice{make([]uint64, pos), make([]uint32, pos)}
+		}
+	}
+	return s
+}
+
+// add writes one record of pair b (numbered in the batch), found by worker w.
+func (s *ampStore) add(w int, b int32, hash uint64, taxon uint32) {
+	i := s.cursor[w][b]
+	s.cursor[w][b] = i + 1
+	s.pairs[b].hash[i], s.pairs[b].tax[i] = hash, taxon
+}
+
+// check panics if a worker did not write exactly the records counted for it:
+// the counting and write passes must enumerate the same amplicons.
+func (s *ampStore) check() {
+	for b := range s.pairs {
+		pos := 0
+		for w, row := range s.plan.count {
+			pos += int(row[s.lo+b])
+			if s.cursor[w][b] != pos {
+				panic(fmt.Sprintf("obiprimer: pair %d: worker %d wrote %d amplicons, %d were counted",
+					s.lo+b, w, s.cursor[w][b]-(pos-int(row[s.lo+b])), row[s.lo+b]))
+			}
+		}
+	}
+}
+
+// pair returns the amplicons of pair b (numbered in the batch).
+func (s *ampStore) pair(b int) ampSlice { return s.pairs[b] }
+
+// ampBatches cuts the pairs 0..len(count)-1 in consecutive ranges [lo,hi)
+// whose amplicons together do not exceed budget records (a pair alone above the
+// budget makes a batch of its own). A non positive budget gives one batch.
+func ampBatches(count []int, budget int) [][2]int {
+	var out [][2]int
+	lo, sum := 0, 0
+	for i, n := range count {
+		if budget > 0 && i > lo && sum+n > budget {
+			out = append(out, [2]int{lo, i})
+			lo, sum = i, 0
+		}
+		sum += n
+	}
+	if lo < len(count) {
+		out = append(out, [2]int{lo, len(count)})
+	}
+	return out
+}

pkg/obiprimer/corpus.go

@@ -0,0 +1,113 @@
+package obiprimer
+
+import (
+	"fmt"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+// maxSeqLen bounds the length of a sequence: the position of a primer site and
+// its strand share 32 bits (see seqMatch).
+const maxSeqLen = 1 << 31
+
+// Counts gives the number of sequences by role.
+type Counts struct {
+	Examples        int
+	CounterExamples int
+	Others          int
+}
+
+// Corpus is the set of sequences of the primer search, in input order, each
+// with its role: the examples (the targets), the counterexamples, and all the
+// other sequences, which are loaded as well since the amplicons they give
+// decide whether a target is identified.
+//
+// The primer search needs several passes over the sequences (words are
+// searched approximately in every one of them), so the corpus lives in memory.
+type Corpus struct {
+	Seqs  []*obiseq.BioSequence // nil entries once the corpus is packed
+	Roles []Role                // Roles[i] is the role of Seqs[i]
+
+	packed []packedSeq // see Pack
+	ids    []string
+}
+
+// Counts returns the number of sequences by role.
+func (c *Corpus) Counts() Counts {
+	var n Counts
+	for _, r := range c.Roles {
+		switch r {
+		case Example:
+			n.Examples++
+		case CounterExample:
+			n.CounterExamples++
+		default:
+			n.Others++
+		}
+	}
+	return n
+}
+
+// IndexOfExample returns the index of the example sequence with the given
+// identifier. It is an error if there is none, or if the sequence is a
+// counterexample.
+func (c *Corpus) IndexOfExample(id string) (int, error) {
+	for i := range c.Roles {
+		if c.ID(i) == id {
+			if c.Roles[i] != Example {
+				return -1, fmt.Errorf("obiprimer: reference sequence %q is a counterexample, it must be an example", id)
+			}
+			return i, nil
+		}
+	}
+	return -1, fmt.Errorf("obiprimer: reference sequence %q is not among the selected example sequences", id)
+}
+
+// Len returns the number of sequences.
+func (c *Corpus) Len() int { return len(c.Roles) }
+
+// SeqLen returns the length of sequence i.
+func (c *Corpus) SeqLen(i int) int {
+	if c.Seqs[i] != nil {
+		return c.Seqs[i].Len()
+	}
+	return c.packed[i].n
+}
+
+// ID returns the identifier of sequence i.
+func (c *Corpus) ID(i int) string {
+	if c.Seqs[i] != nil {
+		return c.Seqs[i].Id()
+	}
+	return c.ids[i]
+}
+
+// bases returns the bases of sequence i, lower case. A packed sequence is
+// unpacked into buf, grown if need be, which the caller keeps for the next call
+// (the second result); an unpacked one is returned as it is. The result must
+// not be modified, and is valid until the next call with the same buffer.
+func (c *Corpus) bases(i int, buf []byte) (raw, newBuf []byte) {
+	if c.Seqs[i] != nil {
+		return c.Seqs[i].Sequence(), buf
+	}
+	buf = c.packed[i].unpack(buf)
+	return buf, buf
+}
+
+// Pack replaces the sequences by their packed form, on 2 bits per base, and
+// frees the sequence objects (their annotations, their slack): the primer search
+// only needs the bases, the identifiers, and the taxa, which must have been
+// assigned before (AssignTaxa). A packed corpus holds about a quarter of the
+// memory of the bases themselves.
+func (c *Corpus) Pack() {
+	if c.packed != nil {
+		return
+	}
+	c.packed = make([]packedSeq, len(c.Seqs))
+	c.ids = make([]string, len(c.Seqs))
+	parallelFor(len(c.Seqs), func(_, i int) {
+		c.packed[i] = packBases(c.Seqs[i].Sequence())
+		c.ids[i] = c.Seqs[i].Id()
+		c.Seqs[i] = nil
+	})
+}

pkg/obiprimer/corpus_load.go

@@ -0,0 +1,137 @@
+package obiprimer
+
+import (
+	"cmp"
+	"fmt"
+	"slices"
+	"sync"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax"
+)
+
+// corpusEntry is a sequence read by one of the loading goroutines: either the
+// sequence itself, or, when it is packed as it is read, what is kept of it.
+type corpusEntry struct {
+	batch, index int
+	role         Role
+
+	seq *obiseq.BioSequence // unpacked loading
+
+	packed packedSeq // packed loading
+	id     string
+	taxon  *obitax.Taxon // at the chosen rank, nil when there is none
+}
+
+// loadEntries reads every sequence of seqs, once, on all the cores, and returns
+// the entries in the order of the input, whatever the number of reader
+// goroutines. keep fills the entry of a sequence (besides its position and its
+// role); it runs in the reading goroutines.
+func loadEntries(seqs obiiter.IBioSequence, sel Selection, keep func(*obiseq.BioSequence, *corpusEntry)) []corpusEntry {
+	var mu sync.Mutex
+	var entries []corpusEntry
+
+	consumer := func(it obiiter.IBioSequence, wg *sync.WaitGroup) {
+		defer wg.Done()
+		var local []corpusEntry
+		for it.Next() {
+			batch := it.Get()
+			for i, seq := range batch.Slice() {
+				e := corpusEntry{batch: batch.Order(), index: i, role: sel.Role(seq)}
+				keep(seq, &e)
+				local = append(local, e)
+			}
+		}
+		mu.Lock()
+		entries = append(entries, local...)
+		mu.Unlock()
+	}
+
+	var wg sync.WaitGroup
+	for i := 1; i < obidefault.ParallelWorkers(); i++ {
+		wg.Add(1)
+		go consumer(seqs.Split(), &wg)
+	}
+	wg.Add(1)
+	go consumer(seqs, &wg)
+	wg.Wait()
+
+	slices.SortFunc(entries, func(a, b corpusEntry) int {
+		if c := cmp.Compare(a.batch, b.batch); c != 0 {
+			return c
+		}
+		return cmp.Compare(a.index, b.index)
+	})
+	return entries
+}
+
+func checkSeqLen(id string, n int) error {
+	if n >= maxSeqLen {
+		return fmt.Errorf("obiprimer: sequence %q is too long (%d bases, the limit is %d)", id, n, maxSeqLen-1)
+	}
+	return nil
+}
+
+// LoadCorpus reads every sequence of seqs, once, and keeps all of them with the
+// role given by the selection. The order of the input is preserved whatever
+// the number of reader goroutines.
+func LoadCorpus(seqs obiiter.IBioSequence, sel Selection) (*Corpus, error) {
+	entries := loadEntries(seqs, sel, func(seq *obiseq.BioSequence, e *corpusEntry) { e.seq = seq })
+
+	c := &Corpus{
+		Seqs:  make([]*obiseq.BioSequence, len(entries)),
+		Roles: make([]Role, len(entries)),
+	}
+	for i, e := range entries {
+		if err := checkSeqLen(e.seq.Id(), e.seq.Len()); err != nil {
+			return nil, err
+		}
+		c.Seqs[i], c.Roles[i] = e.seq, e.role
+	}
+	if c.Counts().Examples == 0 {
+		return nil, fmt.Errorf("obiprimer: no example sequence selected")
+	}
+	return c, nil
+}
+
+// LoadPackedCorpus is LoadCorpus followed by Pack and AssignTaxa, without ever
+// holding the sequences themselves: each one is packed, and its taxon at rank
+// looked up, as it is read, and then dropped. The corpus of a large database
+// would otherwise first take, in memory, the sequences with all their
+// annotations, four times the size of their bases and more.
+func LoadPackedCorpus(seqs obiiter.IBioSequence, sel Selection, taxonomy *obitax.Taxonomy, rank string) (*Corpus, *Taxa, error) {
+	if err := checkRank(taxonomy, rank); err != nil {
+		return nil, nil, err
+	}
+	entries := loadEntries(seqs, sel, func(seq *obiseq.BioSequence, e *corpusEntry) {
+		e.packed = packBases(seq.Sequence())
+		e.id = seq.Id()
+		e.taxon = seq.Taxon(taxonomy).TaxonAtRank(rank)
+	})
+
+	c := &Corpus{
+		Seqs:   make([]*obiseq.BioSequence, len(entries)),
+		Roles:  make([]Role, len(entries)),
+		packed: make([]packedSeq, len(entries)),
+		ids:    make([]string, len(entries)),
+	}
+	taxons := make([]*obitax.Taxon, len(entries))
+	for i := range entries {
+		e := &entries[i]
+		if err := checkSeqLen(e.id, e.packed.n); err != nil {
+			return nil, nil, err
+		}
+		c.Roles[i], c.packed[i], c.ids[i], taxons[i] = e.role, e.packed, e.id, e.taxon
+		*e = corpusEntry{} // frees the entry as it goes
+	}
+	if c.Counts().Examples == 0 {
+		return nil, nil, fmt.Errorf("obiprimer: no example sequence selected")
+	}
+	taxa, err := buildTaxa(rank, taxons, c.Roles)
+	if err != nil {
+		return nil, nil, err
+	}
+	return c, taxa, nil
+}

pkg/obiprimer/match.go

@@ -0,0 +1,180 @@
+package obiprimer
+
+import (
+	"cmp"
+	"fmt"
+	"slices"
+	"sync"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obikmer"
+)
+
+// Match is one approximate occurrence of a word in a corpus sequence.
+//
+// Pos is the 0-based start of the window on the sequence as given (forward
+// coordinates). When Reverse is false the window reads like the word; when it
+// is true the window reads like the reverse complement of the word, i.e. the
+// word occurs on the other strand. Errors is the number of substitutions.
+type Match struct {
+	Seq     uint32 // index in the Corpus
+	Pos     uint32
+	Errors  uint8
+	Reverse bool
+}
+
+// WordMatches gathers every occurrence of one canonical word, sorted by
+// (Seq, Pos, Reverse).
+type WordMatches struct {
+	Word    uint64
+	Matches []Match
+}
+
+// MatchParams parameterizes the approximate search.
+//
+// Rules, those of ecoPrimers: only substitutions are allowed; a base other
+// than A, C, G, T (or U, read as T) in the sequence never matches, it counts as
+// one error; every window with at most MaxErrors errors is reported, windows
+// overlapping each other included; on circular sequences windows may span the
+// junction.
+type MatchParams struct {
+	K         int
+	MaxErrors int
+	Circular  bool
+}
+
+func (p MatchParams) validate() error {
+	if p.K < 2 || p.K > 31 {
+		return fmt.Errorf("obiprimer: word size must be between 2 and 31, got %d", p.K)
+	}
+	if p.MaxErrors < 0 || p.MaxErrors >= p.K {
+		return fmt.Errorf("obiprimer: error count must be in [0,%d[, got %d", p.K, p.MaxErrors)
+	}
+	return nil
+}
+
+// hit is a Match before its sequence is known: the sequence it belongs to is
+// the one being scanned.
+type hit struct {
+	word    uint32
+	pos     uint32
+	errors  uint8
+	reverse bool
+}
+
+// scanFunc finds every occurrence of every word, on both strands, in one
+// sequence (its bases) and reports them through emit.
+type scanFunc func(raw []byte, emit func(word, pos, errors int, reverse bool))
+
+// scannerMaker returns an independent scanFunc; each worker gets its own so
+// that scanners can keep private buffers.
+type scannerMaker func() scanFunc
+
+// wordStrings returns, for each canonical word, the word and its reverse
+// complement as lowercase strings.
+func wordStrings(words []uint64, k int) (fwd, rev []string) {
+	fwd = make([]string, len(words))
+	rev = make([]string, len(words))
+	for i, w := range words {
+		fwd[i] = string(obikmer.DecodeKmer(w, k, nil))
+		rev[i] = string(obikmer.DecodeKmer(obikmer.ReverseComplement(w, k), k, nil))
+	}
+	return
+}
+
+// matchCorpus runs the scanners over every sequence of the corpus in parallel
+// and assembles the per-word results. The output does not depend on the number
+// of workers.
+//
+// The occurrences of a sequence are kept in a slice of their exact size, then
+// counted per word, so that the occurrences of all the words are laid out in
+// one allocation of the exact size too: nothing grows by doubling, nor is held
+// twice for long.
+func matchCorpus(c *Corpus, words []uint64, mk scannerMaker) []WordMatches {
+	perSeq := make([][]hit, c.Len())
+
+	jobs := make(chan int)
+	var wg sync.WaitGroup
+	for range obidefault.ParallelWorkers() {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			scan := mk()
+			var buf []hit
+			var raw, bases []byte
+			for i := range jobs {
+				buf = buf[:0]
+				raw, bases = c.bases(i, bases)
+				scan(raw, func(word, pos, errors int, reverse bool) {
+					buf = append(buf, hit{uint32(word), uint32(pos), uint8(errors), reverse})
+				})
+				if len(buf) > 0 {
+					perSeq[i] = slices.Clone(buf)
+				}
+			}
+		}()
+	}
+	for i := range c.Roles {
+		jobs <- i
+	}
+	close(jobs)
+	wg.Wait()
+
+	res := make([]WordMatches, len(words))
+	count := make([]int, len(words))
+	for _, hits := range perSeq {
+		for _, h := range hits {
+			count[h.word]++
+		}
+	}
+	for i, w := range words {
+		res[i].Word = w
+		if count[i] > 0 {
+			res[i].Matches = make([]Match, 0, count[i])
+		}
+	}
+	for s, hits := range perSeq {
+		for _, h := range hits {
+			res[h.word].Matches = append(res[h.word].Matches, Match{uint32(s), h.pos, h.errors, h.reverse})
+		}
+		perSeq[s] = nil // the hits of the sequence are no longer needed
+	}
+	parallelFor(len(res), func(_, i int) {
+		slices.SortFunc(res[i].Matches, func(a, b Match) int {
+			if c := cmp.Compare(a.Seq, b.Seq); c != 0 {
+				return c
+			}
+			if c := cmp.Compare(a.Pos, b.Pos); c != 0 {
+				return c
+			}
+			if a.Reverse != b.Reverse {
+				if !a.Reverse {
+					return -1
+				}
+				return 1
+			}
+			return 0
+		})
+	})
+	return res
+}
+
+// SequencesMatched returns, for the matches of one word, the number of
+// distinct example and counterexample sequences it occurs in. The other
+// sequences are not counted.
+func (w WordMatches) SequencesMatched(c *Corpus) (examples, counterExamples int) {
+	last := ^uint32(0)
+	for _, m := range w.Matches {
+		if m.Seq == last {
+			continue
+		}
+		last = m.Seq
+		switch c.Roles[m.Seq] {
+		case Example:
+			examples++
+		case CounterExample:
+			counterExamples++
+		}
+	}
+	return
+}

pkg/obiprimer/match_naive.go

@@ -0,0 +1,75 @@
+package obiprimer
+
+// baseCode maps a nucleotide to 0..3 (A,C,G,T; U is T) and everything else to 4,
+// which never equals a word base.
+var baseCode = func() (t [256]byte) {
+	for i := range t {
+		t[i] = 4
+	}
+	for i, c := range "ACGT" {
+		t[c], t[c+'a'-'A'] = byte(i), byte(i)
+	}
+	t['U'], t['u'] = 3, 3
+	return
+}()
+
+func codesOf(s string) []byte {
+	out := make([]byte, len(s))
+	for i := range out {
+		out[i] = baseCode[s[i]]
+	}
+	return out
+}
+
+// matchNaive is the reference implementation: every window of every sequence
+// is compared with every word, base by base. It is slow and exists to define
+// the exact semantics (see MatchParams) that faster implementations must
+// reproduce.
+func matchNaive(c *Corpus, words []uint64, p MatchParams) ([]WordMatches, error) {
+	if err := p.validate(); err != nil {
+		return nil, err
+	}
+	fwd, rev := wordStrings(words, p.K)
+	fc, rc := make([][]byte, len(words)), make([][]byte, len(words))
+	for i := range words {
+		fc[i], rc[i] = codesOf(fwd[i]), codesOf(rev[i])
+	}
+
+	mk := func() scanFunc {
+		var codes []byte
+		return func(raw []byte, emit func(word, pos, errors int, reverse bool)) {
+			n := len(raw)
+			if n < p.K {
+				return
+			}
+			codes = codes[:0]
+			for _, b := range raw {
+				codes = append(codes, baseCode[b])
+			}
+			windows := n - p.K + 1
+			if p.Circular {
+				codes = append(codes, codes[:p.K-1]...)
+				windows = n
+			}
+			for pos := range windows {
+				win := codes[pos : pos+p.K]
+				for w := range words {
+					for strand, pat := range [2][]byte{fc[w], rc[w]} {
+						errs := 0
+						for j, b := range pat {
+							if win[j] != b {
+								if errs++; errs > p.MaxErrors {
+									break
+								}
+							}
+						}
+						if errs <= p.MaxErrors {
+							emit(w, pos, errs, strand == 1)
+						}
+					}
+				}
+			}
+		}
+	}
+	return matchCorpus(c, words, mk), nil
+}

pkg/obiprimer/match_seed.go

@@ -0,0 +1,211 @@
+package obiprimer
+
+import (
+	"math/bits"
+	"slices"
+)
+
+// seedIndex maps the value of one half of a window to the patterns that are
+// candidates for it: those whose same half is within d substitutions of it.
+//
+// Pigeonhole: a window with at most e = 2d+1 errors against a pattern has at
+// most d errors in at least one of the two halves of the pattern, so the
+// windows to verify are exactly those found through one of the two indexes.
+// Indexing the half of every pattern together with its neighbours at distance
+// d (1 + 3L values for d = 1, 1 + 3L + 9L(L-1)/2 for d = 2, L = 9 bases) costs
+// a few tens of entries per pattern, and a window of a random text finds about
+// patterns x neighbours / 4^L candidates, far fewer than the patterns.
+type seedIndex struct {
+	shift   uint     // slot = seed >> shift when hashed, else seed
+	hashed  bool     // the seed space is too large for a direct table
+	offsets []uint32 // CSR: ids[offsets[s]:offsets[s+1]] are the candidates of slot s
+	ids     []int32  // pattern ids (2*word + strand)
+}
+
+const maxDirectSeedBits = 22
+
+func (x *seedIndex) slot(seed uint64) uint32 {
+	if !x.hashed {
+		return uint32(seed)
+	}
+	return uint32((seed * 0x9E3779B97F4A7C15) >> x.shift)
+}
+
+func (x *seedIndex) candidates(seed uint64) []int32 {
+	s := x.slot(seed)
+	return x.ids[x.offsets[s]:x.offsets[s+1]]
+}
+
+// neighbours calls fn for v and every value within d substitutions of it (each
+// once), v being L bases packed on 2 bits.
+func neighbours(v uint64, L, d int, fn func(uint64)) {
+	var rec func(v uint64, from, left int)
+	rec = func(v uint64, from, left int) {
+		fn(v)
+		if left == 0 {
+			return
+		}
+		for pos := from; pos < L; pos++ {
+			for alt := uint64(1); alt <= 3; alt++ {
+				rec(v^(alt<<(2*uint(pos))), pos+1, left-1)
+			}
+		}
+	}
+	rec(v, 0, d)
+}
+
+// newSeedIndex indexes, for every pattern, the half of the given width (in
+// bases) selected by (code >> shiftBases*2) & mask, with its neighbours at
+// distance d.
+func newSeedIndex(patterns []uint64, shiftBases, width, d int) *seedIndex {
+	x := &seedIndex{}
+	bitsNeeded := 2 * width
+	if bitsNeeded > maxDirectSeedBits {
+		x.hashed, bitsNeeded = true, maxDirectSeedBits
+		x.shift = uint(64 - maxDirectSeedBits)
+	}
+	nslots := 1 << uint(bitsNeeded)
+	halfMask := uint64(1)<<(2*uint(width)) - 1
+
+	type entry struct {
+		slot uint32
+		id   int32
+	}
+	var entries []entry
+	var scratch []uint32
+	for id, pat := range patterns {
+		half := (pat >> (2 * uint(shiftBases))) & halfMask
+		scratch = scratch[:0]
+		neighbours(half, width, d, func(v uint64) { scratch = append(scratch, x.slot(v)) })
+		slices.Sort(scratch)
+		scratch = slices.Compact(scratch) // hashing may merge neighbours
+		for _, s := range scratch {
+			entries = append(entries, entry{s, int32(id)})
+		}
+	}
+
+	x.offsets = make([]uint32, nslots+1)
+	for _, e := range entries {
+		x.offsets[e.slot+1]++
+	}
+	for i := 1; i <= nslots; i++ {
+		x.offsets[i] += x.offsets[i-1]
+	}
+	x.ids = make([]int32, len(entries))
+	fill := slices.Clone(x.offsets[:nslots])
+	for _, e := range entries {
+		x.ids[fill[e.slot]] = e.id
+		fill[e.slot]++
+	}
+	return x
+}
+
+// MatchWords finds every approximate occurrence, on both strands, of the given
+// canonical words in the sequences of the corpus, with the rules of MatchParams
+// (substitutions only, ambiguous bases count as errors, circular junction).
+//
+// It uses the pigeonhole seed indexes above. It returns exactly what the
+// reference matchNaive returns, and what ecoPrimers' fuzzy search finds, in a
+// small fraction of the time: on the Teleostei tutorial data (255 words, 3909
+// sequences, 65 Mb, e = 3) 0.42 s on one core, against 113 s for the
+// obiapat (agrep) search it replaces.
+func MatchWords(c *Corpus, words []uint64, p MatchParams) ([]WordMatches, error) {
+	if err := p.validate(); err != nil {
+		return nil, err
+	}
+	k := p.K
+	d := p.MaxErrors / 2
+
+	// pattern 2w is the word, 2w+1 its reverse complement
+	patterns := make([]uint64, 2*len(words))
+	for i, w := range words {
+		patterns[2*i] = codeOf(wordCodes(w, k, false), k)
+		patterns[2*i+1] = codeOf(wordCodes(w, k, true), k)
+	}
+
+	right := k / 2 // the right half holds the low bits
+	left := k - right
+	leftIdx := newSeedIndex(patterns, right, left, d)
+	rightIdx := newSeedIndex(patterns, 0, right, d)
+
+	rightMask := uint64(1)<<(2*uint(right)) - 1
+	windowMask := uint64(1)<<(2*uint(k)) - 1
+	lowBits := uint64(0x5555555555555555) & windowMask
+	maxErr := p.MaxErrors
+
+	mk := func() scanFunc {
+		return func(raw []byte, emit func(word, pos, errors int, reverse bool)) {
+			n := len(raw)
+			if n < k {
+				return
+			}
+			total := n // number of bases fed to the rolling window
+			windows := n - k + 1
+			if p.Circular {
+				total, windows = n+k-1, n
+			}
+
+			var code, bad uint64 // window, and its non-ACGT positions (bit 2j)
+			verify := func(pos int, id int32) {
+				pat := patterns[id]
+				x := code ^ pat
+				if errs := bits.OnesCount64(((x | x>>1) & lowBits) | bad); errs <= maxErr {
+					emit(int(id)>>1, pos, errs, id&1 == 1)
+				}
+			}
+
+			for i := 0; i < total; i++ {
+				j := i
+				if j >= n {
+					j -= n
+				}
+				v := baseCode[raw[j]]
+				isBad := uint64(0)
+				if v > 3 {
+					v, isBad = 0, 1
+				}
+				code = (code<<2 | uint64(v)) & windowMask
+				bad = (bad<<2 | isBad) & windowMask
+
+				if i < k-1 {
+					continue
+				}
+				pos := i - k + 1
+				if pos >= windows {
+					break
+				}
+				leftCands := leftIdx.candidates(code >> (2 * uint(right)))
+				for _, id := range leftCands {
+					verify(pos, id)
+				}
+				for _, id := range rightIdx.candidates(code & rightMask) {
+					// ids are sorted: skip the ones the left index has just verified
+					if _, done := slices.BinarySearch(leftCands, id); !done {
+						verify(pos, id)
+					}
+				}
+			}
+		}
+	}
+	return matchCorpus(c, words, mk), nil
+}
+
+// wordCodes returns the 2-bit codes (A=0..T=3) of the word or of its reverse
+// complement, 5' to 3'.
+func wordCodes(word uint64, k int, reverse bool) []byte {
+	fwd, rev := wordStrings([]uint64{word}, k)
+	s := fwd[0]
+	if reverse {
+		s = rev[0]
+	}
+	return codesOf(s)
+}
+
+// codeOf packs codes (each 0..3) on 2 bits, first base in the high bits.
+func codeOf(codes []byte, k int) uint64 {
+	var v uint64
+	for _, c := range codes[:k] {
+		v = v<<2 | uint64(c)
+	}
+	return v
+}

pkg/obiprimer/match_test.go

@@ -0,0 +1,304 @@
+package obiprimer
+
+import (
+	"bufio"
+	"fmt"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"reflect"
+	"runtime"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiformats"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obikmer"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+func corpusOf(roles []Role, seqs ...string) *Corpus {
+	c := &Corpus{Roles: roles}
+	for i, s := range seqs {
+		c.Seqs = append(c.Seqs, obiseq.NewBioSequence(fmt.Sprintf("s%d", i), []byte(s), ""))
+	}
+	return c
+}
+
+func canon(word string) uint64 { return obikmer.EncodeCanonicalKmer([]byte(word), len(word)) }
+
+// The reference implementation is checked by hand first: everything else is
+// compared with it.
+func TestNaiveMatchSemantics(t *testing.T) {
+	const word = "AACCGGTA" // not a palindrome; reverse complement TACCGGTT
+	cases := []struct {
+		name     string
+		seq      string
+		circular bool
+		e        int
+		want     []Match
+	}{
+		{"exact forward", "ggaaccggtacc", false, 0, []Match{{0, 2, 0, false}}},
+		{"exact reverse strand", "ggtaccggttcc", false, 0, []Match{{0, 2, 0, true}}},
+		{"one error is not found at e=0", "ggaaccgctacc", false, 0, nil},
+		{"one error found at e=1 with its count", "ggaaccgctacc", false, 1, []Match{{0, 2, 1, false}}},
+		{"an N is an error", "ggaaccgntacc", false, 1, []Match{{0, 2, 1, false}}},
+		{"an N is never a match at e=0", "ggaaccgntacc", false, 0, nil},
+		{"an IUPAC code is an error, not a wildcard", "ggaaccgrtacc", false, 0, nil},
+		{"U reads as T", "ggaaccgguacc", false, 0, []Match{{0, 2, 0, false}}},
+		{"junction ignored on a linear sequence", "ccggtagggaa", false, 0, nil},
+		{"junction found on a circular sequence", "ccggtagggaa", true, 0, []Match{{0, 9, 0, false}}},
+		{"overlapping windows are all reported", "aaccggtaaccggta", false, 0, []Match{{0, 0, 0, false}, {0, 7, 0, false}}},
+		{"too short", "aaccgg", true, 3, nil},
+	}
+	for _, tc := range cases {
+		c := corpusOf([]Role{Example}, tc.seq)
+		got, err := matchNaive(c, []uint64{canon(word)}, MatchParams{K: len(word), MaxErrors: tc.e, Circular: tc.circular})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !reflect.DeepEqual(got[0].Matches, tc.want) {
+			t.Errorf("%s: got %v, want %v", tc.name, got[0].Matches, tc.want)
+		}
+	}
+}
+
+func TestMatchParamsValidation(t *testing.T) {
+	c := corpusOf([]Role{Example}, "acgtacgtacgtacgt")
+	for _, p := range []MatchParams{{K: 1}, {K: 32}, {K: 8, MaxErrors: -1}, {K: 8, MaxErrors: 8}} {
+		if _, err := matchNaive(c, nil, p); err == nil {
+			t.Errorf("%+v must be rejected", p)
+		}
+		if _, err := MatchWords(c, nil, p); err == nil {
+			t.Errorf("%+v must be rejected by MatchWords", p)
+		}
+	}
+}
+
+func randSeq(rng *rand.Rand, n int, alphabet string) []byte {
+	b := make([]byte, n)
+	for i := range b {
+		b[i] = alphabet[rng.Intn(len(alphabet))]
+	}
+	return b
+}
+
+func mutate(rng *rand.Rand, s []byte, n int) []byte {
+	out := append([]byte(nil), s...)
+	for _, i := range rng.Perm(len(out))[:n] {
+		out[i] = "acgt"[rng.Intn(4)]
+	}
+	return out
+}
+
+// randomCorpus builds sequences with words planted on both strands, with 0..3
+// substitutions, across the circular junction, and with ambiguity codes.
+func randomCorpus(rng *rand.Rand, words []string, nseq int) *Corpus {
+	c := &Corpus{}
+	for i := 0; i < nseq; i++ {
+		alphabet := "acgt"
+		if i%3 == 0 {
+			alphabet = "acgtacgtacgtacgtacgtnnrykm"
+		}
+		s := randSeq(rng, 30+rng.Intn(500), alphabet)
+		for range 1 + rng.Intn(4) {
+			w := []byte(words[rng.Intn(len(words))])
+			if rng.Intn(2) == 0 {
+				w = []byte(revcomp(string(w)))
+			}
+			w = mutate(rng, []byte(string(w)), rng.Intn(4))
+			for j := range w {
+				w[j] += 'a' - 'A'
+			}
+			at := rng.Intn(len(s) - len(w) + 1)
+			if rng.Intn(5) == 0 { // across the junction
+				s = append(s[:0:0], append(append([]byte(nil), w[len(w)/2:]...), append(s[:len(s)-len(w)/2], w[:len(w)/2]...)...)...)
+				continue
+			}
+			copy(s[at:], w)
+		}
+		c.Seqs = append(c.Seqs, obiseq.NewBioSequence(fmt.Sprintf("r%d", i), s, ""))
+		c.Roles = append(c.Roles, Example)
+	}
+	return c
+}
+
+// Every implementation must return exactly what the reference returns.
+func TestMatchersAgreeWithReference(t *testing.T) {
+	for _, tc := range []struct {
+		k, e     int
+		circular bool
+	}{{18, 0, false}, {18, 1, true}, {18, 3, false}, {18, 3, true}, {12, 2, true}, {25, 4, false}, {18, 2, true}, {18, 5, false}, {31, 3, true}, {24, 3, false}, {9, 2, true}, {18, 6, true}, {18, 7, false}, {20, 8, true}, {12, 11, false}} {
+		t.Run(fmt.Sprintf("k%d-e%d-circ%v", tc.k, tc.e, tc.circular), func(t *testing.T) {
+			rng := rand.New(rand.NewSource(int64(tc.k*100 + tc.e)))
+			var wordStr []string
+			var words []uint64
+			for range 7 {
+				w := string(randSeq(rng, tc.k, "ACGT"))
+				wordStr = append(wordStr, w)
+				words = append(words, canon(w))
+			}
+			c := randomCorpus(rng, wordStr, 40)
+			p := MatchParams{K: tc.k, MaxErrors: tc.e, Circular: tc.circular}
+
+			want, err := matchNaive(c, words, p)
+			if err != nil {
+				t.Fatal(err)
+			}
+			total := 0
+			for _, w := range want {
+				total += len(w.Matches)
+			}
+			if total < 10 {
+				t.Fatalf("test data too poor: only %d reference matches", total)
+			}
+
+			got, err := MatchWords(c, words, p)
+			if err != nil {
+				t.Fatal(err)
+			}
+			for i := range want {
+				if !reflect.DeepEqual(got[i].Matches, want[i].Matches) {
+					t.Fatalf("word %d: MatchWords gives %v\nreference gives %v", i, got[i].Matches, want[i].Matches)
+				}
+			}
+		})
+	}
+}
+
+func TestSequencesMatched(t *testing.T) {
+	c := corpusOf([]Role{Example, Example, CounterExample, CounterExample}, "a", "a", "a", "a")
+	w := WordMatches{Matches: []Match{{0, 1, 0, false}, {0, 9, 0, true}, {1, 3, 0, false}, {3, 0, 0, false}}}
+	if e, x := w.SequencesMatched(c); e != 2 || x != 1 {
+		t.Errorf("got %d examples, %d counterexamples", e, x)
+	}
+}
+
+func readEcoCounts(t *testing.T, path string) map[string]int {
+	t.Helper()
+	f, err := os.Open(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	counts := map[string]int{}
+	sc := bufio.NewScanner(f)
+	for sc.Scan() {
+		var w string
+		var in, out int
+		if _, err := fmt.Sscanf(sc.Text(), "%s\t%d\t%d", &w, &in, &out); err != nil {
+			t.Fatal(err)
+		}
+		counts[w] = in
+	}
+	return counts
+}
+
+// Opt-in comparison with ecoPrimers on the tutorial data (same variable as the
+// strict word test). For every strict word, ecoPrimers logs the number of
+// example sequences in which the fuzzy search (-e 3, both strands, circular)
+// finds it; testdata/ecoprimers_*.words.tsv holds those numbers.
+func TestMatchWordsAgainstEcoPrimers(t *testing.T) {
+	dir := os.Getenv("OBIPRIMERS_TUTORIAL")
+	if dir == "" {
+		t.Skip("OBIPRIMERS_TUTORIAL not set")
+	}
+	if v := os.Getenv("OBIPRIMERS_WORKERS"); v != "" { // e.g. 1, to time a single core
+		n, err := strconv.Atoi(v)
+		if err != nil {
+			t.Fatal(err)
+		}
+		obidefault.SetMaxCPU(n)
+	}
+	taxonomy, err := obiformats.LoadTaxonomy(filepath.Join(dir, "ncbitaxo_20250211.tgz"), false, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, tc := range []struct{ name, taxid string }{{"Cyprinidae", "7953"}, {"Teleostei", "32443"}} {
+		t.Run(tc.name, func(t *testing.T) {
+			restrict, err := BelongingTo(taxonomy, []string{tc.taxid})
+			if err != nil {
+				t.Fatal(err)
+			}
+			seqs, err := obiformats.ReadSequencesFromFile(filepath.Join(dir, "mito.vert.fasta.gz"))
+			if err != nil {
+				t.Fatal(err)
+			}
+			corpus, err := LoadCorpus(seqs, Selection{Restrict: restrict})
+			if err != nil {
+				t.Fatal(err)
+			}
+			sw, err := FindStrictWords(corpus, WordSearch{PrimerLength: 18, Quorum: 0.7, Circular: true, TmpDir: t.TempDir()})
+			if err != nil {
+				t.Fatal(err)
+			}
+			words := make([]uint64, len(sw.Words))
+			for i, w := range sw.Words {
+				words[i] = w.Kmer
+			}
+
+			var before, after runtime.MemStats
+			runtime.GC()
+			runtime.ReadMemStats(&before)
+			start := time.Now()
+			res, err := MatchWords(corpus, words, MatchParams{K: 18, MaxErrors: 3, Circular: true})
+			elapsed := time.Since(start)
+			runtime.ReadMemStats(&after)
+			if err != nil {
+				t.Fatal(err)
+			}
+			nm := 0
+			for _, r := range res {
+				nm += len(r.Matches)
+			}
+			bases := 0
+			for _, s := range corpus.Seqs {
+				bases += s.Len()
+			}
+			t.Logf("%d words x %d sequences (%.1f Mb) e=3 in %s; %d matches; allocated %.0f MB",
+				len(words), len(corpus.Seqs), float64(bases)/1e6, elapsed.Round(time.Millisecond), nm,
+				float64(after.TotalAlloc-before.TotalAlloc)/1e6)
+
+			// On real sequences (with their ambiguity codes), the first ones must give
+			// exactly what the reference implementation gives.
+			sub := &Corpus{Seqs: corpus.Seqs[:40], Roles: corpus.Roles[:40]}
+			wantSub, err := matchNaive(sub, words, MatchParams{K: 18, MaxErrors: 3, Circular: true})
+			if err != nil {
+				t.Fatal(err)
+			}
+			gotSub, err := MatchWords(sub, words, MatchParams{K: 18, MaxErrors: 3, Circular: true})
+			if err != nil {
+				t.Fatal(err)
+			}
+			subMatches := 0
+			for i := range wantSub {
+				subMatches += len(wantSub[i].Matches)
+				if !reflect.DeepEqual(gotSub[i].Matches, wantSub[i].Matches) {
+					t.Fatalf("word %d differs from the reference on the first 40 sequences", i)
+				}
+			}
+			t.Logf("first 40 sequences identical to the reference (%d matches)", subMatches)
+
+			eco := readEcoCounts(t, filepath.Join("testdata", "ecoprimers_"+tc.name+".words.tsv"))
+			compared, differ := 0, 0
+			for i, r := range res {
+				w := strings.ToUpper(string(obikmer.DecodeKmer(words[i], 18, nil)))
+				want, ok := eco[w]
+				if !ok {
+					t.Errorf("word %s is not in the ecoPrimers log", w)
+					continue
+				}
+				got, _ := r.SequencesMatched(corpus)
+				compared++
+				if got != want {
+					differ++
+					t.Errorf("word %s: %d example sequences, ecoPrimers %d", w, got, want)
+				}
+			}
+			t.Logf("%d words compared with ecoPrimers, %d differ", compared, differ)
+		})
+	}
+}

pkg/obiprimer/multimatch.go

@@ -0,0 +1,42 @@
+package obiprimer
+
+import "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obikmer"
+
+// DropRepeatedWords removes the strict words found more than once (on either
+// strand) in at least one example sequence. It is the first half of the
+// ecoPrimers -U option (no multi-match).
+func DropRepeatedWords(words []obikmer.SeqKmerCount) (kept []obikmer.SeqKmerCount, dropped int) {
+	kept = make([]obikmer.SeqKmerCount, 0, len(words))
+	for _, w := range words {
+		if w.Repeated {
+			dropped++
+			continue
+		}
+		kept = append(kept, w)
+	}
+	return kept, dropped
+}
+
+// DropMultiMatch removes the words having two or more approximate occurrences,
+// both strands counted, in at least one example or counterexample sequence. It
+// is the second half of the ecoPrimers -U option: the primers that can anneal
+// at several places of a sequence they must (or must not) amplify are
+// discarded. The other sequences are not considered.
+func DropMultiMatch(wm []WordMatches, c *Corpus) (kept []WordMatches, dropped int) {
+	kept = make([]WordMatches, 0, len(wm))
+	for _, w := range wm {
+		multi := false
+		for i := 1; i < len(w.Matches); i++ {
+			if w.Matches[i].Seq == w.Matches[i-1].Seq && c.Roles[w.Matches[i].Seq] != Other {
+				multi = true
+				break
+			}
+		}
+		if multi {
+			dropped++
+			continue
+		}
+		kept = append(kept, w)
+	}
+	return kept, dropped
+}

pkg/obiprimer/packed.go

@@ -0,0 +1,80 @@
+package obiprimer
+
+import (
+	"encoding/binary"
+	"slices"
+)
+
+// packedSeq holds the bases of a sequence on 2 bits each, a quarter of the
+// bytes of the sequence itself. The bytes that are not a, c, g or t (N, the
+// IUPAC codes, u...) are kept apart in runs, so that unpacking gives back the
+// sequence byte for byte.
+type packedSeq struct {
+	n    int
+	bits []byte   // 4 bases per byte, the first in the low bits
+	exc  []excRun // sorted by position
+}
+
+// excRun is a run of length identical bytes b that are not a, c, g or t,
+// starting at pos.
+type excRun struct {
+	pos, length uint32
+	b           byte
+}
+
+// packCode maps a, c, g, t to 0..3 and every other byte to 4.
+var packCode = func() (t [256]byte) {
+	for i := range t {
+		t[i] = 4
+	}
+	for i, c := range "acgt" {
+		t[c] = byte(i)
+	}
+	return
+}()
+
+// unpackWord gives, for a byte of 4 packed bases, the 4 bytes they stand for.
+var unpackWord = func() (t [256]uint32) {
+	for b := range t {
+		for k := range 4 {
+			t[b] |= uint32("acgt"[(b>>(2*k))&3]) << (8 * k)
+		}
+	}
+	return
+}()
+
+// packBases packs raw (lower case).
+func packBases(raw []byte) packedSeq {
+	p := packedSeq{n: len(raw), bits: make([]byte, (len(raw)+3)/4)}
+	for i, b := range raw {
+		code := packCode[b]
+		if code > 3 {
+			if k := len(p.exc) - 1; k >= 0 && p.exc[k].b == b && int(p.exc[k].pos)+int(p.exc[k].length) == i {
+				p.exc[k].length++
+			} else {
+				p.exc = append(p.exc, excRun{uint32(i), 1, b})
+			}
+			code = 0
+		}
+		p.bits[i>>2] |= code << (2 * (i & 3))
+	}
+	p.exc = slices.Clone(p.exc) // the exact size, nil when there is none
+	return p
+}
+
+// unpack writes the sequence into dst, which it grows if need be, and returns it.
+func (p packedSeq) unpack(dst []byte) []byte {
+	dst = slices.Grow(dst[:0], p.n)[:p.n]
+	for i := range p.n / 4 {
+		binary.LittleEndian.PutUint32(dst[4*i:], unpackWord[p.bits[i]])
+	}
+	for t := p.n &^ 3; t < p.n; t++ {
+		dst[t] = "acgt"[(p.bits[t>>2]>>(2*(t&3)))&3]
+	}
+	for _, e := range p.exc {
+		for k := range e.length {
+			dst[e.pos+k] = e.b
+		}
+	}
+	return dst
+}

pkg/obiprimer/packed_test.go

@@ -0,0 +1,109 @@
+package obiprimer
+
+import (
+	"bytes"
+	"math/rand"
+	"reflect"
+	"strings"
+	"testing"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+func TestPackedSeqRoundTrip(t *testing.T) {
+	rng := rand.New(rand.NewSource(1))
+	alphabet := []byte("acgtacgtacgtacgtnnrykmu-.")
+	for trial := 0; trial < 500; trial++ {
+		n := rng.Intn(70) // all the lengths modulo 4, the empty sequence included
+		raw := make([]byte, n)
+		for i := range raw {
+			raw[i] = alphabet[rng.Intn(len(alphabet))]
+		}
+		if trial%5 == 0 && n > 10 { // a long run of N, and one at each end
+			copy(raw[2:9], "nnnnnnn")
+			raw[0], raw[n-1] = 'n', 'n'
+		}
+		p := packBases(raw)
+		buf := make([]byte, 3, 8) // dirty, too small or too big: whatever
+		if got := p.unpack(buf); !bytes.Equal(got, raw) {
+			t.Fatalf("trial %d: %q unpacked as %q", trial, raw, got)
+		}
+		if len(p.bits) != (n+3)/4 {
+			t.Fatalf("trial %d: %d bytes for %d bases", trial, len(p.bits), n)
+		}
+	}
+	if p := packBases([]byte("acgtacgt")); p.exc != nil {
+		t.Errorf("no exception expected: %v", p.exc)
+	}
+}
+
+// Packing the corpus must change nothing to the search, ambiguous bases and
+// reference included.
+func TestPackedCorpusGivesTheSameSearch(t *testing.T) {
+	build := func() synthetic {
+		s := newSynthetic(10, -1, 7)
+		rng := rand.New(rand.NewSource(3))
+		for k, seq := range s.corpus.Seqs { // sprinkle ambiguous bases, some inside the sites
+			raw := append([]byte(nil), seq.Sequence()...)
+			for range 3 { // in the flanks, and in the insert past the first bases
+				at := rng.Intn(len(raw))
+				if at >= s.pos-2 && at < s.pos+18+2 || at >= s.pos+18+40-2 && at < s.pos+18+40+18+2 {
+					continue
+				}
+				raw[at] = "nrykm"[rng.Intn(5)]
+			}
+			if k < 2 { // one mismatch inside a site, which the search tolerates
+				raw[s.pos+5] = 'n'
+			}
+			s.corpus.Seqs[k] = obiseq.NewBioSequence(seq.Id(), raw, "")
+		}
+		return s
+	}
+	run := func(pack bool, circular bool) []PairStats {
+		s := build()
+		if pack {
+			s.corpus.Pack()
+			for i := range s.corpus.Seqs {
+				if s.corpus.Seqs[i] != nil {
+					t.Fatal("Pack must free the sequences")
+				}
+			}
+		}
+		p := s.params()
+		p.Reference = "s3"
+		p.ThreePrime = 3
+		p.Circular = circular
+		p.MinSpecificity = 0
+		pairs, _, err := Search(s.corpus, s.taxa, p)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return pairs
+	}
+	for _, circular := range []bool{false, true} {
+		want := run(false, circular)
+		if len(want) == 0 {
+			t.Fatal("the test needs pairs")
+		}
+		if got := run(true, circular); !reflect.DeepEqual(got, want) {
+			t.Errorf("circular=%v: packing changes the result (%d pairs, want %d)", circular, len(got), len(want))
+		}
+	}
+}
+
+func TestPackedCorpusAccessors(t *testing.T) {
+	c := &Corpus{
+		Seqs:  []*obiseq.BioSequence{obiseq.NewBioSequence("a", []byte(strings.ToLower("ACGTNNAC")), ""), obiseq.NewBioSequence("b", []byte("gg"), "")},
+		Roles: []Role{Example, Other},
+	}
+	c.Pack()
+	if c.Len() != 2 || c.SeqLen(0) != 8 || c.SeqLen(1) != 2 || c.ID(1) != "b" {
+		t.Errorf("accessors: %d %d %d %q", c.Len(), c.SeqLen(0), c.SeqLen(1), c.ID(1))
+	}
+	if raw, _ := c.bases(0, nil); string(raw) != "acgtnnac" {
+		t.Errorf("bases: %q", raw)
+	}
+	if i, err := c.IndexOfExample("a"); err != nil || i != 0 {
+		t.Errorf("IndexOfExample: %d %v", i, err)
+	}
+}

pkg/obiprimer/pairs.go

@@ -0,0 +1,326 @@
+package obiprimer
+
+import (
+	"cmp"
+	"fmt"
+	"slices"
+	"strings"
+	"sync"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obikmer"
+)
+
+// PairParams holds the parameters of the primer pair search, named after the
+// ecoPrimers options.
+type PairParams struct {
+	PrimerLength   int     // -O, word size
+	MinLength      int     // -l: the amplicon (primers excluded) must be longer than this
+	MaxLength      int     // -L: and shorter than this
+	ThreePrime     int     // -3: number of 3' bases that must match exactly (0 = no constraint)
+	Sensitivity    float64 // -s: a primer must hit more than this fraction of the examples (0.9)
+	FalsePositive  float64 // -x: and less than this fraction of the counterexamples (0.1)
+	MinSpecificity float64 // -T: pairs with a lower Bs are dropped (0.6)
+	Circular       bool    // -c
+	Reference      int     // -R: corpus index of the reference sequence (used when HasReference)
+	HasReference   bool
+	AmpBudget      int    // amplicon records held at once in phase 2 (0 = DefaultAmpBudget)
+	Release        func() // see SearchParams.Release
+}
+
+func (p PairParams) validate() error {
+	switch {
+	case p.PrimerLength < 2 || p.PrimerLength > 31:
+		return fmt.Errorf("obiprimer: primer length must be between 2 and 31, got %d", p.PrimerLength)
+	case p.MaxLength <= p.MinLength+1:
+		return fmt.Errorf("obiprimer: empty amplicon length range ]%d,%d[", p.MinLength, p.MaxLength)
+	case p.ThreePrime < 0 || p.ThreePrime > p.PrimerLength:
+		return fmt.Errorf("obiprimer: 3' strict bases must be in [0,%d], got %d", p.PrimerLength, p.ThreePrime)
+	case p.Sensitivity <= 0 || p.Sensitivity > 1 || p.FalsePositive < 0 || p.FalsePositive > 1:
+		return fmt.Errorf("obiprimer: quorums must be in ]0,1] and [0,1]")
+	}
+	return nil
+}
+
+// Primer is a conserved word: one found, with at most the allowed errors, in
+// enough example sequences to be a candidate primer.
+type Primer struct {
+	Word            uint64 // canonical word
+	Examples        int    // example sequences where it is found (either strand)
+	CounterExamples int
+	// Good means conserved in the examples and rare enough in the
+	// counterexamples; a pair needs at least one good primer.
+	Good bool
+}
+
+// seqMatch is one occurrence of a primer in one sequence, on 8 bytes: there
+// are millions of them.
+type seqMatch struct {
+	primer int32
+	place  uint32 // position << 1, plus 1 when the primer is found reversed
+}
+
+func newSeqMatch(primer int32, pos uint32, reverse bool) seqMatch {
+	return seqMatch{primer, pos<<1 | uint32(b2i(reverse))}
+}
+
+func (m seqMatch) pos() uint32   { return m.place >> 1 }
+func (m seqMatch) reverse() bool { return m.place&1 == 1 }
+
+// primerSet is the conserved words and, for each corpus sequence, their
+// occurrences sorted by position.
+type primerSet struct {
+	primers []Primer
+	perSeq  [][]seqMatch
+}
+
+// quorumCount is floor(n*fraction), robust to the binary form of the fraction.
+func quorumCount(n int, fraction float64) int { return MinSequences(n, fraction) }
+
+// selectPrimers keeps the words found in at least floor(examples*Sensitivity)
+// example sequences and marks as good those found in at most
+// floor(counterexamples*FalsePositive) counterexamples.
+func selectPrimers(c *Corpus, wm []WordMatches, p PairParams) *primerSet {
+	counts := c.Counts()
+	inQ := quorumCount(counts.Examples, p.Sensitivity)
+	outQ := quorumCount(counts.CounterExamples, p.FalsePositive)
+
+	ex, co := make([]int, len(wm)), make([]int, len(wm))
+	parallelFor(len(wm), func(_, i int) { ex[i], co[i] = wm[i].SequencesMatched(c) })
+
+	// the occurrences of the words kept, by sequence, each in a slice of the
+	// exact size (and not in one big array: a few hundred bytes to some kB each
+	// find room in the free memory, which a single block of tens of MB does not)
+	ps := &primerSet{perSeq: make([][]seqMatch, c.Len())}
+	size := make([]int, c.Len())
+	for i, w := range wm {
+		if ex[i] < inQ {
+			continue
+		}
+		ps.primers = append(ps.primers, Primer{w.Word, ex[i], co[i], co[i] <= outQ})
+		for _, m := range w.Matches {
+			size[m.Seq]++
+		}
+	}
+	for s, n := range size {
+		if n > 0 {
+			ps.perSeq[s] = make([]seqMatch, 0, n)
+		}
+	}
+	id := int32(0)
+	for i, w := range wm {
+		if ex[i] < inQ {
+			continue
+		}
+		for _, m := range w.Matches {
+			ps.perSeq[m.Seq] = append(ps.perSeq[m.Seq], newSeqMatch(id, m.Pos, m.Reverse))
+		}
+		id++
+	}
+	parallelFor(c.Len(), func(_, s int) {
+		slices.SortStableFunc(ps.perSeq[s], func(a, b seqMatch) int { return cmp.Compare(a.pos(), b.pos()) })
+	})
+	return ps
+}
+
+// pairKey identifies a pair: the two primers, ordered by canonical word, and
+// for each whether the primer is the canonical word as is (true) or its reverse
+// complement (false). It is the key of ecoPrimers' pair tree.
+type pairKey struct {
+	p1, p2 int32
+	a1, a2 bool
+}
+
+// occurrence is one amplicon: a pair of matches in a sequence.
+type occurrence struct {
+	key      pairKey
+	swapped  bool // true when the upstream match is the second primer of the key
+	up, down seqMatch
+	length   int // amplicon length, primers excluded
+}
+
+// forEachOccurrence calls fn for every amplicon of sequence s, applying the
+// length, good-primer and 3' constraints of ecoPrimers. buf is the buffer in
+// which the sequence is unpacked, if the corpus is packed.
+func (ps *primerSet) forEachOccurrence(c *Corpus, s int, buf []byte, p PairParams, fn func(occurrence)) []byte {
+	ms := ps.perSeq[s]
+	var raw []byte
+	if p.ThreePrime > 0 && len(ms) > 1 {
+		raw, buf = c.bases(s, buf)
+	}
+	ps.forEachOccurrenceIn(raw, ms, p, fn)
+	return buf
+}
+
+// forEachOccurrenceIn is forEachOccurrence over ms, the matches of a sequence
+// sorted by position (all of them, or only those of the primers of interest,
+// which yields the amplicons between these primers and none other), and raw its
+// bases, needed when the 3' bases must match exactly.
+func (ps *primerSet) forEachOccurrenceIn(raw []byte, ms []seqMatch, p PairParams, fn func(occurrence)) {
+	lp := p.PrimerLength
+
+	for i := range ms {
+		for j := i + 1; j < len(ms); j++ {
+			dpos := int(ms[j].pos()) - int(ms[i].pos())
+			if dpos <= lp {
+				continue
+			}
+			length := dpos - lp
+			if length >= p.MaxLength {
+				break
+			}
+			up, down := ms[i], ms[j]
+			if length <= p.MinLength || !(ps.primers[up.primer].Good || ps.primers[down.primer].Good) {
+				continue
+			}
+
+			// The upstream match is read as is (primer 1 of the pair when its word
+			// is smaller), the downstream one as the reverse primer.
+			key := pairKey{up.primer, down.primer, !up.reverse(), down.reverse()}
+			swapped := !(ps.primers[key.p2].Word > ps.primers[key.p1].Word)
+			if swapped {
+				key = pairKey{key.p2, key.p1, key.a2, key.a1}
+			}
+
+			if p.ThreePrime > 0 && !ps.threePrimeMatches(raw, up, down, p) {
+				continue
+			}
+			fn(occurrence{key, swapped, up, down, length})
+		}
+	}
+}
+
+// wordBase returns base t (0 = 5') of a 2-bit packed word of size lp.
+func wordBase(word uint64, lp, t int) byte { return byte(word>>(2*uint(lp-1-t))) & 3 }
+
+// threePrimeMatches reports whether the last p.ThreePrime bases of the forward
+// primer (as found at the upstream match) and of the reverse primer (reverse
+// complement of the downstream window) equal the sequence, i.e. have no
+// mismatch at their 3' end.
+func (ps *primerSet) threePrimeMatches(raw []byte, up, down seqMatch, p PairParams) bool {
+	lp, n := p.PrimerLength, len(raw)
+	at := func(pos int) byte {
+		if p.Circular {
+			pos %= n
+		}
+		return baseCode[raw[pos]]
+	}
+
+	fw := ps.primers[up.primer].Word
+	rv := ps.primers[down.primer].Word
+	for t := lp - p.ThreePrime; t < lp; t++ {
+		// forward primer: the word when found direct, its reverse complement otherwise
+		var f byte
+		if !up.reverse() {
+			f = wordBase(fw, lp, t)
+		} else {
+			f = 3 - wordBase(fw, lp, lp-1-t)
+		}
+		if s := at(int(up.pos()) + t); s > 3 || s != f {
+			return false
+		}
+
+		// reverse primer: the word when found reverse, its reverse complement otherwise
+		var r byte
+		if down.reverse() {
+			r = wordBase(rv, lp, t)
+		} else {
+			r = 3 - wordBase(rv, lp, lp-1-t)
+		}
+		if s := at(int(down.pos()) + lp - 1 - t); s > 3 || 3-s != r {
+			return false
+		}
+	}
+	return true
+}
+
+// pairAcc accumulates, over the sequences, the counts ecoPrimers keeps while it
+// builds the pairs.
+type pairAcc struct {
+	examples, counters uint32
+	last               int32 // 1 + last sequence counted, to count each sequence once
+	amplicons          uint32
+	sumLen             uint64
+	minLen, maxLen     uint32
+}
+
+// countPairs enumerates the amplicons of every sequence, in parallel, and
+// returns for each pair the sequences it amplifies and the length statistics
+// of its example amplicons.
+func (ps *primerSet) countPairs(c *Corpus, p PairParams) map[pairKey]*pairAcc {
+	workers := obidefault.ParallelWorkers()
+	partial := make([]map[pairKey]*pairAcc, workers)
+
+	jobs := make(chan int)
+	var wg sync.WaitGroup
+	for w := range workers {
+		wg.Add(1)
+		partial[w] = map[pairKey]*pairAcc{}
+		go func() {
+			defer wg.Done()
+			acc := partial[w]
+			var buf []byte
+			for s := range jobs {
+				if c.Roles[s] == Other {
+					continue // only the quorums are counted here
+				}
+				isExample := c.Roles[s] == Example
+				buf = ps.forEachOccurrence(c, s, buf, p, func(o occurrence) {
+					a := acc[o.key]
+					if a == nil {
+						a = &pairAcc{minLen: ^uint32(0)}
+						acc[o.key] = a
+					}
+					if a.last != int32(s)+1 {
+						a.last = int32(s) + 1
+						if isExample {
+							a.examples++
+						} else {
+							a.counters++
+						}
+					}
+					if isExample {
+						l := uint32(o.length)
+						a.amplicons++
+						a.sumLen += uint64(l)
+						a.minLen = min(a.minLen, l)
+						a.maxLen = max(a.maxLen, l)
+					}
+				})
+			}
+		}()
+	}
+	for s := range c.Roles {
+		jobs <- s
+	}
+	close(jobs)
+	wg.Wait()
+
+	// A sequence is handled by one worker only, so the sequence counts add up.
+	total := partial[0]
+	for _, m := range partial[1:] {
+		for k, a := range m {
+			t := total[k]
+			if t == nil {
+				total[k] = a
+				continue
+			}
+			t.examples += a.examples
+			t.counters += a.counters
+			t.amplicons += a.amplicons
+			t.sumLen += a.sumLen
+			t.minLen = min(t.minLen, a.minLen)
+			t.maxLen = max(t.maxLen, a.maxLen)
+		}
+	}
+	return total
+}
+
+// orientedWord returns the upper-case primer string 5'->3': the canonical word
+// when direct, its reverse complement otherwise.
+func orientedWord(word uint64, lp int, direct bool) string {
+	if !direct {
+		word = obikmer.ReverseComplement(word, lp)
+	}
+	return strings.ToUpper(string(obikmer.DecodeKmer(word, lp, nil)))
+}

pkg/obiprimer/pairs_batches_test.go

@@ -0,0 +1,78 @@
+package obiprimer
+
+import (
+	"math/rand"
+	"reflect"
+	"strings"
+	"testing"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+func TestAmpBatches(t *testing.T) {
+	cases := []struct {
+		count  []int
+		budget int
+		want   [][2]int
+	}{
+		{[]int{3, 3, 3}, 0, [][2]int{{0, 3}}},
+		{[]int{3, 3, 3}, 100, [][2]int{{0, 3}}},
+		{[]int{3, 3, 3}, 6, [][2]int{{0, 2}, {2, 3}}},
+		{[]int{3, 3, 3}, 1, [][2]int{{0, 1}, {1, 2}, {2, 3}}}, // a pair above the budget goes alone
+		{[]int{9, 1, 1}, 5, [][2]int{{0, 1}, {1, 3}}},
+		{nil, 5, nil},
+		{[]int{0, 0}, 5, [][2]int{{0, 2}}},
+	}
+	for _, c := range cases {
+		if got := ampBatches(c.count, c.budget); !reflect.DeepEqual(got, c.want) {
+			t.Errorf("ampBatches(%v, %d) = %v, want %v", c.count, c.budget, got, c.want)
+		}
+	}
+}
+
+// The batches of phase 2 only bound the memory: whatever their size, the pairs,
+// their indices and their sites on the reference are the same.
+func TestAmpBudgetDoesNotChangeTheResult(t *testing.T) {
+	s := newSynthetic(10, -1, 5)
+	// non-targets sharing barcodes with targets, and counterexamples
+	for k, role := range []Role{Other, Other, CounterExample, Other, CounterExample} {
+		cp := append([]byte(nil), s.corpus.Seqs[k*2].Sequence()...)
+		if role == CounterExample {
+			for i := s.pos; i < s.pos+10; i++ { // no forward site: not amplified
+				cp[i] = "cgta"[strings.IndexByte("acgt", cp[i])]
+			}
+		}
+		s.corpus.Seqs = append(s.corpus.Seqs, obiseq.NewBioSequence(strings.Repeat("x", k+1), cp, ""))
+		s.corpus.Roles = append(s.corpus.Roles, role)
+		s.taxa.Of = append(s.taxa.Of, int32(100+k%3))
+	}
+	// a few unrelated sequences, with a taxon of their own or none
+	rng := rand.New(rand.NewSource(9))
+	for k := 0; k < 3; k++ {
+		s.corpus.Seqs = append(s.corpus.Seqs, obiseq.NewBioSequence("u"+strings.Repeat("y", k), []byte(strings.ToLower(randDNA(rng, 300))), ""))
+		s.corpus.Roles = append(s.corpus.Roles, Other)
+		s.taxa.Of = append(s.taxa.Of, int32(k-1))
+	}
+
+	run := func(budget int) []PairStats {
+		p := s.params()
+		p.MinSpecificity = 0 // keep every pair: the stress case of the batches
+		p.FalsePositive = 1
+		p.Reference = "s3"
+		p.AmpBudget = budget
+		pairs, info, err := Search(s.corpus, s.taxa, p)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if info.Pair.Candidates < 2 {
+			t.Fatalf("budget %d: only %d pairs reach phase 2b, the test needs several", budget, info.Pair.Candidates)
+		}
+		return pairs
+	}
+	want := run(0)
+	for _, budget := range []int{1, 7, 50, 1000} {
+		if got := run(budget); !reflect.DeepEqual(got, want) {
+			t.Errorf("budget %d changes the result:\n got %d pairs\nwant %d pairs", budget, len(got), len(want))
+		}
+	}
+}

pkg/obiprimer/pairs_eval.go

@@ -0,0 +1,211 @@
+package obiprimer
+
+import (
+	"cmp"
+	"fmt"
+	"math"
+	"slices"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obithermo"
+)
+
+// PairStats describes one primer pair and its quality indices. Primers are
+// written 5'->3' and ordered so that Primer1 < Primer2 (as in ecoPrimers).
+type PairStats struct {
+	Primer1, Primer2 string
+	Good1, Good2     bool // primer conserved in the examples and rare in the counterexamples
+
+	Examples        int // example sequences amplified
+	CounterExamples int
+	Yule            float32 // Examples/|E| - CounterExamples/|C|
+
+	ExampleTaxa    int     // target taxa amplified
+	CounterTaxa    int     // counterexample taxa amplified
+	Bc             float32 // coverage: ExampleTaxa over all the target taxa
+	Identified     int     // target taxa all of whose barcodes belong to them alone (see resolution.go)
+	Bs             float32 // specificity: Identified over ExampleTaxa
+	Bd             float32 // barcode discrimination: share of the target barcodes leading to a single taxon
+	MinLen, MaxLen int
+	MeanLen        float64
+	Tm1, Tm2       float64 // melting temperature of the primers against their perfect target
+	MinTm1, MinTm2 float64 // lowest melting temperature over the example amplicons
+	GC1, GC2       int
+
+	// With a reference sequence (-R): where the pair amplifies it. RefLocation
+	// is the GenBank location of the two primer sites (1-based, left to right),
+	// wrapped in complement() when Primer1 binds the other strand; RefAmplicon
+	// is the amplified region, primers excluded, read from Primer1 to Primer2.
+	// Both are empty when the pair does not amplify the reference.
+	RefLocation string
+	RefAmplicon string
+}
+
+// Score is the ranking criterion of ecoPrimers.
+func (p PairStats) Score() float32 { return p.Yule * p.Bs }
+
+// PairInfo summarizes the pair search, for reporting.
+type PairInfo struct {
+	Primers        int // conserved words kept as candidate primers
+	GoodPrimers    int // of which good ones
+	CandidatePairs int // pairs delimiting an amplicon in at least one sequence
+	RetainedPairs  int // pairs passing the sensitivity and false positive quorums, before the Bs filter
+	Candidates     int // of which pairs whose Bs upper bound reaches the threshold (their amplicons are gathered in full)
+}
+
+// FindPairs searches the primer pairs from the approximate occurrences of the
+// strict words (see MatchApat) and evaluates them.
+//
+// A pair is a pair of conserved primers, at least one of them good, that
+// delimit an amplicon of the allowed length. It is retained when it amplifies
+// more than Sensitivity of the examples and less than FalsePositive of the
+// counterexamples, and its specificity Bs reaches MinSpecificity. The result
+// is sorted by decreasing Yule*Bs.
+//
+// model computes the melting temperatures; nil leaves them NaN.
+func FindPairs(c *Corpus, taxa *Taxa, wm []WordMatches, p PairParams, model *obithermo.Model) ([]PairStats, PairInfo, error) {
+	var info PairInfo
+	if err := p.validate(); err != nil {
+		return nil, info, err
+	}
+	if taxa == nil || len(taxa.Of) != c.Len() {
+		return nil, info, fmt.Errorf("obiprimer: taxa do not match the corpus")
+	}
+	ps := selectPrimers(c, wm, p)
+	wm = nil // the occurrences are in ps now
+	info.Primers = len(ps.primers)
+	for _, pr := range ps.primers {
+		if pr.Good {
+			info.GoodPrimers++
+		}
+	}
+	if len(ps.primers) == 0 {
+		return nil, info, nil
+	}
+
+	// Phase 1: which pairs amplify enough examples and few enough counterexamples.
+	counts := c.Counts()
+	acc := ps.countPairs(c, p)
+	info.CandidatePairs = len(acc)
+	sens, fpq := float32(p.Sensitivity), float32(p.FalsePositive)
+	var kept []pairKey
+	for k, a := range acc {
+		q := float32(a.examples) / float32(counts.Examples)
+		qfp := float32(0)
+		if counts.CounterExamples > 0 {
+			qfp = float32(a.counters) / float32(counts.CounterExamples)
+		}
+		if q > sens && qfp < fpq {
+			kept = append(kept, k)
+		}
+	}
+	info.RetainedPairs = len(kept)
+	slices.SortFunc(kept, func(a, b pairKey) int {
+		return cmp.Or(cmp.Compare(a.p1, b.p1), cmp.Compare(a.p2, b.p2),
+			boolCmp(a.a1, b.a1), boolCmp(a.a2, b.a2))
+	})
+
+	// Phase 2a: the amplicons of the target sequences, which give the melting
+	// temperatures, the reference sites, and an upper bound of Bs. It is cheap
+	// and rejects most pairs on the specificity threshold.
+	first := ps.boundPairs(c, taxa, p, kept, model)
+	var survivors []int // indices in kept
+	for i := range kept {
+		if first.targets[i] > 0 && float32(first.identified[i])/float32(first.targets[i]) >= float32(p.MinSpecificity) {
+			survivors = append(survivors, i)
+		}
+	}
+	info.Candidates = len(survivors)
+
+	// Phase 2b: for the survivors only, the amplicons of every sequence, whatever
+	// its role: they decide which target taxa are identified.
+	results := ps.evaluatePairs(c, taxa, p, kept, survivors, acc, first, model)
+
+	var out []PairStats
+	for _, st := range results {
+		if st != nil {
+			out = append(out, *st)
+		}
+	}
+	slices.SortFunc(out, func(a, b PairStats) int {
+		return cmp.Or(-cmp.Compare(a.Score(), b.Score()), cmp.Compare(a.Primer1, b.Primer1), cmp.Compare(a.Primer2, b.Primer2))
+	})
+	return out, info, nil
+}
+
+func boolCmp(a, b bool) int {
+	switch {
+	case a == b:
+		return 0
+	case !a:
+		return -1
+	}
+	return 1
+}
+
+// evaluate puts the indices of one retained pair in a PairStats. It reports
+// false when the pair must be dropped (no target taxon amplified, or
+// specificity under the threshold).
+func (ps *primerSet) evaluate(taxa *Taxa, counts Counts, p PairParams, k pairKey, a *pairAcc, res resolution,
+	minTm1, minTm2 float32, model *obithermo.Model) (PairStats, bool) {
+
+	if res.targetTaxa == 0 {
+		return PairStats{}, false
+	}
+	st := PairStats{
+		Examples:        int(a.examples),
+		CounterExamples: int(a.counters),
+		ExampleTaxa:     res.targetTaxa,
+		CounterTaxa:     res.counterTaxa,
+		Identified:      res.identified,
+		MinLen:          int(a.minLen),
+		MaxLen:          int(a.maxLen),
+		MeanLen:         float64(a.sumLen) / float64(a.amplicons),
+	}
+	q := float32(a.examples) / float32(counts.Examples)
+	qfp := float32(0)
+	if counts.CounterExamples > 0 {
+		qfp = float32(a.counters) / float32(counts.CounterExamples)
+	}
+	st.Yule = q - qfp
+	st.Bc = float32(res.targetTaxa) / float32(taxa.ExampleTaxa)
+	st.Bs = float32(res.identified) / float32(res.targetTaxa)
+	st.Bd = float32(res.discriminant) / float32(res.barcodes)
+	if st.Bs < float32(p.MinSpecificity) {
+		return PairStats{}, false
+	}
+
+	return ps.finish(st, p, k, a, minTm1, minTm2, model), true
+}
+
+// finish adds the primers, their composition and melting temperatures, and puts
+// the two primers in string order.
+func (ps *primerSet) finish(st PairStats, p PairParams, k pairKey, a *pairAcc,
+	minTm1, minTm2 float32, model *obithermo.Model) PairStats {
+
+	lp := p.PrimerLength
+	st.Primer1 = orientedWord(ps.primers[k.p1].Word, lp, k.a1)
+	st.Primer2 = orientedWord(ps.primers[k.p2].Word, lp, k.a2)
+	st.Good1, st.Good2 = ps.primers[k.p1].Good, ps.primers[k.p2].Good
+	st.GC1, st.GC2 = obithermo.GCCount([]byte(st.Primer1)), obithermo.GCCount([]byte(st.Primer2))
+
+	st.Tm1, st.Tm2, st.MinTm1, st.MinTm2 = math.NaN(), math.NaN(), math.NaN(), math.NaN()
+	if model != nil {
+		st.Tm1, _ = model.SelfTm([]byte(st.Primer1))
+		st.Tm2, _ = model.SelfTm([]byte(st.Primer2))
+		if !math.IsInf(float64(minTm1), 1) {
+			st.MinTm1 = float64(minTm1)
+		}
+		if !math.IsInf(float64(minTm2), 1) {
+			st.MinTm2 = float64(minTm2)
+		}
+	}
+
+	if st.Primer2 < st.Primer1 {
+		st.Primer1, st.Primer2 = st.Primer2, st.Primer1
+		st.Good1, st.Good2 = st.Good2, st.Good1
+		st.GC1, st.GC2 = st.GC2, st.GC1
+		st.Tm1, st.Tm2 = st.Tm2, st.Tm1
+		st.MinTm1, st.MinTm2 = st.MinTm2, st.MinTm1
+	}
+	return st
+}

pkg/obiprimer/pairs_phase2.go

@@ -0,0 +1,160 @@
+package obiprimer
+
+import (
+	"sync"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obithermo"
+)
+
+// DefaultAmpBudget is the number of amplicon records (12 bytes each) phase 2
+// holds at once: 20 million, 240 MB. The pairs are processed in batches whose
+// amplicons fit in it. With the usual specificity threshold a search needs one
+// batch; with none (-T 0) no pair is rejected early and there are many more
+// amplicons than any memory could hold.
+const DefaultAmpBudget = 20_000_000
+
+func (p PairParams) release() {
+	if p.Release != nil {
+		p.Release()
+	}
+}
+
+func (p PairParams) ampBudget() int {
+	if p.AmpBudget > 0 {
+		return p.AmpBudget
+	}
+	return DefaultAmpBudget
+}
+
+// batchIndex maps the pairs kept[lo:hi] to their number in the batch.
+func batchIndex(kept []pairKey, lo, hi int) map[pairKey]int32 {
+	index := make(map[pairKey]int32, hi-lo)
+	for i := lo; i < hi; i++ {
+		index[kept[i]] = int32(i - lo)
+	}
+	return index
+}
+
+// parallelFor calls fn(i) for i in [0,n) on obidefault.ParallelWorkers() goroutines.
+func parallelFor(n int, fn func(worker, i int)) {
+	jobs := make(chan int)
+	var wg sync.WaitGroup
+	for w := range obidefault.ParallelWorkers() {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for i := range jobs {
+				fn(w, i)
+			}
+		}()
+	}
+	for i := 0; i < n; i++ {
+		jobs <- i
+	}
+	close(jobs)
+	wg.Wait()
+}
+
+// bound is what phase 2a learns about each retained pair (indexed like kept).
+type bound struct {
+	targets    []int // target taxa amplified
+	identified []int // upper bound of the target taxa identified (lenientIdentified)
+	minTm1     []float32
+	minTm2     []float32
+	refs       map[int32]refHit
+}
+
+// boundPairs is phase 2a: from the amplicons of the target sequences alone, it
+// gives each retained pair the upper bound of its Bs, the lowest melting
+// temperatures and its site on the reference. The pairs go by batches whose
+// amplicons fit in the budget; they are counted first, so that nothing is resized.
+func (ps *primerSet) boundPairs(c *Corpus, taxa *Taxa, p PairParams, kept []pairKey,
+	model *obithermo.Model) *bound {
+
+	plan := ps.planAmplicons(c, p, batchIndex(kept, 0, len(kept)), true)
+	res := &bound{
+		targets:    make([]int, len(kept)),
+		identified: make([]int, len(kept)),
+		minTm1:     make([]float32, len(kept)),
+		minTm2:     make([]float32, len(kept)),
+		refs:       map[int32]refHit{},
+	}
+	resolvers := make([]*resolver, obidefault.ParallelWorkers())
+	for i := range resolvers {
+		resolvers[i] = newResolver()
+	}
+
+	for _, b := range ampBatches(plan.total(), p.ampBudget()) {
+		lo, hi := b[0], b[1]
+		p.release() // the previous batch, or the stage before, is garbage
+		store := newAmpStore(plan, lo, hi)
+		sites := ps.collectAmplicons(c, taxa, p, batchIndex(kept, lo, hi), plan, store,
+			collectOptions{examplesOnly: true, model: model, reference: true})
+		store.check()
+		parallelFor(hi-lo, func(w, i int) {
+			res.targets[lo+i], res.identified[lo+i] = resolvers[w].lenientIdentified(store.pair(i))
+		})
+		copy(res.minTm1[lo:hi], sites.minTm1)
+		copy(res.minTm2[lo:hi], sites.minTm2)
+		for i, h := range sites.refs {
+			res.refs[int32(lo)+i] = h
+		}
+	}
+	return res
+}
+
+// evaluatePairs is phase 2b: for the surviving pairs (indices in kept), it
+// gathers the amplicons of every sequence, whatever its role, by batches whose
+// amplicons fit in the budget, and evaluates the pairs. The result is indexed
+// like survivors; a nil entry is a pair dropped by the specificity threshold.
+func (ps *primerSet) evaluatePairs(c *Corpus, taxa *Taxa, p PairParams, kept []pairKey, survivors []int,
+	acc map[pairKey]*pairAcc, first *bound, model *obithermo.Model) []*PairStats {
+
+	results := make([]*PairStats, len(survivors))
+	if len(survivors) == 0 {
+		return results
+	}
+	keys := make([]pairKey, len(survivors))
+	for j, i := range survivors {
+		keys[j] = kept[i]
+	}
+	// the amplicons of every sequence are counted first: the store is allocated
+	// at its exact size, and the batches are cut on the exact counts
+	plan := ps.planAmplicons(c, p, batchIndex(keys, 0, len(keys)), false)
+
+	counts := c.Counts()
+	var refBases []byte
+	if p.HasReference {
+		refBases, _ = c.bases(p.Reference, nil)
+	}
+	resolvers := make([]*resolver, obidefault.ParallelWorkers())
+	for i := range resolvers {
+		resolvers[i] = newResolver()
+	}
+	for _, b := range ampBatches(plan.total(), p.ampBudget()) {
+		lo, hi := b[0], b[1]
+		p.release()
+		store := newAmpStore(plan, lo, hi)
+		ps.collectAmplicons(c, taxa, p, batchIndex(keys, lo, hi), plan, store, collectOptions{})
+		store.check()
+		parallelFor(hi-lo, func(w, n int) {
+			j := lo + n
+			i := survivors[j]
+			k := keys[j]
+			st, ok := ps.evaluate(taxa, counts, p, k, acc[k], resolvers[w].resolve(store.pair(n)),
+				first.minTm1[i], first.minTm2[i], model)
+			if !ok {
+				return
+			}
+			if h, found := first.refs[int32(i)]; found {
+				// evaluate orders the primers by string; the amplicon is read from the first one
+				s1 := orientedWord(ps.primers[k.p1].Word, p.PrimerLength, k.a1)
+				s2 := orientedWord(ps.primers[k.p2].Word, p.PrimerLength, k.a2)
+				st.RefLocation, st.RefAmplicon = referenceSites(refBases, h, p, s2 < s1)
+			}
+			results[j] = &st
+		})
+	}
+	return results
+}

pkg/obiprimer/pairs_test.go

@@ -0,0 +1,240 @@
+package obiprimer
+
+import (
+	"bufio"
+	"fmt"
+	"math"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiformats"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obikmer"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obithermo"
+)
+
+// ecoRow is one result line of ecoPrimers.
+type ecoRow struct {
+	p1, p2         string
+	tm1, mtm1      float64
+	tm2, mtm2      float64
+	gc1, gc2       int
+	good           string
+	in, out        int
+	yule           float64
+	intaxa, outtax int
+	bc             float64
+	well           int
+	bs             float64
+	mind, maxd     int
+	avg            float64
+}
+
+func readEcoRows(t *testing.T, path string) []ecoRow {
+	t.Helper()
+	f, err := os.Open(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	var rows []ecoRow
+	sc := bufio.NewScanner(f)
+	sc.Buffer(make([]byte, 1<<20), 1<<20)
+	for sc.Scan() {
+		line := sc.Text()
+		if strings.HasPrefix(line, "#") || strings.TrimSpace(line) == "" {
+			continue
+		}
+		f := strings.Split(line, "\t")
+		fl := func(i int) float64 {
+			v, err := strconv.ParseFloat(strings.TrimSpace(f[i]), 64)
+			if err != nil {
+				t.Fatalf("field %d of %q: %v", i, line, err)
+			}
+			return v
+		}
+		in := func(i int) int { return int(fl(i)) }
+		rows = append(rows, ecoRow{
+			p1: f[1], p2: f[2], tm1: fl(3), mtm1: fl(4), tm2: fl(5), mtm2: fl(6),
+			gc1: in(7), gc2: in(8), good: f[9], in: in(10), out: in(11), yule: fl(12),
+			intaxa: in(13), outtax: in(14), bc: fl(15), well: in(16), bs: fl(17),
+			mind: in(18), maxd: in(19), avg: fl(20),
+		})
+	}
+	if err := sc.Err(); err != nil {
+		t.Fatal(err)
+	}
+	return rows
+}
+
+// Opt-in end-to-end comparison with the complete ecoPrimers result tables of
+// the tutorial: same parameters (-e 3 -3 2 -l 30 -L 150 -c, -r taxid).
+//
+// Bs and the number of identified taxa are NOT compared: obiprimers defines
+// them differently (a taxon is identified only if all its barcodes are its own,
+// and every amplified sequence counts, targets or not), so they are lower. The
+// specificity threshold is switched off so that the pairs ecoPrimers dropped
+// with its own Bs are still produced, and only the pairs of ecoPrimers are
+// checked. What is checked: that every pair of ecoPrimers is found, with the
+// same good flags, sequence and taxon counts, Yule, Bc, lengths, GC, Tm and
+// lowest Tm; and that identified taxa never exceed those of ecoPrimers' more
+// lenient rule on the same amplified sequences.
+//
+// Cyprinidae is reproduced exactly on these columns for its 3448 pairs.
+// Teleostei finds the same 407 pairs, and 6 of them count one
+// sequence less than ecoPrimers (NC_033548, whose downstream site is
+// NGGGGAGGCAAGTCGTAA). The 3' check of ecoPrimers reads an N through the
+// shift register of extractSite, where the code 4 of N spills on the next base
+// and ~4&3 reads as T, so an N can be accepted by accident. Emulating that
+// register exactly in the 3' check was verified to reproduce both tables on
+// every column; it is not kept: an ambiguous base cannot be a verified 3' match.
+func TestPairsAgainstEcoPrimers(t *testing.T) {
+	dir := os.Getenv("OBIPRIMERS_TUTORIAL")
+	if dir == "" {
+		t.Skip("OBIPRIMERS_TUTORIAL not set")
+	}
+	taxonomy, err := obiformats.LoadTaxonomy(filepath.Join(dir, "ncbitaxo_20250211.tgz"), false, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, tc := range []struct {
+		name, taxid string
+		species     int
+		divergent   int
+	}{{"Cyprinidae", "7953", 316, 0}, {"Teleostei", "32443", 3876, 6}} {
+		t.Run(tc.name, func(t *testing.T) {
+			var stages []string
+			tick := time.Now()
+			lap := func(name string) {
+				stages = append(stages, fmt.Sprintf("%s %s", name, time.Since(tick).Round(time.Millisecond)))
+				tick = time.Now()
+			}
+			restrict, err := BelongingTo(taxonomy, []string{tc.taxid})
+			if err != nil {
+				t.Fatal(err)
+			}
+			seqs, err := obiformats.ReadSequencesFromFile(filepath.Join(dir, "mito.vert.fasta.gz"))
+			if err != nil {
+				t.Fatal(err)
+			}
+			corpus, err := LoadCorpus(seqs, Selection{Restrict: restrict})
+			if err != nil {
+				t.Fatal(err)
+			}
+			lap("read+select")
+			taxa, err := corpus.AssignTaxa(taxonomy, "species")
+			if err != nil {
+				t.Fatal(err)
+			}
+			lap("taxa")
+			if taxa.ExampleTaxa != tc.species {
+				t.Errorf("%d example species, ecoPrimers reports %d", taxa.ExampleTaxa, tc.species)
+			}
+
+			sw, err := FindStrictWords(corpus, WordSearch{PrimerLength: 18, Quorum: 0.7, Circular: true, TmpDir: t.TempDir()})
+			if err != nil {
+				t.Fatal(err)
+			}
+			lap("strict words")
+			words := make([]uint64, len(sw.Words))
+			for i, w := range sw.Words {
+				words[i] = w.Kmer
+			}
+			wm, err := MatchWords(corpus, words, MatchParams{K: 18, MaxErrors: 3, Circular: true})
+			if err != nil {
+				t.Fatal(err)
+			}
+			lap("approximate search")
+			got, _, err := FindPairs(corpus, taxa, wm, PairParams{
+				PrimerLength: 18, MinLength: 30, MaxLength: 150, ThreePrime: 2,
+				Sensitivity: 0.9, FalsePositive: 0.1, MinSpecificity: 0, Circular: true,
+			}, obithermo.DefaultModel())
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			lap("pairs+indices+Tm")
+			var ms runtime.MemStats
+			runtime.ReadMemStats(&ms)
+			t.Logf("stages: %s; Go heap in use at the end %.0f MB (includes the taxonomy)", strings.Join(stages, ", "), float64(ms.HeapAlloc)/1e6)
+
+			want := readEcoRows(t, filepath.Join(dir, tc.name+".ecoprimers"))
+			t.Logf("%d pairs, ecoPrimers %d", len(got), len(want))
+
+			byKey := map[string]PairStats{}
+			for _, g := range got {
+				byKey[g.Primer1+"/"+g.Primer2] = g
+			}
+			bad := map[string]int{}
+			missing, divergent := 0, 0
+			note := func(field string, w ecoRow, format string, args ...any) {
+				if bad[field]++; bad[field] <= 3 {
+					t.Errorf("%s/%s %s: "+format, append([]any{w.p1, w.p2, field}, args...)...)
+				}
+			}
+			near := func(a, b, tol float64) bool { return math.Abs(a-b) <= tol }
+			for _, w := range want {
+				g, ok := byKey[w.p1+"/"+w.p2]
+				if !ok {
+					if missing++; missing <= 5 {
+						t.Errorf("ecoPrimers pair %s/%s not found", w.p1, w.p2)
+					}
+					continue
+				}
+				delete(byKey, w.p1+"/"+w.p2)
+				good := string("bG"[b2i(g.Good1)]) + string("bG"[b2i(g.Good2)])
+				if good != w.good {
+					note("good", w, "got %s want %s", good, w.good)
+				}
+				// Known divergence, see the comment of the test: ecoPrimers accepts one
+				// amplicon (NC_033548) whose 3' primer site ends with an N. We reject it,
+				// so the counts can be exactly one lower and nothing else may differ.
+				lower := func(got, want int) bool { return got == want || got == want-1 }
+				diverges := g.Examples != w.in || g.ExampleTaxa != w.intaxa
+				if diverges {
+					divergent++
+				}
+				if !lower(g.Examples, w.in) || g.CounterExamples != w.out {
+					note("sequences", w, "got %d/%d want %d/%d", g.Examples, g.CounterExamples, w.in, w.out)
+				}
+				if !lower(g.ExampleTaxa, w.intaxa) || g.CounterTaxa != w.outtax {
+					note("taxa", w, "got %d/%d want %d/%d", g.ExampleTaxa, g.CounterTaxa, w.intaxa, w.outtax)
+				}
+				if !diverges && g.Identified > w.well {
+					note("identified", w, "got %d, more than the %d of ecoPrimers' lenient rule", g.Identified, w.well)
+				}
+				tol := 0.0006
+				if diverges {
+					tol = 0.0021
+				}
+				if !near(float64(g.Yule), w.yule, tol) || !near(float64(g.Bc), w.bc, tol) {
+					note("indices", w, "yule/bc got %.4f/%.4f want %.3f/%.3f", g.Yule, g.Bc, w.yule, w.bc)
+				}
+				if g.MinLen != w.mind || g.MaxLen != w.maxd || !near(g.MeanLen, w.avg, 0.006) {
+					note("lengths", w, "got %d/%d/%.2f want %d/%d/%.2f", g.MinLen, g.MaxLen, g.MeanLen, w.mind, w.maxd, w.avg)
+				}
+				if g.GC1 != w.gc1 || g.GC2 != w.gc2 {
+					note("gc", w, "got %d/%d want %d/%d", g.GC1, g.GC2, w.gc1, w.gc2)
+				}
+				// ecoPrimers prints Kelvin-273.0; obithermo uses 273.15
+				if !near(g.Tm1+0.15, w.tm1, 0.051) || !near(g.Tm2+0.15, w.tm2, 0.051) {
+					note("tm", w, "got %.2f/%.2f want %.1f/%.1f", g.Tm1+0.15, g.Tm2+0.15, w.tm1, w.tm2)
+				}
+				if !near(g.MinTm1+0.15, w.mtm1, 0.051) || !near(g.MinTm2+0.15, w.mtm2, 0.051) {
+					note("mintm", w, "got %.2f/%.2f want %.1f/%.1f", g.MinTm1+0.15, g.MinTm2+0.15, w.mtm1, w.mtm2)
+				}
+			}
+			t.Logf("missing from ours: %d; pairs of ours that ecoPrimers dropped with its Bs: %d; mismatching fields: %v; pairs one sequence below ecoPrimers: %d",
+				missing, len(byKey), bad, divergent)
+			if divergent != tc.divergent {
+				t.Errorf("%d pairs differ from ecoPrimers by one sequence, expected %d", divergent, tc.divergent)
+			}
+		})
+	}
+	_ = obikmer.DecodeKmer
+}

pkg/obiprimer/reference.go

@@ -0,0 +1,44 @@
+package obiprimer
+
+import (
+	"fmt"
+	"strings"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+// span writes the 1-based GenBank range of the lp bases starting at the 0-based
+// position pos. On a circular sequence a range running over the junction is
+// split in two.
+func span(pos, lp, n int, circular bool) string {
+	end := pos + lp
+	if end > n && circular {
+		return fmt.Sprintf("%d..%d,1..%d", pos+1, n, end-n)
+	}
+	return fmt.Sprintf("%d..%d", pos+1, end)
+}
+
+// referenceSites describes an amplicon of a pair on the reference sequence raw.
+//
+// The location lists the two primer sites left to right as join(a..b,c..d),
+// inside complement() when the first primer of the pair (Primer1 once the pair
+// is in string order) binds the reverse strand. The amplicon is the region
+// between the sites, read 5'->3' from the first primer, in lower case.
+//
+// swappedByOrder says whether putting the pair in string order exchanged the
+// two primers of the key.
+func referenceSites(raw []byte, h refHit, p PairParams, swappedByOrder bool) (location, amplicon string) {
+	lp, n := p.PrimerLength, len(raw)
+
+	// The key slot 1 is the upstream primer unless the match order was swapped;
+	// the exchange due to the string order flips it again.
+	firstIsUpstream := h.swapped == swappedByOrder
+
+	location = "join(" + span(int(h.up.pos()), lp, n, p.Circular) + "," + span(int(h.down.pos()), lp, n, p.Circular) + ")"
+	region := []byte(strings.ToLower(string(raw[int(h.up.pos())+lp : int(h.down.pos())])))
+	if firstIsUpstream {
+		return location, string(region)
+	}
+	rc := obiseq.NewBioSequence("", region, "").ReverseComplement(true)
+	return "complement(" + location + ")", string(rc.Sequence())
+}

pkg/obiprimer/resolution.go

@@ -0,0 +1,154 @@
+package obiprimer
+
+// The three quality indices of a pair, over the sequences it amplifies.
+//
+// Every sequence the pair amplifies gives a barcode (the amplicon between the
+// primers); all of them are taken into account, whatever their role: examples
+// (the targets), counterexamples and non-targets.
+//
+//   - Bc, coverage: target taxa amplified / target taxa.
+//   - Bs, specificity: target taxa identified / target taxa amplified. A target
+//     taxon is identified when ALL its barcodes belong to it alone: as soon as a
+//     barcode is shared with another taxon, target or not, counterexample or
+//     not, both taxa are badly identified.
+//   - Bd, barcode discrimination: the share of the barcodes carried by the
+//     targets that lead without ambiguity to one taxon.
+//
+// A sequence with no taxon at the chosen rank is an anonymous taxon, distinct
+// from every other: a barcode it shares with a taxon is ambiguous.
+
+// barcodeState is what is known of one barcode while the amplicons are scanned.
+type barcodeState struct {
+	taxon int32 // 1 + the first taxon met, 0 while none
+	flags uint8
+}
+
+const (
+	barcodeAmbiguous = 1 << iota // two distinct taxa carry it
+	barcodeAnonymous             // a sequence with no taxon carries it
+	barcodeTarget                // a target sequence with a taxon carries it
+)
+
+// ambiguous reports whether the barcode does not lead to a single taxon.
+func (b barcodeState) ambiguous() bool {
+	return b.flags&barcodeAmbiguous != 0 || (b.flags&barcodeAnonymous != 0 && b.taxon != 0)
+}
+
+// resolution is the outcome of the identification of the taxa of one pair.
+type resolution struct {
+	targetTaxa   int // target taxa amplified
+	counterTaxa  int // counterexample taxa amplified
+	identified   int // target taxa whose barcodes all belong to them alone
+	barcodes     int // distinct barcodes carried by the targets
+	discriminant int // of which lead to a single taxon
+}
+
+// resolver computes the indices of pairs one after the other, reusing its maps
+// (a pair needs a few thousand entries, and there are thousands of pairs).
+type resolver struct {
+	state    map[uint64]barcodeState
+	targets  map[int32]struct{}
+	counters map[int32]struct{}
+	bad      map[int32]struct{}
+	owner    map[uint64]int32
+}
+
+func newResolver() *resolver {
+	return &resolver{
+		state:    map[uint64]barcodeState{},
+		targets:  map[int32]struct{}{},
+		counters: map[int32]struct{}{},
+		bad:      map[int32]struct{}{},
+		owner:    map[uint64]int32{},
+	}
+}
+
+// resolve computes the resolution of one pair from its amplicons, in any order.
+func resolve(a ampSlice) resolution { return newResolver().resolve(a) }
+
+func (rs *resolver) resolve(a ampSlice) resolution {
+	var res resolution
+	clear(rs.state)
+	clear(rs.targets)
+	clear(rs.counters)
+	clear(rs.bad)
+
+	for i, h := range a.hash {
+		taxon, role := unpackTaxon(a.tax[i])
+		st := rs.state[h]
+		switch {
+		case taxon < 0:
+			st.flags |= barcodeAnonymous
+		case st.taxon == 0:
+			st.taxon = taxon + 1
+		case st.taxon != taxon+1:
+			st.flags |= barcodeAmbiguous
+		}
+		if taxon >= 0 {
+			switch role {
+			case Example:
+				st.flags |= barcodeTarget
+				rs.targets[taxon] = struct{}{}
+			case CounterExample:
+				rs.counters[taxon] = struct{}{}
+			}
+		}
+		rs.state[h] = st
+	}
+
+	for i, h := range a.hash {
+		taxon, role := unpackTaxon(a.tax[i])
+		if role == Example && taxon >= 0 && rs.state[h].ambiguous() {
+			rs.bad[taxon] = struct{}{}
+		}
+	}
+
+	for _, st := range rs.state {
+		if st.flags&barcodeTarget != 0 {
+			res.barcodes++
+			if !st.ambiguous() {
+				res.discriminant++
+			}
+		}
+	}
+	res.targetTaxa, res.counterTaxa = len(rs.targets), len(rs.counters)
+	res.identified = len(rs.targets) - len(rs.bad)
+	return res
+}
+
+// lenientIdentified is an upper bound of resolution.identified, computed on
+// the target sequences alone and with the lenient rule of ecoPrimers (a taxon
+// counts as soon as ONE of its barcodes belongs to it alone, among the targets).
+//
+// A taxon identified by resolve has all its barcodes alone among every amplified
+// sequence, hence among the targets: it has at least one such barcode and is
+// counted here too. So identified <= the second result, with the same number of
+// target taxa, which lets a pair be rejected on the specificity threshold without
+// gathering the amplicons of the non-target sequences.
+func lenientIdentified(a ampSlice) (targetTaxa, identified int) {
+	return newResolver().lenientIdentified(a)
+}
+
+func (rs *resolver) lenientIdentified(a ampSlice) (targetTaxa, identified int) {
+	clear(rs.owner) // barcode -> its taxon, or -1 when shared
+	clear(rs.targets)
+	for i, h := range a.hash {
+		taxon, role := unpackTaxon(a.tax[i])
+		if role != Example || taxon < 0 {
+			continue
+		}
+		rs.targets[taxon] = struct{}{}
+		if prev, seen := rs.owner[h]; !seen {
+			rs.owner[h] = taxon
+		} else if prev != taxon {
+			rs.owner[h] = -1
+		}
+	}
+	clear(rs.bad) // here: the taxa owning a barcode
+	for _, t := range rs.owner {
+		if t >= 0 {
+			rs.bad[t] = struct{}{}
+		}
+	}
+	return len(rs.targets), len(rs.bad)
+}

pkg/obiprimer/resolution_test.go

@@ -0,0 +1,165 @@
+package obiprimer
+
+import (
+	"math/rand"
+	"testing"
+)
+
+// rec builds an amplicon record: barcode, taxon (-1: none at the rank), role.
+// pack builds an ampSlice from logical records.
+func pack(parts ...[]ampRecord) ampSlice {
+	var a ampSlice
+	for _, part := range parts {
+		for _, r := range part {
+			a.hash = append(a.hash, r.hash)
+			a.tax = append(a.tax, packTaxon(r.taxon, r.role))
+		}
+	}
+	return a
+}
+
+func rec(barcode uint64, taxon int32, role Role) ampRecord {
+	return ampRecord{hash: barcode, taxon: taxon, role: role}
+}
+
+func TestResolutionUserExample(t *testing.T) {
+	// A has the barcodes A1 and A2, B has A2 and B1. A2 is shared: neither A
+	// nor B is identified, but A1 and B1 lead to one species each.
+	const A, B = 0, 1
+	const A1, A2, B1 = 11, 12, 21
+	res := resolve(pack([]ampRecord{
+		rec(A1, A, Example), rec(A2, A, Example),
+		rec(A2, B, Example), rec(B1, B, Example),
+	}))
+	want := resolution{targetTaxa: 2, identified: 0, barcodes: 3, discriminant: 2}
+	if res != want {
+		t.Fatalf("got %+v, want %+v", res, want)
+	}
+}
+
+func TestResolutionAllUnique(t *testing.T) {
+	res := resolve(pack([]ampRecord{rec(1, 0, Example), rec(2, 1, Example), rec(3, 2, Example)}))
+	if res.identified != 3 || res.discriminant != 3 || res.barcodes != 3 || res.targetTaxa != 3 {
+		t.Errorf("all barcodes distinct: %+v", res)
+	}
+}
+
+func TestResolutionSameTaxonSharesItsBarcodeFreely(t *testing.T) {
+	// several sequences of one species with the same barcode are not ambiguous,
+	// and several barcodes for one species are fine when none is shared
+	res := resolve(pack([]ampRecord{
+		rec(1, 0, Example), rec(1, 0, Example), rec(1, 0, Example),
+		rec(2, 0, Example),
+	}))
+	if res.identified != 1 || res.barcodes != 2 || res.discriminant != 2 {
+		t.Errorf("%+v", res)
+	}
+}
+
+// The barcode of a target that is unique among the targets but shared with a
+// non-target, or with a counterexample, does not identify it.
+func TestResolutionNonTargetsCount(t *testing.T) {
+	for _, role := range []Role{Other, CounterExample} {
+		res := resolve(pack([]ampRecord{
+			rec(1, 0, Example), // unique among the targets...
+			rec(1, 5, role),    // ...but shared with a non-target taxon
+			rec(2, 1, Example), // identified
+			rec(3, 6, role),    // a barcode of non-targets only
+		}))
+		if res.targetTaxa != 2 || res.identified != 1 {
+			t.Errorf("role %d: %+v, want 2 target taxa and 1 identified", role, res)
+		}
+		// barcodes carried by the targets: 1 (ambiguous) and 2; barcode 3 leads to a non-target
+		if res.barcodes != 2 || res.discriminant != 1 {
+			t.Errorf("role %d: Bd counts %d/%d, want 1/2", role, res.discriminant, res.barcodes)
+		}
+	}
+}
+
+func TestResolutionAmplifiedTaxaCounts(t *testing.T) {
+	res := resolve(pack([]ampRecord{
+		rec(1, 0, Example), rec(2, 1, CounterExample), rec(3, 1, CounterExample),
+		rec(4, 2, CounterExample), rec(5, 3, Other),
+	}))
+	if res.targetTaxa != 1 || res.counterTaxa != 2 {
+		t.Errorf("target taxa %d (want 1), counterexample taxa %d (want 2: others are not counted)",
+			res.targetTaxa, res.counterTaxa)
+	}
+}
+
+// A sequence with no taxon at the rank is an anonymous taxon, distinct from all
+// the others: a barcode shared with it is ambiguous.
+func TestResolutionAnonymousTaxon(t *testing.T) {
+	res := resolve(pack([]ampRecord{
+		rec(1, 0, Example), rec(1, -1, Other), // shared with an unknown: ambiguous
+		rec(2, 1, Example),
+	}))
+	if res.identified != 1 || res.barcodes != 2 || res.discriminant != 1 {
+		t.Errorf("target sharing with an anonymous sequence: %+v", res)
+	}
+	// two anonymous sequences with the same barcode never count: no target carries it
+	res = resolve(pack([]ampRecord{rec(7, -1, Other), rec(7, -1, Other), rec(8, 0, Example)}))
+	if res.barcodes != 1 || res.discriminant != 1 || res.identified != 1 {
+		t.Errorf("anonymous barcodes alone: %+v", res)
+	}
+	// a target with no taxon is not a target taxon, and is anonymous too
+	res = resolve(pack([]ampRecord{rec(1, -1, Example), rec(1, 0, Example), rec(2, 0, Example)}))
+	if res.targetTaxa != 1 || res.identified != 0 {
+		t.Errorf("an anonymous target sequence sharing a barcode must make the taxon ambiguous: %+v", res)
+	}
+}
+
+func TestResolutionSeveralPartsAndOrder(t *testing.T) {
+	// the amplicons come as one slice per worker, in any order
+	a := []ampRecord{rec(1, 0, Example), rec(2, 1, Example)}
+	b := []ampRecord{rec(2, 2, Other), rec(3, 1, Example)}
+	if res := resolve(pack(a, b)); res.targetTaxa != 2 || res.identified != 1 {
+		t.Errorf("taxon 1 shares barcode 2 with taxon 2: %+v", res)
+	}
+	if res := resolve(pack(b, a)); res.targetTaxa != 2 || res.identified != 1 {
+		t.Errorf("order must not matter: %+v", res)
+	}
+}
+
+// ecoPrimers' lenient rule on the user's example: A and B each own one barcode,
+// so both count (the new rule counts none), and the bound is what lets a pair
+// be rejected early.
+func TestLenientIdentifiedUserExample(t *testing.T) {
+	recs := []ampRecord{
+		rec(11, 0, Example), rec(12, 0, Example),
+		rec(12, 1, Example), rec(21, 1, Example),
+	}
+	if targets, identified := lenientIdentified(pack(recs)); targets != 2 || identified != 2 {
+		t.Errorf("lenient rule: %d of %d, want 2 of 2", identified, targets)
+	}
+	if res := resolve(pack(recs)); res.identified != 0 {
+		t.Errorf("strict rule: %d identified, want 0", res.identified)
+	}
+}
+
+// The pruning of phase 2 is only exact if the bound never falls under the
+// true count, with the same number of target taxa.
+func TestLenientIdentifiedIsAnUpperBound(t *testing.T) {
+	rng := rand.New(rand.NewSource(3))
+	for trial := 0; trial < 3000; trial++ {
+		var recs []ampRecord
+		n := 1 + rng.Intn(40)
+		for range n {
+			role := []Role{Example, Example, Example, CounterExample, Other}[rng.Intn(5)]
+			recs = append(recs, rec(uint64(rng.Intn(12)), int32(rng.Intn(8))-1, role)) // taxon -1: none
+		}
+		// split in a few parts, as the workers do
+		cut := rng.Intn(len(recs) + 1)
+		res := resolve(pack(recs[:cut], recs[cut:]))
+		targets, lenient := lenientIdentified(pack(recs[:cut], recs[cut:]))
+		if targets != res.targetTaxa {
+			t.Fatalf("trial %d: %d target taxa for the bound, %d for resolve", trial, targets, res.targetTaxa)
+		}
+		if res.identified > lenient {
+			t.Fatalf("trial %d: identified %d exceeds the bound %d for %v", trial, res.identified, lenient, recs)
+		}
+		if res.discriminant > res.barcodes || res.identified > res.targetTaxa {
+			t.Fatalf("trial %d: inconsistent %+v", trial, res)
+		}
+	}
+}

pkg/obiprimer/roles.go

@@ -0,0 +1,79 @@
+// Package obiprimer implements the core of obiprimers, the OBITools4
+// re-implementation of ecoPrimers: finding conserved words usable as primers
+// in a set of example sequences, and the primer pairs they form.
+package obiprimer
+
+import (
+	"fmt"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax"
+)
+
+// Role tells how a sequence takes part in the primer search.
+type Role uint8
+
+const (
+	// Other sequences are non-targets: nothing is required of them, so they
+	// count neither in the sensitivity nor in the false positive quorums, but
+	// the amplicons they give are taken into account to decide whether a target
+	// is identified (see Bs and Bd).
+	Other Role = iota
+	// Example sequences are the targets: the primers must amplify them.
+	Example
+	// CounterExample sequences are those the primers must not amplify.
+	CounterExample
+)
+
+// Selection defines the roles from the taxonomy, with the ecoPrimers
+// semantics of -r (restrict), -i (ignore) and -E (exception):
+//
+//   - a sequence is an Example when it belongs to Restrict (all sequences when
+//     Restrict is nil) and not to Except;
+//   - otherwise it is a CounterExample when it belongs to Ignore or to Except;
+//   - otherwise it is Other.
+//
+// Nil predicates mean "no such constraint".
+type Selection struct {
+	Restrict obiseq.SequencePredicate
+	Ignore   obiseq.SequencePredicate
+	Except   obiseq.SequencePredicate
+}
+
+// Role returns the role of seq.
+func (s Selection) Role(seq *obiseq.BioSequence) Role {
+	inRestrict := s.Restrict == nil || s.Restrict(seq)
+	inExcept := s.Except != nil && s.Except(seq)
+
+	switch {
+	case inRestrict && !inExcept:
+		return Example
+	case inExcept, s.Ignore != nil && s.Ignore(seq):
+		return CounterExample
+	default:
+		return Other
+	}
+}
+
+// BelongingTo returns a predicate true for the sequences whose taxon is a
+// sub-clade of at least one of the given taxids. It returns nil (no
+// constraint) when taxids is empty.
+func BelongingTo(taxonomy *obitax.Taxonomy, taxids []string) (obiseq.SequencePredicate, error) {
+	var pred obiseq.SequencePredicate
+	for _, id := range taxids {
+		taxon, _, err := taxonomy.Taxon(id)
+		if err != nil {
+			return nil, fmt.Errorf("obiprimer: taxid %q: %w", id, err)
+		}
+		if taxon == nil {
+			return nil, fmt.Errorf("obiprimer: taxid %q is unknown from the taxonomy", id)
+		}
+		p := obiseq.IsSubCladeOf(taxonomy, taxon)
+		if pred == nil {
+			pred = p
+		} else {
+			pred = pred.Or(p)
+		}
+	}
+	return pred, nil
+}

pkg/obiprimer/search.go

@@ -0,0 +1,123 @@
+package obiprimer
+
+import (
+	"fmt"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obithermo"
+)
+
+// SearchParams gathers every parameter of the primer pair search, named after
+// the ecoPrimers options they come from.
+type SearchParams struct {
+	PrimerLength   int     // -O: size of the primers (default 18)
+	MaxErrors      int     // -e: substitutions allowed between a primer and its sites (default 3)
+	MinLength      int     // -l: the amplicon (primers excluded) must be longer than this
+	MaxLength      int     // -L: and shorter than this
+	ThreePrime     int     // -3: number of 3' bases that must match exactly
+	StrictQuorum   float64 // -q: fraction of the examples containing the exact word (0.7)
+	Sensitivity    float64 // -s: fraction of the examples a primer must match (0.9)
+	FalsePositive  float64 // -x: fraction of the counterexamples it may match (0.1)
+	MinSpecificity float64 // -T: pairs with a lower Bs are dropped (0.6)
+	Circular       bool    // -c: sequences are circular
+	NoMultiMatch   bool    // -U: discard the primers matching several times in a sequence
+	Reference      string  // -R: identifier of an example sequence on which the amplicons are located ("" = none)
+
+	// Release, if not nil, is called when a stage has just freed a lot of memory
+	// and the next one is about to take some: the command makes the runtime hand
+	// it back to the system then, which the garbage collector would only do
+	// slowly, and the peak of memory would be the sum of the stages.
+	Release func()
+
+	AmpBudget int // amplicon records held at once by the pair evaluation (0 = DefaultAmpBudget)
+
+	Partitions int // partitions of the k-mer counter (default 256)
+	TmpDir     string
+
+	Model *obithermo.Model // melting temperatures; nil leaves them NaN
+}
+
+// DefaultSearchParams returns the ecoPrimers defaults.
+func DefaultSearchParams() SearchParams {
+	return SearchParams{
+		PrimerLength: 18, MaxErrors: 3, MinLength: 0, MaxLength: 1000,
+		StrictQuorum: 0.7, Sensitivity: 0.9, FalsePositive: 0.1, MinSpecificity: 0.6,
+		Model: obithermo.DefaultModel(),
+	}
+}
+
+// SearchInfo reports the size of every stage of a search.
+type SearchInfo struct {
+	Counts      Counts
+	StrictWords int // words found without error in at least StrictQuorum of the examples
+	MinStrict   int // that quorum, in sequences
+	Matches     int // approximate occurrences of those words
+	Repeated    int // strict words dropped because repeated in an example (-U)
+	MultiMatch  int // words dropped because matching several times in a sequence (-U)
+	Pair        PairInfo
+}
+
+// Search runs the whole ecoPrimers pipeline on a corpus: strict words,
+// approximate occurrences, pairs and their indices. The pairs are sorted by
+// decreasing Yule*Bs.
+func Search(c *Corpus, taxa *Taxa, p SearchParams) ([]PairStats, *SearchInfo, error) {
+	if p.MaxErrors < 0 {
+		return nil, nil, fmt.Errorf("obiprimer: the allowed mismatch count cannot be negative")
+	}
+
+	sw, err := FindStrictWords(c, WordSearch{
+		PrimerLength: p.PrimerLength,
+		Quorum:       p.StrictQuorum,
+		Circular:     p.Circular,
+		Partitions:   p.Partitions,
+		TmpDir:       p.TmpDir,
+	})
+	if err != nil {
+		return nil, nil, err
+	}
+	info := &SearchInfo{Counts: sw.Counts, StrictWords: len(sw.Words), MinStrict: sw.MinSequences}
+	if p.NoMultiMatch {
+		sw.Words, info.Repeated = DropRepeatedWords(sw.Words)
+	}
+
+	refIndex := -1
+	if p.Reference != "" {
+		if refIndex, err = c.IndexOfExample(p.Reference); err != nil {
+			return nil, info, err
+		}
+	}
+
+	words := make([]uint64, len(sw.Words))
+	for i, w := range sw.Words {
+		words[i] = w.Kmer
+	}
+	if p.Release != nil {
+		p.Release() // the strict word search has just freed its occurrences
+	}
+	wm, err := MatchWords(c, words, MatchParams{K: p.PrimerLength, MaxErrors: p.MaxErrors, Circular: p.Circular})
+	if err != nil {
+		return nil, info, err
+	}
+	if p.NoMultiMatch {
+		wm, info.MultiMatch = DropMultiMatch(wm, c)
+	}
+	for _, w := range wm {
+		info.Matches += len(w.Matches)
+	}
+
+	pairs, pinfo, err := FindPairs(c, taxa, wm, PairParams{
+		PrimerLength:   p.PrimerLength,
+		MinLength:      p.MinLength,
+		MaxLength:      p.MaxLength,
+		ThreePrime:     p.ThreePrime,
+		Sensitivity:    p.Sensitivity,
+		FalsePositive:  p.FalsePositive,
+		MinSpecificity: p.MinSpecificity,
+		Circular:       p.Circular,
+		Reference:      refIndex,
+		HasReference:   refIndex >= 0,
+		AmpBudget:      p.AmpBudget,
+		Release:        p.Release,
+	}, p.Model)
+	info.Pair = pinfo
+	return pairs, info, err
+}

pkg/obiprimer/search_real_test.go

@@ -0,0 +1,258 @@
+package obiprimer
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"testing"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiformats"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax"
+)
+
+// loadTutorial reads the Cyprinidae examples of the tutorial data (the opt-in
+// tests need OBIPRIMERS_TUTORIAL, see TestStrictWordsAgainstEcoPrimers).
+func loadTutorial(t *testing.T) (*Corpus, *Taxa, *obitax.Taxonomy) {
+	t.Helper()
+	return loadTutorialWith(t, nil)
+}
+
+// loadTutorialWith is loadTutorial with counterexample clades (ecoPrimers -i).
+func loadTutorialWith(t *testing.T, ignore []string) (*Corpus, *Taxa, *obitax.Taxonomy) {
+	t.Helper()
+	dir := os.Getenv("OBIPRIMERS_TUTORIAL")
+	if dir == "" {
+		t.Skip("OBIPRIMERS_TUTORIAL not set")
+	}
+	taxonomy, err := obiformats.LoadTaxonomy(filepath.Join(dir, "ncbitaxo_20250211.tgz"), false, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	restrict, err := BelongingTo(taxonomy, []string{"7953"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	seqs, err := obiformats.ReadSequencesFromFile(filepath.Join(dir, "mito.vert.fasta.gz"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	sel := Selection{Restrict: restrict}
+	if sel.Ignore, err = BelongingTo(taxonomy, ignore); err != nil {
+		t.Fatal(err)
+	}
+	corpus, err := LoadCorpus(seqs, sel)
+	if err != nil {
+		t.Fatal(err)
+	}
+	taxa, err := corpus.AssignTaxa(taxonomy, "species")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return corpus, taxa, taxonomy
+}
+
+func tutorialParams(t *testing.T) SearchParams {
+	p := DefaultSearchParams()
+	p.MinLength, p.MaxLength, p.ThreePrime, p.Circular = 30, 150, 2, true
+	p.TmpDir = t.TempDir()
+	return p
+}
+
+// ecoPrimers -U on the Cyprinidae tutorial case (-e 3 -3 2 -l 30 -L 150 -c -r 7953 -U)
+// keeps 1366 pairs. testdata/ecoprimers_Cyprinidae_U.pairs.tsv holds, for each,
+// the primers, the amplified example and counterexample sequences, the amplified
+// example and counterexample taxa, and the identified taxa. The identified taxa
+// are not compared (obiprimers defines them differently, see
+// TestPairsAgainstEcoPrimers) and the specificity threshold is off, so that
+// the pairs ecoPrimers dropped with its own Bs do not hide any of its pairs.
+func TestNoMultiMatchAgainstEcoPrimers(t *testing.T) {
+	corpus, taxa, _ := loadTutorial(t)
+	p := tutorialParams(t)
+	p.MinSpecificity = 0
+	p.NoMultiMatch = true
+	pairs, info, err := Search(corpus, taxa, p)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("%+v", *info)
+
+	f, err := os.Open(filepath.Join("testdata", "ecoprimers_Cyprinidae_U.pairs.tsv"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	want := map[string][5]int{}
+	sc := bufio.NewScanner(f)
+	for sc.Scan() {
+		var p1, p2 string
+		var v [5]int
+		if _, err := fmt.Sscanf(sc.Text(), "%s\t%s\t%d\t%d\t%d\t%d\t%d", &p1, &p2, &v[0], &v[1], &v[2], &v[3], &v[4]); err != nil {
+			t.Fatal(err)
+		}
+		want[p1+"/"+p2] = v
+	}
+
+	ours := map[string]PairStats{}
+	for _, g := range pairs {
+		ours[g.Primer1+"/"+g.Primer2] = g
+	}
+	bad, missing := 0, 0
+	for key, v := range want {
+		g, ok := ours[key]
+		if !ok {
+			if missing++; missing <= 3 {
+				t.Errorf("ecoPrimers -U pair %s is not found", key)
+			}
+			continue
+		}
+		got := [4]int{g.Examples, g.CounterExamples, g.ExampleTaxa, g.CounterTaxa}
+		if got != [4]int{v[0], v[1], v[2], v[3]} || g.Identified > v[4] {
+			if bad++; bad <= 3 {
+				t.Errorf("pair %s: got %v (identified %d), ecoPrimers %v", key, got, g.Identified, v)
+			}
+		}
+	}
+	t.Logf("%d pairs of ecoPrimers -U checked, %d missing, %d different; obiprimers has %d pairs (%d more, dropped by ecoPrimers' own Bs)",
+		len(want), missing, bad, len(pairs), len(pairs)-len(want)+missing)
+}
+
+var locRE = regexp.MustCompile(`(\d+)\.\.(\d+)`)
+
+// ecoPrimers -R is only right for the references that happen to come first: it
+// indexes the amplicons of a pair by sequence number. On NC_083213 only 382 of
+// its 3405 locations are real primer sites, against all 3405 for obiprimers.
+// Here the locations are checked against the sequence itself, so no
+// ecoPrimers output is needed: each location must give two real primer sites
+// (at most 3 substitutions) and the amplicon must be the region between them,
+// read from the first primer.
+func TestReferenceSitesAreRealSites(t *testing.T) {
+	corpus, taxa, _ := loadTutorial(t)
+
+	for _, ref := range []string{"NC_022183", "NC_083213"} {
+		t.Run(ref, func(t *testing.T) {
+			p := tutorialParams(t)
+			p.Reference = ref
+			pairs, _, err := Search(corpus, taxa, p)
+			if err != nil {
+				t.Fatal(err)
+			}
+			idx, err := corpus.IndexOfExample(ref)
+			if err != nil {
+				t.Fatal(err)
+			}
+			seq := strings.ToLower(string(corpus.Seqs[idx].Sequence()))
+
+			hamming := func(a, b string) int {
+				d := 0
+				for i := range a {
+					if a[i] != b[i] {
+						d++
+					}
+				}
+				return d
+			}
+			located, bad := 0, 0
+			for _, pr := range pairs {
+				if pr.RefLocation == "" {
+					continue
+				}
+				located++
+				m := locRE.FindAllStringSubmatch(pr.RefLocation, -1)
+				if len(m) != 2 {
+					t.Fatalf("location %q", pr.RefLocation)
+				}
+				var a1, b1, a2, b2 int
+				fmt.Sscanf(m[0][0], "%d..%d", &a1, &b1)
+				fmt.Sscanf(m[1][0], "%d..%d", &a2, &b2)
+				left, right := seq[a1-1:b1], seq[a2-1:b2]
+				between := seq[b1 : a2-1]
+
+				p1, p2 := strings.ToLower(pr.Primer1), strings.ToLower(pr.Primer2)
+				var ok bool
+				if strings.HasPrefix(pr.RefLocation, "complement(") {
+					ok = hamming(strings.ToLower(revcomp(strings.ToUpper(right))), p1) <= 3 && hamming(left, p2) <= 3 &&
+						pr.RefAmplicon == strings.ToLower(revcomp(strings.ToUpper(between)))
+				} else {
+					ok = hamming(left, p1) <= 3 && hamming(strings.ToLower(revcomp(strings.ToUpper(right))), p2) <= 3 &&
+						pr.RefAmplicon == between
+				}
+				if !ok {
+					if bad++; bad <= 3 {
+						t.Errorf("pair %s/%s: %s is not a real location", pr.Primer1, pr.Primer2, pr.RefLocation)
+					}
+				}
+			}
+			t.Logf("%d of %d pairs amplify %s; %d wrong locations", located, len(pairs), ref, bad)
+			if located == 0 {
+				t.Error("no pair amplifies the reference")
+			}
+		})
+	}
+}
+
+// ecoPrimers with counterexamples: -r 7953 -i 1 (the Cyprinidae against all the
+// other sequences of the database, 7497 counterexamples of 7387 species), -e 3
+// -3 2 -l 30 -L 150 -c. ecoPrimers needed 45 minutes and keeps 173 pairs.
+// testdata/ecoprimers_Cyprinidae_I1.pairs.tsv holds for each: the primers, the
+// good flags, the amplified example and counterexample sequences, the amplified
+// example and counterexample taxa, and the identified taxa.
+func TestCounterExamplesAgainstEcoPrimers(t *testing.T) {
+	corpus, taxa, _ := loadTutorialWith(t, []string{"1"})
+	if c := corpus.Counts(); c.Examples != 325 || c.CounterExamples != 7497 || taxa.CounterExamples != 7387 {
+		t.Fatalf("counts %+v, %d counterexample species (ecoPrimers: 325, 7497, 7387)", c, taxa.CounterExamples)
+	}
+	pc := tutorialParams(t)
+	pc.MinSpecificity = 0 // ecoPrimers' own Bs filter is not reproduced, see TestPairsAgainstEcoPrimers
+	pairs, info, err := Search(corpus, taxa, pc)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("%+v", *info)
+
+	f, err := os.Open(filepath.Join("testdata", "ecoprimers_Cyprinidae_I1.pairs.tsv"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	type row struct {
+		good string
+		v    [5]int
+	}
+	want := map[string]row{}
+	sc := bufio.NewScanner(f)
+	for sc.Scan() {
+		var p1, p2 string
+		var r row
+		if _, err := fmt.Sscanf(sc.Text(), "%s\t%s\t%s\t%d\t%d\t%d\t%d\t%d", &p1, &p2, &r.good,
+			&r.v[0], &r.v[1], &r.v[2], &r.v[3], &r.v[4]); err != nil {
+			t.Fatal(err)
+		}
+		want[p1+"/"+p2] = r
+	}
+
+	ours := map[string]PairStats{}
+	for _, g := range pairs {
+		ours[g.Primer1+"/"+g.Primer2] = g
+	}
+	bad, missing := 0, 0
+	for key, w := range want {
+		g, ok := ours[key]
+		if !ok {
+			if missing++; missing <= 3 {
+				t.Errorf("ecoPrimers pair %s is not found", key)
+			}
+			continue
+		}
+		good := string("bG"[b2i(g.Good1)]) + string("bG"[b2i(g.Good2)])
+		got := [4]int{g.Examples, g.CounterExamples, g.ExampleTaxa, g.CounterTaxa}
+		if got != [4]int{w.v[0], w.v[1], w.v[2], w.v[3]} || good != w.good || g.Identified > w.v[4] {
+			if bad++; bad <= 3 {
+				t.Errorf("pair %s: got %s %v (identified %d), ecoPrimers %s %v", key, good, got, g.Identified, w.good, w.v)
+			}
+		}
+	}
+	t.Logf("%d pairs of ecoPrimers checked, %d missing, %d different; obiprimers has %d pairs", len(want), missing, bad, len(pairs))
+}

pkg/obiprimer/search_test.go

@@ -0,0 +1,274 @@
+package obiprimer
+
+import (
+	"fmt"
+	"math/rand"
+	"strings"
+	"testing"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obikmer"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+// synthetic builds nseq example sequences sharing the block F + insert + rc(R),
+// each with its own flanks and its own substitution inside the insert (so that
+// every sequence has a distinct amplicon, hence Bs = 1, with one taxon each).
+// extraF, if not empty, is appended to the flank of sequence extraIn: that
+// sequence then holds the forward site twice.
+type synthetic struct {
+	corpus  *Corpus
+	taxa    *Taxa
+	f, r, i string // forward primer, reverse primer (5'->3'), insert of sequence 0
+	pos     int    // 0-based start of the block in every sequence
+}
+
+func newSynthetic(nseq int, repeatForwardIn int, seed int64) synthetic {
+	rng := rand.New(rand.NewSource(seed))
+	f := randDNA(rng, 18)
+	r := randDNA(rng, 18)
+	ins := randDNA(rng, 40)
+	const pos = 60
+
+	c := &Corpus{}
+	taxa := &Taxa{Rank: "species", ExampleTaxa: nseq}
+	for k := 0; k < nseq; k++ {
+		in := []byte(ins)
+		in[3+k] = "ACGT"[(strings.IndexByte("ACGT", in[3+k])+1)%4] // one private substitution
+		s := randDNA(rng, pos) + f + string(in) + revcomp(r) + randDNA(rng, 60)
+		if k == repeatForwardIn {
+			s += randDNA(rng, 20) + f // the forward site a second time
+		}
+		c.Seqs = append(c.Seqs, obiseq.NewBioSequence(fmt.Sprintf("s%d", k), []byte(strings.ToLower(s)), ""))
+		c.Roles = append(c.Roles, Example)
+		taxa.Of = append(taxa.Of, int32(k))
+	}
+	return synthetic{c, taxa, f, r, ins, pos}
+}
+
+func (s synthetic) params() SearchParams {
+	p := DefaultSearchParams()
+	p.MaxErrors = 1
+	p.MinLength, p.MaxLength = 20, 100
+	p.Partitions = 8
+	p.TmpDir = ""
+	return p
+}
+
+// find returns the pair made of the primers f and r, whatever their order.
+func find(pairs []PairStats, f, r string) *PairStats {
+	for i := range pairs {
+		if (pairs[i].Primer1 == f && pairs[i].Primer2 == r) || (pairs[i].Primer1 == r && pairs[i].Primer2 == f) {
+			return &pairs[i]
+		}
+	}
+	return nil
+}
+
+func TestSearchFindsThePlantedPair(t *testing.T) {
+	s := newSynthetic(8, -1, 11)
+	pairs, info, err := Search(s.corpus, s.taxa, s.params())
+	if err != nil {
+		t.Fatal(err)
+	}
+	p := find(pairs, s.f, s.r)
+	if p == nil {
+		t.Fatalf("the planted pair was not found among %d pairs (%+v)", len(pairs), info)
+	}
+	if p.Examples != 8 || p.MinLen != 40 || p.MaxLen != 40 || p.Bs != 1 || p.Bc != 1 || p.Bd != 1 {
+		t.Errorf("unexpected indices %+v", *p)
+	}
+	if p.RefLocation != "" {
+		t.Error("no reference was asked: no location expected")
+	}
+}
+
+func TestReferenceLocation(t *testing.T) {
+	forwardFirst, reverseFirst := 0, 0
+	for seed := int64(1); seed <= 24; seed++ {
+		s := newSynthetic(8, -1, seed)
+		p := s.params()
+		p.Reference = "s3"
+		pairs, _, err := Search(s.corpus, s.taxa, p)
+		if err != nil {
+			t.Fatal(err)
+		}
+		pair := find(pairs, s.f, s.r)
+		if pair == nil {
+			t.Fatalf("seed %d: planted pair not found", seed)
+		}
+
+		// 1-based sites: forward primer at 61..78, then 40 bases of insert, then the reverse site
+		wantLoc := "join(61..78,119..136)"
+		insert := strings.ToLower(string(s.corpus.Seqs[3].Sequence()[s.pos+18 : s.pos+58]))
+		if pair.Primer1 == s.f { // the amplicon is read from the first primer
+			forwardFirst++
+			if pair.RefLocation != wantLoc || pair.RefAmplicon != insert {
+				t.Errorf("seed %d, forward first: got %s %s", seed, pair.RefLocation, pair.RefAmplicon)
+			}
+		} else {
+			reverseFirst++
+			rc := strings.ToLower(revcomp(strings.ToUpper(insert)))
+			if pair.RefLocation != "complement("+wantLoc+")" || pair.RefAmplicon != rc {
+				t.Errorf("seed %d, reverse first: got %s %s", seed, pair.RefLocation, pair.RefAmplicon)
+			}
+		}
+		if len(pair.RefAmplicon) != pair.MinLen {
+			t.Errorf("seed %d: amplicon length %d, pair length %d", seed, len(pair.RefAmplicon), pair.MinLen)
+		}
+	}
+	if forwardFirst == 0 || reverseFirst == 0 {
+		t.Errorf("both orientations must be exercised: %d forward first, %d reverse first", forwardFirst, reverseFirst)
+	}
+}
+
+func TestReferenceErrors(t *testing.T) {
+	s := newSynthetic(8, -1, 11)
+	p := s.params()
+	p.Reference = "nope"
+	if _, _, err := Search(s.corpus, s.taxa, p); err == nil {
+		t.Error("an unknown reference must be refused")
+	}
+	s.corpus.Roles[2] = CounterExample
+	p.Reference = "s2"
+	if _, _, err := Search(s.corpus, s.taxa, p); err == nil {
+		t.Error("a counterexample cannot be the reference")
+	}
+}
+
+func TestReferenceOnACircularJunction(t *testing.T) {
+	// span writes a range running over the end of a circular sequence in two parts
+	if got := span(95, 10, 100, true); got != "96..100,1..5" {
+		t.Errorf("circular span %q", got)
+	}
+	if got := span(95, 10, 100, false); got != "96..105" {
+		t.Errorf("linear span %q", got)
+	}
+	if got := span(10, 18, 100, true); got != "11..28" {
+		t.Errorf("inner span %q", got)
+	}
+}
+
+func TestNoMultiMatchDropsRepeatedPrimers(t *testing.T) {
+	// the forward primer is found twice in sequence 2
+	s := newSynthetic(8, 2, 11)
+
+	with, infoWith, err := Search(s.corpus, s.taxa, s.params())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if find(with, s.f, s.r) == nil {
+		t.Fatalf("without -U the pair must be found (%+v)", infoWith)
+	}
+
+	p := s.params()
+	p.NoMultiMatch = true
+	without, info, err := Search(s.corpus, s.taxa, p)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if find(without, s.f, s.r) != nil {
+		t.Error("with -U the repeated forward primer must be discarded, and its pair with it")
+	}
+	if info.Repeated == 0 {
+		t.Errorf("the repeated word was not counted: %+v", info)
+	}
+	if len(without) >= len(with) {
+		t.Errorf("-U must not add pairs: %d with, %d without", len(with), len(without))
+	}
+}
+
+func TestDropMultiMatch(t *testing.T) {
+	wm := []WordMatches{
+		{Word: 1, Matches: []Match{{0, 5, 0, false}, {1, 9, 0, true}}},   // one per sequence: kept
+		{Word: 2, Matches: []Match{{0, 5, 0, false}, {0, 50, 1, false}}}, // two in sequence 0
+		{Word: 3, Matches: []Match{{2, 5, 0, false}, {2, 5, 0, true}}},   // both strands of one sequence
+		{Word: 4}, // no match at all: kept
+		{Word: 5, Matches: []Match{{3, 1, 0, false}, {3, 8, 0, false}}}, // twice, but in a non-target: kept
+	}
+	c := corpusOf([]Role{Example, Example, CounterExample, Other}, "a", "a", "a", "a")
+	kept, dropped := DropMultiMatch(wm, c)
+	if dropped != 2 || len(kept) != 3 || kept[0].Word != 1 || kept[1].Word != 4 || kept[2].Word != 5 {
+		t.Errorf("kept %v, dropped %d", kept, dropped)
+	}
+}
+
+func TestDropRepeatedWords(t *testing.T) {
+	words := []obikmer.SeqKmerCount{{Kmer: 1}, {Kmer: 2, Repeated: true}, {Kmer: 3}}
+	kept, dropped := DropRepeatedWords(words)
+	if dropped != 1 || len(kept) != 2 || kept[0].Kmer != 1 || kept[1].Kmer != 3 {
+		t.Errorf("kept %v, dropped %d", kept, dropped)
+	}
+}
+
+// A non-target amplified with the very barcode of a target taxon makes this
+// target badly identified, although its barcode is unique among the targets;
+// it counts in neither the sensitivity nor the false positive quorums.
+func TestNonTargetSharingABarcodeLowersBsAndBd(t *testing.T) {
+	s := newSynthetic(8, -1, 11)
+
+	alone, _, err := Search(s.corpus, s.taxa, s.params())
+	if err != nil {
+		t.Fatal(err)
+	}
+	p0 := find(alone, s.f, s.r)
+	if p0 == nil || p0.Bs != 1 || p0.Bd != 1 {
+		t.Fatalf("without the non-target: %+v", p0)
+	}
+
+	// a copy of sequence 3, of another taxon, that is not a target
+	copy3 := append([]byte(nil), s.corpus.Seqs[3].Sequence()...)
+	s.corpus.Seqs = append(s.corpus.Seqs, obiseq.NewBioSequence("other", copy3, ""))
+	s.corpus.Roles = append(s.corpus.Roles, Other)
+	s.taxa.Of = append(s.taxa.Of, 100)
+
+	pairs, _, err := Search(s.corpus, s.taxa, s.params())
+	if err != nil {
+		t.Fatal(err)
+	}
+	p := find(pairs, s.f, s.r)
+	if p == nil {
+		t.Fatal("the planted pair must still be found")
+	}
+	if p.Examples != 8 || p.ExampleTaxa != 8 || p.Bc != 1 {
+		t.Errorf("the non-target must not change the quorums nor Bc: %+v", *p)
+	}
+	if p.Identified != 7 || p.Bs != 0.875 {
+		t.Errorf("target 3 shares its barcode with the non-target: identified %d, Bs %v (want 7, 0.875)", p.Identified, p.Bs)
+	}
+	if p.Bd != 0.875 {
+		t.Errorf("Bd %v, want 7 discriminant barcodes of 8", p.Bd)
+	}
+	if p.CounterTaxa != 0 || p.CounterExamples != 0 {
+		t.Errorf("a non-target is not a counterexample: %+v", *p)
+	}
+
+	// Now counterexamples: two copies (of sequences 3 and 5) that the pair amplifies and
+	// one unrelated sequence that it does not. 2 of 3 amplified is under the 0.9 quorum.
+	s.corpus.Roles[len(s.corpus.Roles)-1] = CounterExample
+	copy5 := append([]byte(nil), s.corpus.Seqs[5].Sequence()...)
+	s.corpus.Seqs = append(s.corpus.Seqs, obiseq.NewBioSequence("counter5", copy5, ""))
+	s.corpus.Roles = append(s.corpus.Roles, CounterExample)
+	s.taxa.Of = append(s.taxa.Of, 101)
+	unrelated := []byte(strings.ToLower(randDNA(rand.New(rand.NewSource(5)), 300)))
+	s.corpus.Seqs = append(s.corpus.Seqs, obiseq.NewBioSequence("unrelated", unrelated, ""))
+	s.corpus.Roles = append(s.corpus.Roles, CounterExample)
+	s.taxa.Of = append(s.taxa.Of, 102)
+
+	pq := s.params()
+	pq.FalsePositive = 0.9
+	pairs, _, err = Search(s.corpus, s.taxa, pq)
+	if err != nil {
+		t.Fatal(err)
+	}
+	pc := find(pairs, s.f, s.r)
+	if pc == nil {
+		t.Fatal("the pair amplifies 2 of 3 counterexamples (< 0.9): it must be retained")
+	}
+	if pc.CounterExamples != 2 || pc.CounterTaxa != 2 {
+		t.Errorf("counterexamples amplified %d (taxa %d), want 2 and 2", pc.CounterExamples, pc.CounterTaxa)
+	}
+	if pc.Identified != 6 || pc.Bd != 0.75 {
+		t.Errorf("targets 3 and 5 share their barcode with a counterexample: identified %d (want 6), Bd %v (want 0.75)",
+			pc.Identified, pc.Bd)
+	}
+}

pkg/obiprimer/taxa.go

@@ -0,0 +1,83 @@
+package obiprimer
+
+import (
+	"fmt"
+	"slices"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax"
+)
+
+// Taxa gives every corpus sequence its taxon at one rank (the species by
+// default in ecoPrimers), as a dense index, and the number of distinct taxa
+// among the examples and among the counterexamples.
+//
+// A sequence whose taxon has no ancestor at that rank (or no taxon at all) has
+// the index -1; it is never counted as a taxon.
+type Taxa struct {
+	Rank            string
+	Of              []int32 // Of[i] is the taxon index of Corpus.Seqs[i], or -1
+	ExampleTaxa     int
+	CounterExamples int // distinct taxa among the counterexamples
+	nodes           []*obitax.Taxon
+}
+
+// Name returns the taxon with the given index, for reporting.
+func (t *Taxa) Name(i int32) *obitax.Taxon { return t.nodes[i] }
+
+// checkRank reports an error if the taxonomy has no such rank.
+func checkRank(taxonomy *obitax.Taxonomy, rank string) error {
+	if taxonomy == nil {
+		return fmt.Errorf("obiprimer: a taxonomy is required to compute taxon statistics")
+	}
+	if !slices.Contains(taxonomy.RankList(), rank) {
+		return fmt.Errorf("obiprimer: unknown taxonomic rank %q", rank)
+	}
+	return nil
+}
+
+// AssignTaxa determines the taxon of every sequence of the corpus at rank. The
+// sequences must not be packed (see LoadPackedCorpus).
+func (c *Corpus) AssignTaxa(taxonomy *obitax.Taxonomy, rank string) (*Taxa, error) {
+	if err := checkRank(taxonomy, rank); err != nil {
+		return nil, err
+	}
+	taxons := make([]*obitax.Taxon, len(c.Seqs))
+	for i, seq := range c.Seqs {
+		taxons[i] = seq.Taxon(taxonomy).TaxonAtRank(rank)
+	}
+	return buildTaxa(rank, taxons, c.Roles)
+}
+
+// buildTaxa gives the sequences, whose taxon at rank is taxons[i] (nil when
+// there is none), their dense taxon index.
+func buildTaxa(rank string, taxons []*obitax.Taxon, roles []Role) (*Taxa, error) {
+	t := &Taxa{Rank: rank, Of: make([]int32, len(taxons))}
+	index := map[*obitax.TaxNode]int32{}
+	exSeen := map[int32]struct{}{}
+	coSeen := map[int32]struct{}{}
+
+	for i, taxon := range taxons {
+		t.Of[i] = -1
+		if taxon == nil {
+			continue
+		}
+		id, ok := index[taxon.Node]
+		if !ok {
+			if len(t.nodes) >= maxTaxa {
+				return nil, fmt.Errorf("obiprimer: more than %d taxa at rank %q", maxTaxa, rank)
+			}
+			id = int32(len(t.nodes))
+			index[taxon.Node] = id
+			t.nodes = append(t.nodes, taxon)
+		}
+		t.Of[i] = id
+		switch roles[i] {
+		case Example:
+			exSeen[id] = struct{}{}
+		case CounterExample:
+			coSeen[id] = struct{}{}
+		}
+	}
+	t.ExampleTaxa, t.CounterExamples = len(exSeen), len(coSeen)
+	return t, nil
+}

pkg/obiprimer/testdata/ecoprimers_Cyprinidae_I1.pairs.tsv

@@ -0,0 +1,173 @@
+AGCAATCCAGGTCAGTTT	TGCCACCCCAGTATGTCC	bG	309	454	301	452	197
+GCAATCCAGGTCAGTTTC	TGCCACCCCAGTATGTCC	bG	309	455	301	453	197
+AGCAATCCAGGTCAGTTT	CCACCCCAGTATGTCCTT	bG	311	484	302	482	197
+CCACCCCAGTATGTCCTT	GCAATCCAGGTCAGTTTC	Gb	311	485	302	483	197
+AGCAATCCAGGTCAGTTT	GCCACCCCAGTATGTCCT	bG	308	445	300	444	196
+GCAATCCAGGTCAGTTTC	GCCACCCCAGTATGTCCT	bG	308	446	300	445	196
+ACCGGAGCAATCCAGGTC	TGCCACCCCAGTATGTCC	bG	308	453	300	451	196
+GAGCAATCCAGGTCAGTT	TGCCACCCCAGTATGTCC	bG	308	453	300	451	196
+AGACCGGAGCAATCCAGG	CCACCCCAGTATGTCCTT	bG	310	483	301	481	196
+CCACCCCAGTATGTCCTT	GAGCAATCCAGGTCAGTT	Gb	310	483	301	481	196
+ACCGGAGCAATCCAGGTC	CCACCCCAGTATGTCCTT	bG	310	484	301	482	196
+CCACCCCAGTATGTCCTT	GACCGGAGCAATCCAGGT	Gb	310	484	301	482	196
+CCACCCCAGTATGTCCTT	GGTCAGTTTCTATCTGTA	Gb	309	460	300	459	195
+AGGTCAGTTTCTATCTGT	TGCCACCCCAGTATGTCC	bG	307	438	299	437	195
+CAGGTCAGTTTCTATCTG	TGCCACCCCAGTATGTCC	bG	307	440	299	439	195
+GGTCAGTTTCTATCTGTA	TGCCACCCCAGTATGTCC	bG	307	441	299	440	195
+GAGCAATCCAGGTCAGTT	GCCACCCCAGTATGTCCT	bG	307	444	299	443	195
+ACCGGAGCAATCCAGGTC	GCCACCCCAGTATGTCCT	bG	307	445	299	444	195
+GACCGGAGCAATCCAGGT	GCCACCCCAGTATGTCCT	bG	307	445	299	444	195
+AGGTCAGTTTCTATCTGT	CCACCCCAGTATGTCCTT	bG	309	469	300	467	195
+CAGGTCAGTTTCTATCTG	CCACCCCAGTATGTCCTT	bG	309	470	300	468	195
+GGAGCAATCCAGGTCAGT	TGCCACCCCAGTATGTCC	bG	307	454	299	452	195
+CCAGGTCAGTTTCTATCT	TGCCACCCCAGTATGTCC	bG	307	455	299	453	195
+AATCCAGGTCAGTTTCTA	TGCCACCCCAGTATGTCC	bG	307	455	299	453	195
+CAATCCAGGTCAGTTTCT	TGCCACCCCAGTATGTCC	bG	307	455	299	453	195
+ATCCAGGTCAGTTTCTAT	TGCCACCCCAGTATGTCC	bG	307	455	299	453	195
+TCCAGGTCAGTTTCTATC	TGCCACCCCAGTATGTCC	bG	307	455	299	453	195
+CCACCCCAGTATGTCCTT	GGAGCAATCCAGGTCAGT	Gb	309	484	300	482	195
+CAATCCAGGTCAGTTTCT	CCACCCCAGTATGTCCTT	bG	309	485	300	483	195
+AATCCAGGTCAGTTTCTA	CCACCCCAGTATGTCCTT	bG	309	485	300	483	195
+ATCCAGGTCAGTTTCTAT	CCACCCCAGTATGTCCTT	bG	309	485	300	483	195
+CCACCCCAGTATGTCCTT	TCCAGGTCAGTTTCTATC	Gb	309	485	300	483	195
+CCACCCCAGTATGTCCTT	CCAGGTCAGTTTCTATCT	Gb	309	485	300	483	195
+GCCACCCCAGTATGTCCT	GGTCAGTTTCTATCTGTA	Gb	306	437	298	437	194
+AGGTCAGTTTCTATCTGT	GCCACCCCAGTATGTCCT	bG	306	440	298	439	194
+CAGGTCAGTTTCTATCTG	GCCACCCCAGTATGTCCT	bG	306	442	298	441	194
+GCCACCCCAGTATGTCCT	GGAGCAATCCAGGTCAGT	Gb	306	445	298	444	194
+GCCACCCCAGTATGTCCT	TCCAGGTCAGTTTCTATC	Gb	306	446	298	445	194
+CCAGGTCAGTTTCTATCT	GCCACCCCAGTATGTCCT	bG	306	446	298	445	194
+CAATCCAGGTCAGTTTCT	GCCACCCCAGTATGTCCT	bG	306	446	298	445	194
+AATCCAGGTCAGTTTCTA	GCCACCCCAGTATGTCCT	bG	306	446	298	445	194
+ATCCAGGTCAGTTTCTAT	GCCACCCCAGTATGTCCT	bG	306	446	298	445	194
+CGGAGCAATCCAGGTCAG	TGCCACCCCAGTATGTCC	bG	304	445	296	444	192
+CCGGAGCAATCCAGGTCA	TGCCACCCCAGTATGTCC	bG	304	445	296	444	192
+TGCCACCCCAGTATGTCC	TTTTCCTAGTACGAAAGG	Gb	308	453	300	452	192
+CCACCCCAGTATGTCCTT	CCGGAGCAATCCAGGTCA	Gb	306	476	297	474	192
+CCACCCCAGTATGTCCTT	CGGAGCAATCCAGGTCAG	Gb	306	476	297	474	192
+GACCGGAGCAATCCAGGT	TGCCACCCCAGTATGTCC	bG	304	453	296	451	192
+CCACCCCAGTATGTCCTT	GTCAGTTTCTATCTGTAA	Gb	304	408	295	407	190
+CCACCCCAGTATGTCCTT	TTTTCCTAGTACGAAAGG	Gb	310	484	301	482	192
+CAGACCGGAGCAATCCAG	CCACCCCAGTATGTCCTT	bG	306	484	297	482	192
+GTCAGTTTCTATCTGTAA	TGCCACCCCAGTATGTCC	bG	302	390	294	390	190
+AGCAATCCAGGTCAGTTT	CCCAGTATGTCCTTATTT	bG	301	310	292	309	187
+CCCAGTATGTCCTTATTT	GCAATCCAGGTCAGTTTC	Gb	301	311	292	310	187
+ACTTTTCCTAGTACGAAA	TGCCACCCCAGTATGTCC	bG	307	443	299	442	191
+GCCACCCCAGTATGTCCT	TTTTCCTAGTACGAAAGG	Gb	307	445	299	444	191
+CGGAGCAATCCAGGTCAG	GCCACCCCAGTATGTCCT	bG	303	443	295	442	191
+CCGGAGCAATCCAGGTCA	GCCACCCCAGTATGTCCT	bG	303	443	295	442	191
+CTTTTCCTAGTACGAAAG	TGCCACCCCAGTATGTCC	bG	307	448	299	447	191
+AGACCGGAGCAATCCAGG	GCCACCCCAGTATGTCCT	bG	303	445	295	444	191
+ACTTTTCCTAGTACGAAA	CCACCCCAGTATGTCCTT	bG	309	478	300	476	191
+CCACCCCAGTATGTCCTT	CTTTTCCTAGTACGAAAG	Gb	309	482	300	480	191
+GCCACCCCAGTATGTCCT	GTCAGTTTCTATCTGTAA	Gb	301	390	293	390	189
+AGACCGGAGCAATCCAGG	CCCAGTATGTCCTTATTT	bG	300	310	291	309	186
+CAGACCGGAGCAATCCAG	CCCAGTATGTCCTTATTT	bG	300	310	291	309	186
+CCCAGTATGTCCTTATTT	GAGCAATCCAGGTCAGTT	Gb	300	310	291	309	186
+ACCGGAGCAATCCAGGTC	CCCAGTATGTCCTTATTT	bG	300	311	291	310	186
+CCCAGTATGTCCTTATTT	GACCGGAGCAATCCAGGT	Gb	300	311	291	310	186
+CCCAGTATGTCCTTATTT	TCAGACCGGAGCAATCCA	Gb	300	311	291	310	186
+CCCAGTATGTCCTTATTT	TTCAGACCGGAGCAATCC	Gb	300	311	291	310	186
+CCCAGTATGTCCTTATTT	GTTCAGACCGGAGCAATC	Gb	300	311	291	310	186
+CTTTTCCTAGTACGAAAG	GCCACCCCAGTATGTCCT	bG	306	443	298	442	190
+ACTTTTCCTAGTACGAAA	GCCACCCCAGTATGTCCT	bG	306	443	298	442	190
+AGACCGGAGCAATCCAGG	TGCCACCCCAGTATGTCC	bG	302	453	294	451	190
+CCACCCCAGTATGTCCTT	TCAGACCGGAGCAATCCA	Gb	304	485	295	483	190
+AGGTCAGTTTCTATCTGT	CCCAGTATGTCCTTATTT	bG	299	309	290	308	185
+CAGGTCAGTTTCTATCTG	CCCAGTATGTCCTTATTT	bG	299	309	290	308	185
+CCCAGTATGTCCTTATTT	GGTCAGTTTCTATCTGTA	Gb	299	309	290	308	185
+ATCCAGGTCAGTTTCTAT	CCCAGTATGTCCTTATTT	bG	299	311	290	310	185
+CCAGGTCAGTTTCTATCT	CCCAGTATGTCCTTATTT	bG	299	311	290	310	185
+CCCAGTATGTCCTTATTT	GGAGCAATCCAGGTCAGT	Gb	299	311	290	310	185
+CCCAGTATGTCCTTATTT	TCCAGGTCAGTTTCTATC	Gb	299	311	290	310	185
+CAATCCAGGTCAGTTTCT	CCCAGTATGTCCTTATTT	bG	299	311	290	310	185
+AATCCAGGTCAGTTTCTA	CCCAGTATGTCCTTATTT	bG	299	311	290	310	185
+TGCCACCCCAGTATGTCC	TTTCCTAGTACGAAAGGA	Gb	305	449	297	447	189
+CAGACCGGAGCAATCCAG	GCCACCCCAGTATGTCCT	bG	301	446	293	445	189
+CCACCCCAGTATGTCCTT	TTTCCTAGTACGAAAGGA	Gb	307	479	298	477	189
+CCACCCCAGTATGTCCTT	TACGAAAGGATCGGAAAA	Gb	307	453	298	452	188
+CCCAGTATGTCCTTATTT	CCGGAGCAATCCAGGTCA	Gb	298	309	289	308	184
+CCCAGTATGTCCTTATTT	CGGAGCAATCCAGGTCAG	Gb	298	309	289	308	184
+GCCACCCCAGTATGTCCT	TTTCCTAGTACGAAAGGA	Gb	304	440	296	439	188
+GCAATTTACCATGCTCTG	TTTTCCTAGTACGAAAGG	Gb	308	505	299	501	189
+ACTTTTCCTAGTACGAAA	GCAATTTACCATGCTCTG	bG	307	478	298	474	188
+CAGACCGGAGCAATCCAG	TGCCACCCCAGTATGTCC	bG	300	454	292	452	188
+CCTAGTACGAAAGGATCG	TGCCACCCCAGTATGTCC	bG	303	427	295	425	187
+CCACCCCAGTATGTCCTT	TTCAGACCGGAGCAATCC	Gb	302	485	293	483	188
+TACGAAAGGATCGGAAAA	TGCCACCCCAGTATGTCC	bG	304	430	296	430	187
+GCCACCCCAGTATGTCCT	TACGAAAGGATCGGAAAA	Gb	304	430	296	430	187
+CCACCCCAGTATGTCCTT	CCTAGTACGAAAGGATCG	Gb	305	459	296	457	187
+CTTTTCCTAGTACGAAAG	GCAATTTACCATGCTCTG	bG	307	498	298	494	188
+CGCAATTTACCATGCTCT	TTTTCCTAGTACGAAAGG	Gb	306	463	297	459	187
+GCAATTTACCATGCTCTG	TTTCCTAGTACGAAAGGA	Gb	307	506	298	502	188
+GCAATTTACCATGCTCTG	GGTCAGTTTCTATCTGTA	Gb	301	433	292	429	186
+GCCACCCCAGTATGTCCT	TCAGACCGGAGCAATCCA	Gb	299	446	291	445	187
+CCTAGTACGAAAGGATCG	GCCACCCCAGTATGTCCT	bG	302	419	294	418	186
+CCCAGTATGTCCTTATTT	GTCAGTTTCTATCTGTAA	Gb	295	264	286	263	181
+ACTTTTCCTAGTACGAAA	CGCAATTTACCATGCTCT	bG	305	449	296	445	186
+TCGCAATTTACCATGCTC	TTTTCCTAGTACGAAAGG	Gb	305	456	296	452	186
+TTCGCAATTTACCATGCT	TTTTCCTAGTACGAAAGG	Gb	305	456	296	452	186
+AGTTCAGACCGGAGCAAT	CCCAGTATGTCCTTATTT	bG	296	310	287	309	182
+CGCAATTTACCATGCTCT	CTTTTCCTAGTACGAAAG	Gb	305	460	296	456	186
+CGCAATTTACCATGCTCT	TTTCCTAGTACGAAAGGA	Gb	305	463	296	459	186
+CCTAGTACGAAAGGATCG	GCAATTTACCATGCTCTG	bG	305	472	296	468	186
+TCAGACCGGAGCAATCCA	TGCCACCCCAGTATGTCC	bG	298	454	290	452	186
+ACTTTTCCTAGTACGAAA	TCGCAATTTACCATGCTC	bG	304	447	295	443	185
+ACTTTTCCTAGTACGAAA	TTCGCAATTTACCATGCT	bG	304	447	295	443	185
+CTTTTCCTAGTACGAAAG	TCGCAATTTACCATGCTC	bG	304	453	295	449	185
+CTTTTCCTAGTACGAAAG	TTCGCAATTTACCATGCT	bG	304	453	295	449	185
+CCACCCCAGTATGTCCTT	GTTCAGACCGGAGCAATC	Gb	300	485	291	483	186
+TCGCAATTTACCATGCTC	TTTCCTAGTACGAAAGGA	Gb	304	456	295	452	185
+TTTCGCAATTTACCATGC	TTTTCCTAGTACGAAAGG	Gb	304	456	295	452	185
+TTCGCAATTTACCATGCT	TTTCCTAGTACGAAAGGA	Gb	304	456	295	452	185
+GCAATTTACCATGCTCTG	TACGAAAGGATCGGAAAA	Gb	304	427	295	425	184
+CCTAGTACGAAAGGATCG	CGCAATTTACCATGCTCT	bG	303	433	294	429	184
+AGGTCAGTTTCTATCTGT	GCAATTTACCATGCTCTG	bG	299	431	290	427	184
+GCCACCCCAGTATGTCCT	TTCAGACCGGAGCAATCC	Gb	297	446	289	445	185
+ACTTTTCCTAGTACGAAA	TTTCGCAATTTACCATGC	bG	303	447	294	443	184
+GCAATTTACCATGCTCTG	GTCAGTTTCTATCTGTAA	Gb	295	375	286	373	182
+CTTTTCCTAGTACGAAAG	TTTCGCAATTTACCATGC	bG	303	453	294	449	184
+TTTCCTAGTACGAAAGGA	TTTCGCAATTTACCATGC	bG	303	456	294	452	184
+CCCAGTATGTCCTTATTT	TTTTCCTAGTACGAAAGG	Gb	300	311	291	310	180
+CCCAGTATGTCCTTATTT	GAGTTCAGACCGGAGCAA	Gb	294	310	285	309	180
+CCTAGTACGAAAGGATCG	TCGCAATTTACCATGCTC	bG	302	430	293	426	183
+CCTAGTACGAAAGGATCG	TTCGCAATTTACCATGCT	bG	302	430	293	426	183
+CGCAATTTACCATGCTCT	TACGAAAGGATCGGAAAA	Gb	302	420	293	418	182
+ACTTTTCCTAGTACGAAA	CCCAGTATGTCCTTATTT	bG	299	311	290	310	179
+CCCAGTATGTCCTTATTT	CTTTTCCTAGTACGAAAG	Gb	299	311	290	310	179
+CGCAATTTACCATGCTCT	GGTCAGTTTCTATCTGTA	Gb	297	426	288	422	182
+CCTAGTACGAAAGGATCG	TTTCGCAATTTACCATGC	bG	301	430	292	426	182
+CAGGTCAGTTTCTATCTG	GCAATTTACCATGCTCTG	bG	297	431	288	427	182
+TACGAAAGGATCGGAAAA	TTCGCAATTTACCATGCT	bG	301	418	292	416	181
+TACGAAAGGATCGGAAAA	TCGCAATTTACCATGCTC	bG	301	418	292	416	181
+GGCCTTTCGCAATTTACC	TTTTCCTAGTACGAAAGG	Gb	302	510	293	506	183
+ACTTTTCCTAGTACGAAA	GGCCTTTCGCAATTTACC	bG	301	479	292	475	182
+TACGAAAGGATCGGAAAA	TTTCGCAATTTACCATGC	bG	300	418	291	416	180
+CCCAGTATGTCCTTATTT	TTTCCTAGTACGAAAGGA	Gb	297	310	288	309	177
+CTTTTCCTAGTACGAAAG	GGCCTTTCGCAATTTACC	bG	301	500	292	496	182
+AGGTCAGTTTCTATCTGT	CGCAATTTACCATGCTCT	bG	295	428	286	424	180
+GGCCTTTCGCAATTTACC	TTTCCTAGTACGAAAGGA	Gb	301	511	292	507	182
+GCCTTTCGCAATTTACCA	TTTTCCTAGTACGAAAGG	Gb	299	440	290	436	180
+CTTTCGCAATTTACCATG	TTTTCCTAGTACGAAAGG	Gb	298	412	289	408	179
+CCCAGTATGTCCTTATTT	CCTAGTACGAAAGGATCG	Gb	296	305	287	304	176
+CCCAGTATGTCCTTATTT	TACGAAAGGATCGGAAAA	Gb	297	307	288	306	176
+GGTCAGTTTCTATCTGTA	TCGCAATTTACCATGCTC	bG	294	425	285	421	179
+ACTTTTCCTAGTACGAAA	GCCTTTCGCAATTTACCA	bG	298	436	289	432	179
+CTTTTCCTAGTACGAAAG	GCCTTTCGCAATTTACCA	bG	298	438	289	434	179
+GCCTTTCGCAATTTACCA	TTTCCTAGTACGAAAGGA	Gb	298	440	289	436	179
+CCTAGTACGAAAGGATCG	GGCCTTTCGCAATTTACC	bG	299	484	290	480	180
+ACTTTTCCTAGTACGAAA	CTTTCGCAATTTACCATG	bG	297	411	288	407	178
+CTTTCGCAATTTACCATG	CTTTTCCTAGTACGAAAG	Gb	297	411	288	407	178
+CTTTCGCAATTTACCATG	TTTCCTAGTACGAAAGGA	Gb	297	412	288	408	178
+GGCCTTTCGCAATTTACC	TACGAAAGGATCGGAAAA	Gb	298	425	289	423	178
+AGTTCAGACCGGAGCAAT	CCACCCCAGTATGTCCTT	bG	293	484	284	482	179
+CCTAGTACGAAAGGATCG	GCCTTTCGCAATTTACCA	bG	296	416	287	412	177
+CCTAGTACGAAAGGATCG	CTTTCGCAATTTACCATG	bG	295	390	286	386	176
+CCAGGTCAGTTTCTATCT	GCAATTTACCATGCTCTG	bG	294	513	285	509	179
+CCTTTCGCAATTTACCAT	TTTTCCTAGTACGAAAGG	Gb	295	411	286	407	176
+GCAATTTACCATGCTCTG	TTCCTAGTACGAAAGGAT	Gb	293	423	284	419	176
+CTTTCGCAATTTACCATG	TACGAAAGGATCGGAAAA	Gb	295	398	286	396	175
+ACTTTTCCTAGTACGAAA	CCTTTCGCAATTTACCAT	bG	294	409	285	405	175
+CCTTTCGCAATTTACCAT	CTTTTCCTAGTACGAAAG	Gb	294	409	285	405	175
+CCTTTCGCAATTTACCAT	TTTCCTAGTACGAAAGGA	Gb	294	410	285	406	175
+GCCTTTCGCAATTTACCA	TACGAAAGGATCGGAAAA	Gb	295	413	286	411	175

pkg/obiprimer/testdata/ecoprimers_Teleostei.words.tsv

@@ -0,0 +1,255 @@
+AAAACTCGTGCCAGCCAC	3891	0
+AAACCCAAAGGACTTGGC	3890	0
+AAACGAACCCTTAATAGC	3875	0
+AAACTCGTGCCAGCCACC	3894	0
+AAACTGGGATTAGATACC	3896	0
+AAAGTCCTACGTGATCTG	3901	0
+AAATCATGATGCAAAAGG	3894	0
+AACAAACGAACCCTTAAT	3888	0
+AACATCGAGGTCGTAAAC	3893	0
+AACATGGTAAGTGTACCG	3894	0
+AACCAAAAACTCTTGGTG	3854	0
+AACCCAAAGGACTTGGCG	3888	0
+AACCCTTAATAGCGGCTG	3858	0
+AACCGCGGTGGCTGGCAC	3896	0
+AACGAACCCTTAATAGCG	3860	0
+AACGATTAAAGTCCTACG	3700	0
+AACGTCAGGTCGAGGTGT	3894	0
+AACTCAGATCACGTAGGA	3899	0
+AACTCGTGCCAGCCACCG	3893	0
+AACTGGGATTAGATACCC	3896	0
+AAGAGTTTTTGGTTCCTA	3848	0
+AAGCCTCGCCTGTTTACC	3876	0
+AAGGGTTCGTTTGTTCAA	3889	0
+AAGGTAGCGCAATCACTT	3889	0
+AAGTCCTACGTGATCTGA	3900	0
+AAGTCGTAACATGGTAAG	3893	0
+AAGTGCACCTTCCGGTAC	3888	0
+AAGTGTACCGGAAGGTGC	3889	0
+AATAGCGGCTGCACCATT	3839	0
+AATCACTTGTCTTTTAAA	3856	0
+AATCATGATGCAAAAGGT	3893	0
+AATCCAGGTCAGTTTCTA	3903	0
+AATCGTTGAACAAACGAA	3876	0
+AATGTAGCCCATTTCTTC	3871	0
+ACAAACGAACCCTTAATA	3885	0
+ACAAACTGGGATTAGATA	3893	0
+ACAAGTGATTGCGCTACC	3889	0
+ACACACCGCCCGTCACTC	3891	0
+ACACCGCCCGTCACTCTC	3890	0
+ACACTTACCATGTTACGA	3893	0
+ACATCCTAATGGTGCAGC	3858	0
+ACATCGAGGTCGTAAACC	3894	0
+ACATGGTAAGTGTACCGG	3894	0
+ACCAAAAACTCTTGGTGC	3875	0
+ACCAAGAGTTTTTGGTTC	3857	0
+ACCATGTTACGACTTGCC	3891	0
+ACCATTAGGATGTCCTGA	3891	0
+ACCCAAAGGACTTGGCGG	3886	0
+ACCCGTCTCTGTGGCAAA	3892	0
+ACCCTAGGGATAACAGCG	3895	0
+ACCCTTAATAGCGGCTGC	3861	0
+ACCGCCCGTCACTCTCCC	3893	0
+ACCGCGGTGGCTGGCACG	3895	0
+ACCGGAAGGTGCACTTGG	3887	0
+ACCTCGATGTTGGATCAG	3891	0
+ACCTTCCGGTACACTTAC	3893	0
+ACGAACCCTTAATAGCGG	3859	0
+ACGACCTCGATGTTGGAT	3889	0
+ACGATTAAAGTCCTACGT	3701	0
+ACGTCAGGTCGAGGTGTA	3894	0
+ACGTGATCTGAGTTCAGA	3901	0
+ACTCAGATCACGTAGGAC	3899	0
+ACTCGTGCCAGCCACCGC	3894	0
+ACTCTTTTGCCACAGAGA	3867	0
+ACTGGGATTAGATACCCC	3896	0
+ACTTACCATGTTACGACT	3894	0
+ACTTTAATCGTTGAACAA	3876	0
+AGAACAGGCTCCTCTAGG	3893	0
+AGACAAGTGATTGCGCTA	3883	0
+AGAGGAGCCTGTTCTAGA	3876	0
+AGAGTGACGGGCGGTGTG	3890	0
+AGAGTTTTTGGTTCCTAA	3851	0
+AGATAGAAACTGACCTGG	3900	0
+AGATCACGTAGGACTTTA	3901	0
+AGCCGCTATTAAGGGTTC	3856	0
+AGCCTCGCCTGTTTACCA	3889	0
+AGCCTGTTCTAGAACCGA	3871	0
+AGCGGCTGCACCATTAGG	3846	0
+AGGAACCAAAAACTCTTG	3842	0
+AGGACATCCTAATGGTGC	3866	0
+AGGACTTTAATCGTTGAA	3688	0
+AGGAGCCTGTTCTAGAAC	3874	0
+AGGATGTCCTGATCCAAC	3897	0
+AGGCAAGTCGTAACATGG	3868	0
+AGGCGATGTTTTTGGTAA	3894	0
+AGGGATAACAGCGCAATC	3896	0
+AGGGGAGGCAAGTCGTAA	3890	0
+AGGGGGTTTACGACCTCG	3894	0
+AGGGTTCGTTTGTTCAAC	3888	0
+AGGTAGCGCAATCACTTG	3888	0
+AGTCCTACGTGATCTGAG	3900	0
+AGTGGGGTATCTAATCCC	3896	0
+AGTTACCCTAGGGATAAC	3895	0
+AGTTTTTGGTTCCTAAGA	3848	0
+ATAGAAACTGACCTGGAT	3901	0
+ATAGCGGCTGCACCATTA	3837	0
+ATAGTGGGGTATCTAATC	3895	0
+ATCACGTAGGACTTTAAT	3880	0
+ATCACTTGTCTTTTAAAT	3856	0
+ATCAGGACATCCTAATGG	3892	0
+ATCATGATGCAAAAGGTA	3894	0
+ATCCTAATGGTGCAGCCG	3852	0
+ATCGAGGTCGTAAACCCC	3894	0
+ATCGGTTCTAGAACAGGC	3862	0
+ATCGTTGAACAAACGAAC	3877	0
+ATCTGAGTTCAGACCGGA	3902	0
+ATGATGCAAAAGGTACGA	3891	0
+ATGGTAAGTGTACCGGAA	3895	0
+ATGGTGCAGCCGCTATTA	3840	0
+ATGTAGCCCATTTCTTCC	3871	0
+ATGTCCTGATCCAACATC	3892	0
+ATGTTACGACTTGCCTCC	3864	0
+ATGTTGGATCAGGACATC	3893	0
+ATGTTTTTGGTAAACAGG	3887	0
+ATTAGATACCCCACTATG	3896	0
+ATTAGGATGTCCTGATCC	3893	0
+ATTGCGCTGTTATCCCTA	3896	0
+CAAAAACTCTTGGTGCAA	3886	0
+CAAACGAACCCTTAATAG	3882	0
+CAAACTGGGATTAGATAC	3895	0
+CAACATCGAGGTCGTAAA	3894	0
+CAACGATTAAAGTCCTAC	3691	0
+CAAGTCGTAACATGGTAA	3893	0
+CAAGTGCACCTTCCGGTA	3888	0
+CACCATTAGGATGTCCTG	3890	0
+CACCGCCCGTCACTCTCC	3888	0
+CACCTTCCGGTACACTTA	3891	0
+CACGTAGGACTTTAATCG	3702	0
+CACTCTTTTGCCACAGAG	3856	0
+CACTTACCATGTTACGAC	3894	0
+CATCCTAATGGTGCAGCC	3852	0
+CATCGAGGTCGTAAACCC	3894	0
+CATGATGCAAAAGGTACG	3890	0
+CATGGTAAGTGTACCGGA	3894	0
+CATGTTACGACTTGCCTC	3866	0
+CATTAGGATGTCCTGATC	3893	0
+CCAAAAACTCTTGGTGCA	3883	0
+CCAACATCGAGGTCGTAA	3894	0
+CCAAGAGTTTTTGGTTCC	3844	0
+CCACTCTTTTGCCACAGA	3879	0
+CCCACTCTTTTGCCACAG	3879	0
+CCCGTCTCTGTGGCAAAA	3894	0
+CCCTAGGGATAACAGCGC	3898	0
+CCCTTAATAGCGGCTGCA	3853	0
+CCGCCCGTCACTCTCCCC	3890	0
+CCGCGGTGGCTGGCACGA	3895	0
+CCGGTAAAACTCGTGCCA	3847	0
+CCGGTCTGAACTCAGATC	3904	0
+CCGTCTCTGTGGCAAAAG	3894	0
+CCTACGTGATCTGAGTTC	3899	0
+CCTAGGGATAACAGCGCA	3896	0
+CCTCGATGTTGGATCAGG	3892	0
+CCTCGCCTGTTTACCAAA	3892	0
+CCTGTTCTAGAACCGATA	3862	0
+CCTTAATAGCGGCTGCAC	3850	0
+CCTTCCGGTACACTTACC	3893	0
+CGAACCCTTAATAGCGGC	3855	0
+CGACCTCGATGTTGGATC	3890	0
+CGACGGCGGTATATAGGC	3890	0
+CGATGTTGGATCAGGACA	3892	0
+CGATGTTTTTGGTAAACA	3886	0
+CGCCTGTTTACCAAAAAC	3889	0
+CGCGGTGGCTGGCACGAG	3894	0
+CGCTTAGCTGTTAACTAA	3864	0
+CGGTAAAACTCGTGCCAG	3856	0
+CGGTCTGAACTCAGATCA	3904	0
+CGGTTCTAGAACAGGCTC	3873	0
+CGTAACATGGTAAGTGTA	3892	0
+CGTCAGGTCGAGGTGTAG	3893	0
+CGTCTCTGTGGCAAAAGA	3891	0
+CGTGATCTGAGTTCAGAC	3902	0
+CGTTGAACAAACGAACCC	3887	0
+CGTTTGTTCAACGATTAA	3877	0
+CTAAATCATGATGCAAAA	3896	0
+CTAATGGTGCAGCCGCTA	3840	0
+CTACGTGATCTGAGTTCA	3899	0
+CTAGAACAGGCTCCTCTA	3877	0
+CTAGAGGAGCCTGTTCTA	3891	0
+CTAGGGATAACAGCGCAA	3897	0
+CTCCGGTCTGAACTCAGA	3903	0
+CTCGATGTTGGATCAGGA	3892	0
+CTCGCCTGTTTACCAAAA	3892	0
+CTCGTACCTTTTGCATCA	3889	0
+CTCTTTTGCCACAGAGAC	3869	0
+CTGAACTCAGATCACGTA	3900	0
+CTGCACCATTAGGATGTC	3861	0
+CTGGGATTAGATACCCCA	3895	0
+CTGTTATCCCTAGGGTAA	3897	0
+CTGTTTACCAAAAACATC	3886	0
+CTTAATAGCGGCTGCACC	3848	0
+CTTAGCTGTTAACTAAGA	3878	0
+CTTAGGAACCAAAAACTC	3847	0
+CTTAGTTAACAGCTAAGC	3875	0
+CTTCCGGTACACTTACCA	3891	0
+CTTGGATTTGCACCAAGA	3899	0
+CTTTAATCGTTGAACAAA	3876	0
+CTTTTGCATCATGATTTA	3896	0
+GAACAAACGAACCCTTAA	3890	0
+GAAGGAACTCGGCAAACA	3889	0
+GACAAGTGATTGCGCTAC	3889	0
+GACCTCGATGTTGGATCA	3890	0
+GACTTTAATCGTTGAACA	3686	0
+GAGGAGCCTGTTCTAGAA	3876	0
+GAGGCGATGTTTTTGGTA	3893	0
+GATAGAAACTGACCTGGA	3900	0
+GATCACGTAGGACTTTAA	3900	0
+GATTAAAGTCCTACGTGA	3704	0
+GCAAGTCGTAACATGGTA	3891	0
+GCATAGTGGGGTATCTAA	3896	0
+GCCGGTAAAACTCGTGCC	3827	0
+GCCTCGCCTGTTTACCAA	3890	0
+GCCTGTTTACCAAAAACA	3889	0
+GCGATGTTTTTGGTAAAC	3888	0
+GCGCTTAGCTGTTAACTA	3899	0
+GCGGCTGCACCATTAGGA	3850	0
+GCTAAATCATGATGCAAA	3893	0
+GCTACACCTCGACCTGAC	3893	0
+GCTGGCACGAGTTTTACC	3881	0
+GCTGTTATCCCTAGGGTA	3896	0
+GGACATCCTAATGGTGCA	3865	0
+GGACTTTAATCGTTGAAC	3687	0
+GGAGCCTGTTCTAGAACC	3874	0
+GGATGTCCTGATCCAACA	3895	0
+GGATTAGATACCCCACTA	3896	0
+GGATTGCGCTGTTATCCC	3895	0
+GGCGATGTTTTTGGTAAA	3890	0
+GGCTGGCACGAGTTTTAC	3880	0
+GGGAAGAAATGGGCTACA	3872	0
+GGGAGGCAAGTCGTAACA	3891	0
+GGGGAGGCAAGTCGTAAC	3890	0
+GGGGGTTTACGACCTCGA	3894	0
+GGTACACTTACCATGTTA	3893	0
+GGTCTGAACTCAGATCAC	3902	0
+GGTCTTAGGAACCAAAAA	3862	0
+GGTTCGTTTGTTCAACGA	3877	0
+GTAACATGGTAAGTGTAC	3893	0
+GTACCTTTTGCATCATGA	3892	0
+GTCCTGATCCAACATCGA	3892	0
+GTCTTAGGAACCAAAAAC	3853	0
+GTGGGGTATCTAATCCCA	3895	0
+GTTACCCTAGGGATAACA	3896	0
+GTTTGTTCAACGATTAAA	3877	0
+TAAAACTCGTGCCAGCCA	3887	0
+TAACCGCGGTGGCTGGCA	3895	0
+TAAGGGTTCGTTTGTTCA	3889	0
+TAATCGTTGAACAAACGA	3876	0
+TACGACCTCGATGTTGGA	3892	0
+TAGGACTTTAATCGTTGA	3689	0
+TAGGATGTCCTGATCCAA	3897	0
+TCCCACTCTTTTGCCACA	3866	0
+TCGCCTGTTTACCAAAAA	3891	0
+TGGATCAGGACATCCTAA	3894	0
+TGGTCTTAGGAACCAAAA	3862	0
+TGGTGCAGCCGCTATTAA	3847	0
+TTGGTCTTAGGAACCAAA	3862	0

pkg/obiprimer/words.go

@@ -0,0 +1,97 @@
+package obiprimer
+
+import (
+	"fmt"
+	"math"
+	"sync"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obikmer"
+)
+
+// WordSearch holds the parameters of the strict word search.
+type WordSearch struct {
+	PrimerLength int     // word size (ecoPrimers -O, 18 by default; 2..31)
+	Quorum       float64 // fraction of example sequences that must contain the word (ecoPrimers -q, 0.7)
+	Circular     bool    // sequences are circular (ecoPrimers -c)
+	Partitions   int     // partition files of the counter (0 = chosen from the size of the examples, see obikmer.DefaultPartitions)
+	TmpDir       string  // parent of the temporary directory ("" = system default)
+}
+
+// StrictWords is the result of the strict word search: the canonical words of
+// PrimerLength bases present, without any error, in at least MinSequences
+// example sequences.
+type StrictWords struct {
+	PrimerLength int
+	MinSequences int // quorum in number of example sequences
+	Words        []obikmer.SeqKmerCount
+	Counts       Counts
+}
+
+// MinSequences returns the quorum in sequences: floor(examples * quorum).
+// The tiny epsilon makes exact products (10 * 0.7 = 7) robust to the binary
+// representation of the fraction.
+func MinSequences(examples int, quorum float64) int {
+	return int(math.Floor(float64(examples)*quorum + 1e-9))
+}
+
+// FindStrictWords indexes the example sequences of the corpus and returns the
+// words reaching the quorum. Sequences no longer than the word size are
+// counted as examples but cannot contain any word, as in ecoPrimers. Words are
+// canonical (the smaller of the word and its reverse complement), so both
+// strands are searched.
+func FindStrictWords(c *Corpus, cfg WordSearch) (*StrictWords, error) {
+	if cfg.PrimerLength < 2 || cfg.PrimerLength > 31 {
+		return nil, fmt.Errorf("obiprimer: primer length must be between 2 and 31, got %d", cfg.PrimerLength)
+	}
+	if cfg.Quorum <= 0 || cfg.Quorum > 1 {
+		return nil, fmt.Errorf("obiprimer: quorum must be in ]0,1], got %g", cfg.Quorum)
+	}
+	if cfg.Partitions <= 0 {
+		// the k-mer occurrences to count: those of the examples, which are the
+		// only sequences indexed
+		total := 0
+		for i, r := range c.Roles {
+			if r == Example && c.SeqLen(i) > cfg.PrimerLength {
+				total += c.SeqLen(i)
+			}
+		}
+		cfg.Partitions = obikmer.DefaultPartitions(total)
+	}
+
+	counter, err := obikmer.NewSeqKmerCounter(cfg.TmpDir, cfg.PrimerLength, 0, cfg.Partitions)
+	if err != nil {
+		return nil, err
+	}
+	defer counter.Close()
+
+	jobs := make(chan int)
+	var wg sync.WaitGroup
+	for range obidefault.ParallelWorkers() {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			var buf []byte
+			for i := range jobs {
+				if c.Roles[i] == Example && c.SeqLen(i) > cfg.PrimerLength {
+					var raw []byte
+					raw, buf = c.bases(i, buf)
+					counter.AddSequence(uint32(i), raw, cfg.Circular)
+				}
+			}
+		}()
+	}
+	for i := range c.Roles {
+		jobs <- i
+	}
+	close(jobs)
+	wg.Wait()
+
+	res := &StrictWords{PrimerLength: cfg.PrimerLength, Counts: c.Counts()}
+	res.MinSequences = MinSequences(res.Counts.Examples, cfg.Quorum)
+	res.Words, err = counter.CountAtLeast(max(res.MinSequences, 1))
+	if err != nil {
+		return nil, err
+	}
+	return res, nil
+}

pkg/obiprimer/words_test.go

@@ -0,0 +1,233 @@
+package obiprimer
+
+import (
+	"math/rand"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiformats"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obikmer"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+func idPrefix(prefix string) obiseq.SequencePredicate {
+	return func(s *obiseq.BioSequence) bool { return strings.HasPrefix(s.Id(), prefix) }
+}
+
+func TestSelectionRoles(t *testing.T) {
+	seq := func(id string) *obiseq.BioSequence { return obiseq.NewBioSequence(id, []byte("acgt"), "") }
+	// ids: "r" restricted clade, "x" exception inside it, "i" ignored clade, "o" other
+	cases := []struct {
+		name string
+		sel  Selection
+		want map[string]Role
+	}{
+		{"nothing set: everything is an example",
+			Selection{}, map[string]Role{"r1": Example, "x1": Example, "i1": Example, "o1": Example}},
+		{"restrict only: the rest are non-targets",
+			Selection{Restrict: idPrefix("r")},
+			map[string]Role{"r1": Example, "x1": Other, "i1": Other, "o1": Other}},
+		{"restrict + ignore",
+			Selection{Restrict: idPrefix("r"), Ignore: idPrefix("i")},
+			map[string]Role{"r1": Example, "i1": CounterExample, "o1": Other}},
+		{"exception inside restrict becomes a counterexample",
+			Selection{Restrict: idPrefix("r").Or(idPrefix("x")), Except: idPrefix("x")},
+			map[string]Role{"r1": Example, "x1": CounterExample, "o1": Other}},
+		{"example wins over ignore when both match",
+			Selection{Restrict: idPrefix("r"), Ignore: idPrefix("r").Or(idPrefix("i"))},
+			map[string]Role{"r1": Example, "i1": CounterExample, "o1": Other}},
+	}
+	for _, tc := range cases {
+		for id, want := range tc.want {
+			if got := tc.sel.Role(seq(id)); got != want {
+				t.Errorf("%s: %s: role %d, want %d", tc.name, id, got, want)
+			}
+		}
+	}
+}
+
+func TestMinSequences(t *testing.T) {
+	for _, tc := range []struct {
+		n    int
+		q    float64
+		want int
+	}{{10, 0.7, 7}, {3909, 0.7, 2736}, {325, 0.7, 227}, {100, 0.9, 90}, {7, 0.9, 6}, {1, 1, 1}} {
+		if got := MinSequences(tc.n, tc.q); got != tc.want {
+			t.Errorf("MinSequences(%d,%g) = %d, want %d", tc.n, tc.q, got, tc.want)
+		}
+	}
+}
+
+func revcomp(s string) string {
+	c := map[byte]byte{'A': 'T', 'C': 'G', 'G': 'C', 'T': 'A'}
+	out := make([]byte, len(s))
+	for i := range s {
+		out[len(s)-1-i] = c[s[i]]
+	}
+	return string(out)
+}
+
+func randDNA(rng *rand.Rand, n int) string {
+	b := make([]byte, n)
+	for i := range b {
+		b[i] = "ACGT"[rng.Intn(4)]
+	}
+	return string(b)
+}
+
+func TestFindStrictWords(t *testing.T) {
+	const k = 18
+	rng := rand.New(rand.NewSource(1))
+	motif := randDNA(rng, 30)
+
+	var data obiseq.BioSequenceSlice
+	add := func(id, s string) { data = append(data, obiseq.NewBioSequence(id, []byte(s), "")) }
+
+	// 10 examples; the motif is in 8 of them, 3 on the reverse strand. One
+	// example carries it twice. Another has an ambiguity breaking it.
+	for i := 0; i < 8; i++ {
+		m := motif
+		if i%3 == 0 {
+			m = revcomp(motif)
+		}
+		s := randDNA(rng, 120) + m + randDNA(rng, 120)
+		if i == 5 {
+			s += randDNA(rng, 50) + motif
+		}
+		add("ex"+string(rune('a'+i)), s)
+	}
+	broken := motif[:12] + "N" + motif[13:]
+	add("exh", randDNA(rng, 100)+broken+randDNA(rng, 100))
+	add("exi", randDNA(rng, 300))
+	// counterexamples and ignored sequences must not count
+	add("cx1", randDNA(rng, 50)+motif)
+	add("zz1", randDNA(rng, 50)+motif)
+
+	sel := Selection{Restrict: idPrefix("ex").Or(idPrefix("cx")), Ignore: idPrefix("cx"), Except: idPrefix("cx")}
+	corpus, err := LoadCorpus(obiiter.IBatchOver("t", data, 3), sel)
+	if err != nil {
+		t.Fatal(err)
+	}
+	res, err := FindStrictWords(corpus, WordSearch{PrimerLength: k, Quorum: 0.7, Partitions: 8, TmpDir: t.TempDir()})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if res.Counts != (Counts{Examples: 10, CounterExamples: 1, Others: 1}) {
+		t.Fatalf("counts = %+v", res.Counts)
+	}
+	if res.MinSequences != 7 {
+		t.Fatalf("quorum = %d sequences, want 7", res.MinSequences)
+	}
+
+	want := map[uint64]bool{}
+	for i := 0; i+k <= len(motif); i++ {
+		want[obikmer.EncodeCanonicalKmer([]byte(motif[i:i+k]), k)] = true
+	}
+	// the N breaks the windows that cover it, but 8 sequences still carry all of them
+	if len(res.Words) != len(want) {
+		t.Fatalf("%d words, want %d (the motif windows)", len(res.Words), len(want))
+	}
+	for _, w := range res.Words {
+		if !want[w.Kmer] {
+			t.Errorf("unexpected word %d", w.Kmer)
+		}
+		if w.Sequences != 8 {
+			t.Errorf("word %d in %d sequences, want 8", w.Kmer, w.Sequences)
+		}
+	}
+	repeated := 0
+	for _, w := range res.Words {
+		if w.Repeated {
+			repeated++
+		}
+	}
+	if repeated != len(want) {
+		t.Errorf("%d words flagged Repeated, want all %d (motif twice in one sequence)", repeated, len(want))
+	}
+}
+
+func TestFindStrictWordsParameters(t *testing.T) {
+	it := func() obiiter.IBioSequence {
+		return obiiter.IBatchOver("t", obiseq.BioSequenceSlice{obiseq.NewBioSequence("a", []byte("acgtacgtacgtacgtacgtacgt"), "")}, 1)
+	}
+	corpus, err := LoadCorpus(it(), Selection{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := FindStrictWords(corpus, WordSearch{PrimerLength: 1, Quorum: 0.7}); err == nil {
+		t.Error("primer length 1 must fail")
+	}
+	if _, err := FindStrictWords(corpus, WordSearch{PrimerLength: 18, Quorum: 0}); err == nil {
+		t.Error("quorum 0 must fail")
+	}
+	if _, err := LoadCorpus(it(), Selection{Restrict: idPrefix("zz")}); err == nil {
+		t.Error("no example must fail")
+	}
+}
+
+// Opt-in end-to-end check against ecoPrimers on the "Designing fish primers"
+// tutorial data. Set OBIPRIMERS_TUTORIAL to the directory holding
+// ncbitaxo_20250211.tgz and mito.vert.fasta.gz.
+//
+// ecoPrimers (-e 3 -3 2 -l 30 -L 150 -c, k=18, strict quorum 0.7) reports
+// "Strict primer count": 1208 for Cyprinidae (-r 7953, 325 examples) and 255
+// for Teleostei (-r 32443, 3909 examples).
+//
+// Teleostei matches exactly. For Cyprinidae we find 1207: ecoPrimers keeps
+// TTTATGAAAACAAATAAA, which a brute-force scan (both strands, circular
+// junction) finds in only 225 of the 325 examples, below its own quorum of
+// 227. That word is in none of the 3448 pairs ecoPrimers finally retains, so
+// the divergence has no effect on the reference results.
+func TestStrictWordsAgainstEcoPrimers(t *testing.T) {
+	dir := os.Getenv("OBIPRIMERS_TUTORIAL")
+	if dir == "" {
+		t.Skip("OBIPRIMERS_TUTORIAL not set")
+	}
+
+	taxonomy, err := obiformats.LoadTaxonomy(filepath.Join(dir, "ncbitaxo_20250211.tgz"), false, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, tc := range []struct {
+		name     string
+		taxid    string
+		examples int
+		words    int
+	}{
+		{"Cyprinidae", "7953", 325, 1207}, // ecoPrimers: 1208, see above
+		{"Teleostei", "32443", 3909, 255},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			restrict, err := BelongingTo(taxonomy, []string{tc.taxid})
+			if err != nil {
+				t.Fatal(err)
+			}
+			// The predicates read the taxon through the taxid annotation.
+			seqs, err := obiformats.ReadSequencesFromFile(filepath.Join(dir, "mito.vert.fasta.gz"))
+			if err != nil {
+				t.Fatal(err)
+			}
+			corpus, err := LoadCorpus(seqs, Selection{Restrict: restrict})
+			if err != nil {
+				t.Fatal(err)
+			}
+			res, err := FindStrictWords(corpus,
+				WordSearch{PrimerLength: 18, Quorum: 0.7, Circular: true, TmpDir: t.TempDir()})
+			if err != nil {
+				t.Fatal(err)
+			}
+			t.Logf("%+v min=%d words=%d", res.Counts, res.MinSequences, len(res.Words))
+			if res.Counts.Examples != tc.examples {
+				t.Errorf("%d examples, ecoPrimers reports %d", res.Counts.Examples, tc.examples)
+			}
+			if len(res.Words) != tc.words {
+				t.Errorf("%d strict words, ecoPrimers reports %d", len(res.Words), tc.words)
+			}
+		})
+	}
+}

pkg/obiseq/iupac_nog.go

@@ -26,3 +26,16 @@ func SameIUPACNuc(a, b byte) bool {
 	}
 	return a == b
 }
+
+// IUPACNucMask returns the set of nucleotides denoted by the IUPAC code b
+// (case insensitive, U is read as T) as a 4 bits mask: bit 0 is A, bit 1 is C,
+// bit 2 is G and bit 3 is T. It returns 0 if b is not an IUPAC nucleotide code.
+func IUPACNucMask(b byte) byte {
+	if (b >= 'A') && (b <= 'Z') {
+		b |= 32
+	}
+	if (b < 'a') || (b > 'z') {
+		return 0
+	}
+	return _iupac[b-'a']
+}

pkg/obiseq/taxonomy_methods.go

@@ -176,6 +176,10 @@ func (sequence *BioSequence) SetFamily(taxonomy *obitax.Taxonomy) *obitax.Taxon
 	return sequence.SetTaxonAtRank(taxonomy, "family")
 }
 
+// TaxonomicPathAttribute is the attribute holding the taxonomic path of a
+// sequence, from the root to its taxon (see obiannotate --taxonomic-path).
+const TaxonomicPathAttribute = "taxonomic_path"
+
 func (sequence *BioSequence) SetPath(taxonomy *obitax.Taxonomy) []string {
 	taxon := sequence.Taxon(taxonomy)
 	path := taxon.Path()
@@ -186,13 +190,13 @@ func (sequence *BioSequence) SetPath(taxonomy *obitax.Taxonomy) []string {
 		spath[lpath-i] = path.Get(i).FullString(taxonomy.Code())
 	}
 
-	sequence.SetAttribute("taxonomic_path", spath)
+	sequence.SetAttribute(TaxonomicPathAttribute, spath)
 
 	return spath
 }
 
 func (sequence *BioSequence) Path() []string {
-	path, ok := sequence.GetAttribute("taxonomic_path")
+	path, ok := sequence.GetAttribute(TaxonomicPathAttribute)
 
 	if !ok {
 		if taxo := obitax.DefaultTaxonomy(); taxo != nil {

pkg/obisimilarity/block.go

@@ -0,0 +1,46 @@
+package obisimilarity
+
+// A block stores the 4-mer tables of a few sequences, transposed: the counter
+// of the word w for the lane j is data[w*width+j]. A vector instruction then
+// handles the same word for all the sequences of the block, and only the words
+// that a sequence contains (a hundred for a short amplicon, out of 256) need
+// to be visited to compare it with all the sequences of the block.
+type tableBlock struct {
+	data []uint8
+}
+
+func newTableBlock(width int) tableBlock {
+	return tableBlock{data: make([]uint8, 256*width)}
+}
+
+// set stores the table t as the lane `lane` of the block.
+func (b *tableBlock) set(width, lane int, t *Table4) {
+	for w := range t.Count {
+		b.data[w*width+lane] = t.Count[w]
+	}
+}
+
+// common computes the sums of the minima of the counters of the sequence
+// given by its words with every lane of the block (see blockCommon).
+func (b *tableBlock) common(width int, words []uint16, out []uint8) {
+	blockCommon(b.data, width, words, out)
+}
+
+// SparseWords lists the words of a table as w<<8|count, so that the visit of
+// a block only goes through the words present in the sequence. nil is
+// returned when the counters add up to more than 255, which would overflow the
+// 8-bit sums of the blocks.
+func SparseWords(t *Table4) []uint16 {
+	words := make([]uint16, 0, 128)
+	mass := 0
+	for w, c := range t.Count {
+		if c > 0 {
+			words = append(words, uint16(w)<<8|uint16(c))
+			mass += int(c)
+		}
+	}
+	if mass > 255 {
+		return nil
+	}
+	return words
+}

pkg/obisimilarity/block_nosimd.go

@@ -0,0 +1,10 @@
+//go:build !goexperiment.simd
+
+package obisimilarity
+
+// simdBlockWidth is 0 without SIMD: the blocks are not used.
+func simdBlockWidth() int { return 0 }
+
+func blockCommon(data []uint8, width int, words []uint16, out []uint8) {
+	panic("obisimilarity: blocks of tables require GOEXPERIMENT=simd")
+}

pkg/obisimilarity/block_simd.go

@@ -0,0 +1,25 @@
+//go:build goexperiment.simd
+
+package obisimilarity
+
+import "simd"
+
+// simdBlockWidth is the number of sequences in a block: one per lane of a
+// vector of bytes.
+func simdBlockWidth() int {
+	var probe simd.Uint8s
+	return min(probe.Len(), 64)
+}
+
+// blockCommon computes, for the sequence whose words are given, the sum of
+// the minima of the counters with every lane of the block. It is the value of
+// commonWords without the overflow term, as the sums fit in 8 bits (see
+// SparseWords). out must hold at least `width` bytes.
+func blockCommon(data []uint8, width int, words []uint16, out []uint8) {
+	acc := simd.BroadcastUint8s(0)
+	for _, e := range words {
+		w := simd.LoadUint8s(data[int(e>>8)*width:])
+		acc = acc.Add(w.Min(simd.BroadcastUint8s(uint8(e))))
+	}
+	acc.Store(out)
+}

pkg/obisimilarity/entry.go

@@ -0,0 +1,95 @@
+package obisimilarity
+
+import (
+	"sync"
+	"sync/atomic"
+)
+
+// Entry is a distinct sequence of a dataset, with what the comparisons need.
+type Entry struct {
+	Seq    []byte
+	Length int
+	Count  int // count of the sequence (cumulated for identical sequences)
+	Table  *Table4
+	Sparse []uint16 // words of the table (see SparseWords), nil if unusable
+	// Ambiguous sequences contain IUPAC letters other than a, c, g and t: they
+	// have no 4-mer table, and the filter does not apply to them.
+	Ambiguous bool
+}
+
+const (
+	notNucleotide = iota
+	plainNucleotide
+	ambiguousNucleotide
+)
+
+var nucleotideClass = func() (t [256]uint8) {
+	for _, c := range "acgt" {
+		t[c] = plainNucleotide
+	}
+	for _, c := range "umrwsykvhdbn" {
+		t[c] = ambiguousNucleotide
+	}
+	return t
+}()
+
+// Classify tells if a sequence (in lower case) can be compared. It returns
+// ok=false for an empty sequence and for a sequence with anything else than
+// IUPAC letters; IUPAC letters other than a, c, g and t are only accepted if
+// acceptAmbiguous is set, and then ambiguous is true.
+func Classify(s []byte, acceptAmbiguous bool) (ok, ambiguous bool) {
+	if len(s) == 0 {
+		return false, false
+	}
+	for _, c := range s {
+		switch nucleotideClass[c] {
+		case notNucleotide:
+			return false, false
+		case ambiguousNucleotide:
+			if !acceptAmbiguous {
+				return false, false
+			}
+			ambiguous = true
+		}
+	}
+	return true, ambiguous
+}
+
+// BuildEntries describes n sequences, their 4-mer tables being computed in
+// parallel. get returns the sequence i, its count, and whether it is
+// ambiguous. The tables are stored contiguously.
+func BuildEntries(n int, get func(i int) (seq []byte, count int, ambiguous bool), workers int) []Entry {
+	entries := make([]Entry, n)
+	tables := make([]Table4, n)
+
+	var next atomic.Int64
+	var wg sync.WaitGroup
+	for w := 0; w < max(workers, 1); w++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for {
+				i := int(next.Add(1)) - 1
+				if i >= n {
+					return
+				}
+				seq, count, ambiguous := get(i)
+				var sparse []uint16
+				if !ambiguous {
+					tables[i].Fill(seq)
+					sparse = SparseWords(&tables[i])
+				}
+				entries[i] = Entry{
+					Seq:       seq,
+					Length:    len(seq),
+					Count:     count,
+					Table:     &tables[i],
+					Sparse:    sparse,
+					Ambiguous: ambiguous,
+				}
+			}
+		}()
+	}
+	wg.Wait()
+	return entries
+}

pkg/obisimilarity/index.go

@@ -0,0 +1,106 @@
+package obisimilarity
+
+import "sort"
+
+// Member is a copy of the entry of a sequence in an Index, with its own table.
+type Member struct {
+	Entry
+	ID   int // identifier given by the caller
+	Rank int // rank in the order of insertion
+}
+
+const tabChunk = 1024
+
+// A Bucket holds the members of one length, in the order of insertion, and
+// their tables by blocks (see tableBlock).
+type Bucket struct {
+	Members []Member
+	blocks  []tableBlock
+	width   int
+}
+
+// Commons caches the result of the filter on one block of a bucket.
+type Commons struct {
+	v   [64]uint8
+	blk int
+}
+
+// Reset must be called when changing of bucket.
+func (c *Commons) Reset() { c.blk = -1 }
+
+// Common returns the sum of the minima of the 4-mer counters of the sequence
+// whose Sparse words are given and of the member k of the bucket (without the
+// overflow term), or -1 if it is not available, in which case PairScore
+// computes it.
+func (b *Bucket) Common(k int, sparse []uint16, c *Commons) int {
+	if b.width == 0 || sparse == nil {
+		return -1
+	}
+	if blk := k / b.width; blk != c.blk {
+		b.blocks[blk].common(b.width, sparse, c.v[:])
+		c.blk = blk
+	}
+	return int(c.v[k%b.width])
+}
+
+// Index stores sequences by length. A sequence can only be similar to the ones
+// whose length is compatible with its own (see Scoring.LengthCompatible), so
+// it only visits the buckets of these lengths, instead of all the sequences,
+// and the filter works on blocks of sequences at once.
+//
+// An Index can grow, but not while it is read.
+type Index struct {
+	width int // sequences per block of transposed tables, 0 if unused
+	byLen map[int]*Bucket
+	lens  []int      // lengths that have a bucket, increasing
+	tabs  [][]Table4 // chunks holding the tables of the members
+	n     int
+}
+
+// NewIndex returns an empty index.
+func NewIndex() *Index {
+	return &Index{width: simdBlockWidth(), byLen: make(map[int]*Bucket)}
+}
+
+// Len is the number of sequences in the index.
+func (x *Index) Len() int { return x.n }
+
+// Lengths returns the lengths that have a bucket, in increasing order.
+func (x *Index) Lengths() []int { return x.lens }
+
+// Bucket returns the bucket of the sequences of the given length.
+func (x *Index) Bucket(length int) *Bucket { return x.byLen[length] }
+
+// Add stores a copy of the entry e as the next member of the index, with the
+// identifier id, and returns it.
+func (x *Index) Add(e Entry, id int) Member {
+	if x.n%tabChunk == 0 {
+		x.tabs = append(x.tabs, make([]Table4, tabChunk))
+	}
+	tab := &x.tabs[len(x.tabs)-1][x.n%tabChunk]
+	if e.Table != nil {
+		*tab = *e.Table
+	}
+	e.Table = tab
+
+	b := x.byLen[e.Length]
+	if b == nil {
+		b = &Bucket{width: x.width}
+		x.byLen[e.Length] = b
+		x.lens = append(x.lens, e.Length)
+		sort.Ints(x.lens)
+	}
+	if w := x.width; w > 0 && !e.Ambiguous {
+		if len(b.Members)%w == 0 {
+			b.blocks = append(b.blocks, newTableBlock(w))
+		}
+		b.blocks[len(b.blocks)-1].set(w, len(b.Members)%w, tab)
+	} else if w > 0 && len(b.Members)%w == 0 {
+		b.blocks = append(b.blocks, newTableBlock(w))
+	}
+
+	m := Member{Entry: e, ID: id, Rank: x.n}
+	b.Members = append(b.Members, m)
+	x.n++
+	return m
+}

pkg/obisimilarity/obisimilarity_test.go

@@ -0,0 +1,233 @@
+package obisimilarity
+
+import (
+	"math/rand"
+	"testing"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obialign"
+)
+
+func randomSeq(rng *rand.Rand, n int, alpha string) []byte {
+	s := make([]byte, n)
+	for i := range s {
+		s[i] = alpha[rng.Intn(len(alpha))]
+	}
+	return s
+}
+
+// commonWordsRef is the definition of commonWords, spelled out.
+func commonWordsRef(t1, t2 *Table4) int {
+	total := 0
+	for i := range t1.Count {
+		total += int(min(t1.Count[i], t2.Count[i]))
+	}
+	return total + min(t1.Overflow, t2.Overflow)
+}
+
+func TestCommonWords(t *testing.T) {
+	rng := rand.New(rand.NewSource(3))
+	for it := 0; it < 3000; it++ {
+		alpha := []string{"acgt", "ac", "a"}[it%3]
+		t1 := NewTable4(randomSeq(rng, rng.Intn(1200), alpha))
+		t2 := NewTable4(randomSeq(rng, rng.Intn(1200), alpha))
+		if got, want := commonWords(t1, t2), commonWordsRef(t1, t2); got != want {
+			t.Fatalf("commonWords = %d, want %d", got, want)
+		}
+	}
+}
+
+// TestNewTable4 checks the table against the 4-mers counted the long way,
+// including the saturation of the counters.
+func TestNewTable4(t *testing.T) {
+	code := map[byte]int{'a': 0, 'c': 1, 't': 2, 'g': 3}
+	rng := rand.New(rand.NewSource(4))
+	for it := 0; it < 500; it++ {
+		alpha := []string{"acgt", "a", "ac"}[it%3]
+		s := randomSeq(rng, rng.Intn(900), alpha)
+		var counts [256]int
+		for i := 0; i+4 <= len(s); i++ {
+			w := 0
+			for _, c := range s[i : i+4] {
+				w = w<<2 | code[c]
+			}
+			counts[w]++
+		}
+		tab := NewTable4(s)
+		over := 0
+		for w, c := range counts {
+			if c > 255 {
+				over += c - 255
+				c = 255
+			}
+			if int(tab.Count[w]) != c {
+				t.Fatalf("word %d: got %d, want %d", w, tab.Count[w], c)
+			}
+		}
+		if tab.Overflow != over {
+			t.Fatalf("overflow = %d, want %d", tab.Overflow, over)
+		}
+	}
+}
+
+// TestBlockCommon checks the sums of a block of transposed tables against
+// commonWords.
+func TestBlockCommon(t *testing.T) {
+	width := simdBlockWidth()
+	if width == 0 {
+		t.Skip("blocks of tables are used with GOEXPERIMENT=simd only")
+	}
+	rng := rand.New(rand.NewSource(8))
+	for it := 0; it < 200; it++ {
+		alpha := []string{"acgt", "ac", "a"}[it%3]
+		block := newTableBlock(width)
+		tabs := make([]*Table4, width)
+		for j := range tabs {
+			tabs[j] = NewTable4(randomSeq(rng, rng.Intn(600), alpha))
+			block.set(width, j, tabs[j])
+		}
+		q := NewTable4(randomSeq(rng, rng.Intn(260), alpha))
+		words := SparseWords(q)
+		if words == nil {
+			continue
+		}
+		out := make([]uint8, 64)
+		block.common(width, words, out)
+		for j, tab := range tabs {
+			want := commonWordsRef(q, tab) - min(q.Overflow, tab.Overflow)
+			if int(out[j]) != want {
+				t.Fatalf("lane %d: %d, want %d", j, out[j], want)
+			}
+		}
+	}
+}
+
+func TestClassify(t *testing.T) {
+	cases := []struct {
+		in        string
+		accept    bool
+		kept      bool
+		ambiguous bool
+	}{
+		{"acgt", false, true, false}, {"acgt", true, true, false},
+		{"acgtn", false, false, false}, {"acgtn", true, true, true},
+		{"acgtx", true, false, false}, {"nacgt", false, false, false},
+		{"acgt-", true, false, false}, {"acgu", true, true, true},
+		{"ACGT", true, false, false}, {"", true, false, false},
+		{"rykmswbdhvn", true, true, true}, {"rykmswbdhvn", false, false, false},
+	}
+	for _, c := range cases {
+		kept, amb := Classify([]byte(c.in), c.accept)
+		if kept != c.kept || amb != c.ambiguous {
+			t.Errorf("Classify(%q, %v) = (%v, %v), want kept=%v ambiguous=%v", c.in, c.accept, kept, amb, c.kept, c.ambiguous)
+		}
+	}
+}
+
+func randomEntries(rng *rand.Rand, n int) []Entry {
+	return BuildEntries(n, func(int) ([]byte, int, bool) {
+		return randomSeq(rng, 8+rng.Intn(80), "acgt"), 1 + rng.Intn(20), false
+	}, 2)
+}
+
+// TestLengthTable checks that the values precomputed by length give the
+// scores of the direct computation, in every mode.
+func TestLengthTable(t *testing.T) {
+	rng := rand.New(rand.NewSource(33))
+	// sequences made of the same few roots, so that many pairs are similar
+	roots := [][]byte{randomSeq(rng, 70, "acgt"), randomSeq(rng, 55, "acgt")}
+	entries := BuildEntries(120, func(i int) ([]byte, int, bool) {
+		var s []byte
+		for _, c := range roots[i%2] {
+			switch r := rng.Intn(30); {
+			case r == 0:
+				s = append(s, "acgt"[rng.Intn(4)])
+			case r == 1:
+			default:
+				s = append(s, c)
+			}
+		}
+		if i < 20 { // very short sequences skip the filter
+			s = s[:8+rng.Intn(10)]
+		}
+		return s, 1 + rng.Intn(20), false
+	}, 2)
+
+	var buf obialign.BandedLCSBuffer
+	for _, ref := range []Reference{RefAlignment, RefLongest, RefShortest} {
+		for _, p := range []Params{
+			{Threshold: 0.9, Normalized: true},
+			{Threshold: 0.8, Normalized: true, Distance: true},
+			{Threshold: 30},
+			{Threshold: 6, Distance: true},
+			{Normalized: true}, // no threshold
+		} {
+			p.Reference, p.Ratio = ref, 3
+			table, err := NewScoring(p, entries)
+			if err != nil {
+				t.Fatal(err)
+			}
+			direct := *table
+			direct.byLength = nil // forces the direct computation
+			if (ref == RefShortest) != (table.byLength == nil) {
+				t.Fatalf("ref %v: table presence is wrong", ref)
+			}
+			for i := range entries {
+				for j := range entries {
+					a, aok := direct.PairScore(&entries[i], &entries[j], -1, &buf)
+					b, bok := table.PairScore(&entries[i], &entries[j], -1, &buf)
+					if a != b || aok != bok {
+						t.Fatalf("ref %v params %+v pair %d,%d: (%v,%v) with the table, (%v,%v) directly", ref, p, i, j, b, bok, a, aok)
+					}
+				}
+			}
+		}
+	}
+}
+
+// TestIndexCommon checks that the filter computed by blocks gives the same
+// scores as the direct computation, for every pair, whatever the order the
+// members are read in.
+func TestIndexCommon(t *testing.T) {
+	rng := rand.New(rand.NewSource(7))
+	entries := randomEntries(rng, 150)
+	sc, err := NewScoring(Params{Threshold: 0.6, Normalized: true, Reference: RefAlignment, Ratio: 100}, entries)
+	if err != nil {
+		t.Fatal(err)
+	}
+	index := NewIndex()
+	for i := range entries {
+		index.Add(entries[i], i)
+	}
+	if index.Len() != len(entries) {
+		t.Fatalf("Len = %d", index.Len())
+	}
+
+	var buf obialign.BandedLCSBuffer
+	var commons Commons
+	for i := range entries {
+		q := &entries[i]
+		for _, length := range index.Lengths() {
+			b := index.Bucket(length)
+			commons.Reset()
+			for k := len(b.Members) - 1; k >= 0; k-- { // backwards: blocks are not read in order
+				m := &b.Members[k]
+				got, gok := sc.PairScore(q, &m.Entry, b.Common(k, q.Sparse, &commons), &buf)
+				want, wok := sc.PairScore(q, &entries[m.ID], -1, &buf)
+				if got != want || gok != wok {
+					t.Fatalf("pair %d,%d: (%v,%v) with the blocks, (%v,%v) directly", i, m.ID, got, gok, want, wok)
+				}
+			}
+		}
+	}
+}
+
+func TestNewScoringChecks(t *testing.T) {
+	for _, p := range []Params{{Threshold: -1}, {Threshold: 1.5, Normalized: true}} {
+		if _, err := NewScoring(p, nil); err == nil {
+			t.Errorf("%+v must be rejected", p)
+		}
+	}
+	if _, err := NewScoring(Params{Threshold: 12}, nil); err != nil {
+		t.Errorf("a raw threshold above 1 is valid: %v", err)
+	}
+}

pkg/obisimilarity/scoring.go

@@ -0,0 +1,284 @@
+package obisimilarity
+
+import (
+	"errors"
+	"math"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obialign"
+	log "github.com/sirupsen/logrus"
+)
+
+// Reference is the length used to normalize a score (or to express a raw
+// distance): the alignment length, the longest or the shortest sequence.
+type Reference int
+
+const (
+	RefAlignment Reference = iota
+	RefLongest
+	RefShortest
+)
+
+// Params describe how a pair of sequences is scored and compared to a threshold.
+type Params struct {
+	// Threshold: with a normalized similarity an identity (0.95 for 95%), with
+	// a normalized distance 1 - identity, with a raw similarity the length of
+	// the LCS, with a raw distance the number of differences with the
+	// reference length. 0 means no threshold: all the pairs are reported.
+	Threshold  float64
+	Normalized bool // the score is divided by the reference length
+	Distance   bool // the score is a distance, not a similarity
+	Reference  Reference
+	// Ratio is the maximum ratio between the counts of a sequence and of the
+	// one it is compared to (clustering only): a larger ratio gives no match.
+	Ratio float64
+}
+
+// Scoring scores the pairs of sequences and decides whether they pass the
+// threshold. It must not be modified once built.
+type Scoring struct {
+	threshold float64 // threshold as used for the comparison (1 - t for a normalized distance)
+	normalize bool
+	lcsmode   bool // true: score is a similarity, false: it is a distance
+	ref       Reference
+	ratio     float64
+	worst     float64 // score of a pair that does not pass
+	all       bool    // no threshold: every pair is reported
+
+	// byLength gives, for the pairs whose longest sequence has a length l,
+	// the values that depend on l only. It is built when the reference is not
+	// the shortest sequence, otherwise those values depend on both lengths.
+	byLength []lengthInfo
+}
+
+type lengthInfo struct {
+	lcsMin    int32 // see lcsMin
+	minCommon int32 // see minCommon
+}
+
+// NewScoring builds the scoring of the sequences `entries`, which are the
+// ones to be compared (the longest of them is needed for a raw distance).
+func NewScoring(p Params, entries []Entry) (*Scoring, error) {
+	if p.Threshold < 0 || (p.Normalized && p.Threshold > 1.0) {
+		return nil, errors.New("the threshold must not be negative, and must be at most 1 when scores are normalized")
+	}
+
+	sc := &Scoring{
+		normalize: p.Normalized,
+		lcsmode:   !p.Distance,
+		ref:       p.Reference,
+		ratio:     p.Ratio,
+	}
+	sc.threshold = p.Threshold
+	sc.all = p.Threshold == 0
+	if !sc.lcsmode && sc.normalize && p.Threshold > 0 {
+		sc.threshold = 1.0 - p.Threshold
+	}
+
+	lmax := 0
+	for i := range entries {
+		lmax = max(lmax, entries[i].Length)
+	}
+	if !sc.similarityLike() {
+		sc.worst = float64(lmax)
+	}
+
+	if sc.ref != RefShortest {
+		sc.byLength = make([]lengthInfo, lmax+1)
+		for l := range sc.byLength {
+			sc.byLength[l] = lengthInfo{int32(sc.lcsMin(l, l)), int32(sc.minCommon(l, l))}
+		}
+	}
+	return sc, nil
+}
+
+// Worst is the score of a pair that does not pass the threshold.
+func (sc *Scoring) Worst() float64 { return sc.worst }
+
+// Threshold is the threshold the scores are compared to.
+func (sc *Scoring) Threshold() float64 { return sc.threshold }
+
+// SimilarityLike tells if the comparison with the threshold is "score >= threshold".
+func (sc *Scoring) SimilarityLike() bool { return sc.similarityLike() }
+
+func (sc *Scoring) similarityLike() bool { return sc.lcsmode || sc.normalize }
+
+// lcsMin is the smallest LCS score that a pair of lengths l1 >= l2 can have
+// and still pass the threshold.
+func (sc *Scoring) lcsMin(l1, l2 int) int {
+	switch {
+	case sc.threshold <= 0:
+		return 0
+	case sc.normalize:
+		if sc.ref == RefShortest {
+			return int(sc.threshold * float64(l2))
+		}
+		return int(sc.threshold * float64(l1))
+	case sc.lcsmode:
+		return int(sc.threshold)
+	case sc.ref == RefShortest:
+		return int(float64(l2) - sc.threshold)
+	default:
+		return int(float64(l1) - sc.threshold)
+	}
+}
+
+// minCommon is the number of 4-mers that two sequences of lengths l1 >= l2
+// must share to be possibly similar enough.
+func (sc *Scoring) minCommon(l1, l2 int) int {
+	if sc.all {
+		return 0 // no threshold, no filter
+	}
+	reflen := l2
+	if sc.ref == RefAlignment || sc.ref == RefLongest {
+		reflen = l1
+	}
+	var lcs int
+	if sc.normalize {
+		lcs = int(math.Ceil(float64(reflen) * sc.threshold))
+	} else {
+		th := sc.threshold
+		if !sc.lcsmode {
+			th = float64(reflen) - th
+		}
+		lcs = int(th)
+	}
+	return minCommonWords(l1, lcs)
+}
+
+// lcsMinFor is lcsMin for sequences of lengths l1 >= l2, from the table when it exists.
+func (sc *Scoring) lcsMinFor(l1, l2 int) int {
+	if sc.byLength != nil {
+		return int(sc.byLength[l1].lcsMin)
+	}
+	return sc.lcsMin(l1, l2)
+}
+
+// LengthCompatible tells if two sequences of these lengths may be similar
+// enough: the shortest cannot be shorter than the minimum LCS score of the pair.
+func (sc *Scoring) LengthCompatible(la, lb int) bool {
+	l1, l2 := max(la, lb), min(la, lb)
+	return l2 >= sc.lcsMinFor(l1, l2)
+}
+
+// PairScore scores the pair (q, c). ok is false when the pair is known not to
+// reach the threshold, in which case no score is given.
+//
+// common, if not negative, is the sum of the minima of the 4-mer counters of
+// the two sequences, already computed by a block of tables (see Index).
+func (sc *Scoring) PairScore(q, c *Entry, common int, buf *obialign.BandedLCSBuffer) (score float64, ok bool) {
+	// The quotient of two counts is not above 1 when the first is not above
+	// the second, which spares the division in the usual case.
+	if !(sc.ratio >= 1 && q.Count <= c.Count) && float64(q.Count)/float64(c.Count) > sc.ratio {
+		return 0, false
+	}
+
+	l1, l2 := q.Length, c.Length
+	long, short := q, c
+	if l1 < l2 {
+		l1, l2 = l2, l1
+		long, short = c, q
+	}
+	var lcsMin, minCommon int
+	if sc.byLength != nil {
+		info := sc.byLength[l1]
+		lcsMin, minCommon = int(info.lcsMin), int(info.minCommon)
+	} else {
+		lcsMin, minCommon = sc.lcsMin(l1, l2), sc.minCommon(l1, l2)
+	}
+	if l2 < lcsMin {
+		return 0, false
+	}
+	// The 4-mer filter counts identical words: it does not hold when IUPAC
+	// codes can match different letters. It lets short sequences pass.
+	if l1 >= 12 && l2 >= 12 && !long.Ambiguous && !short.Ambiguous {
+		if common >= 0 {
+			common += min(long.Table.Overflow, short.Table.Overflow)
+		} else {
+			common = commonWords(long.Table, short.Table)
+		}
+		if common < minCommon {
+			return 0, false
+		}
+	}
+
+	// The length of the alignment is needed only to normalize by it, or to
+	// express a distance with it; the score alone is much cheaper.
+	var lcs, alilen int
+	var err error
+	if sc.ref == RefAlignment && (sc.normalize || !sc.lcsmode) {
+		lcs, alilen, err = buf.AtLeast(q.Seq, c.Seq, lcsMin)
+	} else {
+		lcs, err = buf.ScoreAtLeast(q.Seq, c.Seq, lcsMin)
+	}
+	if err != nil {
+		log.Fatalf("Cannot align two sequences: %v", err)
+	}
+	if lcs < 0 {
+		// The best alignment has a score under lcsMin: the pair cannot pass.
+		return 0, false
+	}
+
+	id := float64(lcs)
+	if !sc.lcsmode && !sc.normalize {
+		switch sc.ref {
+		case RefAlignment:
+			id = float64(alilen) - id
+		case RefLongest:
+			id = float64(l1) - id
+		case RefShortest:
+			id = float64(l2) - id
+		}
+	}
+	if sc.normalize {
+		switch sc.ref {
+		case RefAlignment:
+			id /= float64(alilen)
+		case RefLongest:
+			id /= float64(l1)
+		case RefShortest:
+			id /= float64(l2)
+		}
+	}
+	return id, true
+}
+
+// Passes tells if the score s of a pair reaches the threshold.
+func (sc *Scoring) Passes(s float64) bool {
+	if sc.all {
+		return true
+	}
+	if sc.similarityLike() {
+		return s >= sc.threshold
+	}
+	return s <= sc.threshold
+}
+
+// Beats tells if the score s is better than the score best (without any
+// threshold).
+func (sc *Scoring) Beats(s, best float64) bool {
+	if sc.similarityLike() {
+		return s > best
+	}
+	return s < best
+}
+
+// SelfScore is the score of a sequence with itself.
+func (sc *Scoring) SelfScore(length int) float64 {
+	switch {
+	case sc.normalize && sc.lcsmode:
+		return 1.0
+	case !sc.lcsmode:
+		return 0.0
+	default:
+		return float64(length)
+	}
+}
+
+// Reported is the score to report for a pair whose score is s: a normalized
+// distance is computed as a similarity against the converted threshold.
+func (sc *Scoring) Reported(s float64) float64 {
+	if !sc.lcsmode && sc.normalize {
+		return 1.0 - s
+	}
+	return s
+}

pkg/obisimilarity/table4.go

@@ -0,0 +1,54 @@
+// Package obisimilarity holds what the tools that compare nucleotide sequences
+// by the length of their longest common subsequence (obiclust, obialign) have
+// in common: the scoring of a pair against a threshold, with the lossless
+// 4-mer filter that avoids most of the alignments, and the index that finds
+// the sequences a given one can be similar to.
+//
+// The scoring follows the arithmetic of sumaclust and sumatra (integer
+// truncations, floating point comparisons) so that the same pairs are
+// accepted with the same scores.
+package obisimilarity
+
+// Table4 is the 4-mer occurrence table of a sequence: one saturating counter
+// per word, and the number of occurrences that did not fit in the counters.
+type Table4 struct {
+	Count    [256]uint8
+	Overflow int
+}
+
+// NewTable4 counts the 4-mers of a sequence made of the letters a, c, g
+// and t. The 2-bit code of a base is (letter>>1)&3, i.e. a=0, c=1, t=2, g=3.
+func NewTable4(seq []byte) *Table4 {
+	t := new(Table4)
+	t.Fill(seq)
+	return t
+}
+
+// Fill counts the 4-mers of seq in a zeroed table.
+func (t *Table4) Fill(seq []byte) {
+	if len(seq) < 4 {
+		return
+	}
+	w := uint(0)
+	for i, c := range seq {
+		w = (w<<2 | uint(c>>1)&3) & 0xFF
+		if i < 3 {
+			continue
+		}
+		if t.Count[w] < 255 {
+			t.Count[w]++
+		} else {
+			t.Overflow++
+		}
+	}
+}
+
+// minCommonWords is the minimum number of 4-mers that two sequences must
+// share when the longest has length reflen and they have a common
+// subsequence of length lcs.
+func minCommonWords(reflen, lcs int) int {
+	nbfrag := (reflen-lcs)*2 + 1
+	smin := lcs / nbfrag
+	r := lcs - smin*nbfrag
+	return max(smin-2, 0)*r + max(smin-3, 0)*(nbfrag-r)
+}

pkg/obisimilarity/table4_nosimd.go

@@ -0,0 +1,14 @@
+//go:build !goexperiment.simd
+
+package obisimilarity
+
+// commonWords is the upper bound used for the number of 4-mers shared by two
+// sequences: the sum of the minima of the counters, plus the minimum of the
+// two overflows.
+func commonWords(t1, t2 *Table4) int {
+	total := 0
+	for i := range t1.Count {
+		total += int(min(t1.Count[i], t2.Count[i]))
+	}
+	return total + min(t1.Overflow, t2.Overflow)
+}

pkg/obisimilarity/table4_simd.go

@@ -0,0 +1,35 @@
+//go:build goexperiment.simd
+
+package obisimilarity
+
+import "simd"
+
+// commonWords is the upper bound used for the number of 4-mers shared by two
+// sequences: the sum of the minima of the counters, plus the minimum of the
+// two overflows.
+//
+// The minima of the 256 counters are computed by vectors of bytes, and summed
+// in 16-bit lanes (at most 256*255 spread over the lanes, the lanes being at
+// least 8 in number, so that none of them overflows).
+func commonWords(t1, t2 *Table4) int {
+	var probe simd.Uint8s
+	vl := probe.Len()
+
+	acc := simd.BroadcastUint16s(0)
+	low := simd.BroadcastUint16s(0x00FF)
+	for i := 0; i < len(t1.Count); i += vl {
+		m := simd.LoadUint8s(t1.Count[i:]).Min(simd.LoadUint8s(t2.Count[i:])).ReshapeToUint16s()
+		acc = acc.Add(m.And(low)).Add(m.ShiftAllRight(8))
+	}
+
+	// sum the pairs of 16-bit lanes into 32-bit lanes, then add those up
+	wide := acc.ReshapeToUint32s()
+	sums := wide.And(simd.BroadcastUint32s(0xFFFF)).Add(wide.ShiftAllRight(16))
+	var lanes [32]uint32
+	sums.Store(lanes[:])
+	total := 0
+	for _, v := range lanes[:sums.Len()] {
+		total += int(v)
+	}
+	return total + min(t1.Overflow, t2.Overflow)
+}

pkg/obithermo/model.go

@@ -0,0 +1,222 @@
+// Package obithermo computes DNA duplex melting temperatures with the
+// nearest-neighbor model used by ecoPrimers (SantaLucia 1998 parameters, with
+// the SantaLucia or Owczarzy 2008 salt corrections).
+//
+// Only the situations ecoPrimers actually evaluates are supported: a primer
+// against its perfect complement (SelfTm), and a primer against a genomic site
+// of the same length that may carry mismatches (Tm). Gaps, bulges and dangling
+// ends are not modeled.
+package obithermo
+
+import (
+	"fmt"
+	"math"
+)
+
+// SaltMethod selects the monovalent salt correction.
+type SaltMethod int
+
+const (
+	// SantaLucia applies a per-step entropy correction 0.368*ln[Na+].
+	SantaLucia SaltMethod = 1
+	// Owczarzy applies the GC-dependent correction of Owczarzy et al. 2008.
+	Owczarzy SaltMethod = 2
+)
+
+// Default experimental conditions, those of ecoPrimers.
+const (
+	DefaultPrimerConc = 8e-7 // total primer strand concentration (M)
+	DefaultTargetConc = 0.0  // target strand concentration (M)
+	DefaultSalt       = 0.05 // monovalent cation concentration (M)
+)
+
+const (
+	gasConstant       = 1.987   // cal/mol/K
+	initEntropy       = -5.9    // duplex initiation, cal/mol/K
+	kelvinOffset      = 273.15  // ecoPrimers subtracts 273.0 (see package notes)
+	owczarzyGCSlope   = 4.29e-5 // (4.29*fGC - 3.95)e-5 * ln[Na]
+	owczarzyOffset    = 3.95e-5
+	owczarzyQuadratic = 9.40e-6 // 9.4e-6 * ln[Na]^2
+)
+
+// Model holds the thermodynamic tables and the experimental conditions.
+// A Model is immutable after NewModel and safe for concurrent use.
+type Model struct {
+	method SaltMethod
+	rlogc  float64 // R*ln(Ct factor), included in the initial entropy
+	logNa  float64 // ln[Na+]
+
+	// Indexed by (top0<<6 | top1<<4 | bot0<<2 | bot1) with A,C,G,T = 0..3.
+	// dS already includes the SantaLucia salt term when that method is selected.
+	dH [256]float64
+	dS [256]float64
+}
+
+// NewModel builds a Model for the given strand concentrations (M), monovalent
+// salt concentration (M) and salt correction method.
+func NewModel(primerConc, targetConc, salt float64, method SaltMethod) (*Model, error) {
+	if method != SantaLucia && method != Owczarzy {
+		return nil, fmt.Errorf("obithermo: unknown salt method %d", method)
+	}
+	if salt <= 0 {
+		return nil, fmt.Errorf("obithermo: salt concentration must be positive, got %g", salt)
+	}
+
+	// Effective strand concentration, as in nparam_InitParams.
+	var ctFactor float64
+	switch {
+	case primerConc == targetConc:
+		ctFactor = primerConc / 2
+	case primerConc > targetConc:
+		ctFactor = primerConc - targetConc/2
+	default:
+		ctFactor = targetConc - primerConc/2
+	}
+	if ctFactor <= 0 {
+		return nil, fmt.Errorf("obithermo: strand concentrations must be positive (primer %g, target %g)",
+			primerConc, targetConc)
+	}
+
+	m := &Model{
+		method: method,
+		rlogc:  gasConstant * math.Log(ctFactor),
+		logNa:  math.Log(salt),
+	}
+	m.fillTables()
+	return m, nil
+}
+
+// DefaultModel returns a Model with the ecoPrimers default conditions and the
+// SantaLucia salt correction.
+func DefaultModel() *Model {
+	m, err := NewModel(DefaultPrimerConc, DefaultTargetConc, DefaultSalt, SantaLucia)
+	if err != nil {
+		panic(err) // the defaults are valid
+	}
+	return m
+}
+
+var baseCode = func() (t [256]int8) {
+	for i := range t {
+		t[i] = -1
+	}
+	for i, c := range "ACGT" {
+		t[c] = int8(i)
+		t[c+'a'-'A'] = int8(i)
+	}
+	return
+}()
+
+func stepIndex(t0, t1, b0, b1 int) int { return t0<<6 | t1<<4 | b0<<2 | b1 }
+
+func (m *Model) fillTables() {
+	// Double mismatches (no Watson-Crick pair at either end of the step) are
+	// treated as interior-loop expansions. Complement of code c is 3-c.
+	const loopH, loopS = 0.0, -0.97
+	for x := range 4 {
+		for y := range 4 {
+			for a := range 4 {
+				for b := range 4 {
+					if x+a != 3 && y+b != 3 {
+						i := stepIndex(x, y, a, b)
+						m.dH[i], m.dS[i] = loopH, loopS
+					}
+				}
+			}
+		}
+	}
+
+	// Explicit parameters override the defaults (Watson-Crick and single mismatches).
+	for _, e := range nnEntries {
+		i := stepIndex(int(baseCode[e.top[0]]), int(baseCode[e.top[1]]),
+			int(baseCode[e.bottom[0]]), int(baseCode[e.bottom[1]]))
+		m.dH[i], m.dS[i] = e.dH, e.dS
+	}
+
+	// SantaLucia salt correction: 0.5*kfac per side of each step, i.e. kfac.
+	if m.method == SantaLucia {
+		kfac := 0.368 * m.logNa
+		for i := range m.dS {
+			m.dS[i] += kfac
+		}
+	}
+}
+
+// GCCount returns the number of G and C (upper or lower case) in seq.
+func GCCount(seq []byte) int {
+	n := 0
+	for _, c := range seq {
+		switch c {
+		case 'G', 'C', 'g', 'c':
+			n++
+		}
+	}
+	return n
+}
+
+// SelfTm returns the melting temperature (degrees Celsius) of primer against
+// its perfect complement. It returns NaN when the model predicts no stable duplex.
+func (m *Model) SelfTm(primer []byte) (float64, error) {
+	return m.duplex(primer, primer, true)
+}
+
+// Tm returns the melting temperature (degrees Celsius) of primer against the
+// site, a stretch of the target read in the same orientation as the primer
+// (primer and site have the same length; mismatches are primer[i] != site[i]).
+// It returns NaN when the model predicts no stable duplex.
+func (m *Model) Tm(primer, site []byte) (float64, error) {
+	if len(site) != len(primer) {
+		return 0, fmt.Errorf("obithermo: primer (%d) and site (%d) lengths differ", len(primer), len(site))
+	}
+	return m.duplex(primer, site, false)
+}
+
+func (m *Model) duplex(primer, site []byte, perfect bool) (float64, error) {
+	n := len(primer)
+	if n < 2 {
+		return 0, fmt.Errorf("obithermo: primer too short (%d), need at least 2 bases", n)
+	}
+
+	dH := 0.0
+	dS := initEntropy + m.rlogc
+
+	p0 := int(baseCode[primer[0]])
+	s0 := p0
+	if !perfect {
+		s0 = int(baseCode[site[0]])
+	}
+	if p0 < 0 || s0 < 0 {
+		return 0, fmt.Errorf("obithermo: non-ACGT base at position 0")
+	}
+
+	for i := 1; i < n; i++ {
+		p1 := int(baseCode[primer[i]])
+		s1 := p1
+		if !perfect {
+			s1 = int(baseCode[site[i]])
+		}
+		if p1 < 0 || s1 < 0 {
+			return 0, fmt.Errorf("obithermo: non-ACGT base at position %d", i)
+		}
+		// The partner strand carries the complement of the site.
+		idx := stepIndex(p0, p1, 3-s0, 3-s1)
+		dH += m.dH[idx]
+		dS += m.dS[idx]
+		p0, s0 = p1, s1
+	}
+
+	if m.method == Owczarzy {
+		fgc := float64(GCCount(primer)) / float64(n)
+		dS += dH * ((owczarzyGCSlope*fgc-owczarzyOffset)*m.logNa + owczarzyQuadratic*m.logNa*m.logNa)
+	}
+
+	// One-state transition, as nparam_CalcTM: valid only for dS < 0 and Tm >= 0.
+	if dS >= 0 {
+		return math.NaN(), nil
+	}
+	tm := dH / dS
+	if tm < 0 {
+		return math.NaN(), nil
+	}
+	return tm - kelvinOffset, nil
+}

pkg/obithermo/model_test.go

@@ -0,0 +1,203 @@
+package obithermo
+
+import (
+	"bufio"
+	"math"
+	"os"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+func readTSV(t *testing.T, path string) [][]string {
+	t.Helper()
+	f, err := os.Open(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	var rows [][]string
+	sc := bufio.NewScanner(f)
+	for sc.Scan() {
+		if line := strings.TrimSpace(sc.Text()); line != "" {
+			rows = append(rows, strings.Split(line, "\t"))
+		}
+	}
+	if err := sc.Err(); err != nil {
+		t.Fatal(err)
+	}
+	return rows
+}
+
+func mustFloat(t *testing.T, s string) float64 {
+	t.Helper()
+	v, err := strconv.ParseFloat(s, 64)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return v
+}
+
+// checkAgainstC compares one Go result (Celsius) with the raw Kelvin value
+// returned by the C library. The C code returns 0 when the model fails.
+//
+// The tolerance (1e-4 K) covers the single-precision literals of the C source
+// (-21.3f, 1.987f, -5.9f ...): emulating them in float32 brings the gap down
+// to 1e-12 K, so the Go port uses the same model with exact decimal constants.
+func checkAgainstC(t *testing.T, what string, got float64, cKelvin float64) {
+	t.Helper()
+	if cKelvin <= 0 {
+		if !math.IsNaN(got) {
+			t.Errorf("%s: C model failed, Go returned %g", what, got)
+		}
+		return
+	}
+	if math.IsNaN(got) {
+		t.Errorf("%s: Go model failed, C returned %g K", what, cKelvin)
+		return
+	}
+	if d := math.Abs(got + 273.15 - cKelvin); d > 1e-4 {
+		t.Errorf("%s: Go %.9f C %.9f K (diff %g)", what, got+273.15, cKelvin, d)
+	}
+}
+
+// testdata/c_reference.tsv was produced by calling nparam_CalcSelfTM and
+// nparam_CalcTwoTM of ecoPrimers' libthermo on random primers and sites with
+// 0..n mismatches: primer, site, self Tm, two Tm (default conditions), and
+// two Tm with 1e-6 M / 2e-7 M strands and 0.2 M salt.
+func TestAgainstEcoPrimersC(t *testing.T) {
+	def := DefaultModel()
+	alt, err := NewModel(1e-6, 2e-7, 0.2, SantaLucia)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	rows := readTSV(t, "testdata/c_reference.tsv")
+	if len(rows) < 100 {
+		t.Fatalf("reference file too short: %d", len(rows))
+	}
+	for _, r := range rows {
+		primer, site := []byte(r[0]), []byte(r[1])
+
+		got, err := def.SelfTm(primer)
+		if err != nil {
+			t.Fatal(err)
+		}
+		checkAgainstC(t, "self "+r[0], got, mustFloat(t, r[2]))
+
+		got, err = def.Tm(primer, site)
+		if err != nil {
+			t.Fatal(err)
+		}
+		checkAgainstC(t, "two "+r[0]+"/"+r[1], got, mustFloat(t, r[3]))
+
+		got, err = alt.Tm(primer, site)
+		if err != nil {
+			t.Fatal(err)
+		}
+		checkAgainstC(t, "alt "+r[0]+"/"+r[1], got, mustFloat(t, r[4]))
+	}
+}
+
+// ecoPrimers prints Tm as (Kelvin - 273.0) with one decimal; obithermo uses the
+// exact 273.15 offset, so the printed value is our result + 0.15.
+func TestAgainstTutorialOutput(t *testing.T) {
+	m := DefaultModel()
+	rows := readTSV(t, "testdata/ecoprimers_tm.tsv")
+	if len(rows) == 0 {
+		t.Fatal("empty fixture")
+	}
+	for _, r := range rows {
+		got, err := m.SelfTm([]byte(r[0]))
+		if err != nil {
+			t.Fatal(err)
+		}
+		want := mustFloat(t, r[1])
+		if d := math.Abs(got + 0.15 - want); d > 0.0501 {
+			t.Errorf("%s: printed %.1f, got %.3f (+0.15 = %.3f)", r[0], want, got, got+0.15)
+		}
+	}
+}
+
+func TestSaltMethodsAgreeAtOneMolar(t *testing.T) {
+	sl, _ := NewModel(DefaultPrimerConc, DefaultTargetConc, 1.0, SantaLucia)
+	ow, _ := NewModel(DefaultPrimerConc, DefaultTargetConc, 1.0, Owczarzy)
+	p := []byte("ACACCGCCCGTCACTCTC")
+	a, _ := sl.SelfTm(p)
+	b, _ := ow.SelfTm(p)
+	if math.Abs(a-b) > 1e-9 {
+		t.Errorf("at 1 M both corrections vanish: %g vs %g", a, b)
+	}
+}
+
+func TestSaltRaisesTm(t *testing.T) {
+	p := []byte("ACACCGCCCGTCACTCTC")
+	for _, method := range []SaltMethod{SantaLucia, Owczarzy} {
+		low, _ := NewModel(DefaultPrimerConc, DefaultTargetConc, 0.02, method)
+		high, _ := NewModel(DefaultPrimerConc, DefaultTargetConc, 0.5, method)
+		a, _ := low.SelfTm(p)
+		b, _ := high.SelfTm(p)
+		if !(b > a) {
+			t.Errorf("method %d: Tm should grow with salt, got %g then %g", method, a, b)
+		}
+	}
+}
+
+func TestMismatchLowersTm(t *testing.T) {
+	m := DefaultModel()
+	p := []byte("ACACCGCCCGTCACTCTC")
+	perfect, _ := m.Tm(p, p)
+	mm, _ := m.Tm(p, []byte("ACACCGCCCGTGACTCTC"))
+	if !(mm < perfect) {
+		t.Errorf("a mismatch must lower Tm: perfect %g, mismatch %g", perfect, mm)
+	}
+}
+
+func TestInputErrors(t *testing.T) {
+	m := DefaultModel()
+	if _, err := m.SelfTm([]byte("A")); err == nil {
+		t.Error("expected error for a 1-base primer")
+	}
+	if _, err := m.SelfTm([]byte("ACGNT")); err == nil {
+		t.Error("expected error for a non-ACGT base")
+	}
+	if _, err := m.Tm([]byte("ACGT"), []byte("ACG")); err == nil {
+		t.Error("expected error for different lengths")
+	}
+	if _, err := NewModel(0, 0, 0.05, SantaLucia); err == nil {
+		t.Error("expected error for zero strand concentration")
+	}
+	if _, err := NewModel(1e-6, 0, -1, SantaLucia); err == nil {
+		t.Error("expected error for negative salt")
+	}
+	if _, err := NewModel(1e-6, 0, 0.05, SaltMethod(9)); err == nil {
+		t.Error("expected error for unknown salt method")
+	}
+}
+
+func TestLowerCaseAccepted(t *testing.T) {
+	m := DefaultModel()
+	a, _ := m.SelfTm([]byte("ACACCGCCCGTCACTCTC"))
+	b, _ := m.SelfTm([]byte("acaccgcccgtcactctc"))
+	if a != b {
+		t.Errorf("case must not matter: %g vs %g", a, b)
+	}
+}
+
+func TestGCCount(t *testing.T) {
+	if n := GCCount([]byte("ACgtCCaT")); n != 4 {
+		t.Errorf("GCCount = %d, want 4", n)
+	}
+}
+
+func BenchmarkTm(b *testing.B) {
+	m := DefaultModel()
+	p := []byte("ACACCGCCCGTCACTCTC")
+	s := []byte("ACACCGCCCGTGACTCTC")
+	b.ReportAllocs()
+	for b.Loop() {
+		if _, err := m.Tm(p, s); err != nil {
+			b.Fatal(err)
+		}
+	}
+}

pkg/obithermo/nn_table.go

@@ -0,0 +1,126 @@
+// Code generated from ecoPrimers libthermo/nnparams.c (nparam_InitParams); DO NOT EDIT.
+
+package obithermo
+
+// nnEntry is one explicit nearest-neighbor parameter: the 5'->3' dinucleotide of
+// the top strand, the 3'->5' dinucleotide facing it, enthalpy (cal/mol) and
+// entropy (cal/mol/K).
+type nnEntry struct {
+	top, bottom string
+	dH, dS      float64
+}
+
+var nnEntries = [...]nnEntry{
+	{"AA", "TT", -7600.0, -21.3},  // AA/TT
+	{"AC", "TG", -8400.0, -22.4},  // AC/TG
+	{"AG", "TC", -7800.0, -21.0},  // AG/TC
+	{"AT", "TA", -7200.0, -20.4},  // AT/TA
+	{"CA", "GT", -8500.0, -22.7},  // CA/GT
+	{"CC", "GG", -8000.0, -19.9},  // CC/GG
+	{"CG", "GC", -10600.0, -27.2}, // CG/GC
+	{"CT", "GA", -7800.0, -21.0},  // CT/GA
+	{"GA", "CT", -8200.0, -22.2},  // GA/CT
+	{"GC", "CG", -9800.0, -24.4},  // GC/CG
+	{"GG", "CC", -8000.0, -19.9},  // GG/CC
+	{"GT", "CA", -8400.0, -22.4},  // GT/CA
+	{"TA", "AT", -7200.0, -21.3},  // TA/AT
+	{"TC", "AG", -8200.0, -22.2},  // TC/AG
+	{"TG", "AC", -8500.0, -22.7},  // TG/AC
+	{"TT", "AA", -7600.0, -21.3},  // TT/AA
+	{"AA", "CT", 7600.0, 20.2},    // AA/CT
+	{"AA", "TC", 2300.0, 4.6},     // AA/TC
+	{"AC", "CG", -700.0, -3.8},    // AC/CG
+	{"AC", "TA", 5300.0, 14.6},    // AC/TA
+	{"AG", "CC", 600.0, -0.6},     // AG/CC
+	{"AT", "CA", 5300.0, 14.6},    // AT/CA
+	{"CA", "AT", 3400.0, 8.0},     // CA/AT
+	{"CA", "GC", 1900.0, 3.7},     // CA/GC
+	{"CC", "AG", 5200.0, 14.2},    // CC/AG
+	{"CC", "GA", 600.0, -0.6},     // CC/GA
+	{"CG", "AC", 1900.0, 3.7},     // CG/AC
+	{"CT", "AA", 2300.0, 4.6},     // CT/AA
+	{"GA", "CC", 5200.0, 14.2},    // GA/CC
+	{"GC", "CA", -700.0, -3.8},    // GC/CA
+	{"TA", "AC", 3400.0, 8.0},     // TA/AC
+	{"TC", "AA", 7600.0, 20.2},    // TC/AA
+	{"AC", "TT", 700.0, 0.2},      // AC/TT
+	{"AT", "TC", -1200.0, -6.2},   // AT/TC
+	{"CA", "TT", 1000.0, 0.7},     // CA/TT
+	{"CC", "GT", -800.0, -4.5},    // CC/GT
+	{"CC", "TG", 5200.0, 13.5},    // CC/TG
+	{"CG", "TC", -1500.0, -6.1},   // CG/TC
+	{"CT", "GC", -1500.0, -6.1},   // CT/GC
+	{"CT", "TA", -1200.0, -6.2},   // CT/TA
+	{"GC", "CT", 2300.0, 5.4},     // GC/CT
+	{"GT", "CC", 5200.0, 13.5},    // GT/CC
+	{"TA", "CT", 1200.0, 0.7},     // TA/CT
+	{"TC", "CG", 2300.0, 5.4},     // TC/CG
+	{"TC", "AT", 1200.0, 0.7},     // TC/AT
+	{"TG", "CC", -800.0, -4.5},    // TG/CC
+	{"TT", "CA", 700.0, 0.2},      // TT/CA
+	{"TT", "AC", 1000.0, 0.7},     // TT/AC
+	{"AA", "GT", 3000.0, 7.4},     // AA/GT
+	{"AA", "TG", -600.0, -2.3},    // AA/TG
+	{"AC", "GG", 500.0, 3.2},      // AC/GG
+	{"AG", "GC", -4000.0, -13.2},  // AG/GC
+	{"AG", "TA", -700.0, -2.3},    // AG/TA
+	{"AT", "GA", -700.0, -2.3},    // AT/GA
+	{"CA", "GG", -700.0, -2.3},    // CA/GG
+	{"CG", "GA", -4000.0, -13.2},  // CG/GA
+	{"GA", "AT", 700.0, 0.7},      // GA/AT
+	{"GA", "CG", -600.0, -1.0},    // GA/CG
+	{"GC", "AG", -600.0, -1.0},    // GC/AG
+	{"GG", "AC", -700.0, -2.3},    // GG/AC
+	{"GG", "CA", 500.0, 3.2},      // GG/CA
+	{"GT", "AA", -600.0, -2.3},    // GT/AA
+	{"TA", "AG", 700.0, 0.7},      // TA/AG
+	{"TG", "AA", 3000.0, 7.4},     // TG/AA
+	{"AG", "TT", 1000.0, 0.9},     // AG/TT
+	{"AT", "TG", -2500.0, -8.3},   // AT/TG
+	{"CG", "GT", -4100.0, -11.7},  // CG/GT
+	{"CT", "GG", -2800.0, -8.0},   // CT/GG
+	{"GA", "TT", -1300.0, -5.3},   // GA/TT
+	{"GC", "TG", -4400.0, -12.3},  // GC/TG
+	{"GG", "CT", 3300.0, 10.4},    // GG/CT
+	{"GG", "TC", -2800.0, -8.0},   // GG/TC
+	{"GT", "CG", -4400.0, -12.3},  // GT/CG
+	{"GT", "TA", -2500.0, -8.3},   // GT/TA
+	{"TA", "GT", -100.0, -1.7},    // TA/GT
+	{"TC", "GG", 3300.0, 10.4},    // TC/GG
+	{"TG", "AT", -100.0, -1.7},    // TG/AT
+	{"TG", "GC", -4100.0, -11.7},  // TG/GC
+	{"TT", "AG", -1300.0, -5.3},   // TT/AG
+	{"TT", "GA", 1000.0, 0.9},     // TT/GA
+	{"AA", "AT", 4700.0, 12.9},    // AA/AT
+	{"AA", "TA", 1200.0, 1.7},     // AA/TA
+	{"AC", "AG", -2900.0, -9.8},   // AC/AG
+	{"AG", "AC", -900.0, -4.2},    // AG/AC
+	{"AT", "AA", 1200.0, 1.7},     // AT/AA
+	{"CA", "GA", -900.0, -4.2},    // CA/GA
+	{"GA", "CA", -2900.0, -9.8},   // GA/CA
+	{"TA", "AA", 4700.0, 12.9},    // TA/AA
+	{"AC", "TC", 0.0, -4.4},       // AC/TC
+	{"CA", "CT", 6100.0, 16.4},    // CA/CT
+	{"CC", "CG", 3600.0, 8.9},     // CC/CG
+	{"CC", "GC", -1500.0, -7.2},   // CC/GC
+	{"CG", "CC", -1500.0, -7.2},   // CG/CC
+	{"CT", "CA", 0.0, -4.4},       // CT/CA
+	{"GC", "CC", 3600.0, 8.9},     // GC/CC
+	{"TC", "AC", 6100.0, 16.4},    // TC/AC
+	{"AG", "TG", -3100.0, -9.5},   // AG/TG
+	{"CG", "GG", -4900.0, -15.3},  // CG/GG
+	{"GA", "GT", 1600.0, 3.6},     // GA/GT
+	{"GC", "GG", -6000.0, -15.8},  // GC/GG
+	{"GG", "CG", -6000.0, -15.8},  // GG/CG
+	{"GG", "GC", -4900.0, -15.3},  // GG/GC
+	{"GT", "GA", -3100.0, -9.5},   // GT/GA
+	{"TG", "AG", 1600.0, 3.6},     // TG/AG
+	{"AT", "TT", -2700.0, -10.8},  // AT/TT
+	{"CT", "GT", -5000.0, -15.8},  // CT/GT
+	{"GT", "CT", -2200.0, -8.4},   // GT/CT
+	{"TA", "TT", 200.0, -1.5},     // TA/TT
+	{"TC", "TG", -2200.0, -8.4},   // TC/TG
+	{"TG", "TC", -5000.0, -15.8},  // TG/TC
+	{"TT", "AT", 200.0, -1.5},     // TT/AT
+	{"TT", "TA", -2700.0, -10.8},  // TT/TA
+}

pkg/obithermo/testdata/c_reference.tsv

@@ -0,0 +1,400 @@
+GATTCAAATGACGGCAGC	GATTCAAATGACGGCAGC	328.590295502325	328.590295502325	335.520082032693
+GCCGGGAGTCCCTGAGAG	GCCGGGGGTCCCTTAGTG	335.202600908825	307.571309416976	317.582817372006
+CCGGAAATGTGCCATCTGCGTGCGA	CCGGAAATGTGCCATCTGCGTGCGA	341.857354066828	341.857354066828	349.067971091170
+GCAGCGTAAGAGGAGGGC	GCCGAGTGAGAGGACGGC	334.238236820572	292.689122005507	302.959373794547
+ATCGGGATCTCAAAACCATC	ATCCGGATCCCAAAACTATC	328.374164082672	303.012377993885	311.743534784977
+TTTACTTCTCTCAAGGCC	TTTACTTCGCTCACGGCC	324.870543621461	310.322643307651	318.434500513589
+GATATTATCCGG	TTCCTCATTAGG	306.633848409535	131.298936687567	140.958918657051
+ACCAGATTCACCGTTAAA	ACCAGATTCACCGTTAAA	324.666065281835	324.666065281835	331.625141334598
+GCAGAAGGAATTCGTCTTAAAGTTT	GCAGAAGGAATTCGTCTTAAAGTTT	330.221644057193	330.221644057193	337.429293354313
+GTTACGCCCGTGGACAGA	GTTACGCCCGTGGACAGA	333.152006826526	333.152006826526	340.131722535940
+TACTGGCCAAGTGTTTCGGGCTACC	TACTGGACAAGGGTTTCGAGCTACC	338.543873371072	317.159669988483	326.737809053749
+CGGGCGAAAGACCTAACTCGTC	GCCCATGTTTAAACGGTTCAAC	335.961664387167	99.726835837839	110.941608232133
+ACCACAGAAGAGCCTCAG	ACCACAGAAGAGCCTCAG	329.675966490972	329.675966490972	336.774887202068
+TCCGTATCCGTGTGCTAT	TCCGTATCCGTGTGCTAT	328.758681136274	328.758681136274	335.670742367360
+AGCACAACCATCTTGCCA	AACCAGCAATTGGGCAAC	330.726812945211	0.000000000000	0.000000000000
+AAATCCGCGGTTACTTAG	AAATCCCCGGATACTTAG	325.912290117405	310.432539561454	317.816914468819
+TGGTTGCGGAATGGCGACACGCTAG	TGGTTGCGGAATGGCGACACGCTAG	341.368691736083	341.368691736083	348.576311955777
+CCACAAGGCACACCGAGGGCCCAAC	CCACAAGGCACACCGAGGGCCCAAC	343.836245623632	343.836245623632	351.324029371624
+TACCTTTGGTTAACCGCG	TACCTTTGGTTGACCGCG	328.806445188317	321.784293018138	329.972272766148
+ACCGCCAAACGGTAGTCCTT	ACCGCCAAACGTTATTCCTT	334.638358155254	325.067933219113	333.495286023143
+GGAATCACCGAATTC	GGAATCACCAAATTC	319.872552670165	310.456121372350	317.724729105623
+CGCTGGCGGGGACGCTTA	CGCTGGCGGGGACGCTTA	338.850878426355	338.850878426355	345.870729715750
+CCCAGAACCGAGTGCTGA	CCCAGAACCTAGTGCTGA	332.759551850704	324.515696448123	332.085468128098
+AGGACTATAGCC	AGGACTATAGCC	311.340568942550	311.340568942550	317.961816433927
+AAGTAAGCCAAACTC	AAGTAAGCCAAACTC	318.374897033808	318.374897033808	325.174916603994
+GGCAAACAGTGGCGTCTCTA	GGCAAACATTGGCGTCTCGA	333.444435842161	322.587904260466	330.767259471616
+GCGGGCTGGGCGACGCCT	GCGGGCTGGGCGACGCCT	344.353158968164	344.353158968164	351.458901919666
+AAAACCCGTTCTTGAGAA	ATAACCCGTTCTTGAGAA	325.196643716015	318.373852240644	326.309549286768
+TCTACGGGCAGG	TCTACGGGCAGG	320.684836039138	320.684836039138	327.333083788784
+CTGTATCATTCAAGACCTTGTG	CTGGATCATTCATGACCTTCTA	327.587191928324	300.073898600295	309.615050320053
+GAGACCTCGTTTATATGTGCCG	GAGATCTCCTTTATATGTTCCG	331.709967011022	307.489712436204	316.778347789381
+GATGCCATGCAACGCGGTCGAAATA	GATGCCAAGCAACGCCGTCGAAATA	338.385858507321	328.510784583696	335.857144351146
+CACCCATGGGTCGAGGAA	CCTTCATGGGTCAAGGAA	332.434732686040	303.559662982167	313.037414209458
+AACTTTACCACTTTGTCGGA	AACTTTACCACTGTGTCGAA	327.854258269700	313.748415282978	322.159067997528
+CTCCCAAGAATAAGGCCA	CTCCCAAGAATAAGGCTA	327.279305268693	315.945719545119	323.849648970559
+CTGCTCCCGACCTGCCGTTG	CTGCTCCCGACCTGCAGTTG	339.148624544930	330.290260008245	337.867535931326
+TTAGGAGACTCTTCAGCCTCCTTGT	TTAGGACACTATTAAGCCTCCTTGT	334.398890499698	315.037835993209	323.891875192977
+TGGACGATTGCG	AGAATTGCGGCC	320.244905957781	146.501396490593	156.665817684297
+ATTGTCAACTGTAGGTCA	ATTGTCTAGTGTAGGTCA	323.939323737395	300.681567922172	308.863167608541
+ACAATGTATTTTCTCGGG	ACAATGTATTTTCTCGGG	323.362512238862	323.362512238862	330.327169046076
+CGTGTGAGTGGCCCGAGTTGTCTCT	CGTGCGAGTGGCCCGAGTTGTCTTT	340.130848609675	332.245514068114	340.636316881815
+TATCAGACGTGGCAATGC	TATCAGACTTGACAATGT	328.992885301719	304.405743905969	312.701350705029
+CGTCGTCCCTCTCAGCCG	CGTCGTCCCTCTCAGCCG	336.200821936899	336.200821936899	343.186897048375
+TAAGGCTAGCCAGAACCACG	TGAGGCTAGCCAGAACCACG	332.598253740841	326.985556902569	335.280653989150
+GTTCAGGGGAATGAGAGATA	TTACAACTGCCAACTTCGCT	326.722966616537	109.952678878773	121.594862151643
+CCCAGCGCGAATTTGTCA	CCCATCGCGAACTCGGCC	332.540722470730	292.630570524742	303.511783076339
+TGTCGTCGCTACCAGTTA	TGCCTTCCCTACCAGTTA	328.953316297354	307.160700342629	315.442693299698
+CATGCACCGTGAGTCTGC	CGTTCACCGGGAGTCCGC	332.846939286468	291.434652139556	302.526073291148
+GAGAGTTGCTGTTGATTA	TAGAGTTGCTGTTGATTA	323.031379731052	319.294880722061	326.526947280607
+GCTCGGTCCTACGCCGACGT	GCCTGATCTAAATATGCCTG	339.747877109584	158.568208002892	173.058219990461
+TCTTTGAGCTTCCCATCT	TATTTGAGGTTCCCATCT	326.307771557426	300.329179385918	308.617669678883
+GTTTCCTCGGGT	ATGTCCCCGGTC	317.903207078845	250.921863687541	262.030341678318
+AGTCTGCATCCAGGATAGTGTTCAA	AGTCTGCGTCCAGGATAGTGTTCAA	333.474575396610	328.822975749317	336.887405627890
+CTAGGGCACCAACGCACG	CTAGGTCACCAACGCACG	335.143347846422	329.701891848980	337.431099508189
+GATTGCTGGACCCGACAAAC	ATCGACTGCGTCAATCTGAT	332.811424650986	125.989102558259	139.175967010614
+TTATTTCTCCGAGATCCA	TTATTGCTCCGGGATCGA	323.096708880271	281.710374029821	291.583720089868
+AACCTCCGCTCAGCCCAT	AACCTCCGCTCAGCCCAT	334.559812073966	334.559812073966	341.720616471431
+ACATGCGGAGACGATAGA	GAGCATCTTCGTGTCCCC	328.777637355840	13.663795446228	15.557079997899
+GCGATCCGGCCCGAAGGACGGTTAC	GCGATCCGGCCCGAAGGACGGTTAC	343.160858693872	343.160858693872	350.466895660629
+TAGGCGCTACCAAAGAGACC	TACGCGATACCAAAGAGACC	332.314713425979	316.910949741281	324.757961637335
+GCTTGACGGAGGGATTGCATACGTC	GCTTGACGGAGGGCTTGCATAGGTC	337.905459739261	327.298908614928	335.399197844049
+CTTACGGTTTTGTGG	CTTACGGTTTTGTGG	320.490499863910	320.490499863910	327.265086426630
+AGAATGCTTTGCACGGGAAAAAGAC	AGAATGCTTTGCCCGGGAACAACAC	334.833702111547	324.032364174889	333.126901145695
+CCATCCGGGACCGTGTTA	GTGGTGGACAGAAAGCAC	332.523437582445	67.486926898379	75.000548050396
+AGCAAATATTTAAAACTG	AGCAAATATTTAAAACTG	316.766614411011	316.766614411011	323.642414073658
+ACTGACAAAAGGGACCTTTAGA	CCCGAACAAAGGGACCTTGAGG	329.363491007070	296.438968024933	307.489562539207
+ATAGGACTCGGACGAAAG	ATAGGACTCGGACGAAAG	326.913461392949	326.913461392949	333.867158802657
+TATAAGCTAGTTGAG	TACCAGCTACTCGAG	312.936278497921	279.566780789215	288.458312256098
+AAAGCTGCCGTG	GTCGATTGTCGG	320.846677260689	11.139130534401	12.281612095844
+AAAACTGGCACG	AAAACTGGCACG	317.241607884236	317.241607884236	323.710646130150
+ATTCAGATAATGAAGGAC	ATTAAGATAATGAACGAC	319.569864618292	300.532814450507	308.266203286509
+ACTCCGCCAGTAAAGAGCGATC	ACGCCGCCAGTAAAGATCGATC	334.956435857922	324.130818124085	332.402089448274
+AAACAGGTGGTGCTTCTGGT	AAGCAGGTGGCGCTTCCGGT	332.663618849926	317.895372912569	326.451852134041
+ACGTAAACTAGCGATCCTAC	ACGCAAAACAGCGATCCTTC	327.648394671924	306.841804464239	316.109126582555
+ACTCTTCTGTAGTAATTT	ACTCTTCTGTATTAATTT	318.111014934586	311.385476013207	318.854355601485
+GCCTGGATGAAA	CCTCGCATTATA	314.875733564946	135.466989302349	145.485320466877
+ACAAACATCTTT	ATAATTATCTTT	306.751127645023	230.133355279539	240.007445742650
+TTTATGTTGCACCCG	GCCCTCCTGCAAGGG	322.725425600228	198.227797952416	210.186099128768
+CTCTGTGATTGC	CTCTGTGATTAC	313.233837007613	299.818390981614	306.829729109027
+CAGGGGGTGTCTTCTTGCCA	CACGGGGTGCCTTCTTACTA	335.170765818115	304.025892909583	313.686365730924
+ATAAACTTAAGCGGGGCT	ATAAACTTAAGCGGGGCT	327.141275752020	327.141275752020	334.197558650535
+GTGGTACGTCTCGCACCGTATT	GTGGTACGTCTCGCACCGTGTT	335.183465966560	331.876024565197	339.826055532683
+AGCCCCTACCGTGAGGTATGAA	AGCCCCTACCGTGAGCAATGAA	334.930026765640	329.325029384246	337.230038559345
+CTACGTTTGGACTTTCAG	TCCAATTCATGTACACCA	324.309529994597	130.265141330095	141.767454087853
+GGAGGCGTCGTCTAATGTGGCTGCA	GGAGGCGTCGTCTAATGTGGCTGCA	340.715437268799	340.715437268799	347.974364433377
+TCACGCCCGTTACCATGC	TCGCGCCAGTTACCATTC	334.223290685080	301.060585660525	310.242722525263
+GTGTGGAATAAC	GTGTGGAACAAT	308.856970626568	289.055396946314	297.087438077958
+AACTAGGCACAGCGGAAGTGTTACA	AACTACGCACAGCGGAATGGTTACA	336.064499532413	324.487528333723	332.510759640255
+AGTTGGGACCGCGGACCTCAGACGA	ACTTGGGACCGCGGACCTCAGACGA	342.586917960981	339.716507068413	347.297152857653
+CGCAGGCCGCCCTATCCG	GTGGGCAACCGTGATTAT	339.659476639126	111.730621568890	122.520566332669
+ACTCGGTTAAAGAAGTAGCGGG	ACTCGGTTAAAGAAGTAGCGGG	332.661444291378	332.661444291378	339.831677127033
+GTCGCACTCACGCCAAGG	GGCGCACTCACGCCAAGG	335.434179979611	327.968420416066	335.946784667129
+TTATGTGCCGTC	TTATGTGCCGTC	315.550568136423	315.550568136423	321.978396903837
+ATACACTCCCTC	ATACACTCCCTC	311.742937528520	311.742937528520	318.358302050000
+ATATATAAAACGCTT	ATATATAAAACGCTT	310.835103721529	310.835103721529	317.439661067059
+TTGACCCAGCTCTGACAA	TTGACCCAGCTCTGACCA	329.300611062468	323.357283025809	331.214889354503
+TCACCAAGGACATCGGTGAA	CACTAATCTCTAGGCTACTG	331.893316827967	0.000000000000	0.000000000000
+AAATACTAATATCACGAGGCCT	AAATACTAATATCACGAGGCCT	326.991443235419	326.991443235419	334.132256945596
+CATCTAAGGTTCCCCACG	CTCCTAAGGTTCCCCCTA	328.386823878643	301.442125171364	311.286145433228
+TCAGGCAAGGCTAAAGAT	TCATGCAAGGCTGAACCT	326.725811212726	296.261200224359	306.662120559413
+ACCGGTGTCTGACGAAGA	CCCGGAGTCTGACGAAGA	331.461574428762	323.233483550268	330.841087730303
+TGCCATAACAGT	TGCCATAACAGT	313.107834134787	313.107834134787	319.615154304331
+ATAACTCTGCTCTATAAA	ATAACTCTGCTCTATAAA	317.633729090723	317.633729090723	324.531712635958
+TCCTGCGCAAATTACTAGAACCTAA	GTCTCAGCGAGAACTCTATTAGGAT	332.050504643625	0.000000000000	0.000000000000
+GTATCGAGGTCGAGGAAAGCGCTGT	GTATCAAGGTCGAGGAAAGCGCTGT	338.362288220608	334.066037941649	341.579718818818
+CAGCCTTTCAGG	CAGACTTTCAGG	316.248469201820	292.111362062705	300.080584403201
+CTTGCGGTACATCGCAGTGTGAATA	CTTGCGGTACATCGCAGTGTGAATA	335.474585471307	335.474585471307	342.613324305508
+TAAGAACTAGCACGCATAAG	TTAGAACCAGTACGAAGAAT	326.081723019865	254.023237958651	266.323327012842
+TACATGTACATTTCGGCC	TACATGTACATTTCGGCC	325.896662787897	325.896662787897	332.832044747960
+TAGCATGCGATTCCCTAAGAAG	TATCATGCGATTCCCTAGGAAG	330.590797080638	318.020175239024	326.375059746915
+CGGTGTTGTACGCCGACC	CAGTGGTGTATGGAGTCC	335.122388615945	245.504228310044	257.618564855382
+ACTTTACGCCCGTAG	ACTTTACGCCCGTAG	324.104530886916	324.104530886916	330.841602836160
+GGTCTTGGTGGACAGATG	AGTCTTGGTGTACAGATC	328.628141921468	313.674320134018	322.206848229767
+GTTCGGTCATGC	GTTCGGTCATGC	317.988264837295	317.988264837295	324.445499068321
+CTCACGCTTCTCATTATT	CTCACGCTTCTTTTTATT	323.247552234877	309.106689004420	317.389249379550
+TGCGAGCGTGAT	TGTGAGCGTGAT	322.408515345848	302.521839005369	309.729023044907
+AGATAGGGGGCCCTCGAACCGCCCG	AGATAGGGGGCCCTCGAACCGCCCG	345.102909485864	345.102909485864	352.627470314597
+AACCTACACAGG	AACCTACACAGG	313.161040203465	313.161040203465	319.768122899840
+TCCGCCCGCCAC	TCCGCCCGCCTC	330.719187584648	320.094784422350	327.317364425992
+ATATCGTGAGCCGTGTACCCGG	ATAGCGCGATCCGTATACCCGC	336.660788072052	313.231213020956	322.468970039473
+GGAAGCGTTGACACCGAC	GGAAGCGTTGACACCGAC	332.770993213501	332.770993213501	339.705211527389
+CGAGGCAAGGGATACGCAGCTCAGT	CGAGACAAGGGATACGCAGCTCACT	340.305756168638	332.795008096728	340.846081667398
+CTACCGGCGTAGATATGC	CTACCGGCGTAGATGGGA	328.520812530739	311.151850739834	319.645074203379
+GACTTAAGCGACTAC	ACGGCTTAGCGTGGC	318.887440179040	10.664421060955	12.014569510602
+CTTAGGATTCTGGCTAATTTACGGT	CTTAGGTTTCTGGCTAATTTACGGT	330.595304320228	326.378730172844	334.038450345878
+ATCCGCACGGAC	TTACCGGACCCC	323.089768931227	0.000000000000	0.000000000000
+TCCCAGCTTTTGCAGTGGGATT	TCCTAGCTTTTGCAGTGGGATC	335.062847253676	324.545742095465	332.720886054794
+TGGCTTCACGGC	TGGCTTCACGGC	323.785812253507	323.785812253507	330.353929757972
+TAGGTTATTCCGACCATA	ATGCGGGACACTGCCATG	322.422210409371	192.368508212637	205.559476403263
+AGCCTTTCGACTGAC	AGCCTTTCGATTGAC	324.335884950598	312.595672426393	320.534233002808
+CGTGCAGCTTTCATG	CGTGCAGCGTTCTTG	324.919682340793	305.582789800617	313.597104921942
+TCTACGTGGCGGGGACCGGG	TCTACGCGGCTGTGAGCCGG	341.347076387025	299.430239048585	309.018341106927
+TCAAATTAAAGTTGGACTGACA	TCAAATTAAAGTTGGACTGACA	326.350254889911	326.350254889911	333.467128526408
+CGGGGGCCGTGGTCACTGCTTTCAG	CGGGAGCCCTGGTCACTGTTTTGAG	343.571008380319	318.537337281338	328.795128334431
+ACATTGCAGTCA	TACAAATATGCA	314.282094430945	54.422588938214	59.864081596928
+GTTCCATAAGACTTA	GTTCCATATGATTTA	313.492331047121	283.603344310509	292.314146114969
+TATTTGAAAGACATAGGGGCGT	CGGTACACTGGTGCCGGAGGCA	329.927702442206	21.671006660567	25.185742081802
+AATGGCAATTGCGTA	AATGGCAATTGCGTA	322.153531008784	322.153531008784	328.861389966064
+TGACCTCACAAATACGGGGTCG	TGACCTCACAAATAAGGGGTCG	334.840837473162	327.359477549248	335.051513807130
+CTCTGAAAGAACCGGGCA	CTTTGAAAGAACCGGGCA	330.717616406698	321.166071332522	329.380562417680
+TCCCCGATGCTTAAGACACGCGTGT	TTCCCAAGGCTTAATACACTCGTGT	339.959965745411	305.790225053607	316.153725711522
+TGCACTTCGAGG	TGCACTTCGAGG	318.856723935199	318.856723935199	325.349637181717
+AAGGGTCACCAGAAA	AATAAGGTTCCTGCT	322.240165848681	53.802401211576	60.076587053449
+TAATAAGTCGTG	TTATAAGTGTTG	306.473400984667	233.666269447857	244.153793035378
+CATGATGCCTAGTATGAT	CATGATGCCTAGTAAGAT	321.893431646493	315.240330549201	322.344428969018
+TATGCGCGAACCTAC	TATGCGCGAACCTCC	324.777133033446	320.584177151486	328.154789861490
+TGGCAACTCCCGCACATGCCCT	TAGCAACTCCCGCCCCTGCCCT	341.428702525130	330.240007837364	339.446160220950
+AACGGTCCATAGTGT	TTACTGCCTCTATCA	322.301936674505	92.184229884404	101.375195399772
+CTGGTTAGCGACTAGGGCGA	CTGGTTAGCGACCAGGACGA	334.491616458730	325.265096818567	333.491322070668
+CTACGTGACATGGCCCCT	CTACGTGACATGGCCCCT	332.520578697054	332.520578697054	339.639446777780
+CCCTCCTGGAAGAATCCATAGTATC	TCCTCCTGGAAGAATGCATAGTATA	331.837538166233	315.597835592823	324.711900293230
+TCCGCGGTGCGGCGTCTA	TCCGCGGTGCGGCGTCTA	340.070741817013	340.070741817013	346.961988197618
+TGAGGGCGGCTCGAAAGAGATAGGC	TGAAGGCGGCTCGAAAGAGATAGGC	339.908445380864	335.397387022299	343.087954453266
+AGGATTGAATGCGGATTCACAATGA	AGGATTGAATGCGGATTCACCATGA	333.075743741665	329.267518388258	337.096639984117
+CTGGCACGTGAAAGTGTA	CTGGCACGTGAAAGTGTA	328.758681136274	328.758681136274	335.670742367360
+TATGAGCTCGCATTGACGAT	TATCAGCTCGCATTGACGTT	330.199735252295	317.575655447674	325.732088199432
+CTCCGATACGCCCAACGTTCCT	CTCATATACGCCCAACGATCCT	337.111273784768	319.264148345607	327.480755860446
+CTTGTAAGGAACCGATCGTGAAACT	ATAAGATATATTTGTTAAATAGGAG	333.190199559369	0.000000000000	0.000000000000
+CGATCGGGGTTCGGTGCTAAGG	CGATCGGGGTTCGGTGCTAAGG	337.958135639451	337.958135639451	345.185613424927
+GTGTCACTTAGTGGC	GTGTCACTTAGGGGC	322.599730343108	309.940994981687	317.650231320028
+GTAAGGTGCTACGTGGGA	GTAAGGTGGTACGTGGGA	329.791738219571	317.210999985374	324.885748886244
+GGGGACCACGGAGCGCTT	GGACACCACGGAGCGCTT	338.493908487715	328.075398097466	336.508505160201
+ATGATTCGGCTGATGTTG	ATAATCCGACTGACGTTA	326.617373214063	284.388619463998	294.654772560060
+CGTTGCACCTGTGCAGAC	GGATGGACCTGTGGAAAC	333.382064119484	257.494983943418	268.960440677818
+GGGTATGGAACCGAGCTT	GGGTATGGAACCGAGCTT	330.144774353530	330.144774353530	337.264117120300
+CCGTTTAGTACTTGAGCT	TCGTTTAGTTATTGAGCT	325.101412938739	306.110310829587	314.815405000962
+TGCTCGCCGTTGATG	TGCTCGCCGTTGATG	328.982207099090	328.982207099090	335.675432532816
+GCGGTATCCTGTTTAATAGCAC	GCGGTATCCTGTTTAATAGCAC	330.022926941214	330.022926941214	337.146106169306
+ATTGCGTTGCCCAACGGGAAAC	CTTCTAGATCTCGTGTTCTGTC	337.501498070251	30.100236916645	34.669973146917
+GAAGCGTACTAC	TAATCGTACTAC	312.135520730872	290.578334792837	298.230744679790
+TCGCTTAAGTTGGACCCG	TCGCTTAAGTTGGACCCG	330.825518413055	330.825518413055	337.820664158968
+TCAAAATATCGACGATCGGGTA	TCAAAATATCGACTATCGGGAA	329.618484688871	316.847687836049	324.679351875168
+CTGTAAGTGCGT	CTGTAAATGCTT	315.552108712421	284.478519286229	292.079870838245
+ATCCGTTGTCTGTAT	TCCTATCTTGTCTCA	319.236231890115	0.000000000000	0.000000000000
+GCTGACGATCACCACTGAGT	GCTGACGATCACCACTGAGT	332.887137941413	332.887137941413	339.934427106782
+GGGACCGTCTTGCCTGAA	GGGACCGTCTTGCCTGAA	333.069804290103	333.069804290103	340.197004324922
+GACACTTCATGTTGAAGGAA	GAGACATCAGGCTGAAGGAA	326.623921634303	290.160218586875	299.116365655134
+AAAGTTAATACT	AAAGTTAATGGC	300.439776727757	274.074904518474	281.916563483524
+TTCACTCAGGCCGGGACT	TTCACTCAGGCCGGGACT	333.775772265987	333.775772265987	340.923230531829
+TGAGGTGTACCTTCGATAACCCCTG	TGAGGTGTACCTTCGATAACCCCTG	335.429777164410	335.429777164410	342.806589346267
+GGCGTTAGACTGGGACTCAT	GGCGTTAGACTGGGACTCAT	332.314063072189	332.314063072189	339.465122151725
+GCGGAAAATTCCTCCTGGAGGC	GCGGAAAATTCCTCCTGGAGGC	336.555274112350	336.555274112350	343.900546225317
+ACCCGCCGGGTTATG	AACCGCCTGGTTATG	330.420331626846	303.591907390707	312.104888476139
+GTGAATCTGTGCCATGTAAGGAGTA	GTGAATCTGTGCCATGTAAGGAGTA	332.502726598592	332.502726598592	339.772881635043
+TGTGCATATGCGAAGCAGCA	TCTGCATATCCGAATCAGAA	333.922832979848	300.308875168811	310.038091502733
+TTCCGGGAGAGTCATTCAAGCGATT	AAGCGAGGGGAGACCCTTCTTTACG	335.964252182230	0.000000000000	0.000000000000
+TATCACTTACTCCCCGGA	TATCACTTACTCCCCGGA	327.110291836484	327.110291836484	334.201971889462
+ACGCATTAGTCTACCGTCTTGC	ACGCATAAGTCTACCGTCATGC	333.304407097527	323.852674403909	331.529871399707
+ATAATAGATGAT	ATAATAGATGAT	298.724754718266	298.724754718266	305.105559067848
+ACTTCATTGGCCTCTTTC	GGGATAGGCCATTAGAAA	325.919067175754	0.000000000000	0.000000000000
+ATTTGCGTTAAGGAGACTCA	ATTTGCGTTAAGGAGACTCA	327.736465255252	327.736465255252	334.763124999138
+GGGAGCCACGTCCCGCACTT	TGGGTCTGTTCTACGGGTCC	340.351804998615	66.406913767236	75.840004728444
+AATGGTAGCTTGGAACAACT	AATGGTATCTTGGAACAACG	327.871691250641	318.316542767859	326.227958663349
+ATACAATACCAA	GTAAAACCCTAA	304.232095602747	144.099149360347	153.765346779177
+TTCTAGCGTCCCGGTACT	TTCTAGCGTCCCGGTACT	330.589206350792	330.589206350792	337.589326383339
+TGGCCTTTTTTTAAGTAAAT	TGGCCTTTTTTTAAGTAAAT	321.872839855264	321.872839855264	328.960538235470
+ATGCCTCCTTGTAATACAGCTAGGG	ATGCCTCCTTGTAATACAGCTAGGG	333.823387389837	333.823387389837	341.214372867702
+GAGATAACACGCCAGGTTAA	GGGATAACACGTCAGGTTAA	328.361333311021	307.593342393239	316.302295915689
+CACGGGCATCAAATG	CACGGGCATCAAATG	324.403592056034	324.403592056034	331.200512200928
+TTAACTGTCGGTACTCTT	TTAACTGTTGTTATTCTT	323.363845103162	280.866882856653	291.385325188001
+CACGATAGTCGTTGA	TTCAAAGGTGAGTAA	321.130711058809	128.711478226556	138.526211474293
+GCCGGCCTAAGGACG	GCCGGCCTAAGGACG	331.645745178000	331.645745178000	338.559056448220
+ACGCAAAAAGCGCAT	ACGCACAAAGCGCAT	327.022026932331	320.836952659324	328.322467858379
+AGTCTTAAACGTTTATTG	AGTCTTAATCGTTTATCG	318.625758737664	304.665747911265	312.350081264934
+CTGATCCGAGTTTGGCAGCT	CTGAACCGAGTTTGGCAGCC	333.515248215999	325.848131196554	333.400292317441
+AATCCCCCTGCGCGGTGG	AATCTCCCATCGCGGTGG	338.914827359853	316.871536337894	325.825478445471
+CCTATTGCGAGTGGAGTTTTTC	CCTATTGCGAGTGGAGTTTTTC	330.398722345826	330.398722345826	337.555411082006
+TTTGTAGCACTCTCTAAG	TTGGCACCGCTCTCGAGG	321.958335092061	240.192027201542	251.110839474907
+TATAAATCCAAACCG	TATAAATCCAAACCG	314.941565560136	314.941565560136	321.679961211234
+TCTGGTCTACGGACCAGA	TCAGGAATGCGGACCAGA	329.776739773812	300.216331414806	308.948988559444
+GCGATTCTGGAAATT	GCGATTCTGGAAATT	319.582046008775	319.582046008775	326.323905265890
+AACTTATTGTAAGTGGCT	ATCTTATTGTAAGTGGCT	321.893436025311	314.515961362535	321.963655827352
+AGTAGATCCACGCTC	GGTGACGGTATAAGT	322.922652409125	0.000000000000	0.000000000000
+AGACAGCCAATTTGGACC	AGACAGCCAATTTGGACC	328.624785345222	328.624785345222	335.740638937702
+TCTCCCCATCTGGTTGTGACAC	CCTCCCCAACTGGCTGTGACAC	334.475047649994	322.522934177182	331.074093710435
+GTCCGGAGGGAAATT	GTCCGGAGGGAAATT	323.695895292588	323.695895292588	330.657945722303
+GCATTGGTGTCTGAGAGACATC	GCATTGGTGTCTGAGAGACATC	331.823924496399	331.823924496399	338.987388491055
+TACTCTTCGTTG	TACTCTTCGTTG	310.392977904224	310.392977904224	316.757886088883
+ACCTACTACAAT	ACCTACTACAAT	306.714175636802	306.714175636802	313.206330066357
+CCTTACACTATGCACCCG	CCTTACACTATGCACCCG	328.839039592404	328.839039592404	335.870639509527
+ACCTGTAAGATCGAGTGATATCGGA	ACCTGTAATATCGAGTGATATTGGC	332.453410863819	317.742424213661	326.757759926067
+ACCCGGAATCAC	ACCCGGAATCCC	318.062682068531	309.456479129025	317.168301011525
+TTGCATCTAAACATTAAAGCAGAAG	TTATGAGCCTTCCTAATCCCGCAGC	328.558653084235	92.581987798659	103.905955829058
+CCGCGCCAATAT	ATCACAAAGGAA	320.315607213397	0.000000000000	0.000000000000
+ACCAGCCTAACTTTTGTT	GCCAGCCTAACTGTTTTT	325.132735655685	301.134269729448	310.106656597025
+GCTTAATGCTGGCTACGC	GCTTAATGCTGGCTATGC	330.692022417212	320.449633481302	328.448263187090
+TGGTCCGAACGGCAACAC	TGGTCCGAACGGCAACAC	334.376696093757	334.376696093757	341.354168304032
+ACCCGGCCAGGG	ACCCGGCCAGGG	328.950241125024	328.950241125024	335.887941031174
+ATACCATCGCCTAAGATTGGCC	ATACCATAGCCTAAAATTAGCC	332.848856616901	309.476145134398	318.241854893043
+AAACCGGCAGTA	AATCCGGCAGTA	317.201144426162	305.365706144159	312.555666368026
+CACAACAACGTTCAA	CACAACAACGTTCAA	320.677535999798	320.677535999798	327.306012831155
+TGGGTCGCGTCCCAG	CAGTACCCAACCCCG	333.123346113342	246.742853521647	257.809384715704
+CTCCTAAACATGAAGAACCAGAAAT	CTCCTAAACATGAAGAACTAGAAAT	329.191307133388	321.504803533715	329.322148156357
+TACTATTTCGTTTGC	TACTATTTCGACCTC	316.071427737447	286.990612725496	295.372324621468
+TCCCTGGCAGCAGGGTAGTTTTGAA	TGGGCCAACGCTAACTGGTGCCTTG	337.792849188337	121.771136708049	135.395255297683
+GCTACATTACCGGCTCTC	GCTACATTACCGGCTGCC	328.627216292430	317.865179052925	326.512498741613
+TGAGGCGGGTTATGCGTTAATG	TGAGGCGGGTTATGCGTTAATT	333.877486753212	330.936597263706	338.449826872471
+CACTTATCCCGCAATAAT	AACTTATAACGCAAGAAT	323.209491396864	285.261497168294	294.371363704620
+CGCGCCCGACGATCCACT	CCCGCCTGACGTTCCACC	338.845192827705	310.574091806929	319.266524881313
+AGGCTCCTCTATGAA	AGTCTCCTCTATAAA	320.534065615391	298.120651463631	306.645491242189
+TGGTAATTGCCTGAG	TGGTAATTGCCTGAG	320.915868643696	320.915868643696	327.776118922002
+GTTCATTCTCTGGAGGGC	AAGCCGGTACCTGCGGGG	328.693062285185	175.795089391355	189.885811626097
+GAACGCTGGTCGTCACAT	GAACGCTGGTCGTCACAT	331.556430142525	331.556430142525	338.434764177077
+ATTGGGTGATCGTAAGTAAG	ATTGGGTGATCGTAAGTAAG	325.612525475036	325.612525475036	332.668318324958
+AAGATATGTTTGTAGGCCGAAC	AAGATATGTTTGTAGGCCGAAC	329.104322856711	329.104322856711	336.238620405649
+GTGTGGGTGCCC	GTGTGGGTGCCC	324.309319433451	324.309319433451	331.095278789318
+GGCGTCCAGAAT	GGCGTCCAGAAT	318.801851843066	318.801851843066	325.371455269089
+TGGCTCTGGATAGCTCCCCC	AAGGACTATACAAAGCCAGT	336.053481241646	140.181856955550	152.924856339728
+GTAATCTACCATATG	GGAATCTACCATATG	311.238267655019	297.565035184391	305.303043725135
+TAGAGACGTTTCCTCGCCTTTATCG	CTTAGACACTCAGGTGTTGTTGGCA	334.313541801127	188.908254593789	203.829299320958
+TGCCAGCTTGTGGCGAGTAT	TGCCAGCTTGTGGCGAGTAT	335.045486995831	335.045486995831	342.139981881034
+TTCCGAGCCTGCCGAACAGG	TTCCGAGCCTGCACACCCGG	337.981474268738	315.139472928922	324.695198064855
+TAGGCGACGGCTAACTGG	ATTAACACAGAATGTAAC	332.611090573720	117.192190429110	129.003910625976
+CTATCAAGCCTCAGGTATCGCT	CTATCAAGCCACAGGTATCGCT	332.013904222950	327.008869759689	334.361264460119
+GCGTTGAACAGTGCGTACGT	GCGTTGAACAGTGCGTACGG	334.922740354371	331.487402480161	338.817709417777
+ATGACCTGTACGCTG	TCACACCCTCGGTCG	324.096877104678	76.811640858322	84.402210669196
+TAGTTATCTTCTACCTGATGTAAGC	TAATTATCTTCTACCAGATGTTAGT	327.657446380070	306.418404503197	315.329248976153
+CGGTGCAGTGCAAAATGCGAGTAGT	CGGTGCAGTGCAAAGTGCGCGTAGT	338.234766348875	331.605213173016	339.892622743888
+TTGTTGCCATGTAAA	TTGTTGCCATGTAAA	318.513544450161	318.513544450161	325.252135026214
+GTGAAACGTTTGCTGTGG	GTGAAACGTTTGCTGTGG	328.992886751469	328.992886751469	335.904977195626
+GTTGGAATTGCGGCAAAG	GTTGGAATTGCGGCAAAG	329.419693351400	329.419693351400	336.405013425441
+ATTTCCCGCAGCTTCGAC	AGTTCCTGCAGCGTTGAC	331.921604125857	275.704268703669	286.426092962088
+GGGCTTCAACTCTGCAGA	AACTTCCGATAGTACCCG	330.715768712068	0.000000000000	0.000000000000
+GTAGTACGTCACTTAGGGTATA	GTAGTACGTCACTTAGGGTCTA	326.370618283308	323.603568352225	331.248373111091
+GTGATATTTTCACTGCTAGG	GTGATATGTTCACTGCTAGG	324.660738933852	318.097303613935	325.739236970914
+TTAGAGCCTAGATATCGC	TGAGAGCCTACATATCGC	323.737203572163	308.667021683076	316.877776506437
+GGCCGGTTTGAT	GGCCGGTTTGAT	319.628239742093	319.628239742093	326.261640206404
+CGATAGTGGAAG	CGTTTGTGGAAG	311.281036639251	282.148143916626	290.555750610531
+TTTATAGACGGTCCA	TTAATAGAGGGTCCA	318.271295635730	289.396103709646	297.309467179028
+CTGGTGTAAATTTTTATT	CTGGTGTAAATTTTTATT	316.242691168716	316.242691168716	323.159373336702
+CACGCAAAGACGAGTCGAGA	CGCGCTAAGACGAGTCGAGC	333.509565254824	317.485155999984	326.212807278748
+GCCCCTGTCCGGACG	GCCCCTGTCCGGGAG	334.323987481155	318.296185453565	326.861053252518
+AAGCCGGCACTA	TAGCCGGCACTA	320.841087637638	315.681196183552	323.022375994306
+CCCCCTTGTTCT	CCCACTGGTTCT	316.846355213233	271.731156926471	280.431509587266
+TTTGGGTCCGATACGTTA	TTTGGGTCCGACAGGTTC	326.526907539640	304.936001347139	313.235900619784
+CTAGTAAGGTATAGAACT	CTAGTAAGGTATAGAAAT	317.080850733325	306.909216639793	314.322636687809
+ACACTGCAACATGCAAGGCA	TCTCTGCACCATACTAAGCA	334.279193488462	287.410425546330	298.337166942414
+CGGATACGTGAGCTGACCGTCACTG	AGGGTTCATGAGCTGACCGTCACTG	339.003807135660	321.603408699629	331.135793238806
+CACAATCCTTTCATAGGTCTGATGT	CACAAACCTAACATATGTCTGGTGT	330.769444740710	302.850977615703	312.757796843091
+ATAGGTGTATTACTTCCC	ATAAGTGCATTACTTCCC	320.813390192036	307.309069291562	315.330380672069
+TGCCCGTGCTAATTTCGG	TGCCCGTGCGAATTTCGA	331.916704163337	321.788326255281	329.840680268581
+CATACCTCCCGTGTGACC	CAGACCTCCTGTCTGAGC	330.790040366578	275.685519938138	285.797334919463
+CATGCGAATGAACGTGGTCAAC	CATGCGAATGAACGTGGTCAAC	333.744727353783	333.744727353783	340.783101574584
+GGTCCAGCCAGC	GGGCCGGCCAGA	323.997990076396	258.335190180963	269.085542668344
+TCAGCCTCTTACTACGAA	TCAGCCTCTTACTACGAA	325.504908314887	325.504908314887	332.433608839830
+TTCCACTCGGTA	TTTAACTCGTCA	315.163712062905	240.754025130773	251.279252359821
+GTACCCATCCGTTAACCACATT	GTACCCATCAGTTAACCACATT	330.965975035173	322.116695925385	329.685541486863
+TCGGATTTGTGG	TCGTATATGCGT	315.144792522878	261.302945400146	270.745105924733
+CAGGGGTTTGTAGCTTGCAT	AAAAGGTCGACATGAAACTC	331.592676687332	178.298382734095	192.121517719714
+GTATGTTTAGATTCCGGT	GTATGTTCACATTCCGGT	322.407865448171	309.819055451666	317.927267476480
+ATTTGAAGGTTGTTCATT	ATTTGAAGGTTGTTTATT	320.745098479303	308.460034775676	316.604473343016
+ACGGTGGCCGGCGACCTCAGCCGCA	TCGGTCTCCGGCGAGCTCAGTCGCA	349.319476545950	322.805970408218	332.569513311728
+GGTGAAACATGGGTGGTATGATAAT	GGTGAAACATGGGTGATATGATAAT	331.043479784795	326.332708138854	334.184817037240
+AAGTGTAATCGACTA	AAGTGTAATCGACTA	315.631885291475	315.631885291475	322.265747968200
+GAAGAAAATCCGTTTAGT	GAAGAAATTTCGATTAAT	320.919322767873	264.519341261783	274.695121470457
+GTATACCTACCGCCAGGCGTAA	GTATACCTACCGCCAGGCGTAA	334.199822075507	334.199822075507	341.373552239777
+GGGCTTGCAAATGCACTAGT	GGGCTCGCAAACGCACATGT	332.244946736482	316.459896257268	324.959991904042
+AGTAACATTACCCCTCTT	AGTAACATTACCCCTCTT	322.762790159849	322.762790159849	329.855166947719
+ATCGTTCCAGATTCA	ATAGTTCCACATTGA	319.593588349254	263.703483135864	273.889968541208
+ATCTAGACAAAT	CTCTAGACAAAT	303.769798710798	299.975537128382	306.767744251403
+GACTCGTGCCCAAAGGCG	GACTCGGGCCTAAACGCG	335.453032103533	305.944507861049	314.863300937474
+ATACCTGAAAGATACATAGGGAGGG	ATACCCTAACGATACATAGGGAGGG	330.838488925824	319.951646252133	328.421632500441
+CTCCTTCTATATCTTTACGC	CTCCTTCTATATCTTTACGC	323.614245457101	323.614245457101	330.629799804542
+GGAACGACAGCGGAC	GGAGCGACATCGGAG	329.215799373567	299.203726696696	307.870410467643
+AATTAAACTTCACTTGCTGCTA	AATTAAACCGCACTTGCTGCTA	326.774684409898	319.760152873130	327.648310708541
+CGGGGTGGTTTTAACGCCTT	CGGGGTGGTTTTAACGCCTT	334.293634381856	334.293634381856	341.479503422465
+TTACAAGTGTAC	TTACAAGTGTAC	306.490429220235	306.490429220235	312.860219905519
+GGTTTTACGCACTCGTTT	GGTTGTACGCACCCGTTT	327.475666665468	315.253671257952	323.597273287183
+GCGACCGGGATATTCGTG	GCGACTGTGATATTCGTG	332.081729064974	312.612274551236	320.900746018201
+AAATCATAGATGTTTGTGTG	AAATCTTAGACGTTTGTGTG	322.490478539460	311.233326719324	319.056840527317
+CCGCGGCTGGCCAGTGCG	CCGGGGCTGGTCAGTGCG	343.595298549215	320.692138927062	329.428906251676
+TTAGCGCCCCCT	TTAGAGGCCCGT	323.638703533546	223.193289824279	232.649538093037
+CGACTCCTTGCG	CAACTCGTTGCG	321.093094443631	287.627523199553	294.941907978827
+CAAATAGGGGAGAAATAGGGATGAT	CAAATAGGGGAGACATAGGGATGAT	329.829565362939	326.229605874518	334.271529764424
+TTATGACTCTAT	TTATGACTCTAT	302.351970012031	302.351970012031	308.749474038831
+CTTCGCACGGCTCTGAGGTG	CTTCGCACGGCTCTGAGGTG	336.758970783676	336.758970783676	343.832413942225
+GGTAGATCGTCTAACACA	GGTGGATAGTCTAGCACA	324.183145729044	288.591703366550	297.757637466896
+TCCGAAGTTATTTGT	TCCGAAGTTATTCGC	317.020120014950	307.809184902247	315.094442186122
+ACGCTATAGGTATCCAATGTGC	TGAGGGGTTGCCTCGCGCGAGA	330.668816473345	168.517116213496	182.239452513402
+ATCCCGGTGTTT	CTCGCGTTCTTT	316.963839881018	238.291774003272	248.409713749044
+TGTAAATATTGAGAACAT	CTACTTGTAAAAATTATC	316.552099039474	4.196619607978	4.793908669173
+TAAGATGGCGGTGCTAAGGT	TAAGATGGCGGTGCTAAGCT	331.748855931570	328.095422595895	335.422537658975
+CAGAGAATCGTATTG	TAGAGAATCGTATTG	316.058437382107	309.336246420051	316.478881368674
+ACACTATCGCAT	ACACTATTGCAT	313.287559156327	289.002177419194	296.901322522936
+TTTGCAACAAACTGCGGG	TTTGCAACAAACTGCGGG	330.987598215230	330.987598215230	337.964638501570
+TGCGCATTGACCGTATGA	TGAGCATTGCAAATATGA	330.724505651688	291.707022981886	301.075127848151
+CTGCAACGTGGCTGACGTGG	CTGCAACGAGGCTGACGTGG	337.828829590951	332.158127068020	339.396841315526
+AATCTAGGTATTCTGTGCACACTAG	AATCTAGGTATTCGGTAGACACTAG	330.093652788304	312.242546896813	320.851841260874
+ACGGCGAAACTGTCCCGT	ACGGCGAAACTGTCCCGT	335.434181438533	335.434181438533	342.388102607306
+AACGAGTACTCTTTGCACGCGC	AACGAGTACTCTTTGCAAGCGC	336.704192051484	329.670890289599	337.116780737979
+ACGGCCACCCGCGTAAGCCAGT	ACGACCACCCGCGCTAGCCAGT	343.225833795195	335.051197281573	343.935168403898
+AGGTGGAAACACCTCTTTATAG	AGGTGGAAACACCTCTTTATAG	327.767855921026	327.767855921026	335.018188736768
+GATGCACCTCTTCCTTGT	GATGCACCTAGTCCTTGT	328.015528597840	315.813502381439	323.875578738504
+CACACGTTAAATCAGCCGTGAC	CACACGTTAAATCAGCCGTGAC	333.178564800892	333.178564800892	340.213130099053
+GTGAAAAACGCTAAAACT	GAGAAAAACGCTGAAACT	322.651310853696	305.102360687419	313.386535231522
+CCTCTAAGCGTAAACTGGCGTA	CCTCTAAGCGTAAACTGGCGTA	332.731489287653	332.731489287653	339.833318025496
+CTTAACGGATCAGGTGCC	CTTAACGGATGAGGTACC	329.145552153189	309.298607409059	318.329957990940
+ACCAGCGCGGGTCCAAGCCGGG	AACAGCGCGGGTCAAAGCCCGG	346.277068986675	323.989295670246	332.757718044482
+GGTCGCCCTTGGTTG	GGTCGCCCTTGGTTG	329.378530415267	329.378530415267	336.338999066477
+GGTCTTTCCTGATAG	GGTCTTTCCTGCTAG	317.502443546558	312.248143518990	319.891057909696
+ATGGGGTCCCTTGGATAA	ATGGGCTCACTTGGAGAA	327.935969186382	299.834873075580	308.768543906097
+AGACTTTATCTGCGTTAG	CGACTTAATCTGCATTGG	322.530698528657	290.771157085417	299.660782150129
+TTCTAGCCTGTGCCATTCATTCGCG	TGTCTTTAATCATTCACTGCATTTT	337.662752230513	0.000000000000	0.000000000000
+GTGGCGTGATCCGCCTATGATG	GTGGCGTGATACGCCTATGATG	336.412147017699	328.612405834565	336.461928234461
+ATTATCCTGCAGAGCAGTGA	ATTATCCTGCAGAGCAGTGA	329.338095898179	329.338095898179	336.434344794516
+TGGGGTCACGCATGGCACCG	TCGTTTCCGCGTAGACACAT	340.932042495783	219.946305376159	234.105527008232
+GATCAGTCATGCCGGGCA	TCGCTCACTGGGAGCTTT	333.615526310877	70.980108149928	78.760105051004
+GAGAAACTACCAACCACC	GAGAAACTACCAACCACC	326.141893376412	326.141893376412	333.228102476867
+ATTGACTAGCCTGGAGGC	ATTGACTCGCCTGGAGGC	329.814592678115	325.021169884577	332.608693895943
+ACGGAGTCTATTGTC	AAGGAGTCTATTGTC	319.985406470302	306.562131243465	313.933774764732
+TGGCTGATTGGCCATTATTC	TGGGTGATTGGCCATTATTC	329.296931346104	317.833565606159	325.604567901985
+TGTTACGGACGGACG	TGTCAGGGACGCACG	327.360275144725	294.262459737383	302.641243348176
+AACGCTCCTTCTGACGCTCTGTACC	AACTCTCCTGCTGACGCTCTGCACC	338.281645508427	324.671859746526	333.350952165099
+CTTTCTACAAGCTTCCTCAGCCGAT	CTTTCTACAAGCTTCCTCAGCCGAT	334.726254329200	334.726254329200	342.014376175497
+CTGGGAGCGTGT	CTGGGAGCGTGT	322.225101677135	322.225101677135	328.814148322577
+GCAAGTGCTCCATACGTCAGGA	GCAAGTGCTCCATACGTCAGGA	335.151235041562	335.151235041562	342.311691193007
+TTGAACTCGAAGTCATTCCAGCCCA	TTGTACTTGAAGCCATTCCAGCCCA	336.086839523526	320.167401115734	329.871603131488
+CGCGAGTGTTAATTTTCATC	CGCGAGTGTTAATTTTCATC	326.706848503872	326.706848503872	333.607112791387
+CTCGAAATCAGCCTA	CAGGCAGATCCGATA	320.433152901209	106.785241479223	115.721577071620
+CCCACGTTGGATCCCTCA	CACACCCAGCGTTCCCCT	332.434732686040	168.247008949882	182.904021311056
+TCTTCCCGAAGGCAACTA	TTGCTAGCCACCGCTACC	328.742342595200	0.000000000000	0.000000000000
+TCGGTCGCCCGGCCA	TCGGTCGCCCGGCCA	338.280658715332	338.280658715332	345.187830452143
+TTCCGGTCTAGAGAT	GTCCCGTCTAGAGAT	320.290802240471	304.664615132156	311.959714415134
+TGCCACGTTTGG	TTCCACCTTTGG	320.673142294482	289.947588641272	298.070057172917
+CTGAGAACATGG	GTTAAGTTAAAA	311.861058614240	0.000000000000	0.000000000000
+TTCCCATTTGCAACG	TTCCCATTTGCAACG	323.462465099785	323.462465099785	330.249305701134
+AAGAGTTACTGGATTACC	ATTACATGTTGAGCTAAC	321.460730667492	45.295339819983	50.838550888654
+AAAACTCATTACCGTCAATGTGCCG	AAAACTCATTACCGTCAATGTGCCG	334.427238998430	334.427238998430	341.612309670089
+CGGGAAATCATGCACTTC	CGGGAAATCATGCAGTTC	327.837268831791	315.451695083719	322.783155312403
+TCTTGTCCATCAGACAGA	ACTTGTCGACCAGACAGA	325.674390994389	302.134938899268	310.971599041593
+TAACTTTGCCGTGCGTTC	TTGCATAGCATGCAATTT	330.185153783687	164.313401929230	176.255433198370
+GAGATTGACGACTAACCT	GAGATTGACGACTAACCT	324.455556483854	324.455556483854	331.390094199272
+CCGCGAGTCCACGGCGTAAG	CCGCGAGTCCACGGCGTAAG	339.144645429285	339.144645429285	346.143799055883
+TGACGGCGAGTGAGTTTT	GGATTAGTCATCATCGCG	330.479181965474	47.596928980319	53.756086430128
+AGCAACGTCTAAACTCGATT	AGCAACGTCTAAACTCGATT	328.165361220407	328.165361220407	335.096430259205
+GACTGCTAGCTGGCT	GAATTATCACGTTCC	326.800285243793	145.119103179615	155.884495817015
+ACCCTCGTAATTATTCCG	GCCTCGCTAGAGAGGGAA	324.895708998494	137.294573703070	148.579615770821
+AGCGGGCCGGCGCCCACGGTTAAAT	AGCGGGACGGCTCCCACGGATAAAT	346.469332548688	327.821835178262	336.666611815380
+ATACACCAGTCTCCACTGTGTCAGC	ATACACCAGTCTCCACTGTGTCAGC	336.084026510573	336.084026510573	343.378799103379
+AGGCAACGGTGTTTTTAGTTTCAAG	AGGCAACGGTGCTTTTAGTTTCAAG	332.791413039949	329.450267766259	337.042137604359
+CATAGTATGGTTCATAATTTAA	CAAAGACGGGATTATCGGACTT	319.805110852434	183.915523269303	197.052684250500
+ACGCCCCCCGGATTCGGC	ACGCCCCGCGGATTCGGC	341.276243356091	331.161466849465	338.916726408046
+TTGTCTTTCGAGCGG	TTGTGTGTCCAACCA	325.282149224729	222.156190895896	232.598751363606
+CTCCCGGGCGCCAGATGG	CTCCCGGGCGCCAGATGG	339.311369482363	339.311369482363	346.533954436015
+GCACACTTTGTCGGTAAGTGAGGAC	GCACACTTTGTCGATAGCTGAGGAC	336.007697297418	324.718456248143	333.231550897913
+ATATAGTCTCCGTGT	ATATAGTCTCCGTGT	317.670914323835	317.670914323835	324.391677867921
+ATTCGAGGCGGCTCTGAGGT	GTTCGAGGCGGCTCCGTGGT	336.308093587336	324.901517012421	333.408982676338
+CGCTAGCTCTCAGCATAA	CGCTAGCTCTCAGCTTAA	327.750384254086	320.869735908381	328.340764122397
+TACTCACTCTAATTCAAC	TACTAACTCCAATTGATC	319.232623419269	263.254120288054	273.886368603122
+CTTGGAATCTTGTTCCGC	CTTGGAATCTTGTTCCGC	327.748687917447	327.748687917447	334.748570070981
+TCTCTCTCTACTGAAGATGT	TCTCTCTCTACTGAAGATGT	324.955441779254	324.955441779254	332.010856900828
+AGATAGCTAAACTATGGTATTT	AGATAGCTAAACTATTGTATTT	322.379524984208	314.920600083519	322.617736532767
+TCCAGTAGGCCTGCACACCC	TCCTGTAGGCCTGCACACCC	337.082762403533	331.065336200951	338.847431016473
+CCGGTCGGTTAA	CCGAGCGGTCAA	318.298891942113	278.422489888824	287.988343096102
+TGCGGTATTACACAGTAC	TGCGGTACTAGACCGGAC	325.257155822510	277.211655556984	287.317024862518
+TAGGTTGAGATGTTCGGGCG	TAGGTTGAGATGTGCGGGCG	332.954776706522	326.896199598776	334.681687928661
+TCGAACCTCTAA	TCGAACCTCTAA	311.195901304215	311.195901304215	317.645142918670
+GGCCGTTGGGTGAGATGT	GGCCGTTGGGTGAGATGT	333.461534051827	333.461534051827	340.580014223162
+TTACGATTAGTAGAC	TTACGATTAGTAGAC	313.227722774298	313.227722774298	319.837422500332
+GCTGGCAGACTTCCGGGA	GCAGGTAGACTGGCGGGA	335.491766864185	300.268242946003	309.735154150044

pkg/obithermo/testdata/ecoprimers_tm.tsv

@@ -0,0 +1,13 @@
+AAACTGGGATTAGATACC	48.5
+ACACCTCGACCTGACGTT	58.8
+ACTTACCATGTTACGACT	50.8
+CAAACTGGGATTAGATAC	47.5
+CCAAGTGCACCTTCCGGT	60.7
+CCTGTTCTAGAACCGATA	50.8
+CTACACCTCGACCTGACG	57.5
+CTTTTGCCACAGAGACGG	57.0
+GCCAAGTCCTTTGGGTTT	56.2
+GGGATTAGATACCCCACT	52.6
+GTTCTAGAACAGGCTCCT	53.6
+TCGTACCTTTTGCATCAT	52.2
+TTCTAGAACAGGCTCCTC	53.3

pkg/obitools/obialign/obialign.go

@@ -0,0 +1,340 @@
+package obialign
+
+import (
+	"os"
+	"slices"
+	"sort"
+	"sync"
+	"sync/atomic"
+
+	align "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obialign"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiitercsv"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+	sim "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obisimilarity"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obicsv"
+	"github.com/schollz/progressbar/v3"
+	log "github.com/sirupsen/logrus"
+)
+
+// Params are the parameters of the comparison.
+type Params struct {
+	Threshold       float64 // 0: no threshold
+	Normalized      bool
+	Distance        bool
+	Reference       sim.Reference
+	AcceptAmbiguous bool
+	ExtraColumns    bool
+	Workers         int
+}
+
+// ParamsFromCLI collects the parameters from the command line options.
+func ParamsFromCLI() Params {
+	return Params{
+		Threshold:       CLIThreshold(),
+		Normalized:      CLINormalized(),
+		Distance:        CLIDistance(),
+		Reference:       CLIReference(),
+		AcceptAmbiguous: CLIAcceptAmbiguous(),
+		ExtraColumns:    CLIExtraColumns(),
+		Workers:         obidefault.ParallelWorkers(),
+	}
+}
+
+// dataset is a set of usable sequences with their description.
+type dataset struct {
+	seqs    obiseq.BioSequenceSlice
+	entries []sim.Entry
+}
+
+// readDataset keeps the sequences that can be compared.
+func readDataset(db obiseq.BioSequenceSlice, name string, p Params) dataset {
+	var d dataset
+	ambiguous := make([]bool, 0, len(db))
+	for _, s := range db {
+		ok, amb := sim.Classify(s.Sequence(), p.AcceptAmbiguous)
+		if !ok {
+			continue
+		}
+		d.seqs = append(d.seqs, s)
+		ambiguous = append(ambiguous, amb)
+	}
+	if discarded := len(db) - len(d.seqs); discarded > 0 {
+		log.Warnf("Discarded %d sequences of %s that did not contain only IUPAC nucleotides (or only 'acgt' without --accept-ambiguous)",
+			discarded, name)
+	}
+	if len(d.seqs) == 0 {
+		log.Fatalf("No valid sequences in %s.", name)
+	}
+	d.entries = sim.BuildEntries(len(d.seqs), func(i int) ([]byte, int, bool) {
+		return d.seqs[i].Sequence(), d.seqs[i].Count(), ambiguous[i]
+	}, p.Workers)
+	for i := range d.entries {
+		if d.entries[i].Length > align.MaxBandedLCSLength {
+			log.Fatalf("The sequence %s is too long (%d bp, maximum %d bp)",
+				d.seqs[i].Id(), d.entries[i].Length, align.MaxBandedLCSLength)
+		}
+	}
+	return d
+}
+
+// pair is a pair of sequences that reaches the threshold: the index of the
+// second one, and the score.
+type pair struct {
+	j     int
+	score float64
+}
+
+// aligner compares the sequences of a dataset with the ones of an index.
+type aligner struct {
+	queries []sim.Entry
+	index   *sim.Index
+	self    bool // the index holds the queries: only the pairs i < j are reported
+	sc      *sim.Scoring
+}
+
+// row finds the sequences of the index that reach the threshold with the
+// query i, and appends them to out in the order of the index.
+//
+// Only the buckets of compatible lengths are visited, and the 4-mer filter
+// works on blocks of sequences.
+func (a *aligner) row(i int, buf *align.BandedLCSBuffer, commons *sim.Commons, out []pair) []pair {
+	q := &a.queries[i]
+	out = out[:0]
+	for _, length := range a.index.Lengths() {
+		if !a.sc.LengthCompatible(q.Length, length) {
+			continue
+		}
+		b := a.index.Bucket(length)
+		commons.Reset()
+		start := 0
+		if a.self {
+			// the members are in the order of the dataset
+			start = sort.Search(len(b.Members), func(k int) bool { return b.Members[k].ID > i })
+		}
+		for k := start; k < len(b.Members); k++ {
+			m := &b.Members[k]
+			if s, ok := a.sc.PairScore(q, &m.Entry, b.Common(k, q.Sparse, commons), buf); ok && a.sc.Passes(s) {
+				out = append(out, pair{m.ID, a.sc.Reported(s)})
+			}
+		}
+	}
+	slices.SortFunc(out, func(x, y pair) int { return x.j - y.j })
+	return out
+}
+
+func progressBar(n int) *progressbar.ProgressBar {
+	if !obiconvert.CLIProgressBar() {
+		return nil
+	}
+	return progressbar.NewOptions(n,
+		progressbar.OptionSetWriter(os.Stderr),
+		progressbar.OptionSetWidth(15),
+		progressbar.OptionShowIts(),
+		progressbar.OptionSetPredictTime(true),
+		progressbar.OptionSetDescription("[Comparing sequences]"),
+	)
+}
+
+// maxPairsWithoutThreshold is the number of pairs above which a missing
+// threshold is reported, the table having then as many rows.
+const maxPairsWithoutThreshold = 1e8
+
+// maxRowsPerRun is the number of rows of the table that a run is expected to
+// produce at most when every pair is reported.
+const maxRowsPerRun = 100000
+
+// runShape chooses how many sequences a run covers, and how many runs a
+// worker may be ahead of the oldest unfinished one. Without threshold every
+// pair is a row of the table, so the runs are shortened to keep their size
+// reasonable; otherwise many pairs are rejected and the runs are longer, to
+// reduce the synchronizations.
+func runShape(n, targets int, noThreshold bool, workers int) (rowsPerRun, window int) {
+	rowsPerRun = min(max(n/(workers*32), 1), 256)
+	if noThreshold {
+		rowsPerRun = min(rowsPerRun, max(maxRowsPerRun/max(targets, 1), 1))
+	}
+	return rowsPerRun, 4 * workers
+}
+
+// Compare compares every sequence of db1 with every sequence of db2, or with
+// the following ones of db1 if db2 is nil, and returns the pairs that reach the
+// threshold as a table with the columns id1, id2 and score (and count1,
+// count2, length1 and length2 if p.ExtraColumns).
+//
+// The rows are in the order of the sequences of db1, then of the ones of
+// db2, whatever the number of workers.
+//
+// The workers take runs of rows of the matrix of the pairs, as in sumatra,
+// but each run is sent to the writer with its rank: the writer puts them back
+// in order, so that the output neither depends on the timing nor needs a lock
+// on the output.
+func Compare(db1, db2 obiseq.BioSequenceSlice, source string, p Params, onProgress func(done int)) *obiitercsv.ICSVRecord {
+	d1 := readDataset(db1, "the input", p)
+	d2 := d1
+	self := db2 == nil
+	if !self {
+		d2 = readDataset(db2, "the reference file", p)
+	}
+	log.Infof("%d sequences", len(d1.entries))
+	if !self {
+		log.Infof("%d reference sequences", len(d2.entries))
+	}
+
+	all := d1.entries
+	if !self {
+		all = append(slices.Clone(d1.entries), d2.entries...)
+	}
+	sc, err := sim.NewScoring(sim.Params{
+		Threshold:  p.Threshold,
+		Normalized: p.Normalized,
+		Distance:   p.Distance,
+		Reference:  p.Reference,
+		Ratio:      1e300, // no limit on the counts
+	}, all)
+	if err != nil {
+		log.Fatalf("Wrong threshold: %v", err)
+	}
+
+	index := sim.NewIndex()
+	for j := range d2.entries {
+		index.Add(d2.entries[j], j)
+	}
+	a := &aligner{queries: d1.entries, index: index, self: self, sc: sc}
+
+	iterator := obiitercsv.NewICSVRecord()
+	for _, f := range []string{"id1", "id2", "score"} {
+		iterator.AppendField(f)
+	}
+	if p.ExtraColumns {
+		for _, f := range []string{"count1", "count2", "length1", "length2"} {
+			iterator.AppendField(f)
+		}
+	}
+
+	n := len(d1.entries)
+	workers := max(p.Workers, 1)
+
+	pairs := float64(n) * float64(len(d2.entries))
+	if self {
+		pairs = float64(n) * float64(n-1) / 2
+	}
+	log.Infof("%.0f pairs of sequences to compare", pairs)
+	if p.Threshold == 0 && pairs > maxPairsWithoutThreshold {
+		log.Warnf("No threshold has been given: the table will have %.0f rows, "+
+			"use the option --threshold to keep only the similar pairs", pairs)
+	}
+
+	rowsPerRun, window := runShape(n, len(d2.entries), p.Threshold == 0, workers)
+	runs := (n + rowsPerRun - 1) / rowsPerRun
+
+	iterator.Add(workers)
+	go func() {
+		iterator.WaitAndClose()
+	}()
+
+	// The runs are sent to the writer with their rank, and the writer holds
+	// those that arrive before the previous ones. To bound that memory, a
+	// worker does not start a run more than `window` runs after the oldest run
+	// that is not finished.
+	var (
+		lock     sync.Mutex
+		progress = sync.NewCond(&lock)
+		finished = make([]bool, runs)
+		oldest   int
+		rows     int
+	)
+	var next atomic.Int64
+	var wg sync.WaitGroup
+	for w := 0; w < workers; w++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			defer iterator.Done()
+			var buf align.BandedLCSBuffer
+			var commons sim.Commons
+			var pairs []pair
+			for {
+				run := int(next.Add(1)) - 1
+				if run >= runs {
+					return
+				}
+				lock.Lock()
+				for run-oldest >= window {
+					progress.Wait()
+				}
+				lock.Unlock()
+
+				from, to := run*rowsPerRun, min((run+1)*rowsPerRun, n)
+				batch := make([]obiitercsv.CSVRecord, 0, to-from)
+				for i := from; i < to; i++ {
+					pairs = a.row(i, &buf, &commons, pairs)
+					for _, pr := range pairs {
+						rec := obiitercsv.CSVRecord{
+							"id1":   d1.seqs[i].Id(),
+							"id2":   d2.seqs[pr.j].Id(),
+							"score": pr.score,
+						}
+						if p.ExtraColumns {
+							rec["count1"], rec["count2"] = d1.entries[i].Count, d2.entries[pr.j].Count
+							rec["length1"], rec["length2"] = d1.entries[i].Length, d2.entries[pr.j].Length
+						}
+						batch = append(batch, rec)
+					}
+				}
+				iterator.Push(obiitercsv.MakeCSVRecordBatch(source, run, batch))
+
+				lock.Lock()
+				finished[run] = true
+				for oldest < runs && finished[oldest] {
+					oldest++
+				}
+				rows += to - from
+				done := rows
+				progress.Broadcast()
+				lock.Unlock()
+				if onProgress != nil {
+					onProgress(done)
+				}
+			}
+		}()
+	}
+	return iterator
+}
+
+// CLIOBIAlign compares the sequences as the command line asks, and returns the table of the pairs.
+func CLIOBIAlign(iterator obiiter.IBioSequence) *obiitercsv.ICSVRecord {
+	source, db := iterator.Load()
+	log.Infof("Sequence dataset of %d sequences loaded", len(db))
+
+	var db2 obiseq.BioSequenceSlice
+	if CLIHasReferenceDB() {
+		db2 = CLIReferenceDB()
+		if db2 == nil {
+			db2 = obiseq.BioSequenceSlice{}
+		}
+	}
+
+	var bar *progressbar.ProgressBar
+	barChecked := false // the bar is created once, even when it is disabled
+	last := int64(0)
+	var lock sync.Mutex
+	return Compare(db, db2, source, ParamsFromCLI(), func(done int) {
+		lock.Lock()
+		defer lock.Unlock()
+		if !barChecked {
+			bar, barChecked = progressBar(len(db)), true
+		}
+		if bar != nil {
+			bar.Add(int(int64(done) - last))
+		}
+		last = int64(done)
+	})
+}
+
+// CLIWriteTable writes the table to the output file given by --out, or to the standard output.
+func CLIWriteTable(table *obiitercsv.ICSVRecord) {
+	obicsv.CLICSVWriter(table, true)
+}