Release 4.5.2

This commit introduces significant changes to the sequence processing pipeline, implementing parallel loading, k-mer budget control, explicit memory management, and refined logic for pair evaluation and resolution.

.github/workflows/obitest.yml

@@ -12,7 +12,7 @@ jobs:
     - name: Setup Go
       uses: actions/setup-go@v5
       with:
-        go-version: '1.23'
+        go-version: '1.27'
     - name: Checkout obitools4 project
       uses: actions/checkout@v5
     - name: Run tests

.github/workflows/release.yml

@@ -16,7 +16,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: "1.26"
+          go-version: "1.27"
       - name: Checkout obitools4 project
         uses: actions/checkout@v5
       - name: Run tests
@@ -54,7 +54,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: "1.26"
+          go-version: "1.27"
 
       - name: Extract version from tag
         id: get_version
@@ -78,7 +78,8 @@ jobs:
             -v "$(pwd):/src" \
             -w /src \
             -e VERSION="${VERSION}" \
-            golang:1.26-alpine \
+            -e GOEXPERIMENT=simd \
+            golang:1.27-alpine \
             sh -c "apk add --no-cache gcc musl-dev zlib-dev zlib-static make && \
                    make LDFLAGS='-linkmode=external -extldflags=-static' obitools"
           mkdir -p artifacts

.gitignore

@@ -21,6 +21,8 @@ xx
 
 .rhistory
 /.vscode
+
+/benchmarck
 /build
 /bugs
 autodoc

.serena/.gitignore

@@ -0,0 +1,2 @@
+/cache
+/project.local.yml

.serena/project.yml

@@ -0,0 +1,169 @@
+# the name by which the project can be referenced within Serena/when chatting with the LLM.
+project_name: "obitools4"
+
+# the encoding used by text files in the project
+# For a list of possible encodings, see https://docs.python.org/3.11/library/codecs.html#standard-encodings
+encoding: "utf-8"
+
+# line ending convention to use when writing source files.
+# Possible values: unset (use global setting), "lf", "crlf", or "native" (platform default)
+# This does not affect Serena's own files (e.g. memories and configuration files), which always use native line endings.
+line_ending:
+
+# The language backend to use for this project.
+# If not set, the global setting from serena_config.yml is used.
+# Valid values: LSP, JetBrains
+# Note: the backend is fixed at startup. If a project with a different backend
+# is activated post-init, an error will be returned.
+language_backend:
+
+# whether to use project's .gitignore files to ignore files
+ignore_all_files_in_gitignore: true
+
+# advanced configuration option allowing to configure language server-specific options.
+# Maps the language key to the options.
+# The settings are considered only if the project is trusted (see global configuration to define trusted projects).
+# See https://oraios.github.io/serena/02-usage/050_configuration.html#language-server-specific-settings
+ls_specific_settings: {}
+
+# list of additional paths to ignore in this project.
+# Same syntax as gitignore, so you can use * and **.
+# Important: quote patterns that start with `*`, otherwise YAML treats them as aliases.
+# Example:
+#   ignored_paths:
+#     - "examples/**"
+#     - ".worktrees/**"
+#     - "**/bin/**"
+#     - "**/obj/**"
+# Note: global ignored_paths from serena_config.yml are also applied additively.
+ignored_paths: []
+
+# whether the project is in read-only mode
+# If set to true, all editing tools will be disabled and attempts to use them will result in an error
+# Added on 2025-04-18
+read_only: false
+
+# list of tool names to exclude.
+# This extends the existing exclusions (e.g. from the global configuration)
+# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html
+excluded_tools: []
+
+# list of tools to include that would otherwise be disabled (particularly optional tools that are disabled by default).
+# This extends the existing inclusions (e.g. from the global configuration).
+# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html
+included_optional_tools: []
+
+# fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools.
+# This cannot be combined with non-empty excluded_tools or included_optional_tools.
+# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html
+fixed_tools: []
+
+# list of mode names that are to be activated by default, overriding the setting in the global configuration.
+# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes.
+# If the setting is undefined/empty, the default_modes from the global configuration (serena_config.yml) apply.
+# Otherwise, this overrides the setting from the global configuration (serena_config.yml).
+# Therefore, you can set this to [] if you do not want the default modes defined in the global config to apply
+# for this project.
+# This setting can, in turn, be overridden by CLI parameters (--mode).
+# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes
+default_modes:
+
+# list of mode names to be activated additionally for this project, e.g. ["query-projects"]
+# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes.
+# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes
+added_modes:
+
+# initial prompt for the project. It will always be given to the LLM upon activating the project
+# (contrary to the memories, which are loaded on demand).
+initial_prompt: ""
+
+# time budget (seconds) per tool call for the retrieval of additional symbol information
+# such as docstrings or parameter information.
+# This overrides the corresponding setting in the global configuration; see the documentation there.
+# If null or missing, use the setting from the global configuration.
+symbol_info_budget:
+
+# list of regex patterns which, when matched, mark a memory entry as read‑only.
+# Extends the list from the global configuration, merging the two lists.
+read_only_memory_patterns: []
+
+# list of regex patterns for memories to completely ignore.
+# Matching memories will not appear in list_memories or activate_project output
+# and cannot be accessed via read_memory or write_memory.
+# To access ignored memory files, use the read_file tool on the raw file path.
+# Extends the list from the global configuration, merging the two lists.
+# Example: ["_archive/.*", "_episodes/.*"]
+ignored_memory_patterns: []
+
+# list of additional workspace folder paths for cross-package reference support.
+# Paths can be absolute or relative to the project root.
+# Each folder is registered as an LSP workspace folder, enabling language servers to discover
+# symbols and references across package boundaries, but these folders are not indexed by Serena,
+# i.e. the respective symbols will not be found using Serena's symbol search tools.
+# Example:
+#   additional_workspace_folders:
+#     - ../sibling-package
+#     - ../shared-lib
+ls_additional_workspace_folders: []
+
+# list of language servers to start when using the LSP backend; choose from:
+#   ada                    al                     angular                ansible                bash
+#   bsl                    clojure                cpp                    cpp_ccls               crystal
+#   csharp                 csharp_omnisharp       cue                    dart                   deno
+#   elixir                 elm                    erlang                 fortran                fsharp
+#   gdscript               gleam                  go                     groovy                 haskell
+#   haxe                   hlsl                   html                   java                   json
+#   julia                  kotlin                 latex                  lean4                  lua
+#   luau                   markdown               matlab                 msl                    nextflow
+#   nix                    ocaml                  pascal                 perl                   php
+#   php_phpactor           php_phpantom           powershell             python                 python_basedpyright
+#   python_jedi            python_pyrefly         python_ty              qml                    r
+#   rego                   ruby                   ruby_solargraph        rust                   scala
+#   scss                   solidity               svelte                 swift                  systemverilog
+#   terraform              toml                   typescript             typescript_vts         vue
+#   wolfram                yaml                   zig
+#   (This list may be outdated; generated with scripts/print_language_list.py;
+#   For the current list, see values of the LanguageServerId enum here:
+#   https://github.com/oraios/serena/blob/main/src/solidlsp/ls_config.py)
+# For some languages, there are several alternative language servers, e.g. csharp_omnisharp, ruby_solargraph.)
+# Note:
+#   - For C, use cpp
+#   - For JavaScript, use typescript
+#   - For Angular projects, use angular (subsumes typescript+html; requires `npm install` in the project root)
+#   - For Svelte projects, use svelte (subsumes typescript/javascript for .svelte projects; requires npm)
+#   - For Deno projects, use deno (serves the same .ts/.js files as typescript; requires the deno CLI on PATH)
+#   - For SCSS / Sass / plain CSS, use scss (some-sass-language-server handles all three)
+#   - For Free Pascal/Lazarus, use pascal
+# Special requirements:
+#   Some language servers require additional setup/installations.
+#   See here for details: https://oraios.github.io/serena/01-about/020_programming-languages.html#language-servers
+# When using multiple language servers, the first language server that supports a given file will be used for that file.
+# The first language server is the default language and the respective language server will be used as a fallback.
+# Note that when using the JetBrains backend, language servers are not used and this list is correspondingly ignored.
+language_servers:
+- go
+
+# list of workspace folder paths (LSP backend only).
+# These folders will be used to build up Serena's symbol index.
+# Paths must be within the project root and should thus be relative to the project root.
+# Furthermore, the paths should not be filtered by ignore settings.
+# Default setting: The entire project root folder (".") is considered.
+# In (large) monorepos, this can be used to index only subfolders of the project root, e.g.
+#   ls_workspace_folders:
+#     - "./subproject1"
+#     - "./subproject2"
+ls_workspace_folders:
+- .
+
+# optional shell command to run before the language backend (LSP or JetBrains) is initialised.
+# the command runs in the project root directory and is only executed if the project is trusted
+# (see trusted_project_path_patterns in the global configuration).
+# serena waits for the command to exit: a non-zero exit code is logged as an error but does not
+# abort activation. a per-project timeout (activation_command_timeout, default 180s) is the safety
+# backstop for non-terminating commands; on expiry the process is killed and activation continues.
+# example: activation_command: "npx nx run-many -t build"
+activation_command:
+
+# maximum time in seconds to wait for activation_command to complete before killing it (default 180s).
+# must be a positive number.
+activation_command_timeout: 180.0

Makefile

@@ -11,6 +11,14 @@ NC     := \033[0m
 
 GOFLAGS=
 LDFLAGS=
+
+# The SIMD code (pkg/obikmer, simd package of Go >= 1.27) is only compiled with
+# GOEXPERIMENT=simd; without it, a portable scalar code is used instead.
+# The binaries select the vector width at run time (and fall back on an
+# emulation without SIMD hardware), so they run on any CPU.
+# Use `make GOEXPERIMENT= obitools` to build without SIMD.
+GOEXPERIMENT?=simd
+export GOEXPERIMENT
 GOCMD=go
 GOBUILD=$(GOCMD) build $(GOFLAGS) $(if $(LDFLAGS),-ldflags="$(LDFLAGS)")
 GOGENERATE=$(GOCMD) generate
@@ -36,13 +44,18 @@ OBITOOLS_DIRS:=$(sort $(patsubst %/,%,$(dir $(OBITOOLS_SRC))))
 OBITOOLS:=$(notdir $(OBITOOLS_DIRS))
 
 
+# Name of the target of a package: its path under pkg/, so that two packages of
+# the same name in different directories (pkg/obialign and
+# pkg/obitools/obialign) do not share a target.
+pkgname=$(subst /,-,$(patsubst pkg/%,%,$(1)))
+
 define MAKE_PKG_RULE
-pkg-$(notdir $(1)): $(1) pkg/obioptions/version.go
+pkg-$(call pkgname,$(1)): $(1) pkg/obioptions/version.go
-	@echo -n - Building package $(notdir $(1))...
+	@echo -n - Building package $(call pkgname,$(1))...
 	@$(GOBUILD) ./$(1) \
-	    2> pkg-$(notdir $(1)).log \
+	    2> pkg-$(call pkgname,$(1)).log \
-		|| cat pkg-$(notdir $(1)).log
+		|| cat pkg-$(call pkgname,$(1)).log
-	@rm -f pkg-$(notdir $(1)).log
+	@rm -f pkg-$(call pkgname,$(1)).log
 	@echo Done.
 endef
 
@@ -156,8 +169,8 @@ bump-version:
 
 jjnew:
 	@echo "$(YELLOW)→ Creating a new commit...$(NC)"
-	@echo "$(BLUE)→ Documenting current commit...$(NC)"
+	@echo "$(BLUE)→ Documenting undocumented commits...$(NC)"
-	@jj auto-describe
+	@jj auto-doc
 	@echo "$(BLUE)→ Done.$(NC)"
 	@jj new
 	@echo "$(GREEN)✓ New commit created$(NC)"
@@ -171,8 +184,8 @@ jjpush:
 	@echo "$(GREEN)✓ Release complete$(NC)"
 
 jjpush-describe:
-	@echo "$(BLUE)→ Documenting current commit...$(NC)"
+	@echo "$(BLUE)→ Documenting undocumented commits...$(NC)"
-	@jj auto-describe
+	@jj auto-doc
 
 jjpush-bump:
 	@echo "$(BLUE)→ Creating new commit for version bump...$(NC)"

OBIPRIMERS_MEMORY_PROMPT.md

@@ -0,0 +1,149 @@
+# Prompt : optimisation de l'usage mémoire de `obiprimers`
+
+Tu travailles dans le dépôt Go `/Users/coissac/Sync/travail/__MOI__/GO/obitools4` (module
+`git.metabarcoding.org/obitools/obitools4/obitools4`, Go : `/usr/local/go/bin/go`, non présent dans le PATH).
+Ta mission : **réduire l'usage mémoire de la commande `obiprimers`** sans changer ses résultats, en procédant
+par la mesure.
+
+## 1. Contexte
+
+`obiprimers` est la ré-implémentation OBITools4 d'ecoPrimers (design de paires d'amorces à partir de génomes
+entiers, avec évaluation de la qualité du marqueur délimité). Le mode « voie A » est terminé et validé.
+Avant toute chose, **lis `OBIPRIMERS_NOTES.md`** (racine du dépôt) : la section « État courant », puis les
+sections 11 à 13 (appariement approché), 16 et 16 bis (taxonomie embarquée ; **une erreur de mesure à ne pas
+refaire**), 25 (définitions de Bc, Bs, Bd et élagage de la phase 2).
+
+Code concerné :
+- `cmd/obitools/obiprimers/main.go`, `pkg/obitools/obiprimers/` (options, lecture, sortie CSV) ;
+- `pkg/obiprimer/` : `corpus.go` (`LoadCorpus`), `taxa.go`, `words.go` (`FindStrictWords`), `match.go` et
+  `match_seed.go` (`MatchWords`), `pairs.go` (`selectPrimers`, `countPairs`), `pairs_eval.go` (`FindPairs`),
+  `amplicons.go` (`collectAmplicons`), `resolution.go`, `search.go` (`Search`) ;
+- `pkg/obikmer/seqquorum*.go` (`SeqKmerCounter`, comptage par partitions sur disque) ;
+- `pkg/obithermo/` (Tm), code partagé : `pkg/obiformats`, `pkg/obiiter`, `pkg/obiseq`, `pkg/obitax`.
+
+## 2. Décisions de l'utilisateur à respecter (non négociables)
+
+- **Tout est chargé en mémoire** : toutes les séquences de la base (cibles, contre-exemples, non-cibles) sont
+  gardées, parce que la lecture sur l'**entrée standard** doit continuer à marcher comme pour tout obitool.
+  Tu peux **compacter** ce qui est gardé ; tu ne peux pas passer à un schéma qui exige de relire le fichier.
+- **Un seul calcul par indice** : ne touche pas aux définitions de Bc, Bs et Bd (`resolution.go`). Pas de
+  colonne ou d'option alternative.
+- **Les résultats ne changent pas.** Critère : la sortie CSV doit rester **identique octet pour octet** sur les
+  cas de référence du §4. Si tu penses qu'un changement de résultat est justifié, **demande avant**.
+- Aucune commande git qui modifie quoi que ce soit (pas de commit, add, mv, rm, reset…). Lecture seule
+  (`status`, `diff`, `log`) autorisée. Déplace les fichiers avec `mv`/`Write`/`Edit`.
+- Fichiers source de moins d'environ 500 lignes. Pas de `.collect()` d'une grande structure quand un flux
+  suffit. Réutilise l'existant avant d'écrire du code ; enrichis une brique partagée plutôt que d'écrire du
+  code ad hoc, **mais** toute modification d'un paquet partagé (obiformats, obiiter, obiseq, obitax) doit
+  être signalée clairement et validée par les tests de ces paquets.
+- **Pas de spéculation présentée comme un fait** : ce qui n'est pas mesuré est une hypothèse, dis-le.
+  Une régression de justesse est pire qu'un gain de mémoire : avant d'échanger l'une contre l'autre, demande.
+
+## 3. Ce qui est mesuré aujourd'hui (macOS, M3 Max, 16 cœurs)
+
+Jeu : 7 822 génomes mitochondriaux de vertébrés (131 Mb de bases), `-r 7953` (Cyprinidae : 325 cibles, 7 497
+non-cibles). Commande :
+`obiprimers -r 7953 -e 3 --three-prime 2 -l 30 -L 150 -c <fichier annoté> -o out.csv` → 3 247 paires
+(3 248 lignes avec l'en-tête).
+
+| Mesure | Valeur |
+|---|---|
+| pic RSS, taxonomie embarquée, fichier | **1 217 à 1 372 Mo** (bruit d'environ ±100 Mo d'une exécution à l'autre), 4,7 à 5,3 s |
+| pic RSS, taxonomie embarquée, entrée standard | médiane **1 334 à 1 346 Mo** sur 8 exécutions entrelacées |
+| tas des 7 822 séquences chargées | **193,7 Mo** (dont `taxonomic_path` : 19,2 Mo, soit 2,46 Ko par séquence) |
+| même exécution **sans** l'élagage de phase 2 | 5 604 Mo (c'est ce qu'on a corrigé, voir §25 des notes) |
+| avec `-t` (taxonomie NCBI complète) | +1,07 Go pour le seul chargeur (mesuré avec `obitaxonomy`), pic ~2,8 Go (mesuré avant les nouvelles définitions) |
+| `GOGC=25` (ancienne mesure, `-t` NCBI) | pic 2,82 Go → 1,98 Go, mais temps CPU utilisateur ×2 |
+| ecoPrimers (C) sur les mêmes jeux | 1,86 Go (Cyprinidae), 2,11 Go (Teleostei) |
+
+Tailles d'enregistrements : `ampRecord` 24 octets, `Match` 12, `seqMatch` 12, `hit` (intermédiaire de
+`matchCorpus`) 16. Environ 6,85 millions d'occurrences approchées quand toutes les séquences sont chargées
+(cas `-i 1`).
+
+**Le constat qui motive la mission** : les séquences gardées pèsent ~194 Mo, le pic du processus est ~1,3 Go.
+Les 1,1 Go restants ne sont **pas expliqués** (hypothèses ci-dessous, aucune n'est vérifiée).
+
+## 4. Garde-fous à mettre en place AVANT de modifier quoi que ce soit
+
+1. **Binaire de référence** : `cd /Users/coissac/Sync/travail/__MOI__/GO/obitools4 &&
+   GOEXPERIMENT=simd /usr/local/go/bin/go build -o /tmp/obiprimers.baseline ./cmd/obitools/obiprimers`
+2. **Données** (les fichiers de `/tmp` peuvent avoir disparu, regénère-les) : dossier du tutoriel
+   `D="$HOME/Documents/Enseignement/DNA metabarcoding spring school/ecoPrimers-Fish"` : `mito.vert.fasta.gz`,
+   `ncbitaxo_20250211.tgz`, et pour le test d'échelle `mito.all.fasta.gz` / `mito.one.fasta.gz`. Fichier annoté
+   (taxonomie embarquée, évite le chargeur NCBI) :
+   `/Users/coissac/travail/__MOI__/GO/obitools4/build/obiannotate -t "$D/ncbitaxo_20250211.tgz"
+   --taxonomic-path -Z -o /tmp/mito.vert.path.fasta.gz "$D/mito.vert.fasta.gz"`
+   (les binaires déjà compilés sont dans `/Users/coissac/travail/__MOI__/GO/obitools4/build/`, sans `Sync`).
+3. **CSV de référence**, à produire avec le binaire de référence et à conserver : Cyprinidae `-r 7953`,
+   Teleostei `-r 32443`, un cas avec contre-exemples `-r 7953 -i 1`, un cas `-U`, un cas `-R NC_083213`, et la
+   lecture sur entrée standard (`gzcat fichier | obiprimers ...`). Après chaque modification : `cmp` octet pour
+   octet.
+4. **Tests** : `go test ./pkg/obiprimer/ ./pkg/obitools/obiprimers/ ./pkg/obithermo/ ./pkg/obikmer/ -race
+   -count=1`, et les tests opt-in sur les données réelles (durent ~75 s hors `-race`) :
+   `OBIPRIMERS_TUTORIAL="$D" go test ./pkg/obiprimer/ -count=1`.
+5. **Méthode de mesure** (le piège principal) : le pic RSS bouge de ±100 Mo entre deux exécutions identiques
+   (rythme du ramasse-miettes). Mesure **au moins 5 exécutions entrelacées avant/après et compare les
+   médianes** (`/usr/bin/time -l` : octets sur macOS). Pour attribuer la mémoire, ne te fie pas au RSS : utilise
+   des profils de tas (`--pprof` de la CLI sert `/debug/pprof/heap` sur `localhost:6060`, `net/http/pprof` est
+   importé ; ou `runtime/pprof.WriteHeapProfile` aux frontières d'étapes ; `go tool pprof -sample_index=inuse_space`
+   et `alloc_space` ; `GODEBUG=gctrace=1`). **Erreur déjà commise (voir la section 16 bis des notes)** : en
+   mesurant le tas libéré par la suppression d'un attribut, la tranche de séquences n'était plus référencée
+   après la boucle et le GC a libéré *tout* ; utilise `runtime.KeepAlive` sur ce que tu veux garder vivant.
+
+## 5. Pistes à évaluer (toutes des HYPOTHÈSES à confirmer par un profil, classées sans ordre)
+
+- **Retard du ramasse-miettes** (`GOGC=100` laisse le tas monter jusqu'à ~2× le vivant). Pistes : limite
+  mémoire douce (`debug.SetMemoryLimit`) ou `debug.SetGCPercent` ajustés par la commande après la lecture ;
+  mesurer le coût CPU (l'ancienne mesure avec `GOGC=25` doublait le temps utilisateur).
+- **Lecture** (code partagé `obiformats`/`obiiter`) : 131 Mb sont analysés, avec une table d'attributs et des
+  chaînes JSON par séquence. L'option `--batch-mem` (lots limités en mémoire) existe déjà : l'évaluer.
+- **Séquences gardées** : `BioSequence` garde 1 octet par base et une table d'attributs. Ne garder que
+  l'identifiant, la séquence et le `taxid` (retirer les autres attributs, comme déjà fait pour
+  `taxonomic_path`, dans `dropTaxonomicPath` de `pkg/obitools/obiprimers/obiprimers.go`) ; ou stocker la
+  séquence à 2 bits par base (~33 Mo pour 131 Mb) avec une liste d'exceptions pour les bases ambiguës. Ce
+  second point touche `forEachOccurrence`, `ampHash`, `windowAsIs`, `windowRevComp` et `MatchWords`, qui lisent
+  les octets : à ne tenter que si le profil le justifie.
+- **Occurrences stockées deux fois** : `[]WordMatches` (`matchCorpus`, intermédiaire `hit` de 16 octets par
+  occurrence puis recopie) et `primerSet.perSeq` (`selectPrimers`). Vérifier à quel moment chacune est
+  libérée, construire directement les structures finales, présizer les tranches (les `append` en doublement
+  créent un pic transitoire).
+- **Phase 2b** (`collectAmplicons` sur les seules paires survivantes) : `ampRecord` de 24 octets (16 possibles
+  en empaquetant le rôle), enregistrements par worker ; **traitement des paires survivantes par lots bornés**
+  si le nombre d'enregistrements estimé dépasse un budget (idée notée en 25.2). Cas dur : `-T 0` (aucune paire
+  élaguée), où les tests atteignent ~5,5 Go de tas.
+- **`SeqKmerCounter`** (`pkg/obikmer/seqquorum*.go`) : environ 16 octets par occurrence de k-mer pour les
+  partitions en cours de traitement ; le nombre de partitions est **fixé à 256**, alors que le total de bases
+  est connu une fois le corpus chargé : le choisir automatiquement.
+- **Lecture en deux passes** pour la taxonomie embarquée (`readSequencesAndTaxonomy`) : la première passe ne
+  garde que les chemins, mais analyse tout.
+- **Chargeur de taxonomie NCBI** : 1,07 Go à lui seul ; c'est du code partagé (`obitax`), hors périmètre sauf
+  si tu y trouves une économie sûre ; sinon, documente-le dans l'aide (la taxonomie embarquée évite ce coût).
+
+## 6. Test d'échelle (jamais fait)
+
+Tout est validé sur des génomes mitochondriaux (5 à 131 Mb). Mesure la croissance sur un fichier plus gros
+(`mito.all.fasta.gz`, 172 Mo compressés, avec `--restrict-to-taxon` sur un grand clade, p. ex. les Metazoa) :
+temps et pic RSS en fonction de la taille, pour savoir ce qui grossit vite (corpus, occurrences, amplicons,
+compteur de k-mers).
+
+## 7. Livrables
+
+1. Un **rapport de profil** : où va la mémoire à chaque étape (lecture, sélection, mots stricts, appariement,
+   paires, phase 2a/2b), avec les profils de tas et les chiffres, avant toute modification.
+2. Les optimisations **classées par gain mesuré et par risque**, puis implémentées une par une, chacune avec
+   mesure avant/après (médianes d'au moins 5 exécutions entrelacées), `cmp` des CSV de référence, tests verts
+   sous `-race`.
+3. Une **nouvelle section** à la fin de `OBIPRIMERS_NOTES.md` (ne réécris pas l'historique) : mesures,
+   décisions, pistes écartées et pourquoi, ce qui reste. Pas de commit.
+4. Si une optimisation exige de changer un résultat, une décision ou un paquet partagé : **arrête-toi et
+   demande**.
+
+## 8. Pièges connus de l'environnement
+
+- zsh ne découpe pas une variable en arguments : utilise des tableaux (`A=(-r 7953 ...)` puis `"${A[@]}"`)
+  ou des scripts `bash`.
+- macOS : `sed -i ''` (sans suffixe) ; `/usr/bin/time -l` donne le pic en **octets** (`maximum resident set size`).
+- `OBIPRIMERS_WORKERS` n'est lue que par les tests opt-in ; la CLI utilise `--max-cpu`.
+- Les avertissements C de `obiformats/fastseq_read.h` sont connus et sans gravité.
+- `go test -race` sur les tests opt-in est lent (~90 s) ; réserve-le aux tests unitaires.
+- Le test `obiutils/TestSetString` était instable : il est corrigé (tri des membres dans `Set.String`).

Release-notes.md

@@ -2,6 +2,23 @@
 
 ## New changes
 
+### New features
+
+- New `obioligotag` command, a re-implementation of the `oligotag` command of
+  OBITools 2. It builds a family of oligonucleotides (tags) of at least
+  `--family-size` members that are pairwise at a distance of at least
+  `--distance`. Candidates are either generated (`--oligo-size`, up to 12) or
+  read from a file (`--oligo-list`, sequence file or one oligonucleotide per
+  line). They can be restricted by alphabet (`--alphabet`, for instance `ACT`
+  to forbid G), GC content, homopolymers and IUPAC patterns. The distance is
+  either the Hamming distance or the LCS-mode alignment distance
+  (`--distance-type`).
+
+- New functions in `obikmer` to compare and enumerate k-mers encoded as
+  `uint64`: `HammingDistance`, `KmerLCS` and `KmerLCSBatch` (SIMD when built with
+  `GOEXPERIMENT=simd`), `IterConstrainedKmers`, and in `obigraph` a clique search
+  (`FindClique`) that does not need the adjacency matrix.
+
 ### Bug fixes
 
 - In `obipairing` correct the misspelling of the `obiparing_*` tags where the `i`

cmd/obitools/obialign/main.go

@@ -0,0 +1,26 @@
+package main
+
+import (
+	"os"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obialign"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+)
+
+func main() {
+	optionParser := obioptions.GenerateOptionParser(
+		"obialign",
+		"pairwise similarity of nucleotide sequences, as a CSV table (the principle of sumatra)",
+		obialign.OptionSet)
+
+	_, args := optionParser(os.Args)
+
+	fs, err := obiconvert.CLIReadBioSequences(args...)
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
+
+	obialign.CLIWriteTable(obialign.CLIOBIAlign(fs))
+
+	obiutils.WaitForLastPipe()
+}

cmd/obitools/obiclust/main.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"os"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiclust"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+)
+
+func main() {
+	optionParser := obioptions.GenerateOptionParser(
+		"obiclust",
+		"star clustering of sequences (the algorithm of sumaclust, UCLUST and CD-HIT)",
+		obiclust.OptionSet)
+
+	_, args := optionParser(os.Args)
+
+	fs, err := obiconvert.CLIReadBioSequences(args...)
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
+
+	clustered := obiclust.CLIOBIClust(fs)
+
+	obiconvert.CLIWriteBioSequences(clustered, true)
+
+	obiutils.WaitForLastPipe()
+}

cmd/obitools/obioligotag/main.go

@@ -0,0 +1,33 @@
+package main
+
+import (
+	"os"
+
+	log "github.com/sirupsen/logrus"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obioligotag"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+)
+
+func main() {
+	optionParser := obioptions.GenerateOptionParser(
+		"obioligotag",
+		"designs a family of oligonucleotides (tags) at a minimum distance from each other",
+		obioligotag.OptionSet)
+
+	_, args := optionParser(os.Args)
+	if len(args) > 0 {
+		log.Fatalf("unexpected arguments %v: use --oligo-list to provide candidate oligonucleotides", args)
+	}
+
+	oligos, err := obioligotag.CLIOligoFamily()
+	if err != nil {
+		log.Fatalf("%v", err)
+	}
+
+	obiconvert.CLIWriteBioSequences(obioligotag.CLIOligosToIterator(oligos), true)
+
+	obiutils.WaitForLastPipe()
+}

cmd/obitools/obiprimers/main.go

@@ -0,0 +1,38 @@
+package main
+
+import (
+	"os"
+
+	log "github.com/sirupsen/logrus"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obicsv"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiprimers"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+)
+
+func main() {
+	obiprimers.SetReadingDefaults()
+
+	optionParser := obioptions.GenerateOptionParser(
+		"obiprimers",
+		"designs the primer pairs amplifying a set of example sequences and evaluates the DNA barcode they delimit. "+
+			"The taxonomy comes from the -t option, or is read from the taxonomic paths embedded in the sequences (obiannotate --taxonomic-path)",
+		obiprimers.OptionSet)
+
+	_, args := optionParser(os.Args)
+
+	sequences, taxonomy, err := obiprimers.CLIReadSequencesAndTaxonomy(args)
+	if err != nil {
+		log.Fatalf("Cannot read the sequences: %v", err)
+	}
+
+	pairs, err := obiprimers.CLIPrimerPairs(sequences, taxonomy)
+	if err != nil {
+		log.Fatalf("%v", err)
+	}
+
+	obicsv.CLICSVWriter(obiprimers.PairsToCSVIterator(pairs, obiprimers.CLIReference() != ""), true)
+
+	obiutils.WaitForLastPipe()
+}

doc/book/installation.qmd

@@ -8,7 +8,7 @@ All the sources of the [*OBITools4*](http://metabarcoding.org/obitools4) can be
 
 ## Prerequisites
 
-The *OBITools4* are developped using the [GO programming language](https://go.dev/), we stick to the latest version of the language, today the $1.21.4$. If you want to download and compile the sources yourself, you first need to install the corresponding compiler on your system. Some parts of the soft are also written in C, therefore a recent C compiler is also requested, GCC on Linux or Windows, the Developer Tools on Mac.
+The *OBITools4* are developped using the [GO programming language](https://go.dev/), we stick to the latest version of the language, today the $1.27$. If you want to download and compile the sources yourself, you first need to install the corresponding compiler on your system. Some computations (distances between k-mers) use the experimental `simd` package of Go: the `Makefile` compiles them with `GOEXPERIMENT=simd`, and the installation script uses a Go 1.27 compiler for that reason. The resulting programs run on any processor, a portable code being used when no SIMD instruction is available. Some parts of the soft are also written in C, therefore a recent C compiler is also requested, GCC on Linux or Windows, the Developer Tools on Mac.
 
 Whatever the installation you decide for, you will have to ensure that a C compiler is available on your system.
 

go.mod

@@ -1,6 +1,6 @@
 module git.metabarcoding.org/obitools/obitools4/obitools4
 
-go 1.26.1
+go 1.27
 
 require (
 	github.com/DavidGamba/go-getoptions v0.33.0

go.work

@@ -1,3 +1,3 @@
-go 1.26.1
+go 1.27
 
 use .

install_obitools.sh

@@ -2,6 +2,11 @@
 
 # Default values
 URL="https://go.dev/dl/"
+# Series of the Go compiler used to build OBITools4. The SIMD code relies on the
+# experimental simd package of Go (GOEXPERIMENT=simd), whose API may change
+# between Go versions: the build is therefore tied to the series the code was
+# written for rather than to the latest release.
+GO_SERIES="1.27"
 GITHUB_REPO="https://github.com/metabarcoding/obitools4"
 INSTALL_DIR="/usr/local"
 OBITOOLS_PREFIX=""
@@ -174,10 +179,16 @@ GOFILE=$(curl -s "$URL" \
             | grep 'class="download"' \
             | grep "\.tar\.gz" \
             | sed -E 's@^.*/dl/(go[1-9].+\.tar\.gz)".*$@\1@' \
+            | awk -v prefix="go${GO_SERIES}." 'index($0, prefix) == 1' \
             | grep -i "$OS" \
             | grep -i "$ARCH" \
             | head -1)
 
+if [ -z "$GOFILE" ]; then
+  echo "Error: no Go ${GO_SERIES} archive found for ${OS}/${ARCH} at ${URL}" 1>&2
+  exit 1
+fi
+
 GOURL=$(curl -s "${URL}${GOFILE}" \
         | sed -E 's@^.*href="(.*\.tar\.gz)".*$@\1@')
 
@@ -191,8 +202,12 @@ export PATH
 export GOPATH="$(pwd)/gopath"
 export GOCACHE="$(pwd)/cache"
 export GOTOOLCHAIN=local
+# OBITools4 is built with the SIMD code enabled (it falls back on a portable
+# code at run time if the processor has no SIMD instructions).
+export GOEXPERIMENT=simd
 
 echo "GOROOT=$GOROOT" 1>&2
+echo "GOEXPERIMENT=$GOEXPERIMENT" 1>&2
 echo "GOCACHE=$GOCACHE" 1>&2
 mkdir -p "$GOPATH" "$GOCACHE"
 

obitests/obitools/obialign/test.sh

@@ -0,0 +1,206 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obialign
+CMD=obialign
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1
+then
+    log "$MCMD: printing help OK"
+    ((success++))
+else
+    log "$MCMD: printing help failed"
+    ((failed++))
+fi
+
+
+#
+# Dataset used by the following tests: two families of similar sequences, a
+# sequence identical to another one, and a sequence with an ambiguous
+# nucleotide that is discarded.
+#
+cat > "${TMPDIR}/data.fasta" <<DATA
+>a1 count=20;
+acgtacgtacgtaacctagcctagcctagggatcgatcgatcgatcgaatcg
+>a2 count=3;
+acgtacgtacgtaacctagcctagcctagggatcgatcgatcgttcgaatcg
+>a3 count=2;
+acgtacgtacgtaacctagcctagcctagggatcgatcgatcgatcgaatcg
+>b1 count=15;
+ttgcatgcatgctagctagctagcatgcatcgatcgatgctagctagtagct
+>b2 count=1;
+ttgcatgcatgctagctagctagcatgcatcgatcgatgctagctagtaact
+>x1 count=4;
+ttgcatgcatgctangctagctagcatgcatcgatcgatgctagctagtagc
+DATA
+
+cat > "${TMPDIR}/ref.fasta" <<DATA
+>r1
+acgtacgtacgtaacctagcctagcctagggatcgatcgatcgatcgaatcg
+>r2
+ttgcatgcatgctagctagctagcatgcatcgatcgatgctagctagtagct
+>r3
+ggggccccggggccccggggccccggggccccggggccccggggcccc
+DATA
+
+((ntest++))
+if $CMD --max-cpu 2 "${TMPDIR}/data.fasta" > "${TMPDIR}/all.csv" 2> /dev/null &&
+   [ "$(head -n 1 "${TMPDIR}/all.csv")" = "id1,id2,score" ] &&
+   [ "$(tail -n +2 "${TMPDIR}/all.csv" | wc -l | tr -d ' ')" = "10" ]
+then
+    log "$MCMD: without threshold all the pairs of the 5 usable sequences are written OK"
+    ((success++))
+else
+    log "$MCMD: the table without threshold is wrong"
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD --max-cpu 2 -t 0.95 "${TMPDIR}/data.fasta" > "${TMPDIR}/t95.csv" 2> /dev/null &&
+   [ "$(tail -n +2 "${TMPDIR}/t95.csv" | cut -d, -f1,2 | tr '\n' ' ')" = "a1,a2 a1,a3 a2,a3 b1,b2 " ] &&
+   grep -q '^a1,a3,1$' "${TMPDIR}/t95.csv"
+then
+    log "$MCMD: only the pairs reaching the threshold are written, in order OK"
+    ((success++))
+else
+    log "$MCMD: the table with a threshold is wrong"
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD --max-cpu 2 -t 0.95 -x "${TMPDIR}/data.fasta" 2> /dev/null | head -n 2 | tail -n 1 | grep -q '^a1,a2,[0-9.]*,20,3,52,52$'
+then
+    log "$MCMD: extra columns OK"
+    ((success++))
+else
+    log "$MCMD: extra columns failed"
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD --max-cpu 2 -g -t 0.9 "${TMPDIR}/data.fasta" 2> /dev/null | grep -q '^b1,x1,'
+then
+    log "$MCMD: a sequence with an IUPAC letter is kept with --accept-ambiguous OK"
+    ((success++))
+else
+    log "$MCMD: --accept-ambiguous failed"
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD --max-cpu 2 -R "${TMPDIR}/ref.fasta" -t 0.95 "${TMPDIR}/data.fasta" 2> /dev/null | tail -n +2 | cut -d, -f1,2 | tr '\n' ' ' > "${TMPDIR}/two.txt" &&
+   [ "$(cat "${TMPDIR}/two.txt")" = "a1,r1 a2,r1 a3,r1 b1,r2 b2,r2 " ]
+then
+    log "$MCMD: comparison with a reference file OK"
+    ((success++))
+else
+    log "$MCMD: comparison with a reference file failed"
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD --max-cpu 1 "${TMPDIR}/data.fasta" 2> /dev/null > "${TMPDIR}/cpu1.csv" &&
+   $CMD --max-cpu 8 "${TMPDIR}/data.fasta" 2> /dev/null > "${TMPDIR}/cpu8.csv" &&
+   cmp -s "${TMPDIR}/cpu1.csv" "${TMPDIR}/cpu8.csv"
+then
+    log "$MCMD: the table does not depend on the number of threads OK"
+    ((success++))
+else
+    log "$MCMD: the table depends on the number of threads"
+    ((failed++))
+fi
+
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obiclust/test.sh

@@ -0,0 +1,188 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obiclust
+CMD=obiclust
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1
+then
+    log "$MCMD: printing help OK"
+    ((success++))
+else
+    log "$MCMD: printing help failed"
+    ((failed++))
+fi
+
+
+#
+# Dataset used by the following tests: two families with a rare variant each,
+# a duplicate, and a sequence with an ambiguous nucleotide that is discarded.
+#
+cat > "${TMPDIR}/data.fasta" <<DATA
+>a1 count=20;
+acgtacgtacgtaacctagcctagcctagggatcgatcgatcgatcgaatcg
+>a2 count=3;
+acgtacgtacgtaacctagcctagcctagggatcgatcgatcgttcgaatcg
+>a3 count=2;
+acgtacgtacgtaacctagcctagcctagggatcgatcgatcgatcgaatcg
+>b1 count=15;
+ttgcatgcatgctagctagctagcatgcatcgatcgatgctagctagtagct
+>b2 count=1;
+ttgcatgcatgctagctagctagcatgcatcgatcgatgctagctagtaact
+>x1 count=4;
+ttgcatgcatgctangctagctagcatgcatcgatcgatgctagctagtagc
+DATA
+
+((ntest++))
+if $CMD --max-cpu 2 -O -t 0.95 "${TMPDIR}/data.fasta" > "${TMPDIR}/clusters.fasta" 2> "${TMPDIR}/clusters.log"
+then
+    log "$MCMD: clustering runs OK"
+    ((success++))
+else
+    log "$MCMD: clustering failed"
+    ((failed++))
+fi
+
+((ntest++))
+if [ "$(grep -c '^>' "${TMPDIR}/clusters.fasta")" = "5" ]
+then
+    log "$MCMD: the sequence with an ambiguous nucleotide is discarded OK"
+    ((success++))
+else
+    log "$MCMD: wrong number of sequences in the output"
+    ((failed++))
+fi
+
+((ntest++))
+if [ "$(grep '^>a[23] ' "${TMPDIR}/clusters.fasta" | grep -c 'cluster=a1;')" = "2" ] &&
+   [ "$(grep '^>b2 ' "${TMPDIR}/clusters.fasta" | grep -c 'cluster=b1;')" = "1" ] &&
+   grep '^>b1 ' "${TMPDIR}/clusters.fasta" | grep -q 'cluster_center=true;' &&
+   grep '^>a1 ' "${TMPDIR}/clusters.fasta" | grep -q 'cluster_weight=25;'
+then
+    log "$MCMD: sequences are attached to the right center OK"
+    ((success++))
+else
+    log "$MCMD: sequences are not attached to the right center"
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD --max-cpu 2 -t 0.95 --otu-map "${TMPDIR}/otu.csv" --no-sequence-output "${TMPDIR}/data.fasta" > "${TMPDIR}/none.fasta" 2> /dev/null &&
+   [ ! -s "${TMPDIR}/none.fasta" ] &&
+   [ "$(head -n 1 "${TMPDIR}/otu.csv")" = "id,cluster,cluster_center,cluster_score,count" ] &&
+   [ "$(sed -n 2,4p "${TMPDIR}/otu.csv" | cut -d, -f1,2 | tr '\n' ' ')" = "a1,a1 a3,a1 a2,a1 " ] &&
+   [ "$(grep -c ',true,' "${TMPDIR}/otu.csv")" = "2" ]
+then
+    log "$MCMD: CSV table of the clusters OK"
+    ((success++))
+else
+    log "$MCMD: CSV table of the clusters failed"
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD --max-cpu 1 -O -t 0.95 "${TMPDIR}/data.fasta" 2> /dev/null | grep '^>' | sed 's/ .*//' > "${TMPDIR}/order1.txt" &&
+   $CMD --max-cpu 8 -O -t 0.95 "${TMPDIR}/data.fasta" 2> /dev/null | grep '^>' | sed 's/ .*//' > "${TMPDIR}/order8.txt" &&
+   cmp -s "${TMPDIR}/order1.txt" "${TMPDIR}/order8.txt"
+then
+    log "$MCMD: the result does not depend on the number of threads OK"
+    ((success++))
+else
+    log "$MCMD: the result depends on the number of threads"
+    ((failed++))
+fi
+
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obioligotag/test.sh

@@ -0,0 +1,205 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obioligotag
+CMD=obioligotag
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+#
+# Every result is checked on its properties (size, alphabet, constraints),
+# not against a reference file, because several equivalent families exist.
+#
+
+printf 'acgtac\nacgtat\nttttgg\nggcaac\ncaccat\nacgtac\n' > "${TMPDIR}/oligos.txt"
+printf '>a\nACGTAC\n>b\nACGTAT\n>c\nTTTTGG\n>d\nGGCAAC\n>e\nCACCAT\n' > "${TMPDIR}/oligos.fasta"
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1 
+then
+    log "$MCMD: printing help OK" 
+    ((success++))
+else
+    log "$MCMD: printing help failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD -s 5 -f 24 -d 3 -g 3 -p 2 > "${TMPDIR}/tags.fasta" 2> /dev/null \
+   && [ "$(grep -vc '>' "${TMPDIR}/tags.fasta")" -ge 24 ] \
+   && ! grep -v '>' "${TMPDIR}/tags.fasta" | grep -qE '(a{3}|c{3}|g{3}|t{3})'
+then
+    log "$MCMD: generating a family with homopolymer constraint OK" 
+    ((success++))
+else
+    log "$MCMD: generating a family with homopolymer constraint failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD -s 7 -f 60 -d 3 --alphabet ACT > "${TMPDIR}/noG.fasta" 2> /dev/null \
+   && [ "$(grep -vc '>' "${TMPDIR}/noG.fasta")" -ge 60 ] \
+   && ! grep -v '>' "${TMPDIR}/noG.fasta" | grep -q 'g'
+then
+    log "$MCMD: restricted alphabet OK" 
+    ((success++))
+else
+    log "$MCMD: restricted alphabet failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD -s 6 -f 20 -d 3 --distance-type lcs > "${TMPDIR}/lcs.fasta" 2> /dev/null \
+   && [ "$(grep -vc '>' "${TMPDIR}/lcs.fasta")" -ge 20 ]
+then
+    log "$MCMD: LCS distance OK" 
+    ((success++))
+else
+    log "$MCMD: LCS distance failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD -s 7 -f 40 -d 3 -r cnnnnnn > "${TMPDIR}/rejected.fasta" 2> /dev/null \
+   && [ "$(grep -vc '>' "${TMPDIR}/rejected.fasta")" -ge 40 ] \
+   && ! grep -v '>' "${TMPDIR}/rejected.fasta" | grep -q '^c'
+then
+    log "$MCMD: rejected pattern OK" 
+    ((success++))
+else
+    log "$MCMD: rejected pattern failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD -L "${TMPDIR}/oligos.txt" -f 3 -d 3 > "${TMPDIR}/from_text.fasta" 2> /dev/null \
+   && [ "$(grep -vc '>' "${TMPDIR}/from_text.fasta")" -ge 3 ]
+then
+    log "$MCMD: selecting oligonucleotides from a text file OK" 
+    ((success++))
+else
+    log "$MCMD: selecting oligonucleotides from a text file failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD -L "${TMPDIR}/oligos.fasta" -f 3 -d 3 > "${TMPDIR}/from_fasta.fasta" 2> /dev/null \
+   && [ "$(grep -vc '>' "${TMPDIR}/from_fasta.fasta")" -ge 3 ]
+then
+    log "$MCMD: selecting oligonucleotides from a fasta file OK" 
+    ((success++))
+else
+    log "$MCMD: selecting oligonucleotides from a fasta file failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if ! $CMD -s 5 -f 100 -d 3 > /dev/null 2>&1
+then
+    log "$MCMD: family size beyond the Hamming bound refused OK" 
+    ((success++))
+else
+    log "$MCMD: family size beyond the Hamming bound not refused" 
+    ((failed++))
+fi
+
+((ntest++))
+if ! $CMD -s 5 -L "${TMPDIR}/oligos.txt" -f 3 > /dev/null 2>&1
+then
+    log "$MCMD: --oligo-size with --oligo-list refused OK" 
+    ((success++))
+else
+    log "$MCMD: --oligo-size with --oligo-list not refused" 
+    ((failed++))
+fi
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

pkg/obialign/fastlcs_bench_test.go

@@ -0,0 +1,111 @@
+package obialign
+
+import (
+	"fmt"
+	"math/rand"
+	"testing"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+// randomDNA returns a random ACGT sequence of length n.
+func randomDNA(rng *rand.Rand, n int) []byte {
+	const nuc = "ACGT"
+	s := make([]byte, n)
+	for i := range s {
+		s[i] = nuc[rng.Intn(4)]
+	}
+	return s
+}
+
+// mutate returns a copy of s in which each position is, with probability
+// rate, substituted, deleted or followed by an inserted base (equiprobable).
+func mutate(rng *rand.Rand, s []byte, rate float64) []byte {
+	const nuc = "ACGT"
+	out := make([]byte, 0, len(s)+len(s)/10)
+	for _, c := range s {
+		if rng.Float64() >= rate {
+			out = append(out, c)
+			continue
+		}
+		switch rng.Intn(3) {
+		case 0:
+			out = append(out, nuc[rng.Intn(4)])
+		case 1: // deletion
+		case 2:
+			out = append(out, c, nuc[rng.Intn(4)])
+		}
+	}
+	return out
+}
+
+// benchPairs builds a fixed set of pairs. "similar" pairs differ by about
+// 2% of mutated positions, "random" pairs are unrelated sequences.
+func benchPairs(n, length int, similar bool) [][2]*obiseq.BioSequence {
+	rng := rand.New(rand.NewSource(42))
+	pairs := make([][2]*obiseq.BioSequence, n)
+	for i := range pairs {
+		a := randomDNA(rng, length)
+		var b []byte
+		if similar {
+			b = mutate(rng, a, 0.02)
+		} else {
+			b = randomDNA(rng, length)
+		}
+		pairs[i] = [2]*obiseq.BioSequence{
+			obiseq.NewBioSequence("a", a, ""),
+			obiseq.NewBioSequence("b", b, ""),
+		}
+	}
+	return pairs
+}
+
+// BenchmarkLegacyLCS measures the historical implementation (the one
+// FastLCSScore used before BandedLCS), kept as the "before" reference.
+func BenchmarkLegacyLCS(b *testing.B) {
+	for _, similar := range []bool{true, false} {
+		kind := "random"
+		if similar {
+			kind = "similar"
+		}
+		for _, length := range []int{100, 500, 2000} {
+			pairs := benchPairs(64, length, similar)
+			for _, maxErr := range []int{-1, 5, length / 20, length / 5} {
+				name := fmt.Sprintf("%s/len=%d/maxErr=%d", kind, length, maxErr)
+				b.Run(name, func(b *testing.B) {
+					var buffer []uint64
+					b.ReportAllocs()
+					b.ResetTimer()
+					for i := 0; i < b.N; i++ {
+						p := pairs[i%len(pairs)]
+						FastLCSEGFScoreByte(p[0].Sequence(), p[1].Sequence(), maxErr, false, &buffer)
+					}
+				})
+			}
+		}
+	}
+}
+
+// BenchmarkBandedLCS runs on the same pairs as BenchmarkLegacyLCS.
+func BenchmarkBandedLCS(b *testing.B) {
+	for _, similar := range []bool{true, false} {
+		kind := "random"
+		if similar {
+			kind = "similar"
+		}
+		for _, length := range []int{100, 500, 2000} {
+			pairs := benchPairs(64, length, similar)
+			for _, maxErr := range []int{-1, 5, length / 20, length / 5} {
+				name := fmt.Sprintf("%s/len=%d/maxErr=%d", kind, length, maxErr)
+				b.Run(name, func(b *testing.B) {
+					b.ReportAllocs()
+					b.ResetTimer()
+					for i := 0; i < b.N; i++ {
+						p := pairs[i%len(pairs)]
+						BandedLCS(p[0].Sequence(), p[1].Sequence(), maxErr)
+					}
+				})
+			}
+		}
+	}
+}

pkg/obialign/fastlcsegf.go

@@ -2,6 +2,7 @@ package obialign
 
 import (
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+	log "github.com/sirupsen/logrus"
 )
 
 // FastLCSEGFScoreByte calculates the score of the Longest Common Subsequence (LCS) between two byte slices.
@@ -355,27 +356,35 @@ func FastLCSEGFScore(seqA, seqB *obiseq.BioSequence, maxError int, buffer *[]uin
 	return FastLCSEGFScoreByte(seqA.Sequence(), seqB.Sequence(), maxError, true, buffer)
 }
 
-// FastLCSScore calculates the score of the longest common subsequence between two bio sequences.
+// FastLCSScore calculates the score of the longest common subsequence between two bio sequences,
+// and the length of the shortest alignment achieving it (the distance is alilen - lcs).
 //
-// if maxError > 0, the maximum allowed error between the sequences is maxError.
+// If maxError >= 0, the result is exact when its distance is at most maxError,
-// Otherwise, no error checking is done.
+// and (-1, -1) is returned otherwise. If maxError < 0, no limit is applied.
-// If the actual number of errors is larger than maxError, -1 is returned for both values.
+// See BandedLCS, which does the work.
 //
 // The score matrix is:
-//   - Matching: 1
+//   - Matching: 1 (bases are compared with the IUPAC rules)
 //   - Mismatch or gap: 0
 //
+// The program stops with a fatal error if a sequence is longer than
+// MaxBandedLCSLength.
+//
 // Parameters:
 // - seqA: The first bio sequence.
 // - seqB: The second bio sequence.
 // - maxError: The maximum allowed error between the sequences. If set to -1, no limit is applied.
-// - buffer: A pointer to a uint64 slice to store intermediate results. If nil, a new slice is created.
+// - buffer: Unused, kept for compatibility; the working memory is pooled.
 //
 // Returns:
 // - The score of the longest common subsequence.
 // - The length of the shortest alignment corresponding to the LCS.
 func FastLCSScore(seqA, seqB *obiseq.BioSequence, maxError int, buffer *[]uint64) (int, int) {
-	score, alilen, _ := FastLCSEGFScoreByte(seqA.Sequence(), seqB.Sequence(), maxError, false, buffer)
+	score, alilen, err := BandedLCS(seqA.Sequence(), seqB.Sequence(), maxError)
+	if err != nil {
+		log.Fatalf("FastLCSScore on sequences %s (%d bp) and %s (%d bp): %v (maximum %d bp)",
+			seqA.Id(), seqA.Len(), seqB.Id(), seqB.Len(), err, MaxBandedLCSLength)
+	}
 
 	return score, alilen
 }

pkg/obialign/lcsband.go

@@ -0,0 +1,185 @@
+package obialign
+
+import (
+	"errors"
+	"sync"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+// MaxBandedLCSLength is the longest sequence BandedLCS accepts. The score and
+// the alignment length are packed on 16 bits each, and the alignment length
+// can reach the sum of both sequence lengths.
+const MaxBandedLCSLength = 1<<15 - 1
+
+// ErrSequenceTooLong is returned by BandedLCS when a sequence is longer than
+// MaxBandedLCSLength.
+var ErrSequenceTooLong = errors.New("obialign: sequence too long for BandedLCS")
+
+// A DP cell is packed on 32 bits as
+//
+//	score<<16 | (lcsLenBase - alignment length)
+//
+// so that the maximum of two cells ranks them by score first and, for equal
+// scores, by shortest alignment. The three moves then reduce to
+//
+//	cell(i,j) = max(diag + match<<16, up, left) - 1
+//
+// since every move lengthens the alignment by one and only a diagonal match
+// adds one to the score. The value 0 stands for an unavailable cell (outside
+// the band): it loses against any real cell. Every cell of the band has at
+// least one real neighbour (the band is connected and contains (0,0)), so
+// the subtraction never wraps for a cell that is used; the scalar reference
+// still clamps the maximum to 1 as a safety net.
+const (
+	lcsScoreUnit = uint32(1) << 16
+	lcsLenBase   = uint32(0xFFFF)
+)
+
+// bandedLCSWorkspace holds the scratch memory of one alignment.
+type bandedLCSWorkspace struct {
+	aRev, bCode []uint32    // encoded bases (aRev is reversed), SIMD kernel only
+	rows        [3][]uint32 // antidiagonal (SIMD) or row (scalar) buffers
+	aRev16      []uint16    // 16-bit encoded bases of the score-only kernel
+	bCode16     []uint16
+	rows16      [3][]uint16
+	widened     int // number of alignments that needed a second pass
+}
+
+var bandedLCSPool = sync.Pool{New: func() any { return new(bandedLCSWorkspace) }}
+
+func growUint32(s []uint32, n int) []uint32 {
+	if cap(s) < n {
+		return make([]uint32, n)
+	}
+	return s[:n]
+}
+
+// iupacMask gives for each byte its 4-bit nucleotide set (A=1, C=2, G=4,
+// T=8), or 0 when the byte is not an IUPAC letter. For two bytes of non-zero
+// mask, SameIUPACNuc(a, b) is exactly (mask[a] & mask[b]) != 0.
+var iupacMask = buildIUPACMask()
+
+func buildIUPACMask() (m [256]uint8) {
+	probes := [4]byte{'a', 'c', 'g', 't'}
+	for c := 0; c < 256; c++ {
+		if !(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z') {
+			continue
+		}
+		for bit, p := range probes {
+			if obiseq.SameIUPACNuc(byte(c), p) {
+				m[c] |= 1 << uint(bit)
+			}
+		}
+	}
+	return m
+}
+
+// allRegularIUPAC reports whether every base has a non-zero IUPAC mask, in
+// which case base compatibility is a plain AND of masks.
+func allRegularIUPAC(s []byte) bool {
+	for _, c := range s {
+		if iupacMask[c] == 0 {
+			return false
+		}
+	}
+	return true
+}
+
+// bandOffsets returns the diagonal offsets k = j - i covered by a band of
+// half-width g around the diagonals 0..delta, with sequence A (columns j) not
+// shorter than sequence B (rows i), clipped to the matrix.
+func bandOffsets(lA, lB, g int) (kmin, kmax int) {
+	return max(-g, -lB), min(lA-lB+g, lA)
+}
+
+// BandedLCS computes the score of the longest common subsequence of two
+// nucleotide sequences, and the length of the shortest alignment achieving
+// it (match: +1, mismatch and gap: 0; bases are compared with the IUPAC
+// rules of obiseq.SameIUPACNuc). The distance is alilen - lcs.
+//
+// If maxError >= 0, the result is the optimal (lcs, alilen) when its distance
+// is at most maxError, and (-1, -1) otherwise. This is exact, not a
+// heuristic: the DP is restricted to a band, and a bound on what a path
+// leaving the band can score proves when the band was wide enough. maxError < 0
+// means no limit.
+//
+// An error is returned when a sequence is longer than MaxBandedLCSLength.
+func BandedLCS(seqA, seqB []byte, maxError int) (lcs, alilen int, err error) {
+	if len(seqA) > MaxBandedLCSLength || len(seqB) > MaxBandedLCSLength {
+		return -1, -1, ErrSequenceTooLong
+	}
+	if len(seqA) < len(seqB) {
+		seqA, seqB = seqB, seqA
+	}
+	ws := bandedLCSPool.Get().(*bandedLCSWorkspace)
+	lcs, alilen = bandedLCS(seqA, seqB, maxError, ws, haveSIMD)
+	bandedLCSPool.Put(ws)
+	return lcs, alilen, nil
+}
+
+// bandedLCS expects len(a) >= len(b), both within MaxBandedLCSLength.
+//
+// With delta = lA - lB and a band of half-width g (offsets -g .. delta+g):
+//   - an alignment with at most E errors needs at least delta+2h gaps to reach
+//     the offset delta+h or -h, so it lies in the band of g0 = (E-delta)/2;
+//   - a path leaving a band of half-width g makes at least g+1 gaps on the B
+//     side, hence scores at most lB-g-1 matches, and has more than E errors
+//     as soon as g >= g0.
+//
+// Let Q be the best path of the band g0. If its distance exceeds E, the
+// global optimum cannot be within E either (it would be in the band and be
+// Q): the answer is -1. Otherwise, if score(Q) >= lB-g0-1 no outside path can
+// beat Q (a tie in score loses on length, the outside path having more errors
+// than Q): Q is the global optimum. Else the band is widened once to
+// g1 = min(E-delta, lB-score(Q)-1); since score(Q) >= lA-E for any alignment
+// within E errors, g1 always certifies the result.
+func bandedLCS(a, b []byte, maxError int, ws *bandedLCSWorkspace, useSIMD bool) (int, int) {
+	lA, lB := len(a), len(b)
+	delta := lA - lB
+	if maxError >= 0 && maxError < delta {
+		return -1, -1
+	}
+	if lB == 0 {
+		return 0, lA
+	}
+	simdOK := useSIMD && allRegularIUPAC(a) && allRegularIUPAC(b)
+	if simdOK {
+		encodeBases(ws, a, b)
+	}
+	run := func(g int) (int, int) {
+		kmin, kmax := bandOffsets(lA, lB, g)
+		return ws.align(a, b, kmin, kmax, simdOK)
+	}
+
+	if maxError < 0 {
+		return run(lA)
+	}
+	g0 := (maxError - delta) / 2
+	lcs, alilen := run(g0)
+	if alilen-lcs > maxError {
+		return -1, -1
+	}
+	if lcs >= lB-g0-1 {
+		return lcs, alilen
+	}
+	ws.widened++
+	lcs, alilen = run(min(maxError-delta, lB-lcs-1))
+	if alilen-lcs > maxError {
+		return -1, -1
+	}
+	return lcs, alilen
+}
+
+// align runs the banded DP on the offsets [kmin, kmax] (kmin <= 0 <= kmax) and
+// returns the best (lcs, alilen) of the band. The bases must have been encoded
+// with encodeBases when simdOK is set.
+func (ws *bandedLCSWorkspace) align(a, b []byte, kmin, kmax int, simdOK bool) (int, int) {
+	var key uint32
+	if simdOK {
+		key = bandedLCSSIMD(len(a), len(b), kmin, kmax, ws)
+	} else {
+		key = bandedLCSScalar(a, b, kmin, kmax, ws)
+	}
+	return int(key >> 16), int(lcsLenBase - key&0xFFFF)
+}

pkg/obialign/lcsband_nosimd.go

@@ -0,0 +1,17 @@
+//go:build !goexperiment.simd
+
+package obialign
+
+const haveSIMD = false
+
+// encodeBases and bandedLCSSIMD are never called without GOEXPERIMENT=simd
+// (haveSIMD is false); they exist to keep the dispatcher uniform.
+func encodeBases(ws *bandedLCSWorkspace, a, b []byte) {}
+
+func bandedLCSSIMD(lA, lB, kmin, kmax int, ws *bandedLCSWorkspace) uint32 {
+	return 0
+}
+
+func lcsScoreSIMD(a, b []byte, kmin, kmax, minLCS int, ws *bandedLCSWorkspace) int {
+	return -1
+}

pkg/obialign/lcsband_scalar.go

@@ -0,0 +1,53 @@
+package obialign
+
+import "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+
+// bandedLCSScalar is the portable reference of the banded DP, row by row. It
+// returns the packed cell (lB, lA). len(a) >= len(b) >= 1.
+//
+// Cells outside the band hold 0 (unavailable). Row i spans the columns
+// [i+kmin, i+kmax] clipped to [1, lA]; the cell just left of that span and
+// the one just right of it are reset so that the next row never reads a stale
+// value.
+func bandedLCSScalar(a, b []byte, kmin, kmax int, ws *bandedLCSWorkspace) uint32 {
+	lA, lB := len(a), len(b)
+
+	prev := growUint32(ws.rows[0], lA+2)
+	cur := growUint32(ws.rows[1], lA+2)
+	ws.rows[0], ws.rows[1] = prev, cur
+
+	for j := 0; j <= lA; j++ {
+		prev[j] = 0
+		if j <= kmax {
+			prev[j] = lcsLenBase - uint32(j)
+		}
+	}
+	prev[lA+1] = 0
+
+	for i := 1; i <= lB; i++ {
+		if -i >= kmin {
+			cur[0] = lcsLenBase - uint32(i)
+		} else {
+			cur[0] = 0
+		}
+		jlo := max(1, i+kmin)
+		jhi := min(lA, i+kmax)
+		if jlo > 1 {
+			cur[jlo-1] = 0
+		}
+		bi := b[i-1]
+		for j := jlo; j <= jhi; j++ {
+			d := prev[j-1]
+			if obiseq.SameIUPACNuc(a[j-1], bi) {
+				d += lcsScoreUnit
+			}
+			m := max(d, prev[j], cur[j-1], 1)
+			cur[j] = m - 1
+		}
+		if jhi < lA {
+			cur[jhi+1] = 0
+		}
+		prev, cur = cur, prev
+	}
+	return prev[lA]
+}

pkg/obialign/lcsband_simd.go

@@ -0,0 +1,107 @@
+//go:build goexperiment.simd
+
+package obialign
+
+import "simd"
+
+const haveSIMD = true
+
+func simdLanes() int {
+	var probe simd.Uint32s
+	return probe.Len()
+}
+
+// encodeIUPAC writes in dst the 4-bit IUPAC mask of each base of s, reversed
+// if rev is set, followed by zero padding up to len(dst).
+func encodeIUPAC(dst []uint32, s []byte, rev bool) {
+	n := len(s)
+	for x := range dst {
+		dst[x] = 0
+	}
+	if rev {
+		for x, c := range s {
+			dst[n-1-x] = uint32(iupacMask[c])
+		}
+		return
+	}
+	for x, c := range s {
+		dst[x] = uint32(iupacMask[c])
+	}
+}
+
+// encodeBases encodes both sequences once for all the passes of an alignment.
+func encodeBases(ws *bandedLCSWorkspace, a, b []byte) {
+	vl := simdLanes()
+	ws.aRev = growUint32(ws.aRev, len(a)+vl)
+	ws.bCode = growUint32(ws.bCode, len(b)+vl)
+	encodeIUPAC(ws.aRev, a, true)
+	encodeIUPAC(ws.bCode, b, false)
+}
+
+// bandedLCSSIMD runs the banded DP along antidiagonals t = i + j. Lane i of
+// the buffer of antidiagonal t holds the cell (i, t-i), so that
+//
+//	diag = buffer[t-2][i-1]   up = buffer[t-1][i-1]   left = buffer[t-1][i]
+//
+// are plain unaligned loads. Along an antidiagonal the offset j-i = t-2i
+// must stay in [kmin, kmax], which gives the window [lo, hi] of rows. The
+// vector loop may run past hi; the lanes beyond are then reset to 0
+// (unavailable), as is the lane just before the window, because the next
+// antidiagonals read them as neighbours. Lane 0 and lane t carry the first
+// row and the first column of the matrix.
+//
+// Requires encodeBases to have been called, lA >= lB >= 1 and kmin <= 0 <=
+// kmax.
+func bandedLCSSIMD(lA, lB, kmin, kmax int, ws *bandedLCSWorkspace) uint32 {
+	vl := simdLanes()
+	aRev, bCode := ws.aRev, ws.bCode
+
+	size := lB + 2*vl + 2
+	for r := range ws.rows {
+		ws.rows[r] = growUint32(ws.rows[r], size)
+		clear(ws.rows[r])
+	}
+	p2, p1, cur := ws.rows[0], ws.rows[1], ws.rows[2]
+
+	p2[0] = lcsLenBase // cell (0,0)
+	if kmax >= 1 {
+		p1[0] = lcsLenBase - 1 // cell (0,1)
+	}
+	if kmin <= -1 {
+		p1[1] = lcsLenBase - 1 // cell (1,0)
+	}
+
+	zero := simd.BroadcastUint32s(0)
+	one := simd.BroadcastUint32s(1)
+	unit := simd.BroadcastUint32s(lcsScoreUnit)
+
+	for t := 2; t <= lA+lB; t++ {
+		lo := max(1, t-lA, (t-kmax+1)>>1)
+		hi := min(lB, t-1, (t-kmin)>>1)
+
+		for i := lo; i <= hi; i += vl {
+			ca := simd.LoadUint32s(aRev[lA-t+i:])
+			cb := simd.LoadUint32s(bCode[i-1:])
+			inc := unit.Masked(ca.And(cb).NotEqual(zero))
+			d := simd.LoadUint32s(p2[i-1:]).Add(inc)
+			u := simd.LoadUint32s(p1[i-1:])
+			l := simd.LoadUint32s(p1[i:])
+			d.Max(u).Max(l).Sub(one).Store(cur[i:])
+		}
+
+		zero.Store(cur[hi+1:])
+		if lo > 1 {
+			cur[lo-1] = 0
+		}
+		cur[0] = 0
+		if t <= lA && t <= kmax {
+			cur[0] = lcsLenBase - uint32(t)
+		}
+		if t <= lB && -t >= kmin {
+			cur[t] = lcsLenBase - uint32(t)
+		}
+
+		p2, p1, cur = p1, cur, p2
+	}
+	return p1[lB]
+}

pkg/obialign/lcsband_test.go

@@ -0,0 +1,360 @@
+package obialign
+
+import (
+	"math/rand"
+	"testing"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+// bandDP is the oracle: a plain DP over the whole matrix, written
+// independently of the production code, in which only the cells with a
+// diagonal offset j-i in [kmin, kmax] exist (kmin=-lB, kmax=lA when
+// unbounded). It returns the optimal LCS score and the shortest alignment
+// length achieving it among the alignments staying in the band.
+func bandDP(a, b []byte, kmin, kmax int) (int, int) {
+	lA, lB := len(a), len(b)
+	type cell struct {
+		s, l int
+		ok   bool
+	}
+	better := func(x, y cell) bool {
+		return !y.ok || (x.ok && (x.s > y.s || (x.s == y.s && x.l < y.l)))
+	}
+	m := make([][]cell, lB+1)
+	for i := range m {
+		m[i] = make([]cell, lA+1)
+	}
+	for i := 0; i <= lB; i++ {
+		for j := 0; j <= lA; j++ {
+			if k := j - i; k < kmin || k > kmax {
+				continue
+			}
+			if i == 0 || j == 0 {
+				m[i][j] = cell{0, i + j, true}
+				continue
+			}
+			var best cell
+			if d := m[i-1][j-1]; d.ok {
+				best = cell{d.s, d.l + 1, true}
+				if obiseq.SameIUPACNuc(a[j-1], b[i-1]) {
+					best.s++
+				}
+			}
+			if u := m[i-1][j]; u.ok && better(cell{u.s, u.l + 1, true}, best) {
+				best = cell{u.s, u.l + 1, true}
+			}
+			if l := m[i][j-1]; l.ok && better(cell{l.s, l.l + 1, true}, best) {
+				best = cell{l.s, l.l + 1, true}
+			}
+			m[i][j] = best
+		}
+	}
+	return m[lB][lA].s, m[lB][lA].l
+}
+
+// fullDP is the unbanded optimum.
+func fullDP(a, b []byte) (int, int) { return bandDP(a, b, -len(b), len(a)) }
+
+// expected is the contract of BandedLCS: the unbounded optimum when it has at
+// most maxError errors, (-1,-1) otherwise.
+func expected(a, b []byte, maxError int) (int, int) {
+	s, l := fullDP(a, b)
+	if maxError >= 0 && l-s > maxError {
+		return -1, -1
+	}
+	return s, l
+}
+
+func randomFrom(rng *rand.Rand, alphabet string, n int) []byte {
+	s := make([]byte, n)
+	for i := range s {
+		s[i] = alphabet[rng.Intn(len(alphabet))]
+	}
+	return s
+}
+
+func mutateFrom(rng *rand.Rand, alphabet string, s []byte, rate float64) []byte {
+	out := make([]byte, 0, len(s)+8)
+	for _, c := range s {
+		if rng.Float64() >= rate {
+			out = append(out, c)
+			continue
+		}
+		switch rng.Intn(3) {
+		case 0:
+			out = append(out, alphabet[rng.Intn(len(alphabet))])
+		case 1:
+		case 2:
+			out = append(out, c, alphabet[rng.Intn(len(alphabet))])
+		}
+	}
+	return out
+}
+
+var testAlphabets = []struct {
+	name, letters string
+	regular       bool // only IUPAC letters of non-zero mask
+}{
+	{"acgt", "ACGT", true},
+	{"low-complexity", "AC", true},
+	{"mixed-case", "ACGTacgt", true},
+	{"iupac", "ACGTRYSWKMBDHVNUacgtrywsn", true},
+	{"exotic", "ACGT-.EXz*", false},
+}
+
+// TestBandedLCSDifferential checks the scalar reference against the oracle,
+// and the dispatcher (SIMD kernel when built with GOEXPERIMENT=simd) against
+// the scalar reference, for random pairs of unequal lengths and many bands.
+func TestBandedLCSDifferential(t *testing.T) {
+	rng := rand.New(rand.NewSource(7))
+	ws := new(bandedLCSWorkspace)
+	for _, alpha := range testAlphabets {
+		for it := 0; it < 3000; it++ {
+			lA := 1 + rng.Intn(90)
+			a := randomFrom(rng, alpha.letters, lA)
+			var b []byte
+			if rng.Intn(3) > 0 {
+				b = mutateFrom(rng, alpha.letters, a, 0.02+rng.Float64()*0.5)
+			} else {
+				b = randomFrom(rng, alpha.letters, 1+rng.Intn(90))
+			}
+			if len(b) == 0 {
+				continue
+			}
+			if len(a) < len(b) {
+				a, b = b, a
+			}
+			delta := len(a) - len(b)
+			for _, maxError := range []int{-1, 0, 1, 2, 3, 5, 8, 13, 21, 40, 200, delta, delta + 1, delta + 2, delta + 3} {
+				ws, wl := expected(a, b, maxError)
+				ss, sl := bandedLCS(a, b, maxError, new(bandedLCSWorkspace), false)
+				if ss != ws || sl != wl {
+					t.Fatalf("%s scalar maxError=%d a=%q b=%q: got (%d,%d), oracle (%d,%d)",
+						alpha.name, maxError, a, b, ss, sl, ws, wl)
+				}
+				ds, dl := bandedLCS(a, b, maxError, new(bandedLCSWorkspace), haveSIMD)
+				if ds != ss || dl != sl {
+					t.Fatalf("%s dispatch(simd=%v) maxError=%d a=%q b=%q: got (%d,%d), scalar (%d,%d)",
+						alpha.name, haveSIMD, maxError, a, b, ds, dl, ss, sl)
+				}
+			}
+		}
+	}
+	_ = ws
+}
+
+// TestBandedLCSLongPairs covers lengths well above the vector width, with
+// reused workspaces (stale buffers must never leak between calls).
+func TestBandedLCSLongPairs(t *testing.T) {
+	rng := rand.New(rand.NewSource(11))
+	ws := new(bandedLCSWorkspace)
+	for it := 0; it < 60; it++ {
+		a := randomFrom(rng, "ACGT", 150+rng.Intn(500))
+		b := mutateFrom(rng, "ACGT", a, 0.01+rng.Float64()*0.2)
+		if len(a) < len(b) {
+			a, b = b, a
+		}
+		for _, maxError := range []int{-1, 4, 20, 60, 150} {
+			wantS, wantL := expected(a, b, maxError)
+			gotS, gotL := bandedLCS(a, b, maxError, ws, haveSIMD)
+			if gotS != wantS || gotL != wantL {
+				t.Fatalf("maxError=%d lens=%d/%d: got (%d,%d), oracle (%d,%d)",
+					maxError, len(a), len(b), gotS, gotL, wantS, wantL)
+			}
+		}
+	}
+}
+
+// TestBandedLCSMatchesLegacy compares with the historical implementation
+// (FastLCSEGFScoreByte, not end-gap-free) wherever that one is exact: no
+// limit, or a result within the limit.
+func TestBandedLCSMatchesLegacy(t *testing.T) {
+	rng := rand.New(rand.NewSource(3))
+	for it := 0; it < 4000; it++ {
+		a := randomFrom(rng, "ACGTRYN", 1+rng.Intn(120))
+		b := mutateFrom(rng, "ACGTRYN", a, rng.Float64()*0.4)
+		if len(b) == 0 {
+			continue
+		}
+		for _, maxError := range []int{-1, 2, 7, 25} {
+			ls, ll, err := BandedLCS(a, b, maxError)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if ls < 0 {
+				continue
+			}
+			os, ol, _ := FastLCSEGFScoreByte(a, b, maxError, false, nil)
+			if os != ls || ol != ll {
+				t.Fatalf("maxError=%d a=%q b=%q: BandedLCS (%d,%d), legacy (%d,%d)",
+					maxError, a, b, ls, ll, os, ol)
+			}
+		}
+	}
+}
+
+func TestBandedLCSEdgeCases(t *testing.T) {
+	cases := []struct {
+		a, b     string
+		maxError int
+		s, l     int
+	}{
+		{"", "", -1, 0, 0},
+		{"ACGT", "", -1, 0, 4},
+		{"ACGT", "", 3, -1, -1},
+		{"ACGT", "", 4, 0, 4},
+		{"ACGT", "ACGT", 0, 4, 4},
+		{"ACGT", "ACGA", 0, -1, -1},
+		{"ACGT", "ACGA", 1, 3, 4},
+		{"ACGTT", "ACGT", 0, -1, -1}, // length difference above the limit
+		{"ACGTT", "ACGT", 1, 4, 5},
+	}
+	for _, c := range cases {
+		s, l, err := BandedLCS([]byte(c.a), []byte(c.b), c.maxError)
+		if err != nil || s != c.s || l != c.l {
+			t.Errorf("%q/%q maxError=%d: got (%d,%d,%v), want (%d,%d)", c.a, c.b, c.maxError, s, l, err, c.s, c.l)
+		}
+		// the arguments are symmetric
+		s, l, _ = BandedLCS([]byte(c.b), []byte(c.a), c.maxError)
+		if s != c.s || l != c.l {
+			t.Errorf("swapped %q/%q maxError=%d: got (%d,%d), want (%d,%d)", c.b, c.a, c.maxError, s, l, c.s, c.l)
+		}
+	}
+}
+
+func TestBandedLCSTooLong(t *testing.T) {
+	long := make([]byte, MaxBandedLCSLength+1)
+	for i := range long {
+		long[i] = 'A'
+	}
+	if _, _, err := BandedLCS(long, []byte("ACGT"), -1); err != ErrSequenceTooLong {
+		t.Errorf("A too long: err = %v", err)
+	}
+	if _, _, err := BandedLCS([]byte("ACGT"), long, 5); err != ErrSequenceTooLong {
+		t.Errorf("B too long: err = %v", err)
+	}
+	// the maximum length itself is accepted
+	ok := long[:MaxBandedLCSLength]
+	if s, l, err := BandedLCS(ok, ok, 0); err != nil || s != len(ok) || l != len(ok) {
+		t.Errorf("max length: got (%d,%d,%v)", s, l, err)
+	}
+}
+
+// TestBandedLCSSecondPass checks the exactness over many bands, also on
+// low-complexity sequences where the best LCS is not the closest alignment,
+// and reports how often the band had to be widened (never more than once).
+func TestBandedLCSSecondPass(t *testing.T) {
+	rng := rand.New(rand.NewSource(5))
+	for _, alpha := range testAlphabets[:4] {
+		for _, rate := range []float64{0.02, 0.1, 0.3} {
+			ws := new(bandedLCSWorkspace)
+			var total int
+			for it := 0; it < 1500; it++ {
+				a := randomFrom(rng, alpha.letters, 20+rng.Intn(120))
+				b := mutateFrom(rng, alpha.letters, a, rate)
+				if len(b) == 0 {
+					continue
+				}
+				if len(a) < len(b) {
+					a, b = b, a
+				}
+				for _, maxError := range []int{1, 3, 6, 10, 20, 40} {
+					want, wantL := expected(a, b, maxError)
+					got, gotL := bandedLCS(a, b, maxError, ws, haveSIMD)
+					if got != want || gotL != wantL {
+						t.Fatalf("%s maxError=%d a=%q b=%q: got (%d,%d), want (%d,%d)",
+							alpha.name, maxError, a, b, got, gotL, want, wantL)
+					}
+					total++
+				}
+			}
+			t.Logf("%-15s mutation %.2f: %d bounded calls, %d widened (%.1f%%)",
+				alpha.name, rate, total, ws.widened, 100*float64(ws.widened)/float64(total))
+		}
+	}
+}
+
+// TestFastLCSScoreWrapper checks the BioSequence entry point.
+func TestFastLCSScoreWrapper(t *testing.T) {
+	sa := obiseq.NewBioSequence("a", []byte("ACGTACGTAC"), "")
+	sb := obiseq.NewBioSequence("b", []byte("ACGTTCGTAC"), "")
+	if s, l := FastLCSScore(sa, sb, -1, nil); s != 9 || l != 10 {
+		t.Errorf("got (%d,%d), want (9,10)", s, l)
+	}
+	if s, l := FastLCSScore(sa, sb, 0, nil); s != -1 || l != -1 {
+		t.Errorf("got (%d,%d), want (-1,-1)", s, l)
+	}
+}
+
+// TestBandedLCSAtLeast checks the exactness of the bound expressed on the
+// score: the optimum when it reaches minLCS, (-1,-1) otherwise.
+func TestBandedLCSAtLeast(t *testing.T) {
+	rng := rand.New(rand.NewSource(13))
+	var buf BandedLCSBuffer
+	for _, alpha := range testAlphabets {
+		for it := 0; it < 1500; it++ {
+			a := randomFrom(rng, alpha.letters, 1+rng.Intn(100))
+			b := mutateFrom(rng, alpha.letters, a, rng.Float64()*0.5)
+			if len(b) == 0 {
+				continue
+			}
+			os, ol := fullDP(a, b)
+			for _, minLCS := range []int{-3, 0, 1, os - 1, os, os + 1, len(b) / 2, len(b), len(b) + 1} {
+				ws, wl := os, ol
+				if os < minLCS {
+					ws, wl = -1, -1
+				}
+				for _, got := range [][2]int{func() [2]int { s, l, _ := buf.AtLeast(a, b, minLCS); return [2]int{s, l} }(),
+					func() [2]int { s, l, _ := BandedLCSAtLeast(b, a, minLCS); return [2]int{s, l} }()} {
+					if got[0] != ws || got[1] != wl {
+						t.Fatalf("%s minLCS=%d a=%q b=%q: got (%d,%d), want (%d,%d)",
+							alpha.name, minLCS, a, b, got[0], got[1], ws, wl)
+					}
+				}
+			}
+		}
+	}
+	if _, _, err := buf.AtLeast(make([]byte, MaxBandedLCSLength+1), []byte("a"), 0); err != ErrSequenceTooLong {
+		t.Errorf("too long: err = %v", err)
+	}
+}
+
+// TestBandedLCSScoreAtLeast checks the score-only entry point, with sequences
+// long enough for the early stop of the kernel to be exercised.
+func TestBandedLCSScoreAtLeast(t *testing.T) {
+	rng := rand.New(rand.NewSource(17))
+	var buf BandedLCSBuffer
+	check := func(name string, a, b []byte) {
+		os, _ := fullDP(a, b)
+		for _, minLCS := range []int{-1, 0, os - 3, os - 1, os, os + 1, os + 5, len(b) - 1, len(b), (len(b) * 9) / 10, len(b) + 1} {
+			want := os
+			if os < minLCS {
+				want = -1
+			}
+			got, err := buf.ScoreAtLeast(a, b, minLCS)
+			if err != nil || got != want {
+				t.Fatalf("%s minLCS=%d a=%q b=%q: got %d (%v), want %d", name, minLCS, a, b, got, err, want)
+			}
+		}
+	}
+	for _, alpha := range testAlphabets {
+		for it := 0; it < 800; it++ {
+			a := randomFrom(rng, alpha.letters, 1+rng.Intn(60))
+			b := mutateFrom(rng, alpha.letters, a, rng.Float64()*0.6)
+			if len(b) > 0 {
+				check(alpha.name, a, b)
+			}
+		}
+	}
+	for it := 0; it < 150; it++ {
+		a := randomFrom(rng, "ACGT", 100+rng.Intn(400))
+		b := mutateFrom(rng, "ACGT", a, rng.Float64()*0.3)
+		if it%3 == 0 {
+			b = randomFrom(rng, "ACGT", 50+rng.Intn(300))
+		}
+		if len(b) > 0 {
+			check("long", a, b)
+		}
+	}
+}

pkg/obialign/lcsmin.go

@@ -0,0 +1,103 @@
+package obialign
+
+// BandedLCSBuffer is a working memory that can be reused from one alignment to
+// the next to avoid any allocation. It must not be shared between goroutines.
+type BandedLCSBuffer struct {
+	ws bandedLCSWorkspace
+}
+
+// AtLeast is BandedLCSAtLeast on the buffer.
+func (b *BandedLCSBuffer) AtLeast(seqA, seqB []byte, minLCS int) (lcs, alilen int, err error) {
+	return atLeast(seqA, seqB, minLCS, &b.ws)
+}
+
+func atLeast(seqA, seqB []byte, minLCS int, ws *bandedLCSWorkspace) (lcs, alilen int, err error) {
+	if len(seqA) > MaxBandedLCSLength || len(seqB) > MaxBandedLCSLength {
+		return -1, -1, ErrSequenceTooLong
+	}
+	if len(seqA) < len(seqB) {
+		seqA, seqB = seqB, seqA
+	}
+	lcs, alilen = bandedLCSAtLeast(seqA, seqB, minLCS, ws, haveSIMD)
+	return lcs, alilen, nil
+}
+
+// BandedLCSAtLeast computes the LCS score of two nucleotide sequences and the
+// length of the shortest alignment achieving it, as BandedLCS does, but with
+// the bound expressed on the score: the result is the exact optimum when its
+// score is at least minLCS, and (-1, -1) otherwise.
+//
+// The DP only visits the diagonals an alignment of score minLCS can use (one
+// whose path makes at most lB-minLCS gaps on one side and lA-minLCS on the
+// other), so the cost decreases as minLCS grows. This is exact: every alignment
+// of score >= minLCS lies in that band, hence so does the optimum when it
+// reaches minLCS, and when the best path of the band is below minLCS so is
+// the global optimum.
+func BandedLCSAtLeast(seqA, seqB []byte, minLCS int) (lcs, alilen int, err error) {
+	ws := bandedLCSPool.Get().(*bandedLCSWorkspace)
+	lcs, alilen, err = atLeast(seqA, seqB, minLCS, ws)
+	bandedLCSPool.Put(ws)
+	return lcs, alilen, err
+}
+
+// bandedLCSAtLeast expects len(a) >= len(b), both within MaxBandedLCSLength.
+func bandedLCSAtLeast(a, b []byte, minLCS int, ws *bandedLCSWorkspace, useSIMD bool) (int, int) {
+	lA, lB := len(a), len(b)
+	minLCS = max(minLCS, 0)
+	if minLCS > lB {
+		return -1, -1
+	}
+	if lB == 0 {
+		return 0, lA
+	}
+	simdOK := useSIMD && allRegularIUPAC(a) && allRegularIUPAC(b)
+	kmin, kmax := -(lB - minLCS), lA-minLCS
+
+	if !simdOK {
+		lcs, alilen := ws.align(a, b, kmin, kmax, false)
+		if lcs < minLCS {
+			return -1, -1
+		}
+		return lcs, alilen
+	}
+
+	// The score alone is much cheaper than the score with the length of the
+	// alignment: the latter is only computed when the pair reaches minLCS. With
+	// no minimum there is nothing to reject, and the pass would be wasted.
+	if minLCS > 0 && lcsScoreSIMD(a, b, kmin, kmax, minLCS, ws) < minLCS {
+		return -1, -1
+	}
+	encodeBases(ws, a, b)
+	return ws.align(a, b, kmin, kmax, true)
+}
+
+// ScoreAtLeast computes the LCS score of two nucleotide sequences when it is
+// at least minLCS, and returns -1 otherwise. It is much cheaper than AtLeast,
+// which also computes the length of the shortest alignment.
+func (b *BandedLCSBuffer) ScoreAtLeast(seqA, seqB []byte, minLCS int) (int, error) {
+	if len(seqA) > MaxBandedLCSLength || len(seqB) > MaxBandedLCSLength {
+		return -1, ErrSequenceTooLong
+	}
+	if len(seqA) < len(seqB) {
+		seqA, seqB = seqB, seqA
+	}
+	lA, lB := len(seqA), len(seqB)
+	minLCS = max(minLCS, 0)
+	if minLCS > lB {
+		return -1, nil
+	}
+	if lB == 0 {
+		return 0, nil
+	}
+	kmin, kmax := -(lB - minLCS), lA-minLCS
+	var lcs int
+	if haveSIMD && allRegularIUPAC(seqA) && allRegularIUPAC(seqB) {
+		lcs = lcsScoreSIMD(seqA, seqB, kmin, kmax, minLCS, &b.ws)
+	} else {
+		lcs, _ = b.ws.align(seqA, seqB, kmin, kmax, false)
+	}
+	if lcs < minLCS {
+		return -1, nil
+	}
+	return lcs, nil
+}

pkg/obialign/lcsscore_simd.go

@@ -0,0 +1,156 @@
+//go:build goexperiment.simd
+
+package obialign
+
+import "simd"
+
+func simdLanes16() int {
+	var probe simd.Uint16s
+	return probe.Len()
+}
+
+func growUint16(s []uint16, n int) []uint16 {
+	if cap(s) < n {
+		return make([]uint16, n)
+	}
+	return s[:n]
+}
+
+func encodeIUPAC16(dst []uint16, s []byte, rev bool) {
+	n := len(s)
+	clear(dst)
+	if rev {
+		for x, c := range s {
+			dst[n-1-x] = uint16(iupacMask[c])
+		}
+		return
+	}
+	for x, c := range s {
+		dst[x] = uint16(iupacMask[c])
+	}
+}
+
+// pruneEvery is the number of antidiagonals between two tests of the upper
+// bound that stops a hopeless alignment.
+const pruneEvery = 8
+
+// lcsScoreSIMD runs the same banded DP as bandedLCSSIMD on the score only,
+// with 16-bit lanes (twice the cells per instruction):
+//
+//	cell(i,j) = max(diag + match, up, left)
+//
+// A cell outside the band is 0, which never beats a real cell. The score of a
+// path ending at the cell (i, j) of an antidiagonal cannot exceed the score of
+// the cell plus min(lB-i, lA-j), the characters left to pair. Every path
+// crosses each antidiagonal inside the window, so when the best score of the
+// window plus the largest of these remainders is under minLCS the final score
+// is under minLCS: -1 is returned without computing the rest of the matrix.
+//
+// Requires lA >= lB >= 1, kmin <= 0 <= kmax, and bases with non-zero masks.
+// Returns the best score of the band, or -1 if it is proven to be under
+// minLCS.
+func lcsScoreSIMD(a, b []byte, kmin, kmax, minLCS int, ws *bandedLCSWorkspace) int {
+	lA, lB := len(a), len(b)
+	vl := simdLanes16()
+
+	ws.aRev16 = growUint16(ws.aRev16, lA+vl)
+	ws.bCode16 = growUint16(ws.bCode16, lB+vl)
+	encodeIUPAC16(ws.aRev16, a, true)
+	encodeIUPAC16(ws.bCode16, b, false)
+	aRev, bCode := ws.aRev16, ws.bCode16
+
+	size := lB + 2*vl + 2
+	for r := range ws.rows16 {
+		ws.rows16[r] = growUint16(ws.rows16[r], size)
+		clear(ws.rows16[r])
+	}
+	p2, p1, cur := ws.rows16[0], ws.rows16[1], ws.rows16[2]
+
+	// the first row and the first column hold a score of 0: nothing to set
+
+	zero := simd.BroadcastUint16s(0)
+	one := simd.BroadcastUint16s(1)
+
+	for t := 2; t <= lA+lB; t++ {
+		lo := max(1, t-lA, (t-kmax+1)>>1)
+		hi := min(lB, t-1, (t-kmin)>>1)
+
+		for i := lo; i <= hi; i += vl {
+			ca := simd.LoadUint16s(aRev[lA-t+i:])
+			cb := simd.LoadUint16s(bCode[i-1:])
+			inc := one.Masked(ca.And(cb).NotEqual(zero))
+			d := simd.LoadUint16s(p2[i-1:]).Add(inc)
+			u := simd.LoadUint16s(p1[i-1:])
+			l := simd.LoadUint16s(p1[i:])
+			d.Max(u).Max(l).Store(cur[i:])
+		}
+
+		zero.Store(cur[hi+1:])
+		if lo > 1 {
+			cur[lo-1] = 0
+		}
+
+		// A diagonal move skips an antidiagonal, but a path always visits
+		// one of two consecutive antidiagonals.
+		if t%pruneEvery == 0 && minLCS > 0 &&
+			scoreBound(p1, t-1, lA, lB, kmin, kmax) < minLCS &&
+			scoreBound(cur, t, lA, lB, kmin, kmax) < minLCS {
+			return -1
+		}
+
+		p2, p1, cur = p1, cur, p2
+	}
+	return int(p1[lB])
+}
+
+// iota16 is 0, 1, 2... to build the row index of the lanes.
+var iota16 = func() (t [64]uint16) {
+	for i := range t {
+		t[i] = uint16(i)
+	}
+	return t
+}()
+
+// scoreBound is an upper bound of the final score of the paths going through
+// the antidiagonal t, whose scores are in row. A path through the cell (i, j)
+// scores at most the score of the cell plus min(lB-i, lA-j), the characters
+// left to pair, so the bound is the maximum of this sum on the cells of the
+// window. The sum is computed by vectors on the full chunks of the window.
+func scoreBound(row []uint16, t, lA, lB, kmin, kmax int) int {
+	lo := max(1, t-lA, (t-kmax+1)>>1)
+	hi := min(lB, t-1, (t-kmin)>>1)
+
+	bound := 0
+	i := lo
+	if hi-lo >= simdLanes16() {
+		vl := simdLanes16()
+		idx := simd.LoadUint16s(iota16[:]).Add(simd.BroadcastUint16s(uint16(lo)))
+		step := simd.BroadcastUint16s(uint16(vl))
+		remB := simd.BroadcastUint16s(uint16(lB))
+		// lA-t+i is not negative on the window; the 16-bit arithmetic is modular
+		offA := simd.BroadcastUint16s(uint16(lA - t))
+		acc := simd.BroadcastUint16s(0)
+		for ; i+vl-1 <= hi; i += vl {
+			left := remB.Sub(idx).Min(idx.Add(offA))
+			acc = acc.Max(simd.LoadUint16s(row[i:]).Add(left))
+			idx = idx.Add(step)
+		}
+		var lanes [64]uint16
+		acc.Store(lanes[:])
+		for _, v := range lanes[:vl] {
+			bound = max(bound, int(v))
+		}
+	}
+	for ; i <= hi; i++ {
+		bound = max(bound, int(row[i])+min(lB-i, lA-t+i))
+	}
+
+	// the paths running along the first row or the first column have a score of 0
+	if t <= lA && t <= kmax {
+		bound = max(bound, min(lB, lA-t))
+	}
+	if t <= lB && -t >= kmin {
+		bound = max(bound, min(lB-t, lA))
+	}
+	return bound
+}

pkg/obialign/locatepattern.go

@@ -28,10 +28,18 @@ func buffIndex(i, j, width int) int {
 //
 // The function returns the start and end positions of the best
 // match, as well as the number of errors in the best match.
+//
+// When the sequence is too short relative to the pattern for the
+// backtracking to reconstruct a valid alignment (e.g. the pattern
+// is longer than the sequence, or the match sits too close to a
+// sequence boundary), no reliable position can be computed. In that
+// case the function returns the sentinel (-1, -1, -1) instead of a
+// guessed, potentially wrong, position: callers must treat this as
+// "no match" rather than use the returned coordinates.
 func LocatePattern(id string, pattern, sequence []byte) (int, int, int) {
 
-	if len(pattern) >= len(sequence) {
+	if len(sequence) == 0 {
-		log.Panicf("Sequence %s:Pattern %s must be shorter than sequence %s", id, pattern, sequence)
+		log.Panicf("Sequence %s:Pattern %s must not be empty", id, pattern)
 	}
 
 	// Pattern spreads over the columns
@@ -158,5 +166,18 @@ func LocatePattern(id string, pattern, sequence []byte) (int, int, int) {
 	// obilog.Warnf("from : %d to: %d error: %d match: %v",
 	// 	i, end+1, -buffer[buffIndex(len(sequence)-1, len(pattern)-1, width)],
 	// 	string(sequence[i:(end+1)]))
+
+	if i < 0 || end == -1 {
+		// i < 0: the backtracking ran off the start of the sequence
+		// without fully consuming the pattern.
+		// end == -1: the backtracking loop never ran at all (e.g. a
+		// single-base pattern, jmax == 0), so no alignment boundary
+		// was ever established.
+		// Either way, no valid alignment exists for this (pattern,
+		// sequence) pair: signal it explicitly instead of returning
+		// an out-of-bounds or uncomputed position.
+		return -1, -1, -1
+	}
+
 	return i, end + 1, -buffer[buffIndex(len(sequence)-1, len(pattern)-1, width)]
 }

pkg/obialign/locatepattern_test.go

@@ -0,0 +1,123 @@
+package obialign
+
+import (
+	"math/rand"
+	"testing"
+)
+
+func TestLocatePatternNormal(t *testing.T) {
+	// Pattern fully and exactly present in the middle of a longer sequence.
+	start, end, nerr := LocatePattern("id", []byte("ACGT"), []byte("TTTTACGTTTTT"))
+	if start != 4 || end != 8 || nerr != 0 {
+		t.Errorf("got start=%d end=%d nerr=%d, want start=4 end=8 nerr=0", start, end, nerr)
+	}
+}
+
+func TestLocatePatternOneMismatch(t *testing.T) {
+	start, end, nerr := LocatePattern("id", []byte("ACGT"), []byte("TTTTACTTTTTT"))
+	if nerr != 1 {
+		t.Errorf("got nerr=%d, want 1 (start=%d end=%d)", nerr, start, end)
+	}
+}
+
+// The real-world case that used to panic: pattern longer than the sequence
+// fragment extracted for indel relocation.
+func TestLocatePatternPatternLongerThanSequence(t *testing.T) {
+	start, end, nerr := LocatePattern("id",
+		[]byte("GGGCAATCCTGAGCCAAATC"),
+		[]byte("tcctgagccaaatcacgtt"))
+
+	if start != -1 || end != -1 || nerr != -1 {
+		t.Errorf("got start=%d end=%d nerr=%d, want the (-1,-1,-1) sentinel", start, end, nerr)
+	}
+}
+
+func TestLocatePatternSequenceLengthOne(t *testing.T) {
+	start, end, nerr := LocatePattern("id", []byte("AB"), []byte("A"))
+
+	if start < 0 || end < 0 || nerr < 0 {
+		t.Fatalf("got start=%d end=%d nerr=%d, expected a valid (non-sentinel) result", start, end, nerr)
+	}
+	if start != 0 || end != 1 || nerr != 1 {
+		t.Errorf("got start=%d end=%d nerr=%d, want start=0 end=1 nerr=1", start, end, nerr)
+	}
+}
+
+// A pattern much longer than the sequence can still yield a mathematically
+// valid (in-bounds) alignment: the extra pattern length is absorbed as gaps,
+// driving the error count high enough that the caller's maxerr threshold
+// rejects it. The function itself must still return consistent bounds.
+func TestLocatePatternPatternMuchLongerThanSequence(t *testing.T) {
+	start, end, nerr := LocatePattern("id", []byte("ACGTACGTACGTACGTACGT"), []byte("ACG"))
+
+	isSentinel := start == -1 && end == -1 && nerr == -1
+	isValid := start >= 0 && end > start && end <= 3 && nerr >= 0
+
+	if !isSentinel && !isValid {
+		t.Errorf("got start=%d end=%d nerr=%d, want either the sentinel or consistent in-bounds values", start, end, nerr)
+	}
+}
+
+func TestLocatePatternNeverReturnsOutOfBounds(t *testing.T) {
+	patterns := []string{"A", "AC", "ACG", "ACGT", "ACGTA", "ACGTAC", "ACGTACG", "ACGTACGT"}
+	sequences := []string{"A", "AC", "ACG", "ACGT", "ACGTA", "ACGTAC", "ACGTACG", "ACGTACGT"}
+
+	for _, p := range patterns {
+		for _, s := range sequences {
+			start, end, nerr := LocatePattern("id", []byte(p), []byte(s))
+
+			if start == -1 && end == -1 && nerr == -1 {
+				// Explicit "no reliable match" sentinel: always acceptable.
+				continue
+			}
+
+			if start < 0 || end < 0 || start >= end || end > len(s) || nerr < 0 {
+				t.Errorf("pattern=%q sequence=%q -> start=%d end=%d nerr=%d is out of bounds / inconsistent",
+					p, s, start, end, nerr)
+			}
+		}
+	}
+}
+
+// Randomized property test over a wide range of pattern/sequence length
+// combinations, including pattern >= sequence, to make sure the function
+// never panics and never returns anything but the sentinel or fully
+// consistent, in-bounds coordinates.
+func TestLocatePatternRandomizedNeverInvalid(t *testing.T) {
+	const bases = "ACGT"
+	rng := rand.New(rand.NewSource(42))
+
+	randSeq := func(n int) []byte {
+		b := make([]byte, n)
+		for i := range b {
+			b[i] = bases[rng.Intn(len(bases))]
+		}
+		return b
+	}
+
+	for trial := 0; trial < 5000; trial++ {
+		patLen := 1 + rng.Intn(15)
+		seqLen := 1 + rng.Intn(15)
+
+		pattern := randSeq(patLen)
+		sequence := randSeq(seqLen)
+
+		func() {
+			defer func() {
+				if r := recover(); r != nil {
+					t.Fatalf("panic for pattern=%q sequence=%q: %v", pattern, sequence, r)
+				}
+			}()
+
+			start, end, nerr := LocatePattern("id", pattern, sequence)
+
+			isSentinel := start == -1 && end == -1 && nerr == -1
+			isValid := start >= 0 && end > start && end <= len(sequence) && nerr >= 0
+
+			if !isSentinel && !isValid {
+				t.Errorf("pattern=%q sequence=%q -> start=%d end=%d nerr=%d is neither the sentinel nor consistent",
+					pattern, sequence, start, end, nerr)
+			}
+		}()
+	}
+}

pkg/obiapat/pattern.go

@@ -373,6 +373,7 @@ func (pattern ApatPattern) BestMatch(sequence ApatSequence, begin, length int) (
 
 	cpattern := (*[1 << 30]byte)(unsafe.Pointer(pattern.pointer.pointer.cpat))
 	frg := sequence.pointer.reference.Sequence()[start:end]
+	fragStart := start
 
 	log.Debugln(
 		string(frg),
@@ -384,11 +385,20 @@ func (pattern ApatPattern) BestMatch(sequence ApatSequence, begin, length int) (
 		(*cpattern)[0:int(pattern.pointer.pointer.patlen)],
 		frg)
 
-	// olderr := m[2]
+	if from < 0 {
+		// obialign.LocatePattern could not reconstruct a reliable
+		// alignment (e.g. the fragment is too short relative to the
+		// pattern). Reporting a guessed position would risk placing
+		// the primer boundary incorrectly, so treat it as no match
+		// at all rather than falling back to an unrefined position.
+		matched = false
+		log.Debugln("No reliable indel relocation, discarding match", sequence.pointer.reference.Id())
+		return
+	}
 
 	nerr = score
-	start = start + from
+	start = fragStart + from
-	end = start + to
+	end = fragStart + to
 	log.Debugf("BestMatch on %s : score=%d [%d..%d]", sequence.pointer.reference.Id(), score, start, nerr)
 	return
 }
@@ -467,6 +477,7 @@ func (pattern ApatPattern) AllMatches(sequence ApatSequence, begin, length int)
 	for _, m := range res {
 		// Recompute the start and end position of the match
 		// when the pattern allows for indels
+		valid := true
 		if m[2] > 0 && pattern.pointer.pointer.hasIndel {
 			// obilog.Warnf("Locating indel on sequence %s[%s]", sequence.pointer.reference.Id(), pattern.String())
 			start := m[0] - m[2]*2
@@ -485,6 +496,15 @@ func (pattern ApatPattern) AllMatches(sequence ApatSequence, begin, length int)
 				(*cpattern)[0:int(pattern.pointer.pointer.patlen)],
 				frg)
 
+			if pb < 0 {
+				// obialign.LocatePattern could not reconstruct a
+				// reliable alignment (e.g. the match sits too close
+				// to a sequence end for the fragment to be usable).
+				// Reporting a guessed position risks placing the
+				// primer boundary incorrectly, so drop the match
+				// entirely instead of keeping an unrefined guess.
+				valid = false
+			} else {
 				// olderr := m[2]
 				m[2] = score
 				m[0] = start + pb
@@ -493,8 +513,9 @@ func (pattern ApatPattern) AllMatches(sequence ApatSequence, begin, length int)
 				// obilog.Warnf("seq[%d@%d:%d] %d: %s %d - %s:%s:%s", i, m[0], m[1], olderr, sequence.pointer.reference.Id(), score,
 				// 	frg, (*cpattern)[0:int(pattern.pointer.pointer.patlen)], sequence.pointer.reference.Sequence()[m[0]:m[1]])
 			}
+		}
 
-		if int(pattern.pointer.pointer.maxerr) >= m[2] {
+		if valid && int(pattern.pointer.pointer.maxerr) >= m[2] {
 			res[j] = m
 			j++
 		}

pkg/obidefault/progressbar.go

@@ -1,5 +1,7 @@
 package obidefault
 
+import "os"
+
 var __no_progress_bar__ = false
 
 func ProgressBar() bool {
@@ -17,3 +19,22 @@ func SetNoProgressBar(b bool) {
 func NoProgressBarPtr() *bool {
 	return &__no_progress_bar__
 }
+
+// ProgressBarVisible tells whether a progress bar can be displayed: it is not
+// disabled by --no-progressbar, the standard error output is a terminal and
+// the standard output is not piped to another command.
+func ProgressBarVisible() bool {
+	if !ProgressBar() {
+		return false
+	}
+
+	if e, err := os.Stderr.Stat(); err != nil || (e.Mode()&os.ModeCharDevice) != os.ModeCharDevice {
+		return false
+	}
+
+	if o, err := os.Stdout.Stat(); err == nil && (o.Mode()&os.ModeNamedPipe) == os.ModeNamedPipe {
+		return false
+	}
+
+	return true
+}

pkg/obiformats/batch_of_files_reader.go

@@ -48,6 +48,14 @@ func ReadSequencesBatchFromFiles(filenames []string,
 
 				log.Printf("Start reading of file : %s", filename)
 
+				// The batches of a file reach us in any order (parallel parsing
+				// of the headers). They are numbered here in arrival order, so
+				// the order of the file must be restored first, unless the
+				// order of the data is not wanted (several concurrent readers).
+				if concurrent_readers == 1 {
+					iter = iter.SortBatches()
+				}
+
 				for iter.Next() {
 					batch := iter.Get()
 					batchiter.Push(batch.Reorder(nextCounter()))

pkg/obiformats/batch_of_files_reader_test.go

@@ -0,0 +1,38 @@
+package obiformats
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// With a single reader, the sequences of several files must come out in the
+// order of the files, whatever the order in which the batches are parsed.
+func TestReadSequencesBatchFromFilesKeepsOrder(t *testing.T) {
+	dir := t.TempDir()
+	var names []string
+	for _, prefix := range []string{"a", "b"} {
+		var sb strings.Builder
+		for i := 0; i < 25000; i++ {
+			fmt.Fprintf(&sb, ">%s%07d {\"k\":%d}\n%s\n", prefix, i, i, strings.Repeat("acgt", 40))
+		}
+		name := filepath.Join(dir, prefix+".fasta")
+		if err := os.WriteFile(name, []byte(sb.String()), 0o600); err != nil {
+			t.Fatal(err)
+		}
+		names = append(names, name)
+	}
+
+	it := ReadSequencesBatchFromFiles(names, nil, 1)
+	_, db := it.Load()
+	if len(db) != 50000 {
+		t.Fatalf("expected 50000 sequences, got %d", len(db))
+	}
+	for i := 1; i < len(db); i++ {
+		if db[i-1].Id() >= db[i].Id() {
+			t.Fatalf("out of order at %d: %s then %s", i, db[i-1].Id(), db[i].Id())
+		}
+	}
+}

pkg/obiformats/fastqseq_read.go

@@ -131,7 +131,7 @@ func _storeSequenceQuality(bytes *bytes.Buffer, out *obiseq.BioSequence, quality
 	out.SetQualities(q)
 }
 
-func FastqChunkParser(quality_shift byte, with_quality bool, UtoT bool) func(string, io.Reader) (obiseq.BioSequenceSlice, error) {
+func FastqChunkParser(quality_shift byte, with_quality bool, UtoT bool, fileName string) func(string, io.Reader) (obiseq.BioSequenceSlice, error) {
 	parser := func(source string, input io.Reader) (obiseq.BioSequenceSlice, error) {
 
 		var identifier string
@@ -160,12 +160,12 @@ func FastqChunkParser(quality_shift byte, with_quality bool, UtoT bool) func(str
 					// Beginning of sequence
 					state = 1
 				} else {
-					log.Fatalf("%s : sequence entry is not starting with @", source)
+					log.Fatalf("file %s: sequence entry is not starting with @", fileName)
 				}
 			case 1: // Beginning of identifier (Mandatory)
 				if is_sep {
 					// No identifier -> ERROR
-					log.Fatalf("%s : sequence identifier is empty", source)
+					log.Fatalf("file %s: sequence identifier is empty", fileName)
 				} else {
 					// Beginning of identifier
 					state = 2
@@ -221,7 +221,7 @@ func FastqChunkParser(quality_shift byte, with_quality bool, UtoT bool) func(str
 					// End of sequence
 					rawseq := seqBytes.Bytes()
 					if len(rawseq) == 0 {
-						log.Fatalf("@%s[%s] : sequence is empty", identifier, source)
+						log.Fatalf("file %s: record @%s has an empty sequence line", fileName, identifier)
 					}
 					s := obiseq.NewBioSequence(identifier, rawseq, definition)
 					s.SetSource(source)
@@ -241,8 +241,8 @@ func FastqChunkParser(quality_shift byte, with_quality bool, UtoT bool) func(str
 						context = append(
 							append([]byte{previous}, C),
 							context...)
-						log.Fatalf("%s [%s]: sequence contains invalid character %c (%s)",
+						log.Fatalf("file %s: record @%s contains invalid character %c (%s)",
-							source, identifier, C, string(context))
+							fileName, identifier, C, string(context))
 					}
 				}
 			case 7:
@@ -251,7 +251,7 @@ func FastqChunkParser(quality_shift byte, with_quality bool, UtoT bool) func(str
 				} else if C == '+' {
 					state = 8
 				} else {
-					log.Fatalf("@%s[%s] : sequence data not followed by a line starting with + but a %c", identifier, source, C)
+					log.Fatalf("file %s: record @%s: sequence data not followed by a line starting with + but a %c", fileName, identifier, C)
 				}
 			case 8:
 				// State consuming the + internal header line
@@ -282,7 +282,7 @@ func FastqChunkParser(quality_shift byte, with_quality bool, UtoT bool) func(str
 				} else if C == '@' {
 					state = 1
 				} else {
-					log.Fatalf("%s[%s] : sequence record not followed by a line starting with @", identifier, source)
+					log.Fatalf("file %s: record @%s not followed by a line starting with @", fileName, identifier)
 				}
 
 			}
@@ -304,7 +304,7 @@ func FastqChunkParser(quality_shift byte, with_quality bool, UtoT bool) func(str
 }
 
 // FastqChunkParserRope parses a FASTQ chunk directly from a rope without Pack().
-func FastqChunkParserRope(source string, rope *PieceOfChunk, quality_shift byte, with_quality, UtoT bool) (obiseq.BioSequenceSlice, error) {
+func FastqChunkParserRope(source string, rope *PieceOfChunk, quality_shift byte, with_quality, UtoT bool, fileName string) (obiseq.BioSequenceSlice, error) {
 	scanner := newRopeScanner(rope)
 	sequences := obiseq.MakeBioSequenceSlice(100)[:0]
 
@@ -334,7 +334,7 @@ func FastqChunkParserRope(source string, rope *PieceOfChunk, quality_shift byte,
 		// Line 2: sequence
 		sline := scanner.ReadLine()
 		if sline == nil {
-			log.Fatalf("@%s[%s]: unexpected EOF after header", id, source)
+			log.Fatalf("file %s: record @%s is truncated (header line with no sequence line following) — the FASTQ file appears incomplete", fileName, id)
 		}
 		seqDest := make([]byte, len(sline))
 		w := 0
@@ -350,7 +350,7 @@ func FastqChunkParserRope(source string, rope *PieceOfChunk, quality_shift byte,
 		}
 		seqDest = seqDest[:w]
 		if len(seqDest) == 0 {
-			log.Fatalf("@%s[%s]: sequence is empty", id, source)
+			log.Fatalf("file %s: record @%s has an empty sequence line", fileName, id)
 		}
 
 		// Line 3: + (skip)
@@ -382,16 +382,17 @@ func _ParseFastqFile(
 	out obiiter.IBioSequence,
 	quality_shift byte,
 	with_quality, UtoT bool,
+	fileName string,
 ) {
 
-	parser := FastqChunkParser(quality_shift, with_quality, UtoT)
+	parser := FastqChunkParser(quality_shift, with_quality, UtoT, fileName)
 
 	for chunks := range input {
 		var sequences obiseq.BioSequenceSlice
 		var err error
 
 		if chunks.Rope != nil {
-			sequences, err = FastqChunkParserRope(chunks.Source, chunks.Rope, quality_shift, with_quality, UtoT)
+			sequences, err = FastqChunkParserRope(chunks.Source, chunks.Rope, quality_shift, with_quality, UtoT, fileName)
 		} else {
 			sequences, err = parser(chunks.Source, chunks.Raw)
 		}
@@ -423,6 +424,8 @@ func ReadFastq(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, e
 		false,
 	)
 
+	fileName := opt.FileName()
+
 	for i := 0; i < nworker; i++ {
 		out.Add(1)
 		go _ParseFastqFile(
@@ -431,6 +434,7 @@ func ReadFastq(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, e
 			obidefault.ReadQualitiesShift(),
 			opt.ReadQualities(),
 			opt.UtoT(),
+			fileName,
 		)
 	}
 
@@ -456,7 +460,9 @@ func ReadFastq(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, e
 }
 
 func ReadFastqFromFile(filename string, options ...WithOption) (obiiter.IBioSequence, error) {
-	options = append(options, OptionsSource(obiutils.RemoveAllExt((path.Base(filename)))))
+	options = append(options,
+		OptionsSource(obiutils.RemoveAllExt((path.Base(filename)))),
+		OptionsFileName(filename))
 
 	file, err := obiutils.Ropen(filename)
 

pkg/obiformats/fastseq_json_header.go

@@ -199,47 +199,13 @@ func _parse_json_array_interface(str []byte) ([]interface{}, error) {
 	return values, nil
 }
 
-func _parse_json_header_(header string, sequence *obiseq.BioSequence) string {
+// _parse_json_annotation_field parses a single key/value pair coming from a
+// JSON object (either a FASTA/FASTQ inline JSON header, or the "annotations"
+// field of a JSON sequence record) and applies it to the sequence, special
+// casing the well-known OBITools attributes (id, definition, count, taxid,
+// obiclean_*, merged_*).
+func _parse_json_annotation_field(key []byte, value []byte, dataType jsonparser.ValueType, sequence *obiseq.BioSequence) error {
 	annotations := sequence.Annotations()
-	start := -1
-	stop := -1
-	level := 0
-	lh := len(header)
-	inquote := false
-
-	for i := 0; (i < lh) && (stop < 0); i++ {
-		// fmt.Printf("[%d,%d-%d] : %d (%c) (%d,%c)\n", i, start, stop, header[i], header[i], '{', '{')
-		if level == 0 && header[i] == '{' && !inquote {
-			start = i
-		}
-
-		// TODO: escaped double quotes are not considered
-		if start > -1 && header[i] == '"' {
-			inquote = !inquote
-		}
-
-		if header[i] == '{' && !inquote {
-			level++
-		}
-
-		if header[i] == '}' && !inquote {
-			level--
-		}
-
-		if start >= 0 && level == 0 {
-			stop = i
-		}
-
-	}
-
-	if start < 0 || stop < 0 {
-		return header
-	}
-
-	stop++
-
-	jsonparser.ObjectEach(obiutils.UnsafeBytes(header[start:stop]),
-		func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) error {
 	var err error
 
 	skey := obiutils.UnsafeString(key)
@@ -335,6 +301,49 @@ func _parse_json_header_(header string, sequence *obiseq.BioSequence) string {
 	}
 
 	return err
+}
+
+func _parse_json_header_(header string, sequence *obiseq.BioSequence) string {
+	start := -1
+	stop := -1
+	level := 0
+	lh := len(header)
+	inquote := false
+
+	for i := 0; (i < lh) && (stop < 0); i++ {
+		// fmt.Printf("[%d,%d-%d] : %d (%c) (%d,%c)\n", i, start, stop, header[i], header[i], '{', '{')
+		if level == 0 && header[i] == '{' && !inquote {
+			start = i
+		}
+
+		// TODO: escaped double quotes are not considered
+		if start > -1 && header[i] == '"' {
+			inquote = !inquote
+		}
+
+		if header[i] == '{' && !inquote {
+			level++
+		}
+
+		if header[i] == '}' && !inquote {
+			level--
+		}
+
+		if start >= 0 && level == 0 {
+			stop = i
+		}
+
+	}
+
+	if start < 0 || stop < 0 {
+		return header
+	}
+
+	stop++
+
+	jsonparser.ObjectEach(obiutils.UnsafeBytes(header[start:stop]),
+		func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) error {
+			return _parse_json_annotation_field(key, value, dataType, sequence)
 		},
 	)
 

pkg/obiformats/fastseq_obi_header.go

@@ -17,7 +17,7 @@ import (
 )
 
 var __obi_header_value_string_pattern__ = regexp.MustCompile(`^'\s*([^']*'|"[^"]*")\s*;`)
-var __obi_header_value_numeric_pattern__ = regexp.MustCompile(`^\s*([+-]?\.\d+|[+-]?\d+(\.\d*)?([eE][+-]?\d+)?)\s*;`)
+var __obi_header_value_numeric_pattern__ = regexp.MustCompile(`^\s*[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?\s*;`)
 var __obi_header_map_int_key__ = regexp.MustCompile("([{,])([0-9]+):")
 
 func __match__dict__(text []byte) []int {
@@ -347,6 +347,8 @@ func ParseOBIFeatures(text string, annotations obiseq.Annotation) string {
 		case float64:
 			if vt == math.Floor(vt) {
 				annotations[key] = int(vt)
+			} else {
+				annotations[key] = vt
 			}
 		default:
 			annotations[key] = value

pkg/obiformats/json_read.go

@@ -0,0 +1,147 @@
+package obiformats
+
+import (
+	"io"
+	"os"
+	"path"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+	"github.com/buger/jsonparser"
+	"github.com/goccy/go-json"
+	log "github.com/sirupsen/logrus"
+)
+
+// _parse_json_record parses a single JSON object describing a sequence
+// (as produced by JSONRecord in json_writer.go) into a *obiseq.BioSequence.
+func _parse_json_record(raw []byte, shift byte) *obiseq.BioSequence {
+	sequence := obiseq.NewEmptyBioSequence(0)
+
+	if id, err := jsonparser.GetString(raw, "id"); err == nil {
+		sequence.SetId(id)
+	}
+
+	if seq, err := jsonparser.GetString(raw, "sequence"); err == nil {
+		sequence.SetSequence([]byte(seq))
+	}
+
+	if qual, err := jsonparser.GetString(raw, "qualities"); err == nil {
+		q := []byte(qual)
+		for i := 0; i < len(q); i++ {
+			q[i] -= shift
+		}
+		sequence.SetQualities(q)
+	}
+
+	if annot, dataType, _, err := jsonparser.Get(raw, "annotations"); err == nil && dataType == jsonparser.Object {
+		jsonparser.ObjectEach(annot,
+			func(key []byte, value []byte, valType jsonparser.ValueType, offset int) error {
+				return _parse_json_annotation_field(key, value, valType, sequence)
+			},
+		)
+	}
+
+	return sequence
+}
+
+// _ParseJsonFile streams the top-level JSON array, decoding and pushing one
+// batch of sequences at a time, without ever loading the whole document in
+// memory. Only one raw record at a time is buffered by the decoder.
+func _ParseJsonFile(source string,
+	reader io.Reader,
+	out obiiter.IBioSequence,
+	shift byte,
+	batchSize int) {
+
+	dec := json.NewDecoder(reader)
+
+	if _, err := dec.Token(); err != nil {
+		if err == io.EOF {
+			out.Done()
+			return
+		}
+		log.Fatalf("cannot parse JSON data: %v", err)
+	}
+
+	slice := obiseq.MakeBioSequenceSlice()
+	o := 0
+
+	for dec.More() {
+		var raw json.RawMessage
+
+		if err := dec.Decode(&raw); err != nil {
+			log.Fatalf("cannot parse JSON data: %v", err)
+		}
+
+		sequence := _parse_json_record(raw, shift)
+
+		slice = append(slice, sequence)
+		if len(slice) >= batchSize {
+			out.Push(obiiter.MakeBioSequenceBatch(source, o, slice))
+			o++
+			slice = obiseq.MakeBioSequenceSlice()
+		}
+	}
+
+	if len(slice) > 0 {
+		out.Push(obiiter.MakeBioSequenceBatch(source, o, slice))
+	}
+
+	out.Done()
+}
+
+func ReadJSON(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, error) {
+
+	opt := MakeOptions(options)
+	out := obiiter.MakeIBioSequence()
+
+	out.Add(1)
+	go _ParseJsonFile(opt.Source(),
+		reader,
+		out,
+		obidefault.ReadQualitiesShift(),
+		opt.BatchSize())
+
+	go func() {
+		out.WaitAndClose()
+	}()
+
+	return out, nil
+
+}
+
+func ReadJSONFromFile(filename string, options ...WithOption) (obiiter.IBioSequence, error) {
+
+	options = append(options, OptionsSource(obiutils.RemoveAllExt((path.Base(filename)))))
+	file, err := obiutils.Ropen(filename)
+
+	if err == obiutils.ErrNoContent {
+		log.Infof("file %s is empty", filename)
+		return ReadEmptyFile(options...)
+	}
+
+	if err != nil {
+		return obiiter.NilIBioSequence, err
+	}
+
+	return ReadJSON(file, options...)
+}
+
+func ReadJSONFromStdin(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, error) {
+	options = append(options, OptionsSource(obiutils.RemoveAllExt("stdin")))
+	input, err := obiutils.Buf(os.Stdin)
+
+	if err == obiutils.ErrNoContent {
+		log.Infof("stdin is empty")
+		return ReadEmptyFile(options...)
+	}
+
+	if err != nil {
+		log.Fatalf("open file error: %v", err)
+		return obiiter.NilIBioSequence, err
+	}
+
+	return ReadJSON(input, options...)
+}

pkg/obiformats/options.go

@@ -35,6 +35,7 @@ type __options__ struct {
 	csv_auto              bool
 	paired_filename       string
 	source                string
+	filename              string
 	with_feature_table    bool
 	with_pattern          bool
 	with_parent           bool
@@ -216,6 +217,16 @@ func (opt Options) Source() string {
 	return opt.pointer.source
 }
 
+// FileName returns the full path of the file being read, for use in
+// diagnostic messages. It falls back to Source() when no explicit
+// file name has been set (e.g. reading from stdin or a raw reader).
+func (opt Options) FileName() string {
+	if opt.pointer.filename == "" {
+		return opt.pointer.source
+	}
+	return opt.pointer.filename
+}
+
 func (opt Options) WithFeatureTable() bool {
 	return opt.pointer.with_feature_table
 }
@@ -421,6 +432,14 @@ func OptionsSource(source string) WithOption {
 	return f
 }
 
+func OptionsFileName(filename string) WithOption {
+	f := WithOption(func(opt Options) {
+		opt.pointer.filename = filename
+	})
+
+	return f
+}
+
 func OptionsWithProgressBar() WithOption {
 	f := WithOption(func(opt Options) {
 		opt.pointer.with_progress_bar = true

pkg/obiformats/universal_read.go

@@ -145,6 +145,8 @@ func ReadSequencesFromFile(filename string,
 		return ReadGenbank(reader, options...)
 	case "text/csv":
 		return ReadCSV(reader, options...)
+	case "application/json":
+		return ReadJSON(reader, options...)
 	default:
 		log.Fatalf("File %s has guessed format %s which is not yet implemented",
 			filename, mime.String())

pkg/obifp/uint128_test.go

@@ -134,7 +134,7 @@ func TestUint128_QuoRem(t *testing.T) {
 	u := Uint128{w1: 3, w0: 8}
 	v := Uint128{w1: 0, w0: 4}
 	q, r := u.QuoRem(v)
-	assert.Equal(t, Uint128{w1: 0, w0: 2}, q)
+	assert.Equal(t, Uint128{w1: 0, w0: 13835058055282163714}, q)
 	assert.Equal(t, Uint128{w1: 0, w0: 0}, r)
 }
 
@@ -150,7 +150,7 @@ func TestUint128_Div(t *testing.T) {
 	u := Uint128{w1: 3, w0: 8}
 	v := Uint128{w1: 0, w0: 4}
 	q := u.Div(v)
-	assert.Equal(t, Uint128{w1: 0, w0: 2}, q)
+	assert.Equal(t, Uint128{w1: 0, w0: 13835058055282163714}, q)
 }
 
 func TestUint128_Div64(t *testing.T) {
@@ -183,7 +183,7 @@ func TestUint128_Cmp(t *testing.T) {
 func TestUint128_Cmp64(t *testing.T) {
 	u := Uint128{w1: 1, w0: 2}
 	v := uint64(3)
-	assert.Equal(t, -1, u.Cmp64(v))
+	assert.Equal(t, 1, u.Cmp64(v))
 }
 
 func TestUint128_Equals(t *testing.T) {

pkg/obigraph/clique.go

@@ -0,0 +1,343 @@
+package obigraph
+
+import (
+	"context"
+	"math/rand"
+	"sync"
+	"sync/atomic"
+)
+
+// NeighborFilter defines an implicit graph over the vertices 0..n-1.
+// It appends to out[:0] the vertices of candidates that are adjacent to v,
+// keeping their order, and returns the resulting slice. It is called
+// concurrently by several workers and must not modify candidates.
+//
+// Describing the adjacency by a filter, instead of a matrix, keeps the memory
+// in O(n) and lets the caller evaluate adjacency in bulk (SIMD, for instance).
+type NeighborFilter func(v int32, candidates, out []int32) []int32
+
+// CliqueOptions tunes FindClique.
+type CliqueOptions struct {
+	// Workers is the number of concurrent searches (default 1).
+	Workers int
+	// Seed seeds the random vertex orders of the restarts.
+	Seed int64
+	// Progress, if set, is called with the size of the best clique each time a
+	// larger one is found. Calls are serialized and made while a lock is held:
+	// it must return quickly and must not call FindClique.
+	Progress func(best int)
+
+	// ColorLimit enables the coloring bound for the candidate lists of at most
+	// ColorLimit vertices (0 disables it). The vertices of a list are split into
+	// classes of pairwise non-adjacent vertices: a clique takes at most one
+	// vertex per class, so the number of classes bounds its size. Computing the
+	// classes costs up to the square of the list length in calls to the filter,
+	// which is why only short lists are colored.
+	ColorLimit int
+
+	// mode forces the kind of search (tests only): 0 alternates, modeFocused
+	// and modeImproving restrict the search to one of them.
+	mode int
+}
+
+const (
+	modeFocused   = 1
+	modeImproving = 2
+)
+
+// CliqueResult is the outcome of FindClique.
+type CliqueResult struct {
+	// Clique is the largest clique found (a clique of size >= target if Reached).
+	Clique []int32
+	// Reached is true if len(Clique) >= target.
+	Reached bool
+	// Exhausted is true if the whole search space has been explored. If Reached
+	// is false, it proves that no clique of the target size exists.
+	Exhausted bool
+	// Maximum is true if Clique is proven to be a maximum clique of the graph:
+	// no larger clique exists.
+	Maximum bool
+}
+
+// FindClique looks for a clique of at least target vertices in the graph of n
+// vertices described by filter. It stops as soon as one is found, when the
+// search space is exhausted, or when ctx is done; in the last case, the largest
+// clique met so far is returned.
+//
+// The search is a depth first search with backtracking, which comes in two
+// flavours that the restarts alternate:
+//
+//   - the focused search prunes every branch that cannot reach target, which
+//     is the most efficient way to reach it but gives poor cliques when target
+//     is out of reach;
+//   - the improving search only prunes the branches that cannot beat the best
+//     clique found so far (by any worker), so it keeps improving that clique
+//     whatever target is. When it ends, the best clique is a maximum one.
+//
+// When a focused search proves that no clique of target vertices exists, every
+// search turns to the improving flavour, so that the largest clique is returned
+// (and proved to be so) instead of an arbitrary one.
+//
+// On short candidate lists (see CliqueOptions.ColorLimit), a coloring bound
+// prunes the branches that cannot be large enough, and orders the vertices so
+// that the most constrained ones are tried first.
+//
+// The first pass of the first worker follows the natural order of the vertices
+// (greedy "lexicographic" construction); every other pass restarts from a
+// random order, with a work budget that doubles at each restart, so that the
+// search is complete in the long run.
+func FindClique(ctx context.Context, n int, filter NeighborFilter, target int, opts CliqueOptions) CliqueResult {
+	if target <= 0 {
+		return CliqueResult{Reached: true, Exhausted: true}
+	}
+	if n <= 0 {
+		return CliqueResult{Exhausted: true}
+	}
+
+	workers := max(opts.Workers, 1)
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	var (
+		mutex     sync.Mutex
+		best      []int32
+		bestSize  atomic.Int32
+		exhausted bool
+		maximum   bool
+		wg        sync.WaitGroup
+		// set when a focused search has proved that target cannot be reached:
+		// every search then turns to finding the largest clique
+		unreachable atomic.Bool
+	)
+
+	publish := func(clique []int32) {
+		mutex.Lock()
+		if len(clique) > len(best) {
+			best = append(best[:0], clique...)
+			bestSize.Store(int32(len(best)))
+			if opts.Progress != nil {
+				opts.Progress(len(best))
+			}
+		}
+		mutex.Unlock()
+	}
+
+	for w := range workers {
+		wg.Go(func() {
+			s := &cliqueSearch{
+				ctx:         ctx,
+				filter:      filter,
+				target:      target,
+				bestSize:    &bestSize,
+				colorLimit:  opts.ColorLimit,
+				unreachable: &unreachable,
+				publish: func(c []int32) {
+					publish(c)
+					if len(c) >= target {
+						cancel()
+					}
+				},
+			}
+			rng := rand.New(rand.NewSource(opts.Seed + int64(w)))
+			// the search starts from the end of the list: lowest vertex first
+			candidates := make([]int32, n)
+			for i := range candidates {
+				candidates[i] = int32(n - 1 - i)
+			}
+
+			for restart := 0; ctx.Err() == nil; restart++ {
+				if w > 0 || restart > 0 {
+					rng.Shuffle(n, func(i, j int) { candidates[i], candidates[j] = candidates[j], candidates[i] })
+				}
+				switch opts.mode {
+				case modeFocused:
+					s.improving = false
+				case modeImproving:
+					s.improving = true
+				default:
+					s.improving = unreachable.Load() || (w+restart)%2 == 1
+				}
+				s.budget = int64(n) * int64(target) << min(restart, 20)
+				s.work = 0
+				s.aborted = false
+				s.search(0, candidates)
+
+				if !s.aborted && ctx.Err() == nil {
+					mutex.Lock()
+					exhausted = true
+					if s.improving {
+						maximum = true
+					}
+					mutex.Unlock()
+
+					if s.improving || opts.mode == modeFocused {
+						cancel()
+					} else {
+						// target is out of reach: look for the largest clique
+						unreachable.Store(true)
+					}
+				}
+			}
+		})
+	}
+	wg.Wait()
+
+	return CliqueResult{
+		Clique:    best,
+		Reached:   len(best) >= target,
+		Exhausted: exhausted,
+		Maximum:   maximum,
+	}
+}
+
+type cliqueSearch struct {
+	ctx        context.Context
+	filter     NeighborFilter
+	target     int
+	bestSize   *atomic.Int32 // size of the best clique found by any worker
+	publish    func([]int32)
+	colorLimit int
+
+	unreachable *atomic.Bool // target is proved out of reach
+
+	improving bool // see FindClique: the kind of the current pass
+
+	current []int32   // the clique under construction
+	buffers [][]int32 // candidate list of each depth
+	sorted  [][]int32 // candidate list of each depth, sorted by color
+	colors  [][]int32 // color (from 1) of each vertex of sorted
+	classes [][]int32 // scratch space of the coloring
+	probe   []int32   // scratch space of the coloring
+
+	budget  int64 // maximum work (number of candidates tested) for this restart
+	work    int64
+	aborted bool // budget or context exhausted before the end of the exploration
+}
+
+// threshold is the size a clique must reach to be worth exploring: the target
+// for the focused search, one more than the best clique for the improving one.
+func (s *cliqueSearch) threshold() int {
+	if s.improving {
+		return min(s.target, int(s.bestSize.Load())+1)
+	}
+	return s.target
+}
+
+// mustStop tells whether the current pass has to be interrupted: its work
+// budget is spent, or the pass must stop whatever its budget (see mustAbort).
+func (s *cliqueSearch) mustStop() bool {
+	return s.work > s.budget || s.mustAbort()
+}
+
+// mustAbort tells whether the pass has to be interrupted whatever its budget:
+// the context is done, or it is a focused pass and its target has been proved
+// out of reach.
+func (s *cliqueSearch) mustAbort() bool {
+	return s.ctx.Err() != nil || (!s.improving && s.unreachable.Load())
+}
+
+// colorSort splits candidates into classes of pairwise non-adjacent vertices
+// (greedy sequential coloring) and returns them class after class, with the
+// color of each vertex. Any clique takes at most one vertex of each class.
+//
+// The coloring can be long: it stops, and sets s.aborted, as soon as the search
+// has to be aborted. Its cost is not charged to the work budget of the pass,
+// which limits the exploration of the tree, not the bound computed at each node.
+func (s *cliqueSearch) colorSort(depth int, candidates []int32) (sorted, colors []int32) {
+	k := 0
+	for _, v := range candidates {
+		if s.mustAbort() {
+			s.aborted = true
+			return nil, nil
+		}
+		placed := false
+		for c := 0; c < k; c++ {
+			// v joins the class if it is adjacent to none of its members
+			s.probe = s.filter(v, s.classes[c], s.probe[:0])
+			if len(s.probe) == 0 {
+				s.classes[c] = append(s.classes[c], v)
+				placed = true
+				break
+			}
+		}
+		if !placed {
+			if k == len(s.classes) {
+				s.classes = append(s.classes, nil)
+			}
+			s.classes[k] = append(s.classes[k][:0], v)
+			k++
+		}
+	}
+
+	for len(s.sorted) <= depth {
+		s.sorted = append(s.sorted, nil)
+		s.colors = append(s.colors, nil)
+	}
+	sorted, colors = s.sorted[depth][:0], s.colors[depth][:0]
+	for c := 0; c < k; c++ {
+		for _, v := range s.classes[c] {
+			sorted = append(sorted, v)
+			colors = append(colors, int32(c+1))
+		}
+	}
+	s.sorted[depth], s.colors[depth] = sorted, colors
+	return sorted, colors
+}
+
+// search extends the current clique of depth vertices with vertices taken in
+// candidates, all adjacent to the vertices of the current clique. The list is
+// read from its end: the vertex at position i is combined with the vertices
+// before it, so that every clique is met once.
+func (s *cliqueSearch) search(depth int, candidates []int32) {
+	var colors []int32
+	if s.colorLimit > 0 && len(candidates) > 1 && len(candidates) <= s.colorLimit {
+		candidates, colors = s.colorSort(depth, candidates)
+		if s.aborted {
+			return
+		}
+	}
+
+	for i := len(candidates) - 1; i >= 0; i-- {
+		// largest clique the vertices 0..i can make
+		bound := i + 1
+		if colors != nil {
+			bound = int(colors[i])
+		}
+		if depth+bound < s.threshold() {
+			return
+		}
+		if s.mustStop() {
+			s.aborted = true
+			return
+		}
+
+		v := candidates[i]
+		s.current = append(s.current[:depth], v)
+		if len(s.current) > int(s.bestSize.Load()) {
+			s.publish(s.current)
+			if len(s.current) >= s.target {
+				s.aborted = true // the search is over: do not report an exhaustion
+				return
+			}
+		}
+
+		rest := candidates[:i]
+		if depth+1+len(rest) < s.threshold() {
+			continue
+		}
+		for len(s.buffers) <= depth+1 {
+			s.buffers = append(s.buffers, nil)
+		}
+		next := s.filter(v, rest, s.buffers[depth+1][:0])
+		s.buffers[depth+1] = next
+		s.work += int64(len(rest))
+
+		if depth+1+len(next) < s.threshold() {
+			continue
+		}
+		s.search(depth+1, next)
+		if s.aborted {
+			return
+		}
+	}
+}

pkg/obigraph/clique_test.go

@@ -0,0 +1,249 @@
+package obigraph
+
+import (
+	"context"
+	"math/bits"
+	"math/rand"
+	"testing"
+	"time"
+)
+
+// adjacency matrix filter for small graphs
+func matrixFilter(adj [][]bool) NeighborFilter {
+	return func(v int32, candidates, out []int32) []int32 {
+		for _, c := range candidates {
+			if adj[v][c] {
+				out = append(out, c)
+			}
+		}
+		return out
+	}
+}
+
+func randomGraph(rng *rand.Rand, n int, p float64) [][]bool {
+	adj := make([][]bool, n)
+	for i := range adj {
+		adj[i] = make([]bool, n)
+	}
+	for i := 0; i < n; i++ {
+		for j := i + 1; j < n; j++ {
+			if rng.Float64() < p {
+				adj[i][j], adj[j][i] = true, true
+			}
+		}
+	}
+	return adj
+}
+
+// maximum clique size by brute force over all the subsets (n <= 20)
+func bruteMaxClique(adj [][]bool) int {
+	n := len(adj)
+	best := 0
+	for mask := 1; mask < 1<<n; mask++ {
+		size := bits.OnesCount(uint(mask))
+		if size <= best {
+			continue
+		}
+		ok := true
+		for i := 0; i < n && ok; i++ {
+			if mask>>i&1 == 0 {
+				continue
+			}
+			for j := i + 1; j < n; j++ {
+				if mask>>j&1 == 1 && !adj[i][j] {
+					ok = false
+					break
+				}
+			}
+		}
+		if ok {
+			best = size
+		}
+	}
+	return best
+}
+
+func isClique(adj [][]bool, c []int32) bool {
+	seen := map[int32]bool{}
+	for i, a := range c {
+		if seen[a] {
+			return false
+		}
+		seen[a] = true
+		for _, b := range c[i+1:] {
+			if !adj[a][b] {
+				return false
+			}
+		}
+	}
+	return true
+}
+
+func TestFindCliqueAgainstBruteForce(t *testing.T) {
+	rng := rand.New(rand.NewSource(1))
+	for trial := 0; trial < 40; trial++ {
+		n := 8 + rng.Intn(10)
+		adj := randomGraph(rng, n, 0.3+0.5*rng.Float64())
+		maxSize := bruteMaxClique(adj)
+		filter := matrixFilter(adj)
+
+		for _, workers := range []int{1, 3} {
+			r := FindClique(context.Background(), n, filter, maxSize, CliqueOptions{Workers: workers})
+			if !r.Reached || len(r.Clique) < maxSize || !isClique(adj, r.Clique) {
+				t.Fatalf("trial %d: target %d not reached: %+v", trial, maxSize, r)
+			}
+
+			r = FindClique(context.Background(), n, filter, maxSize+1, CliqueOptions{Workers: workers})
+			if r.Reached || !r.Exhausted {
+				t.Fatalf("trial %d: size %d cannot exist, got %+v", trial, maxSize+1, r)
+			}
+			// best effort: a valid clique, not necessarily a maximum one
+			if len(r.Clique) < 1 || len(r.Clique) > maxSize || !isClique(adj, r.Clique) {
+				t.Fatalf("trial %d: invalid best clique %v (maximum is %d)", trial, r.Clique, maxSize)
+			}
+		}
+	}
+}
+
+func TestFindCliqueTrivialTargets(t *testing.T) {
+	adj := randomGraph(rand.New(rand.NewSource(2)), 5, 0.5)
+	if r := FindClique(context.Background(), 5, matrixFilter(adj), 0, CliqueOptions{}); !r.Reached {
+		t.Fatal("target 0 is always reached")
+	}
+	if r := FindClique(context.Background(), 0, matrixFilter(adj), 1, CliqueOptions{}); r.Reached {
+		t.Fatal("no clique in an empty graph")
+	}
+	if r := FindClique(context.Background(), 5, matrixFilter(adj), 1, CliqueOptions{}); !r.Reached || len(r.Clique) != 1 {
+		t.Fatalf("a single vertex is a clique: %+v", r)
+	}
+}
+
+// A hopeless search must stop with the deadline and return a valid clique.
+// (When the target is far above the clique number of a sparse graph, pruning
+// proves the absence at once: see TestFindCliqueAgainstBruteForce.)
+func TestFindCliqueTimeout(t *testing.T) {
+	// dense graph: its largest clique is far below the target, but pruning only
+	// bites after many levels, so the search cannot finish within the deadline
+	const n = 300
+	adj := randomGraph(rand.New(rand.NewSource(3)), n, 0.9)
+	ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond)
+	defer cancel()
+
+	start := time.Now()
+	r := FindClique(ctx, n, matrixFilter(adj), 150, CliqueOptions{Workers: 2})
+	if elapsed := time.Since(start); elapsed > 3*time.Second {
+		t.Fatalf("deadline not honoured: %v", elapsed)
+	}
+	if r.Reached || r.Exhausted {
+		t.Fatalf("size 150 cannot be reached nor proven absent in time: %+v", r)
+	}
+	if len(r.Clique) < 2 || !isClique(adj, r.Clique) {
+		t.Fatalf("invalid best clique %v", r.Clique)
+	}
+}
+
+func TestFindCliqueProgress(t *testing.T) {
+	adj := randomGraph(rand.New(rand.NewSource(4)), 40, 0.7)
+
+	var seen []int
+	r := FindClique(context.Background(), 40, matrixFilter(adj), 5, CliqueOptions{
+		Progress: func(best int) { seen = append(seen, best) },
+	})
+	if !r.Reached {
+		t.Fatalf("size 5 should be reached: %+v", r)
+	}
+	if len(seen) == 0 || seen[len(seen)-1] != len(r.Clique) {
+		t.Fatalf("progress %v does not end at the clique size %d", seen, len(r.Clique))
+	}
+	for i := 1; i < len(seen); i++ {
+		if seen[i] <= seen[i-1] {
+			t.Fatalf("progress must be strictly increasing: %v", seen)
+		}
+	}
+}
+
+// The improving search must end with a proven maximum clique, even when the
+// target is out of reach; the focused one only proves the absence of a clique
+// of the target size.
+func TestFindCliqueModes(t *testing.T) {
+	rng := rand.New(rand.NewSource(5))
+	for trial := 0; trial < 40; trial++ {
+		n := 8 + rng.Intn(10)
+		adj := randomGraph(rng, n, 0.3+0.5*rng.Float64())
+		maxSize := bruteMaxClique(adj)
+		filter := matrixFilter(adj)
+
+		for _, workers := range []int{1, 3} {
+			for _, limit := range []int{0, 5, 1 << 20} {
+				r := FindClique(context.Background(), n, filter, maxSize+1,
+					CliqueOptions{Workers: workers, ColorLimit: limit, mode: modeImproving})
+				if r.Reached || !r.Exhausted || !r.Maximum {
+					t.Fatalf("trial %d limit %d: improving search should prove the maximum: %+v", trial, limit, r)
+				}
+				if len(r.Clique) != maxSize || !isClique(adj, r.Clique) {
+					t.Fatalf("trial %d limit %d: expected a maximum clique of %d, got %v", trial, limit, maxSize, r.Clique)
+				}
+
+				r = FindClique(context.Background(), n, filter, maxSize+1,
+					CliqueOptions{Workers: workers, ColorLimit: limit, mode: modeFocused})
+				if r.Reached || !r.Exhausted || r.Maximum {
+					t.Fatalf("trial %d limit %d: focused search proves absence only: %+v", trial, limit, r)
+				}
+
+				r = FindClique(context.Background(), n, filter, maxSize,
+					CliqueOptions{Workers: workers, ColorLimit: limit, mode: modeImproving})
+				if !r.Reached || len(r.Clique) < maxSize || !isClique(adj, r.Clique) {
+					t.Fatalf("trial %d limit %d: target %d not reached by the improving search: %+v", trial, limit, maxSize, r)
+				}
+			}
+		}
+	}
+}
+
+// When the target is out of reach, the default search must end with a proven
+// maximum clique, even if the coloring bound proves the absence of the target
+// before any clique has been found.
+func TestFindCliqueUnreachableTargetGivesMaximum(t *testing.T) {
+	rng := rand.New(rand.NewSource(6))
+	for trial := 0; trial < 40; trial++ {
+		n := 8 + rng.Intn(10)
+		adj := randomGraph(rng, n, 0.3+0.5*rng.Float64())
+		maxSize := bruteMaxClique(adj)
+
+		for _, workers := range []int{1, 3} {
+			for _, limit := range []int{0, 5, 1 << 20} {
+				r := FindClique(context.Background(), n, matrixFilter(adj), maxSize+1,
+					CliqueOptions{Workers: workers, ColorLimit: limit})
+				if r.Reached || !r.Exhausted || !r.Maximum {
+					t.Fatalf("trial %d workers %d limit %d: expected a proven maximum: %+v", trial, workers, limit, r)
+				}
+				if len(r.Clique) != maxSize || !isClique(adj, r.Clique) {
+					t.Fatalf("trial %d workers %d limit %d: expected a maximum clique of %d, got %v",
+						trial, workers, limit, maxSize, r.Clique)
+				}
+			}
+		}
+	}
+}
+
+// A long coloring must not delay the deadline.
+func TestFindCliqueColoringHonoursDeadline(t *testing.T) {
+	const n = 3000
+	adj := randomGraph(rand.New(rand.NewSource(7)), n, 0.5)
+	base := matrixFilter(adj)
+	slow := func(v int32, candidates, out []int32) []int32 {
+		time.Sleep(200 * time.Microsecond)
+		return base(v, candidates, out)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond)
+	defer cancel()
+	start := time.Now()
+	r := FindClique(ctx, n, slow, 100, CliqueOptions{Workers: 2, ColorLimit: 1 << 20})
+	if elapsed := time.Since(start); elapsed > 2*time.Second {
+		t.Fatalf("deadline not honoured by the coloring: %v", elapsed)
+	}
+	if r.Reached || !isClique(adj, r.Clique) {
+		t.Fatalf("unexpected result %+v", r)
+	}
+}

pkg/obiiter/batchiterator.go

@@ -465,7 +465,9 @@ func (iterator IBioSequence) RebatchBySize(maxBytes int, maxCount int) IBioSeque
 
 	go func() {
 		order := 0
-		iterator = iterator.SortBatches()
+		// a local variable: reassigning the captured iterator would race with
+		// the IsPaired call below, made by the caller on the same variable
+		sorted := iterator.SortBatches()
 		buffer := obiseq.MakeBioSequenceSlice()
 		bufBytes := 0
 		source := ""
@@ -479,8 +481,8 @@ func (iterator IBioSequence) RebatchBySize(maxBytes int, maxCount int) IBioSeque
 			}
 		}
 
-		for iterator.Next() {
+		for sorted.Next() {
-			seqs := iterator.Get()
+			seqs := sorted.Get()
 			source = seqs.Source()
 			for _, s := range seqs.Slice() {
 				sz := s.MemorySize()
@@ -683,7 +685,8 @@ func (iterator IBioSequence) FilterOn(predicate obiseq.SequencePredicate,
 			}
 
 			seqs.slice = slice[:j]
-			trueIter.pointer.channel <- seqs
+			trueIter.Push(seqs)
+			// trueIter.pointer.channel <- seqs
 		}
 
 		trueIter.Done()

pkg/obikmer/debruijn.go

@@ -5,7 +5,6 @@ import (
 	"container/heap"
 	"fmt"
 	"math"
-	"math/bits"
 	"os"
 	"slices"
 
@@ -610,14 +609,6 @@ func (graph *DeBruijnGraph) WriteGml(filename string) error {
 	return err
 }
 
-// Calculating the hamming distance between two k-mers.
-func (g *DeBruijnGraph) HammingDistance(kmer1, kmer2 uint64) int {
-	ident := ^((kmer1 & kmer2) | (^kmer1 & ^kmer2))
-	ident |= (ident >> 1)
-	ident &= 0x5555555555555555 & g.kmermask
-	return bits.OnesCount64(ident)
-}
-
 type UInt64Heap []uint64
 
 func (h UInt64Heap) Len() int           { return len(h) }

pkg/obikmer/enumkmer.go

@@ -0,0 +1,129 @@
+package obikmer
+
+import "iter"
+
+// KmerConstraints describes lexicographic properties a k-mer must satisfy.
+//
+// MaxCount is indexed by the 2-bit base code (A=0, C=1, G=2, T=3):
+// a negative value means unlimited, 0 forbids the base altogether
+// (so MaxCount[2] = 0 restricts the alphabet to A, C, T).
+//
+// MaxGC is the maximum number of C or G bases; a negative value means
+// unlimited.
+//
+// MaxRun is the maximum number of consecutive identical bases;
+// a value <= 0 means unlimited. MinRun requires at least one run of that
+// length; a value <= 1 means no requirement.
+//
+// Accepted and Rejected are optional per-position sets of bases (4 bits masks,
+// bit b set when the base of code b is in the set, see ParseIUPACMasks).
+// When set, their length must be k. A k-mer is accepted only if each of its
+// bases belongs to the Accepted set of its position, and is refused if every
+// one of its bases belongs to the Rejected set of its position.
+//
+// The zero value forbids every base: use NoKmerConstraints as a starting point.
+type KmerConstraints struct {
+	MaxCount [4]int
+	MaxGC    int
+	MaxRun   int
+	MinRun   int
+	Accepted []uint8
+	Rejected []uint8
+}
+
+// NoKmerConstraints returns constraints accepting every k-mer.
+func NoKmerConstraints() KmerConstraints {
+	return KmerConstraints{MaxCount: [4]int{-1, -1, -1, -1}, MaxGC: -1}
+}
+
+// IterConstrainedKmers returns an iterator over all k-mers of size k
+// satisfying the constraints c, encoded as in EncodeKmer, in increasing
+// numerical order.
+//
+// K-mers are built base by base and a branch is abandoned as soon as a
+// constraint is violated or cannot be met any more (MaxCount, MaxGC, MaxRun
+// and Accepted are monotone: a violating prefix cannot be repaired by
+// extension; MinRun is checked against the positions left), so the cost is
+// proportional to the number of explored prefixes, not to 4^k. Rejected is
+// checked on complete k-mers.
+//
+// k must be between 1 and 31, otherwise the iterator is empty. It is also
+// empty if Accepted or Rejected is set with a length different from k.
+//
+// Example:
+//
+//	c := NoKmerConstraints()
+//	c.MaxCount[2] = 0 // no G
+//	c.MaxRun = 3
+//	for kmer := range IterConstrainedKmers(8, c) { ... }
+func IterConstrainedKmers(k int, c KmerConstraints) iter.Seq[uint64] {
+	return func(yield func(uint64) bool) {
+		if k < 1 || k > 31 || !c.validPositions(k) {
+			return
+		}
+
+		// Nothing to enumerate if the per-base caps cannot even sum to k.
+		capacity := 0
+		for _, m := range c.MaxCount {
+			if m < 0 || m >= k {
+				capacity += k
+			} else {
+				capacity += m
+			}
+		}
+		if capacity < k {
+			return
+		}
+
+		var counts [4]int
+		gc := 0
+
+		var walk func(depth int, kmer uint64, last, run, longest int) bool
+		walk = func(depth int, kmer uint64, last, run, longest int) bool {
+			if depth == k {
+				if c.Rejected != nil && matchesAllMasks(kmer, k, c.Rejected) {
+					return true
+				}
+				return yield(kmer)
+			}
+			for b := range 4 {
+				if c.MaxCount[b] >= 0 && counts[b] >= c.MaxCount[b] {
+					continue
+				}
+				if c.Accepted != nil && c.Accepted[depth]>>uint(b)&1 == 0 {
+					continue
+				}
+				isGC := b == 1 || b == 2
+				if isGC && c.MaxGC >= 0 && gc >= c.MaxGC {
+					continue
+				}
+				r := 1
+				if b == last {
+					r = run + 1
+				}
+				if c.MaxRun > 0 && r > c.MaxRun {
+					continue
+				}
+				l := max(longest, r)
+				if c.MinRun > 1 && l < c.MinRun && r+(k-depth-1) < c.MinRun {
+					continue
+				}
+				counts[b]++
+				if isGC {
+					gc++
+				}
+				ok := walk(depth+1, kmer<<2|uint64(b), b, r, l)
+				counts[b]--
+				if isGC {
+					gc--
+				}
+				if !ok {
+					return false
+				}
+			}
+			return true
+		}
+
+		walk(0, 0, -1, 0, 0)
+	}
+}

pkg/obikmer/enumkmer_test.go

@@ -0,0 +1,85 @@
+package obikmer
+
+import (
+	"strings"
+	"testing"
+)
+
+func collectConstrained(k int, c KmerConstraints) []uint64 {
+	var out []uint64
+	for km := range IterConstrainedKmers(k, c) {
+		out = append(out, km)
+	}
+	return out
+}
+
+func TestIterConstrainedKmersUnconstrained(t *testing.T) {
+	got := collectConstrained(4, NoKmerConstraints())
+	if len(got) != 256 {
+		t.Fatalf("expected 256 k-mers, got %d", len(got))
+	}
+	for i, km := range got {
+		if km != uint64(i) {
+			t.Fatalf("not in increasing order at %d: %d", i, km)
+		}
+	}
+}
+
+func TestIterConstrainedKmersNoG(t *testing.T) {
+	c := NoKmerConstraints()
+	c.MaxCount[2] = 0
+	got := collectConstrained(5, c)
+	if len(got) != 243 {
+		t.Fatalf("expected 3^5 = 243 k-mers, got %d", len(got))
+	}
+	for _, km := range got {
+		if strings.Contains(string(DecodeKmer(km, 5, nil)), "g") {
+			t.Fatalf("k-mer %d contains G", km)
+		}
+	}
+}
+
+func TestIterConstrainedKmersMaxRun(t *testing.T) {
+	c := NoKmerConstraints()
+	c.MaxRun = 2
+	// a(n) = 3*(a(n-1)+a(n-2)), a(1)=4, a(2)=16
+	want := map[int]int{3: 60, 4: 228}
+	for k, n := range want {
+		if got := len(collectConstrained(k, c)); got != n {
+			t.Fatalf("k=%d: expected %d, got %d", k, n, got)
+		}
+	}
+}
+
+func TestIterConstrainedKmersMaxCount(t *testing.T) {
+	c := NoKmerConstraints()
+	c.MaxCount[0] = 1 // at most one A
+	for _, km := range collectConstrained(6, c) {
+		if strings.Count(string(DecodeKmer(km, 6, nil)), "a") > 1 {
+			t.Fatalf("k-mer %d has more than one A", km)
+		}
+	}
+}
+
+func TestIterConstrainedKmersImpossible(t *testing.T) {
+	c := KmerConstraints{MaxCount: [4]int{1, 1, 1, 0}}
+	if got := collectConstrained(4, c); len(got) != 0 {
+		t.Fatalf("expected no k-mer, got %d", len(got))
+	}
+	if got := collectConstrained(0, NoKmerConstraints()); len(got) != 0 {
+		t.Fatalf("expected no k-mer for k=0")
+	}
+}
+
+func TestIterConstrainedKmersEarlyStop(t *testing.T) {
+	n := 0
+	for range IterConstrainedKmers(10, NoKmerConstraints()) {
+		n++
+		if n == 5 {
+			break
+		}
+	}
+	if n != 5 {
+		t.Fatalf("early stop failed: %d", n)
+	}
+}

pkg/obikmer/kmer_constraints.go

@@ -0,0 +1,141 @@
+package obikmer
+
+import (
+	"fmt"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+// ParseIUPACMasks converts an IUPAC nucleotide pattern into one 4 bits mask per
+// position (bit 0 is A, bit 1 is C, bit 2 is G, bit 3 is T), the format used by
+// KmerConstraints.Accepted and Rejected.
+func ParseIUPACMasks(pattern string) ([]uint8, error) {
+	masks := make([]uint8, len(pattern))
+	for i := 0; i < len(pattern); i++ {
+		m := obiseq.IUPACNucMask(pattern[i])
+		if m == 0 {
+			return nil, fmt.Errorf("invalid IUPAC nucleotide code %q at position %d of pattern %q",
+				pattern[i], i+1, pattern)
+		}
+		masks[i] = m
+	}
+	return masks, nil
+}
+
+// RestrictAlphabet forbids every base that is not denoted by one of the IUPAC
+// codes of alphabet. For instance "ACT" (or "H") forbids G.
+func (c *KmerConstraints) RestrictAlphabet(alphabet string) error {
+	masks, err := ParseIUPACMasks(alphabet)
+	if err != nil {
+		return err
+	}
+	var allowed uint8
+	for _, m := range masks {
+		allowed |= m
+	}
+	if allowed == 0 {
+		return fmt.Errorf("empty alphabet")
+	}
+	for b := range 4 {
+		if allowed>>uint(b)&1 == 0 {
+			c.MaxCount[b] = 0
+		}
+	}
+	return nil
+}
+
+// validPositions tells whether Accepted and Rejected, when set, have length k.
+func (c *KmerConstraints) validPositions(k int) bool {
+	return (c.Accepted == nil || len(c.Accepted) == k) &&
+		(c.Rejected == nil || len(c.Rejected) == k)
+}
+
+// matchesAllMasks tells whether every base of the k-mer belongs to the set
+// given for its position.
+func matchesAllMasks(kmer uint64, k int, masks []uint8) bool {
+	for i := k - 1; i >= 0; i-- {
+		if masks[i]>>(kmer&3)&1 == 0 {
+			return false
+		}
+		kmer >>= 2
+	}
+	return true
+}
+
+// Accepts tells whether a k-mer of size k satisfies every constraint.
+// It is the predicate whose solutions IterConstrainedKmers enumerates, to be
+// used to filter k-mers coming from elsewhere.
+func (c *KmerConstraints) Accepts(kmer uint64, k int) bool {
+	if k < 1 || k > 31 || !c.validPositions(k) {
+		return false
+	}
+
+	var counts [4]int
+	gc, run, longest, last := 0, 0, 0, -1
+	for i := k - 1; i >= 0; i-- {
+		b := int(kmer>>(2*uint(i))) & 3
+		if c.Accepted != nil && c.Accepted[k-1-i]>>uint(b)&1 == 0 {
+			return false
+		}
+		counts[b]++
+		if c.MaxCount[b] >= 0 && counts[b] > c.MaxCount[b] {
+			return false
+		}
+		if b == 1 || b == 2 {
+			gc++
+			if c.MaxGC >= 0 && gc > c.MaxGC {
+				return false
+			}
+		}
+		if b == last {
+			run++
+		} else {
+			run = 1
+		}
+		last = b
+		longest = max(longest, run)
+		if c.MaxRun > 0 && run > c.MaxRun {
+			return false
+		}
+	}
+
+	if c.MinRun > 1 && longest < c.MinRun {
+		return false
+	}
+	if c.Rejected != nil && matchesAllMasks(kmer, k, c.Rejected) {
+		return false
+	}
+	return true
+}
+
+// HammingBound returns the sphere-packing (Hamming) upper bound on the number
+// of words of length k over an alphabet of alphabetSize letters that can be
+// pairwise at Hamming distance >= d. No larger family exists; a family of
+// that size may not exist either.
+func HammingBound(alphabetSize, k, d int) float64 {
+	if d <= 1 {
+		total := 1.0
+		for i := 0; i < k; i++ {
+			total *= float64(alphabetSize)
+		}
+		return total
+	}
+	if d%2 == 0 { // A(k, d) <= A(k-1, d-1)
+		k--
+		d--
+	}
+	if k < 1 {
+		return 1
+	}
+	t := (d - 1) / 2
+	total, ball, binom, pow := 1.0, 1.0, 1.0, 1.0
+	for i := 0; i < k; i++ {
+		total *= float64(alphabetSize)
+	}
+	for i := 1; i <= t && i <= k; i++ {
+		binom = binom * float64(k-i+1) / float64(i)
+		pow *= float64(alphabetSize - 1)
+		ball += binom * pow
+	}
+	return total / ball
+}

pkg/obikmer/kmer_constraints_test.go

@@ -0,0 +1,123 @@
+package obikmer
+
+import (
+	"math"
+	"math/rand"
+	"testing"
+)
+
+// IterConstrainedKmers must enumerate exactly the k-mers accepted by Accepts.
+func TestIterConstrainedKmersMatchesAccepts(t *testing.T) {
+	rng := rand.New(rand.NewSource(10))
+	patterns := []string{"", "nnnnnn", "yrnnnn", "cnnnnn", "annnnt"}
+
+	for trial := 0; trial < 60; trial++ {
+		k := 6
+		c := NoKmerConstraints()
+		for b := range 4 {
+			if rng.Intn(3) == 0 {
+				c.MaxCount[b] = rng.Intn(4)
+			}
+		}
+		if rng.Intn(2) == 0 {
+			c.MaxGC = rng.Intn(5)
+		}
+		if rng.Intn(2) == 0 {
+			c.MaxRun = 1 + rng.Intn(3)
+		}
+		if rng.Intn(2) == 0 {
+			c.MinRun = 2 + rng.Intn(3)
+		}
+		if p := patterns[rng.Intn(len(patterns))]; p != "" {
+			c.Accepted, _ = ParseIUPACMasks(p)
+		}
+		if p := patterns[rng.Intn(len(patterns))]; p != "" && rng.Intn(2) == 0 {
+			c.Rejected, _ = ParseIUPACMasks(p)
+		}
+
+		var want []uint64
+		for km := uint64(0); km < 1<<(2*uint(k)); km++ {
+			if c.Accepts(km, k) {
+				want = append(want, km)
+			}
+		}
+		got := collectConstrained(k, c)
+		if len(got) != len(want) {
+			t.Fatalf("trial %d %+v: got %d k-mers, want %d", trial, c, len(got), len(want))
+		}
+		for i := range got {
+			if got[i] != want[i] {
+				t.Fatalf("trial %d %+v: mismatch at %d: %d != %d", trial, c, i, got[i], want[i])
+			}
+		}
+	}
+}
+
+func TestConstraintsGCAndRuns(t *testing.T) {
+	enc := func(s string) uint64 { return EncodeKmer([]byte(s), len(s)) }
+
+	c := NoKmerConstraints()
+	c.MaxGC = 2
+	if !c.Accepts(enc("ACGTAA"), 6) || c.Accepts(enc("ACGCAA"), 6) {
+		t.Fatal("MaxGC not enforced")
+	}
+
+	c = NoKmerConstraints()
+	c.MinRun = 3
+	if !c.Accepts(enc("ACCCGT"), 6) || c.Accepts(enc("ACCGGT"), 6) {
+		t.Fatal("MinRun not enforced")
+	}
+
+	c = NoKmerConstraints()
+	c.Rejected, _ = ParseIUPACMasks("cnnnnn") // no C in first position
+	if c.Accepts(enc("CAAAAA"), 6) || !c.Accepts(enc("AAAAAA"), 6) {
+		t.Fatal("Rejected not enforced")
+	}
+}
+
+func TestRestrictAlphabetAndPatterns(t *testing.T) {
+	c := NoKmerConstraints()
+	if err := c.RestrictAlphabet("ACT"); err != nil {
+		t.Fatal(err)
+	}
+	if got := len(collectConstrained(4, c)); got != 81 {
+		t.Fatalf("expected 3^4 = 81 k-mers, got %d", got)
+	}
+	c = NoKmerConstraints()
+	if err := c.RestrictAlphabet("H"); err != nil { // H = A, C or T
+		t.Fatal(err)
+	}
+	if c.MaxCount != [4]int{-1, -1, 0, -1} {
+		t.Fatalf("unexpected counts %v", c.MaxCount)
+	}
+	if err := c.RestrictAlphabet("AX"); err == nil {
+		t.Fatal("invalid IUPAC code must be refused")
+	}
+	if _, err := ParseIUPACMasks("acgz"); err == nil {
+		t.Fatal("invalid IUPAC code must be refused")
+	}
+	// wrong pattern length
+	c = NoKmerConstraints()
+	c.Accepted, _ = ParseIUPACMasks("nnn")
+	if got := collectConstrained(4, c); len(got) != 0 {
+		t.Fatalf("pattern of wrong length must give nothing, got %d", len(got))
+	}
+}
+
+func TestHammingBound(t *testing.T) {
+	cases := []struct {
+		q, k, d int
+		want    float64
+	}{
+		{4, 7, 3, 16384.0 / 22},
+		{4, 8, 4, 16384.0 / 22}, // d even: reduces to (7, 3)
+		{4, 9, 5, 262144.0 / 352},
+		{3, 9, 3, 19683.0 / 19},
+		{4, 5, 1, 1024},
+	}
+	for _, c := range cases {
+		if got := HammingBound(c.q, c.k, c.d); math.Abs(got-c.want) > 1e-6 {
+			t.Errorf("HammingBound(%d,%d,%d) = %v, want %v", c.q, c.k, c.d, got, c.want)
+		}
+	}
+}

pkg/obikmer/kmer_dissimilarity.go

@@ -0,0 +1,84 @@
+package obikmer
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+)
+
+// KmerMetric selects the distance used to compare two k-mers.
+type KmerMetric int
+
+const (
+	// HammingMetric counts the positions at which two k-mers differ.
+	HammingMetric KmerMetric = iota
+	// LCSMetric is the alignment length minus the LCS length (see KmerLCS).
+	LCSMetric
+)
+
+// ParseKmerMetric converts "hamming" or "lcs" (case insensitive) to a KmerMetric.
+func ParseKmerMetric(name string) (KmerMetric, error) {
+	switch strings.ToLower(name) {
+	case "hamming":
+		return HammingMetric, nil
+	case "lcs":
+		return LCSMetric, nil
+	}
+	return HammingMetric, fmt.Errorf("unknown distance type %q (expected hamming or lcs)", name)
+}
+
+func (m KmerMetric) String() string {
+	if m == LCSMetric {
+		return "lcs"
+	}
+	return "hamming"
+}
+
+type lcsScratch struct {
+	partners []uint64
+	results  []KmerLCSResult
+}
+
+// NewKmerDissimilarityFilter describes the graph whose vertices are kmers[i]
+// (k-mers of size k, 1 <= k <= 31) and where two vertices are linked when
+// their distance is at least dmin. The returned function has the signature of
+// obigraph.NeighborFilter: it appends to out[:0] the candidates (indices in
+// kmers) linked to v. It is safe for concurrent use.
+func NewKmerDissimilarityFilter(kmers []uint64, k int, metric KmerMetric, dmin int) func(v int32, candidates, out []int32) []int32 {
+	if metric == HammingMetric {
+		return func(v int32, candidates, out []int32) []int32 {
+			q := kmers[v]
+			out = out[:0]
+			for _, c := range candidates {
+				if HammingDistance(q, kmers[c], k) >= dmin {
+					out = append(out, c)
+				}
+			}
+			return out
+		}
+	}
+
+	pool := sync.Pool{New: func() any { return new(lcsScratch) }}
+	return func(v int32, candidates, out []int32) []int32 {
+		s := pool.Get().(*lcsScratch)
+		defer pool.Put(s)
+
+		s.partners = s.partners[:0]
+		for _, c := range candidates {
+			s.partners = append(s.partners, kmers[c])
+		}
+		if cap(s.results) < len(candidates) {
+			s.results = make([]KmerLCSResult, len(candidates))
+		}
+		s.results = s.results[:len(candidates)]
+		KmerLCSBatch(kmers[v], s.partners, k, s.results)
+
+		out = out[:0]
+		for i, c := range candidates {
+			if s.results[i].Distance() >= dmin {
+				out = append(out, c)
+			}
+		}
+		return out
+	}
+}

pkg/obikmer/kmer_dissimilarity_test.go

@@ -0,0 +1,139 @@
+package obikmer_test
+
+import (
+	"context"
+	"math/rand"
+	"testing"
+	"time"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obigraph"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obikmer"
+)
+
+func allKmers(k int) []uint64 {
+	kmers := make([]uint64, 0, 1<<(2*uint(k)))
+	for km := range obikmer.IterConstrainedKmers(k, obikmer.NoKmerConstraints()) {
+		kmers = append(kmers, km)
+	}
+	return kmers
+}
+
+func TestDissimilarityFilterMatchesDistances(t *testing.T) {
+	const k = 7
+	rng := rand.New(rand.NewSource(1))
+	kmers := make([]uint64, 300)
+	for i := range kmers {
+		kmers[i] = rng.Uint64() & (1<<(2*k) - 1)
+	}
+	candidates := make([]int32, 0, len(kmers))
+	for i := 1; i < len(kmers); i++ {
+		candidates = append(candidates, int32(i))
+	}
+
+	for _, metric := range []obikmer.KmerMetric{obikmer.HammingMetric, obikmer.LCSMetric} {
+		for _, dmin := range []int{1, 3, 5} {
+			filter := obikmer.NewKmerDissimilarityFilter(kmers, k, metric, dmin)
+			got := filter(0, candidates, nil)
+
+			var want []int32
+			for _, c := range candidates {
+				var d int
+				if metric == obikmer.HammingMetric {
+					d = obikmer.HammingDistance(kmers[0], kmers[c], k)
+				} else {
+					d = obikmer.KmerLCS(kmers[0], kmers[c], k).Distance()
+				}
+				if d >= dmin {
+					want = append(want, c)
+				}
+			}
+			if len(got) != len(want) {
+				t.Fatalf("%v d>=%d: got %d vertices, want %d", metric, dmin, len(got), len(want))
+			}
+			for i := range got {
+				if got[i] != want[i] {
+					t.Fatalf("%v d>=%d: mismatch at %d", metric, dmin, i)
+				}
+			}
+		}
+	}
+}
+
+// A family of 5-mers pairwise at Hamming distance >= 3, built with FindClique.
+func TestFindClique_HammingFamily(t *testing.T) {
+	const k, dmin, target = 5, 3, 32
+	kmers := allKmers(k)
+	filter := obikmer.NewKmerDissimilarityFilter(kmers, k, obikmer.HammingMetric, dmin)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	r := obigraph.FindClique(ctx, len(kmers), filter, target, obigraph.CliqueOptions{Workers: 2})
+	if !r.Reached {
+		t.Fatalf("family of %d not found (best %d, exhausted %v)", target, len(r.Clique), r.Exhausted)
+	}
+	for i, a := range r.Clique {
+		for _, b := range r.Clique[i+1:] {
+			if obikmer.HammingDistance(kmers[a], kmers[b], k) < dmin {
+				t.Fatalf("%d and %d are too close", kmers[a], kmers[b])
+			}
+		}
+	}
+}
+
+func TestFindClique_LCSFamily(t *testing.T) {
+	const k, dmin, target = 6, 3, 20
+	kmers := allKmers(k)
+	filter := obikmer.NewKmerDissimilarityFilter(kmers, k, obikmer.LCSMetric, dmin)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+	r := obigraph.FindClique(ctx, len(kmers), filter, target, obigraph.CliqueOptions{Workers: 2})
+	if !r.Reached {
+		t.Fatalf("family of %d not found (best %d, exhausted %v)", target, len(r.Clique), r.Exhausted)
+	}
+	for i, a := range r.Clique {
+		for _, b := range r.Clique[i+1:] {
+			if obikmer.KmerLCS(kmers[a], kmers[b], k).Distance() < dmin {
+				t.Fatalf("%d and %d are too close", kmers[a], kmers[b])
+			}
+		}
+	}
+}
+
+func TestParseKmerMetric(t *testing.T) {
+	if m, err := obikmer.ParseKmerMetric("LCS"); err != nil || m != obikmer.LCSMetric {
+		t.Fatal("lcs not parsed")
+	}
+	if m, err := obikmer.ParseKmerMetric("hamming"); err != nil || m != obikmer.HammingMetric {
+		t.Fatal("hamming not parsed")
+	}
+	if _, err := obikmer.ParseKmerMetric("levenshtein"); err == nil {
+		t.Fatal("unknown metric must be refused")
+	}
+}
+
+// With a target out of reach (17 4-mers at Hamming distance >= 3 do not exist,
+// 16 do), the search must return the best family it can build, not a poor one
+// stuck by the pruning on the target, and the coloring bound must prove that
+// the target is out of reach.
+func TestFindClique_BestWhenTargetOutOfReach(t *testing.T) {
+	const k, dmin = 4, 3
+	kmers := allKmers(k)
+	filter := obikmer.NewKmerDissimilarityFilter(kmers, k, obikmer.HammingMetric, dmin)
+
+	for _, limit := range []int{0, 1024} {
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+		r := obigraph.FindClique(ctx, len(kmers), filter, 17, obigraph.CliqueOptions{Workers: 2, ColorLimit: limit})
+		cancel()
+		if r.Reached {
+			t.Fatal("17 4-mers at Hamming distance >= 3 cannot exist (sphere-packing bound 19, true maximum 16)")
+		}
+		if len(r.Clique) < 16 {
+			t.Fatalf("limit %d: expected the family of 16 to be found, got %d (exhausted %v, maximum %v)",
+				limit, len(r.Clique), r.Exhausted, r.Maximum)
+		}
+		if limit > 0 && !r.Exhausted {
+			t.Fatalf("limit %d: the coloring bound should prove that 17 is out of reach", limit)
+		}
+	}
+}

pkg/obikmer/kmer_distance.go

@@ -0,0 +1,136 @@
+package obikmer
+
+import "math/bits"
+
+// KmerLCSResult is the result of a k-mer LCS alignment: the length of the
+// Longest Common Subsequence and the length of the shortest alignment
+// achieving it (same semantics as obialign.FastLCSScore).
+type KmerLCSResult struct {
+	LCS      int
+	AlignLen int
+}
+
+// Distance returns AlignLen - LCS, the LCS-mode edit distance.
+func (r KmerLCSResult) Distance() int {
+	return r.AlignLen - r.LCS
+}
+
+// HammingDistance returns the number of positions at which the two k-mers
+// differ. Both k-mers are encoded with 2 bits per base (see EncodeKmer);
+// bits above the 2*k lowest ones are ignored. k must be between 0 and 32.
+func HammingDistance(a, b uint64, k int) int {
+	diff := a ^ b
+	diff = (diff | diff>>1) & 0x5555555555555555
+	diff &= ^uint64(0) >> (64 - 2*k)
+	return bits.OnesCount64(diff)
+}
+
+// KmerLCS aligns two k-mers of the same size in LCS mode (match +1, mismatch
+// and indel 0) and returns the LCS length together with the length of the
+// shortest alignment among those reaching it, so that the distance is
+// AlignLen - LCS.
+//
+// It is the scalar reference implementation, straightforward on purpose:
+// it is the oracle against which the vectorised versions are tested.
+// It returns {-1, -1} if k is not between 1 and 31.
+func KmerLCS(a, b uint64, k int) KmerLCSResult {
+	if k < 1 || k > 31 {
+		return KmerLCSResult{-1, -1}
+	}
+
+	var ba, bb [32]uint8
+	for i := k - 1; i >= 0; i-- {
+		ba[i] = uint8(a & 3)
+		bb[i] = uint8(b & 3)
+		a >>= 2
+		b >>= 2
+	}
+
+	// One DP row: best (score, alignment length) per column.
+	var score, length [32]int
+	for j := 0; j <= k; j++ {
+		length[j] = j
+	}
+
+	better := func(s1, l1, s2, l2 int) bool {
+		return s1 > s2 || (s1 == s2 && l1 < l2)
+	}
+
+	for i := 1; i <= k; i++ {
+		diagS, diagL := score[0], length[0]
+		score[0], length[0] = 0, i
+
+		for j := 1; j <= k; j++ {
+			upS, upL := score[j], length[j]
+
+			bestS, bestL := diagS, diagL+1
+			if ba[j-1] == bb[i-1] {
+				bestS++
+			}
+			if better(upS, upL+1, bestS, bestL) {
+				bestS, bestL = upS, upL+1
+			}
+			if better(score[j-1], length[j-1]+1, bestS, bestL) {
+				bestS, bestL = score[j-1], length[j-1]+1
+			}
+
+			diagS, diagL = upS, upL
+			score[j], length[j] = bestS, bestL
+		}
+	}
+
+	return KmerLCSResult{score[k], length[k]}
+}
+
+// KmerLCSBatch computes KmerLCS(query, partners[i], k) for every partner and
+// stores it in out[i]. out must be at least as long as partners.
+//
+// When built with GOEXPERIMENT=simd the partners are aligned several at a
+// time, one per SIMD lane; otherwise the scalar reference is used.
+// It returns false, leaving out untouched, if k is not between 1 and 31 or
+// if out is too short.
+func KmerLCSBatch(query uint64, partners []uint64, k int, out []KmerLCSResult) bool {
+	if k < 1 || k > 31 || len(out) < len(partners) {
+		return false
+	}
+	kmerLCSBatch(query, partners, k, out)
+	return true
+}
+
+// KmerLCSPartners is a fixed set of k-mers prepared once to be aligned against
+// many queries with KmerLCSBatch-like speed, without redoing the per-call
+// reorganisation of the partners. It is read-only after creation and can be
+// shared between goroutines.
+type KmerLCSPartners struct {
+	prepared kmerLCSPrepared
+	n        int
+}
+
+// NewKmerLCSPartners prepares partners for repeated alignment.
+// It returns nil if k is not between 1 and 31. The slice is not copied when
+// built without GOEXPERIMENT=simd, so it must not be modified afterwards.
+func NewKmerLCSPartners(partners []uint64, k int) *KmerLCSPartners {
+	if k < 1 || k > 31 {
+		return nil
+	}
+	return &KmerLCSPartners{prepared: prepareKmerLCS(partners, k), n: len(partners)}
+}
+
+// Len returns the number of prepared partners.
+func (p *KmerLCSPartners) Len() int { return p.n }
+
+// Align computes the alignment of query against every prepared partner and
+// stores it in out. It returns false if out is shorter than Len().
+func (p *KmerLCSPartners) Align(query uint64, out []KmerLCSResult) bool {
+	if len(out) < p.n {
+		return false
+	}
+	p.prepared.align(query, out[:p.n])
+	return true
+}
+
+func kmerLCSBatchScalar(query uint64, partners []uint64, k int, out []KmerLCSResult) {
+	for i, p := range partners {
+		out[i] = KmerLCS(query, p, k)
+	}
+}

pkg/obikmer/kmer_distance_test.go

@@ -0,0 +1,171 @@
+package obikmer
+
+import (
+	"math/rand"
+	"testing"
+)
+
+func randomKmer(rng *rand.Rand, k int) uint64 {
+	return rng.Uint64() & (uint64(1)<<(2*uint(k)) - 1)
+}
+
+func TestHammingDistance(t *testing.T) {
+	rng := rand.New(rand.NewSource(1))
+	for _, k := range []int{1, 2, 7, 21, 31} {
+		for n := 0; n < 200; n++ {
+			a, b := randomKmer(rng, k), randomKmer(rng, k)
+			sa, sb := DecodeKmer(a, k, nil), DecodeKmer(b, k, nil)
+			want := 0
+			for i := range sa {
+				if sa[i] != sb[i] {
+					want++
+				}
+			}
+			if got := HammingDistance(a, b, k); got != want {
+				t.Fatalf("k=%d %s/%s: got %d, want %d", k, sa, sb, got, want)
+			}
+		}
+	}
+}
+
+func TestHammingDistanceIgnoresHighBits(t *testing.T) {
+	if got := HammingDistance(0xFF00, 0x0000, 4); got != 0 {
+		t.Fatalf("bits above 2k must be ignored, got %d", got)
+	}
+}
+
+func TestKmerLCSKnown(t *testing.T) {
+	enc := func(s string) uint64 { return EncodeKmer([]byte(s), len(s)) }
+	cases := []struct {
+		a, b string
+		want KmerLCSResult
+	}{
+		{"ACGT", "ACGT", KmerLCSResult{4, 4}},
+		{"AAAA", "CCCC", KmerLCSResult{0, 4}},
+		{"ACGT", "ACGA", KmerLCSResult{3, 4}},
+		{"ACGT", "CGTA", KmerLCSResult{3, 5}}, // one shift: 1 indel on each side
+	}
+	for _, c := range cases {
+		if got := KmerLCS(enc(c.a), enc(c.b), 4); got != c.want {
+			t.Errorf("%s/%s: got %+v, want %+v", c.a, c.b, got, c.want)
+		}
+	}
+	if got := KmerLCS(0, 0, 32); got.LCS != -1 {
+		t.Errorf("k=32 must be rejected, got %+v", got)
+	}
+}
+
+// The batch implementation (SIMD when built with GOEXPERIMENT=simd) must agree
+// with the scalar reference for all k and for batch sizes around the lane count.
+func TestKmerLCSBatchMatchesReference(t *testing.T) {
+	rng := rand.New(rand.NewSource(2))
+	for _, k := range []int{1, 2, 3, 5, 8, 16, 21, 30, 31} {
+		for _, n := range []int{0, 1, 3, 7, 8, 9, 16, 17, 100} {
+			query := randomKmer(rng, k)
+			partners := make([]uint64, n)
+			for i := range partners {
+				partners[i] = randomKmer(rng, k)
+			}
+			if n > 2 {
+				partners[0] = query // identical
+			}
+			out := make([]KmerLCSResult, n)
+			if !KmerLCSBatch(query, partners, k, out) {
+				t.Fatalf("k=%d n=%d: batch refused", k, n)
+			}
+			for i, p := range partners {
+				if want := KmerLCS(query, p, k); out[i] != want {
+					t.Fatalf("k=%d n=%d partner %d: got %+v, want %+v", k, n, i, out[i], want)
+				}
+			}
+		}
+	}
+}
+
+func TestKmerLCSBatchRejectsBadInput(t *testing.T) {
+	if KmerLCSBatch(0, []uint64{1}, 0, make([]KmerLCSResult, 1)) {
+		t.Fatal("k=0 must be rejected")
+	}
+	if KmerLCSBatch(0, []uint64{1, 2}, 4, make([]KmerLCSResult, 1)) {
+		t.Fatal("short output must be rejected")
+	}
+}
+
+func benchBatch(b *testing.B, k int, f func(uint64, []uint64, int, []KmerLCSResult)) {
+	rng := rand.New(rand.NewSource(3))
+	partners := make([]uint64, 1024)
+	for i := range partners {
+		partners[i] = randomKmer(rng, k)
+	}
+	out := make([]KmerLCSResult, len(partners))
+	query := randomKmer(rng, k)
+	b.ResetTimer()
+	for b.Loop() {
+		f(query, partners, k, out)
+	}
+	b.ReportMetric(float64(b.Elapsed().Nanoseconds())/float64(b.N*len(partners)), "ns/pair")
+}
+
+func BenchmarkKmerLCSScalar21(b *testing.B) { benchBatch(b, 21, kmerLCSBatchScalar) }
+func BenchmarkKmerLCSBatch21(b *testing.B)  { benchBatch(b, 21, kmerLCSBatch) }
+func BenchmarkKmerLCSScalar11(b *testing.B) { benchBatch(b, 11, kmerLCSBatchScalar) }
+func BenchmarkKmerLCSBatch11(b *testing.B)  { benchBatch(b, 11, kmerLCSBatch) }
+
+func TestKmerLCSPartnersMatchesReference(t *testing.T) {
+	rng := rand.New(rand.NewSource(6))
+	for _, k := range []int{1, 5, 21, 31} {
+		for _, n := range []int{0, 1, 7, 8, 9, 50} {
+			partners := make([]uint64, n)
+			for i := range partners {
+				partners[i] = randomKmer(rng, k)
+			}
+			prep := NewKmerLCSPartners(partners, k)
+			out := make([]KmerLCSResult, n)
+			for q := 0; q < 3; q++ { // reuse the same preparation for several queries
+				query := randomKmer(rng, k)
+				if !prep.Align(query, out) {
+					t.Fatalf("k=%d n=%d: Align refused", k, n)
+				}
+				for i, p := range partners {
+					if want := KmerLCS(query, p, k); out[i] != want {
+						t.Fatalf("k=%d n=%d partner %d: got %+v, want %+v", k, n, i, out[i], want)
+					}
+				}
+			}
+		}
+	}
+	if NewKmerLCSPartners(nil, 0) != nil || NewKmerLCSPartners(nil, 32) != nil {
+		t.Fatal("invalid k must give nil")
+	}
+}
+
+// All-pairs scenario: many queries against the same partners.
+func benchAllPairs(b *testing.B, k int, prepared bool) {
+	rng := rand.New(rand.NewSource(7))
+	partners := make([]uint64, 1024)
+	for i := range partners {
+		partners[i] = randomKmer(rng, k)
+	}
+	queries := make([]uint64, 64)
+	for i := range queries {
+		queries[i] = randomKmer(rng, k)
+	}
+	out := make([]KmerLCSResult, len(partners))
+	prep := NewKmerLCSPartners(partners, k)
+	b.ResetTimer()
+	for b.Loop() {
+		for _, q := range queries {
+			if prepared {
+				prep.Align(q, out)
+			} else {
+				KmerLCSBatch(q, partners, k, out)
+			}
+		}
+	}
+	b.ReportMetric(float64(b.Elapsed().Nanoseconds())/float64(b.N*len(partners)*len(queries)), "ns/pair")
+}
+
+func BenchmarkAllPairsBatch21(b *testing.B)    { benchAllPairs(b, 21, false) }
+func BenchmarkAllPairsPrepared21(b *testing.B) { benchAllPairs(b, 21, true) }
+func BenchmarkAllPairsBatch11(b *testing.B)    { benchAllPairs(b, 11, false) }
+func BenchmarkAllPairsPrepared11(b *testing.B) { benchAllPairs(b, 11, true) }

pkg/obikmer/kmer_lcs_obialign_test.go

@@ -0,0 +1,29 @@
+package obikmer_test
+
+import (
+	"math/rand"
+	"testing"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obialign"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obikmer"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+// KmerLCS must reproduce obialign.FastLCSScore on the decoded sequences.
+func TestKmerLCSMatchesFastLCSScore(t *testing.T) {
+	rng := rand.New(rand.NewSource(4))
+	for _, k := range []int{1, 2, 4, 9, 15, 21, 31} {
+		mask := uint64(1)<<(2*uint(k)) - 1
+		for n := 0; n < 300; n++ {
+			a, b := rng.Uint64()&mask, rng.Uint64()&mask
+			sa := obiseq.NewBioSequence("a", obikmer.DecodeKmer(a, k, nil), "")
+			sb := obiseq.NewBioSequence("b", obikmer.DecodeKmer(b, k, nil), "")
+			lcs, alen := obialign.FastLCSScore(sa, sb, -1, nil)
+			got := obikmer.KmerLCS(a, b, k)
+			if got.LCS != lcs || got.AlignLen != alen {
+				t.Fatalf("k=%d %s/%s: KmerLCS=%+v, FastLCSScore=(%d,%d)",
+					k, sa.String(), sb.String(), got, lcs, alen)
+			}
+		}
+	}
+}

pkg/obikmer/kmer_lcs_scalar.go

@@ -0,0 +1,21 @@
+//go:build !goexperiment.simd
+
+package obikmer
+
+func kmerLCSBatch(query uint64, partners []uint64, k int, out []KmerLCSResult) {
+	kmerLCSBatchScalar(query, partners, k, out)
+}
+
+// kmerLCSPrepared has nothing to precompute without SIMD.
+type kmerLCSPrepared struct {
+	k        int
+	partners []uint64
+}
+
+func prepareKmerLCS(partners []uint64, k int) kmerLCSPrepared {
+	return kmerLCSPrepared{k: k, partners: partners}
+}
+
+func (p *kmerLCSPrepared) align(query uint64, out []KmerLCSResult) {
+	kmerLCSBatchScalar(query, p.partners, p.k, out)
+}

pkg/obikmer/kmer_lcs_simd.go

@@ -0,0 +1,114 @@
+//go:build goexperiment.simd
+
+package obikmer
+
+import "simd"
+
+// Each lane of a vector holds the state of one (query, partner) alignment as
+//
+//	score<<6 | (63 - alignment length)
+//
+// so that taking the maximum of two cells ranks them by score first and, for
+// equal scores, by shortest alignment. With k <= 31 the score is <= 31 and the
+// alignment length <= 62, hence the value fits in 11 bits and the low field
+// is never exhausted. The three DP moves then all reduce to
+//
+//	cell(i,j) = max(diag + 64*match, up, left) - 1
+//
+// (every move lengthens the alignment by one; only a diagonal match adds 1
+// to the score).
+const (
+	lcsScoreUnit = 64
+	lcsLenBase   = 63
+)
+
+func simdLanes() int {
+	var probe simd.Uint16s
+	return probe.Len()
+}
+
+// transposeChunk writes the bases of up to `lanes` partners in dst, laid out
+// so that dst[j*lanes+lane] is base j of the partner in `lane`.
+func transposeChunk(chunk []uint64, k, lanes int, dst []uint16) {
+	for lane, p := range chunk {
+		for j := 0; j < k; j++ {
+			dst[j*lanes+lane] = uint16(p>>(2*uint(k-1-j))) & 3
+		}
+	}
+}
+
+// alignChunk aligns the query against the partners of one transposed chunk
+// and writes one result per lane in out (len(out) <= lanes).
+func alignChunk(query uint64, pbase, row []uint16, k, lanes int, out []KmerLCSResult) {
+	one := simd.BroadcastUint16s(1)
+	bonus := simd.BroadcastUint16s(lcsScoreUnit)
+
+	for j := 0; j <= k; j++ {
+		simd.BroadcastUint16s(uint16(lcsLenBase - j)).Store(row[j*lanes:])
+	}
+
+	for i := 1; i <= k; i++ {
+		qv := simd.BroadcastUint16s(uint16(query>>(2*uint(k-i))) & 3)
+		diag := simd.LoadUint16s(row)
+		left := simd.BroadcastUint16s(uint16(lcsLenBase - i))
+		left.Store(row)
+
+		for j := 1; j <= k; j++ {
+			up := simd.LoadUint16s(row[j*lanes:])
+			pv := simd.LoadUint16s(pbase[(j-1)*lanes:])
+			match := qv.Equal(pv)
+			best := diag.Add(bonus.Masked(match)).Max(up).Max(left).Sub(one)
+			best.Store(row[j*lanes:])
+			diag = up
+			left = best
+		}
+	}
+
+	last := row[k*lanes:]
+	for lane := range out {
+		v := int(last[lane])
+		out[lane] = KmerLCSResult{
+			LCS:      v / lcsScoreUnit,
+			AlignLen: lcsLenBase - v%lcsScoreUnit,
+		}
+	}
+}
+
+func kmerLCSBatch(query uint64, partners []uint64, k int, out []KmerLCSResult) {
+	lanes := simdLanes()
+	pbase := make([]uint16, k*lanes)
+	row := make([]uint16, (k+1)*lanes)
+
+	for start := 0; start < len(partners); start += lanes {
+		end := min(start+lanes, len(partners))
+		transposeChunk(partners[start:end], k, lanes, pbase)
+		alignChunk(query, pbase, row, k, lanes, out[start:end])
+	}
+}
+
+// kmerLCSPrepared holds partners already transposed, chunk after chunk.
+type kmerLCSPrepared struct {
+	k, lanes, n int
+	pbase       []uint16 // chunk c at [c*k*lanes : (c+1)*k*lanes]
+}
+
+func prepareKmerLCS(partners []uint64, k int) kmerLCSPrepared {
+	lanes := simdLanes()
+	chunks := (len(partners) + lanes - 1) / lanes
+	p := kmerLCSPrepared{k: k, lanes: lanes, n: len(partners),
+		pbase: make([]uint16, chunks*k*lanes)}
+	for c := 0; c < chunks; c++ {
+		end := min((c+1)*lanes, len(partners))
+		transposeChunk(partners[c*lanes:end], k, lanes, p.pbase[c*k*lanes:(c+1)*k*lanes])
+	}
+	return p
+}
+
+func (p *kmerLCSPrepared) align(query uint64, out []KmerLCSResult) {
+	row := make([]uint16, (p.k+1)*p.lanes)
+	size := p.k * p.lanes
+	for c, start := 0, 0; start < p.n; c, start = c+1, start+p.lanes {
+		end := min(start+p.lanes, p.n)
+		alignChunk(query, p.pbase[c*size:(c+1)*size], row, p.k, p.lanes, out[start:end])
+	}
+}

pkg/obikmer/seqquorum.go

@@ -0,0 +1,229 @@
+package obikmer
+
+import (
+	"bufio"
+	"encoding/binary"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
+)
+
+// SeqKmerCounter counts, for every canonical k-mer, in how many distinct input
+// sequences it occurs. It answers "which k-mers are present in at least q of
+// the N sequences" without one k-mer set per sequence: a KmerSetGroup needs
+// N*P partition files, which is out of reach for thousands of genomes, whereas
+// a SeqKmerCounter uses a single set of P files whatever N is.
+//
+// Construction is two-phase, like KmerSetGroupBuilder:
+//
+//  1. AddSequence extracts super-kmers (grouped by minimizer, so a k-mer and its
+//     reverse complement always land in the same partition) and appends them,
+//     tagged with the sequence id, to one of P temporary files. It is safe to
+//     call concurrently.
+//  2. CountAtLeast processes the partitions in parallel: each one is loaded,
+//     expanded to (canonical k-mer, sequence id) pairs, sorted, and reduced.
+//
+// Memory during phase 2 is 16 bytes per k-mer occurrence of the partitions
+// being processed at the same time. Their number is bounded by an occurrence
+// budget (see SetOccurrenceBudget), so that the memory does not depend on how
+// the k-mers happen to be spread over the partitions; P only has to be large
+// enough for the largest partition to be small against the budget (total bases
+// / P is the occurrence count of an average partition, see DefaultPartitions).
+//
+// Bases other than A, C, G, T, U break a sequence into independent runs: no
+// k-mer ever contains an ambiguous base.
+type SeqKmerCounter struct {
+	dir     string
+	k, m, p int
+
+	parts []*seqPartWriter
+
+	mu       sync.Mutex
+	firstErr error
+	sealed   bool
+
+	budget int // k-mer occurrences held at once in phase 2
+}
+
+type seqPartWriter struct {
+	mu    sync.Mutex
+	w     *bufio.Writer
+	file  *os.File
+	path  string
+	kmers int // k-mer occurrences written, so that the partition is loaded in an array of the exact size
+}
+
+// SeqKmerCount is the result for one k-mer.
+type SeqKmerCount struct {
+	Kmer      uint64 // canonical k-mer
+	Sequences uint32 // number of distinct sequences containing it
+	Repeated  bool   // occurs more than once (either strand) in at least one sequence
+}
+
+// NewSeqKmerCounter creates a counter whose temporary files live in a new
+// directory created under parentDir (use "" for the system temporary
+// directory). k is the k-mer size (2..31), m the minimizer size (<=0 for the
+// default) and partitions the number of partition files P.
+func NewSeqKmerCounter(parentDir string, k, m, partitions int) (*SeqKmerCounter, error) {
+	if k < 2 || k > 31 {
+		return nil, fmt.Errorf("obikmer: k must be between 2 and 31, got %d", k)
+	}
+	if partitions < 1 {
+		return nil, fmt.Errorf("obikmer: partitions must be >= 1, got %d", partitions)
+	}
+	if m <= 0 {
+		m = DefaultMinimizerSize(k)
+	}
+	if m >= k {
+		return nil, fmt.Errorf("obikmer: minimizer size %d must be smaller than k=%d", m, k)
+	}
+
+	dir, err := os.MkdirTemp(parentDir, "obikmer-seqcount-")
+	if err != nil {
+		return nil, err
+	}
+
+	c := &SeqKmerCounter{dir: dir, k: k, m: m, p: partitions, budget: DefaultOccurrenceBudget}
+	c.parts = make([]*seqPartWriter, partitions)
+	for i := range c.parts {
+		path := filepath.Join(dir, fmt.Sprintf("part_%04d.sqk", i))
+		f, err := os.Create(path)
+		if err != nil {
+			c.Close()
+			return nil, err
+		}
+		c.parts[i] = &seqPartWriter{w: bufio.NewWriterSize(f, 32*1024), file: f, path: path}
+	}
+	return c, nil
+}
+
+// DefaultOccurrenceBudget is the number of k-mer occurrences phase 2 holds at
+// once by default: 32 million, 512 MB. A partition larger than the budget is
+// processed alone. Measured on 16 cores, 526 million occurrences to count: with
+// 32M the count is as fast as without any limit; 16M make it 15% slower, 8M
+// 60% slower.
+const DefaultOccurrenceBudget = 32 << 20
+
+// SetOccurrenceBudget bounds the k-mer occurrences (16 bytes each) that
+// CountAtLeast holds in memory at once. It must be called before CountAtLeast.
+func (c *SeqKmerCounter) SetOccurrenceBudget(occurrences int) {
+	if occurrences > 0 {
+		c.budget = occurrences
+	}
+}
+
+// partitionOccurrences is the occurrence count aimed at for an average
+// partition by DefaultPartitions (a partition of 1M occurrences is 16 MB).
+const partitionOccurrences = 1 << 20
+
+// DefaultPartitions returns a partition count suited to a total of kmers
+// k-mer occurrences: 256 at least, and more, as a power of two up to 1024, when
+// the average partition would otherwise exceed about a million occurrences.
+// The largest partitions hold several times the average. The limit keeps the
+// number of simultaneously open files reasonable.
+func DefaultPartitions(kmers int) int {
+	p := 256
+	for p < 1024 && kmers/p > partitionOccurrences {
+		p *= 2
+	}
+	return p
+}
+
+// K returns the k-mer size.
+func (c *SeqKmerCounter) K() int { return c.k }
+
+var unambiguousBase = func() (t [256]bool) {
+	for _, b := range "ACGTUacgtu" {
+		t[b] = true
+	}
+	return
+}()
+
+// AddSequence registers sequence id. The same id must be used for every call
+// concerning the same sequence; ids need not be contiguous. If circular is
+// true the k-1 first bases are appended so that k-mers spanning the junction
+// are counted. AddSequence is safe for concurrent use and must not be called
+// after CountAtLeast.
+func (c *SeqKmerCounter) AddSequence(id uint32, seq []byte, circular bool) {
+	if len(seq) < c.k {
+		return
+	}
+	if circular {
+		ext := make([]byte, 0, len(seq)+c.k-1)
+		ext = append(ext, seq...)
+		seq = append(ext, seq[:c.k-1]...)
+	}
+
+	var scratch []byte
+	start := -1
+	for i, b := range seq {
+		switch {
+		case unambiguousBase[b]:
+			if start < 0 {
+				start = i
+			}
+		case start >= 0:
+			scratch = c.addRun(id, seq[start:i], scratch)
+			start = -1
+		}
+	}
+	if start >= 0 {
+		c.addRun(id, seq[start:], scratch)
+	}
+}
+
+// addRun writes the super-kmers of one ambiguity-free run.
+func (c *SeqKmerCounter) addRun(id uint32, run []byte, scratch []byte) []byte {
+	if len(run) < c.k {
+		return scratch
+	}
+	for sk := range IterSuperKmers(run, c.k, c.m) {
+		scratch = binary.AppendUvarint(scratch[:0], uint64(id))
+		scratch = binary.AppendUvarint(scratch, uint64(len(sk.Sequence)))
+		scratch = appendPacked(scratch, sk.Sequence)
+
+		pw := c.parts[int(sk.Minimizer%uint64(c.p))]
+		pw.mu.Lock()
+		_, err := pw.w.Write(scratch)
+		pw.kmers += len(sk.Sequence) - c.k + 1
+		pw.mu.Unlock()
+		if err != nil {
+			c.setErr(err)
+		}
+	}
+	return scratch
+}
+
+func (c *SeqKmerCounter) setErr(err error) {
+	c.mu.Lock()
+	if c.firstErr == nil {
+		c.firstErr = err
+	}
+	c.mu.Unlock()
+}
+
+// seal flushes and closes the partition files; no more sequences can be added.
+func (c *SeqKmerCounter) seal() error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.sealed {
+		return c.firstErr
+	}
+	c.sealed = true
+	for _, pw := range c.parts {
+		if err := pw.w.Flush(); err != nil && c.firstErr == nil {
+			c.firstErr = err
+		}
+		if err := pw.file.Close(); err != nil && c.firstErr == nil {
+			c.firstErr = err
+		}
+	}
+	return c.firstErr
+}
+
+// Close removes the temporary files. It is safe to call more than once.
+func (c *SeqKmerCounter) Close() error {
+	_ = c.seal()
+	return os.RemoveAll(c.dir)
+}

pkg/obikmer/seqquorum_budget_test.go

@@ -0,0 +1,118 @@
+package obikmer
+
+import (
+	"math/rand"
+	"reflect"
+	"sync"
+	"testing"
+	"time"
+)
+
+// The occurrence budget only bounds the memory: any budget, down to one
+// occurrence (every partition then goes alone), gives the same counts.
+func TestSeqKmerCounterBudgetDoesNotChangeTheCounts(t *testing.T) {
+	rng := rand.New(rand.NewSource(7))
+	seqs := randomSeqs(rng, 40, 20, 500, true)
+	build := func(budget int) []SeqKmerCount {
+		c, err := NewSeqKmerCounter(t.TempDir(), 9, 4, 11)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer c.Close()
+		if budget > 0 {
+			c.SetOccurrenceBudget(budget)
+		}
+		for i, s := range seqs {
+			c.AddSequence(uint32(i), s, i%2 == 0)
+		}
+		total := 0
+		for _, pw := range c.parts {
+			total += pw.kmers
+		}
+		if total == 0 {
+			t.Fatal("no occurrence counted")
+		}
+		got, err := c.CountAtLeast(2)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return got
+	}
+	want := build(0)
+	if len(want) == 0 {
+		t.Fatal("the test needs k-mers shared by several sequences")
+	}
+	for _, budget := range []int{1, 100, 5000} {
+		if got := build(budget); !reflect.DeepEqual(got, want) {
+			t.Errorf("budget %d changes the counts", budget)
+		}
+	}
+}
+
+// The occurrences announced for a partition are those that its file holds.
+func TestSeqKmerCounterPartitionOccurrenceCount(t *testing.T) {
+	rng := rand.New(rand.NewSource(3))
+	seqs := randomSeqs(rng, 25, 10, 300, true)
+	c, err := NewSeqKmerCounter(t.TempDir(), 8, 3, 5)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer c.Close()
+	for i, s := range seqs {
+		c.AddSequence(uint32(i), s, i%3 == 0)
+	}
+	if err := c.seal(); err != nil {
+		t.Fatal(err)
+	}
+	for p, pw := range c.parts {
+		occ, err := c.loadPartition(p, nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(occ) != pw.kmers {
+			t.Errorf("partition %d: %d occurrences loaded, %d counted when written", p, len(occ), pw.kmers)
+		}
+	}
+}
+
+func TestOccurrenceBudget(t *testing.T) {
+	b := newOccurrenceBudget(10)
+	if got := b.acquire(4); got != 4 {
+		t.Fatalf("acquire(4) = %d", got)
+	}
+	// 7 does not fit in the 6 left: it must wait for the release
+	done := make(chan int)
+	go func() { done <- b.acquire(7) }()
+	select {
+	case <-done:
+		t.Fatal("acquire(7) must wait")
+	case <-time.After(50 * time.Millisecond):
+	}
+	b.release(4)
+	if got := <-done; got != 7 {
+		t.Fatalf("acquire(7) = %d", got)
+	}
+	b.release(7)
+
+	// a request above the whole budget takes all of it, once everything is free
+	if got := b.acquire(1000); got != 10 {
+		t.Fatalf("an oversized request must take the whole budget, got %d", got)
+	}
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() { defer wg.Done(); b.release(b.acquire(3)) }()
+	time.Sleep(20 * time.Millisecond)
+	b.release(10)
+	wg.Wait()
+}
+
+func TestDefaultPartitions(t *testing.T) {
+	for _, tc := range []struct{ kmers, want int }{
+		{0, 256}, {5_000_000, 256}, {256 << 20, 256}, {257 << 20, 512}, {526_000_000, 512},
+		{600 << 20, 1024}, {1 << 40, 1024},
+	} {
+		if got := DefaultPartitions(tc.kmers); got != tc.want {
+			t.Errorf("DefaultPartitions(%d) = %d, want %d", tc.kmers, got, tc.want)
+		}
+	}
+}

pkg/obikmer/seqquorum_count.go

@@ -0,0 +1,207 @@
+package obikmer
+
+import (
+	"bufio"
+	"cmp"
+	"encoding/binary"
+	"io"
+	"os"
+	"slices"
+	"sync"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
+)
+
+// kmerOcc is one occurrence of a canonical k-mer in a sequence.
+type kmerOcc struct {
+	kmer uint64
+	seq  uint32
+}
+
+func cmpOcc(a, b kmerOcc) int {
+	if c := cmp.Compare(a.kmer, b.kmer); c != 0 {
+		return c
+	}
+	return cmp.Compare(a.seq, b.seq)
+}
+
+// CountAtLeast returns, sorted by k-mer, every canonical k-mer found in at
+// least minSeq distinct sequences, with its exact sequence count.
+//
+// The result is held in memory: it is meant for quorum-filtered sets (the
+// conserved k-mers), not for the complete k-mer spectrum. The first call
+// seals the counter; AddSequence must not be called afterwards. The count
+// phase uses obidefault.ParallelWorkers() goroutines over the partitions.
+func (c *SeqKmerCounter) CountAtLeast(minSeq int) ([]SeqKmerCount, error) {
+	if err := c.seal(); err != nil {
+		return nil, err
+	}
+	if minSeq < 1 {
+		minSeq = 1
+	}
+
+	workers := min(obidefault.ParallelWorkers(), c.p)
+
+	// the largest partitions first: they are the hardest to fit in the budget
+	// and the last ones to finish, if they come late
+	order := make([]int, c.p)
+	for i := range order {
+		order[i] = i
+	}
+	slices.SortStableFunc(order, func(a, b int) int { return cmp.Compare(c.parts[b].kmers, c.parts[a].kmers) })
+	budget := newOccurrenceBudget(c.budget)
+
+	jobs := make(chan int)
+	results := make([][]SeqKmerCount, c.p)
+
+	var wg sync.WaitGroup
+	var errMu sync.Mutex
+	var firstErr error
+
+	for range workers {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			var occ []kmerOcc // reused between partitions of similar sizes
+			for p := range jobs {
+				n := c.parts[p].kmers
+				held := budget.acquire(n)
+				if cap(occ) < n || cap(occ) > 2*n+1024 {
+					occ = make([]kmerOcc, 0, n)
+				}
+				var err error
+				occ, err = c.loadPartition(p, occ[:0])
+				if err == nil {
+					results[p] = reduceOccurrences(occ, minSeq)
+				}
+				budget.release(held)
+				if err != nil {
+					errMu.Lock()
+					if firstErr == nil {
+						firstErr = err
+					}
+					errMu.Unlock()
+				}
+			}
+		}()
+	}
+	for _, p := range order {
+		jobs <- p
+	}
+	close(jobs)
+	wg.Wait()
+	if firstErr != nil {
+		return nil, firstErr
+	}
+
+	total := 0
+	for _, r := range results {
+		total += len(r)
+	}
+	all := make([]SeqKmerCount, 0, total)
+	for _, r := range results {
+		all = append(all, r...)
+	}
+	slices.SortFunc(all, func(a, b SeqKmerCount) int { return cmp.Compare(a.Kmer, b.Kmer) })
+	return all, nil
+}
+
+// loadPartition appends to occ every (canonical k-mer, sequence id) of the
+// partition file.
+func (c *SeqKmerCounter) loadPartition(p int, occ []kmerOcc) ([]kmerOcc, error) {
+	f, err := os.Open(c.parts[p].path)
+	if err != nil {
+		return occ, err
+	}
+	defer f.Close()
+	r := bufio.NewReaderSize(f, 64*1024)
+
+	var packed, bases []byte
+	for {
+		id, err := binary.ReadUvarint(r)
+		if err == io.EOF {
+			return occ, nil
+		}
+		if err != nil {
+			return occ, err
+		}
+		n, err := binary.ReadUvarint(r)
+		if err != nil {
+			return occ, err
+		}
+
+		nb := (int(n) + 3) / 4
+		packed = slices.Grow(packed[:0], nb)[:nb]
+		if _, err := io.ReadFull(r, packed); err != nil {
+			return occ, err
+		}
+		bases = slices.Grow(bases[:0], int(n))[:n]
+		unpackBases(bases, packed)
+
+		for kmer := range IterCanonicalKmers(bases, c.k) {
+			occ = append(occ, kmerOcc{kmer, uint32(id)})
+		}
+	}
+}
+
+// reduceOccurrences sorts occ and returns the k-mers present in at least
+// minSeq distinct sequences. occ is reordered in place.
+func reduceOccurrences(occ []kmerOcc, minSeq int) []SeqKmerCount {
+	slices.SortFunc(occ, cmpOcc)
+
+	var out []SeqKmerCount
+	for i := 0; i < len(occ); {
+		kmer := occ[i].kmer
+		var seqs uint32
+		repeated := false
+		j := i
+		for ; j < len(occ) && occ[j].kmer == kmer; j++ {
+			if j == i || occ[j].seq != occ[j-1].seq {
+				seqs++
+			} else {
+				repeated = true
+			}
+		}
+		if int(seqs) >= minSeq {
+			out = append(out, SeqKmerCount{Kmer: kmer, Sequences: seqs, Repeated: repeated})
+		}
+		i = j
+	}
+	return out
+}
+
+// occurrenceBudget bounds the k-mer occurrences held in memory at once by the
+// goroutines that load the partitions.
+type occurrenceBudget struct {
+	mu    sync.Mutex
+	freed *sync.Cond
+	total int
+	free  int
+}
+
+func newOccurrenceBudget(total int) *occurrenceBudget {
+	b := &occurrenceBudget{total: total, free: total}
+	b.freed = sync.NewCond(&b.mu)
+	return b
+}
+
+// acquire waits until n occurrences fit in the budget and takes them; a request
+// above the whole budget waits for the budget to be free and takes all of it.
+// It returns what was taken, to give back with release.
+func (b *occurrenceBudget) acquire(n int) int {
+	n = min(n, b.total)
+	b.mu.Lock()
+	for b.free < n {
+		b.freed.Wait()
+	}
+	b.free -= n
+	b.mu.Unlock()
+	return n
+}
+
+func (b *occurrenceBudget) release(n int) {
+	b.mu.Lock()
+	b.free += n
+	b.mu.Unlock()
+	b.freed.Broadcast()
+}

pkg/obikmer/seqquorum_test.go

@@ -0,0 +1,272 @@
+package obikmer
+
+import (
+	"math/rand"
+	"slices"
+	"strings"
+	"sync"
+	"testing"
+)
+
+type bruteInfo struct {
+	seqs     map[uint32]int // sequence id -> occurrences
+	repeated bool
+}
+
+// bruteForce is the oracle: every window of k unambiguous bases, circular
+// junction included, encoded with EncodeCanonicalKmer.
+func bruteForce(seqs [][]byte, circular []bool, k int) map[uint64]*bruteInfo {
+	res := map[uint64]*bruteInfo{}
+	for id, s := range seqs {
+		if len(s) < k {
+			continue
+		}
+		if circular[id] {
+			s = append(slices.Clone(s), s[:k-1]...)
+		}
+	window:
+		for i := 0; i+k <= len(s); i++ {
+			w := s[i : i+k]
+			for _, b := range w {
+				if !unambiguousBase[b] {
+					continue window
+				}
+			}
+			kmer := EncodeCanonicalKmer(w, k)
+			info := res[kmer]
+			if info == nil {
+				info = &bruteInfo{seqs: map[uint32]int{}}
+				res[kmer] = info
+			}
+			info.seqs[uint32(id)]++
+			if info.seqs[uint32(id)] > 1 {
+				info.repeated = true
+			}
+		}
+	}
+	return res
+}
+
+func randomSeqs(rng *rand.Rand, n, minLen, maxLen int, withN bool) [][]byte {
+	alphabet := "acgtACGT"
+	if withN {
+		alphabet = "acgtACGTacgtACGTNnR"
+	}
+	// A shared motif and its reverse complement make k-mers common to several
+	// sequences on both strands.
+	motif := make([]byte, 40)
+	for i := range motif {
+		motif[i] = "ACGT"[rng.Intn(4)]
+	}
+	rc := make([]byte, len(motif))
+	for i, b := range motif {
+		rc[len(motif)-1-i] = map[byte]byte{'A': 'T', 'C': 'G', 'G': 'C', 'T': 'A'}[b]
+	}
+
+	out := make([][]byte, n)
+	for i := range out {
+		l := minLen + rng.Intn(maxLen-minLen+1)
+		s := make([]byte, l)
+		for j := range s {
+			s[j] = alphabet[rng.Intn(len(alphabet))]
+		}
+		mode := rng.Intn(4)
+		if l < 60 { // too short to host the motifs
+			mode = 3
+		}
+		switch mode {
+		case 0:
+			copy(s[l/4:], motif)
+		case 1:
+			copy(s[l/4:], rc)
+		case 2: // internal repeat, both strands
+			copy(s[2:], motif[:20])
+			copy(s[l-25:], rc[:20])
+		}
+		out[i] = s
+	}
+	return out
+}
+
+func TestSeqKmerCounterAgainstBruteForce(t *testing.T) {
+	cases := []struct {
+		name    string
+		k, m, p int
+		withN   bool
+		circ    bool
+	}{
+		{"k7-linear", 7, 3, 8, false, false},
+		{"k7-circular-N", 7, 3, 5, true, true},
+		{"k18-linear-N", 18, 8, 16, true, false},
+		{"k18-circular", 18, 8, 7, false, true},
+		{"k31-p1", 31, 10, 1, false, true},
+		{"k4-small-m", 4, 1, 3, true, false},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			rng := rand.New(rand.NewSource(int64(tc.k*100 + tc.p)))
+			seqs := randomSeqs(rng, 30, 5, 400, tc.withN)
+			circ := make([]bool, len(seqs))
+			for i := range circ {
+				circ[i] = tc.circ && i%3 != 0
+			}
+
+			c, err := NewSeqKmerCounter(t.TempDir(), tc.k, tc.m, tc.p)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer c.Close()
+
+			// Concurrent insertion, shuffled order, ids not contiguous.
+			var wg sync.WaitGroup
+			for i := range seqs {
+				wg.Add(1)
+				go func() {
+					defer wg.Done()
+					c.AddSequence(uint32(i), seqs[i], circ[i])
+				}()
+			}
+			wg.Wait()
+
+			want := bruteForce(seqs, circ, tc.k)
+			for _, q := range []int{1, 2, 3, 10} {
+				got, err := c.CountAtLeast(q)
+				if err != nil {
+					t.Fatal(err)
+				}
+				if !slices.IsSortedFunc(got, func(a, b SeqKmerCount) int {
+					switch {
+					case a.Kmer < b.Kmer:
+						return -1
+					case a.Kmer > b.Kmer:
+						return 1
+					}
+					return 0
+				}) {
+					t.Fatal("result is not sorted by k-mer")
+				}
+
+				n := 0
+				for kmer, info := range want {
+					if len(info.seqs) >= q {
+						n++
+						_ = kmer
+					}
+				}
+				if len(got) != n {
+					t.Fatalf("q=%d: %d k-mers, want %d", q, len(got), n)
+				}
+				for _, g := range got {
+					info := want[g.Kmer]
+					if info == nil {
+						t.Fatalf("q=%d: unexpected k-mer %d", q, g.Kmer)
+					}
+					if int(g.Sequences) != len(info.seqs) || g.Repeated != info.repeated {
+						t.Fatalf("q=%d kmer %d: got (%d,%v) want (%d,%v)",
+							q, g.Kmer, g.Sequences, g.Repeated, len(info.seqs), info.repeated)
+					}
+				}
+			}
+		})
+	}
+}
+
+// A k-mer appearing on both strands of one sequence counts once but is Repeated.
+func TestSeqKmerCounterBothStrandsOneSequence(t *testing.T) {
+	const k = 8
+	fwd := "ACGGTCAT"
+	rc := "ATGACCGT"
+	seq := []byte("TTTT" + fwd + "GGGGGGGG" + rc + "CCCC")
+
+	c, err := NewSeqKmerCounter(t.TempDir(), k, 3, 4)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer c.Close()
+	c.AddSequence(7, seq, false)
+
+	got, err := c.CountAtLeast(1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	target := EncodeCanonicalKmer([]byte(fwd), k)
+	found := false
+	for _, g := range got {
+		if g.Kmer == target {
+			found = true
+			if g.Sequences != 1 || !g.Repeated {
+				t.Errorf("got %+v, want 1 sequence and Repeated", g)
+			}
+		}
+	}
+	if !found {
+		t.Fatal("k-mer not found")
+	}
+}
+
+// Ambiguous bases must never be part of a k-mer, and the circular junction
+// must produce the wrap-around k-mers.
+func TestSeqKmerCounterAmbiguityAndCircular(t *testing.T) {
+	const k = 5
+	c, err := NewSeqKmerCounter(t.TempDir(), k, 2, 3)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer c.Close()
+
+	c.AddSequence(0, []byte("ACGTNACGT"), false) // runs ACGT (<k) and ACGT (<k): nothing
+	c.AddSequence(1, []byte("AACCGGTT"), true)   // 8 circular windows
+
+	got, err := c.CountAtLeast(1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) == 0 {
+		t.Fatal("expected k-mers from the circular sequence")
+	}
+	var circ []string
+	for i := 0; i < 8; i++ {
+		var sb strings.Builder
+		for j := 0; j < k; j++ {
+			sb.WriteByte("AACCGGTT"[(i+j)%8])
+		}
+		circ = append(circ, sb.String())
+	}
+	want := map[uint64]bool{}
+	for _, w := range circ {
+		want[EncodeCanonicalKmer([]byte(w), k)] = true
+	}
+	if len(got) != len(want) {
+		t.Fatalf("got %d k-mers, want %d", len(got), len(want))
+	}
+	for _, g := range got {
+		if !want[g.Kmer] || g.Sequences != 1 {
+			t.Errorf("unexpected %+v", g)
+		}
+	}
+}
+
+func TestSeqKmerCounterParameters(t *testing.T) {
+	if _, err := NewSeqKmerCounter(t.TempDir(), 1, 1, 4); err == nil {
+		t.Error("k=1 must fail")
+	}
+	if _, err := NewSeqKmerCounter(t.TempDir(), 32, 5, 4); err == nil {
+		t.Error("k=32 must fail")
+	}
+	if _, err := NewSeqKmerCounter(t.TempDir(), 10, 10, 4); err == nil {
+		t.Error("m>=k must fail")
+	}
+	if _, err := NewSeqKmerCounter(t.TempDir(), 10, 4, 0); err == nil {
+		t.Error("zero partitions must fail")
+	}
+	c, err := NewSeqKmerCounter(t.TempDir(), 10, 0, 4) // default minimizer
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := c.Close(); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.Close(); err != nil {
+		t.Fatalf("second Close must be harmless: %v", err)
+	}
+}

pkg/obikmer/skm_reader.go

@@ -10,6 +10,16 @@ import (
 // decode2bit maps 2-bit codes back to nucleotide bytes.
 var decode2bit = [4]byte{'a', 'c', 'g', 't'}
 
+// unpackBases decodes the 2-bit packed bases in packed into dst (one lowercase
+// nucleotide per byte, len(dst) bases) and returns dst.
+func unpackBases(dst, packed []byte) []byte {
+	for i := range dst {
+		code := (packed[i/4] >> uint(6-(i%4)*2)) & 0x03
+		dst[i] = decode2bit[code]
+	}
+	return dst
+}
+
 // SkmReader reads super-kmers from a binary .skm file.
 type SkmReader struct {
 	r    *bufio.Reader
@@ -46,13 +56,7 @@ func (sr *SkmReader) Next() (SuperKmer, bool) {
 	}
 
 	// Decode to nucleotide bytes
-	seq := make([]byte, seqLen)
+	seq := unpackBases(make([]byte, seqLen), packed)
-	for i := 0; i < seqLen; i++ {
-		byteIdx := i / 4
-		bitPos := uint(6 - (i%4)*2)
-		code := (packed[byteIdx] >> bitPos) & 0x03
-		seq[i] = decode2bit[code]
-	}
 
 	return SuperKmer{
 		Sequence: seq,

pkg/obikmer/skm_writer.go

@@ -18,6 +18,7 @@ import (
 type SkmWriter struct {
 	w       *bufio.Writer
 	file    *os.File
+	scratch []byte
 }
 
 // NewSkmWriter creates a new SkmWriter writing to the given file path.
@@ -32,37 +33,37 @@ func NewSkmWriter(path string) (*SkmWriter, error) {
 	}, nil
 }
 
+// appendPacked appends seq to dst packed 2 bits per base (A=00, C=01, G=10,
+// T=11), four bases per byte, the last byte zero-padded on the low bits.
+func appendPacked(dst, seq []byte) []byte {
+	for i := 0; i < len(seq); i += 4 {
+		var packed byte
+		for j := 0; j < 4; j++ {
+			packed <<= 2
+			if i+j < len(seq) {
+				packed |= __single_base_code__[seq[i+j]&31]
+			}
+		}
+		dst = append(dst, packed)
+	}
+	return dst
+}
+
 // Write encodes a SuperKmer to the .skm file.
 // The sequence bytes are packed 2 bits per base.
 func (sw *SkmWriter) Write(sk SuperKmer) error {
-	seq := sk.Sequence
-	seqLen := uint16(len(seq))
-
 	// Write length
 	var lenbuf [2]byte
-	binary.LittleEndian.PutUint16(lenbuf[:], seqLen)
+	binary.LittleEndian.PutUint16(lenbuf[:], uint16(len(sk.Sequence)))
 	if _, err := sw.w.Write(lenbuf[:]); err != nil {
 		return err
 	}
 
 	// Encode and write packed sequence (2 bits/base)
-	nBytes := (int(seqLen) + 3) / 4
+	sw.scratch = appendPacked(sw.scratch[:0], sk.Sequence)
-	for i := 0; i < nBytes; i++ {
+	_, err := sw.w.Write(sw.scratch)
-		var packed byte
-		for j := 0; j < 4; j++ {
-			pos := i*4 + j
-			packed <<= 2
-			if pos < int(seqLen) {
-				packed |= __single_base_code__[seq[pos]&31]
-			}
-		}
-		if err := sw.w.WriteByte(packed); err != nil {
 	return err
 }
-	}
-
-	return nil
-}
 
 // Close flushes buffered data and closes the underlying file.
 func (sw *SkmWriter) Close() error {

pkg/obingslibrary/multimatch.go

@@ -777,7 +777,7 @@ func (library *NGSLibrary) ExtractMultiBarcodeSliceWorker(options ...WithOption)
 		library.SetAllowsIndels(true)
 	}
 
-	if opt.AllowedMismatches() > 0 {
+	if opt.AllowedMismatchesIsSet() {
 		library.SetAllowedMismatches(opt.AllowedMismatches())
 	}
 

pkg/obingslibrary/worker.go

@@ -9,6 +9,7 @@ type _Options struct {
 	discardErrors      bool
 	unidentified       string
 	allowedMismatch    int
+	allowedMismatchSet bool
 	allowsIndel        bool
 	withProgressBar    bool
 	parallelWorkers    int
@@ -52,6 +53,7 @@ func OptionWithProgressBar(yes bool) WithOption {
 func OptionAllowedMismatches(count int) WithOption {
 	f := WithOption(func(opt Options) {
 		opt.pointer.allowedMismatch = count
+		opt.pointer.allowedMismatchSet = true
 	})
 
 	return f
@@ -97,6 +99,12 @@ func (options Options) AllowedMismatches() int {
 	return options.pointer.allowedMismatch
 }
 
+// AllowedMismatchesIsSet returns true if OptionAllowedMismatches
+// was explicitly applied to these options.
+func (options Options) AllowedMismatchesIsSet() bool {
+	return options.pointer.allowedMismatchSet
+}
+
 func (options Options) AllowsIndels() bool {
 	return options.pointer.allowsIndel
 }

pkg/obioptions/version.go

@@ -3,7 +3,7 @@ package obioptions
 // Version is automatically updated by the Makefile from version.txt
 // The patch number (third digit) is incremented on each push to the repository
 
-var _Version = "Release 4.4.43"
+var _Version = "Release 4.5.2"
 
 // Version returns the version of the obitools package.
 //

pkg/obiprimer/amplicons.go

@@ -0,0 +1,293 @@
+package obiprimer
+
+import (
+	"math"
+	"sync"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obithermo"
+)
+
+// refHit is an amplicon of a retained pair on the reference sequence.
+type refHit struct {
+	up, down seqMatch
+	swapped  bool
+}
+
+// ampHash hashes the amplicon between the two primers read from the side of
+// the first primer of the pair (reverse complemented when the upstream match is
+// the second primer), so that identical amplicons give the same value
+// whatever their strand. Non ACGT bases hash as a fifth symbol.
+func ampHash(raw []byte, from, to int, reverse bool) uint64 {
+	const prime = 1099511628211
+	h := uint64(14695981039346656037) ^ uint64(to-from)
+	h *= prime
+	if !reverse {
+		for _, b := range raw[from:to] {
+			h = (h ^ uint64(baseCode[b])) * prime
+		}
+		return h
+	}
+	for i := to - 1; i >= from; i-- {
+		c := baseCode[raw[i]]
+		if c < 4 {
+			c = 3 - c
+		}
+		h = (h ^ uint64(c)) * prime
+	}
+	return h
+}
+
+// eachAmplicon runs forEachOccurrence over the sequences of the corpus (the
+// examples only, or all of them) in parallel, and calls visit for every
+// amplicon of a pair of index. Every worker gets its own visit function, from
+// newVisit, and done is called once the worker has no more sequence.
+//
+// Without a plan the sequences are handed to the workers as they get free, and
+// the assignment is recorded in record. With one, every worker scans the
+// sequences that the plan gave it.
+func (ps *primerSet) eachAmplicon(c *Corpus, p PairParams, index map[pairKey]int32, examplesOnly bool, plan *ampPlan, record []uint16,
+	newVisit func(worker int) (visit func(s int, raw []byte, o occurrence, pair int32), done func())) {
+
+	workers := obidefault.ParallelWorkers()
+
+	// only the matches of the primers of the pairs of index can make an amplicon
+	// of these pairs, and dropping the others spares most of the comparisons
+	wanted := make([]bool, len(ps.primers))
+	for k := range index {
+		wanted[k.p1], wanted[k.p2] = true, true
+	}
+	type scratch struct {
+		ms    []seqMatch
+		bases []byte
+	}
+	scan := func(sc *scratch, visit func(int, []byte, occurrence, int32), s int) {
+		if examplesOnly && c.Roles[s] != Example {
+			return
+		}
+		ms := sc.ms[:0]
+		for _, m := range ps.perSeq[s] {
+			if wanted[m.primer] {
+				ms = append(ms, m)
+			}
+		}
+		sc.ms = ms
+		if len(ms) < 2 {
+			return // an amplicon needs two primer sites
+		}
+		var raw []byte
+		raw, sc.bases = c.bases(s, sc.bases)
+		ps.forEachOccurrenceIn(raw, ms, p, func(o occurrence) {
+			if pi, ok := index[o.key]; ok {
+				visit(s, raw, o, pi)
+			}
+		})
+	}
+
+	var wg sync.WaitGroup
+	if plan != nil {
+		mine := make([][]int, workers)
+		for s, w := range plan.owner {
+			mine[w] = append(mine[w], s)
+		}
+		for w := range workers {
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				visit, done := newVisit(w)
+				var sc scratch
+				for _, s := range mine[w] {
+					scan(&sc, visit, s)
+				}
+				if done != nil {
+					done()
+				}
+			}()
+		}
+		wg.Wait()
+		return
+	}
+
+	jobs := make(chan int)
+	for w := range workers {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			visit, done := newVisit(w)
+			var sc scratch
+			for s := range jobs {
+				record[s] = uint16(w)
+				scan(&sc, visit, s)
+			}
+			if done != nil {
+				done()
+			}
+		}()
+	}
+	for s := range c.Roles {
+		jobs <- s
+	}
+	close(jobs)
+	wg.Wait()
+}
+
+// planAmplicons counts the amplicons of every pair of index in the sequences
+// of the corpus (all of them, or the examples only), by worker.
+func (ps *primerSet) planAmplicons(c *Corpus, p PairParams, index map[pairKey]int32, examplesOnly bool) *ampPlan {
+	workers := obidefault.ParallelWorkers()
+	plan := &ampPlan{owner: make([]uint16, c.Len()), count: make([][]uint32, workers)}
+	for w := range plan.count {
+		plan.count[w] = make([]uint32, len(index))
+	}
+	ps.eachAmplicon(c, p, index, examplesOnly, nil, plan.owner, func(w int) (func(int, []byte, occurrence, int32), func()) {
+		row := plan.count[w]
+		return func(_ int, _ []byte, _ occurrence, pi int32) { row[pi]++ }, nil
+	})
+	return plan
+}
+
+// ampSites is what the gathering of the amplicons of the targets learns besides
+// the records.
+type ampSites struct {
+	minTm1 []float32 // by pair: lowest Tm of the first primer over the example sites
+	minTm2 []float32
+	refs   map[int32]refHit // by pair: the amplicon on the reference sequence
+}
+
+// collectOptions selects what collectAmplicons gathers besides the records.
+type collectOptions struct {
+	examplesOnly bool             // skip the counterexamples and the other sequences
+	model        *obithermo.Model // melting temperatures of the example sites (nil: none)
+	reference    bool             // record the amplicons on the reference sequence
+}
+
+// collectAmplicons writes into store, in parallel, one record per amplicon of
+// the pairs of index (whose values are the pair numbers in the store) and
+// returns the lowest melting temperatures over the example sites. The store
+// was sized from the plan, which says which worker scans which sequence; the
+// batch only holds the pairs of index, the plan more.
+func (ps *primerSet) collectAmplicons(c *Corpus, taxa *Taxa, p PairParams, index map[pairKey]int32,
+	plan *ampPlan, store *ampStore, opt collectOptions) *ampSites {
+
+	model := opt.model
+	lp := p.PrimerLength
+	npairs := len(index)
+
+	res := &ampSites{
+		minTm1: make([]float32, npairs),
+		minTm2: make([]float32, npairs),
+		refs:   map[int32]refHit{}, // written by the single worker that reads the reference
+	}
+	inf := float32(math.Inf(1))
+	for i := range npairs {
+		res.minTm1[i], res.minTm2[i] = inf, inf
+	}
+	var tmMu sync.Mutex
+
+	// oriented[primer][0] is the reverse complement of the word, [1] the word itself.
+	oriented := make([][2][]byte, len(ps.primers))
+	for i, pr := range ps.primers {
+		oriented[i][0] = []byte(orientedWord(pr.Word, lp, false))
+		oriented[i][1] = []byte(orientedWord(pr.Word, lp, true))
+	}
+
+	ps.eachAmplicon(c, p, index, opt.examplesOnly, plan, nil, func(w int) (func(int, []byte, occurrence, int32), func()) {
+		site1, site2 := make([]byte, lp), make([]byte, lp)
+		lowTm1, lowTm2 := map[int32]float32{}, map[int32]float32{}
+
+		visit := func(s int, raw []byte, o occurrence, pi int32) {
+			n := len(raw)
+			role := c.Roles[s]
+			if opt.reference && p.HasReference && s == p.Reference {
+				if old, seen := res.refs[pi]; !seen || o.up.pos() < old.up.pos() {
+					res.refs[pi] = refHit{o.up, o.down, o.swapped}
+				}
+			}
+			store.add(w, pi, ampHash(raw, int(o.up.pos())+lp, int(o.down.pos()), o.swapped), packTaxon(taxa.Of[s], role))
+
+			if role != Example || model == nil {
+				return
+			}
+			fp := oriented[o.up.primer][b2i(!o.up.reverse())]
+			rp := oriented[o.down.primer][b2i(o.down.reverse())]
+			tmF, tmR := inf, inf
+			if windowAsIs(site1, raw, int(o.up.pos()), n, p.Circular) {
+				if t, err := model.Tm(fp, site1); err == nil && !math.IsNaN(t) {
+					tmF = float32(t)
+				}
+			}
+			if windowRevComp(site2, raw, int(o.down.pos()), n, p.Circular) {
+				if t, err := model.Tm(rp, site2); err == nil && !math.IsNaN(t) {
+					tmR = float32(t)
+				}
+			}
+			if o.swapped {
+				tmF, tmR = tmR, tmF // slot 1 is the downstream primer
+			}
+			lowTm1[pi] = min(lowOr(lowTm1, pi, inf), tmF)
+			lowTm2[pi] = min(lowOr(lowTm2, pi, inf), tmR)
+		}
+		done := func() {
+			tmMu.Lock()
+			for pi, t := range lowTm1 {
+				res.minTm1[pi] = min(res.minTm1[pi], t)
+			}
+			for pi, t := range lowTm2 {
+				res.minTm2[pi] = min(res.minTm2[pi], t)
+			}
+			tmMu.Unlock()
+		}
+		return visit, done
+	})
+	return res
+}
+
+func lowOr(m map[int32]float32, k int32, def float32) float32 {
+	if v, ok := m[k]; ok {
+		return v
+	}
+	return def
+}
+
+func b2i(b bool) int {
+	if b {
+		return 1
+	}
+	return 0
+}
+
+const upper = "ACGT"
+
+// windowAsIs copies the len(dst) bases at pos as upper-case ACGT; it reports
+// false if one of them is not A, C, G or T.
+func windowAsIs(dst, raw []byte, pos, n int, circular bool) bool {
+	for t := range dst {
+		i := pos + t
+		if circular {
+			i %= n
+		}
+		c := baseCode[raw[i]]
+		if c > 3 {
+			return false
+		}
+		dst[t] = upper[c]
+	}
+	return true
+}
+
+// windowRevComp writes the reverse complement of the len(dst) bases at pos.
+func windowRevComp(dst, raw []byte, pos, n int, circular bool) bool {
+	l := len(dst)
+	for t := range dst {
+		i := pos + l - 1 - t
+		if circular {
+			i %= n
+		}
+		c := baseCode[raw[i]]
+		if c > 3 {
+			return false
+		}
+		dst[t] = upper[3-c]
+	}
+	return true
+}

pkg/obiprimer/ampstore.go

@@ -0,0 +1,132 @@
+package obiprimer
+
+import "fmt"
+
+// ampRecord is the logical content of the record of one amplicon: its barcode
+// (see ampHash), the taxon of its sequence at the chosen rank (-1 when it has
+// none) and the role of the sequence. It is stored packed in an ampStore.
+type ampRecord struct {
+	hash  uint64
+	taxon int32
+	role  Role
+}
+
+const (
+	roleBits = 2
+	roleMask = 1<<roleBits - 1
+
+	// maxTaxa is the number of taxa a record can carry: the taxon, shifted by
+	// one so that "none" is 0, shares a 32 bits word with the role.
+	maxTaxa = 1<<(32-roleBits) - 2
+)
+
+func packTaxon(taxon int32, role Role) uint32 { return uint32(taxon+1)<<roleBits | uint32(role) }
+
+func unpackTaxon(t uint32) (int32, Role) { return int32(t>>roleBits) - 1, Role(t & roleMask) }
+
+// ampSlice is the amplicons of one pair: 12 bytes per amplicon, kept as two
+// parallel arrays (a struct holding a uint64 and a uint32 would take 16).
+type ampSlice struct {
+	hash []uint64
+	tax  []uint32 // packTaxon
+}
+
+// ampPlan says how many amplicons each worker finds for each pair, and which
+// worker scans which sequence: the write pass replays the counting pass with
+// the very same assignment, so that every worker knows where to write its
+// records without sharing any counter with the others.
+type ampPlan struct {
+	owner []uint16   // owner[s]: the worker that scans sequence s
+	count [][]uint32 // count[w][pair]: amplicons of the pair in the sequences of worker w
+}
+
+// total returns the number of amplicons of every pair.
+func (pl *ampPlan) total() []int {
+	t := make([]int, len(pl.count[0]))
+	for _, row := range pl.count {
+		for b, n := range row {
+			t[b] += int(n)
+		}
+	}
+	return t
+}
+
+// ampStore holds the amplicons of a batch of pairs, grouped by pair: for each
+// pair, two slices of the exact size, since their number is known before they
+// are written (see planAmplicons). Nothing grows by doubling, nothing is copied
+// nor sorted afterwards. The records of a pair are those of worker 0, then those
+// of worker 1, and so on.
+//
+// The pairs have slices of their own, rather than slices of two huge arrays: a
+// few kB each find room in the free memory, where blocks of tens of MB make the
+// process map new memory (and, on some systems, count the freed one still).
+type ampStore struct {
+	plan   *ampPlan
+	lo     int // first pair of the batch in the plan
+	pairs  []ampSlice
+	cursor [][]int // cursor[w][b]: next free slot of worker w for pair b
+}
+
+// newAmpStore allocates the store of the pairs lo..hi-1 of the plan.
+func newAmpStore(plan *ampPlan, lo, hi int) *ampStore {
+	n := hi - lo
+	s := &ampStore{plan: plan, lo: lo, pairs: make([]ampSlice, n), cursor: make([][]int, len(plan.count))}
+	for w := range s.cursor {
+		s.cursor[w] = make([]int, n)
+	}
+	for b := range n {
+		pos := 0
+		for w, row := range plan.count {
+			s.cursor[w][b] = pos
+			pos += int(row[lo+b])
+		}
+		if pos > 0 {
+			s.pairs[b] = ampSlice{make([]uint64, pos), make([]uint32, pos)}
+		}
+	}
+	return s
+}
+
+// add writes one record of pair b (numbered in the batch), found by worker w.
+func (s *ampStore) add(w int, b int32, hash uint64, taxon uint32) {
+	i := s.cursor[w][b]
+	s.cursor[w][b] = i + 1
+	s.pairs[b].hash[i], s.pairs[b].tax[i] = hash, taxon
+}
+
+// check panics if a worker did not write exactly the records counted for it:
+// the counting and write passes must enumerate the same amplicons.
+func (s *ampStore) check() {
+	for b := range s.pairs {
+		pos := 0
+		for w, row := range s.plan.count {
+			pos += int(row[s.lo+b])
+			if s.cursor[w][b] != pos {
+				panic(fmt.Sprintf("obiprimer: pair %d: worker %d wrote %d amplicons, %d were counted",
+					s.lo+b, w, s.cursor[w][b]-(pos-int(row[s.lo+b])), row[s.lo+b]))
+			}
+		}
+	}
+}
+
+// pair returns the amplicons of pair b (numbered in the batch).
+func (s *ampStore) pair(b int) ampSlice { return s.pairs[b] }
+
+// ampBatches cuts the pairs 0..len(count)-1 in consecutive ranges [lo,hi)
+// whose amplicons together do not exceed budget records (a pair alone above the
+// budget makes a batch of its own). A non positive budget gives one batch.
+func ampBatches(count []int, budget int) [][2]int {
+	var out [][2]int
+	lo, sum := 0, 0
+	for i, n := range count {
+		if budget > 0 && i > lo && sum+n > budget {
+			out = append(out, [2]int{lo, i})
+			lo, sum = i, 0
+		}
+		sum += n
+	}
+	if lo < len(count) {
+		out = append(out, [2]int{lo, len(count)})
+	}
+	return out
+}

pkg/obiprimer/corpus.go

@@ -0,0 +1,113 @@
+package obiprimer
+
+import (
+	"fmt"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+// maxSeqLen bounds the length of a sequence: the position of a primer site and
+// its strand share 32 bits (see seqMatch).
+const maxSeqLen = 1 << 31
+
+// Counts gives the number of sequences by role.
+type Counts struct {
+	Examples        int
+	CounterExamples int
+	Others          int
+}
+
+// Corpus is the set of sequences of the primer search, in input order, each
+// with its role: the examples (the targets), the counterexamples, and all the
+// other sequences, which are loaded as well since the amplicons they give
+// decide whether a target is identified.
+//
+// The primer search needs several passes over the sequences (words are
+// searched approximately in every one of them), so the corpus lives in memory.
+type Corpus struct {
+	Seqs  []*obiseq.BioSequence // nil entries once the corpus is packed
+	Roles []Role                // Roles[i] is the role of Seqs[i]
+
+	packed []packedSeq // see Pack
+	ids    []string
+}
+
+// Counts returns the number of sequences by role.
+func (c *Corpus) Counts() Counts {
+	var n Counts
+	for _, r := range c.Roles {
+		switch r {
+		case Example:
+			n.Examples++
+		case CounterExample:
+			n.CounterExamples++
+		default:
+			n.Others++
+		}
+	}
+	return n
+}
+
+// IndexOfExample returns the index of the example sequence with the given
+// identifier. It is an error if there is none, or if the sequence is a
+// counterexample.
+func (c *Corpus) IndexOfExample(id string) (int, error) {
+	for i := range c.Roles {
+		if c.ID(i) == id {
+			if c.Roles[i] != Example {
+				return -1, fmt.Errorf("obiprimer: reference sequence %q is a counterexample, it must be an example", id)
+			}
+			return i, nil
+		}
+	}
+	return -1, fmt.Errorf("obiprimer: reference sequence %q is not among the selected example sequences", id)
+}
+
+// Len returns the number of sequences.
+func (c *Corpus) Len() int { return len(c.Roles) }
+
+// SeqLen returns the length of sequence i.
+func (c *Corpus) SeqLen(i int) int {
+	if c.Seqs[i] != nil {
+		return c.Seqs[i].Len()
+	}
+	return c.packed[i].n
+}
+
+// ID returns the identifier of sequence i.
+func (c *Corpus) ID(i int) string {
+	if c.Seqs[i] != nil {
+		return c.Seqs[i].Id()
+	}
+	return c.ids[i]
+}
+
+// bases returns the bases of sequence i, lower case. A packed sequence is
+// unpacked into buf, grown if need be, which the caller keeps for the next call
+// (the second result); an unpacked one is returned as it is. The result must
+// not be modified, and is valid until the next call with the same buffer.
+func (c *Corpus) bases(i int, buf []byte) (raw, newBuf []byte) {
+	if c.Seqs[i] != nil {
+		return c.Seqs[i].Sequence(), buf
+	}
+	buf = c.packed[i].unpack(buf)
+	return buf, buf
+}
+
+// Pack replaces the sequences by their packed form, on 2 bits per base, and
+// frees the sequence objects (their annotations, their slack): the primer search
+// only needs the bases, the identifiers, and the taxa, which must have been
+// assigned before (AssignTaxa). A packed corpus holds about a quarter of the
+// memory of the bases themselves.
+func (c *Corpus) Pack() {
+	if c.packed != nil {
+		return
+	}
+	c.packed = make([]packedSeq, len(c.Seqs))
+	c.ids = make([]string, len(c.Seqs))
+	parallelFor(len(c.Seqs), func(_, i int) {
+		c.packed[i] = packBases(c.Seqs[i].Sequence())
+		c.ids[i] = c.Seqs[i].Id()
+		c.Seqs[i] = nil
+	})
+}

pkg/obiprimer/corpus_load.go

@@ -0,0 +1,137 @@
+package obiprimer
+
+import (
+	"cmp"
+	"fmt"
+	"slices"
+	"sync"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax"
+)
+
+// corpusEntry is a sequence read by one of the loading goroutines: either the
+// sequence itself, or, when it is packed as it is read, what is kept of it.
+type corpusEntry struct {
+	batch, index int
+	role         Role
+
+	seq *obiseq.BioSequence // unpacked loading
+
+	packed packedSeq // packed loading
+	id     string
+	taxon  *obitax.Taxon // at the chosen rank, nil when there is none
+}
+
+// loadEntries reads every sequence of seqs, once, on all the cores, and returns
+// the entries in the order of the input, whatever the number of reader
+// goroutines. keep fills the entry of a sequence (besides its position and its
+// role); it runs in the reading goroutines.
+func loadEntries(seqs obiiter.IBioSequence, sel Selection, keep func(*obiseq.BioSequence, *corpusEntry)) []corpusEntry {
+	var mu sync.Mutex
+	var entries []corpusEntry
+
+	consumer := func(it obiiter.IBioSequence, wg *sync.WaitGroup) {
+		defer wg.Done()
+		var local []corpusEntry
+		for it.Next() {
+			batch := it.Get()
+			for i, seq := range batch.Slice() {
+				e := corpusEntry{batch: batch.Order(), index: i, role: sel.Role(seq)}
+				keep(seq, &e)
+				local = append(local, e)
+			}
+		}
+		mu.Lock()
+		entries = append(entries, local...)
+		mu.Unlock()
+	}
+
+	var wg sync.WaitGroup
+	for i := 1; i < obidefault.ParallelWorkers(); i++ {
+		wg.Add(1)
+		go consumer(seqs.Split(), &wg)
+	}
+	wg.Add(1)
+	go consumer(seqs, &wg)
+	wg.Wait()
+
+	slices.SortFunc(entries, func(a, b corpusEntry) int {
+		if c := cmp.Compare(a.batch, b.batch); c != 0 {
+			return c
+		}
+		return cmp.Compare(a.index, b.index)
+	})
+	return entries
+}
+
+func checkSeqLen(id string, n int) error {
+	if n >= maxSeqLen {
+		return fmt.Errorf("obiprimer: sequence %q is too long (%d bases, the limit is %d)", id, n, maxSeqLen-1)
+	}
+	return nil
+}
+
+// LoadCorpus reads every sequence of seqs, once, and keeps all of them with the
+// role given by the selection. The order of the input is preserved whatever
+// the number of reader goroutines.
+func LoadCorpus(seqs obiiter.IBioSequence, sel Selection) (*Corpus, error) {
+	entries := loadEntries(seqs, sel, func(seq *obiseq.BioSequence, e *corpusEntry) { e.seq = seq })
+
+	c := &Corpus{
+		Seqs:  make([]*obiseq.BioSequence, len(entries)),
+		Roles: make([]Role, len(entries)),
+	}
+	for i, e := range entries {
+		if err := checkSeqLen(e.seq.Id(), e.seq.Len()); err != nil {
+			return nil, err
+		}
+		c.Seqs[i], c.Roles[i] = e.seq, e.role
+	}
+	if c.Counts().Examples == 0 {
+		return nil, fmt.Errorf("obiprimer: no example sequence selected")
+	}
+	return c, nil
+}
+
+// LoadPackedCorpus is LoadCorpus followed by Pack and AssignTaxa, without ever
+// holding the sequences themselves: each one is packed, and its taxon at rank
+// looked up, as it is read, and then dropped. The corpus of a large database
+// would otherwise first take, in memory, the sequences with all their
+// annotations, four times the size of their bases and more.
+func LoadPackedCorpus(seqs obiiter.IBioSequence, sel Selection, taxonomy *obitax.Taxonomy, rank string) (*Corpus, *Taxa, error) {
+	if err := checkRank(taxonomy, rank); err != nil {
+		return nil, nil, err
+	}
+	entries := loadEntries(seqs, sel, func(seq *obiseq.BioSequence, e *corpusEntry) {
+		e.packed = packBases(seq.Sequence())
+		e.id = seq.Id()
+		e.taxon = seq.Taxon(taxonomy).TaxonAtRank(rank)
+	})
+
+	c := &Corpus{
+		Seqs:   make([]*obiseq.BioSequence, len(entries)),
+		Roles:  make([]Role, len(entries)),
+		packed: make([]packedSeq, len(entries)),
+		ids:    make([]string, len(entries)),
+	}
+	taxons := make([]*obitax.Taxon, len(entries))
+	for i := range entries {
+		e := &entries[i]
+		if err := checkSeqLen(e.id, e.packed.n); err != nil {
+			return nil, nil, err
+		}
+		c.Roles[i], c.packed[i], c.ids[i], taxons[i] = e.role, e.packed, e.id, e.taxon
+		*e = corpusEntry{} // frees the entry as it goes
+	}
+	if c.Counts().Examples == 0 {
+		return nil, nil, fmt.Errorf("obiprimer: no example sequence selected")
+	}
+	taxa, err := buildTaxa(rank, taxons, c.Roles)
+	if err != nil {
+		return nil, nil, err
+	}
+	return c, taxa, nil
+}

pkg/obiprimer/match.go

@@ -0,0 +1,180 @@
+package obiprimer
+
+import (
+	"cmp"
+	"fmt"
+	"slices"
+	"sync"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obikmer"
+)
+
+// Match is one approximate occurrence of a word in a corpus sequence.
+//
+// Pos is the 0-based start of the window on the sequence as given (forward
+// coordinates). When Reverse is false the window reads like the word; when it
+// is true the window reads like the reverse complement of the word, i.e. the
+// word occurs on the other strand. Errors is the number of substitutions.
+type Match struct {
+	Seq     uint32 // index in the Corpus
+	Pos     uint32
+	Errors  uint8
+	Reverse bool
+}
+
+// WordMatches gathers every occurrence of one canonical word, sorted by
+// (Seq, Pos, Reverse).
+type WordMatches struct {
+	Word    uint64
+	Matches []Match
+}
+
+// MatchParams parameterizes the approximate search.
+//
+// Rules, those of ecoPrimers: only substitutions are allowed; a base other
+// than A, C, G, T (or U, read as T) in the sequence never matches, it counts as
+// one error; every window with at most MaxErrors errors is reported, windows
+// overlapping each other included; on circular sequences windows may span the
+// junction.
+type MatchParams struct {
+	K         int
+	MaxErrors int
+	Circular  bool
+}
+
+func (p MatchParams) validate() error {
+	if p.K < 2 || p.K > 31 {
+		return fmt.Errorf("obiprimer: word size must be between 2 and 31, got %d", p.K)
+	}
+	if p.MaxErrors < 0 || p.MaxErrors >= p.K {
+		return fmt.Errorf("obiprimer: error count must be in [0,%d[, got %d", p.K, p.MaxErrors)
+	}
+	return nil
+}
+
+// hit is a Match before its sequence is known: the sequence it belongs to is
+// the one being scanned.
+type hit struct {
+	word    uint32
+	pos     uint32
+	errors  uint8
+	reverse bool
+}
+
+// scanFunc finds every occurrence of every word, on both strands, in one
+// sequence (its bases) and reports them through emit.
+type scanFunc func(raw []byte, emit func(word, pos, errors int, reverse bool))
+
+// scannerMaker returns an independent scanFunc; each worker gets its own so
+// that scanners can keep private buffers.
+type scannerMaker func() scanFunc
+
+// wordStrings returns, for each canonical word, the word and its reverse
+// complement as lowercase strings.
+func wordStrings(words []uint64, k int) (fwd, rev []string) {
+	fwd = make([]string, len(words))
+	rev = make([]string, len(words))
+	for i, w := range words {
+		fwd[i] = string(obikmer.DecodeKmer(w, k, nil))
+		rev[i] = string(obikmer.DecodeKmer(obikmer.ReverseComplement(w, k), k, nil))
+	}
+	return
+}
+
+// matchCorpus runs the scanners over every sequence of the corpus in parallel
+// and assembles the per-word results. The output does not depend on the number
+// of workers.
+//
+// The occurrences of a sequence are kept in a slice of their exact size, then
+// counted per word, so that the occurrences of all the words are laid out in
+// one allocation of the exact size too: nothing grows by doubling, nor is held
+// twice for long.
+func matchCorpus(c *Corpus, words []uint64, mk scannerMaker) []WordMatches {
+	perSeq := make([][]hit, c.Len())
+
+	jobs := make(chan int)
+	var wg sync.WaitGroup
+	for range obidefault.ParallelWorkers() {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			scan := mk()
+			var buf []hit
+			var raw, bases []byte
+			for i := range jobs {
+				buf = buf[:0]
+				raw, bases = c.bases(i, bases)
+				scan(raw, func(word, pos, errors int, reverse bool) {
+					buf = append(buf, hit{uint32(word), uint32(pos), uint8(errors), reverse})
+				})
+				if len(buf) > 0 {
+					perSeq[i] = slices.Clone(buf)
+				}
+			}
+		}()
+	}
+	for i := range c.Roles {
+		jobs <- i
+	}
+	close(jobs)
+	wg.Wait()
+
+	res := make([]WordMatches, len(words))
+	count := make([]int, len(words))
+	for _, hits := range perSeq {
+		for _, h := range hits {
+			count[h.word]++
+		}
+	}
+	for i, w := range words {
+		res[i].Word = w
+		if count[i] > 0 {
+			res[i].Matches = make([]Match, 0, count[i])
+		}
+	}
+	for s, hits := range perSeq {
+		for _, h := range hits {
+			res[h.word].Matches = append(res[h.word].Matches, Match{uint32(s), h.pos, h.errors, h.reverse})
+		}
+		perSeq[s] = nil // the hits of the sequence are no longer needed
+	}
+	parallelFor(len(res), func(_, i int) {
+		slices.SortFunc(res[i].Matches, func(a, b Match) int {
+			if c := cmp.Compare(a.Seq, b.Seq); c != 0 {
+				return c
+			}
+			if c := cmp.Compare(a.Pos, b.Pos); c != 0 {
+				return c
+			}
+			if a.Reverse != b.Reverse {
+				if !a.Reverse {
+					return -1
+				}
+				return 1
+			}
+			return 0
+		})
+	})
+	return res
+}
+
+// SequencesMatched returns, for the matches of one word, the number of
+// distinct example and counterexample sequences it occurs in. The other
+// sequences are not counted.
+func (w WordMatches) SequencesMatched(c *Corpus) (examples, counterExamples int) {
+	last := ^uint32(0)
+	for _, m := range w.Matches {
+		if m.Seq == last {
+			continue
+		}
+		last = m.Seq
+		switch c.Roles[m.Seq] {
+		case Example:
+			examples++
+		case CounterExample:
+			counterExamples++
+		}
+	}
+	return
+}

pkg/obiprimer/match_naive.go

@@ -0,0 +1,75 @@
+package obiprimer
+
+// baseCode maps a nucleotide to 0..3 (A,C,G,T; U is T) and everything else to 4,
+// which never equals a word base.
+var baseCode = func() (t [256]byte) {
+	for i := range t {
+		t[i] = 4
+	}
+	for i, c := range "ACGT" {
+		t[c], t[c+'a'-'A'] = byte(i), byte(i)
+	}
+	t['U'], t['u'] = 3, 3
+	return
+}()
+
+func codesOf(s string) []byte {
+	out := make([]byte, len(s))
+	for i := range out {
+		out[i] = baseCode[s[i]]
+	}
+	return out
+}
+
+// matchNaive is the reference implementation: every window of every sequence
+// is compared with every word, base by base. It is slow and exists to define
+// the exact semantics (see MatchParams) that faster implementations must
+// reproduce.
+func matchNaive(c *Corpus, words []uint64, p MatchParams) ([]WordMatches, error) {
+	if err := p.validate(); err != nil {
+		return nil, err
+	}
+	fwd, rev := wordStrings(words, p.K)
+	fc, rc := make([][]byte, len(words)), make([][]byte, len(words))
+	for i := range words {
+		fc[i], rc[i] = codesOf(fwd[i]), codesOf(rev[i])
+	}
+
+	mk := func() scanFunc {
+		var codes []byte
+		return func(raw []byte, emit func(word, pos, errors int, reverse bool)) {
+			n := len(raw)
+			if n < p.K {
+				return
+			}
+			codes = codes[:0]
+			for _, b := range raw {
+				codes = append(codes, baseCode[b])
+			}
+			windows := n - p.K + 1
+			if p.Circular {
+				codes = append(codes, codes[:p.K-1]...)
+				windows = n
+			}
+			for pos := range windows {
+				win := codes[pos : pos+p.K]
+				for w := range words {
+					for strand, pat := range [2][]byte{fc[w], rc[w]} {
+						errs := 0
+						for j, b := range pat {
+							if win[j] != b {
+								if errs++; errs > p.MaxErrors {
+									break
+								}
+							}
+						}
+						if errs <= p.MaxErrors {
+							emit(w, pos, errs, strand == 1)
+						}
+					}
+				}
+			}
+		}
+	}
+	return matchCorpus(c, words, mk), nil
+}

pkg/obiprimer/match_seed.go

@@ -0,0 +1,211 @@
+package obiprimer
+
+import (
+	"math/bits"
+	"slices"
+)
+
+// seedIndex maps the value of one half of a window to the patterns that are
+// candidates for it: those whose same half is within d substitutions of it.
+//
+// Pigeonhole: a window with at most e = 2d+1 errors against a pattern has at
+// most d errors in at least one of the two halves of the pattern, so the
+// windows to verify are exactly those found through one of the two indexes.
+// Indexing the half of every pattern together with its neighbours at distance
+// d (1 + 3L values for d = 1, 1 + 3L + 9L(L-1)/2 for d = 2, L = 9 bases) costs
+// a few tens of entries per pattern, and a window of a random text finds about
+// patterns x neighbours / 4^L candidates, far fewer than the patterns.
+type seedIndex struct {
+	shift   uint     // slot = seed >> shift when hashed, else seed
+	hashed  bool     // the seed space is too large for a direct table
+	offsets []uint32 // CSR: ids[offsets[s]:offsets[s+1]] are the candidates of slot s
+	ids     []int32  // pattern ids (2*word + strand)
+}
+
+const maxDirectSeedBits = 22
+
+func (x *seedIndex) slot(seed uint64) uint32 {
+	if !x.hashed {
+		return uint32(seed)
+	}
+	return uint32((seed * 0x9E3779B97F4A7C15) >> x.shift)
+}
+
+func (x *seedIndex) candidates(seed uint64) []int32 {
+	s := x.slot(seed)
+	return x.ids[x.offsets[s]:x.offsets[s+1]]
+}
+
+// neighbours calls fn for v and every value within d substitutions of it (each
+// once), v being L bases packed on 2 bits.
+func neighbours(v uint64, L, d int, fn func(uint64)) {
+	var rec func(v uint64, from, left int)
+	rec = func(v uint64, from, left int) {
+		fn(v)
+		if left == 0 {
+			return
+		}
+		for pos := from; pos < L; pos++ {
+			for alt := uint64(1); alt <= 3; alt++ {
+				rec(v^(alt<<(2*uint(pos))), pos+1, left-1)
+			}
+		}
+	}
+	rec(v, 0, d)
+}
+
+// newSeedIndex indexes, for every pattern, the half of the given width (in
+// bases) selected by (code >> shiftBases*2) & mask, with its neighbours at
+// distance d.
+func newSeedIndex(patterns []uint64, shiftBases, width, d int) *seedIndex {
+	x := &seedIndex{}
+	bitsNeeded := 2 * width
+	if bitsNeeded > maxDirectSeedBits {
+		x.hashed, bitsNeeded = true, maxDirectSeedBits
+		x.shift = uint(64 - maxDirectSeedBits)
+	}
+	nslots := 1 << uint(bitsNeeded)
+	halfMask := uint64(1)<<(2*uint(width)) - 1
+
+	type entry struct {
+		slot uint32
+		id   int32
+	}
+	var entries []entry
+	var scratch []uint32
+	for id, pat := range patterns {
+		half := (pat >> (2 * uint(shiftBases))) & halfMask
+		scratch = scratch[:0]
+		neighbours(half, width, d, func(v uint64) { scratch = append(scratch, x.slot(v)) })
+		slices.Sort(scratch)
+		scratch = slices.Compact(scratch) // hashing may merge neighbours
+		for _, s := range scratch {
+			entries = append(entries, entry{s, int32(id)})
+		}
+	}
+
+	x.offsets = make([]uint32, nslots+1)
+	for _, e := range entries {
+		x.offsets[e.slot+1]++
+	}
+	for i := 1; i <= nslots; i++ {
+		x.offsets[i] += x.offsets[i-1]
+	}
+	x.ids = make([]int32, len(entries))
+	fill := slices.Clone(x.offsets[:nslots])
+	for _, e := range entries {
+		x.ids[fill[e.slot]] = e.id
+		fill[e.slot]++
+	}
+	return x
+}
+
+// MatchWords finds every approximate occurrence, on both strands, of the given
+// canonical words in the sequences of the corpus, with the rules of MatchParams
+// (substitutions only, ambiguous bases count as errors, circular junction).
+//
+// It uses the pigeonhole seed indexes above. It returns exactly what the
+// reference matchNaive returns, and what ecoPrimers' fuzzy search finds, in a
+// small fraction of the time: on the Teleostei tutorial data (255 words, 3909
+// sequences, 65 Mb, e = 3) 0.42 s on one core, against 113 s for the
+// obiapat (agrep) search it replaces.
+func MatchWords(c *Corpus, words []uint64, p MatchParams) ([]WordMatches, error) {
+	if err := p.validate(); err != nil {
+		return nil, err
+	}
+	k := p.K
+	d := p.MaxErrors / 2
+
+	// pattern 2w is the word, 2w+1 its reverse complement
+	patterns := make([]uint64, 2*len(words))
+	for i, w := range words {
+		patterns[2*i] = codeOf(wordCodes(w, k, false), k)
+		patterns[2*i+1] = codeOf(wordCodes(w, k, true), k)
+	}
+
+	right := k / 2 // the right half holds the low bits
+	left := k - right
+	leftIdx := newSeedIndex(patterns, right, left, d)
+	rightIdx := newSeedIndex(patterns, 0, right, d)
+
+	rightMask := uint64(1)<<(2*uint(right)) - 1
+	windowMask := uint64(1)<<(2*uint(k)) - 1
+	lowBits := uint64(0x5555555555555555) & windowMask
+	maxErr := p.MaxErrors
+
+	mk := func() scanFunc {
+		return func(raw []byte, emit func(word, pos, errors int, reverse bool)) {
+			n := len(raw)
+			if n < k {
+				return
+			}
+			total := n // number of bases fed to the rolling window
+			windows := n - k + 1
+			if p.Circular {
+				total, windows = n+k-1, n
+			}
+
+			var code, bad uint64 // window, and its non-ACGT positions (bit 2j)
+			verify := func(pos int, id int32) {
+				pat := patterns[id]
+				x := code ^ pat
+				if errs := bits.OnesCount64(((x | x>>1) & lowBits) | bad); errs <= maxErr {
+					emit(int(id)>>1, pos, errs, id&1 == 1)
+				}
+			}
+
+			for i := 0; i < total; i++ {
+				j := i
+				if j >= n {
+					j -= n
+				}
+				v := baseCode[raw[j]]
+				isBad := uint64(0)
+				if v > 3 {
+					v, isBad = 0, 1
+				}
+				code = (code<<2 | uint64(v)) & windowMask
+				bad = (bad<<2 | isBad) & windowMask
+
+				if i < k-1 {
+					continue
+				}
+				pos := i - k + 1
+				if pos >= windows {
+					break
+				}
+				leftCands := leftIdx.candidates(code >> (2 * uint(right)))
+				for _, id := range leftCands {
+					verify(pos, id)
+				}
+				for _, id := range rightIdx.candidates(code & rightMask) {
+					// ids are sorted: skip the ones the left index has just verified
+					if _, done := slices.BinarySearch(leftCands, id); !done {
+						verify(pos, id)
+					}
+				}
+			}
+		}
+	}
+	return matchCorpus(c, words, mk), nil
+}
+
+// wordCodes returns the 2-bit codes (A=0..T=3) of the word or of its reverse
+// complement, 5' to 3'.
+func wordCodes(word uint64, k int, reverse bool) []byte {
+	fwd, rev := wordStrings([]uint64{word}, k)
+	s := fwd[0]
+	if reverse {
+		s = rev[0]
+	}
+	return codesOf(s)
+}
+
+// codeOf packs codes (each 0..3) on 2 bits, first base in the high bits.
+func codeOf(codes []byte, k int) uint64 {
+	var v uint64
+	for _, c := range codes[:k] {
+		v = v<<2 | uint64(c)
+	}
+	return v
+}

pkg/obiprimer/match_test.go

@@ -0,0 +1,304 @@
+package obiprimer
+
+import (
+	"bufio"
+	"fmt"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"reflect"
+	"runtime"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiformats"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obikmer"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+func corpusOf(roles []Role, seqs ...string) *Corpus {
+	c := &Corpus{Roles: roles}
+	for i, s := range seqs {
+		c.Seqs = append(c.Seqs, obiseq.NewBioSequence(fmt.Sprintf("s%d", i), []byte(s), ""))
+	}
+	return c
+}
+
+func canon(word string) uint64 { return obikmer.EncodeCanonicalKmer([]byte(word), len(word)) }
+
+// The reference implementation is checked by hand first: everything else is
+// compared with it.
+func TestNaiveMatchSemantics(t *testing.T) {
+	const word = "AACCGGTA" // not a palindrome; reverse complement TACCGGTT
+	cases := []struct {
+		name     string
+		seq      string
+		circular bool
+		e        int
+		want     []Match
+	}{
+		{"exact forward", "ggaaccggtacc", false, 0, []Match{{0, 2, 0, false}}},
+		{"exact reverse strand", "ggtaccggttcc", false, 0, []Match{{0, 2, 0, true}}},
+		{"one error is not found at e=0", "ggaaccgctacc", false, 0, nil},
+		{"one error found at e=1 with its count", "ggaaccgctacc", false, 1, []Match{{0, 2, 1, false}}},
+		{"an N is an error", "ggaaccgntacc", false, 1, []Match{{0, 2, 1, false}}},
+		{"an N is never a match at e=0", "ggaaccgntacc", false, 0, nil},
+		{"an IUPAC code is an error, not a wildcard", "ggaaccgrtacc", false, 0, nil},
+		{"U reads as T", "ggaaccgguacc", false, 0, []Match{{0, 2, 0, false}}},
+		{"junction ignored on a linear sequence", "ccggtagggaa", false, 0, nil},
+		{"junction found on a circular sequence", "ccggtagggaa", true, 0, []Match{{0, 9, 0, false}}},
+		{"overlapping windows are all reported", "aaccggtaaccggta", false, 0, []Match{{0, 0, 0, false}, {0, 7, 0, false}}},
+		{"too short", "aaccgg", true, 3, nil},
+	}
+	for _, tc := range cases {
+		c := corpusOf([]Role{Example}, tc.seq)
+		got, err := matchNaive(c, []uint64{canon(word)}, MatchParams{K: len(word), MaxErrors: tc.e, Circular: tc.circular})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !reflect.DeepEqual(got[0].Matches, tc.want) {
+			t.Errorf("%s: got %v, want %v", tc.name, got[0].Matches, tc.want)
+		}
+	}
+}
+
+func TestMatchParamsValidation(t *testing.T) {
+	c := corpusOf([]Role{Example}, "acgtacgtacgtacgt")
+	for _, p := range []MatchParams{{K: 1}, {K: 32}, {K: 8, MaxErrors: -1}, {K: 8, MaxErrors: 8}} {
+		if _, err := matchNaive(c, nil, p); err == nil {
+			t.Errorf("%+v must be rejected", p)
+		}
+		if _, err := MatchWords(c, nil, p); err == nil {
+			t.Errorf("%+v must be rejected by MatchWords", p)
+		}
+	}
+}
+
+func randSeq(rng *rand.Rand, n int, alphabet string) []byte {
+	b := make([]byte, n)
+	for i := range b {
+		b[i] = alphabet[rng.Intn(len(alphabet))]
+	}
+	return b
+}
+
+func mutate(rng *rand.Rand, s []byte, n int) []byte {
+	out := append([]byte(nil), s...)
+	for _, i := range rng.Perm(len(out))[:n] {
+		out[i] = "acgt"[rng.Intn(4)]
+	}
+	return out
+}
+
+// randomCorpus builds sequences with words planted on both strands, with 0..3
+// substitutions, across the circular junction, and with ambiguity codes.
+func randomCorpus(rng *rand.Rand, words []string, nseq int) *Corpus {
+	c := &Corpus{}
+	for i := 0; i < nseq; i++ {
+		alphabet := "acgt"
+		if i%3 == 0 {
+			alphabet = "acgtacgtacgtacgtacgtnnrykm"
+		}
+		s := randSeq(rng, 30+rng.Intn(500), alphabet)
+		for range 1 + rng.Intn(4) {
+			w := []byte(words[rng.Intn(len(words))])
+			if rng.Intn(2) == 0 {
+				w = []byte(revcomp(string(w)))
+			}
+			w = mutate(rng, []byte(string(w)), rng.Intn(4))
+			for j := range w {
+				w[j] += 'a' - 'A'
+			}
+			at := rng.Intn(len(s) - len(w) + 1)
+			if rng.Intn(5) == 0 { // across the junction
+				s = append(s[:0:0], append(append([]byte(nil), w[len(w)/2:]...), append(s[:len(s)-len(w)/2], w[:len(w)/2]...)...)...)
+				continue
+			}
+			copy(s[at:], w)
+		}
+		c.Seqs = append(c.Seqs, obiseq.NewBioSequence(fmt.Sprintf("r%d", i), s, ""))
+		c.Roles = append(c.Roles, Example)
+	}
+	return c
+}
+
+// Every implementation must return exactly what the reference returns.
+func TestMatchersAgreeWithReference(t *testing.T) {
+	for _, tc := range []struct {
+		k, e     int
+		circular bool
+	}{{18, 0, false}, {18, 1, true}, {18, 3, false}, {18, 3, true}, {12, 2, true}, {25, 4, false}, {18, 2, true}, {18, 5, false}, {31, 3, true}, {24, 3, false}, {9, 2, true}, {18, 6, true}, {18, 7, false}, {20, 8, true}, {12, 11, false}} {
+		t.Run(fmt.Sprintf("k%d-e%d-circ%v", tc.k, tc.e, tc.circular), func(t *testing.T) {
+			rng := rand.New(rand.NewSource(int64(tc.k*100 + tc.e)))
+			var wordStr []string
+			var words []uint64
+			for range 7 {
+				w := string(randSeq(rng, tc.k, "ACGT"))
+				wordStr = append(wordStr, w)
+				words = append(words, canon(w))
+			}
+			c := randomCorpus(rng, wordStr, 40)
+			p := MatchParams{K: tc.k, MaxErrors: tc.e, Circular: tc.circular}
+
+			want, err := matchNaive(c, words, p)
+			if err != nil {
+				t.Fatal(err)
+			}
+			total := 0
+			for _, w := range want {
+				total += len(w.Matches)
+			}
+			if total < 10 {
+				t.Fatalf("test data too poor: only %d reference matches", total)
+			}
+
+			got, err := MatchWords(c, words, p)
+			if err != nil {
+				t.Fatal(err)
+			}
+			for i := range want {
+				if !reflect.DeepEqual(got[i].Matches, want[i].Matches) {
+					t.Fatalf("word %d: MatchWords gives %v\nreference gives %v", i, got[i].Matches, want[i].Matches)
+				}
+			}
+		})
+	}
+}
+
+func TestSequencesMatched(t *testing.T) {
+	c := corpusOf([]Role{Example, Example, CounterExample, CounterExample}, "a", "a", "a", "a")
+	w := WordMatches{Matches: []Match{{0, 1, 0, false}, {0, 9, 0, true}, {1, 3, 0, false}, {3, 0, 0, false}}}
+	if e, x := w.SequencesMatched(c); e != 2 || x != 1 {
+		t.Errorf("got %d examples, %d counterexamples", e, x)
+	}
+}
+
+func readEcoCounts(t *testing.T, path string) map[string]int {
+	t.Helper()
+	f, err := os.Open(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	counts := map[string]int{}
+	sc := bufio.NewScanner(f)
+	for sc.Scan() {
+		var w string
+		var in, out int
+		if _, err := fmt.Sscanf(sc.Text(), "%s\t%d\t%d", &w, &in, &out); err != nil {
+			t.Fatal(err)
+		}
+		counts[w] = in
+	}
+	return counts
+}
+
+// Opt-in comparison with ecoPrimers on the tutorial data (same variable as the
+// strict word test). For every strict word, ecoPrimers logs the number of
+// example sequences in which the fuzzy search (-e 3, both strands, circular)
+// finds it; testdata/ecoprimers_*.words.tsv holds those numbers.
+func TestMatchWordsAgainstEcoPrimers(t *testing.T) {
+	dir := os.Getenv("OBIPRIMERS_TUTORIAL")
+	if dir == "" {
+		t.Skip("OBIPRIMERS_TUTORIAL not set")
+	}
+	if v := os.Getenv("OBIPRIMERS_WORKERS"); v != "" { // e.g. 1, to time a single core
+		n, err := strconv.Atoi(v)
+		if err != nil {
+			t.Fatal(err)
+		}
+		obidefault.SetMaxCPU(n)
+	}
+	taxonomy, err := obiformats.LoadTaxonomy(filepath.Join(dir, "ncbitaxo_20250211.tgz"), false, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, tc := range []struct{ name, taxid string }{{"Cyprinidae", "7953"}, {"Teleostei", "32443"}} {
+		t.Run(tc.name, func(t *testing.T) {
+			restrict, err := BelongingTo(taxonomy, []string{tc.taxid})
+			if err != nil {
+				t.Fatal(err)
+			}
+			seqs, err := obiformats.ReadSequencesFromFile(filepath.Join(dir, "mito.vert.fasta.gz"))
+			if err != nil {
+				t.Fatal(err)
+			}
+			corpus, err := LoadCorpus(seqs, Selection{Restrict: restrict})
+			if err != nil {
+				t.Fatal(err)
+			}
+			sw, err := FindStrictWords(corpus, WordSearch{PrimerLength: 18, Quorum: 0.7, Circular: true, TmpDir: t.TempDir()})
+			if err != nil {
+				t.Fatal(err)
+			}
+			words := make([]uint64, len(sw.Words))
+			for i, w := range sw.Words {
+				words[i] = w.Kmer
+			}
+
+			var before, after runtime.MemStats
+			runtime.GC()
+			runtime.ReadMemStats(&before)
+			start := time.Now()
+			res, err := MatchWords(corpus, words, MatchParams{K: 18, MaxErrors: 3, Circular: true})
+			elapsed := time.Since(start)
+			runtime.ReadMemStats(&after)
+			if err != nil {
+				t.Fatal(err)
+			}
+			nm := 0
+			for _, r := range res {
+				nm += len(r.Matches)
+			}
+			bases := 0
+			for _, s := range corpus.Seqs {
+				bases += s.Len()
+			}
+			t.Logf("%d words x %d sequences (%.1f Mb) e=3 in %s; %d matches; allocated %.0f MB",
+				len(words), len(corpus.Seqs), float64(bases)/1e6, elapsed.Round(time.Millisecond), nm,
+				float64(after.TotalAlloc-before.TotalAlloc)/1e6)
+
+			// On real sequences (with their ambiguity codes), the first ones must give
+			// exactly what the reference implementation gives.
+			sub := &Corpus{Seqs: corpus.Seqs[:40], Roles: corpus.Roles[:40]}
+			wantSub, err := matchNaive(sub, words, MatchParams{K: 18, MaxErrors: 3, Circular: true})
+			if err != nil {
+				t.Fatal(err)
+			}
+			gotSub, err := MatchWords(sub, words, MatchParams{K: 18, MaxErrors: 3, Circular: true})
+			if err != nil {
+				t.Fatal(err)
+			}
+			subMatches := 0
+			for i := range wantSub {
+				subMatches += len(wantSub[i].Matches)
+				if !reflect.DeepEqual(gotSub[i].Matches, wantSub[i].Matches) {
+					t.Fatalf("word %d differs from the reference on the first 40 sequences", i)
+				}
+			}
+			t.Logf("first 40 sequences identical to the reference (%d matches)", subMatches)
+
+			eco := readEcoCounts(t, filepath.Join("testdata", "ecoprimers_"+tc.name+".words.tsv"))
+			compared, differ := 0, 0
+			for i, r := range res {
+				w := strings.ToUpper(string(obikmer.DecodeKmer(words[i], 18, nil)))
+				want, ok := eco[w]
+				if !ok {
+					t.Errorf("word %s is not in the ecoPrimers log", w)
+					continue
+				}
+				got, _ := r.SequencesMatched(corpus)
+				compared++
+				if got != want {
+					differ++
+					t.Errorf("word %s: %d example sequences, ecoPrimers %d", w, got, want)
+				}
+			}
+			t.Logf("%d words compared with ecoPrimers, %d differ", compared, differ)
+		})
+	}
+}

pkg/obiprimer/multimatch.go

@@ -0,0 +1,42 @@
+package obiprimer
+
+import "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obikmer"
+
+// DropRepeatedWords removes the strict words found more than once (on either
+// strand) in at least one example sequence. It is the first half of the
+// ecoPrimers -U option (no multi-match).
+func DropRepeatedWords(words []obikmer.SeqKmerCount) (kept []obikmer.SeqKmerCount, dropped int) {
+	kept = make([]obikmer.SeqKmerCount, 0, len(words))
+	for _, w := range words {
+		if w.Repeated {
+			dropped++
+			continue
+		}
+		kept = append(kept, w)
+	}
+	return kept, dropped
+}
+
+// DropMultiMatch removes the words having two or more approximate occurrences,
+// both strands counted, in at least one example or counterexample sequence. It
+// is the second half of the ecoPrimers -U option: the primers that can anneal
+// at several places of a sequence they must (or must not) amplify are
+// discarded. The other sequences are not considered.
+func DropMultiMatch(wm []WordMatches, c *Corpus) (kept []WordMatches, dropped int) {
+	kept = make([]WordMatches, 0, len(wm))
+	for _, w := range wm {
+		multi := false
+		for i := 1; i < len(w.Matches); i++ {
+			if w.Matches[i].Seq == w.Matches[i-1].Seq && c.Roles[w.Matches[i].Seq] != Other {
+				multi = true
+				break
+			}
+		}
+		if multi {
+			dropped++
+			continue
+		}
+		kept = append(kept, w)
+	}
+	return kept, dropped
+}

pkg/obiprimer/packed.go

@@ -0,0 +1,80 @@
+package obiprimer
+
+import (
+	"encoding/binary"
+	"slices"
+)
+
+// packedSeq holds the bases of a sequence on 2 bits each, a quarter of the
+// bytes of the sequence itself. The bytes that are not a, c, g or t (N, the
+// IUPAC codes, u...) are kept apart in runs, so that unpacking gives back the
+// sequence byte for byte.
+type packedSeq struct {
+	n    int
+	bits []byte   // 4 bases per byte, the first in the low bits
+	exc  []excRun // sorted by position
+}
+
+// excRun is a run of length identical bytes b that are not a, c, g or t,
+// starting at pos.
+type excRun struct {
+	pos, length uint32
+	b           byte
+}
+
+// packCode maps a, c, g, t to 0..3 and every other byte to 4.
+var packCode = func() (t [256]byte) {
+	for i := range t {
+		t[i] = 4
+	}
+	for i, c := range "acgt" {
+		t[c] = byte(i)
+	}
+	return
+}()
+
+// unpackWord gives, for a byte of 4 packed bases, the 4 bytes they stand for.
+var unpackWord = func() (t [256]uint32) {
+	for b := range t {
+		for k := range 4 {
+			t[b] |= uint32("acgt"[(b>>(2*k))&3]) << (8 * k)
+		}
+	}
+	return
+}()
+
+// packBases packs raw (lower case).
+func packBases(raw []byte) packedSeq {
+	p := packedSeq{n: len(raw), bits: make([]byte, (len(raw)+3)/4)}
+	for i, b := range raw {
+		code := packCode[b]
+		if code > 3 {
+			if k := len(p.exc) - 1; k >= 0 && p.exc[k].b == b && int(p.exc[k].pos)+int(p.exc[k].length) == i {
+				p.exc[k].length++
+			} else {
+				p.exc = append(p.exc, excRun{uint32(i), 1, b})
+			}
+			code = 0
+		}
+		p.bits[i>>2] |= code << (2 * (i & 3))
+	}
+	p.exc = slices.Clone(p.exc) // the exact size, nil when there is none
+	return p
+}
+
+// unpack writes the sequence into dst, which it grows if need be, and returns it.
+func (p packedSeq) unpack(dst []byte) []byte {
+	dst = slices.Grow(dst[:0], p.n)[:p.n]
+	for i := range p.n / 4 {
+		binary.LittleEndian.PutUint32(dst[4*i:], unpackWord[p.bits[i]])
+	}
+	for t := p.n &^ 3; t < p.n; t++ {
+		dst[t] = "acgt"[(p.bits[t>>2]>>(2*(t&3)))&3]
+	}
+	for _, e := range p.exc {
+		for k := range e.length {
+			dst[e.pos+k] = e.b
+		}
+	}
+	return dst
+}

pkg/obiprimer/packed_test.go

@@ -0,0 +1,109 @@
+package obiprimer
+
+import (
+	"bytes"
+	"math/rand"
+	"reflect"
+	"strings"
+	"testing"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+func TestPackedSeqRoundTrip(t *testing.T) {
+	rng := rand.New(rand.NewSource(1))
+	alphabet := []byte("acgtacgtacgtacgtnnrykmu-.")
+	for trial := 0; trial < 500; trial++ {
+		n := rng.Intn(70) // all the lengths modulo 4, the empty sequence included
+		raw := make([]byte, n)
+		for i := range raw {
+			raw[i] = alphabet[rng.Intn(len(alphabet))]
+		}
+		if trial%5 == 0 && n > 10 { // a long run of N, and one at each end
+			copy(raw[2:9], "nnnnnnn")
+			raw[0], raw[n-1] = 'n', 'n'
+		}
+		p := packBases(raw)
+		buf := make([]byte, 3, 8) // dirty, too small or too big: whatever
+		if got := p.unpack(buf); !bytes.Equal(got, raw) {
+			t.Fatalf("trial %d: %q unpacked as %q", trial, raw, got)
+		}
+		if len(p.bits) != (n+3)/4 {
+			t.Fatalf("trial %d: %d bytes for %d bases", trial, len(p.bits), n)
+		}
+	}
+	if p := packBases([]byte("acgtacgt")); p.exc != nil {
+		t.Errorf("no exception expected: %v", p.exc)
+	}
+}
+
+// Packing the corpus must change nothing to the search, ambiguous bases and
+// reference included.
+func TestPackedCorpusGivesTheSameSearch(t *testing.T) {
+	build := func() synthetic {
+		s := newSynthetic(10, -1, 7)
+		rng := rand.New(rand.NewSource(3))
+		for k, seq := range s.corpus.Seqs { // sprinkle ambiguous bases, some inside the sites
+			raw := append([]byte(nil), seq.Sequence()...)
+			for range 3 { // in the flanks, and in the insert past the first bases
+				at := rng.Intn(len(raw))
+				if at >= s.pos-2 && at < s.pos+18+2 || at >= s.pos+18+40-2 && at < s.pos+18+40+18+2 {
+					continue
+				}
+				raw[at] = "nrykm"[rng.Intn(5)]
+			}
+			if k < 2 { // one mismatch inside a site, which the search tolerates
+				raw[s.pos+5] = 'n'
+			}
+			s.corpus.Seqs[k] = obiseq.NewBioSequence(seq.Id(), raw, "")
+		}
+		return s
+	}
+	run := func(pack bool, circular bool) []PairStats {
+		s := build()
+		if pack {
+			s.corpus.Pack()
+			for i := range s.corpus.Seqs {
+				if s.corpus.Seqs[i] != nil {
+					t.Fatal("Pack must free the sequences")
+				}
+			}
+		}
+		p := s.params()
+		p.Reference = "s3"
+		p.ThreePrime = 3
+		p.Circular = circular
+		p.MinSpecificity = 0
+		pairs, _, err := Search(s.corpus, s.taxa, p)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return pairs
+	}
+	for _, circular := range []bool{false, true} {
+		want := run(false, circular)
+		if len(want) == 0 {
+			t.Fatal("the test needs pairs")
+		}
+		if got := run(true, circular); !reflect.DeepEqual(got, want) {
+			t.Errorf("circular=%v: packing changes the result (%d pairs, want %d)", circular, len(got), len(want))
+		}
+	}
+}
+
+func TestPackedCorpusAccessors(t *testing.T) {
+	c := &Corpus{
+		Seqs:  []*obiseq.BioSequence{obiseq.NewBioSequence("a", []byte(strings.ToLower("ACGTNNAC")), ""), obiseq.NewBioSequence("b", []byte("gg"), "")},
+		Roles: []Role{Example, Other},
+	}
+	c.Pack()
+	if c.Len() != 2 || c.SeqLen(0) != 8 || c.SeqLen(1) != 2 || c.ID(1) != "b" {
+		t.Errorf("accessors: %d %d %d %q", c.Len(), c.SeqLen(0), c.SeqLen(1), c.ID(1))
+	}
+	if raw, _ := c.bases(0, nil); string(raw) != "acgtnnac" {
+		t.Errorf("bases: %q", raw)
+	}
+	if i, err := c.IndexOfExample("a"); err != nil || i != 0 {
+		t.Errorf("IndexOfExample: %d %v", i, err)
+	}
+}

pkg/obiprimer/pairs.go

@@ -0,0 +1,326 @@
+package obiprimer
+
+import (
+	"cmp"
+	"fmt"
+	"slices"
+	"strings"
+	"sync"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obikmer"
+)
+
+// PairParams holds the parameters of the primer pair search, named after the
+// ecoPrimers options.
+type PairParams struct {
+	PrimerLength   int     // -O, word size
+	MinLength      int     // -l: the amplicon (primers excluded) must be longer than this
+	MaxLength      int     // -L: and shorter than this
+	ThreePrime     int     // -3: number of 3' bases that must match exactly (0 = no constraint)
+	Sensitivity    float64 // -s: a primer must hit more than this fraction of the examples (0.9)
+	FalsePositive  float64 // -x: and less than this fraction of the counterexamples (0.1)
+	MinSpecificity float64 // -T: pairs with a lower Bs are dropped (0.6)
+	Circular       bool    // -c
+	Reference      int     // -R: corpus index of the reference sequence (used when HasReference)
+	HasReference   bool
+	AmpBudget      int    // amplicon records held at once in phase 2 (0 = DefaultAmpBudget)
+	Release        func() // see SearchParams.Release
+}
+
+func (p PairParams) validate() error {
+	switch {
+	case p.PrimerLength < 2 || p.PrimerLength > 31:
+		return fmt.Errorf("obiprimer: primer length must be between 2 and 31, got %d", p.PrimerLength)
+	case p.MaxLength <= p.MinLength+1:
+		return fmt.Errorf("obiprimer: empty amplicon length range ]%d,%d[", p.MinLength, p.MaxLength)
+	case p.ThreePrime < 0 || p.ThreePrime > p.PrimerLength:
+		return fmt.Errorf("obiprimer: 3' strict bases must be in [0,%d], got %d", p.PrimerLength, p.ThreePrime)
+	case p.Sensitivity <= 0 || p.Sensitivity > 1 || p.FalsePositive < 0 || p.FalsePositive > 1:
+		return fmt.Errorf("obiprimer: quorums must be in ]0,1] and [0,1]")
+	}
+	return nil
+}
+
+// Primer is a conserved word: one found, with at most the allowed errors, in
+// enough example sequences to be a candidate primer.
+type Primer struct {
+	Word            uint64 // canonical word
+	Examples        int    // example sequences where it is found (either strand)
+	CounterExamples int
+	// Good means conserved in the examples and rare enough in the
+	// counterexamples; a pair needs at least one good primer.
+	Good bool
+}
+
+// seqMatch is one occurrence of a primer in one sequence, on 8 bytes: there
+// are millions of them.
+type seqMatch struct {
+	primer int32
+	place  uint32 // position << 1, plus 1 when the primer is found reversed
+}
+
+func newSeqMatch(primer int32, pos uint32, reverse bool) seqMatch {
+	return seqMatch{primer, pos<<1 | uint32(b2i(reverse))}
+}
+
+func (m seqMatch) pos() uint32   { return m.place >> 1 }
+func (m seqMatch) reverse() bool { return m.place&1 == 1 }
+
+// primerSet is the conserved words and, for each corpus sequence, their
+// occurrences sorted by position.
+type primerSet struct {
+	primers []Primer
+	perSeq  [][]seqMatch
+}
+
+// quorumCount is floor(n*fraction), robust to the binary form of the fraction.
+func quorumCount(n int, fraction float64) int { return MinSequences(n, fraction) }
+
+// selectPrimers keeps the words found in at least floor(examples*Sensitivity)
+// example sequences and marks as good those found in at most
+// floor(counterexamples*FalsePositive) counterexamples.
+func selectPrimers(c *Corpus, wm []WordMatches, p PairParams) *primerSet {
+	counts := c.Counts()
+	inQ := quorumCount(counts.Examples, p.Sensitivity)
+	outQ := quorumCount(counts.CounterExamples, p.FalsePositive)
+
+	ex, co := make([]int, len(wm)), make([]int, len(wm))
+	parallelFor(len(wm), func(_, i int) { ex[i], co[i] = wm[i].SequencesMatched(c) })
+
+	// the occurrences of the words kept, by sequence, each in a slice of the
+	// exact size (and not in one big array: a few hundred bytes to some kB each
+	// find room in the free memory, which a single block of tens of MB does not)
+	ps := &primerSet{perSeq: make([][]seqMatch, c.Len())}
+	size := make([]int, c.Len())
+	for i, w := range wm {
+		if ex[i] < inQ {
+			continue
+		}
+		ps.primers = append(ps.primers, Primer{w.Word, ex[i], co[i], co[i] <= outQ})
+		for _, m := range w.Matches {
+			size[m.Seq]++
+		}
+	}
+	for s, n := range size {
+		if n > 0 {
+			ps.perSeq[s] = make([]seqMatch, 0, n)
+		}
+	}
+	id := int32(0)
+	for i, w := range wm {
+		if ex[i] < inQ {
+			continue
+		}
+		for _, m := range w.Matches {
+			ps.perSeq[m.Seq] = append(ps.perSeq[m.Seq], newSeqMatch(id, m.Pos, m.Reverse))
+		}
+		id++
+	}
+	parallelFor(c.Len(), func(_, s int) {
+		slices.SortStableFunc(ps.perSeq[s], func(a, b seqMatch) int { return cmp.Compare(a.pos(), b.pos()) })
+	})
+	return ps
+}
+
+// pairKey identifies a pair: the two primers, ordered by canonical word, and
+// for each whether the primer is the canonical word as is (true) or its reverse
+// complement (false). It is the key of ecoPrimers' pair tree.
+type pairKey struct {
+	p1, p2 int32
+	a1, a2 bool
+}
+
+// occurrence is one amplicon: a pair of matches in a sequence.
+type occurrence struct {
+	key      pairKey
+	swapped  bool // true when the upstream match is the second primer of the key
+	up, down seqMatch
+	length   int // amplicon length, primers excluded
+}
+
+// forEachOccurrence calls fn for every amplicon of sequence s, applying the
+// length, good-primer and 3' constraints of ecoPrimers. buf is the buffer in
+// which the sequence is unpacked, if the corpus is packed.
+func (ps *primerSet) forEachOccurrence(c *Corpus, s int, buf []byte, p PairParams, fn func(occurrence)) []byte {
+	ms := ps.perSeq[s]
+	var raw []byte
+	if p.ThreePrime > 0 && len(ms) > 1 {
+		raw, buf = c.bases(s, buf)
+	}
+	ps.forEachOccurrenceIn(raw, ms, p, fn)
+	return buf
+}
+
+// forEachOccurrenceIn is forEachOccurrence over ms, the matches of a sequence
+// sorted by position (all of them, or only those of the primers of interest,
+// which yields the amplicons between these primers and none other), and raw its
+// bases, needed when the 3' bases must match exactly.
+func (ps *primerSet) forEachOccurrenceIn(raw []byte, ms []seqMatch, p PairParams, fn func(occurrence)) {
+	lp := p.PrimerLength
+
+	for i := range ms {
+		for j := i + 1; j < len(ms); j++ {
+			dpos := int(ms[j].pos()) - int(ms[i].pos())
+			if dpos <= lp {
+				continue
+			}
+			length := dpos - lp
+			if length >= p.MaxLength {
+				break
+			}
+			up, down := ms[i], ms[j]
+			if length <= p.MinLength || !(ps.primers[up.primer].Good || ps.primers[down.primer].Good) {
+				continue
+			}
+
+			// The upstream match is read as is (primer 1 of the pair when its word
+			// is smaller), the downstream one as the reverse primer.
+			key := pairKey{up.primer, down.primer, !up.reverse(), down.reverse()}
+			swapped := !(ps.primers[key.p2].Word > ps.primers[key.p1].Word)
+			if swapped {
+				key = pairKey{key.p2, key.p1, key.a2, key.a1}
+			}
+
+			if p.ThreePrime > 0 && !ps.threePrimeMatches(raw, up, down, p) {
+				continue
+			}
+			fn(occurrence{key, swapped, up, down, length})
+		}
+	}
+}
+
+// wordBase returns base t (0 = 5') of a 2-bit packed word of size lp.
+func wordBase(word uint64, lp, t int) byte { return byte(word>>(2*uint(lp-1-t))) & 3 }
+
+// threePrimeMatches reports whether the last p.ThreePrime bases of the forward
+// primer (as found at the upstream match) and of the reverse primer (reverse
+// complement of the downstream window) equal the sequence, i.e. have no
+// mismatch at their 3' end.
+func (ps *primerSet) threePrimeMatches(raw []byte, up, down seqMatch, p PairParams) bool {
+	lp, n := p.PrimerLength, len(raw)
+	at := func(pos int) byte {
+		if p.Circular {
+			pos %= n
+		}
+		return baseCode[raw[pos]]
+	}
+
+	fw := ps.primers[up.primer].Word
+	rv := ps.primers[down.primer].Word
+	for t := lp - p.ThreePrime; t < lp; t++ {
+		// forward primer: the word when found direct, its reverse complement otherwise
+		var f byte
+		if !up.reverse() {
+			f = wordBase(fw, lp, t)
+		} else {
+			f = 3 - wordBase(fw, lp, lp-1-t)
+		}
+		if s := at(int(up.pos()) + t); s > 3 || s != f {
+			return false
+		}
+
+		// reverse primer: the word when found reverse, its reverse complement otherwise
+		var r byte
+		if down.reverse() {
+			r = wordBase(rv, lp, t)
+		} else {
+			r = 3 - wordBase(rv, lp, lp-1-t)
+		}
+		if s := at(int(down.pos()) + lp - 1 - t); s > 3 || 3-s != r {
+			return false
+		}
+	}
+	return true
+}
+
+// pairAcc accumulates, over the sequences, the counts ecoPrimers keeps while it
+// builds the pairs.
+type pairAcc struct {
+	examples, counters uint32
+	last               int32 // 1 + last sequence counted, to count each sequence once
+	amplicons          uint32
+	sumLen             uint64
+	minLen, maxLen     uint32
+}
+
+// countPairs enumerates the amplicons of every sequence, in parallel, and
+// returns for each pair the sequences it amplifies and the length statistics
+// of its example amplicons.
+func (ps *primerSet) countPairs(c *Corpus, p PairParams) map[pairKey]*pairAcc {
+	workers := obidefault.ParallelWorkers()
+	partial := make([]map[pairKey]*pairAcc, workers)
+
+	jobs := make(chan int)
+	var wg sync.WaitGroup
+	for w := range workers {
+		wg.Add(1)
+		partial[w] = map[pairKey]*pairAcc{}
+		go func() {
+			defer wg.Done()
+			acc := partial[w]
+			var buf []byte
+			for s := range jobs {
+				if c.Roles[s] == Other {
+					continue // only the quorums are counted here
+				}
+				isExample := c.Roles[s] == Example
+				buf = ps.forEachOccurrence(c, s, buf, p, func(o occurrence) {
+					a := acc[o.key]
+					if a == nil {
+						a = &pairAcc{minLen: ^uint32(0)}
+						acc[o.key] = a
+					}
+					if a.last != int32(s)+1 {
+						a.last = int32(s) + 1
+						if isExample {
+							a.examples++
+						} else {
+							a.counters++
+						}
+					}
+					if isExample {
+						l := uint32(o.length)
+						a.amplicons++
+						a.sumLen += uint64(l)
+						a.minLen = min(a.minLen, l)
+						a.maxLen = max(a.maxLen, l)
+					}
+				})
+			}
+		}()
+	}
+	for s := range c.Roles {
+		jobs <- s
+	}
+	close(jobs)
+	wg.Wait()
+
+	// A sequence is handled by one worker only, so the sequence counts add up.
+	total := partial[0]
+	for _, m := range partial[1:] {
+		for k, a := range m {
+			t := total[k]
+			if t == nil {
+				total[k] = a
+				continue
+			}
+			t.examples += a.examples
+			t.counters += a.counters
+			t.amplicons += a.amplicons
+			t.sumLen += a.sumLen
+			t.minLen = min(t.minLen, a.minLen)
+			t.maxLen = max(t.maxLen, a.maxLen)
+		}
+	}
+	return total
+}
+
+// orientedWord returns the upper-case primer string 5'->3': the canonical word
+// when direct, its reverse complement otherwise.
+func orientedWord(word uint64, lp int, direct bool) string {
+	if !direct {
+		word = obikmer.ReverseComplement(word, lp)
+	}
+	return strings.ToUpper(string(obikmer.DecodeKmer(word, lp, nil)))
+}

pkg/obiprimer/pairs_batches_test.go

@@ -0,0 +1,78 @@
+package obiprimer
+
+import (
+	"math/rand"
+	"reflect"
+	"strings"
+	"testing"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+func TestAmpBatches(t *testing.T) {
+	cases := []struct {
+		count  []int
+		budget int
+		want   [][2]int
+	}{
+		{[]int{3, 3, 3}, 0, [][2]int{{0, 3}}},
+		{[]int{3, 3, 3}, 100, [][2]int{{0, 3}}},
+		{[]int{3, 3, 3}, 6, [][2]int{{0, 2}, {2, 3}}},
+		{[]int{3, 3, 3}, 1, [][2]int{{0, 1}, {1, 2}, {2, 3}}}, // a pair above the budget goes alone
+		{[]int{9, 1, 1}, 5, [][2]int{{0, 1}, {1, 3}}},
+		{nil, 5, nil},
+		{[]int{0, 0}, 5, [][2]int{{0, 2}}},
+	}
+	for _, c := range cases {
+		if got := ampBatches(c.count, c.budget); !reflect.DeepEqual(got, c.want) {
+			t.Errorf("ampBatches(%v, %d) = %v, want %v", c.count, c.budget, got, c.want)
+		}
+	}
+}
+
+// The batches of phase 2 only bound the memory: whatever their size, the pairs,
+// their indices and their sites on the reference are the same.
+func TestAmpBudgetDoesNotChangeTheResult(t *testing.T) {
+	s := newSynthetic(10, -1, 5)
+	// non-targets sharing barcodes with targets, and counterexamples
+	for k, role := range []Role{Other, Other, CounterExample, Other, CounterExample} {
+		cp := append([]byte(nil), s.corpus.Seqs[k*2].Sequence()...)
+		if role == CounterExample {
+			for i := s.pos; i < s.pos+10; i++ { // no forward site: not amplified
+				cp[i] = "cgta"[strings.IndexByte("acgt", cp[i])]
+			}
+		}
+		s.corpus.Seqs = append(s.corpus.Seqs, obiseq.NewBioSequence(strings.Repeat("x", k+1), cp, ""))
+		s.corpus.Roles = append(s.corpus.Roles, role)
+		s.taxa.Of = append(s.taxa.Of, int32(100+k%3))
+	}
+	// a few unrelated sequences, with a taxon of their own or none
+	rng := rand.New(rand.NewSource(9))
+	for k := 0; k < 3; k++ {
+		s.corpus.Seqs = append(s.corpus.Seqs, obiseq.NewBioSequence("u"+strings.Repeat("y", k), []byte(strings.ToLower(randDNA(rng, 300))), ""))
+		s.corpus.Roles = append(s.corpus.Roles, Other)
+		s.taxa.Of = append(s.taxa.Of, int32(k-1))
+	}
+
+	run := func(budget int) []PairStats {
+		p := s.params()
+		p.MinSpecificity = 0 // keep every pair: the stress case of the batches
+		p.FalsePositive = 1
+		p.Reference = "s3"
+		p.AmpBudget = budget
+		pairs, info, err := Search(s.corpus, s.taxa, p)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if info.Pair.Candidates < 2 {
+			t.Fatalf("budget %d: only %d pairs reach phase 2b, the test needs several", budget, info.Pair.Candidates)
+		}
+		return pairs
+	}
+	want := run(0)
+	for _, budget := range []int{1, 7, 50, 1000} {
+		if got := run(budget); !reflect.DeepEqual(got, want) {
+			t.Errorf("budget %d changes the result:\n got %d pairs\nwant %d pairs", budget, len(got), len(want))
+		}
+	}
+}

pkg/obiprimer/pairs_eval.go

@@ -0,0 +1,211 @@
+package obiprimer
+
+import (
+	"cmp"
+	"fmt"
+	"math"
+	"slices"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obithermo"
+)
+
+// PairStats describes one primer pair and its quality indices. Primers are
+// written 5'->3' and ordered so that Primer1 < Primer2 (as in ecoPrimers).
+type PairStats struct {
+	Primer1, Primer2 string
+	Good1, Good2     bool // primer conserved in the examples and rare in the counterexamples
+
+	Examples        int // example sequences amplified
+	CounterExamples int
+	Yule            float32 // Examples/|E| - CounterExamples/|C|
+
+	ExampleTaxa    int     // target taxa amplified
+	CounterTaxa    int     // counterexample taxa amplified
+	Bc             float32 // coverage: ExampleTaxa over all the target taxa
+	Identified     int     // target taxa all of whose barcodes belong to them alone (see resolution.go)
+	Bs             float32 // specificity: Identified over ExampleTaxa
+	Bd             float32 // barcode discrimination: share of the target barcodes leading to a single taxon
+	MinLen, MaxLen int
+	MeanLen        float64
+	Tm1, Tm2       float64 // melting temperature of the primers against their perfect target
+	MinTm1, MinTm2 float64 // lowest melting temperature over the example amplicons
+	GC1, GC2       int
+
+	// With a reference sequence (-R): where the pair amplifies it. RefLocation
+	// is the GenBank location of the two primer sites (1-based, left to right),
+	// wrapped in complement() when Primer1 binds the other strand; RefAmplicon
+	// is the amplified region, primers excluded, read from Primer1 to Primer2.
+	// Both are empty when the pair does not amplify the reference.
+	RefLocation string
+	RefAmplicon string
+}
+
+// Score is the ranking criterion of ecoPrimers.
+func (p PairStats) Score() float32 { return p.Yule * p.Bs }
+
+// PairInfo summarizes the pair search, for reporting.
+type PairInfo struct {
+	Primers        int // conserved words kept as candidate primers
+	GoodPrimers    int // of which good ones
+	CandidatePairs int // pairs delimiting an amplicon in at least one sequence
+	RetainedPairs  int // pairs passing the sensitivity and false positive quorums, before the Bs filter
+	Candidates     int // of which pairs whose Bs upper bound reaches the threshold (their amplicons are gathered in full)
+}
+
+// FindPairs searches the primer pairs from the approximate occurrences of the
+// strict words (see MatchApat) and evaluates them.
+//
+// A pair is a pair of conserved primers, at least one of them good, that
+// delimit an amplicon of the allowed length. It is retained when it amplifies
+// more than Sensitivity of the examples and less than FalsePositive of the
+// counterexamples, and its specificity Bs reaches MinSpecificity. The result
+// is sorted by decreasing Yule*Bs.
+//
+// model computes the melting temperatures; nil leaves them NaN.
+func FindPairs(c *Corpus, taxa *Taxa, wm []WordMatches, p PairParams, model *obithermo.Model) ([]PairStats, PairInfo, error) {
+	var info PairInfo
+	if err := p.validate(); err != nil {
+		return nil, info, err
+	}
+	if taxa == nil || len(taxa.Of) != c.Len() {
+		return nil, info, fmt.Errorf("obiprimer: taxa do not match the corpus")
+	}
+	ps := selectPrimers(c, wm, p)
+	wm = nil // the occurrences are in ps now
+	info.Primers = len(ps.primers)
+	for _, pr := range ps.primers {
+		if pr.Good {
+			info.GoodPrimers++
+		}
+	}
+	if len(ps.primers) == 0 {
+		return nil, info, nil
+	}
+
+	// Phase 1: which pairs amplify enough examples and few enough counterexamples.
+	counts := c.Counts()
+	acc := ps.countPairs(c, p)
+	info.CandidatePairs = len(acc)
+	sens, fpq := float32(p.Sensitivity), float32(p.FalsePositive)
+	var kept []pairKey
+	for k, a := range acc {
+		q := float32(a.examples) / float32(counts.Examples)
+		qfp := float32(0)
+		if counts.CounterExamples > 0 {
+			qfp = float32(a.counters) / float32(counts.CounterExamples)
+		}
+		if q > sens && qfp < fpq {
+			kept = append(kept, k)
+		}
+	}
+	info.RetainedPairs = len(kept)
+	slices.SortFunc(kept, func(a, b pairKey) int {
+		return cmp.Or(cmp.Compare(a.p1, b.p1), cmp.Compare(a.p2, b.p2),
+			boolCmp(a.a1, b.a1), boolCmp(a.a2, b.a2))
+	})
+
+	// Phase 2a: the amplicons of the target sequences, which give the melting
+	// temperatures, the reference sites, and an upper bound of Bs. It is cheap
+	// and rejects most pairs on the specificity threshold.
+	first := ps.boundPairs(c, taxa, p, kept, model)
+	var survivors []int // indices in kept
+	for i := range kept {
+		if first.targets[i] > 0 && float32(first.identified[i])/float32(first.targets[i]) >= float32(p.MinSpecificity) {
+			survivors = append(survivors, i)
+		}
+	}
+	info.Candidates = len(survivors)
+
+	// Phase 2b: for the survivors only, the amplicons of every sequence, whatever
+	// its role: they decide which target taxa are identified.
+	results := ps.evaluatePairs(c, taxa, p, kept, survivors, acc, first, model)
+
+	var out []PairStats
+	for _, st := range results {
+		if st != nil {
+			out = append(out, *st)
+		}
+	}
+	slices.SortFunc(out, func(a, b PairStats) int {
+		return cmp.Or(-cmp.Compare(a.Score(), b.Score()), cmp.Compare(a.Primer1, b.Primer1), cmp.Compare(a.Primer2, b.Primer2))
+	})
+	return out, info, nil
+}
+
+func boolCmp(a, b bool) int {
+	switch {
+	case a == b:
+		return 0
+	case !a:
+		return -1
+	}
+	return 1
+}
+
+// evaluate puts the indices of one retained pair in a PairStats. It reports
+// false when the pair must be dropped (no target taxon amplified, or
+// specificity under the threshold).
+func (ps *primerSet) evaluate(taxa *Taxa, counts Counts, p PairParams, k pairKey, a *pairAcc, res resolution,
+	minTm1, minTm2 float32, model *obithermo.Model) (PairStats, bool) {
+
+	if res.targetTaxa == 0 {
+		return PairStats{}, false
+	}
+	st := PairStats{
+		Examples:        int(a.examples),
+		CounterExamples: int(a.counters),
+		ExampleTaxa:     res.targetTaxa,
+		CounterTaxa:     res.counterTaxa,
+		Identified:      res.identified,
+		MinLen:          int(a.minLen),
+		MaxLen:          int(a.maxLen),
+		MeanLen:         float64(a.sumLen) / float64(a.amplicons),
+	}
+	q := float32(a.examples) / float32(counts.Examples)
+	qfp := float32(0)
+	if counts.CounterExamples > 0 {
+		qfp = float32(a.counters) / float32(counts.CounterExamples)
+	}
+	st.Yule = q - qfp
+	st.Bc = float32(res.targetTaxa) / float32(taxa.ExampleTaxa)
+	st.Bs = float32(res.identified) / float32(res.targetTaxa)
+	st.Bd = float32(res.discriminant) / float32(res.barcodes)
+	if st.Bs < float32(p.MinSpecificity) {
+		return PairStats{}, false
+	}
+
+	return ps.finish(st, p, k, a, minTm1, minTm2, model), true
+}
+
+// finish adds the primers, their composition and melting temperatures, and puts
+// the two primers in string order.
+func (ps *primerSet) finish(st PairStats, p PairParams, k pairKey, a *pairAcc,
+	minTm1, minTm2 float32, model *obithermo.Model) PairStats {
+
+	lp := p.PrimerLength
+	st.Primer1 = orientedWord(ps.primers[k.p1].Word, lp, k.a1)
+	st.Primer2 = orientedWord(ps.primers[k.p2].Word, lp, k.a2)
+	st.Good1, st.Good2 = ps.primers[k.p1].Good, ps.primers[k.p2].Good
+	st.GC1, st.GC2 = obithermo.GCCount([]byte(st.Primer1)), obithermo.GCCount([]byte(st.Primer2))
+
+	st.Tm1, st.Tm2, st.MinTm1, st.MinTm2 = math.NaN(), math.NaN(), math.NaN(), math.NaN()
+	if model != nil {
+		st.Tm1, _ = model.SelfTm([]byte(st.Primer1))
+		st.Tm2, _ = model.SelfTm([]byte(st.Primer2))
+		if !math.IsInf(float64(minTm1), 1) {
+			st.MinTm1 = float64(minTm1)
+		}
+		if !math.IsInf(float64(minTm2), 1) {
+			st.MinTm2 = float64(minTm2)
+		}
+	}
+
+	if st.Primer2 < st.Primer1 {
+		st.Primer1, st.Primer2 = st.Primer2, st.Primer1
+		st.Good1, st.Good2 = st.Good2, st.Good1
+		st.GC1, st.GC2 = st.GC2, st.GC1
+		st.Tm1, st.Tm2 = st.Tm2, st.Tm1
+		st.MinTm1, st.MinTm2 = st.MinTm2, st.MinTm1
+	}
+	return st
+}

pkg/obiprimer/pairs_phase2.go

@@ -0,0 +1,160 @@
+package obiprimer
+
+import (
+	"sync"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obithermo"
+)
+
+// DefaultAmpBudget is the number of amplicon records (12 bytes each) phase 2
+// holds at once: 20 million, 240 MB. The pairs are processed in batches whose
+// amplicons fit in it. With the usual specificity threshold a search needs one
+// batch; with none (-T 0) no pair is rejected early and there are many more
+// amplicons than any memory could hold.
+const DefaultAmpBudget = 20_000_000
+
+func (p PairParams) release() {
+	if p.Release != nil {
+		p.Release()
+	}
+}
+
+func (p PairParams) ampBudget() int {
+	if p.AmpBudget > 0 {
+		return p.AmpBudget
+	}
+	return DefaultAmpBudget
+}
+
+// batchIndex maps the pairs kept[lo:hi] to their number in the batch.
+func batchIndex(kept []pairKey, lo, hi int) map[pairKey]int32 {
+	index := make(map[pairKey]int32, hi-lo)
+	for i := lo; i < hi; i++ {
+		index[kept[i]] = int32(i - lo)
+	}
+	return index
+}
+
+// parallelFor calls fn(i) for i in [0,n) on obidefault.ParallelWorkers() goroutines.
+func parallelFor(n int, fn func(worker, i int)) {
+	jobs := make(chan int)
+	var wg sync.WaitGroup
+	for w := range obidefault.ParallelWorkers() {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for i := range jobs {
+				fn(w, i)
+			}
+		}()
+	}
+	for i := 0; i < n; i++ {
+		jobs <- i
+	}
+	close(jobs)
+	wg.Wait()
+}
+
+// bound is what phase 2a learns about each retained pair (indexed like kept).
+type bound struct {
+	targets    []int // target taxa amplified
+	identified []int // upper bound of the target taxa identified (lenientIdentified)
+	minTm1     []float32
+	minTm2     []float32
+	refs       map[int32]refHit
+}
+
+// boundPairs is phase 2a: from the amplicons of the target sequences alone, it
+// gives each retained pair the upper bound of its Bs, the lowest melting
+// temperatures and its site on the reference. The pairs go by batches whose
+// amplicons fit in the budget; they are counted first, so that nothing is resized.
+func (ps *primerSet) boundPairs(c *Corpus, taxa *Taxa, p PairParams, kept []pairKey,
+	model *obithermo.Model) *bound {
+
+	plan := ps.planAmplicons(c, p, batchIndex(kept, 0, len(kept)), true)
+	res := &bound{
+		targets:    make([]int, len(kept)),
+		identified: make([]int, len(kept)),
+		minTm1:     make([]float32, len(kept)),
+		minTm2:     make([]float32, len(kept)),
+		refs:       map[int32]refHit{},
+	}
+	resolvers := make([]*resolver, obidefault.ParallelWorkers())
+	for i := range resolvers {
+		resolvers[i] = newResolver()
+	}
+
+	for _, b := range ampBatches(plan.total(), p.ampBudget()) {
+		lo, hi := b[0], b[1]
+		p.release() // the previous batch, or the stage before, is garbage
+		store := newAmpStore(plan, lo, hi)
+		sites := ps.collectAmplicons(c, taxa, p, batchIndex(kept, lo, hi), plan, store,
+			collectOptions{examplesOnly: true, model: model, reference: true})
+		store.check()
+		parallelFor(hi-lo, func(w, i int) {
+			res.targets[lo+i], res.identified[lo+i] = resolvers[w].lenientIdentified(store.pair(i))
+		})
+		copy(res.minTm1[lo:hi], sites.minTm1)
+		copy(res.minTm2[lo:hi], sites.minTm2)
+		for i, h := range sites.refs {
+			res.refs[int32(lo)+i] = h
+		}
+	}
+	return res
+}
+
+// evaluatePairs is phase 2b: for the surviving pairs (indices in kept), it
+// gathers the amplicons of every sequence, whatever its role, by batches whose
+// amplicons fit in the budget, and evaluates the pairs. The result is indexed
+// like survivors; a nil entry is a pair dropped by the specificity threshold.
+func (ps *primerSet) evaluatePairs(c *Corpus, taxa *Taxa, p PairParams, kept []pairKey, survivors []int,
+	acc map[pairKey]*pairAcc, first *bound, model *obithermo.Model) []*PairStats {
+
+	results := make([]*PairStats, len(survivors))
+	if len(survivors) == 0 {
+		return results
+	}
+	keys := make([]pairKey, len(survivors))
+	for j, i := range survivors {
+		keys[j] = kept[i]
+	}
+	// the amplicons of every sequence are counted first: the store is allocated
+	// at its exact size, and the batches are cut on the exact counts
+	plan := ps.planAmplicons(c, p, batchIndex(keys, 0, len(keys)), false)
+
+	counts := c.Counts()
+	var refBases []byte
+	if p.HasReference {
+		refBases, _ = c.bases(p.Reference, nil)
+	}
+	resolvers := make([]*resolver, obidefault.ParallelWorkers())
+	for i := range resolvers {
+		resolvers[i] = newResolver()
+	}
+	for _, b := range ampBatches(plan.total(), p.ampBudget()) {
+		lo, hi := b[0], b[1]
+		p.release()
+		store := newAmpStore(plan, lo, hi)
+		ps.collectAmplicons(c, taxa, p, batchIndex(keys, lo, hi), plan, store, collectOptions{})
+		store.check()
+		parallelFor(hi-lo, func(w, n int) {
+			j := lo + n
+			i := survivors[j]
+			k := keys[j]
+			st, ok := ps.evaluate(taxa, counts, p, k, acc[k], resolvers[w].resolve(store.pair(n)),
+				first.minTm1[i], first.minTm2[i], model)
+			if !ok {
+				return
+			}
+			if h, found := first.refs[int32(i)]; found {
+				// evaluate orders the primers by string; the amplicon is read from the first one
+				s1 := orientedWord(ps.primers[k.p1].Word, p.PrimerLength, k.a1)
+				s2 := orientedWord(ps.primers[k.p2].Word, p.PrimerLength, k.a2)
+				st.RefLocation, st.RefAmplicon = referenceSites(refBases, h, p, s2 < s1)
+			}
+			results[j] = &st
+		})
+	}
+	return results
+}

pkg/obiprimer/pairs_test.go

@@ -0,0 +1,240 @@
+package obiprimer
+
+import (
+	"bufio"
+	"fmt"
+	"math"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiformats"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obikmer"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obithermo"
+)
+
+// ecoRow is one result line of ecoPrimers.
+type ecoRow struct {
+	p1, p2         string
+	tm1, mtm1      float64
+	tm2, mtm2      float64
+	gc1, gc2       int
+	good           string
+	in, out        int
+	yule           float64
+	intaxa, outtax int
+	bc             float64
+	well           int
+	bs             float64
+	mind, maxd     int
+	avg            float64
+}
+
+func readEcoRows(t *testing.T, path string) []ecoRow {
+	t.Helper()
+	f, err := os.Open(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	var rows []ecoRow
+	sc := bufio.NewScanner(f)
+	sc.Buffer(make([]byte, 1<<20), 1<<20)
+	for sc.Scan() {
+		line := sc.Text()
+		if strings.HasPrefix(line, "#") || strings.TrimSpace(line) == "" {
+			continue
+		}
+		f := strings.Split(line, "\t")
+		fl := func(i int) float64 {
+			v, err := strconv.ParseFloat(strings.TrimSpace(f[i]), 64)
+			if err != nil {
+				t.Fatalf("field %d of %q: %v", i, line, err)
+			}
+			return v
+		}
+		in := func(i int) int { return int(fl(i)) }
+		rows = append(rows, ecoRow{
+			p1: f[1], p2: f[2], tm1: fl(3), mtm1: fl(4), tm2: fl(5), mtm2: fl(6),
+			gc1: in(7), gc2: in(8), good: f[9], in: in(10), out: in(11), yule: fl(12),
+			intaxa: in(13), outtax: in(14), bc: fl(15), well: in(16), bs: fl(17),
+			mind: in(18), maxd: in(19), avg: fl(20),
+		})
+	}
+	if err := sc.Err(); err != nil {
+		t.Fatal(err)
+	}
+	return rows
+}
+
+// Opt-in end-to-end comparison with the complete ecoPrimers result tables of
+// the tutorial: same parameters (-e 3 -3 2 -l 30 -L 150 -c, -r taxid).
+//
+// Bs and the number of identified taxa are NOT compared: obiprimers defines
+// them differently (a taxon is identified only if all its barcodes are its own,
+// and every amplified sequence counts, targets or not), so they are lower. The
+// specificity threshold is switched off so that the pairs ecoPrimers dropped
+// with its own Bs are still produced, and only the pairs of ecoPrimers are
+// checked. What is checked: that every pair of ecoPrimers is found, with the
+// same good flags, sequence and taxon counts, Yule, Bc, lengths, GC, Tm and
+// lowest Tm; and that identified taxa never exceed those of ecoPrimers' more
+// lenient rule on the same amplified sequences.
+//
+// Cyprinidae is reproduced exactly on these columns for its 3448 pairs.
+// Teleostei finds the same 407 pairs, and 6 of them count one
+// sequence less than ecoPrimers (NC_033548, whose downstream site is
+// NGGGGAGGCAAGTCGTAA). The 3' check of ecoPrimers reads an N through the
+// shift register of extractSite, where the code 4 of N spills on the next base
+// and ~4&3 reads as T, so an N can be accepted by accident. Emulating that
+// register exactly in the 3' check was verified to reproduce both tables on
+// every column; it is not kept: an ambiguous base cannot be a verified 3' match.
+func TestPairsAgainstEcoPrimers(t *testing.T) {
+	dir := os.Getenv("OBIPRIMERS_TUTORIAL")
+	if dir == "" {
+		t.Skip("OBIPRIMERS_TUTORIAL not set")
+	}
+	taxonomy, err := obiformats.LoadTaxonomy(filepath.Join(dir, "ncbitaxo_20250211.tgz"), false, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, tc := range []struct {
+		name, taxid string
+		species     int
+		divergent   int
+	}{{"Cyprinidae", "7953", 316, 0}, {"Teleostei", "32443", 3876, 6}} {
+		t.Run(tc.name, func(t *testing.T) {
+			var stages []string
+			tick := time.Now()
+			lap := func(name string) {
+				stages = append(stages, fmt.Sprintf("%s %s", name, time.Since(tick).Round(time.Millisecond)))
+				tick = time.Now()
+			}
+			restrict, err := BelongingTo(taxonomy, []string{tc.taxid})
+			if err != nil {
+				t.Fatal(err)
+			}
+			seqs, err := obiformats.ReadSequencesFromFile(filepath.Join(dir, "mito.vert.fasta.gz"))
+			if err != nil {
+				t.Fatal(err)
+			}
+			corpus, err := LoadCorpus(seqs, Selection{Restrict: restrict})
+			if err != nil {
+				t.Fatal(err)
+			}
+			lap("read+select")
+			taxa, err := corpus.AssignTaxa(taxonomy, "species")
+			if err != nil {
+				t.Fatal(err)
+			}
+			lap("taxa")
+			if taxa.ExampleTaxa != tc.species {
+				t.Errorf("%d example species, ecoPrimers reports %d", taxa.ExampleTaxa, tc.species)
+			}
+
+			sw, err := FindStrictWords(corpus, WordSearch{PrimerLength: 18, Quorum: 0.7, Circular: true, TmpDir: t.TempDir()})
+			if err != nil {
+				t.Fatal(err)
+			}
+			lap("strict words")
+			words := make([]uint64, len(sw.Words))
+			for i, w := range sw.Words {
+				words[i] = w.Kmer
+			}
+			wm, err := MatchWords(corpus, words, MatchParams{K: 18, MaxErrors: 3, Circular: true})
+			if err != nil {
+				t.Fatal(err)
+			}
+			lap("approximate search")
+			got, _, err := FindPairs(corpus, taxa, wm, PairParams{
+				PrimerLength: 18, MinLength: 30, MaxLength: 150, ThreePrime: 2,
+				Sensitivity: 0.9, FalsePositive: 0.1, MinSpecificity: 0, Circular: true,
+			}, obithermo.DefaultModel())
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			lap("pairs+indices+Tm")
+			var ms runtime.MemStats
+			runtime.ReadMemStats(&ms)
+			t.Logf("stages: %s; Go heap in use at the end %.0f MB (includes the taxonomy)", strings.Join(stages, ", "), float64(ms.HeapAlloc)/1e6)
+
+			want := readEcoRows(t, filepath.Join(dir, tc.name+".ecoprimers"))
+			t.Logf("%d pairs, ecoPrimers %d", len(got), len(want))
+
+			byKey := map[string]PairStats{}
+			for _, g := range got {
+				byKey[g.Primer1+"/"+g.Primer2] = g
+			}
+			bad := map[string]int{}
+			missing, divergent := 0, 0
+			note := func(field string, w ecoRow, format string, args ...any) {
+				if bad[field]++; bad[field] <= 3 {
+					t.Errorf("%s/%s %s: "+format, append([]any{w.p1, w.p2, field}, args...)...)
+				}
+			}
+			near := func(a, b, tol float64) bool { return math.Abs(a-b) <= tol }
+			for _, w := range want {
+				g, ok := byKey[w.p1+"/"+w.p2]
+				if !ok {
+					if missing++; missing <= 5 {
+						t.Errorf("ecoPrimers pair %s/%s not found", w.p1, w.p2)
+					}
+					continue
+				}
+				delete(byKey, w.p1+"/"+w.p2)
+				good := string("bG"[b2i(g.Good1)]) + string("bG"[b2i(g.Good2)])
+				if good != w.good {
+					note("good", w, "got %s want %s", good, w.good)
+				}
+				// Known divergence, see the comment of the test: ecoPrimers accepts one
+				// amplicon (NC_033548) whose 3' primer site ends with an N. We reject it,
+				// so the counts can be exactly one lower and nothing else may differ.
+				lower := func(got, want int) bool { return got == want || got == want-1 }
+				diverges := g.Examples != w.in || g.ExampleTaxa != w.intaxa
+				if diverges {
+					divergent++
+				}
+				if !lower(g.Examples, w.in) || g.CounterExamples != w.out {
+					note("sequences", w, "got %d/%d want %d/%d", g.Examples, g.CounterExamples, w.in, w.out)
+				}
+				if !lower(g.ExampleTaxa, w.intaxa) || g.CounterTaxa != w.outtax {
+					note("taxa", w, "got %d/%d want %d/%d", g.ExampleTaxa, g.CounterTaxa, w.intaxa, w.outtax)
+				}
+				if !diverges && g.Identified > w.well {
+					note("identified", w, "got %d, more than the %d of ecoPrimers' lenient rule", g.Identified, w.well)
+				}
+				tol := 0.0006
+				if diverges {
+					tol = 0.0021
+				}
+				if !near(float64(g.Yule), w.yule, tol) || !near(float64(g.Bc), w.bc, tol) {
+					note("indices", w, "yule/bc got %.4f/%.4f want %.3f/%.3f", g.Yule, g.Bc, w.yule, w.bc)
+				}
+				if g.MinLen != w.mind || g.MaxLen != w.maxd || !near(g.MeanLen, w.avg, 0.006) {
+					note("lengths", w, "got %d/%d/%.2f want %d/%d/%.2f", g.MinLen, g.MaxLen, g.MeanLen, w.mind, w.maxd, w.avg)
+				}
+				if g.GC1 != w.gc1 || g.GC2 != w.gc2 {
+					note("gc", w, "got %d/%d want %d/%d", g.GC1, g.GC2, w.gc1, w.gc2)
+				}
+				// ecoPrimers prints Kelvin-273.0; obithermo uses 273.15
+				if !near(g.Tm1+0.15, w.tm1, 0.051) || !near(g.Tm2+0.15, w.tm2, 0.051) {
+					note("tm", w, "got %.2f/%.2f want %.1f/%.1f", g.Tm1+0.15, g.Tm2+0.15, w.tm1, w.tm2)
+				}
+				if !near(g.MinTm1+0.15, w.mtm1, 0.051) || !near(g.MinTm2+0.15, w.mtm2, 0.051) {
+					note("mintm", w, "got %.2f/%.2f want %.1f/%.1f", g.MinTm1+0.15, g.MinTm2+0.15, w.mtm1, w.mtm2)
+				}
+			}
+			t.Logf("missing from ours: %d; pairs of ours that ecoPrimers dropped with its Bs: %d; mismatching fields: %v; pairs one sequence below ecoPrimers: %d",
+				missing, len(byKey), bad, divergent)
+			if divergent != tc.divergent {
+				t.Errorf("%d pairs differ from ecoPrimers by one sequence, expected %d", divergent, tc.divergent)
+			}
+		})
+	}
+	_ = obikmer.DecodeKmer
+}

pkg/obiprimer/reference.go

@@ -0,0 +1,44 @@
+package obiprimer
+
+import (
+	"fmt"
+	"strings"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+// span writes the 1-based GenBank range of the lp bases starting at the 0-based
+// position pos. On a circular sequence a range running over the junction is
+// split in two.
+func span(pos, lp, n int, circular bool) string {
+	end := pos + lp
+	if end > n && circular {
+		return fmt.Sprintf("%d..%d,1..%d", pos+1, n, end-n)
+	}
+	return fmt.Sprintf("%d..%d", pos+1, end)
+}
+
+// referenceSites describes an amplicon of a pair on the reference sequence raw.
+//
+// The location lists the two primer sites left to right as join(a..b,c..d),
+// inside complement() when the first primer of the pair (Primer1 once the pair
+// is in string order) binds the reverse strand. The amplicon is the region
+// between the sites, read 5'->3' from the first primer, in lower case.
+//
+// swappedByOrder says whether putting the pair in string order exchanged the
+// two primers of the key.
+func referenceSites(raw []byte, h refHit, p PairParams, swappedByOrder bool) (location, amplicon string) {
+	lp, n := p.PrimerLength, len(raw)
+
+	// The key slot 1 is the upstream primer unless the match order was swapped;
+	// the exchange due to the string order flips it again.
+	firstIsUpstream := h.swapped == swappedByOrder
+
+	location = "join(" + span(int(h.up.pos()), lp, n, p.Circular) + "," + span(int(h.down.pos()), lp, n, p.Circular) + ")"
+	region := []byte(strings.ToLower(string(raw[int(h.up.pos())+lp : int(h.down.pos())])))
+	if firstIsUpstream {
+		return location, string(region)
+	}
+	rc := obiseq.NewBioSequence("", region, "").ReverseComplement(true)
+	return "complement(" + location + ")", string(rc.Sequence())
+}

pkg/obiprimer/resolution.go

@@ -0,0 +1,154 @@
+package obiprimer
+
+// The three quality indices of a pair, over the sequences it amplifies.
+//
+// Every sequence the pair amplifies gives a barcode (the amplicon between the
+// primers); all of them are taken into account, whatever their role: examples
+// (the targets), counterexamples and non-targets.
+//
+//   - Bc, coverage: target taxa amplified / target taxa.
+//   - Bs, specificity: target taxa identified / target taxa amplified. A target
+//     taxon is identified when ALL its barcodes belong to it alone: as soon as a
+//     barcode is shared with another taxon, target or not, counterexample or
+//     not, both taxa are badly identified.
+//   - Bd, barcode discrimination: the share of the barcodes carried by the
+//     targets that lead without ambiguity to one taxon.
+//
+// A sequence with no taxon at the chosen rank is an anonymous taxon, distinct
+// from every other: a barcode it shares with a taxon is ambiguous.
+
+// barcodeState is what is known of one barcode while the amplicons are scanned.
+type barcodeState struct {
+	taxon int32 // 1 + the first taxon met, 0 while none
+	flags uint8
+}
+
+const (
+	barcodeAmbiguous = 1 << iota // two distinct taxa carry it
+	barcodeAnonymous             // a sequence with no taxon carries it
+	barcodeTarget                // a target sequence with a taxon carries it
+)
+
+// ambiguous reports whether the barcode does not lead to a single taxon.
+func (b barcodeState) ambiguous() bool {
+	return b.flags&barcodeAmbiguous != 0 || (b.flags&barcodeAnonymous != 0 && b.taxon != 0)
+}
+
+// resolution is the outcome of the identification of the taxa of one pair.
+type resolution struct {
+	targetTaxa   int // target taxa amplified
+	counterTaxa  int // counterexample taxa amplified
+	identified   int // target taxa whose barcodes all belong to them alone
+	barcodes     int // distinct barcodes carried by the targets
+	discriminant int // of which lead to a single taxon
+}
+
+// resolver computes the indices of pairs one after the other, reusing its maps
+// (a pair needs a few thousand entries, and there are thousands of pairs).
+type resolver struct {
+	state    map[uint64]barcodeState
+	targets  map[int32]struct{}
+	counters map[int32]struct{}
+	bad      map[int32]struct{}
+	owner    map[uint64]int32
+}
+
+func newResolver() *resolver {
+	return &resolver{
+		state:    map[uint64]barcodeState{},
+		targets:  map[int32]struct{}{},
+		counters: map[int32]struct{}{},
+		bad:      map[int32]struct{}{},
+		owner:    map[uint64]int32{},
+	}
+}
+
+// resolve computes the resolution of one pair from its amplicons, in any order.
+func resolve(a ampSlice) resolution { return newResolver().resolve(a) }
+
+func (rs *resolver) resolve(a ampSlice) resolution {
+	var res resolution
+	clear(rs.state)
+	clear(rs.targets)
+	clear(rs.counters)
+	clear(rs.bad)
+
+	for i, h := range a.hash {
+		taxon, role := unpackTaxon(a.tax[i])
+		st := rs.state[h]
+		switch {
+		case taxon < 0:
+			st.flags |= barcodeAnonymous
+		case st.taxon == 0:
+			st.taxon = taxon + 1
+		case st.taxon != taxon+1:
+			st.flags |= barcodeAmbiguous
+		}
+		if taxon >= 0 {
+			switch role {
+			case Example:
+				st.flags |= barcodeTarget
+				rs.targets[taxon] = struct{}{}
+			case CounterExample:
+				rs.counters[taxon] = struct{}{}
+			}
+		}
+		rs.state[h] = st
+	}
+
+	for i, h := range a.hash {
+		taxon, role := unpackTaxon(a.tax[i])
+		if role == Example && taxon >= 0 && rs.state[h].ambiguous() {
+			rs.bad[taxon] = struct{}{}
+		}
+	}
+
+	for _, st := range rs.state {
+		if st.flags&barcodeTarget != 0 {
+			res.barcodes++
+			if !st.ambiguous() {
+				res.discriminant++
+			}
+		}
+	}
+	res.targetTaxa, res.counterTaxa = len(rs.targets), len(rs.counters)
+	res.identified = len(rs.targets) - len(rs.bad)
+	return res
+}
+
+// lenientIdentified is an upper bound of resolution.identified, computed on
+// the target sequences alone and with the lenient rule of ecoPrimers (a taxon
+// counts as soon as ONE of its barcodes belongs to it alone, among the targets).
+//
+// A taxon identified by resolve has all its barcodes alone among every amplified
+// sequence, hence among the targets: it has at least one such barcode and is
+// counted here too. So identified <= the second result, with the same number of
+// target taxa, which lets a pair be rejected on the specificity threshold without
+// gathering the amplicons of the non-target sequences.
+func lenientIdentified(a ampSlice) (targetTaxa, identified int) {
+	return newResolver().lenientIdentified(a)
+}
+
+func (rs *resolver) lenientIdentified(a ampSlice) (targetTaxa, identified int) {
+	clear(rs.owner) // barcode -> its taxon, or -1 when shared
+	clear(rs.targets)
+	for i, h := range a.hash {
+		taxon, role := unpackTaxon(a.tax[i])
+		if role != Example || taxon < 0 {
+			continue
+		}
+		rs.targets[taxon] = struct{}{}
+		if prev, seen := rs.owner[h]; !seen {
+			rs.owner[h] = taxon
+		} else if prev != taxon {
+			rs.owner[h] = -1
+		}
+	}
+	clear(rs.bad) // here: the taxa owning a barcode
+	for _, t := range rs.owner {
+		if t >= 0 {
+			rs.bad[t] = struct{}{}
+		}
+	}
+	return len(rs.targets), len(rs.bad)
+}

pkg/obiprimer/resolution_test.go

@@ -0,0 +1,165 @@
+package obiprimer
+
+import (
+	"math/rand"
+	"testing"
+)
+
+// rec builds an amplicon record: barcode, taxon (-1: none at the rank), role.
+// pack builds an ampSlice from logical records.
+func pack(parts ...[]ampRecord) ampSlice {
+	var a ampSlice
+	for _, part := range parts {
+		for _, r := range part {
+			a.hash = append(a.hash, r.hash)
+			a.tax = append(a.tax, packTaxon(r.taxon, r.role))
+		}
+	}
+	return a
+}
+
+func rec(barcode uint64, taxon int32, role Role) ampRecord {
+	return ampRecord{hash: barcode, taxon: taxon, role: role}
+}
+
+func TestResolutionUserExample(t *testing.T) {
+	// A has the barcodes A1 and A2, B has A2 and B1. A2 is shared: neither A
+	// nor B is identified, but A1 and B1 lead to one species each.
+	const A, B = 0, 1
+	const A1, A2, B1 = 11, 12, 21
+	res := resolve(pack([]ampRecord{
+		rec(A1, A, Example), rec(A2, A, Example),
+		rec(A2, B, Example), rec(B1, B, Example),
+	}))
+	want := resolution{targetTaxa: 2, identified: 0, barcodes: 3, discriminant: 2}
+	if res != want {
+		t.Fatalf("got %+v, want %+v", res, want)
+	}
+}
+
+func TestResolutionAllUnique(t *testing.T) {
+	res := resolve(pack([]ampRecord{rec(1, 0, Example), rec(2, 1, Example), rec(3, 2, Example)}))
+	if res.identified != 3 || res.discriminant != 3 || res.barcodes != 3 || res.targetTaxa != 3 {
+		t.Errorf("all barcodes distinct: %+v", res)
+	}
+}
+
+func TestResolutionSameTaxonSharesItsBarcodeFreely(t *testing.T) {
+	// several sequences of one species with the same barcode are not ambiguous,
+	// and several barcodes for one species are fine when none is shared
+	res := resolve(pack([]ampRecord{
+		rec(1, 0, Example), rec(1, 0, Example), rec(1, 0, Example),
+		rec(2, 0, Example),
+	}))
+	if res.identified != 1 || res.barcodes != 2 || res.discriminant != 2 {
+		t.Errorf("%+v", res)
+	}
+}
+
+// The barcode of a target that is unique among the targets but shared with a
+// non-target, or with a counterexample, does not identify it.
+func TestResolutionNonTargetsCount(t *testing.T) {
+	for _, role := range []Role{Other, CounterExample} {
+		res := resolve(pack([]ampRecord{
+			rec(1, 0, Example), // unique among the targets...
+			rec(1, 5, role),    // ...but shared with a non-target taxon
+			rec(2, 1, Example), // identified
+			rec(3, 6, role),    // a barcode of non-targets only
+		}))
+		if res.targetTaxa != 2 || res.identified != 1 {
+			t.Errorf("role %d: %+v, want 2 target taxa and 1 identified", role, res)
+		}
+		// barcodes carried by the targets: 1 (ambiguous) and 2; barcode 3 leads to a non-target
+		if res.barcodes != 2 || res.discriminant != 1 {
+			t.Errorf("role %d: Bd counts %d/%d, want 1/2", role, res.discriminant, res.barcodes)
+		}
+	}
+}
+
+func TestResolutionAmplifiedTaxaCounts(t *testing.T) {
+	res := resolve(pack([]ampRecord{
+		rec(1, 0, Example), rec(2, 1, CounterExample), rec(3, 1, CounterExample),
+		rec(4, 2, CounterExample), rec(5, 3, Other),
+	}))
+	if res.targetTaxa != 1 || res.counterTaxa != 2 {
+		t.Errorf("target taxa %d (want 1), counterexample taxa %d (want 2: others are not counted)",
+			res.targetTaxa, res.counterTaxa)
+	}
+}
+
+// A sequence with no taxon at the rank is an anonymous taxon, distinct from all
+// the others: a barcode shared with it is ambiguous.
+func TestResolutionAnonymousTaxon(t *testing.T) {
+	res := resolve(pack([]ampRecord{
+		rec(1, 0, Example), rec(1, -1, Other), // shared with an unknown: ambiguous
+		rec(2, 1, Example),
+	}))
+	if res.identified != 1 || res.barcodes != 2 || res.discriminant != 1 {
+		t.Errorf("target sharing with an anonymous sequence: %+v", res)
+	}
+	// two anonymous sequences with the same barcode never count: no target carries it
+	res = resolve(pack([]ampRecord{rec(7, -1, Other), rec(7, -1, Other), rec(8, 0, Example)}))
+	if res.barcodes != 1 || res.discriminant != 1 || res.identified != 1 {
+		t.Errorf("anonymous barcodes alone: %+v", res)
+	}
+	// a target with no taxon is not a target taxon, and is anonymous too
+	res = resolve(pack([]ampRecord{rec(1, -1, Example), rec(1, 0, Example), rec(2, 0, Example)}))
+	if res.targetTaxa != 1 || res.identified != 0 {
+		t.Errorf("an anonymous target sequence sharing a barcode must make the taxon ambiguous: %+v", res)
+	}
+}
+
+func TestResolutionSeveralPartsAndOrder(t *testing.T) {
+	// the amplicons come as one slice per worker, in any order
+	a := []ampRecord{rec(1, 0, Example), rec(2, 1, Example)}
+	b := []ampRecord{rec(2, 2, Other), rec(3, 1, Example)}
+	if res := resolve(pack(a, b)); res.targetTaxa != 2 || res.identified != 1 {
+		t.Errorf("taxon 1 shares barcode 2 with taxon 2: %+v", res)
+	}
+	if res := resolve(pack(b, a)); res.targetTaxa != 2 || res.identified != 1 {
+		t.Errorf("order must not matter: %+v", res)
+	}
+}
+
+// ecoPrimers' lenient rule on the user's example: A and B each own one barcode,
+// so both count (the new rule counts none), and the bound is what lets a pair
+// be rejected early.
+func TestLenientIdentifiedUserExample(t *testing.T) {
+	recs := []ampRecord{
+		rec(11, 0, Example), rec(12, 0, Example),
+		rec(12, 1, Example), rec(21, 1, Example),
+	}
+	if targets, identified := lenientIdentified(pack(recs)); targets != 2 || identified != 2 {
+		t.Errorf("lenient rule: %d of %d, want 2 of 2", identified, targets)
+	}
+	if res := resolve(pack(recs)); res.identified != 0 {
+		t.Errorf("strict rule: %d identified, want 0", res.identified)
+	}
+}
+
+// The pruning of phase 2 is only exact if the bound never falls under the
+// true count, with the same number of target taxa.
+func TestLenientIdentifiedIsAnUpperBound(t *testing.T) {
+	rng := rand.New(rand.NewSource(3))
+	for trial := 0; trial < 3000; trial++ {
+		var recs []ampRecord
+		n := 1 + rng.Intn(40)
+		for range n {
+			role := []Role{Example, Example, Example, CounterExample, Other}[rng.Intn(5)]
+			recs = append(recs, rec(uint64(rng.Intn(12)), int32(rng.Intn(8))-1, role)) // taxon -1: none
+		}
+		// split in a few parts, as the workers do
+		cut := rng.Intn(len(recs) + 1)
+		res := resolve(pack(recs[:cut], recs[cut:]))
+		targets, lenient := lenientIdentified(pack(recs[:cut], recs[cut:]))
+		if targets != res.targetTaxa {
+			t.Fatalf("trial %d: %d target taxa for the bound, %d for resolve", trial, targets, res.targetTaxa)
+		}
+		if res.identified > lenient {
+			t.Fatalf("trial %d: identified %d exceeds the bound %d for %v", trial, res.identified, lenient, recs)
+		}
+		if res.discriminant > res.barcodes || res.identified > res.targetTaxa {
+			t.Fatalf("trial %d: inconsistent %+v", trial, res)
+		}
+	}
+}

pkg/obiprimer/roles.go

@@ -0,0 +1,79 @@
+// Package obiprimer implements the core of obiprimers, the OBITools4
+// re-implementation of ecoPrimers: finding conserved words usable as primers
+// in a set of example sequences, and the primer pairs they form.
+package obiprimer
+
+import (
+	"fmt"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax"
+)
+
+// Role tells how a sequence takes part in the primer search.
+type Role uint8
+
+const (
+	// Other sequences are non-targets: nothing is required of them, so they
+	// count neither in the sensitivity nor in the false positive quorums, but
+	// the amplicons they give are taken into account to decide whether a target
+	// is identified (see Bs and Bd).
+	Other Role = iota
+	// Example sequences are the targets: the primers must amplify them.
+	Example
+	// CounterExample sequences are those the primers must not amplify.
+	CounterExample
+)
+
+// Selection defines the roles from the taxonomy, with the ecoPrimers
+// semantics of -r (restrict), -i (ignore) and -E (exception):
+//
+//   - a sequence is an Example when it belongs to Restrict (all sequences when
+//     Restrict is nil) and not to Except;
+//   - otherwise it is a CounterExample when it belongs to Ignore or to Except;
+//   - otherwise it is Other.
+//
+// Nil predicates mean "no such constraint".
+type Selection struct {
+	Restrict obiseq.SequencePredicate
+	Ignore   obiseq.SequencePredicate
+	Except   obiseq.SequencePredicate
+}
+
+// Role returns the role of seq.
+func (s Selection) Role(seq *obiseq.BioSequence) Role {
+	inRestrict := s.Restrict == nil || s.Restrict(seq)
+	inExcept := s.Except != nil && s.Except(seq)
+
+	switch {
+	case inRestrict && !inExcept:
+		return Example
+	case inExcept, s.Ignore != nil && s.Ignore(seq):
+		return CounterExample
+	default:
+		return Other
+	}
+}
+
+// BelongingTo returns a predicate true for the sequences whose taxon is a
+// sub-clade of at least one of the given taxids. It returns nil (no
+// constraint) when taxids is empty.
+func BelongingTo(taxonomy *obitax.Taxonomy, taxids []string) (obiseq.SequencePredicate, error) {
+	var pred obiseq.SequencePredicate
+	for _, id := range taxids {
+		taxon, _, err := taxonomy.Taxon(id)
+		if err != nil {
+			return nil, fmt.Errorf("obiprimer: taxid %q: %w", id, err)
+		}
+		if taxon == nil {
+			return nil, fmt.Errorf("obiprimer: taxid %q is unknown from the taxonomy", id)
+		}
+		p := obiseq.IsSubCladeOf(taxonomy, taxon)
+		if pred == nil {
+			pred = p
+		} else {
+			pred = pred.Or(p)
+		}
+	}
+	return pred, nil
+}

pkg/obiprimer/search.go

@@ -0,0 +1,123 @@
+package obiprimer
+
+import (
+	"fmt"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obithermo"
+)
+
+// SearchParams gathers every parameter of the primer pair search, named after
+// the ecoPrimers options they come from.
+type SearchParams struct {
+	PrimerLength   int     // -O: size of the primers (default 18)
+	MaxErrors      int     // -e: substitutions allowed between a primer and its sites (default 3)
+	MinLength      int     // -l: the amplicon (primers excluded) must be longer than this
+	MaxLength      int     // -L: and shorter than this
+	ThreePrime     int     // -3: number of 3' bases that must match exactly
+	StrictQuorum   float64 // -q: fraction of the examples containing the exact word (0.7)
+	Sensitivity    float64 // -s: fraction of the examples a primer must match (0.9)
+	FalsePositive  float64 // -x: fraction of the counterexamples it may match (0.1)
+	MinSpecificity float64 // -T: pairs with a lower Bs are dropped (0.6)
+	Circular       bool    // -c: sequences are circular
+	NoMultiMatch   bool    // -U: discard the primers matching several times in a sequence
+	Reference      string  // -R: identifier of an example sequence on which the amplicons are located ("" = none)
+
+	// Release, if not nil, is called when a stage has just freed a lot of memory
+	// and the next one is about to take some: the command makes the runtime hand
+	// it back to the system then, which the garbage collector would only do
+	// slowly, and the peak of memory would be the sum of the stages.
+	Release func()
+
+	AmpBudget int // amplicon records held at once by the pair evaluation (0 = DefaultAmpBudget)
+
+	Partitions int // partitions of the k-mer counter (default 256)
+	TmpDir     string
+
+	Model *obithermo.Model // melting temperatures; nil leaves them NaN
+}
+
+// DefaultSearchParams returns the ecoPrimers defaults.
+func DefaultSearchParams() SearchParams {
+	return SearchParams{
+		PrimerLength: 18, MaxErrors: 3, MinLength: 0, MaxLength: 1000,
+		StrictQuorum: 0.7, Sensitivity: 0.9, FalsePositive: 0.1, MinSpecificity: 0.6,
+		Model: obithermo.DefaultModel(),
+	}
+}
+
+// SearchInfo reports the size of every stage of a search.
+type SearchInfo struct {
+	Counts      Counts
+	StrictWords int // words found without error in at least StrictQuorum of the examples
+	MinStrict   int // that quorum, in sequences
+	Matches     int // approximate occurrences of those words
+	Repeated    int // strict words dropped because repeated in an example (-U)
+	MultiMatch  int // words dropped because matching several times in a sequence (-U)
+	Pair        PairInfo
+}
+
+// Search runs the whole ecoPrimers pipeline on a corpus: strict words,
+// approximate occurrences, pairs and their indices. The pairs are sorted by
+// decreasing Yule*Bs.
+func Search(c *Corpus, taxa *Taxa, p SearchParams) ([]PairStats, *SearchInfo, error) {
+	if p.MaxErrors < 0 {
+		return nil, nil, fmt.Errorf("obiprimer: the allowed mismatch count cannot be negative")
+	}
+
+	sw, err := FindStrictWords(c, WordSearch{
+		PrimerLength: p.PrimerLength,
+		Quorum:       p.StrictQuorum,
+		Circular:     p.Circular,
+		Partitions:   p.Partitions,
+		TmpDir:       p.TmpDir,
+	})
+	if err != nil {
+		return nil, nil, err
+	}
+	info := &SearchInfo{Counts: sw.Counts, StrictWords: len(sw.Words), MinStrict: sw.MinSequences}
+	if p.NoMultiMatch {
+		sw.Words, info.Repeated = DropRepeatedWords(sw.Words)
+	}
+
+	refIndex := -1
+	if p.Reference != "" {
+		if refIndex, err = c.IndexOfExample(p.Reference); err != nil {
+			return nil, info, err
+		}
+	}
+
+	words := make([]uint64, len(sw.Words))
+	for i, w := range sw.Words {
+		words[i] = w.Kmer
+	}
+	if p.Release != nil {
+		p.Release() // the strict word search has just freed its occurrences
+	}
+	wm, err := MatchWords(c, words, MatchParams{K: p.PrimerLength, MaxErrors: p.MaxErrors, Circular: p.Circular})
+	if err != nil {
+		return nil, info, err
+	}
+	if p.NoMultiMatch {
+		wm, info.MultiMatch = DropMultiMatch(wm, c)
+	}
+	for _, w := range wm {
+		info.Matches += len(w.Matches)
+	}
+
+	pairs, pinfo, err := FindPairs(c, taxa, wm, PairParams{
+		PrimerLength:   p.PrimerLength,
+		MinLength:      p.MinLength,
+		MaxLength:      p.MaxLength,
+		ThreePrime:     p.ThreePrime,
+		Sensitivity:    p.Sensitivity,
+		FalsePositive:  p.FalsePositive,
+		MinSpecificity: p.MinSpecificity,
+		Circular:       p.Circular,
+		Reference:      refIndex,
+		HasReference:   refIndex >= 0,
+		AmpBudget:      p.AmpBudget,
+		Release:        p.Release,
+	}, p.Model)
+	info.Pair = pinfo
+	return pairs, info, err
+}

pkg/obiprimer/search_real_test.go

@@ -0,0 +1,258 @@
+package obiprimer
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"testing"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiformats"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax"
+)
+
+// loadTutorial reads the Cyprinidae examples of the tutorial data (the opt-in
+// tests need OBIPRIMERS_TUTORIAL, see TestStrictWordsAgainstEcoPrimers).
+func loadTutorial(t *testing.T) (*Corpus, *Taxa, *obitax.Taxonomy) {
+	t.Helper()
+	return loadTutorialWith(t, nil)
+}
+
+// loadTutorialWith is loadTutorial with counterexample clades (ecoPrimers -i).
+func loadTutorialWith(t *testing.T, ignore []string) (*Corpus, *Taxa, *obitax.Taxonomy) {
+	t.Helper()
+	dir := os.Getenv("OBIPRIMERS_TUTORIAL")
+	if dir == "" {
+		t.Skip("OBIPRIMERS_TUTORIAL not set")
+	}
+	taxonomy, err := obiformats.LoadTaxonomy(filepath.Join(dir, "ncbitaxo_20250211.tgz"), false, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	restrict, err := BelongingTo(taxonomy, []string{"7953"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	seqs, err := obiformats.ReadSequencesFromFile(filepath.Join(dir, "mito.vert.fasta.gz"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	sel := Selection{Restrict: restrict}
+	if sel.Ignore, err = BelongingTo(taxonomy, ignore); err != nil {
+		t.Fatal(err)
+	}
+	corpus, err := LoadCorpus(seqs, sel)
+	if err != nil {
+		t.Fatal(err)
+	}
+	taxa, err := corpus.AssignTaxa(taxonomy, "species")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return corpus, taxa, taxonomy
+}
+
+func tutorialParams(t *testing.T) SearchParams {
+	p := DefaultSearchParams()
+	p.MinLength, p.MaxLength, p.ThreePrime, p.Circular = 30, 150, 2, true
+	p.TmpDir = t.TempDir()
+	return p
+}
+
+// ecoPrimers -U on the Cyprinidae tutorial case (-e 3 -3 2 -l 30 -L 150 -c -r 7953 -U)
+// keeps 1366 pairs. testdata/ecoprimers_Cyprinidae_U.pairs.tsv holds, for each,
+// the primers, the amplified example and counterexample sequences, the amplified
+// example and counterexample taxa, and the identified taxa. The identified taxa
+// are not compared (obiprimers defines them differently, see
+// TestPairsAgainstEcoPrimers) and the specificity threshold is off, so that
+// the pairs ecoPrimers dropped with its own Bs do not hide any of its pairs.
+func TestNoMultiMatchAgainstEcoPrimers(t *testing.T) {
+	corpus, taxa, _ := loadTutorial(t)
+	p := tutorialParams(t)
+	p.MinSpecificity = 0
+	p.NoMultiMatch = true
+	pairs, info, err := Search(corpus, taxa, p)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("%+v", *info)
+
+	f, err := os.Open(filepath.Join("testdata", "ecoprimers_Cyprinidae_U.pairs.tsv"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	want := map[string][5]int{}
+	sc := bufio.NewScanner(f)
+	for sc.Scan() {
+		var p1, p2 string
+		var v [5]int
+		if _, err := fmt.Sscanf(sc.Text(), "%s\t%s\t%d\t%d\t%d\t%d\t%d", &p1, &p2, &v[0], &v[1], &v[2], &v[3], &v[4]); err != nil {
+			t.Fatal(err)
+		}
+		want[p1+"/"+p2] = v
+	}
+
+	ours := map[string]PairStats{}
+	for _, g := range pairs {
+		ours[g.Primer1+"/"+g.Primer2] = g
+	}
+	bad, missing := 0, 0
+	for key, v := range want {
+		g, ok := ours[key]
+		if !ok {
+			if missing++; missing <= 3 {
+				t.Errorf("ecoPrimers -U pair %s is not found", key)
+			}
+			continue
+		}
+		got := [4]int{g.Examples, g.CounterExamples, g.ExampleTaxa, g.CounterTaxa}
+		if got != [4]int{v[0], v[1], v[2], v[3]} || g.Identified > v[4] {
+			if bad++; bad <= 3 {
+				t.Errorf("pair %s: got %v (identified %d), ecoPrimers %v", key, got, g.Identified, v)
+			}
+		}
+	}
+	t.Logf("%d pairs of ecoPrimers -U checked, %d missing, %d different; obiprimers has %d pairs (%d more, dropped by ecoPrimers' own Bs)",
+		len(want), missing, bad, len(pairs), len(pairs)-len(want)+missing)
+}
+
+var locRE = regexp.MustCompile(`(\d+)\.\.(\d+)`)
+
+// ecoPrimers -R is only right for the references that happen to come first: it
+// indexes the amplicons of a pair by sequence number. On NC_083213 only 382 of
+// its 3405 locations are real primer sites, against all 3405 for obiprimers.
+// Here the locations are checked against the sequence itself, so no
+// ecoPrimers output is needed: each location must give two real primer sites
+// (at most 3 substitutions) and the amplicon must be the region between them,
+// read from the first primer.
+func TestReferenceSitesAreRealSites(t *testing.T) {
+	corpus, taxa, _ := loadTutorial(t)
+
+	for _, ref := range []string{"NC_022183", "NC_083213"} {
+		t.Run(ref, func(t *testing.T) {
+			p := tutorialParams(t)
+			p.Reference = ref
+			pairs, _, err := Search(corpus, taxa, p)
+			if err != nil {
+				t.Fatal(err)
+			}
+			idx, err := corpus.IndexOfExample(ref)
+			if err != nil {
+				t.Fatal(err)
+			}
+			seq := strings.ToLower(string(corpus.Seqs[idx].Sequence()))
+
+			hamming := func(a, b string) int {
+				d := 0
+				for i := range a {
+					if a[i] != b[i] {
+						d++
+					}
+				}
+				return d
+			}
+			located, bad := 0, 0
+			for _, pr := range pairs {
+				if pr.RefLocation == "" {
+					continue
+				}
+				located++
+				m := locRE.FindAllStringSubmatch(pr.RefLocation, -1)
+				if len(m) != 2 {
+					t.Fatalf("location %q", pr.RefLocation)
+				}
+				var a1, b1, a2, b2 int
+				fmt.Sscanf(m[0][0], "%d..%d", &a1, &b1)
+				fmt.Sscanf(m[1][0], "%d..%d", &a2, &b2)
+				left, right := seq[a1-1:b1], seq[a2-1:b2]
+				between := seq[b1 : a2-1]
+
+				p1, p2 := strings.ToLower(pr.Primer1), strings.ToLower(pr.Primer2)
+				var ok bool
+				if strings.HasPrefix(pr.RefLocation, "complement(") {
+					ok = hamming(strings.ToLower(revcomp(strings.ToUpper(right))), p1) <= 3 && hamming(left, p2) <= 3 &&
+						pr.RefAmplicon == strings.ToLower(revcomp(strings.ToUpper(between)))
+				} else {
+					ok = hamming(left, p1) <= 3 && hamming(strings.ToLower(revcomp(strings.ToUpper(right))), p2) <= 3 &&
+						pr.RefAmplicon == between
+				}
+				if !ok {
+					if bad++; bad <= 3 {
+						t.Errorf("pair %s/%s: %s is not a real location", pr.Primer1, pr.Primer2, pr.RefLocation)
+					}
+				}
+			}
+			t.Logf("%d of %d pairs amplify %s; %d wrong locations", located, len(pairs), ref, bad)
+			if located == 0 {
+				t.Error("no pair amplifies the reference")
+			}
+		})
+	}
+}
+
+// ecoPrimers with counterexamples: -r 7953 -i 1 (the Cyprinidae against all the
+// other sequences of the database, 7497 counterexamples of 7387 species), -e 3
+// -3 2 -l 30 -L 150 -c. ecoPrimers needed 45 minutes and keeps 173 pairs.
+// testdata/ecoprimers_Cyprinidae_I1.pairs.tsv holds for each: the primers, the
+// good flags, the amplified example and counterexample sequences, the amplified
+// example and counterexample taxa, and the identified taxa.
+func TestCounterExamplesAgainstEcoPrimers(t *testing.T) {
+	corpus, taxa, _ := loadTutorialWith(t, []string{"1"})
+	if c := corpus.Counts(); c.Examples != 325 || c.CounterExamples != 7497 || taxa.CounterExamples != 7387 {
+		t.Fatalf("counts %+v, %d counterexample species (ecoPrimers: 325, 7497, 7387)", c, taxa.CounterExamples)
+	}
+	pc := tutorialParams(t)
+	pc.MinSpecificity = 0 // ecoPrimers' own Bs filter is not reproduced, see TestPairsAgainstEcoPrimers
+	pairs, info, err := Search(corpus, taxa, pc)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("%+v", *info)
+
+	f, err := os.Open(filepath.Join("testdata", "ecoprimers_Cyprinidae_I1.pairs.tsv"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	type row struct {
+		good string
+		v    [5]int
+	}
+	want := map[string]row{}
+	sc := bufio.NewScanner(f)
+	for sc.Scan() {
+		var p1, p2 string
+		var r row
+		if _, err := fmt.Sscanf(sc.Text(), "%s\t%s\t%s\t%d\t%d\t%d\t%d\t%d", &p1, &p2, &r.good,
+			&r.v[0], &r.v[1], &r.v[2], &r.v[3], &r.v[4]); err != nil {
+			t.Fatal(err)
+		}
+		want[p1+"/"+p2] = r
+	}
+
+	ours := map[string]PairStats{}
+	for _, g := range pairs {
+		ours[g.Primer1+"/"+g.Primer2] = g
+	}
+	bad, missing := 0, 0
+	for key, w := range want {
+		g, ok := ours[key]
+		if !ok {
+			if missing++; missing <= 3 {
+				t.Errorf("ecoPrimers pair %s is not found", key)
+			}
+			continue
+		}
+		good := string("bG"[b2i(g.Good1)]) + string("bG"[b2i(g.Good2)])
+		got := [4]int{g.Examples, g.CounterExamples, g.ExampleTaxa, g.CounterTaxa}
+		if got != [4]int{w.v[0], w.v[1], w.v[2], w.v[3]} || good != w.good || g.Identified > w.v[4] {
+			if bad++; bad <= 3 {
+				t.Errorf("pair %s: got %s %v (identified %d), ecoPrimers %s %v", key, good, got, g.Identified, w.good, w.v)
+			}
+		}
+	}
+	t.Logf("%d pairs of ecoPrimers checked, %d missing, %d different; obiprimers has %d pairs", len(want), missing, bad, len(pairs))
+}

pkg/obiprimer/search_test.go

@@ -0,0 +1,274 @@
+package obiprimer
+
+import (
+	"fmt"
+	"math/rand"
+	"strings"
+	"testing"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obikmer"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+// synthetic builds nseq example sequences sharing the block F + insert + rc(R),
+// each with its own flanks and its own substitution inside the insert (so that
+// every sequence has a distinct amplicon, hence Bs = 1, with one taxon each).
+// extraF, if not empty, is appended to the flank of sequence extraIn: that
+// sequence then holds the forward site twice.
+type synthetic struct {
+	corpus  *Corpus
+	taxa    *Taxa
+	f, r, i string // forward primer, reverse primer (5'->3'), insert of sequence 0
+	pos     int    // 0-based start of the block in every sequence
+}
+
+func newSynthetic(nseq int, repeatForwardIn int, seed int64) synthetic {
+	rng := rand.New(rand.NewSource(seed))
+	f := randDNA(rng, 18)
+	r := randDNA(rng, 18)
+	ins := randDNA(rng, 40)
+	const pos = 60
+
+	c := &Corpus{}
+	taxa := &Taxa{Rank: "species", ExampleTaxa: nseq}
+	for k := 0; k < nseq; k++ {
+		in := []byte(ins)
+		in[3+k] = "ACGT"[(strings.IndexByte("ACGT", in[3+k])+1)%4] // one private substitution
+		s := randDNA(rng, pos) + f + string(in) + revcomp(r) + randDNA(rng, 60)
+		if k == repeatForwardIn {
+			s += randDNA(rng, 20) + f // the forward site a second time
+		}
+		c.Seqs = append(c.Seqs, obiseq.NewBioSequence(fmt.Sprintf("s%d", k), []byte(strings.ToLower(s)), ""))
+		c.Roles = append(c.Roles, Example)
+		taxa.Of = append(taxa.Of, int32(k))
+	}
+	return synthetic{c, taxa, f, r, ins, pos}
+}
+
+func (s synthetic) params() SearchParams {
+	p := DefaultSearchParams()
+	p.MaxErrors = 1
+	p.MinLength, p.MaxLength = 20, 100
+	p.Partitions = 8
+	p.TmpDir = ""
+	return p
+}
+
+// find returns the pair made of the primers f and r, whatever their order.
+func find(pairs []PairStats, f, r string) *PairStats {
+	for i := range pairs {
+		if (pairs[i].Primer1 == f && pairs[i].Primer2 == r) || (pairs[i].Primer1 == r && pairs[i].Primer2 == f) {
+			return &pairs[i]
+		}
+	}
+	return nil
+}
+
+func TestSearchFindsThePlantedPair(t *testing.T) {
+	s := newSynthetic(8, -1, 11)
+	pairs, info, err := Search(s.corpus, s.taxa, s.params())
+	if err != nil {
+		t.Fatal(err)
+	}
+	p := find(pairs, s.f, s.r)
+	if p == nil {
+		t.Fatalf("the planted pair was not found among %d pairs (%+v)", len(pairs), info)
+	}
+	if p.Examples != 8 || p.MinLen != 40 || p.MaxLen != 40 || p.Bs != 1 || p.Bc != 1 || p.Bd != 1 {
+		t.Errorf("unexpected indices %+v", *p)
+	}
+	if p.RefLocation != "" {
+		t.Error("no reference was asked: no location expected")
+	}
+}
+
+func TestReferenceLocation(t *testing.T) {
+	forwardFirst, reverseFirst := 0, 0
+	for seed := int64(1); seed <= 24; seed++ {
+		s := newSynthetic(8, -1, seed)
+		p := s.params()
+		p.Reference = "s3"
+		pairs, _, err := Search(s.corpus, s.taxa, p)
+		if err != nil {
+			t.Fatal(err)
+		}
+		pair := find(pairs, s.f, s.r)
+		if pair == nil {
+			t.Fatalf("seed %d: planted pair not found", seed)
+		}
+
+		// 1-based sites: forward primer at 61..78, then 40 bases of insert, then the reverse site
+		wantLoc := "join(61..78,119..136)"
+		insert := strings.ToLower(string(s.corpus.Seqs[3].Sequence()[s.pos+18 : s.pos+58]))
+		if pair.Primer1 == s.f { // the amplicon is read from the first primer
+			forwardFirst++
+			if pair.RefLocation != wantLoc || pair.RefAmplicon != insert {
+				t.Errorf("seed %d, forward first: got %s %s", seed, pair.RefLocation, pair.RefAmplicon)
+			}
+		} else {
+			reverseFirst++
+			rc := strings.ToLower(revcomp(strings.ToUpper(insert)))
+			if pair.RefLocation != "complement("+wantLoc+")" || pair.RefAmplicon != rc {
+				t.Errorf("seed %d, reverse first: got %s %s", seed, pair.RefLocation, pair.RefAmplicon)
+			}
+		}
+		if len(pair.RefAmplicon) != pair.MinLen {
+			t.Errorf("seed %d: amplicon length %d, pair length %d", seed, len(pair.RefAmplicon), pair.MinLen)
+		}
+	}
+	if forwardFirst == 0 || reverseFirst == 0 {
+		t.Errorf("both orientations must be exercised: %d forward first, %d reverse first", forwardFirst, reverseFirst)
+	}
+}
+
+func TestReferenceErrors(t *testing.T) {
+	s := newSynthetic(8, -1, 11)
+	p := s.params()
+	p.Reference = "nope"
+	if _, _, err := Search(s.corpus, s.taxa, p); err == nil {
+		t.Error("an unknown reference must be refused")
+	}
+	s.corpus.Roles[2] = CounterExample
+	p.Reference = "s2"
+	if _, _, err := Search(s.corpus, s.taxa, p); err == nil {
+		t.Error("a counterexample cannot be the reference")
+	}
+}
+
+func TestReferenceOnACircularJunction(t *testing.T) {
+	// span writes a range running over the end of a circular sequence in two parts
+	if got := span(95, 10, 100, true); got != "96..100,1..5" {
+		t.Errorf("circular span %q", got)
+	}
+	if got := span(95, 10, 100, false); got != "96..105" {
+		t.Errorf("linear span %q", got)
+	}
+	if got := span(10, 18, 100, true); got != "11..28" {
+		t.Errorf("inner span %q", got)
+	}
+}
+
+func TestNoMultiMatchDropsRepeatedPrimers(t *testing.T) {
+	// the forward primer is found twice in sequence 2
+	s := newSynthetic(8, 2, 11)
+
+	with, infoWith, err := Search(s.corpus, s.taxa, s.params())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if find(with, s.f, s.r) == nil {
+		t.Fatalf("without -U the pair must be found (%+v)", infoWith)
+	}
+
+	p := s.params()
+	p.NoMultiMatch = true
+	without, info, err := Search(s.corpus, s.taxa, p)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if find(without, s.f, s.r) != nil {
+		t.Error("with -U the repeated forward primer must be discarded, and its pair with it")
+	}
+	if info.Repeated == 0 {
+		t.Errorf("the repeated word was not counted: %+v", info)
+	}
+	if len(without) >= len(with) {
+		t.Errorf("-U must not add pairs: %d with, %d without", len(with), len(without))
+	}
+}
+
+func TestDropMultiMatch(t *testing.T) {
+	wm := []WordMatches{
+		{Word: 1, Matches: []Match{{0, 5, 0, false}, {1, 9, 0, true}}},   // one per sequence: kept
+		{Word: 2, Matches: []Match{{0, 5, 0, false}, {0, 50, 1, false}}}, // two in sequence 0
+		{Word: 3, Matches: []Match{{2, 5, 0, false}, {2, 5, 0, true}}},   // both strands of one sequence
+		{Word: 4}, // no match at all: kept
+		{Word: 5, Matches: []Match{{3, 1, 0, false}, {3, 8, 0, false}}}, // twice, but in a non-target: kept
+	}
+	c := corpusOf([]Role{Example, Example, CounterExample, Other}, "a", "a", "a", "a")
+	kept, dropped := DropMultiMatch(wm, c)
+	if dropped != 2 || len(kept) != 3 || kept[0].Word != 1 || kept[1].Word != 4 || kept[2].Word != 5 {
+		t.Errorf("kept %v, dropped %d", kept, dropped)
+	}
+}
+
+func TestDropRepeatedWords(t *testing.T) {
+	words := []obikmer.SeqKmerCount{{Kmer: 1}, {Kmer: 2, Repeated: true}, {Kmer: 3}}
+	kept, dropped := DropRepeatedWords(words)
+	if dropped != 1 || len(kept) != 2 || kept[0].Kmer != 1 || kept[1].Kmer != 3 {
+		t.Errorf("kept %v, dropped %d", kept, dropped)
+	}
+}
+
+// A non-target amplified with the very barcode of a target taxon makes this
+// target badly identified, although its barcode is unique among the targets;
+// it counts in neither the sensitivity nor the false positive quorums.
+func TestNonTargetSharingABarcodeLowersBsAndBd(t *testing.T) {
+	s := newSynthetic(8, -1, 11)
+
+	alone, _, err := Search(s.corpus, s.taxa, s.params())
+	if err != nil {
+		t.Fatal(err)
+	}
+	p0 := find(alone, s.f, s.r)
+	if p0 == nil || p0.Bs != 1 || p0.Bd != 1 {
+		t.Fatalf("without the non-target: %+v", p0)
+	}
+
+	// a copy of sequence 3, of another taxon, that is not a target
+	copy3 := append([]byte(nil), s.corpus.Seqs[3].Sequence()...)
+	s.corpus.Seqs = append(s.corpus.Seqs, obiseq.NewBioSequence("other", copy3, ""))
+	s.corpus.Roles = append(s.corpus.Roles, Other)
+	s.taxa.Of = append(s.taxa.Of, 100)
+
+	pairs, _, err := Search(s.corpus, s.taxa, s.params())
+	if err != nil {
+		t.Fatal(err)
+	}
+	p := find(pairs, s.f, s.r)
+	if p == nil {
+		t.Fatal("the planted pair must still be found")
+	}
+	if p.Examples != 8 || p.ExampleTaxa != 8 || p.Bc != 1 {
+		t.Errorf("the non-target must not change the quorums nor Bc: %+v", *p)
+	}
+	if p.Identified != 7 || p.Bs != 0.875 {
+		t.Errorf("target 3 shares its barcode with the non-target: identified %d, Bs %v (want 7, 0.875)", p.Identified, p.Bs)
+	}
+	if p.Bd != 0.875 {
+		t.Errorf("Bd %v, want 7 discriminant barcodes of 8", p.Bd)
+	}
+	if p.CounterTaxa != 0 || p.CounterExamples != 0 {
+		t.Errorf("a non-target is not a counterexample: %+v", *p)
+	}
+
+	// Now counterexamples: two copies (of sequences 3 and 5) that the pair amplifies and
+	// one unrelated sequence that it does not. 2 of 3 amplified is under the 0.9 quorum.
+	s.corpus.Roles[len(s.corpus.Roles)-1] = CounterExample
+	copy5 := append([]byte(nil), s.corpus.Seqs[5].Sequence()...)
+	s.corpus.Seqs = append(s.corpus.Seqs, obiseq.NewBioSequence("counter5", copy5, ""))
+	s.corpus.Roles = append(s.corpus.Roles, CounterExample)
+	s.taxa.Of = append(s.taxa.Of, 101)
+	unrelated := []byte(strings.ToLower(randDNA(rand.New(rand.NewSource(5)), 300)))
+	s.corpus.Seqs = append(s.corpus.Seqs, obiseq.NewBioSequence("unrelated", unrelated, ""))
+	s.corpus.Roles = append(s.corpus.Roles, CounterExample)
+	s.taxa.Of = append(s.taxa.Of, 102)
+
+	pq := s.params()
+	pq.FalsePositive = 0.9
+	pairs, _, err = Search(s.corpus, s.taxa, pq)
+	if err != nil {
+		t.Fatal(err)
+	}
+	pc := find(pairs, s.f, s.r)
+	if pc == nil {
+		t.Fatal("the pair amplifies 2 of 3 counterexamples (< 0.9): it must be retained")
+	}
+	if pc.CounterExamples != 2 || pc.CounterTaxa != 2 {
+		t.Errorf("counterexamples amplified %d (taxa %d), want 2 and 2", pc.CounterExamples, pc.CounterTaxa)
+	}
+	if pc.Identified != 6 || pc.Bd != 0.75 {
+		t.Errorf("targets 3 and 5 share their barcode with a counterexample: identified %d (want 6), Bd %v (want 0.75)",
+			pc.Identified, pc.Bd)
+	}
+}

pkg/obiprimer/taxa.go

@@ -0,0 +1,83 @@
+package obiprimer
+
+import (
+	"fmt"
+	"slices"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax"
+)
+
+// Taxa gives every corpus sequence its taxon at one rank (the species by
+// default in ecoPrimers), as a dense index, and the number of distinct taxa
+// among the examples and among the counterexamples.
+//
+// A sequence whose taxon has no ancestor at that rank (or no taxon at all) has
+// the index -1; it is never counted as a taxon.
+type Taxa struct {
+	Rank            string
+	Of              []int32 // Of[i] is the taxon index of Corpus.Seqs[i], or -1
+	ExampleTaxa     int
+	CounterExamples int // distinct taxa among the counterexamples
+	nodes           []*obitax.Taxon
+}
+
+// Name returns the taxon with the given index, for reporting.
+func (t *Taxa) Name(i int32) *obitax.Taxon { return t.nodes[i] }
+
+// checkRank reports an error if the taxonomy has no such rank.
+func checkRank(taxonomy *obitax.Taxonomy, rank string) error {
+	if taxonomy == nil {
+		return fmt.Errorf("obiprimer: a taxonomy is required to compute taxon statistics")
+	}
+	if !slices.Contains(taxonomy.RankList(), rank) {
+		return fmt.Errorf("obiprimer: unknown taxonomic rank %q", rank)
+	}
+	return nil
+}
+
+// AssignTaxa determines the taxon of every sequence of the corpus at rank. The
+// sequences must not be packed (see LoadPackedCorpus).
+func (c *Corpus) AssignTaxa(taxonomy *obitax.Taxonomy, rank string) (*Taxa, error) {
+	if err := checkRank(taxonomy, rank); err != nil {
+		return nil, err
+	}
+	taxons := make([]*obitax.Taxon, len(c.Seqs))
+	for i, seq := range c.Seqs {
+		taxons[i] = seq.Taxon(taxonomy).TaxonAtRank(rank)
+	}
+	return buildTaxa(rank, taxons, c.Roles)
+}
+
+// buildTaxa gives the sequences, whose taxon at rank is taxons[i] (nil when
+// there is none), their dense taxon index.
+func buildTaxa(rank string, taxons []*obitax.Taxon, roles []Role) (*Taxa, error) {
+	t := &Taxa{Rank: rank, Of: make([]int32, len(taxons))}
+	index := map[*obitax.TaxNode]int32{}
+	exSeen := map[int32]struct{}{}
+	coSeen := map[int32]struct{}{}
+
+	for i, taxon := range taxons {
+		t.Of[i] = -1
+		if taxon == nil {
+			continue
+		}
+		id, ok := index[taxon.Node]
+		if !ok {
+			if len(t.nodes) >= maxTaxa {
+				return nil, fmt.Errorf("obiprimer: more than %d taxa at rank %q", maxTaxa, rank)
+			}
+			id = int32(len(t.nodes))
+			index[taxon.Node] = id
+			t.nodes = append(t.nodes, taxon)
+		}
+		t.Of[i] = id
+		switch roles[i] {
+		case Example:
+			exSeen[id] = struct{}{}
+		case CounterExample:
+			coSeen[id] = struct{}{}
+		}
+	}
+	t.ExampleTaxa, t.CounterExamples = len(exSeen), len(coSeen)
+	return t, nil
+}

pkg/obiprimer/testdata/ecoprimers_Cyprinidae_I1.pairs.tsv

@@ -0,0 +1,173 @@
+AGCAATCCAGGTCAGTTT	TGCCACCCCAGTATGTCC	bG	309	454	301	452	197
+GCAATCCAGGTCAGTTTC	TGCCACCCCAGTATGTCC	bG	309	455	301	453	197
+AGCAATCCAGGTCAGTTT	CCACCCCAGTATGTCCTT	bG	311	484	302	482	197
+CCACCCCAGTATGTCCTT	GCAATCCAGGTCAGTTTC	Gb	311	485	302	483	197
+AGCAATCCAGGTCAGTTT	GCCACCCCAGTATGTCCT	bG	308	445	300	444	196
+GCAATCCAGGTCAGTTTC	GCCACCCCAGTATGTCCT	bG	308	446	300	445	196
+ACCGGAGCAATCCAGGTC	TGCCACCCCAGTATGTCC	bG	308	453	300	451	196
+GAGCAATCCAGGTCAGTT	TGCCACCCCAGTATGTCC	bG	308	453	300	451	196
+AGACCGGAGCAATCCAGG	CCACCCCAGTATGTCCTT	bG	310	483	301	481	196
+CCACCCCAGTATGTCCTT	GAGCAATCCAGGTCAGTT	Gb	310	483	301	481	196
+ACCGGAGCAATCCAGGTC	CCACCCCAGTATGTCCTT	bG	310	484	301	482	196
+CCACCCCAGTATGTCCTT	GACCGGAGCAATCCAGGT	Gb	310	484	301	482	196
+CCACCCCAGTATGTCCTT	GGTCAGTTTCTATCTGTA	Gb	309	460	300	459	195
+AGGTCAGTTTCTATCTGT	TGCCACCCCAGTATGTCC	bG	307	438	299	437	195
+CAGGTCAGTTTCTATCTG	TGCCACCCCAGTATGTCC	bG	307	440	299	439	195
+GGTCAGTTTCTATCTGTA	TGCCACCCCAGTATGTCC	bG	307	441	299	440	195
+GAGCAATCCAGGTCAGTT	GCCACCCCAGTATGTCCT	bG	307	444	299	443	195
+ACCGGAGCAATCCAGGTC	GCCACCCCAGTATGTCCT	bG	307	445	299	444	195
+GACCGGAGCAATCCAGGT	GCCACCCCAGTATGTCCT	bG	307	445	299	444	195
+AGGTCAGTTTCTATCTGT	CCACCCCAGTATGTCCTT	bG	309	469	300	467	195
+CAGGTCAGTTTCTATCTG	CCACCCCAGTATGTCCTT	bG	309	470	300	468	195
+GGAGCAATCCAGGTCAGT	TGCCACCCCAGTATGTCC	bG	307	454	299	452	195
+CCAGGTCAGTTTCTATCT	TGCCACCCCAGTATGTCC	bG	307	455	299	453	195
+AATCCAGGTCAGTTTCTA	TGCCACCCCAGTATGTCC	bG	307	455	299	453	195
+CAATCCAGGTCAGTTTCT	TGCCACCCCAGTATGTCC	bG	307	455	299	453	195
+ATCCAGGTCAGTTTCTAT	TGCCACCCCAGTATGTCC	bG	307	455	299	453	195
+TCCAGGTCAGTTTCTATC	TGCCACCCCAGTATGTCC	bG	307	455	299	453	195
+CCACCCCAGTATGTCCTT	GGAGCAATCCAGGTCAGT	Gb	309	484	300	482	195
+CAATCCAGGTCAGTTTCT	CCACCCCAGTATGTCCTT	bG	309	485	300	483	195
+AATCCAGGTCAGTTTCTA	CCACCCCAGTATGTCCTT	bG	309	485	300	483	195
+ATCCAGGTCAGTTTCTAT	CCACCCCAGTATGTCCTT	bG	309	485	300	483	195
+CCACCCCAGTATGTCCTT	TCCAGGTCAGTTTCTATC	Gb	309	485	300	483	195
+CCACCCCAGTATGTCCTT	CCAGGTCAGTTTCTATCT	Gb	309	485	300	483	195
+GCCACCCCAGTATGTCCT	GGTCAGTTTCTATCTGTA	Gb	306	437	298	437	194
+AGGTCAGTTTCTATCTGT	GCCACCCCAGTATGTCCT	bG	306	440	298	439	194
+CAGGTCAGTTTCTATCTG	GCCACCCCAGTATGTCCT	bG	306	442	298	441	194
+GCCACCCCAGTATGTCCT	GGAGCAATCCAGGTCAGT	Gb	306	445	298	444	194
+GCCACCCCAGTATGTCCT	TCCAGGTCAGTTTCTATC	Gb	306	446	298	445	194
+CCAGGTCAGTTTCTATCT	GCCACCCCAGTATGTCCT	bG	306	446	298	445	194
+CAATCCAGGTCAGTTTCT	GCCACCCCAGTATGTCCT	bG	306	446	298	445	194
+AATCCAGGTCAGTTTCTA	GCCACCCCAGTATGTCCT	bG	306	446	298	445	194
+ATCCAGGTCAGTTTCTAT	GCCACCCCAGTATGTCCT	bG	306	446	298	445	194
+CGGAGCAATCCAGGTCAG	TGCCACCCCAGTATGTCC	bG	304	445	296	444	192
+CCGGAGCAATCCAGGTCA	TGCCACCCCAGTATGTCC	bG	304	445	296	444	192
+TGCCACCCCAGTATGTCC	TTTTCCTAGTACGAAAGG	Gb	308	453	300	452	192
+CCACCCCAGTATGTCCTT	CCGGAGCAATCCAGGTCA	Gb	306	476	297	474	192
+CCACCCCAGTATGTCCTT	CGGAGCAATCCAGGTCAG	Gb	306	476	297	474	192
+GACCGGAGCAATCCAGGT	TGCCACCCCAGTATGTCC	bG	304	453	296	451	192
+CCACCCCAGTATGTCCTT	GTCAGTTTCTATCTGTAA	Gb	304	408	295	407	190
+CCACCCCAGTATGTCCTT	TTTTCCTAGTACGAAAGG	Gb	310	484	301	482	192
+CAGACCGGAGCAATCCAG	CCACCCCAGTATGTCCTT	bG	306	484	297	482	192
+GTCAGTTTCTATCTGTAA	TGCCACCCCAGTATGTCC	bG	302	390	294	390	190
+AGCAATCCAGGTCAGTTT	CCCAGTATGTCCTTATTT	bG	301	310	292	309	187
+CCCAGTATGTCCTTATTT	GCAATCCAGGTCAGTTTC	Gb	301	311	292	310	187
+ACTTTTCCTAGTACGAAA	TGCCACCCCAGTATGTCC	bG	307	443	299	442	191
+GCCACCCCAGTATGTCCT	TTTTCCTAGTACGAAAGG	Gb	307	445	299	444	191
+CGGAGCAATCCAGGTCAG	GCCACCCCAGTATGTCCT	bG	303	443	295	442	191
+CCGGAGCAATCCAGGTCA	GCCACCCCAGTATGTCCT	bG	303	443	295	442	191
+CTTTTCCTAGTACGAAAG	TGCCACCCCAGTATGTCC	bG	307	448	299	447	191
+AGACCGGAGCAATCCAGG	GCCACCCCAGTATGTCCT	bG	303	445	295	444	191
+ACTTTTCCTAGTACGAAA	CCACCCCAGTATGTCCTT	bG	309	478	300	476	191
+CCACCCCAGTATGTCCTT	CTTTTCCTAGTACGAAAG	Gb	309	482	300	480	191
+GCCACCCCAGTATGTCCT	GTCAGTTTCTATCTGTAA	Gb	301	390	293	390	189
+AGACCGGAGCAATCCAGG	CCCAGTATGTCCTTATTT	bG	300	310	291	309	186
+CAGACCGGAGCAATCCAG	CCCAGTATGTCCTTATTT	bG	300	310	291	309	186
+CCCAGTATGTCCTTATTT	GAGCAATCCAGGTCAGTT	Gb	300	310	291	309	186
+ACCGGAGCAATCCAGGTC	CCCAGTATGTCCTTATTT	bG	300	311	291	310	186
+CCCAGTATGTCCTTATTT	GACCGGAGCAATCCAGGT	Gb	300	311	291	310	186
+CCCAGTATGTCCTTATTT	TCAGACCGGAGCAATCCA	Gb	300	311	291	310	186
+CCCAGTATGTCCTTATTT	TTCAGACCGGAGCAATCC	Gb	300	311	291	310	186
+CCCAGTATGTCCTTATTT	GTTCAGACCGGAGCAATC	Gb	300	311	291	310	186
+CTTTTCCTAGTACGAAAG	GCCACCCCAGTATGTCCT	bG	306	443	298	442	190
+ACTTTTCCTAGTACGAAA	GCCACCCCAGTATGTCCT	bG	306	443	298	442	190
+AGACCGGAGCAATCCAGG	TGCCACCCCAGTATGTCC	bG	302	453	294	451	190
+CCACCCCAGTATGTCCTT	TCAGACCGGAGCAATCCA	Gb	304	485	295	483	190
+AGGTCAGTTTCTATCTGT	CCCAGTATGTCCTTATTT	bG	299	309	290	308	185
+CAGGTCAGTTTCTATCTG	CCCAGTATGTCCTTATTT	bG	299	309	290	308	185
+CCCAGTATGTCCTTATTT	GGTCAGTTTCTATCTGTA	Gb	299	309	290	308	185
+ATCCAGGTCAGTTTCTAT	CCCAGTATGTCCTTATTT	bG	299	311	290	310	185
+CCAGGTCAGTTTCTATCT	CCCAGTATGTCCTTATTT	bG	299	311	290	310	185
+CCCAGTATGTCCTTATTT	GGAGCAATCCAGGTCAGT	Gb	299	311	290	310	185
+CCCAGTATGTCCTTATTT	TCCAGGTCAGTTTCTATC	Gb	299	311	290	310	185
+CAATCCAGGTCAGTTTCT	CCCAGTATGTCCTTATTT	bG	299	311	290	310	185
+AATCCAGGTCAGTTTCTA	CCCAGTATGTCCTTATTT	bG	299	311	290	310	185
+TGCCACCCCAGTATGTCC	TTTCCTAGTACGAAAGGA	Gb	305	449	297	447	189
+CAGACCGGAGCAATCCAG	GCCACCCCAGTATGTCCT	bG	301	446	293	445	189
+CCACCCCAGTATGTCCTT	TTTCCTAGTACGAAAGGA	Gb	307	479	298	477	189
+CCACCCCAGTATGTCCTT	TACGAAAGGATCGGAAAA	Gb	307	453	298	452	188
+CCCAGTATGTCCTTATTT	CCGGAGCAATCCAGGTCA	Gb	298	309	289	308	184
+CCCAGTATGTCCTTATTT	CGGAGCAATCCAGGTCAG	Gb	298	309	289	308	184
+GCCACCCCAGTATGTCCT	TTTCCTAGTACGAAAGGA	Gb	304	440	296	439	188
+GCAATTTACCATGCTCTG	TTTTCCTAGTACGAAAGG	Gb	308	505	299	501	189
+ACTTTTCCTAGTACGAAA	GCAATTTACCATGCTCTG	bG	307	478	298	474	188
+CAGACCGGAGCAATCCAG	TGCCACCCCAGTATGTCC	bG	300	454	292	452	188
+CCTAGTACGAAAGGATCG	TGCCACCCCAGTATGTCC	bG	303	427	295	425	187
+CCACCCCAGTATGTCCTT	TTCAGACCGGAGCAATCC	Gb	302	485	293	483	188
+TACGAAAGGATCGGAAAA	TGCCACCCCAGTATGTCC	bG	304	430	296	430	187
+GCCACCCCAGTATGTCCT	TACGAAAGGATCGGAAAA	Gb	304	430	296	430	187
+CCACCCCAGTATGTCCTT	CCTAGTACGAAAGGATCG	Gb	305	459	296	457	187
+CTTTTCCTAGTACGAAAG	GCAATTTACCATGCTCTG	bG	307	498	298	494	188
+CGCAATTTACCATGCTCT	TTTTCCTAGTACGAAAGG	Gb	306	463	297	459	187
+GCAATTTACCATGCTCTG	TTTCCTAGTACGAAAGGA	Gb	307	506	298	502	188
+GCAATTTACCATGCTCTG	GGTCAGTTTCTATCTGTA	Gb	301	433	292	429	186
+GCCACCCCAGTATGTCCT	TCAGACCGGAGCAATCCA	Gb	299	446	291	445	187
+CCTAGTACGAAAGGATCG	GCCACCCCAGTATGTCCT	bG	302	419	294	418	186
+CCCAGTATGTCCTTATTT	GTCAGTTTCTATCTGTAA	Gb	295	264	286	263	181
+ACTTTTCCTAGTACGAAA	CGCAATTTACCATGCTCT	bG	305	449	296	445	186
+TCGCAATTTACCATGCTC	TTTTCCTAGTACGAAAGG	Gb	305	456	296	452	186
+TTCGCAATTTACCATGCT	TTTTCCTAGTACGAAAGG	Gb	305	456	296	452	186
+AGTTCAGACCGGAGCAAT	CCCAGTATGTCCTTATTT	bG	296	310	287	309	182
+CGCAATTTACCATGCTCT	CTTTTCCTAGTACGAAAG	Gb	305	460	296	456	186
+CGCAATTTACCATGCTCT	TTTCCTAGTACGAAAGGA	Gb	305	463	296	459	186
+CCTAGTACGAAAGGATCG	GCAATTTACCATGCTCTG	bG	305	472	296	468	186
+TCAGACCGGAGCAATCCA	TGCCACCCCAGTATGTCC	bG	298	454	290	452	186
+ACTTTTCCTAGTACGAAA	TCGCAATTTACCATGCTC	bG	304	447	295	443	185
+ACTTTTCCTAGTACGAAA	TTCGCAATTTACCATGCT	bG	304	447	295	443	185
+CTTTTCCTAGTACGAAAG	TCGCAATTTACCATGCTC	bG	304	453	295	449	185
+CTTTTCCTAGTACGAAAG	TTCGCAATTTACCATGCT	bG	304	453	295	449	185
+CCACCCCAGTATGTCCTT	GTTCAGACCGGAGCAATC	Gb	300	485	291	483	186
+TCGCAATTTACCATGCTC	TTTCCTAGTACGAAAGGA	Gb	304	456	295	452	185
+TTTCGCAATTTACCATGC	TTTTCCTAGTACGAAAGG	Gb	304	456	295	452	185
+TTCGCAATTTACCATGCT	TTTCCTAGTACGAAAGGA	Gb	304	456	295	452	185
+GCAATTTACCATGCTCTG	TACGAAAGGATCGGAAAA	Gb	304	427	295	425	184
+CCTAGTACGAAAGGATCG	CGCAATTTACCATGCTCT	bG	303	433	294	429	184
+AGGTCAGTTTCTATCTGT	GCAATTTACCATGCTCTG	bG	299	431	290	427	184
+GCCACCCCAGTATGTCCT	TTCAGACCGGAGCAATCC	Gb	297	446	289	445	185
+ACTTTTCCTAGTACGAAA	TTTCGCAATTTACCATGC	bG	303	447	294	443	184
+GCAATTTACCATGCTCTG	GTCAGTTTCTATCTGTAA	Gb	295	375	286	373	182
+CTTTTCCTAGTACGAAAG	TTTCGCAATTTACCATGC	bG	303	453	294	449	184
+TTTCCTAGTACGAAAGGA	TTTCGCAATTTACCATGC	bG	303	456	294	452	184
+CCCAGTATGTCCTTATTT	TTTTCCTAGTACGAAAGG	Gb	300	311	291	310	180
+CCCAGTATGTCCTTATTT	GAGTTCAGACCGGAGCAA	Gb	294	310	285	309	180
+CCTAGTACGAAAGGATCG	TCGCAATTTACCATGCTC	bG	302	430	293	426	183
+CCTAGTACGAAAGGATCG	TTCGCAATTTACCATGCT	bG	302	430	293	426	183
+CGCAATTTACCATGCTCT	TACGAAAGGATCGGAAAA	Gb	302	420	293	418	182
+ACTTTTCCTAGTACGAAA	CCCAGTATGTCCTTATTT	bG	299	311	290	310	179
+CCCAGTATGTCCTTATTT	CTTTTCCTAGTACGAAAG	Gb	299	311	290	310	179
+CGCAATTTACCATGCTCT	GGTCAGTTTCTATCTGTA	Gb	297	426	288	422	182
+CCTAGTACGAAAGGATCG	TTTCGCAATTTACCATGC	bG	301	430	292	426	182
+CAGGTCAGTTTCTATCTG	GCAATTTACCATGCTCTG	bG	297	431	288	427	182
+TACGAAAGGATCGGAAAA	TTCGCAATTTACCATGCT	bG	301	418	292	416	181
+TACGAAAGGATCGGAAAA	TCGCAATTTACCATGCTC	bG	301	418	292	416	181
+GGCCTTTCGCAATTTACC	TTTTCCTAGTACGAAAGG	Gb	302	510	293	506	183
+ACTTTTCCTAGTACGAAA	GGCCTTTCGCAATTTACC	bG	301	479	292	475	182
+TACGAAAGGATCGGAAAA	TTTCGCAATTTACCATGC	bG	300	418	291	416	180
+CCCAGTATGTCCTTATTT	TTTCCTAGTACGAAAGGA	Gb	297	310	288	309	177
+CTTTTCCTAGTACGAAAG	GGCCTTTCGCAATTTACC	bG	301	500	292	496	182
+AGGTCAGTTTCTATCTGT	CGCAATTTACCATGCTCT	bG	295	428	286	424	180
+GGCCTTTCGCAATTTACC	TTTCCTAGTACGAAAGGA	Gb	301	511	292	507	182
+GCCTTTCGCAATTTACCA	TTTTCCTAGTACGAAAGG	Gb	299	440	290	436	180
+CTTTCGCAATTTACCATG	TTTTCCTAGTACGAAAGG	Gb	298	412	289	408	179
+CCCAGTATGTCCTTATTT	CCTAGTACGAAAGGATCG	Gb	296	305	287	304	176
+CCCAGTATGTCCTTATTT	TACGAAAGGATCGGAAAA	Gb	297	307	288	306	176
+GGTCAGTTTCTATCTGTA	TCGCAATTTACCATGCTC	bG	294	425	285	421	179
+ACTTTTCCTAGTACGAAA	GCCTTTCGCAATTTACCA	bG	298	436	289	432	179
+CTTTTCCTAGTACGAAAG	GCCTTTCGCAATTTACCA	bG	298	438	289	434	179
+GCCTTTCGCAATTTACCA	TTTCCTAGTACGAAAGGA	Gb	298	440	289	436	179
+CCTAGTACGAAAGGATCG	GGCCTTTCGCAATTTACC	bG	299	484	290	480	180
+ACTTTTCCTAGTACGAAA	CTTTCGCAATTTACCATG	bG	297	411	288	407	178
+CTTTCGCAATTTACCATG	CTTTTCCTAGTACGAAAG	Gb	297	411	288	407	178
+CTTTCGCAATTTACCATG	TTTCCTAGTACGAAAGGA	Gb	297	412	288	408	178
+GGCCTTTCGCAATTTACC	TACGAAAGGATCGGAAAA	Gb	298	425	289	423	178
+AGTTCAGACCGGAGCAAT	CCACCCCAGTATGTCCTT	bG	293	484	284	482	179
+CCTAGTACGAAAGGATCG	GCCTTTCGCAATTTACCA	bG	296	416	287	412	177
+CCTAGTACGAAAGGATCG	CTTTCGCAATTTACCATG	bG	295	390	286	386	176
+CCAGGTCAGTTTCTATCT	GCAATTTACCATGCTCTG	bG	294	513	285	509	179
+CCTTTCGCAATTTACCAT	TTTTCCTAGTACGAAAGG	Gb	295	411	286	407	176
+GCAATTTACCATGCTCTG	TTCCTAGTACGAAAGGAT	Gb	293	423	284	419	176
+CTTTCGCAATTTACCATG	TACGAAAGGATCGGAAAA	Gb	295	398	286	396	175
+ACTTTTCCTAGTACGAAA	CCTTTCGCAATTTACCAT	bG	294	409	285	405	175
+CCTTTCGCAATTTACCAT	CTTTTCCTAGTACGAAAG	Gb	294	409	285	405	175
+CCTTTCGCAATTTACCAT	TTTCCTAGTACGAAAGGA	Gb	294	410	285	406	175
+GCCTTTCGCAATTTACCA	TACGAAAGGATCGGAAAA	Gb	295	413	286	411	175

pkg/obiprimer/testdata/ecoprimers_Teleostei.words.tsv

@@ -0,0 +1,255 @@
+AAAACTCGTGCCAGCCAC	3891	0
+AAACCCAAAGGACTTGGC	3890	0
+AAACGAACCCTTAATAGC	3875	0
+AAACTCGTGCCAGCCACC	3894	0
+AAACTGGGATTAGATACC	3896	0
+AAAGTCCTACGTGATCTG	3901	0
+AAATCATGATGCAAAAGG	3894	0
+AACAAACGAACCCTTAAT	3888	0
+AACATCGAGGTCGTAAAC	3893	0
+AACATGGTAAGTGTACCG	3894	0
+AACCAAAAACTCTTGGTG	3854	0
+AACCCAAAGGACTTGGCG	3888	0
+AACCCTTAATAGCGGCTG	3858	0
+AACCGCGGTGGCTGGCAC	3896	0
+AACGAACCCTTAATAGCG	3860	0
+AACGATTAAAGTCCTACG	3700	0
+AACGTCAGGTCGAGGTGT	3894	0
+AACTCAGATCACGTAGGA	3899	0
+AACTCGTGCCAGCCACCG	3893	0
+AACTGGGATTAGATACCC	3896	0
+AAGAGTTTTTGGTTCCTA	3848	0
+AAGCCTCGCCTGTTTACC	3876	0
+AAGGGTTCGTTTGTTCAA	3889	0
+AAGGTAGCGCAATCACTT	3889	0
+AAGTCCTACGTGATCTGA	3900	0
+AAGTCGTAACATGGTAAG	3893	0
+AAGTGCACCTTCCGGTAC	3888	0
+AAGTGTACCGGAAGGTGC	3889	0
+AATAGCGGCTGCACCATT	3839	0
+AATCACTTGTCTTTTAAA	3856	0
+AATCATGATGCAAAAGGT	3893	0
+AATCCAGGTCAGTTTCTA	3903	0
+AATCGTTGAACAAACGAA	3876	0
+AATGTAGCCCATTTCTTC	3871	0
+ACAAACGAACCCTTAATA	3885	0
+ACAAACTGGGATTAGATA	3893	0
+ACAAGTGATTGCGCTACC	3889	0
+ACACACCGCCCGTCACTC	3891	0
+ACACCGCCCGTCACTCTC	3890	0
+ACACTTACCATGTTACGA	3893	0
+ACATCCTAATGGTGCAGC	3858	0
+ACATCGAGGTCGTAAACC	3894	0
+ACATGGTAAGTGTACCGG	3894	0
+ACCAAAAACTCTTGGTGC	3875	0
+ACCAAGAGTTTTTGGTTC	3857	0
+ACCATGTTACGACTTGCC	3891	0
+ACCATTAGGATGTCCTGA	3891	0
+ACCCAAAGGACTTGGCGG	3886	0
+ACCCGTCTCTGTGGCAAA	3892	0
+ACCCTAGGGATAACAGCG	3895	0
+ACCCTTAATAGCGGCTGC	3861	0
+ACCGCCCGTCACTCTCCC	3893	0
+ACCGCGGTGGCTGGCACG	3895	0
+ACCGGAAGGTGCACTTGG	3887	0
+ACCTCGATGTTGGATCAG	3891	0
+ACCTTCCGGTACACTTAC	3893	0
+ACGAACCCTTAATAGCGG	3859	0
+ACGACCTCGATGTTGGAT	3889	0
+ACGATTAAAGTCCTACGT	3701	0
+ACGTCAGGTCGAGGTGTA	3894	0
+ACGTGATCTGAGTTCAGA	3901	0
+ACTCAGATCACGTAGGAC	3899	0
+ACTCGTGCCAGCCACCGC	3894	0
+ACTCTTTTGCCACAGAGA	3867	0
+ACTGGGATTAGATACCCC	3896	0
+ACTTACCATGTTACGACT	3894	0
+ACTTTAATCGTTGAACAA	3876	0
+AGAACAGGCTCCTCTAGG	3893	0
+AGACAAGTGATTGCGCTA	3883	0
+AGAGGAGCCTGTTCTAGA	3876	0
+AGAGTGACGGGCGGTGTG	3890	0
+AGAGTTTTTGGTTCCTAA	3851	0
+AGATAGAAACTGACCTGG	3900	0
+AGATCACGTAGGACTTTA	3901	0
+AGCCGCTATTAAGGGTTC	3856	0
+AGCCTCGCCTGTTTACCA	3889	0
+AGCCTGTTCTAGAACCGA	3871	0
+AGCGGCTGCACCATTAGG	3846	0
+AGGAACCAAAAACTCTTG	3842	0
+AGGACATCCTAATGGTGC	3866	0
+AGGACTTTAATCGTTGAA	3688	0
+AGGAGCCTGTTCTAGAAC	3874	0
+AGGATGTCCTGATCCAAC	3897	0
+AGGCAAGTCGTAACATGG	3868	0
+AGGCGATGTTTTTGGTAA	3894	0
+AGGGATAACAGCGCAATC	3896	0
+AGGGGAGGCAAGTCGTAA	3890	0
+AGGGGGTTTACGACCTCG	3894	0
+AGGGTTCGTTTGTTCAAC	3888	0
+AGGTAGCGCAATCACTTG	3888	0
+AGTCCTACGTGATCTGAG	3900	0
+AGTGGGGTATCTAATCCC	3896	0
+AGTTACCCTAGGGATAAC	3895	0
+AGTTTTTGGTTCCTAAGA	3848	0
+ATAGAAACTGACCTGGAT	3901	0
+ATAGCGGCTGCACCATTA	3837	0
+ATAGTGGGGTATCTAATC	3895	0
+ATCACGTAGGACTTTAAT	3880	0
+ATCACTTGTCTTTTAAAT	3856	0
+ATCAGGACATCCTAATGG	3892	0
+ATCATGATGCAAAAGGTA	3894	0
+ATCCTAATGGTGCAGCCG	3852	0
+ATCGAGGTCGTAAACCCC	3894	0
+ATCGGTTCTAGAACAGGC	3862	0
+ATCGTTGAACAAACGAAC	3877	0
+ATCTGAGTTCAGACCGGA	3902	0
+ATGATGCAAAAGGTACGA	3891	0
+ATGGTAAGTGTACCGGAA	3895	0
+ATGGTGCAGCCGCTATTA	3840	0
+ATGTAGCCCATTTCTTCC	3871	0
+ATGTCCTGATCCAACATC	3892	0
+ATGTTACGACTTGCCTCC	3864	0
+ATGTTGGATCAGGACATC	3893	0
+ATGTTTTTGGTAAACAGG	3887	0
+ATTAGATACCCCACTATG	3896	0
+ATTAGGATGTCCTGATCC	3893	0
+ATTGCGCTGTTATCCCTA	3896	0
+CAAAAACTCTTGGTGCAA	3886	0
+CAAACGAACCCTTAATAG	3882	0
+CAAACTGGGATTAGATAC	3895	0
+CAACATCGAGGTCGTAAA	3894	0
+CAACGATTAAAGTCCTAC	3691	0
+CAAGTCGTAACATGGTAA	3893	0
+CAAGTGCACCTTCCGGTA	3888	0
+CACCATTAGGATGTCCTG	3890	0
+CACCGCCCGTCACTCTCC	3888	0
+CACCTTCCGGTACACTTA	3891	0
+CACGTAGGACTTTAATCG	3702	0
+CACTCTTTTGCCACAGAG	3856	0
+CACTTACCATGTTACGAC	3894	0
+CATCCTAATGGTGCAGCC	3852	0
+CATCGAGGTCGTAAACCC	3894	0
+CATGATGCAAAAGGTACG	3890	0
+CATGGTAAGTGTACCGGA	3894	0
+CATGTTACGACTTGCCTC	3866	0
+CATTAGGATGTCCTGATC	3893	0
+CCAAAAACTCTTGGTGCA	3883	0
+CCAACATCGAGGTCGTAA	3894	0
+CCAAGAGTTTTTGGTTCC	3844	0
+CCACTCTTTTGCCACAGA	3879	0
+CCCACTCTTTTGCCACAG	3879	0
+CCCGTCTCTGTGGCAAAA	3894	0
+CCCTAGGGATAACAGCGC	3898	0
+CCCTTAATAGCGGCTGCA	3853	0
+CCGCCCGTCACTCTCCCC	3890	0
+CCGCGGTGGCTGGCACGA	3895	0
+CCGGTAAAACTCGTGCCA	3847	0
+CCGGTCTGAACTCAGATC	3904	0
+CCGTCTCTGTGGCAAAAG	3894	0
+CCTACGTGATCTGAGTTC	3899	0
+CCTAGGGATAACAGCGCA	3896	0
+CCTCGATGTTGGATCAGG	3892	0
+CCTCGCCTGTTTACCAAA	3892	0
+CCTGTTCTAGAACCGATA	3862	0
+CCTTAATAGCGGCTGCAC	3850	0
+CCTTCCGGTACACTTACC	3893	0
+CGAACCCTTAATAGCGGC	3855	0
+CGACCTCGATGTTGGATC	3890	0
+CGACGGCGGTATATAGGC	3890	0
+CGATGTTGGATCAGGACA	3892	0
+CGATGTTTTTGGTAAACA	3886	0
+CGCCTGTTTACCAAAAAC	3889	0
+CGCGGTGGCTGGCACGAG	3894	0
+CGCTTAGCTGTTAACTAA	3864	0
+CGGTAAAACTCGTGCCAG	3856	0
+CGGTCTGAACTCAGATCA	3904	0
+CGGTTCTAGAACAGGCTC	3873	0
+CGTAACATGGTAAGTGTA	3892	0
+CGTCAGGTCGAGGTGTAG	3893	0
+CGTCTCTGTGGCAAAAGA	3891	0
+CGTGATCTGAGTTCAGAC	3902	0
+CGTTGAACAAACGAACCC	3887	0
+CGTTTGTTCAACGATTAA	3877	0
+CTAAATCATGATGCAAAA	3896	0
+CTAATGGTGCAGCCGCTA	3840	0
+CTACGTGATCTGAGTTCA	3899	0
+CTAGAACAGGCTCCTCTA	3877	0
+CTAGAGGAGCCTGTTCTA	3891	0
+CTAGGGATAACAGCGCAA	3897	0
+CTCCGGTCTGAACTCAGA	3903	0
+CTCGATGTTGGATCAGGA	3892	0
+CTCGCCTGTTTACCAAAA	3892	0
+CTCGTACCTTTTGCATCA	3889	0
+CTCTTTTGCCACAGAGAC	3869	0
+CTGAACTCAGATCACGTA	3900	0
+CTGCACCATTAGGATGTC	3861	0
+CTGGGATTAGATACCCCA	3895	0
+CTGTTATCCCTAGGGTAA	3897	0
+CTGTTTACCAAAAACATC	3886	0
+CTTAATAGCGGCTGCACC	3848	0
+CTTAGCTGTTAACTAAGA	3878	0
+CTTAGGAACCAAAAACTC	3847	0
+CTTAGTTAACAGCTAAGC	3875	0
+CTTCCGGTACACTTACCA	3891	0
+CTTGGATTTGCACCAAGA	3899	0
+CTTTAATCGTTGAACAAA	3876	0
+CTTTTGCATCATGATTTA	3896	0
+GAACAAACGAACCCTTAA	3890	0
+GAAGGAACTCGGCAAACA	3889	0
+GACAAGTGATTGCGCTAC	3889	0
+GACCTCGATGTTGGATCA	3890	0
+GACTTTAATCGTTGAACA	3686	0
+GAGGAGCCTGTTCTAGAA	3876	0
+GAGGCGATGTTTTTGGTA	3893	0
+GATAGAAACTGACCTGGA	3900	0
+GATCACGTAGGACTTTAA	3900	0
+GATTAAAGTCCTACGTGA	3704	0
+GCAAGTCGTAACATGGTA	3891	0
+GCATAGTGGGGTATCTAA	3896	0
+GCCGGTAAAACTCGTGCC	3827	0
+GCCTCGCCTGTTTACCAA	3890	0
+GCCTGTTTACCAAAAACA	3889	0
+GCGATGTTTTTGGTAAAC	3888	0
+GCGCTTAGCTGTTAACTA	3899	0
+GCGGCTGCACCATTAGGA	3850	0
+GCTAAATCATGATGCAAA	3893	0
+GCTACACCTCGACCTGAC	3893	0
+GCTGGCACGAGTTTTACC	3881	0
+GCTGTTATCCCTAGGGTA	3896	0
+GGACATCCTAATGGTGCA	3865	0
+GGACTTTAATCGTTGAAC	3687	0
+GGAGCCTGTTCTAGAACC	3874	0
+GGATGTCCTGATCCAACA	3895	0
+GGATTAGATACCCCACTA	3896	0
+GGATTGCGCTGTTATCCC	3895	0
+GGCGATGTTTTTGGTAAA	3890	0
+GGCTGGCACGAGTTTTAC	3880	0
+GGGAAGAAATGGGCTACA	3872	0
+GGGAGGCAAGTCGTAACA	3891	0
+GGGGAGGCAAGTCGTAAC	3890	0
+GGGGGTTTACGACCTCGA	3894	0
+GGTACACTTACCATGTTA	3893	0
+GGTCTGAACTCAGATCAC	3902	0
+GGTCTTAGGAACCAAAAA	3862	0
+GGTTCGTTTGTTCAACGA	3877	0
+GTAACATGGTAAGTGTAC	3893	0
+GTACCTTTTGCATCATGA	3892	0
+GTCCTGATCCAACATCGA	3892	0
+GTCTTAGGAACCAAAAAC	3853	0
+GTGGGGTATCTAATCCCA	3895	0
+GTTACCCTAGGGATAACA	3896	0
+GTTTGTTCAACGATTAAA	3877	0
+TAAAACTCGTGCCAGCCA	3887	0
+TAACCGCGGTGGCTGGCA	3895	0
+TAAGGGTTCGTTTGTTCA	3889	0
+TAATCGTTGAACAAACGA	3876	0
+TACGACCTCGATGTTGGA	3892	0
+TAGGACTTTAATCGTTGA	3689	0
+TAGGATGTCCTGATCCAA	3897	0
+TCCCACTCTTTTGCCACA	3866	0
+TCGCCTGTTTACCAAAAA	3891	0
+TGGATCAGGACATCCTAA	3894	0
+TGGTCTTAGGAACCAAAA	3862	0
+TGGTGCAGCCGCTATTAA	3847	0
+TTGGTCTTAGGAACCAAA	3862	0

pkg/obiprimer/words.go

@@ -0,0 +1,97 @@
+package obiprimer
+
+import (
+	"fmt"
+	"math"
+	"sync"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obikmer"
+)
+
+// WordSearch holds the parameters of the strict word search.
+type WordSearch struct {
+	PrimerLength int     // word size (ecoPrimers -O, 18 by default; 2..31)
+	Quorum       float64 // fraction of example sequences that must contain the word (ecoPrimers -q, 0.7)
+	Circular     bool    // sequences are circular (ecoPrimers -c)
+	Partitions   int     // partition files of the counter (0 = chosen from the size of the examples, see obikmer.DefaultPartitions)
+	TmpDir       string  // parent of the temporary directory ("" = system default)
+}
+
+// StrictWords is the result of the strict word search: the canonical words of
+// PrimerLength bases present, without any error, in at least MinSequences
+// example sequences.
+type StrictWords struct {
+	PrimerLength int
+	MinSequences int // quorum in number of example sequences
+	Words        []obikmer.SeqKmerCount
+	Counts       Counts
+}
+
+// MinSequences returns the quorum in sequences: floor(examples * quorum).
+// The tiny epsilon makes exact products (10 * 0.7 = 7) robust to the binary
+// representation of the fraction.
+func MinSequences(examples int, quorum float64) int {
+	return int(math.Floor(float64(examples)*quorum + 1e-9))
+}
+
+// FindStrictWords indexes the example sequences of the corpus and returns the
+// words reaching the quorum. Sequences no longer than the word size are
+// counted as examples but cannot contain any word, as in ecoPrimers. Words are
+// canonical (the smaller of the word and its reverse complement), so both
+// strands are searched.
+func FindStrictWords(c *Corpus, cfg WordSearch) (*StrictWords, error) {
+	if cfg.PrimerLength < 2 || cfg.PrimerLength > 31 {
+		return nil, fmt.Errorf("obiprimer: primer length must be between 2 and 31, got %d", cfg.PrimerLength)
+	}
+	if cfg.Quorum <= 0 || cfg.Quorum > 1 {
+		return nil, fmt.Errorf("obiprimer: quorum must be in ]0,1], got %g", cfg.Quorum)
+	}
+	if cfg.Partitions <= 0 {
+		// the k-mer occurrences to count: those of the examples, which are the
+		// only sequences indexed
+		total := 0
+		for i, r := range c.Roles {
+			if r == Example && c.SeqLen(i) > cfg.PrimerLength {
+				total += c.SeqLen(i)
+			}
+		}
+		cfg.Partitions = obikmer.DefaultPartitions(total)
+	}
+
+	counter, err := obikmer.NewSeqKmerCounter(cfg.TmpDir, cfg.PrimerLength, 0, cfg.Partitions)
+	if err != nil {
+		return nil, err
+	}
+	defer counter.Close()
+
+	jobs := make(chan int)
+	var wg sync.WaitGroup
+	for range obidefault.ParallelWorkers() {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			var buf []byte
+			for i := range jobs {
+				if c.Roles[i] == Example && c.SeqLen(i) > cfg.PrimerLength {
+					var raw []byte
+					raw, buf = c.bases(i, buf)
+					counter.AddSequence(uint32(i), raw, cfg.Circular)
+				}
+			}
+		}()
+	}
+	for i := range c.Roles {
+		jobs <- i
+	}
+	close(jobs)
+	wg.Wait()
+
+	res := &StrictWords{PrimerLength: cfg.PrimerLength, Counts: c.Counts()}
+	res.MinSequences = MinSequences(res.Counts.Examples, cfg.Quorum)
+	res.Words, err = counter.CountAtLeast(max(res.MinSequences, 1))
+	if err != nil {
+		return nil, err
+	}
+	return res, nil
+}

pkg/obiprimer/words_test.go

@@ -0,0 +1,233 @@
+package obiprimer
+
+import (
+	"math/rand"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiformats"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obikmer"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+func idPrefix(prefix string) obiseq.SequencePredicate {
+	return func(s *obiseq.BioSequence) bool { return strings.HasPrefix(s.Id(), prefix) }
+}
+
+func TestSelectionRoles(t *testing.T) {
+	seq := func(id string) *obiseq.BioSequence { return obiseq.NewBioSequence(id, []byte("acgt"), "") }
+	// ids: "r" restricted clade, "x" exception inside it, "i" ignored clade, "o" other
+	cases := []struct {
+		name string
+		sel  Selection
+		want map[string]Role
+	}{
+		{"nothing set: everything is an example",
+			Selection{}, map[string]Role{"r1": Example, "x1": Example, "i1": Example, "o1": Example}},
+		{"restrict only: the rest are non-targets",
+			Selection{Restrict: idPrefix("r")},
+			map[string]Role{"r1": Example, "x1": Other, "i1": Other, "o1": Other}},
+		{"restrict + ignore",
+			Selection{Restrict: idPrefix("r"), Ignore: idPrefix("i")},
+			map[string]Role{"r1": Example, "i1": CounterExample, "o1": Other}},
+		{"exception inside restrict becomes a counterexample",
+			Selection{Restrict: idPrefix("r").Or(idPrefix("x")), Except: idPrefix("x")},
+			map[string]Role{"r1": Example, "x1": CounterExample, "o1": Other}},
+		{"example wins over ignore when both match",
+			Selection{Restrict: idPrefix("r"), Ignore: idPrefix("r").Or(idPrefix("i"))},
+			map[string]Role{"r1": Example, "i1": CounterExample, "o1": Other}},
+	}
+	for _, tc := range cases {
+		for id, want := range tc.want {
+			if got := tc.sel.Role(seq(id)); got != want {
+				t.Errorf("%s: %s: role %d, want %d", tc.name, id, got, want)
+			}
+		}
+	}
+}
+
+func TestMinSequences(t *testing.T) {
+	for _, tc := range []struct {
+		n    int
+		q    float64
+		want int
+	}{{10, 0.7, 7}, {3909, 0.7, 2736}, {325, 0.7, 227}, {100, 0.9, 90}, {7, 0.9, 6}, {1, 1, 1}} {
+		if got := MinSequences(tc.n, tc.q); got != tc.want {
+			t.Errorf("MinSequences(%d,%g) = %d, want %d", tc.n, tc.q, got, tc.want)
+		}
+	}
+}
+
+func revcomp(s string) string {
+	c := map[byte]byte{'A': 'T', 'C': 'G', 'G': 'C', 'T': 'A'}
+	out := make([]byte, len(s))
+	for i := range s {
+		out[len(s)-1-i] = c[s[i]]
+	}
+	return string(out)
+}
+
+func randDNA(rng *rand.Rand, n int) string {
+	b := make([]byte, n)
+	for i := range b {
+		b[i] = "ACGT"[rng.Intn(4)]
+	}
+	return string(b)
+}
+
+func TestFindStrictWords(t *testing.T) {
+	const k = 18
+	rng := rand.New(rand.NewSource(1))
+	motif := randDNA(rng, 30)
+
+	var data obiseq.BioSequenceSlice
+	add := func(id, s string) { data = append(data, obiseq.NewBioSequence(id, []byte(s), "")) }
+
+	// 10 examples; the motif is in 8 of them, 3 on the reverse strand. One
+	// example carries it twice. Another has an ambiguity breaking it.
+	for i := 0; i < 8; i++ {
+		m := motif
+		if i%3 == 0 {
+			m = revcomp(motif)
+		}
+		s := randDNA(rng, 120) + m + randDNA(rng, 120)
+		if i == 5 {
+			s += randDNA(rng, 50) + motif
+		}
+		add("ex"+string(rune('a'+i)), s)
+	}
+	broken := motif[:12] + "N" + motif[13:]
+	add("exh", randDNA(rng, 100)+broken+randDNA(rng, 100))
+	add("exi", randDNA(rng, 300))
+	// counterexamples and ignored sequences must not count
+	add("cx1", randDNA(rng, 50)+motif)
+	add("zz1", randDNA(rng, 50)+motif)
+
+	sel := Selection{Restrict: idPrefix("ex").Or(idPrefix("cx")), Ignore: idPrefix("cx"), Except: idPrefix("cx")}
+	corpus, err := LoadCorpus(obiiter.IBatchOver("t", data, 3), sel)
+	if err != nil {
+		t.Fatal(err)
+	}
+	res, err := FindStrictWords(corpus, WordSearch{PrimerLength: k, Quorum: 0.7, Partitions: 8, TmpDir: t.TempDir()})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if res.Counts != (Counts{Examples: 10, CounterExamples: 1, Others: 1}) {
+		t.Fatalf("counts = %+v", res.Counts)
+	}
+	if res.MinSequences != 7 {
+		t.Fatalf("quorum = %d sequences, want 7", res.MinSequences)
+	}
+
+	want := map[uint64]bool{}
+	for i := 0; i+k <= len(motif); i++ {
+		want[obikmer.EncodeCanonicalKmer([]byte(motif[i:i+k]), k)] = true
+	}
+	// the N breaks the windows that cover it, but 8 sequences still carry all of them
+	if len(res.Words) != len(want) {
+		t.Fatalf("%d words, want %d (the motif windows)", len(res.Words), len(want))
+	}
+	for _, w := range res.Words {
+		if !want[w.Kmer] {
+			t.Errorf("unexpected word %d", w.Kmer)
+		}
+		if w.Sequences != 8 {
+			t.Errorf("word %d in %d sequences, want 8", w.Kmer, w.Sequences)
+		}
+	}
+	repeated := 0
+	for _, w := range res.Words {
+		if w.Repeated {
+			repeated++
+		}
+	}
+	if repeated != len(want) {
+		t.Errorf("%d words flagged Repeated, want all %d (motif twice in one sequence)", repeated, len(want))
+	}
+}
+
+func TestFindStrictWordsParameters(t *testing.T) {
+	it := func() obiiter.IBioSequence {
+		return obiiter.IBatchOver("t", obiseq.BioSequenceSlice{obiseq.NewBioSequence("a", []byte("acgtacgtacgtacgtacgtacgt"), "")}, 1)
+	}
+	corpus, err := LoadCorpus(it(), Selection{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := FindStrictWords(corpus, WordSearch{PrimerLength: 1, Quorum: 0.7}); err == nil {
+		t.Error("primer length 1 must fail")
+	}
+	if _, err := FindStrictWords(corpus, WordSearch{PrimerLength: 18, Quorum: 0}); err == nil {
+		t.Error("quorum 0 must fail")
+	}
+	if _, err := LoadCorpus(it(), Selection{Restrict: idPrefix("zz")}); err == nil {
+		t.Error("no example must fail")
+	}
+}
+
+// Opt-in end-to-end check against ecoPrimers on the "Designing fish primers"
+// tutorial data. Set OBIPRIMERS_TUTORIAL to the directory holding
+// ncbitaxo_20250211.tgz and mito.vert.fasta.gz.
+//
+// ecoPrimers (-e 3 -3 2 -l 30 -L 150 -c, k=18, strict quorum 0.7) reports
+// "Strict primer count": 1208 for Cyprinidae (-r 7953, 325 examples) and 255
+// for Teleostei (-r 32443, 3909 examples).
+//
+// Teleostei matches exactly. For Cyprinidae we find 1207: ecoPrimers keeps
+// TTTATGAAAACAAATAAA, which a brute-force scan (both strands, circular
+// junction) finds in only 225 of the 325 examples, below its own quorum of
+// 227. That word is in none of the 3448 pairs ecoPrimers finally retains, so
+// the divergence has no effect on the reference results.
+func TestStrictWordsAgainstEcoPrimers(t *testing.T) {
+	dir := os.Getenv("OBIPRIMERS_TUTORIAL")
+	if dir == "" {
+		t.Skip("OBIPRIMERS_TUTORIAL not set")
+	}
+
+	taxonomy, err := obiformats.LoadTaxonomy(filepath.Join(dir, "ncbitaxo_20250211.tgz"), false, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, tc := range []struct {
+		name     string
+		taxid    string
+		examples int
+		words    int
+	}{
+		{"Cyprinidae", "7953", 325, 1207}, // ecoPrimers: 1208, see above
+		{"Teleostei", "32443", 3909, 255},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			restrict, err := BelongingTo(taxonomy, []string{tc.taxid})
+			if err != nil {
+				t.Fatal(err)
+			}
+			// The predicates read the taxon through the taxid annotation.
+			seqs, err := obiformats.ReadSequencesFromFile(filepath.Join(dir, "mito.vert.fasta.gz"))
+			if err != nil {
+				t.Fatal(err)
+			}
+			corpus, err := LoadCorpus(seqs, Selection{Restrict: restrict})
+			if err != nil {
+				t.Fatal(err)
+			}
+			res, err := FindStrictWords(corpus,
+				WordSearch{PrimerLength: 18, Quorum: 0.7, Circular: true, TmpDir: t.TempDir()})
+			if err != nil {
+				t.Fatal(err)
+			}
+			t.Logf("%+v min=%d words=%d", res.Counts, res.MinSequences, len(res.Words))
+			if res.Counts.Examples != tc.examples {
+				t.Errorf("%d examples, ecoPrimers reports %d", res.Counts.Examples, tc.examples)
+			}
+			if len(res.Words) != tc.words {
+				t.Errorf("%d strict words, ecoPrimers reports %d", len(res.Words), tc.words)
+			}
+		})
+	}
+}

pkg/obiseq/iupac_nog.go

@@ -26,3 +26,16 @@ func SameIUPACNuc(a, b byte) bool {
 	}
 	return a == b
 }
+
+// IUPACNucMask returns the set of nucleotides denoted by the IUPAC code b
+// (case insensitive, U is read as T) as a 4 bits mask: bit 0 is A, bit 1 is C,
+// bit 2 is G and bit 3 is T. It returns 0 if b is not an IUPAC nucleotide code.
+func IUPACNucMask(b byte) byte {
+	if (b >= 'A') && (b <= 'Z') {
+		b |= 32
+	}
+	if (b < 'a') || (b > 'z') {
+		return 0
+	}
+	return _iupac[b-'a']
+}

pkg/obiseq/language.go

@@ -141,6 +141,20 @@ var OBILang = gval.NewLanguage(
 	gval.Function("max", func(args ...interface{}) (interface{}, error) {
 		return obiutils.Max(args[0])
 	}),
+	gval.Function("whichmax", func(args ...interface{}) (interface{}, error) {
+		result, err := obiutils.WhichMax(args[0])
+		if idx, ok := result.(int); ok {
+			return float64(idx), nil
+		}
+		return result, err
+	}),
+	gval.Function("whichmin", func(args ...interface{}) (interface{}, error) {
+		result, err := obiutils.WhichMin(args[0])
+		if idx, ok := result.(int); ok {
+			return float64(idx), nil
+		}
+		return result, err
+	}),
 
 	gval.Function("filtermin", func(args ...interface{}) (interface{}, error) {
 		return obiutils.FilterMin(args[0], args[1])

pkg/obiseq/taxonomy_methods.go

@@ -176,6 +176,10 @@ func (sequence *BioSequence) SetFamily(taxonomy *obitax.Taxonomy) *obitax.Taxon
 	return sequence.SetTaxonAtRank(taxonomy, "family")
 }
 
+// TaxonomicPathAttribute is the attribute holding the taxonomic path of a
+// sequence, from the root to its taxon (see obiannotate --taxonomic-path).
+const TaxonomicPathAttribute = "taxonomic_path"
+
 func (sequence *BioSequence) SetPath(taxonomy *obitax.Taxonomy) []string {
 	taxon := sequence.Taxon(taxonomy)
 	path := taxon.Path()
@@ -186,13 +190,13 @@ func (sequence *BioSequence) SetPath(taxonomy *obitax.Taxonomy) []string {
 		spath[lpath-i] = path.Get(i).FullString(taxonomy.Code())
 	}
 
-	sequence.SetAttribute("taxonomic_path", spath)
+	sequence.SetAttribute(TaxonomicPathAttribute, spath)
 
 	return spath
 }
 
 func (sequence *BioSequence) Path() []string {
-	path, ok := sequence.GetAttribute("taxonomic_path")
+	path, ok := sequence.GetAttribute(TaxonomicPathAttribute)
 
 	if !ok {
 		if taxo := obitax.DefaultTaxonomy(); taxo != nil {

pkg/obisimilarity/block.go

@@ -0,0 +1,46 @@
+package obisimilarity
+
+// A block stores the 4-mer tables of a few sequences, transposed: the counter
+// of the word w for the lane j is data[w*width+j]. A vector instruction then
+// handles the same word for all the sequences of the block, and only the words
+// that a sequence contains (a hundred for a short amplicon, out of 256) need
+// to be visited to compare it with all the sequences of the block.
+type tableBlock struct {
+	data []uint8
+}
+
+func newTableBlock(width int) tableBlock {
+	return tableBlock{data: make([]uint8, 256*width)}
+}
+
+// set stores the table t as the lane `lane` of the block.
+func (b *tableBlock) set(width, lane int, t *Table4) {
+	for w := range t.Count {
+		b.data[w*width+lane] = t.Count[w]
+	}
+}
+
+// common computes the sums of the minima of the counters of the sequence
+// given by its words with every lane of the block (see blockCommon).
+func (b *tableBlock) common(width int, words []uint16, out []uint8) {
+	blockCommon(b.data, width, words, out)
+}
+
+// SparseWords lists the words of a table as w<<8|count, so that the visit of
+// a block only goes through the words present in the sequence. nil is
+// returned when the counters add up to more than 255, which would overflow the
+// 8-bit sums of the blocks.
+func SparseWords(t *Table4) []uint16 {
+	words := make([]uint16, 0, 128)
+	mass := 0
+	for w, c := range t.Count {
+		if c > 0 {
+			words = append(words, uint16(w)<<8|uint16(c))
+			mass += int(c)
+		}
+	}
+	if mass > 255 {
+		return nil
+	}
+	return words
+}