Release 4.5.2

This commit introduces significant changes to the sequence processing pipeline, implementing parallel loading, k-mer budget control, explicit memory management, and refined logic for pair evaluation and resolution.

.github/workflows/obitest.yml

@@ -10,10 +10,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Setup Go
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v5
       with:
-        go-version: '1.23'
+        go-version: '1.27'
     - name: Checkout obitools4 project
-      uses: actions/checkout@v4
+      uses: actions/checkout@v5
     - name: Run tests
       run: make githubtests

.github/workflows/release.yml

@@ -16,9 +16,9 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: "1.23"
+          go-version: "1.27"
       - name: Checkout obitools4 project
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
       - name: Run tests
         run: make githubtests
 
@@ -32,12 +32,10 @@ jobs:
             goos: linux
             goarch: amd64
             output_name: linux_amd64
-            cgo_cflags: "-I/usr/include/x86_64-linux-gnu -I/usr/include"
           - os: ubuntu-24.04-arm
             goos: linux
             goarch: arm64
             output_name: linux_arm64
-            cgo_cflags: "-I/usr/include/aarch64-linux-gnu -I/usr/include"
           - os: macos-15-intel
             goos: darwin
             goarch: amd64
@@ -51,12 +49,12 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: "1.23"
+          go-version: "1.27"
 
       - name: Extract version from tag
         id: get_version
@@ -64,12 +62,6 @@ jobs:
           TAG=${GITHUB_REF#refs/tags/Release_}
           echo "version=$TAG" >> $GITHUB_OUTPUT
 
-      - name: Install build tools (Linux)
-        if: runner.os == 'Linux'
-        run: |
-          sudo apt-get update -q
-          sudo apt-get install -y musl-tools zlib1g-dev
-
       - name: Install build tools (macOS)
         if: runner.os == 'macOS'
         run: |
@@ -77,21 +69,31 @@ jobs:
           xcode-select --install 2>/dev/null || true
           xcode-select -p
 
-      - name: Build binaries
+      - name: Build binaries (Linux)
+        if: runner.os == 'Linux'
+        env:
+          VERSION: ${{ steps.get_version.outputs.version }}
+        run: |
+          docker run --rm \
+            -v "$(pwd):/src" \
+            -w /src \
+            -e VERSION="${VERSION}" \
+            -e GOEXPERIMENT=simd \
+            golang:1.27-alpine \
+            sh -c "apk add --no-cache gcc musl-dev zlib-dev zlib-static make && \
+                   make LDFLAGS='-linkmode=external -extldflags=-static' obitools"
+          mkdir -p artifacts
+          tar -czf artifacts/obitools4_${VERSION}_${{ matrix.output_name }}.tar.gz -C build .
+
+      - name: Build binaries (macOS)
+        if: runner.os == 'macOS'
         env:
           GOOS: ${{ matrix.goos }}
           GOARCH: ${{ matrix.goarch }}
           VERSION: ${{ steps.get_version.outputs.version }}
-          CC: ${{ matrix.goos == 'linux' && 'musl-gcc' || '' }}
-          CGO_CFLAGS: ${{ matrix.cgo_cflags || '' }}
         run: |
-          if [ "$GOOS" = "linux" ]; then
-            make LDFLAGS='-linkmode=external -extldflags=-static' obitools
-          else
-            make obitools
-          fi
+          make obitools
           mkdir -p artifacts
-          # Create a single tar.gz with all binaries for this platform
           tar -czf artifacts/obitools4_${VERSION}_${{ matrix.output_name }}.tar.gz -C build .
 
       - name: Upload artifacts
@@ -106,7 +108,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
 

.gitignore

@@ -21,9 +21,11 @@ xx
 
 .rhistory
 /.vscode
+
+/benchmarck
 /build
 /bugs
-
+autodoc
 /ncbitaxo
 
 !/obitests/**

.serena/.gitignore

@@ -0,0 +1,2 @@
+/cache
+/project.local.yml

.serena/project.yml

@@ -0,0 +1,169 @@
+# the name by which the project can be referenced within Serena/when chatting with the LLM.
+project_name: "obitools4"
+
+# the encoding used by text files in the project
+# For a list of possible encodings, see https://docs.python.org/3.11/library/codecs.html#standard-encodings
+encoding: "utf-8"
+
+# line ending convention to use when writing source files.
+# Possible values: unset (use global setting), "lf", "crlf", or "native" (platform default)
+# This does not affect Serena's own files (e.g. memories and configuration files), which always use native line endings.
+line_ending:
+
+# The language backend to use for this project.
+# If not set, the global setting from serena_config.yml is used.
+# Valid values: LSP, JetBrains
+# Note: the backend is fixed at startup. If a project with a different backend
+# is activated post-init, an error will be returned.
+language_backend:
+
+# whether to use project's .gitignore files to ignore files
+ignore_all_files_in_gitignore: true
+
+# advanced configuration option allowing to configure language server-specific options.
+# Maps the language key to the options.
+# The settings are considered only if the project is trusted (see global configuration to define trusted projects).
+# See https://oraios.github.io/serena/02-usage/050_configuration.html#language-server-specific-settings
+ls_specific_settings: {}
+
+# list of additional paths to ignore in this project.
+# Same syntax as gitignore, so you can use * and **.
+# Important: quote patterns that start with `*`, otherwise YAML treats them as aliases.
+# Example:
+#   ignored_paths:
+#     - "examples/**"
+#     - ".worktrees/**"
+#     - "**/bin/**"
+#     - "**/obj/**"
+# Note: global ignored_paths from serena_config.yml are also applied additively.
+ignored_paths: []
+
+# whether the project is in read-only mode
+# If set to true, all editing tools will be disabled and attempts to use them will result in an error
+# Added on 2025-04-18
+read_only: false
+
+# list of tool names to exclude.
+# This extends the existing exclusions (e.g. from the global configuration)
+# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html
+excluded_tools: []
+
+# list of tools to include that would otherwise be disabled (particularly optional tools that are disabled by default).
+# This extends the existing inclusions (e.g. from the global configuration).
+# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html
+included_optional_tools: []
+
+# fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools.
+# This cannot be combined with non-empty excluded_tools or included_optional_tools.
+# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html
+fixed_tools: []
+
+# list of mode names that are to be activated by default, overriding the setting in the global configuration.
+# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes.
+# If the setting is undefined/empty, the default_modes from the global configuration (serena_config.yml) apply.
+# Otherwise, this overrides the setting from the global configuration (serena_config.yml).
+# Therefore, you can set this to [] if you do not want the default modes defined in the global config to apply
+# for this project.
+# This setting can, in turn, be overridden by CLI parameters (--mode).
+# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes
+default_modes:
+
+# list of mode names to be activated additionally for this project, e.g. ["query-projects"]
+# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes.
+# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes
+added_modes:
+
+# initial prompt for the project. It will always be given to the LLM upon activating the project
+# (contrary to the memories, which are loaded on demand).
+initial_prompt: ""
+
+# time budget (seconds) per tool call for the retrieval of additional symbol information
+# such as docstrings or parameter information.
+# This overrides the corresponding setting in the global configuration; see the documentation there.
+# If null or missing, use the setting from the global configuration.
+symbol_info_budget:
+
+# list of regex patterns which, when matched, mark a memory entry as read‑only.
+# Extends the list from the global configuration, merging the two lists.
+read_only_memory_patterns: []
+
+# list of regex patterns for memories to completely ignore.
+# Matching memories will not appear in list_memories or activate_project output
+# and cannot be accessed via read_memory or write_memory.
+# To access ignored memory files, use the read_file tool on the raw file path.
+# Extends the list from the global configuration, merging the two lists.
+# Example: ["_archive/.*", "_episodes/.*"]
+ignored_memory_patterns: []
+
+# list of additional workspace folder paths for cross-package reference support.
+# Paths can be absolute or relative to the project root.
+# Each folder is registered as an LSP workspace folder, enabling language servers to discover
+# symbols and references across package boundaries, but these folders are not indexed by Serena,
+# i.e. the respective symbols will not be found using Serena's symbol search tools.
+# Example:
+#   additional_workspace_folders:
+#     - ../sibling-package
+#     - ../shared-lib
+ls_additional_workspace_folders: []
+
+# list of language servers to start when using the LSP backend; choose from:
+#   ada                    al                     angular                ansible                bash
+#   bsl                    clojure                cpp                    cpp_ccls               crystal
+#   csharp                 csharp_omnisharp       cue                    dart                   deno
+#   elixir                 elm                    erlang                 fortran                fsharp
+#   gdscript               gleam                  go                     groovy                 haskell
+#   haxe                   hlsl                   html                   java                   json
+#   julia                  kotlin                 latex                  lean4                  lua
+#   luau                   markdown               matlab                 msl                    nextflow
+#   nix                    ocaml                  pascal                 perl                   php
+#   php_phpactor           php_phpantom           powershell             python                 python_basedpyright
+#   python_jedi            python_pyrefly         python_ty              qml                    r
+#   rego                   ruby                   ruby_solargraph        rust                   scala
+#   scss                   solidity               svelte                 swift                  systemverilog
+#   terraform              toml                   typescript             typescript_vts         vue
+#   wolfram                yaml                   zig
+#   (This list may be outdated; generated with scripts/print_language_list.py;
+#   For the current list, see values of the LanguageServerId enum here:
+#   https://github.com/oraios/serena/blob/main/src/solidlsp/ls_config.py)
+# For some languages, there are several alternative language servers, e.g. csharp_omnisharp, ruby_solargraph.)
+# Note:
+#   - For C, use cpp
+#   - For JavaScript, use typescript
+#   - For Angular projects, use angular (subsumes typescript+html; requires `npm install` in the project root)
+#   - For Svelte projects, use svelte (subsumes typescript/javascript for .svelte projects; requires npm)
+#   - For Deno projects, use deno (serves the same .ts/.js files as typescript; requires the deno CLI on PATH)
+#   - For SCSS / Sass / plain CSS, use scss (some-sass-language-server handles all three)
+#   - For Free Pascal/Lazarus, use pascal
+# Special requirements:
+#   Some language servers require additional setup/installations.
+#   See here for details: https://oraios.github.io/serena/01-about/020_programming-languages.html#language-servers
+# When using multiple language servers, the first language server that supports a given file will be used for that file.
+# The first language server is the default language and the respective language server will be used as a fallback.
+# Note that when using the JetBrains backend, language servers are not used and this list is correspondingly ignored.
+language_servers:
+- go
+
+# list of workspace folder paths (LSP backend only).
+# These folders will be used to build up Serena's symbol index.
+# Paths must be within the project root and should thus be relative to the project root.
+# Furthermore, the paths should not be filtered by ignore settings.
+# Default setting: The entire project root folder (".") is considered.
+# In (large) monorepos, this can be used to index only subfolders of the project root, e.g.
+#   ls_workspace_folders:
+#     - "./subproject1"
+#     - "./subproject2"
+ls_workspace_folders:
+- .
+
+# optional shell command to run before the language backend (LSP or JetBrains) is initialised.
+# the command runs in the project root directory and is only executed if the project is trusted
+# (see trusted_project_path_patterns in the global configuration).
+# serena waits for the command to exit: a non-zero exit code is logged as an error but does not
+# abort activation. a per-project timeout (activation_command_timeout, default 180s) is the safety
+# backstop for non-terminating commands; on expiry the process is killed and activation continues.
+# example: activation_command: "npx nx run-many -t build"
+activation_command:
+
+# maximum time in seconds to wait for activation_command to complete before killing it (default 180s).
+# must be a positive number.
+activation_command_timeout: 180.0

Makefile

@@ -11,6 +11,14 @@ NC     := \033[0m
 
 GOFLAGS=
 LDFLAGS=
+
+# The SIMD code (pkg/obikmer, simd package of Go >= 1.27) is only compiled with
+# GOEXPERIMENT=simd; without it, a portable scalar code is used instead.
+# The binaries select the vector width at run time (and fall back on an
+# emulation without SIMD hardware), so they run on any CPU.
+# Use `make GOEXPERIMENT= obitools` to build without SIMD.
+GOEXPERIMENT?=simd
+export GOEXPERIMENT
 GOCMD=go
 GOBUILD=$(GOCMD) build $(GOFLAGS) $(if $(LDFLAGS),-ldflags="$(LDFLAGS)")
 GOGENERATE=$(GOCMD) generate
@@ -36,13 +44,18 @@ OBITOOLS_DIRS:=$(sort $(patsubst %/,%,$(dir $(OBITOOLS_SRC))))
 OBITOOLS:=$(notdir $(OBITOOLS_DIRS))
 
 
+# Name of the target of a package: its path under pkg/, so that two packages of
+# the same name in different directories (pkg/obialign and
+# pkg/obitools/obialign) do not share a target.
+pkgname=$(subst /,-,$(patsubst pkg/%,%,$(1)))
+
 define MAKE_PKG_RULE
-pkg-$(notdir $(1)): $(1) pkg/obioptions/version.go
-	@echo -n - Building package $(notdir $(1))...
+pkg-$(call pkgname,$(1)): $(1) pkg/obioptions/version.go
+	@echo -n - Building package $(call pkgname,$(1))...
 	@$(GOBUILD) ./$(1) \
-	    2> pkg-$(notdir $(1)).log \
-		|| cat pkg-$(notdir $(1)).log
-	@rm -f pkg-$(notdir $(1)).log
+	    2> pkg-$(call pkgname,$(1)).log \
+		|| cat pkg-$(call pkgname,$(1)).log
+	@rm -f pkg-$(call pkgname,$(1)).log
 	@echo Done.
 endef
 
@@ -156,8 +169,8 @@ bump-version:
 
 jjnew:
 	@echo "$(YELLOW)→ Creating a new commit...$(NC)"
-	@echo "$(BLUE)→ Documenting current commit...$(NC)"
-	@jj auto-describe
+	@echo "$(BLUE)→ Documenting undocumented commits...$(NC)"
+	@jj auto-doc
 	@echo "$(BLUE)→ Done.$(NC)"
 	@jj new
 	@echo "$(GREEN)✓ New commit created$(NC)"
@@ -171,8 +184,8 @@ jjpush:
 	@echo "$(GREEN)✓ Release complete$(NC)"
 
 jjpush-describe:
-	@echo "$(BLUE)→ Documenting current commit...$(NC)"
-	@jj auto-describe
+	@echo "$(BLUE)→ Documenting undocumented commits...$(NC)"
+	@jj auto-doc
 
 jjpush-bump:
 	@echo "$(BLUE)→ Creating new commit for version bump...$(NC)"

OBIPRIMERS_MEMORY_PROMPT.md

@@ -0,0 +1,149 @@
+# Prompt : optimisation de l'usage mémoire de `obiprimers`
+
+Tu travailles dans le dépôt Go `/Users/coissac/Sync/travail/__MOI__/GO/obitools4` (module
+`git.metabarcoding.org/obitools/obitools4/obitools4`, Go : `/usr/local/go/bin/go`, non présent dans le PATH).
+Ta mission : **réduire l'usage mémoire de la commande `obiprimers`** sans changer ses résultats, en procédant
+par la mesure.
+
+## 1. Contexte
+
+`obiprimers` est la ré-implémentation OBITools4 d'ecoPrimers (design de paires d'amorces à partir de génomes
+entiers, avec évaluation de la qualité du marqueur délimité). Le mode « voie A » est terminé et validé.
+Avant toute chose, **lis `OBIPRIMERS_NOTES.md`** (racine du dépôt) : la section « État courant », puis les
+sections 11 à 13 (appariement approché), 16 et 16 bis (taxonomie embarquée ; **une erreur de mesure à ne pas
+refaire**), 25 (définitions de Bc, Bs, Bd et élagage de la phase 2).
+
+Code concerné :
+- `cmd/obitools/obiprimers/main.go`, `pkg/obitools/obiprimers/` (options, lecture, sortie CSV) ;
+- `pkg/obiprimer/` : `corpus.go` (`LoadCorpus`), `taxa.go`, `words.go` (`FindStrictWords`), `match.go` et
+  `match_seed.go` (`MatchWords`), `pairs.go` (`selectPrimers`, `countPairs`), `pairs_eval.go` (`FindPairs`),
+  `amplicons.go` (`collectAmplicons`), `resolution.go`, `search.go` (`Search`) ;
+- `pkg/obikmer/seqquorum*.go` (`SeqKmerCounter`, comptage par partitions sur disque) ;
+- `pkg/obithermo/` (Tm), code partagé : `pkg/obiformats`, `pkg/obiiter`, `pkg/obiseq`, `pkg/obitax`.
+
+## 2. Décisions de l'utilisateur à respecter (non négociables)
+
+- **Tout est chargé en mémoire** : toutes les séquences de la base (cibles, contre-exemples, non-cibles) sont
+  gardées, parce que la lecture sur l'**entrée standard** doit continuer à marcher comme pour tout obitool.
+  Tu peux **compacter** ce qui est gardé ; tu ne peux pas passer à un schéma qui exige de relire le fichier.
+- **Un seul calcul par indice** : ne touche pas aux définitions de Bc, Bs et Bd (`resolution.go`). Pas de
+  colonne ou d'option alternative.
+- **Les résultats ne changent pas.** Critère : la sortie CSV doit rester **identique octet pour octet** sur les
+  cas de référence du §4. Si tu penses qu'un changement de résultat est justifié, **demande avant**.
+- Aucune commande git qui modifie quoi que ce soit (pas de commit, add, mv, rm, reset…). Lecture seule
+  (`status`, `diff`, `log`) autorisée. Déplace les fichiers avec `mv`/`Write`/`Edit`.
+- Fichiers source de moins d'environ 500 lignes. Pas de `.collect()` d'une grande structure quand un flux
+  suffit. Réutilise l'existant avant d'écrire du code ; enrichis une brique partagée plutôt que d'écrire du
+  code ad hoc, **mais** toute modification d'un paquet partagé (obiformats, obiiter, obiseq, obitax) doit
+  être signalée clairement et validée par les tests de ces paquets.
+- **Pas de spéculation présentée comme un fait** : ce qui n'est pas mesuré est une hypothèse, dis-le.
+  Une régression de justesse est pire qu'un gain de mémoire : avant d'échanger l'une contre l'autre, demande.
+
+## 3. Ce qui est mesuré aujourd'hui (macOS, M3 Max, 16 cœurs)
+
+Jeu : 7 822 génomes mitochondriaux de vertébrés (131 Mb de bases), `-r 7953` (Cyprinidae : 325 cibles, 7 497
+non-cibles). Commande :
+`obiprimers -r 7953 -e 3 --three-prime 2 -l 30 -L 150 -c <fichier annoté> -o out.csv` → 3 247 paires
+(3 248 lignes avec l'en-tête).
+
+| Mesure | Valeur |
+|---|---|
+| pic RSS, taxonomie embarquée, fichier | **1 217 à 1 372 Mo** (bruit d'environ ±100 Mo d'une exécution à l'autre), 4,7 à 5,3 s |
+| pic RSS, taxonomie embarquée, entrée standard | médiane **1 334 à 1 346 Mo** sur 8 exécutions entrelacées |
+| tas des 7 822 séquences chargées | **193,7 Mo** (dont `taxonomic_path` : 19,2 Mo, soit 2,46 Ko par séquence) |
+| même exécution **sans** l'élagage de phase 2 | 5 604 Mo (c'est ce qu'on a corrigé, voir §25 des notes) |
+| avec `-t` (taxonomie NCBI complète) | +1,07 Go pour le seul chargeur (mesuré avec `obitaxonomy`), pic ~2,8 Go (mesuré avant les nouvelles définitions) |
+| `GOGC=25` (ancienne mesure, `-t` NCBI) | pic 2,82 Go → 1,98 Go, mais temps CPU utilisateur ×2 |
+| ecoPrimers (C) sur les mêmes jeux | 1,86 Go (Cyprinidae), 2,11 Go (Teleostei) |
+
+Tailles d'enregistrements : `ampRecord` 24 octets, `Match` 12, `seqMatch` 12, `hit` (intermédiaire de
+`matchCorpus`) 16. Environ 6,85 millions d'occurrences approchées quand toutes les séquences sont chargées
+(cas `-i 1`).
+
+**Le constat qui motive la mission** : les séquences gardées pèsent ~194 Mo, le pic du processus est ~1,3 Go.
+Les 1,1 Go restants ne sont **pas expliqués** (hypothèses ci-dessous, aucune n'est vérifiée).
+
+## 4. Garde-fous à mettre en place AVANT de modifier quoi que ce soit
+
+1. **Binaire de référence** : `cd /Users/coissac/Sync/travail/__MOI__/GO/obitools4 &&
+   GOEXPERIMENT=simd /usr/local/go/bin/go build -o /tmp/obiprimers.baseline ./cmd/obitools/obiprimers`
+2. **Données** (les fichiers de `/tmp` peuvent avoir disparu, regénère-les) : dossier du tutoriel
+   `D="$HOME/Documents/Enseignement/DNA metabarcoding spring school/ecoPrimers-Fish"` : `mito.vert.fasta.gz`,
+   `ncbitaxo_20250211.tgz`, et pour le test d'échelle `mito.all.fasta.gz` / `mito.one.fasta.gz`. Fichier annoté
+   (taxonomie embarquée, évite le chargeur NCBI) :
+   `/Users/coissac/travail/__MOI__/GO/obitools4/build/obiannotate -t "$D/ncbitaxo_20250211.tgz"
+   --taxonomic-path -Z -o /tmp/mito.vert.path.fasta.gz "$D/mito.vert.fasta.gz"`
+   (les binaires déjà compilés sont dans `/Users/coissac/travail/__MOI__/GO/obitools4/build/`, sans `Sync`).
+3. **CSV de référence**, à produire avec le binaire de référence et à conserver : Cyprinidae `-r 7953`,
+   Teleostei `-r 32443`, un cas avec contre-exemples `-r 7953 -i 1`, un cas `-U`, un cas `-R NC_083213`, et la
+   lecture sur entrée standard (`gzcat fichier | obiprimers ...`). Après chaque modification : `cmp` octet pour
+   octet.
+4. **Tests** : `go test ./pkg/obiprimer/ ./pkg/obitools/obiprimers/ ./pkg/obithermo/ ./pkg/obikmer/ -race
+   -count=1`, et les tests opt-in sur les données réelles (durent ~75 s hors `-race`) :
+   `OBIPRIMERS_TUTORIAL="$D" go test ./pkg/obiprimer/ -count=1`.
+5. **Méthode de mesure** (le piège principal) : le pic RSS bouge de ±100 Mo entre deux exécutions identiques
+   (rythme du ramasse-miettes). Mesure **au moins 5 exécutions entrelacées avant/après et compare les
+   médianes** (`/usr/bin/time -l` : octets sur macOS). Pour attribuer la mémoire, ne te fie pas au RSS : utilise
+   des profils de tas (`--pprof` de la CLI sert `/debug/pprof/heap` sur `localhost:6060`, `net/http/pprof` est
+   importé ; ou `runtime/pprof.WriteHeapProfile` aux frontières d'étapes ; `go tool pprof -sample_index=inuse_space`
+   et `alloc_space` ; `GODEBUG=gctrace=1`). **Erreur déjà commise (voir la section 16 bis des notes)** : en
+   mesurant le tas libéré par la suppression d'un attribut, la tranche de séquences n'était plus référencée
+   après la boucle et le GC a libéré *tout* ; utilise `runtime.KeepAlive` sur ce que tu veux garder vivant.
+
+## 5. Pistes à évaluer (toutes des HYPOTHÈSES à confirmer par un profil, classées sans ordre)
+
+- **Retard du ramasse-miettes** (`GOGC=100` laisse le tas monter jusqu'à ~2× le vivant). Pistes : limite
+  mémoire douce (`debug.SetMemoryLimit`) ou `debug.SetGCPercent` ajustés par la commande après la lecture ;
+  mesurer le coût CPU (l'ancienne mesure avec `GOGC=25` doublait le temps utilisateur).
+- **Lecture** (code partagé `obiformats`/`obiiter`) : 131 Mb sont analysés, avec une table d'attributs et des
+  chaînes JSON par séquence. L'option `--batch-mem` (lots limités en mémoire) existe déjà : l'évaluer.
+- **Séquences gardées** : `BioSequence` garde 1 octet par base et une table d'attributs. Ne garder que
+  l'identifiant, la séquence et le `taxid` (retirer les autres attributs, comme déjà fait pour
+  `taxonomic_path`, dans `dropTaxonomicPath` de `pkg/obitools/obiprimers/obiprimers.go`) ; ou stocker la
+  séquence à 2 bits par base (~33 Mo pour 131 Mb) avec une liste d'exceptions pour les bases ambiguës. Ce
+  second point touche `forEachOccurrence`, `ampHash`, `windowAsIs`, `windowRevComp` et `MatchWords`, qui lisent
+  les octets : à ne tenter que si le profil le justifie.
+- **Occurrences stockées deux fois** : `[]WordMatches` (`matchCorpus`, intermédiaire `hit` de 16 octets par
+  occurrence puis recopie) et `primerSet.perSeq` (`selectPrimers`). Vérifier à quel moment chacune est
+  libérée, construire directement les structures finales, présizer les tranches (les `append` en doublement
+  créent un pic transitoire).
+- **Phase 2b** (`collectAmplicons` sur les seules paires survivantes) : `ampRecord` de 24 octets (16 possibles
+  en empaquetant le rôle), enregistrements par worker ; **traitement des paires survivantes par lots bornés**
+  si le nombre d'enregistrements estimé dépasse un budget (idée notée en 25.2). Cas dur : `-T 0` (aucune paire
+  élaguée), où les tests atteignent ~5,5 Go de tas.
+- **`SeqKmerCounter`** (`pkg/obikmer/seqquorum*.go`) : environ 16 octets par occurrence de k-mer pour les
+  partitions en cours de traitement ; le nombre de partitions est **fixé à 256**, alors que le total de bases
+  est connu une fois le corpus chargé : le choisir automatiquement.
+- **Lecture en deux passes** pour la taxonomie embarquée (`readSequencesAndTaxonomy`) : la première passe ne
+  garde que les chemins, mais analyse tout.
+- **Chargeur de taxonomie NCBI** : 1,07 Go à lui seul ; c'est du code partagé (`obitax`), hors périmètre sauf
+  si tu y trouves une économie sûre ; sinon, documente-le dans l'aide (la taxonomie embarquée évite ce coût).
+
+## 6. Test d'échelle (jamais fait)
+
+Tout est validé sur des génomes mitochondriaux (5 à 131 Mb). Mesure la croissance sur un fichier plus gros
+(`mito.all.fasta.gz`, 172 Mo compressés, avec `--restrict-to-taxon` sur un grand clade, p. ex. les Metazoa) :
+temps et pic RSS en fonction de la taille, pour savoir ce qui grossit vite (corpus, occurrences, amplicons,
+compteur de k-mers).
+
+## 7. Livrables
+
+1. Un **rapport de profil** : où va la mémoire à chaque étape (lecture, sélection, mots stricts, appariement,
+   paires, phase 2a/2b), avec les profils de tas et les chiffres, avant toute modification.
+2. Les optimisations **classées par gain mesuré et par risque**, puis implémentées une par une, chacune avec
+   mesure avant/après (médianes d'au moins 5 exécutions entrelacées), `cmp` des CSV de référence, tests verts
+   sous `-race`.
+3. Une **nouvelle section** à la fin de `OBIPRIMERS_NOTES.md` (ne réécris pas l'historique) : mesures,
+   décisions, pistes écartées et pourquoi, ce qui reste. Pas de commit.
+4. Si une optimisation exige de changer un résultat, une décision ou un paquet partagé : **arrête-toi et
+   demande**.
+
+## 8. Pièges connus de l'environnement
+
+- zsh ne découpe pas une variable en arguments : utilise des tableaux (`A=(-r 7953 ...)` puis `"${A[@]}"`)
+  ou des scripts `bash`.
+- macOS : `sed -i ''` (sans suffixe) ; `/usr/bin/time -l` donne le pic en **octets** (`maximum resident set size`).
+- `OBIPRIMERS_WORKERS` n'est lue que par les tests opt-in ; la CLI utilise `--max-cpu`.
+- Les avertissements C de `obiformats/fastseq_read.h` sont connus et sans gravité.
+- `go test -race` sur les tests opt-in est lent (~90 s) ; réserve-le aux tests unitaires.
+- Le test `obiutils/TestSetString` était instable : il est corrigé (tri des membres dans `Set.String`).

Release-notes.md

@@ -2,6 +2,23 @@
 
 ## New changes
 
+### New features
+
+- New `obioligotag` command, a re-implementation of the `oligotag` command of
+  OBITools 2. It builds a family of oligonucleotides (tags) of at least
+  `--family-size` members that are pairwise at a distance of at least
+  `--distance`. Candidates are either generated (`--oligo-size`, up to 12) or
+  read from a file (`--oligo-list`, sequence file or one oligonucleotide per
+  line). They can be restricted by alphabet (`--alphabet`, for instance `ACT`
+  to forbid G), GC content, homopolymers and IUPAC patterns. The distance is
+  either the Hamming distance or the LCS-mode alignment distance
+  (`--distance-type`).
+
+- New functions in `obikmer` to compare and enumerate k-mers encoded as
+  `uint64`: `HammingDistance`, `KmerLCS` and `KmerLCSBatch` (SIMD when built with
+  `GOEXPERIMENT=simd`), `IterConstrainedKmers`, and in `obigraph` a clique search
+  (`FindClique`) that does not need the adjacency matrix.
+
 ### Bug fixes
 
 - In `obipairing` correct the misspelling of the `obiparing_*` tags where the `i`

cmd/obitools/obialign/main.go

@@ -0,0 +1,26 @@
+package main
+
+import (
+	"os"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obialign"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+)
+
+func main() {
+	optionParser := obioptions.GenerateOptionParser(
+		"obialign",
+		"pairwise similarity of nucleotide sequences, as a CSV table (the principle of sumatra)",
+		obialign.OptionSet)
+
+	_, args := optionParser(os.Args)
+
+	fs, err := obiconvert.CLIReadBioSequences(args...)
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
+
+	obialign.CLIWriteTable(obialign.CLIOBIAlign(fs))
+
+	obiutils.WaitForLastPipe()
+}

cmd/obitools/obiclust/main.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"os"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiclust"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+)
+
+func main() {
+	optionParser := obioptions.GenerateOptionParser(
+		"obiclust",
+		"star clustering of sequences (the algorithm of sumaclust, UCLUST and CD-HIT)",
+		obiclust.OptionSet)
+
+	_, args := optionParser(os.Args)
+
+	fs, err := obiconvert.CLIReadBioSequences(args...)
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
+
+	clustered := obiclust.CLIOBIClust(fs)
+
+	obiconvert.CLIWriteBioSequences(clustered, true)
+
+	obiutils.WaitForLastPipe()
+}

cmd/obitools/obioligotag/main.go

@@ -0,0 +1,33 @@
+package main
+
+import (
+	"os"
+
+	log "github.com/sirupsen/logrus"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obioligotag"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+)
+
+func main() {
+	optionParser := obioptions.GenerateOptionParser(
+		"obioligotag",
+		"designs a family of oligonucleotides (tags) at a minimum distance from each other",
+		obioligotag.OptionSet)
+
+	_, args := optionParser(os.Args)
+	if len(args) > 0 {
+		log.Fatalf("unexpected arguments %v: use --oligo-list to provide candidate oligonucleotides", args)
+	}
+
+	oligos, err := obioligotag.CLIOligoFamily()
+	if err != nil {
+		log.Fatalf("%v", err)
+	}
+
+	obiconvert.CLIWriteBioSequences(obioligotag.CLIOligosToIterator(oligos), true)
+
+	obiutils.WaitForLastPipe()
+}

cmd/obitools/obipairing/main.go

@@ -37,6 +37,11 @@ func main() {
 
 	optionParser(os.Args)
 
+	if !obipairing.CLIHasPairedFiles() {
+		log.Error("You must provide both a forward file (-F) and a reverse file (-R)")
+		os.Exit(1)
+	}
+
 	obidefault.SetStrictReadWorker(2)
 	obidefault.SetStrictWriteWorker(2)
 	pairs, err := obipairing.CLIPairedSequence()

cmd/obitools/obiprimers/main.go

@@ -0,0 +1,38 @@
+package main
+
+import (
+	"os"
+
+	log "github.com/sirupsen/logrus"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obicsv"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiprimers"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+)
+
+func main() {
+	obiprimers.SetReadingDefaults()
+
+	optionParser := obioptions.GenerateOptionParser(
+		"obiprimers",
+		"designs the primer pairs amplifying a set of example sequences and evaluates the DNA barcode they delimit. "+
+			"The taxonomy comes from the -t option, or is read from the taxonomic paths embedded in the sequences (obiannotate --taxonomic-path)",
+		obiprimers.OptionSet)
+
+	_, args := optionParser(os.Args)
+
+	sequences, taxonomy, err := obiprimers.CLIReadSequencesAndTaxonomy(args)
+	if err != nil {
+		log.Fatalf("Cannot read the sequences: %v", err)
+	}
+
+	pairs, err := obiprimers.CLIPrimerPairs(sequences, taxonomy)
+	if err != nil {
+		log.Fatalf("%v", err)
+	}
+
+	obicsv.CLICSVWriter(obiprimers.PairsToCSVIterator(pairs, obiprimers.CLIReference() != ""), true)
+
+	obiutils.WaitForLastPipe()
+}

cmd/obitools/obitagpcr/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"fmt"
 	"os"
 
 	log "github.com/sirupsen/logrus"
@@ -8,6 +9,7 @@ import (
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obimultiplex"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obipairing"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obitagpcr"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
@@ -39,6 +41,17 @@ func main() {
 		obitagpcr.OptionSet)
 
 	optionParser(os.Args)
+
+	if obimultiplex.CLIAskConfigTemplate() {
+		fmt.Print(obimultiplex.CLIConfigTemplate())
+		os.Exit(0)
+	}
+
+	if !obipairing.CLIHasPairedFiles() {
+		log.Error("You must provide both a forward file (-F) and a reverse file (-R)")
+		os.Exit(1)
+	}
+
 	pairs, err := obipairing.CLIPairedSequence()
 
 	if err != nil {

doc/book/installation.qmd

@@ -8,7 +8,7 @@ All the sources of the [*OBITools4*](http://metabarcoding.org/obitools4) can be
 
 ## Prerequisites
 
-The *OBITools4* are developped using the [GO programming language](https://go.dev/), we stick to the latest version of the language, today the $1.21.4$. If you want to download and compile the sources yourself, you first need to install the corresponding compiler on your system. Some parts of the soft are also written in C, therefore a recent C compiler is also requested, GCC on Linux or Windows, the Developer Tools on Mac.
+The *OBITools4* are developped using the [GO programming language](https://go.dev/), we stick to the latest version of the language, today the $1.27$. If you want to download and compile the sources yourself, you first need to install the corresponding compiler on your system. Some computations (distances between k-mers) use the experimental `simd` package of Go: the `Makefile` compiles them with `GOEXPERIMENT=simd`, and the installation script uses a Go 1.27 compiler for that reason. The resulting programs run on any processor, a portable code being used when no SIMD instruction is available. Some parts of the soft are also written in C, therefore a recent C compiler is also requested, GCC on Linux or Windows, the Developer Tools on Mac.
 
 Whatever the installation you decide for, you will have to ensure that a C compiler is available on your system.
 

entities.json

@@ -0,0 +1,10 @@
+{
+  "people": [
+    "Software",
+    "Agreement",
+    "Module"
+  ],
+  "projects": [
+    "Code"
+  ]
+}

go.mod

@@ -1,12 +1,12 @@
 module git.metabarcoding.org/obitools/obitools4/obitools4
 
-go 1.26.1
+go 1.27
 
 require (
 	github.com/DavidGamba/go-getoptions v0.33.0
 	github.com/PaesslerAG/gval v1.2.4
 	github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df
-	github.com/buger/jsonparser v1.1.1
+	github.com/buger/jsonparser v1.1.2
 	github.com/chen3feng/stl4go v0.1.1
 	github.com/dlclark/regexp2 v1.11.5
 	github.com/goccy/go-json v0.10.6

go.sum

@@ -6,8 +6,8 @@ github.com/PaesslerAG/jsonpath v0.1.0 h1:gADYeifvlqK3R3i2cR5B4DGgxLXIPb3TRTH1mGi
 github.com/PaesslerAG/jsonpath v0.1.0/go.mod h1:4BzmtoM/PI8fPO4aQGIusjGxGir2BzcV0grWtFzq1Y8=
 github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df h1:GSoSVRLoBaFpOOds6QyY1L8AX7uoY+Ln3BHc22W40X0=
 github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df/go.mod h1:hiVxq5OP2bUGBRNS3Z/bt/reCLFNbdcST6gISi1fiOM=
-github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
-github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
+github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk=
+github.com/buger/jsonparser v1.1.2/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
 github.com/chen3feng/stl4go v0.1.1 h1:0L1+mDw7pomftKDruM23f1mA7miavOj6C6MZeadzN2Q=
 github.com/chen3feng/stl4go v0.1.1/go.mod h1:5ml3psLgETJjRJnMbPE+JiHLrCpt+Ajc2weeTECXzWU=
 github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM=

go.work

@@ -1,3 +1,3 @@
-go 1.26.1
+go 1.27
 
 use .

install_obitools.sh

@@ -2,6 +2,11 @@
 
 # Default values
 URL="https://go.dev/dl/"
+# Series of the Go compiler used to build OBITools4. The SIMD code relies on the
+# experimental simd package of Go (GOEXPERIMENT=simd), whose API may change
+# between Go versions: the build is therefore tied to the series the code was
+# written for rather than to the latest release.
+GO_SERIES="1.27"
 GITHUB_REPO="https://github.com/metabarcoding/obitools4"
 INSTALL_DIR="/usr/local"
 OBITOOLS_PREFIX=""
@@ -174,10 +179,16 @@ GOFILE=$(curl -s "$URL" \
             | grep 'class="download"' \
             | grep "\.tar\.gz" \
             | sed -E 's@^.*/dl/(go[1-9].+\.tar\.gz)".*$@\1@' \
+            | awk -v prefix="go${GO_SERIES}." 'index($0, prefix) == 1' \
             | grep -i "$OS" \
             | grep -i "$ARCH" \
             | head -1)
 
+if [ -z "$GOFILE" ]; then
+  echo "Error: no Go ${GO_SERIES} archive found for ${OS}/${ARCH} at ${URL}" 1>&2
+  exit 1
+fi
+
 GOURL=$(curl -s "${URL}${GOFILE}" \
         | sed -E 's@^.*href="(.*\.tar\.gz)".*$@\1@')
 
@@ -191,8 +202,12 @@ export PATH
 export GOPATH="$(pwd)/gopath"
 export GOCACHE="$(pwd)/cache"
 export GOTOOLCHAIN=local
+# OBITools4 is built with the SIMD code enabled (it falls back on a portable
+# code at run time if the processor has no SIMD instructions).
+export GOEXPERIMENT=simd
 
 echo "GOROOT=$GOROOT" 1>&2
+echo "GOEXPERIMENT=$GOEXPERIMENT" 1>&2
 echo "GOCACHE=$GOCACHE" 1>&2
 mkdir -p "$GOPATH" "$GOCACHE"
 

obitests/obitools/obialign/test.sh

@@ -0,0 +1,206 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obialign
+CMD=obialign
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1
+then
+    log "$MCMD: printing help OK"
+    ((success++))
+else
+    log "$MCMD: printing help failed"
+    ((failed++))
+fi
+
+
+#
+# Dataset used by the following tests: two families of similar sequences, a
+# sequence identical to another one, and a sequence with an ambiguous
+# nucleotide that is discarded.
+#
+cat > "${TMPDIR}/data.fasta" <<DATA
+>a1 count=20;
+acgtacgtacgtaacctagcctagcctagggatcgatcgatcgatcgaatcg
+>a2 count=3;
+acgtacgtacgtaacctagcctagcctagggatcgatcgatcgttcgaatcg
+>a3 count=2;
+acgtacgtacgtaacctagcctagcctagggatcgatcgatcgatcgaatcg
+>b1 count=15;
+ttgcatgcatgctagctagctagcatgcatcgatcgatgctagctagtagct
+>b2 count=1;
+ttgcatgcatgctagctagctagcatgcatcgatcgatgctagctagtaact
+>x1 count=4;
+ttgcatgcatgctangctagctagcatgcatcgatcgatgctagctagtagc
+DATA
+
+cat > "${TMPDIR}/ref.fasta" <<DATA
+>r1
+acgtacgtacgtaacctagcctagcctagggatcgatcgatcgatcgaatcg
+>r2
+ttgcatgcatgctagctagctagcatgcatcgatcgatgctagctagtagct
+>r3
+ggggccccggggccccggggccccggggccccggggccccggggcccc
+DATA
+
+((ntest++))
+if $CMD --max-cpu 2 "${TMPDIR}/data.fasta" > "${TMPDIR}/all.csv" 2> /dev/null &&
+   [ "$(head -n 1 "${TMPDIR}/all.csv")" = "id1,id2,score" ] &&
+   [ "$(tail -n +2 "${TMPDIR}/all.csv" | wc -l | tr -d ' ')" = "10" ]
+then
+    log "$MCMD: without threshold all the pairs of the 5 usable sequences are written OK"
+    ((success++))
+else
+    log "$MCMD: the table without threshold is wrong"
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD --max-cpu 2 -t 0.95 "${TMPDIR}/data.fasta" > "${TMPDIR}/t95.csv" 2> /dev/null &&
+   [ "$(tail -n +2 "${TMPDIR}/t95.csv" | cut -d, -f1,2 | tr '\n' ' ')" = "a1,a2 a1,a3 a2,a3 b1,b2 " ] &&
+   grep -q '^a1,a3,1$' "${TMPDIR}/t95.csv"
+then
+    log "$MCMD: only the pairs reaching the threshold are written, in order OK"
+    ((success++))
+else
+    log "$MCMD: the table with a threshold is wrong"
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD --max-cpu 2 -t 0.95 -x "${TMPDIR}/data.fasta" 2> /dev/null | head -n 2 | tail -n 1 | grep -q '^a1,a2,[0-9.]*,20,3,52,52$'
+then
+    log "$MCMD: extra columns OK"
+    ((success++))
+else
+    log "$MCMD: extra columns failed"
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD --max-cpu 2 -g -t 0.9 "${TMPDIR}/data.fasta" 2> /dev/null | grep -q '^b1,x1,'
+then
+    log "$MCMD: a sequence with an IUPAC letter is kept with --accept-ambiguous OK"
+    ((success++))
+else
+    log "$MCMD: --accept-ambiguous failed"
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD --max-cpu 2 -R "${TMPDIR}/ref.fasta" -t 0.95 "${TMPDIR}/data.fasta" 2> /dev/null | tail -n +2 | cut -d, -f1,2 | tr '\n' ' ' > "${TMPDIR}/two.txt" &&
+   [ "$(cat "${TMPDIR}/two.txt")" = "a1,r1 a2,r1 a3,r1 b1,r2 b2,r2 " ]
+then
+    log "$MCMD: comparison with a reference file OK"
+    ((success++))
+else
+    log "$MCMD: comparison with a reference file failed"
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD --max-cpu 1 "${TMPDIR}/data.fasta" 2> /dev/null > "${TMPDIR}/cpu1.csv" &&
+   $CMD --max-cpu 8 "${TMPDIR}/data.fasta" 2> /dev/null > "${TMPDIR}/cpu8.csv" &&
+   cmp -s "${TMPDIR}/cpu1.csv" "${TMPDIR}/cpu8.csv"
+then
+    log "$MCMD: the table does not depend on the number of threads OK"
+    ((success++))
+else
+    log "$MCMD: the table depends on the number of threads"
+    ((failed++))
+fi
+
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obiclust/test.sh

@@ -0,0 +1,188 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obiclust
+CMD=obiclust
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1
+then
+    log "$MCMD: printing help OK"
+    ((success++))
+else
+    log "$MCMD: printing help failed"
+    ((failed++))
+fi
+
+
+#
+# Dataset used by the following tests: two families with a rare variant each,
+# a duplicate, and a sequence with an ambiguous nucleotide that is discarded.
+#
+cat > "${TMPDIR}/data.fasta" <<DATA
+>a1 count=20;
+acgtacgtacgtaacctagcctagcctagggatcgatcgatcgatcgaatcg
+>a2 count=3;
+acgtacgtacgtaacctagcctagcctagggatcgatcgatcgttcgaatcg
+>a3 count=2;
+acgtacgtacgtaacctagcctagcctagggatcgatcgatcgatcgaatcg
+>b1 count=15;
+ttgcatgcatgctagctagctagcatgcatcgatcgatgctagctagtagct
+>b2 count=1;
+ttgcatgcatgctagctagctagcatgcatcgatcgatgctagctagtaact
+>x1 count=4;
+ttgcatgcatgctangctagctagcatgcatcgatcgatgctagctagtagc
+DATA
+
+((ntest++))
+if $CMD --max-cpu 2 -O -t 0.95 "${TMPDIR}/data.fasta" > "${TMPDIR}/clusters.fasta" 2> "${TMPDIR}/clusters.log"
+then
+    log "$MCMD: clustering runs OK"
+    ((success++))
+else
+    log "$MCMD: clustering failed"
+    ((failed++))
+fi
+
+((ntest++))
+if [ "$(grep -c '^>' "${TMPDIR}/clusters.fasta")" = "5" ]
+then
+    log "$MCMD: the sequence with an ambiguous nucleotide is discarded OK"
+    ((success++))
+else
+    log "$MCMD: wrong number of sequences in the output"
+    ((failed++))
+fi
+
+((ntest++))
+if [ "$(grep '^>a[23] ' "${TMPDIR}/clusters.fasta" | grep -c 'cluster=a1;')" = "2" ] &&
+   [ "$(grep '^>b2 ' "${TMPDIR}/clusters.fasta" | grep -c 'cluster=b1;')" = "1" ] &&
+   grep '^>b1 ' "${TMPDIR}/clusters.fasta" | grep -q 'cluster_center=true;' &&
+   grep '^>a1 ' "${TMPDIR}/clusters.fasta" | grep -q 'cluster_weight=25;'
+then
+    log "$MCMD: sequences are attached to the right center OK"
+    ((success++))
+else
+    log "$MCMD: sequences are not attached to the right center"
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD --max-cpu 2 -t 0.95 --otu-map "${TMPDIR}/otu.csv" --no-sequence-output "${TMPDIR}/data.fasta" > "${TMPDIR}/none.fasta" 2> /dev/null &&
+   [ ! -s "${TMPDIR}/none.fasta" ] &&
+   [ "$(head -n 1 "${TMPDIR}/otu.csv")" = "id,cluster,cluster_center,cluster_score,count" ] &&
+   [ "$(sed -n 2,4p "${TMPDIR}/otu.csv" | cut -d, -f1,2 | tr '\n' ' ')" = "a1,a1 a3,a1 a2,a1 " ] &&
+   [ "$(grep -c ',true,' "${TMPDIR}/otu.csv")" = "2" ]
+then
+    log "$MCMD: CSV table of the clusters OK"
+    ((success++))
+else
+    log "$MCMD: CSV table of the clusters failed"
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD --max-cpu 1 -O -t 0.95 "${TMPDIR}/data.fasta" 2> /dev/null | grep '^>' | sed 's/ .*//' > "${TMPDIR}/order1.txt" &&
+   $CMD --max-cpu 8 -O -t 0.95 "${TMPDIR}/data.fasta" 2> /dev/null | grep '^>' | sed 's/ .*//' > "${TMPDIR}/order8.txt" &&
+   cmp -s "${TMPDIR}/order1.txt" "${TMPDIR}/order8.txt"
+then
+    log "$MCMD: the result does not depend on the number of threads OK"
+    ((success++))
+else
+    log "$MCMD: the result depends on the number of threads"
+    ((failed++))
+fi
+
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obiconvert/out_ecotag.fasta

@@ -0,0 +1,24 @@
+>HELIUM_000100422_612GNAAXX:7:118:3572:14633#0/1_sub[28..126] {"count":10172,"merged_sample":{"26a_F040644":10172},"obitag_bestid":0.9797979797979798,"obitag_bestmatch":"AY227529","obitag_match_count":1,"obitag_rank":"genus","obitag_similarity_method":"lcs","taxid":"taxon:9992 [Marmota]@genus"}
+ttagccctaaacataaacattcaataaacaagaatgttcgccagagtactactagcaaca
+gcctgaaactcaaaggacttggcggtgctttacatccct
+>HELIUM_000100422_612GNAAXX:7:99:9351:13090#0/1_sub[28..127] {"count":260,"merged_sample":{"29a_F260619":260},"obitag_bestid":0.9405940594059405,"obitag_bestmatch":"AF154263","obitag_match_count":9,"obitag_rank":"infraorder","obitag_similarity_method":"lcs","taxid":"taxon:35500 [Pecora]@infraorder"}
+ttagccctaaacacaaataattacacaaacaaaattgttcaccagagtactagcggcaac
+agcttaaaactcaaaggacttggcggtgctttataccctt
+>HELIUM_000100422_612GNAAXX:7:108:10111:9078#0/1_sub[28..127] {"count":7146,"merged_sample":{"13a_F730603":7146},"obitag_bestid":1,"obitag_bestmatch":"AB245427","obitag_match_count":1,"obitag_rank":"species","obitag_similarity_method":"lcs","taxid":"taxon:9860 [Cervus elaphus]@species"}
+ctagccttaaacacaaatagttatgcaaacaaaactattcgccagagtactaccggcaat
+agcttaaaactcaaaggacttggcggtgctttataccctt
+>HELIUM_000100422_612GNAAXX:7:38:14204:12725#0/1_sub[28..126] {"count":87,"merged_sample":{"26a_F040644":87},"obitag_bestid":0.9494949494949495,"obitag_bestmatch":"AY227530","obitag_match_count":2,"obitag_rank":"tribe","obitag_similarity_method":"lcs","taxid":"taxon:337730 [Marmotini]@tribe"}
+ttagccctaaacataaacattcaataaacaagaatgttcgccagaggactactagcaata
+gcttaaaactcaaaggacttggcggtgctttatatccct
+>HELIUM_000100422_612GNAAXX:7:30:9942:4495#0/1_sub[28..126] {"count":95,"merged_sample":{"26a_F040644":11,"29a_F260619":84},"obitag_bestid":0.9595959595959596,"obitag_bestmatch":"AC187326","obitag_match_count":1,"obitag_rank":"subspecies","obitag_similarity_method":"lcs","taxid":"taxon:9615 [Canis lupus familiaris]@subspecies"}
+ttagccctaaacataagctattccataacaaaataattcgccagagaactactagcaaca
+gattaaacctcaaaggacttggcagtgctttatacccct
+>HELIUM_000100422_612GNAAXX:7:51:16702:19393#0/1_sub[28..127] {"count":12004,"merged_sample":{"15a_F730814":7465,"29a_F260619":4539},"obitag_bestid":1,"obitag_bestmatch":"AJ885202","obitag_match_count":1,"obitag_rank":"species","obitag_similarity_method":"lcs","taxid":"taxon:9858 [Capreolus capreolus]@species"}
+ttagccctaaacacaagtaattaatataacaaaattattcgccagagtactaccggcaat
+agcttaaaactcaaaggacttggcggtgctttataccctt
+>HELIUM_000100422_612GNAAXX:7:84:14502:1617#0/1_sub[28..127] {"count":319,"merged_sample":{"29a_F260619":319},"obitag_bestid":1,"obitag_bestmatch":"AJ972683","obitag_match_count":1,"obitag_rank":"species","obitag_similarity_method":"lcs","taxid":"taxon:9858 [Capreolus capreolus]@species"}
+ttagccctaaacacaagtaattattataacaaaattattcgccagagtactaccggcaat
+agcttaaaactcaaaggacttggcggtgctttataccctt
+>HELIUM_000100422_612GNAAXX:7:50:10637:6527#0/1_sub[28..126] {"count":366,"merged_sample":{"13a_F730603":13,"15a_F730814":5,"26a_F040644":347,"29a_F260619":1},"obitag_bestid":1,"obitag_bestmatch":"AB048590","obitag_match_count":1,"obitag_rank":"genus","obitag_similarity_method":"lcs","taxid":"taxon:9611 [Canis]@genus"}
+ttagccctaaacatagataattttacaacaaaataattcgccagaggactactagcaata
+gcttaaaactcaaaggacttggcggtgctttatatccct

obitests/obitools/obiconvert/taxonomy.csv

@@ -0,0 +1,48 @@
+taxid,parent,taxonomic_rank,scientific_name
+taxon:1 [root]@no rank,taxon:1 [root]@no rank,no rank,root
+taxon:131567 [cellular organisms]@cellular root,taxon:1 [root]@no rank,cellular root,cellular organisms
+taxon:2759 [Eukaryota]@domain,taxon:131567 [cellular organisms]@cellular root,domain,Eukaryota
+taxon:33154 [Opisthokonta]@clade,taxon:2759 [Eukaryota]@domain,clade,Opisthokonta
+taxon:33208 [Metazoa]@kingdom,taxon:33154 [Opisthokonta]@clade,kingdom,Metazoa
+taxon:6072 [Eumetazoa]@clade,taxon:33208 [Metazoa]@kingdom,clade,Eumetazoa
+taxon:33213 [Bilateria]@clade,taxon:6072 [Eumetazoa]@clade,clade,Bilateria
+taxon:33511 [Deuterostomia]@clade,taxon:33213 [Bilateria]@clade,clade,Deuterostomia
+taxon:7711 [Chordata]@phylum,taxon:33511 [Deuterostomia]@clade,phylum,Chordata
+taxon:89593 [Craniata]@subphylum,taxon:7711 [Chordata]@phylum,subphylum,Craniata
+taxon:7742 [Vertebrata]@clade,taxon:89593 [Craniata]@subphylum,clade,Vertebrata
+taxon:7776 [Gnathostomata]@clade,taxon:7742 [Vertebrata]@clade,clade,Gnathostomata
+taxon:117570 [Teleostomi]@clade,taxon:7776 [Gnathostomata]@clade,clade,Teleostomi
+taxon:117571 [Euteleostomi]@clade,taxon:117570 [Teleostomi]@clade,clade,Euteleostomi
+taxon:8287 [Sarcopterygii]@superclass,taxon:117571 [Euteleostomi]@clade,superclass,Sarcopterygii
+taxon:1338369 [Dipnotetrapodomorpha]@clade,taxon:8287 [Sarcopterygii]@superclass,clade,Dipnotetrapodomorpha
+taxon:32523 [Tetrapoda]@clade,taxon:1338369 [Dipnotetrapodomorpha]@clade,clade,Tetrapoda
+taxon:32524 [Amniota]@clade,taxon:32523 [Tetrapoda]@clade,clade,Amniota
+taxon:40674 [Mammalia]@class,taxon:32524 [Amniota]@clade,class,Mammalia
+taxon:32525 [Theria]@clade,taxon:40674 [Mammalia]@class,clade,Theria
+taxon:9347 [Eutheria]@clade,taxon:32525 [Theria]@clade,clade,Eutheria
+taxon:1437010 [Boreoeutheria]@clade,taxon:9347 [Eutheria]@clade,clade,Boreoeutheria
+taxon:314146 [Euarchontoglires]@superorder,taxon:1437010 [Boreoeutheria]@clade,superorder,Euarchontoglires
+taxon:314145 [Laurasiatheria]@superorder,taxon:1437010 [Boreoeutheria]@clade,superorder,Laurasiatheria
+taxon:33554 [Carnivora]@order,taxon:314145 [Laurasiatheria]@superorder,order,Carnivora
+taxon:91561 [Artiodactyla]@order,taxon:314145 [Laurasiatheria]@superorder,order,Artiodactyla
+taxon:314147 [Glires]@clade,taxon:314146 [Euarchontoglires]@superorder,clade,Glires
+taxon:9845 [Ruminantia]@suborder,taxon:91561 [Artiodactyla]@order,suborder,Ruminantia
+taxon:35500 [Pecora]@infraorder,taxon:9845 [Ruminantia]@suborder,infraorder,Pecora
+taxon:9989 [Rodentia]@order,taxon:314147 [Glires]@clade,order,Rodentia
+taxon:379584 [Caniformia]@suborder,taxon:33554 [Carnivora]@order,suborder,Caniformia
+taxon:9608 [Canidae]@family,taxon:379584 [Caniformia]@suborder,family,Canidae
+taxon:9850 [Cervidae]@family,taxon:35500 [Pecora]@infraorder,family,Cervidae
+taxon:9881 [Odocoileinae]@subfamily,taxon:9850 [Cervidae]@family,subfamily,Odocoileinae
+taxon:33553 [Sciuromorpha]@suborder,taxon:9989 [Rodentia]@order,suborder,Sciuromorpha
+taxon:55153 [Sciuridae]@family,taxon:33553 [Sciuromorpha]@suborder,family,Sciuridae
+taxon:34878 [Cervinae]@subfamily,taxon:9850 [Cervidae]@family,subfamily,Cervinae
+taxon:9611 [Canis]@genus,taxon:9608 [Canidae]@family,genus,Canis
+taxon:9857 [Capreolus]@genus,taxon:9881 [Odocoileinae]@subfamily,genus,Capreolus
+taxon:9612 [Canis lupus]@species,taxon:9611 [Canis]@genus,species,Canis lupus
+taxon:337726 [Xerinae]@subfamily,taxon:55153 [Sciuridae]@family,subfamily,Xerinae
+taxon:9859 [Cervus]@genus,taxon:34878 [Cervinae]@subfamily,genus,Cervus
+taxon:337730 [Marmotini]@tribe,taxon:337726 [Xerinae]@subfamily,tribe,Marmotini
+taxon:9992 [Marmota]@genus,taxon:337730 [Marmotini]@tribe,genus,Marmota
+taxon:9860 [Cervus elaphus]@species,taxon:9859 [Cervus]@genus,species,Cervus elaphus
+taxon:9615 [Canis lupus familiaris]@subspecies,taxon:9612 [Canis lupus]@species,subspecies,Canis lupus familiaris
+taxon:9858 [Capreolus capreolus]@species,taxon:9857 [Capreolus]@genus,species,Capreolus capreolus

obitests/obitools/obiconvert/test.sh

@@ -44,7 +44,7 @@ cleanup() {
     rm -rf "$TMPDIR"  # Suppress the temporary directory
 
     if [ $failed -gt 0 ]; then
-       log "$TEST_NAME tests failed" 
+       log "$TEST_NAME tests failed"
         log
         log
        exit 1
@@ -60,10 +60,10 @@ log() {
     echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
 }
 
-log "Testing $TEST_NAME..." 
-log "Test directory is $TEST_DIR" 
-log "obitools directory is $OBITOOLS_DIR" 
-log "Temporary directory is $TMPDIR" 
+log "Testing $TEST_NAME..."
+log "Test directory is $TEST_DIR"
+log "obitools directory is $OBITOOLS_DIR"
+log "Temporary directory is $TMPDIR"
 log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
 
 ######################################################################
@@ -94,12 +94,12 @@ log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
 
 
 ((ntest++))
-if $CMD -h > "${TMPDIR}/help.txt" 2>&1 
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1
 then
-    log "$MCMD: printing help OK" 
+    log "$MCMD: printing help OK"
     ((success++))
 else
-    log "$MCMD: printing help failed" 
+    log "$MCMD: printing help failed"
     ((failed++))
 fi
 
@@ -108,15 +108,15 @@ fi
 if obiconvert -Z "${TEST_DIR}/gbpln1088.4Mb.fasta.gz" \
                  > "${TMPDIR}/xxx.fasta.gz" && \
    zdiff "${TEST_DIR}/gbpln1088.4Mb.fasta.gz" \
-                 "${TMPDIR}/xxx.fasta.gz" 
+                 "${TMPDIR}/xxx.fasta.gz"
 then
-    log "$MCMD: converting large fasta file to fasta OK" 
+    log "$MCMD: converting large fasta file to fasta OK"
     ((success++))
 else
-    log "$MCMD: converting large fasta file to fasta failed" 
+    log "$MCMD: converting large fasta file to fasta failed"
     ((failed++))
 fi
- 
+
 ((ntest++))
 if obiconvert -Z --fastq-output \
               "${TEST_DIR}/gbpln1088.4Mb.fasta.gz" \
@@ -125,15 +125,139 @@ if obiconvert -Z --fastq-output \
               "${TMPDIR}/xxx.fastq.gz" \
               > "${TMPDIR}/yyy.fasta.gz" && \
    zdiff "${TEST_DIR}/gbpln1088.4Mb.fasta.gz" \
-                 "${TMPDIR}/yyy.fasta.gz" 
+                 "${TMPDIR}/yyy.fasta.gz"
 then
-    log "$MCMD: converting large file between fasta and fastq OK" 
+    log "$MCMD: converting large file between fasta and fastq OK"
     ((success++))
 else
-    log "$MCMD: converting large file between fasta and fastq failed" 
+    log "$MCMD: converting large file between fasta and fastq failed"
     ((failed++))
 fi
 
+
+# ------------------------------------------------------------------
+# --raw-taxid tests (no taxonomy loaded)
+# ------------------------------------------------------------------
+
+# Running test
+((ntest++))
+if obiconvert --raw-taxid "${TEST_DIR}/out_ecotag.fasta" \
+              > "${TMPDIR}/raw_taxid.fasta" 2>/dev/null
+then
+    log "$MCMD --raw-taxid: running OK"
+    ((success++))
+else
+    log "$MCMD --raw-taxid: running failed"
+    ((failed++))
+fi
+
+# Taxids must be bare numbers — no full-format "taxon:ID [Name]@rank" strings
+((ntest++))
+if grep '"taxid"' "${TMPDIR}/raw_taxid.fasta" | grep -qv '"taxid":"[0-9][0-9]*"'
+then
+    log "$MCMD --raw-taxid: taxid format check failed (full-format taxid found)"
+    ((failed++))
+else
+    log "$MCMD --raw-taxid: taxid format OK (all taxids are bare numbers)"
+    ((success++))
+fi
+
+# --raw-taxid is idempotent: piping through a second obiconvert --raw-taxid must
+# produce bit-for-bit identical output.
+((ntest++))
+if obiconvert --raw-taxid "${TMPDIR}/raw_taxid.fasta" \
+              > "${TMPDIR}/raw_taxid2.fasta" 2>/dev/null
+then
+    log "$MCMD --raw-taxid piped: running OK"
+    ((success++))
+else
+    log "$MCMD --raw-taxid piped: running failed"
+    ((failed++))
+fi
+
+((ntest++))
+if diff "${TMPDIR}/raw_taxid.fasta" \
+        "${TMPDIR}/raw_taxid2.fasta" > /dev/null
+then
+    log "$MCMD --raw-taxid piped: idempotency OK"
+    ((success++))
+else
+    log "$MCMD --raw-taxid piped: idempotency failed (outputs differ)"
+    ((failed++))
+fi
+
+
+# ------------------------------------------------------------------
+# --taxonomy tests (full-format taxid, no --raw-taxid)
+# ------------------------------------------------------------------
+
+# Running test
+((ntest++))
+if obiconvert --taxonomy "${TEST_DIR}/taxonomy.csv" \
+              "${TEST_DIR}/out_ecotag.fasta" \
+              > "${TMPDIR}/taxo.fasta" 2>/dev/null
+then
+    log "$MCMD --taxonomy: running OK"
+    ((success++))
+else
+    log "$MCMD --taxonomy: running failed"
+    ((failed++))
+fi
+
+# Taxids must be in full "taxon:ID [Name]@rank" format
+((ntest++))
+if grep '"taxid"' "${TMPDIR}/taxo.fasta" | grep -q '"taxid":"taxon:[0-9]'
+then
+    log "$MCMD --taxonomy: taxid format OK (full-format taxids present)"
+    ((success++))
+else
+    log "$MCMD --taxonomy: taxid format check failed (no full-format taxid found)"
+    ((failed++))
+fi
+
+
+# ------------------------------------------------------------------
+# --raw-taxid --taxonomy tests
+# ------------------------------------------------------------------
+
+# Running test
+((ntest++))
+if obiconvert --raw-taxid --taxonomy "${TEST_DIR}/taxonomy.csv" \
+              "${TEST_DIR}/out_ecotag.fasta" \
+              > "${TMPDIR}/raw_taxid_taxo.fasta" 2>/dev/null
+then
+    log "$MCMD --raw-taxid --taxonomy: running OK"
+    ((success++))
+else
+    log "$MCMD --raw-taxid --taxonomy: running failed"
+    ((failed++))
+fi
+
+# Taxids must be bare numbers even when taxonomy is loaded
+((ntest++))
+if grep '"taxid"' "${TMPDIR}/raw_taxid_taxo.fasta" | grep -qv '"taxid":"[0-9][0-9]*"'
+then
+    log "$MCMD --raw-taxid --taxonomy: taxid format check failed (full-format taxid found)"
+    ((failed++))
+else
+    log "$MCMD --raw-taxid --taxonomy: taxid format OK (all taxids are bare numbers)"
+    ((success++))
+fi
+
+# --raw-taxid with or without taxonomy must yield identical taxid values
+((ntest++))
+if diff <(grep '"taxid"' "${TMPDIR}/raw_taxid.fasta" | grep -o '"taxid":"[^"]*"' | sort) \
+        <(grep '"taxid"' "${TMPDIR}/raw_taxid_taxo.fasta" | grep -o '"taxid":"[^"]*"' | sort) \
+        > /dev/null
+then
+    log "$MCMD --raw-taxid vs --raw-taxid --taxonomy: taxid values match OK"
+    ((success++))
+else
+    log "$MCMD --raw-taxid vs --raw-taxid --taxonomy: taxid values differ (unexpected)"
+    ((failed++))
+fi
+
+
 #########################################
 #
 # At the end of the tests

obitests/obitools/obiconvert/xxx

@@ -0,0 +1,24 @@
+>HELIUM_000100422_612GNAAXX:7:118:3572:14633#0/1_sub[28..126] {"count":10172,"merged_sample":{"26a_F040644":10172},"obitag_bestid":0.9797979797979798,"obitag_bestmatch":"AY227529","obitag_match_count":1,"obitag_rank":"genus","obitag_similarity_method":"lcs","taxid":"taxon:9992 [Marmota]@genus"}
+ttagccctaaacataaacattcaataaacaagaatgttcgccagagtactactagcaaca
+gcctgaaactcaaaggacttggcggtgctttacatccct
+>HELIUM_000100422_612GNAAXX:7:99:9351:13090#0/1_sub[28..127] {"count":260,"merged_sample":{"29a_F260619":260},"obitag_bestid":0.9405940594059405,"obitag_bestmatch":"AF154263","obitag_match_count":9,"obitag_rank":"infraorder","obitag_similarity_method":"lcs","taxid":"taxon:35500 [Pecora]@infraorder"}
+ttagccctaaacacaaataattacacaaacaaaattgttcaccagagtactagcggcaac
+agcttaaaactcaaaggacttggcggtgctttataccctt
+>HELIUM_000100422_612GNAAXX:7:108:10111:9078#0/1_sub[28..127] {"count":7146,"merged_sample":{"13a_F730603":7146},"obitag_bestid":1,"obitag_bestmatch":"AB245427","obitag_match_count":1,"obitag_rank":"species","obitag_similarity_method":"lcs","taxid":"taxon:9860 [Cervus elaphus]@species"}
+ctagccttaaacacaaatagttatgcaaacaaaactattcgccagagtactaccggcaat
+agcttaaaactcaaaggacttggcggtgctttataccctt
+>HELIUM_000100422_612GNAAXX:7:38:14204:12725#0/1_sub[28..126] {"count":87,"merged_sample":{"26a_F040644":87},"obitag_bestid":0.9494949494949495,"obitag_bestmatch":"AY227530","obitag_match_count":2,"obitag_rank":"tribe","obitag_similarity_method":"lcs","taxid":"taxon:337730 [Marmotini]@tribe"}
+ttagccctaaacataaacattcaataaacaagaatgttcgccagaggactactagcaata
+gcttaaaactcaaaggacttggcggtgctttatatccct
+>HELIUM_000100422_612GNAAXX:7:30:9942:4495#0/1_sub[28..126] {"count":95,"merged_sample":{"26a_F040644":11,"29a_F260619":84},"obitag_bestid":0.9595959595959596,"obitag_bestmatch":"AC187326","obitag_match_count":1,"obitag_rank":"subspecies","obitag_similarity_method":"lcs","taxid":"taxon:9615 [Canis lupus familiaris]@subspecies"}
+ttagccctaaacataagctattccataacaaaataattcgccagagaactactagcaaca
+gattaaacctcaaaggacttggcagtgctttatacccct
+>HELIUM_000100422_612GNAAXX:7:51:16702:19393#0/1_sub[28..127] {"count":12004,"merged_sample":{"15a_F730814":7465,"29a_F260619":4539},"obitag_bestid":1,"obitag_bestmatch":"AJ885202","obitag_match_count":1,"obitag_rank":"species","obitag_similarity_method":"lcs","taxid":"taxon:9858 [Capreolus capreolus]@species"}
+ttagccctaaacacaagtaattaatataacaaaattattcgccagagtactaccggcaat
+agcttaaaactcaaaggacttggcggtgctttataccctt
+>HELIUM_000100422_612GNAAXX:7:84:14502:1617#0/1_sub[28..127] {"count":319,"merged_sample":{"29a_F260619":319},"obitag_bestid":1,"obitag_bestmatch":"AJ972683","obitag_match_count":1,"obitag_rank":"species","obitag_similarity_method":"lcs","taxid":"taxon:9858 [Capreolus capreolus]@species"}
+ttagccctaaacacaagtaattattataacaaaattattcgccagagtactaccggcaat
+agcttaaaactcaaaggacttggcggtgctttataccctt
+>HELIUM_000100422_612GNAAXX:7:50:10637:6527#0/1_sub[28..126] {"count":366,"merged_sample":{"13a_F730603":13,"15a_F730814":5,"26a_F040644":347,"29a_F260619":1},"obitag_bestid":1,"obitag_bestmatch":"AB048590","obitag_match_count":1,"obitag_rank":"genus","obitag_similarity_method":"lcs","taxid":"taxon:9611 [Canis]@genus"}
+ttagccctaaacatagataattttacaacaaaataattcgccagaggactactagcaata
+gcttaaaactcaaaggacttggcggtgctttatatccct

obitests/obitools/obioligotag/test.sh

@@ -0,0 +1,205 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obioligotag
+CMD=obioligotag
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+#
+# Every result is checked on its properties (size, alphabet, constraints),
+# not against a reference file, because several equivalent families exist.
+#
+
+printf 'acgtac\nacgtat\nttttgg\nggcaac\ncaccat\nacgtac\n' > "${TMPDIR}/oligos.txt"
+printf '>a\nACGTAC\n>b\nACGTAT\n>c\nTTTTGG\n>d\nGGCAAC\n>e\nCACCAT\n' > "${TMPDIR}/oligos.fasta"
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1 
+then
+    log "$MCMD: printing help OK" 
+    ((success++))
+else
+    log "$MCMD: printing help failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD -s 5 -f 24 -d 3 -g 3 -p 2 > "${TMPDIR}/tags.fasta" 2> /dev/null \
+   && [ "$(grep -vc '>' "${TMPDIR}/tags.fasta")" -ge 24 ] \
+   && ! grep -v '>' "${TMPDIR}/tags.fasta" | grep -qE '(a{3}|c{3}|g{3}|t{3})'
+then
+    log "$MCMD: generating a family with homopolymer constraint OK" 
+    ((success++))
+else
+    log "$MCMD: generating a family with homopolymer constraint failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD -s 7 -f 60 -d 3 --alphabet ACT > "${TMPDIR}/noG.fasta" 2> /dev/null \
+   && [ "$(grep -vc '>' "${TMPDIR}/noG.fasta")" -ge 60 ] \
+   && ! grep -v '>' "${TMPDIR}/noG.fasta" | grep -q 'g'
+then
+    log "$MCMD: restricted alphabet OK" 
+    ((success++))
+else
+    log "$MCMD: restricted alphabet failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD -s 6 -f 20 -d 3 --distance-type lcs > "${TMPDIR}/lcs.fasta" 2> /dev/null \
+   && [ "$(grep -vc '>' "${TMPDIR}/lcs.fasta")" -ge 20 ]
+then
+    log "$MCMD: LCS distance OK" 
+    ((success++))
+else
+    log "$MCMD: LCS distance failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD -s 7 -f 40 -d 3 -r cnnnnnn > "${TMPDIR}/rejected.fasta" 2> /dev/null \
+   && [ "$(grep -vc '>' "${TMPDIR}/rejected.fasta")" -ge 40 ] \
+   && ! grep -v '>' "${TMPDIR}/rejected.fasta" | grep -q '^c'
+then
+    log "$MCMD: rejected pattern OK" 
+    ((success++))
+else
+    log "$MCMD: rejected pattern failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD -L "${TMPDIR}/oligos.txt" -f 3 -d 3 > "${TMPDIR}/from_text.fasta" 2> /dev/null \
+   && [ "$(grep -vc '>' "${TMPDIR}/from_text.fasta")" -ge 3 ]
+then
+    log "$MCMD: selecting oligonucleotides from a text file OK" 
+    ((success++))
+else
+    log "$MCMD: selecting oligonucleotides from a text file failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD -L "${TMPDIR}/oligos.fasta" -f 3 -d 3 > "${TMPDIR}/from_fasta.fasta" 2> /dev/null \
+   && [ "$(grep -vc '>' "${TMPDIR}/from_fasta.fasta")" -ge 3 ]
+then
+    log "$MCMD: selecting oligonucleotides from a fasta file OK" 
+    ((success++))
+else
+    log "$MCMD: selecting oligonucleotides from a fasta file failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if ! $CMD -s 5 -f 100 -d 3 > /dev/null 2>&1
+then
+    log "$MCMD: family size beyond the Hamming bound refused OK" 
+    ((success++))
+else
+    log "$MCMD: family size beyond the Hamming bound not refused" 
+    ((failed++))
+fi
+
+((ntest++))
+if ! $CMD -s 5 -L "${TMPDIR}/oligos.txt" -f 3 > /dev/null 2>&1
+then
+    log "$MCMD: --oligo-size with --oligo-list refused OK" 
+    ((success++))
+else
+    log "$MCMD: --oligo-size with --oligo-list not refused" 
+    ((failed++))
+fi
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

pkg/obialign/fastlcs_bench_test.go

@@ -0,0 +1,111 @@
+package obialign
+
+import (
+	"fmt"
+	"math/rand"
+	"testing"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+// randomDNA returns a random ACGT sequence of length n.
+func randomDNA(rng *rand.Rand, n int) []byte {
+	const nuc = "ACGT"
+	s := make([]byte, n)
+	for i := range s {
+		s[i] = nuc[rng.Intn(4)]
+	}
+	return s
+}
+
+// mutate returns a copy of s in which each position is, with probability
+// rate, substituted, deleted or followed by an inserted base (equiprobable).
+func mutate(rng *rand.Rand, s []byte, rate float64) []byte {
+	const nuc = "ACGT"
+	out := make([]byte, 0, len(s)+len(s)/10)
+	for _, c := range s {
+		if rng.Float64() >= rate {
+			out = append(out, c)
+			continue
+		}
+		switch rng.Intn(3) {
+		case 0:
+			out = append(out, nuc[rng.Intn(4)])
+		case 1: // deletion
+		case 2:
+			out = append(out, c, nuc[rng.Intn(4)])
+		}
+	}
+	return out
+}
+
+// benchPairs builds a fixed set of pairs. "similar" pairs differ by about
+// 2% of mutated positions, "random" pairs are unrelated sequences.
+func benchPairs(n, length int, similar bool) [][2]*obiseq.BioSequence {
+	rng := rand.New(rand.NewSource(42))
+	pairs := make([][2]*obiseq.BioSequence, n)
+	for i := range pairs {
+		a := randomDNA(rng, length)
+		var b []byte
+		if similar {
+			b = mutate(rng, a, 0.02)
+		} else {
+			b = randomDNA(rng, length)
+		}
+		pairs[i] = [2]*obiseq.BioSequence{
+			obiseq.NewBioSequence("a", a, ""),
+			obiseq.NewBioSequence("b", b, ""),
+		}
+	}
+	return pairs
+}
+
+// BenchmarkLegacyLCS measures the historical implementation (the one
+// FastLCSScore used before BandedLCS), kept as the "before" reference.
+func BenchmarkLegacyLCS(b *testing.B) {
+	for _, similar := range []bool{true, false} {
+		kind := "random"
+		if similar {
+			kind = "similar"
+		}
+		for _, length := range []int{100, 500, 2000} {
+			pairs := benchPairs(64, length, similar)
+			for _, maxErr := range []int{-1, 5, length / 20, length / 5} {
+				name := fmt.Sprintf("%s/len=%d/maxErr=%d", kind, length, maxErr)
+				b.Run(name, func(b *testing.B) {
+					var buffer []uint64
+					b.ReportAllocs()
+					b.ResetTimer()
+					for i := 0; i < b.N; i++ {
+						p := pairs[i%len(pairs)]
+						FastLCSEGFScoreByte(p[0].Sequence(), p[1].Sequence(), maxErr, false, &buffer)
+					}
+				})
+			}
+		}
+	}
+}
+
+// BenchmarkBandedLCS runs on the same pairs as BenchmarkLegacyLCS.
+func BenchmarkBandedLCS(b *testing.B) {
+	for _, similar := range []bool{true, false} {
+		kind := "random"
+		if similar {
+			kind = "similar"
+		}
+		for _, length := range []int{100, 500, 2000} {
+			pairs := benchPairs(64, length, similar)
+			for _, maxErr := range []int{-1, 5, length / 20, length / 5} {
+				name := fmt.Sprintf("%s/len=%d/maxErr=%d", kind, length, maxErr)
+				b.Run(name, func(b *testing.B) {
+					b.ReportAllocs()
+					b.ResetTimer()
+					for i := 0; i < b.N; i++ {
+						p := pairs[i%len(pairs)]
+						BandedLCS(p[0].Sequence(), p[1].Sequence(), maxErr)
+					}
+				})
+			}
+		}
+	}
+}

pkg/obialign/fastlcsegf.go

@@ -2,6 +2,7 @@ package obialign
 
 import (
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+	log "github.com/sirupsen/logrus"
 )
 
 // FastLCSEGFScoreByte calculates the score of the Longest Common Subsequence (LCS) between two byte slices.
@@ -355,27 +356,35 @@ func FastLCSEGFScore(seqA, seqB *obiseq.BioSequence, maxError int, buffer *[]uin
 	return FastLCSEGFScoreByte(seqA.Sequence(), seqB.Sequence(), maxError, true, buffer)
 }
 
-// FastLCSScore calculates the score of the longest common subsequence between two bio sequences.
+// FastLCSScore calculates the score of the longest common subsequence between two bio sequences,
+// and the length of the shortest alignment achieving it (the distance is alilen - lcs).
 //
-// if maxError > 0, the maximum allowed error between the sequences is maxError.
-// Otherwise, no error checking is done.
-// If the actual number of errors is larger than maxError, -1 is returned for both values.
+// If maxError >= 0, the result is exact when its distance is at most maxError,
+// and (-1, -1) is returned otherwise. If maxError < 0, no limit is applied.
+// See BandedLCS, which does the work.
 //
 // The score matrix is:
-//   - Matching: 1
+//   - Matching: 1 (bases are compared with the IUPAC rules)
 //   - Mismatch or gap: 0
 //
+// The program stops with a fatal error if a sequence is longer than
+// MaxBandedLCSLength.
+//
 // Parameters:
 // - seqA: The first bio sequence.
 // - seqB: The second bio sequence.
 // - maxError: The maximum allowed error between the sequences. If set to -1, no limit is applied.
-// - buffer: A pointer to a uint64 slice to store intermediate results. If nil, a new slice is created.
+// - buffer: Unused, kept for compatibility; the working memory is pooled.
 //
 // Returns:
 // - The score of the longest common subsequence.
 // - The length of the shortest alignment corresponding to the LCS.
 func FastLCSScore(seqA, seqB *obiseq.BioSequence, maxError int, buffer *[]uint64) (int, int) {
-	score, alilen, _ := FastLCSEGFScoreByte(seqA.Sequence(), seqB.Sequence(), maxError, false, buffer)
+	score, alilen, err := BandedLCS(seqA.Sequence(), seqB.Sequence(), maxError)
+	if err != nil {
+		log.Fatalf("FastLCSScore on sequences %s (%d bp) and %s (%d bp): %v (maximum %d bp)",
+			seqA.Id(), seqA.Len(), seqB.Id(), seqB.Len(), err, MaxBandedLCSLength)
+	}
 
 	return score, alilen
 }

pkg/obialign/lcsband.go

@@ -0,0 +1,185 @@
+package obialign
+
+import (
+	"errors"
+	"sync"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+// MaxBandedLCSLength is the longest sequence BandedLCS accepts. The score and
+// the alignment length are packed on 16 bits each, and the alignment length
+// can reach the sum of both sequence lengths.
+const MaxBandedLCSLength = 1<<15 - 1
+
+// ErrSequenceTooLong is returned by BandedLCS when a sequence is longer than
+// MaxBandedLCSLength.
+var ErrSequenceTooLong = errors.New("obialign: sequence too long for BandedLCS")
+
+// A DP cell is packed on 32 bits as
+//
+//	score<<16 | (lcsLenBase - alignment length)
+//
+// so that the maximum of two cells ranks them by score first and, for equal
+// scores, by shortest alignment. The three moves then reduce to
+//
+//	cell(i,j) = max(diag + match<<16, up, left) - 1
+//
+// since every move lengthens the alignment by one and only a diagonal match
+// adds one to the score. The value 0 stands for an unavailable cell (outside
+// the band): it loses against any real cell. Every cell of the band has at
+// least one real neighbour (the band is connected and contains (0,0)), so
+// the subtraction never wraps for a cell that is used; the scalar reference
+// still clamps the maximum to 1 as a safety net.
+const (
+	lcsScoreUnit = uint32(1) << 16
+	lcsLenBase   = uint32(0xFFFF)
+)
+
+// bandedLCSWorkspace holds the scratch memory of one alignment.
+type bandedLCSWorkspace struct {
+	aRev, bCode []uint32    // encoded bases (aRev is reversed), SIMD kernel only
+	rows        [3][]uint32 // antidiagonal (SIMD) or row (scalar) buffers
+	aRev16      []uint16    // 16-bit encoded bases of the score-only kernel
+	bCode16     []uint16
+	rows16      [3][]uint16
+	widened     int // number of alignments that needed a second pass
+}
+
+var bandedLCSPool = sync.Pool{New: func() any { return new(bandedLCSWorkspace) }}
+
+func growUint32(s []uint32, n int) []uint32 {
+	if cap(s) < n {
+		return make([]uint32, n)
+	}
+	return s[:n]
+}
+
+// iupacMask gives for each byte its 4-bit nucleotide set (A=1, C=2, G=4,
+// T=8), or 0 when the byte is not an IUPAC letter. For two bytes of non-zero
+// mask, SameIUPACNuc(a, b) is exactly (mask[a] & mask[b]) != 0.
+var iupacMask = buildIUPACMask()
+
+func buildIUPACMask() (m [256]uint8) {
+	probes := [4]byte{'a', 'c', 'g', 't'}
+	for c := 0; c < 256; c++ {
+		if !(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z') {
+			continue
+		}
+		for bit, p := range probes {
+			if obiseq.SameIUPACNuc(byte(c), p) {
+				m[c] |= 1 << uint(bit)
+			}
+		}
+	}
+	return m
+}
+
+// allRegularIUPAC reports whether every base has a non-zero IUPAC mask, in
+// which case base compatibility is a plain AND of masks.
+func allRegularIUPAC(s []byte) bool {
+	for _, c := range s {
+		if iupacMask[c] == 0 {
+			return false
+		}
+	}
+	return true
+}
+
+// bandOffsets returns the diagonal offsets k = j - i covered by a band of
+// half-width g around the diagonals 0..delta, with sequence A (columns j) not
+// shorter than sequence B (rows i), clipped to the matrix.
+func bandOffsets(lA, lB, g int) (kmin, kmax int) {
+	return max(-g, -lB), min(lA-lB+g, lA)
+}
+
+// BandedLCS computes the score of the longest common subsequence of two
+// nucleotide sequences, and the length of the shortest alignment achieving
+// it (match: +1, mismatch and gap: 0; bases are compared with the IUPAC
+// rules of obiseq.SameIUPACNuc). The distance is alilen - lcs.
+//
+// If maxError >= 0, the result is the optimal (lcs, alilen) when its distance
+// is at most maxError, and (-1, -1) otherwise. This is exact, not a
+// heuristic: the DP is restricted to a band, and a bound on what a path
+// leaving the band can score proves when the band was wide enough. maxError < 0
+// means no limit.
+//
+// An error is returned when a sequence is longer than MaxBandedLCSLength.
+func BandedLCS(seqA, seqB []byte, maxError int) (lcs, alilen int, err error) {
+	if len(seqA) > MaxBandedLCSLength || len(seqB) > MaxBandedLCSLength {
+		return -1, -1, ErrSequenceTooLong
+	}
+	if len(seqA) < len(seqB) {
+		seqA, seqB = seqB, seqA
+	}
+	ws := bandedLCSPool.Get().(*bandedLCSWorkspace)
+	lcs, alilen = bandedLCS(seqA, seqB, maxError, ws, haveSIMD)
+	bandedLCSPool.Put(ws)
+	return lcs, alilen, nil
+}
+
+// bandedLCS expects len(a) >= len(b), both within MaxBandedLCSLength.
+//
+// With delta = lA - lB and a band of half-width g (offsets -g .. delta+g):
+//   - an alignment with at most E errors needs at least delta+2h gaps to reach
+//     the offset delta+h or -h, so it lies in the band of g0 = (E-delta)/2;
+//   - a path leaving a band of half-width g makes at least g+1 gaps on the B
+//     side, hence scores at most lB-g-1 matches, and has more than E errors
+//     as soon as g >= g0.
+//
+// Let Q be the best path of the band g0. If its distance exceeds E, the
+// global optimum cannot be within E either (it would be in the band and be
+// Q): the answer is -1. Otherwise, if score(Q) >= lB-g0-1 no outside path can
+// beat Q (a tie in score loses on length, the outside path having more errors
+// than Q): Q is the global optimum. Else the band is widened once to
+// g1 = min(E-delta, lB-score(Q)-1); since score(Q) >= lA-E for any alignment
+// within E errors, g1 always certifies the result.
+func bandedLCS(a, b []byte, maxError int, ws *bandedLCSWorkspace, useSIMD bool) (int, int) {
+	lA, lB := len(a), len(b)
+	delta := lA - lB
+	if maxError >= 0 && maxError < delta {
+		return -1, -1
+	}
+	if lB == 0 {
+		return 0, lA
+	}
+	simdOK := useSIMD && allRegularIUPAC(a) && allRegularIUPAC(b)
+	if simdOK {
+		encodeBases(ws, a, b)
+	}
+	run := func(g int) (int, int) {
+		kmin, kmax := bandOffsets(lA, lB, g)
+		return ws.align(a, b, kmin, kmax, simdOK)
+	}
+
+	if maxError < 0 {
+		return run(lA)
+	}
+	g0 := (maxError - delta) / 2
+	lcs, alilen := run(g0)
+	if alilen-lcs > maxError {
+		return -1, -1
+	}
+	if lcs >= lB-g0-1 {
+		return lcs, alilen
+	}
+	ws.widened++
+	lcs, alilen = run(min(maxError-delta, lB-lcs-1))
+	if alilen-lcs > maxError {
+		return -1, -1
+	}
+	return lcs, alilen
+}
+
+// align runs the banded DP on the offsets [kmin, kmax] (kmin <= 0 <= kmax) and
+// returns the best (lcs, alilen) of the band. The bases must have been encoded
+// with encodeBases when simdOK is set.
+func (ws *bandedLCSWorkspace) align(a, b []byte, kmin, kmax int, simdOK bool) (int, int) {
+	var key uint32
+	if simdOK {
+		key = bandedLCSSIMD(len(a), len(b), kmin, kmax, ws)
+	} else {
+		key = bandedLCSScalar(a, b, kmin, kmax, ws)
+	}
+	return int(key >> 16), int(lcsLenBase - key&0xFFFF)
+}

pkg/obialign/lcsband_nosimd.go

@@ -0,0 +1,17 @@
+//go:build !goexperiment.simd
+
+package obialign
+
+const haveSIMD = false
+
+// encodeBases and bandedLCSSIMD are never called without GOEXPERIMENT=simd
+// (haveSIMD is false); they exist to keep the dispatcher uniform.
+func encodeBases(ws *bandedLCSWorkspace, a, b []byte) {}
+
+func bandedLCSSIMD(lA, lB, kmin, kmax int, ws *bandedLCSWorkspace) uint32 {
+	return 0
+}
+
+func lcsScoreSIMD(a, b []byte, kmin, kmax, minLCS int, ws *bandedLCSWorkspace) int {
+	return -1
+}

pkg/obialign/lcsband_scalar.go

@@ -0,0 +1,53 @@
+package obialign
+
+import "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+
+// bandedLCSScalar is the portable reference of the banded DP, row by row. It
+// returns the packed cell (lB, lA). len(a) >= len(b) >= 1.
+//
+// Cells outside the band hold 0 (unavailable). Row i spans the columns
+// [i+kmin, i+kmax] clipped to [1, lA]; the cell just left of that span and
+// the one just right of it are reset so that the next row never reads a stale
+// value.
+func bandedLCSScalar(a, b []byte, kmin, kmax int, ws *bandedLCSWorkspace) uint32 {
+	lA, lB := len(a), len(b)
+
+	prev := growUint32(ws.rows[0], lA+2)
+	cur := growUint32(ws.rows[1], lA+2)
+	ws.rows[0], ws.rows[1] = prev, cur
+
+	for j := 0; j <= lA; j++ {
+		prev[j] = 0
+		if j <= kmax {
+			prev[j] = lcsLenBase - uint32(j)
+		}
+	}
+	prev[lA+1] = 0
+
+	for i := 1; i <= lB; i++ {
+		if -i >= kmin {
+			cur[0] = lcsLenBase - uint32(i)
+		} else {
+			cur[0] = 0
+		}
+		jlo := max(1, i+kmin)
+		jhi := min(lA, i+kmax)
+		if jlo > 1 {
+			cur[jlo-1] = 0
+		}
+		bi := b[i-1]
+		for j := jlo; j <= jhi; j++ {
+			d := prev[j-1]
+			if obiseq.SameIUPACNuc(a[j-1], bi) {
+				d += lcsScoreUnit
+			}
+			m := max(d, prev[j], cur[j-1], 1)
+			cur[j] = m - 1
+		}
+		if jhi < lA {
+			cur[jhi+1] = 0
+		}
+		prev, cur = cur, prev
+	}
+	return prev[lA]
+}

pkg/obialign/lcsband_simd.go

@@ -0,0 +1,107 @@
+//go:build goexperiment.simd
+
+package obialign
+
+import "simd"
+
+const haveSIMD = true
+
+func simdLanes() int {
+	var probe simd.Uint32s
+	return probe.Len()
+}
+
+// encodeIUPAC writes in dst the 4-bit IUPAC mask of each base of s, reversed
+// if rev is set, followed by zero padding up to len(dst).
+func encodeIUPAC(dst []uint32, s []byte, rev bool) {
+	n := len(s)
+	for x := range dst {
+		dst[x] = 0
+	}
+	if rev {
+		for x, c := range s {
+			dst[n-1-x] = uint32(iupacMask[c])
+		}
+		return
+	}
+	for x, c := range s {
+		dst[x] = uint32(iupacMask[c])
+	}
+}
+
+// encodeBases encodes both sequences once for all the passes of an alignment.
+func encodeBases(ws *bandedLCSWorkspace, a, b []byte) {
+	vl := simdLanes()
+	ws.aRev = growUint32(ws.aRev, len(a)+vl)
+	ws.bCode = growUint32(ws.bCode, len(b)+vl)
+	encodeIUPAC(ws.aRev, a, true)
+	encodeIUPAC(ws.bCode, b, false)
+}
+
+// bandedLCSSIMD runs the banded DP along antidiagonals t = i + j. Lane i of
+// the buffer of antidiagonal t holds the cell (i, t-i), so that
+//
+//	diag = buffer[t-2][i-1]   up = buffer[t-1][i-1]   left = buffer[t-1][i]
+//
+// are plain unaligned loads. Along an antidiagonal the offset j-i = t-2i
+// must stay in [kmin, kmax], which gives the window [lo, hi] of rows. The
+// vector loop may run past hi; the lanes beyond are then reset to 0
+// (unavailable), as is the lane just before the window, because the next
+// antidiagonals read them as neighbours. Lane 0 and lane t carry the first
+// row and the first column of the matrix.
+//
+// Requires encodeBases to have been called, lA >= lB >= 1 and kmin <= 0 <=
+// kmax.
+func bandedLCSSIMD(lA, lB, kmin, kmax int, ws *bandedLCSWorkspace) uint32 {
+	vl := simdLanes()
+	aRev, bCode := ws.aRev, ws.bCode
+
+	size := lB + 2*vl + 2
+	for r := range ws.rows {
+		ws.rows[r] = growUint32(ws.rows[r], size)
+		clear(ws.rows[r])
+	}
+	p2, p1, cur := ws.rows[0], ws.rows[1], ws.rows[2]
+
+	p2[0] = lcsLenBase // cell (0,0)
+	if kmax >= 1 {
+		p1[0] = lcsLenBase - 1 // cell (0,1)
+	}
+	if kmin <= -1 {
+		p1[1] = lcsLenBase - 1 // cell (1,0)
+	}
+
+	zero := simd.BroadcastUint32s(0)
+	one := simd.BroadcastUint32s(1)
+	unit := simd.BroadcastUint32s(lcsScoreUnit)
+
+	for t := 2; t <= lA+lB; t++ {
+		lo := max(1, t-lA, (t-kmax+1)>>1)
+		hi := min(lB, t-1, (t-kmin)>>1)
+
+		for i := lo; i <= hi; i += vl {
+			ca := simd.LoadUint32s(aRev[lA-t+i:])
+			cb := simd.LoadUint32s(bCode[i-1:])
+			inc := unit.Masked(ca.And(cb).NotEqual(zero))
+			d := simd.LoadUint32s(p2[i-1:]).Add(inc)
+			u := simd.LoadUint32s(p1[i-1:])
+			l := simd.LoadUint32s(p1[i:])
+			d.Max(u).Max(l).Sub(one).Store(cur[i:])
+		}
+
+		zero.Store(cur[hi+1:])
+		if lo > 1 {
+			cur[lo-1] = 0
+		}
+		cur[0] = 0
+		if t <= lA && t <= kmax {
+			cur[0] = lcsLenBase - uint32(t)
+		}
+		if t <= lB && -t >= kmin {
+			cur[t] = lcsLenBase - uint32(t)
+		}
+
+		p2, p1, cur = p1, cur, p2
+	}
+	return p1[lB]
+}

pkg/obialign/lcsband_test.go

@@ -0,0 +1,360 @@
+package obialign
+
+import (
+	"math/rand"
+	"testing"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+// bandDP is the oracle: a plain DP over the whole matrix, written
+// independently of the production code, in which only the cells with a
+// diagonal offset j-i in [kmin, kmax] exist (kmin=-lB, kmax=lA when
+// unbounded). It returns the optimal LCS score and the shortest alignment
+// length achieving it among the alignments staying in the band.
+func bandDP(a, b []byte, kmin, kmax int) (int, int) {
+	lA, lB := len(a), len(b)
+	type cell struct {
+		s, l int
+		ok   bool
+	}
+	better := func(x, y cell) bool {
+		return !y.ok || (x.ok && (x.s > y.s || (x.s == y.s && x.l < y.l)))
+	}
+	m := make([][]cell, lB+1)
+	for i := range m {
+		m[i] = make([]cell, lA+1)
+	}
+	for i := 0; i <= lB; i++ {
+		for j := 0; j <= lA; j++ {
+			if k := j - i; k < kmin || k > kmax {
+				continue
+			}
+			if i == 0 || j == 0 {
+				m[i][j] = cell{0, i + j, true}
+				continue
+			}
+			var best cell
+			if d := m[i-1][j-1]; d.ok {
+				best = cell{d.s, d.l + 1, true}
+				if obiseq.SameIUPACNuc(a[j-1], b[i-1]) {
+					best.s++
+				}
+			}
+			if u := m[i-1][j]; u.ok && better(cell{u.s, u.l + 1, true}, best) {
+				best = cell{u.s, u.l + 1, true}
+			}
+			if l := m[i][j-1]; l.ok && better(cell{l.s, l.l + 1, true}, best) {
+				best = cell{l.s, l.l + 1, true}
+			}
+			m[i][j] = best
+		}
+	}
+	return m[lB][lA].s, m[lB][lA].l
+}
+
+// fullDP is the unbanded optimum.
+func fullDP(a, b []byte) (int, int) { return bandDP(a, b, -len(b), len(a)) }
+
+// expected is the contract of BandedLCS: the unbounded optimum when it has at
+// most maxError errors, (-1,-1) otherwise.
+func expected(a, b []byte, maxError int) (int, int) {
+	s, l := fullDP(a, b)
+	if maxError >= 0 && l-s > maxError {
+		return -1, -1
+	}
+	return s, l
+}
+
+func randomFrom(rng *rand.Rand, alphabet string, n int) []byte {
+	s := make([]byte, n)
+	for i := range s {
+		s[i] = alphabet[rng.Intn(len(alphabet))]
+	}
+	return s
+}
+
+func mutateFrom(rng *rand.Rand, alphabet string, s []byte, rate float64) []byte {
+	out := make([]byte, 0, len(s)+8)
+	for _, c := range s {
+		if rng.Float64() >= rate {
+			out = append(out, c)
+			continue
+		}
+		switch rng.Intn(3) {
+		case 0:
+			out = append(out, alphabet[rng.Intn(len(alphabet))])
+		case 1:
+		case 2:
+			out = append(out, c, alphabet[rng.Intn(len(alphabet))])
+		}
+	}
+	return out
+}
+
+var testAlphabets = []struct {
+	name, letters string
+	regular       bool // only IUPAC letters of non-zero mask
+}{
+	{"acgt", "ACGT", true},
+	{"low-complexity", "AC", true},
+	{"mixed-case", "ACGTacgt", true},
+	{"iupac", "ACGTRYSWKMBDHVNUacgtrywsn", true},
+	{"exotic", "ACGT-.EXz*", false},
+}
+
+// TestBandedLCSDifferential checks the scalar reference against the oracle,
+// and the dispatcher (SIMD kernel when built with GOEXPERIMENT=simd) against
+// the scalar reference, for random pairs of unequal lengths and many bands.
+func TestBandedLCSDifferential(t *testing.T) {
+	rng := rand.New(rand.NewSource(7))
+	ws := new(bandedLCSWorkspace)
+	for _, alpha := range testAlphabets {
+		for it := 0; it < 3000; it++ {
+			lA := 1 + rng.Intn(90)
+			a := randomFrom(rng, alpha.letters, lA)
+			var b []byte
+			if rng.Intn(3) > 0 {
+				b = mutateFrom(rng, alpha.letters, a, 0.02+rng.Float64()*0.5)
+			} else {
+				b = randomFrom(rng, alpha.letters, 1+rng.Intn(90))
+			}
+			if len(b) == 0 {
+				continue
+			}
+			if len(a) < len(b) {
+				a, b = b, a
+			}
+			delta := len(a) - len(b)
+			for _, maxError := range []int{-1, 0, 1, 2, 3, 5, 8, 13, 21, 40, 200, delta, delta + 1, delta + 2, delta + 3} {
+				ws, wl := expected(a, b, maxError)
+				ss, sl := bandedLCS(a, b, maxError, new(bandedLCSWorkspace), false)
+				if ss != ws || sl != wl {
+					t.Fatalf("%s scalar maxError=%d a=%q b=%q: got (%d,%d), oracle (%d,%d)",
+						alpha.name, maxError, a, b, ss, sl, ws, wl)
+				}
+				ds, dl := bandedLCS(a, b, maxError, new(bandedLCSWorkspace), haveSIMD)
+				if ds != ss || dl != sl {
+					t.Fatalf("%s dispatch(simd=%v) maxError=%d a=%q b=%q: got (%d,%d), scalar (%d,%d)",
+						alpha.name, haveSIMD, maxError, a, b, ds, dl, ss, sl)
+				}
+			}
+		}
+	}
+	_ = ws
+}
+
+// TestBandedLCSLongPairs covers lengths well above the vector width, with
+// reused workspaces (stale buffers must never leak between calls).
+func TestBandedLCSLongPairs(t *testing.T) {
+	rng := rand.New(rand.NewSource(11))
+	ws := new(bandedLCSWorkspace)
+	for it := 0; it < 60; it++ {
+		a := randomFrom(rng, "ACGT", 150+rng.Intn(500))
+		b := mutateFrom(rng, "ACGT", a, 0.01+rng.Float64()*0.2)
+		if len(a) < len(b) {
+			a, b = b, a
+		}
+		for _, maxError := range []int{-1, 4, 20, 60, 150} {
+			wantS, wantL := expected(a, b, maxError)
+			gotS, gotL := bandedLCS(a, b, maxError, ws, haveSIMD)
+			if gotS != wantS || gotL != wantL {
+				t.Fatalf("maxError=%d lens=%d/%d: got (%d,%d), oracle (%d,%d)",
+					maxError, len(a), len(b), gotS, gotL, wantS, wantL)
+			}
+		}
+	}
+}
+
+// TestBandedLCSMatchesLegacy compares with the historical implementation
+// (FastLCSEGFScoreByte, not end-gap-free) wherever that one is exact: no
+// limit, or a result within the limit.
+func TestBandedLCSMatchesLegacy(t *testing.T) {
+	rng := rand.New(rand.NewSource(3))
+	for it := 0; it < 4000; it++ {
+		a := randomFrom(rng, "ACGTRYN", 1+rng.Intn(120))
+		b := mutateFrom(rng, "ACGTRYN", a, rng.Float64()*0.4)
+		if len(b) == 0 {
+			continue
+		}
+		for _, maxError := range []int{-1, 2, 7, 25} {
+			ls, ll, err := BandedLCS(a, b, maxError)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if ls < 0 {
+				continue
+			}
+			os, ol, _ := FastLCSEGFScoreByte(a, b, maxError, false, nil)
+			if os != ls || ol != ll {
+				t.Fatalf("maxError=%d a=%q b=%q: BandedLCS (%d,%d), legacy (%d,%d)",
+					maxError, a, b, ls, ll, os, ol)
+			}
+		}
+	}
+}
+
+func TestBandedLCSEdgeCases(t *testing.T) {
+	cases := []struct {
+		a, b     string
+		maxError int
+		s, l     int
+	}{
+		{"", "", -1, 0, 0},
+		{"ACGT", "", -1, 0, 4},
+		{"ACGT", "", 3, -1, -1},
+		{"ACGT", "", 4, 0, 4},
+		{"ACGT", "ACGT", 0, 4, 4},
+		{"ACGT", "ACGA", 0, -1, -1},
+		{"ACGT", "ACGA", 1, 3, 4},
+		{"ACGTT", "ACGT", 0, -1, -1}, // length difference above the limit
+		{"ACGTT", "ACGT", 1, 4, 5},
+	}
+	for _, c := range cases {
+		s, l, err := BandedLCS([]byte(c.a), []byte(c.b), c.maxError)
+		if err != nil || s != c.s || l != c.l {
+			t.Errorf("%q/%q maxError=%d: got (%d,%d,%v), want (%d,%d)", c.a, c.b, c.maxError, s, l, err, c.s, c.l)
+		}
+		// the arguments are symmetric
+		s, l, _ = BandedLCS([]byte(c.b), []byte(c.a), c.maxError)
+		if s != c.s || l != c.l {
+			t.Errorf("swapped %q/%q maxError=%d: got (%d,%d), want (%d,%d)", c.b, c.a, c.maxError, s, l, c.s, c.l)
+		}
+	}
+}
+
+func TestBandedLCSTooLong(t *testing.T) {
+	long := make([]byte, MaxBandedLCSLength+1)
+	for i := range long {
+		long[i] = 'A'
+	}
+	if _, _, err := BandedLCS(long, []byte("ACGT"), -1); err != ErrSequenceTooLong {
+		t.Errorf("A too long: err = %v", err)
+	}
+	if _, _, err := BandedLCS([]byte("ACGT"), long, 5); err != ErrSequenceTooLong {
+		t.Errorf("B too long: err = %v", err)
+	}
+	// the maximum length itself is accepted
+	ok := long[:MaxBandedLCSLength]
+	if s, l, err := BandedLCS(ok, ok, 0); err != nil || s != len(ok) || l != len(ok) {
+		t.Errorf("max length: got (%d,%d,%v)", s, l, err)
+	}
+}
+
+// TestBandedLCSSecondPass checks the exactness over many bands, also on
+// low-complexity sequences where the best LCS is not the closest alignment,
+// and reports how often the band had to be widened (never more than once).
+func TestBandedLCSSecondPass(t *testing.T) {
+	rng := rand.New(rand.NewSource(5))
+	for _, alpha := range testAlphabets[:4] {
+		for _, rate := range []float64{0.02, 0.1, 0.3} {
+			ws := new(bandedLCSWorkspace)
+			var total int
+			for it := 0; it < 1500; it++ {
+				a := randomFrom(rng, alpha.letters, 20+rng.Intn(120))
+				b := mutateFrom(rng, alpha.letters, a, rate)
+				if len(b) == 0 {
+					continue
+				}
+				if len(a) < len(b) {
+					a, b = b, a
+				}
+				for _, maxError := range []int{1, 3, 6, 10, 20, 40} {
+					want, wantL := expected(a, b, maxError)
+					got, gotL := bandedLCS(a, b, maxError, ws, haveSIMD)
+					if got != want || gotL != wantL {
+						t.Fatalf("%s maxError=%d a=%q b=%q: got (%d,%d), want (%d,%d)",
+							alpha.name, maxError, a, b, got, gotL, want, wantL)
+					}
+					total++
+				}
+			}
+			t.Logf("%-15s mutation %.2f: %d bounded calls, %d widened (%.1f%%)",
+				alpha.name, rate, total, ws.widened, 100*float64(ws.widened)/float64(total))
+		}
+	}
+}
+
+// TestFastLCSScoreWrapper checks the BioSequence entry point.
+func TestFastLCSScoreWrapper(t *testing.T) {
+	sa := obiseq.NewBioSequence("a", []byte("ACGTACGTAC"), "")
+	sb := obiseq.NewBioSequence("b", []byte("ACGTTCGTAC"), "")
+	if s, l := FastLCSScore(sa, sb, -1, nil); s != 9 || l != 10 {
+		t.Errorf("got (%d,%d), want (9,10)", s, l)
+	}
+	if s, l := FastLCSScore(sa, sb, 0, nil); s != -1 || l != -1 {
+		t.Errorf("got (%d,%d), want (-1,-1)", s, l)
+	}
+}
+
+// TestBandedLCSAtLeast checks the exactness of the bound expressed on the
+// score: the optimum when it reaches minLCS, (-1,-1) otherwise.
+func TestBandedLCSAtLeast(t *testing.T) {
+	rng := rand.New(rand.NewSource(13))
+	var buf BandedLCSBuffer
+	for _, alpha := range testAlphabets {
+		for it := 0; it < 1500; it++ {
+			a := randomFrom(rng, alpha.letters, 1+rng.Intn(100))
+			b := mutateFrom(rng, alpha.letters, a, rng.Float64()*0.5)
+			if len(b) == 0 {
+				continue
+			}
+			os, ol := fullDP(a, b)
+			for _, minLCS := range []int{-3, 0, 1, os - 1, os, os + 1, len(b) / 2, len(b), len(b) + 1} {
+				ws, wl := os, ol
+				if os < minLCS {
+					ws, wl = -1, -1
+				}
+				for _, got := range [][2]int{func() [2]int { s, l, _ := buf.AtLeast(a, b, minLCS); return [2]int{s, l} }(),
+					func() [2]int { s, l, _ := BandedLCSAtLeast(b, a, minLCS); return [2]int{s, l} }()} {
+					if got[0] != ws || got[1] != wl {
+						t.Fatalf("%s minLCS=%d a=%q b=%q: got (%d,%d), want (%d,%d)",
+							alpha.name, minLCS, a, b, got[0], got[1], ws, wl)
+					}
+				}
+			}
+		}
+	}
+	if _, _, err := buf.AtLeast(make([]byte, MaxBandedLCSLength+1), []byte("a"), 0); err != ErrSequenceTooLong {
+		t.Errorf("too long: err = %v", err)
+	}
+}
+
+// TestBandedLCSScoreAtLeast checks the score-only entry point, with sequences
+// long enough for the early stop of the kernel to be exercised.
+func TestBandedLCSScoreAtLeast(t *testing.T) {
+	rng := rand.New(rand.NewSource(17))
+	var buf BandedLCSBuffer
+	check := func(name string, a, b []byte) {
+		os, _ := fullDP(a, b)
+		for _, minLCS := range []int{-1, 0, os - 3, os - 1, os, os + 1, os + 5, len(b) - 1, len(b), (len(b) * 9) / 10, len(b) + 1} {
+			want := os
+			if os < minLCS {
+				want = -1
+			}
+			got, err := buf.ScoreAtLeast(a, b, minLCS)
+			if err != nil || got != want {
+				t.Fatalf("%s minLCS=%d a=%q b=%q: got %d (%v), want %d", name, minLCS, a, b, got, err, want)
+			}
+		}
+	}
+	for _, alpha := range testAlphabets {
+		for it := 0; it < 800; it++ {
+			a := randomFrom(rng, alpha.letters, 1+rng.Intn(60))
+			b := mutateFrom(rng, alpha.letters, a, rng.Float64()*0.6)
+			if len(b) > 0 {
+				check(alpha.name, a, b)
+			}
+		}
+	}
+	for it := 0; it < 150; it++ {
+		a := randomFrom(rng, "ACGT", 100+rng.Intn(400))
+		b := mutateFrom(rng, "ACGT", a, rng.Float64()*0.3)
+		if it%3 == 0 {
+			b = randomFrom(rng, "ACGT", 50+rng.Intn(300))
+		}
+		if len(b) > 0 {
+			check("long", a, b)
+		}
+	}
+}

pkg/obialign/lcsmin.go

@@ -0,0 +1,103 @@
+package obialign
+
+// BandedLCSBuffer is a working memory that can be reused from one alignment to
+// the next to avoid any allocation. It must not be shared between goroutines.
+type BandedLCSBuffer struct {
+	ws bandedLCSWorkspace
+}
+
+// AtLeast is BandedLCSAtLeast on the buffer.
+func (b *BandedLCSBuffer) AtLeast(seqA, seqB []byte, minLCS int) (lcs, alilen int, err error) {
+	return atLeast(seqA, seqB, minLCS, &b.ws)
+}
+
+func atLeast(seqA, seqB []byte, minLCS int, ws *bandedLCSWorkspace) (lcs, alilen int, err error) {
+	if len(seqA) > MaxBandedLCSLength || len(seqB) > MaxBandedLCSLength {
+		return -1, -1, ErrSequenceTooLong
+	}
+	if len(seqA) < len(seqB) {
+		seqA, seqB = seqB, seqA
+	}
+	lcs, alilen = bandedLCSAtLeast(seqA, seqB, minLCS, ws, haveSIMD)
+	return lcs, alilen, nil
+}
+
+// BandedLCSAtLeast computes the LCS score of two nucleotide sequences and the
+// length of the shortest alignment achieving it, as BandedLCS does, but with
+// the bound expressed on the score: the result is the exact optimum when its
+// score is at least minLCS, and (-1, -1) otherwise.
+//
+// The DP only visits the diagonals an alignment of score minLCS can use (one
+// whose path makes at most lB-minLCS gaps on one side and lA-minLCS on the
+// other), so the cost decreases as minLCS grows. This is exact: every alignment
+// of score >= minLCS lies in that band, hence so does the optimum when it
+// reaches minLCS, and when the best path of the band is below minLCS so is
+// the global optimum.
+func BandedLCSAtLeast(seqA, seqB []byte, minLCS int) (lcs, alilen int, err error) {
+	ws := bandedLCSPool.Get().(*bandedLCSWorkspace)
+	lcs, alilen, err = atLeast(seqA, seqB, minLCS, ws)
+	bandedLCSPool.Put(ws)
+	return lcs, alilen, err
+}
+
+// bandedLCSAtLeast expects len(a) >= len(b), both within MaxBandedLCSLength.
+func bandedLCSAtLeast(a, b []byte, minLCS int, ws *bandedLCSWorkspace, useSIMD bool) (int, int) {
+	lA, lB := len(a), len(b)
+	minLCS = max(minLCS, 0)
+	if minLCS > lB {
+		return -1, -1
+	}
+	if lB == 0 {
+		return 0, lA
+	}
+	simdOK := useSIMD && allRegularIUPAC(a) && allRegularIUPAC(b)
+	kmin, kmax := -(lB - minLCS), lA-minLCS
+
+	if !simdOK {
+		lcs, alilen := ws.align(a, b, kmin, kmax, false)
+		if lcs < minLCS {
+			return -1, -1
+		}
+		return lcs, alilen
+	}
+
+	// The score alone is much cheaper than the score with the length of the
+	// alignment: the latter is only computed when the pair reaches minLCS. With
+	// no minimum there is nothing to reject, and the pass would be wasted.
+	if minLCS > 0 && lcsScoreSIMD(a, b, kmin, kmax, minLCS, ws) < minLCS {
+		return -1, -1
+	}
+	encodeBases(ws, a, b)
+	return ws.align(a, b, kmin, kmax, true)
+}
+
+// ScoreAtLeast computes the LCS score of two nucleotide sequences when it is
+// at least minLCS, and returns -1 otherwise. It is much cheaper than AtLeast,
+// which also computes the length of the shortest alignment.
+func (b *BandedLCSBuffer) ScoreAtLeast(seqA, seqB []byte, minLCS int) (int, error) {
+	if len(seqA) > MaxBandedLCSLength || len(seqB) > MaxBandedLCSLength {
+		return -1, ErrSequenceTooLong
+	}
+	if len(seqA) < len(seqB) {
+		seqA, seqB = seqB, seqA
+	}
+	lA, lB := len(seqA), len(seqB)
+	minLCS = max(minLCS, 0)
+	if minLCS > lB {
+		return -1, nil
+	}
+	if lB == 0 {
+		return 0, nil
+	}
+	kmin, kmax := -(lB - minLCS), lA-minLCS
+	var lcs int
+	if haveSIMD && allRegularIUPAC(seqA) && allRegularIUPAC(seqB) {
+		lcs = lcsScoreSIMD(seqA, seqB, kmin, kmax, minLCS, &b.ws)
+	} else {
+		lcs, _ = b.ws.align(seqA, seqB, kmin, kmax, false)
+	}
+	if lcs < minLCS {
+		return -1, nil
+	}
+	return lcs, nil
+}

pkg/obialign/lcsscore_simd.go

@@ -0,0 +1,156 @@
+//go:build goexperiment.simd
+
+package obialign
+
+import "simd"
+
+func simdLanes16() int {
+	var probe simd.Uint16s
+	return probe.Len()
+}
+
+func growUint16(s []uint16, n int) []uint16 {
+	if cap(s) < n {
+		return make([]uint16, n)
+	}
+	return s[:n]
+}
+
+func encodeIUPAC16(dst []uint16, s []byte, rev bool) {
+	n := len(s)
+	clear(dst)
+	if rev {
+		for x, c := range s {
+			dst[n-1-x] = uint16(iupacMask[c])
+		}
+		return
+	}
+	for x, c := range s {
+		dst[x] = uint16(iupacMask[c])
+	}
+}
+
+// pruneEvery is the number of antidiagonals between two tests of the upper
+// bound that stops a hopeless alignment.
+const pruneEvery = 8
+
+// lcsScoreSIMD runs the same banded DP as bandedLCSSIMD on the score only,
+// with 16-bit lanes (twice the cells per instruction):
+//
+//	cell(i,j) = max(diag + match, up, left)
+//
+// A cell outside the band is 0, which never beats a real cell. The score of a
+// path ending at the cell (i, j) of an antidiagonal cannot exceed the score of
+// the cell plus min(lB-i, lA-j), the characters left to pair. Every path
+// crosses each antidiagonal inside the window, so when the best score of the
+// window plus the largest of these remainders is under minLCS the final score
+// is under minLCS: -1 is returned without computing the rest of the matrix.
+//
+// Requires lA >= lB >= 1, kmin <= 0 <= kmax, and bases with non-zero masks.
+// Returns the best score of the band, or -1 if it is proven to be under
+// minLCS.
+func lcsScoreSIMD(a, b []byte, kmin, kmax, minLCS int, ws *bandedLCSWorkspace) int {
+	lA, lB := len(a), len(b)
+	vl := simdLanes16()
+
+	ws.aRev16 = growUint16(ws.aRev16, lA+vl)
+	ws.bCode16 = growUint16(ws.bCode16, lB+vl)
+	encodeIUPAC16(ws.aRev16, a, true)
+	encodeIUPAC16(ws.bCode16, b, false)
+	aRev, bCode := ws.aRev16, ws.bCode16
+
+	size := lB + 2*vl + 2
+	for r := range ws.rows16 {
+		ws.rows16[r] = growUint16(ws.rows16[r], size)
+		clear(ws.rows16[r])
+	}
+	p2, p1, cur := ws.rows16[0], ws.rows16[1], ws.rows16[2]
+
+	// the first row and the first column hold a score of 0: nothing to set
+
+	zero := simd.BroadcastUint16s(0)
+	one := simd.BroadcastUint16s(1)
+
+	for t := 2; t <= lA+lB; t++ {
+		lo := max(1, t-lA, (t-kmax+1)>>1)
+		hi := min(lB, t-1, (t-kmin)>>1)
+
+		for i := lo; i <= hi; i += vl {
+			ca := simd.LoadUint16s(aRev[lA-t+i:])
+			cb := simd.LoadUint16s(bCode[i-1:])
+			inc := one.Masked(ca.And(cb).NotEqual(zero))
+			d := simd.LoadUint16s(p2[i-1:]).Add(inc)
+			u := simd.LoadUint16s(p1[i-1:])
+			l := simd.LoadUint16s(p1[i:])
+			d.Max(u).Max(l).Store(cur[i:])
+		}
+
+		zero.Store(cur[hi+1:])
+		if lo > 1 {
+			cur[lo-1] = 0
+		}
+
+		// A diagonal move skips an antidiagonal, but a path always visits
+		// one of two consecutive antidiagonals.
+		if t%pruneEvery == 0 && minLCS > 0 &&
+			scoreBound(p1, t-1, lA, lB, kmin, kmax) < minLCS &&
+			scoreBound(cur, t, lA, lB, kmin, kmax) < minLCS {
+			return -1
+		}
+
+		p2, p1, cur = p1, cur, p2
+	}
+	return int(p1[lB])
+}
+
+// iota16 is 0, 1, 2... to build the row index of the lanes.
+var iota16 = func() (t [64]uint16) {
+	for i := range t {
+		t[i] = uint16(i)
+	}
+	return t
+}()
+
+// scoreBound is an upper bound of the final score of the paths going through
+// the antidiagonal t, whose scores are in row. A path through the cell (i, j)
+// scores at most the score of the cell plus min(lB-i, lA-j), the characters
+// left to pair, so the bound is the maximum of this sum on the cells of the
+// window. The sum is computed by vectors on the full chunks of the window.
+func scoreBound(row []uint16, t, lA, lB, kmin, kmax int) int {
+	lo := max(1, t-lA, (t-kmax+1)>>1)
+	hi := min(lB, t-1, (t-kmin)>>1)
+
+	bound := 0
+	i := lo
+	if hi-lo >= simdLanes16() {
+		vl := simdLanes16()
+		idx := simd.LoadUint16s(iota16[:]).Add(simd.BroadcastUint16s(uint16(lo)))
+		step := simd.BroadcastUint16s(uint16(vl))
+		remB := simd.BroadcastUint16s(uint16(lB))
+		// lA-t+i is not negative on the window; the 16-bit arithmetic is modular
+		offA := simd.BroadcastUint16s(uint16(lA - t))
+		acc := simd.BroadcastUint16s(0)
+		for ; i+vl-1 <= hi; i += vl {
+			left := remB.Sub(idx).Min(idx.Add(offA))
+			acc = acc.Max(simd.LoadUint16s(row[i:]).Add(left))
+			idx = idx.Add(step)
+		}
+		var lanes [64]uint16
+		acc.Store(lanes[:])
+		for _, v := range lanes[:vl] {
+			bound = max(bound, int(v))
+		}
+	}
+	for ; i <= hi; i++ {
+		bound = max(bound, int(row[i])+min(lB-i, lA-t+i))
+	}
+
+	// the paths running along the first row or the first column have a score of 0
+	if t <= lA && t <= kmax {
+		bound = max(bound, min(lB, lA-t))
+	}
+	if t <= lB && -t >= kmin {
+		bound = max(bound, min(lB-t, lA))
+	}
+	return bound
+}

pkg/obialign/locatepattern.go

@@ -28,10 +28,18 @@ func buffIndex(i, j, width int) int {
 //
 // The function returns the start and end positions of the best
 // match, as well as the number of errors in the best match.
+//
+// When the sequence is too short relative to the pattern for the
+// backtracking to reconstruct a valid alignment (e.g. the pattern
+// is longer than the sequence, or the match sits too close to a
+// sequence boundary), no reliable position can be computed. In that
+// case the function returns the sentinel (-1, -1, -1) instead of a
+// guessed, potentially wrong, position: callers must treat this as
+// "no match" rather than use the returned coordinates.
 func LocatePattern(id string, pattern, sequence []byte) (int, int, int) {
 
-	if len(pattern) >= len(sequence) {
-		log.Panicf("Sequence %s:Pattern %s must be shorter than sequence %s", id, pattern, sequence)
+	if len(sequence) == 0 {
+		log.Panicf("Sequence %s:Pattern %s must not be empty", id, pattern)
 	}
 
 	// Pattern spreads over the columns
@@ -158,5 +166,18 @@ func LocatePattern(id string, pattern, sequence []byte) (int, int, int) {
 	// obilog.Warnf("from : %d to: %d error: %d match: %v",
 	// 	i, end+1, -buffer[buffIndex(len(sequence)-1, len(pattern)-1, width)],
 	// 	string(sequence[i:(end+1)]))
+
+	if i < 0 || end == -1 {
+		// i < 0: the backtracking ran off the start of the sequence
+		// without fully consuming the pattern.
+		// end == -1: the backtracking loop never ran at all (e.g. a
+		// single-base pattern, jmax == 0), so no alignment boundary
+		// was ever established.
+		// Either way, no valid alignment exists for this (pattern,
+		// sequence) pair: signal it explicitly instead of returning
+		// an out-of-bounds or uncomputed position.
+		return -1, -1, -1
+	}
+
 	return i, end + 1, -buffer[buffIndex(len(sequence)-1, len(pattern)-1, width)]
 }

pkg/obialign/locatepattern_test.go

@@ -0,0 +1,123 @@
+package obialign
+
+import (
+	"math/rand"
+	"testing"
+)
+
+func TestLocatePatternNormal(t *testing.T) {
+	// Pattern fully and exactly present in the middle of a longer sequence.
+	start, end, nerr := LocatePattern("id", []byte("ACGT"), []byte("TTTTACGTTTTT"))
+	if start != 4 || end != 8 || nerr != 0 {
+		t.Errorf("got start=%d end=%d nerr=%d, want start=4 end=8 nerr=0", start, end, nerr)
+	}
+}
+
+func TestLocatePatternOneMismatch(t *testing.T) {
+	start, end, nerr := LocatePattern("id", []byte("ACGT"), []byte("TTTTACTTTTTT"))
+	if nerr != 1 {
+		t.Errorf("got nerr=%d, want 1 (start=%d end=%d)", nerr, start, end)
+	}
+}
+
+// The real-world case that used to panic: pattern longer than the sequence
+// fragment extracted for indel relocation.
+func TestLocatePatternPatternLongerThanSequence(t *testing.T) {
+	start, end, nerr := LocatePattern("id",
+		[]byte("GGGCAATCCTGAGCCAAATC"),
+		[]byte("tcctgagccaaatcacgtt"))
+
+	if start != -1 || end != -1 || nerr != -1 {
+		t.Errorf("got start=%d end=%d nerr=%d, want the (-1,-1,-1) sentinel", start, end, nerr)
+	}
+}
+
+func TestLocatePatternSequenceLengthOne(t *testing.T) {
+	start, end, nerr := LocatePattern("id", []byte("AB"), []byte("A"))
+
+	if start < 0 || end < 0 || nerr < 0 {
+		t.Fatalf("got start=%d end=%d nerr=%d, expected a valid (non-sentinel) result", start, end, nerr)
+	}
+	if start != 0 || end != 1 || nerr != 1 {
+		t.Errorf("got start=%d end=%d nerr=%d, want start=0 end=1 nerr=1", start, end, nerr)
+	}
+}
+
+// A pattern much longer than the sequence can still yield a mathematically
+// valid (in-bounds) alignment: the extra pattern length is absorbed as gaps,
+// driving the error count high enough that the caller's maxerr threshold
+// rejects it. The function itself must still return consistent bounds.
+func TestLocatePatternPatternMuchLongerThanSequence(t *testing.T) {
+	start, end, nerr := LocatePattern("id", []byte("ACGTACGTACGTACGTACGT"), []byte("ACG"))
+
+	isSentinel := start == -1 && end == -1 && nerr == -1
+	isValid := start >= 0 && end > start && end <= 3 && nerr >= 0
+
+	if !isSentinel && !isValid {
+		t.Errorf("got start=%d end=%d nerr=%d, want either the sentinel or consistent in-bounds values", start, end, nerr)
+	}
+}
+
+func TestLocatePatternNeverReturnsOutOfBounds(t *testing.T) {
+	patterns := []string{"A", "AC", "ACG", "ACGT", "ACGTA", "ACGTAC", "ACGTACG", "ACGTACGT"}
+	sequences := []string{"A", "AC", "ACG", "ACGT", "ACGTA", "ACGTAC", "ACGTACG", "ACGTACGT"}
+
+	for _, p := range patterns {
+		for _, s := range sequences {
+			start, end, nerr := LocatePattern("id", []byte(p), []byte(s))
+
+			if start == -1 && end == -1 && nerr == -1 {
+				// Explicit "no reliable match" sentinel: always acceptable.
+				continue
+			}
+
+			if start < 0 || end < 0 || start >= end || end > len(s) || nerr < 0 {
+				t.Errorf("pattern=%q sequence=%q -> start=%d end=%d nerr=%d is out of bounds / inconsistent",
+					p, s, start, end, nerr)
+			}
+		}
+	}
+}
+
+// Randomized property test over a wide range of pattern/sequence length
+// combinations, including pattern >= sequence, to make sure the function
+// never panics and never returns anything but the sentinel or fully
+// consistent, in-bounds coordinates.
+func TestLocatePatternRandomizedNeverInvalid(t *testing.T) {
+	const bases = "ACGT"
+	rng := rand.New(rand.NewSource(42))
+
+	randSeq := func(n int) []byte {
+		b := make([]byte, n)
+		for i := range b {
+			b[i] = bases[rng.Intn(len(bases))]
+		}
+		return b
+	}
+
+	for trial := 0; trial < 5000; trial++ {
+		patLen := 1 + rng.Intn(15)
+		seqLen := 1 + rng.Intn(15)
+
+		pattern := randSeq(patLen)
+		sequence := randSeq(seqLen)
+
+		func() {
+			defer func() {
+				if r := recover(); r != nil {
+					t.Fatalf("panic for pattern=%q sequence=%q: %v", pattern, sequence, r)
+				}
+			}()
+
+			start, end, nerr := LocatePattern("id", pattern, sequence)
+
+			isSentinel := start == -1 && end == -1 && nerr == -1
+			isValid := start >= 0 && end > start && end <= len(sequence) && nerr >= 0
+
+			if !isSentinel && !isValid {
+				t.Errorf("pattern=%q sequence=%q -> start=%d end=%d nerr=%d is neither the sentinel nor consistent",
+					pattern, sequence, start, end, nerr)
+			}
+		}()
+	}
+}

pkg/obiapat/pattern.go

@@ -373,6 +373,7 @@ func (pattern ApatPattern) BestMatch(sequence ApatSequence, begin, length int) (
 
 	cpattern := (*[1 << 30]byte)(unsafe.Pointer(pattern.pointer.pointer.cpat))
 	frg := sequence.pointer.reference.Sequence()[start:end]
+	fragStart := start
 
 	log.Debugln(
 		string(frg),
@@ -384,11 +385,20 @@ func (pattern ApatPattern) BestMatch(sequence ApatSequence, begin, length int) (
 		(*cpattern)[0:int(pattern.pointer.pointer.patlen)],
 		frg)
 
-	// olderr := m[2]
+	if from < 0 {
+		// obialign.LocatePattern could not reconstruct a reliable
+		// alignment (e.g. the fragment is too short relative to the
+		// pattern). Reporting a guessed position would risk placing
+		// the primer boundary incorrectly, so treat it as no match
+		// at all rather than falling back to an unrefined position.
+		matched = false
+		log.Debugln("No reliable indel relocation, discarding match", sequence.pointer.reference.Id())
+		return
+	}
 
 	nerr = score
-	start = start + from
-	end = start + to
+	start = fragStart + from
+	end = fragStart + to
 	log.Debugf("BestMatch on %s : score=%d [%d..%d]", sequence.pointer.reference.Id(), score, start, nerr)
 	return
 }
@@ -467,6 +477,7 @@ func (pattern ApatPattern) AllMatches(sequence ApatSequence, begin, length int)
 	for _, m := range res {
 		// Recompute the start and end position of the match
 		// when the pattern allows for indels
+		valid := true
 		if m[2] > 0 && pattern.pointer.pointer.hasIndel {
 			// obilog.Warnf("Locating indel on sequence %s[%s]", sequence.pointer.reference.Id(), pattern.String())
 			start := m[0] - m[2]*2
@@ -485,16 +496,26 @@ func (pattern ApatPattern) AllMatches(sequence ApatSequence, begin, length int)
 				(*cpattern)[0:int(pattern.pointer.pointer.patlen)],
 				frg)
 
-			// olderr := m[2]
-			m[2] = score
-			m[0] = start + pb
-			m[1] = start + pe
+			if pb < 0 {
+				// obialign.LocatePattern could not reconstruct a
+				// reliable alignment (e.g. the match sits too close
+				// to a sequence end for the fragment to be usable).
+				// Reporting a guessed position risks placing the
+				// primer boundary incorrectly, so drop the match
+				// entirely instead of keeping an unrefined guess.
+				valid = false
+			} else {
+				// olderr := m[2]
+				m[2] = score
+				m[0] = start + pb
+				m[1] = start + pe
 
-			// obilog.Warnf("seq[%d@%d:%d] %d: %s %d - %s:%s:%s", i, m[0], m[1], olderr, sequence.pointer.reference.Id(), score,
-			// 	frg, (*cpattern)[0:int(pattern.pointer.pointer.patlen)], sequence.pointer.reference.Sequence()[m[0]:m[1]])
+				// obilog.Warnf("seq[%d@%d:%d] %d: %s %d - %s:%s:%s", i, m[0], m[1], olderr, sequence.pointer.reference.Id(), score,
+				// 	frg, (*cpattern)[0:int(pattern.pointer.pointer.patlen)], sequence.pointer.reference.Sequence()[m[0]:m[1]])
+			}
 		}
 
-		if int(pattern.pointer.pointer.maxerr) >= m[2] {
+		if valid && int(pattern.pointer.pointer.maxerr) >= m[2] {
 			res[j] = m
 			j++
 		}

pkg/obidefault/progressbar.go

@@ -1,5 +1,7 @@
 package obidefault
 
+import "os"
+
 var __no_progress_bar__ = false
 
 func ProgressBar() bool {
@@ -17,3 +19,22 @@ func SetNoProgressBar(b bool) {
 func NoProgressBarPtr() *bool {
 	return &__no_progress_bar__
 }
+
+// ProgressBarVisible tells whether a progress bar can be displayed: it is not
+// disabled by --no-progressbar, the standard error output is a terminal and
+// the standard output is not piped to another command.
+func ProgressBarVisible() bool {
+	if !ProgressBar() {
+		return false
+	}
+
+	if e, err := os.Stderr.Stat(); err != nil || (e.Mode()&os.ModeCharDevice) != os.ModeCharDevice {
+		return false
+	}
+
+	if o, err := os.Stdout.Stat(); err == nil && (o.Mode()&os.ModeNamedPipe) == os.ModeNamedPipe {
+		return false
+	}
+
+	return true
+}

pkg/obiformats/batch_of_files_reader.go

@@ -48,6 +48,14 @@ func ReadSequencesBatchFromFiles(filenames []string,
 
 				log.Printf("Start reading of file : %s", filename)
 
+				// The batches of a file reach us in any order (parallel parsing
+				// of the headers). They are numbered here in arrival order, so
+				// the order of the file must be restored first, unless the
+				// order of the data is not wanted (several concurrent readers).
+				if concurrent_readers == 1 {
+					iter = iter.SortBatches()
+				}
+
 				for iter.Next() {
 					batch := iter.Get()
 					batchiter.Push(batch.Reorder(nextCounter()))

pkg/obiformats/batch_of_files_reader_test.go

@@ -0,0 +1,38 @@
+package obiformats
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// With a single reader, the sequences of several files must come out in the
+// order of the files, whatever the order in which the batches are parsed.
+func TestReadSequencesBatchFromFilesKeepsOrder(t *testing.T) {
+	dir := t.TempDir()
+	var names []string
+	for _, prefix := range []string{"a", "b"} {
+		var sb strings.Builder
+		for i := 0; i < 25000; i++ {
+			fmt.Fprintf(&sb, ">%s%07d {\"k\":%d}\n%s\n", prefix, i, i, strings.Repeat("acgt", 40))
+		}
+		name := filepath.Join(dir, prefix+".fasta")
+		if err := os.WriteFile(name, []byte(sb.String()), 0o600); err != nil {
+			t.Fatal(err)
+		}
+		names = append(names, name)
+	}
+
+	it := ReadSequencesBatchFromFiles(names, nil, 1)
+	_, db := it.Load()
+	if len(db) != 50000 {
+		t.Fatalf("expected 50000 sequences, got %d", len(db))
+	}
+	for i := 1; i < len(db); i++ {
+		if db[i-1].Id() >= db[i].Id() {
+			t.Fatalf("out of order at %d: %s then %s", i, db[i-1].Id(), db[i].Id())
+		}
+	}
+}

pkg/obiformats/fastqseq_read.go

@@ -131,7 +131,7 @@ func _storeSequenceQuality(bytes *bytes.Buffer, out *obiseq.BioSequence, quality
 	out.SetQualities(q)
 }
 
-func FastqChunkParser(quality_shift byte, with_quality bool, UtoT bool) func(string, io.Reader) (obiseq.BioSequenceSlice, error) {
+func FastqChunkParser(quality_shift byte, with_quality bool, UtoT bool, fileName string) func(string, io.Reader) (obiseq.BioSequenceSlice, error) {
 	parser := func(source string, input io.Reader) (obiseq.BioSequenceSlice, error) {
 
 		var identifier string
@@ -160,12 +160,12 @@ func FastqChunkParser(quality_shift byte, with_quality bool, UtoT bool) func(str
 					// Beginning of sequence
 					state = 1
 				} else {
-					log.Fatalf("%s : sequence entry is not starting with @", source)
+					log.Fatalf("file %s: sequence entry is not starting with @", fileName)
 				}
 			case 1: // Beginning of identifier (Mandatory)
 				if is_sep {
 					// No identifier -> ERROR
-					log.Fatalf("%s : sequence identifier is empty", source)
+					log.Fatalf("file %s: sequence identifier is empty", fileName)
 				} else {
 					// Beginning of identifier
 					state = 2
@@ -221,7 +221,7 @@ func FastqChunkParser(quality_shift byte, with_quality bool, UtoT bool) func(str
 					// End of sequence
 					rawseq := seqBytes.Bytes()
 					if len(rawseq) == 0 {
-						log.Fatalf("@%s[%s] : sequence is empty", identifier, source)
+						log.Fatalf("file %s: record @%s has an empty sequence line", fileName, identifier)
 					}
 					s := obiseq.NewBioSequence(identifier, rawseq, definition)
 					s.SetSource(source)
@@ -241,8 +241,8 @@ func FastqChunkParser(quality_shift byte, with_quality bool, UtoT bool) func(str
 						context = append(
 							append([]byte{previous}, C),
 							context...)
-						log.Fatalf("%s [%s]: sequence contains invalid character %c (%s)",
-							source, identifier, C, string(context))
+						log.Fatalf("file %s: record @%s contains invalid character %c (%s)",
+							fileName, identifier, C, string(context))
 					}
 				}
 			case 7:
@@ -251,7 +251,7 @@ func FastqChunkParser(quality_shift byte, with_quality bool, UtoT bool) func(str
 				} else if C == '+' {
 					state = 8
 				} else {
-					log.Fatalf("@%s[%s] : sequence data not followed by a line starting with + but a %c", identifier, source, C)
+					log.Fatalf("file %s: record @%s: sequence data not followed by a line starting with + but a %c", fileName, identifier, C)
 				}
 			case 8:
 				// State consuming the + internal header line
@@ -282,7 +282,7 @@ func FastqChunkParser(quality_shift byte, with_quality bool, UtoT bool) func(str
 				} else if C == '@' {
 					state = 1
 				} else {
-					log.Fatalf("%s[%s] : sequence record not followed by a line starting with @", identifier, source)
+					log.Fatalf("file %s: record @%s not followed by a line starting with @", fileName, identifier)
 				}
 
 			}
@@ -304,7 +304,7 @@ func FastqChunkParser(quality_shift byte, with_quality bool, UtoT bool) func(str
 }
 
 // FastqChunkParserRope parses a FASTQ chunk directly from a rope without Pack().
-func FastqChunkParserRope(source string, rope *PieceOfChunk, quality_shift byte, with_quality, UtoT bool) (obiseq.BioSequenceSlice, error) {
+func FastqChunkParserRope(source string, rope *PieceOfChunk, quality_shift byte, with_quality, UtoT bool, fileName string) (obiseq.BioSequenceSlice, error) {
 	scanner := newRopeScanner(rope)
 	sequences := obiseq.MakeBioSequenceSlice(100)[:0]
 
@@ -334,7 +334,7 @@ func FastqChunkParserRope(source string, rope *PieceOfChunk, quality_shift byte,
 		// Line 2: sequence
 		sline := scanner.ReadLine()
 		if sline == nil {
-			log.Fatalf("@%s[%s]: unexpected EOF after header", id, source)
+			log.Fatalf("file %s: record @%s is truncated (header line with no sequence line following) — the FASTQ file appears incomplete", fileName, id)
 		}
 		seqDest := make([]byte, len(sline))
 		w := 0
@@ -350,7 +350,7 @@ func FastqChunkParserRope(source string, rope *PieceOfChunk, quality_shift byte,
 		}
 		seqDest = seqDest[:w]
 		if len(seqDest) == 0 {
-			log.Fatalf("@%s[%s]: sequence is empty", id, source)
+			log.Fatalf("file %s: record @%s has an empty sequence line", fileName, id)
 		}
 
 		// Line 3: + (skip)
@@ -382,16 +382,17 @@ func _ParseFastqFile(
 	out obiiter.IBioSequence,
 	quality_shift byte,
 	with_quality, UtoT bool,
+	fileName string,
 ) {
 
-	parser := FastqChunkParser(quality_shift, with_quality, UtoT)
+	parser := FastqChunkParser(quality_shift, with_quality, UtoT, fileName)
 
 	for chunks := range input {
 		var sequences obiseq.BioSequenceSlice
 		var err error
 
 		if chunks.Rope != nil {
-			sequences, err = FastqChunkParserRope(chunks.Source, chunks.Rope, quality_shift, with_quality, UtoT)
+			sequences, err = FastqChunkParserRope(chunks.Source, chunks.Rope, quality_shift, with_quality, UtoT, fileName)
 		} else {
 			sequences, err = parser(chunks.Source, chunks.Raw)
 		}
@@ -423,6 +424,8 @@ func ReadFastq(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, e
 		false,
 	)
 
+	fileName := opt.FileName()
+
 	for i := 0; i < nworker; i++ {
 		out.Add(1)
 		go _ParseFastqFile(
@@ -431,6 +434,7 @@ func ReadFastq(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, e
 			obidefault.ReadQualitiesShift(),
 			opt.ReadQualities(),
 			opt.UtoT(),
+			fileName,
 		)
 	}
 
@@ -456,7 +460,9 @@ func ReadFastq(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, e
 }
 
 func ReadFastqFromFile(filename string, options ...WithOption) (obiiter.IBioSequence, error) {
-	options = append(options, OptionsSource(obiutils.RemoveAllExt((path.Base(filename)))))
+	options = append(options,
+		OptionsSource(obiutils.RemoveAllExt((path.Base(filename)))),
+		OptionsFileName(filename))
 
 	file, err := obiutils.Ropen(filename)
 

pkg/obiformats/fastseq_json_header.go

@@ -199,8 +199,111 @@ func _parse_json_array_interface(str []byte) ([]interface{}, error) {
 	return values, nil
 }
 
-func _parse_json_header_(header string, sequence *obiseq.BioSequence) string {
+// _parse_json_annotation_field parses a single key/value pair coming from a
+// JSON object (either a FASTA/FASTQ inline JSON header, or the "annotations"
+// field of a JSON sequence record) and applies it to the sequence, special
+// casing the well-known OBITools attributes (id, definition, count, taxid,
+// obiclean_*, merged_*).
+func _parse_json_annotation_field(key []byte, value []byte, dataType jsonparser.ValueType, sequence *obiseq.BioSequence) error {
 	annotations := sequence.Annotations()
+	var err error
+
+	skey := obiutils.UnsafeString(key)
+
+	switch {
+	case skey == "id":
+		sequence.SetId(string(value))
+	case skey == "definition":
+		sequence.SetDefinition(string(value))
+
+	case skey == "count":
+		if dataType != jsonparser.Number {
+			log.Fatalf("%s: Count attribut must be numeric: %s", sequence.Id(), string(value))
+		}
+		count, err := jsonparser.ParseInt(value)
+		if err != nil {
+			log.Fatalf("%s: Cannot parse count %s", sequence.Id(), string(value))
+		}
+		sequence.SetCount(int(count))
+
+	case skey == "obiclean_weight":
+		weight, err := _parse_json_map_int(value)
+		if err != nil {
+			log.Fatalf("%s: Cannot parse obiclean weight %s", sequence.Id(), string(value))
+		}
+		annotations[skey] = weight
+
+	case skey == "obiclean_status":
+		status, err := _parse_json_map_string(value)
+		if err != nil {
+			log.Fatalf("%s: Cannot parse obiclean status %s", sequence.Id(), string(value))
+		}
+		annotations[skey] = status
+
+	case strings.HasPrefix(skey, "merged_"):
+		if dataType == jsonparser.Object {
+			data, err := _parse_json_map_int(value)
+			if err != nil {
+				log.Fatalf("%s: Cannot parse merged slot %s: %v", sequence.Id(), skey, err)
+			} else {
+				annotations[skey] = obiseq.MapAsStatsOnValues(data)
+			}
+		} else {
+			log.Fatalf("%s: Cannot parse merged slot %s", sequence.Id(), skey)
+		}
+
+	case skey == "taxid":
+		if dataType == jsonparser.Number || dataType == jsonparser.String {
+			taxid := string(value)
+			sequence.SetTaxid(taxid)
+		} else {
+			log.Fatalf("%s: Cannot parse taxid %s", sequence.Id(), string(value))
+		}
+
+	case strings.HasSuffix(skey, "_taxid"):
+		if dataType == jsonparser.Number || dataType == jsonparser.String {
+			rank := skey[:len(skey)-len("_taxid")]
+
+			taxid := string(value)
+			sequence.SetTaxid(taxid, rank)
+		} else {
+			log.Fatalf("%s: Cannot parse taxid %s", sequence.Id(), string(value))
+		}
+
+	default:
+		skey = strings.Clone(skey)
+		switch dataType {
+		case jsonparser.String:
+			annotations[skey] = string(value)
+		case jsonparser.Number:
+			// Try to parse the number as an int at first then as float if that fails.
+			annotations[skey], err = jsonparser.ParseInt(value)
+			if err != nil {
+				annotations[skey], err = strconv.ParseFloat(obiutils.UnsafeString(value), 64)
+			}
+		case jsonparser.Array:
+			annotations[skey], err = _parse_json_array_interface(value)
+		case jsonparser.Object:
+			annotations[skey], err = _parse_json_map_interface(value)
+		case jsonparser.Boolean:
+			annotations[skey], err = jsonparser.ParseBoolean(value)
+		case jsonparser.Null:
+			annotations[skey] = nil
+		default:
+			log.Fatalf("Unknown data type %v", dataType)
+		}
+	}
+
+	if err != nil {
+		annotations[skey] = "NaN"
+		log.Fatalf("%s: Cannot parse value %s assicated to key %s into a %s value",
+			sequence.Id(), string(value), skey, dataType.String())
+	}
+
+	return err
+}
+
+func _parse_json_header_(header string, sequence *obiseq.BioSequence) string {
 	start := -1
 	stop := -1
 	level := 0
@@ -240,101 +343,7 @@ func _parse_json_header_(header string, sequence *obiseq.BioSequence) string {
 
 	jsonparser.ObjectEach(obiutils.UnsafeBytes(header[start:stop]),
 		func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) error {
-			var err error
-
-			skey := obiutils.UnsafeString(key)
-
-			switch {
-			case skey == "id":
-				sequence.SetId(string(value))
-			case skey == "definition":
-				sequence.SetDefinition(string(value))
-
-			case skey == "count":
-				if dataType != jsonparser.Number {
-					log.Fatalf("%s: Count attribut must be numeric: %s", sequence.Id(), string(value))
-				}
-				count, err := jsonparser.ParseInt(value)
-				if err != nil {
-					log.Fatalf("%s: Cannot parse count %s", sequence.Id(), string(value))
-				}
-				sequence.SetCount(int(count))
-
-			case skey == "obiclean_weight":
-				weight, err := _parse_json_map_int(value)
-				if err != nil {
-					log.Fatalf("%s: Cannot parse obiclean weight %s", sequence.Id(), string(value))
-				}
-				annotations[skey] = weight
-
-			case skey == "obiclean_status":
-				status, err := _parse_json_map_string(value)
-				if err != nil {
-					log.Fatalf("%s: Cannot parse obiclean status %s", sequence.Id(), string(value))
-				}
-				annotations[skey] = status
-
-			case strings.HasPrefix(skey, "merged_"):
-				if dataType == jsonparser.Object {
-					data, err := _parse_json_map_int(value)
-					if err != nil {
-						log.Fatalf("%s: Cannot parse merged slot %s: %v", sequence.Id(), skey, err)
-					} else {
-						annotations[skey] = obiseq.MapAsStatsOnValues(data)
-					}
-				} else {
-					log.Fatalf("%s: Cannot parse merged slot %s", sequence.Id(), skey)
-				}
-
-			case skey == "taxid":
-				if dataType == jsonparser.Number || dataType == jsonparser.String {
-					taxid := string(value)
-					sequence.SetTaxid(taxid)
-				} else {
-					log.Fatalf("%s: Cannot parse taxid %s", sequence.Id(), string(value))
-				}
-
-			case strings.HasSuffix(skey, "_taxid"):
-				if dataType == jsonparser.Number || dataType == jsonparser.String {
-					rank := skey[:len(skey)-len("_taxid")]
-
-					taxid := string(value)
-					sequence.SetTaxid(taxid, rank)
-				} else {
-					log.Fatalf("%s: Cannot parse taxid %s", sequence.Id(), string(value))
-				}
-
-			default:
-				skey = strings.Clone(skey)
-				switch dataType {
-				case jsonparser.String:
-					annotations[skey] = string(value)
-				case jsonparser.Number:
-					// Try to parse the number as an int at first then as float if that fails.
-					annotations[skey], err = jsonparser.ParseInt(value)
-					if err != nil {
-						annotations[skey], err = strconv.ParseFloat(obiutils.UnsafeString(value), 64)
-					}
-				case jsonparser.Array:
-					annotations[skey], err = _parse_json_array_interface(value)
-				case jsonparser.Object:
-					annotations[skey], err = _parse_json_map_interface(value)
-				case jsonparser.Boolean:
-					annotations[skey], err = jsonparser.ParseBoolean(value)
-				case jsonparser.Null:
-					annotations[skey] = nil
-				default:
-					log.Fatalf("Unknown data type %v", dataType)
-				}
-			}
-
-			if err != nil {
-				annotations[skey] = "NaN"
-				log.Fatalf("%s: Cannot parse value %s assicated to key %s into a %s value",
-					sequence.Id(), string(value), skey, dataType.String())
-			}
-
-			return err
+			return _parse_json_annotation_field(key, value, dataType, sequence)
 		},
 	)
 

pkg/obiformats/fastseq_obi_header.go

@@ -17,7 +17,7 @@ import (
 )
 
 var __obi_header_value_string_pattern__ = regexp.MustCompile(`^'\s*([^']*'|"[^"]*")\s*;`)
-var __obi_header_value_numeric_pattern__ = regexp.MustCompile(`^\s*([+-]?\.\d+|[+-]?\d+(\.\d*)?([eE][+-]?\d+)?)\s*;`)
+var __obi_header_value_numeric_pattern__ = regexp.MustCompile(`^\s*[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?\s*;`)
 var __obi_header_map_int_key__ = regexp.MustCompile("([{,])([0-9]+):")
 
 func __match__dict__(text []byte) []int {
@@ -146,6 +146,65 @@ func __match__key__(text []byte) []int {
 	return []int{} // Not a key
 }
 
+func __match__array__(text []byte) []int {
+
+	state := 0
+	level := 0
+	start := 0
+	instring := byte(0)
+
+	for i, r := range text {
+		if state == 2 {
+			if r == ';' {
+				return []int{start, i + 1}
+			}
+			if r != ' ' && r != '\t' {
+				return []int{}
+			}
+		}
+
+		if state == 0 {
+			if r == '[' {
+				level++
+				state++
+				start = i
+				continue
+			}
+			if r != ' ' && r != '\t' {
+				return []int{}
+			}
+			continue
+		}
+
+		// state == 1: inside the array
+		if instring != 0 {
+			if r == instring {
+				instring = 0
+			}
+			continue
+		}
+
+		if r == '"' || r == '\'' {
+			instring = r
+			continue
+		}
+
+		if r == '[' || r == '{' {
+			level++
+			continue
+		}
+
+		if r == ']' || r == '}' {
+			level--
+			if level == 0 {
+				state++
+			}
+		}
+	}
+
+	return []int{}
+}
+
 func __match__general__(text []byte) []int {
 
 	for i, r := range text {
@@ -242,28 +301,44 @@ func ParseOBIFeatures(text string, annotations obiseq.Annotation) string {
 					stop = m[1] + 1
 				} else {
 
-					// Generic value
-
-					// m = __obi_header_value_general_pattern__.FindIndex(part)
-					m = __match__general__(part)
+					// array value
+					m = __match__array__(part)
 					if len(m) > 0 {
 						bvalue = bytes.TrimSpace(part[m[0]:(m[1] - 1)])
-
-						if __is_false__(bvalue) {
-							value = false
+						j := bytes.ReplaceAll(bvalue, []byte("'"), []byte(`"`))
+						j = __obi_header_map_int_key__.ReplaceAll(j, []byte(`$1"$2":`))
+						arr, err := _parse_json_array_interface(j)
+						if err != nil {
+							value = string(bvalue)
 						} else {
-							if __is_true__(bvalue) {
-								value = true
-							} else {
-								value = string(bvalue)
-							}
+							value = arr
 						}
-
 						stop = m[1] + 1
 					} else {
-						// no value
-						break
-					} // End of No value
+
+						// Generic value
+
+						// m = __obi_header_value_general_pattern__.FindIndex(part)
+						m = __match__general__(part)
+						if len(m) > 0 {
+							bvalue = bytes.TrimSpace(part[m[0]:(m[1] - 1)])
+
+							if __is_false__(bvalue) {
+								value = false
+							} else {
+								if __is_true__(bvalue) {
+									value = true
+								} else {
+									value = string(bvalue)
+								}
+							}
+
+							stop = m[1] + 1
+						} else {
+							// no value
+							break
+						} // End of No value
+					} // End of not array
 				} // End of not dict
 			} // End of not string
 		} // End of not numeric
@@ -272,6 +347,8 @@ func ParseOBIFeatures(text string, annotations obiseq.Annotation) string {
 		case float64:
 			if vt == math.Floor(vt) {
 				annotations[key] = int(vt)
+			} else {
+				annotations[key] = vt
 			}
 		default:
 			annotations[key] = value
@@ -327,19 +404,19 @@ func WriteFastSeqOBIHeade(buffer *bytes.Buffer, sequence *obiseq.BioSequence) {
 					buffer.WriteString(fmt.Sprintf("%s=", key))
 					buffer.Write(tv)
 					buffer.WriteString("; ")
-				case map[string]int,
-					map[string]string,
-					map[string]interface{}:
-					tv, err := obiutils.JsonMarshal(t)
-					if err != nil {
-						log.Fatalf("Cannot convert %v value", value)
-					}
-					tv = bytes.ReplaceAll(tv, []byte(`"`), []byte("'"))
-					buffer.WriteString(fmt.Sprintf("%s=", key))
-					buffer.Write(tv)
-					buffer.WriteString("; ")
 				default:
-					buffer.WriteString(fmt.Sprintf("%s=%v; ", key, value))
+					if obiutils.IsAMap(value) || obiutils.IsASlice(value) || obiutils.IsAnArray(value) {
+						tv, err := obiutils.JsonMarshal(t)
+						if err != nil {
+							log.Fatalf("Cannot convert %v value", value)
+						}
+						tv = bytes.ReplaceAll(tv, []byte(`"`), []byte("'"))
+						buffer.WriteString(fmt.Sprintf("%s=", key))
+						buffer.Write(tv)
+						buffer.WriteString("; ")
+					} else {
+						buffer.WriteString(fmt.Sprintf("%s=%v; ", key, value))
+					}
 				}
 			}
 		}

pkg/obiformats/fastseq_write_fasta.go

@@ -90,6 +90,9 @@ func FormatFastaBatch(batch obiiter.BioSequenceBatch, formater FormatHeader, ski
 	log.Debugf("FormatFastaBatch: #%d : %d seqs", batch.Order(), batch.Len())
 
 	for _, seq := range batch.Slice() {
+		if len(seq.Id()) == 0 {
+			log.Fatalf("Sequence identifier is empty")
+		}
 		if seq.Len() > 0 {
 			// Write header directly into bs — no intermediate string
 			bs.WriteByte('>')

pkg/obiformats/fastseq_write_fastq.go

@@ -64,6 +64,9 @@ func FormatFastqBatch(batch obiiter.BioSequenceBatch,
 	first := true
 
 	for _, seq := range batch.Slice() {
+		if len(seq.Id()) == 0 {
+			log.Fatalf("Sequence identifier is empty")
+		}
 		if seq.Len() > 0 {
 			_formatFastq(&bs, seq, formater)
 

pkg/obiformats/json_read.go

@@ -0,0 +1,147 @@
+package obiformats
+
+import (
+	"io"
+	"os"
+	"path"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+	"github.com/buger/jsonparser"
+	"github.com/goccy/go-json"
+	log "github.com/sirupsen/logrus"
+)
+
+// _parse_json_record parses a single JSON object describing a sequence
+// (as produced by JSONRecord in json_writer.go) into a *obiseq.BioSequence.
+func _parse_json_record(raw []byte, shift byte) *obiseq.BioSequence {
+	sequence := obiseq.NewEmptyBioSequence(0)
+
+	if id, err := jsonparser.GetString(raw, "id"); err == nil {
+		sequence.SetId(id)
+	}
+
+	if seq, err := jsonparser.GetString(raw, "sequence"); err == nil {
+		sequence.SetSequence([]byte(seq))
+	}
+
+	if qual, err := jsonparser.GetString(raw, "qualities"); err == nil {
+		q := []byte(qual)
+		for i := 0; i < len(q); i++ {
+			q[i] -= shift
+		}
+		sequence.SetQualities(q)
+	}
+
+	if annot, dataType, _, err := jsonparser.Get(raw, "annotations"); err == nil && dataType == jsonparser.Object {
+		jsonparser.ObjectEach(annot,
+			func(key []byte, value []byte, valType jsonparser.ValueType, offset int) error {
+				return _parse_json_annotation_field(key, value, valType, sequence)
+			},
+		)
+	}
+
+	return sequence
+}
+
+// _ParseJsonFile streams the top-level JSON array, decoding and pushing one
+// batch of sequences at a time, without ever loading the whole document in
+// memory. Only one raw record at a time is buffered by the decoder.
+func _ParseJsonFile(source string,
+	reader io.Reader,
+	out obiiter.IBioSequence,
+	shift byte,
+	batchSize int) {
+
+	dec := json.NewDecoder(reader)
+
+	if _, err := dec.Token(); err != nil {
+		if err == io.EOF {
+			out.Done()
+			return
+		}
+		log.Fatalf("cannot parse JSON data: %v", err)
+	}
+
+	slice := obiseq.MakeBioSequenceSlice()
+	o := 0
+
+	for dec.More() {
+		var raw json.RawMessage
+
+		if err := dec.Decode(&raw); err != nil {
+			log.Fatalf("cannot parse JSON data: %v", err)
+		}
+
+		sequence := _parse_json_record(raw, shift)
+
+		slice = append(slice, sequence)
+		if len(slice) >= batchSize {
+			out.Push(obiiter.MakeBioSequenceBatch(source, o, slice))
+			o++
+			slice = obiseq.MakeBioSequenceSlice()
+		}
+	}
+
+	if len(slice) > 0 {
+		out.Push(obiiter.MakeBioSequenceBatch(source, o, slice))
+	}
+
+	out.Done()
+}
+
+func ReadJSON(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, error) {
+
+	opt := MakeOptions(options)
+	out := obiiter.MakeIBioSequence()
+
+	out.Add(1)
+	go _ParseJsonFile(opt.Source(),
+		reader,
+		out,
+		obidefault.ReadQualitiesShift(),
+		opt.BatchSize())
+
+	go func() {
+		out.WaitAndClose()
+	}()
+
+	return out, nil
+
+}
+
+func ReadJSONFromFile(filename string, options ...WithOption) (obiiter.IBioSequence, error) {
+
+	options = append(options, OptionsSource(obiutils.RemoveAllExt((path.Base(filename)))))
+	file, err := obiutils.Ropen(filename)
+
+	if err == obiutils.ErrNoContent {
+		log.Infof("file %s is empty", filename)
+		return ReadEmptyFile(options...)
+	}
+
+	if err != nil {
+		return obiiter.NilIBioSequence, err
+	}
+
+	return ReadJSON(file, options...)
+}
+
+func ReadJSONFromStdin(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, error) {
+	options = append(options, OptionsSource(obiutils.RemoveAllExt("stdin")))
+	input, err := obiutils.Buf(os.Stdin)
+
+	if err == obiutils.ErrNoContent {
+		log.Infof("stdin is empty")
+		return ReadEmptyFile(options...)
+	}
+
+	if err != nil {
+		log.Fatalf("open file error: %v", err)
+		return obiiter.NilIBioSequence, err
+	}
+
+	return ReadJSON(input, options...)
+}

pkg/obiformats/ngsfilter_read.go

@@ -631,9 +631,9 @@ func ReadCSVNGSFilter(reader io.Reader) (*obingslibrary.NGSLibrary, error) {
 			return nil, fmt.Errorf("row %d has %d columns, expected %d", len(data), len(fields), len(header))
 		}
 
-		forward_primer := fields[forward_primerColIndex]
-		reverse_primer := fields[reverse_primerColIndex]
-		tags := _parseMainNGSFilterTags(fields[sample_tagColIndex])
+		forward_primer := strings.TrimSpace(fields[forward_primerColIndex])
+		reverse_primer := strings.TrimSpace(fields[reverse_primerColIndex])
+		tags := _parseMainNGSFilterTags(strings.TrimSpace(fields[sample_tagColIndex]))
 
 		marker, _ := ngsfilter.GetMarker(forward_primer, reverse_primer)
 		pcr, ok := marker.GetPCR(tags.Forward, tags.Reverse)
@@ -644,8 +644,8 @@ func ReadCSVNGSFilter(reader io.Reader) (*obingslibrary.NGSLibrary, error) {
 					i, tags.Forward, tags.Reverse, forward_primer, reverse_primer)
 		}
 
-		pcr.Experiment = fields[experimentColIndex]
-		pcr.Sample = fields[sampleColIndex]
+		pcr.Experiment = strings.TrimSpace(fields[experimentColIndex])
+		pcr.Sample = strings.TrimSpace(fields[sampleColIndex])
 
 		if extraColumns != nil {
 			pcr.Annotations = make(obiseq.Annotation)

pkg/obiformats/options.go

@@ -35,6 +35,7 @@ type __options__ struct {
 	csv_auto              bool
 	paired_filename       string
 	source                string
+	filename              string
 	with_feature_table    bool
 	with_pattern          bool
 	with_parent           bool
@@ -216,6 +217,16 @@ func (opt Options) Source() string {
 	return opt.pointer.source
 }
 
+// FileName returns the full path of the file being read, for use in
+// diagnostic messages. It falls back to Source() when no explicit
+// file name has been set (e.g. reading from stdin or a raw reader).
+func (opt Options) FileName() string {
+	if opt.pointer.filename == "" {
+		return opt.pointer.source
+	}
+	return opt.pointer.filename
+}
+
 func (opt Options) WithFeatureTable() bool {
 	return opt.pointer.with_feature_table
 }
@@ -421,6 +432,14 @@ func OptionsSource(source string) WithOption {
 	return f
 }
 
+func OptionsFileName(filename string) WithOption {
+	f := WithOption(func(opt Options) {
+		opt.pointer.filename = filename
+	})
+
+	return f
+}
+
 func OptionsWithProgressBar() WithOption {
 	f := WithOption(func(opt Options) {
 		opt.pointer.with_progress_bar = true

pkg/obiformats/universal_read.go

@@ -145,6 +145,8 @@ func ReadSequencesFromFile(filename string,
 		return ReadGenbank(reader, options...)
 	case "text/csv":
 		return ReadCSV(reader, options...)
+	case "application/json":
+		return ReadJSON(reader, options...)
 	default:
 		log.Fatalf("File %s has guessed format %s which is not yet implemented",
 			filename, mime.String())

pkg/obifp/uint128_test.go

@@ -134,7 +134,7 @@ func TestUint128_QuoRem(t *testing.T) {
 	u := Uint128{w1: 3, w0: 8}
 	v := Uint128{w1: 0, w0: 4}
 	q, r := u.QuoRem(v)
-	assert.Equal(t, Uint128{w1: 0, w0: 2}, q)
+	assert.Equal(t, Uint128{w1: 0, w0: 13835058055282163714}, q)
 	assert.Equal(t, Uint128{w1: 0, w0: 0}, r)
 }
 
@@ -150,7 +150,7 @@ func TestUint128_Div(t *testing.T) {
 	u := Uint128{w1: 3, w0: 8}
 	v := Uint128{w1: 0, w0: 4}
 	q := u.Div(v)
-	assert.Equal(t, Uint128{w1: 0, w0: 2}, q)
+	assert.Equal(t, Uint128{w1: 0, w0: 13835058055282163714}, q)
 }
 
 func TestUint128_Div64(t *testing.T) {
@@ -183,7 +183,7 @@ func TestUint128_Cmp(t *testing.T) {
 func TestUint128_Cmp64(t *testing.T) {
 	u := Uint128{w1: 1, w0: 2}
 	v := uint64(3)
-	assert.Equal(t, -1, u.Cmp64(v))
+	assert.Equal(t, 1, u.Cmp64(v))
 }
 
 func TestUint128_Equals(t *testing.T) {

pkg/obigraph/clique.go

@@ -0,0 +1,343 @@
+package obigraph
+
+import (
+	"context"
+	"math/rand"
+	"sync"
+	"sync/atomic"
+)
+
+// NeighborFilter defines an implicit graph over the vertices 0..n-1.
+// It appends to out[:0] the vertices of candidates that are adjacent to v,
+// keeping their order, and returns the resulting slice. It is called
+// concurrently by several workers and must not modify candidates.
+//
+// Describing the adjacency by a filter, instead of a matrix, keeps the memory
+// in O(n) and lets the caller evaluate adjacency in bulk (SIMD, for instance).
+type NeighborFilter func(v int32, candidates, out []int32) []int32
+
+// CliqueOptions tunes FindClique.
+type CliqueOptions struct {
+	// Workers is the number of concurrent searches (default 1).
+	Workers int
+	// Seed seeds the random vertex orders of the restarts.
+	Seed int64
+	// Progress, if set, is called with the size of the best clique each time a
+	// larger one is found. Calls are serialized and made while a lock is held:
+	// it must return quickly and must not call FindClique.
+	Progress func(best int)
+
+	// ColorLimit enables the coloring bound for the candidate lists of at most
+	// ColorLimit vertices (0 disables it). The vertices of a list are split into
+	// classes of pairwise non-adjacent vertices: a clique takes at most one
+	// vertex per class, so the number of classes bounds its size. Computing the
+	// classes costs up to the square of the list length in calls to the filter,
+	// which is why only short lists are colored.
+	ColorLimit int
+
+	// mode forces the kind of search (tests only): 0 alternates, modeFocused
+	// and modeImproving restrict the search to one of them.
+	mode int
+}
+
+const (
+	modeFocused   = 1
+	modeImproving = 2
+)
+
+// CliqueResult is the outcome of FindClique.
+type CliqueResult struct {
+	// Clique is the largest clique found (a clique of size >= target if Reached).
+	Clique []int32
+	// Reached is true if len(Clique) >= target.
+	Reached bool
+	// Exhausted is true if the whole search space has been explored. If Reached
+	// is false, it proves that no clique of the target size exists.
+	Exhausted bool
+	// Maximum is true if Clique is proven to be a maximum clique of the graph:
+	// no larger clique exists.
+	Maximum bool
+}
+
+// FindClique looks for a clique of at least target vertices in the graph of n
+// vertices described by filter. It stops as soon as one is found, when the
+// search space is exhausted, or when ctx is done; in the last case, the largest
+// clique met so far is returned.
+//
+// The search is a depth first search with backtracking, which comes in two
+// flavours that the restarts alternate:
+//
+//   - the focused search prunes every branch that cannot reach target, which
+//     is the most efficient way to reach it but gives poor cliques when target
+//     is out of reach;
+//   - the improving search only prunes the branches that cannot beat the best
+//     clique found so far (by any worker), so it keeps improving that clique
+//     whatever target is. When it ends, the best clique is a maximum one.
+//
+// When a focused search proves that no clique of target vertices exists, every
+// search turns to the improving flavour, so that the largest clique is returned
+// (and proved to be so) instead of an arbitrary one.
+//
+// On short candidate lists (see CliqueOptions.ColorLimit), a coloring bound
+// prunes the branches that cannot be large enough, and orders the vertices so
+// that the most constrained ones are tried first.
+//
+// The first pass of the first worker follows the natural order of the vertices
+// (greedy "lexicographic" construction); every other pass restarts from a
+// random order, with a work budget that doubles at each restart, so that the
+// search is complete in the long run.
+func FindClique(ctx context.Context, n int, filter NeighborFilter, target int, opts CliqueOptions) CliqueResult {
+	if target <= 0 {
+		return CliqueResult{Reached: true, Exhausted: true}
+	}
+	if n <= 0 {
+		return CliqueResult{Exhausted: true}
+	}
+
+	workers := max(opts.Workers, 1)
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	var (
+		mutex     sync.Mutex
+		best      []int32
+		bestSize  atomic.Int32
+		exhausted bool
+		maximum   bool
+		wg        sync.WaitGroup
+		// set when a focused search has proved that target cannot be reached:
+		// every search then turns to finding the largest clique
+		unreachable atomic.Bool
+	)
+
+	publish := func(clique []int32) {
+		mutex.Lock()
+		if len(clique) > len(best) {
+			best = append(best[:0], clique...)
+			bestSize.Store(int32(len(best)))
+			if opts.Progress != nil {
+				opts.Progress(len(best))
+			}
+		}
+		mutex.Unlock()
+	}
+
+	for w := range workers {
+		wg.Go(func() {
+			s := &cliqueSearch{
+				ctx:         ctx,
+				filter:      filter,
+				target:      target,
+				bestSize:    &bestSize,
+				colorLimit:  opts.ColorLimit,
+				unreachable: &unreachable,
+				publish: func(c []int32) {
+					publish(c)
+					if len(c) >= target {
+						cancel()
+					}
+				},
+			}
+			rng := rand.New(rand.NewSource(opts.Seed + int64(w)))
+			// the search starts from the end of the list: lowest vertex first
+			candidates := make([]int32, n)
+			for i := range candidates {
+				candidates[i] = int32(n - 1 - i)
+			}
+
+			for restart := 0; ctx.Err() == nil; restart++ {
+				if w > 0 || restart > 0 {
+					rng.Shuffle(n, func(i, j int) { candidates[i], candidates[j] = candidates[j], candidates[i] })
+				}
+				switch opts.mode {
+				case modeFocused:
+					s.improving = false
+				case modeImproving:
+					s.improving = true
+				default:
+					s.improving = unreachable.Load() || (w+restart)%2 == 1
+				}
+				s.budget = int64(n) * int64(target) << min(restart, 20)
+				s.work = 0
+				s.aborted = false
+				s.search(0, candidates)
+
+				if !s.aborted && ctx.Err() == nil {
+					mutex.Lock()
+					exhausted = true
+					if s.improving {
+						maximum = true
+					}
+					mutex.Unlock()
+
+					if s.improving || opts.mode == modeFocused {
+						cancel()
+					} else {
+						// target is out of reach: look for the largest clique
+						unreachable.Store(true)
+					}
+				}
+			}
+		})
+	}
+	wg.Wait()
+
+	return CliqueResult{
+		Clique:    best,
+		Reached:   len(best) >= target,
+		Exhausted: exhausted,
+		Maximum:   maximum,
+	}
+}
+
+type cliqueSearch struct {
+	ctx        context.Context
+	filter     NeighborFilter
+	target     int
+	bestSize   *atomic.Int32 // size of the best clique found by any worker
+	publish    func([]int32)
+	colorLimit int
+
+	unreachable *atomic.Bool // target is proved out of reach
+
+	improving bool // see FindClique: the kind of the current pass
+
+	current []int32   // the clique under construction
+	buffers [][]int32 // candidate list of each depth
+	sorted  [][]int32 // candidate list of each depth, sorted by color
+	colors  [][]int32 // color (from 1) of each vertex of sorted
+	classes [][]int32 // scratch space of the coloring
+	probe   []int32   // scratch space of the coloring
+
+	budget  int64 // maximum work (number of candidates tested) for this restart
+	work    int64
+	aborted bool // budget or context exhausted before the end of the exploration
+}
+
+// threshold is the size a clique must reach to be worth exploring: the target
+// for the focused search, one more than the best clique for the improving one.
+func (s *cliqueSearch) threshold() int {
+	if s.improving {
+		return min(s.target, int(s.bestSize.Load())+1)
+	}
+	return s.target
+}
+
+// mustStop tells whether the current pass has to be interrupted: its work
+// budget is spent, or the pass must stop whatever its budget (see mustAbort).
+func (s *cliqueSearch) mustStop() bool {
+	return s.work > s.budget || s.mustAbort()
+}
+
+// mustAbort tells whether the pass has to be interrupted whatever its budget:
+// the context is done, or it is a focused pass and its target has been proved
+// out of reach.
+func (s *cliqueSearch) mustAbort() bool {
+	return s.ctx.Err() != nil || (!s.improving && s.unreachable.Load())
+}
+
+// colorSort splits candidates into classes of pairwise non-adjacent vertices
+// (greedy sequential coloring) and returns them class after class, with the
+// color of each vertex. Any clique takes at most one vertex of each class.
+//
+// The coloring can be long: it stops, and sets s.aborted, as soon as the search
+// has to be aborted. Its cost is not charged to the work budget of the pass,
+// which limits the exploration of the tree, not the bound computed at each node.
+func (s *cliqueSearch) colorSort(depth int, candidates []int32) (sorted, colors []int32) {
+	k := 0
+	for _, v := range candidates {
+		if s.mustAbort() {
+			s.aborted = true
+			return nil, nil
+		}
+		placed := false
+		for c := 0; c < k; c++ {
+			// v joins the class if it is adjacent to none of its members
+			s.probe = s.filter(v, s.classes[c], s.probe[:0])
+			if len(s.probe) == 0 {
+				s.classes[c] = append(s.classes[c], v)
+				placed = true
+				break
+			}
+		}
+		if !placed {
+			if k == len(s.classes) {
+				s.classes = append(s.classes, nil)
+			}
+			s.classes[k] = append(s.classes[k][:0], v)
+			k++
+		}
+	}
+
+	for len(s.sorted) <= depth {
+		s.sorted = append(s.sorted, nil)
+		s.colors = append(s.colors, nil)
+	}
+	sorted, colors = s.sorted[depth][:0], s.colors[depth][:0]
+	for c := 0; c < k; c++ {
+		for _, v := range s.classes[c] {
+			sorted = append(sorted, v)
+			colors = append(colors, int32(c+1))
+		}
+	}
+	s.sorted[depth], s.colors[depth] = sorted, colors
+	return sorted, colors
+}
+
+// search extends the current clique of depth vertices with vertices taken in
+// candidates, all adjacent to the vertices of the current clique. The list is
+// read from its end: the vertex at position i is combined with the vertices
+// before it, so that every clique is met once.
+func (s *cliqueSearch) search(depth int, candidates []int32) {
+	var colors []int32
+	if s.colorLimit > 0 && len(candidates) > 1 && len(candidates) <= s.colorLimit {
+		candidates, colors = s.colorSort(depth, candidates)
+		if s.aborted {
+			return
+		}
+	}
+
+	for i := len(candidates) - 1; i >= 0; i-- {
+		// largest clique the vertices 0..i can make
+		bound := i + 1
+		if colors != nil {
+			bound = int(colors[i])
+		}
+		if depth+bound < s.threshold() {
+			return
+		}
+		if s.mustStop() {
+			s.aborted = true
+			return
+		}
+
+		v := candidates[i]
+		s.current = append(s.current[:depth], v)
+		if len(s.current) > int(s.bestSize.Load()) {
+			s.publish(s.current)
+			if len(s.current) >= s.target {
+				s.aborted = true // the search is over: do not report an exhaustion
+				return
+			}
+		}
+
+		rest := candidates[:i]
+		if depth+1+len(rest) < s.threshold() {
+			continue
+		}
+		for len(s.buffers) <= depth+1 {
+			s.buffers = append(s.buffers, nil)
+		}
+		next := s.filter(v, rest, s.buffers[depth+1][:0])
+		s.buffers[depth+1] = next
+		s.work += int64(len(rest))
+
+		if depth+1+len(next) < s.threshold() {
+			continue
+		}
+		s.search(depth+1, next)
+		if s.aborted {
+			return
+		}
+	}
+}

pkg/obigraph/clique_test.go

@@ -0,0 +1,249 @@
+package obigraph
+
+import (
+	"context"
+	"math/bits"
+	"math/rand"
+	"testing"
+	"time"
+)
+
+// adjacency matrix filter for small graphs
+func matrixFilter(adj [][]bool) NeighborFilter {
+	return func(v int32, candidates, out []int32) []int32 {
+		for _, c := range candidates {
+			if adj[v][c] {
+				out = append(out, c)
+			}
+		}
+		return out
+	}
+}
+
+func randomGraph(rng *rand.Rand, n int, p float64) [][]bool {
+	adj := make([][]bool, n)
+	for i := range adj {
+		adj[i] = make([]bool, n)
+	}
+	for i := 0; i < n; i++ {
+		for j := i + 1; j < n; j++ {
+			if rng.Float64() < p {
+				adj[i][j], adj[j][i] = true, true
+			}
+		}
+	}
+	return adj
+}
+
+// maximum clique size by brute force over all the subsets (n <= 20)
+func bruteMaxClique(adj [][]bool) int {
+	n := len(adj)
+	best := 0
+	for mask := 1; mask < 1<<n; mask++ {
+		size := bits.OnesCount(uint(mask))
+		if size <= best {
+			continue
+		}
+		ok := true
+		for i := 0; i < n && ok; i++ {
+			if mask>>i&1 == 0 {
+				continue
+			}
+			for j := i + 1; j < n; j++ {
+				if mask>>j&1 == 1 && !adj[i][j] {
+					ok = false
+					break
+				}
+			}
+		}
+		if ok {
+			best = size
+		}
+	}
+	return best
+}
+
+func isClique(adj [][]bool, c []int32) bool {
+	seen := map[int32]bool{}
+	for i, a := range c {
+		if seen[a] {
+			return false
+		}
+		seen[a] = true
+		for _, b := range c[i+1:] {
+			if !adj[a][b] {
+				return false
+			}
+		}
+	}
+	return true
+}
+
+func TestFindCliqueAgainstBruteForce(t *testing.T) {
+	rng := rand.New(rand.NewSource(1))
+	for trial := 0; trial < 40; trial++ {
+		n := 8 + rng.Intn(10)
+		adj := randomGraph(rng, n, 0.3+0.5*rng.Float64())
+		maxSize := bruteMaxClique(adj)
+		filter := matrixFilter(adj)
+
+		for _, workers := range []int{1, 3} {
+			r := FindClique(context.Background(), n, filter, maxSize, CliqueOptions{Workers: workers})
+			if !r.Reached || len(r.Clique) < maxSize || !isClique(adj, r.Clique) {
+				t.Fatalf("trial %d: target %d not reached: %+v", trial, maxSize, r)
+			}
+
+			r = FindClique(context.Background(), n, filter, maxSize+1, CliqueOptions{Workers: workers})
+			if r.Reached || !r.Exhausted {
+				t.Fatalf("trial %d: size %d cannot exist, got %+v", trial, maxSize+1, r)
+			}
+			// best effort: a valid clique, not necessarily a maximum one
+			if len(r.Clique) < 1 || len(r.Clique) > maxSize || !isClique(adj, r.Clique) {
+				t.Fatalf("trial %d: invalid best clique %v (maximum is %d)", trial, r.Clique, maxSize)
+			}
+		}
+	}
+}
+
+func TestFindCliqueTrivialTargets(t *testing.T) {
+	adj := randomGraph(rand.New(rand.NewSource(2)), 5, 0.5)
+	if r := FindClique(context.Background(), 5, matrixFilter(adj), 0, CliqueOptions{}); !r.Reached {
+		t.Fatal("target 0 is always reached")
+	}
+	if r := FindClique(context.Background(), 0, matrixFilter(adj), 1, CliqueOptions{}); r.Reached {
+		t.Fatal("no clique in an empty graph")
+	}
+	if r := FindClique(context.Background(), 5, matrixFilter(adj), 1, CliqueOptions{}); !r.Reached || len(r.Clique) != 1 {
+		t.Fatalf("a single vertex is a clique: %+v", r)
+	}
+}
+
+// A hopeless search must stop with the deadline and return a valid clique.
+// (When the target is far above the clique number of a sparse graph, pruning
+// proves the absence at once: see TestFindCliqueAgainstBruteForce.)
+func TestFindCliqueTimeout(t *testing.T) {
+	// dense graph: its largest clique is far below the target, but pruning only
+	// bites after many levels, so the search cannot finish within the deadline
+	const n = 300
+	adj := randomGraph(rand.New(rand.NewSource(3)), n, 0.9)
+	ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond)
+	defer cancel()
+
+	start := time.Now()
+	r := FindClique(ctx, n, matrixFilter(adj), 150, CliqueOptions{Workers: 2})
+	if elapsed := time.Since(start); elapsed > 3*time.Second {
+		t.Fatalf("deadline not honoured: %v", elapsed)
+	}
+	if r.Reached || r.Exhausted {
+		t.Fatalf("size 150 cannot be reached nor proven absent in time: %+v", r)
+	}
+	if len(r.Clique) < 2 || !isClique(adj, r.Clique) {
+		t.Fatalf("invalid best clique %v", r.Clique)
+	}
+}
+
+func TestFindCliqueProgress(t *testing.T) {
+	adj := randomGraph(rand.New(rand.NewSource(4)), 40, 0.7)
+
+	var seen []int
+	r := FindClique(context.Background(), 40, matrixFilter(adj), 5, CliqueOptions{
+		Progress: func(best int) { seen = append(seen, best) },
+	})
+	if !r.Reached {
+		t.Fatalf("size 5 should be reached: %+v", r)
+	}
+	if len(seen) == 0 || seen[len(seen)-1] != len(r.Clique) {
+		t.Fatalf("progress %v does not end at the clique size %d", seen, len(r.Clique))
+	}
+	for i := 1; i < len(seen); i++ {
+		if seen[i] <= seen[i-1] {
+			t.Fatalf("progress must be strictly increasing: %v", seen)
+		}
+	}
+}
+
+// The improving search must end with a proven maximum clique, even when the
+// target is out of reach; the focused one only proves the absence of a clique
+// of the target size.
+func TestFindCliqueModes(t *testing.T) {
+	rng := rand.New(rand.NewSource(5))
+	for trial := 0; trial < 40; trial++ {
+		n := 8 + rng.Intn(10)
+		adj := randomGraph(rng, n, 0.3+0.5*rng.Float64())
+		maxSize := bruteMaxClique(adj)
+		filter := matrixFilter(adj)
+
+		for _, workers := range []int{1, 3} {
+			for _, limit := range []int{0, 5, 1 << 20} {
+				r := FindClique(context.Background(), n, filter, maxSize+1,
+					CliqueOptions{Workers: workers, ColorLimit: limit, mode: modeImproving})
+				if r.Reached || !r.Exhausted || !r.Maximum {
+					t.Fatalf("trial %d limit %d: improving search should prove the maximum: %+v", trial, limit, r)
+				}
+				if len(r.Clique) != maxSize || !isClique(adj, r.Clique) {
+					t.Fatalf("trial %d limit %d: expected a maximum clique of %d, got %v", trial, limit, maxSize, r.Clique)
+				}
+
+				r = FindClique(context.Background(), n, filter, maxSize+1,
+					CliqueOptions{Workers: workers, ColorLimit: limit, mode: modeFocused})
+				if r.Reached || !r.Exhausted || r.Maximum {
+					t.Fatalf("trial %d limit %d: focused search proves absence only: %+v", trial, limit, r)
+				}
+
+				r = FindClique(context.Background(), n, filter, maxSize,
+					CliqueOptions{Workers: workers, ColorLimit: limit, mode: modeImproving})
+				if !r.Reached || len(r.Clique) < maxSize || !isClique(adj, r.Clique) {
+					t.Fatalf("trial %d limit %d: target %d not reached by the improving search: %+v", trial, limit, maxSize, r)
+				}
+			}
+		}
+	}
+}
+
+// When the target is out of reach, the default search must end with a proven
+// maximum clique, even if the coloring bound proves the absence of the target
+// before any clique has been found.
+func TestFindCliqueUnreachableTargetGivesMaximum(t *testing.T) {
+	rng := rand.New(rand.NewSource(6))
+	for trial := 0; trial < 40; trial++ {
+		n := 8 + rng.Intn(10)
+		adj := randomGraph(rng, n, 0.3+0.5*rng.Float64())
+		maxSize := bruteMaxClique(adj)
+
+		for _, workers := range []int{1, 3} {
+			for _, limit := range []int{0, 5, 1 << 20} {
+				r := FindClique(context.Background(), n, matrixFilter(adj), maxSize+1,
+					CliqueOptions{Workers: workers, ColorLimit: limit})
+				if r.Reached || !r.Exhausted || !r.Maximum {
+					t.Fatalf("trial %d workers %d limit %d: expected a proven maximum: %+v", trial, workers, limit, r)
+				}
+				if len(r.Clique) != maxSize || !isClique(adj, r.Clique) {
+					t.Fatalf("trial %d workers %d limit %d: expected a maximum clique of %d, got %v",
+						trial, workers, limit, maxSize, r.Clique)
+				}
+			}
+		}
+	}
+}
+
+// A long coloring must not delay the deadline.
+func TestFindCliqueColoringHonoursDeadline(t *testing.T) {
+	const n = 3000
+	adj := randomGraph(rand.New(rand.NewSource(7)), n, 0.5)
+	base := matrixFilter(adj)
+	slow := func(v int32, candidates, out []int32) []int32 {
+		time.Sleep(200 * time.Microsecond)
+		return base(v, candidates, out)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond)
+	defer cancel()
+	start := time.Now()
+	r := FindClique(ctx, n, slow, 100, CliqueOptions{Workers: 2, ColorLimit: 1 << 20})
+	if elapsed := time.Since(start); elapsed > 2*time.Second {
+		t.Fatalf("deadline not honoured by the coloring: %v", elapsed)
+	}
+	if r.Reached || !isClique(adj, r.Clique) {
+		t.Fatalf("unexpected result %+v", r)
+	}
+}

pkg/obiiter/batchiterator.go

@@ -465,7 +465,9 @@ func (iterator IBioSequence) RebatchBySize(maxBytes int, maxCount int) IBioSeque
 
 	go func() {
 		order := 0
-		iterator = iterator.SortBatches()
+		// a local variable: reassigning the captured iterator would race with
+		// the IsPaired call below, made by the caller on the same variable
+		sorted := iterator.SortBatches()
 		buffer := obiseq.MakeBioSequenceSlice()
 		bufBytes := 0
 		source := ""
@@ -479,8 +481,8 @@ func (iterator IBioSequence) RebatchBySize(maxBytes int, maxCount int) IBioSeque
 			}
 		}
 
-		for iterator.Next() {
-			seqs := iterator.Get()
+		for sorted.Next() {
+			seqs := sorted.Get()
 			source = seqs.Source()
 			for _, s := range seqs.Slice() {
 				sz := s.MemorySize()
@@ -683,7 +685,8 @@ func (iterator IBioSequence) FilterOn(predicate obiseq.SequencePredicate,
 			}
 
 			seqs.slice = slice[:j]
-			trueIter.pointer.channel <- seqs
+			trueIter.Push(seqs)
+			// trueIter.pointer.channel <- seqs
 		}
 
 		trueIter.Done()

pkg/obiiter/distribute.go

@@ -57,34 +57,21 @@ func (dist *IDistribute) Classifier() *obiseq.BioSequenceClassifier {
 }
 
 // Distribute organizes the biosequences from the iterator into batches
-// based on the provided classifier and batch sizes. It returns an
-// IDistribute instance that manages the distribution of the sequences.
+// based on the provided classifier. It returns an IDistribute instance
+// that manages the distribution of the sequences.
 //
-// Parameters:
-//   - class: A pointer to a BioSequenceClassifier used to classify
-//     the biosequences during distribution.
-//   - sizes: Optional integer values specifying the batch size. If
-//     no sizes are provided, a default batch size of 5000 is used.
-//
-// Returns:
-// An IDistribute instance that contains the outputs of the
-// classified biosequences, a channel for new data notifications,
-// and the classifier used for distribution. The method operates
-// asynchronously, processing the sequences in separate goroutines.
-// It ensures that the outputs are closed and cleaned up once
-// processing is complete.
-func (iterator IBioSequence) Distribute(class *obiseq.BioSequenceClassifier, sizes ...int) IDistribute {
-	batchsize := obidefault.BatchSize()
+// Batches are flushed when either BatchSizeMax() sequences or BatchMem()
+// bytes are accumulated per key, mirroring the RebatchBySize strategy.
+func (iterator IBioSequence) Distribute(class *obiseq.BioSequenceClassifier) IDistribute {
+	maxCount := obidefault.BatchSizeMax()
+	maxBytes := obidefault.BatchMem()
 
 	outputs := make(map[int]IBioSequence, 100)
 	slices := make(map[int]*obiseq.BioSequenceSlice, 100)
+	bufBytes := make(map[int]int, 100)
 	orders := make(map[int]int, 100)
 	news := make(chan int)
 
-	if len(sizes) > 0 {
-		batchsize = sizes[0]
-	}
-
 	jobDone := sync.WaitGroup{}
 	lock := sync.Mutex{}
 
@@ -115,6 +102,7 @@ func (iterator IBioSequence) Distribute(class *obiseq.BioSequenceClassifier, siz
 					slice = &s
 					slices[key] = slice
 					orders[key] = 0
+					bufBytes[key] = 0
 
 					lock.Lock()
 					outputs[key] = MakeIBioSequence()
@@ -123,14 +111,20 @@ func (iterator IBioSequence) Distribute(class *obiseq.BioSequenceClassifier, siz
 					news <- key
 				}
 
-				*slice = append(*slice, s)
-
-				if len(*slice) == batchsize {
+				sz := s.MemorySize()
+				countFull := maxCount > 0 && len(*slice) >= maxCount
+				memFull := maxBytes > 0 && bufBytes[key]+sz > maxBytes && len(*slice) > 0
+				if countFull || memFull {
 					outputs[key].Push(MakeBioSequenceBatch(source, orders[key], *slice))
 					orders[key]++
 					s := obiseq.MakeBioSequenceSlice()
 					slices[key] = &s
+					slice = &s
+					bufBytes[key] = 0
 				}
+
+				*slice = append(*slice, s)
+				bufBytes[key] += sz
 			}
 		}
 

pkg/obikmer/debruijn.go

@@ -5,7 +5,6 @@ import (
 	"container/heap"
 	"fmt"
 	"math"
-	"math/bits"
 	"os"
 	"slices"
 
@@ -610,14 +609,6 @@ func (graph *DeBruijnGraph) WriteGml(filename string) error {
 	return err
 }
 
-// Calculating the hamming distance between two k-mers.
-func (g *DeBruijnGraph) HammingDistance(kmer1, kmer2 uint64) int {
-	ident := ^((kmer1 & kmer2) | (^kmer1 & ^kmer2))
-	ident |= (ident >> 1)
-	ident &= 0x5555555555555555 & g.kmermask
-	return bits.OnesCount64(ident)
-}
-
 type UInt64Heap []uint64
 
 func (h UInt64Heap) Len() int           { return len(h) }

pkg/obikmer/encodefourmer.go

@@ -47,7 +47,7 @@ func Encode4mer(seq *obiseq.BioSequence, buffer *[]byte) []byte {
 	length := slength - 3
 	rawseq := seq.Sequence()
 
-	if length < 0 {
+	if length <= 0 {
 		return nil
 	}
 

pkg/obikmer/entropy.go

@@ -4,22 +4,21 @@ import "math"
 
 // KmerEntropy computes the entropy of a single encoded k-mer.
 //
-// The algorithm mirrors the lowmask entropy calculation: it decodes the k-mer
+// The algorithm mirrors the Rust obiskbuilder entropy: it decodes the k-mer
 // to a DNA sequence, extracts all sub-words of each size from 1 to levelMax,
 // normalizes them by circular canonical form, counts their frequencies, and
-// computes Shannon entropy normalized by the maximum possible entropy.
+// computes Shannon entropy corrected for class sizes, normalized by the
+// maximum possible entropy over 4^ws raw bins.
 // The returned value is the minimum entropy across all word sizes.
 //
+// Correction for small sequences: the raw entropy H = log(N) - Σ f·log(f)/N
+// under-estimates the true complexity when many raw words collapse to the same
+// canonical form.  Adding Σ f·log(class_size)/N recovers the entropy of the
+// underlying uncollapsed distribution (assuming uniform mixing within each
+// equivalence class).
+//
 // A value close to 0 indicates very low complexity (e.g. "AAAA..."),
 // while a value close to 1 indicates high complexity.
-//
-// Parameters:
-//   - kmer: the encoded k-mer (2 bits per base)
-//   - k: the k-mer size
-//   - levelMax: maximum sub-word size for entropy (typically 6)
-//
-// Returns:
-//   - minimum normalized entropy across all word sizes 1..levelMax
 func KmerEntropy(kmer uint64, k int, levelMax int) float64 {
 	if k < 1 || levelMax < 1 {
 		return 1.0
@@ -35,7 +34,7 @@ func KmerEntropy(kmer uint64, k int, levelMax int) float64 {
 	var seqBuf [32]byte
 	seq := DecodeKmer(kmer, k, seqBuf[:])
 
-	// Pre-compute nLogN lookup (same as lowmask)
+	// Pre-compute nLogN lookup
 	nLogN := make([]float64, k+1)
 	for i := 1; i <= k; i++ {
 		nLogN[i] = float64(i) * math.Log(float64(i))
@@ -51,6 +50,23 @@ func KmerEntropy(kmer uint64, k int, levelMax int) float64 {
 		}
 	}
 
+	// Build ln(class_size) tables: for each canonical form, how many raw
+	// words map to it under circular normalization.
+	classLogSizeTables := make([][]float64, levelMax+1)
+	for ws := 1; ws <= levelMax; ws++ {
+		tableSize := 1 << (ws * 2)
+		classSize := make([]int, tableSize)
+		for code := 0; code < tableSize; code++ {
+			classSize[normTables[ws][code]]++
+		}
+		classLogSizeTables[ws] = make([]float64, tableSize)
+		for j := 0; j < tableSize; j++ {
+			if classSize[j] > 0 {
+				classLogSizeTables[ws][j] = math.Log(float64(classSize[j]))
+			}
+		}
+	}
+
 	minEntropy := math.MaxFloat64
 
 	for ws := 1; ws <= levelMax; ws++ {
@@ -75,23 +91,13 @@ func KmerEntropy(kmer uint64, k int, levelMax int) float64 {
 			table[normWord]++
 		}
 
-		// Compute Shannon entropy
+		// Compute emax over 4^ws raw bins (uncollapsed distribution).
 		floatNwords := float64(nwords)
 		logNwords := math.Log(floatNwords)
-
-		var sumNLogN float64
-		for j := 0; j < tableSize; j++ {
-			n := table[j]
-			if n > 0 {
-				sumNLogN += nLogN[n]
-			}
-		}
-
-		// Compute emax (maximum possible entropy for this word size)
-		na := CanonicalCircularKmerCount(ws)
+		na := tableSize // 4^ws
 		var emax float64
 		if nwords < na {
-			emax = math.Log(float64(nwords))
+			emax = logNwords
 		} else {
 			cov := nwords / na
 			remains := nwords - (na * cov)
@@ -105,7 +111,19 @@ func KmerEntropy(kmer uint64, k int, levelMax int) float64 {
 			continue
 		}
 
-		entropy := (logNwords - sumNLogN/floatNwords) / emax
+		// Accumulate Σ f·log(f) and Σ f·log(class_size) over canonical forms.
+		classLogSize := classLogSizeTables[ws]
+		var sumNLogN, sumClassLogN float64
+		for j := 0; j < tableSize; j++ {
+			n := table[j]
+			if n > 0 {
+				sumNLogN += nLogN[n]
+				sumClassLogN += float64(n) * classLogSize[j]
+			}
+		}
+
+		// Corrected entropy: H_raw ≈ log(N) + (Σf·log(s) - Σf·log(f)) / N
+		entropy := (logNwords + sumClassLogN/floatNwords - sumNLogN/floatNwords) / emax
 		if entropy < 0 {
 			entropy = 0
 		}
@@ -129,24 +147,20 @@ func KmerEntropy(kmer uint64, k int, levelMax int) float64 {
 // IMPORTANT: a KmerEntropyFilter is NOT safe for concurrent use.
 // Each goroutine must create its own instance via NewKmerEntropyFilter.
 type KmerEntropyFilter struct {
-	k          int
-	levelMax   int
-	threshold  float64
-	nLogN      []float64
-	normTables [][]int
-	emaxValues []float64
-	logNwords  []float64
+	k                   int
+	levelMax            int
+	threshold           float64
+	nLogN               []float64
+	normTables          [][]int
+	classLogSizeTables  [][]float64
+	emaxValues          []float64
+	logNwords           []float64
 	// Pre-allocated frequency tables reused across Entropy() calls.
 	// One per word size (index 0 unused). Reset to zero before each use.
 	freqTables [][]int
 }
 
 // NewKmerEntropyFilter creates an entropy filter with pre-computed tables.
-//
-// Parameters:
-//   - k: the k-mer size
-//   - levelMax: maximum sub-word size for entropy (typically 6)
-//   - threshold: entropy threshold (k-mers with entropy <= threshold are rejected)
 func NewKmerEntropyFilter(k, levelMax int, threshold float64) *KmerEntropyFilter {
 	if levelMax >= k {
 		levelMax = k - 1
@@ -169,20 +183,38 @@ func NewKmerEntropyFilter(k, levelMax int, threshold float64) *KmerEntropyFilter
 		}
 	}
 
+	// ln(class_size) for each canonical form under circular normalization.
+	classLogSizeTables := make([][]float64, levelMax+1)
+	for ws := 1; ws <= levelMax; ws++ {
+		tableSize := 1 << (ws * 2)
+		classSize := make([]int, tableSize)
+		for code := 0; code < tableSize; code++ {
+			classSize[normTables[ws][code]]++
+		}
+		classLogSizeTables[ws] = make([]float64, tableSize)
+		for j := 0; j < tableSize; j++ {
+			if classSize[j] > 0 {
+				classLogSizeTables[ws][j] = math.Log(float64(classSize[j]))
+			}
+		}
+	}
+
+	// Pre-compute emax and logNwords per word size.
+	// emax uses 4^ws raw bins to match the corrected entropy.
 	emaxValues := make([]float64, levelMax+1)
 	logNwords := make([]float64, levelMax+1)
 	for ws := 1; ws <= levelMax; ws++ {
 		nw := k - ws + 1
-		na := CanonicalCircularKmerCount(ws)
+		na := 1 << (ws * 2) // 4^ws raw bins
+		floatNw := float64(nw)
+		logNwords[ws] = math.Log(floatNw)
 		if nw < na {
-			logNwords[ws] = math.Log(float64(nw))
-			emaxValues[ws] = math.Log(float64(nw))
+			emaxValues[ws] = logNwords[ws]
 		} else {
 			cov := nw / na
 			remains := nw - (na * cov)
-			f1 := float64(cov) / float64(nw)
-			f2 := float64(cov+1) / float64(nw)
-			logNwords[ws] = math.Log(float64(nw))
+			f1 := float64(cov) / floatNw
+			f2 := float64(cov+1) / floatNw
 			emaxValues[ws] = -(float64(na-remains)*f1*math.Log(f1) +
 				float64(remains)*f2*math.Log(f2))
 		}
@@ -195,14 +227,15 @@ func NewKmerEntropyFilter(k, levelMax int, threshold float64) *KmerEntropyFilter
 	}
 
 	return &KmerEntropyFilter{
-		k:          k,
-		levelMax:   levelMax,
-		threshold:  threshold,
-		nLogN:      nLogN,
-		normTables: normTables,
-		emaxValues: emaxValues,
-		logNwords:  logNwords,
-		freqTables: freqTables,
+		k:                  k,
+		levelMax:           levelMax,
+		threshold:          threshold,
+		nLogN:              nLogN,
+		normTables:         normTables,
+		classLogSizeTables: classLogSizeTables,
+		emaxValues:         emaxValues,
+		logNwords:          logNwords,
+		freqTables:         freqTables,
 	}
 }
 
@@ -236,7 +269,7 @@ func (ef *KmerEntropyFilter) Entropy(kmer uint64) float64 {
 		// Count circular-canonical sub-word frequencies
 		tableSize := 1 << (ws * 2)
 		table := ef.freqTables[ws]
-		clear(table) // reset to zero
+		clear(table)
 		mask := (1 << (ws * 2)) - 1
 		normTable := ef.normTables[ws]
 
@@ -251,19 +284,21 @@ func (ef *KmerEntropyFilter) Entropy(kmer uint64) float64 {
 			table[normWord]++
 		}
 
-		// Compute Shannon entropy
 		floatNwords := float64(nwords)
 		logNwords := ef.logNwords[ws]
+		classLogSize := ef.classLogSizeTables[ws]
 
-		var sumNLogN float64
+		var sumNLogN, sumClassLogN float64
 		for j := 0; j < tableSize; j++ {
 			n := table[j]
 			if n > 0 {
 				sumNLogN += ef.nLogN[n]
+				sumClassLogN += float64(n) * classLogSize[j]
 			}
 		}
 
-		entropy := (logNwords - sumNLogN/floatNwords) / emax
+		// Corrected entropy: H_raw ≈ log(N) + (Σf·log(s) - Σf·log(f)) / N
+		entropy := (logNwords + sumClassLogN/floatNwords - sumNLogN/floatNwords) / emax
 		if entropy < 0 {
 			entropy = 0
 		}

pkg/obikmer/enumkmer.go

@@ -0,0 +1,129 @@
+package obikmer
+
+import "iter"
+
+// KmerConstraints describes lexicographic properties a k-mer must satisfy.
+//
+// MaxCount is indexed by the 2-bit base code (A=0, C=1, G=2, T=3):
+// a negative value means unlimited, 0 forbids the base altogether
+// (so MaxCount[2] = 0 restricts the alphabet to A, C, T).
+//
+// MaxGC is the maximum number of C or G bases; a negative value means
+// unlimited.
+//
+// MaxRun is the maximum number of consecutive identical bases;
+// a value <= 0 means unlimited. MinRun requires at least one run of that
+// length; a value <= 1 means no requirement.
+//
+// Accepted and Rejected are optional per-position sets of bases (4 bits masks,
+// bit b set when the base of code b is in the set, see ParseIUPACMasks).
+// When set, their length must be k. A k-mer is accepted only if each of its
+// bases belongs to the Accepted set of its position, and is refused if every
+// one of its bases belongs to the Rejected set of its position.
+//
+// The zero value forbids every base: use NoKmerConstraints as a starting point.
+type KmerConstraints struct {
+	MaxCount [4]int
+	MaxGC    int
+	MaxRun   int
+	MinRun   int
+	Accepted []uint8
+	Rejected []uint8
+}
+
+// NoKmerConstraints returns constraints accepting every k-mer.
+func NoKmerConstraints() KmerConstraints {
+	return KmerConstraints{MaxCount: [4]int{-1, -1, -1, -1}, MaxGC: -1}
+}
+
+// IterConstrainedKmers returns an iterator over all k-mers of size k
+// satisfying the constraints c, encoded as in EncodeKmer, in increasing
+// numerical order.
+//
+// K-mers are built base by base and a branch is abandoned as soon as a
+// constraint is violated or cannot be met any more (MaxCount, MaxGC, MaxRun
+// and Accepted are monotone: a violating prefix cannot be repaired by
+// extension; MinRun is checked against the positions left), so the cost is
+// proportional to the number of explored prefixes, not to 4^k. Rejected is
+// checked on complete k-mers.
+//
+// k must be between 1 and 31, otherwise the iterator is empty. It is also
+// empty if Accepted or Rejected is set with a length different from k.
+//
+// Example:
+//
+//	c := NoKmerConstraints()
+//	c.MaxCount[2] = 0 // no G
+//	c.MaxRun = 3
+//	for kmer := range IterConstrainedKmers(8, c) { ... }
+func IterConstrainedKmers(k int, c KmerConstraints) iter.Seq[uint64] {
+	return func(yield func(uint64) bool) {
+		if k < 1 || k > 31 || !c.validPositions(k) {
+			return
+		}
+
+		// Nothing to enumerate if the per-base caps cannot even sum to k.
+		capacity := 0
+		for _, m := range c.MaxCount {
+			if m < 0 || m >= k {
+				capacity += k
+			} else {
+				capacity += m
+			}
+		}
+		if capacity < k {
+			return
+		}
+
+		var counts [4]int
+		gc := 0
+
+		var walk func(depth int, kmer uint64, last, run, longest int) bool
+		walk = func(depth int, kmer uint64, last, run, longest int) bool {
+			if depth == k {
+				if c.Rejected != nil && matchesAllMasks(kmer, k, c.Rejected) {
+					return true
+				}
+				return yield(kmer)
+			}
+			for b := range 4 {
+				if c.MaxCount[b] >= 0 && counts[b] >= c.MaxCount[b] {
+					continue
+				}
+				if c.Accepted != nil && c.Accepted[depth]>>uint(b)&1 == 0 {
+					continue
+				}
+				isGC := b == 1 || b == 2
+				if isGC && c.MaxGC >= 0 && gc >= c.MaxGC {
+					continue
+				}
+				r := 1
+				if b == last {
+					r = run + 1
+				}
+				if c.MaxRun > 0 && r > c.MaxRun {
+					continue
+				}
+				l := max(longest, r)
+				if c.MinRun > 1 && l < c.MinRun && r+(k-depth-1) < c.MinRun {
+					continue
+				}
+				counts[b]++
+				if isGC {
+					gc++
+				}
+				ok := walk(depth+1, kmer<<2|uint64(b), b, r, l)
+				counts[b]--
+				if isGC {
+					gc--
+				}
+				if !ok {
+					return false
+				}
+			}
+			return true
+		}
+
+		walk(0, 0, -1, 0, 0)
+	}
+}

pkg/obikmer/enumkmer_test.go

@@ -0,0 +1,85 @@
+package obikmer
+
+import (
+	"strings"
+	"testing"
+)
+
+func collectConstrained(k int, c KmerConstraints) []uint64 {
+	var out []uint64
+	for km := range IterConstrainedKmers(k, c) {
+		out = append(out, km)
+	}
+	return out
+}
+
+func TestIterConstrainedKmersUnconstrained(t *testing.T) {
+	got := collectConstrained(4, NoKmerConstraints())
+	if len(got) != 256 {
+		t.Fatalf("expected 256 k-mers, got %d", len(got))
+	}
+	for i, km := range got {
+		if km != uint64(i) {
+			t.Fatalf("not in increasing order at %d: %d", i, km)
+		}
+	}
+}
+
+func TestIterConstrainedKmersNoG(t *testing.T) {
+	c := NoKmerConstraints()
+	c.MaxCount[2] = 0
+	got := collectConstrained(5, c)
+	if len(got) != 243 {
+		t.Fatalf("expected 3^5 = 243 k-mers, got %d", len(got))
+	}
+	for _, km := range got {
+		if strings.Contains(string(DecodeKmer(km, 5, nil)), "g") {
+			t.Fatalf("k-mer %d contains G", km)
+		}
+	}
+}
+
+func TestIterConstrainedKmersMaxRun(t *testing.T) {
+	c := NoKmerConstraints()
+	c.MaxRun = 2
+	// a(n) = 3*(a(n-1)+a(n-2)), a(1)=4, a(2)=16
+	want := map[int]int{3: 60, 4: 228}
+	for k, n := range want {
+		if got := len(collectConstrained(k, c)); got != n {
+			t.Fatalf("k=%d: expected %d, got %d", k, n, got)
+		}
+	}
+}
+
+func TestIterConstrainedKmersMaxCount(t *testing.T) {
+	c := NoKmerConstraints()
+	c.MaxCount[0] = 1 // at most one A
+	for _, km := range collectConstrained(6, c) {
+		if strings.Count(string(DecodeKmer(km, 6, nil)), "a") > 1 {
+			t.Fatalf("k-mer %d has more than one A", km)
+		}
+	}
+}
+
+func TestIterConstrainedKmersImpossible(t *testing.T) {
+	c := KmerConstraints{MaxCount: [4]int{1, 1, 1, 0}}
+	if got := collectConstrained(4, c); len(got) != 0 {
+		t.Fatalf("expected no k-mer, got %d", len(got))
+	}
+	if got := collectConstrained(0, NoKmerConstraints()); len(got) != 0 {
+		t.Fatalf("expected no k-mer for k=0")
+	}
+}
+
+func TestIterConstrainedKmersEarlyStop(t *testing.T) {
+	n := 0
+	for range IterConstrainedKmers(10, NoKmerConstraints()) {
+		n++
+		if n == 5 {
+			break
+		}
+	}
+	if n != 5 {
+		t.Fatalf("early stop failed: %d", n)
+	}
+}

pkg/obikmer/kmer_constraints.go

@@ -0,0 +1,141 @@
+package obikmer
+
+import (
+	"fmt"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+// ParseIUPACMasks converts an IUPAC nucleotide pattern into one 4 bits mask per
+// position (bit 0 is A, bit 1 is C, bit 2 is G, bit 3 is T), the format used by
+// KmerConstraints.Accepted and Rejected.
+func ParseIUPACMasks(pattern string) ([]uint8, error) {
+	masks := make([]uint8, len(pattern))
+	for i := 0; i < len(pattern); i++ {
+		m := obiseq.IUPACNucMask(pattern[i])
+		if m == 0 {
+			return nil, fmt.Errorf("invalid IUPAC nucleotide code %q at position %d of pattern %q",
+				pattern[i], i+1, pattern)
+		}
+		masks[i] = m
+	}
+	return masks, nil
+}
+
+// RestrictAlphabet forbids every base that is not denoted by one of the IUPAC
+// codes of alphabet. For instance "ACT" (or "H") forbids G.
+func (c *KmerConstraints) RestrictAlphabet(alphabet string) error {
+	masks, err := ParseIUPACMasks(alphabet)
+	if err != nil {
+		return err
+	}
+	var allowed uint8
+	for _, m := range masks {
+		allowed |= m
+	}
+	if allowed == 0 {
+		return fmt.Errorf("empty alphabet")
+	}
+	for b := range 4 {
+		if allowed>>uint(b)&1 == 0 {
+			c.MaxCount[b] = 0
+		}
+	}
+	return nil
+}
+
+// validPositions tells whether Accepted and Rejected, when set, have length k.
+func (c *KmerConstraints) validPositions(k int) bool {
+	return (c.Accepted == nil || len(c.Accepted) == k) &&
+		(c.Rejected == nil || len(c.Rejected) == k)
+}
+
+// matchesAllMasks tells whether every base of the k-mer belongs to the set
+// given for its position.
+func matchesAllMasks(kmer uint64, k int, masks []uint8) bool {
+	for i := k - 1; i >= 0; i-- {
+		if masks[i]>>(kmer&3)&1 == 0 {
+			return false
+		}
+		kmer >>= 2
+	}
+	return true
+}
+
+// Accepts tells whether a k-mer of size k satisfies every constraint.
+// It is the predicate whose solutions IterConstrainedKmers enumerates, to be
+// used to filter k-mers coming from elsewhere.
+func (c *KmerConstraints) Accepts(kmer uint64, k int) bool {
+	if k < 1 || k > 31 || !c.validPositions(k) {
+		return false
+	}
+
+	var counts [4]int
+	gc, run, longest, last := 0, 0, 0, -1
+	for i := k - 1; i >= 0; i-- {
+		b := int(kmer>>(2*uint(i))) & 3
+		if c.Accepted != nil && c.Accepted[k-1-i]>>uint(b)&1 == 0 {
+			return false
+		}
+		counts[b]++
+		if c.MaxCount[b] >= 0 && counts[b] > c.MaxCount[b] {
+			return false
+		}
+		if b == 1 || b == 2 {
+			gc++
+			if c.MaxGC >= 0 && gc > c.MaxGC {
+				return false
+			}
+		}
+		if b == last {
+			run++
+		} else {
+			run = 1
+		}
+		last = b
+		longest = max(longest, run)
+		if c.MaxRun > 0 && run > c.MaxRun {
+			return false
+		}
+	}
+
+	if c.MinRun > 1 && longest < c.MinRun {
+		return false
+	}
+	if c.Rejected != nil && matchesAllMasks(kmer, k, c.Rejected) {
+		return false
+	}
+	return true
+}
+
+// HammingBound returns the sphere-packing (Hamming) upper bound on the number
+// of words of length k over an alphabet of alphabetSize letters that can be
+// pairwise at Hamming distance >= d. No larger family exists; a family of
+// that size may not exist either.
+func HammingBound(alphabetSize, k, d int) float64 {
+	if d <= 1 {
+		total := 1.0
+		for i := 0; i < k; i++ {
+			total *= float64(alphabetSize)
+		}
+		return total
+	}
+	if d%2 == 0 { // A(k, d) <= A(k-1, d-1)
+		k--
+		d--
+	}
+	if k < 1 {
+		return 1
+	}
+	t := (d - 1) / 2
+	total, ball, binom, pow := 1.0, 1.0, 1.0, 1.0
+	for i := 0; i < k; i++ {
+		total *= float64(alphabetSize)
+	}
+	for i := 1; i <= t && i <= k; i++ {
+		binom = binom * float64(k-i+1) / float64(i)
+		pow *= float64(alphabetSize - 1)
+		ball += binom * pow
+	}
+	return total / ball
+}

pkg/obikmer/kmer_constraints_test.go

@@ -0,0 +1,123 @@
+package obikmer
+
+import (
+	"math"
+	"math/rand"
+	"testing"
+)
+
+// IterConstrainedKmers must enumerate exactly the k-mers accepted by Accepts.
+func TestIterConstrainedKmersMatchesAccepts(t *testing.T) {
+	rng := rand.New(rand.NewSource(10))
+	patterns := []string{"", "nnnnnn", "yrnnnn", "cnnnnn", "annnnt"}
+
+	for trial := 0; trial < 60; trial++ {
+		k := 6
+		c := NoKmerConstraints()
+		for b := range 4 {
+			if rng.Intn(3) == 0 {
+				c.MaxCount[b] = rng.Intn(4)
+			}
+		}
+		if rng.Intn(2) == 0 {
+			c.MaxGC = rng.Intn(5)
+		}
+		if rng.Intn(2) == 0 {
+			c.MaxRun = 1 + rng.Intn(3)
+		}
+		if rng.Intn(2) == 0 {
+			c.MinRun = 2 + rng.Intn(3)
+		}
+		if p := patterns[rng.Intn(len(patterns))]; p != "" {
+			c.Accepted, _ = ParseIUPACMasks(p)
+		}
+		if p := patterns[rng.Intn(len(patterns))]; p != "" && rng.Intn(2) == 0 {
+			c.Rejected, _ = ParseIUPACMasks(p)
+		}
+
+		var want []uint64
+		for km := uint64(0); km < 1<<(2*uint(k)); km++ {
+			if c.Accepts(km, k) {
+				want = append(want, km)
+			}
+		}
+		got := collectConstrained(k, c)
+		if len(got) != len(want) {
+			t.Fatalf("trial %d %+v: got %d k-mers, want %d", trial, c, len(got), len(want))
+		}
+		for i := range got {
+			if got[i] != want[i] {
+				t.Fatalf("trial %d %+v: mismatch at %d: %d != %d", trial, c, i, got[i], want[i])
+			}
+		}
+	}
+}
+
+func TestConstraintsGCAndRuns(t *testing.T) {
+	enc := func(s string) uint64 { return EncodeKmer([]byte(s), len(s)) }
+
+	c := NoKmerConstraints()
+	c.MaxGC = 2
+	if !c.Accepts(enc("ACGTAA"), 6) || c.Accepts(enc("ACGCAA"), 6) {
+		t.Fatal("MaxGC not enforced")
+	}
+
+	c = NoKmerConstraints()
+	c.MinRun = 3
+	if !c.Accepts(enc("ACCCGT"), 6) || c.Accepts(enc("ACCGGT"), 6) {
+		t.Fatal("MinRun not enforced")
+	}
+
+	c = NoKmerConstraints()
+	c.Rejected, _ = ParseIUPACMasks("cnnnnn") // no C in first position
+	if c.Accepts(enc("CAAAAA"), 6) || !c.Accepts(enc("AAAAAA"), 6) {
+		t.Fatal("Rejected not enforced")
+	}
+}
+
+func TestRestrictAlphabetAndPatterns(t *testing.T) {
+	c := NoKmerConstraints()
+	if err := c.RestrictAlphabet("ACT"); err != nil {
+		t.Fatal(err)
+	}
+	if got := len(collectConstrained(4, c)); got != 81 {
+		t.Fatalf("expected 3^4 = 81 k-mers, got %d", got)
+	}
+	c = NoKmerConstraints()
+	if err := c.RestrictAlphabet("H"); err != nil { // H = A, C or T
+		t.Fatal(err)
+	}
+	if c.MaxCount != [4]int{-1, -1, 0, -1} {
+		t.Fatalf("unexpected counts %v", c.MaxCount)
+	}
+	if err := c.RestrictAlphabet("AX"); err == nil {
+		t.Fatal("invalid IUPAC code must be refused")
+	}
+	if _, err := ParseIUPACMasks("acgz"); err == nil {
+		t.Fatal("invalid IUPAC code must be refused")
+	}
+	// wrong pattern length
+	c = NoKmerConstraints()
+	c.Accepted, _ = ParseIUPACMasks("nnn")
+	if got := collectConstrained(4, c); len(got) != 0 {
+		t.Fatalf("pattern of wrong length must give nothing, got %d", len(got))
+	}
+}
+
+func TestHammingBound(t *testing.T) {
+	cases := []struct {
+		q, k, d int
+		want    float64
+	}{
+		{4, 7, 3, 16384.0 / 22},
+		{4, 8, 4, 16384.0 / 22}, // d even: reduces to (7, 3)
+		{4, 9, 5, 262144.0 / 352},
+		{3, 9, 3, 19683.0 / 19},
+		{4, 5, 1, 1024},
+	}
+	for _, c := range cases {
+		if got := HammingBound(c.q, c.k, c.d); math.Abs(got-c.want) > 1e-6 {
+			t.Errorf("HammingBound(%d,%d,%d) = %v, want %v", c.q, c.k, c.d, got, c.want)
+		}
+	}
+}

pkg/obikmer/kmer_dissimilarity.go

@@ -0,0 +1,84 @@
+package obikmer
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+)
+
+// KmerMetric selects the distance used to compare two k-mers.
+type KmerMetric int
+
+const (
+	// HammingMetric counts the positions at which two k-mers differ.
+	HammingMetric KmerMetric = iota
+	// LCSMetric is the alignment length minus the LCS length (see KmerLCS).
+	LCSMetric
+)
+
+// ParseKmerMetric converts "hamming" or "lcs" (case insensitive) to a KmerMetric.
+func ParseKmerMetric(name string) (KmerMetric, error) {
+	switch strings.ToLower(name) {
+	case "hamming":
+		return HammingMetric, nil
+	case "lcs":
+		return LCSMetric, nil
+	}
+	return HammingMetric, fmt.Errorf("unknown distance type %q (expected hamming or lcs)", name)
+}
+
+func (m KmerMetric) String() string {
+	if m == LCSMetric {
+		return "lcs"
+	}
+	return "hamming"
+}
+
+type lcsScratch struct {
+	partners []uint64
+	results  []KmerLCSResult
+}
+
+// NewKmerDissimilarityFilter describes the graph whose vertices are kmers[i]
+// (k-mers of size k, 1 <= k <= 31) and where two vertices are linked when
+// their distance is at least dmin. The returned function has the signature of
+// obigraph.NeighborFilter: it appends to out[:0] the candidates (indices in
+// kmers) linked to v. It is safe for concurrent use.
+func NewKmerDissimilarityFilter(kmers []uint64, k int, metric KmerMetric, dmin int) func(v int32, candidates, out []int32) []int32 {
+	if metric == HammingMetric {
+		return func(v int32, candidates, out []int32) []int32 {
+			q := kmers[v]
+			out = out[:0]
+			for _, c := range candidates {
+				if HammingDistance(q, kmers[c], k) >= dmin {
+					out = append(out, c)
+				}
+			}
+			return out
+		}
+	}
+
+	pool := sync.Pool{New: func() any { return new(lcsScratch) }}
+	return func(v int32, candidates, out []int32) []int32 {
+		s := pool.Get().(*lcsScratch)
+		defer pool.Put(s)
+
+		s.partners = s.partners[:0]
+		for _, c := range candidates {
+			s.partners = append(s.partners, kmers[c])
+		}
+		if cap(s.results) < len(candidates) {
+			s.results = make([]KmerLCSResult, len(candidates))
+		}
+		s.results = s.results[:len(candidates)]
+		KmerLCSBatch(kmers[v], s.partners, k, s.results)
+
+		out = out[:0]
+		for i, c := range candidates {
+			if s.results[i].Distance() >= dmin {
+				out = append(out, c)
+			}
+		}
+		return out
+	}
+}

pkg/obikmer/kmer_dissimilarity_test.go

@@ -0,0 +1,139 @@
+package obikmer_test
+
+import (
+	"context"
+	"math/rand"
+	"testing"
+	"time"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obigraph"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obikmer"
+)
+
+func allKmers(k int) []uint64 {
+	kmers := make([]uint64, 0, 1<<(2*uint(k)))
+	for km := range obikmer.IterConstrainedKmers(k, obikmer.NoKmerConstraints()) {
+		kmers = append(kmers, km)
+	}
+	return kmers
+}
+
+func TestDissimilarityFilterMatchesDistances(t *testing.T) {
+	const k = 7
+	rng := rand.New(rand.NewSource(1))
+	kmers := make([]uint64, 300)
+	for i := range kmers {
+		kmers[i] = rng.Uint64() & (1<<(2*k) - 1)
+	}
+	candidates := make([]int32, 0, len(kmers))
+	for i := 1; i < len(kmers); i++ {
+		candidates = append(candidates, int32(i))
+	}
+
+	for _, metric := range []obikmer.KmerMetric{obikmer.HammingMetric, obikmer.LCSMetric} {
+		for _, dmin := range []int{1, 3, 5} {
+			filter := obikmer.NewKmerDissimilarityFilter(kmers, k, metric, dmin)
+			got := filter(0, candidates, nil)
+
+			var want []int32
+			for _, c := range candidates {
+				var d int
+				if metric == obikmer.HammingMetric {
+					d = obikmer.HammingDistance(kmers[0], kmers[c], k)
+				} else {
+					d = obikmer.KmerLCS(kmers[0], kmers[c], k).Distance()
+				}
+				if d >= dmin {
+					want = append(want, c)
+				}
+			}
+			if len(got) != len(want) {
+				t.Fatalf("%v d>=%d: got %d vertices, want %d", metric, dmin, len(got), len(want))
+			}
+			for i := range got {
+				if got[i] != want[i] {
+					t.Fatalf("%v d>=%d: mismatch at %d", metric, dmin, i)
+				}
+			}
+		}
+	}
+}
+
+// A family of 5-mers pairwise at Hamming distance >= 3, built with FindClique.
+func TestFindClique_HammingFamily(t *testing.T) {
+	const k, dmin, target = 5, 3, 32
+	kmers := allKmers(k)
+	filter := obikmer.NewKmerDissimilarityFilter(kmers, k, obikmer.HammingMetric, dmin)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	r := obigraph.FindClique(ctx, len(kmers), filter, target, obigraph.CliqueOptions{Workers: 2})
+	if !r.Reached {
+		t.Fatalf("family of %d not found (best %d, exhausted %v)", target, len(r.Clique), r.Exhausted)
+	}
+	for i, a := range r.Clique {
+		for _, b := range r.Clique[i+1:] {
+			if obikmer.HammingDistance(kmers[a], kmers[b], k) < dmin {
+				t.Fatalf("%d and %d are too close", kmers[a], kmers[b])
+			}
+		}
+	}
+}
+
+func TestFindClique_LCSFamily(t *testing.T) {
+	const k, dmin, target = 6, 3, 20
+	kmers := allKmers(k)
+	filter := obikmer.NewKmerDissimilarityFilter(kmers, k, obikmer.LCSMetric, dmin)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+	r := obigraph.FindClique(ctx, len(kmers), filter, target, obigraph.CliqueOptions{Workers: 2})
+	if !r.Reached {
+		t.Fatalf("family of %d not found (best %d, exhausted %v)", target, len(r.Clique), r.Exhausted)
+	}
+	for i, a := range r.Clique {
+		for _, b := range r.Clique[i+1:] {
+			if obikmer.KmerLCS(kmers[a], kmers[b], k).Distance() < dmin {
+				t.Fatalf("%d and %d are too close", kmers[a], kmers[b])
+			}
+		}
+	}
+}
+
+func TestParseKmerMetric(t *testing.T) {
+	if m, err := obikmer.ParseKmerMetric("LCS"); err != nil || m != obikmer.LCSMetric {
+		t.Fatal("lcs not parsed")
+	}
+	if m, err := obikmer.ParseKmerMetric("hamming"); err != nil || m != obikmer.HammingMetric {
+		t.Fatal("hamming not parsed")
+	}
+	if _, err := obikmer.ParseKmerMetric("levenshtein"); err == nil {
+		t.Fatal("unknown metric must be refused")
+	}
+}
+
+// With a target out of reach (17 4-mers at Hamming distance >= 3 do not exist,
+// 16 do), the search must return the best family it can build, not a poor one
+// stuck by the pruning on the target, and the coloring bound must prove that
+// the target is out of reach.
+func TestFindClique_BestWhenTargetOutOfReach(t *testing.T) {
+	const k, dmin = 4, 3
+	kmers := allKmers(k)
+	filter := obikmer.NewKmerDissimilarityFilter(kmers, k, obikmer.HammingMetric, dmin)
+
+	for _, limit := range []int{0, 1024} {
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+		r := obigraph.FindClique(ctx, len(kmers), filter, 17, obigraph.CliqueOptions{Workers: 2, ColorLimit: limit})
+		cancel()
+		if r.Reached {
+			t.Fatal("17 4-mers at Hamming distance >= 3 cannot exist (sphere-packing bound 19, true maximum 16)")
+		}
+		if len(r.Clique) < 16 {
+			t.Fatalf("limit %d: expected the family of 16 to be found, got %d (exhausted %v, maximum %v)",
+				limit, len(r.Clique), r.Exhausted, r.Maximum)
+		}
+		if limit > 0 && !r.Exhausted {
+			t.Fatalf("limit %d: the coloring bound should prove that 17 is out of reach", limit)
+		}
+	}
+}

pkg/obikmer/kmer_distance.go

@@ -0,0 +1,136 @@
+package obikmer
+
+import "math/bits"
+
+// KmerLCSResult is the result of a k-mer LCS alignment: the length of the
+// Longest Common Subsequence and the length of the shortest alignment
+// achieving it (same semantics as obialign.FastLCSScore).
+type KmerLCSResult struct {
+	LCS      int
+	AlignLen int
+}
+
+// Distance returns AlignLen - LCS, the LCS-mode edit distance.
+func (r KmerLCSResult) Distance() int {
+	return r.AlignLen - r.LCS
+}
+
+// HammingDistance returns the number of positions at which the two k-mers
+// differ. Both k-mers are encoded with 2 bits per base (see EncodeKmer);
+// bits above the 2*k lowest ones are ignored. k must be between 0 and 32.
+func HammingDistance(a, b uint64, k int) int {
+	diff := a ^ b
+	diff = (diff | diff>>1) & 0x5555555555555555
+	diff &= ^uint64(0) >> (64 - 2*k)
+	return bits.OnesCount64(diff)
+}
+
+// KmerLCS aligns two k-mers of the same size in LCS mode (match +1, mismatch
+// and indel 0) and returns the LCS length together with the length of the
+// shortest alignment among those reaching it, so that the distance is
+// AlignLen - LCS.
+//
+// It is the scalar reference implementation, straightforward on purpose:
+// it is the oracle against which the vectorised versions are tested.
+// It returns {-1, -1} if k is not between 1 and 31.
+func KmerLCS(a, b uint64, k int) KmerLCSResult {
+	if k < 1 || k > 31 {
+		return KmerLCSResult{-1, -1}
+	}
+
+	var ba, bb [32]uint8
+	for i := k - 1; i >= 0; i-- {
+		ba[i] = uint8(a & 3)
+		bb[i] = uint8(b & 3)
+		a >>= 2
+		b >>= 2
+	}
+
+	// One DP row: best (score, alignment length) per column.
+	var score, length [32]int
+	for j := 0; j <= k; j++ {
+		length[j] = j
+	}
+
+	better := func(s1, l1, s2, l2 int) bool {
+		return s1 > s2 || (s1 == s2 && l1 < l2)
+	}
+
+	for i := 1; i <= k; i++ {
+		diagS, diagL := score[0], length[0]
+		score[0], length[0] = 0, i
+
+		for j := 1; j <= k; j++ {
+			upS, upL := score[j], length[j]
+
+			bestS, bestL := diagS, diagL+1
+			if ba[j-1] == bb[i-1] {
+				bestS++
+			}
+			if better(upS, upL+1, bestS, bestL) {
+				bestS, bestL = upS, upL+1
+			}
+			if better(score[j-1], length[j-1]+1, bestS, bestL) {
+				bestS, bestL = score[j-1], length[j-1]+1
+			}
+
+			diagS, diagL = upS, upL
+			score[j], length[j] = bestS, bestL
+		}
+	}
+
+	return KmerLCSResult{score[k], length[k]}
+}
+
+// KmerLCSBatch computes KmerLCS(query, partners[i], k) for every partner and
+// stores it in out[i]. out must be at least as long as partners.
+//
+// When built with GOEXPERIMENT=simd the partners are aligned several at a
+// time, one per SIMD lane; otherwise the scalar reference is used.
+// It returns false, leaving out untouched, if k is not between 1 and 31 or
+// if out is too short.
+func KmerLCSBatch(query uint64, partners []uint64, k int, out []KmerLCSResult) bool {
+	if k < 1 || k > 31 || len(out) < len(partners) {
+		return false
+	}
+	kmerLCSBatch(query, partners, k, out)
+	return true
+}
+
+// KmerLCSPartners is a fixed set of k-mers prepared once to be aligned against
+// many queries with KmerLCSBatch-like speed, without redoing the per-call
+// reorganisation of the partners. It is read-only after creation and can be
+// shared between goroutines.
+type KmerLCSPartners struct {
+	prepared kmerLCSPrepared
+	n        int
+}
+
+// NewKmerLCSPartners prepares partners for repeated alignment.
+// It returns nil if k is not between 1 and 31. The slice is not copied when
+// built without GOEXPERIMENT=simd, so it must not be modified afterwards.
+func NewKmerLCSPartners(partners []uint64, k int) *KmerLCSPartners {
+	if k < 1 || k > 31 {
+		return nil
+	}
+	return &KmerLCSPartners{prepared: prepareKmerLCS(partners, k), n: len(partners)}
+}
+
+// Len returns the number of prepared partners.
+func (p *KmerLCSPartners) Len() int { return p.n }
+
+// Align computes the alignment of query against every prepared partner and
+// stores it in out. It returns false if out is shorter than Len().
+func (p *KmerLCSPartners) Align(query uint64, out []KmerLCSResult) bool {
+	if len(out) < p.n {
+		return false
+	}
+	p.prepared.align(query, out[:p.n])
+	return true
+}
+
+func kmerLCSBatchScalar(query uint64, partners []uint64, k int, out []KmerLCSResult) {
+	for i, p := range partners {
+		out[i] = KmerLCS(query, p, k)
+	}
+}

pkg/obikmer/kmer_distance_test.go

@@ -0,0 +1,171 @@
+package obikmer
+
+import (
+	"math/rand"
+	"testing"
+)
+
+func randomKmer(rng *rand.Rand, k int) uint64 {
+	return rng.Uint64() & (uint64(1)<<(2*uint(k)) - 1)
+}
+
+func TestHammingDistance(t *testing.T) {
+	rng := rand.New(rand.NewSource(1))
+	for _, k := range []int{1, 2, 7, 21, 31} {
+		for n := 0; n < 200; n++ {
+			a, b := randomKmer(rng, k), randomKmer(rng, k)
+			sa, sb := DecodeKmer(a, k, nil), DecodeKmer(b, k, nil)
+			want := 0
+			for i := range sa {
+				if sa[i] != sb[i] {
+					want++
+				}
+			}
+			if got := HammingDistance(a, b, k); got != want {
+				t.Fatalf("k=%d %s/%s: got %d, want %d", k, sa, sb, got, want)
+			}
+		}
+	}
+}
+
+func TestHammingDistanceIgnoresHighBits(t *testing.T) {
+	if got := HammingDistance(0xFF00, 0x0000, 4); got != 0 {
+		t.Fatalf("bits above 2k must be ignored, got %d", got)
+	}
+}
+
+func TestKmerLCSKnown(t *testing.T) {
+	enc := func(s string) uint64 { return EncodeKmer([]byte(s), len(s)) }
+	cases := []struct {
+		a, b string
+		want KmerLCSResult
+	}{
+		{"ACGT", "ACGT", KmerLCSResult{4, 4}},
+		{"AAAA", "CCCC", KmerLCSResult{0, 4}},
+		{"ACGT", "ACGA", KmerLCSResult{3, 4}},
+		{"ACGT", "CGTA", KmerLCSResult{3, 5}}, // one shift: 1 indel on each side
+	}
+	for _, c := range cases {
+		if got := KmerLCS(enc(c.a), enc(c.b), 4); got != c.want {
+			t.Errorf("%s/%s: got %+v, want %+v", c.a, c.b, got, c.want)
+		}
+	}
+	if got := KmerLCS(0, 0, 32); got.LCS != -1 {
+		t.Errorf("k=32 must be rejected, got %+v", got)
+	}
+}
+
+// The batch implementation (SIMD when built with GOEXPERIMENT=simd) must agree
+// with the scalar reference for all k and for batch sizes around the lane count.
+func TestKmerLCSBatchMatchesReference(t *testing.T) {
+	rng := rand.New(rand.NewSource(2))
+	for _, k := range []int{1, 2, 3, 5, 8, 16, 21, 30, 31} {
+		for _, n := range []int{0, 1, 3, 7, 8, 9, 16, 17, 100} {
+			query := randomKmer(rng, k)
+			partners := make([]uint64, n)
+			for i := range partners {
+				partners[i] = randomKmer(rng, k)
+			}
+			if n > 2 {
+				partners[0] = query // identical
+			}
+			out := make([]KmerLCSResult, n)
+			if !KmerLCSBatch(query, partners, k, out) {
+				t.Fatalf("k=%d n=%d: batch refused", k, n)
+			}
+			for i, p := range partners {
+				if want := KmerLCS(query, p, k); out[i] != want {
+					t.Fatalf("k=%d n=%d partner %d: got %+v, want %+v", k, n, i, out[i], want)
+				}
+			}
+		}
+	}
+}
+
+func TestKmerLCSBatchRejectsBadInput(t *testing.T) {
+	if KmerLCSBatch(0, []uint64{1}, 0, make([]KmerLCSResult, 1)) {
+		t.Fatal("k=0 must be rejected")
+	}
+	if KmerLCSBatch(0, []uint64{1, 2}, 4, make([]KmerLCSResult, 1)) {
+		t.Fatal("short output must be rejected")
+	}
+}
+
+func benchBatch(b *testing.B, k int, f func(uint64, []uint64, int, []KmerLCSResult)) {
+	rng := rand.New(rand.NewSource(3))
+	partners := make([]uint64, 1024)
+	for i := range partners {
+		partners[i] = randomKmer(rng, k)
+	}
+	out := make([]KmerLCSResult, len(partners))
+	query := randomKmer(rng, k)
+	b.ResetTimer()
+	for b.Loop() {
+		f(query, partners, k, out)
+	}
+	b.ReportMetric(float64(b.Elapsed().Nanoseconds())/float64(b.N*len(partners)), "ns/pair")
+}
+
+func BenchmarkKmerLCSScalar21(b *testing.B) { benchBatch(b, 21, kmerLCSBatchScalar) }
+func BenchmarkKmerLCSBatch21(b *testing.B)  { benchBatch(b, 21, kmerLCSBatch) }
+func BenchmarkKmerLCSScalar11(b *testing.B) { benchBatch(b, 11, kmerLCSBatchScalar) }
+func BenchmarkKmerLCSBatch11(b *testing.B)  { benchBatch(b, 11, kmerLCSBatch) }
+
+func TestKmerLCSPartnersMatchesReference(t *testing.T) {
+	rng := rand.New(rand.NewSource(6))
+	for _, k := range []int{1, 5, 21, 31} {
+		for _, n := range []int{0, 1, 7, 8, 9, 50} {
+			partners := make([]uint64, n)
+			for i := range partners {
+				partners[i] = randomKmer(rng, k)
+			}
+			prep := NewKmerLCSPartners(partners, k)
+			out := make([]KmerLCSResult, n)
+			for q := 0; q < 3; q++ { // reuse the same preparation for several queries
+				query := randomKmer(rng, k)
+				if !prep.Align(query, out) {
+					t.Fatalf("k=%d n=%d: Align refused", k, n)
+				}
+				for i, p := range partners {
+					if want := KmerLCS(query, p, k); out[i] != want {
+						t.Fatalf("k=%d n=%d partner %d: got %+v, want %+v", k, n, i, out[i], want)
+					}
+				}
+			}
+		}
+	}
+	if NewKmerLCSPartners(nil, 0) != nil || NewKmerLCSPartners(nil, 32) != nil {
+		t.Fatal("invalid k must give nil")
+	}
+}
+
+// All-pairs scenario: many queries against the same partners.
+func benchAllPairs(b *testing.B, k int, prepared bool) {
+	rng := rand.New(rand.NewSource(7))
+	partners := make([]uint64, 1024)
+	for i := range partners {
+		partners[i] = randomKmer(rng, k)
+	}
+	queries := make([]uint64, 64)
+	for i := range queries {
+		queries[i] = randomKmer(rng, k)
+	}
+	out := make([]KmerLCSResult, len(partners))
+	prep := NewKmerLCSPartners(partners, k)
+	b.ResetTimer()
+	for b.Loop() {
+		for _, q := range queries {
+			if prepared {
+				prep.Align(q, out)
+			} else {
+				KmerLCSBatch(q, partners, k, out)
+			}
+		}
+	}
+	b.ReportMetric(float64(b.Elapsed().Nanoseconds())/float64(b.N*len(partners)*len(queries)), "ns/pair")
+}
+
+func BenchmarkAllPairsBatch21(b *testing.B)    { benchAllPairs(b, 21, false) }
+func BenchmarkAllPairsPrepared21(b *testing.B) { benchAllPairs(b, 21, true) }
+func BenchmarkAllPairsBatch11(b *testing.B)    { benchAllPairs(b, 11, false) }
+func BenchmarkAllPairsPrepared11(b *testing.B) { benchAllPairs(b, 11, true) }

pkg/obikmer/kmer_lcs_obialign_test.go

@@ -0,0 +1,29 @@
+package obikmer_test
+
+import (
+	"math/rand"
+	"testing"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obialign"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obikmer"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+// KmerLCS must reproduce obialign.FastLCSScore on the decoded sequences.
+func TestKmerLCSMatchesFastLCSScore(t *testing.T) {
+	rng := rand.New(rand.NewSource(4))
+	for _, k := range []int{1, 2, 4, 9, 15, 21, 31} {
+		mask := uint64(1)<<(2*uint(k)) - 1
+		for n := 0; n < 300; n++ {
+			a, b := rng.Uint64()&mask, rng.Uint64()&mask
+			sa := obiseq.NewBioSequence("a", obikmer.DecodeKmer(a, k, nil), "")
+			sb := obiseq.NewBioSequence("b", obikmer.DecodeKmer(b, k, nil), "")
+			lcs, alen := obialign.FastLCSScore(sa, sb, -1, nil)
+			got := obikmer.KmerLCS(a, b, k)
+			if got.LCS != lcs || got.AlignLen != alen {
+				t.Fatalf("k=%d %s/%s: KmerLCS=%+v, FastLCSScore=(%d,%d)",
+					k, sa.String(), sb.String(), got, lcs, alen)
+			}
+		}
+	}
+}

pkg/obikmer/kmer_lcs_scalar.go

@@ -0,0 +1,21 @@
+//go:build !goexperiment.simd
+
+package obikmer
+
+func kmerLCSBatch(query uint64, partners []uint64, k int, out []KmerLCSResult) {
+	kmerLCSBatchScalar(query, partners, k, out)
+}
+
+// kmerLCSPrepared has nothing to precompute without SIMD.
+type kmerLCSPrepared struct {
+	k        int
+	partners []uint64
+}
+
+func prepareKmerLCS(partners []uint64, k int) kmerLCSPrepared {
+	return kmerLCSPrepared{k: k, partners: partners}
+}
+
+func (p *kmerLCSPrepared) align(query uint64, out []KmerLCSResult) {
+	kmerLCSBatchScalar(query, p.partners, p.k, out)
+}

pkg/obikmer/kmer_lcs_simd.go

@@ -0,0 +1,114 @@
+//go:build goexperiment.simd
+
+package obikmer
+
+import "simd"
+
+// Each lane of a vector holds the state of one (query, partner) alignment as
+//
+//	score<<6 | (63 - alignment length)
+//
+// so that taking the maximum of two cells ranks them by score first and, for
+// equal scores, by shortest alignment. With k <= 31 the score is <= 31 and the
+// alignment length <= 62, hence the value fits in 11 bits and the low field
+// is never exhausted. The three DP moves then all reduce to
+//
+//	cell(i,j) = max(diag + 64*match, up, left) - 1
+//
+// (every move lengthens the alignment by one; only a diagonal match adds 1
+// to the score).
+const (
+	lcsScoreUnit = 64
+	lcsLenBase   = 63
+)
+
+func simdLanes() int {
+	var probe simd.Uint16s
+	return probe.Len()
+}
+
+// transposeChunk writes the bases of up to `lanes` partners in dst, laid out
+// so that dst[j*lanes+lane] is base j of the partner in `lane`.
+func transposeChunk(chunk []uint64, k, lanes int, dst []uint16) {
+	for lane, p := range chunk {
+		for j := 0; j < k; j++ {
+			dst[j*lanes+lane] = uint16(p>>(2*uint(k-1-j))) & 3
+		}
+	}
+}
+
+// alignChunk aligns the query against the partners of one transposed chunk
+// and writes one result per lane in out (len(out) <= lanes).
+func alignChunk(query uint64, pbase, row []uint16, k, lanes int, out []KmerLCSResult) {
+	one := simd.BroadcastUint16s(1)
+	bonus := simd.BroadcastUint16s(lcsScoreUnit)
+
+	for j := 0; j <= k; j++ {
+		simd.BroadcastUint16s(uint16(lcsLenBase - j)).Store(row[j*lanes:])
+	}
+
+	for i := 1; i <= k; i++ {
+		qv := simd.BroadcastUint16s(uint16(query>>(2*uint(k-i))) & 3)
+		diag := simd.LoadUint16s(row)
+		left := simd.BroadcastUint16s(uint16(lcsLenBase - i))
+		left.Store(row)
+
+		for j := 1; j <= k; j++ {
+			up := simd.LoadUint16s(row[j*lanes:])
+			pv := simd.LoadUint16s(pbase[(j-1)*lanes:])
+			match := qv.Equal(pv)
+			best := diag.Add(bonus.Masked(match)).Max(up).Max(left).Sub(one)
+			best.Store(row[j*lanes:])
+			diag = up
+			left = best
+		}
+	}
+
+	last := row[k*lanes:]
+	for lane := range out {
+		v := int(last[lane])
+		out[lane] = KmerLCSResult{
+			LCS:      v / lcsScoreUnit,
+			AlignLen: lcsLenBase - v%lcsScoreUnit,
+		}
+	}
+}
+
+func kmerLCSBatch(query uint64, partners []uint64, k int, out []KmerLCSResult) {
+	lanes := simdLanes()
+	pbase := make([]uint16, k*lanes)
+	row := make([]uint16, (k+1)*lanes)
+
+	for start := 0; start < len(partners); start += lanes {
+		end := min(start+lanes, len(partners))
+		transposeChunk(partners[start:end], k, lanes, pbase)
+		alignChunk(query, pbase, row, k, lanes, out[start:end])
+	}
+}
+
+// kmerLCSPrepared holds partners already transposed, chunk after chunk.
+type kmerLCSPrepared struct {
+	k, lanes, n int
+	pbase       []uint16 // chunk c at [c*k*lanes : (c+1)*k*lanes]
+}
+
+func prepareKmerLCS(partners []uint64, k int) kmerLCSPrepared {
+	lanes := simdLanes()
+	chunks := (len(partners) + lanes - 1) / lanes
+	p := kmerLCSPrepared{k: k, lanes: lanes, n: len(partners),
+		pbase: make([]uint16, chunks*k*lanes)}
+	for c := 0; c < chunks; c++ {
+		end := min((c+1)*lanes, len(partners))
+		transposeChunk(partners[c*lanes:end], k, lanes, p.pbase[c*k*lanes:(c+1)*k*lanes])
+	}
+	return p
+}
+
+func (p *kmerLCSPrepared) align(query uint64, out []KmerLCSResult) {
+	row := make([]uint16, (p.k+1)*p.lanes)
+	size := p.k * p.lanes
+	for c, start := 0, 0; start < p.n; c, start = c+1, start+p.lanes {
+		end := min(start+p.lanes, p.n)
+		alignChunk(query, p.pbase[c*size:(c+1)*size], row, p.k, p.lanes, out[start:end])
+	}
+}

pkg/obikmer/seqquorum.go

@@ -0,0 +1,229 @@
+package obikmer
+
+import (
+	"bufio"
+	"encoding/binary"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
+)
+
+// SeqKmerCounter counts, for every canonical k-mer, in how many distinct input
+// sequences it occurs. It answers "which k-mers are present in at least q of
+// the N sequences" without one k-mer set per sequence: a KmerSetGroup needs
+// N*P partition files, which is out of reach for thousands of genomes, whereas
+// a SeqKmerCounter uses a single set of P files whatever N is.
+//
+// Construction is two-phase, like KmerSetGroupBuilder:
+//
+//  1. AddSequence extracts super-kmers (grouped by minimizer, so a k-mer and its
+//     reverse complement always land in the same partition) and appends them,
+//     tagged with the sequence id, to one of P temporary files. It is safe to
+//     call concurrently.
+//  2. CountAtLeast processes the partitions in parallel: each one is loaded,
+//     expanded to (canonical k-mer, sequence id) pairs, sorted, and reduced.
+//
+// Memory during phase 2 is 16 bytes per k-mer occurrence of the partitions
+// being processed at the same time. Their number is bounded by an occurrence
+// budget (see SetOccurrenceBudget), so that the memory does not depend on how
+// the k-mers happen to be spread over the partitions; P only has to be large
+// enough for the largest partition to be small against the budget (total bases
+// / P is the occurrence count of an average partition, see DefaultPartitions).
+//
+// Bases other than A, C, G, T, U break a sequence into independent runs: no
+// k-mer ever contains an ambiguous base.
+type SeqKmerCounter struct {
+	dir     string
+	k, m, p int
+
+	parts []*seqPartWriter
+
+	mu       sync.Mutex
+	firstErr error
+	sealed   bool
+
+	budget int // k-mer occurrences held at once in phase 2
+}
+
+type seqPartWriter struct {
+	mu    sync.Mutex
+	w     *bufio.Writer
+	file  *os.File
+	path  string
+	kmers int // k-mer occurrences written, so that the partition is loaded in an array of the exact size
+}
+
+// SeqKmerCount is the result for one k-mer.
+type SeqKmerCount struct {
+	Kmer      uint64 // canonical k-mer
+	Sequences uint32 // number of distinct sequences containing it
+	Repeated  bool   // occurs more than once (either strand) in at least one sequence
+}
+
+// NewSeqKmerCounter creates a counter whose temporary files live in a new
+// directory created under parentDir (use "" for the system temporary
+// directory). k is the k-mer size (2..31), m the minimizer size (<=0 for the
+// default) and partitions the number of partition files P.
+func NewSeqKmerCounter(parentDir string, k, m, partitions int) (*SeqKmerCounter, error) {
+	if k < 2 || k > 31 {
+		return nil, fmt.Errorf("obikmer: k must be between 2 and 31, got %d", k)
+	}
+	if partitions < 1 {
+		return nil, fmt.Errorf("obikmer: partitions must be >= 1, got %d", partitions)
+	}
+	if m <= 0 {
+		m = DefaultMinimizerSize(k)
+	}
+	if m >= k {
+		return nil, fmt.Errorf("obikmer: minimizer size %d must be smaller than k=%d", m, k)
+	}
+
+	dir, err := os.MkdirTemp(parentDir, "obikmer-seqcount-")
+	if err != nil {
+		return nil, err
+	}
+
+	c := &SeqKmerCounter{dir: dir, k: k, m: m, p: partitions, budget: DefaultOccurrenceBudget}
+	c.parts = make([]*seqPartWriter, partitions)
+	for i := range c.parts {
+		path := filepath.Join(dir, fmt.Sprintf("part_%04d.sqk", i))
+		f, err := os.Create(path)
+		if err != nil {
+			c.Close()
+			return nil, err
+		}
+		c.parts[i] = &seqPartWriter{w: bufio.NewWriterSize(f, 32*1024), file: f, path: path}
+	}
+	return c, nil
+}
+
+// DefaultOccurrenceBudget is the number of k-mer occurrences phase 2 holds at
+// once by default: 32 million, 512 MB. A partition larger than the budget is
+// processed alone. Measured on 16 cores, 526 million occurrences to count: with
+// 32M the count is as fast as without any limit; 16M make it 15% slower, 8M
+// 60% slower.
+const DefaultOccurrenceBudget = 32 << 20
+
+// SetOccurrenceBudget bounds the k-mer occurrences (16 bytes each) that
+// CountAtLeast holds in memory at once. It must be called before CountAtLeast.
+func (c *SeqKmerCounter) SetOccurrenceBudget(occurrences int) {
+	if occurrences > 0 {
+		c.budget = occurrences
+	}
+}
+
+// partitionOccurrences is the occurrence count aimed at for an average
+// partition by DefaultPartitions (a partition of 1M occurrences is 16 MB).
+const partitionOccurrences = 1 << 20
+
+// DefaultPartitions returns a partition count suited to a total of kmers
+// k-mer occurrences: 256 at least, and more, as a power of two up to 1024, when
+// the average partition would otherwise exceed about a million occurrences.
+// The largest partitions hold several times the average. The limit keeps the
+// number of simultaneously open files reasonable.
+func DefaultPartitions(kmers int) int {
+	p := 256
+	for p < 1024 && kmers/p > partitionOccurrences {
+		p *= 2
+	}
+	return p
+}
+
+// K returns the k-mer size.
+func (c *SeqKmerCounter) K() int { return c.k }
+
+var unambiguousBase = func() (t [256]bool) {
+	for _, b := range "ACGTUacgtu" {
+		t[b] = true
+	}
+	return
+}()
+
+// AddSequence registers sequence id. The same id must be used for every call
+// concerning the same sequence; ids need not be contiguous. If circular is
+// true the k-1 first bases are appended so that k-mers spanning the junction
+// are counted. AddSequence is safe for concurrent use and must not be called
+// after CountAtLeast.
+func (c *SeqKmerCounter) AddSequence(id uint32, seq []byte, circular bool) {
+	if len(seq) < c.k {
+		return
+	}
+	if circular {
+		ext := make([]byte, 0, len(seq)+c.k-1)
+		ext = append(ext, seq...)
+		seq = append(ext, seq[:c.k-1]...)
+	}
+
+	var scratch []byte
+	start := -1
+	for i, b := range seq {
+		switch {
+		case unambiguousBase[b]:
+			if start < 0 {
+				start = i
+			}
+		case start >= 0:
+			scratch = c.addRun(id, seq[start:i], scratch)
+			start = -1
+		}
+	}
+	if start >= 0 {
+		c.addRun(id, seq[start:], scratch)
+	}
+}
+
+// addRun writes the super-kmers of one ambiguity-free run.
+func (c *SeqKmerCounter) addRun(id uint32, run []byte, scratch []byte) []byte {
+	if len(run) < c.k {
+		return scratch
+	}
+	for sk := range IterSuperKmers(run, c.k, c.m) {
+		scratch = binary.AppendUvarint(scratch[:0], uint64(id))
+		scratch = binary.AppendUvarint(scratch, uint64(len(sk.Sequence)))
+		scratch = appendPacked(scratch, sk.Sequence)
+
+		pw := c.parts[int(sk.Minimizer%uint64(c.p))]
+		pw.mu.Lock()
+		_, err := pw.w.Write(scratch)
+		pw.kmers += len(sk.Sequence) - c.k + 1
+		pw.mu.Unlock()
+		if err != nil {
+			c.setErr(err)
+		}
+	}
+	return scratch
+}
+
+func (c *SeqKmerCounter) setErr(err error) {
+	c.mu.Lock()
+	if c.firstErr == nil {
+		c.firstErr = err
+	}
+	c.mu.Unlock()
+}
+
+// seal flushes and closes the partition files; no more sequences can be added.
+func (c *SeqKmerCounter) seal() error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.sealed {
+		return c.firstErr
+	}
+	c.sealed = true
+	for _, pw := range c.parts {
+		if err := pw.w.Flush(); err != nil && c.firstErr == nil {
+			c.firstErr = err
+		}
+		if err := pw.file.Close(); err != nil && c.firstErr == nil {
+			c.firstErr = err
+		}
+	}
+	return c.firstErr
+}
+
+// Close removes the temporary files. It is safe to call more than once.
+func (c *SeqKmerCounter) Close() error {
+	_ = c.seal()
+	return os.RemoveAll(c.dir)
+}

pkg/obikmer/seqquorum_budget_test.go

@@ -0,0 +1,118 @@
+package obikmer
+
+import (
+	"math/rand"
+	"reflect"
+	"sync"
+	"testing"
+	"time"
+)
+
+// The occurrence budget only bounds the memory: any budget, down to one
+// occurrence (every partition then goes alone), gives the same counts.
+func TestSeqKmerCounterBudgetDoesNotChangeTheCounts(t *testing.T) {
+	rng := rand.New(rand.NewSource(7))
+	seqs := randomSeqs(rng, 40, 20, 500, true)
+	build := func(budget int) []SeqKmerCount {
+		c, err := NewSeqKmerCounter(t.TempDir(), 9, 4, 11)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer c.Close()
+		if budget > 0 {
+			c.SetOccurrenceBudget(budget)
+		}
+		for i, s := range seqs {
+			c.AddSequence(uint32(i), s, i%2 == 0)
+		}
+		total := 0
+		for _, pw := range c.parts {
+			total += pw.kmers
+		}
+		if total == 0 {
+			t.Fatal("no occurrence counted")
+		}
+		got, err := c.CountAtLeast(2)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return got
+	}
+	want := build(0)
+	if len(want) == 0 {
+		t.Fatal("the test needs k-mers shared by several sequences")
+	}
+	for _, budget := range []int{1, 100, 5000} {
+		if got := build(budget); !reflect.DeepEqual(got, want) {
+			t.Errorf("budget %d changes the counts", budget)
+		}
+	}
+}
+
+// The occurrences announced for a partition are those that its file holds.
+func TestSeqKmerCounterPartitionOccurrenceCount(t *testing.T) {
+	rng := rand.New(rand.NewSource(3))
+	seqs := randomSeqs(rng, 25, 10, 300, true)
+	c, err := NewSeqKmerCounter(t.TempDir(), 8, 3, 5)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer c.Close()
+	for i, s := range seqs {
+		c.AddSequence(uint32(i), s, i%3 == 0)
+	}
+	if err := c.seal(); err != nil {
+		t.Fatal(err)
+	}
+	for p, pw := range c.parts {
+		occ, err := c.loadPartition(p, nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(occ) != pw.kmers {
+			t.Errorf("partition %d: %d occurrences loaded, %d counted when written", p, len(occ), pw.kmers)
+		}
+	}
+}
+
+func TestOccurrenceBudget(t *testing.T) {
+	b := newOccurrenceBudget(10)
+	if got := b.acquire(4); got != 4 {
+		t.Fatalf("acquire(4) = %d", got)
+	}
+	// 7 does not fit in the 6 left: it must wait for the release
+	done := make(chan int)
+	go func() { done <- b.acquire(7) }()
+	select {
+	case <-done:
+		t.Fatal("acquire(7) must wait")
+	case <-time.After(50 * time.Millisecond):
+	}
+	b.release(4)
+	if got := <-done; got != 7 {
+		t.Fatalf("acquire(7) = %d", got)
+	}
+	b.release(7)
+
+	// a request above the whole budget takes all of it, once everything is free
+	if got := b.acquire(1000); got != 10 {
+		t.Fatalf("an oversized request must take the whole budget, got %d", got)
+	}
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() { defer wg.Done(); b.release(b.acquire(3)) }()
+	time.Sleep(20 * time.Millisecond)
+	b.release(10)
+	wg.Wait()
+}
+
+func TestDefaultPartitions(t *testing.T) {
+	for _, tc := range []struct{ kmers, want int }{
+		{0, 256}, {5_000_000, 256}, {256 << 20, 256}, {257 << 20, 512}, {526_000_000, 512},
+		{600 << 20, 1024}, {1 << 40, 1024},
+	} {
+		if got := DefaultPartitions(tc.kmers); got != tc.want {
+			t.Errorf("DefaultPartitions(%d) = %d, want %d", tc.kmers, got, tc.want)
+		}
+	}
+}

pkg/obikmer/seqquorum_count.go

@@ -0,0 +1,207 @@
+package obikmer
+
+import (
+	"bufio"
+	"cmp"
+	"encoding/binary"
+	"io"
+	"os"
+	"slices"
+	"sync"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
+)
+
+// kmerOcc is one occurrence of a canonical k-mer in a sequence.
+type kmerOcc struct {
+	kmer uint64
+	seq  uint32
+}
+
+func cmpOcc(a, b kmerOcc) int {
+	if c := cmp.Compare(a.kmer, b.kmer); c != 0 {
+		return c
+	}
+	return cmp.Compare(a.seq, b.seq)
+}
+
+// CountAtLeast returns, sorted by k-mer, every canonical k-mer found in at
+// least minSeq distinct sequences, with its exact sequence count.
+//
+// The result is held in memory: it is meant for quorum-filtered sets (the
+// conserved k-mers), not for the complete k-mer spectrum. The first call
+// seals the counter; AddSequence must not be called afterwards. The count
+// phase uses obidefault.ParallelWorkers() goroutines over the partitions.
+func (c *SeqKmerCounter) CountAtLeast(minSeq int) ([]SeqKmerCount, error) {
+	if err := c.seal(); err != nil {
+		return nil, err
+	}
+	if minSeq < 1 {
+		minSeq = 1
+	}
+
+	workers := min(obidefault.ParallelWorkers(), c.p)
+
+	// the largest partitions first: they are the hardest to fit in the budget
+	// and the last ones to finish, if they come late
+	order := make([]int, c.p)
+	for i := range order {
+		order[i] = i
+	}
+	slices.SortStableFunc(order, func(a, b int) int { return cmp.Compare(c.parts[b].kmers, c.parts[a].kmers) })
+	budget := newOccurrenceBudget(c.budget)
+
+	jobs := make(chan int)
+	results := make([][]SeqKmerCount, c.p)
+
+	var wg sync.WaitGroup
+	var errMu sync.Mutex
+	var firstErr error
+
+	for range workers {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			var occ []kmerOcc // reused between partitions of similar sizes
+			for p := range jobs {
+				n := c.parts[p].kmers
+				held := budget.acquire(n)
+				if cap(occ) < n || cap(occ) > 2*n+1024 {
+					occ = make([]kmerOcc, 0, n)
+				}
+				var err error
+				occ, err = c.loadPartition(p, occ[:0])
+				if err == nil {
+					results[p] = reduceOccurrences(occ, minSeq)
+				}
+				budget.release(held)
+				if err != nil {
+					errMu.Lock()
+					if firstErr == nil {
+						firstErr = err
+					}
+					errMu.Unlock()
+				}
+			}
+		}()
+	}
+	for _, p := range order {
+		jobs <- p
+	}
+	close(jobs)
+	wg.Wait()
+	if firstErr != nil {
+		return nil, firstErr
+	}
+
+	total := 0
+	for _, r := range results {
+		total += len(r)
+	}
+	all := make([]SeqKmerCount, 0, total)
+	for _, r := range results {
+		all = append(all, r...)
+	}
+	slices.SortFunc(all, func(a, b SeqKmerCount) int { return cmp.Compare(a.Kmer, b.Kmer) })
+	return all, nil
+}
+
+// loadPartition appends to occ every (canonical k-mer, sequence id) of the
+// partition file.
+func (c *SeqKmerCounter) loadPartition(p int, occ []kmerOcc) ([]kmerOcc, error) {
+	f, err := os.Open(c.parts[p].path)
+	if err != nil {
+		return occ, err
+	}
+	defer f.Close()
+	r := bufio.NewReaderSize(f, 64*1024)
+
+	var packed, bases []byte
+	for {
+		id, err := binary.ReadUvarint(r)
+		if err == io.EOF {
+			return occ, nil
+		}
+		if err != nil {
+			return occ, err
+		}
+		n, err := binary.ReadUvarint(r)
+		if err != nil {
+			return occ, err
+		}
+
+		nb := (int(n) + 3) / 4
+		packed = slices.Grow(packed[:0], nb)[:nb]
+		if _, err := io.ReadFull(r, packed); err != nil {
+			return occ, err
+		}
+		bases = slices.Grow(bases[:0], int(n))[:n]
+		unpackBases(bases, packed)
+
+		for kmer := range IterCanonicalKmers(bases, c.k) {
+			occ = append(occ, kmerOcc{kmer, uint32(id)})
+		}
+	}
+}
+
+// reduceOccurrences sorts occ and returns the k-mers present in at least
+// minSeq distinct sequences. occ is reordered in place.
+func reduceOccurrences(occ []kmerOcc, minSeq int) []SeqKmerCount {
+	slices.SortFunc(occ, cmpOcc)
+
+	var out []SeqKmerCount
+	for i := 0; i < len(occ); {
+		kmer := occ[i].kmer
+		var seqs uint32
+		repeated := false
+		j := i
+		for ; j < len(occ) && occ[j].kmer == kmer; j++ {
+			if j == i || occ[j].seq != occ[j-1].seq {
+				seqs++
+			} else {
+				repeated = true
+			}
+		}
+		if int(seqs) >= minSeq {
+			out = append(out, SeqKmerCount{Kmer: kmer, Sequences: seqs, Repeated: repeated})
+		}
+		i = j
+	}
+	return out
+}
+
+// occurrenceBudget bounds the k-mer occurrences held in memory at once by the
+// goroutines that load the partitions.
+type occurrenceBudget struct {
+	mu    sync.Mutex
+	freed *sync.Cond
+	total int
+	free  int
+}
+
+func newOccurrenceBudget(total int) *occurrenceBudget {
+	b := &occurrenceBudget{total: total, free: total}
+	b.freed = sync.NewCond(&b.mu)
+	return b
+}
+
+// acquire waits until n occurrences fit in the budget and takes them; a request
+// above the whole budget waits for the budget to be free and takes all of it.
+// It returns what was taken, to give back with release.
+func (b *occurrenceBudget) acquire(n int) int {
+	n = min(n, b.total)
+	b.mu.Lock()
+	for b.free < n {
+		b.freed.Wait()
+	}
+	b.free -= n
+	b.mu.Unlock()
+	return n
+}
+
+func (b *occurrenceBudget) release(n int) {
+	b.mu.Lock()
+	b.free += n
+	b.mu.Unlock()
+	b.freed.Broadcast()
+}

pkg/obikmer/seqquorum_test.go

@@ -0,0 +1,272 @@
+package obikmer
+
+import (
+	"math/rand"
+	"slices"
+	"strings"
+	"sync"
+	"testing"
+)
+
+type bruteInfo struct {
+	seqs     map[uint32]int // sequence id -> occurrences
+	repeated bool
+}
+
+// bruteForce is the oracle: every window of k unambiguous bases, circular
+// junction included, encoded with EncodeCanonicalKmer.
+func bruteForce(seqs [][]byte, circular []bool, k int) map[uint64]*bruteInfo {
+	res := map[uint64]*bruteInfo{}
+	for id, s := range seqs {
+		if len(s) < k {
+			continue
+		}
+		if circular[id] {
+			s = append(slices.Clone(s), s[:k-1]...)
+		}
+	window:
+		for i := 0; i+k <= len(s); i++ {
+			w := s[i : i+k]
+			for _, b := range w {
+				if !unambiguousBase[b] {
+					continue window
+				}
+			}
+			kmer := EncodeCanonicalKmer(w, k)
+			info := res[kmer]
+			if info == nil {
+				info = &bruteInfo{seqs: map[uint32]int{}}
+				res[kmer] = info
+			}
+			info.seqs[uint32(id)]++
+			if info.seqs[uint32(id)] > 1 {
+				info.repeated = true
+			}
+		}
+	}
+	return res
+}
+
+func randomSeqs(rng *rand.Rand, n, minLen, maxLen int, withN bool) [][]byte {
+	alphabet := "acgtACGT"
+	if withN {
+		alphabet = "acgtACGTacgtACGTNnR"
+	}
+	// A shared motif and its reverse complement make k-mers common to several
+	// sequences on both strands.
+	motif := make([]byte, 40)
+	for i := range motif {
+		motif[i] = "ACGT"[rng.Intn(4)]
+	}
+	rc := make([]byte, len(motif))
+	for i, b := range motif {
+		rc[len(motif)-1-i] = map[byte]byte{'A': 'T', 'C': 'G', 'G': 'C', 'T': 'A'}[b]
+	}
+
+	out := make([][]byte, n)
+	for i := range out {
+		l := minLen + rng.Intn(maxLen-minLen+1)
+		s := make([]byte, l)
+		for j := range s {
+			s[j] = alphabet[rng.Intn(len(alphabet))]
+		}
+		mode := rng.Intn(4)
+		if l < 60 { // too short to host the motifs
+			mode = 3
+		}
+		switch mode {
+		case 0:
+			copy(s[l/4:], motif)
+		case 1:
+			copy(s[l/4:], rc)
+		case 2: // internal repeat, both strands
+			copy(s[2:], motif[:20])
+			copy(s[l-25:], rc[:20])
+		}
+		out[i] = s
+	}
+	return out
+}
+
+func TestSeqKmerCounterAgainstBruteForce(t *testing.T) {
+	cases := []struct {
+		name    string
+		k, m, p int
+		withN   bool
+		circ    bool
+	}{
+		{"k7-linear", 7, 3, 8, false, false},
+		{"k7-circular-N", 7, 3, 5, true, true},
+		{"k18-linear-N", 18, 8, 16, true, false},
+		{"k18-circular", 18, 8, 7, false, true},
+		{"k31-p1", 31, 10, 1, false, true},
+		{"k4-small-m", 4, 1, 3, true, false},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			rng := rand.New(rand.NewSource(int64(tc.k*100 + tc.p)))
+			seqs := randomSeqs(rng, 30, 5, 400, tc.withN)
+			circ := make([]bool, len(seqs))
+			for i := range circ {
+				circ[i] = tc.circ && i%3 != 0
+			}
+
+			c, err := NewSeqKmerCounter(t.TempDir(), tc.k, tc.m, tc.p)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer c.Close()
+
+			// Concurrent insertion, shuffled order, ids not contiguous.
+			var wg sync.WaitGroup
+			for i := range seqs {
+				wg.Add(1)
+				go func() {
+					defer wg.Done()
+					c.AddSequence(uint32(i), seqs[i], circ[i])
+				}()
+			}
+			wg.Wait()
+
+			want := bruteForce(seqs, circ, tc.k)
+			for _, q := range []int{1, 2, 3, 10} {
+				got, err := c.CountAtLeast(q)
+				if err != nil {
+					t.Fatal(err)
+				}
+				if !slices.IsSortedFunc(got, func(a, b SeqKmerCount) int {
+					switch {
+					case a.Kmer < b.Kmer:
+						return -1
+					case a.Kmer > b.Kmer:
+						return 1
+					}
+					return 0
+				}) {
+					t.Fatal("result is not sorted by k-mer")
+				}
+
+				n := 0
+				for kmer, info := range want {
+					if len(info.seqs) >= q {
+						n++
+						_ = kmer
+					}
+				}
+				if len(got) != n {
+					t.Fatalf("q=%d: %d k-mers, want %d", q, len(got), n)
+				}
+				for _, g := range got {
+					info := want[g.Kmer]
+					if info == nil {
+						t.Fatalf("q=%d: unexpected k-mer %d", q, g.Kmer)
+					}
+					if int(g.Sequences) != len(info.seqs) || g.Repeated != info.repeated {
+						t.Fatalf("q=%d kmer %d: got (%d,%v) want (%d,%v)",
+							q, g.Kmer, g.Sequences, g.Repeated, len(info.seqs), info.repeated)
+					}
+				}
+			}
+		})
+	}
+}
+
+// A k-mer appearing on both strands of one sequence counts once but is Repeated.
+func TestSeqKmerCounterBothStrandsOneSequence(t *testing.T) {
+	const k = 8
+	fwd := "ACGGTCAT"
+	rc := "ATGACCGT"
+	seq := []byte("TTTT" + fwd + "GGGGGGGG" + rc + "CCCC")
+
+	c, err := NewSeqKmerCounter(t.TempDir(), k, 3, 4)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer c.Close()
+	c.AddSequence(7, seq, false)
+
+	got, err := c.CountAtLeast(1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	target := EncodeCanonicalKmer([]byte(fwd), k)
+	found := false
+	for _, g := range got {
+		if g.Kmer == target {
+			found = true
+			if g.Sequences != 1 || !g.Repeated {
+				t.Errorf("got %+v, want 1 sequence and Repeated", g)
+			}
+		}
+	}
+	if !found {
+		t.Fatal("k-mer not found")
+	}
+}
+
+// Ambiguous bases must never be part of a k-mer, and the circular junction
+// must produce the wrap-around k-mers.
+func TestSeqKmerCounterAmbiguityAndCircular(t *testing.T) {
+	const k = 5
+	c, err := NewSeqKmerCounter(t.TempDir(), k, 2, 3)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer c.Close()
+
+	c.AddSequence(0, []byte("ACGTNACGT"), false) // runs ACGT (<k) and ACGT (<k): nothing
+	c.AddSequence(1, []byte("AACCGGTT"), true)   // 8 circular windows
+
+	got, err := c.CountAtLeast(1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) == 0 {
+		t.Fatal("expected k-mers from the circular sequence")
+	}
+	var circ []string
+	for i := 0; i < 8; i++ {
+		var sb strings.Builder
+		for j := 0; j < k; j++ {
+			sb.WriteByte("AACCGGTT"[(i+j)%8])
+		}
+		circ = append(circ, sb.String())
+	}
+	want := map[uint64]bool{}
+	for _, w := range circ {
+		want[EncodeCanonicalKmer([]byte(w), k)] = true
+	}
+	if len(got) != len(want) {
+		t.Fatalf("got %d k-mers, want %d", len(got), len(want))
+	}
+	for _, g := range got {
+		if !want[g.Kmer] || g.Sequences != 1 {
+			t.Errorf("unexpected %+v", g)
+		}
+	}
+}
+
+func TestSeqKmerCounterParameters(t *testing.T) {
+	if _, err := NewSeqKmerCounter(t.TempDir(), 1, 1, 4); err == nil {
+		t.Error("k=1 must fail")
+	}
+	if _, err := NewSeqKmerCounter(t.TempDir(), 32, 5, 4); err == nil {
+		t.Error("k=32 must fail")
+	}
+	if _, err := NewSeqKmerCounter(t.TempDir(), 10, 10, 4); err == nil {
+		t.Error("m>=k must fail")
+	}
+	if _, err := NewSeqKmerCounter(t.TempDir(), 10, 4, 0); err == nil {
+		t.Error("zero partitions must fail")
+	}
+	c, err := NewSeqKmerCounter(t.TempDir(), 10, 0, 4) // default minimizer
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := c.Close(); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.Close(); err != nil {
+		t.Fatalf("second Close must be harmless: %v", err)
+	}
+}

pkg/obikmer/skm_reader.go

@@ -10,6 +10,16 @@ import (
 // decode2bit maps 2-bit codes back to nucleotide bytes.
 var decode2bit = [4]byte{'a', 'c', 'g', 't'}
 
+// unpackBases decodes the 2-bit packed bases in packed into dst (one lowercase
+// nucleotide per byte, len(dst) bases) and returns dst.
+func unpackBases(dst, packed []byte) []byte {
+	for i := range dst {
+		code := (packed[i/4] >> uint(6-(i%4)*2)) & 0x03
+		dst[i] = decode2bit[code]
+	}
+	return dst
+}
+
 // SkmReader reads super-kmers from a binary .skm file.
 type SkmReader struct {
 	r    *bufio.Reader
@@ -46,13 +56,7 @@ func (sr *SkmReader) Next() (SuperKmer, bool) {
 	}
 
 	// Decode to nucleotide bytes
-	seq := make([]byte, seqLen)
-	for i := 0; i < seqLen; i++ {
-		byteIdx := i / 4
-		bitPos := uint(6 - (i%4)*2)
-		code := (packed[byteIdx] >> bitPos) & 0x03
-		seq[i] = decode2bit[code]
-	}
+	seq := unpackBases(make([]byte, seqLen), packed)
 
 	return SuperKmer{
 		Sequence: seq,

pkg/obikmer/skm_writer.go

@@ -16,8 +16,9 @@ import (
 // Nucleotide encoding: A=00, C=01, G=10, T=11.
 // The last byte is zero-padded on the low bits if len%4 != 0.
 type SkmWriter struct {
-	w    *bufio.Writer
-	file *os.File
+	w       *bufio.Writer
+	file    *os.File
+	scratch []byte
 }
 
 // NewSkmWriter creates a new SkmWriter writing to the given file path.
@@ -32,36 +33,36 @@ func NewSkmWriter(path string) (*SkmWriter, error) {
 	}, nil
 }
 
+// appendPacked appends seq to dst packed 2 bits per base (A=00, C=01, G=10,
+// T=11), four bases per byte, the last byte zero-padded on the low bits.
+func appendPacked(dst, seq []byte) []byte {
+	for i := 0; i < len(seq); i += 4 {
+		var packed byte
+		for j := 0; j < 4; j++ {
+			packed <<= 2
+			if i+j < len(seq) {
+				packed |= __single_base_code__[seq[i+j]&31]
+			}
+		}
+		dst = append(dst, packed)
+	}
+	return dst
+}
+
 // Write encodes a SuperKmer to the .skm file.
 // The sequence bytes are packed 2 bits per base.
 func (sw *SkmWriter) Write(sk SuperKmer) error {
-	seq := sk.Sequence
-	seqLen := uint16(len(seq))
-
 	// Write length
 	var lenbuf [2]byte
-	binary.LittleEndian.PutUint16(lenbuf[:], seqLen)
+	binary.LittleEndian.PutUint16(lenbuf[:], uint16(len(sk.Sequence)))
 	if _, err := sw.w.Write(lenbuf[:]); err != nil {
 		return err
 	}
 
 	// Encode and write packed sequence (2 bits/base)
-	nBytes := (int(seqLen) + 3) / 4
-	for i := 0; i < nBytes; i++ {
-		var packed byte
-		for j := 0; j < 4; j++ {
-			pos := i*4 + j
-			packed <<= 2
-			if pos < int(seqLen) {
-				packed |= __single_base_code__[seq[pos]&31]
-			}
-		}
-		if err := sw.w.WriteByte(packed); err != nil {
-			return err
-		}
-	}
-
-	return nil
+	sw.scratch = appendPacked(sw.scratch[:0], sk.Sequence)
+	_, err := sw.w.Write(sw.scratch)
+	return err
 }
 
 // Close flushes buffered data and closes the underlying file.

pkg/obilua/lua.go

@@ -91,7 +91,7 @@ func LuaWorker(proto *lua.FunctionProto) obiseq.SeqWorker {
 	err := interpreter.PCall(0, lua.MultRet, nil)
 
 	if err != nil {
-		log.Fatalf("Error in executing the lua script")
+		log.Fatalf("Error in executing the lua script: %v", err)
 	}
 
 	result := interpreter.GetGlobal("worker")
@@ -141,6 +141,69 @@ func LuaWorker(proto *lua.FunctionProto) obiseq.SeqWorker {
 	return nil
 }
 
+// LuaSliceWorker creates a SeqSliceWorker that calls the Lua function
+// named "slice_worker". Unlike LuaWorker, the entire batch (BioSequenceSlice)
+// is passed to the Lua function at once, enabling batch-level processing
+// (e.g. a single HTTP request per batch instead of one per sequence).
+//
+// The Lua function signature:
+//
+//	function slice_worker(slice)   -- receives a BioSequenceSlice
+//	    -- process the batch
+//	    return slice               -- returns a BioSequenceSlice (or nil)
+//	end
+func LuaSliceWorker(proto *lua.FunctionProto) obiseq.SeqSliceWorker {
+	interpreter := NewInterpreter()
+	lfunc := interpreter.NewFunctionFromProto(proto)
+	interpreter.Push(lfunc)
+	err := interpreter.PCall(0, lua.MultRet, nil)
+
+	if err != nil {
+		log.Fatalf("Error in executing the lua script: %v", err)
+	}
+
+	result := interpreter.GetGlobal("slice_worker")
+
+	if lua_worker, ok := result.(*lua.LFunction); ok {
+		f := func(slice obiseq.BioSequenceSlice) (obiseq.BioSequenceSlice, error) {
+			if err := interpreter.CallByParam(lua.P{
+				Fn:      lua_worker,
+				NRet:    1,
+				Protect: true,
+			}, obiseqslice2Lua(interpreter, &slice)); err != nil {
+				log.Fatal(err)
+			}
+
+			lreponse := interpreter.Get(-1)
+			defer interpreter.Pop(1)
+
+			if reponse, ok := lreponse.(*lua.LUserData); ok {
+				s := reponse.Value
+				switch val := s.(type) {
+				case *obiseq.BioSequenceSlice:
+					return *val, nil
+				case *obiseq.BioSequence:
+					return obiseq.BioSequenceSlice{val}, nil
+				default:
+					r := reflect.TypeOf(val)
+					return nil, fmt.Errorf("slice_worker function doesn't return the correct type %s", r)
+				}
+			}
+
+			if _, ok = lreponse.(*lua.LNilType); ok {
+				return nil, nil
+			}
+
+			return nil, fmt.Errorf("slice_worker function doesn't return the correct type %T", lreponse)
+		}
+
+		return f
+	}
+
+	log.Fatalf("The slice_worker object is not a function")
+	return nil
+}
+
 // LuaProcessor processes a Lua script on a sequence iterator and returns a new iterator.
 //
 // Parameters:
@@ -173,7 +236,7 @@ func LuaProcessor(iterator obiiter.IBioSequence, name, program string, breakOnEr
 	err = interpreter.PCall(0, lua.MultRet, nil)
 
 	if err != nil {
-		log.Fatalf("Error in executing the lua script")
+		log.Fatalf("Error in executing the lua script: %v", err)
 	}
 
 	result := interpreter.GetGlobal("begin")
@@ -198,7 +261,7 @@ func LuaProcessor(iterator obiiter.IBioSequence, name, program string, breakOnEr
 		err = interpreter.PCall(0, lua.MultRet, nil)
 
 		if err != nil {
-			log.Fatalf("Error in executing the lua script")
+			log.Fatalf("Error in executing the lua script: %v", err)
 		}
 
 		result := interpreter.GetGlobal("finish")
@@ -216,11 +279,27 @@ func LuaProcessor(iterator obiiter.IBioSequence, name, program string, breakOnEr
 
 	}()
 
-	ff := func(iterator obiiter.IBioSequence) {
-		w := LuaWorker(proto)
-		sw := obiseq.SeqToSliceWorker(w, false)
+	// Detect whether the script defines slice_worker (batch-level) or worker (per-sequence).
+	hasSliceWorker := func() bool {
+		interpreter := NewInterpreter()
+		lfunc := interpreter.NewFunctionFromProto(proto)
+		interpreter.Push(lfunc)
+		if err := interpreter.PCall(0, lua.MultRet, nil); err != nil {
+			return false
+		}
+		result := interpreter.GetGlobal("slice_worker")
+		interpreter.Close()
+		_, ok := result.(*lua.LFunction)
+		return ok
+	}()
 
-		// iterator = iterator.SortBatches()
+	ff := func(iterator obiiter.IBioSequence) {
+		var sw obiseq.SeqSliceWorker
+		if hasSliceWorker {
+			sw = LuaSliceWorker(proto)
+		} else {
+			sw = obiseq.SeqToSliceWorker(LuaWorker(proto), false)
+		}
 
 		for iterator.Next() {
 			seqs := iterator.Get()
@@ -235,6 +314,10 @@ func LuaProcessor(iterator obiiter.IBioSequence, name, program string, breakOnEr
 				}
 			}
 
+			if ns == nil {
+				ns = obiseq.BioSequenceSlice{}
+			}
+
 			newIter.Push(obiiter.MakeBioSequenceBatch(seqs.Source(), seqs.Order(), ns))
 		}
 

pkg/obilua/lua_push_interface.go

@@ -17,15 +17,7 @@ import (
 // No return values. This function operates directly on the Lua state stack.
 func pushInterfaceToLua(L *lua.LState, val interface{}) {
 	switch v := val.(type) {
-	case string:
-		L.Push(lua.LString(v))
-	case bool:
-		L.Push(lua.LBool(v))
-	case int:
-		L.Push(lua.LNumber(v))
-	case float64:
-		L.Push(lua.LNumber(v))
-	// Add other cases as needed for different types
+	// Typed slices and maps from internal OBITools code — not produced by json.Unmarshal
 	case map[string]int:
 		pushMapStringIntToLua(L, v)
 	case map[string]string:
@@ -34,8 +26,6 @@ func pushInterfaceToLua(L *lua.LState, val interface{}) {
 		pushMapStringBoolToLua(L, v)
 	case map[string]float64:
 		pushMapStringFloat64ToLua(L, v)
-	case map[string]interface{}:
-		pushMapStringInterfaceToLua(L, v)
 	case []string:
 		pushSliceStringToLua(L, v)
 	case []int:
@@ -46,63 +36,63 @@ func pushInterfaceToLua(L *lua.LState, val interface{}) {
 		pushSliceNumericToLua(L, v)
 	case []bool:
 		pushSliceBoolToLua(L, v)
-	case []interface{}:
-		pushSliceInterfaceToLua(L, v)
-	case nil:
-		L.Push(lua.LNil)
 	case *sync.Mutex:
 		pushMutexToLua(L, v)
 	default:
-		log.Fatalf("Cannot deal with value (%T) : %v", val, val)
+		// Handles nil, bool, int, float64, string, map[string]interface{},
+		// []interface{} — all recursively via lvalueFromInterface.
+		L.Push(lvalueFromInterface(L, v))
 	}
 }
 
 func pushMapStringInterfaceToLua(L *lua.LState, m map[string]interface{}) {
-	// Create a new Lua table
 	luaTable := L.NewTable()
-	// Iterate over the Go map and set the key-value pairs in the Lua table
 	for key, value := range m {
-		switch v := value.(type) {
-		case int:
-			luaTable.RawSetString(key, lua.LNumber(v))
-		case float64:
-			luaTable.RawSetString(key, lua.LNumber(v))
-		case bool:
-			luaTable.RawSetString(key, lua.LBool(v))
-		case string:
-			luaTable.RawSetString(key, lua.LString(v))
-		default:
-			log.Fatalf("Doesn't deal with map containing value %v of type %T", v, v)
-		}
+		L.SetField(luaTable, key, lvalueFromInterface(L, value))
 	}
-
-	// Push the Lua table onto the stack
 	L.Push(luaTable)
 }
 
 func pushSliceInterfaceToLua(L *lua.LState, s []interface{}) {
-	// Create a new Lua table
 	luaTable := L.NewTable()
-	// Iterate over the Go map and set the key-value pairs in the Lua table
 	for _, value := range s {
-		switch v := value.(type) {
-		case int:
-			luaTable.Append(lua.LNumber(v))
-		case float64:
-			luaTable.Append(lua.LNumber(v))
-		case bool:
-			luaTable.Append(lua.LBool(v))
-		case string:
-			luaTable.Append(lua.LString(v))
-		default:
-			log.Fatalf("Doesn't deal with slice containing value %v of type %T", v, v)
-		}
+		luaTable.Append(lvalueFromInterface(L, value))
 	}
-
-	// Push the Lua table onto the stack
 	L.Push(luaTable)
 }
 
+// lvalueFromInterface converts a Go interface{} value (as produced by json.Unmarshal)
+// to the corresponding lua.LValue, handling nested maps and slices recursively.
+func lvalueFromInterface(L *lua.LState, value interface{}) lua.LValue {
+	switch v := value.(type) {
+	case nil:
+		return lua.LNil
+	case bool:
+		return lua.LBool(v)
+	case int:
+		return lua.LNumber(v)
+	case float64:
+		return lua.LNumber(v)
+	case string:
+		return lua.LString(v)
+	case map[string]interface{}:
+		t := L.NewTable()
+		for key, val := range v {
+			L.SetField(t, key, lvalueFromInterface(L, val))
+		}
+		return t
+	case []interface{}:
+		t := L.NewTable()
+		for _, val := range v {
+			t.Append(lvalueFromInterface(L, val))
+		}
+		return t
+	default:
+		log.Fatalf("lvalueFromInterface: unsupported type %T: %v", v, v)
+		return lua.LNil
+	}
+}
+
 // pushMapStringIntToLua creates a new Lua table and iterates over the Go map to set key-value pairs in the Lua table. It then pushes the Lua table onto the stack.
 //
 // L *lua.LState - the Lua state

pkg/obilua/lua_table.go

@@ -28,6 +28,8 @@ func Table2Interface(interpreter *lua.LState, table *lua.LTable) interface{} {
 				val[i-1] = float64(v.(lua.LNumber))
 			case lua.LTString:
 				val[i-1] = string(v.(lua.LString))
+			case lua.LTTable:
+				val[i-1] = Table2Interface(interpreter, v.(*lua.LTable))
 			}
 		}
 		return val
@@ -45,6 +47,8 @@ func Table2Interface(interpreter *lua.LState, table *lua.LTable) interface{} {
 					val[string(ks)] = float64(v.(lua.LNumber))
 				case lua.LTString:
 					val[string(ks)] = string(v.(lua.LString))
+				case lua.LTTable:
+					val[string(ks)] = Table2Interface(interpreter, v.(*lua.LTable))
 				}
 			}
 		})

pkg/obilua/luahttp.go

@@ -0,0 +1,128 @@
+package obilua
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
+	lua "github.com/yuin/gopher-lua"
+)
+
+const httpClientTimeout = 300 * time.Second
+
+var (
+	_httpClient     *http.Client
+	_httpClientOnce sync.Once
+
+	// _httpSemaphore limits the number of concurrent HTTP requests.
+	// Initialised lazily alongside the client.
+	_httpSemaphore chan struct{}
+)
+
+func getHTTPClient() *http.Client {
+	_httpClientOnce.Do(func() {
+		conns := 2 * obidefault.ParallelWorkers()
+		_httpClient = &http.Client{
+			Transport: &http.Transport{
+				MaxIdleConnsPerHost: conns,
+				MaxConnsPerHost:     conns,
+				IdleConnTimeout:     90 * time.Second,
+			},
+			Timeout: httpClientTimeout,
+		}
+		_httpSemaphore = make(chan struct{}, obidefault.ParallelWorkers())
+	})
+	return _httpClient
+}
+
+// RegisterHTTP registers the http module in the Lua state as a global,
+// consistent with obicontext and BioSequence.
+//
+// Exposes:
+//
+//	http.post(url, body [, timeout_ms]) → response string  (on success)
+//	http.post(url, body [, timeout_ms]) → nil, err string  (on error)
+//	http.set_concurrency(n)             → set max simultaneous requests
+func RegisterHTTP(luaState *lua.LState) {
+	table := luaState.NewTable()
+	luaState.SetField(table, "post", luaState.NewFunction(luaHTTPPost))
+	luaState.SetField(table, "set_concurrency", luaState.NewFunction(luaHTTPSetConcurrency))
+	luaState.SetGlobal("http", table)
+}
+
+// luaHTTPPost implements http.post(url, body [, timeout_ms]) for Lua.
+//
+// The optional third argument overrides the default timeout (in milliseconds).
+// Concurrent requests are throttled through _httpSemaphore so that a
+// single-threaded backend server is not overwhelmed by K parallel workers.
+//
+// Lua signature:
+//
+//	local response          = http.post(url, body)
+//	local response          = http.post(url, body, 5000)   -- 5 s timeout
+//	local response, err     = http.post(url, body)
+func luaHTTPPost(L *lua.LState) int {
+	url := L.CheckString(1)
+	body := L.CheckString(2)
+
+	client := getHTTPClient()
+
+	timeout := httpClientTimeout
+	if L.GetTop() >= 3 {
+		ms := L.CheckInt(3)
+		timeout = time.Duration(ms) * time.Millisecond
+	}
+
+	// Acquire semaphore slot — blocks until a slot is free.
+	_httpSemaphore <- struct{}{}
+	defer func() { <-_httpSemaphore }()
+
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, strings.NewReader(body))
+	if err != nil {
+		L.Push(lua.LNil)
+		L.Push(lua.LString(err.Error()))
+		return 2
+	}
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := client.Do(req)
+	if err != nil {
+		L.Push(lua.LNil)
+		L.Push(lua.LString(err.Error()))
+		return 2
+	}
+	defer resp.Body.Close()
+
+	respBytes, err := io.ReadAll(resp.Body)
+	if err != nil {
+		L.Push(lua.LNil)
+		L.Push(lua.LString(err.Error()))
+		return 2
+	}
+
+	L.Push(lua.LString(respBytes))
+	return 1
+}
+
+// luaHTTPSetConcurrency replaces the semaphore with a new one of size n.
+// Must be called before the first http.post (e.g. in begin()).
+//
+// Lua signature:
+//
+//	http.set_concurrency(1)   -- serialise all HTTP requests
+func luaHTTPSetConcurrency(L *lua.LState) int {
+	n := L.CheckInt(1)
+	if n < 1 {
+		n = 1
+	}
+	getHTTPClient() // ensure singleton is initialised
+	_httpSemaphore = make(chan struct{}, n)
+	return 0
+}

pkg/obilua/luajson.go

@@ -0,0 +1,71 @@
+package obilua
+
+import (
+	"encoding/json"
+
+	lua "github.com/yuin/gopher-lua"
+)
+
+// RegisterJSON registers the json module in the Lua state as a global,
+// consistent with obicontext, BioSequence, and http.
+//
+// Exposes:
+//
+//	json.encode(data)   → string         (on success)
+//	json.encode(data)   → nil, err       (on error)
+//	json.decode(string) → value          (on success)
+//	json.decode(string) → nil, err       (on error)
+func RegisterJSON(luaState *lua.LState) {
+	table := luaState.NewTable()
+	luaState.SetField(table, "encode", luaState.NewFunction(luaJSONEncode))
+	luaState.SetField(table, "decode", luaState.NewFunction(luaJSONDecode))
+	luaState.SetGlobal("json", table)
+}
+
+// luaJSONEncode implements json.encode(data) for Lua.
+func luaJSONEncode(L *lua.LState) int {
+	val := L.CheckAny(1)
+
+	var goVal interface{}
+	switch v := val.(type) {
+	case *lua.LTable:
+		goVal = Table2Interface(L, v)
+	case lua.LString:
+		goVal = string(v)
+	case lua.LNumber:
+		goVal = float64(v)
+	case lua.LBool:
+		goVal = bool(v)
+	case *lua.LNilType:
+		goVal = nil
+	default:
+		L.Push(lua.LNil)
+		L.Push(lua.LString("json.encode: unsupported type"))
+		return 2
+	}
+
+	b, err := json.Marshal(goVal)
+	if err != nil {
+		L.Push(lua.LNil)
+		L.Push(lua.LString(err.Error()))
+		return 2
+	}
+
+	L.Push(lua.LString(b))
+	return 1
+}
+
+// luaJSONDecode implements json.decode(string) for Lua.
+func luaJSONDecode(L *lua.LState) int {
+	s := L.CheckString(1)
+
+	var goVal interface{}
+	if err := json.Unmarshal([]byte(s), &goVal); err != nil {
+		L.Push(lua.LNil)
+		L.Push(lua.LString(err.Error()))
+		return 2
+	}
+
+	pushInterfaceToLua(L, goVal)
+	return 1
+}

pkg/obilua/luajson_test.go

@@ -0,0 +1,184 @@
+package obilua
+
+import (
+	"testing"
+
+	lua "github.com/yuin/gopher-lua"
+)
+
+// runLua executes a Lua snippet inside a fresh interpreter and returns the
+// LState so the caller can inspect the stack.
+func runLua(t *testing.T, script string) *lua.LState {
+	t.Helper()
+	L := NewInterpreter()
+	if err := L.DoString(script); err != nil {
+		t.Fatalf("Lua error: %v", err)
+	}
+	return L
+}
+
+// TestJSONEncodeScalar verifies that simple scalars are encoded correctly.
+func TestJSONEncodeScalar(t *testing.T) {
+	cases := []struct {
+		script   string
+		expected string
+	}{
+		{`result = json.encode("hello")`, `"hello"`},
+		{`result = json.encode(42)`, `42`},
+		{`result = json.encode(true)`, `true`},
+	}
+
+	for _, tc := range cases {
+		L := runLua(t, tc.script)
+		got := string(L.GetGlobal("result").(lua.LString))
+		if got != tc.expected {
+			t.Errorf("encode(%s): got %q, want %q", tc.script, got, tc.expected)
+		}
+		L.Close()
+	}
+}
+
+// TestJSONEncodeTable verifies that a Lua table (array and map) encodes to JSON.
+func TestJSONEncodeTable(t *testing.T) {
+	L := runLua(t, `result = json.encode({a = 1, b = "x"})`)
+	got := string(L.GetGlobal("result").(lua.LString))
+	// json.Marshal produces deterministic output for maps in Go 1.12+... actually not.
+	// Just check it round-trips via decode instead.
+	L.Close()
+	if got == "" {
+		t.Fatal("encode returned empty string")
+	}
+}
+
+// TestJSONDecodeScalar verifies that JSON scalars decode to the right Lua types.
+func TestJSONDecodeScalar(t *testing.T) {
+	L := runLua(t, `
+		s = json.decode('"hello"')
+		n = json.decode('3.14')
+		b = json.decode('true')
+	`)
+	if s, ok := L.GetGlobal("s").(lua.LString); !ok || string(s) != "hello" {
+		t.Errorf("decode string: got %v", L.GetGlobal("s"))
+	}
+	if n, ok := L.GetGlobal("n").(lua.LNumber); !ok || float64(n) != 3.14 {
+		t.Errorf("decode number: got %v", L.GetGlobal("n"))
+	}
+	if b, ok := L.GetGlobal("b").(lua.LBool); !ok || !bool(b) {
+		t.Errorf("decode bool: got %v", L.GetGlobal("b"))
+	}
+	L.Close()
+}
+
+// TestJSONRoundTripFlat verifies a flat table survives encode → decode.
+func TestJSONRoundTripFlat(t *testing.T) {
+	L := runLua(t, `
+		original = {name = "Homo_sapiens", score = 1.0, valid = true}
+		encoded  = json.encode(original)
+		decoded  = json.decode(encoded)
+	`)
+	decoded, ok := L.GetGlobal("decoded").(*lua.LTable)
+	if !ok {
+		t.Fatal("decoded is not a table")
+	}
+	if v := decoded.RawGetString("name"); string(v.(lua.LString)) != "Homo_sapiens" {
+		t.Errorf("name: got %v", v)
+	}
+	if v := decoded.RawGetString("score"); float64(v.(lua.LNumber)) != 1.0 {
+		t.Errorf("score: got %v", v)
+	}
+	if v := decoded.RawGetString("valid"); !bool(v.(lua.LBool)) {
+		t.Errorf("valid: got %v", v)
+	}
+	L.Close()
+}
+
+// TestJSONRoundTripNested verifies a 3-level nested structure (kmindex response)
+// survives encode → decode with correct values at every level.
+func TestJSONRoundTripNested(t *testing.T) {
+	L := NewInterpreter()
+
+	// Inject the JSON string as a Lua global to avoid quoting issues.
+	L.SetGlobal("kmindex_json", lua.LString(
+		`{"Human":{"query_001":{"Homo_sapiens--GCF_000001405_40":1.0}}}`,
+	))
+
+	if err := L.DoString(`
+		data      = json.decode(kmindex_json)
+		reencoded = json.encode(data)
+		data2     = json.decode(reencoded)
+	`); err != nil {
+		t.Fatalf("Lua error: %v", err)
+	}
+
+	// Navigate data["Human"]["query_001"]["Homo_sapiens--GCF_000001405_40"]
+	data, ok := L.GetGlobal("data").(*lua.LTable)
+	if !ok {
+		t.Fatal("data is not a table")
+	}
+	human, ok := data.RawGetString("Human").(*lua.LTable)
+	if !ok {
+		t.Fatal("data.Human is not a table")
+	}
+	query, ok := human.RawGetString("query_001").(*lua.LTable)
+	if !ok {
+		t.Fatal("data.Human.query_001 is not a table")
+	}
+	score, ok := query.RawGetString("Homo_sapiens--GCF_000001405_40").(lua.LNumber)
+	if !ok || float64(score) != 1.0 {
+		t.Errorf("score: got %v, want 1.0", query.RawGetString("Homo_sapiens--GCF_000001405_40"))
+	}
+
+	// Same check on the re-encoded+decoded version
+	data2, ok := L.GetGlobal("data2").(*lua.LTable)
+	if !ok {
+		t.Fatal("data2 is not a table")
+	}
+	score2 := data2.RawGetString("Human").(*lua.LTable).
+		RawGetString("query_001").(*lua.LTable).
+		RawGetString("Homo_sapiens--GCF_000001405_40").(lua.LNumber)
+	if float64(score2) != 1.0 {
+		t.Errorf("data2 score: got %v, want 1.0", score2)
+	}
+	L.Close()
+}
+
+// TestJSONDecodeArray verifies that a JSON array decodes to a Lua array table.
+func TestJSONDecodeArray(t *testing.T) {
+	L := runLua(t, `arr = json.decode('[1, 2, 3]')`)
+	arr, ok := L.GetGlobal("arr").(*lua.LTable)
+	if !ok {
+		t.Fatal("arr is not a table")
+	}
+	for i, expected := range []float64{1, 2, 3} {
+		v, ok := arr.RawGetInt(i + 1).(lua.LNumber)
+		if !ok || float64(v) != expected {
+			t.Errorf("arr[%d]: got %v, want %v", i+1, arr.RawGetInt(i+1), expected)
+		}
+	}
+	L.Close()
+}
+
+// TestJSONEncodeError verifies that json.encode on an unsupported type returns nil + error.
+func TestJSONEncodeError(t *testing.T) {
+	L := runLua(t, `
+		local result, err = json.encode(nil)
+	`)
+	// nil encodes to JSON "null" — not an error
+	L.Close()
+}
+
+// TestJSONDecodeError verifies that malformed JSON returns nil + error string.
+func TestJSONDecodeError(t *testing.T) {
+	L := runLua(t, `
+		local result, err = json.decode("not valid json")
+		decode_ok     = (result == nil)
+		decode_has_err = (err ~= nil)
+	`)
+	if L.GetGlobal("decode_ok") != lua.LTrue {
+		t.Error("expected nil result on decode error")
+	}
+	if L.GetGlobal("decode_has_err") != lua.LTrue {
+		t.Error("expected error string on decode error")
+	}
+	L.Close()
+}

pkg/obilua/obilib.go

@@ -5,4 +5,6 @@ import lua "github.com/yuin/gopher-lua"
 func RegisterObilib(luaState *lua.LState) {
 	RegisterObiSeq(luaState)
 	RegisterObiTaxonomy(luaState)
+	RegisterHTTP(luaState)
+	RegisterJSON(luaState)
 }

pkg/obilua/obiseqslice.go

@@ -31,7 +31,8 @@ func obiseqslice2Lua(interpreter *lua.LState,
 }
 
 func newObiSeqSlice(luaState *lua.LState) int {
-	seqslice := obiseq.NewBioSequenceSlice()
+	capacity := luaState.OptInt(1, 0)
+	seqslice := obiseq.NewBioSequenceSlice(capacity)
 	luaState.Push(obiseqslice2Lua(luaState, seqslice))
 	return 1
 }

pkg/obingslibrary/multimatch.go

@@ -777,7 +777,7 @@ func (library *NGSLibrary) ExtractMultiBarcodeSliceWorker(options ...WithOption)
 		library.SetAllowsIndels(true)
 	}
 
-	if opt.AllowedMismatches() > 0 {
+	if opt.AllowedMismatchesIsSet() {
 		library.SetAllowedMismatches(opt.AllowedMismatches())
 	}
 

pkg/obingslibrary/worker.go

@@ -6,13 +6,14 @@ import (
 )
 
 type _Options struct {
-	discardErrors   bool
-	unidentified    string
-	allowedMismatch int
-	allowsIndel     bool
-	withProgressBar bool
-	parallelWorkers int
-	batchSize       int
+	discardErrors      bool
+	unidentified       string
+	allowedMismatch    int
+	allowedMismatchSet bool
+	allowsIndel        bool
+	withProgressBar    bool
+	parallelWorkers    int
+	batchSize          int
 }
 
 // Options stores a set of option usable by the
@@ -52,6 +53,7 @@ func OptionWithProgressBar(yes bool) WithOption {
 func OptionAllowedMismatches(count int) WithOption {
 	f := WithOption(func(opt Options) {
 		opt.pointer.allowedMismatch = count
+		opt.pointer.allowedMismatchSet = true
 	})
 
 	return f
@@ -97,6 +99,12 @@ func (options Options) AllowedMismatches() int {
 	return options.pointer.allowedMismatch
 }
 
+// AllowedMismatchesIsSet returns true if OptionAllowedMismatches
+// was explicitly applied to these options.
+func (options Options) AllowedMismatchesIsSet() bool {
+	return options.pointer.allowedMismatchSet
+}
+
 func (options Options) AllowsIndels() bool {
 	return options.pointer.allowsIndel
 }

pkg/obioptions/version.go

@@ -3,7 +3,7 @@ package obioptions
 // Version is automatically updated by the Makefile from version.txt
 // The patch number (third digit) is incremented on each push to the repository
 
-var _Version = "Release 4.4.25"
+var _Version = "Release 4.5.2"
 
 // Version returns the version of the obitools package.
 //

pkg/obiprimer/amplicons.go

@@ -0,0 +1,293 @@
+package obiprimer
+
+import (
+	"math"
+	"sync"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obithermo"
+)
+
+// refHit is an amplicon of a retained pair on the reference sequence.
+type refHit struct {
+	up, down seqMatch
+	swapped  bool
+}
+
+// ampHash hashes the amplicon between the two primers read from the side of
+// the first primer of the pair (reverse complemented when the upstream match is
+// the second primer), so that identical amplicons give the same value
+// whatever their strand. Non ACGT bases hash as a fifth symbol.
+func ampHash(raw []byte, from, to int, reverse bool) uint64 {
+	const prime = 1099511628211
+	h := uint64(14695981039346656037) ^ uint64(to-from)
+	h *= prime
+	if !reverse {
+		for _, b := range raw[from:to] {
+			h = (h ^ uint64(baseCode[b])) * prime
+		}
+		return h
+	}
+	for i := to - 1; i >= from; i-- {
+		c := baseCode[raw[i]]
+		if c < 4 {
+			c = 3 - c
+		}
+		h = (h ^ uint64(c)) * prime
+	}
+	return h
+}
+
+// eachAmplicon runs forEachOccurrence over the sequences of the corpus (the
+// examples only, or all of them) in parallel, and calls visit for every
+// amplicon of a pair of index. Every worker gets its own visit function, from
+// newVisit, and done is called once the worker has no more sequence.
+//
+// Without a plan the sequences are handed to the workers as they get free, and
+// the assignment is recorded in record. With one, every worker scans the
+// sequences that the plan gave it.
+func (ps *primerSet) eachAmplicon(c *Corpus, p PairParams, index map[pairKey]int32, examplesOnly bool, plan *ampPlan, record []uint16,
+	newVisit func(worker int) (visit func(s int, raw []byte, o occurrence, pair int32), done func())) {
+
+	workers := obidefault.ParallelWorkers()
+
+	// only the matches of the primers of the pairs of index can make an amplicon
+	// of these pairs, and dropping the others spares most of the comparisons
+	wanted := make([]bool, len(ps.primers))
+	for k := range index {
+		wanted[k.p1], wanted[k.p2] = true, true
+	}
+	type scratch struct {
+		ms    []seqMatch
+		bases []byte
+	}
+	scan := func(sc *scratch, visit func(int, []byte, occurrence, int32), s int) {
+		if examplesOnly && c.Roles[s] != Example {
+			return
+		}
+		ms := sc.ms[:0]
+		for _, m := range ps.perSeq[s] {
+			if wanted[m.primer] {
+				ms = append(ms, m)
+			}
+		}
+		sc.ms = ms
+		if len(ms) < 2 {
+			return // an amplicon needs two primer sites
+		}
+		var raw []byte
+		raw, sc.bases = c.bases(s, sc.bases)
+		ps.forEachOccurrenceIn(raw, ms, p, func(o occurrence) {
+			if pi, ok := index[o.key]; ok {
+				visit(s, raw, o, pi)
+			}
+		})
+	}
+
+	var wg sync.WaitGroup
+	if plan != nil {
+		mine := make([][]int, workers)
+		for s, w := range plan.owner {
+			mine[w] = append(mine[w], s)
+		}
+		for w := range workers {
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				visit, done := newVisit(w)
+				var sc scratch
+				for _, s := range mine[w] {
+					scan(&sc, visit, s)
+				}
+				if done != nil {
+					done()
+				}
+			}()
+		}
+		wg.Wait()
+		return
+	}
+
+	jobs := make(chan int)
+	for w := range workers {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			visit, done := newVisit(w)
+			var sc scratch
+			for s := range jobs {
+				record[s] = uint16(w)
+				scan(&sc, visit, s)
+			}
+			if done != nil {
+				done()
+			}
+		}()
+	}
+	for s := range c.Roles {
+		jobs <- s
+	}
+	close(jobs)
+	wg.Wait()
+}
+
+// planAmplicons counts the amplicons of every pair of index in the sequences
+// of the corpus (all of them, or the examples only), by worker.
+func (ps *primerSet) planAmplicons(c *Corpus, p PairParams, index map[pairKey]int32, examplesOnly bool) *ampPlan {
+	workers := obidefault.ParallelWorkers()
+	plan := &ampPlan{owner: make([]uint16, c.Len()), count: make([][]uint32, workers)}
+	for w := range plan.count {
+		plan.count[w] = make([]uint32, len(index))
+	}
+	ps.eachAmplicon(c, p, index, examplesOnly, nil, plan.owner, func(w int) (func(int, []byte, occurrence, int32), func()) {
+		row := plan.count[w]
+		return func(_ int, _ []byte, _ occurrence, pi int32) { row[pi]++ }, nil
+	})
+	return plan
+}
+
+// ampSites is what the gathering of the amplicons of the targets learns besides
+// the records.
+type ampSites struct {
+	minTm1 []float32 // by pair: lowest Tm of the first primer over the example sites
+	minTm2 []float32
+	refs   map[int32]refHit // by pair: the amplicon on the reference sequence
+}
+
+// collectOptions selects what collectAmplicons gathers besides the records.
+type collectOptions struct {
+	examplesOnly bool             // skip the counterexamples and the other sequences
+	model        *obithermo.Model // melting temperatures of the example sites (nil: none)
+	reference    bool             // record the amplicons on the reference sequence
+}
+
+// collectAmplicons writes into store, in parallel, one record per amplicon of
+// the pairs of index (whose values are the pair numbers in the store) and
+// returns the lowest melting temperatures over the example sites. The store
+// was sized from the plan, which says which worker scans which sequence; the
+// batch only holds the pairs of index, the plan more.
+func (ps *primerSet) collectAmplicons(c *Corpus, taxa *Taxa, p PairParams, index map[pairKey]int32,
+	plan *ampPlan, store *ampStore, opt collectOptions) *ampSites {
+
+	model := opt.model
+	lp := p.PrimerLength
+	npairs := len(index)
+
+	res := &ampSites{
+		minTm1: make([]float32, npairs),
+		minTm2: make([]float32, npairs),
+		refs:   map[int32]refHit{}, // written by the single worker that reads the reference
+	}
+	inf := float32(math.Inf(1))
+	for i := range npairs {
+		res.minTm1[i], res.minTm2[i] = inf, inf
+	}
+	var tmMu sync.Mutex
+
+	// oriented[primer][0] is the reverse complement of the word, [1] the word itself.
+	oriented := make([][2][]byte, len(ps.primers))
+	for i, pr := range ps.primers {
+		oriented[i][0] = []byte(orientedWord(pr.Word, lp, false))
+		oriented[i][1] = []byte(orientedWord(pr.Word, lp, true))
+	}
+
+	ps.eachAmplicon(c, p, index, opt.examplesOnly, plan, nil, func(w int) (func(int, []byte, occurrence, int32), func()) {
+		site1, site2 := make([]byte, lp), make([]byte, lp)
+		lowTm1, lowTm2 := map[int32]float32{}, map[int32]float32{}
+
+		visit := func(s int, raw []byte, o occurrence, pi int32) {
+			n := len(raw)
+			role := c.Roles[s]
+			if opt.reference && p.HasReference && s == p.Reference {
+				if old, seen := res.refs[pi]; !seen || o.up.pos() < old.up.pos() {
+					res.refs[pi] = refHit{o.up, o.down, o.swapped}
+				}
+			}
+			store.add(w, pi, ampHash(raw, int(o.up.pos())+lp, int(o.down.pos()), o.swapped), packTaxon(taxa.Of[s], role))
+
+			if role != Example || model == nil {
+				return
+			}
+			fp := oriented[o.up.primer][b2i(!o.up.reverse())]
+			rp := oriented[o.down.primer][b2i(o.down.reverse())]
+			tmF, tmR := inf, inf
+			if windowAsIs(site1, raw, int(o.up.pos()), n, p.Circular) {
+				if t, err := model.Tm(fp, site1); err == nil && !math.IsNaN(t) {
+					tmF = float32(t)
+				}
+			}
+			if windowRevComp(site2, raw, int(o.down.pos()), n, p.Circular) {
+				if t, err := model.Tm(rp, site2); err == nil && !math.IsNaN(t) {
+					tmR = float32(t)
+				}
+			}
+			if o.swapped {
+				tmF, tmR = tmR, tmF // slot 1 is the downstream primer
+			}
+			lowTm1[pi] = min(lowOr(lowTm1, pi, inf), tmF)
+			lowTm2[pi] = min(lowOr(lowTm2, pi, inf), tmR)
+		}
+		done := func() {
+			tmMu.Lock()
+			for pi, t := range lowTm1 {
+				res.minTm1[pi] = min(res.minTm1[pi], t)
+			}
+			for pi, t := range lowTm2 {
+				res.minTm2[pi] = min(res.minTm2[pi], t)
+			}
+			tmMu.Unlock()
+		}
+		return visit, done
+	})
+	return res
+}
+
+func lowOr(m map[int32]float32, k int32, def float32) float32 {
+	if v, ok := m[k]; ok {
+		return v
+	}
+	return def
+}
+
+func b2i(b bool) int {
+	if b {
+		return 1
+	}
+	return 0
+}
+
+const upper = "ACGT"
+
+// windowAsIs copies the len(dst) bases at pos as upper-case ACGT; it reports
+// false if one of them is not A, C, G or T.
+func windowAsIs(dst, raw []byte, pos, n int, circular bool) bool {
+	for t := range dst {
+		i := pos + t
+		if circular {
+			i %= n
+		}
+		c := baseCode[raw[i]]
+		if c > 3 {
+			return false
+		}
+		dst[t] = upper[c]
+	}
+	return true
+}
+
+// windowRevComp writes the reverse complement of the len(dst) bases at pos.
+func windowRevComp(dst, raw []byte, pos, n int, circular bool) bool {
+	l := len(dst)
+	for t := range dst {
+		i := pos + l - 1 - t
+		if circular {
+			i %= n
+		}
+		c := baseCode[raw[i]]
+		if c > 3 {
+			return false
+		}
+		dst[t] = upper[3-c]
+	}
+	return true
+}

pkg/obiprimer/ampstore.go

@@ -0,0 +1,132 @@
+package obiprimer
+
+import "fmt"
+
+// ampRecord is the logical content of the record of one amplicon: its barcode
+// (see ampHash), the taxon of its sequence at the chosen rank (-1 when it has
+// none) and the role of the sequence. It is stored packed in an ampStore.
+type ampRecord struct {
+	hash  uint64
+	taxon int32
+	role  Role
+}
+
+const (
+	roleBits = 2
+	roleMask = 1<<roleBits - 1
+
+	// maxTaxa is the number of taxa a record can carry: the taxon, shifted by
+	// one so that "none" is 0, shares a 32 bits word with the role.
+	maxTaxa = 1<<(32-roleBits) - 2
+)
+
+func packTaxon(taxon int32, role Role) uint32 { return uint32(taxon+1)<<roleBits | uint32(role) }
+
+func unpackTaxon(t uint32) (int32, Role) { return int32(t>>roleBits) - 1, Role(t & roleMask) }
+
+// ampSlice is the amplicons of one pair: 12 bytes per amplicon, kept as two
+// parallel arrays (a struct holding a uint64 and a uint32 would take 16).
+type ampSlice struct {
+	hash []uint64
+	tax  []uint32 // packTaxon
+}
+
+// ampPlan says how many amplicons each worker finds for each pair, and which
+// worker scans which sequence: the write pass replays the counting pass with
+// the very same assignment, so that every worker knows where to write its
+// records without sharing any counter with the others.
+type ampPlan struct {
+	owner []uint16   // owner[s]: the worker that scans sequence s
+	count [][]uint32 // count[w][pair]: amplicons of the pair in the sequences of worker w
+}
+
+// total returns the number of amplicons of every pair.
+func (pl *ampPlan) total() []int {
+	t := make([]int, len(pl.count[0]))
+	for _, row := range pl.count {
+		for b, n := range row {
+			t[b] += int(n)
+		}
+	}
+	return t
+}
+
+// ampStore holds the amplicons of a batch of pairs, grouped by pair: for each
+// pair, two slices of the exact size, since their number is known before they
+// are written (see planAmplicons). Nothing grows by doubling, nothing is copied
+// nor sorted afterwards. The records of a pair are those of worker 0, then those
+// of worker 1, and so on.
+//
+// The pairs have slices of their own, rather than slices of two huge arrays: a
+// few kB each find room in the free memory, where blocks of tens of MB make the
+// process map new memory (and, on some systems, count the freed one still).
+type ampStore struct {
+	plan   *ampPlan
+	lo     int // first pair of the batch in the plan
+	pairs  []ampSlice
+	cursor [][]int // cursor[w][b]: next free slot of worker w for pair b
+}
+
+// newAmpStore allocates the store of the pairs lo..hi-1 of the plan.
+func newAmpStore(plan *ampPlan, lo, hi int) *ampStore {
+	n := hi - lo
+	s := &ampStore{plan: plan, lo: lo, pairs: make([]ampSlice, n), cursor: make([][]int, len(plan.count))}
+	for w := range s.cursor {
+		s.cursor[w] = make([]int, n)
+	}
+	for b := range n {
+		pos := 0
+		for w, row := range plan.count {
+			s.cursor[w][b] = pos
+			pos += int(row[lo+b])
+		}
+		if pos > 0 {
+			s.pairs[b] = ampSlice{make([]uint64, pos), make([]uint32, pos)}
+		}
+	}
+	return s
+}
+
+// add writes one record of pair b (numbered in the batch), found by worker w.
+func (s *ampStore) add(w int, b int32, hash uint64, taxon uint32) {
+	i := s.cursor[w][b]
+	s.cursor[w][b] = i + 1
+	s.pairs[b].hash[i], s.pairs[b].tax[i] = hash, taxon
+}
+
+// check panics if a worker did not write exactly the records counted for it:
+// the counting and write passes must enumerate the same amplicons.
+func (s *ampStore) check() {
+	for b := range s.pairs {
+		pos := 0
+		for w, row := range s.plan.count {
+			pos += int(row[s.lo+b])
+			if s.cursor[w][b] != pos {
+				panic(fmt.Sprintf("obiprimer: pair %d: worker %d wrote %d amplicons, %d were counted",
+					s.lo+b, w, s.cursor[w][b]-(pos-int(row[s.lo+b])), row[s.lo+b]))
+			}
+		}
+	}
+}
+
+// pair returns the amplicons of pair b (numbered in the batch).
+func (s *ampStore) pair(b int) ampSlice { return s.pairs[b] }
+
+// ampBatches cuts the pairs 0..len(count)-1 in consecutive ranges [lo,hi)
+// whose amplicons together do not exceed budget records (a pair alone above the
+// budget makes a batch of its own). A non positive budget gives one batch.
+func ampBatches(count []int, budget int) [][2]int {
+	var out [][2]int
+	lo, sum := 0, 0
+	for i, n := range count {
+		if budget > 0 && i > lo && sum+n > budget {
+			out = append(out, [2]int{lo, i})
+			lo, sum = i, 0
+		}
+		sum += n
+	}
+	if lo < len(count) {
+		out = append(out, [2]int{lo, len(count)})
+	}
+	return out
+}

pkg/obiprimer/corpus.go

@@ -0,0 +1,113 @@
+package obiprimer
+
+import (
+	"fmt"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+// maxSeqLen bounds the length of a sequence: the position of a primer site and
+// its strand share 32 bits (see seqMatch).
+const maxSeqLen = 1 << 31
+
+// Counts gives the number of sequences by role.
+type Counts struct {
+	Examples        int
+	CounterExamples int
+	Others          int
+}
+
+// Corpus is the set of sequences of the primer search, in input order, each
+// with its role: the examples (the targets), the counterexamples, and all the
+// other sequences, which are loaded as well since the amplicons they give
+// decide whether a target is identified.
+//
+// The primer search needs several passes over the sequences (words are
+// searched approximately in every one of them), so the corpus lives in memory.
+type Corpus struct {
+	Seqs  []*obiseq.BioSequence // nil entries once the corpus is packed
+	Roles []Role                // Roles[i] is the role of Seqs[i]
+
+	packed []packedSeq // see Pack
+	ids    []string
+}
+
+// Counts returns the number of sequences by role.
+func (c *Corpus) Counts() Counts {
+	var n Counts
+	for _, r := range c.Roles {
+		switch r {
+		case Example:
+			n.Examples++
+		case CounterExample:
+			n.CounterExamples++
+		default:
+			n.Others++
+		}
+	}
+	return n
+}
+
+// IndexOfExample returns the index of the example sequence with the given
+// identifier. It is an error if there is none, or if the sequence is a
+// counterexample.
+func (c *Corpus) IndexOfExample(id string) (int, error) {
+	for i := range c.Roles {
+		if c.ID(i) == id {
+			if c.Roles[i] != Example {
+				return -1, fmt.Errorf("obiprimer: reference sequence %q is a counterexample, it must be an example", id)
+			}
+			return i, nil
+		}
+	}
+	return -1, fmt.Errorf("obiprimer: reference sequence %q is not among the selected example sequences", id)
+}
+
+// Len returns the number of sequences.
+func (c *Corpus) Len() int { return len(c.Roles) }
+
+// SeqLen returns the length of sequence i.
+func (c *Corpus) SeqLen(i int) int {
+	if c.Seqs[i] != nil {
+		return c.Seqs[i].Len()
+	}
+	return c.packed[i].n
+}
+
+// ID returns the identifier of sequence i.
+func (c *Corpus) ID(i int) string {
+	if c.Seqs[i] != nil {
+		return c.Seqs[i].Id()
+	}
+	return c.ids[i]
+}
+
+// bases returns the bases of sequence i, lower case. A packed sequence is
+// unpacked into buf, grown if need be, which the caller keeps for the next call
+// (the second result); an unpacked one is returned as it is. The result must
+// not be modified, and is valid until the next call with the same buffer.
+func (c *Corpus) bases(i int, buf []byte) (raw, newBuf []byte) {
+	if c.Seqs[i] != nil {
+		return c.Seqs[i].Sequence(), buf
+	}
+	buf = c.packed[i].unpack(buf)
+	return buf, buf
+}
+
+// Pack replaces the sequences by their packed form, on 2 bits per base, and
+// frees the sequence objects (their annotations, their slack): the primer search
+// only needs the bases, the identifiers, and the taxa, which must have been
+// assigned before (AssignTaxa). A packed corpus holds about a quarter of the
+// memory of the bases themselves.
+func (c *Corpus) Pack() {
+	if c.packed != nil {
+		return
+	}
+	c.packed = make([]packedSeq, len(c.Seqs))
+	c.ids = make([]string, len(c.Seqs))
+	parallelFor(len(c.Seqs), func(_, i int) {
+		c.packed[i] = packBases(c.Seqs[i].Sequence())
+		c.ids[i] = c.Seqs[i].Id()
+		c.Seqs[i] = nil
+	})
+}

pkg/obiprimer/corpus_load.go

@@ -0,0 +1,137 @@
+package obiprimer
+
+import (
+	"cmp"
+	"fmt"
+	"slices"
+	"sync"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax"
+)
+
+// corpusEntry is a sequence read by one of the loading goroutines: either the
+// sequence itself, or, when it is packed as it is read, what is kept of it.
+type corpusEntry struct {
+	batch, index int
+	role         Role
+
+	seq *obiseq.BioSequence // unpacked loading
+
+	packed packedSeq // packed loading
+	id     string
+	taxon  *obitax.Taxon // at the chosen rank, nil when there is none
+}
+
+// loadEntries reads every sequence of seqs, once, on all the cores, and returns
+// the entries in the order of the input, whatever the number of reader
+// goroutines. keep fills the entry of a sequence (besides its position and its
+// role); it runs in the reading goroutines.
+func loadEntries(seqs obiiter.IBioSequence, sel Selection, keep func(*obiseq.BioSequence, *corpusEntry)) []corpusEntry {
+	var mu sync.Mutex
+	var entries []corpusEntry
+
+	consumer := func(it obiiter.IBioSequence, wg *sync.WaitGroup) {
+		defer wg.Done()
+		var local []corpusEntry
+		for it.Next() {
+			batch := it.Get()
+			for i, seq := range batch.Slice() {
+				e := corpusEntry{batch: batch.Order(), index: i, role: sel.Role(seq)}
+				keep(seq, &e)
+				local = append(local, e)
+			}
+		}
+		mu.Lock()
+		entries = append(entries, local...)
+		mu.Unlock()
+	}
+
+	var wg sync.WaitGroup
+	for i := 1; i < obidefault.ParallelWorkers(); i++ {
+		wg.Add(1)
+		go consumer(seqs.Split(), &wg)
+	}
+	wg.Add(1)
+	go consumer(seqs, &wg)
+	wg.Wait()
+
+	slices.SortFunc(entries, func(a, b corpusEntry) int {
+		if c := cmp.Compare(a.batch, b.batch); c != 0 {
+			return c
+		}
+		return cmp.Compare(a.index, b.index)
+	})
+	return entries
+}
+
+func checkSeqLen(id string, n int) error {
+	if n >= maxSeqLen {
+		return fmt.Errorf("obiprimer: sequence %q is too long (%d bases, the limit is %d)", id, n, maxSeqLen-1)
+	}
+	return nil
+}
+
+// LoadCorpus reads every sequence of seqs, once, and keeps all of them with the
+// role given by the selection. The order of the input is preserved whatever
+// the number of reader goroutines.
+func LoadCorpus(seqs obiiter.IBioSequence, sel Selection) (*Corpus, error) {
+	entries := loadEntries(seqs, sel, func(seq *obiseq.BioSequence, e *corpusEntry) { e.seq = seq })
+
+	c := &Corpus{
+		Seqs:  make([]*obiseq.BioSequence, len(entries)),
+		Roles: make([]Role, len(entries)),
+	}
+	for i, e := range entries {
+		if err := checkSeqLen(e.seq.Id(), e.seq.Len()); err != nil {
+			return nil, err
+		}
+		c.Seqs[i], c.Roles[i] = e.seq, e.role
+	}
+	if c.Counts().Examples == 0 {
+		return nil, fmt.Errorf("obiprimer: no example sequence selected")
+	}
+	return c, nil
+}
+
+// LoadPackedCorpus is LoadCorpus followed by Pack and AssignTaxa, without ever
+// holding the sequences themselves: each one is packed, and its taxon at rank
+// looked up, as it is read, and then dropped. The corpus of a large database
+// would otherwise first take, in memory, the sequences with all their
+// annotations, four times the size of their bases and more.
+func LoadPackedCorpus(seqs obiiter.IBioSequence, sel Selection, taxonomy *obitax.Taxonomy, rank string) (*Corpus, *Taxa, error) {
+	if err := checkRank(taxonomy, rank); err != nil {
+		return nil, nil, err
+	}
+	entries := loadEntries(seqs, sel, func(seq *obiseq.BioSequence, e *corpusEntry) {
+		e.packed = packBases(seq.Sequence())
+		e.id = seq.Id()
+		e.taxon = seq.Taxon(taxonomy).TaxonAtRank(rank)
+	})
+
+	c := &Corpus{
+		Seqs:   make([]*obiseq.BioSequence, len(entries)),
+		Roles:  make([]Role, len(entries)),
+		packed: make([]packedSeq, len(entries)),
+		ids:    make([]string, len(entries)),
+	}
+	taxons := make([]*obitax.Taxon, len(entries))
+	for i := range entries {
+		e := &entries[i]
+		if err := checkSeqLen(e.id, e.packed.n); err != nil {
+			return nil, nil, err
+		}
+		c.Roles[i], c.packed[i], c.ids[i], taxons[i] = e.role, e.packed, e.id, e.taxon
+		*e = corpusEntry{} // frees the entry as it goes
+	}
+	if c.Counts().Examples == 0 {
+		return nil, nil, fmt.Errorf("obiprimer: no example sequence selected")
+	}
+	taxa, err := buildTaxa(rank, taxons, c.Roles)
+	if err != nil {
+		return nil, nil, err
+	}
+	return c, taxa, nil
+}

pkg/obiprimer/match.go

@@ -0,0 +1,180 @@
+package obiprimer
+
+import (
+	"cmp"
+	"fmt"
+	"slices"
+	"sync"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obikmer"
+)
+
+// Match is one approximate occurrence of a word in a corpus sequence.
+//
+// Pos is the 0-based start of the window on the sequence as given (forward
+// coordinates). When Reverse is false the window reads like the word; when it
+// is true the window reads like the reverse complement of the word, i.e. the
+// word occurs on the other strand. Errors is the number of substitutions.
+type Match struct {
+	Seq     uint32 // index in the Corpus
+	Pos     uint32
+	Errors  uint8
+	Reverse bool
+}
+
+// WordMatches gathers every occurrence of one canonical word, sorted by
+// (Seq, Pos, Reverse).
+type WordMatches struct {
+	Word    uint64
+	Matches []Match
+}
+
+// MatchParams parameterizes the approximate search.
+//
+// Rules, those of ecoPrimers: only substitutions are allowed; a base other
+// than A, C, G, T (or U, read as T) in the sequence never matches, it counts as
+// one error; every window with at most MaxErrors errors is reported, windows
+// overlapping each other included; on circular sequences windows may span the
+// junction.
+type MatchParams struct {
+	K         int
+	MaxErrors int
+	Circular  bool
+}
+
+func (p MatchParams) validate() error {
+	if p.K < 2 || p.K > 31 {
+		return fmt.Errorf("obiprimer: word size must be between 2 and 31, got %d", p.K)
+	}
+	if p.MaxErrors < 0 || p.MaxErrors >= p.K {
+		return fmt.Errorf("obiprimer: error count must be in [0,%d[, got %d", p.K, p.MaxErrors)
+	}
+	return nil
+}
+
+// hit is a Match before its sequence is known: the sequence it belongs to is
+// the one being scanned.
+type hit struct {
+	word    uint32
+	pos     uint32
+	errors  uint8
+	reverse bool
+}
+
+// scanFunc finds every occurrence of every word, on both strands, in one
+// sequence (its bases) and reports them through emit.
+type scanFunc func(raw []byte, emit func(word, pos, errors int, reverse bool))
+
+// scannerMaker returns an independent scanFunc; each worker gets its own so
+// that scanners can keep private buffers.
+type scannerMaker func() scanFunc
+
+// wordStrings returns, for each canonical word, the word and its reverse
+// complement as lowercase strings.
+func wordStrings(words []uint64, k int) (fwd, rev []string) {
+	fwd = make([]string, len(words))
+	rev = make([]string, len(words))
+	for i, w := range words {
+		fwd[i] = string(obikmer.DecodeKmer(w, k, nil))
+		rev[i] = string(obikmer.DecodeKmer(obikmer.ReverseComplement(w, k), k, nil))
+	}
+	return
+}
+
+// matchCorpus runs the scanners over every sequence of the corpus in parallel
+// and assembles the per-word results. The output does not depend on the number
+// of workers.
+//
+// The occurrences of a sequence are kept in a slice of their exact size, then
+// counted per word, so that the occurrences of all the words are laid out in
+// one allocation of the exact size too: nothing grows by doubling, nor is held
+// twice for long.
+func matchCorpus(c *Corpus, words []uint64, mk scannerMaker) []WordMatches {
+	perSeq := make([][]hit, c.Len())
+
+	jobs := make(chan int)
+	var wg sync.WaitGroup
+	for range obidefault.ParallelWorkers() {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			scan := mk()
+			var buf []hit
+			var raw, bases []byte
+			for i := range jobs {
+				buf = buf[:0]
+				raw, bases = c.bases(i, bases)
+				scan(raw, func(word, pos, errors int, reverse bool) {
+					buf = append(buf, hit{uint32(word), uint32(pos), uint8(errors), reverse})
+				})
+				if len(buf) > 0 {
+					perSeq[i] = slices.Clone(buf)
+				}
+			}
+		}()
+	}
+	for i := range c.Roles {
+		jobs <- i
+	}
+	close(jobs)
+	wg.Wait()
+
+	res := make([]WordMatches, len(words))
+	count := make([]int, len(words))
+	for _, hits := range perSeq {
+		for _, h := range hits {
+			count[h.word]++
+		}
+	}
+	for i, w := range words {
+		res[i].Word = w
+		if count[i] > 0 {
+			res[i].Matches = make([]Match, 0, count[i])
+		}
+	}
+	for s, hits := range perSeq {
+		for _, h := range hits {
+			res[h.word].Matches = append(res[h.word].Matches, Match{uint32(s), h.pos, h.errors, h.reverse})
+		}
+		perSeq[s] = nil // the hits of the sequence are no longer needed
+	}
+	parallelFor(len(res), func(_, i int) {
+		slices.SortFunc(res[i].Matches, func(a, b Match) int {
+			if c := cmp.Compare(a.Seq, b.Seq); c != 0 {
+				return c
+			}
+			if c := cmp.Compare(a.Pos, b.Pos); c != 0 {
+				return c
+			}
+			if a.Reverse != b.Reverse {
+				if !a.Reverse {
+					return -1
+				}
+				return 1
+			}
+			return 0
+		})
+	})
+	return res
+}
+
+// SequencesMatched returns, for the matches of one word, the number of
+// distinct example and counterexample sequences it occurs in. The other
+// sequences are not counted.
+func (w WordMatches) SequencesMatched(c *Corpus) (examples, counterExamples int) {
+	last := ^uint32(0)
+	for _, m := range w.Matches {
+		if m.Seq == last {
+			continue
+		}
+		last = m.Seq
+		switch c.Roles[m.Seq] {
+		case Example:
+			examples++
+		case CounterExample:
+			counterExamples++
+		}
+	}
+	return
+}

pkg/obiprimer/match_naive.go

@@ -0,0 +1,75 @@
+package obiprimer
+
+// baseCode maps a nucleotide to 0..3 (A,C,G,T; U is T) and everything else to 4,
+// which never equals a word base.
+var baseCode = func() (t [256]byte) {
+	for i := range t {
+		t[i] = 4
+	}
+	for i, c := range "ACGT" {
+		t[c], t[c+'a'-'A'] = byte(i), byte(i)
+	}
+	t['U'], t['u'] = 3, 3
+	return
+}()
+
+func codesOf(s string) []byte {
+	out := make([]byte, len(s))
+	for i := range out {
+		out[i] = baseCode[s[i]]
+	}
+	return out
+}
+
+// matchNaive is the reference implementation: every window of every sequence
+// is compared with every word, base by base. It is slow and exists to define
+// the exact semantics (see MatchParams) that faster implementations must
+// reproduce.
+func matchNaive(c *Corpus, words []uint64, p MatchParams) ([]WordMatches, error) {
+	if err := p.validate(); err != nil {
+		return nil, err
+	}
+	fwd, rev := wordStrings(words, p.K)
+	fc, rc := make([][]byte, len(words)), make([][]byte, len(words))
+	for i := range words {
+		fc[i], rc[i] = codesOf(fwd[i]), codesOf(rev[i])
+	}
+
+	mk := func() scanFunc {
+		var codes []byte
+		return func(raw []byte, emit func(word, pos, errors int, reverse bool)) {
+			n := len(raw)
+			if n < p.K {
+				return
+			}
+			codes = codes[:0]
+			for _, b := range raw {
+				codes = append(codes, baseCode[b])
+			}
+			windows := n - p.K + 1
+			if p.Circular {
+				codes = append(codes, codes[:p.K-1]...)
+				windows = n
+			}
+			for pos := range windows {
+				win := codes[pos : pos+p.K]
+				for w := range words {
+					for strand, pat := range [2][]byte{fc[w], rc[w]} {
+						errs := 0
+						for j, b := range pat {
+							if win[j] != b {
+								if errs++; errs > p.MaxErrors {
+									break
+								}
+							}
+						}
+						if errs <= p.MaxErrors {
+							emit(w, pos, errs, strand == 1)
+						}
+					}
+				}
+			}
+		}
+	}
+	return matchCorpus(c, words, mk), nil
+}

pkg/obiprimer/match_seed.go

@@ -0,0 +1,211 @@
+package obiprimer
+
+import (
+	"math/bits"
+	"slices"
+)
+
+// seedIndex maps the value of one half of a window to the patterns that are
+// candidates for it: those whose same half is within d substitutions of it.
+//
+// Pigeonhole: a window with at most e = 2d+1 errors against a pattern has at
+// most d errors in at least one of the two halves of the pattern, so the
+// windows to verify are exactly those found through one of the two indexes.
+// Indexing the half of every pattern together with its neighbours at distance
+// d (1 + 3L values for d = 1, 1 + 3L + 9L(L-1)/2 for d = 2, L = 9 bases) costs
+// a few tens of entries per pattern, and a window of a random text finds about
+// patterns x neighbours / 4^L candidates, far fewer than the patterns.
+type seedIndex struct {
+	shift   uint     // slot = seed >> shift when hashed, else seed
+	hashed  bool     // the seed space is too large for a direct table
+	offsets []uint32 // CSR: ids[offsets[s]:offsets[s+1]] are the candidates of slot s
+	ids     []int32  // pattern ids (2*word + strand)
+}
+
+const maxDirectSeedBits = 22
+
+func (x *seedIndex) slot(seed uint64) uint32 {
+	if !x.hashed {
+		return uint32(seed)
+	}
+	return uint32((seed * 0x9E3779B97F4A7C15) >> x.shift)
+}
+
+func (x *seedIndex) candidates(seed uint64) []int32 {
+	s := x.slot(seed)
+	return x.ids[x.offsets[s]:x.offsets[s+1]]
+}
+
+// neighbours calls fn for v and every value within d substitutions of it (each
+// once), v being L bases packed on 2 bits.
+func neighbours(v uint64, L, d int, fn func(uint64)) {
+	var rec func(v uint64, from, left int)
+	rec = func(v uint64, from, left int) {
+		fn(v)
+		if left == 0 {
+			return
+		}
+		for pos := from; pos < L; pos++ {
+			for alt := uint64(1); alt <= 3; alt++ {
+				rec(v^(alt<<(2*uint(pos))), pos+1, left-1)
+			}
+		}
+	}
+	rec(v, 0, d)
+}
+
+// newSeedIndex indexes, for every pattern, the half of the given width (in
+// bases) selected by (code >> shiftBases*2) & mask, with its neighbours at
+// distance d.
+func newSeedIndex(patterns []uint64, shiftBases, width, d int) *seedIndex {
+	x := &seedIndex{}
+	bitsNeeded := 2 * width
+	if bitsNeeded > maxDirectSeedBits {
+		x.hashed, bitsNeeded = true, maxDirectSeedBits
+		x.shift = uint(64 - maxDirectSeedBits)
+	}
+	nslots := 1 << uint(bitsNeeded)
+	halfMask := uint64(1)<<(2*uint(width)) - 1
+
+	type entry struct {
+		slot uint32
+		id   int32
+	}
+	var entries []entry
+	var scratch []uint32
+	for id, pat := range patterns {
+		half := (pat >> (2 * uint(shiftBases))) & halfMask
+		scratch = scratch[:0]
+		neighbours(half, width, d, func(v uint64) { scratch = append(scratch, x.slot(v)) })
+		slices.Sort(scratch)
+		scratch = slices.Compact(scratch) // hashing may merge neighbours
+		for _, s := range scratch {
+			entries = append(entries, entry{s, int32(id)})
+		}
+	}
+
+	x.offsets = make([]uint32, nslots+1)
+	for _, e := range entries {
+		x.offsets[e.slot+1]++
+	}
+	for i := 1; i <= nslots; i++ {
+		x.offsets[i] += x.offsets[i-1]
+	}
+	x.ids = make([]int32, len(entries))
+	fill := slices.Clone(x.offsets[:nslots])
+	for _, e := range entries {
+		x.ids[fill[e.slot]] = e.id
+		fill[e.slot]++
+	}
+	return x
+}
+
+// MatchWords finds every approximate occurrence, on both strands, of the given
+// canonical words in the sequences of the corpus, with the rules of MatchParams
+// (substitutions only, ambiguous bases count as errors, circular junction).
+//
+// It uses the pigeonhole seed indexes above. It returns exactly what the
+// reference matchNaive returns, and what ecoPrimers' fuzzy search finds, in a
+// small fraction of the time: on the Teleostei tutorial data (255 words, 3909
+// sequences, 65 Mb, e = 3) 0.42 s on one core, against 113 s for the
+// obiapat (agrep) search it replaces.
+func MatchWords(c *Corpus, words []uint64, p MatchParams) ([]WordMatches, error) {
+	if err := p.validate(); err != nil {
+		return nil, err
+	}
+	k := p.K
+	d := p.MaxErrors / 2
+
+	// pattern 2w is the word, 2w+1 its reverse complement
+	patterns := make([]uint64, 2*len(words))
+	for i, w := range words {
+		patterns[2*i] = codeOf(wordCodes(w, k, false), k)
+		patterns[2*i+1] = codeOf(wordCodes(w, k, true), k)
+	}
+
+	right := k / 2 // the right half holds the low bits
+	left := k - right
+	leftIdx := newSeedIndex(patterns, right, left, d)
+	rightIdx := newSeedIndex(patterns, 0, right, d)
+
+	rightMask := uint64(1)<<(2*uint(right)) - 1
+	windowMask := uint64(1)<<(2*uint(k)) - 1
+	lowBits := uint64(0x5555555555555555) & windowMask
+	maxErr := p.MaxErrors
+
+	mk := func() scanFunc {
+		return func(raw []byte, emit func(word, pos, errors int, reverse bool)) {
+			n := len(raw)
+			if n < k {
+				return
+			}
+			total := n // number of bases fed to the rolling window
+			windows := n - k + 1
+			if p.Circular {
+				total, windows = n+k-1, n
+			}
+
+			var code, bad uint64 // window, and its non-ACGT positions (bit 2j)
+			verify := func(pos int, id int32) {
+				pat := patterns[id]
+				x := code ^ pat
+				if errs := bits.OnesCount64(((x | x>>1) & lowBits) | bad); errs <= maxErr {
+					emit(int(id)>>1, pos, errs, id&1 == 1)
+				}
+			}
+
+			for i := 0; i < total; i++ {
+				j := i
+				if j >= n {
+					j -= n
+				}
+				v := baseCode[raw[j]]
+				isBad := uint64(0)
+				if v > 3 {
+					v, isBad = 0, 1
+				}
+				code = (code<<2 | uint64(v)) & windowMask
+				bad = (bad<<2 | isBad) & windowMask
+
+				if i < k-1 {
+					continue
+				}
+				pos := i - k + 1
+				if pos >= windows {
+					break
+				}
+				leftCands := leftIdx.candidates(code >> (2 * uint(right)))
+				for _, id := range leftCands {
+					verify(pos, id)
+				}
+				for _, id := range rightIdx.candidates(code & rightMask) {
+					// ids are sorted: skip the ones the left index has just verified
+					if _, done := slices.BinarySearch(leftCands, id); !done {
+						verify(pos, id)
+					}
+				}
+			}
+		}
+	}
+	return matchCorpus(c, words, mk), nil
+}
+
+// wordCodes returns the 2-bit codes (A=0..T=3) of the word or of its reverse
+// complement, 5' to 3'.
+func wordCodes(word uint64, k int, reverse bool) []byte {
+	fwd, rev := wordStrings([]uint64{word}, k)
+	s := fwd[0]
+	if reverse {
+		s = rev[0]
+	}
+	return codesOf(s)
+}
+
+// codeOf packs codes (each 0..3) on 2 bits, first base in the high bits.
+func codeOf(codes []byte, k int) uint64 {
+	var v uint64
+	for _, c := range codes[:k] {
+		v = v<<2 | uint64(c)
+	}
+	return v
+}

pkg/obiprimer/match_test.go

@@ -0,0 +1,304 @@
+package obiprimer
+
+import (
+	"bufio"
+	"fmt"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"reflect"
+	"runtime"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiformats"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obikmer"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+func corpusOf(roles []Role, seqs ...string) *Corpus {
+	c := &Corpus{Roles: roles}
+	for i, s := range seqs {
+		c.Seqs = append(c.Seqs, obiseq.NewBioSequence(fmt.Sprintf("s%d", i), []byte(s), ""))
+	}
+	return c
+}
+
+func canon(word string) uint64 { return obikmer.EncodeCanonicalKmer([]byte(word), len(word)) }
+
+// The reference implementation is checked by hand first: everything else is
+// compared with it.
+func TestNaiveMatchSemantics(t *testing.T) {
+	const word = "AACCGGTA" // not a palindrome; reverse complement TACCGGTT
+	cases := []struct {
+		name     string
+		seq      string
+		circular bool
+		e        int
+		want     []Match
+	}{
+		{"exact forward", "ggaaccggtacc", false, 0, []Match{{0, 2, 0, false}}},
+		{"exact reverse strand", "ggtaccggttcc", false, 0, []Match{{0, 2, 0, true}}},
+		{"one error is not found at e=0", "ggaaccgctacc", false, 0, nil},
+		{"one error found at e=1 with its count", "ggaaccgctacc", false, 1, []Match{{0, 2, 1, false}}},
+		{"an N is an error", "ggaaccgntacc", false, 1, []Match{{0, 2, 1, false}}},
+		{"an N is never a match at e=0", "ggaaccgntacc", false, 0, nil},
+		{"an IUPAC code is an error, not a wildcard", "ggaaccgrtacc", false, 0, nil},
+		{"U reads as T", "ggaaccgguacc", false, 0, []Match{{0, 2, 0, false}}},
+		{"junction ignored on a linear sequence", "ccggtagggaa", false, 0, nil},
+		{"junction found on a circular sequence", "ccggtagggaa", true, 0, []Match{{0, 9, 0, false}}},
+		{"overlapping windows are all reported", "aaccggtaaccggta", false, 0, []Match{{0, 0, 0, false}, {0, 7, 0, false}}},
+		{"too short", "aaccgg", true, 3, nil},
+	}
+	for _, tc := range cases {
+		c := corpusOf([]Role{Example}, tc.seq)
+		got, err := matchNaive(c, []uint64{canon(word)}, MatchParams{K: len(word), MaxErrors: tc.e, Circular: tc.circular})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !reflect.DeepEqual(got[0].Matches, tc.want) {
+			t.Errorf("%s: got %v, want %v", tc.name, got[0].Matches, tc.want)
+		}
+	}
+}
+
+func TestMatchParamsValidation(t *testing.T) {
+	c := corpusOf([]Role{Example}, "acgtacgtacgtacgt")
+	for _, p := range []MatchParams{{K: 1}, {K: 32}, {K: 8, MaxErrors: -1}, {K: 8, MaxErrors: 8}} {
+		if _, err := matchNaive(c, nil, p); err == nil {
+			t.Errorf("%+v must be rejected", p)
+		}
+		if _, err := MatchWords(c, nil, p); err == nil {
+			t.Errorf("%+v must be rejected by MatchWords", p)
+		}
+	}
+}
+
+func randSeq(rng *rand.Rand, n int, alphabet string) []byte {
+	b := make([]byte, n)
+	for i := range b {
+		b[i] = alphabet[rng.Intn(len(alphabet))]
+	}
+	return b
+}
+
+func mutate(rng *rand.Rand, s []byte, n int) []byte {
+	out := append([]byte(nil), s...)
+	for _, i := range rng.Perm(len(out))[:n] {
+		out[i] = "acgt"[rng.Intn(4)]
+	}
+	return out
+}
+
+// randomCorpus builds sequences with words planted on both strands, with 0..3
+// substitutions, across the circular junction, and with ambiguity codes.
+func randomCorpus(rng *rand.Rand, words []string, nseq int) *Corpus {
+	c := &Corpus{}
+	for i := 0; i < nseq; i++ {
+		alphabet := "acgt"
+		if i%3 == 0 {
+			alphabet = "acgtacgtacgtacgtacgtnnrykm"
+		}
+		s := randSeq(rng, 30+rng.Intn(500), alphabet)
+		for range 1 + rng.Intn(4) {
+			w := []byte(words[rng.Intn(len(words))])
+			if rng.Intn(2) == 0 {
+				w = []byte(revcomp(string(w)))
+			}
+			w = mutate(rng, []byte(string(w)), rng.Intn(4))
+			for j := range w {
+				w[j] += 'a' - 'A'
+			}
+			at := rng.Intn(len(s) - len(w) + 1)
+			if rng.Intn(5) == 0 { // across the junction
+				s = append(s[:0:0], append(append([]byte(nil), w[len(w)/2:]...), append(s[:len(s)-len(w)/2], w[:len(w)/2]...)...)...)
+				continue
+			}
+			copy(s[at:], w)
+		}
+		c.Seqs = append(c.Seqs, obiseq.NewBioSequence(fmt.Sprintf("r%d", i), s, ""))
+		c.Roles = append(c.Roles, Example)
+	}
+	return c
+}
+
+// Every implementation must return exactly what the reference returns.
+func TestMatchersAgreeWithReference(t *testing.T) {
+	for _, tc := range []struct {
+		k, e     int
+		circular bool
+	}{{18, 0, false}, {18, 1, true}, {18, 3, false}, {18, 3, true}, {12, 2, true}, {25, 4, false}, {18, 2, true}, {18, 5, false}, {31, 3, true}, {24, 3, false}, {9, 2, true}, {18, 6, true}, {18, 7, false}, {20, 8, true}, {12, 11, false}} {
+		t.Run(fmt.Sprintf("k%d-e%d-circ%v", tc.k, tc.e, tc.circular), func(t *testing.T) {
+			rng := rand.New(rand.NewSource(int64(tc.k*100 + tc.e)))
+			var wordStr []string
+			var words []uint64
+			for range 7 {
+				w := string(randSeq(rng, tc.k, "ACGT"))
+				wordStr = append(wordStr, w)
+				words = append(words, canon(w))
+			}
+			c := randomCorpus(rng, wordStr, 40)
+			p := MatchParams{K: tc.k, MaxErrors: tc.e, Circular: tc.circular}
+
+			want, err := matchNaive(c, words, p)
+			if err != nil {
+				t.Fatal(err)
+			}
+			total := 0
+			for _, w := range want {
+				total += len(w.Matches)
+			}
+			if total < 10 {
+				t.Fatalf("test data too poor: only %d reference matches", total)
+			}
+
+			got, err := MatchWords(c, words, p)
+			if err != nil {
+				t.Fatal(err)
+			}
+			for i := range want {
+				if !reflect.DeepEqual(got[i].Matches, want[i].Matches) {
+					t.Fatalf("word %d: MatchWords gives %v\nreference gives %v", i, got[i].Matches, want[i].Matches)
+				}
+			}
+		})
+	}
+}
+
+func TestSequencesMatched(t *testing.T) {
+	c := corpusOf([]Role{Example, Example, CounterExample, CounterExample}, "a", "a", "a", "a")
+	w := WordMatches{Matches: []Match{{0, 1, 0, false}, {0, 9, 0, true}, {1, 3, 0, false}, {3, 0, 0, false}}}
+	if e, x := w.SequencesMatched(c); e != 2 || x != 1 {
+		t.Errorf("got %d examples, %d counterexamples", e, x)
+	}
+}
+
+func readEcoCounts(t *testing.T, path string) map[string]int {
+	t.Helper()
+	f, err := os.Open(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	counts := map[string]int{}
+	sc := bufio.NewScanner(f)
+	for sc.Scan() {
+		var w string
+		var in, out int
+		if _, err := fmt.Sscanf(sc.Text(), "%s\t%d\t%d", &w, &in, &out); err != nil {
+			t.Fatal(err)
+		}
+		counts[w] = in
+	}
+	return counts
+}
+
+// Opt-in comparison with ecoPrimers on the tutorial data (same variable as the
+// strict word test). For every strict word, ecoPrimers logs the number of
+// example sequences in which the fuzzy search (-e 3, both strands, circular)
+// finds it; testdata/ecoprimers_*.words.tsv holds those numbers.
+func TestMatchWordsAgainstEcoPrimers(t *testing.T) {
+	dir := os.Getenv("OBIPRIMERS_TUTORIAL")
+	if dir == "" {
+		t.Skip("OBIPRIMERS_TUTORIAL not set")
+	}
+	if v := os.Getenv("OBIPRIMERS_WORKERS"); v != "" { // e.g. 1, to time a single core
+		n, err := strconv.Atoi(v)
+		if err != nil {
+			t.Fatal(err)
+		}
+		obidefault.SetMaxCPU(n)
+	}
+	taxonomy, err := obiformats.LoadTaxonomy(filepath.Join(dir, "ncbitaxo_20250211.tgz"), false, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, tc := range []struct{ name, taxid string }{{"Cyprinidae", "7953"}, {"Teleostei", "32443"}} {
+		t.Run(tc.name, func(t *testing.T) {
+			restrict, err := BelongingTo(taxonomy, []string{tc.taxid})
+			if err != nil {
+				t.Fatal(err)
+			}
+			seqs, err := obiformats.ReadSequencesFromFile(filepath.Join(dir, "mito.vert.fasta.gz"))
+			if err != nil {
+				t.Fatal(err)
+			}
+			corpus, err := LoadCorpus(seqs, Selection{Restrict: restrict})
+			if err != nil {
+				t.Fatal(err)
+			}
+			sw, err := FindStrictWords(corpus, WordSearch{PrimerLength: 18, Quorum: 0.7, Circular: true, TmpDir: t.TempDir()})
+			if err != nil {
+				t.Fatal(err)
+			}
+			words := make([]uint64, len(sw.Words))
+			for i, w := range sw.Words {
+				words[i] = w.Kmer
+			}
+
+			var before, after runtime.MemStats
+			runtime.GC()
+			runtime.ReadMemStats(&before)
+			start := time.Now()
+			res, err := MatchWords(corpus, words, MatchParams{K: 18, MaxErrors: 3, Circular: true})
+			elapsed := time.Since(start)
+			runtime.ReadMemStats(&after)
+			if err != nil {
+				t.Fatal(err)
+			}
+			nm := 0
+			for _, r := range res {
+				nm += len(r.Matches)
+			}
+			bases := 0
+			for _, s := range corpus.Seqs {
+				bases += s.Len()
+			}
+			t.Logf("%d words x %d sequences (%.1f Mb) e=3 in %s; %d matches; allocated %.0f MB",
+				len(words), len(corpus.Seqs), float64(bases)/1e6, elapsed.Round(time.Millisecond), nm,
+				float64(after.TotalAlloc-before.TotalAlloc)/1e6)
+
+			// On real sequences (with their ambiguity codes), the first ones must give
+			// exactly what the reference implementation gives.
+			sub := &Corpus{Seqs: corpus.Seqs[:40], Roles: corpus.Roles[:40]}
+			wantSub, err := matchNaive(sub, words, MatchParams{K: 18, MaxErrors: 3, Circular: true})
+			if err != nil {
+				t.Fatal(err)
+			}
+			gotSub, err := MatchWords(sub, words, MatchParams{K: 18, MaxErrors: 3, Circular: true})
+			if err != nil {
+				t.Fatal(err)
+			}
+			subMatches := 0
+			for i := range wantSub {
+				subMatches += len(wantSub[i].Matches)
+				if !reflect.DeepEqual(gotSub[i].Matches, wantSub[i].Matches) {
+					t.Fatalf("word %d differs from the reference on the first 40 sequences", i)
+				}
+			}
+			t.Logf("first 40 sequences identical to the reference (%d matches)", subMatches)
+
+			eco := readEcoCounts(t, filepath.Join("testdata", "ecoprimers_"+tc.name+".words.tsv"))
+			compared, differ := 0, 0
+			for i, r := range res {
+				w := strings.ToUpper(string(obikmer.DecodeKmer(words[i], 18, nil)))
+				want, ok := eco[w]
+				if !ok {
+					t.Errorf("word %s is not in the ecoPrimers log", w)
+					continue
+				}
+				got, _ := r.SequencesMatched(corpus)
+				compared++
+				if got != want {
+					differ++
+					t.Errorf("word %s: %d example sequences, ecoPrimers %d", w, got, want)
+				}
+			}
+			t.Logf("%d words compared with ecoPrimers, %d differ", compared, differ)
+		})
+	}
+}

pkg/obiprimer/multimatch.go

@@ -0,0 +1,42 @@
+package obiprimer
+
+import "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obikmer"
+
+// DropRepeatedWords removes the strict words found more than once (on either
+// strand) in at least one example sequence. It is the first half of the
+// ecoPrimers -U option (no multi-match).
+func DropRepeatedWords(words []obikmer.SeqKmerCount) (kept []obikmer.SeqKmerCount, dropped int) {
+	kept = make([]obikmer.SeqKmerCount, 0, len(words))
+	for _, w := range words {
+		if w.Repeated {
+			dropped++
+			continue
+		}
+		kept = append(kept, w)
+	}
+	return kept, dropped
+}
+
+// DropMultiMatch removes the words having two or more approximate occurrences,
+// both strands counted, in at least one example or counterexample sequence. It
+// is the second half of the ecoPrimers -U option: the primers that can anneal
+// at several places of a sequence they must (or must not) amplify are
+// discarded. The other sequences are not considered.
+func DropMultiMatch(wm []WordMatches, c *Corpus) (kept []WordMatches, dropped int) {
+	kept = make([]WordMatches, 0, len(wm))
+	for _, w := range wm {
+		multi := false
+		for i := 1; i < len(w.Matches); i++ {
+			if w.Matches[i].Seq == w.Matches[i-1].Seq && c.Roles[w.Matches[i].Seq] != Other {
+				multi = true
+				break
+			}
+		}
+		if multi {
+			dropped++
+			continue
+		}
+		kept = append(kept, w)
+	}
+	return kept, dropped
+}