Release 4.5.2

This commit introduces significant changes to the sequence processing pipeline, implementing parallel loading, k-mer budget control, explicit memory management, and refined logic for pair evaluation and resolution.

.github/workflows/obitest.yml

@@ -9,11 +9,11 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout obitools4 project
-        uses: actions/checkout@v4
-      - name: Setup Go
-        uses: actions/setup-go@v5
-        with:
-          go-version: "1.23"
-      - name: Run tests
-        run: make githubtests
+    - name: Setup Go
+      uses: actions/setup-go@v5
+      with:
+        go-version: '1.27'
+    - name: Checkout obitools4 project
+      uses: actions/checkout@v5
+    - name: Run tests
+      run: make githubtests

.github/workflows/release.yml

@@ -16,9 +16,9 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: "1.23"
+          go-version: "1.27"
       - name: Checkout obitools4 project
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
       - name: Run tests
         run: make githubtests
 
@@ -49,12 +49,12 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: "1.23"
+          go-version: "1.27"
 
       - name: Extract version from tag
         id: get_version
@@ -69,7 +69,24 @@ jobs:
           xcode-select --install 2>/dev/null || true
           xcode-select -p
 
-      - name: Build binaries
+      - name: Build binaries (Linux)
+        if: runner.os == 'Linux'
+        env:
+          VERSION: ${{ steps.get_version.outputs.version }}
+        run: |
+          docker run --rm \
+            -v "$(pwd):/src" \
+            -w /src \
+            -e VERSION="${VERSION}" \
+            -e GOEXPERIMENT=simd \
+            golang:1.27-alpine \
+            sh -c "apk add --no-cache gcc musl-dev zlib-dev zlib-static make && \
+                   make LDFLAGS='-linkmode=external -extldflags=-static' obitools"
+          mkdir -p artifacts
+          tar -czf artifacts/obitools4_${VERSION}_${{ matrix.output_name }}.tar.gz -C build .
+
+      - name: Build binaries (macOS)
+        if: runner.os == 'macOS'
         env:
           GOOS: ${{ matrix.goos }}
           GOARCH: ${{ matrix.goarch }}
@@ -77,7 +94,6 @@ jobs:
         run: |
           make obitools
           mkdir -p artifacts
-          # Create a single tar.gz with all binaries for this platform
           tar -czf artifacts/obitools4_${VERSION}_${{ matrix.output_name }}.tar.gz -C build .
 
       - name: Upload artifacts
@@ -92,7 +108,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
 

.gitignore

@@ -16,13 +16,16 @@
 **/*.tgz
 **/*.yaml
 **/*.csv
+**/*.pb.gz
 xx
 
 .rhistory
 /.vscode
+
+/benchmarck
 /build
 /bugs
-
+autodoc
 /ncbitaxo
 
 !/obitests/**

.serena/.gitignore

@@ -0,0 +1,2 @@
+/cache
+/project.local.yml

.serena/project.yml

@@ -0,0 +1,169 @@
+# the name by which the project can be referenced within Serena/when chatting with the LLM.
+project_name: "obitools4"
+
+# the encoding used by text files in the project
+# For a list of possible encodings, see https://docs.python.org/3.11/library/codecs.html#standard-encodings
+encoding: "utf-8"
+
+# line ending convention to use when writing source files.
+# Possible values: unset (use global setting), "lf", "crlf", or "native" (platform default)
+# This does not affect Serena's own files (e.g. memories and configuration files), which always use native line endings.
+line_ending:
+
+# The language backend to use for this project.
+# If not set, the global setting from serena_config.yml is used.
+# Valid values: LSP, JetBrains
+# Note: the backend is fixed at startup. If a project with a different backend
+# is activated post-init, an error will be returned.
+language_backend:
+
+# whether to use project's .gitignore files to ignore files
+ignore_all_files_in_gitignore: true
+
+# advanced configuration option allowing to configure language server-specific options.
+# Maps the language key to the options.
+# The settings are considered only if the project is trusted (see global configuration to define trusted projects).
+# See https://oraios.github.io/serena/02-usage/050_configuration.html#language-server-specific-settings
+ls_specific_settings: {}
+
+# list of additional paths to ignore in this project.
+# Same syntax as gitignore, so you can use * and **.
+# Important: quote patterns that start with `*`, otherwise YAML treats them as aliases.
+# Example:
+#   ignored_paths:
+#     - "examples/**"
+#     - ".worktrees/**"
+#     - "**/bin/**"
+#     - "**/obj/**"
+# Note: global ignored_paths from serena_config.yml are also applied additively.
+ignored_paths: []
+
+# whether the project is in read-only mode
+# If set to true, all editing tools will be disabled and attempts to use them will result in an error
+# Added on 2025-04-18
+read_only: false
+
+# list of tool names to exclude.
+# This extends the existing exclusions (e.g. from the global configuration)
+# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html
+excluded_tools: []
+
+# list of tools to include that would otherwise be disabled (particularly optional tools that are disabled by default).
+# This extends the existing inclusions (e.g. from the global configuration).
+# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html
+included_optional_tools: []
+
+# fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools.
+# This cannot be combined with non-empty excluded_tools or included_optional_tools.
+# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html
+fixed_tools: []
+
+# list of mode names that are to be activated by default, overriding the setting in the global configuration.
+# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes.
+# If the setting is undefined/empty, the default_modes from the global configuration (serena_config.yml) apply.
+# Otherwise, this overrides the setting from the global configuration (serena_config.yml).
+# Therefore, you can set this to [] if you do not want the default modes defined in the global config to apply
+# for this project.
+# This setting can, in turn, be overridden by CLI parameters (--mode).
+# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes
+default_modes:
+
+# list of mode names to be activated additionally for this project, e.g. ["query-projects"]
+# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes.
+# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes
+added_modes:
+
+# initial prompt for the project. It will always be given to the LLM upon activating the project
+# (contrary to the memories, which are loaded on demand).
+initial_prompt: ""
+
+# time budget (seconds) per tool call for the retrieval of additional symbol information
+# such as docstrings or parameter information.
+# This overrides the corresponding setting in the global configuration; see the documentation there.
+# If null or missing, use the setting from the global configuration.
+symbol_info_budget:
+
+# list of regex patterns which, when matched, mark a memory entry as read‑only.
+# Extends the list from the global configuration, merging the two lists.
+read_only_memory_patterns: []
+
+# list of regex patterns for memories to completely ignore.
+# Matching memories will not appear in list_memories or activate_project output
+# and cannot be accessed via read_memory or write_memory.
+# To access ignored memory files, use the read_file tool on the raw file path.
+# Extends the list from the global configuration, merging the two lists.
+# Example: ["_archive/.*", "_episodes/.*"]
+ignored_memory_patterns: []
+
+# list of additional workspace folder paths for cross-package reference support.
+# Paths can be absolute or relative to the project root.
+# Each folder is registered as an LSP workspace folder, enabling language servers to discover
+# symbols and references across package boundaries, but these folders are not indexed by Serena,
+# i.e. the respective symbols will not be found using Serena's symbol search tools.
+# Example:
+#   additional_workspace_folders:
+#     - ../sibling-package
+#     - ../shared-lib
+ls_additional_workspace_folders: []
+
+# list of language servers to start when using the LSP backend; choose from:
+#   ada                    al                     angular                ansible                bash
+#   bsl                    clojure                cpp                    cpp_ccls               crystal
+#   csharp                 csharp_omnisharp       cue                    dart                   deno
+#   elixir                 elm                    erlang                 fortran                fsharp
+#   gdscript               gleam                  go                     groovy                 haskell
+#   haxe                   hlsl                   html                   java                   json
+#   julia                  kotlin                 latex                  lean4                  lua
+#   luau                   markdown               matlab                 msl                    nextflow
+#   nix                    ocaml                  pascal                 perl                   php
+#   php_phpactor           php_phpantom           powershell             python                 python_basedpyright
+#   python_jedi            python_pyrefly         python_ty              qml                    r
+#   rego                   ruby                   ruby_solargraph        rust                   scala
+#   scss                   solidity               svelte                 swift                  systemverilog
+#   terraform              toml                   typescript             typescript_vts         vue
+#   wolfram                yaml                   zig
+#   (This list may be outdated; generated with scripts/print_language_list.py;
+#   For the current list, see values of the LanguageServerId enum here:
+#   https://github.com/oraios/serena/blob/main/src/solidlsp/ls_config.py)
+# For some languages, there are several alternative language servers, e.g. csharp_omnisharp, ruby_solargraph.)
+# Note:
+#   - For C, use cpp
+#   - For JavaScript, use typescript
+#   - For Angular projects, use angular (subsumes typescript+html; requires `npm install` in the project root)
+#   - For Svelte projects, use svelte (subsumes typescript/javascript for .svelte projects; requires npm)
+#   - For Deno projects, use deno (serves the same .ts/.js files as typescript; requires the deno CLI on PATH)
+#   - For SCSS / Sass / plain CSS, use scss (some-sass-language-server handles all three)
+#   - For Free Pascal/Lazarus, use pascal
+# Special requirements:
+#   Some language servers require additional setup/installations.
+#   See here for details: https://oraios.github.io/serena/01-about/020_programming-languages.html#language-servers
+# When using multiple language servers, the first language server that supports a given file will be used for that file.
+# The first language server is the default language and the respective language server will be used as a fallback.
+# Note that when using the JetBrains backend, language servers are not used and this list is correspondingly ignored.
+language_servers:
+- go
+
+# list of workspace folder paths (LSP backend only).
+# These folders will be used to build up Serena's symbol index.
+# Paths must be within the project root and should thus be relative to the project root.
+# Furthermore, the paths should not be filtered by ignore settings.
+# Default setting: The entire project root folder (".") is considered.
+# In (large) monorepos, this can be used to index only subfolders of the project root, e.g.
+#   ls_workspace_folders:
+#     - "./subproject1"
+#     - "./subproject2"
+ls_workspace_folders:
+- .
+
+# optional shell command to run before the language backend (LSP or JetBrains) is initialised.
+# the command runs in the project root directory and is only executed if the project is trusted
+# (see trusted_project_path_patterns in the global configuration).
+# serena waits for the command to exit: a non-zero exit code is logged as an error but does not
+# abort activation. a per-project timeout (activation_command_timeout, default 180s) is the safety
+# backstop for non-terminating commands; on expiry the process is killed and activation continues.
+# example: activation_command: "npx nx run-many -t build"
+activation_command:
+
+# maximum time in seconds to wait for activation_command to complete before killing it (default 180s).
+# must be a positive number.
+activation_command_timeout: 180.0

Makefile

@@ -2,9 +2,25 @@
 #export GOBIN=$(GOPATH)/bin
 #export PATH=$(GOBIN):$(shell echo $${PATH})
 
+.DEFAULT_GOAL := all
+
+GREEN  := \033[0;32m
+YELLOW := \033[0;33m
+BLUE   := \033[0;34m
+NC     := \033[0m
+
 GOFLAGS=
+LDFLAGS=
+
+# The SIMD code (pkg/obikmer, simd package of Go >= 1.27) is only compiled with
+# GOEXPERIMENT=simd; without it, a portable scalar code is used instead.
+# The binaries select the vector width at run time (and fall back on an
+# emulation without SIMD hardware), so they run on any CPU.
+# Use `make GOEXPERIMENT= obitools` to build without SIMD.
+GOEXPERIMENT?=simd
+export GOEXPERIMENT
 GOCMD=go
-GOBUILD=$(GOCMD) build $(GOFLAGS)
+GOBUILD=$(GOCMD) build $(GOFLAGS) $(if $(LDFLAGS),-ldflags="$(LDFLAGS)")
 GOGENERATE=$(GOCMD) generate
 GOCLEAN=$(GOCMD) clean
 GOTEST=$(GOCMD) test
@@ -28,13 +44,18 @@ OBITOOLS_DIRS:=$(sort $(patsubst %/,%,$(dir $(OBITOOLS_SRC))))
 OBITOOLS:=$(notdir $(OBITOOLS_DIRS))
 
 
+# Name of the target of a package: its path under pkg/, so that two packages of
+# the same name in different directories (pkg/obialign and
+# pkg/obitools/obialign) do not share a target.
+pkgname=$(subst /,-,$(patsubst pkg/%,%,$(1)))
+
 define MAKE_PKG_RULE
-pkg-$(notdir $(1)): $(1) pkg/obioptions/version.go
-	@echo -n - Building package $(notdir $(1))...
+pkg-$(call pkgname,$(1)): $(1) pkg/obioptions/version.go
+	@echo -n - Building package $(call pkgname,$(1))...
 	@$(GOBUILD) ./$(1) \
-	    2> pkg-$(notdir $(1)).log \
-		|| cat pkg-$(notdir $(1)).log
-	@rm -f pkg-$(notdir $(1)).log
+	    2> pkg-$(call pkgname,$(1)).log \
+		|| cat pkg-$(call pkgname,$(1)).log
+	@rm -f pkg-$(call pkgname,$(1)).log
 	@echo Done.
 endef
 
@@ -43,7 +64,7 @@ $(OBITOOLS_PREFIX)$(notdir $(1)): $(BUILD_DIR) $(1) pkg/obioptions/version.go
 	@echo -n - Building obitool $(notdir $(1))...
 	@$(GOBUILD)  -o $(BUILD_DIR)/$(OBITOOLS_PREFIX)$(notdir $(1)) ./$(1) \
 	             2> $(OBITOOLS_PREFIX)$(notdir $(1)).log \
-				 || cat $(OBITOOLS_PREFIX)$(notdir $(1)).log
+				 || { cat $(OBITOOLS_PREFIX)$(notdir $(1)).log; rm -f $(OBITOOLS_PREFIX)$(notdir $(1)).log; exit 1; }
 	@rm -f $(OBITOOLS_PREFIX)$(notdir $(1)).log
 	@echo Done.
 endef
@@ -60,6 +81,28 @@ endif
 
 OUTPUT:=$(shell mktemp)
 
+help:
+	@printf "$(GREEN)OBITools4 Makefile$(NC)\n\n"
+	@printf "$(BLUE)Main targets:$(NC)\n"
+	@printf "  %-20s %s\n" "all"          "Build all obitools (default)"
+	@printf "  %-20s %s\n" "obitools"     "Build all obitools binaries to build/"
+	@printf "  %-20s %s\n" "test"         "Run Go unit tests"
+	@printf "  %-20s %s\n" "obitests"     "Run integration tests (obitests/)"
+	@printf "  %-20s %s\n" "bump-version" "Increment patch version (or set with VERSION=x.y.z)"
+	@printf "  %-20s %s\n" "update-deps"  "Update all Go dependencies"
+	@printf "\n$(BLUE)Jujutsu workflow:$(NC)\n"
+	@printf "  %-20s %s\n" "jjnew"        "Document current commit and start a new one"
+	@printf "  %-20s %s\n" "jjpush"       "Release: describe, bump, generate notes, push PR, tag (VERSION=x.y.z optional)"
+	@printf "  %-20s %s\n" "jjfetch"      "Fetch latest commits from origin"
+	@printf "\n$(BLUE)Required tools:$(NC)\n"
+	@printf "  %-20s " "go";      command -v go      >/dev/null 2>&1 && printf "$(GREEN)✓$(NC) %s\n" "$$(go version)" || printf "$(YELLOW)✗ not found$(NC)\n"
+	@printf "  %-20s " "git";     command -v git     >/dev/null 2>&1 && printf "$(GREEN)✓$(NC) %s\n" "$$(git --version)" || printf "$(YELLOW)✗ not found$(NC)\n"
+	@printf "  %-20s " "jj";      command -v jj      >/dev/null 2>&1 && printf "$(GREEN)✓$(NC) %s\n" "$$(jj --version)" || printf "$(YELLOW)✗ not found$(NC)\n"
+	@printf "  %-20s " "gh";      command -v gh      >/dev/null 2>&1 && printf "$(GREEN)✓$(NC) %s\n" "$$(gh --version | head -1)" || printf "$(YELLOW)✗ not found$(NC)  (brew install gh)\n"
+	@printf "\n$(BLUE)Optional tools (release notes generation):$(NC)\n"
+	@printf "  %-20s " "aichat";  command -v aichat  >/dev/null 2>&1 && printf "$(GREEN)✓$(NC) %s\n" "$$(aichat --version)" || printf "$(YELLOW)✗ not found$(NC)  (https://github.com/sigoden/aichat)\n"
+	@printf "  %-20s " "jq";      command -v jq      >/dev/null 2>&1 && printf "$(GREEN)✓$(NC) %s\n" "$$(jq --version)" || printf "$(YELLOW)✗ not found$(NC)  (brew install jq)\n"
+
 all: install-githook obitools
 
 obitools: $(patsubst %,$(OBITOOLS_PREFIX)%,$(OBITOOLS))
@@ -106,23 +149,28 @@ pkg/obioptions/version.go: version.txt .FORCE
 	@rm -f $(OUTPUT)
 
 bump-version:
-	@echo "Incrementing version..."
 	@current=$$(cat version.txt); \
-	echo "  Current version: $$current"; \
-	major=$$(echo $$current | cut -d. -f1); \
-	minor=$$(echo $$current | cut -d. -f2); \
-	patch=$$(echo $$current | cut -d. -f3); \
-	new_patch=$$((patch + 1)); \
-	new_version="$$major.$$minor.$$new_patch"; \
-	echo "  New version: $$new_version"; \
+	if [ -n "$(VERSION)" ]; then \
+		new_version="$(VERSION)"; \
+		echo "Setting version to $$new_version (was $$current)"; \
+	else \
+		echo "Incrementing version..."; \
+		echo "  Current version: $$current"; \
+		major=$$(echo $$current | cut -d. -f1); \
+		minor=$$(echo $$current | cut -d. -f2); \
+		patch=$$(echo $$current | cut -d. -f3); \
+		new_patch=$$((patch + 1)); \
+		new_version="$$major.$$minor.$$new_patch"; \
+		echo "  New version: $$new_version"; \
+	fi; \
 	echo "$$new_version" > version.txt
 	@echo "✓ Version updated in version.txt"
 	@$(MAKE) pkg/obioptions/version.go
 
 jjnew:
 	@echo "$(YELLOW)→ Creating a new commit...$(NC)"
-	@echo "$(BLUE)→ Documenting current commit...$(NC)"
-	@jj auto-describe
+	@echo "$(BLUE)→ Documenting undocumented commits...$(NC)"
+	@jj auto-doc
 	@echo "$(BLUE)→ Done.$(NC)"
 	@jj new
 	@echo "$(GREEN)✓ New commit created$(NC)"
@@ -130,51 +178,75 @@ jjnew:
 jjpush:
 	@$(MAKE) jjpush-describe
 	@$(MAKE) jjpush-bump
+	@$(MAKE) jjpush-notes
 	@$(MAKE) jjpush-push
 	@$(MAKE) jjpush-tag
 	@echo "$(GREEN)✓ Release complete$(NC)"
 
 jjpush-describe:
-	@echo "$(BLUE)→ Documenting current commit...$(NC)"
-	@jj auto-describe
+	@echo "$(BLUE)→ Documenting undocumented commits...$(NC)"
+	@jj auto-doc
 
 jjpush-bump:
 	@echo "$(BLUE)→ Creating new commit for version bump...$(NC)"
 	@jj new
 	@$(MAKE) bump-version
-	@echo "$(BLUE)→ Documenting version bump commit...$(NC)"
-	@jj auto-describe
+
+jjpush-notes:
+	@version=$$(cat version.txt); \
+	echo "$(BLUE)→ Generating release notes for version $$version...$(NC)"; \
+	release_title="Release $$version"; \
+	release_body=""; \
+	if command -v aichat >/dev/null 2>&1; then \
+		previous_tag=$$(git describe --tags --abbrev=0 --match 'Release_*' 2>/dev/null); \
+		if [ -z "$$previous_tag" ]; then \
+			echo "$(YELLOW)⚠ No previous Release tag found, skipping release notes$(NC)"; \
+		else \
+			raw_output=$$(git log --format="%h %B" "$$previous_tag..HEAD" | \
+				aichat \
+				"Summarize the following commits into a GitHub release note for version $$version. Ignore commits related to version bumps, .gitignore changes, or any internal housekeeping that is irrelevant to end users. Describe each user-facing change precisely without exposing code. Eliminate redundancy. Output strictly valid JSON with no surrounding text, using this exact schema: {\"title\": \"<short release title>\", \"body\": \"<detailed markdown release notes>\"}" 2>/dev/null) || true; \
+			if [ -n "$$raw_output" ]; then \
+				notes=$$(printf '%s\n' "$$raw_output" | python3 tools/json2md.py 2>/dev/null); \
+				if [ -n "$$notes" ]; then \
+					release_title=$$(echo "$$notes" | head -1); \
+					release_body=$$(echo "$$notes" | tail -n +3); \
+				else \
+					echo "$(YELLOW)⚠ JSON parsing failed, using default release message$(NC)"; \
+				fi; \
+			fi; \
+		fi; \
+	fi; \
+	printf '%s' "$$release_title" > /tmp/obitools4-release-title.txt; \
+	printf '%s' "$$release_body" > /tmp/obitools4-release-body.txt; \
+	echo "$(BLUE)→ Setting release notes as commit description...$(NC)"; \
+	jj desc -m "$$release_title"$$'\n\n'"$$release_body"
 
 jjpush-push:
 	@echo "$(BLUE)→ Pushing commits...$(NC)"
 	@jj git push --change @
+	@echo "$(BLUE)→ Creating/updating PR...$(NC)"
+	@release_title=$$(cat /tmp/obitools4-release-title.txt 2>/dev/null || echo "Release $$(cat version.txt)"); \
+	release_body=$$(cat /tmp/obitools4-release-body.txt 2>/dev/null || echo ""); \
+	branch=$$(jj log -r @ --no-graph -T 'bookmarks.map(|b| b.name()).join("\n")' 2>/dev/null | head -1); \
+	if [ -n "$$branch" ] && command -v gh >/dev/null 2>&1; then \
+		gh pr create --title "$$release_title" --body "$$release_body" --base master --head "$$branch" 2>/dev/null \
+		|| gh pr edit "$$branch" --title "$$release_title" --body "$$release_body" 2>/dev/null \
+		|| echo "$(YELLOW)⚠ Could not create/update PR$(NC)"; \
+	fi
 
 jjpush-tag:
 	@version=$$(cat version.txt); \
 	tag_name="Release_$$version"; \
-	echo "$(BLUE)→ Generating release notes for $$tag_name...$(NC)"; \
-	release_message="Release $$version"; \
-	if command -v orla >/dev/null 2>&1 && command -v jq >/dev/null 2>&1; then \
-		previous_patch=$$((  $$(echo $$version | cut -d. -f3) - 1 )); \
-		previous_tag="Release_$$(echo $$version | cut -d. -f1).$$(echo $$version | cut -d. -f2).$$previous_patch"; \
-		raw_output=$$(jj log -r "$$previous_tag::@" -T 'commit_id.short() ++ " " ++ description' | \
-			ORLA_MAX_TOOL_CALLS=50 orla agent -m ollama:qwen3-coder-next:latest \
-			"Summarize the following commits into a GitHub release note for version $$version. Ignore commits related to version bumps, .gitignore changes, or any internal housekeeping that is irrelevant to end users. Describe each user-facing change precisely without exposing code. Eliminate redundancy. Output strictly valid JSON with no surrounding text, using this exact schema: {\"title\": \"<short release title>\", \"body\": \"<detailed markdown release notes>\"}" 2>/dev/null) || true; \
-		if [ -n "$$raw_output" ]; then \
-			sanitized=$$(echo "$$raw_output" | sed -n '/^{/,/^}/p' | tr -d '\000-\011\013-\014\016-\037'); \
-			release_title=$$(echo "$$sanitized" | jq -r '.title // empty' 2>/dev/null) ; \
-			release_body=$$(echo "$$sanitized" | jq -r '.body // empty' 2>/dev/null) ; \
-			if [ -n "$$release_title" ] && [ -n "$$release_body" ]; then \
-				release_message="$$release_title"$$'\n\n'"$$release_body"; \
-			else \
-				echo "$(YELLOW)⚠ JSON parsing failed, using default release message$(NC)"; \
-			fi; \
-		fi; \
-	fi; \
+	release_title=$$(cat /tmp/obitools4-release-title.txt 2>/dev/null || echo "Release $$version"); \
+	release_body=$$(cat /tmp/obitools4-release-body.txt 2>/dev/null || echo ""); \
+	install_section=$$'\n## Installation\n\n### Pre-built binaries\n\nDownload the appropriate archive for your system from the\n[release assets](https://github.com/metabarcoding/obitools4/releases/tag/Release_'"$$version"')\nand extract it:\n\n#### Linux (AMD64)\n```bash\ntar -xzf obitools4_'"$$version"'_linux_amd64.tar.gz\n```\n\n#### Linux (ARM64)\n```bash\ntar -xzf obitools4_'"$$version"'_linux_arm64.tar.gz\n```\n\n#### macOS (Intel)\n```bash\ntar -xzf obitools4_'"$$version"'_darwin_amd64.tar.gz\n```\n\n#### macOS (Apple Silicon)\n```bash\ntar -xzf obitools4_'"$$version"'_darwin_arm64.tar.gz\n```\n\nAll OBITools4 binaries are included in each archive.\n\n### From source\n\nYou can also compile and install OBITools4 directly from source using the\ninstallation script:\n\n```bash\ncurl -L https://raw.githubusercontent.com/metabarcoding/obitools4/master/install_obitools.sh | bash -s -- --version '"$$version"'\n```\n\nBy default binaries are installed in `/usr/local/bin`. Use `--install-dir` to\nchange the destination and `--obitools-prefix` to add a prefix to command names:\n\n```bash\ncurl -L https://raw.githubusercontent.com/metabarcoding/obitools4/master/install_obitools.sh | \\\n  bash -s -- --version '"$$version"' --install-dir ~/local --obitools-prefix k\n```\n'; \
+	release_message="$$release_title"$$'\n\n'"$$release_body$$install_section"; \
 	echo "$(BLUE)→ Creating tag $$tag_name...$(NC)"; \
-	git tag -a "$$tag_name" -m "$$release_message" 2>/dev/null || echo "$(YELLOW)⚠ Tag $$tag_name already exists$(NC)"; \
+	commit_hash=$$(jj log -r @ --no-graph -T 'commit_id' 2>/dev/null); \
+	git tag -a "$$tag_name" $${commit_hash:+"$$commit_hash"} -m "$$release_message" 2>/dev/null || echo "$(YELLOW)⚠ Tag $$tag_name already exists$(NC)"; \
 	echo "$(BLUE)→ Pushing tag $$tag_name...$(NC)"; \
-	git push origin "$$tag_name" 2>/dev/null || echo "$(YELLOW)⚠ Tag push failed or already pushed$(NC)"
+	git push origin "$$tag_name" 2>/dev/null || echo "$(YELLOW)⚠ Tag push failed or already pushed$(NC)"; \
+	rm -f /tmp/obitools4-release-title.txt /tmp/obitools4-release-body.txt
 
 jjfetch:
 	@echo "$(YELLOW)→ Pulling latest commits...$(NC)"
@@ -182,5 +254,5 @@ jjfetch:
 	@jj new master@origin
 	@echo "$(GREEN)✓ Latest commits pulled$(NC)"
 
-.PHONY: all obitools update-deps obitests githubtests jjnew jjpush jjpush-describe jjpush-bump jjpush-push jjpush-tag jjfetch bump-version .FORCE
+.PHONY: all obitools update-deps obitests githubtests help jjnew jjpush jjpush-describe jjpush-bump jjpush-notes jjpush-push jjpush-tag jjfetch bump-version .FORCE
 .FORCE:

OBIPRIMERS_MEMORY_PROMPT.md

@@ -0,0 +1,149 @@
+# Prompt : optimisation de l'usage mémoire de `obiprimers`
+
+Tu travailles dans le dépôt Go `/Users/coissac/Sync/travail/__MOI__/GO/obitools4` (module
+`git.metabarcoding.org/obitools/obitools4/obitools4`, Go : `/usr/local/go/bin/go`, non présent dans le PATH).
+Ta mission : **réduire l'usage mémoire de la commande `obiprimers`** sans changer ses résultats, en procédant
+par la mesure.
+
+## 1. Contexte
+
+`obiprimers` est la ré-implémentation OBITools4 d'ecoPrimers (design de paires d'amorces à partir de génomes
+entiers, avec évaluation de la qualité du marqueur délimité). Le mode « voie A » est terminé et validé.
+Avant toute chose, **lis `OBIPRIMERS_NOTES.md`** (racine du dépôt) : la section « État courant », puis les
+sections 11 à 13 (appariement approché), 16 et 16 bis (taxonomie embarquée ; **une erreur de mesure à ne pas
+refaire**), 25 (définitions de Bc, Bs, Bd et élagage de la phase 2).
+
+Code concerné :
+- `cmd/obitools/obiprimers/main.go`, `pkg/obitools/obiprimers/` (options, lecture, sortie CSV) ;
+- `pkg/obiprimer/` : `corpus.go` (`LoadCorpus`), `taxa.go`, `words.go` (`FindStrictWords`), `match.go` et
+  `match_seed.go` (`MatchWords`), `pairs.go` (`selectPrimers`, `countPairs`), `pairs_eval.go` (`FindPairs`),
+  `amplicons.go` (`collectAmplicons`), `resolution.go`, `search.go` (`Search`) ;
+- `pkg/obikmer/seqquorum*.go` (`SeqKmerCounter`, comptage par partitions sur disque) ;
+- `pkg/obithermo/` (Tm), code partagé : `pkg/obiformats`, `pkg/obiiter`, `pkg/obiseq`, `pkg/obitax`.
+
+## 2. Décisions de l'utilisateur à respecter (non négociables)
+
+- **Tout est chargé en mémoire** : toutes les séquences de la base (cibles, contre-exemples, non-cibles) sont
+  gardées, parce que la lecture sur l'**entrée standard** doit continuer à marcher comme pour tout obitool.
+  Tu peux **compacter** ce qui est gardé ; tu ne peux pas passer à un schéma qui exige de relire le fichier.
+- **Un seul calcul par indice** : ne touche pas aux définitions de Bc, Bs et Bd (`resolution.go`). Pas de
+  colonne ou d'option alternative.
+- **Les résultats ne changent pas.** Critère : la sortie CSV doit rester **identique octet pour octet** sur les
+  cas de référence du §4. Si tu penses qu'un changement de résultat est justifié, **demande avant**.
+- Aucune commande git qui modifie quoi que ce soit (pas de commit, add, mv, rm, reset…). Lecture seule
+  (`status`, `diff`, `log`) autorisée. Déplace les fichiers avec `mv`/`Write`/`Edit`.
+- Fichiers source de moins d'environ 500 lignes. Pas de `.collect()` d'une grande structure quand un flux
+  suffit. Réutilise l'existant avant d'écrire du code ; enrichis une brique partagée plutôt que d'écrire du
+  code ad hoc, **mais** toute modification d'un paquet partagé (obiformats, obiiter, obiseq, obitax) doit
+  être signalée clairement et validée par les tests de ces paquets.
+- **Pas de spéculation présentée comme un fait** : ce qui n'est pas mesuré est une hypothèse, dis-le.
+  Une régression de justesse est pire qu'un gain de mémoire : avant d'échanger l'une contre l'autre, demande.
+
+## 3. Ce qui est mesuré aujourd'hui (macOS, M3 Max, 16 cœurs)
+
+Jeu : 7 822 génomes mitochondriaux de vertébrés (131 Mb de bases), `-r 7953` (Cyprinidae : 325 cibles, 7 497
+non-cibles). Commande :
+`obiprimers -r 7953 -e 3 --three-prime 2 -l 30 -L 150 -c <fichier annoté> -o out.csv` → 3 247 paires
+(3 248 lignes avec l'en-tête).
+
+| Mesure | Valeur |
+|---|---|
+| pic RSS, taxonomie embarquée, fichier | **1 217 à 1 372 Mo** (bruit d'environ ±100 Mo d'une exécution à l'autre), 4,7 à 5,3 s |
+| pic RSS, taxonomie embarquée, entrée standard | médiane **1 334 à 1 346 Mo** sur 8 exécutions entrelacées |
+| tas des 7 822 séquences chargées | **193,7 Mo** (dont `taxonomic_path` : 19,2 Mo, soit 2,46 Ko par séquence) |
+| même exécution **sans** l'élagage de phase 2 | 5 604 Mo (c'est ce qu'on a corrigé, voir §25 des notes) |
+| avec `-t` (taxonomie NCBI complète) | +1,07 Go pour le seul chargeur (mesuré avec `obitaxonomy`), pic ~2,8 Go (mesuré avant les nouvelles définitions) |
+| `GOGC=25` (ancienne mesure, `-t` NCBI) | pic 2,82 Go → 1,98 Go, mais temps CPU utilisateur ×2 |
+| ecoPrimers (C) sur les mêmes jeux | 1,86 Go (Cyprinidae), 2,11 Go (Teleostei) |
+
+Tailles d'enregistrements : `ampRecord` 24 octets, `Match` 12, `seqMatch` 12, `hit` (intermédiaire de
+`matchCorpus`) 16. Environ 6,85 millions d'occurrences approchées quand toutes les séquences sont chargées
+(cas `-i 1`).
+
+**Le constat qui motive la mission** : les séquences gardées pèsent ~194 Mo, le pic du processus est ~1,3 Go.
+Les 1,1 Go restants ne sont **pas expliqués** (hypothèses ci-dessous, aucune n'est vérifiée).
+
+## 4. Garde-fous à mettre en place AVANT de modifier quoi que ce soit
+
+1. **Binaire de référence** : `cd /Users/coissac/Sync/travail/__MOI__/GO/obitools4 &&
+   GOEXPERIMENT=simd /usr/local/go/bin/go build -o /tmp/obiprimers.baseline ./cmd/obitools/obiprimers`
+2. **Données** (les fichiers de `/tmp` peuvent avoir disparu, regénère-les) : dossier du tutoriel
+   `D="$HOME/Documents/Enseignement/DNA metabarcoding spring school/ecoPrimers-Fish"` : `mito.vert.fasta.gz`,
+   `ncbitaxo_20250211.tgz`, et pour le test d'échelle `mito.all.fasta.gz` / `mito.one.fasta.gz`. Fichier annoté
+   (taxonomie embarquée, évite le chargeur NCBI) :
+   `/Users/coissac/travail/__MOI__/GO/obitools4/build/obiannotate -t "$D/ncbitaxo_20250211.tgz"
+   --taxonomic-path -Z -o /tmp/mito.vert.path.fasta.gz "$D/mito.vert.fasta.gz"`
+   (les binaires déjà compilés sont dans `/Users/coissac/travail/__MOI__/GO/obitools4/build/`, sans `Sync`).
+3. **CSV de référence**, à produire avec le binaire de référence et à conserver : Cyprinidae `-r 7953`,
+   Teleostei `-r 32443`, un cas avec contre-exemples `-r 7953 -i 1`, un cas `-U`, un cas `-R NC_083213`, et la
+   lecture sur entrée standard (`gzcat fichier | obiprimers ...`). Après chaque modification : `cmp` octet pour
+   octet.
+4. **Tests** : `go test ./pkg/obiprimer/ ./pkg/obitools/obiprimers/ ./pkg/obithermo/ ./pkg/obikmer/ -race
+   -count=1`, et les tests opt-in sur les données réelles (durent ~75 s hors `-race`) :
+   `OBIPRIMERS_TUTORIAL="$D" go test ./pkg/obiprimer/ -count=1`.
+5. **Méthode de mesure** (le piège principal) : le pic RSS bouge de ±100 Mo entre deux exécutions identiques
+   (rythme du ramasse-miettes). Mesure **au moins 5 exécutions entrelacées avant/après et compare les
+   médianes** (`/usr/bin/time -l` : octets sur macOS). Pour attribuer la mémoire, ne te fie pas au RSS : utilise
+   des profils de tas (`--pprof` de la CLI sert `/debug/pprof/heap` sur `localhost:6060`, `net/http/pprof` est
+   importé ; ou `runtime/pprof.WriteHeapProfile` aux frontières d'étapes ; `go tool pprof -sample_index=inuse_space`
+   et `alloc_space` ; `GODEBUG=gctrace=1`). **Erreur déjà commise (voir la section 16 bis des notes)** : en
+   mesurant le tas libéré par la suppression d'un attribut, la tranche de séquences n'était plus référencée
+   après la boucle et le GC a libéré *tout* ; utilise `runtime.KeepAlive` sur ce que tu veux garder vivant.
+
+## 5. Pistes à évaluer (toutes des HYPOTHÈSES à confirmer par un profil, classées sans ordre)
+
+- **Retard du ramasse-miettes** (`GOGC=100` laisse le tas monter jusqu'à ~2× le vivant). Pistes : limite
+  mémoire douce (`debug.SetMemoryLimit`) ou `debug.SetGCPercent` ajustés par la commande après la lecture ;
+  mesurer le coût CPU (l'ancienne mesure avec `GOGC=25` doublait le temps utilisateur).
+- **Lecture** (code partagé `obiformats`/`obiiter`) : 131 Mb sont analysés, avec une table d'attributs et des
+  chaînes JSON par séquence. L'option `--batch-mem` (lots limités en mémoire) existe déjà : l'évaluer.
+- **Séquences gardées** : `BioSequence` garde 1 octet par base et une table d'attributs. Ne garder que
+  l'identifiant, la séquence et le `taxid` (retirer les autres attributs, comme déjà fait pour
+  `taxonomic_path`, dans `dropTaxonomicPath` de `pkg/obitools/obiprimers/obiprimers.go`) ; ou stocker la
+  séquence à 2 bits par base (~33 Mo pour 131 Mb) avec une liste d'exceptions pour les bases ambiguës. Ce
+  second point touche `forEachOccurrence`, `ampHash`, `windowAsIs`, `windowRevComp` et `MatchWords`, qui lisent
+  les octets : à ne tenter que si le profil le justifie.
+- **Occurrences stockées deux fois** : `[]WordMatches` (`matchCorpus`, intermédiaire `hit` de 16 octets par
+  occurrence puis recopie) et `primerSet.perSeq` (`selectPrimers`). Vérifier à quel moment chacune est
+  libérée, construire directement les structures finales, présizer les tranches (les `append` en doublement
+  créent un pic transitoire).
+- **Phase 2b** (`collectAmplicons` sur les seules paires survivantes) : `ampRecord` de 24 octets (16 possibles
+  en empaquetant le rôle), enregistrements par worker ; **traitement des paires survivantes par lots bornés**
+  si le nombre d'enregistrements estimé dépasse un budget (idée notée en 25.2). Cas dur : `-T 0` (aucune paire
+  élaguée), où les tests atteignent ~5,5 Go de tas.
+- **`SeqKmerCounter`** (`pkg/obikmer/seqquorum*.go`) : environ 16 octets par occurrence de k-mer pour les
+  partitions en cours de traitement ; le nombre de partitions est **fixé à 256**, alors que le total de bases
+  est connu une fois le corpus chargé : le choisir automatiquement.
+- **Lecture en deux passes** pour la taxonomie embarquée (`readSequencesAndTaxonomy`) : la première passe ne
+  garde que les chemins, mais analyse tout.
+- **Chargeur de taxonomie NCBI** : 1,07 Go à lui seul ; c'est du code partagé (`obitax`), hors périmètre sauf
+  si tu y trouves une économie sûre ; sinon, documente-le dans l'aide (la taxonomie embarquée évite ce coût).
+
+## 6. Test d'échelle (jamais fait)
+
+Tout est validé sur des génomes mitochondriaux (5 à 131 Mb). Mesure la croissance sur un fichier plus gros
+(`mito.all.fasta.gz`, 172 Mo compressés, avec `--restrict-to-taxon` sur un grand clade, p. ex. les Metazoa) :
+temps et pic RSS en fonction de la taille, pour savoir ce qui grossit vite (corpus, occurrences, amplicons,
+compteur de k-mers).
+
+## 7. Livrables
+
+1. Un **rapport de profil** : où va la mémoire à chaque étape (lecture, sélection, mots stricts, appariement,
+   paires, phase 2a/2b), avec les profils de tas et les chiffres, avant toute modification.
+2. Les optimisations **classées par gain mesuré et par risque**, puis implémentées une par une, chacune avec
+   mesure avant/après (médianes d'au moins 5 exécutions entrelacées), `cmp` des CSV de référence, tests verts
+   sous `-race`.
+3. Une **nouvelle section** à la fin de `OBIPRIMERS_NOTES.md` (ne réécris pas l'historique) : mesures,
+   décisions, pistes écartées et pourquoi, ce qui reste. Pas de commit.
+4. Si une optimisation exige de changer un résultat, une décision ou un paquet partagé : **arrête-toi et
+   demande**.
+
+## 8. Pièges connus de l'environnement
+
+- zsh ne découpe pas une variable en arguments : utilise des tableaux (`A=(-r 7953 ...)` puis `"${A[@]}"`)
+  ou des scripts `bash`.
+- macOS : `sed -i ''` (sans suffixe) ; `/usr/bin/time -l` donne le pic en **octets** (`maximum resident set size`).
+- `OBIPRIMERS_WORKERS` n'est lue que par les tests opt-in ; la CLI utilise `--max-cpu`.
+- Les avertissements C de `obiformats/fastseq_read.h` sont connus et sans gravité.
+- `go test -race` sur les tests opt-in est lent (~90 s) ; réserve-le aux tests unitaires.
+- Le test `obiutils/TestSetString` était instable : il est corrigé (tri des membres dans `Set.String`).

README.md

@@ -32,8 +32,12 @@ The installation script offers several options:
 > 
 >  -p, --obitools-prefix   Prefix added to the obitools command names if you
 >                          want to have several versions of obitools at the
->                          same time on your system (as example `-p g` will produce 
+>                          same time on your system (as example `-p g` will produce
 >                          `gobigrep` command instead of `obigrep`).
+>
+>  -j, --jobs              Number of parallel jobs used for compilation
+>                          (default: 1). Increase this value to speed up
+>                          compilation on multi-core systems (e.g., `-j 4`).
 
 ### Examples
 

Release-notes.md

@@ -2,6 +2,23 @@
 
 ## New changes
 
+### New features
+
+- New `obioligotag` command, a re-implementation of the `oligotag` command of
+  OBITools 2. It builds a family of oligonucleotides (tags) of at least
+  `--family-size` members that are pairwise at a distance of at least
+  `--distance`. Candidates are either generated (`--oligo-size`, up to 12) or
+  read from a file (`--oligo-list`, sequence file or one oligonucleotide per
+  line). They can be restricted by alphabet (`--alphabet`, for instance `ACT`
+  to forbid G), GC content, homopolymers and IUPAC patterns. The distance is
+  either the Hamming distance or the LCS-mode alignment distance
+  (`--distance-type`).
+
+- New functions in `obikmer` to compare and enumerate k-mers encoded as
+  `uint64`: `HammingDistance`, `KmerLCS` and `KmerLCSBatch` (SIMD when built with
+  `GOEXPERIMENT=simd`), `IterConstrainedKmers`, and in `obigraph` a clique search
+  (`FindClique`) that does not need the adjacency matrix.
+
 ### Bug fixes
 
 - In `obipairing` correct the misspelling of the `obiparing_*` tags where the `i`

blackboard/Prospective/canonical-super-kmer-strategy.md

@@ -1,755 +0,0 @@
-# Prospective : Index k-mer v3 — Super-kmers canoniques, unitigs, et Aho-Corasick
-
-## 1. Constat sur l'index v1
-
-L'index actuel (`.kdi` delta-varint) stocke 18.6 milliards de k-mers (k=31, m=13, P=4096, 2 sets) en 85 Go, soit 4.8-5.6 bytes/k-mer. Les causes :
-
-- Le canonical standard `min(fwd, rc)` disperse les k-mers sur 62 bits → deltas ~2^40 → 5-6 bytes varint
-- Les k-mers partagés entre sets sont stockés N fois (une fois par set)
-- Le matching nécessite N×P ouvertures de fichier (N passes)
-
-## 2. Observations expérimentales
-
-### 2.1 Déréplication brute
-
-Sur un génome de *Betula exilis* 15× couvert, le pipeline `obik lowmask | obik super | obiuniq` réduit **80 Go de fastq.gz en 5.6 Go de fasta.gz** — un facteur 14×. Cela montre que la déréplication au niveau super-kmer est extrêmement efficace et que les super-kmers forment une représentation naturellement compacte.
-
-### 2.2 Après filtre de fréquence (count > 1)
-
-En éliminant les super-kmers observés une seule fois (erreurs de séquençage), le fichier passe de 5.6 Go à **2.7 Go de fasta.gz**. Les statistiques détaillées (obicount) :
-
-| Métrique | Valeur |
-|----------|--------|
-| Variants (super-kmers uniques) | 37,294,271 |
-| Reads (somme des counts) | 148,828,167 |
-| Symboles (bases totales variants) | 1,415,018,593 |
-| Longueur moyenne super-kmer | **37.9 bases** |
-| K-mers/super-kmer moyen (k=31) | **7.9** |
-| K-mers totaux estimés | **~295M** |
-| Count moyen par super-kmer | **4.0×** |
-
-### 2.3 Comparaison avec l'index v1
-
-| Format | Taille | K-mers | Bytes/k-mer |
-|--------|--------|--------|-------------|
-| Index .kdi v1 (set Human dans Contaminent_idx) | 12.8 Go | ~3B | 4.3 |
-| Delta-varint hypothétique (295M k-mers) | ~1.5 Go | 295M | 5.0 |
-| Super-kmers 2-bit packed (*Betula* count>1) | ~354 Mo | 295M | **1.2** |
-| Super-kmers fasta.gz (*Betula* count>1) | 2.7 Go | 295M | 9.2* |
-
-\* Le fasta.gz inclut les headers, les counts, et la compression gzip — pas directement comparable au format binaire.
-
-**Le format super-kmer 2-bit est ~4× plus compact que le delta-varint** à nombre égal de k-mers. Cette efficacité vient du fait qu'un super-kmer de 38 bases encode 8 k-mers en ~10 bytes au lieu de 8 × 5 = 40 bytes en delta-varint.
-
-Note : la comparaison n'est pas directe (Contaminent_idx = génomes assemblés, *Betula* = reads bruts filtrés), mais le ratio bytes/k-mer est comparable car il dépend de la longueur des super-kmers, pas de la source des données.
-
-## 3. Stratégie proposée : pipeline de construction v3
-
-### 3.1 Définition du k-mer minimizer-canonique
-
-On redéfinit la forme canonique d'un k-mer en fonction de son minimiseur :
-
-```
-CanonicalKmer(kmer, k, m) :
-  minimizer = plus petit m-mer canonique dans le k-mer
-  si minimizer == forward_mmer(minimizer_pos)
-    → garder le k-mer tel quel
-  sinon
-    → prendre le reverse-complement du k-mer
-```
-
-Propriétés :
-- **m impair** → aucun m-mer ne peut être palindromique (`m_mer != RC(m_mer)` toujours) → la canonisation par le minimiseur est toujours non-ambiguë. C'est m, pas k, qui doit être impair : l'ambiguïté viendrait d'un minimiseur palindrome (`min == RC(min)`), auquel cas on ne saurait pas dans quel sens orienter le k-mer/super-kmer.
-- Tous les k-mers d'un super-kmer partagent le même minimiseur
-- **La canonisation peut se faire au niveau du super-kmer entier** : si `minimizer != canonical(minimizer)`, on RC le super-kmer complet. Tous les k-mers qu'il contient deviennent automatiquement minimizer-canoniques.
-
-### 3.2 Pipeline de construction
-
-```
-Séquences brutes ([]byte, 1 byte/base)
-    │
-    ▼
-[0] Encodage 2-bit + nettoyage
-    │  - Encoder chaque séquence en 2 bits/base ([]byte packed)
-    │  - Couper aux bases ambiguës (N, R, Y, W, S, K, M, B, D, H, V)
-    │  - Retirer les fragments de longueur < k
-    │  - Résultat : fragments 2-bit clean, prêts pour toutes les opérations
-    ▼
-[1] Filtre de complexité (lowmask sur vecteurs 2-bit)
-    │  Supprime/masque les régions de faible entropie
-    ▼
-[2] Extraction des super-kmers (sur vecteurs 2-bit, non canonisé)
-    │  Chaque super-kmer a un minimiseur et une séquence 2-bit packed
-    ▼
-[3] Canonisation au niveau super-kmer
-    │  Si minimizer != CanonicalKmer(minimizer) → RC le super-kmer (op bit)
-    │  Résultat : super-kmers canoniques 2-bit packed
-    ▼
-[4] Écriture dans les partitions .skm (partition = minimizer % P)
-    │  Format natif 2-bit → écriture directe, pas de conversion
-    ▼
-[5] Déréplication des super-kmers par partition
-    │  Trier les super-kmers (comparaison uint64 sur données packed → très rapide)
-    │  Compter les occurrences identiques
-    │  Résultat : super-kmers uniques avec count
-    ▼
-[6] Construction des unitigs canoniques par partition
-    │  Assembler les super-kmers qui se chevauchent de (k-1) bases
-    │  en chaînes linéaires non-branchantes (tout en 2-bit)
-    │  Propager les counts : vecteur de poids par unitig
-    ▼
-[7] Filtre de fréquence sur le graphe pondéré (voir section 4)
-    │  Supprimer les k-mers (positions) avec poids < seuil
-    │  Re-calculer les unitigs après filtrage
-    ▼
-[8] Stockage des unitigs avec bitmask multiset
-    │  Format compact sur disque (déjà en 2-bit, écriture directe)
-    ▼
-Index v3
-```
-
-### 3.2bis Pourquoi encoder en 2-bit dès le début ?
-
-**Alternative rejetée** : travailler en `[]byte` (1 byte/base) puis encoder en 2-bit seulement pour le stockage final.
-
-| Aspect | `[]byte` (1 byte/base) | 2-bit packed |
-|--------|----------------------|--------------|
-| Programmation | Simple (slicing natif, pas de bit-shift) | Plus complexe (masques, shifts) |
-| Mémoire par super-kmer (38 bases) | 38 bytes | 10 bytes (**3.8×** moins) |
-| 37M super-kmers en RAM | ~1.4 Go | ~370 Mo |
-| Tri (comparaison) | `bytes.Compare` sur slices | Comparaison uint64 (**beaucoup** plus rapide) |
-| Format .skm | Conversion encode/decode à chaque I/O | Écriture/lecture directe |
-| RC d'un super-kmer | Boucle sur bytes + lookup | Opérations bit (une instruction pour complement) |
-
-L'opération la plus coûteuse du pipeline est le **tri des super-kmers** pour la déréplication (étape 5). En 2-bit packed, un super-kmer de ≤32 bases tient dans un `uint64` → tri par comparaison entière (une instruction CPU). Un super-kmer de 33-64 bases tient dans deux `uint64` → tri en deux comparaisons.
-
-Le code de manipulation 2-bit est plus complexe à écrire mais **s'écrit une seule fois** (bibliothèque de primitives) et bénéficie à toute la chaîne. Le gain en mémoire (4×) et en temps de tri est significatif sur des dizaines de millions de super-kmers.
-
-### 3.3 Canonisation des super-kmers : pourquoi ça marche
-
-**Point crucial** : les super-kmers doivent être construits en utilisant le minimiseur **non-canonique** (le m-mer brut tel qu'il apparaît dans la séquence), et non le minimiseur canonique `min(fwd, rc)`.
-
-**Pourquoi ?** Si on utilise le minimiseur canonique comme critère de regroupement, un même super-kmer pourrait contenir le minimiseur dans ses **deux orientations** à des positions différentes (le m-mer forward à une position, et sa forme RC à une autre position, ayant la même valeur canonique). Dans ce cas, le RC du super-kmer ne résoudrait pas l'ambiguïté.
-
-**Algorithme correct** :
-
-1. **Extraction** : construire les super-kmers en regroupant les k-mers consécutifs qui partagent le même m-mer minimal **non-canonique** (le m-mer brut). Au sein d'un tel super-kmer, le minimiseur apparaît toujours dans **une seule orientation**.
-
-2. **Canonisation** : pour chaque super-kmer, comparer son minimiseur brut à `canonical(minimizer) = min(minimizer, RC(minimizer))` :
-   - Si `minimizer == canonical(minimizer)` → le minimiseur est déjà en forward → garder le super-kmer tel quel
-   - Si `minimizer != canonical(minimizer)` → le minimiseur est en RC → RC le super-kmer entier → le minimiseur apparaît maintenant en forward
-
-Après cette étape, **chaque k-mer du super-kmer** contient le minimiseur canonique en position forward, ce qui correspond exactement à notre définition de k-mer minimizer-canonique.
-
-**Note** : cela signifie que l'algorithme `IterSuperKmers` actuel (qui utilise le minimiseur canonique pour le regroupement) doit être modifié pour utiliser le minimiseur brut. C'est un changement dans le critère de rupture des super-kmers : on casse quand le **m-mer minimal brut** change, pas quand le **m-mer minimal canonique** change. Les super-kmers résultants seront potentiellement plus courts (un changement d'orientation du minimiseur force une coupure), mais c'est le prix de la canonicité absolue.
-
-### 3.4 Déréplication des super-kmers
-
-Deux super-kmers identiques (même séquence, même minimiseur) correspondent aux mêmes k-mers. On peut les dérépliquer en triant :
-
-1. Par minimiseur (déjà partitionné)
-2. Par séquence (tri lexicographique des séquences 2-bit packed)
-
-Les super-kmers identiques deviennent consécutifs dans le tri → comptage linéaire.
-
-Le tri peut se faire sur les fichiers .skm d'une partition, en mémoire si la partition tient en RAM, ou par merge-sort externe sinon.
-
-## 4. Filtre de fréquence
-
-### 4.1 Problème
-
-Le filtre de fréquence (`--min-occurrence N`) élimine les k-mers vus moins de N fois. Avec la déréplication des super-kmers, on a un count par super-kmer, pas par k-mer. Un k-mer peut apparaître dans plusieurs super-kmers différents (aux jonctions, ou quand le minimiseur change), donc le count exact d'un k-mer n'est connu qu'après fusion.
-
-### 4.2 Solution : filtrage sur le graphe de De Bruijn pondéré
-
-Le filtre de fréquence doit être appliqué **après** la construction des unitigs canoniques (section 5), et non avant. Le pipeline devient :
-
-```
-Super-kmers canoniques dérepliqués (avec counts)
-    │
-    ▼
-Construction des unitigs canoniques (section 5)
-    │  Chaque position dans un unitig porte un poids
-    │  = somme des counts des super-kmers couvrant ce k-mer
-    ▼
-Graphe de De Bruijn pondéré (implicite dans les unitigs)
-    │
-    ▼
-Filtrage : supprimer les k-mers (positions) avec poids < seuil
-    │  Cela casse certains unitigs en fragments
-    ▼
-Recalcul des unitigs sur le graphe filtré
-    │
-    ▼
-Unitigs filtrés finaux
-```
-
-**Avantages** :
-- Le filtre opère sur les **k-mers exacts** avec leurs **counts exacts** (pas une approximation par super-kmer)
-- Le graphe de De Bruijn est implicitement contenu dans les unitigs — pas besoin de le construire explicitement avec une `map[uint64]uint`
-- Les k-mers aux jonctions de super-kmers ont leurs counts correctement agrégés
-
-### 4.3 Calcul du poids de chaque position dans un unitig
-
-Un unitig est construit par chaînage de super-kmers. Chaque super-kmer S de longueur L et count C contribue (L-k+1) k-mers, chacun avec poids C. Quand deux super-kmers se chevauchent de (k-1) bases dans l'unitig, les k-mers de la zone de chevauchement reçoivent la **somme** des counts des deux super-kmers.
-
-En pratique, lors de la construction de l'unitig par chaînage, on construit un vecteur de poids `weights[0..nkmers-1]` :
-```
-Pour chaque super-kmer S (count=C) ajouté à l'unitig:
-  Pour chaque position i couverte par S dans l'unitig:
-    weights[i] += C
-```
-
-### 4.4 Filtrage et re-construction
-
-Après filtrage (`weights[i] < seuil` → supprimer position i), l'unitig est potentiellement coupé en fragments. Chaque fragment continu de positions conservées forme un nouvel unitig (ou super-kmer si court).
-
-Le recalcul des unitigs après filtrage est trivial : les fragments sont déjà des chemins linéaires, il suffit de vérifier les conditions de non-branchement aux nouvelles extrémités.
-
-### 4.5 Spectre de fréquence
-
-Le spectre de fréquence exact peut être calculé directement depuis les vecteurs de poids des unitigs : `weights[i]` donne le count exact du k-mer à la position i. C'est un histogramme sur toutes les positions de tous les unitigs.
-
-### 4.6 Faisabilité mémoire : graphe pondéré par partition
-
-Données mesurées sur un index *Betula* (k=31, P=4096 partitions, 1 set, génome assemblé) — distribution des tailles de fichiers .kdi :
-
-| Métrique | Taille fichier .kdi | K-mers estimés (~5 B/kmer) | Super-kmers (~8 kmer/skm) |
-|----------|--------------------|-----------------------------|---------------------------|
-| Mode | 100-200 Ko | 20 000 – 40 000 | 2 500 – 5 000 |
-| Médiane | ~350-400 Ko | ~70 000 – 80 000 | ~9 000 – 10 000 |
-| Max | ~2.3 Mo | ~460 000 | ~57 000 |
-
-Le graphe de De Bruijn pondéré pour une partition nécessite d'extraire tous les k-mers (arêtes) et (k-1)-mers (nœuds) des super-kmers :
-
-| Partition | K-mers (arêtes) | RAM arêtes (~20 B) | RAM nœuds (~16 B) | Total |
-|-----------|-----------------|--------------------|--------------------|-------|
-| Typique (~10K skm, 38 bases avg) | ~80K | ~1.6 Mo | ~1.3 Mo | **~3 Mo** |
-| Maximale (~57K skm) | ~460K | ~9.2 Mo | ~7.4 Mo | **~17 Mo** |
-
-C'est **largement en mémoire**. Les partitions étant indépendantes, elles peuvent être traitées en parallèle par un pool de goroutines. Avec 8 goroutines : **~136 Mo** au pic — négligeable. Les tableaux sont réutilisables entre partitions (allocation unique).
-
-**Conclusion** : la construction du graphe de De Bruijn pondéré partition par partition est non seulement faisable mais triviale en termes de mémoire. C'est un argument fort en faveur de l'approche « filtre après unitigs » plutôt que « filtre sur super-kmers ».
-
-### 4.7 Invariance de la distribution par rapport à la canonisation
-
-La redéfinition du k-mer canonique (par le minimiseur au lieu de `min(fwd, rc)`) ne change **rien** à l'ensemble des k-mers ni à leur répartition par partition :
-
-- C'est une **bijection** : chaque k-mer a toujours exactement un représentant canonique, on change juste lequel des deux brins on choisit
-- Le partitionnement se fait sur `canonical(minimizer) % P` — la valeur du minimiseur canonique est la même dans les deux conventions
-- **Même nombre de k-mers par partition, même distribution de tailles**
-- **Même topologie du graphe de De Bruijn** (mêmes nœuds, mêmes arêtes)
-
-Ce qui change, c'est **l'orientation** : avec la canonicité par minimiseur, les unitigs canoniques ne suivent que les arêtes « forward » (`suffix(S1) == prefix(S2)`, identité exacte). Certaines arêtes traversables en RC dans BCALM2 deviennent des points de cassure. Le graphe n'est pas plus gros — il suffit de ne construire que des unitigs canoniques, ce qui **simplifie** l'algorithme (pas de gestion des traversées de brin).
-
-## 5. Construction des unitigs canoniques
-
-### 5.1 Définition : unitig canonique absolu
-
-Un **unitig canonique** est un chemin linéaire non-branchant dans le graphe de De Bruijn où :
-1. Chaque k-mer est **minimizer-canonique** (le minimiseur y apparaît en forward)
-2. Chaque super-kmer constituant est **canonique** (même convention)
-3. Le chaînage se fait **sans traversée de brin** : `suffix(k-1, S1) == prefix(k-1, S2)` dans le même sens (pas en RC)
-
-C'est plus restrictif que les unitigs BCALM2 (qui autorisent `suffix(S1) == RC(prefix(S2))`), mais cela garantit que **tout k-mer extrait par fenêtre glissante est directement dans sa forme canonique**, sans re-canonisation.
-
-### 5.2 Pourquoi la canonicité absolue est essentielle
-
-**Matching** : les k-mers requête sont canonisés une fois (par le minimiseur), puis comparés directement aux k-mers de l'unitig par scan. Pas de re-canonisation à la volée → plus rapide, plus simple.
-
-**Opérations ensemblistes** : deux index utilisant la même convention produisent les mêmes unitigs canoniques pour les mêmes k-mers. L'intersect/union peut opérer par comparaison directe de séquences triées.
-
-**Bitmask multiset** : la fusion de N sets est triviale — merger des listes de super-kmers/unitigs canoniques triés par séquence.
-
-**Déterminisme** : un ensemble de k-mers produit toujours les mêmes unitigs canoniques, quel que soit l'ordre d'insertion ou la source des données.
-
-### 5.3 Impact sur la compaction
-
-La contrainte canonique interdit les traversées de brin aux jonctions → les unitigs canoniques sont **plus courts** que les unitigs BCALM2. Estimation :
-- BCALM2 (unitigs libres) : 63 bases moyennes (mesuré sur *Betula*)
-- Unitigs canoniques : probablement ~45-55 bases moyennes
-- Super-kmers dérepliqués : 38 bases moyennes
-
-Le facteur de compaction est légèrement réduit mais le gain en simplicité opérationnelle compense largement.
-
-### 5.4 Construction par partition — le vrai graphe de De Bruijn
-
-Les super-kmers canoniques dérepliqués sont des **chemins** dans le graphe de De Bruijn, pas des nœuds. On ne peut pas les chaîner directement comme des nœuds car :
-- Deux super-kmers peuvent **se chevaucher** (partager des k-mers aux jonctions)
-- Un super-kmer court peut avoir ses k-mers **inclus** dans un super-kmer plus long
-
-Un super-kmer de longueur L contient (L-k+1) k-mers, soit (L-k+1) arêtes et (L-k+2) nœuds ((k-1)-mers) dans le graphe de De Bruijn.
-
-#### 5.4.1 Nœuds = (k-1)-mers, Arêtes = k-mers
-
-Le graphe de De Bruijn par partition a :
-- **Nœuds** : les (k-1)-mers uniques (extraits de toutes les positions dans les super-kmers)
-- **Arêtes** : les k-mers (chaque position dans un super-kmer = une arête entre deux (k-1)-mers consécutifs)
-- **Poids** : chaque arête (k-mer) porte le count du super-kmer qui la contient
-
-Les branchements (nœud avec degré entrant > 1 ou degré sortant > 1) peuvent être :
-- Aux **bords** des super-kmers (jonctions entre super-kmers)
-- Aux **positions internes** si un k-mer d'un autre super-kmer rejoint un (k-1)-mer interne
-
-#### 5.4.2 Graphe complet nécessaire
-
-Construire le graphe avec **tous** les (k-1)-mers (internes et bords) est nécessaire pour détecter correctement les branchements. Se limiter aux seuls bords de super-kmers serait incorrect car un (k-1)-mer de bord d'un super-kmer peut correspondre à un nœud interne d'un autre super-kmer.
-
-Pour une partition typique de 10K super-kmers de longueur moyenne 38 bases → ~80K k-mers → ~80K arêtes et ~80K nœuds. Voir section 4.6 pour la faisabilité mémoire (~3 Mo par partition typique, ~17 Mo max).
-
-#### 5.4.3 Structure de données : tableau trié d'arêtes
-
-Plutôt que des hash maps, on utilise un **tableau trié** pour le graphe :
-
-```go
-type Edge struct {
-    srcKmer  uint64    // (k-1)-mer source (prefix du k-mer)
-    dstKmer  uint64    // (k-1)-mer destination (suffix du k-mer)
-    weight   int32     // count du super-kmer contenant ce k-mer
-}
-```
-
-Pour chaque super-kmer S de longueur L et count C, on émet (L-k+1) arêtes. Tableau total pour une partition typique : ~80K × 20 bytes = **~1.6 Mo**.
-
-On trie par `srcKmer` pour obtenir la liste d'adjacence sortante, ou on construit deux vues triées (par src et par dst) pour avoir adjacence entrante et sortante.
-
-#### 5.4.4 Détection des unitigs canoniques
-
-Un unitig canonique est un chemin maximal non-branchant. L'algorithme :
-
-```
-1. Extraire toutes les arêtes des super-kmers → tableau edges[]
-2. Trier edges[] par srcKmer → vue sortante
-   Trier une copie par dstKmer → vue entrante
-3. Pour chaque (k-1)-mer unique :
-   - degré_sortant = nombre d'arêtes avec ce srcKmer
-   - degré_entrant = nombre d'arêtes avec ce dstKmer
-   - Si degré_sortant == 1 ET degré_entrant == 1 → nœud interne d'unitig
-   - Sinon → nœud de branchement (début ou fin d'unitig)
-4. Parcourir les chemins non-branchants pour construire les unitigs
-   - Chaque unitig est une séquence de (k-1)-mers chaînés
-   - Le vecteur de poids est la séquence des weight des arêtes traversées
-```
-
-Les (k-1)-mers ne sont **pas canonisés** — on respecte l'orientation des super-kmers canoniques. Le chaînage est strictement orienté.
-
-#### 5.4.5 Estimation mémoire
-
-| Partition | K-mers (arêtes) | RAM arêtes | RAM nœuds | Total |
-|-----------|-----------------|------------|-----------|-------|
-| Typique (~10K skm, 38 bases avg) | ~80K | ~1.6 Mo | ~1.3 Mo | **~3 Mo** |
-| Maximale (~57K skm) | ~460K | ~9.2 Mo | ~7.4 Mo | **~17 Mo** |
-
-Avec traitement parallèle par un pool de G goroutines : RAM max = G × 17 Mo. Avec G=8 : **~136 Mo** au pic. Le tableau d'arêtes est réutilisable entre partitions (allocation unique, remise à zéro).
-
-Complexité : O(E log E) par partition, avec E = nombre total de k-mers. Dominé par les deux tris.
-
-### 5.5 Graphe par minimiseur, pas par partition
-
-Les k-mers (arêtes) sont partitionnés par minimiseur. Deux k-mers adjacents dans le graphe de De Bruijn peuvent avoir des minimiseurs différents — c'est exactement ce qui définit les frontières de super-kmers. Si on construit le graphe par partition (qui regroupe plusieurs minimiseurs), des (k-1)-mers de jonction entre minimiseurs différents apparaîtraient comme nœuds partagés entre partitions → le graphe par partition n'est pas autonome.
-
-**Solution : construire un graphe par minimiseur.**
-
-Un super-kmer est par définition un chemin dont **tous les k-mers partagent le même minimiseur**. Donc :
-- Toutes les arêtes d'un super-kmer appartiennent à un seul minimiseur
-- Chaque graphe par minimiseur est **100% autonome** : toutes ses arêtes et nœuds internes sont auto-contenus
-- Les (k-1)-mers aux bords des super-kmers qui touchent un autre minimiseur sont des extrémités (degré 0 dans ce graphe) → bouts d'unitig naturels
-- Aucune jonction inter-graphe → pas de cassure artificielle d'unitig
-
-**Taille des graphes** : avec ~16K minimiseurs théoriques par partition (P=4096, m=13), le calcul naïf donne ~5 arêtes/minimiseur. Mais en pratique, beaucoup de minimiseurs ne sont pas représentés (séquences biologiques, pas aléatoires) et la distribution est très inégale. Si seuls ~500-1000 minimiseurs sont effectivement présents dans une partition typique de 80K arêtes, on a plutôt **80-160 arêtes en moyenne** par minimiseur, avec une queue de distribution vers les centaines ou milliers pour les minimiseurs les plus fréquents. Même dans ce cas, les graphes restent petits (quelques Ko à quelques dizaines de Ko).
-
-*À mesurer* : nombre de minimiseurs distincts par partition et distribution du nombre de k-mers par minimiseur sur un index existant.
-
-**Algorithme** : les super-kmers étant déjà triés par minimiseur dans la partition, on itère séquentiellement et on construit/détruit un petit graphe à chaque changement de minimiseur. C'est plus simple que le graphe par partition — pas de tri global de toutes les arêtes, juste un buffer local réutilisé.
-
-**Les unitigs résultants** sont les chemins maximaux non-branchants au sein d'un minimiseur. Un unitig ne traverse jamais une frontière de minimiseur, ce qui est correct : tous les k-mers d'un unitig partagent le même minimiseur canonique, ce qui renforce la propriété de canonicité absolue.
-
-### 5.6 Quand les unitigs canoniques n'aident pas
-
-- Si les super-kmers sont courts (peu de chevauchement entre super-kmers adjacents)
-- Si le graphe est très branché (zones de divergence entre génomes)
-- Si beaucoup de jonctions se font par traversée de brin (la contrainte canonique empêche la fusion)
-- Données metabarcoding avec grande diversité taxonomique → courts unitigs
-
-Dans ces cas, stocker les super-kmers dérepliqués directement est suffisant — ils sont déjà canoniques par construction.
-
-## 6. Construction multiset : super-kmers par set, graphe commun
-
-### 6.1 Pipeline en deux phases
-
-La construction d'un index multiset (N sets) se fait en deux phases distinctes :
-
-**Phase 1 — Par set (indépendant, parallélisable)** :
-
-Chaque set i (i = 0..N-1) produit indépendamment ses super-kmers canoniques :
-```
-Set i : séquences → [0] 2-bit → [1] lowmask → [2] super-kmers → [3] canonisation → [4] partition .skm_i
-```
-Puis déréplication par partition : super-kmers triés avec counts, écrits dans des fichiers `.skm` distincts par set.
-
-**Phase 2 — Par partition, tous sets confondus** :
-
-Pour chaque partition P (parallélisable par goroutine) :
-```
-.skm_0[P], .skm_1[P], ..., .skm_{N-1}[P]   (super-kmers triés de chaque set)
-    │
-    ▼
-[a] N-way merge des super-kmers triés
-    │  Même super-kmer dans sets i et j → fusionner en vecteur de counts [c_0, ..., c_{N-1}]
-    │  Super-kmer uniquement dans set i → counts = [0, ..., c_i, ..., 0]
-    ▼
-[b] Extraction des arêtes du graphe de De Bruijn
-    │  Chaque k-mer (arête) porte un vecteur de poids [w_0, ..., w_{N-1}]
-    │  w_i = count du super-kmer contenant ce k-mer dans le set i
-    ▼
-[c] Construction d'un SEUL graphe de De Bruijn par partition
-    │  Les branchements sont définis par l'UNION de tous les sets :
-    │  si un (k-1)-mer a degré > 1 dans n'importe quel set, c'est un branchement
-    ▼
-[d] Extraction des unitigs canoniques communs
-    │  Même séquence pour tous les sets
-    │  Chaque position porte un vecteur de poids (un count par set)
-    ▼
-[e] Filtre de fréquence (optionnel, par set ou global)
-    ▼
-[f] Encodage du bitmask par runs le long de chaque unitig
-    ▼
-Écriture dans le .sku de la partition
-```
-
-### 6.2 Le graphe est défini par l'union
-
-Point crucial : les unitigs sont déterminés par la **topologie de l'union** de tous les sets. Un branchement dans un seul set force une coupure d'unitig pour tous les sets. Cela garantit que :
-- Les unitigs sont les mêmes quelle que soit l'ordre des sets
-- Un k-mer donné se trouve toujours au même endroit (même unitig, même position)
-- Les opérations ensemblistes (intersect, union, difference) opèrent sur les mêmes unitigs
-
-### 6.3 Arêtes à vecteur de poids
-
-La structure Edge (section 5.4.3) est étendue pour le multiset :
-
-```go
-type Edge struct {
-    srcKmer  uint64      // (k-1)-mer source
-    dstKmer  uint64      // (k-1)-mer destination
-    weights  []int32     // weights[i] = count dans le set i (0 si absent)
-}
-```
-
-Pour la détection des branchements, le degré d'un nœud est le nombre d'arêtes distinctes (par dstKmer pour le degré sortant), **indépendamment** des sets. Une arête présente dans le set 0 mais pas le set 1 compte quand même.
-
-### 6.4 Bitmask par runs le long des unitigs
-
-Le long d'un unitig, le bitmask (quels sets contiennent ce k-mer) change rarement — les régions conservées entre génomes sont longues. On encode :
-
-```
-unitig_bitmask = [(bitmask_1, run_length_1), (bitmask_2, run_length_2), ...]
-```
-
-Où `bitmask_i` a un bit par set (bit j = 1 si `weights[j] > 0` à cette position).
-
-Pour un unitig de 70 k-mers avec 2 sets :
-- Si complètement partagé : 1 run `(0b11, 70)` → 2 bytes
-- Si divergent au milieu : 2-3 runs → 4-6 bytes
-- Pire cas : 70 runs → 140 bytes (très rare)
-
-### 6.5 Impact mémoire du multiset
-
-Le vecteur de poids par arête augmente la taille du graphe :
-- 1 set : `weight int32` → 4 bytes/arête
-- N sets : `weights [N]int32` → 4N bytes/arête
-
-Pour la partition typique (~80K arêtes) avec N=2 sets : overhead = 80K × 4 = **320 Ko** supplémentaires. Négligeable.
-
-Pour N=64 sets (cas extrême) : 80K × 256 = **~20 Mo** par partition. Reste faisable mais les sets très nombreux pourraient nécessiter un encodage plus compact (sparse vector si beaucoup de zéros).
-
-### 6.6 Merge des super-kmers : N-way sur séquences triées
-
-Le merge des N listes de super-kmers triés (par séquence 2-bit) est un N-way merge classique avec min-heap :
-- Chaque .skm est déjà trié par séquence (étape de déréplication)
-- On compare les séquences 2-bit packed (comparaison uint64, très rapide)
-- Quand le même super-kmer apparaît dans plusieurs sets, on fusionne les counts
-- Quand un super-kmer est unique à un set, les autres counts sont 0
-
-C'est analogue au `KWayMerge` existant sur les k-mers triés, étendu aux super-kmers.
-
-## 7. Format de stockage v3 : fichiers parallèles
-
-### 7.1 Architecture : 3 fichiers par partition
-
-Pour chaque partition, trois fichiers alignés :
-
-```
-index_v3/
-  metadata.toml
-  parts/
-    part_PPPP.sku         # séquences 2-bit des unitigs concaténés
-    part_PPPP.skx         # index par minimiseur (offsets dans .sku)
-    part_PPPP.skb         # bitmask multiset (1 entrée par k-mer)
-    ...
-  set_N/spectrum.bin      # spectre de fréquence par set
-```
-
-### 7.2 Fichier .sku — séquences d'unitigs concaténées
-
-Tous les unitigs d'une partition sont concaténés bout à bout en 2-bit packed, **ordonnés par minimiseur**. Entre deux unitigs, pas de séparateur dans le flux 2-bit.
-
-Un **tableau de longueurs** stocké en en-tête ou dans le .skx donne la longueur (en bases) de chaque unitig dans l'ordre. Ce tableau permet :
-- De retrouver les frontières d'unitigs
-- De savoir si un match AC chevauche une jonction (à filtrer)
-- D'indexer directement un unitig par son numéro
-
-```
-Format .sku :
-  Magic: "SKU\x01"           (4 bytes)
-  TotalBases: uint64 LE       (nombre total de bases dans la partition)
-  NUnitigs: uint64 LE         (nombre d'unitigs)
-  Lengths: [NUnitigs]varint   (longueur en bases de chaque unitig)
-  Sequence: ceil(TotalBases/4) bytes  (flux 2-bit continu)
-```
-
-### 7.3 Fichier .skx — index par minimiseur
-
-Pour chaque minimiseur présent dans la partition, l'index donne l'offset (en bases) dans le flux .sku et le nombre d'unitigs :
-
-```
-Format .skx :
-  Magic: "SKX\x01"           (4 bytes)
-  NMinimizers: uint32 LE      (nombre de minimiseurs présents)
-  Entries: [NMinimizers] {
-    Minimizer: uint64 LE      (valeur du minimiseur canonique)
-    BaseOffset: uint64 LE     (offset en bases dans le flux .sku)
-    UnitigOffset: uint32 LE   (index du premier unitig de ce minimiseur dans le tableau de longueurs)
-    NUnitigs: uint32 LE       (nombre d'unitigs pour ce minimiseur)
-  }
-```
-
-Les entrées sont triées par `Minimizer` → recherche binaire en O(log N).
-
-Pour accéder aux unitigs d'un minimiseur donné :
-1. Recherche binaire dans le .skx → `BaseOffset`, `UnitigOffset`, `NUnitigs`
-2. Seek dans le .sku au bit `BaseOffset × 2`
-3. Lecture de `NUnitigs` unitigs (longueurs dans le tableau à partir de `UnitigOffset`)
-
-### 7.4 Fichier .skb — bitmask multiset parallèle
-
-Le fichier bitmask est **aligné position par position** avec le flux de k-mers des unitigs. Chaque k-mer (position dans un unitig) a exactement une entrée dans le .skb, dans le même ordre que les k-mers apparaissent en lisant les unitigs séquentiellement.
-
-```
-Format .skb :
-  Magic: "SKB\x01"           (4 bytes)
-  TotalKmers: uint64 LE       (nombre total de k-mers)
-  NSets: uint8                 (nombre de sets)
-  BitmaskSize: uint8           (ceil(NSets/8) bytes par entrée)
-  Bitmasks: [TotalKmers × BitmaskSize] bytes
-```
-
-**Accès direct** : la position absolue d'un k-mer dans le flux d'unitigs (offset en k-mers depuis le début de la partition) donne directement l'index dans le fichier .skb :
-```
-bitmask_offset = header_size + kmer_position × BitmaskSize
-```
-
-Pour 2 sets : 1 byte par k-mer (6 bits inutilisés).
-Pour ≤8 sets : 1 byte par k-mer.
-Pour ≤16 sets : 2 bytes par k-mer.
-
-**Coût** : pour 295M k-mers (*Betula*, 2 sets) : 295 Mo. Pour l'index Contaminent_idx (18.6B k-mers, 2 sets) : ~18.6 Go. C'est significatif — voir section 7.5 pour la compression.
-
-### 7.5 Compression du bitmask : RLE ou non ?
-
-| Approche | Taille (2 sets, 295M k-mers) | Accès |
-|----------|------------------------------|-------|
-| Non compressé (1 byte/k-mer) | 295 Mo | O(1) direct |
-| RLE par unitig | ~10-50 Mo (estimé) | O(decode) par unitig |
-| Bitset par set (1 bit/k-mer/set) | 74 Mo | O(1) direct |
-
-L'approche **bitset par set** (1 bit par k-mer par set, packed en bytes) est un bon compromis :
-- 2 sets : 2 bits/k-mer → ~74 Mo (vs 295 Mo non compressé)
-- Accès O(1) : `bit = (data[kmer_pos / 4] >> ((kmer_pos % 4) × 2)) & 0x3`
-- Pas besoin de décompression séquentielle
-
-Pour les très grands index (18.6B k-mers), même le bitset fait ~4.6 Go. Le RLE par minimiseur (ou par unitig) pourrait réduire à ~1-2 Go mais perd l'accès O(1).
-
-**Recommandation** : bitset packed pour ≤8 sets (accès O(1)), RLE pour >8 sets ou très grands index.
-
-## 8. Matching avec Aho-Corasick sur le flux d'unitigs
-
-### 8.1 Principe
-
-Pour chaque partition dont les k-mers requête partagent le minimiseur :
-1. Seek dans le .sku au bloc du minimiseur (via .skx)
-2. Construire un automate AC avec les k-mers requête canoniques de ce minimiseur
-3. Scanner le flux 2-bit des unitigs de ce minimiseur
-4. Pour chaque match : vérifier qu'il ne chevauche pas une frontière d'unitig
-5. Pour chaque match valide : lookup dans le .skb à la position correspondante → bitmask
-
-### 8.2 Le problème des faux matches aux jonctions
-
-En 2-bit, pas de 5e lettre pour séparer les unitigs. Le scan AC sur le flux continu peut produire des matches à cheval sur deux unitigs adjacents.
-
-**Solution : post-filtrage par le tableau de longueurs.**
-
-Pendant le scan, on maintient un compteur de position et un index dans le tableau de longueurs (préfixe cumulé). Quand un match est trouvé à la position `p` :
-- Le match couvre les bases `[p, p+k-1]`
-- Si ces bases chevauchent une frontière d'unitig → faux positif, ignorer
-- Sinon → match valide
-
-Le coût du post-filtrage est O(1) par match (le compteur de frontière avance séquentiellement).
-
-**Estimation du taux de faux positifs** : avec des unitigs de ~50 bases en moyenne, une jonction tous les ~50 bases, et k=31 : ~31/50 = ~62% des positions de jonction peuvent produire un faux match. Mais seule une infime fraction de ces positions correspond à un pattern dans l'automate AC. En pratique, le nombre de faux positifs est négligeable.
-
-### 8.3 Du match à la position absolue dans le .skb
-
-Un match AC à la position `p` dans le flux du minimiseur se traduit en position k-mer dans le .skb :
-
-```
-kmer_position_in_partition = base_offset_of_minimizer_in_partition
-                           + p
-                           - (nombre de bases de padding/frontières avant p)
-```
-
-En fait, si le tableau de longueurs donne les longueurs d'unitigs en bases, la position k-mer cumulative est :
-```
-Pour l'unitig i contenant le match :
-  kmer_base = somme des longueurs des unitigs 0..i-1
-  kmer_offset_in_unitig = p - kmer_base
-  kmer_index = somme des (len_j - k + 1) pour j=0..i-1  +  kmer_offset_in_unitig
-```
-
-Ce `kmer_index` est l'index direct dans le fichier .skb.
-
-### 8.4 Comparaison avec le merge-scan v1
-
-| Aspect | Merge-scan (v1) | AC sur unitigs (v3) |
-|--------|----------------|---------------------|
-| Pré-requis | Tri des requêtes O(Q log Q) | Construction automate AC O(Q×k) |
-| Seek | .kdx sparse index | .skx index par minimiseur |
-| Scan | O(Q + K) merge linéaire par set | O(bases_du_minimiseur + matches) |
-| Multi-set | **N passes** (une par set) | **1 seule passe** (bitmask .skb) |
-| I/O | N×P ouvertures de fichier | 1 seek + lecture séquentielle + lookup .skb |
-| Accès bitmask | implicite (chaque .kdi = 1 set) | O(1) dans .skb |
-
-Le gain principal du v3 est l'**élimination des N passes** : au lieu de scanner N fois (une par set), on scanne une seule fois et on consulte le bitmask. Pour N=2 sets et P=4096 partitions, cela réduit les ouvertures de fichier de 2×4096 = 8192 à 4096.
-
-## 9. Estimations de taille et validation expérimentale
-
-### 9.1 Cas mesuré : *Betula exilis* 15× (reads bruts, count > 1)
-
-| Métrique | Valeur |
-|----------|--------|
-| Super-kmers uniques (count > 1) | 37.3M |
-| Longueur moyenne | 37.9 bases |
-| Bases totales | 1.415G |
-
-**Stockage binaire 2-bit packed** :
-- Séquences : 1.415G / 4 = **354 Mo**
-- Headers (longueur varint + minimiseur) : 37.3M × ~4 bytes = **150 Mo**
-- Bitmask (1 set → 0 bytes, ou 2 sets → 1 byte/entrée = 37 Mo)
-- **Total estimé : ~500-550 Mo** pour un set
-
-### 9.2 Extrapolation pour l'index Plants+Human (2 sets)
-
-L'index v1 actuel contient 18.6B k-mers en 85 Go. Avec le pipeline v3 :
-
-**Scénario reads bruts 15× par génome** (extrapolé depuis *Betula exilis*) :
-- *Betula exilis* mesuré : ~37M super-kmers, ~1.4G bases → ~500 Mo
-- Proportionnellement pour l'index Contaminent_idx (18.6B k-mers) : **~2-5 Go**
-
-**Scénario génome assemblé (pas de filtre de fréquence)** :
-- Un génome assemblé de 3 Gbases → estimation ~80M super-kmers × 38 bases → **760 Mo**
-- Un génome assemblé de 10 Gbases → estimation ~350M super-kmers × 38 bases → **3.3 Go**
-- Avec overlap multiset : super-kmers partagés fusionnés (bitmask) → **~4 Go**
-
-**Le gain est spectaculaire dans les deux scénarios** :
-- Reads bruts : facteur **~30-40×** grâce à la déréplication + filtre de fréquence
-- Génomes assemblés : facteur **~20×** grâce au format super-kmer seul
-
-Le format super-kmer est intrinsèquement plus efficace que le delta-varint car il exploite la structure locale du graphe de De Bruijn : des k-mers consécutifs partagent (k-1) bases, encodées une seule fois dans le super-kmer.
-
-### 9.3 Validation expérimentale : unitigs BCALM2
-
-*Betula exilis* 15×, après lowmask + super-kmers canoniques + déréplication + filtre count>1, passé dans BCALM2 (`-kmer-size 31 -abundance-min 1`) :
-
-| Métrique | Super-kmers (count>1) | Unitigs (BCALM2) | Ratio |
-|----------|----------------------|-------------------|-------|
-| Variants | 37,294,271 | 6,473,171 | **5.8×** |
-| Bases totales | 1,415,018,593 | 408,070,894 | **3.5×** |
-| Longueur moyenne | 37.9 bases | 63.0 bases | 1.7× |
-| K-mers estimés | ~295M | ~213M | — |
-
-### Stockage estimé
-
-| Format | Taille estimée | Bytes/k-mer | Facteur vs v1 |
-|--------|---------------|-------------|---------------|
-| .kdi v1 (delta-varint, assemblé) | 12.8 Go | 4.3 | 1× |
-| Super-kmers 2-bit (count>1) | ~500 Mo | 1.7 | 25× |
-| **Unitigs 2-bit (BCALM2)** | **~130 Mo** | **0.6** | **98×** |
-
-### Extrapolation pour l'index Contaminent_idx (Plants+Human, 2 sets)
-
-Le facteur ~100× mesuré sur *Betula exilis* 15× se décompose :
-- Déréplication des reads redondants : facteur ~15× (couverture 15×)
-- Compaction super-kmer/unitig vs delta-varint : facteur ~100/15 ≈ **6.7×**
-
-L'index Contaminent_idx est construit à partir de **génomes assemblés** (sans redondance de séquençage). Seul le facteur de compaction unitig s'applique :
-- Index v1 actuel : 85 Go (Plants 72 Go + Human 12.8 Go)
-- **Estimation unitigs : ~85 / 6.7 ≈ 12-13 Go** (facteur **~6.7×**)
-
-C'est un gain significatif mais bien moins spectaculaire que sur des reads bruts. Le facteur pourrait être meilleur si les unitigs des génomes assemblés sont plus longs que ceux des reads (moins de fragmentation par les erreurs de séquençage).
-
-### Observation sur le nombre de k-mers
-
-Les unitigs contiennent ~213M k-mers vs ~295M estimés dans les super-kmers. La différence (~80M) provient probablement de k-mers qui étaient comptés dans plusieurs super-kmers (aux jonctions) et qui ne sont comptés qu'une fois dans les unitigs (déduplication exacte par le graphe de De Bruijn).
-
-### Conclusion
-
-L'approche unitig est massivement plus compacte que toutes les alternatives. Le format de stockage final devrait être basé sur les unitigs (ou au minimum sur les super-kmers dérepliqués) plutôt que sur des k-mers individuels en delta-varint.
-
-## 10. Questions ouvertes
-
-### 10.1 Le format super-kmer est-il toujours meilleur que delta-varint ?
-
-D'après les estimations révisées (section 8.3), le format super-kmer 2-bit est **toujours plus compact** que le delta-varint, même pour des génomes assemblés :
-- Reads bruts 15× : ~500 Mo vs ~1.5 Go (facteur 3×, à k-mers égaux) + déréplication massive
-- Génomes assemblés : ~1.2 bytes/k-mer vs ~5 bytes/k-mer (facteur 4×)
-
-La raison fondamentale : le delta-varint encode chaque k-mer indépendamment (même avec deltas), tandis que le super-kmer exploite le chevauchement de (k-1) bases entre k-mers consécutifs. C'est un avantage structurel irrattrapable par le delta-varint.
-
-**Le format super-kmer semble donc préférable dans tous les cas.**
-
-### 10.2 L'index doit-il stocker les super-kmers ou les k-mers ?
-
-Stocker les super-kmers/unitigs comme format d'index final a des avantages (compacité, scan naturel) mais des inconvénients :
-- Pas de seek rapide vers un k-mer spécifique (vs .kdx sparse index)
-- Le matching par scan complet est O(total_bases) vs O(Q + K) pour le merge-scan
-- Les opérations ensemblistes (Union, Intersect) deviennent plus complexes
-
-**Approche hybride possible** :
-1. Phase de construction : lowmask → super-kmers canoniques → déréplication → filtre de fréquence
-2. Phase de finalisation : extraire les k-mers uniques des super-kmers filtrés → delta-varint .kdi (v1 ou v2)
-3. Les super-kmers servent de **format intermédiaire efficace**, pas de format d'index final
-
-Cela combine le meilleur des deux mondes :
-- Déréplication ultra-efficace au niveau super-kmer (facteur 16× sur reads bruts)
-- Index final compact et query-efficient en delta-varint
-
-### 10.3 Le filtre de fréquence simple (niveau super-kmer) est-il suffisant ?
-
-À valider expérimentalement :
-- Comparer le nombre de k-mers retenus par filtre super-kmer vs filtre k-mer exact
-- Mesurer l'impact sur les métriques biologiques (Jaccard, match positions)
-- Si la différence est <1%, le filtre simple suffit
-
-### 10.4 Aho-Corasick vs merge-scan pour le matching final ?
-
-Si le format d'index final reste delta-varint (question 9.2), le merge-scan reste la méthode naturelle de matching. L'AC/hash-set n'a d'intérêt que si le format de stockage est basé sur des séquences (unitigs/super-kmers).
-
-## 11. Prochaine étape : validation expérimentale
-
-Avant de modifier l'architecture, valider sur des données réelles :
-
-1. **Taux de compaction super-kmer** : sur un génome assemblé vs reads bruts, mesurer le nombre de super-kmers uniques et leur longueur moyenne
-2. **Impact du filtre super-kmer** : comparer filtre au niveau super-kmer vs filtre au niveau k-mer exact sur un jeu de données de référence
-3. **Taux d'assembly en unitigs** : mesurer la longueur des unitigs obtenus à partir des super-kmers dérepliqués
-4. **Benchmark stockage** : comparer taille index super-kmer vs delta-varint vs unitig sur les mêmes données
-5. **Benchmark matching** : comparer temps de matching AC/hash vs merge-scan sur différentes densités de requêtes

blackboard/Prospective/large_sequence_parsing.md

@@ -0,0 +1,264 @@
+# Optimisation du parsing des grandes séquences
+
+## Contexte
+
+OBITools4 doit pouvoir traiter des séquences de taille chromosomique (plusieurs Gbp), notamment
+issues de fichiers GenBank/EMBL (assemblages de génomes) ou de fichiers FASTA convertis depuis
+ces formats.
+
+## Architecture actuelle
+
+### Pipeline de lecture (`pkg/obiformats/`)
+
+```
+ReadFileChunk (goroutine)
+    → ChannelFileChunk
+    → N × _ParseGenbankFile / _ParseFastaFile (goroutines)
+    → IBioSequence
+```
+
+`ReadFileChunk` (`file_chunk_read.go`) lit le fichier par morceaux via une chaîne de
+`PieceOfChunk` (rope). Chaque nœud fait `fileChunkSize` bytes :
+
+- GenBank/EMBL : 128 MB (`1024*1024*128`)
+- FASTA/FASTQ  : 1 MB (`1024*1024`)
+
+La chaîne est accumulée jusqu'à trouver la fin du dernier enregistrement complet (splitter),
+puis `Pack()` est appelé pour fusionner tous les nœuds en un seul buffer contigu. Ce buffer
+est transmis au parseur via `FileChunk.Raw *bytes.Buffer`.
+
+### Parseur GenBank (`genbank_read.go`)
+
+`GenbankChunkParser` reçoit un `io.Reader` sur le buffer packé, lit ligne par ligne via
+`bufio.NewReader` (buffer 4096 bytes), et pour chaque ligne de la section `ORIGIN` :
+
+```go
+line = string(bline)                        // allocation par ligne
+cleanline := strings.TrimSpace(line)        // allocation
+parts := strings.SplitN(cleanline, " ", 7) // allocation []string + substrings
+for i := 1; i < lparts; i++ {
+    seqBytes.WriteString(parts[i])
+}
+```
+
+Point positif : `seqBytes` est pré-alloué grâce à `lseq` extrait de la ligne `LOCUS`.
+
+### Parseur FASTA (`fastaseq_read.go`)
+
+`FastaChunkParser` lit **octet par octet** via `scanner.ReadByte()`. Pour 3 Gbp :
+3 milliards d'appels. `seqBytes` est un `bytes.Buffer{}` sans pré-allocation.
+
+## Problème principal
+
+Pour une séquence de plusieurs Gbp, `Pack()` fusionne une chaîne de ~N nœuds de 128 MB en
+un seul buffer contigu. C'est une allocation de N × 128 MB suivie d'une copie de toutes les
+données. Bien que l'implémentation de `Pack()` soit efficace (libère les nœuds au fur et à
+mesure via `slices.Grow`), la copie est inévitable avec l'architecture actuelle.
+
+De plus, le parseur GenBank produit des dizaines de millions d'allocations temporaires pour
+parser la section `ORIGIN` (une par ligne).
+
+## Invariant clé découvert
+
+**Si la rope a plus d'un nœud, le premier nœud seul ne se termine pas sur une frontière
+d'enregistrement** (pas de `//\n` en fin de `piece1`).
+
+Preuve par construction dans `ReadFileChunk` :
+- `splitter` est appelé dès le premier nœud (ligne 157)
+- Si `end >= 0` → frontière trouvée dans 128 MB → boucle interne sautée → rope à 1 nœud
+- Si `end < 0` → boucle interne ajoute des nœuds → rope à ≥ 2 nœuds
+
+Corollaire : si rope à 1 nœud, `Pack()` ne fait rien (aucun nœud suivant).
+
+**Attention** : rope à ≥ 2 nœuds ne signifie pas qu'il n'y a qu'une seule séquence dans
+la rope. La rope packée peut contenir plusieurs enregistrements complets. Exemple : records
+de 80 MB → `nextpieces` (48 MB de reste) + nouveau nœud (128 MB) = rope à 2 nœuds
+contenant 2 records complets + début d'un troisième.
+
+L'invariant dit seulement que `piece1` seul est incomplet — pas que la rope entière
+ne contient qu'un seul record.
+
+**Invariant : le dernier FileChunk envoyé finit sur une frontière d'enregistrement.**
+
+Deux chemins dans `ReadFileChunk` :
+
+1. **Chemin normal** (`end >= 0` via `splitter`) : le buffer est explicitement tronqué à
+   `end` (ligne 200 : `pieces.data = pieces.data[:end]`). Frontière garantie par construction
+   pour tous les formats. ✓
+
+2. **Chemin EOF** (`end < 0`, `end = pieces.Len()`) : tout le reste du fichier est envoyé.
+   - **GenBank/EMBL** : présuppose fichier bien formé (se termine par `//\n`). Le parseur
+     lève un `log.Fatalf` sur tout état inattendu — filet de sécurité suffisant. ✓
+   - **FASTQ** : présupposé, vérifié par le parseur. ✓
+   - **FASTA** : garanti par le format lui-même (fin d'enregistrement = EOF ou `>`). ✓
+
+**Hypothèse de travail adoptée** : les fichiers d'entrée sont bien formés. Dans le pire cas,
+le parseur lèvera une erreur explicite. Il n'y a pas de risque de corruption silencieuse.
+
+## Piste d'optimisation : se dispenser de Pack()
+
+### Idée centrale
+
+Au lieu de fusionner la rope avant de la passer au parseur, **parser directement la rope
+nœud par nœud**, et **écrire la séquence compactée in-place dans le premier nœud**.
+
+Pourquoi c'est sûr :
+- Le header (LOCUS, DEFINITION, SOURCE, FEATURES) est **petit** et traité en premier
+- La séquence (ORIGIN) est **à la fin** du record
+- Au moment d'écrire la séquence depuis l'offset 0 de `piece1`, le pointeur de lecture
+  est profond dans la rope (offset >> 0) → jamais de collision
+- La séquence compactée est toujours plus courte que les données brutes
+
+### Pré-allocation
+
+Pour GenBank/EMBL : `lseq` est connu dès la ligne `LOCUS`/`ID` (première ligne, dans
+`piece1`). On peut faire `slices.Grow(piece1.data, lseq)` dès ce moment.
+
+Pour FASTA : pas de taille garantie dans le header, mais `rope.Len()` donne un majorant.
+On peut utiliser `rope.Len() / 2` comme estimation initiale.
+
+### Gestion des jonctions entre nœuds
+
+Une ligne peut chevaucher deux nœuds (rare avec 128 MB, mais possible). Solution : carry
+buffer de ~128 bytes pour les quelques bytes en fin de nœud.
+
+### Cas FASTA/FASTQ multi-séquences
+
+Un FileChunk peut contenir N séquences (notamment FASTA/FASTQ courts). Dans ce cas
+l'écriture in-place dans `piece1` n'est pas applicable directement — on écrase des données
+nécessaires aux séquences suivantes.
+
+Stratégie par cas :
+- **Rope à 1 nœud** (record ≤ 128 MB) : `Pack()` est trivial (no-op), parseur actuel OK
+- **Rope à ≥ 2 nœuds** : par l'invariant, `piece1` ne contient pas de record complet →
+  une seule grande séquence → in-place applicable
+
+### Format d'une ligne séquence GenBank (Après ORIGIN)
+
+```
+/^ *[0-9]+( [nuc]{10}){0,5} [nuc]{1,10}/
+```
+
+### Format d'une ligne séquence GenBank (Après SQ)
+
+La ligne SQ contient aussi la taille de la séquence
+
+```
+/^ *( [nuc]{10}){0,5} [nuc]{1,10} *[0-9]+/
+```
+
+Compactage in-place sur `bline` ([]byte brut, sans conversion `string`) :
+
+```go
+w := 0
+i := 0
+for i < len(bline) && bline[i] == ' '  { i++ }   // skip indentation
+for i < len(bline) && bline[i] <= '9'  { i++ }   // skip position number
+for ; i < len(bline); i++ {
+    if bline[i] != ' ' {
+        bline[w] = bline[i]
+        w++
+    }
+}
+// écrire bline[:w] directement dans piece1.data[seqOffset:]
+```
+
+## Changements nécessaires
+
+1. **`FileChunk`** : exposer la rope `*PieceOfChunk` non-packée en plus (ou à la place)
+   de `Raw *bytes.Buffer`
+2. **`GenbankChunkParser` / `EmblChunkParser`** : accepter `*PieceOfChunk`, parser la
+   rope séquentiellement avec carry buffer pour les jonctions
+3. **`FastaChunkParser`** : idem, avec in-place conditionnel selon taille de la rope
+4. **`ReadFileChunk`** : ne pas appeler `Pack()` avant envoi sur le channel (ou version
+   alternative `ReadFileChunkRope`)
+
+## Fichiers concernés
+
+- `pkg/obiformats/file_chunk_read.go` — structure rope, `ReadFileChunk`
+- `pkg/obiformats/genbank_read.go` — `GenbankChunkParser`, `_ParseGenbankFile`
+- `pkg/obiformats/embl_read.go` — `EmblChunkParser`, `ReadEMBL`
+- `pkg/obiformats/fastaseq_read.go` — `FastaChunkParser`, `_ParseFastaFile`
+- `pkg/obiformats/fastqseq_read.go` — parseur FASTQ (même structure)
+
+## Plan d'implémentation : parseur GenBank sur rope
+
+### Contexte
+
+Baseline mesurée : `obiconvert gbpln640.seq.gz` → 49s real, 42s user, 29s sys, **57 GB RSS**.
+Le sys élevé indique des allocations massives. Deux causes :
+1. `Pack()` : fusionne toute la rope (N × 128 MB) en un buffer contigu avant de parser
+2. Parser ORIGIN : `string(bline)` + `TrimSpace` + `SplitN` × millions de lignes
+
+### 1. `gbRopeScanner`
+
+Struct de lecture ligne par ligne sur la rope, sans allocation heap :
+
+```go
+type gbRopeScanner struct {
+    current *PieceOfChunk
+    pos     int
+    carry   [256]byte  // stack-allocated, max GenBank line = 80 chars
+    carryN  int
+}
+```
+
+`ReadLine()` :
+- Cherche `\n` dans `current.data[pos:]` via `bytes.IndexByte`
+- Si trouvé sans carry : retourne slice direct du node (zéro alloc)
+- Si trouvé avec carry : copie dans carry buffer, retourne `carry[:n]`
+- Si non trouvé : copie le reste dans carry, avance au node suivant, recommence
+- EOF : retourne `carry[:carryN]` puis nil
+
+`extractSequence(dest []byte, UtoT bool) int` :
+- Scan direct des bytes pour section ORIGIN, sans passer par ReadLine
+- Machine d'états : lineStart → skip espaces/digits → copier nucléotides dans dest
+- Stop sur `//` en début de ligne
+- Zéro allocation, UtoT inline
+
+### 2. `GenbankChunkParserRope`
+
+```go
+func GenbankChunkParserRope(source string, rope *PieceOfChunk,
+    withFeatureTable, UtoT bool) (obiseq.BioSequenceSlice, error)
+```
+
+- Même machine d'états que `GenbankChunkParser`, sur `[]byte` (`bytes.HasPrefix`)
+- LOCUS : extrait `id` et `lseq` par scan direct (remplace `_seqlenght_rx`)
+- FEATURES / default inFeature : taxid extrait par scan de `/db_xref="taxon:`
+  dans la source feature ; `featBytes` rempli seulement si `withFeatureTable=true`
+- DEFINITION : toujours conservée
+- ORIGIN : `dest = make([]byte, 0, lseq+20)` puis `s.extractSequence(dest, UtoT)`
+
+### 3. Modifications `_ParseGenbankFile` et `ReadGenbank`
+
+`_ParseGenbankFile` utilise `chunk.Rope` :
+```go
+sequences, err := GenbankChunkParserRope(chunk.Source, chunk.Rope, ...)
+```
+
+`ReadGenbank` passe `pack=false` :
+```go
+entry_channel := ReadFileChunk(..., false)
+```
+
+### 4. Ce qui NE change pas
+
+- `GenbankChunkParser` reste (référence, tests)
+- `ReadFileChunk`, `Pack()`, autres parseurs (EMBL, FASTA, FASTQ) : inchangés
+
+### 5. Gains attendus
+
+- **RSS** : pic ≈ 128 MB × workers (au lieu de N × 128 MB)
+- **Temps sys** : élimination des mmap/munmap pour les gros buffers
+- **Temps user** : ~50M allocations éliminées
+
+### 6. Vérification
+
+```bash
+/usr/local/go/bin/go build ./...
+diff <(obiconvert gbpln640.seq.gz) gbpln640.reference.fasta
+cd bugs/genbank && ./benchmark.sh gbpln640.seq.gz
+```
+
+Cible : RSS < 1 GB, temps comparable ou meilleur.

blackboard/ToDo/Canonical-superkmers.md

@@ -1,3 +0,0 @@
-lit le ficier [@canonical-super-kmer-strategy.md](file:///Users/coissac/Sync/travail/__MOI__/GO/obitools4/blackboard/Prospective/canonical-super-kmer-strategy.md).
-
-Dans le fichier [@superkmer_iter.go](file:///Users/coissac/Sync/travail/__MOI__/GO/obitools4/pkg/obikmer/superkmer_iter.go) implemente une nouvelle fonction IterCanonicalSuperKmers sur le modèle de IterSuperKmers, qui implémente la notion de SuperKmers canonique présenté dans le document d'architecture.

cmd/obitools/obialign/main.go

@@ -0,0 +1,26 @@
+package main
+
+import (
+	"os"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obialign"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+)
+
+func main() {
+	optionParser := obioptions.GenerateOptionParser(
+		"obialign",
+		"pairwise similarity of nucleotide sequences, as a CSV table (the principle of sumatra)",
+		obialign.OptionSet)
+
+	_, args := optionParser(os.Args)
+
+	fs, err := obiconvert.CLIReadBioSequences(args...)
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
+
+	obialign.CLIWriteTable(obialign.CLIOBIAlign(fs))
+
+	obiutils.WaitForLastPipe()
+}

cmd/obitools/obiclust/main.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"os"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiclust"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+)
+
+func main() {
+	optionParser := obioptions.GenerateOptionParser(
+		"obiclust",
+		"star clustering of sequences (the algorithm of sumaclust, UCLUST and CD-HIT)",
+		obiclust.OptionSet)
+
+	_, args := optionParser(os.Args)
+
+	fs, err := obiconvert.CLIReadBioSequences(args...)
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
+
+	clustered := obiclust.CLIOBIClust(fs)
+
+	obiconvert.CLIWriteBioSequences(clustered, true)
+
+	obiutils.WaitForLastPipe()
+}

cmd/obitools/obioligotag/main.go

@@ -0,0 +1,33 @@
+package main
+
+import (
+	"os"
+
+	log "github.com/sirupsen/logrus"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obioligotag"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+)
+
+func main() {
+	optionParser := obioptions.GenerateOptionParser(
+		"obioligotag",
+		"designs a family of oligonucleotides (tags) at a minimum distance from each other",
+		obioligotag.OptionSet)
+
+	_, args := optionParser(os.Args)
+	if len(args) > 0 {
+		log.Fatalf("unexpected arguments %v: use --oligo-list to provide candidate oligonucleotides", args)
+	}
+
+	oligos, err := obioligotag.CLIOligoFamily()
+	if err != nil {
+		log.Fatalf("%v", err)
+	}
+
+	obiconvert.CLIWriteBioSequences(obioligotag.CLIOligosToIterator(oligos), true)
+
+	obiutils.WaitForLastPipe()
+}

cmd/obitools/obipairing/main.go

@@ -37,6 +37,11 @@ func main() {
 
 	optionParser(os.Args)
 
+	if !obipairing.CLIHasPairedFiles() {
+		log.Error("You must provide both a forward file (-F) and a reverse file (-R)")
+		os.Exit(1)
+	}
+
 	obidefault.SetStrictReadWorker(2)
 	obidefault.SetStrictWriteWorker(2)
 	pairs, err := obipairing.CLIPairedSequence()

cmd/obitools/obiprimers/main.go

@@ -0,0 +1,38 @@
+package main
+
+import (
+	"os"
+
+	log "github.com/sirupsen/logrus"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obicsv"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiprimers"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+)
+
+func main() {
+	obiprimers.SetReadingDefaults()
+
+	optionParser := obioptions.GenerateOptionParser(
+		"obiprimers",
+		"designs the primer pairs amplifying a set of example sequences and evaluates the DNA barcode they delimit. "+
+			"The taxonomy comes from the -t option, or is read from the taxonomic paths embedded in the sequences (obiannotate --taxonomic-path)",
+		obiprimers.OptionSet)
+
+	_, args := optionParser(os.Args)
+
+	sequences, taxonomy, err := obiprimers.CLIReadSequencesAndTaxonomy(args)
+	if err != nil {
+		log.Fatalf("Cannot read the sequences: %v", err)
+	}
+
+	pairs, err := obiprimers.CLIPrimerPairs(sequences, taxonomy)
+	if err != nil {
+		log.Fatalf("%v", err)
+	}
+
+	obicsv.CLICSVWriter(obiprimers.PairsToCSVIterator(pairs, obiprimers.CLIReference() != ""), true)
+
+	obiutils.WaitForLastPipe()
+}

cmd/obitools/obitagpcr/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"fmt"
 	"os"
 
 	log "github.com/sirupsen/logrus"
@@ -8,6 +9,7 @@ import (
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obimultiplex"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obipairing"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obitagpcr"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
@@ -39,6 +41,17 @@ func main() {
 		obitagpcr.OptionSet)
 
 	optionParser(os.Args)
+
+	if obimultiplex.CLIAskConfigTemplate() {
+		fmt.Print(obimultiplex.CLIConfigTemplate())
+		os.Exit(0)
+	}
+
+	if !obipairing.CLIHasPairedFiles() {
+		log.Error("You must provide both a forward file (-F) and a reverse file (-R)")
+		os.Exit(1)
+	}
+
 	pairs, err := obipairing.CLIPairedSequence()
 
 	if err != nil {

doc/book/installation.qmd

@@ -8,7 +8,7 @@ All the sources of the [*OBITools4*](http://metabarcoding.org/obitools4) can be
 
 ## Prerequisites
 
-The *OBITools4* are developped using the [GO programming language](https://go.dev/), we stick to the latest version of the language, today the $1.21.4$. If you want to download and compile the sources yourself, you first need to install the corresponding compiler on your system. Some parts of the soft are also written in C, therefore a recent C compiler is also requested, GCC on Linux or Windows, the Developer Tools on Mac.
+The *OBITools4* are developped using the [GO programming language](https://go.dev/), we stick to the latest version of the language, today the $1.27$. If you want to download and compile the sources yourself, you first need to install the corresponding compiler on your system. Some computations (distances between k-mers) use the experimental `simd` package of Go: the `Makefile` compiles them with `GOEXPERIMENT=simd`, and the installation script uses a Go 1.27 compiler for that reason. The resulting programs run on any processor, a portable code being used when no SIMD instruction is available. Some parts of the soft are also written in C, therefore a recent C compiler is also requested, GCC on Linux or Windows, the Developer Tools on Mac.
 
 Whatever the installation you decide for, you will have to ensure that a C compiler is available on your system.
 

entities.json

@@ -0,0 +1,10 @@
+{
+  "people": [
+    "Software",
+    "Agreement",
+    "Module"
+  ],
+  "projects": [
+    "Code"
+  ]
+}

go.mod

@@ -1,35 +1,33 @@
 module git.metabarcoding.org/obitools/obitools4/obitools4
 
-go 1.23.4
-
-toolchain go1.24.2
+go 1.27
 
 require (
-	github.com/DavidGamba/go-getoptions v0.28.0
-	github.com/PaesslerAG/gval v1.2.2
+	github.com/DavidGamba/go-getoptions v0.33.0
+	github.com/PaesslerAG/gval v1.2.4
 	github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df
-	github.com/buger/jsonparser v1.1.1
+	github.com/buger/jsonparser v1.1.2
 	github.com/chen3feng/stl4go v0.1.1
-	github.com/dlclark/regexp2 v1.11.4
-	github.com/goccy/go-json v0.10.3
+	github.com/dlclark/regexp2 v1.11.5
+	github.com/goccy/go-json v0.10.6
 	github.com/klauspost/pgzip v1.2.6
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
 	github.com/pelletier/go-toml/v2 v2.2.4
 	github.com/rrethy/ahocorasick v1.0.0
-	github.com/schollz/progressbar/v3 v3.13.1
-	github.com/sirupsen/logrus v1.9.3
-	github.com/stretchr/testify v1.8.4
+	github.com/schollz/progressbar/v3 v3.19.0
+	github.com/sirupsen/logrus v1.9.4
+	github.com/stretchr/testify v1.10.0
 	github.com/tevino/abool/v2 v2.1.0
 	github.com/yuin/gopher-lua v1.1.1
-	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa
-	gonum.org/v1/gonum v0.14.0
+	golang.org/x/exp v0.0.0-20260312153236-7ab1446f8b90
+	gonum.org/v1/gonum v0.17.0
 	gopkg.in/yaml.v3 v3.0.1
 	scientificgo.org/special v0.0.0
 )
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/goombaio/orderedmap v0.0.0-20180924084748-ba921b7e2419 // indirect
+	github.com/goombaio/orderedmap v0.0.0-20180925151256-3da0e2f905f9 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
@@ -38,16 +36,15 @@ require (
 
 require (
 	github.com/dsnet/compress v0.0.1
-	github.com/gabriel-vasile/mimetype v1.4.3
+	github.com/gabriel-vasile/mimetype v1.4.13
 	github.com/goombaio/orderedset v0.0.0-20180925151225-8e67b20a9b77
-	github.com/klauspost/compress v1.17.2
-	github.com/mattn/go-runewidth v0.0.15 // indirect
+	github.com/klauspost/compress v1.18.4
+	github.com/mattn/go-runewidth v0.0.21 // indirect
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
-	github.com/rivo/uniseg v0.4.4 // indirect
-	github.com/shopspring/decimal v1.3.1 // indirect
-	github.com/ulikunitz/xz v0.5.11
-	golang.org/x/net v0.35.0 // indirect
-	golang.org/x/sys v0.30.0 // indirect
-	golang.org/x/term v0.29.0 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/shopspring/decimal v1.4.0 // indirect
+	github.com/ulikunitz/xz v0.5.15
+	golang.org/x/sys v0.42.0 // indirect
+	golang.org/x/term v0.41.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c
 )

go.sum

@@ -1,36 +1,41 @@
-github.com/DavidGamba/go-getoptions v0.28.0 h1:18wgEvfZdrlfIhVDGEBO3Dl0fkOyXqXLa0tLMCKxM1c=
-github.com/DavidGamba/go-getoptions v0.28.0/go.mod h1:zE97E3PR9P3BI/HKyNYgdMlYxodcuiC6W68KIgeYT84=
-github.com/PaesslerAG/gval v1.2.2 h1:Y7iBzhgE09IGTt5QgGQ2IdaYYYOU134YGHBThD+wm9E=
-github.com/PaesslerAG/gval v1.2.2/go.mod h1:XRFLwvmkTEdYziLdaCeCa5ImcGVrfQbeNUbVR+C6xac=
+github.com/DavidGamba/go-getoptions v0.33.0 h1:8xCPH87Yy5avYenygyHVlqqm8RpymH0YFe4a7IWlarE=
+github.com/DavidGamba/go-getoptions v0.33.0/go.mod h1:zE97E3PR9P3BI/HKyNYgdMlYxodcuiC6W68KIgeYT84=
+github.com/PaesslerAG/gval v1.2.4 h1:rhX7MpjJlcxYwL2eTTYIOBUyEKZ+A96T9vQySWkVUiU=
+github.com/PaesslerAG/gval v1.2.4/go.mod h1:XRFLwvmkTEdYziLdaCeCa5ImcGVrfQbeNUbVR+C6xac=
 github.com/PaesslerAG/jsonpath v0.1.0 h1:gADYeifvlqK3R3i2cR5B4DGgxLXIPb3TRTH1mGi0jPI=
 github.com/PaesslerAG/jsonpath v0.1.0/go.mod h1:4BzmtoM/PI8fPO4aQGIusjGxGir2BzcV0grWtFzq1Y8=
 github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df h1:GSoSVRLoBaFpOOds6QyY1L8AX7uoY+Ln3BHc22W40X0=
 github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df/go.mod h1:hiVxq5OP2bUGBRNS3Z/bt/reCLFNbdcST6gISi1fiOM=
-github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
-github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
+github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk=
+github.com/buger/jsonparser v1.1.2/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
 github.com/chen3feng/stl4go v0.1.1 h1:0L1+mDw7pomftKDruM23f1mA7miavOj6C6MZeadzN2Q=
 github.com/chen3feng/stl4go v0.1.1/go.mod h1:5ml3psLgETJjRJnMbPE+JiHLrCpt+Ajc2weeTECXzWU=
+github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM=
+github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY=
+github.com/clipperhouse/uax29/v2 v2.2.0 h1:ChwIKnQN3kcZteTXMgb1wztSgaU+ZemkgWdohwgs8tY=
+github.com/clipperhouse/uax29/v2 v2.2.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo=
-github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
+github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ=
+github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
 github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q=
 github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo=
 github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
-github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
-github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
-github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
-github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
-github.com/goombaio/orderedmap v0.0.0-20180924084748-ba921b7e2419 h1:SajEQ6tktpF9SRIuzbiPOX9AEZZ53Bvw0k9Mzrts8Lg=
+github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM=
+github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
+github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU=
+github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/goombaio/orderedmap v0.0.0-20180924084748-ba921b7e2419/go.mod h1:YKu81H3RSd1cFh0d7NhvUoTtUC9IY/vBX0WUQb1/o4Y=
+github.com/goombaio/orderedmap v0.0.0-20180925151256-3da0e2f905f9 h1:vFjPvFavIiDY71bQ9HIxPQBANvNl1SmFC4fgg5xRkho=
+github.com/goombaio/orderedmap v0.0.0-20180925151256-3da0e2f905f9/go.mod h1:YKu81H3RSd1cFh0d7NhvUoTtUC9IY/vBX0WUQb1/o4Y=
 github.com/goombaio/orderedset v0.0.0-20180925151225-8e67b20a9b77 h1:4dvq1tGHn1Y9KSRY0OZ24Khki4+4U+ZrA//YYsdUlJU=
 github.com/goombaio/orderedset v0.0.0-20180925151225-8e67b20a9b77/go.mod h1:HPelMYpOyy0XvglpBbmZ3krZpwaHmszj/vQNlnETPTM=
-github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
 github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
-github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
-github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
+github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
 github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU=
 github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
@@ -41,10 +46,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
-github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/mattn/go-runewidth v0.0.21 h1:jJKAZiQH+2mIinzCJIaIG9Be1+0NR+5sz/lYEEjdM8w=
+github.com/mattn/go-runewidth v0.0.21/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
@@ -54,50 +57,40 @@ github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
-github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
-github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
 github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
 github.com/rrethy/ahocorasick v1.0.0 h1:YKkCB+E5PXc0xmLfMrWbfNht8vG9Re97IHSWZk/Lk8E=
 github.com/rrethy/ahocorasick v1.0.0/go.mod h1:nq8oScE7Vy1rOppoQxpQiiDmPHuKCuk9rXrNcxUV3R0=
-github.com/schollz/progressbar/v3 v3.13.1 h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE=
-github.com/schollz/progressbar/v3 v3.13.1/go.mod h1:xvrbki8kfT1fzWzBT/UZd9L6GA+jdL7HAgq2RFnO6fQ=
-github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
+github.com/schollz/progressbar/v3 v3.19.0 h1:Ea18xuIRQXLAUidVDox3AbwfUhD0/1IvohyTutOIFoc=
+github.com/schollz/progressbar/v3 v3.19.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
 github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
-github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
-github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
+github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
+github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
+github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/tevino/abool/v2 v2.1.0 h1:7w+Vf9f/5gmKT4m4qkayb33/92M+Um45F2BkHOR+L/c=
 github.com/tevino/abool/v2 v2.1.0/go.mod h1:+Lmlqk6bHDWHqN1cbxqhwEAwMPXgc8I1SDEamtseuXY=
 github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
-github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
-github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
+github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY=
+github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
 github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
-golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ=
-golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE=
-golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8=
-golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk=
-golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
-golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
-golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU=
-golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s=
-gonum.org/v1/gonum v0.14.0 h1:2NiG67LD1tEH0D7kM+ps2V+fXmsAnpUeec7n8tcr4S0=
-gonum.org/v1/gonum v0.14.0/go.mod h1:AoWeoz0becf9QMWtE8iWXNXc27fK4fNeHNf/oMejGfU=
+golang.org/x/exp v0.0.0-20260312153236-7ab1446f8b90 h1:jiDhWWeC7jfWqR9c/uplMOqJ0sbNlNWv0UkzE0vX1MA=
+golang.org/x/exp v0.0.0-20260312153236-7ab1446f8b90/go.mod h1:xE1HEv6b+1SCZ5/uscMRjUBKtIxworgEcEi+/n9NQDQ=
+golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
+golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU=
+golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A=
+gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
+gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 scientificgo.org/special v0.0.0 h1:P6WJkECo6tgtvZAEfNXl+KEB9ReAatjKAeX8U07mjSc=

go.work

@@ -1,5 +1,3 @@
-go 1.23.4
-
-toolchain go1.24.2
+go 1.27
 
 use .

go.work.sum

@@ -52,6 +52,8 @@ golang.org/x/image v0.6.0/go.mod h1:MXLdDR43H7cDJq5GEGXEVeeNhPgi+YYEQ2pC1byI1x0=
 golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY=
 golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
+golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
 golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=

install_obitools.sh

@@ -2,11 +2,17 @@
 
 # Default values
 URL="https://go.dev/dl/"
+# Series of the Go compiler used to build OBITools4. The SIMD code relies on the
+# experimental simd package of Go (GOEXPERIMENT=simd), whose API may change
+# between Go versions: the build is therefore tied to the series the code was
+# written for rather than to the latest release.
+GO_SERIES="1.27"
 GITHUB_REPO="https://github.com/metabarcoding/obitools4"
 INSTALL_DIR="/usr/local"
 OBITOOLS_PREFIX=""
 VERSION=""
 LIST_VERSIONS=false
+JOBS=1
 
 # Help message
 function display_help {
@@ -21,6 +27,7 @@ function display_help {
   echo "                          gobigrep command instead of obigrep)."
   echo "  -v, --version           Install a specific version (e.g., 4.4.8)."
   echo "                          If not specified, installs the latest version."
+  echo "  -j, --jobs              Number of parallel jobs for compilation (default: 1)."
   echo "  -l, --list              List all available versions and exit."
   echo "  -h, --help              Display this help message."
   echo ""
@@ -65,6 +72,10 @@ while [ "$#" -gt 0 ]; do
       VERSION="$2"
       shift 2
       ;;
+    -j|--jobs)
+      JOBS="$2"
+      shift 2
+      ;;
     -l|--list)
       LIST_VERSIONS=true
       shift
@@ -122,9 +133,15 @@ mkdir -p "${WORK_DIR}/cache" \
       exit 1)
 
 # Create installation directory
-mkdir -p "${INSTALL_DIR}/bin" 2> /dev/null \
-  || (echo "Please enter your password for installing obitools in ${INSTALL_DIR}" 1>&2
-      sudo mkdir -p "${INSTALL_DIR}/bin")
+if ! mkdir -p "${INSTALL_DIR}/bin" 2>/dev/null; then
+  if [ ! -w "$(dirname "${INSTALL_DIR}")" ] && [ ! -w "${INSTALL_DIR}" ]; then
+    echo "Please enter your password for installing obitools in ${INSTALL_DIR}" 1>&2
+    sudo mkdir -p "${INSTALL_DIR}/bin"
+  else
+    echo "Error: Could not create ${INSTALL_DIR}/bin (check path or disk space)" 1>&2
+    exit 1
+  fi
+fi
 
 if [[ ! -d "${INSTALL_DIR}/bin" ]]; then
   echo "Could not create ${INSTALL_DIR}/bin directory for installing obitools" 1>&2
@@ -162,31 +179,43 @@ GOFILE=$(curl -s "$URL" \
             | grep 'class="download"' \
             | grep "\.tar\.gz" \
             | sed -E 's@^.*/dl/(go[1-9].+\.tar\.gz)".*$@\1@' \
+            | awk -v prefix="go${GO_SERIES}." 'index($0, prefix) == 1' \
             | grep -i "$OS" \
             | grep -i "$ARCH" \
             | head -1)
 
+if [ -z "$GOFILE" ]; then
+  echo "Error: no Go ${GO_SERIES} archive found for ${OS}/${ARCH} at ${URL}" 1>&2
+  exit 1
+fi
+
 GOURL=$(curl -s "${URL}${GOFILE}" \
         | sed -E 's@^.*href="(.*\.tar\.gz)".*$@\1@')
 
 echo "Installing Go from: $GOURL" 1>&2
 
-curl -s "$GOURL" | tar zxf -
+curl --progress-bar "$GOURL" | tar zxf -
 
-PATH="$(pwd)/go/bin:$PATH"
+export GOROOT="$(pwd)/go"
+PATH="${GOROOT}/bin:$PATH"
 export PATH
-GOPATH="$(pwd)/go"
-export GOPATH
+export GOPATH="$(pwd)/gopath"
 export GOCACHE="$(pwd)/cache"
+export GOTOOLCHAIN=local
+# OBITools4 is built with the SIMD code enabled (it falls back on a portable
+# code at run time if the processor has no SIMD instructions).
+export GOEXPERIMENT=simd
 
+echo "GOROOT=$GOROOT" 1>&2
+echo "GOEXPERIMENT=$GOEXPERIMENT" 1>&2
 echo "GOCACHE=$GOCACHE" 1>&2
-mkdir -p "$GOCACHE"
+mkdir -p "$GOPATH" "$GOCACHE"
 
 # Download OBITools4 source
 echo "Downloading OBITools4 v${VERSION}..." 1>&2
 echo "Source URL: $OBIURL4" 1>&2
 
-if ! curl -sL "$OBIURL4" > obitools4.zip; then
+if ! curl --progress-bar -L "$OBIURL4" > obitools4.zip; then
   echo "Error: Could not download OBITools4 version ${VERSION}" 1>&2
   echo "Please check that this version exists with: $0 --list" 1>&2
   exit 1
@@ -208,16 +237,29 @@ mkdir -p vendor
 
 # Build with or without prefix
 if [[ -z "$OBITOOLS_PREFIX" ]] ; then
-  make GOFLAGS="-buildvcs=false"
+  make -j"${JOBS}" obitools GOFLAGS="-buildvcs=false"
 else
-  make GOFLAGS="-buildvcs=false" OBITOOLS_PREFIX="${OBITOOLS_PREFIX}"
+  make -j"${JOBS}" obitools GOFLAGS="-buildvcs=false" OBITOOLS_PREFIX="${OBITOOLS_PREFIX}"
 fi
 
 # Install binaries
 echo "Installing binaries to ${INSTALL_DIR}/bin..." 1>&2
-(cp build/* "${INSTALL_DIR}/bin" 2> /dev/null) \
-   || (echo "Please enter your password for installing obitools in ${INSTALL_DIR}" 1>&2
-       sudo cp build/* "${INSTALL_DIR}/bin")
+if ! cp build/* "${INSTALL_DIR}/bin" 2>/dev/null; then
+  if [ ! -w "${INSTALL_DIR}/bin" ]; then
+    echo "Please enter your password for installing obitools in ${INSTALL_DIR}" 1>&2
+    sudo cp build/* "${INSTALL_DIR}/bin"
+  else
+    echo "Error: Could not copy binaries to ${INSTALL_DIR}/bin" 1>&2
+    echo "  Source files: $(ls build/ 2>/dev/null || echo 'none found')" 1>&2
+    echo "" 1>&2
+    echo "The build directory has been preserved for manual recovery:" 1>&2
+    echo "  $(pwd)/build/" 1>&2
+    echo "You can install manually with:" 1>&2
+    echo "  cp $(pwd)/build/* ${INSTALL_DIR}/bin/" 1>&2
+    popd > /dev/null || true
+    exit 1
+  fi
+fi
 
 popd > /dev/null || exit
 

obitests/obitools/obialign/test.sh

@@ -0,0 +1,206 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obialign
+CMD=obialign
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1
+then
+    log "$MCMD: printing help OK"
+    ((success++))
+else
+    log "$MCMD: printing help failed"
+    ((failed++))
+fi
+
+
+#
+# Dataset used by the following tests: two families of similar sequences, a
+# sequence identical to another one, and a sequence with an ambiguous
+# nucleotide that is discarded.
+#
+cat > "${TMPDIR}/data.fasta" <<DATA
+>a1 count=20;
+acgtacgtacgtaacctagcctagcctagggatcgatcgatcgatcgaatcg
+>a2 count=3;
+acgtacgtacgtaacctagcctagcctagggatcgatcgatcgttcgaatcg
+>a3 count=2;
+acgtacgtacgtaacctagcctagcctagggatcgatcgatcgatcgaatcg
+>b1 count=15;
+ttgcatgcatgctagctagctagcatgcatcgatcgatgctagctagtagct
+>b2 count=1;
+ttgcatgcatgctagctagctagcatgcatcgatcgatgctagctagtaact
+>x1 count=4;
+ttgcatgcatgctangctagctagcatgcatcgatcgatgctagctagtagc
+DATA
+
+cat > "${TMPDIR}/ref.fasta" <<DATA
+>r1
+acgtacgtacgtaacctagcctagcctagggatcgatcgatcgatcgaatcg
+>r2
+ttgcatgcatgctagctagctagcatgcatcgatcgatgctagctagtagct
+>r3
+ggggccccggggccccggggccccggggccccggggccccggggcccc
+DATA
+
+((ntest++))
+if $CMD --max-cpu 2 "${TMPDIR}/data.fasta" > "${TMPDIR}/all.csv" 2> /dev/null &&
+   [ "$(head -n 1 "${TMPDIR}/all.csv")" = "id1,id2,score" ] &&
+   [ "$(tail -n +2 "${TMPDIR}/all.csv" | wc -l | tr -d ' ')" = "10" ]
+then
+    log "$MCMD: without threshold all the pairs of the 5 usable sequences are written OK"
+    ((success++))
+else
+    log "$MCMD: the table without threshold is wrong"
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD --max-cpu 2 -t 0.95 "${TMPDIR}/data.fasta" > "${TMPDIR}/t95.csv" 2> /dev/null &&
+   [ "$(tail -n +2 "${TMPDIR}/t95.csv" | cut -d, -f1,2 | tr '\n' ' ')" = "a1,a2 a1,a3 a2,a3 b1,b2 " ] &&
+   grep -q '^a1,a3,1$' "${TMPDIR}/t95.csv"
+then
+    log "$MCMD: only the pairs reaching the threshold are written, in order OK"
+    ((success++))
+else
+    log "$MCMD: the table with a threshold is wrong"
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD --max-cpu 2 -t 0.95 -x "${TMPDIR}/data.fasta" 2> /dev/null | head -n 2 | tail -n 1 | grep -q '^a1,a2,[0-9.]*,20,3,52,52$'
+then
+    log "$MCMD: extra columns OK"
+    ((success++))
+else
+    log "$MCMD: extra columns failed"
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD --max-cpu 2 -g -t 0.9 "${TMPDIR}/data.fasta" 2> /dev/null | grep -q '^b1,x1,'
+then
+    log "$MCMD: a sequence with an IUPAC letter is kept with --accept-ambiguous OK"
+    ((success++))
+else
+    log "$MCMD: --accept-ambiguous failed"
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD --max-cpu 2 -R "${TMPDIR}/ref.fasta" -t 0.95 "${TMPDIR}/data.fasta" 2> /dev/null | tail -n +2 | cut -d, -f1,2 | tr '\n' ' ' > "${TMPDIR}/two.txt" &&
+   [ "$(cat "${TMPDIR}/two.txt")" = "a1,r1 a2,r1 a3,r1 b1,r2 b2,r2 " ]
+then
+    log "$MCMD: comparison with a reference file OK"
+    ((success++))
+else
+    log "$MCMD: comparison with a reference file failed"
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD --max-cpu 1 "${TMPDIR}/data.fasta" 2> /dev/null > "${TMPDIR}/cpu1.csv" &&
+   $CMD --max-cpu 8 "${TMPDIR}/data.fasta" 2> /dev/null > "${TMPDIR}/cpu8.csv" &&
+   cmp -s "${TMPDIR}/cpu1.csv" "${TMPDIR}/cpu8.csv"
+then
+    log "$MCMD: the table does not depend on the number of threads OK"
+    ((success++))
+else
+    log "$MCMD: the table depends on the number of threads"
+    ((failed++))
+fi
+
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obiclust/test.sh

@@ -0,0 +1,188 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obiclust
+CMD=obiclust
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1
+then
+    log "$MCMD: printing help OK"
+    ((success++))
+else
+    log "$MCMD: printing help failed"
+    ((failed++))
+fi
+
+
+#
+# Dataset used by the following tests: two families with a rare variant each,
+# a duplicate, and a sequence with an ambiguous nucleotide that is discarded.
+#
+cat > "${TMPDIR}/data.fasta" <<DATA
+>a1 count=20;
+acgtacgtacgtaacctagcctagcctagggatcgatcgatcgatcgaatcg
+>a2 count=3;
+acgtacgtacgtaacctagcctagcctagggatcgatcgatcgttcgaatcg
+>a3 count=2;
+acgtacgtacgtaacctagcctagcctagggatcgatcgatcgatcgaatcg
+>b1 count=15;
+ttgcatgcatgctagctagctagcatgcatcgatcgatgctagctagtagct
+>b2 count=1;
+ttgcatgcatgctagctagctagcatgcatcgatcgatgctagctagtaact
+>x1 count=4;
+ttgcatgcatgctangctagctagcatgcatcgatcgatgctagctagtagc
+DATA
+
+((ntest++))
+if $CMD --max-cpu 2 -O -t 0.95 "${TMPDIR}/data.fasta" > "${TMPDIR}/clusters.fasta" 2> "${TMPDIR}/clusters.log"
+then
+    log "$MCMD: clustering runs OK"
+    ((success++))
+else
+    log "$MCMD: clustering failed"
+    ((failed++))
+fi
+
+((ntest++))
+if [ "$(grep -c '^>' "${TMPDIR}/clusters.fasta")" = "5" ]
+then
+    log "$MCMD: the sequence with an ambiguous nucleotide is discarded OK"
+    ((success++))
+else
+    log "$MCMD: wrong number of sequences in the output"
+    ((failed++))
+fi
+
+((ntest++))
+if [ "$(grep '^>a[23] ' "${TMPDIR}/clusters.fasta" | grep -c 'cluster=a1;')" = "2" ] &&
+   [ "$(grep '^>b2 ' "${TMPDIR}/clusters.fasta" | grep -c 'cluster=b1;')" = "1" ] &&
+   grep '^>b1 ' "${TMPDIR}/clusters.fasta" | grep -q 'cluster_center=true;' &&
+   grep '^>a1 ' "${TMPDIR}/clusters.fasta" | grep -q 'cluster_weight=25;'
+then
+    log "$MCMD: sequences are attached to the right center OK"
+    ((success++))
+else
+    log "$MCMD: sequences are not attached to the right center"
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD --max-cpu 2 -t 0.95 --otu-map "${TMPDIR}/otu.csv" --no-sequence-output "${TMPDIR}/data.fasta" > "${TMPDIR}/none.fasta" 2> /dev/null &&
+   [ ! -s "${TMPDIR}/none.fasta" ] &&
+   [ "$(head -n 1 "${TMPDIR}/otu.csv")" = "id,cluster,cluster_center,cluster_score,count" ] &&
+   [ "$(sed -n 2,4p "${TMPDIR}/otu.csv" | cut -d, -f1,2 | tr '\n' ' ')" = "a1,a1 a3,a1 a2,a1 " ] &&
+   [ "$(grep -c ',true,' "${TMPDIR}/otu.csv")" = "2" ]
+then
+    log "$MCMD: CSV table of the clusters OK"
+    ((success++))
+else
+    log "$MCMD: CSV table of the clusters failed"
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD --max-cpu 1 -O -t 0.95 "${TMPDIR}/data.fasta" 2> /dev/null | grep '^>' | sed 's/ .*//' > "${TMPDIR}/order1.txt" &&
+   $CMD --max-cpu 8 -O -t 0.95 "${TMPDIR}/data.fasta" 2> /dev/null | grep '^>' | sed 's/ .*//' > "${TMPDIR}/order8.txt" &&
+   cmp -s "${TMPDIR}/order1.txt" "${TMPDIR}/order8.txt"
+then
+    log "$MCMD: the result does not depend on the number of threads OK"
+    ((success++))
+else
+    log "$MCMD: the result depends on the number of threads"
+    ((failed++))
+fi
+
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obiconvert/out_ecotag.fasta

@@ -0,0 +1,24 @@
+>HELIUM_000100422_612GNAAXX:7:118:3572:14633#0/1_sub[28..126] {"count":10172,"merged_sample":{"26a_F040644":10172},"obitag_bestid":0.9797979797979798,"obitag_bestmatch":"AY227529","obitag_match_count":1,"obitag_rank":"genus","obitag_similarity_method":"lcs","taxid":"taxon:9992 [Marmota]@genus"}
+ttagccctaaacataaacattcaataaacaagaatgttcgccagagtactactagcaaca
+gcctgaaactcaaaggacttggcggtgctttacatccct
+>HELIUM_000100422_612GNAAXX:7:99:9351:13090#0/1_sub[28..127] {"count":260,"merged_sample":{"29a_F260619":260},"obitag_bestid":0.9405940594059405,"obitag_bestmatch":"AF154263","obitag_match_count":9,"obitag_rank":"infraorder","obitag_similarity_method":"lcs","taxid":"taxon:35500 [Pecora]@infraorder"}
+ttagccctaaacacaaataattacacaaacaaaattgttcaccagagtactagcggcaac
+agcttaaaactcaaaggacttggcggtgctttataccctt
+>HELIUM_000100422_612GNAAXX:7:108:10111:9078#0/1_sub[28..127] {"count":7146,"merged_sample":{"13a_F730603":7146},"obitag_bestid":1,"obitag_bestmatch":"AB245427","obitag_match_count":1,"obitag_rank":"species","obitag_similarity_method":"lcs","taxid":"taxon:9860 [Cervus elaphus]@species"}
+ctagccttaaacacaaatagttatgcaaacaaaactattcgccagagtactaccggcaat
+agcttaaaactcaaaggacttggcggtgctttataccctt
+>HELIUM_000100422_612GNAAXX:7:38:14204:12725#0/1_sub[28..126] {"count":87,"merged_sample":{"26a_F040644":87},"obitag_bestid":0.9494949494949495,"obitag_bestmatch":"AY227530","obitag_match_count":2,"obitag_rank":"tribe","obitag_similarity_method":"lcs","taxid":"taxon:337730 [Marmotini]@tribe"}
+ttagccctaaacataaacattcaataaacaagaatgttcgccagaggactactagcaata
+gcttaaaactcaaaggacttggcggtgctttatatccct
+>HELIUM_000100422_612GNAAXX:7:30:9942:4495#0/1_sub[28..126] {"count":95,"merged_sample":{"26a_F040644":11,"29a_F260619":84},"obitag_bestid":0.9595959595959596,"obitag_bestmatch":"AC187326","obitag_match_count":1,"obitag_rank":"subspecies","obitag_similarity_method":"lcs","taxid":"taxon:9615 [Canis lupus familiaris]@subspecies"}
+ttagccctaaacataagctattccataacaaaataattcgccagagaactactagcaaca
+gattaaacctcaaaggacttggcagtgctttatacccct
+>HELIUM_000100422_612GNAAXX:7:51:16702:19393#0/1_sub[28..127] {"count":12004,"merged_sample":{"15a_F730814":7465,"29a_F260619":4539},"obitag_bestid":1,"obitag_bestmatch":"AJ885202","obitag_match_count":1,"obitag_rank":"species","obitag_similarity_method":"lcs","taxid":"taxon:9858 [Capreolus capreolus]@species"}
+ttagccctaaacacaagtaattaatataacaaaattattcgccagagtactaccggcaat
+agcttaaaactcaaaggacttggcggtgctttataccctt
+>HELIUM_000100422_612GNAAXX:7:84:14502:1617#0/1_sub[28..127] {"count":319,"merged_sample":{"29a_F260619":319},"obitag_bestid":1,"obitag_bestmatch":"AJ972683","obitag_match_count":1,"obitag_rank":"species","obitag_similarity_method":"lcs","taxid":"taxon:9858 [Capreolus capreolus]@species"}
+ttagccctaaacacaagtaattattataacaaaattattcgccagagtactaccggcaat
+agcttaaaactcaaaggacttggcggtgctttataccctt
+>HELIUM_000100422_612GNAAXX:7:50:10637:6527#0/1_sub[28..126] {"count":366,"merged_sample":{"13a_F730603":13,"15a_F730814":5,"26a_F040644":347,"29a_F260619":1},"obitag_bestid":1,"obitag_bestmatch":"AB048590","obitag_match_count":1,"obitag_rank":"genus","obitag_similarity_method":"lcs","taxid":"taxon:9611 [Canis]@genus"}
+ttagccctaaacatagataattttacaacaaaataattcgccagaggactactagcaata
+gcttaaaactcaaaggacttggcggtgctttatatccct

obitests/obitools/obiconvert/taxonomy.csv

@@ -0,0 +1,48 @@
+taxid,parent,taxonomic_rank,scientific_name
+taxon:1 [root]@no rank,taxon:1 [root]@no rank,no rank,root
+taxon:131567 [cellular organisms]@cellular root,taxon:1 [root]@no rank,cellular root,cellular organisms
+taxon:2759 [Eukaryota]@domain,taxon:131567 [cellular organisms]@cellular root,domain,Eukaryota
+taxon:33154 [Opisthokonta]@clade,taxon:2759 [Eukaryota]@domain,clade,Opisthokonta
+taxon:33208 [Metazoa]@kingdom,taxon:33154 [Opisthokonta]@clade,kingdom,Metazoa
+taxon:6072 [Eumetazoa]@clade,taxon:33208 [Metazoa]@kingdom,clade,Eumetazoa
+taxon:33213 [Bilateria]@clade,taxon:6072 [Eumetazoa]@clade,clade,Bilateria
+taxon:33511 [Deuterostomia]@clade,taxon:33213 [Bilateria]@clade,clade,Deuterostomia
+taxon:7711 [Chordata]@phylum,taxon:33511 [Deuterostomia]@clade,phylum,Chordata
+taxon:89593 [Craniata]@subphylum,taxon:7711 [Chordata]@phylum,subphylum,Craniata
+taxon:7742 [Vertebrata]@clade,taxon:89593 [Craniata]@subphylum,clade,Vertebrata
+taxon:7776 [Gnathostomata]@clade,taxon:7742 [Vertebrata]@clade,clade,Gnathostomata
+taxon:117570 [Teleostomi]@clade,taxon:7776 [Gnathostomata]@clade,clade,Teleostomi
+taxon:117571 [Euteleostomi]@clade,taxon:117570 [Teleostomi]@clade,clade,Euteleostomi
+taxon:8287 [Sarcopterygii]@superclass,taxon:117571 [Euteleostomi]@clade,superclass,Sarcopterygii
+taxon:1338369 [Dipnotetrapodomorpha]@clade,taxon:8287 [Sarcopterygii]@superclass,clade,Dipnotetrapodomorpha
+taxon:32523 [Tetrapoda]@clade,taxon:1338369 [Dipnotetrapodomorpha]@clade,clade,Tetrapoda
+taxon:32524 [Amniota]@clade,taxon:32523 [Tetrapoda]@clade,clade,Amniota
+taxon:40674 [Mammalia]@class,taxon:32524 [Amniota]@clade,class,Mammalia
+taxon:32525 [Theria]@clade,taxon:40674 [Mammalia]@class,clade,Theria
+taxon:9347 [Eutheria]@clade,taxon:32525 [Theria]@clade,clade,Eutheria
+taxon:1437010 [Boreoeutheria]@clade,taxon:9347 [Eutheria]@clade,clade,Boreoeutheria
+taxon:314146 [Euarchontoglires]@superorder,taxon:1437010 [Boreoeutheria]@clade,superorder,Euarchontoglires
+taxon:314145 [Laurasiatheria]@superorder,taxon:1437010 [Boreoeutheria]@clade,superorder,Laurasiatheria
+taxon:33554 [Carnivora]@order,taxon:314145 [Laurasiatheria]@superorder,order,Carnivora
+taxon:91561 [Artiodactyla]@order,taxon:314145 [Laurasiatheria]@superorder,order,Artiodactyla
+taxon:314147 [Glires]@clade,taxon:314146 [Euarchontoglires]@superorder,clade,Glires
+taxon:9845 [Ruminantia]@suborder,taxon:91561 [Artiodactyla]@order,suborder,Ruminantia
+taxon:35500 [Pecora]@infraorder,taxon:9845 [Ruminantia]@suborder,infraorder,Pecora
+taxon:9989 [Rodentia]@order,taxon:314147 [Glires]@clade,order,Rodentia
+taxon:379584 [Caniformia]@suborder,taxon:33554 [Carnivora]@order,suborder,Caniformia
+taxon:9608 [Canidae]@family,taxon:379584 [Caniformia]@suborder,family,Canidae
+taxon:9850 [Cervidae]@family,taxon:35500 [Pecora]@infraorder,family,Cervidae
+taxon:9881 [Odocoileinae]@subfamily,taxon:9850 [Cervidae]@family,subfamily,Odocoileinae
+taxon:33553 [Sciuromorpha]@suborder,taxon:9989 [Rodentia]@order,suborder,Sciuromorpha
+taxon:55153 [Sciuridae]@family,taxon:33553 [Sciuromorpha]@suborder,family,Sciuridae
+taxon:34878 [Cervinae]@subfamily,taxon:9850 [Cervidae]@family,subfamily,Cervinae
+taxon:9611 [Canis]@genus,taxon:9608 [Canidae]@family,genus,Canis
+taxon:9857 [Capreolus]@genus,taxon:9881 [Odocoileinae]@subfamily,genus,Capreolus
+taxon:9612 [Canis lupus]@species,taxon:9611 [Canis]@genus,species,Canis lupus
+taxon:337726 [Xerinae]@subfamily,taxon:55153 [Sciuridae]@family,subfamily,Xerinae
+taxon:9859 [Cervus]@genus,taxon:34878 [Cervinae]@subfamily,genus,Cervus
+taxon:337730 [Marmotini]@tribe,taxon:337726 [Xerinae]@subfamily,tribe,Marmotini
+taxon:9992 [Marmota]@genus,taxon:337730 [Marmotini]@tribe,genus,Marmota
+taxon:9860 [Cervus elaphus]@species,taxon:9859 [Cervus]@genus,species,Cervus elaphus
+taxon:9615 [Canis lupus familiaris]@subspecies,taxon:9612 [Canis lupus]@species,subspecies,Canis lupus familiaris
+taxon:9858 [Capreolus capreolus]@species,taxon:9857 [Capreolus]@genus,species,Capreolus capreolus

obitests/obitools/obiconvert/test.sh

@@ -44,7 +44,7 @@ cleanup() {
     rm -rf "$TMPDIR"  # Suppress the temporary directory
 
     if [ $failed -gt 0 ]; then
-       log "$TEST_NAME tests failed" 
+       log "$TEST_NAME tests failed"
         log
         log
        exit 1
@@ -60,10 +60,10 @@ log() {
     echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
 }
 
-log "Testing $TEST_NAME..." 
-log "Test directory is $TEST_DIR" 
-log "obitools directory is $OBITOOLS_DIR" 
-log "Temporary directory is $TMPDIR" 
+log "Testing $TEST_NAME..."
+log "Test directory is $TEST_DIR"
+log "obitools directory is $OBITOOLS_DIR"
+log "Temporary directory is $TMPDIR"
 log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
 
 ######################################################################
@@ -94,12 +94,12 @@ log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
 
 
 ((ntest++))
-if $CMD -h > "${TMPDIR}/help.txt" 2>&1 
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1
 then
-    log "$MCMD: printing help OK" 
+    log "$MCMD: printing help OK"
     ((success++))
 else
-    log "$MCMD: printing help failed" 
+    log "$MCMD: printing help failed"
     ((failed++))
 fi
 
@@ -108,15 +108,15 @@ fi
 if obiconvert -Z "${TEST_DIR}/gbpln1088.4Mb.fasta.gz" \
                  > "${TMPDIR}/xxx.fasta.gz" && \
    zdiff "${TEST_DIR}/gbpln1088.4Mb.fasta.gz" \
-                 "${TMPDIR}/xxx.fasta.gz" 
+                 "${TMPDIR}/xxx.fasta.gz"
 then
-    log "$MCMD: converting large fasta file to fasta OK" 
+    log "$MCMD: converting large fasta file to fasta OK"
     ((success++))
 else
-    log "$MCMD: converting large fasta file to fasta failed" 
+    log "$MCMD: converting large fasta file to fasta failed"
     ((failed++))
 fi
- 
+
 ((ntest++))
 if obiconvert -Z --fastq-output \
               "${TEST_DIR}/gbpln1088.4Mb.fasta.gz" \
@@ -125,15 +125,139 @@ if obiconvert -Z --fastq-output \
               "${TMPDIR}/xxx.fastq.gz" \
               > "${TMPDIR}/yyy.fasta.gz" && \
    zdiff "${TEST_DIR}/gbpln1088.4Mb.fasta.gz" \
-                 "${TMPDIR}/yyy.fasta.gz" 
+                 "${TMPDIR}/yyy.fasta.gz"
 then
-    log "$MCMD: converting large file between fasta and fastq OK" 
+    log "$MCMD: converting large file between fasta and fastq OK"
     ((success++))
 else
-    log "$MCMD: converting large file between fasta and fastq failed" 
+    log "$MCMD: converting large file between fasta and fastq failed"
     ((failed++))
 fi
 
+
+# ------------------------------------------------------------------
+# --raw-taxid tests (no taxonomy loaded)
+# ------------------------------------------------------------------
+
+# Running test
+((ntest++))
+if obiconvert --raw-taxid "${TEST_DIR}/out_ecotag.fasta" \
+              > "${TMPDIR}/raw_taxid.fasta" 2>/dev/null
+then
+    log "$MCMD --raw-taxid: running OK"
+    ((success++))
+else
+    log "$MCMD --raw-taxid: running failed"
+    ((failed++))
+fi
+
+# Taxids must be bare numbers — no full-format "taxon:ID [Name]@rank" strings
+((ntest++))
+if grep '"taxid"' "${TMPDIR}/raw_taxid.fasta" | grep -qv '"taxid":"[0-9][0-9]*"'
+then
+    log "$MCMD --raw-taxid: taxid format check failed (full-format taxid found)"
+    ((failed++))
+else
+    log "$MCMD --raw-taxid: taxid format OK (all taxids are bare numbers)"
+    ((success++))
+fi
+
+# --raw-taxid is idempotent: piping through a second obiconvert --raw-taxid must
+# produce bit-for-bit identical output.
+((ntest++))
+if obiconvert --raw-taxid "${TMPDIR}/raw_taxid.fasta" \
+              > "${TMPDIR}/raw_taxid2.fasta" 2>/dev/null
+then
+    log "$MCMD --raw-taxid piped: running OK"
+    ((success++))
+else
+    log "$MCMD --raw-taxid piped: running failed"
+    ((failed++))
+fi
+
+((ntest++))
+if diff "${TMPDIR}/raw_taxid.fasta" \
+        "${TMPDIR}/raw_taxid2.fasta" > /dev/null
+then
+    log "$MCMD --raw-taxid piped: idempotency OK"
+    ((success++))
+else
+    log "$MCMD --raw-taxid piped: idempotency failed (outputs differ)"
+    ((failed++))
+fi
+
+
+# ------------------------------------------------------------------
+# --taxonomy tests (full-format taxid, no --raw-taxid)
+# ------------------------------------------------------------------
+
+# Running test
+((ntest++))
+if obiconvert --taxonomy "${TEST_DIR}/taxonomy.csv" \
+              "${TEST_DIR}/out_ecotag.fasta" \
+              > "${TMPDIR}/taxo.fasta" 2>/dev/null
+then
+    log "$MCMD --taxonomy: running OK"
+    ((success++))
+else
+    log "$MCMD --taxonomy: running failed"
+    ((failed++))
+fi
+
+# Taxids must be in full "taxon:ID [Name]@rank" format
+((ntest++))
+if grep '"taxid"' "${TMPDIR}/taxo.fasta" | grep -q '"taxid":"taxon:[0-9]'
+then
+    log "$MCMD --taxonomy: taxid format OK (full-format taxids present)"
+    ((success++))
+else
+    log "$MCMD --taxonomy: taxid format check failed (no full-format taxid found)"
+    ((failed++))
+fi
+
+
+# ------------------------------------------------------------------
+# --raw-taxid --taxonomy tests
+# ------------------------------------------------------------------
+
+# Running test
+((ntest++))
+if obiconvert --raw-taxid --taxonomy "${TEST_DIR}/taxonomy.csv" \
+              "${TEST_DIR}/out_ecotag.fasta" \
+              > "${TMPDIR}/raw_taxid_taxo.fasta" 2>/dev/null
+then
+    log "$MCMD --raw-taxid --taxonomy: running OK"
+    ((success++))
+else
+    log "$MCMD --raw-taxid --taxonomy: running failed"
+    ((failed++))
+fi
+
+# Taxids must be bare numbers even when taxonomy is loaded
+((ntest++))
+if grep '"taxid"' "${TMPDIR}/raw_taxid_taxo.fasta" | grep -qv '"taxid":"[0-9][0-9]*"'
+then
+    log "$MCMD --raw-taxid --taxonomy: taxid format check failed (full-format taxid found)"
+    ((failed++))
+else
+    log "$MCMD --raw-taxid --taxonomy: taxid format OK (all taxids are bare numbers)"
+    ((success++))
+fi
+
+# --raw-taxid with or without taxonomy must yield identical taxid values
+((ntest++))
+if diff <(grep '"taxid"' "${TMPDIR}/raw_taxid.fasta" | grep -o '"taxid":"[^"]*"' | sort) \
+        <(grep '"taxid"' "${TMPDIR}/raw_taxid_taxo.fasta" | grep -o '"taxid":"[^"]*"' | sort) \
+        > /dev/null
+then
+    log "$MCMD --raw-taxid vs --raw-taxid --taxonomy: taxid values match OK"
+    ((success++))
+else
+    log "$MCMD --raw-taxid vs --raw-taxid --taxonomy: taxid values differ (unexpected)"
+    ((failed++))
+fi
+
+
 #########################################
 #
 # At the end of the tests

obitests/obitools/obiconvert/xxx

@@ -0,0 +1,24 @@
+>HELIUM_000100422_612GNAAXX:7:118:3572:14633#0/1_sub[28..126] {"count":10172,"merged_sample":{"26a_F040644":10172},"obitag_bestid":0.9797979797979798,"obitag_bestmatch":"AY227529","obitag_match_count":1,"obitag_rank":"genus","obitag_similarity_method":"lcs","taxid":"taxon:9992 [Marmota]@genus"}
+ttagccctaaacataaacattcaataaacaagaatgttcgccagagtactactagcaaca
+gcctgaaactcaaaggacttggcggtgctttacatccct
+>HELIUM_000100422_612GNAAXX:7:99:9351:13090#0/1_sub[28..127] {"count":260,"merged_sample":{"29a_F260619":260},"obitag_bestid":0.9405940594059405,"obitag_bestmatch":"AF154263","obitag_match_count":9,"obitag_rank":"infraorder","obitag_similarity_method":"lcs","taxid":"taxon:35500 [Pecora]@infraorder"}
+ttagccctaaacacaaataattacacaaacaaaattgttcaccagagtactagcggcaac
+agcttaaaactcaaaggacttggcggtgctttataccctt
+>HELIUM_000100422_612GNAAXX:7:108:10111:9078#0/1_sub[28..127] {"count":7146,"merged_sample":{"13a_F730603":7146},"obitag_bestid":1,"obitag_bestmatch":"AB245427","obitag_match_count":1,"obitag_rank":"species","obitag_similarity_method":"lcs","taxid":"taxon:9860 [Cervus elaphus]@species"}
+ctagccttaaacacaaatagttatgcaaacaaaactattcgccagagtactaccggcaat
+agcttaaaactcaaaggacttggcggtgctttataccctt
+>HELIUM_000100422_612GNAAXX:7:38:14204:12725#0/1_sub[28..126] {"count":87,"merged_sample":{"26a_F040644":87},"obitag_bestid":0.9494949494949495,"obitag_bestmatch":"AY227530","obitag_match_count":2,"obitag_rank":"tribe","obitag_similarity_method":"lcs","taxid":"taxon:337730 [Marmotini]@tribe"}
+ttagccctaaacataaacattcaataaacaagaatgttcgccagaggactactagcaata
+gcttaaaactcaaaggacttggcggtgctttatatccct
+>HELIUM_000100422_612GNAAXX:7:30:9942:4495#0/1_sub[28..126] {"count":95,"merged_sample":{"26a_F040644":11,"29a_F260619":84},"obitag_bestid":0.9595959595959596,"obitag_bestmatch":"AC187326","obitag_match_count":1,"obitag_rank":"subspecies","obitag_similarity_method":"lcs","taxid":"taxon:9615 [Canis lupus familiaris]@subspecies"}
+ttagccctaaacataagctattccataacaaaataattcgccagagaactactagcaaca
+gattaaacctcaaaggacttggcagtgctttatacccct
+>HELIUM_000100422_612GNAAXX:7:51:16702:19393#0/1_sub[28..127] {"count":12004,"merged_sample":{"15a_F730814":7465,"29a_F260619":4539},"obitag_bestid":1,"obitag_bestmatch":"AJ885202","obitag_match_count":1,"obitag_rank":"species","obitag_similarity_method":"lcs","taxid":"taxon:9858 [Capreolus capreolus]@species"}
+ttagccctaaacacaagtaattaatataacaaaattattcgccagagtactaccggcaat
+agcttaaaactcaaaggacttggcggtgctttataccctt
+>HELIUM_000100422_612GNAAXX:7:84:14502:1617#0/1_sub[28..127] {"count":319,"merged_sample":{"29a_F260619":319},"obitag_bestid":1,"obitag_bestmatch":"AJ972683","obitag_match_count":1,"obitag_rank":"species","obitag_similarity_method":"lcs","taxid":"taxon:9858 [Capreolus capreolus]@species"}
+ttagccctaaacacaagtaattattataacaaaattattcgccagagtactaccggcaat
+agcttaaaactcaaaggacttggcggtgctttataccctt
+>HELIUM_000100422_612GNAAXX:7:50:10637:6527#0/1_sub[28..126] {"count":366,"merged_sample":{"13a_F730603":13,"15a_F730814":5,"26a_F040644":347,"29a_F260619":1},"obitag_bestid":1,"obitag_bestmatch":"AB048590","obitag_match_count":1,"obitag_rank":"genus","obitag_similarity_method":"lcs","taxid":"taxon:9611 [Canis]@genus"}
+ttagccctaaacatagataattttacaacaaaataattcgccagaggactactagcaata
+gcttaaaactcaaaggacttggcggtgctttatatccct

obitests/obitools/obioligotag/test.sh

@@ -0,0 +1,205 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obioligotag
+CMD=obioligotag
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+#
+# Every result is checked on its properties (size, alphabet, constraints),
+# not against a reference file, because several equivalent families exist.
+#
+
+printf 'acgtac\nacgtat\nttttgg\nggcaac\ncaccat\nacgtac\n' > "${TMPDIR}/oligos.txt"
+printf '>a\nACGTAC\n>b\nACGTAT\n>c\nTTTTGG\n>d\nGGCAAC\n>e\nCACCAT\n' > "${TMPDIR}/oligos.fasta"
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1 
+then
+    log "$MCMD: printing help OK" 
+    ((success++))
+else
+    log "$MCMD: printing help failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD -s 5 -f 24 -d 3 -g 3 -p 2 > "${TMPDIR}/tags.fasta" 2> /dev/null \
+   && [ "$(grep -vc '>' "${TMPDIR}/tags.fasta")" -ge 24 ] \
+   && ! grep -v '>' "${TMPDIR}/tags.fasta" | grep -qE '(a{3}|c{3}|g{3}|t{3})'
+then
+    log "$MCMD: generating a family with homopolymer constraint OK" 
+    ((success++))
+else
+    log "$MCMD: generating a family with homopolymer constraint failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD -s 7 -f 60 -d 3 --alphabet ACT > "${TMPDIR}/noG.fasta" 2> /dev/null \
+   && [ "$(grep -vc '>' "${TMPDIR}/noG.fasta")" -ge 60 ] \
+   && ! grep -v '>' "${TMPDIR}/noG.fasta" | grep -q 'g'
+then
+    log "$MCMD: restricted alphabet OK" 
+    ((success++))
+else
+    log "$MCMD: restricted alphabet failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD -s 6 -f 20 -d 3 --distance-type lcs > "${TMPDIR}/lcs.fasta" 2> /dev/null \
+   && [ "$(grep -vc '>' "${TMPDIR}/lcs.fasta")" -ge 20 ]
+then
+    log "$MCMD: LCS distance OK" 
+    ((success++))
+else
+    log "$MCMD: LCS distance failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD -s 7 -f 40 -d 3 -r cnnnnnn > "${TMPDIR}/rejected.fasta" 2> /dev/null \
+   && [ "$(grep -vc '>' "${TMPDIR}/rejected.fasta")" -ge 40 ] \
+   && ! grep -v '>' "${TMPDIR}/rejected.fasta" | grep -q '^c'
+then
+    log "$MCMD: rejected pattern OK" 
+    ((success++))
+else
+    log "$MCMD: rejected pattern failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD -L "${TMPDIR}/oligos.txt" -f 3 -d 3 > "${TMPDIR}/from_text.fasta" 2> /dev/null \
+   && [ "$(grep -vc '>' "${TMPDIR}/from_text.fasta")" -ge 3 ]
+then
+    log "$MCMD: selecting oligonucleotides from a text file OK" 
+    ((success++))
+else
+    log "$MCMD: selecting oligonucleotides from a text file failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD -L "${TMPDIR}/oligos.fasta" -f 3 -d 3 > "${TMPDIR}/from_fasta.fasta" 2> /dev/null \
+   && [ "$(grep -vc '>' "${TMPDIR}/from_fasta.fasta")" -ge 3 ]
+then
+    log "$MCMD: selecting oligonucleotides from a fasta file OK" 
+    ((success++))
+else
+    log "$MCMD: selecting oligonucleotides from a fasta file failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if ! $CMD -s 5 -f 100 -d 3 > /dev/null 2>&1
+then
+    log "$MCMD: family size beyond the Hamming bound refused OK" 
+    ((success++))
+else
+    log "$MCMD: family size beyond the Hamming bound not refused" 
+    ((failed++))
+fi
+
+((ntest++))
+if ! $CMD -s 5 -L "${TMPDIR}/oligos.txt" -f 3 > /dev/null 2>&1
+then
+    log "$MCMD: --oligo-size with --oligo-list refused OK" 
+    ((success++))
+else
+    log "$MCMD: --oligo-size with --oligo-list not refused" 
+    ((failed++))
+fi
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

pkg/obialign/fastlcs_bench_test.go

@@ -0,0 +1,111 @@
+package obialign
+
+import (
+	"fmt"
+	"math/rand"
+	"testing"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+// randomDNA returns a random ACGT sequence of length n.
+func randomDNA(rng *rand.Rand, n int) []byte {
+	const nuc = "ACGT"
+	s := make([]byte, n)
+	for i := range s {
+		s[i] = nuc[rng.Intn(4)]
+	}
+	return s
+}
+
+// mutate returns a copy of s in which each position is, with probability
+// rate, substituted, deleted or followed by an inserted base (equiprobable).
+func mutate(rng *rand.Rand, s []byte, rate float64) []byte {
+	const nuc = "ACGT"
+	out := make([]byte, 0, len(s)+len(s)/10)
+	for _, c := range s {
+		if rng.Float64() >= rate {
+			out = append(out, c)
+			continue
+		}
+		switch rng.Intn(3) {
+		case 0:
+			out = append(out, nuc[rng.Intn(4)])
+		case 1: // deletion
+		case 2:
+			out = append(out, c, nuc[rng.Intn(4)])
+		}
+	}
+	return out
+}
+
+// benchPairs builds a fixed set of pairs. "similar" pairs differ by about
+// 2% of mutated positions, "random" pairs are unrelated sequences.
+func benchPairs(n, length int, similar bool) [][2]*obiseq.BioSequence {
+	rng := rand.New(rand.NewSource(42))
+	pairs := make([][2]*obiseq.BioSequence, n)
+	for i := range pairs {
+		a := randomDNA(rng, length)
+		var b []byte
+		if similar {
+			b = mutate(rng, a, 0.02)
+		} else {
+			b = randomDNA(rng, length)
+		}
+		pairs[i] = [2]*obiseq.BioSequence{
+			obiseq.NewBioSequence("a", a, ""),
+			obiseq.NewBioSequence("b", b, ""),
+		}
+	}
+	return pairs
+}
+
+// BenchmarkLegacyLCS measures the historical implementation (the one
+// FastLCSScore used before BandedLCS), kept as the "before" reference.
+func BenchmarkLegacyLCS(b *testing.B) {
+	for _, similar := range []bool{true, false} {
+		kind := "random"
+		if similar {
+			kind = "similar"
+		}
+		for _, length := range []int{100, 500, 2000} {
+			pairs := benchPairs(64, length, similar)
+			for _, maxErr := range []int{-1, 5, length / 20, length / 5} {
+				name := fmt.Sprintf("%s/len=%d/maxErr=%d", kind, length, maxErr)
+				b.Run(name, func(b *testing.B) {
+					var buffer []uint64
+					b.ReportAllocs()
+					b.ResetTimer()
+					for i := 0; i < b.N; i++ {
+						p := pairs[i%len(pairs)]
+						FastLCSEGFScoreByte(p[0].Sequence(), p[1].Sequence(), maxErr, false, &buffer)
+					}
+				})
+			}
+		}
+	}
+}
+
+// BenchmarkBandedLCS runs on the same pairs as BenchmarkLegacyLCS.
+func BenchmarkBandedLCS(b *testing.B) {
+	for _, similar := range []bool{true, false} {
+		kind := "random"
+		if similar {
+			kind = "similar"
+		}
+		for _, length := range []int{100, 500, 2000} {
+			pairs := benchPairs(64, length, similar)
+			for _, maxErr := range []int{-1, 5, length / 20, length / 5} {
+				name := fmt.Sprintf("%s/len=%d/maxErr=%d", kind, length, maxErr)
+				b.Run(name, func(b *testing.B) {
+					b.ReportAllocs()
+					b.ResetTimer()
+					for i := 0; i < b.N; i++ {
+						p := pairs[i%len(pairs)]
+						BandedLCS(p[0].Sequence(), p[1].Sequence(), maxErr)
+					}
+				})
+			}
+		}
+	}
+}

pkg/obialign/fastlcsegf.go

@@ -2,6 +2,7 @@ package obialign
 
 import (
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+	log "github.com/sirupsen/logrus"
 )
 
 // FastLCSEGFScoreByte calculates the score of the Longest Common Subsequence (LCS) between two byte slices.
@@ -355,27 +356,35 @@ func FastLCSEGFScore(seqA, seqB *obiseq.BioSequence, maxError int, buffer *[]uin
 	return FastLCSEGFScoreByte(seqA.Sequence(), seqB.Sequence(), maxError, true, buffer)
 }
 
-// FastLCSScore calculates the score of the longest common subsequence between two bio sequences.
+// FastLCSScore calculates the score of the longest common subsequence between two bio sequences,
+// and the length of the shortest alignment achieving it (the distance is alilen - lcs).
 //
-// if maxError > 0, the maximum allowed error between the sequences is maxError.
-// Otherwise, no error checking is done.
-// If the actual number of errors is larger than maxError, -1 is returned for both values.
+// If maxError >= 0, the result is exact when its distance is at most maxError,
+// and (-1, -1) is returned otherwise. If maxError < 0, no limit is applied.
+// See BandedLCS, which does the work.
 //
 // The score matrix is:
-//   - Matching: 1
+//   - Matching: 1 (bases are compared with the IUPAC rules)
 //   - Mismatch or gap: 0
 //
+// The program stops with a fatal error if a sequence is longer than
+// MaxBandedLCSLength.
+//
 // Parameters:
 // - seqA: The first bio sequence.
 // - seqB: The second bio sequence.
 // - maxError: The maximum allowed error between the sequences. If set to -1, no limit is applied.
-// - buffer: A pointer to a uint64 slice to store intermediate results. If nil, a new slice is created.
+// - buffer: Unused, kept for compatibility; the working memory is pooled.
 //
 // Returns:
 // - The score of the longest common subsequence.
 // - The length of the shortest alignment corresponding to the LCS.
 func FastLCSScore(seqA, seqB *obiseq.BioSequence, maxError int, buffer *[]uint64) (int, int) {
-	score, alilen, _ := FastLCSEGFScoreByte(seqA.Sequence(), seqB.Sequence(), maxError, false, buffer)
+	score, alilen, err := BandedLCS(seqA.Sequence(), seqB.Sequence(), maxError)
+	if err != nil {
+		log.Fatalf("FastLCSScore on sequences %s (%d bp) and %s (%d bp): %v (maximum %d bp)",
+			seqA.Id(), seqA.Len(), seqB.Id(), seqB.Len(), err, MaxBandedLCSLength)
+	}
 
 	return score, alilen
 }

pkg/obialign/lcsband.go

@@ -0,0 +1,185 @@
+package obialign
+
+import (
+	"errors"
+	"sync"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+// MaxBandedLCSLength is the longest sequence BandedLCS accepts. The score and
+// the alignment length are packed on 16 bits each, and the alignment length
+// can reach the sum of both sequence lengths.
+const MaxBandedLCSLength = 1<<15 - 1
+
+// ErrSequenceTooLong is returned by BandedLCS when a sequence is longer than
+// MaxBandedLCSLength.
+var ErrSequenceTooLong = errors.New("obialign: sequence too long for BandedLCS")
+
+// A DP cell is packed on 32 bits as
+//
+//	score<<16 | (lcsLenBase - alignment length)
+//
+// so that the maximum of two cells ranks them by score first and, for equal
+// scores, by shortest alignment. The three moves then reduce to
+//
+//	cell(i,j) = max(diag + match<<16, up, left) - 1
+//
+// since every move lengthens the alignment by one and only a diagonal match
+// adds one to the score. The value 0 stands for an unavailable cell (outside
+// the band): it loses against any real cell. Every cell of the band has at
+// least one real neighbour (the band is connected and contains (0,0)), so
+// the subtraction never wraps for a cell that is used; the scalar reference
+// still clamps the maximum to 1 as a safety net.
+const (
+	lcsScoreUnit = uint32(1) << 16
+	lcsLenBase   = uint32(0xFFFF)
+)
+
+// bandedLCSWorkspace holds the scratch memory of one alignment.
+type bandedLCSWorkspace struct {
+	aRev, bCode []uint32    // encoded bases (aRev is reversed), SIMD kernel only
+	rows        [3][]uint32 // antidiagonal (SIMD) or row (scalar) buffers
+	aRev16      []uint16    // 16-bit encoded bases of the score-only kernel
+	bCode16     []uint16
+	rows16      [3][]uint16
+	widened     int // number of alignments that needed a second pass
+}
+
+var bandedLCSPool = sync.Pool{New: func() any { return new(bandedLCSWorkspace) }}
+
+func growUint32(s []uint32, n int) []uint32 {
+	if cap(s) < n {
+		return make([]uint32, n)
+	}
+	return s[:n]
+}
+
+// iupacMask gives for each byte its 4-bit nucleotide set (A=1, C=2, G=4,
+// T=8), or 0 when the byte is not an IUPAC letter. For two bytes of non-zero
+// mask, SameIUPACNuc(a, b) is exactly (mask[a] & mask[b]) != 0.
+var iupacMask = buildIUPACMask()
+
+func buildIUPACMask() (m [256]uint8) {
+	probes := [4]byte{'a', 'c', 'g', 't'}
+	for c := 0; c < 256; c++ {
+		if !(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z') {
+			continue
+		}
+		for bit, p := range probes {
+			if obiseq.SameIUPACNuc(byte(c), p) {
+				m[c] |= 1 << uint(bit)
+			}
+		}
+	}
+	return m
+}
+
+// allRegularIUPAC reports whether every base has a non-zero IUPAC mask, in
+// which case base compatibility is a plain AND of masks.
+func allRegularIUPAC(s []byte) bool {
+	for _, c := range s {
+		if iupacMask[c] == 0 {
+			return false
+		}
+	}
+	return true
+}
+
+// bandOffsets returns the diagonal offsets k = j - i covered by a band of
+// half-width g around the diagonals 0..delta, with sequence A (columns j) not
+// shorter than sequence B (rows i), clipped to the matrix.
+func bandOffsets(lA, lB, g int) (kmin, kmax int) {
+	return max(-g, -lB), min(lA-lB+g, lA)
+}
+
+// BandedLCS computes the score of the longest common subsequence of two
+// nucleotide sequences, and the length of the shortest alignment achieving
+// it (match: +1, mismatch and gap: 0; bases are compared with the IUPAC
+// rules of obiseq.SameIUPACNuc). The distance is alilen - lcs.
+//
+// If maxError >= 0, the result is the optimal (lcs, alilen) when its distance
+// is at most maxError, and (-1, -1) otherwise. This is exact, not a
+// heuristic: the DP is restricted to a band, and a bound on what a path
+// leaving the band can score proves when the band was wide enough. maxError < 0
+// means no limit.
+//
+// An error is returned when a sequence is longer than MaxBandedLCSLength.
+func BandedLCS(seqA, seqB []byte, maxError int) (lcs, alilen int, err error) {
+	if len(seqA) > MaxBandedLCSLength || len(seqB) > MaxBandedLCSLength {
+		return -1, -1, ErrSequenceTooLong
+	}
+	if len(seqA) < len(seqB) {
+		seqA, seqB = seqB, seqA
+	}
+	ws := bandedLCSPool.Get().(*bandedLCSWorkspace)
+	lcs, alilen = bandedLCS(seqA, seqB, maxError, ws, haveSIMD)
+	bandedLCSPool.Put(ws)
+	return lcs, alilen, nil
+}
+
+// bandedLCS expects len(a) >= len(b), both within MaxBandedLCSLength.
+//
+// With delta = lA - lB and a band of half-width g (offsets -g .. delta+g):
+//   - an alignment with at most E errors needs at least delta+2h gaps to reach
+//     the offset delta+h or -h, so it lies in the band of g0 = (E-delta)/2;
+//   - a path leaving a band of half-width g makes at least g+1 gaps on the B
+//     side, hence scores at most lB-g-1 matches, and has more than E errors
+//     as soon as g >= g0.
+//
+// Let Q be the best path of the band g0. If its distance exceeds E, the
+// global optimum cannot be within E either (it would be in the band and be
+// Q): the answer is -1. Otherwise, if score(Q) >= lB-g0-1 no outside path can
+// beat Q (a tie in score loses on length, the outside path having more errors
+// than Q): Q is the global optimum. Else the band is widened once to
+// g1 = min(E-delta, lB-score(Q)-1); since score(Q) >= lA-E for any alignment
+// within E errors, g1 always certifies the result.
+func bandedLCS(a, b []byte, maxError int, ws *bandedLCSWorkspace, useSIMD bool) (int, int) {
+	lA, lB := len(a), len(b)
+	delta := lA - lB
+	if maxError >= 0 && maxError < delta {
+		return -1, -1
+	}
+	if lB == 0 {
+		return 0, lA
+	}
+	simdOK := useSIMD && allRegularIUPAC(a) && allRegularIUPAC(b)
+	if simdOK {
+		encodeBases(ws, a, b)
+	}
+	run := func(g int) (int, int) {
+		kmin, kmax := bandOffsets(lA, lB, g)
+		return ws.align(a, b, kmin, kmax, simdOK)
+	}
+
+	if maxError < 0 {
+		return run(lA)
+	}
+	g0 := (maxError - delta) / 2
+	lcs, alilen := run(g0)
+	if alilen-lcs > maxError {
+		return -1, -1
+	}
+	if lcs >= lB-g0-1 {
+		return lcs, alilen
+	}
+	ws.widened++
+	lcs, alilen = run(min(maxError-delta, lB-lcs-1))
+	if alilen-lcs > maxError {
+		return -1, -1
+	}
+	return lcs, alilen
+}
+
+// align runs the banded DP on the offsets [kmin, kmax] (kmin <= 0 <= kmax) and
+// returns the best (lcs, alilen) of the band. The bases must have been encoded
+// with encodeBases when simdOK is set.
+func (ws *bandedLCSWorkspace) align(a, b []byte, kmin, kmax int, simdOK bool) (int, int) {
+	var key uint32
+	if simdOK {
+		key = bandedLCSSIMD(len(a), len(b), kmin, kmax, ws)
+	} else {
+		key = bandedLCSScalar(a, b, kmin, kmax, ws)
+	}
+	return int(key >> 16), int(lcsLenBase - key&0xFFFF)
+}

pkg/obialign/lcsband_nosimd.go

@@ -0,0 +1,17 @@
+//go:build !goexperiment.simd
+
+package obialign
+
+const haveSIMD = false
+
+// encodeBases and bandedLCSSIMD are never called without GOEXPERIMENT=simd
+// (haveSIMD is false); they exist to keep the dispatcher uniform.
+func encodeBases(ws *bandedLCSWorkspace, a, b []byte) {}
+
+func bandedLCSSIMD(lA, lB, kmin, kmax int, ws *bandedLCSWorkspace) uint32 {
+	return 0
+}
+
+func lcsScoreSIMD(a, b []byte, kmin, kmax, minLCS int, ws *bandedLCSWorkspace) int {
+	return -1
+}

pkg/obialign/lcsband_scalar.go

@@ -0,0 +1,53 @@
+package obialign
+
+import "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+
+// bandedLCSScalar is the portable reference of the banded DP, row by row. It
+// returns the packed cell (lB, lA). len(a) >= len(b) >= 1.
+//
+// Cells outside the band hold 0 (unavailable). Row i spans the columns
+// [i+kmin, i+kmax] clipped to [1, lA]; the cell just left of that span and
+// the one just right of it are reset so that the next row never reads a stale
+// value.
+func bandedLCSScalar(a, b []byte, kmin, kmax int, ws *bandedLCSWorkspace) uint32 {
+	lA, lB := len(a), len(b)
+
+	prev := growUint32(ws.rows[0], lA+2)
+	cur := growUint32(ws.rows[1], lA+2)
+	ws.rows[0], ws.rows[1] = prev, cur
+
+	for j := 0; j <= lA; j++ {
+		prev[j] = 0
+		if j <= kmax {
+			prev[j] = lcsLenBase - uint32(j)
+		}
+	}
+	prev[lA+1] = 0
+
+	for i := 1; i <= lB; i++ {
+		if -i >= kmin {
+			cur[0] = lcsLenBase - uint32(i)
+		} else {
+			cur[0] = 0
+		}
+		jlo := max(1, i+kmin)
+		jhi := min(lA, i+kmax)
+		if jlo > 1 {
+			cur[jlo-1] = 0
+		}
+		bi := b[i-1]
+		for j := jlo; j <= jhi; j++ {
+			d := prev[j-1]
+			if obiseq.SameIUPACNuc(a[j-1], bi) {
+				d += lcsScoreUnit
+			}
+			m := max(d, prev[j], cur[j-1], 1)
+			cur[j] = m - 1
+		}
+		if jhi < lA {
+			cur[jhi+1] = 0
+		}
+		prev, cur = cur, prev
+	}
+	return prev[lA]
+}

pkg/obialign/lcsband_simd.go

@@ -0,0 +1,107 @@
+//go:build goexperiment.simd
+
+package obialign
+
+import "simd"
+
+const haveSIMD = true
+
+func simdLanes() int {
+	var probe simd.Uint32s
+	return probe.Len()
+}
+
+// encodeIUPAC writes in dst the 4-bit IUPAC mask of each base of s, reversed
+// if rev is set, followed by zero padding up to len(dst).
+func encodeIUPAC(dst []uint32, s []byte, rev bool) {
+	n := len(s)
+	for x := range dst {
+		dst[x] = 0
+	}
+	if rev {
+		for x, c := range s {
+			dst[n-1-x] = uint32(iupacMask[c])
+		}
+		return
+	}
+	for x, c := range s {
+		dst[x] = uint32(iupacMask[c])
+	}
+}
+
+// encodeBases encodes both sequences once for all the passes of an alignment.
+func encodeBases(ws *bandedLCSWorkspace, a, b []byte) {
+	vl := simdLanes()
+	ws.aRev = growUint32(ws.aRev, len(a)+vl)
+	ws.bCode = growUint32(ws.bCode, len(b)+vl)
+	encodeIUPAC(ws.aRev, a, true)
+	encodeIUPAC(ws.bCode, b, false)
+}
+
+// bandedLCSSIMD runs the banded DP along antidiagonals t = i + j. Lane i of
+// the buffer of antidiagonal t holds the cell (i, t-i), so that
+//
+//	diag = buffer[t-2][i-1]   up = buffer[t-1][i-1]   left = buffer[t-1][i]
+//
+// are plain unaligned loads. Along an antidiagonal the offset j-i = t-2i
+// must stay in [kmin, kmax], which gives the window [lo, hi] of rows. The
+// vector loop may run past hi; the lanes beyond are then reset to 0
+// (unavailable), as is the lane just before the window, because the next
+// antidiagonals read them as neighbours. Lane 0 and lane t carry the first
+// row and the first column of the matrix.
+//
+// Requires encodeBases to have been called, lA >= lB >= 1 and kmin <= 0 <=
+// kmax.
+func bandedLCSSIMD(lA, lB, kmin, kmax int, ws *bandedLCSWorkspace) uint32 {
+	vl := simdLanes()
+	aRev, bCode := ws.aRev, ws.bCode
+
+	size := lB + 2*vl + 2
+	for r := range ws.rows {
+		ws.rows[r] = growUint32(ws.rows[r], size)
+		clear(ws.rows[r])
+	}
+	p2, p1, cur := ws.rows[0], ws.rows[1], ws.rows[2]
+
+	p2[0] = lcsLenBase // cell (0,0)
+	if kmax >= 1 {
+		p1[0] = lcsLenBase - 1 // cell (0,1)
+	}
+	if kmin <= -1 {
+		p1[1] = lcsLenBase - 1 // cell (1,0)
+	}
+
+	zero := simd.BroadcastUint32s(0)
+	one := simd.BroadcastUint32s(1)
+	unit := simd.BroadcastUint32s(lcsScoreUnit)
+
+	for t := 2; t <= lA+lB; t++ {
+		lo := max(1, t-lA, (t-kmax+1)>>1)
+		hi := min(lB, t-1, (t-kmin)>>1)
+
+		for i := lo; i <= hi; i += vl {
+			ca := simd.LoadUint32s(aRev[lA-t+i:])
+			cb := simd.LoadUint32s(bCode[i-1:])
+			inc := unit.Masked(ca.And(cb).NotEqual(zero))
+			d := simd.LoadUint32s(p2[i-1:]).Add(inc)
+			u := simd.LoadUint32s(p1[i-1:])
+			l := simd.LoadUint32s(p1[i:])
+			d.Max(u).Max(l).Sub(one).Store(cur[i:])
+		}
+
+		zero.Store(cur[hi+1:])
+		if lo > 1 {
+			cur[lo-1] = 0
+		}
+		cur[0] = 0
+		if t <= lA && t <= kmax {
+			cur[0] = lcsLenBase - uint32(t)
+		}
+		if t <= lB && -t >= kmin {
+			cur[t] = lcsLenBase - uint32(t)
+		}
+
+		p2, p1, cur = p1, cur, p2
+	}
+	return p1[lB]
+}

pkg/obialign/lcsband_test.go

@@ -0,0 +1,360 @@
+package obialign
+
+import (
+	"math/rand"
+	"testing"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+// bandDP is the oracle: a plain DP over the whole matrix, written
+// independently of the production code, in which only the cells with a
+// diagonal offset j-i in [kmin, kmax] exist (kmin=-lB, kmax=lA when
+// unbounded). It returns the optimal LCS score and the shortest alignment
+// length achieving it among the alignments staying in the band.
+func bandDP(a, b []byte, kmin, kmax int) (int, int) {
+	lA, lB := len(a), len(b)
+	type cell struct {
+		s, l int
+		ok   bool
+	}
+	better := func(x, y cell) bool {
+		return !y.ok || (x.ok && (x.s > y.s || (x.s == y.s && x.l < y.l)))
+	}
+	m := make([][]cell, lB+1)
+	for i := range m {
+		m[i] = make([]cell, lA+1)
+	}
+	for i := 0; i <= lB; i++ {
+		for j := 0; j <= lA; j++ {
+			if k := j - i; k < kmin || k > kmax {
+				continue
+			}
+			if i == 0 || j == 0 {
+				m[i][j] = cell{0, i + j, true}
+				continue
+			}
+			var best cell
+			if d := m[i-1][j-1]; d.ok {
+				best = cell{d.s, d.l + 1, true}
+				if obiseq.SameIUPACNuc(a[j-1], b[i-1]) {
+					best.s++
+				}
+			}
+			if u := m[i-1][j]; u.ok && better(cell{u.s, u.l + 1, true}, best) {
+				best = cell{u.s, u.l + 1, true}
+			}
+			if l := m[i][j-1]; l.ok && better(cell{l.s, l.l + 1, true}, best) {
+				best = cell{l.s, l.l + 1, true}
+			}
+			m[i][j] = best
+		}
+	}
+	return m[lB][lA].s, m[lB][lA].l
+}
+
+// fullDP is the unbanded optimum.
+func fullDP(a, b []byte) (int, int) { return bandDP(a, b, -len(b), len(a)) }
+
+// expected is the contract of BandedLCS: the unbounded optimum when it has at
+// most maxError errors, (-1,-1) otherwise.
+func expected(a, b []byte, maxError int) (int, int) {
+	s, l := fullDP(a, b)
+	if maxError >= 0 && l-s > maxError {
+		return -1, -1
+	}
+	return s, l
+}
+
+func randomFrom(rng *rand.Rand, alphabet string, n int) []byte {
+	s := make([]byte, n)
+	for i := range s {
+		s[i] = alphabet[rng.Intn(len(alphabet))]
+	}
+	return s
+}
+
+func mutateFrom(rng *rand.Rand, alphabet string, s []byte, rate float64) []byte {
+	out := make([]byte, 0, len(s)+8)
+	for _, c := range s {
+		if rng.Float64() >= rate {
+			out = append(out, c)
+			continue
+		}
+		switch rng.Intn(3) {
+		case 0:
+			out = append(out, alphabet[rng.Intn(len(alphabet))])
+		case 1:
+		case 2:
+			out = append(out, c, alphabet[rng.Intn(len(alphabet))])
+		}
+	}
+	return out
+}
+
+var testAlphabets = []struct {
+	name, letters string
+	regular       bool // only IUPAC letters of non-zero mask
+}{
+	{"acgt", "ACGT", true},
+	{"low-complexity", "AC", true},
+	{"mixed-case", "ACGTacgt", true},
+	{"iupac", "ACGTRYSWKMBDHVNUacgtrywsn", true},
+	{"exotic", "ACGT-.EXz*", false},
+}
+
+// TestBandedLCSDifferential checks the scalar reference against the oracle,
+// and the dispatcher (SIMD kernel when built with GOEXPERIMENT=simd) against
+// the scalar reference, for random pairs of unequal lengths and many bands.
+func TestBandedLCSDifferential(t *testing.T) {
+	rng := rand.New(rand.NewSource(7))
+	ws := new(bandedLCSWorkspace)
+	for _, alpha := range testAlphabets {
+		for it := 0; it < 3000; it++ {
+			lA := 1 + rng.Intn(90)
+			a := randomFrom(rng, alpha.letters, lA)
+			var b []byte
+			if rng.Intn(3) > 0 {
+				b = mutateFrom(rng, alpha.letters, a, 0.02+rng.Float64()*0.5)
+			} else {
+				b = randomFrom(rng, alpha.letters, 1+rng.Intn(90))
+			}
+			if len(b) == 0 {
+				continue
+			}
+			if len(a) < len(b) {
+				a, b = b, a
+			}
+			delta := len(a) - len(b)
+			for _, maxError := range []int{-1, 0, 1, 2, 3, 5, 8, 13, 21, 40, 200, delta, delta + 1, delta + 2, delta + 3} {
+				ws, wl := expected(a, b, maxError)
+				ss, sl := bandedLCS(a, b, maxError, new(bandedLCSWorkspace), false)
+				if ss != ws || sl != wl {
+					t.Fatalf("%s scalar maxError=%d a=%q b=%q: got (%d,%d), oracle (%d,%d)",
+						alpha.name, maxError, a, b, ss, sl, ws, wl)
+				}
+				ds, dl := bandedLCS(a, b, maxError, new(bandedLCSWorkspace), haveSIMD)
+				if ds != ss || dl != sl {
+					t.Fatalf("%s dispatch(simd=%v) maxError=%d a=%q b=%q: got (%d,%d), scalar (%d,%d)",
+						alpha.name, haveSIMD, maxError, a, b, ds, dl, ss, sl)
+				}
+			}
+		}
+	}
+	_ = ws
+}
+
+// TestBandedLCSLongPairs covers lengths well above the vector width, with
+// reused workspaces (stale buffers must never leak between calls).
+func TestBandedLCSLongPairs(t *testing.T) {
+	rng := rand.New(rand.NewSource(11))
+	ws := new(bandedLCSWorkspace)
+	for it := 0; it < 60; it++ {
+		a := randomFrom(rng, "ACGT", 150+rng.Intn(500))
+		b := mutateFrom(rng, "ACGT", a, 0.01+rng.Float64()*0.2)
+		if len(a) < len(b) {
+			a, b = b, a
+		}
+		for _, maxError := range []int{-1, 4, 20, 60, 150} {
+			wantS, wantL := expected(a, b, maxError)
+			gotS, gotL := bandedLCS(a, b, maxError, ws, haveSIMD)
+			if gotS != wantS || gotL != wantL {
+				t.Fatalf("maxError=%d lens=%d/%d: got (%d,%d), oracle (%d,%d)",
+					maxError, len(a), len(b), gotS, gotL, wantS, wantL)
+			}
+		}
+	}
+}
+
+// TestBandedLCSMatchesLegacy compares with the historical implementation
+// (FastLCSEGFScoreByte, not end-gap-free) wherever that one is exact: no
+// limit, or a result within the limit.
+func TestBandedLCSMatchesLegacy(t *testing.T) {
+	rng := rand.New(rand.NewSource(3))
+	for it := 0; it < 4000; it++ {
+		a := randomFrom(rng, "ACGTRYN", 1+rng.Intn(120))
+		b := mutateFrom(rng, "ACGTRYN", a, rng.Float64()*0.4)
+		if len(b) == 0 {
+			continue
+		}
+		for _, maxError := range []int{-1, 2, 7, 25} {
+			ls, ll, err := BandedLCS(a, b, maxError)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if ls < 0 {
+				continue
+			}
+			os, ol, _ := FastLCSEGFScoreByte(a, b, maxError, false, nil)
+			if os != ls || ol != ll {
+				t.Fatalf("maxError=%d a=%q b=%q: BandedLCS (%d,%d), legacy (%d,%d)",
+					maxError, a, b, ls, ll, os, ol)
+			}
+		}
+	}
+}
+
+func TestBandedLCSEdgeCases(t *testing.T) {
+	cases := []struct {
+		a, b     string
+		maxError int
+		s, l     int
+	}{
+		{"", "", -1, 0, 0},
+		{"ACGT", "", -1, 0, 4},
+		{"ACGT", "", 3, -1, -1},
+		{"ACGT", "", 4, 0, 4},
+		{"ACGT", "ACGT", 0, 4, 4},
+		{"ACGT", "ACGA", 0, -1, -1},
+		{"ACGT", "ACGA", 1, 3, 4},
+		{"ACGTT", "ACGT", 0, -1, -1}, // length difference above the limit
+		{"ACGTT", "ACGT", 1, 4, 5},
+	}
+	for _, c := range cases {
+		s, l, err := BandedLCS([]byte(c.a), []byte(c.b), c.maxError)
+		if err != nil || s != c.s || l != c.l {
+			t.Errorf("%q/%q maxError=%d: got (%d,%d,%v), want (%d,%d)", c.a, c.b, c.maxError, s, l, err, c.s, c.l)
+		}
+		// the arguments are symmetric
+		s, l, _ = BandedLCS([]byte(c.b), []byte(c.a), c.maxError)
+		if s != c.s || l != c.l {
+			t.Errorf("swapped %q/%q maxError=%d: got (%d,%d), want (%d,%d)", c.b, c.a, c.maxError, s, l, c.s, c.l)
+		}
+	}
+}
+
+func TestBandedLCSTooLong(t *testing.T) {
+	long := make([]byte, MaxBandedLCSLength+1)
+	for i := range long {
+		long[i] = 'A'
+	}
+	if _, _, err := BandedLCS(long, []byte("ACGT"), -1); err != ErrSequenceTooLong {
+		t.Errorf("A too long: err = %v", err)
+	}
+	if _, _, err := BandedLCS([]byte("ACGT"), long, 5); err != ErrSequenceTooLong {
+		t.Errorf("B too long: err = %v", err)
+	}
+	// the maximum length itself is accepted
+	ok := long[:MaxBandedLCSLength]
+	if s, l, err := BandedLCS(ok, ok, 0); err != nil || s != len(ok) || l != len(ok) {
+		t.Errorf("max length: got (%d,%d,%v)", s, l, err)
+	}
+}
+
+// TestBandedLCSSecondPass checks the exactness over many bands, also on
+// low-complexity sequences where the best LCS is not the closest alignment,
+// and reports how often the band had to be widened (never more than once).
+func TestBandedLCSSecondPass(t *testing.T) {
+	rng := rand.New(rand.NewSource(5))
+	for _, alpha := range testAlphabets[:4] {
+		for _, rate := range []float64{0.02, 0.1, 0.3} {
+			ws := new(bandedLCSWorkspace)
+			var total int
+			for it := 0; it < 1500; it++ {
+				a := randomFrom(rng, alpha.letters, 20+rng.Intn(120))
+				b := mutateFrom(rng, alpha.letters, a, rate)
+				if len(b) == 0 {
+					continue
+				}
+				if len(a) < len(b) {
+					a, b = b, a
+				}
+				for _, maxError := range []int{1, 3, 6, 10, 20, 40} {
+					want, wantL := expected(a, b, maxError)
+					got, gotL := bandedLCS(a, b, maxError, ws, haveSIMD)
+					if got != want || gotL != wantL {
+						t.Fatalf("%s maxError=%d a=%q b=%q: got (%d,%d), want (%d,%d)",
+							alpha.name, maxError, a, b, got, gotL, want, wantL)
+					}
+					total++
+				}
+			}
+			t.Logf("%-15s mutation %.2f: %d bounded calls, %d widened (%.1f%%)",
+				alpha.name, rate, total, ws.widened, 100*float64(ws.widened)/float64(total))
+		}
+	}
+}
+
+// TestFastLCSScoreWrapper checks the BioSequence entry point.
+func TestFastLCSScoreWrapper(t *testing.T) {
+	sa := obiseq.NewBioSequence("a", []byte("ACGTACGTAC"), "")
+	sb := obiseq.NewBioSequence("b", []byte("ACGTTCGTAC"), "")
+	if s, l := FastLCSScore(sa, sb, -1, nil); s != 9 || l != 10 {
+		t.Errorf("got (%d,%d), want (9,10)", s, l)
+	}
+	if s, l := FastLCSScore(sa, sb, 0, nil); s != -1 || l != -1 {
+		t.Errorf("got (%d,%d), want (-1,-1)", s, l)
+	}
+}
+
+// TestBandedLCSAtLeast checks the exactness of the bound expressed on the
+// score: the optimum when it reaches minLCS, (-1,-1) otherwise.
+func TestBandedLCSAtLeast(t *testing.T) {
+	rng := rand.New(rand.NewSource(13))
+	var buf BandedLCSBuffer
+	for _, alpha := range testAlphabets {
+		for it := 0; it < 1500; it++ {
+			a := randomFrom(rng, alpha.letters, 1+rng.Intn(100))
+			b := mutateFrom(rng, alpha.letters, a, rng.Float64()*0.5)
+			if len(b) == 0 {
+				continue
+			}
+			os, ol := fullDP(a, b)
+			for _, minLCS := range []int{-3, 0, 1, os - 1, os, os + 1, len(b) / 2, len(b), len(b) + 1} {
+				ws, wl := os, ol
+				if os < minLCS {
+					ws, wl = -1, -1
+				}
+				for _, got := range [][2]int{func() [2]int { s, l, _ := buf.AtLeast(a, b, minLCS); return [2]int{s, l} }(),
+					func() [2]int { s, l, _ := BandedLCSAtLeast(b, a, minLCS); return [2]int{s, l} }()} {
+					if got[0] != ws || got[1] != wl {
+						t.Fatalf("%s minLCS=%d a=%q b=%q: got (%d,%d), want (%d,%d)",
+							alpha.name, minLCS, a, b, got[0], got[1], ws, wl)
+					}
+				}
+			}
+		}
+	}
+	if _, _, err := buf.AtLeast(make([]byte, MaxBandedLCSLength+1), []byte("a"), 0); err != ErrSequenceTooLong {
+		t.Errorf("too long: err = %v", err)
+	}
+}
+
+// TestBandedLCSScoreAtLeast checks the score-only entry point, with sequences
+// long enough for the early stop of the kernel to be exercised.
+func TestBandedLCSScoreAtLeast(t *testing.T) {
+	rng := rand.New(rand.NewSource(17))
+	var buf BandedLCSBuffer
+	check := func(name string, a, b []byte) {
+		os, _ := fullDP(a, b)
+		for _, minLCS := range []int{-1, 0, os - 3, os - 1, os, os + 1, os + 5, len(b) - 1, len(b), (len(b) * 9) / 10, len(b) + 1} {
+			want := os
+			if os < minLCS {
+				want = -1
+			}
+			got, err := buf.ScoreAtLeast(a, b, minLCS)
+			if err != nil || got != want {
+				t.Fatalf("%s minLCS=%d a=%q b=%q: got %d (%v), want %d", name, minLCS, a, b, got, err, want)
+			}
+		}
+	}
+	for _, alpha := range testAlphabets {
+		for it := 0; it < 800; it++ {
+			a := randomFrom(rng, alpha.letters, 1+rng.Intn(60))
+			b := mutateFrom(rng, alpha.letters, a, rng.Float64()*0.6)
+			if len(b) > 0 {
+				check(alpha.name, a, b)
+			}
+		}
+	}
+	for it := 0; it < 150; it++ {
+		a := randomFrom(rng, "ACGT", 100+rng.Intn(400))
+		b := mutateFrom(rng, "ACGT", a, rng.Float64()*0.3)
+		if it%3 == 0 {
+			b = randomFrom(rng, "ACGT", 50+rng.Intn(300))
+		}
+		if len(b) > 0 {
+			check("long", a, b)
+		}
+	}
+}

pkg/obialign/lcsmin.go

@@ -0,0 +1,103 @@
+package obialign
+
+// BandedLCSBuffer is a working memory that can be reused from one alignment to
+// the next to avoid any allocation. It must not be shared between goroutines.
+type BandedLCSBuffer struct {
+	ws bandedLCSWorkspace
+}
+
+// AtLeast is BandedLCSAtLeast on the buffer.
+func (b *BandedLCSBuffer) AtLeast(seqA, seqB []byte, minLCS int) (lcs, alilen int, err error) {
+	return atLeast(seqA, seqB, minLCS, &b.ws)
+}
+
+func atLeast(seqA, seqB []byte, minLCS int, ws *bandedLCSWorkspace) (lcs, alilen int, err error) {
+	if len(seqA) > MaxBandedLCSLength || len(seqB) > MaxBandedLCSLength {
+		return -1, -1, ErrSequenceTooLong
+	}
+	if len(seqA) < len(seqB) {
+		seqA, seqB = seqB, seqA
+	}
+	lcs, alilen = bandedLCSAtLeast(seqA, seqB, minLCS, ws, haveSIMD)
+	return lcs, alilen, nil
+}
+
+// BandedLCSAtLeast computes the LCS score of two nucleotide sequences and the
+// length of the shortest alignment achieving it, as BandedLCS does, but with
+// the bound expressed on the score: the result is the exact optimum when its
+// score is at least minLCS, and (-1, -1) otherwise.
+//
+// The DP only visits the diagonals an alignment of score minLCS can use (one
+// whose path makes at most lB-minLCS gaps on one side and lA-minLCS on the
+// other), so the cost decreases as minLCS grows. This is exact: every alignment
+// of score >= minLCS lies in that band, hence so does the optimum when it
+// reaches minLCS, and when the best path of the band is below minLCS so is
+// the global optimum.
+func BandedLCSAtLeast(seqA, seqB []byte, minLCS int) (lcs, alilen int, err error) {
+	ws := bandedLCSPool.Get().(*bandedLCSWorkspace)
+	lcs, alilen, err = atLeast(seqA, seqB, minLCS, ws)
+	bandedLCSPool.Put(ws)
+	return lcs, alilen, err
+}
+
+// bandedLCSAtLeast expects len(a) >= len(b), both within MaxBandedLCSLength.
+func bandedLCSAtLeast(a, b []byte, minLCS int, ws *bandedLCSWorkspace, useSIMD bool) (int, int) {
+	lA, lB := len(a), len(b)
+	minLCS = max(minLCS, 0)
+	if minLCS > lB {
+		return -1, -1
+	}
+	if lB == 0 {
+		return 0, lA
+	}
+	simdOK := useSIMD && allRegularIUPAC(a) && allRegularIUPAC(b)
+	kmin, kmax := -(lB - minLCS), lA-minLCS
+
+	if !simdOK {
+		lcs, alilen := ws.align(a, b, kmin, kmax, false)
+		if lcs < minLCS {
+			return -1, -1
+		}
+		return lcs, alilen
+	}
+
+	// The score alone is much cheaper than the score with the length of the
+	// alignment: the latter is only computed when the pair reaches minLCS. With
+	// no minimum there is nothing to reject, and the pass would be wasted.
+	if minLCS > 0 && lcsScoreSIMD(a, b, kmin, kmax, minLCS, ws) < minLCS {
+		return -1, -1
+	}
+	encodeBases(ws, a, b)
+	return ws.align(a, b, kmin, kmax, true)
+}
+
+// ScoreAtLeast computes the LCS score of two nucleotide sequences when it is
+// at least minLCS, and returns -1 otherwise. It is much cheaper than AtLeast,
+// which also computes the length of the shortest alignment.
+func (b *BandedLCSBuffer) ScoreAtLeast(seqA, seqB []byte, minLCS int) (int, error) {
+	if len(seqA) > MaxBandedLCSLength || len(seqB) > MaxBandedLCSLength {
+		return -1, ErrSequenceTooLong
+	}
+	if len(seqA) < len(seqB) {
+		seqA, seqB = seqB, seqA
+	}
+	lA, lB := len(seqA), len(seqB)
+	minLCS = max(minLCS, 0)
+	if minLCS > lB {
+		return -1, nil
+	}
+	if lB == 0 {
+		return 0, nil
+	}
+	kmin, kmax := -(lB - minLCS), lA-minLCS
+	var lcs int
+	if haveSIMD && allRegularIUPAC(seqA) && allRegularIUPAC(seqB) {
+		lcs = lcsScoreSIMD(seqA, seqB, kmin, kmax, minLCS, &b.ws)
+	} else {
+		lcs, _ = b.ws.align(seqA, seqB, kmin, kmax, false)
+	}
+	if lcs < minLCS {
+		return -1, nil
+	}
+	return lcs, nil
+}

pkg/obialign/lcsscore_simd.go

@@ -0,0 +1,156 @@
+//go:build goexperiment.simd
+
+package obialign
+
+import "simd"
+
+func simdLanes16() int {
+	var probe simd.Uint16s
+	return probe.Len()
+}
+
+func growUint16(s []uint16, n int) []uint16 {
+	if cap(s) < n {
+		return make([]uint16, n)
+	}
+	return s[:n]
+}
+
+func encodeIUPAC16(dst []uint16, s []byte, rev bool) {
+	n := len(s)
+	clear(dst)
+	if rev {
+		for x, c := range s {
+			dst[n-1-x] = uint16(iupacMask[c])
+		}
+		return
+	}
+	for x, c := range s {
+		dst[x] = uint16(iupacMask[c])
+	}
+}
+
+// pruneEvery is the number of antidiagonals between two tests of the upper
+// bound that stops a hopeless alignment.
+const pruneEvery = 8
+
+// lcsScoreSIMD runs the same banded DP as bandedLCSSIMD on the score only,
+// with 16-bit lanes (twice the cells per instruction):
+//
+//	cell(i,j) = max(diag + match, up, left)
+//
+// A cell outside the band is 0, which never beats a real cell. The score of a
+// path ending at the cell (i, j) of an antidiagonal cannot exceed the score of
+// the cell plus min(lB-i, lA-j), the characters left to pair. Every path
+// crosses each antidiagonal inside the window, so when the best score of the
+// window plus the largest of these remainders is under minLCS the final score
+// is under minLCS: -1 is returned without computing the rest of the matrix.
+//
+// Requires lA >= lB >= 1, kmin <= 0 <= kmax, and bases with non-zero masks.
+// Returns the best score of the band, or -1 if it is proven to be under
+// minLCS.
+func lcsScoreSIMD(a, b []byte, kmin, kmax, minLCS int, ws *bandedLCSWorkspace) int {
+	lA, lB := len(a), len(b)
+	vl := simdLanes16()
+
+	ws.aRev16 = growUint16(ws.aRev16, lA+vl)
+	ws.bCode16 = growUint16(ws.bCode16, lB+vl)
+	encodeIUPAC16(ws.aRev16, a, true)
+	encodeIUPAC16(ws.bCode16, b, false)
+	aRev, bCode := ws.aRev16, ws.bCode16
+
+	size := lB + 2*vl + 2
+	for r := range ws.rows16 {
+		ws.rows16[r] = growUint16(ws.rows16[r], size)
+		clear(ws.rows16[r])
+	}
+	p2, p1, cur := ws.rows16[0], ws.rows16[1], ws.rows16[2]
+
+	// the first row and the first column hold a score of 0: nothing to set
+
+	zero := simd.BroadcastUint16s(0)
+	one := simd.BroadcastUint16s(1)
+
+	for t := 2; t <= lA+lB; t++ {
+		lo := max(1, t-lA, (t-kmax+1)>>1)
+		hi := min(lB, t-1, (t-kmin)>>1)
+
+		for i := lo; i <= hi; i += vl {
+			ca := simd.LoadUint16s(aRev[lA-t+i:])
+			cb := simd.LoadUint16s(bCode[i-1:])
+			inc := one.Masked(ca.And(cb).NotEqual(zero))
+			d := simd.LoadUint16s(p2[i-1:]).Add(inc)
+			u := simd.LoadUint16s(p1[i-1:])
+			l := simd.LoadUint16s(p1[i:])
+			d.Max(u).Max(l).Store(cur[i:])
+		}
+
+		zero.Store(cur[hi+1:])
+		if lo > 1 {
+			cur[lo-1] = 0
+		}
+
+		// A diagonal move skips an antidiagonal, but a path always visits
+		// one of two consecutive antidiagonals.
+		if t%pruneEvery == 0 && minLCS > 0 &&
+			scoreBound(p1, t-1, lA, lB, kmin, kmax) < minLCS &&
+			scoreBound(cur, t, lA, lB, kmin, kmax) < minLCS {
+			return -1
+		}
+
+		p2, p1, cur = p1, cur, p2
+	}
+	return int(p1[lB])
+}
+
+// iota16 is 0, 1, 2... to build the row index of the lanes.
+var iota16 = func() (t [64]uint16) {
+	for i := range t {
+		t[i] = uint16(i)
+	}
+	return t
+}()
+
+// scoreBound is an upper bound of the final score of the paths going through
+// the antidiagonal t, whose scores are in row. A path through the cell (i, j)
+// scores at most the score of the cell plus min(lB-i, lA-j), the characters
+// left to pair, so the bound is the maximum of this sum on the cells of the
+// window. The sum is computed by vectors on the full chunks of the window.
+func scoreBound(row []uint16, t, lA, lB, kmin, kmax int) int {
+	lo := max(1, t-lA, (t-kmax+1)>>1)
+	hi := min(lB, t-1, (t-kmin)>>1)
+
+	bound := 0
+	i := lo
+	if hi-lo >= simdLanes16() {
+		vl := simdLanes16()
+		idx := simd.LoadUint16s(iota16[:]).Add(simd.BroadcastUint16s(uint16(lo)))
+		step := simd.BroadcastUint16s(uint16(vl))
+		remB := simd.BroadcastUint16s(uint16(lB))
+		// lA-t+i is not negative on the window; the 16-bit arithmetic is modular
+		offA := simd.BroadcastUint16s(uint16(lA - t))
+		acc := simd.BroadcastUint16s(0)
+		for ; i+vl-1 <= hi; i += vl {
+			left := remB.Sub(idx).Min(idx.Add(offA))
+			acc = acc.Max(simd.LoadUint16s(row[i:]).Add(left))
+			idx = idx.Add(step)
+		}
+		var lanes [64]uint16
+		acc.Store(lanes[:])
+		for _, v := range lanes[:vl] {
+			bound = max(bound, int(v))
+		}
+	}
+	for ; i <= hi; i++ {
+		bound = max(bound, int(row[i])+min(lB-i, lA-t+i))
+	}
+
+	// the paths running along the first row or the first column have a score of 0
+	if t <= lA && t <= kmax {
+		bound = max(bound, min(lB, lA-t))
+	}
+	if t <= lB && -t >= kmin {
+		bound = max(bound, min(lB-t, lA))
+	}
+	return bound
+}

pkg/obialign/locatepattern.go

@@ -28,10 +28,18 @@ func buffIndex(i, j, width int) int {
 //
 // The function returns the start and end positions of the best
 // match, as well as the number of errors in the best match.
+//
+// When the sequence is too short relative to the pattern for the
+// backtracking to reconstruct a valid alignment (e.g. the pattern
+// is longer than the sequence, or the match sits too close to a
+// sequence boundary), no reliable position can be computed. In that
+// case the function returns the sentinel (-1, -1, -1) instead of a
+// guessed, potentially wrong, position: callers must treat this as
+// "no match" rather than use the returned coordinates.
 func LocatePattern(id string, pattern, sequence []byte) (int, int, int) {
 
-	if len(pattern) >= len(sequence) {
-		log.Panicf("Sequence %s:Pattern %s must be shorter than sequence %s", id, pattern, sequence)
+	if len(sequence) == 0 {
+		log.Panicf("Sequence %s:Pattern %s must not be empty", id, pattern)
 	}
 
 	// Pattern spreads over the columns
@@ -158,5 +166,18 @@ func LocatePattern(id string, pattern, sequence []byte) (int, int, int) {
 	// obilog.Warnf("from : %d to: %d error: %d match: %v",
 	// 	i, end+1, -buffer[buffIndex(len(sequence)-1, len(pattern)-1, width)],
 	// 	string(sequence[i:(end+1)]))
+
+	if i < 0 || end == -1 {
+		// i < 0: the backtracking ran off the start of the sequence
+		// without fully consuming the pattern.
+		// end == -1: the backtracking loop never ran at all (e.g. a
+		// single-base pattern, jmax == 0), so no alignment boundary
+		// was ever established.
+		// Either way, no valid alignment exists for this (pattern,
+		// sequence) pair: signal it explicitly instead of returning
+		// an out-of-bounds or uncomputed position.
+		return -1, -1, -1
+	}
+
 	return i, end + 1, -buffer[buffIndex(len(sequence)-1, len(pattern)-1, width)]
 }

pkg/obialign/locatepattern_test.go

@@ -0,0 +1,123 @@
+package obialign
+
+import (
+	"math/rand"
+	"testing"
+)
+
+func TestLocatePatternNormal(t *testing.T) {
+	// Pattern fully and exactly present in the middle of a longer sequence.
+	start, end, nerr := LocatePattern("id", []byte("ACGT"), []byte("TTTTACGTTTTT"))
+	if start != 4 || end != 8 || nerr != 0 {
+		t.Errorf("got start=%d end=%d nerr=%d, want start=4 end=8 nerr=0", start, end, nerr)
+	}
+}
+
+func TestLocatePatternOneMismatch(t *testing.T) {
+	start, end, nerr := LocatePattern("id", []byte("ACGT"), []byte("TTTTACTTTTTT"))
+	if nerr != 1 {
+		t.Errorf("got nerr=%d, want 1 (start=%d end=%d)", nerr, start, end)
+	}
+}
+
+// The real-world case that used to panic: pattern longer than the sequence
+// fragment extracted for indel relocation.
+func TestLocatePatternPatternLongerThanSequence(t *testing.T) {
+	start, end, nerr := LocatePattern("id",
+		[]byte("GGGCAATCCTGAGCCAAATC"),
+		[]byte("tcctgagccaaatcacgtt"))
+
+	if start != -1 || end != -1 || nerr != -1 {
+		t.Errorf("got start=%d end=%d nerr=%d, want the (-1,-1,-1) sentinel", start, end, nerr)
+	}
+}
+
+func TestLocatePatternSequenceLengthOne(t *testing.T) {
+	start, end, nerr := LocatePattern("id", []byte("AB"), []byte("A"))
+
+	if start < 0 || end < 0 || nerr < 0 {
+		t.Fatalf("got start=%d end=%d nerr=%d, expected a valid (non-sentinel) result", start, end, nerr)
+	}
+	if start != 0 || end != 1 || nerr != 1 {
+		t.Errorf("got start=%d end=%d nerr=%d, want start=0 end=1 nerr=1", start, end, nerr)
+	}
+}
+
+// A pattern much longer than the sequence can still yield a mathematically
+// valid (in-bounds) alignment: the extra pattern length is absorbed as gaps,
+// driving the error count high enough that the caller's maxerr threshold
+// rejects it. The function itself must still return consistent bounds.
+func TestLocatePatternPatternMuchLongerThanSequence(t *testing.T) {
+	start, end, nerr := LocatePattern("id", []byte("ACGTACGTACGTACGTACGT"), []byte("ACG"))
+
+	isSentinel := start == -1 && end == -1 && nerr == -1
+	isValid := start >= 0 && end > start && end <= 3 && nerr >= 0
+
+	if !isSentinel && !isValid {
+		t.Errorf("got start=%d end=%d nerr=%d, want either the sentinel or consistent in-bounds values", start, end, nerr)
+	}
+}
+
+func TestLocatePatternNeverReturnsOutOfBounds(t *testing.T) {
+	patterns := []string{"A", "AC", "ACG", "ACGT", "ACGTA", "ACGTAC", "ACGTACG", "ACGTACGT"}
+	sequences := []string{"A", "AC", "ACG", "ACGT", "ACGTA", "ACGTAC", "ACGTACG", "ACGTACGT"}
+
+	for _, p := range patterns {
+		for _, s := range sequences {
+			start, end, nerr := LocatePattern("id", []byte(p), []byte(s))
+
+			if start == -1 && end == -1 && nerr == -1 {
+				// Explicit "no reliable match" sentinel: always acceptable.
+				continue
+			}
+
+			if start < 0 || end < 0 || start >= end || end > len(s) || nerr < 0 {
+				t.Errorf("pattern=%q sequence=%q -> start=%d end=%d nerr=%d is out of bounds / inconsistent",
+					p, s, start, end, nerr)
+			}
+		}
+	}
+}
+
+// Randomized property test over a wide range of pattern/sequence length
+// combinations, including pattern >= sequence, to make sure the function
+// never panics and never returns anything but the sentinel or fully
+// consistent, in-bounds coordinates.
+func TestLocatePatternRandomizedNeverInvalid(t *testing.T) {
+	const bases = "ACGT"
+	rng := rand.New(rand.NewSource(42))
+
+	randSeq := func(n int) []byte {
+		b := make([]byte, n)
+		for i := range b {
+			b[i] = bases[rng.Intn(len(bases))]
+		}
+		return b
+	}
+
+	for trial := 0; trial < 5000; trial++ {
+		patLen := 1 + rng.Intn(15)
+		seqLen := 1 + rng.Intn(15)
+
+		pattern := randSeq(patLen)
+		sequence := randSeq(seqLen)
+
+		func() {
+			defer func() {
+				if r := recover(); r != nil {
+					t.Fatalf("panic for pattern=%q sequence=%q: %v", pattern, sequence, r)
+				}
+			}()
+
+			start, end, nerr := LocatePattern("id", pattern, sequence)
+
+			isSentinel := start == -1 && end == -1 && nerr == -1
+			isValid := start >= 0 && end > start && end <= len(sequence) && nerr >= 0
+
+			if !isSentinel && !isValid {
+				t.Errorf("pattern=%q sequence=%q -> start=%d end=%d nerr=%d is neither the sentinel nor consistent",
+					pattern, sequence, start, end, nerr)
+			}
+		}()
+	}
+}

pkg/obiapat/pattern.go

@@ -373,6 +373,7 @@ func (pattern ApatPattern) BestMatch(sequence ApatSequence, begin, length int) (
 
 	cpattern := (*[1 << 30]byte)(unsafe.Pointer(pattern.pointer.pointer.cpat))
 	frg := sequence.pointer.reference.Sequence()[start:end]
+	fragStart := start
 
 	log.Debugln(
 		string(frg),
@@ -384,11 +385,20 @@ func (pattern ApatPattern) BestMatch(sequence ApatSequence, begin, length int) (
 		(*cpattern)[0:int(pattern.pointer.pointer.patlen)],
 		frg)
 
-	// olderr := m[2]
+	if from < 0 {
+		// obialign.LocatePattern could not reconstruct a reliable
+		// alignment (e.g. the fragment is too short relative to the
+		// pattern). Reporting a guessed position would risk placing
+		// the primer boundary incorrectly, so treat it as no match
+		// at all rather than falling back to an unrefined position.
+		matched = false
+		log.Debugln("No reliable indel relocation, discarding match", sequence.pointer.reference.Id())
+		return
+	}
 
 	nerr = score
-	start = start + from
-	end = start + to
+	start = fragStart + from
+	end = fragStart + to
 	log.Debugf("BestMatch on %s : score=%d [%d..%d]", sequence.pointer.reference.Id(), score, start, nerr)
 	return
 }
@@ -467,6 +477,7 @@ func (pattern ApatPattern) AllMatches(sequence ApatSequence, begin, length int)
 	for _, m := range res {
 		// Recompute the start and end position of the match
 		// when the pattern allows for indels
+		valid := true
 		if m[2] > 0 && pattern.pointer.pointer.hasIndel {
 			// obilog.Warnf("Locating indel on sequence %s[%s]", sequence.pointer.reference.Id(), pattern.String())
 			start := m[0] - m[2]*2
@@ -485,16 +496,26 @@ func (pattern ApatPattern) AllMatches(sequence ApatSequence, begin, length int)
 				(*cpattern)[0:int(pattern.pointer.pointer.patlen)],
 				frg)
 
-			// olderr := m[2]
-			m[2] = score
-			m[0] = start + pb
-			m[1] = start + pe
+			if pb < 0 {
+				// obialign.LocatePattern could not reconstruct a
+				// reliable alignment (e.g. the match sits too close
+				// to a sequence end for the fragment to be usable).
+				// Reporting a guessed position risks placing the
+				// primer boundary incorrectly, so drop the match
+				// entirely instead of keeping an unrefined guess.
+				valid = false
+			} else {
+				// olderr := m[2]
+				m[2] = score
+				m[0] = start + pb
+				m[1] = start + pe
 
-			// obilog.Warnf("seq[%d@%d:%d] %d: %s %d - %s:%s:%s", i, m[0], m[1], olderr, sequence.pointer.reference.Id(), score,
-			// 	frg, (*cpattern)[0:int(pattern.pointer.pointer.patlen)], sequence.pointer.reference.Sequence()[m[0]:m[1]])
+				// obilog.Warnf("seq[%d@%d:%d] %d: %s %d - %s:%s:%s", i, m[0], m[1], olderr, sequence.pointer.reference.Id(), score,
+				// 	frg, (*cpattern)[0:int(pattern.pointer.pointer.patlen)], sequence.pointer.reference.Sequence()[m[0]:m[1]])
+			}
 		}
 
-		if int(pattern.pointer.pointer.maxerr) >= m[2] {
+		if valid && int(pattern.pointer.pointer.maxerr) >= m[2] {
 			res[j] = m
 			j++
 		}

pkg/obidefault/batch.go

@@ -1,6 +1,12 @@
 package obidefault
 
-var _BatchSize = 2000
+// _BatchSize is the minimum number of sequences per batch (floor).
+// Used as the minSeqs argument to RebatchBySize.
+var _BatchSize = 1
+
+// _BatchSizeMax is the maximum number of sequences per batch (ceiling).
+// A batch is flushed when this count is reached regardless of memory usage.
+var _BatchSizeMax = 2000
 
 // SetBatchSize sets the size of the sequence batches.
 //
@@ -24,3 +30,42 @@ func BatchSize() int {
 func BatchSizePtr() *int {
 	return &_BatchSize
 }
+
+// BatchSizeMax returns the maximum number of sequences per batch.
+func BatchSizeMax() int {
+	return _BatchSizeMax
+}
+
+func BatchSizeMaxPtr() *int {
+	return &_BatchSizeMax
+}
+
+// _BatchMem holds the maximum cumulative memory (in bytes) per batch when
+// memory-based batching is requested. A value of 0 disables memory-based
+// batching and falls back to count-based batching.
+var _BatchMem = 128 * 1024 * 1024 // 128 MB default; set to 0 to disable
+var _BatchMemStr = ""
+
+// SetBatchMem sets the memory budget per batch in bytes.
+func SetBatchMem(n int) {
+	_BatchMem = n
+}
+
+// BatchMem returns the current memory budget per batch in bytes.
+// A value of 0 means memory-based batching is disabled.
+func BatchMem() int {
+	return _BatchMem
+}
+
+func BatchMemPtr() *int {
+	return &_BatchMem
+}
+
+// BatchMemStr returns the raw --batch-mem string value as provided on the CLI.
+func BatchMemStr() string {
+	return _BatchMemStr
+}
+
+func BatchMemStrPtr() *string {
+	return &_BatchMemStr
+}

pkg/obidefault/progressbar.go

@@ -1,5 +1,7 @@
 package obidefault
 
+import "os"
+
 var __no_progress_bar__ = false
 
 func ProgressBar() bool {
@@ -17,3 +19,22 @@ func SetNoProgressBar(b bool) {
 func NoProgressBarPtr() *bool {
 	return &__no_progress_bar__
 }
+
+// ProgressBarVisible tells whether a progress bar can be displayed: it is not
+// disabled by --no-progressbar, the standard error output is a terminal and
+// the standard output is not piped to another command.
+func ProgressBarVisible() bool {
+	if !ProgressBar() {
+		return false
+	}
+
+	if e, err := os.Stderr.Stat(); err != nil || (e.Mode()&os.ModeCharDevice) != os.ModeCharDevice {
+		return false
+	}
+
+	if o, err := os.Stdout.Stat(); err == nil && (o.Mode()&os.ModeNamedPipe) == os.ModeNamedPipe {
+		return false
+	}
+
+	return true
+}

pkg/obiformats/batch_of_files_reader.go

@@ -48,6 +48,14 @@ func ReadSequencesBatchFromFiles(filenames []string,
 
 				log.Printf("Start reading of file : %s", filename)
 
+				// The batches of a file reach us in any order (parallel parsing
+				// of the headers). They are numbered here in arrival order, so
+				// the order of the file must be restored first, unless the
+				// order of the data is not wanted (several concurrent readers).
+				if concurrent_readers == 1 {
+					iter = iter.SortBatches()
+				}
+
 				for iter.Next() {
 					batch := iter.Get()
 					batchiter.Push(batch.Reorder(nextCounter()))

pkg/obiformats/batch_of_files_reader_test.go

@@ -0,0 +1,38 @@
+package obiformats
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// With a single reader, the sequences of several files must come out in the
+// order of the files, whatever the order in which the batches are parsed.
+func TestReadSequencesBatchFromFilesKeepsOrder(t *testing.T) {
+	dir := t.TempDir()
+	var names []string
+	for _, prefix := range []string{"a", "b"} {
+		var sb strings.Builder
+		for i := 0; i < 25000; i++ {
+			fmt.Fprintf(&sb, ">%s%07d {\"k\":%d}\n%s\n", prefix, i, i, strings.Repeat("acgt", 40))
+		}
+		name := filepath.Join(dir, prefix+".fasta")
+		if err := os.WriteFile(name, []byte(sb.String()), 0o600); err != nil {
+			t.Fatal(err)
+		}
+		names = append(names, name)
+	}
+
+	it := ReadSequencesBatchFromFiles(names, nil, 1)
+	_, db := it.Load()
+	if len(db) != 50000 {
+		t.Fatalf("expected 50000 sequences, got %d", len(db))
+	}
+	for i := 1; i < len(db); i++ {
+		if db[i-1].Id() >= db[i].Id() {
+			t.Fatalf("out of order at %d: %s then %s", i, db[i-1].Id(), db[i].Id())
+		}
+	}
+}

pkg/obiformats/embl_read.go

@@ -161,6 +161,149 @@ func EmblChunkParser(withFeatureTable, UtoT bool) func(string, io.Reader) (obise
 	return parser
 }
 
+// extractEmblSeq scans the sequence section of an EMBL record directly on the
+// rope. EMBL sequence lines start with 5 spaces followed by bases in groups of
+// 10, separated by spaces, with a position number at the end. The section ends
+// with "//".
+func (s *ropeScanner) extractEmblSeq(dest []byte, UtoT bool) []byte {
+	// We use ReadLine and scan each line for bases (skip digits, spaces, newlines).
+	for {
+		line := s.ReadLine()
+		if line == nil {
+			break
+		}
+		if len(line) >= 2 && line[0] == '/' && line[1] == '/' {
+			break
+		}
+		// Lines start with 5 spaces; bases follow separated by single spaces.
+		// Digits at the end are the position counter — skip them.
+		// Simplest: take every byte that is a letter.
+		for _, b := range line {
+			if b >= 'A' && b <= 'Z' {
+				b += 'a' - 'A'
+			}
+			if UtoT && b == 'u' {
+				b = 't'
+			}
+			if b >= 'a' && b <= 'z' {
+				dest = append(dest, b)
+			}
+		}
+	}
+	return dest
+}
+
+// EmblChunkParserRope parses an EMBL chunk directly from a rope without Pack().
+func EmblChunkParserRope(source string, rope *PieceOfChunk, withFeatureTable, UtoT bool) (obiseq.BioSequenceSlice, error) {
+	scanner := newRopeScanner(rope)
+	sequences := obiseq.MakeBioSequenceSlice(100)[:0]
+
+	var id string
+	var scientificName string
+	defBytes := make([]byte, 0, 256)
+	featBytes := make([]byte, 0, 1024)
+	var taxid int
+	inSeq := false
+
+	for {
+		line := scanner.ReadLine()
+		if line == nil {
+			break
+		}
+
+		if inSeq {
+			// Should not happen — extractEmblSeq consumed up to "//"
+			inSeq = false
+			continue
+		}
+
+		switch {
+		case bytes.HasPrefix(line, []byte("ID   ")):
+			id = string(bytes.SplitN(line[5:], []byte(";"), 2)[0])
+		case bytes.HasPrefix(line, []byte("OS   ")):
+			scientificName = string(bytes.TrimSpace(line[5:]))
+		case bytes.HasPrefix(line, []byte("DE   ")):
+			if len(defBytes) > 0 {
+				defBytes = append(defBytes, ' ')
+			}
+			defBytes = append(defBytes, bytes.TrimSpace(line[5:])...)
+		case withFeatureTable && bytes.HasPrefix(line, []byte("FH   ")):
+			featBytes = append(featBytes, line...)
+		case withFeatureTable && bytes.Equal(line, []byte("FH")):
+			featBytes = append(featBytes, '\n')
+			featBytes = append(featBytes, line...)
+		case bytes.HasPrefix(line, []byte("FT   ")):
+			if withFeatureTable {
+				featBytes = append(featBytes, '\n')
+				featBytes = append(featBytes, line...)
+			}
+			if bytes.HasPrefix(line, []byte(`FT                   /db_xref="taxon:`)) {
+				rest := line[37:]
+				end := bytes.IndexByte(rest, '"')
+				if end > 0 {
+					taxid, _ = strconv.Atoi(string(rest[:end]))
+				}
+			}
+		case bytes.HasPrefix(line, []byte("     ")):
+			// First sequence line: extract all bases via extractEmblSeq,
+			// which also consumes this line's remaining content.
+			// But ReadLine already consumed this line — we need to process it
+			// plus subsequent lines. Process this line inline then call helper.
+			seqDest := make([]byte, 0, 4096)
+			for _, b := range line {
+				if b >= 'A' && b <= 'Z' {
+					b += 'a' - 'A'
+				}
+				if UtoT && b == 'u' {
+					b = 't'
+				}
+				if b >= 'a' && b <= 'z' {
+					seqDest = append(seqDest, b)
+				}
+			}
+			seqDest = scanner.extractEmblSeq(seqDest, UtoT)
+
+			seq := obiseq.NewBioSequenceOwning(id, seqDest, string(defBytes))
+			seq.SetSource(source)
+			if withFeatureTable {
+				seq.SetFeatures(featBytes)
+			}
+			annot := seq.Annotations()
+			annot["scientific_name"] = scientificName
+			annot["taxid"] = taxid
+			sequences = append(sequences, seq)
+
+			// Reset state
+			id = ""
+			scientificName = ""
+			defBytes = defBytes[:0]
+			featBytes = featBytes[:0]
+			taxid = 1
+
+		case bytes.Equal(line, []byte("//")):
+			// record ended without SQ/sequence section (e.g. WGS entries)
+			if id != "" {
+				seq := obiseq.NewBioSequenceOwning(id, []byte{}, string(defBytes))
+				seq.SetSource(source)
+				if withFeatureTable {
+					seq.SetFeatures(featBytes)
+				}
+				annot := seq.Annotations()
+				annot["scientific_name"] = scientificName
+				annot["taxid"] = taxid
+				sequences = append(sequences, seq)
+			}
+			id = ""
+			scientificName = ""
+			defBytes = defBytes[:0]
+			featBytes = featBytes[:0]
+			taxid = 1
+		}
+	}
+
+	return sequences, nil
+}
+
 func _ParseEmblFile(
 	input ChannelFileChunk,
 	out obiiter.IBioSequence,
@@ -171,7 +314,14 @@ func _ParseEmblFile(
 
 	for chunks := range input {
 		order := chunks.Order
-		sequences, err := parser(chunks.Source, chunks.Raw)
+		var sequences obiseq.BioSequenceSlice
+		var err error
+
+		if chunks.Rope != nil {
+			sequences, err = EmblChunkParserRope(chunks.Source, chunks.Rope, withFeatureTable, UtoT)
+		} else {
+			sequences, err = parser(chunks.Source, chunks.Raw)
+		}
 
 		if err != nil {
 			log.Fatalf("%s : Cannot parse the embl file : %v", chunks.Source, err)
@@ -196,6 +346,7 @@ func ReadEMBL(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, er
 		1024*1024*128,
 		EndOfLastFlatFileEntry,
 		"\nID   ",
+		false,
 	)
 
 	newIter := obiiter.MakeIBioSequence()

pkg/obiformats/fastaseq_read.go

@@ -209,28 +209,121 @@ func FastaChunkParser(UtoT bool) func(string, io.Reader) (obiseq.BioSequenceSlic
 	return parser
 }
 
+// extractFastaSeq scans sequence bytes from the rope directly into dest,
+// appending valid nucleotide characters and skipping whitespace.
+// Stops when '>' is found at the start of a line (next record) or at EOF.
+// Returns (dest with appended bases, hasMore).
+// hasMore=true means scanner is now positioned at '>' of the next record.
+func (s *ropeScanner) extractFastaSeq(dest []byte, UtoT bool) ([]byte, bool) {
+	lineStart := true
+
+	for s.current != nil {
+		data := s.current.data[s.pos:]
+		for i, b := range data {
+			if lineStart && b == '>' {
+				s.pos += i
+				if s.pos >= len(s.current.data) {
+					s.current = s.current.Next()
+					s.pos = 0
+				}
+				return dest, true
+			}
+			if b == '\n' || b == '\r' {
+				lineStart = true
+				continue
+			}
+			lineStart = false
+			if b == ' ' || b == '\t' {
+				continue
+			}
+			if b >= 'A' && b <= 'Z' {
+				b += 'a' - 'A'
+			}
+			if UtoT && b == 'u' {
+				b = 't'
+			}
+			dest = append(dest, b)
+		}
+		s.current = s.current.Next()
+		s.pos = 0
+	}
+	return dest, false
+}
+
+// FastaChunkParserRope parses a FASTA chunk directly from the rope without Pack().
+func FastaChunkParserRope(source string, rope *PieceOfChunk, UtoT bool) (obiseq.BioSequenceSlice, error) {
+	scanner := newRopeScanner(rope)
+	sequences := obiseq.MakeBioSequenceSlice(100)[:0]
+
+	for {
+		bline := scanner.ReadLine()
+		if bline == nil {
+			break
+		}
+		if len(bline) == 0 || bline[0] != '>' {
+			continue
+		}
+
+		// Parse header: ">id definition"
+		header := bline[1:]
+		var id string
+		var definition string
+		sp := bytes.IndexByte(header, ' ')
+		if sp < 0 {
+			sp = bytes.IndexByte(header, '\t')
+		}
+		if sp < 0 {
+			id = string(header)
+		} else {
+			id = string(header[:sp])
+			definition = string(bytes.TrimSpace(header[sp+1:]))
+		}
+
+		seqDest := make([]byte, 0, 4096)
+		var hasMore bool
+		seqDest, hasMore = scanner.extractFastaSeq(seqDest, UtoT)
+
+		if len(seqDest) == 0 {
+			log.Fatalf("%s [%s]: sequence is empty", source, id)
+		}
+
+		seq := obiseq.NewBioSequenceOwning(id, seqDest, definition)
+		seq.SetSource(source)
+		sequences = append(sequences, seq)
+
+		if !hasMore {
+			break
+		}
+	}
+
+	return sequences, nil
+}
+
 func _ParseFastaFile(
 	input ChannelFileChunk,
 	out obiiter.IBioSequence,
 	UtoT bool,
 ) {
-
 	parser := FastaChunkParser(UtoT)
 
 	for chunks := range input {
-		sequences, err := parser(chunks.Source, chunks.Raw)
-		// obilog.Warnf("Chunck(%d:%d) -%d- ", chunks.Order, l, sequences.Len())
+		var sequences obiseq.BioSequenceSlice
+		var err error
+
+		if chunks.Rope != nil {
+			sequences, err = FastaChunkParserRope(chunks.Source, chunks.Rope, UtoT)
+		} else {
+			sequences, err = parser(chunks.Source, chunks.Raw)
+		}
 
 		if err != nil {
 			log.Fatalf("File %s : Cannot parse the fasta file : %v", chunks.Source, err)
 		}
 
 		out.Push(obiiter.MakeBioSequenceBatch(chunks.Source, chunks.Order, sequences))
-
 	}
 
 	out.Done()
-
 }
 
 func ReadFasta(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, error) {
@@ -245,6 +338,7 @@ func ReadFasta(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, e
 		1024*1024,
 		EndOfLastFastaEntry,
 		"\n>",
+		false,
 	)
 
 	for i := 0; i < nworker; i++ {

pkg/obiformats/fastqseq_read.go

@@ -131,7 +131,7 @@ func _storeSequenceQuality(bytes *bytes.Buffer, out *obiseq.BioSequence, quality
 	out.SetQualities(q)
 }
 
-func FastqChunkParser(quality_shift byte, with_quality bool, UtoT bool) func(string, io.Reader) (obiseq.BioSequenceSlice, error) {
+func FastqChunkParser(quality_shift byte, with_quality bool, UtoT bool, fileName string) func(string, io.Reader) (obiseq.BioSequenceSlice, error) {
 	parser := func(source string, input io.Reader) (obiseq.BioSequenceSlice, error) {
 
 		var identifier string
@@ -160,12 +160,12 @@ func FastqChunkParser(quality_shift byte, with_quality bool, UtoT bool) func(str
 					// Beginning of sequence
 					state = 1
 				} else {
-					log.Fatalf("%s : sequence entry is not starting with @", source)
+					log.Fatalf("file %s: sequence entry is not starting with @", fileName)
 				}
 			case 1: // Beginning of identifier (Mandatory)
 				if is_sep {
 					// No identifier -> ERROR
-					log.Fatalf("%s : sequence identifier is empty", source)
+					log.Fatalf("file %s: sequence identifier is empty", fileName)
 				} else {
 					// Beginning of identifier
 					state = 2
@@ -221,7 +221,7 @@ func FastqChunkParser(quality_shift byte, with_quality bool, UtoT bool) func(str
 					// End of sequence
 					rawseq := seqBytes.Bytes()
 					if len(rawseq) == 0 {
-						log.Fatalf("@%s[%s] : sequence is empty", identifier, source)
+						log.Fatalf("file %s: record @%s has an empty sequence line", fileName, identifier)
 					}
 					s := obiseq.NewBioSequence(identifier, rawseq, definition)
 					s.SetSource(source)
@@ -241,8 +241,8 @@ func FastqChunkParser(quality_shift byte, with_quality bool, UtoT bool) func(str
 						context = append(
 							append([]byte{previous}, C),
 							context...)
-						log.Fatalf("%s [%s]: sequence contains invalid character %c (%s)",
-							source, identifier, C, string(context))
+						log.Fatalf("file %s: record @%s contains invalid character %c (%s)",
+							fileName, identifier, C, string(context))
 					}
 				}
 			case 7:
@@ -251,7 +251,7 @@ func FastqChunkParser(quality_shift byte, with_quality bool, UtoT bool) func(str
 				} else if C == '+' {
 					state = 8
 				} else {
-					log.Fatalf("@%s[%s] : sequence data not followed by a line starting with + but a %c", identifier, source, C)
+					log.Fatalf("file %s: record @%s: sequence data not followed by a line starting with + but a %c", fileName, identifier, C)
 				}
 			case 8:
 				// State consuming the + internal header line
@@ -282,7 +282,7 @@ func FastqChunkParser(quality_shift byte, with_quality bool, UtoT bool) func(str
 				} else if C == '@' {
 					state = 1
 				} else {
-					log.Fatalf("%s[%s] : sequence record not followed by a line starting with @", identifier, source)
+					log.Fatalf("file %s: record @%s not followed by a line starting with @", fileName, identifier)
 				}
 
 			}
@@ -303,17 +303,99 @@ func FastqChunkParser(quality_shift byte, with_quality bool, UtoT bool) func(str
 	return parser
 }
 
+// FastqChunkParserRope parses a FASTQ chunk directly from a rope without Pack().
+func FastqChunkParserRope(source string, rope *PieceOfChunk, quality_shift byte, with_quality, UtoT bool, fileName string) (obiseq.BioSequenceSlice, error) {
+	scanner := newRopeScanner(rope)
+	sequences := obiseq.MakeBioSequenceSlice(100)[:0]
+
+	for {
+		// Line 1: @id [definition]
+		hline := scanner.ReadLine()
+		if hline == nil {
+			break
+		}
+		if len(hline) == 0 || hline[0] != '@' {
+			continue
+		}
+		header := hline[1:]
+		var id string
+		var definition string
+		sp := bytes.IndexByte(header, ' ')
+		if sp < 0 {
+			sp = bytes.IndexByte(header, '\t')
+		}
+		if sp < 0 {
+			id = string(header)
+		} else {
+			id = string(header[:sp])
+			definition = string(bytes.TrimSpace(header[sp+1:]))
+		}
+
+		// Line 2: sequence
+		sline := scanner.ReadLine()
+		if sline == nil {
+			log.Fatalf("file %s: record @%s is truncated (header line with no sequence line following) — the FASTQ file appears incomplete", fileName, id)
+		}
+		seqDest := make([]byte, len(sline))
+		w := 0
+		for _, b := range sline {
+			if b >= 'A' && b <= 'Z' {
+				b += 'a' - 'A'
+			}
+			if UtoT && b == 'u' {
+				b = 't'
+			}
+			seqDest[w] = b
+			w++
+		}
+		seqDest = seqDest[:w]
+		if len(seqDest) == 0 {
+			log.Fatalf("file %s: record @%s has an empty sequence line", fileName, id)
+		}
+
+		// Line 3: + (skip)
+		scanner.ReadLine()
+
+		// Line 4: quality
+		qline := scanner.ReadLine()
+
+		seq := obiseq.NewBioSequenceOwning(id, seqDest, definition)
+		seq.SetSource(source)
+
+		if with_quality && qline != nil {
+			qDest := make([]byte, len(qline))
+			copy(qDest, qline)
+			for i := range qDest {
+				qDest[i] -= quality_shift
+			}
+			seq.TakeQualities(qDest)
+		}
+
+		sequences = append(sequences, seq)
+	}
+
+	return sequences, nil
+}
+
 func _ParseFastqFile(
 	input ChannelFileChunk,
 	out obiiter.IBioSequence,
 	quality_shift byte,
 	with_quality, UtoT bool,
+	fileName string,
 ) {
 
-	parser := FastqChunkParser(quality_shift, with_quality, UtoT)
+	parser := FastqChunkParser(quality_shift, with_quality, UtoT, fileName)
 
 	for chunks := range input {
-		sequences, err := parser(chunks.Source, chunks.Raw)
+		var sequences obiseq.BioSequenceSlice
+		var err error
+
+		if chunks.Rope != nil {
+			sequences, err = FastqChunkParserRope(chunks.Source, chunks.Rope, quality_shift, with_quality, UtoT, fileName)
+		} else {
+			sequences, err = parser(chunks.Source, chunks.Raw)
+		}
 
 		if err != nil {
 			log.Fatalf("File %s : Cannot parse the fastq file : %v", chunks.Source, err)
@@ -339,8 +421,11 @@ func ReadFastq(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, e
 		1024*1024,
 		EndOfLastFastqEntry,
 		"\n@",
+		false,
 	)
 
+	fileName := opt.FileName()
+
 	for i := 0; i < nworker; i++ {
 		out.Add(1)
 		go _ParseFastqFile(
@@ -349,6 +434,7 @@ func ReadFastq(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, e
 			obidefault.ReadQualitiesShift(),
 			opt.ReadQualities(),
 			opt.UtoT(),
+			fileName,
 		)
 	}
 
@@ -374,7 +460,9 @@ func ReadFastq(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, e
 }
 
 func ReadFastqFromFile(filename string, options ...WithOption) (obiiter.IBioSequence, error) {
-	options = append(options, OptionsSource(obiutils.RemoveAllExt((path.Base(filename)))))
+	options = append(options,
+		OptionsSource(obiutils.RemoveAllExt((path.Base(filename)))),
+		OptionsFileName(filename))
 
 	file, err := obiutils.Ropen(filename)
 

pkg/obiformats/fastseq_json_header.go

@@ -199,8 +199,111 @@ func _parse_json_array_interface(str []byte) ([]interface{}, error) {
 	return values, nil
 }
 
-func _parse_json_header_(header string, sequence *obiseq.BioSequence) string {
+// _parse_json_annotation_field parses a single key/value pair coming from a
+// JSON object (either a FASTA/FASTQ inline JSON header, or the "annotations"
+// field of a JSON sequence record) and applies it to the sequence, special
+// casing the well-known OBITools attributes (id, definition, count, taxid,
+// obiclean_*, merged_*).
+func _parse_json_annotation_field(key []byte, value []byte, dataType jsonparser.ValueType, sequence *obiseq.BioSequence) error {
 	annotations := sequence.Annotations()
+	var err error
+
+	skey := obiutils.UnsafeString(key)
+
+	switch {
+	case skey == "id":
+		sequence.SetId(string(value))
+	case skey == "definition":
+		sequence.SetDefinition(string(value))
+
+	case skey == "count":
+		if dataType != jsonparser.Number {
+			log.Fatalf("%s: Count attribut must be numeric: %s", sequence.Id(), string(value))
+		}
+		count, err := jsonparser.ParseInt(value)
+		if err != nil {
+			log.Fatalf("%s: Cannot parse count %s", sequence.Id(), string(value))
+		}
+		sequence.SetCount(int(count))
+
+	case skey == "obiclean_weight":
+		weight, err := _parse_json_map_int(value)
+		if err != nil {
+			log.Fatalf("%s: Cannot parse obiclean weight %s", sequence.Id(), string(value))
+		}
+		annotations[skey] = weight
+
+	case skey == "obiclean_status":
+		status, err := _parse_json_map_string(value)
+		if err != nil {
+			log.Fatalf("%s: Cannot parse obiclean status %s", sequence.Id(), string(value))
+		}
+		annotations[skey] = status
+
+	case strings.HasPrefix(skey, "merged_"):
+		if dataType == jsonparser.Object {
+			data, err := _parse_json_map_int(value)
+			if err != nil {
+				log.Fatalf("%s: Cannot parse merged slot %s: %v", sequence.Id(), skey, err)
+			} else {
+				annotations[skey] = obiseq.MapAsStatsOnValues(data)
+			}
+		} else {
+			log.Fatalf("%s: Cannot parse merged slot %s", sequence.Id(), skey)
+		}
+
+	case skey == "taxid":
+		if dataType == jsonparser.Number || dataType == jsonparser.String {
+			taxid := string(value)
+			sequence.SetTaxid(taxid)
+		} else {
+			log.Fatalf("%s: Cannot parse taxid %s", sequence.Id(), string(value))
+		}
+
+	case strings.HasSuffix(skey, "_taxid"):
+		if dataType == jsonparser.Number || dataType == jsonparser.String {
+			rank := skey[:len(skey)-len("_taxid")]
+
+			taxid := string(value)
+			sequence.SetTaxid(taxid, rank)
+		} else {
+			log.Fatalf("%s: Cannot parse taxid %s", sequence.Id(), string(value))
+		}
+
+	default:
+		skey = strings.Clone(skey)
+		switch dataType {
+		case jsonparser.String:
+			annotations[skey] = string(value)
+		case jsonparser.Number:
+			// Try to parse the number as an int at first then as float if that fails.
+			annotations[skey], err = jsonparser.ParseInt(value)
+			if err != nil {
+				annotations[skey], err = strconv.ParseFloat(obiutils.UnsafeString(value), 64)
+			}
+		case jsonparser.Array:
+			annotations[skey], err = _parse_json_array_interface(value)
+		case jsonparser.Object:
+			annotations[skey], err = _parse_json_map_interface(value)
+		case jsonparser.Boolean:
+			annotations[skey], err = jsonparser.ParseBoolean(value)
+		case jsonparser.Null:
+			annotations[skey] = nil
+		default:
+			log.Fatalf("Unknown data type %v", dataType)
+		}
+	}
+
+	if err != nil {
+		annotations[skey] = "NaN"
+		log.Fatalf("%s: Cannot parse value %s assicated to key %s into a %s value",
+			sequence.Id(), string(value), skey, dataType.String())
+	}
+
+	return err
+}
+
+func _parse_json_header_(header string, sequence *obiseq.BioSequence) string {
 	start := -1
 	stop := -1
 	level := 0
@@ -240,101 +343,7 @@ func _parse_json_header_(header string, sequence *obiseq.BioSequence) string {
 
 	jsonparser.ObjectEach(obiutils.UnsafeBytes(header[start:stop]),
 		func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) error {
-			var err error
-
-			skey := obiutils.UnsafeString(key)
-
-			switch {
-			case skey == "id":
-				sequence.SetId(string(value))
-			case skey == "definition":
-				sequence.SetDefinition(string(value))
-
-			case skey == "count":
-				if dataType != jsonparser.Number {
-					log.Fatalf("%s: Count attribut must be numeric: %s", sequence.Id(), string(value))
-				}
-				count, err := jsonparser.ParseInt(value)
-				if err != nil {
-					log.Fatalf("%s: Cannot parse count %s", sequence.Id(), string(value))
-				}
-				sequence.SetCount(int(count))
-
-			case skey == "obiclean_weight":
-				weight, err := _parse_json_map_int(value)
-				if err != nil {
-					log.Fatalf("%s: Cannot parse obiclean weight %s", sequence.Id(), string(value))
-				}
-				annotations[skey] = weight
-
-			case skey == "obiclean_status":
-				status, err := _parse_json_map_string(value)
-				if err != nil {
-					log.Fatalf("%s: Cannot parse obiclean status %s", sequence.Id(), string(value))
-				}
-				annotations[skey] = status
-
-			case strings.HasPrefix(skey, "merged_"):
-				if dataType == jsonparser.Object {
-					data, err := _parse_json_map_int(value)
-					if err != nil {
-						log.Fatalf("%s: Cannot parse merged slot %s: %v", sequence.Id(), skey, err)
-					} else {
-						annotations[skey] = obiseq.MapAsStatsOnValues(data)
-					}
-				} else {
-					log.Fatalf("%s: Cannot parse merged slot %s", sequence.Id(), skey)
-				}
-
-			case skey == "taxid":
-				if dataType == jsonparser.Number || dataType == jsonparser.String {
-					taxid := string(value)
-					sequence.SetTaxid(taxid)
-				} else {
-					log.Fatalf("%s: Cannot parse taxid %s", sequence.Id(), string(value))
-				}
-
-			case strings.HasSuffix(skey, "_taxid"):
-				if dataType == jsonparser.Number || dataType == jsonparser.String {
-					rank, _ := obiutils.SplitInTwo(skey, '_')
-
-					taxid := string(value)
-					sequence.SetTaxid(taxid, rank)
-				} else {
-					log.Fatalf("%s: Cannot parse taxid %s", sequence.Id(), string(value))
-				}
-
-			default:
-				skey = strings.Clone(skey)
-				switch dataType {
-				case jsonparser.String:
-					annotations[skey] = string(value)
-				case jsonparser.Number:
-					// Try to parse the number as an int at first then as float if that fails.
-					annotations[skey], err = jsonparser.ParseInt(value)
-					if err != nil {
-						annotations[skey], err = strconv.ParseFloat(obiutils.UnsafeString(value), 64)
-					}
-				case jsonparser.Array:
-					annotations[skey], err = _parse_json_array_interface(value)
-				case jsonparser.Object:
-					annotations[skey], err = _parse_json_map_interface(value)
-				case jsonparser.Boolean:
-					annotations[skey], err = jsonparser.ParseBoolean(value)
-				case jsonparser.Null:
-					annotations[skey] = nil
-				default:
-					log.Fatalf("Unknown data type %v", dataType)
-				}
-			}
-
-			if err != nil {
-				annotations[skey] = "NaN"
-				log.Fatalf("%s: Cannot parse value %s assicated to key %s into a %s value",
-					sequence.Id(), string(value), skey, dataType.String())
-			}
-
-			return err
+			return _parse_json_annotation_field(key, value, dataType, sequence)
 		},
 	)
 

pkg/obiformats/fastseq_obi_header.go

@@ -17,7 +17,7 @@ import (
 )
 
 var __obi_header_value_string_pattern__ = regexp.MustCompile(`^'\s*([^']*'|"[^"]*")\s*;`)
-var __obi_header_value_numeric_pattern__ = regexp.MustCompile(`^\s*([+-]?\.\d+|[+-]?\d+(\.\d*)?([eE][+-]?\d+)?)\s*;`)
+var __obi_header_value_numeric_pattern__ = regexp.MustCompile(`^\s*[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?\s*;`)
 var __obi_header_map_int_key__ = regexp.MustCompile("([{,])([0-9]+):")
 
 func __match__dict__(text []byte) []int {
@@ -146,6 +146,65 @@ func __match__key__(text []byte) []int {
 	return []int{} // Not a key
 }
 
+func __match__array__(text []byte) []int {
+
+	state := 0
+	level := 0
+	start := 0
+	instring := byte(0)
+
+	for i, r := range text {
+		if state == 2 {
+			if r == ';' {
+				return []int{start, i + 1}
+			}
+			if r != ' ' && r != '\t' {
+				return []int{}
+			}
+		}
+
+		if state == 0 {
+			if r == '[' {
+				level++
+				state++
+				start = i
+				continue
+			}
+			if r != ' ' && r != '\t' {
+				return []int{}
+			}
+			continue
+		}
+
+		// state == 1: inside the array
+		if instring != 0 {
+			if r == instring {
+				instring = 0
+			}
+			continue
+		}
+
+		if r == '"' || r == '\'' {
+			instring = r
+			continue
+		}
+
+		if r == '[' || r == '{' {
+			level++
+			continue
+		}
+
+		if r == ']' || r == '}' {
+			level--
+			if level == 0 {
+				state++
+			}
+		}
+	}
+
+	return []int{}
+}
+
 func __match__general__(text []byte) []int {
 
 	for i, r := range text {
@@ -242,28 +301,44 @@ func ParseOBIFeatures(text string, annotations obiseq.Annotation) string {
 					stop = m[1] + 1
 				} else {
 
-					// Generic value
-
-					// m = __obi_header_value_general_pattern__.FindIndex(part)
-					m = __match__general__(part)
+					// array value
+					m = __match__array__(part)
 					if len(m) > 0 {
 						bvalue = bytes.TrimSpace(part[m[0]:(m[1] - 1)])
-
-						if __is_false__(bvalue) {
-							value = false
+						j := bytes.ReplaceAll(bvalue, []byte("'"), []byte(`"`))
+						j = __obi_header_map_int_key__.ReplaceAll(j, []byte(`$1"$2":`))
+						arr, err := _parse_json_array_interface(j)
+						if err != nil {
+							value = string(bvalue)
 						} else {
-							if __is_true__(bvalue) {
-								value = true
-							} else {
-								value = string(bvalue)
-							}
+							value = arr
 						}
-
 						stop = m[1] + 1
 					} else {
-						// no value
-						break
-					} // End of No value
+
+						// Generic value
+
+						// m = __obi_header_value_general_pattern__.FindIndex(part)
+						m = __match__general__(part)
+						if len(m) > 0 {
+							bvalue = bytes.TrimSpace(part[m[0]:(m[1] - 1)])
+
+							if __is_false__(bvalue) {
+								value = false
+							} else {
+								if __is_true__(bvalue) {
+									value = true
+								} else {
+									value = string(bvalue)
+								}
+							}
+
+							stop = m[1] + 1
+						} else {
+							// no value
+							break
+						} // End of No value
+					} // End of not array
 				} // End of not dict
 			} // End of not string
 		} // End of not numeric
@@ -272,6 +347,8 @@ func ParseOBIFeatures(text string, annotations obiseq.Annotation) string {
 		case float64:
 			if vt == math.Floor(vt) {
 				annotations[key] = int(vt)
+			} else {
+				annotations[key] = vt
 			}
 		default:
 			annotations[key] = value
@@ -327,19 +404,19 @@ func WriteFastSeqOBIHeade(buffer *bytes.Buffer, sequence *obiseq.BioSequence) {
 					buffer.WriteString(fmt.Sprintf("%s=", key))
 					buffer.Write(tv)
 					buffer.WriteString("; ")
-				case map[string]int,
-					map[string]string,
-					map[string]interface{}:
-					tv, err := obiutils.JsonMarshal(t)
-					if err != nil {
-						log.Fatalf("Cannot convert %v value", value)
-					}
-					tv = bytes.ReplaceAll(tv, []byte(`"`), []byte("'"))
-					buffer.WriteString(fmt.Sprintf("%s=", key))
-					buffer.Write(tv)
-					buffer.WriteString("; ")
 				default:
-					buffer.WriteString(fmt.Sprintf("%s=%v; ", key, value))
+					if obiutils.IsAMap(value) || obiutils.IsASlice(value) || obiutils.IsAnArray(value) {
+						tv, err := obiutils.JsonMarshal(t)
+						if err != nil {
+							log.Fatalf("Cannot convert %v value", value)
+						}
+						tv = bytes.ReplaceAll(tv, []byte(`"`), []byte("'"))
+						buffer.WriteString(fmt.Sprintf("%s=", key))
+						buffer.Write(tv)
+						buffer.WriteString("; ")
+					} else {
+						buffer.WriteString(fmt.Sprintf("%s=%v; ", key, value))
+					}
 				}
 			}
 		}

pkg/obiformats/fastseq_write_fasta.go

@@ -77,45 +77,50 @@ func FormatFasta(seq *obiseq.BioSequence, formater FormatHeader) string {
 //
 // It returns a byte array containing the formatted sequences.
 func FormatFastaBatch(batch obiiter.BioSequenceBatch, formater FormatHeader, skipEmpty bool) *bytes.Buffer {
-	// Create a buffer to store the formatted sequences
 	var bs bytes.Buffer
 
 	lt := 0
-
 	for _, seq := range batch.Slice() {
 		lt += seq.Len()
 	}
 
-	// Iterate over each sequence in the batch
+	// Pre-allocate: sequence data + newlines every 60 chars + ~100 bytes header per sequence
+	bs.Grow(lt + lt/60 + 100*batch.Len() + 1)
+
 	log.Debugf("FormatFastaBatch: #%d : %d seqs", batch.Order(), batch.Len())
-	first := true
+
 	for _, seq := range batch.Slice() {
-		// Check if the sequence is empty
+		if len(seq.Id()) == 0 {
+			log.Fatalf("Sequence identifier is empty")
+		}
 		if seq.Len() > 0 {
-			// Format the sequence using the provided formater function
-			formattedSeq := FormatFasta(seq, formater)
-
-			if first {
-				bs.Grow(lt + (len(formattedSeq)-seq.Len())*batch.Len()*5/4)
-				first = false
-			}
-
-			// Append the formatted sequence to the buffer
-			bs.WriteString(formattedSeq)
+			// Write header directly into bs — no intermediate string
+			bs.WriteByte('>')
+			bs.WriteString(seq.Id())
+			bs.WriteByte(' ')
+			bs.WriteString(formater(seq))
 			bs.WriteByte('\n')
+
+			// Write folded sequence directly into bs — no copies
+			s := seq.Sequence()
+			l := len(s)
+			for i := 0; i < l; i += 60 {
+				to := i + 60
+				if to > l {
+					to = l
+				}
+				bs.Write(s[i:to])
+				bs.WriteByte('\n')
+			}
 		} else {
-			// Handle empty sequences
 			if skipEmpty {
-				// Skip empty sequences if skipEmpty is true
 				obilog.Warnf("Sequence %s is empty and skipped in output", seq.Id())
 			} else {
-				// Terminate the program if skipEmpty is false
 				log.Fatalf("Sequence %s is empty", seq.Id())
 			}
 		}
 	}
 
-	// Return the byte array representation of the buffer
 	return &bs
 }
 

pkg/obiformats/fastseq_write_fastq.go

@@ -64,6 +64,9 @@ func FormatFastqBatch(batch obiiter.BioSequenceBatch,
 	first := true
 
 	for _, seq := range batch.Slice() {
+		if len(seq.Id()) == 0 {
+			log.Fatalf("Sequence identifier is empty")
+		}
 		if seq.Len() > 0 {
 			_formatFastq(&bs, seq, formater)
 

pkg/obiformats/file_chunk_read.go

@@ -16,6 +16,7 @@ type SeqFileChunkParser func(string, io.Reader) (obiseq.BioSequenceSlice, error)
 type FileChunk struct {
 	Source string
 	Raw    *bytes.Buffer
+	Rope   *PieceOfChunk
 	Order  int
 }
 
@@ -97,11 +98,17 @@ func (piece *PieceOfChunk) IsLast() bool {
 	return piece.next == nil
 }
 
-func (piece *PieceOfChunk) FileChunk(source string, order int) FileChunk {
-	piece.Pack()
+func (piece *PieceOfChunk) FileChunk(source string, order int, pack bool) FileChunk {
+	piece = piece.Head()
+	var raw *bytes.Buffer
+	if pack {
+		piece.Pack()
+		raw = bytes.NewBuffer(piece.data)
+	}
 	return FileChunk{
 		Source: source,
-		Raw:    bytes.NewBuffer(piece.data),
+		Raw:    raw,
+		Rope:   piece,
 		Order:  order,
 	}
 }
@@ -133,7 +140,8 @@ func ReadFileChunk(
 	reader io.Reader,
 	fileChunkSize int,
 	splitter LastSeqRecord,
-	probe string) ChannelFileChunk {
+	probe string,
+	pack bool) ChannelFileChunk {
 
 	chunk_channel := make(ChannelFileChunk)
 
@@ -205,7 +213,7 @@ func ReadFileChunk(
 
 				if len(pieces.data) > 0 {
 					// obilog.Warnf("chuck %d :Read %d bytes from file %s", i, io.Len(), source)
-					chunk_channel <- pieces.FileChunk(source, i)
+					chunk_channel <- pieces.FileChunk(source, i, pack)
 					i++
 				}
 
@@ -222,7 +230,7 @@ func ReadFileChunk(
 
 		// Send the last chunk to the channel
 		if pieces.Len() > 0 {
-			chunk_channel <- pieces.FileChunk(source, i)
+			chunk_channel <- pieces.FileChunk(source, i, pack)
 		}
 
 		// Close the readers channel when the end of the file is reached

pkg/obiformats/genbank_read.go

@@ -29,6 +29,265 @@ const (
 
 var _seqlenght_rx = regexp.MustCompile(" +([0-9]+) bp")
 
+// extractSequence scans the ORIGIN section byte-by-byte directly on the rope,
+// appending compacted bases to dest. Returns the extended slice.
+// Stops and returns when "//" is found at the start of a line.
+// The scanner is left positioned after the "//" line.
+func (s *ropeScanner) extractSequence(dest []byte, UtoT bool) []byte {
+	lineStart := true
+	skipDigits := true
+
+	for s.current != nil {
+		data := s.current.data[s.pos:]
+		for i, b := range data {
+			if lineStart {
+				if b == '/' {
+					// End-of-record marker "//"
+					s.pos += i + 1
+					if s.pos >= len(s.current.data) {
+						s.current = s.current.Next()
+						s.pos = 0
+					}
+					s.skipToNewline()
+					return dest
+				}
+				lineStart = false
+				skipDigits = true
+			}
+			switch {
+			case b == '\n':
+				lineStart = true
+			case b == '\r':
+				// skip
+			case skipDigits:
+				if b != ' ' && (b < '0' || b > '9') {
+					skipDigits = false
+					if UtoT && b == 'u' {
+						b = 't'
+					}
+					dest = append(dest, b)
+				}
+			case b != ' ':
+				if UtoT && b == 'u' {
+					b = 't'
+				}
+				dest = append(dest, b)
+			}
+		}
+		s.current = s.current.Next()
+		s.pos = 0
+	}
+	return dest
+}
+
+// parseLseqFromLocus extracts the declared sequence length from a LOCUS line.
+// Format: "LOCUS       <id> <length> bp ..."
+// Returns -1 if not found or parse error.
+func parseLseqFromLocus(line []byte) int {
+	if len(line) < 13 {
+		return -1
+	}
+	i := 12
+	for i < len(line) && line[i] != ' ' {
+		i++
+	}
+	for i < len(line) && line[i] == ' ' {
+		i++
+	}
+	start := i
+	for i < len(line) && line[i] >= '0' && line[i] <= '9' {
+		i++
+	}
+	if i == start {
+		return -1
+	}
+	n, err := strconv.Atoi(string(line[start:i]))
+	if err != nil {
+		return -1
+	}
+	return n
+}
+
+// Prefix constants for GenBank section headers (byte slices for zero-alloc comparison).
+var (
+	gbPfxLocus      = []byte("LOCUS       ")
+	gbPfxDefinition = []byte("DEFINITION  ")
+	gbPfxContinue   = []byte("            ")
+	gbPfxSource     = []byte("SOURCE      ")
+	gbPfxFeatures   = []byte("FEATURES    ")
+	gbPfxOrigin     = []byte("ORIGIN")
+	gbPfxContig     = []byte("CONTIG")
+	gbPfxEnd        = []byte("//")
+	gbPfxDbXref     = []byte(`                     /db_xref="taxon:`)
+)
+
+// GenbankChunkParserRope parses a GenBank FileChunk directly from the rope
+// (PieceOfChunk linked list) without calling Pack(). This eliminates the large
+// contiguous allocation required for chromosomal-scale sequences.
+func GenbankChunkParserRope(source string, rope *PieceOfChunk,
+	withFeatureTable, UtoT bool) (obiseq.BioSequenceSlice, error) {
+
+	state := inHeader
+	scanner := newRopeScanner(rope)
+	sequences := obiseq.MakeBioSequenceSlice(100)[:0]
+
+	id := ""
+	lseq := -1
+	scientificName := ""
+	defBytes := new(bytes.Buffer)
+	featBytes := new(bytes.Buffer)
+	var seqDest []byte
+	taxid := 1
+	nl := 0
+
+	for bline := scanner.ReadLine(); bline != nil; bline = scanner.ReadLine() {
+		nl++
+		processed := false
+		for !processed {
+			switch {
+
+			case bytes.HasPrefix(bline, gbPfxLocus):
+				if state != inHeader {
+					log.Fatalf("Line %d - Unexpected state %d while reading LOCUS: %s", nl, state, bline)
+				}
+				rest := bline[12:]
+				sp := bytes.IndexByte(rest, ' ')
+				if sp < 0 {
+					id = string(rest)
+				} else {
+					id = string(rest[:sp])
+				}
+				lseq = parseLseqFromLocus(bline)
+				cap0 := lseq + 20
+				if cap0 < 1024 {
+					cap0 = 1024
+				}
+				seqDest = make([]byte, 0, cap0)
+				state = inEntry
+				processed = true
+
+			case bytes.HasPrefix(bline, gbPfxDefinition):
+				if state != inEntry {
+					log.Fatalf("Line %d - Unexpected state %d while reading DEFINITION: %s", nl, state, bline)
+				}
+				defBytes.Write(bytes.TrimSpace(bline[12:]))
+				state = inDefinition
+				processed = true
+
+			case state == inDefinition:
+				if bytes.HasPrefix(bline, gbPfxContinue) {
+					defBytes.WriteByte(' ')
+					defBytes.Write(bytes.TrimSpace(bline[12:]))
+					processed = true
+				} else {
+					state = inEntry
+				}
+
+			case bytes.HasPrefix(bline, gbPfxSource):
+				if state != inEntry {
+					log.Fatalf("Line %d - Unexpected state %d while reading SOURCE: %s", nl, state, bline)
+				}
+				scientificName = string(bytes.TrimSpace(bline[12:]))
+				processed = true
+
+			case bytes.HasPrefix(bline, gbPfxFeatures):
+				if state != inEntry {
+					log.Fatalf("Line %d - Unexpected state %d while reading FEATURES: %s", nl, state, bline)
+				}
+				if withFeatureTable {
+					featBytes.Write(bline)
+				}
+				state = inFeature
+				processed = true
+
+			case bytes.HasPrefix(bline, gbPfxOrigin):
+				if state != inFeature && state != inContig {
+					log.Fatalf("Line %d - Unexpected state %d while reading ORIGIN: %s", nl, state, bline)
+				}
+				// Use fast byte-scan to extract sequence and consume through "//"
+				seqDest = scanner.extractSequence(seqDest, UtoT)
+				// Emit record
+				if id == "" {
+					log.Warn("Empty id when parsing genbank file")
+				}
+				sequence := obiseq.NewBioSequenceOwning(id, seqDest, defBytes.String())
+				sequence.SetSource(source)
+				if withFeatureTable {
+					sequence.SetFeatures(featBytes.Bytes())
+				}
+				annot := sequence.Annotations()
+				annot["scientific_name"] = scientificName
+				annot["taxid"] = taxid
+				sequences = append(sequences, sequence)
+
+				defBytes = bytes.NewBuffer(obiseq.GetSlice(200))
+				featBytes = new(bytes.Buffer)
+				nl = 0
+				taxid = 1
+				seqDest = nil
+				state = inHeader
+				processed = true
+
+			case bytes.HasPrefix(bline, gbPfxContig):
+				if state != inFeature && state != inContig {
+					log.Fatalf("Line %d - Unexpected state %d while reading CONTIG: %s", nl, state, bline)
+				}
+				state = inContig
+				processed = true
+
+			case bytes.Equal(bline, gbPfxEnd):
+				// Reached for CONTIG records (no ORIGIN section)
+				if state != inContig {
+					log.Fatalf("Line %d - Unexpected state %d while reading end of record %s", nl, state, id)
+				}
+				if id == "" {
+					log.Warn("Empty id when parsing genbank file")
+				}
+				sequence := obiseq.NewBioSequenceOwning(id, seqDest, defBytes.String())
+				sequence.SetSource(source)
+				if withFeatureTable {
+					sequence.SetFeatures(featBytes.Bytes())
+				}
+				annot := sequence.Annotations()
+				annot["scientific_name"] = scientificName
+				annot["taxid"] = taxid
+				sequences = append(sequences, sequence)
+
+				defBytes = bytes.NewBuffer(obiseq.GetSlice(200))
+				featBytes = new(bytes.Buffer)
+				nl = 0
+				taxid = 1
+				seqDest = nil
+				state = inHeader
+				processed = true
+
+			default:
+				switch state {
+				case inFeature:
+					if withFeatureTable {
+						featBytes.WriteByte('\n')
+						featBytes.Write(bline)
+					}
+					if bytes.HasPrefix(bline, gbPfxDbXref) {
+						rest := bline[len(gbPfxDbXref):]
+						q := bytes.IndexByte(rest, '"')
+						if q >= 0 {
+							taxid, _ = strconv.Atoi(string(rest[:q]))
+						}
+					}
+					processed = true
+				case inHeader, inEntry, inContig:
+					processed = true
+				default:
+					log.Fatalf("Unexpected state %d while reading: %s", state, bline)
+				}
+			}
+		}
+	}
+
+	return sequences, nil
+}
+
 func GenbankChunkParser(withFeatureTable, UtoT bool) func(string, io.Reader) (obiseq.BioSequenceSlice, error) {
 	return func(source string, input io.Reader) (obiseq.BioSequenceSlice, error) {
 		state := inHeader
@@ -125,13 +384,10 @@ func GenbankChunkParser(withFeatureTable, UtoT bool) func(string, io.Reader) (ob
 					if state != inSequence && state != inContig {
 						log.Fatalf("Line %d - Unexpected state %d while reading end of record %s", nl, state, id)
 					}
-					// log.Debugln("Total lines := ", nl)
 					if id == "" {
 						log.Warn("Empty id when parsing genbank file")
 					}
 
-					// log.Debugf("End of sequence %s: %dbp ", id, seqBytes.Len())
-
 					sequence := obiseq.NewBioSequence(id,
 						seqBytes.Bytes(),
 						defBytes.String())
@@ -144,9 +400,6 @@ func GenbankChunkParser(withFeatureTable, UtoT bool) func(string, io.Reader) (ob
 					annot := sequence.Annotations()
 					annot["scientific_name"] = scientificName
 					annot["taxid"] = taxid
-					// log.Println(FormatFasta(sequence, FormatFastSeqJsonHeader))
-					// log.Debugf("Read sequences %s: %dbp (%d)", sequence.Id(),
-					//	sequence.Len(), seqBytes.Len())
 
 					sequences = append(sequences, sequence)
 
@@ -159,8 +412,6 @@ func GenbankChunkParser(withFeatureTable, UtoT bool) func(string, io.Reader) (ob
 					processed = true
 
 				case state == inSequence:
-					// log.Debugf("Chunk %d : Genbank: line %d, state = %d : %s", chunks.order, nl, state, line)
-
 					sl++
 					cleanline := strings.TrimSpace(line)
 					parts := strings.SplitN(cleanline, " ", 7)
@@ -198,6 +449,7 @@ func GenbankChunkParser(withFeatureTable, UtoT bool) func(string, io.Reader) (ob
 
 		}
 
+		_ = sl
 		return sequences, nil
 	}
 }
@@ -206,10 +458,16 @@ func _ParseGenbankFile(input ChannelFileChunk,
 	out obiiter.IBioSequence,
 	withFeatureTable, UtoT bool) {
 
-	parser := GenbankChunkParser(withFeatureTable, UtoT)
-
 	for chunks := range input {
-		sequences, err := parser(chunks.Source, chunks.Raw)
+		var sequences obiseq.BioSequenceSlice
+		var err error
+
+		if chunks.Rope != nil {
+			sequences, err = GenbankChunkParserRope(chunks.Source, chunks.Rope, withFeatureTable, UtoT)
+		} else {
+			parser := GenbankChunkParser(withFeatureTable, UtoT)
+			sequences, err = parser(chunks.Source, chunks.Raw)
+		}
 
 		if err != nil {
 			log.Fatalf("File %s : Cannot parse the genbank file : %v", chunks.Source, err)
@@ -225,7 +483,6 @@ func _ParseGenbankFile(input ChannelFileChunk,
 
 func ReadGenbank(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, error) {
 	opt := MakeOptions(options)
-	// entry_channel := make(chan _FileChunk)
 
 	entry_channel := ReadFileChunk(
 		opt.Source(),
@@ -233,13 +490,13 @@ func ReadGenbank(reader io.Reader, options ...WithOption) (obiiter.IBioSequence,
 		1024*1024*128,
 		EndOfLastFlatFileEntry,
 		"\nLOCUS       ",
+		false, // do not pack: rope-based parser avoids contiguous allocation
 	)
 
 	newIter := obiiter.MakeIBioSequence()
 
 	nworkers := opt.ParallelWorkers()
 
-	// for j := 0; j < opt.ParallelWorkers(); j++ {
 	for j := 0; j < nworkers; j++ {
 		newIter.Add(1)
 		go _ParseGenbankFile(
@@ -250,8 +507,6 @@ func ReadGenbank(reader io.Reader, options ...WithOption) (obiiter.IBioSequence,
 		)
 	}
 
-	// go _ReadFlatFileChunk(reader, entry_channel)
-
 	go func() {
 		newIter.WaitAndClose()
 		log.Debug("End of the genbank file ", opt.Source())

pkg/obiformats/json_read.go

@@ -0,0 +1,147 @@
+package obiformats
+
+import (
+	"io"
+	"os"
+	"path"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+	"github.com/buger/jsonparser"
+	"github.com/goccy/go-json"
+	log "github.com/sirupsen/logrus"
+)
+
+// _parse_json_record parses a single JSON object describing a sequence
+// (as produced by JSONRecord in json_writer.go) into a *obiseq.BioSequence.
+func _parse_json_record(raw []byte, shift byte) *obiseq.BioSequence {
+	sequence := obiseq.NewEmptyBioSequence(0)
+
+	if id, err := jsonparser.GetString(raw, "id"); err == nil {
+		sequence.SetId(id)
+	}
+
+	if seq, err := jsonparser.GetString(raw, "sequence"); err == nil {
+		sequence.SetSequence([]byte(seq))
+	}
+
+	if qual, err := jsonparser.GetString(raw, "qualities"); err == nil {
+		q := []byte(qual)
+		for i := 0; i < len(q); i++ {
+			q[i] -= shift
+		}
+		sequence.SetQualities(q)
+	}
+
+	if annot, dataType, _, err := jsonparser.Get(raw, "annotations"); err == nil && dataType == jsonparser.Object {
+		jsonparser.ObjectEach(annot,
+			func(key []byte, value []byte, valType jsonparser.ValueType, offset int) error {
+				return _parse_json_annotation_field(key, value, valType, sequence)
+			},
+		)
+	}
+
+	return sequence
+}
+
+// _ParseJsonFile streams the top-level JSON array, decoding and pushing one
+// batch of sequences at a time, without ever loading the whole document in
+// memory. Only one raw record at a time is buffered by the decoder.
+func _ParseJsonFile(source string,
+	reader io.Reader,
+	out obiiter.IBioSequence,
+	shift byte,
+	batchSize int) {
+
+	dec := json.NewDecoder(reader)
+
+	if _, err := dec.Token(); err != nil {
+		if err == io.EOF {
+			out.Done()
+			return
+		}
+		log.Fatalf("cannot parse JSON data: %v", err)
+	}
+
+	slice := obiseq.MakeBioSequenceSlice()
+	o := 0
+
+	for dec.More() {
+		var raw json.RawMessage
+
+		if err := dec.Decode(&raw); err != nil {
+			log.Fatalf("cannot parse JSON data: %v", err)
+		}
+
+		sequence := _parse_json_record(raw, shift)
+
+		slice = append(slice, sequence)
+		if len(slice) >= batchSize {
+			out.Push(obiiter.MakeBioSequenceBatch(source, o, slice))
+			o++
+			slice = obiseq.MakeBioSequenceSlice()
+		}
+	}
+
+	if len(slice) > 0 {
+		out.Push(obiiter.MakeBioSequenceBatch(source, o, slice))
+	}
+
+	out.Done()
+}
+
+func ReadJSON(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, error) {
+
+	opt := MakeOptions(options)
+	out := obiiter.MakeIBioSequence()
+
+	out.Add(1)
+	go _ParseJsonFile(opt.Source(),
+		reader,
+		out,
+		obidefault.ReadQualitiesShift(),
+		opt.BatchSize())
+
+	go func() {
+		out.WaitAndClose()
+	}()
+
+	return out, nil
+
+}
+
+func ReadJSONFromFile(filename string, options ...WithOption) (obiiter.IBioSequence, error) {
+
+	options = append(options, OptionsSource(obiutils.RemoveAllExt((path.Base(filename)))))
+	file, err := obiutils.Ropen(filename)
+
+	if err == obiutils.ErrNoContent {
+		log.Infof("file %s is empty", filename)
+		return ReadEmptyFile(options...)
+	}
+
+	if err != nil {
+		return obiiter.NilIBioSequence, err
+	}
+
+	return ReadJSON(file, options...)
+}
+
+func ReadJSONFromStdin(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, error) {
+	options = append(options, OptionsSource(obiutils.RemoveAllExt("stdin")))
+	input, err := obiutils.Buf(os.Stdin)
+
+	if err == obiutils.ErrNoContent {
+		log.Infof("stdin is empty")
+		return ReadEmptyFile(options...)
+	}
+
+	if err != nil {
+		log.Fatalf("open file error: %v", err)
+		return obiiter.NilIBioSequence, err
+	}
+
+	return ReadJSON(input, options...)
+}

pkg/obiformats/ngsfilter_read.go

@@ -631,9 +631,9 @@ func ReadCSVNGSFilter(reader io.Reader) (*obingslibrary.NGSLibrary, error) {
 			return nil, fmt.Errorf("row %d has %d columns, expected %d", len(data), len(fields), len(header))
 		}
 
-		forward_primer := fields[forward_primerColIndex]
-		reverse_primer := fields[reverse_primerColIndex]
-		tags := _parseMainNGSFilterTags(fields[sample_tagColIndex])
+		forward_primer := strings.TrimSpace(fields[forward_primerColIndex])
+		reverse_primer := strings.TrimSpace(fields[reverse_primerColIndex])
+		tags := _parseMainNGSFilterTags(strings.TrimSpace(fields[sample_tagColIndex]))
 
 		marker, _ := ngsfilter.GetMarker(forward_primer, reverse_primer)
 		pcr, ok := marker.GetPCR(tags.Forward, tags.Reverse)
@@ -644,8 +644,8 @@ func ReadCSVNGSFilter(reader io.Reader) (*obingslibrary.NGSLibrary, error) {
 					i, tags.Forward, tags.Reverse, forward_primer, reverse_primer)
 		}
 
-		pcr.Experiment = fields[experimentColIndex]
-		pcr.Sample = fields[sampleColIndex]
+		pcr.Experiment = strings.TrimSpace(fields[experimentColIndex])
+		pcr.Sample = strings.TrimSpace(fields[sampleColIndex])
 
 		if extraColumns != nil {
 			pcr.Annotations = make(obiseq.Annotation)

pkg/obiformats/options.go

@@ -35,6 +35,7 @@ type __options__ struct {
 	csv_auto              bool
 	paired_filename       string
 	source                string
+	filename              string
 	with_feature_table    bool
 	with_pattern          bool
 	with_parent           bool
@@ -216,6 +217,16 @@ func (opt Options) Source() string {
 	return opt.pointer.source
 }
 
+// FileName returns the full path of the file being read, for use in
+// diagnostic messages. It falls back to Source() when no explicit
+// file name has been set (e.g. reading from stdin or a raw reader).
+func (opt Options) FileName() string {
+	if opt.pointer.filename == "" {
+		return opt.pointer.source
+	}
+	return opt.pointer.filename
+}
+
 func (opt Options) WithFeatureTable() bool {
 	return opt.pointer.with_feature_table
 }
@@ -421,6 +432,14 @@ func OptionsSource(source string) WithOption {
 	return f
 }
 
+func OptionsFileName(filename string) WithOption {
+	f := WithOption(func(opt Options) {
+		opt.pointer.filename = filename
+	})
+
+	return f
+}
+
 func OptionsWithProgressBar() WithOption {
 	f := WithOption(func(opt Options) {
 		opt.pointer.with_progress_bar = true

pkg/obiformats/rope_scanner.go

@@ -0,0 +1,77 @@
+package obiformats
+
+import "bytes"
+
+// ropeScanner reads lines from a PieceOfChunk rope.
+// The carry buffer handles lines that span two rope nodes; it grows as needed.
+type ropeScanner struct {
+	current *PieceOfChunk
+	pos     int
+	carry   []byte
+}
+
+func newRopeScanner(rope *PieceOfChunk) *ropeScanner {
+	return &ropeScanner{current: rope}
+}
+
+// ReadLine returns the next line without the trailing \n (or \r\n).
+// Returns nil at end of rope. The returned slice aliases carry[] or the node
+// data and is valid only until the next ReadLine call.
+func (s *ropeScanner) ReadLine() []byte {
+	for {
+		if s.current == nil {
+			if len(s.carry) > 0 {
+				line := s.carry
+				s.carry = s.carry[:0]
+				return line
+			}
+			return nil
+		}
+
+		data := s.current.data[s.pos:]
+		idx := bytes.IndexByte(data, '\n')
+
+		if idx >= 0 {
+			var line []byte
+			if len(s.carry) == 0 {
+				line = data[:idx]
+			} else {
+				s.carry = append(s.carry, data[:idx]...)
+				line = s.carry
+				s.carry = s.carry[:0]
+			}
+			s.pos += idx + 1
+			if s.pos >= len(s.current.data) {
+				s.current = s.current.Next()
+				s.pos = 0
+			}
+			if len(line) > 0 && line[len(line)-1] == '\r' {
+				line = line[:len(line)-1]
+			}
+			return line
+		}
+
+		// No \n in this node: accumulate into carry and advance
+		s.carry = append(s.carry, data...)
+		s.current = s.current.Next()
+		s.pos = 0
+	}
+}
+
+// skipToNewline advances the scanner past the next '\n'.
+func (s *ropeScanner) skipToNewline() {
+	for s.current != nil {
+		data := s.current.data[s.pos:]
+		idx := bytes.IndexByte(data, '\n')
+		if idx >= 0 {
+			s.pos += idx + 1
+			if s.pos >= len(s.current.data) {
+				s.current = s.current.Next()
+				s.pos = 0
+			}
+			return
+		}
+		s.current = s.current.Next()
+		s.pos = 0
+	}
+}

pkg/obiformats/universal_read.go

@@ -145,6 +145,8 @@ func ReadSequencesFromFile(filename string,
 		return ReadGenbank(reader, options...)
 	case "text/csv":
 		return ReadCSV(reader, options...)
+	case "application/json":
+		return ReadJSON(reader, options...)
 	default:
 		log.Fatalf("File %s has guessed format %s which is not yet implemented",
 			filename, mime.String())

pkg/obifp/uint128_test.go

@@ -134,7 +134,7 @@ func TestUint128_QuoRem(t *testing.T) {
 	u := Uint128{w1: 3, w0: 8}
 	v := Uint128{w1: 0, w0: 4}
 	q, r := u.QuoRem(v)
-	assert.Equal(t, Uint128{w1: 0, w0: 2}, q)
+	assert.Equal(t, Uint128{w1: 0, w0: 13835058055282163714}, q)
 	assert.Equal(t, Uint128{w1: 0, w0: 0}, r)
 }
 
@@ -150,7 +150,7 @@ func TestUint128_Div(t *testing.T) {
 	u := Uint128{w1: 3, w0: 8}
 	v := Uint128{w1: 0, w0: 4}
 	q := u.Div(v)
-	assert.Equal(t, Uint128{w1: 0, w0: 2}, q)
+	assert.Equal(t, Uint128{w1: 0, w0: 13835058055282163714}, q)
 }
 
 func TestUint128_Div64(t *testing.T) {
@@ -183,7 +183,7 @@ func TestUint128_Cmp(t *testing.T) {
 func TestUint128_Cmp64(t *testing.T) {
 	u := Uint128{w1: 1, w0: 2}
 	v := uint64(3)
-	assert.Equal(t, -1, u.Cmp64(v))
+	assert.Equal(t, 1, u.Cmp64(v))
 }
 
 func TestUint128_Equals(t *testing.T) {

pkg/obigraph/clique.go

@@ -0,0 +1,343 @@
+package obigraph
+
+import (
+	"context"
+	"math/rand"
+	"sync"
+	"sync/atomic"
+)
+
+// NeighborFilter defines an implicit graph over the vertices 0..n-1.
+// It appends to out[:0] the vertices of candidates that are adjacent to v,
+// keeping their order, and returns the resulting slice. It is called
+// concurrently by several workers and must not modify candidates.
+//
+// Describing the adjacency by a filter, instead of a matrix, keeps the memory
+// in O(n) and lets the caller evaluate adjacency in bulk (SIMD, for instance).
+type NeighborFilter func(v int32, candidates, out []int32) []int32
+
+// CliqueOptions tunes FindClique.
+type CliqueOptions struct {
+	// Workers is the number of concurrent searches (default 1).
+	Workers int
+	// Seed seeds the random vertex orders of the restarts.
+	Seed int64
+	// Progress, if set, is called with the size of the best clique each time a
+	// larger one is found. Calls are serialized and made while a lock is held:
+	// it must return quickly and must not call FindClique.
+	Progress func(best int)
+
+	// ColorLimit enables the coloring bound for the candidate lists of at most
+	// ColorLimit vertices (0 disables it). The vertices of a list are split into
+	// classes of pairwise non-adjacent vertices: a clique takes at most one
+	// vertex per class, so the number of classes bounds its size. Computing the
+	// classes costs up to the square of the list length in calls to the filter,
+	// which is why only short lists are colored.
+	ColorLimit int
+
+	// mode forces the kind of search (tests only): 0 alternates, modeFocused
+	// and modeImproving restrict the search to one of them.
+	mode int
+}
+
+const (
+	modeFocused   = 1
+	modeImproving = 2
+)
+
+// CliqueResult is the outcome of FindClique.
+type CliqueResult struct {
+	// Clique is the largest clique found (a clique of size >= target if Reached).
+	Clique []int32
+	// Reached is true if len(Clique) >= target.
+	Reached bool
+	// Exhausted is true if the whole search space has been explored. If Reached
+	// is false, it proves that no clique of the target size exists.
+	Exhausted bool
+	// Maximum is true if Clique is proven to be a maximum clique of the graph:
+	// no larger clique exists.
+	Maximum bool
+}
+
+// FindClique looks for a clique of at least target vertices in the graph of n
+// vertices described by filter. It stops as soon as one is found, when the
+// search space is exhausted, or when ctx is done; in the last case, the largest
+// clique met so far is returned.
+//
+// The search is a depth first search with backtracking, which comes in two
+// flavours that the restarts alternate:
+//
+//   - the focused search prunes every branch that cannot reach target, which
+//     is the most efficient way to reach it but gives poor cliques when target
+//     is out of reach;
+//   - the improving search only prunes the branches that cannot beat the best
+//     clique found so far (by any worker), so it keeps improving that clique
+//     whatever target is. When it ends, the best clique is a maximum one.
+//
+// When a focused search proves that no clique of target vertices exists, every
+// search turns to the improving flavour, so that the largest clique is returned
+// (and proved to be so) instead of an arbitrary one.
+//
+// On short candidate lists (see CliqueOptions.ColorLimit), a coloring bound
+// prunes the branches that cannot be large enough, and orders the vertices so
+// that the most constrained ones are tried first.
+//
+// The first pass of the first worker follows the natural order of the vertices
+// (greedy "lexicographic" construction); every other pass restarts from a
+// random order, with a work budget that doubles at each restart, so that the
+// search is complete in the long run.
+func FindClique(ctx context.Context, n int, filter NeighborFilter, target int, opts CliqueOptions) CliqueResult {
+	if target <= 0 {
+		return CliqueResult{Reached: true, Exhausted: true}
+	}
+	if n <= 0 {
+		return CliqueResult{Exhausted: true}
+	}
+
+	workers := max(opts.Workers, 1)
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	var (
+		mutex     sync.Mutex
+		best      []int32
+		bestSize  atomic.Int32
+		exhausted bool
+		maximum   bool
+		wg        sync.WaitGroup
+		// set when a focused search has proved that target cannot be reached:
+		// every search then turns to finding the largest clique
+		unreachable atomic.Bool
+	)
+
+	publish := func(clique []int32) {
+		mutex.Lock()
+		if len(clique) > len(best) {
+			best = append(best[:0], clique...)
+			bestSize.Store(int32(len(best)))
+			if opts.Progress != nil {
+				opts.Progress(len(best))
+			}
+		}
+		mutex.Unlock()
+	}
+
+	for w := range workers {
+		wg.Go(func() {
+			s := &cliqueSearch{
+				ctx:         ctx,
+				filter:      filter,
+				target:      target,
+				bestSize:    &bestSize,
+				colorLimit:  opts.ColorLimit,
+				unreachable: &unreachable,
+				publish: func(c []int32) {
+					publish(c)
+					if len(c) >= target {
+						cancel()
+					}
+				},
+			}
+			rng := rand.New(rand.NewSource(opts.Seed + int64(w)))
+			// the search starts from the end of the list: lowest vertex first
+			candidates := make([]int32, n)
+			for i := range candidates {
+				candidates[i] = int32(n - 1 - i)
+			}
+
+			for restart := 0; ctx.Err() == nil; restart++ {
+				if w > 0 || restart > 0 {
+					rng.Shuffle(n, func(i, j int) { candidates[i], candidates[j] = candidates[j], candidates[i] })
+				}
+				switch opts.mode {
+				case modeFocused:
+					s.improving = false
+				case modeImproving:
+					s.improving = true
+				default:
+					s.improving = unreachable.Load() || (w+restart)%2 == 1
+				}
+				s.budget = int64(n) * int64(target) << min(restart, 20)
+				s.work = 0
+				s.aborted = false
+				s.search(0, candidates)
+
+				if !s.aborted && ctx.Err() == nil {
+					mutex.Lock()
+					exhausted = true
+					if s.improving {
+						maximum = true
+					}
+					mutex.Unlock()
+
+					if s.improving || opts.mode == modeFocused {
+						cancel()
+					} else {
+						// target is out of reach: look for the largest clique
+						unreachable.Store(true)
+					}
+				}
+			}
+		})
+	}
+	wg.Wait()
+
+	return CliqueResult{
+		Clique:    best,
+		Reached:   len(best) >= target,
+		Exhausted: exhausted,
+		Maximum:   maximum,
+	}
+}
+
+type cliqueSearch struct {
+	ctx        context.Context
+	filter     NeighborFilter
+	target     int
+	bestSize   *atomic.Int32 // size of the best clique found by any worker
+	publish    func([]int32)
+	colorLimit int
+
+	unreachable *atomic.Bool // target is proved out of reach
+
+	improving bool // see FindClique: the kind of the current pass
+
+	current []int32   // the clique under construction
+	buffers [][]int32 // candidate list of each depth
+	sorted  [][]int32 // candidate list of each depth, sorted by color
+	colors  [][]int32 // color (from 1) of each vertex of sorted
+	classes [][]int32 // scratch space of the coloring
+	probe   []int32   // scratch space of the coloring
+
+	budget  int64 // maximum work (number of candidates tested) for this restart
+	work    int64
+	aborted bool // budget or context exhausted before the end of the exploration
+}
+
+// threshold is the size a clique must reach to be worth exploring: the target
+// for the focused search, one more than the best clique for the improving one.
+func (s *cliqueSearch) threshold() int {
+	if s.improving {
+		return min(s.target, int(s.bestSize.Load())+1)
+	}
+	return s.target
+}
+
+// mustStop tells whether the current pass has to be interrupted: its work
+// budget is spent, or the pass must stop whatever its budget (see mustAbort).
+func (s *cliqueSearch) mustStop() bool {
+	return s.work > s.budget || s.mustAbort()
+}
+
+// mustAbort tells whether the pass has to be interrupted whatever its budget:
+// the context is done, or it is a focused pass and its target has been proved
+// out of reach.
+func (s *cliqueSearch) mustAbort() bool {
+	return s.ctx.Err() != nil || (!s.improving && s.unreachable.Load())
+}
+
+// colorSort splits candidates into classes of pairwise non-adjacent vertices
+// (greedy sequential coloring) and returns them class after class, with the
+// color of each vertex. Any clique takes at most one vertex of each class.
+//
+// The coloring can be long: it stops, and sets s.aborted, as soon as the search
+// has to be aborted. Its cost is not charged to the work budget of the pass,
+// which limits the exploration of the tree, not the bound computed at each node.
+func (s *cliqueSearch) colorSort(depth int, candidates []int32) (sorted, colors []int32) {
+	k := 0
+	for _, v := range candidates {
+		if s.mustAbort() {
+			s.aborted = true
+			return nil, nil
+		}
+		placed := false
+		for c := 0; c < k; c++ {
+			// v joins the class if it is adjacent to none of its members
+			s.probe = s.filter(v, s.classes[c], s.probe[:0])
+			if len(s.probe) == 0 {
+				s.classes[c] = append(s.classes[c], v)
+				placed = true
+				break
+			}
+		}
+		if !placed {
+			if k == len(s.classes) {
+				s.classes = append(s.classes, nil)
+			}
+			s.classes[k] = append(s.classes[k][:0], v)
+			k++
+		}
+	}
+
+	for len(s.sorted) <= depth {
+		s.sorted = append(s.sorted, nil)
+		s.colors = append(s.colors, nil)
+	}
+	sorted, colors = s.sorted[depth][:0], s.colors[depth][:0]
+	for c := 0; c < k; c++ {
+		for _, v := range s.classes[c] {
+			sorted = append(sorted, v)
+			colors = append(colors, int32(c+1))
+		}
+	}
+	s.sorted[depth], s.colors[depth] = sorted, colors
+	return sorted, colors
+}
+
+// search extends the current clique of depth vertices with vertices taken in
+// candidates, all adjacent to the vertices of the current clique. The list is
+// read from its end: the vertex at position i is combined with the vertices
+// before it, so that every clique is met once.
+func (s *cliqueSearch) search(depth int, candidates []int32) {
+	var colors []int32
+	if s.colorLimit > 0 && len(candidates) > 1 && len(candidates) <= s.colorLimit {
+		candidates, colors = s.colorSort(depth, candidates)
+		if s.aborted {
+			return
+		}
+	}
+
+	for i := len(candidates) - 1; i >= 0; i-- {
+		// largest clique the vertices 0..i can make
+		bound := i + 1
+		if colors != nil {
+			bound = int(colors[i])
+		}
+		if depth+bound < s.threshold() {
+			return
+		}
+		if s.mustStop() {
+			s.aborted = true
+			return
+		}
+
+		v := candidates[i]
+		s.current = append(s.current[:depth], v)
+		if len(s.current) > int(s.bestSize.Load()) {
+			s.publish(s.current)
+			if len(s.current) >= s.target {
+				s.aborted = true // the search is over: do not report an exhaustion
+				return
+			}
+		}
+
+		rest := candidates[:i]
+		if depth+1+len(rest) < s.threshold() {
+			continue
+		}
+		for len(s.buffers) <= depth+1 {
+			s.buffers = append(s.buffers, nil)
+		}
+		next := s.filter(v, rest, s.buffers[depth+1][:0])
+		s.buffers[depth+1] = next
+		s.work += int64(len(rest))
+
+		if depth+1+len(next) < s.threshold() {
+			continue
+		}
+		s.search(depth+1, next)
+		if s.aborted {
+			return
+		}
+	}
+}

pkg/obigraph/clique_test.go

@@ -0,0 +1,249 @@
+package obigraph
+
+import (
+	"context"
+	"math/bits"
+	"math/rand"
+	"testing"
+	"time"
+)
+
+// adjacency matrix filter for small graphs
+func matrixFilter(adj [][]bool) NeighborFilter {
+	return func(v int32, candidates, out []int32) []int32 {
+		for _, c := range candidates {
+			if adj[v][c] {
+				out = append(out, c)
+			}
+		}
+		return out
+	}
+}
+
+func randomGraph(rng *rand.Rand, n int, p float64) [][]bool {
+	adj := make([][]bool, n)
+	for i := range adj {
+		adj[i] = make([]bool, n)
+	}
+	for i := 0; i < n; i++ {
+		for j := i + 1; j < n; j++ {
+			if rng.Float64() < p {
+				adj[i][j], adj[j][i] = true, true
+			}
+		}
+	}
+	return adj
+}
+
+// maximum clique size by brute force over all the subsets (n <= 20)
+func bruteMaxClique(adj [][]bool) int {
+	n := len(adj)
+	best := 0
+	for mask := 1; mask < 1<<n; mask++ {
+		size := bits.OnesCount(uint(mask))
+		if size <= best {
+			continue
+		}
+		ok := true
+		for i := 0; i < n && ok; i++ {
+			if mask>>i&1 == 0 {
+				continue
+			}
+			for j := i + 1; j < n; j++ {
+				if mask>>j&1 == 1 && !adj[i][j] {
+					ok = false
+					break
+				}
+			}
+		}
+		if ok {
+			best = size
+		}
+	}
+	return best
+}
+
+func isClique(adj [][]bool, c []int32) bool {
+	seen := map[int32]bool{}
+	for i, a := range c {
+		if seen[a] {
+			return false
+		}
+		seen[a] = true
+		for _, b := range c[i+1:] {
+			if !adj[a][b] {
+				return false
+			}
+		}
+	}
+	return true
+}
+
+func TestFindCliqueAgainstBruteForce(t *testing.T) {
+	rng := rand.New(rand.NewSource(1))
+	for trial := 0; trial < 40; trial++ {
+		n := 8 + rng.Intn(10)
+		adj := randomGraph(rng, n, 0.3+0.5*rng.Float64())
+		maxSize := bruteMaxClique(adj)
+		filter := matrixFilter(adj)
+
+		for _, workers := range []int{1, 3} {
+			r := FindClique(context.Background(), n, filter, maxSize, CliqueOptions{Workers: workers})
+			if !r.Reached || len(r.Clique) < maxSize || !isClique(adj, r.Clique) {
+				t.Fatalf("trial %d: target %d not reached: %+v", trial, maxSize, r)
+			}
+
+			r = FindClique(context.Background(), n, filter, maxSize+1, CliqueOptions{Workers: workers})
+			if r.Reached || !r.Exhausted {
+				t.Fatalf("trial %d: size %d cannot exist, got %+v", trial, maxSize+1, r)
+			}
+			// best effort: a valid clique, not necessarily a maximum one
+			if len(r.Clique) < 1 || len(r.Clique) > maxSize || !isClique(adj, r.Clique) {
+				t.Fatalf("trial %d: invalid best clique %v (maximum is %d)", trial, r.Clique, maxSize)
+			}
+		}
+	}
+}
+
+func TestFindCliqueTrivialTargets(t *testing.T) {
+	adj := randomGraph(rand.New(rand.NewSource(2)), 5, 0.5)
+	if r := FindClique(context.Background(), 5, matrixFilter(adj), 0, CliqueOptions{}); !r.Reached {
+		t.Fatal("target 0 is always reached")
+	}
+	if r := FindClique(context.Background(), 0, matrixFilter(adj), 1, CliqueOptions{}); r.Reached {
+		t.Fatal("no clique in an empty graph")
+	}
+	if r := FindClique(context.Background(), 5, matrixFilter(adj), 1, CliqueOptions{}); !r.Reached || len(r.Clique) != 1 {
+		t.Fatalf("a single vertex is a clique: %+v", r)
+	}
+}
+
+// A hopeless search must stop with the deadline and return a valid clique.
+// (When the target is far above the clique number of a sparse graph, pruning
+// proves the absence at once: see TestFindCliqueAgainstBruteForce.)
+func TestFindCliqueTimeout(t *testing.T) {
+	// dense graph: its largest clique is far below the target, but pruning only
+	// bites after many levels, so the search cannot finish within the deadline
+	const n = 300
+	adj := randomGraph(rand.New(rand.NewSource(3)), n, 0.9)
+	ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond)
+	defer cancel()
+
+	start := time.Now()
+	r := FindClique(ctx, n, matrixFilter(adj), 150, CliqueOptions{Workers: 2})
+	if elapsed := time.Since(start); elapsed > 3*time.Second {
+		t.Fatalf("deadline not honoured: %v", elapsed)
+	}
+	if r.Reached || r.Exhausted {
+		t.Fatalf("size 150 cannot be reached nor proven absent in time: %+v", r)
+	}
+	if len(r.Clique) < 2 || !isClique(adj, r.Clique) {
+		t.Fatalf("invalid best clique %v", r.Clique)
+	}
+}
+
+func TestFindCliqueProgress(t *testing.T) {
+	adj := randomGraph(rand.New(rand.NewSource(4)), 40, 0.7)
+
+	var seen []int
+	r := FindClique(context.Background(), 40, matrixFilter(adj), 5, CliqueOptions{
+		Progress: func(best int) { seen = append(seen, best) },
+	})
+	if !r.Reached {
+		t.Fatalf("size 5 should be reached: %+v", r)
+	}
+	if len(seen) == 0 || seen[len(seen)-1] != len(r.Clique) {
+		t.Fatalf("progress %v does not end at the clique size %d", seen, len(r.Clique))
+	}
+	for i := 1; i < len(seen); i++ {
+		if seen[i] <= seen[i-1] {
+			t.Fatalf("progress must be strictly increasing: %v", seen)
+		}
+	}
+}
+
+// The improving search must end with a proven maximum clique, even when the
+// target is out of reach; the focused one only proves the absence of a clique
+// of the target size.
+func TestFindCliqueModes(t *testing.T) {
+	rng := rand.New(rand.NewSource(5))
+	for trial := 0; trial < 40; trial++ {
+		n := 8 + rng.Intn(10)
+		adj := randomGraph(rng, n, 0.3+0.5*rng.Float64())
+		maxSize := bruteMaxClique(adj)
+		filter := matrixFilter(adj)
+
+		for _, workers := range []int{1, 3} {
+			for _, limit := range []int{0, 5, 1 << 20} {
+				r := FindClique(context.Background(), n, filter, maxSize+1,
+					CliqueOptions{Workers: workers, ColorLimit: limit, mode: modeImproving})
+				if r.Reached || !r.Exhausted || !r.Maximum {
+					t.Fatalf("trial %d limit %d: improving search should prove the maximum: %+v", trial, limit, r)
+				}
+				if len(r.Clique) != maxSize || !isClique(adj, r.Clique) {
+					t.Fatalf("trial %d limit %d: expected a maximum clique of %d, got %v", trial, limit, maxSize, r.Clique)
+				}
+
+				r = FindClique(context.Background(), n, filter, maxSize+1,
+					CliqueOptions{Workers: workers, ColorLimit: limit, mode: modeFocused})
+				if r.Reached || !r.Exhausted || r.Maximum {
+					t.Fatalf("trial %d limit %d: focused search proves absence only: %+v", trial, limit, r)
+				}
+
+				r = FindClique(context.Background(), n, filter, maxSize,
+					CliqueOptions{Workers: workers, ColorLimit: limit, mode: modeImproving})
+				if !r.Reached || len(r.Clique) < maxSize || !isClique(adj, r.Clique) {
+					t.Fatalf("trial %d limit %d: target %d not reached by the improving search: %+v", trial, limit, maxSize, r)
+				}
+			}
+		}
+	}
+}
+
+// When the target is out of reach, the default search must end with a proven
+// maximum clique, even if the coloring bound proves the absence of the target
+// before any clique has been found.
+func TestFindCliqueUnreachableTargetGivesMaximum(t *testing.T) {
+	rng := rand.New(rand.NewSource(6))
+	for trial := 0; trial < 40; trial++ {
+		n := 8 + rng.Intn(10)
+		adj := randomGraph(rng, n, 0.3+0.5*rng.Float64())
+		maxSize := bruteMaxClique(adj)
+
+		for _, workers := range []int{1, 3} {
+			for _, limit := range []int{0, 5, 1 << 20} {
+				r := FindClique(context.Background(), n, matrixFilter(adj), maxSize+1,
+					CliqueOptions{Workers: workers, ColorLimit: limit})
+				if r.Reached || !r.Exhausted || !r.Maximum {
+					t.Fatalf("trial %d workers %d limit %d: expected a proven maximum: %+v", trial, workers, limit, r)
+				}
+				if len(r.Clique) != maxSize || !isClique(adj, r.Clique) {
+					t.Fatalf("trial %d workers %d limit %d: expected a maximum clique of %d, got %v",
+						trial, workers, limit, maxSize, r.Clique)
+				}
+			}
+		}
+	}
+}
+
+// A long coloring must not delay the deadline.
+func TestFindCliqueColoringHonoursDeadline(t *testing.T) {
+	const n = 3000
+	adj := randomGraph(rand.New(rand.NewSource(7)), n, 0.5)
+	base := matrixFilter(adj)
+	slow := func(v int32, candidates, out []int32) []int32 {
+		time.Sleep(200 * time.Microsecond)
+		return base(v, candidates, out)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond)
+	defer cancel()
+	start := time.Now()
+	r := FindClique(ctx, n, slow, 100, CliqueOptions{Workers: 2, ColorLimit: 1 << 20})
+	if elapsed := time.Since(start); elapsed > 2*time.Second {
+		t.Fatalf("deadline not honoured by the coloring: %v", elapsed)
+	}
+	if r.Reached || !isClique(adj, r.Clique) {
+		t.Fatalf("unexpected result %+v", r)
+	}
+}

pkg/obiiter/batchiterator.go

@@ -444,6 +444,69 @@ func (iterator IBioSequence) Rebatch(size int) IBioSequence {
 	return newIter
 }
 
+// RebatchBySize reorganises the stream into batches bounded by two independent
+// upper limits: maxCount (max number of sequences) and maxBytes (max cumulative
+// estimated memory). A batch is flushed as soon as either limit would be
+// exceeded. A single sequence larger than maxBytes is always emitted alone.
+// Passing 0 for a limit disables that constraint; if both are 0 it falls back
+// to Rebatch(obidefault.BatchSizeMax()).
+func (iterator IBioSequence) RebatchBySize(maxBytes int, maxCount int) IBioSequence {
+	if maxBytes <= 0 && maxCount <= 0 {
+		return iterator.Rebatch(obidefault.BatchSizeMax())
+	}
+
+	newIter := MakeIBioSequence()
+
+	newIter.Add(1)
+
+	go func() {
+		newIter.WaitAndClose()
+	}()
+
+	go func() {
+		order := 0
+		// a local variable: reassigning the captured iterator would race with
+		// the IsPaired call below, made by the caller on the same variable
+		sorted := iterator.SortBatches()
+		buffer := obiseq.MakeBioSequenceSlice()
+		bufBytes := 0
+		source := ""
+
+		flush := func() {
+			if len(buffer) > 0 {
+				newIter.Push(MakeBioSequenceBatch(source, order, buffer))
+				order++
+				buffer = obiseq.MakeBioSequenceSlice()
+				bufBytes = 0
+			}
+		}
+
+		for sorted.Next() {
+			seqs := sorted.Get()
+			source = seqs.Source()
+			for _, s := range seqs.Slice() {
+				sz := s.MemorySize()
+				countFull := maxCount > 0 && len(buffer) >= maxCount
+				memFull := maxBytes > 0 && bufBytes+sz > maxBytes && len(buffer) > 0
+				if countFull || memFull {
+					flush()
+				}
+				buffer = append(buffer, s)
+				bufBytes += sz
+			}
+		}
+		flush()
+
+		newIter.Done()
+	}()
+
+	if iterator.IsPaired() {
+		newIter.MarkAsPaired()
+	}
+
+	return newIter
+}
+
 func (iterator IBioSequence) FilterEmpty() IBioSequence {
 
 	newIter := MakeIBioSequence()
@@ -622,7 +685,8 @@ func (iterator IBioSequence) FilterOn(predicate obiseq.SequencePredicate,
 			}
 
 			seqs.slice = slice[:j]
-			trueIter.pointer.channel <- seqs
+			trueIter.Push(seqs)
+			// trueIter.pointer.channel <- seqs
 		}
 
 		trueIter.Done()
@@ -638,7 +702,7 @@ func (iterator IBioSequence) FilterOn(predicate obiseq.SequencePredicate,
 		trueIter.MarkAsPaired()
 	}
 
-	return trueIter.Rebatch(size)
+	return trueIter.RebatchBySize(obidefault.BatchMem(), obidefault.BatchSizeMax())
 }
 
 func (iterator IBioSequence) FilterAnd(predicate obiseq.SequencePredicate,
@@ -694,7 +758,7 @@ func (iterator IBioSequence) FilterAnd(predicate obiseq.SequencePredicate,
 		trueIter.MarkAsPaired()
 	}
 
-	return trueIter.Rebatch(size)
+	return trueIter.RebatchBySize(obidefault.BatchMem(), obidefault.BatchSizeMax())
 }
 
 // Load all sequences availables from an IBioSequenceBatch iterator into

pkg/obiiter/distribute.go

@@ -57,34 +57,21 @@ func (dist *IDistribute) Classifier() *obiseq.BioSequenceClassifier {
 }
 
 // Distribute organizes the biosequences from the iterator into batches
-// based on the provided classifier and batch sizes. It returns an
-// IDistribute instance that manages the distribution of the sequences.
+// based on the provided classifier. It returns an IDistribute instance
+// that manages the distribution of the sequences.
 //
-// Parameters:
-//   - class: A pointer to a BioSequenceClassifier used to classify
-//     the biosequences during distribution.
-//   - sizes: Optional integer values specifying the batch size. If
-//     no sizes are provided, a default batch size of 5000 is used.
-//
-// Returns:
-// An IDistribute instance that contains the outputs of the
-// classified biosequences, a channel for new data notifications,
-// and the classifier used for distribution. The method operates
-// asynchronously, processing the sequences in separate goroutines.
-// It ensures that the outputs are closed and cleaned up once
-// processing is complete.
-func (iterator IBioSequence) Distribute(class *obiseq.BioSequenceClassifier, sizes ...int) IDistribute {
-	batchsize := obidefault.BatchSize()
+// Batches are flushed when either BatchSizeMax() sequences or BatchMem()
+// bytes are accumulated per key, mirroring the RebatchBySize strategy.
+func (iterator IBioSequence) Distribute(class *obiseq.BioSequenceClassifier) IDistribute {
+	maxCount := obidefault.BatchSizeMax()
+	maxBytes := obidefault.BatchMem()
 
 	outputs := make(map[int]IBioSequence, 100)
 	slices := make(map[int]*obiseq.BioSequenceSlice, 100)
+	bufBytes := make(map[int]int, 100)
 	orders := make(map[int]int, 100)
 	news := make(chan int)
 
-	if len(sizes) > 0 {
-		batchsize = sizes[0]
-	}
-
 	jobDone := sync.WaitGroup{}
 	lock := sync.Mutex{}
 
@@ -115,6 +102,7 @@ func (iterator IBioSequence) Distribute(class *obiseq.BioSequenceClassifier, siz
 					slice = &s
 					slices[key] = slice
 					orders[key] = 0
+					bufBytes[key] = 0
 
 					lock.Lock()
 					outputs[key] = MakeIBioSequence()
@@ -123,14 +111,20 @@ func (iterator IBioSequence) Distribute(class *obiseq.BioSequenceClassifier, siz
 					news <- key
 				}
 
-				*slice = append(*slice, s)
-
-				if len(*slice) == batchsize {
+				sz := s.MemorySize()
+				countFull := maxCount > 0 && len(*slice) >= maxCount
+				memFull := maxBytes > 0 && bufBytes[key]+sz > maxBytes && len(*slice) > 0
+				if countFull || memFull {
 					outputs[key].Push(MakeBioSequenceBatch(source, orders[key], *slice))
 					orders[key]++
 					s := obiseq.MakeBioSequenceSlice()
 					slices[key] = &s
+					slice = &s
+					bufBytes[key] = 0
 				}
+
+				*slice = append(*slice, s)
+				bufBytes[key] += sz
 			}
 		}
 

pkg/obiiter/fragment.go

@@ -3,6 +3,7 @@ package obiiter
 import (
 	log "github.com/sirupsen/logrus"
 
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 )
 
@@ -70,7 +71,7 @@ func IFragments(minsize, length, overlap, size, nworkers int) Pipeable {
 		}
 		go f(iterator)
 
-		return newiter.SortBatches().Rebatch(size)
+		return newiter.SortBatches().RebatchBySize(obidefault.BatchMem(), obidefault.BatchSizeMax())
 	}
 
 	return ifrg

pkg/obikmer/debruijn.go

@@ -5,7 +5,6 @@ import (
 	"container/heap"
 	"fmt"
 	"math"
-	"math/bits"
 	"os"
 	"slices"
 
@@ -610,14 +609,6 @@ func (graph *DeBruijnGraph) WriteGml(filename string) error {
 	return err
 }
 
-// Calculating the hamming distance between two k-mers.
-func (g *DeBruijnGraph) HammingDistance(kmer1, kmer2 uint64) int {
-	ident := ^((kmer1 & kmer2) | (^kmer1 & ^kmer2))
-	ident |= (ident >> 1)
-	ident &= 0x5555555555555555 & g.kmermask
-	return bits.OnesCount64(ident)
-}
-
 type UInt64Heap []uint64
 
 func (h UInt64Heap) Len() int           { return len(h) }

pkg/obikmer/encodefourmer.go

@@ -47,7 +47,7 @@ func Encode4mer(seq *obiseq.BioSequence, buffer *[]byte) []byte {
 	length := slength - 3
 	rawseq := seq.Sequence()
 
-	if length < 0 {
+	if length <= 0 {
 		return nil
 	}
 

pkg/obikmer/entropy.go

@@ -4,22 +4,21 @@ import "math"
 
 // KmerEntropy computes the entropy of a single encoded k-mer.
 //
-// The algorithm mirrors the lowmask entropy calculation: it decodes the k-mer
+// The algorithm mirrors the Rust obiskbuilder entropy: it decodes the k-mer
 // to a DNA sequence, extracts all sub-words of each size from 1 to levelMax,
 // normalizes them by circular canonical form, counts their frequencies, and
-// computes Shannon entropy normalized by the maximum possible entropy.
+// computes Shannon entropy corrected for class sizes, normalized by the
+// maximum possible entropy over 4^ws raw bins.
 // The returned value is the minimum entropy across all word sizes.
 //
+// Correction for small sequences: the raw entropy H = log(N) - Σ f·log(f)/N
+// under-estimates the true complexity when many raw words collapse to the same
+// canonical form.  Adding Σ f·log(class_size)/N recovers the entropy of the
+// underlying uncollapsed distribution (assuming uniform mixing within each
+// equivalence class).
+//
 // A value close to 0 indicates very low complexity (e.g. "AAAA..."),
 // while a value close to 1 indicates high complexity.
-//
-// Parameters:
-//   - kmer: the encoded k-mer (2 bits per base)
-//   - k: the k-mer size
-//   - levelMax: maximum sub-word size for entropy (typically 6)
-//
-// Returns:
-//   - minimum normalized entropy across all word sizes 1..levelMax
 func KmerEntropy(kmer uint64, k int, levelMax int) float64 {
 	if k < 1 || levelMax < 1 {
 		return 1.0
@@ -35,7 +34,7 @@ func KmerEntropy(kmer uint64, k int, levelMax int) float64 {
 	var seqBuf [32]byte
 	seq := DecodeKmer(kmer, k, seqBuf[:])
 
-	// Pre-compute nLogN lookup (same as lowmask)
+	// Pre-compute nLogN lookup
 	nLogN := make([]float64, k+1)
 	for i := 1; i <= k; i++ {
 		nLogN[i] = float64(i) * math.Log(float64(i))
@@ -51,6 +50,23 @@ func KmerEntropy(kmer uint64, k int, levelMax int) float64 {
 		}
 	}
 
+	// Build ln(class_size) tables: for each canonical form, how many raw
+	// words map to it under circular normalization.
+	classLogSizeTables := make([][]float64, levelMax+1)
+	for ws := 1; ws <= levelMax; ws++ {
+		tableSize := 1 << (ws * 2)
+		classSize := make([]int, tableSize)
+		for code := 0; code < tableSize; code++ {
+			classSize[normTables[ws][code]]++
+		}
+		classLogSizeTables[ws] = make([]float64, tableSize)
+		for j := 0; j < tableSize; j++ {
+			if classSize[j] > 0 {
+				classLogSizeTables[ws][j] = math.Log(float64(classSize[j]))
+			}
+		}
+	}
+
 	minEntropy := math.MaxFloat64
 
 	for ws := 1; ws <= levelMax; ws++ {
@@ -75,23 +91,13 @@ func KmerEntropy(kmer uint64, k int, levelMax int) float64 {
 			table[normWord]++
 		}
 
-		// Compute Shannon entropy
+		// Compute emax over 4^ws raw bins (uncollapsed distribution).
 		floatNwords := float64(nwords)
 		logNwords := math.Log(floatNwords)
-
-		var sumNLogN float64
-		for j := 0; j < tableSize; j++ {
-			n := table[j]
-			if n > 0 {
-				sumNLogN += nLogN[n]
-			}
-		}
-
-		// Compute emax (maximum possible entropy for this word size)
-		na := CanonicalCircularKmerCount(ws)
+		na := tableSize // 4^ws
 		var emax float64
 		if nwords < na {
-			emax = math.Log(float64(nwords))
+			emax = logNwords
 		} else {
 			cov := nwords / na
 			remains := nwords - (na * cov)
@@ -105,7 +111,19 @@ func KmerEntropy(kmer uint64, k int, levelMax int) float64 {
 			continue
 		}
 
-		entropy := (logNwords - sumNLogN/floatNwords) / emax
+		// Accumulate Σ f·log(f) and Σ f·log(class_size) over canonical forms.
+		classLogSize := classLogSizeTables[ws]
+		var sumNLogN, sumClassLogN float64
+		for j := 0; j < tableSize; j++ {
+			n := table[j]
+			if n > 0 {
+				sumNLogN += nLogN[n]
+				sumClassLogN += float64(n) * classLogSize[j]
+			}
+		}
+
+		// Corrected entropy: H_raw ≈ log(N) + (Σf·log(s) - Σf·log(f)) / N
+		entropy := (logNwords + sumClassLogN/floatNwords - sumNLogN/floatNwords) / emax
 		if entropy < 0 {
 			entropy = 0
 		}
@@ -129,24 +147,20 @@ func KmerEntropy(kmer uint64, k int, levelMax int) float64 {
 // IMPORTANT: a KmerEntropyFilter is NOT safe for concurrent use.
 // Each goroutine must create its own instance via NewKmerEntropyFilter.
 type KmerEntropyFilter struct {
-	k          int
-	levelMax   int
-	threshold  float64
-	nLogN      []float64
-	normTables [][]int
-	emaxValues []float64
-	logNwords  []float64
+	k                   int
+	levelMax            int
+	threshold           float64
+	nLogN               []float64
+	normTables          [][]int
+	classLogSizeTables  [][]float64
+	emaxValues          []float64
+	logNwords           []float64
 	// Pre-allocated frequency tables reused across Entropy() calls.
 	// One per word size (index 0 unused). Reset to zero before each use.
 	freqTables [][]int
 }
 
 // NewKmerEntropyFilter creates an entropy filter with pre-computed tables.
-//
-// Parameters:
-//   - k: the k-mer size
-//   - levelMax: maximum sub-word size for entropy (typically 6)
-//   - threshold: entropy threshold (k-mers with entropy <= threshold are rejected)
 func NewKmerEntropyFilter(k, levelMax int, threshold float64) *KmerEntropyFilter {
 	if levelMax >= k {
 		levelMax = k - 1
@@ -169,20 +183,38 @@ func NewKmerEntropyFilter(k, levelMax int, threshold float64) *KmerEntropyFilter
 		}
 	}
 
+	// ln(class_size) for each canonical form under circular normalization.
+	classLogSizeTables := make([][]float64, levelMax+1)
+	for ws := 1; ws <= levelMax; ws++ {
+		tableSize := 1 << (ws * 2)
+		classSize := make([]int, tableSize)
+		for code := 0; code < tableSize; code++ {
+			classSize[normTables[ws][code]]++
+		}
+		classLogSizeTables[ws] = make([]float64, tableSize)
+		for j := 0; j < tableSize; j++ {
+			if classSize[j] > 0 {
+				classLogSizeTables[ws][j] = math.Log(float64(classSize[j]))
+			}
+		}
+	}
+
+	// Pre-compute emax and logNwords per word size.
+	// emax uses 4^ws raw bins to match the corrected entropy.
 	emaxValues := make([]float64, levelMax+1)
 	logNwords := make([]float64, levelMax+1)
 	for ws := 1; ws <= levelMax; ws++ {
 		nw := k - ws + 1
-		na := CanonicalCircularKmerCount(ws)
+		na := 1 << (ws * 2) // 4^ws raw bins
+		floatNw := float64(nw)
+		logNwords[ws] = math.Log(floatNw)
 		if nw < na {
-			logNwords[ws] = math.Log(float64(nw))
-			emaxValues[ws] = math.Log(float64(nw))
+			emaxValues[ws] = logNwords[ws]
 		} else {
 			cov := nw / na
 			remains := nw - (na * cov)
-			f1 := float64(cov) / float64(nw)
-			f2 := float64(cov+1) / float64(nw)
-			logNwords[ws] = math.Log(float64(nw))
+			f1 := float64(cov) / floatNw
+			f2 := float64(cov+1) / floatNw
 			emaxValues[ws] = -(float64(na-remains)*f1*math.Log(f1) +
 				float64(remains)*f2*math.Log(f2))
 		}
@@ -195,14 +227,15 @@ func NewKmerEntropyFilter(k, levelMax int, threshold float64) *KmerEntropyFilter
 	}
 
 	return &KmerEntropyFilter{
-		k:          k,
-		levelMax:   levelMax,
-		threshold:  threshold,
-		nLogN:      nLogN,
-		normTables: normTables,
-		emaxValues: emaxValues,
-		logNwords:  logNwords,
-		freqTables: freqTables,
+		k:                  k,
+		levelMax:           levelMax,
+		threshold:          threshold,
+		nLogN:              nLogN,
+		normTables:         normTables,
+		classLogSizeTables: classLogSizeTables,
+		emaxValues:         emaxValues,
+		logNwords:          logNwords,
+		freqTables:         freqTables,
 	}
 }
 
@@ -236,7 +269,7 @@ func (ef *KmerEntropyFilter) Entropy(kmer uint64) float64 {
 		// Count circular-canonical sub-word frequencies
 		tableSize := 1 << (ws * 2)
 		table := ef.freqTables[ws]
-		clear(table) // reset to zero
+		clear(table)
 		mask := (1 << (ws * 2)) - 1
 		normTable := ef.normTables[ws]
 
@@ -251,19 +284,21 @@ func (ef *KmerEntropyFilter) Entropy(kmer uint64) float64 {
 			table[normWord]++
 		}
 
-		// Compute Shannon entropy
 		floatNwords := float64(nwords)
 		logNwords := ef.logNwords[ws]
+		classLogSize := ef.classLogSizeTables[ws]
 
-		var sumNLogN float64
+		var sumNLogN, sumClassLogN float64
 		for j := 0; j < tableSize; j++ {
 			n := table[j]
 			if n > 0 {
 				sumNLogN += ef.nLogN[n]
+				sumClassLogN += float64(n) * classLogSize[j]
 			}
 		}
 
-		entropy := (logNwords - sumNLogN/floatNwords) / emax
+		// Corrected entropy: H_raw ≈ log(N) + (Σf·log(s) - Σf·log(f)) / N
+		entropy := (logNwords + sumClassLogN/floatNwords - sumNLogN/floatNwords) / emax
 		if entropy < 0 {
 			entropy = 0
 		}

pkg/obikmer/enumkmer.go

@@ -0,0 +1,129 @@
+package obikmer
+
+import "iter"
+
+// KmerConstraints describes lexicographic properties a k-mer must satisfy.
+//
+// MaxCount is indexed by the 2-bit base code (A=0, C=1, G=2, T=3):
+// a negative value means unlimited, 0 forbids the base altogether
+// (so MaxCount[2] = 0 restricts the alphabet to A, C, T).
+//
+// MaxGC is the maximum number of C or G bases; a negative value means
+// unlimited.
+//
+// MaxRun is the maximum number of consecutive identical bases;
+// a value <= 0 means unlimited. MinRun requires at least one run of that
+// length; a value <= 1 means no requirement.
+//
+// Accepted and Rejected are optional per-position sets of bases (4 bits masks,
+// bit b set when the base of code b is in the set, see ParseIUPACMasks).
+// When set, their length must be k. A k-mer is accepted only if each of its
+// bases belongs to the Accepted set of its position, and is refused if every
+// one of its bases belongs to the Rejected set of its position.
+//
+// The zero value forbids every base: use NoKmerConstraints as a starting point.
+type KmerConstraints struct {
+	MaxCount [4]int
+	MaxGC    int
+	MaxRun   int
+	MinRun   int
+	Accepted []uint8
+	Rejected []uint8
+}
+
+// NoKmerConstraints returns constraints accepting every k-mer.
+func NoKmerConstraints() KmerConstraints {
+	return KmerConstraints{MaxCount: [4]int{-1, -1, -1, -1}, MaxGC: -1}
+}
+
+// IterConstrainedKmers returns an iterator over all k-mers of size k
+// satisfying the constraints c, encoded as in EncodeKmer, in increasing
+// numerical order.
+//
+// K-mers are built base by base and a branch is abandoned as soon as a
+// constraint is violated or cannot be met any more (MaxCount, MaxGC, MaxRun
+// and Accepted are monotone: a violating prefix cannot be repaired by
+// extension; MinRun is checked against the positions left), so the cost is
+// proportional to the number of explored prefixes, not to 4^k. Rejected is
+// checked on complete k-mers.
+//
+// k must be between 1 and 31, otherwise the iterator is empty. It is also
+// empty if Accepted or Rejected is set with a length different from k.
+//
+// Example:
+//
+//	c := NoKmerConstraints()
+//	c.MaxCount[2] = 0 // no G
+//	c.MaxRun = 3
+//	for kmer := range IterConstrainedKmers(8, c) { ... }
+func IterConstrainedKmers(k int, c KmerConstraints) iter.Seq[uint64] {
+	return func(yield func(uint64) bool) {
+		if k < 1 || k > 31 || !c.validPositions(k) {
+			return
+		}
+
+		// Nothing to enumerate if the per-base caps cannot even sum to k.
+		capacity := 0
+		for _, m := range c.MaxCount {
+			if m < 0 || m >= k {
+				capacity += k
+			} else {
+				capacity += m
+			}
+		}
+		if capacity < k {
+			return
+		}
+
+		var counts [4]int
+		gc := 0
+
+		var walk func(depth int, kmer uint64, last, run, longest int) bool
+		walk = func(depth int, kmer uint64, last, run, longest int) bool {
+			if depth == k {
+				if c.Rejected != nil && matchesAllMasks(kmer, k, c.Rejected) {
+					return true
+				}
+				return yield(kmer)
+			}
+			for b := range 4 {
+				if c.MaxCount[b] >= 0 && counts[b] >= c.MaxCount[b] {
+					continue
+				}
+				if c.Accepted != nil && c.Accepted[depth]>>uint(b)&1 == 0 {
+					continue
+				}
+				isGC := b == 1 || b == 2
+				if isGC && c.MaxGC >= 0 && gc >= c.MaxGC {
+					continue
+				}
+				r := 1
+				if b == last {
+					r = run + 1
+				}
+				if c.MaxRun > 0 && r > c.MaxRun {
+					continue
+				}
+				l := max(longest, r)
+				if c.MinRun > 1 && l < c.MinRun && r+(k-depth-1) < c.MinRun {
+					continue
+				}
+				counts[b]++
+				if isGC {
+					gc++
+				}
+				ok := walk(depth+1, kmer<<2|uint64(b), b, r, l)
+				counts[b]--
+				if isGC {
+					gc--
+				}
+				if !ok {
+					return false
+				}
+			}
+			return true
+		}
+
+		walk(0, 0, -1, 0, 0)
+	}
+}

pkg/obikmer/enumkmer_test.go

@@ -0,0 +1,85 @@
+package obikmer
+
+import (
+	"strings"
+	"testing"
+)
+
+func collectConstrained(k int, c KmerConstraints) []uint64 {
+	var out []uint64
+	for km := range IterConstrainedKmers(k, c) {
+		out = append(out, km)
+	}
+	return out
+}
+
+func TestIterConstrainedKmersUnconstrained(t *testing.T) {
+	got := collectConstrained(4, NoKmerConstraints())
+	if len(got) != 256 {
+		t.Fatalf("expected 256 k-mers, got %d", len(got))
+	}
+	for i, km := range got {
+		if km != uint64(i) {
+			t.Fatalf("not in increasing order at %d: %d", i, km)
+		}
+	}
+}
+
+func TestIterConstrainedKmersNoG(t *testing.T) {
+	c := NoKmerConstraints()
+	c.MaxCount[2] = 0
+	got := collectConstrained(5, c)
+	if len(got) != 243 {
+		t.Fatalf("expected 3^5 = 243 k-mers, got %d", len(got))
+	}
+	for _, km := range got {
+		if strings.Contains(string(DecodeKmer(km, 5, nil)), "g") {
+			t.Fatalf("k-mer %d contains G", km)
+		}
+	}
+}
+
+func TestIterConstrainedKmersMaxRun(t *testing.T) {
+	c := NoKmerConstraints()
+	c.MaxRun = 2
+	// a(n) = 3*(a(n-1)+a(n-2)), a(1)=4, a(2)=16
+	want := map[int]int{3: 60, 4: 228}
+	for k, n := range want {
+		if got := len(collectConstrained(k, c)); got != n {
+			t.Fatalf("k=%d: expected %d, got %d", k, n, got)
+		}
+	}
+}
+
+func TestIterConstrainedKmersMaxCount(t *testing.T) {
+	c := NoKmerConstraints()
+	c.MaxCount[0] = 1 // at most one A
+	for _, km := range collectConstrained(6, c) {
+		if strings.Count(string(DecodeKmer(km, 6, nil)), "a") > 1 {
+			t.Fatalf("k-mer %d has more than one A", km)
+		}
+	}
+}
+
+func TestIterConstrainedKmersImpossible(t *testing.T) {
+	c := KmerConstraints{MaxCount: [4]int{1, 1, 1, 0}}
+	if got := collectConstrained(4, c); len(got) != 0 {
+		t.Fatalf("expected no k-mer, got %d", len(got))
+	}
+	if got := collectConstrained(0, NoKmerConstraints()); len(got) != 0 {
+		t.Fatalf("expected no k-mer for k=0")
+	}
+}
+
+func TestIterConstrainedKmersEarlyStop(t *testing.T) {
+	n := 0
+	for range IterConstrainedKmers(10, NoKmerConstraints()) {
+		n++
+		if n == 5 {
+			break
+		}
+	}
+	if n != 5 {
+		t.Fatalf("early stop failed: %d", n)
+	}
+}

pkg/obikmer/kmer_constraints.go

@@ -0,0 +1,141 @@
+package obikmer
+
+import (
+	"fmt"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+// ParseIUPACMasks converts an IUPAC nucleotide pattern into one 4 bits mask per
+// position (bit 0 is A, bit 1 is C, bit 2 is G, bit 3 is T), the format used by
+// KmerConstraints.Accepted and Rejected.
+func ParseIUPACMasks(pattern string) ([]uint8, error) {
+	masks := make([]uint8, len(pattern))
+	for i := 0; i < len(pattern); i++ {
+		m := obiseq.IUPACNucMask(pattern[i])
+		if m == 0 {
+			return nil, fmt.Errorf("invalid IUPAC nucleotide code %q at position %d of pattern %q",
+				pattern[i], i+1, pattern)
+		}
+		masks[i] = m
+	}
+	return masks, nil
+}
+
+// RestrictAlphabet forbids every base that is not denoted by one of the IUPAC
+// codes of alphabet. For instance "ACT" (or "H") forbids G.
+func (c *KmerConstraints) RestrictAlphabet(alphabet string) error {
+	masks, err := ParseIUPACMasks(alphabet)
+	if err != nil {
+		return err
+	}
+	var allowed uint8
+	for _, m := range masks {
+		allowed |= m
+	}
+	if allowed == 0 {
+		return fmt.Errorf("empty alphabet")
+	}
+	for b := range 4 {
+		if allowed>>uint(b)&1 == 0 {
+			c.MaxCount[b] = 0
+		}
+	}
+	return nil
+}
+
+// validPositions tells whether Accepted and Rejected, when set, have length k.
+func (c *KmerConstraints) validPositions(k int) bool {
+	return (c.Accepted == nil || len(c.Accepted) == k) &&
+		(c.Rejected == nil || len(c.Rejected) == k)
+}
+
+// matchesAllMasks tells whether every base of the k-mer belongs to the set
+// given for its position.
+func matchesAllMasks(kmer uint64, k int, masks []uint8) bool {
+	for i := k - 1; i >= 0; i-- {
+		if masks[i]>>(kmer&3)&1 == 0 {
+			return false
+		}
+		kmer >>= 2
+	}
+	return true
+}
+
+// Accepts tells whether a k-mer of size k satisfies every constraint.
+// It is the predicate whose solutions IterConstrainedKmers enumerates, to be
+// used to filter k-mers coming from elsewhere.
+func (c *KmerConstraints) Accepts(kmer uint64, k int) bool {
+	if k < 1 || k > 31 || !c.validPositions(k) {
+		return false
+	}
+
+	var counts [4]int
+	gc, run, longest, last := 0, 0, 0, -1
+	for i := k - 1; i >= 0; i-- {
+		b := int(kmer>>(2*uint(i))) & 3
+		if c.Accepted != nil && c.Accepted[k-1-i]>>uint(b)&1 == 0 {
+			return false
+		}
+		counts[b]++
+		if c.MaxCount[b] >= 0 && counts[b] > c.MaxCount[b] {
+			return false
+		}
+		if b == 1 || b == 2 {
+			gc++
+			if c.MaxGC >= 0 && gc > c.MaxGC {
+				return false
+			}
+		}
+		if b == last {
+			run++
+		} else {
+			run = 1
+		}
+		last = b
+		longest = max(longest, run)
+		if c.MaxRun > 0 && run > c.MaxRun {
+			return false
+		}
+	}
+
+	if c.MinRun > 1 && longest < c.MinRun {
+		return false
+	}
+	if c.Rejected != nil && matchesAllMasks(kmer, k, c.Rejected) {
+		return false
+	}
+	return true
+}
+
+// HammingBound returns the sphere-packing (Hamming) upper bound on the number
+// of words of length k over an alphabet of alphabetSize letters that can be
+// pairwise at Hamming distance >= d. No larger family exists; a family of
+// that size may not exist either.
+func HammingBound(alphabetSize, k, d int) float64 {
+	if d <= 1 {
+		total := 1.0
+		for i := 0; i < k; i++ {
+			total *= float64(alphabetSize)
+		}
+		return total
+	}
+	if d%2 == 0 { // A(k, d) <= A(k-1, d-1)
+		k--
+		d--
+	}
+	if k < 1 {
+		return 1
+	}
+	t := (d - 1) / 2
+	total, ball, binom, pow := 1.0, 1.0, 1.0, 1.0
+	for i := 0; i < k; i++ {
+		total *= float64(alphabetSize)
+	}
+	for i := 1; i <= t && i <= k; i++ {
+		binom = binom * float64(k-i+1) / float64(i)
+		pow *= float64(alphabetSize - 1)
+		ball += binom * pow
+	}
+	return total / ball
+}

pkg/obikmer/kmer_constraints_test.go

@@ -0,0 +1,123 @@
+package obikmer
+
+import (
+	"math"
+	"math/rand"
+	"testing"
+)
+
+// IterConstrainedKmers must enumerate exactly the k-mers accepted by Accepts.
+func TestIterConstrainedKmersMatchesAccepts(t *testing.T) {
+	rng := rand.New(rand.NewSource(10))
+	patterns := []string{"", "nnnnnn", "yrnnnn", "cnnnnn", "annnnt"}
+
+	for trial := 0; trial < 60; trial++ {
+		k := 6
+		c := NoKmerConstraints()
+		for b := range 4 {
+			if rng.Intn(3) == 0 {
+				c.MaxCount[b] = rng.Intn(4)
+			}
+		}
+		if rng.Intn(2) == 0 {
+			c.MaxGC = rng.Intn(5)
+		}
+		if rng.Intn(2) == 0 {
+			c.MaxRun = 1 + rng.Intn(3)
+		}
+		if rng.Intn(2) == 0 {
+			c.MinRun = 2 + rng.Intn(3)
+		}
+		if p := patterns[rng.Intn(len(patterns))]; p != "" {
+			c.Accepted, _ = ParseIUPACMasks(p)
+		}
+		if p := patterns[rng.Intn(len(patterns))]; p != "" && rng.Intn(2) == 0 {
+			c.Rejected, _ = ParseIUPACMasks(p)
+		}
+
+		var want []uint64
+		for km := uint64(0); km < 1<<(2*uint(k)); km++ {
+			if c.Accepts(km, k) {
+				want = append(want, km)
+			}
+		}
+		got := collectConstrained(k, c)
+		if len(got) != len(want) {
+			t.Fatalf("trial %d %+v: got %d k-mers, want %d", trial, c, len(got), len(want))
+		}
+		for i := range got {
+			if got[i] != want[i] {
+				t.Fatalf("trial %d %+v: mismatch at %d: %d != %d", trial, c, i, got[i], want[i])
+			}
+		}
+	}
+}
+
+func TestConstraintsGCAndRuns(t *testing.T) {
+	enc := func(s string) uint64 { return EncodeKmer([]byte(s), len(s)) }
+
+	c := NoKmerConstraints()
+	c.MaxGC = 2
+	if !c.Accepts(enc("ACGTAA"), 6) || c.Accepts(enc("ACGCAA"), 6) {
+		t.Fatal("MaxGC not enforced")
+	}
+
+	c = NoKmerConstraints()
+	c.MinRun = 3
+	if !c.Accepts(enc("ACCCGT"), 6) || c.Accepts(enc("ACCGGT"), 6) {
+		t.Fatal("MinRun not enforced")
+	}
+
+	c = NoKmerConstraints()
+	c.Rejected, _ = ParseIUPACMasks("cnnnnn") // no C in first position
+	if c.Accepts(enc("CAAAAA"), 6) || !c.Accepts(enc("AAAAAA"), 6) {
+		t.Fatal("Rejected not enforced")
+	}
+}
+
+func TestRestrictAlphabetAndPatterns(t *testing.T) {
+	c := NoKmerConstraints()
+	if err := c.RestrictAlphabet("ACT"); err != nil {
+		t.Fatal(err)
+	}
+	if got := len(collectConstrained(4, c)); got != 81 {
+		t.Fatalf("expected 3^4 = 81 k-mers, got %d", got)
+	}
+	c = NoKmerConstraints()
+	if err := c.RestrictAlphabet("H"); err != nil { // H = A, C or T
+		t.Fatal(err)
+	}
+	if c.MaxCount != [4]int{-1, -1, 0, -1} {
+		t.Fatalf("unexpected counts %v", c.MaxCount)
+	}
+	if err := c.RestrictAlphabet("AX"); err == nil {
+		t.Fatal("invalid IUPAC code must be refused")
+	}
+	if _, err := ParseIUPACMasks("acgz"); err == nil {
+		t.Fatal("invalid IUPAC code must be refused")
+	}
+	// wrong pattern length
+	c = NoKmerConstraints()
+	c.Accepted, _ = ParseIUPACMasks("nnn")
+	if got := collectConstrained(4, c); len(got) != 0 {
+		t.Fatalf("pattern of wrong length must give nothing, got %d", len(got))
+	}
+}
+
+func TestHammingBound(t *testing.T) {
+	cases := []struct {
+		q, k, d int
+		want    float64
+	}{
+		{4, 7, 3, 16384.0 / 22},
+		{4, 8, 4, 16384.0 / 22}, // d even: reduces to (7, 3)
+		{4, 9, 5, 262144.0 / 352},
+		{3, 9, 3, 19683.0 / 19},
+		{4, 5, 1, 1024},
+	}
+	for _, c := range cases {
+		if got := HammingBound(c.q, c.k, c.d); math.Abs(got-c.want) > 1e-6 {
+			t.Errorf("HammingBound(%d,%d,%d) = %v, want %v", c.q, c.k, c.d, got, c.want)
+		}
+	}
+}

pkg/obikmer/kmer_dissimilarity.go

@@ -0,0 +1,84 @@
+package obikmer
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+)
+
+// KmerMetric selects the distance used to compare two k-mers.
+type KmerMetric int
+
+const (
+	// HammingMetric counts the positions at which two k-mers differ.
+	HammingMetric KmerMetric = iota
+	// LCSMetric is the alignment length minus the LCS length (see KmerLCS).
+	LCSMetric
+)
+
+// ParseKmerMetric converts "hamming" or "lcs" (case insensitive) to a KmerMetric.
+func ParseKmerMetric(name string) (KmerMetric, error) {
+	switch strings.ToLower(name) {
+	case "hamming":
+		return HammingMetric, nil
+	case "lcs":
+		return LCSMetric, nil
+	}
+	return HammingMetric, fmt.Errorf("unknown distance type %q (expected hamming or lcs)", name)
+}
+
+func (m KmerMetric) String() string {
+	if m == LCSMetric {
+		return "lcs"
+	}
+	return "hamming"
+}
+
+type lcsScratch struct {
+	partners []uint64
+	results  []KmerLCSResult
+}
+
+// NewKmerDissimilarityFilter describes the graph whose vertices are kmers[i]
+// (k-mers of size k, 1 <= k <= 31) and where two vertices are linked when
+// their distance is at least dmin. The returned function has the signature of
+// obigraph.NeighborFilter: it appends to out[:0] the candidates (indices in
+// kmers) linked to v. It is safe for concurrent use.
+func NewKmerDissimilarityFilter(kmers []uint64, k int, metric KmerMetric, dmin int) func(v int32, candidates, out []int32) []int32 {
+	if metric == HammingMetric {
+		return func(v int32, candidates, out []int32) []int32 {
+			q := kmers[v]
+			out = out[:0]
+			for _, c := range candidates {
+				if HammingDistance(q, kmers[c], k) >= dmin {
+					out = append(out, c)
+				}
+			}
+			return out
+		}
+	}
+
+	pool := sync.Pool{New: func() any { return new(lcsScratch) }}
+	return func(v int32, candidates, out []int32) []int32 {
+		s := pool.Get().(*lcsScratch)
+		defer pool.Put(s)
+
+		s.partners = s.partners[:0]
+		for _, c := range candidates {
+			s.partners = append(s.partners, kmers[c])
+		}
+		if cap(s.results) < len(candidates) {
+			s.results = make([]KmerLCSResult, len(candidates))
+		}
+		s.results = s.results[:len(candidates)]
+		KmerLCSBatch(kmers[v], s.partners, k, s.results)
+
+		out = out[:0]
+		for i, c := range candidates {
+			if s.results[i].Distance() >= dmin {
+				out = append(out, c)
+			}
+		}
+		return out
+	}
+}

pkg/obikmer/kmer_dissimilarity_test.go

@@ -0,0 +1,139 @@
+package obikmer_test
+
+import (
+	"context"
+	"math/rand"
+	"testing"
+	"time"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obigraph"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obikmer"
+)
+
+func allKmers(k int) []uint64 {
+	kmers := make([]uint64, 0, 1<<(2*uint(k)))
+	for km := range obikmer.IterConstrainedKmers(k, obikmer.NoKmerConstraints()) {
+		kmers = append(kmers, km)
+	}
+	return kmers
+}
+
+func TestDissimilarityFilterMatchesDistances(t *testing.T) {
+	const k = 7
+	rng := rand.New(rand.NewSource(1))
+	kmers := make([]uint64, 300)
+	for i := range kmers {
+		kmers[i] = rng.Uint64() & (1<<(2*k) - 1)
+	}
+	candidates := make([]int32, 0, len(kmers))
+	for i := 1; i < len(kmers); i++ {
+		candidates = append(candidates, int32(i))
+	}
+
+	for _, metric := range []obikmer.KmerMetric{obikmer.HammingMetric, obikmer.LCSMetric} {
+		for _, dmin := range []int{1, 3, 5} {
+			filter := obikmer.NewKmerDissimilarityFilter(kmers, k, metric, dmin)
+			got := filter(0, candidates, nil)
+
+			var want []int32
+			for _, c := range candidates {
+				var d int
+				if metric == obikmer.HammingMetric {
+					d = obikmer.HammingDistance(kmers[0], kmers[c], k)
+				} else {
+					d = obikmer.KmerLCS(kmers[0], kmers[c], k).Distance()
+				}
+				if d >= dmin {
+					want = append(want, c)
+				}
+			}
+			if len(got) != len(want) {
+				t.Fatalf("%v d>=%d: got %d vertices, want %d", metric, dmin, len(got), len(want))
+			}
+			for i := range got {
+				if got[i] != want[i] {
+					t.Fatalf("%v d>=%d: mismatch at %d", metric, dmin, i)
+				}
+			}
+		}
+	}
+}
+
+// A family of 5-mers pairwise at Hamming distance >= 3, built with FindClique.
+func TestFindClique_HammingFamily(t *testing.T) {
+	const k, dmin, target = 5, 3, 32
+	kmers := allKmers(k)
+	filter := obikmer.NewKmerDissimilarityFilter(kmers, k, obikmer.HammingMetric, dmin)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	r := obigraph.FindClique(ctx, len(kmers), filter, target, obigraph.CliqueOptions{Workers: 2})
+	if !r.Reached {
+		t.Fatalf("family of %d not found (best %d, exhausted %v)", target, len(r.Clique), r.Exhausted)
+	}
+	for i, a := range r.Clique {
+		for _, b := range r.Clique[i+1:] {
+			if obikmer.HammingDistance(kmers[a], kmers[b], k) < dmin {
+				t.Fatalf("%d and %d are too close", kmers[a], kmers[b])
+			}
+		}
+	}
+}
+
+func TestFindClique_LCSFamily(t *testing.T) {
+	const k, dmin, target = 6, 3, 20
+	kmers := allKmers(k)
+	filter := obikmer.NewKmerDissimilarityFilter(kmers, k, obikmer.LCSMetric, dmin)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+	r := obigraph.FindClique(ctx, len(kmers), filter, target, obigraph.CliqueOptions{Workers: 2})
+	if !r.Reached {
+		t.Fatalf("family of %d not found (best %d, exhausted %v)", target, len(r.Clique), r.Exhausted)
+	}
+	for i, a := range r.Clique {
+		for _, b := range r.Clique[i+1:] {
+			if obikmer.KmerLCS(kmers[a], kmers[b], k).Distance() < dmin {
+				t.Fatalf("%d and %d are too close", kmers[a], kmers[b])
+			}
+		}
+	}
+}
+
+func TestParseKmerMetric(t *testing.T) {
+	if m, err := obikmer.ParseKmerMetric("LCS"); err != nil || m != obikmer.LCSMetric {
+		t.Fatal("lcs not parsed")
+	}
+	if m, err := obikmer.ParseKmerMetric("hamming"); err != nil || m != obikmer.HammingMetric {
+		t.Fatal("hamming not parsed")
+	}
+	if _, err := obikmer.ParseKmerMetric("levenshtein"); err == nil {
+		t.Fatal("unknown metric must be refused")
+	}
+}
+
+// With a target out of reach (17 4-mers at Hamming distance >= 3 do not exist,
+// 16 do), the search must return the best family it can build, not a poor one
+// stuck by the pruning on the target, and the coloring bound must prove that
+// the target is out of reach.
+func TestFindClique_BestWhenTargetOutOfReach(t *testing.T) {
+	const k, dmin = 4, 3
+	kmers := allKmers(k)
+	filter := obikmer.NewKmerDissimilarityFilter(kmers, k, obikmer.HammingMetric, dmin)
+
+	for _, limit := range []int{0, 1024} {
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+		r := obigraph.FindClique(ctx, len(kmers), filter, 17, obigraph.CliqueOptions{Workers: 2, ColorLimit: limit})
+		cancel()
+		if r.Reached {
+			t.Fatal("17 4-mers at Hamming distance >= 3 cannot exist (sphere-packing bound 19, true maximum 16)")
+		}
+		if len(r.Clique) < 16 {
+			t.Fatalf("limit %d: expected the family of 16 to be found, got %d (exhausted %v, maximum %v)",
+				limit, len(r.Clique), r.Exhausted, r.Maximum)
+		}
+		if limit > 0 && !r.Exhausted {
+			t.Fatalf("limit %d: the coloring bound should prove that 17 is out of reach", limit)
+		}
+	}
+}

pkg/obikmer/kmer_distance.go

@@ -0,0 +1,136 @@
+package obikmer
+
+import "math/bits"
+
+// KmerLCSResult is the result of a k-mer LCS alignment: the length of the
+// Longest Common Subsequence and the length of the shortest alignment
+// achieving it (same semantics as obialign.FastLCSScore).
+type KmerLCSResult struct {
+	LCS      int
+	AlignLen int
+}
+
+// Distance returns AlignLen - LCS, the LCS-mode edit distance.
+func (r KmerLCSResult) Distance() int {
+	return r.AlignLen - r.LCS
+}
+
+// HammingDistance returns the number of positions at which the two k-mers
+// differ. Both k-mers are encoded with 2 bits per base (see EncodeKmer);
+// bits above the 2*k lowest ones are ignored. k must be between 0 and 32.
+func HammingDistance(a, b uint64, k int) int {
+	diff := a ^ b
+	diff = (diff | diff>>1) & 0x5555555555555555
+	diff &= ^uint64(0) >> (64 - 2*k)
+	return bits.OnesCount64(diff)
+}
+
+// KmerLCS aligns two k-mers of the same size in LCS mode (match +1, mismatch
+// and indel 0) and returns the LCS length together with the length of the
+// shortest alignment among those reaching it, so that the distance is
+// AlignLen - LCS.
+//
+// It is the scalar reference implementation, straightforward on purpose:
+// it is the oracle against which the vectorised versions are tested.
+// It returns {-1, -1} if k is not between 1 and 31.
+func KmerLCS(a, b uint64, k int) KmerLCSResult {
+	if k < 1 || k > 31 {
+		return KmerLCSResult{-1, -1}
+	}
+
+	var ba, bb [32]uint8
+	for i := k - 1; i >= 0; i-- {
+		ba[i] = uint8(a & 3)
+		bb[i] = uint8(b & 3)
+		a >>= 2
+		b >>= 2
+	}
+
+	// One DP row: best (score, alignment length) per column.
+	var score, length [32]int
+	for j := 0; j <= k; j++ {
+		length[j] = j
+	}
+
+	better := func(s1, l1, s2, l2 int) bool {
+		return s1 > s2 || (s1 == s2 && l1 < l2)
+	}
+
+	for i := 1; i <= k; i++ {
+		diagS, diagL := score[0], length[0]
+		score[0], length[0] = 0, i
+
+		for j := 1; j <= k; j++ {
+			upS, upL := score[j], length[j]
+
+			bestS, bestL := diagS, diagL+1
+			if ba[j-1] == bb[i-1] {
+				bestS++
+			}
+			if better(upS, upL+1, bestS, bestL) {
+				bestS, bestL = upS, upL+1
+			}
+			if better(score[j-1], length[j-1]+1, bestS, bestL) {
+				bestS, bestL = score[j-1], length[j-1]+1
+			}
+
+			diagS, diagL = upS, upL
+			score[j], length[j] = bestS, bestL
+		}
+	}
+
+	return KmerLCSResult{score[k], length[k]}
+}
+
+// KmerLCSBatch computes KmerLCS(query, partners[i], k) for every partner and
+// stores it in out[i]. out must be at least as long as partners.
+//
+// When built with GOEXPERIMENT=simd the partners are aligned several at a
+// time, one per SIMD lane; otherwise the scalar reference is used.
+// It returns false, leaving out untouched, if k is not between 1 and 31 or
+// if out is too short.
+func KmerLCSBatch(query uint64, partners []uint64, k int, out []KmerLCSResult) bool {
+	if k < 1 || k > 31 || len(out) < len(partners) {
+		return false
+	}
+	kmerLCSBatch(query, partners, k, out)
+	return true
+}
+
+// KmerLCSPartners is a fixed set of k-mers prepared once to be aligned against
+// many queries with KmerLCSBatch-like speed, without redoing the per-call
+// reorganisation of the partners. It is read-only after creation and can be
+// shared between goroutines.
+type KmerLCSPartners struct {
+	prepared kmerLCSPrepared
+	n        int
+}
+
+// NewKmerLCSPartners prepares partners for repeated alignment.
+// It returns nil if k is not between 1 and 31. The slice is not copied when
+// built without GOEXPERIMENT=simd, so it must not be modified afterwards.
+func NewKmerLCSPartners(partners []uint64, k int) *KmerLCSPartners {
+	if k < 1 || k > 31 {
+		return nil
+	}
+	return &KmerLCSPartners{prepared: prepareKmerLCS(partners, k), n: len(partners)}
+}
+
+// Len returns the number of prepared partners.
+func (p *KmerLCSPartners) Len() int { return p.n }
+
+// Align computes the alignment of query against every prepared partner and
+// stores it in out. It returns false if out is shorter than Len().
+func (p *KmerLCSPartners) Align(query uint64, out []KmerLCSResult) bool {
+	if len(out) < p.n {
+		return false
+	}
+	p.prepared.align(query, out[:p.n])
+	return true
+}
+
+func kmerLCSBatchScalar(query uint64, partners []uint64, k int, out []KmerLCSResult) {
+	for i, p := range partners {
+		out[i] = KmerLCS(query, p, k)
+	}
+}

pkg/obikmer/kmer_distance_test.go

@@ -0,0 +1,171 @@
+package obikmer
+
+import (
+	"math/rand"
+	"testing"
+)
+
+func randomKmer(rng *rand.Rand, k int) uint64 {
+	return rng.Uint64() & (uint64(1)<<(2*uint(k)) - 1)
+}
+
+func TestHammingDistance(t *testing.T) {
+	rng := rand.New(rand.NewSource(1))
+	for _, k := range []int{1, 2, 7, 21, 31} {
+		for n := 0; n < 200; n++ {
+			a, b := randomKmer(rng, k), randomKmer(rng, k)
+			sa, sb := DecodeKmer(a, k, nil), DecodeKmer(b, k, nil)
+			want := 0
+			for i := range sa {
+				if sa[i] != sb[i] {
+					want++
+				}
+			}
+			if got := HammingDistance(a, b, k); got != want {
+				t.Fatalf("k=%d %s/%s: got %d, want %d", k, sa, sb, got, want)
+			}
+		}
+	}
+}
+
+func TestHammingDistanceIgnoresHighBits(t *testing.T) {
+	if got := HammingDistance(0xFF00, 0x0000, 4); got != 0 {
+		t.Fatalf("bits above 2k must be ignored, got %d", got)
+	}
+}
+
+func TestKmerLCSKnown(t *testing.T) {
+	enc := func(s string) uint64 { return EncodeKmer([]byte(s), len(s)) }
+	cases := []struct {
+		a, b string
+		want KmerLCSResult
+	}{
+		{"ACGT", "ACGT", KmerLCSResult{4, 4}},
+		{"AAAA", "CCCC", KmerLCSResult{0, 4}},
+		{"ACGT", "ACGA", KmerLCSResult{3, 4}},
+		{"ACGT", "CGTA", KmerLCSResult{3, 5}}, // one shift: 1 indel on each side
+	}
+	for _, c := range cases {
+		if got := KmerLCS(enc(c.a), enc(c.b), 4); got != c.want {
+			t.Errorf("%s/%s: got %+v, want %+v", c.a, c.b, got, c.want)
+		}
+	}
+	if got := KmerLCS(0, 0, 32); got.LCS != -1 {
+		t.Errorf("k=32 must be rejected, got %+v", got)
+	}
+}
+
+// The batch implementation (SIMD when built with GOEXPERIMENT=simd) must agree
+// with the scalar reference for all k and for batch sizes around the lane count.
+func TestKmerLCSBatchMatchesReference(t *testing.T) {
+	rng := rand.New(rand.NewSource(2))
+	for _, k := range []int{1, 2, 3, 5, 8, 16, 21, 30, 31} {
+		for _, n := range []int{0, 1, 3, 7, 8, 9, 16, 17, 100} {
+			query := randomKmer(rng, k)
+			partners := make([]uint64, n)
+			for i := range partners {
+				partners[i] = randomKmer(rng, k)
+			}
+			if n > 2 {
+				partners[0] = query // identical
+			}
+			out := make([]KmerLCSResult, n)
+			if !KmerLCSBatch(query, partners, k, out) {
+				t.Fatalf("k=%d n=%d: batch refused", k, n)
+			}
+			for i, p := range partners {
+				if want := KmerLCS(query, p, k); out[i] != want {
+					t.Fatalf("k=%d n=%d partner %d: got %+v, want %+v", k, n, i, out[i], want)
+				}
+			}
+		}
+	}
+}
+
+func TestKmerLCSBatchRejectsBadInput(t *testing.T) {
+	if KmerLCSBatch(0, []uint64{1}, 0, make([]KmerLCSResult, 1)) {
+		t.Fatal("k=0 must be rejected")
+	}
+	if KmerLCSBatch(0, []uint64{1, 2}, 4, make([]KmerLCSResult, 1)) {
+		t.Fatal("short output must be rejected")
+	}
+}
+
+func benchBatch(b *testing.B, k int, f func(uint64, []uint64, int, []KmerLCSResult)) {
+	rng := rand.New(rand.NewSource(3))
+	partners := make([]uint64, 1024)
+	for i := range partners {
+		partners[i] = randomKmer(rng, k)
+	}
+	out := make([]KmerLCSResult, len(partners))
+	query := randomKmer(rng, k)
+	b.ResetTimer()
+	for b.Loop() {
+		f(query, partners, k, out)
+	}
+	b.ReportMetric(float64(b.Elapsed().Nanoseconds())/float64(b.N*len(partners)), "ns/pair")
+}
+
+func BenchmarkKmerLCSScalar21(b *testing.B) { benchBatch(b, 21, kmerLCSBatchScalar) }
+func BenchmarkKmerLCSBatch21(b *testing.B)  { benchBatch(b, 21, kmerLCSBatch) }
+func BenchmarkKmerLCSScalar11(b *testing.B) { benchBatch(b, 11, kmerLCSBatchScalar) }
+func BenchmarkKmerLCSBatch11(b *testing.B)  { benchBatch(b, 11, kmerLCSBatch) }
+
+func TestKmerLCSPartnersMatchesReference(t *testing.T) {
+	rng := rand.New(rand.NewSource(6))
+	for _, k := range []int{1, 5, 21, 31} {
+		for _, n := range []int{0, 1, 7, 8, 9, 50} {
+			partners := make([]uint64, n)
+			for i := range partners {
+				partners[i] = randomKmer(rng, k)
+			}
+			prep := NewKmerLCSPartners(partners, k)
+			out := make([]KmerLCSResult, n)
+			for q := 0; q < 3; q++ { // reuse the same preparation for several queries
+				query := randomKmer(rng, k)
+				if !prep.Align(query, out) {
+					t.Fatalf("k=%d n=%d: Align refused", k, n)
+				}
+				for i, p := range partners {
+					if want := KmerLCS(query, p, k); out[i] != want {
+						t.Fatalf("k=%d n=%d partner %d: got %+v, want %+v", k, n, i, out[i], want)
+					}
+				}
+			}
+		}
+	}
+	if NewKmerLCSPartners(nil, 0) != nil || NewKmerLCSPartners(nil, 32) != nil {
+		t.Fatal("invalid k must give nil")
+	}
+}
+
+// All-pairs scenario: many queries against the same partners.
+func benchAllPairs(b *testing.B, k int, prepared bool) {
+	rng := rand.New(rand.NewSource(7))
+	partners := make([]uint64, 1024)
+	for i := range partners {
+		partners[i] = randomKmer(rng, k)
+	}
+	queries := make([]uint64, 64)
+	for i := range queries {
+		queries[i] = randomKmer(rng, k)
+	}
+	out := make([]KmerLCSResult, len(partners))
+	prep := NewKmerLCSPartners(partners, k)
+	b.ResetTimer()
+	for b.Loop() {
+		for _, q := range queries {
+			if prepared {
+				prep.Align(q, out)
+			} else {
+				KmerLCSBatch(q, partners, k, out)
+			}
+		}
+	}
+	b.ReportMetric(float64(b.Elapsed().Nanoseconds())/float64(b.N*len(partners)*len(queries)), "ns/pair")
+}
+
+func BenchmarkAllPairsBatch21(b *testing.B)    { benchAllPairs(b, 21, false) }
+func BenchmarkAllPairsPrepared21(b *testing.B) { benchAllPairs(b, 21, true) }
+func BenchmarkAllPairsBatch11(b *testing.B)    { benchAllPairs(b, 11, false) }
+func BenchmarkAllPairsPrepared11(b *testing.B) { benchAllPairs(b, 11, true) }

pkg/obikmer/kmer_lcs_obialign_test.go

@@ -0,0 +1,29 @@
+package obikmer_test
+
+import (
+	"math/rand"
+	"testing"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obialign"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obikmer"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+// KmerLCS must reproduce obialign.FastLCSScore on the decoded sequences.
+func TestKmerLCSMatchesFastLCSScore(t *testing.T) {
+	rng := rand.New(rand.NewSource(4))
+	for _, k := range []int{1, 2, 4, 9, 15, 21, 31} {
+		mask := uint64(1)<<(2*uint(k)) - 1
+		for n := 0; n < 300; n++ {
+			a, b := rng.Uint64()&mask, rng.Uint64()&mask
+			sa := obiseq.NewBioSequence("a", obikmer.DecodeKmer(a, k, nil), "")
+			sb := obiseq.NewBioSequence("b", obikmer.DecodeKmer(b, k, nil), "")
+			lcs, alen := obialign.FastLCSScore(sa, sb, -1, nil)
+			got := obikmer.KmerLCS(a, b, k)
+			if got.LCS != lcs || got.AlignLen != alen {
+				t.Fatalf("k=%d %s/%s: KmerLCS=%+v, FastLCSScore=(%d,%d)",
+					k, sa.String(), sb.String(), got, lcs, alen)
+			}
+		}
+	}
+}

pkg/obikmer/kmer_lcs_scalar.go

@@ -0,0 +1,21 @@
+//go:build !goexperiment.simd
+
+package obikmer
+
+func kmerLCSBatch(query uint64, partners []uint64, k int, out []KmerLCSResult) {
+	kmerLCSBatchScalar(query, partners, k, out)
+}
+
+// kmerLCSPrepared has nothing to precompute without SIMD.
+type kmerLCSPrepared struct {
+	k        int
+	partners []uint64
+}
+
+func prepareKmerLCS(partners []uint64, k int) kmerLCSPrepared {
+	return kmerLCSPrepared{k: k, partners: partners}
+}
+
+func (p *kmerLCSPrepared) align(query uint64, out []KmerLCSResult) {
+	kmerLCSBatchScalar(query, p.partners, p.k, out)
+}

pkg/obikmer/kmer_lcs_simd.go

@@ -0,0 +1,114 @@
+//go:build goexperiment.simd
+
+package obikmer
+
+import "simd"
+
+// Each lane of a vector holds the state of one (query, partner) alignment as
+//
+//	score<<6 | (63 - alignment length)
+//
+// so that taking the maximum of two cells ranks them by score first and, for
+// equal scores, by shortest alignment. With k <= 31 the score is <= 31 and the
+// alignment length <= 62, hence the value fits in 11 bits and the low field
+// is never exhausted. The three DP moves then all reduce to
+//
+//	cell(i,j) = max(diag + 64*match, up, left) - 1
+//
+// (every move lengthens the alignment by one; only a diagonal match adds 1
+// to the score).
+const (
+	lcsScoreUnit = 64
+	lcsLenBase   = 63
+)
+
+func simdLanes() int {
+	var probe simd.Uint16s
+	return probe.Len()
+}
+
+// transposeChunk writes the bases of up to `lanes` partners in dst, laid out
+// so that dst[j*lanes+lane] is base j of the partner in `lane`.
+func transposeChunk(chunk []uint64, k, lanes int, dst []uint16) {
+	for lane, p := range chunk {
+		for j := 0; j < k; j++ {
+			dst[j*lanes+lane] = uint16(p>>(2*uint(k-1-j))) & 3
+		}
+	}
+}
+
+// alignChunk aligns the query against the partners of one transposed chunk
+// and writes one result per lane in out (len(out) <= lanes).
+func alignChunk(query uint64, pbase, row []uint16, k, lanes int, out []KmerLCSResult) {
+	one := simd.BroadcastUint16s(1)
+	bonus := simd.BroadcastUint16s(lcsScoreUnit)
+
+	for j := 0; j <= k; j++ {
+		simd.BroadcastUint16s(uint16(lcsLenBase - j)).Store(row[j*lanes:])
+	}
+
+	for i := 1; i <= k; i++ {
+		qv := simd.BroadcastUint16s(uint16(query>>(2*uint(k-i))) & 3)
+		diag := simd.LoadUint16s(row)
+		left := simd.BroadcastUint16s(uint16(lcsLenBase - i))
+		left.Store(row)
+
+		for j := 1; j <= k; j++ {
+			up := simd.LoadUint16s(row[j*lanes:])
+			pv := simd.LoadUint16s(pbase[(j-1)*lanes:])
+			match := qv.Equal(pv)
+			best := diag.Add(bonus.Masked(match)).Max(up).Max(left).Sub(one)
+			best.Store(row[j*lanes:])
+			diag = up
+			left = best
+		}
+	}
+
+	last := row[k*lanes:]
+	for lane := range out {
+		v := int(last[lane])
+		out[lane] = KmerLCSResult{
+			LCS:      v / lcsScoreUnit,
+			AlignLen: lcsLenBase - v%lcsScoreUnit,
+		}
+	}
+}
+
+func kmerLCSBatch(query uint64, partners []uint64, k int, out []KmerLCSResult) {
+	lanes := simdLanes()
+	pbase := make([]uint16, k*lanes)
+	row := make([]uint16, (k+1)*lanes)
+
+	for start := 0; start < len(partners); start += lanes {
+		end := min(start+lanes, len(partners))
+		transposeChunk(partners[start:end], k, lanes, pbase)
+		alignChunk(query, pbase, row, k, lanes, out[start:end])
+	}
+}
+
+// kmerLCSPrepared holds partners already transposed, chunk after chunk.
+type kmerLCSPrepared struct {
+	k, lanes, n int
+	pbase       []uint16 // chunk c at [c*k*lanes : (c+1)*k*lanes]
+}
+
+func prepareKmerLCS(partners []uint64, k int) kmerLCSPrepared {
+	lanes := simdLanes()
+	chunks := (len(partners) + lanes - 1) / lanes
+	p := kmerLCSPrepared{k: k, lanes: lanes, n: len(partners),
+		pbase: make([]uint16, chunks*k*lanes)}
+	for c := 0; c < chunks; c++ {
+		end := min((c+1)*lanes, len(partners))
+		transposeChunk(partners[c*lanes:end], k, lanes, p.pbase[c*k*lanes:(c+1)*k*lanes])
+	}
+	return p
+}
+
+func (p *kmerLCSPrepared) align(query uint64, out []KmerLCSResult) {
+	row := make([]uint16, (p.k+1)*p.lanes)
+	size := p.k * p.lanes
+	for c, start := 0, 0; start < p.n; c, start = c+1, start+p.lanes {
+		end := min(start+p.lanes, p.n)
+		alignChunk(query, p.pbase[c*size:(c+1)*size], row, p.k, p.lanes, out[start:end])
+	}
+}

pkg/obikmer/seqquorum.go

@@ -0,0 +1,229 @@
+package obikmer
+
+import (
+	"bufio"
+	"encoding/binary"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
+)
+
+// SeqKmerCounter counts, for every canonical k-mer, in how many distinct input
+// sequences it occurs. It answers "which k-mers are present in at least q of
+// the N sequences" without one k-mer set per sequence: a KmerSetGroup needs
+// N*P partition files, which is out of reach for thousands of genomes, whereas
+// a SeqKmerCounter uses a single set of P files whatever N is.
+//
+// Construction is two-phase, like KmerSetGroupBuilder:
+//
+//  1. AddSequence extracts super-kmers (grouped by minimizer, so a k-mer and its
+//     reverse complement always land in the same partition) and appends them,
+//     tagged with the sequence id, to one of P temporary files. It is safe to
+//     call concurrently.
+//  2. CountAtLeast processes the partitions in parallel: each one is loaded,
+//     expanded to (canonical k-mer, sequence id) pairs, sorted, and reduced.
+//
+// Memory during phase 2 is 16 bytes per k-mer occurrence of the partitions
+// being processed at the same time. Their number is bounded by an occurrence
+// budget (see SetOccurrenceBudget), so that the memory does not depend on how
+// the k-mers happen to be spread over the partitions; P only has to be large
+// enough for the largest partition to be small against the budget (total bases
+// / P is the occurrence count of an average partition, see DefaultPartitions).
+//
+// Bases other than A, C, G, T, U break a sequence into independent runs: no
+// k-mer ever contains an ambiguous base.
+type SeqKmerCounter struct {
+	dir     string
+	k, m, p int
+
+	parts []*seqPartWriter
+
+	mu       sync.Mutex
+	firstErr error
+	sealed   bool
+
+	budget int // k-mer occurrences held at once in phase 2
+}
+
+type seqPartWriter struct {
+	mu    sync.Mutex
+	w     *bufio.Writer
+	file  *os.File
+	path  string
+	kmers int // k-mer occurrences written, so that the partition is loaded in an array of the exact size
+}
+
+// SeqKmerCount is the result for one k-mer.
+type SeqKmerCount struct {
+	Kmer      uint64 // canonical k-mer
+	Sequences uint32 // number of distinct sequences containing it
+	Repeated  bool   // occurs more than once (either strand) in at least one sequence
+}
+
+// NewSeqKmerCounter creates a counter whose temporary files live in a new
+// directory created under parentDir (use "" for the system temporary
+// directory). k is the k-mer size (2..31), m the minimizer size (<=0 for the
+// default) and partitions the number of partition files P.
+func NewSeqKmerCounter(parentDir string, k, m, partitions int) (*SeqKmerCounter, error) {
+	if k < 2 || k > 31 {
+		return nil, fmt.Errorf("obikmer: k must be between 2 and 31, got %d", k)
+	}
+	if partitions < 1 {
+		return nil, fmt.Errorf("obikmer: partitions must be >= 1, got %d", partitions)
+	}
+	if m <= 0 {
+		m = DefaultMinimizerSize(k)
+	}
+	if m >= k {
+		return nil, fmt.Errorf("obikmer: minimizer size %d must be smaller than k=%d", m, k)
+	}
+
+	dir, err := os.MkdirTemp(parentDir, "obikmer-seqcount-")
+	if err != nil {
+		return nil, err
+	}
+
+	c := &SeqKmerCounter{dir: dir, k: k, m: m, p: partitions, budget: DefaultOccurrenceBudget}
+	c.parts = make([]*seqPartWriter, partitions)
+	for i := range c.parts {
+		path := filepath.Join(dir, fmt.Sprintf("part_%04d.sqk", i))
+		f, err := os.Create(path)
+		if err != nil {
+			c.Close()
+			return nil, err
+		}
+		c.parts[i] = &seqPartWriter{w: bufio.NewWriterSize(f, 32*1024), file: f, path: path}
+	}
+	return c, nil
+}
+
+// DefaultOccurrenceBudget is the number of k-mer occurrences phase 2 holds at
+// once by default: 32 million, 512 MB. A partition larger than the budget is
+// processed alone. Measured on 16 cores, 526 million occurrences to count: with
+// 32M the count is as fast as without any limit; 16M make it 15% slower, 8M
+// 60% slower.
+const DefaultOccurrenceBudget = 32 << 20
+
+// SetOccurrenceBudget bounds the k-mer occurrences (16 bytes each) that
+// CountAtLeast holds in memory at once. It must be called before CountAtLeast.
+func (c *SeqKmerCounter) SetOccurrenceBudget(occurrences int) {
+	if occurrences > 0 {
+		c.budget = occurrences
+	}
+}
+
+// partitionOccurrences is the occurrence count aimed at for an average
+// partition by DefaultPartitions (a partition of 1M occurrences is 16 MB).
+const partitionOccurrences = 1 << 20
+
+// DefaultPartitions returns a partition count suited to a total of kmers
+// k-mer occurrences: 256 at least, and more, as a power of two up to 1024, when
+// the average partition would otherwise exceed about a million occurrences.
+// The largest partitions hold several times the average. The limit keeps the
+// number of simultaneously open files reasonable.
+func DefaultPartitions(kmers int) int {
+	p := 256
+	for p < 1024 && kmers/p > partitionOccurrences {
+		p *= 2
+	}
+	return p
+}
+
+// K returns the k-mer size.
+func (c *SeqKmerCounter) K() int { return c.k }
+
+var unambiguousBase = func() (t [256]bool) {
+	for _, b := range "ACGTUacgtu" {
+		t[b] = true
+	}
+	return
+}()
+
+// AddSequence registers sequence id. The same id must be used for every call
+// concerning the same sequence; ids need not be contiguous. If circular is
+// true the k-1 first bases are appended so that k-mers spanning the junction
+// are counted. AddSequence is safe for concurrent use and must not be called
+// after CountAtLeast.
+func (c *SeqKmerCounter) AddSequence(id uint32, seq []byte, circular bool) {
+	if len(seq) < c.k {
+		return
+	}
+	if circular {
+		ext := make([]byte, 0, len(seq)+c.k-1)
+		ext = append(ext, seq...)
+		seq = append(ext, seq[:c.k-1]...)
+	}
+
+	var scratch []byte
+	start := -1
+	for i, b := range seq {
+		switch {
+		case unambiguousBase[b]:
+			if start < 0 {
+				start = i
+			}
+		case start >= 0:
+			scratch = c.addRun(id, seq[start:i], scratch)
+			start = -1
+		}
+	}
+	if start >= 0 {
+		c.addRun(id, seq[start:], scratch)
+	}
+}
+
+// addRun writes the super-kmers of one ambiguity-free run.
+func (c *SeqKmerCounter) addRun(id uint32, run []byte, scratch []byte) []byte {
+	if len(run) < c.k {
+		return scratch
+	}
+	for sk := range IterSuperKmers(run, c.k, c.m) {
+		scratch = binary.AppendUvarint(scratch[:0], uint64(id))
+		scratch = binary.AppendUvarint(scratch, uint64(len(sk.Sequence)))
+		scratch = appendPacked(scratch, sk.Sequence)
+
+		pw := c.parts[int(sk.Minimizer%uint64(c.p))]
+		pw.mu.Lock()
+		_, err := pw.w.Write(scratch)
+		pw.kmers += len(sk.Sequence) - c.k + 1
+		pw.mu.Unlock()
+		if err != nil {
+			c.setErr(err)
+		}
+	}
+	return scratch
+}
+
+func (c *SeqKmerCounter) setErr(err error) {
+	c.mu.Lock()
+	if c.firstErr == nil {
+		c.firstErr = err
+	}
+	c.mu.Unlock()
+}
+
+// seal flushes and closes the partition files; no more sequences can be added.
+func (c *SeqKmerCounter) seal() error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.sealed {
+		return c.firstErr
+	}
+	c.sealed = true
+	for _, pw := range c.parts {
+		if err := pw.w.Flush(); err != nil && c.firstErr == nil {
+			c.firstErr = err
+		}
+		if err := pw.file.Close(); err != nil && c.firstErr == nil {
+			c.firstErr = err
+		}
+	}
+	return c.firstErr
+}
+
+// Close removes the temporary files. It is safe to call more than once.
+func (c *SeqKmerCounter) Close() error {
+	_ = c.seal()
+	return os.RemoveAll(c.dir)
+}

pkg/obikmer/seqquorum_budget_test.go

@@ -0,0 +1,118 @@
+package obikmer
+
+import (
+	"math/rand"
+	"reflect"
+	"sync"
+	"testing"
+	"time"
+)
+
+// The occurrence budget only bounds the memory: any budget, down to one
+// occurrence (every partition then goes alone), gives the same counts.
+func TestSeqKmerCounterBudgetDoesNotChangeTheCounts(t *testing.T) {
+	rng := rand.New(rand.NewSource(7))
+	seqs := randomSeqs(rng, 40, 20, 500, true)
+	build := func(budget int) []SeqKmerCount {
+		c, err := NewSeqKmerCounter(t.TempDir(), 9, 4, 11)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer c.Close()
+		if budget > 0 {
+			c.SetOccurrenceBudget(budget)
+		}
+		for i, s := range seqs {
+			c.AddSequence(uint32(i), s, i%2 == 0)
+		}
+		total := 0
+		for _, pw := range c.parts {
+			total += pw.kmers
+		}
+		if total == 0 {
+			t.Fatal("no occurrence counted")
+		}
+		got, err := c.CountAtLeast(2)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return got
+	}
+	want := build(0)
+	if len(want) == 0 {
+		t.Fatal("the test needs k-mers shared by several sequences")
+	}
+	for _, budget := range []int{1, 100, 5000} {
+		if got := build(budget); !reflect.DeepEqual(got, want) {
+			t.Errorf("budget %d changes the counts", budget)
+		}
+	}
+}
+
+// The occurrences announced for a partition are those that its file holds.
+func TestSeqKmerCounterPartitionOccurrenceCount(t *testing.T) {
+	rng := rand.New(rand.NewSource(3))
+	seqs := randomSeqs(rng, 25, 10, 300, true)
+	c, err := NewSeqKmerCounter(t.TempDir(), 8, 3, 5)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer c.Close()
+	for i, s := range seqs {
+		c.AddSequence(uint32(i), s, i%3 == 0)
+	}
+	if err := c.seal(); err != nil {
+		t.Fatal(err)
+	}
+	for p, pw := range c.parts {
+		occ, err := c.loadPartition(p, nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(occ) != pw.kmers {
+			t.Errorf("partition %d: %d occurrences loaded, %d counted when written", p, len(occ), pw.kmers)
+		}
+	}
+}
+
+func TestOccurrenceBudget(t *testing.T) {
+	b := newOccurrenceBudget(10)
+	if got := b.acquire(4); got != 4 {
+		t.Fatalf("acquire(4) = %d", got)
+	}
+	// 7 does not fit in the 6 left: it must wait for the release
+	done := make(chan int)
+	go func() { done <- b.acquire(7) }()
+	select {
+	case <-done:
+		t.Fatal("acquire(7) must wait")
+	case <-time.After(50 * time.Millisecond):
+	}
+	b.release(4)
+	if got := <-done; got != 7 {
+		t.Fatalf("acquire(7) = %d", got)
+	}
+	b.release(7)
+
+	// a request above the whole budget takes all of it, once everything is free
+	if got := b.acquire(1000); got != 10 {
+		t.Fatalf("an oversized request must take the whole budget, got %d", got)
+	}
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() { defer wg.Done(); b.release(b.acquire(3)) }()
+	time.Sleep(20 * time.Millisecond)
+	b.release(10)
+	wg.Wait()
+}
+
+func TestDefaultPartitions(t *testing.T) {
+	for _, tc := range []struct{ kmers, want int }{
+		{0, 256}, {5_000_000, 256}, {256 << 20, 256}, {257 << 20, 512}, {526_000_000, 512},
+		{600 << 20, 1024}, {1 << 40, 1024},
+	} {
+		if got := DefaultPartitions(tc.kmers); got != tc.want {
+			t.Errorf("DefaultPartitions(%d) = %d, want %d", tc.kmers, got, tc.want)
+		}
+	}
+}

pkg/obikmer/seqquorum_count.go

@@ -0,0 +1,207 @@
+package obikmer
+
+import (
+	"bufio"
+	"cmp"
+	"encoding/binary"
+	"io"
+	"os"
+	"slices"
+	"sync"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
+)
+
+// kmerOcc is one occurrence of a canonical k-mer in a sequence.
+type kmerOcc struct {
+	kmer uint64
+	seq  uint32
+}
+
+func cmpOcc(a, b kmerOcc) int {
+	if c := cmp.Compare(a.kmer, b.kmer); c != 0 {
+		return c
+	}
+	return cmp.Compare(a.seq, b.seq)
+}
+
+// CountAtLeast returns, sorted by k-mer, every canonical k-mer found in at
+// least minSeq distinct sequences, with its exact sequence count.
+//
+// The result is held in memory: it is meant for quorum-filtered sets (the
+// conserved k-mers), not for the complete k-mer spectrum. The first call
+// seals the counter; AddSequence must not be called afterwards. The count
+// phase uses obidefault.ParallelWorkers() goroutines over the partitions.
+func (c *SeqKmerCounter) CountAtLeast(minSeq int) ([]SeqKmerCount, error) {
+	if err := c.seal(); err != nil {
+		return nil, err
+	}
+	if minSeq < 1 {
+		minSeq = 1
+	}
+
+	workers := min(obidefault.ParallelWorkers(), c.p)
+
+	// the largest partitions first: they are the hardest to fit in the budget
+	// and the last ones to finish, if they come late
+	order := make([]int, c.p)
+	for i := range order {
+		order[i] = i
+	}
+	slices.SortStableFunc(order, func(a, b int) int { return cmp.Compare(c.parts[b].kmers, c.parts[a].kmers) })
+	budget := newOccurrenceBudget(c.budget)
+
+	jobs := make(chan int)
+	results := make([][]SeqKmerCount, c.p)
+
+	var wg sync.WaitGroup
+	var errMu sync.Mutex
+	var firstErr error
+
+	for range workers {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			var occ []kmerOcc // reused between partitions of similar sizes
+			for p := range jobs {
+				n := c.parts[p].kmers
+				held := budget.acquire(n)
+				if cap(occ) < n || cap(occ) > 2*n+1024 {
+					occ = make([]kmerOcc, 0, n)
+				}
+				var err error
+				occ, err = c.loadPartition(p, occ[:0])
+				if err == nil {
+					results[p] = reduceOccurrences(occ, minSeq)
+				}
+				budget.release(held)
+				if err != nil {
+					errMu.Lock()
+					if firstErr == nil {
+						firstErr = err
+					}
+					errMu.Unlock()
+				}
+			}
+		}()
+	}
+	for _, p := range order {
+		jobs <- p
+	}
+	close(jobs)
+	wg.Wait()
+	if firstErr != nil {
+		return nil, firstErr
+	}
+
+	total := 0
+	for _, r := range results {
+		total += len(r)
+	}
+	all := make([]SeqKmerCount, 0, total)
+	for _, r := range results {
+		all = append(all, r...)
+	}
+	slices.SortFunc(all, func(a, b SeqKmerCount) int { return cmp.Compare(a.Kmer, b.Kmer) })
+	return all, nil
+}
+
+// loadPartition appends to occ every (canonical k-mer, sequence id) of the
+// partition file.
+func (c *SeqKmerCounter) loadPartition(p int, occ []kmerOcc) ([]kmerOcc, error) {
+	f, err := os.Open(c.parts[p].path)
+	if err != nil {
+		return occ, err
+	}
+	defer f.Close()
+	r := bufio.NewReaderSize(f, 64*1024)
+
+	var packed, bases []byte
+	for {
+		id, err := binary.ReadUvarint(r)
+		if err == io.EOF {
+			return occ, nil
+		}
+		if err != nil {
+			return occ, err
+		}
+		n, err := binary.ReadUvarint(r)
+		if err != nil {
+			return occ, err
+		}
+
+		nb := (int(n) + 3) / 4
+		packed = slices.Grow(packed[:0], nb)[:nb]
+		if _, err := io.ReadFull(r, packed); err != nil {
+			return occ, err
+		}
+		bases = slices.Grow(bases[:0], int(n))[:n]
+		unpackBases(bases, packed)
+
+		for kmer := range IterCanonicalKmers(bases, c.k) {
+			occ = append(occ, kmerOcc{kmer, uint32(id)})
+		}
+	}
+}
+
+// reduceOccurrences sorts occ and returns the k-mers present in at least
+// minSeq distinct sequences. occ is reordered in place.
+func reduceOccurrences(occ []kmerOcc, minSeq int) []SeqKmerCount {
+	slices.SortFunc(occ, cmpOcc)
+
+	var out []SeqKmerCount
+	for i := 0; i < len(occ); {
+		kmer := occ[i].kmer
+		var seqs uint32
+		repeated := false
+		j := i
+		for ; j < len(occ) && occ[j].kmer == kmer; j++ {
+			if j == i || occ[j].seq != occ[j-1].seq {
+				seqs++
+			} else {
+				repeated = true
+			}
+		}
+		if int(seqs) >= minSeq {
+			out = append(out, SeqKmerCount{Kmer: kmer, Sequences: seqs, Repeated: repeated})
+		}
+		i = j
+	}
+	return out
+}
+
+// occurrenceBudget bounds the k-mer occurrences held in memory at once by the
+// goroutines that load the partitions.
+type occurrenceBudget struct {
+	mu    sync.Mutex
+	freed *sync.Cond
+	total int
+	free  int
+}
+
+func newOccurrenceBudget(total int) *occurrenceBudget {
+	b := &occurrenceBudget{total: total, free: total}
+	b.freed = sync.NewCond(&b.mu)
+	return b
+}
+
+// acquire waits until n occurrences fit in the budget and takes them; a request
+// above the whole budget waits for the budget to be free and takes all of it.
+// It returns what was taken, to give back with release.
+func (b *occurrenceBudget) acquire(n int) int {
+	n = min(n, b.total)
+	b.mu.Lock()
+	for b.free < n {
+		b.freed.Wait()
+	}
+	b.free -= n
+	b.mu.Unlock()
+	return n
+}
+
+func (b *occurrenceBudget) release(n int) {
+	b.mu.Lock()
+	b.free += n
+	b.mu.Unlock()
+	b.freed.Broadcast()
+}

pkg/obikmer/seqquorum_test.go

@@ -0,0 +1,272 @@
+package obikmer
+
+import (
+	"math/rand"
+	"slices"
+	"strings"
+	"sync"
+	"testing"
+)
+
+type bruteInfo struct {
+	seqs     map[uint32]int // sequence id -> occurrences
+	repeated bool
+}
+
+// bruteForce is the oracle: every window of k unambiguous bases, circular
+// junction included, encoded with EncodeCanonicalKmer.
+func bruteForce(seqs [][]byte, circular []bool, k int) map[uint64]*bruteInfo {
+	res := map[uint64]*bruteInfo{}
+	for id, s := range seqs {
+		if len(s) < k {
+			continue
+		}
+		if circular[id] {
+			s = append(slices.Clone(s), s[:k-1]...)
+		}
+	window:
+		for i := 0; i+k <= len(s); i++ {
+			w := s[i : i+k]
+			for _, b := range w {
+				if !unambiguousBase[b] {
+					continue window
+				}
+			}
+			kmer := EncodeCanonicalKmer(w, k)
+			info := res[kmer]
+			if info == nil {
+				info = &bruteInfo{seqs: map[uint32]int{}}
+				res[kmer] = info
+			}
+			info.seqs[uint32(id)]++
+			if info.seqs[uint32(id)] > 1 {
+				info.repeated = true
+			}
+		}
+	}
+	return res
+}
+
+func randomSeqs(rng *rand.Rand, n, minLen, maxLen int, withN bool) [][]byte {
+	alphabet := "acgtACGT"
+	if withN {
+		alphabet = "acgtACGTacgtACGTNnR"
+	}
+	// A shared motif and its reverse complement make k-mers common to several
+	// sequences on both strands.
+	motif := make([]byte, 40)
+	for i := range motif {
+		motif[i] = "ACGT"[rng.Intn(4)]
+	}
+	rc := make([]byte, len(motif))
+	for i, b := range motif {
+		rc[len(motif)-1-i] = map[byte]byte{'A': 'T', 'C': 'G', 'G': 'C', 'T': 'A'}[b]
+	}
+
+	out := make([][]byte, n)
+	for i := range out {
+		l := minLen + rng.Intn(maxLen-minLen+1)
+		s := make([]byte, l)
+		for j := range s {
+			s[j] = alphabet[rng.Intn(len(alphabet))]
+		}
+		mode := rng.Intn(4)
+		if l < 60 { // too short to host the motifs
+			mode = 3
+		}
+		switch mode {
+		case 0:
+			copy(s[l/4:], motif)
+		case 1:
+			copy(s[l/4:], rc)
+		case 2: // internal repeat, both strands
+			copy(s[2:], motif[:20])
+			copy(s[l-25:], rc[:20])
+		}
+		out[i] = s
+	}
+	return out
+}
+
+func TestSeqKmerCounterAgainstBruteForce(t *testing.T) {
+	cases := []struct {
+		name    string
+		k, m, p int
+		withN   bool
+		circ    bool
+	}{
+		{"k7-linear", 7, 3, 8, false, false},
+		{"k7-circular-N", 7, 3, 5, true, true},
+		{"k18-linear-N", 18, 8, 16, true, false},
+		{"k18-circular", 18, 8, 7, false, true},
+		{"k31-p1", 31, 10, 1, false, true},
+		{"k4-small-m", 4, 1, 3, true, false},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			rng := rand.New(rand.NewSource(int64(tc.k*100 + tc.p)))
+			seqs := randomSeqs(rng, 30, 5, 400, tc.withN)
+			circ := make([]bool, len(seqs))
+			for i := range circ {
+				circ[i] = tc.circ && i%3 != 0
+			}
+
+			c, err := NewSeqKmerCounter(t.TempDir(), tc.k, tc.m, tc.p)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer c.Close()
+
+			// Concurrent insertion, shuffled order, ids not contiguous.
+			var wg sync.WaitGroup
+			for i := range seqs {
+				wg.Add(1)
+				go func() {
+					defer wg.Done()
+					c.AddSequence(uint32(i), seqs[i], circ[i])
+				}()
+			}
+			wg.Wait()
+
+			want := bruteForce(seqs, circ, tc.k)
+			for _, q := range []int{1, 2, 3, 10} {
+				got, err := c.CountAtLeast(q)
+				if err != nil {
+					t.Fatal(err)
+				}
+				if !slices.IsSortedFunc(got, func(a, b SeqKmerCount) int {
+					switch {
+					case a.Kmer < b.Kmer:
+						return -1
+					case a.Kmer > b.Kmer:
+						return 1
+					}
+					return 0
+				}) {
+					t.Fatal("result is not sorted by k-mer")
+				}
+
+				n := 0
+				for kmer, info := range want {
+					if len(info.seqs) >= q {
+						n++
+						_ = kmer
+					}
+				}
+				if len(got) != n {
+					t.Fatalf("q=%d: %d k-mers, want %d", q, len(got), n)
+				}
+				for _, g := range got {
+					info := want[g.Kmer]
+					if info == nil {
+						t.Fatalf("q=%d: unexpected k-mer %d", q, g.Kmer)
+					}
+					if int(g.Sequences) != len(info.seqs) || g.Repeated != info.repeated {
+						t.Fatalf("q=%d kmer %d: got (%d,%v) want (%d,%v)",
+							q, g.Kmer, g.Sequences, g.Repeated, len(info.seqs), info.repeated)
+					}
+				}
+			}
+		})
+	}
+}
+
+// A k-mer appearing on both strands of one sequence counts once but is Repeated.
+func TestSeqKmerCounterBothStrandsOneSequence(t *testing.T) {
+	const k = 8
+	fwd := "ACGGTCAT"
+	rc := "ATGACCGT"
+	seq := []byte("TTTT" + fwd + "GGGGGGGG" + rc + "CCCC")
+
+	c, err := NewSeqKmerCounter(t.TempDir(), k, 3, 4)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer c.Close()
+	c.AddSequence(7, seq, false)
+
+	got, err := c.CountAtLeast(1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	target := EncodeCanonicalKmer([]byte(fwd), k)
+	found := false
+	for _, g := range got {
+		if g.Kmer == target {
+			found = true
+			if g.Sequences != 1 || !g.Repeated {
+				t.Errorf("got %+v, want 1 sequence and Repeated", g)
+			}
+		}
+	}
+	if !found {
+		t.Fatal("k-mer not found")
+	}
+}
+
+// Ambiguous bases must never be part of a k-mer, and the circular junction
+// must produce the wrap-around k-mers.
+func TestSeqKmerCounterAmbiguityAndCircular(t *testing.T) {
+	const k = 5
+	c, err := NewSeqKmerCounter(t.TempDir(), k, 2, 3)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer c.Close()
+
+	c.AddSequence(0, []byte("ACGTNACGT"), false) // runs ACGT (<k) and ACGT (<k): nothing
+	c.AddSequence(1, []byte("AACCGGTT"), true)   // 8 circular windows
+
+	got, err := c.CountAtLeast(1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) == 0 {
+		t.Fatal("expected k-mers from the circular sequence")
+	}
+	var circ []string
+	for i := 0; i < 8; i++ {
+		var sb strings.Builder
+		for j := 0; j < k; j++ {
+			sb.WriteByte("AACCGGTT"[(i+j)%8])
+		}
+		circ = append(circ, sb.String())
+	}
+	want := map[uint64]bool{}
+	for _, w := range circ {
+		want[EncodeCanonicalKmer([]byte(w), k)] = true
+	}
+	if len(got) != len(want) {
+		t.Fatalf("got %d k-mers, want %d", len(got), len(want))
+	}
+	for _, g := range got {
+		if !want[g.Kmer] || g.Sequences != 1 {
+			t.Errorf("unexpected %+v", g)
+		}
+	}
+}
+
+func TestSeqKmerCounterParameters(t *testing.T) {
+	if _, err := NewSeqKmerCounter(t.TempDir(), 1, 1, 4); err == nil {
+		t.Error("k=1 must fail")
+	}
+	if _, err := NewSeqKmerCounter(t.TempDir(), 32, 5, 4); err == nil {
+		t.Error("k=32 must fail")
+	}
+	if _, err := NewSeqKmerCounter(t.TempDir(), 10, 10, 4); err == nil {
+		t.Error("m>=k must fail")
+	}
+	if _, err := NewSeqKmerCounter(t.TempDir(), 10, 4, 0); err == nil {
+		t.Error("zero partitions must fail")
+	}
+	c, err := NewSeqKmerCounter(t.TempDir(), 10, 0, 4) // default minimizer
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := c.Close(); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.Close(); err != nil {
+		t.Fatalf("second Close must be harmless: %v", err)
+	}
+}

pkg/obikmer/skm_reader.go

@@ -10,6 +10,16 @@ import (
 // decode2bit maps 2-bit codes back to nucleotide bytes.
 var decode2bit = [4]byte{'a', 'c', 'g', 't'}
 
+// unpackBases decodes the 2-bit packed bases in packed into dst (one lowercase
+// nucleotide per byte, len(dst) bases) and returns dst.
+func unpackBases(dst, packed []byte) []byte {
+	for i := range dst {
+		code := (packed[i/4] >> uint(6-(i%4)*2)) & 0x03
+		dst[i] = decode2bit[code]
+	}
+	return dst
+}
+
 // SkmReader reads super-kmers from a binary .skm file.
 type SkmReader struct {
 	r    *bufio.Reader
@@ -46,13 +56,7 @@ func (sr *SkmReader) Next() (SuperKmer, bool) {
 	}
 
 	// Decode to nucleotide bytes
-	seq := make([]byte, seqLen)
-	for i := 0; i < seqLen; i++ {
-		byteIdx := i / 4
-		bitPos := uint(6 - (i%4)*2)
-		code := (packed[byteIdx] >> bitPos) & 0x03
-		seq[i] = decode2bit[code]
-	}
+	seq := unpackBases(make([]byte, seqLen), packed)
 
 	return SuperKmer{
 		Sequence: seq,

pkg/obikmer/skm_writer.go

@@ -16,8 +16,9 @@ import (
 // Nucleotide encoding: A=00, C=01, G=10, T=11.
 // The last byte is zero-padded on the low bits if len%4 != 0.
 type SkmWriter struct {
-	w    *bufio.Writer
-	file *os.File
+	w       *bufio.Writer
+	file    *os.File
+	scratch []byte
 }
 
 // NewSkmWriter creates a new SkmWriter writing to the given file path.
@@ -32,36 +33,36 @@ func NewSkmWriter(path string) (*SkmWriter, error) {
 	}, nil
 }
 
+// appendPacked appends seq to dst packed 2 bits per base (A=00, C=01, G=10,
+// T=11), four bases per byte, the last byte zero-padded on the low bits.
+func appendPacked(dst, seq []byte) []byte {
+	for i := 0; i < len(seq); i += 4 {
+		var packed byte
+		for j := 0; j < 4; j++ {
+			packed <<= 2
+			if i+j < len(seq) {
+				packed |= __single_base_code__[seq[i+j]&31]
+			}
+		}
+		dst = append(dst, packed)
+	}
+	return dst
+}
+
 // Write encodes a SuperKmer to the .skm file.
 // The sequence bytes are packed 2 bits per base.
 func (sw *SkmWriter) Write(sk SuperKmer) error {
-	seq := sk.Sequence
-	seqLen := uint16(len(seq))
-
 	// Write length
 	var lenbuf [2]byte
-	binary.LittleEndian.PutUint16(lenbuf[:], seqLen)
+	binary.LittleEndian.PutUint16(lenbuf[:], uint16(len(sk.Sequence)))
 	if _, err := sw.w.Write(lenbuf[:]); err != nil {
 		return err
 	}
 
 	// Encode and write packed sequence (2 bits/base)
-	nBytes := (int(seqLen) + 3) / 4
-	for i := 0; i < nBytes; i++ {
-		var packed byte
-		for j := 0; j < 4; j++ {
-			pos := i*4 + j
-			packed <<= 2
-			if pos < int(seqLen) {
-				packed |= __single_base_code__[seq[pos]&31]
-			}
-		}
-		if err := sw.w.WriteByte(packed); err != nil {
-			return err
-		}
-	}
-
-	return nil
+	sw.scratch = appendPacked(sw.scratch[:0], sk.Sequence)
+	_, err := sw.w.Write(sw.scratch)
+	return err
 }
 
 // Close flushes buffered data and closes the underlying file.

pkg/obilua/lua.go

@@ -91,7 +91,7 @@ func LuaWorker(proto *lua.FunctionProto) obiseq.SeqWorker {
 	err := interpreter.PCall(0, lua.MultRet, nil)
 
 	if err != nil {
-		log.Fatalf("Error in executing the lua script")
+		log.Fatalf("Error in executing the lua script: %v", err)
 	}
 
 	result := interpreter.GetGlobal("worker")
@@ -141,6 +141,69 @@ func LuaWorker(proto *lua.FunctionProto) obiseq.SeqWorker {
 	return nil
 }
 
+// LuaSliceWorker creates a SeqSliceWorker that calls the Lua function
+// named "slice_worker". Unlike LuaWorker, the entire batch (BioSequenceSlice)
+// is passed to the Lua function at once, enabling batch-level processing
+// (e.g. a single HTTP request per batch instead of one per sequence).
+//
+// The Lua function signature:
+//
+//	function slice_worker(slice)   -- receives a BioSequenceSlice
+//	    -- process the batch
+//	    return slice               -- returns a BioSequenceSlice (or nil)
+//	end
+func LuaSliceWorker(proto *lua.FunctionProto) obiseq.SeqSliceWorker {
+	interpreter := NewInterpreter()
+	lfunc := interpreter.NewFunctionFromProto(proto)
+	interpreter.Push(lfunc)
+	err := interpreter.PCall(0, lua.MultRet, nil)
+
+	if err != nil {
+		log.Fatalf("Error in executing the lua script: %v", err)
+	}
+
+	result := interpreter.GetGlobal("slice_worker")
+
+	if lua_worker, ok := result.(*lua.LFunction); ok {
+		f := func(slice obiseq.BioSequenceSlice) (obiseq.BioSequenceSlice, error) {
+			if err := interpreter.CallByParam(lua.P{
+				Fn:      lua_worker,
+				NRet:    1,
+				Protect: true,
+			}, obiseqslice2Lua(interpreter, &slice)); err != nil {
+				log.Fatal(err)
+			}
+
+			lreponse := interpreter.Get(-1)
+			defer interpreter.Pop(1)
+
+			if reponse, ok := lreponse.(*lua.LUserData); ok {
+				s := reponse.Value
+				switch val := s.(type) {
+				case *obiseq.BioSequenceSlice:
+					return *val, nil
+				case *obiseq.BioSequence:
+					return obiseq.BioSequenceSlice{val}, nil
+				default:
+					r := reflect.TypeOf(val)
+					return nil, fmt.Errorf("slice_worker function doesn't return the correct type %s", r)
+				}
+			}
+
+			if _, ok = lreponse.(*lua.LNilType); ok {
+				return nil, nil
+			}
+
+			return nil, fmt.Errorf("slice_worker function doesn't return the correct type %T", lreponse)
+		}
+
+		return f
+	}
+
+	log.Fatalf("The slice_worker object is not a function")
+	return nil
+}
+
 // LuaProcessor processes a Lua script on a sequence iterator and returns a new iterator.
 //
 // Parameters:
@@ -173,7 +236,7 @@ func LuaProcessor(iterator obiiter.IBioSequence, name, program string, breakOnEr
 	err = interpreter.PCall(0, lua.MultRet, nil)
 
 	if err != nil {
-		log.Fatalf("Error in executing the lua script")
+		log.Fatalf("Error in executing the lua script: %v", err)
 	}
 
 	result := interpreter.GetGlobal("begin")
@@ -198,7 +261,7 @@ func LuaProcessor(iterator obiiter.IBioSequence, name, program string, breakOnEr
 		err = interpreter.PCall(0, lua.MultRet, nil)
 
 		if err != nil {
-			log.Fatalf("Error in executing the lua script")
+			log.Fatalf("Error in executing the lua script: %v", err)
 		}
 
 		result := interpreter.GetGlobal("finish")
@@ -216,11 +279,27 @@ func LuaProcessor(iterator obiiter.IBioSequence, name, program string, breakOnEr
 
 	}()
 
-	ff := func(iterator obiiter.IBioSequence) {
-		w := LuaWorker(proto)
-		sw := obiseq.SeqToSliceWorker(w, false)
+	// Detect whether the script defines slice_worker (batch-level) or worker (per-sequence).
+	hasSliceWorker := func() bool {
+		interpreter := NewInterpreter()
+		lfunc := interpreter.NewFunctionFromProto(proto)
+		interpreter.Push(lfunc)
+		if err := interpreter.PCall(0, lua.MultRet, nil); err != nil {
+			return false
+		}
+		result := interpreter.GetGlobal("slice_worker")
+		interpreter.Close()
+		_, ok := result.(*lua.LFunction)
+		return ok
+	}()
 
-		// iterator = iterator.SortBatches()
+	ff := func(iterator obiiter.IBioSequence) {
+		var sw obiseq.SeqSliceWorker
+		if hasSliceWorker {
+			sw = LuaSliceWorker(proto)
+		} else {
+			sw = obiseq.SeqToSliceWorker(LuaWorker(proto), false)
+		}
 
 		for iterator.Next() {
 			seqs := iterator.Get()
@@ -235,6 +314,10 @@ func LuaProcessor(iterator obiiter.IBioSequence, name, program string, breakOnEr
 				}
 			}
 
+			if ns == nil {
+				ns = obiseq.BioSequenceSlice{}
+			}
+
 			newIter.Push(obiiter.MakeBioSequenceBatch(seqs.Source(), seqs.Order(), ns))
 		}
 

pkg/obilua/lua_push_interface.go

@@ -17,15 +17,7 @@ import (
 // No return values. This function operates directly on the Lua state stack.
 func pushInterfaceToLua(L *lua.LState, val interface{}) {
 	switch v := val.(type) {
-	case string:
-		L.Push(lua.LString(v))
-	case bool:
-		L.Push(lua.LBool(v))
-	case int:
-		L.Push(lua.LNumber(v))
-	case float64:
-		L.Push(lua.LNumber(v))
-	// Add other cases as needed for different types
+	// Typed slices and maps from internal OBITools code — not produced by json.Unmarshal
 	case map[string]int:
 		pushMapStringIntToLua(L, v)
 	case map[string]string:
@@ -34,8 +26,6 @@ func pushInterfaceToLua(L *lua.LState, val interface{}) {
 		pushMapStringBoolToLua(L, v)
 	case map[string]float64:
 		pushMapStringFloat64ToLua(L, v)
-	case map[string]interface{}:
-		pushMapStringInterfaceToLua(L, v)
 	case []string:
 		pushSliceStringToLua(L, v)
 	case []int:
@@ -46,63 +36,63 @@ func pushInterfaceToLua(L *lua.LState, val interface{}) {
 		pushSliceNumericToLua(L, v)
 	case []bool:
 		pushSliceBoolToLua(L, v)
-	case []interface{}:
-		pushSliceInterfaceToLua(L, v)
-	case nil:
-		L.Push(lua.LNil)
 	case *sync.Mutex:
 		pushMutexToLua(L, v)
 	default:
-		log.Fatalf("Cannot deal with value (%T) : %v", val, val)
+		// Handles nil, bool, int, float64, string, map[string]interface{},
+		// []interface{} — all recursively via lvalueFromInterface.
+		L.Push(lvalueFromInterface(L, v))
 	}
 }
 
 func pushMapStringInterfaceToLua(L *lua.LState, m map[string]interface{}) {
-	// Create a new Lua table
 	luaTable := L.NewTable()
-	// Iterate over the Go map and set the key-value pairs in the Lua table
 	for key, value := range m {
-		switch v := value.(type) {
-		case int:
-			luaTable.RawSetString(key, lua.LNumber(v))
-		case float64:
-			luaTable.RawSetString(key, lua.LNumber(v))
-		case bool:
-			luaTable.RawSetString(key, lua.LBool(v))
-		case string:
-			luaTable.RawSetString(key, lua.LString(v))
-		default:
-			log.Fatalf("Doesn't deal with map containing value %v of type %T", v, v)
-		}
+		L.SetField(luaTable, key, lvalueFromInterface(L, value))
 	}
-
-	// Push the Lua table onto the stack
 	L.Push(luaTable)
 }
 
 func pushSliceInterfaceToLua(L *lua.LState, s []interface{}) {
-	// Create a new Lua table
 	luaTable := L.NewTable()
-	// Iterate over the Go map and set the key-value pairs in the Lua table
 	for _, value := range s {
-		switch v := value.(type) {
-		case int:
-			luaTable.Append(lua.LNumber(v))
-		case float64:
-			luaTable.Append(lua.LNumber(v))
-		case bool:
-			luaTable.Append(lua.LBool(v))
-		case string:
-			luaTable.Append(lua.LString(v))
-		default:
-			log.Fatalf("Doesn't deal with slice containing value %v of type %T", v, v)
-		}
+		luaTable.Append(lvalueFromInterface(L, value))
 	}
-
-	// Push the Lua table onto the stack
 	L.Push(luaTable)
 }
 
+// lvalueFromInterface converts a Go interface{} value (as produced by json.Unmarshal)
+// to the corresponding lua.LValue, handling nested maps and slices recursively.
+func lvalueFromInterface(L *lua.LState, value interface{}) lua.LValue {
+	switch v := value.(type) {
+	case nil:
+		return lua.LNil
+	case bool:
+		return lua.LBool(v)
+	case int:
+		return lua.LNumber(v)
+	case float64:
+		return lua.LNumber(v)
+	case string:
+		return lua.LString(v)
+	case map[string]interface{}:
+		t := L.NewTable()
+		for key, val := range v {
+			L.SetField(t, key, lvalueFromInterface(L, val))
+		}
+		return t
+	case []interface{}:
+		t := L.NewTable()
+		for _, val := range v {
+			t.Append(lvalueFromInterface(L, val))
+		}
+		return t
+	default:
+		log.Fatalf("lvalueFromInterface: unsupported type %T: %v", v, v)
+		return lua.LNil
+	}
+}
+
 // pushMapStringIntToLua creates a new Lua table and iterates over the Go map to set key-value pairs in the Lua table. It then pushes the Lua table onto the stack.
 //
 // L *lua.LState - the Lua state

pkg/obilua/lua_table.go

@@ -28,6 +28,8 @@ func Table2Interface(interpreter *lua.LState, table *lua.LTable) interface{} {
 				val[i-1] = float64(v.(lua.LNumber))
 			case lua.LTString:
 				val[i-1] = string(v.(lua.LString))
+			case lua.LTTable:
+				val[i-1] = Table2Interface(interpreter, v.(*lua.LTable))
 			}
 		}
 		return val
@@ -45,6 +47,8 @@ func Table2Interface(interpreter *lua.LState, table *lua.LTable) interface{} {
 					val[string(ks)] = float64(v.(lua.LNumber))
 				case lua.LTString:
 					val[string(ks)] = string(v.(lua.LString))
+				case lua.LTTable:
+					val[string(ks)] = Table2Interface(interpreter, v.(*lua.LTable))
 				}
 			}
 		})

pkg/obilua/luahttp.go

@@ -0,0 +1,128 @@
+package obilua
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
+	lua "github.com/yuin/gopher-lua"
+)
+
+const httpClientTimeout = 300 * time.Second
+
+var (
+	_httpClient     *http.Client
+	_httpClientOnce sync.Once
+
+	// _httpSemaphore limits the number of concurrent HTTP requests.
+	// Initialised lazily alongside the client.
+	_httpSemaphore chan struct{}
+)
+
+func getHTTPClient() *http.Client {
+	_httpClientOnce.Do(func() {
+		conns := 2 * obidefault.ParallelWorkers()
+		_httpClient = &http.Client{
+			Transport: &http.Transport{
+				MaxIdleConnsPerHost: conns,
+				MaxConnsPerHost:     conns,
+				IdleConnTimeout:     90 * time.Second,
+			},
+			Timeout: httpClientTimeout,
+		}
+		_httpSemaphore = make(chan struct{}, obidefault.ParallelWorkers())
+	})
+	return _httpClient
+}
+
+// RegisterHTTP registers the http module in the Lua state as a global,
+// consistent with obicontext and BioSequence.
+//
+// Exposes:
+//
+//	http.post(url, body [, timeout_ms]) → response string  (on success)
+//	http.post(url, body [, timeout_ms]) → nil, err string  (on error)
+//	http.set_concurrency(n)             → set max simultaneous requests
+func RegisterHTTP(luaState *lua.LState) {
+	table := luaState.NewTable()
+	luaState.SetField(table, "post", luaState.NewFunction(luaHTTPPost))
+	luaState.SetField(table, "set_concurrency", luaState.NewFunction(luaHTTPSetConcurrency))
+	luaState.SetGlobal("http", table)
+}
+
+// luaHTTPPost implements http.post(url, body [, timeout_ms]) for Lua.
+//
+// The optional third argument overrides the default timeout (in milliseconds).
+// Concurrent requests are throttled through _httpSemaphore so that a
+// single-threaded backend server is not overwhelmed by K parallel workers.
+//
+// Lua signature:
+//
+//	local response          = http.post(url, body)
+//	local response          = http.post(url, body, 5000)   -- 5 s timeout
+//	local response, err     = http.post(url, body)
+func luaHTTPPost(L *lua.LState) int {
+	url := L.CheckString(1)
+	body := L.CheckString(2)
+
+	client := getHTTPClient()
+
+	timeout := httpClientTimeout
+	if L.GetTop() >= 3 {
+		ms := L.CheckInt(3)
+		timeout = time.Duration(ms) * time.Millisecond
+	}
+
+	// Acquire semaphore slot — blocks until a slot is free.
+	_httpSemaphore <- struct{}{}
+	defer func() { <-_httpSemaphore }()
+
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, strings.NewReader(body))
+	if err != nil {
+		L.Push(lua.LNil)
+		L.Push(lua.LString(err.Error()))
+		return 2
+	}
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := client.Do(req)
+	if err != nil {
+		L.Push(lua.LNil)
+		L.Push(lua.LString(err.Error()))
+		return 2
+	}
+	defer resp.Body.Close()
+
+	respBytes, err := io.ReadAll(resp.Body)
+	if err != nil {
+		L.Push(lua.LNil)
+		L.Push(lua.LString(err.Error()))
+		return 2
+	}
+
+	L.Push(lua.LString(respBytes))
+	return 1
+}
+
+// luaHTTPSetConcurrency replaces the semaphore with a new one of size n.
+// Must be called before the first http.post (e.g. in begin()).
+//
+// Lua signature:
+//
+//	http.set_concurrency(1)   -- serialise all HTTP requests
+func luaHTTPSetConcurrency(L *lua.LState) int {
+	n := L.CheckInt(1)
+	if n < 1 {
+		n = 1
+	}
+	getHTTPClient() // ensure singleton is initialised
+	_httpSemaphore = make(chan struct{}, n)
+	return 0
+}

pkg/obilua/luajson.go

@@ -0,0 +1,71 @@
+package obilua
+
+import (
+	"encoding/json"
+
+	lua "github.com/yuin/gopher-lua"
+)
+
+// RegisterJSON registers the json module in the Lua state as a global,
+// consistent with obicontext, BioSequence, and http.
+//
+// Exposes:
+//
+//	json.encode(data)   → string         (on success)
+//	json.encode(data)   → nil, err       (on error)
+//	json.decode(string) → value          (on success)
+//	json.decode(string) → nil, err       (on error)
+func RegisterJSON(luaState *lua.LState) {
+	table := luaState.NewTable()
+	luaState.SetField(table, "encode", luaState.NewFunction(luaJSONEncode))
+	luaState.SetField(table, "decode", luaState.NewFunction(luaJSONDecode))
+	luaState.SetGlobal("json", table)
+}
+
+// luaJSONEncode implements json.encode(data) for Lua.
+func luaJSONEncode(L *lua.LState) int {
+	val := L.CheckAny(1)
+
+	var goVal interface{}
+	switch v := val.(type) {
+	case *lua.LTable:
+		goVal = Table2Interface(L, v)
+	case lua.LString:
+		goVal = string(v)
+	case lua.LNumber:
+		goVal = float64(v)
+	case lua.LBool:
+		goVal = bool(v)
+	case *lua.LNilType:
+		goVal = nil
+	default:
+		L.Push(lua.LNil)
+		L.Push(lua.LString("json.encode: unsupported type"))
+		return 2
+	}
+
+	b, err := json.Marshal(goVal)
+	if err != nil {
+		L.Push(lua.LNil)
+		L.Push(lua.LString(err.Error()))
+		return 2
+	}
+
+	L.Push(lua.LString(b))
+	return 1
+}
+
+// luaJSONDecode implements json.decode(string) for Lua.
+func luaJSONDecode(L *lua.LState) int {
+	s := L.CheckString(1)
+
+	var goVal interface{}
+	if err := json.Unmarshal([]byte(s), &goVal); err != nil {
+		L.Push(lua.LNil)
+		L.Push(lua.LString(err.Error()))
+		return 2
+	}
+
+	pushInterfaceToLua(L, goVal)
+	return 1
+}

pkg/obilua/luajson_test.go

@@ -0,0 +1,184 @@
+package obilua
+
+import (
+	"testing"
+
+	lua "github.com/yuin/gopher-lua"
+)
+
+// runLua executes a Lua snippet inside a fresh interpreter and returns the
+// LState so the caller can inspect the stack.
+func runLua(t *testing.T, script string) *lua.LState {
+	t.Helper()
+	L := NewInterpreter()
+	if err := L.DoString(script); err != nil {
+		t.Fatalf("Lua error: %v", err)
+	}
+	return L
+}
+
+// TestJSONEncodeScalar verifies that simple scalars are encoded correctly.
+func TestJSONEncodeScalar(t *testing.T) {
+	cases := []struct {
+		script   string
+		expected string
+	}{
+		{`result = json.encode("hello")`, `"hello"`},
+		{`result = json.encode(42)`, `42`},
+		{`result = json.encode(true)`, `true`},
+	}
+
+	for _, tc := range cases {
+		L := runLua(t, tc.script)
+		got := string(L.GetGlobal("result").(lua.LString))
+		if got != tc.expected {
+			t.Errorf("encode(%s): got %q, want %q", tc.script, got, tc.expected)
+		}
+		L.Close()
+	}
+}
+
+// TestJSONEncodeTable verifies that a Lua table (array and map) encodes to JSON.
+func TestJSONEncodeTable(t *testing.T) {
+	L := runLua(t, `result = json.encode({a = 1, b = "x"})`)
+	got := string(L.GetGlobal("result").(lua.LString))
+	// json.Marshal produces deterministic output for maps in Go 1.12+... actually not.
+	// Just check it round-trips via decode instead.
+	L.Close()
+	if got == "" {
+		t.Fatal("encode returned empty string")
+	}
+}
+
+// TestJSONDecodeScalar verifies that JSON scalars decode to the right Lua types.
+func TestJSONDecodeScalar(t *testing.T) {
+	L := runLua(t, `
+		s = json.decode('"hello"')
+		n = json.decode('3.14')
+		b = json.decode('true')
+	`)
+	if s, ok := L.GetGlobal("s").(lua.LString); !ok || string(s) != "hello" {
+		t.Errorf("decode string: got %v", L.GetGlobal("s"))
+	}
+	if n, ok := L.GetGlobal("n").(lua.LNumber); !ok || float64(n) != 3.14 {
+		t.Errorf("decode number: got %v", L.GetGlobal("n"))
+	}
+	if b, ok := L.GetGlobal("b").(lua.LBool); !ok || !bool(b) {
+		t.Errorf("decode bool: got %v", L.GetGlobal("b"))
+	}
+	L.Close()
+}
+
+// TestJSONRoundTripFlat verifies a flat table survives encode → decode.
+func TestJSONRoundTripFlat(t *testing.T) {
+	L := runLua(t, `
+		original = {name = "Homo_sapiens", score = 1.0, valid = true}
+		encoded  = json.encode(original)
+		decoded  = json.decode(encoded)
+	`)
+	decoded, ok := L.GetGlobal("decoded").(*lua.LTable)
+	if !ok {
+		t.Fatal("decoded is not a table")
+	}
+	if v := decoded.RawGetString("name"); string(v.(lua.LString)) != "Homo_sapiens" {
+		t.Errorf("name: got %v", v)
+	}
+	if v := decoded.RawGetString("score"); float64(v.(lua.LNumber)) != 1.0 {
+		t.Errorf("score: got %v", v)
+	}
+	if v := decoded.RawGetString("valid"); !bool(v.(lua.LBool)) {
+		t.Errorf("valid: got %v", v)
+	}
+	L.Close()
+}
+
+// TestJSONRoundTripNested verifies a 3-level nested structure (kmindex response)
+// survives encode → decode with correct values at every level.
+func TestJSONRoundTripNested(t *testing.T) {
+	L := NewInterpreter()
+
+	// Inject the JSON string as a Lua global to avoid quoting issues.
+	L.SetGlobal("kmindex_json", lua.LString(
+		`{"Human":{"query_001":{"Homo_sapiens--GCF_000001405_40":1.0}}}`,
+	))
+
+	if err := L.DoString(`
+		data      = json.decode(kmindex_json)
+		reencoded = json.encode(data)
+		data2     = json.decode(reencoded)
+	`); err != nil {
+		t.Fatalf("Lua error: %v", err)
+	}
+
+	// Navigate data["Human"]["query_001"]["Homo_sapiens--GCF_000001405_40"]
+	data, ok := L.GetGlobal("data").(*lua.LTable)
+	if !ok {
+		t.Fatal("data is not a table")
+	}
+	human, ok := data.RawGetString("Human").(*lua.LTable)
+	if !ok {
+		t.Fatal("data.Human is not a table")
+	}
+	query, ok := human.RawGetString("query_001").(*lua.LTable)
+	if !ok {
+		t.Fatal("data.Human.query_001 is not a table")
+	}
+	score, ok := query.RawGetString("Homo_sapiens--GCF_000001405_40").(lua.LNumber)
+	if !ok || float64(score) != 1.0 {
+		t.Errorf("score: got %v, want 1.0", query.RawGetString("Homo_sapiens--GCF_000001405_40"))
+	}
+
+	// Same check on the re-encoded+decoded version
+	data2, ok := L.GetGlobal("data2").(*lua.LTable)
+	if !ok {
+		t.Fatal("data2 is not a table")
+	}
+	score2 := data2.RawGetString("Human").(*lua.LTable).
+		RawGetString("query_001").(*lua.LTable).
+		RawGetString("Homo_sapiens--GCF_000001405_40").(lua.LNumber)
+	if float64(score2) != 1.0 {
+		t.Errorf("data2 score: got %v, want 1.0", score2)
+	}
+	L.Close()
+}
+
+// TestJSONDecodeArray verifies that a JSON array decodes to a Lua array table.
+func TestJSONDecodeArray(t *testing.T) {
+	L := runLua(t, `arr = json.decode('[1, 2, 3]')`)
+	arr, ok := L.GetGlobal("arr").(*lua.LTable)
+	if !ok {
+		t.Fatal("arr is not a table")
+	}
+	for i, expected := range []float64{1, 2, 3} {
+		v, ok := arr.RawGetInt(i + 1).(lua.LNumber)
+		if !ok || float64(v) != expected {
+			t.Errorf("arr[%d]: got %v, want %v", i+1, arr.RawGetInt(i+1), expected)
+		}
+	}
+	L.Close()
+}
+
+// TestJSONEncodeError verifies that json.encode on an unsupported type returns nil + error.
+func TestJSONEncodeError(t *testing.T) {
+	L := runLua(t, `
+		local result, err = json.encode(nil)
+	`)
+	// nil encodes to JSON "null" — not an error
+	L.Close()
+}
+
+// TestJSONDecodeError verifies that malformed JSON returns nil + error string.
+func TestJSONDecodeError(t *testing.T) {
+	L := runLua(t, `
+		local result, err = json.decode("not valid json")
+		decode_ok     = (result == nil)
+		decode_has_err = (err ~= nil)
+	`)
+	if L.GetGlobal("decode_ok") != lua.LTrue {
+		t.Error("expected nil result on decode error")
+	}
+	if L.GetGlobal("decode_has_err") != lua.LTrue {
+		t.Error("expected error string on decode error")
+	}
+	L.Close()
+}

pkg/obilua/obilib.go

@@ -5,4 +5,6 @@ import lua "github.com/yuin/gopher-lua"
 func RegisterObilib(luaState *lua.LState) {
 	RegisterObiSeq(luaState)
 	RegisterObiTaxonomy(luaState)
+	RegisterHTTP(luaState)
+	RegisterJSON(luaState)
 }

pkg/obilua/obiseqslice.go

@@ -31,7 +31,8 @@ func obiseqslice2Lua(interpreter *lua.LState,
 }
 
 func newObiSeqSlice(luaState *lua.LState) int {
-	seqslice := obiseq.NewBioSequenceSlice()
+	capacity := luaState.OptInt(1, 0)
+	seqslice := obiseq.NewBioSequenceSlice(capacity)
 	luaState.Push(obiseqslice2Lua(luaState, seqslice))
 	return 1
 }