Release 4.5.2

This commit introduces significant changes to the sequence processing pipeline, implementing parallel loading, k-mer budget control, explicit memory management, and refined logic for pair evaluation and resolution.

.github/workflows/obitest.yml

@@ -10,10 +10,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Setup Go
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v5
       with:
-        go-version: '1.23'
+        go-version: '1.27'
     - name: Checkout obitools4 project
-      uses: actions/checkout@v4
+      uses: actions/checkout@v5
     - name: Run tests
       run: make githubtests

.github/workflows/release.yml

@@ -16,103 +16,120 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: "1.23"
+          go-version: "1.27"
       - name: Checkout obitools4 project
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
       - name: Run tests
         run: make githubtests
 
-  # Then create release only if tests pass
+  # Build binaries for each platform
-  create-release:
+  build:
     needs: test
-    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - os: ubuntu-latest
+            goos: linux
+            goarch: amd64
+            output_name: linux_amd64
+          - os: ubuntu-24.04-arm
+            goos: linux
+            goarch: arm64
+            output_name: linux_arm64
+          - os: macos-15-intel
+            goos: darwin
+            goarch: amd64
+            output_name: darwin_amd64
+          - os: macos-latest
+            goos: darwin
+            goarch: arm64
+            output_name: darwin_arm64
+
+    runs-on: ${{ matrix.os }}
+
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
-        with:
-          fetch-depth: 0
 
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: "1.23"
+          go-version: "1.27"
 
       - name: Extract version from tag
         id: get_version
         run: |
           TAG=${GITHUB_REF#refs/tags/Release_}
           echo "version=$TAG" >> $GITHUB_OUTPUT
-          echo "tag_name=Release_$TAG" >> $GITHUB_OUTPUT
 
-      - name: Build binaries for multiple platforms
+      - name: Install build tools (macOS)
+        if: runner.os == 'macOS'
+        run: |
+          # Ensure Xcode Command Line Tools are installed
+          xcode-select --install 2>/dev/null || true
+          xcode-select -p
+
+      - name: Build binaries (Linux)
+        if: runner.os == 'Linux'
         env:
           VERSION: ${{ steps.get_version.outputs.version }}
+        run: |
+          docker run --rm \
+            -v "$(pwd):/src" \
+            -w /src \
+            -e VERSION="${VERSION}" \
+            -e GOEXPERIMENT=simd \
+            golang:1.27-alpine \
+            sh -c "apk add --no-cache gcc musl-dev zlib-dev zlib-static make && \
+                   make LDFLAGS='-linkmode=external -extldflags=-static' obitools"
+          mkdir -p artifacts
+          tar -czf artifacts/obitools4_${VERSION}_${{ matrix.output_name }}.tar.gz -C build .
+
+      - name: Build binaries (macOS)
+        if: runner.os == 'macOS'
+        env:
+          GOOS: ${{ matrix.goos }}
+          GOARCH: ${{ matrix.goarch }}
+          VERSION: ${{ steps.get_version.outputs.version }}
+        run: |
+          make obitools
+          mkdir -p artifacts
+          tar -czf artifacts/obitools4_${VERSION}_${{ matrix.output_name }}.tar.gz -C build .
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: binaries-${{ matrix.output_name }}
+          path: artifacts/*
+
+  # Create the release
+  create-release:
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+
+      - name: Extract version from tag
+        id: get_version
+        run: |
+          TAG=${GITHUB_REF#refs/tags/Release_}
+          echo "version=$TAG" >> $GITHUB_OUTPUT
+
+      - name: Download all artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: release-artifacts
+
+      - name: Prepare release directory
         run: |
           mkdir -p release
-
+          find release-artifacts -type f -name "*.tar.gz" -exec cp {} release/ \;
-          # Build for Linux AMD64
-          echo "Building for Linux AMD64..."
-          GOOS=linux GOARCH=amd64 make
-          cd build
-          for binary in *; do
-            tar -czf ../release/${binary}_${VERSION}_linux_amd64.tar.gz ${binary}
-          done
-          cd ..
-          rm -rf build
-          
-
-          # Build for Linux ARM64
-          echo "Building for Linux ARM64..."
-          GOOS=linux GOARCH=arm64 make
-          cd build
-          for binary in *; do
-            tar -czf ../release/${binary}_${VERSION}_linux_arm64.tar.gz ${binary}
-          done
-          cd ..
-          rm -rf build
-          
-
-          # Build for macOS AMD64 (Intel)
-          echo "Building for macOS AMD64..."
-          GOOS=darwin GOARCH=amd64 make
-          cd build
-          for binary in *; do
-            tar -czf ../release/${binary}_${VERSION}_darwin_amd64.tar.gz ${binary}
-          done
-          cd ..
-          rm -rf build
-          
-
-          # Build for macOS ARM64 (Apple Silicon)
-          echo "Building for macOS ARM64..."
-          GOOS=darwin GOARCH=arm64 make
-          cd build
-          for binary in *; do
-            tar -czf ../release/${binary}_${VERSION}_darwin_arm64.tar.gz ${binary}
-          done
-          cd ..
-          rm -rf build
-          
-
-          # Build for Windows AMD64
-          echo "Building for Windows AMD64..."
-          GOOS=windows GOARCH=amd64 make
-          cd build
-          for binary in *; do
-            # Windows binaries have .exe extension
-            if [ -f "${binary}.exe" ]; then
-              zip ../release/${binary}_${VERSION}_windows_amd64.zip ${binary}.exe
-            else
-              zip ../release/${binary}_${VERSION}_windows_amd64.zip ${binary}
-            fi
-          done
-          cd ..
-
-          echo "Built archives:"
           ls -lh release/
 
       - name: Generate Release Notes
-        id: release_notes
         env:
           VERSION: ${{ steps.get_version.outputs.version }}
         run: |
@@ -135,34 +152,29 @@ jobs:
           echo "" >> release_notes.md
           echo "## Installation" >> release_notes.md
           echo "" >> release_notes.md
-          echo "Download the appropriate binary for your system and extract it:" >> release_notes.md
+          echo "Download the appropriate archive for your system and extract it:" >> release_notes.md
           echo "" >> release_notes.md
           echo "### Linux (AMD64)" >> release_notes.md
           echo '```bash' >> release_notes.md
-          echo "tar -xzf <tool>_${VERSION}_linux_amd64.tar.gz" >> release_notes.md
+          echo "tar -xzf obitools4_${VERSION}_linux_amd64.tar.gz" >> release_notes.md
           echo '```' >> release_notes.md
           echo "" >> release_notes.md
           echo "### Linux (ARM64)" >> release_notes.md
           echo '```bash' >> release_notes.md
-          echo "tar -xzf <tool>_${VERSION}_linux_arm64.tar.gz" >> release_notes.md
+          echo "tar -xzf obitools4_${VERSION}_linux_arm64.tar.gz" >> release_notes.md
           echo '```' >> release_notes.md
           echo "" >> release_notes.md
           echo "### macOS (Intel)" >> release_notes.md
           echo '```bash' >> release_notes.md
-          echo "tar -xzf <tool>_${VERSION}_darwin_amd64.tar.gz" >> release_notes.md
+          echo "tar -xzf obitools4_${VERSION}_darwin_amd64.tar.gz" >> release_notes.md
           echo '```' >> release_notes.md
           echo "" >> release_notes.md
           echo "### macOS (Apple Silicon)" >> release_notes.md
           echo '```bash' >> release_notes.md
-          echo "tar -xzf <tool>_${VERSION}_darwin_arm64.tar.gz" >> release_notes.md
+          echo "tar -xzf obitools4_${VERSION}_darwin_arm64.tar.gz" >> release_notes.md
           echo '```' >> release_notes.md
           echo "" >> release_notes.md
-          echo "### Windows (AMD64)" >> release_notes.md
+          echo "All OBITools4 binaries are included in each archive." >> release_notes.md
-          echo '```powershell' >> release_notes.md
-          echo "Expand-Archive <tool>_${VERSION}_windows_amd64.zip" >> release_notes.md
-          echo '```' >> release_notes.md
-          echo "" >> release_notes.md
-          echo "Available tools: Replace \`<tool>\` with one of the obitools commands." >> release_notes.md
 
       - name: Create GitHub Release
         uses: softprops/action-gh-release@v1

.gitignore

@@ -16,13 +16,16 @@
 **/*.tgz
 **/*.yaml
 **/*.csv
+**/*.pb.gz
 xx
 
 .rhistory
 /.vscode
+
+/benchmarck
 /build
 /bugs
-
+autodoc
 /ncbitaxo
 
 !/obitests/**
@@ -31,3 +34,6 @@ LLM/**
 *_files
 
 entropy.html
+bug_id.txt
+obilowmask_ref
+test_*

.serena/.gitignore

@@ -0,0 +1,2 @@
+/cache
+/project.local.yml

.serena/project.yml

@@ -0,0 +1,169 @@
+# the name by which the project can be referenced within Serena/when chatting with the LLM.
+project_name: "obitools4"
+
+# the encoding used by text files in the project
+# For a list of possible encodings, see https://docs.python.org/3.11/library/codecs.html#standard-encodings
+encoding: "utf-8"
+
+# line ending convention to use when writing source files.
+# Possible values: unset (use global setting), "lf", "crlf", or "native" (platform default)
+# This does not affect Serena's own files (e.g. memories and configuration files), which always use native line endings.
+line_ending:
+
+# The language backend to use for this project.
+# If not set, the global setting from serena_config.yml is used.
+# Valid values: LSP, JetBrains
+# Note: the backend is fixed at startup. If a project with a different backend
+# is activated post-init, an error will be returned.
+language_backend:
+
+# whether to use project's .gitignore files to ignore files
+ignore_all_files_in_gitignore: true
+
+# advanced configuration option allowing to configure language server-specific options.
+# Maps the language key to the options.
+# The settings are considered only if the project is trusted (see global configuration to define trusted projects).
+# See https://oraios.github.io/serena/02-usage/050_configuration.html#language-server-specific-settings
+ls_specific_settings: {}
+
+# list of additional paths to ignore in this project.
+# Same syntax as gitignore, so you can use * and **.
+# Important: quote patterns that start with `*`, otherwise YAML treats them as aliases.
+# Example:
+#   ignored_paths:
+#     - "examples/**"
+#     - ".worktrees/**"
+#     - "**/bin/**"
+#     - "**/obj/**"
+# Note: global ignored_paths from serena_config.yml are also applied additively.
+ignored_paths: []
+
+# whether the project is in read-only mode
+# If set to true, all editing tools will be disabled and attempts to use them will result in an error
+# Added on 2025-04-18
+read_only: false
+
+# list of tool names to exclude.
+# This extends the existing exclusions (e.g. from the global configuration)
+# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html
+excluded_tools: []
+
+# list of tools to include that would otherwise be disabled (particularly optional tools that are disabled by default).
+# This extends the existing inclusions (e.g. from the global configuration).
+# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html
+included_optional_tools: []
+
+# fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools.
+# This cannot be combined with non-empty excluded_tools or included_optional_tools.
+# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html
+fixed_tools: []
+
+# list of mode names that are to be activated by default, overriding the setting in the global configuration.
+# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes.
+# If the setting is undefined/empty, the default_modes from the global configuration (serena_config.yml) apply.
+# Otherwise, this overrides the setting from the global configuration (serena_config.yml).
+# Therefore, you can set this to [] if you do not want the default modes defined in the global config to apply
+# for this project.
+# This setting can, in turn, be overridden by CLI parameters (--mode).
+# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes
+default_modes:
+
+# list of mode names to be activated additionally for this project, e.g. ["query-projects"]
+# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes.
+# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes
+added_modes:
+
+# initial prompt for the project. It will always be given to the LLM upon activating the project
+# (contrary to the memories, which are loaded on demand).
+initial_prompt: ""
+
+# time budget (seconds) per tool call for the retrieval of additional symbol information
+# such as docstrings or parameter information.
+# This overrides the corresponding setting in the global configuration; see the documentation there.
+# If null or missing, use the setting from the global configuration.
+symbol_info_budget:
+
+# list of regex patterns which, when matched, mark a memory entry as read‑only.
+# Extends the list from the global configuration, merging the two lists.
+read_only_memory_patterns: []
+
+# list of regex patterns for memories to completely ignore.
+# Matching memories will not appear in list_memories or activate_project output
+# and cannot be accessed via read_memory or write_memory.
+# To access ignored memory files, use the read_file tool on the raw file path.
+# Extends the list from the global configuration, merging the two lists.
+# Example: ["_archive/.*", "_episodes/.*"]
+ignored_memory_patterns: []
+
+# list of additional workspace folder paths for cross-package reference support.
+# Paths can be absolute or relative to the project root.
+# Each folder is registered as an LSP workspace folder, enabling language servers to discover
+# symbols and references across package boundaries, but these folders are not indexed by Serena,
+# i.e. the respective symbols will not be found using Serena's symbol search tools.
+# Example:
+#   additional_workspace_folders:
+#     - ../sibling-package
+#     - ../shared-lib
+ls_additional_workspace_folders: []
+
+# list of language servers to start when using the LSP backend; choose from:
+#   ada                    al                     angular                ansible                bash
+#   bsl                    clojure                cpp                    cpp_ccls               crystal
+#   csharp                 csharp_omnisharp       cue                    dart                   deno
+#   elixir                 elm                    erlang                 fortran                fsharp
+#   gdscript               gleam                  go                     groovy                 haskell
+#   haxe                   hlsl                   html                   java                   json
+#   julia                  kotlin                 latex                  lean4                  lua
+#   luau                   markdown               matlab                 msl                    nextflow
+#   nix                    ocaml                  pascal                 perl                   php
+#   php_phpactor           php_phpantom           powershell             python                 python_basedpyright
+#   python_jedi            python_pyrefly         python_ty              qml                    r
+#   rego                   ruby                   ruby_solargraph        rust                   scala
+#   scss                   solidity               svelte                 swift                  systemverilog
+#   terraform              toml                   typescript             typescript_vts         vue
+#   wolfram                yaml                   zig
+#   (This list may be outdated; generated with scripts/print_language_list.py;
+#   For the current list, see values of the LanguageServerId enum here:
+#   https://github.com/oraios/serena/blob/main/src/solidlsp/ls_config.py)
+# For some languages, there are several alternative language servers, e.g. csharp_omnisharp, ruby_solargraph.)
+# Note:
+#   - For C, use cpp
+#   - For JavaScript, use typescript
+#   - For Angular projects, use angular (subsumes typescript+html; requires `npm install` in the project root)
+#   - For Svelte projects, use svelte (subsumes typescript/javascript for .svelte projects; requires npm)
+#   - For Deno projects, use deno (serves the same .ts/.js files as typescript; requires the deno CLI on PATH)
+#   - For SCSS / Sass / plain CSS, use scss (some-sass-language-server handles all three)
+#   - For Free Pascal/Lazarus, use pascal
+# Special requirements:
+#   Some language servers require additional setup/installations.
+#   See here for details: https://oraios.github.io/serena/01-about/020_programming-languages.html#language-servers
+# When using multiple language servers, the first language server that supports a given file will be used for that file.
+# The first language server is the default language and the respective language server will be used as a fallback.
+# Note that when using the JetBrains backend, language servers are not used and this list is correspondingly ignored.
+language_servers:
+- go
+
+# list of workspace folder paths (LSP backend only).
+# These folders will be used to build up Serena's symbol index.
+# Paths must be within the project root and should thus be relative to the project root.
+# Furthermore, the paths should not be filtered by ignore settings.
+# Default setting: The entire project root folder (".") is considered.
+# In (large) monorepos, this can be used to index only subfolders of the project root, e.g.
+#   ls_workspace_folders:
+#     - "./subproject1"
+#     - "./subproject2"
+ls_workspace_folders:
+- .
+
+# optional shell command to run before the language backend (LSP or JetBrains) is initialised.
+# the command runs in the project root directory and is only executed if the project is trusted
+# (see trusted_project_path_patterns in the global configuration).
+# serena waits for the command to exit: a non-zero exit code is logged as an error but does not
+# abort activation. a per-project timeout (activation_command_timeout, default 180s) is the safety
+# backstop for non-terminating commands; on expiry the process is killed and activation continues.
+# example: activation_command: "npx nx run-many -t build"
+activation_command:
+
+# maximum time in seconds to wait for activation_command to complete before killing it (default 180s).
+# must be a positive number.
+activation_command_timeout: 180.0

Makefile

@@ -2,9 +2,25 @@
 #export GOBIN=$(GOPATH)/bin
 #export PATH=$(GOBIN):$(shell echo $${PATH})
 
+.DEFAULT_GOAL := all
+
+GREEN  := \033[0;32m
+YELLOW := \033[0;33m
+BLUE   := \033[0;34m
+NC     := \033[0m
+
 GOFLAGS=
+LDFLAGS=
+
+# The SIMD code (pkg/obikmer, simd package of Go >= 1.27) is only compiled with
+# GOEXPERIMENT=simd; without it, a portable scalar code is used instead.
+# The binaries select the vector width at run time (and fall back on an
+# emulation without SIMD hardware), so they run on any CPU.
+# Use `make GOEXPERIMENT= obitools` to build without SIMD.
+GOEXPERIMENT?=simd
+export GOEXPERIMENT
 GOCMD=go
-GOBUILD=$(GOCMD) build $(GOFLAGS)
+GOBUILD=$(GOCMD) build $(GOFLAGS) $(if $(LDFLAGS),-ldflags="$(LDFLAGS)")
 GOGENERATE=$(GOCMD) generate
 GOCLEAN=$(GOCMD) clean
 GOTEST=$(GOCMD) test
@@ -28,13 +44,18 @@ OBITOOLS_DIRS:=$(sort $(patsubst %/,%,$(dir $(OBITOOLS_SRC))))
 OBITOOLS:=$(notdir $(OBITOOLS_DIRS))
 
 
+# Name of the target of a package: its path under pkg/, so that two packages of
+# the same name in different directories (pkg/obialign and
+# pkg/obitools/obialign) do not share a target.
+pkgname=$(subst /,-,$(patsubst pkg/%,%,$(1)))
+
 define MAKE_PKG_RULE
-pkg-$(notdir $(1)): $(1) pkg/obioptions/version.go
+pkg-$(call pkgname,$(1)): $(1) pkg/obioptions/version.go
-	@echo -n - Building package $(notdir $(1))...
+	@echo -n - Building package $(call pkgname,$(1))...
 	@$(GOBUILD) ./$(1) \
-	    2> pkg-$(notdir $(1)).log \
+	    2> pkg-$(call pkgname,$(1)).log \
-		|| cat pkg-$(notdir $(1)).log
+		|| cat pkg-$(call pkgname,$(1)).log
-	@rm -f pkg-$(notdir $(1)).log
+	@rm -f pkg-$(call pkgname,$(1)).log
 	@echo Done.
 endef
 
@@ -43,7 +64,7 @@ $(OBITOOLS_PREFIX)$(notdir $(1)): $(BUILD_DIR) $(1) pkg/obioptions/version.go
 	@echo -n - Building obitool $(notdir $(1))...
 	@$(GOBUILD)  -o $(BUILD_DIR)/$(OBITOOLS_PREFIX)$(notdir $(1)) ./$(1) \
 	             2> $(OBITOOLS_PREFIX)$(notdir $(1)).log \
-				 || cat $(OBITOOLS_PREFIX)$(notdir $(1)).log
+				 || { cat $(OBITOOLS_PREFIX)$(notdir $(1)).log; rm -f $(OBITOOLS_PREFIX)$(notdir $(1)).log; exit 1; }
 	@rm -f $(OBITOOLS_PREFIX)$(notdir $(1)).log
 	@echo Done.
 endef
@@ -60,6 +81,28 @@ endif
 
 OUTPUT:=$(shell mktemp)
 
+help:
+	@printf "$(GREEN)OBITools4 Makefile$(NC)\n\n"
+	@printf "$(BLUE)Main targets:$(NC)\n"
+	@printf "  %-20s %s\n" "all"          "Build all obitools (default)"
+	@printf "  %-20s %s\n" "obitools"     "Build all obitools binaries to build/"
+	@printf "  %-20s %s\n" "test"         "Run Go unit tests"
+	@printf "  %-20s %s\n" "obitests"     "Run integration tests (obitests/)"
+	@printf "  %-20s %s\n" "bump-version" "Increment patch version (or set with VERSION=x.y.z)"
+	@printf "  %-20s %s\n" "update-deps"  "Update all Go dependencies"
+	@printf "\n$(BLUE)Jujutsu workflow:$(NC)\n"
+	@printf "  %-20s %s\n" "jjnew"        "Document current commit and start a new one"
+	@printf "  %-20s %s\n" "jjpush"       "Release: describe, bump, generate notes, push PR, tag (VERSION=x.y.z optional)"
+	@printf "  %-20s %s\n" "jjfetch"      "Fetch latest commits from origin"
+	@printf "\n$(BLUE)Required tools:$(NC)\n"
+	@printf "  %-20s " "go";      command -v go      >/dev/null 2>&1 && printf "$(GREEN)✓$(NC) %s\n" "$$(go version)" || printf "$(YELLOW)✗ not found$(NC)\n"
+	@printf "  %-20s " "git";     command -v git     >/dev/null 2>&1 && printf "$(GREEN)✓$(NC) %s\n" "$$(git --version)" || printf "$(YELLOW)✗ not found$(NC)\n"
+	@printf "  %-20s " "jj";      command -v jj      >/dev/null 2>&1 && printf "$(GREEN)✓$(NC) %s\n" "$$(jj --version)" || printf "$(YELLOW)✗ not found$(NC)\n"
+	@printf "  %-20s " "gh";      command -v gh      >/dev/null 2>&1 && printf "$(GREEN)✓$(NC) %s\n" "$$(gh --version | head -1)" || printf "$(YELLOW)✗ not found$(NC)  (brew install gh)\n"
+	@printf "\n$(BLUE)Optional tools (release notes generation):$(NC)\n"
+	@printf "  %-20s " "aichat";  command -v aichat  >/dev/null 2>&1 && printf "$(GREEN)✓$(NC) %s\n" "$$(aichat --version)" || printf "$(YELLOW)✗ not found$(NC)  (https://github.com/sigoden/aichat)\n"
+	@printf "  %-20s " "jq";      command -v jq      >/dev/null 2>&1 && printf "$(GREEN)✓$(NC) %s\n" "$$(jq --version)" || printf "$(YELLOW)✗ not found$(NC)  (brew install jq)\n"
+
 all: install-githook obitools
 
 obitools: $(patsubst %,$(OBITOOLS_PREFIX)%,$(OBITOOLS))
@@ -106,43 +149,104 @@ pkg/obioptions/version.go: version.txt .FORCE
 	@rm -f $(OUTPUT)
 
 bump-version:
-	@echo "Incrementing version..."
 	@current=$$(cat version.txt); \
-	echo "  Current version: $$current"; \
+	if [ -n "$(VERSION)" ]; then \
-	major=$$(echo $$current | cut -d. -f1); \
+		new_version="$(VERSION)"; \
-	minor=$$(echo $$current | cut -d. -f2); \
+		echo "Setting version to $$new_version (was $$current)"; \
-	patch=$$(echo $$current | cut -d. -f3); \
+	else \
-	new_patch=$$((patch + 1)); \
+		echo "Incrementing version..."; \
-	new_version="$$major.$$minor.$$new_patch"; \
+		echo "  Current version: $$current"; \
-	echo "  New version: $$new_version"; \
+		major=$$(echo $$current | cut -d. -f1); \
+		minor=$$(echo $$current | cut -d. -f2); \
+		patch=$$(echo $$current | cut -d. -f3); \
+		new_patch=$$((patch + 1)); \
+		new_version="$$major.$$minor.$$new_patch"; \
+		echo "  New version: $$new_version"; \
+	fi; \
 	echo "$$new_version" > version.txt
 	@echo "✓ Version updated in version.txt"
 	@$(MAKE) pkg/obioptions/version.go
 
 jjnew:
 	@echo "$(YELLOW)→ Creating a new commit...$(NC)"
-	@echo "$(BLUE)→ Documenting current commit...$(NC)"
+	@echo "$(BLUE)→ Documenting undocumented commits...$(NC)"
-	@jj auto-describe
+	@jj auto-doc
 	@echo "$(BLUE)→ Done.$(NC)"
 	@jj new
 	@echo "$(GREEN)✓ New commit created$(NC)"
 
 jjpush:
-	@echo "$(YELLOW)→ Pushing commit to repository...$(NC)"
+	@$(MAKE) jjpush-describe
-	@echo "$(BLUE)→ Documenting current commit...$(NC)"
+	@$(MAKE) jjpush-bump
-	@jj auto-describe
+	@$(MAKE) jjpush-notes
+	@$(MAKE) jjpush-push
+	@$(MAKE) jjpush-tag
+	@echo "$(GREEN)✓ Release complete$(NC)"
+
+jjpush-describe:
+	@echo "$(BLUE)→ Documenting undocumented commits...$(NC)"
+	@jj auto-doc
+
+jjpush-bump:
 	@echo "$(BLUE)→ Creating new commit for version bump...$(NC)"
 	@jj new
 	@$(MAKE) bump-version
-	@echo "$(BLUE)→ Documenting version bump commit...$(NC)"
+
-	@jj auto-describe
+jjpush-notes:
+	@version=$$(cat version.txt); \
+	echo "$(BLUE)→ Generating release notes for version $$version...$(NC)"; \
+	release_title="Release $$version"; \
+	release_body=""; \
+	if command -v aichat >/dev/null 2>&1; then \
+		previous_tag=$$(git describe --tags --abbrev=0 --match 'Release_*' 2>/dev/null); \
+		if [ -z "$$previous_tag" ]; then \
+			echo "$(YELLOW)⚠ No previous Release tag found, skipping release notes$(NC)"; \
+		else \
+			raw_output=$$(git log --format="%h %B" "$$previous_tag..HEAD" | \
+				aichat \
+				"Summarize the following commits into a GitHub release note for version $$version. Ignore commits related to version bumps, .gitignore changes, or any internal housekeeping that is irrelevant to end users. Describe each user-facing change precisely without exposing code. Eliminate redundancy. Output strictly valid JSON with no surrounding text, using this exact schema: {\"title\": \"<short release title>\", \"body\": \"<detailed markdown release notes>\"}" 2>/dev/null) || true; \
+			if [ -n "$$raw_output" ]; then \
+				notes=$$(printf '%s\n' "$$raw_output" | python3 tools/json2md.py 2>/dev/null); \
+				if [ -n "$$notes" ]; then \
+					release_title=$$(echo "$$notes" | head -1); \
+					release_body=$$(echo "$$notes" | tail -n +3); \
+				else \
+					echo "$(YELLOW)⚠ JSON parsing failed, using default release message$(NC)"; \
+				fi; \
+			fi; \
+		fi; \
+	fi; \
+	printf '%s' "$$release_title" > /tmp/obitools4-release-title.txt; \
+	printf '%s' "$$release_body" > /tmp/obitools4-release-body.txt; \
+	echo "$(BLUE)→ Setting release notes as commit description...$(NC)"; \
+	jj desc -m "$$release_title"$$'\n\n'"$$release_body"
+
+jjpush-push:
+	@echo "$(BLUE)→ Pushing commits...$(NC)"
+	@jj git push --change @
+	@echo "$(BLUE)→ Creating/updating PR...$(NC)"
+	@release_title=$$(cat /tmp/obitools4-release-title.txt 2>/dev/null || echo "Release $$(cat version.txt)"); \
+	release_body=$$(cat /tmp/obitools4-release-body.txt 2>/dev/null || echo ""); \
+	branch=$$(jj log -r @ --no-graph -T 'bookmarks.map(|b| b.name()).join("\n")' 2>/dev/null | head -1); \
+	if [ -n "$$branch" ] && command -v gh >/dev/null 2>&1; then \
+		gh pr create --title "$$release_title" --body "$$release_body" --base master --head "$$branch" 2>/dev/null \
+		|| gh pr edit "$$branch" --title "$$release_title" --body "$$release_body" 2>/dev/null \
+		|| echo "$(YELLOW)⚠ Could not create/update PR$(NC)"; \
+	fi
+
+jjpush-tag:
 	@version=$$(cat version.txt); \
 	tag_name="Release_$$version"; \
-	echo "$(BLUE)→ Pushing commits and creating tag $$tag_name...$(NC)"; \
+	release_title=$$(cat /tmp/obitools4-release-title.txt 2>/dev/null || echo "Release $$version"); \
-	jj git push --change @; \
+	release_body=$$(cat /tmp/obitools4-release-body.txt 2>/dev/null || echo ""); \
-	git tag -a "$$tag_name" -m "Release $$version" 2>/dev/null || echo "Tag $$tag_name already exists"; \
+	install_section=$$'\n## Installation\n\n### Pre-built binaries\n\nDownload the appropriate archive for your system from the\n[release assets](https://github.com/metabarcoding/obitools4/releases/tag/Release_'"$$version"')\nand extract it:\n\n#### Linux (AMD64)\n```bash\ntar -xzf obitools4_'"$$version"'_linux_amd64.tar.gz\n```\n\n#### Linux (ARM64)\n```bash\ntar -xzf obitools4_'"$$version"'_linux_arm64.tar.gz\n```\n\n#### macOS (Intel)\n```bash\ntar -xzf obitools4_'"$$version"'_darwin_amd64.tar.gz\n```\n\n#### macOS (Apple Silicon)\n```bash\ntar -xzf obitools4_'"$$version"'_darwin_arm64.tar.gz\n```\n\nAll OBITools4 binaries are included in each archive.\n\n### From source\n\nYou can also compile and install OBITools4 directly from source using the\ninstallation script:\n\n```bash\ncurl -L https://raw.githubusercontent.com/metabarcoding/obitools4/master/install_obitools.sh | bash -s -- --version '"$$version"'\n```\n\nBy default binaries are installed in `/usr/local/bin`. Use `--install-dir` to\nchange the destination and `--obitools-prefix` to add a prefix to command names:\n\n```bash\ncurl -L https://raw.githubusercontent.com/metabarcoding/obitools4/master/install_obitools.sh | \\\n  bash -s -- --version '"$$version"' --install-dir ~/local --obitools-prefix k\n```\n'; \
-	git push origin "$$tag_name" 2>/dev/null || echo "Tag already pushed"
+	release_message="$$release_title"$$'\n\n'"$$release_body$$install_section"; \
-	@echo "$(GREEN)✓ Commits and tag pushed to repository$(NC)"
+	echo "$(BLUE)→ Creating tag $$tag_name...$(NC)"; \
+	commit_hash=$$(jj log -r @ --no-graph -T 'commit_id' 2>/dev/null); \
+	git tag -a "$$tag_name" $${commit_hash:+"$$commit_hash"} -m "$$release_message" 2>/dev/null || echo "$(YELLOW)⚠ Tag $$tag_name already exists$(NC)"; \
+	echo "$(BLUE)→ Pushing tag $$tag_name...$(NC)"; \
+	git push origin "$$tag_name" 2>/dev/null || echo "$(YELLOW)⚠ Tag push failed or already pushed$(NC)"; \
+	rm -f /tmp/obitools4-release-title.txt /tmp/obitools4-release-body.txt
 
 jjfetch:
 	@echo "$(YELLOW)→ Pulling latest commits...$(NC)"
@@ -150,5 +254,5 @@ jjfetch:
 	@jj new master@origin
 	@echo "$(GREEN)✓ Latest commits pulled$(NC)"
 
-.PHONY: all obitools update-deps obitests githubtests jjnew jjpush jjfetch bump-version .FORCE
+.PHONY: all obitools update-deps obitests githubtests help jjnew jjpush jjpush-describe jjpush-bump jjpush-notes jjpush-push jjpush-tag jjfetch bump-version .FORCE
 .FORCE:

OBIPRIMERS_MEMORY_PROMPT.md

@@ -0,0 +1,149 @@
+# Prompt : optimisation de l'usage mémoire de `obiprimers`
+
+Tu travailles dans le dépôt Go `/Users/coissac/Sync/travail/__MOI__/GO/obitools4` (module
+`git.metabarcoding.org/obitools/obitools4/obitools4`, Go : `/usr/local/go/bin/go`, non présent dans le PATH).
+Ta mission : **réduire l'usage mémoire de la commande `obiprimers`** sans changer ses résultats, en procédant
+par la mesure.
+
+## 1. Contexte
+
+`obiprimers` est la ré-implémentation OBITools4 d'ecoPrimers (design de paires d'amorces à partir de génomes
+entiers, avec évaluation de la qualité du marqueur délimité). Le mode « voie A » est terminé et validé.
+Avant toute chose, **lis `OBIPRIMERS_NOTES.md`** (racine du dépôt) : la section « État courant », puis les
+sections 11 à 13 (appariement approché), 16 et 16 bis (taxonomie embarquée ; **une erreur de mesure à ne pas
+refaire**), 25 (définitions de Bc, Bs, Bd et élagage de la phase 2).
+
+Code concerné :
+- `cmd/obitools/obiprimers/main.go`, `pkg/obitools/obiprimers/` (options, lecture, sortie CSV) ;
+- `pkg/obiprimer/` : `corpus.go` (`LoadCorpus`), `taxa.go`, `words.go` (`FindStrictWords`), `match.go` et
+  `match_seed.go` (`MatchWords`), `pairs.go` (`selectPrimers`, `countPairs`), `pairs_eval.go` (`FindPairs`),
+  `amplicons.go` (`collectAmplicons`), `resolution.go`, `search.go` (`Search`) ;
+- `pkg/obikmer/seqquorum*.go` (`SeqKmerCounter`, comptage par partitions sur disque) ;
+- `pkg/obithermo/` (Tm), code partagé : `pkg/obiformats`, `pkg/obiiter`, `pkg/obiseq`, `pkg/obitax`.
+
+## 2. Décisions de l'utilisateur à respecter (non négociables)
+
+- **Tout est chargé en mémoire** : toutes les séquences de la base (cibles, contre-exemples, non-cibles) sont
+  gardées, parce que la lecture sur l'**entrée standard** doit continuer à marcher comme pour tout obitool.
+  Tu peux **compacter** ce qui est gardé ; tu ne peux pas passer à un schéma qui exige de relire le fichier.
+- **Un seul calcul par indice** : ne touche pas aux définitions de Bc, Bs et Bd (`resolution.go`). Pas de
+  colonne ou d'option alternative.
+- **Les résultats ne changent pas.** Critère : la sortie CSV doit rester **identique octet pour octet** sur les
+  cas de référence du §4. Si tu penses qu'un changement de résultat est justifié, **demande avant**.
+- Aucune commande git qui modifie quoi que ce soit (pas de commit, add, mv, rm, reset…). Lecture seule
+  (`status`, `diff`, `log`) autorisée. Déplace les fichiers avec `mv`/`Write`/`Edit`.
+- Fichiers source de moins d'environ 500 lignes. Pas de `.collect()` d'une grande structure quand un flux
+  suffit. Réutilise l'existant avant d'écrire du code ; enrichis une brique partagée plutôt que d'écrire du
+  code ad hoc, **mais** toute modification d'un paquet partagé (obiformats, obiiter, obiseq, obitax) doit
+  être signalée clairement et validée par les tests de ces paquets.
+- **Pas de spéculation présentée comme un fait** : ce qui n'est pas mesuré est une hypothèse, dis-le.
+  Une régression de justesse est pire qu'un gain de mémoire : avant d'échanger l'une contre l'autre, demande.
+
+## 3. Ce qui est mesuré aujourd'hui (macOS, M3 Max, 16 cœurs)
+
+Jeu : 7 822 génomes mitochondriaux de vertébrés (131 Mb de bases), `-r 7953` (Cyprinidae : 325 cibles, 7 497
+non-cibles). Commande :
+`obiprimers -r 7953 -e 3 --three-prime 2 -l 30 -L 150 -c <fichier annoté> -o out.csv` → 3 247 paires
+(3 248 lignes avec l'en-tête).
+
+| Mesure | Valeur |
+|---|---|
+| pic RSS, taxonomie embarquée, fichier | **1 217 à 1 372 Mo** (bruit d'environ ±100 Mo d'une exécution à l'autre), 4,7 à 5,3 s |
+| pic RSS, taxonomie embarquée, entrée standard | médiane **1 334 à 1 346 Mo** sur 8 exécutions entrelacées |
+| tas des 7 822 séquences chargées | **193,7 Mo** (dont `taxonomic_path` : 19,2 Mo, soit 2,46 Ko par séquence) |
+| même exécution **sans** l'élagage de phase 2 | 5 604 Mo (c'est ce qu'on a corrigé, voir §25 des notes) |
+| avec `-t` (taxonomie NCBI complète) | +1,07 Go pour le seul chargeur (mesuré avec `obitaxonomy`), pic ~2,8 Go (mesuré avant les nouvelles définitions) |
+| `GOGC=25` (ancienne mesure, `-t` NCBI) | pic 2,82 Go → 1,98 Go, mais temps CPU utilisateur ×2 |
+| ecoPrimers (C) sur les mêmes jeux | 1,86 Go (Cyprinidae), 2,11 Go (Teleostei) |
+
+Tailles d'enregistrements : `ampRecord` 24 octets, `Match` 12, `seqMatch` 12, `hit` (intermédiaire de
+`matchCorpus`) 16. Environ 6,85 millions d'occurrences approchées quand toutes les séquences sont chargées
+(cas `-i 1`).
+
+**Le constat qui motive la mission** : les séquences gardées pèsent ~194 Mo, le pic du processus est ~1,3 Go.
+Les 1,1 Go restants ne sont **pas expliqués** (hypothèses ci-dessous, aucune n'est vérifiée).
+
+## 4. Garde-fous à mettre en place AVANT de modifier quoi que ce soit
+
+1. **Binaire de référence** : `cd /Users/coissac/Sync/travail/__MOI__/GO/obitools4 &&
+   GOEXPERIMENT=simd /usr/local/go/bin/go build -o /tmp/obiprimers.baseline ./cmd/obitools/obiprimers`
+2. **Données** (les fichiers de `/tmp` peuvent avoir disparu, regénère-les) : dossier du tutoriel
+   `D="$HOME/Documents/Enseignement/DNA metabarcoding spring school/ecoPrimers-Fish"` : `mito.vert.fasta.gz`,
+   `ncbitaxo_20250211.tgz`, et pour le test d'échelle `mito.all.fasta.gz` / `mito.one.fasta.gz`. Fichier annoté
+   (taxonomie embarquée, évite le chargeur NCBI) :
+   `/Users/coissac/travail/__MOI__/GO/obitools4/build/obiannotate -t "$D/ncbitaxo_20250211.tgz"
+   --taxonomic-path -Z -o /tmp/mito.vert.path.fasta.gz "$D/mito.vert.fasta.gz"`
+   (les binaires déjà compilés sont dans `/Users/coissac/travail/__MOI__/GO/obitools4/build/`, sans `Sync`).
+3. **CSV de référence**, à produire avec le binaire de référence et à conserver : Cyprinidae `-r 7953`,
+   Teleostei `-r 32443`, un cas avec contre-exemples `-r 7953 -i 1`, un cas `-U`, un cas `-R NC_083213`, et la
+   lecture sur entrée standard (`gzcat fichier | obiprimers ...`). Après chaque modification : `cmp` octet pour
+   octet.
+4. **Tests** : `go test ./pkg/obiprimer/ ./pkg/obitools/obiprimers/ ./pkg/obithermo/ ./pkg/obikmer/ -race
+   -count=1`, et les tests opt-in sur les données réelles (durent ~75 s hors `-race`) :
+   `OBIPRIMERS_TUTORIAL="$D" go test ./pkg/obiprimer/ -count=1`.
+5. **Méthode de mesure** (le piège principal) : le pic RSS bouge de ±100 Mo entre deux exécutions identiques
+   (rythme du ramasse-miettes). Mesure **au moins 5 exécutions entrelacées avant/après et compare les
+   médianes** (`/usr/bin/time -l` : octets sur macOS). Pour attribuer la mémoire, ne te fie pas au RSS : utilise
+   des profils de tas (`--pprof` de la CLI sert `/debug/pprof/heap` sur `localhost:6060`, `net/http/pprof` est
+   importé ; ou `runtime/pprof.WriteHeapProfile` aux frontières d'étapes ; `go tool pprof -sample_index=inuse_space`
+   et `alloc_space` ; `GODEBUG=gctrace=1`). **Erreur déjà commise (voir la section 16 bis des notes)** : en
+   mesurant le tas libéré par la suppression d'un attribut, la tranche de séquences n'était plus référencée
+   après la boucle et le GC a libéré *tout* ; utilise `runtime.KeepAlive` sur ce que tu veux garder vivant.
+
+## 5. Pistes à évaluer (toutes des HYPOTHÈSES à confirmer par un profil, classées sans ordre)
+
+- **Retard du ramasse-miettes** (`GOGC=100` laisse le tas monter jusqu'à ~2× le vivant). Pistes : limite
+  mémoire douce (`debug.SetMemoryLimit`) ou `debug.SetGCPercent` ajustés par la commande après la lecture ;
+  mesurer le coût CPU (l'ancienne mesure avec `GOGC=25` doublait le temps utilisateur).
+- **Lecture** (code partagé `obiformats`/`obiiter`) : 131 Mb sont analysés, avec une table d'attributs et des
+  chaînes JSON par séquence. L'option `--batch-mem` (lots limités en mémoire) existe déjà : l'évaluer.
+- **Séquences gardées** : `BioSequence` garde 1 octet par base et une table d'attributs. Ne garder que
+  l'identifiant, la séquence et le `taxid` (retirer les autres attributs, comme déjà fait pour
+  `taxonomic_path`, dans `dropTaxonomicPath` de `pkg/obitools/obiprimers/obiprimers.go`) ; ou stocker la
+  séquence à 2 bits par base (~33 Mo pour 131 Mb) avec une liste d'exceptions pour les bases ambiguës. Ce
+  second point touche `forEachOccurrence`, `ampHash`, `windowAsIs`, `windowRevComp` et `MatchWords`, qui lisent
+  les octets : à ne tenter que si le profil le justifie.
+- **Occurrences stockées deux fois** : `[]WordMatches` (`matchCorpus`, intermédiaire `hit` de 16 octets par
+  occurrence puis recopie) et `primerSet.perSeq` (`selectPrimers`). Vérifier à quel moment chacune est
+  libérée, construire directement les structures finales, présizer les tranches (les `append` en doublement
+  créent un pic transitoire).
+- **Phase 2b** (`collectAmplicons` sur les seules paires survivantes) : `ampRecord` de 24 octets (16 possibles
+  en empaquetant le rôle), enregistrements par worker ; **traitement des paires survivantes par lots bornés**
+  si le nombre d'enregistrements estimé dépasse un budget (idée notée en 25.2). Cas dur : `-T 0` (aucune paire
+  élaguée), où les tests atteignent ~5,5 Go de tas.
+- **`SeqKmerCounter`** (`pkg/obikmer/seqquorum*.go`) : environ 16 octets par occurrence de k-mer pour les
+  partitions en cours de traitement ; le nombre de partitions est **fixé à 256**, alors que le total de bases
+  est connu une fois le corpus chargé : le choisir automatiquement.
+- **Lecture en deux passes** pour la taxonomie embarquée (`readSequencesAndTaxonomy`) : la première passe ne
+  garde que les chemins, mais analyse tout.
+- **Chargeur de taxonomie NCBI** : 1,07 Go à lui seul ; c'est du code partagé (`obitax`), hors périmètre sauf
+  si tu y trouves une économie sûre ; sinon, documente-le dans l'aide (la taxonomie embarquée évite ce coût).
+
+## 6. Test d'échelle (jamais fait)
+
+Tout est validé sur des génomes mitochondriaux (5 à 131 Mb). Mesure la croissance sur un fichier plus gros
+(`mito.all.fasta.gz`, 172 Mo compressés, avec `--restrict-to-taxon` sur un grand clade, p. ex. les Metazoa) :
+temps et pic RSS en fonction de la taille, pour savoir ce qui grossit vite (corpus, occurrences, amplicons,
+compteur de k-mers).
+
+## 7. Livrables
+
+1. Un **rapport de profil** : où va la mémoire à chaque étape (lecture, sélection, mots stricts, appariement,
+   paires, phase 2a/2b), avec les profils de tas et les chiffres, avant toute modification.
+2. Les optimisations **classées par gain mesuré et par risque**, puis implémentées une par une, chacune avec
+   mesure avant/après (médianes d'au moins 5 exécutions entrelacées), `cmp` des CSV de référence, tests verts
+   sous `-race`.
+3. Une **nouvelle section** à la fin de `OBIPRIMERS_NOTES.md` (ne réécris pas l'historique) : mesures,
+   décisions, pistes écartées et pourquoi, ce qui reste. Pas de commit.
+4. Si une optimisation exige de changer un résultat, une décision ou un paquet partagé : **arrête-toi et
+   demande**.
+
+## 8. Pièges connus de l'environnement
+
+- zsh ne découpe pas une variable en arguments : utilise des tableaux (`A=(-r 7953 ...)` puis `"${A[@]}"`)
+  ou des scripts `bash`.
+- macOS : `sed -i ''` (sans suffixe) ; `/usr/bin/time -l` donne le pic en **octets** (`maximum resident set size`).
+- `OBIPRIMERS_WORKERS` n'est lue que par les tests opt-in ; la CLI utilise `--max-cpu`.
+- Les avertissements C de `obiformats/fastseq_read.h` sont connus et sans gravité.
+- `go test -race` sur les tests opt-in est lent (~90 s) ; réserve-le aux tests unitaires.
+- Le test `obiutils/TestSetString` était instable : il est corrigé (tri des membres dans `Set.String`).

README.md

@@ -16,12 +16,17 @@ The easiest way to run it is to copy and paste the following command into your t
 curl -L https://raw.githubusercontent.com/metabarcoding/obitools4/master/install_obitools.sh | bash
 ```
 
-By default, the script installs the *OBITools* commands and other associated files into the `/usr/local` directory.
+By default, the script installs the latest version of *OBITools* commands and other associated files into the `/usr/local` directory.
-The names of the commands in the new *OBITools4* are mostly identical to those in *OBITools2*.
-Therefore, installing the new *OBITools* may hide or delete the old ones. If you want both versions to be 
-available on your system, the installation script offers two options:
 
+### Installation Options
 
+The installation script offers several options:
+
+>  -l, --list              List all available versions and exit.
+> 
+>  -v, --version           Install a specific version (e.g., `-v 4.4.3`).
+>                          By default, the latest version is installed.
+> 
 >  -i, --install-dir       Directory where obitools are installed 
 >                          (as example use `/usr/local` not `/usr/local/bin`).
 > 
@@ -29,15 +34,36 @@ available on your system, the installation script offers two options:
 >                          want to have several versions of obitools at the
 >                          same time on your system (as example `-p g` will produce
 >                          `gobigrep` command instead of `obigrep`).
+>
+>  -j, --jobs              Number of parallel jobs used for compilation
+>                          (default: 1). Increase this value to speed up
+>                          compilation on multi-core systems (e.g., `-j 4`).
 
-You can use these options by following the installation command:
+### Examples
 
+List all available versions:
+```{bash}
+curl -L https://raw.githubusercontent.com/metabarcoding/obitools4/master/install_obitools.sh | bash -s -- --list
+```
+
+Install a specific version:
+```{bash}
+curl -L https://raw.githubusercontent.com/metabarcoding/obitools4/master/install_obitools.sh | bash -s -- --version 4.4.3
+```
+
+Install in a custom directory with command prefix:
 ```{bash}
 curl -L https://raw.githubusercontent.com/metabarcoding/obitools4/master/install_obitools.sh | \
       bash -s -- --install-dir test_install --obitools-prefix k
 ```
 
-In this case, the binaries will be installed in the `test_install` directory and all command names will be prefixed with the letter `k`. Thus, `obigrep` will be named `kobigrep`.
+In this last example, the binaries will be installed in the `test_install` directory and all command names will be prefixed with the letter `k`. Thus, `obigrep` will be named `kobigrep`.
+
+### Note on Version Compatibility
+
+The names of the commands in the new *OBITools4* are mostly identical to those in *OBITools2*.
+Therefore, installing the new *OBITools* may hide or delete the old ones. If you want both versions to be 
+available on your system, use the `--install-dir` and `--obitools-prefix` options as shown above.
 
 ## Continuing the analysis...
 

Release-notes.md

@@ -2,6 +2,23 @@
 
 ## New changes
 
+### New features
+
+- New `obioligotag` command, a re-implementation of the `oligotag` command of
+  OBITools 2. It builds a family of oligonucleotides (tags) of at least
+  `--family-size` members that are pairwise at a distance of at least
+  `--distance`. Candidates are either generated (`--oligo-size`, up to 12) or
+  read from a file (`--oligo-list`, sequence file or one oligonucleotide per
+  line). They can be restricted by alphabet (`--alphabet`, for instance `ACT`
+  to forbid G), GC content, homopolymers and IUPAC patterns. The distance is
+  either the Hamming distance or the LCS-mode alignment distance
+  (`--distance-type`).
+
+- New functions in `obikmer` to compare and enumerate k-mers encoded as
+  `uint64`: `HammingDistance`, `KmerLCS` and `KmerLCSBatch` (SIMD when built with
+  `GOEXPERIMENT=simd`), `IterConstrainedKmers`, and in `obigraph` a clique search
+  (`FindClique`) that does not need the adjacency matrix.
+
 ### Bug fixes
 
 - In `obipairing` correct the misspelling of the `obiparing_*` tags where the `i`

blackboard/Prospective/kmer_disk_index_plan.md

@@ -0,0 +1,508 @@
+# Plan de refonte du package obikmer : index disk-based par partitions minimizer
+
+## Constat
+
+Les roaring64 bitmaps ne sont pas adaptés au stockage de 10^10 k-mers
+(k=31) dispersés sur un espace de 2^62. L'overhead structurel (containers
+roaring par high key 32 bits) dépasse la taille des données elles-mêmes,
+et les opérations `Or()` entre bitmaps fragmentés ne terminent pas en
+temps raisonnable.
+
+## Principe de la nouvelle architecture
+
+Un `KmerSet` est un ensemble trié de k-mers canoniques (uint64) stocké
+sur disque, partitionné par minimizer. Chaque partition est un fichier
+binaire contenant des uint64 triés, compressés par delta-varint.
+
+Un `KmerSetGroup` est un répertoire contenant N ensembles partitionnés
+de la même façon (même k, même m, même P).
+
+Un `KmerSet` est un `KmerSetGroup` de taille 1 (singleton).
+
+Les opérations ensemblistes se font partition par partition, en merge
+streaming, sans charger l'index complet en mémoire.
+
+## Cycle de vie d'un index
+
+L'index a deux phases distinctes :
+
+1. **Phase de construction (mutable)** : on ouvre un index, on y ajoute
+   des séquences. Pour chaque séquence, les super-kmers sont extraits
+   et écrits de manière compacte (2 bits/base) dans le fichier
+   temporaire de partition correspondant (`minimizer % P`). Les
+   super-kmers sont une représentation compressée naturelle des k-mers
+   chevauchants : un super-kmer de longueur L encode L-k+1 k-mers en
+   ne stockant que ~L/4 bytes au lieu de (L-k+1) × 8 bytes.
+
+2. **Phase de clôture (optimisation)** : on ferme l'index, ce qui
+   déclenche le traitement **partition par partition** (indépendant,
+   parallélisable) :
+   - Charger les super-kmers de la partition
+   - En extraire tous les k-mers canoniques
+   - Trier le tableau de k-mers
+   - Dédupliquer (et compter si FrequencyFilter)
+   - Delta-encoder et écrire le fichier .kdi final
+   Après clôture, l'index est statique et immuable.
+
+3. **Phase de lecture (immutable)** : opérations ensemblistes,
+   Jaccard, Quorum, Contains, itération. Toutes en streaming.
+
+---
+
+## Format sur disque
+
+### Index finalisé
+
+```
+index_dir/
+  metadata.toml
+  set_0/
+    part_0000.kdi
+    part_0001.kdi
+    ...
+    part_{P-1}.kdi
+  set_1/
+    part_0000.kdi
+    ...
+  ...
+  set_{N-1}/
+    ...
+```
+
+### Fichiers temporaires pendant la construction
+
+```
+index_dir/
+  .build/
+    set_0/
+      part_0000.skm          # super-kmers encodés 2 bits/base
+      part_0001.skm
+      ...
+    set_1/
+      ...
+```
+
+Le répertoire `.build/` est supprimé après Close().
+
+### metadata.toml
+
+```toml
+id = "mon_index"
+k = 31
+m = 13
+partitions = 1024
+type = "KmerSetGroup"       # ou "KmerSet" (N=1)
+size = 3                    # nombre de sets (N)
+sets_ids = ["genome_A", "genome_B", "genome_C"]
+
+[user_metadata]
+organism = "Triticum aestivum"
+
+[sets_metadata]
+# métadonnées individuelles par set si nécessaire
+```
+
+### Fichier .kdi (Kmer Delta Index)
+
+Format binaire :
+
+```
+[magic: 4 bytes "KDI\x01"]
+[count: uint64 little-endian]       # nombre de k-mers dans cette partition
+[first: uint64 little-endian]       # premier k-mer (valeur absolue)
+[delta_1: varint]                   # arr[1] - arr[0]
+[delta_2: varint]                   # arr[2] - arr[1]
+...
+[delta_{count-1}: varint]           # arr[count-1] - arr[count-2]
+```
+
+Varint : encoding unsigned, 7 bits utiles par byte, bit de poids fort
+= continuation (identique au varint protobuf).
+
+Fichier vide (partition sans k-mer) : magic + count=0.
+
+### Fichier .skm (Super-Kmer temporaire)
+
+Format binaire, séquence de super-kmers encodés :
+
+```
+[len: uint16 little-endian]         # longueur du super-kmer en bases
+[sequence: ceil(len/4) bytes]       # séquence encodée 2 bits/base, packed
+...
+```
+
+**Compression par rapport au stockage de k-mers bruts** :
+
+Un super-kmer de longueur L contient L-k+1 k-mers.
+- Stockage super-kmer : 2 + ceil(L/4) bytes
+- Stockage k-mers bruts : (L-k+1) × 8 bytes
+
+Exemple avec k=31, super-kmer typique L=50 :
+- Super-kmer : 2 + 13 = 15 bytes → encode 20 k-mers
+- K-mers bruts : 20 × 8 = 160 bytes
+- **Facteur de compression : ~10×**
+
+Pour un génome de 10 Gbases (~10^10 k-mers bruts) :
+- K-mers bruts : ~80 Go par set temporaire
+- Super-kmers : **~8 Go** par set temporaire
+
+Avec FrequencyFilter et couverture 30× :
+- K-mers bruts : ~2.4 To
+- Super-kmers : **~240 Go**
+
+---
+
+## FrequencyFilter
+
+Le FrequencyFilter n'est plus un type de données séparé. C'est un
+**mode de construction** du builder. Le résultat est un KmerSetGroup
+standard.
+
+### Principe
+
+Pendant la construction, tous les super-kmers sont écrits dans les
+fichiers temporaires .skm, y compris les doublons (chaque occurrence
+de chaque séquence est écrite).
+
+Pendant Close(), pour chaque partition :
+1. Charger tous les super-kmers de la partition
+2. Extraire tous les k-mers canoniques dans un tableau []uint64
+3. Trier le tableau
+4. Parcourir linéairement : les k-mers identiques sont consécutifs
+5. Compter les occurrences de chaque k-mer
+6. Si count >= minFreq → écrire dans le .kdi final (une seule fois)
+7. Sinon → ignorer
+
+### Dimensionnement
+
+Pour un génome de 10 Gbases avec couverture 30× :
+- N_brut ≈ 3×10^11 k-mers bruts
+- Espace temporaire .skm ≈ 240 Go (compressé super-kmer)
+- RAM par partition pendant Close() :
+  Avec P=1024 : ~3×10^8 k-mers/partition × 8 = **~2.4 Go**
+  Avec P=4096 : ~7.3×10^7 k-mers/partition × 8 = **~600 Mo**
+
+Le choix de P détermine le compromis nombre de fichiers vs RAM par
+partition.
+
+### Sans FrequencyFilter (déduplication simple)
+
+Pour de la déduplication simple (chaque k-mer écrit une fois), le
+builder peut dédupliquer au niveau des buffers en RAM avant flush.
+Cela réduit significativement l'espace temporaire car les doublons
+au sein d'un même buffer (provenant de séquences proches) sont
+éliminés immédiatement.
+
+---
+
+## API publique visée
+
+### Structures
+
+```go
+// KmerSetGroup est l'entité de base.
+// Un KmerSet est un KmerSetGroup avec Size() == 1.
+type KmerSetGroup struct {
+    // champs internes : path, k, m, P, N, metadata, état
+}
+
+// KmerSetGroupBuilder construit un KmerSetGroup mutable.
+type KmerSetGroupBuilder struct {
+    // champs internes : buffers I/O par partition et par set,
+    // fichiers temporaires .skm, paramètres (minFreq, etc.)
+}
+```
+
+### Construction
+
+```go
+// NewKmerSetGroupBuilder crée un builder pour un nouveau KmerSetGroup.
+//   directory : répertoire de destination
+//   k : taille des k-mers (1-31)
+//   m : taille des minimizers (-1 pour auto = ceil(k/2.5))
+//   n : nombre de sets dans le groupe
+//   P : nombre de partitions (-1 pour auto)
+//   options : options de construction (FrequencyFilter, etc.)
+func NewKmerSetGroupBuilder(directory string, k, m, n, P int,
+    options ...BuilderOption) (*KmerSetGroupBuilder, error)
+
+// WithMinFrequency active le mode FrequencyFilter.
+// Seuls les k-mers vus >= minFreq fois sont conservés dans l'index
+// final. Les super-kmers sont écrits avec leurs doublons pendant
+// la construction ; le comptage exact se fait au Close().
+func WithMinFrequency(minFreq int) BuilderOption
+
+// AddSequence extrait les super-kmers d'une séquence et les écrit
+// dans les fichiers temporaires de partition du set i.
+func (b *KmerSetGroupBuilder) AddSequence(setIndex int, seq *obiseq.BioSequence)
+
+// AddSuperKmer écrit un super-kmer dans le fichier temporaire de
+// sa partition pour le set i.
+func (b *KmerSetGroupBuilder) AddSuperKmer(setIndex int, sk SuperKmer)
+
+// Close finalise la construction :
+//   - flush des buffers d'écriture
+//   - pour chaque partition de chaque set (parallélisable) :
+//     - charger les super-kmers depuis le .skm
+//     - extraire les k-mers canoniques
+//     - trier, dédupliquer (compter si freq filter)
+//     - delta-encoder et écrire le .kdi
+//   - écrire metadata.toml
+//   - supprimer le répertoire .build/
+// Retourne le KmerSetGroup en lecture seule.
+func (b *KmerSetGroupBuilder) Close() (*KmerSetGroup, error)
+```
+
+### Lecture et opérations
+
+```go
+// OpenKmerSetGroup ouvre un index finalisé en lecture seule.
+func OpenKmerSetGroup(directory string) (*KmerSetGroup, error)
+
+// --- Métadonnées (API inchangée) ---
+func (ksg *KmerSetGroup) K() int
+func (ksg *KmerSetGroup) M() int          // nouveau : taille du minimizer
+func (ksg *KmerSetGroup) Partitions() int  // nouveau : nombre de partitions
+func (ksg *KmerSetGroup) Size() int
+func (ksg *KmerSetGroup) Id() string
+func (ksg *KmerSetGroup) SetId(id string)
+func (ksg *KmerSetGroup) HasAttribute(key string) bool
+func (ksg *KmerSetGroup) GetAttribute(key string) (interface{}, bool)
+func (ksg *KmerSetGroup) SetAttribute(key string, value interface{})
+// ... etc (toute l'API attributs actuelle est conservée)
+
+// --- Opérations ensemblistes ---
+// Toutes produisent un nouveau KmerSetGroup singleton sur disque.
+// Opèrent partition par partition en streaming.
+
+func (ksg *KmerSetGroup) Union(outputDir string) (*KmerSetGroup, error)
+func (ksg *KmerSetGroup) Intersect(outputDir string) (*KmerSetGroup, error)
+func (ksg *KmerSetGroup) Difference(outputDir string) (*KmerSetGroup, error)
+func (ksg *KmerSetGroup) QuorumAtLeast(q int, outputDir string) (*KmerSetGroup, error)
+func (ksg *KmerSetGroup) QuorumExactly(q int, outputDir string) (*KmerSetGroup, error)
+func (ksg *KmerSetGroup) QuorumAtMost(q int, outputDir string) (*KmerSetGroup, error)
+
+// --- Opérations entre deux KmerSetGroups ---
+// Les deux groupes doivent avoir les mêmes k, m, P.
+
+func (ksg *KmerSetGroup) UnionWith(other *KmerSetGroup, outputDir string) (*KmerSetGroup, error)
+func (ksg *KmerSetGroup) IntersectWith(other *KmerSetGroup, outputDir string) (*KmerSetGroup, error)
+
+// --- Métriques (résultat en mémoire, pas de sortie disque) ---
+
+func (ksg *KmerSetGroup) JaccardDistanceMatrix() *obidist.DistMatrix
+func (ksg *KmerSetGroup) JaccardSimilarityMatrix() *obidist.DistMatrix
+
+// --- Accès individuel ---
+
+func (ksg *KmerSetGroup) Len(setIndex ...int) uint64
+func (ksg *KmerSetGroup) Contains(setIndex int, kmer uint64) bool
+func (ksg *KmerSetGroup) Iterator(setIndex int) iter.Seq[uint64]
+```
+
+---
+
+## Implémentation interne
+
+### Primitives bas niveau
+
+**`varint.go`** : encode/decode varint uint64
+
+```go
+func EncodeVarint(w io.Writer, v uint64) (int, error)
+func DecodeVarint(r io.Reader) (uint64, error)
+```
+
+### Format .kdi
+
+**`kdi_writer.go`** : écriture d'un fichier .kdi à partir d'un flux
+trié de uint64 (delta-encode au vol).
+
+```go
+type KdiWriter struct { ... }
+func NewKdiWriter(path string) (*KdiWriter, error)
+func (w *KdiWriter) Write(kmer uint64) error
+func (w *KdiWriter) Close() error
+```
+
+**`kdi_reader.go`** : lecture streaming d'un fichier .kdi (décode
+les deltas au vol).
+
+```go
+type KdiReader struct { ... }
+func NewKdiReader(path string) (*KdiReader, error)
+func (r *KdiReader) Next() (uint64, bool)
+func (r *KdiReader) Count() uint64
+func (r *KdiReader) Close() error
+```
+
+### Format .skm
+
+**`skm_writer.go`** : écriture de super-kmers encodés 2 bits/base.
+
+```go
+type SkmWriter struct { ... }
+func NewSkmWriter(path string) (*SkmWriter, error)
+func (w *SkmWriter) Write(sk SuperKmer) error
+func (w *SkmWriter) Close() error
+```
+
+**`skm_reader.go`** : lecture de super-kmers depuis un fichier .skm.
+
+```go
+type SkmReader struct { ... }
+func NewSkmReader(path string) (*SkmReader, error)
+func (r *SkmReader) Next() (SuperKmer, bool)
+func (r *SkmReader) Close() error
+```
+
+### Merge streaming
+
+**`kdi_merge.go`** : k-way merge de plusieurs flux triés.
+
+```go
+type KWayMerge struct { ... }
+func NewKWayMerge(readers []*KdiReader) *KWayMerge
+func (m *KWayMerge) Next() (kmer uint64, count int, ok bool)
+func (m *KWayMerge) Close() error
+```
+
+### Builder
+
+**`kmer_set_builder.go`** : construction d'un KmerSetGroup.
+
+Le builder gère :
+- P × N écrivains .skm bufferisés (un par partition × set)
+- À la clôture : traitement partition par partition
+  (parallélisable sur plusieurs cores)
+
+Gestion mémoire des buffers d'écriture :
+- Chaque SkmWriter a un buffer I/O de taille raisonnable (~64 Ko)
+- Avec P=1024 et N=1 : 1024 × 64 Ko = 64 Mo de buffers
+- Avec P=1024 et N=10 : 640 Mo de buffers
+- Pas de buffer de k-mers en RAM : tout est écrit sur disque
+  immédiatement via les super-kmers
+
+RAM pendant Close() (tri d'une partition) :
+- Charger les super-kmers → extraire les k-mers → tableau []uint64
+- Avec P=1024 et 10^10 k-mers/set : ~10^7 k-mers/partition × 8 = ~80 Mo
+- Avec FrequencyFilter (doublons) et couverture 30× :
+  ~3×10^8/partition × 8 = ~2.4 Go (ajustable via P)
+
+### Structure disk-based
+
+**`kmer_set_disk.go`** : KmerSetGroup en lecture seule.
+
+**`kmer_set_disk_ops.go`** : opérations ensemblistes par merge
+streaming partition par partition.
+
+---
+
+## Ce qui change par rapport à l'API actuelle
+
+### Changements de sémantique
+
+| Aspect | Ancien (roaring) | Nouveau (disk-based) |
+|---|---|---|
+| Stockage | En mémoire (roaring64.Bitmap) | Sur disque (.kdi delta-encoded) |
+| Temporaire construction | En mémoire | Super-kmers sur disque (.skm 2 bits/base) |
+| Mutabilité | Mutable à tout moment | Builder → Close() → immutable |
+| Opérations ensemblistes | Résultat en mémoire | Résultat sur disque (nouveau répertoire) |
+| Contains | O(1) roaring lookup | O(log n) recherche binaire sur .kdi |
+| Itération | Roaring iterator | Streaming décodage delta-varint |
+
+### API conservée (signatures identiques ou quasi-identiques)
+
+- `KmerSetGroup` : `K()`, `Size()`, `Id()`, `SetId()`
+- Toute l'API attributs
+- `JaccardDistanceMatrix()`, `JaccardSimilarityMatrix()`
+- `Len()`, `Contains()`
+
+### API modifiée
+
+- `Union()`, `Intersect()`, etc. : ajout du paramètre `outputDir`
+- `QuorumAtLeast()`, etc. : idem
+- Construction : `NewKmerSetGroupBuilder()` + `AddSequence()` + `Close()`
+  au lieu de manipulation directe
+
+### API supprimée
+
+- `KmerSet` comme type distinct (remplacé par KmerSetGroup singleton)
+- `FrequencyFilter` comme type distinct (mode du Builder)
+- Tout accès direct à `roaring64.Bitmap`
+- `KmerSet.Copy()` (copie de répertoire à la place)
+- `KmerSet.Union()`, `.Intersect()`, `.Difference()` (deviennent méthodes
+  de KmerSetGroup avec outputDir)
+
+---
+
+## Fichiers à créer / modifier dans pkg/obikmer
+
+### Nouveaux fichiers
+
+| Fichier | Contenu |
+|---|---|
+| `varint.go` | Encode/Decode varint uint64 |
+| `kdi_writer.go` | Écrivain de fichiers .kdi (delta-encoded) |
+| `kdi_reader.go` | Lecteur streaming de fichiers .kdi |
+| `skm_writer.go` | Écrivain de super-kmers encodés 2 bits/base |
+| `skm_reader.go` | Lecteur de super-kmers depuis .skm |
+| `kdi_merge.go` | K-way merge streaming de flux triés |
+| `kmer_set_builder.go` | KmerSetGroupBuilder (construction) |
+| `kmer_set_disk.go` | KmerSetGroup disk-based (lecture, métadonnées) |
+| `kmer_set_disk_ops.go` | Opérations ensemblistes streaming |
+
+### Fichiers à supprimer
+
+| Fichier | Raison |
+|---|---|
+| `kmer_set.go` | Remplacé par kmer_set_disk.go |
+| `kmer_set_group.go` | Idem |
+| `kmer_set_attributes.go` | Intégré dans kmer_set_disk.go |
+| `kmer_set_persistence.go` | L'index est nativement sur disque |
+| `kmer_set_group_quorum.go` | Intégré dans kmer_set_disk_ops.go |
+| `frequency_filter.go` | Mode du Builder, plus de type séparé |
+| `kmer_index_builder.go` | Remplacé par kmer_set_builder.go |
+
+### Fichiers conservés tels quels
+
+| Fichier | Contenu |
+|---|---|
+| `encodekmer.go` | Encodage/décodage k-mers |
+| `superkmer.go` | Structure SuperKmer |
+| `superkmer_iter.go` | IterSuperKmers, IterCanonicalKmers |
+| `encodefourmer.go` | Encode4mer |
+| `counting.go` | Count4Mer |
+| `kmermap.go` | KmerMap (usage indépendant) |
+| `debruijn.go` | Graphe de de Bruijn |
+
+---
+
+## Ordre d'implémentation
+
+1. `varint.go` + tests
+2. `skm_writer.go` + `skm_reader.go` + tests
+3. `kdi_writer.go` + `kdi_reader.go` + tests
+4. `kdi_merge.go` + tests
+5. `kmer_set_builder.go` + tests (construction + Close)
+6. `kmer_set_disk.go` (structure, métadonnées, Open)
+7. `kmer_set_disk_ops.go` + tests (Union, Intersect, Quorum, Jaccard)
+8. Adaptation de `pkg/obitools/obikindex/`
+9. Suppression des anciens fichiers roaring
+10. Adaptation des tests existants
+
+Chaque étape est testable indépendamment.
+
+---
+
+## Dépendances externes
+
+### Supprimées
+
+- `github.com/RoaringBitmap/roaring` : plus nécessaire pour les
+  index k-mers (vérifier si d'autres packages l'utilisent encore)
+
+### Ajoutées
+
+- Aucune. Varint, delta-encoding, merge, encodage 2 bits/base :
+  tout est implémentable en Go standard.

blackboard/Prospective/large_sequence_parsing.md

@@ -0,0 +1,264 @@
+# Optimisation du parsing des grandes séquences
+
+## Contexte
+
+OBITools4 doit pouvoir traiter des séquences de taille chromosomique (plusieurs Gbp), notamment
+issues de fichiers GenBank/EMBL (assemblages de génomes) ou de fichiers FASTA convertis depuis
+ces formats.
+
+## Architecture actuelle
+
+### Pipeline de lecture (`pkg/obiformats/`)
+
+```
+ReadFileChunk (goroutine)
+    → ChannelFileChunk
+    → N × _ParseGenbankFile / _ParseFastaFile (goroutines)
+    → IBioSequence
+```
+
+`ReadFileChunk` (`file_chunk_read.go`) lit le fichier par morceaux via une chaîne de
+`PieceOfChunk` (rope). Chaque nœud fait `fileChunkSize` bytes :
+
+- GenBank/EMBL : 128 MB (`1024*1024*128`)
+- FASTA/FASTQ  : 1 MB (`1024*1024`)
+
+La chaîne est accumulée jusqu'à trouver la fin du dernier enregistrement complet (splitter),
+puis `Pack()` est appelé pour fusionner tous les nœuds en un seul buffer contigu. Ce buffer
+est transmis au parseur via `FileChunk.Raw *bytes.Buffer`.
+
+### Parseur GenBank (`genbank_read.go`)
+
+`GenbankChunkParser` reçoit un `io.Reader` sur le buffer packé, lit ligne par ligne via
+`bufio.NewReader` (buffer 4096 bytes), et pour chaque ligne de la section `ORIGIN` :
+
+```go
+line = string(bline)                        // allocation par ligne
+cleanline := strings.TrimSpace(line)        // allocation
+parts := strings.SplitN(cleanline, " ", 7) // allocation []string + substrings
+for i := 1; i < lparts; i++ {
+    seqBytes.WriteString(parts[i])
+}
+```
+
+Point positif : `seqBytes` est pré-alloué grâce à `lseq` extrait de la ligne `LOCUS`.
+
+### Parseur FASTA (`fastaseq_read.go`)
+
+`FastaChunkParser` lit **octet par octet** via `scanner.ReadByte()`. Pour 3 Gbp :
+3 milliards d'appels. `seqBytes` est un `bytes.Buffer{}` sans pré-allocation.
+
+## Problème principal
+
+Pour une séquence de plusieurs Gbp, `Pack()` fusionne une chaîne de ~N nœuds de 128 MB en
+un seul buffer contigu. C'est une allocation de N × 128 MB suivie d'une copie de toutes les
+données. Bien que l'implémentation de `Pack()` soit efficace (libère les nœuds au fur et à
+mesure via `slices.Grow`), la copie est inévitable avec l'architecture actuelle.
+
+De plus, le parseur GenBank produit des dizaines de millions d'allocations temporaires pour
+parser la section `ORIGIN` (une par ligne).
+
+## Invariant clé découvert
+
+**Si la rope a plus d'un nœud, le premier nœud seul ne se termine pas sur une frontière
+d'enregistrement** (pas de `//\n` en fin de `piece1`).
+
+Preuve par construction dans `ReadFileChunk` :
+- `splitter` est appelé dès le premier nœud (ligne 157)
+- Si `end >= 0` → frontière trouvée dans 128 MB → boucle interne sautée → rope à 1 nœud
+- Si `end < 0` → boucle interne ajoute des nœuds → rope à ≥ 2 nœuds
+
+Corollaire : si rope à 1 nœud, `Pack()` ne fait rien (aucun nœud suivant).
+
+**Attention** : rope à ≥ 2 nœuds ne signifie pas qu'il n'y a qu'une seule séquence dans
+la rope. La rope packée peut contenir plusieurs enregistrements complets. Exemple : records
+de 80 MB → `nextpieces` (48 MB de reste) + nouveau nœud (128 MB) = rope à 2 nœuds
+contenant 2 records complets + début d'un troisième.
+
+L'invariant dit seulement que `piece1` seul est incomplet — pas que la rope entière
+ne contient qu'un seul record.
+
+**Invariant : le dernier FileChunk envoyé finit sur une frontière d'enregistrement.**
+
+Deux chemins dans `ReadFileChunk` :
+
+1. **Chemin normal** (`end >= 0` via `splitter`) : le buffer est explicitement tronqué à
+   `end` (ligne 200 : `pieces.data = pieces.data[:end]`). Frontière garantie par construction
+   pour tous les formats. ✓
+
+2. **Chemin EOF** (`end < 0`, `end = pieces.Len()`) : tout le reste du fichier est envoyé.
+   - **GenBank/EMBL** : présuppose fichier bien formé (se termine par `//\n`). Le parseur
+     lève un `log.Fatalf` sur tout état inattendu — filet de sécurité suffisant. ✓
+   - **FASTQ** : présupposé, vérifié par le parseur. ✓
+   - **FASTA** : garanti par le format lui-même (fin d'enregistrement = EOF ou `>`). ✓
+
+**Hypothèse de travail adoptée** : les fichiers d'entrée sont bien formés. Dans le pire cas,
+le parseur lèvera une erreur explicite. Il n'y a pas de risque de corruption silencieuse.
+
+## Piste d'optimisation : se dispenser de Pack()
+
+### Idée centrale
+
+Au lieu de fusionner la rope avant de la passer au parseur, **parser directement la rope
+nœud par nœud**, et **écrire la séquence compactée in-place dans le premier nœud**.
+
+Pourquoi c'est sûr :
+- Le header (LOCUS, DEFINITION, SOURCE, FEATURES) est **petit** et traité en premier
+- La séquence (ORIGIN) est **à la fin** du record
+- Au moment d'écrire la séquence depuis l'offset 0 de `piece1`, le pointeur de lecture
+  est profond dans la rope (offset >> 0) → jamais de collision
+- La séquence compactée est toujours plus courte que les données brutes
+
+### Pré-allocation
+
+Pour GenBank/EMBL : `lseq` est connu dès la ligne `LOCUS`/`ID` (première ligne, dans
+`piece1`). On peut faire `slices.Grow(piece1.data, lseq)` dès ce moment.
+
+Pour FASTA : pas de taille garantie dans le header, mais `rope.Len()` donne un majorant.
+On peut utiliser `rope.Len() / 2` comme estimation initiale.
+
+### Gestion des jonctions entre nœuds
+
+Une ligne peut chevaucher deux nœuds (rare avec 128 MB, mais possible). Solution : carry
+buffer de ~128 bytes pour les quelques bytes en fin de nœud.
+
+### Cas FASTA/FASTQ multi-séquences
+
+Un FileChunk peut contenir N séquences (notamment FASTA/FASTQ courts). Dans ce cas
+l'écriture in-place dans `piece1` n'est pas applicable directement — on écrase des données
+nécessaires aux séquences suivantes.
+
+Stratégie par cas :
+- **Rope à 1 nœud** (record ≤ 128 MB) : `Pack()` est trivial (no-op), parseur actuel OK
+- **Rope à ≥ 2 nœuds** : par l'invariant, `piece1` ne contient pas de record complet →
+  une seule grande séquence → in-place applicable
+
+### Format d'une ligne séquence GenBank (Après ORIGIN)
+
+```
+/^ *[0-9]+( [nuc]{10}){0,5} [nuc]{1,10}/
+```
+
+### Format d'une ligne séquence GenBank (Après SQ)
+
+La ligne SQ contient aussi la taille de la séquence
+
+```
+/^ *( [nuc]{10}){0,5} [nuc]{1,10} *[0-9]+/
+```
+
+Compactage in-place sur `bline` ([]byte brut, sans conversion `string`) :
+
+```go
+w := 0
+i := 0
+for i < len(bline) && bline[i] == ' '  { i++ }   // skip indentation
+for i < len(bline) && bline[i] <= '9'  { i++ }   // skip position number
+for ; i < len(bline); i++ {
+    if bline[i] != ' ' {
+        bline[w] = bline[i]
+        w++
+    }
+}
+// écrire bline[:w] directement dans piece1.data[seqOffset:]
+```
+
+## Changements nécessaires
+
+1. **`FileChunk`** : exposer la rope `*PieceOfChunk` non-packée en plus (ou à la place)
+   de `Raw *bytes.Buffer`
+2. **`GenbankChunkParser` / `EmblChunkParser`** : accepter `*PieceOfChunk`, parser la
+   rope séquentiellement avec carry buffer pour les jonctions
+3. **`FastaChunkParser`** : idem, avec in-place conditionnel selon taille de la rope
+4. **`ReadFileChunk`** : ne pas appeler `Pack()` avant envoi sur le channel (ou version
+   alternative `ReadFileChunkRope`)
+
+## Fichiers concernés
+
+- `pkg/obiformats/file_chunk_read.go` — structure rope, `ReadFileChunk`
+- `pkg/obiformats/genbank_read.go` — `GenbankChunkParser`, `_ParseGenbankFile`
+- `pkg/obiformats/embl_read.go` — `EmblChunkParser`, `ReadEMBL`
+- `pkg/obiformats/fastaseq_read.go` — `FastaChunkParser`, `_ParseFastaFile`
+- `pkg/obiformats/fastqseq_read.go` — parseur FASTQ (même structure)
+
+## Plan d'implémentation : parseur GenBank sur rope
+
+### Contexte
+
+Baseline mesurée : `obiconvert gbpln640.seq.gz` → 49s real, 42s user, 29s sys, **57 GB RSS**.
+Le sys élevé indique des allocations massives. Deux causes :
+1. `Pack()` : fusionne toute la rope (N × 128 MB) en un buffer contigu avant de parser
+2. Parser ORIGIN : `string(bline)` + `TrimSpace` + `SplitN` × millions de lignes
+
+### 1. `gbRopeScanner`
+
+Struct de lecture ligne par ligne sur la rope, sans allocation heap :
+
+```go
+type gbRopeScanner struct {
+    current *PieceOfChunk
+    pos     int
+    carry   [256]byte  // stack-allocated, max GenBank line = 80 chars
+    carryN  int
+}
+```
+
+`ReadLine()` :
+- Cherche `\n` dans `current.data[pos:]` via `bytes.IndexByte`
+- Si trouvé sans carry : retourne slice direct du node (zéro alloc)
+- Si trouvé avec carry : copie dans carry buffer, retourne `carry[:n]`
+- Si non trouvé : copie le reste dans carry, avance au node suivant, recommence
+- EOF : retourne `carry[:carryN]` puis nil
+
+`extractSequence(dest []byte, UtoT bool) int` :
+- Scan direct des bytes pour section ORIGIN, sans passer par ReadLine
+- Machine d'états : lineStart → skip espaces/digits → copier nucléotides dans dest
+- Stop sur `//` en début de ligne
+- Zéro allocation, UtoT inline
+
+### 2. `GenbankChunkParserRope`
+
+```go
+func GenbankChunkParserRope(source string, rope *PieceOfChunk,
+    withFeatureTable, UtoT bool) (obiseq.BioSequenceSlice, error)
+```
+
+- Même machine d'états que `GenbankChunkParser`, sur `[]byte` (`bytes.HasPrefix`)
+- LOCUS : extrait `id` et `lseq` par scan direct (remplace `_seqlenght_rx`)
+- FEATURES / default inFeature : taxid extrait par scan de `/db_xref="taxon:`
+  dans la source feature ; `featBytes` rempli seulement si `withFeatureTable=true`
+- DEFINITION : toujours conservée
+- ORIGIN : `dest = make([]byte, 0, lseq+20)` puis `s.extractSequence(dest, UtoT)`
+
+### 3. Modifications `_ParseGenbankFile` et `ReadGenbank`
+
+`_ParseGenbankFile` utilise `chunk.Rope` :
+```go
+sequences, err := GenbankChunkParserRope(chunk.Source, chunk.Rope, ...)
+```
+
+`ReadGenbank` passe `pack=false` :
+```go
+entry_channel := ReadFileChunk(..., false)
+```
+
+### 4. Ce qui NE change pas
+
+- `GenbankChunkParser` reste (référence, tests)
+- `ReadFileChunk`, `Pack()`, autres parseurs (EMBL, FASTA, FASTQ) : inchangés
+
+### 5. Gains attendus
+
+- **RSS** : pic ≈ 128 MB × workers (au lieu de N × 128 MB)
+- **Temps sys** : élimination des mmap/munmap pour les gros buffers
+- **Temps user** : ~50M allocations éliminées
+
+### 6. Vérification
+
+```bash
+/usr/local/go/bin/go build ./...
+diff <(obiconvert gbpln640.seq.gz) gbpln640.reference.fasta
+cd bugs/genbank && ./benchmark.sh gbpln640.seq.gz
+```
+
+Cible : RSS < 1 GB, temps comparable ou meilleur.

blackboard/architechture/architecture-commande-obitools.md

@@ -0,0 +1,735 @@
+# Architecture d'une commande OBITools
+
+## Vue d'ensemble
+
+Une commande OBITools suit une architecture modulaire et standardisée qui sépare clairement les responsabilités entre :
+- Le package de la commande dans `pkg/obitools/<nom_commande>/`
+- L'exécutable dans `cmd/obitools/<nom_commande>/`
+
+Cette architecture favorise la réutilisabilité du code, la testabilité et la cohérence entre les différentes commandes de la suite OBITools.
+
+## Structure du projet
+
+```
+obitools4/
+├── pkg/obitools/
+│   ├── obiconvert/          # Commande de conversion (base pour toutes)
+│   │   ├── obiconvert.go    # Fonctions vides (pas d'implémentation)
+│   │   ├── options.go       # Définition des options CLI
+│   │   ├── sequence_reader.go  # Lecture des séquences
+│   │   └── sequence_writer.go  # Écriture des séquences
+│   ├── obiuniq/             # Commande de déréplication
+│   │   ├── obiuniq.go       # (fichier vide)
+│   │   ├── options.go       # Options spécifiques à obiuniq
+│   │   └── unique.go        # Implémentation du traitement
+│   ├── obipairing/          # Assemblage de lectures paired-end
+│   ├── obisummary/          # Résumé de fichiers de séquences
+│   └── obimicrosat/         # Détection de microsatellites
+└── cmd/obitools/
+    ├── obiconvert/
+    │   └── main.go          # Point d'entrée de la commande
+    ├── obiuniq/
+    │   └── main.go
+    ├── obipairing/
+    │   └── main.go
+    ├── obisummary/
+    │   └── main.go
+    └── obimicrosat/
+        └── main.go
+```
+
+## Composants de l'architecture
+
+### 1. Package `pkg/obitools/<commande>/`
+
+Chaque commande possède son propre package dans `pkg/obitools/` qui contient l'implémentation complète de la logique métier. Ce package est structuré en plusieurs fichiers :
+
+#### a) `options.go` - Gestion des options CLI
+
+Ce fichier définit :
+- Les **variables globales** privées (préfixées par `_`) stockant les valeurs des options
+- La fonction **`OptionSet()`** qui configure toutes les options pour la commande
+- Les fonctions **`CLI*()`** qui retournent les valeurs des options (getters)
+- Les fonctions **`Set*()`** qui permettent de définir les options programmatiquement (setters)
+
+**Exemple (obiuniq/options.go) :**
+
+```go
+package obiuniq
+
+import (
+    "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+    "github.com/DavidGamba/go-getoptions"
+)
+
+// Variables globales privées pour stocker les options
+var _StatsOn = make([]string, 0, 10)
+var _Keys = make([]string, 0, 10)
+var _InMemory = false
+var _chunks = 100
+
+// Configuration des options spécifiques à la commande
+func UniqueOptionSet(options *getoptions.GetOpt) {
+    options.StringSliceVar(&_StatsOn, "merge", 1, 1,
+        options.Alias("m"),
+        options.ArgName("KEY"),
+        options.Description("Adds a merged attribute..."))
+    
+    options.BoolVar(&_InMemory, "in-memory", _InMemory,
+        options.Description("Use memory instead of disk..."))
+    
+    options.IntVar(&_chunks, "chunk-count", _chunks,
+        options.Description("In how many chunks..."))
+}
+
+// OptionSet combine les options de base + les options spécifiques
+func OptionSet(options *getoptions.GetOpt) {
+    obiconvert.OptionSet(false)(options)  // Options de base
+    UniqueOptionSet(options)              // Options spécifiques
+}
+
+// Getters pour accéder aux valeurs des options
+func CLIStatsOn() []string {
+    return _StatsOn
+}
+
+func CLIUniqueInMemory() bool {
+    return _InMemory
+}
+
+// Setters pour définir les options programmatiquement
+func SetUniqueInMemory(inMemory bool) {
+    _InMemory = inMemory
+}
+```
+
+**Convention de nommage :**
+- Variables privées : `_NomOption` (underscore préfixe)
+- Getters : `CLINomOption()` (préfixe CLI)
+- Setters : `SetNomOption()` (préfixe Set)
+
+#### b) Fichier(s) d'implémentation
+
+Un ou plusieurs fichiers contenant la logique métier de la commande :
+
+**Exemple (obiuniq/unique.go) :**
+
+```go
+package obiuniq
+
+import (
+    "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+    "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obichunk"
+)
+
+// Fonction CLI principale qui orchestre le traitement
+func CLIUnique(sequences obiiter.IBioSequence) obiiter.IBioSequence {
+    // Récupération des options via les getters CLI*()
+    options := make([]obichunk.WithOption, 0, 30)
+    
+    options = append(options,
+        obichunk.OptionBatchCount(CLINumberOfChunks()),
+    )
+    
+    if CLIUniqueInMemory() {
+        options = append(options, obichunk.OptionSortOnMemory())
+    } else {
+        options = append(options, obichunk.OptionSortOnDisk())
+    }
+    
+    // Appel de la fonction de traitement réelle
+    iUnique, err := obichunk.IUniqueSequence(sequences, options...)
+    
+    if err != nil {
+        log.Fatal(err)
+    }
+    
+    return iUnique
+}
+```
+
+**Autres exemples d'implémentation :**
+
+- **obimicrosat/microsat.go** : Contient `MakeMicrosatWorker()` et `CLIAnnotateMicrosat()`
+- **obisummary/obisummary.go** : Contient `ISummary()` et les structures de données
+
+#### c) Fichiers utilitaires (optionnel)
+
+Certaines commandes ont des fichiers additionnels pour des fonctionnalités spécifiques.
+
+**Exemple (obipairing/options.go) :**
+
+```go
+// Fonction spéciale pour créer un itérateur de séquences pairées
+func CLIPairedSequence() (obiiter.IBioSequence, error) {
+    forward, err := obiconvert.CLIReadBioSequences(_ForwardFile)
+    if err != nil {
+        return obiiter.NilIBioSequence, err
+    }
+    
+    reverse, err := obiconvert.CLIReadBioSequences(_ReverseFile)
+    if err != nil {
+        return obiiter.NilIBioSequence, err
+    }
+    
+    paired := forward.PairTo(reverse)
+    return paired, nil
+}
+```
+
+### 2. Package `obiconvert` - La base commune
+
+Le package `obiconvert` est spécial car il fournit les fonctionnalités de base utilisées par toutes les autres commandes :
+
+#### Fonctionnalités fournies :
+
+1. **Lecture de séquences** (`sequence_reader.go`)
+   - `CLIReadBioSequences()` : lecture depuis fichiers ou stdin
+   - Support de multiples formats (FASTA, FASTQ, EMBL, GenBank, etc.)
+   - Gestion des fichiers multiples
+   - Barre de progression optionnelle
+
+2. **Écriture de séquences** (`sequence_writer.go`)
+   - `CLIWriteBioSequences()` : écriture vers fichiers ou stdout
+   - Support de multiples formats
+   - Gestion des lectures pairées
+   - Compression optionnelle
+
+3. **Options communes** (`options.go`)
+   - Options d'entrée (format, skip, etc.)
+   - Options de sortie (format, fichier, compression)
+   - Options de mode (barre de progression, etc.)
+
+#### Utilisation par les autres commandes :
+
+Toutes les commandes incluent les options de `obiconvert` via :
+
+```go
+func OptionSet(options *getoptions.GetOpt) {
+    obiconvert.OptionSet(false)(options)  // false = pas de fichiers pairés
+    MaCommandeOptionSet(options)          // Options spécifiques
+}
+```
+
+### 3. Exécutable `cmd/obitools/<commande>/main.go`
+
+Le fichier `main.go` de chaque commande est volontairement **minimaliste** et suit toujours le même pattern :
+
+```go
+package main
+
+import (
+    "os"
+    "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
+    "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+    "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+    "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/macommande"
+    "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+)
+
+func main() {
+    // 1. Configuration optionnelle de paramètres par défaut
+    obidefault.SetBatchSize(10)
+    
+    // 2. Génération du parser d'options
+    optionParser := obioptions.GenerateOptionParser(
+        "macommande",                    // Nom de la commande
+        "description de la commande",    // Description
+        macommande.OptionSet)            // Fonction de configuration des options
+    
+    // 3. Parsing des arguments
+    _, args := optionParser(os.Args)
+    
+    // 4. Lecture des séquences d'entrée
+    sequences, err := obiconvert.CLIReadBioSequences(args...)
+    obiconvert.OpenSequenceDataErrorMessage(args, err)
+    
+    // 5. Traitement spécifique de la commande
+    resultat := macommande.CLITraitement(sequences)
+    
+    // 6. Écriture des résultats
+    obiconvert.CLIWriteBioSequences(resultat, true)
+    
+    // 7. Attente de la fin du pipeline
+    obiutils.WaitForLastPipe()
+}
+```
+
+## Patterns architecturaux
+
+### Pattern 1 : Pipeline de traitement de séquences
+
+La plupart des commandes suivent ce pattern :
+
+```
+Lecture → Traitement → Écriture
+```
+
+**Exemples :**
+- **obiconvert** : Lecture → Écriture (conversion de format)
+- **obiuniq** : Lecture → Déréplication → Écriture
+- **obimicrosat** : Lecture → Annotation → Filtrage → Écriture
+
+### Pattern 2 : Traitement avec entrées multiples
+
+Certaines commandes acceptent plusieurs fichiers d'entrée :
+
+**obipairing** :
+```
+Lecture Forward + Lecture Reverse → Pairing → Assemblage → Écriture
+```
+
+### Pattern 3 : Traitement sans écriture de séquences
+
+**obisummary** : produit un résumé JSON/YAML au lieu de séquences
+
+```go
+func main() {
+    // ... parsing options et lecture ...
+    
+    summary := obisummary.ISummary(fs, obisummary.CLIMapSummary())
+    
+    // Formatage et affichage direct
+    if obisummary.CLIOutFormat() == "json" {
+        output, _ := json.MarshalIndent(summary, "", "  ")
+        fmt.Print(string(output))
+    } else {
+        output, _ := yaml.Marshal(summary)
+        fmt.Print(string(output))
+    }
+}
+```
+
+### Pattern 4 : Utilisation de Workers
+
+Les commandes qui transforment des séquences utilisent souvent le pattern Worker :
+
+```go
+// Création d'un worker
+worker := MakeMicrosatWorker(
+    CLIMinUnitLength(),
+    CLIMaxUnitLength(),
+    // ... autres paramètres
+)
+
+// Application du worker sur l'itérateur
+newIter = iterator.MakeIWorker(
+    worker, 
+    false,                              // merge results
+    obidefault.ParallelWorkers()        // parallélisation
+)
+```
+
+## Étapes d'implémentation d'une nouvelle commande
+
+### Étape 1 : Créer le package dans `pkg/obitools/`
+
+```bash
+mkdir -p pkg/obitools/macommande
+```
+
+### Étape 2 : Créer `options.go`
+
+```go
+package macommande
+
+import (
+    "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+    "github.com/DavidGamba/go-getoptions"
+)
+
+// Variables privées pour les options
+var _MonOption = "valeur_par_defaut"
+
+// Configuration des options spécifiques
+func MaCommandeOptionSet(options *getoptions.GetOpt) {
+    options.StringVar(&_MonOption, "mon-option", _MonOption,
+        options.Alias("o"),
+        options.Description("Description de l'option"))
+}
+
+// OptionSet combine options de base + spécifiques
+func OptionSet(options *getoptions.GetOpt) {
+    obiconvert.OptionSet(false)(options)  // false si pas de fichiers pairés
+    MaCommandeOptionSet(options)
+}
+
+// Getters
+func CLIMonOption() string {
+    return _MonOption
+}
+
+// Setters
+func SetMonOption(value string) {
+    _MonOption = value
+}
+```
+
+### Étape 3 : Créer le fichier d'implémentation
+
+Créer `macommande.go` (ou un nom plus descriptif) :
+
+```go
+package macommande
+
+import (
+    "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+    "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+// Fonction de traitement principale
+func CLIMaCommande(sequences obiiter.IBioSequence) obiiter.IBioSequence {
+    // Récupération des options
+    option := CLIMonOption()
+    
+    // Implémentation du traitement
+    // ...
+    
+    return resultat
+}
+```
+
+### Étape 4 : Créer l'exécutable dans `cmd/obitools/`
+
+```bash
+mkdir -p cmd/obitools/macommande
+```
+
+Créer `main.go` :
+
+```go
+package main
+
+import (
+    "os"
+    "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+    "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+    "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/macommande"
+    "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+)
+
+func main() {
+    // Parser d'options
+    optionParser := obioptions.GenerateOptionParser(
+        "macommande",
+        "Description courte de ma commande",
+        macommande.OptionSet)
+    
+    _, args := optionParser(os.Args)
+    
+    // Lecture
+    sequences, err := obiconvert.CLIReadBioSequences(args...)
+    obiconvert.OpenSequenceDataErrorMessage(args, err)
+    
+    // Traitement
+    resultat := macommande.CLIMaCommande(sequences)
+    
+    // Écriture
+    obiconvert.CLIWriteBioSequences(resultat, true)
+    
+    // Attente
+    obiutils.WaitForLastPipe()
+}
+```
+
+### Étape 5 : Configurations optionnelles
+
+Dans `main.go`, avant le parsing des options, on peut configurer :
+
+```go
+// Taille des batchs de séquences
+obidefault.SetBatchSize(10)
+
+// Nombre de workers en lecture (strict)
+obidefault.SetStrictReadWorker(2)
+
+// Nombre de workers en écriture
+obidefault.SetStrictWriteWorker(2)
+
+// Désactiver la lecture des qualités
+obidefault.SetReadQualities(false)
+```
+
+### Étape 6 : Gestion des erreurs
+
+Utiliser les fonctions utilitaires pour les messages d'erreur cohérents :
+
+```go
+// Pour les erreurs d'ouverture de fichiers
+obiconvert.OpenSequenceDataErrorMessage(args, err)
+
+// Pour les erreurs générales
+if err != nil {
+    log.Errorf("Message d'erreur: %v", err)
+    os.Exit(1)
+}
+```
+
+### Étape 7 : Tests et debugging (optionnel)
+
+Des commentaires dans le code montrent comment activer le profiling :
+
+```go
+// go tool pprof -http=":8000" ./macommande ./cpu.pprof
+// f, err := os.Create("cpu.pprof")
+// if err != nil {
+//     log.Fatal(err)
+// }
+// pprof.StartCPUProfile(f)
+// defer pprof.StopCPUProfile()
+
+// go tool trace cpu.trace
+// ftrace, err := os.Create("cpu.trace")
+// if err != nil {
+//     log.Fatal(err)
+// }
+// trace.Start(ftrace)
+// defer trace.Stop()
+```
+
+## Bonnes pratiques observées
+
+### 1. Séparation des responsabilités
+
+- **`main.go`** : orchestration minimale
+- **`options.go`** : définition et gestion des options
+- **Fichiers d'implémentation** : logique métier
+
+### 2. Convention de nommage cohérente
+
+- Variables d'options : `_NomOption`
+- Getters CLI : `CLINomOption()`
+- Setters : `SetNomOption()`
+- Fonctions de traitement CLI : `CLITraitement()`
+
+### 3. Réutilisation du code
+
+- Toutes les commandes réutilisent `obiconvert` pour l'I/O
+- Les options communes sont partagées
+- Les fonctions utilitaires sont centralisées
+
+### 4. Configuration par défaut
+
+Les valeurs par défaut sont :
+- Définies lors de l'initialisation des variables
+- Modifiables via les options CLI
+- Modifiables programmatiquement via les setters
+
+### 5. Gestion des formats
+
+Support automatique de multiples formats :
+- FASTA / FASTQ (avec compression gzip)
+- EMBL / GenBank
+- ecoPCR
+- CSV
+- JSON (avec différents formats d'en-têtes)
+
+### 6. Parallélisation
+
+Les commandes utilisent les workers parallèles via :
+- `obidefault.ParallelWorkers()`
+- `obidefault.SetStrictReadWorker(n)`
+- `obidefault.SetStrictWriteWorker(n)`
+
+### 7. Logging cohérent
+
+Utilisation de `logrus` pour tous les logs :
+```go
+log.Printf("Message informatif")
+log.Errorf("Message d'erreur: %v", err)
+log.Fatal(err)  // Arrêt du programme
+```
+
+## Dépendances principales
+
+### Packages internes OBITools
+
+- `pkg/obidefault` : valeurs par défaut et configuration globale
+- `pkg/obioptions` : génération du parser d'options
+- `pkg/obiiter` : itérateurs de séquences biologiques
+- `pkg/obiseq` : structures et fonctions pour séquences biologiques
+- `pkg/obiformats` : lecture/écriture de différents formats
+- `pkg/obiutils` : fonctions utilitaires diverses
+- `pkg/obichunk` : traitement par chunks (pour dereplication, etc.)
+
+### Packages externes
+
+- `github.com/DavidGamba/go-getoptions` : parsing des options CLI
+- `github.com/sirupsen/logrus` : logging structuré
+- `gopkg.in/yaml.v3` : encodage/décodage YAML
+- `github.com/dlclark/regexp2` : expressions régulières avancées
+
+## Cas spéciaux
+
+### Commande avec fichiers pairés (obipairing)
+
+```go
+func OptionSet(options *getoptions.GetOpt) {
+    obiconvert.OutputOptionSet(options)
+    obiconvert.InputOptionSet(options)
+    PairingOptionSet(options)  // Options spécifiques au pairing
+}
+
+func CLIPairedSequence() (obiiter.IBioSequence, error) {
+    forward, err := obiconvert.CLIReadBioSequences(_ForwardFile)
+    // ...
+    reverse, err := obiconvert.CLIReadBioSequences(_ReverseFile)
+    // ...
+    paired := forward.PairTo(reverse)
+    return paired, nil
+}
+```
+
+Dans `main.go` :
+```go
+pairs, err := obipairing.CLIPairedSequence()  // Lecture spéciale
+if err != nil {
+    log.Errorf("Cannot open file (%v)", err)
+    os.Exit(1)
+}
+
+paired := obipairing.IAssemblePESequencesBatch(
+    pairs,
+    obipairing.CLIGapPenality(),
+    // ... autres paramètres
+)
+```
+
+### Commande sans sortie de séquences (obisummary)
+
+Au lieu de `obiconvert.CLIWriteBioSequences()`, affichage direct :
+
+```go
+summary := obisummary.ISummary(fs, obisummary.CLIMapSummary())
+
+if obisummary.CLIOutFormat() == "json" {
+    output, _ := json.MarshalIndent(summary, "", "  ")
+    fmt.Print(string(output))
+} else {
+    output, _ := yaml.Marshal(summary)
+    fmt.Print(string(output))
+}
+fmt.Printf("\n")
+```
+
+### Commande avec Workers personnalisés (obimicrosat)
+
+```go
+func CLIAnnotateMicrosat(iterator obiiter.IBioSequence) obiiter.IBioSequence {
+    // Création du worker
+    worker := MakeMicrosatWorker(
+        CLIMinUnitLength(),
+        CLIMaxUnitLength(),
+        CLIMinUnitCount(),
+        CLIMinLength(),
+        CLIMinFlankLength(),
+        CLIReoriented(),
+    )
+    
+    // Application du worker
+    newIter := iterator.MakeIWorker(
+        worker, 
+        false,                           // pas de merge
+        obidefault.ParallelWorkers(),    // parallélisation
+    )
+    
+    return newIter.FilterEmpty()  // Filtrage des résultats vides
+}
+```
+
+## Diagramme de flux d'exécution
+
+```
+┌─────────────────────────────────────────────────────────────┐
+│                      cmd/obitools/macommande/main.go        │
+└─────────────────────────────────────────────────────────────┘
+                              │
+                              ▼
+┌─────────────────────────────────────────────────────────────┐
+│  1. Génération du parser d'options                          │
+│     obioptions.GenerateOptionParser(                        │
+│         "macommande",                                       │
+│         "description",                                      │
+│         macommande.OptionSet)                               │
+└─────────────────────────────────────────────────────────────┘
+                              │
+                              ▼
+┌─────────────────────────────────────────────────────────────┐
+│  pkg/obitools/macommande/options.go                         │
+│  ┌─────────────────────────────────────────────────────┐   │
+│  │ func OptionSet(options *getoptions.GetOpt)          │   │
+│  │   obiconvert.OptionSet(false)(options) ───────────┐ │   │
+│  │   MaCommandeOptionSet(options)                    │ │   │
+│  └───────────────────────────────────────────────────┼─┘   │
+└────────────────────────────────────────────────────────┼─────┘
+                              │                         │
+                              │                         │
+                ┌─────────────┘                         │
+                │                                       │
+                ▼                                       ▼
+┌─────────────────────────────────┐  ┌───────────────────────────────┐
+│ 2. Parsing des arguments        │  │ pkg/obitools/obiconvert/      │
+│    _, args := optionParser(...) │  │    options.go                 │
+└─────────────────────────────────┘  │  - InputOptionSet()           │
+                │                     │  - OutputOptionSet()          │
+                ▼                     │  - PairedFilesOptionSet()     │
+┌─────────────────────────────────┐  └───────────────────────────────┘
+│ 3. Lecture des séquences        │
+│    CLIReadBioSequences(args)    │
+└─────────────────────────────────┘
+                │
+                ▼
+┌─────────────────────────────────────────────────────────────┐
+│ pkg/obitools/obiconvert/sequence_reader.go                  │
+│  - ExpandListOfFiles()                                      │
+│  - ReadSequencesFromFile() / ReadSequencesFromStdin()       │
+│  - Support: FASTA, FASTQ, EMBL, GenBank, ecoPCR, CSV        │
+└─────────────────────────────────────────────────────────────┘
+                │
+                ▼ obiiter.IBioSequence
+┌─────────────────────────────────────────────────────────────┐
+│ 4. Traitement spécifique                                    │
+│    macommande.CLITraitement(sequences)                      │
+└─────────────────────────────────────────────────────────────┘
+                │
+                ▼
+┌─────────────────────────────────────────────────────────────┐
+│ pkg/obitools/macommande/<implementation>.go                 │
+│  - Récupération des options via CLI*() getters             │
+│  - Application de la logique métier                         │
+│  - Retour d'un nouvel iterator                              │
+└─────────────────────────────────────────────────────────────┘
+                │
+                ▼ obiiter.IBioSequence
+┌─────────────────────────────────────────────────────────────┐
+│ 5. Écriture des résultats                                   │
+│    CLIWriteBioSequences(resultat, true)                     │
+└─────────────────────────────────────────────────────────────┘
+                │
+                ▼
+┌─────────────────────────────────────────────────────────────┐
+│ pkg/obitools/obiconvert/sequence_writer.go                  │
+│  - WriteSequencesToFile() / WriteSequencesToStdout()        │
+│  - Support: FASTA, FASTQ, JSON                              │
+│  - Gestion des lectures pairées                             │
+│  - Compression optionnelle                                  │
+└─────────────────────────────────────────────────────────────┘
+                │
+                ▼
+┌─────────────────────────────────────────────────────────────┐
+│ 6. Attente de fin du pipeline                               │
+│    obiutils.WaitForLastPipe()                               │
+└─────────────────────────────────────────────────────────────┘
+```
+
+## Conclusion
+
+L'architecture des commandes OBITools est conçue pour :
+
+1. **Maximiser la réutilisation** : `obiconvert` fournit les fonctionnalités communes
+2. **Simplifier l'ajout de nouvelles commandes** : pattern standardisé et minimaliste
+3. **Faciliter la maintenance** : séparation claire des responsabilités
+4. **Garantir la cohérence** : conventions de nommage et structure uniforme
+5. **Optimiser les performances** : parallélisation intégrée et traitement par batch
+
+Cette architecture modulaire permet de créer rapidement de nouvelles commandes tout en maintenant une qualité et une cohérence élevées dans toute la suite OBITools.

blackboard/architechture/definition-superkmer.md

@@ -0,0 +1,99 @@
+# Définition du super k-mer
+
+## Définition
+
+Un **super k-mer** est une **sous-séquence MAXIMALE** d'une séquence dans laquelle **tous les k-mers consécutifs partagent le même minimiseur**.
+
+### Termes
+
+- **k-mer** : sous-séquence de longueur k
+- **minimiseur** : le plus petit m-mer canonique parmi tous les m-mers d'un k-mer
+- **k-mers consécutifs** : k-mers aux positions i et i+1 (chevauchement de k-1 nucléotides)
+- **MAXIMALE** : ne peut être étendue ni à gauche ni à droite
+
+## RÈGLES ABSOLUES
+
+### RÈGLE 1 : Longueur minimum = k
+
+Un super k-mer contient au minimum k nucléotides.
+
+```
+longueur(super-kmer) >= k
+```
+
+### RÈGLE 2 : Chevauchement obligatoire = k-1
+
+Deux super-kmers consécutifs se chevauchent d'EXACTEMENT k-1 nucléotides.
+
+```
+SK1.End - SK2.Start = k - 1
+```
+
+### RÈGLE 3 : Bijection séquence ↔ minimiseur
+
+Une séquence de super k-mer a UN et UN SEUL minimiseur.
+
+```
+Même séquence → Même minimiseur (TOUJOURS)
+```
+
+**Si vous observez la même séquence avec deux minimiseurs différents, c'est un BUG.**
+
+### RÈGLE 4 : Tous les k-mers partagent le minimiseur
+
+TOUS les k-mers contenus dans un super k-mer ont le même minimiseur.
+
+```
+∀ k-mer K dans SK : minimiseur(K) = SK.minimizer
+```
+
+### RÈGLE 5 : Maximalité
+
+Un super k-mer ne peut pas être étendu.
+
+- Si on ajoute un nucléotide à gauche : le nouveau k-mer a un minimiseur différent
+- Si on ajoute un nucléotide à droite : le nouveau k-mer a un minimiseur différent
+
+## VIOLATIONS INTERDITES
+
+❌ **Super k-mer de longueur < k**
+❌ **Chevauchement ≠ k-1 entre consécutifs**
+❌ **Même séquence avec minimiseurs différents**
+❌ **K-mer dans le super k-mer avec minimiseur différent**
+❌ **Super k-mer extensible (non-maximal)**
+
+## CONSÉQUENCES PRATIQUES
+
+### Pour l'extraction
+
+L'algorithme doit :
+1. Calculer le minimiseur de chaque k-mer
+2. Découper quand le minimiseur change
+3. Assigner au super k-mer le minimiseur commun à tous ses k-mers
+4. Garantir que chaque super k-mer contient au moins k nucléotides
+5. Garantir le chevauchement de k-1 entre consécutifs
+
+### Pour la validation
+
+Si après déduplication (obiuniq) on observe :
+```
+Séquence: ACGT...
+Minimiseurs: {M1, M2}  // plusieurs minimiseurs
+```
+
+C'est la PREUVE d'un bug : l'algorithme a produit cette séquence avec des minimiseurs différents, ce qui viole la RÈGLE 3.
+
+## DIAGNOSTIC DU BUG
+
+**Bug observé** : Même séquence avec minimiseurs différents après obiuniq
+
+**Cause possible** : L'algorithme assigne le mauvais minimiseur OU découpe mal les super-kmers
+
+**Ce que le bug NE PEUT PAS être** :
+- Un problème d'obiuniq (révèle le bug, ne le crée pas)
+- Un problème de chevauchement légitime (k-1 est correct)
+
+**Ce que le bug DOIT être** :
+- Minimiseur mal calculé ou mal assigné
+- Découpage incorrect (mauvais endPos)
+- Copie incorrecte des données

blackboard/architechture/guide-redaction-obitest.md

@@ -0,0 +1,316 @@
+# Guide de rédaction d'un obitest
+
+## Règles essentielles
+
+1. **Données < 1 KB** - Fichiers de test très petits
+2. **Exécution < 10 sec** - Tests rapides pour CI/CD
+3. **Auto-contenu** - Pas de dépendances externes
+4. **Auto-nettoyage** - Pas de fichiers résiduels
+
+## Structure minimale
+
+```
+obitests/obitools/<commande>/
+├── test.sh          # Script exécutable
+└── data.fasta       # Données minimales (optionnel)
+```
+
+## Template de test.sh
+
+```bash
+#!/bin/bash
+
+TEST_NAME=<commande>
+CMD=<commande>
+
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+########## TESTS ##########
+
+# Test 1: Help (OBLIGATOIRE)
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1 
+then
+    log "$MCMD: printing help OK" 
+    ((success++))
+else
+    log "$MCMD: printing help failed" 
+    ((failed++))
+fi
+
+# Ajoutez vos tests ici...
+
+###########################
+
+cleanup
+```
+
+## Pattern de test
+
+```bash
+((ntest++))
+if commande args > "${TMPDIR}/output.txt" 2>&1
+then
+    log "$MCMD: description OK" 
+    ((success++))
+else
+    log "$MCMD: description failed"
+    ((failed++))
+fi
+```
+
+## Tests courants
+
+### Exécution basique
+```bash
+((ntest++))
+if $CMD "${TEST_DIR}/input.fasta" > "${TMPDIR}/output.fasta" 2>&1
+then
+    log "$MCMD: basic execution OK" 
+    ((success++))
+else
+    log "$MCMD: basic execution failed"
+    ((failed++))
+fi
+```
+
+### Sortie non vide
+```bash
+((ntest++))
+if [ -s "${TMPDIR}/output.fasta" ]
+then
+    log "$MCMD: output not empty OK"
+    ((success++))
+else
+    log "$MCMD: output empty - failed"
+    ((failed++))
+fi
+```
+
+### Comptage
+```bash
+((ntest++))
+count=$(grep -c "^>" "${TMPDIR}/output.fasta")
+if [ "$count" -gt 0 ]
+then
+    log "$MCMD: extracted $count sequences OK"
+    ((success++))
+else
+    log "$MCMD: no sequences - failed"
+    ((failed++))
+fi
+```
+
+### Présence de contenu
+```bash
+((ntest++))
+if grep -q "expected_string" "${TMPDIR}/output.fasta"
+then
+    log "$MCMD: expected content found OK"
+    ((success++))
+else
+    log "$MCMD: content not found - failed"
+    ((failed++))
+fi
+```
+
+### Comparaison avec référence
+```bash
+((ntest++))
+if diff "${TEST_DIR}/expected.fasta" "${TMPDIR}/output.fasta" > /dev/null
+then
+    log "$MCMD: matches reference OK"
+    ((success++))
+else
+    log "$MCMD: differs from reference - failed"
+    ((failed++))
+fi
+```
+
+### Test avec options
+```bash
+((ntest++))
+if $CMD --opt value "${TEST_DIR}/input.fasta" > "${TMPDIR}/out.fasta" 2>&1
+then
+    log "$MCMD: with option OK" 
+    ((success++))
+else
+    log "$MCMD: with option failed"
+    ((failed++))
+fi
+```
+
+## Variables importantes
+
+- **TEST_DIR** - Répertoire du test (données d'entrée)
+- **TMPDIR** - Répertoire temporaire (sorties)
+- **CMD** - Nom de la commande
+- **MCMD** - Nom formaté pour les logs
+
+## Règles d'or
+
+✅ **Entrées** → `${TEST_DIR}/`
+✅ **Sorties** → `${TMPDIR}/`
+✅ **Toujours rediriger** → `> file 2>&1`
+✅ **Incrémenter ntest** → Avant chaque test
+✅ **Messages clairs** → Descriptions explicites
+
+❌ **Pas de chemins en dur**
+❌ **Pas de /tmp direct**
+❌ **Pas de sortie vers TEST_DIR**
+❌ **Pas de commandes sans redirection**
+
+## Données de test
+
+Créer un fichier minimal (< 500 bytes) :
+
+```fasta
+>seq1
+ACGTACGTACGTACGT
+>seq2
+AAAACCCCGGGGTTTT
+>seq3
+ATCGATCGATCGATCG
+```
+
+## Création rapide
+
+```bash
+# 1. Créer le répertoire
+mkdir -p obitests/obitools/<commande>
+cd obitests/obitools/<commande>
+
+# 2. Créer les données de test
+cat > test_data.fasta << 'EOF'
+>seq1
+ACGTACGTACGTACGT
+>seq2
+AAAACCCCGGGGTTTT
+EOF
+
+# 3. Copier le template dans test.sh
+# 4. Adapter le TEST_NAME et CMD
+# 5. Ajouter les tests
+# 6. Rendre exécutable
+chmod +x test.sh
+
+# 7. Tester
+./test.sh
+```
+
+## Checklist
+
+- [ ] `test.sh` exécutable (`chmod +x`)
+- [ ] Test d'aide inclus
+- [ ] Données < 1 KB
+- [ ] Sorties vers `${TMPDIR}/`
+- [ ] Entrées depuis `${TEST_DIR}/`
+- [ ] Redirections `2>&1`
+- [ ] Messages clairs
+- [ ] Testé localement
+- [ ] Exit code 0 si succès
+
+## Debug
+
+Conserver TMPDIR pour inspection :
+```bash
+cleanup() {
+    echo "Temporary directory: $TMPDIR" 1>&2
+    # rm -rf "$TMPDIR"  # Commenté
+    ...
+}
+```
+
+Mode verbose :
+```bash
+set -x  # Au début du script
+```
+
+## Exemples
+
+**Simple (1 test)** - obimicrosat
+```bash
+# Juste l'aide
+```
+
+**Moyen (4-5 tests)** - obisuperkmer
+```bash
+# Aide + exécution + validation sortie + contenu
+```
+
+**Complet (7+ tests)** - obiuniq
+```bash
+# Aide + exécution + comparaison CSV + options + multiples cas
+```
+
+## Commandes utiles
+
+```bash
+# Compter séquences
+grep -c "^>" file.fasta
+
+# Fichier non vide
+[ -s file ]
+
+# Comparer
+diff file1 file2 > /dev/null
+
+# Comparer compressés
+zdiff file1.gz file2.gz
+
+# Compter bases
+grep -v "^>" file | tr -d '\n' | wc -c
+```
+
+## Ce qu'il faut retenir
+
+Un bon test est **COURT**, **RAPIDE** et **SIMPLE** :
+- 3-10 tests maximum
+- Données < 1 KB
+- Exécution < 10 secondes
+- Pattern standard respecté

blackboard/architechture/obisuperkmer-implementation.md

@@ -0,0 +1,268 @@
+# Implémentation de la commande obisuperkmer
+
+## Vue d'ensemble
+
+La commande `obisuperkmer` a été implémentée en suivant l'architecture standard des commandes OBITools décrite dans `architecture-commande-obitools.md`. Cette commande permet d'extraire les super k-mers de fichiers de séquences biologiques.
+
+## Qu'est-ce qu'un super k-mer ?
+
+Un super k-mer est une sous-séquence maximale dans laquelle tous les k-mers consécutifs partagent le même minimiseur. Cette décomposition est utile pour :
+- L'indexation efficace de k-mers
+- La réduction de la redondance dans les analyses
+- L'optimisation de la mémoire pour les structures de données de k-mers
+
+## Structure de l'implémentation
+
+### 1. Package `pkg/obitools/obisuperkmer/`
+
+Le package contient trois fichiers :
+
+#### `obisuperkmer.go`
+Documentation du package avec une description de son rôle.
+
+#### `options.go`
+Définit les options de ligne de commande :
+
+```go
+var _KmerSize = 21          // Taille des k-mers (par défaut 21)
+var _MinimizerSize = 11     // Taille des minimiseurs (par défaut 11)
+```
+
+**Options CLI disponibles :**
+- `--kmer-size` / `-k` : Taille des k-mers (entre m+1 et 31)
+- `--minimizer-size` / `-m` : Taille des minimiseurs (entre 1 et k-1)
+
+**Fonctions d'accès :**
+- `CLIKmerSize()` : retourne la taille des k-mers
+- `CLIMinimizerSize()` : retourne la taille des minimiseurs
+- `SetKmerSize(k int)` : définit la taille des k-mers
+- `SetMinimizerSize(m int)` : définit la taille des minimiseurs
+
+#### `superkmer.go`
+Implémente la logique de traitement :
+
+```go
+func CLIExtractSuperKmers(iterator obiiter.IBioSequence) obiiter.IBioSequence
+```
+
+Cette fonction :
+1. Récupère les paramètres k et m depuis les options CLI
+2. Valide les paramètres (m < k, k <= 31, etc.)
+3. Crée un worker utilisant `obikmer.SuperKmerWorker(k, m)`
+4. Applique le worker en parallèle sur l'itérateur de séquences
+5. Retourne un itérateur de super k-mers
+
+### 2. Exécutable `cmd/obitools/obisuperkmer/main.go`
+
+L'exécutable suit le pattern standard minimal :
+
+```go
+func main() {
+    // 1. Génération du parser d'options
+    optionParser := obioptions.GenerateOptionParser(
+        "obisuperkmer",
+        "extract super k-mers from sequence files",
+        obisuperkmer.OptionSet)
+    
+    // 2. Parsing des arguments
+    _, args := optionParser(os.Args)
+    
+    // 3. Lecture des séquences
+    sequences, err := obiconvert.CLIReadBioSequences(args...)
+    obiconvert.OpenSequenceDataErrorMessage(args, err)
+    
+    // 4. Extraction des super k-mers
+    superkmers := obisuperkmer.CLIExtractSuperKmers(sequences)
+    
+    // 5. Écriture des résultats
+    obiconvert.CLIWriteBioSequences(superkmers, true)
+    
+    // 6. Attente de la fin du pipeline
+    obiutils.WaitForLastPipe()
+}
+```
+
+## Utilisation du package `obikmer`
+
+L'implémentation s'appuie sur le package `obikmer` qui fournit :
+
+### `SuperKmerWorker(k int, m int) obiseq.SeqWorker`
+
+Crée un worker qui :
+- Extrait les super k-mers d'une BioSequence
+- Retourne une slice de BioSequence, une par super k-mer
+- Chaque super k-mer contient les attributs suivants :
+
+```go
+// Métadonnées ajoutées à chaque super k-mer :
+{
+    "minimizer_value": uint64,  // Valeur canonique du minimiseur
+    "minimizer_seq": string,    // Séquence ADN du minimiseur
+    "k": int,                   // Taille des k-mers utilisée
+    "m": int,                   // Taille des minimiseurs utilisée
+    "start": int,               // Position de début (0-indexé)
+    "end": int,                 // Position de fin (exclusif)
+    "parent_id": string,        // ID de la séquence parente
+}
+```
+
+### Algorithme sous-jacent
+
+Le package `obikmer` utilise :
+- `IterSuperKmers(seq []byte, k int, m int)` : itérateur sur les super k-mers
+- Une deque monotone pour suivre les minimiseurs dans une fenêtre glissante
+- Complexité temporelle : O(n) où n est la longueur de la séquence
+- Complexité spatiale : O(k-m+1) pour la deque
+
+## Exemple d'utilisation
+
+### Ligne de commande
+
+```bash
+# Extraction avec paramètres par défaut (k=21, m=11)
+obisuperkmer sequences.fasta > superkmers.fasta
+
+# Spécifier les tailles de k-mers et minimiseurs
+obisuperkmer -k 25 -m 13 sequences.fasta -o superkmers.fasta
+
+# Avec plusieurs fichiers d'entrée
+obisuperkmer --kmer-size 31 --minimizer-size 15 file1.fasta file2.fasta > output.fasta
+
+# Format FASTQ en entrée, FASTA en sortie
+obisuperkmer sequences.fastq --fasta-output -o superkmers.fasta
+
+# Avec compression
+obisuperkmer sequences.fasta -o superkmers.fasta.gz --compress
+```
+
+### Exemple de sortie
+
+Pour une séquence d'entrée :
+```
+>seq1
+ACGTACGTACGTACGTACGTACGT
+```
+
+La sortie contiendra plusieurs super k-mers :
+```
+>seq1_superkmer_0_15 {"minimizer_value":123456,"minimizer_seq":"acgtacgt","k":21,"m":11,"start":0,"end":15,"parent_id":"seq1"}
+ACGTACGTACGTACG
+>seq1_superkmer_8_24 {"minimizer_value":789012,"minimizer_seq":"gtacgtac","k":21,"m":11,"start":8,"end":24,"parent_id":"seq1"}
+TACGTACGTACGTACGT
+```
+
+## Options héritées de `obiconvert`
+
+La commande hérite de toutes les options standard d'OBITools :
+
+### Options d'entrée
+- `--fasta` : forcer le format FASTA
+- `--fastq` : forcer le format FASTQ
+- `--ecopcr` : format ecoPCR
+- `--embl` : format EMBL
+- `--genbank` : format GenBank
+- `--input-json-header` : en-têtes JSON
+- `--input-OBI-header` : en-têtes OBI
+
+### Options de sortie
+- `--out` / `-o` : fichier de sortie (défaut : stdout)
+- `--fasta-output` : sortie en format FASTA
+- `--fastq-output` : sortie en format FASTQ
+- `--json-output` : sortie en format JSON
+- `--output-json-header` : en-têtes JSON en sortie
+- `--output-OBI-header` / `-O` : en-têtes OBI en sortie
+- `--compress` / `-Z` : compression gzip
+- `--skip-empty` : ignorer les séquences vides
+- `--no-progressbar` : désactiver la barre de progression
+
+## Compilation
+
+Pour compiler la commande :
+
+```bash
+cd /chemin/vers/obitools4
+go build -o bin/obisuperkmer ./cmd/obitools/obisuperkmer/
+```
+
+## Tests
+
+Pour tester la commande :
+
+```bash
+# Créer un fichier de test
+echo -e ">test\nACGTACGTACGTACGTACGTACGTACGTACGT" > test.fasta
+
+# Exécuter obisuperkmer
+obisuperkmer test.fasta
+
+# Vérifier avec des paramètres différents
+obisuperkmer -k 15 -m 7 test.fasta
+```
+
+## Validation des paramètres
+
+La commande valide automatiquement :
+- `1 <= m < k` : le minimiseur doit être plus petit que le k-mer
+- `2 <= k <= 31` : contrainte du codage sur 64 bits
+- `len(sequence) >= k` : la séquence doit être assez longue
+
+En cas de paramètres invalides, la commande affiche une erreur explicite et s'arrête.
+
+## Intégration avec le pipeline OBITools
+
+La commande s'intègre naturellement dans les pipelines OBITools :
+
+```bash
+# Pipeline complet d'analyse
+obiconvert sequences.fastq --fasta-output | \
+  obisuperkmer -k 21 -m 11 | \
+  obiuniq | \
+  obigrep -p "minimizer_value>1000" > filtered_superkmers.fasta
+```
+
+## Parallélisation
+
+La commande utilise automatiquement :
+- `obidefault.ParallelWorkers()` pour le traitement parallèle
+- Les workers sont distribués sur les séquences d'entrée
+- La parallélisation est transparente pour l'utilisateur
+
+## Conformité avec l'architecture OBITools
+
+L'implémentation respecte tous les principes de l'architecture :
+
+✅ Séparation des responsabilités (package + commande)
+✅ Convention de nommage cohérente (CLI*, Set*, _variables)
+✅ Réutilisation de `obiconvert` pour l'I/O
+✅ Options standard partagées
+✅ Pattern Worker pour le traitement
+✅ Validation des paramètres
+✅ Logging avec `logrus`
+✅ Gestion d'erreurs cohérente
+✅ Documentation complète
+
+## Fichiers créés
+
+```
+pkg/obitools/obisuperkmer/
+├── obisuperkmer.go      # Documentation du package
+├── options.go           # Définition des options CLI
+└── superkmer.go         # Implémentation du traitement
+
+cmd/obitools/obisuperkmer/
+└── main.go              # Point d'entrée de la commande
+```
+
+## Prochaines étapes
+
+1. **Compilation** : Compiler la commande avec `go build`
+2. **Tests unitaires** : Créer des tests dans `pkg/obitools/obisuperkmer/superkmer_test.go`
+3. **Documentation utilisateur** : Ajouter la documentation de la commande
+4. **Intégration CI/CD** : Ajouter aux tests d'intégration
+5. **Benchmarks** : Mesurer les performances sur différents jeux de données
+
+## Références
+
+- Architecture des commandes OBITools : `architecture-commande-obitools.md`
+- Package `obikmer` : `pkg/obikmer/`
+- Tests du package : `pkg/obikmer/superkmer_iter_test.go`

blackboard/architechture/obisuperkmer-tests.md

@@ -0,0 +1,440 @@
+# Tests automatisés pour obisuperkmer
+
+## Vue d'ensemble
+
+Des tests automatisés ont été créés pour la commande `obisuperkmer` dans le répertoire `obitests/obitools/obisuperkmer/`. Ces tests suivent le pattern standard utilisé par toutes les commandes OBITools et sont conçus pour être exécutés dans un environnement CI/CD.
+
+## Fichiers créés
+
+```
+obitests/obitools/obisuperkmer/
+├── test.sh                    # Script de test principal (6.7 KB)
+├── test_sequences.fasta       # Données de test (117 bytes)
+└── README.md                  # Documentation (4.1 KB)
+```
+
+### Taille totale : ~11 KB
+
+Cette taille minimale est idéale pour un dépôt Git et des tests CI/CD rapides.
+
+## Jeu de données de test
+
+### Fichier : `test_sequences.fasta` (117 bytes)
+
+Le fichier contient 3 séquences de 32 nucléotides chacune :
+
+```fasta
+>seq1
+ACGTACGTACGTACGTACGTACGTACGTACGT
+>seq2
+AAAACCCCGGGGTTTTAAAACCCCGGGGTTTT
+>seq3
+ATCGATCGATCGATCGATCGATCGATCGATCG
+```
+
+#### Justification du choix
+
+1. **seq1** : Motif répétitif simple (ACGT)
+   - Teste l'extraction de super k-mers sur une séquence avec faible complexité
+   - Les minimiseurs devraient être assez réguliers
+
+2. **seq2** : Blocs homopolymères
+   - Teste le comportement avec des régions de très faible complexité
+   - Les minimiseurs varieront entre les blocs A, C, G et T
+
+3. **seq3** : Motif différent (ATCG)
+   - Teste la diversité des super k-mers extraits
+   - Différent de seq1 pour vérifier la distinction
+
+#### Caractéristiques
+
+- **Longueur** : 32 nucléotides par séquence
+- **Taille totale** : 96 nucléotides (3 × 32)
+- **Format** : FASTA avec en-têtes JSON compatibles
+- **Alphabet** : A, C, G, T uniquement (pas de bases ambiguës)
+- **Taille du fichier** : 117 bytes
+
+Avec k=21 (défaut), chaque séquence de 32 bp peut produire :
+- 32 - 21 + 1 = 12 k-mers
+- Plusieurs super k-mers selon les minimiseurs
+
+## Script de test : `test.sh`
+
+### Structure
+
+Le script suit le pattern standard OBITools :
+
+```bash
+#!/bin/bash
+
+TEST_NAME=obisuperkmer
+CMD=obisuperkmer
+
+# Variables et fonctions standard
+TEST_DIR="..."
+OBITOOLS_DIR="..."
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() { ... }
+log() { ... }
+
+# Tests (12 au total)
+# ...
+
+cleanup
+```
+
+### Tests implémentés
+
+#### 1. Test d'aide (`-h`)
+```bash
+obisuperkmer -h
+```
+Vérifie que la commande peut afficher son aide sans erreur.
+
+#### 2. Extraction basique avec paramètres par défaut
+```bash
+obisuperkmer test_sequences.fasta > output_default.fasta
+```
+Teste l'exécution avec k=21, m=11 (défaut).
+
+#### 3. Vérification de sortie non vide
+```bash
+[ -s output_default.fasta ]
+```
+S'assure que la commande produit un résultat.
+
+#### 4. Comptage des super k-mers
+```bash
+grep -c "^>" output_default.fasta
+```
+Vérifie qu'au moins un super k-mer a été extrait.
+
+#### 5. Présence des métadonnées
+```bash
+grep -q "minimizer_value" output_default.fasta
+grep -q "minimizer_seq" output_default.fasta
+grep -q "parent_id" output_default.fasta
+```
+Vérifie que les attributs requis sont présents.
+
+#### 6. Extraction avec paramètres personnalisés
+```bash
+obisuperkmer -k 15 -m 7 test_sequences.fasta > output_k15_m7.fasta
+```
+Teste la configuration de k et m.
+
+#### 7. Validation des paramètres personnalisés
+```bash
+grep -q '"k":15' output_k15_m7.fasta
+grep -q '"m":7' output_k15_m7.fasta
+```
+Vérifie que les paramètres sont correctement enregistrés.
+
+#### 8. Format de sortie FASTA
+```bash
+obisuperkmer --fasta-output test_sequences.fasta > output_fasta.fasta
+```
+Teste l'option de format explicite.
+
+#### 9. Vérification des IDs
+```bash
+grep "^>" output_default.fasta | grep -q "superkmer"
+```
+S'assure que les IDs contiennent "superkmer".
+
+#### 10. Préservation des IDs parents
+```bash
+grep -q "seq1" output_default.fasta
+grep -q "seq2" output_default.fasta
+grep -q "seq3" output_default.fasta
+```
+Vérifie que les IDs des séquences parentes sont préservés.
+
+#### 11. Option de fichier de sortie (`-o`)
+```bash
+obisuperkmer -o output_file.fasta test_sequences.fasta
+```
+Teste la redirection vers un fichier.
+
+#### 12. Vérification de création du fichier
+```bash
+[ -s output_file.fasta ]
+```
+S'assure que le fichier a été créé.
+
+#### 13. Cohérence des longueurs
+```bash
+# Vérifie que longueur(output) <= longueur(input)
+```
+S'assure que les super k-mers ne sont pas plus longs que l'entrée.
+
+### Compteurs
+
+- **ntest** : Nombre de tests exécutés
+- **success** : Nombre de tests réussis
+- **failed** : Nombre de tests échoués
+
+### Sortie du script
+
+#### En cas de succès
+```
+========================================
+## Results of the obisuperkmer tests:
+
+- 12 tests run
+- 12 successfully completed
+- 0 failed tests
+
+Cleaning up the temporary directory...
+
+========================================
+```
+
+Exit code : **0**
+
+#### En cas d'échec
+```
+========================================
+## Results of the obisuperkmer tests:
+
+- 12 tests run
+- 10 successfully completed
+- 2 failed tests
+
+Cleaning up the temporary directory...
+
+========================================
+```
+
+Exit code : **1**
+
+## Intégration CI/CD
+
+### Exécution automatique
+
+Le script est conçu pour être exécuté automatiquement dans un pipeline CI/CD :
+
+1. Le build produit l'exécutable dans `build/obisuperkmer`
+2. Le script de test ajoute `build/` au PATH
+3. Les tests s'exécutent
+4. Le code de retour indique le succès (0) ou l'échec (1)
+
+### Exemple de configuration CI/CD
+
+```yaml
+# .github/workflows/test.yml ou équivalent
+test-obisuperkmer:
+  runs-on: ubuntu-latest
+  steps:
+    - uses: actions/checkout@v2
+    - name: Build obitools
+      run: make build
+    - name: Test obisuperkmer
+      run: ./obitests/obitools/obisuperkmer/test.sh
+```
+
+### Avantages
+
+✅ **Rapidité** : Données de test minimales (117 bytes)
+✅ **Fiabilité** : Tests reproductibles
+✅ **Isolation** : Utilisation d'un répertoire temporaire
+✅ **Nettoyage automatique** : Pas de fichiers résiduels
+✅ **Logging** : Messages horodatés et détaillés
+✅ **Compatibilité** : Pattern standard OBITools
+
+## Exécution locale
+
+### Prérequis
+
+1. Compiler obisuperkmer :
+   ```bash
+   cd /chemin/vers/obitools4
+   go build -o build/obisuperkmer ./cmd/obitools/obisuperkmer/
+   ```
+
+2. Se placer dans le répertoire de test :
+   ```bash
+   cd obitests/obitools/obisuperkmer
+   ```
+
+3. Exécuter le script :
+   ```bash
+   ./test.sh
+   ```
+
+### Exemple de sortie
+
+```
+[obisuperkmer @ Fri Feb  7 13:00:00 CET 2026] Testing obisuperkmer...
+[obisuperkmer @ Fri Feb  7 13:00:00 CET 2026] Test directory is /path/to/obitests/obitools/obisuperkmer
+[obisuperkmer @ Fri Feb  7 13:00:00 CET 2026] obitools directory is /path/to/build
+[obisuperkmer @ Fri Feb  7 13:00:00 CET 2026] Temporary directory is /tmp/tmp.abc123
+[obisuperkmer @ Fri Feb  7 13:00:00 CET 2026] files: README.md test.sh test_sequences.fasta
+[obisuperkmer @ Fri Feb  7 13:00:01 CET 2026] OBISuperkmer: printing help OK
+[obisuperkmer @ Fri Feb  7 13:00:02 CET 2026] OBISuperkmer: basic extraction with default parameters OK
+[obisuperkmer @ Fri Feb  7 13:00:02 CET 2026] OBISuperkmer: output file is not empty OK
+[obisuperkmer @ Fri Feb  7 13:00:02 CET 2026] OBISuperkmer: extracted 8 super k-mers OK
+[obisuperkmer @ Fri Feb  7 13:00:02 CET 2026] OBISuperkmer: super k-mers contain required metadata OK
+[obisuperkmer @ Fri Feb  7 13:00:03 CET 2026] OBISuperkmer: extraction with custom k=15, m=7 OK
+[obisuperkmer @ Fri Feb  7 13:00:03 CET 2026] OBISuperkmer: custom parameters correctly set in metadata OK
+[obisuperkmer @ Fri Feb  7 13:00:03 CET 2026] OBISuperkmer: FASTA output format OK
+[obisuperkmer @ Fri Feb  7 13:00:03 CET 2026] OBISuperkmer: super k-mer IDs contain 'superkmer' OK
+[obisuperkmer @ Fri Feb  7 13:00:03 CET 2026] OBISuperkmer: parent sequence IDs preserved OK
+[obisuperkmer @ Fri Feb  7 13:00:04 CET 2026] OBISuperkmer: output to file with -o option OK
+[obisuperkmer @ Fri Feb  7 13:00:04 CET 2026] OBISuperkmer: output file created with -o option OK
+[obisuperkmer @ Fri Feb  7 13:00:04 CET 2026] OBISuperkmer: super k-mer total length <= input length OK
+========================================
+## Results of the obisuperkmer tests:
+
+- 12 tests run
+- 12 successfully completed
+- 0 failed tests
+
+Cleaning up the temporary directory...
+
+========================================
+```
+
+## Debugging des tests
+
+### Conserver les fichiers temporaires
+
+Modifier temporairement la fonction `cleanup()` :
+
+```bash
+cleanup() {
+    echo "Temporary directory: $TMPDIR" 1>&2
+    # Commenter cette ligne pour conserver les fichiers
+    # rm -rf "$TMPDIR"
+    ...
+}
+```
+
+### Activer le mode verbose
+
+Ajouter au début du script :
+
+```bash
+set -x  # Active l'affichage de toutes les commandes
+```
+
+### Tester une seule commande
+
+Extraire et exécuter manuellement :
+
+```bash
+export TEST_DIR=/chemin/vers/obitests/obitools/obisuperkmer
+export TMPDIR=$(mktemp -d)
+obisuperkmer "${TEST_DIR}/test_sequences.fasta" > "${TMPDIR}/output.fasta"
+cat "${TMPDIR}/output.fasta"
+```
+
+## Ajout de nouveaux tests
+
+Pour ajouter un test supplémentaire :
+
+1. Incrémenter le compteur `ntest`
+2. Écrire la condition de test
+3. Logger le succès ou l'échec
+4. Incrémenter le bon compteur
+
+```bash
+((ntest++))
+if ma_nouvelle_commande_de_test
+then
+    log "Description du test: OK" 
+    ((success++))
+else
+    log "Description du test: failed"
+    ((failed++))
+fi
+```
+
+## Comparaison avec d'autres tests
+
+### Taille des données de test
+
+| Commande | Taille des données | Nombre de fichiers |
+|----------|-------------------|-------------------|
+| obiconvert | 925 KB | 1 fichier |
+| obiuniq | ~600 bytes | 4 fichiers |
+| obimicrosat | 0 bytes | 0 fichiers (génère à la volée) |
+| **obisuperkmer** | **117 bytes** | **1 fichier** |
+
+Notre test `obisuperkmer` est parmi les plus légers, ce qui est optimal pour CI/CD.
+
+### Nombre de tests
+
+| Commande | Nombre de tests |
+|----------|----------------|
+| obiconvert | 3 tests |
+| obiuniq | 7 tests |
+| obimicrosat | 1 test |
+| **obisuperkmer** | **12 tests** |
+
+Notre test `obisuperkmer` offre une couverture complète avec 12 tests différents.
+
+## Couverture de test
+
+Les tests couvrent :
+
+✅ Affichage de l'aide  
+✅ Exécution basique  
+✅ Paramètres par défaut (k=21, m=11)  
+✅ Paramètres personnalisés (k=15, m=7)  
+✅ Formats de sortie (FASTA)  
+✅ Redirection vers fichier (`-o`)  
+✅ Présence des métadonnées  
+✅ Validation des IDs  
+✅ Préservation des IDs parents  
+✅ Cohérence des longueurs  
+✅ Production de résultats non vides  
+
+## Maintenance
+
+### Mise à jour des tests
+
+Si l'implémentation de `obisuperkmer` change :
+
+1. Vérifier que les tests existants passent toujours
+2. Ajouter de nouveaux tests pour les nouvelles fonctionnalités
+3. Mettre à jour `README.md` si nécessaire
+4. Documenter les changements
+
+### Vérification régulière
+
+Exécuter périodiquement :
+
+```bash
+cd obitests/obitools/obisuperkmer
+./test.sh
+```
+
+Ou via l'ensemble des tests :
+
+```bash
+cd obitests
+for dir in obitools/*/; do
+    if [ -f "$dir/test.sh" ]; then
+        echo "Testing $(basename $dir)..."
+        (cd "$dir" && ./test.sh) || echo "FAILED: $(basename $dir)"
+    fi
+done
+```
+
+## Conclusion
+
+Les tests pour `obisuperkmer` sont :
+
+- ✅ **Complets** : 12 tests couvrant toutes les fonctionnalités principales
+- ✅ **Légers** : 117 bytes de données de test
+- ✅ **Rapides** : Exécution en quelques secondes
+- ✅ **Fiables** : Pattern éprouvé utilisé par toutes les commandes OBITools
+- ✅ **Maintenables** : Structure claire et documentée
+- ✅ **CI/CD ready** : Code de retour approprié et nettoyage automatique
+
+Ils garantissent que la commande fonctionne correctement à chaque commit et facilitent la détection précoce des régressions.

cmd/obitools/obialign/main.go

@@ -0,0 +1,26 @@
+package main
+
+import (
+	"os"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obialign"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+)
+
+func main() {
+	optionParser := obioptions.GenerateOptionParser(
+		"obialign",
+		"pairwise similarity of nucleotide sequences, as a CSV table (the principle of sumatra)",
+		obialign.OptionSet)
+
+	_, args := optionParser(os.Args)
+
+	fs, err := obiconvert.CLIReadBioSequences(args...)
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
+
+	obialign.CLIWriteTable(obialign.CLIOBIAlign(fs))
+
+	obiutils.WaitForLastPipe()
+}

cmd/obitools/obiclust/main.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"os"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiclust"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+)
+
+func main() {
+	optionParser := obioptions.GenerateOptionParser(
+		"obiclust",
+		"star clustering of sequences (the algorithm of sumaclust, UCLUST and CD-HIT)",
+		obiclust.OptionSet)
+
+	_, args := optionParser(os.Args)
+
+	fs, err := obiconvert.CLIReadBioSequences(args...)
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
+
+	clustered := obiclust.CLIOBIClust(fs)
+
+	obiconvert.CLIWriteBioSequences(clustered, true)
+
+	obiutils.WaitForLastPipe()
+}

cmd/obitools/obik/main.go

@@ -0,0 +1,34 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"os"
+
+	log "github.com/sirupsen/logrus"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obik"
+	"github.com/DavidGamba/go-getoptions"
+)
+
+func main() {
+	defer obiseq.LogBioSeqStatus()
+
+	opt, parser := obioptions.GenerateSubcommandParser(
+		"obik",
+		"Manage disk-based kmer indices",
+		obik.OptionSet,
+	)
+
+	_, remaining := parser(os.Args)
+
+	err := opt.Dispatch(context.Background(), remaining)
+	if err != nil {
+		if errors.Is(err, getoptions.ErrorHelpCalled) {
+			os.Exit(0)
+		}
+		log.Fatalf("Error: %v", err)
+	}
+}

cmd/obitools/obilowmask/main.go

@@ -1,47 +0,0 @@
-package main
-
-import (
-	"os"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obilowmask"
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
-)
-
-func main() {
-
-	defer obiseq.LogBioSeqStatus()
-
-	// go tool pprof -http=":8000" ./obipairing ./cpu.pprof
-	// f, err := os.Create("cpu.pprof")
-	// if err != nil {
-	// 	log.Fatal(err)
-	// }
-	// pprof.StartCPUProfile(f)
-	// defer pprof.StopCPUProfile()
-
-	// go tool trace cpu.trace
-	// ftrace, err := os.Create("cpu.trace")
-	// if err != nil {
-	// 	log.Fatal(err)
-	// }
-	// trace.Start(ftrace)
-	// defer trace.Stop()
-
-	optionParser := obioptions.GenerateOptionParser(
-		"obimicrosat",
-		"looks for microsatellites sequences in a sequence file",
-		obilowmask.OptionSet)
-
-	_, args := optionParser(os.Args)
-
-	sequences, err := obiconvert.CLIReadBioSequences(args...)
-	obiconvert.OpenSequenceDataErrorMessage(args, err)
-
-	selected := obilowmask.CLISequenceEntropyMasker(sequences)
-	obiconvert.CLIWriteBioSequences(selected, true)
-	obiutils.WaitForLastPipe()
-
-}

cmd/obitools/obioligotag/main.go

@@ -0,0 +1,33 @@
+package main
+
+import (
+	"os"
+
+	log "github.com/sirupsen/logrus"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obioligotag"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+)
+
+func main() {
+	optionParser := obioptions.GenerateOptionParser(
+		"obioligotag",
+		"designs a family of oligonucleotides (tags) at a minimum distance from each other",
+		obioligotag.OptionSet)
+
+	_, args := optionParser(os.Args)
+	if len(args) > 0 {
+		log.Fatalf("unexpected arguments %v: use --oligo-list to provide candidate oligonucleotides", args)
+	}
+
+	oligos, err := obioligotag.CLIOligoFamily()
+	if err != nil {
+		log.Fatalf("%v", err)
+	}
+
+	obiconvert.CLIWriteBioSequences(obioligotag.CLIOligosToIterator(oligos), true)
+
+	obiutils.WaitForLastPipe()
+}

cmd/obitools/obipairing/main.go

@@ -37,6 +37,11 @@ func main() {
 
 	optionParser(os.Args)
 
+	if !obipairing.CLIHasPairedFiles() {
+		log.Error("You must provide both a forward file (-F) and a reverse file (-R)")
+		os.Exit(1)
+	}
+
 	obidefault.SetStrictReadWorker(2)
 	obidefault.SetStrictWriteWorker(2)
 	pairs, err := obipairing.CLIPairedSequence()

cmd/obitools/obiprimers/main.go

@@ -0,0 +1,38 @@
+package main
+
+import (
+	"os"
+
+	log "github.com/sirupsen/logrus"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obicsv"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiprimers"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+)
+
+func main() {
+	obiprimers.SetReadingDefaults()
+
+	optionParser := obioptions.GenerateOptionParser(
+		"obiprimers",
+		"designs the primer pairs amplifying a set of example sequences and evaluates the DNA barcode they delimit. "+
+			"The taxonomy comes from the -t option, or is read from the taxonomic paths embedded in the sequences (obiannotate --taxonomic-path)",
+		obiprimers.OptionSet)
+
+	_, args := optionParser(os.Args)
+
+	sequences, taxonomy, err := obiprimers.CLIReadSequencesAndTaxonomy(args)
+	if err != nil {
+		log.Fatalf("Cannot read the sequences: %v", err)
+	}
+
+	pairs, err := obiprimers.CLIPrimerPairs(sequences, taxonomy)
+	if err != nil {
+		log.Fatalf("%v", err)
+	}
+
+	obicsv.CLICSVWriter(obiprimers.PairsToCSVIterator(pairs, obiprimers.CLIReference() != ""), true)
+
+	obiutils.WaitForLastPipe()
+}

cmd/obitools/obitagpcr/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"fmt"
 	"os"
 
 	log "github.com/sirupsen/logrus"
@@ -8,6 +9,7 @@ import (
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obimultiplex"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obipairing"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obitagpcr"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
@@ -39,6 +41,17 @@ func main() {
 		obitagpcr.OptionSet)
 
 	optionParser(os.Args)
+
+	if obimultiplex.CLIAskConfigTemplate() {
+		fmt.Print(obimultiplex.CLIConfigTemplate())
+		os.Exit(0)
+	}
+
+	if !obipairing.CLIHasPairedFiles() {
+		log.Error("You must provide both a forward file (-F) and a reverse file (-R)")
+		os.Exit(1)
+	}
+
 	pairs, err := obipairing.CLIPairedSequence()
 
 	if err != nil {

doc/book/installation.qmd

@@ -8,7 +8,7 @@ All the sources of the [*OBITools4*](http://metabarcoding.org/obitools4) can be
 
 ## Prerequisites
 
-The *OBITools4* are developped using the [GO programming language](https://go.dev/), we stick to the latest version of the language, today the $1.21.4$. If you want to download and compile the sources yourself, you first need to install the corresponding compiler on your system. Some parts of the soft are also written in C, therefore a recent C compiler is also requested, GCC on Linux or Windows, the Developer Tools on Mac.
+The *OBITools4* are developped using the [GO programming language](https://go.dev/), we stick to the latest version of the language, today the $1.27$. If you want to download and compile the sources yourself, you first need to install the corresponding compiler on your system. Some computations (distances between k-mers) use the experimental `simd` package of Go: the `Makefile` compiles them with `GOEXPERIMENT=simd`, and the installation script uses a Go 1.27 compiler for that reason. The resulting programs run on any processor, a portable code being used when no SIMD instruction is available. Some parts of the soft are also written in C, therefore a recent C compiler is also requested, GCC on Linux or Windows, the Developer Tools on Mac.
 
 Whatever the installation you decide for, you will have to ensure that a C compiler is available on your system.
 

entities.json

@@ -0,0 +1,10 @@
+{
+  "people": [
+    "Software",
+    "Agreement",
+    "Module"
+  ],
+  "projects": [
+    "Code"
+  ]
+}

go.mod

@@ -1,56 +1,50 @@
 module git.metabarcoding.org/obitools/obitools4/obitools4
 
-go 1.23.4
+go 1.27
-
-toolchain go1.24.2
 
 require (
-	github.com/DavidGamba/go-getoptions v0.28.0
+	github.com/DavidGamba/go-getoptions v0.33.0
-	github.com/PaesslerAG/gval v1.2.2
+	github.com/PaesslerAG/gval v1.2.4
 	github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df
-	github.com/buger/jsonparser v1.1.1
+	github.com/buger/jsonparser v1.1.2
 	github.com/chen3feng/stl4go v0.1.1
-	github.com/dlclark/regexp2 v1.11.4
+	github.com/dlclark/regexp2 v1.11.5
-	github.com/goccy/go-json v0.10.3
+	github.com/goccy/go-json v0.10.6
 	github.com/klauspost/pgzip v1.2.6
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
+	github.com/pelletier/go-toml/v2 v2.2.4
 	github.com/rrethy/ahocorasick v1.0.0
-	github.com/schollz/progressbar/v3 v3.13.1
+	github.com/schollz/progressbar/v3 v3.19.0
-	github.com/sirupsen/logrus v1.9.3
+	github.com/sirupsen/logrus v1.9.4
-	github.com/stretchr/testify v1.8.4
+	github.com/stretchr/testify v1.10.0
 	github.com/tevino/abool/v2 v2.1.0
 	github.com/yuin/gopher-lua v1.1.1
-	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa
+	golang.org/x/exp v0.0.0-20260312153236-7ab1446f8b90
-	gonum.org/v1/gonum v0.14.0
+	gonum.org/v1/gonum v0.17.0
 	gopkg.in/yaml.v3 v3.0.1
 	scientificgo.org/special v0.0.0
 )
 
 require (
-	github.com/RoaringBitmap/roaring v1.9.4 // indirect
-	github.com/bits-and-blooms/bitset v1.12.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/goombaio/orderedmap v0.0.0-20180924084748-ba921b7e2419 // indirect
+	github.com/goombaio/orderedmap v0.0.0-20180925151256-3da0e2f905f9 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
-	github.com/mschoch/smat v0.2.0 // indirect
-	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rogpeppe/go-internal v1.12.0 // indirect
 )
 
 require (
 	github.com/dsnet/compress v0.0.1
-	github.com/gabriel-vasile/mimetype v1.4.3
+	github.com/gabriel-vasile/mimetype v1.4.13
 	github.com/goombaio/orderedset v0.0.0-20180925151225-8e67b20a9b77
-	github.com/klauspost/compress v1.17.2
+	github.com/klauspost/compress v1.18.4
-	github.com/mattn/go-runewidth v0.0.15 // indirect
+	github.com/mattn/go-runewidth v0.0.21 // indirect
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
-	github.com/rivo/uniseg v0.4.4 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
-	github.com/shopspring/decimal v1.3.1 // indirect
+	github.com/shopspring/decimal v1.4.0 // indirect
-	github.com/ulikunitz/xz v0.5.11
+	github.com/ulikunitz/xz v0.5.15
-	golang.org/x/net v0.35.0 // indirect
+	golang.org/x/sys v0.42.0 // indirect
-	golang.org/x/sys v0.30.0 // indirect
+	golang.org/x/term v0.41.0 // indirect
-	golang.org/x/term v0.29.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c
 )

go.sum

@@ -1,40 +1,41 @@
-github.com/DavidGamba/go-getoptions v0.28.0 h1:18wgEvfZdrlfIhVDGEBO3Dl0fkOyXqXLa0tLMCKxM1c=
+github.com/DavidGamba/go-getoptions v0.33.0 h1:8xCPH87Yy5avYenygyHVlqqm8RpymH0YFe4a7IWlarE=
-github.com/DavidGamba/go-getoptions v0.28.0/go.mod h1:zE97E3PR9P3BI/HKyNYgdMlYxodcuiC6W68KIgeYT84=
+github.com/DavidGamba/go-getoptions v0.33.0/go.mod h1:zE97E3PR9P3BI/HKyNYgdMlYxodcuiC6W68KIgeYT84=
-github.com/PaesslerAG/gval v1.2.2 h1:Y7iBzhgE09IGTt5QgGQ2IdaYYYOU134YGHBThD+wm9E=
+github.com/PaesslerAG/gval v1.2.4 h1:rhX7MpjJlcxYwL2eTTYIOBUyEKZ+A96T9vQySWkVUiU=
-github.com/PaesslerAG/gval v1.2.2/go.mod h1:XRFLwvmkTEdYziLdaCeCa5ImcGVrfQbeNUbVR+C6xac=
+github.com/PaesslerAG/gval v1.2.4/go.mod h1:XRFLwvmkTEdYziLdaCeCa5ImcGVrfQbeNUbVR+C6xac=
 github.com/PaesslerAG/jsonpath v0.1.0 h1:gADYeifvlqK3R3i2cR5B4DGgxLXIPb3TRTH1mGi0jPI=
 github.com/PaesslerAG/jsonpath v0.1.0/go.mod h1:4BzmtoM/PI8fPO4aQGIusjGxGir2BzcV0grWtFzq1Y8=
-github.com/RoaringBitmap/roaring v1.9.4 h1:yhEIoH4YezLYT04s1nHehNO64EKFTop/wBhxv2QzDdQ=
-github.com/RoaringBitmap/roaring v1.9.4/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90=
 github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df h1:GSoSVRLoBaFpOOds6QyY1L8AX7uoY+Ln3BHc22W40X0=
 github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df/go.mod h1:hiVxq5OP2bUGBRNS3Z/bt/reCLFNbdcST6gISi1fiOM=
-github.com/bits-and-blooms/bitset v1.12.0 h1:U/q1fAF7xXRhFCrhROzIfffYnu+dlS38vCZtmFVPHmA=
+github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk=
-github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
+github.com/buger/jsonparser v1.1.2/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
-github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
-github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
 github.com/chen3feng/stl4go v0.1.1 h1:0L1+mDw7pomftKDruM23f1mA7miavOj6C6MZeadzN2Q=
 github.com/chen3feng/stl4go v0.1.1/go.mod h1:5ml3psLgETJjRJnMbPE+JiHLrCpt+Ajc2weeTECXzWU=
+github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM=
+github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY=
+github.com/clipperhouse/uax29/v2 v2.2.0 h1:ChwIKnQN3kcZteTXMgb1wztSgaU+ZemkgWdohwgs8tY=
+github.com/clipperhouse/uax29/v2 v2.2.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo=
+github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ=
-github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
+github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
 github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q=
 github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo=
 github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
-github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
+github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM=
-github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
+github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
-github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
+github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU=
-github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
+github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
-github.com/goombaio/orderedmap v0.0.0-20180924084748-ba921b7e2419 h1:SajEQ6tktpF9SRIuzbiPOX9AEZZ53Bvw0k9Mzrts8Lg=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/goombaio/orderedmap v0.0.0-20180924084748-ba921b7e2419/go.mod h1:YKu81H3RSd1cFh0d7NhvUoTtUC9IY/vBX0WUQb1/o4Y=
+github.com/goombaio/orderedmap v0.0.0-20180925151256-3da0e2f905f9 h1:vFjPvFavIiDY71bQ9HIxPQBANvNl1SmFC4fgg5xRkho=
+github.com/goombaio/orderedmap v0.0.0-20180925151256-3da0e2f905f9/go.mod h1:YKu81H3RSd1cFh0d7NhvUoTtUC9IY/vBX0WUQb1/o4Y=
 github.com/goombaio/orderedset v0.0.0-20180925151225-8e67b20a9b77 h1:4dvq1tGHn1Y9KSRY0OZ24Khki4+4U+ZrA//YYsdUlJU=
 github.com/goombaio/orderedset v0.0.0-20180925151225-8e67b20a9b77/go.mod h1:HPelMYpOyy0XvglpBbmZ3krZpwaHmszj/vQNlnETPTM=
-github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
 github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
-github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
+github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
-github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
 github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU=
 github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
@@ -45,14 +46,10 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-runewidth v0.0.21 h1:jJKAZiQH+2mIinzCJIaIG9Be1+0NR+5sz/lYEEjdM8w=
-github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/mattn/go-runewidth v0.0.21/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
-github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
-github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
-github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM=
-github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
 github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
@@ -60,50 +57,40 @@ github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
-github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
-github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
 github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
 github.com/rrethy/ahocorasick v1.0.0 h1:YKkCB+E5PXc0xmLfMrWbfNht8vG9Re97IHSWZk/Lk8E=
 github.com/rrethy/ahocorasick v1.0.0/go.mod h1:nq8oScE7Vy1rOppoQxpQiiDmPHuKCuk9rXrNcxUV3R0=
-github.com/schollz/progressbar/v3 v3.13.1 h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE=
+github.com/schollz/progressbar/v3 v3.19.0 h1:Ea18xuIRQXLAUidVDox3AbwfUhD0/1IvohyTutOIFoc=
-github.com/schollz/progressbar/v3 v3.13.1/go.mod h1:xvrbki8kfT1fzWzBT/UZd9L6GA+jdL7HAgq2RFnO6fQ=
+github.com/schollz/progressbar/v3 v3.19.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
-github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
 github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
-github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
-github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
-github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/tevino/abool/v2 v2.1.0 h1:7w+Vf9f/5gmKT4m4qkayb33/92M+Um45F2BkHOR+L/c=
 github.com/tevino/abool/v2 v2.1.0/go.mod h1:+Lmlqk6bHDWHqN1cbxqhwEAwMPXgc8I1SDEamtseuXY=
 github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
-github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
+github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY=
-github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
+github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
 github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
-golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ=
+golang.org/x/exp v0.0.0-20260312153236-7ab1446f8b90 h1:jiDhWWeC7jfWqR9c/uplMOqJ0sbNlNWv0UkzE0vX1MA=
-golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE=
+golang.org/x/exp v0.0.0-20260312153236-7ab1446f8b90/go.mod h1:xE1HEv6b+1SCZ5/uscMRjUBKtIxworgEcEi+/n9NQDQ=
-golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8=
+golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
-golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk=
+golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
-golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU=
-golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A=
-golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
-golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
+gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
-golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
-golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU=
-golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s=
-gonum.org/v1/gonum v0.14.0 h1:2NiG67LD1tEH0D7kM+ps2V+fXmsAnpUeec7n8tcr4S0=
-gonum.org/v1/gonum v0.14.0/go.mod h1:AoWeoz0becf9QMWtE8iWXNXc27fK4fNeHNf/oMejGfU=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 scientificgo.org/special v0.0.0 h1:P6WJkECo6tgtvZAEfNXl+KEB9ReAatjKAeX8U07mjSc=

go.work

@@ -1,5 +1,3 @@
-go 1.23.4
+go 1.27
-
-toolchain go1.24.2
 
 use .

go.work.sum

@@ -52,6 +52,8 @@ golang.org/x/image v0.6.0/go.mod h1:MXLdDR43H7cDJq5GEGXEVeeNhPgi+YYEQ2pC1byI1x0=
 golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY=
 golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
+golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
 golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=

install_obitools.sh

@@ -1,27 +1,63 @@
 #!/bin/bash
 
-INSTALL_DIR="/usr/local"
+# Default values
-OBITOOLS_PREFIX=""
-# default values
 URL="https://go.dev/dl/"
-OBIURL4="https://github.com/metabarcoding/obitools4/archive/refs/heads/master.zip"
+# Series of the Go compiler used to build OBITools4. The SIMD code relies on the
+# experimental simd package of Go (GOEXPERIMENT=simd), whose API may change
+# between Go versions: the build is therefore tied to the series the code was
+# written for rather than to the latest release.
+GO_SERIES="1.27"
+GITHUB_REPO="https://github.com/metabarcoding/obitools4"
 INSTALL_DIR="/usr/local"
 OBITOOLS_PREFIX=""
+VERSION=""
+LIST_VERSIONS=false
+JOBS=1
 
-# help message
+# Help message
 function display_help {
   echo "Usage: $0 [OPTIONS]"
   echo ""
   echo "Options:"
   echo "  -i, --install-dir       Directory where obitools are installed "
-  echo "                          (as example use /usr/local not /usr/local/bin)."
+  echo "                          (e.g., use /usr/local not /usr/local/bin)."
   echo "  -p, --obitools-prefix   Prefix added to the obitools command names if you"
   echo "                          want to have several versions of obitools at the"
-  echo "                          same time on your system (as example -p g will produce "
+  echo "                          same time on your system (e.g., -p g will produce "
   echo "                          gobigrep command instead of obigrep)."
+  echo "  -v, --version           Install a specific version (e.g., 4.4.8)."
+  echo "                          If not specified, installs the latest version."
+  echo "  -j, --jobs              Number of parallel jobs for compilation (default: 1)."
+  echo "  -l, --list              List all available versions and exit."
   echo "  -h, --help              Display this help message."
+  echo ""
+  echo "Examples:"
+  echo "  $0                      # Install latest version"
+  echo "  $0 -l                   # List available versions"
+  echo "  $0 -v 4.4.8             # Install specific version"
+  echo "  $0 -i /opt/local        # Install to custom directory"
 }
 
+# List available versions from GitHub releases
+function list_versions {
+  echo "Fetching available versions..." 1>&2
+  echo ""
+  curl -s "https://api.github.com/repos/metabarcoding/obitools4/releases" \
+    | grep '"tag_name":' \
+    | sed -E 's/.*"tag_name": "Release_([0-9.]+)".*/\1/' \
+    | sort -V -r
+}
+
+# Get latest version from GitHub releases
+function get_latest_version {
+  curl -s "https://api.github.com/repos/metabarcoding/obitools4/releases" \
+    | grep '"tag_name":' \
+    | sed -E 's/.*"tag_name": "Release_([0-9.]+)".*/\1/' \
+    | sort -V -r \
+    | head -1
+}
+
+# Parse command line arguments
 while [ "$#" -gt 0 ]; do
   case "$1" in
     -i|--install-dir)
@@ -32,30 +68,63 @@ while [ "$#" -gt 0 ]; do
       OBITOOLS_PREFIX="$2"
       shift 2
       ;;
+    -v|--version)
+      VERSION="$2"
+      shift 2
+      ;;
+    -j|--jobs)
+      JOBS="$2"
+      shift 2
+      ;;
+    -l|--list)
+      LIST_VERSIONS=true
+      shift
+      ;;
     -h|--help)
-      display_help  1>&2 
+      display_help
       exit 0
       ;;
     *)
-      echo "Error: Unsupported option $1"  1>&2
+      echo "Error: Unsupported option $1" 1>&2
+      display_help 1>&2
       exit 1
       ;;
   esac
 done
 
-# the directory from where the script is run
+# List versions and exit if requested
+if [ "$LIST_VERSIONS" = true ]; then
+  echo "Available OBITools4 versions:"
+  echo "=============================="
+  list_versions
+  exit 0
+fi
+
+# Determine version to install
+if [ -z "$VERSION" ]; then
+  echo "Fetching latest version..." 1>&2
+  VERSION=$(get_latest_version)
+  if [ -z "$VERSION" ]; then
+    echo "Error: Could not determine latest version" 1>&2
+    exit 1
+  fi
+  echo "Latest version: $VERSION" 1>&2
+else
+  echo "Installing version: $VERSION" 1>&2
+fi
+
+# Construct source URL for the specified version
+OBIURL4="${GITHUB_REPO}/archive/refs/tags/Release_${VERSION}.zip"
+
+# The directory from where the script is run
 DIR="$(pwd)"
 
-# the temp directory used, within $DIR
+# Create temporary directory
-# omit the -p parameter to create a temporal directory in the default location
-# WORK_DIR=$(mktemp -d -p "$DIR"  "obitools4.XXXXXX" 2> /dev/null || \
-#            mktemp -d -t "$DIR"  "obitools4.XXXXXX")
-
 WORK_DIR=$(mktemp -d "obitools4.XXXXXX")
 
-# check if tmp dir was created
+# Check if tmp dir was created
 if [[ ! "$WORK_DIR" || ! -d "$WORK_DIR" ]]; then
-  echo "Could not create temp dir"  1>&2
+  echo "Could not create temp dir" 1>&2
   exit 1
 fi
 
@@ -63,24 +132,36 @@ mkdir -p "${WORK_DIR}/cache" \
   || (echo "Cannot create ${WORK_DIR}/cache directory" 1>&2
       exit 1)
 
-
+# Create installation directory
-mkdir -p "${INSTALL_DIR}/bin" 2> /dev/null \
+if ! mkdir -p "${INSTALL_DIR}/bin" 2>/dev/null; then
-  || (echo "Please enter your password for installing obitools in ${INSTALL_DIR}"  1>&2
+  if [ ! -w "$(dirname "${INSTALL_DIR}")" ] && [ ! -w "${INSTALL_DIR}" ]; then
-      sudo mkdir -p "${INSTALL_DIR}/bin")
+    echo "Please enter your password for installing obitools in ${INSTALL_DIR}" 1>&2
+    sudo mkdir -p "${INSTALL_DIR}/bin"
+  else
+    echo "Error: Could not create ${INSTALL_DIR}/bin (check path or disk space)" 1>&2
+    exit 1
+  fi
+fi
 
 if [[ ! -d "${INSTALL_DIR}/bin" ]]; then
-  echo "Could not create ${INSTALL_DIR}/bin directory for installing obitools"  1>&2
+  echo "Could not create ${INSTALL_DIR}/bin directory for installing obitools" 1>&2
   exit 1
 fi
 
 INSTALL_DIR="$(cd ${INSTALL_DIR} && pwd)"
 
-echo "WORK_DIR=$WORK_DIR"  1>&2
+echo "================================" 1>&2
-echo "INSTALL_DIR=$INSTALL_DIR"  1>&2
+echo "OBITools4 Installation" 1>&2
-echo "OBITOOLS_PREFIX=$OBITOOLS_PREFIX"  1>&2
+echo "================================" 1>&2
+echo "VERSION=$VERSION" 1>&2
+echo "WORK_DIR=$WORK_DIR" 1>&2
+echo "INSTALL_DIR=$INSTALL_DIR" 1>&2
+echo "OBITOOLS_PREFIX=$OBITOOLS_PREFIX" 1>&2
+echo "================================" 1>&2
 
-pushd "$WORK_DIR"|| exit
+pushd "$WORK_DIR" > /dev/null || exit
 
+# Detect OS and architecture
 OS=$(uname -a | awk '{print $1}')
 ARCH=$(uname -m)
 
@@ -92,52 +173,106 @@ if [[ "$ARCH" == "aarch64" ]] ; then
     ARCH="arm64"
 fi
 
-GOFILE=$(curl "$URL" \
+# Download and install Go
+echo "Downloading Go..." 1>&2
+GOFILE=$(curl -s "$URL" \
             | grep 'class="download"' \
             | grep "\.tar\.gz" \
             | sed -E 's@^.*/dl/(go[1-9].+\.tar\.gz)".*$@\1@' \
+            | awk -v prefix="go${GO_SERIES}." 'index($0, prefix) == 1' \
             | grep -i "$OS" \
             | grep -i "$ARCH" \
             | head -1)
 
-GOURL=$(curl "${URL}${GOFILE}" \
+if [ -z "$GOFILE" ]; then
-        | sed -E 's@^.*href="(.*\.tar\.gz)".*$@\1@')
+  echo "Error: no Go ${GO_SERIES} archive found for ${OS}/${ARCH} at ${URL}" 1>&2
-
+  exit 1
-echo "Install GO from : $GOURL" 1>&2
-        
-curl "$GOURL" \
-    | tar zxf -
-
-PATH="$(pwd)/go/bin:$PATH"
-export PATH
-GOPATH="$(pwd)/go"
-export GOPATH
-
-export GOCACHE="$(pwd)/cache"
-echo "GOCACHE=$GOCACHE" 1>&2@
-mkdir -p "$GOCACHE"
-
-
-curl -L "$OBIURL4" > master.zip
-unzip master.zip
-
-echo "Install OBITOOLS from : $OBIURL4"
-
-cd obitools4-master || exit
-mkdir vendor
-
-if [[ -z "$OBITOOLS_PREFIX" ]] ; then
-  make GOFLAGS="-buildvcs=false" 
-else
-  make GOFLAGS="-buildvcs=false" OBITOOLS_PREFIX="${OBITOOLS_PREFIX}"
 fi
 
-(cp build/* "${INSTALL_DIR}/bin" 2> /dev/null) \
+GOURL=$(curl -s "${URL}${GOFILE}" \
-   || (echo "Please enter your password for installing obitools in ${INSTALL_DIR}" 
+        | sed -E 's@^.*href="(.*\.tar\.gz)".*$@\1@')
-       sudo cp build/* "${INSTALL_DIR}/bin")
 
-popd || exit
+echo "Installing Go from: $GOURL" 1>&2
 
+curl --progress-bar "$GOURL" | tar zxf -
+
+export GOROOT="$(pwd)/go"
+PATH="${GOROOT}/bin:$PATH"
+export PATH
+export GOPATH="$(pwd)/gopath"
+export GOCACHE="$(pwd)/cache"
+export GOTOOLCHAIN=local
+# OBITools4 is built with the SIMD code enabled (it falls back on a portable
+# code at run time if the processor has no SIMD instructions).
+export GOEXPERIMENT=simd
+
+echo "GOROOT=$GOROOT" 1>&2
+echo "GOEXPERIMENT=$GOEXPERIMENT" 1>&2
+echo "GOCACHE=$GOCACHE" 1>&2
+mkdir -p "$GOPATH" "$GOCACHE"
+
+# Download OBITools4 source
+echo "Downloading OBITools4 v${VERSION}..." 1>&2
+echo "Source URL: $OBIURL4" 1>&2
+
+if ! curl --progress-bar -L "$OBIURL4" > obitools4.zip; then
+  echo "Error: Could not download OBITools4 version ${VERSION}" 1>&2
+  echo "Please check that this version exists with: $0 --list" 1>&2
+  exit 1
+fi
+
+unzip -q obitools4.zip
+
+# Find the extracted directory
+OBITOOLS_DIR=$(ls -d obitools4-* 2>/dev/null | head -1)
+
+if [ -z "$OBITOOLS_DIR" ] || [ ! -d "$OBITOOLS_DIR" ]; then
+  echo "Error: Could not find extracted OBITools4 directory" 1>&2
+  exit 1
+fi
+
+echo "Building OBITools4..." 1>&2
+cd "$OBITOOLS_DIR" || exit
+mkdir -p vendor
+
+# Build with or without prefix
+if [[ -z "$OBITOOLS_PREFIX" ]] ; then
+  make -j"${JOBS}" obitools GOFLAGS="-buildvcs=false"
+else
+  make -j"${JOBS}" obitools GOFLAGS="-buildvcs=false" OBITOOLS_PREFIX="${OBITOOLS_PREFIX}"
+fi
+
+# Install binaries
+echo "Installing binaries to ${INSTALL_DIR}/bin..." 1>&2
+if ! cp build/* "${INSTALL_DIR}/bin" 2>/dev/null; then
+  if [ ! -w "${INSTALL_DIR}/bin" ]; then
+    echo "Please enter your password for installing obitools in ${INSTALL_DIR}" 1>&2
+    sudo cp build/* "${INSTALL_DIR}/bin"
+  else
+    echo "Error: Could not copy binaries to ${INSTALL_DIR}/bin" 1>&2
+    echo "  Source files: $(ls build/ 2>/dev/null || echo 'none found')" 1>&2
+    echo "" 1>&2
+    echo "The build directory has been preserved for manual recovery:" 1>&2
+    echo "  $(pwd)/build/" 1>&2
+    echo "You can install manually with:" 1>&2
+    echo "  cp $(pwd)/build/* ${INSTALL_DIR}/bin/" 1>&2
+    popd > /dev/null || true
+    exit 1
+  fi
+fi
+
+popd > /dev/null || exit
+
+# Cleanup
+echo "Cleaning up..." 1>&2
 chmod -R +w "$WORK_DIR"
 rm -rf "$WORK_DIR"
 
+echo "" 1>&2
+echo "================================" 1>&2
+echo "OBITools4 v${VERSION} installed successfully!" 1>&2
+echo "Binaries location: ${INSTALL_DIR}/bin" 1>&2
+if [[ -n "$OBITOOLS_PREFIX" ]] ; then
+  echo "Command prefix: ${OBITOOLS_PREFIX}" 1>&2
+fi
+echo "================================" 1>&2

kmer_roaring_index/FREQUENCY_FILTER_FINAL.md

@@ -1,292 +0,0 @@
-# Filtre de Fréquence avec v Niveaux de Roaring Bitmaps
-
-## Algorithme
-
-```go
-Pour chaque k-mer rencontré dans les données:
-    c = 0
-    tant que (k-mer ∈ index[c] ET c < v):
-        c++
-    
-    si c < v:
-        index[c].insert(k-mer)
-```
-
-**Résultat** : `index[v-1]` contient les k-mers vus **≥ v fois**
-
----
-
-## Exemple d'exécution (v=3)
-
-```
-Données:
-  Read1: kmer X
-  Read2: kmer X
-  Read3: kmer X  (X vu 3 fois)
-  Read4: kmer Y
-  Read5: kmer Y  (Y vu 2 fois)
-  Read6: kmer Z  (Z vu 1 fois)
-
-Exécution:
-
-Read1 (X):
-  c=0: X ∉ index[0] → index[0].add(X)
-  État: index[0]={X}, index[1]={}, index[2]={}
-
-Read2 (X):
-  c=0: X ∈ index[0] → c=1
-  c=1: X ∉ index[1] → index[1].add(X)
-  État: index[0]={X}, index[1]={X}, index[2]={}
-
-Read3 (X):
-  c=0: X ∈ index[0] → c=1
-  c=1: X ∈ index[1] → c=2
-  c=2: X ∉ index[2] → index[2].add(X)
-  État: index[0]={X}, index[1]={X}, index[2]={X}
-
-Read4 (Y):
-  c=0: Y ∉ index[0] → index[0].add(Y)
-  État: index[0]={X,Y}, index[1]={X}, index[2]={X}
-
-Read5 (Y):
-  c=0: Y ∈ index[0] → c=1
-  c=1: Y ∉ index[1] → index[1].add(Y)
-  État: index[0]={X,Y}, index[1]={X,Y}, index[2]={X}
-
-Read6 (Z):
-  c=0: Z ∉ index[0] → index[0].add(Z)
-  État: index[0]={X,Y,Z}, index[1]={X,Y}, index[2]={X}
-
-Résultat final:
-  index[0] (freq≥1): {X, Y, Z}
-  index[1] (freq≥2): {X, Y}
-  index[2] (freq≥3): {X}  ← K-mers filtrés ✓
-```
-
----
-
-## Utilisation
-
-```go
-// Créer le filtre
-filter := obikmer.NewFrequencyFilter(31, 3) // k=31, minFreq=3
-
-// Ajouter les séquences
-for _, read := range reads {
-    filter.AddSequence(read)
-}
-
-// Récupérer les k-mers filtrés (freq ≥ 3)
-filtered := filter.GetFilteredSet("filtered")
-fmt.Printf("K-mers de qualité: %d\n", filtered.Cardinality())
-
-// Statistiques
-stats := filter.Stats()
-fmt.Println(stats.String())
-```
-
----
-
-## Performance
-
-### Complexité
-
-**Par k-mer** :
-- Lookups : Moyenne ~v/2, pire cas v
-- Insertions : 1 Add
-- **Pas de Remove** ✅
-
-**Total pour n k-mers** :
-- Temps : O(n × v/2)
-- Mémoire : O(unique_kmers × v × 2 bytes)
-
-### Early exit pour distribution skewed
-
-Avec distribution typique (séquençage) :
-```
-80% singletons → 1 lookup (early exit)
-15% freq 2-3   → 2-3 lookups
-5% freq ≥4     → jusqu'à v lookups
-
-Moyenne réelle : ~2 lookups/kmer (au lieu de v/2)
-```
-
----
-
-## Mémoire
-
-### Pour 10^8 k-mers uniques
-
-| v (minFreq) | Nombre bitmaps | Mémoire | vs map simple |
-|-------------|----------------|---------|---------------|
-| v=2 | 2 | ~400 MB | 6x moins |
-| v=3 | 3 | ~600 MB | 4x moins |
-| v=5 | 5 | ~1 GB | 2.4x moins |
-| v=10 | 10 | ~2 GB | 1.2x moins |
-| v=20 | 20 | ~4 GB | ~égal |
-
-**Note** : Avec distribution skewed (beaucoup de singletons), la mémoire réelle est bien plus faible car les niveaux hauts ont peu d'éléments.
-
-### Exemple réaliste (séquençage)
-
-Pour 10^8 k-mers totaux, v=3 :
-```
-Distribution:
-  80% singletons  → 80M dans index[0]
-  15% freq 2-3    → 15M dans index[1]
-  5% freq ≥3      → 5M dans index[2]
-
-Mémoire:
-  index[0]: 80M × 2 bytes = 160 MB
-  index[1]: 15M × 2 bytes = 30 MB
-  index[2]: 5M × 2 bytes = 10 MB
-  Total: ~200 MB ✅
-
-vs map simple: 80M × 24 bytes = ~2 GB
-Réduction: 10x
-```
-
----
-
-## Comparaison des approches
-
-| Approche | Mémoire (10^8 kmers) | Passes | Lookups/kmer | Quand utiliser |
-|----------|----------------------|--------|--------------|----------------|
-| **v-Bitmaps** | **200-600 MB** | **1** | **~2 (avg)** | **Standard** ✅ |
-| Map simple | 2.4 GB | 1 | 1 | Si RAM illimitée |
-| Multi-pass | 400 MB | v | v | Si I/O pas cher |
-
----
-
-## Avantages de v-Bitmaps
-
-✅ **Une seule passe** sur les données  
-✅ **Mémoire optimale** avec Roaring bitmaps  
-✅ **Pas de Remove** (seulement Contains + Add)  
-✅ **Early exit** efficace sur singletons  
-✅ **Scalable** jusqu'à v~10-20  
-✅ **Simple** à implémenter et comprendre  
-
----
-
-## Cas d'usage typiques
-
-### 1. Éliminer erreurs de séquençage
-
-```go
-filter := obikmer.NewFrequencyFilter(31, 3)
-
-// Traiter FASTQ
-for read := range StreamFastq("sample.fastq") {
-    filter.AddSequence(read)
-}
-
-// K-mers de qualité (pas d'erreurs)
-cleaned := filter.GetFilteredSet("cleaned")
-```
-
-**Résultat** : Élimine 70-80% des k-mers (erreurs)
-
-### 2. Assemblage de génome
-
-```go
-filter := obikmer.NewFrequencyFilter(31, 2)
-
-// Filtrer avant l'assemblage
-for read := range reads {
-    filter.AddSequence(read)
-}
-
-solidKmers := filter.GetFilteredSet("solid")
-// Utiliser solidKmers pour le graphe de Bruijn
-```
-
-### 3. Comparaison de génomes
-
-```go
-collection := obikmer.NewKmerSetCollection(31)
-
-for _, genome := range genomes {
-    filter := obikmer.NewFrequencyFilter(31, 3)
-    filter.AddSequences(genome.Reads)
-    
-    cleaned := filter.GetFilteredSet(genome.ID)
-    collection.Add(cleaned)
-}
-
-// Analyses comparatives sur k-mers de qualité
-matrix := collection.ParallelPairwiseJaccard(8)
-```
-
----
-
-## Limites
-
-**Pour v > 20** :
-- Trop de lookups (v lookups/kmer)
-- Mémoire importante (v × 200MB pour 10^8 kmers)
-
-**Solutions alternatives pour v > 20** :
-- Utiliser map simple (9 bytes/kmer) si RAM disponible
-- Algorithme différent (sketch, probabiliste)
-
----
-
-## Optimisations possibles
-
-### 1. Parallélisation
-
-```go
-// Traiter plusieurs fichiers en parallèle
-filters := make([]*FrequencyFilter, numFiles)
-
-var wg sync.WaitGroup
-for i, file := range files {
-    wg.Add(1)
-    go func(idx int, f string) {
-        defer wg.Done()
-        filters[idx] = ProcessFile(f, k, minFreq)
-    }(i, file)
-}
-wg.Wait()
-
-// Merger les résultats
-merged := MergeFilters(filters)
-```
-
-### 2. Streaming avec seuil adaptatif
-
-```go
-// Commencer avec v=5, réduire progressivement
-filter := obikmer.NewFrequencyFilter(31, 5)
-
-// ... traitement ...
-
-// Si trop de mémoire, réduire à v=3
-if filter.MemoryUsage() > threshold {
-    filter = ConvertToLowerThreshold(filter, 3)
-}
-```
-
----
-
-## Récapitulatif final
-
-**Pour filtrer les k-mers par fréquence ≥ v :**
-
-1. **Créer** : `filter := NewFrequencyFilter(k, v)`
-2. **Traiter** : `filter.AddSequence(read)` pour chaque read
-3. **Résultat** : `filtered := filter.GetFilteredSet(id)`
-
-**Mémoire** : ~2v MB par million de k-mers uniques  
-**Temps** : Une seule passe, ~2 lookups/kmer en moyenne  
-**Optimal pour** : v ≤ 20, distribution skewed (séquençage)  
-
----
-
-## Code fourni
-
-1. **frequency_filter.go** - Implémentation complète
-2. **examples_frequency_filter_final.go** - Exemples d'utilisation
-
-**Tout est prêt à utiliser !** 🚀

kmer_roaring_index/examples_frequency_filter_final.go

@@ -1,320 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"obikmer"
-)
-
-func main() {
-	// ==========================================
-	// EXEMPLE 1 : Utilisation basique
-	// ==========================================
-	fmt.Println("=== EXEMPLE 1 : Utilisation basique ===\n")
-
-	k := 31
-	minFreq := 3 // Garder les k-mers vus ≥3 fois
-
-	// Créer le filtre
-	filter := obikmer.NewFrequencyFilter(k, minFreq)
-
-	// Simuler des séquences avec différentes fréquences
-	sequences := [][]byte{
-		[]byte("ACGTACGTACGTACGTACGTACGTACGTACG"), // Kmer X
-		[]byte("ACGTACGTACGTACGTACGTACGTACGTACG"), // Kmer X (freq=2)
-		[]byte("ACGTACGTACGTACGTACGTACGTACGTACG"), // Kmer X (freq=3) ✓
-		[]byte("TTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTT"), // Kmer Y
-		[]byte("TTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTT"), // Kmer Y (freq=2) ✗
-		[]byte("GGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGG"), // Kmer Z (freq=1) ✗
-	}
-
-	fmt.Printf("Traitement de %d séquences...\n", len(sequences))
-	for _, seq := range sequences {
-		filter.AddSequence(seq)
-	}
-
-	// Récupérer les k-mers filtrés
-	filtered := filter.GetFilteredSet("filtered")
-	fmt.Printf("\nK-mers avec freq ≥ %d: %d\n", minFreq, filtered.Cardinality())
-
-	// Statistiques
-	stats := filter.Stats()
-	fmt.Println("\n" + stats.String())
-
-	// ==========================================
-	// EXEMPLE 2 : Vérifier les niveaux
-	// ==========================================
-	fmt.Println("\n=== EXEMPLE 2 : Inspection des niveaux ===\n")
-
-	// Vérifier chaque niveau
-	for level := 0; level < minFreq; level++ {
-		levelSet := filter.GetKmersAtLevel(level)
-		fmt.Printf("Niveau %d (freq≥%d): %d k-mers\n",
-			level+1, level+1, levelSet.Cardinality())
-	}
-
-	// ==========================================
-	// EXEMPLE 3 : Données réalistes
-	// ==========================================
-	fmt.Println("\n=== EXEMPLE 3 : Simulation données séquençage ===\n")
-
-	filter2 := obikmer.NewFrequencyFilter(31, 3)
-
-	// Simuler un dataset réaliste :
-	// - 1000 reads
-	// - 80% contiennent des erreurs (singletons)
-	// - 15% vrais k-mers à basse fréquence
-	// - 5% vrais k-mers à haute fréquence
-
-	// Vraie séquence répétée
-	trueSeq := []byte("ACGTACGTACGTACGTACGTACGTACGTACG")
-	for i := 0; i < 50; i++ {
-		filter2.AddSequence(trueSeq)
-	}
-
-	// Séquence à fréquence moyenne
-	mediumSeq := []byte("CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC")
-	for i := 0; i < 5; i++ {
-		filter2.AddSequence(mediumSeq)
-	}
-
-	// Erreurs de séquençage (singletons)
-	for i := 0; i < 100; i++ {
-		errorSeq := []byte(fmt.Sprintf("TTTTTTTTTTTTTTTTTTTTTTTTTTTT%03d", i))
-		filter2.AddSequence(errorSeq)
-	}
-
-	stats2 := filter2.Stats()
-	fmt.Println(stats2.String())
-
-	fmt.Println("Distribution attendue:")
-	fmt.Println("  - Beaucoup de singletons (erreurs)")
-	fmt.Println("  - Peu de k-mers à haute fréquence (signal)")
-	fmt.Println("  → Filtrage efficace !")
-
-	// ==========================================
-	// EXEMPLE 4 : Tester différents seuils
-	// ==========================================
-	fmt.Println("\n=== EXEMPLE 4 : Comparaison de seuils ===\n")
-
-	testSeqs := [][]byte{
-		[]byte("ACGTACGTACGTACGTACGTACGTACGTACG"),
-		[]byte("ACGTACGTACGTACGTACGTACGTACGTACG"),
-		[]byte("ACGTACGTACGTACGTACGTACGTACGTACG"),
-		[]byte("ACGTACGTACGTACGTACGTACGTACGTACG"),
-		[]byte("ACGTACGTACGTACGTACGTACGTACGTACG"), // freq=5
-		[]byte("TTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTT"),
-		[]byte("TTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTT"),
-		[]byte("TTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTT"), // freq=3
-		[]byte("GGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGG"), // freq=1
-	}
-
-	for _, minFreq := range []int{2, 3, 5} {
-		f := obikmer.NewFrequencyFilter(31, minFreq)
-		f.AddSequences(testSeqs)
-
-		fmt.Printf("minFreq=%d: %d k-mers retenus (%.2f MB)\n",
-			minFreq,
-			f.Cardinality(),
-			float64(f.MemoryUsage())/1024/1024)
-	}
-
-	// ==========================================
-	// EXEMPLE 5 : Comparaison mémoire
-	// ==========================================
-	fmt.Println("\n=== EXEMPLE 5 : Comparaison mémoire ===\n")
-
-	filter3 := obikmer.NewFrequencyFilter(31, 3)
-
-	// Simuler 10000 séquences
-	for i := 0; i < 10000; i++ {
-		seq := make([]byte, 100)
-		for j := range seq {
-			seq[j] = "ACGT"[(i+j)%4]
-		}
-		filter3.AddSequence(seq)
-	}
-
-	fmt.Println(filter3.CompareWithSimpleMap())
-
-	// ==========================================
-	// EXEMPLE 6 : Workflow complet
-	// ==========================================
-	fmt.Println("\n=== EXEMPLE 6 : Workflow complet ===\n")
-
-	fmt.Println("1. Créer le filtre")
-	finalFilter := obikmer.NewFrequencyFilter(31, 3)
-
-	fmt.Println("2. Traiter les données (simulation)")
-	// En pratique : lire depuis FASTQ
-	// for read := range ReadFastq("data.fastq") {
-	//     finalFilter.AddSequence(read)
-	// }
-
-	// Simulation
-	for i := 0; i < 1000; i++ {
-		seq := []byte("ACGTACGTACGTACGTACGTACGTACGTACG")
-		finalFilter.AddSequence(seq)
-	}
-
-	fmt.Println("3. Récupérer les k-mers filtrés")
-	result := finalFilter.GetFilteredSet("final")
-
-	fmt.Println("4. Utiliser le résultat")
-	fmt.Printf("   K-mers de qualité: %d\n", result.Cardinality())
-	fmt.Printf("   Mémoire utilisée: %.2f MB\n", float64(finalFilter.MemoryUsage())/1024/1024)
-
-	fmt.Println("5. Sauvegarder (optionnel)")
-	// result.Save("filtered_kmers.bin")
-
-	// ==========================================
-	// EXEMPLE 7 : Vérification individuelle
-	// ==========================================
-	fmt.Println("\n=== EXEMPLE 7 : Vérification de k-mers spécifiques ===\n")
-
-	checkFilter := obikmer.NewFrequencyFilter(31, 3)
-
-	testSeq := []byte("ACGTACGTACGTACGTACGTACGTACGTACG")
-	for i := 0; i < 5; i++ {
-		checkFilter.AddSequence(testSeq)
-	}
-
-	var kmers []uint64
-	kmers = obikmer.EncodeKmers(testSeq, 31, &kmers)
-
-	if len(kmers) > 0 {
-		testKmer := kmers[0]
-
-		fmt.Printf("K-mer test: 0x%016X\n", testKmer)
-		fmt.Printf("  Présent dans filtre: %v\n", checkFilter.Contains(testKmer))
-		fmt.Printf("  Fréquence approx: %d\n", checkFilter.GetFrequency(testKmer))
-	}
-
-	// ==========================================
-	// EXEMPLE 8 : Intégration avec collection
-	// ==========================================
-	fmt.Println("\n=== EXEMPLE 8 : Intégration avec KmerSetCollection ===\n")
-
-	// Créer une collection de génomes filtrés
-	collection := obikmer.NewKmerSetCollection(31)
-
-	genomes := map[string][][]byte{
-		"Genome1": {
-			[]byte("ACGTACGTACGTACGTACGTACGTACGTACG"),
-			[]byte("ACGTACGTACGTACGTACGTACGTACGTACG"),
-			[]byte("ACGTACGTACGTACGTACGTACGTACGTACG"),
-			[]byte("TTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTT"), // Erreur
-		},
-		"Genome2": {
-			[]byte("ACGTACGTACGTACGTACGTACGTACGTACG"),
-			[]byte("ACGTACGTACGTACGTACGTACGTACGTACG"),
-			[]byte("ACGTACGTACGTACGTACGTACGTACGTACG"),
-			[]byte("GGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGG"), // Erreur
-		},
-	}
-
-	for id, sequences := range genomes {
-		// Filtrer chaque génome
-		genomeFilter := obikmer.NewFrequencyFilter(31, 3)
-		genomeFilter.AddSequences(sequences)
-
-		// Ajouter à la collection
-		filteredSet := genomeFilter.GetFilteredSet(id)
-		collection.Add(filteredSet)
-
-		fmt.Printf("%s: %d k-mers de qualité\n", id, filteredSet.Cardinality())
-	}
-
-	// Analyser la collection
-	fmt.Println("\nAnalyse comparative:")
-	collectionStats := collection.ComputeStats()
-	fmt.Printf("  Core genome: %d k-mers\n", collectionStats.CoreSize)
-	fmt.Printf("  Pan genome: %d k-mers\n", collectionStats.PanGenomeSize)
-
-	// ==========================================
-	// RÉSUMÉ
-	// ==========================================
-	fmt.Println("\n=== RÉSUMÉ ===\n")
-	fmt.Println("Le FrequencyFilter permet de:")
-	fmt.Println("  ✓ Filtrer les k-mers par fréquence minimale")
-	fmt.Println("  ✓ Utiliser une mémoire optimale avec Roaring bitmaps")
-	fmt.Println("  ✓ Une seule passe sur les données")
-	fmt.Println("  ✓ Éliminer efficacement les erreurs de séquençage")
-	fmt.Println("")
-	fmt.Println("Workflow typique:")
-	fmt.Println("  1. filter := NewFrequencyFilter(k, minFreq)")
-	fmt.Println("  2. for each sequence: filter.AddSequence(seq)")
-	fmt.Println("  3. filtered := filter.GetFilteredSet(id)")
-	fmt.Println("  4. Utiliser filtered dans vos analyses")
-}
-
-// ==================================
-// FONCTION HELPER POUR BENCHMARKS
-// ==================================
-
-func BenchmarkFrequencyFilter() {
-	k := 31
-	minFreq := 3
-
-	// Test avec différentes tailles
-	sizes := []int{1000, 10000, 100000}
-
-	fmt.Println("\n=== BENCHMARK ===\n")
-
-	for _, size := range sizes {
-		filter := obikmer.NewFrequencyFilter(k, minFreq)
-
-		// Générer des séquences
-		for i := 0; i < size; i++ {
-			seq := make([]byte, 100)
-			for j := range seq {
-				seq[j] = "ACGT"[(i+j)%4]
-			}
-			filter.AddSequence(seq)
-		}
-
-		fmt.Printf("Size=%d reads:\n", size)
-		fmt.Printf("  Filtered k-mers: %d\n", filter.Cardinality())
-		fmt.Printf("  Memory: %.2f MB\n", float64(filter.MemoryUsage())/1024/1024)
-		fmt.Println()
-	}
-}
-
-// ==================================
-// FONCTION POUR DONNÉES RÉELLES
-// ==================================
-
-func ProcessRealData() {
-	// Exemple pour traiter de vraies données FASTQ
-
-	k := 31
-	minFreq := 3
-
-	filter := obikmer.NewFrequencyFilter(k, minFreq)
-
-	// Pseudo-code pour lire un FASTQ
-	/*
-	fastqFile := "sample.fastq"
-	reader := NewFastqReader(fastqFile)
-
-	for reader.HasNext() {
-		read := reader.Next()
-		filter.AddSequence(read.Sequence)
-	}
-
-	// Récupérer le résultat
-	filtered := filter.GetFilteredSet("sample_filtered")
-	filtered.Save("sample_filtered_kmers.bin")
-
-	// Stats
-	stats := filter.Stats()
-	fmt.Println(stats.String())
-	*/
-
-	fmt.Println("Workflow pour données réelles:")
-	fmt.Println("  1. Créer le filtre avec minFreq approprié (2-5 typique)")
-	fmt.Println("  2. Stream les reads depuis FASTQ")
-	fmt.Println("  3. Récupérer les k-mers filtrés")
-	fmt.Println("  4. Utiliser pour assemblage/comparaison/etc.")
-
-	_ = filter // unused
-}

obitests/obitools/obialign/test.sh

@@ -0,0 +1,206 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obialign
+CMD=obialign
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1
+then
+    log "$MCMD: printing help OK"
+    ((success++))
+else
+    log "$MCMD: printing help failed"
+    ((failed++))
+fi
+
+
+#
+# Dataset used by the following tests: two families of similar sequences, a
+# sequence identical to another one, and a sequence with an ambiguous
+# nucleotide that is discarded.
+#
+cat > "${TMPDIR}/data.fasta" <<DATA
+>a1 count=20;
+acgtacgtacgtaacctagcctagcctagggatcgatcgatcgatcgaatcg
+>a2 count=3;
+acgtacgtacgtaacctagcctagcctagggatcgatcgatcgttcgaatcg
+>a3 count=2;
+acgtacgtacgtaacctagcctagcctagggatcgatcgatcgatcgaatcg
+>b1 count=15;
+ttgcatgcatgctagctagctagcatgcatcgatcgatgctagctagtagct
+>b2 count=1;
+ttgcatgcatgctagctagctagcatgcatcgatcgatgctagctagtaact
+>x1 count=4;
+ttgcatgcatgctangctagctagcatgcatcgatcgatgctagctagtagc
+DATA
+
+cat > "${TMPDIR}/ref.fasta" <<DATA
+>r1
+acgtacgtacgtaacctagcctagcctagggatcgatcgatcgatcgaatcg
+>r2
+ttgcatgcatgctagctagctagcatgcatcgatcgatgctagctagtagct
+>r3
+ggggccccggggccccggggccccggggccccggggccccggggcccc
+DATA
+
+((ntest++))
+if $CMD --max-cpu 2 "${TMPDIR}/data.fasta" > "${TMPDIR}/all.csv" 2> /dev/null &&
+   [ "$(head -n 1 "${TMPDIR}/all.csv")" = "id1,id2,score" ] &&
+   [ "$(tail -n +2 "${TMPDIR}/all.csv" | wc -l | tr -d ' ')" = "10" ]
+then
+    log "$MCMD: without threshold all the pairs of the 5 usable sequences are written OK"
+    ((success++))
+else
+    log "$MCMD: the table without threshold is wrong"
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD --max-cpu 2 -t 0.95 "${TMPDIR}/data.fasta" > "${TMPDIR}/t95.csv" 2> /dev/null &&
+   [ "$(tail -n +2 "${TMPDIR}/t95.csv" | cut -d, -f1,2 | tr '\n' ' ')" = "a1,a2 a1,a3 a2,a3 b1,b2 " ] &&
+   grep -q '^a1,a3,1$' "${TMPDIR}/t95.csv"
+then
+    log "$MCMD: only the pairs reaching the threshold are written, in order OK"
+    ((success++))
+else
+    log "$MCMD: the table with a threshold is wrong"
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD --max-cpu 2 -t 0.95 -x "${TMPDIR}/data.fasta" 2> /dev/null | head -n 2 | tail -n 1 | grep -q '^a1,a2,[0-9.]*,20,3,52,52$'
+then
+    log "$MCMD: extra columns OK"
+    ((success++))
+else
+    log "$MCMD: extra columns failed"
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD --max-cpu 2 -g -t 0.9 "${TMPDIR}/data.fasta" 2> /dev/null | grep -q '^b1,x1,'
+then
+    log "$MCMD: a sequence with an IUPAC letter is kept with --accept-ambiguous OK"
+    ((success++))
+else
+    log "$MCMD: --accept-ambiguous failed"
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD --max-cpu 2 -R "${TMPDIR}/ref.fasta" -t 0.95 "${TMPDIR}/data.fasta" 2> /dev/null | tail -n +2 | cut -d, -f1,2 | tr '\n' ' ' > "${TMPDIR}/two.txt" &&
+   [ "$(cat "${TMPDIR}/two.txt")" = "a1,r1 a2,r1 a3,r1 b1,r2 b2,r2 " ]
+then
+    log "$MCMD: comparison with a reference file OK"
+    ((success++))
+else
+    log "$MCMD: comparison with a reference file failed"
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD --max-cpu 1 "${TMPDIR}/data.fasta" 2> /dev/null > "${TMPDIR}/cpu1.csv" &&
+   $CMD --max-cpu 8 "${TMPDIR}/data.fasta" 2> /dev/null > "${TMPDIR}/cpu8.csv" &&
+   cmp -s "${TMPDIR}/cpu1.csv" "${TMPDIR}/cpu8.csv"
+then
+    log "$MCMD: the table does not depend on the number of threads OK"
+    ((success++))
+else
+    log "$MCMD: the table depends on the number of threads"
+    ((failed++))
+fi
+
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obiclust/test.sh

@@ -0,0 +1,188 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obiclust
+CMD=obiclust
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1
+then
+    log "$MCMD: printing help OK"
+    ((success++))
+else
+    log "$MCMD: printing help failed"
+    ((failed++))
+fi
+
+
+#
+# Dataset used by the following tests: two families with a rare variant each,
+# a duplicate, and a sequence with an ambiguous nucleotide that is discarded.
+#
+cat > "${TMPDIR}/data.fasta" <<DATA
+>a1 count=20;
+acgtacgtacgtaacctagcctagcctagggatcgatcgatcgatcgaatcg
+>a2 count=3;
+acgtacgtacgtaacctagcctagcctagggatcgatcgatcgttcgaatcg
+>a3 count=2;
+acgtacgtacgtaacctagcctagcctagggatcgatcgatcgatcgaatcg
+>b1 count=15;
+ttgcatgcatgctagctagctagcatgcatcgatcgatgctagctagtagct
+>b2 count=1;
+ttgcatgcatgctagctagctagcatgcatcgatcgatgctagctagtaact
+>x1 count=4;
+ttgcatgcatgctangctagctagcatgcatcgatcgatgctagctagtagc
+DATA
+
+((ntest++))
+if $CMD --max-cpu 2 -O -t 0.95 "${TMPDIR}/data.fasta" > "${TMPDIR}/clusters.fasta" 2> "${TMPDIR}/clusters.log"
+then
+    log "$MCMD: clustering runs OK"
+    ((success++))
+else
+    log "$MCMD: clustering failed"
+    ((failed++))
+fi
+
+((ntest++))
+if [ "$(grep -c '^>' "${TMPDIR}/clusters.fasta")" = "5" ]
+then
+    log "$MCMD: the sequence with an ambiguous nucleotide is discarded OK"
+    ((success++))
+else
+    log "$MCMD: wrong number of sequences in the output"
+    ((failed++))
+fi
+
+((ntest++))
+if [ "$(grep '^>a[23] ' "${TMPDIR}/clusters.fasta" | grep -c 'cluster=a1;')" = "2" ] &&
+   [ "$(grep '^>b2 ' "${TMPDIR}/clusters.fasta" | grep -c 'cluster=b1;')" = "1" ] &&
+   grep '^>b1 ' "${TMPDIR}/clusters.fasta" | grep -q 'cluster_center=true;' &&
+   grep '^>a1 ' "${TMPDIR}/clusters.fasta" | grep -q 'cluster_weight=25;'
+then
+    log "$MCMD: sequences are attached to the right center OK"
+    ((success++))
+else
+    log "$MCMD: sequences are not attached to the right center"
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD --max-cpu 2 -t 0.95 --otu-map "${TMPDIR}/otu.csv" --no-sequence-output "${TMPDIR}/data.fasta" > "${TMPDIR}/none.fasta" 2> /dev/null &&
+   [ ! -s "${TMPDIR}/none.fasta" ] &&
+   [ "$(head -n 1 "${TMPDIR}/otu.csv")" = "id,cluster,cluster_center,cluster_score,count" ] &&
+   [ "$(sed -n 2,4p "${TMPDIR}/otu.csv" | cut -d, -f1,2 | tr '\n' ' ')" = "a1,a1 a3,a1 a2,a1 " ] &&
+   [ "$(grep -c ',true,' "${TMPDIR}/otu.csv")" = "2" ]
+then
+    log "$MCMD: CSV table of the clusters OK"
+    ((success++))
+else
+    log "$MCMD: CSV table of the clusters failed"
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD --max-cpu 1 -O -t 0.95 "${TMPDIR}/data.fasta" 2> /dev/null | grep '^>' | sed 's/ .*//' > "${TMPDIR}/order1.txt" &&
+   $CMD --max-cpu 8 -O -t 0.95 "${TMPDIR}/data.fasta" 2> /dev/null | grep '^>' | sed 's/ .*//' > "${TMPDIR}/order8.txt" &&
+   cmp -s "${TMPDIR}/order1.txt" "${TMPDIR}/order8.txt"
+then
+    log "$MCMD: the result does not depend on the number of threads OK"
+    ((success++))
+else
+    log "$MCMD: the result depends on the number of threads"
+    ((failed++))
+fi
+
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obiconvert/out_ecotag.fasta

@@ -0,0 +1,24 @@
+>HELIUM_000100422_612GNAAXX:7:118:3572:14633#0/1_sub[28..126] {"count":10172,"merged_sample":{"26a_F040644":10172},"obitag_bestid":0.9797979797979798,"obitag_bestmatch":"AY227529","obitag_match_count":1,"obitag_rank":"genus","obitag_similarity_method":"lcs","taxid":"taxon:9992 [Marmota]@genus"}
+ttagccctaaacataaacattcaataaacaagaatgttcgccagagtactactagcaaca
+gcctgaaactcaaaggacttggcggtgctttacatccct
+>HELIUM_000100422_612GNAAXX:7:99:9351:13090#0/1_sub[28..127] {"count":260,"merged_sample":{"29a_F260619":260},"obitag_bestid":0.9405940594059405,"obitag_bestmatch":"AF154263","obitag_match_count":9,"obitag_rank":"infraorder","obitag_similarity_method":"lcs","taxid":"taxon:35500 [Pecora]@infraorder"}
+ttagccctaaacacaaataattacacaaacaaaattgttcaccagagtactagcggcaac
+agcttaaaactcaaaggacttggcggtgctttataccctt
+>HELIUM_000100422_612GNAAXX:7:108:10111:9078#0/1_sub[28..127] {"count":7146,"merged_sample":{"13a_F730603":7146},"obitag_bestid":1,"obitag_bestmatch":"AB245427","obitag_match_count":1,"obitag_rank":"species","obitag_similarity_method":"lcs","taxid":"taxon:9860 [Cervus elaphus]@species"}
+ctagccttaaacacaaatagttatgcaaacaaaactattcgccagagtactaccggcaat
+agcttaaaactcaaaggacttggcggtgctttataccctt
+>HELIUM_000100422_612GNAAXX:7:38:14204:12725#0/1_sub[28..126] {"count":87,"merged_sample":{"26a_F040644":87},"obitag_bestid":0.9494949494949495,"obitag_bestmatch":"AY227530","obitag_match_count":2,"obitag_rank":"tribe","obitag_similarity_method":"lcs","taxid":"taxon:337730 [Marmotini]@tribe"}
+ttagccctaaacataaacattcaataaacaagaatgttcgccagaggactactagcaata
+gcttaaaactcaaaggacttggcggtgctttatatccct
+>HELIUM_000100422_612GNAAXX:7:30:9942:4495#0/1_sub[28..126] {"count":95,"merged_sample":{"26a_F040644":11,"29a_F260619":84},"obitag_bestid":0.9595959595959596,"obitag_bestmatch":"AC187326","obitag_match_count":1,"obitag_rank":"subspecies","obitag_similarity_method":"lcs","taxid":"taxon:9615 [Canis lupus familiaris]@subspecies"}
+ttagccctaaacataagctattccataacaaaataattcgccagagaactactagcaaca
+gattaaacctcaaaggacttggcagtgctttatacccct
+>HELIUM_000100422_612GNAAXX:7:51:16702:19393#0/1_sub[28..127] {"count":12004,"merged_sample":{"15a_F730814":7465,"29a_F260619":4539},"obitag_bestid":1,"obitag_bestmatch":"AJ885202","obitag_match_count":1,"obitag_rank":"species","obitag_similarity_method":"lcs","taxid":"taxon:9858 [Capreolus capreolus]@species"}
+ttagccctaaacacaagtaattaatataacaaaattattcgccagagtactaccggcaat
+agcttaaaactcaaaggacttggcggtgctttataccctt
+>HELIUM_000100422_612GNAAXX:7:84:14502:1617#0/1_sub[28..127] {"count":319,"merged_sample":{"29a_F260619":319},"obitag_bestid":1,"obitag_bestmatch":"AJ972683","obitag_match_count":1,"obitag_rank":"species","obitag_similarity_method":"lcs","taxid":"taxon:9858 [Capreolus capreolus]@species"}
+ttagccctaaacacaagtaattattataacaaaattattcgccagagtactaccggcaat
+agcttaaaactcaaaggacttggcggtgctttataccctt
+>HELIUM_000100422_612GNAAXX:7:50:10637:6527#0/1_sub[28..126] {"count":366,"merged_sample":{"13a_F730603":13,"15a_F730814":5,"26a_F040644":347,"29a_F260619":1},"obitag_bestid":1,"obitag_bestmatch":"AB048590","obitag_match_count":1,"obitag_rank":"genus","obitag_similarity_method":"lcs","taxid":"taxon:9611 [Canis]@genus"}
+ttagccctaaacatagataattttacaacaaaataattcgccagaggactactagcaata
+gcttaaaactcaaaggacttggcggtgctttatatccct

obitests/obitools/obiconvert/taxonomy.csv

@@ -0,0 +1,48 @@
+taxid,parent,taxonomic_rank,scientific_name
+taxon:1 [root]@no rank,taxon:1 [root]@no rank,no rank,root
+taxon:131567 [cellular organisms]@cellular root,taxon:1 [root]@no rank,cellular root,cellular organisms
+taxon:2759 [Eukaryota]@domain,taxon:131567 [cellular organisms]@cellular root,domain,Eukaryota
+taxon:33154 [Opisthokonta]@clade,taxon:2759 [Eukaryota]@domain,clade,Opisthokonta
+taxon:33208 [Metazoa]@kingdom,taxon:33154 [Opisthokonta]@clade,kingdom,Metazoa
+taxon:6072 [Eumetazoa]@clade,taxon:33208 [Metazoa]@kingdom,clade,Eumetazoa
+taxon:33213 [Bilateria]@clade,taxon:6072 [Eumetazoa]@clade,clade,Bilateria
+taxon:33511 [Deuterostomia]@clade,taxon:33213 [Bilateria]@clade,clade,Deuterostomia
+taxon:7711 [Chordata]@phylum,taxon:33511 [Deuterostomia]@clade,phylum,Chordata
+taxon:89593 [Craniata]@subphylum,taxon:7711 [Chordata]@phylum,subphylum,Craniata
+taxon:7742 [Vertebrata]@clade,taxon:89593 [Craniata]@subphylum,clade,Vertebrata
+taxon:7776 [Gnathostomata]@clade,taxon:7742 [Vertebrata]@clade,clade,Gnathostomata
+taxon:117570 [Teleostomi]@clade,taxon:7776 [Gnathostomata]@clade,clade,Teleostomi
+taxon:117571 [Euteleostomi]@clade,taxon:117570 [Teleostomi]@clade,clade,Euteleostomi
+taxon:8287 [Sarcopterygii]@superclass,taxon:117571 [Euteleostomi]@clade,superclass,Sarcopterygii
+taxon:1338369 [Dipnotetrapodomorpha]@clade,taxon:8287 [Sarcopterygii]@superclass,clade,Dipnotetrapodomorpha
+taxon:32523 [Tetrapoda]@clade,taxon:1338369 [Dipnotetrapodomorpha]@clade,clade,Tetrapoda
+taxon:32524 [Amniota]@clade,taxon:32523 [Tetrapoda]@clade,clade,Amniota
+taxon:40674 [Mammalia]@class,taxon:32524 [Amniota]@clade,class,Mammalia
+taxon:32525 [Theria]@clade,taxon:40674 [Mammalia]@class,clade,Theria
+taxon:9347 [Eutheria]@clade,taxon:32525 [Theria]@clade,clade,Eutheria
+taxon:1437010 [Boreoeutheria]@clade,taxon:9347 [Eutheria]@clade,clade,Boreoeutheria
+taxon:314146 [Euarchontoglires]@superorder,taxon:1437010 [Boreoeutheria]@clade,superorder,Euarchontoglires
+taxon:314145 [Laurasiatheria]@superorder,taxon:1437010 [Boreoeutheria]@clade,superorder,Laurasiatheria
+taxon:33554 [Carnivora]@order,taxon:314145 [Laurasiatheria]@superorder,order,Carnivora
+taxon:91561 [Artiodactyla]@order,taxon:314145 [Laurasiatheria]@superorder,order,Artiodactyla
+taxon:314147 [Glires]@clade,taxon:314146 [Euarchontoglires]@superorder,clade,Glires
+taxon:9845 [Ruminantia]@suborder,taxon:91561 [Artiodactyla]@order,suborder,Ruminantia
+taxon:35500 [Pecora]@infraorder,taxon:9845 [Ruminantia]@suborder,infraorder,Pecora
+taxon:9989 [Rodentia]@order,taxon:314147 [Glires]@clade,order,Rodentia
+taxon:379584 [Caniformia]@suborder,taxon:33554 [Carnivora]@order,suborder,Caniformia
+taxon:9608 [Canidae]@family,taxon:379584 [Caniformia]@suborder,family,Canidae
+taxon:9850 [Cervidae]@family,taxon:35500 [Pecora]@infraorder,family,Cervidae
+taxon:9881 [Odocoileinae]@subfamily,taxon:9850 [Cervidae]@family,subfamily,Odocoileinae
+taxon:33553 [Sciuromorpha]@suborder,taxon:9989 [Rodentia]@order,suborder,Sciuromorpha
+taxon:55153 [Sciuridae]@family,taxon:33553 [Sciuromorpha]@suborder,family,Sciuridae
+taxon:34878 [Cervinae]@subfamily,taxon:9850 [Cervidae]@family,subfamily,Cervinae
+taxon:9611 [Canis]@genus,taxon:9608 [Canidae]@family,genus,Canis
+taxon:9857 [Capreolus]@genus,taxon:9881 [Odocoileinae]@subfamily,genus,Capreolus
+taxon:9612 [Canis lupus]@species,taxon:9611 [Canis]@genus,species,Canis lupus
+taxon:337726 [Xerinae]@subfamily,taxon:55153 [Sciuridae]@family,subfamily,Xerinae
+taxon:9859 [Cervus]@genus,taxon:34878 [Cervinae]@subfamily,genus,Cervus
+taxon:337730 [Marmotini]@tribe,taxon:337726 [Xerinae]@subfamily,tribe,Marmotini
+taxon:9992 [Marmota]@genus,taxon:337730 [Marmotini]@tribe,genus,Marmota
+taxon:9860 [Cervus elaphus]@species,taxon:9859 [Cervus]@genus,species,Cervus elaphus
+taxon:9615 [Canis lupus familiaris]@subspecies,taxon:9612 [Canis lupus]@species,subspecies,Canis lupus familiaris
+taxon:9858 [Capreolus capreolus]@species,taxon:9857 [Capreolus]@genus,species,Capreolus capreolus

obitests/obitools/obiconvert/test.sh

@@ -134,6 +134,130 @@ else
     ((failed++))
 fi
 
+
+# ------------------------------------------------------------------
+# --raw-taxid tests (no taxonomy loaded)
+# ------------------------------------------------------------------
+
+# Running test
+((ntest++))
+if obiconvert --raw-taxid "${TEST_DIR}/out_ecotag.fasta" \
+              > "${TMPDIR}/raw_taxid.fasta" 2>/dev/null
+then
+    log "$MCMD --raw-taxid: running OK"
+    ((success++))
+else
+    log "$MCMD --raw-taxid: running failed"
+    ((failed++))
+fi
+
+# Taxids must be bare numbers — no full-format "taxon:ID [Name]@rank" strings
+((ntest++))
+if grep '"taxid"' "${TMPDIR}/raw_taxid.fasta" | grep -qv '"taxid":"[0-9][0-9]*"'
+then
+    log "$MCMD --raw-taxid: taxid format check failed (full-format taxid found)"
+    ((failed++))
+else
+    log "$MCMD --raw-taxid: taxid format OK (all taxids are bare numbers)"
+    ((success++))
+fi
+
+# --raw-taxid is idempotent: piping through a second obiconvert --raw-taxid must
+# produce bit-for-bit identical output.
+((ntest++))
+if obiconvert --raw-taxid "${TMPDIR}/raw_taxid.fasta" \
+              > "${TMPDIR}/raw_taxid2.fasta" 2>/dev/null
+then
+    log "$MCMD --raw-taxid piped: running OK"
+    ((success++))
+else
+    log "$MCMD --raw-taxid piped: running failed"
+    ((failed++))
+fi
+
+((ntest++))
+if diff "${TMPDIR}/raw_taxid.fasta" \
+        "${TMPDIR}/raw_taxid2.fasta" > /dev/null
+then
+    log "$MCMD --raw-taxid piped: idempotency OK"
+    ((success++))
+else
+    log "$MCMD --raw-taxid piped: idempotency failed (outputs differ)"
+    ((failed++))
+fi
+
+
+# ------------------------------------------------------------------
+# --taxonomy tests (full-format taxid, no --raw-taxid)
+# ------------------------------------------------------------------
+
+# Running test
+((ntest++))
+if obiconvert --taxonomy "${TEST_DIR}/taxonomy.csv" \
+              "${TEST_DIR}/out_ecotag.fasta" \
+              > "${TMPDIR}/taxo.fasta" 2>/dev/null
+then
+    log "$MCMD --taxonomy: running OK"
+    ((success++))
+else
+    log "$MCMD --taxonomy: running failed"
+    ((failed++))
+fi
+
+# Taxids must be in full "taxon:ID [Name]@rank" format
+((ntest++))
+if grep '"taxid"' "${TMPDIR}/taxo.fasta" | grep -q '"taxid":"taxon:[0-9]'
+then
+    log "$MCMD --taxonomy: taxid format OK (full-format taxids present)"
+    ((success++))
+else
+    log "$MCMD --taxonomy: taxid format check failed (no full-format taxid found)"
+    ((failed++))
+fi
+
+
+# ------------------------------------------------------------------
+# --raw-taxid --taxonomy tests
+# ------------------------------------------------------------------
+
+# Running test
+((ntest++))
+if obiconvert --raw-taxid --taxonomy "${TEST_DIR}/taxonomy.csv" \
+              "${TEST_DIR}/out_ecotag.fasta" \
+              > "${TMPDIR}/raw_taxid_taxo.fasta" 2>/dev/null
+then
+    log "$MCMD --raw-taxid --taxonomy: running OK"
+    ((success++))
+else
+    log "$MCMD --raw-taxid --taxonomy: running failed"
+    ((failed++))
+fi
+
+# Taxids must be bare numbers even when taxonomy is loaded
+((ntest++))
+if grep '"taxid"' "${TMPDIR}/raw_taxid_taxo.fasta" | grep -qv '"taxid":"[0-9][0-9]*"'
+then
+    log "$MCMD --raw-taxid --taxonomy: taxid format check failed (full-format taxid found)"
+    ((failed++))
+else
+    log "$MCMD --raw-taxid --taxonomy: taxid format OK (all taxids are bare numbers)"
+    ((success++))
+fi
+
+# --raw-taxid with or without taxonomy must yield identical taxid values
+((ntest++))
+if diff <(grep '"taxid"' "${TMPDIR}/raw_taxid.fasta" | grep -o '"taxid":"[^"]*"' | sort) \
+        <(grep '"taxid"' "${TMPDIR}/raw_taxid_taxo.fasta" | grep -o '"taxid":"[^"]*"' | sort) \
+        > /dev/null
+then
+    log "$MCMD --raw-taxid vs --raw-taxid --taxonomy: taxid values match OK"
+    ((success++))
+else
+    log "$MCMD --raw-taxid vs --raw-taxid --taxonomy: taxid values differ (unexpected)"
+    ((failed++))
+fi
+
+
 #########################################
 #
 # At the end of the tests

obitests/obitools/obiconvert/xxx

@@ -0,0 +1,24 @@
+>HELIUM_000100422_612GNAAXX:7:118:3572:14633#0/1_sub[28..126] {"count":10172,"merged_sample":{"26a_F040644":10172},"obitag_bestid":0.9797979797979798,"obitag_bestmatch":"AY227529","obitag_match_count":1,"obitag_rank":"genus","obitag_similarity_method":"lcs","taxid":"taxon:9992 [Marmota]@genus"}
+ttagccctaaacataaacattcaataaacaagaatgttcgccagagtactactagcaaca
+gcctgaaactcaaaggacttggcggtgctttacatccct
+>HELIUM_000100422_612GNAAXX:7:99:9351:13090#0/1_sub[28..127] {"count":260,"merged_sample":{"29a_F260619":260},"obitag_bestid":0.9405940594059405,"obitag_bestmatch":"AF154263","obitag_match_count":9,"obitag_rank":"infraorder","obitag_similarity_method":"lcs","taxid":"taxon:35500 [Pecora]@infraorder"}
+ttagccctaaacacaaataattacacaaacaaaattgttcaccagagtactagcggcaac
+agcttaaaactcaaaggacttggcggtgctttataccctt
+>HELIUM_000100422_612GNAAXX:7:108:10111:9078#0/1_sub[28..127] {"count":7146,"merged_sample":{"13a_F730603":7146},"obitag_bestid":1,"obitag_bestmatch":"AB245427","obitag_match_count":1,"obitag_rank":"species","obitag_similarity_method":"lcs","taxid":"taxon:9860 [Cervus elaphus]@species"}
+ctagccttaaacacaaatagttatgcaaacaaaactattcgccagagtactaccggcaat
+agcttaaaactcaaaggacttggcggtgctttataccctt
+>HELIUM_000100422_612GNAAXX:7:38:14204:12725#0/1_sub[28..126] {"count":87,"merged_sample":{"26a_F040644":87},"obitag_bestid":0.9494949494949495,"obitag_bestmatch":"AY227530","obitag_match_count":2,"obitag_rank":"tribe","obitag_similarity_method":"lcs","taxid":"taxon:337730 [Marmotini]@tribe"}
+ttagccctaaacataaacattcaataaacaagaatgttcgccagaggactactagcaata
+gcttaaaactcaaaggacttggcggtgctttatatccct
+>HELIUM_000100422_612GNAAXX:7:30:9942:4495#0/1_sub[28..126] {"count":95,"merged_sample":{"26a_F040644":11,"29a_F260619":84},"obitag_bestid":0.9595959595959596,"obitag_bestmatch":"AC187326","obitag_match_count":1,"obitag_rank":"subspecies","obitag_similarity_method":"lcs","taxid":"taxon:9615 [Canis lupus familiaris]@subspecies"}
+ttagccctaaacataagctattccataacaaaataattcgccagagaactactagcaaca
+gattaaacctcaaaggacttggcagtgctttatacccct
+>HELIUM_000100422_612GNAAXX:7:51:16702:19393#0/1_sub[28..127] {"count":12004,"merged_sample":{"15a_F730814":7465,"29a_F260619":4539},"obitag_bestid":1,"obitag_bestmatch":"AJ885202","obitag_match_count":1,"obitag_rank":"species","obitag_similarity_method":"lcs","taxid":"taxon:9858 [Capreolus capreolus]@species"}
+ttagccctaaacacaagtaattaatataacaaaattattcgccagagtactaccggcaat
+agcttaaaactcaaaggacttggcggtgctttataccctt
+>HELIUM_000100422_612GNAAXX:7:84:14502:1617#0/1_sub[28..127] {"count":319,"merged_sample":{"29a_F260619":319},"obitag_bestid":1,"obitag_bestmatch":"AJ972683","obitag_match_count":1,"obitag_rank":"species","obitag_similarity_method":"lcs","taxid":"taxon:9858 [Capreolus capreolus]@species"}
+ttagccctaaacacaagtaattattataacaaaattattcgccagagtactaccggcaat
+agcttaaaactcaaaggacttggcggtgctttataccctt
+>HELIUM_000100422_612GNAAXX:7:50:10637:6527#0/1_sub[28..126] {"count":366,"merged_sample":{"13a_F730603":13,"15a_F730814":5,"26a_F040644":347,"29a_F260619":1},"obitag_bestid":1,"obitag_bestmatch":"AB048590","obitag_match_count":1,"obitag_rank":"genus","obitag_similarity_method":"lcs","taxid":"taxon:9611 [Canis]@genus"}
+ttagccctaaacatagataattttacaacaaaataattcgccagaggactactagcaata
+gcttaaaactcaaaggacttggcggtgctttatatccct

obitests/obitools/obioligotag/test.sh

@@ -0,0 +1,205 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obioligotag
+CMD=obioligotag
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+#
+# Every result is checked on its properties (size, alphabet, constraints),
+# not against a reference file, because several equivalent families exist.
+#
+
+printf 'acgtac\nacgtat\nttttgg\nggcaac\ncaccat\nacgtac\n' > "${TMPDIR}/oligos.txt"
+printf '>a\nACGTAC\n>b\nACGTAT\n>c\nTTTTGG\n>d\nGGCAAC\n>e\nCACCAT\n' > "${TMPDIR}/oligos.fasta"
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1 
+then
+    log "$MCMD: printing help OK" 
+    ((success++))
+else
+    log "$MCMD: printing help failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD -s 5 -f 24 -d 3 -g 3 -p 2 > "${TMPDIR}/tags.fasta" 2> /dev/null \
+   && [ "$(grep -vc '>' "${TMPDIR}/tags.fasta")" -ge 24 ] \
+   && ! grep -v '>' "${TMPDIR}/tags.fasta" | grep -qE '(a{3}|c{3}|g{3}|t{3})'
+then
+    log "$MCMD: generating a family with homopolymer constraint OK" 
+    ((success++))
+else
+    log "$MCMD: generating a family with homopolymer constraint failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD -s 7 -f 60 -d 3 --alphabet ACT > "${TMPDIR}/noG.fasta" 2> /dev/null \
+   && [ "$(grep -vc '>' "${TMPDIR}/noG.fasta")" -ge 60 ] \
+   && ! grep -v '>' "${TMPDIR}/noG.fasta" | grep -q 'g'
+then
+    log "$MCMD: restricted alphabet OK" 
+    ((success++))
+else
+    log "$MCMD: restricted alphabet failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD -s 6 -f 20 -d 3 --distance-type lcs > "${TMPDIR}/lcs.fasta" 2> /dev/null \
+   && [ "$(grep -vc '>' "${TMPDIR}/lcs.fasta")" -ge 20 ]
+then
+    log "$MCMD: LCS distance OK" 
+    ((success++))
+else
+    log "$MCMD: LCS distance failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD -s 7 -f 40 -d 3 -r cnnnnnn > "${TMPDIR}/rejected.fasta" 2> /dev/null \
+   && [ "$(grep -vc '>' "${TMPDIR}/rejected.fasta")" -ge 40 ] \
+   && ! grep -v '>' "${TMPDIR}/rejected.fasta" | grep -q '^c'
+then
+    log "$MCMD: rejected pattern OK" 
+    ((success++))
+else
+    log "$MCMD: rejected pattern failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD -L "${TMPDIR}/oligos.txt" -f 3 -d 3 > "${TMPDIR}/from_text.fasta" 2> /dev/null \
+   && [ "$(grep -vc '>' "${TMPDIR}/from_text.fasta")" -ge 3 ]
+then
+    log "$MCMD: selecting oligonucleotides from a text file OK" 
+    ((success++))
+else
+    log "$MCMD: selecting oligonucleotides from a text file failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if $CMD -L "${TMPDIR}/oligos.fasta" -f 3 -d 3 > "${TMPDIR}/from_fasta.fasta" 2> /dev/null \
+   && [ "$(grep -vc '>' "${TMPDIR}/from_fasta.fasta")" -ge 3 ]
+then
+    log "$MCMD: selecting oligonucleotides from a fasta file OK" 
+    ((success++))
+else
+    log "$MCMD: selecting oligonucleotides from a fasta file failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if ! $CMD -s 5 -f 100 -d 3 > /dev/null 2>&1
+then
+    log "$MCMD: family size beyond the Hamming bound refused OK" 
+    ((success++))
+else
+    log "$MCMD: family size beyond the Hamming bound not refused" 
+    ((failed++))
+fi
+
+((ntest++))
+if ! $CMD -s 5 -L "${TMPDIR}/oligos.txt" -f 3 > /dev/null 2>&1
+then
+    log "$MCMD: --oligo-size with --oligo-list refused OK" 
+    ((success++))
+else
+    log "$MCMD: --oligo-size with --oligo-list not refused" 
+    ((failed++))
+fi
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obisuperkmer/README.md

@@ -0,0 +1,148 @@
+# Tests pour obisuperkmer
+
+## Description
+
+Ce répertoire contient les tests automatisés pour la commande `obisuperkmer`.
+
+## Fichiers
+
+- `test.sh` : Script de test principal (exécutable)
+- `test_sequences.fasta` : Jeu de données de test minimal (3 séquences courtes)
+- `README.md` : Ce fichier
+
+## Jeu de données de test
+
+Le fichier `test_sequences.fasta` contient 3 séquences de 32 nucléotides chacune :
+
+1. **seq1** : Répétition du motif ACGT (séquence régulière)
+2. **seq2** : Alternance de blocs homopolymères (AAAA, CCCC, GGGG, TTTT)
+3. **seq3** : Répétition du motif ATCG (différent de seq1)
+
+Ces séquences sont volontairement courtes pour :
+- Minimiser la taille du dépôt Git
+- Accélérer l'exécution des tests en CI/CD
+- Tester différents cas d'extraction de super k-mers
+
+## Tests effectués
+
+Le script `test.sh` effectue 12 tests :
+
+### Test 1 : Affichage de l'aide
+Vérifie que `obisuperkmer -h` s'exécute correctement.
+
+### Test 2 : Extraction basique avec paramètres par défaut
+Exécute `obisuperkmer` avec k=21, m=11 (valeurs par défaut).
+
+### Test 3 : Vérification du fichier de sortie non vide
+S'assure que la commande produit une sortie.
+
+### Test 4 : Comptage des super k-mers extraits
+Vérifie qu'au moins un super k-mer a été extrait.
+
+### Test 5 : Présence des métadonnées requises
+Vérifie que chaque super k-mer contient :
+- `minimizer_value`
+- `minimizer_seq`
+- `parent_id`
+
+### Test 6 : Extraction avec paramètres personnalisés
+Teste avec k=15 et m=7.
+
+### Test 7 : Vérification des paramètres dans les métadonnées
+S'assure que les valeurs k=15 et m=7 sont présentes dans la sortie.
+
+### Test 8 : Format de sortie FASTA explicite
+Teste l'option `--fasta-output`.
+
+### Test 9 : Vérification des IDs des super k-mers
+S'assure que tous les IDs contiennent "superkmer".
+
+### Test 10 : Préservation des IDs parents
+Vérifie que seq1, seq2 et seq3 apparaissent dans la sortie.
+
+### Test 11 : Option -o pour fichier de sortie
+Teste la redirection vers un fichier avec `-o`.
+
+### Test 12 : Vérification de la création du fichier avec -o
+S'assure que le fichier de sortie a été créé.
+
+### Test 13 : Cohérence des longueurs
+Vérifie que la somme des longueurs des super k-mers est inférieure ou égale à la longueur totale des séquences d'entrée.
+
+## Exécution des tests
+
+### Localement
+
+```bash
+cd /chemin/vers/obitools4/obitests/obitools/obisuperkmer
+./test.sh
+```
+
+### En CI/CD
+
+Les tests sont automatiquement exécutés lors de chaque commit via le système CI/CD configuré pour le projet.
+
+### Prérequis
+
+- La commande `obisuperkmer` doit être compilée et disponible dans `../../build/`
+- Les dépendances système : bash, grep, etc.
+
+## Structure du script de test
+
+Le script suit le pattern standard utilisé par tous les tests OBITools :
+
+1. **En-tête** : Définition du nom du test et de la commande
+2. **Variables** : Configuration des chemins et compteurs
+3. **Fonction cleanup()** : Affiche les résultats et nettoie le répertoire temporaire
+4. **Fonction log()** : Affiche les messages horodatés
+5. **Tests** : Série de tests avec incrémentation des compteurs
+6. **Appel cleanup()** : Nettoyage et sortie avec code de retour approprié
+
+## Format de sortie
+
+Chaque test affiche :
+```
+[obisuperkmer @ date] message
+```
+
+En fin d'exécution :
+```
+========================================
+## Results of the obisuperkmer tests:
+
+- 12 tests run
+- 12 successfully completed
+- 0 failed tests
+
+Cleaning up the temporary directory...
+
+========================================
+```
+
+## Codes de retour
+
+- **0** : Tous les tests ont réussi
+- **1** : Au moins un test a échoué
+
+## Ajout de nouveaux tests
+
+Pour ajouter un nouveau test, suivre le pattern :
+
+```bash
+((ntest++))
+if commande_test arguments
+then
+    log "Description: OK" 
+    ((success++))
+else
+    log "Description: failed"
+    ((failed++))
+fi
+```
+
+## Notes
+
+- Les fichiers temporaires sont créés dans `$TMPDIR` (créé par mktemp)
+- Les fichiers de données sont dans `$TEST_DIR`
+- La commande testée doit être dans `$OBITOOLS_DIR` (../../build/)
+- Le répertoire temporaire est automatiquement nettoyé à la fin

obitests/obitools/obisuperkmer/test.sh

@@ -0,0 +1,232 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obik-super
+CMD=obik
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="OBIk-super"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed"
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..."
+log "Test directory is $TEST_DIR"
+log "obitools directory is $OBITOOLS_DIR"
+log "Temporary directory is $TMPDIR"
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+######################################################################
+
+((ntest++))
+if $CMD super -h > "${TMPDIR}/help.txt" 2>&1
+then
+    log "$MCMD: printing help OK"
+    ((success++))
+else
+    log "$MCMD: printing help failed"
+    ((failed++))
+fi
+
+# Test 1: Basic super k-mer extraction with default parameters
+((ntest++))
+if $CMD super "${TEST_DIR}/test_sequences.fasta" \
+    > "${TMPDIR}/output_default.fasta" 2>&1
+then
+    log "$MCMD: basic extraction with default parameters OK"
+    ((success++))
+else
+    log "$MCMD: basic extraction with default parameters failed"
+    ((failed++))
+fi
+
+# Test 2: Verify output is not empty
+((ntest++))
+if [ -s "${TMPDIR}/output_default.fasta" ]
+then
+    log "$MCMD: output file is not empty OK"
+    ((success++))
+else
+    log "$MCMD: output file is empty - failed"
+    ((failed++))
+fi
+
+# Test 3: Count number of super k-mers extracted (should be > 0)
+((ntest++))
+num_sequences=$(grep -c "^>" "${TMPDIR}/output_default.fasta")
+if [ "$num_sequences" -gt 0 ]
+then
+    log "$MCMD: extracted $num_sequences super k-mers OK"
+    ((success++))
+else
+    log "$MCMD: no super k-mers extracted - failed"
+    ((failed++))
+fi
+
+# Test 4: Verify super k-mers have required metadata attributes
+((ntest++))
+if grep -q "minimizer_value" "${TMPDIR}/output_default.fasta" && \
+   grep -q "minimizer_seq" "${TMPDIR}/output_default.fasta" && \
+   grep -q "parent_id" "${TMPDIR}/output_default.fasta"
+then
+    log "$MCMD: super k-mers contain required metadata OK"
+    ((success++))
+else
+    log "$MCMD: super k-mers missing metadata - failed"
+    ((failed++))
+fi
+
+# Test 5: Extract super k-mers with custom k and m parameters
+((ntest++))
+if $CMD super -k 15 -m 7 "${TEST_DIR}/test_sequences.fasta" \
+    > "${TMPDIR}/output_k15_m7.fasta" 2>&1
+then
+    log "$MCMD: extraction with custom k=15, m=7 OK"
+    ((success++))
+else
+    log "$MCMD: extraction with custom k=15, m=7 failed"
+    ((failed++))
+fi
+
+# Test 6: Verify custom parameters in output metadata
+((ntest++))
+if grep -q '"k":15' "${TMPDIR}/output_k15_m7.fasta" && \
+   grep -q '"m":7' "${TMPDIR}/output_k15_m7.fasta"
+then
+    log "$MCMD: custom parameters correctly set in metadata OK"
+    ((success++))
+else
+    log "$MCMD: custom parameters not in metadata - failed"
+    ((failed++))
+fi
+
+# Test 7: Test with different output format (FASTA output explicitly)
+((ntest++))
+if $CMD super --fasta-output -k 21 -m 11 \
+    "${TEST_DIR}/test_sequences.fasta" \
+    > "${TMPDIR}/output_fasta.fasta" 2>&1
+then
+    log "$MCMD: FASTA output format OK"
+    ((success++))
+else
+    log "$MCMD: FASTA output format failed"
+    ((failed++))
+fi
+
+# Test 8: Verify all super k-mers have superkmer in their ID
+((ntest++))
+if grep "^>" "${TMPDIR}/output_default.fasta" | grep -q "superkmer"
+then
+    log "$MCMD: super k-mer IDs contain 'superkmer' OK"
+    ((success++))
+else
+    log "$MCMD: super k-mer IDs missing 'superkmer' - failed"
+    ((failed++))
+fi
+
+# Test 9: Verify parent sequence IDs are preserved
+((ntest++))
+if grep -q "seq1" "${TMPDIR}/output_default.fasta" && \
+   grep -q "seq2" "${TMPDIR}/output_default.fasta" && \
+   grep -q "seq3" "${TMPDIR}/output_default.fasta"
+then
+    log "$MCMD: parent sequence IDs preserved OK"
+    ((success++))
+else
+    log "$MCMD: parent sequence IDs not preserved - failed"
+    ((failed++))
+fi
+
+# Test 10: Test with output file option
+((ntest++))
+if $CMD super -o "${TMPDIR}/output_file.fasta" \
+    "${TEST_DIR}/test_sequences.fasta" 2>&1
+then
+    log "$MCMD: output to file with -o option OK"
+    ((success++))
+else
+    log "$MCMD: output to file with -o option failed"
+    ((failed++))
+fi
+
+# Test 11: Verify output file was created with -o option
+((ntest++))
+if [ -s "${TMPDIR}/output_file.fasta" ]
+then
+    log "$MCMD: output file created with -o option OK"
+    ((success++))
+else
+    log "$MCMD: output file not created with -o option - failed"
+    ((failed++))
+fi
+
+# Test 12: Verify each super k-mer length is >= k (default k=31)
+((ntest++))
+min_len=$(grep -v "^>" "${TMPDIR}/output_default.fasta" | awk '{print length}' | sort -n | head -1)
+
+if [ "$min_len" -ge 31 ]
+then
+    log "$MCMD: all super k-mers have length >= k OK"
+    ((success++))
+else
+    log "$MCMD: some super k-mers shorter than k ($min_len < 31) - failed"
+    ((failed++))
+fi
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obisuperkmer/test_sequences.fasta

@@ -0,0 +1,6 @@
+>seq1
+ACGTACGTACGTACGTACGTACGTACGTACGT
+>seq2
+AAAACCCCGGGGTTTTAAAACCCCGGGGTTTT
+>seq3
+ATCGATCGATCGATCGATCGATCGATCGATCG

obitests/obitools/obiuniq/test.sh

@@ -195,6 +195,59 @@ else
     ((failed++))
 fi
 
+##
+## Test merge attributes consistency between in-memory and on-disk paths
+## This test catches the bug where the shared classifier in the on-disk
+## dereplication path caused incorrect merged attributes.
+##
+
+((ntest++))
+if obiuniq -m a -m b --in-memory \
+    "${TEST_DIR}/touniq.fasta" \
+    > "${TMPDIR}/touniq_u_merge_mem.fasta" 2>/dev/null
+then
+    log "OBIUniq merge in-memory: running OK"
+    ((success++))
+else
+    log "OBIUniq merge in-memory: running failed"
+    ((failed++))
+fi
+
+((ntest++))
+if obiuniq -m a -m b --chunk-count 4 \
+    "${TEST_DIR}/touniq.fasta" \
+    > "${TMPDIR}/touniq_u_merge_disk.fasta" 2>/dev/null
+then
+    log "OBIUniq merge on-disk: running OK"
+    ((success++))
+else
+    log "OBIUniq merge on-disk: running failed"
+    ((failed++))
+fi
+
+# Extract sorted annotations (JSON attributes) from both outputs
+# to compare merge results independently of sequence ordering
+grep '^>' "${TMPDIR}/touniq_u_merge_mem.fasta" \
+| sed 's/^>seq[0-9]* //' \
+| sort \
+> "${TMPDIR}/touniq_u_merge_mem.json"
+
+grep '^>' "${TMPDIR}/touniq_u_merge_disk.fasta" \
+| sed 's/^>seq[0-9]* //' \
+| sort \
+> "${TMPDIR}/touniq_u_merge_disk.json"
+
+((ntest++))
+if diff "${TMPDIR}/touniq_u_merge_mem.json" \
+        "${TMPDIR}/touniq_u_merge_disk.json" > /dev/null
+then
+    log "OBIUniq merge on-disk vs in-memory: result OK"
+    ((success++))
+else
+    log "OBIUniq merge on-disk vs in-memory: result failed"
+    ((failed++))
+fi
+
 #########################################
 #
 # At the end of the tests

pkg/obialign/fastlcs_bench_test.go

@@ -0,0 +1,111 @@
+package obialign
+
+import (
+	"fmt"
+	"math/rand"
+	"testing"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+// randomDNA returns a random ACGT sequence of length n.
+func randomDNA(rng *rand.Rand, n int) []byte {
+	const nuc = "ACGT"
+	s := make([]byte, n)
+	for i := range s {
+		s[i] = nuc[rng.Intn(4)]
+	}
+	return s
+}
+
+// mutate returns a copy of s in which each position is, with probability
+// rate, substituted, deleted or followed by an inserted base (equiprobable).
+func mutate(rng *rand.Rand, s []byte, rate float64) []byte {
+	const nuc = "ACGT"
+	out := make([]byte, 0, len(s)+len(s)/10)
+	for _, c := range s {
+		if rng.Float64() >= rate {
+			out = append(out, c)
+			continue
+		}
+		switch rng.Intn(3) {
+		case 0:
+			out = append(out, nuc[rng.Intn(4)])
+		case 1: // deletion
+		case 2:
+			out = append(out, c, nuc[rng.Intn(4)])
+		}
+	}
+	return out
+}
+
+// benchPairs builds a fixed set of pairs. "similar" pairs differ by about
+// 2% of mutated positions, "random" pairs are unrelated sequences.
+func benchPairs(n, length int, similar bool) [][2]*obiseq.BioSequence {
+	rng := rand.New(rand.NewSource(42))
+	pairs := make([][2]*obiseq.BioSequence, n)
+	for i := range pairs {
+		a := randomDNA(rng, length)
+		var b []byte
+		if similar {
+			b = mutate(rng, a, 0.02)
+		} else {
+			b = randomDNA(rng, length)
+		}
+		pairs[i] = [2]*obiseq.BioSequence{
+			obiseq.NewBioSequence("a", a, ""),
+			obiseq.NewBioSequence("b", b, ""),
+		}
+	}
+	return pairs
+}
+
+// BenchmarkLegacyLCS measures the historical implementation (the one
+// FastLCSScore used before BandedLCS), kept as the "before" reference.
+func BenchmarkLegacyLCS(b *testing.B) {
+	for _, similar := range []bool{true, false} {
+		kind := "random"
+		if similar {
+			kind = "similar"
+		}
+		for _, length := range []int{100, 500, 2000} {
+			pairs := benchPairs(64, length, similar)
+			for _, maxErr := range []int{-1, 5, length / 20, length / 5} {
+				name := fmt.Sprintf("%s/len=%d/maxErr=%d", kind, length, maxErr)
+				b.Run(name, func(b *testing.B) {
+					var buffer []uint64
+					b.ReportAllocs()
+					b.ResetTimer()
+					for i := 0; i < b.N; i++ {
+						p := pairs[i%len(pairs)]
+						FastLCSEGFScoreByte(p[0].Sequence(), p[1].Sequence(), maxErr, false, &buffer)
+					}
+				})
+			}
+		}
+	}
+}
+
+// BenchmarkBandedLCS runs on the same pairs as BenchmarkLegacyLCS.
+func BenchmarkBandedLCS(b *testing.B) {
+	for _, similar := range []bool{true, false} {
+		kind := "random"
+		if similar {
+			kind = "similar"
+		}
+		for _, length := range []int{100, 500, 2000} {
+			pairs := benchPairs(64, length, similar)
+			for _, maxErr := range []int{-1, 5, length / 20, length / 5} {
+				name := fmt.Sprintf("%s/len=%d/maxErr=%d", kind, length, maxErr)
+				b.Run(name, func(b *testing.B) {
+					b.ReportAllocs()
+					b.ResetTimer()
+					for i := 0; i < b.N; i++ {
+						p := pairs[i%len(pairs)]
+						BandedLCS(p[0].Sequence(), p[1].Sequence(), maxErr)
+					}
+				})
+			}
+		}
+	}
+}

pkg/obialign/fastlcsegf.go

@@ -2,6 +2,7 @@ package obialign
 
 import (
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+	log "github.com/sirupsen/logrus"
 )
 
 // FastLCSEGFScoreByte calculates the score of the Longest Common Subsequence (LCS) between two byte slices.
@@ -355,27 +356,35 @@ func FastLCSEGFScore(seqA, seqB *obiseq.BioSequence, maxError int, buffer *[]uin
 	return FastLCSEGFScoreByte(seqA.Sequence(), seqB.Sequence(), maxError, true, buffer)
 }
 
-// FastLCSScore calculates the score of the longest common subsequence between two bio sequences.
+// FastLCSScore calculates the score of the longest common subsequence between two bio sequences,
+// and the length of the shortest alignment achieving it (the distance is alilen - lcs).
 //
-// if maxError > 0, the maximum allowed error between the sequences is maxError.
+// If maxError >= 0, the result is exact when its distance is at most maxError,
-// Otherwise, no error checking is done.
+// and (-1, -1) is returned otherwise. If maxError < 0, no limit is applied.
-// If the actual number of errors is larger than maxError, -1 is returned for both values.
+// See BandedLCS, which does the work.
 //
 // The score matrix is:
-//   - Matching: 1
+//   - Matching: 1 (bases are compared with the IUPAC rules)
 //   - Mismatch or gap: 0
 //
+// The program stops with a fatal error if a sequence is longer than
+// MaxBandedLCSLength.
+//
 // Parameters:
 // - seqA: The first bio sequence.
 // - seqB: The second bio sequence.
 // - maxError: The maximum allowed error between the sequences. If set to -1, no limit is applied.
-// - buffer: A pointer to a uint64 slice to store intermediate results. If nil, a new slice is created.
+// - buffer: Unused, kept for compatibility; the working memory is pooled.
 //
 // Returns:
 // - The score of the longest common subsequence.
 // - The length of the shortest alignment corresponding to the LCS.
 func FastLCSScore(seqA, seqB *obiseq.BioSequence, maxError int, buffer *[]uint64) (int, int) {
-	score, alilen, _ := FastLCSEGFScoreByte(seqA.Sequence(), seqB.Sequence(), maxError, false, buffer)
+	score, alilen, err := BandedLCS(seqA.Sequence(), seqB.Sequence(), maxError)
+	if err != nil {
+		log.Fatalf("FastLCSScore on sequences %s (%d bp) and %s (%d bp): %v (maximum %d bp)",
+			seqA.Id(), seqA.Len(), seqB.Id(), seqB.Len(), err, MaxBandedLCSLength)
+	}
 
 	return score, alilen
 }

pkg/obialign/lcsband.go

@@ -0,0 +1,185 @@
+package obialign
+
+import (
+	"errors"
+	"sync"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+// MaxBandedLCSLength is the longest sequence BandedLCS accepts. The score and
+// the alignment length are packed on 16 bits each, and the alignment length
+// can reach the sum of both sequence lengths.
+const MaxBandedLCSLength = 1<<15 - 1
+
+// ErrSequenceTooLong is returned by BandedLCS when a sequence is longer than
+// MaxBandedLCSLength.
+var ErrSequenceTooLong = errors.New("obialign: sequence too long for BandedLCS")
+
+// A DP cell is packed on 32 bits as
+//
+//	score<<16 | (lcsLenBase - alignment length)
+//
+// so that the maximum of two cells ranks them by score first and, for equal
+// scores, by shortest alignment. The three moves then reduce to
+//
+//	cell(i,j) = max(diag + match<<16, up, left) - 1
+//
+// since every move lengthens the alignment by one and only a diagonal match
+// adds one to the score. The value 0 stands for an unavailable cell (outside
+// the band): it loses against any real cell. Every cell of the band has at
+// least one real neighbour (the band is connected and contains (0,0)), so
+// the subtraction never wraps for a cell that is used; the scalar reference
+// still clamps the maximum to 1 as a safety net.
+const (
+	lcsScoreUnit = uint32(1) << 16
+	lcsLenBase   = uint32(0xFFFF)
+)
+
+// bandedLCSWorkspace holds the scratch memory of one alignment.
+type bandedLCSWorkspace struct {
+	aRev, bCode []uint32    // encoded bases (aRev is reversed), SIMD kernel only
+	rows        [3][]uint32 // antidiagonal (SIMD) or row (scalar) buffers
+	aRev16      []uint16    // 16-bit encoded bases of the score-only kernel
+	bCode16     []uint16
+	rows16      [3][]uint16
+	widened     int // number of alignments that needed a second pass
+}
+
+var bandedLCSPool = sync.Pool{New: func() any { return new(bandedLCSWorkspace) }}
+
+func growUint32(s []uint32, n int) []uint32 {
+	if cap(s) < n {
+		return make([]uint32, n)
+	}
+	return s[:n]
+}
+
+// iupacMask gives for each byte its 4-bit nucleotide set (A=1, C=2, G=4,
+// T=8), or 0 when the byte is not an IUPAC letter. For two bytes of non-zero
+// mask, SameIUPACNuc(a, b) is exactly (mask[a] & mask[b]) != 0.
+var iupacMask = buildIUPACMask()
+
+func buildIUPACMask() (m [256]uint8) {
+	probes := [4]byte{'a', 'c', 'g', 't'}
+	for c := 0; c < 256; c++ {
+		if !(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z') {
+			continue
+		}
+		for bit, p := range probes {
+			if obiseq.SameIUPACNuc(byte(c), p) {
+				m[c] |= 1 << uint(bit)
+			}
+		}
+	}
+	return m
+}
+
+// allRegularIUPAC reports whether every base has a non-zero IUPAC mask, in
+// which case base compatibility is a plain AND of masks.
+func allRegularIUPAC(s []byte) bool {
+	for _, c := range s {
+		if iupacMask[c] == 0 {
+			return false
+		}
+	}
+	return true
+}
+
+// bandOffsets returns the diagonal offsets k = j - i covered by a band of
+// half-width g around the diagonals 0..delta, with sequence A (columns j) not
+// shorter than sequence B (rows i), clipped to the matrix.
+func bandOffsets(lA, lB, g int) (kmin, kmax int) {
+	return max(-g, -lB), min(lA-lB+g, lA)
+}
+
+// BandedLCS computes the score of the longest common subsequence of two
+// nucleotide sequences, and the length of the shortest alignment achieving
+// it (match: +1, mismatch and gap: 0; bases are compared with the IUPAC
+// rules of obiseq.SameIUPACNuc). The distance is alilen - lcs.
+//
+// If maxError >= 0, the result is the optimal (lcs, alilen) when its distance
+// is at most maxError, and (-1, -1) otherwise. This is exact, not a
+// heuristic: the DP is restricted to a band, and a bound on what a path
+// leaving the band can score proves when the band was wide enough. maxError < 0
+// means no limit.
+//
+// An error is returned when a sequence is longer than MaxBandedLCSLength.
+func BandedLCS(seqA, seqB []byte, maxError int) (lcs, alilen int, err error) {
+	if len(seqA) > MaxBandedLCSLength || len(seqB) > MaxBandedLCSLength {
+		return -1, -1, ErrSequenceTooLong
+	}
+	if len(seqA) < len(seqB) {
+		seqA, seqB = seqB, seqA
+	}
+	ws := bandedLCSPool.Get().(*bandedLCSWorkspace)
+	lcs, alilen = bandedLCS(seqA, seqB, maxError, ws, haveSIMD)
+	bandedLCSPool.Put(ws)
+	return lcs, alilen, nil
+}
+
+// bandedLCS expects len(a) >= len(b), both within MaxBandedLCSLength.
+//
+// With delta = lA - lB and a band of half-width g (offsets -g .. delta+g):
+//   - an alignment with at most E errors needs at least delta+2h gaps to reach
+//     the offset delta+h or -h, so it lies in the band of g0 = (E-delta)/2;
+//   - a path leaving a band of half-width g makes at least g+1 gaps on the B
+//     side, hence scores at most lB-g-1 matches, and has more than E errors
+//     as soon as g >= g0.
+//
+// Let Q be the best path of the band g0. If its distance exceeds E, the
+// global optimum cannot be within E either (it would be in the band and be
+// Q): the answer is -1. Otherwise, if score(Q) >= lB-g0-1 no outside path can
+// beat Q (a tie in score loses on length, the outside path having more errors
+// than Q): Q is the global optimum. Else the band is widened once to
+// g1 = min(E-delta, lB-score(Q)-1); since score(Q) >= lA-E for any alignment
+// within E errors, g1 always certifies the result.
+func bandedLCS(a, b []byte, maxError int, ws *bandedLCSWorkspace, useSIMD bool) (int, int) {
+	lA, lB := len(a), len(b)
+	delta := lA - lB
+	if maxError >= 0 && maxError < delta {
+		return -1, -1
+	}
+	if lB == 0 {
+		return 0, lA
+	}
+	simdOK := useSIMD && allRegularIUPAC(a) && allRegularIUPAC(b)
+	if simdOK {
+		encodeBases(ws, a, b)
+	}
+	run := func(g int) (int, int) {
+		kmin, kmax := bandOffsets(lA, lB, g)
+		return ws.align(a, b, kmin, kmax, simdOK)
+	}
+
+	if maxError < 0 {
+		return run(lA)
+	}
+	g0 := (maxError - delta) / 2
+	lcs, alilen := run(g0)
+	if alilen-lcs > maxError {
+		return -1, -1
+	}
+	if lcs >= lB-g0-1 {
+		return lcs, alilen
+	}
+	ws.widened++
+	lcs, alilen = run(min(maxError-delta, lB-lcs-1))
+	if alilen-lcs > maxError {
+		return -1, -1
+	}
+	return lcs, alilen
+}
+
+// align runs the banded DP on the offsets [kmin, kmax] (kmin <= 0 <= kmax) and
+// returns the best (lcs, alilen) of the band. The bases must have been encoded
+// with encodeBases when simdOK is set.
+func (ws *bandedLCSWorkspace) align(a, b []byte, kmin, kmax int, simdOK bool) (int, int) {
+	var key uint32
+	if simdOK {
+		key = bandedLCSSIMD(len(a), len(b), kmin, kmax, ws)
+	} else {
+		key = bandedLCSScalar(a, b, kmin, kmax, ws)
+	}
+	return int(key >> 16), int(lcsLenBase - key&0xFFFF)
+}

pkg/obialign/lcsband_nosimd.go

@@ -0,0 +1,17 @@
+//go:build !goexperiment.simd
+
+package obialign
+
+const haveSIMD = false
+
+// encodeBases and bandedLCSSIMD are never called without GOEXPERIMENT=simd
+// (haveSIMD is false); they exist to keep the dispatcher uniform.
+func encodeBases(ws *bandedLCSWorkspace, a, b []byte) {}
+
+func bandedLCSSIMD(lA, lB, kmin, kmax int, ws *bandedLCSWorkspace) uint32 {
+	return 0
+}
+
+func lcsScoreSIMD(a, b []byte, kmin, kmax, minLCS int, ws *bandedLCSWorkspace) int {
+	return -1
+}

pkg/obialign/lcsband_scalar.go

@@ -0,0 +1,53 @@
+package obialign
+
+import "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+
+// bandedLCSScalar is the portable reference of the banded DP, row by row. It
+// returns the packed cell (lB, lA). len(a) >= len(b) >= 1.
+//
+// Cells outside the band hold 0 (unavailable). Row i spans the columns
+// [i+kmin, i+kmax] clipped to [1, lA]; the cell just left of that span and
+// the one just right of it are reset so that the next row never reads a stale
+// value.
+func bandedLCSScalar(a, b []byte, kmin, kmax int, ws *bandedLCSWorkspace) uint32 {
+	lA, lB := len(a), len(b)
+
+	prev := growUint32(ws.rows[0], lA+2)
+	cur := growUint32(ws.rows[1], lA+2)
+	ws.rows[0], ws.rows[1] = prev, cur
+
+	for j := 0; j <= lA; j++ {
+		prev[j] = 0
+		if j <= kmax {
+			prev[j] = lcsLenBase - uint32(j)
+		}
+	}
+	prev[lA+1] = 0
+
+	for i := 1; i <= lB; i++ {
+		if -i >= kmin {
+			cur[0] = lcsLenBase - uint32(i)
+		} else {
+			cur[0] = 0
+		}
+		jlo := max(1, i+kmin)
+		jhi := min(lA, i+kmax)
+		if jlo > 1 {
+			cur[jlo-1] = 0
+		}
+		bi := b[i-1]
+		for j := jlo; j <= jhi; j++ {
+			d := prev[j-1]
+			if obiseq.SameIUPACNuc(a[j-1], bi) {
+				d += lcsScoreUnit
+			}
+			m := max(d, prev[j], cur[j-1], 1)
+			cur[j] = m - 1
+		}
+		if jhi < lA {
+			cur[jhi+1] = 0
+		}
+		prev, cur = cur, prev
+	}
+	return prev[lA]
+}

pkg/obialign/lcsband_simd.go

@@ -0,0 +1,107 @@
+//go:build goexperiment.simd
+
+package obialign
+
+import "simd"
+
+const haveSIMD = true
+
+func simdLanes() int {
+	var probe simd.Uint32s
+	return probe.Len()
+}
+
+// encodeIUPAC writes in dst the 4-bit IUPAC mask of each base of s, reversed
+// if rev is set, followed by zero padding up to len(dst).
+func encodeIUPAC(dst []uint32, s []byte, rev bool) {
+	n := len(s)
+	for x := range dst {
+		dst[x] = 0
+	}
+	if rev {
+		for x, c := range s {
+			dst[n-1-x] = uint32(iupacMask[c])
+		}
+		return
+	}
+	for x, c := range s {
+		dst[x] = uint32(iupacMask[c])
+	}
+}
+
+// encodeBases encodes both sequences once for all the passes of an alignment.
+func encodeBases(ws *bandedLCSWorkspace, a, b []byte) {
+	vl := simdLanes()
+	ws.aRev = growUint32(ws.aRev, len(a)+vl)
+	ws.bCode = growUint32(ws.bCode, len(b)+vl)
+	encodeIUPAC(ws.aRev, a, true)
+	encodeIUPAC(ws.bCode, b, false)
+}
+
+// bandedLCSSIMD runs the banded DP along antidiagonals t = i + j. Lane i of
+// the buffer of antidiagonal t holds the cell (i, t-i), so that
+//
+//	diag = buffer[t-2][i-1]   up = buffer[t-1][i-1]   left = buffer[t-1][i]
+//
+// are plain unaligned loads. Along an antidiagonal the offset j-i = t-2i
+// must stay in [kmin, kmax], which gives the window [lo, hi] of rows. The
+// vector loop may run past hi; the lanes beyond are then reset to 0
+// (unavailable), as is the lane just before the window, because the next
+// antidiagonals read them as neighbours. Lane 0 and lane t carry the first
+// row and the first column of the matrix.
+//
+// Requires encodeBases to have been called, lA >= lB >= 1 and kmin <= 0 <=
+// kmax.
+func bandedLCSSIMD(lA, lB, kmin, kmax int, ws *bandedLCSWorkspace) uint32 {
+	vl := simdLanes()
+	aRev, bCode := ws.aRev, ws.bCode
+
+	size := lB + 2*vl + 2
+	for r := range ws.rows {
+		ws.rows[r] = growUint32(ws.rows[r], size)
+		clear(ws.rows[r])
+	}
+	p2, p1, cur := ws.rows[0], ws.rows[1], ws.rows[2]
+
+	p2[0] = lcsLenBase // cell (0,0)
+	if kmax >= 1 {
+		p1[0] = lcsLenBase - 1 // cell (0,1)
+	}
+	if kmin <= -1 {
+		p1[1] = lcsLenBase - 1 // cell (1,0)
+	}
+
+	zero := simd.BroadcastUint32s(0)
+	one := simd.BroadcastUint32s(1)
+	unit := simd.BroadcastUint32s(lcsScoreUnit)
+
+	for t := 2; t <= lA+lB; t++ {
+		lo := max(1, t-lA, (t-kmax+1)>>1)
+		hi := min(lB, t-1, (t-kmin)>>1)
+
+		for i := lo; i <= hi; i += vl {
+			ca := simd.LoadUint32s(aRev[lA-t+i:])
+			cb := simd.LoadUint32s(bCode[i-1:])
+			inc := unit.Masked(ca.And(cb).NotEqual(zero))
+			d := simd.LoadUint32s(p2[i-1:]).Add(inc)
+			u := simd.LoadUint32s(p1[i-1:])
+			l := simd.LoadUint32s(p1[i:])
+			d.Max(u).Max(l).Sub(one).Store(cur[i:])
+		}
+
+		zero.Store(cur[hi+1:])
+		if lo > 1 {
+			cur[lo-1] = 0
+		}
+		cur[0] = 0
+		if t <= lA && t <= kmax {
+			cur[0] = lcsLenBase - uint32(t)
+		}
+		if t <= lB && -t >= kmin {
+			cur[t] = lcsLenBase - uint32(t)
+		}
+
+		p2, p1, cur = p1, cur, p2
+	}
+	return p1[lB]
+}

pkg/obialign/lcsband_test.go

@@ -0,0 +1,360 @@
+package obialign
+
+import (
+	"math/rand"
+	"testing"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+// bandDP is the oracle: a plain DP over the whole matrix, written
+// independently of the production code, in which only the cells with a
+// diagonal offset j-i in [kmin, kmax] exist (kmin=-lB, kmax=lA when
+// unbounded). It returns the optimal LCS score and the shortest alignment
+// length achieving it among the alignments staying in the band.
+func bandDP(a, b []byte, kmin, kmax int) (int, int) {
+	lA, lB := len(a), len(b)
+	type cell struct {
+		s, l int
+		ok   bool
+	}
+	better := func(x, y cell) bool {
+		return !y.ok || (x.ok && (x.s > y.s || (x.s == y.s && x.l < y.l)))
+	}
+	m := make([][]cell, lB+1)
+	for i := range m {
+		m[i] = make([]cell, lA+1)
+	}
+	for i := 0; i <= lB; i++ {
+		for j := 0; j <= lA; j++ {
+			if k := j - i; k < kmin || k > kmax {
+				continue
+			}
+			if i == 0 || j == 0 {
+				m[i][j] = cell{0, i + j, true}
+				continue
+			}
+			var best cell
+			if d := m[i-1][j-1]; d.ok {
+				best = cell{d.s, d.l + 1, true}
+				if obiseq.SameIUPACNuc(a[j-1], b[i-1]) {
+					best.s++
+				}
+			}
+			if u := m[i-1][j]; u.ok && better(cell{u.s, u.l + 1, true}, best) {
+				best = cell{u.s, u.l + 1, true}
+			}
+			if l := m[i][j-1]; l.ok && better(cell{l.s, l.l + 1, true}, best) {
+				best = cell{l.s, l.l + 1, true}
+			}
+			m[i][j] = best
+		}
+	}
+	return m[lB][lA].s, m[lB][lA].l
+}
+
+// fullDP is the unbanded optimum.
+func fullDP(a, b []byte) (int, int) { return bandDP(a, b, -len(b), len(a)) }
+
+// expected is the contract of BandedLCS: the unbounded optimum when it has at
+// most maxError errors, (-1,-1) otherwise.
+func expected(a, b []byte, maxError int) (int, int) {
+	s, l := fullDP(a, b)
+	if maxError >= 0 && l-s > maxError {
+		return -1, -1
+	}
+	return s, l
+}
+
+func randomFrom(rng *rand.Rand, alphabet string, n int) []byte {
+	s := make([]byte, n)
+	for i := range s {
+		s[i] = alphabet[rng.Intn(len(alphabet))]
+	}
+	return s
+}
+
+func mutateFrom(rng *rand.Rand, alphabet string, s []byte, rate float64) []byte {
+	out := make([]byte, 0, len(s)+8)
+	for _, c := range s {
+		if rng.Float64() >= rate {
+			out = append(out, c)
+			continue
+		}
+		switch rng.Intn(3) {
+		case 0:
+			out = append(out, alphabet[rng.Intn(len(alphabet))])
+		case 1:
+		case 2:
+			out = append(out, c, alphabet[rng.Intn(len(alphabet))])
+		}
+	}
+	return out
+}
+
+var testAlphabets = []struct {
+	name, letters string
+	regular       bool // only IUPAC letters of non-zero mask
+}{
+	{"acgt", "ACGT", true},
+	{"low-complexity", "AC", true},
+	{"mixed-case", "ACGTacgt", true},
+	{"iupac", "ACGTRYSWKMBDHVNUacgtrywsn", true},
+	{"exotic", "ACGT-.EXz*", false},
+}
+
+// TestBandedLCSDifferential checks the scalar reference against the oracle,
+// and the dispatcher (SIMD kernel when built with GOEXPERIMENT=simd) against
+// the scalar reference, for random pairs of unequal lengths and many bands.
+func TestBandedLCSDifferential(t *testing.T) {
+	rng := rand.New(rand.NewSource(7))
+	ws := new(bandedLCSWorkspace)
+	for _, alpha := range testAlphabets {
+		for it := 0; it < 3000; it++ {
+			lA := 1 + rng.Intn(90)
+			a := randomFrom(rng, alpha.letters, lA)
+			var b []byte
+			if rng.Intn(3) > 0 {
+				b = mutateFrom(rng, alpha.letters, a, 0.02+rng.Float64()*0.5)
+			} else {
+				b = randomFrom(rng, alpha.letters, 1+rng.Intn(90))
+			}
+			if len(b) == 0 {
+				continue
+			}
+			if len(a) < len(b) {
+				a, b = b, a
+			}
+			delta := len(a) - len(b)
+			for _, maxError := range []int{-1, 0, 1, 2, 3, 5, 8, 13, 21, 40, 200, delta, delta + 1, delta + 2, delta + 3} {
+				ws, wl := expected(a, b, maxError)
+				ss, sl := bandedLCS(a, b, maxError, new(bandedLCSWorkspace), false)
+				if ss != ws || sl != wl {
+					t.Fatalf("%s scalar maxError=%d a=%q b=%q: got (%d,%d), oracle (%d,%d)",
+						alpha.name, maxError, a, b, ss, sl, ws, wl)
+				}
+				ds, dl := bandedLCS(a, b, maxError, new(bandedLCSWorkspace), haveSIMD)
+				if ds != ss || dl != sl {
+					t.Fatalf("%s dispatch(simd=%v) maxError=%d a=%q b=%q: got (%d,%d), scalar (%d,%d)",
+						alpha.name, haveSIMD, maxError, a, b, ds, dl, ss, sl)
+				}
+			}
+		}
+	}
+	_ = ws
+}
+
+// TestBandedLCSLongPairs covers lengths well above the vector width, with
+// reused workspaces (stale buffers must never leak between calls).
+func TestBandedLCSLongPairs(t *testing.T) {
+	rng := rand.New(rand.NewSource(11))
+	ws := new(bandedLCSWorkspace)
+	for it := 0; it < 60; it++ {
+		a := randomFrom(rng, "ACGT", 150+rng.Intn(500))
+		b := mutateFrom(rng, "ACGT", a, 0.01+rng.Float64()*0.2)
+		if len(a) < len(b) {
+			a, b = b, a
+		}
+		for _, maxError := range []int{-1, 4, 20, 60, 150} {
+			wantS, wantL := expected(a, b, maxError)
+			gotS, gotL := bandedLCS(a, b, maxError, ws, haveSIMD)
+			if gotS != wantS || gotL != wantL {
+				t.Fatalf("maxError=%d lens=%d/%d: got (%d,%d), oracle (%d,%d)",
+					maxError, len(a), len(b), gotS, gotL, wantS, wantL)
+			}
+		}
+	}
+}
+
+// TestBandedLCSMatchesLegacy compares with the historical implementation
+// (FastLCSEGFScoreByte, not end-gap-free) wherever that one is exact: no
+// limit, or a result within the limit.
+func TestBandedLCSMatchesLegacy(t *testing.T) {
+	rng := rand.New(rand.NewSource(3))
+	for it := 0; it < 4000; it++ {
+		a := randomFrom(rng, "ACGTRYN", 1+rng.Intn(120))
+		b := mutateFrom(rng, "ACGTRYN", a, rng.Float64()*0.4)
+		if len(b) == 0 {
+			continue
+		}
+		for _, maxError := range []int{-1, 2, 7, 25} {
+			ls, ll, err := BandedLCS(a, b, maxError)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if ls < 0 {
+				continue
+			}
+			os, ol, _ := FastLCSEGFScoreByte(a, b, maxError, false, nil)
+			if os != ls || ol != ll {
+				t.Fatalf("maxError=%d a=%q b=%q: BandedLCS (%d,%d), legacy (%d,%d)",
+					maxError, a, b, ls, ll, os, ol)
+			}
+		}
+	}
+}
+
+func TestBandedLCSEdgeCases(t *testing.T) {
+	cases := []struct {
+		a, b     string
+		maxError int
+		s, l     int
+	}{
+		{"", "", -1, 0, 0},
+		{"ACGT", "", -1, 0, 4},
+		{"ACGT", "", 3, -1, -1},
+		{"ACGT", "", 4, 0, 4},
+		{"ACGT", "ACGT", 0, 4, 4},
+		{"ACGT", "ACGA", 0, -1, -1},
+		{"ACGT", "ACGA", 1, 3, 4},
+		{"ACGTT", "ACGT", 0, -1, -1}, // length difference above the limit
+		{"ACGTT", "ACGT", 1, 4, 5},
+	}
+	for _, c := range cases {
+		s, l, err := BandedLCS([]byte(c.a), []byte(c.b), c.maxError)
+		if err != nil || s != c.s || l != c.l {
+			t.Errorf("%q/%q maxError=%d: got (%d,%d,%v), want (%d,%d)", c.a, c.b, c.maxError, s, l, err, c.s, c.l)
+		}
+		// the arguments are symmetric
+		s, l, _ = BandedLCS([]byte(c.b), []byte(c.a), c.maxError)
+		if s != c.s || l != c.l {
+			t.Errorf("swapped %q/%q maxError=%d: got (%d,%d), want (%d,%d)", c.b, c.a, c.maxError, s, l, c.s, c.l)
+		}
+	}
+}
+
+func TestBandedLCSTooLong(t *testing.T) {
+	long := make([]byte, MaxBandedLCSLength+1)
+	for i := range long {
+		long[i] = 'A'
+	}
+	if _, _, err := BandedLCS(long, []byte("ACGT"), -1); err != ErrSequenceTooLong {
+		t.Errorf("A too long: err = %v", err)
+	}
+	if _, _, err := BandedLCS([]byte("ACGT"), long, 5); err != ErrSequenceTooLong {
+		t.Errorf("B too long: err = %v", err)
+	}
+	// the maximum length itself is accepted
+	ok := long[:MaxBandedLCSLength]
+	if s, l, err := BandedLCS(ok, ok, 0); err != nil || s != len(ok) || l != len(ok) {
+		t.Errorf("max length: got (%d,%d,%v)", s, l, err)
+	}
+}
+
+// TestBandedLCSSecondPass checks the exactness over many bands, also on
+// low-complexity sequences where the best LCS is not the closest alignment,
+// and reports how often the band had to be widened (never more than once).
+func TestBandedLCSSecondPass(t *testing.T) {
+	rng := rand.New(rand.NewSource(5))
+	for _, alpha := range testAlphabets[:4] {
+		for _, rate := range []float64{0.02, 0.1, 0.3} {
+			ws := new(bandedLCSWorkspace)
+			var total int
+			for it := 0; it < 1500; it++ {
+				a := randomFrom(rng, alpha.letters, 20+rng.Intn(120))
+				b := mutateFrom(rng, alpha.letters, a, rate)
+				if len(b) == 0 {
+					continue
+				}
+				if len(a) < len(b) {
+					a, b = b, a
+				}
+				for _, maxError := range []int{1, 3, 6, 10, 20, 40} {
+					want, wantL := expected(a, b, maxError)
+					got, gotL := bandedLCS(a, b, maxError, ws, haveSIMD)
+					if got != want || gotL != wantL {
+						t.Fatalf("%s maxError=%d a=%q b=%q: got (%d,%d), want (%d,%d)",
+							alpha.name, maxError, a, b, got, gotL, want, wantL)
+					}
+					total++
+				}
+			}
+			t.Logf("%-15s mutation %.2f: %d bounded calls, %d widened (%.1f%%)",
+				alpha.name, rate, total, ws.widened, 100*float64(ws.widened)/float64(total))
+		}
+	}
+}
+
+// TestFastLCSScoreWrapper checks the BioSequence entry point.
+func TestFastLCSScoreWrapper(t *testing.T) {
+	sa := obiseq.NewBioSequence("a", []byte("ACGTACGTAC"), "")
+	sb := obiseq.NewBioSequence("b", []byte("ACGTTCGTAC"), "")
+	if s, l := FastLCSScore(sa, sb, -1, nil); s != 9 || l != 10 {
+		t.Errorf("got (%d,%d), want (9,10)", s, l)
+	}
+	if s, l := FastLCSScore(sa, sb, 0, nil); s != -1 || l != -1 {
+		t.Errorf("got (%d,%d), want (-1,-1)", s, l)
+	}
+}
+
+// TestBandedLCSAtLeast checks the exactness of the bound expressed on the
+// score: the optimum when it reaches minLCS, (-1,-1) otherwise.
+func TestBandedLCSAtLeast(t *testing.T) {
+	rng := rand.New(rand.NewSource(13))
+	var buf BandedLCSBuffer
+	for _, alpha := range testAlphabets {
+		for it := 0; it < 1500; it++ {
+			a := randomFrom(rng, alpha.letters, 1+rng.Intn(100))
+			b := mutateFrom(rng, alpha.letters, a, rng.Float64()*0.5)
+			if len(b) == 0 {
+				continue
+			}
+			os, ol := fullDP(a, b)
+			for _, minLCS := range []int{-3, 0, 1, os - 1, os, os + 1, len(b) / 2, len(b), len(b) + 1} {
+				ws, wl := os, ol
+				if os < minLCS {
+					ws, wl = -1, -1
+				}
+				for _, got := range [][2]int{func() [2]int { s, l, _ := buf.AtLeast(a, b, minLCS); return [2]int{s, l} }(),
+					func() [2]int { s, l, _ := BandedLCSAtLeast(b, a, minLCS); return [2]int{s, l} }()} {
+					if got[0] != ws || got[1] != wl {
+						t.Fatalf("%s minLCS=%d a=%q b=%q: got (%d,%d), want (%d,%d)",
+							alpha.name, minLCS, a, b, got[0], got[1], ws, wl)
+					}
+				}
+			}
+		}
+	}
+	if _, _, err := buf.AtLeast(make([]byte, MaxBandedLCSLength+1), []byte("a"), 0); err != ErrSequenceTooLong {
+		t.Errorf("too long: err = %v", err)
+	}
+}
+
+// TestBandedLCSScoreAtLeast checks the score-only entry point, with sequences
+// long enough for the early stop of the kernel to be exercised.
+func TestBandedLCSScoreAtLeast(t *testing.T) {
+	rng := rand.New(rand.NewSource(17))
+	var buf BandedLCSBuffer
+	check := func(name string, a, b []byte) {
+		os, _ := fullDP(a, b)
+		for _, minLCS := range []int{-1, 0, os - 3, os - 1, os, os + 1, os + 5, len(b) - 1, len(b), (len(b) * 9) / 10, len(b) + 1} {
+			want := os
+			if os < minLCS {
+				want = -1
+			}
+			got, err := buf.ScoreAtLeast(a, b, minLCS)
+			if err != nil || got != want {
+				t.Fatalf("%s minLCS=%d a=%q b=%q: got %d (%v), want %d", name, minLCS, a, b, got, err, want)
+			}
+		}
+	}
+	for _, alpha := range testAlphabets {
+		for it := 0; it < 800; it++ {
+			a := randomFrom(rng, alpha.letters, 1+rng.Intn(60))
+			b := mutateFrom(rng, alpha.letters, a, rng.Float64()*0.6)
+			if len(b) > 0 {
+				check(alpha.name, a, b)
+			}
+		}
+	}
+	for it := 0; it < 150; it++ {
+		a := randomFrom(rng, "ACGT", 100+rng.Intn(400))
+		b := mutateFrom(rng, "ACGT", a, rng.Float64()*0.3)
+		if it%3 == 0 {
+			b = randomFrom(rng, "ACGT", 50+rng.Intn(300))
+		}
+		if len(b) > 0 {
+			check("long", a, b)
+		}
+	}
+}

pkg/obialign/lcsmin.go

@@ -0,0 +1,103 @@
+package obialign
+
+// BandedLCSBuffer is a working memory that can be reused from one alignment to
+// the next to avoid any allocation. It must not be shared between goroutines.
+type BandedLCSBuffer struct {
+	ws bandedLCSWorkspace
+}
+
+// AtLeast is BandedLCSAtLeast on the buffer.
+func (b *BandedLCSBuffer) AtLeast(seqA, seqB []byte, minLCS int) (lcs, alilen int, err error) {
+	return atLeast(seqA, seqB, minLCS, &b.ws)
+}
+
+func atLeast(seqA, seqB []byte, minLCS int, ws *bandedLCSWorkspace) (lcs, alilen int, err error) {
+	if len(seqA) > MaxBandedLCSLength || len(seqB) > MaxBandedLCSLength {
+		return -1, -1, ErrSequenceTooLong
+	}
+	if len(seqA) < len(seqB) {
+		seqA, seqB = seqB, seqA
+	}
+	lcs, alilen = bandedLCSAtLeast(seqA, seqB, minLCS, ws, haveSIMD)
+	return lcs, alilen, nil
+}
+
+// BandedLCSAtLeast computes the LCS score of two nucleotide sequences and the
+// length of the shortest alignment achieving it, as BandedLCS does, but with
+// the bound expressed on the score: the result is the exact optimum when its
+// score is at least minLCS, and (-1, -1) otherwise.
+//
+// The DP only visits the diagonals an alignment of score minLCS can use (one
+// whose path makes at most lB-minLCS gaps on one side and lA-minLCS on the
+// other), so the cost decreases as minLCS grows. This is exact: every alignment
+// of score >= minLCS lies in that band, hence so does the optimum when it
+// reaches minLCS, and when the best path of the band is below minLCS so is
+// the global optimum.
+func BandedLCSAtLeast(seqA, seqB []byte, minLCS int) (lcs, alilen int, err error) {
+	ws := bandedLCSPool.Get().(*bandedLCSWorkspace)
+	lcs, alilen, err = atLeast(seqA, seqB, minLCS, ws)
+	bandedLCSPool.Put(ws)
+	return lcs, alilen, err
+}
+
+// bandedLCSAtLeast expects len(a) >= len(b), both within MaxBandedLCSLength.
+func bandedLCSAtLeast(a, b []byte, minLCS int, ws *bandedLCSWorkspace, useSIMD bool) (int, int) {
+	lA, lB := len(a), len(b)
+	minLCS = max(minLCS, 0)
+	if minLCS > lB {
+		return -1, -1
+	}
+	if lB == 0 {
+		return 0, lA
+	}
+	simdOK := useSIMD && allRegularIUPAC(a) && allRegularIUPAC(b)
+	kmin, kmax := -(lB - minLCS), lA-minLCS
+
+	if !simdOK {
+		lcs, alilen := ws.align(a, b, kmin, kmax, false)
+		if lcs < minLCS {
+			return -1, -1
+		}
+		return lcs, alilen
+	}
+
+	// The score alone is much cheaper than the score with the length of the
+	// alignment: the latter is only computed when the pair reaches minLCS. With
+	// no minimum there is nothing to reject, and the pass would be wasted.
+	if minLCS > 0 && lcsScoreSIMD(a, b, kmin, kmax, minLCS, ws) < minLCS {
+		return -1, -1
+	}
+	encodeBases(ws, a, b)
+	return ws.align(a, b, kmin, kmax, true)
+}
+
+// ScoreAtLeast computes the LCS score of two nucleotide sequences when it is
+// at least minLCS, and returns -1 otherwise. It is much cheaper than AtLeast,
+// which also computes the length of the shortest alignment.
+func (b *BandedLCSBuffer) ScoreAtLeast(seqA, seqB []byte, minLCS int) (int, error) {
+	if len(seqA) > MaxBandedLCSLength || len(seqB) > MaxBandedLCSLength {
+		return -1, ErrSequenceTooLong
+	}
+	if len(seqA) < len(seqB) {
+		seqA, seqB = seqB, seqA
+	}
+	lA, lB := len(seqA), len(seqB)
+	minLCS = max(minLCS, 0)
+	if minLCS > lB {
+		return -1, nil
+	}
+	if lB == 0 {
+		return 0, nil
+	}
+	kmin, kmax := -(lB - minLCS), lA-minLCS
+	var lcs int
+	if haveSIMD && allRegularIUPAC(seqA) && allRegularIUPAC(seqB) {
+		lcs = lcsScoreSIMD(seqA, seqB, kmin, kmax, minLCS, &b.ws)
+	} else {
+		lcs, _ = b.ws.align(seqA, seqB, kmin, kmax, false)
+	}
+	if lcs < minLCS {
+		return -1, nil
+	}
+	return lcs, nil
+}

pkg/obialign/lcsscore_simd.go

@@ -0,0 +1,156 @@
+//go:build goexperiment.simd
+
+package obialign
+
+import "simd"
+
+func simdLanes16() int {
+	var probe simd.Uint16s
+	return probe.Len()
+}
+
+func growUint16(s []uint16, n int) []uint16 {
+	if cap(s) < n {
+		return make([]uint16, n)
+	}
+	return s[:n]
+}
+
+func encodeIUPAC16(dst []uint16, s []byte, rev bool) {
+	n := len(s)
+	clear(dst)
+	if rev {
+		for x, c := range s {
+			dst[n-1-x] = uint16(iupacMask[c])
+		}
+		return
+	}
+	for x, c := range s {
+		dst[x] = uint16(iupacMask[c])
+	}
+}
+
+// pruneEvery is the number of antidiagonals between two tests of the upper
+// bound that stops a hopeless alignment.
+const pruneEvery = 8
+
+// lcsScoreSIMD runs the same banded DP as bandedLCSSIMD on the score only,
+// with 16-bit lanes (twice the cells per instruction):
+//
+//	cell(i,j) = max(diag + match, up, left)
+//
+// A cell outside the band is 0, which never beats a real cell. The score of a
+// path ending at the cell (i, j) of an antidiagonal cannot exceed the score of
+// the cell plus min(lB-i, lA-j), the characters left to pair. Every path
+// crosses each antidiagonal inside the window, so when the best score of the
+// window plus the largest of these remainders is under minLCS the final score
+// is under minLCS: -1 is returned without computing the rest of the matrix.
+//
+// Requires lA >= lB >= 1, kmin <= 0 <= kmax, and bases with non-zero masks.
+// Returns the best score of the band, or -1 if it is proven to be under
+// minLCS.
+func lcsScoreSIMD(a, b []byte, kmin, kmax, minLCS int, ws *bandedLCSWorkspace) int {
+	lA, lB := len(a), len(b)
+	vl := simdLanes16()
+
+	ws.aRev16 = growUint16(ws.aRev16, lA+vl)
+	ws.bCode16 = growUint16(ws.bCode16, lB+vl)
+	encodeIUPAC16(ws.aRev16, a, true)
+	encodeIUPAC16(ws.bCode16, b, false)
+	aRev, bCode := ws.aRev16, ws.bCode16
+
+	size := lB + 2*vl + 2
+	for r := range ws.rows16 {
+		ws.rows16[r] = growUint16(ws.rows16[r], size)
+		clear(ws.rows16[r])
+	}
+	p2, p1, cur := ws.rows16[0], ws.rows16[1], ws.rows16[2]
+
+	// the first row and the first column hold a score of 0: nothing to set
+
+	zero := simd.BroadcastUint16s(0)
+	one := simd.BroadcastUint16s(1)
+
+	for t := 2; t <= lA+lB; t++ {
+		lo := max(1, t-lA, (t-kmax+1)>>1)
+		hi := min(lB, t-1, (t-kmin)>>1)
+
+		for i := lo; i <= hi; i += vl {
+			ca := simd.LoadUint16s(aRev[lA-t+i:])
+			cb := simd.LoadUint16s(bCode[i-1:])
+			inc := one.Masked(ca.And(cb).NotEqual(zero))
+			d := simd.LoadUint16s(p2[i-1:]).Add(inc)
+			u := simd.LoadUint16s(p1[i-1:])
+			l := simd.LoadUint16s(p1[i:])
+			d.Max(u).Max(l).Store(cur[i:])
+		}
+
+		zero.Store(cur[hi+1:])
+		if lo > 1 {
+			cur[lo-1] = 0
+		}
+
+		// A diagonal move skips an antidiagonal, but a path always visits
+		// one of two consecutive antidiagonals.
+		if t%pruneEvery == 0 && minLCS > 0 &&
+			scoreBound(p1, t-1, lA, lB, kmin, kmax) < minLCS &&
+			scoreBound(cur, t, lA, lB, kmin, kmax) < minLCS {
+			return -1
+		}
+
+		p2, p1, cur = p1, cur, p2
+	}
+	return int(p1[lB])
+}
+
+// iota16 is 0, 1, 2... to build the row index of the lanes.
+var iota16 = func() (t [64]uint16) {
+	for i := range t {
+		t[i] = uint16(i)
+	}
+	return t
+}()
+
+// scoreBound is an upper bound of the final score of the paths going through
+// the antidiagonal t, whose scores are in row. A path through the cell (i, j)
+// scores at most the score of the cell plus min(lB-i, lA-j), the characters
+// left to pair, so the bound is the maximum of this sum on the cells of the
+// window. The sum is computed by vectors on the full chunks of the window.
+func scoreBound(row []uint16, t, lA, lB, kmin, kmax int) int {
+	lo := max(1, t-lA, (t-kmax+1)>>1)
+	hi := min(lB, t-1, (t-kmin)>>1)
+
+	bound := 0
+	i := lo
+	if hi-lo >= simdLanes16() {
+		vl := simdLanes16()
+		idx := simd.LoadUint16s(iota16[:]).Add(simd.BroadcastUint16s(uint16(lo)))
+		step := simd.BroadcastUint16s(uint16(vl))
+		remB := simd.BroadcastUint16s(uint16(lB))
+		// lA-t+i is not negative on the window; the 16-bit arithmetic is modular
+		offA := simd.BroadcastUint16s(uint16(lA - t))
+		acc := simd.BroadcastUint16s(0)
+		for ; i+vl-1 <= hi; i += vl {
+			left := remB.Sub(idx).Min(idx.Add(offA))
+			acc = acc.Max(simd.LoadUint16s(row[i:]).Add(left))
+			idx = idx.Add(step)
+		}
+		var lanes [64]uint16
+		acc.Store(lanes[:])
+		for _, v := range lanes[:vl] {
+			bound = max(bound, int(v))
+		}
+	}
+	for ; i <= hi; i++ {
+		bound = max(bound, int(row[i])+min(lB-i, lA-t+i))
+	}
+
+	// the paths running along the first row or the first column have a score of 0
+	if t <= lA && t <= kmax {
+		bound = max(bound, min(lB, lA-t))
+	}
+	if t <= lB && -t >= kmin {
+		bound = max(bound, min(lB-t, lA))
+	}
+	return bound
+}

pkg/obialign/locatepattern.go

@@ -28,10 +28,18 @@ func buffIndex(i, j, width int) int {
 //
 // The function returns the start and end positions of the best
 // match, as well as the number of errors in the best match.
+//
+// When the sequence is too short relative to the pattern for the
+// backtracking to reconstruct a valid alignment (e.g. the pattern
+// is longer than the sequence, or the match sits too close to a
+// sequence boundary), no reliable position can be computed. In that
+// case the function returns the sentinel (-1, -1, -1) instead of a
+// guessed, potentially wrong, position: callers must treat this as
+// "no match" rather than use the returned coordinates.
 func LocatePattern(id string, pattern, sequence []byte) (int, int, int) {
 
-	if len(pattern) >= len(sequence) {
+	if len(sequence) == 0 {
-		log.Panicf("Sequence %s:Pattern %s must be shorter than sequence %s", id, pattern, sequence)
+		log.Panicf("Sequence %s:Pattern %s must not be empty", id, pattern)
 	}
 
 	// Pattern spreads over the columns
@@ -158,5 +166,18 @@ func LocatePattern(id string, pattern, sequence []byte) (int, int, int) {
 	// obilog.Warnf("from : %d to: %d error: %d match: %v",
 	// 	i, end+1, -buffer[buffIndex(len(sequence)-1, len(pattern)-1, width)],
 	// 	string(sequence[i:(end+1)]))
+
+	if i < 0 || end == -1 {
+		// i < 0: the backtracking ran off the start of the sequence
+		// without fully consuming the pattern.
+		// end == -1: the backtracking loop never ran at all (e.g. a
+		// single-base pattern, jmax == 0), so no alignment boundary
+		// was ever established.
+		// Either way, no valid alignment exists for this (pattern,
+		// sequence) pair: signal it explicitly instead of returning
+		// an out-of-bounds or uncomputed position.
+		return -1, -1, -1
+	}
+
 	return i, end + 1, -buffer[buffIndex(len(sequence)-1, len(pattern)-1, width)]
 }

pkg/obialign/locatepattern_test.go

@@ -0,0 +1,123 @@
+package obialign
+
+import (
+	"math/rand"
+	"testing"
+)
+
+func TestLocatePatternNormal(t *testing.T) {
+	// Pattern fully and exactly present in the middle of a longer sequence.
+	start, end, nerr := LocatePattern("id", []byte("ACGT"), []byte("TTTTACGTTTTT"))
+	if start != 4 || end != 8 || nerr != 0 {
+		t.Errorf("got start=%d end=%d nerr=%d, want start=4 end=8 nerr=0", start, end, nerr)
+	}
+}
+
+func TestLocatePatternOneMismatch(t *testing.T) {
+	start, end, nerr := LocatePattern("id", []byte("ACGT"), []byte("TTTTACTTTTTT"))
+	if nerr != 1 {
+		t.Errorf("got nerr=%d, want 1 (start=%d end=%d)", nerr, start, end)
+	}
+}
+
+// The real-world case that used to panic: pattern longer than the sequence
+// fragment extracted for indel relocation.
+func TestLocatePatternPatternLongerThanSequence(t *testing.T) {
+	start, end, nerr := LocatePattern("id",
+		[]byte("GGGCAATCCTGAGCCAAATC"),
+		[]byte("tcctgagccaaatcacgtt"))
+
+	if start != -1 || end != -1 || nerr != -1 {
+		t.Errorf("got start=%d end=%d nerr=%d, want the (-1,-1,-1) sentinel", start, end, nerr)
+	}
+}
+
+func TestLocatePatternSequenceLengthOne(t *testing.T) {
+	start, end, nerr := LocatePattern("id", []byte("AB"), []byte("A"))
+
+	if start < 0 || end < 0 || nerr < 0 {
+		t.Fatalf("got start=%d end=%d nerr=%d, expected a valid (non-sentinel) result", start, end, nerr)
+	}
+	if start != 0 || end != 1 || nerr != 1 {
+		t.Errorf("got start=%d end=%d nerr=%d, want start=0 end=1 nerr=1", start, end, nerr)
+	}
+}
+
+// A pattern much longer than the sequence can still yield a mathematically
+// valid (in-bounds) alignment: the extra pattern length is absorbed as gaps,
+// driving the error count high enough that the caller's maxerr threshold
+// rejects it. The function itself must still return consistent bounds.
+func TestLocatePatternPatternMuchLongerThanSequence(t *testing.T) {
+	start, end, nerr := LocatePattern("id", []byte("ACGTACGTACGTACGTACGT"), []byte("ACG"))
+
+	isSentinel := start == -1 && end == -1 && nerr == -1
+	isValid := start >= 0 && end > start && end <= 3 && nerr >= 0
+
+	if !isSentinel && !isValid {
+		t.Errorf("got start=%d end=%d nerr=%d, want either the sentinel or consistent in-bounds values", start, end, nerr)
+	}
+}
+
+func TestLocatePatternNeverReturnsOutOfBounds(t *testing.T) {
+	patterns := []string{"A", "AC", "ACG", "ACGT", "ACGTA", "ACGTAC", "ACGTACG", "ACGTACGT"}
+	sequences := []string{"A", "AC", "ACG", "ACGT", "ACGTA", "ACGTAC", "ACGTACG", "ACGTACGT"}
+
+	for _, p := range patterns {
+		for _, s := range sequences {
+			start, end, nerr := LocatePattern("id", []byte(p), []byte(s))
+
+			if start == -1 && end == -1 && nerr == -1 {
+				// Explicit "no reliable match" sentinel: always acceptable.
+				continue
+			}
+
+			if start < 0 || end < 0 || start >= end || end > len(s) || nerr < 0 {
+				t.Errorf("pattern=%q sequence=%q -> start=%d end=%d nerr=%d is out of bounds / inconsistent",
+					p, s, start, end, nerr)
+			}
+		}
+	}
+}
+
+// Randomized property test over a wide range of pattern/sequence length
+// combinations, including pattern >= sequence, to make sure the function
+// never panics and never returns anything but the sentinel or fully
+// consistent, in-bounds coordinates.
+func TestLocatePatternRandomizedNeverInvalid(t *testing.T) {
+	const bases = "ACGT"
+	rng := rand.New(rand.NewSource(42))
+
+	randSeq := func(n int) []byte {
+		b := make([]byte, n)
+		for i := range b {
+			b[i] = bases[rng.Intn(len(bases))]
+		}
+		return b
+	}
+
+	for trial := 0; trial < 5000; trial++ {
+		patLen := 1 + rng.Intn(15)
+		seqLen := 1 + rng.Intn(15)
+
+		pattern := randSeq(patLen)
+		sequence := randSeq(seqLen)
+
+		func() {
+			defer func() {
+				if r := recover(); r != nil {
+					t.Fatalf("panic for pattern=%q sequence=%q: %v", pattern, sequence, r)
+				}
+			}()
+
+			start, end, nerr := LocatePattern("id", pattern, sequence)
+
+			isSentinel := start == -1 && end == -1 && nerr == -1
+			isValid := start >= 0 && end > start && end <= len(sequence) && nerr >= 0
+
+			if !isSentinel && !isValid {
+				t.Errorf("pattern=%q sequence=%q -> start=%d end=%d nerr=%d is neither the sentinel nor consistent",
+					pattern, sequence, start, end, nerr)
+			}
+		}()
+	}
+}

pkg/obiapat/pattern.go

@@ -373,6 +373,7 @@ func (pattern ApatPattern) BestMatch(sequence ApatSequence, begin, length int) (
 
 	cpattern := (*[1 << 30]byte)(unsafe.Pointer(pattern.pointer.pointer.cpat))
 	frg := sequence.pointer.reference.Sequence()[start:end]
+	fragStart := start
 
 	log.Debugln(
 		string(frg),
@@ -384,11 +385,20 @@ func (pattern ApatPattern) BestMatch(sequence ApatSequence, begin, length int) (
 		(*cpattern)[0:int(pattern.pointer.pointer.patlen)],
 		frg)
 
-	// olderr := m[2]
+	if from < 0 {
+		// obialign.LocatePattern could not reconstruct a reliable
+		// alignment (e.g. the fragment is too short relative to the
+		// pattern). Reporting a guessed position would risk placing
+		// the primer boundary incorrectly, so treat it as no match
+		// at all rather than falling back to an unrefined position.
+		matched = false
+		log.Debugln("No reliable indel relocation, discarding match", sequence.pointer.reference.Id())
+		return
+	}
 
 	nerr = score
-	start = start + from
+	start = fragStart + from
-	end = start + to
+	end = fragStart + to
 	log.Debugf("BestMatch on %s : score=%d [%d..%d]", sequence.pointer.reference.Id(), score, start, nerr)
 	return
 }
@@ -467,6 +477,7 @@ func (pattern ApatPattern) AllMatches(sequence ApatSequence, begin, length int)
 	for _, m := range res {
 		// Recompute the start and end position of the match
 		// when the pattern allows for indels
+		valid := true
 		if m[2] > 0 && pattern.pointer.pointer.hasIndel {
 			// obilog.Warnf("Locating indel on sequence %s[%s]", sequence.pointer.reference.Id(), pattern.String())
 			start := m[0] - m[2]*2
@@ -485,16 +496,26 @@ func (pattern ApatPattern) AllMatches(sequence ApatSequence, begin, length int)
 				(*cpattern)[0:int(pattern.pointer.pointer.patlen)],
 				frg)
 
-			// olderr := m[2]
+			if pb < 0 {
-			m[2] = score
+				// obialign.LocatePattern could not reconstruct a
-			m[0] = start + pb
+				// reliable alignment (e.g. the match sits too close
-			m[1] = start + pe
+				// to a sequence end for the fragment to be usable).
+				// Reporting a guessed position risks placing the
+				// primer boundary incorrectly, so drop the match
+				// entirely instead of keeping an unrefined guess.
+				valid = false
+			} else {
+				// olderr := m[2]
+				m[2] = score
+				m[0] = start + pb
+				m[1] = start + pe
 
-			// obilog.Warnf("seq[%d@%d:%d] %d: %s %d - %s:%s:%s", i, m[0], m[1], olderr, sequence.pointer.reference.Id(), score,
+				// obilog.Warnf("seq[%d@%d:%d] %d: %s %d - %s:%s:%s", i, m[0], m[1], olderr, sequence.pointer.reference.Id(), score,
-			// 	frg, (*cpattern)[0:int(pattern.pointer.pointer.patlen)], sequence.pointer.reference.Sequence()[m[0]:m[1]])
+				// 	frg, (*cpattern)[0:int(pattern.pointer.pointer.patlen)], sequence.pointer.reference.Sequence()[m[0]:m[1]])
+			}
 		}
 
-		if int(pattern.pointer.pointer.maxerr) >= m[2] {
+		if valid && int(pattern.pointer.pointer.maxerr) >= m[2] {
 			res[j] = m
 			j++
 		}

pkg/obichunk/chunk_on_disk.go

@@ -110,6 +110,7 @@ func ISequenceChunkOnDisk(iterator obiiter.IBioSequence,
 	log.Infof("Data splitted over %d batches", nbatch)
 
 	go func() {
+		localClassifier := uniqueClassifier.Clone()
 
 		for order, file := range fileNames {
 			iseq, err := obiformats.ReadSequencesFromFile(file)
@@ -121,7 +122,7 @@ func ISequenceChunkOnDisk(iterator obiiter.IBioSequence,
 			if dereplicate {
 				u := make(map[string]*obiseq.BioSequence)
 				var source string
-				uniqueClassifier.Reset()
+				localClassifier.Reset()
 
 				for iseq.Next() {
 					batch := iseq.Get()
@@ -129,8 +130,8 @@ func ISequenceChunkOnDisk(iterator obiiter.IBioSequence,
 
 					for _, seq := range batch.Slice() {
 						// Use composite key: sequence + categories
-						code := uniqueClassifier.Code(seq)
+						code := localClassifier.Code(seq)
-						key := uniqueClassifier.Value(code)
+						key := localClassifier.Value(code)
 						prev, ok := u[key]
 						if ok {
 							prev.Merge(seq, na, true, statsOn)

pkg/obicorazick/worker.go

@@ -27,22 +27,26 @@ func AhoCorazickWorker(slot string, patterns []string) obiseq.SeqWorker {
 	npar := min(obidefault.ParallelWorkers(), nmatcher)
 	mutex.Add(npar)
 
-	pbopt := make([]progressbar.Option, 0, 5)
+	var bar *progressbar.ProgressBar
-	pbopt = append(pbopt,
+	if obidefault.ProgressBar() {
-		progressbar.OptionSetWriter(os.Stderr),
+		pbopt := make([]progressbar.Option, 0, 5)
-		progressbar.OptionSetWidth(15),
+		pbopt = append(pbopt,
-		progressbar.OptionShowCount(),
+			progressbar.OptionSetWriter(os.Stderr),
-		progressbar.OptionShowIts(),
+			progressbar.OptionSetWidth(15),
-		progressbar.OptionSetDescription("Building AhoCorasick matcher..."),
+			progressbar.OptionShowCount(),
-	)
+			progressbar.OptionShowIts(),
+			progressbar.OptionSetDescription("Building AhoCorasick matcher..."),
+		)
 
-	bar := progressbar.NewOptions(nmatcher, pbopt...)
+		bar = progressbar.NewOptions(nmatcher, pbopt...)
-	bar.Add(0)
+	}
 
 	builder := func() {
 		for i := range ieme  {
 		matchers[i] = ahocorasick.CompileStrings(patterns[i*sizebatch:min((i+1)*sizebatch,len(patterns))])
-		bar.Add(1)
+		if bar != nil {
+			bar.Add(1)
+		}
 		}
 		mutex.Done()
 	}

pkg/obidefault/batch.go

@@ -1,6 +1,12 @@
 package obidefault
 
-var _BatchSize = 2000
+// _BatchSize is the minimum number of sequences per batch (floor).
+// Used as the minSeqs argument to RebatchBySize.
+var _BatchSize = 1
+
+// _BatchSizeMax is the maximum number of sequences per batch (ceiling).
+// A batch is flushed when this count is reached regardless of memory usage.
+var _BatchSizeMax = 2000
 
 // SetBatchSize sets the size of the sequence batches.
 //
@@ -24,3 +30,42 @@ func BatchSize() int {
 func BatchSizePtr() *int {
 	return &_BatchSize
 }
+
+// BatchSizeMax returns the maximum number of sequences per batch.
+func BatchSizeMax() int {
+	return _BatchSizeMax
+}
+
+func BatchSizeMaxPtr() *int {
+	return &_BatchSizeMax
+}
+
+// _BatchMem holds the maximum cumulative memory (in bytes) per batch when
+// memory-based batching is requested. A value of 0 disables memory-based
+// batching and falls back to count-based batching.
+var _BatchMem = 128 * 1024 * 1024 // 128 MB default; set to 0 to disable
+var _BatchMemStr = ""
+
+// SetBatchMem sets the memory budget per batch in bytes.
+func SetBatchMem(n int) {
+	_BatchMem = n
+}
+
+// BatchMem returns the current memory budget per batch in bytes.
+// A value of 0 means memory-based batching is disabled.
+func BatchMem() int {
+	return _BatchMem
+}
+
+func BatchMemPtr() *int {
+	return &_BatchMem
+}
+
+// BatchMemStr returns the raw --batch-mem string value as provided on the CLI.
+func BatchMemStr() string {
+	return _BatchMemStr
+}
+
+func BatchMemStrPtr() *string {
+	return &_BatchMemStr
+}

pkg/obidefault/progressbar.go

@@ -0,0 +1,40 @@
+package obidefault
+
+import "os"
+
+var __no_progress_bar__ = false
+
+func ProgressBar() bool {
+	return !__no_progress_bar__
+}
+
+func NoProgressBar() bool {
+	return __no_progress_bar__
+}
+
+func SetNoProgressBar(b bool) {
+	__no_progress_bar__ = b
+}
+
+func NoProgressBarPtr() *bool {
+	return &__no_progress_bar__
+}
+
+// ProgressBarVisible tells whether a progress bar can be displayed: it is not
+// disabled by --no-progressbar, the standard error output is a terminal and
+// the standard output is not piped to another command.
+func ProgressBarVisible() bool {
+	if !ProgressBar() {
+		return false
+	}
+
+	if e, err := os.Stderr.Stat(); err != nil || (e.Mode()&os.ModeCharDevice) != os.ModeCharDevice {
+		return false
+	}
+
+	if o, err := os.Stdout.Stat(); err == nil && (o.Mode()&os.ModeNamedPipe) == os.ModeNamedPipe {
+		return false
+	}
+
+	return true
+}

pkg/obiformats/batch_of_files_reader.go

@@ -48,6 +48,14 @@ func ReadSequencesBatchFromFiles(filenames []string,
 
 				log.Printf("Start reading of file : %s", filename)
 
+				// The batches of a file reach us in any order (parallel parsing
+				// of the headers). They are numbered here in arrival order, so
+				// the order of the file must be restored first, unless the
+				// order of the data is not wanted (several concurrent readers).
+				if concurrent_readers == 1 {
+					iter = iter.SortBatches()
+				}
+
 				for iter.Next() {
 					batch := iter.Get()
 					batchiter.Push(batch.Reorder(nextCounter()))

pkg/obiformats/batch_of_files_reader_test.go

@@ -0,0 +1,38 @@
+package obiformats
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// With a single reader, the sequences of several files must come out in the
+// order of the files, whatever the order in which the batches are parsed.
+func TestReadSequencesBatchFromFilesKeepsOrder(t *testing.T) {
+	dir := t.TempDir()
+	var names []string
+	for _, prefix := range []string{"a", "b"} {
+		var sb strings.Builder
+		for i := 0; i < 25000; i++ {
+			fmt.Fprintf(&sb, ">%s%07d {\"k\":%d}\n%s\n", prefix, i, i, strings.Repeat("acgt", 40))
+		}
+		name := filepath.Join(dir, prefix+".fasta")
+		if err := os.WriteFile(name, []byte(sb.String()), 0o600); err != nil {
+			t.Fatal(err)
+		}
+		names = append(names, name)
+	}
+
+	it := ReadSequencesBatchFromFiles(names, nil, 1)
+	_, db := it.Load()
+	if len(db) != 50000 {
+		t.Fatalf("expected 50000 sequences, got %d", len(db))
+	}
+	for i := 1; i < len(db); i++ {
+		if db[i-1].Id() >= db[i].Id() {
+			t.Fatalf("out of order at %d: %s then %s", i, db[i-1].Id(), db[i].Id())
+		}
+	}
+}

pkg/obiformats/embl_read.go

@@ -161,6 +161,149 @@ func EmblChunkParser(withFeatureTable, UtoT bool) func(string, io.Reader) (obise
 	return parser
 }
 
+// extractEmblSeq scans the sequence section of an EMBL record directly on the
+// rope. EMBL sequence lines start with 5 spaces followed by bases in groups of
+// 10, separated by spaces, with a position number at the end. The section ends
+// with "//".
+func (s *ropeScanner) extractEmblSeq(dest []byte, UtoT bool) []byte {
+	// We use ReadLine and scan each line for bases (skip digits, spaces, newlines).
+	for {
+		line := s.ReadLine()
+		if line == nil {
+			break
+		}
+		if len(line) >= 2 && line[0] == '/' && line[1] == '/' {
+			break
+		}
+		// Lines start with 5 spaces; bases follow separated by single spaces.
+		// Digits at the end are the position counter — skip them.
+		// Simplest: take every byte that is a letter.
+		for _, b := range line {
+			if b >= 'A' && b <= 'Z' {
+				b += 'a' - 'A'
+			}
+			if UtoT && b == 'u' {
+				b = 't'
+			}
+			if b >= 'a' && b <= 'z' {
+				dest = append(dest, b)
+			}
+		}
+	}
+	return dest
+}
+
+// EmblChunkParserRope parses an EMBL chunk directly from a rope without Pack().
+func EmblChunkParserRope(source string, rope *PieceOfChunk, withFeatureTable, UtoT bool) (obiseq.BioSequenceSlice, error) {
+	scanner := newRopeScanner(rope)
+	sequences := obiseq.MakeBioSequenceSlice(100)[:0]
+
+	var id string
+	var scientificName string
+	defBytes := make([]byte, 0, 256)
+	featBytes := make([]byte, 0, 1024)
+	var taxid int
+	inSeq := false
+
+	for {
+		line := scanner.ReadLine()
+		if line == nil {
+			break
+		}
+
+		if inSeq {
+			// Should not happen — extractEmblSeq consumed up to "//"
+			inSeq = false
+			continue
+		}
+
+		switch {
+		case bytes.HasPrefix(line, []byte("ID   ")):
+			id = string(bytes.SplitN(line[5:], []byte(";"), 2)[0])
+		case bytes.HasPrefix(line, []byte("OS   ")):
+			scientificName = string(bytes.TrimSpace(line[5:]))
+		case bytes.HasPrefix(line, []byte("DE   ")):
+			if len(defBytes) > 0 {
+				defBytes = append(defBytes, ' ')
+			}
+			defBytes = append(defBytes, bytes.TrimSpace(line[5:])...)
+		case withFeatureTable && bytes.HasPrefix(line, []byte("FH   ")):
+			featBytes = append(featBytes, line...)
+		case withFeatureTable && bytes.Equal(line, []byte("FH")):
+			featBytes = append(featBytes, '\n')
+			featBytes = append(featBytes, line...)
+		case bytes.HasPrefix(line, []byte("FT   ")):
+			if withFeatureTable {
+				featBytes = append(featBytes, '\n')
+				featBytes = append(featBytes, line...)
+			}
+			if bytes.HasPrefix(line, []byte(`FT                   /db_xref="taxon:`)) {
+				rest := line[37:]
+				end := bytes.IndexByte(rest, '"')
+				if end > 0 {
+					taxid, _ = strconv.Atoi(string(rest[:end]))
+				}
+			}
+		case bytes.HasPrefix(line, []byte("     ")):
+			// First sequence line: extract all bases via extractEmblSeq,
+			// which also consumes this line's remaining content.
+			// But ReadLine already consumed this line — we need to process it
+			// plus subsequent lines. Process this line inline then call helper.
+			seqDest := make([]byte, 0, 4096)
+			for _, b := range line {
+				if b >= 'A' && b <= 'Z' {
+					b += 'a' - 'A'
+				}
+				if UtoT && b == 'u' {
+					b = 't'
+				}
+				if b >= 'a' && b <= 'z' {
+					seqDest = append(seqDest, b)
+				}
+			}
+			seqDest = scanner.extractEmblSeq(seqDest, UtoT)
+
+			seq := obiseq.NewBioSequenceOwning(id, seqDest, string(defBytes))
+			seq.SetSource(source)
+			if withFeatureTable {
+				seq.SetFeatures(featBytes)
+			}
+			annot := seq.Annotations()
+			annot["scientific_name"] = scientificName
+			annot["taxid"] = taxid
+			sequences = append(sequences, seq)
+
+			// Reset state
+			id = ""
+			scientificName = ""
+			defBytes = defBytes[:0]
+			featBytes = featBytes[:0]
+			taxid = 1
+
+		case bytes.Equal(line, []byte("//")):
+			// record ended without SQ/sequence section (e.g. WGS entries)
+			if id != "" {
+				seq := obiseq.NewBioSequenceOwning(id, []byte{}, string(defBytes))
+				seq.SetSource(source)
+				if withFeatureTable {
+					seq.SetFeatures(featBytes)
+				}
+				annot := seq.Annotations()
+				annot["scientific_name"] = scientificName
+				annot["taxid"] = taxid
+				sequences = append(sequences, seq)
+			}
+			id = ""
+			scientificName = ""
+			defBytes = defBytes[:0]
+			featBytes = featBytes[:0]
+			taxid = 1
+		}
+	}
+
+	return sequences, nil
+}
+
 func _ParseEmblFile(
 	input ChannelFileChunk,
 	out obiiter.IBioSequence,
@@ -171,7 +314,14 @@ func _ParseEmblFile(
 
 	for chunks := range input {
 		order := chunks.Order
-		sequences, err := parser(chunks.Source, chunks.Raw)
+		var sequences obiseq.BioSequenceSlice
+		var err error
+
+		if chunks.Rope != nil {
+			sequences, err = EmblChunkParserRope(chunks.Source, chunks.Rope, withFeatureTable, UtoT)
+		} else {
+			sequences, err = parser(chunks.Source, chunks.Raw)
+		}
 
 		if err != nil {
 			log.Fatalf("%s : Cannot parse the embl file : %v", chunks.Source, err)
@@ -196,6 +346,7 @@ func ReadEMBL(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, er
 		1024*1024*128,
 		EndOfLastFlatFileEntry,
 		"\nID   ",
+		false,
 	)
 
 	newIter := obiiter.MakeIBioSequence()

pkg/obiformats/fastaseq_read.go

@@ -209,28 +209,121 @@ func FastaChunkParser(UtoT bool) func(string, io.Reader) (obiseq.BioSequenceSlic
 	return parser
 }
 
+// extractFastaSeq scans sequence bytes from the rope directly into dest,
+// appending valid nucleotide characters and skipping whitespace.
+// Stops when '>' is found at the start of a line (next record) or at EOF.
+// Returns (dest with appended bases, hasMore).
+// hasMore=true means scanner is now positioned at '>' of the next record.
+func (s *ropeScanner) extractFastaSeq(dest []byte, UtoT bool) ([]byte, bool) {
+	lineStart := true
+
+	for s.current != nil {
+		data := s.current.data[s.pos:]
+		for i, b := range data {
+			if lineStart && b == '>' {
+				s.pos += i
+				if s.pos >= len(s.current.data) {
+					s.current = s.current.Next()
+					s.pos = 0
+				}
+				return dest, true
+			}
+			if b == '\n' || b == '\r' {
+				lineStart = true
+				continue
+			}
+			lineStart = false
+			if b == ' ' || b == '\t' {
+				continue
+			}
+			if b >= 'A' && b <= 'Z' {
+				b += 'a' - 'A'
+			}
+			if UtoT && b == 'u' {
+				b = 't'
+			}
+			dest = append(dest, b)
+		}
+		s.current = s.current.Next()
+		s.pos = 0
+	}
+	return dest, false
+}
+
+// FastaChunkParserRope parses a FASTA chunk directly from the rope without Pack().
+func FastaChunkParserRope(source string, rope *PieceOfChunk, UtoT bool) (obiseq.BioSequenceSlice, error) {
+	scanner := newRopeScanner(rope)
+	sequences := obiseq.MakeBioSequenceSlice(100)[:0]
+
+	for {
+		bline := scanner.ReadLine()
+		if bline == nil {
+			break
+		}
+		if len(bline) == 0 || bline[0] != '>' {
+			continue
+		}
+
+		// Parse header: ">id definition"
+		header := bline[1:]
+		var id string
+		var definition string
+		sp := bytes.IndexByte(header, ' ')
+		if sp < 0 {
+			sp = bytes.IndexByte(header, '\t')
+		}
+		if sp < 0 {
+			id = string(header)
+		} else {
+			id = string(header[:sp])
+			definition = string(bytes.TrimSpace(header[sp+1:]))
+		}
+
+		seqDest := make([]byte, 0, 4096)
+		var hasMore bool
+		seqDest, hasMore = scanner.extractFastaSeq(seqDest, UtoT)
+
+		if len(seqDest) == 0 {
+			log.Fatalf("%s [%s]: sequence is empty", source, id)
+		}
+
+		seq := obiseq.NewBioSequenceOwning(id, seqDest, definition)
+		seq.SetSource(source)
+		sequences = append(sequences, seq)
+
+		if !hasMore {
+			break
+		}
+	}
+
+	return sequences, nil
+}
+
 func _ParseFastaFile(
 	input ChannelFileChunk,
 	out obiiter.IBioSequence,
 	UtoT bool,
 ) {
-
 	parser := FastaChunkParser(UtoT)
 
 	for chunks := range input {
-		sequences, err := parser(chunks.Source, chunks.Raw)
+		var sequences obiseq.BioSequenceSlice
-		// obilog.Warnf("Chunck(%d:%d) -%d- ", chunks.Order, l, sequences.Len())
+		var err error
+
+		if chunks.Rope != nil {
+			sequences, err = FastaChunkParserRope(chunks.Source, chunks.Rope, UtoT)
+		} else {
+			sequences, err = parser(chunks.Source, chunks.Raw)
+		}
 
 		if err != nil {
 			log.Fatalf("File %s : Cannot parse the fasta file : %v", chunks.Source, err)
 		}
 
 		out.Push(obiiter.MakeBioSequenceBatch(chunks.Source, chunks.Order, sequences))
-
 	}
 
 	out.Done()
-
 }
 
 func ReadFasta(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, error) {
@@ -245,6 +338,7 @@ func ReadFasta(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, e
 		1024*1024,
 		EndOfLastFastaEntry,
 		"\n>",
+		false,
 	)
 
 	for i := 0; i < nworker; i++ {

pkg/obiformats/fastqseq_read.go

@@ -131,7 +131,7 @@ func _storeSequenceQuality(bytes *bytes.Buffer, out *obiseq.BioSequence, quality
 	out.SetQualities(q)
 }
 
-func FastqChunkParser(quality_shift byte, with_quality bool, UtoT bool) func(string, io.Reader) (obiseq.BioSequenceSlice, error) {
+func FastqChunkParser(quality_shift byte, with_quality bool, UtoT bool, fileName string) func(string, io.Reader) (obiseq.BioSequenceSlice, error) {
 	parser := func(source string, input io.Reader) (obiseq.BioSequenceSlice, error) {
 
 		var identifier string
@@ -160,12 +160,12 @@ func FastqChunkParser(quality_shift byte, with_quality bool, UtoT bool) func(str
 					// Beginning of sequence
 					state = 1
 				} else {
-					log.Fatalf("%s : sequence entry is not starting with @", source)
+					log.Fatalf("file %s: sequence entry is not starting with @", fileName)
 				}
 			case 1: // Beginning of identifier (Mandatory)
 				if is_sep {
 					// No identifier -> ERROR
-					log.Fatalf("%s : sequence identifier is empty", source)
+					log.Fatalf("file %s: sequence identifier is empty", fileName)
 				} else {
 					// Beginning of identifier
 					state = 2
@@ -221,7 +221,7 @@ func FastqChunkParser(quality_shift byte, with_quality bool, UtoT bool) func(str
 					// End of sequence
 					rawseq := seqBytes.Bytes()
 					if len(rawseq) == 0 {
-						log.Fatalf("@%s[%s] : sequence is empty", identifier, source)
+						log.Fatalf("file %s: record @%s has an empty sequence line", fileName, identifier)
 					}
 					s := obiseq.NewBioSequence(identifier, rawseq, definition)
 					s.SetSource(source)
@@ -241,8 +241,8 @@ func FastqChunkParser(quality_shift byte, with_quality bool, UtoT bool) func(str
 						context = append(
 							append([]byte{previous}, C),
 							context...)
-						log.Fatalf("%s [%s]: sequence contains invalid character %c (%s)",
+						log.Fatalf("file %s: record @%s contains invalid character %c (%s)",
-							source, identifier, C, string(context))
+							fileName, identifier, C, string(context))
 					}
 				}
 			case 7:
@@ -251,7 +251,7 @@ func FastqChunkParser(quality_shift byte, with_quality bool, UtoT bool) func(str
 				} else if C == '+' {
 					state = 8
 				} else {
-					log.Fatalf("@%s[%s] : sequence data not followed by a line starting with + but a %c", identifier, source, C)
+					log.Fatalf("file %s: record @%s: sequence data not followed by a line starting with + but a %c", fileName, identifier, C)
 				}
 			case 8:
 				// State consuming the + internal header line
@@ -282,7 +282,7 @@ func FastqChunkParser(quality_shift byte, with_quality bool, UtoT bool) func(str
 				} else if C == '@' {
 					state = 1
 				} else {
-					log.Fatalf("%s[%s] : sequence record not followed by a line starting with @", identifier, source)
+					log.Fatalf("file %s: record @%s not followed by a line starting with @", fileName, identifier)
 				}
 
 			}
@@ -303,17 +303,99 @@ func FastqChunkParser(quality_shift byte, with_quality bool, UtoT bool) func(str
 	return parser
 }
 
+// FastqChunkParserRope parses a FASTQ chunk directly from a rope without Pack().
+func FastqChunkParserRope(source string, rope *PieceOfChunk, quality_shift byte, with_quality, UtoT bool, fileName string) (obiseq.BioSequenceSlice, error) {
+	scanner := newRopeScanner(rope)
+	sequences := obiseq.MakeBioSequenceSlice(100)[:0]
+
+	for {
+		// Line 1: @id [definition]
+		hline := scanner.ReadLine()
+		if hline == nil {
+			break
+		}
+		if len(hline) == 0 || hline[0] != '@' {
+			continue
+		}
+		header := hline[1:]
+		var id string
+		var definition string
+		sp := bytes.IndexByte(header, ' ')
+		if sp < 0 {
+			sp = bytes.IndexByte(header, '\t')
+		}
+		if sp < 0 {
+			id = string(header)
+		} else {
+			id = string(header[:sp])
+			definition = string(bytes.TrimSpace(header[sp+1:]))
+		}
+
+		// Line 2: sequence
+		sline := scanner.ReadLine()
+		if sline == nil {
+			log.Fatalf("file %s: record @%s is truncated (header line with no sequence line following) — the FASTQ file appears incomplete", fileName, id)
+		}
+		seqDest := make([]byte, len(sline))
+		w := 0
+		for _, b := range sline {
+			if b >= 'A' && b <= 'Z' {
+				b += 'a' - 'A'
+			}
+			if UtoT && b == 'u' {
+				b = 't'
+			}
+			seqDest[w] = b
+			w++
+		}
+		seqDest = seqDest[:w]
+		if len(seqDest) == 0 {
+			log.Fatalf("file %s: record @%s has an empty sequence line", fileName, id)
+		}
+
+		// Line 3: + (skip)
+		scanner.ReadLine()
+
+		// Line 4: quality
+		qline := scanner.ReadLine()
+
+		seq := obiseq.NewBioSequenceOwning(id, seqDest, definition)
+		seq.SetSource(source)
+
+		if with_quality && qline != nil {
+			qDest := make([]byte, len(qline))
+			copy(qDest, qline)
+			for i := range qDest {
+				qDest[i] -= quality_shift
+			}
+			seq.TakeQualities(qDest)
+		}
+
+		sequences = append(sequences, seq)
+	}
+
+	return sequences, nil
+}
+
 func _ParseFastqFile(
 	input ChannelFileChunk,
 	out obiiter.IBioSequence,
 	quality_shift byte,
 	with_quality, UtoT bool,
+	fileName string,
 ) {
 
-	parser := FastqChunkParser(quality_shift, with_quality, UtoT)
+	parser := FastqChunkParser(quality_shift, with_quality, UtoT, fileName)
 
 	for chunks := range input {
-		sequences, err := parser(chunks.Source, chunks.Raw)
+		var sequences obiseq.BioSequenceSlice
+		var err error
+
+		if chunks.Rope != nil {
+			sequences, err = FastqChunkParserRope(chunks.Source, chunks.Rope, quality_shift, with_quality, UtoT, fileName)
+		} else {
+			sequences, err = parser(chunks.Source, chunks.Raw)
+		}
 
 		if err != nil {
 			log.Fatalf("File %s : Cannot parse the fastq file : %v", chunks.Source, err)
@@ -339,8 +421,11 @@ func ReadFastq(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, e
 		1024*1024,
 		EndOfLastFastqEntry,
 		"\n@",
+		false,
 	)
 
+	fileName := opt.FileName()
+
 	for i := 0; i < nworker; i++ {
 		out.Add(1)
 		go _ParseFastqFile(
@@ -349,6 +434,7 @@ func ReadFastq(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, e
 			obidefault.ReadQualitiesShift(),
 			opt.ReadQualities(),
 			opt.UtoT(),
+			fileName,
 		)
 	}
 
@@ -374,7 +460,9 @@ func ReadFastq(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, e
 }
 
 func ReadFastqFromFile(filename string, options ...WithOption) (obiiter.IBioSequence, error) {
-	options = append(options, OptionsSource(obiutils.RemoveAllExt((path.Base(filename)))))
+	options = append(options,
+		OptionsSource(obiutils.RemoveAllExt((path.Base(filename)))),
+		OptionsFileName(filename))
 
 	file, err := obiutils.Ropen(filename)
 

pkg/obiformats/fastseq_json_header.go

@@ -199,8 +199,111 @@ func _parse_json_array_interface(str []byte) ([]interface{}, error) {
 	return values, nil
 }
 
-func _parse_json_header_(header string, sequence *obiseq.BioSequence) string {
+// _parse_json_annotation_field parses a single key/value pair coming from a
+// JSON object (either a FASTA/FASTQ inline JSON header, or the "annotations"
+// field of a JSON sequence record) and applies it to the sequence, special
+// casing the well-known OBITools attributes (id, definition, count, taxid,
+// obiclean_*, merged_*).
+func _parse_json_annotation_field(key []byte, value []byte, dataType jsonparser.ValueType, sequence *obiseq.BioSequence) error {
 	annotations := sequence.Annotations()
+	var err error
+
+	skey := obiutils.UnsafeString(key)
+
+	switch {
+	case skey == "id":
+		sequence.SetId(string(value))
+	case skey == "definition":
+		sequence.SetDefinition(string(value))
+
+	case skey == "count":
+		if dataType != jsonparser.Number {
+			log.Fatalf("%s: Count attribut must be numeric: %s", sequence.Id(), string(value))
+		}
+		count, err := jsonparser.ParseInt(value)
+		if err != nil {
+			log.Fatalf("%s: Cannot parse count %s", sequence.Id(), string(value))
+		}
+		sequence.SetCount(int(count))
+
+	case skey == "obiclean_weight":
+		weight, err := _parse_json_map_int(value)
+		if err != nil {
+			log.Fatalf("%s: Cannot parse obiclean weight %s", sequence.Id(), string(value))
+		}
+		annotations[skey] = weight
+
+	case skey == "obiclean_status":
+		status, err := _parse_json_map_string(value)
+		if err != nil {
+			log.Fatalf("%s: Cannot parse obiclean status %s", sequence.Id(), string(value))
+		}
+		annotations[skey] = status
+
+	case strings.HasPrefix(skey, "merged_"):
+		if dataType == jsonparser.Object {
+			data, err := _parse_json_map_int(value)
+			if err != nil {
+				log.Fatalf("%s: Cannot parse merged slot %s: %v", sequence.Id(), skey, err)
+			} else {
+				annotations[skey] = obiseq.MapAsStatsOnValues(data)
+			}
+		} else {
+			log.Fatalf("%s: Cannot parse merged slot %s", sequence.Id(), skey)
+		}
+
+	case skey == "taxid":
+		if dataType == jsonparser.Number || dataType == jsonparser.String {
+			taxid := string(value)
+			sequence.SetTaxid(taxid)
+		} else {
+			log.Fatalf("%s: Cannot parse taxid %s", sequence.Id(), string(value))
+		}
+
+	case strings.HasSuffix(skey, "_taxid"):
+		if dataType == jsonparser.Number || dataType == jsonparser.String {
+			rank := skey[:len(skey)-len("_taxid")]
+
+			taxid := string(value)
+			sequence.SetTaxid(taxid, rank)
+		} else {
+			log.Fatalf("%s: Cannot parse taxid %s", sequence.Id(), string(value))
+		}
+
+	default:
+		skey = strings.Clone(skey)
+		switch dataType {
+		case jsonparser.String:
+			annotations[skey] = string(value)
+		case jsonparser.Number:
+			// Try to parse the number as an int at first then as float if that fails.
+			annotations[skey], err = jsonparser.ParseInt(value)
+			if err != nil {
+				annotations[skey], err = strconv.ParseFloat(obiutils.UnsafeString(value), 64)
+			}
+		case jsonparser.Array:
+			annotations[skey], err = _parse_json_array_interface(value)
+		case jsonparser.Object:
+			annotations[skey], err = _parse_json_map_interface(value)
+		case jsonparser.Boolean:
+			annotations[skey], err = jsonparser.ParseBoolean(value)
+		case jsonparser.Null:
+			annotations[skey] = nil
+		default:
+			log.Fatalf("Unknown data type %v", dataType)
+		}
+	}
+
+	if err != nil {
+		annotations[skey] = "NaN"
+		log.Fatalf("%s: Cannot parse value %s assicated to key %s into a %s value",
+			sequence.Id(), string(value), skey, dataType.String())
+	}
+
+	return err
+}
+
+func _parse_json_header_(header string, sequence *obiseq.BioSequence) string {
 	start := -1
 	stop := -1
 	level := 0
@@ -240,101 +343,7 @@ func _parse_json_header_(header string, sequence *obiseq.BioSequence) string {
 
 	jsonparser.ObjectEach(obiutils.UnsafeBytes(header[start:stop]),
 		func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) error {
-			var err error
+			return _parse_json_annotation_field(key, value, dataType, sequence)
-
-			skey := obiutils.UnsafeString(key)
-
-			switch {
-			case skey == "id":
-				sequence.SetId(string(value))
-			case skey == "definition":
-				sequence.SetDefinition(string(value))
-
-			case skey == "count":
-				if dataType != jsonparser.Number {
-					log.Fatalf("%s: Count attribut must be numeric: %s", sequence.Id(), string(value))
-				}
-				count, err := jsonparser.ParseInt(value)
-				if err != nil {
-					log.Fatalf("%s: Cannot parse count %s", sequence.Id(), string(value))
-				}
-				sequence.SetCount(int(count))
-
-			case skey == "obiclean_weight":
-				weight, err := _parse_json_map_int(value)
-				if err != nil {
-					log.Fatalf("%s: Cannot parse obiclean weight %s", sequence.Id(), string(value))
-				}
-				annotations[skey] = weight
-
-			case skey == "obiclean_status":
-				status, err := _parse_json_map_string(value)
-				if err != nil {
-					log.Fatalf("%s: Cannot parse obiclean status %s", sequence.Id(), string(value))
-				}
-				annotations[skey] = status
-
-			case strings.HasPrefix(skey, "merged_"):
-				if dataType == jsonparser.Object {
-					data, err := _parse_json_map_int(value)
-					if err != nil {
-						log.Fatalf("%s: Cannot parse merged slot %s: %v", sequence.Id(), skey, err)
-					} else {
-						annotations[skey] = obiseq.MapAsStatsOnValues(data)
-					}
-				} else {
-					log.Fatalf("%s: Cannot parse merged slot %s", sequence.Id(), skey)
-				}
-
-			case skey == "taxid":
-				if dataType == jsonparser.Number || dataType == jsonparser.String {
-					taxid := string(value)
-					sequence.SetTaxid(taxid)
-				} else {
-					log.Fatalf("%s: Cannot parse taxid %s", sequence.Id(), string(value))
-				}
-
-			case strings.HasSuffix(skey, "_taxid"):
-				if dataType == jsonparser.Number || dataType == jsonparser.String {
-					rank, _ := obiutils.SplitInTwo(skey, '_')
-
-					taxid := string(value)
-					sequence.SetTaxid(taxid, rank)
-				} else {
-					log.Fatalf("%s: Cannot parse taxid %s", sequence.Id(), string(value))
-				}
-
-			default:
-				skey = strings.Clone(skey)
-				switch dataType {
-				case jsonparser.String:
-					annotations[skey] = string(value)
-				case jsonparser.Number:
-					// Try to parse the number as an int at first then as float if that fails.
-					annotations[skey], err = jsonparser.ParseInt(value)
-					if err != nil {
-						annotations[skey], err = strconv.ParseFloat(obiutils.UnsafeString(value), 64)
-					}
-				case jsonparser.Array:
-					annotations[skey], err = _parse_json_array_interface(value)
-				case jsonparser.Object:
-					annotations[skey], err = _parse_json_map_interface(value)
-				case jsonparser.Boolean:
-					annotations[skey], err = jsonparser.ParseBoolean(value)
-				case jsonparser.Null:
-					annotations[skey] = nil
-				default:
-					log.Fatalf("Unknown data type %v", dataType)
-				}
-			}
-
-			if err != nil {
-				annotations[skey] = "NaN"
-				log.Fatalf("%s: Cannot parse value %s assicated to key %s into a %s value",
-					sequence.Id(), string(value), skey, dataType.String())
-			}
-
-			return err
 		},
 	)
 

pkg/obiformats/fastseq_obi_header.go

@@ -17,7 +17,7 @@ import (
 )
 
 var __obi_header_value_string_pattern__ = regexp.MustCompile(`^'\s*([^']*'|"[^"]*")\s*;`)
-var __obi_header_value_numeric_pattern__ = regexp.MustCompile(`^\s*([+-]?\.\d+|[+-]?\d+(\.\d*)?([eE][+-]?\d+)?)\s*;`)
+var __obi_header_value_numeric_pattern__ = regexp.MustCompile(`^\s*[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?\s*;`)
 var __obi_header_map_int_key__ = regexp.MustCompile("([{,])([0-9]+):")
 
 func __match__dict__(text []byte) []int {
@@ -146,6 +146,65 @@ func __match__key__(text []byte) []int {
 	return []int{} // Not a key
 }
 
+func __match__array__(text []byte) []int {
+
+	state := 0
+	level := 0
+	start := 0
+	instring := byte(0)
+
+	for i, r := range text {
+		if state == 2 {
+			if r == ';' {
+				return []int{start, i + 1}
+			}
+			if r != ' ' && r != '\t' {
+				return []int{}
+			}
+		}
+
+		if state == 0 {
+			if r == '[' {
+				level++
+				state++
+				start = i
+				continue
+			}
+			if r != ' ' && r != '\t' {
+				return []int{}
+			}
+			continue
+		}
+
+		// state == 1: inside the array
+		if instring != 0 {
+			if r == instring {
+				instring = 0
+			}
+			continue
+		}
+
+		if r == '"' || r == '\'' {
+			instring = r
+			continue
+		}
+
+		if r == '[' || r == '{' {
+			level++
+			continue
+		}
+
+		if r == ']' || r == '}' {
+			level--
+			if level == 0 {
+				state++
+			}
+		}
+	}
+
+	return []int{}
+}
+
 func __match__general__(text []byte) []int {
 
 	for i, r := range text {
@@ -242,28 +301,44 @@ func ParseOBIFeatures(text string, annotations obiseq.Annotation) string {
 					stop = m[1] + 1
 				} else {
 
-					// Generic value
+					// array value
-
+					m = __match__array__(part)
-					// m = __obi_header_value_general_pattern__.FindIndex(part)
-					m = __match__general__(part)
 					if len(m) > 0 {
 						bvalue = bytes.TrimSpace(part[m[0]:(m[1] - 1)])
-
+						j := bytes.ReplaceAll(bvalue, []byte("'"), []byte(`"`))
-						if __is_false__(bvalue) {
+						j = __obi_header_map_int_key__.ReplaceAll(j, []byte(`$1"$2":`))
-							value = false
+						arr, err := _parse_json_array_interface(j)
+						if err != nil {
+							value = string(bvalue)
 						} else {
-							if __is_true__(bvalue) {
+							value = arr
-								value = true
-							} else {
-								value = string(bvalue)
-							}
 						}
-
 						stop = m[1] + 1
 					} else {
-						// no value
+
-						break
+						// Generic value
-					} // End of No value
+
+						// m = __obi_header_value_general_pattern__.FindIndex(part)
+						m = __match__general__(part)
+						if len(m) > 0 {
+							bvalue = bytes.TrimSpace(part[m[0]:(m[1] - 1)])
+
+							if __is_false__(bvalue) {
+								value = false
+							} else {
+								if __is_true__(bvalue) {
+									value = true
+								} else {
+									value = string(bvalue)
+								}
+							}
+
+							stop = m[1] + 1
+						} else {
+							// no value
+							break
+						} // End of No value
+					} // End of not array
 				} // End of not dict
 			} // End of not string
 		} // End of not numeric
@@ -272,6 +347,8 @@ func ParseOBIFeatures(text string, annotations obiseq.Annotation) string {
 		case float64:
 			if vt == math.Floor(vt) {
 				annotations[key] = int(vt)
+			} else {
+				annotations[key] = vt
 			}
 		default:
 			annotations[key] = value
@@ -327,19 +404,19 @@ func WriteFastSeqOBIHeade(buffer *bytes.Buffer, sequence *obiseq.BioSequence) {
 					buffer.WriteString(fmt.Sprintf("%s=", key))
 					buffer.Write(tv)
 					buffer.WriteString("; ")
-				case map[string]int,
-					map[string]string,
-					map[string]interface{}:
-					tv, err := obiutils.JsonMarshal(t)
-					if err != nil {
-						log.Fatalf("Cannot convert %v value", value)
-					}
-					tv = bytes.ReplaceAll(tv, []byte(`"`), []byte("'"))
-					buffer.WriteString(fmt.Sprintf("%s=", key))
-					buffer.Write(tv)
-					buffer.WriteString("; ")
 				default:
-					buffer.WriteString(fmt.Sprintf("%s=%v; ", key, value))
+					if obiutils.IsAMap(value) || obiutils.IsASlice(value) || obiutils.IsAnArray(value) {
+						tv, err := obiutils.JsonMarshal(t)
+						if err != nil {
+							log.Fatalf("Cannot convert %v value", value)
+						}
+						tv = bytes.ReplaceAll(tv, []byte(`"`), []byte("'"))
+						buffer.WriteString(fmt.Sprintf("%s=", key))
+						buffer.Write(tv)
+						buffer.WriteString("; ")
+					} else {
+						buffer.WriteString(fmt.Sprintf("%s=%v; ", key, value))
+					}
 				}
 			}
 		}

pkg/obiformats/fastseq_write_fasta.go

@@ -77,45 +77,50 @@ func FormatFasta(seq *obiseq.BioSequence, formater FormatHeader) string {
 //
 // It returns a byte array containing the formatted sequences.
 func FormatFastaBatch(batch obiiter.BioSequenceBatch, formater FormatHeader, skipEmpty bool) *bytes.Buffer {
-	// Create a buffer to store the formatted sequences
 	var bs bytes.Buffer
 
 	lt := 0
-
 	for _, seq := range batch.Slice() {
 		lt += seq.Len()
 	}
 
-	// Iterate over each sequence in the batch
+	// Pre-allocate: sequence data + newlines every 60 chars + ~100 bytes header per sequence
+	bs.Grow(lt + lt/60 + 100*batch.Len() + 1)
+
 	log.Debugf("FormatFastaBatch: #%d : %d seqs", batch.Order(), batch.Len())
-	first := true
+
 	for _, seq := range batch.Slice() {
-		// Check if the sequence is empty
+		if len(seq.Id()) == 0 {
+			log.Fatalf("Sequence identifier is empty")
+		}
 		if seq.Len() > 0 {
-			// Format the sequence using the provided formater function
+			// Write header directly into bs — no intermediate string
-			formattedSeq := FormatFasta(seq, formater)
+			bs.WriteByte('>')
-
+			bs.WriteString(seq.Id())
-			if first {
+			bs.WriteByte(' ')
-				bs.Grow(lt + (len(formattedSeq)-seq.Len())*batch.Len()*5/4)
+			bs.WriteString(formater(seq))
-				first = false
-			}
-
-			// Append the formatted sequence to the buffer
-			bs.WriteString(formattedSeq)
 			bs.WriteByte('\n')
+
+			// Write folded sequence directly into bs — no copies
+			s := seq.Sequence()
+			l := len(s)
+			for i := 0; i < l; i += 60 {
+				to := i + 60
+				if to > l {
+					to = l
+				}
+				bs.Write(s[i:to])
+				bs.WriteByte('\n')
+			}
 		} else {
-			// Handle empty sequences
 			if skipEmpty {
-				// Skip empty sequences if skipEmpty is true
 				obilog.Warnf("Sequence %s is empty and skipped in output", seq.Id())
 			} else {
-				// Terminate the program if skipEmpty is false
 				log.Fatalf("Sequence %s is empty", seq.Id())
 			}
 		}
 	}
 
-	// Return the byte array representation of the buffer
 	return &bs
 }
 

pkg/obiformats/fastseq_write_fastq.go

@@ -64,6 +64,9 @@ func FormatFastqBatch(batch obiiter.BioSequenceBatch,
 	first := true
 
 	for _, seq := range batch.Slice() {
+		if len(seq.Id()) == 0 {
+			log.Fatalf("Sequence identifier is empty")
+		}
 		if seq.Len() > 0 {
 			_formatFastq(&bs, seq, formater)
 

pkg/obiformats/file_chunk_read.go

@@ -16,6 +16,7 @@ type SeqFileChunkParser func(string, io.Reader) (obiseq.BioSequenceSlice, error)
 type FileChunk struct {
 	Source string
 	Raw    *bytes.Buffer
+	Rope   *PieceOfChunk
 	Order  int
 }
 
@@ -97,11 +98,17 @@ func (piece *PieceOfChunk) IsLast() bool {
 	return piece.next == nil
 }
 
-func (piece *PieceOfChunk) FileChunk(source string, order int) FileChunk {
+func (piece *PieceOfChunk) FileChunk(source string, order int, pack bool) FileChunk {
-	piece.Pack()
+	piece = piece.Head()
+	var raw *bytes.Buffer
+	if pack {
+		piece.Pack()
+		raw = bytes.NewBuffer(piece.data)
+	}
 	return FileChunk{
 		Source: source,
-		Raw:    bytes.NewBuffer(piece.data),
+		Raw:    raw,
+		Rope:   piece,
 		Order:  order,
 	}
 }
@@ -133,7 +140,8 @@ func ReadFileChunk(
 	reader io.Reader,
 	fileChunkSize int,
 	splitter LastSeqRecord,
-	probe string) ChannelFileChunk {
+	probe string,
+	pack bool) ChannelFileChunk {
 
 	chunk_channel := make(ChannelFileChunk)
 
@@ -205,7 +213,7 @@ func ReadFileChunk(
 
 				if len(pieces.data) > 0 {
 					// obilog.Warnf("chuck %d :Read %d bytes from file %s", i, io.Len(), source)
-					chunk_channel <- pieces.FileChunk(source, i)
+					chunk_channel <- pieces.FileChunk(source, i, pack)
 					i++
 				}
 
@@ -222,7 +230,7 @@ func ReadFileChunk(
 
 		// Send the last chunk to the channel
 		if pieces.Len() > 0 {
-			chunk_channel <- pieces.FileChunk(source, i)
+			chunk_channel <- pieces.FileChunk(source, i, pack)
 		}
 
 		// Close the readers channel when the end of the file is reached

pkg/obiformats/genbank_read.go

@@ -29,6 +29,265 @@ const (
 
 var _seqlenght_rx = regexp.MustCompile(" +([0-9]+) bp")
 
+// extractSequence scans the ORIGIN section byte-by-byte directly on the rope,
+// appending compacted bases to dest. Returns the extended slice.
+// Stops and returns when "//" is found at the start of a line.
+// The scanner is left positioned after the "//" line.
+func (s *ropeScanner) extractSequence(dest []byte, UtoT bool) []byte {
+	lineStart := true
+	skipDigits := true
+
+	for s.current != nil {
+		data := s.current.data[s.pos:]
+		for i, b := range data {
+			if lineStart {
+				if b == '/' {
+					// End-of-record marker "//"
+					s.pos += i + 1
+					if s.pos >= len(s.current.data) {
+						s.current = s.current.Next()
+						s.pos = 0
+					}
+					s.skipToNewline()
+					return dest
+				}
+				lineStart = false
+				skipDigits = true
+			}
+			switch {
+			case b == '\n':
+				lineStart = true
+			case b == '\r':
+				// skip
+			case skipDigits:
+				if b != ' ' && (b < '0' || b > '9') {
+					skipDigits = false
+					if UtoT && b == 'u' {
+						b = 't'
+					}
+					dest = append(dest, b)
+				}
+			case b != ' ':
+				if UtoT && b == 'u' {
+					b = 't'
+				}
+				dest = append(dest, b)
+			}
+		}
+		s.current = s.current.Next()
+		s.pos = 0
+	}
+	return dest
+}
+
+// parseLseqFromLocus extracts the declared sequence length from a LOCUS line.
+// Format: "LOCUS       <id> <length> bp ..."
+// Returns -1 if not found or parse error.
+func parseLseqFromLocus(line []byte) int {
+	if len(line) < 13 {
+		return -1
+	}
+	i := 12
+	for i < len(line) && line[i] != ' ' {
+		i++
+	}
+	for i < len(line) && line[i] == ' ' {
+		i++
+	}
+	start := i
+	for i < len(line) && line[i] >= '0' && line[i] <= '9' {
+		i++
+	}
+	if i == start {
+		return -1
+	}
+	n, err := strconv.Atoi(string(line[start:i]))
+	if err != nil {
+		return -1
+	}
+	return n
+}
+
+// Prefix constants for GenBank section headers (byte slices for zero-alloc comparison).
+var (
+	gbPfxLocus      = []byte("LOCUS       ")
+	gbPfxDefinition = []byte("DEFINITION  ")
+	gbPfxContinue   = []byte("            ")
+	gbPfxSource     = []byte("SOURCE      ")
+	gbPfxFeatures   = []byte("FEATURES    ")
+	gbPfxOrigin     = []byte("ORIGIN")
+	gbPfxContig     = []byte("CONTIG")
+	gbPfxEnd        = []byte("//")
+	gbPfxDbXref     = []byte(`                     /db_xref="taxon:`)
+)
+
+// GenbankChunkParserRope parses a GenBank FileChunk directly from the rope
+// (PieceOfChunk linked list) without calling Pack(). This eliminates the large
+// contiguous allocation required for chromosomal-scale sequences.
+func GenbankChunkParserRope(source string, rope *PieceOfChunk,
+	withFeatureTable, UtoT bool) (obiseq.BioSequenceSlice, error) {
+
+	state := inHeader
+	scanner := newRopeScanner(rope)
+	sequences := obiseq.MakeBioSequenceSlice(100)[:0]
+
+	id := ""
+	lseq := -1
+	scientificName := ""
+	defBytes := new(bytes.Buffer)
+	featBytes := new(bytes.Buffer)
+	var seqDest []byte
+	taxid := 1
+	nl := 0
+
+	for bline := scanner.ReadLine(); bline != nil; bline = scanner.ReadLine() {
+		nl++
+		processed := false
+		for !processed {
+			switch {
+
+			case bytes.HasPrefix(bline, gbPfxLocus):
+				if state != inHeader {
+					log.Fatalf("Line %d - Unexpected state %d while reading LOCUS: %s", nl, state, bline)
+				}
+				rest := bline[12:]
+				sp := bytes.IndexByte(rest, ' ')
+				if sp < 0 {
+					id = string(rest)
+				} else {
+					id = string(rest[:sp])
+				}
+				lseq = parseLseqFromLocus(bline)
+				cap0 := lseq + 20
+				if cap0 < 1024 {
+					cap0 = 1024
+				}
+				seqDest = make([]byte, 0, cap0)
+				state = inEntry
+				processed = true
+
+			case bytes.HasPrefix(bline, gbPfxDefinition):
+				if state != inEntry {
+					log.Fatalf("Line %d - Unexpected state %d while reading DEFINITION: %s", nl, state, bline)
+				}
+				defBytes.Write(bytes.TrimSpace(bline[12:]))
+				state = inDefinition
+				processed = true
+
+			case state == inDefinition:
+				if bytes.HasPrefix(bline, gbPfxContinue) {
+					defBytes.WriteByte(' ')
+					defBytes.Write(bytes.TrimSpace(bline[12:]))
+					processed = true
+				} else {
+					state = inEntry
+				}
+
+			case bytes.HasPrefix(bline, gbPfxSource):
+				if state != inEntry {
+					log.Fatalf("Line %d - Unexpected state %d while reading SOURCE: %s", nl, state, bline)
+				}
+				scientificName = string(bytes.TrimSpace(bline[12:]))
+				processed = true
+
+			case bytes.HasPrefix(bline, gbPfxFeatures):
+				if state != inEntry {
+					log.Fatalf("Line %d - Unexpected state %d while reading FEATURES: %s", nl, state, bline)
+				}
+				if withFeatureTable {
+					featBytes.Write(bline)
+				}
+				state = inFeature
+				processed = true
+
+			case bytes.HasPrefix(bline, gbPfxOrigin):
+				if state != inFeature && state != inContig {
+					log.Fatalf("Line %d - Unexpected state %d while reading ORIGIN: %s", nl, state, bline)
+				}
+				// Use fast byte-scan to extract sequence and consume through "//"
+				seqDest = scanner.extractSequence(seqDest, UtoT)
+				// Emit record
+				if id == "" {
+					log.Warn("Empty id when parsing genbank file")
+				}
+				sequence := obiseq.NewBioSequenceOwning(id, seqDest, defBytes.String())
+				sequence.SetSource(source)
+				if withFeatureTable {
+					sequence.SetFeatures(featBytes.Bytes())
+				}
+				annot := sequence.Annotations()
+				annot["scientific_name"] = scientificName
+				annot["taxid"] = taxid
+				sequences = append(sequences, sequence)
+
+				defBytes = bytes.NewBuffer(obiseq.GetSlice(200))
+				featBytes = new(bytes.Buffer)
+				nl = 0
+				taxid = 1
+				seqDest = nil
+				state = inHeader
+				processed = true
+
+			case bytes.HasPrefix(bline, gbPfxContig):
+				if state != inFeature && state != inContig {
+					log.Fatalf("Line %d - Unexpected state %d while reading CONTIG: %s", nl, state, bline)
+				}
+				state = inContig
+				processed = true
+
+			case bytes.Equal(bline, gbPfxEnd):
+				// Reached for CONTIG records (no ORIGIN section)
+				if state != inContig {
+					log.Fatalf("Line %d - Unexpected state %d while reading end of record %s", nl, state, id)
+				}
+				if id == "" {
+					log.Warn("Empty id when parsing genbank file")
+				}
+				sequence := obiseq.NewBioSequenceOwning(id, seqDest, defBytes.String())
+				sequence.SetSource(source)
+				if withFeatureTable {
+					sequence.SetFeatures(featBytes.Bytes())
+				}
+				annot := sequence.Annotations()
+				annot["scientific_name"] = scientificName
+				annot["taxid"] = taxid
+				sequences = append(sequences, sequence)
+
+				defBytes = bytes.NewBuffer(obiseq.GetSlice(200))
+				featBytes = new(bytes.Buffer)
+				nl = 0
+				taxid = 1
+				seqDest = nil
+				state = inHeader
+				processed = true
+
+			default:
+				switch state {
+				case inFeature:
+					if withFeatureTable {
+						featBytes.WriteByte('\n')
+						featBytes.Write(bline)
+					}
+					if bytes.HasPrefix(bline, gbPfxDbXref) {
+						rest := bline[len(gbPfxDbXref):]
+						q := bytes.IndexByte(rest, '"')
+						if q >= 0 {
+							taxid, _ = strconv.Atoi(string(rest[:q]))
+						}
+					}
+					processed = true
+				case inHeader, inEntry, inContig:
+					processed = true
+				default:
+					log.Fatalf("Unexpected state %d while reading: %s", state, bline)
+				}
+			}
+		}
+	}
+
+	return sequences, nil
+}
+
 func GenbankChunkParser(withFeatureTable, UtoT bool) func(string, io.Reader) (obiseq.BioSequenceSlice, error) {
 	return func(source string, input io.Reader) (obiseq.BioSequenceSlice, error) {
 		state := inHeader
@@ -125,13 +384,10 @@ func GenbankChunkParser(withFeatureTable, UtoT bool) func(string, io.Reader) (ob
 					if state != inSequence && state != inContig {
 						log.Fatalf("Line %d - Unexpected state %d while reading end of record %s", nl, state, id)
 					}
-					// log.Debugln("Total lines := ", nl)
 					if id == "" {
 						log.Warn("Empty id when parsing genbank file")
 					}
 
-					// log.Debugf("End of sequence %s: %dbp ", id, seqBytes.Len())
-
 					sequence := obiseq.NewBioSequence(id,
 						seqBytes.Bytes(),
 						defBytes.String())
@@ -144,9 +400,6 @@ func GenbankChunkParser(withFeatureTable, UtoT bool) func(string, io.Reader) (ob
 					annot := sequence.Annotations()
 					annot["scientific_name"] = scientificName
 					annot["taxid"] = taxid
-					// log.Println(FormatFasta(sequence, FormatFastSeqJsonHeader))
-					// log.Debugf("Read sequences %s: %dbp (%d)", sequence.Id(),
-					//	sequence.Len(), seqBytes.Len())
 
 					sequences = append(sequences, sequence)
 
@@ -159,12 +412,11 @@ func GenbankChunkParser(withFeatureTable, UtoT bool) func(string, io.Reader) (ob
 					processed = true
 
 				case state == inSequence:
-					// log.Debugf("Chunk %d : Genbank: line %d, state = %d : %s", chunks.order, nl, state, line)
-
 					sl++
-					parts := strings.SplitN(line[10:], " ", 6)
+					cleanline := strings.TrimSpace(line)
+					parts := strings.SplitN(cleanline, " ", 7)
 					lparts := len(parts)
-					for i := 0; i < lparts; i++ {
+					for i := 1; i < lparts; i++ {
 						if UtoT {
 							parts[i] = strings.ReplaceAll(parts[i], "u", "t")
 						}
@@ -197,6 +449,7 @@ func GenbankChunkParser(withFeatureTable, UtoT bool) func(string, io.Reader) (ob
 
 		}
 
+		_ = sl
 		return sequences, nil
 	}
 }
@@ -205,10 +458,16 @@ func _ParseGenbankFile(input ChannelFileChunk,
 	out obiiter.IBioSequence,
 	withFeatureTable, UtoT bool) {
 
-	parser := GenbankChunkParser(withFeatureTable, UtoT)
-
 	for chunks := range input {
-		sequences, err := parser(chunks.Source, chunks.Raw)
+		var sequences obiseq.BioSequenceSlice
+		var err error
+
+		if chunks.Rope != nil {
+			sequences, err = GenbankChunkParserRope(chunks.Source, chunks.Rope, withFeatureTable, UtoT)
+		} else {
+			parser := GenbankChunkParser(withFeatureTable, UtoT)
+			sequences, err = parser(chunks.Source, chunks.Raw)
+		}
 
 		if err != nil {
 			log.Fatalf("File %s : Cannot parse the genbank file : %v", chunks.Source, err)
@@ -224,7 +483,6 @@ func _ParseGenbankFile(input ChannelFileChunk,
 
 func ReadGenbank(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, error) {
 	opt := MakeOptions(options)
-	// entry_channel := make(chan _FileChunk)
 
 	entry_channel := ReadFileChunk(
 		opt.Source(),
@@ -232,13 +490,13 @@ func ReadGenbank(reader io.Reader, options ...WithOption) (obiiter.IBioSequence,
 		1024*1024*128,
 		EndOfLastFlatFileEntry,
 		"\nLOCUS       ",
+		false, // do not pack: rope-based parser avoids contiguous allocation
 	)
 
 	newIter := obiiter.MakeIBioSequence()
 
 	nworkers := opt.ParallelWorkers()
 
-	// for j := 0; j < opt.ParallelWorkers(); j++ {
 	for j := 0; j < nworkers; j++ {
 		newIter.Add(1)
 		go _ParseGenbankFile(
@@ -249,8 +507,6 @@ func ReadGenbank(reader io.Reader, options ...WithOption) (obiiter.IBioSequence,
 		)
 	}
 
-	// go _ReadFlatFileChunk(reader, entry_channel)
-
 	go func() {
 		newIter.WaitAndClose()
 		log.Debug("End of the genbank file ", opt.Source())

pkg/obiformats/json_read.go

@@ -0,0 +1,147 @@
+package obiformats
+
+import (
+	"io"
+	"os"
+	"path"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+	"github.com/buger/jsonparser"
+	"github.com/goccy/go-json"
+	log "github.com/sirupsen/logrus"
+)
+
+// _parse_json_record parses a single JSON object describing a sequence
+// (as produced by JSONRecord in json_writer.go) into a *obiseq.BioSequence.
+func _parse_json_record(raw []byte, shift byte) *obiseq.BioSequence {
+	sequence := obiseq.NewEmptyBioSequence(0)
+
+	if id, err := jsonparser.GetString(raw, "id"); err == nil {
+		sequence.SetId(id)
+	}
+
+	if seq, err := jsonparser.GetString(raw, "sequence"); err == nil {
+		sequence.SetSequence([]byte(seq))
+	}
+
+	if qual, err := jsonparser.GetString(raw, "qualities"); err == nil {
+		q := []byte(qual)
+		for i := 0; i < len(q); i++ {
+			q[i] -= shift
+		}
+		sequence.SetQualities(q)
+	}
+
+	if annot, dataType, _, err := jsonparser.Get(raw, "annotations"); err == nil && dataType == jsonparser.Object {
+		jsonparser.ObjectEach(annot,
+			func(key []byte, value []byte, valType jsonparser.ValueType, offset int) error {
+				return _parse_json_annotation_field(key, value, valType, sequence)
+			},
+		)
+	}
+
+	return sequence
+}
+
+// _ParseJsonFile streams the top-level JSON array, decoding and pushing one
+// batch of sequences at a time, without ever loading the whole document in
+// memory. Only one raw record at a time is buffered by the decoder.
+func _ParseJsonFile(source string,
+	reader io.Reader,
+	out obiiter.IBioSequence,
+	shift byte,
+	batchSize int) {
+
+	dec := json.NewDecoder(reader)
+
+	if _, err := dec.Token(); err != nil {
+		if err == io.EOF {
+			out.Done()
+			return
+		}
+		log.Fatalf("cannot parse JSON data: %v", err)
+	}
+
+	slice := obiseq.MakeBioSequenceSlice()
+	o := 0
+
+	for dec.More() {
+		var raw json.RawMessage
+
+		if err := dec.Decode(&raw); err != nil {
+			log.Fatalf("cannot parse JSON data: %v", err)
+		}
+
+		sequence := _parse_json_record(raw, shift)
+
+		slice = append(slice, sequence)
+		if len(slice) >= batchSize {
+			out.Push(obiiter.MakeBioSequenceBatch(source, o, slice))
+			o++
+			slice = obiseq.MakeBioSequenceSlice()
+		}
+	}
+
+	if len(slice) > 0 {
+		out.Push(obiiter.MakeBioSequenceBatch(source, o, slice))
+	}
+
+	out.Done()
+}
+
+func ReadJSON(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, error) {
+
+	opt := MakeOptions(options)
+	out := obiiter.MakeIBioSequence()
+
+	out.Add(1)
+	go _ParseJsonFile(opt.Source(),
+		reader,
+		out,
+		obidefault.ReadQualitiesShift(),
+		opt.BatchSize())
+
+	go func() {
+		out.WaitAndClose()
+	}()
+
+	return out, nil
+
+}
+
+func ReadJSONFromFile(filename string, options ...WithOption) (obiiter.IBioSequence, error) {
+
+	options = append(options, OptionsSource(obiutils.RemoveAllExt((path.Base(filename)))))
+	file, err := obiutils.Ropen(filename)
+
+	if err == obiutils.ErrNoContent {
+		log.Infof("file %s is empty", filename)
+		return ReadEmptyFile(options...)
+	}
+
+	if err != nil {
+		return obiiter.NilIBioSequence, err
+	}
+
+	return ReadJSON(file, options...)
+}
+
+func ReadJSONFromStdin(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, error) {
+	options = append(options, OptionsSource(obiutils.RemoveAllExt("stdin")))
+	input, err := obiutils.Buf(os.Stdin)
+
+	if err == obiutils.ErrNoContent {
+		log.Infof("stdin is empty")
+		return ReadEmptyFile(options...)
+	}
+
+	if err != nil {
+		log.Fatalf("open file error: %v", err)
+		return obiiter.NilIBioSequence, err
+	}
+
+	return ReadJSON(input, options...)
+}

pkg/obiformats/ngsfilter_read.go

@@ -631,9 +631,9 @@ func ReadCSVNGSFilter(reader io.Reader) (*obingslibrary.NGSLibrary, error) {
 			return nil, fmt.Errorf("row %d has %d columns, expected %d", len(data), len(fields), len(header))
 		}
 
-		forward_primer := fields[forward_primerColIndex]
+		forward_primer := strings.TrimSpace(fields[forward_primerColIndex])
-		reverse_primer := fields[reverse_primerColIndex]
+		reverse_primer := strings.TrimSpace(fields[reverse_primerColIndex])
-		tags := _parseMainNGSFilterTags(fields[sample_tagColIndex])
+		tags := _parseMainNGSFilterTags(strings.TrimSpace(fields[sample_tagColIndex]))
 
 		marker, _ := ngsfilter.GetMarker(forward_primer, reverse_primer)
 		pcr, ok := marker.GetPCR(tags.Forward, tags.Reverse)
@@ -644,8 +644,8 @@ func ReadCSVNGSFilter(reader io.Reader) (*obingslibrary.NGSLibrary, error) {
 					i, tags.Forward, tags.Reverse, forward_primer, reverse_primer)
 		}
 
-		pcr.Experiment = fields[experimentColIndex]
+		pcr.Experiment = strings.TrimSpace(fields[experimentColIndex])
-		pcr.Sample = fields[sampleColIndex]
+		pcr.Sample = strings.TrimSpace(fields[sampleColIndex])
 
 		if extraColumns != nil {
 			pcr.Annotations = make(obiseq.Annotation)

pkg/obiformats/options.go

@@ -35,6 +35,7 @@ type __options__ struct {
 	csv_auto              bool
 	paired_filename       string
 	source                string
+	filename              string
 	with_feature_table    bool
 	with_pattern          bool
 	with_parent           bool
@@ -216,6 +217,16 @@ func (opt Options) Source() string {
 	return opt.pointer.source
 }
 
+// FileName returns the full path of the file being read, for use in
+// diagnostic messages. It falls back to Source() when no explicit
+// file name has been set (e.g. reading from stdin or a raw reader).
+func (opt Options) FileName() string {
+	if opt.pointer.filename == "" {
+		return opt.pointer.source
+	}
+	return opt.pointer.filename
+}
+
 func (opt Options) WithFeatureTable() bool {
 	return opt.pointer.with_feature_table
 }
@@ -421,6 +432,14 @@ func OptionsSource(source string) WithOption {
 	return f
 }
 
+func OptionsFileName(filename string) WithOption {
+	f := WithOption(func(opt Options) {
+		opt.pointer.filename = filename
+	})
+
+	return f
+}
+
 func OptionsWithProgressBar() WithOption {
 	f := WithOption(func(opt Options) {
 		opt.pointer.with_progress_bar = true

pkg/obiformats/rope_scanner.go

@@ -0,0 +1,77 @@
+package obiformats
+
+import "bytes"
+
+// ropeScanner reads lines from a PieceOfChunk rope.
+// The carry buffer handles lines that span two rope nodes; it grows as needed.
+type ropeScanner struct {
+	current *PieceOfChunk
+	pos     int
+	carry   []byte
+}
+
+func newRopeScanner(rope *PieceOfChunk) *ropeScanner {
+	return &ropeScanner{current: rope}
+}
+
+// ReadLine returns the next line without the trailing \n (or \r\n).
+// Returns nil at end of rope. The returned slice aliases carry[] or the node
+// data and is valid only until the next ReadLine call.
+func (s *ropeScanner) ReadLine() []byte {
+	for {
+		if s.current == nil {
+			if len(s.carry) > 0 {
+				line := s.carry
+				s.carry = s.carry[:0]
+				return line
+			}
+			return nil
+		}
+
+		data := s.current.data[s.pos:]
+		idx := bytes.IndexByte(data, '\n')
+
+		if idx >= 0 {
+			var line []byte
+			if len(s.carry) == 0 {
+				line = data[:idx]
+			} else {
+				s.carry = append(s.carry, data[:idx]...)
+				line = s.carry
+				s.carry = s.carry[:0]
+			}
+			s.pos += idx + 1
+			if s.pos >= len(s.current.data) {
+				s.current = s.current.Next()
+				s.pos = 0
+			}
+			if len(line) > 0 && line[len(line)-1] == '\r' {
+				line = line[:len(line)-1]
+			}
+			return line
+		}
+
+		// No \n in this node: accumulate into carry and advance
+		s.carry = append(s.carry, data...)
+		s.current = s.current.Next()
+		s.pos = 0
+	}
+}
+
+// skipToNewline advances the scanner past the next '\n'.
+func (s *ropeScanner) skipToNewline() {
+	for s.current != nil {
+		data := s.current.data[s.pos:]
+		idx := bytes.IndexByte(data, '\n')
+		if idx >= 0 {
+			s.pos += idx + 1
+			if s.pos >= len(s.current.data) {
+				s.current = s.current.Next()
+				s.pos = 0
+			}
+			return
+		}
+		s.current = s.current.Next()
+		s.pos = 0
+	}
+}

pkg/obiformats/universal_read.go

@@ -145,6 +145,8 @@ func ReadSequencesFromFile(filename string,
 		return ReadGenbank(reader, options...)
 	case "text/csv":
 		return ReadCSV(reader, options...)
+	case "application/json":
+		return ReadJSON(reader, options...)
 	default:
 		log.Fatalf("File %s has guessed format %s which is not yet implemented",
 			filename, mime.String())

pkg/obifp/uint128_test.go

@@ -134,7 +134,7 @@ func TestUint128_QuoRem(t *testing.T) {
 	u := Uint128{w1: 3, w0: 8}
 	v := Uint128{w1: 0, w0: 4}
 	q, r := u.QuoRem(v)
-	assert.Equal(t, Uint128{w1: 0, w0: 2}, q)
+	assert.Equal(t, Uint128{w1: 0, w0: 13835058055282163714}, q)
 	assert.Equal(t, Uint128{w1: 0, w0: 0}, r)
 }
 
@@ -150,7 +150,7 @@ func TestUint128_Div(t *testing.T) {
 	u := Uint128{w1: 3, w0: 8}
 	v := Uint128{w1: 0, w0: 4}
 	q := u.Div(v)
-	assert.Equal(t, Uint128{w1: 0, w0: 2}, q)
+	assert.Equal(t, Uint128{w1: 0, w0: 13835058055282163714}, q)
 }
 
 func TestUint128_Div64(t *testing.T) {
@@ -183,7 +183,7 @@ func TestUint128_Cmp(t *testing.T) {
 func TestUint128_Cmp64(t *testing.T) {
 	u := Uint128{w1: 1, w0: 2}
 	v := uint64(3)
-	assert.Equal(t, -1, u.Cmp64(v))
+	assert.Equal(t, 1, u.Cmp64(v))
 }
 
 func TestUint128_Equals(t *testing.T) {

pkg/obigraph/clique.go

@@ -0,0 +1,343 @@
+package obigraph
+
+import (
+	"context"
+	"math/rand"
+	"sync"
+	"sync/atomic"
+)
+
+// NeighborFilter defines an implicit graph over the vertices 0..n-1.
+// It appends to out[:0] the vertices of candidates that are adjacent to v,
+// keeping their order, and returns the resulting slice. It is called
+// concurrently by several workers and must not modify candidates.
+//
+// Describing the adjacency by a filter, instead of a matrix, keeps the memory
+// in O(n) and lets the caller evaluate adjacency in bulk (SIMD, for instance).
+type NeighborFilter func(v int32, candidates, out []int32) []int32
+
+// CliqueOptions tunes FindClique.
+type CliqueOptions struct {
+	// Workers is the number of concurrent searches (default 1).
+	Workers int
+	// Seed seeds the random vertex orders of the restarts.
+	Seed int64
+	// Progress, if set, is called with the size of the best clique each time a
+	// larger one is found. Calls are serialized and made while a lock is held:
+	// it must return quickly and must not call FindClique.
+	Progress func(best int)
+
+	// ColorLimit enables the coloring bound for the candidate lists of at most
+	// ColorLimit vertices (0 disables it). The vertices of a list are split into
+	// classes of pairwise non-adjacent vertices: a clique takes at most one
+	// vertex per class, so the number of classes bounds its size. Computing the
+	// classes costs up to the square of the list length in calls to the filter,
+	// which is why only short lists are colored.
+	ColorLimit int
+
+	// mode forces the kind of search (tests only): 0 alternates, modeFocused
+	// and modeImproving restrict the search to one of them.
+	mode int
+}
+
+const (
+	modeFocused   = 1
+	modeImproving = 2
+)
+
+// CliqueResult is the outcome of FindClique.
+type CliqueResult struct {
+	// Clique is the largest clique found (a clique of size >= target if Reached).
+	Clique []int32
+	// Reached is true if len(Clique) >= target.
+	Reached bool
+	// Exhausted is true if the whole search space has been explored. If Reached
+	// is false, it proves that no clique of the target size exists.
+	Exhausted bool
+	// Maximum is true if Clique is proven to be a maximum clique of the graph:
+	// no larger clique exists.
+	Maximum bool
+}
+
+// FindClique looks for a clique of at least target vertices in the graph of n
+// vertices described by filter. It stops as soon as one is found, when the
+// search space is exhausted, or when ctx is done; in the last case, the largest
+// clique met so far is returned.
+//
+// The search is a depth first search with backtracking, which comes in two
+// flavours that the restarts alternate:
+//
+//   - the focused search prunes every branch that cannot reach target, which
+//     is the most efficient way to reach it but gives poor cliques when target
+//     is out of reach;
+//   - the improving search only prunes the branches that cannot beat the best
+//     clique found so far (by any worker), so it keeps improving that clique
+//     whatever target is. When it ends, the best clique is a maximum one.
+//
+// When a focused search proves that no clique of target vertices exists, every
+// search turns to the improving flavour, so that the largest clique is returned
+// (and proved to be so) instead of an arbitrary one.
+//
+// On short candidate lists (see CliqueOptions.ColorLimit), a coloring bound
+// prunes the branches that cannot be large enough, and orders the vertices so
+// that the most constrained ones are tried first.
+//
+// The first pass of the first worker follows the natural order of the vertices
+// (greedy "lexicographic" construction); every other pass restarts from a
+// random order, with a work budget that doubles at each restart, so that the
+// search is complete in the long run.
+func FindClique(ctx context.Context, n int, filter NeighborFilter, target int, opts CliqueOptions) CliqueResult {
+	if target <= 0 {
+		return CliqueResult{Reached: true, Exhausted: true}
+	}
+	if n <= 0 {
+		return CliqueResult{Exhausted: true}
+	}
+
+	workers := max(opts.Workers, 1)
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	var (
+		mutex     sync.Mutex
+		best      []int32
+		bestSize  atomic.Int32
+		exhausted bool
+		maximum   bool
+		wg        sync.WaitGroup
+		// set when a focused search has proved that target cannot be reached:
+		// every search then turns to finding the largest clique
+		unreachable atomic.Bool
+	)
+
+	publish := func(clique []int32) {
+		mutex.Lock()
+		if len(clique) > len(best) {
+			best = append(best[:0], clique...)
+			bestSize.Store(int32(len(best)))
+			if opts.Progress != nil {
+				opts.Progress(len(best))
+			}
+		}
+		mutex.Unlock()
+	}
+
+	for w := range workers {
+		wg.Go(func() {
+			s := &cliqueSearch{
+				ctx:         ctx,
+				filter:      filter,
+				target:      target,
+				bestSize:    &bestSize,
+				colorLimit:  opts.ColorLimit,
+				unreachable: &unreachable,
+				publish: func(c []int32) {
+					publish(c)
+					if len(c) >= target {
+						cancel()
+					}
+				},
+			}
+			rng := rand.New(rand.NewSource(opts.Seed + int64(w)))
+			// the search starts from the end of the list: lowest vertex first
+			candidates := make([]int32, n)
+			for i := range candidates {
+				candidates[i] = int32(n - 1 - i)
+			}
+
+			for restart := 0; ctx.Err() == nil; restart++ {
+				if w > 0 || restart > 0 {
+					rng.Shuffle(n, func(i, j int) { candidates[i], candidates[j] = candidates[j], candidates[i] })
+				}
+				switch opts.mode {
+				case modeFocused:
+					s.improving = false
+				case modeImproving:
+					s.improving = true
+				default:
+					s.improving = unreachable.Load() || (w+restart)%2 == 1
+				}
+				s.budget = int64(n) * int64(target) << min(restart, 20)
+				s.work = 0
+				s.aborted = false
+				s.search(0, candidates)
+
+				if !s.aborted && ctx.Err() == nil {
+					mutex.Lock()
+					exhausted = true
+					if s.improving {
+						maximum = true
+					}
+					mutex.Unlock()
+
+					if s.improving || opts.mode == modeFocused {
+						cancel()
+					} else {
+						// target is out of reach: look for the largest clique
+						unreachable.Store(true)
+					}
+				}
+			}
+		})
+	}
+	wg.Wait()
+
+	return CliqueResult{
+		Clique:    best,
+		Reached:   len(best) >= target,
+		Exhausted: exhausted,
+		Maximum:   maximum,
+	}
+}
+
+type cliqueSearch struct {
+	ctx        context.Context
+	filter     NeighborFilter
+	target     int
+	bestSize   *atomic.Int32 // size of the best clique found by any worker
+	publish    func([]int32)
+	colorLimit int
+
+	unreachable *atomic.Bool // target is proved out of reach
+
+	improving bool // see FindClique: the kind of the current pass
+
+	current []int32   // the clique under construction
+	buffers [][]int32 // candidate list of each depth
+	sorted  [][]int32 // candidate list of each depth, sorted by color
+	colors  [][]int32 // color (from 1) of each vertex of sorted
+	classes [][]int32 // scratch space of the coloring
+	probe   []int32   // scratch space of the coloring
+
+	budget  int64 // maximum work (number of candidates tested) for this restart
+	work    int64
+	aborted bool // budget or context exhausted before the end of the exploration
+}
+
+// threshold is the size a clique must reach to be worth exploring: the target
+// for the focused search, one more than the best clique for the improving one.
+func (s *cliqueSearch) threshold() int {
+	if s.improving {
+		return min(s.target, int(s.bestSize.Load())+1)
+	}
+	return s.target
+}
+
+// mustStop tells whether the current pass has to be interrupted: its work
+// budget is spent, or the pass must stop whatever its budget (see mustAbort).
+func (s *cliqueSearch) mustStop() bool {
+	return s.work > s.budget || s.mustAbort()
+}
+
+// mustAbort tells whether the pass has to be interrupted whatever its budget:
+// the context is done, or it is a focused pass and its target has been proved
+// out of reach.
+func (s *cliqueSearch) mustAbort() bool {
+	return s.ctx.Err() != nil || (!s.improving && s.unreachable.Load())
+}
+
+// colorSort splits candidates into classes of pairwise non-adjacent vertices
+// (greedy sequential coloring) and returns them class after class, with the
+// color of each vertex. Any clique takes at most one vertex of each class.
+//
+// The coloring can be long: it stops, and sets s.aborted, as soon as the search
+// has to be aborted. Its cost is not charged to the work budget of the pass,
+// which limits the exploration of the tree, not the bound computed at each node.
+func (s *cliqueSearch) colorSort(depth int, candidates []int32) (sorted, colors []int32) {
+	k := 0
+	for _, v := range candidates {
+		if s.mustAbort() {
+			s.aborted = true
+			return nil, nil
+		}
+		placed := false
+		for c := 0; c < k; c++ {
+			// v joins the class if it is adjacent to none of its members
+			s.probe = s.filter(v, s.classes[c], s.probe[:0])
+			if len(s.probe) == 0 {
+				s.classes[c] = append(s.classes[c], v)
+				placed = true
+				break
+			}
+		}
+		if !placed {
+			if k == len(s.classes) {
+				s.classes = append(s.classes, nil)
+			}
+			s.classes[k] = append(s.classes[k][:0], v)
+			k++
+		}
+	}
+
+	for len(s.sorted) <= depth {
+		s.sorted = append(s.sorted, nil)
+		s.colors = append(s.colors, nil)
+	}
+	sorted, colors = s.sorted[depth][:0], s.colors[depth][:0]
+	for c := 0; c < k; c++ {
+		for _, v := range s.classes[c] {
+			sorted = append(sorted, v)
+			colors = append(colors, int32(c+1))
+		}
+	}
+	s.sorted[depth], s.colors[depth] = sorted, colors
+	return sorted, colors
+}
+
+// search extends the current clique of depth vertices with vertices taken in
+// candidates, all adjacent to the vertices of the current clique. The list is
+// read from its end: the vertex at position i is combined with the vertices
+// before it, so that every clique is met once.
+func (s *cliqueSearch) search(depth int, candidates []int32) {
+	var colors []int32
+	if s.colorLimit > 0 && len(candidates) > 1 && len(candidates) <= s.colorLimit {
+		candidates, colors = s.colorSort(depth, candidates)
+		if s.aborted {
+			return
+		}
+	}
+
+	for i := len(candidates) - 1; i >= 0; i-- {
+		// largest clique the vertices 0..i can make
+		bound := i + 1
+		if colors != nil {
+			bound = int(colors[i])
+		}
+		if depth+bound < s.threshold() {
+			return
+		}
+		if s.mustStop() {
+			s.aborted = true
+			return
+		}
+
+		v := candidates[i]
+		s.current = append(s.current[:depth], v)
+		if len(s.current) > int(s.bestSize.Load()) {
+			s.publish(s.current)
+			if len(s.current) >= s.target {
+				s.aborted = true // the search is over: do not report an exhaustion
+				return
+			}
+		}
+
+		rest := candidates[:i]
+		if depth+1+len(rest) < s.threshold() {
+			continue
+		}
+		for len(s.buffers) <= depth+1 {
+			s.buffers = append(s.buffers, nil)
+		}
+		next := s.filter(v, rest, s.buffers[depth+1][:0])
+		s.buffers[depth+1] = next
+		s.work += int64(len(rest))
+
+		if depth+1+len(next) < s.threshold() {
+			continue
+		}
+		s.search(depth+1, next)
+		if s.aborted {
+			return
+		}
+	}
+}

pkg/obigraph/clique_test.go

@@ -0,0 +1,249 @@
+package obigraph
+
+import (
+	"context"
+	"math/bits"
+	"math/rand"
+	"testing"
+	"time"
+)
+
+// adjacency matrix filter for small graphs
+func matrixFilter(adj [][]bool) NeighborFilter {
+	return func(v int32, candidates, out []int32) []int32 {
+		for _, c := range candidates {
+			if adj[v][c] {
+				out = append(out, c)
+			}
+		}
+		return out
+	}
+}
+
+func randomGraph(rng *rand.Rand, n int, p float64) [][]bool {
+	adj := make([][]bool, n)
+	for i := range adj {
+		adj[i] = make([]bool, n)
+	}
+	for i := 0; i < n; i++ {
+		for j := i + 1; j < n; j++ {
+			if rng.Float64() < p {
+				adj[i][j], adj[j][i] = true, true
+			}
+		}
+	}
+	return adj
+}
+
+// maximum clique size by brute force over all the subsets (n <= 20)
+func bruteMaxClique(adj [][]bool) int {
+	n := len(adj)
+	best := 0
+	for mask := 1; mask < 1<<n; mask++ {
+		size := bits.OnesCount(uint(mask))
+		if size <= best {
+			continue
+		}
+		ok := true
+		for i := 0; i < n && ok; i++ {
+			if mask>>i&1 == 0 {
+				continue
+			}
+			for j := i + 1; j < n; j++ {
+				if mask>>j&1 == 1 && !adj[i][j] {
+					ok = false
+					break
+				}
+			}
+		}
+		if ok {
+			best = size
+		}
+	}
+	return best
+}
+
+func isClique(adj [][]bool, c []int32) bool {
+	seen := map[int32]bool{}
+	for i, a := range c {
+		if seen[a] {
+			return false
+		}
+		seen[a] = true
+		for _, b := range c[i+1:] {
+			if !adj[a][b] {
+				return false
+			}
+		}
+	}
+	return true
+}
+
+func TestFindCliqueAgainstBruteForce(t *testing.T) {
+	rng := rand.New(rand.NewSource(1))
+	for trial := 0; trial < 40; trial++ {
+		n := 8 + rng.Intn(10)
+		adj := randomGraph(rng, n, 0.3+0.5*rng.Float64())
+		maxSize := bruteMaxClique(adj)
+		filter := matrixFilter(adj)
+
+		for _, workers := range []int{1, 3} {
+			r := FindClique(context.Background(), n, filter, maxSize, CliqueOptions{Workers: workers})
+			if !r.Reached || len(r.Clique) < maxSize || !isClique(adj, r.Clique) {
+				t.Fatalf("trial %d: target %d not reached: %+v", trial, maxSize, r)
+			}
+
+			r = FindClique(context.Background(), n, filter, maxSize+1, CliqueOptions{Workers: workers})
+			if r.Reached || !r.Exhausted {
+				t.Fatalf("trial %d: size %d cannot exist, got %+v", trial, maxSize+1, r)
+			}
+			// best effort: a valid clique, not necessarily a maximum one
+			if len(r.Clique) < 1 || len(r.Clique) > maxSize || !isClique(adj, r.Clique) {
+				t.Fatalf("trial %d: invalid best clique %v (maximum is %d)", trial, r.Clique, maxSize)
+			}
+		}
+	}
+}
+
+func TestFindCliqueTrivialTargets(t *testing.T) {
+	adj := randomGraph(rand.New(rand.NewSource(2)), 5, 0.5)
+	if r := FindClique(context.Background(), 5, matrixFilter(adj), 0, CliqueOptions{}); !r.Reached {
+		t.Fatal("target 0 is always reached")
+	}
+	if r := FindClique(context.Background(), 0, matrixFilter(adj), 1, CliqueOptions{}); r.Reached {
+		t.Fatal("no clique in an empty graph")
+	}
+	if r := FindClique(context.Background(), 5, matrixFilter(adj), 1, CliqueOptions{}); !r.Reached || len(r.Clique) != 1 {
+		t.Fatalf("a single vertex is a clique: %+v", r)
+	}
+}
+
+// A hopeless search must stop with the deadline and return a valid clique.
+// (When the target is far above the clique number of a sparse graph, pruning
+// proves the absence at once: see TestFindCliqueAgainstBruteForce.)
+func TestFindCliqueTimeout(t *testing.T) {
+	// dense graph: its largest clique is far below the target, but pruning only
+	// bites after many levels, so the search cannot finish within the deadline
+	const n = 300
+	adj := randomGraph(rand.New(rand.NewSource(3)), n, 0.9)
+	ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond)
+	defer cancel()
+
+	start := time.Now()
+	r := FindClique(ctx, n, matrixFilter(adj), 150, CliqueOptions{Workers: 2})
+	if elapsed := time.Since(start); elapsed > 3*time.Second {
+		t.Fatalf("deadline not honoured: %v", elapsed)
+	}
+	if r.Reached || r.Exhausted {
+		t.Fatalf("size 150 cannot be reached nor proven absent in time: %+v", r)
+	}
+	if len(r.Clique) < 2 || !isClique(adj, r.Clique) {
+		t.Fatalf("invalid best clique %v", r.Clique)
+	}
+}
+
+func TestFindCliqueProgress(t *testing.T) {
+	adj := randomGraph(rand.New(rand.NewSource(4)), 40, 0.7)
+
+	var seen []int
+	r := FindClique(context.Background(), 40, matrixFilter(adj), 5, CliqueOptions{
+		Progress: func(best int) { seen = append(seen, best) },
+	})
+	if !r.Reached {
+		t.Fatalf("size 5 should be reached: %+v", r)
+	}
+	if len(seen) == 0 || seen[len(seen)-1] != len(r.Clique) {
+		t.Fatalf("progress %v does not end at the clique size %d", seen, len(r.Clique))
+	}
+	for i := 1; i < len(seen); i++ {
+		if seen[i] <= seen[i-1] {
+			t.Fatalf("progress must be strictly increasing: %v", seen)
+		}
+	}
+}
+
+// The improving search must end with a proven maximum clique, even when the
+// target is out of reach; the focused one only proves the absence of a clique
+// of the target size.
+func TestFindCliqueModes(t *testing.T) {
+	rng := rand.New(rand.NewSource(5))
+	for trial := 0; trial < 40; trial++ {
+		n := 8 + rng.Intn(10)
+		adj := randomGraph(rng, n, 0.3+0.5*rng.Float64())
+		maxSize := bruteMaxClique(adj)
+		filter := matrixFilter(adj)
+
+		for _, workers := range []int{1, 3} {
+			for _, limit := range []int{0, 5, 1 << 20} {
+				r := FindClique(context.Background(), n, filter, maxSize+1,
+					CliqueOptions{Workers: workers, ColorLimit: limit, mode: modeImproving})
+				if r.Reached || !r.Exhausted || !r.Maximum {
+					t.Fatalf("trial %d limit %d: improving search should prove the maximum: %+v", trial, limit, r)
+				}
+				if len(r.Clique) != maxSize || !isClique(adj, r.Clique) {
+					t.Fatalf("trial %d limit %d: expected a maximum clique of %d, got %v", trial, limit, maxSize, r.Clique)
+				}
+
+				r = FindClique(context.Background(), n, filter, maxSize+1,
+					CliqueOptions{Workers: workers, ColorLimit: limit, mode: modeFocused})
+				if r.Reached || !r.Exhausted || r.Maximum {
+					t.Fatalf("trial %d limit %d: focused search proves absence only: %+v", trial, limit, r)
+				}
+
+				r = FindClique(context.Background(), n, filter, maxSize,
+					CliqueOptions{Workers: workers, ColorLimit: limit, mode: modeImproving})
+				if !r.Reached || len(r.Clique) < maxSize || !isClique(adj, r.Clique) {
+					t.Fatalf("trial %d limit %d: target %d not reached by the improving search: %+v", trial, limit, maxSize, r)
+				}
+			}
+		}
+	}
+}
+
+// When the target is out of reach, the default search must end with a proven
+// maximum clique, even if the coloring bound proves the absence of the target
+// before any clique has been found.
+func TestFindCliqueUnreachableTargetGivesMaximum(t *testing.T) {
+	rng := rand.New(rand.NewSource(6))
+	for trial := 0; trial < 40; trial++ {
+		n := 8 + rng.Intn(10)
+		adj := randomGraph(rng, n, 0.3+0.5*rng.Float64())
+		maxSize := bruteMaxClique(adj)
+
+		for _, workers := range []int{1, 3} {
+			for _, limit := range []int{0, 5, 1 << 20} {
+				r := FindClique(context.Background(), n, matrixFilter(adj), maxSize+1,
+					CliqueOptions{Workers: workers, ColorLimit: limit})
+				if r.Reached || !r.Exhausted || !r.Maximum {
+					t.Fatalf("trial %d workers %d limit %d: expected a proven maximum: %+v", trial, workers, limit, r)
+				}
+				if len(r.Clique) != maxSize || !isClique(adj, r.Clique) {
+					t.Fatalf("trial %d workers %d limit %d: expected a maximum clique of %d, got %v",
+						trial, workers, limit, maxSize, r.Clique)
+				}
+			}
+		}
+	}
+}
+
+// A long coloring must not delay the deadline.
+func TestFindCliqueColoringHonoursDeadline(t *testing.T) {
+	const n = 3000
+	adj := randomGraph(rand.New(rand.NewSource(7)), n, 0.5)
+	base := matrixFilter(adj)
+	slow := func(v int32, candidates, out []int32) []int32 {
+		time.Sleep(200 * time.Microsecond)
+		return base(v, candidates, out)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond)
+	defer cancel()
+	start := time.Now()
+	r := FindClique(ctx, n, slow, 100, CliqueOptions{Workers: 2, ColorLimit: 1 << 20})
+	if elapsed := time.Since(start); elapsed > 2*time.Second {
+		t.Fatalf("deadline not honoured by the coloring: %v", elapsed)
+	}
+	if r.Reached || !isClique(adj, r.Clique) {
+		t.Fatalf("unexpected result %+v", r)
+	}
+}

pkg/obiiter/batchiterator.go

@@ -444,6 +444,69 @@ func (iterator IBioSequence) Rebatch(size int) IBioSequence {
 	return newIter
 }
 
+// RebatchBySize reorganises the stream into batches bounded by two independent
+// upper limits: maxCount (max number of sequences) and maxBytes (max cumulative
+// estimated memory). A batch is flushed as soon as either limit would be
+// exceeded. A single sequence larger than maxBytes is always emitted alone.
+// Passing 0 for a limit disables that constraint; if both are 0 it falls back
+// to Rebatch(obidefault.BatchSizeMax()).
+func (iterator IBioSequence) RebatchBySize(maxBytes int, maxCount int) IBioSequence {
+	if maxBytes <= 0 && maxCount <= 0 {
+		return iterator.Rebatch(obidefault.BatchSizeMax())
+	}
+
+	newIter := MakeIBioSequence()
+
+	newIter.Add(1)
+
+	go func() {
+		newIter.WaitAndClose()
+	}()
+
+	go func() {
+		order := 0
+		// a local variable: reassigning the captured iterator would race with
+		// the IsPaired call below, made by the caller on the same variable
+		sorted := iterator.SortBatches()
+		buffer := obiseq.MakeBioSequenceSlice()
+		bufBytes := 0
+		source := ""
+
+		flush := func() {
+			if len(buffer) > 0 {
+				newIter.Push(MakeBioSequenceBatch(source, order, buffer))
+				order++
+				buffer = obiseq.MakeBioSequenceSlice()
+				bufBytes = 0
+			}
+		}
+
+		for sorted.Next() {
+			seqs := sorted.Get()
+			source = seqs.Source()
+			for _, s := range seqs.Slice() {
+				sz := s.MemorySize()
+				countFull := maxCount > 0 && len(buffer) >= maxCount
+				memFull := maxBytes > 0 && bufBytes+sz > maxBytes && len(buffer) > 0
+				if countFull || memFull {
+					flush()
+				}
+				buffer = append(buffer, s)
+				bufBytes += sz
+			}
+		}
+		flush()
+
+		newIter.Done()
+	}()
+
+	if iterator.IsPaired() {
+		newIter.MarkAsPaired()
+	}
+
+	return newIter
+}
+
 func (iterator IBioSequence) FilterEmpty() IBioSequence {
 
 	newIter := MakeIBioSequence()
@@ -622,7 +685,8 @@ func (iterator IBioSequence) FilterOn(predicate obiseq.SequencePredicate,
 			}
 
 			seqs.slice = slice[:j]
-			trueIter.pointer.channel <- seqs
+			trueIter.Push(seqs)
+			// trueIter.pointer.channel <- seqs
 		}
 
 		trueIter.Done()
@@ -638,7 +702,7 @@ func (iterator IBioSequence) FilterOn(predicate obiseq.SequencePredicate,
 		trueIter.MarkAsPaired()
 	}
 
-	return trueIter.Rebatch(size)
+	return trueIter.RebatchBySize(obidefault.BatchMem(), obidefault.BatchSizeMax())
 }
 
 func (iterator IBioSequence) FilterAnd(predicate obiseq.SequencePredicate,
@@ -694,7 +758,7 @@ func (iterator IBioSequence) FilterAnd(predicate obiseq.SequencePredicate,
 		trueIter.MarkAsPaired()
 	}
 
-	return trueIter.Rebatch(size)
+	return trueIter.RebatchBySize(obidefault.BatchMem(), obidefault.BatchSizeMax())
 }
 
 // Load all sequences availables from an IBioSequenceBatch iterator into

pkg/obiiter/distribute.go

@@ -57,34 +57,21 @@ func (dist *IDistribute) Classifier() *obiseq.BioSequenceClassifier {
 }
 
 // Distribute organizes the biosequences from the iterator into batches
-// based on the provided classifier and batch sizes. It returns an
+// based on the provided classifier. It returns an IDistribute instance
-// IDistribute instance that manages the distribution of the sequences.
+// that manages the distribution of the sequences.
 //
-// Parameters:
+// Batches are flushed when either BatchSizeMax() sequences or BatchMem()
-//   - class: A pointer to a BioSequenceClassifier used to classify
+// bytes are accumulated per key, mirroring the RebatchBySize strategy.
-//     the biosequences during distribution.
+func (iterator IBioSequence) Distribute(class *obiseq.BioSequenceClassifier) IDistribute {
-//   - sizes: Optional integer values specifying the batch size. If
+	maxCount := obidefault.BatchSizeMax()
-//     no sizes are provided, a default batch size of 5000 is used.
+	maxBytes := obidefault.BatchMem()
-//
-// Returns:
-// An IDistribute instance that contains the outputs of the
-// classified biosequences, a channel for new data notifications,
-// and the classifier used for distribution. The method operates
-// asynchronously, processing the sequences in separate goroutines.
-// It ensures that the outputs are closed and cleaned up once
-// processing is complete.
-func (iterator IBioSequence) Distribute(class *obiseq.BioSequenceClassifier, sizes ...int) IDistribute {
-	batchsize := obidefault.BatchSize()
 
 	outputs := make(map[int]IBioSequence, 100)
 	slices := make(map[int]*obiseq.BioSequenceSlice, 100)
+	bufBytes := make(map[int]int, 100)
 	orders := make(map[int]int, 100)
 	news := make(chan int)
 
-	if len(sizes) > 0 {
-		batchsize = sizes[0]
-	}
-
 	jobDone := sync.WaitGroup{}
 	lock := sync.Mutex{}
 
@@ -115,6 +102,7 @@ func (iterator IBioSequence) Distribute(class *obiseq.BioSequenceClassifier, siz
 					slice = &s
 					slices[key] = slice
 					orders[key] = 0
+					bufBytes[key] = 0
 
 					lock.Lock()
 					outputs[key] = MakeIBioSequence()
@@ -123,14 +111,20 @@ func (iterator IBioSequence) Distribute(class *obiseq.BioSequenceClassifier, siz
 					news <- key
 				}
 
-				*slice = append(*slice, s)
+				sz := s.MemorySize()
-
+				countFull := maxCount > 0 && len(*slice) >= maxCount
-				if len(*slice) == batchsize {
+				memFull := maxBytes > 0 && bufBytes[key]+sz > maxBytes && len(*slice) > 0
+				if countFull || memFull {
 					outputs[key].Push(MakeBioSequenceBatch(source, orders[key], *slice))
 					orders[key]++
 					s := obiseq.MakeBioSequenceSlice()
 					slices[key] = &s
+					slice = &s
+					bufBytes[key] = 0
 				}
+
+				*slice = append(*slice, s)
+				bufBytes[key] += sz
 			}
 		}
 

pkg/obiiter/fragment.go

@@ -3,6 +3,7 @@ package obiiter
 import (
 	log "github.com/sirupsen/logrus"
 
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 )
 
@@ -70,7 +71,7 @@ func IFragments(minsize, length, overlap, size, nworkers int) Pipeable {
 		}
 		go f(iterator)
 
-		return newiter.SortBatches().Rebatch(size)
+		return newiter.SortBatches().RebatchBySize(obidefault.BatchMem(), obidefault.BatchSizeMax())
 	}
 
 	return ifrg

pkg/obiiter/speed.go

@@ -5,18 +5,30 @@ import (
 	"os"
 	"time"
 
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"github.com/schollz/progressbar/v3"
 )
 
 func (iterator IBioSequence) Speed(message string, size ...int) IBioSequence {
 
-	// If the STDERR is redicted and doesn't end up to a terminal
+	// If the progress bar is disabled via --no-progressbar option
+	if !obidefault.ProgressBar() {
+		return iterator
+	}
+
+	// If the STDERR is redirected and doesn't end up to a terminal
 	// No progress bar is printed.
 	o, _ := os.Stderr.Stat()
 	if (o.Mode() & os.ModeCharDevice) != os.ModeCharDevice {
 		return iterator
 	}
 
+	// If stdout is piped, no progress bar is printed.
+	oo, _ := os.Stdout.Stat()
+	if (oo.Mode() & os.ModeNamedPipe) == os.ModeNamedPipe {
+		return iterator
+	}
+
 	newIter := MakeIBioSequence()
 
 	newIter.Add(1)

pkg/obikmer/debruijn.go

@@ -5,7 +5,6 @@ import (
 	"container/heap"
 	"fmt"
 	"math"
-	"math/bits"
 	"os"
 	"slices"
 
@@ -610,14 +609,6 @@ func (graph *DeBruijnGraph) WriteGml(filename string) error {
 	return err
 }
 
-// Calculating the hamming distance between two k-mers.
-func (g *DeBruijnGraph) HammingDistance(kmer1, kmer2 uint64) int {
-	ident := ^((kmer1 & kmer2) | (^kmer1 & ^kmer2))
-	ident |= (ident >> 1)
-	ident &= 0x5555555555555555 & g.kmermask
-	return bits.OnesCount64(ident)
-}
-
 type UInt64Heap []uint64
 
 func (h UInt64Heap) Len() int           { return len(h) }

pkg/obikmer/encodefourmer.go

@@ -47,7 +47,7 @@ func Encode4mer(seq *obiseq.BioSequence, buffer *[]byte) []byte {
 	length := slength - 3
 	rawseq := seq.Sequence()
 
-	if length < 0 {
+	if length <= 0 {
 		return nil
 	}
 

pkg/obikmer/encodekmer.go

@@ -447,141 +447,6 @@ func IterCanonicalKmers(seq []byte, k int) iter.Seq[uint64] {
 		}
 	}
 }
-
-// SuperKmer represents a maximal subsequence where all consecutive k-mers
-// share the same minimizer. A minimizer is the smallest canonical m-mer
-// among the (k-m+1) m-mers contained in a k-mer.
-type SuperKmer struct {
-	Minimizer uint64 // The canonical minimizer value (normalized m-mer)
-	Start     int    // Starting position in the original sequence (0-indexed)
-	End       int    // Ending position (exclusive, like Go slice notation)
-	Sequence  []byte // The actual DNA subsequence [Start:End]
-}
-
-// dequeItem represents an element in the monotone deque used for
-// tracking minimizers in a sliding window.
-type dequeItem struct {
-	position  int    // Position of the m-mer in the sequence
-	canonical uint64 // Canonical (normalized) m-mer value
-}
-
-// ExtractSuperKmers extracts super k-mers from a DNA sequence.
-// A super k-mer is a maximal subsequence where all consecutive k-mers
-// share the same minimizer. The minimizer of a k-mer is the smallest
-// canonical m-mer among its (k-m+1) constituent m-mers.
-//
-// The algorithm uses:
-// - Simultaneous forward/reverse m-mer encoding for O(1) canonical m-mer computation
-// - Monotone deque for O(1) amortized minimizer tracking per position
-//
-// The maximum k-mer size is 31 (using 62 bits), leaving the top 2 bits
-// available for error markers if needed.
-//
-// Parameters:
-//   - seq: DNA sequence as a byte slice (case insensitive, supports A, C, G, T, U)
-//   - k: k-mer size (must be between m+1 and 31)
-//   - m: minimizer size (must be between 1 and k-1)
-//   - buffer: optional pre-allocated buffer for results. If nil, a new slice is created.
-//
-// Returns:
-//   - slice of SuperKmer structs representing maximal subsequences
-//   - nil if parameters are invalid or sequence is too short
-//
-// Time complexity: O(n) where n is the sequence length
-// Space complexity: O(k-m+1) for the deque + O(number of super k-mers) for results
-func ExtractSuperKmers(seq []byte, k int, m int, buffer *[]SuperKmer) []SuperKmer {
-	if m < 1 || m >= k || k < 2 || k > 31 || len(seq) < k {
-		return nil
-	}
-
-	var result []SuperKmer
-	if buffer == nil {
-		estimatedSize := len(seq) / k
-		if estimatedSize < 1 {
-			estimatedSize = 1
-		}
-		result = make([]SuperKmer, 0, estimatedSize)
-	} else {
-		result = (*buffer)[:0]
-	}
-
-	deque := make([]dequeItem, 0, k-m+1)
-
-	mMask := uint64(1)<<(m*2) - 1
-	rcShift := uint((m - 1) * 2)
-
-	var fwdMmer, rvcMmer uint64
-	for i := 0; i < m-1 && i < len(seq); i++ {
-		code := uint64(__single_base_code__[seq[i]&31])
-		fwdMmer = (fwdMmer << 2) | code
-		rvcMmer = (rvcMmer >> 2) | ((code ^ 3) << rcShift)
-	}
-
-	superKmerStart := 0
-	var currentMinimizer uint64
-	firstKmer := true
-
-	for pos := m - 1; pos < len(seq); pos++ {
-		code := uint64(__single_base_code__[seq[pos]&31])
-		fwdMmer = ((fwdMmer << 2) | code) & mMask
-		rvcMmer = (rvcMmer >> 2) | ((code ^ 3) << rcShift)
-
-		canonical := fwdMmer
-		if rvcMmer < fwdMmer {
-			canonical = rvcMmer
-		}
-
-		mmerPos := pos - m + 1
-
-		if pos >= k-1 {
-			windowStart := pos - k + 1
-			for len(deque) > 0 && deque[0].position < windowStart {
-				deque = deque[1:]
-			}
-		}
-
-		for len(deque) > 0 && deque[len(deque)-1].canonical >= canonical {
-			deque = deque[:len(deque)-1]
-		}
-
-		deque = append(deque, dequeItem{position: mmerPos, canonical: canonical})
-
-		if pos >= k-1 {
-			newMinimizer := deque[0].canonical
-			kmerStart := pos - k + 1
-
-			if firstKmer {
-				currentMinimizer = newMinimizer
-				firstKmer = false
-			} else if newMinimizer != currentMinimizer {
-				endPos := kmerStart + k - 1
-				superKmer := SuperKmer{
-					Minimizer: currentMinimizer,
-					Start:     superKmerStart,
-					End:       endPos,
-					Sequence:  seq[superKmerStart:endPos],
-				}
-				result = append(result, superKmer)
-
-				superKmerStart = kmerStart
-				currentMinimizer = newMinimizer
-			}
-		}
-	}
-
-	if !firstKmer {
-		superKmer := SuperKmer{
-			Minimizer: currentMinimizer,
-			Start:     superKmerStart,
-			End:       len(seq),
-			Sequence:  seq[superKmerStart:],
-		}
-		result = append(result, superKmer)
-	}
-
-	return result
-}
-
 // ReverseComplement computes the reverse complement of an encoded k-mer.
 // The k-mer is encoded with 2 bits per nucleotide (A=00, C=01, G=10, T=11).
 // The complement is: A↔T (00↔11), C↔G (01↔10), which is simply XOR with 11.

pkg/obikmer/entropy.go

@@ -0,0 +1,316 @@
+package obikmer
+
+import "math"
+
+// KmerEntropy computes the entropy of a single encoded k-mer.
+//
+// The algorithm mirrors the Rust obiskbuilder entropy: it decodes the k-mer
+// to a DNA sequence, extracts all sub-words of each size from 1 to levelMax,
+// normalizes them by circular canonical form, counts their frequencies, and
+// computes Shannon entropy corrected for class sizes, normalized by the
+// maximum possible entropy over 4^ws raw bins.
+// The returned value is the minimum entropy across all word sizes.
+//
+// Correction for small sequences: the raw entropy H = log(N) - Σ f·log(f)/N
+// under-estimates the true complexity when many raw words collapse to the same
+// canonical form.  Adding Σ f·log(class_size)/N recovers the entropy of the
+// underlying uncollapsed distribution (assuming uniform mixing within each
+// equivalence class).
+//
+// A value close to 0 indicates very low complexity (e.g. "AAAA..."),
+// while a value close to 1 indicates high complexity.
+func KmerEntropy(kmer uint64, k int, levelMax int) float64 {
+	if k < 1 || levelMax < 1 {
+		return 1.0
+	}
+	if levelMax >= k {
+		levelMax = k - 1
+	}
+	if levelMax < 1 {
+		return 1.0
+	}
+
+	// Decode k-mer to DNA sequence
+	var seqBuf [32]byte
+	seq := DecodeKmer(kmer, k, seqBuf[:])
+
+	// Pre-compute nLogN lookup
+	nLogN := make([]float64, k+1)
+	for i := 1; i <= k; i++ {
+		nLogN[i] = float64(i) * math.Log(float64(i))
+	}
+
+	// Build circular-canonical normalization tables per word size
+	normTables := make([][]int, levelMax+1)
+	for ws := 1; ws <= levelMax; ws++ {
+		size := 1 << (ws * 2)
+		normTables[ws] = make([]int, size)
+		for code := 0; code < size; code++ {
+			normTables[ws][code] = int(NormalizeCircular(uint64(code), ws))
+		}
+	}
+
+	// Build ln(class_size) tables: for each canonical form, how many raw
+	// words map to it under circular normalization.
+	classLogSizeTables := make([][]float64, levelMax+1)
+	for ws := 1; ws <= levelMax; ws++ {
+		tableSize := 1 << (ws * 2)
+		classSize := make([]int, tableSize)
+		for code := 0; code < tableSize; code++ {
+			classSize[normTables[ws][code]]++
+		}
+		classLogSizeTables[ws] = make([]float64, tableSize)
+		for j := 0; j < tableSize; j++ {
+			if classSize[j] > 0 {
+				classLogSizeTables[ws][j] = math.Log(float64(classSize[j]))
+			}
+		}
+	}
+
+	minEntropy := math.MaxFloat64
+
+	for ws := 1; ws <= levelMax; ws++ {
+		nwords := k - ws + 1
+		if nwords < 1 {
+			continue
+		}
+
+		// Count circular-canonical sub-word frequencies
+		tableSize := 1 << (ws * 2)
+		table := make([]int, tableSize)
+		mask := (1 << (ws * 2)) - 1
+
+		wordIndex := 0
+		for i := 0; i < ws-1; i++ {
+			wordIndex = (wordIndex << 2) + int(EncodeNucleotide(seq[i]))
+		}
+
+		for i, j := 0, ws-1; j < k; i, j = i+1, j+1 {
+			wordIndex = ((wordIndex << 2) & mask) + int(EncodeNucleotide(seq[j]))
+			normWord := normTables[ws][wordIndex]
+			table[normWord]++
+		}
+
+		// Compute emax over 4^ws raw bins (uncollapsed distribution).
+		floatNwords := float64(nwords)
+		logNwords := math.Log(floatNwords)
+		na := tableSize // 4^ws
+		var emax float64
+		if nwords < na {
+			emax = logNwords
+		} else {
+			cov := nwords / na
+			remains := nwords - (na * cov)
+			f1 := float64(cov) / floatNwords
+			f2 := float64(cov+1) / floatNwords
+			emax = -(float64(na-remains)*f1*math.Log(f1) +
+				float64(remains)*f2*math.Log(f2))
+		}
+
+		if emax <= 0 {
+			continue
+		}
+
+		// Accumulate Σ f·log(f) and Σ f·log(class_size) over canonical forms.
+		classLogSize := classLogSizeTables[ws]
+		var sumNLogN, sumClassLogN float64
+		for j := 0; j < tableSize; j++ {
+			n := table[j]
+			if n > 0 {
+				sumNLogN += nLogN[n]
+				sumClassLogN += float64(n) * classLogSize[j]
+			}
+		}
+
+		// Corrected entropy: H_raw ≈ log(N) + (Σf·log(s) - Σf·log(f)) / N
+		entropy := (logNwords + sumClassLogN/floatNwords - sumNLogN/floatNwords) / emax
+		if entropy < 0 {
+			entropy = 0
+		}
+
+		if entropy < minEntropy {
+			minEntropy = entropy
+		}
+	}
+
+	if minEntropy == math.MaxFloat64 {
+		return 1.0
+	}
+
+	return math.Round(minEntropy*10000) / 10000
+}
+
+// KmerEntropyFilter is a reusable entropy filter for batch processing.
+// It pre-computes normalization tables and lookup values to avoid repeated
+// allocation across millions of k-mers.
+//
+// IMPORTANT: a KmerEntropyFilter is NOT safe for concurrent use.
+// Each goroutine must create its own instance via NewKmerEntropyFilter.
+type KmerEntropyFilter struct {
+	k                   int
+	levelMax            int
+	threshold           float64
+	nLogN               []float64
+	normTables          [][]int
+	classLogSizeTables  [][]float64
+	emaxValues          []float64
+	logNwords           []float64
+	// Pre-allocated frequency tables reused across Entropy() calls.
+	// One per word size (index 0 unused). Reset to zero before each use.
+	freqTables [][]int
+}
+
+// NewKmerEntropyFilter creates an entropy filter with pre-computed tables.
+func NewKmerEntropyFilter(k, levelMax int, threshold float64) *KmerEntropyFilter {
+	if levelMax >= k {
+		levelMax = k - 1
+	}
+	if levelMax < 1 {
+		levelMax = 1
+	}
+
+	nLogN := make([]float64, k+1)
+	for i := 1; i <= k; i++ {
+		nLogN[i] = float64(i) * math.Log(float64(i))
+	}
+
+	normTables := make([][]int, levelMax+1)
+	for ws := 1; ws <= levelMax; ws++ {
+		size := 1 << (ws * 2)
+		normTables[ws] = make([]int, size)
+		for code := 0; code < size; code++ {
+			normTables[ws][code] = int(NormalizeCircular(uint64(code), ws))
+		}
+	}
+
+	// ln(class_size) for each canonical form under circular normalization.
+	classLogSizeTables := make([][]float64, levelMax+1)
+	for ws := 1; ws <= levelMax; ws++ {
+		tableSize := 1 << (ws * 2)
+		classSize := make([]int, tableSize)
+		for code := 0; code < tableSize; code++ {
+			classSize[normTables[ws][code]]++
+		}
+		classLogSizeTables[ws] = make([]float64, tableSize)
+		for j := 0; j < tableSize; j++ {
+			if classSize[j] > 0 {
+				classLogSizeTables[ws][j] = math.Log(float64(classSize[j]))
+			}
+		}
+	}
+
+	// Pre-compute emax and logNwords per word size.
+	// emax uses 4^ws raw bins to match the corrected entropy.
+	emaxValues := make([]float64, levelMax+1)
+	logNwords := make([]float64, levelMax+1)
+	for ws := 1; ws <= levelMax; ws++ {
+		nw := k - ws + 1
+		na := 1 << (ws * 2) // 4^ws raw bins
+		floatNw := float64(nw)
+		logNwords[ws] = math.Log(floatNw)
+		if nw < na {
+			emaxValues[ws] = logNwords[ws]
+		} else {
+			cov := nw / na
+			remains := nw - (na * cov)
+			f1 := float64(cov) / floatNw
+			f2 := float64(cov+1) / floatNw
+			emaxValues[ws] = -(float64(na-remains)*f1*math.Log(f1) +
+				float64(remains)*f2*math.Log(f2))
+		}
+	}
+
+	// Pre-allocate frequency tables per word size
+	freqTables := make([][]int, levelMax+1)
+	for ws := 1; ws <= levelMax; ws++ {
+		freqTables[ws] = make([]int, 1<<(ws*2))
+	}
+
+	return &KmerEntropyFilter{
+		k:                  k,
+		levelMax:           levelMax,
+		threshold:          threshold,
+		nLogN:              nLogN,
+		normTables:         normTables,
+		classLogSizeTables: classLogSizeTables,
+		emaxValues:         emaxValues,
+		logNwords:          logNwords,
+		freqTables:         freqTables,
+	}
+}
+
+// Accept returns true if the k-mer has entropy strictly above the threshold.
+// Low-complexity k-mers (entropy <= threshold) are rejected.
+func (ef *KmerEntropyFilter) Accept(kmer uint64) bool {
+	return ef.Entropy(kmer) > ef.threshold
+}
+
+// Entropy computes the entropy for a single k-mer using pre-computed tables.
+func (ef *KmerEntropyFilter) Entropy(kmer uint64) float64 {
+	k := ef.k
+
+	// Decode k-mer to DNA sequence
+	var seqBuf [32]byte
+	seq := DecodeKmer(kmer, k, seqBuf[:])
+
+	minEntropy := math.MaxFloat64
+
+	for ws := 1; ws <= ef.levelMax; ws++ {
+		nwords := k - ws + 1
+		if nwords < 1 {
+			continue
+		}
+
+		emax := ef.emaxValues[ws]
+		if emax <= 0 {
+			continue
+		}
+
+		// Count circular-canonical sub-word frequencies
+		tableSize := 1 << (ws * 2)
+		table := ef.freqTables[ws]
+		clear(table)
+		mask := (1 << (ws * 2)) - 1
+		normTable := ef.normTables[ws]
+
+		wordIndex := 0
+		for i := 0; i < ws-1; i++ {
+			wordIndex = (wordIndex << 2) + int(EncodeNucleotide(seq[i]))
+		}
+
+		for i, j := 0, ws-1; j < k; i, j = i+1, j+1 {
+			wordIndex = ((wordIndex << 2) & mask) + int(EncodeNucleotide(seq[j]))
+			normWord := normTable[wordIndex]
+			table[normWord]++
+		}
+
+		floatNwords := float64(nwords)
+		logNwords := ef.logNwords[ws]
+		classLogSize := ef.classLogSizeTables[ws]
+
+		var sumNLogN, sumClassLogN float64
+		for j := 0; j < tableSize; j++ {
+			n := table[j]
+			if n > 0 {
+				sumNLogN += ef.nLogN[n]
+				sumClassLogN += float64(n) * classLogSize[j]
+			}
+		}
+
+		// Corrected entropy: H_raw ≈ log(N) + (Σf·log(s) - Σf·log(f)) / N
+		entropy := (logNwords + sumClassLogN/floatNwords - sumNLogN/floatNwords) / emax
+		if entropy < 0 {
+			entropy = 0
+		}
+
+		if entropy < minEntropy {
+			minEntropy = entropy
+		}
+	}
+
+	if minEntropy == math.MaxFloat64 {
+		return 1.0
+	}
+
+	return math.Round(minEntropy*10000) / 10000
+}

pkg/obikmer/enumkmer.go

@@ -0,0 +1,129 @@
+package obikmer
+
+import "iter"
+
+// KmerConstraints describes lexicographic properties a k-mer must satisfy.
+//
+// MaxCount is indexed by the 2-bit base code (A=0, C=1, G=2, T=3):
+// a negative value means unlimited, 0 forbids the base altogether
+// (so MaxCount[2] = 0 restricts the alphabet to A, C, T).
+//
+// MaxGC is the maximum number of C or G bases; a negative value means
+// unlimited.
+//
+// MaxRun is the maximum number of consecutive identical bases;
+// a value <= 0 means unlimited. MinRun requires at least one run of that
+// length; a value <= 1 means no requirement.
+//
+// Accepted and Rejected are optional per-position sets of bases (4 bits masks,
+// bit b set when the base of code b is in the set, see ParseIUPACMasks).
+// When set, their length must be k. A k-mer is accepted only if each of its
+// bases belongs to the Accepted set of its position, and is refused if every
+// one of its bases belongs to the Rejected set of its position.
+//
+// The zero value forbids every base: use NoKmerConstraints as a starting point.
+type KmerConstraints struct {
+	MaxCount [4]int
+	MaxGC    int
+	MaxRun   int
+	MinRun   int
+	Accepted []uint8
+	Rejected []uint8
+}
+
+// NoKmerConstraints returns constraints accepting every k-mer.
+func NoKmerConstraints() KmerConstraints {
+	return KmerConstraints{MaxCount: [4]int{-1, -1, -1, -1}, MaxGC: -1}
+}
+
+// IterConstrainedKmers returns an iterator over all k-mers of size k
+// satisfying the constraints c, encoded as in EncodeKmer, in increasing
+// numerical order.
+//
+// K-mers are built base by base and a branch is abandoned as soon as a
+// constraint is violated or cannot be met any more (MaxCount, MaxGC, MaxRun
+// and Accepted are monotone: a violating prefix cannot be repaired by
+// extension; MinRun is checked against the positions left), so the cost is
+// proportional to the number of explored prefixes, not to 4^k. Rejected is
+// checked on complete k-mers.
+//
+// k must be between 1 and 31, otherwise the iterator is empty. It is also
+// empty if Accepted or Rejected is set with a length different from k.
+//
+// Example:
+//
+//	c := NoKmerConstraints()
+//	c.MaxCount[2] = 0 // no G
+//	c.MaxRun = 3
+//	for kmer := range IterConstrainedKmers(8, c) { ... }
+func IterConstrainedKmers(k int, c KmerConstraints) iter.Seq[uint64] {
+	return func(yield func(uint64) bool) {
+		if k < 1 || k > 31 || !c.validPositions(k) {
+			return
+		}
+
+		// Nothing to enumerate if the per-base caps cannot even sum to k.
+		capacity := 0
+		for _, m := range c.MaxCount {
+			if m < 0 || m >= k {
+				capacity += k
+			} else {
+				capacity += m
+			}
+		}
+		if capacity < k {
+			return
+		}
+
+		var counts [4]int
+		gc := 0
+
+		var walk func(depth int, kmer uint64, last, run, longest int) bool
+		walk = func(depth int, kmer uint64, last, run, longest int) bool {
+			if depth == k {
+				if c.Rejected != nil && matchesAllMasks(kmer, k, c.Rejected) {
+					return true
+				}
+				return yield(kmer)
+			}
+			for b := range 4 {
+				if c.MaxCount[b] >= 0 && counts[b] >= c.MaxCount[b] {
+					continue
+				}
+				if c.Accepted != nil && c.Accepted[depth]>>uint(b)&1 == 0 {
+					continue
+				}
+				isGC := b == 1 || b == 2
+				if isGC && c.MaxGC >= 0 && gc >= c.MaxGC {
+					continue
+				}
+				r := 1
+				if b == last {
+					r = run + 1
+				}
+				if c.MaxRun > 0 && r > c.MaxRun {
+					continue
+				}
+				l := max(longest, r)
+				if c.MinRun > 1 && l < c.MinRun && r+(k-depth-1) < c.MinRun {
+					continue
+				}
+				counts[b]++
+				if isGC {
+					gc++
+				}
+				ok := walk(depth+1, kmer<<2|uint64(b), b, r, l)
+				counts[b]--
+				if isGC {
+					gc--
+				}
+				if !ok {
+					return false
+				}
+			}
+			return true
+		}
+
+		walk(0, 0, -1, 0, 0)
+	}
+}

pkg/obikmer/enumkmer_test.go

@@ -0,0 +1,85 @@
+package obikmer
+
+import (
+	"strings"
+	"testing"
+)
+
+func collectConstrained(k int, c KmerConstraints) []uint64 {
+	var out []uint64
+	for km := range IterConstrainedKmers(k, c) {
+		out = append(out, km)
+	}
+	return out
+}
+
+func TestIterConstrainedKmersUnconstrained(t *testing.T) {
+	got := collectConstrained(4, NoKmerConstraints())
+	if len(got) != 256 {
+		t.Fatalf("expected 256 k-mers, got %d", len(got))
+	}
+	for i, km := range got {
+		if km != uint64(i) {
+			t.Fatalf("not in increasing order at %d: %d", i, km)
+		}
+	}
+}
+
+func TestIterConstrainedKmersNoG(t *testing.T) {
+	c := NoKmerConstraints()
+	c.MaxCount[2] = 0
+	got := collectConstrained(5, c)
+	if len(got) != 243 {
+		t.Fatalf("expected 3^5 = 243 k-mers, got %d", len(got))
+	}
+	for _, km := range got {
+		if strings.Contains(string(DecodeKmer(km, 5, nil)), "g") {
+			t.Fatalf("k-mer %d contains G", km)
+		}
+	}
+}
+
+func TestIterConstrainedKmersMaxRun(t *testing.T) {
+	c := NoKmerConstraints()
+	c.MaxRun = 2
+	// a(n) = 3*(a(n-1)+a(n-2)), a(1)=4, a(2)=16
+	want := map[int]int{3: 60, 4: 228}
+	for k, n := range want {
+		if got := len(collectConstrained(k, c)); got != n {
+			t.Fatalf("k=%d: expected %d, got %d", k, n, got)
+		}
+	}
+}
+
+func TestIterConstrainedKmersMaxCount(t *testing.T) {
+	c := NoKmerConstraints()
+	c.MaxCount[0] = 1 // at most one A
+	for _, km := range collectConstrained(6, c) {
+		if strings.Count(string(DecodeKmer(km, 6, nil)), "a") > 1 {
+			t.Fatalf("k-mer %d has more than one A", km)
+		}
+	}
+}
+
+func TestIterConstrainedKmersImpossible(t *testing.T) {
+	c := KmerConstraints{MaxCount: [4]int{1, 1, 1, 0}}
+	if got := collectConstrained(4, c); len(got) != 0 {
+		t.Fatalf("expected no k-mer, got %d", len(got))
+	}
+	if got := collectConstrained(0, NoKmerConstraints()); len(got) != 0 {
+		t.Fatalf("expected no k-mer for k=0")
+	}
+}
+
+func TestIterConstrainedKmersEarlyStop(t *testing.T) {
+	n := 0
+	for range IterConstrainedKmers(10, NoKmerConstraints()) {
+		n++
+		if n == 5 {
+			break
+		}
+	}
+	if n != 5 {
+		t.Fatalf("early stop failed: %d", n)
+	}
+}

pkg/obikmer/frequency_filter.go

@@ -1,310 +0,0 @@
-package obikmer
-
-import (
-	"fmt"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
-)
-
-// FrequencyFilter filters k-mers by minimum frequency
-// Specialization of KmerSetGroup where index[i] contains k-mers seen at least i+1 times
-type FrequencyFilter struct {
-	*KmerSetGroup          // Group of KmerSet (one per frequency level)
-	MinFreq       int      // v - minimum required frequency
-}
-
-// NewFrequencyFilter creates a new frequency filter
-// minFreq: minimum number d'occurrences required (v)
-func NewFrequencyFilter(k, minFreq int) *FrequencyFilter {
-	ff := &FrequencyFilter{
-		KmerSetGroup: NewKmerSetGroup(k, minFreq),
-		MinFreq:      minFreq,
-	}
-
-	// Initialize group metadata
-	ff.SetAttribute("type", "FrequencyFilter")
-	ff.SetAttribute("min_freq", minFreq)
-
-	// Initialize metadata for each level
-	for i := 0; i < minFreq; i++ {
-		level := ff.Get(i)
-		level.SetAttribute("level", i)
-		level.SetAttribute("min_occurrences", i+1)
-		level.SetId(fmt.Sprintf("level_%d", i))
-	}
-
-	return ff
-}
-
-// AddSequence adds all k-mers from a sequence to the filter
-// Uses an iterator to avoid allocating an intermediate vector
-func (ff *FrequencyFilter) AddSequence(seq *obiseq.BioSequence) {
-	rawSeq := seq.Sequence()
-	for canonical := range IterCanonicalKmers(rawSeq, ff.K()) {
-		ff.AddKmerCode(canonical)
-	}
-}
-
-// AddKmerCode adds an encoded k-mer to the filter (main algorithm)
-func (ff *FrequencyFilter) AddKmerCode(kmer uint64) {
-	// Find the current level of the k-mer
-	c := 0
-	for c < ff.MinFreq && ff.Get(c).Contains(kmer) {
-		c++
-	}
-
-	// Add to next level (if not yet at maximum)
-	if c < ff.MinFreq {
-		ff.Get(c).AddKmerCode(kmer)
-	}
-}
-
-// AddCanonicalKmerCode adds an encoded canonical k-mer to the filter
-func (ff *FrequencyFilter) AddCanonicalKmerCode(kmer uint64) {
-	canonical := CanonicalKmer(kmer, ff.K())
-	ff.AddKmerCode(canonical)
-}
-
-// AddKmer adds a k-mer to the filter by encoding the sequence
-// The sequence must have exactly k nucleotides
-// Zero-allocation: encodes directly without creating an intermediate slice
-func (ff *FrequencyFilter) AddKmer(seq []byte) {
-	kmer := EncodeKmer(seq, ff.K())
-	ff.AddKmerCode(kmer)
-}
-
-// AddCanonicalKmer adds a canonical k-mer to the filter by encoding the sequence
-// The sequence must have exactly k nucleotides
-// Zero-allocation: encodes directly in canonical form without creating an intermediate slice
-func (ff *FrequencyFilter) AddCanonicalKmer(seq []byte) {
-	canonical := EncodeCanonicalKmer(seq, ff.K())
-	ff.AddKmerCode(canonical)
-}
-
-// GetFilteredSet returns a KmerSet of k-mers with frequency ≥ minFreq
-func (ff *FrequencyFilter) GetFilteredSet() *KmerSet {
-	// Filtered k-mers are in the last level
-	return ff.Get(ff.MinFreq - 1).Copy()
-}
-
-// GetKmersAtLevel returns a KmerSet of k-mers seen at least (level+1) times
-// level doit être dans [0, minFreq-1]
-func (ff *FrequencyFilter) GetKmersAtLevel(level int) *KmerSet {
-	ks := ff.Get(level)
-	if ks == nil {
-		return NewKmerSet(ff.K())
-	}
-	return ks.Copy()
-}
-
-// Stats returns statistics on frequency levels
-func (ff *FrequencyFilter) Stats() FrequencyFilterStats {
-	stats := FrequencyFilterStats{
-		MinFreq: ff.MinFreq,
-		Levels:  make([]LevelStats, ff.MinFreq),
-	}
-
-	for i := 0; i < ff.MinFreq; i++ {
-		ks := ff.Get(i)
-		card := ks.Len()
-		sizeBytes := ks.MemoryUsage()
-
-		stats.Levels[i] = LevelStats{
-			Level:       i + 1, // Level 1 = freq ≥ 1
-			Cardinality: card,
-			SizeBytes:   sizeBytes,
-		}
-
-		stats.TotalBytes += sizeBytes
-	}
-
-	// The last level contains the result
-	stats.FilteredKmers = stats.Levels[ff.MinFreq-1].Cardinality
-
-	return stats
-}
-
-// FrequencyFilterStats contains the filter statistics
-type FrequencyFilterStats struct {
-	MinFreq       int
-	FilteredKmers uint64      // K-mers with freq ≥ minFreq
-	TotalBytes    uint64      // Total memory used
-	Levels        []LevelStats
-}
-
-// LevelStats contains the stats of a level
-type LevelStats struct {
-	Level       int    // freq ≥ Level
-	Cardinality uint64 // Number of k-mers
-	SizeBytes   uint64 // Size in bytes
-}
-
-func (ffs FrequencyFilterStats) String() string {
-	result := fmt.Sprintf(`Frequency Filter Statistics (minFreq=%d):
-  Filtered k-mers (freq≥%d): %d
-  Total memory: %.2f MB
-
-Level breakdown:
-`, ffs.MinFreq, ffs.MinFreq, ffs.FilteredKmers, float64(ffs.TotalBytes)/1024/1024)
-
-	for _, level := range ffs.Levels {
-		result += fmt.Sprintf("  freq≥%d: %d k-mers (%.2f MB)\n",
-			level.Level,
-			level.Cardinality,
-			float64(level.SizeBytes)/1024/1024)
-	}
-
-	return result
-}
-
-// Clear libère la mémoire de tous les niveaux
-// (héritée de KmerSetGroup mais redéfinie pour clarté)
-func (ff *FrequencyFilter) Clear() {
-	ff.KmerSetGroup.Clear()
-}
-
-// ==================================
-// BATCH PROCESSING
-// ==================================
-
-// AddSequences adds multiple sequences in batch
-func (ff *FrequencyFilter) AddSequences(sequences *obiseq.BioSequenceSlice) {
-	for _, seq := range *sequences {
-		ff.AddSequence(seq)
-	}
-}
-
-// ==================================
-// PERSISTANCE
-// ==================================
-
-// Save sauvegarde le FrequencyFilter dans un répertoire
-// Utilise le format de sérialisation du KmerSetGroup sous-jacent
-// Les métadonnées incluent le type "FrequencyFilter" et min_freq
-//
-// Format:
-//   - directory/metadata.{toml,yaml,json} - métadonnées du filtre
-//   - directory/set_0.roaring - k-mers vus ≥1 fois
-//   - directory/set_1.roaring - k-mers vus ≥2 fois
-//   - ...
-//   - directory/set_{minFreq-1}.roaring - k-mers vus ≥minFreq fois
-//
-// Parameters:
-//   - directory: répertoire de destination
-//   - format: format des métadonnées (FormatTOML, FormatYAML, FormatJSON)
-//
-// Example:
-//
-//	err := ff.Save("./my_filter", obikmer.FormatTOML)
-func (ff *FrequencyFilter) Save(directory string, format MetadataFormat) error {
-	// Déléguer à KmerSetGroup qui gère déjà tout
-	return ff.KmerSetGroup.Save(directory, format)
-}
-
-// LoadFrequencyFilter charge un FrequencyFilter depuis un répertoire
-// Vérifie que les métadonnées correspondent à un FrequencyFilter
-//
-// Parameters:
-//   - directory: répertoire source
-//
-// Returns:
-//   - *FrequencyFilter: le filtre chargé
-//   - error: erreur si le chargement échoue ou si ce n'est pas un FrequencyFilter
-//
-// Example:
-//
-//	ff, err := obikmer.LoadFrequencyFilter("./my_filter")
-func LoadFrequencyFilter(directory string) (*FrequencyFilter, error) {
-	// Charger le KmerSetGroup
-	ksg, err := LoadKmerSetGroup(directory)
-	if err != nil {
-		return nil, err
-	}
-
-	// Vérifier que c'est bien un FrequencyFilter
-	if typeAttr, ok := ksg.GetAttribute("type"); !ok || typeAttr != "FrequencyFilter" {
-		return nil, fmt.Errorf("loaded data is not a FrequencyFilter (type=%v)", typeAttr)
-	}
-
-	// Récupérer min_freq
-	minFreqAttr, ok := ksg.GetIntAttribute("min_freq")
-	if !ok {
-		return nil, fmt.Errorf("FrequencyFilter missing min_freq attribute")
-	}
-
-	// Créer le FrequencyFilter
-	ff := &FrequencyFilter{
-		KmerSetGroup: ksg,
-		MinFreq:      minFreqAttr,
-	}
-
-	return ff, nil
-}
-
-// ==================================
-// UTILITAIRES
-// ==================================
-
-// Contains vérifie si un k-mer a atteint la fréquence minimale
-func (ff *FrequencyFilter) Contains(kmer uint64) bool {
-	canonical := CanonicalKmer(kmer, ff.K())
-	return ff.Get(ff.MinFreq - 1).Contains(canonical)
-}
-
-// GetFrequency returns the approximate frequency of a k-mer
-// Retourne le niveau maximum atteint (freq ≥ niveau)
-func (ff *FrequencyFilter) GetFrequency(kmer uint64) int {
-	canonical := CanonicalKmer(kmer, ff.K())
-
-	freq := 0
-	for i := 0; i < ff.MinFreq; i++ {
-		if ff.Get(i).Contains(canonical) {
-			freq = i + 1
-		} else {
-			break
-		}
-	}
-
-	return freq
-}
-
-// Len returns the number of filtered k-mers or at a specific level
-// Without argument: returns the number of k-mers with freq ≥ minFreq (last level)
-// With argument level: returns the number of k-mers with freq ≥ (level+1)
-// Exemple: Len() pour les k-mers filtrés, Len(2) pour freq ≥ 3
-// (héritée de KmerSetGroup mais redéfinie pour la documentation)
-func (ff *FrequencyFilter) Len(level ...int) uint64 {
-	return ff.KmerSetGroup.Len(level...)
-}
-
-// MemoryUsage returns memory usage in bytes
-// (héritée de KmerSetGroup mais redéfinie pour clarté)
-func (ff *FrequencyFilter) MemoryUsage() uint64 {
-	return ff.KmerSetGroup.MemoryUsage()
-}
-
-// ==================================
-// COMPARAISON AVEC D'AUTRES APPROCHES
-// ==================================
-
-// CompareWithSimpleMap compare la mémoire avec une simple map
-func (ff *FrequencyFilter) CompareWithSimpleMap() string {
-	totalKmers := ff.Get(0).Len()
-
-	simpleMapBytes := totalKmers * 24 // ~24 bytes par entrée
-	roaringBytes := ff.MemoryUsage()
-
-	reduction := float64(simpleMapBytes) / float64(roaringBytes)
-
-	return fmt.Sprintf(`Memory Comparison for %d k-mers:
-  Simple map[uint64]uint32: %.2f MB
-  Roaring filter (v=%d):    %.2f MB
-  Reduction:                %.1fx
-`,
-		totalKmers,
-		float64(simpleMapBytes)/1024/1024,
-		ff.MinFreq,
-		float64(roaringBytes)/1024/1024,
-		reduction,
-	)
-}

pkg/obikmer/kdi_merge.go

@@ -0,0 +1,86 @@
+package obikmer
+
+import "container/heap"
+
+// mergeItem represents an element in the min-heap for k-way merge.
+type mergeItem struct {
+	value uint64
+	idx   int // index of the reader that produced this value
+}
+
+// mergeHeap implements heap.Interface for k-way merge.
+type mergeHeap []mergeItem
+
+func (h mergeHeap) Len() int            { return len(h) }
+func (h mergeHeap) Less(i, j int) bool  { return h[i].value < h[j].value }
+func (h mergeHeap) Swap(i, j int)       { h[i], h[j] = h[j], h[i] }
+func (h *mergeHeap) Push(x interface{}) { *h = append(*h, x.(mergeItem)) }
+func (h *mergeHeap) Pop() interface{} {
+	old := *h
+	n := len(old)
+	x := old[n-1]
+	*h = old[:n-1]
+	return x
+}
+
+// KWayMerge performs a k-way merge of multiple sorted KdiReader streams.
+// For each unique k-mer value, it reports the value and the number of
+// input streams that contained it (count).
+type KWayMerge struct {
+	h       mergeHeap
+	readers []*KdiReader
+}
+
+// NewKWayMerge creates a k-way merge from multiple KdiReaders.
+// Each reader must produce values in sorted (ascending) order.
+func NewKWayMerge(readers []*KdiReader) *KWayMerge {
+	m := &KWayMerge{
+		h:       make(mergeHeap, 0, len(readers)),
+		readers: readers,
+	}
+
+	// Initialize heap with first value from each reader
+	for i, r := range readers {
+		if v, ok := r.Next(); ok {
+			m.h = append(m.h, mergeItem{value: v, idx: i})
+		}
+	}
+	heap.Init(&m.h)
+
+	return m
+}
+
+// Next returns the next smallest k-mer value, the number of readers
+// that contained this value (count), and true.
+// Returns (0, 0, false) when all streams are exhausted.
+func (m *KWayMerge) Next() (kmer uint64, count int, ok bool) {
+	if len(m.h) == 0 {
+		return 0, 0, false
+	}
+
+	minVal := m.h[0].value
+	count = 0
+
+	// Pop all items with the same value
+	for len(m.h) > 0 && m.h[0].value == minVal {
+		item := heap.Pop(&m.h).(mergeItem)
+		count++
+		// Advance that reader
+		if v, ok := m.readers[item.idx].Next(); ok {
+			heap.Push(&m.h, mergeItem{value: v, idx: item.idx})
+		}
+	}
+
+	return minVal, count, true
+}
+
+// Close closes all underlying readers.
+func (m *KWayMerge) Close() error {
+	var firstErr error
+	for _, r := range m.readers {
+		if err := r.Close(); err != nil && firstErr == nil {
+			firstErr = err
+		}
+	}
+	return firstErr
+}

pkg/obikmer/kdi_merge_test.go

@@ -0,0 +1,159 @@
+package obikmer
+
+import (
+	"path/filepath"
+	"testing"
+)
+
+// writeKdi is a helper that writes sorted kmers to a .kdi file.
+func writeKdi(t *testing.T, dir, name string, kmers []uint64) string {
+	t.Helper()
+	path := filepath.Join(dir, name)
+	w, err := NewKdiWriter(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, v := range kmers {
+		if err := w.Write(v); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+	return path
+}
+
+func TestKWayMergeBasic(t *testing.T) {
+	dir := t.TempDir()
+
+	// Three sorted streams
+	p1 := writeKdi(t, dir, "a.kdi", []uint64{1, 3, 5, 7})
+	p2 := writeKdi(t, dir, "b.kdi", []uint64{2, 3, 6, 7})
+	p3 := writeKdi(t, dir, "c.kdi", []uint64{3, 4, 7, 8})
+
+	r1, _ := NewKdiReader(p1)
+	r2, _ := NewKdiReader(p2)
+	r3, _ := NewKdiReader(p3)
+
+	m := NewKWayMerge([]*KdiReader{r1, r2, r3})
+	defer m.Close()
+
+	type result struct {
+		kmer  uint64
+		count int
+	}
+	var results []result
+	for {
+		kmer, count, ok := m.Next()
+		if !ok {
+			break
+		}
+		results = append(results, result{kmer, count})
+	}
+
+	expected := []result{
+		{1, 1}, {2, 1}, {3, 3}, {4, 1}, {5, 1}, {6, 1}, {7, 3}, {8, 1},
+	}
+	if len(results) != len(expected) {
+		t.Fatalf("got %d results, want %d", len(results), len(expected))
+	}
+	for i, exp := range expected {
+		if results[i] != exp {
+			t.Errorf("result %d: got %+v, want %+v", i, results[i], exp)
+		}
+	}
+}
+
+func TestKWayMergeSingleStream(t *testing.T) {
+	dir := t.TempDir()
+	p := writeKdi(t, dir, "a.kdi", []uint64{10, 20, 30})
+
+	r, _ := NewKdiReader(p)
+	m := NewKWayMerge([]*KdiReader{r})
+	defer m.Close()
+
+	vals := []uint64{10, 20, 30}
+	for _, expected := range vals {
+		kmer, count, ok := m.Next()
+		if !ok {
+			t.Fatal("unexpected EOF")
+		}
+		if kmer != expected || count != 1 {
+			t.Fatalf("got (%d, %d), want (%d, 1)", kmer, count, expected)
+		}
+	}
+	_, _, ok := m.Next()
+	if ok {
+		t.Fatal("expected EOF")
+	}
+}
+
+func TestKWayMergeEmpty(t *testing.T) {
+	dir := t.TempDir()
+
+	p1 := writeKdi(t, dir, "a.kdi", nil)
+	p2 := writeKdi(t, dir, "b.kdi", nil)
+
+	r1, _ := NewKdiReader(p1)
+	r2, _ := NewKdiReader(p2)
+
+	m := NewKWayMerge([]*KdiReader{r1, r2})
+	defer m.Close()
+
+	_, _, ok := m.Next()
+	if ok {
+		t.Fatal("expected no results from empty streams")
+	}
+}
+
+func TestKWayMergeDisjoint(t *testing.T) {
+	dir := t.TempDir()
+
+	p1 := writeKdi(t, dir, "a.kdi", []uint64{1, 2, 3})
+	p2 := writeKdi(t, dir, "b.kdi", []uint64{10, 20, 30})
+
+	r1, _ := NewKdiReader(p1)
+	r2, _ := NewKdiReader(p2)
+
+	m := NewKWayMerge([]*KdiReader{r1, r2})
+	defer m.Close()
+
+	expected := []uint64{1, 2, 3, 10, 20, 30}
+	for _, exp := range expected {
+		kmer, count, ok := m.Next()
+		if !ok {
+			t.Fatal("unexpected EOF")
+		}
+		if kmer != exp || count != 1 {
+			t.Fatalf("got (%d, %d), want (%d, 1)", kmer, count, exp)
+		}
+	}
+}
+
+func TestKWayMergeAllSame(t *testing.T) {
+	dir := t.TempDir()
+
+	p1 := writeKdi(t, dir, "a.kdi", []uint64{42})
+	p2 := writeKdi(t, dir, "b.kdi", []uint64{42})
+	p3 := writeKdi(t, dir, "c.kdi", []uint64{42})
+
+	r1, _ := NewKdiReader(p1)
+	r2, _ := NewKdiReader(p2)
+	r3, _ := NewKdiReader(p3)
+
+	m := NewKWayMerge([]*KdiReader{r1, r2, r3})
+	defer m.Close()
+
+	kmer, count, ok := m.Next()
+	if !ok {
+		t.Fatal("expected one result")
+	}
+	if kmer != 42 || count != 3 {
+		t.Fatalf("got (%d, %d), want (42, 3)", kmer, count)
+	}
+	_, _, ok = m.Next()
+	if ok {
+		t.Fatal("expected EOF")
+	}
+}

pkg/obikmer/kdi_reader.go

@@ -0,0 +1,170 @@
+package obikmer
+
+import (
+	"bufio"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"os"
+)
+
+// KdiReader reads k-mers from a .kdi file using streaming delta-varint decoding.
+type KdiReader struct {
+	r       *bufio.Reader
+	file    *os.File
+	count   uint64    // total number of k-mers
+	read    uint64    // number of k-mers already consumed
+	prev    uint64    // last decoded value
+	started bool      // whether first value has been read
+	index   *KdxIndex // optional sparse index for seeking
+}
+
+// NewKdiReader opens a .kdi file for streaming reading (no index).
+func NewKdiReader(path string) (*KdiReader, error) {
+	return openKdiReader(path, nil)
+}
+
+// NewKdiIndexedReader opens a .kdi file with its companion .kdx index
+// loaded for fast seeking. If the .kdx file does not exist, it gracefully
+// falls back to sequential reading.
+func NewKdiIndexedReader(path string) (*KdiReader, error) {
+	kdxPath := KdxPathForKdi(path)
+	idx, err := LoadKdxIndex(kdxPath)
+	if err != nil {
+		// Index load failed — fall back to non-indexed
+		return openKdiReader(path, nil)
+	}
+	// idx may be nil if file does not exist — that's fine
+	return openKdiReader(path, idx)
+}
+
+func openKdiReader(path string, idx *KdxIndex) (*KdiReader, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	r := bufio.NewReaderSize(f, 65536)
+
+	// Read and verify magic
+	var magic [4]byte
+	if _, err := io.ReadFull(r, magic[:]); err != nil {
+		f.Close()
+		return nil, fmt.Errorf("kdi: read magic: %w", err)
+	}
+	if magic != kdiMagic {
+		f.Close()
+		return nil, fmt.Errorf("kdi: bad magic %v", magic)
+	}
+
+	// Read count
+	var countBuf [8]byte
+	if _, err := io.ReadFull(r, countBuf[:]); err != nil {
+		f.Close()
+		return nil, fmt.Errorf("kdi: read count: %w", err)
+	}
+	count := binary.LittleEndian.Uint64(countBuf[:])
+
+	return &KdiReader{
+		r:     r,
+		file:  f,
+		count: count,
+		index: idx,
+	}, nil
+}
+
+// Next returns the next k-mer and true, or (0, false) when exhausted.
+func (kr *KdiReader) Next() (uint64, bool) {
+	if kr.read >= kr.count {
+		return 0, false
+	}
+
+	if !kr.started {
+		// Read first value as absolute uint64 LE
+		var buf [8]byte
+		if _, err := io.ReadFull(kr.r, buf[:]); err != nil {
+			return 0, false
+		}
+		kr.prev = binary.LittleEndian.Uint64(buf[:])
+		kr.started = true
+		kr.read++
+		return kr.prev, true
+	}
+
+	// Read delta varint
+	delta, err := DecodeVarint(kr.r)
+	if err != nil {
+		return 0, false
+	}
+	kr.prev += delta
+	kr.read++
+	return kr.prev, true
+}
+
+// SeekTo positions the reader near the target k-mer using the sparse .kdx index.
+// After SeekTo, the reader is positioned so that the next call to Next()
+// returns the k-mer immediately after the indexed entry at or before target.
+//
+// If the reader has no index, or the target is before the current position,
+// SeekTo does nothing (linear scan continues from current position).
+func (kr *KdiReader) SeekTo(target uint64) error {
+	if kr.index == nil {
+		return nil
+	}
+
+	// If we've already passed the target, we can't seek backwards
+	if kr.started && kr.prev >= target {
+		return nil
+	}
+
+	offset, skipCount, ok := kr.index.FindOffset(target)
+	if !ok {
+		return nil
+	}
+
+	// skipCount is the number of k-mers consumed at the indexed position.
+	// The index was recorded AFTER writing the k-mer at position skipCount-1
+	// (since count%stride==0 after incrementing count). So the actual number
+	// of k-mers consumed is skipCount (the entry's kmer is the last one
+	// before the offset).
+
+	// Only seek if it would skip significant work
+	if kr.started && skipCount <= kr.read {
+		return nil
+	}
+
+	// The index entry stores (kmer_value, byte_offset_after_that_kmer).
+	// skipCount = (entryIdx+1)*stride, so entryIdx = skipCount/stride - 1
+	// We seek to that offset, set prev = indexedKmer, and the next Next()
+	// call will read the delta-varint of the following k-mer.
+	entryIdx := int(skipCount)/kr.index.stride - 1
+	if entryIdx < 0 || entryIdx >= len(kr.index.entries) {
+		return nil
+	}
+	indexedKmer := kr.index.entries[entryIdx].kmer
+
+	if _, err := kr.file.Seek(int64(offset), io.SeekStart); err != nil {
+		return fmt.Errorf("kdi: seek: %w", err)
+	}
+	kr.r.Reset(kr.file)
+
+	kr.prev = indexedKmer
+	kr.started = true
+	kr.read = skipCount
+
+	return nil
+}
+
+// Count returns the total number of k-mers in this partition.
+func (kr *KdiReader) Count() uint64 {
+	return kr.count
+}
+
+// Remaining returns how many k-mers have not been read yet.
+func (kr *KdiReader) Remaining() uint64 {
+	return kr.count - kr.read
+}
+
+// Close closes the underlying file.
+func (kr *KdiReader) Close() error {
+	return kr.file.Close()
+}

pkg/obikmer/kdi_test.go

@@ -0,0 +1,255 @@
+package obikmer
+
+import (
+	"os"
+	"path/filepath"
+	"sort"
+	"testing"
+)
+
+func TestKdiRoundTrip(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "test.kdi")
+
+	// Sorted k-mer values
+	kmers := []uint64{10, 20, 30, 100, 200, 500, 10000, 1 << 40, 1<<62 - 1}
+
+	w, err := NewKdiWriter(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, v := range kmers {
+		if err := w.Write(v); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if w.Count() != uint64(len(kmers)) {
+		t.Fatalf("writer count: got %d, want %d", w.Count(), len(kmers))
+	}
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	// Read back
+	r, err := NewKdiReader(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer r.Close()
+
+	if r.Count() != uint64(len(kmers)) {
+		t.Fatalf("reader count: got %d, want %d", r.Count(), len(kmers))
+	}
+
+	for i, expected := range kmers {
+		got, ok := r.Next()
+		if !ok {
+			t.Fatalf("unexpected EOF at index %d", i)
+		}
+		if got != expected {
+			t.Fatalf("kmer %d: got %d, want %d", i, got, expected)
+		}
+	}
+
+	_, ok := r.Next()
+	if ok {
+		t.Fatal("expected EOF after all k-mers")
+	}
+}
+
+func TestKdiEmpty(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "empty.kdi")
+
+	w, err := NewKdiWriter(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	r, err := NewKdiReader(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer r.Close()
+
+	if r.Count() != 0 {
+		t.Fatalf("expected count 0, got %d", r.Count())
+	}
+
+	_, ok := r.Next()
+	if ok {
+		t.Fatal("expected no k-mers in empty file")
+	}
+}
+
+func TestKdiSingleValue(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "single.kdi")
+
+	w, err := NewKdiWriter(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := w.Write(42); err != nil {
+		t.Fatal(err)
+	}
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	r, err := NewKdiReader(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer r.Close()
+
+	if r.Count() != 1 {
+		t.Fatalf("expected count 1, got %d", r.Count())
+	}
+
+	v, ok := r.Next()
+	if !ok {
+		t.Fatal("expected one k-mer")
+	}
+	if v != 42 {
+		t.Fatalf("got %d, want 42", v)
+	}
+}
+
+func TestKdiFileSize(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "size.kdi")
+
+	// Write: magic(4) + count(8) + first(8) = 20 bytes
+	w, err := NewKdiWriter(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := w.Write(0); err != nil {
+		t.Fatal(err)
+	}
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	info, err := os.Stat(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// magic(4) + count(8) + first(8) = 20
+	if info.Size() != 20 {
+		t.Fatalf("file size: got %d, want 20", info.Size())
+	}
+}
+
+func TestKdiDeltaCompression(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "delta.kdi")
+
+	// Dense consecutive values should compress well
+	n := 10000
+	kmers := make([]uint64, n)
+	for i := range kmers {
+		kmers[i] = uint64(i * 2) // even numbers
+	}
+
+	w, err := NewKdiWriter(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, v := range kmers {
+		if err := w.Write(v); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	// Each delta is 2, encoded as 1 byte varint
+	// Total: magic(4) + count(8) + first(8) + (n-1)*1 = 20 + 9999 bytes
+	info, err := os.Stat(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expected := int64(20 + n - 1)
+	if info.Size() != expected {
+		t.Fatalf("file size: got %d, want %d", info.Size(), expected)
+	}
+
+	// Verify round-trip
+	r, err := NewKdiReader(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer r.Close()
+
+	for i, expected := range kmers {
+		got, ok := r.Next()
+		if !ok {
+			t.Fatalf("unexpected EOF at index %d", i)
+		}
+		if got != expected {
+			t.Fatalf("kmer %d: got %d, want %d", i, got, expected)
+		}
+	}
+}
+
+func TestKdiFromRealKmers(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "real.kdi")
+
+	// Extract k-mers from a sequence, sort, dedup, write to KDI
+	seq := []byte("ACGTACGTACGTACGTACGTACGTACGTACGTACGTACGT")
+	k := 15
+
+	var kmers []uint64
+	for kmer := range IterCanonicalKmers(seq, k) {
+		kmers = append(kmers, kmer)
+	}
+	sort.Slice(kmers, func(i, j int) bool { return kmers[i] < kmers[j] })
+	// Dedup
+	deduped := kmers[:0]
+	for i, v := range kmers {
+		if i == 0 || v != kmers[i-1] {
+			deduped = append(deduped, v)
+		}
+	}
+
+	w, err := NewKdiWriter(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, v := range deduped {
+		if err := w.Write(v); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	// Read back and verify
+	r, err := NewKdiReader(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer r.Close()
+
+	if r.Count() != uint64(len(deduped)) {
+		t.Fatalf("count: got %d, want %d", r.Count(), len(deduped))
+	}
+
+	for i, expected := range deduped {
+		got, ok := r.Next()
+		if !ok {
+			t.Fatalf("unexpected EOF at index %d", i)
+		}
+		if got != expected {
+			t.Fatalf("kmer %d: got %d, want %d", i, got, expected)
+		}
+	}
+}

pkg/obikmer/kdi_writer.go

@@ -0,0 +1,151 @@
+package obikmer
+
+import (
+	"bufio"
+	"encoding/binary"
+	"os"
+)
+
+// KDI file magic bytes: "KDI\x01"
+var kdiMagic = [4]byte{'K', 'D', 'I', 0x01}
+
+// kdiHeaderSize is the size of the KDI header: magic(4) + count(8) = 12 bytes.
+const kdiHeaderSize = 12
+
+// KdiWriter writes a sorted sequence of uint64 k-mers to a .kdi file
+// using delta-varint encoding.
+//
+// Format:
+//
+//	[magic: 4 bytes "KDI\x01"]
+//	[count: uint64 LE]        number of k-mers
+//	[first: uint64 LE]        first k-mer (absolute value)
+//	[delta_1: varint]          arr[1] - arr[0]
+//	[delta_2: varint]          arr[2] - arr[1]
+//	...
+//
+// The caller must write k-mers in strictly increasing order.
+//
+// On Close(), a companion .kdx sparse index file is written alongside
+// the .kdi file for fast random access.
+type KdiWriter struct {
+	w            *bufio.Writer
+	file         *os.File
+	count        uint64
+	prev         uint64
+	first        bool
+	path         string
+	bytesWritten uint64     // bytes written after header (data section offset)
+	indexEntries []kdxEntry // sparse index entries collected during writes
+}
+
+// NewKdiWriter creates a new KdiWriter writing to the given file path.
+// The header (magic + count placeholder) is written immediately.
+// Count is patched on Close().
+func NewKdiWriter(path string) (*KdiWriter, error) {
+	f, err := os.Create(path)
+	if err != nil {
+		return nil, err
+	}
+	w := bufio.NewWriterSize(f, 65536)
+
+	// Write magic
+	if _, err := w.Write(kdiMagic[:]); err != nil {
+		f.Close()
+		return nil, err
+	}
+	// Write placeholder for count (will be patched on Close)
+	var countBuf [8]byte
+	if _, err := w.Write(countBuf[:]); err != nil {
+		f.Close()
+		return nil, err
+	}
+
+	return &KdiWriter{
+		w:            w,
+		file:         f,
+		first:        true,
+		path:         path,
+		bytesWritten: 0,
+		indexEntries: make([]kdxEntry, 0, 256),
+	}, nil
+}
+
+// Write adds a k-mer to the file. K-mers must be written in strictly
+// increasing order.
+func (kw *KdiWriter) Write(kmer uint64) error {
+	if kw.first {
+		// Write first value as absolute uint64 LE
+		var buf [8]byte
+		binary.LittleEndian.PutUint64(buf[:], kmer)
+		if _, err := kw.w.Write(buf[:]); err != nil {
+			return err
+		}
+		kw.bytesWritten += 8
+		kw.prev = kmer
+		kw.first = false
+	} else {
+		delta := kmer - kw.prev
+		n, err := EncodeVarint(kw.w, delta)
+		if err != nil {
+			return err
+		}
+		kw.bytesWritten += uint64(n)
+		kw.prev = kmer
+	}
+	kw.count++
+
+	// Record sparse index entry every defaultKdxStride k-mers.
+	// The offset recorded is AFTER writing this k-mer, so it points to
+	// where the next k-mer's data will start. SeekTo uses this: it seeks
+	// to the recorded offset, sets prev = indexedKmer, and Next() reads
+	// the delta of the following k-mer.
+	if kw.count%defaultKdxStride == 0 {
+		kw.indexEntries = append(kw.indexEntries, kdxEntry{
+			kmer:   kmer,
+			offset: kdiHeaderSize + kw.bytesWritten,
+		})
+	}
+
+	return nil
+}
+
+// Count returns the number of k-mers written so far.
+func (kw *KdiWriter) Count() uint64 {
+	return kw.count
+}
+
+// Close flushes buffered data, patches the count in the header,
+// writes the companion .kdx index file, and closes the file.
+func (kw *KdiWriter) Close() error {
+	if err := kw.w.Flush(); err != nil {
+		kw.file.Close()
+		return err
+	}
+
+	// Patch count at offset 4 (after magic)
+	if _, err := kw.file.Seek(4, 0); err != nil {
+		kw.file.Close()
+		return err
+	}
+	var countBuf [8]byte
+	binary.LittleEndian.PutUint64(countBuf[:], kw.count)
+	if _, err := kw.file.Write(countBuf[:]); err != nil {
+		kw.file.Close()
+		return err
+	}
+
+	if err := kw.file.Close(); err != nil {
+		return err
+	}
+
+	// Write .kdx index file if there are entries to index
+	if len(kw.indexEntries) > 0 {
+		kdxPath := KdxPathForKdi(kw.path)
+		if err := WriteKdxIndex(kdxPath, defaultKdxStride, kw.indexEntries); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}