Refactor k-mer matching pipeline with improved concurrency and memory management

Refactor k-mer matching to use a pipeline architecture with improved concurrency and memory management:

- Replace sort.Slice with slices.SortFunc and cmp.Compare for better performance
- Introduce PreparedQueries struct to encapsulate query buckets with metadata
- Implement MergeQueries function to merge query buckets from multiple batches
- Rewrite MatchBatch to use pre-allocated results and mutexes instead of map-based accumulation
- Add seek optimization in matchPartition to reduce linear scanning
- Refactor match command to use a multi-stage pipeline with proper batching and merging
- Add index directory option for match command
- Improve parallel processing of sequence batches

This refactoring improves performance by reducing memory allocations, optimizing k-mer lookup, and implementing a more efficient pipeline for large-scale k-mer matching operations.

.github/workflows/obitest.yml

@@ -0,0 +1,19 @@
+name: "Run the obitools command test suite"
+
+on:
+  push:
+    branches:
+      - master
+      - V*
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Setup Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: '1.23'
+    - name: Checkout obitools4 project
+      uses: actions/checkout@v4
+    - name: Run tests
+      run: make githubtests

.github/workflows/release.yml

@@ -0,0 +1,172 @@
+name: Create Release on Tag
+
+on:
+  push:
+    tags:
+      - "Release_*"
+
+permissions:
+  contents: write
+
+jobs:
+  # First run tests
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.23"
+      - name: Checkout obitools4 project
+        uses: actions/checkout@v4
+      - name: Run tests
+        run: make githubtests
+
+  # Build binaries for each platform
+  build:
+    needs: test
+    strategy:
+      matrix:
+        include:
+          - os: ubuntu-latest
+            goos: linux
+            goarch: amd64
+            output_name: linux_amd64
+          - os: ubuntu-24.04-arm
+            goos: linux
+            goarch: arm64
+            output_name: linux_arm64
+          - os: macos-15-intel
+            goos: darwin
+            goarch: amd64
+            output_name: darwin_amd64
+          - os: macos-latest
+            goos: darwin
+            goarch: arm64
+            output_name: darwin_arm64
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.23"
+
+      - name: Extract version from tag
+        id: get_version
+        run: |
+          TAG=${GITHUB_REF#refs/tags/Release_}
+          echo "version=$TAG" >> $GITHUB_OUTPUT
+
+      - name: Install build tools (macOS)
+        if: runner.os == 'macOS'
+        run: |
+          # Ensure Xcode Command Line Tools are installed
+          xcode-select --install 2>/dev/null || true
+          xcode-select -p
+
+      - name: Build binaries
+        env:
+          GOOS: ${{ matrix.goos }}
+          GOARCH: ${{ matrix.goarch }}
+          VERSION: ${{ steps.get_version.outputs.version }}
+        run: |
+          make obitools
+          mkdir -p artifacts
+          # Create a single tar.gz with all binaries for this platform
+          tar -czf artifacts/obitools4_${VERSION}_${{ matrix.output_name }}.tar.gz -C build .
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: binaries-${{ matrix.output_name }}
+          path: artifacts/*
+
+  # Create the release
+  create-release:
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Extract version from tag
+        id: get_version
+        run: |
+          TAG=${GITHUB_REF#refs/tags/Release_}
+          echo "version=$TAG" >> $GITHUB_OUTPUT
+
+      - name: Download all artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: release-artifacts
+
+      - name: Prepare release directory
+        run: |
+          mkdir -p release
+          find release-artifacts -type f -name "*.tar.gz" -exec cp {} release/ \;
+          ls -lh release/
+
+      - name: Generate Release Notes
+        env:
+          VERSION: ${{ steps.get_version.outputs.version }}
+        run: |
+          PREV_TAG=$(git describe --tags --abbrev=0 HEAD^ 2>/dev/null || echo "")
+
+          echo "# OBITools4 Release ${VERSION}" > release_notes.md
+          echo "" >> release_notes.md
+
+          if [ -n "$PREV_TAG" ]; then
+            echo "## Changes since ${PREV_TAG}" >> release_notes.md
+            echo "" >> release_notes.md
+            git log ${PREV_TAG}..HEAD --pretty=format:"- %s" >> release_notes.md
+          else
+            echo "## Changes" >> release_notes.md
+            echo "" >> release_notes.md
+            git log --pretty=format:"- %s" -n 20 >> release_notes.md
+          fi
+
+          echo "" >> release_notes.md
+          echo "" >> release_notes.md
+          echo "## Installation" >> release_notes.md
+          echo "" >> release_notes.md
+          echo "Download the appropriate archive for your system and extract it:" >> release_notes.md
+          echo "" >> release_notes.md
+          echo "### Linux (AMD64)" >> release_notes.md
+          echo '```bash' >> release_notes.md
+          echo "tar -xzf obitools4_${VERSION}_linux_amd64.tar.gz" >> release_notes.md
+          echo '```' >> release_notes.md
+          echo "" >> release_notes.md
+          echo "### Linux (ARM64)" >> release_notes.md
+          echo '```bash' >> release_notes.md
+          echo "tar -xzf obitools4_${VERSION}_linux_arm64.tar.gz" >> release_notes.md
+          echo '```' >> release_notes.md
+          echo "" >> release_notes.md
+          echo "### macOS (Intel)" >> release_notes.md
+          echo '```bash' >> release_notes.md
+          echo "tar -xzf obitools4_${VERSION}_darwin_amd64.tar.gz" >> release_notes.md
+          echo '```' >> release_notes.md
+          echo "" >> release_notes.md
+          echo "### macOS (Apple Silicon)" >> release_notes.md
+          echo '```bash' >> release_notes.md
+          echo "tar -xzf obitools4_${VERSION}_darwin_arm64.tar.gz" >> release_notes.md
+          echo '```' >> release_notes.md
+          echo "" >> release_notes.md
+          echo "All OBITools4 binaries are included in each archive." >> release_notes.md
+
+      - name: Create GitHub Release
+        uses: softprops/action-gh-release@v1
+        with:
+          name: Release ${{ steps.get_version.outputs.version }}
+          body_path: release_notes.md
+          files: release/*
+          draft: false
+          prerelease: false
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -1,120 +1,36 @@
-cpu.pprof
+**/cpu.pprof
-cpu.trace
+**/cpu.trace
-test
+**/test
-bin
+**/bin
-vendor
+**/vendor
-*.fastq
+**/*.fastq
-*.fasta
+**/*.fasta
-*.fastq.gz
+**/*.fastq.gz
-*.fasta.gz
+**/*.fasta.gz
-.DS_Store
+**/.DS_Store
-*.gml
+**/*.gml
-*.log
+**/*.log
-/argaly
+**/xxx*
-
+**/*.sav
-/obiconvert
+**/*.old
-/obicount
+**/*.tgz
-/obimultiplex
+**/*.yaml
-/obipairing
+**/*.csv
-/obipcr
-/obifind
-/obidistribute
-/obiuniq
-/build
-/Makefile.old
-.Rproj.user
-obitools.Rproj
-Stat_error.knit.md
-.Rhistory
-Stat_error.nb.html
-Stat_error.Rmd
-
-/.luarc.json
-/doc/TAXO/
-/doc/results/
-/doc/_main.log
-/doc/_book/_main.tex
-/doc/_freeze/
-/doc/tutorial_files/
-/doc/wolf_data/
-/taxdump/
-/.vscode/
-
-/Algo-Alignement.numbers
-/Estimate_proba_true_seq.html
-/Estimate_proba_true_seq.nb.html
-/Estimate_proba_true_seq.Rmd
-/modele_error_euka.qmd
-/obitools.code-workspace
-.DS_Store
-.RData
-x
-xxx
-y
-/doc/wolf_diet.tgz
-/doc/man/depends
-/sample/wolf_R1.fasta.gz
-/sample/wolf_R2.fasta.gz
-/sample/euka03.ecotag.fasta.gz
-/sample/ratio.csv
-/sample/STD_PLN_1.dat
-/sample/STD_PLN_2.dat
-/sample/subset_Pasvik_R1.fastq.gz
-/sample/subset_Pasvik_R2.fastq.gz
-/sample/test_gobitools.fasta.bz2
-euka03.csv*
-gbbct793.seq.gz
-gbinv1003.seq.gz
-gbpln210.seq
-/doc/book/OBITools-V4.aux
-/doc/book/OBITools-V4.fdb_latexmk
-/doc/book/OBITools-V4.fls
-/doc/book/OBITools-V4.log
-/doc/book/OBITools-V4.pdf
-/doc/book/OBITools-V4.synctex.gz
-/doc/book/OBITools-V4.tex
-/doc/book/OBITools-V4.toc
-getoptions.adoc
-Archive.zip
-.DS_Store
-sample/.DS_Store
-sample/consensus_graphs/specimen_hac_plants_Vern_disicolor_.gml
-93954
-Bact03.e5.gb_R254.obipcr.idx.fasta.save
-sample/test.obipcr.log
-Bact02.e3.gb_R254.obipcr.fasta.gz
-Example_Arth03.ngsfilter
-SPER01.csv
-SPER03.csv
-wolf_diet_ngsfilter.txt
 xx
-xxx.gb
+
-yyy_geom.csv
+.rhistory
-yyy_LCS.csv
+/.vscode
-yyy.json
+/build
-bug_obimultiplex/toto
+/bugs
-bug_obimultiplex/toto_mapping
+
-bug_obimultiplex/tutu
+/ncbitaxo
-bug_obimultiplex/tutu_mapping
+
-bug_obipairing/GIT1_GH_ngsfilter.txt
+!/obitests/**
-doc/book/TAXO/citations.dmp
+!/sample/**
-doc/book/TAXO/delnodes.dmp
+LLM/**
-doc/book/TAXO/division.dmp
+*_files
-doc/book/TAXO/gc.prt
+
-doc/book/TAXO/gencode.dmp
+entropy.html
-doc/book/TAXO/merged.dmp
+bug_id.txt
-doc/book/TAXO/names.dmp
+obilowmask_ref
-doc/book/TAXO/nodes.dmp
+test_*
-doc/book/TAXO/readme.txt
-doc/book/wolf_data/Release-253/ncbitaxo/citations.dmp
-doc/book/wolf_data/Release-253/ncbitaxo/delnodes.dmp
-doc/book/wolf_data/Release-253/ncbitaxo/division.dmp
-doc/book/wolf_data/Release-253/ncbitaxo/gc.prt
-doc/book/wolf_data/Release-253/ncbitaxo/gencode.dmp
-doc/book/wolf_data/Release-253/ncbitaxo/merged.dmp
-doc/book/wolf_data/Release-253/ncbitaxo/names.dmp
-doc/book/wolf_data/Release-253/ncbitaxo/nodes.dmp
-doc/book/wolf_data/Release-253/ncbitaxo/readme.txt
-doc/book/results/toto.tasta
-sample/.DS_Store
-GO

Makefile

@@ -2,8 +2,9 @@
 #export GOBIN=$(GOPATH)/bin
 #export PATH=$(GOBIN):$(shell echo $${PATH})
 
+GOFLAGS=
 GOCMD=go
-GOBUILD=$(GOCMD) build # -compiler gccgo -gccgoflags -O3
+GOBUILD=$(GOCMD) build $(GOFLAGS)
 GOGENERATE=$(GOCMD) generate
 GOCLEAN=$(GOCMD) clean
 GOTEST=$(GOCMD) test
@@ -16,6 +17,12 @@ PACKAGES_SRC:= $(wildcard pkg/*/*.go pkg/*/*/*.go)
 PACKAGE_DIRS:=$(sort $(patsubst %/,%,$(dir $(PACKAGES_SRC))))
 PACKAGES:=$(notdir $(PACKAGE_DIRS))
 
+GITHOOK_SRC_DIR=git-hooks
+GITHOOKS_SRC:=$(wildcard $(GITHOOK_SRC_DIR)/*)
+
+GITHOOK_DIR=.git/hooks
+GITHOOKS:=$(patsubst $(GITHOOK_SRC_DIR)/%,$(GITHOOK_DIR)/%,$(GITHOOKS_SRC))
+
 OBITOOLS_SRC:= $(wildcard cmd/obitools/*/*.go)
 OBITOOLS_DIRS:=$(sort $(patsubst %/,%,$(dir $(OBITOOLS_SRC))))
 OBITOOLS:=$(notdir $(OBITOOLS_DIRS))
@@ -53,27 +60,31 @@ endif
 
 OUTPUT:=$(shell mktemp)
 
-all: obitools
+all: install-githook obitools
+
+obitools: $(patsubst %,$(OBITOOLS_PREFIX)%,$(OBITOOLS))
+
+install-githook: $(GITHOOKS)
+
+$(GITHOOK_DIR)/%: $(GITHOOK_SRC_DIR)/%
+	@echo installing $$(basename $@)...
+	@mkdir -p $(GITHOOK_DIR)
+	@cp $< $@
+	@chmod +x $@
 
-packages: $(patsubst %,pkg-%,$(PACKAGES))
-obitools: $(patsubst %,$(OBITOOLS_PREFIX)%,$(OBITOOLS)) 
 
 update-deps:
 	go get -u ./...
 
-test:
+test: .FORCE
 	$(GOTEST) ./...
-	
-man: 
-	make -C doc man
-obibook: 
-	make -C doc obibook
-doc: man obibook
 
-macos-pkg: 
+obitests:
-	@bash pkgs/macos/macos-installer-builder-master/macOS-x64/build-macos-x64.sh \
+	@for t in $$(find obitests -name test.sh -print) ; do \
-		OBITools \
+		bash $${t} || exit 1;\
-		0.0.1
+	done
+
+githubtests: obitools obitests
 
 $(BUILD_DIR):
 	mkdir -p $@
@@ -83,19 +94,80 @@ $(foreach P,$(PACKAGE_DIRS),$(eval $(call MAKE_PKG_RULE,$(P))))
 
 $(foreach P,$(OBITOOLS_DIRS),$(eval $(call MAKE_OBITOOLS_RULE,$(P))))
 
-pkg/obioptions/version.go: .FORCE
+pkg/obioptions/version.go: version.txt .FORCE
-ifneq ($(strip $(COMMIT_ID)),)
+	@version=$$(cat version.txt); \
-	@cat $@ \
+	cat $@ \
-	| sed  -E 's/^var _Commit = "[^"]*"/var _Commit = "'$(COMMIT_ID)'"/' \
+	| sed  -E 's/^var _Version = "[^"]*"/var _Version = "Release '$$version'"/' \
-	| sed  -E 's/^var _Version = "[^"]*"/var _Version = "'"$(LAST_TAG)"'"/' \
 	> $(OUTPUT)
 
 	@diff $@ $(OUTPUT) 2>&1 > /dev/null \
-    	|| echo "Update version.go : $@ to $(LAST_TAG) ($(COMMIT_ID))" \
+    	|| (echo "Update version.go to $$(cat version.txt)" && mv $(OUTPUT) $@)
-    	&& mv $(OUTPUT) $@
 
 	@rm -f $(OUTPUT)
-endif
 
-.PHONY: all packages obitools man obibook doc update-deps .FORCE
+bump-version:
-.FORCE:
+	@echo "Incrementing version..."
+	@current=$$(cat version.txt); \
+	echo "  Current version: $$current"; \
+	major=$$(echo $$current | cut -d. -f1); \
+	minor=$$(echo $$current | cut -d. -f2); \
+	patch=$$(echo $$current | cut -d. -f3); \
+	new_patch=$$((patch + 1)); \
+	new_version="$$major.$$minor.$$new_patch"; \
+	echo "  New version: $$new_version"; \
+	echo "$$new_version" > version.txt
+	@echo "✓ Version updated in version.txt"
+	@$(MAKE) pkg/obioptions/version.go
+
+jjnew:
+	@echo "$(YELLOW)→ Creating a new commit...$(NC)"
+	@echo "$(BLUE)→ Documenting current commit...$(NC)"
+	@jj auto-describe
+	@echo "$(BLUE)→ Done.$(NC)"
+	@jj new
+	@echo "$(GREEN)✓ New commit created$(NC)"
+
+jjpush:
+	@echo "$(YELLOW)→ Pushing commit to repository...$(NC)"
+	@echo "$(BLUE)→ Documenting current commit...$(NC)"
+	@jj auto-describe
+	@echo "$(BLUE)→ Creating new commit for version bump...$(NC)"
+	@jj new
+	@previous_version=$$(cat version.txt); \
+	$(MAKE) bump-version; \
+	version=$$(cat version.txt); \
+	tag_name="Release_$$version"; \
+	previous_tag="Release_$$previous_version"; \
+	echo "$(BLUE)→ Documenting version bump commit...$(NC)"; \
+	jj auto-describe; \
+	echo "$(BLUE)→ Generating release notes from $$previous_tag to current commit...$(NC)"; \
+	if command -v orla >/dev/null 2>&1 && command -v jq >/dev/null 2>&1; then \
+		release_json=$$(ORLA_MAX_TOOL_CALLS=50 jj log -r "$$previous_tag::@" -T 'commit_id.short() ++ " " ++ description' | \
+			orla agent -m ollama:qwen3-coder-next:latest \
+			"Summarize the following commits into a GitHub release note for version $$version. Ignore commits related to version bumps, .gitignore changes, or any internal housekeeping that is irrelevant to end users. Describe each user-facing change precisely without exposing code. Eliminate redundancy. Output strictly valid JSON with no surrounding text, using this exact schema: {\"title\": \"<short release title>\", \"body\": \"<detailed markdown release notes>\"}"); \
+		release_json=$$(echo "$$release_json" | sed -n '/^{/,/^}/p'); \
+		release_title=$$(echo "$$release_json" | jq -r '.title // empty') ; \
+		release_body=$$(echo "$$release_json" | jq -r '.body // empty') ; \
+		if [ -n "$$release_title" ] && [ -n "$$release_body" ]; then \
+			release_message="$$release_title"$$'\n\n'"$$release_body"; \
+		else \
+			echo "$(YELLOW)⚠ JSON parsing failed, falling back to raw output$(NC)"; \
+			release_message="Release $$version"$$'\n\n'"$$release_json"; \
+		fi; \
+	else \
+		release_message="Release $$version"; \
+	fi; \
+	echo "$(BLUE)→ Pushing commits and creating tag $$tag_name...$(NC)"; \
+	jj git push --change @; \
+	git tag -a "$$tag_name" -m "$$release_message" 2>/dev/null || echo "Tag $$tag_name already exists"; \
+	git push origin "$$tag_name" 2>/dev/null || echo "Tag already pushed"
+	@echo "$(GREEN)✓ Commits and tag pushed to repository$(NC)"
+
+jjfetch:
+	@echo "$(YELLOW)→ Pulling latest commits...$(NC)"
+	@jj git fetch
+	@jj new master@origin
+	@echo "$(GREEN)✓ Latest commits pulled$(NC)"
+
+.PHONY: all obitools update-deps obitests githubtests jjnew jjpush jjfetch bump-version .FORCE
+.FORCE:

README.md

@@ -16,12 +16,17 @@ The easiest way to run it is to copy and paste the following command into your t
 curl -L https://raw.githubusercontent.com/metabarcoding/obitools4/master/install_obitools.sh | bash
 ```
 
-By default, the script installs the *OBITools* commands and other associated files into the `/usr/local` directory.
+By default, the script installs the latest version of *OBITools* commands and other associated files into the `/usr/local` directory.
-The names of the commands in the new *OBITools4* are mostly identical to those in *OBITools2*.
-Therefore, installing the new *OBITools* may hide or delete the old ones. If you want both versions to be 
-available on your system, the installation script offers two options:
 
+### Installation Options
 
+The installation script offers several options:
+
+>  -l, --list              List all available versions and exit.
+> 
+>  -v, --version           Install a specific version (e.g., `-v 4.4.3`).
+>                          By default, the latest version is installed.
+> 
 >  -i, --install-dir       Directory where obitools are installed 
 >                          (as example use `/usr/local` not `/usr/local/bin`).
 > 
@@ -30,14 +35,31 @@ available on your system, the installation script offers two options:
 >                          same time on your system (as example `-p g` will produce 
 >                          `gobigrep` command instead of `obigrep`).
 
-You can use these options by following the installation command:
+### Examples
 
+List all available versions:
+```{bash}
+curl -L https://raw.githubusercontent.com/metabarcoding/obitools4/master/install_obitools.sh | bash -s -- --list
+```
+
+Install a specific version:
+```{bash}
+curl -L https://raw.githubusercontent.com/metabarcoding/obitools4/master/install_obitools.sh | bash -s -- --version 4.4.3
+```
+
+Install in a custom directory with command prefix:
 ```{bash}
 curl -L https://raw.githubusercontent.com/metabarcoding/obitools4/master/install_obitools.sh | \
       bash -s -- --install-dir test_install --obitools-prefix k
 ```
 
-In this case, the binaries will be installed in the `test_install` directory and all command names will be prefixed with the letter `k`. Thus `obigrep` will be named `kobigrep`.
+In this last example, the binaries will be installed in the `test_install` directory and all command names will be prefixed with the letter `k`. Thus, `obigrep` will be named `kobigrep`.
+
+### Note on Version Compatibility
+
+The names of the commands in the new *OBITools4* are mostly identical to those in *OBITools2*.
+Therefore, installing the new *OBITools* may hide or delete the old ones. If you want both versions to be 
+available on your system, use the `--install-dir` and `--obitools-prefix` options as shown above.
 
 ## Continuing the analysis...
 

Release-notes.md

@@ -1,51 +1,190 @@
 # OBITools release notes
 
-## Latest changes
+## New changes
+
+### Bug fixes
+
+- In `obipairing` correct the misspelling of the `obiparing_*` tags where the `i`
+  was missing to `obipairing_`.  
+
+- In `obigrep` the **-C** option that excludes sequences too abundant was not 
+  functional.
+
+- In `obitaxonomy` the **-l** option that lists all the taxonomic rank defined by
+  a taxonomy was not functional
+
+- The file type guesser was not using enough data to be able to correctly detect
+  file format when sequences were too long in fastq and fasta or when lines were
+  to long in CSV files. That's now corrected
+
+- Options **--fasta** or **--fastq** usable to specify input format were ignored.
+  They are now correctly considered
+
+- The `obiannotate` command were crashing when a selection option was used but
+  no editing option.
+  
+- The `--fail-on-taxonomy` led to an error on merged taxa even when the
+  `--update-taxid` option was used.
+
+- The `--compressed` option was not correctly named. It was renamed to `--compress`
+  
+### Enhancement
+
+- Some sequences in the Genbank and EMBL databases are several gigabases long. The 
+  sequence parser had to reallocate and recopy memory many times to read them, 
+  resulting in a complexity of O(N^2) for reading such large sequences. 
+  The new file chunk reader has a linear algorithm that speeds up the reading 
+  of very long sequences.
+
+- A new option **--csv** is added to every obitools to indicate that the input
+  format is CSV
+
+- The new version of obitools are now printing the taxids in a fancy way
+  including the scientific name and the taxonomic rank (`"taxon:9606 [Homo
+  sapiens]@species"`). But if you need the old fashion raw taxid, a new option
+  **--raw-taxid** has been added to get obitools printing the taxids without any
+  decorations (`"9606"`).
+
+
+## March 1st, 2025. Release 4.4.0
+
+A new documentation website is available at https://obitools4.metabarcoding.org.
+Its development is still in progress. 
+
+The biggest step forward in this new version is taxonomy management. The new
+version is now able to handle taxonomic identifiers that are not just integer
+values. This is a first step towards an easy way to handle other taxonomy
+databases soon, such as the GBIF or Catalog of Life taxonomies. This version
+is able to handle files containing taxonomic information created by previous
+versions of OBITools, but files created by this new version may have some
+problems to be analyzed by previous versions, at least for the taxonomic
+information.
+
+
+### Breaking changes
+
+- In `obimultiplex`, the short version of the **--tag-list** option used to
+  specify the list of tags and primers to be used for the demultiplexing has
+  been changed from `-t` to `-s`.
+
+- The command `obifind` is now renamed `obitaxonomy`.
+
+- The **--taxdump** option used to specify the path to the taxdump containing
+  the NCBI taxonomy has been renamed to **--taxonomy**.
+
+### Bug fixes
+
+- Correction of a bug when using paired sequence file with the **--out** option.
+
+- Correction of a bug in `obitag` when trying to annotate very short sequences of
+  4 bases or less.
+  
+
+- In `obipairing`, correct the stats `seq_a_single` and `seq_b_single` when
+  on right alignment mode
+
+- Not really a bug but the memory impact of `obiuniq` has been reduced by reducing
+  the batch size and not reading the qualities from the fastq files as `obiuniq`
+  is producing only fasta output without qualities.
+
+-   In `obitag`, correct the wrong assignment of the **obitag_bestmatch**
+    attribute.
+
+-   In `obiclean`, the **--no-progress-bar** option disables all progress bars,
+    not just the data.
+
+-   Several fixes in reading FASTA and FASTQ files, including some code
+    simplification and factorization.
+
+-   Fixed a bug in all obitools that caused the same file to be processed
+    multiple times, when specifying a directory name as input.
+
+
+### New features
+
+-   `obigrep` add a new **--valid-taxid** option to keep only sequence with a
+    valid taxid
+
+-   `obiclean` add a new **--min-sample-count** option with a default value of 1,
+    asking to filter out sequences which are not occurring in at least the
+    specified number of samples.
+
+-   `obitoaxonomy` a new **--dump|D** option allows for dumping a sub-taxonomy.
+  
+-   Taxonomy dump can now be provided as a four-columns CSV file to the
+    **--taxonomy** option.
+
+-   NCBI Taxonomy dump does not need to be uncompressed and unarchived anymore. The
+    path of the tar and gziped dump file can be directly specified using the
+    **--taxonomy** option.
+
+-   Most of the time obitools identify automatically sequence file format. But
+    it fails sometimes. Two new option **--fasta** and **--fastq** are added to
+    allow the processing of the rare fasta and fastq files not recognized.
+    
+-   In `obiscript`, adds new methods to the Lua sequence object:
+    - `md5_string()`: returning the MD5 check sum as a hexadecimal string, 
+    - `subsequence(from,to)`: allows extracting a subsequence on a 0 based
+      coordinate system, upper bound excluded like in go. 
+    - `reverse_complement`: returning a sequence object corresponding to the
+      reverse complement of the current sequence.
+
+### Enhancement
+
+-   All obitools now have a **--taxonomy** option. If specified, the taxonomy is
+    loaded first and taxids annotating the sequences are validated against that
+    taxonomy. A warning is issued for any invalid taxid and for any taxid that
+    is transferred to a new taxid. The **--update-taxid** option allows these
+    old taxids to be replaced with their new equivalent in the result of the
+    obitools command.
+
+-   The scoring system used by the `obipairing` command has been changed to be
+    more coherent. In the new version, the scores associated to a match and a
+    mismatch involving a nucleotide with a quality score of 0 are equal. Which
+    is normal as a zero quality score means a perfect indecision on the read
+    nucleotide, therefore there is no reason to penalize a match differently
+    from a mismatch (see 
+    https://obitools4.metabarcoding.org/docs/commands/alignments/obipairing/exact-alignment/).
+
+-   In every *OBITools* command, the progress bar is automatically deactivated
+    when the standard error output is redirected.
+
+-   Because Genbank and ENA:EMBL contain very large sequences, while OBITools4
+    are optimized As Genbank and ENA:EMBL contain very large sequences, while
+    OBITools4 is optimized for short sequences, `obipcr` faces some problems
+    with excessive consumption of computer resources, especially memory. Several
+    improvements in the tuning of the default `obipcr` parameters and some new
+    features, currently only available for FASTA and FASTQ file readers, have
+    been implemented to limit the memory impact of `obipcr` without changing the
+    computational efficiency too much.
+
+-   Logging system and therefore format, have been homogenized.
 
 ## August 2nd, 2024. Release 4.3.0
 
-### Change of git repositiory
+### Change of git repository
 
--   The OBITools4 git repository has been moved to the github repository. 
+-   The OBITools4 git repository has been moved to the GitHub repository. 
     The new address is: https://github.com/metabarcoding/obitools4.
     Take care for using the new install script for retrieving the new version.
 
     ```bash
-    curl -L https://raw.githubusercontent.com/metabarcoding/obitools4/master/install_obitools.sh \
+    curl -L https://metabarcoding.org/obitools4/install.sh \
       | bash
     ```
 
     or with options:
 
     ```bash
-    curl -L https://raw.githubusercontent.com/metabarcoding/obitools4/master/install_obitools.sh \
+    curl -L https://metabarcoding.org/obitools4/install.sh \
       | bash -s -- --install-dir test_install --obitools-prefix k
     ```
-    
-### CPU limitation
-
--   By default, *OBITools4* tries to use all the computing power available on
-    your computer. In some circumstances this can be problematic (e.g. if you
-    are running on a computer cluster managed by your university). You can limit
-    the number of CPU cores used by *OBITools4* or by using the **--max-cpu**
-    option or by setting the **OBIMAXCPU** environment variable. Some strange
-    behaviour of *OBITools4* has been observed when users try to limit the
-    maximum number of usable CPU cores to one. This seems to be caused by the Go
-    language, and it is not obvious to get *OBITools4* to run correctly on a
-    single core in all circumstances. Therefore, if you ask to use a single
-    core, **OBITools4** will print a warning message and actually set this
-    parameter to two cores. If you really want a single core, you can use the
-    **--force-one-core** option. But be aware that this can lead to incorrect
-    calculations.
-
-### New features
-
 -   The output of the obitools will evolve to produce results only in standard
     formats such as fasta and fastq. For non-sequential data, the output will be
     in CSV format, with the separator `,`, the decimal separator `.`, and a
     header line with the column names. It is more convenient to use the output
     in other programs. For example, you can use the `csvtomd` command to
-    reformat the csv output into a markdown table. The first command to initiate
+    reformat the CSV output into a Markdown table. The first command to initiate
     this change is `obicount`, which now produces a 3-line CSV output.
 
     ```bash
@@ -57,7 +196,7 @@
     database for `obitag` is to use `obipcr` on a local copy of Genbank or EMBL.
     However, these sequence databases are known to contain many taxonomic
     errors, such as bacterial sequences annotated with the taxid of their host
-    species. obicleandb tries to detect these errors. To do this, it first keeps
+    species. `obicleandb` tries to detect these errors. To do this, it first keeps
     only sequences annotated with the taxid to which a species, genus, and
     family taxid can be assigned. Then, for each sequence, it compares the
     distance of the sequence to the other sequences belonging to the same genus
@@ -68,7 +207,7 @@
     with the p-value of the Mann-Whitney U test in the **obicleandb_trusted**
     slot. Later, the distribution of this p-value can be analyzed to determine a
     threshold. Empirically, a threshold of 0.05 is a good compromise and allows
-    to filter out less than 1‰ of the sequences. These sequences can then be
+    filtering out less than 1‰ of the sequences. These sequences can then be
     removed using `obigrep`.
 
 -   Adds a new `obijoin` utility to join information contained in a sequence
@@ -78,16 +217,16 @@
 
 -   Adds a new tool `obidemerge` to demerge a `merge_xxx` slot by recreating the 
     multiple identical sequences having the slot `xxx` recreated with its initial
-    value and the sequence count set to the number of occurences refered in the
+    value and the sequence count set to the number of occurrences referred in the
     `merge_xxx` slot. During the operation, the `merge_xxx` slot is removed.
 
 -   Adds CSV as one of the input format for every obitools command. To encode
-    sequence the CSV file must includes a column named `sequence` and another
+    sequence the CSV file must include a column named `sequence` and another
     column named `id`. An extra column named `qualities` can be added to specify 
-    the quality scores of the sequence following the same ascii encoding than the
+    the quality scores of the sequence following the same ASCII encoding than the
     fastq format. All the other columns will be considered as annotations and will
     be interpreted as JSON objects encoding potentially for atomic values. If a 
-    calumn value can not be decoded as JSON it will be considered as a string.
+    column value can not be decoded as JSON it will be considered as a string.
 
 -   A new option **--version** has been added to every obitools command. It will
     print the version of the command.
@@ -96,8 +235,8 @@
     quality scores from a BioSequence object.\
 
 -   In `obimultuplex` the ngsfilter file describing the samples can be no provided
-    not only using the classical nfsfilter format but also using the csv format.
+    not only using the classical ngsfilter format but also using the CSV format.
-    When using csv, the first line must contain the column names. 5 columns are
+    When using CSV, the first line must contain the column names. 5 columns are
     expected:
 
     -   `experiment` the name of the experiment
@@ -113,43 +252,34 @@
 
     Supplementary columns are allowed. Their names and content will be used to
     annotate the sequence corresponding to the sample, as the `key=value;` did
-    in the nfsfilter format.
+    in the ngsfilter format.
 
     The CSV format used allows for comment lines starting with `#` character.
-    Special data lines starting with `@param` in the first column allow to
+    Special data lines starting with `@param` in the first column allow configuring the algorithm. The options **--template** provided an over
-    configure the algorithm. The options **--template** provided an over
+    commented example of the CSV format, including all the possible options.
-    commented example of the csv format, including all the possible options.
+    
+### CPU limitation
 
-### Enhancement
+-   By default, *OBITools4* tries to use all the computing power available on
+    your computer. In some circumstances this can be problematic (e.g. if you
+    are running on a computer cluster managed by your university). You can limit
+    the number of CPU cores used by *OBITools4* or by using the **--max-cpu**
+    option or by setting the **OBIMAXCPU** environment variable. Some strange
+    behavior of *OBITools4* has been observed when users try to limit the
+    maximum number of usable CPU cores to one. This seems to be caused by the Go
+    language, and it is not obvious to get *OBITools4* to run correctly on a
+    single core in all circumstances. Therefore, if you ask to use a single
+    core, **OBITools4** will print a warning message and actually set this
+    parameter to two cores. If you really want a single core, you can use the
+    **--force-one-core** option. But be aware that this can lead to incorrect
+    calculations.
 
--   In every *OBITools* command, the progress bar are automatically deactivated
-    when the standard error output is redirected.
--   Because Genbank and ENA:EMBL contain very large sequences, while OBITools4
-    are optimized As Genbank and ENA:EMBL contain very large sequences, while
-    OBITools4 is optimised for short sequences, `obipcr` faces some problems
-    with excessive consumption of computer resources, especially memory. Several
-    improvements in the tuning of the default `obipcr` parameters and some new
-    features, currently only available for FASTA and FASTQ file readers, have
-    been implemented to limit the memory impact of `obipcr` without changing the
-    computational efficiency too much.
--   Logging system and therefore format, have been homogenized.
-
-### Bug
-
--   In `obitag`, correct the wrong assignment of the **obitag_bestmatch**
-    attribute.
--   In `obiclean`, the **--no-progress-bar** option disables all progress bars,
-    not just the data.
--   Several fixes in reading FASTA and FASTQ files, including some code
-    simplification and and factorization.
--   Fixed a bug in all obitools that caused the same file to be processed
-    multiple times. when specifying a directory name as input.
 
 ## April 2nd, 2024. Release 4.2.0
 
 ### New features
 
--   A new OBITools named `obiscript` allows to process each sequence according
+-   A new OBITools named `obiscript` allows processing each sequence according
     to a Lua script. This is an experimental tool. The **--template** option
     allows for generating an example script on the `stdout`.
 
@@ -157,7 +287,7 @@
 
 -   Two of the main class `obiseq.SeqWorker` and `obiseq.SeqWorker` have their
     declaration changed. Both now return two values a `obiseq.BioSequenceSlice`
-    and an `error`. This allow a worker to return potentially several sequences
+    and an `error`. This allows a worker to return potentially several sequences
     as the result of the processing of a single sequence, or zero, which is
     equivalent to filter out the input sequence.
 
@@ -165,12 +295,12 @@
 
 -   In `obitag` if the reference database contains sequences annotated by taxid
     not referenced in the taxonomy, the corresponding sequences are discarded
-    from the reference database and a warning indicating the sequence id and the
+    from the reference database and a warning indicating the sequence *id* and the
     wrong taxid is emitted.
 -   The bug corrected in the parsing of EMBL and Genbank files as implemented in
     version 4.1.2 of OBITools4, potentially induced some reduction in the
     performance of the parsing. This should have been now fixed.
--   In the same idea, parsing of genbank and EMBL files were reading and storing
+-   In the same idea, parsing of Genbank and EMBL files were reading and storing
     in memory not only the sequence but also the annotations (features table).
     Up to now none of the OBITools are using this information, but with large
     complete genomes, it is occupying a lot of memory. To reduce this impact,
@@ -209,7 +339,7 @@
 
 ### New feature
 
--   In `obimatrix` a **--transpose** option allows to transpose the produced
+-   In `obimatrix` a **--transpose** option allows transposing the produced
     matrix table in CSV format.
 -   In `obitpairing` and `obipcrtag` two new options **--exact-mode** and
     **--fast-absolute** to control the heuristic used in the alignment
@@ -217,7 +347,7 @@
     the exact algorithm at the cost of a speed. **--fast-absolute** change the
     scoring schema of the heuristic.
 -   In `obiannotate` adds the possibility to annotate the first match of a
-    pattern using the same algorithm than the one used in `obipcr` and
+    pattern using the same algorithm as the one used in `obipcr` and
     `obimultiplex`. For that four option were added :
     -   **--pattern** : to specify the pattern. It can use IUPAC codes and
         position with no error tolerated has to be followed by a `#` character.
@@ -298,7 +428,7 @@
 
 ### Bugs
 
--   in the obitools language, the `composition` function now returns a map
+-   In the obitools language, the `composition` function now returns a map
     indexed by lowercase string "a", "c", "g", "t" and "o" for other instead of
     being indexed by the ASCII codes of the corresponding letters.
 -   Correction of the reverse-complement operation. Every reverse complement of
@@ -311,18 +441,18 @@
     duplicating the quality values. This made `obimultiplex` to produce fastq
     files with sequences having quality values duplicated.
 
-### Becareful
+### Be careful
 
 GO 1.21.0 is out, and it includes new functionalities which are used in the
-OBITools4 code. If you use the recommanded method for compiling OBITools on your
+OBITools4 code. If you use the recommended method for compiling OBITools on your
-computer, their is no problem, as the script always load the latest GO version.
+computer, there is no problem, as the script always load the latest GO version.
-If you rely on you personnal GO install, please think to update.
+If you rely on your personal GO install, please think to update.
 
 ## August 29th, 2023. Release 4.0.5
 
 ### Bugs
 
--   Patch a bug in the `obiseq.BioSequence` constructor leading to a error on
+-   Patch a bug in the `obiseq.BioSequence` constructor leading to an error on
     almost every obitools. The error message indicates : `fatal error: sync:
     unlock of unlocked mutex` This bug was introduced in the release 4.0.4
 
@@ -341,7 +471,7 @@ If you rely on you personnal GO install, please think to update.
     data structure to limit the number of alignments actually computed. This
     increase a bit the speed of both the software. `obirefidx` is nevertheless
     still too slow compared to my expectation.
--   Switch to a parallel version of the gzip library, allowing for high speed
+-   Switch to a parallel version of the GZIP library, allowing for high speed
     compress and decompress operation on files.
 
 ### New feature
@@ -385,12 +515,12 @@ If you rely on you personnal GO install, please think to update.
               --unidentified not_assigned.fastq
     ```
 
-    the command produced four files : `tagged_library_R1.fastq` and
+    The command produced four files : `tagged_library_R1.fastq` and
     `tagged_library_R2.fastq` containing the assigned reads and
     `not_assigned_R1.fastq` and `not_assigned_R2.fastq` containing the
     unassignable reads.
 
-    the tagged library files can then be split using `obidistribute`:
+    The tagged library files can then be split using `obidistribute`:
 
     ```{bash}
     mkdir pcr_reads
@@ -400,9 +530,9 @@ If you rely on you personnal GO install, please think to update.
 
 -   Adding of two options **--add-lca-in** and **--lca-error** to `obiannotate`.
     These options aim to help during construction of reference database using
-    `obipcr`. On obipcr output, it is commonly run obiuniq. To merge identical
+    `obipcr`. On `obipcr` output, it is commonly run `obiuniq`. To merge identical
     sequences annotated with different taxids, it is now possible to use the
-    following strategie :
+    following strategies :
 
     ```{bash}
     obiuniq -m taxid myrefdb.obipcr.fasta \
@@ -433,7 +563,7 @@ If you rely on you personnal GO install, please think to update.
 -   Correction of a bug in `obiconsensus` leading into the deletion of a base
     close to the beginning of the consensus sequence.
 
-## March 31th, 2023. Release 4.0.2
+## March 31st, 2023. Release 4.0.2
 
 ### Compiler change
 
@@ -444,15 +574,15 @@ If you rely on you personnal GO install, please think to update.
 -   Add the possibility for looking pattern with indels. This has been added to
     `obimultiplex` through the **--with-indels** option.
 -   Every obitools command has a **--pprof** option making the command
-    publishing a profiling web site available at the address :
+    publishing a profiling website available at the address :
     <http://localhost:8080/debug/pprof/>
 -   A new `obiconsensus` command has been added. It is a prototype. It aims to
     build a consensus sequence from a set of reads. The consensus is estimated
     for all the sequences contained in the input file. If several input files,
     or a directory name are provided the result contains a consensus per file.
-    The id of the sequence is the name of the input file depleted of its
+    The *id* of the sequence is the name of the input file depleted of its
     directory name and of all its extensions.
--   In `obipcr` an experimental option **--fragmented** allows for spliting very
+-   In `obipcr` an experimental option **--fragmented** allows for splitting very
     long query sequences into shorter fragments with an overlap between the two
     contiguous fragment insuring that no amplicons are missed despite the split.
     As a site effect some amplicon can be identified twice.
@@ -495,7 +625,7 @@ If you rely on you personnal GO install, please think to update.
 ### Enhancement
 
 -   *OBITools* are automatically processing all the sequences files contained in
-    a directory and its sub-directory\
+    a directory and its subdirectory\
     recursively if its name is provided as input. To process easily Genbank
     files, the corresponding filename extensions have been added. Today the
     following extensions are recognized as sequence files : `.fasta`, `.fastq`,
@@ -512,7 +642,7 @@ If you rely on you personnal GO install, please think to update.
     export OBICPUMAX=4
     ```
 
--   Adds a new option --out\|-o allowing to specify the name of an outpout file.
+-   Adds a new option --out\|-o allowing to specify the name of an output file.
 
     ``` bash
     obiconvert -o xyz.fasta xxx.fastq
@@ -534,10 +664,10 @@ If you rely on you personnal GO install, please think to update.
     matched files remain consistent when processed.
 
 -   Adding of the function `ifelse` to the expression language for computing
-    conditionnal values.
+    conditional values.
 
 -   Adding two function to the expression language related to sequence
-    conposition : `composition` and `gcskew`. Both are taking a sequence as
+    composition : `composition` and `gcskew`. Both are taking a sequence as
     single argument.
 
 ## February 18th, 2023. Release 4.0.0
@@ -545,8 +675,8 @@ If you rely on you personnal GO install, please think to update.
 It is the first version of the *OBITools* version 4. I decided to tag then
 following two weeks of intensive data analysis with them allowing to discover
 many small bugs present in the previous non-official version. Obviously other
-bugs are certainly persent in the code, and you are welcome to use the git
+bugs are certainly present in the code, and you are welcome to use the git
-ticket system to mention them. But they seems to produce now reliable results.
+ticket system to mention them. But they seem to produce now reliable results.
 
 ### Corrected bugs
 
@@ -554,11 +684,11 @@ ticket system to mention them. But they seems to produce now reliable results.
     of sequences and to the production of incorrect file because of the last
     sequence record, sometime truncated in its middle. This was only occurring
     when more than a single CPU was used. It was affecting every obitools.
--   The `obiparing` software had a bug in the right aligment procedure. This led
+-   The `obiparing` software had a bug in the right alignment procedure. This led
-    to the non alignment of very sort barcode during the paring of the forward
+    to the non-alignment of very sort barcode during the paring of the forward
     and reverse reads.
--   The `obipairing` tools had a non deterministic comportment when aligning a
+-   The `obipairing` tools had a non-deterministic comportment when aligning a
-    paor very low quality reads. This induced that the result of the same low
+    pair very low quality reads. This induced that the result of the same low
     quality read pair was not the same from run to run.
 
 ### New features
@@ -566,11 +696,10 @@ ticket system to mention them. But they seems to produce now reliable results.
 -   Adding of a `--compress|-Z` option to every obitools allowing to produce
     `gz` compressed output. OBITools were already able to deal with gziped input
     files transparently. They can now produce their results in the same format.
-    - Adding of a `--append|-A` option to the `obidistribute` tool. It allows to
+    - Adding of a `--append|-A` option to the `obidistribute` tool. It allows appending the result of an `obidistribute` execution to preexisting files. -
-    append the result of an `obidistribute` execution to preexisting files. -
     Adding of a `--directory|-d` option to the `obidistribute` tool. It allows
-    to declare a secondary classification key over the one defined by the
+    declaring a secondary classification key over the one defined by the
-    '--category\|-c\` option. This extra key leads to produce directories in
+    `--category\|-c\` option. This extra key leads to produce directories in
     which files produced according to the primary criterion are stored.
 -   Adding of the functions `subspc`, `printf`, `int`, `numeric`, and `bool` to
     the expression language.

blackboard/Prospective/kmer_disk_index_plan.md

@@ -0,0 +1,508 @@
+# Plan de refonte du package obikmer : index disk-based par partitions minimizer
+
+## Constat
+
+Les roaring64 bitmaps ne sont pas adaptés au stockage de 10^10 k-mers
+(k=31) dispersés sur un espace de 2^62. L'overhead structurel (containers
+roaring par high key 32 bits) dépasse la taille des données elles-mêmes,
+et les opérations `Or()` entre bitmaps fragmentés ne terminent pas en
+temps raisonnable.
+
+## Principe de la nouvelle architecture
+
+Un `KmerSet` est un ensemble trié de k-mers canoniques (uint64) stocké
+sur disque, partitionné par minimizer. Chaque partition est un fichier
+binaire contenant des uint64 triés, compressés par delta-varint.
+
+Un `KmerSetGroup` est un répertoire contenant N ensembles partitionnés
+de la même façon (même k, même m, même P).
+
+Un `KmerSet` est un `KmerSetGroup` de taille 1 (singleton).
+
+Les opérations ensemblistes se font partition par partition, en merge
+streaming, sans charger l'index complet en mémoire.
+
+## Cycle de vie d'un index
+
+L'index a deux phases distinctes :
+
+1. **Phase de construction (mutable)** : on ouvre un index, on y ajoute
+   des séquences. Pour chaque séquence, les super-kmers sont extraits
+   et écrits de manière compacte (2 bits/base) dans le fichier
+   temporaire de partition correspondant (`minimizer % P`). Les
+   super-kmers sont une représentation compressée naturelle des k-mers
+   chevauchants : un super-kmer de longueur L encode L-k+1 k-mers en
+   ne stockant que ~L/4 bytes au lieu de (L-k+1) × 8 bytes.
+
+2. **Phase de clôture (optimisation)** : on ferme l'index, ce qui
+   déclenche le traitement **partition par partition** (indépendant,
+   parallélisable) :
+   - Charger les super-kmers de la partition
+   - En extraire tous les k-mers canoniques
+   - Trier le tableau de k-mers
+   - Dédupliquer (et compter si FrequencyFilter)
+   - Delta-encoder et écrire le fichier .kdi final
+   Après clôture, l'index est statique et immuable.
+
+3. **Phase de lecture (immutable)** : opérations ensemblistes,
+   Jaccard, Quorum, Contains, itération. Toutes en streaming.
+
+---
+
+## Format sur disque
+
+### Index finalisé
+
+```
+index_dir/
+  metadata.toml
+  set_0/
+    part_0000.kdi
+    part_0001.kdi
+    ...
+    part_{P-1}.kdi
+  set_1/
+    part_0000.kdi
+    ...
+  ...
+  set_{N-1}/
+    ...
+```
+
+### Fichiers temporaires pendant la construction
+
+```
+index_dir/
+  .build/
+    set_0/
+      part_0000.skm          # super-kmers encodés 2 bits/base
+      part_0001.skm
+      ...
+    set_1/
+      ...
+```
+
+Le répertoire `.build/` est supprimé après Close().
+
+### metadata.toml
+
+```toml
+id = "mon_index"
+k = 31
+m = 13
+partitions = 1024
+type = "KmerSetGroup"       # ou "KmerSet" (N=1)
+size = 3                    # nombre de sets (N)
+sets_ids = ["genome_A", "genome_B", "genome_C"]
+
+[user_metadata]
+organism = "Triticum aestivum"
+
+[sets_metadata]
+# métadonnées individuelles par set si nécessaire
+```
+
+### Fichier .kdi (Kmer Delta Index)
+
+Format binaire :
+
+```
+[magic: 4 bytes "KDI\x01"]
+[count: uint64 little-endian]       # nombre de k-mers dans cette partition
+[first: uint64 little-endian]       # premier k-mer (valeur absolue)
+[delta_1: varint]                   # arr[1] - arr[0]
+[delta_2: varint]                   # arr[2] - arr[1]
+...
+[delta_{count-1}: varint]           # arr[count-1] - arr[count-2]
+```
+
+Varint : encoding unsigned, 7 bits utiles par byte, bit de poids fort
+= continuation (identique au varint protobuf).
+
+Fichier vide (partition sans k-mer) : magic + count=0.
+
+### Fichier .skm (Super-Kmer temporaire)
+
+Format binaire, séquence de super-kmers encodés :
+
+```
+[len: uint16 little-endian]         # longueur du super-kmer en bases
+[sequence: ceil(len/4) bytes]       # séquence encodée 2 bits/base, packed
+...
+```
+
+**Compression par rapport au stockage de k-mers bruts** :
+
+Un super-kmer de longueur L contient L-k+1 k-mers.
+- Stockage super-kmer : 2 + ceil(L/4) bytes
+- Stockage k-mers bruts : (L-k+1) × 8 bytes
+
+Exemple avec k=31, super-kmer typique L=50 :
+- Super-kmer : 2 + 13 = 15 bytes → encode 20 k-mers
+- K-mers bruts : 20 × 8 = 160 bytes
+- **Facteur de compression : ~10×**
+
+Pour un génome de 10 Gbases (~10^10 k-mers bruts) :
+- K-mers bruts : ~80 Go par set temporaire
+- Super-kmers : **~8 Go** par set temporaire
+
+Avec FrequencyFilter et couverture 30× :
+- K-mers bruts : ~2.4 To
+- Super-kmers : **~240 Go**
+
+---
+
+## FrequencyFilter
+
+Le FrequencyFilter n'est plus un type de données séparé. C'est un
+**mode de construction** du builder. Le résultat est un KmerSetGroup
+standard.
+
+### Principe
+
+Pendant la construction, tous les super-kmers sont écrits dans les
+fichiers temporaires .skm, y compris les doublons (chaque occurrence
+de chaque séquence est écrite).
+
+Pendant Close(), pour chaque partition :
+1. Charger tous les super-kmers de la partition
+2. Extraire tous les k-mers canoniques dans un tableau []uint64
+3. Trier le tableau
+4. Parcourir linéairement : les k-mers identiques sont consécutifs
+5. Compter les occurrences de chaque k-mer
+6. Si count >= minFreq → écrire dans le .kdi final (une seule fois)
+7. Sinon → ignorer
+
+### Dimensionnement
+
+Pour un génome de 10 Gbases avec couverture 30× :
+- N_brut ≈ 3×10^11 k-mers bruts
+- Espace temporaire .skm ≈ 240 Go (compressé super-kmer)
+- RAM par partition pendant Close() :
+  Avec P=1024 : ~3×10^8 k-mers/partition × 8 = **~2.4 Go**
+  Avec P=4096 : ~7.3×10^7 k-mers/partition × 8 = **~600 Mo**
+
+Le choix de P détermine le compromis nombre de fichiers vs RAM par
+partition.
+
+### Sans FrequencyFilter (déduplication simple)
+
+Pour de la déduplication simple (chaque k-mer écrit une fois), le
+builder peut dédupliquer au niveau des buffers en RAM avant flush.
+Cela réduit significativement l'espace temporaire car les doublons
+au sein d'un même buffer (provenant de séquences proches) sont
+éliminés immédiatement.
+
+---
+
+## API publique visée
+
+### Structures
+
+```go
+// KmerSetGroup est l'entité de base.
+// Un KmerSet est un KmerSetGroup avec Size() == 1.
+type KmerSetGroup struct {
+    // champs internes : path, k, m, P, N, metadata, état
+}
+
+// KmerSetGroupBuilder construit un KmerSetGroup mutable.
+type KmerSetGroupBuilder struct {
+    // champs internes : buffers I/O par partition et par set,
+    // fichiers temporaires .skm, paramètres (minFreq, etc.)
+}
+```
+
+### Construction
+
+```go
+// NewKmerSetGroupBuilder crée un builder pour un nouveau KmerSetGroup.
+//   directory : répertoire de destination
+//   k : taille des k-mers (1-31)
+//   m : taille des minimizers (-1 pour auto = ceil(k/2.5))
+//   n : nombre de sets dans le groupe
+//   P : nombre de partitions (-1 pour auto)
+//   options : options de construction (FrequencyFilter, etc.)
+func NewKmerSetGroupBuilder(directory string, k, m, n, P int,
+    options ...BuilderOption) (*KmerSetGroupBuilder, error)
+
+// WithMinFrequency active le mode FrequencyFilter.
+// Seuls les k-mers vus >= minFreq fois sont conservés dans l'index
+// final. Les super-kmers sont écrits avec leurs doublons pendant
+// la construction ; le comptage exact se fait au Close().
+func WithMinFrequency(minFreq int) BuilderOption
+
+// AddSequence extrait les super-kmers d'une séquence et les écrit
+// dans les fichiers temporaires de partition du set i.
+func (b *KmerSetGroupBuilder) AddSequence(setIndex int, seq *obiseq.BioSequence)
+
+// AddSuperKmer écrit un super-kmer dans le fichier temporaire de
+// sa partition pour le set i.
+func (b *KmerSetGroupBuilder) AddSuperKmer(setIndex int, sk SuperKmer)
+
+// Close finalise la construction :
+//   - flush des buffers d'écriture
+//   - pour chaque partition de chaque set (parallélisable) :
+//     - charger les super-kmers depuis le .skm
+//     - extraire les k-mers canoniques
+//     - trier, dédupliquer (compter si freq filter)
+//     - delta-encoder et écrire le .kdi
+//   - écrire metadata.toml
+//   - supprimer le répertoire .build/
+// Retourne le KmerSetGroup en lecture seule.
+func (b *KmerSetGroupBuilder) Close() (*KmerSetGroup, error)
+```
+
+### Lecture et opérations
+
+```go
+// OpenKmerSetGroup ouvre un index finalisé en lecture seule.
+func OpenKmerSetGroup(directory string) (*KmerSetGroup, error)
+
+// --- Métadonnées (API inchangée) ---
+func (ksg *KmerSetGroup) K() int
+func (ksg *KmerSetGroup) M() int          // nouveau : taille du minimizer
+func (ksg *KmerSetGroup) Partitions() int  // nouveau : nombre de partitions
+func (ksg *KmerSetGroup) Size() int
+func (ksg *KmerSetGroup) Id() string
+func (ksg *KmerSetGroup) SetId(id string)
+func (ksg *KmerSetGroup) HasAttribute(key string) bool
+func (ksg *KmerSetGroup) GetAttribute(key string) (interface{}, bool)
+func (ksg *KmerSetGroup) SetAttribute(key string, value interface{})
+// ... etc (toute l'API attributs actuelle est conservée)
+
+// --- Opérations ensemblistes ---
+// Toutes produisent un nouveau KmerSetGroup singleton sur disque.
+// Opèrent partition par partition en streaming.
+
+func (ksg *KmerSetGroup) Union(outputDir string) (*KmerSetGroup, error)
+func (ksg *KmerSetGroup) Intersect(outputDir string) (*KmerSetGroup, error)
+func (ksg *KmerSetGroup) Difference(outputDir string) (*KmerSetGroup, error)
+func (ksg *KmerSetGroup) QuorumAtLeast(q int, outputDir string) (*KmerSetGroup, error)
+func (ksg *KmerSetGroup) QuorumExactly(q int, outputDir string) (*KmerSetGroup, error)
+func (ksg *KmerSetGroup) QuorumAtMost(q int, outputDir string) (*KmerSetGroup, error)
+
+// --- Opérations entre deux KmerSetGroups ---
+// Les deux groupes doivent avoir les mêmes k, m, P.
+
+func (ksg *KmerSetGroup) UnionWith(other *KmerSetGroup, outputDir string) (*KmerSetGroup, error)
+func (ksg *KmerSetGroup) IntersectWith(other *KmerSetGroup, outputDir string) (*KmerSetGroup, error)
+
+// --- Métriques (résultat en mémoire, pas de sortie disque) ---
+
+func (ksg *KmerSetGroup) JaccardDistanceMatrix() *obidist.DistMatrix
+func (ksg *KmerSetGroup) JaccardSimilarityMatrix() *obidist.DistMatrix
+
+// --- Accès individuel ---
+
+func (ksg *KmerSetGroup) Len(setIndex ...int) uint64
+func (ksg *KmerSetGroup) Contains(setIndex int, kmer uint64) bool
+func (ksg *KmerSetGroup) Iterator(setIndex int) iter.Seq[uint64]
+```
+
+---
+
+## Implémentation interne
+
+### Primitives bas niveau
+
+**`varint.go`** : encode/decode varint uint64
+
+```go
+func EncodeVarint(w io.Writer, v uint64) (int, error)
+func DecodeVarint(r io.Reader) (uint64, error)
+```
+
+### Format .kdi
+
+**`kdi_writer.go`** : écriture d'un fichier .kdi à partir d'un flux
+trié de uint64 (delta-encode au vol).
+
+```go
+type KdiWriter struct { ... }
+func NewKdiWriter(path string) (*KdiWriter, error)
+func (w *KdiWriter) Write(kmer uint64) error
+func (w *KdiWriter) Close() error
+```
+
+**`kdi_reader.go`** : lecture streaming d'un fichier .kdi (décode
+les deltas au vol).
+
+```go
+type KdiReader struct { ... }
+func NewKdiReader(path string) (*KdiReader, error)
+func (r *KdiReader) Next() (uint64, bool)
+func (r *KdiReader) Count() uint64
+func (r *KdiReader) Close() error
+```
+
+### Format .skm
+
+**`skm_writer.go`** : écriture de super-kmers encodés 2 bits/base.
+
+```go
+type SkmWriter struct { ... }
+func NewSkmWriter(path string) (*SkmWriter, error)
+func (w *SkmWriter) Write(sk SuperKmer) error
+func (w *SkmWriter) Close() error
+```
+
+**`skm_reader.go`** : lecture de super-kmers depuis un fichier .skm.
+
+```go
+type SkmReader struct { ... }
+func NewSkmReader(path string) (*SkmReader, error)
+func (r *SkmReader) Next() (SuperKmer, bool)
+func (r *SkmReader) Close() error
+```
+
+### Merge streaming
+
+**`kdi_merge.go`** : k-way merge de plusieurs flux triés.
+
+```go
+type KWayMerge struct { ... }
+func NewKWayMerge(readers []*KdiReader) *KWayMerge
+func (m *KWayMerge) Next() (kmer uint64, count int, ok bool)
+func (m *KWayMerge) Close() error
+```
+
+### Builder
+
+**`kmer_set_builder.go`** : construction d'un KmerSetGroup.
+
+Le builder gère :
+- P × N écrivains .skm bufferisés (un par partition × set)
+- À la clôture : traitement partition par partition
+  (parallélisable sur plusieurs cores)
+
+Gestion mémoire des buffers d'écriture :
+- Chaque SkmWriter a un buffer I/O de taille raisonnable (~64 Ko)
+- Avec P=1024 et N=1 : 1024 × 64 Ko = 64 Mo de buffers
+- Avec P=1024 et N=10 : 640 Mo de buffers
+- Pas de buffer de k-mers en RAM : tout est écrit sur disque
+  immédiatement via les super-kmers
+
+RAM pendant Close() (tri d'une partition) :
+- Charger les super-kmers → extraire les k-mers → tableau []uint64
+- Avec P=1024 et 10^10 k-mers/set : ~10^7 k-mers/partition × 8 = ~80 Mo
+- Avec FrequencyFilter (doublons) et couverture 30× :
+  ~3×10^8/partition × 8 = ~2.4 Go (ajustable via P)
+
+### Structure disk-based
+
+**`kmer_set_disk.go`** : KmerSetGroup en lecture seule.
+
+**`kmer_set_disk_ops.go`** : opérations ensemblistes par merge
+streaming partition par partition.
+
+---
+
+## Ce qui change par rapport à l'API actuelle
+
+### Changements de sémantique
+
+| Aspect | Ancien (roaring) | Nouveau (disk-based) |
+|---|---|---|
+| Stockage | En mémoire (roaring64.Bitmap) | Sur disque (.kdi delta-encoded) |
+| Temporaire construction | En mémoire | Super-kmers sur disque (.skm 2 bits/base) |
+| Mutabilité | Mutable à tout moment | Builder → Close() → immutable |
+| Opérations ensemblistes | Résultat en mémoire | Résultat sur disque (nouveau répertoire) |
+| Contains | O(1) roaring lookup | O(log n) recherche binaire sur .kdi |
+| Itération | Roaring iterator | Streaming décodage delta-varint |
+
+### API conservée (signatures identiques ou quasi-identiques)
+
+- `KmerSetGroup` : `K()`, `Size()`, `Id()`, `SetId()`
+- Toute l'API attributs
+- `JaccardDistanceMatrix()`, `JaccardSimilarityMatrix()`
+- `Len()`, `Contains()`
+
+### API modifiée
+
+- `Union()`, `Intersect()`, etc. : ajout du paramètre `outputDir`
+- `QuorumAtLeast()`, etc. : idem
+- Construction : `NewKmerSetGroupBuilder()` + `AddSequence()` + `Close()`
+  au lieu de manipulation directe
+
+### API supprimée
+
+- `KmerSet` comme type distinct (remplacé par KmerSetGroup singleton)
+- `FrequencyFilter` comme type distinct (mode du Builder)
+- Tout accès direct à `roaring64.Bitmap`
+- `KmerSet.Copy()` (copie de répertoire à la place)
+- `KmerSet.Union()`, `.Intersect()`, `.Difference()` (deviennent méthodes
+  de KmerSetGroup avec outputDir)
+
+---
+
+## Fichiers à créer / modifier dans pkg/obikmer
+
+### Nouveaux fichiers
+
+| Fichier | Contenu |
+|---|---|
+| `varint.go` | Encode/Decode varint uint64 |
+| `kdi_writer.go` | Écrivain de fichiers .kdi (delta-encoded) |
+| `kdi_reader.go` | Lecteur streaming de fichiers .kdi |
+| `skm_writer.go` | Écrivain de super-kmers encodés 2 bits/base |
+| `skm_reader.go` | Lecteur de super-kmers depuis .skm |
+| `kdi_merge.go` | K-way merge streaming de flux triés |
+| `kmer_set_builder.go` | KmerSetGroupBuilder (construction) |
+| `kmer_set_disk.go` | KmerSetGroup disk-based (lecture, métadonnées) |
+| `kmer_set_disk_ops.go` | Opérations ensemblistes streaming |
+
+### Fichiers à supprimer
+
+| Fichier | Raison |
+|---|---|
+| `kmer_set.go` | Remplacé par kmer_set_disk.go |
+| `kmer_set_group.go` | Idem |
+| `kmer_set_attributes.go` | Intégré dans kmer_set_disk.go |
+| `kmer_set_persistence.go` | L'index est nativement sur disque |
+| `kmer_set_group_quorum.go` | Intégré dans kmer_set_disk_ops.go |
+| `frequency_filter.go` | Mode du Builder, plus de type séparé |
+| `kmer_index_builder.go` | Remplacé par kmer_set_builder.go |
+
+### Fichiers conservés tels quels
+
+| Fichier | Contenu |
+|---|---|
+| `encodekmer.go` | Encodage/décodage k-mers |
+| `superkmer.go` | Structure SuperKmer |
+| `superkmer_iter.go` | IterSuperKmers, IterCanonicalKmers |
+| `encodefourmer.go` | Encode4mer |
+| `counting.go` | Count4Mer |
+| `kmermap.go` | KmerMap (usage indépendant) |
+| `debruijn.go` | Graphe de de Bruijn |
+
+---
+
+## Ordre d'implémentation
+
+1. `varint.go` + tests
+2. `skm_writer.go` + `skm_reader.go` + tests
+3. `kdi_writer.go` + `kdi_reader.go` + tests
+4. `kdi_merge.go` + tests
+5. `kmer_set_builder.go` + tests (construction + Close)
+6. `kmer_set_disk.go` (structure, métadonnées, Open)
+7. `kmer_set_disk_ops.go` + tests (Union, Intersect, Quorum, Jaccard)
+8. Adaptation de `pkg/obitools/obikindex/`
+9. Suppression des anciens fichiers roaring
+10. Adaptation des tests existants
+
+Chaque étape est testable indépendamment.
+
+---
+
+## Dépendances externes
+
+### Supprimées
+
+- `github.com/RoaringBitmap/roaring` : plus nécessaire pour les
+  index k-mers (vérifier si d'autres packages l'utilisent encore)
+
+### Ajoutées
+
+- Aucune. Varint, delta-encoding, merge, encodage 2 bits/base :
+  tout est implémentable en Go standard.

blackboard/Prospective/kmer_index_design.md

@@ -0,0 +1,213 @@
+# Index de k-mers pour génomes de grande taille
+
+## Contexte et objectifs
+
+### Cas d'usage
+
+- Indexation de k-mers longs (k=31) pour des génomes de grande taille (< 10 Go par génome)
+- Nombre de génomes : plusieurs dizaines à quelques centaines
+- Indexation en parallèle
+- Stockage sur disque
+- Possibilité d'ajouter des génomes, mais pas de modifier un génome existant
+
+### Requêtes cibles
+
+- **Présence/absence** d'un k-mer dans un génome
+- **Intersection** entre génomes
+- **Distances** : Jaccard (présence/absence) et potentiellement Bray-Curtis (comptage)
+
+### Ressources disponibles
+
+- 128 Go de RAM
+- Stockage disque
+
+---
+
+## Estimation des volumes
+
+### Par génome
+
+- **10 Go de séquence** → ~10¹⁰ k-mers bruts (chevauchants)
+- **Après déduplication** : typiquement 10-50% de k-mers uniques → **~1-5 × 10⁹ k-mers distincts**
+
+### Espace théorique
+
+- **k=31** → 62 bits → ~4.6 × 10¹⁸ k-mers possibles
+- Table d'indexation directe impossible
+
+---
+
+## Métriques de distance
+
+### Présence/absence (binaire)
+
+- **Jaccard** : |A ∩ B| / |A ∪ B|
+- **Sørensen-Dice** : 2|A ∩ B| / (|A| + |B|)
+
+### Comptage (abondance)
+
+- **Bray-Curtis** : 1 - (2 × Σ min(aᵢ, bᵢ)) / (Σ aᵢ + Σ bᵢ)
+
+Note : Pour Bray-Curtis, le stockage des comptages est nécessaire, ce qui augmente significativement la taille de l'index.
+
+---
+
+## Options d'indexation
+
+### Option 1 : Bloom Filter par génome
+
+**Principe** : Structure probabiliste pour test d'appartenance.
+
+**Avantages :**
+- Très compact : ~10 bits/élément pour FPR ~1%
+- Construction rapide, streaming
+- Facile à sérialiser/désérialiser
+- Intersection et Jaccard estimables via formules analytiques
+
+**Inconvénients :**
+- Faux positifs (pas de faux négatifs)
+- Distances approximatives
+
+**Taille estimée** : 1-6 Go par génome (selon FPR cible)
+
+#### Dimensionnement des Bloom filters
+
+```
+\mathrm{FPR} ;=; \left(1 - e^{-h n / m}\right)^h
+```
+
+
+| Bits/élément | FPR optimal | k (hash functions) |
+|--------------|-------------|---------------------|
+| 8            | ~2%         | 5-6                 |
+| 10           | ~1%         | 7                   |
+| 12           | ~0.3%       | 8                   |
+| 16           | ~0.01%      | 11                  |
+
+Formule du taux de faux positifs :
+```
+FPR ≈ (1 - e^(-kn/m))^k
+```
+Où n = nombre d'éléments, m = nombre de bits, k = nombre de hash functions.
+
+### Option 2 : Ensemble trié de k-mers
+
+**Principe** : Stocker les k-mers (uint64) triés, avec compression possible.
+
+**Avantages :**
+- Exact (pas de faux positifs)
+- Intersection/union par merge sort O(n+m)
+- Compression efficace (delta encoding sur k-mers triés)
+
+**Inconvénients :**
+- Plus volumineux : 8 octets/k-mer
+- Construction plus lente (tri nécessaire)
+
+**Taille estimée** : 8-40 Go par génome (non compressé)
+
+### Option 3 : MPHF (Minimal Perfect Hash Function)
+
+**Principe** : Fonction de hash parfaite minimale pour les k-mers présents.
+
+**Avantages :**
+- Très compact : ~3-4 bits/élément
+- Lookup O(1)
+- Exact pour les k-mers présents
+
+**Inconvénients :**
+- Construction coûteuse (plusieurs passes)
+- Statique (pas d'ajout de k-mers après construction)
+- Ne distingue pas "absent" vs "jamais vu" sans structure auxiliaire
+
+### Option 4 : Hybride MPHF + Bloom filter
+
+- MPHF pour mapping compact des k-mers présents
+- Bloom filter pour pré-filtrage des absents
+
+---
+
+## Optimisation : Indexation de (k-2)-mers pour requêtes k-mers
+
+### Principe
+
+Au lieu d'indexer directement les 31-mers dans un Bloom filter, on indexe les 29-mers. Pour tester la présence d'un 31-mer, on vérifie que les **trois 29-mers** qu'il contient sont présents :
+
+- positions 0-28
+- positions 1-29
+- positions 2-30
+
+### Analyse probabiliste
+
+Si le Bloom filter a un FPR de p pour un 29-mer individuel, le FPR effectif pour un 31-mer devient **p³** (les trois requêtes doivent toutes être des faux positifs).
+
+| FPR 29-mer | FPR 31-mer effectif |
+|------------|---------------------|
+| 10%        | 0.1%                |
+| 5%         | 0.0125%             |
+| 1%         | 0.0001%             |
+
+### Avantages
+
+1. **Moins d'éléments à stocker** : il y a moins de 29-mers distincts que de 31-mers distincts dans un génome (deux 31-mers différents peuvent partager un même 29-mer)
+
+2. **FPR drastiquement réduit** : FPR³ avec seulement 3 requêtes
+
+3. **Index plus compact** : on peut utiliser moins de bits par élément (FPR plus élevé acceptable sur le 29-mer) tout en obtenant un FPR très bas sur le 31-mer
+
+### Trade-off
+
+Un Bloom filter à **5-6 bits/élément** pour les 29-mers donnerait un FPR effectif < 0.01% pour les 31-mers, soit environ **2× plus compact** que l'approche directe à qualité égale.
+
+**Coût** : 3× plus de requêtes par lookup (mais les requêtes Bloom sont très rapides).
+
+---
+
+## Accélération des calculs de distance : MinHash
+
+### Principe
+
+Pré-calculer une "signature" compacte (sketch) de chaque génome permettant d'estimer rapidement Jaccard sans charger les index complets.
+
+### Avantages
+
+- Matrice de distances entre 100+ génomes en quelques secondes
+- Signature de taille fixe (ex: 1000-10000 hash values) quel que soit le génome
+- Stockage minimal
+
+### Utilisation
+
+1. Construction : une passe sur les k-mers de chaque génome
+2. Distance : comparaison des sketches en O(taille du sketch)
+
+---
+
+## Architecture recommandée
+
+### Pour présence/absence + Jaccard
+
+1. **Index principal** : Bloom filter de (k-2)-mers avec l'optimisation décrite
+   - Compact (~3-5 Go par génome)
+   - FPR très bas pour les k-mers grâce aux requêtes triples
+
+2. **Sketches MinHash** : pour calcul rapide des distances entre génomes
+   - Quelques Ko par génome
+   - Permet exploration rapide de la matrice de distances
+
+### Pour comptage + Bray-Curtis
+
+1. **Index principal** : k-mers triés + comptages
+   - uint64 (k-mer) + uint8/uint16 (count)
+   - Compression delta possible
+   - Plus volumineux mais exact
+
+2. **Sketches** : variantes de MinHash pour données pondérées (ex: HyperMinHash)
+
+---
+
+## Prochaines étapes
+
+1. Implémenter un Bloom filter optimisé pour k-mers
+2. Implémenter l'optimisation (k-2)-mer → k-mer
+3. Implémenter MinHash pour les sketches
+4. Définir le format de sérialisation sur disque
+5. Benchmarker sur des génomes réels

blackboard/architechture/architecture-commande-obitools.md

@@ -0,0 +1,735 @@
+# Architecture d'une commande OBITools
+
+## Vue d'ensemble
+
+Une commande OBITools suit une architecture modulaire et standardisée qui sépare clairement les responsabilités entre :
+- Le package de la commande dans `pkg/obitools/<nom_commande>/`
+- L'exécutable dans `cmd/obitools/<nom_commande>/`
+
+Cette architecture favorise la réutilisabilité du code, la testabilité et la cohérence entre les différentes commandes de la suite OBITools.
+
+## Structure du projet
+
+```
+obitools4/
+├── pkg/obitools/
+│   ├── obiconvert/          # Commande de conversion (base pour toutes)
+│   │   ├── obiconvert.go    # Fonctions vides (pas d'implémentation)
+│   │   ├── options.go       # Définition des options CLI
+│   │   ├── sequence_reader.go  # Lecture des séquences
+│   │   └── sequence_writer.go  # Écriture des séquences
+│   ├── obiuniq/             # Commande de déréplication
+│   │   ├── obiuniq.go       # (fichier vide)
+│   │   ├── options.go       # Options spécifiques à obiuniq
+│   │   └── unique.go        # Implémentation du traitement
+│   ├── obipairing/          # Assemblage de lectures paired-end
+│   ├── obisummary/          # Résumé de fichiers de séquences
+│   └── obimicrosat/         # Détection de microsatellites
+└── cmd/obitools/
+    ├── obiconvert/
+    │   └── main.go          # Point d'entrée de la commande
+    ├── obiuniq/
+    │   └── main.go
+    ├── obipairing/
+    │   └── main.go
+    ├── obisummary/
+    │   └── main.go
+    └── obimicrosat/
+        └── main.go
+```
+
+## Composants de l'architecture
+
+### 1. Package `pkg/obitools/<commande>/`
+
+Chaque commande possède son propre package dans `pkg/obitools/` qui contient l'implémentation complète de la logique métier. Ce package est structuré en plusieurs fichiers :
+
+#### a) `options.go` - Gestion des options CLI
+
+Ce fichier définit :
+- Les **variables globales** privées (préfixées par `_`) stockant les valeurs des options
+- La fonction **`OptionSet()`** qui configure toutes les options pour la commande
+- Les fonctions **`CLI*()`** qui retournent les valeurs des options (getters)
+- Les fonctions **`Set*()`** qui permettent de définir les options programmatiquement (setters)
+
+**Exemple (obiuniq/options.go) :**
+
+```go
+package obiuniq
+
+import (
+    "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+    "github.com/DavidGamba/go-getoptions"
+)
+
+// Variables globales privées pour stocker les options
+var _StatsOn = make([]string, 0, 10)
+var _Keys = make([]string, 0, 10)
+var _InMemory = false
+var _chunks = 100
+
+// Configuration des options spécifiques à la commande
+func UniqueOptionSet(options *getoptions.GetOpt) {
+    options.StringSliceVar(&_StatsOn, "merge", 1, 1,
+        options.Alias("m"),
+        options.ArgName("KEY"),
+        options.Description("Adds a merged attribute..."))
+    
+    options.BoolVar(&_InMemory, "in-memory", _InMemory,
+        options.Description("Use memory instead of disk..."))
+    
+    options.IntVar(&_chunks, "chunk-count", _chunks,
+        options.Description("In how many chunks..."))
+}
+
+// OptionSet combine les options de base + les options spécifiques
+func OptionSet(options *getoptions.GetOpt) {
+    obiconvert.OptionSet(false)(options)  // Options de base
+    UniqueOptionSet(options)              // Options spécifiques
+}
+
+// Getters pour accéder aux valeurs des options
+func CLIStatsOn() []string {
+    return _StatsOn
+}
+
+func CLIUniqueInMemory() bool {
+    return _InMemory
+}
+
+// Setters pour définir les options programmatiquement
+func SetUniqueInMemory(inMemory bool) {
+    _InMemory = inMemory
+}
+```
+
+**Convention de nommage :**
+- Variables privées : `_NomOption` (underscore préfixe)
+- Getters : `CLINomOption()` (préfixe CLI)
+- Setters : `SetNomOption()` (préfixe Set)
+
+#### b) Fichier(s) d'implémentation
+
+Un ou plusieurs fichiers contenant la logique métier de la commande :
+
+**Exemple (obiuniq/unique.go) :**
+
+```go
+package obiuniq
+
+import (
+    "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+    "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obichunk"
+)
+
+// Fonction CLI principale qui orchestre le traitement
+func CLIUnique(sequences obiiter.IBioSequence) obiiter.IBioSequence {
+    // Récupération des options via les getters CLI*()
+    options := make([]obichunk.WithOption, 0, 30)
+    
+    options = append(options,
+        obichunk.OptionBatchCount(CLINumberOfChunks()),
+    )
+    
+    if CLIUniqueInMemory() {
+        options = append(options, obichunk.OptionSortOnMemory())
+    } else {
+        options = append(options, obichunk.OptionSortOnDisk())
+    }
+    
+    // Appel de la fonction de traitement réelle
+    iUnique, err := obichunk.IUniqueSequence(sequences, options...)
+    
+    if err != nil {
+        log.Fatal(err)
+    }
+    
+    return iUnique
+}
+```
+
+**Autres exemples d'implémentation :**
+
+- **obimicrosat/microsat.go** : Contient `MakeMicrosatWorker()` et `CLIAnnotateMicrosat()`
+- **obisummary/obisummary.go** : Contient `ISummary()` et les structures de données
+
+#### c) Fichiers utilitaires (optionnel)
+
+Certaines commandes ont des fichiers additionnels pour des fonctionnalités spécifiques.
+
+**Exemple (obipairing/options.go) :**
+
+```go
+// Fonction spéciale pour créer un itérateur de séquences pairées
+func CLIPairedSequence() (obiiter.IBioSequence, error) {
+    forward, err := obiconvert.CLIReadBioSequences(_ForwardFile)
+    if err != nil {
+        return obiiter.NilIBioSequence, err
+    }
+    
+    reverse, err := obiconvert.CLIReadBioSequences(_ReverseFile)
+    if err != nil {
+        return obiiter.NilIBioSequence, err
+    }
+    
+    paired := forward.PairTo(reverse)
+    return paired, nil
+}
+```
+
+### 2. Package `obiconvert` - La base commune
+
+Le package `obiconvert` est spécial car il fournit les fonctionnalités de base utilisées par toutes les autres commandes :
+
+#### Fonctionnalités fournies :
+
+1. **Lecture de séquences** (`sequence_reader.go`)
+   - `CLIReadBioSequences()` : lecture depuis fichiers ou stdin
+   - Support de multiples formats (FASTA, FASTQ, EMBL, GenBank, etc.)
+   - Gestion des fichiers multiples
+   - Barre de progression optionnelle
+
+2. **Écriture de séquences** (`sequence_writer.go`)
+   - `CLIWriteBioSequences()` : écriture vers fichiers ou stdout
+   - Support de multiples formats
+   - Gestion des lectures pairées
+   - Compression optionnelle
+
+3. **Options communes** (`options.go`)
+   - Options d'entrée (format, skip, etc.)
+   - Options de sortie (format, fichier, compression)
+   - Options de mode (barre de progression, etc.)
+
+#### Utilisation par les autres commandes :
+
+Toutes les commandes incluent les options de `obiconvert` via :
+
+```go
+func OptionSet(options *getoptions.GetOpt) {
+    obiconvert.OptionSet(false)(options)  // false = pas de fichiers pairés
+    MaCommandeOptionSet(options)          // Options spécifiques
+}
+```
+
+### 3. Exécutable `cmd/obitools/<commande>/main.go`
+
+Le fichier `main.go` de chaque commande est volontairement **minimaliste** et suit toujours le même pattern :
+
+```go
+package main
+
+import (
+    "os"
+    "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
+    "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+    "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+    "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/macommande"
+    "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+)
+
+func main() {
+    // 1. Configuration optionnelle de paramètres par défaut
+    obidefault.SetBatchSize(10)
+    
+    // 2. Génération du parser d'options
+    optionParser := obioptions.GenerateOptionParser(
+        "macommande",                    // Nom de la commande
+        "description de la commande",    // Description
+        macommande.OptionSet)            // Fonction de configuration des options
+    
+    // 3. Parsing des arguments
+    _, args := optionParser(os.Args)
+    
+    // 4. Lecture des séquences d'entrée
+    sequences, err := obiconvert.CLIReadBioSequences(args...)
+    obiconvert.OpenSequenceDataErrorMessage(args, err)
+    
+    // 5. Traitement spécifique de la commande
+    resultat := macommande.CLITraitement(sequences)
+    
+    // 6. Écriture des résultats
+    obiconvert.CLIWriteBioSequences(resultat, true)
+    
+    // 7. Attente de la fin du pipeline
+    obiutils.WaitForLastPipe()
+}
+```
+
+## Patterns architecturaux
+
+### Pattern 1 : Pipeline de traitement de séquences
+
+La plupart des commandes suivent ce pattern :
+
+```
+Lecture → Traitement → Écriture
+```
+
+**Exemples :**
+- **obiconvert** : Lecture → Écriture (conversion de format)
+- **obiuniq** : Lecture → Déréplication → Écriture
+- **obimicrosat** : Lecture → Annotation → Filtrage → Écriture
+
+### Pattern 2 : Traitement avec entrées multiples
+
+Certaines commandes acceptent plusieurs fichiers d'entrée :
+
+**obipairing** :
+```
+Lecture Forward + Lecture Reverse → Pairing → Assemblage → Écriture
+```
+
+### Pattern 3 : Traitement sans écriture de séquences
+
+**obisummary** : produit un résumé JSON/YAML au lieu de séquences
+
+```go
+func main() {
+    // ... parsing options et lecture ...
+    
+    summary := obisummary.ISummary(fs, obisummary.CLIMapSummary())
+    
+    // Formatage et affichage direct
+    if obisummary.CLIOutFormat() == "json" {
+        output, _ := json.MarshalIndent(summary, "", "  ")
+        fmt.Print(string(output))
+    } else {
+        output, _ := yaml.Marshal(summary)
+        fmt.Print(string(output))
+    }
+}
+```
+
+### Pattern 4 : Utilisation de Workers
+
+Les commandes qui transforment des séquences utilisent souvent le pattern Worker :
+
+```go
+// Création d'un worker
+worker := MakeMicrosatWorker(
+    CLIMinUnitLength(),
+    CLIMaxUnitLength(),
+    // ... autres paramètres
+)
+
+// Application du worker sur l'itérateur
+newIter = iterator.MakeIWorker(
+    worker, 
+    false,                              // merge results
+    obidefault.ParallelWorkers()        // parallélisation
+)
+```
+
+## Étapes d'implémentation d'une nouvelle commande
+
+### Étape 1 : Créer le package dans `pkg/obitools/`
+
+```bash
+mkdir -p pkg/obitools/macommande
+```
+
+### Étape 2 : Créer `options.go`
+
+```go
+package macommande
+
+import (
+    "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+    "github.com/DavidGamba/go-getoptions"
+)
+
+// Variables privées pour les options
+var _MonOption = "valeur_par_defaut"
+
+// Configuration des options spécifiques
+func MaCommandeOptionSet(options *getoptions.GetOpt) {
+    options.StringVar(&_MonOption, "mon-option", _MonOption,
+        options.Alias("o"),
+        options.Description("Description de l'option"))
+}
+
+// OptionSet combine options de base + spécifiques
+func OptionSet(options *getoptions.GetOpt) {
+    obiconvert.OptionSet(false)(options)  // false si pas de fichiers pairés
+    MaCommandeOptionSet(options)
+}
+
+// Getters
+func CLIMonOption() string {
+    return _MonOption
+}
+
+// Setters
+func SetMonOption(value string) {
+    _MonOption = value
+}
+```
+
+### Étape 3 : Créer le fichier d'implémentation
+
+Créer `macommande.go` (ou un nom plus descriptif) :
+
+```go
+package macommande
+
+import (
+    "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+    "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+// Fonction de traitement principale
+func CLIMaCommande(sequences obiiter.IBioSequence) obiiter.IBioSequence {
+    // Récupération des options
+    option := CLIMonOption()
+    
+    // Implémentation du traitement
+    // ...
+    
+    return resultat
+}
+```
+
+### Étape 4 : Créer l'exécutable dans `cmd/obitools/`
+
+```bash
+mkdir -p cmd/obitools/macommande
+```
+
+Créer `main.go` :
+
+```go
+package main
+
+import (
+    "os"
+    "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+    "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+    "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/macommande"
+    "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+)
+
+func main() {
+    // Parser d'options
+    optionParser := obioptions.GenerateOptionParser(
+        "macommande",
+        "Description courte de ma commande",
+        macommande.OptionSet)
+    
+    _, args := optionParser(os.Args)
+    
+    // Lecture
+    sequences, err := obiconvert.CLIReadBioSequences(args...)
+    obiconvert.OpenSequenceDataErrorMessage(args, err)
+    
+    // Traitement
+    resultat := macommande.CLIMaCommande(sequences)
+    
+    // Écriture
+    obiconvert.CLIWriteBioSequences(resultat, true)
+    
+    // Attente
+    obiutils.WaitForLastPipe()
+}
+```
+
+### Étape 5 : Configurations optionnelles
+
+Dans `main.go`, avant le parsing des options, on peut configurer :
+
+```go
+// Taille des batchs de séquences
+obidefault.SetBatchSize(10)
+
+// Nombre de workers en lecture (strict)
+obidefault.SetStrictReadWorker(2)
+
+// Nombre de workers en écriture
+obidefault.SetStrictWriteWorker(2)
+
+// Désactiver la lecture des qualités
+obidefault.SetReadQualities(false)
+```
+
+### Étape 6 : Gestion des erreurs
+
+Utiliser les fonctions utilitaires pour les messages d'erreur cohérents :
+
+```go
+// Pour les erreurs d'ouverture de fichiers
+obiconvert.OpenSequenceDataErrorMessage(args, err)
+
+// Pour les erreurs générales
+if err != nil {
+    log.Errorf("Message d'erreur: %v", err)
+    os.Exit(1)
+}
+```
+
+### Étape 7 : Tests et debugging (optionnel)
+
+Des commentaires dans le code montrent comment activer le profiling :
+
+```go
+// go tool pprof -http=":8000" ./macommande ./cpu.pprof
+// f, err := os.Create("cpu.pprof")
+// if err != nil {
+//     log.Fatal(err)
+// }
+// pprof.StartCPUProfile(f)
+// defer pprof.StopCPUProfile()
+
+// go tool trace cpu.trace
+// ftrace, err := os.Create("cpu.trace")
+// if err != nil {
+//     log.Fatal(err)
+// }
+// trace.Start(ftrace)
+// defer trace.Stop()
+```
+
+## Bonnes pratiques observées
+
+### 1. Séparation des responsabilités
+
+- **`main.go`** : orchestration minimale
+- **`options.go`** : définition et gestion des options
+- **Fichiers d'implémentation** : logique métier
+
+### 2. Convention de nommage cohérente
+
+- Variables d'options : `_NomOption`
+- Getters CLI : `CLINomOption()`
+- Setters : `SetNomOption()`
+- Fonctions de traitement CLI : `CLITraitement()`
+
+### 3. Réutilisation du code
+
+- Toutes les commandes réutilisent `obiconvert` pour l'I/O
+- Les options communes sont partagées
+- Les fonctions utilitaires sont centralisées
+
+### 4. Configuration par défaut
+
+Les valeurs par défaut sont :
+- Définies lors de l'initialisation des variables
+- Modifiables via les options CLI
+- Modifiables programmatiquement via les setters
+
+### 5. Gestion des formats
+
+Support automatique de multiples formats :
+- FASTA / FASTQ (avec compression gzip)
+- EMBL / GenBank
+- ecoPCR
+- CSV
+- JSON (avec différents formats d'en-têtes)
+
+### 6. Parallélisation
+
+Les commandes utilisent les workers parallèles via :
+- `obidefault.ParallelWorkers()`
+- `obidefault.SetStrictReadWorker(n)`
+- `obidefault.SetStrictWriteWorker(n)`
+
+### 7. Logging cohérent
+
+Utilisation de `logrus` pour tous les logs :
+```go
+log.Printf("Message informatif")
+log.Errorf("Message d'erreur: %v", err)
+log.Fatal(err)  // Arrêt du programme
+```
+
+## Dépendances principales
+
+### Packages internes OBITools
+
+- `pkg/obidefault` : valeurs par défaut et configuration globale
+- `pkg/obioptions` : génération du parser d'options
+- `pkg/obiiter` : itérateurs de séquences biologiques
+- `pkg/obiseq` : structures et fonctions pour séquences biologiques
+- `pkg/obiformats` : lecture/écriture de différents formats
+- `pkg/obiutils` : fonctions utilitaires diverses
+- `pkg/obichunk` : traitement par chunks (pour dereplication, etc.)
+
+### Packages externes
+
+- `github.com/DavidGamba/go-getoptions` : parsing des options CLI
+- `github.com/sirupsen/logrus` : logging structuré
+- `gopkg.in/yaml.v3` : encodage/décodage YAML
+- `github.com/dlclark/regexp2` : expressions régulières avancées
+
+## Cas spéciaux
+
+### Commande avec fichiers pairés (obipairing)
+
+```go
+func OptionSet(options *getoptions.GetOpt) {
+    obiconvert.OutputOptionSet(options)
+    obiconvert.InputOptionSet(options)
+    PairingOptionSet(options)  // Options spécifiques au pairing
+}
+
+func CLIPairedSequence() (obiiter.IBioSequence, error) {
+    forward, err := obiconvert.CLIReadBioSequences(_ForwardFile)
+    // ...
+    reverse, err := obiconvert.CLIReadBioSequences(_ReverseFile)
+    // ...
+    paired := forward.PairTo(reverse)
+    return paired, nil
+}
+```
+
+Dans `main.go` :
+```go
+pairs, err := obipairing.CLIPairedSequence()  // Lecture spéciale
+if err != nil {
+    log.Errorf("Cannot open file (%v)", err)
+    os.Exit(1)
+}
+
+paired := obipairing.IAssemblePESequencesBatch(
+    pairs,
+    obipairing.CLIGapPenality(),
+    // ... autres paramètres
+)
+```
+
+### Commande sans sortie de séquences (obisummary)
+
+Au lieu de `obiconvert.CLIWriteBioSequences()`, affichage direct :
+
+```go
+summary := obisummary.ISummary(fs, obisummary.CLIMapSummary())
+
+if obisummary.CLIOutFormat() == "json" {
+    output, _ := json.MarshalIndent(summary, "", "  ")
+    fmt.Print(string(output))
+} else {
+    output, _ := yaml.Marshal(summary)
+    fmt.Print(string(output))
+}
+fmt.Printf("\n")
+```
+
+### Commande avec Workers personnalisés (obimicrosat)
+
+```go
+func CLIAnnotateMicrosat(iterator obiiter.IBioSequence) obiiter.IBioSequence {
+    // Création du worker
+    worker := MakeMicrosatWorker(
+        CLIMinUnitLength(),
+        CLIMaxUnitLength(),
+        CLIMinUnitCount(),
+        CLIMinLength(),
+        CLIMinFlankLength(),
+        CLIReoriented(),
+    )
+    
+    // Application du worker
+    newIter := iterator.MakeIWorker(
+        worker, 
+        false,                           // pas de merge
+        obidefault.ParallelWorkers(),    // parallélisation
+    )
+    
+    return newIter.FilterEmpty()  // Filtrage des résultats vides
+}
+```
+
+## Diagramme de flux d'exécution
+
+```
+┌─────────────────────────────────────────────────────────────┐
+│                      cmd/obitools/macommande/main.go        │
+└─────────────────────────────────────────────────────────────┘
+                              │
+                              ▼
+┌─────────────────────────────────────────────────────────────┐
+│  1. Génération du parser d'options                          │
+│     obioptions.GenerateOptionParser(                        │
+│         "macommande",                                       │
+│         "description",                                      │
+│         macommande.OptionSet)                               │
+└─────────────────────────────────────────────────────────────┘
+                              │
+                              ▼
+┌─────────────────────────────────────────────────────────────┐
+│  pkg/obitools/macommande/options.go                         │
+│  ┌─────────────────────────────────────────────────────┐   │
+│  │ func OptionSet(options *getoptions.GetOpt)          │   │
+│  │   obiconvert.OptionSet(false)(options) ───────────┐ │   │
+│  │   MaCommandeOptionSet(options)                    │ │   │
+│  └───────────────────────────────────────────────────┼─┘   │
+└────────────────────────────────────────────────────────┼─────┘
+                              │                         │
+                              │                         │
+                ┌─────────────┘                         │
+                │                                       │
+                ▼                                       ▼
+┌─────────────────────────────────┐  ┌───────────────────────────────┐
+│ 2. Parsing des arguments        │  │ pkg/obitools/obiconvert/      │
+│    _, args := optionParser(...) │  │    options.go                 │
+└─────────────────────────────────┘  │  - InputOptionSet()           │
+                │                     │  - OutputOptionSet()          │
+                ▼                     │  - PairedFilesOptionSet()     │
+┌─────────────────────────────────┐  └───────────────────────────────┘
+│ 3. Lecture des séquences        │
+│    CLIReadBioSequences(args)    │
+└─────────────────────────────────┘
+                │
+                ▼
+┌─────────────────────────────────────────────────────────────┐
+│ pkg/obitools/obiconvert/sequence_reader.go                  │
+│  - ExpandListOfFiles()                                      │
+│  - ReadSequencesFromFile() / ReadSequencesFromStdin()       │
+│  - Support: FASTA, FASTQ, EMBL, GenBank, ecoPCR, CSV        │
+└─────────────────────────────────────────────────────────────┘
+                │
+                ▼ obiiter.IBioSequence
+┌─────────────────────────────────────────────────────────────┐
+│ 4. Traitement spécifique                                    │
+│    macommande.CLITraitement(sequences)                      │
+└─────────────────────────────────────────────────────────────┘
+                │
+                ▼
+┌─────────────────────────────────────────────────────────────┐
+│ pkg/obitools/macommande/<implementation>.go                 │
+│  - Récupération des options via CLI*() getters             │
+│  - Application de la logique métier                         │
+│  - Retour d'un nouvel iterator                              │
+└─────────────────────────────────────────────────────────────┘
+                │
+                ▼ obiiter.IBioSequence
+┌─────────────────────────────────────────────────────────────┐
+│ 5. Écriture des résultats                                   │
+│    CLIWriteBioSequences(resultat, true)                     │
+└─────────────────────────────────────────────────────────────┘
+                │
+                ▼
+┌─────────────────────────────────────────────────────────────┐
+│ pkg/obitools/obiconvert/sequence_writer.go                  │
+│  - WriteSequencesToFile() / WriteSequencesToStdout()        │
+│  - Support: FASTA, FASTQ, JSON                              │
+│  - Gestion des lectures pairées                             │
+│  - Compression optionnelle                                  │
+└─────────────────────────────────────────────────────────────┘
+                │
+                ▼
+┌─────────────────────────────────────────────────────────────┐
+│ 6. Attente de fin du pipeline                               │
+│    obiutils.WaitForLastPipe()                               │
+└─────────────────────────────────────────────────────────────┘
+```
+
+## Conclusion
+
+L'architecture des commandes OBITools est conçue pour :
+
+1. **Maximiser la réutilisation** : `obiconvert` fournit les fonctionnalités communes
+2. **Simplifier l'ajout de nouvelles commandes** : pattern standardisé et minimaliste
+3. **Faciliter la maintenance** : séparation claire des responsabilités
+4. **Garantir la cohérence** : conventions de nommage et structure uniforme
+5. **Optimiser les performances** : parallélisation intégrée et traitement par batch
+
+Cette architecture modulaire permet de créer rapidement de nouvelles commandes tout en maintenant une qualité et une cohérence élevées dans toute la suite OBITools.

blackboard/architechture/definition-superkmer.md

@@ -0,0 +1,99 @@
+# Définition du super k-mer
+
+## Définition
+
+Un **super k-mer** est une **sous-séquence MAXIMALE** d'une séquence dans laquelle **tous les k-mers consécutifs partagent le même minimiseur**.
+
+### Termes
+
+- **k-mer** : sous-séquence de longueur k
+- **minimiseur** : le plus petit m-mer canonique parmi tous les m-mers d'un k-mer
+- **k-mers consécutifs** : k-mers aux positions i et i+1 (chevauchement de k-1 nucléotides)
+- **MAXIMALE** : ne peut être étendue ni à gauche ni à droite
+
+## RÈGLES ABSOLUES
+
+### RÈGLE 1 : Longueur minimum = k
+
+Un super k-mer contient au minimum k nucléotides.
+
+```
+longueur(super-kmer) >= k
+```
+
+### RÈGLE 2 : Chevauchement obligatoire = k-1
+
+Deux super-kmers consécutifs se chevauchent d'EXACTEMENT k-1 nucléotides.
+
+```
+SK1.End - SK2.Start = k - 1
+```
+
+### RÈGLE 3 : Bijection séquence ↔ minimiseur
+
+Une séquence de super k-mer a UN et UN SEUL minimiseur.
+
+```
+Même séquence → Même minimiseur (TOUJOURS)
+```
+
+**Si vous observez la même séquence avec deux minimiseurs différents, c'est un BUG.**
+
+### RÈGLE 4 : Tous les k-mers partagent le minimiseur
+
+TOUS les k-mers contenus dans un super k-mer ont le même minimiseur.
+
+```
+∀ k-mer K dans SK : minimiseur(K) = SK.minimizer
+```
+
+### RÈGLE 5 : Maximalité
+
+Un super k-mer ne peut pas être étendu.
+
+- Si on ajoute un nucléotide à gauche : le nouveau k-mer a un minimiseur différent
+- Si on ajoute un nucléotide à droite : le nouveau k-mer a un minimiseur différent
+
+## VIOLATIONS INTERDITES
+
+❌ **Super k-mer de longueur < k**
+❌ **Chevauchement ≠ k-1 entre consécutifs**
+❌ **Même séquence avec minimiseurs différents**
+❌ **K-mer dans le super k-mer avec minimiseur différent**
+❌ **Super k-mer extensible (non-maximal)**
+
+## CONSÉQUENCES PRATIQUES
+
+### Pour l'extraction
+
+L'algorithme doit :
+1. Calculer le minimiseur de chaque k-mer
+2. Découper quand le minimiseur change
+3. Assigner au super k-mer le minimiseur commun à tous ses k-mers
+4. Garantir que chaque super k-mer contient au moins k nucléotides
+5. Garantir le chevauchement de k-1 entre consécutifs
+
+### Pour la validation
+
+Si après déduplication (obiuniq) on observe :
+```
+Séquence: ACGT...
+Minimiseurs: {M1, M2}  // plusieurs minimiseurs
+```
+
+C'est la PREUVE d'un bug : l'algorithme a produit cette séquence avec des minimiseurs différents, ce qui viole la RÈGLE 3.
+
+## DIAGNOSTIC DU BUG
+
+**Bug observé** : Même séquence avec minimiseurs différents après obiuniq
+
+**Cause possible** : L'algorithme assigne le mauvais minimiseur OU découpe mal les super-kmers
+
+**Ce que le bug NE PEUT PAS être** :
+- Un problème d'obiuniq (révèle le bug, ne le crée pas)
+- Un problème de chevauchement légitime (k-1 est correct)
+
+**Ce que le bug DOIT être** :
+- Minimiseur mal calculé ou mal assigné
+- Découpage incorrect (mauvais endPos)
+- Copie incorrecte des données

blackboard/architechture/guide-redaction-obitest.md

@@ -0,0 +1,316 @@
+# Guide de rédaction d'un obitest
+
+## Règles essentielles
+
+1. **Données < 1 KB** - Fichiers de test très petits
+2. **Exécution < 10 sec** - Tests rapides pour CI/CD
+3. **Auto-contenu** - Pas de dépendances externes
+4. **Auto-nettoyage** - Pas de fichiers résiduels
+
+## Structure minimale
+
+```
+obitests/obitools/<commande>/
+├── test.sh          # Script exécutable
+└── data.fasta       # Données minimales (optionnel)
+```
+
+## Template de test.sh
+
+```bash
+#!/bin/bash
+
+TEST_NAME=<commande>
+CMD=<commande>
+
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+########## TESTS ##########
+
+# Test 1: Help (OBLIGATOIRE)
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1 
+then
+    log "$MCMD: printing help OK" 
+    ((success++))
+else
+    log "$MCMD: printing help failed" 
+    ((failed++))
+fi
+
+# Ajoutez vos tests ici...
+
+###########################
+
+cleanup
+```
+
+## Pattern de test
+
+```bash
+((ntest++))
+if commande args > "${TMPDIR}/output.txt" 2>&1
+then
+    log "$MCMD: description OK" 
+    ((success++))
+else
+    log "$MCMD: description failed"
+    ((failed++))
+fi
+```
+
+## Tests courants
+
+### Exécution basique
+```bash
+((ntest++))
+if $CMD "${TEST_DIR}/input.fasta" > "${TMPDIR}/output.fasta" 2>&1
+then
+    log "$MCMD: basic execution OK" 
+    ((success++))
+else
+    log "$MCMD: basic execution failed"
+    ((failed++))
+fi
+```
+
+### Sortie non vide
+```bash
+((ntest++))
+if [ -s "${TMPDIR}/output.fasta" ]
+then
+    log "$MCMD: output not empty OK"
+    ((success++))
+else
+    log "$MCMD: output empty - failed"
+    ((failed++))
+fi
+```
+
+### Comptage
+```bash
+((ntest++))
+count=$(grep -c "^>" "${TMPDIR}/output.fasta")
+if [ "$count" -gt 0 ]
+then
+    log "$MCMD: extracted $count sequences OK"
+    ((success++))
+else
+    log "$MCMD: no sequences - failed"
+    ((failed++))
+fi
+```
+
+### Présence de contenu
+```bash
+((ntest++))
+if grep -q "expected_string" "${TMPDIR}/output.fasta"
+then
+    log "$MCMD: expected content found OK"
+    ((success++))
+else
+    log "$MCMD: content not found - failed"
+    ((failed++))
+fi
+```
+
+### Comparaison avec référence
+```bash
+((ntest++))
+if diff "${TEST_DIR}/expected.fasta" "${TMPDIR}/output.fasta" > /dev/null
+then
+    log "$MCMD: matches reference OK"
+    ((success++))
+else
+    log "$MCMD: differs from reference - failed"
+    ((failed++))
+fi
+```
+
+### Test avec options
+```bash
+((ntest++))
+if $CMD --opt value "${TEST_DIR}/input.fasta" > "${TMPDIR}/out.fasta" 2>&1
+then
+    log "$MCMD: with option OK" 
+    ((success++))
+else
+    log "$MCMD: with option failed"
+    ((failed++))
+fi
+```
+
+## Variables importantes
+
+- **TEST_DIR** - Répertoire du test (données d'entrée)
+- **TMPDIR** - Répertoire temporaire (sorties)
+- **CMD** - Nom de la commande
+- **MCMD** - Nom formaté pour les logs
+
+## Règles d'or
+
+✅ **Entrées** → `${TEST_DIR}/`
+✅ **Sorties** → `${TMPDIR}/`
+✅ **Toujours rediriger** → `> file 2>&1`
+✅ **Incrémenter ntest** → Avant chaque test
+✅ **Messages clairs** → Descriptions explicites
+
+❌ **Pas de chemins en dur**
+❌ **Pas de /tmp direct**
+❌ **Pas de sortie vers TEST_DIR**
+❌ **Pas de commandes sans redirection**
+
+## Données de test
+
+Créer un fichier minimal (< 500 bytes) :
+
+```fasta
+>seq1
+ACGTACGTACGTACGT
+>seq2
+AAAACCCCGGGGTTTT
+>seq3
+ATCGATCGATCGATCG
+```
+
+## Création rapide
+
+```bash
+# 1. Créer le répertoire
+mkdir -p obitests/obitools/<commande>
+cd obitests/obitools/<commande>
+
+# 2. Créer les données de test
+cat > test_data.fasta << 'EOF'
+>seq1
+ACGTACGTACGTACGT
+>seq2
+AAAACCCCGGGGTTTT
+EOF
+
+# 3. Copier le template dans test.sh
+# 4. Adapter le TEST_NAME et CMD
+# 5. Ajouter les tests
+# 6. Rendre exécutable
+chmod +x test.sh
+
+# 7. Tester
+./test.sh
+```
+
+## Checklist
+
+- [ ] `test.sh` exécutable (`chmod +x`)
+- [ ] Test d'aide inclus
+- [ ] Données < 1 KB
+- [ ] Sorties vers `${TMPDIR}/`
+- [ ] Entrées depuis `${TEST_DIR}/`
+- [ ] Redirections `2>&1`
+- [ ] Messages clairs
+- [ ] Testé localement
+- [ ] Exit code 0 si succès
+
+## Debug
+
+Conserver TMPDIR pour inspection :
+```bash
+cleanup() {
+    echo "Temporary directory: $TMPDIR" 1>&2
+    # rm -rf "$TMPDIR"  # Commenté
+    ...
+}
+```
+
+Mode verbose :
+```bash
+set -x  # Au début du script
+```
+
+## Exemples
+
+**Simple (1 test)** - obimicrosat
+```bash
+# Juste l'aide
+```
+
+**Moyen (4-5 tests)** - obisuperkmer
+```bash
+# Aide + exécution + validation sortie + contenu
+```
+
+**Complet (7+ tests)** - obiuniq
+```bash
+# Aide + exécution + comparaison CSV + options + multiples cas
+```
+
+## Commandes utiles
+
+```bash
+# Compter séquences
+grep -c "^>" file.fasta
+
+# Fichier non vide
+[ -s file ]
+
+# Comparer
+diff file1 file2 > /dev/null
+
+# Comparer compressés
+zdiff file1.gz file2.gz
+
+# Compter bases
+grep -v "^>" file | tr -d '\n' | wc -c
+```
+
+## Ce qu'il faut retenir
+
+Un bon test est **COURT**, **RAPIDE** et **SIMPLE** :
+- 3-10 tests maximum
+- Données < 1 KB
+- Exécution < 10 secondes
+- Pattern standard respecté

blackboard/architechture/obisuperkmer-implementation.md

@@ -0,0 +1,268 @@
+# Implémentation de la commande obisuperkmer
+
+## Vue d'ensemble
+
+La commande `obisuperkmer` a été implémentée en suivant l'architecture standard des commandes OBITools décrite dans `architecture-commande-obitools.md`. Cette commande permet d'extraire les super k-mers de fichiers de séquences biologiques.
+
+## Qu'est-ce qu'un super k-mer ?
+
+Un super k-mer est une sous-séquence maximale dans laquelle tous les k-mers consécutifs partagent le même minimiseur. Cette décomposition est utile pour :
+- L'indexation efficace de k-mers
+- La réduction de la redondance dans les analyses
+- L'optimisation de la mémoire pour les structures de données de k-mers
+
+## Structure de l'implémentation
+
+### 1. Package `pkg/obitools/obisuperkmer/`
+
+Le package contient trois fichiers :
+
+#### `obisuperkmer.go`
+Documentation du package avec une description de son rôle.
+
+#### `options.go`
+Définit les options de ligne de commande :
+
+```go
+var _KmerSize = 21          // Taille des k-mers (par défaut 21)
+var _MinimizerSize = 11     // Taille des minimiseurs (par défaut 11)
+```
+
+**Options CLI disponibles :**
+- `--kmer-size` / `-k` : Taille des k-mers (entre m+1 et 31)
+- `--minimizer-size` / `-m` : Taille des minimiseurs (entre 1 et k-1)
+
+**Fonctions d'accès :**
+- `CLIKmerSize()` : retourne la taille des k-mers
+- `CLIMinimizerSize()` : retourne la taille des minimiseurs
+- `SetKmerSize(k int)` : définit la taille des k-mers
+- `SetMinimizerSize(m int)` : définit la taille des minimiseurs
+
+#### `superkmer.go`
+Implémente la logique de traitement :
+
+```go
+func CLIExtractSuperKmers(iterator obiiter.IBioSequence) obiiter.IBioSequence
+```
+
+Cette fonction :
+1. Récupère les paramètres k et m depuis les options CLI
+2. Valide les paramètres (m < k, k <= 31, etc.)
+3. Crée un worker utilisant `obikmer.SuperKmerWorker(k, m)`
+4. Applique le worker en parallèle sur l'itérateur de séquences
+5. Retourne un itérateur de super k-mers
+
+### 2. Exécutable `cmd/obitools/obisuperkmer/main.go`
+
+L'exécutable suit le pattern standard minimal :
+
+```go
+func main() {
+    // 1. Génération du parser d'options
+    optionParser := obioptions.GenerateOptionParser(
+        "obisuperkmer",
+        "extract super k-mers from sequence files",
+        obisuperkmer.OptionSet)
+    
+    // 2. Parsing des arguments
+    _, args := optionParser(os.Args)
+    
+    // 3. Lecture des séquences
+    sequences, err := obiconvert.CLIReadBioSequences(args...)
+    obiconvert.OpenSequenceDataErrorMessage(args, err)
+    
+    // 4. Extraction des super k-mers
+    superkmers := obisuperkmer.CLIExtractSuperKmers(sequences)
+    
+    // 5. Écriture des résultats
+    obiconvert.CLIWriteBioSequences(superkmers, true)
+    
+    // 6. Attente de la fin du pipeline
+    obiutils.WaitForLastPipe()
+}
+```
+
+## Utilisation du package `obikmer`
+
+L'implémentation s'appuie sur le package `obikmer` qui fournit :
+
+### `SuperKmerWorker(k int, m int) obiseq.SeqWorker`
+
+Crée un worker qui :
+- Extrait les super k-mers d'une BioSequence
+- Retourne une slice de BioSequence, une par super k-mer
+- Chaque super k-mer contient les attributs suivants :
+
+```go
+// Métadonnées ajoutées à chaque super k-mer :
+{
+    "minimizer_value": uint64,  // Valeur canonique du minimiseur
+    "minimizer_seq": string,    // Séquence ADN du minimiseur
+    "k": int,                   // Taille des k-mers utilisée
+    "m": int,                   // Taille des minimiseurs utilisée
+    "start": int,               // Position de début (0-indexé)
+    "end": int,                 // Position de fin (exclusif)
+    "parent_id": string,        // ID de la séquence parente
+}
+```
+
+### Algorithme sous-jacent
+
+Le package `obikmer` utilise :
+- `IterSuperKmers(seq []byte, k int, m int)` : itérateur sur les super k-mers
+- Une deque monotone pour suivre les minimiseurs dans une fenêtre glissante
+- Complexité temporelle : O(n) où n est la longueur de la séquence
+- Complexité spatiale : O(k-m+1) pour la deque
+
+## Exemple d'utilisation
+
+### Ligne de commande
+
+```bash
+# Extraction avec paramètres par défaut (k=21, m=11)
+obisuperkmer sequences.fasta > superkmers.fasta
+
+# Spécifier les tailles de k-mers et minimiseurs
+obisuperkmer -k 25 -m 13 sequences.fasta -o superkmers.fasta
+
+# Avec plusieurs fichiers d'entrée
+obisuperkmer --kmer-size 31 --minimizer-size 15 file1.fasta file2.fasta > output.fasta
+
+# Format FASTQ en entrée, FASTA en sortie
+obisuperkmer sequences.fastq --fasta-output -o superkmers.fasta
+
+# Avec compression
+obisuperkmer sequences.fasta -o superkmers.fasta.gz --compress
+```
+
+### Exemple de sortie
+
+Pour une séquence d'entrée :
+```
+>seq1
+ACGTACGTACGTACGTACGTACGT
+```
+
+La sortie contiendra plusieurs super k-mers :
+```
+>seq1_superkmer_0_15 {"minimizer_value":123456,"minimizer_seq":"acgtacgt","k":21,"m":11,"start":0,"end":15,"parent_id":"seq1"}
+ACGTACGTACGTACG
+>seq1_superkmer_8_24 {"minimizer_value":789012,"minimizer_seq":"gtacgtac","k":21,"m":11,"start":8,"end":24,"parent_id":"seq1"}
+TACGTACGTACGTACGT
+```
+
+## Options héritées de `obiconvert`
+
+La commande hérite de toutes les options standard d'OBITools :
+
+### Options d'entrée
+- `--fasta` : forcer le format FASTA
+- `--fastq` : forcer le format FASTQ
+- `--ecopcr` : format ecoPCR
+- `--embl` : format EMBL
+- `--genbank` : format GenBank
+- `--input-json-header` : en-têtes JSON
+- `--input-OBI-header` : en-têtes OBI
+
+### Options de sortie
+- `--out` / `-o` : fichier de sortie (défaut : stdout)
+- `--fasta-output` : sortie en format FASTA
+- `--fastq-output` : sortie en format FASTQ
+- `--json-output` : sortie en format JSON
+- `--output-json-header` : en-têtes JSON en sortie
+- `--output-OBI-header` / `-O` : en-têtes OBI en sortie
+- `--compress` / `-Z` : compression gzip
+- `--skip-empty` : ignorer les séquences vides
+- `--no-progressbar` : désactiver la barre de progression
+
+## Compilation
+
+Pour compiler la commande :
+
+```bash
+cd /chemin/vers/obitools4
+go build -o bin/obisuperkmer ./cmd/obitools/obisuperkmer/
+```
+
+## Tests
+
+Pour tester la commande :
+
+```bash
+# Créer un fichier de test
+echo -e ">test\nACGTACGTACGTACGTACGTACGTACGTACGT" > test.fasta
+
+# Exécuter obisuperkmer
+obisuperkmer test.fasta
+
+# Vérifier avec des paramètres différents
+obisuperkmer -k 15 -m 7 test.fasta
+```
+
+## Validation des paramètres
+
+La commande valide automatiquement :
+- `1 <= m < k` : le minimiseur doit être plus petit que le k-mer
+- `2 <= k <= 31` : contrainte du codage sur 64 bits
+- `len(sequence) >= k` : la séquence doit être assez longue
+
+En cas de paramètres invalides, la commande affiche une erreur explicite et s'arrête.
+
+## Intégration avec le pipeline OBITools
+
+La commande s'intègre naturellement dans les pipelines OBITools :
+
+```bash
+# Pipeline complet d'analyse
+obiconvert sequences.fastq --fasta-output | \
+  obisuperkmer -k 21 -m 11 | \
+  obiuniq | \
+  obigrep -p "minimizer_value>1000" > filtered_superkmers.fasta
+```
+
+## Parallélisation
+
+La commande utilise automatiquement :
+- `obidefault.ParallelWorkers()` pour le traitement parallèle
+- Les workers sont distribués sur les séquences d'entrée
+- La parallélisation est transparente pour l'utilisateur
+
+## Conformité avec l'architecture OBITools
+
+L'implémentation respecte tous les principes de l'architecture :
+
+✅ Séparation des responsabilités (package + commande)
+✅ Convention de nommage cohérente (CLI*, Set*, _variables)
+✅ Réutilisation de `obiconvert` pour l'I/O
+✅ Options standard partagées
+✅ Pattern Worker pour le traitement
+✅ Validation des paramètres
+✅ Logging avec `logrus`
+✅ Gestion d'erreurs cohérente
+✅ Documentation complète
+
+## Fichiers créés
+
+```
+pkg/obitools/obisuperkmer/
+├── obisuperkmer.go      # Documentation du package
+├── options.go           # Définition des options CLI
+└── superkmer.go         # Implémentation du traitement
+
+cmd/obitools/obisuperkmer/
+└── main.go              # Point d'entrée de la commande
+```
+
+## Prochaines étapes
+
+1. **Compilation** : Compiler la commande avec `go build`
+2. **Tests unitaires** : Créer des tests dans `pkg/obitools/obisuperkmer/superkmer_test.go`
+3. **Documentation utilisateur** : Ajouter la documentation de la commande
+4. **Intégration CI/CD** : Ajouter aux tests d'intégration
+5. **Benchmarks** : Mesurer les performances sur différents jeux de données
+
+## Références
+
+- Architecture des commandes OBITools : `architecture-commande-obitools.md`
+- Package `obikmer` : `pkg/obikmer/`
+- Tests du package : `pkg/obikmer/superkmer_iter_test.go`

blackboard/architechture/obisuperkmer-tests.md

@@ -0,0 +1,440 @@
+# Tests automatisés pour obisuperkmer
+
+## Vue d'ensemble
+
+Des tests automatisés ont été créés pour la commande `obisuperkmer` dans le répertoire `obitests/obitools/obisuperkmer/`. Ces tests suivent le pattern standard utilisé par toutes les commandes OBITools et sont conçus pour être exécutés dans un environnement CI/CD.
+
+## Fichiers créés
+
+```
+obitests/obitools/obisuperkmer/
+├── test.sh                    # Script de test principal (6.7 KB)
+├── test_sequences.fasta       # Données de test (117 bytes)
+└── README.md                  # Documentation (4.1 KB)
+```
+
+### Taille totale : ~11 KB
+
+Cette taille minimale est idéale pour un dépôt Git et des tests CI/CD rapides.
+
+## Jeu de données de test
+
+### Fichier : `test_sequences.fasta` (117 bytes)
+
+Le fichier contient 3 séquences de 32 nucléotides chacune :
+
+```fasta
+>seq1
+ACGTACGTACGTACGTACGTACGTACGTACGT
+>seq2
+AAAACCCCGGGGTTTTAAAACCCCGGGGTTTT
+>seq3
+ATCGATCGATCGATCGATCGATCGATCGATCG
+```
+
+#### Justification du choix
+
+1. **seq1** : Motif répétitif simple (ACGT)
+   - Teste l'extraction de super k-mers sur une séquence avec faible complexité
+   - Les minimiseurs devraient être assez réguliers
+
+2. **seq2** : Blocs homopolymères
+   - Teste le comportement avec des régions de très faible complexité
+   - Les minimiseurs varieront entre les blocs A, C, G et T
+
+3. **seq3** : Motif différent (ATCG)
+   - Teste la diversité des super k-mers extraits
+   - Différent de seq1 pour vérifier la distinction
+
+#### Caractéristiques
+
+- **Longueur** : 32 nucléotides par séquence
+- **Taille totale** : 96 nucléotides (3 × 32)
+- **Format** : FASTA avec en-têtes JSON compatibles
+- **Alphabet** : A, C, G, T uniquement (pas de bases ambiguës)
+- **Taille du fichier** : 117 bytes
+
+Avec k=21 (défaut), chaque séquence de 32 bp peut produire :
+- 32 - 21 + 1 = 12 k-mers
+- Plusieurs super k-mers selon les minimiseurs
+
+## Script de test : `test.sh`
+
+### Structure
+
+Le script suit le pattern standard OBITools :
+
+```bash
+#!/bin/bash
+
+TEST_NAME=obisuperkmer
+CMD=obisuperkmer
+
+# Variables et fonctions standard
+TEST_DIR="..."
+OBITOOLS_DIR="..."
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() { ... }
+log() { ... }
+
+# Tests (12 au total)
+# ...
+
+cleanup
+```
+
+### Tests implémentés
+
+#### 1. Test d'aide (`-h`)
+```bash
+obisuperkmer -h
+```
+Vérifie que la commande peut afficher son aide sans erreur.
+
+#### 2. Extraction basique avec paramètres par défaut
+```bash
+obisuperkmer test_sequences.fasta > output_default.fasta
+```
+Teste l'exécution avec k=21, m=11 (défaut).
+
+#### 3. Vérification de sortie non vide
+```bash
+[ -s output_default.fasta ]
+```
+S'assure que la commande produit un résultat.
+
+#### 4. Comptage des super k-mers
+```bash
+grep -c "^>" output_default.fasta
+```
+Vérifie qu'au moins un super k-mer a été extrait.
+
+#### 5. Présence des métadonnées
+```bash
+grep -q "minimizer_value" output_default.fasta
+grep -q "minimizer_seq" output_default.fasta
+grep -q "parent_id" output_default.fasta
+```
+Vérifie que les attributs requis sont présents.
+
+#### 6. Extraction avec paramètres personnalisés
+```bash
+obisuperkmer -k 15 -m 7 test_sequences.fasta > output_k15_m7.fasta
+```
+Teste la configuration de k et m.
+
+#### 7. Validation des paramètres personnalisés
+```bash
+grep -q '"k":15' output_k15_m7.fasta
+grep -q '"m":7' output_k15_m7.fasta
+```
+Vérifie que les paramètres sont correctement enregistrés.
+
+#### 8. Format de sortie FASTA
+```bash
+obisuperkmer --fasta-output test_sequences.fasta > output_fasta.fasta
+```
+Teste l'option de format explicite.
+
+#### 9. Vérification des IDs
+```bash
+grep "^>" output_default.fasta | grep -q "superkmer"
+```
+S'assure que les IDs contiennent "superkmer".
+
+#### 10. Préservation des IDs parents
+```bash
+grep -q "seq1" output_default.fasta
+grep -q "seq2" output_default.fasta
+grep -q "seq3" output_default.fasta
+```
+Vérifie que les IDs des séquences parentes sont préservés.
+
+#### 11. Option de fichier de sortie (`-o`)
+```bash
+obisuperkmer -o output_file.fasta test_sequences.fasta
+```
+Teste la redirection vers un fichier.
+
+#### 12. Vérification de création du fichier
+```bash
+[ -s output_file.fasta ]
+```
+S'assure que le fichier a été créé.
+
+#### 13. Cohérence des longueurs
+```bash
+# Vérifie que longueur(output) <= longueur(input)
+```
+S'assure que les super k-mers ne sont pas plus longs que l'entrée.
+
+### Compteurs
+
+- **ntest** : Nombre de tests exécutés
+- **success** : Nombre de tests réussis
+- **failed** : Nombre de tests échoués
+
+### Sortie du script
+
+#### En cas de succès
+```
+========================================
+## Results of the obisuperkmer tests:
+
+- 12 tests run
+- 12 successfully completed
+- 0 failed tests
+
+Cleaning up the temporary directory...
+
+========================================
+```
+
+Exit code : **0**
+
+#### En cas d'échec
+```
+========================================
+## Results of the obisuperkmer tests:
+
+- 12 tests run
+- 10 successfully completed
+- 2 failed tests
+
+Cleaning up the temporary directory...
+
+========================================
+```
+
+Exit code : **1**
+
+## Intégration CI/CD
+
+### Exécution automatique
+
+Le script est conçu pour être exécuté automatiquement dans un pipeline CI/CD :
+
+1. Le build produit l'exécutable dans `build/obisuperkmer`
+2. Le script de test ajoute `build/` au PATH
+3. Les tests s'exécutent
+4. Le code de retour indique le succès (0) ou l'échec (1)
+
+### Exemple de configuration CI/CD
+
+```yaml
+# .github/workflows/test.yml ou équivalent
+test-obisuperkmer:
+  runs-on: ubuntu-latest
+  steps:
+    - uses: actions/checkout@v2
+    - name: Build obitools
+      run: make build
+    - name: Test obisuperkmer
+      run: ./obitests/obitools/obisuperkmer/test.sh
+```
+
+### Avantages
+
+✅ **Rapidité** : Données de test minimales (117 bytes)
+✅ **Fiabilité** : Tests reproductibles
+✅ **Isolation** : Utilisation d'un répertoire temporaire
+✅ **Nettoyage automatique** : Pas de fichiers résiduels
+✅ **Logging** : Messages horodatés et détaillés
+✅ **Compatibilité** : Pattern standard OBITools
+
+## Exécution locale
+
+### Prérequis
+
+1. Compiler obisuperkmer :
+   ```bash
+   cd /chemin/vers/obitools4
+   go build -o build/obisuperkmer ./cmd/obitools/obisuperkmer/
+   ```
+
+2. Se placer dans le répertoire de test :
+   ```bash
+   cd obitests/obitools/obisuperkmer
+   ```
+
+3. Exécuter le script :
+   ```bash
+   ./test.sh
+   ```
+
+### Exemple de sortie
+
+```
+[obisuperkmer @ Fri Feb  7 13:00:00 CET 2026] Testing obisuperkmer...
+[obisuperkmer @ Fri Feb  7 13:00:00 CET 2026] Test directory is /path/to/obitests/obitools/obisuperkmer
+[obisuperkmer @ Fri Feb  7 13:00:00 CET 2026] obitools directory is /path/to/build
+[obisuperkmer @ Fri Feb  7 13:00:00 CET 2026] Temporary directory is /tmp/tmp.abc123
+[obisuperkmer @ Fri Feb  7 13:00:00 CET 2026] files: README.md test.sh test_sequences.fasta
+[obisuperkmer @ Fri Feb  7 13:00:01 CET 2026] OBISuperkmer: printing help OK
+[obisuperkmer @ Fri Feb  7 13:00:02 CET 2026] OBISuperkmer: basic extraction with default parameters OK
+[obisuperkmer @ Fri Feb  7 13:00:02 CET 2026] OBISuperkmer: output file is not empty OK
+[obisuperkmer @ Fri Feb  7 13:00:02 CET 2026] OBISuperkmer: extracted 8 super k-mers OK
+[obisuperkmer @ Fri Feb  7 13:00:02 CET 2026] OBISuperkmer: super k-mers contain required metadata OK
+[obisuperkmer @ Fri Feb  7 13:00:03 CET 2026] OBISuperkmer: extraction with custom k=15, m=7 OK
+[obisuperkmer @ Fri Feb  7 13:00:03 CET 2026] OBISuperkmer: custom parameters correctly set in metadata OK
+[obisuperkmer @ Fri Feb  7 13:00:03 CET 2026] OBISuperkmer: FASTA output format OK
+[obisuperkmer @ Fri Feb  7 13:00:03 CET 2026] OBISuperkmer: super k-mer IDs contain 'superkmer' OK
+[obisuperkmer @ Fri Feb  7 13:00:03 CET 2026] OBISuperkmer: parent sequence IDs preserved OK
+[obisuperkmer @ Fri Feb  7 13:00:04 CET 2026] OBISuperkmer: output to file with -o option OK
+[obisuperkmer @ Fri Feb  7 13:00:04 CET 2026] OBISuperkmer: output file created with -o option OK
+[obisuperkmer @ Fri Feb  7 13:00:04 CET 2026] OBISuperkmer: super k-mer total length <= input length OK
+========================================
+## Results of the obisuperkmer tests:
+
+- 12 tests run
+- 12 successfully completed
+- 0 failed tests
+
+Cleaning up the temporary directory...
+
+========================================
+```
+
+## Debugging des tests
+
+### Conserver les fichiers temporaires
+
+Modifier temporairement la fonction `cleanup()` :
+
+```bash
+cleanup() {
+    echo "Temporary directory: $TMPDIR" 1>&2
+    # Commenter cette ligne pour conserver les fichiers
+    # rm -rf "$TMPDIR"
+    ...
+}
+```
+
+### Activer le mode verbose
+
+Ajouter au début du script :
+
+```bash
+set -x  # Active l'affichage de toutes les commandes
+```
+
+### Tester une seule commande
+
+Extraire et exécuter manuellement :
+
+```bash
+export TEST_DIR=/chemin/vers/obitests/obitools/obisuperkmer
+export TMPDIR=$(mktemp -d)
+obisuperkmer "${TEST_DIR}/test_sequences.fasta" > "${TMPDIR}/output.fasta"
+cat "${TMPDIR}/output.fasta"
+```
+
+## Ajout de nouveaux tests
+
+Pour ajouter un test supplémentaire :
+
+1. Incrémenter le compteur `ntest`
+2. Écrire la condition de test
+3. Logger le succès ou l'échec
+4. Incrémenter le bon compteur
+
+```bash
+((ntest++))
+if ma_nouvelle_commande_de_test
+then
+    log "Description du test: OK" 
+    ((success++))
+else
+    log "Description du test: failed"
+    ((failed++))
+fi
+```
+
+## Comparaison avec d'autres tests
+
+### Taille des données de test
+
+| Commande | Taille des données | Nombre de fichiers |
+|----------|-------------------|-------------------|
+| obiconvert | 925 KB | 1 fichier |
+| obiuniq | ~600 bytes | 4 fichiers |
+| obimicrosat | 0 bytes | 0 fichiers (génère à la volée) |
+| **obisuperkmer** | **117 bytes** | **1 fichier** |
+
+Notre test `obisuperkmer` est parmi les plus légers, ce qui est optimal pour CI/CD.
+
+### Nombre de tests
+
+| Commande | Nombre de tests |
+|----------|----------------|
+| obiconvert | 3 tests |
+| obiuniq | 7 tests |
+| obimicrosat | 1 test |
+| **obisuperkmer** | **12 tests** |
+
+Notre test `obisuperkmer` offre une couverture complète avec 12 tests différents.
+
+## Couverture de test
+
+Les tests couvrent :
+
+✅ Affichage de l'aide  
+✅ Exécution basique  
+✅ Paramètres par défaut (k=21, m=11)  
+✅ Paramètres personnalisés (k=15, m=7)  
+✅ Formats de sortie (FASTA)  
+✅ Redirection vers fichier (`-o`)  
+✅ Présence des métadonnées  
+✅ Validation des IDs  
+✅ Préservation des IDs parents  
+✅ Cohérence des longueurs  
+✅ Production de résultats non vides  
+
+## Maintenance
+
+### Mise à jour des tests
+
+Si l'implémentation de `obisuperkmer` change :
+
+1. Vérifier que les tests existants passent toujours
+2. Ajouter de nouveaux tests pour les nouvelles fonctionnalités
+3. Mettre à jour `README.md` si nécessaire
+4. Documenter les changements
+
+### Vérification régulière
+
+Exécuter périodiquement :
+
+```bash
+cd obitests/obitools/obisuperkmer
+./test.sh
+```
+
+Ou via l'ensemble des tests :
+
+```bash
+cd obitests
+for dir in obitools/*/; do
+    if [ -f "$dir/test.sh" ]; then
+        echo "Testing $(basename $dir)..."
+        (cd "$dir" && ./test.sh) || echo "FAILED: $(basename $dir)"
+    fi
+done
+```
+
+## Conclusion
+
+Les tests pour `obisuperkmer` sont :
+
+- ✅ **Complets** : 12 tests couvrant toutes les fonctionnalités principales
+- ✅ **Légers** : 117 bytes de données de test
+- ✅ **Rapides** : Exécution en quelques secondes
+- ✅ **Fiables** : Pattern éprouvé utilisé par toutes les commandes OBITools
+- ✅ **Maintenables** : Structure claire et documentée
+- ✅ **CI/CD ready** : Code de retour approprié et nettoyage automatique
+
+Ils garantissent que la commande fonctionne correctement à chaque commit et facilitent la détection précoce des régressions.

cmd/obitools/obiannotate/main.go

@@ -3,13 +3,11 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiannotate"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
 
 func main() {
@@ -32,20 +30,25 @@ func main() {
 	// trace.Start(ftrace)
 	// defer trace.Stop()
 
-	optionParser := obioptions.GenerateOptionParser(obiannotate.OptionSet)
+	optionParser := obioptions.GenerateOptionParser(
+		"obiannotate",
+		"edits the sequence annotations",
+		obiannotate.OptionSet,
+	)
 
 	_, args := optionParser(os.Args)
 
 	sequences, err := obiconvert.CLIReadBioSequences(args...)
-
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 
 	annotator := obiannotate.CLIAnnotationPipeline()
+
+	if obiannotate.CLIHasSetNumberFlag() {
+		sequences = sequences.NumberSequences(1, !obiconvert.CLINoInputOrder())
+	}
+
 	obiconvert.CLIWriteBioSequences(sequences.Pipe(annotator), true)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obiclean/main.go

@@ -3,31 +3,28 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiclean"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 )
 
 func main() {
-	optionParser := obioptions.GenerateOptionParser(obiclean.OptionSet)
+	optionParser := obioptions.GenerateOptionParser(
+		"obiclean",
+		"",
+		obiclean.OptionSet)
 
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 
 	cleaned := obiclean.CLIOBIClean(fs)
 
 	obiconvert.CLIWriteBioSequences(cleaned, true)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obicleandb/main.go

@@ -3,33 +3,31 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obicleandb"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 )
 
 func main() {
-	obioptions.SetBatchSize(10)
+	obidefault.SetBatchSize(10)
 
-	optionParser := obioptions.GenerateOptionParser(obicleandb.OptionSet)
+	optionParser := obioptions.GenerateOptionParser(
+		"obicleandb",
+		"clean-up reference databases",
+		obicleandb.OptionSet)
 
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 
 	cleaned := obicleandb.ICleanDB(fs)
 
 	toconsume, _ := obiconvert.CLIWriteBioSequences(cleaned, false)
 	toconsume.Consume()
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 }

cmd/obitools/obicomplement/main.go

@@ -3,30 +3,27 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 )
 
 func main() {
-	optionParser := obioptions.GenerateOptionParser(obiconvert.OptionSet)
+	optionParser := obioptions.GenerateOptionParser(
+		"obicomplement",
+		"reverse complement of sequences",
+		obiconvert.OptionSet(true))
 
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 
 	comp := fs.MakeIWorker(obiseq.ReverseComplementWorker(true), true)
 	obiconvert.CLIWriteBioSequences(comp, true)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obiconsensus/main.go

@@ -3,31 +3,28 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconsensus"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 )
 
 func main() {
-	optionParser := obioptions.GenerateOptionParser(obiconsensus.OptionSet)
+	optionParser := obioptions.GenerateOptionParser(
+		"obiconsensus",
+		"ONT reads denoising",
+		obiconsensus.OptionSet)
 
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 
 	cleaned := obiconsensus.CLIOBIMinion(fs)
 
 	obiconvert.CLIWriteBioSequences(cleaned, true)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obiconvert/main.go

@@ -3,31 +3,29 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 )
 
 func main() {
-	obioptions.SetStrictReadWorker(2)
+	obidefault.SetStrictReadWorker(2)
-	obioptions.SetStrictWriteWorker(2)
+	obidefault.SetStrictWriteWorker(2)
 
-	optionParser := obioptions.GenerateOptionParser(obiconvert.OptionSet)
+	optionParser := obioptions.GenerateOptionParser(
+		"obiconvert",
+		"convertion of sequence files to various formats",
+		obiconvert.OptionSet(true))
 
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 
 	obiconvert.CLIWriteBioSequences(fs, true)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obicount/main.go

@@ -4,8 +4,7 @@ import (
 	"fmt"
 	"os"
 
-	log "github.com/sirupsen/logrus"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
-
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obicount"
 
@@ -29,23 +28,21 @@ func main() {
 	// defer trace.Stop()
 
 	optionParser := obioptions.GenerateOptionParser(
+		"obicount",
+		"counts the sequences present in a file of sequences",
 		obiconvert.InputOptionSet,
 		obicount.OptionSet,
 	)
 
 	_, args := optionParser(os.Args)
 
-	obioptions.SetStrictReadWorker(min(4, obioptions.CLIParallelWorkers()))
+	obidefault.SetStrictReadWorker(min(4, obidefault.ParallelWorkers()))
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 
 	nvariant, nread, nsymbol := fs.Count(true)
 
-	fmt.Print("entites,n\n")
+	fmt.Print("entities,n\n")
 
 	if obicount.CLIIsPrintingVariantCount() {
 		fmt.Printf("variants,%d\n", nvariant)

cmd/obitools/obicsv/main.go

@@ -3,28 +3,25 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obicsv"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
 
 func main() {
-	optionParser := obioptions.GenerateOptionParser(obicsv.OptionSet)
+	optionParser := obioptions.GenerateOptionParser(
+		"obicsv",
+		"converts sequence files to CSV format",
+		obicsv.OptionSet)
 
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
 
-	if err != nil {
+	obicsv.CLIWriteSequenceCSV(fs, true)
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 
-	obicsv.CLIWriteCSV(fs, true)
+	obiutils.WaitForLastPipe()
-
-	obiiter.WaitForLastPipe()
 
 }

cmd/obitools/obidemerge/main.go

@@ -3,34 +3,32 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obidemerge"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 )
 
 func main() {
-	obioptions.SetStrictReadWorker(2)
+	obidefault.SetStrictReadWorker(2)
-	obioptions.SetStrictWriteWorker(2)
+	obidefault.SetStrictWriteWorker(2)
 
-	optionParser := obioptions.GenerateOptionParser(obidemerge.OptionSet)
+	optionParser := obioptions.GenerateOptionParser(
+		"obidemerge",
+		"",
+		obidemerge.OptionSet)
 
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 
 	demerged := obidemerge.CLIDemergeSequences(fs)
 
 	obiconvert.CLIWriteBioSequences(demerged, true)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obidistribute/main.go

@@ -3,29 +3,26 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obidistribute"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 )
 
 func main() {
-	optionParser := obioptions.GenerateOptionParser(obidistribute.OptionSet)
+	optionParser := obioptions.GenerateOptionParser(
+		"obidistribute",
+		"divided an input set of sequences into subsets",
+		obidistribute.OptionSet)
 
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 
 	obidistribute.CLIDistributeSequence(fs)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obifind/main.go

@@ -1,68 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"os"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obifind"
-)
-
-func main() {
-	optionParser := obioptions.GenerateOptionParser(obifind.OptionSet)
-
-	_, args := optionParser(os.Args)
-
-	//prof, _ := os.Create("obifind.prof")
-	//pprof.StartCPUProfile(prof)
-
-	restrictions, err := obifind.ITaxonRestrictions()
-	if err != nil {
-		fmt.Printf("%+v", err)
-	}
-
-	switch {
-	case obifind.CLIRequestsPathForTaxid() >= 0:
-		taxonomy, err := obifind.CLILoadSelectedTaxonomy()
-		if err != nil {
-			fmt.Printf("%+v", err)
-		}
-
-		taxon, err := taxonomy.Taxon(obifind.CLIRequestsPathForTaxid())
-
-		if err != nil {
-			fmt.Printf("%+v", err)
-		}
-
-		s, err := taxon.Path()
-
-		if err != nil {
-			fmt.Printf("%+v", err)
-		}
-
-		obifind.TaxonWriter(s.Iterator(),
-			fmt.Sprintf("path:%d", taxon.Taxid()))
-
-	case len(args) == 0:
-		taxonomy, err := obifind.CLILoadSelectedTaxonomy()
-		if err != nil {
-			fmt.Printf("%+v", err)
-		}
-
-		obifind.TaxonWriter(restrictions(taxonomy.Iterator()), "")
-
-	default:
-		matcher, err := obifind.ITaxonNameMatcher()
-
-		if err != nil {
-			fmt.Printf("%+v", err)
-		}
-
-		for _, pattern := range args {
-			s := restrictions(matcher(pattern))
-			obifind.TaxonWriter(s, pattern)
-		}
-	}
-
-	//pprof.StopCPUProfile()
-}

cmd/obitools/obigrep/main.go

@@ -3,13 +3,11 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obigrep"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
 
 func main() {
@@ -32,18 +30,18 @@ func main() {
 	// trace.Start(ftrace)
 	// defer trace.Stop()
 
-	optionParser := obioptions.GenerateOptionParser(obigrep.OptionSet)
+	optionParser := obioptions.GenerateOptionParser(
+		"obigrep",
+		"select a subset of sequences on various criteria",
+		obigrep.OptionSet)
 
 	_, args := optionParser(os.Args)
 
 	sequences, err := obiconvert.CLIReadBioSequences(args...)
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
 
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 	selected := obigrep.CLIFilterSequence(sequences)
 	obiconvert.CLIWriteBioSequences(selected, true)
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obijoin/main.go

@@ -3,34 +3,32 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obijoin"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 )
 
 func main() {
-	obioptions.SetStrictReadWorker(2)
+	obidefault.SetStrictReadWorker(2)
-	obioptions.SetStrictWriteWorker(2)
+	obidefault.SetStrictWriteWorker(2)
 
-	optionParser := obioptions.GenerateOptionParser(obijoin.OptionSet)
+	optionParser := obioptions.GenerateOptionParser(
+		"obijoin",
+		"merge annotations contained in a file to another file",
+		obijoin.OptionSet)
 
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 
 	joined := obijoin.CLIJoinSequences(fs)
 
 	obiconvert.CLIWriteBioSequences(joined, true)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obik/main.go

@@ -0,0 +1,34 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"os"
+
+	log "github.com/sirupsen/logrus"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obik"
+	"github.com/DavidGamba/go-getoptions"
+)
+
+func main() {
+	defer obiseq.LogBioSeqStatus()
+
+	opt, parser := obioptions.GenerateSubcommandParser(
+		"obik",
+		"Manage disk-based kmer indices",
+		obik.OptionSet,
+	)
+
+	_, remaining := parser(os.Args)
+
+	err := opt.Dispatch(context.Background(), remaining)
+	if err != nil {
+		if errors.Is(err, getoptions.ErrorHelpCalled) {
+			os.Exit(0)
+		}
+		log.Fatalf("Error: %v", err)
+	}
+}

cmd/obitools/obikmermatch/main.go

@@ -0,0 +1,54 @@
+package main
+
+import (
+	"os"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obikmersim"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+)
+
+func main() {
+
+	defer obiseq.LogBioSeqStatus()
+
+	// go tool pprof -http=":8000" ./obipairing ./cpu.pprof
+	// f, err := os.Create("cpu.pprof")
+	// if err != nil {
+	// 	log.Fatal(err)
+	// }
+	// pprof.StartCPUProfile(f)
+	// defer pprof.StopCPUProfile()
+
+	// go tool trace cpu.trace
+	// ftrace, err := os.Create("cpu.trace")
+	// if err != nil {
+	// 	log.Fatal(err)
+	// }
+	// trace.Start(ftrace)
+	// defer trace.Stop()
+
+	optionParser := obioptions.GenerateOptionParser(
+		"obikmermatch",
+		"",
+		obikmersim.MatchOptionSet)
+
+	_, args := optionParser(os.Args)
+
+	var err error
+	sequences := obiiter.NilIBioSequence
+
+	if !obikmersim.CLISelf() {
+		sequences, err = obiconvert.CLIReadBioSequences(args...)
+	}
+
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
+
+	selected := obikmersim.CLIAlignSequences(sequences)
+	obiconvert.CLIWriteBioSequences(selected, true)
+	obiutils.WaitForLastPipe()
+
+}

cmd/obitools/obikmersimcount/main.go

@@ -0,0 +1,62 @@
+package main
+
+import (
+	"log"
+	"os"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obikmersim"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+)
+
+func main() {
+
+	defer obiseq.LogBioSeqStatus()
+
+	// go tool pprof -http=":8000" ./obipairing ./cpu.pprof
+	// f, err := os.Create("cpu.pprof")
+	// if err != nil {
+	// 	log.Fatal(err)
+	// }
+	// pprof.StartCPUProfile(f)
+	// defer pprof.StopCPUProfile()
+
+	// go tool trace cpu.trace
+	// ftrace, err := os.Create("cpu.trace")
+	// if err != nil {
+	// 	log.Fatal(err)
+	// }
+	// trace.Start(ftrace)
+	// defer trace.Stop()
+
+	optionParser := obioptions.GenerateOptionParser(
+		"obikmersimcount",
+		"",
+		obikmersim.CountOptionSet)
+
+	_, args := optionParser(os.Args)
+
+	var err error
+	sequences := obiiter.NilIBioSequence
+
+	if !obikmersim.CLISelf() {
+		sequences, err = obiconvert.CLIReadBioSequences(args...)
+	}
+
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
+
+	counted := obikmersim.CLILookForSharedKmers(sequences)
+	topull, err := obiconvert.CLIWriteBioSequences(counted, false)
+
+	if err != nil {
+		log.Panic(err)
+	}
+
+	topull.Consume()
+
+	obiutils.WaitForLastPipe()
+
+}

cmd/obitools/obilandmark/main.go

@@ -3,30 +3,27 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obilandmark"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 )
 
 func main() {
-	optionParser := obioptions.GenerateOptionParser(obilandmark.OptionSet)
+	optionParser := obioptions.GenerateOptionParser(
+		"obilandmark",
+		"",
+		obilandmark.OptionSet)
 
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 
 	indexed := obilandmark.CLISelectLandmarkSequences(fs)
 
 	obiconvert.CLIWriteBioSequences(indexed, true)
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obimatrix/main.go

@@ -4,8 +4,6 @@ import (
 	"fmt"
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
@@ -33,17 +31,15 @@ func main() {
 	// defer trace.Stop()
 
 	optionParser := obioptions.GenerateOptionParser(
+		"obimatrix",
+		"",
 		obimatrix.OptionSet,
 	)
 
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 
 	matrix := obimatrix.IMatrix(fs)
 

cmd/obitools/obimicrosat/main.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	"os"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obimicrosat"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+)
+
+func main() {
+
+	defer obiseq.LogBioSeqStatus()
+
+	// go tool pprof -http=":8000" ./obipairing ./cpu.pprof
+	// f, err := os.Create("cpu.pprof")
+	// if err != nil {
+	// 	log.Fatal(err)
+	// }
+	// pprof.StartCPUProfile(f)
+	// defer pprof.StopCPUProfile()
+
+	// go tool trace cpu.trace
+	// ftrace, err := os.Create("cpu.trace")
+	// if err != nil {
+	// 	log.Fatal(err)
+	// }
+	// trace.Start(ftrace)
+	// defer trace.Stop()
+
+	optionParser := obioptions.GenerateOptionParser(
+		"obimicrosat",
+		"looks for microsatellites sequences in a sequence file",
+		obimicrosat.OptionSet)
+
+	_, args := optionParser(os.Args)
+
+	sequences, err := obiconvert.CLIReadBioSequences(args...)
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
+
+	selected := obimicrosat.CLIAnnotateMicrosat(sequences)
+	obiconvert.CLIWriteBioSequences(selected, true)
+	obiutils.WaitForLastPipe()
+
+}

cmd/obitools/obimultiplex/main.go

@@ -6,10 +6,10 @@ import (
 
 	log "github.com/sirupsen/logrus"
 
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obimultiplex"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
 
 func main() {
@@ -28,7 +28,10 @@ func main() {
 	// trace.Start(ftrace)
 	// defer trace.Stop()
 
-	optionParser := obioptions.GenerateOptionParser(obimultiplex.OptionSet)
+	optionParser := obioptions.GenerateOptionParser(
+		"obimultiplex",
+		"demultiplex amplicons",
+		obimultiplex.OptionSet)
 
 	_, args := optionParser(os.Args)
 
@@ -43,14 +46,11 @@ func main() {
 	}
 
 	sequences, err := obiconvert.CLIReadBioSequences(args...)
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
 
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 	amplicons, _ := obimultiplex.IExtractBarcode(sequences)
 	obiconvert.CLIWriteBioSequences(amplicons, true)
 	amplicons.Wait()
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obipairing/main.go

@@ -5,10 +5,11 @@ import (
 
 	log "github.com/sirupsen/logrus"
 
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obipairing"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
 
 func main() {
@@ -29,12 +30,15 @@ func main() {
 	// trace.Start(ftrace)
 	// defer trace.Stop()
 
-	optionParser := obioptions.GenerateOptionParser(obipairing.OptionSet)
+	optionParser := obioptions.GenerateOptionParser(
+		"obipairing",
+		"align forward with reverse reads with paired reads",
+		obipairing.OptionSet)
 
 	optionParser(os.Args)
 
-	obioptions.SetStrictReadWorker(2)
+	obidefault.SetStrictReadWorker(2)
-	obioptions.SetStrictWriteWorker(2)
+	obidefault.SetStrictWriteWorker(2)
 	pairs, err := obipairing.CLIPairedSequence()
 
 	if err != nil {
@@ -51,10 +55,10 @@ func main() {
 		obipairing.CLIFastMode(),
 		obipairing.CLIFastRelativeScore(),
 		obipairing.CLIWithStats(),
-		obioptions.CLIParallelWorkers(),
+		obidefault.ParallelWorkers(),
 	)
 
 	obiconvert.CLIWriteBioSequences(paired, true)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 }

cmd/obitools/obipcr/main.go

@@ -3,12 +3,11 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obipcr"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
 
 func main() {
@@ -25,24 +24,23 @@ func main() {
 	// trace.Start(ftrace)
 	// defer trace.Stop()
 
-	obioptions.SetWorkerPerCore(2)
+	obidefault.SetWorkerPerCore(2)
-	obioptions.SetReadWorkerPerCore(0.5)
+	obidefault.SetReadWorkerPerCore(0.5)
-	obioptions.SetParallelFilesRead(obioptions.CLIParallelWorkers() / 4)
+	obidefault.SetParallelFilesRead(obidefault.ParallelWorkers() / 4)
-	obioptions.SetBatchSize(10)
+	obidefault.SetBatchSize(10)
 
-	optionParser := obioptions.GenerateOptionParser(obipcr.OptionSet)
+	optionParser := obioptions.GenerateOptionParser(
+		"obipcr",
+		"simulates a PCR on a sequence files",
+		obipcr.OptionSet)
 
 	_, args := optionParser(os.Args)
 
 	sequences, err := obiconvert.CLIReadBioSequences(args...)
-
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 
 	amplicons, _ := obipcr.CLIPCR(sequences)
 	obiconvert.CLIWriteBioSequences(amplicons, true)
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obireffamidx/main.go

@@ -3,30 +3,27 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obirefidx"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 )
 
 func main() {
-	optionParser := obioptions.GenerateOptionParser(obirefidx.OptionSet)
+	optionParser := obioptions.GenerateOptionParser(
+		"obireffamidx",
+		"",
+		obirefidx.OptionSet)
 
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 
 	indexed := obirefidx.IndexFamilyDB(fs)
 
 	obiconvert.CLIWriteBioSequences(indexed, true)
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obirefidx/main.go

@@ -3,29 +3,27 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obirefidx"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 )
 
 func main() {
-	optionParser := obioptions.GenerateOptionParser(obirefidx.OptionSet)
+	optionParser := obioptions.GenerateOptionParser(
+		"obirefidx",
+		"",
+		obirefidx.OptionSet)
 
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
 
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 	indexed := obirefidx.IndexReferenceDB(fs)
 
 	obiconvert.CLIWriteBioSequences(indexed, true)
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obiscript/main.go

@@ -4,13 +4,11 @@ import (
 	"fmt"
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiscript"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
 
 func main() {
@@ -33,7 +31,10 @@ func main() {
 	// trace.Start(ftrace)
 	// defer trace.Stop()
 
-	optionParser := obioptions.GenerateOptionParser(obiscript.OptionSet)
+	optionParser := obioptions.GenerateOptionParser(
+		"obiscript",
+		"executes a lua script on the input sequences",
+		obiscript.OptionSet)
 
 	_, args := optionParser(os.Args)
 
@@ -43,15 +44,11 @@ func main() {
 	}
 
 	sequences, err := obiconvert.CLIReadBioSequences(args...)
-
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 
 	annotator := obiscript.CLIScriptPipeline()
 	obiconvert.CLIWriteBioSequences(sequences.Pipe(annotator), true)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obisplit/main.go

@@ -4,13 +4,11 @@ import (
 	"fmt"
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obisplit"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
 
 func main() {
@@ -33,7 +31,10 @@ func main() {
 	// trace.Start(ftrace)
 	// defer trace.Stop()
 
-	optionParser := obioptions.GenerateOptionParser(obisplit.OptionSet)
+	optionParser := obioptions.GenerateOptionParser(
+		"obisplit",
+		"",
+		obisplit.OptionSet)
 
 	_, args := optionParser(os.Args)
 
@@ -43,15 +44,11 @@ func main() {
 	}
 
 	sequences, err := obiconvert.CLIReadBioSequences(args...)
-
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 
 	annotator := obisplit.CLISlitPipeline()
 	obiconvert.CLIWriteBioSequences(sequences.Pipe(annotator), true)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obisummary/main.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"os"
 
-	log "github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v3"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
@@ -34,16 +33,15 @@ func main() {
 	// trace.Start(ftrace)
 	// defer trace.Stop()
 
-	optionParser := obioptions.GenerateOptionParser(obisummary.OptionSet)
+	optionParser := obioptions.GenerateOptionParser(
+		"obisummary",
+		"resume main information from a sequence file",
+		obisummary.OptionSet)
 
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 
 	summary := obisummary.ISummary(fs, obisummary.CLIMapSummary())
 

cmd/obitools/obitag/main.go

@@ -6,10 +6,13 @@ import (
 
 	log "github.com/sirupsen/logrus"
 
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obifind"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obitag"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obitaxonomy"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 )
@@ -32,39 +35,55 @@ func main() {
 	// trace.Start(ftrace)
 	// defer trace.Stop()
 
-	obioptions.SetWorkerPerCore(2)
+	obidefault.SetWorkerPerCore(2)
-	obioptions.SetStrictReadWorker(1)
+	obidefault.SetStrictReadWorker(1)
-	obioptions.SetStrictWriteWorker(1)
+	obidefault.SetStrictWriteWorker(1)
-	obioptions.SetBatchSize(10)
+	obidefault.SetBatchSize(10)
 
-	optionParser := obioptions.GenerateOptionParser(obitag.OptionSet)
+	optionParser := obioptions.GenerateOptionParser(
+		"obitag",
+		"realizes taxonomic assignment",
+		obitag.OptionSet)
 
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
 
-	if err != nil {
+	taxo := obitax.DefaultTaxonomy()
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
-
-	taxo, error := obifind.CLILoadSelectedTaxonomy()
-	if error != nil {
-		log.Panicln(error)
-	}
 
 	references := obitag.CLIRefDB()
 
+	if references == nil {
+		log.Panicln("No loaded reference database")
+	}
+
+	if taxo == nil {
+		taxo, err = references.ExtractTaxonomy(nil, obitaxonomy.CLINewickWithLeaves())
+
+		if err != nil {
+			log.Fatalf("No taxonomy specified or extractable from reference database: %v", err)
+		}
+
+		taxo.SetAsDefault()
+	}
+
+	if taxo == nil {
+		log.Panicln("No loaded taxonomy")
+	}
+
 	var identified obiiter.IBioSequence
 
+	fsrb := fs.Rebatch(obidefault.BatchSize())
+
 	if obitag.CLIGeometricMode() {
-		identified = obitag.CLIGeomAssignTaxonomy(fs, references, taxo)
+		identified = obitag.CLIGeomAssignTaxonomy(fsrb, references, taxo)
 	} else {
-		identified = obitag.CLIAssignTaxonomy(fs, references, taxo)
+		identified = obitag.CLIAssignTaxonomy(fsrb, references, taxo)
 	}
 
 	obiconvert.CLIWriteBioSequences(identified, true)
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 	obitag.CLISaveRefetenceDB(references)
 

cmd/obitools/obitag2/main.go

@@ -1,65 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"os"
-
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obifind"
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obitag"
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obitag2"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
-)
-
-func main() {
-
-	// go tool pprof -http=":8000" ./build/obitag ./cpu.pprof
-	// f, err := os.Create("cpu.pprof")
-	// if err != nil {
-	// 	log.Fatal(err)
-	// }
-	// pprof.StartCPUProfile(f)
-	// defer pprof.StopCPUProfile()
-
-	// go tool trace cpu.trace
-	// ftrace, err := os.Create("cpu.trace")
-	// if err != nil {
-	// 	log.Fatal(err)
-	// }
-	// trace.Start(ftrace)
-	// defer trace.Stop()
-
-	obioptions.SetWorkerPerCore(2)
-	obioptions.SetStrictReadWorker(1)
-	obioptions.SetStrictWriteWorker(1)
-	obioptions.SetBatchSize(10)
-
-	optionParser := obioptions.GenerateOptionParser(obitag.OptionSet)
-
-	_, args := optionParser(os.Args)
-
-	fs, err := obiconvert.CLIReadBioSequences(args...)
-
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
-
-	taxo, error := obifind.CLILoadSelectedTaxonomy()
-	if error != nil {
-		log.Panicln(error)
-	}
-
-	references := obitag.CLIRefDB()
-
-	identified := obitag2.CLIAssignTaxonomy(fs, references, taxo)
-
-	obiconvert.CLIWriteBioSequences(identified, true)
-	obiiter.WaitForLastPipe()
-
-	fmt.Println("")
-}

cmd/obitools/obitagpcr/main.go

@@ -5,11 +5,12 @@ import (
 
 	log "github.com/sirupsen/logrus"
 
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obipairing"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obitagpcr"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
 
 func main() {
@@ -30,9 +31,12 @@ func main() {
 	// trace.Start(ftrace)
 	// defer trace.Stop()
 
-	obioptions.SetWorkerPerCore(1)
+	obidefault.SetWorkerPerCore(1)
 
-	optionParser := obioptions.GenerateOptionParser(obitagpcr.OptionSet)
+	optionParser := obioptions.GenerateOptionParser(
+		"obitagpcr",
+		"split a paired raw read data set per sample",
+		obitagpcr.OptionSet)
 
 	optionParser(os.Args)
 	pairs, err := obipairing.CLIPairedSequence()
@@ -54,5 +58,5 @@ func main() {
 
 	obiconvert.CLIWriteBioSequences(paired, true)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 }

cmd/obitools/obitaxonomy/main.go

@@ -0,0 +1,142 @@
+package main
+
+import (
+	"os"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiitercsv"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obicsv"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obitaxonomy"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+
+	log "github.com/sirupsen/logrus"
+)
+
+func main() {
+	optionParser := obioptions.GenerateOptionParser(
+		"obitaxonomy",
+		"manipulates and queries taxonomy",
+		obitaxonomy.OptionSet)
+
+	_, args := optionParser(os.Args)
+
+	var iterator *obitax.ITaxon
+
+	if obitaxonomy.CLIDownloadNCBI() {
+		err := obitaxonomy.CLIDownloadNCBITaxdump()
+		if err != nil {
+			log.Errorf("Cannot download NCBI taxonomy: %s", err.Error())
+			os.Exit(1)
+		}
+		os.Exit(0)
+	}
+
+	if !obidefault.HasSelectedTaxonomy() {
+		log.Fatal("you must indicate a taxonomy using the -t or --taxonomy option")
+	}
+
+	switch {
+	case obitaxonomy.CLIAskForRankList():
+		newIter := obiitercsv.NewICSVRecord()
+		newIter.Add(1)
+		newIter.AppendField("rank")
+		go func() {
+			ranks := obitax.DefaultTaxonomy().RankList()
+			data := make([]obiitercsv.CSVRecord, len(ranks))
+
+			for i, rank := range ranks {
+				record := make(obiitercsv.CSVRecord)
+				record["rank"] = rank
+				data[i] = record
+			}
+			newIter.Push(obiitercsv.MakeCSVRecordBatch(obitax.DefaultTaxonomy().Name(), 0, data))
+			newIter.Close()
+			newIter.Done()
+		}()
+		obicsv.CLICSVWriter(newIter, true)
+		obiutils.WaitForLastPipe()
+		os.Exit(0)
+
+	case obitaxonomy.CLIExtractTaxonomy():
+		iter, err := obiconvert.CLIReadBioSequences(args...)
+		iter = iter.NumberSequences(1, true)
+
+		if err != nil {
+			log.Fatalf("Cannot extract taxonomy: %v", err)
+		}
+
+		taxonomy, err := iter.ExtractTaxonomy(obitaxonomy.CLINewickWithLeaves())
+
+		if err != nil {
+			log.Fatalf("Cannot extract taxonomy: %v", err)
+		}
+
+		taxonomy.SetAsDefault()
+
+		log.Infof("Number of extracted taxa: %d", taxonomy.Len())
+		iterator = taxonomy.AsTaxonSet().Sort().Iterator()
+
+	case obitaxonomy.CLIDumpSubtaxonomy():
+		iterator = obitaxonomy.CLISubTaxonomyIterator()
+
+	case obitaxonomy.CLIRequestsPathForTaxid() != "NA":
+
+		taxon, isAlias, err := obitax.DefaultTaxonomy().Taxon(obitaxonomy.CLIRequestsPathForTaxid())
+
+		if err != nil {
+			log.Fatalf("Cannot identify the requested taxon: %s (%v)",
+				obitaxonomy.CLIRequestsPathForTaxid(), err)
+		}
+
+		if isAlias {
+			if obidefault.FailOnTaxonomy() {
+				log.Fatalf("Taxon %s is an alias for %s", taxon.String(), taxon.Parent().String())
+			}
+		}
+
+		s := taxon.Path()
+
+		if s == nil {
+			log.Fatalf("Cannot extract taxonomic path describing %s", taxon.String())
+		}
+
+		iterator = s.Iterator()
+
+		if obitaxonomy.CLIWithQuery() {
+			iterator = iterator.AddMetadata("query", taxon.String())
+		}
+
+	case len(args) == 0:
+		iterator = obitax.DefaultTaxonomy().Iterator()
+	default:
+		iters := make([]*obitax.ITaxon, len(args))
+
+		for i, pat := range args {
+			ii := obitax.DefaultTaxonomy().IFilterOnName(pat, obitaxonomy.CLIFixedPattern(), true)
+			if obitaxonomy.CLIWithQuery() {
+				ii = ii.AddMetadata("query", pat)
+			}
+			iters[i] = ii
+		}
+
+		iterator = iters[0]
+
+		if len(iters) > 1 {
+			iterator = iterator.Concat(iters[1:]...)
+		}
+	}
+
+	iterator = obitaxonomy.CLITaxonRestrictions(iterator)
+
+	if obitaxonomy.CLIAsNewick() {
+		obitaxonomy.CLINewickWriter(iterator, true)
+	} else {
+		obitaxonomy.CLICSVTaxaWriter(iterator, true)
+	}
+
+	obiutils.WaitForLastPipe()
+
+}

cmd/obitools/obiuniq/main.go

@@ -3,13 +3,12 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiuniq"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
 
 func main() {
@@ -32,20 +31,21 @@ func main() {
 	// trace.Start(ftrace)
 	// defer trace.Stop()
 
-	optionParser := obioptions.GenerateOptionParser(obiuniq.OptionSet)
+	obidefault.SetBatchSize(10)
+	obidefault.SetReadQualities(false)
+	optionParser := obioptions.GenerateOptionParser(
+		"obiuniq",
+		"dereplicate sequence data sets",
+		obiuniq.OptionSet)
 
 	_, args := optionParser(os.Args)
 
 	sequences, err := obiconvert.CLIReadBioSequences(args...)
-
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 
 	unique := obiuniq.CLIUnique(sequences)
 	obiconvert.CLIWriteBioSequences(unique, true)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/test/main.go

@@ -3,36 +3,14 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiformats"
-
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 )
 
 func main() {
-	optionParser := obioptions.GenerateOptionParser(obiconvert.OptionSet)
 
-	_, args := optionParser(os.Args)
+	obiformats.DetectTaxonomyFormat(os.Args[1])
-
+	println(obiutils.RemoveAllExt("toto/tutu/test.txt"))
-	fs, err := obiconvert.CLIReadBioSequences(args...)
+	println(obiutils.Basename("toto/tutu/test.txt"))
-
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
-
-	frags := obiiter.IFragments(
-		1000,
-		100,
-		10,
-		100,
-		obioptions.CLIParallelWorkers(),
-	)
-
-	obiconvert.CLIWriteBioSequences(fs.Pipe(frags), true)
-
-	obiiter.WaitForLastPipe()
 
 }

git-hooks/pre-push

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+remote="$1"
+#url="$2"
+
+log() {
+    echo -e "[Pre-Push tests @ $(date)] $*" 1>&2
+}
+
+current_branch=$(git symbolic-ref --short head)
+
+cmd="make githubtests"
+
+if [[ $current_branch = "master" ]]; then
+    log "you are on $current_branch, running build test"
+    if ! eval "$cmd"; then
+        log "Pre-push tests failed $cmd" 
+        exit 1
+    fi
+fi
+
+log "Tests are OK, ready to push on $remote"
+exit 0

go.mod

@@ -1,42 +1,39 @@
 module git.metabarcoding.org/obitools/obitools4/obitools4
 
-go 1.22.1
+go 1.23.4
+
+toolchain go1.24.2
 
 require (
 	github.com/DavidGamba/go-getoptions v0.28.0
 	github.com/PaesslerAG/gval v1.2.2
 	github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df
+	github.com/buger/jsonparser v1.1.1
 	github.com/chen3feng/stl4go v0.1.1
+	github.com/dlclark/regexp2 v1.11.4
 	github.com/goccy/go-json v0.10.3
 	github.com/klauspost/pgzip v1.2.6
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
+	github.com/pelletier/go-toml/v2 v2.2.4
 	github.com/rrethy/ahocorasick v1.0.0
 	github.com/schollz/progressbar/v3 v3.13.1
 	github.com/sirupsen/logrus v1.9.3
 	github.com/stretchr/testify v1.8.4
 	github.com/tevino/abool/v2 v2.1.0
 	github.com/yuin/gopher-lua v1.1.1
-	golang.org/x/exp v0.0.0-20231006140011-7918f672742d
+	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa
 	gonum.org/v1/gonum v0.14.0
 	gopkg.in/yaml.v3 v3.0.1
 	scientificgo.org/special v0.0.0
 )
 
 require (
-	github.com/bytedance/sonic v1.11.9 // indirect
-	github.com/bytedance/sonic/loader v0.1.1 // indirect
-	github.com/cloudwego/base64x v0.1.4 // indirect
-	github.com/cloudwego/iasm v0.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/goombaio/orderedmap v0.0.0-20180924084748-ba921b7e2419 // indirect
-	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
-	github.com/kr/pretty v0.3.0 // indirect
 	github.com/kr/text v0.2.0 // indirect
-	github.com/montanaflynn/stats v0.7.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/rogpeppe/go-internal v1.6.1 // indirect
+	github.com/rogpeppe/go-internal v1.12.0 // indirect
-	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
-	golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect
 )
 
 require (
@@ -49,8 +46,8 @@ require (
 	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/shopspring/decimal v1.3.1 // indirect
 	github.com/ulikunitz/xz v0.5.11
-	golang.org/x/net v0.17.0 // indirect
+	golang.org/x/net v0.35.0 // indirect
-	golang.org/x/sys v0.17.0 // indirect
+	golang.org/x/sys v0.30.0 // indirect
-	golang.org/x/term v0.13.0 // indirect
+	golang.org/x/term v0.29.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c
 )

go.sum

@@ -6,27 +6,21 @@ github.com/PaesslerAG/jsonpath v0.1.0 h1:gADYeifvlqK3R3i2cR5B4DGgxLXIPb3TRTH1mGi
 github.com/PaesslerAG/jsonpath v0.1.0/go.mod h1:4BzmtoM/PI8fPO4aQGIusjGxGir2BzcV0grWtFzq1Y8=
 github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df h1:GSoSVRLoBaFpOOds6QyY1L8AX7uoY+Ln3BHc22W40X0=
 github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df/go.mod h1:hiVxq5OP2bUGBRNS3Z/bt/reCLFNbdcST6gISi1fiOM=
-github.com/bytedance/sonic v1.11.9 h1:LFHENlIY/SLzDWverzdOvgMztTxcfcF+cqNsz9pK5zg=
+github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
-github.com/bytedance/sonic v1.11.9/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4=
+github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
-github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM=
-github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
 github.com/chen3feng/stl4go v0.1.1 h1:0L1+mDw7pomftKDruM23f1mA7miavOj6C6MZeadzN2Q=
 github.com/chen3feng/stl4go v0.1.1/go.mod h1:5ml3psLgETJjRJnMbPE+JiHLrCpt+Ajc2weeTECXzWU=
-github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y=
-github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
-github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg=
-github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo=
+github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
 github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q=
 github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo=
 github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
 github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
 github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
-github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
-github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
 github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
 github.com/goombaio/orderedmap v0.0.0-20180924084748-ba921b7e2419 h1:SajEQ6tktpF9SRIuzbiPOX9AEZZ53Bvw0k9Mzrts8Lg=
@@ -37,17 +31,12 @@ github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1
 github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
 github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
-github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE=
 github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
-github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
-github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU=
 github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
-github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
-github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
-github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -58,17 +47,19 @@ github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZ
 github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
-github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
-github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
+github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
+github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
+github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
 github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
-github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=
+github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
-github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
+github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
+github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
 github.com/rrethy/ahocorasick v1.0.0 h1:YKkCB+E5PXc0xmLfMrWbfNht8vG9Re97IHSWZk/Lk8E=
 github.com/rrethy/ahocorasick v1.0.0/go.mod h1:nq8oScE7Vy1rOppoQxpQiiDmPHuKCuk9rXrNcxUV3R0=
 github.com/schollz/progressbar/v3 v3.13.1 h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE=
@@ -78,50 +69,37 @@ github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFR
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/tevino/abool/v2 v2.1.0 h1:7w+Vf9f/5gmKT4m4qkayb33/92M+Um45F2BkHOR+L/c=
 github.com/tevino/abool/v2 v2.1.0/go.mod h1:+Lmlqk6bHDWHqN1cbxqhwEAwMPXgc8I1SDEamtseuXY=
-github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
-github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
 github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
 github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
 github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
 github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
-golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU=
+golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ=
-golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
+golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE=
-golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
+golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8=
-golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
+golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk=
-golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
-golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
+golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
-golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
-golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
+golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU=
-golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
+golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s=
 gonum.org/v1/gonum v0.14.0 h1:2NiG67LD1tEH0D7kM+ps2V+fXmsAnpUeec7n8tcr4S0=
 gonum.org/v1/gonum v0.14.0/go.mod h1:AoWeoz0becf9QMWtE8iWXNXc27fK4fNeHNf/oMejGfU=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
-gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=
-rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
 scientificgo.org/special v0.0.0 h1:P6WJkECo6tgtvZAEfNXl+KEB9ReAatjKAeX8U07mjSc=
 scientificgo.org/special v0.0.0/go.mod h1:LoGVh9tS431RLTJo7gFlYDKFWq44cEb7QqL+M0EKtZU=
 scientificgo.org/testutil v0.0.0 h1:y356DHRo0tAz9zIFmxlhZoKDlHPHaWW/DCm9k3PhIMA=

go.work

@@ -1,3 +1,5 @@
-go 1.22.1
+go 1.23.4
+
+toolchain go1.24.2
 
 use .

go.work.sum

@@ -2,7 +2,6 @@ git.sr.ht/~sbinet/gg v0.3.1 h1:LNhjNn8DerC8f9DHLz6lS0YYul/b602DUxDgGkd/Aik=
 git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc=
 github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b h1:slYM766cy2nI3BwyRiyQj/Ud48djTMtMebDqepE95rw=
 github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGWcpt8ov532z81sp/kMMUG485J2InIOyADM=
-github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
 github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/logex v1.2.0 h1:+eqR0HfOetur4tgnC8ftU5imRnhi4te+BadWS95c5AM=
@@ -20,15 +19,20 @@ github.com/go-latex/latex v0.0.0-20230307184459-12ec69307ad9 h1:NxXI5pTAtpEaU49b
 github.com/go-latex/latex v0.0.0-20230307184459-12ec69307ad9/go.mod h1:gWuR/CrFDDeVRFQwHPvsv9soJVB/iqymhuZQuJ3a9OM=
 github.com/go-pdf/fpdf v0.6.0 h1:MlgtGIfsdMEEQJr2le6b/HNr1ZlQwxyWr77r2aj2U/8=
 github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M=
+github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow=
 github.com/goccmack/gocc v0.0.0-20230228185258-2292f9e40198 h1:FSii2UQeSLngl3jFoR4tUKZLprO7qUlh/TKKticc0BM=
 github.com/goccmack/gocc v0.0.0-20230228185258-2292f9e40198/go.mod h1:DTh/Y2+NbnOVVoypCCQrovMPDKUGp4yZpSbWg5D0XIM=
 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g=
 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
+github.com/google/go-cmdtest v0.4.1-0.20220921163831-55ab3332a786/go.mod h1:apVn/GCasLZUVpAJ6oWAuyP7Ne7CEsQbTnc0plM3m+o=
 github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
 github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/safehtml v0.1.0/go.mod h1:L4KWwDsUJdECRAEpZoBn3O64bQaywRscowZjJAzjHnU=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
 github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2 h1:rcanfLhLDA8nozr/K289V1zcntHr3V+SHlXwzz1ZI2g=
+github.com/jba/templatecheck v0.7.1/go.mod h1:n1Etw+Rrw1mDDD8dDRsEKTwMZsJ98EkktgNJC6wLUGo=
 github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 h1:qGQQKEcAR99REcMpsXCp3lJ03zYT1PkRd3kQGPn9GVg=
 github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE=
 github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw=
@@ -39,17 +43,22 @@ github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
 github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
+golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
 golang.org/x/image v0.6.0 h1:bR8b5okrPI3g/gyZakLZHeWxAR8Dn5CyxXv1hLH5g/4=
 golang.org/x/image v0.6.0/go.mod h1:MXLdDR43H7cDJq5GEGXEVeeNhPgi+YYEQ2pC1byI1x0=
 golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY=
 golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
 golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
 golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc=
 golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
+golang.org/x/tools v0.15.0/go.mod h1:hpksKq4dtpQWS1uQ61JkdqWM3LscIS6Slf+VVkm+wQk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc=
 gonum.org/v1/plot v0.10.1 h1:dnifSs43YJuNMDzB7v8wV64O4ABBHReuAVAoBxqBqS4=
 gonum.org/v1/plot v0.10.1/go.mod h1:VZW5OlhkL1mysU9vaqNHnsy86inf6Ot+jB3r+BczCEo=

install_obitools.sh

@@ -1,27 +1,56 @@
 #!/bin/bash
 
-INSTALL_DIR="/usr/local"
+# Default values
-OBITOOLS_PREFIX=""
-# default values
 URL="https://go.dev/dl/"
-OBIURL4="https://github.com/metabarcoding/obitools4/archive/refs/heads/master.zip"
+GITHUB_REPO="https://github.com/metabarcoding/obitools4"
 INSTALL_DIR="/usr/local"
 OBITOOLS_PREFIX=""
+VERSION=""
+LIST_VERSIONS=false
 
-# help message
+# Help message
 function display_help {
   echo "Usage: $0 [OPTIONS]"
   echo ""
   echo "Options:"
   echo "  -i, --install-dir       Directory where obitools are installed "
-  echo "                          (as example use /usr/local not /usr/local/bin)."
+  echo "                          (e.g., use /usr/local not /usr/local/bin)."
   echo "  -p, --obitools-prefix   Prefix added to the obitools command names if you"
   echo "                          want to have several versions of obitools at the"
-  echo "                          same time on your system (as example -p g will produce "
+  echo "                          same time on your system (e.g., -p g will produce "
   echo "                          gobigrep command instead of obigrep)."
+  echo "  -v, --version           Install a specific version (e.g., 4.4.8)."
+  echo "                          If not specified, installs the latest version."
+  echo "  -l, --list              List all available versions and exit."
   echo "  -h, --help              Display this help message."
+  echo ""
+  echo "Examples:"
+  echo "  $0                      # Install latest version"
+  echo "  $0 -l                   # List available versions"
+  echo "  $0 -v 4.4.8             # Install specific version"
+  echo "  $0 -i /opt/local        # Install to custom directory"
 }
 
+# List available versions from GitHub releases
+function list_versions {
+  echo "Fetching available versions..." 1>&2
+  echo ""
+  curl -s "https://api.github.com/repos/metabarcoding/obitools4/releases" \
+    | grep '"tag_name":' \
+    | sed -E 's/.*"tag_name": "Release_([0-9.]+)".*/\1/' \
+    | sort -V -r
+}
+
+# Get latest version from GitHub releases
+function get_latest_version {
+  curl -s "https://api.github.com/repos/metabarcoding/obitools4/releases" \
+    | grep '"tag_name":' \
+    | sed -E 's/.*"tag_name": "Release_([0-9.]+)".*/\1/' \
+    | sort -V -r \
+    | head -1
+}
+
+# Parse command line arguments
 while [ "$#" -gt 0 ]; do
   case "$1" in
     -i|--install-dir)
@@ -32,62 +61,104 @@ while [ "$#" -gt 0 ]; do
       OBITOOLS_PREFIX="$2"
       shift 2
       ;;
+    -v|--version)
+      VERSION="$2"
+      shift 2
+      ;;
+    -l|--list)
+      LIST_VERSIONS=true
+      shift
+      ;;
     -h|--help)
-      display_help  1>&2 
+      display_help
       exit 0
       ;;
     *)
-      echo "Error: Unsupported option $1"  1>&2
+      echo "Error: Unsupported option $1" 1>&2
+      display_help 1>&2
       exit 1
       ;;
   esac
 done
 
-# the directory from where the script is run
+# List versions and exit if requested
+if [ "$LIST_VERSIONS" = true ]; then
+  echo "Available OBITools4 versions:"
+  echo "=============================="
+  list_versions
+  exit 0
+fi
+
+# Determine version to install
+if [ -z "$VERSION" ]; then
+  echo "Fetching latest version..." 1>&2
+  VERSION=$(get_latest_version)
+  if [ -z "$VERSION" ]; then
+    echo "Error: Could not determine latest version" 1>&2
+    exit 1
+  fi
+  echo "Latest version: $VERSION" 1>&2
+else
+  echo "Installing version: $VERSION" 1>&2
+fi
+
+# Construct source URL for the specified version
+OBIURL4="${GITHUB_REPO}/archive/refs/tags/Release_${VERSION}.zip"
+
+# The directory from where the script is run
 DIR="$(pwd)"
 
-# the temp directory used, within $DIR
+# Create temporary directory
-# omit the -p parameter to create a temporal directory in the default location
-# WORK_DIR=$(mktemp -d -p "$DIR"  "obitools4.XXXXXX" 2> /dev/null || \
-#            mktemp -d -t "$DIR"  "obitools4.XXXXXX")
-
 WORK_DIR=$(mktemp -d "obitools4.XXXXXX")
 
-# check if tmp dir was created
+# Check if tmp dir was created
 if [[ ! "$WORK_DIR" || ! -d "$WORK_DIR" ]]; then
-  echo "Could not create temp dir"  1>&2
+  echo "Could not create temp dir" 1>&2
   exit 1
 fi
 
+mkdir -p "${WORK_DIR}/cache" \
+  || (echo "Cannot create ${WORK_DIR}/cache directory" 1>&2
+      exit 1)
+
+# Create installation directory
 mkdir -p "${INSTALL_DIR}/bin" 2> /dev/null \
-  || (echo "Please enter your password for installing obitools in ${INSTALL_DIR}"  1>&2
+  || (echo "Please enter your password for installing obitools in ${INSTALL_DIR}" 1>&2
       sudo mkdir -p "${INSTALL_DIR}/bin")
 
 if [[ ! -d "${INSTALL_DIR}/bin" ]]; then
-  echo "Could not create ${INSTALL_DIR}/bin directory for installing obitools"  1>&2
+  echo "Could not create ${INSTALL_DIR}/bin directory for installing obitools" 1>&2
   exit 1
 fi
 
-INSTALL_DIR="$(cd $INSTALL_DIR && pwd)"
+INSTALL_DIR="$(cd ${INSTALL_DIR} && pwd)"
 
-echo WORK_DIR=$WORK_DIR  1>&2
+echo "================================" 1>&2
-echo INSTALL_DIR=$INSTALL_DIR  1>&2
+echo "OBITools4 Installation" 1>&2
-echo OBITOOLS_PREFIX=$OBITOOLS_PREFIX  1>&2
+echo "================================" 1>&2
+echo "VERSION=$VERSION" 1>&2
+echo "WORK_DIR=$WORK_DIR" 1>&2
+echo "INSTALL_DIR=$INSTALL_DIR" 1>&2
+echo "OBITOOLS_PREFIX=$OBITOOLS_PREFIX" 1>&2
+echo "================================" 1>&2
 
-pushd "$WORK_DIR"|| exit
+pushd "$WORK_DIR" > /dev/null || exit
 
+# Detect OS and architecture
 OS=$(uname -a | awk '{print $1}')
 ARCH=$(uname -m)
 
-if [[ "$ARCH" == "x86_64" ]] ; then 
+if [[ "$ARCH" == "x86_64" ]] ; then
-    ARCH="amd64" 
+    ARCH="amd64"
 fi
 
-if [[ "$ARCH" == "aarch64" ]] ; then 
+if [[ "$ARCH" == "aarch64" ]] ; then
-    ARCH="arm64" 
+    ARCH="arm64"
 fi
 
-GOFILE=$(curl "$URL" \
+# Download and install Go
+echo "Downloading Go..." 1>&2
+GOFILE=$(curl -s "$URL" \
             | grep 'class="download"' \
             | grep "\.tar\.gz" \
             | sed -E 's@^.*/dl/(go[1-9].+\.tar\.gz)".*$@\1@' \
@@ -95,35 +166,71 @@ GOFILE=$(curl "$URL" \
             | grep -i "$ARCH" \
             | head -1)
 
-GOURL=$(curl "${URL}${GOFILE}" \
+GOURL=$(curl -s "${URL}${GOFILE}" \
         | sed -E 's@^.*href="(.*\.tar\.gz)".*$@\1@')
 
-echo "Install GO from : $GOURL" 1>&2
+echo "Installing Go from: $GOURL" 1>&2
-        
+
-curl "$GOURL" \
+curl -s "$GOURL" | tar zxf -
-    | tar zxf -
 
 PATH="$(pwd)/go/bin:$PATH"
 export PATH
+GOPATH="$(pwd)/go"
+export GOPATH
+export GOCACHE="$(pwd)/cache"
 
-curl -L "$OBIURL4" > master.zip
+echo "GOCACHE=$GOCACHE" 1>&2
-unzip master.zip
+mkdir -p "$GOCACHE"
 
-echo "Install OBITOOLS from : $OBIURL4"
+# Download OBITools4 source
+echo "Downloading OBITools4 v${VERSION}..." 1>&2
+echo "Source URL: $OBIURL4" 1>&2
 
-cd obitools4-master || exit
+if ! curl -sL "$OBIURL4" > obitools4.zip; then
-
+  echo "Error: Could not download OBITools4 version ${VERSION}" 1>&2
-if [[ -z "$OBITOOLS_PREFIX" ]] ; then
+  echo "Please check that this version exists with: $0 --list" 1>&2
-  make
+  exit 1
-else
-  make OBITOOLS_PREFIX="${OBITOOLS_PREFIX}"
 fi
 
+unzip -q obitools4.zip
+
+# Find the extracted directory
+OBITOOLS_DIR=$(ls -d obitools4-* 2>/dev/null | head -1)
+
+if [ -z "$OBITOOLS_DIR" ] || [ ! -d "$OBITOOLS_DIR" ]; then
+  echo "Error: Could not find extracted OBITools4 directory" 1>&2
+  exit 1
+fi
+
+echo "Building OBITools4..." 1>&2
+cd "$OBITOOLS_DIR" || exit
+mkdir -p vendor
+
+# Build with or without prefix
+if [[ -z "$OBITOOLS_PREFIX" ]] ; then
+  make GOFLAGS="-buildvcs=false"
+else
+  make GOFLAGS="-buildvcs=false" OBITOOLS_PREFIX="${OBITOOLS_PREFIX}"
+fi
+
+# Install binaries
+echo "Installing binaries to ${INSTALL_DIR}/bin..." 1>&2
 (cp build/* "${INSTALL_DIR}/bin" 2> /dev/null) \
-   || (echo "Please enter your password for installing obitools in ${INSTALL_DIR}" 
+   || (echo "Please enter your password for installing obitools in ${INSTALL_DIR}" 1>&2
        sudo cp build/* "${INSTALL_DIR}/bin")
 
-popd || exit
+popd > /dev/null || exit
 
+# Cleanup
+echo "Cleaning up..." 1>&2
+chmod -R +w "$WORK_DIR"
 rm -rf "$WORK_DIR"
 
+echo "" 1>&2
+echo "================================" 1>&2
+echo "OBITools4 v${VERSION} installed successfully!" 1>&2
+echo "Binaries location: ${INSTALL_DIR}/bin" 1>&2
+if [[ -n "$OBITOOLS_PREFIX" ]] ; then
+  echo "Command prefix: ${OBITOOLS_PREFIX}" 1>&2
+fi
+echo "================================" 1>&2

obitests/obitools/obiannotate/test.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obiannotate
+CMD=obiannotate
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1 
+then
+    log "$MCMD: printing help OK" 
+    ((success++))
+else
+    log "$MCMD: printing help failed" 
+    ((failed++))
+fi
+
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obiclean/test.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obiclean
+CMD=obiclean
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1 
+then
+    log "$MCMD: printing help OK" 
+    ((success++))
+else
+    log "$MCMD: printing help failed" 
+    ((failed++))
+fi
+
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obicleandb/test.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obicleandb
+CMD=obicleandb
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1 
+then
+    log "$MCMD: printing help OK" 
+    ((success++))
+else
+    log "$MCMD: printing help failed" 
+    ((failed++))
+fi
+
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obicomplement/test.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obicomplement
+CMD=obicomplement
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1 
+then
+    log "$MCMD: printing help OK" 
+    ((success++))
+else
+    log "$MCMD: printing help failed" 
+    ((failed++))
+fi
+
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obiconsensus/test.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obiconsensus
+CMD=obiconsensus
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1 
+then
+    log "$MCMD: printing help OK" 
+    ((success++))
+else
+    log "$MCMD: printing help failed" 
+    ((failed++))
+fi
+
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obiconvert/test.sh

@@ -0,0 +1,144 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obiconvert
+CMD=obiconvert
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+
+if [ -z "$TEST_DIR" ] ; then
+   TEST_DIR="."
+fi
+
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1 
+then
+    log "$MCMD: printing help OK" 
+    ((success++))
+else
+    log "$MCMD: printing help failed" 
+    ((failed++))
+fi
+
+
+((ntest++))
+if obiconvert -Z "${TEST_DIR}/gbpln1088.4Mb.fasta.gz" \
+                 > "${TMPDIR}/xxx.fasta.gz" && \
+   zdiff "${TEST_DIR}/gbpln1088.4Mb.fasta.gz" \
+                 "${TMPDIR}/xxx.fasta.gz" 
+then
+    log "$MCMD: converting large fasta file to fasta OK" 
+    ((success++))
+else
+    log "$MCMD: converting large fasta file to fasta failed" 
+    ((failed++))
+fi
+ 
+((ntest++))
+if obiconvert -Z --fastq-output \
+              "${TEST_DIR}/gbpln1088.4Mb.fasta.gz" \
+                 > "${TMPDIR}/xxx.fastq.gz" && \
+   obiconvert -Z --fasta-output \
+              "${TMPDIR}/xxx.fastq.gz" \
+              > "${TMPDIR}/yyy.fasta.gz" && \
+   zdiff "${TEST_DIR}/gbpln1088.4Mb.fasta.gz" \
+                 "${TMPDIR}/yyy.fasta.gz" 
+then
+    log "$MCMD: converting large file between fasta and fastq OK" 
+    ((success++))
+else
+    log "$MCMD: converting large file between fasta and fastq failed" 
+    ((failed++))
+fi
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obicount/test.sh

@@ -0,0 +1,163 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obicount
+CMD=obicount
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1 
+then
+    log "$MCMD: printing help OK" 
+    ((success++))
+else
+    log "$MCMD: printing help failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if obicount "${TEST_DIR}/wolf_F.fasta.gz" \
+    > "${TMPDIR}/wolf_F.fasta_count.csv" 
+then
+    log "OBICount: fasta reading OK" 
+    ((success++))
+else
+    log "OBICount: fasta reading failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if obicount "${TEST_DIR}/wolf_F.fastq.gz" \
+    > "${TMPDIR}/wolf_F.fastq_count.csv"
+then
+    log "OBICount: fastq reading OK"
+    ((success++))
+else
+    log "OBICount: fastq reading failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if obicount "${TEST_DIR}/wolf_F.csv.gz" \
+    > "${TMPDIR}/wolf_F.csv_count.csv"
+then
+    log "OBICount: csv reading OK" 
+    ((success++))
+else
+    log "OBICount: csv reading failed"
+    ((failed++))
+fi
+
+((ntest++))
+if diff "${TMPDIR}/wolf_F.fasta_count.csv" \
+        "${TMPDIR}/wolf_F.fastq_count.csv"  > /dev/null
+then
+    log "OBICount: counting on fasta and fastq are identical OK"
+    ((success++))
+else
+    log "OBICount: counting on fasta and fastq are different failed"
+    ((failed++))
+fi
+
+((ntest++))
+if diff "${TMPDIR}/wolf_F.fasta_count.csv" \
+        "${TMPDIR}/wolf_F.csv_count.csv" > /dev/null
+then
+    log "OBICount: counting on fasta and csv are identical OK"
+    ((success++))
+else
+    log "OBICount: counting on fasta and csv are different failed"
+    ((failed++))
+fi
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obicsv/test.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obicsv
+CMD=obicsv
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1 
+then
+    log "$MCMD: printing help OK" 
+    ((success++))
+else
+    log "$MCMD: printing help failed" 
+    ((failed++))
+fi
+
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obidemerge/test.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obidemerge
+CMD=obidemerge
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1 
+then
+    log "$MCMD: printing help OK" 
+    ((success++))
+else
+    log "$MCMD: printing help failed" 
+    ((failed++))
+fi
+
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obidistribute/test.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obidistribute
+CMD=obidistribute
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1 
+then
+    log "$MCMD: printing help OK" 
+    ((success++))
+else
+    log "$MCMD: printing help failed" 
+    ((failed++))
+fi
+
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obigrep/test.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obigrep
+CMD=obigrep
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1 
+then
+    log "$MCMD: printing help OK" 
+    ((success++))
+else
+    log "$MCMD: printing help failed" 
+    ((failed++))
+fi
+
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obijoin/test.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obijoin
+CMD=obijoin
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1 
+then
+    log "$MCMD: printing help OK" 
+    ((success++))
+else
+    log "$MCMD: printing help failed" 
+    ((failed++))
+fi
+
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obikmermatch/test.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obikmermatch
+CMD=obikmermatch
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1 
+then
+    log "$MCMD: printing help OK" 
+    ((success++))
+else
+    log "$MCMD: printing help failed" 
+    ((failed++))
+fi
+
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obikmersimcount/test.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obikmersimcount
+CMD=obikmersimcount
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1 
+then
+    log "$MCMD: printing help OK" 
+    ((success++))
+else
+    log "$MCMD: printing help failed" 
+    ((failed++))
+fi
+
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obilandmark/test.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obilandmark
+CMD=obilandmark
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1 
+then
+    log "$MCMD: printing help OK" 
+    ((success++))
+else
+    log "$MCMD: printing help failed" 
+    ((failed++))
+fi
+
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obimatrix/test.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obimatrix
+CMD=obimatrix
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1 
+then
+    log "$MCMD: printing help OK" 
+    ((success++))
+else
+    log "$MCMD: printing help failed" 
+    ((failed++))
+fi
+
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obimicrosat/test.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obimicrosat
+CMD=obimicrosat
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1 
+then
+    log "$MCMD: printing help OK" 
+    ((success++))
+else
+    log "$MCMD: printing help failed" 
+    ((failed++))
+fi
+
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obimultiplex/test.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obimultiplex
+CMD=obimultiplex
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1 
+then
+    log "$MCMD: printing help OK" 
+    ((success++))
+else
+    log "$MCMD: printing help failed" 
+    ((failed++))
+fi
+
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obipairing/test.sh

@@ -0,0 +1,150 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obipairing
+CMD=obipairing
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+       log
+       log
+       exit 1
+    fi
+
+    log
+    log
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1 
+then
+    log "$MCMD: printing help OK" 
+    ((success++))
+else
+    log "$MCMD: printing help failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if obipairing -F "${TEST_DIR}/wolf_F.fastq.gz" \
+              -R "${TEST_DIR}/wolf_R.fastq.gz" \
+    | obidistribute -Z -c mode \
+                    -p "${TMPDIR}/wolf_paired_%s.fastq.gz" 
+then
+    log "OBIPairing: sequence pairing OK" 
+    ((success++))
+else
+    log "OBIPairing: sequence pairing failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if obicsv -Z -s -i \
+          -k ali_dir -k ali_length -k pairing_fast_count \
+          -k pairing_fast_overlap -k pairing_fast_score \
+          -k score -k score_norm -k seq_a_single \
+          -k seq_b_single -k seq_ab_match \
+          "${TMPDIR}/wolf_paired_alignment.fastq.gz" \
+    > "${TMPDIR}/wolf_paired_alignment.csv.gz" \
+    && zdiff -c "${TEST_DIR}/wolf_paired_alignment.csv.gz" \
+                "${TMPDIR}/wolf_paired_alignment.csv.gz" 
+then
+    log "OBIPairing: check aligned sequences OK" 
+    ((success++))
+else
+    log "OBIPairing: check aligned sequences failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if obicsv -Z -s -i \
+          "${TMPDIR}/wolf_paired_join.fastq.gz" \
+    > "${TMPDIR}/wolf_paired_join.csv.gz" \
+    && zdiff -c "${TEST_DIR}/wolf_paired_join.csv.gz" \
+                "${TMPDIR}/wolf_paired_join.csv.gz"
+then
+    log "OBIPairing: check joined sequences OK" 
+    ((success++))
+else
+    log "OBIPairing: check joined sequences failed" 
+    ((failed++))
+fi
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obipcr/test.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obipcr
+CMD=obipcr
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1 
+then
+    log "$MCMD: printing help OK" 
+    ((success++))
+else
+    log "$MCMD: printing help failed" 
+    ((failed++))
+fi
+
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obirefidx/test.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obirefidx
+CMD=obirefidx
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1 
+then
+    log "$MCMD: printing help OK" 
+    ((success++))
+else
+    log "$MCMD: printing help failed" 
+    ((failed++))
+fi
+
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obiscript/test.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obiscript
+CMD=obiscript
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1 
+then
+    log "$MCMD: printing help OK" 
+    ((success++))
+else
+    log "$MCMD: printing help failed" 
+    ((failed++))
+fi
+
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obisplit/test.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obisplit
+CMD=obisplit
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1 
+then
+    log "$MCMD: printing help OK" 
+    ((success++))
+else
+    log "$MCMD: printing help failed" 
+    ((failed++))
+fi
+
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obisummary/some_uniq_seq.fasta

@@ -0,0 +1,9 @@
+>Seq_1 {"count":2,"merged_sample":{"15a_F730814":1,"29a_F260619":1}}
+ttagccctaaacacaagtaattaatataacaaaattattcgccagagtactaccggcaat
+agctyaaaactcaaaggacttggcggtgctttataccctt
+>Seq_2 {"count":22,"merged_sample":{"15a_F730814":12,"29a_F260619":10}}
+ttagccctaaacacaagtaattaatataacaaaattattcgccagagtactaccggcaat
+atcttaaaactcaaaggacttggcggtgctttataccctt
+>Seq_3 {"count":22,"merged_sample":{"15a_F730814":15,"29a_F260619":7}}
+ttagccctaaacacaagtaattaatataacaaaattattcgccagagtactaccggcgat
+agcttaaaactcaaaggacttggcggtgctttataccctt

obitests/obitools/obisummary/some_uniq_seq.json

@@ -0,0 +1,35 @@
+{
+  "annotations": {
+    "keys": {
+      "map": {
+        "merged_sample": 3
+      },
+      "scalar": {
+        "count": 3
+      }
+    },
+    "map_attributes": 1,
+    "scalar_attributes": 1,
+    "vector_attributes": 0
+  },
+  "count": {
+    "reads": 46,
+    "total_length": 300,
+    "variants": 3
+  },
+  "samples": {
+    "sample_count": 2,
+    "sample_stats": {
+      "15a_F730814": {
+        "reads": 28,
+        "singletons": 1,
+        "variants": 3
+      },
+      "29a_F260619": {
+        "reads": 18,
+        "singletons": 1,
+        "variants": 3
+      }
+    }
+  }
+}

obitests/obitools/obisummary/some_uniq_seq.yaml

@@ -0,0 +1,25 @@
+annotations:
+    keys:
+        map:
+            merged_sample: 3
+        scalar:
+            count: 3
+    map_attributes: 1
+    scalar_attributes: 1
+    vector_attributes: 0
+count:
+    reads: 46
+    total_length: 300
+    variants: 3
+samples:
+    sample_count: 2
+    sample_stats:
+        15a_F730814:
+            reads: 28
+            singletons: 1
+            variants: 3
+        29a_F260619:
+            reads: 18
+            singletons: 1
+            variants: 3
+

obitests/obitools/obisummary/test.sh

@@ -0,0 +1,152 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obisummary
+CMD=obisummary
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1 
+then
+    log "$MCMD: printing help OK" 
+    ((success++))
+else
+    log "$MCMD: printing help failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if obisummary "${TEST_DIR}/some_uniq_seq.fasta" \
+              > "${TMPDIR}/some_uniq_seq.json"
+then
+    log "$MCMD: formating json execution OK" 
+    ((success++))
+else
+    log "$MCMD: formating json execution failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if diff "${TEST_DIR}/some_uniq_seq.json" \
+        "${TMPDIR}/some_uniq_seq.json" > /dev/null
+then
+    log "$MCMD: formating json OK" 
+    ((success++))
+else
+    log "$MCMD: formating json failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if obisummary --yaml "${TEST_DIR}/some_uniq_seq.fasta" \
+              > "${TMPDIR}/some_uniq_seq.yaml"
+then
+    log "$MCMD: formating yaml execution OK" 
+    ((success++))
+else
+    log "$MCMD: formating yaml execution failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if diff "${TEST_DIR}/some_uniq_seq.yaml" \
+        "${TMPDIR}/some_uniq_seq.yaml" > /dev/null
+then
+    log "$MCMD: formating yaml OK" 
+    ((success++))
+else
+    log "$MCMD: formating yaml failed" 
+    ((failed++))
+fi
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obisuperkmer/README.md

@@ -0,0 +1,148 @@
+# Tests pour obisuperkmer
+
+## Description
+
+Ce répertoire contient les tests automatisés pour la commande `obisuperkmer`.
+
+## Fichiers
+
+- `test.sh` : Script de test principal (exécutable)
+- `test_sequences.fasta` : Jeu de données de test minimal (3 séquences courtes)
+- `README.md` : Ce fichier
+
+## Jeu de données de test
+
+Le fichier `test_sequences.fasta` contient 3 séquences de 32 nucléotides chacune :
+
+1. **seq1** : Répétition du motif ACGT (séquence régulière)
+2. **seq2** : Alternance de blocs homopolymères (AAAA, CCCC, GGGG, TTTT)
+3. **seq3** : Répétition du motif ATCG (différent de seq1)
+
+Ces séquences sont volontairement courtes pour :
+- Minimiser la taille du dépôt Git
+- Accélérer l'exécution des tests en CI/CD
+- Tester différents cas d'extraction de super k-mers
+
+## Tests effectués
+
+Le script `test.sh` effectue 12 tests :
+
+### Test 1 : Affichage de l'aide
+Vérifie que `obisuperkmer -h` s'exécute correctement.
+
+### Test 2 : Extraction basique avec paramètres par défaut
+Exécute `obisuperkmer` avec k=21, m=11 (valeurs par défaut).
+
+### Test 3 : Vérification du fichier de sortie non vide
+S'assure que la commande produit une sortie.
+
+### Test 4 : Comptage des super k-mers extraits
+Vérifie qu'au moins un super k-mer a été extrait.
+
+### Test 5 : Présence des métadonnées requises
+Vérifie que chaque super k-mer contient :
+- `minimizer_value`
+- `minimizer_seq`
+- `parent_id`
+
+### Test 6 : Extraction avec paramètres personnalisés
+Teste avec k=15 et m=7.
+
+### Test 7 : Vérification des paramètres dans les métadonnées
+S'assure que les valeurs k=15 et m=7 sont présentes dans la sortie.
+
+### Test 8 : Format de sortie FASTA explicite
+Teste l'option `--fasta-output`.
+
+### Test 9 : Vérification des IDs des super k-mers
+S'assure que tous les IDs contiennent "superkmer".
+
+### Test 10 : Préservation des IDs parents
+Vérifie que seq1, seq2 et seq3 apparaissent dans la sortie.
+
+### Test 11 : Option -o pour fichier de sortie
+Teste la redirection vers un fichier avec `-o`.
+
+### Test 12 : Vérification de la création du fichier avec -o
+S'assure que le fichier de sortie a été créé.
+
+### Test 13 : Cohérence des longueurs
+Vérifie que la somme des longueurs des super k-mers est inférieure ou égale à la longueur totale des séquences d'entrée.
+
+## Exécution des tests
+
+### Localement
+
+```bash
+cd /chemin/vers/obitools4/obitests/obitools/obisuperkmer
+./test.sh
+```
+
+### En CI/CD
+
+Les tests sont automatiquement exécutés lors de chaque commit via le système CI/CD configuré pour le projet.
+
+### Prérequis
+
+- La commande `obisuperkmer` doit être compilée et disponible dans `../../build/`
+- Les dépendances système : bash, grep, etc.
+
+## Structure du script de test
+
+Le script suit le pattern standard utilisé par tous les tests OBITools :
+
+1. **En-tête** : Définition du nom du test et de la commande
+2. **Variables** : Configuration des chemins et compteurs
+3. **Fonction cleanup()** : Affiche les résultats et nettoie le répertoire temporaire
+4. **Fonction log()** : Affiche les messages horodatés
+5. **Tests** : Série de tests avec incrémentation des compteurs
+6. **Appel cleanup()** : Nettoyage et sortie avec code de retour approprié
+
+## Format de sortie
+
+Chaque test affiche :
+```
+[obisuperkmer @ date] message
+```
+
+En fin d'exécution :
+```
+========================================
+## Results of the obisuperkmer tests:
+
+- 12 tests run
+- 12 successfully completed
+- 0 failed tests
+
+Cleaning up the temporary directory...
+
+========================================
+```
+
+## Codes de retour
+
+- **0** : Tous les tests ont réussi
+- **1** : Au moins un test a échoué
+
+## Ajout de nouveaux tests
+
+Pour ajouter un nouveau test, suivre le pattern :
+
+```bash
+((ntest++))
+if commande_test arguments
+then
+    log "Description: OK" 
+    ((success++))
+else
+    log "Description: failed"
+    ((failed++))
+fi
+```
+
+## Notes
+
+- Les fichiers temporaires sont créés dans `$TMPDIR` (créé par mktemp)
+- Les fichiers de données sont dans `$TEST_DIR`
+- La commande testée doit être dans `$OBITOOLS_DIR` (../../build/)
+- Le répertoire temporaire est automatiquement nettoyé à la fin

obitests/obitools/obisuperkmer/test.sh

@@ -0,0 +1,253 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obisuperkmer
+CMD=obisuperkmer
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed"
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..."
+log "Test directory is $TEST_DIR"
+log "obitools directory is $OBITOOLS_DIR"
+log "Temporary directory is $TMPDIR"
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1
+then
+    log "$MCMD: printing help OK"
+    ((success++))
+else
+    log "$MCMD: printing help failed"
+    ((failed++))
+fi
+
+# Test 1: Basic super k-mer extraction with default parameters
+((ntest++))
+if obisuperkmer "${TEST_DIR}/test_sequences.fasta" \
+    > "${TMPDIR}/output_default.fasta" 2>&1
+then
+    log "$MCMD: basic extraction with default parameters OK"
+    ((success++))
+else
+    log "$MCMD: basic extraction with default parameters failed"
+    ((failed++))
+fi
+
+# Test 2: Verify output is not empty
+((ntest++))
+if [ -s "${TMPDIR}/output_default.fasta" ]
+then
+    log "$MCMD: output file is not empty OK"
+    ((success++))
+else
+    log "$MCMD: output file is empty - failed"
+    ((failed++))
+fi
+
+# Test 3: Count number of super k-mers extracted (should be > 0)
+((ntest++))
+num_sequences=$(grep -c "^>" "${TMPDIR}/output_default.fasta")
+if [ "$num_sequences" -gt 0 ]
+then
+    log "$MCMD: extracted $num_sequences super k-mers OK"
+    ((success++))
+else
+    log "$MCMD: no super k-mers extracted - failed"
+    ((failed++))
+fi
+
+# Test 4: Verify super k-mers have required metadata attributes
+((ntest++))
+if grep -q "minimizer_value" "${TMPDIR}/output_default.fasta" && \
+   grep -q "minimizer_seq" "${TMPDIR}/output_default.fasta" && \
+   grep -q "parent_id" "${TMPDIR}/output_default.fasta"
+then
+    log "$MCMD: super k-mers contain required metadata OK"
+    ((success++))
+else
+    log "$MCMD: super k-mers missing metadata - failed"
+    ((failed++))
+fi
+
+# Test 5: Extract super k-mers with custom k and m parameters
+((ntest++))
+if obisuperkmer -k 15 -m 7 "${TEST_DIR}/test_sequences.fasta" \
+    > "${TMPDIR}/output_k15_m7.fasta" 2>&1
+then
+    log "$MCMD: extraction with custom k=15, m=7 OK"
+    ((success++))
+else
+    log "$MCMD: extraction with custom k=15, m=7 failed"
+    ((failed++))
+fi
+
+# Test 6: Verify custom parameters in output metadata
+((ntest++))
+if grep -q '"k":15' "${TMPDIR}/output_k15_m7.fasta" && \
+   grep -q '"m":7' "${TMPDIR}/output_k15_m7.fasta"
+then
+    log "$MCMD: custom parameters correctly set in metadata OK"
+    ((success++))
+else
+    log "$MCMD: custom parameters not in metadata - failed"
+    ((failed++))
+fi
+
+# Test 7: Test with different output format (FASTA output explicitly)
+((ntest++))
+if obisuperkmer --fasta-output -k 21 -m 11 \
+    "${TEST_DIR}/test_sequences.fasta" \
+    > "${TMPDIR}/output_fasta.fasta" 2>&1
+then
+    log "$MCMD: FASTA output format OK"
+    ((success++))
+else
+    log "$MCMD: FASTA output format failed"
+    ((failed++))
+fi
+
+# Test 8: Verify all super k-mers have superkmer in their ID
+((ntest++))
+if grep "^>" "${TMPDIR}/output_default.fasta" | grep -q "superkmer"
+then
+    log "$MCMD: super k-mer IDs contain 'superkmer' OK"
+    ((success++))
+else
+    log "$MCMD: super k-mer IDs missing 'superkmer' - failed"
+    ((failed++))
+fi
+
+# Test 9: Verify parent sequence IDs are preserved
+((ntest++))
+if grep -q "seq1" "${TMPDIR}/output_default.fasta" && \
+   grep -q "seq2" "${TMPDIR}/output_default.fasta" && \
+   grep -q "seq3" "${TMPDIR}/output_default.fasta"
+then
+    log "$MCMD: parent sequence IDs preserved OK"
+    ((success++))
+else
+    log "$MCMD: parent sequence IDs not preserved - failed"
+    ((failed++))
+fi
+
+# Test 10: Test with output file option
+((ntest++))
+if obisuperkmer -o "${TMPDIR}/output_file.fasta" \
+    "${TEST_DIR}/test_sequences.fasta" 2>&1
+then
+    log "$MCMD: output to file with -o option OK"
+    ((success++))
+else
+    log "$MCMD: output to file with -o option failed"
+    ((failed++))
+fi
+
+# Test 11: Verify output file was created with -o option
+((ntest++))
+if [ -s "${TMPDIR}/output_file.fasta" ]
+then
+    log "$MCMD: output file created with -o option OK"
+    ((success++))
+else
+    log "$MCMD: output file not created with -o option - failed"
+    ((failed++))
+fi
+
+# Test 12: Verify each super k-mer length is >= k (default k=31)
+((ntest++))
+min_len=$(grep -v "^>" "${TMPDIR}/output_default.fasta" | awk '{print length}' | sort -n | head -1)
+
+if [ "$min_len" -ge 31 ]
+then
+    log "$MCMD: all super k-mers have length >= k OK"
+    ((success++))
+else
+    log "$MCMD: some super k-mers shorter than k ($min_len < 31) - failed"
+    ((failed++))
+fi
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obisuperkmer/test_sequences.fasta

@@ -0,0 +1,6 @@
+>seq1
+ACGTACGTACGTACGTACGTACGTACGTACGT
+>seq2
+AAAACCCCGGGGTTTTAAAACCCCGGGGTTTT
+>seq3
+ATCGATCGATCGATCGATCGATCGATCGATCG

obitests/obitools/obitag/test.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obitag
+CMD=obitag
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1 
+then
+    log "$MCMD: printing help OK" 
+    ((success++))
+else
+    log "$MCMD: printing help failed" 
+    ((failed++))
+fi
+
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obitagpcr/test.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obipcr
+CMD=obipcr
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1 
+then
+    log "$MCMD: printing help OK" 
+    ((success++))
+else
+    log "$MCMD: printing help failed" 
+    ((failed++))
+fi
+
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obitaxonomy/test.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obitaxonomy
+CMD=obitaxonomy
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1 
+then
+    log "$MCMD: printing help OK" 
+    ((success++))
+else
+    log "$MCMD: printing help failed" 
+    ((failed++))
+fi
+
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obiuniq/test.sh

@@ -0,0 +1,258 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obiuniq
+CMD=obiuniq
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed"
+        log
+        log
+       exit 1
+    fi
+
+    log
+    log
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..."
+log "Test directory is $TEST_DIR"
+log "obitools directory is $OBITOOLS_DIR"
+log "Temporary directory is $TMPDIR"
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+
+
+((ntest++))
+if $CMD -h > "${TMPDIR}/help.txt" 2>&1
+then
+    log "$MCMD: printing help OK"
+    ((success++))
+else
+    log "$MCMD: printing help failed"
+    ((failed++))
+fi
+
+((ntest++))
+if obiuniq "${TEST_DIR}/touniq.fasta" \
+    > "${TMPDIR}/touniq_u.fasta"
+then
+    log "OBIUniq simple: running OK"
+    ((success++))
+else
+    log "OBIUniq simple: running failed"
+    ((failed++))
+fi
+
+obicsv -s --auto ${TEST_DIR}/touniq_u.fasta \
+| tail -n +2 \
+| sort \
+> "${TMPDIR}/touniq_u_ref.csv"
+
+obicsv -s --auto ${TMPDIR}/touniq_u.fasta \
+| tail -n +2 \
+| sort \
+> "${TMPDIR}/touniq_u.csv"
+
+((ntest++))
+if diff "${TMPDIR}/touniq_u_ref.csv" \
+        "${TMPDIR}/touniq_u.csv"  > /dev/null
+then
+    log "OBIUniq simple: result OK"
+    ((success++))
+else
+    log "OBIUniq simple: result failed"
+    ((failed++))
+fi
+
+((ntest++))
+if obiuniq -c a "${TEST_DIR}/touniq.fasta" \
+    > "${TMPDIR}/touniq_u_a.fasta"
+then
+    log "OBIUniq one category: running OK"
+    ((success++))
+else
+    log "OBIUniq one category: running failed"
+    ((failed++))
+fi
+
+obicsv -s --auto ${TEST_DIR}/touniq_u_a.fasta \
+| tail -n +2 \
+| sort \
+> "${TMPDIR}/touniq_u_a_ref.csv"
+
+obicsv -s --auto ${TMPDIR}/touniq_u_a.fasta \
+| tail -n +2 \
+| sort \
+> "${TMPDIR}/touniq_u_a.csv"
+
+
+((ntest++))
+if diff "${TMPDIR}/touniq_u_a_ref.csv" \
+        "${TMPDIR}/touniq_u_a.csv"  > /dev/null
+then
+    log "OBIUniq one category: result OK"
+    ((success++))
+else
+    log "OBIUniq one category: result failed"
+    ((failed++))
+fi
+
+((ntest++))
+if obiuniq -c a -c b "${TEST_DIR}/touniq.fasta" \
+    > "${TMPDIR}/touniq_u_a_b.fasta"
+then
+    log "OBIUniq two categories: running OK"
+    ((success++))
+else
+    log "OBIUniq two categories: running failed"
+    ((failed++))
+fi
+
+obicsv -s --auto ${TEST_DIR}/touniq_u_a_b.fasta \
+| tail -n +2 \
+| sort \
+> "${TMPDIR}/touniq_u_a_b_ref.csv"
+
+obicsv -s --auto ${TMPDIR}/touniq_u_a_b.fasta \
+| tail -n +2 \
+| sort \
+> "${TMPDIR}/touniq_u_a_b.csv"
+
+((ntest++))
+if diff "${TMPDIR}/touniq_u_a_b_ref.csv" \
+        "${TMPDIR}/touniq_u_a_b.csv"  > /dev/null
+then
+    log "OBIUniq two categories: result OK"
+    ((success++))
+else
+    log "OBIUniq two categories: result failed"
+    ((failed++))
+fi
+
+##
+## Test merge attributes consistency between in-memory and on-disk paths
+## This test catches the bug where the shared classifier in the on-disk
+## dereplication path caused incorrect merged attributes.
+##
+
+((ntest++))
+if obiuniq -m a -m b --in-memory \
+    "${TEST_DIR}/touniq.fasta" \
+    > "${TMPDIR}/touniq_u_merge_mem.fasta" 2>/dev/null
+then
+    log "OBIUniq merge in-memory: running OK"
+    ((success++))
+else
+    log "OBIUniq merge in-memory: running failed"
+    ((failed++))
+fi
+
+((ntest++))
+if obiuniq -m a -m b --chunk-count 4 \
+    "${TEST_DIR}/touniq.fasta" \
+    > "${TMPDIR}/touniq_u_merge_disk.fasta" 2>/dev/null
+then
+    log "OBIUniq merge on-disk: running OK"
+    ((success++))
+else
+    log "OBIUniq merge on-disk: running failed"
+    ((failed++))
+fi
+
+# Extract sorted annotations (JSON attributes) from both outputs
+# to compare merge results independently of sequence ordering
+grep '^>' "${TMPDIR}/touniq_u_merge_mem.fasta" \
+| sed 's/^>seq[0-9]* //' \
+| sort \
+> "${TMPDIR}/touniq_u_merge_mem.json"
+
+grep '^>' "${TMPDIR}/touniq_u_merge_disk.fasta" \
+| sed 's/^>seq[0-9]* //' \
+| sort \
+> "${TMPDIR}/touniq_u_merge_disk.json"
+
+((ntest++))
+if diff "${TMPDIR}/touniq_u_merge_mem.json" \
+        "${TMPDIR}/touniq_u_merge_disk.json" > /dev/null
+then
+    log "OBIUniq merge on-disk vs in-memory: result OK"
+    ((success++))
+else
+    log "OBIUniq merge on-disk vs in-memory: result failed"
+    ((failed++))
+fi
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obiuniq/touniq.fasta

@@ -0,0 +1,16 @@
+>seq1 {"a":2, "b":4,"c":5}
+aaacccgggttt
+>seq2 {"a":3, "b":4,"c":5}
+aaacccgggttt
+>seq3 {"a":3, "b":5,"c":5}
+aaacccgggttt
+>seq4 {"a":3, "b":5,"c":6}
+aaacccgggttt
+>seq5 {"a":2, "b":4,"c":5}
+aaacccgggtttca
+>seq6 {"a":3, "b":4,"c":5}
+aaacccgggtttca
+>seq7 {"a":3, "b":5,"c":5}
+aaacccgggtttca
+>seq8 {"a":3, "b":5,"c":6}
+aaacccgggtttca

obitests/obitools/obiuniq/touniq_u.fasta

@@ -0,0 +1,4 @@
+>seq5 {"count":4}
+aaacccgggtttca
+>seq1 {"count":4}
+aaacccgggttt

obitests/obitools/obiuniq/touniq_u_a.fasta

@@ -0,0 +1,8 @@
+>seq5 {"a":2,"b":4,"c":5,"count":1}
+aaacccgggtttca
+>seq6 {"a":3,"count":3}
+aaacccgggtttca
+>seq1 {"a":2,"b":4,"c":5,"count":1}
+aaacccgggttt
+>seq2 {"a":3,"count":3}
+aaacccgggttt

obitests/obitools/obiuniq/touniq_u_a_b.fasta

@@ -0,0 +1,12 @@
+>seq5 {"a":2,"b":4,"c":5,"count":1}
+aaacccgggtttca
+>seq6 {"a":3,"b":4,"c":5,"count":1}
+aaacccgggtttca
+>seq7 {"a":3,"b":5,"count":2}
+aaacccgggtttca
+>seq1 {"a":2,"b":4,"c":5,"count":1}
+aaacccgggttt
+>seq2 {"a":3,"b":4,"c":5,"count":1}
+aaacccgggttt
+>seq3 {"a":3,"b":5,"count":2}
+aaacccgggttt

pkg/obialign/alignment.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
 
 // // A pool of byte slices.
@@ -158,12 +159,30 @@ func BuildQualityConsensus(seqA, seqB *obiseq.BioSequence, path []int, statOnMis
 
 	match := 0
 
+	left := obiutils.Abs(path[0])
+	right := 0
+	if path[len(path)-1] == 0 {
+		right = path[len(path)-2]
+	}
+
+	right = obiutils.Abs(right)
+
+	right = len(*bufferQA) - right
+
+	// obilog.Warnf("BuildQualityConsensus: left = %d right = %d\n", left, right)
+
 	for i, qA = range *bufferQA {
 		nA := (*bufferSA)[i]
 		nB := (*bufferSB)[i]
 		qB = (*bufferQB)[i]
 
-		if statOnMismatch && nA != nB && nA != ' ' && nB != ' ' {
+		if statOnMismatch && i >= left && i < right && nA != nB {
+			if nA == ' ' {
+				nA = '-'
+			}
+			if nB == ' ' {
+				nB = '-'
+			}
 			mismatches[strings.ToUpper(fmt.Sprintf("(%c:%02d)->(%c:%02d)", nA, qA, nB, qB))] = i + 1
 		}
 
@@ -183,13 +202,12 @@ func BuildQualityConsensus(seqA, seqB *obiseq.BioSequence, path []int, statOnMis
 
 		q := qA + qB
 
-		if qA > 0 && qB > 0 {
+		if nA != nB {
-			if nA != nB {
+			q = qM - byte(math.Log10(1-math.Pow(10, -float64(qm)/40))*10+0.5)
-				q = qM - byte(math.Log10(1-math.Pow(10, -float64(qm)/30))*10+0.5)
+		}
-			}
+
-			if nA == nB {
+		if nA == nB {
-				match++
+			match++
-			}
 		}
 
 		if q > 90 {

pkg/obialign/backtracking.go

@@ -12,7 +12,7 @@ func _Backtracking(pathMatrix []int, lseqA, lseqB int, path *[]int) []int {
 	cp := cap(*path)
 	(*path) = slices.Grow((*path), needed)
 	if cp < cap(*path) {
-		log.Infof("Resized path from %d to %d\n", cp, cap(*path))
+		log.Debugf("Resized path from %d to %d\n", cp, cap(*path))
 	}
 	p := cap(*path)
 	*path = (*path)[:p]