obitools/obitools4
2
0
Fork 0
mirror of https://github.com/metabarcoding/obitools4.git synced 2026-03-25 21:40:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Compare commits

base: obitools:Release_4.4.23
Branches Tags
obitools:master obitools:push-wpwqvsqsrpwz obitools:push-yxknulsuqnvz obitools:push-pvunnvzrzslw obitools:push-kzmrqmplznrn obitools:push-snmwxkwkqxrm obitools:push-mvpuxnxoyypu obitools:push-uotrstkymowq obitools:push-uzpqqoqvpnxw obitools:V4.3 obitools:microasm obitools:taxonomy obitools:blackboard
obitools:Release_4.4.29 obitools:Release_4.4.28 obitools:Release_4.4.27 obitools:Release_4.4.26 obitools:Release_4.4.25 obitools:Release_4.4.24 obitools:Release_4.4.23 obitools:Release_4.4.22 obitools:Release_4.4.21 obitools:Release_4.4.20 obitools:Release_4.4.19 obitools:Release_4.4.18 obitools:Release_4.4.17 obitools:Release_4.4.16 obitools:Release_4.4.15 obitools:Release_4.4.14 obitools:Release_4.4.13 obitools:Release_4.4.12 obitools:Release_4.4.11 obitools:Release_4.4.10 obitools:Release_4.4.9 obitools:Release_4.4.8 obitools:Release_4.4.7 obitools:Release_4.4.6 obitools:Release_4.4.5 obitools:Release_4.4.0 obitools:Release_4.3.0
..
compare: obitools:microasm
Branches Tags
obitools:master obitools:push-wpwqvsqsrpwz obitools:push-yxknulsuqnvz obitools:push-pvunnvzrzslw obitools:push-kzmrqmplznrn obitools:push-snmwxkwkqxrm obitools:push-mvpuxnxoyypu obitools:push-uotrstkymowq obitools:push-uzpqqoqvpnxw obitools:V4.3 obitools:microasm obitools:taxonomy obitools:blackboard
obitools:Release_4.4.29 obitools:Release_4.4.28 obitools:Release_4.4.27 obitools:Release_4.4.26 obitools:Release_4.4.25 obitools:Release_4.4.24 obitools:Release_4.4.23 obitools:Release_4.4.22 obitools:Release_4.4.21 obitools:Release_4.4.20 obitools:Release_4.4.19 obitools:Release_4.4.18 obitools:Release_4.4.17 obitools:Release_4.4.16 obitools:Release_4.4.15 obitools:Release_4.4.14 obitools:Release_4.4.13 obitools:Release_4.4.12 obitools:Release_4.4.11 obitools:Release_4.4.10 obitools:Release_4.4.9 obitools:Release_4.4.8 obitools:Release_4.4.7 obitools:Release_4.4.6 obitools:Release_4.4.5 obitools:Release_4.4.0 obitools:Release_4.3.0

1 Commits
Release_4. ... microasm

Author SHA1 Message Date
Eric Coissac
d70b0a5b42 A first version of obimicroasm... 2025-02-16 21:47:49 +01:00
281 changed files with 3100 additions and 23234 deletions
Expand all files Collapse all files

19
.github/workflows/obitest.yml vendored
View File

@@ -1,19 +0,0 @@
name: "Run the obitools command test suite"
on:
push:
branches:
- master
- V*
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Setup Go
uses: actions/setup-go@v2
with:
go-version: '1.23'
- name: Checkout obitools4 project
uses: actions/checkout@v4
- name: Run tests
run: make githubtests

183
.github/workflows/release.yml vendored
View File

@@ -1,183 +0,0 @@
name: Create Release on Tag
on:
push:
tags:
- "Release_*"
permissions:
contents: write
jobs:
# First run tests
test:
runs-on: ubuntu-latest
steps:
- name: Setup Go
uses: actions/setup-go@v5
with:
go-version: "1.23"
- name: Checkout obitools4 project
uses: actions/checkout@v4
- name: Run tests
run: make githubtests
# Build binaries for each platform
build:
needs: test
strategy:
matrix:
include:
- os: ubuntu-latest
goos: linux
goarch: amd64
output_name: linux_amd64
- os: ubuntu-24.04-arm
goos: linux
goarch: arm64
output_name: linux_arm64
- os: macos-15-intel
goos: darwin
goarch: amd64
output_name: darwin_amd64
- os: macos-latest
goos: darwin
goarch: arm64
output_name: darwin_arm64
runs-on: ${{ matrix.os }}
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Setup Go
uses: actions/setup-go@v5
with:
go-version: "1.23"
- name: Extract version from tag
id: get_version
run: |
TAG=${GITHUB_REF#refs/tags/Release_}
echo "version=$TAG" >> $GITHUB_OUTPUT
- name: Install build tools (Linux)
if: runner.os == 'Linux'
run: |
sudo apt-get update -q
sudo apt-get install -y musl-tools
- name: Install build tools (macOS)
if: runner.os == 'macOS'
run: |
# Ensure Xcode Command Line Tools are installed
xcode-select --install 2>/dev/null || true
xcode-select -p
- name: Build binaries
env:
GOOS: ${{ matrix.goos }}
GOARCH: ${{ matrix.goarch }}
VERSION: ${{ steps.get_version.outputs.version }}
CC: ${{ matrix.goos == 'linux' && 'musl-gcc' || '' }}
run: |
if [ "$GOOS" = "linux" ]; then
make LDFLAGS='-linkmode=external -extldflags=-static' obitools
else
make obitools
fi
mkdir -p artifacts
# Create a single tar.gz with all binaries for this platform
tar -czf artifacts/obitools4_${VERSION}_${{ matrix.output_name }}.tar.gz -C build .
- name: Upload artifacts
uses: actions/upload-artifact@v4
with:
name: binaries-${{ matrix.output_name }}
path: artifacts/*
# Create the release
create-release:
needs: build
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Extract version from tag
id: get_version
run: |
TAG=${GITHUB_REF#refs/tags/Release_}
echo "version=$TAG" >> $GITHUB_OUTPUT
- name: Download all artifacts
uses: actions/download-artifact@v4
with:
path: release-artifacts
- name: Prepare release directory
run: |
mkdir -p release
find release-artifacts -type f -name "*.tar.gz" -exec cp {} release/ \;
ls -lh release/
- name: Generate Release Notes
env:
VERSION: ${{ steps.get_version.outputs.version }}
run: |
PREV_TAG=$(git describe --tags --abbrev=0 HEAD^ 2>/dev/null || echo "")
echo "# OBITools4 Release ${VERSION}" > release_notes.md
echo "" >> release_notes.md
if [ -n "$PREV_TAG" ]; then
echo "## Changes since ${PREV_TAG}" >> release_notes.md
echo "" >> release_notes.md
git log ${PREV_TAG}..HEAD --pretty=format:"- %s" >> release_notes.md
else
echo "## Changes" >> release_notes.md
echo "" >> release_notes.md
git log --pretty=format:"- %s" -n 20 >> release_notes.md
fi
echo "" >> release_notes.md
echo "" >> release_notes.md
echo "## Installation" >> release_notes.md
echo "" >> release_notes.md
echo "Download the appropriate archive for your system and extract it:" >> release_notes.md
echo "" >> release_notes.md
echo "### Linux (AMD64)" >> release_notes.md
echo '```bash' >> release_notes.md
echo "tar -xzf obitools4_${VERSION}_linux_amd64.tar.gz" >> release_notes.md
echo '```' >> release_notes.md
echo "" >> release_notes.md
echo "### Linux (ARM64)" >> release_notes.md
echo '```bash' >> release_notes.md
echo "tar -xzf obitools4_${VERSION}_linux_arm64.tar.gz" >> release_notes.md
echo '```' >> release_notes.md
echo "" >> release_notes.md
echo "### macOS (Intel)" >> release_notes.md
echo '```bash' >> release_notes.md
echo "tar -xzf obitools4_${VERSION}_darwin_amd64.tar.gz" >> release_notes.md
echo '```' >> release_notes.md
echo "" >> release_notes.md
echo "### macOS (Apple Silicon)" >> release_notes.md
echo '```bash' >> release_notes.md
echo "tar -xzf obitools4_${VERSION}_darwin_arm64.tar.gz" >> release_notes.md
echo '```' >> release_notes.md
echo "" >> release_notes.md
echo "All OBITools4 binaries are included in each archive." >> release_notes.md
- name: Create GitHub Release
uses: softprops/action-gh-release@v1
with:
name: Release ${{ steps.get_version.outputs.version }}
body_path: release_notes.md
files: release/*
draft: false
prerelease: false
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

164
.gitignore vendored
View File

@@ -1,37 +1,135 @@
**/cpu.pprof
**/cpu.trace
**/test
**/bin
**/vendor
**/*.fastq
**/*.fasta
**/*.fastq.gz
**/*.fasta.gz
**/.DS_Store
**/*.gml
**/*.log
**/xxx*
**/*.sav
**/*.old
**/*.tgz
**/*.yaml
**/*.csv
**/*.pb.gz
xx
cpu.pprof
cpu.trace
test
bin
vendor
*.fastq
*.fasta
*.fastq.gz
*.fasta.gz
.DS_Store
*.gml
*.log
/argaly
.rhistory
/.vscode
/obiconvert
/obicount
/obimultiplex
/obipairing
/obipcr
/obifind
/obidistribute
/obiuniq
/build
/bugs
/Makefile.old
.Rproj.user
obitools.Rproj
Stat_error.knit.md
.Rhistory
Stat_error.nb.html
Stat_error.Rmd
/ncbitaxo
/.luarc.json
/doc/TAXO/
/doc/results/
/doc/_main.log
/doc/_book/_main.tex
/doc/_freeze/
/doc/tutorial_files/
/doc/wolf_data/
/taxdump/
/.vscode/
!/obitests/**
!/sample/**
LLM/**
*_files
entropy.html
bug_id.txt
obilowmask_ref
test_*
/Algo-Alignement.numbers
/Estimate_proba_true_seq.html
/Estimate_proba_true_seq.nb.html
/Estimate_proba_true_seq.Rmd
/modele_error_euka.qmd
/obitools.code-workspace
.DS_Store
.RData
x
xxx
y
/doc/wolf_diet.tgz
/doc/man/depends
/sample/wolf_R1.fasta.gz
/sample/wolf_R2.fasta.gz
/sample/euka03.ecotag.fasta.gz
/sample/ratio.csv
/sample/STD_PLN_1.dat
/sample/STD_PLN_2.dat
/sample/subset_Pasvik_R1.fastq.gz
/sample/subset_Pasvik_R2.fastq.gz
/sample/test_gobitools.fasta.bz2
euka03.csv*
gbbct793.seq.gz
gbinv1003.seq.gz
gbpln210.seq
/doc/book/OBITools-V4.aux
/doc/book/OBITools-V4.fdb_latexmk
/doc/book/OBITools-V4.fls
/doc/book/OBITools-V4.log
/doc/book/OBITools-V4.pdf
/doc/book/OBITools-V4.synctex.gz
/doc/book/OBITools-V4.tex
/doc/book/OBITools-V4.toc
getoptions.adoc
Archive.zip
.DS_Store
sample/.DS_Store
sample/consensus_graphs/specimen_hac_plants_Vern_disicolor_.gml
93954
Bact03.e5.gb_R254.obipcr.idx.fasta.save
sample/test.obipcr.log
Bact02.e3.gb_R254.obipcr.fasta.gz
Example_Arth03.ngsfilter
SPER01.csv
SPER03.csv
wolf_diet_ngsfilter.txt
xx
xxx.gb
yyy_geom.csv
yyy_LCS.csv
yyy.json
bug_obimultiplex/toto
bug_obimultiplex/toto_mapping
bug_obimultiplex/tutu
bug_obimultiplex/tutu_mapping
bug_obipairing/GIT1_GH_ngsfilter.txt
doc/book/TAXO/citations.dmp
doc/book/TAXO/delnodes.dmp
doc/book/TAXO/division.dmp
doc/book/TAXO/gc.prt
doc/book/TAXO/gencode.dmp
doc/book/TAXO/merged.dmp
doc/book/TAXO/names.dmp
doc/book/TAXO/nodes.dmp
doc/book/TAXO/readme.txt
doc/book/wolf_data/Release-253/ncbitaxo/citations.dmp
doc/book/wolf_data/Release-253/ncbitaxo/delnodes.dmp
doc/book/wolf_data/Release-253/ncbitaxo/division.dmp
doc/book/wolf_data/Release-253/ncbitaxo/gc.prt
doc/book/wolf_data/Release-253/ncbitaxo/gencode.dmp
doc/book/wolf_data/Release-253/ncbitaxo/merged.dmp
doc/book/wolf_data/Release-253/ncbitaxo/names.dmp
doc/book/wolf_data/Release-253/ncbitaxo/nodes.dmp
doc/book/wolf_data/Release-253/ncbitaxo/readme.txt
doc/book/results/toto.tasta
sample/.DS_Store
GO
ncbitaxo/citations.dmp
ncbitaxo/delnodes.dmp
ncbitaxo/division.dmp
ncbitaxo/gc.prt
ncbitaxo/gencode.dmp
ncbitaxo/merged.dmp
ncbitaxo/names.dmp
ncbitaxo/nodes.dmp
ncbitaxo/readme.txt
template.16S
xxx.gz
*.sav
*.old
ncbitaxo.tgz
*.csv

189
Makefile
View File

@@ -2,17 +2,8 @@
#export GOBIN=$(GOPATH)/bin
#export PATH=$(GOBIN):$(shell echo $${PATH})
.DEFAULT_GOAL := all
GREEN := \033[0;32m
YELLOW := \033[0;33m
BLUE := \033[0;34m
NC := \033[0m
GOFLAGS=
LDFLAGS=
GOCMD=go
GOBUILD=$(GOCMD) build $(GOFLAGS) $(if $(LDFLAGS),-ldflags="$(LDFLAGS)")
GOBUILD=$(GOCMD) build # -compiler gccgo -gccgoflags -O3
GOGENERATE=$(GOCMD) generate
GOCLEAN=$(GOCMD) clean
GOTEST=$(GOCMD) test
@@ -25,12 +16,6 @@ PACKAGES_SRC:= $(wildcard pkg/*/*.go pkg/*/*/*.go)
PACKAGE_DIRS:=$(sort $(patsubst %/,%,$(dir $(PACKAGES_SRC))))
PACKAGES:=$(notdir $(PACKAGE_DIRS))
GITHOOK_SRC_DIR=git-hooks
GITHOOKS_SRC:=$(wildcard $(GITHOOK_SRC_DIR)/*)
GITHOOK_DIR=.git/hooks
GITHOOKS:=$(patsubst $(GITHOOK_SRC_DIR)/%,$(GITHOOK_DIR)/%,$(GITHOOKS_SRC))
OBITOOLS_SRC:= $(wildcard cmd/obitools/*/*.go)
OBITOOLS_DIRS:=$(sort $(patsubst %/,%,$(dir $(OBITOOLS_SRC))))
OBITOOLS:=$(notdir $(OBITOOLS_DIRS))
@@ -51,7 +36,7 @@ $(OBITOOLS_PREFIX)$(notdir $(1)): $(BUILD_DIR) $(1) pkg/obioptions/version.go
@echo -n - Building obitool $(notdir $(1))...
@$(GOBUILD) -o $(BUILD_DIR)/$(OBITOOLS_PREFIX)$(notdir $(1)) ./$(1) \
2> $(OBITOOLS_PREFIX)$(notdir $(1)).log \
|| { cat $(OBITOOLS_PREFIX)$(notdir $(1)).log; rm -f $(OBITOOLS_PREFIX)$(notdir $(1)).log; exit 1; }
|| cat $(OBITOOLS_PREFIX)$(notdir $(1)).log
@rm -f $(OBITOOLS_PREFIX)$(notdir $(1)).log
@echo Done.
endef
@@ -68,53 +53,27 @@ endif
OUTPUT:=$(shell mktemp)
help:
@printf "$(GREEN)OBITools4 Makefile$(NC)\n\n"
@printf "$(BLUE)Main targets:$(NC)\n"
@printf " %-20s %s\n" "all" "Build all obitools (default)"
@printf " %-20s %s\n" "obitools" "Build all obitools binaries to build/"
@printf " %-20s %s\n" "test" "Run Go unit tests"
@printf " %-20s %s\n" "obitests" "Run integration tests (obitests/)"
@printf " %-20s %s\n" "bump-version" "Increment patch version (or set with VERSION=x.y.z)"
@printf " %-20s %s\n" "update-deps" "Update all Go dependencies"
@printf "\n$(BLUE)Jujutsu workflow:$(NC)\n"
@printf " %-20s %s\n" "jjnew" "Document current commit and start a new one"
@printf " %-20s %s\n" "jjpush" "Release: describe, bump, generate notes, push PR, tag (VERSION=x.y.z optional)"
@printf " %-20s %s\n" "jjfetch" "Fetch latest commits from origin"
@printf "\n$(BLUE)Required tools:$(NC)\n"
@printf " %-20s " "go"; command -v go >/dev/null 2>&1 && printf "$(GREEN)✓$(NC) %s\n" "$$(go version)" || printf "$(YELLOW)✗ not found$(NC)\n"
@printf " %-20s " "git"; command -v git >/dev/null 2>&1 && printf "$(GREEN)✓$(NC) %s\n" "$$(git --version)" || printf "$(YELLOW)✗ not found$(NC)\n"
@printf " %-20s " "jj"; command -v jj >/dev/null 2>&1 && printf "$(GREEN)✓$(NC) %s\n" "$$(jj --version)" || printf "$(YELLOW)✗ not found$(NC)\n"
@printf " %-20s " "gh"; command -v gh >/dev/null 2>&1 && printf "$(GREEN)✓$(NC) %s\n" "$$(gh --version | head -1)" || printf "$(YELLOW)✗ not found$(NC) (brew install gh)\n"
@printf "\n$(BLUE)Optional tools (release notes generation):$(NC)\n"
@printf " %-20s " "aichat"; command -v aichat >/dev/null 2>&1 && printf "$(GREEN)✓$(NC) %s\n" "$$(aichat --version)" || printf "$(YELLOW)✗ not found$(NC) (https://github.com/sigoden/aichat)\n"
@printf " %-20s " "jq"; command -v jq >/dev/null 2>&1 && printf "$(GREEN)✓$(NC) %s\n" "$$(jq --version)" || printf "$(YELLOW)✗ not found$(NC) (brew install jq)\n"
all: install-githook obitools
all: obitools
packages: $(patsubst %,pkg-%,$(PACKAGES))
obitools: $(patsubst %,$(OBITOOLS_PREFIX)%,$(OBITOOLS))
install-githook: $(GITHOOKS)
$(GITHOOK_DIR)/%: $(GITHOOK_SRC_DIR)/%
@echo installing $$(basename $@)...
@mkdir -p $(GITHOOK_DIR)
@cp $< $@
@chmod +x $@
update-deps:
go get -u ./...
test: .FORCE
test:
$(GOTEST) ./...
obitests:
@for t in $$(find obitests -name test.sh -print) ; do \
bash $${t} || exit 1;\
done
man:
make -C doc man
obibook:
make -C doc obibook
doc: man obibook
githubtests: obitools obitests
macos-pkg:
@bash pkgs/macos/macos-installer-builder-master/macOS-x64/build-macos-x64.sh \
OBITools \
0.0.1
$(BUILD_DIR):
mkdir -p $@
@@ -124,121 +83,19 @@ $(foreach P,$(PACKAGE_DIRS),$(eval $(call MAKE_PKG_RULE,$(P))))
$(foreach P,$(OBITOOLS_DIRS),$(eval $(call MAKE_OBITOOLS_RULE,$(P))))
pkg/obioptions/version.go: version.txt .FORCE
@version=$$(cat version.txt); \
cat $@ \
| sed -E 's/^var _Version = "[^"]*"/var _Version = "Release '$$version'"/' \
pkg/obioptions/version.go: .FORCE
ifneq ($(strip $(COMMIT_ID)),)
@cat $@ \
| sed -E 's/^var _Commit = "[^"]*"/var _Commit = "'$(COMMIT_ID)'"/' \
| sed -E 's/^var _Version = "[^"]*"/var _Version = "'"$(LAST_TAG)"'"/' \
> $(OUTPUT)
@diff $@ $(OUTPUT) 2>&1 > /dev/null \
|| (echo "Update version.go to $$(cat version.txt)" && mv $(OUTPUT) $@)
|| echo "Update version.go : $@ to $(LAST_TAG) ($(COMMIT_ID))" \
&& mv $(OUTPUT) $@
@rm -f $(OUTPUT)
endif
bump-version:
@current=$$(cat version.txt); \
if [ -n "$(VERSION)" ]; then \
new_version="$(VERSION)"; \
echo "Setting version to $$new_version (was $$current)"; \
else \
echo "Incrementing version..."; \
echo " Current version: $$current"; \
major=$$(echo $$current | cut -d. -f1); \
minor=$$(echo $$current | cut -d. -f2); \
patch=$$(echo $$current | cut -d. -f3); \
new_patch=$$((patch + 1)); \
new_version="$$major.$$minor.$$new_patch"; \
echo " New version: $$new_version"; \
fi; \
echo "$$new_version" > version.txt
@echo "✓ Version updated in version.txt"
@$(MAKE) pkg/obioptions/version.go
jjnew:
@echo "$(YELLOW)→ Creating a new commit...$(NC)"
@echo "$(BLUE)→ Documenting current commit...$(NC)"
@jj auto-describe
@echo "$(BLUE)→ Done.$(NC)"
@jj new
@echo "$(GREEN)✓ New commit created$(NC)"
jjpush:
@$(MAKE) jjpush-describe
@$(MAKE) jjpush-bump
@$(MAKE) jjpush-notes
@$(MAKE) jjpush-push
@$(MAKE) jjpush-tag
@echo "$(GREEN)✓ Release complete$(NC)"
jjpush-describe:
@echo "$(BLUE)→ Documenting current commit...$(NC)"
@jj auto-describe
jjpush-bump:
@echo "$(BLUE)→ Creating new commit for version bump...$(NC)"
@jj new
@$(MAKE) bump-version
jjpush-notes:
@version=$$(cat version.txt); \
echo "$(BLUE)→ Generating release notes for version $$version...$(NC)"; \
release_title="Release $$version"; \
release_body=""; \
if command -v aichat >/dev/null 2>&1; then \
previous_tag=$$(git describe --tags --abbrev=0 --match 'Release_*' 2>/dev/null); \
if [ -z "$$previous_tag" ]; then \
echo "$(YELLOW)⚠ No previous Release tag found, skipping release notes$(NC)"; \
else \
raw_output=$$(git log --format="%h %B" "$$previous_tag..HEAD" | \
aichat \
"Summarize the following commits into a GitHub release note for version $$version. Ignore commits related to version bumps, .gitignore changes, or any internal housekeeping that is irrelevant to end users. Describe each user-facing change precisely without exposing code. Eliminate redundancy. Output strictly valid JSON with no surrounding text, using this exact schema: {\"title\": \"<short release title>\", \"body\": \"<detailed markdown release notes>\"}" 2>/dev/null) || true; \
if [ -n "$$raw_output" ]; then \
notes=$$(printf '%s\n' "$$raw_output" | python3 tools/json2md.py 2>/dev/null); \
if [ -n "$$notes" ]; then \
release_title=$$(echo "$$notes" | head -1); \
release_body=$$(echo "$$notes" | tail -n +3); \
else \
echo "$(YELLOW)⚠ JSON parsing failed, using default release message$(NC)"; \
fi; \
fi; \
fi; \
fi; \
printf '%s' "$$release_title" > /tmp/obitools4-release-title.txt; \
printf '%s' "$$release_body" > /tmp/obitools4-release-body.txt; \
echo "$(BLUE)→ Setting release notes as commit description...$(NC)"; \
jj desc -m "$$release_title"$$'\n\n'"$$release_body"
jjpush-push:
@echo "$(BLUE)→ Pushing commits...$(NC)"
@jj git push --change @
@echo "$(BLUE)→ Creating/updating PR...$(NC)"
@release_title=$$(cat /tmp/obitools4-release-title.txt 2>/dev/null || echo "Release $$(cat version.txt)"); \
release_body=$$(cat /tmp/obitools4-release-body.txt 2>/dev/null || echo ""); \
branch=$$(jj log -r @ --no-graph -T 'bookmarks.map(|b| b.name()).join("\n")' 2>/dev/null | head -1); \
if [ -n "$$branch" ] && command -v gh >/dev/null 2>&1; then \
gh pr create --title "$$release_title" --body "$$release_body" --base master --head "$$branch" 2>/dev/null \
|| gh pr edit "$$branch" --title "$$release_title" --body "$$release_body" 2>/dev/null \
|| echo "$(YELLOW)⚠ Could not create/update PR$(NC)"; \
fi
jjpush-tag:
@version=$$(cat version.txt); \
tag_name="Release_$$version"; \
release_title=$$(cat /tmp/obitools4-release-title.txt 2>/dev/null || echo "Release $$version"); \
release_body=$$(cat /tmp/obitools4-release-body.txt 2>/dev/null || echo ""); \
install_section=$$'\n## Installation\n\n### Pre-built binaries\n\nDownload the appropriate archive for your system from the\n[release assets](https://github.com/metabarcoding/obitools4/releases/tag/Release_'"$$version"')\nand extract it:\n\n#### Linux (AMD64)\n```bash\ntar -xzf obitools4_'"$$version"'_linux_amd64.tar.gz\n```\n\n#### Linux (ARM64)\n```bash\ntar -xzf obitools4_'"$$version"'_linux_arm64.tar.gz\n```\n\n#### macOS (Intel)\n```bash\ntar -xzf obitools4_'"$$version"'_darwin_amd64.tar.gz\n```\n\n#### macOS (Apple Silicon)\n```bash\ntar -xzf obitools4_'"$$version"'_darwin_arm64.tar.gz\n```\n\nAll OBITools4 binaries are included in each archive.\n\n### From source\n\nYou can also compile and install OBITools4 directly from source using the\ninstallation script:\n\n```bash\ncurl -L https://raw.githubusercontent.com/metabarcoding/obitools4/master/install_obitools.sh | bash -s -- --version '"$$version"'\n```\n\nBy default binaries are installed in `/usr/local/bin`. Use `--install-dir` to\nchange the destination and `--obitools-prefix` to add a prefix to command names:\n\n```bash\ncurl -L https://raw.githubusercontent.com/metabarcoding/obitools4/master/install_obitools.sh | \\\n bash -s -- --version '"$$version"' --install-dir ~/local --obitools-prefix k\n```\n'; \
release_message="$$release_title"$$'\n\n'"$$release_body$$install_section"; \
echo "$(BLUE)→ Creating tag $$tag_name...$(NC)"; \
git tag -a "$$tag_name" -m "$$release_message" 2>/dev/null || echo "$(YELLOW)⚠ Tag $$tag_name already exists$(NC)"; \
echo "$(BLUE)→ Pushing tag $$tag_name...$(NC)"; \
git push origin "$$tag_name" 2>/dev/null || echo "$(YELLOW)⚠ Tag push failed or already pushed$(NC)"; \
rm -f /tmp/obitools4-release-title.txt /tmp/obitools4-release-body.txt
jjfetch:
@echo "$(YELLOW)→ Pulling latest commits...$(NC)"
@jj git fetch
@jj new master@origin
@echo "$(GREEN)✓ Latest commits pulled$(NC)"
.PHONY: all obitools update-deps obitests githubtests help jjnew jjpush jjpush-describe jjpush-bump jjpush-notes jjpush-push jjpush-tag jjfetch bump-version .FORCE
.PHONY: all packages obitools man obibook doc update-deps .FORCE
.FORCE:

38
README.md
View File

@@ -16,17 +16,12 @@ The easiest way to run it is to copy and paste the following command into your t
curl -L https://raw.githubusercontent.com/metabarcoding/obitools4/master/install_obitools.sh | bash
```
By default, the script installs the latest version of *OBITools* commands and other associated files into the `/usr/local` directory.
By default, the script installs the *OBITools* commands and other associated files into the `/usr/local` directory.
The names of the commands in the new *OBITools4* are mostly identical to those in *OBITools2*.
Therefore, installing the new *OBITools* may hide or delete the old ones. If you want both versions to be
available on your system, the installation script offers two options:
### Installation Options
The installation script offers several options:
> -l, --list List all available versions and exit.
>
> -v, --version Install a specific version (e.g., `-v 4.4.3`).
> By default, the latest version is installed.
>
> -i, --install-dir Directory where obitools are installed
> (as example use `/usr/local` not `/usr/local/bin`).
>
@@ -34,36 +29,15 @@ The installation script offers several options:
> want to have several versions of obitools at the
> same time on your system (as example `-p g` will produce
> `gobigrep` command instead of `obigrep`).
>
> -j, --jobs Number of parallel jobs used for compilation
> (default: 1). Increase this value to speed up
> compilation on multi-core systems (e.g., `-j 4`).
### Examples
You can use these options by following the installation command:
List all available versions:
```{bash}
curl -L https://raw.githubusercontent.com/metabarcoding/obitools4/master/install_obitools.sh | bash -s -- --list
```
Install a specific version:
```{bash}
curl -L https://raw.githubusercontent.com/metabarcoding/obitools4/master/install_obitools.sh | bash -s -- --version 4.4.3
```
Install in a custom directory with command prefix:
```{bash}
curl -L https://raw.githubusercontent.com/metabarcoding/obitools4/master/install_obitools.sh | \
bash -s -- --install-dir test_install --obitools-prefix k
```
In this last example, the binaries will be installed in the `test_install` directory and all command names will be prefixed with the letter `k`. Thus, `obigrep` will be named `kobigrep`.
### Note on Version Compatibility
The names of the commands in the new *OBITools4* are mostly identical to those in *OBITools2*.
Therefore, installing the new *OBITools* may hide or delete the old ones. If you want both versions to be
available on your system, use the `--install-dir` and `--obitools-prefix` options as shown above.
In this case, the binaries will be installed in the `test_install` directory and all command names will be prefixed with the letter `k`. Thus `obigrep` will be named `kobigrep`.
## Continuing the analysis...

300
Release-notes.md
View File

@@ -1,85 +1,19 @@
# OBITools release notes
## New changes
### Bug fixes
- In `obipairing` correct the misspelling of the `obiparing_*` tags where the `i`
was missing to `obipairing_`.
- In `obigrep` the **-C** option that excludes sequences too abundant was not
functional.
- In `obitaxonomy` the **-l** option that lists all the taxonomic rank defined by
a taxonomy was not functional
- The file type guesser was not using enough data to be able to correctly detect
file format when sequences were too long in fastq and fasta or when lines were
to long in CSV files. That's now corrected
- Options **--fasta** or **--fastq** usable to specify input format were ignored.
They are now correctly considered
- The `obiannotate` command were crashing when a selection option was used but
no editing option.
- The `--fail-on-taxonomy` led to an error on merged taxa even when the
`--update-taxid` option was used.
- The `--compressed` option was not correctly named. It was renamed to `--compress`
### Enhancement
- Some sequences in the Genbank and EMBL databases are several gigabases long. The
sequence parser had to reallocate and recopy memory many times to read them,
resulting in a complexity of O(N^2) for reading such large sequences.
The new file chunk reader has a linear algorithm that speeds up the reading
of very long sequences.
- A new option **--csv** is added to every obitools to indicate that the input
format is CSV
- The new version of obitools are now printing the taxids in a fancy way
including the scientific name and the taxonomic rank (`"taxon:9606 [Homo
sapiens]@species"`). But if you need the old fashion raw taxid, a new option
**--raw-taxid** has been added to get obitools printing the taxids without any
decorations (`"9606"`).
## March 1st, 2025. Release 4.4.0
A new documentation website is available at https://obitools4.metabarcoding.org.
Its development is still in progress.
The biggest step forward in this new version is taxonomy management. The new
version is now able to handle taxonomic identifiers that are not just integer
values. This is a first step towards an easy way to handle other taxonomy
databases soon, such as the GBIF or Catalog of Life taxonomies. This version
is able to handle files containing taxonomic information created by previous
versions of OBITools, but files created by this new version may have some
problems to be analyzed by previous versions, at least for the taxonomic
information.
## Latest changes
### Breaking changes
- In `obimultiplex`, the short version of the **--tag-list** option used to
specify the list of tags and primers to be used for the demultiplexing has
been changed from `-t` to `-s`.
- In `obimultiplex`, the short version of the **--tag-list** option used to specify the list
of tags and primers to be used for the demultiplexing has been changed from `-t` to `-s`.
- The command `obifind` is now renamed `obitaxonomy`.
- The **--taxdump** option used to specify the path to the taxdump containing
the NCBI taxonomy has been renamed to **--taxonomy**.
- The **--taxdump** option used to specify the path to the taxdump containing the NCBI taxonomy
has been renamed to **--taxonomy**.
### Bug fixes
- Correction of a bug when using paired sequence file with the **--out** option.
- Correction of a bug in `obitag` when trying to annotate very short sequences of
4 bases or less.
- In `obipairing`, correct the stats `seq_a_single` and `seq_b_single` when
on right alignment mode
@@ -87,32 +21,12 @@ information.
the batch size and not reading the qualities from the fastq files as `obiuniq`
is producing only fasta output without qualities.
- In `obitag`, correct the wrong assignment of the **obitag_bestmatch**
attribute.
- In `obiclean`, the **--no-progress-bar** option disables all progress bars,
not just the data.
- Several fixes in reading FASTA and FASTQ files, including some code
simplification and factorization.
- Fixed a bug in all obitools that caused the same file to be processed
multiple times, when specifying a directory name as input.
### New features
- `obigrep` add a new **--valid-taxid** option to keep only sequence with a
valid taxid
- `obiclean` add a new **--min-sample-count** option with a default value of 1,
asking to filter out sequences which are not occurring in at least the
specified number of samples.
- `obitoaxonomy` a new **--dump|D** option allows for dumping a sub-taxonomy.
- Taxonomy dump can now be provided as a four-columns CSV file to the
**--taxonomy** option.
- Taxonomy dump can now be provided as a four-columns CSV file to the **--taxonomy**
option.
- NCBI Taxonomy dump does not need to be uncompressed and unarchived anymore. The
path of the tar and gziped dump file can be directly specified using the
@@ -123,68 +37,54 @@ information.
allow the processing of the rare fasta and fastq files not recognized.
- In `obiscript`, adds new methods to the Lua sequence object:
- `md5_string()`: returning the MD5 check sum as a hexadecimal string,
- `subsequence(from,to)`: allows extracting a subsequence on a 0 based
coordinate system, upper bound excluded like in go.
- `reverse_complement`: returning a sequence object corresponding to the
reverse complement of the current sequence.
- `md5_string()`: returning the MD5 check sum as an hexadecimal string,
- `subsequence(from,to)`: allows to extract a subsequence on a 0 based
coordinate system, upper bound expluded like in go.
- `reverse_complement`: returning a sequence object corresponding to the reverse complement
of the current sequence.
### Enhancement
### Change of git repositiory
- All obitools now have a **--taxonomy** option. If specified, the taxonomy is
loaded first and taxids annotating the sequences are validated against that
taxonomy. A warning is issued for any invalid taxid and for any taxid that
is transferred to a new taxid. The **--update-taxid** option allows these
old taxids to be replaced with their new equivalent in the result of the
obitools command.
- The scoring system used by the `obipairing` command has been changed to be
more coherent. In the new version, the scores associated to a match and a
mismatch involving a nucleotide with a quality score of 0 are equal. Which
is normal as a zero quality score means a perfect indecision on the read
nucleotide, therefore there is no reason to penalize a match differently
from a mismatch (see
https://obitools4.metabarcoding.org/docs/commands/alignments/obipairing/exact-alignment/).
- In every *OBITools* command, the progress bar is automatically deactivated
when the standard error output is redirected.
- Because Genbank and ENA:EMBL contain very large sequences, while OBITools4
are optimized As Genbank and ENA:EMBL contain very large sequences, while
OBITools4 is optimized for short sequences, `obipcr` faces some problems
with excessive consumption of computer resources, especially memory. Several
improvements in the tuning of the default `obipcr` parameters and some new
features, currently only available for FASTA and FASTQ file readers, have
been implemented to limit the memory impact of `obipcr` without changing the
computational efficiency too much.
- Logging system and therefore format, have been homogenized.
## August 2nd, 2024. Release 4.3.0
### Change of git repository
- The OBITools4 git repository has been moved to the GitHub repository.
- The OBITools4 git repository has been moved to the github repository.
The new address is: https://github.com/metabarcoding/obitools4.
Take care for using the new install script for retrieving the new version.
```bash
curl -L https://metabarcoding.org/obitools4/install.sh \
curl -L https://raw.githubusercontent.com/metabarcoding/obitools4/master/install_obitools.sh \
| bash
```
or with options:
```bash
curl -L https://metabarcoding.org/obitools4/install.sh \
curl -L https://raw.githubusercontent.com/metabarcoding/obitools4/master/install_obitools.sh \
| bash -s -- --install-dir test_install --obitools-prefix k
```
### CPU limitation
- By default, *OBITools4* tries to use all the computing power available on
your computer. In some circumstances this can be problematic (e.g. if you
are running on a computer cluster managed by your university). You can limit
the number of CPU cores used by *OBITools4* or by using the **--max-cpu**
option or by setting the **OBIMAXCPU** environment variable. Some strange
behaviour of *OBITools4* has been observed when users try to limit the
maximum number of usable CPU cores to one. This seems to be caused by the Go
language, and it is not obvious to get *OBITools4* to run correctly on a
single core in all circumstances. Therefore, if you ask to use a single
core, **OBITools4** will print a warning message and actually set this
parameter to two cores. If you really want a single core, you can use the
**--force-one-core** option. But be aware that this can lead to incorrect
calculations.
### New features
- The output of the obitools will evolve to produce results only in standard
formats such as fasta and fastq. For non-sequential data, the output will be
in CSV format, with the separator `,`, the decimal separator `.`, and a
header line with the column names. It is more convenient to use the output
in other programs. For example, you can use the `csvtomd` command to
reformat the CSV output into a Markdown table. The first command to initiate
reformat the csv output into a markdown table. The first command to initiate
this change is `obicount`, which now produces a 3-line CSV output.
```bash
@@ -196,7 +96,7 @@ information.
database for `obitag` is to use `obipcr` on a local copy of Genbank or EMBL.
However, these sequence databases are known to contain many taxonomic
errors, such as bacterial sequences annotated with the taxid of their host
species. `obicleandb` tries to detect these errors. To do this, it first keeps
species. obicleandb tries to detect these errors. To do this, it first keeps
only sequences annotated with the taxid to which a species, genus, and
family taxid can be assigned. Then, for each sequence, it compares the
distance of the sequence to the other sequences belonging to the same genus
@@ -207,7 +107,7 @@ information.
with the p-value of the Mann-Whitney U test in the **obicleandb_trusted**
slot. Later, the distribution of this p-value can be analyzed to determine a
threshold. Empirically, a threshold of 0.05 is a good compromise and allows
filtering out less than 1‰ of the sequences. These sequences can then be
to filter out less than 1‰ of the sequences. These sequences can then be
removed using `obigrep`.
- Adds a new `obijoin` utility to join information contained in a sequence
@@ -217,16 +117,16 @@ information.
- Adds a new tool `obidemerge` to demerge a `merge_xxx` slot by recreating the
multiple identical sequences having the slot `xxx` recreated with its initial
value and the sequence count set to the number of occurrences referred in the
value and the sequence count set to the number of occurences refered in the
`merge_xxx` slot. During the operation, the `merge_xxx` slot is removed.
- Adds CSV as one of the input format for every obitools command. To encode
sequence the CSV file must include a column named `sequence` and another
sequence the CSV file must includes a column named `sequence` and another
column named `id`. An extra column named `qualities` can be added to specify
the quality scores of the sequence following the same ASCII encoding than the
the quality scores of the sequence following the same ascii encoding than the
fastq format. All the other columns will be considered as annotations and will
be interpreted as JSON objects encoding potentially for atomic values. If a
column value can not be decoded as JSON it will be considered as a string.
calumn value can not be decoded as JSON it will be considered as a string.
- A new option **--version** has been added to every obitools command. It will
print the version of the command.
@@ -235,8 +135,8 @@ information.
quality scores from a BioSequence object.\
- In `obimultuplex` the ngsfilter file describing the samples can be no provided
not only using the classical ngsfilter format but also using the CSV format.
When using CSV, the first line must contain the column names. 5 columns are
not only using the classical nfsfilter format but also using the csv format.
When using csv, the first line must contain the column names. 5 columns are
expected:
- `experiment` the name of the experiment
@@ -252,34 +152,43 @@ information.
Supplementary columns are allowed. Their names and content will be used to
annotate the sequence corresponding to the sample, as the `key=value;` did
in the ngsfilter format.
in the nfsfilter format.
The CSV format used allows for comment lines starting with `#` character.
Special data lines starting with `@param` in the first column allow configuring the algorithm. The options **--template** provided an over
commented example of the CSV format, including all the possible options.
Special data lines starting with `@param` in the first column allow to
configure the algorithm. The options **--template** provided an over
commented example of the csv format, including all the possible options.
### CPU limitation
### Enhancement
- By default, *OBITools4* tries to use all the computing power available on
your computer. In some circumstances this can be problematic (e.g. if you
are running on a computer cluster managed by your university). You can limit
the number of CPU cores used by *OBITools4* or by using the **--max-cpu**
option or by setting the **OBIMAXCPU** environment variable. Some strange
behavior of *OBITools4* has been observed when users try to limit the
maximum number of usable CPU cores to one. This seems to be caused by the Go
language, and it is not obvious to get *OBITools4* to run correctly on a
single core in all circumstances. Therefore, if you ask to use a single
core, **OBITools4** will print a warning message and actually set this
parameter to two cores. If you really want a single core, you can use the
**--force-one-core** option. But be aware that this can lead to incorrect
calculations.
- In every *OBITools* command, the progress bar are automatically deactivated
when the standard error output is redirected.
- Because Genbank and ENA:EMBL contain very large sequences, while OBITools4
are optimized As Genbank and ENA:EMBL contain very large sequences, while
OBITools4 is optimised for short sequences, `obipcr` faces some problems
with excessive consumption of computer resources, especially memory. Several
improvements in the tuning of the default `obipcr` parameters and some new
features, currently only available for FASTA and FASTQ file readers, have
been implemented to limit the memory impact of `obipcr` without changing the
computational efficiency too much.
- Logging system and therefore format, have been homogenized.
### Bug
- In `obitag`, correct the wrong assignment of the **obitag_bestmatch**
attribute.
- In `obiclean`, the **--no-progress-bar** option disables all progress bars,
not just the data.
- Several fixes in reading FASTA and FASTQ files, including some code
simplification and and factorization.
- Fixed a bug in all obitools that caused the same file to be processed
multiple times. when specifying a directory name as input.
## April 2nd, 2024. Release 4.2.0
### New features
- A new OBITools named `obiscript` allows processing each sequence according
- A new OBITools named `obiscript` allows to process each sequence according
to a Lua script. This is an experimental tool. The **--template** option
allows for generating an example script on the `stdout`.
@@ -287,7 +196,7 @@ information.
- Two of the main class `obiseq.SeqWorker` and `obiseq.SeqWorker` have their
declaration changed. Both now return two values a `obiseq.BioSequenceSlice`
and an `error`. This allows a worker to return potentially several sequences
and an `error`. This allow a worker to return potentially several sequences
as the result of the processing of a single sequence, or zero, which is
equivalent to filter out the input sequence.
@@ -295,12 +204,12 @@ information.
- In `obitag` if the reference database contains sequences annotated by taxid
not referenced in the taxonomy, the corresponding sequences are discarded
from the reference database and a warning indicating the sequence *id* and the
from the reference database and a warning indicating the sequence id and the
wrong taxid is emitted.
- The bug corrected in the parsing of EMBL and Genbank files as implemented in
version 4.1.2 of OBITools4, potentially induced some reduction in the
performance of the parsing. This should have been now fixed.
- In the same idea, parsing of Genbank and EMBL files were reading and storing
- In the same idea, parsing of genbank and EMBL files were reading and storing
in memory not only the sequence but also the annotations (features table).
Up to now none of the OBITools are using this information, but with large
complete genomes, it is occupying a lot of memory. To reduce this impact,
@@ -339,7 +248,7 @@ information.
### New feature
- In `obimatrix` a **--transpose** option allows transposing the produced
- In `obimatrix` a **--transpose** option allows to transpose the produced
matrix table in CSV format.
- In `obitpairing` and `obipcrtag` two new options **--exact-mode** and
**--fast-absolute** to control the heuristic used in the alignment
@@ -347,7 +256,7 @@ information.
the exact algorithm at the cost of a speed. **--fast-absolute** change the
scoring schema of the heuristic.
- In `obiannotate` adds the possibility to annotate the first match of a
pattern using the same algorithm as the one used in `obipcr` and
pattern using the same algorithm than the one used in `obipcr` and
`obimultiplex`. For that four option were added :
- **--pattern** : to specify the pattern. It can use IUPAC codes and
position with no error tolerated has to be followed by a `#` character.
@@ -428,7 +337,7 @@ information.
### Bugs
- In the obitools language, the `composition` function now returns a map
- in the obitools language, the `composition` function now returns a map
indexed by lowercase string "a", "c", "g", "t" and "o" for other instead of
being indexed by the ASCII codes of the corresponding letters.
- Correction of the reverse-complement operation. Every reverse complement of
@@ -441,18 +350,18 @@ information.
duplicating the quality values. This made `obimultiplex` to produce fastq
files with sequences having quality values duplicated.
### Be careful
### Becareful
GO 1.21.0 is out, and it includes new functionalities which are used in the
OBITools4 code. If you use the recommended method for compiling OBITools on your
computer, there is no problem, as the script always load the latest GO version.
If you rely on your personal GO install, please think to update.
OBITools4 code. If you use the recommanded method for compiling OBITools on your
computer, their is no problem, as the script always load the latest GO version.
If you rely on you personnal GO install, please think to update.
## August 29th, 2023. Release 4.0.5
### Bugs
- Patch a bug in the `obiseq.BioSequence` constructor leading to an error on
- Patch a bug in the `obiseq.BioSequence` constructor leading to a error on
almost every obitools. The error message indicates : `fatal error: sync:
unlock of unlocked mutex` This bug was introduced in the release 4.0.4
@@ -471,7 +380,7 @@ If you rely on your personal GO install, please think to update.
data structure to limit the number of alignments actually computed. This
increase a bit the speed of both the software. `obirefidx` is nevertheless
still too slow compared to my expectation.
- Switch to a parallel version of the GZIP library, allowing for high speed
- Switch to a parallel version of the gzip library, allowing for high speed
compress and decompress operation on files.
### New feature
@@ -515,12 +424,12 @@ If you rely on your personal GO install, please think to update.
--unidentified not_assigned.fastq
```
The command produced four files : `tagged_library_R1.fastq` and
the command produced four files : `tagged_library_R1.fastq` and
`tagged_library_R2.fastq` containing the assigned reads and
`not_assigned_R1.fastq` and `not_assigned_R2.fastq` containing the
unassignable reads.
The tagged library files can then be split using `obidistribute`:
the tagged library files can then be split using `obidistribute`:
```{bash}
mkdir pcr_reads
@@ -530,9 +439,9 @@ If you rely on your personal GO install, please think to update.
- Adding of two options **--add-lca-in** and **--lca-error** to `obiannotate`.
These options aim to help during construction of reference database using
`obipcr`. On `obipcr` output, it is commonly run `obiuniq`. To merge identical
`obipcr`. On obipcr output, it is commonly run obiuniq. To merge identical
sequences annotated with different taxids, it is now possible to use the
following strategies :
following strategie :
```{bash}
obiuniq -m taxid myrefdb.obipcr.fasta \
@@ -563,7 +472,7 @@ If you rely on your personal GO install, please think to update.
- Correction of a bug in `obiconsensus` leading into the deletion of a base
close to the beginning of the consensus sequence.
## March 31st, 2023. Release 4.0.2
## March 31th, 2023. Release 4.0.2
### Compiler change
@@ -574,15 +483,15 @@ If you rely on your personal GO install, please think to update.
- Add the possibility for looking pattern with indels. This has been added to
`obimultiplex` through the **--with-indels** option.
- Every obitools command has a **--pprof** option making the command
publishing a profiling website available at the address :
publishing a profiling web site available at the address :
<http://localhost:8080/debug/pprof/>
- A new `obiconsensus` command has been added. It is a prototype. It aims to
build a consensus sequence from a set of reads. The consensus is estimated
for all the sequences contained in the input file. If several input files,
or a directory name are provided the result contains a consensus per file.
The *id* of the sequence is the name of the input file depleted of its
The id of the sequence is the name of the input file depleted of its
directory name and of all its extensions.
- In `obipcr` an experimental option **--fragmented** allows for splitting very
- In `obipcr` an experimental option **--fragmented** allows for spliting very
long query sequences into shorter fragments with an overlap between the two
contiguous fragment insuring that no amplicons are missed despite the split.
As a site effect some amplicon can be identified twice.
@@ -625,7 +534,7 @@ If you rely on your personal GO install, please think to update.
### Enhancement
- *OBITools* are automatically processing all the sequences files contained in
a directory and its subdirectory\
a directory and its sub-directory\
recursively if its name is provided as input. To process easily Genbank
files, the corresponding filename extensions have been added. Today the
following extensions are recognized as sequence files : `.fasta`, `.fastq`,
@@ -642,7 +551,7 @@ If you rely on your personal GO install, please think to update.
export OBICPUMAX=4
```
- Adds a new option --out\|-o allowing to specify the name of an output file.
- Adds a new option --out\|-o allowing to specify the name of an outpout file.
``` bash
obiconvert -o xyz.fasta xxx.fastq
@@ -664,10 +573,10 @@ If you rely on your personal GO install, please think to update.
matched files remain consistent when processed.
- Adding of the function `ifelse` to the expression language for computing
conditional values.
conditionnal values.
- Adding two function to the expression language related to sequence
composition : `composition` and `gcskew`. Both are taking a sequence as
conposition : `composition` and `gcskew`. Both are taking a sequence as
single argument.
## February 18th, 2023. Release 4.0.0
@@ -675,8 +584,8 @@ If you rely on your personal GO install, please think to update.
It is the first version of the *OBITools* version 4. I decided to tag then
following two weeks of intensive data analysis with them allowing to discover
many small bugs present in the previous non-official version. Obviously other
bugs are certainly present in the code, and you are welcome to use the git
ticket system to mention them. But they seem to produce now reliable results.
bugs are certainly persent in the code, and you are welcome to use the git
ticket system to mention them. But they seems to produce now reliable results.
### Corrected bugs
@@ -684,11 +593,11 @@ ticket system to mention them. But they seem to produce now reliable results.
of sequences and to the production of incorrect file because of the last
sequence record, sometime truncated in its middle. This was only occurring
when more than a single CPU was used. It was affecting every obitools.
- The `obiparing` software had a bug in the right alignment procedure. This led
to the non-alignment of very sort barcode during the paring of the forward
- The `obiparing` software had a bug in the right aligment procedure. This led
to the non alignment of very sort barcode during the paring of the forward
and reverse reads.
- The `obipairing` tools had a non-deterministic comportment when aligning a
pair very low quality reads. This induced that the result of the same low
- The `obipairing` tools had a non deterministic comportment when aligning a
paor very low quality reads. This induced that the result of the same low
quality read pair was not the same from run to run.
### New features
@@ -696,10 +605,11 @@ ticket system to mention them. But they seem to produce now reliable results.
- Adding of a `--compress|-Z` option to every obitools allowing to produce
`gz` compressed output. OBITools were already able to deal with gziped input
files transparently. They can now produce their results in the same format.
- Adding of a `--append|-A` option to the `obidistribute` tool. It allows appending the result of an `obidistribute` execution to preexisting files. -
- Adding of a `--append|-A` option to the `obidistribute` tool. It allows to
append the result of an `obidistribute` execution to preexisting files. -
Adding of a `--directory|-d` option to the `obidistribute` tool. It allows
declaring a secondary classification key over the one defined by the
`--category\|-c\` option. This extra key leads to produce directories in
to declare a secondary classification key over the one defined by the
'--category\|-c\` option. This extra key leads to produce directories in
which files produced according to the primary criterion are stored.
- Adding of the functions `subspc`, `printf`, `int`, `numeric`, and `bool` to
the expression language.

508
blackboard/Prospective/kmer_disk_index_plan.md
View File

@@ -1,508 +0,0 @@
# Plan de refonte du package obikmer : index disk-based par partitions minimizer
## Constat
Les roaring64 bitmaps ne sont pas adaptés au stockage de 10^10 k-mers
(k=31) dispersés sur un espace de 2^62. L'overhead structurel (containers
roaring par high key 32 bits) dépasse la taille des données elles-mêmes,
et les opérations `Or()` entre bitmaps fragmentés ne terminent pas en
temps raisonnable.
## Principe de la nouvelle architecture
Un `KmerSet` est un ensemble trié de k-mers canoniques (uint64) stocké
sur disque, partitionné par minimizer. Chaque partition est un fichier
binaire contenant des uint64 triés, compressés par delta-varint.
Un `KmerSetGroup` est un répertoire contenant N ensembles partitionnés
de la même façon (même k, même m, même P).
Un `KmerSet` est un `KmerSetGroup` de taille 1 (singleton).
Les opérations ensemblistes se font partition par partition, en merge
streaming, sans charger l'index complet en mémoire.
## Cycle de vie d'un index
L'index a deux phases distinctes :
1. **Phase de construction (mutable)** : on ouvre un index, on y ajoute
des séquences. Pour chaque séquence, les super-kmers sont extraits
et écrits de manière compacte (2 bits/base) dans le fichier
temporaire de partition correspondant (`minimizer % P`). Les
super-kmers sont une représentation compressée naturelle des k-mers
chevauchants : un super-kmer de longueur L encode L-k+1 k-mers en
ne stockant que ~L/4 bytes au lieu de (L-k+1) × 8 bytes.
2. **Phase de clôture (optimisation)** : on ferme l'index, ce qui
déclenche le traitement **partition par partition** (indépendant,
parallélisable) :
- Charger les super-kmers de la partition
- En extraire tous les k-mers canoniques
- Trier le tableau de k-mers
- Dédupliquer (et compter si FrequencyFilter)
- Delta-encoder et écrire le fichier .kdi final
Après clôture, l'index est statique et immuable.
3. **Phase de lecture (immutable)** : opérations ensemblistes,
Jaccard, Quorum, Contains, itération. Toutes en streaming.
---
## Format sur disque
### Index finalisé
```
index_dir/
metadata.toml
set_0/
part_0000.kdi
part_0001.kdi
...
part_{P-1}.kdi
set_1/
part_0000.kdi
...
...
set_{N-1}/
...
```
### Fichiers temporaires pendant la construction
```
index_dir/
.build/
set_0/
part_0000.skm # super-kmers encodés 2 bits/base
part_0001.skm
...
set_1/
...
```
Le répertoire `.build/` est supprimé après Close().
### metadata.toml
```toml
id = "mon_index"
k = 31
m = 13
partitions = 1024
type = "KmerSetGroup" # ou "KmerSet" (N=1)
size = 3 # nombre de sets (N)
sets_ids = ["genome_A", "genome_B", "genome_C"]
[user_metadata]
organism = "Triticum aestivum"
[sets_metadata]
# métadonnées individuelles par set si nécessaire
```
### Fichier .kdi (Kmer Delta Index)
Format binaire :
```
[magic: 4 bytes "KDI\x01"]
[count: uint64 little-endian] # nombre de k-mers dans cette partition
[first: uint64 little-endian] # premier k-mer (valeur absolue)
[delta_1: varint] # arr[1] - arr[0]
[delta_2: varint] # arr[2] - arr[1]
...
[delta_{count-1}: varint] # arr[count-1] - arr[count-2]
```
Varint : encoding unsigned, 7 bits utiles par byte, bit de poids fort
= continuation (identique au varint protobuf).
Fichier vide (partition sans k-mer) : magic + count=0.
### Fichier .skm (Super-Kmer temporaire)
Format binaire, séquence de super-kmers encodés :
```
[len: uint16 little-endian] # longueur du super-kmer en bases
[sequence: ceil(len/4) bytes] # séquence encodée 2 bits/base, packed
...
```
**Compression par rapport au stockage de k-mers bruts** :
Un super-kmer de longueur L contient L-k+1 k-mers.
- Stockage super-kmer : 2 + ceil(L/4) bytes
- Stockage k-mers bruts : (L-k+1) × 8 bytes
Exemple avec k=31, super-kmer typique L=50 :
- Super-kmer : 2 + 13 = 15 bytes → encode 20 k-mers
- K-mers bruts : 20 × 8 = 160 bytes
- **Facteur de compression : ~10×**
Pour un génome de 10 Gbases (~10^10 k-mers bruts) :
- K-mers bruts : ~80 Go par set temporaire
- Super-kmers : **~8 Go** par set temporaire
Avec FrequencyFilter et couverture 30× :
- K-mers bruts : ~2.4 To
- Super-kmers : **~240 Go**
---
## FrequencyFilter
Le FrequencyFilter n'est plus un type de données séparé. C'est un
**mode de construction** du builder. Le résultat est un KmerSetGroup
standard.
### Principe
Pendant la construction, tous les super-kmers sont écrits dans les
fichiers temporaires .skm, y compris les doublons (chaque occurrence
de chaque séquence est écrite).
Pendant Close(), pour chaque partition :
1. Charger tous les super-kmers de la partition
2. Extraire tous les k-mers canoniques dans un tableau []uint64
3. Trier le tableau
4. Parcourir linéairement : les k-mers identiques sont consécutifs
5. Compter les occurrences de chaque k-mer
6. Si count >= minFreq → écrire dans le .kdi final (une seule fois)
7. Sinon → ignorer
### Dimensionnement
Pour un génome de 10 Gbases avec couverture 30× :
- N_brut ≈ 3×10^11 k-mers bruts
- Espace temporaire .skm ≈ 240 Go (compressé super-kmer)
- RAM par partition pendant Close() :
Avec P=1024 : ~3×10^8 k-mers/partition × 8 = **~2.4 Go**
Avec P=4096 : ~7.3×10^7 k-mers/partition × 8 = **~600 Mo**
Le choix de P détermine le compromis nombre de fichiers vs RAM par
partition.
### Sans FrequencyFilter (déduplication simple)
Pour de la déduplication simple (chaque k-mer écrit une fois), le
builder peut dédupliquer au niveau des buffers en RAM avant flush.
Cela réduit significativement l'espace temporaire car les doublons
au sein d'un même buffer (provenant de séquences proches) sont
éliminés immédiatement.
---
## API publique visée
### Structures
```go
// KmerSetGroup est l'entité de base.
// Un KmerSet est un KmerSetGroup avec Size() == 1.
type KmerSetGroup struct {
// champs internes : path, k, m, P, N, metadata, état
}
// KmerSetGroupBuilder construit un KmerSetGroup mutable.
type KmerSetGroupBuilder struct {
// champs internes : buffers I/O par partition et par set,
// fichiers temporaires .skm, paramètres (minFreq, etc.)
}
```
### Construction
```go
// NewKmerSetGroupBuilder crée un builder pour un nouveau KmerSetGroup.
// directory : répertoire de destination
// k : taille des k-mers (1-31)
// m : taille des minimizers (-1 pour auto = ceil(k/2.5))
// n : nombre de sets dans le groupe
// P : nombre de partitions (-1 pour auto)
// options : options de construction (FrequencyFilter, etc.)
func NewKmerSetGroupBuilder(directory string, k, m, n, P int,
options ...BuilderOption) (*KmerSetGroupBuilder, error)
// WithMinFrequency active le mode FrequencyFilter.
// Seuls les k-mers vus >= minFreq fois sont conservés dans l'index
// final. Les super-kmers sont écrits avec leurs doublons pendant
// la construction ; le comptage exact se fait au Close().
func WithMinFrequency(minFreq int) BuilderOption
// AddSequence extrait les super-kmers d'une séquence et les écrit
// dans les fichiers temporaires de partition du set i.
func (b *KmerSetGroupBuilder) AddSequence(setIndex int, seq *obiseq.BioSequence)
// AddSuperKmer écrit un super-kmer dans le fichier temporaire de
// sa partition pour le set i.
func (b *KmerSetGroupBuilder) AddSuperKmer(setIndex int, sk SuperKmer)
// Close finalise la construction :
// - flush des buffers d'écriture
// - pour chaque partition de chaque set (parallélisable) :
// - charger les super-kmers depuis le .skm
// - extraire les k-mers canoniques
// - trier, dédupliquer (compter si freq filter)
// - delta-encoder et écrire le .kdi
// - écrire metadata.toml
// - supprimer le répertoire .build/
// Retourne le KmerSetGroup en lecture seule.
func (b *KmerSetGroupBuilder) Close() (*KmerSetGroup, error)
```
### Lecture et opérations
```go
// OpenKmerSetGroup ouvre un index finalisé en lecture seule.
func OpenKmerSetGroup(directory string) (*KmerSetGroup, error)
// --- Métadonnées (API inchangée) ---
func (ksg *KmerSetGroup) K() int
func (ksg *KmerSetGroup) M() int // nouveau : taille du minimizer
func (ksg *KmerSetGroup) Partitions() int // nouveau : nombre de partitions
func (ksg *KmerSetGroup) Size() int
func (ksg *KmerSetGroup) Id() string
func (ksg *KmerSetGroup) SetId(id string)
func (ksg *KmerSetGroup) HasAttribute(key string) bool
func (ksg *KmerSetGroup) GetAttribute(key string) (interface{}, bool)
func (ksg *KmerSetGroup) SetAttribute(key string, value interface{})
// ... etc (toute l'API attributs actuelle est conservée)
// --- Opérations ensemblistes ---
// Toutes produisent un nouveau KmerSetGroup singleton sur disque.
// Opèrent partition par partition en streaming.
func (ksg *KmerSetGroup) Union(outputDir string) (*KmerSetGroup, error)
func (ksg *KmerSetGroup) Intersect(outputDir string) (*KmerSetGroup, error)
func (ksg *KmerSetGroup) Difference(outputDir string) (*KmerSetGroup, error)
func (ksg *KmerSetGroup) QuorumAtLeast(q int, outputDir string) (*KmerSetGroup, error)
func (ksg *KmerSetGroup) QuorumExactly(q int, outputDir string) (*KmerSetGroup, error)
func (ksg *KmerSetGroup) QuorumAtMost(q int, outputDir string) (*KmerSetGroup, error)
// --- Opérations entre deux KmerSetGroups ---
// Les deux groupes doivent avoir les mêmes k, m, P.
func (ksg *KmerSetGroup) UnionWith(other *KmerSetGroup, outputDir string) (*KmerSetGroup, error)
func (ksg *KmerSetGroup) IntersectWith(other *KmerSetGroup, outputDir string) (*KmerSetGroup, error)
// --- Métriques (résultat en mémoire, pas de sortie disque) ---
func (ksg *KmerSetGroup) JaccardDistanceMatrix() *obidist.DistMatrix
func (ksg *KmerSetGroup) JaccardSimilarityMatrix() *obidist.DistMatrix
// --- Accès individuel ---
func (ksg *KmerSetGroup) Len(setIndex ...int) uint64
func (ksg *KmerSetGroup) Contains(setIndex int, kmer uint64) bool
func (ksg *KmerSetGroup) Iterator(setIndex int) iter.Seq[uint64]
```
---
## Implémentation interne
### Primitives bas niveau
**`varint.go`** : encode/decode varint uint64
```go
func EncodeVarint(w io.Writer, v uint64) (int, error)
func DecodeVarint(r io.Reader) (uint64, error)
```
### Format .kdi
**`kdi_writer.go`** : écriture d'un fichier .kdi à partir d'un flux
trié de uint64 (delta-encode au vol).
```go
type KdiWriter struct { ... }
func NewKdiWriter(path string) (*KdiWriter, error)
func (w *KdiWriter) Write(kmer uint64) error
func (w *KdiWriter) Close() error
```
**`kdi_reader.go`** : lecture streaming d'un fichier .kdi (décode
les deltas au vol).
```go
type KdiReader struct { ... }
func NewKdiReader(path string) (*KdiReader, error)
func (r *KdiReader) Next() (uint64, bool)
func (r *KdiReader) Count() uint64
func (r *KdiReader) Close() error
```
### Format .skm
**`skm_writer.go`** : écriture de super-kmers encodés 2 bits/base.
```go
type SkmWriter struct { ... }
func NewSkmWriter(path string) (*SkmWriter, error)
func (w *SkmWriter) Write(sk SuperKmer) error
func (w *SkmWriter) Close() error
```
**`skm_reader.go`** : lecture de super-kmers depuis un fichier .skm.
```go
type SkmReader struct { ... }
func NewSkmReader(path string) (*SkmReader, error)
func (r *SkmReader) Next() (SuperKmer, bool)
func (r *SkmReader) Close() error
```
### Merge streaming
**`kdi_merge.go`** : k-way merge de plusieurs flux triés.
```go
type KWayMerge struct { ... }
func NewKWayMerge(readers []*KdiReader) *KWayMerge
func (m *KWayMerge) Next() (kmer uint64, count int, ok bool)
func (m *KWayMerge) Close() error
```
### Builder
**`kmer_set_builder.go`** : construction d'un KmerSetGroup.
Le builder gère :
- P × N écrivains .skm bufferisés (un par partition × set)
- À la clôture : traitement partition par partition
(parallélisable sur plusieurs cores)
Gestion mémoire des buffers d'écriture :
- Chaque SkmWriter a un buffer I/O de taille raisonnable (~64 Ko)
- Avec P=1024 et N=1 : 1024 × 64 Ko = 64 Mo de buffers
- Avec P=1024 et N=10 : 640 Mo de buffers
- Pas de buffer de k-mers en RAM : tout est écrit sur disque
immédiatement via les super-kmers
RAM pendant Close() (tri d'une partition) :
- Charger les super-kmers → extraire les k-mers → tableau []uint64
- Avec P=1024 et 10^10 k-mers/set : ~10^7 k-mers/partition × 8 = ~80 Mo
- Avec FrequencyFilter (doublons) et couverture 30× :
~3×10^8/partition × 8 = ~2.4 Go (ajustable via P)
### Structure disk-based
**`kmer_set_disk.go`** : KmerSetGroup en lecture seule.
**`kmer_set_disk_ops.go`** : opérations ensemblistes par merge
streaming partition par partition.
---
## Ce qui change par rapport à l'API actuelle
### Changements de sémantique
| Aspect | Ancien (roaring) | Nouveau (disk-based) |
|---|---|---|
| Stockage | En mémoire (roaring64.Bitmap) | Sur disque (.kdi delta-encoded) |
| Temporaire construction | En mémoire | Super-kmers sur disque (.skm 2 bits/base) |
| Mutabilité | Mutable à tout moment | Builder → Close() → immutable |
| Opérations ensemblistes | Résultat en mémoire | Résultat sur disque (nouveau répertoire) |
| Contains | O(1) roaring lookup | O(log n) recherche binaire sur .kdi |
| Itération | Roaring iterator | Streaming décodage delta-varint |
### API conservée (signatures identiques ou quasi-identiques)
- `KmerSetGroup` : `K()`, `Size()`, `Id()`, `SetId()`
- Toute l'API attributs
- `JaccardDistanceMatrix()`, `JaccardSimilarityMatrix()`
- `Len()`, `Contains()`
### API modifiée
- `Union()`, `Intersect()`, etc. : ajout du paramètre `outputDir`
- `QuorumAtLeast()`, etc. : idem
- Construction : `NewKmerSetGroupBuilder()` + `AddSequence()` + `Close()`
au lieu de manipulation directe
### API supprimée
- `KmerSet` comme type distinct (remplacé par KmerSetGroup singleton)
- `FrequencyFilter` comme type distinct (mode du Builder)
- Tout accès direct à `roaring64.Bitmap`
- `KmerSet.Copy()` (copie de répertoire à la place)
- `KmerSet.Union()`, `.Intersect()`, `.Difference()` (deviennent méthodes
de KmerSetGroup avec outputDir)
---
## Fichiers à créer / modifier dans pkg/obikmer
### Nouveaux fichiers
| Fichier | Contenu |
|---|---|
| `varint.go` | Encode/Decode varint uint64 |
| `kdi_writer.go` | Écrivain de fichiers .kdi (delta-encoded) |
| `kdi_reader.go` | Lecteur streaming de fichiers .kdi |
| `skm_writer.go` | Écrivain de super-kmers encodés 2 bits/base |
| `skm_reader.go` | Lecteur de super-kmers depuis .skm |
| `kdi_merge.go` | K-way merge streaming de flux triés |
| `kmer_set_builder.go` | KmerSetGroupBuilder (construction) |
| `kmer_set_disk.go` | KmerSetGroup disk-based (lecture, métadonnées) |
| `kmer_set_disk_ops.go` | Opérations ensemblistes streaming |
### Fichiers à supprimer
| Fichier | Raison |
|---|---|
| `kmer_set.go` | Remplacé par kmer_set_disk.go |
| `kmer_set_group.go` | Idem |
| `kmer_set_attributes.go` | Intégré dans kmer_set_disk.go |
| `kmer_set_persistence.go` | L'index est nativement sur disque |
| `kmer_set_group_quorum.go` | Intégré dans kmer_set_disk_ops.go |
| `frequency_filter.go` | Mode du Builder, plus de type séparé |
| `kmer_index_builder.go` | Remplacé par kmer_set_builder.go |
### Fichiers conservés tels quels
| Fichier | Contenu |
|---|---|
| `encodekmer.go` | Encodage/décodage k-mers |
| `superkmer.go` | Structure SuperKmer |
| `superkmer_iter.go` | IterSuperKmers, IterCanonicalKmers |
| `encodefourmer.go` | Encode4mer |
| `counting.go` | Count4Mer |
| `kmermap.go` | KmerMap (usage indépendant) |
| `debruijn.go` | Graphe de de Bruijn |
---
## Ordre d'implémentation
1. `varint.go` + tests
2. `skm_writer.go` + `skm_reader.go` + tests
3. `kdi_writer.go` + `kdi_reader.go` + tests
4. `kdi_merge.go` + tests
5. `kmer_set_builder.go` + tests (construction + Close)
6. `kmer_set_disk.go` (structure, métadonnées, Open)
7. `kmer_set_disk_ops.go` + tests (Union, Intersect, Quorum, Jaccard)
8. Adaptation de `pkg/obitools/obikindex/`
9. Suppression des anciens fichiers roaring
10. Adaptation des tests existants
Chaque étape est testable indépendamment.
---
## Dépendances externes
### Supprimées
- `github.com/RoaringBitmap/roaring` : plus nécessaire pour les
index k-mers (vérifier si d'autres packages l'utilisent encore)
### Ajoutées
- Aucune. Varint, delta-encoding, merge, encodage 2 bits/base :
tout est implémentable en Go standard.

213
blackboard/Prospective/kmer_index_design.md
View File

@@ -1,213 +0,0 @@
# Index de k-mers pour génomes de grande taille
## Contexte et objectifs
### Cas d'usage
- Indexation de k-mers longs (k=31) pour des génomes de grande taille (< 10 Go par génome)
- Nombre de génomes : plusieurs dizaines à quelques centaines
- Indexation en parallèle
- Stockage sur disque
- Possibilité d'ajouter des génomes, mais pas de modifier un génome existant
### Requêtes cibles
- **Présence/absence** d'un k-mer dans un génome
- **Intersection** entre génomes
- **Distances** : Jaccard (présence/absence) et potentiellement Bray-Curtis (comptage)
### Ressources disponibles
- 128 Go de RAM
- Stockage disque
---
## Estimation des volumes
### Par génome
- **10 Go de séquence** → ~10¹⁰ k-mers bruts (chevauchants)
- **Après déduplication** : typiquement 10-50% de k-mers uniques → **~1-5 × 10⁹ k-mers distincts**
### Espace théorique
- **k=31** → 62 bits → ~4.6 × 10¹⁸ k-mers possibles
- Table d'indexation directe impossible
---
## Métriques de distance
### Présence/absence (binaire)
- **Jaccard** : |A ∩ B| / |A ∪ B|
- **Sørensen-Dice** : 2|A ∩ B| / (|A| + |B|)
### Comptage (abondance)
- **Bray-Curtis** : 1 - (2 × Σ min(aᵢ, bᵢ)) / (Σ aᵢ + Σ bᵢ)
Note : Pour Bray-Curtis, le stockage des comptages est nécessaire, ce qui augmente significativement la taille de l'index.
---
## Options d'indexation
### Option 1 : Bloom Filter par génome
**Principe** : Structure probabiliste pour test d'appartenance.
**Avantages :**
- Très compact : ~10 bits/élément pour FPR ~1%
- Construction rapide, streaming
- Facile à sérialiser/désérialiser
- Intersection et Jaccard estimables via formules analytiques
**Inconvénients :**
- Faux positifs (pas de faux négatifs)
- Distances approximatives
**Taille estimée** : 1-6 Go par génome (selon FPR cible)
#### Dimensionnement des Bloom filters
```
\mathrm{FPR} ;=; \left(1 - e^{-h n / m}\right)^h
```
| Bits/élément | FPR optimal | k (hash functions) |
|--------------|-------------|---------------------|
| 8 | ~2% | 5-6 |
| 10 | ~1% | 7 |
| 12 | ~0.3% | 8 |
| 16 | ~0.01% | 11 |
Formule du taux de faux positifs :
```
FPR ≈ (1 - e^(-kn/m))^k
```
Où n = nombre d'éléments, m = nombre de bits, k = nombre de hash functions.
### Option 2 : Ensemble trié de k-mers
**Principe** : Stocker les k-mers (uint64) triés, avec compression possible.
**Avantages :**
- Exact (pas de faux positifs)
- Intersection/union par merge sort O(n+m)
- Compression efficace (delta encoding sur k-mers triés)
**Inconvénients :**
- Plus volumineux : 8 octets/k-mer
- Construction plus lente (tri nécessaire)
**Taille estimée** : 8-40 Go par génome (non compressé)
### Option 3 : MPHF (Minimal Perfect Hash Function)
**Principe** : Fonction de hash parfaite minimale pour les k-mers présents.
**Avantages :**
- Très compact : ~3-4 bits/élément
- Lookup O(1)
- Exact pour les k-mers présents
**Inconvénients :**
- Construction coûteuse (plusieurs passes)
- Statique (pas d'ajout de k-mers après construction)
- Ne distingue pas "absent" vs "jamais vu" sans structure auxiliaire
### Option 4 : Hybride MPHF + Bloom filter
- MPHF pour mapping compact des k-mers présents
- Bloom filter pour pré-filtrage des absents
---
## Optimisation : Indexation de (k-2)-mers pour requêtes k-mers
### Principe
Au lieu d'indexer directement les 31-mers dans un Bloom filter, on indexe les 29-mers. Pour tester la présence d'un 31-mer, on vérifie que les **trois 29-mers** qu'il contient sont présents :
- positions 0-28
- positions 1-29
- positions 2-30
### Analyse probabiliste
Si le Bloom filter a un FPR de p pour un 29-mer individuel, le FPR effectif pour un 31-mer devient **p³** (les trois requêtes doivent toutes être des faux positifs).
| FPR 29-mer | FPR 31-mer effectif |
|------------|---------------------|
| 10% | 0.1% |
| 5% | 0.0125% |
| 1% | 0.0001% |
### Avantages
1. **Moins d'éléments à stocker** : il y a moins de 29-mers distincts que de 31-mers distincts dans un génome (deux 31-mers différents peuvent partager un même 29-mer)
2. **FPR drastiquement réduit** : FPR³ avec seulement 3 requêtes
3. **Index plus compact** : on peut utiliser moins de bits par élément (FPR plus élevé acceptable sur le 29-mer) tout en obtenant un FPR très bas sur le 31-mer
### Trade-off
Un Bloom filter à **5-6 bits/élément** pour les 29-mers donnerait un FPR effectif < 0.01% pour les 31-mers, soit environ **2× plus compact** que l'approche directe à qualité égale.
**Coût** : 3× plus de requêtes par lookup (mais les requêtes Bloom sont très rapides).
---
## Accélération des calculs de distance : MinHash
### Principe
Pré-calculer une "signature" compacte (sketch) de chaque génome permettant d'estimer rapidement Jaccard sans charger les index complets.
### Avantages
- Matrice de distances entre 100+ génomes en quelques secondes
- Signature de taille fixe (ex: 1000-10000 hash values) quel que soit le génome
- Stockage minimal
### Utilisation
1. Construction : une passe sur les k-mers de chaque génome
2. Distance : comparaison des sketches en O(taille du sketch)
---
## Architecture recommandée
### Pour présence/absence + Jaccard
1. **Index principal** : Bloom filter de (k-2)-mers avec l'optimisation décrite
- Compact (~3-5 Go par génome)
- FPR très bas pour les k-mers grâce aux requêtes triples
2. **Sketches MinHash** : pour calcul rapide des distances entre génomes
- Quelques Ko par génome
- Permet exploration rapide de la matrice de distances
### Pour comptage + Bray-Curtis
1. **Index principal** : k-mers triés + comptages
- uint64 (k-mer) + uint8/uint16 (count)
- Compression delta possible
- Plus volumineux mais exact
2. **Sketches** : variantes de MinHash pour données pondérées (ex: HyperMinHash)
---
## Prochaines étapes
1. Implémenter un Bloom filter optimisé pour k-mers
2. Implémenter l'optimisation (k-2)-mer → k-mer
3. Implémenter MinHash pour les sketches
4. Définir le format de sérialisation sur disque
5. Benchmarker sur des génomes réels

264
blackboard/Prospective/large_sequence_parsing.md
View File

@@ -1,264 +0,0 @@
# Optimisation du parsing des grandes séquences
## Contexte
OBITools4 doit pouvoir traiter des séquences de taille chromosomique (plusieurs Gbp), notamment
issues de fichiers GenBank/EMBL (assemblages de génomes) ou de fichiers FASTA convertis depuis
ces formats.
## Architecture actuelle
### Pipeline de lecture (`pkg/obiformats/`)
```
ReadFileChunk (goroutine)
→ ChannelFileChunk
→ N × _ParseGenbankFile / _ParseFastaFile (goroutines)
→ IBioSequence
```
`ReadFileChunk` (`file_chunk_read.go`) lit le fichier par morceaux via une chaîne de
`PieceOfChunk` (rope). Chaque nœud fait `fileChunkSize` bytes :
- GenBank/EMBL : 128 MB (`1024*1024*128`)
- FASTA/FASTQ : 1 MB (`1024*1024`)
La chaîne est accumulée jusqu'à trouver la fin du dernier enregistrement complet (splitter),
puis `Pack()` est appelé pour fusionner tous les nœuds en un seul buffer contigu. Ce buffer
est transmis au parseur via `FileChunk.Raw *bytes.Buffer`.
### Parseur GenBank (`genbank_read.go`)
`GenbankChunkParser` reçoit un `io.Reader` sur le buffer packé, lit ligne par ligne via
`bufio.NewReader` (buffer 4096 bytes), et pour chaque ligne de la section `ORIGIN` :
```go
line = string(bline) // allocation par ligne
cleanline := strings.TrimSpace(line) // allocation
parts := strings.SplitN(cleanline, " ", 7) // allocation []string + substrings
for i := 1; i < lparts; i++ {
seqBytes.WriteString(parts[i])
}
```
Point positif : `seqBytes` est pré-alloué grâce à `lseq` extrait de la ligne `LOCUS`.
### Parseur FASTA (`fastaseq_read.go`)
`FastaChunkParser` lit **octet par octet** via `scanner.ReadByte()`. Pour 3 Gbp :
3 milliards d'appels. `seqBytes` est un `bytes.Buffer{}` sans pré-allocation.
## Problème principal
Pour une séquence de plusieurs Gbp, `Pack()` fusionne une chaîne de ~N nœuds de 128 MB en
un seul buffer contigu. C'est une allocation de N × 128 MB suivie d'une copie de toutes les
données. Bien que l'implémentation de `Pack()` soit efficace (libère les nœuds au fur et à
mesure via `slices.Grow`), la copie est inévitable avec l'architecture actuelle.
De plus, le parseur GenBank produit des dizaines de millions d'allocations temporaires pour
parser la section `ORIGIN` (une par ligne).
## Invariant clé découvert
**Si la rope a plus d'un nœud, le premier nœud seul ne se termine pas sur une frontière
d'enregistrement** (pas de `//\n` en fin de `piece1`).
Preuve par construction dans `ReadFileChunk` :
- `splitter` est appelé dès le premier nœud (ligne 157)
- Si `end >= 0` → frontière trouvée dans 128 MB → boucle interne sautée → rope à 1 nœud
- Si `end < 0` → boucle interne ajoute des nœuds → rope à ≥ 2 nœuds
Corollaire : si rope à 1 nœud, `Pack()` ne fait rien (aucun nœud suivant).
**Attention** : rope à ≥ 2 nœuds ne signifie pas qu'il n'y a qu'une seule séquence dans
la rope. La rope packée peut contenir plusieurs enregistrements complets. Exemple : records
de 80 MB → `nextpieces` (48 MB de reste) + nouveau nœud (128 MB) = rope à 2 nœuds
contenant 2 records complets + début d'un troisième.
L'invariant dit seulement que `piece1` seul est incomplet — pas que la rope entière
ne contient qu'un seul record.
**Invariant : le dernier FileChunk envoyé finit sur une frontière d'enregistrement.**
Deux chemins dans `ReadFileChunk` :
1. **Chemin normal** (`end >= 0` via `splitter`) : le buffer est explicitement tronqué à
`end` (ligne 200 : `pieces.data = pieces.data[:end]`). Frontière garantie par construction
pour tous les formats. ✓
2. **Chemin EOF** (`end < 0`, `end = pieces.Len()`) : tout le reste du fichier est envoyé.
- **GenBank/EMBL** : présuppose fichier bien formé (se termine par `//\n`). Le parseur
lève un `log.Fatalf` sur tout état inattendu — filet de sécurité suffisant. ✓
- **FASTQ** : présupposé, vérifié par le parseur. ✓
- **FASTA** : garanti par le format lui-même (fin d'enregistrement = EOF ou `>`). ✓
**Hypothèse de travail adoptée** : les fichiers d'entrée sont bien formés. Dans le pire cas,
le parseur lèvera une erreur explicite. Il n'y a pas de risque de corruption silencieuse.
## Piste d'optimisation : se dispenser de Pack()
### Idée centrale
Au lieu de fusionner la rope avant de la passer au parseur, **parser directement la rope
nœud par nœud**, et **écrire la séquence compactée in-place dans le premier nœud**.
Pourquoi c'est sûr :
- Le header (LOCUS, DEFINITION, SOURCE, FEATURES) est **petit** et traité en premier
- La séquence (ORIGIN) est **à la fin** du record
- Au moment d'écrire la séquence depuis l'offset 0 de `piece1`, le pointeur de lecture
est profond dans la rope (offset >> 0) → jamais de collision
- La séquence compactée est toujours plus courte que les données brutes
### Pré-allocation
Pour GenBank/EMBL : `lseq` est connu dès la ligne `LOCUS`/`ID` (première ligne, dans
`piece1`). On peut faire `slices.Grow(piece1.data, lseq)` dès ce moment.
Pour FASTA : pas de taille garantie dans le header, mais `rope.Len()` donne un majorant.
On peut utiliser `rope.Len() / 2` comme estimation initiale.
### Gestion des jonctions entre nœuds
Une ligne peut chevaucher deux nœuds (rare avec 128 MB, mais possible). Solution : carry
buffer de ~128 bytes pour les quelques bytes en fin de nœud.
### Cas FASTA/FASTQ multi-séquences
Un FileChunk peut contenir N séquences (notamment FASTA/FASTQ courts). Dans ce cas
l'écriture in-place dans `piece1` n'est pas applicable directement — on écrase des données
nécessaires aux séquences suivantes.
Stratégie par cas :
- **Rope à 1 nœud** (record ≤ 128 MB) : `Pack()` est trivial (no-op), parseur actuel OK
- **Rope à ≥ 2 nœuds** : par l'invariant, `piece1` ne contient pas de record complet →
une seule grande séquence → in-place applicable
### Format d'une ligne séquence GenBank (Après ORIGIN)
```
/^ *[0-9]+( [nuc]{10}){0,5} [nuc]{1,10}/
```
### Format d'une ligne séquence GenBank (Après SQ)
La ligne SQ contient aussi la taille de la séquence
```
/^ *( [nuc]{10}){0,5} [nuc]{1,10} *[0-9]+/
```
Compactage in-place sur `bline` ([]byte brut, sans conversion `string`) :
```go
w := 0
i := 0
for i < len(bline) && bline[i] == ' ' { i++ } // skip indentation
for i < len(bline) && bline[i] <= '9' { i++ } // skip position number
for ; i < len(bline); i++ {
if bline[i] != ' ' {
bline[w] = bline[i]
w++
}
}
// écrire bline[:w] directement dans piece1.data[seqOffset:]
```
## Changements nécessaires
1. **`FileChunk`** : exposer la rope `*PieceOfChunk` non-packée en plus (ou à la place)
de `Raw *bytes.Buffer`
2. **`GenbankChunkParser` / `EmblChunkParser`** : accepter `*PieceOfChunk`, parser la
rope séquentiellement avec carry buffer pour les jonctions
3. **`FastaChunkParser`** : idem, avec in-place conditionnel selon taille de la rope
4. **`ReadFileChunk`** : ne pas appeler `Pack()` avant envoi sur le channel (ou version
alternative `ReadFileChunkRope`)
## Fichiers concernés
- `pkg/obiformats/file_chunk_read.go` — structure rope, `ReadFileChunk`
- `pkg/obiformats/genbank_read.go` — `GenbankChunkParser`, `_ParseGenbankFile`
- `pkg/obiformats/embl_read.go` — `EmblChunkParser`, `ReadEMBL`
- `pkg/obiformats/fastaseq_read.go` — `FastaChunkParser`, `_ParseFastaFile`
- `pkg/obiformats/fastqseq_read.go` — parseur FASTQ (même structure)
## Plan d'implémentation : parseur GenBank sur rope
### Contexte
Baseline mesurée : `obiconvert gbpln640.seq.gz` → 49s real, 42s user, 29s sys, **57 GB RSS**.
Le sys élevé indique des allocations massives. Deux causes :
1. `Pack()` : fusionne toute la rope (N × 128 MB) en un buffer contigu avant de parser
2. Parser ORIGIN : `string(bline)` + `TrimSpace` + `SplitN` × millions de lignes
### 1. `gbRopeScanner`
Struct de lecture ligne par ligne sur la rope, sans allocation heap :
```go
type gbRopeScanner struct {
current *PieceOfChunk
pos int
carry [256]byte // stack-allocated, max GenBank line = 80 chars
carryN int
}
```
`ReadLine()` :
- Cherche `\n` dans `current.data[pos:]` via `bytes.IndexByte`
- Si trouvé sans carry : retourne slice direct du node (zéro alloc)
- Si trouvé avec carry : copie dans carry buffer, retourne `carry[:n]`
- Si non trouvé : copie le reste dans carry, avance au node suivant, recommence
- EOF : retourne `carry[:carryN]` puis nil
`extractSequence(dest []byte, UtoT bool) int` :
- Scan direct des bytes pour section ORIGIN, sans passer par ReadLine
- Machine d'états : lineStart → skip espaces/digits → copier nucléotides dans dest
- Stop sur `//` en début de ligne
- Zéro allocation, UtoT inline
### 2. `GenbankChunkParserRope`
```go
func GenbankChunkParserRope(source string, rope *PieceOfChunk,
withFeatureTable, UtoT bool) (obiseq.BioSequenceSlice, error)
```
- Même machine d'états que `GenbankChunkParser`, sur `[]byte` (`bytes.HasPrefix`)
- LOCUS : extrait `id` et `lseq` par scan direct (remplace `_seqlenght_rx`)
- FEATURES / default inFeature : taxid extrait par scan de `/db_xref="taxon:`
dans la source feature ; `featBytes` rempli seulement si `withFeatureTable=true`
- DEFINITION : toujours conservée
- ORIGIN : `dest = make([]byte, 0, lseq+20)` puis `s.extractSequence(dest, UtoT)`
### 3. Modifications `_ParseGenbankFile` et `ReadGenbank`
`_ParseGenbankFile` utilise `chunk.Rope` :
```go
sequences, err := GenbankChunkParserRope(chunk.Source, chunk.Rope, ...)
```
`ReadGenbank` passe `pack=false` :
```go
entry_channel := ReadFileChunk(..., false)
```
### 4. Ce qui NE change pas
- `GenbankChunkParser` reste (référence, tests)
- `ReadFileChunk`, `Pack()`, autres parseurs (EMBL, FASTA, FASTQ) : inchangés
### 5. Gains attendus
- **RSS** : pic ≈ 128 MB × workers (au lieu de N × 128 MB)
- **Temps sys** : élimination des mmap/munmap pour les gros buffers
- **Temps user** : ~50M allocations éliminées
### 6. Vérification
```bash
/usr/local/go/bin/go build ./...
diff <(obiconvert gbpln640.seq.gz) gbpln640.reference.fasta
cd bugs/genbank && ./benchmark.sh gbpln640.seq.gz
```
Cible : RSS < 1 GB, temps comparable ou meilleur.

735
blackboard/architechture/architecture-commande-obitools.md
View File

@@ -1,735 +0,0 @@
# Architecture d'une commande OBITools
## Vue d'ensemble
Une commande OBITools suit une architecture modulaire et standardisée qui sépare clairement les responsabilités entre :
- Le package de la commande dans `pkg/obitools/<nom_commande>/`
- L'exécutable dans `cmd/obitools/<nom_commande>/`
Cette architecture favorise la réutilisabilité du code, la testabilité et la cohérence entre les différentes commandes de la suite OBITools.
## Structure du projet
```
obitools4/
├── pkg/obitools/
│ ├── obiconvert/ # Commande de conversion (base pour toutes)
│ │ ├── obiconvert.go # Fonctions vides (pas d'implémentation)
│ │ ├── options.go # Définition des options CLI
│ │ ├── sequence_reader.go # Lecture des séquences
│ │ └── sequence_writer.go # Écriture des séquences
│ ├── obiuniq/ # Commande de déréplication
│ │ ├── obiuniq.go # (fichier vide)
│ │ ├── options.go # Options spécifiques à obiuniq
│ │ └── unique.go # Implémentation du traitement
│ ├── obipairing/ # Assemblage de lectures paired-end
│ ├── obisummary/ # Résumé de fichiers de séquences
│ └── obimicrosat/ # Détection de microsatellites
└── cmd/obitools/
├── obiconvert/
│ └── main.go # Point d'entrée de la commande
├── obiuniq/
│ └── main.go
├── obipairing/
│ └── main.go
├── obisummary/
│ └── main.go
└── obimicrosat/
└── main.go
```
## Composants de l'architecture
### 1. Package `pkg/obitools/<commande>/`
Chaque commande possède son propre package dans `pkg/obitools/` qui contient l'implémentation complète de la logique métier. Ce package est structuré en plusieurs fichiers :
#### a) `options.go` - Gestion des options CLI
Ce fichier définit :
- Les **variables globales** privées (préfixées par `_`) stockant les valeurs des options
- La fonction **`OptionSet()`** qui configure toutes les options pour la commande
- Les fonctions **`CLI*()`** qui retournent les valeurs des options (getters)
- Les fonctions **`Set*()`** qui permettent de définir les options programmatiquement (setters)
**Exemple (obiuniq/options.go) :**
```go
package obiuniq
import (
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
"github.com/DavidGamba/go-getoptions"
)
// Variables globales privées pour stocker les options
var _StatsOn = make([]string, 0, 10)
var _Keys = make([]string, 0, 10)
var _InMemory = false
var _chunks = 100
// Configuration des options spécifiques à la commande
func UniqueOptionSet(options *getoptions.GetOpt) {
options.StringSliceVar(&_StatsOn, "merge", 1, 1,
options.Alias("m"),
options.ArgName("KEY"),
options.Description("Adds a merged attribute..."))
options.BoolVar(&_InMemory, "in-memory", _InMemory,
options.Description("Use memory instead of disk..."))
options.IntVar(&_chunks, "chunk-count", _chunks,
options.Description("In how many chunks..."))
}
// OptionSet combine les options de base + les options spécifiques
func OptionSet(options *getoptions.GetOpt) {
obiconvert.OptionSet(false)(options) // Options de base
UniqueOptionSet(options) // Options spécifiques
}
// Getters pour accéder aux valeurs des options
func CLIStatsOn() []string {
return _StatsOn
}
func CLIUniqueInMemory() bool {
return _InMemory
}
// Setters pour définir les options programmatiquement
func SetUniqueInMemory(inMemory bool) {
_InMemory = inMemory
}
```
**Convention de nommage :**
- Variables privées : `_NomOption` (underscore préfixe)
- Getters : `CLINomOption()` (préfixe CLI)
- Setters : `SetNomOption()` (préfixe Set)
#### b) Fichier(s) d'implémentation
Un ou plusieurs fichiers contenant la logique métier de la commande :
**Exemple (obiuniq/unique.go) :**
```go
package obiuniq
import (
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obichunk"
)
// Fonction CLI principale qui orchestre le traitement
func CLIUnique(sequences obiiter.IBioSequence) obiiter.IBioSequence {
// Récupération des options via les getters CLI*()
options := make([]obichunk.WithOption, 0, 30)
options = append(options,
obichunk.OptionBatchCount(CLINumberOfChunks()),
)
if CLIUniqueInMemory() {
options = append(options, obichunk.OptionSortOnMemory())
} else {
options = append(options, obichunk.OptionSortOnDisk())
}
// Appel de la fonction de traitement réelle
iUnique, err := obichunk.IUniqueSequence(sequences, options...)
if err != nil {
log.Fatal(err)
}
return iUnique
}
```
**Autres exemples d'implémentation :**
- **obimicrosat/microsat.go** : Contient `MakeMicrosatWorker()` et `CLIAnnotateMicrosat()`
- **obisummary/obisummary.go** : Contient `ISummary()` et les structures de données
#### c) Fichiers utilitaires (optionnel)
Certaines commandes ont des fichiers additionnels pour des fonctionnalités spécifiques.
**Exemple (obipairing/options.go) :**
```go
// Fonction spéciale pour créer un itérateur de séquences pairées
func CLIPairedSequence() (obiiter.IBioSequence, error) {
forward, err := obiconvert.CLIReadBioSequences(_ForwardFile)
if err != nil {
return obiiter.NilIBioSequence, err
}
reverse, err := obiconvert.CLIReadBioSequences(_ReverseFile)
if err != nil {
return obiiter.NilIBioSequence, err
}
paired := forward.PairTo(reverse)
return paired, nil
}
```
### 2. Package `obiconvert` - La base commune
Le package `obiconvert` est spécial car il fournit les fonctionnalités de base utilisées par toutes les autres commandes :
#### Fonctionnalités fournies :
1. **Lecture de séquences** (`sequence_reader.go`)
- `CLIReadBioSequences()` : lecture depuis fichiers ou stdin
- Support de multiples formats (FASTA, FASTQ, EMBL, GenBank, etc.)
- Gestion des fichiers multiples
- Barre de progression optionnelle
2. **Écriture de séquences** (`sequence_writer.go`)
- `CLIWriteBioSequences()` : écriture vers fichiers ou stdout
- Support de multiples formats
- Gestion des lectures pairées
- Compression optionnelle
3. **Options communes** (`options.go`)
- Options d'entrée (format, skip, etc.)
- Options de sortie (format, fichier, compression)
- Options de mode (barre de progression, etc.)
#### Utilisation par les autres commandes :
Toutes les commandes incluent les options de `obiconvert` via :
```go
func OptionSet(options *getoptions.GetOpt) {
obiconvert.OptionSet(false)(options) // false = pas de fichiers pairés
MaCommandeOptionSet(options) // Options spécifiques
}
```
### 3. Exécutable `cmd/obitools/<commande>/main.go`
Le fichier `main.go` de chaque commande est volontairement **minimaliste** et suit toujours le même pattern :
```go
package main
import (
"os"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/macommande"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
)
func main() {
// 1. Configuration optionnelle de paramètres par défaut
obidefault.SetBatchSize(10)
// 2. Génération du parser d'options
optionParser := obioptions.GenerateOptionParser(
"macommande", // Nom de la commande
"description de la commande", // Description
macommande.OptionSet) // Fonction de configuration des options
// 3. Parsing des arguments
_, args := optionParser(os.Args)
// 4. Lecture des séquences d'entrée
sequences, err := obiconvert.CLIReadBioSequences(args...)
obiconvert.OpenSequenceDataErrorMessage(args, err)
// 5. Traitement spécifique de la commande
resultat := macommande.CLITraitement(sequences)
// 6. Écriture des résultats
obiconvert.CLIWriteBioSequences(resultat, true)
// 7. Attente de la fin du pipeline
obiutils.WaitForLastPipe()
}
```
## Patterns architecturaux
### Pattern 1 : Pipeline de traitement de séquences
La plupart des commandes suivent ce pattern :
```
Lecture → Traitement → Écriture
```
**Exemples :**
- **obiconvert** : Lecture → Écriture (conversion de format)
- **obiuniq** : Lecture → Déréplication → Écriture
- **obimicrosat** : Lecture → Annotation → Filtrage → Écriture
### Pattern 2 : Traitement avec entrées multiples
Certaines commandes acceptent plusieurs fichiers d'entrée :
**obipairing** :
```
Lecture Forward + Lecture Reverse → Pairing → Assemblage → Écriture
```
### Pattern 3 : Traitement sans écriture de séquences
**obisummary** : produit un résumé JSON/YAML au lieu de séquences
```go
func main() {
// ... parsing options et lecture ...
summary := obisummary.ISummary(fs, obisummary.CLIMapSummary())
// Formatage et affichage direct
if obisummary.CLIOutFormat() == "json" {
output, _ := json.MarshalIndent(summary, "", " ")
fmt.Print(string(output))
} else {
output, _ := yaml.Marshal(summary)
fmt.Print(string(output))
}
}
```
### Pattern 4 : Utilisation de Workers
Les commandes qui transforment des séquences utilisent souvent le pattern Worker :
```go
// Création d'un worker
worker := MakeMicrosatWorker(
CLIMinUnitLength(),
CLIMaxUnitLength(),
// ... autres paramètres
)
// Application du worker sur l'itérateur
newIter = iterator.MakeIWorker(
worker,
false, // merge results
obidefault.ParallelWorkers() // parallélisation
)
```
## Étapes d'implémentation d'une nouvelle commande
### Étape 1 : Créer le package dans `pkg/obitools/`
```bash
mkdir -p pkg/obitools/macommande
```
### Étape 2 : Créer `options.go`
```go
package macommande
import (
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
"github.com/DavidGamba/go-getoptions"
)
// Variables privées pour les options
var _MonOption = "valeur_par_defaut"
// Configuration des options spécifiques
func MaCommandeOptionSet(options *getoptions.GetOpt) {
options.StringVar(&_MonOption, "mon-option", _MonOption,
options.Alias("o"),
options.Description("Description de l'option"))
}
// OptionSet combine options de base + spécifiques
func OptionSet(options *getoptions.GetOpt) {
obiconvert.OptionSet(false)(options) // false si pas de fichiers pairés
MaCommandeOptionSet(options)
}
// Getters
func CLIMonOption() string {
return _MonOption
}
// Setters
func SetMonOption(value string) {
_MonOption = value
}
```
### Étape 3 : Créer le fichier d'implémentation
Créer `macommande.go` (ou un nom plus descriptif) :
```go
package macommande
import (
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
)
// Fonction de traitement principale
func CLIMaCommande(sequences obiiter.IBioSequence) obiiter.IBioSequence {
// Récupération des options
option := CLIMonOption()
// Implémentation du traitement
// ...
return resultat
}
```
### Étape 4 : Créer l'exécutable dans `cmd/obitools/`
```bash
mkdir -p cmd/obitools/macommande
```
Créer `main.go` :
```go
package main
import (
"os"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/macommande"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
)
func main() {
// Parser d'options
optionParser := obioptions.GenerateOptionParser(
"macommande",
"Description courte de ma commande",
macommande.OptionSet)
_, args := optionParser(os.Args)
// Lecture
sequences, err := obiconvert.CLIReadBioSequences(args...)
obiconvert.OpenSequenceDataErrorMessage(args, err)
// Traitement
resultat := macommande.CLIMaCommande(sequences)
// Écriture
obiconvert.CLIWriteBioSequences(resultat, true)
// Attente
obiutils.WaitForLastPipe()
}
```
### Étape 5 : Configurations optionnelles
Dans `main.go`, avant le parsing des options, on peut configurer :
```go
// Taille des batchs de séquences
obidefault.SetBatchSize(10)
// Nombre de workers en lecture (strict)
obidefault.SetStrictReadWorker(2)
// Nombre de workers en écriture
obidefault.SetStrictWriteWorker(2)
// Désactiver la lecture des qualités
obidefault.SetReadQualities(false)
```
### Étape 6 : Gestion des erreurs
Utiliser les fonctions utilitaires pour les messages d'erreur cohérents :
```go
// Pour les erreurs d'ouverture de fichiers
obiconvert.OpenSequenceDataErrorMessage(args, err)
// Pour les erreurs générales
if err != nil {
log.Errorf("Message d'erreur: %v", err)
os.Exit(1)
}
```
### Étape 7 : Tests et debugging (optionnel)
Des commentaires dans le code montrent comment activer le profiling :
```go
// go tool pprof -http=":8000" ./macommande ./cpu.pprof
// f, err := os.Create("cpu.pprof")
// if err != nil {
// log.Fatal(err)
// }
// pprof.StartCPUProfile(f)
// defer pprof.StopCPUProfile()
// go tool trace cpu.trace
// ftrace, err := os.Create("cpu.trace")
// if err != nil {
// log.Fatal(err)
// }
// trace.Start(ftrace)
// defer trace.Stop()
```
## Bonnes pratiques observées
### 1. Séparation des responsabilités
- **`main.go`** : orchestration minimale
- **`options.go`** : définition et gestion des options
- **Fichiers d'implémentation** : logique métier
### 2. Convention de nommage cohérente
- Variables d'options : `_NomOption`
- Getters CLI : `CLINomOption()`
- Setters : `SetNomOption()`
- Fonctions de traitement CLI : `CLITraitement()`
### 3. Réutilisation du code
- Toutes les commandes réutilisent `obiconvert` pour l'I/O
- Les options communes sont partagées
- Les fonctions utilitaires sont centralisées
### 4. Configuration par défaut
Les valeurs par défaut sont :
- Définies lors de l'initialisation des variables
- Modifiables via les options CLI
- Modifiables programmatiquement via les setters
### 5. Gestion des formats
Support automatique de multiples formats :
- FASTA / FASTQ (avec compression gzip)
- EMBL / GenBank
- ecoPCR
- CSV
- JSON (avec différents formats d'en-têtes)
### 6. Parallélisation
Les commandes utilisent les workers parallèles via :
- `obidefault.ParallelWorkers()`
- `obidefault.SetStrictReadWorker(n)`
- `obidefault.SetStrictWriteWorker(n)`
### 7. Logging cohérent
Utilisation de `logrus` pour tous les logs :
```go
log.Printf("Message informatif")
log.Errorf("Message d'erreur: %v", err)
log.Fatal(err) // Arrêt du programme
```
## Dépendances principales
### Packages internes OBITools
- `pkg/obidefault` : valeurs par défaut et configuration globale
- `pkg/obioptions` : génération du parser d'options
- `pkg/obiiter` : itérateurs de séquences biologiques
- `pkg/obiseq` : structures et fonctions pour séquences biologiques
- `pkg/obiformats` : lecture/écriture de différents formats
- `pkg/obiutils` : fonctions utilitaires diverses
- `pkg/obichunk` : traitement par chunks (pour dereplication, etc.)
### Packages externes
- `github.com/DavidGamba/go-getoptions` : parsing des options CLI
- `github.com/sirupsen/logrus` : logging structuré
- `gopkg.in/yaml.v3` : encodage/décodage YAML
- `github.com/dlclark/regexp2` : expressions régulières avancées
## Cas spéciaux
### Commande avec fichiers pairés (obipairing)
```go
func OptionSet(options *getoptions.GetOpt) {
obiconvert.OutputOptionSet(options)
obiconvert.InputOptionSet(options)
PairingOptionSet(options) // Options spécifiques au pairing
}
func CLIPairedSequence() (obiiter.IBioSequence, error) {
forward, err := obiconvert.CLIReadBioSequences(_ForwardFile)
// ...
reverse, err := obiconvert.CLIReadBioSequences(_ReverseFile)
// ...
paired := forward.PairTo(reverse)
return paired, nil
}
```
Dans `main.go` :
```go
pairs, err := obipairing.CLIPairedSequence() // Lecture spéciale
if err != nil {
log.Errorf("Cannot open file (%v)", err)
os.Exit(1)
}
paired := obipairing.IAssemblePESequencesBatch(
pairs,
obipairing.CLIGapPenality(),
// ... autres paramètres
)
```
### Commande sans sortie de séquences (obisummary)
Au lieu de `obiconvert.CLIWriteBioSequences()`, affichage direct :
```go
summary := obisummary.ISummary(fs, obisummary.CLIMapSummary())
if obisummary.CLIOutFormat() == "json" {
output, _ := json.MarshalIndent(summary, "", " ")
fmt.Print(string(output))
} else {
output, _ := yaml.Marshal(summary)
fmt.Print(string(output))
}
fmt.Printf("\n")
```
### Commande avec Workers personnalisés (obimicrosat)
```go
func CLIAnnotateMicrosat(iterator obiiter.IBioSequence) obiiter.IBioSequence {
// Création du worker
worker := MakeMicrosatWorker(
CLIMinUnitLength(),
CLIMaxUnitLength(),
CLIMinUnitCount(),
CLIMinLength(),
CLIMinFlankLength(),
CLIReoriented(),
)
// Application du worker
newIter := iterator.MakeIWorker(
worker,
false, // pas de merge
obidefault.ParallelWorkers(), // parallélisation
)
return newIter.FilterEmpty() // Filtrage des résultats vides
}
```
## Diagramme de flux d'exécution
```
┌─────────────────────────────────────────────────────────────┐
│ cmd/obitools/macommande/main.go │
└─────────────────────────────────────────────────────────────┘
│
▼
┌─────────────────────────────────────────────────────────────┐
│ 1. Génération du parser d'options │
│ obioptions.GenerateOptionParser( │
│ "macommande", │
│ "description", │
│ macommande.OptionSet) │
└─────────────────────────────────────────────────────────────┘
│
▼
┌─────────────────────────────────────────────────────────────┐
│ pkg/obitools/macommande/options.go │
│ ┌─────────────────────────────────────────────────────┐ │
│ │ func OptionSet(options *getoptions.GetOpt) │ │
│ │ obiconvert.OptionSet(false)(options) ───────────┐ │ │
│ │ MaCommandeOptionSet(options) │ │ │
│ └───────────────────────────────────────────────────┼─┘ │
└────────────────────────────────────────────────────────┼─────┘
│ │
│ │
┌─────────────┘ │
│ │
▼ ▼
┌─────────────────────────────────┐ ┌───────────────────────────────┐
│ 2. Parsing des arguments │ │ pkg/obitools/obiconvert/ │
│ _, args := optionParser(...) │ │ options.go │
└─────────────────────────────────┘ │ - InputOptionSet() │
│ │ - OutputOptionSet() │
▼ │ - PairedFilesOptionSet() │
┌─────────────────────────────────┐ └───────────────────────────────┘
│ 3. Lecture des séquences │
│ CLIReadBioSequences(args) │
└─────────────────────────────────┘
│
▼
┌─────────────────────────────────────────────────────────────┐
│ pkg/obitools/obiconvert/sequence_reader.go │
│ - ExpandListOfFiles() │
│ - ReadSequencesFromFile() / ReadSequencesFromStdin() │
│ - Support: FASTA, FASTQ, EMBL, GenBank, ecoPCR, CSV │
└─────────────────────────────────────────────────────────────┘
│
▼ obiiter.IBioSequence
┌─────────────────────────────────────────────────────────────┐
│ 4. Traitement spécifique │
│ macommande.CLITraitement(sequences) │
└─────────────────────────────────────────────────────────────┘
│
▼
┌─────────────────────────────────────────────────────────────┐
│ pkg/obitools/macommande/<implementation>.go │
│ - Récupération des options via CLI*() getters │
│ - Application de la logique métier │
│ - Retour d'un nouvel iterator │
└─────────────────────────────────────────────────────────────┘
│
▼ obiiter.IBioSequence
┌─────────────────────────────────────────────────────────────┐
│ 5. Écriture des résultats │
│ CLIWriteBioSequences(resultat, true) │
└─────────────────────────────────────────────────────────────┘
│
▼
┌─────────────────────────────────────────────────────────────┐
│ pkg/obitools/obiconvert/sequence_writer.go │
│ - WriteSequencesToFile() / WriteSequencesToStdout() │
│ - Support: FASTA, FASTQ, JSON │
│ - Gestion des lectures pairées │
│ - Compression optionnelle │
└─────────────────────────────────────────────────────────────┘
│
▼
┌─────────────────────────────────────────────────────────────┐
│ 6. Attente de fin du pipeline │
│ obiutils.WaitForLastPipe() │
└─────────────────────────────────────────────────────────────┘
```
## Conclusion
L'architecture des commandes OBITools est conçue pour :
1. **Maximiser la réutilisation** : `obiconvert` fournit les fonctionnalités communes
2. **Simplifier l'ajout de nouvelles commandes** : pattern standardisé et minimaliste
3. **Faciliter la maintenance** : séparation claire des responsabilités
4. **Garantir la cohérence** : conventions de nommage et structure uniforme
5. **Optimiser les performances** : parallélisation intégrée et traitement par batch
Cette architecture modulaire permet de créer rapidement de nouvelles commandes tout en maintenant une qualité et une cohérence élevées dans toute la suite OBITools.

99
blackboard/architechture/definition-superkmer.md
View File

@@ -1,99 +0,0 @@
# Définition du super k-mer
## Définition
Un **super k-mer** est une **sous-séquence MAXIMALE** d'une séquence dans laquelle **tous les k-mers consécutifs partagent le même minimiseur**.
### Termes
- **k-mer** : sous-séquence de longueur k
- **minimiseur** : le plus petit m-mer canonique parmi tous les m-mers d'un k-mer
- **k-mers consécutifs** : k-mers aux positions i et i+1 (chevauchement de k-1 nucléotides)
- **MAXIMALE** : ne peut être étendue ni à gauche ni à droite
## RÈGLES ABSOLUES
### RÈGLE 1 : Longueur minimum = k
Un super k-mer contient au minimum k nucléotides.
```
longueur(super-kmer) >= k
```
### RÈGLE 2 : Chevauchement obligatoire = k-1
Deux super-kmers consécutifs se chevauchent d'EXACTEMENT k-1 nucléotides.
```
SK1.End - SK2.Start = k - 1
```
### RÈGLE 3 : Bijection séquence ↔ minimiseur
Une séquence de super k-mer a UN et UN SEUL minimiseur.
```
Même séquence → Même minimiseur (TOUJOURS)
```
**Si vous observez la même séquence avec deux minimiseurs différents, c'est un BUG.**
### RÈGLE 4 : Tous les k-mers partagent le minimiseur
TOUS les k-mers contenus dans un super k-mer ont le même minimiseur.
```
∀ k-mer K dans SK : minimiseur(K) = SK.minimizer
```
### RÈGLE 5 : Maximalité
Un super k-mer ne peut pas être étendu.
- Si on ajoute un nucléotide à gauche : le nouveau k-mer a un minimiseur différent
- Si on ajoute un nucléotide à droite : le nouveau k-mer a un minimiseur différent
## VIOLATIONS INTERDITES
❌ **Super k-mer de longueur < k**
❌ **Chevauchement ≠ k-1 entre consécutifs**
❌ **Même séquence avec minimiseurs différents**
❌ **K-mer dans le super k-mer avec minimiseur différent**
❌ **Super k-mer extensible (non-maximal)**
## CONSÉQUENCES PRATIQUES
### Pour l'extraction
L'algorithme doit :
1. Calculer le minimiseur de chaque k-mer
2. Découper quand le minimiseur change
3. Assigner au super k-mer le minimiseur commun à tous ses k-mers
4. Garantir que chaque super k-mer contient au moins k nucléotides
5. Garantir le chevauchement de k-1 entre consécutifs
### Pour la validation
Si après déduplication (obiuniq) on observe :
```
Séquence: ACGT...
Minimiseurs: {M1, M2} // plusieurs minimiseurs
```
C'est la PREUVE d'un bug : l'algorithme a produit cette séquence avec des minimiseurs différents, ce qui viole la RÈGLE 3.
## DIAGNOSTIC DU BUG
**Bug observé** : Même séquence avec minimiseurs différents après obiuniq
**Cause possible** : L'algorithme assigne le mauvais minimiseur OU découpe mal les super-kmers
**Ce que le bug NE PEUT PAS être** :
- Un problème d'obiuniq (révèle le bug, ne le crée pas)
- Un problème de chevauchement légitime (k-1 est correct)
**Ce que le bug DOIT être** :
- Minimiseur mal calculé ou mal assigné
- Découpage incorrect (mauvais endPos)
- Copie incorrecte des données

316
blackboard/architechture/guide-redaction-obitest.md
View File

@@ -1,316 +0,0 @@
# Guide de rédaction d'un obitest
## Règles essentielles
1. **Données < 1 KB** - Fichiers de test très petits
2. **Exécution < 10 sec** - Tests rapides pour CI/CD
3. **Auto-contenu** - Pas de dépendances externes
4. **Auto-nettoyage** - Pas de fichiers résiduels
## Structure minimale
```
obitests/obitools/<commande>/
├── test.sh # Script exécutable
└── data.fasta # Données minimales (optionnel)
```
## Template de test.sh
```bash
#!/bin/bash
TEST_NAME=<commande>
CMD=<commande>
TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
export PATH="${OBITOOLS_DIR}:${PATH}"
MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
TMPDIR="$(mktemp -d)"
ntest=0
success=0
failed=0
cleanup() {
echo "========================================" 1>&2
echo "## Results of the $TEST_NAME tests:" 1>&2
echo 1>&2
echo "- $ntest tests run" 1>&2
echo "- $success successfully completed" 1>&2
echo "- $failed failed tests" 1>&2
echo 1>&2
echo "Cleaning up the temporary directory..." 1>&2
echo 1>&2
echo "========================================" 1>&2
rm -rf "$TMPDIR"
if [ $failed -gt 0 ]; then
log "$TEST_NAME tests failed"
log
log
exit 1
fi
log
log
exit 0
}
log() {
echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
}
log "Testing $TEST_NAME..."
log "Test directory is $TEST_DIR"
log "obitools directory is $OBITOOLS_DIR"
log "Temporary directory is $TMPDIR"
log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
########## TESTS ##########
# Test 1: Help (OBLIGATOIRE)
((ntest++))
if $CMD -h > "${TMPDIR}/help.txt" 2>&1
then
log "$MCMD: printing help OK"
((success++))
else
log "$MCMD: printing help failed"
((failed++))
fi
# Ajoutez vos tests ici...
###########################
cleanup
```
## Pattern de test
```bash
((ntest++))
if commande args > "${TMPDIR}/output.txt" 2>&1
then
log "$MCMD: description OK"
((success++))
else
log "$MCMD: description failed"
((failed++))
fi
```
## Tests courants
### Exécution basique
```bash
((ntest++))
if $CMD "${TEST_DIR}/input.fasta" > "${TMPDIR}/output.fasta" 2>&1
then
log "$MCMD: basic execution OK"
((success++))
else
log "$MCMD: basic execution failed"
((failed++))
fi
```
### Sortie non vide
```bash
((ntest++))
if [ -s "${TMPDIR}/output.fasta" ]
then
log "$MCMD: output not empty OK"
((success++))
else
log "$MCMD: output empty - failed"
((failed++))
fi
```
### Comptage
```bash
((ntest++))
count=$(grep -c "^>" "${TMPDIR}/output.fasta")
if [ "$count" -gt 0 ]
then
log "$MCMD: extracted $count sequences OK"
((success++))
else
log "$MCMD: no sequences - failed"
((failed++))
fi
```
### Présence de contenu
```bash
((ntest++))
if grep -q "expected_string" "${TMPDIR}/output.fasta"
then
log "$MCMD: expected content found OK"
((success++))
else
log "$MCMD: content not found - failed"
((failed++))
fi
```
### Comparaison avec référence
```bash
((ntest++))
if diff "${TEST_DIR}/expected.fasta" "${TMPDIR}/output.fasta" > /dev/null
then
log "$MCMD: matches reference OK"
((success++))
else
log "$MCMD: differs from reference - failed"
((failed++))
fi
```
### Test avec options
```bash
((ntest++))
if $CMD --opt value "${TEST_DIR}/input.fasta" > "${TMPDIR}/out.fasta" 2>&1
then
log "$MCMD: with option OK"
((success++))
else
log "$MCMD: with option failed"
((failed++))
fi
```
## Variables importantes
- **TEST_DIR** - Répertoire du test (données d'entrée)
- **TMPDIR** - Répertoire temporaire (sorties)
- **CMD** - Nom de la commande
- **MCMD** - Nom formaté pour les logs
## Règles d'or
✅ **Entrées** → `${TEST_DIR}/`
✅ **Sorties** → `${TMPDIR}/`
✅ **Toujours rediriger** → `> file 2>&1`
✅ **Incrémenter ntest** → Avant chaque test
✅ **Messages clairs** → Descriptions explicites
❌ **Pas de chemins en dur**
❌ **Pas de /tmp direct**
❌ **Pas de sortie vers TEST_DIR**
❌ **Pas de commandes sans redirection**
## Données de test
Créer un fichier minimal (< 500 bytes) :
```fasta
>seq1
ACGTACGTACGTACGT
>seq2
AAAACCCCGGGGTTTT
>seq3
ATCGATCGATCGATCG
```
## Création rapide
```bash
# 1. Créer le répertoire
mkdir -p obitests/obitools/<commande>
cd obitests/obitools/<commande>
# 2. Créer les données de test
cat > test_data.fasta << 'EOF'
>seq1
ACGTACGTACGTACGT
>seq2
AAAACCCCGGGGTTTT
EOF
# 3. Copier le template dans test.sh
# 4. Adapter le TEST_NAME et CMD
# 5. Ajouter les tests
# 6. Rendre exécutable
chmod +x test.sh
# 7. Tester
./test.sh
```
## Checklist
- [ ] `test.sh` exécutable (`chmod +x`)
- [ ] Test d'aide inclus
- [ ] Données < 1 KB
- [ ] Sorties vers `${TMPDIR}/`
- [ ] Entrées depuis `${TEST_DIR}/`
- [ ] Redirections `2>&1`
- [ ] Messages clairs
- [ ] Testé localement
- [ ] Exit code 0 si succès
## Debug
Conserver TMPDIR pour inspection :
```bash
cleanup() {
echo "Temporary directory: $TMPDIR" 1>&2
# rm -rf "$TMPDIR" # Commenté
...
}
```
Mode verbose :
```bash
set -x # Au début du script
```
## Exemples
**Simple (1 test)** - obimicrosat
```bash
# Juste l'aide
```
**Moyen (4-5 tests)** - obisuperkmer
```bash
# Aide + exécution + validation sortie + contenu
```
**Complet (7+ tests)** - obiuniq
```bash
# Aide + exécution + comparaison CSV + options + multiples cas
```
## Commandes utiles
```bash
# Compter séquences
grep -c "^>" file.fasta
# Fichier non vide
[ -s file ]
# Comparer
diff file1 file2 > /dev/null
# Comparer compressés
zdiff file1.gz file2.gz
# Compter bases
grep -v "^>" file | tr -d '\n' | wc -c
```
## Ce qu'il faut retenir
Un bon test est **COURT**, **RAPIDE** et **SIMPLE** :
- 3-10 tests maximum
- Données < 1 KB
- Exécution < 10 secondes
- Pattern standard respecté

268
blackboard/architechture/obisuperkmer-implementation.md
View File

@@ -1,268 +0,0 @@
# Implémentation de la commande obisuperkmer
## Vue d'ensemble
La commande `obisuperkmer` a été implémentée en suivant l'architecture standard des commandes OBITools décrite dans `architecture-commande-obitools.md`. Cette commande permet d'extraire les super k-mers de fichiers de séquences biologiques.
## Qu'est-ce qu'un super k-mer ?
Un super k-mer est une sous-séquence maximale dans laquelle tous les k-mers consécutifs partagent le même minimiseur. Cette décomposition est utile pour :
- L'indexation efficace de k-mers
- La réduction de la redondance dans les analyses
- L'optimisation de la mémoire pour les structures de données de k-mers
## Structure de l'implémentation
### 1. Package `pkg/obitools/obisuperkmer/`
Le package contient trois fichiers :
#### `obisuperkmer.go`
Documentation du package avec une description de son rôle.
#### `options.go`
Définit les options de ligne de commande :
```go
var _KmerSize = 21 // Taille des k-mers (par défaut 21)
var _MinimizerSize = 11 // Taille des minimiseurs (par défaut 11)
```
**Options CLI disponibles :**
- `--kmer-size` / `-k` : Taille des k-mers (entre m+1 et 31)
- `--minimizer-size` / `-m` : Taille des minimiseurs (entre 1 et k-1)
**Fonctions d'accès :**
- `CLIKmerSize()` : retourne la taille des k-mers
- `CLIMinimizerSize()` : retourne la taille des minimiseurs
- `SetKmerSize(k int)` : définit la taille des k-mers
- `SetMinimizerSize(m int)` : définit la taille des minimiseurs
#### `superkmer.go`
Implémente la logique de traitement :
```go
func CLIExtractSuperKmers(iterator obiiter.IBioSequence) obiiter.IBioSequence
```
Cette fonction :
1. Récupère les paramètres k et m depuis les options CLI
2. Valide les paramètres (m < k, k <= 31, etc.)
3. Crée un worker utilisant `obikmer.SuperKmerWorker(k, m)`
4. Applique le worker en parallèle sur l'itérateur de séquences
5. Retourne un itérateur de super k-mers
### 2. Exécutable `cmd/obitools/obisuperkmer/main.go`
L'exécutable suit le pattern standard minimal :
```go
func main() {
// 1. Génération du parser d'options
optionParser := obioptions.GenerateOptionParser(
"obisuperkmer",
"extract super k-mers from sequence files",
obisuperkmer.OptionSet)
// 2. Parsing des arguments
_, args := optionParser(os.Args)
// 3. Lecture des séquences
sequences, err := obiconvert.CLIReadBioSequences(args...)
obiconvert.OpenSequenceDataErrorMessage(args, err)
// 4. Extraction des super k-mers
superkmers := obisuperkmer.CLIExtractSuperKmers(sequences)
// 5. Écriture des résultats
obiconvert.CLIWriteBioSequences(superkmers, true)
// 6. Attente de la fin du pipeline
obiutils.WaitForLastPipe()
}
```
## Utilisation du package `obikmer`
L'implémentation s'appuie sur le package `obikmer` qui fournit :
### `SuperKmerWorker(k int, m int) obiseq.SeqWorker`
Crée un worker qui :
- Extrait les super k-mers d'une BioSequence
- Retourne une slice de BioSequence, une par super k-mer
- Chaque super k-mer contient les attributs suivants :
```go
// Métadonnées ajoutées à chaque super k-mer :
{
"minimizer_value": uint64, // Valeur canonique du minimiseur
"minimizer_seq": string, // Séquence ADN du minimiseur
"k": int, // Taille des k-mers utilisée
"m": int, // Taille des minimiseurs utilisée
"start": int, // Position de début (0-indexé)
"end": int, // Position de fin (exclusif)
"parent_id": string, // ID de la séquence parente
}
```
### Algorithme sous-jacent
Le package `obikmer` utilise :
- `IterSuperKmers(seq []byte, k int, m int)` : itérateur sur les super k-mers
- Une deque monotone pour suivre les minimiseurs dans une fenêtre glissante
- Complexité temporelle : O(n) où n est la longueur de la séquence
- Complexité spatiale : O(k-m+1) pour la deque
## Exemple d'utilisation
### Ligne de commande
```bash
# Extraction avec paramètres par défaut (k=21, m=11)
obisuperkmer sequences.fasta > superkmers.fasta
# Spécifier les tailles de k-mers et minimiseurs
obisuperkmer -k 25 -m 13 sequences.fasta -o superkmers.fasta
# Avec plusieurs fichiers d'entrée
obisuperkmer --kmer-size 31 --minimizer-size 15 file1.fasta file2.fasta > output.fasta
# Format FASTQ en entrée, FASTA en sortie
obisuperkmer sequences.fastq --fasta-output -o superkmers.fasta
# Avec compression
obisuperkmer sequences.fasta -o superkmers.fasta.gz --compress
```
### Exemple de sortie
Pour une séquence d'entrée :
```
>seq1
ACGTACGTACGTACGTACGTACGT
```
La sortie contiendra plusieurs super k-mers :
```
>seq1_superkmer_0_15 {"minimizer_value":123456,"minimizer_seq":"acgtacgt","k":21,"m":11,"start":0,"end":15,"parent_id":"seq1"}
ACGTACGTACGTACG
>seq1_superkmer_8_24 {"minimizer_value":789012,"minimizer_seq":"gtacgtac","k":21,"m":11,"start":8,"end":24,"parent_id":"seq1"}
TACGTACGTACGTACGT
```
## Options héritées de `obiconvert`
La commande hérite de toutes les options standard d'OBITools :
### Options d'entrée
- `--fasta` : forcer le format FASTA
- `--fastq` : forcer le format FASTQ
- `--ecopcr` : format ecoPCR
- `--embl` : format EMBL
- `--genbank` : format GenBank
- `--input-json-header` : en-têtes JSON
- `--input-OBI-header` : en-têtes OBI
### Options de sortie
- `--out` / `-o` : fichier de sortie (défaut : stdout)
- `--fasta-output` : sortie en format FASTA
- `--fastq-output` : sortie en format FASTQ
- `--json-output` : sortie en format JSON
- `--output-json-header` : en-têtes JSON en sortie
- `--output-OBI-header` / `-O` : en-têtes OBI en sortie
- `--compress` / `-Z` : compression gzip
- `--skip-empty` : ignorer les séquences vides
- `--no-progressbar` : désactiver la barre de progression
## Compilation
Pour compiler la commande :
```bash
cd /chemin/vers/obitools4
go build -o bin/obisuperkmer ./cmd/obitools/obisuperkmer/
```
## Tests
Pour tester la commande :
```bash
# Créer un fichier de test
echo -e ">test\nACGTACGTACGTACGTACGTACGTACGTACGT" > test.fasta
# Exécuter obisuperkmer
obisuperkmer test.fasta
# Vérifier avec des paramètres différents
obisuperkmer -k 15 -m 7 test.fasta
```
## Validation des paramètres
La commande valide automatiquement :
- `1 <= m < k` : le minimiseur doit être plus petit que le k-mer
- `2 <= k <= 31` : contrainte du codage sur 64 bits
- `len(sequence) >= k` : la séquence doit être assez longue
En cas de paramètres invalides, la commande affiche une erreur explicite et s'arrête.
## Intégration avec le pipeline OBITools
La commande s'intègre naturellement dans les pipelines OBITools :
```bash
# Pipeline complet d'analyse
obiconvert sequences.fastq --fasta-output | \
obisuperkmer -k 21 -m 11 | \
obiuniq | \
obigrep -p "minimizer_value>1000" > filtered_superkmers.fasta
```
## Parallélisation
La commande utilise automatiquement :
- `obidefault.ParallelWorkers()` pour le traitement parallèle
- Les workers sont distribués sur les séquences d'entrée
- La parallélisation est transparente pour l'utilisateur
## Conformité avec l'architecture OBITools
L'implémentation respecte tous les principes de l'architecture :
✅ Séparation des responsabilités (package + commande)
✅ Convention de nommage cohérente (CLI*, Set*, _variables)
✅ Réutilisation de `obiconvert` pour l'I/O
✅ Options standard partagées
✅ Pattern Worker pour le traitement
✅ Validation des paramètres
✅ Logging avec `logrus`
✅ Gestion d'erreurs cohérente
✅ Documentation complète
## Fichiers créés
```
pkg/obitools/obisuperkmer/
├── obisuperkmer.go # Documentation du package
├── options.go # Définition des options CLI
└── superkmer.go # Implémentation du traitement
cmd/obitools/obisuperkmer/
└── main.go # Point d'entrée de la commande
```
## Prochaines étapes
1. **Compilation** : Compiler la commande avec `go build`
2. **Tests unitaires** : Créer des tests dans `pkg/obitools/obisuperkmer/superkmer_test.go`
3. **Documentation utilisateur** : Ajouter la documentation de la commande
4. **Intégration CI/CD** : Ajouter aux tests d'intégration
5. **Benchmarks** : Mesurer les performances sur différents jeux de données
## Références
- Architecture des commandes OBITools : `architecture-commande-obitools.md`
- Package `obikmer` : `pkg/obikmer/`
- Tests du package : `pkg/obikmer/superkmer_iter_test.go`

440
blackboard/architechture/obisuperkmer-tests.md
View File

@@ -1,440 +0,0 @@
# Tests automatisés pour obisuperkmer
## Vue d'ensemble
Des tests automatisés ont été créés pour la commande `obisuperkmer` dans le répertoire `obitests/obitools/obisuperkmer/`. Ces tests suivent le pattern standard utilisé par toutes les commandes OBITools et sont conçus pour être exécutés dans un environnement CI/CD.
## Fichiers créés
```
obitests/obitools/obisuperkmer/
├── test.sh # Script de test principal (6.7 KB)
├── test_sequences.fasta # Données de test (117 bytes)
└── README.md # Documentation (4.1 KB)
```
### Taille totale : ~11 KB
Cette taille minimale est idéale pour un dépôt Git et des tests CI/CD rapides.
## Jeu de données de test
### Fichier : `test_sequences.fasta` (117 bytes)
Le fichier contient 3 séquences de 32 nucléotides chacune :
```fasta
>seq1
ACGTACGTACGTACGTACGTACGTACGTACGT
>seq2
AAAACCCCGGGGTTTTAAAACCCCGGGGTTTT
>seq3
ATCGATCGATCGATCGATCGATCGATCGATCG
```
#### Justification du choix
1. **seq1** : Motif répétitif simple (ACGT)
- Teste l'extraction de super k-mers sur une séquence avec faible complexité
- Les minimiseurs devraient être assez réguliers
2. **seq2** : Blocs homopolymères
- Teste le comportement avec des régions de très faible complexité
- Les minimiseurs varieront entre les blocs A, C, G et T
3. **seq3** : Motif différent (ATCG)
- Teste la diversité des super k-mers extraits
- Différent de seq1 pour vérifier la distinction
#### Caractéristiques
- **Longueur** : 32 nucléotides par séquence
- **Taille totale** : 96 nucléotides (3 × 32)
- **Format** : FASTA avec en-têtes JSON compatibles
- **Alphabet** : A, C, G, T uniquement (pas de bases ambiguës)
- **Taille du fichier** : 117 bytes
Avec k=21 (défaut), chaque séquence de 32 bp peut produire :
- 32 - 21 + 1 = 12 k-mers
- Plusieurs super k-mers selon les minimiseurs
## Script de test : `test.sh`
### Structure
Le script suit le pattern standard OBITools :
```bash
#!/bin/bash
TEST_NAME=obisuperkmer
CMD=obisuperkmer
# Variables et fonctions standard
TEST_DIR="..."
OBITOOLS_DIR="..."
TMPDIR="$(mktemp -d)"
ntest=0
success=0
failed=0
cleanup() { ... }
log() { ... }
# Tests (12 au total)
# ...
cleanup
```
### Tests implémentés
#### 1. Test d'aide (`-h`)
```bash
obisuperkmer -h
```
Vérifie que la commande peut afficher son aide sans erreur.
#### 2. Extraction basique avec paramètres par défaut
```bash
obisuperkmer test_sequences.fasta > output_default.fasta
```
Teste l'exécution avec k=21, m=11 (défaut).
#### 3. Vérification de sortie non vide
```bash
[ -s output_default.fasta ]
```
S'assure que la commande produit un résultat.
#### 4. Comptage des super k-mers
```bash
grep -c "^>" output_default.fasta
```
Vérifie qu'au moins un super k-mer a été extrait.
#### 5. Présence des métadonnées
```bash
grep -q "minimizer_value" output_default.fasta
grep -q "minimizer_seq" output_default.fasta
grep -q "parent_id" output_default.fasta
```
Vérifie que les attributs requis sont présents.
#### 6. Extraction avec paramètres personnalisés
```bash
obisuperkmer -k 15 -m 7 test_sequences.fasta > output_k15_m7.fasta
```
Teste la configuration de k et m.
#### 7. Validation des paramètres personnalisés
```bash
grep -q '"k":15' output_k15_m7.fasta
grep -q '"m":7' output_k15_m7.fasta
```
Vérifie que les paramètres sont correctement enregistrés.
#### 8. Format de sortie FASTA
```bash
obisuperkmer --fasta-output test_sequences.fasta > output_fasta.fasta
```
Teste l'option de format explicite.
#### 9. Vérification des IDs
```bash
grep "^>" output_default.fasta | grep -q "superkmer"
```
S'assure que les IDs contiennent "superkmer".
#### 10. Préservation des IDs parents
```bash
grep -q "seq1" output_default.fasta
grep -q "seq2" output_default.fasta
grep -q "seq3" output_default.fasta
```
Vérifie que les IDs des séquences parentes sont préservés.
#### 11. Option de fichier de sortie (`-o`)
```bash
obisuperkmer -o output_file.fasta test_sequences.fasta
```
Teste la redirection vers un fichier.
#### 12. Vérification de création du fichier
```bash
[ -s output_file.fasta ]
```
S'assure que le fichier a été créé.
#### 13. Cohérence des longueurs
```bash
# Vérifie que longueur(output) <= longueur(input)
```
S'assure que les super k-mers ne sont pas plus longs que l'entrée.
### Compteurs
- **ntest** : Nombre de tests exécutés
- **success** : Nombre de tests réussis
- **failed** : Nombre de tests échoués
### Sortie du script
#### En cas de succès
```
========================================
## Results of the obisuperkmer tests:
- 12 tests run
- 12 successfully completed
- 0 failed tests
Cleaning up the temporary directory...
========================================
```
Exit code : **0**
#### En cas d'échec
```
========================================
## Results of the obisuperkmer tests:
- 12 tests run
- 10 successfully completed
- 2 failed tests
Cleaning up the temporary directory...
========================================
```
Exit code : **1**
## Intégration CI/CD
### Exécution automatique
Le script est conçu pour être exécuté automatiquement dans un pipeline CI/CD :
1. Le build produit l'exécutable dans `build/obisuperkmer`
2. Le script de test ajoute `build/` au PATH
3. Les tests s'exécutent
4. Le code de retour indique le succès (0) ou l'échec (1)
### Exemple de configuration CI/CD
```yaml
# .github/workflows/test.yml ou équivalent
test-obisuperkmer:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Build obitools
run: make build
- name: Test obisuperkmer
run: ./obitests/obitools/obisuperkmer/test.sh
```
### Avantages
✅ **Rapidité** : Données de test minimales (117 bytes)
✅ **Fiabilité** : Tests reproductibles
✅ **Isolation** : Utilisation d'un répertoire temporaire
✅ **Nettoyage automatique** : Pas de fichiers résiduels
✅ **Logging** : Messages horodatés et détaillés
✅ **Compatibilité** : Pattern standard OBITools
## Exécution locale
### Prérequis
1. Compiler obisuperkmer :
```bash
cd /chemin/vers/obitools4
go build -o build/obisuperkmer ./cmd/obitools/obisuperkmer/
```
2. Se placer dans le répertoire de test :
```bash
cd obitests/obitools/obisuperkmer
```
3. Exécuter le script :
```bash
./test.sh
```
### Exemple de sortie
```
[obisuperkmer @ Fri Feb 7 13:00:00 CET 2026] Testing obisuperkmer...
[obisuperkmer @ Fri Feb 7 13:00:00 CET 2026] Test directory is /path/to/obitests/obitools/obisuperkmer
[obisuperkmer @ Fri Feb 7 13:00:00 CET 2026] obitools directory is /path/to/build
[obisuperkmer @ Fri Feb 7 13:00:00 CET 2026] Temporary directory is /tmp/tmp.abc123
[obisuperkmer @ Fri Feb 7 13:00:00 CET 2026] files: README.md test.sh test_sequences.fasta
[obisuperkmer @ Fri Feb 7 13:00:01 CET 2026] OBISuperkmer: printing help OK
[obisuperkmer @ Fri Feb 7 13:00:02 CET 2026] OBISuperkmer: basic extraction with default parameters OK
[obisuperkmer @ Fri Feb 7 13:00:02 CET 2026] OBISuperkmer: output file is not empty OK
[obisuperkmer @ Fri Feb 7 13:00:02 CET 2026] OBISuperkmer: extracted 8 super k-mers OK
[obisuperkmer @ Fri Feb 7 13:00:02 CET 2026] OBISuperkmer: super k-mers contain required metadata OK
[obisuperkmer @ Fri Feb 7 13:00:03 CET 2026] OBISuperkmer: extraction with custom k=15, m=7 OK
[obisuperkmer @ Fri Feb 7 13:00:03 CET 2026] OBISuperkmer: custom parameters correctly set in metadata OK
[obisuperkmer @ Fri Feb 7 13:00:03 CET 2026] OBISuperkmer: FASTA output format OK
[obisuperkmer @ Fri Feb 7 13:00:03 CET 2026] OBISuperkmer: super k-mer IDs contain 'superkmer' OK
[obisuperkmer @ Fri Feb 7 13:00:03 CET 2026] OBISuperkmer: parent sequence IDs preserved OK
[obisuperkmer @ Fri Feb 7 13:00:04 CET 2026] OBISuperkmer: output to file with -o option OK
[obisuperkmer @ Fri Feb 7 13:00:04 CET 2026] OBISuperkmer: output file created with -o option OK
[obisuperkmer @ Fri Feb 7 13:00:04 CET 2026] OBISuperkmer: super k-mer total length <= input length OK
========================================
## Results of the obisuperkmer tests:
- 12 tests run
- 12 successfully completed
- 0 failed tests
Cleaning up the temporary directory...
========================================
```
## Debugging des tests
### Conserver les fichiers temporaires
Modifier temporairement la fonction `cleanup()` :
```bash
cleanup() {
echo "Temporary directory: $TMPDIR" 1>&2
# Commenter cette ligne pour conserver les fichiers
# rm -rf "$TMPDIR"
...
}
```
### Activer le mode verbose
Ajouter au début du script :
```bash
set -x # Active l'affichage de toutes les commandes
```
### Tester une seule commande
Extraire et exécuter manuellement :
```bash
export TEST_DIR=/chemin/vers/obitests/obitools/obisuperkmer
export TMPDIR=$(mktemp -d)
obisuperkmer "${TEST_DIR}/test_sequences.fasta" > "${TMPDIR}/output.fasta"
cat "${TMPDIR}/output.fasta"
```
## Ajout de nouveaux tests
Pour ajouter un test supplémentaire :
1. Incrémenter le compteur `ntest`
2. Écrire la condition de test
3. Logger le succès ou l'échec
4. Incrémenter le bon compteur
```bash
((ntest++))
if ma_nouvelle_commande_de_test
then
log "Description du test: OK"
((success++))
else
log "Description du test: failed"
((failed++))
fi
```
## Comparaison avec d'autres tests
### Taille des données de test
| Commande | Taille des données | Nombre de fichiers |
|----------|-------------------|-------------------|
| obiconvert | 925 KB | 1 fichier |
| obiuniq | ~600 bytes | 4 fichiers |
| obimicrosat | 0 bytes | 0 fichiers (génère à la volée) |
| **obisuperkmer** | **117 bytes** | **1 fichier** |
Notre test `obisuperkmer` est parmi les plus légers, ce qui est optimal pour CI/CD.
### Nombre de tests
| Commande | Nombre de tests |
|----------|----------------|
| obiconvert | 3 tests |
| obiuniq | 7 tests |
| obimicrosat | 1 test |
| **obisuperkmer** | **12 tests** |
Notre test `obisuperkmer` offre une couverture complète avec 12 tests différents.
## Couverture de test
Les tests couvrent :
✅ Affichage de l'aide
✅ Exécution basique
✅ Paramètres par défaut (k=21, m=11)
✅ Paramètres personnalisés (k=15, m=7)
✅ Formats de sortie (FASTA)
✅ Redirection vers fichier (`-o`)
✅ Présence des métadonnées
✅ Validation des IDs
✅ Préservation des IDs parents
✅ Cohérence des longueurs
✅ Production de résultats non vides
## Maintenance
### Mise à jour des tests
Si l'implémentation de `obisuperkmer` change :
1. Vérifier que les tests existants passent toujours
2. Ajouter de nouveaux tests pour les nouvelles fonctionnalités
3. Mettre à jour `README.md` si nécessaire
4. Documenter les changements
### Vérification régulière
Exécuter périodiquement :
```bash
cd obitests/obitools/obisuperkmer
./test.sh
```
Ou via l'ensemble des tests :
```bash
cd obitests
for dir in obitools/*/; do
if [ -f "$dir/test.sh" ]; then
echo "Testing $(basename $dir)..."
(cd "$dir" && ./test.sh) || echo "FAILED: $(basename $dir)"
fi
done
```
## Conclusion
Les tests pour `obisuperkmer` sont :
- ✅ **Complets** : 12 tests couvrant toutes les fonctionnalités principales
- ✅ **Légers** : 117 bytes de données de test
- ✅ **Rapides** : Exécution en quelques secondes
- ✅ **Fiables** : Pattern éprouvé utilisé par toutes les commandes OBITools
- ✅ **Maintenables** : Structure claire et documentée
- ✅ **CI/CD ready** : Code de retour approprié et nettoyage automatique
Ils garantissent que la commande fonctionne correctement à chaque commit et facilitent la détection précoce des régressions.

11
cmd/obitools/obiannotate/main.go
View File

@@ -30,11 +30,7 @@ func main() {
// trace.Start(ftrace)
// defer trace.Stop()
optionParser := obioptions.GenerateOptionParser(
"obiannotate",
"edits the sequence annotations",
obiannotate.OptionSet,
)
optionParser := obioptions.GenerateOptionParser(obiannotate.OptionSet)
_, args := optionParser(os.Args)
@@ -42,11 +38,6 @@ func main() {
obiconvert.OpenSequenceDataErrorMessage(args, err)
annotator := obiannotate.CLIAnnotationPipeline()
if obiannotate.CLIHasSetNumberFlag() {
sequences = sequences.NumberSequences(1, !obiconvert.CLINoInputOrder())
}
obiconvert.CLIWriteBioSequences(sequences.Pipe(annotator), true)
obiutils.WaitForLastPipe()

5
cmd/obitools/obiclean/main.go
View File

@@ -11,10 +11,7 @@ import (
)
func main() {
optionParser := obioptions.GenerateOptionParser(
"obiclean",
"",
obiclean.OptionSet)
optionParser := obioptions.GenerateOptionParser(obiclean.OptionSet)
_, args := optionParser(os.Args)

5
cmd/obitools/obicleandb/main.go
View File

@@ -14,10 +14,7 @@ import (
func main() {
obidefault.SetBatchSize(10)
optionParser := obioptions.GenerateOptionParser(
"obicleandb",
"clean-up reference databases",
obicleandb.OptionSet)
optionParser := obioptions.GenerateOptionParser(obicleandb.OptionSet)
_, args := optionParser(os.Args)

5
cmd/obitools/obicomplement/main.go
View File

@@ -11,10 +11,7 @@ import (
)
func main() {
optionParser := obioptions.GenerateOptionParser(
"obicomplement",
"reverse complement of sequences",
obiconvert.OptionSet(true))
optionParser := obioptions.GenerateOptionParser(obiconvert.OptionSet)
_, args := optionParser(os.Args)

5
cmd/obitools/obiconsensus/main.go
View File

@@ -11,10 +11,7 @@ import (
)
func main() {
optionParser := obioptions.GenerateOptionParser(
"obiconsensus",
"ONT reads denoising",
obiconsensus.OptionSet)
optionParser := obioptions.GenerateOptionParser(obiconsensus.OptionSet)
_, args := optionParser(os.Args)

5
cmd/obitools/obiconvert/main.go
View File

@@ -14,10 +14,7 @@ func main() {
obidefault.SetStrictReadWorker(2)
obidefault.SetStrictWriteWorker(2)
optionParser := obioptions.GenerateOptionParser(
"obiconvert",
"convertion of sequence files to various formats",
obiconvert.OptionSet(true))
optionParser := obioptions.GenerateOptionParser(obiconvert.OptionSet)
_, args := optionParser(os.Args)

2
cmd/obitools/obicount/main.go
View File

@@ -28,8 +28,6 @@ func main() {
// defer trace.Stop()
optionParser := obioptions.GenerateOptionParser(
"obicount",
"counts the sequences present in a file of sequences",
obiconvert.InputOptionSet,
obicount.OptionSet,
)

5
cmd/obitools/obicsv/main.go
View File

@@ -10,10 +10,7 @@ import (
)
func main() {
optionParser := obioptions.GenerateOptionParser(
"obicsv",
"converts sequence files to CSV format",
obicsv.OptionSet)
optionParser := obioptions.GenerateOptionParser(obicsv.OptionSet)
_, args := optionParser(os.Args)

5
cmd/obitools/obidemerge/main.go
View File

@@ -15,10 +15,7 @@ func main() {
obidefault.SetStrictReadWorker(2)
obidefault.SetStrictWriteWorker(2)
optionParser := obioptions.GenerateOptionParser(
"obidemerge",
"",
obidemerge.OptionSet)
optionParser := obioptions.GenerateOptionParser(obidemerge.OptionSet)
_, args := optionParser(os.Args)

5
cmd/obitools/obidistribute/main.go
View File

@@ -11,10 +11,7 @@ import (
)
func main() {
optionParser := obioptions.GenerateOptionParser(
"obidistribute",
"divided an input set of sequences into subsets",
obidistribute.OptionSet)
optionParser := obioptions.GenerateOptionParser(obidistribute.OptionSet)
_, args := optionParser(os.Args)

5
cmd/obitools/obigrep/main.go
View File

@@ -30,10 +30,7 @@ func main() {
// trace.Start(ftrace)
// defer trace.Stop()
optionParser := obioptions.GenerateOptionParser(
"obigrep",
"select a subset of sequences on various criteria",
obigrep.OptionSet)
optionParser := obioptions.GenerateOptionParser(obigrep.OptionSet)
_, args := optionParser(os.Args)

5
cmd/obitools/obijoin/main.go
View File

@@ -15,10 +15,7 @@ func main() {
obidefault.SetStrictReadWorker(2)
obidefault.SetStrictWriteWorker(2)
optionParser := obioptions.GenerateOptionParser(
"obijoin",
"merge annotations contained in a file to another file",
obijoin.OptionSet)
optionParser := obioptions.GenerateOptionParser(obijoin.OptionSet)
_, args := optionParser(os.Args)

34
cmd/obitools/obik/main.go
View File

@@ -1,34 +0,0 @@
package main
import (
"context"
"errors"
"os"
log "github.com/sirupsen/logrus"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obik"
"github.com/DavidGamba/go-getoptions"
)
func main() {
defer obiseq.LogBioSeqStatus()
opt, parser := obioptions.GenerateSubcommandParser(
"obik",
"Manage disk-based kmer indices",
obik.OptionSet,
)
_, remaining := parser(os.Args)
err := opt.Dispatch(context.Background(), remaining)
if err != nil {
if errors.Is(err, getoptions.ErrorHelpCalled) {
os.Exit(0)
}
log.Fatalf("Error: %v", err)
}
}

5
cmd/obitools/obikmermatch/main.go
View File

@@ -31,10 +31,7 @@ func main() {
// trace.Start(ftrace)
// defer trace.Stop()
optionParser := obioptions.GenerateOptionParser(
"obikmermatch",
"",
obikmersim.MatchOptionSet)
optionParser := obioptions.GenerateOptionParser(obikmersim.MatchOptionSet)
_, args := optionParser(os.Args)

5
cmd/obitools/obikmersimcount/main.go
View File

@@ -32,10 +32,7 @@ func main() {
// trace.Start(ftrace)
// defer trace.Stop()
optionParser := obioptions.GenerateOptionParser(
"obikmersimcount",
"",
obikmersim.CountOptionSet)
optionParser := obioptions.GenerateOptionParser(obikmersim.CountOptionSet)
_, args := optionParser(os.Args)

5
cmd/obitools/obilandmark/main.go
View File

@@ -11,10 +11,7 @@ import (
)
func main() {
optionParser := obioptions.GenerateOptionParser(
"obilandmark",
"",
obilandmark.OptionSet)
optionParser := obioptions.GenerateOptionParser(obilandmark.OptionSet)
_, args := optionParser(os.Args)

2
cmd/obitools/obimatrix/main.go
View File

@@ -31,8 +31,6 @@ func main() {
// defer trace.Stop()
optionParser := obioptions.GenerateOptionParser(
"obimatrix",
"",
obimatrix.OptionSet,
)

42
cmd/obitools/obimicroasm/main.go Normal file
View File

@@ -0,0 +1,42 @@
package main
import (
"os"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiformats"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obimicroasm"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
)
func main() {
// go tool pprof -http=":8000" ./obipairing ./cpu.pprof
// f, err := os.Create("cpu.pprof")
// if err != nil {
// log.Fatal(err)
// }
// pprof.StartCPUProfile(f)
// defer pprof.StopCPUProfile()
// go tool trace cpu.trace
// ftrace, err := os.Create("cpu.trace")
// if err != nil {
// log.Fatal(err)
// }
// trace.Start(ftrace)
// defer trace.Stop()
optionParser := obioptions.GenerateOptionParser(obimicroasm.OptionSet)
optionParser(os.Args)
obidefault.SetStrictReadWorker(2)
obidefault.SetStrictWriteWorker(2)
seq := obimicroasm.CLIAssemblePCR()
println(obiformats.FormatFasta(seq, obiformats.FormatFastSeqJsonHeader))
obiutils.WaitForLastPipe()
}

5
cmd/obitools/obimicrosat/main.go
View File

@@ -30,10 +30,7 @@ func main() {
// trace.Start(ftrace)
// defer trace.Stop()
optionParser := obioptions.GenerateOptionParser(
"obimicrosat",
"looks for microsatellites sequences in a sequence file",
obimicrosat.OptionSet)
optionParser := obioptions.GenerateOptionParser(obimicrosat.OptionSet)
_, args := optionParser(os.Args)

5
cmd/obitools/obimultiplex/main.go
View File

@@ -28,10 +28,7 @@ func main() {
// trace.Start(ftrace)
// defer trace.Stop()
optionParser := obioptions.GenerateOptionParser(
"obimultiplex",
"demultiplex amplicons",
obimultiplex.OptionSet)
optionParser := obioptions.GenerateOptionParser(obimultiplex.OptionSet)
_, args := optionParser(os.Args)

5
cmd/obitools/obipairing/main.go
View File

@@ -30,10 +30,7 @@ func main() {
// trace.Start(ftrace)
// defer trace.Stop()
optionParser := obioptions.GenerateOptionParser(
"obipairing",
"align forward with reverse reads with paired reads",
obipairing.OptionSet)
optionParser := obioptions.GenerateOptionParser(obipairing.OptionSet)
optionParser(os.Args)

5
cmd/obitools/obipcr/main.go
View File

@@ -29,10 +29,7 @@ func main() {
obidefault.SetParallelFilesRead(obidefault.ParallelWorkers() / 4)
obidefault.SetBatchSize(10)
optionParser := obioptions.GenerateOptionParser(
"obipcr",
"simulates a PCR on a sequence files",
obipcr.OptionSet)
optionParser := obioptions.GenerateOptionParser(obipcr.OptionSet)
_, args := optionParser(os.Args)

5
cmd/obitools/obireffamidx/main.go
View File

@@ -11,10 +11,7 @@ import (
)
func main() {
optionParser := obioptions.GenerateOptionParser(
"obireffamidx",
"",
obirefidx.OptionSet)
optionParser := obioptions.GenerateOptionParser(obirefidx.OptionSet)
_, args := optionParser(os.Args)

5
cmd/obitools/obirefidx/main.go
View File

@@ -11,10 +11,7 @@ import (
)
func main() {
optionParser := obioptions.GenerateOptionParser(
"obirefidx",
"",
obirefidx.OptionSet)
optionParser := obioptions.GenerateOptionParser(obirefidx.OptionSet)
_, args := optionParser(os.Args)

5
cmd/obitools/obiscript/main.go
View File

@@ -31,10 +31,7 @@ func main() {
// trace.Start(ftrace)
// defer trace.Stop()
optionParser := obioptions.GenerateOptionParser(
"obiscript",
"executes a lua script on the input sequences",
obiscript.OptionSet)
optionParser := obioptions.GenerateOptionParser(obiscript.OptionSet)
_, args := optionParser(os.Args)

5
cmd/obitools/obisplit/main.go
View File

@@ -31,10 +31,7 @@ func main() {
// trace.Start(ftrace)
// defer trace.Stop()
optionParser := obioptions.GenerateOptionParser(
"obisplit",
"",
obisplit.OptionSet)
optionParser := obioptions.GenerateOptionParser(obisplit.OptionSet)
_, args := optionParser(os.Args)

5
cmd/obitools/obisummary/main.go
View File

@@ -33,10 +33,7 @@ func main() {
// trace.Start(ftrace)
// defer trace.Stop()
optionParser := obioptions.GenerateOptionParser(
"obisummary",
"resume main information from a sequence file",
obisummary.OptionSet)
optionParser := obioptions.GenerateOptionParser(obisummary.OptionSet)
_, args := optionParser(os.Args)

14
cmd/obitools/obitag/main.go
View File

@@ -11,7 +11,6 @@ import (
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obitag"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obitaxonomy"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
@@ -40,10 +39,7 @@ func main() {
obidefault.SetStrictWriteWorker(1)
obidefault.SetBatchSize(10)
optionParser := obioptions.GenerateOptionParser(
"obitag",
"realizes taxonomic assignment",
obitag.OptionSet)
optionParser := obioptions.GenerateOptionParser(obitag.OptionSet)
_, args := optionParser(os.Args)
@@ -59,7 +55,7 @@ func main() {
}
if taxo == nil {
taxo, err = references.ExtractTaxonomy(nil, obitaxonomy.CLINewickWithLeaves())
taxo, err = references.ExtractTaxonomy(nil)
if err != nil {
log.Fatalf("No taxonomy specified or extractable from reference database: %v", err)
@@ -74,12 +70,10 @@ func main() {
var identified obiiter.IBioSequence
fsrb := fs.Rebatch(obidefault.BatchSize())
if obitag.CLIGeometricMode() {
identified = obitag.CLIGeomAssignTaxonomy(fsrb, references, taxo)
identified = obitag.CLIGeomAssignTaxonomy(fs, references, taxo)
} else {
identified = obitag.CLIAssignTaxonomy(fsrb, references, taxo)
identified = obitag.CLIAssignTaxonomy(fs, references, taxo)
}
obiconvert.CLIWriteBioSequences(identified, true)

5
cmd/obitools/obitagpcr/main.go
View File

@@ -33,10 +33,7 @@ func main() {
obidefault.SetWorkerPerCore(1)
optionParser := obioptions.GenerateOptionParser(
"obitagpcr",
"split a paired raw read data set per sample",
obitagpcr.OptionSet)
optionParser := obioptions.GenerateOptionParser(obitagpcr.OptionSet)
optionParser(os.Args)
pairs, err := obipairing.CLIPairedSequence()

46
cmd/obitools/obitaxonomy/main.go
View File

@@ -4,11 +4,9 @@ import (
"os"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiitercsv"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obicsv"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obitaxonomy"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
@@ -16,59 +14,30 @@ import (
)
func main() {
optionParser := obioptions.GenerateOptionParser(
"obitaxonomy",
"manipulates and queries taxonomy",
obitaxonomy.OptionSet)
optionParser := obioptions.GenerateOptionParser(obitaxonomy.OptionSet)
_, args := optionParser(os.Args)
var iterator *obitax.ITaxon
if obitaxonomy.CLIDownloadNCBI() {
switch {
case obitaxonomy.CLIDownloadNCBI():
err := obitaxonomy.CLIDownloadNCBITaxdump()
if err != nil {
log.Errorf("Cannot download NCBI taxonomy: %s", err.Error())
os.Exit(1)
}
os.Exit(0)
}
if !obidefault.HasSelectedTaxonomy() {
log.Fatal("you must indicate a taxonomy using the -t or --taxonomy option")
}
switch {
case obitaxonomy.CLIAskForRankList():
newIter := obiitercsv.NewICSVRecord()
newIter.Add(1)
newIter.AppendField("rank")
go func() {
ranks := obitax.DefaultTaxonomy().RankList()
data := make([]obiitercsv.CSVRecord, len(ranks))
for i, rank := range ranks {
record := make(obiitercsv.CSVRecord)
record["rank"] = rank
data[i] = record
}
newIter.Push(obiitercsv.MakeCSVRecordBatch(obitax.DefaultTaxonomy().Name(), 0, data))
newIter.Close()
newIter.Done()
}()
obicsv.CLICSVWriter(newIter, true)
obiutils.WaitForLastPipe()
os.Exit(0)
case obitaxonomy.CLIExtractTaxonomy():
iter, err := obiconvert.CLIReadBioSequences(args...)
iter = iter.NumberSequences(1, true)
if err != nil {
log.Fatalf("Cannot extract taxonomy: %v", err)
}
taxonomy, err := iter.ExtractTaxonomy(obitaxonomy.CLINewickWithLeaves())
taxonomy, err := iter.ExtractTaxonomy()
if err != nil {
log.Fatalf("Cannot extract taxonomy: %v", err)
@@ -130,12 +99,7 @@ func main() {
}
iterator = obitaxonomy.CLITaxonRestrictions(iterator)
if obitaxonomy.CLIAsNewick() {
obitaxonomy.CLINewickWriter(iterator, true)
} else {
obitaxonomy.CLICSVTaxaWriter(iterator, true)
}
obitaxonomy.CLICSVTaxaWriter(iterator, true)
obiutils.WaitForLastPipe()

5
cmd/obitools/obiuniq/main.go
View File

@@ -33,10 +33,7 @@ func main() {
obidefault.SetBatchSize(10)
obidefault.SetReadQualities(false)
optionParser := obioptions.GenerateOptionParser(
"obiuniq",
"dereplicate sequence data sets",
obiuniq.OptionSet)
optionParser := obioptions.GenerateOptionParser(obiuniq.OptionSet)
_, args := optionParser(os.Args)

4
cmd/test/main.go
View File

@@ -3,13 +3,13 @@ package main
import (
"os"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiformats"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
)
func main() {
obiformats.DetectTaxonomyFormat(os.Args[1])
obitax.DetectTaxonomyFormat(os.Args[1])
println(obiutils.RemoveAllExt("toto/tutu/test.txt"))
println(obiutils.Basename("toto/tutu/test.txt"))

23
git-hooks/pre-push
View File

@@ -1,23 +0,0 @@
#!/bin/bash
remote="$1"
#url="$2"
log() {
echo -e "[Pre-Push tests @ $(date)] $*" 1>&2
}
current_branch=$(git symbolic-ref --short head)
cmd="make githubtests"
if [[ $current_branch = "master" ]]; then
log "you are on $current_branch, running build test"
if ! eval "$cmd"; then
log "Pre-push tests failed $cmd"
exit 1
fi
fi
log "Tests are OK, ready to push on $remote"
exit 0

49
go.mod
View File

@@ -1,50 +1,55 @@
module git.metabarcoding.org/obitools/obitools4/obitools4
go 1.26.1
go 1.23.1
require (
github.com/DavidGamba/go-getoptions v0.33.0
github.com/PaesslerAG/gval v1.2.4
github.com/DavidGamba/go-getoptions v0.28.0
github.com/PaesslerAG/gval v1.2.2
github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df
github.com/buger/jsonparser v1.1.1
github.com/chen3feng/stl4go v0.1.1
github.com/dlclark/regexp2 v1.11.5
github.com/goccy/go-json v0.10.6
github.com/dlclark/regexp2 v1.11.4
github.com/goccy/go-json v0.10.3
github.com/klauspost/pgzip v1.2.6
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
github.com/pelletier/go-toml/v2 v2.2.4
github.com/rrethy/ahocorasick v1.0.0
github.com/schollz/progressbar/v3 v3.19.0
github.com/sirupsen/logrus v1.9.4
github.com/schollz/progressbar/v3 v3.13.1
github.com/sirupsen/logrus v1.9.3
github.com/stretchr/testify v1.10.0
github.com/tevino/abool/v2 v2.1.0
github.com/yuin/gopher-lua v1.1.1
golang.org/x/exp v0.0.0-20260312153236-7ab1446f8b90
gonum.org/v1/gonum v0.17.0
golang.org/x/exp v0.0.0-20231006140011-7918f672742d
gonum.org/v1/gonum v0.14.0
gopkg.in/yaml.v3 v3.0.1
scientificgo.org/special v0.0.0
)
require (
github.com/Clever/csvlint v0.3.0 // indirect
github.com/TuftsBCB/io v0.0.0-20140121014543-22b94e9b23f9 // indirect
github.com/buger/jsonparser v1.1.1 // indirect
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/goombaio/orderedmap v0.0.0-20180925151256-3da0e2f905f9 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/ef-ds/deque/v2 v2.0.2 // indirect
github.com/goombaio/orderedmap v0.0.0-20180924084748-ba921b7e2419 // indirect
github.com/kr/pretty v0.3.0 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rogpeppe/go-internal v1.12.0 // indirect
github.com/rogpeppe/go-internal v1.6.1 // indirect
go.etcd.io/bbolt v1.4.0 // indirect
)
require (
github.com/dsnet/compress v0.0.1
github.com/gabriel-vasile/mimetype v1.4.13
github.com/gabriel-vasile/mimetype v1.4.3
github.com/goombaio/orderedset v0.0.0-20180925151225-8e67b20a9b77
github.com/klauspost/compress v1.18.4
github.com/mattn/go-runewidth v0.0.21 // indirect
github.com/klauspost/compress v1.17.2
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/shopspring/decimal v1.4.0 // indirect
github.com/ulikunitz/xz v0.5.15
golang.org/x/sys v0.42.0 // indirect
golang.org/x/term v0.41.0 // indirect
github.com/rivo/uniseg v0.4.4 // indirect
github.com/shopspring/decimal v1.3.1 // indirect
github.com/ulikunitz/xz v0.5.11
golang.org/x/net v0.17.0 // indirect
golang.org/x/sys v0.29.0 // indirect
golang.org/x/term v0.13.0 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c
)

117
go.sum
View File

@@ -1,96 +1,117 @@
github.com/DavidGamba/go-getoptions v0.33.0 h1:8xCPH87Yy5avYenygyHVlqqm8RpymH0YFe4a7IWlarE=
github.com/DavidGamba/go-getoptions v0.33.0/go.mod h1:zE97E3PR9P3BI/HKyNYgdMlYxodcuiC6W68KIgeYT84=
github.com/PaesslerAG/gval v1.2.4 h1:rhX7MpjJlcxYwL2eTTYIOBUyEKZ+A96T9vQySWkVUiU=
github.com/PaesslerAG/gval v1.2.4/go.mod h1:XRFLwvmkTEdYziLdaCeCa5ImcGVrfQbeNUbVR+C6xac=
github.com/Clever/csvlint v0.3.0 h1:58WEFXWy+i0fCbxTXscR2QwYESRuAUFjEGLgZs6j2iU=
github.com/Clever/csvlint v0.3.0/go.mod h1:+wLRuW/bI8NhpRoeyUBxqKsK35OhvgJhXHSWdKp5XJU=
github.com/DavidGamba/go-getoptions v0.28.0 h1:18wgEvfZdrlfIhVDGEBO3Dl0fkOyXqXLa0tLMCKxM1c=
github.com/DavidGamba/go-getoptions v0.28.0/go.mod h1:zE97E3PR9P3BI/HKyNYgdMlYxodcuiC6W68KIgeYT84=
github.com/PaesslerAG/gval v1.2.2 h1:Y7iBzhgE09IGTt5QgGQ2IdaYYYOU134YGHBThD+wm9E=
github.com/PaesslerAG/gval v1.2.2/go.mod h1:XRFLwvmkTEdYziLdaCeCa5ImcGVrfQbeNUbVR+C6xac=
github.com/PaesslerAG/jsonpath v0.1.0 h1:gADYeifvlqK3R3i2cR5B4DGgxLXIPb3TRTH1mGi0jPI=
github.com/PaesslerAG/jsonpath v0.1.0/go.mod h1:4BzmtoM/PI8fPO4aQGIusjGxGir2BzcV0grWtFzq1Y8=
github.com/TuftsBCB/io v0.0.0-20140121014543-22b94e9b23f9 h1:Zc1/GNsUpgZR9qm1EmRSKrnOHA7CCd0bIzGdq0cREN0=
github.com/TuftsBCB/io v0.0.0-20140121014543-22b94e9b23f9/go.mod h1:PZyV4WA3NpqtezSY0h6E6NARAmdDm0qwrydveOyR5Gc=
github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df h1:GSoSVRLoBaFpOOds6QyY1L8AX7uoY+Ln3BHc22W40X0=
github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df/go.mod h1:hiVxq5OP2bUGBRNS3Z/bt/reCLFNbdcST6gISi1fiOM=
github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
github.com/chen3feng/stl4go v0.1.1 h1:0L1+mDw7pomftKDruM23f1mA7miavOj6C6MZeadzN2Q=
github.com/chen3feng/stl4go v0.1.1/go.mod h1:5ml3psLgETJjRJnMbPE+JiHLrCpt+Ajc2weeTECXzWU=
github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM=
github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY=
github.com/clipperhouse/uax29/v2 v2.2.0 h1:ChwIKnQN3kcZteTXMgb1wztSgaU+ZemkgWdohwgs8tY=
github.com/clipperhouse/uax29/v2 v2.2.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ=
github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo=
github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q=
github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo=
github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM=
github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU=
github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/ef-ds/deque/v2 v2.0.2 h1:GQtDK1boBMu/qsNbSLQsqzwNptaioxZI39X3UxT5ALA=
github.com/ef-ds/deque/v2 v2.0.2/go.mod h1:hoZy4VooWLhRT4uS+sSCilfgBQUNptJU2FGqr08a5sc=
github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
github.com/goombaio/orderedmap v0.0.0-20180924084748-ba921b7e2419 h1:SajEQ6tktpF9SRIuzbiPOX9AEZZ53Bvw0k9Mzrts8Lg=
github.com/goombaio/orderedmap v0.0.0-20180924084748-ba921b7e2419/go.mod h1:YKu81H3RSd1cFh0d7NhvUoTtUC9IY/vBX0WUQb1/o4Y=
github.com/goombaio/orderedmap v0.0.0-20180925151256-3da0e2f905f9 h1:vFjPvFavIiDY71bQ9HIxPQBANvNl1SmFC4fgg5xRkho=
github.com/goombaio/orderedmap v0.0.0-20180925151256-3da0e2f905f9/go.mod h1:YKu81H3RSd1cFh0d7NhvUoTtUC9IY/vBX0WUQb1/o4Y=
github.com/goombaio/orderedset v0.0.0-20180925151225-8e67b20a9b77 h1:4dvq1tGHn1Y9KSRY0OZ24Khki4+4U+ZrA//YYsdUlJU=
github.com/goombaio/orderedset v0.0.0-20180925151225-8e67b20a9b77/go.mod h1:HPelMYpOyy0XvglpBbmZ3krZpwaHmszj/vQNlnETPTM=
github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU=
github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/mattn/go-runewidth v0.0.21 h1:jJKAZiQH+2mIinzCJIaIG9Be1+0NR+5sz/lYEEjdM8w=
github.com/mattn/go-runewidth v0.0.21/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rrethy/ahocorasick v1.0.0 h1:YKkCB+E5PXc0xmLfMrWbfNht8vG9Re97IHSWZk/Lk8E=
github.com/rrethy/ahocorasick v1.0.0/go.mod h1:nq8oScE7Vy1rOppoQxpQiiDmPHuKCuk9rXrNcxUV3R0=
github.com/schollz/progressbar/v3 v3.19.0 h1:Ea18xuIRQXLAUidVDox3AbwfUhD0/1IvohyTutOIFoc=
github.com/schollz/progressbar/v3 v3.19.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
github.com/schollz/progressbar/v3 v3.13.1 h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE=
github.com/schollz/progressbar/v3 v3.13.1/go.mod h1:xvrbki8kfT1fzWzBT/UZd9L6GA+jdL7HAgq2RFnO6fQ=
github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/tevino/abool/v2 v2.1.0 h1:7w+Vf9f/5gmKT4m4qkayb33/92M+Um45F2BkHOR+L/c=
github.com/tevino/abool/v2 v2.1.0/go.mod h1:+Lmlqk6bHDWHqN1cbxqhwEAwMPXgc8I1SDEamtseuXY=
github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY=
github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
golang.org/x/exp v0.0.0-20260312153236-7ab1446f8b90 h1:jiDhWWeC7jfWqR9c/uplMOqJ0sbNlNWv0UkzE0vX1MA=
golang.org/x/exp v0.0.0-20260312153236-7ab1446f8b90/go.mod h1:xE1HEv6b+1SCZ5/uscMRjUBKtIxworgEcEi+/n9NQDQ=
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU=
golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A=
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk=
go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk=
golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
gonum.org/v1/gonum v0.14.0 h1:2NiG67LD1tEH0D7kM+ps2V+fXmsAnpUeec7n8tcr4S0=
gonum.org/v1/gonum v0.14.0/go.mod h1:AoWeoz0becf9QMWtE8iWXNXc27fK4fNeHNf/oMejGfU=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
scientificgo.org/special v0.0.0 h1:P6WJkECo6tgtvZAEfNXl+KEB9ReAatjKAeX8U07mjSc=

2
go.work
View File

@@ -1,3 +1,3 @@
go 1.26.1
go 1.23.1
use .

19
go.work.sum
View File

@@ -5,8 +5,6 @@ github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGW
github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/logex v1.2.0 h1:+eqR0HfOetur4tgnC8ftU5imRnhi4te+BadWS95c5AM=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/chzyer/test v0.0.0-20210722231415-061457976a23 h1:dZ0/VyGgQdVGAss6Ju0dt5P0QltE0SFY5Woh6hbIfiQ=
@@ -19,47 +17,42 @@ github.com/go-latex/latex v0.0.0-20230307184459-12ec69307ad9 h1:NxXI5pTAtpEaU49b
github.com/go-latex/latex v0.0.0-20230307184459-12ec69307ad9/go.mod h1:gWuR/CrFDDeVRFQwHPvsv9soJVB/iqymhuZQuJ3a9OM=
github.com/go-pdf/fpdf v0.6.0 h1:MlgtGIfsdMEEQJr2le6b/HNr1ZlQwxyWr77r2aj2U/8=
github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M=
github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow=
github.com/goccmack/gocc v0.0.0-20230228185258-2292f9e40198 h1:FSii2UQeSLngl3jFoR4tUKZLprO7qUlh/TKKticc0BM=
github.com/goccmack/gocc v0.0.0-20230228185258-2292f9e40198/go.mod h1:DTh/Y2+NbnOVVoypCCQrovMPDKUGp4yZpSbWg5D0XIM=
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g=
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
github.com/google/go-cmdtest v0.4.1-0.20220921163831-55ab3332a786/go.mod h1:apVn/GCasLZUVpAJ6oWAuyP7Ne7CEsQbTnc0plM3m+o=
github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/safehtml v0.1.0/go.mod h1:L4KWwDsUJdECRAEpZoBn3O64bQaywRscowZjJAzjHnU=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2 h1:rcanfLhLDA8nozr/K289V1zcntHr3V+SHlXwzz1ZI2g=
github.com/jba/templatecheck v0.7.1/go.mod h1:n1Etw+Rrw1mDDD8dDRsEKTwMZsJ98EkktgNJC6wLUGo=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 h1:qGQQKEcAR99REcMpsXCp3lJ03zYT1PkRd3kQGPn9GVg=
github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE=
github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
github.com/smallnest/goroutine v1.1.1/go.mod h1:Fp8f6ZReubfdj0m4+NcUnW4IsAqKa+Pnrv9opEiD43E=
github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.etcd.io/gofail v0.2.0/go.mod h1:nL3ILMGfkXTekKI3clMBNazKnjUZjYLKmBHzsVAnC1o=
golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
golang.org/x/image v0.6.0 h1:bR8b5okrPI3g/gyZakLZHeWxAR8Dn5CyxXv1hLH5g/4=
golang.org/x/image v0.6.0/go.mod h1:MXLdDR43H7cDJq5GEGXEVeeNhPgi+YYEQ2pC1byI1x0=
golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY=
golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc=
golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
golang.org/x/tools v0.15.0/go.mod h1:hpksKq4dtpQWS1uQ61JkdqWM3LscIS6Slf+VVkm+wQk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc=
gonum.org/v1/plot v0.10.1 h1:dnifSs43YJuNMDzB7v8wV64O4ABBHReuAVAoBxqBqS4=
gonum.org/v1/plot v0.10.1/go.mod h1:VZW5OlhkL1mysU9vaqNHnsy86inf6Ot+jB3r+BczCEo=

220
install_obitools.sh
View File

@@ -1,58 +1,27 @@
#!/bin/bash
# Default values
URL="https://go.dev/dl/"
GITHUB_REPO="https://github.com/metabarcoding/obitools4"
INSTALL_DIR="/usr/local"
OBITOOLS_PREFIX=""
VERSION=""
LIST_VERSIONS=false
JOBS=1
# default values
URL="https://go.dev/dl/"
OBIURL4="https://github.com/metabarcoding/obitools4/archive/refs/heads/master.zip"
INSTALL_DIR="/usr/local"
OBITOOLS_PREFIX=""
# Help message
# help message
function display_help {
echo "Usage: $0 [OPTIONS]"
echo ""
echo "Options:"
echo " -i, --install-dir Directory where obitools are installed "
echo " (e.g., use /usr/local not /usr/local/bin)."
echo " (as example use /usr/local not /usr/local/bin)."
echo " -p, --obitools-prefix Prefix added to the obitools command names if you"
echo " want to have several versions of obitools at the"
echo " same time on your system (e.g., -p g will produce "
echo " same time on your system (as example -p g will produce "
echo " gobigrep command instead of obigrep)."
echo " -v, --version Install a specific version (e.g., 4.4.8)."
echo " If not specified, installs the latest version."
echo " -j, --jobs Number of parallel jobs for compilation (default: 1)."
echo " -l, --list List all available versions and exit."
echo " -h, --help Display this help message."
echo ""
echo "Examples:"
echo " $0 # Install latest version"
echo " $0 -l # List available versions"
echo " $0 -v 4.4.8 # Install specific version"
echo " $0 -i /opt/local # Install to custom directory"
}
# List available versions from GitHub releases
function list_versions {
echo "Fetching available versions..." 1>&2
echo ""
curl -s "https://api.github.com/repos/metabarcoding/obitools4/releases" \
| grep '"tag_name":' \
| sed -E 's/.*"tag_name": "Release_([0-9.]+)".*/\1/' \
| sort -V -r
}
# Get latest version from GitHub releases
function get_latest_version {
curl -s "https://api.github.com/repos/metabarcoding/obitools4/releases" \
| grep '"tag_name":' \
| sed -E 's/.*"tag_name": "Release_([0-9.]+)".*/\1/' \
| sort -V -r \
| head -1
}
# Parse command line arguments
while [ "$#" -gt 0 ]; do
case "$1" in
-i|--install-dir)
@@ -63,100 +32,50 @@ while [ "$#" -gt 0 ]; do
OBITOOLS_PREFIX="$2"
shift 2
;;
-v|--version)
VERSION="$2"
shift 2
;;
-j|--jobs)
JOBS="$2"
shift 2
;;
-l|--list)
LIST_VERSIONS=true
shift
;;
-h|--help)
display_help
display_help 1>&2
exit 0
;;
*)
echo "Error: Unsupported option $1" 1>&2
display_help 1>&2
echo "Error: Unsupported option $1" 1>&2
exit 1
;;
esac
done
# List versions and exit if requested
if [ "$LIST_VERSIONS" = true ]; then
echo "Available OBITools4 versions:"
echo "=============================="
list_versions
exit 0
fi
# Determine version to install
if [ -z "$VERSION" ]; then
echo "Fetching latest version..." 1>&2
VERSION=$(get_latest_version)
if [ -z "$VERSION" ]; then
echo "Error: Could not determine latest version" 1>&2
exit 1
fi
echo "Latest version: $VERSION" 1>&2
else
echo "Installing version: $VERSION" 1>&2
fi
# Construct source URL for the specified version
OBIURL4="${GITHUB_REPO}/archive/refs/tags/Release_${VERSION}.zip"
# The directory from where the script is run
# the directory from where the script is run
DIR="$(pwd)"
# Create temporary directory
# the temp directory used, within $DIR
# omit the -p parameter to create a temporal directory in the default location
# WORK_DIR=$(mktemp -d -p "$DIR" "obitools4.XXXXXX" 2> /dev/null || \
# mktemp -d -t "$DIR" "obitools4.XXXXXX")
WORK_DIR=$(mktemp -d "obitools4.XXXXXX")
# Check if tmp dir was created
# check if tmp dir was created
if [[ ! "$WORK_DIR" || ! -d "$WORK_DIR" ]]; then
echo "Could not create temp dir" 1>&2
echo "Could not create temp dir" 1>&2
exit 1
fi
mkdir -p "${WORK_DIR}/cache" \
|| (echo "Cannot create ${WORK_DIR}/cache directory" 1>&2
exit 1)
# Create installation directory
if ! mkdir -p "${INSTALL_DIR}/bin" 2>/dev/null; then
if [ ! -w "$(dirname "${INSTALL_DIR}")" ] && [ ! -w "${INSTALL_DIR}" ]; then
echo "Please enter your password for installing obitools in ${INSTALL_DIR}" 1>&2
sudo mkdir -p "${INSTALL_DIR}/bin"
else
echo "Error: Could not create ${INSTALL_DIR}/bin (check path or disk space)" 1>&2
exit 1
fi
fi
mkdir -p "${INSTALL_DIR}/bin" 2> /dev/null \
|| (echo "Please enter your password for installing obitools in ${INSTALL_DIR}" 1>&2
sudo mkdir -p "${INSTALL_DIR}/bin")
if [[ ! -d "${INSTALL_DIR}/bin" ]]; then
echo "Could not create ${INSTALL_DIR}/bin directory for installing obitools" 1>&2
echo "Could not create ${INSTALL_DIR}/bin directory for installing obitools" 1>&2
exit 1
fi
INSTALL_DIR="$(cd ${INSTALL_DIR} && pwd)"
INSTALL_DIR="$(cd $INSTALL_DIR && pwd)"
echo "================================" 1>&2
echo "OBITools4 Installation" 1>&2
echo "================================" 1>&2
echo "VERSION=$VERSION" 1>&2
echo "WORK_DIR=$WORK_DIR" 1>&2
echo "INSTALL_DIR=$INSTALL_DIR" 1>&2
echo "OBITOOLS_PREFIX=$OBITOOLS_PREFIX" 1>&2
echo "================================" 1>&2
echo WORK_DIR=$WORK_DIR 1>&2
echo INSTALL_DIR=$INSTALL_DIR 1>&2
echo OBITOOLS_PREFIX=$OBITOOLS_PREFIX 1>&2
pushd "$WORK_DIR" > /dev/null || exit
pushd "$WORK_DIR"|| exit
# Detect OS and architecture
OS=$(uname -a | awk '{print $1}')
ARCH=$(uname -m)
@@ -168,9 +87,7 @@ if [[ "$ARCH" == "aarch64" ]] ; then
ARCH="arm64"
fi
# Download and install Go
echo "Downloading Go..." 1>&2
GOFILE=$(curl -s "$URL" \
GOFILE=$(curl "$URL" \
| grep 'class="download"' \
| grep "\.tar\.gz" \
| sed -E 's@^.*/dl/(go[1-9].+\.tar\.gz)".*$@\1@' \
@@ -178,86 +95,35 @@ GOFILE=$(curl -s "$URL" \
| grep -i "$ARCH" \
| head -1)
GOURL=$(curl -s "${URL}${GOFILE}" \
GOURL=$(curl "${URL}${GOFILE}" \
| sed -E 's@^.*href="(.*\.tar\.gz)".*$@\1@')
echo "Installing Go from: $GOURL" 1>&2
echo "Install GO from : $GOURL" 1>&2
curl --progress-bar "$GOURL" | tar zxf -
curl "$GOURL" \
| tar zxf -
export GOROOT="$(pwd)/go"
PATH="${GOROOT}/bin:$PATH"
PATH="$(pwd)/go/bin:$PATH"
export PATH
export GOPATH="$(pwd)/gopath"
export GOCACHE="$(pwd)/cache"
export GOTOOLCHAIN=local
echo "GOROOT=$GOROOT" 1>&2
echo "GOCACHE=$GOCACHE" 1>&2
mkdir -p "$GOPATH" "$GOCACHE"
curl -L "$OBIURL4" > master.zip
unzip master.zip
# Download OBITools4 source
echo "Downloading OBITools4 v${VERSION}..." 1>&2
echo "Source URL: $OBIURL4" 1>&2
echo "Install OBITOOLS from : $OBIURL4"
if ! curl --progress-bar -L "$OBIURL4" > obitools4.zip; then
echo "Error: Could not download OBITools4 version ${VERSION}" 1>&2
echo "Please check that this version exists with: $0 --list" 1>&2
exit 1
fi
cd obitools4-master || exit
unzip -q obitools4.zip
# Find the extracted directory
OBITOOLS_DIR=$(ls -d obitools4-* 2>/dev/null | head -1)
if [ -z "$OBITOOLS_DIR" ] || [ ! -d "$OBITOOLS_DIR" ]; then
echo "Error: Could not find extracted OBITools4 directory" 1>&2
exit 1
fi
echo "Building OBITools4..." 1>&2
cd "$OBITOOLS_DIR" || exit
mkdir -p vendor
# Build with or without prefix
if [[ -z "$OBITOOLS_PREFIX" ]] ; then
make -j"${JOBS}" obitools GOFLAGS="-buildvcs=false"
make
else
make -j"${JOBS}" obitools GOFLAGS="-buildvcs=false" OBITOOLS_PREFIX="${OBITOOLS_PREFIX}"
make OBITOOLS_PREFIX="${OBITOOLS_PREFIX}"
fi
# Install binaries
echo "Installing binaries to ${INSTALL_DIR}/bin..." 1>&2
if ! cp build/* "${INSTALL_DIR}/bin" 2>/dev/null; then
if [ ! -w "${INSTALL_DIR}/bin" ]; then
echo "Please enter your password for installing obitools in ${INSTALL_DIR}" 1>&2
sudo cp build/* "${INSTALL_DIR}/bin"
else
echo "Error: Could not copy binaries to ${INSTALL_DIR}/bin" 1>&2
echo " Source files: $(ls build/ 2>/dev/null || echo 'none found')" 1>&2
echo "" 1>&2
echo "The build directory has been preserved for manual recovery:" 1>&2
echo " $(pwd)/build/" 1>&2
echo "You can install manually with:" 1>&2
echo " cp $(pwd)/build/* ${INSTALL_DIR}/bin/" 1>&2
popd > /dev/null || true
exit 1
fi
fi
(cp build/* "${INSTALL_DIR}/bin" 2> /dev/null) \
|| (echo "Please enter your password for installing obitools in ${INSTALL_DIR}"
sudo cp build/* "${INSTALL_DIR}/bin")
popd > /dev/null || exit
popd || exit
# Cleanup
echo "Cleaning up..." 1>&2
chmod -R +w "$WORK_DIR"
rm -rf "$WORK_DIR"
echo "" 1>&2
echo "================================" 1>&2
echo "OBITools4 v${VERSION} installed successfully!" 1>&2
echo "Binaries location: ${INSTALL_DIR}/bin" 1>&2
if [[ -n "$OBITOOLS_PREFIX" ]] ; then
echo "Command prefix: ${OBITOOLS_PREFIX}" 1>&2
fi
echo "================================" 1>&2

109
obitests/obitools/obiannotate/test.sh
View File

@@ -1,109 +0,0 @@
#!/bin/bash
#
# Here give the name of the test serie
#
TEST_NAME=obiannotate
CMD=obiannotate
######
#
# Some variable and function definitions: please don't change them
#
######
TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
export PATH="${OBITOOLS_DIR}:${PATH}"
MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
TMPDIR="$(mktemp -d)"
ntest=0
success=0
failed=0
cleanup() {
echo "========================================" 1>&2
echo "## Results of the $TEST_NAME tests:" 1>&2
echo 1>&2
echo "- $ntest tests run" 1>&2
echo "- $success successfully completed" 1>&2
echo "- $failed failed tests" 1>&2
echo 1>&2
echo "Cleaning up the temporary directory..." 1>&2
echo 1>&2
echo "========================================" 1>&2
rm -rf "$TMPDIR" # Suppress the temporary directory
if [ $failed -gt 0 ]; then
log "$TEST_NAME tests failed"
log
log
exit 1
fi
log
log
exit 0
}
log() {
echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
}
log "Testing $TEST_NAME..."
log "Test directory is $TEST_DIR"
log "obitools directory is $OBITOOLS_DIR"
log "Temporary directory is $TMPDIR"
log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
######################################################################
####
#### Below are the tests
####
#### Before each test :
#### - increment the variable ntest
####
#### Run the command as the condition of an if / then /else
#### - The command must return 0 on success
#### - The command must return an exit code different from 0 on failure
#### - The datafiles are stored in the same directory than the test script
#### - The test script directory is stored in the TEST_DIR variable
#### - If result files have to be produced they must be stored
#### in the temporary directory (TMPDIR variable)
####
#### then clause is executed on success of the command
#### - Write a success message using the log function
#### - increment the variable success
####
#### else clause is executed on failure of the command
#### - Write a failure message using the log function
#### - increment the variable failed
####
######################################################################
((ntest++))
if $CMD -h > "${TMPDIR}/help.txt" 2>&1
then
log "$MCMD: printing help OK"
((success++))
else
log "$MCMD: printing help failed"
((failed++))
fi
#########################################
#
# At the end of the tests
# the cleanup function is called
#
#########################################
cleanup

109
obitests/obitools/obiclean/test.sh
View File

@@ -1,109 +0,0 @@
#!/bin/bash
#
# Here give the name of the test serie
#
TEST_NAME=obiclean
CMD=obiclean
######
#
# Some variable and function definitions: please don't change them
#
######
TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
export PATH="${OBITOOLS_DIR}:${PATH}"
MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
TMPDIR="$(mktemp -d)"
ntest=0
success=0
failed=0
cleanup() {
echo "========================================" 1>&2
echo "## Results of the $TEST_NAME tests:" 1>&2
echo 1>&2
echo "- $ntest tests run" 1>&2
echo "- $success successfully completed" 1>&2
echo "- $failed failed tests" 1>&2
echo 1>&2
echo "Cleaning up the temporary directory..." 1>&2
echo 1>&2
echo "========================================" 1>&2
rm -rf "$TMPDIR" # Suppress the temporary directory
if [ $failed -gt 0 ]; then
log "$TEST_NAME tests failed"
log
log
exit 1
fi
log
log
exit 0
}
log() {
echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
}
log "Testing $TEST_NAME..."
log "Test directory is $TEST_DIR"
log "obitools directory is $OBITOOLS_DIR"
log "Temporary directory is $TMPDIR"
log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
######################################################################
####
#### Below are the tests
####
#### Before each test :
#### - increment the variable ntest
####
#### Run the command as the condition of an if / then /else
#### - The command must return 0 on success
#### - The command must return an exit code different from 0 on failure
#### - The datafiles are stored in the same directory than the test script
#### - The test script directory is stored in the TEST_DIR variable
#### - If result files have to be produced they must be stored
#### in the temporary directory (TMPDIR variable)
####
#### then clause is executed on success of the command
#### - Write a success message using the log function
#### - increment the variable success
####
#### else clause is executed on failure of the command
#### - Write a failure message using the log function
#### - increment the variable failed
####
######################################################################
((ntest++))
if $CMD -h > "${TMPDIR}/help.txt" 2>&1
then
log "$MCMD: printing help OK"
((success++))
else
log "$MCMD: printing help failed"
((failed++))
fi
#########################################
#
# At the end of the tests
# the cleanup function is called
#
#########################################
cleanup

109
obitests/obitools/obicleandb/test.sh
View File

@@ -1,109 +0,0 @@
#!/bin/bash
#
# Here give the name of the test serie
#
TEST_NAME=obicleandb
CMD=obicleandb
######
#
# Some variable and function definitions: please don't change them
#
######
TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
export PATH="${OBITOOLS_DIR}:${PATH}"
MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
TMPDIR="$(mktemp -d)"
ntest=0
success=0
failed=0
cleanup() {
echo "========================================" 1>&2
echo "## Results of the $TEST_NAME tests:" 1>&2
echo 1>&2
echo "- $ntest tests run" 1>&2
echo "- $success successfully completed" 1>&2
echo "- $failed failed tests" 1>&2
echo 1>&2
echo "Cleaning up the temporary directory..." 1>&2
echo 1>&2
echo "========================================" 1>&2
rm -rf "$TMPDIR" # Suppress the temporary directory
if [ $failed -gt 0 ]; then
log "$TEST_NAME tests failed"
log
log
exit 1
fi
log
log
exit 0
}
log() {
echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
}
log "Testing $TEST_NAME..."
log "Test directory is $TEST_DIR"
log "obitools directory is $OBITOOLS_DIR"
log "Temporary directory is $TMPDIR"
log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
######################################################################
####
#### Below are the tests
####
#### Before each test :
#### - increment the variable ntest
####
#### Run the command as the condition of an if / then /else
#### - The command must return 0 on success
#### - The command must return an exit code different from 0 on failure
#### - The datafiles are stored in the same directory than the test script
#### - The test script directory is stored in the TEST_DIR variable
#### - If result files have to be produced they must be stored
#### in the temporary directory (TMPDIR variable)
####
#### then clause is executed on success of the command
#### - Write a success message using the log function
#### - increment the variable success
####
#### else clause is executed on failure of the command
#### - Write a failure message using the log function
#### - increment the variable failed
####
######################################################################
((ntest++))
if $CMD -h > "${TMPDIR}/help.txt" 2>&1
then
log "$MCMD: printing help OK"
((success++))
else
log "$MCMD: printing help failed"
((failed++))
fi
#########################################
#
# At the end of the tests
# the cleanup function is called
#
#########################################
cleanup

109
obitests/obitools/obicomplement/test.sh
View File

@@ -1,109 +0,0 @@
#!/bin/bash
#
# Here give the name of the test serie
#
TEST_NAME=obicomplement
CMD=obicomplement
######
#
# Some variable and function definitions: please don't change them
#
######
TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
export PATH="${OBITOOLS_DIR}:${PATH}"
MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
TMPDIR="$(mktemp -d)"
ntest=0
success=0
failed=0
cleanup() {
echo "========================================" 1>&2
echo "## Results of the $TEST_NAME tests:" 1>&2
echo 1>&2
echo "- $ntest tests run" 1>&2
echo "- $success successfully completed" 1>&2
echo "- $failed failed tests" 1>&2
echo 1>&2
echo "Cleaning up the temporary directory..." 1>&2
echo 1>&2
echo "========================================" 1>&2
rm -rf "$TMPDIR" # Suppress the temporary directory
if [ $failed -gt 0 ]; then
log "$TEST_NAME tests failed"
log
log
exit 1
fi
log
log
exit 0
}
log() {
echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
}
log "Testing $TEST_NAME..."
log "Test directory is $TEST_DIR"
log "obitools directory is $OBITOOLS_DIR"
log "Temporary directory is $TMPDIR"
log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
######################################################################
####
#### Below are the tests
####
#### Before each test :
#### - increment the variable ntest
####
#### Run the command as the condition of an if / then /else
#### - The command must return 0 on success
#### - The command must return an exit code different from 0 on failure
#### - The datafiles are stored in the same directory than the test script
#### - The test script directory is stored in the TEST_DIR variable
#### - If result files have to be produced they must be stored
#### in the temporary directory (TMPDIR variable)
####
#### then clause is executed on success of the command
#### - Write a success message using the log function
#### - increment the variable success
####
#### else clause is executed on failure of the command
#### - Write a failure message using the log function
#### - increment the variable failed
####
######################################################################
((ntest++))
if $CMD -h > "${TMPDIR}/help.txt" 2>&1
then
log "$MCMD: printing help OK"
((success++))
else
log "$MCMD: printing help failed"
((failed++))
fi
#########################################
#
# At the end of the tests
# the cleanup function is called
#
#########################################
cleanup

109
obitests/obitools/obiconsensus/test.sh
View File

@@ -1,109 +0,0 @@
#!/bin/bash
#
# Here give the name of the test serie
#
TEST_NAME=obiconsensus
CMD=obiconsensus
######
#
# Some variable and function definitions: please don't change them
#
######
TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
export PATH="${OBITOOLS_DIR}:${PATH}"
MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
TMPDIR="$(mktemp -d)"
ntest=0
success=0
failed=0
cleanup() {
echo "========================================" 1>&2
echo "## Results of the $TEST_NAME tests:" 1>&2
echo 1>&2
echo "- $ntest tests run" 1>&2
echo "- $success successfully completed" 1>&2
echo "- $failed failed tests" 1>&2
echo 1>&2
echo "Cleaning up the temporary directory..." 1>&2
echo 1>&2
echo "========================================" 1>&2
rm -rf "$TMPDIR" # Suppress the temporary directory
if [ $failed -gt 0 ]; then
log "$TEST_NAME tests failed"
log
log
exit 1
fi
log
log
exit 0
}
log() {
echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
}
log "Testing $TEST_NAME..."
log "Test directory is $TEST_DIR"
log "obitools directory is $OBITOOLS_DIR"
log "Temporary directory is $TMPDIR"
log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
######################################################################
####
#### Below are the tests
####
#### Before each test :
#### - increment the variable ntest
####
#### Run the command as the condition of an if / then /else
#### - The command must return 0 on success
#### - The command must return an exit code different from 0 on failure
#### - The datafiles are stored in the same directory than the test script
#### - The test script directory is stored in the TEST_DIR variable
#### - If result files have to be produced they must be stored
#### in the temporary directory (TMPDIR variable)
####
#### then clause is executed on success of the command
#### - Write a success message using the log function
#### - increment the variable success
####
#### else clause is executed on failure of the command
#### - Write a failure message using the log function
#### - increment the variable failed
####
######################################################################
((ntest++))
if $CMD -h > "${TMPDIR}/help.txt" 2>&1
then
log "$MCMD: printing help OK"
((success++))
else
log "$MCMD: printing help failed"
((failed++))
fi
#########################################
#
# At the end of the tests
# the cleanup function is called
#
#########################################
cleanup

BIN
obitests/obitools/obiconvert/gbpln1088.4Mb.fasta.gz
View File

Binary file not shown.

144
obitests/obitools/obiconvert/test.sh
View File

@@ -1,144 +0,0 @@
#!/bin/bash
#
# Here give the name of the test serie
#
TEST_NAME=obiconvert
CMD=obiconvert
######
#
# Some variable and function definitions: please don't change them
#
######
TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
if [ -z "$TEST_DIR" ] ; then
TEST_DIR="."
fi
OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
export PATH="${OBITOOLS_DIR}:${PATH}"
MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
TMPDIR="$(mktemp -d)"
ntest=0
success=0
failed=0
cleanup() {
echo "========================================" 1>&2
echo "## Results of the $TEST_NAME tests:" 1>&2
echo 1>&2
echo "- $ntest tests run" 1>&2
echo "- $success successfully completed" 1>&2
echo "- $failed failed tests" 1>&2
echo 1>&2
echo "Cleaning up the temporary directory..." 1>&2
echo 1>&2
echo "========================================" 1>&2
rm -rf "$TMPDIR" # Suppress the temporary directory
if [ $failed -gt 0 ]; then
log "$TEST_NAME tests failed"
log
log
exit 1
fi
log
log
exit 0
}
log() {
echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
}
log "Testing $TEST_NAME..."
log "Test directory is $TEST_DIR"
log "obitools directory is $OBITOOLS_DIR"
log "Temporary directory is $TMPDIR"
log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
######################################################################
####
#### Below are the tests
####
#### Before each test :
#### - increment the variable ntest
####
#### Run the command as the condition of an if / then /else
#### - The command must return 0 on success
#### - The command must return an exit code different from 0 on failure
#### - The datafiles are stored in the same directory than the test script
#### - The test script directory is stored in the TEST_DIR variable
#### - If result files have to be produced they must be stored
#### in the temporary directory (TMPDIR variable)
####
#### then clause is executed on success of the command
#### - Write a success message using the log function
#### - increment the variable success
####
#### else clause is executed on failure of the command
#### - Write a failure message using the log function
#### - increment the variable failed
####
######################################################################
((ntest++))
if $CMD -h > "${TMPDIR}/help.txt" 2>&1
then
log "$MCMD: printing help OK"
((success++))
else
log "$MCMD: printing help failed"
((failed++))
fi
((ntest++))
if obiconvert -Z "${TEST_DIR}/gbpln1088.4Mb.fasta.gz" \
> "${TMPDIR}/xxx.fasta.gz" && \
zdiff "${TEST_DIR}/gbpln1088.4Mb.fasta.gz" \
"${TMPDIR}/xxx.fasta.gz"
then
log "$MCMD: converting large fasta file to fasta OK"
((success++))
else
log "$MCMD: converting large fasta file to fasta failed"
((failed++))
fi
((ntest++))
if obiconvert -Z --fastq-output \
"${TEST_DIR}/gbpln1088.4Mb.fasta.gz" \
> "${TMPDIR}/xxx.fastq.gz" && \
obiconvert -Z --fasta-output \
"${TMPDIR}/xxx.fastq.gz" \
> "${TMPDIR}/yyy.fasta.gz" && \
zdiff "${TEST_DIR}/gbpln1088.4Mb.fasta.gz" \
"${TMPDIR}/yyy.fasta.gz"
then
log "$MCMD: converting large file between fasta and fastq OK"
((success++))
else
log "$MCMD: converting large file between fasta and fastq failed"
((failed++))
fi
#########################################
#
# At the end of the tests
# the cleanup function is called
#
#########################################
cleanup

163
obitests/obitools/obicount/test.sh
View File

@@ -1,163 +0,0 @@
#!/bin/bash
#
# Here give the name of the test serie
#
TEST_NAME=obicount
CMD=obicount
######
#
# Some variable and function definitions: please don't change them
#
######
TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
export PATH="${OBITOOLS_DIR}:${PATH}"
MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
TMPDIR="$(mktemp -d)"
ntest=0
success=0
failed=0
cleanup() {
echo "========================================" 1>&2
echo "## Results of the $TEST_NAME tests:" 1>&2
echo 1>&2
echo "- $ntest tests run" 1>&2
echo "- $success successfully completed" 1>&2
echo "- $failed failed tests" 1>&2
echo 1>&2
echo "Cleaning up the temporary directory..." 1>&2
echo 1>&2
echo "========================================" 1>&2
rm -rf "$TMPDIR" # Suppress the temporary directory
if [ $failed -gt 0 ]; then
log "$TEST_NAME tests failed"
log
log
exit 1
fi
log
log
exit 0
}
log() {
echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
}
log "Testing $TEST_NAME..."
log "Test directory is $TEST_DIR"
log "obitools directory is $OBITOOLS_DIR"
log "Temporary directory is $TMPDIR"
log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
######################################################################
####
#### Below are the tests
####
#### Before each test :
#### - increment the variable ntest
####
#### Run the command as the condition of an if / then /else
#### - The command must return 0 on success
#### - The command must return an exit code different from 0 on failure
#### - The datafiles are stored in the same directory than the test script
#### - The test script directory is stored in the TEST_DIR variable
#### - If result files have to be produced they must be stored
#### in the temporary directory (TMPDIR variable)
####
#### then clause is executed on success of the command
#### - Write a success message using the log function
#### - increment the variable success
####
#### else clause is executed on failure of the command
#### - Write a failure message using the log function
#### - increment the variable failed
####
######################################################################
((ntest++))
if $CMD -h > "${TMPDIR}/help.txt" 2>&1
then
log "$MCMD: printing help OK"
((success++))
else
log "$MCMD: printing help failed"
((failed++))
fi
((ntest++))
if obicount "${TEST_DIR}/wolf_F.fasta.gz" \
> "${TMPDIR}/wolf_F.fasta_count.csv"
then
log "OBICount: fasta reading OK"
((success++))
else
log "OBICount: fasta reading failed"
((failed++))
fi
((ntest++))
if obicount "${TEST_DIR}/wolf_F.fastq.gz" \
> "${TMPDIR}/wolf_F.fastq_count.csv"
then
log "OBICount: fastq reading OK"
((success++))
else
log "OBICount: fastq reading failed"
((failed++))
fi
((ntest++))
if obicount "${TEST_DIR}/wolf_F.csv.gz" \
> "${TMPDIR}/wolf_F.csv_count.csv"
then
log "OBICount: csv reading OK"
((success++))
else
log "OBICount: csv reading failed"
((failed++))
fi
((ntest++))
if diff "${TMPDIR}/wolf_F.fasta_count.csv" \
"${TMPDIR}/wolf_F.fastq_count.csv" > /dev/null
then
log "OBICount: counting on fasta and fastq are identical OK"
((success++))
else
log "OBICount: counting on fasta and fastq are different failed"
((failed++))
fi
((ntest++))
if diff "${TMPDIR}/wolf_F.fasta_count.csv" \
"${TMPDIR}/wolf_F.csv_count.csv" > /dev/null
then
log "OBICount: counting on fasta and csv are identical OK"
((success++))
else
log "OBICount: counting on fasta and csv are different failed"
((failed++))
fi
#########################################
#
# At the end of the tests
# the cleanup function is called
#
#########################################
cleanup

BIN
obitests/obitools/obicount/wolf_F.csv.gz
View File

Binary file not shown.

BIN
obitests/obitools/obicount/wolf_F.fasta.gz
View File

Binary file not shown.

BIN
obitests/obitools/obicount/wolf_F.fastq.gz
View File

Binary file not shown.

109
obitests/obitools/obicsv/test.sh
View File

@@ -1,109 +0,0 @@
#!/bin/bash
#
# Here give the name of the test serie
#
TEST_NAME=obicsv
CMD=obicsv
######
#
# Some variable and function definitions: please don't change them
#
######
TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
export PATH="${OBITOOLS_DIR}:${PATH}"
MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
TMPDIR="$(mktemp -d)"
ntest=0
success=0
failed=0
cleanup() {
echo "========================================" 1>&2
echo "## Results of the $TEST_NAME tests:" 1>&2
echo 1>&2
echo "- $ntest tests run" 1>&2
echo "- $success successfully completed" 1>&2
echo "- $failed failed tests" 1>&2
echo 1>&2
echo "Cleaning up the temporary directory..." 1>&2
echo 1>&2
echo "========================================" 1>&2
rm -rf "$TMPDIR" # Suppress the temporary directory
if [ $failed -gt 0 ]; then
log "$TEST_NAME tests failed"
log
log
exit 1
fi
log
log
exit 0
}
log() {
echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
}
log "Testing $TEST_NAME..."
log "Test directory is $TEST_DIR"
log "obitools directory is $OBITOOLS_DIR"
log "Temporary directory is $TMPDIR"
log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
######################################################################
####
#### Below are the tests
####
#### Before each test :
#### - increment the variable ntest
####
#### Run the command as the condition of an if / then /else
#### - The command must return 0 on success
#### - The command must return an exit code different from 0 on failure
#### - The datafiles are stored in the same directory than the test script
#### - The test script directory is stored in the TEST_DIR variable
#### - If result files have to be produced they must be stored
#### in the temporary directory (TMPDIR variable)
####
#### then clause is executed on success of the command
#### - Write a success message using the log function
#### - increment the variable success
####
#### else clause is executed on failure of the command
#### - Write a failure message using the log function
#### - increment the variable failed
####
######################################################################
((ntest++))
if $CMD -h > "${TMPDIR}/help.txt" 2>&1
then
log "$MCMD: printing help OK"
((success++))
else
log "$MCMD: printing help failed"
((failed++))
fi
#########################################
#
# At the end of the tests
# the cleanup function is called
#
#########################################
cleanup

109
obitests/obitools/obidemerge/test.sh
View File

@@ -1,109 +0,0 @@
#!/bin/bash
#
# Here give the name of the test serie
#
TEST_NAME=obidemerge
CMD=obidemerge
######
#
# Some variable and function definitions: please don't change them
#
######
TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
export PATH="${OBITOOLS_DIR}:${PATH}"
MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
TMPDIR="$(mktemp -d)"
ntest=0
success=0
failed=0
cleanup() {
echo "========================================" 1>&2
echo "## Results of the $TEST_NAME tests:" 1>&2
echo 1>&2
echo "- $ntest tests run" 1>&2
echo "- $success successfully completed" 1>&2
echo "- $failed failed tests" 1>&2
echo 1>&2
echo "Cleaning up the temporary directory..." 1>&2
echo 1>&2
echo "========================================" 1>&2
rm -rf "$TMPDIR" # Suppress the temporary directory
if [ $failed -gt 0 ]; then
log "$TEST_NAME tests failed"
log
log
exit 1
fi
log
log
exit 0
}
log() {
echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
}
log "Testing $TEST_NAME..."
log "Test directory is $TEST_DIR"
log "obitools directory is $OBITOOLS_DIR"
log "Temporary directory is $TMPDIR"
log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
######################################################################
####
#### Below are the tests
####
#### Before each test :
#### - increment the variable ntest
####
#### Run the command as the condition of an if / then /else
#### - The command must return 0 on success
#### - The command must return an exit code different from 0 on failure
#### - The datafiles are stored in the same directory than the test script
#### - The test script directory is stored in the TEST_DIR variable
#### - If result files have to be produced they must be stored
#### in the temporary directory (TMPDIR variable)
####
#### then clause is executed on success of the command
#### - Write a success message using the log function
#### - increment the variable success
####
#### else clause is executed on failure of the command
#### - Write a failure message using the log function
#### - increment the variable failed
####
######################################################################
((ntest++))
if $CMD -h > "${TMPDIR}/help.txt" 2>&1
then
log "$MCMD: printing help OK"
((success++))
else
log "$MCMD: printing help failed"
((failed++))
fi
#########################################
#
# At the end of the tests
# the cleanup function is called
#
#########################################
cleanup

109
obitests/obitools/obidistribute/test.sh
View File

@@ -1,109 +0,0 @@
#!/bin/bash
#
# Here give the name of the test serie
#
TEST_NAME=obidistribute
CMD=obidistribute
######
#
# Some variable and function definitions: please don't change them
#
######
TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
export PATH="${OBITOOLS_DIR}:${PATH}"
MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
TMPDIR="$(mktemp -d)"
ntest=0
success=0
failed=0
cleanup() {
echo "========================================" 1>&2
echo "## Results of the $TEST_NAME tests:" 1>&2
echo 1>&2
echo "- $ntest tests run" 1>&2
echo "- $success successfully completed" 1>&2
echo "- $failed failed tests" 1>&2
echo 1>&2
echo "Cleaning up the temporary directory..." 1>&2
echo 1>&2
echo "========================================" 1>&2
rm -rf "$TMPDIR" # Suppress the temporary directory
if [ $failed -gt 0 ]; then
log "$TEST_NAME tests failed"
log
log
exit 1
fi
log
log
exit 0
}
log() {
echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
}
log "Testing $TEST_NAME..."
log "Test directory is $TEST_DIR"
log "obitools directory is $OBITOOLS_DIR"
log "Temporary directory is $TMPDIR"
log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
######################################################################
####
#### Below are the tests
####
#### Before each test :
#### - increment the variable ntest
####
#### Run the command as the condition of an if / then /else
#### - The command must return 0 on success
#### - The command must return an exit code different from 0 on failure
#### - The datafiles are stored in the same directory than the test script
#### - The test script directory is stored in the TEST_DIR variable
#### - If result files have to be produced they must be stored
#### in the temporary directory (TMPDIR variable)
####
#### then clause is executed on success of the command
#### - Write a success message using the log function
#### - increment the variable success
####
#### else clause is executed on failure of the command
#### - Write a failure message using the log function
#### - increment the variable failed
####
######################################################################
((ntest++))
if $CMD -h > "${TMPDIR}/help.txt" 2>&1
then
log "$MCMD: printing help OK"
((success++))
else
log "$MCMD: printing help failed"
((failed++))
fi
#########################################
#
# At the end of the tests
# the cleanup function is called
#
#########################################
cleanup

109
obitests/obitools/obigrep/test.sh
View File

@@ -1,109 +0,0 @@
#!/bin/bash
#
# Here give the name of the test serie
#
TEST_NAME=obigrep
CMD=obigrep
######
#
# Some variable and function definitions: please don't change them
#
######
TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
export PATH="${OBITOOLS_DIR}:${PATH}"
MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
TMPDIR="$(mktemp -d)"
ntest=0
success=0
failed=0
cleanup() {
echo "========================================" 1>&2
echo "## Results of the $TEST_NAME tests:" 1>&2
echo 1>&2
echo "- $ntest tests run" 1>&2
echo "- $success successfully completed" 1>&2
echo "- $failed failed tests" 1>&2
echo 1>&2
echo "Cleaning up the temporary directory..." 1>&2
echo 1>&2
echo "========================================" 1>&2
rm -rf "$TMPDIR" # Suppress the temporary directory
if [ $failed -gt 0 ]; then
log "$TEST_NAME tests failed"
log
log
exit 1
fi
log
log
exit 0
}
log() {
echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
}
log "Testing $TEST_NAME..."
log "Test directory is $TEST_DIR"
log "obitools directory is $OBITOOLS_DIR"
log "Temporary directory is $TMPDIR"
log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
######################################################################
####
#### Below are the tests
####
#### Before each test :
#### - increment the variable ntest
####
#### Run the command as the condition of an if / then /else
#### - The command must return 0 on success
#### - The command must return an exit code different from 0 on failure
#### - The datafiles are stored in the same directory than the test script
#### - The test script directory is stored in the TEST_DIR variable
#### - If result files have to be produced they must be stored
#### in the temporary directory (TMPDIR variable)
####
#### then clause is executed on success of the command
#### - Write a success message using the log function
#### - increment the variable success
####
#### else clause is executed on failure of the command
#### - Write a failure message using the log function
#### - increment the variable failed
####
######################################################################
((ntest++))
if $CMD -h > "${TMPDIR}/help.txt" 2>&1
then
log "$MCMD: printing help OK"
((success++))
else
log "$MCMD: printing help failed"
((failed++))
fi
#########################################
#
# At the end of the tests
# the cleanup function is called
#
#########################################
cleanup

109
obitests/obitools/obijoin/test.sh
View File

@@ -1,109 +0,0 @@
#!/bin/bash
#
# Here give the name of the test serie
#
TEST_NAME=obijoin
CMD=obijoin
######
#
# Some variable and function definitions: please don't change them
#
######
TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
export PATH="${OBITOOLS_DIR}:${PATH}"
MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
TMPDIR="$(mktemp -d)"
ntest=0
success=0
failed=0
cleanup() {
echo "========================================" 1>&2
echo "## Results of the $TEST_NAME tests:" 1>&2
echo 1>&2
echo "- $ntest tests run" 1>&2
echo "- $success successfully completed" 1>&2
echo "- $failed failed tests" 1>&2
echo 1>&2
echo "Cleaning up the temporary directory..." 1>&2
echo 1>&2
echo "========================================" 1>&2
rm -rf "$TMPDIR" # Suppress the temporary directory
if [ $failed -gt 0 ]; then
log "$TEST_NAME tests failed"
log
log
exit 1
fi
log
log
exit 0
}
log() {
echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
}
log "Testing $TEST_NAME..."
log "Test directory is $TEST_DIR"
log "obitools directory is $OBITOOLS_DIR"
log "Temporary directory is $TMPDIR"
log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
######################################################################
####
#### Below are the tests
####
#### Before each test :
#### - increment the variable ntest
####
#### Run the command as the condition of an if / then /else
#### - The command must return 0 on success
#### - The command must return an exit code different from 0 on failure
#### - The datafiles are stored in the same directory than the test script
#### - The test script directory is stored in the TEST_DIR variable
#### - If result files have to be produced they must be stored
#### in the temporary directory (TMPDIR variable)
####
#### then clause is executed on success of the command
#### - Write a success message using the log function
#### - increment the variable success
####
#### else clause is executed on failure of the command
#### - Write a failure message using the log function
#### - increment the variable failed
####
######################################################################
((ntest++))
if $CMD -h > "${TMPDIR}/help.txt" 2>&1
then
log "$MCMD: printing help OK"
((success++))
else
log "$MCMD: printing help failed"
((failed++))
fi
#########################################
#
# At the end of the tests
# the cleanup function is called
#
#########################################
cleanup

109
obitests/obitools/obikmermatch/test.sh
View File

@@ -1,109 +0,0 @@
#!/bin/bash
#
# Here give the name of the test serie
#
TEST_NAME=obikmermatch
CMD=obikmermatch
######
#
# Some variable and function definitions: please don't change them
#
######
TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
export PATH="${OBITOOLS_DIR}:${PATH}"
MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
TMPDIR="$(mktemp -d)"
ntest=0
success=0
failed=0
cleanup() {
echo "========================================" 1>&2
echo "## Results of the $TEST_NAME tests:" 1>&2
echo 1>&2
echo "- $ntest tests run" 1>&2
echo "- $success successfully completed" 1>&2
echo "- $failed failed tests" 1>&2
echo 1>&2
echo "Cleaning up the temporary directory..." 1>&2
echo 1>&2
echo "========================================" 1>&2
rm -rf "$TMPDIR" # Suppress the temporary directory
if [ $failed -gt 0 ]; then
log "$TEST_NAME tests failed"
log
log
exit 1
fi
log
log
exit 0
}
log() {
echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
}
log "Testing $TEST_NAME..."
log "Test directory is $TEST_DIR"
log "obitools directory is $OBITOOLS_DIR"
log "Temporary directory is $TMPDIR"
log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
######################################################################
####
#### Below are the tests
####
#### Before each test :
#### - increment the variable ntest
####
#### Run the command as the condition of an if / then /else
#### - The command must return 0 on success
#### - The command must return an exit code different from 0 on failure
#### - The datafiles are stored in the same directory than the test script
#### - The test script directory is stored in the TEST_DIR variable
#### - If result files have to be produced they must be stored
#### in the temporary directory (TMPDIR variable)
####
#### then clause is executed on success of the command
#### - Write a success message using the log function
#### - increment the variable success
####
#### else clause is executed on failure of the command
#### - Write a failure message using the log function
#### - increment the variable failed
####
######################################################################
((ntest++))
if $CMD -h > "${TMPDIR}/help.txt" 2>&1
then
log "$MCMD: printing help OK"
((success++))
else
log "$MCMD: printing help failed"
((failed++))
fi
#########################################
#
# At the end of the tests
# the cleanup function is called
#
#########################################
cleanup

109
obitests/obitools/obikmersimcount/test.sh
View File

@@ -1,109 +0,0 @@
#!/bin/bash
#
# Here give the name of the test serie
#
TEST_NAME=obikmersimcount
CMD=obikmersimcount
######
#
# Some variable and function definitions: please don't change them
#
######
TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
export PATH="${OBITOOLS_DIR}:${PATH}"
MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
TMPDIR="$(mktemp -d)"
ntest=0
success=0
failed=0
cleanup() {
echo "========================================" 1>&2
echo "## Results of the $TEST_NAME tests:" 1>&2
echo 1>&2
echo "- $ntest tests run" 1>&2
echo "- $success successfully completed" 1>&2
echo "- $failed failed tests" 1>&2
echo 1>&2
echo "Cleaning up the temporary directory..." 1>&2
echo 1>&2
echo "========================================" 1>&2
rm -rf "$TMPDIR" # Suppress the temporary directory
if [ $failed -gt 0 ]; then
log "$TEST_NAME tests failed"
log
log
exit 1
fi
log
log
exit 0
}
log() {
echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
}
log "Testing $TEST_NAME..."
log "Test directory is $TEST_DIR"
log "obitools directory is $OBITOOLS_DIR"
log "Temporary directory is $TMPDIR"
log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
######################################################################
####
#### Below are the tests
####
#### Before each test :
#### - increment the variable ntest
####
#### Run the command as the condition of an if / then /else
#### - The command must return 0 on success
#### - The command must return an exit code different from 0 on failure
#### - The datafiles are stored in the same directory than the test script
#### - The test script directory is stored in the TEST_DIR variable
#### - If result files have to be produced they must be stored
#### in the temporary directory (TMPDIR variable)
####
#### then clause is executed on success of the command
#### - Write a success message using the log function
#### - increment the variable success
####
#### else clause is executed on failure of the command
#### - Write a failure message using the log function
#### - increment the variable failed
####
######################################################################
((ntest++))
if $CMD -h > "${TMPDIR}/help.txt" 2>&1
then
log "$MCMD: printing help OK"
((success++))
else
log "$MCMD: printing help failed"
((failed++))
fi
#########################################
#
# At the end of the tests
# the cleanup function is called
#
#########################################
cleanup

109
obitests/obitools/obilandmark/test.sh
View File

@@ -1,109 +0,0 @@
#!/bin/bash
#
# Here give the name of the test serie
#
TEST_NAME=obilandmark
CMD=obilandmark
######
#
# Some variable and function definitions: please don't change them
#
######
TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
export PATH="${OBITOOLS_DIR}:${PATH}"
MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
TMPDIR="$(mktemp -d)"
ntest=0
success=0
failed=0
cleanup() {
echo "========================================" 1>&2
echo "## Results of the $TEST_NAME tests:" 1>&2
echo 1>&2
echo "- $ntest tests run" 1>&2
echo "- $success successfully completed" 1>&2
echo "- $failed failed tests" 1>&2
echo 1>&2
echo "Cleaning up the temporary directory..." 1>&2
echo 1>&2
echo "========================================" 1>&2
rm -rf "$TMPDIR" # Suppress the temporary directory
if [ $failed -gt 0 ]; then
log "$TEST_NAME tests failed"
log
log
exit 1
fi
log
log
exit 0
}
log() {
echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
}
log "Testing $TEST_NAME..."
log "Test directory is $TEST_DIR"
log "obitools directory is $OBITOOLS_DIR"
log "Temporary directory is $TMPDIR"
log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
######################################################################
####
#### Below are the tests
####
#### Before each test :
#### - increment the variable ntest
####
#### Run the command as the condition of an if / then /else
#### - The command must return 0 on success
#### - The command must return an exit code different from 0 on failure
#### - The datafiles are stored in the same directory than the test script
#### - The test script directory is stored in the TEST_DIR variable
#### - If result files have to be produced they must be stored
#### in the temporary directory (TMPDIR variable)
####
#### then clause is executed on success of the command
#### - Write a success message using the log function
#### - increment the variable success
####
#### else clause is executed on failure of the command
#### - Write a failure message using the log function
#### - increment the variable failed
####
######################################################################
((ntest++))
if $CMD -h > "${TMPDIR}/help.txt" 2>&1
then
log "$MCMD: printing help OK"
((success++))
else
log "$MCMD: printing help failed"
((failed++))
fi
#########################################
#
# At the end of the tests
# the cleanup function is called
#
#########################################
cleanup

109
obitests/obitools/obimatrix/test.sh
View File

@@ -1,109 +0,0 @@
#!/bin/bash
#
# Here give the name of the test serie
#
TEST_NAME=obimatrix
CMD=obimatrix
######
#
# Some variable and function definitions: please don't change them
#
######
TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
export PATH="${OBITOOLS_DIR}:${PATH}"
MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
TMPDIR="$(mktemp -d)"
ntest=0
success=0
failed=0
cleanup() {
echo "========================================" 1>&2
echo "## Results of the $TEST_NAME tests:" 1>&2
echo 1>&2
echo "- $ntest tests run" 1>&2
echo "- $success successfully completed" 1>&2
echo "- $failed failed tests" 1>&2
echo 1>&2
echo "Cleaning up the temporary directory..." 1>&2
echo 1>&2
echo "========================================" 1>&2
rm -rf "$TMPDIR" # Suppress the temporary directory
if [ $failed -gt 0 ]; then
log "$TEST_NAME tests failed"
log
log
exit 1
fi
log
log
exit 0
}
log() {
echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
}
log "Testing $TEST_NAME..."
log "Test directory is $TEST_DIR"
log "obitools directory is $OBITOOLS_DIR"
log "Temporary directory is $TMPDIR"
log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
######################################################################
####
#### Below are the tests
####
#### Before each test :
#### - increment the variable ntest
####
#### Run the command as the condition of an if / then /else
#### - The command must return 0 on success
#### - The command must return an exit code different from 0 on failure
#### - The datafiles are stored in the same directory than the test script
#### - The test script directory is stored in the TEST_DIR variable
#### - If result files have to be produced they must be stored
#### in the temporary directory (TMPDIR variable)
####
#### then clause is executed on success of the command
#### - Write a success message using the log function
#### - increment the variable success
####
#### else clause is executed on failure of the command
#### - Write a failure message using the log function
#### - increment the variable failed
####
######################################################################
((ntest++))
if $CMD -h > "${TMPDIR}/help.txt" 2>&1
then
log "$MCMD: printing help OK"
((success++))
else
log "$MCMD: printing help failed"
((failed++))
fi
#########################################
#
# At the end of the tests
# the cleanup function is called
#
#########################################
cleanup

109
obitests/obitools/obimicrosat/test.sh
View File

@@ -1,109 +0,0 @@
#!/bin/bash
#
# Here give the name of the test serie
#
TEST_NAME=obimicrosat
CMD=obimicrosat
######
#
# Some variable and function definitions: please don't change them
#
######
TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
export PATH="${OBITOOLS_DIR}:${PATH}"
MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
TMPDIR="$(mktemp -d)"
ntest=0
success=0
failed=0
cleanup() {
echo "========================================" 1>&2
echo "## Results of the $TEST_NAME tests:" 1>&2
echo 1>&2
echo "- $ntest tests run" 1>&2
echo "- $success successfully completed" 1>&2
echo "- $failed failed tests" 1>&2
echo 1>&2
echo "Cleaning up the temporary directory..." 1>&2
echo 1>&2
echo "========================================" 1>&2
rm -rf "$TMPDIR" # Suppress the temporary directory
if [ $failed -gt 0 ]; then
log "$TEST_NAME tests failed"
log
log
exit 1
fi
log
log
exit 0
}
log() {
echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
}
log "Testing $TEST_NAME..."
log "Test directory is $TEST_DIR"
log "obitools directory is $OBITOOLS_DIR"
log "Temporary directory is $TMPDIR"
log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
######################################################################
####
#### Below are the tests
####
#### Before each test :
#### - increment the variable ntest
####
#### Run the command as the condition of an if / then /else
#### - The command must return 0 on success
#### - The command must return an exit code different from 0 on failure
#### - The datafiles are stored in the same directory than the test script
#### - The test script directory is stored in the TEST_DIR variable
#### - If result files have to be produced they must be stored
#### in the temporary directory (TMPDIR variable)
####
#### then clause is executed on success of the command
#### - Write a success message using the log function
#### - increment the variable success
####
#### else clause is executed on failure of the command
#### - Write a failure message using the log function
#### - increment the variable failed
####
######################################################################
((ntest++))
if $CMD -h > "${TMPDIR}/help.txt" 2>&1
then
log "$MCMD: printing help OK"
((success++))
else
log "$MCMD: printing help failed"
((failed++))
fi
#########################################
#
# At the end of the tests
# the cleanup function is called
#
#########################################
cleanup

109
obitests/obitools/obimultiplex/test.sh
View File

@@ -1,109 +0,0 @@
#!/bin/bash
#
# Here give the name of the test serie
#
TEST_NAME=obimultiplex
CMD=obimultiplex
######
#
# Some variable and function definitions: please don't change them
#
######
TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
export PATH="${OBITOOLS_DIR}:${PATH}"
MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
TMPDIR="$(mktemp -d)"
ntest=0
success=0
failed=0
cleanup() {
echo "========================================" 1>&2
echo "## Results of the $TEST_NAME tests:" 1>&2
echo 1>&2
echo "- $ntest tests run" 1>&2
echo "- $success successfully completed" 1>&2
echo "- $failed failed tests" 1>&2
echo 1>&2
echo "Cleaning up the temporary directory..." 1>&2
echo 1>&2
echo "========================================" 1>&2
rm -rf "$TMPDIR" # Suppress the temporary directory
if [ $failed -gt 0 ]; then
log "$TEST_NAME tests failed"
log
log
exit 1
fi
log
log
exit 0
}
log() {
echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
}
log "Testing $TEST_NAME..."
log "Test directory is $TEST_DIR"
log "obitools directory is $OBITOOLS_DIR"
log "Temporary directory is $TMPDIR"
log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
######################################################################
####
#### Below are the tests
####
#### Before each test :
#### - increment the variable ntest
####
#### Run the command as the condition of an if / then /else
#### - The command must return 0 on success
#### - The command must return an exit code different from 0 on failure
#### - The datafiles are stored in the same directory than the test script
#### - The test script directory is stored in the TEST_DIR variable
#### - If result files have to be produced they must be stored
#### in the temporary directory (TMPDIR variable)
####
#### then clause is executed on success of the command
#### - Write a success message using the log function
#### - increment the variable success
####
#### else clause is executed on failure of the command
#### - Write a failure message using the log function
#### - increment the variable failed
####
######################################################################
((ntest++))
if $CMD -h > "${TMPDIR}/help.txt" 2>&1
then
log "$MCMD: printing help OK"
((success++))
else
log "$MCMD: printing help failed"
((failed++))
fi
#########################################
#
# At the end of the tests
# the cleanup function is called
#
#########################################
cleanup

150
obitests/obitools/obipairing/test.sh
View File

@@ -1,150 +0,0 @@
#!/bin/bash
#
# Here give the name of the test serie
#
TEST_NAME=obipairing
CMD=obipairing
######
#
# Some variable and function definitions: please don't change them
#
######
TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
export PATH="${OBITOOLS_DIR}:${PATH}"
MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
TMPDIR="$(mktemp -d)"
ntest=0
success=0
failed=0
cleanup() {
echo "========================================" 1>&2
echo "## Results of the $TEST_NAME tests:" 1>&2
echo 1>&2
echo "- $ntest tests run" 1>&2
echo "- $success successfully completed" 1>&2
echo "- $failed failed tests" 1>&2
echo 1>&2
echo "Cleaning up the temporary directory..." 1>&2
echo 1>&2
echo "========================================" 1>&2
rm -rf "$TMPDIR" # Suppress the temporary directory
if [ $failed -gt 0 ]; then
log "$TEST_NAME tests failed"
log
log
exit 1
fi
log
log
exit 0
}
log() {
echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
}
log "Testing $TEST_NAME..."
log "Test directory is $TEST_DIR"
log "obitools directory is $OBITOOLS_DIR"
log "Temporary directory is $TMPDIR"
log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
######################################################################
####
#### Below are the tests
####
#### Before each test :
#### - increment the variable ntest
####
#### Run the command as the condition of an if / then /else
#### - The command must return 0 on success
#### - The command must return an exit code different from 0 on failure
#### - The datafiles are stored in the same directory than the test script
#### - The test script directory is stored in the TEST_DIR variable
#### - If result files have to be produced they must be stored
#### in the temporary directory (TMPDIR variable)
####
#### then clause is executed on success of the command
#### - Write a success message using the log function
#### - increment the variable success
####
#### else clause is executed on failure of the command
#### - Write a failure message using the log function
#### - increment the variable failed
####
######################################################################
((ntest++))
if $CMD -h > "${TMPDIR}/help.txt" 2>&1
then
log "$MCMD: printing help OK"
((success++))
else
log "$MCMD: printing help failed"
((failed++))
fi
((ntest++))
if obipairing -F "${TEST_DIR}/wolf_F.fastq.gz" \
-R "${TEST_DIR}/wolf_R.fastq.gz" \
| obidistribute -Z -c mode \
-p "${TMPDIR}/wolf_paired_%s.fastq.gz"
then
log "OBIPairing: sequence pairing OK"
((success++))
else
log "OBIPairing: sequence pairing failed"
((failed++))
fi
((ntest++))
if obicsv -Z -s -i \
-k ali_dir -k ali_length -k pairing_fast_count \
-k pairing_fast_overlap -k pairing_fast_score \
-k score -k score_norm -k seq_a_single \
-k seq_b_single -k seq_ab_match \
"${TMPDIR}/wolf_paired_alignment.fastq.gz" \
> "${TMPDIR}/wolf_paired_alignment.csv.gz" \
&& zdiff -c "${TEST_DIR}/wolf_paired_alignment.csv.gz" \
"${TMPDIR}/wolf_paired_alignment.csv.gz"
then
log "OBIPairing: check aligned sequences OK"
((success++))
else
log "OBIPairing: check aligned sequences failed"
((failed++))
fi
((ntest++))
if obicsv -Z -s -i \
"${TMPDIR}/wolf_paired_join.fastq.gz" \
> "${TMPDIR}/wolf_paired_join.csv.gz" \
&& zdiff -c "${TEST_DIR}/wolf_paired_join.csv.gz" \
"${TMPDIR}/wolf_paired_join.csv.gz"
then
log "OBIPairing: check joined sequences OK"
((success++))
else
log "OBIPairing: check joined sequences failed"
((failed++))
fi
#########################################
#
# At the end of the tests
# the cleanup function is called
#
#########################################
cleanup

BIN
obitests/obitools/obipairing/wolf_F.fastq.gz
View File

Binary file not shown.

BIN
obitests/obitools/obipairing/wolf_R.fastq.gz
View File

Binary file not shown.

BIN
obitests/obitools/obipairing/wolf_paired_alignment.csv.gz
View File

Binary file not shown.

BIN
obitests/obitools/obipairing/wolf_paired_join.csv.gz
View File

Binary file not shown.

109
obitests/obitools/obipcr/test.sh
View File

@@ -1,109 +0,0 @@
#!/bin/bash
#
# Here give the name of the test serie
#
TEST_NAME=obipcr
CMD=obipcr
######
#
# Some variable and function definitions: please don't change them
#
######
TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
export PATH="${OBITOOLS_DIR}:${PATH}"
MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
TMPDIR="$(mktemp -d)"
ntest=0
success=0
failed=0
cleanup() {
echo "========================================" 1>&2
echo "## Results of the $TEST_NAME tests:" 1>&2
echo 1>&2
echo "- $ntest tests run" 1>&2
echo "- $success successfully completed" 1>&2
echo "- $failed failed tests" 1>&2
echo 1>&2
echo "Cleaning up the temporary directory..." 1>&2
echo 1>&2
echo "========================================" 1>&2
rm -rf "$TMPDIR" # Suppress the temporary directory
if [ $failed -gt 0 ]; then
log "$TEST_NAME tests failed"
log
log
exit 1
fi
log
log
exit 0
}
log() {
echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
}
log "Testing $TEST_NAME..."
log "Test directory is $TEST_DIR"
log "obitools directory is $OBITOOLS_DIR"
log "Temporary directory is $TMPDIR"
log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
######################################################################
####
#### Below are the tests
####
#### Before each test :
#### - increment the variable ntest
####
#### Run the command as the condition of an if / then /else
#### - The command must return 0 on success
#### - The command must return an exit code different from 0 on failure
#### - The datafiles are stored in the same directory than the test script
#### - The test script directory is stored in the TEST_DIR variable
#### - If result files have to be produced they must be stored
#### in the temporary directory (TMPDIR variable)
####
#### then clause is executed on success of the command
#### - Write a success message using the log function
#### - increment the variable success
####
#### else clause is executed on failure of the command
#### - Write a failure message using the log function
#### - increment the variable failed
####
######################################################################
((ntest++))
if $CMD -h > "${TMPDIR}/help.txt" 2>&1
then
log "$MCMD: printing help OK"
((success++))
else
log "$MCMD: printing help failed"
((failed++))
fi
#########################################
#
# At the end of the tests
# the cleanup function is called
#
#########################################
cleanup

109
obitests/obitools/obirefidx/test.sh
View File

@@ -1,109 +0,0 @@
#!/bin/bash
#
# Here give the name of the test serie
#
TEST_NAME=obirefidx
CMD=obirefidx
######
#
# Some variable and function definitions: please don't change them
#
######
TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
export PATH="${OBITOOLS_DIR}:${PATH}"
MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
TMPDIR="$(mktemp -d)"
ntest=0
success=0
failed=0
cleanup() {
echo "========================================" 1>&2
echo "## Results of the $TEST_NAME tests:" 1>&2
echo 1>&2
echo "- $ntest tests run" 1>&2
echo "- $success successfully completed" 1>&2
echo "- $failed failed tests" 1>&2
echo 1>&2
echo "Cleaning up the temporary directory..." 1>&2
echo 1>&2
echo "========================================" 1>&2
rm -rf "$TMPDIR" # Suppress the temporary directory
if [ $failed -gt 0 ]; then
log "$TEST_NAME tests failed"
log
log
exit 1
fi
log
log
exit 0
}
log() {
echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
}
log "Testing $TEST_NAME..."
log "Test directory is $TEST_DIR"
log "obitools directory is $OBITOOLS_DIR"
log "Temporary directory is $TMPDIR"
log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
######################################################################
####
#### Below are the tests
####
#### Before each test :
#### - increment the variable ntest
####
#### Run the command as the condition of an if / then /else
#### - The command must return 0 on success
#### - The command must return an exit code different from 0 on failure
#### - The datafiles are stored in the same directory than the test script
#### - The test script directory is stored in the TEST_DIR variable
#### - If result files have to be produced they must be stored
#### in the temporary directory (TMPDIR variable)
####
#### then clause is executed on success of the command
#### - Write a success message using the log function
#### - increment the variable success
####
#### else clause is executed on failure of the command
#### - Write a failure message using the log function
#### - increment the variable failed
####
######################################################################
((ntest++))
if $CMD -h > "${TMPDIR}/help.txt" 2>&1
then
log "$MCMD: printing help OK"
((success++))
else
log "$MCMD: printing help failed"
((failed++))
fi
#########################################
#
# At the end of the tests
# the cleanup function is called
#
#########################################
cleanup

109
obitests/obitools/obiscript/test.sh
View File

@@ -1,109 +0,0 @@
#!/bin/bash
#
# Here give the name of the test serie
#
TEST_NAME=obiscript
CMD=obiscript
######
#
# Some variable and function definitions: please don't change them
#
######
TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
export PATH="${OBITOOLS_DIR}:${PATH}"
MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
TMPDIR="$(mktemp -d)"
ntest=0
success=0
failed=0
cleanup() {
echo "========================================" 1>&2
echo "## Results of the $TEST_NAME tests:" 1>&2
echo 1>&2
echo "- $ntest tests run" 1>&2
echo "- $success successfully completed" 1>&2
echo "- $failed failed tests" 1>&2
echo 1>&2
echo "Cleaning up the temporary directory..." 1>&2
echo 1>&2
echo "========================================" 1>&2
rm -rf "$TMPDIR" # Suppress the temporary directory
if [ $failed -gt 0 ]; then
log "$TEST_NAME tests failed"
log
log
exit 1
fi
log
log
exit 0
}
log() {
echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
}
log "Testing $TEST_NAME..."
log "Test directory is $TEST_DIR"
log "obitools directory is $OBITOOLS_DIR"
log "Temporary directory is $TMPDIR"
log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
######################################################################
####
#### Below are the tests
####
#### Before each test :
#### - increment the variable ntest
####
#### Run the command as the condition of an if / then /else
#### - The command must return 0 on success
#### - The command must return an exit code different from 0 on failure
#### - The datafiles are stored in the same directory than the test script
#### - The test script directory is stored in the TEST_DIR variable
#### - If result files have to be produced they must be stored
#### in the temporary directory (TMPDIR variable)
####
#### then clause is executed on success of the command
#### - Write a success message using the log function
#### - increment the variable success
####
#### else clause is executed on failure of the command
#### - Write a failure message using the log function
#### - increment the variable failed
####
######################################################################
((ntest++))
if $CMD -h > "${TMPDIR}/help.txt" 2>&1
then
log "$MCMD: printing help OK"
((success++))
else
log "$MCMD: printing help failed"
((failed++))
fi
#########################################
#
# At the end of the tests
# the cleanup function is called
#
#########################################
cleanup

109
obitests/obitools/obisplit/test.sh
View File

@@ -1,109 +0,0 @@
#!/bin/bash
#
# Here give the name of the test serie
#
TEST_NAME=obisplit
CMD=obisplit
######
#
# Some variable and function definitions: please don't change them
#
######
TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
export PATH="${OBITOOLS_DIR}:${PATH}"
MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
TMPDIR="$(mktemp -d)"
ntest=0
success=0
failed=0
cleanup() {
echo "========================================" 1>&2
echo "## Results of the $TEST_NAME tests:" 1>&2
echo 1>&2
echo "- $ntest tests run" 1>&2
echo "- $success successfully completed" 1>&2
echo "- $failed failed tests" 1>&2
echo 1>&2
echo "Cleaning up the temporary directory..." 1>&2
echo 1>&2
echo "========================================" 1>&2
rm -rf "$TMPDIR" # Suppress the temporary directory
if [ $failed -gt 0 ]; then
log "$TEST_NAME tests failed"
log
log
exit 1
fi
log
log
exit 0
}
log() {
echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
}
log "Testing $TEST_NAME..."
log "Test directory is $TEST_DIR"
log "obitools directory is $OBITOOLS_DIR"
log "Temporary directory is $TMPDIR"
log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
######################################################################
####
#### Below are the tests
####
#### Before each test :
#### - increment the variable ntest
####
#### Run the command as the condition of an if / then /else
#### - The command must return 0 on success
#### - The command must return an exit code different from 0 on failure
#### - The datafiles are stored in the same directory than the test script
#### - The test script directory is stored in the TEST_DIR variable
#### - If result files have to be produced they must be stored
#### in the temporary directory (TMPDIR variable)
####
#### then clause is executed on success of the command
#### - Write a success message using the log function
#### - increment the variable success
####
#### else clause is executed on failure of the command
#### - Write a failure message using the log function
#### - increment the variable failed
####
######################################################################
((ntest++))
if $CMD -h > "${TMPDIR}/help.txt" 2>&1
then
log "$MCMD: printing help OK"
((success++))
else
log "$MCMD: printing help failed"
((failed++))
fi
#########################################
#
# At the end of the tests
# the cleanup function is called
#
#########################################
cleanup

9
obitests/obitools/obisummary/some_uniq_seq.fasta
View File

@@ -1,9 +0,0 @@
>Seq_1 {"count":2,"merged_sample":{"15a_F730814":1,"29a_F260619":1}}
ttagccctaaacacaagtaattaatataacaaaattattcgccagagtactaccggcaat
agctyaaaactcaaaggacttggcggtgctttataccctt
>Seq_2 {"count":22,"merged_sample":{"15a_F730814":12,"29a_F260619":10}}
ttagccctaaacacaagtaattaatataacaaaattattcgccagagtactaccggcaat
atcttaaaactcaaaggacttggcggtgctttataccctt
>Seq_3 {"count":22,"merged_sample":{"15a_F730814":15,"29a_F260619":7}}
ttagccctaaacacaagtaattaatataacaaaattattcgccagagtactaccggcgat
agcttaaaactcaaaggacttggcggtgctttataccctt

35
obitests/obitools/obisummary/some_uniq_seq.json
View File

@@ -1,35 +0,0 @@
{
"annotations": {
"keys": {
"map": {
"merged_sample": 3
},
"scalar": {
"count": 3
}
},
"map_attributes": 1,
"scalar_attributes": 1,
"vector_attributes": 0
},
"count": {
"reads": 46,
"total_length": 300,
"variants": 3
},
"samples": {
"sample_count": 2,
"sample_stats": {
"15a_F730814": {
"reads": 28,
"singletons": 1,
"variants": 3
},
"29a_F260619": {
"reads": 18,
"singletons": 1,
"variants": 3
}
}
}
}

25
obitests/obitools/obisummary/some_uniq_seq.yaml
View File

@@ -1,25 +0,0 @@
annotations:
keys:
map:
merged_sample: 3
scalar:
count: 3
map_attributes: 1
scalar_attributes: 1
vector_attributes: 0
count:
reads: 46
total_length: 300
variants: 3
samples:
sample_count: 2
sample_stats:
15a_F730814:
reads: 28
singletons: 1
variants: 3
29a_F260619:
reads: 18
singletons: 1
variants: 3

152
obitests/obitools/obisummary/test.sh
View File

@@ -1,152 +0,0 @@
#!/bin/bash
#
# Here give the name of the test serie
#
TEST_NAME=obisummary
CMD=obisummary
######
#
# Some variable and function definitions: please don't change them
#
######
TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
export PATH="${OBITOOLS_DIR}:${PATH}"
MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
TMPDIR="$(mktemp -d)"
ntest=0
success=0
failed=0
cleanup() {
echo "========================================" 1>&2
echo "## Results of the $TEST_NAME tests:" 1>&2
echo 1>&2
echo "- $ntest tests run" 1>&2
echo "- $success successfully completed" 1>&2
echo "- $failed failed tests" 1>&2
echo 1>&2
echo "Cleaning up the temporary directory..." 1>&2
echo 1>&2
echo "========================================" 1>&2
rm -rf "$TMPDIR" # Suppress the temporary directory
if [ $failed -gt 0 ]; then
log "$TEST_NAME tests failed"
log
log
exit 1
fi
log
log
exit 0
}
log() {
echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
}
log "Testing $TEST_NAME..."
log "Test directory is $TEST_DIR"
log "obitools directory is $OBITOOLS_DIR"
log "Temporary directory is $TMPDIR"
log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
######################################################################
####
#### Below are the tests
####
#### Before each test :
#### - increment the variable ntest
####
#### Run the command as the condition of an if / then /else
#### - The command must return 0 on success
#### - The command must return an exit code different from 0 on failure
#### - The datafiles are stored in the same directory than the test script
#### - The test script directory is stored in the TEST_DIR variable
#### - If result files have to be produced they must be stored
#### in the temporary directory (TMPDIR variable)
####
#### then clause is executed on success of the command
#### - Write a success message using the log function
#### - increment the variable success
####
#### else clause is executed on failure of the command
#### - Write a failure message using the log function
#### - increment the variable failed
####
######################################################################
((ntest++))
if $CMD -h > "${TMPDIR}/help.txt" 2>&1
then
log "$MCMD: printing help OK"
((success++))
else
log "$MCMD: printing help failed"
((failed++))
fi
((ntest++))
if obisummary "${TEST_DIR}/some_uniq_seq.fasta" \
> "${TMPDIR}/some_uniq_seq.json"
then
log "$MCMD: formating json execution OK"
((success++))
else
log "$MCMD: formating json execution failed"
((failed++))
fi
((ntest++))
if diff "${TEST_DIR}/some_uniq_seq.json" \
"${TMPDIR}/some_uniq_seq.json" > /dev/null
then
log "$MCMD: formating json OK"
((success++))
else
log "$MCMD: formating json failed"
((failed++))
fi
((ntest++))
if obisummary --yaml "${TEST_DIR}/some_uniq_seq.fasta" \
> "${TMPDIR}/some_uniq_seq.yaml"
then
log "$MCMD: formating yaml execution OK"
((success++))
else
log "$MCMD: formating yaml execution failed"
((failed++))
fi
((ntest++))
if diff "${TEST_DIR}/some_uniq_seq.yaml" \
"${TMPDIR}/some_uniq_seq.yaml" > /dev/null
then
log "$MCMD: formating yaml OK"
((success++))
else
log "$MCMD: formating yaml failed"
((failed++))
fi
#########################################
#
# At the end of the tests
# the cleanup function is called
#
#########################################
cleanup

148
obitests/obitools/obisuperkmer/README.md
View File

@@ -1,148 +0,0 @@
# Tests pour obisuperkmer
## Description
Ce répertoire contient les tests automatisés pour la commande `obisuperkmer`.
## Fichiers
- `test.sh` : Script de test principal (exécutable)
- `test_sequences.fasta` : Jeu de données de test minimal (3 séquences courtes)
- `README.md` : Ce fichier
## Jeu de données de test
Le fichier `test_sequences.fasta` contient 3 séquences de 32 nucléotides chacune :
1. **seq1** : Répétition du motif ACGT (séquence régulière)
2. **seq2** : Alternance de blocs homopolymères (AAAA, CCCC, GGGG, TTTT)
3. **seq3** : Répétition du motif ATCG (différent de seq1)
Ces séquences sont volontairement courtes pour :
- Minimiser la taille du dépôt Git
- Accélérer l'exécution des tests en CI/CD
- Tester différents cas d'extraction de super k-mers
## Tests effectués
Le script `test.sh` effectue 12 tests :
### Test 1 : Affichage de l'aide
Vérifie que `obisuperkmer -h` s'exécute correctement.
### Test 2 : Extraction basique avec paramètres par défaut
Exécute `obisuperkmer` avec k=21, m=11 (valeurs par défaut).
### Test 3 : Vérification du fichier de sortie non vide
S'assure que la commande produit une sortie.
### Test 4 : Comptage des super k-mers extraits
Vérifie qu'au moins un super k-mer a été extrait.
### Test 5 : Présence des métadonnées requises
Vérifie que chaque super k-mer contient :
- `minimizer_value`
- `minimizer_seq`
- `parent_id`
### Test 6 : Extraction avec paramètres personnalisés
Teste avec k=15 et m=7.
### Test 7 : Vérification des paramètres dans les métadonnées
S'assure que les valeurs k=15 et m=7 sont présentes dans la sortie.
### Test 8 : Format de sortie FASTA explicite
Teste l'option `--fasta-output`.
### Test 9 : Vérification des IDs des super k-mers
S'assure que tous les IDs contiennent "superkmer".
### Test 10 : Préservation des IDs parents
Vérifie que seq1, seq2 et seq3 apparaissent dans la sortie.
### Test 11 : Option -o pour fichier de sortie
Teste la redirection vers un fichier avec `-o`.
### Test 12 : Vérification de la création du fichier avec -o
S'assure que le fichier de sortie a été créé.
### Test 13 : Cohérence des longueurs
Vérifie que la somme des longueurs des super k-mers est inférieure ou égale à la longueur totale des séquences d'entrée.
## Exécution des tests
### Localement
```bash
cd /chemin/vers/obitools4/obitests/obitools/obisuperkmer
./test.sh
```
### En CI/CD
Les tests sont automatiquement exécutés lors de chaque commit via le système CI/CD configuré pour le projet.
### Prérequis
- La commande `obisuperkmer` doit être compilée et disponible dans `../../build/`
- Les dépendances système : bash, grep, etc.
## Structure du script de test
Le script suit le pattern standard utilisé par tous les tests OBITools :
1. **En-tête** : Définition du nom du test et de la commande
2. **Variables** : Configuration des chemins et compteurs
3. **Fonction cleanup()** : Affiche les résultats et nettoie le répertoire temporaire
4. **Fonction log()** : Affiche les messages horodatés
5. **Tests** : Série de tests avec incrémentation des compteurs
6. **Appel cleanup()** : Nettoyage et sortie avec code de retour approprié
## Format de sortie
Chaque test affiche :
```
[obisuperkmer @ date] message
```
En fin d'exécution :
```
========================================
## Results of the obisuperkmer tests:
- 12 tests run
- 12 successfully completed
- 0 failed tests
Cleaning up the temporary directory...
========================================
```
## Codes de retour
- **0** : Tous les tests ont réussi
- **1** : Au moins un test a échoué
## Ajout de nouveaux tests
Pour ajouter un nouveau test, suivre le pattern :
```bash
((ntest++))
if commande_test arguments
then
log "Description: OK"
((success++))
else
log "Description: failed"
((failed++))
fi
```
## Notes
- Les fichiers temporaires sont créés dans `$TMPDIR` (créé par mktemp)
- Les fichiers de données sont dans `$TEST_DIR`
- La commande testée doit être dans `$OBITOOLS_DIR` (../../build/)
- Le répertoire temporaire est automatiquement nettoyé à la fin

232
obitests/obitools/obisuperkmer/test.sh
View File

@@ -1,232 +0,0 @@
#!/bin/bash
#
# Here give the name of the test serie
#
TEST_NAME=obik-super
CMD=obik
######
#
# Some variable and function definitions: please don't change them
#
######
TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
export PATH="${OBITOOLS_DIR}:${PATH}"
MCMD="OBIk-super"
TMPDIR="$(mktemp -d)"
ntest=0
success=0
failed=0
cleanup() {
echo "========================================" 1>&2
echo "## Results of the $TEST_NAME tests:" 1>&2
echo 1>&2
echo "- $ntest tests run" 1>&2
echo "- $success successfully completed" 1>&2
echo "- $failed failed tests" 1>&2
echo 1>&2
echo "Cleaning up the temporary directory..." 1>&2
echo 1>&2
echo "========================================" 1>&2
rm -rf "$TMPDIR" # Suppress the temporary directory
if [ $failed -gt 0 ]; then
log "$TEST_NAME tests failed"
log
log
exit 1
fi
log
log
exit 0
}
log() {
echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
}
log "Testing $TEST_NAME..."
log "Test directory is $TEST_DIR"
log "obitools directory is $OBITOOLS_DIR"
log "Temporary directory is $TMPDIR"
log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
######################################################################
####
#### Below are the tests
####
######################################################################
((ntest++))
if $CMD super -h > "${TMPDIR}/help.txt" 2>&1
then
log "$MCMD: printing help OK"
((success++))
else
log "$MCMD: printing help failed"
((failed++))
fi
# Test 1: Basic super k-mer extraction with default parameters
((ntest++))
if $CMD super "${TEST_DIR}/test_sequences.fasta" \
> "${TMPDIR}/output_default.fasta" 2>&1
then
log "$MCMD: basic extraction with default parameters OK"
((success++))
else
log "$MCMD: basic extraction with default parameters failed"
((failed++))
fi
# Test 2: Verify output is not empty
((ntest++))
if [ -s "${TMPDIR}/output_default.fasta" ]
then
log "$MCMD: output file is not empty OK"
((success++))
else
log "$MCMD: output file is empty - failed"
((failed++))
fi
# Test 3: Count number of super k-mers extracted (should be > 0)
((ntest++))
num_sequences=$(grep -c "^>" "${TMPDIR}/output_default.fasta")
if [ "$num_sequences" -gt 0 ]
then
log "$MCMD: extracted $num_sequences super k-mers OK"
((success++))
else
log "$MCMD: no super k-mers extracted - failed"
((failed++))
fi
# Test 4: Verify super k-mers have required metadata attributes
((ntest++))
if grep -q "minimizer_value" "${TMPDIR}/output_default.fasta" && \
grep -q "minimizer_seq" "${TMPDIR}/output_default.fasta" && \
grep -q "parent_id" "${TMPDIR}/output_default.fasta"
then
log "$MCMD: super k-mers contain required metadata OK"
((success++))
else
log "$MCMD: super k-mers missing metadata - failed"
((failed++))
fi
# Test 5: Extract super k-mers with custom k and m parameters
((ntest++))
if $CMD super -k 15 -m 7 "${TEST_DIR}/test_sequences.fasta" \
> "${TMPDIR}/output_k15_m7.fasta" 2>&1
then
log "$MCMD: extraction with custom k=15, m=7 OK"
((success++))
else
log "$MCMD: extraction with custom k=15, m=7 failed"
((failed++))
fi
# Test 6: Verify custom parameters in output metadata
((ntest++))
if grep -q '"k":15' "${TMPDIR}/output_k15_m7.fasta" && \
grep -q '"m":7' "${TMPDIR}/output_k15_m7.fasta"
then
log "$MCMD: custom parameters correctly set in metadata OK"
((success++))
else
log "$MCMD: custom parameters not in metadata - failed"
((failed++))
fi
# Test 7: Test with different output format (FASTA output explicitly)
((ntest++))
if $CMD super --fasta-output -k 21 -m 11 \
"${TEST_DIR}/test_sequences.fasta" \
> "${TMPDIR}/output_fasta.fasta" 2>&1
then
log "$MCMD: FASTA output format OK"
((success++))
else
log "$MCMD: FASTA output format failed"
((failed++))
fi
# Test 8: Verify all super k-mers have superkmer in their ID
((ntest++))
if grep "^>" "${TMPDIR}/output_default.fasta" | grep -q "superkmer"
then
log "$MCMD: super k-mer IDs contain 'superkmer' OK"
((success++))
else
log "$MCMD: super k-mer IDs missing 'superkmer' - failed"
((failed++))
fi
# Test 9: Verify parent sequence IDs are preserved
((ntest++))
if grep -q "seq1" "${TMPDIR}/output_default.fasta" && \
grep -q "seq2" "${TMPDIR}/output_default.fasta" && \
grep -q "seq3" "${TMPDIR}/output_default.fasta"
then
log "$MCMD: parent sequence IDs preserved OK"
((success++))
else
log "$MCMD: parent sequence IDs not preserved - failed"
((failed++))
fi
# Test 10: Test with output file option
((ntest++))
if $CMD super -o "${TMPDIR}/output_file.fasta" \
"${TEST_DIR}/test_sequences.fasta" 2>&1
then
log "$MCMD: output to file with -o option OK"
((success++))
else
log "$MCMD: output to file with -o option failed"
((failed++))
fi
# Test 11: Verify output file was created with -o option
((ntest++))
if [ -s "${TMPDIR}/output_file.fasta" ]
then
log "$MCMD: output file created with -o option OK"
((success++))
else
log "$MCMD: output file not created with -o option - failed"
((failed++))
fi
# Test 12: Verify each super k-mer length is >= k (default k=31)
((ntest++))
min_len=$(grep -v "^>" "${TMPDIR}/output_default.fasta" | awk '{print length}' | sort -n | head -1)
if [ "$min_len" -ge 31 ]
then
log "$MCMD: all super k-mers have length >= k OK"
((success++))
else
log "$MCMD: some super k-mers shorter than k ($min_len < 31) - failed"
((failed++))
fi
#########################################
#
# At the end of the tests
# the cleanup function is called
#
#########################################
cleanup

6
obitests/obitools/obisuperkmer/test_sequences.fasta
View File

@@ -1,6 +0,0 @@
>seq1
ACGTACGTACGTACGTACGTACGTACGTACGT
>seq2
AAAACCCCGGGGTTTTAAAACCCCGGGGTTTT
>seq3
ATCGATCGATCGATCGATCGATCGATCGATCG

109
obitests/obitools/obitag/test.sh
View File

@@ -1,109 +0,0 @@
#!/bin/bash
#
# Here give the name of the test serie
#
TEST_NAME=obitag
CMD=obitag
######
#
# Some variable and function definitions: please don't change them
#
######
TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
export PATH="${OBITOOLS_DIR}:${PATH}"
MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
TMPDIR="$(mktemp -d)"
ntest=0
success=0
failed=0
cleanup() {
echo "========================================" 1>&2
echo "## Results of the $TEST_NAME tests:" 1>&2
echo 1>&2
echo "- $ntest tests run" 1>&2
echo "- $success successfully completed" 1>&2
echo "- $failed failed tests" 1>&2
echo 1>&2
echo "Cleaning up the temporary directory..." 1>&2
echo 1>&2
echo "========================================" 1>&2
rm -rf "$TMPDIR" # Suppress the temporary directory
if [ $failed -gt 0 ]; then
log "$TEST_NAME tests failed"
log
log
exit 1
fi
log
log
exit 0
}
log() {
echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
}
log "Testing $TEST_NAME..."
log "Test directory is $TEST_DIR"
log "obitools directory is $OBITOOLS_DIR"
log "Temporary directory is $TMPDIR"
log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
######################################################################
####
#### Below are the tests
####
#### Before each test :
#### - increment the variable ntest
####
#### Run the command as the condition of an if / then /else
#### - The command must return 0 on success
#### - The command must return an exit code different from 0 on failure
#### - The datafiles are stored in the same directory than the test script
#### - The test script directory is stored in the TEST_DIR variable
#### - If result files have to be produced they must be stored
#### in the temporary directory (TMPDIR variable)
####
#### then clause is executed on success of the command
#### - Write a success message using the log function
#### - increment the variable success
####
#### else clause is executed on failure of the command
#### - Write a failure message using the log function
#### - increment the variable failed
####
######################################################################
((ntest++))
if $CMD -h > "${TMPDIR}/help.txt" 2>&1
then
log "$MCMD: printing help OK"
((success++))
else
log "$MCMD: printing help failed"
((failed++))
fi
#########################################
#
# At the end of the tests
# the cleanup function is called
#
#########################################
cleanup

109
obitests/obitools/obitagpcr/test.sh
View File

@@ -1,109 +0,0 @@
#!/bin/bash
#
# Here give the name of the test serie
#
TEST_NAME=obipcr
CMD=obipcr
######
#
# Some variable and function definitions: please don't change them
#
######
TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
export PATH="${OBITOOLS_DIR}:${PATH}"
MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
TMPDIR="$(mktemp -d)"
ntest=0
success=0
failed=0
cleanup() {
echo "========================================" 1>&2
echo "## Results of the $TEST_NAME tests:" 1>&2
echo 1>&2
echo "- $ntest tests run" 1>&2
echo "- $success successfully completed" 1>&2
echo "- $failed failed tests" 1>&2
echo 1>&2
echo "Cleaning up the temporary directory..." 1>&2
echo 1>&2
echo "========================================" 1>&2
rm -rf "$TMPDIR" # Suppress the temporary directory
if [ $failed -gt 0 ]; then
log "$TEST_NAME tests failed"
log
log
exit 1
fi
log
log
exit 0
}
log() {
echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
}
log "Testing $TEST_NAME..."
log "Test directory is $TEST_DIR"
log "obitools directory is $OBITOOLS_DIR"
log "Temporary directory is $TMPDIR"
log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
######################################################################
####
#### Below are the tests
####
#### Before each test :
#### - increment the variable ntest
####
#### Run the command as the condition of an if / then /else
#### - The command must return 0 on success
#### - The command must return an exit code different from 0 on failure
#### - The datafiles are stored in the same directory than the test script
#### - The test script directory is stored in the TEST_DIR variable
#### - If result files have to be produced they must be stored
#### in the temporary directory (TMPDIR variable)
####
#### then clause is executed on success of the command
#### - Write a success message using the log function
#### - increment the variable success
####
#### else clause is executed on failure of the command
#### - Write a failure message using the log function
#### - increment the variable failed
####
######################################################################
((ntest++))
if $CMD -h > "${TMPDIR}/help.txt" 2>&1
then
log "$MCMD: printing help OK"
((success++))
else
log "$MCMD: printing help failed"
((failed++))
fi
#########################################
#
# At the end of the tests
# the cleanup function is called
#
#########################################
cleanup

109
obitests/obitools/obitaxonomy/test.sh
View File

@@ -1,109 +0,0 @@
#!/bin/bash
#
# Here give the name of the test serie
#
TEST_NAME=obitaxonomy
CMD=obitaxonomy
######
#
# Some variable and function definitions: please don't change them
#
######
TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
export PATH="${OBITOOLS_DIR}:${PATH}"
MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
TMPDIR="$(mktemp -d)"
ntest=0
success=0
failed=0
cleanup() {
echo "========================================" 1>&2
echo "## Results of the $TEST_NAME tests:" 1>&2
echo 1>&2
echo "- $ntest tests run" 1>&2
echo "- $success successfully completed" 1>&2
echo "- $failed failed tests" 1>&2
echo 1>&2
echo "Cleaning up the temporary directory..." 1>&2
echo 1>&2
echo "========================================" 1>&2
rm -rf "$TMPDIR" # Suppress the temporary directory
if [ $failed -gt 0 ]; then
log "$TEST_NAME tests failed"
log
log
exit 1
fi
log
log
exit 0
}
log() {
echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
}
log "Testing $TEST_NAME..."
log "Test directory is $TEST_DIR"
log "obitools directory is $OBITOOLS_DIR"
log "Temporary directory is $TMPDIR"
log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
######################################################################
####
#### Below are the tests
####
#### Before each test :
#### - increment the variable ntest
####
#### Run the command as the condition of an if / then /else
#### - The command must return 0 on success
#### - The command must return an exit code different from 0 on failure
#### - The datafiles are stored in the same directory than the test script
#### - The test script directory is stored in the TEST_DIR variable
#### - If result files have to be produced they must be stored
#### in the temporary directory (TMPDIR variable)
####
#### then clause is executed on success of the command
#### - Write a success message using the log function
#### - increment the variable success
####
#### else clause is executed on failure of the command
#### - Write a failure message using the log function
#### - increment the variable failed
####
######################################################################
((ntest++))
if $CMD -h > "${TMPDIR}/help.txt" 2>&1
then
log "$MCMD: printing help OK"
((success++))
else
log "$MCMD: printing help failed"
((failed++))
fi
#########################################
#
# At the end of the tests
# the cleanup function is called
#
#########################################
cleanup

258
obitests/obitools/obiuniq/test.sh
View File

@@ -1,258 +0,0 @@
#!/bin/bash
#
# Here give the name of the test serie
#
TEST_NAME=obiuniq
CMD=obiuniq
######
#
# Some variable and function definitions: please don't change them
#
######
TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
export PATH="${OBITOOLS_DIR}:${PATH}"
MCMD="$(echo "${CMD:0:4}" | tr '[:lower:]' '[:upper:]')$(echo "${CMD:4}" | tr '[:upper:]' '[:lower:]')"
TMPDIR="$(mktemp -d)"
ntest=0
success=0
failed=0
cleanup() {
echo "========================================" 1>&2
echo "## Results of the $TEST_NAME tests:" 1>&2
echo 1>&2
echo "- $ntest tests run" 1>&2
echo "- $success successfully completed" 1>&2
echo "- $failed failed tests" 1>&2
echo 1>&2
echo "Cleaning up the temporary directory..." 1>&2
echo 1>&2
echo "========================================" 1>&2
rm -rf "$TMPDIR" # Suppress the temporary directory
if [ $failed -gt 0 ]; then
log "$TEST_NAME tests failed"
log
log
exit 1
fi
log
log
exit 0
}
log() {
echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
}
log "Testing $TEST_NAME..."
log "Test directory is $TEST_DIR"
log "obitools directory is $OBITOOLS_DIR"
log "Temporary directory is $TMPDIR"
log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
######################################################################
####
#### Below are the tests
####
#### Before each test :
#### - increment the variable ntest
####
#### Run the command as the condition of an if / then /else
#### - The command must return 0 on success
#### - The command must return an exit code different from 0 on failure
#### - The datafiles are stored in the same directory than the test script
#### - The test script directory is stored in the TEST_DIR variable
#### - If result files have to be produced they must be stored
#### in the temporary directory (TMPDIR variable)
####
#### then clause is executed on success of the command
#### - Write a success message using the log function
#### - increment the variable success
####
#### else clause is executed on failure of the command
#### - Write a failure message using the log function
#### - increment the variable failed
####
######################################################################
((ntest++))
if $CMD -h > "${TMPDIR}/help.txt" 2>&1
then
log "$MCMD: printing help OK"
((success++))
else
log "$MCMD: printing help failed"
((failed++))
fi
((ntest++))
if obiuniq "${TEST_DIR}/touniq.fasta" \
> "${TMPDIR}/touniq_u.fasta"
then
log "OBIUniq simple: running OK"
((success++))
else
log "OBIUniq simple: running failed"
((failed++))
fi
obicsv -s --auto ${TEST_DIR}/touniq_u.fasta \
| tail -n +2 \
| sort \
> "${TMPDIR}/touniq_u_ref.csv"
obicsv -s --auto ${TMPDIR}/touniq_u.fasta \
| tail -n +2 \
| sort \
> "${TMPDIR}/touniq_u.csv"
((ntest++))
if diff "${TMPDIR}/touniq_u_ref.csv" \
"${TMPDIR}/touniq_u.csv" > /dev/null
then
log "OBIUniq simple: result OK"
((success++))
else
log "OBIUniq simple: result failed"
((failed++))
fi
((ntest++))
if obiuniq -c a "${TEST_DIR}/touniq.fasta" \
> "${TMPDIR}/touniq_u_a.fasta"
then
log "OBIUniq one category: running OK"
((success++))
else
log "OBIUniq one category: running failed"
((failed++))
fi
obicsv -s --auto ${TEST_DIR}/touniq_u_a.fasta \
| tail -n +2 \
| sort \
> "${TMPDIR}/touniq_u_a_ref.csv"
obicsv -s --auto ${TMPDIR}/touniq_u_a.fasta \
| tail -n +2 \
| sort \
> "${TMPDIR}/touniq_u_a.csv"
((ntest++))
if diff "${TMPDIR}/touniq_u_a_ref.csv" \
"${TMPDIR}/touniq_u_a.csv" > /dev/null
then
log "OBIUniq one category: result OK"
((success++))
else
log "OBIUniq one category: result failed"
((failed++))
fi
((ntest++))
if obiuniq -c a -c b "${TEST_DIR}/touniq.fasta" \
> "${TMPDIR}/touniq_u_a_b.fasta"
then
log "OBIUniq two categories: running OK"
((success++))
else
log "OBIUniq two categories: running failed"
((failed++))
fi
obicsv -s --auto ${TEST_DIR}/touniq_u_a_b.fasta \
| tail -n +2 \
| sort \
> "${TMPDIR}/touniq_u_a_b_ref.csv"
obicsv -s --auto ${TMPDIR}/touniq_u_a_b.fasta \
| tail -n +2 \
| sort \
> "${TMPDIR}/touniq_u_a_b.csv"
((ntest++))
if diff "${TMPDIR}/touniq_u_a_b_ref.csv" \
"${TMPDIR}/touniq_u_a_b.csv" > /dev/null
then
log "OBIUniq two categories: result OK"
((success++))
else
log "OBIUniq two categories: result failed"
((failed++))
fi
##
## Test merge attributes consistency between in-memory and on-disk paths
## This test catches the bug where the shared classifier in the on-disk
## dereplication path caused incorrect merged attributes.
##
((ntest++))
if obiuniq -m a -m b --in-memory \
"${TEST_DIR}/touniq.fasta" \
> "${TMPDIR}/touniq_u_merge_mem.fasta" 2>/dev/null
then
log "OBIUniq merge in-memory: running OK"
((success++))
else
log "OBIUniq merge in-memory: running failed"
((failed++))
fi
((ntest++))
if obiuniq -m a -m b --chunk-count 4 \
"${TEST_DIR}/touniq.fasta" \
> "${TMPDIR}/touniq_u_merge_disk.fasta" 2>/dev/null
then
log "OBIUniq merge on-disk: running OK"
((success++))
else
log "OBIUniq merge on-disk: running failed"
((failed++))
fi
# Extract sorted annotations (JSON attributes) from both outputs
# to compare merge results independently of sequence ordering
grep '^>' "${TMPDIR}/touniq_u_merge_mem.fasta" \
| sed 's/^>seq[0-9]* //' \
| sort \
> "${TMPDIR}/touniq_u_merge_mem.json"
grep '^>' "${TMPDIR}/touniq_u_merge_disk.fasta" \
| sed 's/^>seq[0-9]* //' \
| sort \
> "${TMPDIR}/touniq_u_merge_disk.json"
((ntest++))
if diff "${TMPDIR}/touniq_u_merge_mem.json" \
"${TMPDIR}/touniq_u_merge_disk.json" > /dev/null
then
log "OBIUniq merge on-disk vs in-memory: result OK"
((success++))
else
log "OBIUniq merge on-disk vs in-memory: result failed"
((failed++))
fi
#########################################
#
# At the end of the tests
# the cleanup function is called
#
#########################################
cleanup

16
obitests/obitools/obiuniq/touniq.fasta
View File

@@ -1,16 +0,0 @@
>seq1 {"a":2, "b":4,"c":5}
aaacccgggttt
>seq2 {"a":3, "b":4,"c":5}
aaacccgggttt
>seq3 {"a":3, "b":5,"c":5}
aaacccgggttt
>seq4 {"a":3, "b":5,"c":6}
aaacccgggttt
>seq5 {"a":2, "b":4,"c":5}
aaacccgggtttca
>seq6 {"a":3, "b":4,"c":5}
aaacccgggtttca
>seq7 {"a":3, "b":5,"c":5}
aaacccgggtttca
>seq8 {"a":3, "b":5,"c":6}
aaacccgggtttca

4
obitests/obitools/obiuniq/touniq_u.fasta
View File

@@ -1,4 +0,0 @@
>seq5 {"count":4}
aaacccgggtttca
>seq1 {"count":4}
aaacccgggttt

8
obitests/obitools/obiuniq/touniq_u_a.fasta
View File

@@ -1,8 +0,0 @@
>seq5 {"a":2,"b":4,"c":5,"count":1}
aaacccgggtttca
>seq6 {"a":3,"count":3}
aaacccgggtttca
>seq1 {"a":2,"b":4,"c":5,"count":1}
aaacccgggttt
>seq2 {"a":3,"count":3}
aaacccgggttt

12
obitests/obitools/obiuniq/touniq_u_a_b.fasta
View File

@@ -1,12 +0,0 @@
>seq5 {"a":2,"b":4,"c":5,"count":1}
aaacccgggtttca
>seq6 {"a":3,"b":4,"c":5,"count":1}
aaacccgggtttca
>seq7 {"a":3,"b":5,"count":2}
aaacccgggtttca
>seq1 {"a":2,"b":4,"c":5,"count":1}
aaacccgggttt
>seq2 {"a":3,"b":4,"c":5,"count":1}
aaacccgggttt
>seq3 {"a":3,"b":5,"count":2}
aaacccgggttt

34
pkg/obialign/alignment.go
View File

@@ -10,7 +10,6 @@ import (
"strings"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
)
// // A pool of byte slices.
@@ -159,30 +158,12 @@ func BuildQualityConsensus(seqA, seqB *obiseq.BioSequence, path []int, statOnMis
match := 0
left := obiutils.Abs(path[0])
right := 0
if path[len(path)-1] == 0 {
right = path[len(path)-2]
}
right = obiutils.Abs(right)
right = len(*bufferQA) - right
// obilog.Warnf("BuildQualityConsensus: left = %d right = %d\n", left, right)
for i, qA = range *bufferQA {
nA := (*bufferSA)[i]
nB := (*bufferSB)[i]
qB = (*bufferQB)[i]
if statOnMismatch && i >= left && i < right && nA != nB {
if nA == ' ' {
nA = '-'
}
if nB == ' ' {
nB = '-'
}
if statOnMismatch && nA != nB && nA != ' ' && nB != ' ' {
mismatches[strings.ToUpper(fmt.Sprintf("(%c:%02d)->(%c:%02d)", nA, qA, nB, qB))] = i + 1
}
@@ -202,12 +183,13 @@ func BuildQualityConsensus(seqA, seqB *obiseq.BioSequence, path []int, statOnMis
q := qA + qB
if nA != nB {
q = qM - byte(math.Log10(1-math.Pow(10, -float64(qm)/40))*10+0.5)
}
if nA == nB {
match++
if qA > 0 && qB > 0 {
if nA != nB {
q = qM - byte(math.Log10(1-math.Pow(10, -float64(qm)/30))*10+0.5)
}
if nA == nB {
match++
}
}
if q > 90 {

52
pkg/obialign/dnamatrix.go
View File

@@ -74,30 +74,6 @@ func _Logaddexp(a, b float64) float64 {
return b + math.Log1p(math.Exp(a-b))
}
func _Log1mexp(a float64) float64 {
if a > 0 {
log.Panic("Log1mexp: a > 0")
}
if a == 0 {
return 0
}
return (math.Log(-math.Expm1(a)))
}
func _Logdiffexp(a, b float64) float64 {
if a < b {
log.Panic("Log1mexp: a < b")
}
if a == b {
return math.Inf(-1)
}
return a + _Log1mexp(b-a)
}
// _MatchScoreRatio calculates the match score ratio between two bytes.
//
// Parameters:
@@ -107,25 +83,25 @@ func _Logdiffexp(a, b float64) float64 {
// Returns:
// - float64: the match score ratio when a match is observed
// - float64: the match score ratio when a mismatch is observed
func _MatchScoreRatio(QF, QR byte) (float64, float64) {
func _MatchScoreRatio(a, b byte) (float64, float64) {
l2 := math.Log(2)
l3 := math.Log(3)
l4 := math.Log(4)
l10 := math.Log(10)
qF := -float64(QF) / 10 * l10
qR := -float64(QR) / 10 * l10
term1 := _Logaddexp(qF, qR)
term2 := _Logdiffexp(term1, qF+qR)
lalea := math.Log(4) // 1 /(change of the random model)
lE1 := -float64(a)/10*l10 - l3 // log proba of sequencing error on A/3
lE2 := -float64(b)/10*l10 - l3 // log proba of sequencing error on B/3
lO1 := math.Log1p(-math.Exp(lE1 + l3)) // log proba no being an error on A
lO2 := math.Log1p(-math.Exp(lE2 + l3)) // log proba no being an error on B
lO1O2 := lO1 + lO2
lE1E2 := lE1 + lE2
lO1E2 := lO1 + lE2
lO2E1 := lO2 + lE1
// obilog.Warnf("MatchScoreRatio: %v, %v , %v, %v", QF, QR, term1, term2)
MM := _Logaddexp(lO1O2, lE1E2+l3) // Proba match when match observed
Mm := _Logaddexp(_Logaddexp(lO1E2, lO2E1), lE1E2+l2) // Proba match when mismatch observed
match_logp := _Log1mexp(term2 + l3 - l4)
match_score := match_logp - _Log1mexp(match_logp)
mismatch_logp := term2 - l4
mismatch_score := mismatch_logp - _Log1mexp(mismatch_logp)
return match_score, mismatch_score
return MM + lalea, Mm + lalea
}
func _InitNucPartMatch() {

Some files were not shown because too many files have changed in this diff Show More