Some typos

modified:   .gitignore

.github/workflows/obitest.yml

@@ -0,0 +1,19 @@
+name: "Run the obitools command test suite"
+
+on:
+  push:
+    branches:
+      - master
+      - V*
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Setup Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: '1.23'
+    - name: Checkout obitools4 project
+      uses: actions/checkout@v4
+    - name: Run tests
+      run: make githubtests

.gitignore

@@ -1,120 +1,27 @@
-cpu.pprof
+**/cpu.pprof
-cpu.trace
+**/cpu.trace
-test
+**/test
-bin
+**/bin
-vendor
+**/vendor
-*.fastq
+**/*.fastq
-*.fasta
+**/*.fasta
-*.fastq.gz
+**/*.fastq.gz
-*.fasta.gz
+**/*.fasta.gz
-.DS_Store
+**/.DS_Store
-*.gml
+**/*.gml
-*.log
+**/*.log
-/argaly
+**/xxx*
+**/*.sav
+**/*.old
+**/*.tgz
+**/*.yaml
+**/*.csv
 
-/obiconvert
+.rhistory
-/obicount
+/.vscode
-/obimultiplex
-/obipairing
-/obipcr
-/obifind
-/obidistribute
-/obiuniq
 /build
-/Makefile.old
-.Rproj.user
-obitools.Rproj
-Stat_error.knit.md
-.Rhistory
-Stat_error.nb.html
-Stat_error.Rmd
 
-/.luarc.json
+/ncbitaxo
-/doc/TAXO/
-/doc/results/
-/doc/_main.log
-/doc/_book/_main.tex
-/doc/_freeze/
-/doc/tutorial_files/
-/doc/wolf_data/
-/taxdump/
-/.vscode/
 
-/Algo-Alignement.numbers
+!/obitests/**
-/Estimate_proba_true_seq.html
+!/sample/**
-/Estimate_proba_true_seq.nb.html
-/Estimate_proba_true_seq.Rmd
-/modele_error_euka.qmd
-/obitools.code-workspace
-.DS_Store
-.RData
-x
-xxx
-y
-/doc/wolf_diet.tgz
-/doc/man/depends
-/sample/wolf_R1.fasta.gz
-/sample/wolf_R2.fasta.gz
-/sample/euka03.ecotag.fasta.gz
-/sample/ratio.csv
-/sample/STD_PLN_1.dat
-/sample/STD_PLN_2.dat
-/sample/subset_Pasvik_R1.fastq.gz
-/sample/subset_Pasvik_R2.fastq.gz
-/sample/test_gobitools.fasta.bz2
-euka03.csv*
-gbbct793.seq.gz
-gbinv1003.seq.gz
-gbpln210.seq
-/doc/book/OBITools-V4.aux
-/doc/book/OBITools-V4.fdb_latexmk
-/doc/book/OBITools-V4.fls
-/doc/book/OBITools-V4.log
-/doc/book/OBITools-V4.pdf
-/doc/book/OBITools-V4.synctex.gz
-/doc/book/OBITools-V4.tex
-/doc/book/OBITools-V4.toc
-getoptions.adoc
-Archive.zip
-.DS_Store
-sample/.DS_Store
-sample/consensus_graphs/specimen_hac_plants_Vern_disicolor_.gml
-93954
-Bact03.e5.gb_R254.obipcr.idx.fasta.save
-sample/test.obipcr.log
-Bact02.e3.gb_R254.obipcr.fasta.gz
-Example_Arth03.ngsfilter
-SPER01.csv
-SPER03.csv
-wolf_diet_ngsfilter.txt
-xx
-xxx.gb
-yyy_geom.csv
-yyy_LCS.csv
-yyy.json
-bug_obimultiplex/toto
-bug_obimultiplex/toto_mapping
-bug_obimultiplex/tutu
-bug_obimultiplex/tutu_mapping
-bug_obipairing/GIT1_GH_ngsfilter.txt
-doc/book/TAXO/citations.dmp
-doc/book/TAXO/delnodes.dmp
-doc/book/TAXO/division.dmp
-doc/book/TAXO/gc.prt
-doc/book/TAXO/gencode.dmp
-doc/book/TAXO/merged.dmp
-doc/book/TAXO/names.dmp
-doc/book/TAXO/nodes.dmp
-doc/book/TAXO/readme.txt
-doc/book/wolf_data/Release-253/ncbitaxo/citations.dmp
-doc/book/wolf_data/Release-253/ncbitaxo/delnodes.dmp
-doc/book/wolf_data/Release-253/ncbitaxo/division.dmp
-doc/book/wolf_data/Release-253/ncbitaxo/gc.prt
-doc/book/wolf_data/Release-253/ncbitaxo/gencode.dmp
-doc/book/wolf_data/Release-253/ncbitaxo/merged.dmp
-doc/book/wolf_data/Release-253/ncbitaxo/names.dmp
-doc/book/wolf_data/Release-253/ncbitaxo/nodes.dmp
-doc/book/wolf_data/Release-253/ncbitaxo/readme.txt
-doc/book/results/toto.tasta
-sample/.DS_Store
-GO

Makefile

@@ -63,6 +63,13 @@ update-deps:
 
 test:
 	$(GOTEST) ./...
+
+obitests: 
+	@for t in $$(find obitests -name test.sh -print) ; do \
+		bash $${t} ;\
+	done 
+
+githubtests: obitools obitests
 	
 man: 
 	make -C doc man
@@ -97,5 +104,5 @@ ifneq ($(strip $(COMMIT_ID)),)
 	@rm -f $(OUTPUT)
 endif
 
-.PHONY: all packages obitools man obibook doc update-deps .FORCE
+.PHONY: all packages obitools man obibook doc update-deps obitests githubtests .FORCE
 .FORCE:

README.md

@@ -5,7 +5,7 @@ They are implemented in *GO* and are tens of times faster than OBITools2.
 
 The git for *OBITools4* is available at :
 
-> https://metabarcoding.org/obitools4
+> https://github.com/metabarcoding/obitools4
 
 ## Installing *OBITools V4*
 
@@ -13,7 +13,7 @@ An installation script that compiles the new *OBITools* on your Unix-like system
 The easiest way to run it is to copy and paste the following command into your terminal
 
 ```{bash}
-curl -L https://metabarcoding.org/obitools4/install.sh | bash
+curl -L https://raw.githubusercontent.com/metabarcoding/obitools4/master/install_obitools.sh | bash
 ```
 
 By default, the script installs the *OBITools* commands and other associated files into the `/usr/local` directory.
@@ -33,11 +33,11 @@ available on your system, the installation script offers two options:
 You can use these options by following the installation command:
 
 ```{bash}
-curl -L https://metabarcoding.org/obitools4/install.sh | \
+curl -L https://raw.githubusercontent.com/metabarcoding/obitools4/master/install_obitools.sh | \
       bash -s -- --install-dir test_install --obitools-prefix k
 ```
 
-In this case, the binaries will be installed in the `test_install` directory and all command names will be prefixed with the letter `k`. Thus `obigrep` will be named `kobigrep`.
+In this case, the binaries will be installed in the `test_install` directory and all command names will be prefixed with the letter `k`. Thus, `obigrep` will be named `kobigrep`.
 
 ## Continuing the analysis...
 

Release-notes.md

@@ -1,25 +1,141 @@
 # OBITools release notes
 
-## Latest changes
+## March 2nd, 2025. Release 4.3.0
 
-### CPU limitation
+A new documentation website is available at https://obitools4.metabarcoding.org.
+Its development is still in progress. 
+
+### Breaking changes
+
+- In `obimultiplex`, the short version of the **--tag-list** option used to
+  specify the list of tags and primers to be used for the demultiplexing has
+  been changed from `-t` to `-s`.
+
+- The command `obifind` is now renamed `obitaxonomy`.
+
+- The **--taxdump** option used to specify the path to the taxdump containing
+  the NCBI taxonomy has been renamed to **--taxonomy**.
+
+### Bug fixes
+
+- Correction of a bug when using paired sequence file with the **--out** option.
+
+- Correction of a bug in `obitag` when trying to annotate very short sequences of
+  4 bases or less.
+  
+
+- In `obipairing`, correct the stats `seq_a_single` and `seq_b_single` when
+  on right alignment mode
+
+- Not really a bug but the memory impact of `obiuniq` has been reduced by reducing
+  the batch size and not reading the qualities from the fastq files as `obiuniq`
+  is producing only fasta output without qualities.
+
+-   In `obitag`, correct the wrong assignment of the **obitag_bestmatch**
+    attribute.
+
+-   In `obiclean`, the **--no-progress-bar** option disables all progress bars,
+    not just the data.
+
+-   Several fixes in reading FASTA and FASTQ files, including some code
+    simplification and factorization.
+
+-   Fixed a bug in all obitools that caused the same file to be processed
+    multiple times, when specifying a directory name as input.
 
--   By default, *OBITools4* tries to use all the computing power available on
-    your computer. In some circumstances this can be problematic (e.g. if you
-    are running on a computer cluster managed by your university). You can limit
-    the number of CPU cores used by *OBITools4* or by using the **--max-cpu**
-    option or by setting the **OBIMAXCPU** environment variable. Some strange
-    behaviour of *OBITools4* has been observed when users try to limit the
-    maximum number of usable CPU cores to one. This seems to be caused by the Go
-    language, and it is not obvious to get *OBITools4* to run correctly on a
-    single core in all circumstances. Therefore, if you ask to use a single
-    core, **OBITools4** will print a warning message and actually set this
-    parameter to two cores. If you really want a single core, you can use the
-    **--force-one-core** option. But be aware that this can lead to incorrect
-    calculations.
 
 ### New features
 
+-   `obigrep` add a new **--valid-taxid** option to keep only sequence with a
+    valid taxid
+
+-   `obiclean` add a new **--min-sample-count** option with a default value of 1,
+    asking to filter out sequences which are not occurring in at least the
+    specified number of samples.
+
+-   `obitoaxonomy` a new **--dump|D** option allows for dumping a sub-taxonomy.
+  
+-   Taxonomy dump can now be provided as a four-columns CSV file to the
+    **--taxonomy** option.
+
+-   NCBI Taxonomy dump does not need to be uncompressed and unarchived anymore. The
+    path of the tar and gziped dump file can be directly specified using the
+    **--taxonomy** option.
+
+-   Most of the time obitools identify automatically sequence file format. But
+    it fails sometimes. Two new option **--fasta** and **--fastq** are added to
+    allow the processing of the rare fasta and fastq files not recognized.
+    
+-   In `obiscript`, adds new methods to the Lua sequence object:
+    - `md5_string()`: returning the MD5 check sum as a hexadecimal string, 
+    - `subsequence(from,to)`: allows extracting a subsequence on a 0 based
+      coordinate system, upper bound excluded like in go. 
+    - `reverse_complement`: returning a sequence object corresponding to the
+      reverse complement of the current sequence.
+
+### Enhancement
+
+-   In every *OBITools* command, the progress bar is automatically deactivated
+    when the standard error output is redirected.
+-   Because Genbank and ENA:EMBL contain very large sequences, while OBITools4
+    are optimized As Genbank and ENA:EMBL contain very large sequences, while
+    OBITools4 is optimized for short sequences, `obipcr` faces some problems
+    with excessive consumption of computer resources, especially memory. Several
+    improvements in the tuning of the default `obipcr` parameters and some new
+    features, currently only available for FASTA and FASTQ file readers, have
+    been implemented to limit the memory impact of `obipcr` without changing the
+    computational efficiency too much.
+-   Logging system and therefore format, have been homogenized.
+
+
+### Change of git repository
+
+-   The OBITools4 git repository has been moved to the GitHub repository. 
+    The new address is: https://github.com/metabarcoding/obitools4.
+    Take care for using the new install script for retrieving the new version.
+
+    ```bash
+    curl -L https://metabarcoding.org/obitools4/install.sh \
+      | bash
+    ```
+
+    or with options:
+
+    ```bash
+    curl -L https://metabarcoding.org/obitools4/install.sh \
+      | bash -s -- --install-dir test_install --obitools-prefix k
+    ```
+-   The output of the obitools will evolve to produce results only in standard
+    formats such as fasta and fastq. For non-sequential data, the output will be
+    in CSV format, with the separator `,`, the decimal separator `.`, and a
+    header line with the column names. It is more convenient to use the output
+    in other programs. For example, you can use the `csvtomd` command to
+    reformat the CSV output into a Markdown table. The first command to initiate
+    this change is `obicount`, which now produces a 3-line CSV output.
+
+    ```bash
+    obicount data.csv | csvtomd 
+    ```
+
+-   Adds the new experimental `obicleandb` utility to clean up reference
+    database files created with `obipcr`. An easy way to create a reference
+    database for `obitag` is to use `obipcr` on a local copy of Genbank or EMBL.
+    However, these sequence databases are known to contain many taxonomic
+    errors, such as bacterial sequences annotated with the taxid of their host
+    species. `obicleandb` tries to detect these errors. To do this, it first keeps
+    only sequences annotated with the taxid to which a species, genus, and
+    family taxid can be assigned. Then, for each sequence, it compares the
+    distance of the sequence to the other sequences belonging to the same genus
+    to the same number of distances between the considered sequence and a
+    randomly selected set of sequences belonging to another family using a
+    Mann-Whitney U test. The alternative hypothesis is that out-of-family
+    distances are greater than intrageneric distances. Sequences are annotated
+    with the p-value of the Mann-Whitney U test in the **obicleandb_trusted**
+    slot. Later, the distribution of this p-value can be analyzed to determine a
+    threshold. Empirically, a threshold of 0.05 is a good compromise and allows
+    filtering out less than 1‰ of the sequences. These sequences can then be
+    removed using `obigrep`.
+
 -   Adds a new `obijoin` utility to join information contained in a sequence
     file with that contained in another sequence or CSV file. The command allows
     you to specify the names of the keys in the main sequence file and in the
@@ -27,16 +143,16 @@
 
 -   Adds a new tool `obidemerge` to demerge a `merge_xxx` slot by recreating the 
     multiple identical sequences having the slot `xxx` recreated with its initial
-    value and the sequence count set to the number of occurences refered in the
+    value and the sequence count set to the number of occurrences referred in the
     `merge_xxx` slot. During the operation, the `merge_xxx` slot is removed.
 
 -   Adds CSV as one of the input format for every obitools command. To encode
-    sequence the CSV file must includes a column named `sequence` and another
+    sequence the CSV file must include a column named `sequence` and another
     column named `id`. An extra column named `qualities` can be added to specify 
-    the quality scores of the sequence following the same ascii encoding than the
+    the quality scores of the sequence following the same ASCII encoding than the
     fastq format. All the other columns will be considered as annotations and will
     be interpreted as JSON objects encoding potentially for atomic values. If a 
-    calumn value can not be decoded as JSON it will be considered as a string.
+    column value can not be decoded as JSON it will be considered as a string.
 
 -   A new option **--version** has been added to every obitools command. It will
     print the version of the command.
@@ -45,8 +161,8 @@
     quality scores from a BioSequence object.\
 
 -   In `obimultuplex` the ngsfilter file describing the samples can be no provided
-    not only using the classical nfsfilter format but also using the csv format.
+    not only using the classical ngsfilter format but also using the CSV format.
-    When using csv, the first line must contain the column names. 5 columns are
+    When using CSV, the first line must contain the column names. 5 columns are
     expected:
 
     -   `experiment` the name of the experiment
@@ -62,43 +178,34 @@
 
     Supplementary columns are allowed. Their names and content will be used to
     annotate the sequence corresponding to the sample, as the `key=value;` did
-    in the nfsfilter format.
+    in the ngsfilter format.
 
     The CSV format used allows for comment lines starting with `#` character.
-    Special data lines starting with `@param` in the first column allow to
+    Special data lines starting with `@param` in the first column allow configuring the algorithm. The options **--template** provided an over
-    configure the algorithm. The options **--template** provided an over
+    commented example of the CSV format, including all the possible options.
-    commented example of the csv format, including all the possible options.
+    
+### CPU limitation
 
-### Enhancement
+-   By default, *OBITools4* tries to use all the computing power available on
+    your computer. In some circumstances this can be problematic (e.g. if you
+    are running on a computer cluster managed by your university). You can limit
+    the number of CPU cores used by *OBITools4* or by using the **--max-cpu**
+    option or by setting the **OBIMAXCPU** environment variable. Some strange
+    behavior of *OBITools4* has been observed when users try to limit the
+    maximum number of usable CPU cores to one. This seems to be caused by the Go
+    language, and it is not obvious to get *OBITools4* to run correctly on a
+    single core in all circumstances. Therefore, if you ask to use a single
+    core, **OBITools4** will print a warning message and actually set this
+    parameter to two cores. If you really want a single core, you can use the
+    **--force-one-core** option. But be aware that this can lead to incorrect
+    calculations.
 
--   In every *OBITools* command, the progress bar are automatically deactivated
-    when the standard error output is redirected.
--   Because Genbank and ENA:EMBL contain very large sequences, while OBITools4
-    are optimized As Genbank and ENA:EMBL contain very large sequences, while
-    OBITools4 is optimised for short sequences, `obipcr` faces some problems
-    with excessive consumption of computer resources, especially memory. Several
-    improvements in the tuning of the default `obipcr` parameters and some new
-    features, currently only available for FASTA and FASTQ file readers, have
-    been implemented to limit the memory impact of `obipcr` without changing the
-    computational efficiency too much.
--   Logging system and therefore format, have been homogenized.
-
-### Bug
-
--   In `obitag`, correct the wrong assignment of the **obitag_bestmatch**
-    attribute.
--   In `obiclean`, the **--no-progress-bar** option disables all progress bars,
-    not just the data.
--   Several fixes in reading FASTA and FASTQ files, including some code
-    simplification and and factorization.
--   Fixed a bug in all obitools that caused the same file to be processed
-    multiple times. when specifying a directory name as input.
 
 ## April 2nd, 2024. Release 4.2.0
 
 ### New features
 
--   A new OBITools named `obiscript` allows to process each sequence according
+-   A new OBITools named `obiscript` allows processing each sequence according
     to a Lua script. This is an experimental tool. The **--template** option
     allows for generating an example script on the `stdout`.
 
@@ -106,7 +213,7 @@
 
 -   Two of the main class `obiseq.SeqWorker` and `obiseq.SeqWorker` have their
     declaration changed. Both now return two values a `obiseq.BioSequenceSlice`
-    and an `error`. This allow a worker to return potentially several sequences
+    and an `error`. This allows a worker to return potentially several sequences
     as the result of the processing of a single sequence, or zero, which is
     equivalent to filter out the input sequence.
 
@@ -114,12 +221,12 @@
 
 -   In `obitag` if the reference database contains sequences annotated by taxid
     not referenced in the taxonomy, the corresponding sequences are discarded
-    from the reference database and a warning indicating the sequence id and the
+    from the reference database and a warning indicating the sequence *id* and the
     wrong taxid is emitted.
 -   The bug corrected in the parsing of EMBL and Genbank files as implemented in
     version 4.1.2 of OBITools4, potentially induced some reduction in the
     performance of the parsing. This should have been now fixed.
--   In the same idea, parsing of genbank and EMBL files were reading and storing
+-   In the same idea, parsing of Genbank and EMBL files were reading and storing
     in memory not only the sequence but also the annotations (features table).
     Up to now none of the OBITools are using this information, but with large
     complete genomes, it is occupying a lot of memory. To reduce this impact,
@@ -158,7 +265,7 @@
 
 ### New feature
 
--   In `obimatrix` a **--transpose** option allows to transpose the produced
+-   In `obimatrix` a **--transpose** option allows transposing the produced
     matrix table in CSV format.
 -   In `obitpairing` and `obipcrtag` two new options **--exact-mode** and
     **--fast-absolute** to control the heuristic used in the alignment
@@ -166,7 +273,7 @@
     the exact algorithm at the cost of a speed. **--fast-absolute** change the
     scoring schema of the heuristic.
 -   In `obiannotate` adds the possibility to annotate the first match of a
-    pattern using the same algorithm than the one used in `obipcr` and
+    pattern using the same algorithm as the one used in `obipcr` and
     `obimultiplex`. For that four option were added :
     -   **--pattern** : to specify the pattern. It can use IUPAC codes and
         position with no error tolerated has to be followed by a `#` character.
@@ -247,7 +354,7 @@
 
 ### Bugs
 
--   in the obitools language, the `composition` function now returns a map
+-   In the obitools language, the `composition` function now returns a map
     indexed by lowercase string "a", "c", "g", "t" and "o" for other instead of
     being indexed by the ASCII codes of the corresponding letters.
 -   Correction of the reverse-complement operation. Every reverse complement of
@@ -260,18 +367,18 @@
     duplicating the quality values. This made `obimultiplex` to produce fastq
     files with sequences having quality values duplicated.
 
-### Becareful
+### Be careful
 
 GO 1.21.0 is out, and it includes new functionalities which are used in the
-OBITools4 code. If you use the recommanded method for compiling OBITools on your
+OBITools4 code. If you use the recommended method for compiling OBITools on your
-computer, their is no problem, as the script always load the latest GO version.
+computer, there is no problem, as the script always load the latest GO version.
-If you rely on you personnal GO install, please think to update.
+If you rely on your personal GO install, please think to update.
 
 ## August 29th, 2023. Release 4.0.5
 
 ### Bugs
 
--   Patch a bug in the `obiseq.BioSequence` constructor leading to a error on
+-   Patch a bug in the `obiseq.BioSequence` constructor leading to an error on
     almost every obitools. The error message indicates : `fatal error: sync:
     unlock of unlocked mutex` This bug was introduced in the release 4.0.4
 
@@ -290,7 +397,7 @@ If you rely on you personnal GO install, please think to update.
     data structure to limit the number of alignments actually computed. This
     increase a bit the speed of both the software. `obirefidx` is nevertheless
     still too slow compared to my expectation.
--   Switch to a parallel version of the gzip library, allowing for high speed
+-   Switch to a parallel version of the GZIP library, allowing for high speed
     compress and decompress operation on files.
 
 ### New feature
@@ -334,12 +441,12 @@ If you rely on you personnal GO install, please think to update.
               --unidentified not_assigned.fastq
     ```
 
-    the command produced four files : `tagged_library_R1.fastq` and
+    The command produced four files : `tagged_library_R1.fastq` and
     `tagged_library_R2.fastq` containing the assigned reads and
     `not_assigned_R1.fastq` and `not_assigned_R2.fastq` containing the
     unassignable reads.
 
-    the tagged library files can then be split using `obidistribute`:
+    The tagged library files can then be split using `obidistribute`:
 
     ```{bash}
     mkdir pcr_reads
@@ -349,9 +456,9 @@ If you rely on you personnal GO install, please think to update.
 
 -   Adding of two options **--add-lca-in** and **--lca-error** to `obiannotate`.
     These options aim to help during construction of reference database using
-    `obipcr`. On obipcr output, it is commonly run obiuniq. To merge identical
+    `obipcr`. On `obipcr` output, it is commonly run `obiuniq`. To merge identical
     sequences annotated with different taxids, it is now possible to use the
-    following strategie :
+    following strategies :
 
     ```{bash}
     obiuniq -m taxid myrefdb.obipcr.fasta \
@@ -382,7 +489,7 @@ If you rely on you personnal GO install, please think to update.
 -   Correction of a bug in `obiconsensus` leading into the deletion of a base
     close to the beginning of the consensus sequence.
 
-## March 31th, 2023. Release 4.0.2
+## March 31st, 2023. Release 4.0.2
 
 ### Compiler change
 
@@ -393,15 +500,15 @@ If you rely on you personnal GO install, please think to update.
 -   Add the possibility for looking pattern with indels. This has been added to
     `obimultiplex` through the **--with-indels** option.
 -   Every obitools command has a **--pprof** option making the command
-    publishing a profiling web site available at the address :
+    publishing a profiling website available at the address :
     <http://localhost:8080/debug/pprof/>
 -   A new `obiconsensus` command has been added. It is a prototype. It aims to
     build a consensus sequence from a set of reads. The consensus is estimated
     for all the sequences contained in the input file. If several input files,
     or a directory name are provided the result contains a consensus per file.
-    The id of the sequence is the name of the input file depleted of its
+    The *id* of the sequence is the name of the input file depleted of its
     directory name and of all its extensions.
--   In `obipcr` an experimental option **--fragmented** allows for spliting very
+-   In `obipcr` an experimental option **--fragmented** allows for splitting very
     long query sequences into shorter fragments with an overlap between the two
     contiguous fragment insuring that no amplicons are missed despite the split.
     As a site effect some amplicon can be identified twice.
@@ -444,7 +551,7 @@ If you rely on you personnal GO install, please think to update.
 ### Enhancement
 
 -   *OBITools* are automatically processing all the sequences files contained in
-    a directory and its sub-directory\
+    a directory and its subdirectory\
     recursively if its name is provided as input. To process easily Genbank
     files, the corresponding filename extensions have been added. Today the
     following extensions are recognized as sequence files : `.fasta`, `.fastq`,
@@ -461,7 +568,7 @@ If you rely on you personnal GO install, please think to update.
     export OBICPUMAX=4
     ```
 
--   Adds a new option --out\|-o allowing to specify the name of an outpout file.
+-   Adds a new option --out\|-o allowing to specify the name of an output file.
 
     ``` bash
     obiconvert -o xyz.fasta xxx.fastq
@@ -483,10 +590,10 @@ If you rely on you personnal GO install, please think to update.
     matched files remain consistent when processed.
 
 -   Adding of the function `ifelse` to the expression language for computing
-    conditionnal values.
+    conditional values.
 
 -   Adding two function to the expression language related to sequence
-    conposition : `composition` and `gcskew`. Both are taking a sequence as
+    composition : `composition` and `gcskew`. Both are taking a sequence as
     single argument.
 
 ## February 18th, 2023. Release 4.0.0
@@ -494,8 +601,8 @@ If you rely on you personnal GO install, please think to update.
 It is the first version of the *OBITools* version 4. I decided to tag then
 following two weeks of intensive data analysis with them allowing to discover
 many small bugs present in the previous non-official version. Obviously other
-bugs are certainly persent in the code, and you are welcome to use the git
+bugs are certainly present in the code, and you are welcome to use the git
-ticket system to mention them. But they seems to produce now reliable results.
+ticket system to mention them. But they seem to produce now reliable results.
 
 ### Corrected bugs
 
@@ -503,11 +610,11 @@ ticket system to mention them. But they seems to produce now reliable results.
     of sequences and to the production of incorrect file because of the last
     sequence record, sometime truncated in its middle. This was only occurring
     when more than a single CPU was used. It was affecting every obitools.
--   The `obiparing` software had a bug in the right aligment procedure. This led
+-   The `obiparing` software had a bug in the right alignment procedure. This led
-    to the non alignment of very sort barcode during the paring of the forward
+    to the non-alignment of very sort barcode during the paring of the forward
     and reverse reads.
--   The `obipairing` tools had a non deterministic comportment when aligning a
+-   The `obipairing` tools had a non-deterministic comportment when aligning a
-    paor very low quality reads. This induced that the result of the same low
+    pair very low quality reads. This induced that the result of the same low
     quality read pair was not the same from run to run.
 
 ### New features
@@ -515,11 +622,10 @@ ticket system to mention them. But they seems to produce now reliable results.
 -   Adding of a `--compress|-Z` option to every obitools allowing to produce
     `gz` compressed output. OBITools were already able to deal with gziped input
     files transparently. They can now produce their results in the same format.
-    - Adding of a `--append|-A` option to the `obidistribute` tool. It allows to
+    - Adding of a `--append|-A` option to the `obidistribute` tool. It allows appending the result of an `obidistribute` execution to preexisting files. -
-    append the result of an `obidistribute` execution to preexisting files. -
     Adding of a `--directory|-d` option to the `obidistribute` tool. It allows
-    to declare a secondary classification key over the one defined by the
+    declaring a secondary classification key over the one defined by the
-    '--category\|-c\` option. This extra key leads to produce directories in
+    `--category\|-c\` option. This extra key leads to produce directories in
     which files produced according to the primary criterion are stored.
 -   Adding of the functions `subspc`, `printf`, `int`, `numeric`, and `bool` to
     the expression language.

cmd/obitools/obiannotate/main.go

@@ -3,13 +3,11 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiannotate"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
 
 func main() {
@@ -37,15 +35,11 @@ func main() {
 	_, args := optionParser(os.Args)
 
 	sequences, err := obiconvert.CLIReadBioSequences(args...)
-
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 
 	annotator := obiannotate.CLIAnnotationPipeline()
 	obiconvert.CLIWriteBioSequences(sequences.Pipe(annotator), true)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obiclean/main.go

@@ -3,11 +3,9 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiclean"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 )
@@ -18,16 +16,12 @@ func main() {
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 
 	cleaned := obiclean.CLIOBIClean(fs)
 
 	obiconvert.CLIWriteBioSequences(cleaned, true)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obicleandb/main.go

@@ -3,33 +3,28 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obicleandb"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 )
 
 func main() {
-	obioptions.SetBatchSize(10)
+	obidefault.SetBatchSize(10)
 
 	optionParser := obioptions.GenerateOptionParser(obicleandb.OptionSet)
 
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 
 	cleaned := obicleandb.ICleanDB(fs)
 
 	toconsume, _ := obiconvert.CLIWriteBioSequences(cleaned, false)
 	toconsume.Consume()
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 }

cmd/obitools/obicomplement/main.go

@@ -3,11 +3,9 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 )
@@ -18,15 +16,11 @@ func main() {
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 
 	comp := fs.MakeIWorker(obiseq.ReverseComplementWorker(true), true)
 	obiconvert.CLIWriteBioSequences(comp, true)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obiconsensus/main.go

@@ -3,11 +3,9 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconsensus"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 )
@@ -18,16 +16,12 @@ func main() {
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 
 	cleaned := obiconsensus.CLIOBIMinion(fs)
 
 	obiconvert.CLIWriteBioSequences(cleaned, true)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obiconvert/main.go

@@ -3,31 +3,26 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 )
 
 func main() {
-	obioptions.SetStrictReadWorker(2)
+	obidefault.SetStrictReadWorker(2)
-	obioptions.SetStrictWriteWorker(2)
+	obidefault.SetStrictWriteWorker(2)
 
 	optionParser := obioptions.GenerateOptionParser(obiconvert.OptionSet)
 
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 
 	obiconvert.CLIWriteBioSequences(fs, true)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obicount/main.go

@@ -4,8 +4,7 @@ import (
 	"fmt"
 	"os"
 
-	log "github.com/sirupsen/logrus"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
-
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obicount"
 
@@ -35,27 +34,24 @@ func main() {
 
 	_, args := optionParser(os.Args)
 
-	obioptions.SetStrictReadWorker(min(4, obioptions.CLIParallelWorkers()))
+	obidefault.SetStrictReadWorker(min(4, obidefault.ParallelWorkers()))
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 
 	nvariant, nread, nsymbol := fs.Count(true)
 
+	fmt.Print("entities,n\n")
+
 	if obicount.CLIIsPrintingVariantCount() {
-		fmt.Printf(" %d", nvariant)
+		fmt.Printf("variants,%d\n", nvariant)
 	}
 
 	if obicount.CLIIsPrintingReadCount() {
-		fmt.Printf(" %d", nread)
+		fmt.Printf("reads,%d\n", nread)
 	}
 
 	if obicount.CLIIsPrintingSymbolCount() {
-		fmt.Printf(" %d", nsymbol)
+		fmt.Printf("symbols,%d\n", nsymbol)
 	}
 
-	fmt.Printf("\n")
 }

cmd/obitools/obicsv/main.go

@@ -3,12 +3,10 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obicsv"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
 
 func main() {
@@ -17,14 +15,10 @@ func main() {
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
 
-	if err != nil {
+	obicsv.CLIWriteSequenceCSV(fs, true)
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 
-	obicsv.CLIWriteCSV(fs, true)
+	obiutils.WaitForLastPipe()
-
-	obiiter.WaitForLastPipe()
 
 }

cmd/obitools/obidemerge/main.go

@@ -3,34 +3,29 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obidemerge"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 )
 
 func main() {
-	obioptions.SetStrictReadWorker(2)
+	obidefault.SetStrictReadWorker(2)
-	obioptions.SetStrictWriteWorker(2)
+	obidefault.SetStrictWriteWorker(2)
 
 	optionParser := obioptions.GenerateOptionParser(obidemerge.OptionSet)
 
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 
 	demerged := obidemerge.CLIDemergeSequences(fs)
 
 	obiconvert.CLIWriteBioSequences(demerged, true)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obidistribute/main.go

@@ -3,11 +3,9 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obidistribute"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 )
@@ -18,14 +16,10 @@ func main() {
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 
 	obidistribute.CLIDistributeSequence(fs)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obifind/main.go

@@ -1,68 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"os"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obifind"
-)
-
-func main() {
-	optionParser := obioptions.GenerateOptionParser(obifind.OptionSet)
-
-	_, args := optionParser(os.Args)
-
-	//prof, _ := os.Create("obifind.prof")
-	//pprof.StartCPUProfile(prof)
-
-	restrictions, err := obifind.ITaxonRestrictions()
-	if err != nil {
-		fmt.Printf("%+v", err)
-	}
-
-	switch {
-	case obifind.CLIRequestsPathForTaxid() >= 0:
-		taxonomy, err := obifind.CLILoadSelectedTaxonomy()
-		if err != nil {
-			fmt.Printf("%+v", err)
-		}
-
-		taxon, err := taxonomy.Taxon(obifind.CLIRequestsPathForTaxid())
-
-		if err != nil {
-			fmt.Printf("%+v", err)
-		}
-
-		s, err := taxon.Path()
-
-		if err != nil {
-			fmt.Printf("%+v", err)
-		}
-
-		obifind.TaxonWriter(s.Iterator(),
-			fmt.Sprintf("path:%d", taxon.Taxid()))
-
-	case len(args) == 0:
-		taxonomy, err := obifind.CLILoadSelectedTaxonomy()
-		if err != nil {
-			fmt.Printf("%+v", err)
-		}
-
-		obifind.TaxonWriter(restrictions(taxonomy.Iterator()), "")
-
-	default:
-		matcher, err := obifind.ITaxonNameMatcher()
-
-		if err != nil {
-			fmt.Printf("%+v", err)
-		}
-
-		for _, pattern := range args {
-			s := restrictions(matcher(pattern))
-			obifind.TaxonWriter(s, pattern)
-		}
-	}
-
-	//pprof.StopCPUProfile()
-}

cmd/obitools/obigrep/main.go

@@ -3,13 +3,11 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obigrep"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
 
 func main() {
@@ -37,13 +35,10 @@ func main() {
 	_, args := optionParser(os.Args)
 
 	sequences, err := obiconvert.CLIReadBioSequences(args...)
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
 
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 	selected := obigrep.CLIFilterSequence(sequences)
 	obiconvert.CLIWriteBioSequences(selected, true)
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obijoin/main.go

@@ -3,34 +3,29 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obijoin"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 )
 
 func main() {
-	obioptions.SetStrictReadWorker(2)
+	obidefault.SetStrictReadWorker(2)
-	obioptions.SetStrictWriteWorker(2)
+	obidefault.SetStrictWriteWorker(2)
 
 	optionParser := obioptions.GenerateOptionParser(obijoin.OptionSet)
 
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 
 	joined := obijoin.CLIJoinSequences(fs)
 
 	obiconvert.CLIWriteBioSequences(joined, true)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obikmermatch/main.go

@@ -0,0 +1,51 @@
+package main
+
+import (
+	"os"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obikmersim"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+)
+
+func main() {
+
+	defer obiseq.LogBioSeqStatus()
+
+	// go tool pprof -http=":8000" ./obipairing ./cpu.pprof
+	// f, err := os.Create("cpu.pprof")
+	// if err != nil {
+	// 	log.Fatal(err)
+	// }
+	// pprof.StartCPUProfile(f)
+	// defer pprof.StopCPUProfile()
+
+	// go tool trace cpu.trace
+	// ftrace, err := os.Create("cpu.trace")
+	// if err != nil {
+	// 	log.Fatal(err)
+	// }
+	// trace.Start(ftrace)
+	// defer trace.Stop()
+
+	optionParser := obioptions.GenerateOptionParser(obikmersim.MatchOptionSet)
+
+	_, args := optionParser(os.Args)
+
+	var err error
+	sequences := obiiter.NilIBioSequence
+
+	if !obikmersim.CLISelf() {
+		sequences, err = obiconvert.CLIReadBioSequences(args...)
+	}
+
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
+
+	selected := obikmersim.CLIAlignSequences(sequences)
+	obiconvert.CLIWriteBioSequences(selected, true)
+	obiutils.WaitForLastPipe()
+
+}

cmd/obitools/obikmersimcount/main.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"log"
+	"os"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obikmersim"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+)
+
+func main() {
+
+	defer obiseq.LogBioSeqStatus()
+
+	// go tool pprof -http=":8000" ./obipairing ./cpu.pprof
+	// f, err := os.Create("cpu.pprof")
+	// if err != nil {
+	// 	log.Fatal(err)
+	// }
+	// pprof.StartCPUProfile(f)
+	// defer pprof.StopCPUProfile()
+
+	// go tool trace cpu.trace
+	// ftrace, err := os.Create("cpu.trace")
+	// if err != nil {
+	// 	log.Fatal(err)
+	// }
+	// trace.Start(ftrace)
+	// defer trace.Stop()
+
+	optionParser := obioptions.GenerateOptionParser(obikmersim.CountOptionSet)
+
+	_, args := optionParser(os.Args)
+
+	var err error
+	sequences := obiiter.NilIBioSequence
+
+	if !obikmersim.CLISelf() {
+		sequences, err = obiconvert.CLIReadBioSequences(args...)
+	}
+
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
+
+	counted := obikmersim.CLILookForSharedKmers(sequences)
+	topull, err := obiconvert.CLIWriteBioSequences(counted, false)
+
+	if err != nil {
+		log.Panic(err)
+	}
+
+	topull.Consume()
+
+	obiutils.WaitForLastPipe()
+
+}

cmd/obitools/obilandmark/main.go

@@ -3,11 +3,9 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obilandmark"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 )
@@ -18,15 +16,11 @@ func main() {
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 
 	indexed := obilandmark.CLISelectLandmarkSequences(fs)
 
 	obiconvert.CLIWriteBioSequences(indexed, true)
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obimatrix/main.go

@@ -4,8 +4,6 @@ import (
 	"fmt"
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
@@ -39,11 +37,7 @@ func main() {
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 
 	matrix := obimatrix.IMatrix(fs)
 

cmd/obitools/obimicrosat/main.go

@@ -0,0 +1,44 @@
+package main
+
+import (
+	"os"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obimicrosat"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+)
+
+func main() {
+
+	defer obiseq.LogBioSeqStatus()
+
+	// go tool pprof -http=":8000" ./obipairing ./cpu.pprof
+	// f, err := os.Create("cpu.pprof")
+	// if err != nil {
+	// 	log.Fatal(err)
+	// }
+	// pprof.StartCPUProfile(f)
+	// defer pprof.StopCPUProfile()
+
+	// go tool trace cpu.trace
+	// ftrace, err := os.Create("cpu.trace")
+	// if err != nil {
+	// 	log.Fatal(err)
+	// }
+	// trace.Start(ftrace)
+	// defer trace.Stop()
+
+	optionParser := obioptions.GenerateOptionParser(obimicrosat.OptionSet)
+
+	_, args := optionParser(os.Args)
+
+	sequences, err := obiconvert.CLIReadBioSequences(args...)
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
+
+	selected := obimicrosat.CLIAnnotateMicrosat(sequences)
+	obiconvert.CLIWriteBioSequences(selected, true)
+	obiutils.WaitForLastPipe()
+
+}

cmd/obitools/obimultiplex/main.go

@@ -6,10 +6,10 @@ import (
 
 	log "github.com/sirupsen/logrus"
 
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obimultiplex"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
 
 func main() {
@@ -43,14 +43,11 @@ func main() {
 	}
 
 	sequences, err := obiconvert.CLIReadBioSequences(args...)
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
 
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 	amplicons, _ := obimultiplex.IExtractBarcode(sequences)
 	obiconvert.CLIWriteBioSequences(amplicons, true)
 	amplicons.Wait()
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obipairing/main.go

@@ -5,10 +5,11 @@ import (
 
 	log "github.com/sirupsen/logrus"
 
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obipairing"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
 
 func main() {
@@ -33,8 +34,8 @@ func main() {
 
 	optionParser(os.Args)
 
-	obioptions.SetStrictReadWorker(2)
+	obidefault.SetStrictReadWorker(2)
-	obioptions.SetStrictWriteWorker(2)
+	obidefault.SetStrictWriteWorker(2)
 	pairs, err := obipairing.CLIPairedSequence()
 
 	if err != nil {
@@ -51,10 +52,10 @@ func main() {
 		obipairing.CLIFastMode(),
 		obipairing.CLIFastRelativeScore(),
 		obipairing.CLIWithStats(),
-		obioptions.CLIParallelWorkers(),
+		obidefault.ParallelWorkers(),
 	)
 
 	obiconvert.CLIWriteBioSequences(paired, true)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 }

cmd/obitools/obipcr/main.go

@@ -3,12 +3,11 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obipcr"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
 
 func main() {
@@ -25,24 +24,20 @@ func main() {
 	// trace.Start(ftrace)
 	// defer trace.Stop()
 
-	obioptions.SetWorkerPerCore(2)
+	obidefault.SetWorkerPerCore(2)
-	obioptions.SetReadWorkerPerCore(0.5)
+	obidefault.SetReadWorkerPerCore(0.5)
-	obioptions.SetParallelFilesRead(obioptions.CLIParallelWorkers() / 4)
+	obidefault.SetParallelFilesRead(obidefault.ParallelWorkers() / 4)
-	obioptions.SetBatchSize(10)
+	obidefault.SetBatchSize(10)
 
 	optionParser := obioptions.GenerateOptionParser(obipcr.OptionSet)
 
 	_, args := optionParser(os.Args)
 
 	sequences, err := obiconvert.CLIReadBioSequences(args...)
-
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 
 	amplicons, _ := obipcr.CLIPCR(sequences)
 	obiconvert.CLIWriteBioSequences(amplicons, true)
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obireffamidx/main.go

@@ -3,11 +3,9 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obirefidx"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 )
@@ -18,15 +16,11 @@ func main() {
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 
 	indexed := obirefidx.IndexFamilyDB(fs)
 
 	obiconvert.CLIWriteBioSequences(indexed, true)
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obirefidx/main.go

@@ -3,11 +3,9 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obirefidx"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 )
@@ -18,14 +16,11 @@ func main() {
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
 
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 	indexed := obirefidx.IndexReferenceDB(fs)
 
 	obiconvert.CLIWriteBioSequences(indexed, true)
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obiscript/main.go

@@ -4,13 +4,11 @@ import (
 	"fmt"
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiscript"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
 
 func main() {
@@ -43,15 +41,11 @@ func main() {
 	}
 
 	sequences, err := obiconvert.CLIReadBioSequences(args...)
-
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 
 	annotator := obiscript.CLIScriptPipeline()
 	obiconvert.CLIWriteBioSequences(sequences.Pipe(annotator), true)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obisplit/main.go

@@ -4,13 +4,11 @@ import (
 	"fmt"
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obisplit"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
 
 func main() {
@@ -43,15 +41,11 @@ func main() {
 	}
 
 	sequences, err := obiconvert.CLIReadBioSequences(args...)
-
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 
 	annotator := obisplit.CLISlitPipeline()
 	obiconvert.CLIWriteBioSequences(sequences.Pipe(annotator), true)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obisummary/main.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"os"
 
-	log "github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v3"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
@@ -39,11 +38,7 @@ func main() {
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 
 	summary := obisummary.ISummary(fs, obisummary.CLIMapSummary())
 

cmd/obitools/obitag/main.go

@@ -6,10 +6,12 @@ import (
 
 	log "github.com/sirupsen/logrus"
 
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obifind"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obitag"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 )
@@ -32,29 +34,40 @@ func main() {
 	// trace.Start(ftrace)
 	// defer trace.Stop()
 
-	obioptions.SetWorkerPerCore(2)
+	obidefault.SetWorkerPerCore(2)
-	obioptions.SetStrictReadWorker(1)
+	obidefault.SetStrictReadWorker(1)
-	obioptions.SetStrictWriteWorker(1)
+	obidefault.SetStrictWriteWorker(1)
-	obioptions.SetBatchSize(10)
+	obidefault.SetBatchSize(10)
 
 	optionParser := obioptions.GenerateOptionParser(obitag.OptionSet)
 
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
 
-	if err != nil {
+	taxo := obitax.DefaultTaxonomy()
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
-
-	taxo, error := obifind.CLILoadSelectedTaxonomy()
-	if error != nil {
-		log.Panicln(error)
-	}
 
 	references := obitag.CLIRefDB()
 
+	if references == nil {
+		log.Panicln("No loaded reference database")
+	}
+
+	if taxo == nil {
+		taxo, err = references.ExtractTaxonomy(nil)
+
+		if err != nil {
+			log.Fatalf("No taxonomy specified or extractable from reference database: %v", err)
+		}
+
+		taxo.SetAsDefault()
+	}
+
+	if taxo == nil {
+		log.Panicln("No loaded taxonomy")
+	}
+
 	var identified obiiter.IBioSequence
 
 	if obitag.CLIGeometricMode() {
@@ -64,7 +77,7 @@ func main() {
 	}
 
 	obiconvert.CLIWriteBioSequences(identified, true)
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 	obitag.CLISaveRefetenceDB(references)
 

cmd/obitools/obitagpcr/main.go

@@ -5,11 +5,12 @@ import (
 
 	log "github.com/sirupsen/logrus"
 
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obipairing"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obitagpcr"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
 
 func main() {
@@ -30,7 +31,7 @@ func main() {
 	// trace.Start(ftrace)
 	// defer trace.Stop()
 
-	obioptions.SetWorkerPerCore(1)
+	obidefault.SetWorkerPerCore(1)
 
 	optionParser := obioptions.GenerateOptionParser(obitagpcr.OptionSet)
 
@@ -54,5 +55,5 @@ func main() {
 
 	obiconvert.CLIWriteBioSequences(paired, true)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 }

cmd/obitools/obitaxonomy/main.go

@@ -0,0 +1,106 @@
+package main
+
+import (
+	"os"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obitaxonomy"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+
+	log "github.com/sirupsen/logrus"
+)
+
+func main() {
+	optionParser := obioptions.GenerateOptionParser(obitaxonomy.OptionSet)
+
+	_, args := optionParser(os.Args)
+
+	var iterator *obitax.ITaxon
+
+	switch {
+	case obitaxonomy.CLIDownloadNCBI():
+		err := obitaxonomy.CLIDownloadNCBITaxdump()
+		if err != nil {
+			log.Errorf("Cannot download NCBI taxonomy: %s", err.Error())
+			os.Exit(1)
+		}
+
+		os.Exit(0)
+
+	case obitaxonomy.CLIExtractTaxonomy():
+		iter, err := obiconvert.CLIReadBioSequences(args...)
+
+		if err != nil {
+			log.Fatalf("Cannot extract taxonomy: %v", err)
+		}
+
+		taxonomy, err := iter.ExtractTaxonomy()
+
+		if err != nil {
+			log.Fatalf("Cannot extract taxonomy: %v", err)
+		}
+
+		taxonomy.SetAsDefault()
+
+		log.Infof("Number of extracted taxa: %d", taxonomy.Len())
+		iterator = taxonomy.AsTaxonSet().Sort().Iterator()
+
+	case obitaxonomy.CLIDumpSubtaxonomy():
+		iterator = obitaxonomy.CLISubTaxonomyIterator()
+
+	case obitaxonomy.CLIRequestsPathForTaxid() != "NA":
+
+		taxon, isAlias, err := obitax.DefaultTaxonomy().Taxon(obitaxonomy.CLIRequestsPathForTaxid())
+
+		if err != nil {
+			log.Fatalf("Cannot identify the requested taxon: %s (%v)",
+				obitaxonomy.CLIRequestsPathForTaxid(), err)
+		}
+
+		if isAlias {
+			if obidefault.FailOnTaxonomy() {
+				log.Fatalf("Taxon %s is an alias for %s", taxon.String(), taxon.Parent().String())
+			}
+		}
+
+		s := taxon.Path()
+
+		if s == nil {
+			log.Fatalf("Cannot extract taxonomic path describing %s", taxon.String())
+		}
+
+		iterator = s.Iterator()
+
+		if obitaxonomy.CLIWithQuery() {
+			iterator = iterator.AddMetadata("query", taxon.String())
+		}
+
+	case len(args) == 0:
+		iterator = obitax.DefaultTaxonomy().Iterator()
+	default:
+		iters := make([]*obitax.ITaxon, len(args))
+
+		for i, pat := range args {
+			ii := obitax.DefaultTaxonomy().IFilterOnName(pat, obitaxonomy.CLIFixedPattern(), true)
+			if obitaxonomy.CLIWithQuery() {
+				ii = ii.AddMetadata("query", pat)
+			}
+			iters[i] = ii
+		}
+
+		iterator = iters[0]
+
+		if len(iters) > 1 {
+			iterator = iterator.Concat(iters[1:]...)
+		}
+	}
+
+	iterator = obitaxonomy.CLITaxonRestrictions(iterator)
+	obitaxonomy.CLICSVTaxaWriter(iterator, true)
+
+	obiutils.WaitForLastPipe()
+
+}

cmd/obitools/obiuniq/main.go

@@ -3,13 +3,12 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiuniq"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
 
 func main() {
@@ -32,20 +31,18 @@ func main() {
 	// trace.Start(ftrace)
 	// defer trace.Stop()
 
+	obidefault.SetBatchSize(10)
+	obidefault.SetReadQualities(false)
 	optionParser := obioptions.GenerateOptionParser(obiuniq.OptionSet)
 
 	_, args := optionParser(os.Args)
 
 	sequences, err := obiconvert.CLIReadBioSequences(args...)
-
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 
 	unique := obiuniq.CLIUnique(sequences)
 	obiconvert.CLIWriteBioSequences(unique, true)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/test/main.go

@@ -3,36 +3,14 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax"
-
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 )
 
 func main() {
-	optionParser := obioptions.GenerateOptionParser(obiconvert.OptionSet)
 
-	_, args := optionParser(os.Args)
+	obitax.DetectTaxonomyFormat(os.Args[1])
-
+	println(obiutils.RemoveAllExt("toto/tutu/test.txt"))
-	fs, err := obiconvert.CLIReadBioSequences(args...)
+	println(obiutils.Basename("toto/tutu/test.txt"))
-
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
-
-	frags := obiiter.IFragments(
-		1000,
-		100,
-		10,
-		100,
-		obioptions.CLIParallelWorkers(),
-	)
-
-	obiconvert.CLIWriteBioSequences(fs.Pipe(frags), true)
-
-	obiiter.WaitForLastPipe()
 
 }

go.mod

@@ -1,12 +1,15 @@
 module git.metabarcoding.org/obitools/obitools4/obitools4
 
-go 1.22.1
+go 1.23.1
 
 require (
 	github.com/DavidGamba/go-getoptions v0.28.0
 	github.com/PaesslerAG/gval v1.2.2
+	github.com/TuftsBCB/io v0.0.0-20140121014543-22b94e9b23f9
 	github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df
+	github.com/buger/jsonparser v1.1.1
 	github.com/chen3feng/stl4go v0.1.1
+	github.com/dlclark/regexp2 v1.11.4
 	github.com/goccy/go-json v0.10.3
 	github.com/klauspost/pgzip v1.2.6
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
@@ -23,20 +26,12 @@ require (
 )
 
 require (
-	github.com/bytedance/sonic v1.11.9 // indirect
-	github.com/bytedance/sonic/loader v0.1.1 // indirect
-	github.com/cloudwego/base64x v0.1.4 // indirect
-	github.com/cloudwego/iasm v0.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/goombaio/orderedmap v0.0.0-20180924084748-ba921b7e2419 // indirect
-	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
 	github.com/kr/pretty v0.3.0 // indirect
 	github.com/kr/text v0.2.0 // indirect
-	github.com/montanaflynn/stats v0.7.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rogpeppe/go-internal v1.6.1 // indirect
-	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
-	golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect
 )
 
 require (

go.sum

@@ -4,29 +4,25 @@ github.com/PaesslerAG/gval v1.2.2 h1:Y7iBzhgE09IGTt5QgGQ2IdaYYYOU134YGHBThD+wm9E
 github.com/PaesslerAG/gval v1.2.2/go.mod h1:XRFLwvmkTEdYziLdaCeCa5ImcGVrfQbeNUbVR+C6xac=
 github.com/PaesslerAG/jsonpath v0.1.0 h1:gADYeifvlqK3R3i2cR5B4DGgxLXIPb3TRTH1mGi0jPI=
 github.com/PaesslerAG/jsonpath v0.1.0/go.mod h1:4BzmtoM/PI8fPO4aQGIusjGxGir2BzcV0grWtFzq1Y8=
+github.com/TuftsBCB/io v0.0.0-20140121014543-22b94e9b23f9 h1:Zc1/GNsUpgZR9qm1EmRSKrnOHA7CCd0bIzGdq0cREN0=
+github.com/TuftsBCB/io v0.0.0-20140121014543-22b94e9b23f9/go.mod h1:PZyV4WA3NpqtezSY0h6E6NARAmdDm0qwrydveOyR5Gc=
 github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df h1:GSoSVRLoBaFpOOds6QyY1L8AX7uoY+Ln3BHc22W40X0=
 github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df/go.mod h1:hiVxq5OP2bUGBRNS3Z/bt/reCLFNbdcST6gISi1fiOM=
-github.com/bytedance/sonic v1.11.9 h1:LFHENlIY/SLzDWverzdOvgMztTxcfcF+cqNsz9pK5zg=
+github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
-github.com/bytedance/sonic v1.11.9/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4=
+github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
-github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM=
-github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
 github.com/chen3feng/stl4go v0.1.1 h1:0L1+mDw7pomftKDruM23f1mA7miavOj6C6MZeadzN2Q=
 github.com/chen3feng/stl4go v0.1.1/go.mod h1:5ml3psLgETJjRJnMbPE+JiHLrCpt+Ajc2weeTECXzWU=
-github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y=
-github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
-github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg=
-github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo=
+github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
 github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q=
 github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo=
 github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
 github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
 github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
-github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
-github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
 github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
 github.com/goombaio/orderedmap v0.0.0-20180924084748-ba921b7e2419 h1:SajEQ6tktpF9SRIuzbiPOX9AEZZ53Bvw0k9Mzrts8Lg=
@@ -37,13 +33,9 @@ github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1
 github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
 github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
-github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE=
 github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
-github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
-github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU=
 github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
-github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
@@ -58,8 +50,6 @@ github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZ
 github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
-github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
-github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -78,26 +68,17 @@ github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFR
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/tevino/abool/v2 v2.1.0 h1:7w+Vf9f/5gmKT4m4qkayb33/92M+Um45F2BkHOR+L/c=
 github.com/tevino/abool/v2 v2.1.0/go.mod h1:+Lmlqk6bHDWHqN1cbxqhwEAwMPXgc8I1SDEamtseuXY=
-github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
-github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
 github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
 github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
 github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
 github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
-golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU=
-golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
 golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
 golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
 golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
@@ -120,8 +101,6 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=
-rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
 scientificgo.org/special v0.0.0 h1:P6WJkECo6tgtvZAEfNXl+KEB9ReAatjKAeX8U07mjSc=
 scientificgo.org/special v0.0.0/go.mod h1:LoGVh9tS431RLTJo7gFlYDKFWq44cEb7QqL+M0EKtZU=
 scientificgo.org/testutil v0.0.0 h1:y356DHRo0tAz9zIFmxlhZoKDlHPHaWW/DCm9k3PhIMA=

go.work

@@ -1,3 +1,3 @@
-go 1.22.1
+go 1.23.1
 
 use .

install_obitools.sh

@@ -4,7 +4,7 @@ INSTALL_DIR="/usr/local"
 OBITOOLS_PREFIX=""
 # default values
 URL="https://go.dev/dl/"
-OBIURL4="https://git.metabarcoding.org/obitools/obitools4/obitools4/-/archive/master/obitools4-master.tar.gz"
+OBIURL4="https://github.com/metabarcoding/obitools4/archive/refs/heads/master.zip"
 INSTALL_DIR="/usr/local"
 OBITOOLS_PREFIX=""
 
@@ -106,8 +106,10 @@ curl "$GOURL" \
 PATH="$(pwd)/go/bin:$PATH"
 export PATH
 
-curl -L "$OBIURL4" \
+curl -L "$OBIURL4" > master.zip
-    | tar zxf - 
+unzip master.zip
+
+echo "Install OBITOOLS from : $OBIURL4"
 
 cd obitools4-master || exit
 

obitests/obitools/obicount/test.sh

@@ -0,0 +1,144 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obicount
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+       exit 1
+    fi
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+((ntest++))
+if obicount "${TEST_DIR}/wolf_F.fasta.gz" \
+    > "${TMPDIR}/wolf_F.fasta_count.csv" 
+then
+    log "OBICount: fasta reading OK" 
+    ((success++))
+else
+    log "OBICount: fasta reading failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if obicount "${TEST_DIR}/wolf_F.fastq.gz" \
+    > "${TMPDIR}/wolf_F.fastq_count.csv"
+then
+    log "OBICount: fastq reading OK"
+    ((success++))
+else
+    log "OBICount: fastq reading failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if obicount "${TEST_DIR}/wolf_F.csv.gz" \
+    > "${TMPDIR}/wolf_F.csv_count.csv"
+then
+    log "OBICount: csv reading OK" 
+    ((success++))
+else
+    log "OBICount: csv reading failed"
+    ((failed++))
+fi
+
+((ntest++))
+if diff "${TMPDIR}/wolf_F.fasta_count.csv" \
+        "${TMPDIR}/wolf_F.fastq_count.csv"  > /dev/null
+then
+    log "OBICount: counting on fasta and fastq are identical OK"
+    ((success++))
+else
+    log "OBICount: counting on fasta and fastq are different failed"
+    ((failed++))
+fi
+
+((ntest++))
+if diff "${TMPDIR}/wolf_F.fasta_count.csv" \
+        "${TMPDIR}/wolf_F.csv_count.csv" > /dev/null
+then
+    log "OBICount: counting on fasta and csv are identical OK"
+    ((success++))
+else
+    log "OBICount: counting on fasta and csv are different failed"
+    ((failed++))
+fi
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

obitests/obitools/obiparing/test.sh

@@ -0,0 +1,134 @@
+#!/bin/bash
+
+#
+# Here give the name of the test serie
+#
+
+TEST_NAME=obiparing
+
+######
+#
+# Some variable and function definitions: please don't change them
+#
+######
+TEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+OBITOOLS_DIR="${TEST_DIR/obitest*/}build"
+export PATH="${OBITOOLS_DIR}:${PATH}"
+
+
+TMPDIR="$(mktemp -d)"
+ntest=0
+success=0
+failed=0
+
+cleanup() {
+    echo "========================================" 1>&2
+    echo "## Results of the $TEST_NAME tests:" 1>&2
+
+    echo 1>&2
+    echo "- $ntest tests run" 1>&2
+    echo "- $success successfully completed" 1>&2
+    echo "- $failed failed tests" 1>&2
+    echo 1>&2
+    echo "Cleaning up the temporary directory..." 1>&2
+    echo 1>&2
+    echo "========================================" 1>&2
+
+    rm -rf "$TMPDIR"  # Suppress the temporary directory
+
+    if [ $failed -gt 0 ]; then
+       log "$TEST_NAME tests failed" 
+       exit 1
+    fi
+
+    exit 0
+}
+
+log() {
+    echo -e "[$TEST_NAME @ $(date)] $*" 1>&2
+}
+
+log "Testing $TEST_NAME..." 
+log "Test directory is $TEST_DIR" 
+log "obitools directory is $OBITOOLS_DIR" 
+log "Temporary directory is $TMPDIR" 
+log "files: $(find $TEST_DIR | awk -F'/' '{print $NF}' | tail -n +2)"
+
+######################################################################
+####
+#### Below are the tests
+####
+#### Before each test :
+####  - increment the variable ntest
+####
+#### Run the command as the condition of an if / then /else
+####  - The command must return 0 on success
+####  - The command must return an exit code different from 0 on failure
+####  - The datafiles are stored in the same directory than the test script
+####  - The test script directory is stored in the TEST_DIR variable
+####  - If result files have to be produced they must be stored
+####    in the temporary directory (TMPDIR variable)
+####
+#### then clause is executed on success of the command
+####  - Write a success message using the log function
+####  - increment the variable success
+####
+#### else clause is executed on failure of the command
+####  - Write a failure message using the log function
+####  - increment the variable failed
+####
+######################################################################
+
+((ntest++))
+if obipairing -F "${TEST_DIR}/wolf_F.fastq.gz" \
+              -R "${TEST_DIR}/wolf_R.fastq.gz" \
+    | obidistribute -Z -c mode \
+                    -p "${TMPDIR}/wolf_paired_%s.fastq.gz" 
+then
+    log "OBIPairing: sequence pairing OK" 
+    ((success++))
+else
+    log "OBIPairing: sequence pairing failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if obicsv -Z -s -i \
+          -k ali_dir -k ali_length -k paring_fast_count \
+          -k paring_fast_overlap -k paring_fast_score \
+          -k score -k score_norm -k seq_a_single \
+          -k seq_b_single -k seq_ab_match \
+          "${TMPDIR}/wolf_paired_alignment.fastq.gz" \
+    > "${TMPDIR}/wolf_paired_alignment.csv.gz" \
+    && zdiff -c "${TEST_DIR}/wolf_paired_alignment.csv.gz" \
+                "${TMPDIR}/wolf_paired_alignment.csv.gz" 
+then
+    log "OBIPairing: check aligned sequences OK" 
+    ((success++))
+else
+    log "OBIPairing: check aligned sequences failed" 
+    ((failed++))
+fi
+
+((ntest++))
+if obicsv -Z -s -i \
+          "${TMPDIR}/wolf_paired_join.fastq.gz" \
+    > "${TMPDIR}/wolf_paired_join.csv.gz" \
+    && zdiff -c "${TEST_DIR}/wolf_paired_join.csv.gz" \
+                "${TMPDIR}/wolf_paired_join.csv.gz"
+then
+    log "OBIPairing: check joined sequences OK" 
+    ((success++))
+else
+    log "OBIPairing: check joined sequences failed" 
+    ((failed++))
+fi
+
+#########################################
+#
+# At the end of the tests
+# the cleanup function is called
+#
+#########################################
+
+cleanup

pkg/obialign/alignment.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
 
 // // A pool of byte slices.
@@ -158,12 +159,30 @@ func BuildQualityConsensus(seqA, seqB *obiseq.BioSequence, path []int, statOnMis
 
 	match := 0
 
+	left := obiutils.Abs(path[0])
+	right := 0
+	if path[len(path)-1] == 0 {
+		right = path[len(path)-2]
+	}
+
+	right = obiutils.Abs(right)
+
+	right = len(*bufferQA) - right
+
+	// log.Warnf("BuildQualityConsensus: left = %d right = %d\n", left, right)
+
 	for i, qA = range *bufferQA {
 		nA := (*bufferSA)[i]
 		nB := (*bufferSB)[i]
 		qB = (*bufferQB)[i]
 
-		if statOnMismatch && nA != nB && nA != ' ' && nB != ' ' {
+		if statOnMismatch && i >= left && i < right && nA != nB {
+			if nA == ' ' {
+				nA = '-'
+			}
+			if nB == ' ' {
+				nB = '-'
+			}
 			mismatches[strings.ToUpper(fmt.Sprintf("(%c:%02d)->(%c:%02d)", nA, qA, nB, qB))] = i + 1
 		}
 
@@ -183,13 +202,12 @@ func BuildQualityConsensus(seqA, seqB *obiseq.BioSequence, path []int, statOnMis
 
 		q := qA + qB
 
-		if qA > 0 && qB > 0 {
+		if nA != nB {
-			if nA != nB {
+			q = qM - byte(math.Log10(1-math.Pow(10, -float64(qm)/40))*10+0.5)
-				q = qM - byte(math.Log10(1-math.Pow(10, -float64(qm)/30))*10+0.5)
+		}
-			}
+
-			if nA == nB {
+		if nA == nB {
-				match++
+			match++
-			}
 		}
 
 		if q > 90 {

pkg/obialign/backtracking.go

@@ -12,7 +12,7 @@ func _Backtracking(pathMatrix []int, lseqA, lseqB int, path *[]int) []int {
 	cp := cap(*path)
 	(*path) = slices.Grow((*path), needed)
 	if cp < cap(*path) {
-		log.Infof("Resized path from %d to %d\n", cp, cap(*path))
+		log.Debugf("Resized path from %d to %d\n", cp, cap(*path))
 	}
 	p := cap(*path)
 	*path = (*path)[:p]

pkg/obialign/dnamatrix.go

@@ -74,6 +74,30 @@ func _Logaddexp(a, b float64) float64 {
 	return b + math.Log1p(math.Exp(a-b))
 }
 
+func _Log1mexp(a float64) float64 {
+	if a > 0 {
+		log.Panic("Log1mexp: a > 0")
+	}
+
+	if a == 0 {
+		return 0
+	}
+
+	return (math.Log(-math.Expm1(a)))
+}
+
+func _Logdiffexp(a, b float64) float64 {
+	if a < b {
+		log.Panic("Log1mexp: a < b")
+	}
+
+	if a == b {
+		return math.Inf(-1)
+	}
+
+	return a + _Log1mexp(b-a)
+}
+
 // _MatchScoreRatio calculates the match score ratio between two bytes.
 //
 // Parameters:
@@ -83,25 +107,25 @@ func _Logaddexp(a, b float64) float64 {
 // Returns:
 // - float64: the match score ratio when a match is observed
 // - float64: the match score ratio when a mismatch is observed
-func _MatchScoreRatio(a, b byte) (float64, float64) {
+func _MatchScoreRatio(QF, QR byte) (float64, float64) {
 
-	l2 := math.Log(2)
 	l3 := math.Log(3)
+	l4 := math.Log(4)
 	l10 := math.Log(10)
-	lalea := math.Log(4)                   // 1 /(change of the random model)
+	qF := -float64(QF) / 10 * l10
-	lE1 := -float64(a)/10*l10 - l3         // log proba of sequencing error on A/3
+	qR := -float64(QR) / 10 * l10
-	lE2 := -float64(b)/10*l10 - l3         // log proba of sequencing error on B/3
+	term1 := _Logaddexp(qF, qR)
-	lO1 := math.Log1p(-math.Exp(lE1 + l3)) // log proba no being an error on A
+	term2 := _Logdiffexp(term1, qF+qR)
-	lO2 := math.Log1p(-math.Exp(lE2 + l3)) // log proba no being an error on B
-	lO1O2 := lO1 + lO2
-	lE1E2 := lE1 + lE2
-	lO1E2 := lO1 + lE2
-	lO2E1 := lO2 + lE1
 
-	MM := _Logaddexp(lO1O2, lE1E2+l3)                    // Proba match when match observed
+	// log.Warnf("MatchScoreRatio: %v, %v , %v, %v", QF, QR, term1, term2)
-	Mm := _Logaddexp(_Logaddexp(lO1E2, lO2E1), lE1E2+l2) // Proba match when mismatch observed
 
-	return MM + lalea, Mm + lalea
+	match_logp := _Log1mexp(term2 + l3 - l4)
+	match_score := match_logp - _Log1mexp(match_logp)
+
+	mismatch_logp := term2 - l4
+	mismatch_score := mismatch_logp - _Log1mexp(mismatch_logp)
+
+	return match_score, mismatch_score
 }
 
 func _InitNucPartMatch() {

pkg/obialign/fastlcs.go

@@ -21,15 +21,15 @@ func encodeValues(score, length int, out bool) uint64 {
 	return fo
 }
 
-func _isout(value uint64) bool {
+// func _isout(value uint64) bool {
-	const outmask = uint64(1) << dwsize
+// 	const outmask = uint64(1) << dwsize
-	return (value & outmask) == 0
+// 	return (value & outmask) == 0
-}
+// }
 
-func _lpath(value uint64) int {
+// func _lpath(value uint64) int {
-	const mask = uint64(1<<wsize) - 1
+// 	const mask = uint64(1<<wsize) - 1
-	return int(((value + 1) ^ mask) & mask)
+// 	return int(((value + 1) ^ mask) & mask)
-}
+// }
 
 func decodeValues(value uint64) (int, int, bool) {
 	const mask = uint64(1<<wsize) - 1
@@ -57,4 +57,3 @@ func _setout(value uint64) uint64 {
 var _empty = encodeValues(0, 0, false)
 var _out = encodeValues(0, 30000, true)
 var _notavail = encodeValues(0, 30000, false)
-

pkg/obialign/locatepattern.go

@@ -1,30 +1,73 @@
 package obialign
 
+import log "github.com/sirupsen/logrus"
+
+// buffIndex converts a pair of coordinates (i, j) into a linear index in a matrix
+// of size width x width. The coordinates are (-1)-indexed, and the linear index
+// is 0-indexed as well. The function first adds 1 to both coordinates to make
+// sure the (-1,-1) coordinate is at position 0 in the matrix, and then computes
+// the linear index by multiplying the first coordinate by the width and adding
+// the second coordinate.
 func buffIndex(i, j, width int) int {
 	return (i+1)*width + (j + 1)
 }
-func LocatePattern(pattern, sequence []byte) (int, int, int) {
+
+// LocatePattern is a function to locate a pattern in a sequence.
+//
+// It uses a dynamic programming approach to build a matrix of scores.
+// The score at each cell is the maximum of the score of the cell
+// above it (representing a deletion), the score of the cell to its
+// left (representing an insertion), and the score of the cell
+// diagonally above it (representing a match).
+//
+// The score of a match is 0 if the two characters are the same,
+// and -1 if they are different.
+//
+// The function returns the start and end positions of the best
+// match, as well as the number of errors in the best match.
+func LocatePattern(id string, pattern, sequence []byte) (int, int, int) {
+
+	if len(pattern) >= len(sequence) {
+		log.Panicf("Sequence %s:Pattern %s must be shorter than sequence %s", id, pattern, sequence)
+	}
+
+	// Pattern spreads over the columns
+	// Sequence spreads over the rows
 	width := len(pattern) + 1
 	buffsize := (len(pattern) + 1) * (len(sequence) + 1)
 	buffer := make([]int, buffsize)
+
+	// The path matrix keeps track of the best path through the matrix
+	//  0 : indicate the diagonal path
+	//  1 : indicate the up path
+	// -1 : indicate the left path
 	path := make([]int, buffsize)
 
+	// Initialize the first row of the matrix
 	for j := 0; j < len(pattern); j++ {
 		idx := buffIndex(-1, j, width)
 		buffer[idx] = -j - 1
 		path[idx] = -1
 	}
 
+	// Initialize the first column of the matrix
+	// Alignment is endgap free so first column = 0
+	// to allow primer to shift freely along the sequence
 	for i := -1; i < len(sequence); i++ {
 		idx := buffIndex(i, -1, width)
 		buffer[idx] = 0
 		path[idx] = +1
 	}
 
+	// Fills the matrix except the last column
+	// where gaps must be free too.
 	path[0] = 0
 	jmax := len(pattern) - 1
 	for i := 0; i < len(sequence); i++ {
 		for j := 0; j < jmax; j++ {
+
+			// Mismatch score = -1
+			// Match score = 0
 			match := -1
 			if _samenuc(pattern[j], sequence[i]) {
 				match = 0
@@ -33,6 +76,8 @@ func LocatePattern(pattern, sequence []byte) (int, int, int) {
 			idx := buffIndex(i, j, width)
 
 			diag := buffer[buffIndex(i-1, j-1, width)] + match
+
+			// Each gap cost -1
 			left := buffer[buffIndex(i, j-1, width)] - 1
 			up := buffer[buffIndex(i-1, j, width)] - 1
 
@@ -51,9 +96,12 @@ func LocatePattern(pattern, sequence []byte) (int, int, int) {
 		}
 	}
 
+	// Fills the last column considering the free up gap
 	for i := 0; i < len(sequence); i++ {
 		idx := buffIndex(i, jmax, width)
 
+		// Mismatch score = -1
+		// Match score = 0
 		match := -1
 		if _samenuc(pattern[jmax], sequence[i]) {
 			match = 0
@@ -65,6 +113,7 @@ func LocatePattern(pattern, sequence []byte) (int, int, int) {
 
 		score := max(diag, up, left)
 		buffer[idx] = score
+
 		switch {
 		case score == left:
 			path[idx] = -1
@@ -75,11 +124,13 @@ func LocatePattern(pattern, sequence []byte) (int, int, int) {
 		}
 	}
 
+	// Bactracking of the aligment
+
 	i := len(sequence) - 1
 	j := jmax
 	end := -1
 	lali := 0
-	for i > -1 && j > 0 {
+	for j > 0 { // C'était i > -1 && j > 0
 		lali++
 		switch path[buffIndex(i, j, width)] {
 		case 0:
@@ -100,5 +151,9 @@ func LocatePattern(pattern, sequence []byte) (int, int, int) {
 		}
 
 	}
+
+	// log.Warnf("from : %d to: %d error: %d match: %v",
+	// 	i, end+1, -buffer[buffIndex(len(sequence)-1, len(pattern)-1, width)],
+	// 	string(sequence[i:(end+1)]))
 	return i, end + 1, -buffer[buffIndex(len(sequence)-1, len(pattern)-1, width)]
 }

pkg/obialign/pairedendalign.go

@@ -1,6 +1,8 @@
 package obialign
 
 import (
+	"log"
+
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obikmer"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 )
@@ -313,6 +315,105 @@ func _FillMatrixPeRightAlign(seqA, qualA, seqB, qualB []byte, gap, scale float64
 
 }
 
+// Gaps at the beginning and at the end of seqA are free
+// With seqA spanning over lines and seqB over columns
+//
+// SeqA must be the longer sequence. If that constraint is not
+// respected, the function will panic.
+//
+// TO BE FINISHED
+//   - First column gap = 0
+//   - Last column gaps = 0
+//
+// Paths are encoded :
+//   - 0  :  for diagonal
+//   - -1 :  for top
+//   - +1 :  for left
+func _FillMatrixPeCenterAlign(seqA, qualA, seqB, qualB []byte, gap, scale float64,
+	scoreMatrix, pathMatrix *[]int) int {
+
+	la := len(seqA)
+	lb := len(seqB)
+
+	if len(seqA) < len(seqB) {
+		log.Panicf("len(seqA) < len(seqB) : %d < %d", len(seqA), len(seqB))
+	}
+
+	// The actual gap score is the gap score times the mismatch between
+	// two bases with a score of 40
+	gapPenalty := int(scale*gap*float64(_NucScorePartMatchMismatch[40][40]) + 0.5)
+
+	needed := (la + 1) * (lb + 1)
+
+	if needed > cap(*scoreMatrix) {
+		*scoreMatrix = make([]int, needed)
+	}
+
+	if needed > cap(*pathMatrix) {
+		*pathMatrix = make([]int, needed)
+	}
+
+	*scoreMatrix = (*scoreMatrix)[:needed]
+	*pathMatrix = (*pathMatrix)[:needed]
+
+	// Sets the first position of the matrix with 0 score
+	_SetMatrices(scoreMatrix, pathMatrix, la, -1, -1, 0, 0)
+
+	// Fills the first column with score 0
+	for i := 0; i < la; i++ {
+		_SetMatrices(scoreMatrix, pathMatrix, la, i, -1, 0, -1)
+	}
+
+	// la1 := la - 1 // Except the last line (gaps are free on it)
+	lb1 := lb - 1 // Except the last column (gaps are free on it)
+
+	for j := 0; j < lb1; j++ {
+
+		// Fill the first line with scores corresponding to a set of gaps
+		_SetMatrices(scoreMatrix, pathMatrix, la, -1, j, (j+1)*gapPenalty, 1)
+
+		for i := 0; i < la; i++ {
+			left, diag, top := _GetMatrixFrom(scoreMatrix, la, i, j)
+			// log.Infof("LA: i : %d j : %d left : %d diag : %d top : %d\n", i, j, left, diag, top)
+
+			diag += _PairingScorePeAlign(seqA[i], qualA[i], seqB[j], qualB[j], scale)
+			left += gapPenalty
+			top += gapPenalty
+
+			switch {
+			case diag >= left && diag >= top:
+				_SetMatrices(scoreMatrix, pathMatrix, la, i, j, diag, 0)
+			case left >= diag && left >= top:
+				_SetMatrices(scoreMatrix, pathMatrix, la, i, j, left, +1)
+			default:
+				_SetMatrices(scoreMatrix, pathMatrix, la, i, j, top, -1)
+			}
+			// log.Infof("LA: i : %d j : %d left : %d diag : %d top : %d [%d]\n", i, j, left, diag, top, _GetMatrix(scoreMatrix, la, i, j))
+		}
+
+	}
+
+	for i := 0; i < la; i++ {
+		left, diag, top := _GetMatrixFrom(scoreMatrix, la, i, lb1)
+		// log.Infof("LA: i : %d j : %d left : %d diag : %d top : %d\n", i, j, left, diag, top)
+
+		diag += _PairingScorePeAlign(seqA[i], qualA[i], seqB[lb1], qualB[lb1], scale)
+		left += gapPenalty
+
+		switch {
+		case diag >= left && diag >= top:
+			_SetMatrices(scoreMatrix, pathMatrix, la, i, lb1, diag, 0)
+		case left >= diag && left >= top:
+			_SetMatrices(scoreMatrix, pathMatrix, la, i, lb1, left, +1)
+		default:
+			_SetMatrices(scoreMatrix, pathMatrix, la, i, lb1, top, -1)
+		}
+		// log.Infof("LA: i : %d j : %d left : %d diag : %d top : %d [%d]\n", i, j, left, diag, top, _GetMatrix(scoreMatrix, la, i, j))
+	}
+
+	return _GetMatrix(scoreMatrix, la, la-1, lb1)
+}
+
 func PELeftAlign(seqA, seqB *obiseq.BioSequence, gap, scale float64,
 	arena PEAlignArena) (int, []int) {
 
@@ -359,9 +460,33 @@ func PERightAlign(seqA, seqB *obiseq.BioSequence, gap, scale float64,
 	return score, path
 }
 
+func PECenterAlign(seqA, seqB *obiseq.BioSequence, gap, scale float64,
+	arena PEAlignArena) (int, []int) {
+
+	if !_InitializedDnaScore {
+		_InitDNAScoreMatrix()
+	}
+
+	if arena.pointer == nil {
+		arena = MakePEAlignArena(seqA.Len(), seqB.Len())
+	}
+
+	score := _FillMatrixPeCenterAlign(seqA.Sequence(), seqA.Qualities(),
+		seqB.Sequence(), seqB.Qualities(), gap, scale,
+		&arena.pointer.scoreMatrix,
+		&arena.pointer.pathMatrix)
+
+	path := _Backtracking(arena.pointer.pathMatrix,
+		seqA.Len(), seqB.Len(),
+		&arena.pointer.path)
+
+	return score, path
+}
+
 func PEAlign(seqA, seqB *obiseq.BioSequence,
 	gap, scale float64, fastAlign bool, delta int, fastScoreRel bool,
-	arena PEAlignArena, shift_buff *map[int]int) (int, []int, int, int, float64) {
+	arena PEAlignArena, shift_buff *map[int]int) (bool, int, []int, int, int, float64) {
+	var isLeftAlign bool
 	var score, shift int
 	var startA, startB int
 	var partLen, over int
@@ -412,6 +537,7 @@ func PEAlign(seqA, seqB *obiseq.BioSequence,
 				rawSeqB = seqB.Sequence()[0:partLen]
 				qualSeqB = seqB.Qualities()[0:partLen]
 				extra3 = seqB.Len() - partLen
+				isLeftAlign = true
 				score = _FillMatrixPeLeftAlign(
 					rawSeqA, qualSeqA, rawSeqB, qualSeqB, gap, scale,
 					&arena.pointer.scoreMatrix,
@@ -433,7 +559,7 @@ func PEAlign(seqA, seqB *obiseq.BioSequence,
 				rawSeqA = seqA.Sequence()[:partLen]
 				qualSeqA = seqA.Qualities()[:partLen]
 				extra3 = partLen - seqA.Len()
-
+				isLeftAlign = false
 				score = _FillMatrixPeRightAlign(
 					rawSeqA, qualSeqA, rawSeqB, qualSeqB, gap, scale,
 					&arena.pointer.scoreMatrix,
@@ -457,6 +583,7 @@ func PEAlign(seqA, seqB *obiseq.BioSequence,
 				qualSeqB = seqB.Qualities()[0:partLen]
 				extra3 = seqB.Len() - partLen
 				score = 0
+				isLeftAlign = true
 			} else {
 				startA = 0
 				startB = -shift
@@ -465,6 +592,7 @@ func PEAlign(seqA, seqB *obiseq.BioSequence,
 				partLen = len(qualSeqB)
 				extra3 = partLen - seqA.Len()
 				qualSeqA = seqA.Qualities()[:partLen]
+				isLeftAlign = false
 			}
 			score = 0
 			for i, qualA := range qualSeqA {
@@ -497,10 +625,14 @@ func PEAlign(seqA, seqB *obiseq.BioSequence,
 			&arena.pointer.scoreMatrix,
 			&arena.pointer.pathMatrix)
 
+		score = scoreR
+
 		path = _Backtracking(arena.pointer.pathMatrix,
 			len(rawSeqA), len(rawSeqB),
 			&(arena.pointer.path))
 
+		isLeftAlign = false
+
 		scoreL := _FillMatrixPeLeftAlign(
 			rawSeqA, qualSeqA, rawSeqB, qualSeqB, gap, scale,
 			&arena.pointer.scoreMatrix,
@@ -510,9 +642,11 @@ func PEAlign(seqA, seqB *obiseq.BioSequence,
 			path = _Backtracking(arena.pointer.pathMatrix,
 				len(rawSeqA), len(rawSeqB),
 				&(arena.pointer.path))
+			isLeftAlign = true
+			score = scoreL
 		}
 
 	}
 
-	return score, path, fastCount, over, fastScore
+	return isLeftAlign, score, path, fastCount, over, fastScore
 }

pkg/obialign/readalign.go

@@ -0,0 +1,154 @@
+package obialign
+
+import (
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obikmer"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+func ReadAlign(seqA, seqB *obiseq.BioSequence,
+	gap, scale float64, delta int, fastScoreRel bool,
+	arena PEAlignArena, shift_buff *map[int]int) (int, []int, int, int, float64, bool) {
+	var score, shift int
+	var startA, startB int
+	var partLen, over int
+	var rawSeqA, qualSeqA []byte
+	var rawSeqB, qualSeqB []byte
+	var extra5, extra3 int
+
+	var path []int
+
+	if !_InitializedDnaScore {
+		_InitDNAScoreMatrix()
+	}
+
+	fastCount := -1
+	fastScore := -1.0
+
+	directAlignment := true
+
+	index := obikmer.Index4mer(seqA,
+		&arena.pointer.fastIndex,
+		&arena.pointer.fastBuffer)
+
+	shift, fastCount, fastScore = obikmer.FastShiftFourMer(index, shift_buff, seqA.Len(), seqB, fastScoreRel, nil)
+
+	seqBR := seqB.ReverseComplement(false)
+	shiftR, fastCountR, fastScoreR := obikmer.FastShiftFourMer(index, shift_buff, seqA.Len(), seqBR, fastScoreRel, nil)
+
+	if fastCount < fastCountR {
+		shift = shiftR
+		fastCount = fastCountR
+		fastScore = fastScoreR
+		seqB = seqBR
+		directAlignment = false
+	}
+
+	// Compute the overlapping region length
+	switch {
+	case shift > 0:
+		over = seqA.Len() - shift
+	case shift < 0:
+		over = seqB.Len() + shift
+	default:
+		over = min(seqA.Len(), seqB.Len())
+	}
+
+	// log.Warnf("fw/fw: %v shift=%d fastCount=%d/over=%d fastScore=%f",
+	// 	directAlignment, shift, fastCount, over, fastScore)
+
+	// log.Warnf(("seqA: %s\nseqB: %s\n"), seqA.String(), seqB.String())
+
+	// At least one mismatch exists in the overlaping region
+	if fastCount+3 < over {
+
+		if shift > 0 || (shift == 0 && seqB.Len() >= seqA.Len()) {
+			startA = shift - delta
+			if startA < 0 {
+				startA = 0
+			}
+			extra5 = -startA
+			startB = 0
+
+			rawSeqA = seqA.Sequence()[startA:]
+			qualSeqA = seqA.Qualities()[startA:]
+			partLen = len(rawSeqA)
+			if partLen > seqB.Len() {
+				partLen = seqB.Len()
+			}
+			rawSeqB = seqB.Sequence()[0:partLen]
+			qualSeqB = seqB.Qualities()[0:partLen]
+			extra3 = seqB.Len() - partLen
+			score = _FillMatrixPeLeftAlign(
+				rawSeqA, qualSeqA, rawSeqB, qualSeqB, gap, scale,
+				&arena.pointer.scoreMatrix,
+				&arena.pointer.pathMatrix)
+		} else {
+
+			startA = 0
+			startB = -shift - delta
+			if startB < 0 {
+				startB = 0
+			}
+			extra5 = startB
+			rawSeqB = seqB.Sequence()[startB:]
+			qualSeqB = seqB.Qualities()[startB:]
+			partLen = len(rawSeqB)
+			if partLen > seqA.Len() {
+				partLen = seqA.Len()
+			}
+			rawSeqA = seqA.Sequence()[:partLen]
+			qualSeqA = seqA.Qualities()[:partLen]
+			extra3 = partLen - seqA.Len()
+
+			score = _FillMatrixPeRightAlign(
+				rawSeqA, qualSeqA, rawSeqB, qualSeqB, gap, scale,
+				&arena.pointer.scoreMatrix,
+				&arena.pointer.pathMatrix)
+		}
+
+		path = _Backtracking(arena.pointer.pathMatrix,
+			len(rawSeqA), len(rawSeqB),
+			&arena.pointer.path)
+
+	} else {
+
+		// Both overlaping regions are identicals
+
+		if shift > 0 || (shift == 0 && seqB.Len() >= seqA.Len()) {
+			startA = shift
+			startB = 0
+			extra5 = -startA
+			qualSeqA = seqA.Qualities()[startA:]
+			partLen = len(qualSeqA)
+			qualSeqB = seqB.Qualities()[0:partLen]
+			extra3 = seqB.Len() - partLen
+			score = 0
+		} else {
+			startA = 0
+			startB = -shift
+			extra5 = startB
+			qualSeqB = seqB.Qualities()[startB:]
+			partLen = len(qualSeqB)
+			extra3 = partLen - seqA.Len()
+			qualSeqA = seqA.Qualities()[:partLen]
+		}
+
+		score = 0
+		for i, qualA := range qualSeqA {
+			qualB := qualSeqB[i]
+			score += _NucScorePartMatchMatch[qualA][qualB]
+		}
+
+		path = arena.pointer.path[:0]
+		path = append(path, 0, partLen)
+	}
+
+	path[0] += extra5
+	if path[len(path)-1] == 0 {
+		path[len(path)-2] += extra3
+	} else {
+		path = append(path, extra3, 0)
+	}
+
+	return score, path, fastCount, over, fastScore, directAlignment
+}

pkg/obiapat/obiapat.c

@@ -137,6 +137,28 @@ char *reverseSequence(char *str,char isPattern)
         return str;
 }
 
+/* -------------------------------------------- */
+/* lowercase sequence                           */
+/* -------------------------------------------- */
+
+#define IS_UPPER(c) (((c) >= 'A') && ((c) <= 'A'))
+#define TO_LOWER(c) ((c) - 'A' + 'a')
+
+char *LowerSequence(char *seq)
+{
+        char *cseq;
+
+        for (cseq = seq ; *cseq ; cseq++) 
+            if (IS_UPPER(*cseq))
+                *cseq = TO_LOWER(*cseq);
+
+		return seq;
+}
+ 
+#undef IS_UPPER
+#undef TO_LOWER
+
+
 char *ecoComplementPattern(char *nucAcSeq)
 {
     return reverseSequence(LXBioSeqComplement(nucAcSeq),1);
@@ -165,6 +187,7 @@ void UpperSequence(char *seq)
 
 
 
+
 /* -------------------------------------------- */
 /* encode sequence                              */
 /* IS_UPPER is slightly faster than isupper     */

pkg/obiapat/pattern.go

@@ -9,6 +9,7 @@ import "C"
 import (
 	"errors"
 	"runtime"
+	"strings"
 	"unsafe"
 
 	log "github.com/sirupsen/logrus"
@@ -114,7 +115,7 @@ func (pattern ApatPattern) ReverseComplement() (ApatPattern, error) {
 		C.free(unsafe.Pointer(errmsg))
 		return ApatPattern{nil}, errors.New(message)
 	}
-	spat := C.GoString(apc.cpat)
+	spat := strings.ToLower(C.GoString(apc.cpat))
 	ap := _ApatPattern{apc, spat}
 
 	runtime.SetFinalizer(&ap, func(p *_ApatPattern) {
@@ -296,6 +297,24 @@ func (pattern ApatPattern) FindAllIndex(sequence ApatSequence, begin, length int
 	return loc
 }
 
+func (pattern ApatPattern) IsMatching(sequence ApatSequence, begin, length int) bool {
+	if begin < 0 {
+		begin = 0
+	}
+
+	if length < 0 {
+		length = sequence.Len()
+	}
+
+	nhits := int(C.ManberAll(sequence.pointer.pointer,
+		pattern.pointer.pointer,
+		0,
+		C.int32_t(begin),
+		C.int32_t(length+C.MAX_PAT_LEN)))
+
+	return nhits > 0
+}
+
 // BestMatch finds the best match of a given pattern in a sequence.
 //
 // THe function identify the first occurrence of the pattern in the sequence.
@@ -335,6 +354,11 @@ func (pattern ApatPattern) BestMatch(sequence ApatSequence, begin, length int) (
 	nerr = best[2]
 	end = best[1]
 
+	if best[0] < 0 || best[1] > sequence.Len() {
+		matched = false
+		return
+	}
+
 	if nerr == 0 || !pattern.pointer.pointer.hasIndel {
 		start = best[0]
 		log.Debugln("No nws ", start, nerr)
@@ -355,17 +379,31 @@ func (pattern ApatPattern) BestMatch(sequence ApatSequence, begin, length int) (
 		best[0], nerr, int(pattern.pointer.pointer.patlen),
 		sequence.Len(), start, end)
 
-	from, to, score := obialign.LocatePattern((*cpattern)[0:int(pattern.pointer.pointer.patlen)], frg)
+	from, to, score := obialign.LocatePattern(sequence.pointer.reference.Id(),
+		(*cpattern)[0:int(pattern.pointer.pointer.patlen)],
+		frg)
 
 	// olderr := m[2]
 
 	nerr = score
 	start = start + from
 	end = start + to
-	log.Debugln("results", score, start, nerr)
+	log.Debugf("BestMatch on %s : score=%d [%d..%d]", sequence.pointer.reference.Id(), score, start, nerr)
 	return
 }
 
+// FilterBestMatch filters the best non overlapping matches of a given pattern in a sequence.
+//
+// It takes the following parameters:
+// - pattern: the pattern to search for (ApatPattern).
+// - sequence: the sequence to search in (ApatSequence).
+// - begin: the starting index of the search (int).
+// - length: the length of the search (int).
+//
+// It returns a slice of [3]int representing the locations of all non-overlapping matches in the sequence.
+// The two firsts values of the [3]int indicate respectively the start and the end position of
+// the match. Following the GO convention the end position is not included in the
+// match. The third value indicates the number of error detected for this occurrence.
 func (pattern ApatPattern) FilterBestMatch(sequence ApatSequence, begin, length int) (loc [][3]int) {
 	res := pattern.FindAllIndex(sequence, begin, length)
 	filtered := make([][3]int, 0, len(res))
@@ -424,13 +462,15 @@ func (pattern ApatPattern) FilterBestMatch(sequence ApatSequence, begin, length
 func (pattern ApatPattern) AllMatches(sequence ApatSequence, begin, length int) (loc [][3]int) {
 	res := pattern.FilterBestMatch(sequence, begin, length)
 
+	j := 0
 	for _, m := range res {
 		// Recompute the start and end position of the match
 		// when the pattern allows for indels
 		if m[2] > 0 && pattern.pointer.pointer.hasIndel {
-			start := m[0] - m[2]
+			// log.Warnf("Locating indel on sequence %s[%s]", sequence.pointer.reference.Id(), pattern.String())
+			start := m[0] - m[2]*2
 			start = max(start, 0)
-			end := start + int(pattern.pointer.pointer.patlen) + 2*m[2]
+			end := start + int(pattern.pointer.pointer.patlen) + 4*m[2]
 			end = min(end, sequence.Len())
 			// 1 << 30 = 1,073,741,824 = 1Gb
 			// It's a virtual array mapping the sequence to the pattern
@@ -439,18 +479,24 @@ func (pattern ApatPattern) AllMatches(sequence ApatSequence, begin, length int)
 			cpattern := (*[1 << 30]byte)(unsafe.Pointer(pattern.pointer.pointer.cpat))
 			frg := sequence.pointer.reference.Sequence()[start:end]
 
-			begin, end, score := obialign.LocatePattern((*cpattern)[0:int(pattern.pointer.pointer.patlen)], frg)
+			pb, pe, score := obialign.LocatePattern(
+				sequence.pointer.reference.Id(),
+				(*cpattern)[0:int(pattern.pointer.pointer.patlen)],
+				frg)
 
 			// olderr := m[2]
 			m[2] = score
-			m[0] = start + begin
+			m[0] = start + pb
-			m[1] = start + end
+			m[1] = start + pe
+
 			// log.Warnf("seq[%d@%d:%d] %d: %s %d - %s:%s:%s", i, m[0], m[1], olderr, sequence.pointer.reference.Id(), score,
 			// 	frg, (*cpattern)[0:int(pattern.pointer.pointer.patlen)], sequence.pointer.reference.Sequence()[m[0]:m[1]])
 		}
+
+		if int(pattern.pointer.pointer.maxerr) >= m[2] {
+			res[j] = m
+			j++
+		}
 	}
-
+	return res[0:j]
-	// log.Debugf("All matches : %v", res)
-
-	return res
 }

pkg/obiapat/pcr.go

@@ -3,7 +3,7 @@ package obiapat
 import (
 	log "github.com/sirupsen/logrus"
 
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
@@ -104,7 +104,7 @@ func MakeOptions(setters []WithOption) Options {
 		extension:       -1,
 		fullExtension:   false,
 		circular:        false,
-		parallelWorkers: obioptions.CLIParallelWorkers(),
+		parallelWorkers: obidefault.ParallelWorkers(),
 		batchSize:       100,
 		forward:         NilApatPattern,
 		cfwd:            NilApatPattern,
@@ -529,7 +529,6 @@ func PCRSliceWorker(options ...WithOption) obiseq.SeqSliceWorker {
 	opt := MakeOptions(options)
 	worker := func(sequences obiseq.BioSequenceSlice) (obiseq.BioSequenceSlice, error) {
 		result := _PCRSlice(sequences, opt)
-		sequences.Recycle(true)
 		return result, nil
 	}
 

pkg/obiapat/predicat.go

@@ -0,0 +1,40 @@
+package obiapat
+
+import (
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+	log "github.com/sirupsen/logrus"
+)
+
+func IsPatternMatchSequence(pattern string, errormax int, bothStrand, allowIndels bool) obiseq.SequencePredicate {
+
+	pat, err := MakeApatPattern(pattern, errormax, allowIndels)
+
+	if err != nil {
+		log.Fatalf("error in sequence regular pattern syntax : %v", err)
+	}
+
+	cpat, err := pat.ReverseComplement()
+
+	if err != nil {
+		log.Fatalf("cannot reverse complement the pattern : %v", err)
+	}
+
+	f := func(sequence *obiseq.BioSequence) bool {
+		aseq, err := MakeApatSequence(sequence, false)
+
+		if err != nil {
+			log.Panicf("Cannot convert sequence %s to apat format", sequence.Id())
+		}
+
+		match := pat.IsMatching(aseq, 0, aseq.Len())
+
+		if !match && bothStrand {
+
+			match = cpat.IsMatching(aseq, 0, aseq.Len())
+		}
+
+		return match
+	}
+
+	return f
+}

pkg/obichunk/chunk_on_disk.go

@@ -12,6 +12,13 @@ import (
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 )
 
+// tempDir creates a temporary directory with a prefix "obiseq_chunks_"
+// in the system's temporary directory. It returns the path of the
+// created directory and any error encountered during the creation process.
+//
+// If the directory creation is successful, the path to the new
+// temporary directory is returned. If there is an error, it returns
+// an empty string and the error encountered.
 func tempDir() (string, error) {
 	dir, err := os.MkdirTemp(os.TempDir(), "obiseq_chunks_")
 	if err != nil {
@@ -20,6 +27,19 @@ func tempDir() (string, error) {
 	return dir, nil
 }
 
+// find searches for files with a specific extension in the given root directory
+// and its subdirectories. It returns a slice of strings containing the paths
+// of the found files.
+//
+// Parameters:
+// - root: The root directory to start the search from.
+// - ext: The file extension to look for (including the leading dot, e.g., ".txt").
+//
+// Returns:
+// A slice of strings containing the paths of files that match the specified
+// extension. If no files are found, an empty slice is returned. Any errors
+// encountered during the directory traversal will be returned as part of the
+// WalkDir function's error handling.
 func find(root, ext string) []string {
 	var a []string
 	filepath.WalkDir(root, func(s string, d fs.DirEntry, e error) error {
@@ -34,6 +54,24 @@ func find(root, ext string) []string {
 	return a
 }
 
+// ISequenceChunkOnDisk processes a sequence iterator by distributing the sequences
+// into chunks stored on disk. It uses a classifier to determine how to distribute
+// the sequences and returns a new iterator for the processed sequences.
+//
+// Parameters:
+//   - iterator: An iterator of biosequences to be processed.
+//   - classifier: A pointer to a BioSequenceClassifier used to classify the sequences
+//     during distribution.
+//
+// Returns:
+// An iterator of biosequences representing the processed chunks. If an error occurs
+// during the creation of the temporary directory or any other operation, it returns
+// an error along with a nil iterator.
+//
+// The function operates asynchronously, creating a temporary directory to store
+// the sequence chunks. Once the processing is complete, the temporary directory
+// is removed. The function logs the number of batches created and the processing
+// status of each batch.
 func ISequenceChunkOnDisk(iterator obiiter.IBioSequence,
 	classifier *obiseq.BioSequenceClassifier) (obiiter.IBioSequence, error) {
 	dir, err := tempDir()
@@ -73,11 +111,11 @@ func ISequenceChunkOnDisk(iterator obiiter.IBioSequence,
 				panic(err)
 			}
 
-			chunck := iseq.Load()
+			source, chunk := iseq.Load()
 
-			newIter.Push(obiiter.MakeBioSequenceBatch(order, chunck))
+			newIter.Push(obiiter.MakeBioSequenceBatch(source, order, chunk))
 			log.Infof("Start processing of batch %d/%d : %d sequences",
-				order, nbatch, len(chunck))
+				order, nbatch, len(chunk))
 
 		}
 

pkg/obichunk/chunks.go

@@ -28,6 +28,7 @@ func ISequenceChunk(iterator obiiter.IBioSequence,
 
 		jobDone := sync.WaitGroup{}
 		chunks := make(map[int]*obiseq.BioSequenceSlice, 1000)
+		sources := make(map[int]string, 1000)
 
 		for newflux := range dispatcher.News() {
 			jobDone.Add(1)
@@ -43,12 +44,17 @@ func ISequenceChunk(iterator obiiter.IBioSequence,
 				chunks[newflux] = chunk
 				lock.Unlock()
 
+				source := ""
 				for data.Next() {
 					b := data.Get()
+					source = b.Source()
 					*chunk = append(*chunk, b.Slice()...)
-					b.Recycle(false)
 				}
 
+				lock.Lock()
+				sources[newflux] = source
+				lock.Unlock()
+
 				jobDone.Done()
 			}(newflux)
 		}
@@ -56,10 +62,10 @@ func ISequenceChunk(iterator obiiter.IBioSequence,
 		jobDone.Wait()
 		order := 0
 
-		for _, chunck := range chunks {
+		for i, chunk := range chunks {
 
-			if len(*chunck) > 0 {
+			if len(*chunk) > 0 {
-				newIter.Push(obiiter.MakeBioSequenceBatch(order, *chunck))
+				newIter.Push(obiiter.MakeBioSequenceBatch(sources[i], order, *chunk))
 				order++
 			}
 

pkg/obichunk/options.go

@@ -1,7 +1,7 @@
 package obichunk
 
 import (
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 )
 
@@ -29,8 +29,8 @@ func MakeOptions(setters []WithOption) Options {
 		navalue:         "NA",
 		cacheOnDisk:     false,
 		batchCount:      100,
-		batchSize:       obioptions.CLIBatchSize(),
+		batchSize:       obidefault.BatchSize(),
-		parallelWorkers: obioptions.CLIParallelWorkers(),
+		parallelWorkers: obidefault.ParallelWorkers(),
 		noSingleton:     false,
 	}
 

pkg/obichunk/subchunks.go

@@ -6,8 +6,8 @@ import (
 
 	log "github.com/sirupsen/logrus"
 
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 )
 
@@ -62,7 +62,7 @@ func ISequenceSubChunk(iterator obiiter.IBioSequence,
 	nworkers int) (obiiter.IBioSequence, error) {
 
 	if nworkers <= 0 {
-		nworkers = obioptions.CLIParallelWorkers()
+		nworkers = obidefault.ParallelWorkers()
 	}
 
 	newIter := obiiter.MakeIBioSequence()
@@ -90,7 +90,7 @@ func ISequenceSubChunk(iterator obiiter.IBioSequence,
 		for iterator.Next() {
 
 			batch := iterator.Get()
-
+			source := batch.Source()
 			if batch.Len() > 1 {
 				classifier.Reset()
 
@@ -107,8 +107,6 @@ func ISequenceSubChunk(iterator obiiter.IBioSequence,
 					batch.Slice()[i] = nil
 				}
 
-				batch.Recycle(false)
-
 				_By(func(p1, p2 *sSS) bool {
 					return p1.code < p2.code
 				}).Sort(ordered)
@@ -117,7 +115,7 @@ func ISequenceSubChunk(iterator obiiter.IBioSequence,
 				ss := obiseq.MakeBioSequenceSlice()
 				for i, v := range ordered {
 					if v.code != last {
-						newIter.Push(obiiter.MakeBioSequenceBatch(nextOrder(), ss))
+						newIter.Push(obiiter.MakeBioSequenceBatch(source, nextOrder(), ss))
 						ss = obiseq.MakeBioSequenceSlice()
 						last = v.code
 					}
@@ -127,7 +125,7 @@ func ISequenceSubChunk(iterator obiiter.IBioSequence,
 				}
 
 				if len(ss) > 0 {
-					newIter.Push(obiiter.MakeBioSequenceBatch(nextOrder(), ss))
+					newIter.Push(obiiter.MakeBioSequenceBatch(source, nextOrder(), ss))
 				}
 			} else {
 				newIter.Push(batch.Reorder(nextOrder()))

pkg/obichunk/unique.go

@@ -95,10 +95,7 @@ func IUniqueSequence(iterator obiiter.IBioSequence,
 
 			if icat < 0 || len(batch.Slice()) == 1 {
 				// No more sub classification of sequence or only a single sequence
-				if opts.NoSingleton() && len(batch.Slice()) == 1 && batch.Slice()[0].Count() == 1 {
+				if !(opts.NoSingleton() && len(batch.Slice()) == 1 && batch.Slice()[0].Count() == 1) {
-					// We remove singleton from output
-					batch.Recycle(true)
-				} else {
 					iUnique.Push(batch.Reorder(nextOrder()))
 				}
 			} else {

pkg/obidefault/batch.go

@@ -0,0 +1,26 @@
+package obidefault
+
+var _BatchSize = 2000
+
+// SetBatchSize sets the size of the sequence batches.
+//
+// n - an integer representing the size of the sequence batches.
+func SetBatchSize(n int) {
+	_BatchSize = n
+}
+
+// CLIBatchSize returns the expected size of the sequence batches.
+//
+// In Obitools, the sequences are processed in parallel by batches.
+// The number of sequence in each batch is determined by the command line option
+// --batch-size and the environment variable OBIBATCHSIZE.
+//
+// No parameters.
+// Returns an integer value.
+func BatchSize() int {
+	return _BatchSize
+}
+
+func BatchSizePtr() *int {
+	return &_BatchSize
+}

pkg/obidefault/compressed.go

@@ -0,0 +1,15 @@
+package obidefault
+
+var __compressed__ = false
+
+func CompressOutput() bool {
+	return __compressed__
+}
+
+func SetCompressOutput(b bool) {
+	__compressed__ = b
+}
+
+func CompressedPtr() *bool {
+	return &__compressed__
+}

pkg/obidefault/quality.go

@@ -0,0 +1,29 @@
+package obidefault
+
+var _Quality_Shift_Input = byte(33)
+var _Quality_Shift_Output = byte(33)
+var _Read_Qualities = true
+
+func SetReadQualitiesShift(shift byte) {
+	_Quality_Shift_Input = shift
+}
+
+func ReadQualitiesShift() byte {
+	return _Quality_Shift_Input
+}
+
+func SetWriteQualitiesShift(shift byte) {
+	_Quality_Shift_Output = shift
+}
+
+func WriteQualitiesShift() byte {
+	return _Quality_Shift_Output
+}
+
+func SetReadQualities(read bool) {
+	_Read_Qualities = read
+}
+
+func ReadQualities() bool {
+	return _Read_Qualities
+}

pkg/obidefault/taxonomy.go

@@ -0,0 +1,58 @@
+package obidefault
+
+var __taxonomy__ = ""
+var __alternative_name__ = false
+var __fail_on_taxonomy__ = false
+var __update_taxid__ = false
+
+func SelectedTaxonomy() string {
+	return __taxonomy__
+}
+
+func HasSelectedTaxonomy() bool {
+	return __taxonomy__ != ""
+}
+
+func AreAlternativeNamesSelected() bool {
+	return __alternative_name__
+}
+
+func SelectedTaxonomyPtr() *string {
+	return &__taxonomy__
+}
+
+func AlternativeNamesSelectedPtr() *bool {
+	return &__alternative_name__
+}
+
+func SetSelectedTaxonomy(taxonomy string) {
+	__taxonomy__ = taxonomy
+}
+
+func SetAlternativeNamesSelected(alt bool) {
+	__alternative_name__ = alt
+}
+
+func SetFailOnTaxonomy(fail bool) {
+	__fail_on_taxonomy__ = fail
+}
+
+func SetUpdateTaxid(update bool) {
+	__update_taxid__ = update
+}
+
+func FailOnTaxonomyPtr() *bool {
+	return &__fail_on_taxonomy__
+}
+
+func UpdateTaxidPtr() *bool {
+	return &__update_taxid__
+}
+
+func FailOnTaxonomy() bool {
+	return __fail_on_taxonomy__
+}
+
+func UpdateTaxid() bool {
+	return __update_taxid__
+}

pkg/obidefault/workers.go

@@ -0,0 +1,170 @@
+package obidefault
+
+import "runtime"
+
+var _MaxAllowedCPU = runtime.NumCPU()
+var _WorkerPerCore = 1.0
+
+var _ReadWorkerPerCore = 0.25
+var _WriteWorkerPerCore = 0.25
+
+var _StrictReadWorker = 0
+var _StrictWriteWorker = 0
+
+var _ParallelFilesRead = 0
+
+// CLIParallelWorkers returns the number of parallel workers used for
+// computing the result.
+//
+// The number of parallel workers is determined by the command line option
+// --max-cpu|-m and the environment variable OBIMAXCPU. This number is
+// multiplied by the variable _WorkerPerCore.
+//
+// No parameters.
+// Returns an integer representing the number of parallel workers.
+func ParallelWorkers() int {
+	return int(float64(MaxCPU()) * float64(WorkerPerCore()))
+}
+
+// CLIMaxCPU returns the maximum number of CPU cores allowed.
+//
+// The maximum number of CPU cores is determined by the command line option
+// --max-cpu|-m and the environment variable OBIMAXCPU.
+//
+// No parameters.
+// Returns an integer representing the maximum number of CPU cores allowed.
+func MaxCPU() int {
+	return _MaxAllowedCPU
+}
+
+func MaxCPUPtr() *int {
+	return &_MaxAllowedCPU
+}
+
+// WorkerPerCore returns the number of workers per CPU core.
+//
+// No parameters.
+// Returns a float64 representing the number of workers per CPU core.
+func WorkerPerCore() float64 {
+	return _WorkerPerCore
+}
+
+// SetWorkerPerCore sets the number of workers per CPU core.
+//
+// It takes a float64 parameter representing the number of workers
+// per CPU core and does not return any value.
+func SetWorkerPerCore(n float64) {
+	_WorkerPerCore = n
+}
+
+// SetMaxCPU sets the maximum number of CPU cores allowed.
+//
+// n - an integer representing the new maximum number of CPU cores.
+func SetMaxCPU(n int) {
+	_MaxAllowedCPU = n
+}
+
+// SetReadWorker sets the number of workers for reading files.
+//
+// The number of worker dedicated to reading files is determined
+// as the number of allowed CPU cores multiplied by number of read workers per core.
+// Setting the number of read workers using this function allows to decouple the number
+// of read workers from the number of CPU cores.
+//
+// n - an integer representing the number of workers to be set.
+func SetStrictReadWorker(n int) {
+	_StrictReadWorker = n
+}
+
+func SetStrictWriteWorker(n int) {
+	_StrictWriteWorker = n
+}
+
+// SetReadWorkerPerCore sets the number of worker per CPU
+// core for reading files.
+//
+// n float64
+func SetReadWorkerPerCore(n float64) {
+	_ReadWorkerPerCore = n
+}
+
+func SetWriteWorkerPerCore(n float64) {
+	_WriteWorkerPerCore = n
+}
+
+// ReadWorker returns the number of workers for reading files.
+//
+// No parameters.
+// Returns an integer representing the number of workers.
+func StrictReadWorker() int {
+	return _StrictReadWorker
+}
+
+func StrictWriteWorker() int {
+	return _StrictWriteWorker
+}
+
+// CLIReadParallelWorkers returns the number of parallel workers used for
+// reading files.
+//
+// The number of parallel workers is determined by the command line option
+// --max-cpu|-m and the environment variable OBIMAXCPU. This number is
+// multiplied by the variable _ReadWorkerPerCore.
+//
+// No parameters.
+// Returns an integer representing the number of parallel workers.
+func ReadParallelWorkers() int {
+	if StrictReadWorker() == 0 {
+		n := int(float64(MaxCPU()) * ReadWorkerPerCore())
+		if n == 0 {
+			n = 1
+		}
+		return n
+	} else {
+		return StrictReadWorker()
+	}
+}
+
+func WriteParallelWorkers() int {
+	if StrictWriteWorker() == 0 {
+		n := int(float64(MaxCPU()) * WriteWorkerPerCore())
+		if n == 0 {
+			n = 1
+		}
+		return n
+	} else {
+		return StrictReadWorker()
+	}
+}
+
+// ReadWorkerPerCore returns the number of worker per CPU core for
+// computing the result.
+//
+// No parameters.
+// Returns a float64 representing the number of worker per CPU core.
+func ReadWorkerPerCore() float64 {
+	return _ReadWorkerPerCore
+}
+
+func WriteWorkerPerCore() float64 {
+	return _ReadWorkerPerCore
+}
+
+// ParallelFilesRead returns the number of files to be read in parallel.
+//
+// No parameters.
+// Returns an integer representing the number of files to be read.
+func ParallelFilesRead() int {
+	if _ParallelFilesRead == 0 {
+		return ReadParallelWorkers()
+	} else {
+		return _ParallelFilesRead
+	}
+}
+
+// SetParallelFilesRead sets the number of files to be read in parallel.
+//
+// n - an integer representing the number of files to be set.
+func SetParallelFilesRead(n int) {
+	_ParallelFilesRead = n
+}

pkg/obiformats/csv_read.go

@@ -5,10 +5,10 @@ import (
 	"io"
 	"os"
 	"path"
-	"unsafe"
+	"strings"
 
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 	"github.com/goccy/go-json"
@@ -94,31 +94,40 @@ func _ParseCsvFile(source string,
 				continue
 			}
 
-			err := json.Unmarshal(unsafe.Slice(unsafe.StringData(field), len(field)), &val)
+			ft := header[i]
 
-			if err != nil {
+			switch {
-				val = field
+			case ft == "taxid":
-			} else {
+				sequence.SetTaxid(field)
-				if _, ok := val.(float64); ok {
+			case strings.HasSuffix(ft, "_taxid"):
-					if obiutils.IsIntegral(val.(float64)) {
+				sequence.SetTaxid(field, strings.TrimSuffix(ft, "_taxid"))
-						val = int(val.(float64))
+			default:
+				err := json.Unmarshal(obiutils.UnsafeBytes(field), &val)
+
+				if err != nil {
+					val = field
+				} else {
+					if _, ok := val.(float64); ok {
+						if obiutils.IsIntegral(val.(float64)) {
+							val = int(val.(float64))
+						}
 					}
 				}
-			}
 
-			sequence.SetAttribute(header[i], val)
+				sequence.SetAttribute(ft, val)
+			}
 		}
 
 		slice = append(slice, sequence)
 		if len(slice) >= batchSize {
-			out.Push(obiiter.MakeBioSequenceBatch(o, slice))
+			out.Push(obiiter.MakeBioSequenceBatch(source, o, slice))
 			o++
 			slice = obiseq.MakeBioSequenceSlice()
 		}
 	}
 
 	if len(slice) > 0 {
-		out.Push(obiiter.MakeBioSequenceBatch(o, slice))
+		out.Push(obiiter.MakeBioSequenceBatch(source, o, slice))
 	}
 
 	out.Done()
@@ -134,7 +143,7 @@ func ReadCSV(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, err
 	go _ParseCsvFile(opt.Source(),
 		reader,
 		out,
-		byte(obioptions.InputQualityShift()),
+		obidefault.ReadQualitiesShift(),
 		opt.BatchSize())
 
 	go func() {
@@ -148,9 +157,9 @@ func ReadCSV(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, err
 func ReadCSVFromFile(filename string, options ...WithOption) (obiiter.IBioSequence, error) {
 
 	options = append(options, OptionsSource(obiutils.RemoveAllExt((path.Base(filename)))))
-	file, err := Ropen(filename)
+	file, err := obiutils.Ropen(filename)
 
-	if err == ErrNoContent {
+	if err == obiutils.ErrNoContent {
 		log.Infof("file %s is empty", filename)
 		return ReadEmptyFile(options...)
 	}
@@ -164,9 +173,9 @@ func ReadCSVFromFile(filename string, options ...WithOption) (obiiter.IBioSequen
 
 func ReadCSVFromStdin(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, error) {
 	options = append(options, OptionsSource(obiutils.RemoveAllExt("stdin")))
-	input, err := Buf(os.Stdin)
+	input, err := obiutils.Buf(os.Stdin)
 
-	if err == ErrNoContent {
+	if err == obiutils.ErrNoContent {
 		log.Infof("stdin is empty")
 		return ReadEmptyFile(options...)
 	}

pkg/obiformats/csv_writer.go

@@ -1,22 +1,14 @@
 package obiformats
 
 import (
-	"bytes"
-	"encoding/csv"
 	"fmt"
-	"io"
-	"os"
-	"sync"
-	"time"
 
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
-	log "github.com/sirupsen/logrus"
 )
 
-func CSVRecord(sequence *obiseq.BioSequence, opt Options) []string {
+func CSVSequenceRecord(sequence *obiseq.BioSequence, opt Options) []string {
 	keys := opt.CSVKeys()
 	record := make([]string, 0, len(keys)+4)
 
@@ -30,14 +22,10 @@ func CSVRecord(sequence *obiseq.BioSequence, opt Options) []string {
 
 	if opt.CSVTaxon() {
 		taxid := sequence.Taxid()
-		sn, ok := sequence.GetAttribute("scientific_name")
+		sn, ok := sequence.GetStringAttribute("scientific_name")
 
 		if !ok {
-			if taxid == 1 {
+			sn = opt.CSVNAValue()
-				sn = "root"
-			} else {
-				sn = opt.CSVNAValue()
-			}
 		}
 
 		record = append(record, fmt.Sprint(taxid), fmt.Sprint(sn))
@@ -66,7 +54,7 @@ func CSVRecord(sequence *obiseq.BioSequence, opt Options) []string {
 			l := sequence.Len()
 			q := sequence.Qualities()
 			ascii := make([]byte, l)
-			quality_shift := obioptions.OutputQualityShift()
+			quality_shift := obidefault.WriteQualitiesShift()
 			for j := 0; j < l; j++ {
 				ascii[j] = uint8(q[j]) + uint8(quality_shift)
 			}
@@ -78,182 +66,3 @@ func CSVRecord(sequence *obiseq.BioSequence, opt Options) []string {
 
 	return record
 }
-
-func CSVHeader(opt Options) []string {
-	keys := opt.CSVKeys()
-	record := make([]string, 0, len(keys)+4)
-
-	if opt.CSVId() {
-		record = append(record, "id")
-	}
-
-	if opt.CSVCount() {
-		record = append(record, "count")
-	}
-
-	if opt.CSVTaxon() {
-		record = append(record, "taxid", "scientific_name")
-	}
-
-	if opt.CSVDefinition() {
-		record = append(record, "definition")
-	}
-
-	record = append(record, opt.CSVKeys()...)
-
-	if opt.CSVSequence() {
-		record = append(record, "sequence")
-	}
-
-	if opt.CSVQuality() {
-		record = append(record, "quality")
-	}
-
-	return record
-}
-
-func FormatCVSBatch(batch obiiter.BioSequenceBatch, opt Options) []byte {
-	buff := new(bytes.Buffer)
-	csv := csv.NewWriter(buff)
-
-	if batch.Order() == 0 {
-		csv.Write(CSVHeader(opt))
-	}
-	for _, s := range batch.Slice() {
-		csv.Write(CSVRecord(s, opt))
-	}
-
-	csv.Flush()
-
-	return buff.Bytes()
-}
-
-func WriteCSV(iterator obiiter.IBioSequence,
-	file io.WriteCloser,
-	options ...WithOption) (obiiter.IBioSequence, error) {
-
-	var auto_slot obiutils.Set[string]
-	opt := MakeOptions(options)
-
-	file, _ = obiutils.CompressStream(file, opt.CompressedFile(), opt.CloseFile())
-
-	newIter := obiiter.MakeIBioSequence()
-
-	nwriters := opt.ParallelWorkers()
-
-	obiiter.RegisterAPipe()
-	chunkchan := make(chan FileChunck)
-
-	newIter.Add(nwriters)
-	var waitWriter sync.WaitGroup
-
-	go func() {
-		newIter.WaitAndClose()
-		for len(chunkchan) > 0 {
-			time.Sleep(time.Millisecond)
-		}
-		close(chunkchan)
-		waitWriter.Wait()
-	}()
-
-	ff := func(iterator obiiter.IBioSequence) {
-		for iterator.Next() {
-
-			batch := iterator.Get()
-
-			chunkchan <- FileChunck{
-				FormatCVSBatch(batch, opt),
-				batch.Order(),
-			}
-			newIter.Push(batch)
-		}
-		newIter.Done()
-	}
-
-	next_to_send := 0
-	received := make(map[int]FileChunck, 100)
-
-	waitWriter.Add(1)
-	go func() {
-		for chunk := range chunkchan {
-			if chunk.order == next_to_send {
-				file.Write(chunk.text)
-				next_to_send++
-				chunk, ok := received[next_to_send]
-				for ok {
-					file.Write(chunk.text)
-					delete(received, next_to_send)
-					next_to_send++
-					chunk, ok = received[next_to_send]
-				}
-			} else {
-				received[chunk.order] = chunk
-			}
-
-		}
-
-		file.Close()
-
-		log.Debugln("End of the CSV file writing")
-		obiiter.UnregisterPipe()
-		waitWriter.Done()
-	}()
-
-	if opt.pointer.csv_auto {
-		if iterator.Next() {
-			batch := iterator.Get()
-			auto_slot = batch.Slice().AttributeKeys(true)
-			CSVKeys(auto_slot.Members())(opt)
-			iterator.PushBack()
-		}
-	}
-
-	log.Debugln("Start of the CSV file writing")
-	go ff(iterator)
-	for i := 0; i < nwriters-1; i++ {
-		go ff(iterator.Split())
-	}
-
-	return newIter, nil
-}
-
-func WriteCSVToStdout(iterator obiiter.IBioSequence,
-	options ...WithOption) (obiiter.IBioSequence, error) {
-	options = append(options, OptionDontCloseFile())
-	return WriteCSV(iterator, os.Stdout, options...)
-}
-
-func WriteCSVToFile(iterator obiiter.IBioSequence,
-	filename string,
-	options ...WithOption) (obiiter.IBioSequence, error) {
-
-	opt := MakeOptions(options)
-	flags := os.O_WRONLY | os.O_CREATE
-
-	if opt.AppendFile() {
-		flags |= os.O_APPEND
-	}
-	file, err := os.OpenFile(filename, flags, 0660)
-
-	if err != nil {
-		log.Fatalf("open file error: %v", err)
-		return obiiter.NilIBioSequence, err
-	}
-
-	options = append(options, OptionCloseFile())
-
-	iterator, err = WriteCSV(iterator, file, options...)
-
-	if opt.HaveToSavePaired() {
-		var revfile *os.File
-
-		revfile, err = os.OpenFile(opt.PairedFileName(), flags, 0660)
-		if err != nil {
-			log.Fatalf("open file error: %v", err)
-			return obiiter.NilIBioSequence, err
-		}
-		iterator, err = WriteCSV(iterator.PairedWith(), revfile, options...)
-	}
-
-	return iterator, err
-}

pkg/obiformats/dispatcher.go

@@ -14,10 +14,40 @@ import (
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 )
 
+// SequenceBatchWriterToFile is a function type that defines a method for writing
+// a batch of biosequences to a specified file. It takes an iterator of biosequences,
+// a filename, and optional configuration options, and returns an iterator of biosequences
+// along with any error encountered during the writing process.
+//
+// Parameters:
+// - iterator: An iterator of biosequences to be written to the file.
+// - filename: The name of the file where the sequences will be written.
+// - options: Optional configuration options for the writing process.
+//
+// Returns:
+// An iterator of biosequences that may have been modified during the writing process
+// and an error if the writing operation fails.
 type SequenceBatchWriterToFile func(iterator obiiter.IBioSequence,
 	filename string,
 	options ...WithOption) (obiiter.IBioSequence, error)
 
+// WriterDispatcher manages the writing of data to files based on a given
+// prototype name and a dispatcher for distributing the sequences. It
+// processes incoming data from the dispatcher in separate goroutines,
+// formatting and writing the data to files as specified.
+//
+// Parameters:
+//   - prototypename: A string that serves as a template for naming the output files.
+//   - dispatcher: An instance of IDistribute that provides the data to be written
+//     and manages the distribution of sequences.
+//   - formater: A function of type SequenceBatchWriterToFile that formats and writes
+//     the sequences to the specified file.
+//   - options: Optional configuration options for the writing process.
+//
+// The function operates asynchronously, launching goroutines for each new data
+// channel received from the dispatcher. It ensures that directories are created
+// as needed and handles errors during the writing process. The function blocks
+// until all writing jobs are completed.
 func WriterDispatcher(prototypename string,
 	dispatcher obiiter.IDistribute,
 	formater SequenceBatchWriterToFile,
@@ -34,7 +64,7 @@ func WriterDispatcher(prototypename string,
 				data, err := dispatcher.Outputs(newflux)
 
 				if err != nil {
-					log.Fatalf("Cannot retreive the new chanel : %v", err)
+					log.Fatalf("Cannot retrieve the new channel: %v", err)
 				}
 
 				key := dispatcher.Classifier().Value(newflux)
@@ -58,7 +88,7 @@ func WriterDispatcher(prototypename string,
 					info, err := os.Stat(directory)
 					switch {
 					case !os.IsNotExist(err) && !info.IsDir():
-						log.Fatalf("Cannot Create the directory %s", directory)
+						log.Fatalf("Cannot create the directory %s", directory)
 					case os.IsNotExist(err):
 						os.Mkdir(directory, 0755)
 					}
@@ -71,7 +101,7 @@ func WriterDispatcher(prototypename string,
 					options...)
 
 				if err != nil {
-					log.Fatalf("cannot open the output file for key %s",
+					log.Fatalf("Cannot open the output file for key %s",
 						dispatcher.Classifier().Value(newflux))
 				}
 

pkg/obiformats/ecopcr_read.go

@@ -122,7 +122,7 @@ func __read_ecopcr_bioseq__(file *__ecopcr_file__) (*obiseq.BioSequence, error)
 	return bseq, nil
 }
 
-func ReadEcoPCR(reader io.Reader, options ...WithOption) obiiter.IBioSequence {
+func ReadEcoPCR(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, error) {
 	tag := make([]byte, 11)
 	n, _ := reader.Read(tag)
 
@@ -187,7 +187,7 @@ func ReadEcoPCR(reader io.Reader, options ...WithOption) obiiter.IBioSequence {
 			slice = append(slice, seq)
 			ii++
 			if ii >= opt.BatchSize() {
-				newIter.Push(obiiter.MakeBioSequenceBatch(i, slice))
+				newIter.Push(obiiter.MakeBioSequenceBatch(opt.Source(), i, slice))
 				slice = obiseq.MakeBioSequenceSlice()
 				i++
 				ii = 0
@@ -198,7 +198,7 @@ func ReadEcoPCR(reader io.Reader, options ...WithOption) obiiter.IBioSequence {
 		}
 
 		if len(slice) > 0 {
-			newIter.Push(obiiter.MakeBioSequenceBatch(i, slice))
+			newIter.Push(obiiter.MakeBioSequenceBatch(opt.Source(), i, slice))
 		}
 
 		newIter.Done()
@@ -213,7 +213,7 @@ func ReadEcoPCR(reader io.Reader, options ...WithOption) obiiter.IBioSequence {
 		newIter = newIter.CompleteFileIterator()
 	}
 
-	return newIter
+	return newIter, nil
 }
 
 func ReadEcoPCRFromFile(filename string, options ...WithOption) (obiiter.IBioSequence, error) {
@@ -235,5 +235,5 @@ func ReadEcoPCRFromFile(filename string, options ...WithOption) (obiiter.IBioSeq
 		reader = greader
 	}
 
-	return ReadEcoPCR(reader, options...), nil
+	return ReadEcoPCR(reader, options...)
 }

pkg/obiformats/embl_read.go

@@ -15,7 +15,7 @@ import (
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
 
-// _EndOfLastEntry finds the index of the last entry in the given byte slice 'buff'
+// EndOfLastFlatFileEntry finds the index of the last entry in the given byte slice 'buff'
 // using a pattern match of the form:
 // <CR>?<LF>//<CR>?<LF>
 // where <CR> and <LF> are the ASCII codes for carriage return and line feed,
@@ -27,7 +27,7 @@ import (
 //
 // Returns:
 // int - the index of the end of the last entry or -1 if no match is found.
-func _EndOfLastEntry(buff []byte) int {
+func EndOfLastFlatFileEntry(buff []byte) int {
 	//  6    5  43 2    1
 	// <CR>?<LF>//<CR>?<LF>
 	var i int
@@ -87,15 +87,9 @@ func _EndOfLastEntry(buff []byte) int {
 	return -1
 }
 
-func _ParseEmblFile(source string, input ChannelSeqFileChunk,
+func EmblChunkParser(withFeatureTable bool) func(string, io.Reader) (obiseq.BioSequenceSlice, error) {
-	out obiiter.IBioSequence,
+	parser := func(source string, input io.Reader) (obiseq.BioSequenceSlice, error) {
-	withFeatureTable bool,
+		scanner := bufio.NewScanner(input)
-	batch_size int,
-	total_seq_size int) {
-
-	for chunks := range input {
-		scanner := bufio.NewScanner(chunks.raw)
-		order := chunks.order
 		sequences := make(obiseq.BioSequenceSlice, 0, 100)
 		id := ""
 		scientificName := ""
@@ -156,7 +150,31 @@ func _ParseEmblFile(source string, input ChannelSeqFileChunk,
 				seqBytes = new(bytes.Buffer)
 			}
 		}
-		out.Push(obiiter.MakeBioSequenceBatch(order, sequences))
+
+		return sequences, nil
+
+	}
+
+	return parser
+}
+
+func _ParseEmblFile(
+	input ChannelFileChunk,
+	out obiiter.IBioSequence,
+	withFeatureTable bool,
+) {
+
+	parser := EmblChunkParser(withFeatureTable)
+
+	for chunks := range input {
+		order := chunks.Order
+		sequences, err := parser(chunks.Source, chunks.Raw)
+
+		if err != nil {
+			log.Fatalf("%s : Cannot parse the embl file : %v", chunks.Source, err)
+		}
+
+		out.Push(obiiter.MakeBioSequenceBatch(chunks.Source, order, sequences))
 	}
 
 	out.Done()
@@ -166,12 +184,18 @@ func _ParseEmblFile(source string, input ChannelSeqFileChunk,
 //	6    5  43 2    1
 //
 // <CR>?<LF>//<CR>?<LF>
-func ReadEMBL(reader io.Reader, options ...WithOption) obiiter.IBioSequence {
+func ReadEMBL(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, error) {
 	opt := MakeOptions(options)
 
-	buff := make([]byte, 1024*1024*1024*256)
+	buff := make([]byte, 1024*1024*128) // 128 MB
+
+	entry_channel := ReadFileChunk(
+		opt.Source(),
+		reader,
+		buff,
+		EndOfLastFlatFileEntry,
+	)
 
-	entry_channel := ReadSeqFileChunk(reader, buff, _EndOfLastEntry)
 	newIter := obiiter.MakeIBioSequence()
 
 	nworkers := opt.ParallelWorkers()
@@ -179,10 +203,11 @@ func ReadEMBL(reader io.Reader, options ...WithOption) obiiter.IBioSequence {
 	// for j := 0; j < opt.ParallelWorkers(); j++ {
 	for j := 0; j < nworkers; j++ {
 		newIter.Add(1)
-		go _ParseEmblFile(opt.Source(), entry_channel, newIter,
+		go _ParseEmblFile(
+			entry_channel,
+			newIter,
 			opt.WithFeatureTable(),
-			opt.BatchSize(),
+		)
-			opt.TotalSeqSize())
 	}
 
 	go func() {
@@ -193,7 +218,7 @@ func ReadEMBL(reader io.Reader, options ...WithOption) obiiter.IBioSequence {
 		newIter = newIter.CompleteFileIterator()
 	}
 
-	return newIter
+	return newIter, nil
 }
 
 func ReadEMBLFromFile(filename string, options ...WithOption) (obiiter.IBioSequence, error) {
@@ -202,9 +227,9 @@ func ReadEMBLFromFile(filename string, options ...WithOption) (obiiter.IBioSeque
 
 	options = append(options, OptionsSource(obiutils.RemoveAllExt((path.Base(filename)))))
 
-	reader, err = Ropen(filename)
+	reader, err = obiutils.Ropen(filename)
 
-	if err == ErrNoContent {
+	if err == obiutils.ErrNoContent {
 		log.Infof("file %s is empty", filename)
 		return ReadEmptyFile(options...)
 	}
@@ -214,5 +239,5 @@ func ReadEMBLFromFile(filename string, options ...WithOption) (obiiter.IBioSeque
 		return obiiter.NilIBioSequence, err
 	}
 
-	return ReadEMBL(reader, options...), nil
+	return ReadEMBL(reader, options...)
 }

pkg/obiformats/fastaseq_read.go

@@ -14,7 +14,7 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
-func _EndOfLastFastaEntry(buffer []byte) int {
+func EndOfLastFastaEntry(buffer []byte) int {
 	var i int
 
 	imax := len(buffer)
@@ -39,24 +39,18 @@ func _EndOfLastFastaEntry(buffer []byte) int {
 	return last
 }
 
-func _ParseFastaFile(source string,
+func FastaChunkParser() func(string, io.Reader) (obiseq.BioSequenceSlice, error) {
-	input ChannelSeqFileChunk,
-	out obiiter.IBioSequence,
-	no_order bool,
-	batch_size int,
-	chunck_order func() int,
-) {
 
-	var identifier string
+	parser := func(source string, input io.Reader) (obiseq.BioSequenceSlice, error) {
-	var definition string
+		var identifier string
+		var definition string
 
-	idBytes := bytes.Buffer{}
+		idBytes := bytes.Buffer{}
-	defBytes := bytes.Buffer{}
+		defBytes := bytes.Buffer{}
-	seqBytes := bytes.Buffer{}
+		seqBytes := bytes.Buffer{}
 
-	for chunks := range input {
 		state := 0
-		scanner := bufio.NewReader(chunks.raw)
+		scanner := bufio.NewReader(input)
 		start, _ := scanner.Peek(20)
 		if start[0] != '>' {
 			log.Fatalf("%s : first character is not '>'", string(start))
@@ -64,7 +58,8 @@ func _ParseFastaFile(source string,
 		if start[1] == ' ' {
 			log.Fatalf("%s :Strange", string(start))
 		}
-		sequences := make(obiseq.BioSequenceSlice, 0, batch_size)
+
+		sequences := obiseq.MakeBioSequenceSlice(100)[:0]
 
 		previous := byte(0)
 
@@ -160,12 +155,6 @@ func _ParseFastaFile(source string,
 						s := obiseq.NewBioSequence(identifier, rawseq, definition)
 						s.SetSource(source)
 						sequences = append(sequences, s)
-						if no_order {
-							if len(sequences) == batch_size {
-								out.Push(obiiter.MakeBioSequenceBatch(chunck_order(), sequences))
-								sequences = make(obiseq.BioSequenceSlice, 0, batch_size)
-							}
-						}
 						state = 1
 					} else {
 						// Error
@@ -209,13 +198,29 @@ func _ParseFastaFile(source string,
 			sequences = append(sequences, s)
 		}
 
-		if len(sequences) > 0 {
+		return sequences, nil
-			co := chunks.order
+	}
-			if no_order {
+
-				co = chunck_order()
+	return parser
-			}
+}
-			out.Push(obiiter.MakeBioSequenceBatch(co, sequences))
+
+func _ParseFastaFile(
+	input ChannelFileChunk,
+	out obiiter.IBioSequence,
+) {
+
+	parser := FastaChunkParser()
+
+	for chunks := range input {
+		sequences, err := parser(chunks.Source, chunks.Raw)
+		// log.Warnf("Chunck(%d:%d) -%d- ", chunks.Order, l, sequences.Len())
+
+		if err != nil {
+			log.Fatalf("File %s : Cannot parse the fasta file : %v", chunks.Source, err)
 		}
+
+		out.Push(obiiter.MakeBioSequenceBatch(chunks.Source, chunks.Order, sequences))
+
 	}
 
 	out.Done()
@@ -228,26 +233,25 @@ func ReadFasta(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, e
 
 	nworker := opt.ParallelWorkers()
 
-	buff := make([]byte, 1024*1024*1024)
+	buff := make([]byte, 1024*1024)
 
-	chkchan := ReadSeqFileChunk(reader, buff, _EndOfLastFastaEntry)
+	chkchan := ReadFileChunk(
-	chunck_order := obiutils.AtomicCounter()
+		opt.Source(),
+		reader,
+		buff,
+		EndOfLastFastaEntry,
+	)
 
 	for i := 0; i < nworker; i++ {
 		out.Add(1)
-		go _ParseFastaFile(opt.Source(),
+		go _ParseFastaFile(chkchan, out)
-			chkchan,
-			out,
-			opt.NoOrder(),
-			opt.BatchSize(),
-			chunck_order)
 	}
 
 	go func() {
 		out.WaitAndClose()
 	}()
 
-	newIter := out.SortBatches().Rebatch(opt.BatchSize())
+	newIter := out.SortBatches()
 
 	log.Debugln("Full file batch mode : ", opt.FullFileBatch())
 
@@ -267,9 +271,9 @@ func ReadFasta(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, e
 func ReadFastaFromFile(filename string, options ...WithOption) (obiiter.IBioSequence, error) {
 	options = append(options, OptionsSource(obiutils.RemoveAllExt((path.Base(filename)))))
 
-	file, err := Ropen(filename)
+	file, err := obiutils.Ropen(filename)
 
-	if err == ErrNoContent {
+	if err == obiutils.ErrNoContent {
 		log.Infof("file %s is empty", filename)
 		return ReadEmptyFile(options...)
 	}
@@ -282,10 +286,10 @@ func ReadFastaFromFile(filename string, options ...WithOption) (obiiter.IBioSequ
 }
 
 func ReadFastaFromStdin(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, error) {
-	options = append(options, OptionsSource(obiutils.RemoveAllExt("stdin")))
+	options = append(options, OptionsSource("stdin"))
-	input, err := Buf(os.Stdin)
+	input, err := obiutils.Buf(os.Stdin)
 
-	if err == ErrNoContent {
+	if err == obiutils.ErrNoContent {
 		log.Infof("stdin is empty")
 		return ReadEmptyFile(options...)
 	}

pkg/obiformats/fastqseq_read.go

@@ -7,16 +7,17 @@ import (
 	"os"
 	"path"
 
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 	log "github.com/sirupsen/logrus"
 )
 
-func _EndOfLastFastqEntry(buffer []byte) int {
+func EndOfLastFastqEntry(buffer []byte) int {
 	var i int
 
+	// log.Warnf("EndOfLastFastqEntry(%d): %s", len(buffer), string(buffer[0:20]))
 	imax := len(buffer)
 	state := 0
 	restart := imax - 1
@@ -32,39 +33,48 @@ func _EndOfLastFastqEntry(buffer []byte) int {
 		case 0:
 			if C == '+' {
 				// Potential start of quality part step 1
+				// log.Warn("Potential start of quality part step 1 - +")
 				state = 1
 				restart = i
 			}
 		case 1:
 			if is_end_of_line {
 				// Potential start of quality part step 2
+				// log.Warn("Potential start of quality part step 2 - +/end of line")
 				state = 2
 			} else {
 				// it was not the start of quality part
+				// log.Warn("it was not the start of quality part")
 				state = 0
 				i = restart
 			}
 		case 2:
 			if is_sep {
 				// Potential start of quality part step 2 (stay in the same state)
+				// log.Warn("Potential start of quality part step 2 - skipping separator")
 				state = 2
 			} else if (C >= 'a' && C <= 'z') || (C >= 'A' && C <= 'Z') || C == '-' || C == '.' || C == '[' || C == ']' {
-				// End of the sequence
+				// progressing along of the sequence
+				// log.Warn("Detected the end of the sequence switching to state 3")
 				state = 3
 			} else {
 				// it was not the start of quality part
+				// log.Warn("it was not the start of quality part because is not preceded by sequence")
 				state = 0
 				i = restart
 			}
 		case 3:
 			if is_end_of_line {
 				// Entrering in the header line
+				// log.Warn("Potentially entrering in the header line")
 				state = 4
 			} else if (C >= 'a' && C <= 'z') || (C >= 'A' && C <= 'Z') || C == '-' || C == '.' || C == '[' || C == ']' {
 				// progressing along of the sequence
+				// log.Warn("Progressing along of the sequence")
 				state = 3
 			} else {
 				// it was not the sequence part
+				// log.Warnf("it was not the sequence part : %c", C)
 				state = 0
 				i = restart
 			}
@@ -72,6 +82,7 @@ func _EndOfLastFastqEntry(buffer []byte) int {
 			if is_end_of_line {
 				state = 4
 			} else {
+
 				state = 5
 			}
 		case 5:
@@ -80,15 +91,18 @@ func _EndOfLastFastqEntry(buffer []byte) int {
 				state = 0
 				i = restart
 			} else if C == '@' {
+				// It was the header line
+				// log.Warn("It was the header line")
 				state = 6
 				cut = i
 			}
 		case 6:
 			if is_end_of_line {
+				// log.Warn("====> End of the last sequence")
 				state = 7
 			} else {
-				state = 0
+				// log.Warnf("%s: Strange it was not the end of the last sequence : %c : %s", string(buffer[0:40]), C, string(buffer[i-20:i+5]))
-				i = restart
+				state = 5
 			}
 		}
 	}
@@ -117,27 +131,20 @@ func _storeSequenceQuality(bytes *bytes.Buffer, out *obiseq.BioSequence, quality
 	out.SetQualities(q)
 }
 
-func _ParseFastqFile(source string,
+func FastqChunkParser(quality_shift byte, with_quality bool) func(string, io.Reader) (obiseq.BioSequenceSlice, error) {
-	input ChannelSeqFileChunk,
+	parser := func(source string, input io.Reader) (obiseq.BioSequenceSlice, error) {
-	out obiiter.IBioSequence,
-	quality_shift byte,
-	no_order bool,
-	batch_size int,
-	chunck_order func() int,
-) {
 
-	var identifier string
+		var identifier string
-	var definition string
+		var definition string
 
-	idBytes := bytes.Buffer{}
+		idBytes := bytes.Buffer{}
-	defBytes := bytes.Buffer{}
+		defBytes := bytes.Buffer{}
-	qualBytes := bytes.Buffer{}
+		qualBytes := bytes.Buffer{}
-	seqBytes := bytes.Buffer{}
+		seqBytes := bytes.Buffer{}
 
-	for chunks := range input {
 		state := 0
-		scanner := bufio.NewReader(chunks.raw)
+		scanner := bufio.NewReader(input)
-		sequences := make(obiseq.BioSequenceSlice, 0, 100)
+		sequences := obiseq.MakeBioSequenceSlice(100)[:0]
 		previous := byte(0)
 
 		for C, err := scanner.ReadByte(); err != io.EOF; C, err = scanner.ReadByte() {
@@ -256,15 +263,9 @@ func _ParseFastqFile(source string,
 				}
 			case 10:
 				if is_end_of_line {
-					_storeSequenceQuality(&qualBytes, sequences[len(sequences)-1], quality_shift)
+					if with_quality {
-
+						_storeSequenceQuality(&qualBytes, sequences[len(sequences)-1], quality_shift)
-					if no_order {
-						if len(sequences) == batch_size {
-							out.Push(obiiter.MakeBioSequenceBatch(chunck_order(), sequences))
-							sequences = make(obiseq.BioSequenceSlice, 0, batch_size)
-						}
 					}
-
 					state = 11
 				} else {
 					qualBytes.WriteByte(C)
@@ -288,14 +289,32 @@ func _ParseFastqFile(source string,
 				_storeSequenceQuality(&qualBytes, sequences[len(sequences)-1], quality_shift)
 				state = 1
 			}
-
-			co := chunks.order
-			if no_order {
-				co = chunck_order()
-			}
-			out.Push(obiiter.MakeBioSequenceBatch(co, sequences))
 		}
 
+		return sequences, nil
+	}
+
+	return parser
+}
+
+func _ParseFastqFile(
+	input ChannelFileChunk,
+	out obiiter.IBioSequence,
+	quality_shift byte,
+	with_quality bool,
+) {
+
+	parser := FastqChunkParser(quality_shift, with_quality)
+
+	for chunks := range input {
+		sequences, err := parser(chunks.Source, chunks.Raw)
+
+		if err != nil {
+			log.Fatalf("File %s : Cannot parse the fastq file : %v", chunks.Source, err)
+		}
+
+		out.Push(obiiter.MakeBioSequenceBatch(chunks.Source, chunks.Order, sequences))
+
 	}
 
 	out.Done()
@@ -307,28 +326,31 @@ func ReadFastq(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, e
 	out := obiiter.MakeIBioSequence()
 
 	nworker := opt.ParallelWorkers()
-	chunkorder := obiutils.AtomicCounter()
 
-	buff := make([]byte, 1024*1024*1024)
+	buff := make([]byte, 1024*1024)
 
-	chkchan := ReadSeqFileChunk(reader, buff, _EndOfLastFastqEntry)
+	chkchan := ReadFileChunk(
+		opt.Source(),
+		reader,
+		buff,
+		EndOfLastFastqEntry,
+	)
 
 	for i := 0; i < nworker; i++ {
 		out.Add(1)
-		go _ParseFastqFile(opt.Source(),
+		go _ParseFastqFile(
 			chkchan,
 			out,
-			byte(obioptions.InputQualityShift()),
+			obidefault.ReadQualitiesShift(),
-			opt.NoOrder(),
+			opt.ReadQualities(),
-			opt.BatchSize(),
+		)
-			chunkorder)
 	}
 
 	go func() {
 		out.WaitAndClose()
 	}()
 
-	newIter := out.SortBatches().Rebatch(opt.BatchSize())
+	newIter := out.SortBatches()
 
 	log.Debugln("Full file batch mode : ", opt.FullFileBatch())
 
@@ -348,9 +370,9 @@ func ReadFastq(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, e
 func ReadFastqFromFile(filename string, options ...WithOption) (obiiter.IBioSequence, error) {
 	options = append(options, OptionsSource(obiutils.RemoveAllExt((path.Base(filename)))))
 
-	file, err := Ropen(filename)
+	file, err := obiutils.Ropen(filename)
 
-	if err == ErrNoContent {
+	if err == obiutils.ErrNoContent {
 		log.Infof("file %s is empty", filename)
 		return ReadEmptyFile(options...)
 	}
@@ -364,9 +386,9 @@ func ReadFastqFromFile(filename string, options ...WithOption) (obiiter.IBioSequ
 
 func ReadFastqFromStdin(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, error) {
 	options = append(options, OptionsSource(obiutils.RemoveAllExt("stdin")))
-	input, err := Buf(os.Stdin)
+	input, err := obiutils.Buf(os.Stdin)
 
-	if err == ErrNoContent {
+	if err == obiutils.ErrNoContent {
 		log.Infof("stdin is empty")
 		return ReadEmptyFile(options...)
 	}

pkg/obiformats/fastseq_json_header.go

@@ -2,7 +2,7 @@ package obiformats
 
 import (
 	"bytes"
-	"math"
+	"strconv"
 	"strings"
 	"unsafe"
 
@@ -10,10 +10,197 @@ import (
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
-	"github.com/goccy/go-json"
+	"github.com/buger/jsonparser"
 )
 
-func _parse_json_header_(header string, annotations obiseq.Annotation) string {
+func _parse_json_map_string(str []byte) (map[string]string, error) {
+	values := make(map[string]string)
+	jsonparser.ObjectEach(str,
+		func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) (err error) {
+			skey := string(key)
+			values[skey] = string(value)
+			return
+		},
+	)
+	return values, nil
+}
+
+func _parse_json_map_int(str []byte) (map[string]int, error) {
+	values := make(map[string]int)
+	jsonparser.ObjectEach(str,
+		func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) (err error) {
+			skey := string(key)
+			intval, err := jsonparser.ParseInt(value)
+			if err != nil {
+				return err
+			}
+			values[skey] = int(intval)
+			return nil
+		},
+	)
+	return values, nil
+}
+
+func _parse_json_map_float(str []byte) (map[string]float64, error) {
+	values := make(map[string]float64)
+	jsonparser.ObjectEach(str,
+		func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) (err error) {
+			skey := string(key)
+			floatval, err := strconv.ParseFloat(obiutils.UnsafeString(value), 64)
+			if err != nil {
+				return err
+			}
+			values[skey] = float64(floatval)
+			return nil
+		},
+	)
+	return values, nil
+}
+
+func _parse_json_map_bool(str []byte) (map[string]bool, error) {
+	values := make(map[string]bool)
+	jsonparser.ObjectEach(str,
+		func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) (err error) {
+			skey := string(key)
+			boolval, err := jsonparser.ParseBoolean(value)
+			if err != nil {
+				return err
+			}
+			values[skey] = boolval
+			return nil
+		},
+	)
+	return values, nil
+}
+
+func _parse_json_map_interface(str []byte) (map[string]interface{}, error) {
+	values := make(map[string]interface{})
+	jsonparser.ObjectEach(str,
+		func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) (err error) {
+			skey := string(key)
+			switch dataType {
+			case jsonparser.String:
+				values[skey] = string(value)
+			case jsonparser.Number:
+				// Try to parse the number as an int at first then as float if that fails.
+				values[skey], err = jsonparser.ParseInt(value)
+				if err != nil {
+					values[skey], err = strconv.ParseFloat(obiutils.UnsafeString(value), 64)
+				}
+				if err != nil {
+					return
+				}
+			case jsonparser.Boolean:
+			default:
+				values[skey] = string(value)
+			}
+			return
+		},
+	)
+	return values, nil
+}
+
+func _parse_json_array_string(str []byte) ([]string, error) {
+	values := make([]string, 0)
+	jsonparser.ArrayEach(str,
+		func(value []byte, dataType jsonparser.ValueType, offset int, err error) {
+			if dataType == jsonparser.String {
+				skey := string(value)
+				values = append(values, skey)
+			}
+		},
+	)
+	return values, nil
+}
+
+func _parse_json_array_int(str []byte, sequence *obiseq.BioSequence) ([]int, error) {
+	values := make([]int, 0)
+	jsonparser.ArrayEach(str,
+		func(value []byte, dataType jsonparser.ValueType, offset int, err error) {
+			if dataType == jsonparser.Number {
+				intval, err := jsonparser.ParseInt(value)
+				if err != nil {
+					log.Fatalf("%s: Parsing int failed on value %s: %s", sequence.Id(), value, err)
+				}
+				values = append(values, int(intval))
+			}
+		},
+	)
+	return values, nil
+}
+
+func _parse_json_array_float(str []byte, sequence *obiseq.BioSequence) ([]float64, error) {
+	values := make([]float64, 0)
+	jsonparser.ArrayEach(str,
+		func(value []byte, dataType jsonparser.ValueType, offset int, err error) {
+			if dataType == jsonparser.Number {
+				floatval, err := strconv.ParseFloat(obiutils.UnsafeString(value), 64)
+				if err == nil {
+					values = append(values, float64(floatval))
+				} else {
+					log.Fatalf("%s: Parsing float failed on value %s: %s", sequence.Id(), value, err)
+				}
+			}
+		},
+	)
+	return values, nil
+}
+
+func _parse_json_array_bool(str []byte, sequence *obiseq.BioSequence) ([]bool, error) {
+	values := make([]bool, 0)
+	jsonparser.ArrayEach(str,
+		func(value []byte, dataType jsonparser.ValueType, offset int, err error) {
+			if dataType == jsonparser.Boolean {
+				boolval, err := jsonparser.ParseBoolean(value)
+				if err != nil {
+					log.Fatalf("%s: Parsing bool failed on value %s: %s", sequence.Id(), value, err)
+				}
+				values = append(values, boolval)
+			}
+		},
+	)
+	return values, nil
+}
+
+func _parse_json_array_interface(str []byte) ([]interface{}, error) {
+	values := make([]interface{}, 0)
+	jsonparser.ArrayEach(str,
+		func(value []byte, dataType jsonparser.ValueType, offset int, err error) {
+			switch dataType {
+			case jsonparser.String:
+				values = append(values, string(value))
+			case jsonparser.Number:
+				// Try to parse the number as an int at first then as float if that fails.
+				intval, err := jsonparser.ParseInt(value)
+				if err != nil {
+					floatval, err := strconv.ParseFloat(obiutils.UnsafeString(value), 64)
+					if err != nil {
+						values = append(values, string(value))
+					} else {
+						values = append(values, floatval)
+					}
+				} else {
+					values = append(values, intval)
+				}
+			case jsonparser.Boolean:
+				boolval, err := jsonparser.ParseBoolean(value)
+				if err != nil {
+					values = append(values, string(value))
+				} else {
+					values = append(values, boolval)
+				}
+
+			default:
+				values = append(values, string(value))
+			}
+
+		},
+	)
+	return values, nil
+}
+
+func _parse_json_header_(header string, sequence *obiseq.BioSequence) string {
+	annotations := sequence.Annotations()
 	start := -1
 	stop := -1
 	level := 0
@@ -51,23 +238,123 @@ func _parse_json_header_(header string, annotations obiseq.Annotation) string {
 
 	stop++
 
-	err := json.Unmarshal([]byte(header)[start:stop], &annotations)
+	jsonparser.ObjectEach(obiutils.UnsafeBytes(header[start:stop]),
+		func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) error {
+			var err error
 
-	for k, v := range annotations {
+			skey := obiutils.UnsafeString(key)
-		switch vt := v.(type) {
-		case float64:
-			if vt == math.Floor(vt) {
-				annotations[k] = int(vt)
-			}
-			{
-				annotations[k] = vt
-			}
-		}
-	}
 
-	if err != nil {
+			switch {
-		log.Fatalf("annotation parsing error on %s : %v\n", header, err)
+			case skey == "id":
-	}
+				sequence.SetId(string(value))
+			case skey == "definition":
+				sequence.SetDefinition(string(value))
+
+			case skey == "count":
+				if dataType != jsonparser.Number {
+					log.Fatalf("%s: Count attribut must be numeric: %s", sequence.Id(), string(value))
+				}
+				count, err := jsonparser.ParseInt(value)
+				if err != nil {
+					log.Fatalf("%s: Cannot parse count %s", sequence.Id(), string(value))
+				}
+				sequence.SetCount(int(count))
+
+			case skey == "obiclean_weight":
+				weight, err := _parse_json_map_int(value)
+				if err != nil {
+					log.Fatalf("%s: Cannot parse obiclean weight %s", sequence.Id(), string(value))
+				}
+				annotations[skey] = weight
+
+			case skey == "obiclean_status":
+				status, err := _parse_json_map_string(value)
+				if err != nil {
+					log.Fatalf("%s: Cannot parse obiclean status %s", sequence.Id(), string(value))
+				}
+				annotations[skey] = status
+
+			case strings.HasPrefix(skey, "merged_"):
+				if dataType == jsonparser.Object {
+					data, err := _parse_json_map_int(value)
+					if err != nil {
+						log.Fatalf("%s: Cannot parse merged slot %s: %v", sequence.Id(), skey, err)
+					} else {
+						annotations[skey] = data
+					}
+				} else {
+					log.Fatalf("%s: Cannot parse merged slot %s", sequence.Id(), skey)
+				}
+
+			case skey == "taxid":
+				if dataType == jsonparser.Number || dataType == jsonparser.String {
+					taxid := string(value)
+					sequence.SetTaxid(taxid)
+				} else {
+					log.Fatalf("%s: Cannot parse taxid %s", sequence.Id(), string(value))
+				}
+
+			case strings.HasSuffix(skey, "_taxid"):
+				if dataType == jsonparser.Number || dataType == jsonparser.String {
+					rank, _ := obiutils.SplitInTwo(skey, '_')
+
+					taxid := string(value)
+					sequence.SetTaxid(taxid, rank)
+				} else {
+					log.Fatalf("%s: Cannot parse taxid %s", sequence.Id(), string(value))
+				}
+
+			default:
+				skey = strings.Clone(skey)
+				switch dataType {
+				case jsonparser.String:
+					annotations[skey] = string(value)
+				case jsonparser.Number:
+					// Try to parse the number as an int at first then as float if that fails.
+					annotations[skey], err = jsonparser.ParseInt(value)
+					if err != nil {
+						annotations[skey], err = strconv.ParseFloat(obiutils.UnsafeString(value), 64)
+					}
+				case jsonparser.Array:
+					annotations[skey], err = _parse_json_array_interface(value)
+				case jsonparser.Object:
+					annotations[skey], err = _parse_json_map_interface(value)
+				case jsonparser.Boolean:
+					annotations[skey], err = jsonparser.ParseBoolean(value)
+				case jsonparser.Null:
+					annotations[skey] = nil
+				default:
+					log.Fatalf("Unknown data type %v", dataType)
+				}
+			}
+
+			if err != nil {
+				annotations[skey] = "NaN"
+				log.Fatalf("%s: Cannot parse value %s assicated to key %s into a %s value",
+					sequence.Id(), string(value), skey, dataType.String())
+			}
+
+			return err
+		},
+	)
+
+	// err := json.Unmarshal([]byte(header)[start:stop], &annotations)
+
+	// for k, v := range annotations {
+	// 	switch vt := v.(type) {
+	// 	case float64:
+	// 		if vt == math.Floor(vt) {
+	// 			annotations[k] = int(vt)
+	// 		}
+	// 		{
+	// 			annotations[k] = vt
+	// 		}
+	// 	}
+	// }
+
+	// if err != nil {
+	// 	log.Fatalf("annotation parsing error on %s : %v\n", header, err)
+	// }
 
 	return strings.TrimSpace(header[stop:])
 }
@@ -78,7 +365,9 @@ func ParseFastSeqJsonHeader(sequence *obiseq.BioSequence) {
 
 	definition_part := _parse_json_header_(
 		definition,
-		sequence.Annotations())
+		sequence,
+	)
+
 	if len(definition_part) > 0 {
 		if sequence.HasDefinition() {
 			definition_part = sequence.Definition() + " " + definition_part

pkg/obiformats/fastseq_read.go

@@ -14,8 +14,8 @@ import (
 
 	log "github.com/sirupsen/logrus"
 
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
@@ -69,7 +69,7 @@ func _FastseqReader(source string,
 		slice = append(slice, rep)
 		ii++
 		if ii >= batch_size {
-			iterator.Push(obiiter.MakeBioSequenceBatch(i, slice))
+			iterator.Push(obiiter.MakeBioSequenceBatch(source, i, slice))
 			slice = obiseq.MakeBioSequenceSlice()
 			i++
 			ii = 0
@@ -77,7 +77,7 @@ func _FastseqReader(source string,
 
 	}
 	if len(slice) > 0 {
-		iterator.Push(obiiter.MakeBioSequenceBatch(i, slice))
+		iterator.Push(obiiter.MakeBioSequenceBatch(source, i, slice))
 	}
 	iterator.Done()
 
@@ -92,7 +92,7 @@ func ReadFastSeqFromFile(filename string, options ...WithOption) (obiiter.IBioSe
 	name := C.CString(filename)
 	defer C.free(unsafe.Pointer(name))
 
-	pointer := C.open_fast_sek_file(name, C.int32_t(obioptions.InputQualityShift()))
+	pointer := C.open_fast_sek_file(name, C.int32_t(obidefault.ReadQualitiesShift()))
 
 	var err error
 	err = nil
@@ -151,7 +151,7 @@ func ReadFastSeqFromStdin(options ...WithOption) obiiter.IBioSequence {
 	}(newIter)
 
 	go _FastseqReader(opt.Source(),
-		C.open_fast_sek_stdin(C.int32_t(obioptions.InputQualityShift())),
+		C.open_fast_sek_stdin(C.int32_t(obidefault.ReadQualitiesShift())),
 		newIter, opt.BatchSize())
 
 	log.Debugln("Full file batch mode : ", opt.FullFileBatch())

pkg/obiformats/fastseq_write_fasta.go

@@ -7,7 +7,6 @@ import (
 	"io"
 	"os"
 	"strings"
-	"sync"
 	"time"
 
 	log "github.com/sirupsen/logrus"
@@ -76,7 +75,7 @@ func FormatFasta(seq *obiseq.BioSequence, formater FormatHeader) string {
 // - skipEmpty: a boolean indicating whether empty sequences should be skipped or not.
 //
 // It returns a byte array containing the formatted sequences.
-func FormatFastaBatch(batch obiiter.BioSequenceBatch, formater FormatHeader, skipEmpty bool) []byte {
+func FormatFastaBatch(batch obiiter.BioSequenceBatch, formater FormatHeader, skipEmpty bool) *bytes.Buffer {
 	// Create a buffer to store the formatted sequences
 	var bs bytes.Buffer
 
@@ -116,7 +115,7 @@ func FormatFastaBatch(batch obiiter.BioSequenceBatch, formater FormatHeader, ski
 	}
 
 	// Return the byte array representation of the buffer
-	return bs.Bytes()
+	return &bs
 }
 
 // WriteFasta writes a given iterator of bio sequences to a file in FASTA format.
@@ -128,20 +127,17 @@ func WriteFasta(iterator obiiter.IBioSequence,
 	options ...WithOption) (obiiter.IBioSequence, error) {
 	opt := MakeOptions(options)
 
-	iterator = iterator.Rebatch(opt.BatchSize())
 	file, _ = obiutils.CompressStream(file, opt.CompressedFile(), opt.CloseFile())
 
 	newIter := obiiter.MakeIBioSequence()
 
 	nwriters := opt.ParallelWorkers()
 
-	obiiter.RegisterAPipe()
+	chunkchan := WriteFileChunk(file, opt.CloseFile())
-	chunkchan := make(chan FileChunck)
 
 	header_format := opt.FormatFastSeqHeader()
 
 	newIter.Add(nwriters)
-	var waitWriter sync.WaitGroup
 
 	go func() {
 		newIter.WaitAndClose()
@@ -149,7 +145,7 @@ func WriteFasta(iterator obiiter.IBioSequence,
 			time.Sleep(time.Millisecond)
 		}
 		close(chunkchan)
-		waitWriter.Wait()
+		log.Debugf("Writing fasta file done")
 	}()
 
 	ff := func(iterator obiiter.IBioSequence) {
@@ -159,10 +155,12 @@ func WriteFasta(iterator obiiter.IBioSequence,
 
 			log.Debugf("Formating fasta chunk %d", batch.Order())
 
-			chunkchan <- FileChunck{
+			chunkchan <- FileChunk{
-				FormatFastaBatch(batch, header_format, opt.SkipEmptySequence()),
+				Source: batch.Source(),
-				batch.Order(),
+				Raw:    FormatFastaBatch(batch, header_format, opt.SkipEmptySequence()),
+				Order:  batch.Order(),
 			}
+
 			log.Debugf("Fasta chunk %d formated", batch.Order())
 
 			newIter.Push(batch)
@@ -172,43 +170,10 @@ func WriteFasta(iterator obiiter.IBioSequence,
 
 	log.Debugln("Start of the fasta file writing")
 	go ff(iterator)
-	for i := 0; i < nwriters-1; i++ {
+	for i := 1; i < nwriters; i++ {
 		go ff(iterator.Split())
 	}
 
-	next_to_send := 0
-	received := make(map[int]FileChunck, 100)
-
-	waitWriter.Add(1)
-	go func() {
-		for chunk := range chunkchan {
-			if chunk.order == next_to_send {
-				file.Write(chunk.text)
-				log.Debugf("Fasta chunk %d written", chunk.order)
-				next_to_send++
-				chunk, ok := received[next_to_send]
-				for ok {
-					file.Write(chunk.text)
-					log.Debugf("Fasta chunk %d written", chunk.order)
-					delete(received, next_to_send)
-					next_to_send++
-					chunk, ok = received[next_to_send]
-				}
-			} else {
-				log.Debugf("Store Fasta chunk %d", chunk.order)
-				received[chunk.order] = chunk
-			}
-
-		}
-
-		file.Close()
-
-		log.Debugln("End of the fasta file writing")
-		obiiter.UnregisterPipe()
-		waitWriter.Done()
-
-	}()
-
 	return newIter, nil
 }
 
@@ -222,7 +187,8 @@ func WriteFasta(iterator obiiter.IBioSequence,
 // The function returns the same bio sequence iterator and an error if any occurred.
 func WriteFastaToStdout(iterator obiiter.IBioSequence,
 	options ...WithOption) (obiiter.IBioSequence, error) {
-	options = append(options, OptionDontCloseFile())
+	// options = append(options, OptionDontCloseFile())
+	options = append(options, OptionCloseFile())
 	return WriteFasta(iterator, os.Stdout, options...)
 }
 

pkg/obiformats/fastseq_write_fastq.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"io"
 	"os"
-	"sync"
 	"time"
 
 	log "github.com/sirupsen/logrus"
@@ -14,6 +13,8 @@ import (
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
 
+type FormatSeqBatch func(batch obiiter.BioSequenceBatch, formater FormatHeader, skipEmpty bool) *bytes.Buffer
+
 func _formatFastq(buff *bytes.Buffer, seq *obiseq.BioSequence, formater FormatHeader) {
 
 	info := ""
@@ -49,7 +50,7 @@ func FormatFastq(seq *obiseq.BioSequence, formater FormatHeader) string {
 }
 
 func FormatFastqBatch(batch obiiter.BioSequenceBatch,
-	formater FormatHeader, skipEmpty bool) []byte {
+	formater FormatHeader, skipEmpty bool) *bytes.Buffer {
 	var bs bytes.Buffer
 
 	lt := 0
@@ -82,14 +83,7 @@ func FormatFastqBatch(batch obiiter.BioSequenceBatch,
 
 	}
 
-	chunk := bs.Bytes()
+	return &bs
-
-	return chunk
-}
-
-type FileChunck struct {
-	text  []byte
-	order int
 }
 
 func WriteFastq(iterator obiiter.IBioSequence,
@@ -97,7 +91,6 @@ func WriteFastq(iterator obiiter.IBioSequence,
 	options ...WithOption) (obiiter.IBioSequence, error) {
 
 	opt := MakeOptions(options)
-	iterator = iterator.Rebatch(opt.BatchSize())
 
 	file, _ = obiutils.CompressStream(file, opt.CompressedFile(), opt.CloseFile())
 
@@ -105,30 +98,28 @@ func WriteFastq(iterator obiiter.IBioSequence,
 
 	nwriters := opt.ParallelWorkers()
 
-	obiiter.RegisterAPipe()
+	chunkchan := WriteFileChunk(file, opt.CloseFile())
-	chunkchan := make(chan FileChunck)
 
 	header_format := opt.FormatFastSeqHeader()
 
 	newIter.Add(nwriters)
 
-	var waitWriter sync.WaitGroup
-
 	go func() {
 		newIter.WaitAndClose()
 		for len(chunkchan) > 0 {
 			time.Sleep(time.Millisecond)
 		}
 		close(chunkchan)
-		waitWriter.Wait()
+		log.Debugf("Writing fastq file done")
 	}()
 
 	ff := func(iterator obiiter.IBioSequence) {
 		for iterator.Next() {
 			batch := iterator.Get()
-			chunk := FileChunck{
+			chunk := FileChunk{
-				FormatFastqBatch(batch, header_format, opt.SkipEmptySequence()),
+				Source: batch.Source(),
-				batch.Order(),
+				Raw:    FormatFastqBatch(batch, header_format, opt.SkipEmptySequence()),
+				Order:  batch.Order(),
 			}
 			chunkchan <- chunk
 			newIter.Push(batch)
@@ -138,54 +129,18 @@ func WriteFastq(iterator obiiter.IBioSequence,
 
 	log.Debugln("Start of the fastq file writing")
 	go ff(iterator)
-	for i := 0; i < nwriters-1; i++ {
+	for i := 1; i < nwriters; i++ {
 		go ff(iterator.Split())
 	}
 
-	next_to_send := 0
-	received := make(map[int]FileChunck, 100)
-
-	waitWriter.Add(1)
-	go func() {
-		for chunk := range chunkchan {
-			if chunk.order == next_to_send {
-				if chunk.text[0] != '@' {
-					log.Panicln("WriteFastq: FASTQ format error")
-				}
-				file.Write(chunk.text)
-				next_to_send++
-				chunk, ok := received[next_to_send]
-				for ok {
-					if chunk.text[0] != '@' {
-						log.Panicln("WriteFastq: FASTQ format error")
-					}
-					file.Write(chunk.text)
-					delete(received, next_to_send)
-					next_to_send++
-					chunk, ok = received[next_to_send]
-				}
-			} else {
-				if _, ok := received[chunk.order]; ok {
-					log.Panicln("WriteFastq: Two chunks with the same number")
-				}
-				received[chunk.order] = chunk
-			}
-
-		}
-
-		file.Close()
-
-		log.Debugln("End of the fastq file writing")
-		obiiter.UnregisterPipe()
-		waitWriter.Done()
-	}()
-
 	return newIter, nil
 }
 
 func WriteFastqToStdout(iterator obiiter.IBioSequence,
 	options ...WithOption) (obiiter.IBioSequence, error) {
-	options = append(options, OptionDontCloseFile())
+	//	options = append(options, OptionDontCloseFile())
+	options = append(options, OptionCloseFile())
+
 	return WriteFastq(iterator, os.Stdout, options...)
 }
 

pkg/obiformats/file_chunk_read.go

@@ -5,17 +5,19 @@ import (
 	"io"
 	"slices"
 
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 	log "github.com/sirupsen/logrus"
 )
 
-var _FileChunkSize = 1 << 28
+type SeqFileChunkParser func(string, io.Reader) (obiseq.BioSequenceSlice, error)
 
-type SeqFileChunk struct {
+type FileChunk struct {
-	raw   io.Reader
+	Source string
-	order int
+	Raw    *bytes.Buffer
+	Order  int
 }
 
-type ChannelSeqFileChunk chan SeqFileChunk
+type ChannelFileChunk chan FileChunk
 
 type LastSeqRecord func([]byte) int
 
@@ -32,13 +34,17 @@ type LastSeqRecord func([]byte) int
 //
 // Returns:
 // None
-func ReadSeqFileChunk(reader io.Reader,
+func ReadFileChunk(
+	source string,
+	reader io.Reader,
 	buff []byte,
-	splitter LastSeqRecord) ChannelSeqFileChunk {
+	splitter LastSeqRecord) ChannelFileChunk {
 	var err error
 	var fullbuff []byte
 
-	chunk_channel := make(ChannelSeqFileChunk)
+	chunk_channel := make(ChannelFileChunk)
+
+	fileChunkSize := len(buff)
 
 	go func() {
 		size := 0
@@ -65,11 +71,13 @@ func ReadSeqFileChunk(reader io.Reader,
 			// Read from the reader in 1 MB increments until the end of the last entry is found
 			for end = splitter(buff); err == nil && end < 0; end = splitter(buff) {
 				ic++
-				buff = slices.Grow(buff, _FileChunkSize)
+				buff = slices.Grow(buff, fileChunkSize)
 				l := len(buff)
-				extbuff := buff[l:(l + _FileChunkSize - 1)]
+				extbuff := buff[l:(l + fileChunkSize - 1)]
 				size, err = io.ReadFull(reader, extbuff)
 				buff = buff[0:(l + size)]
+				// log.Warnf("Splitter not found, attempting %d to read in %d B increments : len(buff) = %d/%d", ic, fileChunkSize, len(extbuff), len(buff))
+
 			}
 
 			fullbuff = buff
@@ -87,8 +95,10 @@ func ReadSeqFileChunk(reader io.Reader,
 				}
 
 				if len(buff) > 0 {
-					io := bytes.NewBuffer(slices.Clone(buff))
+					cbuff := slices.Clone(buff)
-					chunk_channel <- SeqFileChunk{io, i}
+					io := bytes.NewBuffer(cbuff)
+					// log.Warnf("chuck %d :Read %d bytes from file %s", i, io.Len(), source)
+					chunk_channel <- FileChunk{source, io, i}
 					i++
 				}
 
@@ -96,7 +106,7 @@ func ReadSeqFileChunk(reader io.Reader,
 					buff = fullbuff[0:lremain]
 					lcp := copy(buff, fullbuff[pnext:])
 					if lcp < lremain {
-						log.Fatalf("Error copying remaining data of chunck %d : %d < %d", i, lcp, lremain)
+						log.Fatalf("Error copying remaining data of chunk %d : %d < %d", i, lcp, lremain)
 					}
 				} else {
 					buff = buff[:0]
@@ -112,7 +122,7 @@ func ReadSeqFileChunk(reader io.Reader,
 		// Send the last chunk to the channel
 		if len(buff) > 0 {
 			io := bytes.NewBuffer(slices.Clone(buff))
-			chunk_channel <- SeqFileChunk{io, i}
+			chunk_channel <- FileChunk{source, io, i}
 		}
 
 		// Close the readers channel when the end of the file is reached

pkg/obiformats/file_chunk_write.go

@@ -0,0 +1,61 @@
+package obiformats
+
+import (
+	"io"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+
+	log "github.com/sirupsen/logrus"
+)
+
+func WriteFileChunk(
+	writer io.WriteCloser,
+	toBeClosed bool) ChannelFileChunk {
+
+	obiutils.RegisterAPipe()
+	chunk_channel := make(ChannelFileChunk)
+
+	go func() {
+		nextToPrint := 0
+		toBePrinted := make(map[int]FileChunk)
+		for chunk := range chunk_channel {
+			if chunk.Order == nextToPrint {
+				log.Debugf("Writing chunk: %d of length %d bytes",
+					chunk.Order,
+					len(chunk.Raw.Bytes()))
+
+				n, err := writer.Write(chunk.Raw.Bytes())
+
+				if err != nil {
+					log.Fatalf("Cannot write chunk %d only %d bytes written on %d sended : %v",
+						chunk.Order, n, len(chunk.Raw.Bytes()), err)
+				}
+				nextToPrint++
+
+				chunk, ok := toBePrinted[nextToPrint]
+				for ok {
+					log.Debug("Writing buffered chunk : ", chunk.Order)
+					_, _ = writer.Write(chunk.Raw.Bytes())
+					delete(toBePrinted, nextToPrint)
+					nextToPrint++
+					chunk, ok = toBePrinted[nextToPrint]
+				}
+			} else {
+				toBePrinted[chunk.Order] = chunk
+			}
+		}
+
+		log.Debugf("FIle have to be closed : %v", toBeClosed)
+		if toBeClosed {
+			err := writer.Close()
+			if err != nil {
+				log.Fatalf("Cannot close the writer : %v", err)
+			}
+		}
+
+		obiutils.UnregisterPipe()
+		log.Debugf("The writer has been closed")
+	}()
+
+	return chunk_channel
+}

pkg/obiformats/genbank_read.go

@@ -29,27 +29,11 @@ const (
 
 var _seqlenght_rx = regexp.MustCompile(" +([0-9]+) bp")
 
-func _ParseGenbankFile(source string,
+func GenbankChunkParser(withFeatureTable bool) func(string, io.Reader) (obiseq.BioSequenceSlice, error) {
-	input ChannelSeqFileChunk,
+	return func(source string, input io.Reader) (obiseq.BioSequenceSlice, error) {
-	out obiiter.IBioSequence,
+		state := inHeader
-	chunck_order func() int,
+		scanner := bufio.NewReader(input)
-	withFeatureTable bool,
+		sequences := obiseq.MakeBioSequenceSlice(100)[:0]
-	batch_size int,
-	total_seq_size int) {
-	state := inHeader
-	previous_chunk := -1
-
-	for chunks := range input {
-
-		if state != inHeader {
-			log.Fatalf("Unexpected state %d starting new chunk (id = %d, previous_chunk = %d)",
-				state, chunks.order, previous_chunk)
-		}
-
-		previous_chunk = chunks.order
-		scanner := bufio.NewReader(chunks.raw)
-		sequences := make(obiseq.BioSequenceSlice, 0, 100)
-		sumlength := 0
 		id := ""
 		lseq := -1
 		scientificName := ""
@@ -64,7 +48,7 @@ func _ParseGenbankFile(source string,
 			nl++
 			line = string(bline)
 			if is_prefix || len(line) > 100 {
-				log.Fatalf("Chunk %d : Line too long: %s", chunks.order, line)
+				log.Fatalf("From %s:Line too long: %s", source, line)
 			}
 			processed := false
 			for !processed {
@@ -165,15 +149,6 @@ func _ParseGenbankFile(source string,
 					//	sequence.Len(), seqBytes.Len())
 
 					sequences = append(sequences, sequence)
-					sumlength += sequence.Len()
-
-					if len(sequences) == batch_size || sumlength > total_seq_size {
-						oo := chunck_order()
-						log.Debugln("Pushing sequence batch ", oo, " with ", len(sequences), " sequences")
-						out.Push(obiiter.MakeBioSequenceBatch(oo, sequences))
-						sequences = make(obiseq.BioSequenceSlice, 0, 100)
-						sumlength = 0
-					}
 
 					defBytes = bytes.NewBuffer(obiseq.GetSlice(200))
 					featBytes = new(bytes.Buffer)
@@ -219,11 +194,24 @@ func _ParseGenbankFile(source string,
 
 		}
 
-		if len(sequences) > 0 {
+		return sequences, nil
-			oo := chunck_order()
+	}
-			log.Debugln("Pushing sequence batch ", oo, " with ", len(sequences), " sequences")
+}
-			out.Push(obiiter.MakeBioSequenceBatch(oo, sequences))
+
+func _ParseGenbankFile(input ChannelFileChunk,
+	out obiiter.IBioSequence,
+	withFeatureTable bool) {
+
+	parser := GenbankChunkParser(withFeatureTable)
+
+	for chunks := range input {
+		sequences, err := parser(chunks.Source, chunks.Raw)
+
+		if err != nil {
+			log.Fatalf("File %s : Cannot parse the genbank file : %v", chunks.Source, err)
 		}
+
+		out.Push(obiiter.MakeBioSequenceBatch(chunks.Source, chunks.Order, sequences))
 	}
 
 	log.Debug("End of the Genbank thread")
@@ -231,26 +219,31 @@ func _ParseGenbankFile(source string,
 
 }
 
-func ReadGenbank(reader io.Reader, options ...WithOption) obiiter.IBioSequence {
+func ReadGenbank(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, error) {
 	opt := MakeOptions(options)
 	// entry_channel := make(chan _FileChunk)
 
-	buff := make([]byte, 1024*1024*1024*256)
+	buff := make([]byte, 1024*1024*128) // 128 MB
+
+	entry_channel := ReadFileChunk(
+		opt.Source(),
+		reader,
+		buff,
+		EndOfLastFlatFileEntry,
+	)
 
-	entry_channel := ReadSeqFileChunk(reader, buff, _EndOfLastEntry)
 	newIter := obiiter.MakeIBioSequence()
 
 	nworkers := opt.ParallelWorkers()
-	chunck_order := obiutils.AtomicCounter()
 
 	// for j := 0; j < opt.ParallelWorkers(); j++ {
 	for j := 0; j < nworkers; j++ {
 		newIter.Add(1)
-		go _ParseGenbankFile(opt.Source(),
+		go _ParseGenbankFile(
-			entry_channel, newIter, chunck_order,
+			entry_channel,
+			newIter,
 			opt.WithFeatureTable(),
-			opt.BatchSize(),
+		)
-			opt.TotalSeqSize())
 	}
 
 	// go _ReadFlatFileChunk(reader, entry_channel)
@@ -264,7 +257,7 @@ func ReadGenbank(reader io.Reader, options ...WithOption) obiiter.IBioSequence {
 		newIter = newIter.CompleteFileIterator()
 	}
 
-	return newIter
+	return newIter, nil
 }
 
 func ReadGenbankFromFile(filename string, options ...WithOption) (obiiter.IBioSequence, error) {
@@ -273,9 +266,9 @@ func ReadGenbankFromFile(filename string, options ...WithOption) (obiiter.IBioSe
 
 	options = append(options, OptionsSource(obiutils.RemoveAllExt((path.Base(filename)))))
 
-	reader, err = Ropen(filename)
+	reader, err = obiutils.Ropen(filename)
 
-	if err == ErrNoContent {
+	if err == obiutils.ErrNoContent {
 		log.Infof("file %s is empty", filename)
 		return ReadEmptyFile(options...)
 	}
@@ -285,5 +278,5 @@ func ReadGenbankFromFile(filename string, options ...WithOption) (obiiter.IBioSe
 		return obiiter.NilIBioSequence, err
 	}
 
-	return ReadGenbank(reader, options...), nil
+	return ReadGenbank(reader, options...)
 }

pkg/obiformats/json_writer.go

@@ -3,14 +3,15 @@ package obiformats
 import (
 	"bufio"
 	"bytes"
-	"github.com/goccy/go-json"
 	"io"
 	"os"
 	"strconv"
 	"strings"
-	"sync"
+	"sync/atomic"
 	"time"
 
+	"github.com/goccy/go-json"
+
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
@@ -57,9 +58,17 @@ func JSONRecord(sequence *obiseq.BioSequence) []byte {
 	return text
 }
 
-func FormatJSONBatch(batch obiiter.BioSequenceBatch) []byte {
+func FormatJSONBatch(batch obiiter.BioSequenceBatch) *bytes.Buffer {
 	buff := new(bytes.Buffer)
+
 	json := bufio.NewWriter(buff)
+
+	if batch.Order() == 0 {
+		json.WriteString("[\n")
+	} else {
+		json.WriteString(",\n")
+	}
+
 	n := batch.Slice().Len() - 1
 	for i, s := range batch.Slice() {
 		json.WriteString("  ")
@@ -70,35 +79,36 @@ func FormatJSONBatch(batch obiiter.BioSequenceBatch) []byte {
 	}
 
 	json.Flush()
-
+	return buff
-	return buff.Bytes()
 }
 
 func WriteJSON(iterator obiiter.IBioSequence,
 	file io.WriteCloser,
 	options ...WithOption) (obiiter.IBioSequence, error) {
+	var latestChunk atomic.Int64
 
 	opt := MakeOptions(options)
 
 	file, _ = obiutils.CompressStream(file, opt.CompressedFile(), opt.CloseFile())
 
 	newIter := obiiter.MakeIBioSequence()
-
 	nwriters := opt.ParallelWorkers()
 
-	obiiter.RegisterAPipe()
+	chunkchan := WriteFileChunk(file, opt.CloseFile())
-	chunkchan := make(chan FileChunck)
-
 	newIter.Add(nwriters)
-	var waitWriter sync.WaitGroup
 
 	go func() {
 		newIter.WaitAndClose()
+
+		chunkchan <- FileChunk{
+			Source: "end",
+			Raw:    bytes.NewBuffer([]byte("\n]\n")),
+			Order:  int(latestChunk.Load()) + 1,
+		}
 		for len(chunkchan) > 0 {
 			time.Sleep(time.Millisecond)
 		}
 		close(chunkchan)
-		waitWriter.Wait()
 	}()
 
 	ff := func(iterator obiiter.IBioSequence) {
@@ -106,62 +116,32 @@ func WriteJSON(iterator obiiter.IBioSequence,
 
 			batch := iterator.Get()
 
-			chunkchan <- FileChunck{
+			ss := FileChunk{
-				FormatJSONBatch(batch),
+				Source: batch.Source(),
-				batch.Order(),
+				Raw:    FormatJSONBatch(batch),
+				Order:  batch.Order(),
 			}
+
+			chunkchan <- ss
+			latestChunk.Store(int64(batch.Order()))
 			newIter.Push(batch)
 		}
 		newIter.Done()
 	}
 
-	next_to_send := 0
-	received := make(map[int]FileChunck, 100)
-
-	waitWriter.Add(1)
-	go func() {
-		for chunk := range chunkchan {
-			if chunk.order == next_to_send {
-				if next_to_send > 0 {
-					file.Write([]byte(",\n"))
-				}
-				file.Write(chunk.text)
-				next_to_send++
-				chunk, ok := received[next_to_send]
-				for ok {
-					file.Write(chunk.text)
-					delete(received, next_to_send)
-					next_to_send++
-					chunk, ok = received[next_to_send]
-				}
-			} else {
-				received[chunk.order] = chunk
-			}
-
-		}
-
-		file.Write([]byte("\n]\n"))
-		file.Close()
-
-		log.Debugln("End of the JSON file writing")
-		obiiter.UnregisterPipe()
-		waitWriter.Done()
-
-	}()
-
 	log.Debugln("Start of the JSON file writing")
-	file.Write([]byte("[\n"))
+	for i := 1; i < nwriters; i++ {
-	go ff(iterator)
-	for i := 0; i < nwriters-1; i++ {
 		go ff(iterator.Split())
 	}
+	go ff(iterator)
 
 	return newIter, nil
 }
 
 func WriteJSONToStdout(iterator obiiter.IBioSequence,
 	options ...WithOption) (obiiter.IBioSequence, error) {
-	options = append(options, OptionDontCloseFile())
+	options = append(options, OptionCloseFile())
+
 	return WriteJSON(iterator, os.Stdout, options...)
 }
 

pkg/obiformats/ncbitaxdump/read.go

@@ -1,169 +0,0 @@
-package ncbitaxdump
-
-import (
-	"bufio"
-	"encoding/csv"
-	"fmt"
-	"io"
-	"os"
-	"path"
-	"strconv"
-	"strings"
-
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax"
-)
-
-func loadNodeTable(reader io.Reader, taxonomy *obitax.Taxonomy) {
-	file := csv.NewReader(reader)
-	file.Comma = '|'
-	file.Comment = '#'
-	file.TrimLeadingSpace = true
-	file.ReuseRecord = true
-
-	n := 0
-
-	for record, err := file.Read(); err == nil; record, err = file.Read() {
-		n++
-		taxid, err := strconv.Atoi(strings.TrimSpace(record[0]))
-
-		if err != nil {
-			log.Panicf("Cannot read taxon taxid at line %d: %v", n, err)
-		}
-
-		parent, err := strconv.Atoi(strings.TrimSpace(record[1]))
-
-		if err != nil {
-			log.Panicf("Cannot read taxon parent taxid at line %d: %v", n, err)
-		}
-
-		rank := strings.TrimSpace(record[2])
-
-		taxonomy.AddNewTaxa(taxid, parent, rank, true, true)
-	}
-
-	taxonomy.ReindexParent()
-}
-
-func loadNameTable(reader io.Reader, taxonomy *obitax.Taxonomy, onlysn bool) int {
-	// file := csv.NewReader(reader)
-	// file.Comma = '|'
-	// file.Comment = '#'
-	// file.TrimLeadingSpace = true
-	// file.ReuseRecord = true
-	// file.LazyQuotes = true
-	file := bufio.NewReader(reader)
-
-	n := 0
-	l := 0
-
-	for line, prefix, err := file.ReadLine(); err == nil; line, prefix, err = file.ReadLine() {
-		l++
-		if prefix {
-			return -1
-		}
-
-		record := strings.Split(string(line), "|")
-		taxid, err := strconv.Atoi(strings.TrimSpace(record[0]))
-
-		if err != nil {
-			log.Panicf("Cannot read taxon name taxid at line %d: %v", l, err)
-		}
-
-		name := strings.TrimSpace(record[1])
-		classname := strings.TrimSpace(record[3])
-
-		if !onlysn || classname == "scientific name" {
-			n++
-			taxonomy.AddNewName(taxid, &name, &classname)
-		}
-	}
-
-	return n
-}
-
-func loadMergedTable(reader io.Reader, taxonomy *obitax.Taxonomy) int {
-	file := csv.NewReader(reader)
-	file.Comma = '|'
-	file.Comment = '#'
-	file.TrimLeadingSpace = true
-	file.ReuseRecord = true
-
-	n := 0
-
-	for record, err := file.Read(); err == nil; record, err = file.Read() {
-		n++
-		oldtaxid, err := strconv.Atoi(strings.TrimSpace(record[0]))
-
-		if err != nil {
-			log.Panicf("Cannot read alias taxid at line %d: %v", n, err)
-		}
-		newtaxid, err := strconv.Atoi(strings.TrimSpace(record[1]))
-
-		if err != nil {
-			log.Panicf("Cannot read alias new taxid at line %d: %v", n, err)
-		}
-
-		taxonomy.AddNewAlias(newtaxid, oldtaxid)
-	}
-
-	return n
-}
-
-func LoadNCBITaxDump(directory string, onlysn bool) (*obitax.Taxonomy, error) {
-
-	taxonomy := obitax.NewTaxonomy()
-
-	//
-	// Load the Taxonomy nodes
-	//
-
-	log.Printf("Loading Taxonomy nodes\n")
-
-	nodefile, err := os.Open(path.Join(directory, "nodes.dmp"))
-	if err != nil {
-		return nil, fmt.Errorf("cannot open nodes file from '%s'",
-			directory)
-	}
-	defer nodefile.Close()
-
-	buffered := bufio.NewReader(nodefile)
-	loadNodeTable(buffered, taxonomy)
-	log.Printf("%d Taxonomy nodes read\n", taxonomy.Len())
-
-	//
-	// Load the Taxonomy nodes
-	//
-
-	log.Printf("Loading Taxon names\n")
-
-	namefile, nerr := os.Open(path.Join(directory, "names.dmp"))
-	if nerr != nil {
-		return nil, fmt.Errorf("cannot open names file from '%s'",
-			directory)
-	}
-	defer namefile.Close()
-
-	n := loadNameTable(namefile, taxonomy, onlysn)
-	log.Printf("%d taxon names read\n", n)
-
-	//
-	// Load the merged taxa
-	//
-
-	log.Printf("Loading Merged taxa\n")
-
-	aliasfile, aerr := os.Open(path.Join(directory, "merged.dmp"))
-	if aerr != nil {
-		return nil, fmt.Errorf("cannot open merged file from '%s'",
-			directory)
-	}
-	defer aliasfile.Close()
-
-	buffered = bufio.NewReader(aliasfile)
-	n = loadMergedTable(buffered, taxonomy)
-	log.Printf("%d merged taxa read\n", n)
-
-	return taxonomy, nil
-}

pkg/obiformats/ngsfilter_read.go

@@ -536,6 +536,24 @@ var library_parameter = map[string]func(library *obingslibrary.NGSLibrary, value
 	},
 }
 
+// ReadCSVNGSFilter reads an NGS filter configuration from a CSV file and returns
+// an NGSLibrary. The CSV file must include columns for 'experiment', 'sample',
+// 'sample_tag', 'forward_primer', and 'reverse_primer'. Additional columns are
+// used to annotate PCR samples.
+//
+// Parameters:
+//   - reader: an io.Reader providing the CSV input.
+//
+// Returns:
+//   - A pointer to an NGSLibrary populated with the data from the CSV file.
+//   - An error if the CSV is malformed or required columns are missing.
+//
+// The function processes both data records and parameter lines starting with
+// '@param'. Parameter lines configure various aspects of the library.
+//
+// Each row in the CSV is validated to ensure it has the correct number of columns.
+// Duplicate tag pairs for the same marker result in an error. Primer unicity is
+// checked, and any unknown parameters are logged as warnings.
 func ReadCSVNGSFilter(reader io.Reader) (*obingslibrary.NGSLibrary, error) {
 	ngsfilter := obingslibrary.MakeNGSLibrary()
 	file := csv.NewReader(reader)
@@ -576,6 +594,7 @@ func ReadCSVNGSFilter(reader io.Reader) (*obingslibrary.NGSLibrary, error) {
 	extraColumns := make([]int, 0)
 
 	for i, colName := range header {
+
 		switch colName {
 		case "experiment":
 			experimentColIndex = i
@@ -642,6 +661,8 @@ func ReadCSVNGSFilter(reader io.Reader) (*obingslibrary.NGSLibrary, error) {
 
 	}
 
+	ngsfilter.CheckPrimerUnicity()
+
 	for i := 0; i < len(params); i++ {
 		param := params[i][1]
 		if len(params[i]) < 3 {

pkg/obiformats/options.go

@@ -1,13 +1,14 @@
 package obiformats
 
 import (
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 )
 
 type __options__ struct {
 	fastseq_header_parser obiseq.SeqAnnotator
-	fastseq_header_writer func(*obiseq.BioSequence) string
+	fastseq_header_writer BioSequenceFormater
+	seqBatchFormater      FormatSeqBatch
 	with_progress_bar     bool
 	buffer_size           int
 	batch_size            int
@@ -19,6 +20,7 @@ type __options__ struct {
 	appendfile            bool
 	compressed            bool
 	skip_empty            bool
+	with_quality          bool
 	csv_id                bool
 	csv_sequence          bool
 	csv_quality           bool
@@ -44,10 +46,11 @@ func MakeOptions(setters []WithOption) Options {
 	o := __options__{
 		fastseq_header_parser: ParseGuessedFastSeqHeader,
 		fastseq_header_writer: FormatFastSeqJsonHeader,
+		seqBatchFormater:      nil,
 		with_progress_bar:     false,
 		buffer_size:           2,
-		parallel_workers:      obioptions.CLIReadParallelWorkers(),
+		parallel_workers:      obidefault.ReadParallelWorkers(),
-		batch_size:            obioptions.CLIBatchSize(),
+		batch_size:            obidefault.BatchSize(),
 		total_seq_size:        1024 * 1024 * 100, // 100 MB by default
 		no_order:              false,
 		full_file_batch:       false,
@@ -55,6 +58,7 @@ func MakeOptions(setters []WithOption) Options {
 		appendfile:            false,
 		compressed:            false,
 		skip_empty:            false,
+		with_quality:          true,
 		csv_id:                true,
 		csv_definition:        false,
 		csv_count:             false,
@@ -103,6 +107,10 @@ func (opt Options) FormatFastSeqHeader() func(*obiseq.BioSequence) string {
 	return opt.pointer.fastseq_header_writer
 }
 
+func (opt Options) SequenceFormater() FormatSeqBatch {
+	return opt.pointer.seqBatchFormater
+}
+
 func (opt Options) NoOrder() bool {
 	return opt.pointer.no_order
 }
@@ -127,6 +135,10 @@ func (opt Options) SkipEmptySequence() bool {
 	return opt.pointer.skip_empty
 }
 
+func (opt Options) ReadQualities() bool {
+	return opt.pointer.with_quality
+}
+
 func (opt Options) CSVId() bool {
 	return opt.pointer.csv_id
 }
@@ -219,8 +231,6 @@ func OptionNoOrder(no_order bool) WithOption {
 	return f
 }
 
-
-
 func OptionsCompressed(compressed bool) WithOption {
 	f := WithOption(func(opt Options) {
 		opt.pointer.compressed = compressed
@@ -237,6 +247,14 @@ func OptionsSkipEmptySequence(skip bool) WithOption {
 	return f
 }
 
+func OptionsReadQualities(read bool) WithOption {
+	f := WithOption(func(opt Options) {
+		opt.pointer.with_quality = read
+	})
+
+	return f
+}
+
 func OptionsNewFile() WithOption {
 	f := WithOption(func(opt Options) {
 		opt.pointer.appendfile = false
@@ -271,6 +289,14 @@ func OptionsFastSeqHeaderFormat(format func(*obiseq.BioSequence) string) WithOpt
 	return f
 }
 
+func OptionsSequenceFormater(formater FormatSeqBatch) WithOption {
+	f := WithOption(func(opt Options) {
+		opt.pointer.seqBatchFormater = formater
+	})
+
+	return f
+}
+
 func OptionsParallelWorkers(nworkers int) WithOption {
 	f := WithOption(func(opt Options) {
 		opt.pointer.parallel_workers = nworkers

pkg/obiformats/universal_read.go

@@ -3,6 +3,8 @@ package obiformats
 import (
 	"bufio"
 	"bytes"
+	"encoding/csv"
+	"errors"
 	"io"
 	"path"
 	"regexp"
@@ -15,6 +17,8 @@ import (
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
 
+type SequenceReader func(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, error)
+
 // OBIMimeTypeGuesser is a function that takes an io.Reader as input and guesses the MIME type of the data.
 // It uses several detectors to identify specific file formats, such as FASTA, FASTQ, ecoPCR2, GenBank, and EMBL.
 // The function reads data from the input stream and analyzes it using the mimetype library.
@@ -37,13 +41,38 @@ import (
 // - io.Reader: A modified reader with the read data.
 // - error: Any error encountered during the process.
 func OBIMimeTypeGuesser(stream io.Reader) (*mimetype.MIME, io.Reader, error) {
+	csv := func(in []byte, limit uint32) bool {
+		in = dropLastLine(in, limit)
+
+		br := bytes.NewReader(in)
+		r := csv.NewReader(br)
+		r.Comma = ','
+		r.ReuseRecord = true
+		r.LazyQuotes = true
+		r.Comment = '#'
+
+		lines := 0
+		for {
+			_, err := r.Read()
+			if errors.Is(err, io.EOF) {
+				break
+			}
+			if err != nil {
+				return false
+			}
+			lines++
+		}
+
+		return r.FieldsPerRecord > 1 && lines > 1
+	}
+
 	fastaDetector := func(raw []byte, limit uint32) bool {
 		ok, err := regexp.Match("^>[^ ]", raw)
 		return ok && err == nil
 	}
 
 	fastqDetector := func(raw []byte, limit uint32) bool {
-		ok, err := regexp.Match("^@[^ ].*\n[^ ]+\n\\+", raw)
+		ok, err := regexp.Match("^@[^ ].*\n[A-Za-z.-]+", raw)
 		return ok && err == nil
 	}
 
@@ -68,15 +97,17 @@ func OBIMimeTypeGuesser(stream io.Reader) (*mimetype.MIME, io.Reader, error) {
 	mimetype.Lookup("text/plain").Extend(ecoPCR2Detector, "text/ecopcr2", ".ecopcr")
 	mimetype.Lookup("text/plain").Extend(genbankDetector, "text/genbank", ".seq")
 	mimetype.Lookup("text/plain").Extend(emblDetector, "text/embl", ".dat")
+	mimetype.Lookup("text/plain").Extend(csv, "text/csv", ".csv")
 
 	mimetype.Lookup("application/octet-stream").Extend(fastaDetector, "text/fasta", ".fasta")
 	mimetype.Lookup("application/octet-stream").Extend(fastqDetector, "text/fastq", ".fastq")
 	mimetype.Lookup("application/octet-stream").Extend(ecoPCR2Detector, "text/ecopcr2", ".ecopcr")
 	mimetype.Lookup("application/octet-stream").Extend(genbankDetector, "text/genbank", ".seq")
 	mimetype.Lookup("application/octet-stream").Extend(emblDetector, "text/embl", ".dat")
+	mimetype.Lookup("application/octet-stream").Extend(csv, "text/csv", ".csv")
 
 	// Create a buffer to store the read data
-	buf := make([]byte, 1024*128)
+	buf := make([]byte, 1024*1024)
 	n, err := io.ReadFull(stream, buf)
 
 	if err != nil && err != io.ErrUnexpectedEOF {
@@ -85,6 +116,7 @@ func OBIMimeTypeGuesser(stream io.Reader) (*mimetype.MIME, io.Reader, error) {
 
 	// Detect the MIME type using the mimetype library
 	mimeType := mimetype.Detect(buf)
+
 	if mimeType == nil {
 		return nil, nil, err
 	}
@@ -140,15 +172,15 @@ func OBIMimeTypeGuesser(stream io.Reader) (*mimetype.MIME, io.Reader, error) {
 // - error: An error if any occurred during the reading process.
 func ReadSequencesFromFile(filename string,
 	options ...WithOption) (obiiter.IBioSequence, error) {
-	var file *Reader
+	var file *obiutils.Reader
 	var reader io.Reader
 	var err error
 
 	options = append(options, OptionsSource(obiutils.RemoveAllExt((path.Base(filename)))))
 
-	file, err = Ropen(filename)
+	file, err = obiutils.Ropen(filename)
 
-	if err == ErrNoContent {
+	if err == obiutils.ErrNoContent {
 		log.Infof("file %s is empty", filename)
 		return ReadEmptyFile(options...)
 	}
@@ -172,11 +204,11 @@ func ReadSequencesFromFile(filename string,
 	case "text/fasta":
 		return ReadFasta(reader, options...)
 	case "text/ecopcr2":
-		return ReadEcoPCR(reader, options...), nil
+		return ReadEcoPCR(reader, options...)
 	case "text/embl":
-		return ReadEMBL(reader, options...), nil
+		return ReadEMBL(reader, options...)
 	case "text/genbank":
-		return ReadGenbank(reader, options...), nil
+		return ReadGenbank(reader, options...)
 	case "text/csv":
 		return ReadCSV(reader, options...)
 	default:

pkg/obiformats/universal_write.go

@@ -45,7 +45,8 @@ func WriteSequence(iterator obiiter.IBioSequence,
 
 func WriteSequencesToStdout(iterator obiiter.IBioSequence,
 	options ...WithOption) (obiiter.IBioSequence, error) {
-	options = append(options, OptionDontCloseFile())
+	// options = append(options, OptionDontCloseFile())
+	options = append(options, OptionCloseFile())
 	return WriteSequence(iterator, os.Stdout, options...)
 }
 

pkg/obifp/uint128.go

@@ -0,0 +1,289 @@
+package obifp
+
+import (
+	"math"
+	"math/bits"
+
+	log "github.com/sirupsen/logrus"
+)
+
+type Uint128 struct {
+	w1 uint64
+	w0 uint64
+}
+
+// Zero returns a zero-valued uint128.
+//
+// No parameters.
+// Returns a Uint128 value.
+func (u Uint128) Zero() Uint128 {
+	return Uint128{w1: 0, w0: 0}
+}
+
+// MaxValue returns the maximum possible value for a Uint128.
+//
+// It returns a Uint128 value with the highest possible values for high and low fields.
+func (u Uint128) MaxValue() Uint128 {
+	return Uint128{
+		w1: math.MaxUint64,
+		w0: math.MaxUint64,
+	}
+
+}
+
+// IsZero checks if the Uint128 value is zero.
+//
+// It returns a boolean indicating whether the Uint128 value is zero.
+func (u Uint128) IsZero() bool {
+	return u.w0 == 0 && u.w1 == 0
+}
+
+// Cast a Uint128 to a Uint64.
+//
+// A Warning will be logged if an overflow occurs.
+//
+// No parameters.
+// Returns a Uint64 value.
+func (u Uint128) Uint64() Uint64 {
+	if u.w1 != 0 {
+		log.Warnf("Uint128 overflow at Uint64(%v)", u)
+	}
+	return Uint64{w0: u.w0}
+}
+
+// Uint128 cast a Uint128 to a Uint128.
+//
+// Which is a no-op.
+//
+// No parameters.
+// Returns a Uint128 value.
+func (u Uint128) Uint128() Uint128 {
+	return u
+}
+
+// Cast a Uint128 to a Uint256.
+//
+// A Warning will be logged if an overflow occurs.
+//
+// No parameters.
+// Returns a Uint256 value.
+func (u Uint128) Uint256() Uint256 {
+	return Uint256{0, 0, u.w1, u.w0}
+}
+
+func (u Uint128) Set64(v uint64) Uint128 {
+
+	return Uint128{
+		w1: 0,
+		w0: v,
+	}
+}
+
+// LeftShift performs a left shift operation on the Uint128 value by the specified number of bits.
+//
+// Parameters:
+//   - n: the number of bits to shift the Uint128 value to the left.
+//
+// Returns:
+//   - Uint128: the result of the left shift operation.
+func (u Uint128) LeftShift(n uint) Uint128 {
+	lo, carry := Uint64{w0: u.w0}.LeftShift64(n, 0)
+	hi, _ := Uint64{w0: u.w1}.LeftShift64(n, carry)
+	return Uint128{w1: hi, w0: lo}
+}
+
+// RightShift performs a right shift operation on the Uint128 value by the specified number of bits.
+//
+// Parameters:
+//   - n: the number of bits to shift the Uint128 value to the right.
+//
+// Returns:
+//   - Uint128: the result of the right shift operation.
+func (u Uint128) RightShift(n uint) Uint128 {
+	hi, carry := Uint64{w0: u.w1}.RightShift64(n, 0)
+	lo, _ := Uint64{w0: u.w0}.RightShift64(n, carry)
+	return Uint128{w1: hi, w0: lo}
+}
+
+// Add performs addition of two Uint128 values and returns the result.
+//
+// Parameters:
+//   - v: the Uint128 value to add to the receiver.
+//
+// Returns:
+//   - Uint128: the result of the addition.
+func (u Uint128) Add(v Uint128) Uint128 {
+	lo, carry := bits.Add64(u.w0, v.w0, 0)
+	hi, carry := bits.Add64(u.w1, v.w1, carry)
+	if carry != 0 {
+		log.Panicf("Uint128 overflow at Add(%v, %v)", u, v)
+	}
+	return Uint128{w1: hi, w0: lo}
+}
+
+func (u Uint128) Add64(v uint64) Uint128 {
+	lo, carry := bits.Add64(u.w0, v, 0)
+	hi, carry := bits.Add64(u.w1, 0, carry)
+	if carry != 0 {
+		log.Panicf("Uint128 overflow at Add64(%v, %v)", u, v)
+	}
+	return Uint128{w1: hi, w0: lo}
+}
+
+func (u Uint128) Sub(v Uint128) Uint128 {
+	lo, borrow := bits.Sub64(u.w0, v.w0, 0)
+	hi, borrow := bits.Sub64(u.w1, v.w1, borrow)
+	if borrow != 0 {
+		log.Panicf("Uint128 underflow at Sub(%v, %v)", u, v)
+	}
+	return Uint128{w1: hi, w0: lo}
+}
+
+func (u Uint128) Mul(v Uint128) Uint128 {
+	hi, lo := bits.Mul64(u.w0, v.w0)
+	p0, p1 := bits.Mul64(u.w1, v.w0)
+	p2, p3 := bits.Mul64(u.w0, v.w1)
+	hi, c0 := bits.Add64(hi, p1, 0)
+	hi, c1 := bits.Add64(hi, p3, c0)
+	if p0 != 0 || p2 != 0 || c1 != 0 {
+		log.Panicf("Uint128 overflow at Mul(%v, %v)", u, v)
+	}
+	return Uint128{w1: hi, w0: lo}
+}
+
+func (u Uint128) Mul64(v uint64) Uint128 {
+	hi, lo := bits.Mul64(u.w0, v)
+	p0, p1 := bits.Mul64(u.w1, v)
+	hi, c0 := bits.Add64(hi, p1, 0)
+	if p0 != 0 || c0 != 0 {
+		log.Panicf("Uint128 overflow at Mul64(%v, %v)", u, v)
+	}
+	return Uint128{w1: hi, w0: lo}
+}
+
+func (u Uint128) QuoRem(v Uint128) (q, r Uint128) {
+	if v.w1 == 0 {
+		var r64 uint64
+		q, r64 = u.QuoRem64(v.w0)
+		r = Uint128{w1: 0, w0: r64}
+	} else {
+		// generate a "trial quotient," guaranteed to be within 1 of the actual
+		// quotient, then adjust.
+		n := uint(bits.LeadingZeros64(v.w1))
+		v1 := v.LeftShift(n)
+		u1 := u.RightShift(1)
+		tq, _ := bits.Div64(u1.w1, u1.w0, v1.w1)
+		tq >>= 63 - n
+		if tq != 0 {
+			tq--
+		}
+		q = Uint128{w1: 0, w0: tq}
+		// calculate remainder using trial quotient, then adjust if remainder is
+		// greater than divisor
+		r = u.Sub(v.Mul64(tq))
+		if r.Cmp(v) >= 0 {
+			q = q.Add64(1)
+			r = r.Sub(v)
+		}
+	}
+	return
+}
+
+// QuoRem64 returns q = u/v and r = u%v.
+func (u Uint128) QuoRem64(v uint64) (q Uint128, r uint64) {
+	if u.w1 < v {
+		q.w0, r = bits.Div64(u.w1, u.w0, v)
+	} else {
+		q.w1, r = bits.Div64(0, u.w1, v)
+		q.w0, r = bits.Div64(r, u.w0, v)
+	}
+	return
+}
+
+func (u Uint128) Div(v Uint128) Uint128 {
+	q, _ := u.QuoRem(v)
+	return q
+}
+
+func (u Uint128) Div64(v uint64) Uint128 {
+	q, _ := u.QuoRem64(v)
+	return q
+}
+
+func (u Uint128) Mod(v Uint128) Uint128 {
+	_, r := u.QuoRem(v)
+	return r
+}
+
+func (u Uint128) Mod64(v uint64) uint64 {
+	_, r := u.QuoRem64(v)
+	return r
+}
+
+func (u Uint128) Cmp(v Uint128) int {
+	switch {
+	case u.w1 > v.w1:
+		return 1
+	case u.w1 < v.w1:
+		return -1
+	case u.w0 > v.w0:
+		return 1
+	case u.w0 < v.w0:
+		return -1
+	default:
+		return 0
+	}
+}
+
+func (u Uint128) Cmp64(v uint64) int {
+	switch {
+	case u.w1 > 0:
+		return 1
+	case u.w0 > v:
+		return 1
+	case u.w0 < v:
+		return -1
+	default:
+		return 0
+	}
+}
+
+func (u Uint128) Equals(v Uint128) bool {
+	return u.Cmp(v) == 0
+}
+
+func (u Uint128) LessThan(v Uint128) bool {
+	return u.Cmp(v) < 0
+}
+
+func (u Uint128) GreaterThan(v Uint128) bool {
+	return u.Cmp(v) > 0
+}
+
+func (u Uint128) LessThanOrEqual(v Uint128) bool {
+	return !u.GreaterThan(v)
+}
+
+func (u Uint128) GreaterThanOrEqual(v Uint128) bool {
+	return !u.LessThan(v)
+}
+
+func (u Uint128) And(v Uint128) Uint128 {
+	return Uint128{w1: u.w1 & v.w1, w0: u.w0 & v.w0}
+}
+
+func (u Uint128) Or(v Uint128) Uint128 {
+	return Uint128{w1: u.w1 | v.w1, w0: u.w0 | v.w0}
+}
+
+func (u Uint128) Xor(v Uint128) Uint128 {
+	return Uint128{w1: u.w1 ^ v.w1, w0: u.w0 ^ v.w0}
+}
+
+func (u Uint128) Not() Uint128 {
+	return Uint128{w1: ^u.w1, w0: ^u.w0}
+}
+
+func (u Uint128) AsUint64() uint64 {
+	return u.w0
+}

pkg/obifp/uint128_test.go

@@ -0,0 +1,250 @@
+package obifp
+
+import (
+	"math"
+	"reflect"
+
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestUint128_Add(t *testing.T) {
+	u := Uint128{w1: 1, w0: 2}
+	v := Uint128{w1: 3, w0: 4}
+	w := u.Add(v)
+	assert.Equal(t, Uint128{w1: 4, w0: 6}, w)
+
+	u = Uint128{w1: 0, w0: 0}
+	v = Uint128{w1: 0, w0: 0}
+	w = u.Add(v)
+	assert.Equal(t, Uint128{w1: 0, w0: 0}, w)
+
+	u = Uint128{w1: 0, w0: math.MaxUint64}
+	v = Uint128{w1: 0, w0: 1}
+	w = u.Add(v)
+	assert.Equal(t, Uint128{w1: 1, w0: 0}, w)
+}
+
+func TestUint128_Add64(t *testing.T) {
+	u := Uint128{w1: 1, w0: 2}
+	v := uint64(3)
+	w := u.Add64(v)
+	assert.Equal(t, Uint128{w1: 1, w0: 5}, w)
+
+	u = Uint128{w1: 0, w0: 0}
+	v = uint64(0)
+	w = u.Add64(v)
+	assert.Equal(t, Uint128{w1: 0, w0: 0}, w)
+
+	u = Uint128{w1: 0, w0: math.MaxUint64}
+	v = uint64(1)
+	w = u.Add64(v)
+	assert.Equal(t, Uint128{w1: 1, w0: 0}, w)
+}
+
+func TestUint128_Sub(t *testing.T) {
+	u := Uint128{w1: 4, w0: 6}
+	v := Uint128{w1: 3, w0: 4}
+	w := u.Sub(v)
+	assert.Equal(t, Uint128{w1: 1, w0: 2}, w)
+
+	u = Uint128{w1: 0, w0: 0}
+	v = Uint128{w1: 0, w0: 0}
+	w = u.Sub(v)
+	assert.Equal(t, Uint128{w1: 0, w0: 0}, w)
+
+	u = Uint128{w1: 1, w0: 0}
+	v = Uint128{w1: 0, w0: 1}
+	w = u.Sub(v)
+	assert.Equal(t, Uint128{w1: 0, w0: math.MaxUint64}, w)
+}
+
+func TestUint128_Mul64(t *testing.T) {
+	u := Uint128{w1: 1, w0: 2}
+	v := uint64(3)
+	w := u.Mul64(v)
+
+	if w.w1 != 3 || w.w0 != 6 {
+		t.Errorf("Mul64(%v, %v) = %v, want %v", u, v, w, Uint128{w1: 3, w0: 6})
+	}
+
+	u = Uint128{w1: 0, w0: 0}
+	v = uint64(0)
+	w = u.Mul64(v)
+
+	if w.w1 != 0 || w.w0 != 0 {
+		t.Errorf("Mul64(%v, %v) = %v, want %v", u, v, w, Uint128{w1: 0, w0: 0})
+	}
+
+	u = Uint128{w1: 0, w0: math.MaxUint64}
+	v = uint64(2)
+	w = u.Mul64(v)
+
+	if w.w1 != 1 || w.w0 != 18446744073709551614 {
+		t.Errorf("Mul64(%v, %v) = %v, want %v", u, v, w, Uint128{w1: 1, w0: 18446744073709551614})
+	}
+
+}
+
+func TestUint128_Mul(t *testing.T) {
+	tests := []struct {
+		name     string
+		u        Uint128
+		v        Uint128
+		expected Uint128
+	}{
+		{
+			name:     "simple multiplication",
+			u:        Uint128{w1: 1, w0: 2},
+			v:        Uint128{w1: 3, w0: 4},
+			expected: Uint128{w1: 10, w0: 8},
+		},
+		{
+			name:     "multiplication with overflow",
+			u:        Uint128{w1: 0, w0: math.MaxUint64},
+			v:        Uint128{w1: 0, w0: 2},
+			expected: Uint128{w1: 1, w0: 18446744073709551614},
+		},
+		{
+			name:     "multiplication with zero",
+			u:        Uint128{w1: 0, w0: 0},
+			v:        Uint128{w1: 0, w0: 0},
+			expected: Uint128{w1: 0, w0: 0},
+		},
+		{
+			name:     "multiplication with large numbers",
+			u:        Uint128{w1: 100, w0: 200},
+			v:        Uint128{w1: 300, w0: 400},
+			expected: Uint128{w1: 100000, w0: 80000},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			actual := tt.u.Mul(tt.v)
+			if !reflect.DeepEqual(actual, tt.expected) {
+				t.Errorf("Mul(%v, %v) = %v, want %v", tt.u, tt.v, actual, tt.expected)
+			}
+		})
+	}
+}
+
+func TestUint128_QuoRem(t *testing.T) {
+	u := Uint128{w1: 3, w0: 8}
+	v := Uint128{w1: 0, w0: 4}
+	q, r := u.QuoRem(v)
+	assert.Equal(t, Uint128{w1: 0, w0: 2}, q)
+	assert.Equal(t, Uint128{w1: 0, w0: 0}, r)
+}
+
+func TestUint128_QuoRem64(t *testing.T) {
+	u := Uint128{w1: 0, w0: 6}
+	v := uint64(3)
+	q, r := u.QuoRem64(v)
+	assert.Equal(t, Uint128{w1: 0, w0: 2}, q)
+	assert.Equal(t, uint64(0), r)
+}
+
+func TestUint128_Div(t *testing.T) {
+	u := Uint128{w1: 3, w0: 8}
+	v := Uint128{w1: 0, w0: 4}
+	q := u.Div(v)
+	assert.Equal(t, Uint128{w1: 0, w0: 2}, q)
+}
+
+func TestUint128_Div64(t *testing.T) {
+	u := Uint128{w1: 0, w0: 6}
+	v := uint64(3)
+	q := u.Div64(v)
+	assert.Equal(t, Uint128{w1: 0, w0: 2}, q)
+}
+
+func TestUint128_Mod(t *testing.T) {
+	u := Uint128{w1: 3, w0: 8}
+	v := Uint128{w1: 0, w0: 4}
+	r := u.Mod(v)
+	assert.Equal(t, Uint128{w1: 0, w0: 0}, r)
+}
+
+func TestUint128_Mod64(t *testing.T) {
+	u := Uint128{w1: 0, w0: 6}
+	v := uint64(3)
+	r := u.Mod64(v)
+	assert.Equal(t, uint64(0), r)
+}
+
+func TestUint128_Cmp(t *testing.T) {
+	u := Uint128{w1: 1, w0: 2}
+	v := Uint128{w1: 3, w0: 4}
+	assert.Equal(t, -1, u.Cmp(v))
+}
+
+func TestUint128_Cmp64(t *testing.T) {
+	u := Uint128{w1: 1, w0: 2}
+	v := uint64(3)
+	assert.Equal(t, -1, u.Cmp64(v))
+}
+
+func TestUint128_Equals(t *testing.T) {
+	u := Uint128{w1: 1, w0: 2}
+	v := Uint128{w1: 1, w0: 2}
+	assert.Equal(t, true, u.Equals(v))
+}
+
+func TestUint128_LessThan(t *testing.T) {
+	u := Uint128{w1: 1, w0: 2}
+	v := Uint128{w1: 3, w0: 4}
+	assert.Equal(t, true, u.LessThan(v))
+}
+
+func TestUint128_GreaterThan(t *testing.T) {
+	u := Uint128{w1: 1, w0: 2}
+	v := Uint128{w1: 3, w0: 4}
+	assert.Equal(t, false, u.GreaterThan(v))
+}
+
+func TestUint128_LessThanOrEqual(t *testing.T) {
+	u := Uint128{w1: 1, w0: 2}
+	v := Uint128{w1: 3, w0: 4}
+	assert.Equal(t, true, u.LessThanOrEqual(v))
+}
+
+func TestUint128_GreaterThanOrEqual(t *testing.T) {
+	u := Uint128{w1: 1, w0: 2}
+	v := Uint128{w1: 3, w0: 4}
+	assert.Equal(t, false, u.GreaterThanOrEqual(v))
+}
+
+func TestUint128_And(t *testing.T) {
+	u := Uint128{w1: 1, w0: 2}
+	v := Uint128{w1: 3, w0: 4}
+	w := u.And(v)
+	assert.Equal(t, Uint128{w1: 1, w0: 0}, w)
+}
+
+func TestUint128_Or(t *testing.T) {
+	u := Uint128{w1: 1, w0: 2}
+	v := Uint128{w1: 3, w0: 4}
+	w := u.Or(v)
+	assert.Equal(t, Uint128{w1: 3, w0: 6}, w)
+}
+
+func TestUint128_Xor(t *testing.T) {
+	u := Uint128{w1: 1, w0: 2}
+	v := Uint128{w1: 3, w0: 4}
+	w := u.Xor(v)
+	assert.Equal(t, Uint128{w1: 2, w0: 6}, w)
+}
+
+func TestUint128_Not(t *testing.T) {
+	u := Uint128{w1: 1, w0: 2}
+	w := u.Not()
+	assert.Equal(t, Uint128{w1: math.MaxUint64 - 1, w0: math.MaxUint64 - 2}, w)
+}
+
+func TestUint128_AsUint64(t *testing.T) {
+	u := Uint128{w1: 0, w0: 5}
+	v := u.AsUint64()
+	assert.Equal(t, uint64(5), v)
+}

pkg/obifp/uint256.go

@@ -0,0 +1,307 @@
+package obifp
+
+import (
+	"math"
+	"math/bits"
+
+	log "github.com/sirupsen/logrus"
+)
+
+type Uint256 struct {
+	w3 uint64
+	w2 uint64
+	w1 uint64
+	w0 uint64
+}
+
+// Zero returns a zero value of type Uint256.
+//
+// No parameters.
+// Returns a Uint256 value of 0.
+func (u Uint256) Zero() Uint256 {
+	return Uint256{}
+}
+
+// MaxValue returns the maximum possible value of type Uint256.
+//
+// No parameters.
+// Returns the maximum value of type Uint256.
+func (u Uint256) MaxValue() Uint256 {
+	return Uint256{
+		w3: math.MaxUint64,
+		w2: math.MaxUint64,
+		w1: math.MaxUint64,
+		w0: math.MaxUint64,
+	}
+}
+
+// IsZero checks if the Uint256 value is zero.
+//
+// No parameters.
+// Returns a boolean indicating if the value is zero.
+func (u Uint256) IsZero() bool {
+	return u == Uint256{}
+}
+
+// Cast a Uint256 to a Uint64.
+//
+// A Warning will be logged if an overflow occurs.
+//
+// No parameters.
+// Returns a Uint64 value.
+func (u Uint256) Uint64() Uint64 {
+	if u.w3 != 0 || u.w2 != 0 || u.w1 != 0 {
+		log.Warnf("Uint256 overflow at Uint64(%v)", u)
+	}
+	return Uint64{w0: u.w0}
+}
+
+// Cast a Uint256 to a Uint128.
+//
+// A Warning will be logged if an overflow occurs.
+//
+// No parameters.
+// Returns a Uint128 value.
+func (u Uint256) Uint128() Uint128 {
+	if u.w3 != 0 || u.w2 != 0 {
+		log.Warnf("Uint256 overflow at Uint128(%v)", u)
+	}
+	return Uint128{u.w1, u.w0}
+}
+
+// Cast a Uint128 to a Uint256.
+//
+// A Warning will be logged if an overflow occurs.
+//
+// No parameters.
+// Returns a Uint256 value.
+func (u Uint256) Uint256() Uint256 {
+	return u
+}
+
+func (u Uint256) Set64(v uint64) Uint256 {
+
+	return Uint256{
+		w3: 0,
+		w2: 0,
+		w1: 0,
+		w0: v,
+	}
+}
+
+func (u Uint256) LeftShift(n uint) Uint256 {
+	w0, carry := Uint64{w0: u.w0}.LeftShift64(n, 0)
+	w1, carry := Uint64{w0: u.w1}.LeftShift64(n, carry)
+	w2, carry := Uint64{w0: u.w2}.LeftShift64(n, carry)
+	w3, _ := Uint64{w0: u.w3}.LeftShift64(n, carry)
+	return Uint256{w3, w2, w1, w0}
+}
+
+func (u Uint256) RightShift(n uint) Uint256 {
+	w3, carry := Uint64{w0: u.w3}.RightShift64(n, 0)
+	w2, carry := Uint64{w0: u.w2}.RightShift64(n, carry)
+	w1, carry := Uint64{w0: u.w1}.RightShift64(n, carry)
+	w0, _ := Uint64{w0: u.w0}.RightShift64(n, carry)
+	return Uint256{w3, w2, w1, w0}
+}
+
+func (u Uint256) Cmp(v Uint256) int {
+	switch {
+	case u.w3 > v.w3:
+		return 1
+	case u.w3 < v.w3:
+		return -1
+	case u.w2 > v.w2:
+		return 1
+	case u.w2 < v.w2:
+		return -1
+	case u.w1 > v.w1:
+		return 1
+	case u.w1 < v.w1:
+		return -1
+	case u.w0 > v.w0:
+		return 1
+	case u.w0 < v.w0:
+		return -1
+	default:
+		return 0
+	}
+}
+
+// Add performs addition of two Uint256 values and returns the result.
+//
+// Parameters:
+//   - v: the Uint256 value to add to the receiver.
+//
+// Returns:
+//   - Uint256: the result of the addition.
+func (u Uint256) Add(v Uint256) Uint256 {
+	w0, carry := bits.Add64(u.w0, v.w0, 0)
+	w1, carry := bits.Add64(u.w1, v.w1, carry)
+	w2, carry := bits.Add64(u.w2, v.w2, carry)
+	w3, carry := bits.Add64(u.w3, v.w3, carry)
+	if carry != 0 {
+		log.Panicf("Uint256 overflow at Add(%v, %v)", u, v)
+	}
+	return Uint256{w3, w2, w1, w0}
+}
+
+// Sub performs subtraction of two Uint256 values and returns the result.
+//
+// Parameters:
+//   - v: the Uint256 value to subtract from the receiver.
+//
+// Returns:
+//   - Uint256: the result of the subtraction.
+func (u Uint256) Sub(v Uint256) Uint256 {
+	w0, borrow := bits.Sub64(u.w0, v.w0, 0)
+	w1, borrow := bits.Sub64(u.w1, v.w1, borrow)
+	w2, borrow := bits.Sub64(u.w2, v.w2, borrow)
+	w3, borrow := bits.Sub64(u.w3, v.w3, borrow)
+	if borrow != 0 {
+		log.Panicf("Uint256 overflow at Sub(%v, %v)", u, v)
+	}
+	return Uint256{w3, w2, w1, w0}
+}
+
+// Mul performs multiplication of two Uint256 values and returns the result.
+//
+// Parameters:
+//   - v: the Uint256 value to multiply with the receiver.
+//
+// Returns:
+//   - Uint256: the result of the multiplication.
+func (u Uint256) Mul(v Uint256) Uint256 {
+	var w0, w1, w2, w3, carry uint64
+
+	w0Low, w0High := bits.Mul64(u.w0, v.w0)
+	w1Low1, w1High1 := bits.Mul64(u.w0, v.w1)
+	w1Low2, w1High2 := bits.Mul64(u.w1, v.w0)
+	w2Low1, w2High1 := bits.Mul64(u.w0, v.w2)
+	w2Low2, w2High2 := bits.Mul64(u.w1, v.w1)
+	w2Low3, w2High3 := bits.Mul64(u.w2, v.w0)
+	w3Low1, w3High1 := bits.Mul64(u.w0, v.w3)
+	w3Low2, w3High2 := bits.Mul64(u.w1, v.w2)
+	w3Low3, w3High3 := bits.Mul64(u.w2, v.w1)
+	w3Low4, w3High4 := bits.Mul64(u.w3, v.w0)
+
+	w0 = w0Low
+
+	w1, carry = bits.Add64(w1Low1, w1Low2, 0)
+	w1, _ = bits.Add64(w1, w0High, carry)
+
+	w2, carry = bits.Add64(w2Low1, w2Low2, 0)
+	w2, carry = bits.Add64(w2, w2Low3, carry)
+	w2, carry = bits.Add64(w2, w1High1, carry)
+	w2, _ = bits.Add64(w2, w1High2, carry)
+
+	w3, carry = bits.Add64(w3Low1, w3Low2, 0)
+	w3, carry = bits.Add64(w3, w3Low3, carry)
+	w3, carry = bits.Add64(w3, w3Low4, carry)
+	w3, carry = bits.Add64(w3, w2High1, carry)
+	w3, carry = bits.Add64(w3, w2High2, carry)
+	w3, carry = bits.Add64(w3, w2High3, carry)
+
+	if w3High1 != 0 || w3High2 != 0 || w3High3 != 0 || w3High4 != 0 || carry != 0 {
+		log.Panicf("Uint256 overflow at Mul(%v, %v)", u, v)
+	}
+
+	return Uint256{w3, w2, w1, w0}
+}
+
+// Div performs division of two Uint256 values and returns the result.
+//
+// Parameters:
+//   - v: the Uint256 value to divide with the receiver.
+//
+// Returns:
+//   - Uint256: the result of the division.
+func (u Uint256) Div(v Uint256) Uint256 {
+	if v.IsZero() {
+		log.Panicf("division by zero")
+	}
+
+	if u.IsZero() || u.LessThan(v) {
+		return Uint256{}
+	}
+
+	if v.Equals(Uint256{0, 0, 0, 1}) {
+		return u // Division by 1
+	}
+
+	var q, r Uint256
+	r = u
+
+	for r.GreaterThanOrEqual(v) {
+		var t Uint256 = v
+		var m Uint256 = Uint256{0, 0, 0, 1}
+		for t.LeftShift(1).LessThanOrEqual(r) {
+			t = t.LeftShift(1)
+			m = m.LeftShift(1)
+		}
+		r = r.Sub(t)
+		q = q.Add(m)
+	}
+
+	return q
+}
+
+func (u Uint256) Equals(v Uint256) bool {
+	return u.Cmp(v) == 0
+}
+
+func (u Uint256) LessThan(v Uint256) bool {
+	return u.Cmp(v) < 0
+}
+
+func (u Uint256) GreaterThan(v Uint256) bool {
+	return u.Cmp(v) > 0
+}
+
+func (u Uint256) LessThanOrEqual(v Uint256) bool {
+	return !u.GreaterThan(v)
+}
+
+func (u Uint256) GreaterThanOrEqual(v Uint256) bool {
+	return !u.LessThan(v)
+}
+
+func (u Uint256) And(v Uint256) Uint256 {
+	return Uint256{
+		w3: u.w3 & v.w3,
+		w2: u.w2 & v.w2,
+		w1: u.w1 & v.w1,
+		w0: u.w0 & v.w0,
+	}
+}
+
+func (u Uint256) Or(v Uint256) Uint256 {
+	return Uint256{
+		w3: u.w3 | v.w3,
+		w2: u.w2 | v.w2,
+		w1: u.w1 | v.w1,
+		w0: u.w0 | v.w0,
+	}
+}
+
+func (u Uint256) Xor(v Uint256) Uint256 {
+	return Uint256{
+		w3: u.w3 ^ v.w3,
+		w2: u.w2 ^ v.w2,
+		w1: u.w1 ^ v.w1,
+		w0: u.w0 ^ v.w0,
+	}
+}
+
+func (u Uint256) Not() Uint256 {
+	return Uint256{
+		w3: ^u.w3,
+		w2: ^u.w2,
+		w1: ^u.w1,
+		w0: ^u.w0,
+	}
+}
+
+func (u Uint256) AsUint64() uint64 {
+	return u.w0
+}

pkg/obifp/uint64.go

@@ -0,0 +1,237 @@
+package obifp
+
+import (
+	"math"
+	"math/bits"
+
+	log "github.com/sirupsen/logrus"
+)
+
+type Uint64 struct {
+	w0 uint64
+}
+
+// Zero returns a zero value of type Uint64.
+//
+// No parameters.
+// Returns a Uint64 value of 0.
+func (u Uint64) Zero() Uint64 {
+	return Uint64{0}
+}
+
+// MaxValue returns the maximum possible value of type Uint64.
+//
+// No parameters.
+// Returns the maximum value of type Uint64.
+func (u Uint64) MaxValue() Uint64 {
+	return Uint64{math.MaxUint64}
+}
+
+// IsZero checks if the Uint64 value is zero.
+//
+// No parameters.
+// Returns a boolean indicating if the value is zero.
+func (u Uint64) IsZero() bool {
+	return u.w0 == 0
+}
+
+// Cast a Uint64 to a Uint64.
+//
+// Which is a no-op.
+//
+// No parameters.
+// Returns the Uint64 value itself.
+func (u Uint64) Uint64() Uint64 {
+	return u
+}
+
+// Cast a Uint64 to a Uint128.
+//
+// No parameters.
+// Returns a Uint128 value with the high field set to 0 and the low field set to the value of the Uint64.
+func (u Uint64) Uint128() Uint128 {
+	return Uint128{w1: 0, w0: u.w0}
+}
+
+// Cast a Uint64 to a Uint256.
+//
+// No parameters.
+// Returns a Uint256 value with the high fields set to 0 and the low fields set to the value of the Uint64.
+func (u Uint64) Uint256() Uint256 {
+	return Uint256{w3: 0, w2: 0, w1: 0, w0: u.w0}
+}
+
+func (u Uint64) Set64(v uint64) Uint64 {
+
+	return Uint64{
+		w0: v,
+	}
+}
+
+// LeftShift64 performs a left shift operation on the Uint64 value by n bits, with carry-in from carryIn.
+//
+// The carry-in value is used as the first bit of the shifted value.
+//
+// The function returns u << n | (carryIn & ((1 << n) - 1)).
+//
+// This is a shift left operation, lowest bits are set with the lowest bits of
+// the carry-in value instead of 0 as they would be in classical a left shift operation.
+//
+// Parameters:
+// - n: the number of bits to shift by.
+// - carryIn: the carry-in value.
+//
+// Returns:
+// - value: the result of the left shift operation.
+// - carry: the carry-out value.
+func (u Uint64) LeftShift64(n uint, carryIn uint64) (value, carry uint64) {
+	switch {
+	case n == 0:
+		return u.w0, 0
+
+	case n < 64:
+		return u.w0<<n | (carryIn & ((1 << n) - 1)), u.w0 >> (64 - n)
+
+	case n == 64:
+		return carryIn, u.w0
+
+	case n < 128:
+		return carryIn, u.w0 << (n - 64)
+
+	}
+
+	log.Warnf("Uint64 overflow at LeftShift64(%v, %v)", u, n)
+	return 0, 0
+}
+
+// RightShift64 performs a right shift operation on the Uint64 value by n bits, with carry-out to carry.
+//
+// The function returns the result of the right shift operation and the carry-out value.
+//
+// Parameters:
+// - n: the number of bits to shift by.
+//
+// Returns:
+// - value: the result of the right shift operation.
+// - carry: the carry-out value.
+func (u Uint64) RightShift64(n uint, carryIn uint64) (value, carry uint64) {
+	switch {
+	case n == 0:
+		return u.w0, 0
+
+	case n < 64:
+		return u.w0>>n | (carryIn & ^((1 << (64 - n)) - 1)), u.w0 << (64 - n)
+
+	case n == 64:
+		return carryIn, u.w0
+
+	case n < 128:
+		return carryIn, u.w0 >> (n - 64)
+	}
+
+	log.Warnf("Uint64 overflow at RightShift64(%v, %v)", u, n)
+	return 0, 0
+}
+
+func (u Uint64) Add64(v Uint64, carryIn uint64) (value, carry uint64) {
+	return bits.Add64(u.w0, v.w0, uint64(carryIn))
+}
+
+func (u Uint64) Sub64(v Uint64, carryIn uint64) (value, carry uint64) {
+	return bits.Sub64(u.w0, v.w0, uint64(carryIn))
+}
+
+func (u Uint64) Mul64(v Uint64) (value, carry uint64) {
+	return bits.Mul64(u.w0, v.w0)
+}
+
+func (u Uint64) LeftShift(n uint) Uint64 {
+	sl, _ := u.LeftShift64(n, 0)
+	return Uint64{w0: sl}
+}
+
+func (u Uint64) RightShift(n uint) Uint64 {
+	sr, _ := u.RightShift64(n, 0)
+	return Uint64{w0: sr}
+}
+
+func (u Uint64) Add(v Uint64) Uint64 {
+	value, carry := u.Add64(v, 0)
+
+	if carry != 0 {
+		log.Panicf("Uint64 overflow at Add(%v, %v)", u, v)
+	}
+
+	return Uint64{w0: value}
+}
+
+func (u Uint64) Sub(v Uint64) Uint64 {
+	value, carry := u.Sub64(v, 0)
+
+	if carry != 0 {
+		log.Panicf("Uint64 overflow at Sub(%v, %v)", u, v)
+	}
+
+	return Uint64{w0: value}
+}
+
+func (u Uint64) Mul(v Uint64) Uint64 {
+	value, carry := u.Mul64(v)
+
+	if carry != 0 {
+		log.Panicf("Uint64 overflow at Mul(%v, %v)", u, v)
+	}
+
+	return Uint64{w0: value}
+}
+
+func (u Uint64) Cmp(v Uint64) int {
+	switch {
+	case u.w0 < v.w0:
+		return -1
+	case u.w0 > v.w0:
+		return 1
+	default:
+		return 0
+	}
+}
+
+func (u Uint64) Equals(v Uint64) bool {
+	return u.Cmp(v) == 0
+}
+
+func (u Uint64) LessThan(v Uint64) bool {
+	return u.Cmp(v) < 0
+}
+
+func (u Uint64) GreaterThan(v Uint64) bool {
+	return u.Cmp(v) > 0
+}
+
+func (u Uint64) LessThanOrEqual(v Uint64) bool {
+	return !u.GreaterThan(v)
+}
+
+func (u Uint64) GreaterThanOrEqual(v Uint64) bool {
+	return !u.LessThan(v)
+}
+
+func (u Uint64) And(v Uint64) Uint64 {
+	return Uint64{w0: u.w0 & v.w0}
+}
+
+func (u Uint64) Or(v Uint64) Uint64 {
+	return Uint64{w0: u.w0 | v.w0}
+}
+
+func (u Uint64) Xor(v Uint64) Uint64 {
+	return Uint64{w0: u.w0 ^ v.w0}
+}
+
+func (u Uint64) Not() Uint64 {
+	return Uint64{w0: ^u.w0}
+}
+
+func (u Uint64) AsUint64() uint64 {
+	return u.w0
+}

pkg/obifp/unint.go

@@ -0,0 +1,41 @@
+package obifp
+
+type FPUint[T Uint64 | Uint128 | Uint256] interface {
+	Zero() T
+	Set64(v uint64) T
+
+	IsZero() bool
+	LeftShift(n uint) T
+	RightShift(n uint) T
+
+	Add(v T) T
+	Sub(v T) T
+	Mul(v T) T
+	//Div(v T) T
+
+	And(v T) T
+	Or(v T) T
+	Xor(v T) T
+	Not() T
+
+	LessThan(v T) bool
+	LessThanOrEqual(v T) bool
+	GreaterThan(v T) bool
+	GreaterThanOrEqual(v T) bool
+
+	AsUint64() uint64
+
+	Uint64 | Uint128 | Uint256
+}
+
+func ZeroUint[T FPUint[T]]() T {
+	return *new(T)
+}
+
+func OneUint[T FPUint[T]]() T {
+	return ZeroUint[T]().Set64(1)
+}
+
+func From64[T FPUint[T]](v uint64) T {
+	return ZeroUint[T]().Set64(v)
+}

pkg/obiiter/batch.go

@@ -3,51 +3,108 @@ package obiiter
 import "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 
 type BioSequenceBatch struct {
-	slice obiseq.BioSequenceSlice
+	source string
-	order int
+	slice  obiseq.BioSequenceSlice
+	order  int
 }
 
-var NilBioSequenceBatch = BioSequenceBatch{nil, -1}
+var NilBioSequenceBatch = BioSequenceBatch{"", nil, -1}
 
-func MakeBioSequenceBatch(order int,
+// MakeBioSequenceBatch creates a new BioSequenceBatch with the given source, order, and sequences.
+//
+// Parameters:
+// - source: The source of the BioSequenceBatch.
+// - order: The order of the BioSequenceBatch.
+// - sequences: The slice of BioSequence.
+//
+// Returns:
+// - BioSequenceBatch: The newly created BioSequenceBatch.
+func MakeBioSequenceBatch(
+	source string,
+	order int,
 	sequences obiseq.BioSequenceSlice) BioSequenceBatch {
 
 	return BioSequenceBatch{
-		slice: sequences,
+		source: source,
-		order: order,
+		slice:  sequences,
+		order:  order,
 	}
 }
 
+// Order returns the order of the BioSequenceBatch.
+//
+// Returns:
+// - int: The order of the BioSequenceBatch.
 func (batch BioSequenceBatch) Order() int {
 	return batch.order
 }
 
+// Source returns the source of the BioSequenceBatch.
+//
+// Returns:
+// - string: The source of the BioSequenceBatch.
+func (batch BioSequenceBatch) Source() string {
+	return batch.source
+}
+
+// Reorder updates the order of the BioSequenceBatch and returns the updated batch.
+//
+// Parameters:
+// - newOrder: The new order value to assign to the BioSequenceBatch.
+//
+// Returns:
+// - BioSequenceBatch: The updated BioSequenceBatch with the new order value.
 func (batch BioSequenceBatch) Reorder(newOrder int) BioSequenceBatch {
 	batch.order = newOrder
 	return batch
 }
 
+// Slice returns the BioSequenceSlice contained within the BioSequenceBatch.
+//
+// Returns:
+// - obiseq.BioSequenceSlice: The BioSequenceSlice contained within the BioSequenceBatch.
 func (batch BioSequenceBatch) Slice() obiseq.BioSequenceSlice {
 	return batch.slice
 }
 
+// Len returns the number of BioSequence elements in the given BioSequenceBatch.
+//
+// Parameters:
+// - batch: The BioSequenceBatch to get the length from.
+//
+// Return type:
+// - int: The number of BioSequence elements in the BioSequenceBatch.
 func (batch BioSequenceBatch) Len() int {
 	return len(batch.slice)
 }
 
+// NotEmpty returns whether the BioSequenceBatch is empty or not.
+//
+// It checks if the BioSequenceSlice contained within the BioSequenceBatch is not empty.
+//
+// Returns:
+// - bool: True if the BioSequenceBatch is not empty, false otherwise.
 func (batch BioSequenceBatch) NotEmpty() bool {
 	return batch.slice.NotEmpty()
 }
 
+// Pop0 returns and removes the first element of the BioSequenceBatch.
+//
+// It does not take any parameters.
+// It returns a pointer to a BioSequence object.
 func (batch BioSequenceBatch) Pop0() *obiseq.BioSequence {
 	return batch.slice.Pop0()
 }
 
+// IsNil checks if the BioSequenceBatch's slice is nil.
+//
+// This function takes a BioSequenceBatch as a parameter and returns a boolean value indicating whether the slice of the BioSequenceBatch is nil or not.
+//
+// Parameters:
+// - batch: The BioSequenceBatch to check for nil slice.
+//
+// Returns:
+// - bool: True if the BioSequenceBatch's slice is nil, false otherwise.
 func (batch BioSequenceBatch) IsNil() bool {
 	return batch.slice == nil
 }
-
-func (batch BioSequenceBatch) Recycle(including_seq bool) {
-	batch.slice.Recycle(including_seq)
-	batch.slice = nil
-}

pkg/obiiter/batchiterator.go

@@ -10,31 +10,12 @@ import (
 
 	log "github.com/sirupsen/logrus"
 
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 	"github.com/tevino/abool/v2"
 )
 
-var globalLocker sync.WaitGroup
-var globalLockerCounter = 0
-
-func RegisterAPipe() {
-	globalLocker.Add(1)
-	globalLockerCounter++
-	log.Debugln(globalLockerCounter, " Pipes are registered now")
-}
-
-func UnregisterPipe() {
-	globalLocker.Done()
-	globalLockerCounter--
-	log.Debugln(globalLockerCounter, "are still registered")
-}
-
-func WaitForLastPipe() {
-	globalLocker.Wait()
-}
-
 // Structure implementing an iterator over bioseq.BioSequenceBatch
 // based on a channel.
 type _IBioSequence struct {
@@ -78,7 +59,7 @@ func MakeIBioSequence() IBioSequence {
 	i.lock = &lock
 	ii := IBioSequence{&i}
 
-	RegisterAPipe()
+	obiutils.RegisterAPipe()
 
 	return ii
 }
@@ -245,7 +226,7 @@ func (iterator IBioSequence) Push(batch BioSequenceBatch) {
 
 func (iterator IBioSequence) Close() {
 	close(iterator.pointer.channel)
-	UnregisterPipe()
+	obiutils.UnregisterPipe()
 }
 
 func (iterator IBioSequence) WaitAndClose() {
@@ -424,9 +405,11 @@ func (iterator IBioSequence) Rebatch(size int) IBioSequence {
 		order := 0
 		iterator = iterator.SortBatches()
 		buffer := obiseq.MakeBioSequenceSlice()
+		source := ""
 
 		for iterator.Next() {
 			seqs := iterator.Get()
+			source = seqs.Source()
 			lc := seqs.Len()
 			remains := lc
 			i := 0
@@ -436,18 +419,17 @@ func (iterator IBioSequence) Rebatch(size int) IBioSequence {
 				remains = lc - to_push - i
 				buffer = append(buffer, seqs.Slice()[i:(i+to_push)]...)
 				if len(buffer) == size {
-					newIter.Push(MakeBioSequenceBatch(order, buffer))
+					newIter.Push(MakeBioSequenceBatch(source, order, buffer))
 					log.Debugf("Rebatch #%d pushd", order)
 					order++
 					buffer = obiseq.MakeBioSequenceSlice()
 				}
 				i += to_push
 			}
-			seqs.Recycle(false)
 		}
 		log.Debug("End of the rebatch loop")
 		if len(buffer) > 0 {
-			newIter.Push(MakeBioSequenceBatch(order, buffer))
+			newIter.Push(MakeBioSequenceBatch(source, order, buffer))
 			log.Debugf("Final Rebatch #%d pushd", order)
 		}
 
@@ -462,6 +444,39 @@ func (iterator IBioSequence) Rebatch(size int) IBioSequence {
 	return newIter
 }
 
+func (iterator IBioSequence) FilterEmpty() IBioSequence {
+
+	newIter := MakeIBioSequence()
+
+	newIter.Add(1)
+
+	go func() {
+		newIter.WaitAndClose()
+	}()
+
+	go func() {
+		order := 0
+		iterator = iterator.SortBatches()
+
+		for iterator.Next() {
+			seqs := iterator.Get()
+			lc := seqs.Len()
+
+			if lc > 0 {
+				newIter.Push(seqs.Reorder(order))
+				order++
+			}
+		}
+
+		newIter.Done()
+	}()
+
+	if iterator.IsPaired() {
+		newIter.MarkAsPaired()
+	}
+
+	return newIter
+}
 func (iterator IBioSequence) Recycle() {
 
 	log.Debugln("Start recycling of Bioseq objects")
@@ -472,7 +487,6 @@ func (iterator IBioSequence) Recycle() {
 		o := batch.Order()
 		log.Debugln("Recycling batch #", o)
 		recycled += batch.Len()
-		batch.Recycle(true)
 		log.Debugln("Batch #", o, " recycled")
 	}
 	log.Debugf("End of the recycling of %d Bioseq objects", recycled)
@@ -480,8 +494,7 @@ func (iterator IBioSequence) Recycle() {
 
 func (iterator IBioSequence) Consume() {
 	for iterator.Next() {
-		batch := iterator.Get()
+		iterator.Get()
-		batch.Recycle(false)
 	}
 }
 
@@ -499,7 +512,6 @@ func (iterator IBioSequence) Count(recycle bool) (int, int, int) {
 			reads += seq.Count()
 			nucleotides += seq.Len()
 		}
-		batch.Recycle(recycle)
 	}
 	log.Debugf("End of the counting of %d Bioseq objects", variants)
 	return variants, reads, nucleotides
@@ -526,12 +538,14 @@ func (iterator IBioSequence) DivideOn(predicate obiseq.SequencePredicate,
 		trueOrder := 0
 		falseOrder := 0
 		iterator = iterator.SortBatches()
+		source := ""
 
 		trueSlice := obiseq.MakeBioSequenceSlice()
 		falseSlice := obiseq.MakeBioSequenceSlice()
 
 		for iterator.Next() {
 			seqs := iterator.Get()
+			source = seqs.Source()
 			for _, s := range seqs.slice {
 				if predicate(s) {
 					trueSlice = append(trueSlice, s)
@@ -540,26 +554,25 @@ func (iterator IBioSequence) DivideOn(predicate obiseq.SequencePredicate,
 				}
 
 				if len(trueSlice) == size {
-					trueIter.Push(MakeBioSequenceBatch(trueOrder, trueSlice))
+					trueIter.Push(MakeBioSequenceBatch(source, trueOrder, trueSlice))
 					trueOrder++
 					trueSlice = obiseq.MakeBioSequenceSlice()
 				}
 
 				if len(falseSlice) == size {
-					falseIter.Push(MakeBioSequenceBatch(falseOrder, falseSlice))
+					falseIter.Push(MakeBioSequenceBatch(source, falseOrder, falseSlice))
 					falseOrder++
 					falseSlice = obiseq.MakeBioSequenceSlice()
 				}
 			}
-			seqs.Recycle(false)
 		}
 
 		if len(trueSlice) > 0 {
-			trueIter.Push(MakeBioSequenceBatch(trueOrder, trueSlice))
+			trueIter.Push(MakeBioSequenceBatch(source, trueOrder, trueSlice))
 		}
 
 		if len(falseSlice) > 0 {
-			falseIter.Push(MakeBioSequenceBatch(falseOrder, falseSlice))
+			falseIter.Push(MakeBioSequenceBatch(source, falseOrder, falseSlice))
 		}
 
 		trueIter.Done()
@@ -578,7 +591,7 @@ func (iterator IBioSequence) DivideOn(predicate obiseq.SequencePredicate,
 // A function that takes a predicate and a batch of sequences and returns a filtered batch of sequences.
 func (iterator IBioSequence) FilterOn(predicate obiseq.SequencePredicate,
 	size int, sizes ...int) IBioSequence {
-	nworkers := obioptions.CLIReadParallelWorkers()
+	nworkers := obidefault.ReadParallelWorkers()
 
 	if len(sizes) > 0 {
 		nworkers = sizes[0]
@@ -630,7 +643,7 @@ func (iterator IBioSequence) FilterOn(predicate obiseq.SequencePredicate,
 
 func (iterator IBioSequence) FilterAnd(predicate obiseq.SequencePredicate,
 	size int, sizes ...int) IBioSequence {
-	nworkers := obioptions.CLIReadParallelWorkers()
+	nworkers := obidefault.ReadParallelWorkers()
 
 	if len(sizes) > 0 {
 		nworkers = sizes[0]
@@ -686,17 +699,21 @@ func (iterator IBioSequence) FilterAnd(predicate obiseq.SequencePredicate,
 
 // Load all sequences availables from an IBioSequenceBatch iterator into
 // a large obiseq.BioSequenceSlice.
-func (iterator IBioSequence) Load() obiseq.BioSequenceSlice {
+func (iterator IBioSequence) Load() (string, obiseq.BioSequenceSlice) {
+
+	chunk := obiseq.MakeBioSequenceSlice()
+	source := ""
 
-	chunck := obiseq.MakeBioSequenceSlice()
 	for iterator.Next() {
 		b := iterator.Get()
+		if source == "" {
+			source = b.Source()
+		}
 		log.Debugf("append %d sequences", b.Len())
-		chunck = append(chunck, b.Slice()...)
+		chunk = append(chunk, b.Slice()...)
-		b.Recycle(false)
 	}
 
-	return chunck
+	return source, chunk
 }
 
 // CompleteFileIterator generates a new iterator for reading a complete file.
@@ -718,10 +735,10 @@ func (iterator IBioSequence) CompleteFileIterator() IBioSequence {
 	}()
 
 	go func() {
-		slice := iterator.Load()
+		source, slice := iterator.Load()
 		log.Printf("A batch of  %d sequence is read", len(slice))
 		if len(slice) > 0 {
-			newIter.Push(MakeBioSequenceBatch(0, slice))
+			newIter.Push(MakeBioSequenceBatch(source, 0, slice))
 		}
 		newIter.Done()
 	}()
@@ -735,7 +752,7 @@ func (iterator IBioSequence) CompleteFileIterator() IBioSequence {
 
 // It takes a slice of BioSequence objects, and returns an iterator that will return batches of
 // BioSequence objects
-func IBatchOver(data obiseq.BioSequenceSlice,
+func IBatchOver(source string, data obiseq.BioSequenceSlice,
 	size int, sizes ...int) IBioSequence {
 
 	newIter := MakeIBioSequence()
@@ -755,7 +772,7 @@ func IBatchOver(data obiseq.BioSequenceSlice,
 			if next > ldata {
 				next = ldata
 			}
-			newIter.Push(MakeBioSequenceBatch(batchid, data[i:next]))
+			newIter.Push(MakeBioSequenceBatch(source, batchid, data[i:next]))
 			batchid++
 		}
 

pkg/obiiter/distribute.go

@@ -4,9 +4,25 @@ import (
 	"fmt"
 	"sync"
 
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 )
 
+// IDistribute represents a distribution mechanism for biosequences.
+// It manages the outputs of biosequences, provides a channel for
+// new data notifications, and maintains a classifier for sequence
+// classification. It is designed to facilitate the distribution
+// of biosequences to various processing components.
+//
+// Fields:
+//   - outputs: A map that associates integer keys with corresponding
+//     biosequence outputs (IBioSequence).
+//   - news: A channel that sends notifications of new data available
+//     for processing, represented by integer identifiers.
+//   - classifier: A pointer to a BioSequenceClassifier used to classify
+//     the biosequences during distribution.
+//   - lock: A mutex for synchronizing access to the outputs and other
+//     shared resources to ensure thread safety.
 type IDistribute struct {
 	outputs    map[int]IBioSequence
 	news       chan int
@@ -26,16 +42,39 @@ func (dist *IDistribute) Outputs(key int) (IBioSequence, error) {
 	return iter, nil
 }
 
+// News returns a channel that provides notifications of new data
+// available for processing. The channel sends integer identifiers
+// representing the new data.
 func (dist *IDistribute) News() chan int {
 	return dist.news
 }
 
+// Classifier returns a pointer to the BioSequenceClassifier
+// associated with the distribution mechanism. This classifier
+// is used to classify biosequences during the distribution process.
 func (dist *IDistribute) Classifier() *obiseq.BioSequenceClassifier {
 	return dist.classifier
 }
 
+// Distribute organizes the biosequences from the iterator into batches
+// based on the provided classifier and batch sizes. It returns an
+// IDistribute instance that manages the distribution of the sequences.
+//
+// Parameters:
+//   - class: A pointer to a BioSequenceClassifier used to classify
+//     the biosequences during distribution.
+//   - sizes: Optional integer values specifying the batch size. If
+//     no sizes are provided, a default batch size of 5000 is used.
+//
+// Returns:
+// An IDistribute instance that contains the outputs of the
+// classified biosequences, a channel for new data notifications,
+// and the classifier used for distribution. The method operates
+// asynchronously, processing the sequences in separate goroutines.
+// It ensures that the outputs are closed and cleaned up once
+// processing is complete.
 func (iterator IBioSequence) Distribute(class *obiseq.BioSequenceClassifier, sizes ...int) IDistribute {
-	batchsize := 5000
+	batchsize := obidefault.BatchSize()
 
 	outputs := make(map[int]IBioSequence, 100)
 	slices := make(map[int]*obiseq.BioSequenceSlice, 100)
@@ -61,9 +100,12 @@ func (iterator IBioSequence) Distribute(class *obiseq.BioSequenceClassifier, siz
 
 	go func() {
 		iterator = iterator.SortBatches()
+		source := ""
 
 		for iterator.Next() {
 			seqs := iterator.Get()
+			source = seqs.Source()
+
 			for _, s := range seqs.Slice() {
 				key := class.Code(s)
 				slice, ok := slices[key]
@@ -84,18 +126,17 @@ func (iterator IBioSequence) Distribute(class *obiseq.BioSequenceClassifier, siz
 				*slice = append(*slice, s)
 
 				if len(*slice) == batchsize {
-					outputs[key].Push(MakeBioSequenceBatch(orders[key], *slice))
+					outputs[key].Push(MakeBioSequenceBatch(source, orders[key], *slice))
 					orders[key]++
 					s := obiseq.MakeBioSequenceSlice()
 					slices[key] = &s
 				}
 			}
-			seqs.Recycle(false)
 		}
 
 		for key, slice := range slices {
 			if len(*slice) > 0 {
-				outputs[key].Push(MakeBioSequenceBatch(orders[key], *slice))
+				outputs[key].Push(MakeBioSequenceBatch(source, orders[key], *slice))
 			}
 		}
 

pkg/obiiter/extract_taxonomy.go

@@ -0,0 +1,18 @@
+package obiiter
+
+import "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax"
+
+func (iterator *IBioSequence) ExtractTaxonomy() (taxonomy *obitax.Taxonomy, err error) {
+
+	for iterator.Next() {
+		slice := iterator.Get().Slice()
+
+		taxonomy, err = slice.ExtractTaxonomy(taxonomy)
+
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}

pkg/obiiter/fragment.go

@@ -19,10 +19,12 @@ func IFragments(minsize, length, overlap, size, nworkers int) Pipeable {
 			newiter.WaitAndClose()
 		}()
 
-		f := func(iterator IBioSequence, id int) {
+		f := func(iterator IBioSequence) {
+			source := ""
 			for iterator.Next() {
 				news := obiseq.MakeBioSequenceSlice()
 				sl := iterator.Get()
+				source = sl.Source()
 				for _, s := range sl.Slice() {
 
 					if s.Len() <= minsize {
@@ -52,8 +54,7 @@ func IFragments(minsize, length, overlap, size, nworkers int) Pipeable {
 						s.Recycle()
 					}
 				} // End of the slice loop
-				newiter.Push(MakeBioSequenceBatch(sl.Order(), news))
+				newiter.Push(MakeBioSequenceBatch(source, sl.Order(), news))
-				sl.Recycle(false)
 			} // End of the iterator loop
 
 			// if len(news) > 0 {
@@ -65,9 +66,9 @@ func IFragments(minsize, length, overlap, size, nworkers int) Pipeable {
 		}
 
 		for i := 1; i < nworkers; i++ {
-			go f(iterator.Split(), i)
+			go f(iterator.Split())
 		}
-		go f(iterator, 0)
+		go f(iterator)
 
 		return newiter.SortBatches().Rebatch(size)
 	}