A first version of obimicroasm...

modified:   pkg/obiformats/universal_read.go
	modified:   pkg/obioptions/version.go
	modified:   pkg/obiseq/taxonomy_methods.go

.gitignore

@@ -118,3 +118,18 @@ doc/book/wolf_data/Release-253/ncbitaxo/readme.txt
 doc/book/results/toto.tasta
 sample/.DS_Store
 GO
+ncbitaxo/citations.dmp
+ncbitaxo/delnodes.dmp
+ncbitaxo/division.dmp
+ncbitaxo/gc.prt
+ncbitaxo/gencode.dmp
+ncbitaxo/merged.dmp
+ncbitaxo/names.dmp
+ncbitaxo/nodes.dmp
+ncbitaxo/readme.txt
+template.16S
+xxx.gz
+*.sav
+*.old
+ncbitaxo.tgz
+*.csv

Release-notes.md

@@ -2,7 +2,46 @@
 
 ## Latest changes
 
-## August 2nd, 2024. Release 4.3.0
+### Breaking changes
+
+- In `obimultiplex`, the short version of the **--tag-list** option used to specify the list 
+  of tags and primers to be used for the demultiplexing has been changed from `-t` to `-s`.
+
+- The command `obifind` is now renamed `obitaxonomy`.
+
+- The **--taxdump** option used to specify the path to the taxdump containing the NCBI taxonomy
+  has been renamed to **--taxonomy**.
+
+### Bug fixes
+
+- In `obipairing`, correct the stats `seq_a_single` and `seq_b_single` when
+  on right alignment mode
+
+- Not really a bug but the memory impact of `obiuniq` has been reduced by reducing
+  the batch size and not reading the qualities from the fastq files as `obiuniq`
+  is producing only fasta output without qualities.
+
+### New features
+
+-   `obitoaxonomy` a new **--dump|D** option allows for dumping a sub-taxonomy.
+  
+-   Taxonomy dump can now be provided as a four-columns CSV file to the **--taxonomy**
+    option.
+
+-   NCBI Taxonomy dump does not need to be uncompressed and unarchived anymore. The
+    path of the tar and gziped dump file can be directly specified using the
+    **--taxonomy** option.
+
+-   Most of the time obitools identify automatically sequence file format. But
+    it fails sometimes. Two new option **--fasta** and **--fastq** are added to
+    allow the processing of the rare fasta and fastq files not recognized.
+    
+-   In `obiscript`, adds new methods to the Lua sequence object:
+    - `md5_string()`:  returning the MD5 check sum as an hexadecimal string,
+	- `subsequence(from,to)`: allows to extract a subsequence on a 0 based 
+            coordinate system, upper bound expluded like in go.
+	- `reverse_complement`: returning a sequence object corresponding to the reverse complement
+            of the current sequence.
 
 ### Change of git repositiory
 

cmd/obitools/obiannotate/main.go

@@ -3,13 +3,11 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiannotate"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
 
 func main() {
@@ -37,15 +35,11 @@ func main() {
 	_, args := optionParser(os.Args)
 
 	sequences, err := obiconvert.CLIReadBioSequences(args...)
-
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
 
 	annotator := obiannotate.CLIAnnotationPipeline()
 	obiconvert.CLIWriteBioSequences(sequences.Pipe(annotator), true)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obiclean/main.go

@@ -3,11 +3,9 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiclean"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 )
@@ -18,16 +16,12 @@ func main() {
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
 
 	cleaned := obiclean.CLIOBIClean(fs)
 
 	obiconvert.CLIWriteBioSequences(cleaned, true)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obicleandb/main.go

@@ -3,33 +3,28 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obicleandb"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 )
 
 func main() {
-	obioptions.SetBatchSize(10)
+	obidefault.SetBatchSize(10)
 
 	optionParser := obioptions.GenerateOptionParser(obicleandb.OptionSet)
 
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
 
 	cleaned := obicleandb.ICleanDB(fs)
 
 	toconsume, _ := obiconvert.CLIWriteBioSequences(cleaned, false)
 	toconsume.Consume()
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 }

cmd/obitools/obicomplement/main.go

@@ -3,11 +3,9 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 )
@@ -18,15 +16,11 @@ func main() {
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
 
 	comp := fs.MakeIWorker(obiseq.ReverseComplementWorker(true), true)
 	obiconvert.CLIWriteBioSequences(comp, true)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obiconsensus/main.go

@@ -3,11 +3,9 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconsensus"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 )
@@ -18,16 +16,12 @@ func main() {
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
 
 	cleaned := obiconsensus.CLIOBIMinion(fs)
 
 	obiconvert.CLIWriteBioSequences(cleaned, true)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obiconvert/main.go

@@ -3,31 +3,26 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 )
 
 func main() {
-	obioptions.SetStrictReadWorker(2)
-	obioptions.SetStrictWriteWorker(2)
+	obidefault.SetStrictReadWorker(2)
+	obidefault.SetStrictWriteWorker(2)
 
 	optionParser := obioptions.GenerateOptionParser(obiconvert.OptionSet)
 
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
 
 	obiconvert.CLIWriteBioSequences(fs, true)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obicount/main.go

@@ -4,8 +4,7 @@ import (
 	"fmt"
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obicount"
 
@@ -35,17 +34,13 @@ func main() {
 
 	_, args := optionParser(os.Args)
 
-	obioptions.SetStrictReadWorker(min(4, obioptions.CLIParallelWorkers()))
+	obidefault.SetStrictReadWorker(min(4, obidefault.ParallelWorkers()))
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
 
 	nvariant, nread, nsymbol := fs.Count(true)
 
-	fmt.Print("entites,n\n")
+	fmt.Print("entities,n\n")
 
 	if obicount.CLIIsPrintingVariantCount() {
 		fmt.Printf("variants,%d\n", nvariant)

cmd/obitools/obicsv/main.go

@@ -3,12 +3,10 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obicsv"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
 
 func main() {
@@ -17,14 +15,10 @@ func main() {
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
 
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
+	obicsv.CLIWriteSequenceCSV(fs, true)
 
-	obicsv.CLIWriteCSV(fs, true)
-
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obidemerge/main.go

@@ -3,34 +3,29 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obidemerge"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 )
 
 func main() {
-	obioptions.SetStrictReadWorker(2)
-	obioptions.SetStrictWriteWorker(2)
+	obidefault.SetStrictReadWorker(2)
+	obidefault.SetStrictWriteWorker(2)
 
 	optionParser := obioptions.GenerateOptionParser(obidemerge.OptionSet)
 
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
 
 	demerged := obidemerge.CLIDemergeSequences(fs)
 
 	obiconvert.CLIWriteBioSequences(demerged, true)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obidistribute/main.go

@@ -3,11 +3,9 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obidistribute"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 )
@@ -18,14 +16,10 @@ func main() {
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
 
 	obidistribute.CLIDistributeSequence(fs)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obifind/main.go

@@ -1,68 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"os"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obifind"
-)
-
-func main() {
-	optionParser := obioptions.GenerateOptionParser(obifind.OptionSet)
-
-	_, args := optionParser(os.Args)
-
-	//prof, _ := os.Create("obifind.prof")
-	//pprof.StartCPUProfile(prof)
-
-	restrictions, err := obifind.ITaxonRestrictions()
-	if err != nil {
-		fmt.Printf("%+v", err)
-	}
-
-	switch {
-	case obifind.CLIRequestsPathForTaxid() >= 0:
-		taxonomy, err := obifind.CLILoadSelectedTaxonomy()
-		if err != nil {
-			fmt.Printf("%+v", err)
-		}
-
-		taxon, err := taxonomy.Taxon(obifind.CLIRequestsPathForTaxid())
-
-		if err != nil {
-			fmt.Printf("%+v", err)
-		}
-
-		s, err := taxon.Path()
-
-		if err != nil {
-			fmt.Printf("%+v", err)
-		}
-
-		obifind.TaxonWriter(s.Iterator(),
-			fmt.Sprintf("path:%d", taxon.Taxid()))
-
-	case len(args) == 0:
-		taxonomy, err := obifind.CLILoadSelectedTaxonomy()
-		if err != nil {
-			fmt.Printf("%+v", err)
-		}
-
-		obifind.TaxonWriter(restrictions(taxonomy.Iterator()), "")
-
-	default:
-		matcher, err := obifind.ITaxonNameMatcher()
-
-		if err != nil {
-			fmt.Printf("%+v", err)
-		}
-
-		for _, pattern := range args {
-			s := restrictions(matcher(pattern))
-			obifind.TaxonWriter(s, pattern)
-		}
-	}
-
-	//pprof.StopCPUProfile()
-}

cmd/obitools/obigrep/main.go

@@ -3,13 +3,11 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obigrep"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
 
 func main() {
@@ -37,13 +35,10 @@ func main() {
 	_, args := optionParser(os.Args)
 
 	sequences, err := obiconvert.CLIReadBioSequences(args...)
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
 
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 	selected := obigrep.CLIFilterSequence(sequences)
 	obiconvert.CLIWriteBioSequences(selected, true)
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obijoin/main.go

@@ -3,34 +3,29 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obijoin"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 )
 
 func main() {
-	obioptions.SetStrictReadWorker(2)
-	obioptions.SetStrictWriteWorker(2)
+	obidefault.SetStrictReadWorker(2)
+	obidefault.SetStrictWriteWorker(2)
 
 	optionParser := obioptions.GenerateOptionParser(obijoin.OptionSet)
 
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
 
 	joined := obijoin.CLIJoinSequences(fs)
 
 	obiconvert.CLIWriteBioSequences(joined, true)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obikmermatch/main.go

@@ -0,0 +1,51 @@
+package main
+
+import (
+	"os"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obikmersim"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+)
+
+func main() {
+
+	defer obiseq.LogBioSeqStatus()
+
+	// go tool pprof -http=":8000" ./obipairing ./cpu.pprof
+	// f, err := os.Create("cpu.pprof")
+	// if err != nil {
+	// 	log.Fatal(err)
+	// }
+	// pprof.StartCPUProfile(f)
+	// defer pprof.StopCPUProfile()
+
+	// go tool trace cpu.trace
+	// ftrace, err := os.Create("cpu.trace")
+	// if err != nil {
+	// 	log.Fatal(err)
+	// }
+	// trace.Start(ftrace)
+	// defer trace.Stop()
+
+	optionParser := obioptions.GenerateOptionParser(obikmersim.MatchOptionSet)
+
+	_, args := optionParser(os.Args)
+
+	var err error
+	sequences := obiiter.NilIBioSequence
+
+	if !obikmersim.CLISelf() {
+		sequences, err = obiconvert.CLIReadBioSequences(args...)
+	}
+
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
+
+	selected := obikmersim.CLIAlignSequences(sequences)
+	obiconvert.CLIWriteBioSequences(selected, true)
+	obiutils.WaitForLastPipe()
+
+}

cmd/obitools/obikmersimcount/main.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"log"
+	"os"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obikmersim"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+)
+
+func main() {
+
+	defer obiseq.LogBioSeqStatus()
+
+	// go tool pprof -http=":8000" ./obipairing ./cpu.pprof
+	// f, err := os.Create("cpu.pprof")
+	// if err != nil {
+	// 	log.Fatal(err)
+	// }
+	// pprof.StartCPUProfile(f)
+	// defer pprof.StopCPUProfile()
+
+	// go tool trace cpu.trace
+	// ftrace, err := os.Create("cpu.trace")
+	// if err != nil {
+	// 	log.Fatal(err)
+	// }
+	// trace.Start(ftrace)
+	// defer trace.Stop()
+
+	optionParser := obioptions.GenerateOptionParser(obikmersim.CountOptionSet)
+
+	_, args := optionParser(os.Args)
+
+	var err error
+	sequences := obiiter.NilIBioSequence
+
+	if !obikmersim.CLISelf() {
+		sequences, err = obiconvert.CLIReadBioSequences(args...)
+	}
+
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
+
+	counted := obikmersim.CLILookForSharedKmers(sequences)
+	topull, err := obiconvert.CLIWriteBioSequences(counted, false)
+
+	if err != nil {
+		log.Panic(err)
+	}
+
+	topull.Consume()
+
+	obiutils.WaitForLastPipe()
+
+}

cmd/obitools/obilandmark/main.go

@@ -3,11 +3,9 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obilandmark"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 )
@@ -18,15 +16,11 @@ func main() {
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
 
 	indexed := obilandmark.CLISelectLandmarkSequences(fs)
 
 	obiconvert.CLIWriteBioSequences(indexed, true)
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obimatrix/main.go

@@ -4,8 +4,6 @@ import (
 	"fmt"
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
@@ -39,11 +37,7 @@ func main() {
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
 
 	matrix := obimatrix.IMatrix(fs)
 

cmd/obitools/obimicroasm/main.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"os"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiformats"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obimicroasm"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+)
+
+func main() {
+
+	// go tool pprof -http=":8000" ./obipairing ./cpu.pprof
+	// f, err := os.Create("cpu.pprof")
+	// if err != nil {
+	// 	log.Fatal(err)
+	// }
+	// pprof.StartCPUProfile(f)
+	// defer pprof.StopCPUProfile()
+
+	// go tool trace cpu.trace
+	// ftrace, err := os.Create("cpu.trace")
+	// if err != nil {
+	// 	log.Fatal(err)
+	// }
+	// trace.Start(ftrace)
+	// defer trace.Stop()
+
+	optionParser := obioptions.GenerateOptionParser(obimicroasm.OptionSet)
+
+	optionParser(os.Args)
+
+	obidefault.SetStrictReadWorker(2)
+	obidefault.SetStrictWriteWorker(2)
+
+	seq := obimicroasm.CLIAssemblePCR()
+
+	println(obiformats.FormatFasta(seq, obiformats.FormatFastSeqJsonHeader))
+	obiutils.WaitForLastPipe()
+}

cmd/obitools/obimicrosat/main.go

@@ -0,0 +1,44 @@
+package main
+
+import (
+	"os"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obimicrosat"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+)
+
+func main() {
+
+	defer obiseq.LogBioSeqStatus()
+
+	// go tool pprof -http=":8000" ./obipairing ./cpu.pprof
+	// f, err := os.Create("cpu.pprof")
+	// if err != nil {
+	// 	log.Fatal(err)
+	// }
+	// pprof.StartCPUProfile(f)
+	// defer pprof.StopCPUProfile()
+
+	// go tool trace cpu.trace
+	// ftrace, err := os.Create("cpu.trace")
+	// if err != nil {
+	// 	log.Fatal(err)
+	// }
+	// trace.Start(ftrace)
+	// defer trace.Stop()
+
+	optionParser := obioptions.GenerateOptionParser(obimicrosat.OptionSet)
+
+	_, args := optionParser(os.Args)
+
+	sequences, err := obiconvert.CLIReadBioSequences(args...)
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
+
+	selected := obimicrosat.CLIAnnotateMicrosat(sequences)
+	obiconvert.CLIWriteBioSequences(selected, true)
+	obiutils.WaitForLastPipe()
+
+}

cmd/obitools/obimultiplex/main.go

@@ -6,10 +6,10 @@ import (
 
 	log "github.com/sirupsen/logrus"
 
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obimultiplex"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
 
 func main() {
@@ -43,14 +43,11 @@ func main() {
 	}
 
 	sequences, err := obiconvert.CLIReadBioSequences(args...)
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
 
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 	amplicons, _ := obimultiplex.IExtractBarcode(sequences)
 	obiconvert.CLIWriteBioSequences(amplicons, true)
 	amplicons.Wait()
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obipairing/main.go

@@ -5,10 +5,11 @@ import (
 
 	log "github.com/sirupsen/logrus"
 
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obipairing"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
 
 func main() {
@@ -33,8 +34,8 @@ func main() {
 
 	optionParser(os.Args)
 
-	obioptions.SetStrictReadWorker(2)
-	obioptions.SetStrictWriteWorker(2)
+	obidefault.SetStrictReadWorker(2)
+	obidefault.SetStrictWriteWorker(2)
 	pairs, err := obipairing.CLIPairedSequence()
 
 	if err != nil {
@@ -51,10 +52,10 @@ func main() {
 		obipairing.CLIFastMode(),
 		obipairing.CLIFastRelativeScore(),
 		obipairing.CLIWithStats(),
-		obioptions.CLIParallelWorkers(),
+		obidefault.ParallelWorkers(),
 	)
 
 	obiconvert.CLIWriteBioSequences(paired, true)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 }

cmd/obitools/obipcr/main.go

@@ -3,12 +3,11 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obipcr"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
 
 func main() {
@@ -25,24 +24,20 @@ func main() {
 	// trace.Start(ftrace)
 	// defer trace.Stop()
 
-	obioptions.SetWorkerPerCore(2)
-	obioptions.SetReadWorkerPerCore(0.5)
-	obioptions.SetParallelFilesRead(obioptions.CLIParallelWorkers() / 4)
-	obioptions.SetBatchSize(10)
+	obidefault.SetWorkerPerCore(2)
+	obidefault.SetReadWorkerPerCore(0.5)
+	obidefault.SetParallelFilesRead(obidefault.ParallelWorkers() / 4)
+	obidefault.SetBatchSize(10)
 
 	optionParser := obioptions.GenerateOptionParser(obipcr.OptionSet)
 
 	_, args := optionParser(os.Args)
 
 	sequences, err := obiconvert.CLIReadBioSequences(args...)
-
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
 
 	amplicons, _ := obipcr.CLIPCR(sequences)
 	obiconvert.CLIWriteBioSequences(amplicons, true)
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obireffamidx/main.go

@@ -3,11 +3,9 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obirefidx"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 )
@@ -18,15 +16,11 @@ func main() {
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
 
 	indexed := obirefidx.IndexFamilyDB(fs)
 
 	obiconvert.CLIWriteBioSequences(indexed, true)
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obirefidx/main.go

@@ -3,11 +3,9 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obirefidx"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 )
@@ -18,14 +16,11 @@ func main() {
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
 
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
 	indexed := obirefidx.IndexReferenceDB(fs)
 
 	obiconvert.CLIWriteBioSequences(indexed, true)
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obiscript/main.go

@@ -4,13 +4,11 @@ import (
 	"fmt"
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiscript"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
 
 func main() {
@@ -43,15 +41,11 @@ func main() {
 	}
 
 	sequences, err := obiconvert.CLIReadBioSequences(args...)
-
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
 
 	annotator := obiscript.CLIScriptPipeline()
 	obiconvert.CLIWriteBioSequences(sequences.Pipe(annotator), true)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obisplit/main.go

@@ -4,13 +4,11 @@ import (
 	"fmt"
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obisplit"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
 
 func main() {
@@ -43,15 +41,11 @@ func main() {
 	}
 
 	sequences, err := obiconvert.CLIReadBioSequences(args...)
-
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
 
 	annotator := obisplit.CLISlitPipeline()
 	obiconvert.CLIWriteBioSequences(sequences.Pipe(annotator), true)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/obitools/obisummary/main.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"os"
 
-	log "github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v3"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
@@ -39,11 +38,7 @@ func main() {
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
-
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
 
 	summary := obisummary.ISummary(fs, obisummary.CLIMapSummary())
 

cmd/obitools/obitag/main.go

@@ -6,10 +6,12 @@ import (
 
 	log "github.com/sirupsen/logrus"
 
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obifind"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obitag"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 )
@@ -32,29 +34,40 @@ func main() {
 	// trace.Start(ftrace)
 	// defer trace.Stop()
 
-	obioptions.SetWorkerPerCore(2)
-	obioptions.SetStrictReadWorker(1)
-	obioptions.SetStrictWriteWorker(1)
-	obioptions.SetBatchSize(10)
+	obidefault.SetWorkerPerCore(2)
+	obidefault.SetStrictReadWorker(1)
+	obidefault.SetStrictWriteWorker(1)
+	obidefault.SetBatchSize(10)
 
 	optionParser := obioptions.GenerateOptionParser(obitag.OptionSet)
 
 	_, args := optionParser(os.Args)
 
 	fs, err := obiconvert.CLIReadBioSequences(args...)
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
 
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
-
-	taxo, error := obifind.CLILoadSelectedTaxonomy()
-	if error != nil {
-		log.Panicln(error)
-	}
+	taxo := obitax.DefaultTaxonomy()
 
 	references := obitag.CLIRefDB()
 
+	if references == nil {
+		log.Panicln("No loaded reference database")
+	}
+
+	if taxo == nil {
+		taxo, err = references.ExtractTaxonomy(nil)
+
+		if err != nil {
+			log.Fatalf("No taxonomy specified or extractable from reference database: %v", err)
+		}
+
+		taxo.SetAsDefault()
+	}
+
+	if taxo == nil {
+		log.Panicln("No loaded taxonomy")
+	}
+
 	var identified obiiter.IBioSequence
 
 	if obitag.CLIGeometricMode() {
@@ -64,7 +77,7 @@ func main() {
 	}
 
 	obiconvert.CLIWriteBioSequences(identified, true)
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 	obitag.CLISaveRefetenceDB(references)
 

cmd/obitools/obitag2/main.go

@@ -1,65 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"os"
-
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obifind"
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obitag"
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obitag2"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
-)
-
-func main() {
-
-	// go tool pprof -http=":8000" ./build/obitag ./cpu.pprof
-	// f, err := os.Create("cpu.pprof")
-	// if err != nil {
-	// 	log.Fatal(err)
-	// }
-	// pprof.StartCPUProfile(f)
-	// defer pprof.StopCPUProfile()
-
-	// go tool trace cpu.trace
-	// ftrace, err := os.Create("cpu.trace")
-	// if err != nil {
-	// 	log.Fatal(err)
-	// }
-	// trace.Start(ftrace)
-	// defer trace.Stop()
-
-	obioptions.SetWorkerPerCore(2)
-	obioptions.SetStrictReadWorker(1)
-	obioptions.SetStrictWriteWorker(1)
-	obioptions.SetBatchSize(10)
-
-	optionParser := obioptions.GenerateOptionParser(obitag.OptionSet)
-
-	_, args := optionParser(os.Args)
-
-	fs, err := obiconvert.CLIReadBioSequences(args...)
-
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
-
-	taxo, error := obifind.CLILoadSelectedTaxonomy()
-	if error != nil {
-		log.Panicln(error)
-	}
-
-	references := obitag.CLIRefDB()
-
-	identified := obitag2.CLIAssignTaxonomy(fs, references, taxo)
-
-	obiconvert.CLIWriteBioSequences(identified, true)
-	obiiter.WaitForLastPipe()
-
-	fmt.Println("")
-}

cmd/obitools/obitagpcr/main.go

@@ -5,11 +5,12 @@ import (
 
 	log "github.com/sirupsen/logrus"
 
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obipairing"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obitagpcr"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
 
 func main() {
@@ -30,7 +31,7 @@ func main() {
 	// trace.Start(ftrace)
 	// defer trace.Stop()
 
-	obioptions.SetWorkerPerCore(1)
+	obidefault.SetWorkerPerCore(1)
 
 	optionParser := obioptions.GenerateOptionParser(obitagpcr.OptionSet)
 
@@ -54,5 +55,5 @@ func main() {
 
 	obiconvert.CLIWriteBioSequences(paired, true)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 }

cmd/obitools/obitaxonomy/main.go

@@ -0,0 +1,106 @@
+package main
+
+import (
+	"os"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obitaxonomy"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+
+	log "github.com/sirupsen/logrus"
+)
+
+func main() {
+	optionParser := obioptions.GenerateOptionParser(obitaxonomy.OptionSet)
+
+	_, args := optionParser(os.Args)
+
+	var iterator *obitax.ITaxon
+
+	switch {
+	case obitaxonomy.CLIDownloadNCBI():
+		err := obitaxonomy.CLIDownloadNCBITaxdump()
+		if err != nil {
+			log.Errorf("Cannot download NCBI taxonomy: %s", err.Error())
+			os.Exit(1)
+		}
+
+		os.Exit(0)
+
+	case obitaxonomy.CLIExtractTaxonomy():
+		iter, err := obiconvert.CLIReadBioSequences(args...)
+
+		if err != nil {
+			log.Fatalf("Cannot extract taxonomy: %v", err)
+		}
+
+		taxonomy, err := iter.ExtractTaxonomy()
+
+		if err != nil {
+			log.Fatalf("Cannot extract taxonomy: %v", err)
+		}
+
+		taxonomy.SetAsDefault()
+
+		log.Infof("Number of extracted taxa: %d", taxonomy.Len())
+		iterator = taxonomy.AsTaxonSet().Sort().Iterator()
+
+	case obitaxonomy.CLIDumpSubtaxonomy():
+		iterator = obitaxonomy.CLISubTaxonomyIterator()
+
+	case obitaxonomy.CLIRequestsPathForTaxid() != "NA":
+
+		taxon, isAlias, err := obitax.DefaultTaxonomy().Taxon(obitaxonomy.CLIRequestsPathForTaxid())
+
+		if err != nil {
+			log.Fatalf("Cannot identify the requested taxon: %s (%v)",
+				obitaxonomy.CLIRequestsPathForTaxid(), err)
+		}
+
+		if isAlias {
+			if obidefault.FailOnTaxonomy() {
+				log.Fatalf("Taxon %s is an alias for %s", taxon.String(), taxon.Parent().String())
+			}
+		}
+
+		s := taxon.Path()
+
+		if s == nil {
+			log.Fatalf("Cannot extract taxonomic path describing %s", taxon.String())
+		}
+
+		iterator = s.Iterator()
+
+		if obitaxonomy.CLIWithQuery() {
+			iterator = iterator.AddMetadata("query", taxon.String())
+		}
+
+	case len(args) == 0:
+		iterator = obitax.DefaultTaxonomy().Iterator()
+	default:
+		iters := make([]*obitax.ITaxon, len(args))
+
+		for i, pat := range args {
+			ii := obitax.DefaultTaxonomy().IFilterOnName(pat, obitaxonomy.CLIFixedPattern(), true)
+			if obitaxonomy.CLIWithQuery() {
+				ii = ii.AddMetadata("query", pat)
+			}
+			iters[i] = ii
+		}
+
+		iterator = iters[0]
+
+		if len(iters) > 1 {
+			iterator = iterator.Concat(iters[1:]...)
+		}
+	}
+
+	iterator = obitaxonomy.CLITaxonRestrictions(iterator)
+	obitaxonomy.CLICSVTaxaWriter(iterator, true)
+
+	obiutils.WaitForLastPipe()
+
+}

cmd/obitools/obiuniq/main.go

@@ -3,13 +3,12 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiuniq"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
 
 func main() {
@@ -32,20 +31,18 @@ func main() {
 	// trace.Start(ftrace)
 	// defer trace.Stop()
 
+	obidefault.SetBatchSize(10)
+	obidefault.SetReadQualities(false)
 	optionParser := obioptions.GenerateOptionParser(obiuniq.OptionSet)
 
 	_, args := optionParser(os.Args)
 
 	sequences, err := obiconvert.CLIReadBioSequences(args...)
-
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
+	obiconvert.OpenSequenceDataErrorMessage(args, err)
 
 	unique := obiuniq.CLIUnique(sequences)
 	obiconvert.CLIWriteBioSequences(unique, true)
 
-	obiiter.WaitForLastPipe()
+	obiutils.WaitForLastPipe()
 
 }

cmd/test/main.go

@@ -3,36 +3,14 @@ package main
 import (
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
 
 func main() {
-	optionParser := obioptions.GenerateOptionParser(obiconvert.OptionSet)
 
-	_, args := optionParser(os.Args)
-
-	fs, err := obiconvert.CLIReadBioSequences(args...)
-
-	if err != nil {
-		log.Errorf("Cannot open file (%v)", err)
-		os.Exit(1)
-	}
-
-	frags := obiiter.IFragments(
-		1000,
-		100,
-		10,
-		100,
-		obioptions.CLIParallelWorkers(),
-	)
-
-	obiconvert.CLIWriteBioSequences(fs.Pipe(frags), true)
-
-	obiiter.WaitForLastPipe()
+	obitax.DetectTaxonomyFormat(os.Args[1])
+	println(obiutils.RemoveAllExt("toto/tutu/test.txt"))
+	println(obiutils.Basename("toto/tutu/test.txt"))
 
 }

go.mod

@@ -1,19 +1,20 @@
 module git.metabarcoding.org/obitools/obitools4/obitools4
 
-go 1.22.1
+go 1.23.1
 
 require (
 	github.com/DavidGamba/go-getoptions v0.28.0
 	github.com/PaesslerAG/gval v1.2.2
 	github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df
 	github.com/chen3feng/stl4go v0.1.1
+	github.com/dlclark/regexp2 v1.11.4
 	github.com/goccy/go-json v0.10.3
 	github.com/klauspost/pgzip v1.2.6
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
 	github.com/rrethy/ahocorasick v1.0.0
 	github.com/schollz/progressbar/v3 v3.13.1
 	github.com/sirupsen/logrus v1.9.3
-	github.com/stretchr/testify v1.8.4
+	github.com/stretchr/testify v1.10.0
 	github.com/tevino/abool/v2 v2.1.0
 	github.com/yuin/gopher-lua v1.1.1
 	golang.org/x/exp v0.0.0-20231006140011-7918f672742d
@@ -23,20 +24,18 @@ require (
 )
 
 require (
-	github.com/bytedance/sonic v1.11.9 // indirect
-	github.com/bytedance/sonic/loader v0.1.1 // indirect
-	github.com/cloudwego/base64x v0.1.4 // indirect
-	github.com/cloudwego/iasm v0.2.0 // indirect
+	github.com/Clever/csvlint v0.3.0 // indirect
+	github.com/TuftsBCB/io v0.0.0-20140121014543-22b94e9b23f9 // indirect
+	github.com/buger/jsonparser v1.1.1 // indirect
+	github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/ef-ds/deque/v2 v2.0.2 // indirect
 	github.com/goombaio/orderedmap v0.0.0-20180924084748-ba921b7e2419 // indirect
-	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
 	github.com/kr/pretty v0.3.0 // indirect
 	github.com/kr/text v0.2.0 // indirect
-	github.com/montanaflynn/stats v0.7.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rogpeppe/go-internal v1.6.1 // indirect
-	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
-	golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect
+	go.etcd.io/bbolt v1.4.0 // indirect
 )
 
 require (
@@ -50,7 +49,7 @@ require (
 	github.com/shopspring/decimal v1.3.1 // indirect
 	github.com/ulikunitz/xz v0.5.11
 	golang.org/x/net v0.17.0 // indirect
-	golang.org/x/sys v0.17.0 // indirect
+	golang.org/x/sys v0.29.0 // indirect
 	golang.org/x/term v0.13.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c
 )

go.sum

@@ -1,32 +1,34 @@
+github.com/Clever/csvlint v0.3.0 h1:58WEFXWy+i0fCbxTXscR2QwYESRuAUFjEGLgZs6j2iU=
+github.com/Clever/csvlint v0.3.0/go.mod h1:+wLRuW/bI8NhpRoeyUBxqKsK35OhvgJhXHSWdKp5XJU=
 github.com/DavidGamba/go-getoptions v0.28.0 h1:18wgEvfZdrlfIhVDGEBO3Dl0fkOyXqXLa0tLMCKxM1c=
 github.com/DavidGamba/go-getoptions v0.28.0/go.mod h1:zE97E3PR9P3BI/HKyNYgdMlYxodcuiC6W68KIgeYT84=
 github.com/PaesslerAG/gval v1.2.2 h1:Y7iBzhgE09IGTt5QgGQ2IdaYYYOU134YGHBThD+wm9E=
 github.com/PaesslerAG/gval v1.2.2/go.mod h1:XRFLwvmkTEdYziLdaCeCa5ImcGVrfQbeNUbVR+C6xac=
 github.com/PaesslerAG/jsonpath v0.1.0 h1:gADYeifvlqK3R3i2cR5B4DGgxLXIPb3TRTH1mGi0jPI=
 github.com/PaesslerAG/jsonpath v0.1.0/go.mod h1:4BzmtoM/PI8fPO4aQGIusjGxGir2BzcV0grWtFzq1Y8=
+github.com/TuftsBCB/io v0.0.0-20140121014543-22b94e9b23f9 h1:Zc1/GNsUpgZR9qm1EmRSKrnOHA7CCd0bIzGdq0cREN0=
+github.com/TuftsBCB/io v0.0.0-20140121014543-22b94e9b23f9/go.mod h1:PZyV4WA3NpqtezSY0h6E6NARAmdDm0qwrydveOyR5Gc=
 github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df h1:GSoSVRLoBaFpOOds6QyY1L8AX7uoY+Ln3BHc22W40X0=
 github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df/go.mod h1:hiVxq5OP2bUGBRNS3Z/bt/reCLFNbdcST6gISi1fiOM=
-github.com/bytedance/sonic v1.11.9 h1:LFHENlIY/SLzDWverzdOvgMztTxcfcF+cqNsz9pK5zg=
-github.com/bytedance/sonic v1.11.9/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4=
-github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM=
-github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
+github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
+github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
 github.com/chen3feng/stl4go v0.1.1 h1:0L1+mDw7pomftKDruM23f1mA7miavOj6C6MZeadzN2Q=
 github.com/chen3feng/stl4go v0.1.1/go.mod h1:5ml3psLgETJjRJnMbPE+JiHLrCpt+Ajc2weeTECXzWU=
-github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y=
-github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
-github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg=
-github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo=
+github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
 github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q=
 github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo=
 github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
+github.com/ef-ds/deque/v2 v2.0.2 h1:GQtDK1boBMu/qsNbSLQsqzwNptaioxZI39X3UxT5ALA=
+github.com/ef-ds/deque/v2 v2.0.2/go.mod h1:hoZy4VooWLhRT4uS+sSCilfgBQUNptJU2FGqr08a5sc=
 github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
 github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
-github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
-github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
 github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
 github.com/goombaio/orderedmap v0.0.0-20180924084748-ba921b7e2419 h1:SajEQ6tktpF9SRIuzbiPOX9AEZZ53Bvw0k9Mzrts8Lg=
@@ -37,13 +39,9 @@ github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1
 github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
 github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
-github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE=
 github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
-github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
-github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU=
 github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
-github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
@@ -58,8 +56,6 @@ github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZ
 github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
-github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
-github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -78,26 +74,22 @@ github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFR
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/tevino/abool/v2 v2.1.0 h1:7w+Vf9f/5gmKT4m4qkayb33/92M+Um45F2BkHOR+L/c=
 github.com/tevino/abool/v2 v2.1.0/go.mod h1:+Lmlqk6bHDWHqN1cbxqhwEAwMPXgc8I1SDEamtseuXY=
-github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
-github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
 github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
 github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
 github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
 github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
-golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU=
-golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
+go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk=
+go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk=
 golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
 golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
 golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
@@ -107,6 +99,8 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
+golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
 golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
 golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
@@ -120,8 +114,6 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=
-rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
 scientificgo.org/special v0.0.0 h1:P6WJkECo6tgtvZAEfNXl+KEB9ReAatjKAeX8U07mjSc=
 scientificgo.org/special v0.0.0/go.mod h1:LoGVh9tS431RLTJo7gFlYDKFWq44cEb7QqL+M0EKtZU=
 scientificgo.org/testutil v0.0.0 h1:y356DHRo0tAz9zIFmxlhZoKDlHPHaWW/DCm9k3PhIMA=

go.work

@@ -1,3 +1,3 @@
-go 1.22.1
+go 1.23.1
 
 use .

go.work.sum

@@ -2,12 +2,9 @@ git.sr.ht/~sbinet/gg v0.3.1 h1:LNhjNn8DerC8f9DHLz6lS0YYul/b602DUxDgGkd/Aik=
 git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc=
 github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b h1:slYM766cy2nI3BwyRiyQj/Ud48djTMtMebDqepE95rw=
 github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGWcpt8ov532z81sp/kMMUG485J2InIOyADM=
-github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
 github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/logex v1.2.0 h1:+eqR0HfOetur4tgnC8ftU5imRnhi4te+BadWS95c5AM=
-github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8=
-github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/chzyer/test v0.0.0-20210722231415-061457976a23 h1:dZ0/VyGgQdVGAss6Ju0dt5P0QltE0SFY5Woh6hbIfiQ=
@@ -29,16 +26,21 @@ github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
 github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2 h1:rcanfLhLDA8nozr/K289V1zcntHr3V+SHlXwzz1ZI2g=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 h1:qGQQKEcAR99REcMpsXCp3lJ03zYT1PkRd3kQGPn9GVg=
 github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE=
 github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
 github.com/smallnest/goroutine v1.1.1/go.mod h1:Fp8f6ZReubfdj0m4+NcUnW4IsAqKa+Pnrv9opEiD43E=
+github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
+github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
 github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE=
+go.etcd.io/gofail v0.2.0/go.mod h1:nL3ILMGfkXTekKI3clMBNazKnjUZjYLKmBHzsVAnC1o=
 golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
 golang.org/x/image v0.6.0 h1:bR8b5okrPI3g/gyZakLZHeWxAR8Dn5CyxXv1hLH5g/4=
@@ -46,6 +48,7 @@ golang.org/x/image v0.6.0/go.mod h1:MXLdDR43H7cDJq5GEGXEVeeNhPgi+YYEQ2pC1byI1x0=
 golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY=
 golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
+golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc=

pkg/obialign/backtracking.go

@@ -12,7 +12,7 @@ func _Backtracking(pathMatrix []int, lseqA, lseqB int, path *[]int) []int {
 	cp := cap(*path)
 	(*path) = slices.Grow((*path), needed)
 	if cp < cap(*path) {
-		log.Infof("Resized path from %d to %d\n", cp, cap(*path))
+		log.Debugf("Resized path from %d to %d\n", cp, cap(*path))
 	}
 	p := cap(*path)
 	*path = (*path)[:p]

pkg/obialign/locatepattern.go

@@ -1,30 +1,73 @@
 package obialign
 
+import log "github.com/sirupsen/logrus"
+
+// buffIndex converts a pair of coordinates (i, j) into a linear index in a matrix
+// of size width x width. The coordinates are (-1)-indexed, and the linear index
+// is 0-indexed as well. The function first adds 1 to both coordinates to make
+// sure the (-1,-1) coordinate is at position 0 in the matrix, and then computes
+// the linear index by multiplying the first coordinate by the width and adding
+// the second coordinate.
 func buffIndex(i, j, width int) int {
 	return (i+1)*width + (j + 1)
 }
-func LocatePattern(pattern, sequence []byte) (int, int, int) {
+
+// LocatePattern is a function to locate a pattern in a sequence.
+//
+// It uses a dynamic programming approach to build a matrix of scores.
+// The score at each cell is the maximum of the score of the cell
+// above it (representing a deletion), the score of the cell to its
+// left (representing an insertion), and the score of the cell
+// diagonally above it (representing a match).
+//
+// The score of a match is 0 if the two characters are the same,
+// and -1 if they are different.
+//
+// The function returns the start and end positions of the best
+// match, as well as the number of errors in the best match.
+func LocatePattern(id string, pattern, sequence []byte) (int, int, int) {
+
+	if len(pattern) >= len(sequence) {
+		log.Panicf("Sequence %s:Pattern %s must be shorter than sequence %s", id, pattern, sequence)
+	}
+
+	// Pattern spreads over the columns
+	// Sequence spreads over the rows
 	width := len(pattern) + 1
 	buffsize := (len(pattern) + 1) * (len(sequence) + 1)
 	buffer := make([]int, buffsize)
+
+	// The path matrix keeps track of the best path through the matrix
+	//  0 : indicate the diagonal path
+	//  1 : indicate the up path
+	// -1 : indicate the left path
 	path := make([]int, buffsize)
 
+	// Initialize the first row of the matrix
 	for j := 0; j < len(pattern); j++ {
 		idx := buffIndex(-1, j, width)
 		buffer[idx] = -j - 1
 		path[idx] = -1
 	}
 
+	// Initialize the first column of the matrix
+	// Alignment is endgap free so first column = 0
+	// to allow primer to shift freely along the sequence
 	for i := -1; i < len(sequence); i++ {
 		idx := buffIndex(i, -1, width)
 		buffer[idx] = 0
 		path[idx] = +1
 	}
 
+	// Fills the matrix except the last column
+	// where gaps must be free too.
 	path[0] = 0
 	jmax := len(pattern) - 1
 	for i := 0; i < len(sequence); i++ {
 		for j := 0; j < jmax; j++ {
+
+			// Mismatch score = -1
+			// Match score = 0
 			match := -1
 			if _samenuc(pattern[j], sequence[i]) {
 				match = 0
@@ -33,6 +76,8 @@ func LocatePattern(pattern, sequence []byte) (int, int, int) {
 			idx := buffIndex(i, j, width)
 
 			diag := buffer[buffIndex(i-1, j-1, width)] + match
+
+			// Each gap cost -1
 			left := buffer[buffIndex(i, j-1, width)] - 1
 			up := buffer[buffIndex(i-1, j, width)] - 1
 
@@ -51,9 +96,12 @@ func LocatePattern(pattern, sequence []byte) (int, int, int) {
 		}
 	}
 
+	// Fills the last column considering the free up gap
 	for i := 0; i < len(sequence); i++ {
 		idx := buffIndex(i, jmax, width)
 
+		// Mismatch score = -1
+		// Match score = 0
 		match := -1
 		if _samenuc(pattern[jmax], sequence[i]) {
 			match = 0
@@ -65,6 +113,7 @@ func LocatePattern(pattern, sequence []byte) (int, int, int) {
 
 		score := max(diag, up, left)
 		buffer[idx] = score
+
 		switch {
 		case score == left:
 			path[idx] = -1
@@ -75,11 +124,13 @@ func LocatePattern(pattern, sequence []byte) (int, int, int) {
 		}
 	}
 
+	// Bactracking of the aligment
+
 	i := len(sequence) - 1
 	j := jmax
 	end := -1
 	lali := 0
-	for i > -1 && j > 0 {
+	for j > 0 { // C'était i > -1 && j > 0
 		lali++
 		switch path[buffIndex(i, j, width)] {
 		case 0:
@@ -100,5 +151,9 @@ func LocatePattern(pattern, sequence []byte) (int, int, int) {
 		}
 
 	}
+
+	// log.Warnf("from : %d to: %d error: %d match: %v",
+	// 	i, end+1, -buffer[buffIndex(len(sequence)-1, len(pattern)-1, width)],
+	// 	string(sequence[i:(end+1)]))
 	return i, end + 1, -buffer[buffIndex(len(sequence)-1, len(pattern)-1, width)]
 }

pkg/obialign/pairedendalign.go

@@ -1,6 +1,8 @@
 package obialign
 
 import (
+	"log"
+
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obikmer"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 )
@@ -313,6 +315,105 @@ func _FillMatrixPeRightAlign(seqA, qualA, seqB, qualB []byte, gap, scale float64
 
 }
 
+// Gaps at the beginning and at the end of seqA are free
+// With seqA spanning over lines and seqB over columns
+//
+// SeqA must be the longer sequence. If that constraint is not
+// respected, the function will panic.
+//
+// TO BE FINISHED
+//   - First column gap = 0
+//   - Last column gaps = 0
+//
+// Paths are encoded :
+//   - 0  :  for diagonal
+//   - -1 :  for top
+//   - +1 :  for left
+func _FillMatrixPeCenterAlign(seqA, qualA, seqB, qualB []byte, gap, scale float64,
+	scoreMatrix, pathMatrix *[]int) int {
+
+	la := len(seqA)
+	lb := len(seqB)
+
+	if len(seqA) < len(seqB) {
+		log.Panicf("len(seqA) < len(seqB) : %d < %d", len(seqA), len(seqB))
+	}
+
+	// The actual gap score is the gap score times the mismatch between
+	// two bases with a score of 40
+	gapPenalty := int(scale*gap*float64(_NucScorePartMatchMismatch[40][40]) + 0.5)
+
+	needed := (la + 1) * (lb + 1)
+
+	if needed > cap(*scoreMatrix) {
+		*scoreMatrix = make([]int, needed)
+	}
+
+	if needed > cap(*pathMatrix) {
+		*pathMatrix = make([]int, needed)
+	}
+
+	*scoreMatrix = (*scoreMatrix)[:needed]
+	*pathMatrix = (*pathMatrix)[:needed]
+
+	// Sets the first position of the matrix with 0 score
+	_SetMatrices(scoreMatrix, pathMatrix, la, -1, -1, 0, 0)
+
+	// Fills the first column with score 0
+	for i := 0; i < la; i++ {
+		_SetMatrices(scoreMatrix, pathMatrix, la, i, -1, 0, -1)
+	}
+
+	// la1 := la - 1 // Except the last line (gaps are free on it)
+	lb1 := lb - 1 // Except the last column (gaps are free on it)
+
+	for j := 0; j < lb1; j++ {
+
+		// Fill the first line with scores corresponding to a set of gaps
+		_SetMatrices(scoreMatrix, pathMatrix, la, -1, j, (j+1)*gapPenalty, 1)
+
+		for i := 0; i < la; i++ {
+			left, diag, top := _GetMatrixFrom(scoreMatrix, la, i, j)
+			// log.Infof("LA: i : %d j : %d left : %d diag : %d top : %d\n", i, j, left, diag, top)
+
+			diag += _PairingScorePeAlign(seqA[i], qualA[i], seqB[j], qualB[j], scale)
+			left += gapPenalty
+			top += gapPenalty
+
+			switch {
+			case diag >= left && diag >= top:
+				_SetMatrices(scoreMatrix, pathMatrix, la, i, j, diag, 0)
+			case left >= diag && left >= top:
+				_SetMatrices(scoreMatrix, pathMatrix, la, i, j, left, +1)
+			default:
+				_SetMatrices(scoreMatrix, pathMatrix, la, i, j, top, -1)
+			}
+			// log.Infof("LA: i : %d j : %d left : %d diag : %d top : %d [%d]\n", i, j, left, diag, top, _GetMatrix(scoreMatrix, la, i, j))
+		}
+
+	}
+
+	for i := 0; i < la; i++ {
+		left, diag, top := _GetMatrixFrom(scoreMatrix, la, i, lb1)
+		// log.Infof("LA: i : %d j : %d left : %d diag : %d top : %d\n", i, j, left, diag, top)
+
+		diag += _PairingScorePeAlign(seqA[i], qualA[i], seqB[lb1], qualB[lb1], scale)
+		left += gapPenalty
+
+		switch {
+		case diag >= left && diag >= top:
+			_SetMatrices(scoreMatrix, pathMatrix, la, i, lb1, diag, 0)
+		case left >= diag && left >= top:
+			_SetMatrices(scoreMatrix, pathMatrix, la, i, lb1, left, +1)
+		default:
+			_SetMatrices(scoreMatrix, pathMatrix, la, i, lb1, top, -1)
+		}
+		// log.Infof("LA: i : %d j : %d left : %d diag : %d top : %d [%d]\n", i, j, left, diag, top, _GetMatrix(scoreMatrix, la, i, j))
+	}
+
+	return _GetMatrix(scoreMatrix, la, la-1, lb1)
+}
+
 func PELeftAlign(seqA, seqB *obiseq.BioSequence, gap, scale float64,
 	arena PEAlignArena) (int, []int) {
 
@@ -359,9 +460,33 @@ func PERightAlign(seqA, seqB *obiseq.BioSequence, gap, scale float64,
 	return score, path
 }
 
+func PECenterAlign(seqA, seqB *obiseq.BioSequence, gap, scale float64,
+	arena PEAlignArena) (int, []int) {
+
+	if !_InitializedDnaScore {
+		_InitDNAScoreMatrix()
+	}
+
+	if arena.pointer == nil {
+		arena = MakePEAlignArena(seqA.Len(), seqB.Len())
+	}
+
+	score := _FillMatrixPeCenterAlign(seqA.Sequence(), seqA.Qualities(),
+		seqB.Sequence(), seqB.Qualities(), gap, scale,
+		&arena.pointer.scoreMatrix,
+		&arena.pointer.pathMatrix)
+
+	path := _Backtracking(arena.pointer.pathMatrix,
+		seqA.Len(), seqB.Len(),
+		&arena.pointer.path)
+
+	return score, path
+}
+
 func PEAlign(seqA, seqB *obiseq.BioSequence,
 	gap, scale float64, fastAlign bool, delta int, fastScoreRel bool,
-	arena PEAlignArena, shift_buff *map[int]int) (int, []int, int, int, float64) {
+	arena PEAlignArena, shift_buff *map[int]int) (bool, int, []int, int, int, float64) {
+	var isLeftAlign bool
 	var score, shift int
 	var startA, startB int
 	var partLen, over int
@@ -412,6 +537,7 @@ func PEAlign(seqA, seqB *obiseq.BioSequence,
 				rawSeqB = seqB.Sequence()[0:partLen]
 				qualSeqB = seqB.Qualities()[0:partLen]
 				extra3 = seqB.Len() - partLen
+				isLeftAlign = true
 				score = _FillMatrixPeLeftAlign(
 					rawSeqA, qualSeqA, rawSeqB, qualSeqB, gap, scale,
 					&arena.pointer.scoreMatrix,
@@ -433,7 +559,7 @@ func PEAlign(seqA, seqB *obiseq.BioSequence,
 				rawSeqA = seqA.Sequence()[:partLen]
 				qualSeqA = seqA.Qualities()[:partLen]
 				extra3 = partLen - seqA.Len()
-
+				isLeftAlign = false
 				score = _FillMatrixPeRightAlign(
 					rawSeqA, qualSeqA, rawSeqB, qualSeqB, gap, scale,
 					&arena.pointer.scoreMatrix,
@@ -457,6 +583,7 @@ func PEAlign(seqA, seqB *obiseq.BioSequence,
 				qualSeqB = seqB.Qualities()[0:partLen]
 				extra3 = seqB.Len() - partLen
 				score = 0
+				isLeftAlign = true
 			} else {
 				startA = 0
 				startB = -shift
@@ -465,6 +592,7 @@ func PEAlign(seqA, seqB *obiseq.BioSequence,
 				partLen = len(qualSeqB)
 				extra3 = partLen - seqA.Len()
 				qualSeqA = seqA.Qualities()[:partLen]
+				isLeftAlign = false
 			}
 			score = 0
 			for i, qualA := range qualSeqA {
@@ -501,6 +629,8 @@ func PEAlign(seqA, seqB *obiseq.BioSequence,
 			len(rawSeqA), len(rawSeqB),
 			&(arena.pointer.path))
 
+		isLeftAlign = false
+
 		scoreL := _FillMatrixPeLeftAlign(
 			rawSeqA, qualSeqA, rawSeqB, qualSeqB, gap, scale,
 			&arena.pointer.scoreMatrix,
@@ -510,9 +640,10 @@ func PEAlign(seqA, seqB *obiseq.BioSequence,
 			path = _Backtracking(arena.pointer.pathMatrix,
 				len(rawSeqA), len(rawSeqB),
 				&(arena.pointer.path))
+			isLeftAlign = true
 		}
 
 	}
 
-	return score, path, fastCount, over, fastScore
+	return isLeftAlign, score, path, fastCount, over, fastScore
 }

pkg/obialign/readalign.go

@@ -0,0 +1,154 @@
+package obialign
+
+import (
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obikmer"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+)
+
+func ReadAlign(seqA, seqB *obiseq.BioSequence,
+	gap, scale float64, delta int, fastScoreRel bool,
+	arena PEAlignArena, shift_buff *map[int]int) (int, []int, int, int, float64, bool) {
+	var score, shift int
+	var startA, startB int
+	var partLen, over int
+	var rawSeqA, qualSeqA []byte
+	var rawSeqB, qualSeqB []byte
+	var extra5, extra3 int
+
+	var path []int
+
+	if !_InitializedDnaScore {
+		_InitDNAScoreMatrix()
+	}
+
+	fastCount := -1
+	fastScore := -1.0
+
+	directAlignment := true
+
+	index := obikmer.Index4mer(seqA,
+		&arena.pointer.fastIndex,
+		&arena.pointer.fastBuffer)
+
+	shift, fastCount, fastScore = obikmer.FastShiftFourMer(index, shift_buff, seqA.Len(), seqB, fastScoreRel, nil)
+
+	seqBR := seqB.ReverseComplement(false)
+	shiftR, fastCountR, fastScoreR := obikmer.FastShiftFourMer(index, shift_buff, seqA.Len(), seqBR, fastScoreRel, nil)
+
+	if fastCount < fastCountR {
+		shift = shiftR
+		fastCount = fastCountR
+		fastScore = fastScoreR
+		seqB = seqBR
+		directAlignment = false
+	}
+
+	// Compute the overlapping region length
+	switch {
+	case shift > 0:
+		over = seqA.Len() - shift
+	case shift < 0:
+		over = seqB.Len() + shift
+	default:
+		over = min(seqA.Len(), seqB.Len())
+	}
+
+	// log.Warnf("fw/fw: %v shift=%d fastCount=%d/over=%d fastScore=%f",
+	// 	directAlignment, shift, fastCount, over, fastScore)
+
+	// log.Warnf(("seqA: %s\nseqB: %s\n"), seqA.String(), seqB.String())
+
+	// At least one mismatch exists in the overlaping region
+	if fastCount+3 < over {
+
+		if shift > 0 || (shift == 0 && seqB.Len() >= seqA.Len()) {
+			startA = shift - delta
+			if startA < 0 {
+				startA = 0
+			}
+			extra5 = -startA
+			startB = 0
+
+			rawSeqA = seqA.Sequence()[startA:]
+			qualSeqA = seqA.Qualities()[startA:]
+			partLen = len(rawSeqA)
+			if partLen > seqB.Len() {
+				partLen = seqB.Len()
+			}
+			rawSeqB = seqB.Sequence()[0:partLen]
+			qualSeqB = seqB.Qualities()[0:partLen]
+			extra3 = seqB.Len() - partLen
+			score = _FillMatrixPeLeftAlign(
+				rawSeqA, qualSeqA, rawSeqB, qualSeqB, gap, scale,
+				&arena.pointer.scoreMatrix,
+				&arena.pointer.pathMatrix)
+		} else {
+
+			startA = 0
+			startB = -shift - delta
+			if startB < 0 {
+				startB = 0
+			}
+			extra5 = startB
+			rawSeqB = seqB.Sequence()[startB:]
+			qualSeqB = seqB.Qualities()[startB:]
+			partLen = len(rawSeqB)
+			if partLen > seqA.Len() {
+				partLen = seqA.Len()
+			}
+			rawSeqA = seqA.Sequence()[:partLen]
+			qualSeqA = seqA.Qualities()[:partLen]
+			extra3 = partLen - seqA.Len()
+
+			score = _FillMatrixPeRightAlign(
+				rawSeqA, qualSeqA, rawSeqB, qualSeqB, gap, scale,
+				&arena.pointer.scoreMatrix,
+				&arena.pointer.pathMatrix)
+		}
+
+		path = _Backtracking(arena.pointer.pathMatrix,
+			len(rawSeqA), len(rawSeqB),
+			&arena.pointer.path)
+
+	} else {
+
+		// Both overlaping regions are identicals
+
+		if shift > 0 || (shift == 0 && seqB.Len() >= seqA.Len()) {
+			startA = shift
+			startB = 0
+			extra5 = -startA
+			qualSeqA = seqA.Qualities()[startA:]
+			partLen = len(qualSeqA)
+			qualSeqB = seqB.Qualities()[0:partLen]
+			extra3 = seqB.Len() - partLen
+			score = 0
+		} else {
+			startA = 0
+			startB = -shift
+			extra5 = startB
+			qualSeqB = seqB.Qualities()[startB:]
+			partLen = len(qualSeqB)
+			extra3 = partLen - seqA.Len()
+			qualSeqA = seqA.Qualities()[:partLen]
+		}
+
+		score = 0
+		for i, qualA := range qualSeqA {
+			qualB := qualSeqB[i]
+			score += _NucScorePartMatchMatch[qualA][qualB]
+		}
+
+		path = arena.pointer.path[:0]
+		path = append(path, 0, partLen)
+	}
+
+	path[0] += extra5
+	if path[len(path)-1] == 0 {
+		path[len(path)-2] += extra3
+	} else {
+		path = append(path, extra3, 0)
+	}
+
+	return score, path, fastCount, over, fastScore, directAlignment
+}

pkg/obiapat/obiapat.c

@@ -137,6 +137,28 @@ char *reverseSequence(char *str,char isPattern)
         return str;
 }
 
+/* -------------------------------------------- */
+/* lowercase sequence                           */
+/* -------------------------------------------- */
+
+#define IS_UPPER(c) (((c) >= 'A') && ((c) <= 'A'))
+#define TO_LOWER(c) ((c) - 'A' + 'a')
+
+char *LowerSequence(char *seq)
+{
+        char *cseq;
+
+        for (cseq = seq ; *cseq ; cseq++) 
+            if (IS_UPPER(*cseq))
+                *cseq = TO_LOWER(*cseq);
+
+		return seq;
+}
+ 
+#undef IS_UPPER
+#undef TO_LOWER
+
+
 char *ecoComplementPattern(char *nucAcSeq)
 {
     return reverseSequence(LXBioSeqComplement(nucAcSeq),1);
@@ -165,6 +187,7 @@ void UpperSequence(char *seq)
 
 
 
+
 /* -------------------------------------------- */
 /* encode sequence                              */
 /* IS_UPPER is slightly faster than isupper     */

pkg/obiapat/pattern.go

@@ -9,6 +9,7 @@ import "C"
 import (
 	"errors"
 	"runtime"
+	"strings"
 	"unsafe"
 
 	log "github.com/sirupsen/logrus"
@@ -114,7 +115,7 @@ func (pattern ApatPattern) ReverseComplement() (ApatPattern, error) {
 		C.free(unsafe.Pointer(errmsg))
 		return ApatPattern{nil}, errors.New(message)
 	}
-	spat := C.GoString(apc.cpat)
+	spat := strings.ToLower(C.GoString(apc.cpat))
 	ap := _ApatPattern{apc, spat}
 
 	runtime.SetFinalizer(&ap, func(p *_ApatPattern) {
@@ -296,6 +297,24 @@ func (pattern ApatPattern) FindAllIndex(sequence ApatSequence, begin, length int
 	return loc
 }
 
+func (pattern ApatPattern) IsMatching(sequence ApatSequence, begin, length int) bool {
+	if begin < 0 {
+		begin = 0
+	}
+
+	if length < 0 {
+		length = sequence.Len()
+	}
+
+	nhits := int(C.ManberAll(sequence.pointer.pointer,
+		pattern.pointer.pointer,
+		0,
+		C.int32_t(begin),
+		C.int32_t(length+C.MAX_PAT_LEN)))
+
+	return nhits > 0
+}
+
 // BestMatch finds the best match of a given pattern in a sequence.
 //
 // THe function identify the first occurrence of the pattern in the sequence.
@@ -335,6 +354,11 @@ func (pattern ApatPattern) BestMatch(sequence ApatSequence, begin, length int) (
 	nerr = best[2]
 	end = best[1]
 
+	if best[0] < 0 || best[1] > sequence.Len() {
+		matched = false
+		return
+	}
+
 	if nerr == 0 || !pattern.pointer.pointer.hasIndel {
 		start = best[0]
 		log.Debugln("No nws ", start, nerr)
@@ -355,17 +379,31 @@ func (pattern ApatPattern) BestMatch(sequence ApatSequence, begin, length int) (
 		best[0], nerr, int(pattern.pointer.pointer.patlen),
 		sequence.Len(), start, end)
 
-	from, to, score := obialign.LocatePattern((*cpattern)[0:int(pattern.pointer.pointer.patlen)], frg)
+	from, to, score := obialign.LocatePattern(sequence.pointer.reference.Id(),
+		(*cpattern)[0:int(pattern.pointer.pointer.patlen)],
+		frg)
 
 	// olderr := m[2]
 
 	nerr = score
 	start = start + from
 	end = start + to
-	log.Debugln("results", score, start, nerr)
+	log.Debugf("BestMatch on %s : score=%d [%d..%d]", sequence.pointer.reference.Id(), score, start, nerr)
 	return
 }
 
+// FilterBestMatch filters the best non overlapping matches of a given pattern in a sequence.
+//
+// It takes the following parameters:
+// - pattern: the pattern to search for (ApatPattern).
+// - sequence: the sequence to search in (ApatSequence).
+// - begin: the starting index of the search (int).
+// - length: the length of the search (int).
+//
+// It returns a slice of [3]int representing the locations of all non-overlapping matches in the sequence.
+// The two firsts values of the [3]int indicate respectively the start and the end position of
+// the match. Following the GO convention the end position is not included in the
+// match. The third value indicates the number of error detected for this occurrence.
 func (pattern ApatPattern) FilterBestMatch(sequence ApatSequence, begin, length int) (loc [][3]int) {
 	res := pattern.FindAllIndex(sequence, begin, length)
 	filtered := make([][3]int, 0, len(res))
@@ -424,13 +462,15 @@ func (pattern ApatPattern) FilterBestMatch(sequence ApatSequence, begin, length
 func (pattern ApatPattern) AllMatches(sequence ApatSequence, begin, length int) (loc [][3]int) {
 	res := pattern.FilterBestMatch(sequence, begin, length)
 
+	j := 0
 	for _, m := range res {
 		// Recompute the start and end position of the match
 		// when the pattern allows for indels
 		if m[2] > 0 && pattern.pointer.pointer.hasIndel {
-			start := m[0] - m[2]
+			// log.Warnf("Locating indel on sequence %s[%s]", sequence.pointer.reference.Id(), pattern.String())
+			start := m[0] - m[2]*2
 			start = max(start, 0)
-			end := start + int(pattern.pointer.pointer.patlen) + 2*m[2]
+			end := start + int(pattern.pointer.pointer.patlen) + 4*m[2]
 			end = min(end, sequence.Len())
 			// 1 << 30 = 1,073,741,824 = 1Gb
 			// It's a virtual array mapping the sequence to the pattern
@@ -439,18 +479,24 @@ func (pattern ApatPattern) AllMatches(sequence ApatSequence, begin, length int)
 			cpattern := (*[1 << 30]byte)(unsafe.Pointer(pattern.pointer.pointer.cpat))
 			frg := sequence.pointer.reference.Sequence()[start:end]
 
-			begin, end, score := obialign.LocatePattern((*cpattern)[0:int(pattern.pointer.pointer.patlen)], frg)
+			pb, pe, score := obialign.LocatePattern(
+				sequence.pointer.reference.Id(),
+				(*cpattern)[0:int(pattern.pointer.pointer.patlen)],
+				frg)
 
 			// olderr := m[2]
 			m[2] = score
-			m[0] = start + begin
-			m[1] = start + end
+			m[0] = start + pb
+			m[1] = start + pe
+
 			// log.Warnf("seq[%d@%d:%d] %d: %s %d - %s:%s:%s", i, m[0], m[1], olderr, sequence.pointer.reference.Id(), score,
 			// 	frg, (*cpattern)[0:int(pattern.pointer.pointer.patlen)], sequence.pointer.reference.Sequence()[m[0]:m[1]])
 		}
+
+		if int(pattern.pointer.pointer.maxerr) >= m[2] {
+			res[j] = m
+			j++
+		}
 	}
-
-	// log.Debugf("All matches : %v", res)
-
-	return res
+	return res[0:j]
 }

pkg/obiapat/pcr.go

@@ -3,7 +3,7 @@ package obiapat
 import (
 	log "github.com/sirupsen/logrus"
 
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
@@ -104,7 +104,7 @@ func MakeOptions(setters []WithOption) Options {
 		extension:       -1,
 		fullExtension:   false,
 		circular:        false,
-		parallelWorkers: obioptions.CLIParallelWorkers(),
+		parallelWorkers: obidefault.ParallelWorkers(),
 		batchSize:       100,
 		forward:         NilApatPattern,
 		cfwd:            NilApatPattern,
@@ -529,7 +529,6 @@ func PCRSliceWorker(options ...WithOption) obiseq.SeqSliceWorker {
 	opt := MakeOptions(options)
 	worker := func(sequences obiseq.BioSequenceSlice) (obiseq.BioSequenceSlice, error) {
 		result := _PCRSlice(sequences, opt)
-		sequences.Recycle(true)
 		return result, nil
 	}
 

pkg/obiapat/predicat.go

@@ -0,0 +1,40 @@
+package obiapat
+
+import (
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+	log "github.com/sirupsen/logrus"
+)
+
+func IsPatternMatchSequence(pattern string, errormax int, bothStrand, allowIndels bool) obiseq.SequencePredicate {
+
+	pat, err := MakeApatPattern(pattern, errormax, allowIndels)
+
+	if err != nil {
+		log.Fatalf("error in sequence regular pattern syntax : %v", err)
+	}
+
+	cpat, err := pat.ReverseComplement()
+
+	if err != nil {
+		log.Fatalf("cannot reverse complement the pattern : %v", err)
+	}
+
+	f := func(sequence *obiseq.BioSequence) bool {
+		aseq, err := MakeApatSequence(sequence, false)
+
+		if err != nil {
+			log.Panicf("Cannot convert sequence %s to apat format", sequence.Id())
+		}
+
+		match := pat.IsMatching(aseq, 0, aseq.Len())
+
+		if !match && bothStrand {
+
+			match = cpat.IsMatching(aseq, 0, aseq.Len())
+		}
+
+		return match
+	}
+
+	return f
+}

pkg/obichunk/chunk_on_disk.go

@@ -12,6 +12,13 @@ import (
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 )
 
+// tempDir creates a temporary directory with a prefix "obiseq_chunks_"
+// in the system's temporary directory. It returns the path of the
+// created directory and any error encountered during the creation process.
+//
+// If the directory creation is successful, the path to the new
+// temporary directory is returned. If there is an error, it returns
+// an empty string and the error encountered.
 func tempDir() (string, error) {
 	dir, err := os.MkdirTemp(os.TempDir(), "obiseq_chunks_")
 	if err != nil {
@@ -20,6 +27,19 @@ func tempDir() (string, error) {
 	return dir, nil
 }
 
+// find searches for files with a specific extension in the given root directory
+// and its subdirectories. It returns a slice of strings containing the paths
+// of the found files.
+//
+// Parameters:
+// - root: The root directory to start the search from.
+// - ext: The file extension to look for (including the leading dot, e.g., ".txt").
+//
+// Returns:
+// A slice of strings containing the paths of files that match the specified
+// extension. If no files are found, an empty slice is returned. Any errors
+// encountered during the directory traversal will be returned as part of the
+// WalkDir function's error handling.
 func find(root, ext string) []string {
 	var a []string
 	filepath.WalkDir(root, func(s string, d fs.DirEntry, e error) error {
@@ -34,6 +54,24 @@ func find(root, ext string) []string {
 	return a
 }
 
+// ISequenceChunkOnDisk processes a sequence iterator by distributing the sequences
+// into chunks stored on disk. It uses a classifier to determine how to distribute
+// the sequences and returns a new iterator for the processed sequences.
+//
+// Parameters:
+//   - iterator: An iterator of biosequences to be processed.
+//   - classifier: A pointer to a BioSequenceClassifier used to classify the sequences
+//     during distribution.
+//
+// Returns:
+// An iterator of biosequences representing the processed chunks. If an error occurs
+// during the creation of the temporary directory or any other operation, it returns
+// an error along with a nil iterator.
+//
+// The function operates asynchronously, creating a temporary directory to store
+// the sequence chunks. Once the processing is complete, the temporary directory
+// is removed. The function logs the number of batches created and the processing
+// status of each batch.
 func ISequenceChunkOnDisk(iterator obiiter.IBioSequence,
 	classifier *obiseq.BioSequenceClassifier) (obiiter.IBioSequence, error) {
 	dir, err := tempDir()

pkg/obichunk/chunks.go

@@ -49,7 +49,6 @@ func ISequenceChunk(iterator obiiter.IBioSequence,
 					b := data.Get()
 					source = b.Source()
 					*chunk = append(*chunk, b.Slice()...)
-					b.Recycle(false)
 				}
 
 				lock.Lock()

pkg/obichunk/options.go

@@ -1,7 +1,7 @@
 package obichunk
 
 import (
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 )
 
@@ -29,8 +29,8 @@ func MakeOptions(setters []WithOption) Options {
 		navalue:         "NA",
 		cacheOnDisk:     false,
 		batchCount:      100,
-		batchSize:       obioptions.CLIBatchSize(),
-		parallelWorkers: obioptions.CLIParallelWorkers(),
+		batchSize:       obidefault.BatchSize(),
+		parallelWorkers: obidefault.ParallelWorkers(),
 		noSingleton:     false,
 	}
 

pkg/obichunk/subchunks.go

@@ -6,8 +6,8 @@ import (
 
 	log "github.com/sirupsen/logrus"
 
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 )
 
@@ -62,7 +62,7 @@ func ISequenceSubChunk(iterator obiiter.IBioSequence,
 	nworkers int) (obiiter.IBioSequence, error) {
 
 	if nworkers <= 0 {
-		nworkers = obioptions.CLIParallelWorkers()
+		nworkers = obidefault.ParallelWorkers()
 	}
 
 	newIter := obiiter.MakeIBioSequence()
@@ -107,8 +107,6 @@ func ISequenceSubChunk(iterator obiiter.IBioSequence,
 					batch.Slice()[i] = nil
 				}
 
-				batch.Recycle(false)
-
 				_By(func(p1, p2 *sSS) bool {
 					return p1.code < p2.code
 				}).Sort(ordered)

pkg/obichunk/unique.go

@@ -95,10 +95,7 @@ func IUniqueSequence(iterator obiiter.IBioSequence,
 
 			if icat < 0 || len(batch.Slice()) == 1 {
 				// No more sub classification of sequence or only a single sequence
-				if opts.NoSingleton() && len(batch.Slice()) == 1 && batch.Slice()[0].Count() == 1 {
-					// We remove singleton from output
-					batch.Recycle(true)
-				} else {
+				if !(opts.NoSingleton() && len(batch.Slice()) == 1 && batch.Slice()[0].Count() == 1) {
 					iUnique.Push(batch.Reorder(nextOrder()))
 				}
 			} else {

pkg/obidefault/batch.go

@@ -0,0 +1,26 @@
+package obidefault
+
+var _BatchSize = 2000
+
+// SetBatchSize sets the size of the sequence batches.
+//
+// n - an integer representing the size of the sequence batches.
+func SetBatchSize(n int) {
+	_BatchSize = n
+}
+
+// CLIBatchSize returns the expected size of the sequence batches.
+//
+// In Obitools, the sequences are processed in parallel by batches.
+// The number of sequence in each batch is determined by the command line option
+// --batch-size and the environment variable OBIBATCHSIZE.
+//
+// No parameters.
+// Returns an integer value.
+func BatchSize() int {
+	return _BatchSize
+}
+
+func BatchSizePtr() *int {
+	return &_BatchSize
+}

pkg/obidefault/compressed.go

@@ -0,0 +1,15 @@
+package obidefault
+
+var __compressed__ = false
+
+func CompressOutput() bool {
+	return __compressed__
+}
+
+func SetCompressOutput(b bool) {
+	__compressed__ = b
+}
+
+func CompressedPtr() *bool {
+	return &__compressed__
+}

pkg/obidefault/quality.go

@@ -0,0 +1,29 @@
+package obidefault
+
+var _Quality_Shift_Input = byte(33)
+var _Quality_Shift_Output = byte(33)
+var _Read_Qualities = true
+
+func SetReadQualitiesShift(shift byte) {
+	_Quality_Shift_Input = shift
+}
+
+func ReadQualitiesShift() byte {
+	return _Quality_Shift_Input
+}
+
+func SetWriteQualitiesShift(shift byte) {
+	_Quality_Shift_Output = shift
+}
+
+func WriteQualitiesShift() byte {
+	return _Quality_Shift_Output
+}
+
+func SetReadQualities(read bool) {
+	_Read_Qualities = read
+}
+
+func ReadQualities() bool {
+	return _Read_Qualities
+}

pkg/obidefault/taxonomy.go

@@ -0,0 +1,58 @@
+package obidefault
+
+var __taxonomy__ = ""
+var __alternative_name__ = false
+var __fail_on_taxonomy__ = false
+var __update_taxid__ = false
+
+func SelectedTaxonomy() string {
+	return __taxonomy__
+}
+
+func HasSelectedTaxonomy() bool {
+	return __taxonomy__ != ""
+}
+
+func AreAlternativeNamesSelected() bool {
+	return __alternative_name__
+}
+
+func SelectedTaxonomyPtr() *string {
+	return &__taxonomy__
+}
+
+func AlternativeNamesSelectedPtr() *bool {
+	return &__alternative_name__
+}
+
+func SetSelectedTaxonomy(taxonomy string) {
+	__taxonomy__ = taxonomy
+}
+
+func SetAlternativeNamesSelected(alt bool) {
+	__alternative_name__ = alt
+}
+
+func SetFailOnTaxonomy(fail bool) {
+	__fail_on_taxonomy__ = fail
+}
+
+func SetUpdateTaxid(update bool) {
+	__update_taxid__ = update
+}
+
+func FailOnTaxonomyPtr() *bool {
+	return &__fail_on_taxonomy__
+}
+
+func UpdateTaxidPtr() *bool {
+	return &__update_taxid__
+}
+
+func FailOnTaxonomy() bool {
+	return __fail_on_taxonomy__
+}
+
+func UpdateTaxid() bool {
+	return __update_taxid__
+}

pkg/obidefault/workers.go

@@ -0,0 +1,170 @@
+package obidefault
+
+import "runtime"
+
+var _MaxAllowedCPU = runtime.NumCPU()
+var _WorkerPerCore = 1.0
+
+var _ReadWorkerPerCore = 0.25
+var _WriteWorkerPerCore = 0.25
+
+var _StrictReadWorker = 0
+var _StrictWriteWorker = 0
+
+var _ParallelFilesRead = 0
+
+// CLIParallelWorkers returns the number of parallel workers used for
+// computing the result.
+//
+// The number of parallel workers is determined by the command line option
+// --max-cpu|-m and the environment variable OBIMAXCPU. This number is
+// multiplied by the variable _WorkerPerCore.
+//
+// No parameters.
+// Returns an integer representing the number of parallel workers.
+func ParallelWorkers() int {
+	return int(float64(MaxCPU()) * float64(WorkerPerCore()))
+}
+
+// CLIMaxCPU returns the maximum number of CPU cores allowed.
+//
+// The maximum number of CPU cores is determined by the command line option
+// --max-cpu|-m and the environment variable OBIMAXCPU.
+//
+// No parameters.
+// Returns an integer representing the maximum number of CPU cores allowed.
+func MaxCPU() int {
+	return _MaxAllowedCPU
+}
+
+func MaxCPUPtr() *int {
+	return &_MaxAllowedCPU
+}
+
+// WorkerPerCore returns the number of workers per CPU core.
+//
+// No parameters.
+// Returns a float64 representing the number of workers per CPU core.
+func WorkerPerCore() float64 {
+	return _WorkerPerCore
+}
+
+// SetWorkerPerCore sets the number of workers per CPU core.
+//
+// It takes a float64 parameter representing the number of workers
+// per CPU core and does not return any value.
+func SetWorkerPerCore(n float64) {
+	_WorkerPerCore = n
+}
+
+// SetMaxCPU sets the maximum number of CPU cores allowed.
+//
+// n - an integer representing the new maximum number of CPU cores.
+func SetMaxCPU(n int) {
+	_MaxAllowedCPU = n
+}
+
+// SetReadWorker sets the number of workers for reading files.
+//
+// The number of worker dedicated to reading files is determined
+// as the number of allowed CPU cores multiplied by number of read workers per core.
+// Setting the number of read workers using this function allows to decouple the number
+// of read workers from the number of CPU cores.
+//
+// n - an integer representing the number of workers to be set.
+func SetStrictReadWorker(n int) {
+	_StrictReadWorker = n
+}
+
+func SetStrictWriteWorker(n int) {
+	_StrictWriteWorker = n
+}
+
+// SetReadWorkerPerCore sets the number of worker per CPU
+// core for reading files.
+//
+// n float64
+func SetReadWorkerPerCore(n float64) {
+	_ReadWorkerPerCore = n
+}
+
+func SetWriteWorkerPerCore(n float64) {
+	_WriteWorkerPerCore = n
+}
+
+// ReadWorker returns the number of workers for reading files.
+//
+// No parameters.
+// Returns an integer representing the number of workers.
+func StrictReadWorker() int {
+	return _StrictReadWorker
+}
+
+func StrictWriteWorker() int {
+	return _StrictWriteWorker
+}
+
+// CLIReadParallelWorkers returns the number of parallel workers used for
+// reading files.
+//
+// The number of parallel workers is determined by the command line option
+// --max-cpu|-m and the environment variable OBIMAXCPU. This number is
+// multiplied by the variable _ReadWorkerPerCore.
+//
+// No parameters.
+// Returns an integer representing the number of parallel workers.
+func ReadParallelWorkers() int {
+	if StrictReadWorker() == 0 {
+		n := int(float64(MaxCPU()) * ReadWorkerPerCore())
+		if n == 0 {
+			n = 1
+		}
+		return n
+	} else {
+		return StrictReadWorker()
+	}
+}
+
+func WriteParallelWorkers() int {
+	if StrictWriteWorker() == 0 {
+		n := int(float64(MaxCPU()) * WriteWorkerPerCore())
+		if n == 0 {
+			n = 1
+		}
+		return n
+	} else {
+		return StrictReadWorker()
+	}
+}
+
+// ReadWorkerPerCore returns the number of worker per CPU core for
+// computing the result.
+//
+// No parameters.
+// Returns a float64 representing the number of worker per CPU core.
+func ReadWorkerPerCore() float64 {
+	return _ReadWorkerPerCore
+}
+
+func WriteWorkerPerCore() float64 {
+	return _ReadWorkerPerCore
+}
+
+// ParallelFilesRead returns the number of files to be read in parallel.
+//
+// No parameters.
+// Returns an integer representing the number of files to be read.
+func ParallelFilesRead() int {
+	if _ParallelFilesRead == 0 {
+		return ReadParallelWorkers()
+	} else {
+		return _ParallelFilesRead
+	}
+}
+
+// SetParallelFilesRead sets the number of files to be read in parallel.
+//
+// n - an integer representing the number of files to be set.
+func SetParallelFilesRead(n int) {
+	_ParallelFilesRead = n
+}

pkg/obiformats/csv_read.go

@@ -5,10 +5,10 @@ import (
 	"io"
 	"os"
 	"path"
-	"unsafe"
+	"strings"
 
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 	"github.com/goccy/go-json"
@@ -94,19 +94,28 @@ func _ParseCsvFile(source string,
 				continue
 			}
 
-			err := json.Unmarshal(unsafe.Slice(unsafe.StringData(field), len(field)), &val)
+			ft := header[i]
 
-			if err != nil {
-				val = field
-			} else {
-				if _, ok := val.(float64); ok {
-					if obiutils.IsIntegral(val.(float64)) {
-						val = int(val.(float64))
+			switch {
+			case ft == "taxid":
+				sequence.SetTaxid(field)
+			case strings.HasSuffix(ft, "_taxid"):
+				sequence.SetTaxid(field, strings.TrimSuffix(ft, "_taxid"))
+			default:
+				err := json.Unmarshal(obiutils.UnsafeBytes(field), &val)
+
+				if err != nil {
+					val = field
+				} else {
+					if _, ok := val.(float64); ok {
+						if obiutils.IsIntegral(val.(float64)) {
+							val = int(val.(float64))
+						}
 					}
 				}
-			}
 
-			sequence.SetAttribute(header[i], val)
+				sequence.SetAttribute(ft, val)
+			}
 		}
 
 		slice = append(slice, sequence)
@@ -134,7 +143,7 @@ func ReadCSV(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, err
 	go _ParseCsvFile(opt.Source(),
 		reader,
 		out,
-		byte(obioptions.InputQualityShift()),
+		obidefault.ReadQualitiesShift(),
 		opt.BatchSize())
 
 	go func() {
@@ -148,9 +157,9 @@ func ReadCSV(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, err
 func ReadCSVFromFile(filename string, options ...WithOption) (obiiter.IBioSequence, error) {
 
 	options = append(options, OptionsSource(obiutils.RemoveAllExt((path.Base(filename)))))
-	file, err := Ropen(filename)
+	file, err := obiutils.Ropen(filename)
 
-	if err == ErrNoContent {
+	if err == obiutils.ErrNoContent {
 		log.Infof("file %s is empty", filename)
 		return ReadEmptyFile(options...)
 	}
@@ -164,9 +173,9 @@ func ReadCSVFromFile(filename string, options ...WithOption) (obiiter.IBioSequen
 
 func ReadCSVFromStdin(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, error) {
 	options = append(options, OptionsSource(obiutils.RemoveAllExt("stdin")))
-	input, err := Buf(os.Stdin)
+	input, err := obiutils.Buf(os.Stdin)
 
-	if err == ErrNoContent {
+	if err == obiutils.ErrNoContent {
 		log.Infof("stdin is empty")
 		return ReadEmptyFile(options...)
 	}

pkg/obiformats/csv_writer.go

@@ -1,22 +1,14 @@
 package obiformats
 
 import (
-	"bytes"
-	"encoding/csv"
 	"fmt"
-	"io"
-	"os"
-	"sync"
-	"time"
 
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
-	log "github.com/sirupsen/logrus"
 )
 
-func CSVRecord(sequence *obiseq.BioSequence, opt Options) []string {
+func CSVSequenceRecord(sequence *obiseq.BioSequence, opt Options) []string {
 	keys := opt.CSVKeys()
 	record := make([]string, 0, len(keys)+4)
 
@@ -30,14 +22,10 @@ func CSVRecord(sequence *obiseq.BioSequence, opt Options) []string {
 
 	if opt.CSVTaxon() {
 		taxid := sequence.Taxid()
-		sn, ok := sequence.GetAttribute("scientific_name")
+		sn, ok := sequence.GetStringAttribute("scientific_name")
 
 		if !ok {
-			if taxid == 1 {
-				sn = "root"
-			} else {
-				sn = opt.CSVNAValue()
-			}
+			sn = opt.CSVNAValue()
 		}
 
 		record = append(record, fmt.Sprint(taxid), fmt.Sprint(sn))
@@ -66,7 +54,7 @@ func CSVRecord(sequence *obiseq.BioSequence, opt Options) []string {
 			l := sequence.Len()
 			q := sequence.Qualities()
 			ascii := make([]byte, l)
-			quality_shift := obioptions.OutputQualityShift()
+			quality_shift := obidefault.WriteQualitiesShift()
 			for j := 0; j < l; j++ {
 				ascii[j] = uint8(q[j]) + uint8(quality_shift)
 			}
@@ -78,182 +66,3 @@ func CSVRecord(sequence *obiseq.BioSequence, opt Options) []string {
 
 	return record
 }
-
-func CSVHeader(opt Options) []string {
-	keys := opt.CSVKeys()
-	record := make([]string, 0, len(keys)+4)
-
-	if opt.CSVId() {
-		record = append(record, "id")
-	}
-
-	if opt.CSVCount() {
-		record = append(record, "count")
-	}
-
-	if opt.CSVTaxon() {
-		record = append(record, "taxid", "scientific_name")
-	}
-
-	if opt.CSVDefinition() {
-		record = append(record, "definition")
-	}
-
-	record = append(record, opt.CSVKeys()...)
-
-	if opt.CSVSequence() {
-		record = append(record, "sequence")
-	}
-
-	if opt.CSVQuality() {
-		record = append(record, "quality")
-	}
-
-	return record
-}
-
-func FormatCVSBatch(batch obiiter.BioSequenceBatch, opt Options) []byte {
-	buff := new(bytes.Buffer)
-	csv := csv.NewWriter(buff)
-
-	if batch.Order() == 0 {
-		csv.Write(CSVHeader(opt))
-	}
-	for _, s := range batch.Slice() {
-		csv.Write(CSVRecord(s, opt))
-	}
-
-	csv.Flush()
-
-	return buff.Bytes()
-}
-
-func WriteCSV(iterator obiiter.IBioSequence,
-	file io.WriteCloser,
-	options ...WithOption) (obiiter.IBioSequence, error) {
-
-	var auto_slot obiutils.Set[string]
-	opt := MakeOptions(options)
-
-	file, _ = obiutils.CompressStream(file, opt.CompressedFile(), opt.CloseFile())
-
-	newIter := obiiter.MakeIBioSequence()
-
-	nwriters := opt.ParallelWorkers()
-
-	obiiter.RegisterAPipe()
-	chunkchan := make(chan FileChunk)
-
-	newIter.Add(nwriters)
-	var waitWriter sync.WaitGroup
-
-	go func() {
-		newIter.WaitAndClose()
-		for len(chunkchan) > 0 {
-			time.Sleep(time.Millisecond)
-		}
-		close(chunkchan)
-		waitWriter.Wait()
-	}()
-
-	ff := func(iterator obiiter.IBioSequence) {
-		for iterator.Next() {
-
-			batch := iterator.Get()
-
-			chunkchan <- FileChunk{
-				FormatCVSBatch(batch, opt),
-				batch.Order(),
-			}
-			newIter.Push(batch)
-		}
-		newIter.Done()
-	}
-
-	next_to_send := 0
-	received := make(map[int]FileChunk, 100)
-
-	waitWriter.Add(1)
-	go func() {
-		for chunk := range chunkchan {
-			if chunk.order == next_to_send {
-				file.Write(chunk.text)
-				next_to_send++
-				chunk, ok := received[next_to_send]
-				for ok {
-					file.Write(chunk.text)
-					delete(received, next_to_send)
-					next_to_send++
-					chunk, ok = received[next_to_send]
-				}
-			} else {
-				received[chunk.order] = chunk
-			}
-
-		}
-
-		file.Close()
-
-		log.Debugln("End of the CSV file writing")
-		obiiter.UnregisterPipe()
-		waitWriter.Done()
-	}()
-
-	if opt.pointer.csv_auto {
-		if iterator.Next() {
-			batch := iterator.Get()
-			auto_slot = batch.Slice().AttributeKeys(true)
-			CSVKeys(auto_slot.Members())(opt)
-			iterator.PushBack()
-		}
-	}
-
-	log.Debugln("Start of the CSV file writing")
-	go ff(iterator)
-	for i := 0; i < nwriters-1; i++ {
-		go ff(iterator.Split())
-	}
-
-	return newIter, nil
-}
-
-func WriteCSVToStdout(iterator obiiter.IBioSequence,
-	options ...WithOption) (obiiter.IBioSequence, error) {
-	options = append(options, OptionDontCloseFile())
-	return WriteCSV(iterator, os.Stdout, options...)
-}
-
-func WriteCSVToFile(iterator obiiter.IBioSequence,
-	filename string,
-	options ...WithOption) (obiiter.IBioSequence, error) {
-
-	opt := MakeOptions(options)
-	flags := os.O_WRONLY | os.O_CREATE
-
-	if opt.AppendFile() {
-		flags |= os.O_APPEND
-	}
-	file, err := os.OpenFile(filename, flags, 0660)
-
-	if err != nil {
-		log.Fatalf("open file error: %v", err)
-		return obiiter.NilIBioSequence, err
-	}
-
-	options = append(options, OptionCloseFile())
-
-	iterator, err = WriteCSV(iterator, file, options...)
-
-	if opt.HaveToSavePaired() {
-		var revfile *os.File
-
-		revfile, err = os.OpenFile(opt.PairedFileName(), flags, 0660)
-		if err != nil {
-			log.Fatalf("open file error: %v", err)
-			return obiiter.NilIBioSequence, err
-		}
-		iterator, err = WriteCSV(iterator.PairedWith(), revfile, options...)
-	}
-
-	return iterator, err
-}

pkg/obiformats/dispatcher.go

@@ -14,10 +14,40 @@ import (
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
 )
 
+// SequenceBatchWriterToFile is a function type that defines a method for writing
+// a batch of biosequences to a specified file. It takes an iterator of biosequences,
+// a filename, and optional configuration options, and returns an iterator of biosequences
+// along with any error encountered during the writing process.
+//
+// Parameters:
+// - iterator: An iterator of biosequences to be written to the file.
+// - filename: The name of the file where the sequences will be written.
+// - options: Optional configuration options for the writing process.
+//
+// Returns:
+// An iterator of biosequences that may have been modified during the writing process
+// and an error if the writing operation fails.
 type SequenceBatchWriterToFile func(iterator obiiter.IBioSequence,
 	filename string,
 	options ...WithOption) (obiiter.IBioSequence, error)
 
+// WriterDispatcher manages the writing of data to files based on a given
+// prototype name and a dispatcher for distributing the sequences. It
+// processes incoming data from the dispatcher in separate goroutines,
+// formatting and writing the data to files as specified.
+//
+// Parameters:
+//   - prototypename: A string that serves as a template for naming the output files.
+//   - dispatcher: An instance of IDistribute that provides the data to be written
+//     and manages the distribution of sequences.
+//   - formater: A function of type SequenceBatchWriterToFile that formats and writes
+//     the sequences to the specified file.
+//   - options: Optional configuration options for the writing process.
+//
+// The function operates asynchronously, launching goroutines for each new data
+// channel received from the dispatcher. It ensures that directories are created
+// as needed and handles errors during the writing process. The function blocks
+// until all writing jobs are completed.
 func WriterDispatcher(prototypename string,
 	dispatcher obiiter.IDistribute,
 	formater SequenceBatchWriterToFile,
@@ -34,7 +64,7 @@ func WriterDispatcher(prototypename string,
 				data, err := dispatcher.Outputs(newflux)
 
 				if err != nil {
-					log.Fatalf("Cannot retreive the new chanel : %v", err)
+					log.Fatalf("Cannot retrieve the new channel: %v", err)
 				}
 
 				key := dispatcher.Classifier().Value(newflux)
@@ -58,7 +88,7 @@ func WriterDispatcher(prototypename string,
 					info, err := os.Stat(directory)
 					switch {
 					case !os.IsNotExist(err) && !info.IsDir():
-						log.Fatalf("Cannot Create the directory %s", directory)
+						log.Fatalf("Cannot create the directory %s", directory)
 					case os.IsNotExist(err):
 						os.Mkdir(directory, 0755)
 					}
@@ -71,7 +101,7 @@ func WriterDispatcher(prototypename string,
 					options...)
 
 				if err != nil {
-					log.Fatalf("cannot open the output file for key %s",
+					log.Fatalf("Cannot open the output file for key %s",
 						dispatcher.Classifier().Value(newflux))
 				}
 

pkg/obiformats/embl_read.go

@@ -159,7 +159,7 @@ func EmblChunkParser(withFeatureTable bool) func(string, io.Reader) (obiseq.BioS
 }
 
 func _ParseEmblFile(
-	input ChannelSeqFileChunk,
+	input ChannelFileChunk,
 	out obiiter.IBioSequence,
 	withFeatureTable bool,
 ) {
@@ -187,9 +187,9 @@ func _ParseEmblFile(
 func ReadEMBL(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, error) {
 	opt := MakeOptions(options)
 
-	buff := make([]byte, 1024*1024*1024*256)
+	buff := make([]byte, 1024*1024*128) // 128 MB
 
-	entry_channel := ReadSeqFileChunk(
+	entry_channel := ReadFileChunk(
 		opt.Source(),
 		reader,
 		buff,
@@ -227,9 +227,9 @@ func ReadEMBLFromFile(filename string, options ...WithOption) (obiiter.IBioSeque
 
 	options = append(options, OptionsSource(obiutils.RemoveAllExt((path.Base(filename)))))
 
-	reader, err = Ropen(filename)
+	reader, err = obiutils.Ropen(filename)
 
-	if err == ErrNoContent {
+	if err == obiutils.ErrNoContent {
 		log.Infof("file %s is empty", filename)
 		return ReadEmptyFile(options...)
 	}

pkg/obiformats/fastaseq_read.go

@@ -205,7 +205,7 @@ func FastaChunkParser() func(string, io.Reader) (obiseq.BioSequenceSlice, error)
 }
 
 func _ParseFastaFile(
-	input ChannelSeqFileChunk,
+	input ChannelFileChunk,
 	out obiiter.IBioSequence,
 ) {
 
@@ -213,6 +213,7 @@ func _ParseFastaFile(
 
 	for chunks := range input {
 		sequences, err := parser(chunks.Source, chunks.Raw)
+		// log.Warnf("Chunck(%d:%d) -%d- ", chunks.Order, l, sequences.Len())
 
 		if err != nil {
 			log.Fatalf("File %s : Cannot parse the fasta file : %v", chunks.Source, err)
@@ -232,9 +233,9 @@ func ReadFasta(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, e
 
 	nworker := opt.ParallelWorkers()
 
-	buff := make([]byte, 1024*1024*1024)
+	buff := make([]byte, 1024*1024)
 
-	chkchan := ReadSeqFileChunk(
+	chkchan := ReadFileChunk(
 		opt.Source(),
 		reader,
 		buff,
@@ -250,7 +251,7 @@ func ReadFasta(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, e
 		out.WaitAndClose()
 	}()
 
-	newIter := out.SortBatches().Rebatch(opt.BatchSize())
+	newIter := out.SortBatches()
 
 	log.Debugln("Full file batch mode : ", opt.FullFileBatch())
 
@@ -270,9 +271,9 @@ func ReadFasta(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, e
 func ReadFastaFromFile(filename string, options ...WithOption) (obiiter.IBioSequence, error) {
 	options = append(options, OptionsSource(obiutils.RemoveAllExt((path.Base(filename)))))
 
-	file, err := Ropen(filename)
+	file, err := obiutils.Ropen(filename)
 
-	if err == ErrNoContent {
+	if err == obiutils.ErrNoContent {
 		log.Infof("file %s is empty", filename)
 		return ReadEmptyFile(options...)
 	}
@@ -286,9 +287,9 @@ func ReadFastaFromFile(filename string, options ...WithOption) (obiiter.IBioSequ
 
 func ReadFastaFromStdin(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, error) {
 	options = append(options, OptionsSource("stdin"))
-	input, err := Buf(os.Stdin)
+	input, err := obiutils.Buf(os.Stdin)
 
-	if err == ErrNoContent {
+	if err == obiutils.ErrNoContent {
 		log.Infof("stdin is empty")
 		return ReadEmptyFile(options...)
 	}

pkg/obiformats/fastqseq_read.go

@@ -7,8 +7,8 @@ import (
 	"os"
 	"path"
 
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 	log "github.com/sirupsen/logrus"
@@ -17,6 +17,7 @@ import (
 func EndOfLastFastqEntry(buffer []byte) int {
 	var i int
 
+	// log.Warnf("EndOfLastFastqEntry(%d): %s", len(buffer), string(buffer[0:20]))
 	imax := len(buffer)
 	state := 0
 	restart := imax - 1
@@ -32,39 +33,48 @@ func EndOfLastFastqEntry(buffer []byte) int {
 		case 0:
 			if C == '+' {
 				// Potential start of quality part step 1
+				// log.Warn("Potential start of quality part step 1 - +")
 				state = 1
 				restart = i
 			}
 		case 1:
 			if is_end_of_line {
 				// Potential start of quality part step 2
+				// log.Warn("Potential start of quality part step 2 - +/end of line")
 				state = 2
 			} else {
 				// it was not the start of quality part
+				// log.Warn("it was not the start of quality part")
 				state = 0
 				i = restart
 			}
 		case 2:
 			if is_sep {
 				// Potential start of quality part step 2 (stay in the same state)
+				// log.Warn("Potential start of quality part step 2 - skipping separator")
 				state = 2
 			} else if (C >= 'a' && C <= 'z') || (C >= 'A' && C <= 'Z') || C == '-' || C == '.' || C == '[' || C == ']' {
-				// End of the sequence
+				// progressing along of the sequence
+				// log.Warn("Detected the end of the sequence switching to state 3")
 				state = 3
 			} else {
 				// it was not the start of quality part
+				// log.Warn("it was not the start of quality part because is not preceded by sequence")
 				state = 0
 				i = restart
 			}
 		case 3:
 			if is_end_of_line {
 				// Entrering in the header line
+				// log.Warn("Potentially entrering in the header line")
 				state = 4
 			} else if (C >= 'a' && C <= 'z') || (C >= 'A' && C <= 'Z') || C == '-' || C == '.' || C == '[' || C == ']' {
 				// progressing along of the sequence
+				// log.Warn("Progressing along of the sequence")
 				state = 3
 			} else {
 				// it was not the sequence part
+				// log.Warnf("it was not the sequence part : %c", C)
 				state = 0
 				i = restart
 			}
@@ -72,6 +82,7 @@ func EndOfLastFastqEntry(buffer []byte) int {
 			if is_end_of_line {
 				state = 4
 			} else {
+
 				state = 5
 			}
 		case 5:
@@ -80,15 +91,18 @@ func EndOfLastFastqEntry(buffer []byte) int {
 				state = 0
 				i = restart
 			} else if C == '@' {
+				// It was the header line
+				// log.Warn("It was the header line")
 				state = 6
 				cut = i
 			}
 		case 6:
 			if is_end_of_line {
+				// log.Warn("====> End of the last sequence")
 				state = 7
 			} else {
-				state = 0
-				i = restart
+				// log.Warnf("%s: Strange it was not the end of the last sequence : %c : %s", string(buffer[0:40]), C, string(buffer[i-20:i+5]))
+				state = 5
 			}
 		}
 	}
@@ -117,7 +131,7 @@ func _storeSequenceQuality(bytes *bytes.Buffer, out *obiseq.BioSequence, quality
 	out.SetQualities(q)
 }
 
-func FastqChunkParser(quality_shift byte) func(string, io.Reader) (obiseq.BioSequenceSlice, error) {
+func FastqChunkParser(quality_shift byte, with_quality bool) func(string, io.Reader) (obiseq.BioSequenceSlice, error) {
 	parser := func(source string, input io.Reader) (obiseq.BioSequenceSlice, error) {
 
 		var identifier string
@@ -249,7 +263,9 @@ func FastqChunkParser(quality_shift byte) func(string, io.Reader) (obiseq.BioSeq
 				}
 			case 10:
 				if is_end_of_line {
-					_storeSequenceQuality(&qualBytes, sequences[len(sequences)-1], quality_shift)
+					if with_quality {
+						_storeSequenceQuality(&qualBytes, sequences[len(sequences)-1], quality_shift)
+					}
 					state = 11
 				} else {
 					qualBytes.WriteByte(C)
@@ -282,12 +298,13 @@ func FastqChunkParser(quality_shift byte) func(string, io.Reader) (obiseq.BioSeq
 }
 
 func _ParseFastqFile(
-	input ChannelSeqFileChunk,
+	input ChannelFileChunk,
 	out obiiter.IBioSequence,
 	quality_shift byte,
+	with_quality bool,
 ) {
 
-	parser := FastqChunkParser(quality_shift)
+	parser := FastqChunkParser(quality_shift, with_quality)
 
 	for chunks := range input {
 		sequences, err := parser(chunks.Source, chunks.Raw)
@@ -310,9 +327,9 @@ func ReadFastq(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, e
 
 	nworker := opt.ParallelWorkers()
 
-	buff := make([]byte, 1024*1024*1024)
+	buff := make([]byte, 1024*1024)
 
-	chkchan := ReadSeqFileChunk(
+	chkchan := ReadFileChunk(
 		opt.Source(),
 		reader,
 		buff,
@@ -324,7 +341,8 @@ func ReadFastq(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, e
 		go _ParseFastqFile(
 			chkchan,
 			out,
-			byte(obioptions.InputQualityShift()),
+			obidefault.ReadQualitiesShift(),
+			opt.ReadQualities(),
 		)
 	}
 
@@ -332,7 +350,7 @@ func ReadFastq(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, e
 		out.WaitAndClose()
 	}()
 
-	newIter := out.SortBatches().Rebatch(opt.BatchSize())
+	newIter := out.SortBatches()
 
 	log.Debugln("Full file batch mode : ", opt.FullFileBatch())
 
@@ -352,9 +370,9 @@ func ReadFastq(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, e
 func ReadFastqFromFile(filename string, options ...WithOption) (obiiter.IBioSequence, error) {
 	options = append(options, OptionsSource(obiutils.RemoveAllExt((path.Base(filename)))))
 
-	file, err := Ropen(filename)
+	file, err := obiutils.Ropen(filename)
 
-	if err == ErrNoContent {
+	if err == obiutils.ErrNoContent {
 		log.Infof("file %s is empty", filename)
 		return ReadEmptyFile(options...)
 	}
@@ -368,9 +386,9 @@ func ReadFastqFromFile(filename string, options ...WithOption) (obiiter.IBioSequ
 
 func ReadFastqFromStdin(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, error) {
 	options = append(options, OptionsSource(obiutils.RemoveAllExt("stdin")))
-	input, err := Buf(os.Stdin)
+	input, err := obiutils.Buf(os.Stdin)
 
-	if err == ErrNoContent {
+	if err == obiutils.ErrNoContent {
 		log.Infof("stdin is empty")
 		return ReadEmptyFile(options...)
 	}

pkg/obiformats/fastseq_json_header.go

@@ -2,7 +2,7 @@ package obiformats
 
 import (
 	"bytes"
-	"math"
+	"strconv"
 	"strings"
 	"unsafe"
 
@@ -10,10 +10,197 @@ import (
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
-	"github.com/goccy/go-json"
+	"github.com/buger/jsonparser"
 )
 
-func _parse_json_header_(header string, annotations obiseq.Annotation) string {
+func _parse_json_map_string(str []byte, sequence *obiseq.BioSequence) (map[string]string, error) {
+	values := make(map[string]string)
+	jsonparser.ObjectEach(str,
+		func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) (err error) {
+			skey := string(key)
+			values[skey] = string(value)
+			return
+		},
+	)
+	return values, nil
+}
+
+func _parse_json_map_int(str []byte, sequence *obiseq.BioSequence) (map[string]int, error) {
+	values := make(map[string]int)
+	jsonparser.ObjectEach(str,
+		func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) (err error) {
+			skey := string(key)
+			intval, err := jsonparser.ParseInt(value)
+			if err != nil {
+				return err
+			}
+			values[skey] = int(intval)
+			return nil
+		},
+	)
+	return values, nil
+}
+
+func _parse_json_map_float(str []byte, sequence *obiseq.BioSequence) (map[string]float64, error) {
+	values := make(map[string]float64)
+	jsonparser.ObjectEach(str,
+		func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) (err error) {
+			skey := string(key)
+			floatval, err := strconv.ParseFloat(obiutils.UnsafeString(value), 64)
+			if err != nil {
+				return err
+			}
+			values[skey] = float64(floatval)
+			return nil
+		},
+	)
+	return values, nil
+}
+
+func _parse_json_map_bool(str []byte, sequence *obiseq.BioSequence) (map[string]bool, error) {
+	values := make(map[string]bool)
+	jsonparser.ObjectEach(str,
+		func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) (err error) {
+			skey := string(key)
+			boolval, err := jsonparser.ParseBoolean(value)
+			if err != nil {
+				return err
+			}
+			values[skey] = boolval
+			return nil
+		},
+	)
+	return values, nil
+}
+
+func _parse_json_map_interface(str []byte, sequence *obiseq.BioSequence) (map[string]interface{}, error) {
+	values := make(map[string]interface{})
+	jsonparser.ObjectEach(str,
+		func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) (err error) {
+			skey := string(key)
+			switch dataType {
+			case jsonparser.String:
+				values[skey] = string(value)
+			case jsonparser.Number:
+				// Try to parse the number as an int at first then as float if that fails.
+				values[skey], err = jsonparser.ParseInt(value)
+				if err != nil {
+					values[skey], err = strconv.ParseFloat(obiutils.UnsafeString(value), 64)
+				}
+				if err != nil {
+					return
+				}
+			case jsonparser.Boolean:
+			default:
+				values[skey] = string(value)
+			}
+			return
+		},
+	)
+	return values, nil
+}
+
+func _parse_json_array_string(str []byte, sequence *obiseq.BioSequence) ([]string, error) {
+	values := make([]string, 0)
+	jsonparser.ArrayEach(str,
+		func(value []byte, dataType jsonparser.ValueType, offset int, err error) {
+			if dataType == jsonparser.String {
+				skey := string(value)
+				values = append(values, skey)
+			}
+		},
+	)
+	return values, nil
+}
+
+func _parse_json_array_int(str []byte, sequence *obiseq.BioSequence) ([]int, error) {
+	values := make([]int, 0)
+	jsonparser.ArrayEach(str,
+		func(value []byte, dataType jsonparser.ValueType, offset int, err error) {
+			if dataType == jsonparser.Number {
+				intval, err := jsonparser.ParseInt(value)
+				if err != nil {
+					log.Fatalf("%s: Parsing int failed on value %s: %s", sequence.Id(), value, err)
+				}
+				values = append(values, int(intval))
+			}
+		},
+	)
+	return values, nil
+}
+
+func _parse_json_array_float(str []byte, sequence *obiseq.BioSequence) ([]float64, error) {
+	values := make([]float64, 0)
+	jsonparser.ArrayEach(str,
+		func(value []byte, dataType jsonparser.ValueType, offset int, err error) {
+			if dataType == jsonparser.Number {
+				floatval, err := strconv.ParseFloat(obiutils.UnsafeString(value), 64)
+				if err == nil {
+					values = append(values, float64(floatval))
+				} else {
+					log.Fatalf("%s: Parsing float failed on value %s: %s", sequence.Id(), value, err)
+				}
+			}
+		},
+	)
+	return values, nil
+}
+
+func _parse_json_array_bool(str []byte, sequence *obiseq.BioSequence) ([]bool, error) {
+	values := make([]bool, 0)
+	jsonparser.ArrayEach(str,
+		func(value []byte, dataType jsonparser.ValueType, offset int, err error) {
+			if dataType == jsonparser.Boolean {
+				boolval, err := jsonparser.ParseBoolean(value)
+				if err != nil {
+					log.Fatalf("%s: Parsing bool failed on value %s: %s", sequence.Id(), value, err)
+				}
+				values = append(values, boolval)
+			}
+		},
+	)
+	return values, nil
+}
+
+func _parse_json_array_interface(str []byte, sequence *obiseq.BioSequence) ([]interface{}, error) {
+	values := make([]interface{}, 0)
+	jsonparser.ArrayEach(str,
+		func(value []byte, dataType jsonparser.ValueType, offset int, err error) {
+			switch dataType {
+			case jsonparser.String:
+				values = append(values, string(value))
+			case jsonparser.Number:
+				// Try to parse the number as an int at first then as float if that fails.
+				intval, err := jsonparser.ParseInt(value)
+				if err != nil {
+					floatval, err := strconv.ParseFloat(obiutils.UnsafeString(value), 64)
+					if err != nil {
+						values = append(values, string(value))
+					} else {
+						values = append(values, floatval)
+					}
+				} else {
+					values = append(values, intval)
+				}
+			case jsonparser.Boolean:
+				boolval, err := jsonparser.ParseBoolean(value)
+				if err != nil {
+					values = append(values, string(value))
+				} else {
+					values = append(values, boolval)
+				}
+
+			default:
+				values = append(values, string(value))
+			}
+
+		},
+	)
+	return values, nil
+}
+
+func _parse_json_header_(header string, sequence *obiseq.BioSequence) string {
+	annotations := sequence.Annotations()
 	start := -1
 	stop := -1
 	level := 0
@@ -51,23 +238,123 @@ func _parse_json_header_(header string, annotations obiseq.Annotation) string {
 
 	stop++
 
-	err := json.Unmarshal([]byte(header)[start:stop], &annotations)
+	jsonparser.ObjectEach(obiutils.UnsafeBytes(header[start:stop]),
+		func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) error {
+			var err error
 
-	for k, v := range annotations {
-		switch vt := v.(type) {
-		case float64:
-			if vt == math.Floor(vt) {
-				annotations[k] = int(vt)
-			}
-			{
-				annotations[k] = vt
-			}
-		}
-	}
+			skey := obiutils.UnsafeString(key)
 
-	if err != nil {
-		log.Fatalf("annotation parsing error on %s : %v\n", header, err)
-	}
+			switch {
+			case skey == "id":
+				sequence.SetId(string(value))
+			case skey == "definition":
+				sequence.SetDefinition(string(value))
+
+			case skey == "count":
+				if dataType != jsonparser.Number {
+					log.Fatalf("%s: Count attribut must be numeric: %s", sequence.Id(), string(value))
+				}
+				count, err := jsonparser.ParseInt(value)
+				if err != nil {
+					log.Fatalf("%s: Cannot parse count %s", sequence.Id(), string(value))
+				}
+				sequence.SetCount(int(count))
+
+			case skey == "obiclean_weight":
+				weight, err := _parse_json_map_int(value, sequence)
+				if err != nil {
+					log.Fatalf("%s: Cannot parse obiclean weight %s", sequence.Id(), string(value))
+				}
+				annotations[skey] = weight
+
+			case skey == "obiclean_status":
+				status, err := _parse_json_map_string(value, sequence)
+				if err != nil {
+					log.Fatalf("%s: Cannot parse obiclean status %s", sequence.Id(), string(value))
+				}
+				annotations[skey] = status
+
+			case strings.HasPrefix(skey, "merged_"):
+				if dataType == jsonparser.Object {
+					data, err := _parse_json_map_int(value, sequence)
+					if err != nil {
+						log.Fatalf("%s: Cannot parse merged slot %s: %v", sequence.Id(), skey, err)
+					} else {
+						annotations[skey] = data
+					}
+				} else {
+					log.Fatalf("%s: Cannot parse merged slot %s", sequence.Id(), skey)
+				}
+
+			case skey == "taxid":
+				if dataType == jsonparser.Number || dataType == jsonparser.String {
+					taxid := string(value)
+					sequence.SetTaxid(taxid)
+				} else {
+					log.Fatalf("%s: Cannot parse taxid %s", sequence.Id(), string(value))
+				}
+
+			case strings.HasSuffix(skey, "_taxid"):
+				if dataType == jsonparser.Number || dataType == jsonparser.String {
+					rank, _ := obiutils.SplitInTwo(skey, '_')
+
+					taxid := string(value)
+					sequence.SetTaxid(taxid, rank)
+				} else {
+					log.Fatalf("%s: Cannot parse taxid %s", sequence.Id(), string(value))
+				}
+
+			default:
+				skey = strings.Clone(skey)
+				switch dataType {
+				case jsonparser.String:
+					annotations[skey] = string(value)
+				case jsonparser.Number:
+					// Try to parse the number as an int at first then as float if that fails.
+					annotations[skey], err = jsonparser.ParseInt(value)
+					if err != nil {
+						annotations[skey], err = strconv.ParseFloat(obiutils.UnsafeString(value), 64)
+					}
+				case jsonparser.Array:
+					annotations[skey], err = _parse_json_array_interface(value, sequence)
+				case jsonparser.Object:
+					annotations[skey], err = _parse_json_map_interface(value, sequence)
+				case jsonparser.Boolean:
+					annotations[skey], err = jsonparser.ParseBoolean(value)
+				case jsonparser.Null:
+					annotations[skey] = nil
+				default:
+					log.Fatalf("Unknown data type %v", dataType)
+				}
+			}
+
+			if err != nil {
+				annotations[skey] = "NaN"
+				log.Fatalf("%s: Cannot parse value %s assicated to key %s into a %s value",
+					sequence.Id(), string(value), skey, dataType.String())
+			}
+
+			return err
+		},
+	)
+
+	// err := json.Unmarshal([]byte(header)[start:stop], &annotations)
+
+	// for k, v := range annotations {
+	// 	switch vt := v.(type) {
+	// 	case float64:
+	// 		if vt == math.Floor(vt) {
+	// 			annotations[k] = int(vt)
+	// 		}
+	// 		{
+	// 			annotations[k] = vt
+	// 		}
+	// 	}
+	// }
+
+	// if err != nil {
+	// 	log.Fatalf("annotation parsing error on %s : %v\n", header, err)
+	// }
 
 	return strings.TrimSpace(header[stop:])
 }
@@ -78,7 +365,9 @@ func ParseFastSeqJsonHeader(sequence *obiseq.BioSequence) {
 
 	definition_part := _parse_json_header_(
 		definition,
-		sequence.Annotations())
+		sequence,
+	)
+
 	if len(definition_part) > 0 {
 		if sequence.HasDefinition() {
 			definition_part = sequence.Definition() + " " + definition_part

pkg/obiformats/fastseq_read.go

@@ -14,8 +14,8 @@ import (
 
 	log "github.com/sirupsen/logrus"
 
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 )
@@ -92,7 +92,7 @@ func ReadFastSeqFromFile(filename string, options ...WithOption) (obiiter.IBioSe
 	name := C.CString(filename)
 	defer C.free(unsafe.Pointer(name))
 
-	pointer := C.open_fast_sek_file(name, C.int32_t(obioptions.InputQualityShift()))
+	pointer := C.open_fast_sek_file(name, C.int32_t(obidefault.ReadQualitiesShift()))
 
 	var err error
 	err = nil
@@ -151,7 +151,7 @@ func ReadFastSeqFromStdin(options ...WithOption) obiiter.IBioSequence {
 	}(newIter)
 
 	go _FastseqReader(opt.Source(),
-		C.open_fast_sek_stdin(C.int32_t(obioptions.InputQualityShift())),
+		C.open_fast_sek_stdin(C.int32_t(obidefault.ReadQualitiesShift())),
 		newIter, opt.BatchSize())
 
 	log.Debugln("Full file batch mode : ", opt.FullFileBatch())

pkg/obiformats/fastseq_write_fasta.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"os"
 	"strings"
+	"time"
 
 	log "github.com/sirupsen/logrus"
 
@@ -126,14 +127,13 @@ func WriteFasta(iterator obiiter.IBioSequence,
 	options ...WithOption) (obiiter.IBioSequence, error) {
 	opt := MakeOptions(options)
 
-	iterator = iterator.Rebatch(opt.BatchSize())
 	file, _ = obiutils.CompressStream(file, opt.CompressedFile(), opt.CloseFile())
 
 	newIter := obiiter.MakeIBioSequence()
 
 	nwriters := opt.ParallelWorkers()
 
-	chunkchan := WriteSeqFileChunk(file, opt.CloseFile())
+	chunkchan := WriteFileChunk(file, opt.CloseFile())
 
 	header_format := opt.FormatFastSeqHeader()
 
@@ -141,8 +141,11 @@ func WriteFasta(iterator obiiter.IBioSequence,
 
 	go func() {
 		newIter.WaitAndClose()
+		for len(chunkchan) > 0 {
+			time.Sleep(time.Millisecond)
+		}
 		close(chunkchan)
-		log.Warnf("Writing fasta file done")
+		log.Debugf("Writing fasta file done")
 	}()
 
 	ff := func(iterator obiiter.IBioSequence) {
@@ -152,7 +155,7 @@ func WriteFasta(iterator obiiter.IBioSequence,
 
 			log.Debugf("Formating fasta chunk %d", batch.Order())
 
-			chunkchan <- SeqFileChunk{
+			chunkchan <- FileChunk{
 				Source: batch.Source(),
 				Raw:    FormatFastaBatch(batch, header_format, opt.SkipEmptySequence()),
 				Order:  batch.Order(),
@@ -167,7 +170,7 @@ func WriteFasta(iterator obiiter.IBioSequence,
 
 	log.Debugln("Start of the fasta file writing")
 	go ff(iterator)
-	for i := 0; i < nwriters-1; i++ {
+	for i := 1; i < nwriters; i++ {
 		go ff(iterator.Split())
 	}
 
@@ -184,7 +187,8 @@ func WriteFasta(iterator obiiter.IBioSequence,
 // The function returns the same bio sequence iterator and an error if any occurred.
 func WriteFastaToStdout(iterator obiiter.IBioSequence,
 	options ...WithOption) (obiiter.IBioSequence, error) {
-	options = append(options, OptionDontCloseFile())
+	// options = append(options, OptionDontCloseFile())
+	options = append(options, OptionCloseFile())
 	return WriteFasta(iterator, os.Stdout, options...)
 }
 

pkg/obiformats/fastseq_write_fastq.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"io"
 	"os"
-	"sync"
 	"time"
 
 	log "github.com/sirupsen/logrus"
@@ -87,17 +86,11 @@ func FormatFastqBatch(batch obiiter.BioSequenceBatch,
 	return &bs
 }
 
-type FileChunk struct {
-	text  []byte
-	order int
-}
-
 func WriteFastq(iterator obiiter.IBioSequence,
 	file io.WriteCloser,
 	options ...WithOption) (obiiter.IBioSequence, error) {
 
 	opt := MakeOptions(options)
-	iterator = iterator.Rebatch(opt.BatchSize())
 
 	file, _ = obiutils.CompressStream(file, opt.CompressedFile(), opt.CloseFile())
 
@@ -105,27 +98,25 @@ func WriteFastq(iterator obiiter.IBioSequence,
 
 	nwriters := opt.ParallelWorkers()
 
-	chunkchan := WriteSeqFileChunk(file, opt.CloseFile())
+	chunkchan := WriteFileChunk(file, opt.CloseFile())
 
 	header_format := opt.FormatFastSeqHeader()
 
 	newIter.Add(nwriters)
 
-	var waitWriter sync.WaitGroup
-
 	go func() {
 		newIter.WaitAndClose()
 		for len(chunkchan) > 0 {
 			time.Sleep(time.Millisecond)
 		}
 		close(chunkchan)
-		waitWriter.Wait()
+		log.Debugf("Writing fastq file done")
 	}()
 
 	ff := func(iterator obiiter.IBioSequence) {
 		for iterator.Next() {
 			batch := iterator.Get()
-			chunk := SeqFileChunk{
+			chunk := FileChunk{
 				Source: batch.Source(),
 				Raw:    FormatFastqBatch(batch, header_format, opt.SkipEmptySequence()),
 				Order:  batch.Order(),
@@ -138,7 +129,7 @@ func WriteFastq(iterator obiiter.IBioSequence,
 
 	log.Debugln("Start of the fastq file writing")
 	go ff(iterator)
-	for i := 0; i < nwriters-1; i++ {
+	for i := 1; i < nwriters; i++ {
 		go ff(iterator.Split())
 	}
 
@@ -147,7 +138,9 @@ func WriteFastq(iterator obiiter.IBioSequence,
 
 func WriteFastqToStdout(iterator obiiter.IBioSequence,
 	options ...WithOption) (obiiter.IBioSequence, error) {
-	options = append(options, OptionDontCloseFile())
+	//	options = append(options, OptionDontCloseFile())
+	options = append(options, OptionCloseFile())
+
 	return WriteFastq(iterator, os.Stdout, options...)
 }
 

pkg/obiformats/file_chunk_read.go

@@ -9,17 +9,15 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
-var _FileChunkSize = 1024 * 1024 * 10
-
 type SeqFileChunkParser func(string, io.Reader) (obiseq.BioSequenceSlice, error)
 
-type SeqFileChunk struct {
+type FileChunk struct {
 	Source string
 	Raw    *bytes.Buffer
 	Order  int
 }
 
-type ChannelSeqFileChunk chan SeqFileChunk
+type ChannelFileChunk chan FileChunk
 
 type LastSeqRecord func([]byte) int
 
@@ -36,15 +34,17 @@ type LastSeqRecord func([]byte) int
 //
 // Returns:
 // None
-func ReadSeqFileChunk(
+func ReadFileChunk(
 	source string,
 	reader io.Reader,
 	buff []byte,
-	splitter LastSeqRecord) ChannelSeqFileChunk {
+	splitter LastSeqRecord) ChannelFileChunk {
 	var err error
 	var fullbuff []byte
 
-	chunk_channel := make(ChannelSeqFileChunk)
+	chunk_channel := make(ChannelFileChunk)
+
+	fileChunkSize := len(buff)
 
 	go func() {
 		size := 0
@@ -71,11 +71,13 @@ func ReadSeqFileChunk(
 			// Read from the reader in 1 MB increments until the end of the last entry is found
 			for end = splitter(buff); err == nil && end < 0; end = splitter(buff) {
 				ic++
-				buff = slices.Grow(buff, _FileChunkSize)
+				buff = slices.Grow(buff, fileChunkSize)
 				l := len(buff)
-				extbuff := buff[l:(l + _FileChunkSize - 1)]
+				extbuff := buff[l:(l + fileChunkSize - 1)]
 				size, err = io.ReadFull(reader, extbuff)
 				buff = buff[0:(l + size)]
+				// log.Warnf("Splitter not found, attempting %d to read in %d B increments : len(buff) = %d/%d", ic, fileChunkSize, len(extbuff), len(buff))
+
 			}
 
 			fullbuff = buff
@@ -93,8 +95,10 @@ func ReadSeqFileChunk(
 				}
 
 				if len(buff) > 0 {
-					io := bytes.NewBuffer(slices.Clone(buff))
-					chunk_channel <- SeqFileChunk{source, io, i}
+					cbuff := slices.Clone(buff)
+					io := bytes.NewBuffer(cbuff)
+					// log.Warnf("chuck %d :Read %d bytes from file %s", i, io.Len(), source)
+					chunk_channel <- FileChunk{source, io, i}
 					i++
 				}
 
@@ -118,7 +122,7 @@ func ReadSeqFileChunk(
 		// Send the last chunk to the channel
 		if len(buff) > 0 {
 			io := bytes.NewBuffer(slices.Clone(buff))
-			chunk_channel <- SeqFileChunk{source, io, i}
+			chunk_channel <- FileChunk{source, io, i}
 		}
 
 		// Close the readers channel when the end of the file is reached

pkg/obiformats/file_chunk_write.go

@@ -0,0 +1,61 @@
+package obiformats
+
+import (
+	"io"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+
+	log "github.com/sirupsen/logrus"
+)
+
+func WriteFileChunk(
+	writer io.WriteCloser,
+	toBeClosed bool) ChannelFileChunk {
+
+	obiutils.RegisterAPipe()
+	chunk_channel := make(ChannelFileChunk)
+
+	go func() {
+		nextToPrint := 0
+		toBePrinted := make(map[int]FileChunk)
+		for chunk := range chunk_channel {
+			if chunk.Order == nextToPrint {
+				log.Debugf("Writing chunk: %d of length %d bytes",
+					chunk.Order,
+					len(chunk.Raw.Bytes()))
+
+				n, err := writer.Write(chunk.Raw.Bytes())
+
+				if err != nil {
+					log.Fatalf("Cannot write chunk %d only %d bytes written on %d sended : %v",
+						chunk.Order, n, len(chunk.Raw.Bytes()), err)
+				}
+				nextToPrint++
+
+				chunk, ok := toBePrinted[nextToPrint]
+				for ok {
+					log.Debug("Writing buffered chunk : ", chunk.Order)
+					_, _ = writer.Write(chunk.Raw.Bytes())
+					delete(toBePrinted, nextToPrint)
+					nextToPrint++
+					chunk, ok = toBePrinted[nextToPrint]
+				}
+			} else {
+				toBePrinted[chunk.Order] = chunk
+			}
+		}
+
+		log.Debugf("FIle have to be closed : %v", toBeClosed)
+		if toBeClosed {
+			err := writer.Close()
+			if err != nil {
+				log.Fatalf("Cannot close the writer : %v", err)
+			}
+		}
+
+		obiutils.UnregisterPipe()
+		log.Debugf("The writer has been closed")
+	}()
+
+	return chunk_channel
+}

pkg/obiformats/genbank_read.go

@@ -198,7 +198,7 @@ func GenbankChunkParser(withFeatureTable bool) func(string, io.Reader) (obiseq.B
 	}
 }
 
-func _ParseGenbankFile(input ChannelSeqFileChunk,
+func _ParseGenbankFile(input ChannelFileChunk,
 	out obiiter.IBioSequence,
 	withFeatureTable bool) {
 
@@ -223,9 +223,9 @@ func ReadGenbank(reader io.Reader, options ...WithOption) (obiiter.IBioSequence,
 	opt := MakeOptions(options)
 	// entry_channel := make(chan _FileChunk)
 
-	buff := make([]byte, 1024*1024*1024*256)
+	buff := make([]byte, 1024*1024*128) // 128 MB
 
-	entry_channel := ReadSeqFileChunk(
+	entry_channel := ReadFileChunk(
 		opt.Source(),
 		reader,
 		buff,
@@ -266,9 +266,9 @@ func ReadGenbankFromFile(filename string, options ...WithOption) (obiiter.IBioSe
 
 	options = append(options, OptionsSource(obiutils.RemoveAllExt((path.Base(filename)))))
 
-	reader, err = Ropen(filename)
+	reader, err = obiutils.Ropen(filename)
 
-	if err == ErrNoContent {
+	if err == obiutils.ErrNoContent {
 		log.Infof("file %s is empty", filename)
 		return ReadEmptyFile(options...)
 	}

pkg/obiformats/json_writer.go

@@ -7,7 +7,7 @@ import (
 	"os"
 	"strconv"
 	"strings"
-	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/goccy/go-json"
@@ -58,9 +58,17 @@ func JSONRecord(sequence *obiseq.BioSequence) []byte {
 	return text
 }
 
-func FormatJSONBatch(batch obiiter.BioSequenceBatch) []byte {
+func FormatJSONBatch(batch obiiter.BioSequenceBatch) *bytes.Buffer {
 	buff := new(bytes.Buffer)
+
 	json := bufio.NewWriter(buff)
+
+	if batch.Order() == 0 {
+		json.WriteString("[\n")
+	} else {
+		json.WriteString(",\n")
+	}
+
 	n := batch.Slice().Len() - 1
 	for i, s := range batch.Slice() {
 		json.WriteString("  ")
@@ -71,35 +79,36 @@ func FormatJSONBatch(batch obiiter.BioSequenceBatch) []byte {
 	}
 
 	json.Flush()
-
-	return buff.Bytes()
+	return buff
 }
 
 func WriteJSON(iterator obiiter.IBioSequence,
 	file io.WriteCloser,
 	options ...WithOption) (obiiter.IBioSequence, error) {
+	var latestChunk atomic.Int64
 
 	opt := MakeOptions(options)
 
 	file, _ = obiutils.CompressStream(file, opt.CompressedFile(), opt.CloseFile())
 
 	newIter := obiiter.MakeIBioSequence()
-
 	nwriters := opt.ParallelWorkers()
 
-	obiiter.RegisterAPipe()
-	chunkchan := make(chan FileChunk)
-
+	chunkchan := WriteFileChunk(file, opt.CloseFile())
 	newIter.Add(nwriters)
-	var waitWriter sync.WaitGroup
 
 	go func() {
 		newIter.WaitAndClose()
+
+		chunkchan <- FileChunk{
+			Source: "end",
+			Raw:    bytes.NewBuffer([]byte("\n]\n")),
+			Order:  int(latestChunk.Load()) + 1,
+		}
 		for len(chunkchan) > 0 {
 			time.Sleep(time.Millisecond)
 		}
 		close(chunkchan)
-		waitWriter.Wait()
 	}()
 
 	ff := func(iterator obiiter.IBioSequence) {
@@ -107,62 +116,32 @@ func WriteJSON(iterator obiiter.IBioSequence,
 
 			batch := iterator.Get()
 
-			chunkchan <- FileChunk{
-				FormatJSONBatch(batch),
-				batch.Order(),
+			ss := FileChunk{
+				Source: batch.Source(),
+				Raw:    FormatJSONBatch(batch),
+				Order:  batch.Order(),
 			}
+
+			chunkchan <- ss
+			latestChunk.Store(int64(batch.Order()))
 			newIter.Push(batch)
 		}
 		newIter.Done()
 	}
 
-	next_to_send := 0
-	received := make(map[int]FileChunk, 100)
-
-	waitWriter.Add(1)
-	go func() {
-		for chunk := range chunkchan {
-			if chunk.order == next_to_send {
-				if next_to_send > 0 {
-					file.Write([]byte(",\n"))
-				}
-				file.Write(chunk.text)
-				next_to_send++
-				chunk, ok := received[next_to_send]
-				for ok {
-					file.Write(chunk.text)
-					delete(received, next_to_send)
-					next_to_send++
-					chunk, ok = received[next_to_send]
-				}
-			} else {
-				received[chunk.order] = chunk
-			}
-
-		}
-
-		file.Write([]byte("\n]\n"))
-		file.Close()
-
-		log.Debugln("End of the JSON file writing")
-		obiiter.UnregisterPipe()
-		waitWriter.Done()
-
-	}()
-
 	log.Debugln("Start of the JSON file writing")
-	file.Write([]byte("[\n"))
-	go ff(iterator)
-	for i := 0; i < nwriters-1; i++ {
+	for i := 1; i < nwriters; i++ {
 		go ff(iterator.Split())
 	}
+	go ff(iterator)
 
 	return newIter, nil
 }
 
 func WriteJSONToStdout(iterator obiiter.IBioSequence,
 	options ...WithOption) (obiiter.IBioSequence, error) {
-	options = append(options, OptionDontCloseFile())
+	options = append(options, OptionCloseFile())
+
 	return WriteJSON(iterator, os.Stdout, options...)
 }
 

pkg/obiformats/ncbitaxdump/read.go

@@ -1,169 +0,0 @@
-package ncbitaxdump
-
-import (
-	"bufio"
-	"encoding/csv"
-	"fmt"
-	"io"
-	"os"
-	"path"
-	"strconv"
-	"strings"
-
-	log "github.com/sirupsen/logrus"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax"
-)
-
-func loadNodeTable(reader io.Reader, taxonomy *obitax.Taxonomy) {
-	file := csv.NewReader(reader)
-	file.Comma = '|'
-	file.Comment = '#'
-	file.TrimLeadingSpace = true
-	file.ReuseRecord = true
-
-	n := 0
-
-	for record, err := file.Read(); err == nil; record, err = file.Read() {
-		n++
-		taxid, err := strconv.Atoi(strings.TrimSpace(record[0]))
-
-		if err != nil {
-			log.Panicf("Cannot read taxon taxid at line %d: %v", n, err)
-		}
-
-		parent, err := strconv.Atoi(strings.TrimSpace(record[1]))
-
-		if err != nil {
-			log.Panicf("Cannot read taxon parent taxid at line %d: %v", n, err)
-		}
-
-		rank := strings.TrimSpace(record[2])
-
-		taxonomy.AddNewTaxa(taxid, parent, rank, true, true)
-	}
-
-	taxonomy.ReindexParent()
-}
-
-func loadNameTable(reader io.Reader, taxonomy *obitax.Taxonomy, onlysn bool) int {
-	// file := csv.NewReader(reader)
-	// file.Comma = '|'
-	// file.Comment = '#'
-	// file.TrimLeadingSpace = true
-	// file.ReuseRecord = true
-	// file.LazyQuotes = true
-	file := bufio.NewReader(reader)
-
-	n := 0
-	l := 0
-
-	for line, prefix, err := file.ReadLine(); err == nil; line, prefix, err = file.ReadLine() {
-		l++
-		if prefix {
-			return -1
-		}
-
-		record := strings.Split(string(line), "|")
-		taxid, err := strconv.Atoi(strings.TrimSpace(record[0]))
-
-		if err != nil {
-			log.Panicf("Cannot read taxon name taxid at line %d: %v", l, err)
-		}
-
-		name := strings.TrimSpace(record[1])
-		classname := strings.TrimSpace(record[3])
-
-		if !onlysn || classname == "scientific name" {
-			n++
-			taxonomy.AddNewName(taxid, &name, &classname)
-		}
-	}
-
-	return n
-}
-
-func loadMergedTable(reader io.Reader, taxonomy *obitax.Taxonomy) int {
-	file := csv.NewReader(reader)
-	file.Comma = '|'
-	file.Comment = '#'
-	file.TrimLeadingSpace = true
-	file.ReuseRecord = true
-
-	n := 0
-
-	for record, err := file.Read(); err == nil; record, err = file.Read() {
-		n++
-		oldtaxid, err := strconv.Atoi(strings.TrimSpace(record[0]))
-
-		if err != nil {
-			log.Panicf("Cannot read alias taxid at line %d: %v", n, err)
-		}
-		newtaxid, err := strconv.Atoi(strings.TrimSpace(record[1]))
-
-		if err != nil {
-			log.Panicf("Cannot read alias new taxid at line %d: %v", n, err)
-		}
-
-		taxonomy.AddNewAlias(newtaxid, oldtaxid)
-	}
-
-	return n
-}
-
-func LoadNCBITaxDump(directory string, onlysn bool) (*obitax.Taxonomy, error) {
-
-	taxonomy := obitax.NewTaxonomy()
-
-	//
-	// Load the Taxonomy nodes
-	//
-
-	log.Printf("Loading Taxonomy nodes\n")
-
-	nodefile, err := os.Open(path.Join(directory, "nodes.dmp"))
-	if err != nil {
-		return nil, fmt.Errorf("cannot open nodes file from '%s'",
-			directory)
-	}
-	defer nodefile.Close()
-
-	buffered := bufio.NewReader(nodefile)
-	loadNodeTable(buffered, taxonomy)
-	log.Printf("%d Taxonomy nodes read\n", taxonomy.Len())
-
-	//
-	// Load the Taxonomy nodes
-	//
-
-	log.Printf("Loading Taxon names\n")
-
-	namefile, nerr := os.Open(path.Join(directory, "names.dmp"))
-	if nerr != nil {
-		return nil, fmt.Errorf("cannot open names file from '%s'",
-			directory)
-	}
-	defer namefile.Close()
-
-	n := loadNameTable(namefile, taxonomy, onlysn)
-	log.Printf("%d taxon names read\n", n)
-
-	//
-	// Load the merged taxa
-	//
-
-	log.Printf("Loading Merged taxa\n")
-
-	aliasfile, aerr := os.Open(path.Join(directory, "merged.dmp"))
-	if aerr != nil {
-		return nil, fmt.Errorf("cannot open merged file from '%s'",
-			directory)
-	}
-	defer aliasfile.Close()
-
-	buffered = bufio.NewReader(aliasfile)
-	n = loadMergedTable(buffered, taxonomy)
-	log.Printf("%d merged taxa read\n", n)
-
-	return taxonomy, nil
-}

pkg/obiformats/ngsfilter_read.go

@@ -536,6 +536,24 @@ var library_parameter = map[string]func(library *obingslibrary.NGSLibrary, value
 	},
 }
 
+// ReadCSVNGSFilter reads an NGS filter configuration from a CSV file and returns
+// an NGSLibrary. The CSV file must include columns for 'experiment', 'sample',
+// 'sample_tag', 'forward_primer', and 'reverse_primer'. Additional columns are
+// used to annotate PCR samples.
+//
+// Parameters:
+//   - reader: an io.Reader providing the CSV input.
+//
+// Returns:
+//   - A pointer to an NGSLibrary populated with the data from the CSV file.
+//   - An error if the CSV is malformed or required columns are missing.
+//
+// The function processes both data records and parameter lines starting with
+// '@param'. Parameter lines configure various aspects of the library.
+//
+// Each row in the CSV is validated to ensure it has the correct number of columns.
+// Duplicate tag pairs for the same marker result in an error. Primer unicity is
+// checked, and any unknown parameters are logged as warnings.
 func ReadCSVNGSFilter(reader io.Reader) (*obingslibrary.NGSLibrary, error) {
 	ngsfilter := obingslibrary.MakeNGSLibrary()
 	file := csv.NewReader(reader)
@@ -576,6 +594,7 @@ func ReadCSVNGSFilter(reader io.Reader) (*obingslibrary.NGSLibrary, error) {
 	extraColumns := make([]int, 0)
 
 	for i, colName := range header {
+
 		switch colName {
 		case "experiment":
 			experimentColIndex = i
@@ -642,6 +661,8 @@ func ReadCSVNGSFilter(reader io.Reader) (*obingslibrary.NGSLibrary, error) {
 
 	}
 
+	ngsfilter.CheckPrimerUnicity()
+
 	for i := 0; i < len(params); i++ {
 		param := params[i][1]
 		if len(params[i]) < 3 {

pkg/obiformats/options.go

@@ -1,7 +1,7 @@
 package obiformats
 
 import (
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 )
 
@@ -20,6 +20,7 @@ type __options__ struct {
 	appendfile            bool
 	compressed            bool
 	skip_empty            bool
+	with_quality          bool
 	csv_id                bool
 	csv_sequence          bool
 	csv_quality           bool
@@ -48,8 +49,8 @@ func MakeOptions(setters []WithOption) Options {
 		seqBatchFormater:      nil,
 		with_progress_bar:     false,
 		buffer_size:           2,
-		parallel_workers:      obioptions.CLIReadParallelWorkers(),
-		batch_size:            obioptions.CLIBatchSize(),
+		parallel_workers:      obidefault.ReadParallelWorkers(),
+		batch_size:            obidefault.BatchSize(),
 		total_seq_size:        1024 * 1024 * 100, // 100 MB by default
 		no_order:              false,
 		full_file_batch:       false,
@@ -57,6 +58,7 @@ func MakeOptions(setters []WithOption) Options {
 		appendfile:            false,
 		compressed:            false,
 		skip_empty:            false,
+		with_quality:          true,
 		csv_id:                true,
 		csv_definition:        false,
 		csv_count:             false,
@@ -133,6 +135,10 @@ func (opt Options) SkipEmptySequence() bool {
 	return opt.pointer.skip_empty
 }
 
+func (opt Options) ReadQualities() bool {
+	return opt.pointer.with_quality
+}
+
 func (opt Options) CSVId() bool {
 	return opt.pointer.csv_id
 }
@@ -241,6 +247,14 @@ func OptionsSkipEmptySequence(skip bool) WithOption {
 	return f
 }
 
+func OptionsReadQualities(read bool) WithOption {
+	f := WithOption(func(opt Options) {
+		opt.pointer.with_quality = read
+	})
+
+	return f
+}
+
 func OptionsNewFile() WithOption {
 	f := WithOption(func(opt Options) {
 		opt.pointer.appendfile = false

pkg/obiformats/seqfile_chunk_write.go

@@ -1,51 +0,0 @@
-package obiformats
-
-import (
-	"io"
-
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
-
-	log "github.com/sirupsen/logrus"
-)
-
-func WriteSeqFileChunk(
-	writer io.WriteCloser,
-	toBeClosed bool) ChannelSeqFileChunk {
-
-	obiiter.RegisterAPipe()
-
-	chunk_channel := make(ChannelSeqFileChunk)
-
-	go func() {
-		nextToPrint := 0
-		toBePrinted := make(map[int]SeqFileChunk)
-		for chunk := range chunk_channel {
-			if chunk.Order == nextToPrint {
-				_, _ = writer.Write(chunk.Raw.Bytes())
-				nextToPrint++
-
-				chunk, ok := toBePrinted[nextToPrint]
-				for ok {
-					_, _ = writer.Write(chunk.Raw.Bytes())
-					delete(toBePrinted, nextToPrint)
-					nextToPrint++
-					chunk, ok = toBePrinted[nextToPrint]
-				}
-			} else {
-				toBePrinted[chunk.Order] = chunk
-			}
-		}
-
-		if toBeClosed {
-			err := writer.Close()
-			if err != nil {
-				log.Fatalf("Cannot close the writer : %v", err)
-			}
-		}
-
-		obiiter.UnregisterPipe()
-		log.Warnf("The writer has been closed")
-	}()
-
-	return chunk_channel
-}

pkg/obiformats/universal_read.go

@@ -3,6 +3,8 @@ package obiformats
 import (
 	"bufio"
 	"bytes"
+	"encoding/csv"
+	"errors"
 	"io"
 	"path"
 	"regexp"
@@ -39,13 +41,38 @@ type SequenceReader func(reader io.Reader, options ...WithOption) (obiiter.IBioS
 // - io.Reader: A modified reader with the read data.
 // - error: Any error encountered during the process.
 func OBIMimeTypeGuesser(stream io.Reader) (*mimetype.MIME, io.Reader, error) {
+	csv := func(in []byte, limit uint32) bool {
+		in = dropLastLine(in, limit)
+
+		br := bytes.NewReader(in)
+		r := csv.NewReader(br)
+		r.Comma = ','
+		r.ReuseRecord = true
+		r.LazyQuotes = true
+		r.Comment = '#'
+
+		lines := 0
+		for {
+			_, err := r.Read()
+			if errors.Is(err, io.EOF) {
+				break
+			}
+			if err != nil {
+				return false
+			}
+			lines++
+		}
+
+		return r.FieldsPerRecord > 1 && lines > 1
+	}
+
 	fastaDetector := func(raw []byte, limit uint32) bool {
 		ok, err := regexp.Match("^>[^ ]", raw)
 		return ok && err == nil
 	}
 
 	fastqDetector := func(raw []byte, limit uint32) bool {
-		ok, err := regexp.Match("^@[^ ].*\n[^ ]+\n\\+", raw)
+		ok, err := regexp.Match("^@[^ ].*\n[A-Za-z.-]+", raw)
 		return ok && err == nil
 	}
 
@@ -70,15 +97,17 @@ func OBIMimeTypeGuesser(stream io.Reader) (*mimetype.MIME, io.Reader, error) {
 	mimetype.Lookup("text/plain").Extend(ecoPCR2Detector, "text/ecopcr2", ".ecopcr")
 	mimetype.Lookup("text/plain").Extend(genbankDetector, "text/genbank", ".seq")
 	mimetype.Lookup("text/plain").Extend(emblDetector, "text/embl", ".dat")
+	mimetype.Lookup("text/plain").Extend(csv, "text/csv", ".csv")
 
 	mimetype.Lookup("application/octet-stream").Extend(fastaDetector, "text/fasta", ".fasta")
 	mimetype.Lookup("application/octet-stream").Extend(fastqDetector, "text/fastq", ".fastq")
 	mimetype.Lookup("application/octet-stream").Extend(ecoPCR2Detector, "text/ecopcr2", ".ecopcr")
 	mimetype.Lookup("application/octet-stream").Extend(genbankDetector, "text/genbank", ".seq")
 	mimetype.Lookup("application/octet-stream").Extend(emblDetector, "text/embl", ".dat")
+	mimetype.Lookup("application/octet-stream").Extend(csv, "text/csv", ".csv")
 
 	// Create a buffer to store the read data
-	buf := make([]byte, 1024*128)
+	buf := make([]byte, 1024*1024)
 	n, err := io.ReadFull(stream, buf)
 
 	if err != nil && err != io.ErrUnexpectedEOF {
@@ -87,6 +116,7 @@ func OBIMimeTypeGuesser(stream io.Reader) (*mimetype.MIME, io.Reader, error) {
 
 	// Detect the MIME type using the mimetype library
 	mimeType := mimetype.Detect(buf)
+
 	if mimeType == nil {
 		return nil, nil, err
 	}
@@ -142,15 +172,15 @@ func OBIMimeTypeGuesser(stream io.Reader) (*mimetype.MIME, io.Reader, error) {
 // - error: An error if any occurred during the reading process.
 func ReadSequencesFromFile(filename string,
 	options ...WithOption) (obiiter.IBioSequence, error) {
-	var file *Reader
+	var file *obiutils.Reader
 	var reader io.Reader
 	var err error
 
 	options = append(options, OptionsSource(obiutils.RemoveAllExt((path.Base(filename)))))
 
-	file, err = Ropen(filename)
+	file, err = obiutils.Ropen(filename)
 
-	if err == ErrNoContent {
+	if err == obiutils.ErrNoContent {
 		log.Infof("file %s is empty", filename)
 		return ReadEmptyFile(options...)
 	}

pkg/obiformats/universal_write.go

@@ -45,7 +45,8 @@ func WriteSequence(iterator obiiter.IBioSequence,
 
 func WriteSequencesToStdout(iterator obiiter.IBioSequence,
 	options ...WithOption) (obiiter.IBioSequence, error) {
-	options = append(options, OptionDontCloseFile())
+	// options = append(options, OptionDontCloseFile())
+	options = append(options, OptionCloseFile())
 	return WriteSequence(iterator, os.Stdout, options...)
 }
 

pkg/obifp/uint128.go

@@ -0,0 +1,289 @@
+package obifp
+
+import (
+	"math"
+	"math/bits"
+
+	log "github.com/sirupsen/logrus"
+)
+
+type Uint128 struct {
+	w1 uint64
+	w0 uint64
+}
+
+// Zero returns a zero-valued uint128.
+//
+// No parameters.
+// Returns a Uint128 value.
+func (u Uint128) Zero() Uint128 {
+	return Uint128{w1: 0, w0: 0}
+}
+
+// MaxValue returns the maximum possible value for a Uint128.
+//
+// It returns a Uint128 value with the highest possible values for high and low fields.
+func (u Uint128) MaxValue() Uint128 {
+	return Uint128{
+		w1: math.MaxUint64,
+		w0: math.MaxUint64,
+	}
+
+}
+
+// IsZero checks if the Uint128 value is zero.
+//
+// It returns a boolean indicating whether the Uint128 value is zero.
+func (u Uint128) IsZero() bool {
+	return u.w0 == 0 && u.w1 == 0
+}
+
+// Cast a Uint128 to a Uint64.
+//
+// A Warning will be logged if an overflow occurs.
+//
+// No parameters.
+// Returns a Uint64 value.
+func (u Uint128) Uint64() Uint64 {
+	if u.w1 != 0 {
+		log.Warnf("Uint128 overflow at Uint64(%v)", u)
+	}
+	return Uint64{w0: u.w0}
+}
+
+// Uint128 cast a Uint128 to a Uint128.
+//
+// Which is a no-op.
+//
+// No parameters.
+// Returns a Uint128 value.
+func (u Uint128) Uint128() Uint128 {
+	return u
+}
+
+// Cast a Uint128 to a Uint256.
+//
+// A Warning will be logged if an overflow occurs.
+//
+// No parameters.
+// Returns a Uint256 value.
+func (u Uint128) Uint256() Uint256 {
+	return Uint256{0, 0, u.w1, u.w0}
+}
+
+func (u Uint128) Set64(v uint64) Uint128 {
+
+	return Uint128{
+		w1: 0,
+		w0: v,
+	}
+}
+
+// LeftShift performs a left shift operation on the Uint128 value by the specified number of bits.
+//
+// Parameters:
+//   - n: the number of bits to shift the Uint128 value to the left.
+//
+// Returns:
+//   - Uint128: the result of the left shift operation.
+func (u Uint128) LeftShift(n uint) Uint128 {
+	lo, carry := Uint64{w0: u.w0}.LeftShift64(n, 0)
+	hi, _ := Uint64{w0: u.w1}.LeftShift64(n, carry)
+	return Uint128{w1: hi, w0: lo}
+}
+
+// RightShift performs a right shift operation on the Uint128 value by the specified number of bits.
+//
+// Parameters:
+//   - n: the number of bits to shift the Uint128 value to the right.
+//
+// Returns:
+//   - Uint128: the result of the right shift operation.
+func (u Uint128) RightShift(n uint) Uint128 {
+	hi, carry := Uint64{w0: u.w1}.RightShift64(n, 0)
+	lo, _ := Uint64{w0: u.w0}.RightShift64(n, carry)
+	return Uint128{w1: hi, w0: lo}
+}
+
+// Add performs addition of two Uint128 values and returns the result.
+//
+// Parameters:
+//   - v: the Uint128 value to add to the receiver.
+//
+// Returns:
+//   - Uint128: the result of the addition.
+func (u Uint128) Add(v Uint128) Uint128 {
+	lo, carry := bits.Add64(u.w0, v.w0, 0)
+	hi, carry := bits.Add64(u.w1, v.w1, carry)
+	if carry != 0 {
+		log.Panicf("Uint128 overflow at Add(%v, %v)", u, v)
+	}
+	return Uint128{w1: hi, w0: lo}
+}
+
+func (u Uint128) Add64(v uint64) Uint128 {
+	lo, carry := bits.Add64(u.w0, v, 0)
+	hi, carry := bits.Add64(u.w1, 0, carry)
+	if carry != 0 {
+		log.Panicf("Uint128 overflow at Add64(%v, %v)", u, v)
+	}
+	return Uint128{w1: hi, w0: lo}
+}
+
+func (u Uint128) Sub(v Uint128) Uint128 {
+	lo, borrow := bits.Sub64(u.w0, v.w0, 0)
+	hi, borrow := bits.Sub64(u.w1, v.w1, borrow)
+	if borrow != 0 {
+		log.Panicf("Uint128 underflow at Sub(%v, %v)", u, v)
+	}
+	return Uint128{w1: hi, w0: lo}
+}
+
+func (u Uint128) Mul(v Uint128) Uint128 {
+	hi, lo := bits.Mul64(u.w0, v.w0)
+	p0, p1 := bits.Mul64(u.w1, v.w0)
+	p2, p3 := bits.Mul64(u.w0, v.w1)
+	hi, c0 := bits.Add64(hi, p1, 0)
+	hi, c1 := bits.Add64(hi, p3, c0)
+	if p0 != 0 || p2 != 0 || c1 != 0 {
+		log.Panicf("Uint128 overflow at Mul(%v, %v)", u, v)
+	}
+	return Uint128{w1: hi, w0: lo}
+}
+
+func (u Uint128) Mul64(v uint64) Uint128 {
+	hi, lo := bits.Mul64(u.w0, v)
+	p0, p1 := bits.Mul64(u.w1, v)
+	hi, c0 := bits.Add64(hi, p1, 0)
+	if p0 != 0 || c0 != 0 {
+		log.Panicf("Uint128 overflow at Mul64(%v, %v)", u, v)
+	}
+	return Uint128{w1: hi, w0: lo}
+}
+
+func (u Uint128) QuoRem(v Uint128) (q, r Uint128) {
+	if v.w1 == 0 {
+		var r64 uint64
+		q, r64 = u.QuoRem64(v.w0)
+		r = Uint128{w1: 0, w0: r64}
+	} else {
+		// generate a "trial quotient," guaranteed to be within 1 of the actual
+		// quotient, then adjust.
+		n := uint(bits.LeadingZeros64(v.w1))
+		v1 := v.LeftShift(n)
+		u1 := u.RightShift(1)
+		tq, _ := bits.Div64(u1.w1, u1.w0, v1.w1)
+		tq >>= 63 - n
+		if tq != 0 {
+			tq--
+		}
+		q = Uint128{w1: 0, w0: tq}
+		// calculate remainder using trial quotient, then adjust if remainder is
+		// greater than divisor
+		r = u.Sub(v.Mul64(tq))
+		if r.Cmp(v) >= 0 {
+			q = q.Add64(1)
+			r = r.Sub(v)
+		}
+	}
+	return
+}
+
+// QuoRem64 returns q = u/v and r = u%v.
+func (u Uint128) QuoRem64(v uint64) (q Uint128, r uint64) {
+	if u.w1 < v {
+		q.w0, r = bits.Div64(u.w1, u.w0, v)
+	} else {
+		q.w1, r = bits.Div64(0, u.w1, v)
+		q.w0, r = bits.Div64(r, u.w0, v)
+	}
+	return
+}
+
+func (u Uint128) Div(v Uint128) Uint128 {
+	q, _ := u.QuoRem(v)
+	return q
+}
+
+func (u Uint128) Div64(v uint64) Uint128 {
+	q, _ := u.QuoRem64(v)
+	return q
+}
+
+func (u Uint128) Mod(v Uint128) Uint128 {
+	_, r := u.QuoRem(v)
+	return r
+}
+
+func (u Uint128) Mod64(v uint64) uint64 {
+	_, r := u.QuoRem64(v)
+	return r
+}
+
+func (u Uint128) Cmp(v Uint128) int {
+	switch {
+	case u.w1 > v.w1:
+		return 1
+	case u.w1 < v.w1:
+		return -1
+	case u.w0 > v.w0:
+		return 1
+	case u.w0 < v.w0:
+		return -1
+	default:
+		return 0
+	}
+}
+
+func (u Uint128) Cmp64(v uint64) int {
+	switch {
+	case u.w1 > 0:
+		return 1
+	case u.w0 > v:
+		return 1
+	case u.w0 < v:
+		return -1
+	default:
+		return 0
+	}
+}
+
+func (u Uint128) Equals(v Uint128) bool {
+	return u.Cmp(v) == 0
+}
+
+func (u Uint128) LessThan(v Uint128) bool {
+	return u.Cmp(v) < 0
+}
+
+func (u Uint128) GreaterThan(v Uint128) bool {
+	return u.Cmp(v) > 0
+}
+
+func (u Uint128) LessThanOrEqual(v Uint128) bool {
+	return !u.GreaterThan(v)
+}
+
+func (u Uint128) GreaterThanOrEqual(v Uint128) bool {
+	return !u.LessThan(v)
+}
+
+func (u Uint128) And(v Uint128) Uint128 {
+	return Uint128{w1: u.w1 & v.w1, w0: u.w0 & v.w0}
+}
+
+func (u Uint128) Or(v Uint128) Uint128 {
+	return Uint128{w1: u.w1 | v.w1, w0: u.w0 | v.w0}
+}
+
+func (u Uint128) Xor(v Uint128) Uint128 {
+	return Uint128{w1: u.w1 ^ v.w1, w0: u.w0 ^ v.w0}
+}
+
+func (u Uint128) Not() Uint128 {
+	return Uint128{w1: ^u.w1, w0: ^u.w0}
+}
+
+func (u Uint128) AsUint64() uint64 {
+	return u.w0
+}

pkg/obifp/uint128_test.go

@@ -0,0 +1,250 @@
+package obifp
+
+import (
+	"math"
+	"reflect"
+
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestUint128_Add(t *testing.T) {
+	u := Uint128{w1: 1, w0: 2}
+	v := Uint128{w1: 3, w0: 4}
+	w := u.Add(v)
+	assert.Equal(t, Uint128{w1: 4, w0: 6}, w)
+
+	u = Uint128{w1: 0, w0: 0}
+	v = Uint128{w1: 0, w0: 0}
+	w = u.Add(v)
+	assert.Equal(t, Uint128{w1: 0, w0: 0}, w)
+
+	u = Uint128{w1: 0, w0: math.MaxUint64}
+	v = Uint128{w1: 0, w0: 1}
+	w = u.Add(v)
+	assert.Equal(t, Uint128{w1: 1, w0: 0}, w)
+}
+
+func TestUint128_Add64(t *testing.T) {
+	u := Uint128{w1: 1, w0: 2}
+	v := uint64(3)
+	w := u.Add64(v)
+	assert.Equal(t, Uint128{w1: 1, w0: 5}, w)
+
+	u = Uint128{w1: 0, w0: 0}
+	v = uint64(0)
+	w = u.Add64(v)
+	assert.Equal(t, Uint128{w1: 0, w0: 0}, w)
+
+	u = Uint128{w1: 0, w0: math.MaxUint64}
+	v = uint64(1)
+	w = u.Add64(v)
+	assert.Equal(t, Uint128{w1: 1, w0: 0}, w)
+}
+
+func TestUint128_Sub(t *testing.T) {
+	u := Uint128{w1: 4, w0: 6}
+	v := Uint128{w1: 3, w0: 4}
+	w := u.Sub(v)
+	assert.Equal(t, Uint128{w1: 1, w0: 2}, w)
+
+	u = Uint128{w1: 0, w0: 0}
+	v = Uint128{w1: 0, w0: 0}
+	w = u.Sub(v)
+	assert.Equal(t, Uint128{w1: 0, w0: 0}, w)
+
+	u = Uint128{w1: 1, w0: 0}
+	v = Uint128{w1: 0, w0: 1}
+	w = u.Sub(v)
+	assert.Equal(t, Uint128{w1: 0, w0: math.MaxUint64}, w)
+}
+
+func TestUint128_Mul64(t *testing.T) {
+	u := Uint128{w1: 1, w0: 2}
+	v := uint64(3)
+	w := u.Mul64(v)
+
+	if w.w1 != 3 || w.w0 != 6 {
+		t.Errorf("Mul64(%v, %v) = %v, want %v", u, v, w, Uint128{w1: 3, w0: 6})
+	}
+
+	u = Uint128{w1: 0, w0: 0}
+	v = uint64(0)
+	w = u.Mul64(v)
+
+	if w.w1 != 0 || w.w0 != 0 {
+		t.Errorf("Mul64(%v, %v) = %v, want %v", u, v, w, Uint128{w1: 0, w0: 0})
+	}
+
+	u = Uint128{w1: 0, w0: math.MaxUint64}
+	v = uint64(2)
+	w = u.Mul64(v)
+
+	if w.w1 != 1 || w.w0 != 18446744073709551614 {
+		t.Errorf("Mul64(%v, %v) = %v, want %v", u, v, w, Uint128{w1: 1, w0: 18446744073709551614})
+	}
+
+}
+
+func TestUint128_Mul(t *testing.T) {
+	tests := []struct {
+		name     string
+		u        Uint128
+		v        Uint128
+		expected Uint128
+	}{
+		{
+			name:     "simple multiplication",
+			u:        Uint128{w1: 1, w0: 2},
+			v:        Uint128{w1: 3, w0: 4},
+			expected: Uint128{w1: 10, w0: 8},
+		},
+		{
+			name:     "multiplication with overflow",
+			u:        Uint128{w1: 0, w0: math.MaxUint64},
+			v:        Uint128{w1: 0, w0: 2},
+			expected: Uint128{w1: 1, w0: 18446744073709551614},
+		},
+		{
+			name:     "multiplication with zero",
+			u:        Uint128{w1: 0, w0: 0},
+			v:        Uint128{w1: 0, w0: 0},
+			expected: Uint128{w1: 0, w0: 0},
+		},
+		{
+			name:     "multiplication with large numbers",
+			u:        Uint128{w1: 100, w0: 200},
+			v:        Uint128{w1: 300, w0: 400},
+			expected: Uint128{w1: 100000, w0: 80000},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			actual := tt.u.Mul(tt.v)
+			if !reflect.DeepEqual(actual, tt.expected) {
+				t.Errorf("Mul(%v, %v) = %v, want %v", tt.u, tt.v, actual, tt.expected)
+			}
+		})
+	}
+}
+
+func TestUint128_QuoRem(t *testing.T) {
+	u := Uint128{w1: 3, w0: 8}
+	v := Uint128{w1: 0, w0: 4}
+	q, r := u.QuoRem(v)
+	assert.Equal(t, Uint128{w1: 0, w0: 2}, q)
+	assert.Equal(t, Uint128{w1: 0, w0: 0}, r)
+}
+
+func TestUint128_QuoRem64(t *testing.T) {
+	u := Uint128{w1: 0, w0: 6}
+	v := uint64(3)
+	q, r := u.QuoRem64(v)
+	assert.Equal(t, Uint128{w1: 0, w0: 2}, q)
+	assert.Equal(t, uint64(0), r)
+}
+
+func TestUint128_Div(t *testing.T) {
+	u := Uint128{w1: 3, w0: 8}
+	v := Uint128{w1: 0, w0: 4}
+	q := u.Div(v)
+	assert.Equal(t, Uint128{w1: 0, w0: 2}, q)
+}
+
+func TestUint128_Div64(t *testing.T) {
+	u := Uint128{w1: 0, w0: 6}
+	v := uint64(3)
+	q := u.Div64(v)
+	assert.Equal(t, Uint128{w1: 0, w0: 2}, q)
+}
+
+func TestUint128_Mod(t *testing.T) {
+	u := Uint128{w1: 3, w0: 8}
+	v := Uint128{w1: 0, w0: 4}
+	r := u.Mod(v)
+	assert.Equal(t, Uint128{w1: 0, w0: 0}, r)
+}
+
+func TestUint128_Mod64(t *testing.T) {
+	u := Uint128{w1: 0, w0: 6}
+	v := uint64(3)
+	r := u.Mod64(v)
+	assert.Equal(t, uint64(0), r)
+}
+
+func TestUint128_Cmp(t *testing.T) {
+	u := Uint128{w1: 1, w0: 2}
+	v := Uint128{w1: 3, w0: 4}
+	assert.Equal(t, -1, u.Cmp(v))
+}
+
+func TestUint128_Cmp64(t *testing.T) {
+	u := Uint128{w1: 1, w0: 2}
+	v := uint64(3)
+	assert.Equal(t, -1, u.Cmp64(v))
+}
+
+func TestUint128_Equals(t *testing.T) {
+	u := Uint128{w1: 1, w0: 2}
+	v := Uint128{w1: 1, w0: 2}
+	assert.Equal(t, true, u.Equals(v))
+}
+
+func TestUint128_LessThan(t *testing.T) {
+	u := Uint128{w1: 1, w0: 2}
+	v := Uint128{w1: 3, w0: 4}
+	assert.Equal(t, true, u.LessThan(v))
+}
+
+func TestUint128_GreaterThan(t *testing.T) {
+	u := Uint128{w1: 1, w0: 2}
+	v := Uint128{w1: 3, w0: 4}
+	assert.Equal(t, false, u.GreaterThan(v))
+}
+
+func TestUint128_LessThanOrEqual(t *testing.T) {
+	u := Uint128{w1: 1, w0: 2}
+	v := Uint128{w1: 3, w0: 4}
+	assert.Equal(t, true, u.LessThanOrEqual(v))
+}
+
+func TestUint128_GreaterThanOrEqual(t *testing.T) {
+	u := Uint128{w1: 1, w0: 2}
+	v := Uint128{w1: 3, w0: 4}
+	assert.Equal(t, false, u.GreaterThanOrEqual(v))
+}
+
+func TestUint128_And(t *testing.T) {
+	u := Uint128{w1: 1, w0: 2}
+	v := Uint128{w1: 3, w0: 4}
+	w := u.And(v)
+	assert.Equal(t, Uint128{w1: 1, w0: 0}, w)
+}
+
+func TestUint128_Or(t *testing.T) {
+	u := Uint128{w1: 1, w0: 2}
+	v := Uint128{w1: 3, w0: 4}
+	w := u.Or(v)
+	assert.Equal(t, Uint128{w1: 3, w0: 6}, w)
+}
+
+func TestUint128_Xor(t *testing.T) {
+	u := Uint128{w1: 1, w0: 2}
+	v := Uint128{w1: 3, w0: 4}
+	w := u.Xor(v)
+	assert.Equal(t, Uint128{w1: 2, w0: 6}, w)
+}
+
+func TestUint128_Not(t *testing.T) {
+	u := Uint128{w1: 1, w0: 2}
+	w := u.Not()
+	assert.Equal(t, Uint128{w1: math.MaxUint64 - 1, w0: math.MaxUint64 - 2}, w)
+}
+
+func TestUint128_AsUint64(t *testing.T) {
+	u := Uint128{w1: 0, w0: 5}
+	v := u.AsUint64()
+	assert.Equal(t, uint64(5), v)
+}

pkg/obifp/uint256.go

@@ -0,0 +1,307 @@
+package obifp
+
+import (
+	"math"
+	"math/bits"
+
+	log "github.com/sirupsen/logrus"
+)
+
+type Uint256 struct {
+	w3 uint64
+	w2 uint64
+	w1 uint64
+	w0 uint64
+}
+
+// Zero returns a zero value of type Uint256.
+//
+// No parameters.
+// Returns a Uint256 value of 0.
+func (u Uint256) Zero() Uint256 {
+	return Uint256{}
+}
+
+// MaxValue returns the maximum possible value of type Uint256.
+//
+// No parameters.
+// Returns the maximum value of type Uint256.
+func (u Uint256) MaxValue() Uint256 {
+	return Uint256{
+		w3: math.MaxUint64,
+		w2: math.MaxUint64,
+		w1: math.MaxUint64,
+		w0: math.MaxUint64,
+	}
+}
+
+// IsZero checks if the Uint256 value is zero.
+//
+// No parameters.
+// Returns a boolean indicating if the value is zero.
+func (u Uint256) IsZero() bool {
+	return u == Uint256{}
+}
+
+// Cast a Uint256 to a Uint64.
+//
+// A Warning will be logged if an overflow occurs.
+//
+// No parameters.
+// Returns a Uint64 value.
+func (u Uint256) Uint64() Uint64 {
+	if u.w3 != 0 || u.w2 != 0 || u.w1 != 0 {
+		log.Warnf("Uint256 overflow at Uint64(%v)", u)
+	}
+	return Uint64{w0: u.w0}
+}
+
+// Cast a Uint256 to a Uint128.
+//
+// A Warning will be logged if an overflow occurs.
+//
+// No parameters.
+// Returns a Uint128 value.
+func (u Uint256) Uint128() Uint128 {
+	if u.w3 != 0 || u.w2 != 0 {
+		log.Warnf("Uint256 overflow at Uint128(%v)", u)
+	}
+	return Uint128{u.w1, u.w0}
+}
+
+// Cast a Uint128 to a Uint256.
+//
+// A Warning will be logged if an overflow occurs.
+//
+// No parameters.
+// Returns a Uint256 value.
+func (u Uint256) Uint256() Uint256 {
+	return u
+}
+
+func (u Uint256) Set64(v uint64) Uint256 {
+
+	return Uint256{
+		w3: 0,
+		w2: 0,
+		w1: 0,
+		w0: v,
+	}
+}
+
+func (u Uint256) LeftShift(n uint) Uint256 {
+	w0, carry := Uint64{w0: u.w0}.LeftShift64(n, 0)
+	w1, carry := Uint64{w0: u.w1}.LeftShift64(n, carry)
+	w2, carry := Uint64{w0: u.w2}.LeftShift64(n, carry)
+	w3, _ := Uint64{w0: u.w3}.LeftShift64(n, carry)
+	return Uint256{w3, w2, w1, w0}
+}
+
+func (u Uint256) RightShift(n uint) Uint256 {
+	w3, carry := Uint64{w0: u.w3}.RightShift64(n, 0)
+	w2, carry := Uint64{w0: u.w2}.RightShift64(n, carry)
+	w1, carry := Uint64{w0: u.w1}.RightShift64(n, carry)
+	w0, _ := Uint64{w0: u.w0}.RightShift64(n, carry)
+	return Uint256{w3, w2, w1, w0}
+}
+
+func (u Uint256) Cmp(v Uint256) int {
+	switch {
+	case u.w3 > v.w3:
+		return 1
+	case u.w3 < v.w3:
+		return -1
+	case u.w2 > v.w2:
+		return 1
+	case u.w2 < v.w2:
+		return -1
+	case u.w1 > v.w1:
+		return 1
+	case u.w1 < v.w1:
+		return -1
+	case u.w0 > v.w0:
+		return 1
+	case u.w0 < v.w0:
+		return -1
+	default:
+		return 0
+	}
+}
+
+// Add performs addition of two Uint256 values and returns the result.
+//
+// Parameters:
+//   - v: the Uint256 value to add to the receiver.
+//
+// Returns:
+//   - Uint256: the result of the addition.
+func (u Uint256) Add(v Uint256) Uint256 {
+	w0, carry := bits.Add64(u.w0, v.w0, 0)
+	w1, carry := bits.Add64(u.w1, v.w1, carry)
+	w2, carry := bits.Add64(u.w2, v.w2, carry)
+	w3, carry := bits.Add64(u.w3, v.w3, carry)
+	if carry != 0 {
+		log.Panicf("Uint256 overflow at Add(%v, %v)", u, v)
+	}
+	return Uint256{w3, w2, w1, w0}
+}
+
+// Sub performs subtraction of two Uint256 values and returns the result.
+//
+// Parameters:
+//   - v: the Uint256 value to subtract from the receiver.
+//
+// Returns:
+//   - Uint256: the result of the subtraction.
+func (u Uint256) Sub(v Uint256) Uint256 {
+	w0, borrow := bits.Sub64(u.w0, v.w0, 0)
+	w1, borrow := bits.Sub64(u.w1, v.w1, borrow)
+	w2, borrow := bits.Sub64(u.w2, v.w2, borrow)
+	w3, borrow := bits.Sub64(u.w3, v.w3, borrow)
+	if borrow != 0 {
+		log.Panicf("Uint256 overflow at Sub(%v, %v)", u, v)
+	}
+	return Uint256{w3, w2, w1, w0}
+}
+
+// Mul performs multiplication of two Uint256 values and returns the result.
+//
+// Parameters:
+//   - v: the Uint256 value to multiply with the receiver.
+//
+// Returns:
+//   - Uint256: the result of the multiplication.
+func (u Uint256) Mul(v Uint256) Uint256 {
+	var w0, w1, w2, w3, carry uint64
+
+	w0Low, w0High := bits.Mul64(u.w0, v.w0)
+	w1Low1, w1High1 := bits.Mul64(u.w0, v.w1)
+	w1Low2, w1High2 := bits.Mul64(u.w1, v.w0)
+	w2Low1, w2High1 := bits.Mul64(u.w0, v.w2)
+	w2Low2, w2High2 := bits.Mul64(u.w1, v.w1)
+	w2Low3, w2High3 := bits.Mul64(u.w2, v.w0)
+	w3Low1, w3High1 := bits.Mul64(u.w0, v.w3)
+	w3Low2, w3High2 := bits.Mul64(u.w1, v.w2)
+	w3Low3, w3High3 := bits.Mul64(u.w2, v.w1)
+	w3Low4, w3High4 := bits.Mul64(u.w3, v.w0)
+
+	w0 = w0Low
+
+	w1, carry = bits.Add64(w1Low1, w1Low2, 0)
+	w1, _ = bits.Add64(w1, w0High, carry)
+
+	w2, carry = bits.Add64(w2Low1, w2Low2, 0)
+	w2, carry = bits.Add64(w2, w2Low3, carry)
+	w2, carry = bits.Add64(w2, w1High1, carry)
+	w2, _ = bits.Add64(w2, w1High2, carry)
+
+	w3, carry = bits.Add64(w3Low1, w3Low2, 0)
+	w3, carry = bits.Add64(w3, w3Low3, carry)
+	w3, carry = bits.Add64(w3, w3Low4, carry)
+	w3, carry = bits.Add64(w3, w2High1, carry)
+	w3, carry = bits.Add64(w3, w2High2, carry)
+	w3, carry = bits.Add64(w3, w2High3, carry)
+
+	if w3High1 != 0 || w3High2 != 0 || w3High3 != 0 || w3High4 != 0 || carry != 0 {
+		log.Panicf("Uint256 overflow at Mul(%v, %v)", u, v)
+	}
+
+	return Uint256{w3, w2, w1, w0}
+}
+
+// Div performs division of two Uint256 values and returns the result.
+//
+// Parameters:
+//   - v: the Uint256 value to divide with the receiver.
+//
+// Returns:
+//   - Uint256: the result of the division.
+func (u Uint256) Div(v Uint256) Uint256 {
+	if v.IsZero() {
+		log.Panicf("division by zero")
+	}
+
+	if u.IsZero() || u.LessThan(v) {
+		return Uint256{}
+	}
+
+	if v.Equals(Uint256{0, 0, 0, 1}) {
+		return u // Division by 1
+	}
+
+	var q, r Uint256
+	r = u
+
+	for r.GreaterThanOrEqual(v) {
+		var t Uint256 = v
+		var m Uint256 = Uint256{0, 0, 0, 1}
+		for t.LeftShift(1).LessThanOrEqual(r) {
+			t = t.LeftShift(1)
+			m = m.LeftShift(1)
+		}
+		r = r.Sub(t)
+		q = q.Add(m)
+	}
+
+	return q
+}
+
+func (u Uint256) Equals(v Uint256) bool {
+	return u.Cmp(v) == 0
+}
+
+func (u Uint256) LessThan(v Uint256) bool {
+	return u.Cmp(v) < 0
+}
+
+func (u Uint256) GreaterThan(v Uint256) bool {
+	return u.Cmp(v) > 0
+}
+
+func (u Uint256) LessThanOrEqual(v Uint256) bool {
+	return !u.GreaterThan(v)
+}
+
+func (u Uint256) GreaterThanOrEqual(v Uint256) bool {
+	return !u.LessThan(v)
+}
+
+func (u Uint256) And(v Uint256) Uint256 {
+	return Uint256{
+		w3: u.w3 & v.w3,
+		w2: u.w2 & v.w2,
+		w1: u.w1 & v.w1,
+		w0: u.w0 & v.w0,
+	}
+}
+
+func (u Uint256) Or(v Uint256) Uint256 {
+	return Uint256{
+		w3: u.w3 | v.w3,
+		w2: u.w2 | v.w2,
+		w1: u.w1 | v.w1,
+		w0: u.w0 | v.w0,
+	}
+}
+
+func (u Uint256) Xor(v Uint256) Uint256 {
+	return Uint256{
+		w3: u.w3 ^ v.w3,
+		w2: u.w2 ^ v.w2,
+		w1: u.w1 ^ v.w1,
+		w0: u.w0 ^ v.w0,
+	}
+}
+
+func (u Uint256) Not() Uint256 {
+	return Uint256{
+		w3: ^u.w3,
+		w2: ^u.w2,
+		w1: ^u.w1,
+		w0: ^u.w0,
+	}
+}
+
+func (u Uint256) AsUint64() uint64 {
+	return u.w0
+}

pkg/obifp/uint64.go

@@ -0,0 +1,237 @@
+package obifp
+
+import (
+	"math"
+	"math/bits"
+
+	log "github.com/sirupsen/logrus"
+)
+
+type Uint64 struct {
+	w0 uint64
+}
+
+// Zero returns a zero value of type Uint64.
+//
+// No parameters.
+// Returns a Uint64 value of 0.
+func (u Uint64) Zero() Uint64 {
+	return Uint64{0}
+}
+
+// MaxValue returns the maximum possible value of type Uint64.
+//
+// No parameters.
+// Returns the maximum value of type Uint64.
+func (u Uint64) MaxValue() Uint64 {
+	return Uint64{math.MaxUint64}
+}
+
+// IsZero checks if the Uint64 value is zero.
+//
+// No parameters.
+// Returns a boolean indicating if the value is zero.
+func (u Uint64) IsZero() bool {
+	return u.w0 == 0
+}
+
+// Cast a Uint64 to a Uint64.
+//
+// Which is a no-op.
+//
+// No parameters.
+// Returns the Uint64 value itself.
+func (u Uint64) Uint64() Uint64 {
+	return u
+}
+
+// Cast a Uint64 to a Uint128.
+//
+// No parameters.
+// Returns a Uint128 value with the high field set to 0 and the low field set to the value of the Uint64.
+func (u Uint64) Uint128() Uint128 {
+	return Uint128{w1: 0, w0: u.w0}
+}
+
+// Cast a Uint64 to a Uint256.
+//
+// No parameters.
+// Returns a Uint256 value with the high fields set to 0 and the low fields set to the value of the Uint64.
+func (u Uint64) Uint256() Uint256 {
+	return Uint256{w3: 0, w2: 0, w1: 0, w0: u.w0}
+}
+
+func (u Uint64) Set64(v uint64) Uint64 {
+
+	return Uint64{
+		w0: v,
+	}
+}
+
+// LeftShift64 performs a left shift operation on the Uint64 value by n bits, with carry-in from carryIn.
+//
+// The carry-in value is used as the first bit of the shifted value.
+//
+// The function returns u << n | (carryIn & ((1 << n) - 1)).
+//
+// This is a shift left operation, lowest bits are set with the lowest bits of
+// the carry-in value instead of 0 as they would be in classical a left shift operation.
+//
+// Parameters:
+// - n: the number of bits to shift by.
+// - carryIn: the carry-in value.
+//
+// Returns:
+// - value: the result of the left shift operation.
+// - carry: the carry-out value.
+func (u Uint64) LeftShift64(n uint, carryIn uint64) (value, carry uint64) {
+	switch {
+	case n == 0:
+		return u.w0, 0
+
+	case n < 64:
+		return u.w0<<n | (carryIn & ((1 << n) - 1)), u.w0 >> (64 - n)
+
+	case n == 64:
+		return carryIn, u.w0
+
+	case n < 128:
+		return carryIn, u.w0 << (n - 64)
+
+	}
+
+	log.Warnf("Uint64 overflow at LeftShift64(%v, %v)", u, n)
+	return 0, 0
+}
+
+// RightShift64 performs a right shift operation on the Uint64 value by n bits, with carry-out to carry.
+//
+// The function returns the result of the right shift operation and the carry-out value.
+//
+// Parameters:
+// - n: the number of bits to shift by.
+//
+// Returns:
+// - value: the result of the right shift operation.
+// - carry: the carry-out value.
+func (u Uint64) RightShift64(n uint, carryIn uint64) (value, carry uint64) {
+	switch {
+	case n == 0:
+		return u.w0, 0
+
+	case n < 64:
+		return u.w0>>n | (carryIn & ^((1 << (64 - n)) - 1)), u.w0 << (64 - n)
+
+	case n == 64:
+		return carryIn, u.w0
+
+	case n < 128:
+		return carryIn, u.w0 >> (n - 64)
+	}
+
+	log.Warnf("Uint64 overflow at RightShift64(%v, %v)", u, n)
+	return 0, 0
+}
+
+func (u Uint64) Add64(v Uint64, carryIn uint64) (value, carry uint64) {
+	return bits.Add64(u.w0, v.w0, uint64(carryIn))
+}
+
+func (u Uint64) Sub64(v Uint64, carryIn uint64) (value, carry uint64) {
+	return bits.Sub64(u.w0, v.w0, uint64(carryIn))
+}
+
+func (u Uint64) Mul64(v Uint64) (value, carry uint64) {
+	return bits.Mul64(u.w0, v.w0)
+}
+
+func (u Uint64) LeftShift(n uint) Uint64 {
+	sl, _ := u.LeftShift64(n, 0)
+	return Uint64{w0: sl}
+}
+
+func (u Uint64) RightShift(n uint) Uint64 {
+	sr, _ := u.RightShift64(n, 0)
+	return Uint64{w0: sr}
+}
+
+func (u Uint64) Add(v Uint64) Uint64 {
+	value, carry := u.Add64(v, 0)
+
+	if carry != 0 {
+		log.Panicf("Uint64 overflow at Add(%v, %v)", u, v)
+	}
+
+	return Uint64{w0: value}
+}
+
+func (u Uint64) Sub(v Uint64) Uint64 {
+	value, carry := u.Sub64(v, 0)
+
+	if carry != 0 {
+		log.Panicf("Uint64 overflow at Sub(%v, %v)", u, v)
+	}
+
+	return Uint64{w0: value}
+}
+
+func (u Uint64) Mul(v Uint64) Uint64 {
+	value, carry := u.Mul64(v)
+
+	if carry != 0 {
+		log.Panicf("Uint64 overflow at Mul(%v, %v)", u, v)
+	}
+
+	return Uint64{w0: value}
+}
+
+func (u Uint64) Cmp(v Uint64) int {
+	switch {
+	case u.w0 < v.w0:
+		return -1
+	case u.w0 > v.w0:
+		return 1
+	default:
+		return 0
+	}
+}
+
+func (u Uint64) Equals(v Uint64) bool {
+	return u.Cmp(v) == 0
+}
+
+func (u Uint64) LessThan(v Uint64) bool {
+	return u.Cmp(v) < 0
+}
+
+func (u Uint64) GreaterThan(v Uint64) bool {
+	return u.Cmp(v) > 0
+}
+
+func (u Uint64) LessThanOrEqual(v Uint64) bool {
+	return !u.GreaterThan(v)
+}
+
+func (u Uint64) GreaterThanOrEqual(v Uint64) bool {
+	return !u.LessThan(v)
+}
+
+func (u Uint64) And(v Uint64) Uint64 {
+	return Uint64{w0: u.w0 & v.w0}
+}
+
+func (u Uint64) Or(v Uint64) Uint64 {
+	return Uint64{w0: u.w0 | v.w0}
+}
+
+func (u Uint64) Xor(v Uint64) Uint64 {
+	return Uint64{w0: u.w0 ^ v.w0}
+}
+
+func (u Uint64) Not() Uint64 {
+	return Uint64{w0: ^u.w0}
+}
+
+func (u Uint64) AsUint64() uint64 {
+	return u.w0
+}

pkg/obifp/unint.go

@@ -0,0 +1,41 @@
+package obifp
+
+type FPUint[T Uint64 | Uint128 | Uint256] interface {
+	Zero() T
+	Set64(v uint64) T
+
+	IsZero() bool
+	LeftShift(n uint) T
+	RightShift(n uint) T
+
+	Add(v T) T
+	Sub(v T) T
+	Mul(v T) T
+	//Div(v T) T
+
+	And(v T) T
+	Or(v T) T
+	Xor(v T) T
+	Not() T
+
+	LessThan(v T) bool
+	LessThanOrEqual(v T) bool
+	GreaterThan(v T) bool
+	GreaterThanOrEqual(v T) bool
+
+	AsUint64() uint64
+
+	Uint64 | Uint128 | Uint256
+}
+
+func ZeroUint[T FPUint[T]]() T {
+	return *new(T)
+}
+
+func OneUint[T FPUint[T]]() T {
+	return ZeroUint[T]().Set64(1)
+}
+
+func From64[T FPUint[T]](v uint64) T {
+	return ZeroUint[T]().Set64(v)
+}

pkg/obiiter/batch.go

@@ -108,14 +108,3 @@ func (batch BioSequenceBatch) Pop0() *obiseq.BioSequence {
 func (batch BioSequenceBatch) IsNil() bool {
 	return batch.slice == nil
 }
-
-// Recycle cleans up the BioSequenceBatch by recycling its elements and resetting its slice.
-//
-// If including_seq is true, each element of the BioSequenceBatch's slice is recycled using the Recycle method,
-// and then set to nil. If including_seq is false, each element is simply set to nil.
-//
-// This function does not return anything.
-func (batch BioSequenceBatch) Recycle(including_seq bool) {
-	batch.slice.Recycle(including_seq)
-	batch.slice = nil
-}

pkg/obiiter/batchiterator.go

@@ -10,31 +10,12 @@ import (
 
 	log "github.com/sirupsen/logrus"
 
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
 	"github.com/tevino/abool/v2"
 )
 
-var globalLocker sync.WaitGroup
-var globalLockerCounter = 0
-
-func RegisterAPipe() {
-	globalLocker.Add(1)
-	globalLockerCounter++
-	log.Debugln(globalLockerCounter, " Pipes are registered now")
-}
-
-func UnregisterPipe() {
-	globalLocker.Done()
-	globalLockerCounter--
-	log.Debugln(globalLockerCounter, "are still registered")
-}
-
-func WaitForLastPipe() {
-	globalLocker.Wait()
-}
-
 // Structure implementing an iterator over bioseq.BioSequenceBatch
 // based on a channel.
 type _IBioSequence struct {
@@ -78,7 +59,7 @@ func MakeIBioSequence() IBioSequence {
 	i.lock = &lock
 	ii := IBioSequence{&i}
 
-	RegisterAPipe()
+	obiutils.RegisterAPipe()
 
 	return ii
 }
@@ -245,7 +226,7 @@ func (iterator IBioSequence) Push(batch BioSequenceBatch) {
 
 func (iterator IBioSequence) Close() {
 	close(iterator.pointer.channel)
-	UnregisterPipe()
+	obiutils.UnregisterPipe()
 }
 
 func (iterator IBioSequence) WaitAndClose() {
@@ -445,7 +426,6 @@ func (iterator IBioSequence) Rebatch(size int) IBioSequence {
 				}
 				i += to_push
 			}
-			seqs.Recycle(false)
 		}
 		log.Debug("End of the rebatch loop")
 		if len(buffer) > 0 {
@@ -464,6 +444,39 @@ func (iterator IBioSequence) Rebatch(size int) IBioSequence {
 	return newIter
 }
 
+func (iterator IBioSequence) FilterEmpty() IBioSequence {
+
+	newIter := MakeIBioSequence()
+
+	newIter.Add(1)
+
+	go func() {
+		newIter.WaitAndClose()
+	}()
+
+	go func() {
+		order := 0
+		iterator = iterator.SortBatches()
+
+		for iterator.Next() {
+			seqs := iterator.Get()
+			lc := seqs.Len()
+
+			if lc > 0 {
+				newIter.Push(seqs.Reorder(order))
+				order++
+			}
+		}
+
+		newIter.Done()
+	}()
+
+	if iterator.IsPaired() {
+		newIter.MarkAsPaired()
+	}
+
+	return newIter
+}
 func (iterator IBioSequence) Recycle() {
 
 	log.Debugln("Start recycling of Bioseq objects")
@@ -474,7 +487,6 @@ func (iterator IBioSequence) Recycle() {
 		o := batch.Order()
 		log.Debugln("Recycling batch #", o)
 		recycled += batch.Len()
-		batch.Recycle(true)
 		log.Debugln("Batch #", o, " recycled")
 	}
 	log.Debugf("End of the recycling of %d Bioseq objects", recycled)
@@ -482,8 +494,7 @@ func (iterator IBioSequence) Recycle() {
 
 func (iterator IBioSequence) Consume() {
 	for iterator.Next() {
-		batch := iterator.Get()
-		batch.Recycle(false)
+		iterator.Get()
 	}
 }
 
@@ -501,7 +512,6 @@ func (iterator IBioSequence) Count(recycle bool) (int, int, int) {
 			reads += seq.Count()
 			nucleotides += seq.Len()
 		}
-		batch.Recycle(recycle)
 	}
 	log.Debugf("End of the counting of %d Bioseq objects", variants)
 	return variants, reads, nucleotides
@@ -555,7 +565,6 @@ func (iterator IBioSequence) DivideOn(predicate obiseq.SequencePredicate,
 					falseSlice = obiseq.MakeBioSequenceSlice()
 				}
 			}
-			seqs.Recycle(false)
 		}
 
 		if len(trueSlice) > 0 {
@@ -582,7 +591,7 @@ func (iterator IBioSequence) DivideOn(predicate obiseq.SequencePredicate,
 // A function that takes a predicate and a batch of sequences and returns a filtered batch of sequences.
 func (iterator IBioSequence) FilterOn(predicate obiseq.SequencePredicate,
 	size int, sizes ...int) IBioSequence {
-	nworkers := obioptions.CLIReadParallelWorkers()
+	nworkers := obidefault.ReadParallelWorkers()
 
 	if len(sizes) > 0 {
 		nworkers = sizes[0]
@@ -634,7 +643,7 @@ func (iterator IBioSequence) FilterOn(predicate obiseq.SequencePredicate,
 
 func (iterator IBioSequence) FilterAnd(predicate obiseq.SequencePredicate,
 	size int, sizes ...int) IBioSequence {
-	nworkers := obioptions.CLIReadParallelWorkers()
+	nworkers := obidefault.ReadParallelWorkers()
 
 	if len(sizes) > 0 {
 		nworkers = sizes[0]
@@ -702,7 +711,6 @@ func (iterator IBioSequence) Load() (string, obiseq.BioSequenceSlice) {
 		}
 		log.Debugf("append %d sequences", b.Len())
 		chunk = append(chunk, b.Slice()...)
-		b.Recycle(false)
 	}
 
 	return source, chunk

pkg/obiiter/distribute.go

@@ -4,9 +4,25 @@ import (
 	"fmt"
 	"sync"
 
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 )
 
+// IDistribute represents a distribution mechanism for biosequences.
+// It manages the outputs of biosequences, provides a channel for
+// new data notifications, and maintains a classifier for sequence
+// classification. It is designed to facilitate the distribution
+// of biosequences to various processing components.
+//
+// Fields:
+//   - outputs: A map that associates integer keys with corresponding
+//     biosequence outputs (IBioSequence).
+//   - news: A channel that sends notifications of new data available
+//     for processing, represented by integer identifiers.
+//   - classifier: A pointer to a BioSequenceClassifier used to classify
+//     the biosequences during distribution.
+//   - lock: A mutex for synchronizing access to the outputs and other
+//     shared resources to ensure thread safety.
 type IDistribute struct {
 	outputs    map[int]IBioSequence
 	news       chan int
@@ -26,16 +42,39 @@ func (dist *IDistribute) Outputs(key int) (IBioSequence, error) {
 	return iter, nil
 }
 
+// News returns a channel that provides notifications of new data
+// available for processing. The channel sends integer identifiers
+// representing the new data.
 func (dist *IDistribute) News() chan int {
 	return dist.news
 }
 
+// Classifier returns a pointer to the BioSequenceClassifier
+// associated with the distribution mechanism. This classifier
+// is used to classify biosequences during the distribution process.
 func (dist *IDistribute) Classifier() *obiseq.BioSequenceClassifier {
 	return dist.classifier
 }
 
+// Distribute organizes the biosequences from the iterator into batches
+// based on the provided classifier and batch sizes. It returns an
+// IDistribute instance that manages the distribution of the sequences.
+//
+// Parameters:
+//   - class: A pointer to a BioSequenceClassifier used to classify
+//     the biosequences during distribution.
+//   - sizes: Optional integer values specifying the batch size. If
+//     no sizes are provided, a default batch size of 5000 is used.
+//
+// Returns:
+// An IDistribute instance that contains the outputs of the
+// classified biosequences, a channel for new data notifications,
+// and the classifier used for distribution. The method operates
+// asynchronously, processing the sequences in separate goroutines.
+// It ensures that the outputs are closed and cleaned up once
+// processing is complete.
 func (iterator IBioSequence) Distribute(class *obiseq.BioSequenceClassifier, sizes ...int) IDistribute {
-	batchsize := 5000
+	batchsize := obidefault.BatchSize()
 
 	outputs := make(map[int]IBioSequence, 100)
 	slices := make(map[int]*obiseq.BioSequenceSlice, 100)
@@ -93,7 +132,6 @@ func (iterator IBioSequence) Distribute(class *obiseq.BioSequenceClassifier, siz
 					slices[key] = &s
 				}
 			}
-			seqs.Recycle(false)
 		}
 
 		for key, slice := range slices {

pkg/obiiter/extract_taxonomy.go

@@ -0,0 +1,18 @@
+package obiiter
+
+import "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax"
+
+func (iterator *IBioSequence) ExtractTaxonomy() (taxonomy *obitax.Taxonomy, err error) {
+
+	for iterator.Next() {
+		slice := iterator.Get().Slice()
+
+		taxonomy, err = slice.ExtractTaxonomy(taxonomy)
+
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}

pkg/obiiter/fragment.go

@@ -55,7 +55,6 @@ func IFragments(minsize, length, overlap, size, nworkers int) Pipeable {
 					}
 				} // End of the slice loop
 				newiter.Push(MakeBioSequenceBatch(source, sl.Order(), news))
-				sl.Recycle(false)
 			} // End of the iterator loop
 
 			// if len(news) > 0 {

pkg/obiiter/paired.go

@@ -1,6 +1,7 @@
 package obiiter
 
 import (
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -40,8 +41,8 @@ func (iter IBioSequence) PairTo(p IBioSequence) IBioSequence {
 
 	newIter := MakeIBioSequence()
 
-	iter = iter.SortBatches()
-	p = p.SortBatches()
+	iter = iter.SortBatches().Rebatch(obidefault.BatchSize())
+	p = p.SortBatches().Rebatch(obidefault.BatchSize())
 
 	newIter.Add(1)
 

pkg/obiiter/workers.go

@@ -3,7 +3,7 @@ package obiiter
 import (
 	log "github.com/sirupsen/logrus"
 
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 )
 
@@ -18,7 +18,7 @@ import (
 func (iterator IBioSequence) MakeIWorker(worker obiseq.SeqWorker,
 	breakOnError bool,
 	sizes ...int) IBioSequence {
-	nworkers := obioptions.CLIParallelWorkers()
+	nworkers := obidefault.ParallelWorkers()
 
 	if len(sizes) > 0 {
 		nworkers = sizes[0]
@@ -34,13 +34,13 @@ func (iterator IBioSequence) MakeIWorker(worker obiseq.SeqWorker,
 // Parameters:
 // - predicate: A function that takes a sequence and returns a boolean value indicating whether the sequence satisfies a certain condition.
 // - worker: A function that takes a sequence and returns a modified version of the sequence.
-// - sizes: Optional. One or more integers representing the number of workers to be used for parallel processing. If not provided, the number of workers will be determined by the obioptions.CLIReadParallelWorkers() function.
+// - sizes: Optional. One or more integers representing the number of workers to be used for parallel processing. If not provided, the number of workers will be determined by the obidefault.ReadParallelWorkers() function.
 //
 // Return:
 // - newIter: A new IBioSequence iterator with the modified sequences.
 func (iterator IBioSequence) MakeIConditionalWorker(predicate obiseq.SequencePredicate,
 	worker obiseq.SeqWorker, breakOnError bool, sizes ...int) IBioSequence {
-	nworkers := obioptions.CLIReadParallelWorkers()
+	nworkers := obidefault.ReadParallelWorkers()
 
 	if len(sizes) > 0 {
 		nworkers = sizes[0]
@@ -63,7 +63,7 @@ func (iterator IBioSequence) MakeIConditionalWorker(predicate obiseq.SequencePre
 //
 // The function returns a new IBioSequence containing the modified slices.
 func (iterator IBioSequence) MakeISliceWorker(worker obiseq.SeqSliceWorker, breakOnError bool, sizes ...int) IBioSequence {
-	nworkers := obioptions.CLIParallelWorkers()
+	nworkers := obidefault.ParallelWorkers()
 
 	if len(sizes) > 0 {
 		nworkers = sizes[0]

pkg/obiitercsv/csv.go

@@ -0,0 +1,348 @@
+package obiitercsv
+
+import (
+	"fmt"
+	"slices"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+	"github.com/tevino/abool/v2"
+
+	log "github.com/sirupsen/logrus"
+)
+
+type CSVHeader []string
+type CSVRecord map[string]interface{}
+type CSVRecordBatch struct {
+	source string
+	data   []CSVRecord
+	order  int
+}
+
+var NilCSVRecordBatch = CSVRecordBatch{"", nil, -1}
+
+// Structure implementing an iterator over bioseq.BioSequenceBatch
+// based on a channel.
+type ICSVRecord struct {
+	channel         chan CSVRecordBatch
+	current         CSVRecordBatch
+	pushBack        *abool.AtomicBool
+	all_done        *sync.WaitGroup
+	lock            *sync.RWMutex
+	buffer_size     int32
+	batch_size      int32
+	sequence_format string
+	finished        *abool.AtomicBool
+	header          CSVHeader
+}
+
+var NilIBioSequenceBatch = (*ICSVRecord)(nil)
+
+func NewICSVRecord() *ICSVRecord {
+
+	i := ICSVRecord{
+		channel:         make(chan CSVRecordBatch),
+		current:         NilCSVRecordBatch,
+		pushBack:        abool.New(),
+		batch_size:      -1,
+		sequence_format: "",
+		finished:        abool.New(),
+		header:          make(CSVHeader, 0),
+	}
+
+	waiting := sync.WaitGroup{}
+	i.all_done = &waiting
+	lock := sync.RWMutex{}
+	i.lock = &lock
+
+	obiutils.RegisterAPipe()
+
+	return &i
+}
+
+func MakeCSVRecordBatch(source string, order int, data []CSVRecord) CSVRecordBatch {
+	return CSVRecordBatch{
+		source: source,
+		order:  order,
+		data:   data,
+	}
+}
+
+func (batch *CSVRecordBatch) Order() int {
+	return batch.order
+}
+
+func (batch *CSVRecordBatch) Source() string {
+	return batch.source
+}
+
+func (batch *CSVRecordBatch) Slice() []CSVRecord {
+	return batch.data
+}
+
+// NotEmpty returns whether the BioSequenceBatch is empty or not.
+//
+// It checks if the BioSequenceSlice contained within the BioSequenceBatch is not empty.
+//
+// Returns:
+// - bool: True if the BioSequenceBatch is not empty, false otherwise.
+func (batch *CSVRecordBatch) NotEmpty() bool {
+	return len(batch.data) > 0
+}
+
+// IsNil checks if the BioSequenceBatch's slice is nil.
+//
+// This function takes a BioSequenceBatch as a parameter and returns a boolean value indicating whether the slice of the BioSequenceBatch is nil or not.
+//
+// Parameters:
+// - batch: The BioSequenceBatch to check for nil slice.
+//
+// Returns:
+// - bool: True if the BioSequenceBatch's slice is nil, false otherwise.
+func (batch *CSVRecordBatch) IsNil() bool {
+	return batch.data == nil
+}
+
+func (iterator *ICSVRecord) Add(n int) {
+	if iterator == nil {
+		log.Panic("call of ICSVRecord.Add method on NilIBioSequenceBatch")
+	}
+
+	iterator.all_done.Add(n)
+}
+
+func (iterator *ICSVRecord) Done() {
+	if iterator == nil {
+		log.Panic("call of ICSVRecord.Done method on NilIBioSequenceBatch")
+	}
+
+	iterator.all_done.Done()
+}
+
+func (iterator *ICSVRecord) Unlock() {
+	if iterator == nil {
+		log.Panic("call of ICSVRecord.Unlock method on NilIBioSequenceBatch")
+	}
+
+	iterator.lock.Unlock()
+}
+
+func (iterator *ICSVRecord) Lock() {
+	if iterator == nil {
+		log.Panic("call of ICSVRecord.Lock method on NilIBioSequenceBatch")
+	}
+
+	iterator.lock.Lock()
+}
+
+func (iterator *ICSVRecord) RLock() {
+	if iterator == nil {
+		log.Panic("call of ICSVRecord.RLock method on NilIBioSequenceBatch")
+	}
+
+	iterator.lock.RLock()
+}
+
+func (iterator *ICSVRecord) RUnlock() {
+	if iterator == nil {
+		log.Panic("call of ICSVRecord.RUnlock method on NilIBioSequenceBatch")
+	}
+
+	iterator.lock.RUnlock()
+}
+
+func (iterator *ICSVRecord) Wait() {
+	if iterator == nil {
+		log.Panic("call of ICSVRecord.Wait method on NilIBioSequenceBatch")
+	}
+
+	iterator.all_done.Wait()
+}
+
+func (iterator *ICSVRecord) Channel() chan CSVRecordBatch {
+	if iterator == nil {
+		log.Panic("call of ICSVRecord.Channel method on NilIBioSequenceBatch")
+	}
+
+	return iterator.channel
+}
+
+func (iterator *ICSVRecord) IsNil() bool {
+	if iterator == nil {
+		log.Panic("call of ICSVRecord.IsNil method on NilIBioSequenceBatch")
+	}
+
+	return iterator == nil
+}
+
+func (iterator *ICSVRecord) BatchSize() int {
+	if iterator == nil {
+		log.Panic("call of ICSVRecord.BatchSize method on NilIBioSequenceBatch")
+	}
+
+	return int(atomic.LoadInt32(&iterator.batch_size))
+}
+
+func (iterator *ICSVRecord) SetBatchSize(size int) error {
+	if size >= 0 {
+		atomic.StoreInt32(&iterator.batch_size, int32(size))
+		return nil
+	}
+
+	return fmt.Errorf("size (%d) cannot be negative", size)
+}
+
+func (iterator *ICSVRecord) Split() *ICSVRecord {
+	iterator.lock.RLock()
+	defer iterator.lock.RUnlock()
+	i := ICSVRecord{
+		channel:         iterator.channel,
+		current:         NilCSVRecordBatch,
+		pushBack:        abool.New(),
+		all_done:        iterator.all_done,
+		buffer_size:     iterator.buffer_size,
+		batch_size:      iterator.batch_size,
+		sequence_format: iterator.sequence_format,
+		finished:        iterator.finished,
+		header:          iterator.header,
+	}
+	lock := sync.RWMutex{}
+	i.lock = &lock
+
+	return &i
+}
+
+func (iterator *ICSVRecord) Header() CSVHeader {
+	return iterator.header
+}
+
+func (iterator *ICSVRecord) SetHeader(header CSVHeader) {
+	iterator.header = header
+}
+
+func (iterator *ICSVRecord) AppendField(field string) {
+	iterator.header.AppendField(field)
+}
+
+func (iterator *ICSVRecord) Next() bool {
+	if iterator.pushBack.IsSet() {
+		iterator.pushBack.UnSet()
+		return true
+	}
+
+	if iterator.finished.IsSet() {
+		return false
+	}
+
+	next, ok := (<-iterator.channel)
+
+	if ok {
+		iterator.current = next
+		return true
+	}
+
+	iterator.current = NilCSVRecordBatch
+	iterator.finished.Set()
+	return false
+}
+
+func (iterator *ICSVRecord) PushBack() {
+	if !iterator.current.IsNil() {
+		iterator.pushBack.Set()
+	}
+}
+
+// The 'Get' method returns the instance of BioSequenceBatch
+// currently pointed by the iterator. You have to use the
+// 'Next' method to move to the next entry before calling
+// 'Get' to retreive the following instance.
+func (iterator *ICSVRecord) Get() CSVRecordBatch {
+	return iterator.current
+}
+
+func (iterator *ICSVRecord) Push(batch CSVRecordBatch) {
+	if batch.IsNil() {
+		log.Panicln("A Nil batch is pushed on the channel")
+	}
+	// if batch.Len() == 0 {
+	// 	log.Panicln("An empty batch is pushed on the channel")
+	// }
+
+	iterator.channel <- batch
+}
+
+func (iterator *ICSVRecord) Close() {
+	close(iterator.channel)
+	obiutils.UnregisterPipe()
+}
+
+func (iterator *ICSVRecord) WaitAndClose() {
+	iterator.Wait()
+
+	for len(iterator.Channel()) > 0 {
+		time.Sleep(time.Millisecond)
+	}
+
+	iterator.Close()
+}
+
+// Finished returns 'true' value if no more data is available
+// from the iterator.
+func (iterator *ICSVRecord) Finished() bool {
+	return iterator.finished.IsSet()
+}
+
+// Sorting the batches of sequences.
+func (iterator *ICSVRecord) SortBatches(sizes ...int) *ICSVRecord {
+
+	newIter := NewICSVRecord()
+
+	newIter.Add(1)
+
+	go func() {
+		newIter.WaitAndClose()
+	}()
+
+	next_to_send := 0
+	//log.Println("wait for batch #", next_to_send)
+	received := make(map[int]CSVRecordBatch)
+	go func() {
+		for iterator.Next() {
+			batch := iterator.Get()
+			// log.Println("\nPushd seq #\n", batch.order, next_to_send)
+
+			if batch.order == next_to_send {
+				newIter.channel <- batch
+				next_to_send++
+				//log.Println("\nwait for batch #\n", next_to_send)
+				batch, ok := received[next_to_send]
+				for ok {
+					newIter.channel <- batch
+					delete(received, next_to_send)
+					next_to_send++
+					batch, ok = received[next_to_send]
+				}
+			} else {
+				received[batch.order] = batch
+			}
+		}
+		newIter.Done()
+	}()
+
+	return newIter
+
+}
+
+func (iterator *ICSVRecord) Consume() {
+	for iterator.Next() {
+		iterator.Get()
+	}
+}
+
+func (head *CSVHeader) AppendField(field string) {
+	if !slices.Contains(*head, field) {
+		*head = append(*head, field)
+	}
+}

pkg/obikmer/debruijn.go

@@ -8,8 +8,12 @@ import (
 	"math/bits"
 	"os"
 	"slices"
+	"sort"
 
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obistats"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+	"github.com/ef-ds/deque/v2"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -39,6 +43,25 @@ var iupac = map[byte][]uint64{
 	'n': {0, 1, 2, 3},
 }
 
+var revcompnuc = map[byte]byte{
+	'a': 't',
+	'c': 'g',
+	'g': 'c',
+	't': 'a',
+	'u': 'a',
+	'r': 'y',
+	'y': 'r',
+	's': 's',
+	'w': 'w',
+	'k': 'm',
+	'm': 'k',
+	'b': 'v',
+	'd': 'h',
+	'h': 'd',
+	'v': 'b',
+	'n': 'n',
+}
+
 var decode = map[uint64]byte{
 	0: 'a',
 	1: 'c',
@@ -69,12 +92,18 @@ type DeBruijnGraph struct {
 //
 //	*DeBruijnGraph - a pointer to the created De Bruijn's Graph
 func MakeDeBruijnGraph(kmersize int) *DeBruijnGraph {
+	if kmersize > 31 {
+		log.Panicf("k-mer size %d is too large", kmersize)
+	}
+
+	kmermask := (^uint64(0) << (uint64(kmersize) * 2))
+
 	g := DeBruijnGraph{
 		kmersize: kmersize,
-		kmermask: ^(^uint64(0) << (uint64(kmersize) * 2)), // k-mer mask used to set to 0 the bits that are not in the k-mer
-		prevc:    uint64(1) << (uint64(kmersize-1) * 2),
-		prevg:    uint64(2) << (uint64(kmersize-1) * 2),
-		prevt:    uint64(3) << (uint64(kmersize-1) * 2),
+		kmermask: kmermask, // k-mer mask used to set to 1 the bits that are not in the k-mer
+		prevc:    (uint64(1) << (uint64(kmersize-1) * 2)) | kmermask,
+		prevg:    (uint64(2) << (uint64(kmersize-1) * 2)) | kmermask,
+		prevt:    (uint64(3) << (uint64(kmersize-1) * 2)) | kmermask,
 		graph:    make(map[uint64]uint),
 	}
 
@@ -141,19 +170,34 @@ func (g *DeBruijnGraph) FilterMinWeight(min int) {
 	}
 }
 
+// FilterMinWeight filters the DeBruijnGraph by removing nodes with weight less than the specified minimum.
+//
+// min: an integer representing the minimum count threshold.
+func (g *DeBruijnGraph) FilterMaxWeight(min int) {
+	umin := uint(min)
+	for idx, count := range g.graph {
+		if count > umin {
+			delete(g.graph, idx)
+		}
+	}
+}
+
 func (g *DeBruijnGraph) Previouses(index uint64) []uint64 {
 	if _, ok := g.graph[index]; !ok {
 		log.Panicf("k-mer %s (index %d) is not in graph", g.DecodeNode(index), index)
 	}
 
 	rep := make([]uint64, 0, 4)
+
+	index &= ^g.kmermask
 	index >>= 2
 
-	if _, ok := g.graph[index]; ok {
-		rep = append(rep, index)
+	key := index | g.kmermask
+	if _, ok := g.graph[key]; ok {
+		rep = append(rep, key)
 	}
 
-	key := index | g.prevc
+	key = index | g.prevc
 	if _, ok := g.graph[key]; ok {
 		rep = append(rep, key)
 	}
@@ -177,7 +221,7 @@ func (g *DeBruijnGraph) Nexts(index uint64) []uint64 {
 	}
 
 	rep := make([]uint64, 0, 4)
-	index = (index << 2) & g.kmermask
+	index = (index << 2) | g.kmermask
 
 	if _, ok := g.graph[index]; ok {
 		rep = append(rep, index)
@@ -248,6 +292,33 @@ func (g *DeBruijnGraph) MaxHead() (uint64, int, bool) {
 	return rep, int(max), found
 }
 
+func (g *DeBruijnGraph) Terminals() []uint64 {
+	rep := make([]uint64, 0, 10)
+
+	for k := range g.graph {
+		if len(g.Nexts(k)) == 0 {
+			rep = append(rep, k)
+		}
+	}
+
+	return rep
+}
+
+func (g *DeBruijnGraph) MaxTerminal() (uint64, int, bool) {
+	rep := uint64(0)
+	max := uint(0)
+	found := false
+	for k, w := range g.graph {
+		if len(g.Nexts(k)) == 0 && w > max {
+			rep = k
+			max = w
+			found = true
+		}
+	}
+
+	return rep, int(max), found
+}
+
 func (g *DeBruijnGraph) MaxPath() []uint64 {
 	path := make([]uint64, 0, 1000)
 	ok := false
@@ -293,10 +364,52 @@ func (g *DeBruijnGraph) LongestPath(max_length int) []uint64 {
 	return path
 }
 
-func (g *DeBruijnGraph) LongestConsensus(id string) (*obiseq.BioSequence, error) {
+func (g *DeBruijnGraph) LongestConsensus(id string, min_cov float64) (*obiseq.BioSequence, error) {
+	if g.Len() == 0 {
+		return nil, fmt.Errorf("graph is empty")
+	}
 	//path := g.LongestPath(max_length)
-	path := g.HaviestPath()
-	s := g.DecodePath(path)
+	path, err := g.HaviestPath(nil, nil, false)
+
+	if err != nil {
+		return nil, err
+	}
+
+	spath := path
+
+	if min_cov > 0 {
+		wp := make([]uint, len(path))
+
+		for i, n := range path {
+			wp[i] = g.graph[n]
+		}
+
+		mp := uint(float64(obistats.Mode(wp))*min_cov + 0.5)
+
+		from := 0
+		for i, n := range path {
+			if g.graph[n] < mp {
+				from = i + 1
+			} else {
+				break
+			}
+		}
+
+		to := len(path)
+
+		for i := len(path) - 1; i >= 0; i-- {
+			n := path[i]
+			if g.graph[n] < mp {
+				to = i
+			} else {
+				break
+			}
+		}
+
+		spath = path[from:to]
+	}
+
+	s := g.DecodePath(spath)
 
 	if len(s) > 0 {
 		seq := obiseq.NewBioSequence(
@@ -368,6 +481,48 @@ func (g *DeBruijnGraph) Weight(index uint64) int {
 	return int(val)
 }
 
+// WeightMode returns the mode weight of the nodes in the DeBruijnGraph.
+//
+// It iterates over each count in the graph map and updates the max value if the current count is greater.
+// Finally, it returns the mode weight as an integer.
+//
+// Returns:
+// - int: the mode weight value.
+func (g *DeBruijnGraph) WeightMode() int {
+	dist := make(map[uint]int)
+
+	for _, w := range g.graph {
+		dist[w]++
+	}
+
+	maxi := 0
+	wmax := uint(0)
+
+	for k, v := range dist {
+		if v > maxi && k > 1 {
+			maxi = v
+			wmax = k
+		}
+	}
+
+	return int(wmax)
+}
+
+// WeightMean returns the mean weight of the nodes in the DeBruijnGraph.
+//
+// Returns:
+// - float64: the mean weight of the nodes in the graph.
+func (g *DeBruijnGraph) WeightMean() float64 {
+	sum := 0
+
+	for _, w := range g.graph {
+		i := int(w)
+		sum += i
+	}
+
+	return float64(sum) / float64(len(g.graph))
+}
+
 // append appends a sequence of nucleotides to the DeBruijnGraph.
 //
 // Parameters:
@@ -381,7 +536,7 @@ func (graph *DeBruijnGraph) append(sequence []byte, current uint64, weight int)
 	}
 
 	current <<= 2
-	current &= graph.kmermask
+	current |= graph.kmermask
 	b := iupac[sequence[0]]
 	current |= b[0]
 	graph.graph[current] = uint(graph.Weight(current) + weight)
@@ -395,6 +550,36 @@ func (graph *DeBruijnGraph) append(sequence []byte, current uint64, weight int)
 	}
 }
 
+// func (graph *DeBruijnGraph) search(current uint64, extension []byte, path []uint64, error,errormax int) ([]uint64,error) {
+
+// 	path = append(path, current)
+
+// 	if len(extension) == 0 {
+// 		return path,nil
+// 	}
+
+// 	current <<= 2
+// 	current &= graph.kmermask
+// 	b := iupac[extension[0]]
+
+// 	newPath := path
+// 	if len(b) > 1 {
+// 		newPath = slices.Clone(path)
+// 	}
+
+// 	current |= b[0]
+
+// 	_, ok := graph.graph[current]
+// 	if ok {
+// 		newPath = append(newPath, current)
+// 	}
+// 	rep, err := graph.search(current, extension[1:], newPath, error,errormax)
+// 	if err != nil {
+// 		return path,err
+// 	}
+
+// }
+
 // Push appends a BioSequence to the DeBruijnGraph.
 //
 // Parameters:
@@ -423,6 +608,7 @@ func (graph *DeBruijnGraph) Push(sequence *obiseq.BioSequence) {
 				initFirstKmer(start+1, key)
 			}
 		} else {
+			key |= graph.kmermask
 			graph.graph[key] = uint(graph.Weight(key) + w)
 			graph.append(s[graph.kmersize:], key, w)
 		}
@@ -433,6 +619,110 @@ func (graph *DeBruijnGraph) Push(sequence *obiseq.BioSequence) {
 	}
 }
 
+func (graph *DeBruijnGraph) search(sequence []byte, mismatch, errormax int) []uint64 {
+	var initFirstKmer func(start int, key uint64) []uint64
+
+	initFirstKmer = func(start int, key uint64) []uint64 {
+		if start == graph.kmersize {
+			key |= graph.kmermask
+			if _, ok := graph.graph[key]; ok {
+				return []uint64{key}
+			} else {
+				return []uint64{}
+			}
+		}
+
+		keys := make([]uint64, 0, 1000)
+
+		if start == 0 {
+			key = 0
+		}
+
+		key <<= 2
+		b := iupac[sequence[start]]
+
+		for _, code := range b {
+			key &= ^uint64(3)
+			key |= code
+			keys = append(keys, initFirstKmer(start+1, key)...)
+		}
+
+		// w := []string{}
+		// for _, k := range keys {
+		// 	w = append(w, graph.DecodeNode(k))
+		// }
+		// // log.Warnf("For %s found %d matches : %v", sequence, len(keys), w)
+
+		return keys
+	}
+
+	rep := initFirstKmer(0, 0)
+
+	return rep
+}
+
+func (graph *DeBruijnGraph) Search(sequence *obiseq.BioSequence, errormax int) []uint64 {
+
+	s := sequence.Sequence() // Get the sequence as a byte slice
+
+	if len(s) < graph.KmerSize() {
+		s = slices.Clone(s)
+		for len(s) < graph.KmerSize() {
+			s = append(s, 'n')
+		}
+	}
+
+	log.Warnf("searching for %s", s)
+	keys := graph.search(s, 0, errormax)
+
+	for mismatch := 1; mismatch <= errormax; mismatch++ {
+		log.Warnf("searching with %d error for %s", mismatch, s)
+		for probe := range IterateOneError(s[0:graph.kmersize]) {
+			keys = append(keys,
+				graph.search(probe, mismatch, errormax)...,
+			)
+		}
+	}
+	keys = obiutils.Unique(keys)
+
+	return keys
+}
+
+func (graph *DeBruijnGraph) BackSearch(sequence *obiseq.BioSequence, errormax int) []uint64 {
+	lkmer := graph.KmerSize()
+
+	s := sequence.Sequence() // Get the sequence as a byte slice
+
+	if len(s) < lkmer {
+		sn := []byte{}
+		ls := len(s)
+		for ls < lkmer {
+			sn = append(sn, 'n')
+			ls++
+		}
+		s = append(sn, s...)
+	} else {
+		s = s[(len(s) - lkmer):]
+	}
+
+	log.Warnf("back-searching for %s", s)
+
+	keys := graph.search(s, 0, errormax)
+
+	for mismatch := 1; mismatch <= errormax; mismatch++ {
+		log.Warnf("searching with %d error for %s", mismatch, s)
+		for probe := range IterateOneError(s[0:graph.kmersize]) {
+			// log.Warnf("searching with %d error for %s", mismatch, probe)
+			keys = append(keys,
+				graph.search(probe, mismatch, errormax)...,
+			)
+		}
+	}
+
+	keys = obiutils.Unique(keys)
+	return keys
+}
+
 func (graph *DeBruijnGraph) Gml() string {
 	buffer := bytes.NewBuffer(make([]byte, 0, 1000))
 
@@ -514,7 +804,7 @@ func (graph *DeBruijnGraph) WriteGml(filename string) error {
 func (g *DeBruijnGraph) HammingDistance(kmer1, kmer2 uint64) int {
 	ident := ^((kmer1 & kmer2) | (^kmer1 & ^kmer2))
 	ident |= (ident >> 1)
-	ident &= 0x5555555555555555 & g.kmermask
+	ident &= 0x5555555555555555 & ^g.kmermask
 	return bits.OnesCount64(ident)
 }
 
@@ -538,11 +828,23 @@ func (h *UInt64Heap) Pop() any {
 	return x
 }
 
-func (g *DeBruijnGraph) HaviestPath() []uint64 {
+func (g *DeBruijnGraph) HaviestPath(starts, stops []uint64, backPath bool) ([]uint64, error) {
 
-	if g.HasCycle() {
-		return nil
+	// if g.HasCycle() {
+	// 	return nil, fmt.Errorf("graph has a cycle")
+	// }
+
+	following := g.Nexts
+
+	if backPath {
+		following = g.Previouses
 	}
+
+	stopNodes := make(map[uint64]bool, len(stops))
+	for _, n := range stops {
+		stopNodes[n] = true
+	}
+
 	// Initialize the distance array and visited set
 	distances := make(map[uint64]int)
 	visited := make(map[uint64]bool)
@@ -554,7 +856,11 @@ func (g *DeBruijnGraph) HaviestPath() []uint64 {
 	heap.Init(queue)
 
 	startNodes := make(map[uint64]struct{})
-	for _, n := range g.Heads() {
+	if starts == nil {
+		starts = g.Heads()
+	}
+
+	for _, n := range starts {
 		startNodes[n] = struct{}{}
 		heap.Push(queue, n)
 		distances[n] = g.Weight(n)
@@ -586,7 +892,11 @@ func (g *DeBruijnGraph) HaviestPath() []uint64 {
 			log.Warn("current node is 0")
 		}
 		// Update the distance of the neighbors
-		nextNodes := g.Nexts(currentNode)
+
+		nextNodes := following(currentNode)
+		if _, ok := stopNodes[currentNode]; ok {
+			nextNodes = []uint64{}
+		}
 		for _, nextNode := range nextNodes {
 			if nextNode == 0 {
 				log.Warn("next node is 0")
@@ -618,16 +928,178 @@ func (g *DeBruijnGraph) HaviestPath() []uint64 {
 	}
 
 	if slices.Contains(heaviestPath, currentNode) {
-		log.Fatalf("Cycle detected %v -> %v (%v) len(%v)", heaviestPath, currentNode, startNodes, len(heaviestPath))
-		return nil
+		return nil, fmt.Errorf("cycle detected in heaviest path")
 	}
 
 	heaviestPath = append(heaviestPath, currentNode)
 
 	// Reverse the path
-	slices.Reverse(heaviestPath)
+	if !backPath {
+		slices.Reverse(heaviestPath)
+	}
 
-	return heaviestPath
+	return heaviestPath, nil
+}
+
+func (g *DeBruijnGraph) HaviestPathDSU(starts, stops []uint64, backPath bool) ([]uint64, error) {
+	// Collect and sort edges
+	type Edge struct {
+		weight float64
+		u, v   uint64
+	}
+	edges := make([]Edge, 0)
+
+	// Function to get next nodes (either Nexts or Previouses based on backPath)
+	following := g.Nexts
+	previouses := g.Previouses
+	if backPath {
+		following = g.Previouses
+		previouses = g.Nexts
+	}
+
+	// Collect all edges
+	for u := range g.graph {
+		for _, v := range following(u) {
+			edges = append(edges, Edge{
+				weight: float64(min(g.Weight(u), g.Weight(v))),
+				u:      u,
+				v:      v,
+			})
+		}
+	}
+
+	// Sort edges by weight in descending order
+	sort.Slice(edges, func(i, j int) bool {
+		return edges[i].weight > edges[j].weight
+	})
+
+	// Initialize disjoint set data structure
+	parent := make(map[uint64]uint64)
+	for u := range g.graph {
+		parent[u] = u
+	}
+
+	// Find with path compression
+	var find func(uint64) uint64
+	find = func(node uint64) uint64 {
+		if parent[node] != node {
+			parent[node] = find(parent[node])
+		}
+		return parent[node]
+	}
+
+	// Union function that returns true if cycle is detected
+	union := func(u, v uint64) bool {
+		rootU := find(u)
+		rootV := find(v)
+		if rootU == rootV {
+			return true // Cycle detected
+		}
+		parent[rootV] = rootU
+		return false
+	}
+
+	// If no specific starts provided, use graph heads
+	if starts == nil {
+		if !backPath {
+			starts = g.Heads()
+		} else {
+			starts = g.Terminals()
+		}
+	}
+
+	// If no specific stops provided, use graph terminals
+	if stops == nil {
+		if !backPath {
+			stops = g.Terminals()
+		} else {
+			stops = g.Heads()
+		}
+	}
+
+	// Convert stops to a map for O(1) lookup
+	stopNodes := make(map[uint64]bool)
+	for _, stop := range stops {
+		stopNodes[stop] = false
+	}
+
+	var path []uint64
+	maxCapacity := math.Inf(-1)
+	stopEdge := []Edge{}
+
+	// Process edges in descending order of weight
+	for _, edge := range edges {
+		if stopNodes[edge.u] {
+			continue // Skip edges from stop nodes
+		}
+
+		if in, ok := stopNodes[edge.v]; ok {
+			if !in {
+				stopEdge = append(stopEdge, edge)
+				stopNodes[edge.v] = true
+			}
+		}
+
+		if union(edge.u, edge.v) {
+			continue // Skip if creates cycle
+		}
+
+		pathFound := false
+		for _, sedge := range stopEdge {
+			// Check if any start-stop pair is connected
+			fv := find(sedge.v)
+			for _, s := range starts {
+				fs := find(s)
+				//				log.Warnf("Start: %d, Stop: %d", fs, fv)
+				if fs == fv {
+					pathFound = true
+					maxCapacity = edge.weight
+
+					// Reconstruct path
+					current := sedge.v
+					path = []uint64{current}
+					for current != s {
+						oldcurrent := current
+						//						log.Warnf("Start: %d, Current: %d, Previous: %v", s, current, previouses(current))
+						for _, prev := range previouses(current) {
+							if find(prev) == fs {
+								path = append(path, prev)
+								current = prev
+								break
+							}
+						}
+						if current == oldcurrent {
+							log.Fatalf("We are stuck")
+						}
+
+					}
+					//					log.Warnf("Built path: %v", path)
+					break
+				}
+			}
+			if pathFound {
+				break
+			}
+		}
+		if pathFound {
+			break
+		}
+	}
+
+	//	log.Warnf("Stop edge: %v", stopEdge)
+
+	// Process edges in descending order of weight
+
+	if path == nil {
+		return nil, fmt.Errorf("no valid path found")
+	}
+
+	if !backPath {
+		slices.Reverse(path)
+	}
+	log.Warnf("Max capacity: %5.0f: %v", maxCapacity, g.DecodePath(path))
+
+	return path, nil
 }
 
 func (g *DeBruijnGraph) HasCycle() bool {
@@ -665,3 +1137,59 @@ func (g *DeBruijnGraph) HasCycle() bool {
 	}
 	return false
 }
+
+// HasCycleInDegree détecte la présence d'un cycle dans le graphe en utilisant la méthode des degrés entrants.
+// Cette méthode est basée sur le tri topologique : si on ne peut pas trier tous les nœuds,
+// alors il y a un cycle.
+//
+// Returns:
+// - bool: true si le graphe contient un cycle, false sinon
+func (g *DeBruijnGraph) HasCycleInDegree() bool {
+	// Créer une map pour stocker les degrés entrants de chaque nœud
+	inDegree := make(map[uint64]int)
+
+	// Initialiser les degrés entrants à 0 pour tous les nœuds
+	for node := range g.graph {
+		inDegree[node] = 0
+	}
+
+	// Calculer les degrés entrants
+	for node := range g.graph {
+		for _, next := range g.Nexts(node) {
+			inDegree[next]++
+		}
+	}
+
+	// Créer une deque pour stocker les nœuds avec un degré entrant de 0
+	queue := deque.Deque[uint64]{}
+
+	// Ajouter tous les nœuds avec un degré entrant de 0 à la deque
+	for node := range g.graph {
+		if inDegree[node] == 0 {
+			queue.PushBack(node)
+		}
+	}
+
+	visited := 0 // Compteur de nœuds visités
+
+	// Parcours BFS
+	for queue.Len() > 0 {
+		// Retirer le premier nœud de la deque
+		node, _ := queue.PopFront()
+		visited++
+
+		// Pour chaque nœud adjacent
+		for _, next := range g.Nexts(node) {
+			// Réduire son degré entrant
+			inDegree[next]--
+
+			// Si le degré entrant devient 0, l'ajouter à la deque
+			if inDegree[next] == 0 {
+				queue.PushBack(next)
+			}
+		}
+	}
+
+	// S'il y a un cycle, on n'aura pas pu visiter tous les nœuds
+	return visited != len(g.graph)
+}

pkg/obikmer/encodefourmer.go

@@ -97,8 +97,19 @@ func Index4mer(seq *obiseq.BioSequence, index *[][]int, buffer *[]byte) [][]int
 }
 
 // FastShiftFourMer runs a Fast algorithm (similar to the one used in FASTA) to compare two sequences.
-// The returned values are two integer values. The shift between both the sequences and the count of
-// matching 4mer when this shift is applied between both the sequences.
+//
+// Parameters:
+// - index: A precomputed index of 4mer positions in a reference sequence.
+// - shifts: A map to store the shift and count of matching 4mers.
+// - lindex: The length of the indexed reference sequence.
+// - seq: The sequence to be compared with the reference sequence.
+// - relscore: A boolean indicating whether to calculate the relative score.
+// - buffer: A byte buffer for encoding the sequence.
+//
+// Return type:
+// - int: The shift between the two sequences with the maximum score.
+// - int: The count of matching 4mers at the maximum score.
+// - float64: The maximum score.
 func FastShiftFourMer(index [][]int, shifts *map[int]int, lindex int, seq *obiseq.BioSequence, relscore bool, buffer *[]byte) (int, int, float64) {
 
 	iternal_buffer := Encode4mer(seq, buffer)
@@ -126,12 +137,15 @@ func FastShiftFourMer(index [][]int, shifts *map[int]int, lindex int, seq *obise
 		score := float64(count)
 		if relscore {
 			over := -shift
-			if shift > 0 {
+			switch {
+			case shift > 0:
 				over += lindex
-			} else {
+			case shift < 0:
 				over = seq.Len() - over
+			default:
+				over = min(lindex, seq.Len())
 			}
-			score = score / float64(over)
+			score = score / float64(over-3)
 		}
 		if score > maxscore {
 			maxshift = shift

pkg/obikmer/kmermap.go

@@ -0,0 +1,297 @@
+package obikmer
+
+import (
+	"os"
+	"sort"
+	"unsafe"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obifp"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+	"github.com/schollz/progressbar/v3"
+	log "github.com/sirupsen/logrus"
+)
+
+type KmerMap[T obifp.FPUint[T]] struct {
+	index    map[T][]*obiseq.BioSequence
+	Kmersize uint
+	kmermask T
+
+	leftMask   T
+	rightMask  T
+	sparseMask T
+
+	SparseAt int
+}
+
+type KmerMatch map[*obiseq.BioSequence]int
+
+func (k *KmerMap[T]) KmerSize() uint {
+	return k.Kmersize
+}
+
+func (k *KmerMap[T]) Len() int {
+	return len(k.index)
+}
+
+func (k *KmerMap[T]) KmerAsString(kmer T) string {
+	buff := make([]byte, k.Kmersize)
+	ks := int(k.Kmersize)
+
+	if k.SparseAt >= 0 {
+		ks--
+	}
+
+	for i, j := 0, int(k.Kmersize)-1; i < ks; i++ {
+		code := kmer.And(obifp.From64[T](3)).AsUint64()
+		buff[j] = decode[code]
+		j--
+		if k.SparseAt >= 0 && j == k.SparseAt {
+			buff[j] = '#'
+			j--
+		}
+		kmer = kmer.RightShift(2)
+	}
+
+	return string(buff)
+}
+
+func (k *KmerMap[T]) NormalizedKmerSlice(sequence *obiseq.BioSequence, buff *[]T) []T {
+
+	if sequence.Len() < int(k.Kmersize) {
+		return nil
+	}
+
+	makeSparseAt := func(kmer T) T {
+
+		if k.SparseAt == -1 {
+			return kmer
+		}
+
+		return kmer.And(k.leftMask).RightShift(2).Or(kmer.And(k.rightMask))
+	}
+
+	normalizedKmer := func(fw, rv T) T {
+
+		fw = makeSparseAt(fw)
+		rv = makeSparseAt(rv)
+
+		if fw.LessThan(rv) {
+			return fw
+		}
+
+		return rv
+	}
+
+	current := obifp.ZeroUint[T]()
+	ccurrent := obifp.ZeroUint[T]()
+	lshift := uint(2 * (k.Kmersize - 1))
+
+	sup := sequence.Len() - int(k.Kmersize) + 1
+
+	var rep []T
+	if buff == nil {
+		rep = make([]T, 0, sup)
+	} else {
+		rep = (*buff)[:0]
+	}
+
+	nuc := sequence.Sequence()
+
+	size := 0
+	for i := 0; i < len(nuc); i++ {
+		current = current.LeftShift(2)
+		ccurrent = ccurrent.RightShift(2)
+
+		code := iupac[nuc[i]]
+		ccode := iupac[revcompnuc[nuc[i]]]
+
+		if len(code) != 1 {
+			current = obifp.ZeroUint[T]()
+			ccurrent = obifp.ZeroUint[T]()
+			size = 0
+			continue
+		}
+
+		current = current.Or(obifp.From64[T](uint64(code[0])))
+		ccurrent = ccurrent.Or(obifp.From64[T](uint64(ccode[0])).LeftShift(lshift))
+
+		size++
+
+		if size == int(k.Kmersize) {
+
+			kmer := normalizedKmer(current, ccurrent)
+			rep = append(rep, kmer)
+			size--
+		}
+
+	}
+
+	return rep
+}
+
+func (k *KmerMap[T]) Push(sequence *obiseq.BioSequence, maxocc ...int) {
+	maxoccurs := -1
+	if len(maxocc) > 0 {
+		maxoccurs = maxocc[0]
+	}
+	kmers := k.NormalizedKmerSlice(sequence, nil)
+	for _, kmer := range kmers {
+		seqs := k.index[kmer]
+		if maxoccurs == -1 || len(seqs) <= maxoccurs {
+			seqs = append(seqs, sequence)
+			k.index[kmer] = seqs
+		}
+	}
+}
+
+func (k *KmerMap[T]) Query(sequence *obiseq.BioSequence) KmerMatch {
+	kmers := k.NormalizedKmerSlice(sequence, nil)
+	seqs := make([]*obiseq.BioSequence, 0)
+	rep := make(KmerMatch)
+
+	for _, kmer := range kmers {
+		if candidates, ok := k.index[kmer]; ok {
+			seqs = append(seqs, candidates...)
+		}
+	}
+
+	sort.Slice(seqs,
+		func(i, j int) bool {
+			return uintptr(unsafe.Pointer(seqs[i])) < uintptr(unsafe.Pointer(seqs[j]))
+		})
+
+	prevseq := (*obiseq.BioSequence)(nil)
+	n := 0
+	for _, seq := range seqs {
+
+		if seq != prevseq {
+			if prevseq != nil && prevseq != sequence {
+				rep[prevseq] = n
+			}
+			n = 1
+			prevseq = seq
+		}
+
+		n++
+	}
+
+	if prevseq != nil {
+		rep[prevseq] = n
+	}
+
+	return rep
+}
+
+func (k *KmerMatch) FilterMinCount(mincount int) {
+	for seq, count := range *k {
+		if count < mincount {
+			delete(*k, seq)
+		}
+	}
+}
+
+func (k *KmerMatch) Len() int {
+	return len(*k)
+}
+
+func (k *KmerMatch) Sequences() obiseq.BioSequenceSlice {
+	ks := make([]*obiseq.BioSequence, 0, len(*k))
+
+	for seq := range *k {
+		ks = append(ks, seq)
+	}
+
+	return ks
+}
+
+func (k *KmerMatch) Max() *obiseq.BioSequence {
+	max := 0
+	var maxseq *obiseq.BioSequence
+	for seq, n := range *k {
+		if max < n {
+			max = n
+			maxseq = seq
+		}
+	}
+	return maxseq
+}
+
+func NewKmerMap[T obifp.FPUint[T]](
+	sequences obiseq.BioSequenceSlice,
+	kmersize uint,
+	sparse bool,
+	maxoccurs int) *KmerMap[T] {
+	idx := make(map[T][]*obiseq.BioSequence)
+
+	sparseAt := -1
+
+	if sparse && kmersize%2 == 0 {
+		log.Warnf("Kmer size must be odd when using sparse mode")
+		kmersize++
+	}
+
+	if !sparse && kmersize%2 == 1 {
+		log.Warnf("Kmer size must be even when not using sparse mode")
+		kmersize--
+
+	}
+
+	if sparse {
+		sparseAt = int(kmersize / 2)
+	}
+
+	kmermask := obifp.OneUint[T]().LeftShift(kmersize * 2).Sub(obifp.OneUint[T]())
+	leftMask := obifp.ZeroUint[T]()
+	rightMask := obifp.ZeroUint[T]()
+
+	if sparseAt >= 0 {
+		if sparseAt >= int(kmersize) {
+			sparseAt = -1
+		} else {
+			pos := kmersize - 1 - uint(sparseAt)
+			left := uint(sparseAt) * 2
+			right := pos * 2
+
+			leftMask = obifp.OneUint[T]().LeftShift(left).Sub(obifp.OneUint[T]()).LeftShift(right + 2)
+			rightMask = obifp.OneUint[T]().LeftShift(right).Sub(obifp.OneUint[T]())
+		}
+	}
+
+	kmap := &KmerMap[T]{
+		Kmersize:  kmersize,
+		kmermask:  kmermask,
+		leftMask:  leftMask,
+		rightMask: rightMask,
+		index:     idx,
+		SparseAt:  sparseAt,
+	}
+
+	n := len(sequences)
+	pbopt := make([]progressbar.Option, 0, 5)
+	pbopt = append(pbopt,
+		progressbar.OptionSetWriter(os.Stderr),
+		progressbar.OptionSetWidth(15),
+		progressbar.OptionShowCount(),
+		progressbar.OptionShowIts(),
+		progressbar.OptionSetDescription("Indexing kmers"),
+	)
+
+	bar := progressbar.NewOptions(n, pbopt...)
+
+	for i, sequence := range sequences {
+		kmap.Push(sequence, maxoccurs)
+		if i%100 == 0 {
+			bar.Add(100)
+		}
+	}
+
+	if maxoccurs >= 0 {
+		for k, s := range kmap.index {
+			if len(s) >= maxoccurs {
+				delete(kmap.index, k)
+			}
+		}
+	}
+
+	return kmap
+}

pkg/obikmer/oneerror.go

@@ -0,0 +1,45 @@
+package obikmer
+
+import (
+	"iter"
+	"slices"
+)
+
+var baseError = map[byte]byte{
+	'a': 'b',
+	'c': 'd',
+	'g': 'h',
+	't': 'v',
+	'r': 'y',
+	'y': 'r',
+	's': 'w',
+	'w': 's',
+	'k': 'm',
+	'm': 'k',
+	'd': 'c',
+	'v': 't',
+	'h': 'g',
+	'b': 'a',
+}
+
+type BytesItem []byte
+
+func IterateOneError(kmer []byte) iter.Seq[BytesItem] {
+	lkmer := len(kmer)
+	return func(yield func(BytesItem) bool) {
+		for p := 0; p < lkmer; p++ {
+			for p < lkmer && kmer[p] == 'n' {
+				p++
+			}
+
+			if p < lkmer {
+				nkmer := slices.Clone(kmer)
+				nkmer[p] = baseError[kmer[p]]
+				if !yield(nkmer) {
+					return
+				}
+			}
+		}
+	}
+
+}

pkg/obilua/lua.go

@@ -4,9 +4,10 @@ import (
 	"bytes"
 	"fmt"
 	"os"
+	"reflect"
 
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
-	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
 	log "github.com/sirupsen/logrus"
 	lua "github.com/yuin/gopher-lua"
@@ -20,6 +21,8 @@ import (
 func NewInterpreter() *lua.LState {
 	lua := lua.NewState()
 
+	registerMutexType(lua)
+
 	RegisterObilib(lua)
 	RegisterObiContext(lua)
 
@@ -117,11 +120,17 @@ func LuaWorker(proto *lua.FunctionProto) obiseq.SeqWorker {
 				case *obiseq.BioSequenceSlice:
 					return *val, err
 				default:
-					return nil, fmt.Errorf("worker function doesn't return the correct type")
+					r := reflect.TypeOf(val)
+					return nil, fmt.Errorf("worker function doesn't return the correct type %s", r)
 				}
 			}
 
-			return nil, fmt.Errorf("worker function doesn't return the correct type")
+			// If worker retuns nothing then it is considered as nil biosequence
+			if _, ok = lreponse.(*lua.LNilType); ok {
+				return nil, nil
+			}
+
+			return nil, fmt.Errorf("worker function doesn't return the correct type %T", lreponse)
 		}
 
 		return f
@@ -145,7 +154,7 @@ func LuaProcessor(iterator obiiter.IBioSequence, name, program string, breakOnEr
 	newIter := obiiter.MakeIBioSequence()
 
 	if nworkers <= 0 {
-		nworkers = obioptions.CLIParallelWorkers()
+		nworkers = obidefault.ParallelWorkers()
 	}
 
 	newIter.Add(nworkers)
@@ -226,7 +235,6 @@ func LuaProcessor(iterator obiiter.IBioSequence, name, program string, breakOnEr
 			}
 
 			newIter.Push(obiiter.MakeBioSequenceBatch(seqs.Source(), seqs.Order(), ns))
-			seqs.Recycle(false)
 		}
 
 		newIter.Done()

pkg/obilua/lua_obicontext.go

@@ -36,6 +36,8 @@ func obicontextGetSet(interpreter *lua.LState) int {
 			__lua_obicontext.Store(key, float64(val))
 		case lua.LString:
 			__lua_obicontext.Store(key, string(val))
+		case *lua.LUserData:
+			__lua_obicontext.Store(key, val.Value)
 		case *lua.LTable:
 			__lua_obicontext.Store(key, Table2Interface(interpreter, val))
 		default:

pkg/obilua/lua_push_interface.go

@@ -1,6 +1,8 @@
 package obilua
 
 import (
+	"sync"
+
 	log "github.com/sirupsen/logrus"
 
 	lua "github.com/yuin/gopher-lua"
@@ -44,8 +46,12 @@ func pushInterfaceToLua(L *lua.LState, val interface{}) {
 		pushSliceNumericToLua(L, v)
 	case []bool:
 		pushSliceBoolToLua(L, v)
+	case []interface{}:
+		pushSliceInterfaceToLua(L, v)
 	case nil:
 		L.Push(lua.LNil)
+	case *sync.Mutex:
+		pushMutexToLua(L, v)
 	default:
 		log.Fatalf("Cannot deal with value (%T) : %v", val, val)
 	}
@@ -74,6 +80,29 @@ func pushMapStringInterfaceToLua(L *lua.LState, m map[string]interface{}) {
 	L.Push(luaTable)
 }
 
+func pushSliceInterfaceToLua(L *lua.LState, s []interface{}) {
+	// Create a new Lua table
+	luaTable := L.NewTable()
+	// Iterate over the Go map and set the key-value pairs in the Lua table
+	for _, value := range s {
+		switch v := value.(type) {
+		case int:
+			luaTable.Append(lua.LNumber(v))
+		case float64:
+			luaTable.Append(lua.LNumber(v))
+		case bool:
+			luaTable.Append(lua.LBool(v))
+		case string:
+			luaTable.Append(lua.LString(v))
+		default:
+			log.Fatalf("Doesn't deal with slice containing value %v of type %T", v, v)
+		}
+	}
+
+	// Push the Lua table onto the stack
+	L.Push(luaTable)
+}
+
 // pushMapStringIntToLua creates a new Lua table and iterates over the Go map to set key-value pairs in the Lua table. It then pushes the Lua table onto the stack.
 //
 // L *lua.LState - the Lua state

pkg/obilua/mutex.go

@@ -0,0 +1,63 @@
+package obilua
+
+import (
+	lua "github.com/yuin/gopher-lua"
+
+	"sync"
+)
+
+const luaMutexTypeName = "Mutex"
+
+func registerMutexType(luaState *lua.LState) {
+	mutexType := luaState.NewTypeMetatable(luaMutexTypeName)
+	luaState.SetGlobal(luaMutexTypeName, mutexType)
+
+	luaState.SetField(mutexType, "new", luaState.NewFunction(newMutex))
+
+	luaState.SetField(mutexType, "__index",
+		luaState.SetFuncs(luaState.NewTable(),
+			mutexMethods))
+}
+
+func mutex2Lua(interpreter *lua.LState, mutex *sync.Mutex) lua.LValue {
+	ud := interpreter.NewUserData()
+	ud.Value = mutex
+	interpreter.SetMetatable(ud, interpreter.GetTypeMetatable(luaMutexTypeName))
+
+	return ud
+}
+
+func pushMutexToLua(luaState *lua.LState, mutex *sync.Mutex) {
+	luaState.Push(mutex2Lua(luaState, mutex))
+}
+func newMutex(luaState *lua.LState) int {
+	m := &sync.Mutex{}
+	pushMutexToLua(luaState, m)
+	return 1
+}
+
+var mutexMethods = map[string]lua.LGFunction{
+	"lock":   mutexLock,
+	"unlock": mutexUnlock,
+}
+
+func checkMutex(L *lua.LState) *sync.Mutex {
+	ud := L.CheckUserData(1)
+	if mutex, ok := ud.Value.(*sync.Mutex); ok {
+		return mutex
+	}
+	L.ArgError(1, "Mutex expected")
+	return nil
+}
+
+func mutexLock(L *lua.LState) int {
+	mutex := checkMutex(L)
+	mutex.Lock()
+	return 0
+}
+
+func mutexUnlock(L *lua.LState) int {
+	mutex := checkMutex(L)
+	mutex.Unlock()
+	return 0
+}

pkg/obilua/obilib.go

@@ -4,4 +4,5 @@ import lua "github.com/yuin/gopher-lua"
 
 func RegisterObilib(luaState *lua.LState) {
 	RegisterObiSeq(luaState)
+	RegisterObiTaxonomy(luaState)
 }

pkg/obilua/obiseq.go

@@ -1,7 +1,9 @@
 package obilua
 
 import (
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiformats"
 	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax"
 	lua "github.com/yuin/gopher-lua"
 )
 
@@ -16,6 +18,7 @@ func registerBioSequenceType(luaState *lua.LState) {
 	bioSequenceType := luaState.NewTypeMetatable(luaBioSequenceTypeName)
 	luaState.SetGlobal(luaBioSequenceTypeName, bioSequenceType)
 	luaState.SetField(bioSequenceType, "new", luaState.NewFunction(newObiSeq))
+	luaState.SetField(bioSequenceType, "nil", obiseq2Lua(luaState, nil))
 
 	luaState.SetField(bioSequenceType, "__index",
 		luaState.SetFuncs(luaState.NewTable(),
@@ -47,18 +50,25 @@ func newObiSeq(luaState *lua.LState) int {
 }
 
 var bioSequenceMethods = map[string]lua.LGFunction{
-	"id":            bioSequenceGetSetId,
-	"sequence":      bioSequenceGetSetSequence,
-	"qualities":     bioSequenceGetSetQualities,
-	"definition":    bioSequenceGetSetDefinition,
-	"count":         bioSequenceGetSetCount,
-	"taxid":         bioSequenceGetSetTaxid,
-	"attribute":     bioSequenceGetSetAttribute,
-	"len":           bioSequenceGetLength,
-	"has_sequence":  bioSequenceHasSequence,
-	"has_qualities": bioSequenceHasQualities,
-	"source":        bioSequenceGetSource,
-	"md5":           bioSequenceGetMD5,
+	"id":                 bioSequenceGetSetId,
+	"sequence":           bioSequenceGetSetSequence,
+	"qualities":          bioSequenceGetSetQualities,
+	"definition":         bioSequenceGetSetDefinition,
+	"count":              bioSequenceGetSetCount,
+	"taxid":              bioSequenceGetSetTaxid,
+	"taxon":              bioSequenceGetSetTaxon,
+	"attribute":          bioSequenceGetSetAttribute,
+	"len":                bioSequenceGetLength,
+	"has_sequence":       bioSequenceHasSequence,
+	"has_qualities":      bioSequenceHasQualities,
+	"source":             bioSequenceGetSource,
+	"md5":                bioSequenceGetMD5,
+	"md5_string":         bioSequenceGetMD5String,
+	"subsequence":        bioSequenceGetSubsequence,
+	"reverse_complement": bioSequenceGetRevcomp,
+	"fasta":              bioSequenceGetFasta,
+	"fastq":              bioSequenceGetFastq,
+	"string":             bioSequenceAsString,
 }
 
 // checkBioSequence checks if the first argument in the Lua stack is a *obiseq.BioSequence.
@@ -147,10 +157,10 @@ func bioSequenceGetSetCount(luaState *lua.LState) int {
 func bioSequenceGetSetTaxid(luaState *lua.LState) int {
 	s := checkBioSequence(luaState)
 	if luaState.GetTop() == 2 {
-		s.SetTaxid(luaState.CheckInt(2))
+		s.SetTaxid(luaState.CheckString(2))
 		return 0
 	}
-	luaState.Push(lua.LNumber(s.Taxid()))
+	luaState.Push(lua.LString(s.Taxid()))
 	return 1
 }
 
@@ -224,3 +234,115 @@ func bioSequenceGetMD5(luaState *lua.LState) int {
 	luaState.Push(rt)
 	return 1
 }
+
+func bioSequenceGetMD5String(luaState *lua.LState) int {
+	s := checkBioSequence(luaState)
+	md5 := s.MD5String()
+	luaState.Push(lua.LString(md5))
+	return 1
+}
+
+func bioSequenceGetSubsequence(luaState *lua.LState) int {
+	s := checkBioSequence(luaState)
+	start := luaState.CheckInt(2)
+	end := luaState.CheckInt(3)
+	subseq, err := s.Subsequence(start, end, false)
+	if err != nil {
+		luaState.RaiseError("%s : Error on subseq: %v", s.Id(), err)
+		return 0
+	}
+	luaState.Push(obiseq2Lua(luaState, subseq))
+	return 1
+}
+
+func bioSequenceGetRevcomp(luaState *lua.LState) int {
+	s := checkBioSequence(luaState)
+	revcomp := s.ReverseComplement(false)
+	luaState.Push(obiseq2Lua(luaState, revcomp))
+	return 1
+}
+
+func bioSequenceGetSetTaxon(luaState *lua.LState) int {
+	s := checkBioSequence(luaState)
+
+	if luaState.GetTop() > 1 {
+		taxon := checkTaxon(luaState, 2)
+
+		s.SetTaxon(taxon)
+
+		return 0
+	}
+
+	taxon := s.Taxon(obitax.DefaultTaxonomy())
+	luaState.Push(taxon2Lua(luaState, taxon))
+
+	return 1
+}
+
+func bioSequenceGetFasta(luaState *lua.LState) int {
+	s := checkBioSequence(luaState)
+
+	formater := obiformats.FormatFastSeqJsonHeader
+
+	if luaState.GetTop() > 1 {
+		format := luaState.CheckString(2)
+		switch format {
+		case "json":
+			formater = obiformats.FormatFastSeqJsonHeader
+		case "obi":
+			formater = obiformats.FormatFastSeqOBIHeader
+		}
+	}
+
+	txt := obiformats.FormatFasta(s, formater)
+
+	luaState.Push(lua.LString(txt))
+	return 1
+}
+
+func bioSequenceGetFastq(luaState *lua.LState) int {
+	s := checkBioSequence(luaState)
+
+	formater := obiformats.FormatFastSeqJsonHeader
+
+	if luaState.GetTop() > 1 {
+		format := luaState.CheckString(2)
+		switch format {
+		case "json":
+			formater = obiformats.FormatFastSeqJsonHeader
+		case "obi":
+			formater = obiformats.FormatFastSeqOBIHeader
+		}
+	}
+
+	txt := obiformats.FormatFastq(s, formater)
+
+	luaState.Push(lua.LString(txt))
+	return 1
+}
+
+func bioSequenceAsString(luaState *lua.LState) int {
+	s := checkBioSequence(luaState)
+
+	formater := obiformats.FormatFastSeqJsonHeader
+	format := obiformats.FormatFasta
+
+	if s.HasQualities() {
+		format = obiformats.FormatFastq
+	}
+
+	if luaState.GetTop() > 1 {
+		format := luaState.CheckString(2)
+		switch format {
+		case "json":
+			formater = obiformats.FormatFastSeqJsonHeader
+		case "obi":
+			formater = obiformats.FormatFastSeqOBIHeader
+		}
+	}
+
+	txt := format(s, formater)
+
+	luaState.Push(lua.LString(txt))
+	return 1
+}