refactor code and change algorithm used to read from many files

pkg/obiformats/batch_of_files_reader.go

@@ -0,0 +1,64 @@
+package obiformats
+
+import (
+	"log"
+
+	"git.metabarcoding.org/lecasofts/go/obitools/pkg/goutils"
+	"git.metabarcoding.org/lecasofts/go/obitools/pkg/obiiter"
+)
+
+func ReadSequencesBatchFromFiles(filenames []string,
+	reader IBatchReader,
+	concurrent_readers int,
+	options ...WithOption) obiiter.IBioSequenceBatch {
+
+	if reader == nil {
+		reader = ReadSequencesBatchFromFile
+	}
+
+	batchiter := obiiter.MakeIBioSequenceBatch(0)
+	nextCounter := goutils.AtomicCounter()
+
+	batchiter.Add(concurrent_readers)
+
+	go func() {
+		batchiter.WaitAndClose()
+		log.Println("Finnished to read every files")
+	}()
+
+	filenameChan := make(chan string)
+
+	go func() {
+		for _, filename := range filenames {
+			filenameChan <- filename
+		}
+
+		close(filenameChan)
+	}()
+
+	for i := 0; i < concurrent_readers; i++ {
+		go func() {
+
+			for filename := range filenameChan {
+				iter, err := reader(filename, options...)
+
+				if err != nil {
+					log.Panicf("Cannot open file %s : %v", filename, err)
+				}
+
+				log.Printf("Start reading of file : %s", filename)
+
+				for iter.Next() {
+					batch := iter.Get()
+					batchiter.Push(batch.Reorder(nextCounter()))
+				}
+
+				log.Printf("End of reading of file : %s", filename)
+
+			}
+			batchiter.Done()
+		}()
+	}
+
+	return batchiter
+}

pkg/obiformats/batch_reader_type.go

@@ -0,0 +1,5 @@
+package obiformats
+
+import "git.metabarcoding.org/lecasofts/go/obitools/pkg/obiiter"
+
+type IBatchReader func(string, ...WithOption) (obiiter.IBioSequenceBatch, error)

pkg/obiformats/fastseq_write_fasta.go

@@ -120,7 +120,9 @@ func WriteFastaBatch(iterator obiiter.IBioSequenceBatch,
 
 	ff := func(iterator obiiter.IBioSequenceBatch) {
 		for iterator.Next() {
+
 			batch := iterator.Get()
+
 			chunkchan <- FileChunck{
 				FormatFastaBatch(batch, header_format),
 				batch.Order(),