Start to use leveled log

pkg/obichunk/chunk_on_disk.go

@@ -3,10 +3,11 @@ package obichunk
 import (
 	"io/fs"
 	"io/ioutil"
-	"log"
 	"os"
 	"path/filepath"
 
+	log "github.com/sirupsen/logrus"
+
 	"git.metabarcoding.org/lecasofts/go/obitools/pkg/obiformats"
 	"git.metabarcoding.org/lecasofts/go/obitools/pkg/obiiter"
 	"git.metabarcoding.org/lecasofts/go/obitools/pkg/obiseq"
@@ -55,7 +56,7 @@ func ISequenceChunkOnDisk(iterator obiiter.IBioSequenceBatch,
 	go func() {
 		defer func() {
 			os.RemoveAll(dir)
-			log.Println("Clear the cache directory")
+			log.Debugln("Clear the cache directory")
 		}()
 
 		newIter.Wait()
@@ -68,7 +69,8 @@ func ISequenceChunkOnDisk(iterator obiiter.IBioSequenceBatch,
 	)
 
 	fileNames := find(dir, ".fastx")
-	log.Println("batch count ", len(fileNames))
+	nbatch := len(fileNames)
+	log.Infof("Data splitted over %d batches", nbatch)
 
 	go func() {
 
@@ -88,6 +90,8 @@ func ISequenceChunkOnDisk(iterator obiiter.IBioSequenceBatch,
 			}
 
 			newIter.Push(obiiter.MakeBioSequenceBatch(order, chunck))
+			log.Infof("Start processing of batch %d/%d : %d sequences",
+				order, nbatch, len(chunck))
 
 		}
 

pkg/obichunk/chunks.go

@@ -1,7 +1,7 @@
 package obichunk
 
 import (
-	"log"
+	log "github.com/sirupsen/logrus"
 	"sync"
 
 	"git.metabarcoding.org/lecasofts/go/obitools/pkg/obiiter"

pkg/obichunk/options.go

@@ -9,6 +9,7 @@ type __options__ struct {
 	bufferSize      int
 	batchSize       int
 	parallelWorkers int
+	noSingleton     bool
 }
 
 type Options struct {
@@ -27,6 +28,7 @@ func MakeOptions(setters []WithOption) Options {
 		bufferSize:      2,
 		batchSize:       5000,
 		parallelWorkers: 4,
+		noSingleton:     false,
 	}
 
 	opt := Options{&o}
@@ -79,6 +81,10 @@ func (opt Options) SortOnDisk() bool {
 	return opt.pointer.cacheOnDisk
 }
 
+func (opt Options) NoSingleton() bool {
+	return opt.pointer.noSingleton
+}
+
 func OptionSortOnDisk() WithOption {
 	f := WithOption(func(opt Options) {
 		opt.pointer.cacheOnDisk = true
@@ -149,3 +155,19 @@ func OptionsBufferSize(size int) WithOption {
 
 	return f
 }
+
+func OptionsNoSingleton() WithOption {
+	f := WithOption(func(opt Options) {
+		opt.pointer.noSingleton = true
+	})
+
+	return f
+}
+
+func OptionsWithSingleton() WithOption {
+	f := WithOption(func(opt Options) {
+		opt.pointer.noSingleton = false
+	})
+
+	return f
+}

pkg/obichunk/subchunks.go

@@ -1,10 +1,11 @@
 package obichunk
 
 import (
-	"log"
 	"sort"
 	"sync/atomic"
 
+	log "github.com/sirupsen/logrus"
+
 	"git.metabarcoding.org/lecasofts/go/obitools/pkg/obiiter"
 	"git.metabarcoding.org/lecasofts/go/obitools/pkg/obiseq"
 )
@@ -100,7 +101,7 @@ func ISequenceSubChunk(iterator obiiter.IBioSequenceBatch,
 				classifier.Reset()
 
 				if cap(ordered) < batch.Length() {
-					log.Println("Allocate a new ordered sequences : ", batch.Length())
+					log.Debugln("Allocate a new ordered sequences : ", batch.Length())
 					ordered = make([]sSS, batch.Length())
 				} else {
 					ordered = ordered[:batch.Length()]

pkg/obichunk/unique.go

@@ -3,6 +3,8 @@ package obichunk
 import (
 	"sync"
 
+	log "github.com/sirupsen/logrus"
+
 	"git.metabarcoding.org/lecasofts/go/obitools/pkg/obiiter"
 	"git.metabarcoding.org/lecasofts/go/obitools/pkg/obiseq"
 )
@@ -16,6 +18,10 @@ func IUniqueSequence(iterator obiiter.IBioSequenceBatch,
 
 	iUnique := obiiter.MakeIBioSequenceBatch(opts.BufferSize())
 
+	iterator = iterator.Speed("Splitting data set")
+
+	log.Infoln("Starting data splitting")
+
 	if opts.SortOnDisk() {
 		nworkers = 1
 		iterator, err = ISequenceChunkOnDisk(iterator,
@@ -36,6 +42,8 @@ func IUniqueSequence(iterator obiiter.IBioSequenceBatch,
 		}
 	}
 
+	log.Infoln("End of the data splitting")
+
 	iUnique.Add(nworkers)
 
 	go func() {
@@ -83,7 +91,12 @@ func IUniqueSequence(iterator obiiter.IBioSequenceBatch,
 			batch := input.Get()
 
 			if icat < 0 || len(batch.Slice()) == 1 {
-				iUnique.Push(batch.Reorder(nextOrder()))
+				if opts.NoSingleton() && len(batch.Slice()) == 1 && batch.Slice()[0].Count() == 1 {
+					batch.Slice()[0].Recycle()
+					batch.Recycle()
+				} else {
+					iUnique.Push(batch.Reorder(nextOrder()))
+				}
 			} else {
 				next.Push(batch.Reorder(o))
 				o++
@@ -111,5 +124,5 @@ func IUniqueSequence(iterator obiiter.IBioSequenceBatch,
 		opts.BufferSize(),
 	)
 
-	return iMerged.Speed(), nil
+	return iMerged.Speed("Variants identified"), nil
 }