diff --git a/pkg/obicorazick/worker.go b/pkg/obicorazick/worker.go index a18791f..bae1c68 100644 --- a/pkg/obicorazick/worker.go +++ b/pkg/obicorazick/worker.go @@ -14,35 +14,39 @@ func AhoCorazickWorker(slot string, patterns []string) obiseq.SeqWorker { sizebatch:=10000000 nmatcher := len(patterns) / sizebatch + 1 - log.Infof("Building AhoCorasick %d matcher for %d patterns in slot %s", + log.Infof("Building AhoCorasick %d matcher for %d patterns in slot %s", nmatcher, len(patterns), slot) if nmatcher == 0 { log.Errorln("No patterns provided") } - + matchers := make([]*ahocorasick.Matcher, nmatcher) - ieme := make(chan int) + ieme := make(chan int) mutex := &sync.WaitGroup{} npar := min(obidefault.ParallelWorkers(), nmatcher) mutex.Add(npar) - pbopt := make([]progressbar.Option, 0, 5) - pbopt = append(pbopt, - progressbar.OptionSetWriter(os.Stderr), - progressbar.OptionSetWidth(15), - progressbar.OptionShowCount(), - progressbar.OptionShowIts(), - progressbar.OptionSetDescription("Building AhoCorasick matcher..."), - ) + var bar *progressbar.ProgressBar + if obidefault.ProgressBar() { + pbopt := make([]progressbar.Option, 0, 5) + pbopt = append(pbopt, + progressbar.OptionSetWriter(os.Stderr), + progressbar.OptionSetWidth(15), + progressbar.OptionShowCount(), + progressbar.OptionShowIts(), + progressbar.OptionSetDescription("Building AhoCorasick matcher..."), + ) - bar := progressbar.NewOptions(nmatcher, pbopt...) - bar.Add(0) + bar = progressbar.NewOptions(nmatcher, pbopt...) + } builder := func() { - for i := range ieme { + for i := range ieme { matchers[i] = ahocorasick.CompileStrings(patterns[i*sizebatch:min((i+1)*sizebatch,len(patterns))]) - bar.Add(1) + if bar != nil { + bar.Add(1) + } } mutex.Done() } diff --git a/pkg/obidefault/progressbar.go b/pkg/obidefault/progressbar.go new file mode 100644 index 0000000..3bce334 --- /dev/null +++ b/pkg/obidefault/progressbar.go @@ -0,0 +1,19 @@ +package obidefault + +var __no_progress_bar__ = false + +func ProgressBar() bool { + return !__no_progress_bar__ +} + +func NoProgressBar() bool { + return __no_progress_bar__ +} + +func SetNoProgressBar(b bool) { + __no_progress_bar__ = b +} + +func NoProgressBarPtr() *bool { + return &__no_progress_bar__ +} diff --git a/pkg/obiiter/speed.go b/pkg/obiiter/speed.go index 19dfd04..6f93b02 100644 --- a/pkg/obiiter/speed.go +++ b/pkg/obiiter/speed.go @@ -5,18 +5,30 @@ import ( "os" "time" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault" "github.com/schollz/progressbar/v3" ) func (iterator IBioSequence) Speed(message string, size ...int) IBioSequence { - // If the STDERR is redicted and doesn't end up to a terminal + // If the progress bar is disabled via --no-progressbar option + if !obidefault.ProgressBar() { + return iterator + } + + // If the STDERR is redirected and doesn't end up to a terminal // No progress bar is printed. o, _ := os.Stderr.Stat() if (o.Mode() & os.ModeCharDevice) != os.ModeCharDevice { return iterator } + // If stdout is piped, no progress bar is printed. + oo, _ := os.Stdout.Stat() + if (oo.Mode() & os.ModeNamedPipe) == os.ModeNamedPipe { + return iterator + } + newIter := MakeIBioSequence() newIter.Add(1) diff --git a/pkg/obikmer/kmermap.go b/pkg/obikmer/kmermap.go index 8ab2752..363165f 100644 --- a/pkg/obikmer/kmermap.go +++ b/pkg/obikmer/kmermap.go @@ -5,6 +5,7 @@ import ( "sort" "unsafe" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obifp" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obilog" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq" @@ -267,20 +268,23 @@ func NewKmerMap[T obifp.FPUint[T]]( } n := len(sequences) - pbopt := make([]progressbar.Option, 0, 5) - pbopt = append(pbopt, - progressbar.OptionSetWriter(os.Stderr), - progressbar.OptionSetWidth(15), - progressbar.OptionShowCount(), - progressbar.OptionShowIts(), - progressbar.OptionSetDescription("Indexing kmers"), - ) + var bar *progressbar.ProgressBar + if obidefault.ProgressBar() { + pbopt := make([]progressbar.Option, 0, 5) + pbopt = append(pbopt, + progressbar.OptionSetWriter(os.Stderr), + progressbar.OptionSetWidth(15), + progressbar.OptionShowCount(), + progressbar.OptionShowIts(), + progressbar.OptionSetDescription("Indexing kmers"), + ) - bar := progressbar.NewOptions(n, pbopt...) + bar = progressbar.NewOptions(n, pbopt...) + } for i, sequence := range sequences { kmap.Push(sequence, maxoccurs) - if i%100 == 0 { + if bar != nil && i%100 == 0 { bar.Add(100) } } diff --git a/pkg/obitools/obiclean/graph.go b/pkg/obitools/obiclean/graph.go index 9d1ba4b..9952c5e 100644 --- a/pkg/obitools/obiclean/graph.go +++ b/pkg/obitools/obiclean/graph.go @@ -13,6 +13,7 @@ import ( log "github.com/sirupsen/logrus" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obialign" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" "github.com/schollz/progressbar/v3" ) @@ -69,16 +70,18 @@ func EmpiricalDistCsv(filename string, data [][]Ratio, compressed bool) { } defer destfile.Close() - pbopt := make([]progressbar.Option, 0, 5) - pbopt = append(pbopt, - progressbar.OptionSetWriter(os.Stderr), - progressbar.OptionSetWidth(15), - progressbar.OptionShowIts(), - progressbar.OptionSetPredictTime(true), - progressbar.OptionSetDescription("[Save CSV stat ratio file]"), - ) - - bar := progressbar.NewOptions(len(data), pbopt...) + var bar *progressbar.ProgressBar + if obidefault.ProgressBar() { + pbopt := make([]progressbar.Option, 0, 5) + pbopt = append(pbopt, + progressbar.OptionSetWriter(os.Stderr), + progressbar.OptionSetWidth(15), + progressbar.OptionShowIts(), + progressbar.OptionSetPredictTime(true), + progressbar.OptionSetDescription("[Save CSV stat ratio file]"), + ) + bar = progressbar.NewOptions(len(data), pbopt...) + } fmt.Fprintln(destfile, "Sample,Origin_id,Origin_status,Origin,Mutant,Origin_Weight,Mutant_Weight,Origin_Count,Mutant_Count,Position,Origin_length,A,C,G,T") for code, dist := range data { @@ -101,7 +104,9 @@ func EmpiricalDistCsv(filename string, data [][]Ratio, compressed bool) { ratio.T, ) } - bar.Add(1) + if bar != nil { + bar.Add(1) + } } } @@ -116,7 +121,7 @@ func Gml(seqs *[]*seqPCR, sample string, statThreshold int) string { directed 1 {{range $index, $data:= .}} {{ if or $data.Edges (gt $data.SonCount 0)}} - node [ id {{$index}} + node [ id {{$index}} graphics [ type "{{ Shape $data.Count }}" fill "{{ if and (gt $data.SonCount 0) (not $data.Edges)}}#0000FF{{ else }}#00FF00{{ end }}" @@ -130,15 +135,15 @@ func Gml(seqs *[]*seqPCR, sample string, statThreshold int) string { {{range $index, $data:= .}} {{range $i, $edge:= $data.Edges}} - edge [ source {{$index}} - target {{$edge.Father}} + edge [ source {{$index}} + target {{$edge.Father}} color "{{ if gt (index $data.Edges $i).Dist 1 }}#FF0000{{ else }}#00FF00{{ end }}" label "{{(index $data.Edges $i).Dist}}" ] {{ end }} {{ end }} - ] - + ] + ` tmpl, err := digraphTpl.Funcs(template.FuncMap{ @@ -181,16 +186,18 @@ func SaveGMLGraphs(dirname string, } } - pbopt := make([]progressbar.Option, 0, 5) - pbopt = append(pbopt, - progressbar.OptionSetWriter(os.Stderr), - progressbar.OptionSetWidth(15), - progressbar.OptionShowIts(), - progressbar.OptionSetPredictTime(true), - progressbar.OptionSetDescription("[Save GML Graph files]"), - ) - - bar := progressbar.NewOptions(len(samples), pbopt...) + var bar *progressbar.ProgressBar + if obidefault.ProgressBar() { + pbopt := make([]progressbar.Option, 0, 5) + pbopt = append(pbopt, + progressbar.OptionSetWriter(os.Stderr), + progressbar.OptionSetWidth(15), + progressbar.OptionShowIts(), + progressbar.OptionSetPredictTime(true), + progressbar.OptionSetDescription("[Save GML Graph files]"), + ) + bar = progressbar.NewOptions(len(samples), pbopt...) + } for name, seqs := range samples { @@ -204,7 +211,9 @@ func SaveGMLGraphs(dirname string, file.WriteString(Gml(seqs, name, statThreshold)) file.Close() - bar.Add(1) + if bar != nil { + bar.Add(1) + } } } @@ -495,37 +504,44 @@ func BuildSeqGraph(samples map[string]*[]*seqPCR, npairs += nseq * (nseq - 1) / 2 } - pbopt := make([]progressbar.Option, 0, 5) - pbopt = append(pbopt, - progressbar.OptionSetWriter(os.Stderr), - progressbar.OptionSetWidth(15), - progressbar.OptionShowIts(), - progressbar.OptionSetPredictTime(true), - progressbar.OptionSetDescription("[One error graph]"), - ) - - bar := progressbar.NewOptions(npairs, pbopt...) - for _, seqs := range samples { - np := buildSamplePairs(seqs, workers) - - bar.Add(np) - } - - if maxError > 1 { - pbopt = make([]progressbar.Option, 0, 5) + var bar *progressbar.ProgressBar + if obidefault.ProgressBar() { + pbopt := make([]progressbar.Option, 0, 5) pbopt = append(pbopt, progressbar.OptionSetWriter(os.Stderr), progressbar.OptionSetWidth(15), progressbar.OptionShowIts(), progressbar.OptionSetPredictTime(true), - progressbar.OptionSetDescription("[Adds multiple errors]"), + progressbar.OptionSetDescription("[One error graph]"), ) - bar = progressbar.NewOptions(npairs, pbopt...) + } - for _, seqs := range samples { - np := extendSimilarityGraph(seqs, maxError, workers) + for _, seqs := range samples { + np := buildSamplePairs(seqs, workers) + if bar != nil { bar.Add(np) } } + + if maxError > 1 { + if obidefault.ProgressBar() { + pbopt := make([]progressbar.Option, 0, 5) + pbopt = append(pbopt, + progressbar.OptionSetWriter(os.Stderr), + progressbar.OptionSetWidth(15), + progressbar.OptionShowIts(), + progressbar.OptionSetPredictTime(true), + progressbar.OptionSetDescription("[Adds multiple errors]"), + ) + bar = progressbar.NewOptions(npairs, pbopt...) + } + + for _, seqs := range samples { + np := extendSimilarityGraph(seqs, maxError, workers) + if bar != nil { + bar.Add(np) + } + } + } } diff --git a/pkg/obitools/obiconvert/options.go b/pkg/obitools/obiconvert/options.go index 4c5f0b2..a860194 100644 --- a/pkg/obitools/obiconvert/options.go +++ b/pkg/obitools/obiconvert/options.go @@ -31,7 +31,6 @@ var __output_in_json__ = false var __output_fastjson_format__ = false var __output_fastobi_format__ = false -var __no_progress_bar__ = false var __skip_empty__ = false var __skip_on_error__ = false @@ -82,7 +81,7 @@ func InputOptionSet(options *getoptions.GetOpt) { } func OutputModeOptionSet(options *getoptions.GetOpt, compressed bool) { - options.BoolVar(&__no_progress_bar__, "no-progressbar", false, + options.BoolVar(obidefault.NoProgressBarPtr(), "no-progressbar", obidefault.NoProgressBar(), options.Description("Disable the progress bar printing")) if compressed { @@ -233,7 +232,7 @@ func CLIProgressBar() bool { oo, _ := os.Stdout.Stat() toPipe := (oo.Mode() & os.ModeNamedPipe) == os.ModeNamedPipe - return onTerminal && !toPipe && !__no_progress_bar__ + return onTerminal && !toPipe && obidefault.ProgressBar() } func CLIOutPutFileName() string { diff --git a/pkg/obitools/obiconvert/sequence_reader.go b/pkg/obitools/obiconvert/sequence_reader.go index c985a18..aff14fd 100644 --- a/pkg/obitools/obiconvert/sequence_reader.go +++ b/pkg/obitools/obiconvert/sequence_reader.go @@ -210,9 +210,7 @@ func CLIReadBioSequences(filenames ...string) (obiiter.IBioSequence, error) { } - if CLIProgressBar() { - iterator = iterator.Speed("Reading sequences") - } + iterator = iterator.Speed("Reading sequences") return iterator, nil } diff --git a/pkg/obitools/obicsv/obicsv.go b/pkg/obitools/obicsv/obicsv.go index daf6704..859e8cc 100644 --- a/pkg/obitools/obicsv/obicsv.go +++ b/pkg/obitools/obicsv/obicsv.go @@ -12,9 +12,7 @@ import ( func CLIWriteSequenceCSV(iterator obiiter.IBioSequence, terminalAction bool, filenames ...string) *obiitercsv.ICSVRecord { - if obiconvert.CLIProgressBar() { - iterator = iterator.Speed("Writing CSV") - } + iterator = iterator.Speed("Writing CSV") opts := make([]WithOption, 0, 10) diff --git a/pkg/obitools/obilandmark/obilandmark.go b/pkg/obitools/obilandmark/obilandmark.go index c49b400..175ee0e 100644 --- a/pkg/obitools/obilandmark/obilandmark.go +++ b/pkg/obitools/obilandmark/obilandmark.go @@ -42,16 +42,19 @@ func MapOnLandmarkSequences(library obiseq.BioSequenceSlice, landmark_idx []int, seqworld := obiutils.Make2DArray[float64](library_size, n_landmark) - pbopt := make([]progressbar.Option, 0, 5) - pbopt = append(pbopt, - progressbar.OptionSetWriter(os.Stderr), - progressbar.OptionSetWidth(15), - progressbar.OptionShowCount(), - progressbar.OptionShowIts(), - progressbar.OptionSetDescription("[Sequence mapping]"), - ) + var bar *progressbar.ProgressBar + if obidefault.ProgressBar() { + pbopt := make([]progressbar.Option, 0, 5) + pbopt = append(pbopt, + progressbar.OptionSetWriter(os.Stderr), + progressbar.OptionSetWidth(15), + progressbar.OptionShowCount(), + progressbar.OptionShowIts(), + progressbar.OptionSetDescription("[Sequence mapping]"), + ) - bar := progressbar.NewOptions(library_size, pbopt...) + bar = progressbar.NewOptions(library_size, pbopt...) + } waiting := sync.WaitGroup{} waiting.Add(nworkers) @@ -66,7 +69,9 @@ func MapOnLandmarkSequences(library obiseq.BioSequenceSlice, landmark_idx []int, match, lalign := obialign.FastLCSScore(landmark, seq, -1, &buffer) coord[j] = float64(lalign - match) } - bar.Add(1) + if bar != nil { + bar.Add(1) + } } waiting.Done() } @@ -170,23 +175,26 @@ func CLISelectLandmarkSequences(iterator obiiter.IBioSequence) obiiter.IBioSeque taxa.Set(i, taxon) } - pbopt := make([]progressbar.Option, 0, 5) - pbopt = append(pbopt, - progressbar.OptionSetWriter(os.Stderr), - progressbar.OptionSetWidth(15), - progressbar.OptionShowCount(), - progressbar.OptionShowIts(), - progressbar.OptionSetDescription("[Sequence Indexing]"), - ) + var bar2 *progressbar.ProgressBar + if obidefault.ProgressBar() { + pbopt := make([]progressbar.Option, 0, 5) + pbopt = append(pbopt, + progressbar.OptionSetWriter(os.Stderr), + progressbar.OptionSetWidth(15), + progressbar.OptionShowCount(), + progressbar.OptionShowIts(), + progressbar.OptionSetDescription("[Sequence Indexing]"), + ) - bar := progressbar.NewOptions(len(library), pbopt...) + bar2 = progressbar.NewOptions(len(library), pbopt...) + } for i, seq := range library { idx := obirefidx.GeomIndexSesquence(i, library, taxa, taxo) seq.SetOBITagGeomRefIndex(idx) - if i%10 == 0 { - bar.Add(10) + if bar2 != nil && i%10 == 0 { + bar2.Add(10) } } } diff --git a/pkg/obitools/obirefidx/famlilyindexing.go b/pkg/obitools/obirefidx/famlilyindexing.go index c862948..190c30c 100644 --- a/pkg/obitools/obirefidx/famlilyindexing.go +++ b/pkg/obitools/obirefidx/famlilyindexing.go @@ -207,16 +207,19 @@ func IndexFamilyDB(iterator obiiter.IBioSequence) obiiter.IBioSequence { log.Infof("Done. Found %d clusters", clusters.Len()) - pbopt := make([]progressbar.Option, 0, 5) - pbopt = append(pbopt, - progressbar.OptionSetWriter(os.Stderr), - progressbar.OptionSetWidth(15), - progressbar.OptionShowCount(), - progressbar.OptionShowIts(), - progressbar.OptionSetDescription("Cluster indexing"), - ) + var bar *progressbar.ProgressBar + if obidefault.ProgressBar() { + pbopt := make([]progressbar.Option, 0, 5) + pbopt = append(pbopt, + progressbar.OptionSetWriter(os.Stderr), + progressbar.OptionSetWidth(15), + progressbar.OptionShowCount(), + progressbar.OptionShowIts(), + progressbar.OptionSetDescription("Cluster indexing"), + ) - bar := progressbar.NewOptions(len(clusters), pbopt...) + bar = progressbar.NewOptions(len(clusters), pbopt...) + } limits := make(chan [2]int) waiting := sync.WaitGroup{} @@ -233,7 +236,9 @@ func IndexFamilyDB(iterator obiiter.IBioSequence) obiiter.IBioSequence { for i := l[0]; i < l[1]; i++ { idx := IndexSequence(i, clusters, &kcluster, taxa, taxonomy) clusters[i].SetOBITagRefIndex(idx) - bar.Add(1) + if bar != nil { + bar.Add(1) + } } } diff --git a/pkg/obitools/obirefidx/obirefidx.go b/pkg/obitools/obirefidx/obirefidx.go index 1e4dd74..fa29d29 100644 --- a/pkg/obitools/obirefidx/obirefidx.go +++ b/pkg/obitools/obirefidx/obirefidx.go @@ -239,16 +239,19 @@ func IndexReferenceDB(iterator obiiter.IBioSequence) obiiter.IBioSequence { log.Info("done") - pbopt := make([]progressbar.Option, 0, 5) - pbopt = append(pbopt, - progressbar.OptionSetWriter(os.Stderr), - progressbar.OptionSetWidth(15), - progressbar.OptionShowCount(), - progressbar.OptionShowIts(), - progressbar.OptionSetDescription("[Sequence Processing]"), - ) + var bar *progressbar.ProgressBar + if obidefault.ProgressBar() { + pbopt := make([]progressbar.Option, 0, 5) + pbopt = append(pbopt, + progressbar.OptionSetWriter(os.Stderr), + progressbar.OptionSetWidth(15), + progressbar.OptionShowCount(), + progressbar.OptionShowIts(), + progressbar.OptionSetDescription("[Sequence Processing]"), + ) - bar := progressbar.NewOptions(len(references), pbopt...) + bar = progressbar.NewOptions(len(references), pbopt...) + } limits := make(chan [2]int) indexed := obiiter.MakeIBioSequence() @@ -267,7 +270,9 @@ func IndexReferenceDB(iterator obiiter.IBioSequence) obiiter.IBioSequence { iref := references[i].Copy() iref.SetOBITagRefIndex(idx) sl = append(sl, iref) - bar.Add(1) + if bar != nil { + bar.Add(1) + } } indexed.Push(obiiter.MakeBioSequenceBatch(source, l[0]/10, sl)) }