Small change in parallelisation tuning

Former-commit-id: 3fe2495b7fd86a0ba47dd87907323a457bae481a
This commit is contained in:
2023-04-05 14:15:47 +02:00
parent 2e7fc9a589
commit d70bb45f3f
5 changed files with 25 additions and 9 deletions

View File

@ -25,7 +25,8 @@ func main() {
// trace.Start(ftrace) // trace.Start(ftrace)
// defer trace.Stop() // defer trace.Stop()
obioptions.SetWorkerPerCore(1) obioptions.SetWorkerPerCore(2)
obioptions.SetReadWorkerPerCore(0.5)
optionParser := obioptions.GenerateOptionParser(obipcr.OptionSet) optionParser := obioptions.GenerateOptionParser(obipcr.OptionSet)

View File

@ -14,7 +14,8 @@ import (
) )
var _Debug = false var _Debug = false
var _WorkerPerCore = 2 var _WorkerPerCore = 2.0
var _ReadWorkerPerCore = 1.0
var _MaxAllowedCPU = runtime.NumCPU() var _MaxAllowedCPU = runtime.NumCPU()
var _BatchSize = 5000 var _BatchSize = 5000
var _Pprof = false var _Pprof = false
@ -92,9 +93,14 @@ func CLIIsDebugMode() bool {
// CLIParallelWorkers returns the number of parallel workers requested by // CLIParallelWorkers returns the number of parallel workers requested by
// the command line option --workers|-w. // the command line option --workers|-w.
func CLIParallelWorkers() int { func CLIParallelWorkers() int {
return _MaxAllowedCPU * _WorkerPerCore return int(float64(_MaxAllowedCPU) * float64(_WorkerPerCore))
} }
func CLIReadParallelWorkers() int {
return int(float64(_MaxAllowedCPU) * float64(_ReadWorkerPerCore))
}
// CLIParallelWorkers returns the number of parallel workers requested by // CLIParallelWorkers returns the number of parallel workers requested by
// the command line option --workers|-w. // the command line option --workers|-w.
func CLIMaxCPU() int { func CLIMaxCPU() int {
@ -116,14 +122,22 @@ func DebugOff() {
_Debug = false _Debug = false
} }
func SetWorkerPerCore(n int) { func SetWorkerPerCore(n float64) {
_WorkerPerCore = n _WorkerPerCore = n
} }
func WorkerPerCore() int { func SetReadWorkerPerCore(n float64) {
_ReadWorkerPerCore = n
}
func WorkerPerCore() float64 {
return _WorkerPerCore return _WorkerPerCore
} }
func ReadWorkerPerCore() float64 {
return _ReadWorkerPerCore
}
func SetBatchSize(n int) { func SetBatchSize(n int) {
_BatchSize = n _BatchSize = n
} }

View File

@ -90,7 +90,7 @@ func CLIReadBioSequences(filenames ...string) (obiiter.IBioSequence, error) {
opts = append(opts, obiformats.OptionsFastSeqHeaderParser(obiformats.ParseGuessedFastSeqHeader)) opts = append(opts, obiformats.OptionsFastSeqHeaderParser(obiformats.ParseGuessedFastSeqHeader))
} }
nworkers := obioptions.CLIParallelWorkers() nworkers := obioptions.CLIReadParallelWorkers()
if nworkers < 2 { if nworkers < 2 {
nworkers = 2 nworkers = 2
} }

View File

@ -56,5 +56,5 @@ func IExtractBarcode(iterator obiiter.IBioSequence) (obiiter.IBioSequence, error
} }
log.Printf("Sequence demultiplexing using %d workers\n", obioptions.CLIParallelWorkers()) log.Printf("Sequence demultiplexing using %d workers\n", obioptions.CLIParallelWorkers())
return newIter.Speed("Demultiplexing"), nil return newIter, nil
} }

View File

@ -1,11 +1,12 @@
package obitag package obitag
import ( import (
"log"
"math" "math"
"strconv" "strconv"
"strings" "strings"
log "github.com/sirupsen/logrus"
"git.metabarcoding.org/lecasofts/go/obitools/pkg/obialign" "git.metabarcoding.org/lecasofts/go/obitools/pkg/obialign"
"git.metabarcoding.org/lecasofts/go/obitools/pkg/obiiter" "git.metabarcoding.org/lecasofts/go/obitools/pkg/obiiter"
"git.metabarcoding.org/lecasofts/go/obitools/pkg/obikmer" "git.metabarcoding.org/lecasofts/go/obitools/pkg/obikmer"