obitools/obitools4
2
0
Fork 0
mirror of https://github.com/metabarcoding/obitools4.git synced 2025-12-13 02:30:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

introduce obidefault

Browse Source
This commit is contained in:
Eric Coissac
2025-01-27 17:12:45 +01:00
parent 0a567f621c
commit 7c4042df6b
69 changed files with 505 additions and 479 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
pkg/obitools/obiannotate/obiannotate.go
View File

@@ -7,8 +7,8 @@ import (
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiapat"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obicorazick"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obigrep"
@@ -345,7 +345,7 @@ func CLIAnnotationPipeline() obiiter.Pipeable {
worker := CLIAnnotationWorker()
annotator := obiseq.SeqToSliceConditionalWorker(predicate, worker, false)
f := obiiter.SliceWorkerPipe(annotator, false, obioptions.CLIParallelWorkers())
f := obiiter.SliceWorkerPipe(annotator, false, obidefault.ParallelWorkers())
return f
}

4
pkg/obitools/obiclean/obiclean.go
View File

@@ -4,8 +4,8 @@ import (
"fmt"
"os"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
@@ -298,7 +298,7 @@ func CLIOBIClean(itertator obiiter.IBioSequence) obiiter.IBioSequence {
BuildSeqGraph(samples,
DistStepMax(),
obioptions.CLIParallelWorkers())
obidefault.ParallelWorkers())
if RatioMax() < 1.0 {
bar := (*progressbar.ProgressBar)(nil)

20
pkg/obitools/obicleandb/obicleandb.go
View File

@@ -7,8 +7,8 @@ import (
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obialign"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obichunk"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obistats"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax"
@@ -234,9 +234,9 @@ func ICleanDB(itertator obiiter.IBioSequence) obiiter.IBioSequence {
obichunk.OptionSortOnMemory(),
obichunk.OptionSubCategory("taxid"),
obichunk.OptionsParallelWorkers(
obioptions.CLIParallelWorkers()),
obidefault.ParallelWorkers()),
obichunk.OptionsBatchSize(
obioptions.CLIBatchSize()),
obidefault.BatchSize()),
obichunk.OptionNAValue("NA"),
)
@@ -261,22 +261,22 @@ func ICleanDB(itertator obiiter.IBioSequence) obiiter.IBioSequence {
goodTaxa := obiseq.IsAValidTaxon(taxonomy, CLIUpdateTaxids()).And(rankPredicate)
usable := unique.FilterOn(goodTaxa,
obioptions.CLIBatchSize(),
obioptions.CLIParallelWorkers())
obidefault.BatchSize(),
obidefault.ParallelWorkers())
annotated := usable.MakeIWorker(obiseq.MakeSetSpeciesWorker(taxonomy),
false,
obioptions.CLIParallelWorkers(),
obidefault.ParallelWorkers(),
).MakeIWorker(obiseq.MakeSetGenusWorker(taxonomy),
false,
obioptions.CLIParallelWorkers(),
obidefault.ParallelWorkers(),
).MakeIWorker(obiseq.MakeSetFamilyWorker(taxonomy),
false,
obioptions.CLIParallelWorkers(),
obidefault.ParallelWorkers(),
)
// .MakeIWorker(SequenceTrust,
// false,
// obioptions.CLIParallelWorkers(),
// obidefault.ParallelWorkers(),
// )
source, references := annotated.Load()
@@ -284,7 +284,7 @@ func ICleanDB(itertator obiiter.IBioSequence) obiiter.IBioSequence {
mannwithney := MakeSequenceFamilyGenusWorker(references)
partof := obiiter.IBatchOver(source, references,
obioptions.CLIBatchSize())
obidefault.BatchSize())
// genera_iterator, err := obichunk.ISequenceChunk(
// annotated,

4
pkg/obitools/obiconsensus/obiconsensus.go
View File

@@ -8,11 +8,11 @@ import (
"sync"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obialign"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiformats"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obigraph"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obikmer"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obisuffix"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiannotate"
@@ -472,7 +472,7 @@ func CLIOBIMinion(itertator obiiter.IBioSequence) obiiter.IBioSequence {
CLISampleAttribute(),
seqs,
CLIDistStepMax(),
obioptions.CLIParallelWorkers())
obidefault.ParallelWorkers())
if bar != nil {
bar.Add(1)
}

7
pkg/obitools/obiconvert/sequence_reader.go
View File

@@ -12,7 +12,6 @@ import (
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiformats"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
)
func ExpandListOfFiles(check_ext bool, filenames ...string) ([]string, error) {
@@ -102,13 +101,13 @@ func CLIReadBioSequences(filenames ...string) (obiiter.IBioSequence, error) {
opts = append(opts, obiformats.OptionsReadQualities(obidefault.ReadQualities()))
nworkers := obioptions.CLIReadParallelWorkers()
nworkers := obidefault.ReadParallelWorkers()
if nworkers < 2 {
nworkers = 2
}
opts = append(opts, obiformats.OptionsParallelWorkers(nworkers))
opts = append(opts, obiformats.OptionsBatchSize(obioptions.CLIBatchSize()))
opts = append(opts, obiformats.OptionsBatchSize(obidefault.BatchSize()))
opts = append(opts, obiformats.OptionsFullFileBatch(FullFileBatch()))
@@ -159,7 +158,7 @@ func CLIReadBioSequences(filenames ...string) (obiiter.IBioSequence, error) {
nreader := 1
if CLINoInputOrder() {
nreader = obioptions.ParallelFilesRead()
nreader = obidefault.ParallelFilesRead()
}
iterator = obiformats.ReadSequencesBatchFromFiles(

6
pkg/obitools/obiconvert/sequence_writer.go
View File

@@ -8,9 +8,9 @@ import (
log "github.com/sirupsen/logrus"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiformats"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
)
func BuildPairedFileNames(filename string) (string, string) {
@@ -53,10 +53,10 @@ func CLIWriteBioSequences(iterator obiiter.IBioSequence,
opts = append(opts, obiformats.OptionsFastSeqHeaderFormat(obiformats.FormatFastSeqJsonHeader))
}
nworkers := obioptions.CLIWriteParallelWorkers()
nworkers := obidefault.WriteParallelWorkers()
opts = append(opts, obiformats.OptionsParallelWorkers(nworkers))
opts = append(opts, obiformats.OptionsBatchSize(obioptions.CLIBatchSize()))
opts = append(opts, obiformats.OptionsBatchSize(obidefault.BatchSize()))
opts = append(opts, obiformats.OptionsCompressed(CLICompressed()))

9
pkg/obitools/obicsv/csvoption.go
View File

@@ -1,7 +1,8 @@
package obicsv
import (
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
)
// __options__ holds configuration options for processing.
@@ -53,8 +54,8 @@ func MakeOptions(setters []WithOption) Options {
with_progress_bar: false,
filename: "-",
buffer_size: 2,
parallel_workers: obioptions.CLIReadParallelWorkers(),
batch_size: obioptions.CLIBatchSize(),
parallel_workers: obidefault.ReadParallelWorkers(),
batch_size: obidefault.BatchSize(),
no_order: false,
full_file_batch: false,
closefile: false,
@@ -69,7 +70,7 @@ func MakeOptions(setters []WithOption) Options {
csv_quality: false,
csv_separator: ",",
csv_navalue: "NA",
csv_keys: make(CSVHeader, 0),
csv_keys: make(obiiter.CSVHeader, 0),
csv_auto: false,
source: "unknown",
}

348
pkg/obitools/obicsv/iter.go
View File

@@ -1,348 +0,0 @@
package obicsv
import (
"fmt"
"slices"
"sync"
"sync/atomic"
"time"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
"github.com/tevino/abool/v2"
log "github.com/sirupsen/logrus"
)
type CSVHeader []string
type CSVRecord map[string]interface{}
type CSVRecordBatch struct {
source string
data []CSVRecord
order int
}
var NilCSVRecordBatch = CSVRecordBatch{"", nil, -1}
// Structure implementing an iterator over bioseq.BioSequenceBatch
// based on a channel.
type ICSVRecord struct {
channel chan CSVRecordBatch
current CSVRecordBatch
pushBack *abool.AtomicBool
all_done *sync.WaitGroup
lock *sync.RWMutex
buffer_size int32
batch_size int32
sequence_format string
finished *abool.AtomicBool
header CSVHeader
}
var NilIBioSequenceBatch = (*ICSVRecord)(nil)
func NewICSVRecord() *ICSVRecord {
i := ICSVRecord{
channel: make(chan CSVRecordBatch),
current: NilCSVRecordBatch,
pushBack: abool.New(),
batch_size: -1,
sequence_format: "",
finished: abool.New(),
header: make(CSVHeader, 0),
}
waiting := sync.WaitGroup{}
i.all_done = &waiting
lock := sync.RWMutex{}
i.lock = &lock
obiutils.RegisterAPipe()
return &i
}
func MakeCSVRecordBatch(source string, order int, data []CSVRecord) CSVRecordBatch {
return CSVRecordBatch{
source: source,
order: order,
data: data,
}
}
func (batch *CSVRecordBatch) Order() int {
return batch.order
}
func (batch *CSVRecordBatch) Source() string {
return batch.source
}
func (batch *CSVRecordBatch) Slice() []CSVRecord {
return batch.data
}
// NotEmpty returns whether the BioSequenceBatch is empty or not.
//
// It checks if the BioSequenceSlice contained within the BioSequenceBatch is not empty.
//
// Returns:
// - bool: True if the BioSequenceBatch is not empty, false otherwise.
func (batch *CSVRecordBatch) NotEmpty() bool {
return len(batch.data) > 0
}
// IsNil checks if the BioSequenceBatch's slice is nil.
//
// This function takes a BioSequenceBatch as a parameter and returns a boolean value indicating whether the slice of the BioSequenceBatch is nil or not.
//
// Parameters:
// - batch: The BioSequenceBatch to check for nil slice.
//
// Returns:
// - bool: True if the BioSequenceBatch's slice is nil, false otherwise.
func (batch *CSVRecordBatch) IsNil() bool {
return batch.data == nil
}
func (iterator *ICSVRecord) Add(n int) {
if iterator == nil {
log.Panic("call of ICSVRecord.Add method on NilIBioSequenceBatch")
}
iterator.all_done.Add(n)
}
func (iterator *ICSVRecord) Done() {
if iterator == nil {
log.Panic("call of ICSVRecord.Done method on NilIBioSequenceBatch")
}
iterator.all_done.Done()
}
func (iterator *ICSVRecord) Unlock() {
if iterator == nil {
log.Panic("call of ICSVRecord.Unlock method on NilIBioSequenceBatch")
}
iterator.lock.Unlock()
}
func (iterator *ICSVRecord) Lock() {
if iterator == nil {
log.Panic("call of ICSVRecord.Lock method on NilIBioSequenceBatch")
}
iterator.lock.Lock()
}
func (iterator *ICSVRecord) RLock() {
if iterator == nil {
log.Panic("call of ICSVRecord.RLock method on NilIBioSequenceBatch")
}
iterator.lock.RLock()
}
func (iterator *ICSVRecord) RUnlock() {
if iterator == nil {
log.Panic("call of ICSVRecord.RUnlock method on NilIBioSequenceBatch")
}
iterator.lock.RUnlock()
}
func (iterator *ICSVRecord) Wait() {
if iterator == nil {
log.Panic("call of ICSVRecord.Wait method on NilIBioSequenceBatch")
}
iterator.all_done.Wait()
}
func (iterator *ICSVRecord) Channel() chan CSVRecordBatch {
if iterator == nil {
log.Panic("call of ICSVRecord.Channel method on NilIBioSequenceBatch")
}
return iterator.channel
}
func (iterator *ICSVRecord) IsNil() bool {
if iterator == nil {
log.Panic("call of ICSVRecord.IsNil method on NilIBioSequenceBatch")
}
return iterator == nil
}
func (iterator *ICSVRecord) BatchSize() int {
if iterator == nil {
log.Panic("call of ICSVRecord.BatchSize method on NilIBioSequenceBatch")
}
return int(atomic.LoadInt32(&iterator.batch_size))
}
func (iterator *ICSVRecord) SetBatchSize(size int) error {
if size >= 0 {
atomic.StoreInt32(&iterator.batch_size, int32(size))
return nil
}
return fmt.Errorf("size (%d) cannot be negative", size)
}
func (iterator *ICSVRecord) Split() *ICSVRecord {
iterator.lock.RLock()
defer iterator.lock.RUnlock()
i := ICSVRecord{
channel: iterator.channel,
current: NilCSVRecordBatch,
pushBack: abool.New(),
all_done: iterator.all_done,
buffer_size: iterator.buffer_size,
batch_size: iterator.batch_size,
sequence_format: iterator.sequence_format,
finished: iterator.finished,
header: iterator.header,
}
lock := sync.RWMutex{}
i.lock = &lock
return &i
}
func (iterator *ICSVRecord) Header() CSVHeader {
return iterator.header
}
func (iterator *ICSVRecord) SetHeader(header CSVHeader) {
iterator.header = header
}
func (iterator *ICSVRecord) AppendField(field string) {
iterator.header.AppendField(field)
}
func (iterator *ICSVRecord) Next() bool {
if iterator.pushBack.IsSet() {
iterator.pushBack.UnSet()
return true
}
if iterator.finished.IsSet() {
return false
}
next, ok := (<-iterator.channel)
if ok {
iterator.current = next
return true
}
iterator.current = NilCSVRecordBatch
iterator.finished.Set()
return false
}
func (iterator *ICSVRecord) PushBack() {
if !iterator.current.IsNil() {
iterator.pushBack.Set()
}
}
// The 'Get' method returns the instance of BioSequenceBatch
// currently pointed by the iterator. You have to use the
// 'Next' method to move to the next entry before calling
// 'Get' to retreive the following instance.
func (iterator *ICSVRecord) Get() CSVRecordBatch {
return iterator.current
}
func (iterator *ICSVRecord) Push(batch CSVRecordBatch) {
if batch.IsNil() {
log.Panicln("A Nil batch is pushed on the channel")
}
// if batch.Len() == 0 {
// log.Panicln("An empty batch is pushed on the channel")
// }
iterator.channel <- batch
}
func (iterator *ICSVRecord) Close() {
close(iterator.channel)
obiutils.UnregisterPipe()
}
func (iterator *ICSVRecord) WaitAndClose() {
iterator.Wait()
for len(iterator.Channel()) > 0 {
time.Sleep(time.Millisecond)
}
iterator.Close()
}
// Finished returns 'true' value if no more data is available
// from the iterator.
func (iterator *ICSVRecord) Finished() bool {
return iterator.finished.IsSet()
}
// Sorting the batches of sequences.
func (iterator *ICSVRecord) SortBatches(sizes ...int) *ICSVRecord {
newIter := NewICSVRecord()
newIter.Add(1)
go func() {
newIter.WaitAndClose()
}()
next_to_send := 0
//log.Println("wait for batch #", next_to_send)
received := make(map[int]CSVRecordBatch)
go func() {
for iterator.Next() {
batch := iterator.Get()
// log.Println("\nPushd seq #\n", batch.order, next_to_send)
if batch.order == next_to_send {
newIter.channel <- batch
next_to_send++
//log.Println("\nwait for batch #\n", next_to_send)
batch, ok := received[next_to_send]
for ok {
newIter.channel <- batch
delete(received, next_to_send)
next_to_send++
batch, ok = received[next_to_send]
}
} else {
received[batch.order] = batch
}
}
newIter.Done()
}()
return newIter
}
func (iterator *ICSVRecord) Consume() {
for iterator.Next() {
iterator.Get()
}
}
func (head *CSVHeader) AppendField(field string) {
if !slices.Contains(*head, field) {
*head = append(*head, field)
}
}

16
pkg/obitools/obicsv/obicsv.go
View File

@@ -3,13 +3,13 @@ package obicsv
import (
log "github.com/sirupsen/logrus"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
)
func CLIWriteSequenceCSV(iterator obiiter.IBioSequence,
terminalAction bool, filenames ...string) *ICSVRecord {
terminalAction bool, filenames ...string) *obiiter.ICSVRecord {
if obiconvert.CLIProgressBar() {
iterator = iterator.Speed("Writing CSV")
@@ -17,14 +17,14 @@ func CLIWriteSequenceCSV(iterator obiiter.IBioSequence,
opts := make([]WithOption, 0, 10)
nworkers := obioptions.CLIParallelWorkers() / 4
nworkers := obidefault.ParallelWorkers() / 4
if nworkers < 2 {
nworkers = 2
}
opts = append(opts, OptionsParallelWorkers(nworkers))
opts = append(opts, OptionsBatchSize(obioptions.CLIBatchSize()))
opts = append(opts, OptionsCompressed(obiconvert.CLICompressed()))
opts = append(opts, OptionsBatchSize(obidefault.BatchSize()))
opts = append(opts, OptionsCompressed(obidefault.CompressOutput()))
opts = append(opts, CSVId(CLIPrintId()),
CSVCount(CLIPrintCount()),
@@ -42,12 +42,12 @@ func CLIWriteSequenceCSV(iterator obiiter.IBioSequence,
}
func CLICSVWriter(iterator *ICSVRecord,
func CLICSVWriter(iterator *obiiter.ICSVRecord,
terminalAction bool,
options ...WithOption) *ICSVRecord {
options ...WithOption) *obiiter.ICSVRecord {
var err error
var newIter *ICSVRecord
var newIter *obiiter.ICSVRecord
if obiconvert.CLIOutPutFileName() != "-" {
options = append(options, OptionFileName(obiconvert.CLIOutPutFileName()))

16
pkg/obitools/obicsv/sequence.go
View File

@@ -8,9 +8,9 @@ import (
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
)
func CSVSequenceHeader(opt Options) CSVHeader {
func CSVSequenceHeader(opt Options) obiiter.CSVHeader {
keys := opt.CSVKeys()
record := make(CSVHeader, 0, len(keys)+4)
record := make(obiiter.CSVHeader, 0, len(keys)+4)
if opt.CSVId() {
record.AppendField("id")
@@ -45,12 +45,12 @@ func CSVSequenceHeader(opt Options) CSVHeader {
return record
}
func CSVBatchFromSequences(batch obiiter.BioSequenceBatch, opt Options) CSVRecordBatch {
func CSVBatchFromSequences(batch obiiter.BioSequenceBatch, opt Options) obiiter.CSVRecordBatch {
keys := opt.CSVKeys()
csvslice := make([]CSVRecord, batch.Len())
csvslice := make([]obiiter.CSVRecord, batch.Len())
for i, sequence := range batch.Slice() {
record := make(CSVRecord)
record := make(obiiter.CSVRecord)
if opt.CSVId() {
record["id"] = sequence.Id()
@@ -108,10 +108,10 @@ func CSVBatchFromSequences(batch obiiter.BioSequenceBatch, opt Options) CSVRecor
csvslice[i] = record
}
return MakeCSVRecordBatch(batch.Source(), batch.Order(), csvslice)
return obiiter.MakeCSVRecordBatch(batch.Source(), batch.Order(), csvslice)
}
func NewCSVSequenceIterator(iter obiiter.IBioSequence, options ...WithOption) *ICSVRecord {
func NewCSVSequenceIterator(iter obiiter.IBioSequence, options ...WithOption) *obiiter.ICSVRecord {
opt := MakeOptions(options)
@@ -128,7 +128,7 @@ func NewCSVSequenceIterator(iter obiiter.IBioSequence, options ...WithOption) *I
}
}
newIter := NewICSVRecord()
newIter := obiiter.NewICSVRecord()
newIter.SetHeader(CSVSequenceHeader(opt))
nwriters := opt.ParallelWorkers()

19
pkg/obitools/obicsv/writer.go
View File

@@ -7,12 +7,13 @@ import (
"os"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiformats"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
log "github.com/sirupsen/logrus"
)
func FormatCVSBatch(batch CSVRecordBatch, header CSVHeader, navalue string) *bytes.Buffer {
func FormatCVSBatch(batch obiiter.CSVRecordBatch, header obiiter.CSVHeader, navalue string) *bytes.Buffer {
buff := new(bytes.Buffer)
csv := csv.NewWriter(buff)
@@ -44,14 +45,14 @@ func FormatCVSBatch(batch CSVRecordBatch, header CSVHeader, navalue string) *byt
return buff
}
func WriteCSV(iterator *ICSVRecord,
func WriteCSV(iterator *obiiter.ICSVRecord,
file io.WriteCloser,
options ...WithOption) (*ICSVRecord, error) {
options ...WithOption) (*obiiter.ICSVRecord, error) {
opt := MakeOptions(options)
file, _ = obiutils.CompressStream(file, opt.CompressedFile(), opt.CloseFile())
newIter := NewICSVRecord()
newIter := obiiter.NewICSVRecord()
nwriters := opt.ParallelWorkers()
@@ -65,7 +66,7 @@ func WriteCSV(iterator *ICSVRecord,
log.Debugf("Writing CSV file done")
}()
ff := func(iterator *ICSVRecord) {
ff := func(iterator *obiiter.ICSVRecord) {
for iterator.Next() {
batch := iterator.Get()
@@ -108,8 +109,8 @@ func WriteCSV(iterator *ICSVRecord,
// os.Stdout as the output file, and the options slice.
//
// The function returns the same bio sequence iterator and an error if any occurred.
func WriteCSVToStdout(iterator *ICSVRecord,
options ...WithOption) (*ICSVRecord, error) {
func WriteCSVToStdout(iterator *obiiter.ICSVRecord,
options ...WithOption) (*obiiter.ICSVRecord, error) {
// options = append(options, OptionDontCloseFile())
options = append(options, OptionCloseFile())
return WriteCSV(iterator, os.Stdout, options...)
@@ -126,9 +127,9 @@ func WriteCSVToStdout(iterator *ICSVRecord,
// Returns:
// - obiiter.IBioSequence: The updated biosequence iterator.
// - error: Any error that occurred during the writing process.
func WriteCSVToFile(iterator *ICSVRecord,
func WriteCSVToFile(iterator *obiiter.ICSVRecord,
filename string,
options ...WithOption) (*ICSVRecord, error) {
options ...WithOption) (*obiiter.ICSVRecord, error) {
opt := MakeOptions(options)
flags := os.O_WRONLY | os.O_CREATE

4
pkg/obitools/obidemerge/demerge.go
View File

@@ -1,8 +1,8 @@
package obidemerge
import (
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
)
@@ -34,5 +34,5 @@ func MakeDemergeWorker(key string) obiseq.SeqWorker {
func CLIDemergeSequences(iterator obiiter.IBioSequence) obiiter.IBioSequence {
worker := MakeDemergeWorker(CLIDemergeSlot())
return iterator.MakeIWorker(worker, false, obioptions.CLIParallelWorkers(), 0)
return iterator.MakeIWorker(worker, false, obidefault.ParallelWorkers(), 0)
}

8
pkg/obitools/obidistribute/distribute.go
View File

@@ -3,9 +3,9 @@ package obidistribute
import (
log "github.com/sirupsen/logrus"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiformats"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
)
@@ -25,13 +25,13 @@ func CLIDistributeSequence(sequences obiiter.IBioSequence) {
opts = append(opts, obiformats.OptionsFastSeqHeaderFormat(obiformats.FormatFastSeqJsonHeader))
}
nworkers := obioptions.CLIParallelWorkers() / 4
nworkers := obidefault.ParallelWorkers() / 4
if nworkers < 2 {
nworkers = 2
}
opts = append(opts, obiformats.OptionsParallelWorkers(nworkers),
obiformats.OptionsBatchSize(obioptions.CLIBatchSize()),
obiformats.OptionsBatchSize(obidefault.BatchSize()),
obiformats.OptionsAppendFile(CLIAppendSequences()),
obiformats.OptionsCompressed(obiconvert.CLICompressed()))
@@ -47,7 +47,7 @@ func CLIDistributeSequence(sequences obiiter.IBioSequence) {
}
dispatcher := sequences.Distribute(CLISequenceClassifier(),
obioptions.CLIBatchSize())
obidefault.BatchSize())
obiformats.WriterDispatcher(CLIFileNamePattern(),
dispatcher, formater, opts...,

20
pkg/obitools/obifind/iterator.go
View File

@@ -3,9 +3,9 @@ package obifind
import (
"slices"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obicsv"
)
type __options__ struct {
@@ -38,7 +38,7 @@ type WithOption func(Options)
// - An Options instance with the specified settings.
func MakeOptions(setters []WithOption) Options {
o := __options__{
batch_size: obioptions.CLIBatchSize(), // Number of items to process in a batch
batch_size: obidefault.BatchSize(), // Number of items to process in a batch
with_pattern: true,
with_parent: false,
with_path: false,
@@ -192,12 +192,12 @@ func OptionsWithMetadata(values ...string) WithOption {
return f
}
func NewCSVTaxaIterator(iterator *obitax.ITaxon, options ...WithOption) *obicsv.ICSVRecord {
func NewCSVTaxaIterator(iterator *obitax.ITaxon, options ...WithOption) *obiiter.ICSVRecord {
opt := MakeOptions(options)
metakeys := make([]string, 0)
newIter := obicsv.NewICSVRecord()
newIter := obiiter.NewICSVRecord()
newIter.Add(1)
@@ -240,11 +240,11 @@ func NewCSVTaxaIterator(iterator *obitax.ITaxon, options ...WithOption) *obicsv.
go func() {
o := 0
data := make([]obicsv.CSVRecord, 0, batch_size)
data := make([]obiiter.CSVRecord, 0, batch_size)
for iterator.Next() {
taxon := iterator.Get()
record := make(obicsv.CSVRecord)
record := make(obiiter.CSVRecord)
if opt.WithPattern() {
record["query"] = taxon.MetadataAsString("query")
@@ -282,15 +282,15 @@ func NewCSVTaxaIterator(iterator *obitax.ITaxon, options ...WithOption) *obicsv.
data = append(data, record)
if len(data) >= batch_size {
newIter.Push(obicsv.MakeCSVRecordBatch(opt.Source(), o, data))
data = make([]obicsv.CSVRecord, 0, batch_size)
newIter.Push(obiiter.MakeCSVRecordBatch(opt.Source(), o, data))
data = make([]obiiter.CSVRecord, 0, batch_size)
o++
}
}
if len(data) > 0 {
newIter.Push(obicsv.MakeCSVRecordBatch(opt.Source(), o, data))
newIter.Push(obiiter.MakeCSVRecordBatch(opt.Source(), o, data))
}
newIter.Done()

9
pkg/obitools/obifind/obifind.go
View File

@@ -1,7 +1,8 @@
package obifind
import (
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obicsv"
@@ -39,7 +40,7 @@ func CLIFilterRankRestriction(iterator *obitax.ITaxon) *obitax.ITaxon {
return iterator
}
func CLICSVTaxaIterator(iterator *obitax.ITaxon) *obicsv.ICSVRecord {
func CLICSVTaxaIterator(iterator *obitax.ITaxon) *obiiter.ICSVRecord {
if iterator == nil {
return nil
}
@@ -53,12 +54,12 @@ func CLICSVTaxaIterator(iterator *obitax.ITaxon) *obicsv.ICSVRecord {
OptionsWithScientificName(CLIWithScientificName()),
OptionsWithPath(CLIWithPath()),
OptionsRawTaxid(CLIRawTaxid()),
OptionsSource(obioptions.CLISelectedTaxonomy()),
OptionsSource(obidefault.SelectedTaxonomy()),
)
return NewCSVTaxaIterator(iterator, options...)
}
func CLICSVTaxaWriter(iterator *obitax.ITaxon, terminalAction bool) *obicsv.ICSVRecord {
func CLICSVTaxaWriter(iterator *obitax.ITaxon, terminalAction bool) *obiiter.ICSVRecord {
return obicsv.CLICSVWriter(CLICSVTaxaIterator(iterator), terminalAction)
}

8
pkg/obitools/obigrep/grep.go
View File

@@ -3,8 +3,8 @@ package obigrep
import (
log "github.com/sirupsen/logrus"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
)
@@ -23,7 +23,7 @@ func CLIFilterSequence(iterator obiiter.IBioSequence) obiiter.IBioSequence {
log.Printf("Discarded sequences saved in file: %s\n", CLIDiscardedFileName())
newIter, discarded = iterator.DivideOn(predicate,
obioptions.CLIBatchSize())
obidefault.BatchSize())
go func() {
_, err := obiconvert.CLIWriteBioSequences(discarded,
@@ -37,8 +37,8 @@ func CLIFilterSequence(iterator obiiter.IBioSequence) obiiter.IBioSequence {
} else {
newIter = iterator.FilterOn(predicate,
obioptions.CLIBatchSize(),
obioptions.CLIParallelWorkers(),
obidefault.BatchSize(),
obidefault.ParallelWorkers(),
)
}
} else {

4
pkg/obitools/obijoin/join.go
View File

@@ -1,9 +1,9 @@
package obijoin
import (
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiformats"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
@@ -137,7 +137,7 @@ func CLIJoinSequences(iterator obiiter.IBioSequence) obiiter.IBioSequence {
worker := MakeJoinWorker(keys.Left, index, CLIUpdateId(), CLIUpdateSequence(), CLIUpdateQuality())
iterator = iterator.MakeIWorker(worker, false, obioptions.CLIParallelWorkers())
iterator = iterator.MakeIWorker(worker, false, obidefault.ParallelWorkers())
return iterator
}

10
pkg/obitools/obikmersim/obikmersim.go
View File

@@ -4,10 +4,10 @@ import (
"math"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obialign"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obifp"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obikmer"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
)
@@ -147,7 +147,7 @@ func CLILookForSharedKmers(iterator obiiter.IBioSequence) obiiter.IBioSequence {
source, references := CLIReference()
if iterator == obiiter.NilIBioSequence {
iterator = obiiter.IBatchOver(source, references, obioptions.CLIBatchSize())
iterator = obiiter.IBatchOver(source, references, obidefault.BatchSize())
}
if CLISelf() {
@@ -163,7 +163,7 @@ func CLILookForSharedKmers(iterator obiiter.IBioSequence) obiiter.IBioSequence {
CLIMaxKmerOccurs())
worker := MakeCountMatchWorker(kmerMatch, CLIMinSharedKmers())
newIter = iterator.MakeIWorker(worker, false, obioptions.CLIParallelWorkers())
newIter = iterator.MakeIWorker(worker, false, obidefault.ParallelWorkers())
return newIter.FilterEmpty()
}
@@ -174,7 +174,7 @@ func CLIAlignSequences(iterator obiiter.IBioSequence) obiiter.IBioSequence {
source, references := CLIReference()
if iterator == obiiter.NilIBioSequence {
iterator = obiiter.IBatchOver(source, references, obioptions.CLIBatchSize())
iterator = obiiter.IBatchOver(source, references, obidefault.BatchSize())
}
if CLISelf() {
@@ -188,7 +188,7 @@ func CLIAlignSequences(iterator obiiter.IBioSequence) obiiter.IBioSequence {
CLISparseMode(),
CLIMaxKmerOccurs())
worker := MakeKmerAlignWorker(kmerMatch, CLIMinSharedKmers(), CLIGap(), CLIScale(), CLIDelta(), CLIFastRelativeScore(), 0.8, true)
newIter = iterator.MakeIWorker(worker, false, obioptions.CLIParallelWorkers())
newIter = iterator.MakeIWorker(worker, false, obidefault.ParallelWorkers())
return newIter.FilterEmpty()
}

4
pkg/obitools/obikmersim/options.go
View File

@@ -1,8 +1,8 @@
package obikmersim
import (
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiformats"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
"github.com/DavidGamba/go-getoptions"
@@ -105,7 +105,7 @@ func CLIReference() (string, obiseq.BioSequenceSlice) {
nreader := 1
if obiconvert.CLINoInputOrder() {
nreader = obioptions.StrictReadWorker()
nreader = obidefault.StrictReadWorker()
}
source, references := obiformats.ReadSequencesBatchFromFiles(

8
pkg/obitools/obilandmark/obilandmark.go
View File

@@ -6,8 +6,8 @@ import (
"sync"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obialign"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obistats"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax"
@@ -30,7 +30,7 @@ import (
// Returns:
// - seqworld: A matrix of float64 values representing the mapped coordinates.
func MapOnLandmarkSequences(library obiseq.BioSequenceSlice, landmark_idx []int, sizes ...int) obiutils.Matrix[float64] {
nworkers := obioptions.CLIParallelWorkers()
nworkers := obidefault.ParallelWorkers()
if len(sizes) > 0 {
nworkers = sizes[0]
@@ -154,7 +154,7 @@ func CLISelectLandmarkSequences(iterator obiiter.IBioSequence) obiiter.IBioSeque
}
}
if obioptions.CLIHasSelectedTaxonomy() {
if obidefault.HasSelectedTaxonomy() {
taxo := obitax.DefaultTaxonomy()
if taxo == nil {
log.Fatal("No taxonomy available")
@@ -191,6 +191,6 @@ func CLISelectLandmarkSequences(iterator obiiter.IBioSequence) obiiter.IBioSeque
}
}
return obiiter.IBatchOver(source, library, obioptions.CLIBatchSize())
return obiiter.IBatchOver(source, library, obidefault.BatchSize())
}

4
pkg/obitools/obimatrix/obimatrix.go
View File

@@ -8,8 +8,8 @@ import (
log "github.com/sirupsen/logrus"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
"golang.org/x/exp/maps"
@@ -93,7 +93,7 @@ func (data *MatrixData) Update(s *obiseq.BioSequence, mapkey string) *MatrixData
func IMatrix(iterator obiiter.IBioSequence) *MatrixData {
nproc := obioptions.CLIParallelWorkers()
nproc := obidefault.ParallelWorkers()
waiter := sync.WaitGroup{}
mapAttribute := CLIMapAttribute()

4
pkg/obitools/obimicrosat/microsat.go
View File

@@ -4,8 +4,8 @@ import (
"fmt"
"sort"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
"github.com/dlclark/regexp2"
)
@@ -168,7 +168,7 @@ func CLIAnnotateMicrosat(iterator obiiter.IBioSequence) obiiter.IBioSequence {
CLIMinFlankLength(),
CLIReoriented())
newIter = iterator.MakeIWorker(worker, false, obioptions.CLIParallelWorkers())
newIter = iterator.MakeIWorker(worker, false, obidefault.ParallelWorkers())
return newIter.FilterEmpty()

12
pkg/obitools/obimultiplex/demultiplex.go
View File

@@ -3,9 +3,9 @@ package obimultiplex
import (
log "github.com/sirupsen/logrus"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obingslibrary"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
)
@@ -19,8 +19,8 @@ func IExtractBarcode(iterator obiiter.IBioSequence) (obiiter.IBioSequence, error
obingslibrary.OptionAllowedIndel(CLIAllowsIndel()),
obingslibrary.OptionUnidentified(CLIUnidentifiedFileName()),
obingslibrary.OptionDiscardErrors(!CLIConservedErrors()),
obingslibrary.OptionParallelWorkers(obioptions.CLIParallelWorkers()),
obingslibrary.OptionBatchSize(obioptions.CLIBatchSize()),
obingslibrary.OptionParallelWorkers(obidefault.ParallelWorkers()),
obingslibrary.OptionBatchSize(obidefault.BatchSize()),
)
ngsfilter, err := CLINGSFIlter()
@@ -35,14 +35,14 @@ func IExtractBarcode(iterator obiiter.IBioSequence) (obiiter.IBioSequence, error
if !CLIConservedErrors() {
log.Infoln("Discards unassigned sequences")
out = out.FilterOn(obiseq.HasAttribute("obimultiplex_error").Not(), obioptions.CLIBatchSize())
out = out.FilterOn(obiseq.HasAttribute("obimultiplex_error").Not(), obidefault.BatchSize())
}
var unidentified obiiter.IBioSequence
if CLIUnidentifiedFileName() != "" {
log.Printf("Unassigned sequences saved in file: %s\n", CLIUnidentifiedFileName())
unidentified, out = newIter.DivideOn(obiseq.HasAttribute("obimultiplex_error"),
obioptions.CLIBatchSize())
obidefault.BatchSize())
go func() {
_, err := obiconvert.CLIWriteBioSequences(unidentified,
@@ -55,7 +55,7 @@ func IExtractBarcode(iterator obiiter.IBioSequence) (obiiter.IBioSequence, error
}()
}
log.Printf("Sequence demultiplexing using %d workers\n", obioptions.CLIParallelWorkers())
log.Printf("Sequence demultiplexing using %d workers\n", obidefault.ParallelWorkers())
return out, nil
}

4
pkg/obitools/obipairing/pairing.go
View File

@@ -6,8 +6,8 @@ import (
log "github.com/sirupsen/logrus"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obialign"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
)
@@ -222,7 +222,7 @@ func IAssemblePESequencesBatch(iterator obiiter.IBioSequence,
log.Fatalln("Sequence data must be paired")
}
nworkers := obioptions.CLIParallelWorkers()
nworkers := obidefault.ParallelWorkers()
if len(sizes) > 0 {
nworkers = sizes[0]

6
pkg/obitools/obipcr/pcr.go
View File

@@ -2,8 +2,8 @@ package obipcr
import (
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiapat"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
log "github.com/sirupsen/logrus"
)
@@ -50,7 +50,7 @@ func CLIPCR(iterator obiiter.IBioSequence) (obiiter.IBioSequence, error) {
len(CLIReversePrimer()))+min(len(CLIForwardPrimer()),
len(CLIReversePrimer()))/2,
100,
obioptions.CLIParallelWorkers(),
obidefault.ParallelWorkers(),
)
log.Infof("Fragmenting sequence longer than %dbp into chuncks of %dbp",
CLIMaxLength()*1000,
@@ -59,5 +59,5 @@ func CLIPCR(iterator obiiter.IBioSequence) (obiiter.IBioSequence, error) {
iterator = iterator.Pipe(frags)
}
return iterator.LimitMemory(0.5).MakeISliceWorker(worker, false, obioptions.CLIParallelWorkers()), nil
return iterator.LimitMemory(0.5).MakeISliceWorker(worker, false, obidefault.ParallelWorkers()), nil
}

26
pkg/obitools/obirefidx/famlilyindexing.go
View File

@@ -8,9 +8,9 @@ import (
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obialign"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obichunk"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obikmer"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax"
"github.com/schollz/progressbar/v3"
@@ -111,7 +111,7 @@ func MakeIndexingSliceWorker(indexslot, idslot string,
waiting.Done()
}
nworkers := max(min(obioptions.CLIParallelWorkers(), len(sequences)/10), 1)
nworkers := max(min(obidefault.ParallelWorkers(), len(sequences)/10), 1)
waiting.Add(nworkers)
@@ -134,9 +134,9 @@ func IndexFamilyDB(iterator obiiter.IBioSequence) obiiter.IBioSequence {
nref := len(references)
log.Infof("Done. Database contains %d sequences", nref)
taxonomy, error := obioptions.CLILoadSelectedTaxonomy()
if error != nil {
log.Panicln(error)
taxonomy := obitax.DefaultTaxonomy()
if taxonomy == nil {
log.Panicln("No taxonomy available use the --taxonomy option")
}
log.Infoln("Indexing database kmers...")
@@ -155,15 +155,15 @@ func IndexFamilyDB(iterator obiiter.IBioSequence) obiiter.IBioSequence {
log.Info("done")
partof := obiiter.IBatchOver(source, references,
obioptions.CLIBatchSize()).MakeIWorker(obiseq.MakeSetSpeciesWorker(taxonomy),
obidefault.BatchSize()).MakeIWorker(obiseq.MakeSetSpeciesWorker(taxonomy),
false,
obioptions.CLIParallelWorkers(),
obidefault.ParallelWorkers(),
).MakeIWorker(obiseq.MakeSetGenusWorker(taxonomy),
false,
obioptions.CLIParallelWorkers(),
obidefault.ParallelWorkers(),
).MakeIWorker(obiseq.MakeSetFamilyWorker(taxonomy),
false,
obioptions.CLIParallelWorkers(),
obidefault.ParallelWorkers(),
)
family_iterator, err := obichunk.ISequenceChunk(
@@ -178,11 +178,11 @@ func IndexFamilyDB(iterator obiiter.IBioSequence) obiiter.IBioSequence {
family_iterator.MakeISliceWorker(
MakeStartClusterSliceWorker("reffamidx", 0.9),
false,
obioptions.CLIParallelWorkers(),
obidefault.ParallelWorkers(),
).MakeISliceWorker(
MakeIndexingSliceWorker("reffamidx_in", "reffamidx_id", &refcounts, taxonomy),
false,
obioptions.CLIParallelWorkers(),
obidefault.ParallelWorkers(),
).Speed("Family Indexing", nref).Consume()
clusters := obiseq.MakeBioSequenceSlice(0)
@@ -240,7 +240,7 @@ func IndexFamilyDB(iterator obiiter.IBioSequence) obiiter.IBioSequence {
waiting.Done()
}
nworkers := obioptions.CLIParallelWorkers()
nworkers := obidefault.ParallelWorkers()
waiting.Add(nworkers)
for w := 0; w < nworkers; w++ {
@@ -250,7 +250,7 @@ func IndexFamilyDB(iterator obiiter.IBioSequence) obiiter.IBioSequence {
waiting.Wait()
results := obiiter.IBatchOver(source, references,
obioptions.CLIBatchSize()).Speed("Writing db", nref)
obidefault.BatchSize()).Speed("Writing db", nref)
return results
}

4
pkg/obitools/obirefidx/geomindexing.go
View File

@@ -7,7 +7,7 @@ import (
log "github.com/sirupsen/logrus"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obistats"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax"
@@ -32,7 +32,7 @@ func GeomIndexSesquence(seqidx int,
iseq_channel := make(chan int)
for k := 0; k < obioptions.CLIParallelWorkers(); k++ {
for k := 0; k < obidefault.ParallelWorkers(); k++ {
wg.Add(1)
go func() {
defer wg.Done()

6
pkg/obitools/obirefidx/obirefidx.go
View File

@@ -6,9 +6,9 @@ import (
log "github.com/sirupsen/logrus"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obialign"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obikmer"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
@@ -274,7 +274,7 @@ func IndexReferenceDB(iterator obiiter.IBioSequence) obiiter.IBioSequence {
indexed.Done()
}
nworkers := obioptions.CLIParallelWorkers()
nworkers := obidefault.ParallelWorkers()
indexed.Add(nworkers)
go func() {
@@ -285,5 +285,5 @@ func IndexReferenceDB(iterator obiiter.IBioSequence) obiiter.IBioSequence {
go f()
}
return indexed.Rebatch(obioptions.CLIBatchSize())
return indexed.Rebatch(obidefault.BatchSize())
}

4
pkg/obitools/obiscript/obiscript.go
View File

@@ -1,14 +1,14 @@
package obiscript
import (
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obilua"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
)
func CLIScriptPipeline() obiiter.Pipeable {
pipe := obilua.LuaScriptPipe(CLIScriptFilename(), true, obioptions.CLIParallelWorkers())
pipe := obilua.LuaScriptPipe(CLIScriptFilename(), true, obidefault.ParallelWorkers())
return pipe
}

4
pkg/obitools/obisplit/obisplit.go
View File

@@ -7,8 +7,8 @@ import (
log "github.com/sirupsen/logrus"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiapat"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
)
@@ -295,7 +295,7 @@ func CLISlitPipeline() obiiter.Pipeable {
worker := SplitPatternWorker(CLIConfig())
annotator := obiseq.SeqToSliceWorker(worker, false)
f := obiiter.SliceWorkerPipe(annotator, false, obioptions.CLIParallelWorkers())
f := obiiter.SliceWorkerPipe(annotator, false, obidefault.ParallelWorkers())
return f
}

4
pkg/obitools/obisummary/obisummary.go
View File

@@ -3,8 +3,8 @@ package obisummary
import (
"sync"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
)
@@ -154,7 +154,7 @@ func (data *DataSummary) Update(s *obiseq.BioSequence) *DataSummary {
func ISummary(iterator obiiter.IBioSequence, summarise []string) map[string]interface{} {
nproc := obioptions.CLIParallelWorkers()
nproc := obidefault.ParallelWorkers()
waiter := sync.WaitGroup{}
summaries := make([]*DataSummary, nproc)

4
pkg/obitools/obitag/obigeomtag.go
View File

@@ -6,8 +6,8 @@ import (
log "github.com/sirupsen/logrus"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obialign"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obistats"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax"
@@ -200,5 +200,5 @@ func CLIGeomAssignTaxonomy(iterator obiiter.IBioSequence,
) obiiter.IBioSequence {
worker := GeomIdentifySeqWorker(&references, taxo)
return iterator.MakeIWorker(worker, false, obioptions.CLIParallelWorkers(), 0)
return iterator.MakeIWorker(worker, false, obidefault.ParallelWorkers(), 0)
}

4
pkg/obitools/obitag/obitag.go
View File

@@ -8,9 +8,9 @@ import (
"golang.org/x/exp/slices"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obialign"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obikmer"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obirefidx"
@@ -266,5 +266,5 @@ func CLIAssignTaxonomy(iterator obiiter.IBioSequence,
worker := IdentifySeqWorker(references, refcounts, taxa, taxo, CLIRunExact())
return iterator.MakeIWorker(worker, false, obioptions.CLIParallelWorkers(), 0)
return iterator.MakeIWorker(worker, false, obidefault.ParallelWorkers(), 0)
}

7
pkg/obitools/obitag/options.go
View File

@@ -3,6 +3,7 @@ package obitag
import (
log "github.com/sirupsen/logrus"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiformats"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
@@ -87,15 +88,15 @@ func CLISaveRefetenceDB(db obiseq.BioSequenceSlice) {
opts = append(opts, obiformats.OptionsFastSeqHeaderFormat(obiformats.FormatFastSeqJsonHeader))
}
nworkers := obioptions.CLIParallelWorkers() / 4
nworkers := obidefault.ParallelWorkers() / 4
if nworkers < 2 {
nworkers = 2
}
opts = append(opts, obiformats.OptionsParallelWorkers(nworkers))
opts = append(opts, obiformats.OptionsBatchSize(obioptions.CLIBatchSize()))
opts = append(opts, obiformats.OptionsBatchSize(obidefault.BatchSize()))
opts = append(opts, obiformats.OptionsCompressed(obiconvert.CLICompressed()))
opts = append(opts, obiformats.OptionsCompressed(obidefault.CompressOutput()))
var err error

8
pkg/obitools/obitagpcr/pcrtag.go
View File

@@ -4,8 +4,8 @@ import (
log "github.com/sirupsen/logrus"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obialign"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obimultiplex"
@@ -44,7 +44,7 @@ func IPCRTagPESequencesBatch(iterator obiiter.IBioSequence,
log.Fatalln("Sequence data must be paired")
}
nworkers := obioptions.CLIParallelWorkers()
nworkers := obidefault.ParallelWorkers()
ngsfilter, err := obimultiplex.CLINGSFIlter()
if err != nil {
@@ -180,14 +180,14 @@ func IPCRTagPESequencesBatch(iterator obiiter.IBioSequence,
if !obimultiplex.CLIConservedErrors() {
log.Println("Discards unassigned sequences")
iout = iout.FilterOn(obiseq.HasAttribute("obimultiplex_error").Not(), obioptions.CLIBatchSize())
iout = iout.FilterOn(obiseq.HasAttribute("obimultiplex_error").Not(), obidefault.BatchSize())
}
var unidentified obiiter.IBioSequence
if obimultiplex.CLIUnidentifiedFileName() != "" {
log.Printf("Unassigned sequences saved in file: %s\n", obimultiplex.CLIUnidentifiedFileName())
unidentified, iout = iout.DivideOn(obiseq.HasAttribute("obimultiplex_error"),
obioptions.CLIBatchSize())
obidefault.BatchSize())
go func() {
_, err := obiconvert.CLIWriteBioSequences(unidentified,

6
pkg/obitools/obiuniq/unique.go
View File

@@ -4,8 +4,8 @@ import (
log "github.com/sirupsen/logrus"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obichunk"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
)
func CLIUnique(sequences obiiter.IBioSequence) obiiter.IBioSequence {
@@ -50,9 +50,9 @@ func CLIUnique(sequences obiiter.IBioSequence) obiiter.IBioSequence {
options = append(options,
obichunk.OptionsParallelWorkers(
obioptions.CLIParallelWorkers()),
obidefault.ParallelWorkers()),
obichunk.OptionsBatchSize(
obioptions.CLIBatchSize()),
obidefault.BatchSize()),
obichunk.OptionNAValue(CLINAValue()),
)