From 9acb4a85a869de2cb2d4a24be8e7989265b10af9 Mon Sep 17 00:00:00 2001 From: Eric Coissac Date: Fri, 24 Jan 2025 18:09:59 +0100 Subject: [PATCH] Refactoring of the default values --- cmd/obitools/obiannotate/main.go | 4 +- cmd/obitools/obiclean/main.go | 4 +- cmd/obitools/obicleandb/main.go | 4 +- cmd/obitools/obicomplement/main.go | 4 +- cmd/obitools/obiconsensus/main.go | 4 +- cmd/obitools/obiconvert/main.go | 4 +- cmd/obitools/obicsv/main.go | 4 +- cmd/obitools/obidemerge/main.go | 4 +- cmd/obitools/obidistribute/main.go | 4 +- cmd/obitools/obifind/main.go | 4 +- cmd/obitools/obigrep/main.go | 4 +- cmd/obitools/obijoin/main.go | 4 +- cmd/obitools/obikmermatch/main.go | 3 +- cmd/obitools/obikmersimcount/main.go | 3 +- cmd/obitools/obilandmark/main.go | 4 +- cmd/obitools/obimicrosat/main.go | 4 +- cmd/obitools/obimultiplex/main.go | 4 +- cmd/obitools/obipairing/main.go | 4 +- cmd/obitools/obipcr/main.go | 4 +- cmd/obitools/obireffamidx/main.go | 4 +- cmd/obitools/obirefidx/main.go | 4 +- cmd/obitools/obiscript/main.go | 4 +- cmd/obitools/obisplit/main.go | 4 +- cmd/obitools/obitag/main.go | 3 +- cmd/obitools/obitagpcr/main.go | 4 +- cmd/obitools/obiuniq/main.go | 7 +- cmd/test/main.go | 4 + pkg/obidefault/quality.go | 29 ++++++ pkg/obiformats/csv_read.go | 4 +- pkg/obiformats/csv_writer.go | 4 +- pkg/obiformats/fastqseq_read.go | 4 +- pkg/obiformats/fastseq_read.go | 6 +- pkg/obiformats/file_chunk_write.go | 6 +- pkg/obiiter/batchiterator.go | 37 +------- pkg/obioptions/options.go | 51 +--------- pkg/obioptions/version.go | 2 +- pkg/obiseq/biosequence.go | 4 +- pkg/obitax/iterator.go | 47 ++++++++++ pkg/obitaxformat/csvtaxdump/read.go | 104 +++++++++++++++++++++ pkg/obitaxformat/csvtaxdump/write.go | 12 +++ pkg/obitaxformat/taxonomy_read.go | 8 +- pkg/obitools/obiconvert/sequence_reader.go | 3 +- pkg/obitools/obicsv/iter.go | 6 +- pkg/obitools/obicsv/sequence.go | 4 +- pkg/obitools/obimatrix/obimatrix.go | 2 +- pkg/obitools/obisummary/obisummary.go | 2 +- pkg/obitools/obitag/options.go | 3 +- pkg/obiutils/abs_test.go | 3 +- pkg/obiutils/path.go | 14 +++ pkg/obiutils/pipe.go | 40 ++++++++ 50 files changed, 337 insertions(+), 166 deletions(-) create mode 100644 pkg/obidefault/quality.go create mode 100644 pkg/obitaxformat/csvtaxdump/read.go create mode 100644 pkg/obitaxformat/csvtaxdump/write.go create mode 100644 pkg/obiutils/pipe.go diff --git a/cmd/obitools/obiannotate/main.go b/cmd/obitools/obiannotate/main.go index 8e9076a..bd1638d 100644 --- a/cmd/obitools/obiannotate/main.go +++ b/cmd/obitools/obiannotate/main.go @@ -3,11 +3,11 @@ package main import ( "os" - "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiannotate" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" ) func main() { @@ -40,6 +40,6 @@ func main() { annotator := obiannotate.CLIAnnotationPipeline() obiconvert.CLIWriteBioSequences(sequences.Pipe(annotator), true) - obiiter.WaitForLastPipe() + obiutils.WaitForLastPipe() } diff --git a/cmd/obitools/obiclean/main.go b/cmd/obitools/obiclean/main.go index db81f4e..88b1daf 100644 --- a/cmd/obitools/obiclean/main.go +++ b/cmd/obitools/obiclean/main.go @@ -3,9 +3,9 @@ package main import ( "os" - "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiclean" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions" ) @@ -22,6 +22,6 @@ func main() { obiconvert.CLIWriteBioSequences(cleaned, true) - obiiter.WaitForLastPipe() + obiutils.WaitForLastPipe() } diff --git a/cmd/obitools/obicleandb/main.go b/cmd/obitools/obicleandb/main.go index ba7ea6b..938e04d 100644 --- a/cmd/obitools/obicleandb/main.go +++ b/cmd/obitools/obicleandb/main.go @@ -3,9 +3,9 @@ package main import ( "os" - "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obicleandb" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions" ) @@ -25,5 +25,5 @@ func main() { toconsume, _ := obiconvert.CLIWriteBioSequences(cleaned, false) toconsume.Consume() - obiiter.WaitForLastPipe() + obiutils.WaitForLastPipe() } diff --git a/cmd/obitools/obicomplement/main.go b/cmd/obitools/obicomplement/main.go index 883e305..5c4e76d 100644 --- a/cmd/obitools/obicomplement/main.go +++ b/cmd/obitools/obicomplement/main.go @@ -3,9 +3,9 @@ package main import ( "os" - "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions" ) @@ -21,6 +21,6 @@ func main() { comp := fs.MakeIWorker(obiseq.ReverseComplementWorker(true), true) obiconvert.CLIWriteBioSequences(comp, true) - obiiter.WaitForLastPipe() + obiutils.WaitForLastPipe() } diff --git a/cmd/obitools/obiconsensus/main.go b/cmd/obitools/obiconsensus/main.go index 2ba9827..bb99855 100644 --- a/cmd/obitools/obiconsensus/main.go +++ b/cmd/obitools/obiconsensus/main.go @@ -3,9 +3,9 @@ package main import ( "os" - "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconsensus" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions" ) @@ -22,6 +22,6 @@ func main() { obiconvert.CLIWriteBioSequences(cleaned, true) - obiiter.WaitForLastPipe() + obiutils.WaitForLastPipe() } diff --git a/cmd/obitools/obiconvert/main.go b/cmd/obitools/obiconvert/main.go index fdad6a8..84bb4bc 100644 --- a/cmd/obitools/obiconvert/main.go +++ b/cmd/obitools/obiconvert/main.go @@ -3,8 +3,8 @@ package main import ( "os" - "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions" ) @@ -22,6 +22,6 @@ func main() { obiconvert.CLIWriteBioSequences(fs, true) - obiiter.WaitForLastPipe() + obiutils.WaitForLastPipe() } diff --git a/cmd/obitools/obicsv/main.go b/cmd/obitools/obicsv/main.go index 9a75062..4277126 100644 --- a/cmd/obitools/obicsv/main.go +++ b/cmd/obitools/obicsv/main.go @@ -3,10 +3,10 @@ package main import ( "os" - "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obicsv" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" ) func main() { @@ -19,6 +19,6 @@ func main() { obicsv.CLIWriteSequenceCSV(fs, true) - obiiter.WaitForLastPipe() + obiutils.WaitForLastPipe() } diff --git a/cmd/obitools/obidemerge/main.go b/cmd/obitools/obidemerge/main.go index dd4c398..054c1e0 100644 --- a/cmd/obitools/obidemerge/main.go +++ b/cmd/obitools/obidemerge/main.go @@ -3,9 +3,9 @@ package main import ( "os" - "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obidemerge" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions" ) @@ -25,6 +25,6 @@ func main() { obiconvert.CLIWriteBioSequences(demerged, true) - obiiter.WaitForLastPipe() + obiutils.WaitForLastPipe() } diff --git a/cmd/obitools/obidistribute/main.go b/cmd/obitools/obidistribute/main.go index c9ef90e..c9aeaaf 100644 --- a/cmd/obitools/obidistribute/main.go +++ b/cmd/obitools/obidistribute/main.go @@ -3,9 +3,9 @@ package main import ( "os" - "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obidistribute" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions" ) @@ -20,6 +20,6 @@ func main() { obidistribute.CLIDistributeSequence(fs) - obiiter.WaitForLastPipe() + obiutils.WaitForLastPipe() } diff --git a/cmd/obitools/obifind/main.go b/cmd/obitools/obifind/main.go index 72c7f27..7e2ac0f 100644 --- a/cmd/obitools/obifind/main.go +++ b/cmd/obitools/obifind/main.go @@ -4,10 +4,10 @@ import ( "log" "os" - "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obifind" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" ) func main() { @@ -62,6 +62,6 @@ func main() { iterator = obifind.CLITaxonRestrictions(iterator) obifind.CLICSVTaxaWriter(iterator, true) - obiiter.WaitForLastPipe() + obiutils.WaitForLastPipe() } diff --git a/cmd/obitools/obigrep/main.go b/cmd/obitools/obigrep/main.go index 7477d80..e07782c 100644 --- a/cmd/obitools/obigrep/main.go +++ b/cmd/obitools/obigrep/main.go @@ -3,11 +3,11 @@ package main import ( "os" - "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obigrep" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" ) func main() { @@ -39,6 +39,6 @@ func main() { selected := obigrep.CLIFilterSequence(sequences) obiconvert.CLIWriteBioSequences(selected, true) - obiiter.WaitForLastPipe() + obiutils.WaitForLastPipe() } diff --git a/cmd/obitools/obijoin/main.go b/cmd/obitools/obijoin/main.go index eda016d..69af504 100644 --- a/cmd/obitools/obijoin/main.go +++ b/cmd/obitools/obijoin/main.go @@ -3,9 +3,9 @@ package main import ( "os" - "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obijoin" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions" ) @@ -25,6 +25,6 @@ func main() { obiconvert.CLIWriteBioSequences(joined, true) - obiiter.WaitForLastPipe() + obiutils.WaitForLastPipe() } diff --git a/cmd/obitools/obikmermatch/main.go b/cmd/obitools/obikmermatch/main.go index 4e286fd..4c8bf90 100644 --- a/cmd/obitools/obikmermatch/main.go +++ b/cmd/obitools/obikmermatch/main.go @@ -8,6 +8,7 @@ import ( "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obikmersim" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" ) func main() { @@ -45,6 +46,6 @@ func main() { selected := obikmersim.CLIAlignSequences(sequences) obiconvert.CLIWriteBioSequences(selected, true) - obiiter.WaitForLastPipe() + obiutils.WaitForLastPipe() } diff --git a/cmd/obitools/obikmersimcount/main.go b/cmd/obitools/obikmersimcount/main.go index d155ef8..a9df4f5 100644 --- a/cmd/obitools/obikmersimcount/main.go +++ b/cmd/obitools/obikmersimcount/main.go @@ -9,6 +9,7 @@ import ( "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obikmersim" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" ) func main() { @@ -53,6 +54,6 @@ func main() { topull.Consume() - obiiter.WaitForLastPipe() + obiutils.WaitForLastPipe() } diff --git a/cmd/obitools/obilandmark/main.go b/cmd/obitools/obilandmark/main.go index 974041a..a1f2c26 100644 --- a/cmd/obitools/obilandmark/main.go +++ b/cmd/obitools/obilandmark/main.go @@ -3,9 +3,9 @@ package main import ( "os" - "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obilandmark" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions" ) @@ -21,6 +21,6 @@ func main() { indexed := obilandmark.CLISelectLandmarkSequences(fs) obiconvert.CLIWriteBioSequences(indexed, true) - obiiter.WaitForLastPipe() + obiutils.WaitForLastPipe() } diff --git a/cmd/obitools/obimicrosat/main.go b/cmd/obitools/obimicrosat/main.go index 24849b5..e8c4a95 100644 --- a/cmd/obitools/obimicrosat/main.go +++ b/cmd/obitools/obimicrosat/main.go @@ -3,11 +3,11 @@ package main import ( "os" - "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obimicrosat" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" ) func main() { @@ -39,6 +39,6 @@ func main() { selected := obimicrosat.CLIAnnotateMicrosat(sequences) obiconvert.CLIWriteBioSequences(selected, true) - obiiter.WaitForLastPipe() + obiutils.WaitForLastPipe() } diff --git a/cmd/obitools/obimultiplex/main.go b/cmd/obitools/obimultiplex/main.go index b460e30..9c00c4e 100644 --- a/cmd/obitools/obimultiplex/main.go +++ b/cmd/obitools/obimultiplex/main.go @@ -6,10 +6,10 @@ import ( log "github.com/sirupsen/logrus" - "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obimultiplex" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" ) func main() { @@ -48,6 +48,6 @@ func main() { amplicons, _ := obimultiplex.IExtractBarcode(sequences) obiconvert.CLIWriteBioSequences(amplicons, true) amplicons.Wait() - obiiter.WaitForLastPipe() + obiutils.WaitForLastPipe() } diff --git a/cmd/obitools/obipairing/main.go b/cmd/obitools/obipairing/main.go index 9e9ff02..1e1c3bf 100644 --- a/cmd/obitools/obipairing/main.go +++ b/cmd/obitools/obipairing/main.go @@ -5,10 +5,10 @@ import ( log "github.com/sirupsen/logrus" - "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obipairing" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" ) func main() { @@ -56,5 +56,5 @@ func main() { obiconvert.CLIWriteBioSequences(paired, true) - obiiter.WaitForLastPipe() + obiutils.WaitForLastPipe() } diff --git a/cmd/obitools/obipcr/main.go b/cmd/obitools/obipcr/main.go index 1eb864e..f439fcd 100644 --- a/cmd/obitools/obipcr/main.go +++ b/cmd/obitools/obipcr/main.go @@ -3,10 +3,10 @@ package main import ( "os" - "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obipcr" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" ) func main() { @@ -37,6 +37,6 @@ func main() { amplicons, _ := obipcr.CLIPCR(sequences) obiconvert.CLIWriteBioSequences(amplicons, true) - obiiter.WaitForLastPipe() + obiutils.WaitForLastPipe() } diff --git a/cmd/obitools/obireffamidx/main.go b/cmd/obitools/obireffamidx/main.go index 5377f32..96e85fa 100644 --- a/cmd/obitools/obireffamidx/main.go +++ b/cmd/obitools/obireffamidx/main.go @@ -3,9 +3,9 @@ package main import ( "os" - "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obirefidx" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions" ) @@ -21,6 +21,6 @@ func main() { indexed := obirefidx.IndexFamilyDB(fs) obiconvert.CLIWriteBioSequences(indexed, true) - obiiter.WaitForLastPipe() + obiutils.WaitForLastPipe() } diff --git a/cmd/obitools/obirefidx/main.go b/cmd/obitools/obirefidx/main.go index 65eb5b8..9899b7f 100644 --- a/cmd/obitools/obirefidx/main.go +++ b/cmd/obitools/obirefidx/main.go @@ -3,9 +3,9 @@ package main import ( "os" - "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obirefidx" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions" ) @@ -21,6 +21,6 @@ func main() { indexed := obirefidx.IndexReferenceDB(fs) obiconvert.CLIWriteBioSequences(indexed, true) - obiiter.WaitForLastPipe() + obiutils.WaitForLastPipe() } diff --git a/cmd/obitools/obiscript/main.go b/cmd/obitools/obiscript/main.go index 453b130..861a2ad 100644 --- a/cmd/obitools/obiscript/main.go +++ b/cmd/obitools/obiscript/main.go @@ -4,11 +4,11 @@ import ( "fmt" "os" - "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiscript" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" ) func main() { @@ -46,6 +46,6 @@ func main() { annotator := obiscript.CLIScriptPipeline() obiconvert.CLIWriteBioSequences(sequences.Pipe(annotator), true) - obiiter.WaitForLastPipe() + obiutils.WaitForLastPipe() } diff --git a/cmd/obitools/obisplit/main.go b/cmd/obitools/obisplit/main.go index 44e8b17..9c6ed94 100644 --- a/cmd/obitools/obisplit/main.go +++ b/cmd/obitools/obisplit/main.go @@ -4,11 +4,11 @@ import ( "fmt" "os" - "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obisplit" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" ) func main() { @@ -46,6 +46,6 @@ func main() { annotator := obisplit.CLISlitPipeline() obiconvert.CLIWriteBioSequences(sequences.Pipe(annotator), true) - obiiter.WaitForLastPipe() + obiutils.WaitForLastPipe() } diff --git a/cmd/obitools/obitag/main.go b/cmd/obitools/obitag/main.go index 81b17ab..3b76ee2 100644 --- a/cmd/obitools/obitag/main.go +++ b/cmd/obitools/obitag/main.go @@ -10,6 +10,7 @@ import ( "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obitag" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions" ) @@ -60,7 +61,7 @@ func main() { } obiconvert.CLIWriteBioSequences(identified, true) - obiiter.WaitForLastPipe() + obiutils.WaitForLastPipe() obitag.CLISaveRefetenceDB(references) diff --git a/cmd/obitools/obitagpcr/main.go b/cmd/obitools/obitagpcr/main.go index bafa137..bc2e096 100644 --- a/cmd/obitools/obitagpcr/main.go +++ b/cmd/obitools/obitagpcr/main.go @@ -5,11 +5,11 @@ import ( log "github.com/sirupsen/logrus" - "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obipairing" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obitagpcr" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" ) func main() { @@ -54,5 +54,5 @@ func main() { obiconvert.CLIWriteBioSequences(paired, true) - obiiter.WaitForLastPipe() + obiutils.WaitForLastPipe() } diff --git a/cmd/obitools/obiuniq/main.go b/cmd/obitools/obiuniq/main.go index 510e8eb..9b18c46 100644 --- a/cmd/obitools/obiuniq/main.go +++ b/cmd/obitools/obiuniq/main.go @@ -3,11 +3,12 @@ package main import ( "os" - "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiuniq" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" ) func main() { @@ -31,7 +32,7 @@ func main() { // defer trace.Stop() obioptions.SetBatchSize(10) - obioptions.SetReadQualities(false) + obidefault.SetReadQualities(false) optionParser := obioptions.GenerateOptionParser(obiuniq.OptionSet) _, args := optionParser(os.Args) @@ -42,6 +43,6 @@ func main() { unique := obiuniq.CLIUnique(sequences) obiconvert.CLIWriteBioSequences(unique, true) - obiiter.WaitForLastPipe() + obiutils.WaitForLastPipe() } diff --git a/cmd/test/main.go b/cmd/test/main.go index 84ab35d..2e07616 100644 --- a/cmd/test/main.go +++ b/cmd/test/main.go @@ -4,9 +4,13 @@ import ( "os" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitaxformat" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" ) func main() { obitaxformat.DetectTaxonomyFormat(os.Args[1]) + println(obiutils.RemoveAllExt("toto/tutu/test.txt")) + println(obiutils.Basename("toto/tutu/test.txt")) + } diff --git a/pkg/obidefault/quality.go b/pkg/obidefault/quality.go new file mode 100644 index 0000000..8f56c93 --- /dev/null +++ b/pkg/obidefault/quality.go @@ -0,0 +1,29 @@ +package obidefault + +var _Quality_Shift_Input = byte(33) +var _Quality_Shift_Output = byte(33) +var _Read_Qualities = true + +func SetReadQualitiesShift(shift byte) { + _Quality_Shift_Input = shift +} + +func ReadQualitiesShift() byte { + return _Quality_Shift_Input +} + +func SetWriteQualitiesShift(shift byte) { + _Quality_Shift_Output = shift +} + +func WriteQualitiesShift() byte { + return _Quality_Shift_Output +} + +func SetReadQualities(read bool) { + _Read_Qualities = read +} + +func ReadQualities() bool { + return _Read_Qualities +} diff --git a/pkg/obiformats/csv_read.go b/pkg/obiformats/csv_read.go index 239989c..09d2545 100644 --- a/pkg/obiformats/csv_read.go +++ b/pkg/obiformats/csv_read.go @@ -7,8 +7,8 @@ import ( "path" "strings" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter" - "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" "github.com/goccy/go-json" @@ -143,7 +143,7 @@ func ReadCSV(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, err go _ParseCsvFile(opt.Source(), reader, out, - byte(obioptions.InputQualityShift()), + obidefault.ReadQualitiesShift(), opt.BatchSize()) go func() { diff --git a/pkg/obiformats/csv_writer.go b/pkg/obiformats/csv_writer.go index 42ba6e6..2052175 100644 --- a/pkg/obiformats/csv_writer.go +++ b/pkg/obiformats/csv_writer.go @@ -3,7 +3,7 @@ package obiformats import ( "fmt" - "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" ) @@ -54,7 +54,7 @@ func CSVSequenceRecord(sequence *obiseq.BioSequence, opt Options) []string { l := sequence.Len() q := sequence.Qualities() ascii := make([]byte, l) - quality_shift := obioptions.OutputQualityShift() + quality_shift := obidefault.WriteQualitiesShift() for j := 0; j < l; j++ { ascii[j] = uint8(q[j]) + uint8(quality_shift) } diff --git a/pkg/obiformats/fastqseq_read.go b/pkg/obiformats/fastqseq_read.go index 55ba783..0000e73 100644 --- a/pkg/obiformats/fastqseq_read.go +++ b/pkg/obiformats/fastqseq_read.go @@ -7,8 +7,8 @@ import ( "os" "path" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter" - "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" log "github.com/sirupsen/logrus" @@ -341,7 +341,7 @@ func ReadFastq(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, e go _ParseFastqFile( chkchan, out, - byte(obioptions.InputQualityShift()), + obidefault.ReadQualitiesShift(), opt.ReadQualities(), ) } diff --git a/pkg/obiformats/fastseq_read.go b/pkg/obiformats/fastseq_read.go index f43e361..c94e71c 100644 --- a/pkg/obiformats/fastseq_read.go +++ b/pkg/obiformats/fastseq_read.go @@ -14,8 +14,8 @@ import ( log "github.com/sirupsen/logrus" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter" - "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" ) @@ -92,7 +92,7 @@ func ReadFastSeqFromFile(filename string, options ...WithOption) (obiiter.IBioSe name := C.CString(filename) defer C.free(unsafe.Pointer(name)) - pointer := C.open_fast_sek_file(name, C.int32_t(obioptions.InputQualityShift())) + pointer := C.open_fast_sek_file(name, C.int32_t(obidefault.ReadQualitiesShift())) var err error err = nil @@ -151,7 +151,7 @@ func ReadFastSeqFromStdin(options ...WithOption) obiiter.IBioSequence { }(newIter) go _FastseqReader(opt.Source(), - C.open_fast_sek_stdin(C.int32_t(obioptions.InputQualityShift())), + C.open_fast_sek_stdin(C.int32_t(obidefault.ReadQualitiesShift())), newIter, opt.BatchSize()) log.Debugln("Full file batch mode : ", opt.FullFileBatch()) diff --git a/pkg/obiformats/file_chunk_write.go b/pkg/obiformats/file_chunk_write.go index 820a821..028deb6 100644 --- a/pkg/obiformats/file_chunk_write.go +++ b/pkg/obiformats/file_chunk_write.go @@ -3,7 +3,7 @@ package obiformats import ( "io" - "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" log "github.com/sirupsen/logrus" ) @@ -12,7 +12,7 @@ func WriteFileChunk( writer io.WriteCloser, toBeClosed bool) ChannelFileChunk { - obiiter.RegisterAPipe() + obiutils.RegisterAPipe() chunk_channel := make(ChannelFileChunk) go func() { @@ -53,7 +53,7 @@ func WriteFileChunk( } } - obiiter.UnregisterPipe() + obiutils.UnregisterPipe() log.Debugf("The writer has been closed") }() diff --git a/pkg/obiiter/batchiterator.go b/pkg/obiiter/batchiterator.go index 4a49403..95fc044 100644 --- a/pkg/obiiter/batchiterator.go +++ b/pkg/obiiter/batchiterator.go @@ -16,39 +16,6 @@ import ( "github.com/tevino/abool/v2" ) -var globalLocker sync.WaitGroup -var globalLockerCounter = 0 - -// RegisterAPipe increments the global lock counter and adds a new pipe to the global wait group. -// -// No parameters. -// No return values. -func RegisterAPipe() { - globalLocker.Add(1) - globalLockerCounter++ - log.Debugln(globalLockerCounter, " Pipes are registered now") -} - -// UnregisterPipe decrements the global lock counter and signals that a pipe has finished. -// -// No parameters. -// No return values. -func UnregisterPipe() { - globalLocker.Done() - globalLockerCounter-- - log.Debugln(globalLockerCounter, "are still registered") -} - -// WaitForLastPipe waits until all registered pipes have finished. -// -// THe function have to be called at the end of every main function. -// -// No parameters. -// No return values. -func WaitForLastPipe() { - globalLocker.Wait() -} - // Structure implementing an iterator over bioseq.BioSequenceBatch // based on a channel. type _IBioSequence struct { @@ -92,7 +59,7 @@ func MakeIBioSequence() IBioSequence { i.lock = &lock ii := IBioSequence{&i} - RegisterAPipe() + obiutils.RegisterAPipe() return ii } @@ -259,7 +226,7 @@ func (iterator IBioSequence) Push(batch BioSequenceBatch) { func (iterator IBioSequence) Close() { close(iterator.pointer.channel) - UnregisterPipe() + obiutils.UnregisterPipe() } func (iterator IBioSequence) WaitAndClose() { diff --git a/pkg/obioptions/options.go b/pkg/obioptions/options.go index 4951afe..674b86c 100644 --- a/pkg/obioptions/options.go +++ b/pkg/obioptions/options.go @@ -6,6 +6,7 @@ import ( "os" "runtime" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitaxformat" log "github.com/sirupsen/logrus" @@ -28,9 +29,6 @@ var _BatchSize = 2000 var _Pprof = false var _PprofMudex = 10 var _PprofGoroutine = 6060 -var _Quality_Shift_Input = byte(33) -var _Quality_Shift_Output = byte(33) -var _Read_Qualities = true var __taxonomy__ = "" var __alternative_name__ = false @@ -177,7 +175,7 @@ func GenerateOptionParser(optionset ...func(*getoptions.GetOpt)) ArgumentParser // } if options.Called("solexa") { - SetInputQualityShift(64) + obidefault.SetReadQualitiesShift(64) } return options, remaining @@ -292,10 +290,6 @@ func CLIBatchSize() int { return _BatchSize } -func CLIReadQualities() bool { - return _Read_Qualities -} - // SetDebugOn sets the debug mode on. func SetDebugOn() { _Debug = true @@ -306,10 +300,6 @@ func SetDebugOff() { _Debug = false } -func SetReadQualities(status bool) { - _Read_Qualities = status -} - // SetWorkerPerCore sets the number of workers per CPU core. // // It takes a float64 parameter representing the number of workers @@ -359,43 +349,6 @@ func SetBatchSize(n int) { _BatchSize = n } -// InputQualityShift returns the quality shift value for input. -// -// It can be set programmatically by the SetInputQualityShift() function. -// This value is used to decode the quality scores in FASTQ files. -// The quality shift value defaults to 33, which is the correct value for -// Sanger formated FASTQ files. -// The quality shift value can be modified to 64 by the command line option -// --solexa, for decoding old Solexa formated FASTQ files. -// -// No parameters. -// Returns an integer representing the quality shift value for input. -func InputQualityShift() byte { - return _Quality_Shift_Input -} - -// OutputQualityShift returns the quality shift value used for FASTQ output. -// -// No parameters. -// Returns an integer representing the quality shift value for output. -func OutputQualityShift() byte { - return _Quality_Shift_Output -} - -// SetInputQualityShift sets the quality shift value for decoding FASTQ. -// -// n - an integer representing the quality shift value to be set. -func SetInputQualityShift[T int | byte](n T) { - _Quality_Shift_Input = byte(n) -} - -// SetOutputQualityShift sets the quality shift value used for FASTQ output. -// -// n - an integer representing the quality shift value to be set. -func SetOutputQualityShift[T int | byte](n T) { - _Quality_Shift_Output = byte(n) -} - // SetMaxCPU sets the maximum number of CPU cores allowed. // // n - an integer representing the new maximum number of CPU cores. diff --git a/pkg/obioptions/version.go b/pkg/obioptions/version.go index 72251f0..362ed36 100644 --- a/pkg/obioptions/version.go +++ b/pkg/obioptions/version.go @@ -8,7 +8,7 @@ import ( // corresponds to the last commit, and not the one when the file will be // commited -var _Commit = "ffd6725" +var _Commit = "3137c1f" var _Version = "Release 4.2.0" // Version returns the version of the obitools package. diff --git a/pkg/obiseq/biosequence.go b/pkg/obiseq/biosequence.go index 60448ec..1864f29 100644 --- a/pkg/obiseq/biosequence.go +++ b/pkg/obiseq/biosequence.go @@ -18,7 +18,7 @@ import ( "sync/atomic" "unsafe" - "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" log "github.com/sirupsen/logrus" @@ -287,7 +287,7 @@ func (s *BioSequence) Qualities() Quality { // // Returns a string representing the qualities of the BioSequence after applying the shift. func (s *BioSequence) QualitiesString() string { - quality_shift := obioptions.OutputQualityShift() + quality_shift := obidefault.WriteQualitiesShift() qual := s.Qualities() qual_ascii := make([]byte, len(qual)) diff --git a/pkg/obitax/iterator.go b/pkg/obitax/iterator.go index e2c80a7..a4cf039 100644 --- a/pkg/obitax/iterator.go +++ b/pkg/obitax/iterator.go @@ -1,5 +1,9 @@ package obitax +import ( + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" +) + // ITaxon represents an iterator for traversing Taxon instances. // It provides methods to retrieve the next Taxon and check if the iteration is finished. type ITaxon struct { @@ -168,3 +172,46 @@ func (iterator *ITaxon) Concat(iterators ...*ITaxon) *ITaxon { return newIter } + +func (taxon *Taxon) ISubTaxonomy() *ITaxon { + + taxo := taxon.Taxonomy + + path := taxon.Path() + lpath := path.Len() + + iter := NewITaxon() + + parents := map[*TaxNode]bool{taxon.Node: true} + + obiutils.RegisterAPipe() + + go func() { + for i := lpath - 1; i >= 0; i-- { + taxon := path.Taxon(i) + parents[taxon.Node] = true + iter.Push(taxon) + } + + pushed := true + + for pushed { + itaxo := taxo.Iterator() + pushed = false + for itaxo.Next() { + taxon := itaxo.Get() + + if !parents[taxon.Node] && parents[taxon.Parent().Node] { + parents[taxon.Node] = true + iter.Push(taxon) + pushed = true + } + } + } + + iter.Close() + obiutils.UnregisterPipe() + }() + + return iter +} diff --git a/pkg/obitaxformat/csvtaxdump/read.go b/pkg/obitaxformat/csvtaxdump/read.go new file mode 100644 index 0000000..51c36d0 --- /dev/null +++ b/pkg/obitaxformat/csvtaxdump/read.go @@ -0,0 +1,104 @@ +package csvtaxdump + +import ( + "encoding/csv" + "errors" + "strings" + + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" + log "github.com/sirupsen/logrus" +) + +func LoadCSVTaxonomy(path string, onlysn bool) (*obitax.Taxonomy, error) { + + file, err := obiutils.Ropen(path) + + if err != nil { + return nil, err + } + + defer file.Close() + + csvfile := csv.NewReader(file) + + csvfile.Comma = ',' + csvfile.ReuseRecord = false + csvfile.LazyQuotes = true + csvfile.Comment = '#' + csvfile.FieldsPerRecord = -1 + csvfile.TrimLeadingSpace = true + + header, err := csvfile.Read() + + if err != nil { + log.Fatal(err) + } + + taxidColIndex := -1 + parentColIndex := -1 + scientific_nameColIndex := -1 + rankColIndex := -1 + + for i, colName := range header { + switch colName { + case "taxid": + taxidColIndex = i + case "parent": + parentColIndex = i + case "scientific_name": + scientific_nameColIndex = i + case "rank": + rankColIndex = i + } + } + + if taxidColIndex == -1 { + return nil, errors.New("taxonomy file does not contain taxid column") + } + + if parentColIndex == -1 { + return nil, errors.New("taxonomy file does not contain parent column") + } + + if scientific_nameColIndex == -1 { + return nil, errors.New("taxonomy file does not contain scientific_name column") + } + + if rankColIndex == -1 { + return nil, errors.New("taxonomy file does not contain rank column") + } + + name := obiutils.RemoveAllExt(path) + short := obiutils.Basename(path) + taxonomy := obitax.NewTaxonomy(name, short, obiutils.AsciiAlphaNumSet) + + line, err := csvfile.Read() + + for err != nil { + taxid := line[taxidColIndex] + parent := line[parentColIndex] + scientific_name := line[scientific_nameColIndex] + rank := line[rankColIndex] + + parts := strings.Split(rank, ":") + + rank = parts[0] + + root := len(parts) > 1 && parts[1] == "root" + + taxon, err := taxonomy.AddTaxon(taxid, parent, rank, false, root) + taxon.SetName(scientific_name, "scientific name") + + if err != nil { + return nil, err + } + + } + + if !taxonomy.HasRoot() { + return nil, errors.New("taxonomy file does not contain root node") + } + + return taxonomy, nil +} diff --git a/pkg/obitaxformat/csvtaxdump/write.go b/pkg/obitaxformat/csvtaxdump/write.go new file mode 100644 index 0000000..82c9b1b --- /dev/null +++ b/pkg/obitaxformat/csvtaxdump/write.go @@ -0,0 +1,12 @@ +package csvtaxdump + +// import ( +// "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter" +// "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obicsv" +// ) + +// func WriteTaxonomyCSV(iterator obiiter.IBioSequence, +// terminalAction bool, filenames ...string) *obicsv.ICSVRecord { + +// return nil +// } diff --git a/pkg/obitaxformat/taxonomy_read.go b/pkg/obitaxformat/taxonomy_read.go index 58c26ef..4a2a102 100644 --- a/pkg/obitaxformat/taxonomy_read.go +++ b/pkg/obitaxformat/taxonomy_read.go @@ -5,6 +5,7 @@ import ( "os" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitax" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitaxformat/csvtaxdump" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitaxformat/ncbitaxdump" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" "github.com/gabriel-vasile/mimetype" @@ -62,7 +63,7 @@ func DetectTaxonomyFormat(path string) (TaxonomyLoader, error) { switch mimetype.String() { case "text/csv": - return LoadCSVTaxonomy, nil + return csvtaxdump.LoadCSVTaxonomy, nil case "application/x-tar": return DetectTaxonomyTarFormat(path) } @@ -73,11 +74,6 @@ func DetectTaxonomyFormat(path string) (TaxonomyLoader, error) { return nil, nil } -func LoadCSVTaxonomy(path string, onlysn bool) (*obitax.Taxonomy, error) { - - return nil, nil -} - func LoadTaxonomy(path string, onlysn bool) (*obitax.Taxonomy, error) { loader, err := DetectTaxonomyFormat(path) diff --git a/pkg/obitools/obiconvert/sequence_reader.go b/pkg/obitools/obiconvert/sequence_reader.go index 9c1eaa7..5cec695 100644 --- a/pkg/obitools/obiconvert/sequence_reader.go +++ b/pkg/obitools/obiconvert/sequence_reader.go @@ -9,6 +9,7 @@ import ( "github.com/goombaio/orderedset" log "github.com/sirupsen/logrus" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiformats" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions" @@ -99,7 +100,7 @@ func CLIReadBioSequences(filenames ...string) (obiiter.IBioSequence, error) { opts = append(opts, obiformats.OptionsFastSeqHeaderParser(obiformats.ParseGuessedFastSeqHeader)) } - opts = append(opts, obiformats.OptionsReadQualities(obioptions.CLIReadQualities())) + opts = append(opts, obiformats.OptionsReadQualities(obidefault.ReadQualities())) nworkers := obioptions.CLIReadParallelWorkers() if nworkers < 2 { diff --git a/pkg/obitools/obicsv/iter.go b/pkg/obitools/obicsv/iter.go index 77405ba..5d994d4 100644 --- a/pkg/obitools/obicsv/iter.go +++ b/pkg/obitools/obicsv/iter.go @@ -7,7 +7,7 @@ import ( "sync/atomic" "time" - "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" "github.com/tevino/abool/v2" log "github.com/sirupsen/logrus" @@ -57,7 +57,7 @@ func NewICSVRecord() *ICSVRecord { lock := sync.RWMutex{} i.lock = &lock - obiiter.RegisterAPipe() + obiutils.RegisterAPipe() return &i } @@ -275,7 +275,7 @@ func (iterator *ICSVRecord) Push(batch CSVRecordBatch) { func (iterator *ICSVRecord) Close() { close(iterator.channel) - obiiter.UnregisterPipe() + obiutils.UnregisterPipe() } func (iterator *ICSVRecord) WaitAndClose() { diff --git a/pkg/obitools/obicsv/sequence.go b/pkg/obitools/obicsv/sequence.go index cc3023b..fc57a1f 100644 --- a/pkg/obitools/obicsv/sequence.go +++ b/pkg/obitools/obicsv/sequence.go @@ -4,8 +4,8 @@ import ( "log" "slices" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter" - "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions" ) func CSVSequenceHeader(opt Options) CSVHeader { @@ -95,7 +95,7 @@ func CSVBatchFromSequences(batch obiiter.BioSequenceBatch, opt Options) CSVRecor l := sequence.Len() q := sequence.Qualities() ascii := make([]byte, l) - quality_shift := obioptions.OutputQualityShift() + quality_shift := obidefault.WriteQualitiesShift() for j := 0; j < l; j++ { ascii[j] = uint8(q[j]) + uint8(quality_shift) } diff --git a/pkg/obitools/obimatrix/obimatrix.go b/pkg/obitools/obimatrix/obimatrix.go index 04b30c6..5456b6b 100644 --- a/pkg/obitools/obimatrix/obimatrix.go +++ b/pkg/obitools/obimatrix/obimatrix.go @@ -122,7 +122,7 @@ func IMatrix(iterator obiiter.IBioSequence) *MatrixData { } waiter.Wait() - obiiter.WaitForLastPipe() + obiutils.WaitForLastPipe() rep := summaries[0] diff --git a/pkg/obitools/obisummary/obisummary.go b/pkg/obitools/obisummary/obisummary.go index cc5ecdf..4dd1d5f 100644 --- a/pkg/obitools/obisummary/obisummary.go +++ b/pkg/obitools/obisummary/obisummary.go @@ -187,7 +187,7 @@ func ISummary(iterator obiiter.IBioSequence, summarise []string) map[string]inte } waiter.Wait() - obiiter.WaitForLastPipe() + obiutils.WaitForLastPipe() rep := summaries[0] diff --git a/pkg/obitools/obitag/options.go b/pkg/obitools/obitag/options.go index 1f162f3..53c5011 100644 --- a/pkg/obitools/obitag/options.go +++ b/pkg/obitools/obitag/options.go @@ -8,6 +8,7 @@ import ( "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert" + "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" "github.com/DavidGamba/go-getoptions" ) @@ -112,7 +113,7 @@ func CLISaveRefetenceDB(db obiseq.BioSequenceSlice) { } newIter.Recycle() - obiiter.WaitForLastPipe() + obiutils.WaitForLastPipe() } } diff --git a/pkg/obiutils/abs_test.go b/pkg/obiutils/abs_test.go index 9fd1746..ca7e44b 100644 --- a/pkg/obiutils/abs_test.go +++ b/pkg/obiutils/abs_test.go @@ -1,9 +1,8 @@ package obiutils_test import ( - "testing" - "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" + "testing" ) func TestAbs(t *testing.T) { diff --git a/pkg/obiutils/path.go b/pkg/obiutils/path.go index 598afd3..2e30fb8 100644 --- a/pkg/obiutils/path.go +++ b/pkg/obiutils/path.go @@ -2,6 +2,7 @@ package obiutils import ( "path" + "path/filepath" "strings" ) @@ -21,3 +22,16 @@ func RemoveAllExt(p string) string { return p } + +func Basename(path string) string { + filename := filepath.Base(path) + ext := filepath.Ext(filename) + + // Keep removing extensions until there are no more + for ext != "" { + filename = strings.TrimSuffix(filename, ext) + ext = filepath.Ext(filename) + } + + return filename +} diff --git a/pkg/obiutils/pipe.go b/pkg/obiutils/pipe.go new file mode 100644 index 0000000..fef4447 --- /dev/null +++ b/pkg/obiutils/pipe.go @@ -0,0 +1,40 @@ +package obiutils + +import ( + "sync" + + log "github.com/sirupsen/logrus" +) + +var globalLocker sync.WaitGroup +var globalLockerCounter = 0 + +// RegisterAPipe increments the global lock counter and adds a new pipe to the global wait group. +// +// No parameters. +// No return values. +func RegisterAPipe() { + globalLocker.Add(1) + globalLockerCounter++ + log.Debugln(globalLockerCounter, " Pipes are registered now") +} + +// UnregisterPipe decrements the global lock counter and signals that a pipe has finished. +// +// No parameters. +// No return values. +func UnregisterPipe() { + globalLocker.Done() + globalLockerCounter-- + log.Debugln(globalLockerCounter, "are still registered") +} + +// WaitForLastPipe waits until all registered pipes have finished. +// +// THe function have to be called at the end of every main function. +// +// No parameters. +// No return values. +func WaitForLastPipe() { + globalLocker.Wait() +}