obitools/obitools4
2
0
Fork 0
mirror of https://github.com/metabarcoding/obitools4.git synced 2025-12-08 16:50:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add capacity to obidistribute to save gzipped files

Browse Source
This commit is contained in:
Eric Coissac
2023-02-17 12:52:19 +01:00
parent 6fd426d8be
commit 9554a32490
9 changed files with 124 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
pkg/obiformats/dispatcher.go
View File

@@ -2,6 +2,7 @@ package obiformats
import (
"fmt"
"strings"
"sync"
log "github.com/sirupsen/logrus"
@@ -22,6 +23,7 @@ func WriterDispatcher(prototypename string,
jobDone.Add(1)
go func() {
opt := MakeOptions(options)
for newflux := range dispatcher.News() {
jobDone.Add(1)
go func(newflux int) {
@@ -31,8 +33,13 @@ func WriterDispatcher(prototypename string,
log.Fatalf("Cannot retreive the new chanel : %v", err)
}
name:=fmt.Sprintf(prototypename, dispatcher.Classifier().Value(newflux))
if opt.CompressedFile() && ! strings.HasSuffix(name,".gz") {
name = name + ".gz"
}
out, err := formater(data,
fmt.Sprintf(prototypename, dispatcher.Classifier().Value(newflux)),
name,
options...)
if err != nil {

17
pkg/obiformats/fastseq_write_fasta.go
View File

@@ -11,6 +11,7 @@ import (
log "github.com/sirupsen/logrus"
"git.metabarcoding.org/lecasofts/go/obitools/pkg/goutils"
"git.metabarcoding.org/lecasofts/go/obitools/pkg/obiiter"
"git.metabarcoding.org/lecasofts/go/obitools/pkg/obiseq"
)
@@ -138,6 +139,8 @@ func WriteFasta(iterator obiiter.IBioSequence,
switch file := file.(type) {
case *os.File:
file.Close()
case *goutils.Wfile:
file.Close()
}
}
waitWriter.Done()
@@ -157,19 +160,13 @@ func WriteFastaToFile(iterator obiiter.IBioSequence,
filename string,
options ...WithOption) (obiiter.IBioSequence, error) {
var file *os.File
var err error
opt := MakeOptions(options)
if opt.AppendFile() {
log.Debug("Open files in appending mode")
file, err = os.OpenFile(filename,
os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
} else {
file, err = os.Create(filename)
}
file,err := goutils.OpenWritingFile(filename,
opt.CompressedFile(),
opt.AppendFile(),
)
if err != nil {
log.Fatalf("open file error: %v", err)

17
pkg/obiformats/fastseq_write_fastq.go
View File

@@ -10,6 +10,7 @@ import (
log "github.com/sirupsen/logrus"
"git.metabarcoding.org/lecasofts/go/obitools/pkg/goutils"
"git.metabarcoding.org/lecasofts/go/obitools/pkg/obiiter"
"git.metabarcoding.org/lecasofts/go/obitools/pkg/obiseq"
)
@@ -128,6 +129,8 @@ func WriteFastq(iterator obiiter.IBioSequence,
switch file := file.(type) {
case *os.File:
file.Close()
case *goutils.Wfile:
file.Close()
}
}
@@ -147,18 +150,12 @@ func WriteFastqToFile(iterator obiiter.IBioSequence,
filename string,
options ...WithOption) (obiiter.IBioSequence, error) {
var file *os.File
var err error
opt := MakeOptions(options)
if opt.AppendFile() {
log.Debug("Open files in appending mode")
file, err = os.OpenFile(filename,
os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
} else {
file, err = os.Create(filename)
}
file, err := goutils.OpenWritingFile(filename,
opt.CompressedFile(),
opt.AppendFile(),
)
if err != nil {
log.Fatalf("open file error: %v", err)

14
pkg/obiformats/options.go
View File

@@ -14,6 +14,7 @@ type __options__ struct {
parallel_workers int
closefile bool
appendfile bool
compressed bool
}
type Options struct {
@@ -33,6 +34,7 @@ func MakeOptions(setters []WithOption) Options {
batch_size: 5000,
closefile: false,
appendfile: false,
compressed: false,
}
opt := Options{&o}
@@ -80,6 +82,10 @@ func (opt Options) AppendFile() bool {
return opt.pointer.appendfile
}
func (opt Options) CompressedFile() bool {
return opt.pointer.compressed
}
func OptionsBufferSize(size int) WithOption {
f := WithOption(func(opt Options) {
opt.pointer.buffer_size = size
@@ -112,6 +118,14 @@ func OptionsAppendFile() WithOption {
return f
}
func OptionsCompressed() WithOption {
f := WithOption(func(opt Options) {
opt.pointer.compressed = true
})
return f
}
func OptionsNewFile() WithOption {
f := WithOption(func(opt Options) {
opt.pointer.appendfile = false

14
pkg/obiformats/universal_write.go
View File

@@ -7,6 +7,7 @@ import (
log "github.com/sirupsen/logrus"
"git.metabarcoding.org/lecasofts/go/obitools/pkg/goutils"
"git.metabarcoding.org/lecasofts/go/obitools/pkg/obiiter"
)
@@ -55,18 +56,13 @@ func WriteSequencesToFile(iterator obiiter.IBioSequence,
filename string,
options ...WithOption) (obiiter.IBioSequence, error) {
var file *os.File
var err error
opt := MakeOptions(options)
if opt.AppendFile() {
log.Debug("Open files in appending mode")
file, err = os.OpenFile(filename,
os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
} else {
file, err = os.Create(filename)
}
file, err := goutils.OpenWritingFile(filename,
opt.CompressedFile(),
opt.AppendFile(),
)
if err != nil {
log.Fatalf("open file error: %v", err)