obitools/obitools4
2
0
Fork 0
mirror of https://github.com/metabarcoding/obitools4.git synced 2025-06-29 16:20:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add capacity to obidistribute to save gzipped files

Browse Source
This commit is contained in:
Eric Coissac
2023-02-17 12:52:19 +01:00
parent 6fd426d8be
commit 9554a32490
9 changed files with 124 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
pkg/goutils/gzipfile.go Normal file
View File

@ -0,0 +1,69 @@
package goutils
import (
"bufio"
"compress/gzip"
"os"
)
type Wfile struct {
compressed bool
f *os.File
gf *gzip.Writer
fw *bufio.Writer
}
func OpenWritingFile(name string, compressed bool, append bool) (*Wfile, error) {
flags := os.O_WRONLY | os.O_CREATE
if append {
flags |= os.O_APPEND
}
fi, err := os.OpenFile(name, flags, 0660)
if err != nil {
return nil, err
}
var gf *gzip.Writer
var fw *bufio.Writer
if compressed {
gf = gzip.NewWriter(fi)
fw = bufio.NewWriter(gf)
} else {
gf = nil
fw = bufio.NewWriter(fi)
}
return &Wfile{compressed: compressed,
f: fi,
gf: gf,
fw: fw,
}, nil
}
func (w *Wfile) Write(p []byte) (n int, err error) {
return w.fw.Write(p)
}
func (w *Wfile) WriteString(s string) (n int, err error) {
return w.fw.Write([]byte(s))
}
func (w *Wfile) Close() error {
var err error
err = nil
if w.compressed {
err = w.gf.Close()
}
err2 := w.f.Close()
if err == nil {
err = err2
}
return err
}

9
pkg/obiformats/dispatcher.go
View File

@ -2,6 +2,7 @@ package obiformats
import (
"fmt"
"strings"
"sync"
log "github.com/sirupsen/logrus"
@ -22,6 +23,7 @@ func WriterDispatcher(prototypename string,
jobDone.Add(1)
go func() {
opt := MakeOptions(options)
for newflux := range dispatcher.News() {
jobDone.Add(1)
go func(newflux int) {
@ -31,8 +33,13 @@ func WriterDispatcher(prototypename string,
log.Fatalf("Cannot retreive the new chanel : %v", err)
}
name:=fmt.Sprintf(prototypename, dispatcher.Classifier().Value(newflux))
if opt.CompressedFile() && ! strings.HasSuffix(name,".gz") {
name = name + ".gz"
}
out, err := formater(data,
fmt.Sprintf(prototypename, dispatcher.Classifier().Value(newflux)),
name,
options...)
if err != nil {

17
pkg/obiformats/fastseq_write_fasta.go
View File

@ -11,6 +11,7 @@ import (
log "github.com/sirupsen/logrus"
"git.metabarcoding.org/lecasofts/go/obitools/pkg/goutils"
"git.metabarcoding.org/lecasofts/go/obitools/pkg/obiiter"
"git.metabarcoding.org/lecasofts/go/obitools/pkg/obiseq"
)
@ -138,6 +139,8 @@ func WriteFasta(iterator obiiter.IBioSequence,
switch file := file.(type) {
case *os.File:
file.Close()
case *goutils.Wfile:
file.Close()
}
}
waitWriter.Done()
@ -157,19 +160,13 @@ func WriteFastaToFile(iterator obiiter.IBioSequence,
filename string,
options ...WithOption) (obiiter.IBioSequence, error) {
var file *os.File
var err error
opt := MakeOptions(options)
if opt.AppendFile() {
log.Debug("Open files in appending mode")
file, err = os.OpenFile(filename,
os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
} else {
file, err = os.Create(filename)
}
file,err := goutils.OpenWritingFile(filename,
opt.CompressedFile(),
opt.AppendFile(),
)
if err != nil {
log.Fatalf("open file error: %v", err)

17
pkg/obiformats/fastseq_write_fastq.go
View File

@ -10,6 +10,7 @@ import (
log "github.com/sirupsen/logrus"
"git.metabarcoding.org/lecasofts/go/obitools/pkg/goutils"
"git.metabarcoding.org/lecasofts/go/obitools/pkg/obiiter"
"git.metabarcoding.org/lecasofts/go/obitools/pkg/obiseq"
)
@ -128,6 +129,8 @@ func WriteFastq(iterator obiiter.IBioSequence,
switch file := file.(type) {
case *os.File:
file.Close()
case *goutils.Wfile:
file.Close()
}
}
@ -147,18 +150,12 @@ func WriteFastqToFile(iterator obiiter.IBioSequence,
filename string,
options ...WithOption) (obiiter.IBioSequence, error) {
var file *os.File
var err error
opt := MakeOptions(options)
if opt.AppendFile() {
log.Debug("Open files in appending mode")
file, err = os.OpenFile(filename,
os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
} else {
file, err = os.Create(filename)
}
file, err := goutils.OpenWritingFile(filename,
opt.CompressedFile(),
opt.AppendFile(),
)
if err != nil {
log.Fatalf("open file error: %v", err)

14
pkg/obiformats/options.go
View File

@ -14,6 +14,7 @@ type __options__ struct {
parallel_workers int
closefile bool
appendfile bool
compressed bool
}
type Options struct {
@ -33,6 +34,7 @@ func MakeOptions(setters []WithOption) Options {
batch_size: 5000,
closefile: false,
appendfile: false,
compressed: false,
}
opt := Options{&o}
@ -80,6 +82,10 @@ func (opt Options) AppendFile() bool {
return opt.pointer.appendfile
}
func (opt Options) CompressedFile() bool {
return opt.pointer.compressed
}
func OptionsBufferSize(size int) WithOption {
f := WithOption(func(opt Options) {
opt.pointer.buffer_size = size
@ -112,6 +118,14 @@ func OptionsAppendFile() WithOption {
return f
}
func OptionsCompressed() WithOption {
f := WithOption(func(opt Options) {
opt.pointer.compressed = true
})
return f
}
func OptionsNewFile() WithOption {
f := WithOption(func(opt Options) {
opt.pointer.appendfile = false

14
pkg/obiformats/universal_write.go
View File

@ -7,6 +7,7 @@ import (
log "github.com/sirupsen/logrus"
"git.metabarcoding.org/lecasofts/go/obitools/pkg/goutils"
"git.metabarcoding.org/lecasofts/go/obitools/pkg/obiiter"
)
@ -55,18 +56,13 @@ func WriteSequencesToFile(iterator obiiter.IBioSequence,
filename string,
options ...WithOption) (obiiter.IBioSequence, error) {
var file *os.File
var err error
opt := MakeOptions(options)
if opt.AppendFile() {
log.Debug("Open files in appending mode")
file, err = os.OpenFile(filename,
os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
} else {
file, err = os.Create(filename)
}
file, err := goutils.OpenWritingFile(filename,
opt.CompressedFile(),
opt.AppendFile(),
)
if err != nil {
log.Fatalf("open file error: %v", err)

1
pkg/obitools/obiconvert/options.go
View File

@ -71,6 +71,7 @@ func OutputOptionSet(options *getoptions.GetOpt) {
options.BoolVar(&__no_progress_bar__, "no-progressbar", false,
options.Description("Disable the progress bar printing"))
}
func OptionSet(options *getoptions.GetOpt) {

3
pkg/obitools/obidistribute/distribute.go
View File

@ -40,6 +40,9 @@ func DistributeSequence(sequences obiiter.IBioSequence) {
opts = append(opts, obiformats.OptionsAppendFile())
}
if CLICompressed() {
opts = append(opts, obiformats.OptionsCompressed())
}
var formater obiformats.SequenceBatchWriterToFile
switch obiconvert.CLIOutputFormat() {

10
pkg/obitools/obidistribute/options.go
View File

@ -17,6 +17,8 @@ var _BatchCount = 0
var _HashSize = 0
var _NAValue = "NA"
var _append = false
var _compressed = false
func DistributeOptionSet(options *getoptions.GetOpt) {
options.StringVar(&_FilenamePattern, "pattern", _FilenamePattern,
@ -43,6 +45,10 @@ func DistributeOptionSet(options *getoptions.GetOpt) {
options.Alias("A"),
options.Description("Indicates to append sequence to files if they already exist."))
options.BoolVar(&_compressed, "--compress", false,
options.Alias("Z"),
options.Description("Output is compressed"))
options.IntVar(&_HashSize, "hash", 0,
options.Alias("H"),
options.Description("Indicates to split the input into at most <n> batch based on a hash code of the seequence."))
@ -58,6 +64,10 @@ func CLIAppendSequences() bool {
return _append
}
func CLICompressed() bool {
return _compressed
}
func CLISequenceClassifier() *obiseq.BioSequenceClassifier {
switch {
case _SequenceClassifierTag != "":