obitools/obitools4
2
0
Fork 0
mirror of https://github.com/metabarcoding/obitools4.git synced 2025-12-08 16:50:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

rename the iterator class

Browse Source
This commit is contained in:
Eric Coissac
2023-01-22 22:04:17 +01:00
parent 4592855095
commit f97f92df72
37 changed files with 399 additions and 354 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
pkg/obiformats/batch_of_files_reader.go
View File

@@ -10,13 +10,13 @@ import (
func ReadSequencesBatchFromFiles(filenames []string,
reader IBatchReader,
concurrent_readers int,
options ...WithOption) obiiter.IBioSequenceBatch {
options ...WithOption) obiiter.IBioSequence {
if reader == nil {
reader = ReadSequencesFromFile
}
batchiter := obiiter.MakeIBioSequenceBatch(0)
batchiter := obiiter.MakeIBioSequence(0)
nextCounter := goutils.AtomicCounter()
batchiter.Add(concurrent_readers)

2
pkg/obiformats/batch_reader_type.go
View File

@@ -2,4 +2,4 @@ package obiformats
import "git.metabarcoding.org/lecasofts/go/obitools/pkg/obiiter"
type IBatchReader func(string, ...WithOption) (obiiter.IBioSequenceBatch, error)
type IBatchReader func(string, ...WithOption) (obiiter.IBioSequence, error)

7
pkg/obiformats/dispatcher.go
View File

@@ -2,15 +2,16 @@ package obiformats
import (
"fmt"
log "github.com/sirupsen/logrus"
"sync"
log "github.com/sirupsen/logrus"
"git.metabarcoding.org/lecasofts/go/obitools/pkg/obiiter"
)
type SequenceBatchWriterToFile func(iterator obiiter.IBioSequenceBatch,
type SequenceBatchWriterToFile func(iterator obiiter.IBioSequence,
filename string,
options ...WithOption) (obiiter.IBioSequenceBatch, error)
options ...WithOption) (obiiter.IBioSequence, error)
func WriterDispatcher(prototypename string,
dispatcher obiiter.IDistribute,

8
pkg/obiformats/ecopcr_read.go
View File

@@ -120,7 +120,7 @@ func __read_ecopcr_bioseq__(file *__ecopcr_file__) (*obiseq.BioSequence, error)
return bseq, nil
}
func ReadEcoPCR(reader io.Reader, options ...WithOption) obiiter.IBioSequenceBatch {
func ReadEcoPCR(reader io.Reader, options ...WithOption) obiiter.IBioSequence {
tag := make([]byte, 11)
n, _ := reader.Read(tag)
@@ -166,7 +166,7 @@ func ReadEcoPCR(reader io.Reader, options ...WithOption) obiiter.IBioSequenceBat
opt := MakeOptions(options)
newIter := obiiter.MakeIBioSequenceBatch(opt.BufferSize())
newIter := obiiter.MakeIBioSequence(opt.BufferSize())
newIter.Add(1)
go func() {
@@ -208,7 +208,7 @@ func ReadEcoPCR(reader io.Reader, options ...WithOption) obiiter.IBioSequenceBat
return newIter
}
func ReadEcoPCRBatchFromFile(filename string, options ...WithOption) (obiiter.IBioSequenceBatch, error) {
func ReadEcoPCRBatchFromFile(filename string, options ...WithOption) (obiiter.IBioSequence, error) {
var reader io.Reader
var greader io.Reader
var err error
@@ -216,7 +216,7 @@ func ReadEcoPCRBatchFromFile(filename string, options ...WithOption) (obiiter.IB
reader, err = os.Open(filename)
if err != nil {
log.Printf("open file error: %+v", err)
return obiiter.NilIBioSequenceBatch, err
return obiiter.NilIBioSequence, err
}
// Test if the flux is compressed by gzip

10
pkg/obiformats/embl_read.go
View File

@@ -84,7 +84,7 @@ func _EndOfLastEntry(buff []byte) int {
return -1
}
func _ParseEmblFile(input <-chan _FileChunk, out obiiter.IBioSequenceBatch) {
func _ParseEmblFile(input <-chan _FileChunk, out obiiter.IBioSequence) {
for chunks := range input {
scanner := bufio.NewScanner(chunks.raw)
@@ -201,11 +201,11 @@ func _ReadFlatFileChunk(reader io.Reader, readers chan _FileChunk) {
// 6 5 43 2 1
//
// <CR>?<LF>//<CR>?<LF>
func ReadEMBL(reader io.Reader, options ...WithOption) obiiter.IBioSequenceBatch {
func ReadEMBL(reader io.Reader, options ...WithOption) obiiter.IBioSequence {
opt := MakeOptions(options)
entry_channel := make(chan _FileChunk, opt.BufferSize())
newIter := obiiter.MakeIBioSequenceBatch(opt.BufferSize())
newIter := obiiter.MakeIBioSequence(opt.BufferSize())
nworkers := opt.ParallelWorkers()
newIter.Add(nworkers)
@@ -224,7 +224,7 @@ func ReadEMBL(reader io.Reader, options ...WithOption) obiiter.IBioSequenceBatch
return newIter
}
func ReadEMBLFromFile(filename string, options ...WithOption) (obiiter.IBioSequenceBatch, error) {
func ReadEMBLFromFile(filename string, options ...WithOption) (obiiter.IBioSequence, error) {
var reader io.Reader
var greader io.Reader
var err error
@@ -232,7 +232,7 @@ func ReadEMBLFromFile(filename string, options ...WithOption) (obiiter.IBioSeque
reader, err = os.Open(filename)
if err != nil {
log.Printf("open file error: %+v", err)
return obiiter.NilIBioSequenceBatch, err
return obiiter.NilIBioSequence, err
}
// Test if the flux is compressed by gzip

4
pkg/obiformats/fastseq_header.go
View File

@@ -15,8 +15,8 @@ func ParseGuessedFastSeqHeader(sequence *obiseq.BioSequence) {
}
}
func IParseFastSeqHeaderBatch(iterator obiiter.IBioSequenceBatch,
options ...WithOption) obiiter.IBioSequenceBatch {
func IParseFastSeqHeaderBatch(iterator obiiter.IBioSequence,
options ...WithOption) obiiter.IBioSequence {
opt := MakeOptions(options)
return iterator.MakeIWorker(obiiter.AnnotatorToSeqWorker(opt.ParseFastSeqHeader()),
opt.ParallelWorkers(),

13
pkg/obiformats/fastseq_read.go
View File

@@ -19,7 +19,7 @@ import (
)
func _FastseqReader(seqfile C.fast_kseq_p,
iterator obiiter.IBioSequenceBatch,
iterator obiiter.IBioSequence,
batch_size int) {
var comment string
i := 0
@@ -72,7 +72,6 @@ func _FastseqReader(seqfile C.fast_kseq_p,
ii = 0
}
}
if len(slice) > 0 {
iterator.Push(obiiter.MakeBioSequenceBatch(i, slice))
@@ -81,7 +80,7 @@ func _FastseqReader(seqfile C.fast_kseq_p,
}
func ReadFastSeqFromFile(filename string, options ...WithOption) (obiiter.IBioSequenceBatch, error) {
func ReadFastSeqFromFile(filename string, options ...WithOption) (obiiter.IBioSequence, error) {
opt := MakeOptions(options)
name := C.CString(filename)
@@ -94,7 +93,7 @@ func ReadFastSeqFromFile(filename string, options ...WithOption) (obiiter.IBioSe
if pointer == nil {
err = fmt.Errorf("cannot open file %s", filename)
return obiiter.NilIBioSequenceBatch, err
return obiiter.NilIBioSequence, err
}
size := int64(-1)
@@ -106,7 +105,7 @@ func ReadFastSeqFromFile(filename string, options ...WithOption) (obiiter.IBioSe
size = -1
}
newIter := obiiter.MakeIBioSequenceBatch(opt.BufferSize())
newIter := obiiter.MakeIBioSequence(opt.BufferSize())
newIter.Add(1)
go func() {
@@ -126,9 +125,9 @@ func ReadFastSeqFromFile(filename string, options ...WithOption) (obiiter.IBioSe
return newIter, err
}
func ReadFastSeqFromStdin(options ...WithOption) obiiter.IBioSequenceBatch {
func ReadFastSeqFromStdin(options ...WithOption) obiiter.IBioSequence {
opt := MakeOptions(options)
newIter := obiiter.MakeIBioSequenceBatch(opt.BufferSize())
newIter := obiiter.MakeIBioSequence(opt.BufferSize())
newIter.Add(1)

18
pkg/obiformats/fastseq_write_fasta.go
View File

@@ -61,13 +61,13 @@ func FormatFastaBatch(batch obiiter.BioSequenceBatch, formater FormatHeader) []b
return bs.Bytes()
}
func WriteFasta(iterator obiiter.IBioSequenceBatch,
func WriteFasta(iterator obiiter.IBioSequence,
file io.Writer,
options ...WithOption) (obiiter.IBioSequenceBatch, error) {
options ...WithOption) (obiiter.IBioSequence, error) {
opt := MakeOptions(options)
buffsize := iterator.BufferSize()
newIter := obiiter.MakeIBioSequenceBatch(buffsize)
newIter := obiiter.MakeIBioSequence(buffsize)
nwriters := opt.ParallelWorkers()
@@ -83,7 +83,7 @@ func WriteFasta(iterator obiiter.IBioSequenceBatch,
log.Debugln("End of the fasta file writing")
}()
ff := func(iterator obiiter.IBioSequenceBatch) {
ff := func(iterator obiiter.IBioSequence) {
for iterator.Next() {
batch := iterator.Get()
@@ -136,21 +136,21 @@ func WriteFasta(iterator obiiter.IBioSequenceBatch,
return newIter, nil
}
func WriteFastaToStdout(iterator obiiter.IBioSequenceBatch,
options ...WithOption) (obiiter.IBioSequenceBatch, error) {
func WriteFastaToStdout(iterator obiiter.IBioSequence,
options ...WithOption) (obiiter.IBioSequence, error) {
options = append(options, OptionDontCloseFile())
return WriteFasta(iterator, os.Stdout, options...)
}
func WriteFastaToFile(iterator obiiter.IBioSequenceBatch,
func WriteFastaToFile(iterator obiiter.IBioSequence,
filename string,
options ...WithOption) (obiiter.IBioSequenceBatch, error) {
options ...WithOption) (obiiter.IBioSequence, error) {
file, err := os.Create(filename)
if err != nil {
log.Fatalf("open file error: %v", err)
return obiiter.NilIBioSequenceBatch, err
return obiiter.NilIBioSequence, err
}
options = append(options, OptionCloseFile())

18
pkg/obiformats/fastseq_write_fastq.go
View File

@@ -51,13 +51,13 @@ type FileChunck struct {
order int
}
func WriteFastq(iterator obiiter.IBioSequenceBatch,
func WriteFastq(iterator obiiter.IBioSequence,
file io.Writer,
options ...WithOption) (obiiter.IBioSequenceBatch, error) {
options ...WithOption) (obiiter.IBioSequence, error) {
opt := MakeOptions(options)
buffsize := iterator.BufferSize()
newIter := obiiter.MakeIBioSequenceBatch(buffsize)
newIter := obiiter.MakeIBioSequence(buffsize)
nwriters := opt.ParallelWorkers()
@@ -77,7 +77,7 @@ func WriteFastq(iterator obiiter.IBioSequenceBatch,
log.Debugln("End of the fastq file writing")
}()
ff := func(iterator obiiter.IBioSequenceBatch) {
ff := func(iterator obiiter.IBioSequence) {
for iterator.Next() {
batch := iterator.Get()
chunk := FileChunck{
@@ -129,21 +129,21 @@ func WriteFastq(iterator obiiter.IBioSequenceBatch,
return newIter, nil
}
func WriteFastqToStdout(iterator obiiter.IBioSequenceBatch,
options ...WithOption) (obiiter.IBioSequenceBatch, error) {
func WriteFastqToStdout(iterator obiiter.IBioSequence,
options ...WithOption) (obiiter.IBioSequence, error) {
options = append(options, OptionDontCloseFile())
return WriteFastq(iterator, os.Stdout, options...)
}
func WriteFastqToFile(iterator obiiter.IBioSequenceBatch,
func WriteFastqToFile(iterator obiiter.IBioSequence,
filename string,
options ...WithOption) (obiiter.IBioSequenceBatch, error) {
options ...WithOption) (obiiter.IBioSequence, error) {
file, err := os.Create(filename)
if err != nil {
log.Fatalf("open file error: %v", err)
return obiiter.NilIBioSequenceBatch, err
return obiiter.NilIBioSequence, err
}
options = append(options, OptionCloseFile())

10
pkg/obiformats/genbank_read.go
View File

@@ -26,7 +26,7 @@ const (
inSequence gbstate = 4
)
func _ParseGenbankFile(input <-chan _FileChunk, out obiiter.IBioSequenceBatch) {
func _ParseGenbankFile(input <-chan _FileChunk, out obiiter.IBioSequence) {
state := inHeader
@@ -107,11 +107,11 @@ func _ParseGenbankFile(input <-chan _FileChunk, out obiiter.IBioSequenceBatch) {
}
func ReadGenbank(reader io.Reader, options ...WithOption) obiiter.IBioSequenceBatch {
func ReadGenbank(reader io.Reader, options ...WithOption) obiiter.IBioSequence {
opt := MakeOptions(options)
entry_channel := make(chan _FileChunk, opt.BufferSize())
newIter := obiiter.MakeIBioSequenceBatch(opt.BufferSize())
newIter := obiiter.MakeIBioSequence(opt.BufferSize())
nworkers := opt.ParallelWorkers()
newIter.Add(nworkers)
@@ -130,7 +130,7 @@ func ReadGenbank(reader io.Reader, options ...WithOption) obiiter.IBioSequenceBa
return newIter
}
func ReadGenbankFromFile(filename string, options ...WithOption) (obiiter.IBioSequenceBatch, error) {
func ReadGenbankFromFile(filename string, options ...WithOption) (obiiter.IBioSequence, error) {
var reader io.Reader
var greader io.Reader
var err error
@@ -138,7 +138,7 @@ func ReadGenbankFromFile(filename string, options ...WithOption) (obiiter.IBioSe
reader, err = os.Open(filename)
if err != nil {
log.Printf("open file error: %+v", err)
return obiiter.NilIBioSequenceBatch, err
return obiiter.NilIBioSequence, err
}
// Test if the flux is compressed by gzip

8
pkg/obiformats/universal_read.go
View File

@@ -43,7 +43,7 @@ func GuessSeqFileType(firstline string) string {
}
func ReadSequencesFromFile(filename string,
options ...WithOption) (obiiter.IBioSequenceBatch, error) {
options ...WithOption) (obiiter.IBioSequence, error) {
var file *os.File
var reader io.Reader
var greader io.Reader
@@ -53,7 +53,7 @@ func ReadSequencesFromFile(filename string,
if err != nil {
log.Fatalf("open file error: %v", err)
return obiiter.NilIBioSequenceBatch, err
return obiiter.NilIBioSequence, err
}
reader = file
@@ -72,7 +72,7 @@ func ReadSequencesFromFile(filename string,
tag, _ := breader.Peek(30)
if len(tag) < 30 {
newIter := obiiter.MakeIBioSequenceBatch()
newIter := obiiter.MakeIBioSequence()
newIter.Close()
return newIter, nil
}
@@ -98,5 +98,5 @@ func ReadSequencesFromFile(filename string,
filename, filetype)
}
return obiiter.NilIBioSequenceBatch, nil
return obiiter.NilIBioSequence, nil
}

18
pkg/obiformats/universal_write.go
View File

@@ -10,9 +10,9 @@ import (
"git.metabarcoding.org/lecasofts/go/obitools/pkg/obiiter"
)
func WriteSequence(iterator obiiter.IBioSequenceBatch,
func WriteSequence(iterator obiiter.IBioSequence,
file io.Writer,
options ...WithOption) (obiiter.IBioSequenceBatch, error) {
options ...WithOption) (obiiter.IBioSequence, error) {
iterator = iterator.Rebatch(1000)
@@ -22,7 +22,7 @@ func WriteSequence(iterator obiiter.IBioSequenceBatch,
batch := iterator.Get()
iterator.PushBack()
var newIter obiiter.IBioSequenceBatch
var newIter obiiter.IBioSequence
var err error
if len(batch.Slice()) > 0 {
@@ -42,24 +42,24 @@ func WriteSequence(iterator obiiter.IBioSequenceBatch,
return iterator, nil
}
return obiiter.NilIBioSequenceBatch, fmt.Errorf("input iterator not ready")
return obiiter.NilIBioSequence, fmt.Errorf("input iterator not ready")
}
func WriteSequencesToStdout(iterator obiiter.IBioSequenceBatch,
options ...WithOption) (obiiter.IBioSequenceBatch, error) {
func WriteSequencesToStdout(iterator obiiter.IBioSequence,
options ...WithOption) (obiiter.IBioSequence, error) {
options = append(options, OptionDontCloseFile())
return WriteSequence(iterator, os.Stdout, options...)
}
func WriteSequencesToFile(iterator obiiter.IBioSequenceBatch,
func WriteSequencesToFile(iterator obiiter.IBioSequence,
filename string,
options ...WithOption) (obiiter.IBioSequenceBatch, error) {
options ...WithOption) (obiiter.IBioSequence, error) {
file, err := os.Create(filename)
if err != nil {
log.Fatalf("open file error: %v", err)
return obiiter.NilIBioSequenceBatch, err
return obiiter.NilIBioSequence, err
}
options = append(options, OptionCloseFile())