mirror of
https://github.com/metabarcoding/obitools4.git
synced 2025-06-29 16:20:46 +00:00
Muspelling chunck -> chunk
This commit is contained in:
@ -73,11 +73,11 @@ func ISequenceChunkOnDisk(iterator obiiter.IBioSequence,
|
|||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
source, chunck := iseq.Load()
|
source, chunk := iseq.Load()
|
||||||
|
|
||||||
newIter.Push(obiiter.MakeBioSequenceBatch(source, order, chunck))
|
newIter.Push(obiiter.MakeBioSequenceBatch(source, order, chunk))
|
||||||
log.Infof("Start processing of batch %d/%d : %d sequences",
|
log.Infof("Start processing of batch %d/%d : %d sequences",
|
||||||
order, nbatch, len(chunck))
|
order, nbatch, len(chunk))
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -63,10 +63,10 @@ func ISequenceChunk(iterator obiiter.IBioSequence,
|
|||||||
jobDone.Wait()
|
jobDone.Wait()
|
||||||
order := 0
|
order := 0
|
||||||
|
|
||||||
for i, chunck := range chunks {
|
for i, chunk := range chunks {
|
||||||
|
|
||||||
if len(*chunck) > 0 {
|
if len(*chunk) > 0 {
|
||||||
newIter.Push(obiiter.MakeBioSequenceBatch(sources[i], order, *chunck))
|
newIter.Push(obiiter.MakeBioSequenceBatch(sources[i], order, *chunk))
|
||||||
order++
|
order++
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -142,7 +142,7 @@ func WriteCSV(iterator obiiter.IBioSequence,
|
|||||||
nwriters := opt.ParallelWorkers()
|
nwriters := opt.ParallelWorkers()
|
||||||
|
|
||||||
obiiter.RegisterAPipe()
|
obiiter.RegisterAPipe()
|
||||||
chunkchan := make(chan FileChunck)
|
chunkchan := make(chan FileChunk)
|
||||||
|
|
||||||
newIter.Add(nwriters)
|
newIter.Add(nwriters)
|
||||||
var waitWriter sync.WaitGroup
|
var waitWriter sync.WaitGroup
|
||||||
@ -161,7 +161,7 @@ func WriteCSV(iterator obiiter.IBioSequence,
|
|||||||
|
|
||||||
batch := iterator.Get()
|
batch := iterator.Get()
|
||||||
|
|
||||||
chunkchan <- FileChunck{
|
chunkchan <- FileChunk{
|
||||||
FormatCVSBatch(batch, opt),
|
FormatCVSBatch(batch, opt),
|
||||||
batch.Order(),
|
batch.Order(),
|
||||||
}
|
}
|
||||||
@ -171,7 +171,7 @@ func WriteCSV(iterator obiiter.IBioSequence,
|
|||||||
}
|
}
|
||||||
|
|
||||||
next_to_send := 0
|
next_to_send := 0
|
||||||
received := make(map[int]FileChunck, 100)
|
received := make(map[int]FileChunk, 100)
|
||||||
|
|
||||||
waitWriter.Add(1)
|
waitWriter.Add(1)
|
||||||
go func() {
|
go func() {
|
||||||
|
@ -136,7 +136,7 @@ func WriteFasta(iterator obiiter.IBioSequence,
|
|||||||
nwriters := opt.ParallelWorkers()
|
nwriters := opt.ParallelWorkers()
|
||||||
|
|
||||||
obiiter.RegisterAPipe()
|
obiiter.RegisterAPipe()
|
||||||
chunkchan := make(chan FileChunck)
|
chunkchan := make(chan FileChunk)
|
||||||
|
|
||||||
header_format := opt.FormatFastSeqHeader()
|
header_format := opt.FormatFastSeqHeader()
|
||||||
|
|
||||||
@ -159,7 +159,7 @@ func WriteFasta(iterator obiiter.IBioSequence,
|
|||||||
|
|
||||||
log.Debugf("Formating fasta chunk %d", batch.Order())
|
log.Debugf("Formating fasta chunk %d", batch.Order())
|
||||||
|
|
||||||
chunkchan <- FileChunck{
|
chunkchan <- FileChunk{
|
||||||
FormatFastaBatch(batch, header_format, opt.SkipEmptySequence()),
|
FormatFastaBatch(batch, header_format, opt.SkipEmptySequence()),
|
||||||
batch.Order(),
|
batch.Order(),
|
||||||
}
|
}
|
||||||
@ -177,7 +177,7 @@ func WriteFasta(iterator obiiter.IBioSequence,
|
|||||||
}
|
}
|
||||||
|
|
||||||
next_to_send := 0
|
next_to_send := 0
|
||||||
received := make(map[int]FileChunck, 100)
|
received := make(map[int]FileChunk, 100)
|
||||||
|
|
||||||
waitWriter.Add(1)
|
waitWriter.Add(1)
|
||||||
go func() {
|
go func() {
|
||||||
|
@ -87,7 +87,7 @@ func FormatFastqBatch(batch obiiter.BioSequenceBatch,
|
|||||||
return chunk
|
return chunk
|
||||||
}
|
}
|
||||||
|
|
||||||
type FileChunck struct {
|
type FileChunk struct {
|
||||||
text []byte
|
text []byte
|
||||||
order int
|
order int
|
||||||
}
|
}
|
||||||
@ -106,7 +106,7 @@ func WriteFastq(iterator obiiter.IBioSequence,
|
|||||||
nwriters := opt.ParallelWorkers()
|
nwriters := opt.ParallelWorkers()
|
||||||
|
|
||||||
obiiter.RegisterAPipe()
|
obiiter.RegisterAPipe()
|
||||||
chunkchan := make(chan FileChunck)
|
chunkchan := make(chan FileChunk)
|
||||||
|
|
||||||
header_format := opt.FormatFastSeqHeader()
|
header_format := opt.FormatFastSeqHeader()
|
||||||
|
|
||||||
@ -126,7 +126,7 @@ func WriteFastq(iterator obiiter.IBioSequence,
|
|||||||
ff := func(iterator obiiter.IBioSequence) {
|
ff := func(iterator obiiter.IBioSequence) {
|
||||||
for iterator.Next() {
|
for iterator.Next() {
|
||||||
batch := iterator.Get()
|
batch := iterator.Get()
|
||||||
chunk := FileChunck{
|
chunk := FileChunk{
|
||||||
FormatFastqBatch(batch, header_format, opt.SkipEmptySequence()),
|
FormatFastqBatch(batch, header_format, opt.SkipEmptySequence()),
|
||||||
batch.Order(),
|
batch.Order(),
|
||||||
}
|
}
|
||||||
@ -143,7 +143,7 @@ func WriteFastq(iterator obiiter.IBioSequence,
|
|||||||
}
|
}
|
||||||
|
|
||||||
next_to_send := 0
|
next_to_send := 0
|
||||||
received := make(map[int]FileChunck, 100)
|
received := make(map[int]FileChunk, 100)
|
||||||
|
|
||||||
waitWriter.Add(1)
|
waitWriter.Add(1)
|
||||||
go func() {
|
go func() {
|
||||||
|
@ -3,7 +3,6 @@ package obiformats
|
|||||||
import (
|
import (
|
||||||
"bufio"
|
"bufio"
|
||||||
"bytes"
|
"bytes"
|
||||||
"github.com/goccy/go-json"
|
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"strconv"
|
"strconv"
|
||||||
@ -11,6 +10,8 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/goccy/go-json"
|
||||||
|
|
||||||
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
|
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
|
||||||
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
|
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
|
||||||
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
|
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
|
||||||
@ -87,7 +88,7 @@ func WriteJSON(iterator obiiter.IBioSequence,
|
|||||||
nwriters := opt.ParallelWorkers()
|
nwriters := opt.ParallelWorkers()
|
||||||
|
|
||||||
obiiter.RegisterAPipe()
|
obiiter.RegisterAPipe()
|
||||||
chunkchan := make(chan FileChunck)
|
chunkchan := make(chan FileChunk)
|
||||||
|
|
||||||
newIter.Add(nwriters)
|
newIter.Add(nwriters)
|
||||||
var waitWriter sync.WaitGroup
|
var waitWriter sync.WaitGroup
|
||||||
@ -106,7 +107,7 @@ func WriteJSON(iterator obiiter.IBioSequence,
|
|||||||
|
|
||||||
batch := iterator.Get()
|
batch := iterator.Get()
|
||||||
|
|
||||||
chunkchan <- FileChunck{
|
chunkchan <- FileChunk{
|
||||||
FormatJSONBatch(batch),
|
FormatJSONBatch(batch),
|
||||||
batch.Order(),
|
batch.Order(),
|
||||||
}
|
}
|
||||||
@ -116,7 +117,7 @@ func WriteJSON(iterator obiiter.IBioSequence,
|
|||||||
}
|
}
|
||||||
|
|
||||||
next_to_send := 0
|
next_to_send := 0
|
||||||
received := make(map[int]FileChunck, 100)
|
received := make(map[int]FileChunk, 100)
|
||||||
|
|
||||||
waitWriter.Add(1)
|
waitWriter.Add(1)
|
||||||
go func() {
|
go func() {
|
||||||
|
@ -102,7 +102,7 @@ func ReadSeqFileChunk(
|
|||||||
buff = fullbuff[0:lremain]
|
buff = fullbuff[0:lremain]
|
||||||
lcp := copy(buff, fullbuff[pnext:])
|
lcp := copy(buff, fullbuff[pnext:])
|
||||||
if lcp < lremain {
|
if lcp < lremain {
|
||||||
log.Fatalf("Error copying remaining data of chunck %d : %d < %d", i, lcp, lremain)
|
log.Fatalf("Error copying remaining data of chunk %d : %d < %d", i, lcp, lremain)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
buff = buff[:0]
|
buff = buff[:0]
|
@ -692,7 +692,7 @@ func (iterator IBioSequence) FilterAnd(predicate obiseq.SequencePredicate,
|
|||||||
// a large obiseq.BioSequenceSlice.
|
// a large obiseq.BioSequenceSlice.
|
||||||
func (iterator IBioSequence) Load() (string, obiseq.BioSequenceSlice) {
|
func (iterator IBioSequence) Load() (string, obiseq.BioSequenceSlice) {
|
||||||
|
|
||||||
chunck := obiseq.MakeBioSequenceSlice()
|
chunk := obiseq.MakeBioSequenceSlice()
|
||||||
source := ""
|
source := ""
|
||||||
|
|
||||||
for iterator.Next() {
|
for iterator.Next() {
|
||||||
@ -701,11 +701,11 @@ func (iterator IBioSequence) Load() (string, obiseq.BioSequenceSlice) {
|
|||||||
source = b.Source()
|
source = b.Source()
|
||||||
}
|
}
|
||||||
log.Debugf("append %d sequences", b.Len())
|
log.Debugf("append %d sequences", b.Len())
|
||||||
chunck = append(chunck, b.Slice()...)
|
chunk = append(chunk, b.Slice()...)
|
||||||
b.Recycle(false)
|
b.Recycle(false)
|
||||||
}
|
}
|
||||||
|
|
||||||
return source, chunck
|
return source, chunk
|
||||||
}
|
}
|
||||||
|
|
||||||
// CompleteFileIterator generates a new iterator for reading a complete file.
|
// CompleteFileIterator generates a new iterator for reading a complete file.
|
||||||
|
@ -52,7 +52,7 @@ func CLIPCR(iterator obiiter.IBioSequence) (obiiter.IBioSequence, error) {
|
|||||||
100,
|
100,
|
||||||
obioptions.CLIParallelWorkers(),
|
obioptions.CLIParallelWorkers(),
|
||||||
)
|
)
|
||||||
log.Infof("Fragmenting sequence longer than %dbp into chuncks of %dbp",
|
log.Infof("Fragmenting sequence longer than %dbp into chunks of %dbp",
|
||||||
CLIMaxLength()*1000,
|
CLIMaxLength()*1000,
|
||||||
CLIMaxLength()*100,
|
CLIMaxLength()*100,
|
||||||
)
|
)
|
||||||
|
Reference in New Issue
Block a user