Muspelling chunck -> chunk

This commit is contained in:
Eric Coissac
2024-08-01 16:43:23 +02:00
parent 882cc82f23
commit 776b8f75b7
9 changed files with 26 additions and 25 deletions

View File

@ -73,11 +73,11 @@ func ISequenceChunkOnDisk(iterator obiiter.IBioSequence,
panic(err) panic(err)
} }
source, chunck := iseq.Load() source, chunk := iseq.Load()
newIter.Push(obiiter.MakeBioSequenceBatch(source, order, chunck)) newIter.Push(obiiter.MakeBioSequenceBatch(source, order, chunk))
log.Infof("Start processing of batch %d/%d : %d sequences", log.Infof("Start processing of batch %d/%d : %d sequences",
order, nbatch, len(chunck)) order, nbatch, len(chunk))
} }

View File

@ -63,10 +63,10 @@ func ISequenceChunk(iterator obiiter.IBioSequence,
jobDone.Wait() jobDone.Wait()
order := 0 order := 0
for i, chunck := range chunks { for i, chunk := range chunks {
if len(*chunck) > 0 { if len(*chunk) > 0 {
newIter.Push(obiiter.MakeBioSequenceBatch(sources[i], order, *chunck)) newIter.Push(obiiter.MakeBioSequenceBatch(sources[i], order, *chunk))
order++ order++
} }

View File

@ -142,7 +142,7 @@ func WriteCSV(iterator obiiter.IBioSequence,
nwriters := opt.ParallelWorkers() nwriters := opt.ParallelWorkers()
obiiter.RegisterAPipe() obiiter.RegisterAPipe()
chunkchan := make(chan FileChunck) chunkchan := make(chan FileChunk)
newIter.Add(nwriters) newIter.Add(nwriters)
var waitWriter sync.WaitGroup var waitWriter sync.WaitGroup
@ -161,7 +161,7 @@ func WriteCSV(iterator obiiter.IBioSequence,
batch := iterator.Get() batch := iterator.Get()
chunkchan <- FileChunck{ chunkchan <- FileChunk{
FormatCVSBatch(batch, opt), FormatCVSBatch(batch, opt),
batch.Order(), batch.Order(),
} }
@ -171,7 +171,7 @@ func WriteCSV(iterator obiiter.IBioSequence,
} }
next_to_send := 0 next_to_send := 0
received := make(map[int]FileChunck, 100) received := make(map[int]FileChunk, 100)
waitWriter.Add(1) waitWriter.Add(1)
go func() { go func() {

View File

@ -136,7 +136,7 @@ func WriteFasta(iterator obiiter.IBioSequence,
nwriters := opt.ParallelWorkers() nwriters := opt.ParallelWorkers()
obiiter.RegisterAPipe() obiiter.RegisterAPipe()
chunkchan := make(chan FileChunck) chunkchan := make(chan FileChunk)
header_format := opt.FormatFastSeqHeader() header_format := opt.FormatFastSeqHeader()
@ -159,7 +159,7 @@ func WriteFasta(iterator obiiter.IBioSequence,
log.Debugf("Formating fasta chunk %d", batch.Order()) log.Debugf("Formating fasta chunk %d", batch.Order())
chunkchan <- FileChunck{ chunkchan <- FileChunk{
FormatFastaBatch(batch, header_format, opt.SkipEmptySequence()), FormatFastaBatch(batch, header_format, opt.SkipEmptySequence()),
batch.Order(), batch.Order(),
} }
@ -177,7 +177,7 @@ func WriteFasta(iterator obiiter.IBioSequence,
} }
next_to_send := 0 next_to_send := 0
received := make(map[int]FileChunck, 100) received := make(map[int]FileChunk, 100)
waitWriter.Add(1) waitWriter.Add(1)
go func() { go func() {

View File

@ -87,7 +87,7 @@ func FormatFastqBatch(batch obiiter.BioSequenceBatch,
return chunk return chunk
} }
type FileChunck struct { type FileChunk struct {
text []byte text []byte
order int order int
} }
@ -106,7 +106,7 @@ func WriteFastq(iterator obiiter.IBioSequence,
nwriters := opt.ParallelWorkers() nwriters := opt.ParallelWorkers()
obiiter.RegisterAPipe() obiiter.RegisterAPipe()
chunkchan := make(chan FileChunck) chunkchan := make(chan FileChunk)
header_format := opt.FormatFastSeqHeader() header_format := opt.FormatFastSeqHeader()
@ -126,7 +126,7 @@ func WriteFastq(iterator obiiter.IBioSequence,
ff := func(iterator obiiter.IBioSequence) { ff := func(iterator obiiter.IBioSequence) {
for iterator.Next() { for iterator.Next() {
batch := iterator.Get() batch := iterator.Get()
chunk := FileChunck{ chunk := FileChunk{
FormatFastqBatch(batch, header_format, opt.SkipEmptySequence()), FormatFastqBatch(batch, header_format, opt.SkipEmptySequence()),
batch.Order(), batch.Order(),
} }
@ -143,7 +143,7 @@ func WriteFastq(iterator obiiter.IBioSequence,
} }
next_to_send := 0 next_to_send := 0
received := make(map[int]FileChunck, 100) received := make(map[int]FileChunk, 100)
waitWriter.Add(1) waitWriter.Add(1)
go func() { go func() {

View File

@ -3,7 +3,6 @@ package obiformats
import ( import (
"bufio" "bufio"
"bytes" "bytes"
"github.com/goccy/go-json"
"io" "io"
"os" "os"
"strconv" "strconv"
@ -11,6 +10,8 @@ import (
"sync" "sync"
"time" "time"
"github.com/goccy/go-json"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
@ -87,7 +88,7 @@ func WriteJSON(iterator obiiter.IBioSequence,
nwriters := opt.ParallelWorkers() nwriters := opt.ParallelWorkers()
obiiter.RegisterAPipe() obiiter.RegisterAPipe()
chunkchan := make(chan FileChunck) chunkchan := make(chan FileChunk)
newIter.Add(nwriters) newIter.Add(nwriters)
var waitWriter sync.WaitGroup var waitWriter sync.WaitGroup
@ -106,7 +107,7 @@ func WriteJSON(iterator obiiter.IBioSequence,
batch := iterator.Get() batch := iterator.Get()
chunkchan <- FileChunck{ chunkchan <- FileChunk{
FormatJSONBatch(batch), FormatJSONBatch(batch),
batch.Order(), batch.Order(),
} }
@ -116,7 +117,7 @@ func WriteJSON(iterator obiiter.IBioSequence,
} }
next_to_send := 0 next_to_send := 0
received := make(map[int]FileChunck, 100) received := make(map[int]FileChunk, 100)
waitWriter.Add(1) waitWriter.Add(1)
go func() { go func() {

View File

@ -102,7 +102,7 @@ func ReadSeqFileChunk(
buff = fullbuff[0:lremain] buff = fullbuff[0:lremain]
lcp := copy(buff, fullbuff[pnext:]) lcp := copy(buff, fullbuff[pnext:])
if lcp < lremain { if lcp < lremain {
log.Fatalf("Error copying remaining data of chunck %d : %d < %d", i, lcp, lremain) log.Fatalf("Error copying remaining data of chunk %d : %d < %d", i, lcp, lremain)
} }
} else { } else {
buff = buff[:0] buff = buff[:0]

View File

@ -692,7 +692,7 @@ func (iterator IBioSequence) FilterAnd(predicate obiseq.SequencePredicate,
// a large obiseq.BioSequenceSlice. // a large obiseq.BioSequenceSlice.
func (iterator IBioSequence) Load() (string, obiseq.BioSequenceSlice) { func (iterator IBioSequence) Load() (string, obiseq.BioSequenceSlice) {
chunck := obiseq.MakeBioSequenceSlice() chunk := obiseq.MakeBioSequenceSlice()
source := "" source := ""
for iterator.Next() { for iterator.Next() {
@ -701,11 +701,11 @@ func (iterator IBioSequence) Load() (string, obiseq.BioSequenceSlice) {
source = b.Source() source = b.Source()
} }
log.Debugf("append %d sequences", b.Len()) log.Debugf("append %d sequences", b.Len())
chunck = append(chunck, b.Slice()...) chunk = append(chunk, b.Slice()...)
b.Recycle(false) b.Recycle(false)
} }
return source, chunck return source, chunk
} }
// CompleteFileIterator generates a new iterator for reading a complete file. // CompleteFileIterator generates a new iterator for reading a complete file.

View File

@ -52,7 +52,7 @@ func CLIPCR(iterator obiiter.IBioSequence) (obiiter.IBioSequence, error) {
100, 100,
obioptions.CLIParallelWorkers(), obioptions.CLIParallelWorkers(),
) )
log.Infof("Fragmenting sequence longer than %dbp into chuncks of %dbp", log.Infof("Fragmenting sequence longer than %dbp into chunks of %dbp",
CLIMaxLength()*1000, CLIMaxLength()*1000,
CLIMaxLength()*100, CLIMaxLength()*100,
) )