obitools/obitools4
2
0
Fork 0
mirror of https://github.com/metabarcoding/obitools4.git synced 2025-12-08 08:40:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Optimize memory for readers and writers

Browse Source
This commit is contained in:
Eric Coissac
2024-08-05 10:48:28 +02:00
parent f83032e643
commit 886b5d9a96
11 changed files with 48 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/obiformats/embl_read.go
View File

@@ -187,7 +187,7 @@ func _ParseEmblFile(
func ReadEMBL(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, error) {
opt := MakeOptions(options)
buff := make([]byte, 1024*1024*1024*256)
buff := make([]byte, 1024*1024*512)
entry_channel := ReadSeqFileChunk(
opt.Source(),

4
pkg/obiformats/fastaseq_read.go
View File

@@ -232,7 +232,7 @@ func ReadFasta(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, e
nworker := opt.ParallelWorkers()
buff := make([]byte, 1024*1024*1024)
buff := make([]byte, 1024*1024)
chkchan := ReadSeqFileChunk(
opt.Source(),
@@ -250,7 +250,7 @@ func ReadFasta(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, e
out.WaitAndClose()
}()
newIter := out.SortBatches().Rebatch(opt.BatchSize())
newIter := out.SortBatches()
log.Debugln("Full file batch mode : ", opt.FullFileBatch())

4
pkg/obiformats/fastqseq_read.go
View File

@@ -310,7 +310,7 @@ func ReadFastq(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, e
nworker := opt.ParallelWorkers()
buff := make([]byte, 1024*1024*1024)
buff := make([]byte, 1024*1024)
chkchan := ReadSeqFileChunk(
opt.Source(),
@@ -332,7 +332,7 @@ func ReadFastq(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, e
out.WaitAndClose()
}()
newIter := out.SortBatches().Rebatch(opt.BatchSize())
newIter := out.SortBatches()
log.Debugln("Full file batch mode : ", opt.FullFileBatch())

3
pkg/obiformats/fastseq_write_fasta.go
View File

@@ -126,7 +126,6 @@ func WriteFasta(iterator obiiter.IBioSequence,
options ...WithOption) (obiiter.IBioSequence, error) {
opt := MakeOptions(options)
iterator = iterator.Rebatch(opt.BatchSize())
file, _ = obiutils.CompressStream(file, opt.CompressedFile(), opt.CloseFile())
newIter := obiiter.MakeIBioSequence()
@@ -142,7 +141,7 @@ func WriteFasta(iterator obiiter.IBioSequence,
go func() {
newIter.WaitAndClose()
close(chunkchan)
log.Warnf("Writing fasta file done")
log.Debugf("Writing fasta file done")
}()
ff := func(iterator obiiter.IBioSequence) {

2
pkg/obiformats/fastseq_write_fastq.go
View File

@@ -97,7 +97,7 @@ func WriteFastq(iterator obiiter.IBioSequence,
options ...WithOption) (obiiter.IBioSequence, error) {
opt := MakeOptions(options)
iterator = iterator.Rebatch(opt.BatchSize())
iterator = iterator
file, _ = obiutils.CompressStream(file, opt.CompressedFile(), opt.CloseFile())

2
pkg/obiformats/genbank_read.go
View File

@@ -223,7 +223,7 @@ func ReadGenbank(reader io.Reader, options ...WithOption) (obiiter.IBioSequence,
opt := MakeOptions(options)
// entry_channel := make(chan _FileChunk)
buff := make([]byte, 1024*1024*1024*256)
buff := make([]byte, 1024*1024*512)
entry_channel := ReadSeqFileChunk(
opt.Source(),

2
pkg/obiformats/seqfile_chunk_read.go
View File

@@ -46,6 +46,8 @@ func ReadSeqFileChunk(
chunk_channel := make(ChannelSeqFileChunk)
_FileChunkSize := len(buff)
go func() {
size := 0
l := 0

2
pkg/obiformats/seqfile_chunk_write.go
View File

@@ -44,7 +44,7 @@ func WriteSeqFileChunk(
}
obiiter.UnregisterPipe()
log.Warnf("The writer has been closed")
log.Debugf("The writer has been closed")
}()
return chunk_channel