Debug fasta and fastq writer when the first sequence is hudge

Former-commit-id: d208ff838abb7e19e117067f6243298492d60f14

pkg/obiformats/fastseq_write_fasta.go

@@ -80,15 +80,24 @@ func FormatFastaBatch(batch obiiter.BioSequenceBatch, formater FormatHeader, ski
 	// Create a buffer to store the formatted sequences
 	var bs bytes.Buffer
 
+	lt := 0
+
+	for _, seq := range batch.Slice() {
+		lt += seq.Len()
+	}
+
 	// Iterate over each sequence in the batch
-	for i, seq := range batch.Slice() {
+	log.Debugf("FormatFastaBatch: #%d : %d seqs", batch.Order(), batch.Len())
+	first := true
+	for _, seq := range batch.Slice() {
 		// Check if the sequence is empty
 		if seq.Len() > 0 {
 			// Format the sequence using the provided formater function
 			formattedSeq := FormatFasta(seq, formater)
 
-			if i == 0 {
-				bs.Grow(len(formattedSeq) * len(batch.Slice()) * 5 / 4)
+			if first {
+				bs.Grow(lt + (len(formattedSeq)-seq.Len())*batch.Len()*5/4)
+				first = false
 			}
 
 			// Append the formatted sequence to the buffer
@@ -148,10 +157,14 @@ func WriteFasta(iterator obiiter.IBioSequence,
 
 			batch := iterator.Get()
 
+			log.Debugf("Formating fasta chunk %d", batch.Order())
+
 			chunkchan <- FileChunck{
 				FormatFastaBatch(batch, header_format, opt.SkipEmptySequence()),
 				batch.Order(),
 			}
+			log.Debugf("Fasta chunk %d formated", batch.Order())
+
 			newIter.Push(batch)
 		}
 		newIter.Done()
@@ -171,15 +184,18 @@ func WriteFasta(iterator obiiter.IBioSequence,
 		for chunk := range chunkchan {
 			if chunk.order == next_to_send {
 				file.Write(chunk.text)
+				log.Debugf("Fasta chunk %d written", chunk.order)
 				next_to_send++
 				chunk, ok := received[next_to_send]
 				for ok {
 					file.Write(chunk.text)
+					log.Debugf("Fasta chunk %d written", chunk.order)
 					delete(received, next_to_send)
 					next_to_send++
 					chunk, ok = received[next_to_send]
 				}
 			} else {
+				log.Debugf("Store Fasta chunk %d", chunk.order)
 				received[chunk.order] = chunk
 			}
 

pkg/obiformats/fastseq_write_fastq.go

@@ -52,14 +52,24 @@ func FormatFastqBatch(batch obiiter.BioSequenceBatch,
 	formater FormatHeader, skipEmpty bool) []byte {
 	var bs bytes.Buffer
 
-	for i, seq := range batch.Slice() {
+	lt := 0
+
+	for _, seq := range batch.Slice() {
+		lt += seq.Len()
+	}
+
+	// Iterate over each sequence in the batch
+	first := true
+
+	for _, seq := range batch.Slice() {
 		if seq.Len() > 0 {
 			_formatFastq(&bs, seq, formater)
 
-			if i == 0 {
-
-				bs.Grow(len(bs.Bytes()) * len(batch.Slice()) * 5 / 4)
+			if first {
+				bs.Grow(lt + (len(bs.Bytes())-seq.Len())*batch.Len()*5/4)
+				first = false
 			}
+
 		} else {
 			if skipEmpty {
 				log.Warnf("Sequence %s is empty and skiped in output", seq.Id())

pkg/obiformats/genbank_read.go

@@ -146,7 +146,7 @@ func _ParseGenbankFile(source string,
 						log.Warn("Empty id when parsing genbank file")
 					}
 
-					log.Debugf("End of sequence %s: %dbp ", id, seqBytes.Len())
+					// log.Debugf("End of sequence %s: %dbp ", id, seqBytes.Len())
 
 					sequence := obiseq.NewBioSequence(id,
 						seqBytes.Bytes(),
@@ -168,8 +168,9 @@ func _ParseGenbankFile(source string,
 					sumlength += sequence.Len()
 
 					if len(sequences) == batch_size || sumlength > total_seq_size {
-						log.Debugln("Pushing sequences")
-						out.Push(obiiter.MakeBioSequenceBatch(chunck_order(), sequences))
+						oo := chunck_order()
+						log.Debugln("Pushing sequence batch ", oo, " with ", len(sequences), " sequences")
+						out.Push(obiiter.MakeBioSequenceBatch(oo, sequences))
 						sequences = make(obiseq.BioSequenceSlice, 0, 100)
 						sumlength = 0
 					}
@@ -218,13 +219,14 @@ func _ParseGenbankFile(source string,
 
 		}
 
-		log.Debugf("End of chunk %d : %s", chunks.order, line)
 		if len(sequences) > 0 {
-			log.Debugln("Pushing sequences")
-			out.Push(obiiter.MakeBioSequenceBatch(chunck_order(), sequences))
+			oo := chunck_order()
+			log.Debugln("Pushing sequence batch ", oo, " with ", len(sequences), " sequences")
+			out.Push(obiiter.MakeBioSequenceBatch(oo, sequences))
 		}
 	}
 
+	log.Debug("End of the Genbank thread")
 	out.Done()
 
 }
@@ -255,6 +257,7 @@ func ReadGenbank(reader io.Reader, options ...WithOption) obiiter.IBioSequence {
 
 	go func() {
 		newIter.WaitAndClose()
+		log.Debug("End of the genbank file ", opt.Source())
 	}()
 
 	if opt.FullFileBatch() {