obitools/obitools4
2
0
Fork 0
mirror of https://github.com/metabarcoding/obitools4.git synced 2025-06-29 16:20:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

reduce the memory impact of obiuniq.

Browse Source
This commit is contained in:
Eric Coissac
2024-11-27 13:30:16 +01:00
parent d29a56dcbf
commit 40fb4e9767
9 changed files with 145 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
pkg/obiformats/dispatcher.go
View File

@ -14,10 +14,40 @@ import (
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
)
// SequenceBatchWriterToFile is a function type that defines a method for writing
// a batch of biosequences to a specified file. It takes an iterator of biosequences,
// a filename, and optional configuration options, and returns an iterator of biosequences
// along with any error encountered during the writing process.
//
// Parameters:
// - iterator: An iterator of biosequences to be written to the file.
// - filename: The name of the file where the sequences will be written.
// - options: Optional configuration options for the writing process.
//
// Returns:
// An iterator of biosequences that may have been modified during the writing process
// and an error if the writing operation fails.
type SequenceBatchWriterToFile func(iterator obiiter.IBioSequence,
filename string,
options ...WithOption) (obiiter.IBioSequence, error)
// WriterDispatcher manages the writing of data to files based on a given
// prototype name and a dispatcher for distributing the sequences. It
// processes incoming data from the dispatcher in separate goroutines,
// formatting and writing the data to files as specified.
//
// Parameters:
// - prototypename: A string that serves as a template for naming the output files.
// - dispatcher: An instance of IDistribute that provides the data to be written
// and manages the distribution of sequences.
// - formater: A function of type SequenceBatchWriterToFile that formats and writes
// the sequences to the specified file.
// - options: Optional configuration options for the writing process.
//
// The function operates asynchronously, launching goroutines for each new data
// channel received from the dispatcher. It ensures that directories are created
// as needed and handles errors during the writing process. The function blocks
// until all writing jobs are completed.
func WriterDispatcher(prototypename string,
dispatcher obiiter.IDistribute,
formater SequenceBatchWriterToFile,
@ -34,7 +64,7 @@ func WriterDispatcher(prototypename string,
data, err := dispatcher.Outputs(newflux)
if err != nil {
log.Fatalf("Cannot retreive the new chanel : %v", err)
log.Fatalf("Cannot retrieve the new channel: %v", err)
}
key := dispatcher.Classifier().Value(newflux)
@ -58,7 +88,7 @@ func WriterDispatcher(prototypename string,
info, err := os.Stat(directory)
switch {
case !os.IsNotExist(err) && !info.IsDir():
log.Fatalf("Cannot Create the directory %s", directory)
log.Fatalf("Cannot create the directory %s", directory)
case os.IsNotExist(err):
os.Mkdir(directory, 0755)
}
@ -71,7 +101,7 @@ func WriterDispatcher(prototypename string,
options...)
if err != nil {
log.Fatalf("cannot open the output file for key %s",
log.Fatalf("Cannot open the output file for key %s",
dispatcher.Classifier().Value(newflux))
}