mirror of
https://github.com/metabarcoding/obitools4.git
synced 2026-03-25 13:30:52 +00:00
Add memory-based batching support
Implement memory-aware batch sizing with --batch-mem CLI option, enabling adaptive batching based on estimated sequence memory footprint. Key changes: - Added _BatchMem and related getters/setters in pkg/obidefault - Implemented RebatchBySize() in pkg/obiter for memory-constrained batching - Added BioSequence.MemorySize() for conservative memory estimation - Integrated batch-mem option in pkg/obioptions with human-readable size parsing (e.g., 128K, 64M, 1G) - Added obiutils.ParseMemSize/FormatMemSize for unit conversion - Enhanced pool GC in pkg/obiseq/pool.go to trigger explicit GC for large slice discards - Updated sequence_reader.go to apply memory-based rebatching when enabled
This commit is contained in:
@@ -24,3 +24,33 @@ func BatchSize() int {
|
||||
func BatchSizePtr() *int {
|
||||
return &_BatchSize
|
||||
}
|
||||
|
||||
// _BatchMem holds the maximum cumulative memory (in bytes) per batch when
|
||||
// memory-based batching is requested. A value of 0 disables memory-based
|
||||
// batching and falls back to count-based batching.
|
||||
var _BatchMem = 0
|
||||
var _BatchMemStr = ""
|
||||
|
||||
// SetBatchMem sets the memory budget per batch in bytes.
|
||||
func SetBatchMem(n int) {
|
||||
_BatchMem = n
|
||||
}
|
||||
|
||||
// BatchMem returns the current memory budget per batch in bytes.
|
||||
// A value of 0 means memory-based batching is disabled.
|
||||
func BatchMem() int {
|
||||
return _BatchMem
|
||||
}
|
||||
|
||||
func BatchMemPtr() *int {
|
||||
return &_BatchMem
|
||||
}
|
||||
|
||||
// BatchMemStr returns the raw --batch-mem string value as provided on the CLI.
|
||||
func BatchMemStr() string {
|
||||
return _BatchMemStr
|
||||
}
|
||||
|
||||
func BatchMemStrPtr() *string {
|
||||
return &_BatchMemStr
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user