obitools/obitools4
2
0
Fork 0
mirror of https://github.com/metabarcoding/obitools4.git synced 2025-06-29 16:20:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add some code refactoring from the blackboard branch

Browse Source
This commit is contained in:
Eric Coissac
2024-08-02 12:35:46 +02:00
parent bc1aaaf7d9
commit 1b1cd41fd3
38 changed files with 491 additions and 330 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
pkg/obichunk/chunk_on_disk.go
View File

@ -73,11 +73,11 @@ func ISequenceChunkOnDisk(iterator obiiter.IBioSequence,
panic(err)
}
chunck := iseq.Load()
source, chunk := iseq.Load()
newIter.Push(obiiter.MakeBioSequenceBatch(order, chunck))
newIter.Push(obiiter.MakeBioSequenceBatch(source, order, chunk))
log.Infof("Start processing of batch %d/%d : %d sequences",
order, nbatch, len(chunck))
order, nbatch, len(chunk))
}

13
pkg/obichunk/chunks.go
View File

@ -28,6 +28,7 @@ func ISequenceChunk(iterator obiiter.IBioSequence,
jobDone := sync.WaitGroup{}
chunks := make(map[int]*obiseq.BioSequenceSlice, 1000)
sources := make(map[int]string, 1000)
for newflux := range dispatcher.News() {
jobDone.Add(1)
@ -43,12 +44,18 @@ func ISequenceChunk(iterator obiiter.IBioSequence,
chunks[newflux] = chunk
lock.Unlock()
source := ""
for data.Next() {
b := data.Get()
source = b.Source()
*chunk = append(*chunk, b.Slice()...)
b.Recycle(false)
}
lock.Lock()
sources[newflux] = source
lock.Unlock()
jobDone.Done()
}(newflux)
}
@ -56,10 +63,10 @@ func ISequenceChunk(iterator obiiter.IBioSequence,
jobDone.Wait()
order := 0
for _, chunck := range chunks {
for i, chunk := range chunks {
if len(*chunck) > 0 {
newIter.Push(obiiter.MakeBioSequenceBatch(order, *chunck))
if len(*chunk) > 0 {
newIter.Push(obiiter.MakeBioSequenceBatch(sources[i], order, *chunk))
order++
}

6
pkg/obichunk/subchunks.go
View File

@ -90,7 +90,7 @@ func ISequenceSubChunk(iterator obiiter.IBioSequence,
for iterator.Next() {
batch := iterator.Get()
source := batch.Source()
if batch.Len() > 1 {
classifier.Reset()
@ -117,7 +117,7 @@ func ISequenceSubChunk(iterator obiiter.IBioSequence,
ss := obiseq.MakeBioSequenceSlice()
for i, v := range ordered {
if v.code != last {
newIter.Push(obiiter.MakeBioSequenceBatch(nextOrder(), ss))
newIter.Push(obiiter.MakeBioSequenceBatch(source, nextOrder(), ss))
ss = obiseq.MakeBioSequenceSlice()
last = v.code
}
@ -127,7 +127,7 @@ func ISequenceSubChunk(iterator obiiter.IBioSequence,
}
if len(ss) > 0 {
newIter.Push(obiiter.MakeBioSequenceBatch(nextOrder(), ss))
newIter.Push(obiiter.MakeBioSequenceBatch(source, nextOrder(), ss))
}
} else {
newIter.Push(batch.Reorder(nextOrder()))