obitools/obitools4
2
0
Fork 0
mirror of https://github.com/metabarcoding/obitools4.git synced 2025-06-29 16:20:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

First functional version of a blackboard and a blackboard based obicount

Browse Source
This commit is contained in:
Eric Coissac
2024-07-31 23:16:45 +02:00
parent dfe2fc3d43
commit f36b39bfa3
40 changed files with 1236 additions and 257 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
pkg/obichunk/chunk_on_disk.go
View File

@ -73,9 +73,9 @@ func ISequenceChunkOnDisk(iterator obiiter.IBioSequence,
panic(err)
}
chunck := iseq.Load()
source, chunck := iseq.Load()
newIter.Push(obiiter.MakeBioSequenceBatch(order, chunck))
newIter.Push(obiiter.MakeBioSequenceBatch(source, order, chunck))
log.Infof("Start processing of batch %d/%d : %d sequences",
order, nbatch, len(chunck))

11
pkg/obichunk/chunks.go
View File

@ -28,6 +28,7 @@ func ISequenceChunk(iterator obiiter.IBioSequence,
jobDone := sync.WaitGroup{}
chunks := make(map[int]*obiseq.BioSequenceSlice, 1000)
sources := make(map[int]string, 1000)
for newflux := range dispatcher.News() {
jobDone.Add(1)
@ -43,12 +44,18 @@ func ISequenceChunk(iterator obiiter.IBioSequence,
chunks[newflux] = chunk
lock.Unlock()
source := ""
for data.Next() {
b := data.Get()
source = b.Source()
*chunk = append(*chunk, b.Slice()...)
b.Recycle(false)
}
lock.Lock()
sources[newflux] = source
lock.Unlock()
jobDone.Done()
}(newflux)
}
@ -56,10 +63,10 @@ func ISequenceChunk(iterator obiiter.IBioSequence,
jobDone.Wait()
order := 0
for _, chunck := range chunks {
for i, chunck := range chunks {
if len(*chunck) > 0 {
newIter.Push(obiiter.MakeBioSequenceBatch(order, *chunck))
newIter.Push(obiiter.MakeBioSequenceBatch(sources[i], order, *chunck))
order++
}

6
pkg/obichunk/subchunks.go
View File

@ -90,7 +90,7 @@ func ISequenceSubChunk(iterator obiiter.IBioSequence,
for iterator.Next() {
batch := iterator.Get()
source := batch.Source()
if batch.Len() > 1 {
classifier.Reset()
@ -117,7 +117,7 @@ func ISequenceSubChunk(iterator obiiter.IBioSequence,
ss := obiseq.MakeBioSequenceSlice()
for i, v := range ordered {
if v.code != last {
newIter.Push(obiiter.MakeBioSequenceBatch(nextOrder(), ss))
newIter.Push(obiiter.MakeBioSequenceBatch(source, nextOrder(), ss))
ss = obiseq.MakeBioSequenceSlice()
last = v.code
}
@ -127,7 +127,7 @@ func ISequenceSubChunk(iterator obiiter.IBioSequence,
}
if len(ss) > 0 {
newIter.Push(obiiter.MakeBioSequenceBatch(nextOrder(), ss))
newIter.Push(obiiter.MakeBioSequenceBatch(source, nextOrder(), ss))
}
} else {
newIter.Push(batch.Reorder(nextOrder()))