diff --git a/go.mod b/go.mod index c56db47..e545222 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/PaesslerAG/gval v1.2.2 github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df github.com/chen3feng/stl4go v0.1.1 - github.com/goccy/go-json v0.10.2 + github.com/goccy/go-json v0.10.3 github.com/klauspost/pgzip v1.2.6 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 github.com/rrethy/ahocorasick v1.0.0 @@ -23,12 +23,19 @@ require ( ) require ( + github.com/bytedance/sonic v1.11.9 // indirect + github.com/bytedance/sonic/loader v0.1.1 // indirect + github.com/cloudwego/base64x v0.1.4 // indirect + github.com/cloudwego/iasm v0.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/goombaio/orderedmap v0.0.0-20180924084748-ba921b7e2419 // indirect + github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/kr/pretty v0.3.0 // indirect github.com/kr/text v0.2.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.6.1 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect ) require ( diff --git a/go.sum b/go.sum index 6091bb7..6c2eae6 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,16 @@ github.com/PaesslerAG/jsonpath v0.1.0 h1:gADYeifvlqK3R3i2cR5B4DGgxLXIPb3TRTH1mGi github.com/PaesslerAG/jsonpath v0.1.0/go.mod h1:4BzmtoM/PI8fPO4aQGIusjGxGir2BzcV0grWtFzq1Y8= github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df h1:GSoSVRLoBaFpOOds6QyY1L8AX7uoY+Ln3BHc22W40X0= github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df/go.mod h1:hiVxq5OP2bUGBRNS3Z/bt/reCLFNbdcST6gISi1fiOM= +github.com/bytedance/sonic v1.11.9 h1:LFHENlIY/SLzDWverzdOvgMztTxcfcF+cqNsz9pK5zg= +github.com/bytedance/sonic v1.11.9/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= +github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/chen3feng/stl4go v0.1.1 h1:0L1+mDw7pomftKDruM23f1mA7miavOj6C6MZeadzN2Q= github.com/chen3feng/stl4go v0.1.1/go.mod h1:5ml3psLgETJjRJnMbPE+JiHLrCpt+Ajc2weeTECXzWU= +github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= +github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -19,6 +27,8 @@ github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uq github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= +github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/goombaio/orderedmap v0.0.0-20180924084748-ba921b7e2419 h1:SajEQ6tktpF9SRIuzbiPOX9AEZZ53Bvw0k9Mzrts8Lg= github.com/goombaio/orderedmap v0.0.0-20180924084748-ba921b7e2419/go.mod h1:YKu81H3RSd1cFh0d7NhvUoTtUC9IY/vBX0WUQb1/o4Y= github.com/goombaio/orderedset v0.0.0-20180925151225-8e67b20a9b77 h1:4dvq1tGHn1Y9KSRY0OZ24Khki4+4U+ZrA//YYsdUlJU= @@ -27,9 +37,13 @@ github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1 github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= @@ -62,17 +76,26 @@ github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFR github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/tevino/abool/v2 v2.1.0 h1:7w+Vf9f/5gmKT4m4qkayb33/92M+Um45F2BkHOR+L/c= github.com/tevino/abool/v2 v2.1.0/go.mod h1:+Lmlqk6bHDWHqN1cbxqhwEAwMPXgc8I1SDEamtseuXY= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= @@ -95,6 +118,8 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= scientificgo.org/special v0.0.0 h1:P6WJkECo6tgtvZAEfNXl+KEB9ReAatjKAeX8U07mjSc= scientificgo.org/special v0.0.0/go.mod h1:LoGVh9tS431RLTJo7gFlYDKFWq44cEb7QqL+M0EKtZU= scientificgo.org/testutil v0.0.0 h1:y356DHRo0tAz9zIFmxlhZoKDlHPHaWW/DCm9k3PhIMA= diff --git a/pkg/obialign/backtracking.go b/pkg/obialign/backtracking.go index 9ab913f..d67b239 100644 --- a/pkg/obialign/backtracking.go +++ b/pkg/obialign/backtracking.go @@ -1,12 +1,19 @@ package obialign -import "slices" +import ( + log "github.com/sirupsen/logrus" + "slices" +) func _Backtracking(pathMatrix []int, lseqA, lseqB int, path *[]int) []int { needed := (lseqA + lseqB) * 2 (*path) = (*path)[:0] + cp := cap(*path) (*path) = slices.Grow((*path), needed) + if cp < cap(*path) { + log.Infof("Resized path from %d to %d\n", cp, cap(*path)) + } p := cap(*path) *path = (*path)[:p] @@ -88,7 +95,5 @@ func _Backtracking(pathMatrix []int, lseqA, lseqB int, path *[]int) []int { (*path)[p] = 0 } - *path = (*path)[p:cap((*path))] - - return *path + return (*path)[p:cap((*path))] } diff --git a/pkg/obialign/dnamatrix.go b/pkg/obialign/dnamatrix.go index fdb982c..9ebf17e 100644 --- a/pkg/obialign/dnamatrix.go +++ b/pkg/obialign/dnamatrix.go @@ -2,6 +2,9 @@ package obialign import ( "math" + "sync" + + log "github.com/sirupsen/logrus" ) var _FourBitsCount = []float64{ @@ -28,6 +31,7 @@ var _InitializedDnaScore = false var _NucPartMatch [32][32]float64 var _NucScorePartMatchMatch [100][100]int var _NucScorePartMatchMismatch [100][100]int +var _InitDNAScoreMatrixMutex = &sync.Mutex{} // _MatchRatio calculates the match ratio between two bytes. // @@ -120,7 +124,11 @@ func _InitNucScorePartMatch() { } func _InitDNAScoreMatrix() { + _InitDNAScoreMatrixMutex.Lock() + defer _InitDNAScoreMatrixMutex.Unlock() if !_InitializedDnaScore { + log.Info("Initializing the DNA Scoring matrix") + _InitNucPartMatch() _InitNucScorePartMatch() _InitializedDnaScore = true diff --git a/pkg/obialign/pairedendalign.go b/pkg/obialign/pairedendalign.go index 6817aac..c03dae3 100644 --- a/pkg/obialign/pairedendalign.go +++ b/pkg/obialign/pairedendalign.go @@ -1,8 +1,6 @@ package obialign import ( - log "github.com/sirupsen/logrus" - "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obikmer" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq" ) @@ -319,7 +317,6 @@ func PELeftAlign(seqA, seqB *obiseq.BioSequence, gap, scale float64, arena PEAlignArena) (int, []int) { if !_InitializedDnaScore { - log.Println("Initializing the DNA Scoring matrix") _InitDNAScoreMatrix() } @@ -332,18 +329,17 @@ func PELeftAlign(seqA, seqB *obiseq.BioSequence, gap, scale float64, &arena.pointer.scoreMatrix, &arena.pointer.pathMatrix) - arena.pointer.path = _Backtracking(arena.pointer.pathMatrix, + path := _Backtracking(arena.pointer.pathMatrix, seqA.Len(), seqB.Len(), &arena.pointer.path) - return score, arena.pointer.path + return score, path } func PERightAlign(seqA, seqB *obiseq.BioSequence, gap, scale float64, arena PEAlignArena) (int, []int) { if !_InitializedDnaScore { - log.Println("Initializing the DNA Scoring matrix") _InitDNAScoreMatrix() } @@ -356,11 +352,11 @@ func PERightAlign(seqA, seqB *obiseq.BioSequence, gap, scale float64, &arena.pointer.scoreMatrix, &arena.pointer.pathMatrix) - arena.pointer.path = _Backtracking(arena.pointer.pathMatrix, + path := _Backtracking(arena.pointer.pathMatrix, seqA.Len(), seqB.Len(), &arena.pointer.path) - return score, arena.pointer.path + return score, path } func PEAlign(seqA, seqB *obiseq.BioSequence, @@ -373,8 +369,9 @@ func PEAlign(seqA, seqB *obiseq.BioSequence, var rawSeqB, qualSeqB []byte var extra5, extra3 int + var path []int + if !_InitializedDnaScore { - log.Println("Initializing the DNA Scoring matrix") _InitDNAScoreMatrix() } @@ -443,7 +440,7 @@ func PEAlign(seqA, seqB *obiseq.BioSequence, &arena.pointer.pathMatrix) } - arena.pointer.path = _Backtracking(arena.pointer.pathMatrix, + path = _Backtracking(arena.pointer.pathMatrix, len(rawSeqA), len(rawSeqB), &arena.pointer.path) @@ -474,15 +471,16 @@ func PEAlign(seqA, seqB *obiseq.BioSequence, qualB := qualSeqB[i] score += _NucScorePartMatchMatch[qualA][qualB] } - arena.pointer.path = arena.pointer.path[:0] - arena.pointer.path = append(arena.pointer.path, 0, partLen) + + path = arena.pointer.path[:0] + path = append(path, 0, partLen) } - arena.pointer.path[0] += extra5 - if arena.pointer.path[len(arena.pointer.path)-1] == 0 { - arena.pointer.path[len(arena.pointer.path)-2] += extra3 + path[0] += extra5 + if path[len(path)-1] == 0 { + path[len(path)-2] += extra3 } else { - arena.pointer.path = append(arena.pointer.path, extra3, 0) + path = append(path, extra3, 0) } } else { // @@ -499,9 +497,9 @@ func PEAlign(seqA, seqB *obiseq.BioSequence, &arena.pointer.scoreMatrix, &arena.pointer.pathMatrix) - arena.pointer.path = _Backtracking(arena.pointer.pathMatrix, + path = _Backtracking(arena.pointer.pathMatrix, len(rawSeqA), len(rawSeqB), - &arena.pointer.path) + &(arena.pointer.path)) scoreL := _FillMatrixPeLeftAlign( rawSeqA, qualSeqA, rawSeqB, qualSeqB, gap, scale, @@ -509,12 +507,12 @@ func PEAlign(seqA, seqB *obiseq.BioSequence, &arena.pointer.pathMatrix) if scoreL > scoreR { - arena.pointer.path = _Backtracking(arena.pointer.pathMatrix, + path = _Backtracking(arena.pointer.pathMatrix, len(rawSeqA), len(rawSeqB), - &arena.pointer.path) + &(arena.pointer.path)) } } - return score, arena.pointer.path, fastCount, over, fastScore + return score, path, fastCount, over, fastScore } diff --git a/pkg/obiformats/dispatcher.go b/pkg/obiformats/dispatcher.go index de4c051..0aedf8a 100644 --- a/pkg/obiformats/dispatcher.go +++ b/pkg/obiformats/dispatcher.go @@ -1,13 +1,14 @@ package obiformats import ( - "encoding/json" "fmt" "os" "path/filepath" "strings" "sync" + "github.com/goccy/go-json" + log "github.com/sirupsen/logrus" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter" diff --git a/pkg/obiformats/json_writer.go b/pkg/obiformats/json_writer.go index fac57fa..d9576a8 100644 --- a/pkg/obiformats/json_writer.go +++ b/pkg/obiformats/json_writer.go @@ -3,7 +3,7 @@ package obiformats import ( "bufio" "bytes" - "encoding/json" + "github.com/goccy/go-json" "io" "os" "strconv" diff --git a/pkg/obioptions/version.go b/pkg/obioptions/version.go index b4a591e..b04e191 100644 --- a/pkg/obioptions/version.go +++ b/pkg/obioptions/version.go @@ -7,7 +7,7 @@ import ( // TODO: The version number is extracted from git. This induces that the version // corresponds to the last commit, and not the one when the file will be // commited -var _Commit = "fbdb2af" +var _Commit = "c94e79b" var _Version = "Release 4.2.0" // Version returns the version of the obitools package. diff --git a/pkg/obiutils/goutils.go b/pkg/obiutils/goutils.go index 50dbd1e..01aade5 100644 --- a/pkg/obiutils/goutils.go +++ b/pkg/obiutils/goutils.go @@ -3,12 +3,13 @@ package obiutils import ( "bufio" "bytes" - "encoding/json" "io" "os" "reflect" "sync" + "github.com/bytedance/sonic" + "github.com/barkimedes/go-deepcopy" ) @@ -363,9 +364,12 @@ func AtomicCounter(initial ...int) func() int { return nextCounter } +// JsonMarshalByteBuffer marshals an interface into JSON format. +// +// It takes a bytes.Buffer as a buffer and an interface{} as i. +// Returns an error. func JsonMarshalByteBuffer(buffer *bytes.Buffer, i interface{}) error { - encoder := json.NewEncoder(buffer) - encoder.SetEscapeHTML(false) + encoder := sonic.ConfigDefault.NewEncoder(buffer) err := encoder.Encode(i) b := buffer.Bytes() b = bytes.TrimRight(b, "\n")