mirror of
https://github.com/metabarcoding/obitools4.git
synced 2026-11-09 09:24:41 +00:00
Refactor similarity engine and implement parallel clustering
Introduces parallel processing for clustering, refines scoring mechanisms, and expands test coverage for the alignment functionality. This change introduces parallel processing capabilities within the clustering logic, updates the core similarity engine with new data structures and scoring methods, and implements a comprehensive test suite for the alignment tool. It also adds new command-line options for configuring alignment parameters and reference lengths.
This commit is contained in:
Eric Coissac committed
1 parent
49225030ca
commit
e511650f5b
61 files changed
+13397
-801
No files matched your search
@@ -0,0 +1,46 @@
|
||||
package obisimilarity
|
||||
|
||||
// A block stores the 4-mer tables of a few sequences, transposed: the counter
|
||||
// of the word w for the lane j is data[w*width+j]. A vector instruction then
|
||||
// handles the same word for all the sequences of the block, and only the words
|
||||
// that a sequence contains (a hundred for a short amplicon, out of 256) need
|
||||
// to be visited to compare it with all the sequences of the block.
|
||||
type tableBlock struct {
|
||||
data []uint8
|
||||
}
|
||||
|
||||
func newTableBlock(width int) tableBlock {
|
||||
return tableBlock{data: make([]uint8, 256*width)}
|
||||
}
|
||||
|
||||
// set stores the table t as the lane `lane` of the block.
|
||||
func (b *tableBlock) set(width, lane int, t *Table4) {
|
||||
for w := range t.Count {
|
||||
b.data[w*width+lane] = t.Count[w]
|
||||
}
|
||||
}
|
||||
|
||||
// common computes the sums of the minima of the counters of the sequence
|
||||
// given by its words with every lane of the block (see blockCommon).
|
||||
func (b *tableBlock) common(width int, words []uint16, out []uint8) {
|
||||
blockCommon(b.data, width, words, out)
|
||||
}
|
||||
|
||||
// SparseWords lists the words of a table as w<<8|count, so that the visit of
|
||||
// a block only goes through the words present in the sequence. nil is
|
||||
// returned when the counters add up to more than 255, which would overflow the
|
||||
// 8-bit sums of the blocks.
|
||||
func SparseWords(t *Table4) []uint16 {
|
||||
words := make([]uint16, 0, 128)
|
||||
mass := 0
|
||||
for w, c := range t.Count {
|
||||
if c > 0 {
|
||||
words = append(words, uint16(w)<<8|uint16(c))
|
||||
mass += int(c)
|
||||
}
|
||||
}
|
||||
if mass > 255 {
|
||||
return nil
|
||||
}
|
||||
return words
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
//go:build !goexperiment.simd
|
||||
|
||||
package obisimilarity
|
||||
|
||||
// simdBlockWidth is 0 without SIMD: the blocks are not used.
|
||||
func simdBlockWidth() int { return 0 }
|
||||
|
||||
func blockCommon(data []uint8, width int, words []uint16, out []uint8) {
|
||||
panic("obisimilarity: blocks of tables require GOEXPERIMENT=simd")
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
//go:build goexperiment.simd
|
||||
|
||||
package obisimilarity
|
||||
|
||||
import "simd"
|
||||
|
||||
// simdBlockWidth is the number of sequences in a block: one per lane of a
|
||||
// vector of bytes.
|
||||
func simdBlockWidth() int {
|
||||
var probe simd.Uint8s
|
||||
return min(probe.Len(), 64)
|
||||
}
|
||||
|
||||
// blockCommon computes, for the sequence whose words are given, the sum of
|
||||
// the minima of the counters with every lane of the block. It is the value of
|
||||
// commonWords without the overflow term, as the sums fit in 8 bits (see
|
||||
// SparseWords). out must hold at least `width` bytes.
|
||||
func blockCommon(data []uint8, width int, words []uint16, out []uint8) {
|
||||
acc := simd.BroadcastUint8s(0)
|
||||
for _, e := range words {
|
||||
w := simd.LoadUint8s(data[int(e>>8)*width:])
|
||||
acc = acc.Add(w.Min(simd.BroadcastUint8s(uint8(e))))
|
||||
}
|
||||
acc.Store(out)
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package obisimilarity
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
// Entry is a distinct sequence of a dataset, with what the comparisons need.
|
||||
type Entry struct {
|
||||
Seq []byte
|
||||
Length int
|
||||
Count int // count of the sequence (cumulated for identical sequences)
|
||||
Table *Table4
|
||||
Sparse []uint16 // words of the table (see SparseWords), nil if unusable
|
||||
// Ambiguous sequences contain IUPAC letters other than a, c, g and t: they
|
||||
// have no 4-mer table, and the filter does not apply to them.
|
||||
Ambiguous bool
|
||||
}
|
||||
|
||||
const (
|
||||
notNucleotide = iota
|
||||
plainNucleotide
|
||||
ambiguousNucleotide
|
||||
)
|
||||
|
||||
var nucleotideClass = func() (t [256]uint8) {
|
||||
for _, c := range "acgt" {
|
||||
t[c] = plainNucleotide
|
||||
}
|
||||
for _, c := range "umrwsykvhdbn" {
|
||||
t[c] = ambiguousNucleotide
|
||||
}
|
||||
return t
|
||||
}()
|
||||
|
||||
// Classify tells if a sequence (in lower case) can be compared. It returns
|
||||
// ok=false for an empty sequence and for a sequence with anything else than
|
||||
// IUPAC letters; IUPAC letters other than a, c, g and t are only accepted if
|
||||
// acceptAmbiguous is set, and then ambiguous is true.
|
||||
func Classify(s []byte, acceptAmbiguous bool) (ok, ambiguous bool) {
|
||||
if len(s) == 0 {
|
||||
return false, false
|
||||
}
|
||||
for _, c := range s {
|
||||
switch nucleotideClass[c] {
|
||||
case notNucleotide:
|
||||
return false, false
|
||||
case ambiguousNucleotide:
|
||||
if !acceptAmbiguous {
|
||||
return false, false
|
||||
}
|
||||
ambiguous = true
|
||||
}
|
||||
}
|
||||
return true, ambiguous
|
||||
}
|
||||
|
||||
// BuildEntries describes n sequences, their 4-mer tables being computed in
|
||||
// parallel. get returns the sequence i, its count, and whether it is
|
||||
// ambiguous. The tables are stored contiguously.
|
||||
func BuildEntries(n int, get func(i int) (seq []byte, count int, ambiguous bool), workers int) []Entry {
|
||||
entries := make([]Entry, n)
|
||||
tables := make([]Table4, n)
|
||||
|
||||
var next atomic.Int64
|
||||
var wg sync.WaitGroup
|
||||
for w := 0; w < max(workers, 1); w++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for {
|
||||
i := int(next.Add(1)) - 1
|
||||
if i >= n {
|
||||
return
|
||||
}
|
||||
seq, count, ambiguous := get(i)
|
||||
var sparse []uint16
|
||||
if !ambiguous {
|
||||
tables[i].Fill(seq)
|
||||
sparse = SparseWords(&tables[i])
|
||||
}
|
||||
entries[i] = Entry{
|
||||
Seq: seq,
|
||||
Length: len(seq),
|
||||
Count: count,
|
||||
Table: &tables[i],
|
||||
Sparse: sparse,
|
||||
Ambiguous: ambiguous,
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
return entries
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package obisimilarity
|
||||
|
||||
import "sort"
|
||||
|
||||
// Member is a copy of the entry of a sequence in an Index, with its own table.
|
||||
type Member struct {
|
||||
Entry
|
||||
ID int // identifier given by the caller
|
||||
Rank int // rank in the order of insertion
|
||||
}
|
||||
|
||||
const tabChunk = 1024
|
||||
|
||||
// A Bucket holds the members of one length, in the order of insertion, and
|
||||
// their tables by blocks (see tableBlock).
|
||||
type Bucket struct {
|
||||
Members []Member
|
||||
blocks []tableBlock
|
||||
width int
|
||||
}
|
||||
|
||||
// Commons caches the result of the filter on one block of a bucket.
|
||||
type Commons struct {
|
||||
v [64]uint8
|
||||
blk int
|
||||
}
|
||||
|
||||
// Reset must be called when changing of bucket.
|
||||
func (c *Commons) Reset() { c.blk = -1 }
|
||||
|
||||
// Common returns the sum of the minima of the 4-mer counters of the sequence
|
||||
// whose Sparse words are given and of the member k of the bucket (without the
|
||||
// overflow term), or -1 if it is not available, in which case PairScore
|
||||
// computes it.
|
||||
func (b *Bucket) Common(k int, sparse []uint16, c *Commons) int {
|
||||
if b.width == 0 || sparse == nil {
|
||||
return -1
|
||||
}
|
||||
if blk := k / b.width; blk != c.blk {
|
||||
b.blocks[blk].common(b.width, sparse, c.v[:])
|
||||
c.blk = blk
|
||||
}
|
||||
return int(c.v[k%b.width])
|
||||
}
|
||||
|
||||
// Index stores sequences by length. A sequence can only be similar to the ones
|
||||
// whose length is compatible with its own (see Scoring.LengthCompatible), so
|
||||
// it only visits the buckets of these lengths, instead of all the sequences,
|
||||
// and the filter works on blocks of sequences at once.
|
||||
//
|
||||
// An Index can grow, but not while it is read.
|
||||
type Index struct {
|
||||
width int // sequences per block of transposed tables, 0 if unused
|
||||
byLen map[int]*Bucket
|
||||
lens []int // lengths that have a bucket, increasing
|
||||
tabs [][]Table4 // chunks holding the tables of the members
|
||||
n int
|
||||
}
|
||||
|
||||
// NewIndex returns an empty index.
|
||||
func NewIndex() *Index {
|
||||
return &Index{width: simdBlockWidth(), byLen: make(map[int]*Bucket)}
|
||||
}
|
||||
|
||||
// Len is the number of sequences in the index.
|
||||
func (x *Index) Len() int { return x.n }
|
||||
|
||||
// Lengths returns the lengths that have a bucket, in increasing order.
|
||||
func (x *Index) Lengths() []int { return x.lens }
|
||||
|
||||
// Bucket returns the bucket of the sequences of the given length.
|
||||
func (x *Index) Bucket(length int) *Bucket { return x.byLen[length] }
|
||||
|
||||
// Add stores a copy of the entry e as the next member of the index, with the
|
||||
// identifier id, and returns it.
|
||||
func (x *Index) Add(e Entry, id int) Member {
|
||||
if x.n%tabChunk == 0 {
|
||||
x.tabs = append(x.tabs, make([]Table4, tabChunk))
|
||||
}
|
||||
tab := &x.tabs[len(x.tabs)-1][x.n%tabChunk]
|
||||
if e.Table != nil {
|
||||
*tab = *e.Table
|
||||
}
|
||||
e.Table = tab
|
||||
|
||||
b := x.byLen[e.Length]
|
||||
if b == nil {
|
||||
b = &Bucket{width: x.width}
|
||||
x.byLen[e.Length] = b
|
||||
x.lens = append(x.lens, e.Length)
|
||||
sort.Ints(x.lens)
|
||||
}
|
||||
if w := x.width; w > 0 && !e.Ambiguous {
|
||||
if len(b.Members)%w == 0 {
|
||||
b.blocks = append(b.blocks, newTableBlock(w))
|
||||
}
|
||||
b.blocks[len(b.blocks)-1].set(w, len(b.Members)%w, tab)
|
||||
} else if w > 0 && len(b.Members)%w == 0 {
|
||||
b.blocks = append(b.blocks, newTableBlock(w))
|
||||
}
|
||||
|
||||
m := Member{Entry: e, ID: id, Rank: x.n}
|
||||
b.Members = append(b.Members, m)
|
||||
x.n++
|
||||
return m
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
package obisimilarity
|
||||
|
||||
import (
|
||||
"math/rand"
|
||||
"testing"
|
||||
|
||||
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obialign"
|
||||
)
|
||||
|
||||
func randomSeq(rng *rand.Rand, n int, alpha string) []byte {
|
||||
s := make([]byte, n)
|
||||
for i := range s {
|
||||
s[i] = alpha[rng.Intn(len(alpha))]
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// commonWordsRef is the definition of commonWords, spelled out.
|
||||
func commonWordsRef(t1, t2 *Table4) int {
|
||||
total := 0
|
||||
for i := range t1.Count {
|
||||
total += int(min(t1.Count[i], t2.Count[i]))
|
||||
}
|
||||
return total + min(t1.Overflow, t2.Overflow)
|
||||
}
|
||||
|
||||
func TestCommonWords(t *testing.T) {
|
||||
rng := rand.New(rand.NewSource(3))
|
||||
for it := 0; it < 3000; it++ {
|
||||
alpha := []string{"acgt", "ac", "a"}[it%3]
|
||||
t1 := NewTable4(randomSeq(rng, rng.Intn(1200), alpha))
|
||||
t2 := NewTable4(randomSeq(rng, rng.Intn(1200), alpha))
|
||||
if got, want := commonWords(t1, t2), commonWordsRef(t1, t2); got != want {
|
||||
t.Fatalf("commonWords = %d, want %d", got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewTable4 checks the table against the 4-mers counted the long way,
|
||||
// including the saturation of the counters.
|
||||
func TestNewTable4(t *testing.T) {
|
||||
code := map[byte]int{'a': 0, 'c': 1, 't': 2, 'g': 3}
|
||||
rng := rand.New(rand.NewSource(4))
|
||||
for it := 0; it < 500; it++ {
|
||||
alpha := []string{"acgt", "a", "ac"}[it%3]
|
||||
s := randomSeq(rng, rng.Intn(900), alpha)
|
||||
var counts [256]int
|
||||
for i := 0; i+4 <= len(s); i++ {
|
||||
w := 0
|
||||
for _, c := range s[i : i+4] {
|
||||
w = w<<2 | code[c]
|
||||
}
|
||||
counts[w]++
|
||||
}
|
||||
tab := NewTable4(s)
|
||||
over := 0
|
||||
for w, c := range counts {
|
||||
if c > 255 {
|
||||
over += c - 255
|
||||
c = 255
|
||||
}
|
||||
if int(tab.Count[w]) != c {
|
||||
t.Fatalf("word %d: got %d, want %d", w, tab.Count[w], c)
|
||||
}
|
||||
}
|
||||
if tab.Overflow != over {
|
||||
t.Fatalf("overflow = %d, want %d", tab.Overflow, over)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestBlockCommon checks the sums of a block of transposed tables against
|
||||
// commonWords.
|
||||
func TestBlockCommon(t *testing.T) {
|
||||
width := simdBlockWidth()
|
||||
if width == 0 {
|
||||
t.Skip("blocks of tables are used with GOEXPERIMENT=simd only")
|
||||
}
|
||||
rng := rand.New(rand.NewSource(8))
|
||||
for it := 0; it < 200; it++ {
|
||||
alpha := []string{"acgt", "ac", "a"}[it%3]
|
||||
block := newTableBlock(width)
|
||||
tabs := make([]*Table4, width)
|
||||
for j := range tabs {
|
||||
tabs[j] = NewTable4(randomSeq(rng, rng.Intn(600), alpha))
|
||||
block.set(width, j, tabs[j])
|
||||
}
|
||||
q := NewTable4(randomSeq(rng, rng.Intn(260), alpha))
|
||||
words := SparseWords(q)
|
||||
if words == nil {
|
||||
continue
|
||||
}
|
||||
out := make([]uint8, 64)
|
||||
block.common(width, words, out)
|
||||
for j, tab := range tabs {
|
||||
want := commonWordsRef(q, tab) - min(q.Overflow, tab.Overflow)
|
||||
if int(out[j]) != want {
|
||||
t.Fatalf("lane %d: %d, want %d", j, out[j], want)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassify(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
accept bool
|
||||
kept bool
|
||||
ambiguous bool
|
||||
}{
|
||||
{"acgt", false, true, false}, {"acgt", true, true, false},
|
||||
{"acgtn", false, false, false}, {"acgtn", true, true, true},
|
||||
{"acgtx", true, false, false}, {"nacgt", false, false, false},
|
||||
{"acgt-", true, false, false}, {"acgu", true, true, true},
|
||||
{"ACGT", true, false, false}, {"", true, false, false},
|
||||
{"rykmswbdhvn", true, true, true}, {"rykmswbdhvn", false, false, false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
kept, amb := Classify([]byte(c.in), c.accept)
|
||||
if kept != c.kept || amb != c.ambiguous {
|
||||
t.Errorf("Classify(%q, %v) = (%v, %v), want kept=%v ambiguous=%v", c.in, c.accept, kept, amb, c.kept, c.ambiguous)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func randomEntries(rng *rand.Rand, n int) []Entry {
|
||||
return BuildEntries(n, func(int) ([]byte, int, bool) {
|
||||
return randomSeq(rng, 8+rng.Intn(80), "acgt"), 1 + rng.Intn(20), false
|
||||
}, 2)
|
||||
}
|
||||
|
||||
// TestLengthTable checks that the values precomputed by length give the
|
||||
// scores of the direct computation, in every mode.
|
||||
func TestLengthTable(t *testing.T) {
|
||||
rng := rand.New(rand.NewSource(33))
|
||||
// sequences made of the same few roots, so that many pairs are similar
|
||||
roots := [][]byte{randomSeq(rng, 70, "acgt"), randomSeq(rng, 55, "acgt")}
|
||||
entries := BuildEntries(120, func(i int) ([]byte, int, bool) {
|
||||
var s []byte
|
||||
for _, c := range roots[i%2] {
|
||||
switch r := rng.Intn(30); {
|
||||
case r == 0:
|
||||
s = append(s, "acgt"[rng.Intn(4)])
|
||||
case r == 1:
|
||||
default:
|
||||
s = append(s, c)
|
||||
}
|
||||
}
|
||||
if i < 20 { // very short sequences skip the filter
|
||||
s = s[:8+rng.Intn(10)]
|
||||
}
|
||||
return s, 1 + rng.Intn(20), false
|
||||
}, 2)
|
||||
|
||||
var buf obialign.BandedLCSBuffer
|
||||
for _, ref := range []Reference{RefAlignment, RefLongest, RefShortest} {
|
||||
for _, p := range []Params{
|
||||
{Threshold: 0.9, Normalized: true},
|
||||
{Threshold: 0.8, Normalized: true, Distance: true},
|
||||
{Threshold: 30},
|
||||
{Threshold: 6, Distance: true},
|
||||
{Normalized: true}, // no threshold
|
||||
} {
|
||||
p.Reference, p.Ratio = ref, 3
|
||||
table, err := NewScoring(p, entries)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
direct := *table
|
||||
direct.byLength = nil // forces the direct computation
|
||||
if (ref == RefShortest) != (table.byLength == nil) {
|
||||
t.Fatalf("ref %v: table presence is wrong", ref)
|
||||
}
|
||||
for i := range entries {
|
||||
for j := range entries {
|
||||
a, aok := direct.PairScore(&entries[i], &entries[j], -1, &buf)
|
||||
b, bok := table.PairScore(&entries[i], &entries[j], -1, &buf)
|
||||
if a != b || aok != bok {
|
||||
t.Fatalf("ref %v params %+v pair %d,%d: (%v,%v) with the table, (%v,%v) directly", ref, p, i, j, b, bok, a, aok)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestIndexCommon checks that the filter computed by blocks gives the same
|
||||
// scores as the direct computation, for every pair, whatever the order the
|
||||
// members are read in.
|
||||
func TestIndexCommon(t *testing.T) {
|
||||
rng := rand.New(rand.NewSource(7))
|
||||
entries := randomEntries(rng, 150)
|
||||
sc, err := NewScoring(Params{Threshold: 0.6, Normalized: true, Reference: RefAlignment, Ratio: 100}, entries)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
index := NewIndex()
|
||||
for i := range entries {
|
||||
index.Add(entries[i], i)
|
||||
}
|
||||
if index.Len() != len(entries) {
|
||||
t.Fatalf("Len = %d", index.Len())
|
||||
}
|
||||
|
||||
var buf obialign.BandedLCSBuffer
|
||||
var commons Commons
|
||||
for i := range entries {
|
||||
q := &entries[i]
|
||||
for _, length := range index.Lengths() {
|
||||
b := index.Bucket(length)
|
||||
commons.Reset()
|
||||
for k := len(b.Members) - 1; k >= 0; k-- { // backwards: blocks are not read in order
|
||||
m := &b.Members[k]
|
||||
got, gok := sc.PairScore(q, &m.Entry, b.Common(k, q.Sparse, &commons), &buf)
|
||||
want, wok := sc.PairScore(q, &entries[m.ID], -1, &buf)
|
||||
if got != want || gok != wok {
|
||||
t.Fatalf("pair %d,%d: (%v,%v) with the blocks, (%v,%v) directly", i, m.ID, got, gok, want, wok)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewScoringChecks(t *testing.T) {
|
||||
for _, p := range []Params{{Threshold: -1}, {Threshold: 1.5, Normalized: true}} {
|
||||
if _, err := NewScoring(p, nil); err == nil {
|
||||
t.Errorf("%+v must be rejected", p)
|
||||
}
|
||||
}
|
||||
if _, err := NewScoring(Params{Threshold: 12}, nil); err != nil {
|
||||
t.Errorf("a raw threshold above 1 is valid: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,284 @@
|
||||
package obisimilarity
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"math"
|
||||
|
||||
"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obialign"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// Reference is the length used to normalize a score (or to express a raw
|
||||
// distance): the alignment length, the longest or the shortest sequence.
|
||||
type Reference int
|
||||
|
||||
const (
|
||||
RefAlignment Reference = iota
|
||||
RefLongest
|
||||
RefShortest
|
||||
)
|
||||
|
||||
// Params describe how a pair of sequences is scored and compared to a threshold.
|
||||
type Params struct {
|
||||
// Threshold: with a normalized similarity an identity (0.95 for 95%), with
|
||||
// a normalized distance 1 - identity, with a raw similarity the length of
|
||||
// the LCS, with a raw distance the number of differences with the
|
||||
// reference length. 0 means no threshold: all the pairs are reported.
|
||||
Threshold float64
|
||||
Normalized bool // the score is divided by the reference length
|
||||
Distance bool // the score is a distance, not a similarity
|
||||
Reference Reference
|
||||
// Ratio is the maximum ratio between the counts of a sequence and of the
|
||||
// one it is compared to (clustering only): a larger ratio gives no match.
|
||||
Ratio float64
|
||||
}
|
||||
|
||||
// Scoring scores the pairs of sequences and decides whether they pass the
|
||||
// threshold. It must not be modified once built.
|
||||
type Scoring struct {
|
||||
threshold float64 // threshold as used for the comparison (1 - t for a normalized distance)
|
||||
normalize bool
|
||||
lcsmode bool // true: score is a similarity, false: it is a distance
|
||||
ref Reference
|
||||
ratio float64
|
||||
worst float64 // score of a pair that does not pass
|
||||
all bool // no threshold: every pair is reported
|
||||
|
||||
// byLength gives, for the pairs whose longest sequence has a length l,
|
||||
// the values that depend on l only. It is built when the reference is not
|
||||
// the shortest sequence, otherwise those values depend on both lengths.
|
||||
byLength []lengthInfo
|
||||
}
|
||||
|
||||
type lengthInfo struct {
|
||||
lcsMin int32 // see lcsMin
|
||||
minCommon int32 // see minCommon
|
||||
}
|
||||
|
||||
// NewScoring builds the scoring of the sequences `entries`, which are the
|
||||
// ones to be compared (the longest of them is needed for a raw distance).
|
||||
func NewScoring(p Params, entries []Entry) (*Scoring, error) {
|
||||
if p.Threshold < 0 || (p.Normalized && p.Threshold > 1.0) {
|
||||
return nil, errors.New("the threshold must not be negative, and must be at most 1 when scores are normalized")
|
||||
}
|
||||
|
||||
sc := &Scoring{
|
||||
normalize: p.Normalized,
|
||||
lcsmode: !p.Distance,
|
||||
ref: p.Reference,
|
||||
ratio: p.Ratio,
|
||||
}
|
||||
sc.threshold = p.Threshold
|
||||
sc.all = p.Threshold == 0
|
||||
if !sc.lcsmode && sc.normalize && p.Threshold > 0 {
|
||||
sc.threshold = 1.0 - p.Threshold
|
||||
}
|
||||
|
||||
lmax := 0
|
||||
for i := range entries {
|
||||
lmax = max(lmax, entries[i].Length)
|
||||
}
|
||||
if !sc.similarityLike() {
|
||||
sc.worst = float64(lmax)
|
||||
}
|
||||
|
||||
if sc.ref != RefShortest {
|
||||
sc.byLength = make([]lengthInfo, lmax+1)
|
||||
for l := range sc.byLength {
|
||||
sc.byLength[l] = lengthInfo{int32(sc.lcsMin(l, l)), int32(sc.minCommon(l, l))}
|
||||
}
|
||||
}
|
||||
return sc, nil
|
||||
}
|
||||
|
||||
// Worst is the score of a pair that does not pass the threshold.
|
||||
func (sc *Scoring) Worst() float64 { return sc.worst }
|
||||
|
||||
// Threshold is the threshold the scores are compared to.
|
||||
func (sc *Scoring) Threshold() float64 { return sc.threshold }
|
||||
|
||||
// SimilarityLike tells if the comparison with the threshold is "score >= threshold".
|
||||
func (sc *Scoring) SimilarityLike() bool { return sc.similarityLike() }
|
||||
|
||||
func (sc *Scoring) similarityLike() bool { return sc.lcsmode || sc.normalize }
|
||||
|
||||
// lcsMin is the smallest LCS score that a pair of lengths l1 >= l2 can have
|
||||
// and still pass the threshold.
|
||||
func (sc *Scoring) lcsMin(l1, l2 int) int {
|
||||
switch {
|
||||
case sc.threshold <= 0:
|
||||
return 0
|
||||
case sc.normalize:
|
||||
if sc.ref == RefShortest {
|
||||
return int(sc.threshold * float64(l2))
|
||||
}
|
||||
return int(sc.threshold * float64(l1))
|
||||
case sc.lcsmode:
|
||||
return int(sc.threshold)
|
||||
case sc.ref == RefShortest:
|
||||
return int(float64(l2) - sc.threshold)
|
||||
default:
|
||||
return int(float64(l1) - sc.threshold)
|
||||
}
|
||||
}
|
||||
|
||||
// minCommon is the number of 4-mers that two sequences of lengths l1 >= l2
|
||||
// must share to be possibly similar enough.
|
||||
func (sc *Scoring) minCommon(l1, l2 int) int {
|
||||
if sc.all {
|
||||
return 0 // no threshold, no filter
|
||||
}
|
||||
reflen := l2
|
||||
if sc.ref == RefAlignment || sc.ref == RefLongest {
|
||||
reflen = l1
|
||||
}
|
||||
var lcs int
|
||||
if sc.normalize {
|
||||
lcs = int(math.Ceil(float64(reflen) * sc.threshold))
|
||||
} else {
|
||||
th := sc.threshold
|
||||
if !sc.lcsmode {
|
||||
th = float64(reflen) - th
|
||||
}
|
||||
lcs = int(th)
|
||||
}
|
||||
return minCommonWords(l1, lcs)
|
||||
}
|
||||
|
||||
// lcsMinFor is lcsMin for sequences of lengths l1 >= l2, from the table when it exists.
|
||||
func (sc *Scoring) lcsMinFor(l1, l2 int) int {
|
||||
if sc.byLength != nil {
|
||||
return int(sc.byLength[l1].lcsMin)
|
||||
}
|
||||
return sc.lcsMin(l1, l2)
|
||||
}
|
||||
|
||||
// LengthCompatible tells if two sequences of these lengths may be similar
|
||||
// enough: the shortest cannot be shorter than the minimum LCS score of the pair.
|
||||
func (sc *Scoring) LengthCompatible(la, lb int) bool {
|
||||
l1, l2 := max(la, lb), min(la, lb)
|
||||
return l2 >= sc.lcsMinFor(l1, l2)
|
||||
}
|
||||
|
||||
// PairScore scores the pair (q, c). ok is false when the pair is known not to
|
||||
// reach the threshold, in which case no score is given.
|
||||
//
|
||||
// common, if not negative, is the sum of the minima of the 4-mer counters of
|
||||
// the two sequences, already computed by a block of tables (see Index).
|
||||
func (sc *Scoring) PairScore(q, c *Entry, common int, buf *obialign.BandedLCSBuffer) (score float64, ok bool) {
|
||||
// The quotient of two counts is not above 1 when the first is not above
|
||||
// the second, which spares the division in the usual case.
|
||||
if !(sc.ratio >= 1 && q.Count <= c.Count) && float64(q.Count)/float64(c.Count) > sc.ratio {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
l1, l2 := q.Length, c.Length
|
||||
long, short := q, c
|
||||
if l1 < l2 {
|
||||
l1, l2 = l2, l1
|
||||
long, short = c, q
|
||||
}
|
||||
var lcsMin, minCommon int
|
||||
if sc.byLength != nil {
|
||||
info := sc.byLength[l1]
|
||||
lcsMin, minCommon = int(info.lcsMin), int(info.minCommon)
|
||||
} else {
|
||||
lcsMin, minCommon = sc.lcsMin(l1, l2), sc.minCommon(l1, l2)
|
||||
}
|
||||
if l2 < lcsMin {
|
||||
return 0, false
|
||||
}
|
||||
// The 4-mer filter counts identical words: it does not hold when IUPAC
|
||||
// codes can match different letters. It lets short sequences pass.
|
||||
if l1 >= 12 && l2 >= 12 && !long.Ambiguous && !short.Ambiguous {
|
||||
if common >= 0 {
|
||||
common += min(long.Table.Overflow, short.Table.Overflow)
|
||||
} else {
|
||||
common = commonWords(long.Table, short.Table)
|
||||
}
|
||||
if common < minCommon {
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
|
||||
// The length of the alignment is needed only to normalize by it, or to
|
||||
// express a distance with it; the score alone is much cheaper.
|
||||
var lcs, alilen int
|
||||
var err error
|
||||
if sc.ref == RefAlignment && (sc.normalize || !sc.lcsmode) {
|
||||
lcs, alilen, err = buf.AtLeast(q.Seq, c.Seq, lcsMin)
|
||||
} else {
|
||||
lcs, err = buf.ScoreAtLeast(q.Seq, c.Seq, lcsMin)
|
||||
}
|
||||
if err != nil {
|
||||
log.Fatalf("Cannot align two sequences: %v", err)
|
||||
}
|
||||
if lcs < 0 {
|
||||
// The best alignment has a score under lcsMin: the pair cannot pass.
|
||||
return 0, false
|
||||
}
|
||||
|
||||
id := float64(lcs)
|
||||
if !sc.lcsmode && !sc.normalize {
|
||||
switch sc.ref {
|
||||
case RefAlignment:
|
||||
id = float64(alilen) - id
|
||||
case RefLongest:
|
||||
id = float64(l1) - id
|
||||
case RefShortest:
|
||||
id = float64(l2) - id
|
||||
}
|
||||
}
|
||||
if sc.normalize {
|
||||
switch sc.ref {
|
||||
case RefAlignment:
|
||||
id /= float64(alilen)
|
||||
case RefLongest:
|
||||
id /= float64(l1)
|
||||
case RefShortest:
|
||||
id /= float64(l2)
|
||||
}
|
||||
}
|
||||
return id, true
|
||||
}
|
||||
|
||||
// Passes tells if the score s of a pair reaches the threshold.
|
||||
func (sc *Scoring) Passes(s float64) bool {
|
||||
if sc.all {
|
||||
return true
|
||||
}
|
||||
if sc.similarityLike() {
|
||||
return s >= sc.threshold
|
||||
}
|
||||
return s <= sc.threshold
|
||||
}
|
||||
|
||||
// Beats tells if the score s is better than the score best (without any
|
||||
// threshold).
|
||||
func (sc *Scoring) Beats(s, best float64) bool {
|
||||
if sc.similarityLike() {
|
||||
return s > best
|
||||
}
|
||||
return s < best
|
||||
}
|
||||
|
||||
// SelfScore is the score of a sequence with itself.
|
||||
func (sc *Scoring) SelfScore(length int) float64 {
|
||||
switch {
|
||||
case sc.normalize && sc.lcsmode:
|
||||
return 1.0
|
||||
case !sc.lcsmode:
|
||||
return 0.0
|
||||
default:
|
||||
return float64(length)
|
||||
}
|
||||
}
|
||||
|
||||
// Reported is the score to report for a pair whose score is s: a normalized
|
||||
// distance is computed as a similarity against the converted threshold.
|
||||
func (sc *Scoring) Reported(s float64) float64 {
|
||||
if !sc.lcsmode && sc.normalize {
|
||||
return 1.0 - s
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
// Package obisimilarity holds what the tools that compare nucleotide sequences
|
||||
// by the length of their longest common subsequence (obiclust, obialign) have
|
||||
// in common: the scoring of a pair against a threshold, with the lossless
|
||||
// 4-mer filter that avoids most of the alignments, and the index that finds
|
||||
// the sequences a given one can be similar to.
|
||||
//
|
||||
// The scoring follows the arithmetic of sumaclust and sumatra (integer
|
||||
// truncations, floating point comparisons) so that the same pairs are
|
||||
// accepted with the same scores.
|
||||
package obisimilarity
|
||||
|
||||
// Table4 is the 4-mer occurrence table of a sequence: one saturating counter
|
||||
// per word, and the number of occurrences that did not fit in the counters.
|
||||
type Table4 struct {
|
||||
Count [256]uint8
|
||||
Overflow int
|
||||
}
|
||||
|
||||
// NewTable4 counts the 4-mers of a sequence made of the letters a, c, g
|
||||
// and t. The 2-bit code of a base is (letter>>1)&3, i.e. a=0, c=1, t=2, g=3.
|
||||
func NewTable4(seq []byte) *Table4 {
|
||||
t := new(Table4)
|
||||
t.Fill(seq)
|
||||
return t
|
||||
}
|
||||
|
||||
// Fill counts the 4-mers of seq in a zeroed table.
|
||||
func (t *Table4) Fill(seq []byte) {
|
||||
if len(seq) < 4 {
|
||||
return
|
||||
}
|
||||
w := uint(0)
|
||||
for i, c := range seq {
|
||||
w = (w<<2 | uint(c>>1)&3) & 0xFF
|
||||
if i < 3 {
|
||||
continue
|
||||
}
|
||||
if t.Count[w] < 255 {
|
||||
t.Count[w]++
|
||||
} else {
|
||||
t.Overflow++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// minCommonWords is the minimum number of 4-mers that two sequences must
|
||||
// share when the longest has length reflen and they have a common
|
||||
// subsequence of length lcs.
|
||||
func minCommonWords(reflen, lcs int) int {
|
||||
nbfrag := (reflen-lcs)*2 + 1
|
||||
smin := lcs / nbfrag
|
||||
r := lcs - smin*nbfrag
|
||||
return max(smin-2, 0)*r + max(smin-3, 0)*(nbfrag-r)
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
//go:build !goexperiment.simd
|
||||
|
||||
package obisimilarity
|
||||
|
||||
// commonWords is the upper bound used for the number of 4-mers shared by two
|
||||
// sequences: the sum of the minima of the counters, plus the minimum of the
|
||||
// two overflows.
|
||||
func commonWords(t1, t2 *Table4) int {
|
||||
total := 0
|
||||
for i := range t1.Count {
|
||||
total += int(min(t1.Count[i], t2.Count[i]))
|
||||
}
|
||||
return total + min(t1.Overflow, t2.Overflow)
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
//go:build goexperiment.simd
|
||||
|
||||
package obisimilarity
|
||||
|
||||
import "simd"
|
||||
|
||||
// commonWords is the upper bound used for the number of 4-mers shared by two
|
||||
// sequences: the sum of the minima of the counters, plus the minimum of the
|
||||
// two overflows.
|
||||
//
|
||||
// The minima of the 256 counters are computed by vectors of bytes, and summed
|
||||
// in 16-bit lanes (at most 256*255 spread over the lanes, the lanes being at
|
||||
// least 8 in number, so that none of them overflows).
|
||||
func commonWords(t1, t2 *Table4) int {
|
||||
var probe simd.Uint8s
|
||||
vl := probe.Len()
|
||||
|
||||
acc := simd.BroadcastUint16s(0)
|
||||
low := simd.BroadcastUint16s(0x00FF)
|
||||
for i := 0; i < len(t1.Count); i += vl {
|
||||
m := simd.LoadUint8s(t1.Count[i:]).Min(simd.LoadUint8s(t2.Count[i:])).ReshapeToUint16s()
|
||||
acc = acc.Add(m.And(low)).Add(m.ShiftAllRight(8))
|
||||
}
|
||||
|
||||
// sum the pairs of 16-bit lanes into 32-bit lanes, then add those up
|
||||
wide := acc.ReshapeToUint32s()
|
||||
sums := wide.And(simd.BroadcastUint32s(0xFFFF)).Add(wide.ShiftAllRight(16))
|
||||
var lanes [32]uint32
|
||||
sums.Store(lanes[:])
|
||||
total := 0
|
||||
for _, v := range lanes[:sums.Len()] {
|
||||
total += int(v)
|
||||
}
|
||||
return total + min(t1.Overflow, t2.Overflow)
|
||||
}
|
||||
Reference in new issue
Block a user