Introduce sequence roles and filtering logic

Implement new data structures for corpus management, multi-match filtering, and barcode resolution.

pkg/obiprimer/amplicons.go

@@ -0,0 +1,232 @@
+package obiprimer
+
+import (
+	"cmp"
+	"math"
+	"slices"
+	"sync"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obithermo"
+)
+
+// ampRecord is an amplicon of a retained pair, in one sequence.
+type ampRecord struct {
+	hash  uint64 // the barcode: see ampHash
+	pair  int32
+	taxon int32 // taxon of the sequence at the chosen rank, -1 when it has none
+	role  Role
+}
+
+// refHit is an amplicon of a retained pair on the reference sequence.
+type refHit struct {
+	up, down seqMatch
+	swapped  bool
+}
+
+// ampHash hashes the amplicon between the two primers read from the side of
+// the first primer of the pair (reverse complemented when the upstream match is
+// the second primer), so that identical amplicons give the same value
+// whatever their strand. Non ACGT bases hash as a fifth symbol.
+func ampHash(raw []byte, from, to int, reverse bool) uint64 {
+	const prime = 1099511628211
+	h := uint64(14695981039346656037) ^ uint64(to-from)
+	h *= prime
+	if !reverse {
+		for _, b := range raw[from:to] {
+			h = (h ^ uint64(baseCode[b])) * prime
+		}
+		return h
+	}
+	for i := to - 1; i >= from; i-- {
+		c := baseCode[raw[i]]
+		if c < 4 {
+			c = 3 - c
+		}
+		h = (h ^ uint64(c)) * prime
+	}
+	return h
+}
+
+// amplicons holds what the second phase gathers on the retained pairs.
+type amplicons struct {
+	parts  [][]ampRecord // one slice per worker, each sorted by pair
+	minTm1 []float32     // by pair: lowest Tm of the first primer over the example sites
+	minTm2 []float32
+	refs   map[int32]refHit
+}
+
+// pair returns the amplicons of one retained pair, as one slice per worker.
+func (a *amplicons) pair(i int32) [][]ampRecord {
+	out := make([][]ampRecord, 0, len(a.parts))
+	byPair := func(r ampRecord, p int32) int { return cmp.Compare(r.pair, p) }
+	for _, part := range a.parts {
+		lo, _ := slices.BinarySearchFunc(part, i, byPair)
+		hi, _ := slices.BinarySearchFunc(part, i+1, byPair)
+		if hi > lo {
+			out = append(out, part[lo:hi])
+		}
+	}
+	return out
+}
+
+// collectOptions selects what collectAmplicons gathers.
+type collectOptions struct {
+	examplesOnly bool             // skip the counterexamples and the other sequences
+	model        *obithermo.Model // melting temperatures of the example sites (nil: none)
+	reference    bool             // record the amplicons on the reference sequence
+}
+
+// collectAmplicons gathers, in parallel, one record per amplicon of a retained
+// pair, in the sequences of the corpus (all of them, or the examples only), and
+// the lowest melting temperatures over the example sites. The records are not
+// copied nor regrouped: each worker keeps its own, sorted by pair.
+func (ps *primerSet) collectAmplicons(c *Corpus, taxa *Taxa, p PairParams, index map[pairKey]int32, opt collectOptions) *amplicons {
+	model := opt.model
+	lp := p.PrimerLength
+	workers := obidefault.ParallelWorkers()
+	npairs := len(index)
+
+	res := &amplicons{
+		parts:  make([][]ampRecord, workers),
+		minTm1: make([]float32, npairs),
+		minTm2: make([]float32, npairs),
+		refs:   map[int32]refHit{}, // written by the single worker that reads the reference
+	}
+	inf := float32(math.Inf(1))
+	for i := range npairs {
+		res.minTm1[i], res.minTm2[i] = inf, inf
+	}
+	var tmMu sync.Mutex
+
+	// oriented[primer][0] is the reverse complement of the word, [1] the word itself.
+	oriented := make([][2][]byte, len(ps.primers))
+	for i, pr := range ps.primers {
+		oriented[i][0] = []byte(orientedWord(pr.Word, lp, false))
+		oriented[i][1] = []byte(orientedWord(pr.Word, lp, true))
+	}
+
+	jobs := make(chan int)
+	var wg sync.WaitGroup
+	for w := range workers {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			site1, site2 := make([]byte, lp), make([]byte, lp)
+			lowTm1, lowTm2 := map[int32]float32{}, map[int32]float32{}
+			for s := range jobs {
+				raw := c.Seqs[s].Sequence()
+				n := len(raw)
+				role := c.Roles[s]
+				if opt.examplesOnly && role != Example {
+					continue
+				}
+				ps.forEachOccurrence(c, s, p, func(o occurrence) {
+					pi, ok := index[o.key]
+					if !ok {
+						return
+					}
+					if opt.reference && p.HasReference && s == p.Reference {
+						if old, seen := res.refs[pi]; !seen || o.up.pos < old.up.pos {
+							res.refs[pi] = refHit{o.up, o.down, o.swapped}
+						}
+					}
+					res.parts[w] = append(res.parts[w], ampRecord{
+						hash:  ampHash(raw, int(o.up.pos)+lp, int(o.down.pos), o.swapped),
+						pair:  pi,
+						taxon: taxa.Of[s],
+						role:  role,
+					})
+
+					if role != Example || model == nil {
+						return
+					}
+					fp := oriented[o.up.primer][b2i(!o.up.reverse)]
+					rp := oriented[o.down.primer][b2i(o.down.reverse)]
+					tmF, tmR := inf, inf
+					if windowAsIs(site1, raw, int(o.up.pos), n, p.Circular) {
+						if t, err := model.Tm(fp, site1); err == nil && !math.IsNaN(t) {
+							tmF = float32(t)
+						}
+					}
+					if windowRevComp(site2, raw, int(o.down.pos), n, p.Circular) {
+						if t, err := model.Tm(rp, site2); err == nil && !math.IsNaN(t) {
+							tmR = float32(t)
+						}
+					}
+					if o.swapped {
+						tmF, tmR = tmR, tmF // slot 1 is the downstream primer
+					}
+					lowTm1[pi] = min(lowOr(lowTm1, pi, inf), tmF)
+					lowTm2[pi] = min(lowOr(lowTm2, pi, inf), tmR)
+				})
+			}
+
+			slices.SortFunc(res.parts[w], func(a, b ampRecord) int { return cmp.Compare(a.pair, b.pair) })
+			tmMu.Lock()
+			for pi, t := range lowTm1 {
+				res.minTm1[pi] = min(res.minTm1[pi], t)
+			}
+			for pi, t := range lowTm2 {
+				res.minTm2[pi] = min(res.minTm2[pi], t)
+			}
+			tmMu.Unlock()
+		}()
+	}
+	for s := range c.Seqs {
+		jobs <- s
+	}
+	close(jobs)
+	wg.Wait()
+	return res
+}
+
+func lowOr(m map[int32]float32, k int32, def float32) float32 {
+	if v, ok := m[k]; ok {
+		return v
+	}
+	return def
+}
+
+func b2i(b bool) int {
+	if b {
+		return 1
+	}
+	return 0
+}
+
+const upper = "ACGT"
+
+// windowAsIs copies the len(dst) bases at pos as upper-case ACGT; it reports
+// false if one of them is not A, C, G or T.
+func windowAsIs(dst, raw []byte, pos, n int, circular bool) bool {
+	for t := range dst {
+		i := pos + t
+		if circular {
+			i %= n
+		}
+		c := baseCode[raw[i]]
+		if c > 3 {
+			return false
+		}
+		dst[t] = upper[c]
+	}
+	return true
+}
+
+// windowRevComp writes the reverse complement of the len(dst) bases at pos.
+func windowRevComp(dst, raw []byte, pos, n int, circular bool) bool {
+	l := len(dst)
+	for t := range dst {
+		i := pos + l - 1 - t
+		if circular {
+			i %= n
+		}
+		c := baseCode[raw[i]]
+		if c > 3 {
+			return false
+		}
+		dst[t] = upper[3-c]
+	}
+	return true
+}

pkg/obiprimer/corpus.go

@@ -15,29 +15,32 @@ import (
 type Counts struct {
 	Examples        int
 	CounterExamples int
-	Ignored         int
+	Others          int
 }
 
-// Corpus is the set of sequences used by the primer search: the examples and
-// the counterexamples, in input order. Sequences with the Ignored role are not
-// kept (as in ecoPrimers), only counted.
+// Corpus is the set of sequences of the primer search, in input order, each
+// with its role: the examples (the targets), the counterexamples, and all the
+// other sequences, which are loaded as well since the amplicons they give
+// decide whether a target is identified.
 //
 // The primer search needs several passes over the sequences (words are
 // searched approximately in every one of them), so the corpus lives in memory.
 type Corpus struct {
-	Seqs    []*obiseq.BioSequence
-	Roles   []Role // Roles[i] is Example or CounterExample
-	Ignored int    // number of sequences dropped by the selection
+	Seqs  []*obiseq.BioSequence
+	Roles []Role // Roles[i] is the role of Seqs[i]
 }
 
 // Counts returns the number of sequences by role.
 func (c *Corpus) Counts() Counts {
-	n := Counts{Ignored: c.Ignored}
+	var n Counts
 	for _, r := range c.Roles {
-		if r == Example {
+		switch r {
+		case Example:
 			n.Examples++
-		} else {
+		case CounterExample:
 			n.CounterExamples++
+		default:
+			n.Others++
 		}
 	}
 	return n
@@ -49,31 +52,24 @@ type corpusEntry struct {
 	role         Role
 }
 
-// LoadCorpus reads every sequence of seqs, once, and keeps those selected as
-// examples or counterexamples. The order of the input is preserved whatever
+// LoadCorpus reads every sequence of seqs, once, and keeps all of them with the
+// role given by the selection. The order of the input is preserved whatever
 // the number of reader goroutines.
 func LoadCorpus(seqs obiiter.IBioSequence, sel Selection) (*Corpus, error) {
 	var mu sync.Mutex
 	var entries []corpusEntry
-	ignored := 0
 
 	consumer := func(it obiiter.IBioSequence, wg *sync.WaitGroup) {
 		defer wg.Done()
 		var local []corpusEntry
-		localIgnored := 0
 		for it.Next() {
 			batch := it.Get()
 			for i, seq := range batch.Slice() {
-				if role := sel.Role(seq); role == Ignored {
-					localIgnored++
-				} else {
-					local = append(local, corpusEntry{batch.Order(), i, seq, role})
-				}
+				local = append(local, corpusEntry{batch.Order(), i, seq, sel.Role(seq)})
 			}
 		}
 		mu.Lock()
 		entries = append(entries, local...)
-		ignored += localIgnored
 		mu.Unlock()
 	}
 
@@ -94,9 +90,8 @@ func LoadCorpus(seqs obiiter.IBioSequence, sel Selection) (*Corpus, error) {
 	})
 
 	c := &Corpus{
-		Seqs:    make([]*obiseq.BioSequence, len(entries)),
-		Roles:   make([]Role, len(entries)),
-		Ignored: ignored,
+		Seqs:  make([]*obiseq.BioSequence, len(entries)),
+		Roles: make([]Role, len(entries)),
 	}
 	for i, e := range entries {
 		c.Seqs[i], c.Roles[i] = e.seq, e.role

pkg/obiprimer/match.go

@@ -138,7 +138,8 @@ func matchCorpus(c *Corpus, words []uint64, mk scannerMaker) []WordMatches {
 }
 
 // SequencesMatched returns, for the matches of one word, the number of
-// distinct example and counterexample sequences it occurs in.
+// distinct example and counterexample sequences it occurs in. The other
+// sequences are not counted.
 func (w WordMatches) SequencesMatched(c *Corpus) (examples, counterExamples int) {
 	last := ^uint32(0)
 	for _, m := range w.Matches {
@@ -146,9 +147,10 @@ func (w WordMatches) SequencesMatched(c *Corpus) (examples, counterExamples int)
 			continue
 		}
 		last = m.Seq
-		if c.Roles[m.Seq] == Example {
+		switch c.Roles[m.Seq] {
+		case Example:
 			examples++
-		} else {
+		case CounterExample:
 			counterExamples++
 		}
 	}

pkg/obiprimer/multimatch.go

@@ -18,15 +18,16 @@ func DropRepeatedWords(words []obikmer.SeqKmerCount) (kept []obikmer.SeqKmerCoun
 }
 
 // DropMultiMatch removes the words having two or more approximate occurrences,
-// both strands counted, in at least one sequence of the corpus, whatever its
-// role. It is the second half of the ecoPrimers -U option: the primers that
-// can anneal at several places of a sequence are discarded.
-func DropMultiMatch(wm []WordMatches) (kept []WordMatches, dropped int) {
+// both strands counted, in at least one example or counterexample sequence. It
+// is the second half of the ecoPrimers -U option: the primers that can anneal
+// at several places of a sequence they must (or must not) amplify are
+// discarded. The other sequences are not considered.
+func DropMultiMatch(wm []WordMatches, c *Corpus) (kept []WordMatches, dropped int) {
 	kept = make([]WordMatches, 0, len(wm))
 	for _, w := range wm {
 		multi := false
 		for i := 1; i < len(w.Matches); i++ {
-			if w.Matches[i].Seq == w.Matches[i-1].Seq {
+			if w.Matches[i].Seq == w.Matches[i-1].Seq && c.Roles[w.Matches[i].Seq] != Other {
 				multi = true
 				break
 			}

pkg/obiprimer/pairs.go

@@ -221,6 +221,9 @@ func (ps *primerSet) countPairs(c *Corpus, p PairParams) map[pairKey]*pairAcc {
 			defer wg.Done()
 			acc := partial[w]
 			for s := range jobs {
+				if c.Roles[s] == Other {
+					continue // only the quorums are counted here
+				}
 				isExample := c.Roles[s] == Example
 				ps.forEachOccurrence(c, s, p, func(o occurrence) {
 					a := acc[o.key]

pkg/obiprimer/pairs_eval.go

@@ -21,11 +21,12 @@ type PairStats struct {
 	CounterExamples int
 	Yule            float32 // Examples/|E| - CounterExamples/|C|
 
-	ExampleTaxa    int     // taxa amplified among the examples
-	CounterTaxa    int     // taxa amplified among the counterexamples
-	Bc             float32 // coverage: ExampleTaxa over all the example taxa
-	Identified     int     // example taxa with an amplicon no other taxon shares
+	ExampleTaxa    int     // target taxa amplified
+	CounterTaxa    int     // counterexample taxa amplified
+	Bc             float32 // coverage: ExampleTaxa over all the target taxa
+	Identified     int     // target taxa all of whose barcodes belong to them alone (see resolution.go)
 	Bs             float32 // specificity: Identified over ExampleTaxa
+	Bd             float32 // barcode discrimination: share of the target barcodes leading to a single taxon
 	MinLen, MaxLen int
 	MeanLen        float64
 	Tm1, Tm2       float64 // melting temperature of the primers against their perfect target
@@ -50,44 +51,7 @@ type PairInfo struct {
 	GoodPrimers    int // of which good ones
 	CandidatePairs int // pairs delimiting an amplicon in at least one sequence
 	RetainedPairs  int // pairs passing the sensitivity and false positive quorums, before the Bs filter
-}
-
-// ampRecord is an amplicon of a retained pair, in one sequence.
-type ampRecord struct {
-	pair     int32
-	seq      int32
-	hash     uint64
-	tm1, tm2 float32 // NaN when unavailable
-}
-
-// refHit is an amplicon of a retained pair on the reference sequence.
-type refHit struct {
-	up, down seqMatch
-	swapped  bool
-}
-
-// ampHash hashes the amplicon between the two primers read from the side of
-// the first primer of the pair (reverse complemented when the upstream match is
-// the second primer), so that identical amplicons give the same value
-// whatever their strand. Non ACGT bases hash as a fifth symbol.
-func ampHash(raw []byte, from, to int, reverse bool) uint64 {
-	const prime = 1099511628211
-	h := uint64(14695981039346656037) ^ uint64(to-from)
-	h *= prime
-	if !reverse {
-		for _, b := range raw[from:to] {
-			h = (h ^ uint64(baseCode[b])) * prime
-		}
-		return h
-	}
-	for i := to - 1; i >= from; i-- {
-		c := baseCode[raw[i]]
-		if c < 4 {
-			c = 3 - c
-		}
-		h = (h ^ uint64(c)) * prime
-	}
-	return h
+	Candidates     int // of which pairs whose Bs upper bound reaches the threshold (their amplicons are gathered in full)
 }
 
 // FindPairs searches the primer pairs from the approximate occurrences of the
@@ -145,24 +109,63 @@ func FindPairs(c *Corpus, taxa *Taxa, wm []WordMatches, p PairParams, model *obi
 		index[k] = int32(i)
 	}
 
-	// Phase 2: the amplicons of the retained pairs only.
-	records, refs := ps.collectAmplicons(c, p, index, model)
-	byPair := make([][]ampRecord, len(kept))
-	for _, r := range records {
-		byPair[r.pair] = append(byPair[r.pair], r)
+	// Phase 2a: the amplicons of the target sequences, which give the melting
+	// temperatures, the reference sites, and an upper bound of Bs. It is cheap
+	// and rejects most pairs on the specificity threshold.
+	first := ps.collectAmplicons(c, taxa, p, index, collectOptions{examplesOnly: true, model: model, reference: true})
+	var survivors []int // indices in kept
+	for i := range kept {
+		targets, identified := lenientIdentified(first.pair(int32(i))...)
+		if targets > 0 && float32(identified)/float32(targets) >= float32(p.MinSpecificity) {
+			survivors = append(survivors, i)
+		}
 	}
+	first.parts = nil
+	info.Candidates = len(survivors)
+
+	// Phase 2b: for the survivors only, the amplicons of every sequence, whatever
+	// its role: they decide which target taxa are identified.
+	index2 := make(map[pairKey]int32, len(survivors))
+	for j, i := range survivors {
+		index2[kept[i]] = int32(j)
+	}
+	amps := ps.collectAmplicons(c, taxa, p, index2, collectOptions{})
+
+	results := make([]*PairStats, len(survivors))
+	jobs := make(chan int)
+	var wg sync.WaitGroup
+	for range obidefault.ParallelWorkers() {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for j := range jobs {
+				i := survivors[j]
+				k := kept[i]
+				st, ok := ps.evaluate(taxa, counts, p, k, acc[k], resolve(amps.pair(int32(j))...),
+					first.minTm1[i], first.minTm2[i], model)
+				if !ok {
+					continue
+				}
+				if h, found := first.refs[int32(i)]; found {
+					// evaluate orders the primers by string; the amplicon is read from the first one
+					s1 := orientedWord(ps.primers[k.p1].Word, p.PrimerLength, k.a1)
+					s2 := orientedWord(ps.primers[k.p2].Word, p.PrimerLength, k.a2)
+					st.RefLocation, st.RefAmplicon = referenceSites(c.Seqs[p.Reference].Sequence(), h, p, s2 < s1)
+				}
+				results[j] = &st
+			}
+		}()
+	}
+	for j := range survivors {
+		jobs <- j
+	}
+	close(jobs)
+	wg.Wait()
 
 	var out []PairStats
-	for i, k := range kept {
-		st, ok := ps.evaluate(c, taxa, p, k, acc[k], byPair[i], model)
-		if ok {
-			if h, found := refs[int32(i)]; found {
-				// evaluate orders the primers by string; the amplicon is read from the first one
-				s1 := orientedWord(ps.primers[k.p1].Word, p.PrimerLength, k.a1)
-				s2 := orientedWord(ps.primers[k.p2].Word, p.PrimerLength, k.a2)
-				st.RefLocation, st.RefAmplicon = referenceSites(c.Seqs[p.Reference].Sequence(), h, p, s2 < s1)
-			}
-			out = append(out, st)
+	for _, st := range results {
+		if st != nil {
+			out = append(out, *st)
 		}
 	}
 	slices.SortFunc(out, func(a, b PairStats) int {
@@ -181,174 +184,21 @@ func boolCmp(a, b bool) int {
 	return 1
 }
 
-// collectAmplicons gathers, in parallel, one record per amplicon of a retained pair.
-func (ps *primerSet) collectAmplicons(c *Corpus, p PairParams, index map[pairKey]int32, model *obithermo.Model) ([]ampRecord, map[int32]refHit) {
-	lp := p.PrimerLength
-	workers := obidefault.ParallelWorkers()
-	parts := make([][]ampRecord, workers)
+// evaluate puts the indices of one retained pair in a PairStats. It reports
+// false when the pair must be dropped (no target taxon amplified, or
+// specificity under the threshold).
+func (ps *primerSet) evaluate(taxa *Taxa, counts Counts, p PairParams, k pairKey, a *pairAcc, res resolution,
+	minTm1, minTm2 float32, model *obithermo.Model) (PairStats, bool) {
 
-	// oriented[primer][0] is the reverse complement of the word, [1] the word itself.
-	oriented := make([][2][]byte, len(ps.primers))
-	for i, pr := range ps.primers {
-		oriented[i][0] = []byte(orientedWord(pr.Word, lp, false))
-		oriented[i][1] = []byte(orientedWord(pr.Word, lp, true))
-	}
-	nan := float32(math.NaN())
-
-	// The reference sequence is handled by a single worker, which is the only
-	// writer of refs; the first (leftmost) amplicon of each pair is kept.
-	refs := map[int32]refHit{}
-
-	jobs := make(chan int)
-	var wg sync.WaitGroup
-	for w := range workers {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			var site1, site2 [32]byte
-			for s := range jobs {
-				raw := c.Seqs[s].Sequence()
-				n := len(raw)
-				isExample := c.Roles[s] == Example
-				ps.forEachOccurrence(c, s, p, func(o occurrence) {
-					pi, ok := index[o.key]
-					if !ok {
-						return
-					}
-					if p.HasReference && s == p.Reference {
-						if old, seen := refs[pi]; !seen || o.up.pos < old.up.pos {
-							refs[pi] = refHit{o.up, o.down, o.swapped}
-						}
-					}
-					rec := ampRecord{pair: pi, seq: int32(s), tm1: nan, tm2: nan}
-					if isExample {
-						rec.hash = ampHash(raw, int(o.up.pos)+lp, int(o.down.pos), o.swapped)
-						if model != nil {
-							fp := oriented[o.up.primer][b2i(!o.up.reverse)]
-							rp := oriented[o.down.primer][b2i(o.down.reverse)]
-							fsite, rsite := site1[:lp], site2[:lp]
-							okF := windowAsIs(fsite, raw, int(o.up.pos), n, p.Circular)
-							okR := windowRevComp(rsite, raw, int(o.down.pos), n, p.Circular)
-							tmF, tmR := nan, nan
-							if okF {
-								if t, err := model.Tm(fp, fsite); err == nil {
-									tmF = float32(t)
-								}
-							}
-							if okR {
-								if t, err := model.Tm(rp, rsite); err == nil {
-									tmR = float32(t)
-								}
-							}
-							if o.swapped {
-								rec.tm1, rec.tm2 = tmR, tmF
-							} else {
-								rec.tm1, rec.tm2 = tmF, tmR
-							}
-						}
-					}
-					parts[w] = append(parts[w], rec)
-				})
-			}
-		}()
-	}
-	for s := range c.Seqs {
-		jobs <- s
-	}
-	close(jobs)
-	wg.Wait()
-
-	var all []ampRecord
-	for _, part := range parts {
-		all = append(all, part...)
-	}
-	return all, refs
-}
-
-func b2i(b bool) int {
-	if b {
-		return 1
-	}
-	return 0
-}
-
-const upper = "ACGT"
-
-// windowAsIs copies the len(dst) bases at pos as upper-case ACGT; it reports
-// false if one of them is not A, C, G or T.
-func windowAsIs(dst, raw []byte, pos, n int, circular bool) bool {
-	for t := range dst {
-		i := pos + t
-		if circular {
-			i %= n
-		}
-		c := baseCode[raw[i]]
-		if c > 3 {
-			return false
-		}
-		dst[t] = upper[c]
-	}
-	return true
-}
-
-// windowRevComp writes the reverse complement of the len(dst) bases at pos.
-func windowRevComp(dst, raw []byte, pos, n int, circular bool) bool {
-	l := len(dst)
-	for t := range dst {
-		i := pos + l - 1 - t
-		if circular {
-			i %= n
-		}
-		c := baseCode[raw[i]]
-		if c > 3 {
-			return false
-		}
-		dst[t] = upper[3-c]
-	}
-	return true
-}
-
-// evaluate computes the indices of one retained pair. It reports false when the
-// pair must be dropped (no taxon amplified, or specificity under the threshold).
-func (ps *primerSet) evaluate(c *Corpus, taxa *Taxa, p PairParams, k pairKey, a *pairAcc,
-	recs []ampRecord, model *obithermo.Model) (PairStats, bool) {
-
-	exTaxa := map[int32]struct{}{}
-	coTaxa := map[int32]struct{}{}
-	ampTaxon := map[uint64]int32{} // amplicon -> its taxon, or -2 when shared
-	for _, r := range recs {
-		t := taxa.Of[r.seq]
-		if t < 0 {
-			continue
-		}
-		if c.Roles[r.seq] != Example {
-			coTaxa[t] = struct{}{}
-			continue
-		}
-		exTaxa[t] = struct{}{}
-		if prev, seen := ampTaxon[r.hash]; !seen {
-			ampTaxon[r.hash] = t
-		} else if prev != t {
-			ampTaxon[r.hash] = -2
-		}
-	}
-	if len(exTaxa) == 0 {
+	if res.targetTaxa == 0 {
 		return PairStats{}, false
 	}
-	identified := map[int32]struct{}{}
-	for _, t := range ampTaxon {
-		if t >= 0 {
-			identified[t] = struct{}{}
-		}
-	}
-
-	counts := c.Counts()
 	st := PairStats{
 		Examples:        int(a.examples),
 		CounterExamples: int(a.counters),
-		ExampleTaxa:     len(exTaxa),
-		CounterTaxa:     len(coTaxa),
-		Identified:      len(identified),
+		ExampleTaxa:     res.targetTaxa,
+		CounterTaxa:     res.counterTaxa,
+		Identified:      res.identified,
 		MinLen:          int(a.minLen),
 		MaxLen:          int(a.maxLen),
 		MeanLen:         float64(a.sumLen) / float64(a.amplicons),
@@ -359,12 +209,21 @@ func (ps *primerSet) evaluate(c *Corpus, taxa *Taxa, p PairParams, k pairKey, a
 		qfp = float32(a.counters) / float32(counts.CounterExamples)
 	}
 	st.Yule = q - qfp
-	st.Bc = float32(st.ExampleTaxa) / float32(taxa.ExampleTaxa)
-	st.Bs = float32(st.Identified) / float32(st.ExampleTaxa)
+	st.Bc = float32(res.targetTaxa) / float32(taxa.ExampleTaxa)
+	st.Bs = float32(res.identified) / float32(res.targetTaxa)
+	st.Bd = float32(res.discriminant) / float32(res.barcodes)
 	if st.Bs < float32(p.MinSpecificity) {
 		return PairStats{}, false
 	}
 
+	return ps.finish(st, p, k, a, minTm1, minTm2, model), true
+}
+
+// finish adds the primers, their composition and melting temperatures, and puts
+// the two primers in string order.
+func (ps *primerSet) finish(st PairStats, p PairParams, k pairKey, a *pairAcc,
+	minTm1, minTm2 float32, model *obithermo.Model) PairStats {
+
 	lp := p.PrimerLength
 	st.Primer1 = orientedWord(ps.primers[k.p1].Word, lp, k.a1)
 	st.Primer2 = orientedWord(ps.primers[k.p2].Word, lp, k.a2)
@@ -375,23 +234,11 @@ func (ps *primerSet) evaluate(c *Corpus, taxa *Taxa, p PairParams, k pairKey, a
 	if model != nil {
 		st.Tm1, _ = model.SelfTm([]byte(st.Primer1))
 		st.Tm2, _ = model.SelfTm([]byte(st.Primer2))
-		m1, m2 := float32(math.Inf(1)), float32(math.Inf(1))
-		for _, r := range recs {
-			if c.Roles[r.seq] != Example {
-				continue
-			}
-			if r.tm1 == r.tm1 { // not NaN
-				m1 = min(m1, r.tm1)
-			}
-			if r.tm2 == r.tm2 {
-				m2 = min(m2, r.tm2)
-			}
+		if !math.IsInf(float64(minTm1), 1) {
+			st.MinTm1 = float64(minTm1)
 		}
-		if !math.IsInf(float64(m1), 1) {
-			st.MinTm1 = float64(m1)
-		}
-		if !math.IsInf(float64(m2), 1) {
-			st.MinTm2 = float64(m2)
+		if !math.IsInf(float64(minTm2), 1) {
+			st.MinTm2 = float64(minTm2)
 		}
 	}
 
@@ -402,5 +249,5 @@ func (ps *primerSet) evaluate(c *Corpus, taxa *Taxa, p PairParams, k pairKey, a
 		st.Tm1, st.Tm2 = st.Tm2, st.Tm1
 		st.MinTm1, st.MinTm2 = st.MinTm2, st.MinTm1
 	}
-	return st, true
+	return st
 }

pkg/obiprimer/pairs_test.go

@@ -74,9 +74,18 @@ func readEcoRows(t *testing.T, path string) []ecoRow {
 // Opt-in end-to-end comparison with the complete ecoPrimers result tables of
 // the tutorial: same parameters (-e 3 -3 2 -l 30 -L 150 -c, -r taxid).
 //
-// Cyprinidae is reproduced exactly: its 3448 pairs agree on every column
-// (primers, good flags, sequence and taxon counts, Bc, Bs, lengths, GC, Tm and
-// lowest Tm). Teleostei finds the same 407 pairs, and 6 of them count one
+// Bs and the number of identified taxa are NOT compared: obiprimers defines
+// them differently (a taxon is identified only if all its barcodes are its own,
+// and every amplified sequence counts, targets or not), so they are lower. The
+// specificity threshold is switched off so that the pairs ecoPrimers dropped
+// with its own Bs are still produced, and only the pairs of ecoPrimers are
+// checked. What is checked: that every pair of ecoPrimers is found, with the
+// same good flags, sequence and taxon counts, Yule, Bc, lengths, GC, Tm and
+// lowest Tm; and that identified taxa never exceed those of ecoPrimers' more
+// lenient rule on the same amplified sequences.
+//
+// Cyprinidae is reproduced exactly on these columns for its 3448 pairs.
+// Teleostei finds the same 407 pairs, and 6 of them count one
 // sequence less than ecoPrimers (NC_033548, whose downstream site is
 // NGGGGAGGCAAGTCGTAA). The 3' check of ecoPrimers reads an N through the
 // shift register of extractSite, where the code 4 of N spills on the next base
@@ -143,7 +152,7 @@ func TestPairsAgainstEcoPrimers(t *testing.T) {
 			lap("approximate search")
 			got, _, err := FindPairs(corpus, taxa, wm, PairParams{
 				PrimerLength: 18, MinLength: 30, MaxLength: 150, ThreePrime: 2,
-				Sensitivity: 0.9, FalsePositive: 0.1, MinSpecificity: 0.6, Circular: true,
+				Sensitivity: 0.9, FalsePositive: 0.1, MinSpecificity: 0, Circular: true,
 			}, obithermo.DefaultModel())
 			if err != nil {
 				t.Fatal(err)
@@ -186,7 +195,7 @@ func TestPairsAgainstEcoPrimers(t *testing.T) {
 				// amplicon (NC_033548) whose 3' primer site ends with an N. We reject it,
 				// so the counts can be exactly one lower and nothing else may differ.
 				lower := func(got, want int) bool { return got == want || got == want-1 }
-				diverges := g.Examples != w.in || g.ExampleTaxa != w.intaxa || g.Identified != w.well
+				diverges := g.Examples != w.in || g.ExampleTaxa != w.intaxa
 				if diverges {
 					divergent++
 				}
@@ -196,15 +205,15 @@ func TestPairsAgainstEcoPrimers(t *testing.T) {
 				if !lower(g.ExampleTaxa, w.intaxa) || g.CounterTaxa != w.outtax {
 					note("taxa", w, "got %d/%d want %d/%d", g.ExampleTaxa, g.CounterTaxa, w.intaxa, w.outtax)
 				}
-				if !lower(g.Identified, w.well) {
-					note("identified", w, "got %d want %d", g.Identified, w.well)
+				if !diverges && g.Identified > w.well {
+					note("identified", w, "got %d, more than the %d of ecoPrimers' lenient rule", g.Identified, w.well)
 				}
 				tol := 0.0006
 				if diverges {
 					tol = 0.0021
 				}
-				if !near(float64(g.Yule), w.yule, tol) || !near(float64(g.Bc), w.bc, tol) || !near(float64(g.Bs), w.bs, tol) {
-					note("indices", w, "yule/bc/bs got %.4f/%.4f/%.4f want %.3f/%.3f/%.3f", g.Yule, g.Bc, g.Bs, w.yule, w.bc, w.bs)
+				if !near(float64(g.Yule), w.yule, tol) || !near(float64(g.Bc), w.bc, tol) {
+					note("indices", w, "yule/bc got %.4f/%.4f want %.3f/%.3f", g.Yule, g.Bc, w.yule, w.bc)
 				}
 				if g.MinLen != w.mind || g.MaxLen != w.maxd || !near(g.MeanLen, w.avg, 0.006) {
 					note("lengths", w, "got %d/%d/%.2f want %d/%d/%.2f", g.MinLen, g.MaxLen, g.MeanLen, w.mind, w.maxd, w.avg)
@@ -220,17 +229,11 @@ func TestPairsAgainstEcoPrimers(t *testing.T) {
 					note("mintm", w, "got %.2f/%.2f want %.1f/%.1f", g.MinTm1+0.15, g.MinTm2+0.15, w.mtm1, w.mtm2)
 				}
 			}
-			t.Logf("missing from ours: %d; extra in ours: %d; mismatching fields: %v; pairs one sequence below ecoPrimers: %d",
+			t.Logf("missing from ours: %d; pairs of ours that ecoPrimers dropped with its Bs: %d; mismatching fields: %v; pairs one sequence below ecoPrimers: %d",
 				missing, len(byKey), bad, divergent)
 			if divergent != tc.divergent {
 				t.Errorf("%d pairs differ from ecoPrimers by one sequence, expected %d", divergent, tc.divergent)
 			}
-			for k := range byKey {
-				if len(byKey) > 0 {
-					t.Errorf("extra pair %s", k)
-				}
-				break
-			}
 		})
 	}
 	_ = obikmer.DecodeKmer

pkg/obiprimer/resolution.go

@@ -0,0 +1,133 @@
+package obiprimer
+
+// The three quality indices of a pair, over the sequences it amplifies.
+//
+// Every sequence the pair amplifies gives a barcode (the amplicon between the
+// primers); all of them are taken into account, whatever their role: examples
+// (the targets), counterexamples and non-targets.
+//
+//   - Bc, coverage: target taxa amplified / target taxa.
+//   - Bs, specificity: target taxa identified / target taxa amplified. A target
+//     taxon is identified when ALL its barcodes belong to it alone: as soon as a
+//     barcode is shared with another taxon, target or not, counterexample or
+//     not, both taxa are badly identified.
+//   - Bd, barcode discrimination: the share of the barcodes carried by the
+//     targets that lead without ambiguity to one taxon.
+//
+// A sequence with no taxon at the chosen rank is an anonymous taxon, distinct
+// from every other: a barcode it shares with a taxon is ambiguous.
+
+// barcodeState is what is known of one barcode while the amplicons are scanned.
+type barcodeState struct {
+	taxon int32 // 1 + the first taxon met, 0 while none
+	flags uint8
+}
+
+const (
+	barcodeAmbiguous = 1 << iota // two distinct taxa carry it
+	barcodeAnonymous             // a sequence with no taxon carries it
+	barcodeTarget                // a target sequence with a taxon carries it
+)
+
+// ambiguous reports whether the barcode does not lead to a single taxon.
+func (b barcodeState) ambiguous() bool {
+	return b.flags&barcodeAmbiguous != 0 || (b.flags&barcodeAnonymous != 0 && b.taxon != 0)
+}
+
+// resolution is the outcome of the identification of the taxa of one pair.
+type resolution struct {
+	targetTaxa   int // target taxa amplified
+	counterTaxa  int // counterexample taxa amplified
+	identified   int // target taxa whose barcodes all belong to them alone
+	barcodes     int // distinct barcodes carried by the targets
+	discriminant int // of which lead to a single taxon
+}
+
+// resolve computes the resolution of one pair from its amplicons. The slices
+// need not be sorted; they are only read.
+func resolve(parts ...[]ampRecord) resolution {
+	var res resolution
+
+	state := map[uint64]barcodeState{}
+	targets := map[int32]struct{}{}
+	counters := map[int32]struct{}{}
+
+	for _, part := range parts {
+		for _, r := range part {
+			st := state[r.hash]
+			switch {
+			case r.taxon < 0:
+				st.flags |= barcodeAnonymous
+			case st.taxon == 0:
+				st.taxon = r.taxon + 1
+			case st.taxon != r.taxon+1:
+				st.flags |= barcodeAmbiguous
+			}
+			if r.taxon >= 0 {
+				switch r.role {
+				case Example:
+					st.flags |= barcodeTarget
+					targets[r.taxon] = struct{}{}
+				case CounterExample:
+					counters[r.taxon] = struct{}{}
+				}
+			}
+			state[r.hash] = st
+		}
+	}
+
+	bad := map[int32]struct{}{}
+	for _, part := range parts {
+		for _, r := range part {
+			if r.role == Example && r.taxon >= 0 && state[r.hash].ambiguous() {
+				bad[r.taxon] = struct{}{}
+			}
+		}
+	}
+
+	for _, st := range state {
+		if st.flags&barcodeTarget != 0 {
+			res.barcodes++
+			if !st.ambiguous() {
+				res.discriminant++
+			}
+		}
+	}
+	res.targetTaxa, res.counterTaxa = len(targets), len(counters)
+	res.identified = len(targets) - len(bad)
+	return res
+}
+
+// lenientIdentified is an upper bound of resolution.identified, computed on
+// the target sequences alone and with the lenient rule of ecoPrimers (a taxon
+// counts as soon as ONE of its barcodes belongs to it alone, among the targets).
+//
+// A taxon identified by resolve has all its barcodes alone among every amplified
+// sequence, hence among the targets: it has at least one such barcode and is
+// counted here too. So identified <= the second result, with the same number of
+// target taxa, which lets a pair be rejected on the specificity threshold without
+// gathering the amplicons of the non-target sequences.
+func lenientIdentified(parts ...[]ampRecord) (targetTaxa, identified int) {
+	owner := map[uint64]int32{} // barcode -> its taxon, or -1 when shared
+	targets := map[int32]struct{}{}
+	for _, part := range parts {
+		for _, r := range part {
+			if r.role != Example || r.taxon < 0 {
+				continue
+			}
+			targets[r.taxon] = struct{}{}
+			if prev, seen := owner[r.hash]; !seen {
+				owner[r.hash] = r.taxon
+			} else if prev != r.taxon {
+				owner[r.hash] = -1
+			}
+		}
+	}
+	alone := map[int32]struct{}{}
+	for _, t := range owner {
+		if t >= 0 {
+			alone[t] = struct{}{}
+		}
+	}
+	return len(targets), len(alone)
+}

pkg/obiprimer/resolution_test.go

@@ -0,0 +1,153 @@
+package obiprimer
+
+import (
+	"math/rand"
+	"testing"
+)
+
+// rec builds an amplicon record: barcode, taxon (-1: none at the rank), role.
+func rec(barcode uint64, taxon int32, role Role) ampRecord {
+	return ampRecord{hash: barcode, taxon: taxon, role: role}
+}
+
+func TestResolutionUserExample(t *testing.T) {
+	// A has the barcodes A1 and A2, B has A2 and B1. A2 is shared: neither A
+	// nor B is identified, but A1 and B1 lead to one species each.
+	const A, B = 0, 1
+	const A1, A2, B1 = 11, 12, 21
+	res := resolve([]ampRecord{
+		rec(A1, A, Example), rec(A2, A, Example),
+		rec(A2, B, Example), rec(B1, B, Example),
+	})
+	want := resolution{targetTaxa: 2, identified: 0, barcodes: 3, discriminant: 2}
+	if res != want {
+		t.Fatalf("got %+v, want %+v", res, want)
+	}
+}
+
+func TestResolutionAllUnique(t *testing.T) {
+	res := resolve([]ampRecord{rec(1, 0, Example), rec(2, 1, Example), rec(3, 2, Example)})
+	if res.identified != 3 || res.discriminant != 3 || res.barcodes != 3 || res.targetTaxa != 3 {
+		t.Errorf("all barcodes distinct: %+v", res)
+	}
+}
+
+func TestResolutionSameTaxonSharesItsBarcodeFreely(t *testing.T) {
+	// several sequences of one species with the same barcode are not ambiguous,
+	// and several barcodes for one species are fine when none is shared
+	res := resolve([]ampRecord{
+		rec(1, 0, Example), rec(1, 0, Example), rec(1, 0, Example),
+		rec(2, 0, Example),
+	})
+	if res.identified != 1 || res.barcodes != 2 || res.discriminant != 2 {
+		t.Errorf("%+v", res)
+	}
+}
+
+// The barcode of a target that is unique among the targets but shared with a
+// non-target, or with a counterexample, does not identify it.
+func TestResolutionNonTargetsCount(t *testing.T) {
+	for _, role := range []Role{Other, CounterExample} {
+		res := resolve([]ampRecord{
+			rec(1, 0, Example), // unique among the targets...
+			rec(1, 5, role),    // ...but shared with a non-target taxon
+			rec(2, 1, Example), // identified
+			rec(3, 6, role),    // a barcode of non-targets only
+		})
+		if res.targetTaxa != 2 || res.identified != 1 {
+			t.Errorf("role %d: %+v, want 2 target taxa and 1 identified", role, res)
+		}
+		// barcodes carried by the targets: 1 (ambiguous) and 2; barcode 3 leads to a non-target
+		if res.barcodes != 2 || res.discriminant != 1 {
+			t.Errorf("role %d: Bd counts %d/%d, want 1/2", role, res.discriminant, res.barcodes)
+		}
+	}
+}
+
+func TestResolutionAmplifiedTaxaCounts(t *testing.T) {
+	res := resolve([]ampRecord{
+		rec(1, 0, Example), rec(2, 1, CounterExample), rec(3, 1, CounterExample),
+		rec(4, 2, CounterExample), rec(5, 3, Other),
+	})
+	if res.targetTaxa != 1 || res.counterTaxa != 2 {
+		t.Errorf("target taxa %d (want 1), counterexample taxa %d (want 2: others are not counted)",
+			res.targetTaxa, res.counterTaxa)
+	}
+}
+
+// A sequence with no taxon at the rank is an anonymous taxon, distinct from all
+// the others: a barcode shared with it is ambiguous.
+func TestResolutionAnonymousTaxon(t *testing.T) {
+	res := resolve([]ampRecord{
+		rec(1, 0, Example), rec(1, -1, Other), // shared with an unknown: ambiguous
+		rec(2, 1, Example),
+	})
+	if res.identified != 1 || res.barcodes != 2 || res.discriminant != 1 {
+		t.Errorf("target sharing with an anonymous sequence: %+v", res)
+	}
+	// two anonymous sequences with the same barcode never count: no target carries it
+	res = resolve([]ampRecord{rec(7, -1, Other), rec(7, -1, Other), rec(8, 0, Example)})
+	if res.barcodes != 1 || res.discriminant != 1 || res.identified != 1 {
+		t.Errorf("anonymous barcodes alone: %+v", res)
+	}
+	// a target with no taxon is not a target taxon, and is anonymous too
+	res = resolve([]ampRecord{rec(1, -1, Example), rec(1, 0, Example), rec(2, 0, Example)})
+	if res.targetTaxa != 1 || res.identified != 0 {
+		t.Errorf("an anonymous target sequence sharing a barcode must make the taxon ambiguous: %+v", res)
+	}
+}
+
+func TestResolutionSeveralPartsAndOrder(t *testing.T) {
+	// the amplicons come as one slice per worker, in any order
+	a := []ampRecord{rec(1, 0, Example), rec(2, 1, Example)}
+	b := []ampRecord{rec(2, 2, Other), rec(3, 1, Example)}
+	if res := resolve(a, b); res.targetTaxa != 2 || res.identified != 1 {
+		t.Errorf("taxon 1 shares barcode 2 with taxon 2: %+v", res)
+	}
+	if res := resolve(b, a); res.targetTaxa != 2 || res.identified != 1 {
+		t.Errorf("order must not matter: %+v", res)
+	}
+}
+
+// ecoPrimers' lenient rule on the user's example: A and B each own one barcode,
+// so both count (the new rule counts none), and the bound is what lets a pair
+// be rejected early.
+func TestLenientIdentifiedUserExample(t *testing.T) {
+	recs := []ampRecord{
+		rec(11, 0, Example), rec(12, 0, Example),
+		rec(12, 1, Example), rec(21, 1, Example),
+	}
+	if targets, identified := lenientIdentified(recs); targets != 2 || identified != 2 {
+		t.Errorf("lenient rule: %d of %d, want 2 of 2", identified, targets)
+	}
+	if res := resolve(recs); res.identified != 0 {
+		t.Errorf("strict rule: %d identified, want 0", res.identified)
+	}
+}
+
+// The pruning of phase 2 is only exact if the bound never falls under the
+// true count, with the same number of target taxa.
+func TestLenientIdentifiedIsAnUpperBound(t *testing.T) {
+	rng := rand.New(rand.NewSource(3))
+	for trial := 0; trial < 3000; trial++ {
+		var recs []ampRecord
+		n := 1 + rng.Intn(40)
+		for range n {
+			role := []Role{Example, Example, Example, CounterExample, Other}[rng.Intn(5)]
+			recs = append(recs, rec(uint64(rng.Intn(12)), int32(rng.Intn(8))-1, role)) // taxon -1: none
+		}
+		// split in a few parts, as the workers do
+		cut := rng.Intn(len(recs) + 1)
+		res := resolve(recs[:cut], recs[cut:])
+		targets, lenient := lenientIdentified(recs[:cut], recs[cut:])
+		if targets != res.targetTaxa {
+			t.Fatalf("trial %d: %d target taxa for the bound, %d for resolve", trial, targets, res.targetTaxa)
+		}
+		if res.identified > lenient {
+			t.Fatalf("trial %d: identified %d exceeds the bound %d for %v", trial, res.identified, lenient, recs)
+		}
+		if res.discriminant > res.barcodes || res.identified > res.targetTaxa {
+			t.Fatalf("trial %d: inconsistent %+v", trial, res)
+		}
+	}
+}

pkg/obiprimer/roles.go

@@ -14,9 +14,12 @@ import (
 type Role uint8
 
 const (
-	// Ignored sequences are not used at all.
-	Ignored Role = iota
-	// Example sequences are those the primers must amplify.
+	// Other sequences are non-targets: nothing is required of them, so they
+	// count neither in the sensitivity nor in the false positive quorums, but
+	// the amplicons they give are taken into account to decide whether a target
+	// is identified (see Bs and Bd).
+	Other Role = iota
+	// Example sequences are the targets: the primers must amplify them.
 	Example
 	// CounterExample sequences are those the primers must not amplify.
 	CounterExample
@@ -28,7 +31,7 @@ const (
 //   - a sequence is an Example when it belongs to Restrict (all sequences when
 //     Restrict is nil) and not to Except;
 //   - otherwise it is a CounterExample when it belongs to Ignore or to Except;
-//   - otherwise it is Ignored.
+//   - otherwise it is Other.
 //
 // Nil predicates mean "no such constraint".
 type Selection struct {
@@ -48,7 +51,7 @@ func (s Selection) Role(seq *obiseq.BioSequence) Role {
 	case inExcept, s.Ignore != nil && s.Ignore(seq):
 		return CounterExample
 	default:
-		return Ignored
+		return Other
 	}
 }
 

pkg/obiprimer/search.go

@@ -87,7 +87,7 @@ func Search(c *Corpus, taxa *Taxa, p SearchParams) ([]PairStats, *SearchInfo, er
 		return nil, info, err
 	}
 	if p.NoMultiMatch {
-		wm, info.MultiMatch = DropMultiMatch(wm)
+		wm, info.MultiMatch = DropMultiMatch(wm, c)
 	}
 	for _, w := range wm {
 		info.Matches += len(w.Matches)

pkg/obiprimer/search_real_test.go

@@ -64,10 +64,14 @@ func tutorialParams(t *testing.T) SearchParams {
 // ecoPrimers -U on the Cyprinidae tutorial case (-e 3 -3 2 -l 30 -L 150 -c -r 7953 -U)
 // keeps 1366 pairs. testdata/ecoprimers_Cyprinidae_U.pairs.tsv holds, for each,
 // the primers, the amplified example and counterexample sequences, the amplified
-// example and counterexample taxa, and the identified taxa.
+// example and counterexample taxa, and the identified taxa. The identified taxa
+// are not compared (obiprimers defines them differently, see
+// TestPairsAgainstEcoPrimers) and the specificity threshold is off, so that
+// the pairs ecoPrimers dropped with its own Bs do not hide any of its pairs.
 func TestNoMultiMatchAgainstEcoPrimers(t *testing.T) {
 	corpus, taxa, _ := loadTutorial(t)
 	p := tutorialParams(t)
+	p.MinSpecificity = 0
 	p.NoMultiMatch = true
 	pairs, info, err := Search(corpus, taxa, p)
 	if err != nil {
@@ -91,23 +95,28 @@ func TestNoMultiMatchAgainstEcoPrimers(t *testing.T) {
 		want[p1+"/"+p2] = v
 	}
 
-	if len(pairs) != len(want) {
-		t.Errorf("%d pairs, ecoPrimers -U reports %d", len(pairs), len(want))
-	}
-	bad := 0
+	ours := map[string]PairStats{}
 	for _, g := range pairs {
-		v, ok := want[g.Primer1+"/"+g.Primer2]
+		ours[g.Primer1+"/"+g.Primer2] = g
+	}
+	bad, missing := 0, 0
+	for key, v := range want {
+		g, ok := ours[key]
 		if !ok {
-			t.Errorf("pair %s/%s is not in the ecoPrimers -U result", g.Primer1, g.Primer2)
+			if missing++; missing <= 3 {
+				t.Errorf("ecoPrimers -U pair %s is not found", key)
+			}
 			continue
 		}
-		got := [5]int{g.Examples, g.CounterExamples, g.ExampleTaxa, g.CounterTaxa, g.Identified}
-		if got != v {
+		got := [4]int{g.Examples, g.CounterExamples, g.ExampleTaxa, g.CounterTaxa}
+		if got != [4]int{v[0], v[1], v[2], v[3]} || g.Identified > v[4] {
 			if bad++; bad <= 3 {
-				t.Errorf("pair %s/%s: got %v, ecoPrimers %v", g.Primer1, g.Primer2, got, v)
+				t.Errorf("pair %s: got %v (identified %d), ecoPrimers %v", key, got, g.Identified, v)
 			}
 		}
 	}
+	t.Logf("%d pairs of ecoPrimers -U checked, %d missing, %d different; obiprimers has %d pairs (%d more, dropped by ecoPrimers' own Bs)",
+		len(want), missing, bad, len(pairs), len(pairs)-len(want)+missing)
 }
 
 var locRE = regexp.MustCompile(`(\d+)\.\.(\d+)`)
@@ -195,7 +204,9 @@ func TestCounterExamplesAgainstEcoPrimers(t *testing.T) {
 	if c := corpus.Counts(); c.Examples != 325 || c.CounterExamples != 7497 || taxa.CounterExamples != 7387 {
 		t.Fatalf("counts %+v, %d counterexample species (ecoPrimers: 325, 7497, 7387)", c, taxa.CounterExamples)
 	}
-	pairs, info, err := Search(corpus, taxa, tutorialParams(t))
+	pc := tutorialParams(t)
+	pc.MinSpecificity = 0 // ecoPrimers' own Bs filter is not reproduced, see TestPairsAgainstEcoPrimers
+	pairs, info, err := Search(corpus, taxa, pc)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -222,22 +233,26 @@ func TestCounterExamplesAgainstEcoPrimers(t *testing.T) {
 		want[p1+"/"+p2] = r
 	}
 
-	if len(pairs) != len(want) {
-		t.Errorf("%d pairs, ecoPrimers reports %d", len(pairs), len(want))
-	}
-	bad := 0
+	ours := map[string]PairStats{}
 	for _, g := range pairs {
-		w, ok := want[g.Primer1+"/"+g.Primer2]
+		ours[g.Primer1+"/"+g.Primer2] = g
+	}
+	bad, missing := 0, 0
+	for key, w := range want {
+		g, ok := ours[key]
 		if !ok {
-			t.Errorf("pair %s/%s is not in the ecoPrimers result", g.Primer1, g.Primer2)
+			if missing++; missing <= 3 {
+				t.Errorf("ecoPrimers pair %s is not found", key)
+			}
 			continue
 		}
 		good := string("bG"[b2i(g.Good1)]) + string("bG"[b2i(g.Good2)])
-		got := [5]int{g.Examples, g.CounterExamples, g.ExampleTaxa, g.CounterTaxa, g.Identified}
-		if got != w.v || good != w.good {
+		got := [4]int{g.Examples, g.CounterExamples, g.ExampleTaxa, g.CounterTaxa}
+		if got != [4]int{w.v[0], w.v[1], w.v[2], w.v[3]} || good != w.good || g.Identified > w.v[4] {
 			if bad++; bad <= 3 {
-				t.Errorf("pair %s/%s: got %s %v, ecoPrimers %s %v", g.Primer1, g.Primer2, good, got, w.good, w.v)
+				t.Errorf("pair %s: got %s %v (identified %d), ecoPrimers %s %v", key, good, got, g.Identified, w.good, w.v)
 			}
 		}
 	}
+	t.Logf("%d pairs of ecoPrimers checked, %d missing, %d different; obiprimers has %d pairs", len(want), missing, bad, len(pairs))
 }

pkg/obiprimer/search_test.go

@@ -74,7 +74,7 @@ func TestSearchFindsThePlantedPair(t *testing.T) {
 	if p == nil {
 		t.Fatalf("the planted pair was not found among %d pairs (%+v)", len(pairs), info)
 	}
-	if p.Examples != 8 || p.MinLen != 40 || p.MaxLen != 40 || p.Bs != 1 || p.Bc != 1 {
+	if p.Examples != 8 || p.MinLen != 40 || p.MaxLen != 40 || p.Bs != 1 || p.Bc != 1 || p.Bd != 1 {
 		t.Errorf("unexpected indices %+v", *p)
 	}
 	if p.RefLocation != "" {
@@ -183,9 +183,11 @@ func TestDropMultiMatch(t *testing.T) {
 		{Word: 2, Matches: []Match{{0, 5, 0, false}, {0, 50, 1, false}}}, // two in sequence 0
 		{Word: 3, Matches: []Match{{2, 5, 0, false}, {2, 5, 0, true}}},   // both strands of one sequence
 		{Word: 4}, // no match at all: kept
+		{Word: 5, Matches: []Match{{3, 1, 0, false}, {3, 8, 0, false}}}, // twice, but in a non-target: kept
 	}
-	kept, dropped := DropMultiMatch(wm)
-	if dropped != 2 || len(kept) != 2 || kept[0].Word != 1 || kept[1].Word != 4 {
+	c := corpusOf([]Role{Example, Example, CounterExample, Other}, "a", "a", "a", "a")
+	kept, dropped := DropMultiMatch(wm, c)
+	if dropped != 2 || len(kept) != 3 || kept[0].Word != 1 || kept[1].Word != 4 || kept[2].Word != 5 {
 		t.Errorf("kept %v, dropped %d", kept, dropped)
 	}
 }
@@ -197,3 +199,76 @@ func TestDropRepeatedWords(t *testing.T) {
 		t.Errorf("kept %v, dropped %d", kept, dropped)
 	}
 }
+
+// A non-target amplified with the very barcode of a target taxon makes this
+// target badly identified, although its barcode is unique among the targets;
+// it counts in neither the sensitivity nor the false positive quorums.
+func TestNonTargetSharingABarcodeLowersBsAndBd(t *testing.T) {
+	s := newSynthetic(8, -1, 11)
+
+	alone, _, err := Search(s.corpus, s.taxa, s.params())
+	if err != nil {
+		t.Fatal(err)
+	}
+	p0 := find(alone, s.f, s.r)
+	if p0 == nil || p0.Bs != 1 || p0.Bd != 1 {
+		t.Fatalf("without the non-target: %+v", p0)
+	}
+
+	// a copy of sequence 3, of another taxon, that is not a target
+	copy3 := append([]byte(nil), s.corpus.Seqs[3].Sequence()...)
+	s.corpus.Seqs = append(s.corpus.Seqs, obiseq.NewBioSequence("other", copy3, ""))
+	s.corpus.Roles = append(s.corpus.Roles, Other)
+	s.taxa.Of = append(s.taxa.Of, 100)
+
+	pairs, _, err := Search(s.corpus, s.taxa, s.params())
+	if err != nil {
+		t.Fatal(err)
+	}
+	p := find(pairs, s.f, s.r)
+	if p == nil {
+		t.Fatal("the planted pair must still be found")
+	}
+	if p.Examples != 8 || p.ExampleTaxa != 8 || p.Bc != 1 {
+		t.Errorf("the non-target must not change the quorums nor Bc: %+v", *p)
+	}
+	if p.Identified != 7 || p.Bs != 0.875 {
+		t.Errorf("target 3 shares its barcode with the non-target: identified %d, Bs %v (want 7, 0.875)", p.Identified, p.Bs)
+	}
+	if p.Bd != 0.875 {
+		t.Errorf("Bd %v, want 7 discriminant barcodes of 8", p.Bd)
+	}
+	if p.CounterTaxa != 0 || p.CounterExamples != 0 {
+		t.Errorf("a non-target is not a counterexample: %+v", *p)
+	}
+
+	// Now counterexamples: two copies (of sequences 3 and 5) that the pair amplifies and
+	// one unrelated sequence that it does not. 2 of 3 amplified is under the 0.9 quorum.
+	s.corpus.Roles[len(s.corpus.Roles)-1] = CounterExample
+	copy5 := append([]byte(nil), s.corpus.Seqs[5].Sequence()...)
+	s.corpus.Seqs = append(s.corpus.Seqs, obiseq.NewBioSequence("counter5", copy5, ""))
+	s.corpus.Roles = append(s.corpus.Roles, CounterExample)
+	s.taxa.Of = append(s.taxa.Of, 101)
+	unrelated := []byte(strings.ToLower(randDNA(rand.New(rand.NewSource(5)), 300)))
+	s.corpus.Seqs = append(s.corpus.Seqs, obiseq.NewBioSequence("unrelated", unrelated, ""))
+	s.corpus.Roles = append(s.corpus.Roles, CounterExample)
+	s.taxa.Of = append(s.taxa.Of, 102)
+
+	pq := s.params()
+	pq.FalsePositive = 0.9
+	pairs, _, err = Search(s.corpus, s.taxa, pq)
+	if err != nil {
+		t.Fatal(err)
+	}
+	pc := find(pairs, s.f, s.r)
+	if pc == nil {
+		t.Fatal("the pair amplifies 2 of 3 counterexamples (< 0.9): it must be retained")
+	}
+	if pc.CounterExamples != 2 || pc.CounterTaxa != 2 {
+		t.Errorf("counterexamples amplified %d (taxa %d), want 2 and 2", pc.CounterExamples, pc.CounterTaxa)
+	}
+	if pc.Identified != 6 || pc.Bd != 0.75 {
+		t.Errorf("targets 3 and 5 share their barcode with a counterexample: identified %d (want 6), Bd %v (want 0.75)",
+			pc.Identified, pc.Bd)
+	}
+}

pkg/obiprimer/taxa.go

@@ -58,9 +58,10 @@ func (c *Corpus) AssignTaxa(taxonomy *obitax.Taxonomy, rank string) (*Taxa, erro
 			t.nodes = append(t.nodes, taxon)
 		}
 		t.Of[i] = id
-		if c.Roles[i] == Example {
+		switch c.Roles[i] {
+		case Example:
 			exSeen[id] = struct{}{}
-		} else {
+		case CounterExample:
 			coSeen[id] = struct{}{}
 		}
 	}

pkg/obiprimer/words_test.go

@@ -27,18 +27,18 @@ func TestSelectionRoles(t *testing.T) {
 	}{
 		{"nothing set: everything is an example",
 			Selection{}, map[string]Role{"r1": Example, "x1": Example, "i1": Example, "o1": Example}},
-		{"restrict only: the rest is dropped",
+		{"restrict only: the rest are non-targets",
 			Selection{Restrict: idPrefix("r")},
-			map[string]Role{"r1": Example, "x1": Ignored, "i1": Ignored, "o1": Ignored}},
+			map[string]Role{"r1": Example, "x1": Other, "i1": Other, "o1": Other}},
 		{"restrict + ignore",
 			Selection{Restrict: idPrefix("r"), Ignore: idPrefix("i")},
-			map[string]Role{"r1": Example, "i1": CounterExample, "o1": Ignored}},
+			map[string]Role{"r1": Example, "i1": CounterExample, "o1": Other}},
 		{"exception inside restrict becomes a counterexample",
 			Selection{Restrict: idPrefix("r").Or(idPrefix("x")), Except: idPrefix("x")},
-			map[string]Role{"r1": Example, "x1": CounterExample, "o1": Ignored}},
+			map[string]Role{"r1": Example, "x1": CounterExample, "o1": Other}},
 		{"example wins over ignore when both match",
 			Selection{Restrict: idPrefix("r"), Ignore: idPrefix("r").Or(idPrefix("i"))},
-			map[string]Role{"r1": Example, "i1": CounterExample, "o1": Ignored}},
+			map[string]Role{"r1": Example, "i1": CounterExample, "o1": Other}},
 	}
 	for _, tc := range cases {
 		for id, want := range tc.want {
@@ -116,7 +116,7 @@ func TestFindStrictWords(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if res.Counts != (Counts{Examples: 10, CounterExamples: 1, Ignored: 1}) {
+	if res.Counts != (Counts{Examples: 10, CounterExamples: 1, Others: 1}) {
 		t.Fatalf("counts = %+v", res.Counts)
 	}
 	if res.MinSequences != 7 {