Refactor similarity engine and implement parallel clustering

Introduces parallel processing for clustering, refines scoring mechanisms, and expands test coverage for the alignment functionality.

This change introduces parallel processing capabilities within the clustering logic, updates the core similarity engine with new data structures and scoring methods, and implements a comprehensive test suite for the alignment tool. It also adds new command-line options for configuring alignment parameters and reference lengths.

pkg/obiformats/batch_of_files_reader.go

@@ -48,6 +48,14 @@ func ReadSequencesBatchFromFiles(filenames []string,
 
 				log.Printf("Start reading of file : %s", filename)
 
+				// The batches of a file reach us in any order (parallel parsing
+				// of the headers). They are numbered here in arrival order, so
+				// the order of the file must be restored first, unless the
+				// order of the data is not wanted (several concurrent readers).
+				if concurrent_readers == 1 {
+					iter = iter.SortBatches()
+				}
+
 				for iter.Next() {
 					batch := iter.Get()
 					batchiter.Push(batch.Reorder(nextCounter()))

pkg/obiformats/batch_of_files_reader_test.go

@@ -0,0 +1,38 @@
+package obiformats
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// With a single reader, the sequences of several files must come out in the
+// order of the files, whatever the order in which the batches are parsed.
+func TestReadSequencesBatchFromFilesKeepsOrder(t *testing.T) {
+	dir := t.TempDir()
+	var names []string
+	for _, prefix := range []string{"a", "b"} {
+		var sb strings.Builder
+		for i := 0; i < 25000; i++ {
+			fmt.Fprintf(&sb, ">%s%07d {\"k\":%d}\n%s\n", prefix, i, i, strings.Repeat("acgt", 40))
+		}
+		name := filepath.Join(dir, prefix+".fasta")
+		if err := os.WriteFile(name, []byte(sb.String()), 0o600); err != nil {
+			t.Fatal(err)
+		}
+		names = append(names, name)
+	}
+
+	it := ReadSequencesBatchFromFiles(names, nil, 1)
+	_, db := it.Load()
+	if len(db) != 50000 {
+		t.Fatalf("expected 50000 sequences, got %d", len(db))
+	}
+	for i := 1; i < len(db); i++ {
+		if db[i-1].Id() >= db[i].Id() {
+			t.Fatalf("out of order at %d: %s then %s", i, db[i-1].Id(), db[i].Id())
+		}
+	}
+}