feat: add JSON input support for biological sequences

Introduce a streaming JSON parser that decodes biological sequences using `goccy/go-json` with configurable batching to minimize memory overhead. Extend the CLI, file suffix filters, and MIME type detection to automatically recognize and route JSON inputs. Refactor header parsing into a centralized switch-case handler for improved maintainability.

pkg/obiformats/fastseq_json_header.go

@@ -199,47 +199,13 @@ func _parse_json_array_interface(str []byte) ([]interface{}, error) {
 	return values, nil
 }
 
-func _parse_json_header_(header string, sequence *obiseq.BioSequence) string {
+// _parse_json_annotation_field parses a single key/value pair coming from a
+// JSON object (either a FASTA/FASTQ inline JSON header, or the "annotations"
+// field of a JSON sequence record) and applies it to the sequence, special
+// casing the well-known OBITools attributes (id, definition, count, taxid,
+// obiclean_*, merged_*).
+func _parse_json_annotation_field(key []byte, value []byte, dataType jsonparser.ValueType, sequence *obiseq.BioSequence) error {
 	annotations := sequence.Annotations()
-	start := -1
-	stop := -1
-	level := 0
-	lh := len(header)
-	inquote := false
-
-	for i := 0; (i < lh) && (stop < 0); i++ {
-		// fmt.Printf("[%d,%d-%d] : %d (%c) (%d,%c)\n", i, start, stop, header[i], header[i], '{', '{')
-		if level == 0 && header[i] == '{' && !inquote {
-			start = i
-		}
-
-		// TODO: escaped double quotes are not considered
-		if start > -1 && header[i] == '"' {
-			inquote = !inquote
-		}
-
-		if header[i] == '{' && !inquote {
-			level++
-		}
-
-		if header[i] == '}' && !inquote {
-			level--
-		}
-
-		if start >= 0 && level == 0 {
-			stop = i
-		}
-
-	}
-
-	if start < 0 || stop < 0 {
-		return header
-	}
-
-	stop++
-
-	jsonparser.ObjectEach(obiutils.UnsafeBytes(header[start:stop]),
-		func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) error {
 	var err error
 
 	skey := obiutils.UnsafeString(key)
@@ -335,6 +301,49 @@ func _parse_json_header_(header string, sequence *obiseq.BioSequence) string {
 	}
 
 	return err
+}
+
+func _parse_json_header_(header string, sequence *obiseq.BioSequence) string {
+	start := -1
+	stop := -1
+	level := 0
+	lh := len(header)
+	inquote := false
+
+	for i := 0; (i < lh) && (stop < 0); i++ {
+		// fmt.Printf("[%d,%d-%d] : %d (%c) (%d,%c)\n", i, start, stop, header[i], header[i], '{', '{')
+		if level == 0 && header[i] == '{' && !inquote {
+			start = i
+		}
+
+		// TODO: escaped double quotes are not considered
+		if start > -1 && header[i] == '"' {
+			inquote = !inquote
+		}
+
+		if header[i] == '{' && !inquote {
+			level++
+		}
+
+		if header[i] == '}' && !inquote {
+			level--
+		}
+
+		if start >= 0 && level == 0 {
+			stop = i
+		}
+
+	}
+
+	if start < 0 || stop < 0 {
+		return header
+	}
+
+	stop++
+
+	jsonparser.ObjectEach(obiutils.UnsafeBytes(header[start:stop]),
+		func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) error {
+			return _parse_json_annotation_field(key, value, dataType, sequence)
 		},
 	)
 

pkg/obiformats/json_read.go

@@ -0,0 +1,147 @@
+package obiformats
+
+import (
+	"io"
+	"os"
+	"path"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obidefault"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils"
+	"github.com/buger/jsonparser"
+	"github.com/goccy/go-json"
+	log "github.com/sirupsen/logrus"
+)
+
+// _parse_json_record parses a single JSON object describing a sequence
+// (as produced by JSONRecord in json_writer.go) into a *obiseq.BioSequence.
+func _parse_json_record(raw []byte, shift byte) *obiseq.BioSequence {
+	sequence := obiseq.NewEmptyBioSequence(0)
+
+	if id, err := jsonparser.GetString(raw, "id"); err == nil {
+		sequence.SetId(id)
+	}
+
+	if seq, err := jsonparser.GetString(raw, "sequence"); err == nil {
+		sequence.SetSequence([]byte(seq))
+	}
+
+	if qual, err := jsonparser.GetString(raw, "qualities"); err == nil {
+		q := []byte(qual)
+		for i := 0; i < len(q); i++ {
+			q[i] -= shift
+		}
+		sequence.SetQualities(q)
+	}
+
+	if annot, dataType, _, err := jsonparser.Get(raw, "annotations"); err == nil && dataType == jsonparser.Object {
+		jsonparser.ObjectEach(annot,
+			func(key []byte, value []byte, valType jsonparser.ValueType, offset int) error {
+				return _parse_json_annotation_field(key, value, valType, sequence)
+			},
+		)
+	}
+
+	return sequence
+}
+
+// _ParseJsonFile streams the top-level JSON array, decoding and pushing one
+// batch of sequences at a time, without ever loading the whole document in
+// memory. Only one raw record at a time is buffered by the decoder.
+func _ParseJsonFile(source string,
+	reader io.Reader,
+	out obiiter.IBioSequence,
+	shift byte,
+	batchSize int) {
+
+	dec := json.NewDecoder(reader)
+
+	if _, err := dec.Token(); err != nil {
+		if err == io.EOF {
+			out.Done()
+			return
+		}
+		log.Fatalf("cannot parse JSON data: %v", err)
+	}
+
+	slice := obiseq.MakeBioSequenceSlice()
+	o := 0
+
+	for dec.More() {
+		var raw json.RawMessage
+
+		if err := dec.Decode(&raw); err != nil {
+			log.Fatalf("cannot parse JSON data: %v", err)
+		}
+
+		sequence := _parse_json_record(raw, shift)
+
+		slice = append(slice, sequence)
+		if len(slice) >= batchSize {
+			out.Push(obiiter.MakeBioSequenceBatch(source, o, slice))
+			o++
+			slice = obiseq.MakeBioSequenceSlice()
+		}
+	}
+
+	if len(slice) > 0 {
+		out.Push(obiiter.MakeBioSequenceBatch(source, o, slice))
+	}
+
+	out.Done()
+}
+
+func ReadJSON(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, error) {
+
+	opt := MakeOptions(options)
+	out := obiiter.MakeIBioSequence()
+
+	out.Add(1)
+	go _ParseJsonFile(opt.Source(),
+		reader,
+		out,
+		obidefault.ReadQualitiesShift(),
+		opt.BatchSize())
+
+	go func() {
+		out.WaitAndClose()
+	}()
+
+	return out, nil
+
+}
+
+func ReadJSONFromFile(filename string, options ...WithOption) (obiiter.IBioSequence, error) {
+
+	options = append(options, OptionsSource(obiutils.RemoveAllExt((path.Base(filename)))))
+	file, err := obiutils.Ropen(filename)
+
+	if err == obiutils.ErrNoContent {
+		log.Infof("file %s is empty", filename)
+		return ReadEmptyFile(options...)
+	}
+
+	if err != nil {
+		return obiiter.NilIBioSequence, err
+	}
+
+	return ReadJSON(file, options...)
+}
+
+func ReadJSONFromStdin(reader io.Reader, options ...WithOption) (obiiter.IBioSequence, error) {
+	options = append(options, OptionsSource(obiutils.RemoveAllExt("stdin")))
+	input, err := obiutils.Buf(os.Stdin)
+
+	if err == obiutils.ErrNoContent {
+		log.Infof("stdin is empty")
+		return ReadEmptyFile(options...)
+	}
+
+	if err != nil {
+		log.Fatalf("open file error: %v", err)
+		return obiiter.NilIBioSequence, err
+	}
+
+	return ReadJSON(input, options...)
+}

pkg/obiformats/universal_read.go

@@ -145,6 +145,8 @@ func ReadSequencesFromFile(filename string,
 		return ReadGenbank(reader, options...)
 	case "text/csv":
 		return ReadCSV(reader, options...)
+	case "application/json":
+		return ReadJSON(reader, options...)
 	default:
 		log.Fatalf("File %s has guessed format %s which is not yet implemented",
 			filename, mime.String())

pkg/obitools/obiconvert/options.go

@@ -24,6 +24,7 @@ var __input_genbank_format__ = false
 var __input_fastq_format__ = false
 var __input_fasta_format__ = false
 var __input_csv_format__ = false
+var __input_json_format__ = false
 
 var __output_in_fasta__ = false
 var __output_in_fastq__ = false
@@ -71,6 +72,9 @@ func InputOptionSet(options *getoptions.GetOpt) {
 	options.BoolVar(&__input_csv_format__, "csv", __input_csv_format__,
 		options.Description("Read data following the CSV format."))
 
+	options.BoolVar(&__input_json_format__, "json", __input_json_format__,
+		options.Description("Read data following the JSON format."))
+
 	options.BoolVar(&__no_ordered_input__, "no-order", __no_ordered_input__,
 		options.Description("When several input files are provided, "+
 			"indicates that there is no order among them."))
@@ -158,6 +162,8 @@ func CLIInputFormat() string {
 		return "genbank"
 	case __input_csv_format__:
 		return "csv"
+	case __input_json_format__:
+		return "json"
 	default:
 		return "guessed"
 	}

pkg/obitools/obiconvert/sequence_reader.go

@@ -73,7 +73,9 @@ func ExpandListOfFiles(check_ext bool, filenames ...string) ([]string, error) {
 						strings.HasSuffix(path, "dat") ||
 						strings.HasSuffix(path, "dat.gz") ||
 						strings.HasSuffix(path, "ecopcr") ||
-						strings.HasSuffix(path, "ecopcr.gz") {
+						strings.HasSuffix(path, "ecopcr.gz") ||
+						strings.HasSuffix(path, "json") ||
+						strings.HasSuffix(path, "json.gz") {
 						log.Debugf("Appending %s file\n", path)
 						list_of_files.Add(path)
 					}
@@ -142,6 +144,8 @@ func CLIReadBioSequences(filenames ...string) (obiiter.IBioSequence, error) {
 			iterator, err = obiformats.ReadFastq(os.Stdin, opts...)
 		case "csv":
 			iterator, err = obiformats.ReadCSV(os.Stdin, opts...)
+		case "json":
+			iterator, err = obiformats.ReadJSON(os.Stdin, opts...)
 		default:
 			iterator, err = obiformats.ReadSequencesFromStdin(opts...)
 		}
@@ -163,6 +167,8 @@ func CLIReadBioSequences(filenames ...string) (obiiter.IBioSequence, error) {
 			reader = obiformats.ReadFastaFromFile
 		case "csv":
 			reader = obiformats.ReadCSVFromFile
+		case "json":
+			reader = obiformats.ReadJSONFromFile
 		case "ecopcr":
 			reader = obiformats.ReadEcoPCRFromFile
 		case "embl":

pkg/obiutils/mimetypes.go

@@ -102,6 +102,11 @@ func RegisterOBIMimeType() {
 			return ok
 		}
 
+		jsonDetector := func(raw []byte, limit uint32) bool {
+			raw = bytes.TrimLeft(raw, " \t\r\n")
+			return len(raw) > 0 && (raw[0] == '[' || raw[0] == '{')
+		}
+
 		mimetype.Lookup("text/plain").Extend(fastaDetector, "text/fasta", ".fasta")
 		mimetype.Lookup("text/plain").Extend(fastqDetector, "text/fastq", ".fastq")
 		mimetype.Lookup("text/plain").Extend(ecoPCR2Detector, "text/ecopcr2", ".ecopcr")
@@ -115,6 +120,7 @@ func RegisterOBIMimeType() {
 		mimetype.Lookup("application/octet-stream").Extend(genbankDetector, "text/genbank", ".seq")
 		mimetype.Lookup("application/octet-stream").Extend(emblDetector, "text/embl", ".dat")
 		mimetype.Lookup("application/octet-stream").Extend(csv, "text/csv", ".csv")
+		mimetype.Lookup("application/octet-stream").Extend(jsonDetector, "application/json", ".json")
 	}
 	__obimimetype_registred__ = true
 }