mirror of
https://github.com/metabarcoding/obitools4.git
synced 2025-06-29 16:20:46 +00:00
Patch a genbank & embl parsing error.
Former-commit-id: 060e80f42d176e6982e63d1128993fbcb4ad395f
This commit is contained in:
@ -31,12 +31,18 @@ var _seqlenght_rx = regexp.MustCompile(" +([0-9]+) bp")
|
||||
func _ParseGenbankFile(source string,
|
||||
input <-chan _FileChunk, out obiiter.IBioSequence,
|
||||
chunck_order func() int) {
|
||||
var err error
|
||||
state := inHeader
|
||||
previous_chunk := -1
|
||||
|
||||
for chunks := range input {
|
||||
// log.Debugln("Chunk size", (chunks.raw.(*bytes.Buffer)).Len())
|
||||
scanner := bufio.NewScanner(chunks.raw)
|
||||
|
||||
if state != inHeader {
|
||||
log.Fatalf("Unexpected state %d starting new chunk (id = %d, previous_chunk = %d)",
|
||||
state, chunks.order, previous_chunk)
|
||||
}
|
||||
|
||||
previous_chunk = chunks.order
|
||||
scanner := bufio.NewReader(chunks.raw)
|
||||
sequences := make(obiseq.BioSequenceSlice, 0, 100)
|
||||
sumlength := 0
|
||||
id := ""
|
||||
@ -48,90 +54,155 @@ func _ParseGenbankFile(source string,
|
||||
taxid := 1
|
||||
nl := 0
|
||||
sl := 0
|
||||
for scanner.Scan() {
|
||||
var line string
|
||||
for bline, is_prefix, err := scanner.ReadLine(); err != io.EOF; bline, is_prefix, err = scanner.ReadLine() {
|
||||
nl++
|
||||
line := scanner.Text()
|
||||
switch {
|
||||
case state == inDefinition && !strings.HasPrefix(line, " "):
|
||||
state = inEntry
|
||||
fallthrough
|
||||
case strings.HasPrefix(line, "LOCUS "):
|
||||
state = inEntry
|
||||
id = strings.SplitN(line[12:], " ", 2)[0]
|
||||
match_length := _seqlenght_rx.FindStringSubmatch(line)
|
||||
if len(match_length) > 0 {
|
||||
lseq, err = strconv.Atoi(match_length[1])
|
||||
if err != nil {
|
||||
lseq = -1
|
||||
line = string(bline)
|
||||
if is_prefix || len(line) > 100 {
|
||||
log.Fatalf("Chunk %d : Line too long: %s", chunks.order, line)
|
||||
}
|
||||
processed := false
|
||||
for !processed {
|
||||
switch {
|
||||
|
||||
case strings.HasPrefix(line, "LOCUS "):
|
||||
if state != inHeader {
|
||||
log.Fatalf("Unexpected state %d while reading LOCUS: %s", state, line)
|
||||
}
|
||||
}
|
||||
if lseq > 0 {
|
||||
seqBytes = bytes.NewBuffer(obiseq.GetSlice(lseq + 20))
|
||||
} else {
|
||||
seqBytes = new(bytes.Buffer)
|
||||
}
|
||||
case strings.HasPrefix(line, "SOURCE "):
|
||||
scientificName = strings.TrimSpace(line[12:])
|
||||
case strings.HasPrefix(line, "DEFINITION "):
|
||||
defBytes.WriteString(strings.TrimSpace(line[12:]))
|
||||
state = inDefinition
|
||||
case strings.HasPrefix(line, "FEATURES "):
|
||||
featBytes.WriteString(line)
|
||||
state = inFeature
|
||||
case strings.HasPrefix(line, "ORIGIN"):
|
||||
state = inSequence
|
||||
case line == "//":
|
||||
// log.Debugln("Total lines := ", nl)
|
||||
sequence := obiseq.NewBioSequence(id,
|
||||
seqBytes.Bytes(),
|
||||
defBytes.String())
|
||||
sequence.SetSource(source)
|
||||
state = inHeader
|
||||
id = strings.SplitN(line[12:], " ", 2)[0]
|
||||
match_length := _seqlenght_rx.FindStringSubmatch(line)
|
||||
if len(match_length) > 0 {
|
||||
lseq, err = strconv.Atoi(match_length[1])
|
||||
if err != nil {
|
||||
lseq = -1
|
||||
}
|
||||
}
|
||||
if lseq > 0 {
|
||||
seqBytes = bytes.NewBuffer(obiseq.GetSlice(lseq + 20))
|
||||
} else {
|
||||
seqBytes = new(bytes.Buffer)
|
||||
}
|
||||
state = inEntry
|
||||
processed = true
|
||||
|
||||
sequence.SetFeatures(featBytes.Bytes())
|
||||
case strings.HasPrefix(line, "DEFINITION "):
|
||||
if state != inEntry {
|
||||
log.Fatalf("Unexpected state %d while reading DEFINITION: %s", state, line)
|
||||
}
|
||||
defBytes.WriteString(strings.TrimSpace(line[12:]))
|
||||
state = inDefinition
|
||||
processed = true
|
||||
|
||||
annot := sequence.Annotations()
|
||||
annot["scientific_name"] = scientificName
|
||||
annot["taxid"] = taxid
|
||||
// log.Println(FormatFasta(sequence, FormatFastSeqJsonHeader))
|
||||
// log.Debugf("Read sequences %s: %dbp (%d)", sequence.Id(),
|
||||
// sequence.Len(), seqBytes.Len())
|
||||
case state == inDefinition:
|
||||
if strings.HasPrefix(line, " ") {
|
||||
defBytes.WriteByte(' ')
|
||||
defBytes.WriteString(strings.TrimSpace(line[12:]))
|
||||
processed = true
|
||||
} else {
|
||||
state = inEntry
|
||||
}
|
||||
|
||||
sequences = append(sequences, sequence)
|
||||
sumlength += sequence.Len()
|
||||
case strings.HasPrefix(line, "SOURCE "):
|
||||
if state != inEntry {
|
||||
log.Fatalf("Unexpected state %d while reading SOURCE: %s", state, line)
|
||||
}
|
||||
scientificName = strings.TrimSpace(line[12:])
|
||||
processed = true
|
||||
|
||||
if len(sequences) == 100 || sumlength > 1e7 {
|
||||
out.Push(obiiter.MakeBioSequenceBatch(chunck_order(), sequences))
|
||||
sequences = make(obiseq.BioSequenceSlice, 0, 100)
|
||||
sumlength = 0
|
||||
}
|
||||
defBytes = bytes.NewBuffer(obiseq.GetSlice(200))
|
||||
featBytes = new(bytes.Buffer)
|
||||
nl = 0
|
||||
sl = 0
|
||||
default:
|
||||
switch state {
|
||||
case inDefinition:
|
||||
defBytes.WriteByte(' ')
|
||||
defBytes.WriteString(strings.TrimSpace(line[5:]))
|
||||
case inFeature:
|
||||
featBytes.WriteByte('\n')
|
||||
case strings.HasPrefix(line, "FEATURES "):
|
||||
if state != inEntry {
|
||||
log.Fatalf("Unexpected state %d while reading FEATURES: %s", state, line)
|
||||
}
|
||||
featBytes.WriteString(line)
|
||||
if strings.HasPrefix(line, ` /db_xref="taxon:`) {
|
||||
taxid, _ = strconv.Atoi(strings.SplitN(line[37:], `"`, 2)[0])
|
||||
state = inFeature
|
||||
processed = true
|
||||
|
||||
case strings.HasPrefix(line, "ORIGIN"):
|
||||
if state != inFeature {
|
||||
log.Fatalf("Unexpected state %d while reading ORIGIN: %s", state, line)
|
||||
}
|
||||
case inSequence:
|
||||
state = inSequence
|
||||
processed = true
|
||||
|
||||
case line == "//":
|
||||
|
||||
if state != inSequence {
|
||||
log.Fatalf("Unexpected state %d while reading end of record %s", state, id)
|
||||
}
|
||||
// log.Debugln("Total lines := ", nl)
|
||||
if id == "" {
|
||||
log.Warn("Empty id when parsing genbank file")
|
||||
}
|
||||
if seqBytes.Len() == 0 {
|
||||
log.Warn("Empty sequence when parsing genbank file")
|
||||
}
|
||||
|
||||
log.Debugf("End of sequence %s: %dbp ", id, seqBytes.Len())
|
||||
|
||||
sequence := obiseq.NewBioSequence(id,
|
||||
seqBytes.Bytes(),
|
||||
defBytes.String())
|
||||
sequence.SetSource(source)
|
||||
sequence.SetFeatures(featBytes.Bytes())
|
||||
|
||||
annot := sequence.Annotations()
|
||||
annot["scientific_name"] = scientificName
|
||||
annot["taxid"] = taxid
|
||||
// log.Println(FormatFasta(sequence, FormatFastSeqJsonHeader))
|
||||
// log.Debugf("Read sequences %s: %dbp (%d)", sequence.Id(),
|
||||
// sequence.Len(), seqBytes.Len())
|
||||
|
||||
sequences = append(sequences, sequence)
|
||||
sumlength += sequence.Len()
|
||||
|
||||
if len(sequences) == 100 || sumlength > 1e7 {
|
||||
log.Debugln("Pushing sequences")
|
||||
out.Push(obiiter.MakeBioSequenceBatch(chunck_order(), sequences))
|
||||
sequences = make(obiseq.BioSequenceSlice, 0, 100)
|
||||
sumlength = 0
|
||||
}
|
||||
|
||||
defBytes = bytes.NewBuffer(obiseq.GetSlice(200))
|
||||
featBytes = new(bytes.Buffer)
|
||||
nl = 0
|
||||
sl = 0
|
||||
|
||||
state = inHeader
|
||||
processed = true
|
||||
|
||||
case state == inSequence:
|
||||
log.Debugf("Chunk %d : Genbank: line %d, state = %d : %s", chunks.order, nl, state, line)
|
||||
|
||||
sl++
|
||||
parts := strings.SplitN(line[10:], " ", 7)
|
||||
parts := strings.SplitN(line[10:], " ", 6)
|
||||
lparts := len(parts)
|
||||
for i := 0; i < lparts; i++ {
|
||||
seqBytes.WriteString(parts[i])
|
||||
}
|
||||
processed = true
|
||||
|
||||
default:
|
||||
switch state {
|
||||
case inFeature:
|
||||
featBytes.WriteByte('\n')
|
||||
featBytes.WriteString(line)
|
||||
if strings.HasPrefix(line, ` /db_xref="taxon:`) {
|
||||
taxid, _ = strconv.Atoi(strings.SplitN(line[37:], `"`, 2)[0])
|
||||
}
|
||||
processed = true
|
||||
case inHeader:
|
||||
processed = true
|
||||
case inEntry:
|
||||
processed = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
log.Debugf("End of chunk %d : %s", chunks.order, line)
|
||||
if len(sequences) > 0 {
|
||||
log.Debugln("Pushing sequences")
|
||||
out.Push(obiiter.MakeBioSequenceBatch(chunck_order(), sequences))
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user