package obiformats import ( "bufio" "bytes" "io" "path" "strconv" "strings" log "github.com/sirupsen/logrus" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq" "git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiutils" ) var _FileChunkSize = 1 << 28 type _FileChunk struct { raw io.Reader order int } // _EndOfLastEntry finds the index of the last entry in the given byte slice 'buff' // using a pattern match of the form: // ?//? // where and are the ASCII codes for carriage return and line feed, // respectively. The function returns the index of the end of the last entry // or -1 if no match is found. // // Arguments: // buff []byte - a byte slice to search for the end of the last entry // // Returns: // int - the index of the end of the last entry or -1 if no match is found. func _EndOfLastEntry(buff []byte) int { // 6 5 43 2 1 // ?//? var i int var state = 0 var start = 0 for i = len(buff) - 1; i >= 0 && state < 5; i-- { switch state { case 0: // outside of the pattern if buff[i] == '\n' { state = 1 } case 1: // a \n have been matched start = i + 2 switch buff[i] { case '\r': state = 2 case '/': state = 3 case '\n': state = 1 default: state = 0 } case 2: // a \r have been matched switch buff[i] { case '/': state = 3 case '\n': state = 1 default: state = 0 } case 3: // the first / have been matched switch buff[i] { case '/': state = 4 case '\n': state = 1 default: state = 0 } case 4: // the second / have been matched switch buff[i] { case '\n': state = 5 default: state = 0 } } } if i > 0 { return start } return -1 } func _ParseEmblFile(source string, input <-chan _FileChunk, out obiiter.IBioSequence, withFeatureTable bool, batch_size int, total_seq_size int) { for chunks := range input { scanner := bufio.NewScanner(chunks.raw) order := chunks.order sequences := make(obiseq.BioSequenceSlice, 0, 100) id := "" scientificName := "" defBytes := new(bytes.Buffer) featBytes := new(bytes.Buffer) seqBytes := new(bytes.Buffer) taxid := 1 for scanner.Scan() { line := scanner.Text() switch { case strings.HasPrefix(line, "ID "): id = strings.SplitN(line[5:], ";", 2)[0] case strings.HasPrefix(line, "OS "): scientificName = strings.TrimSpace(line[5:]) case strings.HasPrefix(line, "DE "): if defBytes.Len() > 0 { defBytes.WriteByte(' ') } defBytes.WriteString(strings.TrimSpace(line[5:])) case withFeatureTable && strings.HasPrefix(line, "FH "): featBytes.WriteString(line) case withFeatureTable && line == "FH": featBytes.WriteByte('\n') featBytes.WriteString(line) case strings.HasPrefix(line, "FT "): if withFeatureTable { featBytes.WriteByte('\n') featBytes.WriteString(line) } if strings.HasPrefix(line, `FT /db_xref="taxon:`) { taxid, _ = strconv.Atoi(strings.SplitN(line[37:], `"`, 2)[0]) } case strings.HasPrefix(line, " "): parts := strings.SplitN(line[5:], " ", 7) np := len(parts) - 1 for i := 0; i < np; i++ { seqBytes.WriteString(parts[i]) } case line == "//": sequence := obiseq.NewBioSequence(id, seqBytes.Bytes(), defBytes.String()) sequence.SetSource(source) if withFeatureTable { sequence.SetFeatures(featBytes.Bytes()) } annot := sequence.Annotations() annot["scientific_name"] = scientificName annot["taxid"] = taxid // log.Println(FormatFasta(sequence, FormatFastSeqJsonHeader)) sequences = append(sequences, sequence) defBytes = new(bytes.Buffer) featBytes = new(bytes.Buffer) seqBytes = new(bytes.Buffer) } } out.Push(obiiter.MakeBioSequenceBatch(order, sequences)) } out.Done() } // _ReadFlatFileChunk reads a chunk of data from the given 'reader' and sends it to the // 'readers' channel as a _FileChunk struct. The function reads from the reader until // the end of the last entry is found, then sends the chunk to the channel. If the end // of the last entry is not found in the current chunk, the function reads from the reader // in 1 MB increments until the end of the last entry is found. The function repeats this // process until the end of the file is reached. // // Arguments: // reader io.Reader - an io.Reader to read data from // readers chan _FileChunk - a channel to send the data as a _FileChunk struct // // Returns: // None func _ReadFlatFileChunk(reader io.Reader, readers chan _FileChunk) { var err error var buff []byte size := 0 l := 0 i := 0 // Initialize the buffer to the size of a chunk of data buff = make([]byte, _FileChunkSize) // Read from the reader until the buffer is full or the end of the file is reached l, err = io.ReadFull(reader, buff) buff = buff[:l] if err == io.ErrUnexpectedEOF { err = nil } // Read from the reader until the end of the last entry is found or the end of the file is reached for err == nil { // Create an extended buffer to read from if the end of the last entry is not found in the current buffer extbuff := make([]byte, _FileChunkSize) end := 0 ic := 0 // Read from the reader in 1 MB increments until the end of the last entry is found for end = _EndOfLastEntry(buff); err == nil && end < 0; end = _EndOfLastEntry(buff) { ic++ size, err = io.ReadFull(reader, extbuff) buff = append(buff, extbuff[:size]...) } if len(buff) > 0 { if end < 0 { end = len(buff) } lremain := len(buff) - end remains := make([]byte, max(lremain, _FileChunkSize)) lcp := copy(remains, buff[end:]) remains = remains[:lcp] if lcp < lremain { log.Fatalf("Error copying remaining data of chunck %d : %d < %d", i, lcp, len(remains)) } buff = buff[:end] for len(buff) > 0 && (buff[len(buff)-1] == '\n' || buff[len(buff)-1] == '\r') { buff = buff[:len(buff)-1] } if len(buff) > 0 { io := bytes.NewBuffer(buff) if string(buff[io.Len()-2:]) != "//" { log.Fatalf("File chunck ends with 3 bytes : %s", io.Bytes()[io.Len()-3:]) } readers <- _FileChunk{io, i} i++ buff = remains } } } // Close the readers channel when the end of the file is reached close(readers) } // 6 5 43 2 1 // // ?//? func ReadEMBL(reader io.Reader, options ...WithOption) obiiter.IBioSequence { opt := MakeOptions(options) entry_channel := make(chan _FileChunk) newIter := obiiter.MakeIBioSequence() nworkers := opt.ParallelWorkers() newIter.Add(nworkers) go func() { newIter.WaitAndClose() }() // for j := 0; j < opt.ParallelWorkers(); j++ { for j := 0; j < nworkers; j++ { go _ParseEmblFile(opt.Source(), entry_channel, newIter, opt.WithFeatureTable(), opt.BatchSize(), opt.TotalSeqSize()) } go _ReadFlatFileChunk(reader, entry_channel) if opt.pointer.full_file_batch { newIter = newIter.CompleteFileIterator() } return newIter } func ReadEMBLFromFile(filename string, options ...WithOption) (obiiter.IBioSequence, error) { var reader io.Reader var err error options = append(options, OptionsSource(obiutils.RemoveAllExt((path.Base(filename))))) reader, err = Ropen(filename) if err == ErrNoContent { log.Infof("file %s is empty", filename) return ReadEmptyFile(options...) } if err != nil { log.Printf("open file error: %+v", err) return obiiter.NilIBioSequence, err } return ReadEMBL(reader, options...), nil }