Genbank file parser functions that should have been included in a
previous commit
This commit is contained in:
Celine Mercier
110
python/obitools3/parsers/genbank.cfiles
Normal file
110
python/obitools3/parsers/genbank.cfiles
Normal file
@ -0,0 +1,110 @@
|
||||
../../../src/obi_lcs.h
|
||||
../../../src/obi_lcs.c
|
||||
../../../src/obierrno.h
|
||||
../../../src/obierrno.c
|
||||
../../../src/upperband.h
|
||||
../../../src/upperband.c
|
||||
../../../src/sse_banded_LCS_alignment.h
|
||||
../../../src/sse_banded_LCS_alignment.c
|
||||
../../../src/obiblob.h
|
||||
../../../src/obiblob.c
|
||||
../../../src/utils.h
|
||||
../../../src/utils.c
|
||||
../../../src/obidms.h
|
||||
../../../src/obidms.c
|
||||
../../../src/libjson/json_utils.h
|
||||
../../../src/libjson/json_utils.c
|
||||
../../../src/libjson/cJSON.h
|
||||
../../../src/libjson/cJSON.c
|
||||
../../../src/obiavl.h
|
||||
../../../src/obiavl.c
|
||||
../../../src/bloom.h
|
||||
../../../src/bloom.c
|
||||
../../../src/crc64.h
|
||||
../../../src/crc64.c
|
||||
../../../src/murmurhash2.h
|
||||
../../../src/murmurhash2.c
|
||||
../../../src/obidmscolumn.h
|
||||
../../../src/obidmscolumn.c
|
||||
../../../src/obitypes.h
|
||||
../../../src/obitypes.c
|
||||
../../../src/obidmscolumndir.h
|
||||
../../../src/obidmscolumndir.c
|
||||
../../../src/obiblob_indexer.h
|
||||
../../../src/obiblob_indexer.c
|
||||
../../../src/obiview.h
|
||||
../../../src/obiview.c
|
||||
../../../src/hashtable.h
|
||||
../../../src/hashtable.c
|
||||
../../../src/linked_list.h
|
||||
../../../src/linked_list.c
|
||||
../../../src/obidmscolumn_array.h
|
||||
../../../src/obidmscolumn_array.c
|
||||
../../../src/obidmscolumn_blob.h
|
||||
../../../src/obidmscolumn_blob.c
|
||||
../../../src/obidmscolumn_idx.h
|
||||
../../../src/obidmscolumn_idx.c
|
||||
../../../src/obidmscolumn_bool.h
|
||||
../../../src/obidmscolumn_bool.c
|
||||
../../../src/obidmscolumn_char.h
|
||||
../../../src/obidmscolumn_char.c
|
||||
../../../src/obidmscolumn_float.h
|
||||
../../../src/obidmscolumn_float.c
|
||||
../../../src/obidmscolumn_int.h
|
||||
../../../src/obidmscolumn_int.c
|
||||
../../../src/obidmscolumn_qual.h
|
||||
../../../src/obidmscolumn_qual.c
|
||||
../../../src/obidmscolumn_seq.h
|
||||
../../../src/obidmscolumn_seq.c
|
||||
../../../src/obidmscolumn_str.h
|
||||
../../../src/obidmscolumn_str.c
|
||||
../../../src/array_indexer.h
|
||||
../../../src/array_indexer.c
|
||||
../../../src/char_str_indexer.h
|
||||
../../../src/char_str_indexer.c
|
||||
../../../src/dna_seq_indexer.h
|
||||
../../../src/dna_seq_indexer.c
|
||||
../../../src/encode.c
|
||||
../../../src/encode.h
|
||||
../../../src/uint8_indexer.c
|
||||
../../../src/uint8_indexer.h
|
||||
../../../src/build_reference_db.c
|
||||
../../../src/build_reference_db.h
|
||||
../../../src/kmer_similarity.c
|
||||
../../../src/kmer_similarity.h
|
||||
../../../src/obi_clean.c
|
||||
../../../src/obi_clean.h
|
||||
../../../src/obi_ecopcr.c
|
||||
../../../src/obi_ecopcr.h
|
||||
../../../src/obi_ecotag.c
|
||||
../../../src/obi_ecotag.h
|
||||
../../../src/obidms_taxonomy.c
|
||||
../../../src/obidms_taxonomy.h
|
||||
../../../src/obilittlebigman.c
|
||||
../../../src/obilittlebigman.h
|
||||
../../../src/_sse.h
|
||||
../../../src/obidebug.h
|
||||
../../../src/libecoPCR/libapat/CODES/dft_code.h
|
||||
../../../src/libecoPCR/libapat/CODES/dna_code.h
|
||||
../../../src/libecoPCR/libapat/CODES/prot_code.h
|
||||
../../../src/libecoPCR/libapat/apat_parse.c
|
||||
../../../src/libecoPCR/libapat/apat_search.c
|
||||
../../../src/libecoPCR/libapat/apat.h
|
||||
../../../src/libecoPCR/libapat/Gmach.h
|
||||
../../../src/libecoPCR/libapat/Gtypes.h
|
||||
../../../src/libecoPCR/libapat/libstki.c
|
||||
../../../src/libecoPCR/libapat/libstki.h
|
||||
../../../src/libecoPCR/libthermo/nnparams.h
|
||||
../../../src/libecoPCR/libthermo/nnparams.c
|
||||
../../../src/libecoPCR/ecoapat.c
|
||||
../../../src/libecoPCR/ecodna.c
|
||||
../../../src/libecoPCR/ecoError.c
|
||||
../../../src/libecoPCR/ecoMalloc.c
|
||||
../../../src/libecoPCR/ecoPCR.h
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
9
python/obitools3/parsers/genbank.pxd
Executable file
9
python/obitools3/parsers/genbank.pxd
Executable file
@ -0,0 +1,9 @@
|
||||
#cython: language_level=3
|
||||
|
||||
from ..utils cimport str2bytes
|
||||
from .header cimport parseHeader
|
||||
from ..files.universalopener cimport uopen
|
||||
from ..files.linebuffer cimport LineBuffer
|
||||
|
||||
|
||||
|
||||
193
python/obitools3/parsers/genbank.pyx
Executable file
193
python/obitools3/parsers/genbank.pyx
Executable file
@ -0,0 +1,193 @@
|
||||
#cython: language_level=3
|
||||
|
||||
'''
|
||||
Created on June 12th 2018
|
||||
|
||||
@author: coissac/mercier
|
||||
'''
|
||||
|
||||
|
||||
import types
|
||||
import re
|
||||
import sys
|
||||
import os
|
||||
import glob
|
||||
|
||||
from obitools3.files.universalopener cimport uopen
|
||||
from obitools3.utils cimport tostr
|
||||
from obitools3.dms.obiseq cimport Nuc_Seq
|
||||
from .embl_genbank_features import extractTaxon
|
||||
|
||||
from libc.stdlib cimport free, malloc, realloc
|
||||
from libc.string cimport strcpy, strlen
|
||||
|
||||
|
||||
_featureMatcher = re.compile(b'^FEATURES.+\n(?=ORIGIN)',re.DOTALL + re.M)
|
||||
|
||||
_headerMatcher = re.compile(b'^LOCUS.+(?=\nFEATURES)', re.DOTALL + re.M)
|
||||
_seqMatcher = re.compile(b'(?<=ORIGIN).+(?=//\n)', re.DOTALL + re.M)
|
||||
_cleanSeq = re.compile(b'[ \n0-9]+')
|
||||
_acMatcher = re.compile(b'(?<=^ACCESSION ).+',re.M)
|
||||
_deMatcher = re.compile(b'(?<=^DEFINITION ).+\n( .+\n)*',re.M)
|
||||
_cleanDe = re.compile(b'\n *')
|
||||
|
||||
|
||||
def genbankParser(bytes text):
|
||||
|
||||
cdef Nuc_Seq seq
|
||||
|
||||
try:
|
||||
header = _headerMatcher.search(text).group()
|
||||
|
||||
ft = _featureMatcher.search(text).group()
|
||||
|
||||
s = _seqMatcher.search(text).group()
|
||||
s = _cleanSeq.sub(b'', s).upper()
|
||||
|
||||
acs = _acMatcher.search(text).group()
|
||||
acs = acs.split()
|
||||
ac = acs[0]
|
||||
acs = acs[1:]
|
||||
|
||||
de = _deMatcher.search(header).group()
|
||||
de = _cleanDe.sub(b' ',de).strip().strip(b'.')
|
||||
|
||||
except Exception as e:
|
||||
print("\nCould not import sequence id:", text.split()[1], "(error raised:", e, ")")
|
||||
# Do not raise any Exception if you need the possibility to resume the generator
|
||||
# (Python generators can't resume after any exception is raised)
|
||||
return None
|
||||
|
||||
tags = {}
|
||||
extractTaxon(ft, tags)
|
||||
|
||||
seq = Nuc_Seq(ac,
|
||||
s,
|
||||
definition=de,
|
||||
quality=None,
|
||||
offset=-1,
|
||||
tags=tags)
|
||||
|
||||
return seq
|
||||
|
||||
|
||||
def genbankIterator_file(lineiterator,
|
||||
int skip=0,
|
||||
only=None,
|
||||
firstline=None,
|
||||
int buffersize=100000000
|
||||
):
|
||||
cdef int lines_to_skip, ionly, read
|
||||
cdef Nuc_Seq seq
|
||||
cdef char* entry = NULL
|
||||
cdef size_t entry_buffer_size
|
||||
cdef int entry_len
|
||||
cdef int line_len
|
||||
|
||||
entry_buffer_size = 2048
|
||||
|
||||
entry = <char*> malloc(entry_buffer_size*sizeof(char))
|
||||
|
||||
if only is None:
|
||||
ionly = -1
|
||||
else:
|
||||
ionly = int(only)
|
||||
|
||||
if isinstance(lineiterator, (str, bytes)):
|
||||
lineiterator=uopen(lineiterator)
|
||||
if isinstance(lineiterator, LineBuffer):
|
||||
iterator = iter(lineiterator)
|
||||
else:
|
||||
if hasattr(lineiterator, "readlines"):
|
||||
iterator = iter(LineBuffer(lineiterator, buffersize))
|
||||
elif hasattr(lineiterator, '__next__'):
|
||||
iterator = lineiterator
|
||||
else:
|
||||
raise Exception("Invalid line iterator")
|
||||
|
||||
skipped = 0
|
||||
read = 0
|
||||
|
||||
if firstline is None:
|
||||
line = next(iterator)
|
||||
else:
|
||||
line = firstline
|
||||
|
||||
while True:
|
||||
|
||||
if ionly >= 0 and read >= ionly-1:
|
||||
break
|
||||
|
||||
while skipped < skip:
|
||||
line = next(iterator)
|
||||
try:
|
||||
while line[:2] != b'//':
|
||||
line = next(iterator)
|
||||
line = next(iterator)
|
||||
except StopIteration:
|
||||
break
|
||||
skipped += 1
|
||||
|
||||
try:
|
||||
entry_len = 0
|
||||
while line[:2] != b'//':
|
||||
line_len = strlen(line)
|
||||
while (entry_len + line_len) >= entry_buffer_size:
|
||||
entry_buffer_size*=2
|
||||
entry = <char*>realloc(entry, entry_buffer_size)
|
||||
strcpy(entry+entry_len, line)
|
||||
entry_len+=line_len
|
||||
line = next(iterator)
|
||||
# Add last line too because need the // flag to parse
|
||||
line_len = strlen(line)
|
||||
while (entry_len + line_len) >= entry_buffer_size:
|
||||
entry_buffer_size*=2
|
||||
entry = <char*>realloc(entry, entry_buffer_size)
|
||||
strcpy(entry+entry_len, line)
|
||||
line = next(iterator)
|
||||
except StopIteration:
|
||||
break
|
||||
|
||||
seq = genbankParser(entry)
|
||||
|
||||
yield seq
|
||||
read+=1
|
||||
|
||||
yield seq
|
||||
|
||||
free(entry)
|
||||
|
||||
|
||||
def genbankIterator_dir(dir_path,
|
||||
int skip=0,
|
||||
only=None,
|
||||
firstline=None,
|
||||
int buffersize=100000000
|
||||
):
|
||||
path = dir_path
|
||||
read = 0
|
||||
for filename in glob.glob(os.path.join(path, b'*.gbff*')):
|
||||
if read==only:
|
||||
return
|
||||
f = uopen(filename)
|
||||
if only is not None:
|
||||
only_f = only-read
|
||||
else:
|
||||
only_f = None
|
||||
for seq in genbankIterator_file(f, skip=skip, only=only_f, buffersize=buffersize):
|
||||
yield seq
|
||||
read+=1
|
||||
|
||||
|
||||
def genbankIterator(obj,
|
||||
int skip=0,
|
||||
only=None,
|
||||
firstline=None,
|
||||
int buffersize=100000000
|
||||
):
|
||||
if type(obj) == bytes or type(obj) == str :
|
||||
return genbankIterator_dir(obj, skip=skip, only=only, firstline=firstline, buffersize=buffersize)
|
||||
else:
|
||||
return genbankIterator_file(obj, skip=skip, only=only, firstline=firstline, buffersize=buffersize)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user