obitools/obitools3
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 66 Wiki Activity

Added signal catching and handling in C and Cython

Browse Source
This commit is contained in:
Celine Mercier
2019-09-21 16:47:22 +02:00
parent 06f9d6da60
commit ec0737a600
23 changed files with 178 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
python/obitools3/commands/alignpairedend.pyx
View File

@ -19,6 +19,7 @@ from obitools3.commands.ngsfilter import REVERSE_SEQ_COLUMN_NAME, REVERSE_QUALIT
import sys
import os
from cpython.exc cimport PyErr_CheckSignals
__title__="Aligns paired-ended reads"
@ -95,6 +96,7 @@ def alignmentIterator(entries, aligner):
entries_len = len(entries)
for i in range(entries_len):
if two_views:
seqF = forward[i]
seqR = reverse[i]
@ -180,24 +182,24 @@ def run(config):
# Initialize the progress bar
pb = ProgressBar(entries_len, config, seconde=5)
if config['alignpairedend']['trueali']:
kmer_ali = False
aligner = buildAlignment
else :
kmer_ali = True
if type(entries) == list:
forward = entries[0]
reverse = entries[1]
aligner = Kmer_similarity(forward, \
view2=reverse, \
kmer_size=config['alignpairedend']['kmersize'], \
reversed_column=None)
else:
aligner = Kmer_similarity(entries, \
column2=entries[REVERSE_SEQ_COLUMN_NAME], \
qual_column2=entries[REVERSE_QUALITY_COLUMN_NAME], \
kmer_size=config['alignpairedend']['kmersize'], \
reversed_column=entries[b'reversed']) # column created by the ngsfilter tool
#if config['alignpairedend']['trueali']:
# kmer_ali = False
# aligner = buildAlignment
#else :
kmer_ali = True
if type(entries) == list:
forward = entries[0]
reverse = entries[1]
aligner = Kmer_similarity(forward, \
view2=reverse, \
kmer_size=config['alignpairedend']['kmersize'], \
reversed_column=None)
else:
aligner = Kmer_similarity(entries, \
column2=entries[REVERSE_SEQ_COLUMN_NAME], \
qual_column2=entries[REVERSE_QUALITY_COLUMN_NAME], \
kmer_size=config['alignpairedend']['kmersize'], \
reversed_column=entries[b'reversed']) # column created by the ngsfilter tool
ba = alignmentIterator(entries, aligner)
@ -206,6 +208,8 @@ def run(config):
pb(i)
PyErr_CheckSignals()
consensus = view[i]
if not two_views:

3
python/obitools3/commands/annotate.pyx
View File

@ -19,6 +19,8 @@ import time
import math
import sys
from cpython.exc cimport PyErr_CheckSignals
__title__="Annotate views with new tags and edit existing annotations"
@ -351,6 +353,7 @@ def run(config):
# Editions at line level
sequenceTagger = sequenceTaggerGenerator(config, taxo=taxo)
for i in range(len(o_view)):
PyErr_CheckSignals()
pb(i)
sequenceTagger(o_view[i])

3
python/obitools3/commands/count.pyx
View File

@ -7,6 +7,8 @@ from obitools3.dms import DMS
from obitools3.apps.optiongroups import addMinimalInputOption
from obitools3.dms.capi.obiview cimport COUNT_COLUMN
from cpython.exc cimport PyErr_CheckSignals
__title__="Counts sequence records"
@ -45,6 +47,7 @@ def run(config):
if COUNT_COLUMN in entries and ((config['count']['sequence'] == config['count']['all']) or (config['count']['all'])) :
for e in entries:
PyErr_CheckSignals()
count2+=e[COUNT_COLUMN]
if COUNT_COLUMN in entries and (config['count']['sequence'] == config['count']['all']):

3
python/obitools3/commands/export.pyx
View File

@ -14,6 +14,8 @@ from obitools3.apps.optiongroups import addMinimalInputOption, \
import sys
import io
from cpython.exc cimport PyErr_CheckSignals
__title__="Export a view to a different file format"
@ -66,6 +68,7 @@ def run(config):
i=0
for seq in iview :
PyErr_CheckSignals()
if pb is not None:
pb(i)
try:

2
python/obitools3/commands/grep.pyx
View File

@ -13,6 +13,7 @@ from functools import reduce
import time
import re
import sys
from cpython.exc cimport PyErr_CheckSignals
__title__="Grep view lines that match the given predicates"
@ -308,6 +309,7 @@ def run(config):
filter = Filter_generator(config["grep"], tax_filter)
selection = Line_selection(i_view)
for i in range(len(i_view)):
PyErr_CheckSignals()
pb(i)
line = i_view[i]

3
python/obitools3/commands/head.pyx
View File

@ -12,6 +12,8 @@ from obitools3.utils cimport str2bytes
import time
import sys
from cpython.exc cimport PyErr_CheckSignals
__title__="Keep the N first lines of a view."
@ -70,6 +72,7 @@ def run(config):
selection = Line_selection(i_view)
for i in range(n):
PyErr_CheckSignals()
pb(i)
selection.append(i)

8
python/obitools3/commands/import.pyx
View File

@ -43,6 +43,9 @@ from obitools3.uri.decode import open_uri
from obitools3.apps.config import logger
from cpython.exc cimport PyErr_CheckSignals
__title__="Imports sequences from different formats into a DMS"
@ -189,6 +192,8 @@ def run(config):
i = 0
for entry in entries :
PyErr_CheckSignals()
if entry is None: # error or exception handled at lower level, not raised because Python generators can't resume after any exception is raised
if config['obi']['skiperror']:
i-=1
@ -200,8 +205,7 @@ def run(config):
pb(i)
elif not i%50000:
logger("info", "Imported %d entries", i)
if NUC_SEQS_view:
id_col[i] = entry.id
def_col[i] = entry.definition

2
python/obitools3/commands/ngsfilter.pyx
View File

@ -19,6 +19,7 @@ from libc.stdint cimport INT32_MAX
from functools import reduce
import math
import sys
from cpython.exc cimport PyErr_CheckSignals
REVERSE_SEQ_COLUMN_NAME = b"REVERSE_SEQUENCE" # used by alignpairedend tool
@ -573,6 +574,7 @@ def run(config):
u = 0
try:
for i in range(entries_len):
PyErr_CheckSignals()
pb(i)
if not_aligned:
modseq = [Nuc_Seq.new_from_stored(forward[i]), Nuc_Seq.new_from_stored(reverse[i])]

3
python/obitools3/commands/sort.pyx
View File

@ -23,6 +23,7 @@ from obitools3.dms.capi.obitypes cimport OBI_BOOL, \
import time
import sys
from cpython.exc cimport PyErr_CheckSignals
NULL_VALUE = {OBI_BOOL: OBIBool_NA,
@ -104,9 +105,11 @@ def run(config):
selection = Line_selection(i_view)
for i in range(len(i_view)): # TODO special function?
PyErr_CheckSignals()
selection.append(i)
for k in keys: # TODO order?
PyErr_CheckSignals()
selection.sort(key=lambda line_idx: line_cmp(i_view[line_idx], k, pb(line_idx)), reverse=config['sort']['reverse'])
pb(len(i_view), force=True)

4
python/obitools3/commands/stats.pyx
View File

@ -13,7 +13,8 @@ from functools import reduce
import math
import time
import sys
from cpython.exc cimport PyErr_CheckSignals
__title__="Compute basic statistics for attribute values."
@ -164,6 +165,7 @@ def run(config):
pb = ProgressBar(len(i_view), config, seconde=5)
for i in range(len(i_view)):
PyErr_CheckSignals()
pb(i)
line = i_view[i]

2
python/obitools3/commands/tail.pyx
View File

@ -11,6 +11,7 @@ from obitools3.utils cimport str2bytes
import time
import sys
from cpython.exc cimport PyErr_CheckSignals
__title__="Keep the N last lines of a view."
@ -70,6 +71,7 @@ def run(config):
selection = Line_selection(i_view)
for i in range(start, len(i_view)):
PyErr_CheckSignals()
pb(i)
selection.append(i)

3
python/obitools3/commands/test.pyx
View File

@ -23,7 +23,7 @@ from obitools3.dms.capi.obiview cimport NUC_SEQUENCE_COLUMN, \
import shutil
import string
import random
#import subprocess
from cpython.exc cimport PyErr_CheckSignals
VIEW_TYPES = [b"", b"NUC_SEQS_VIEW"]
@ -512,6 +512,7 @@ def run(config):
i = 0
for t in range(config['test']['nbtests']):
PyErr_CheckSignals()
random_test(config, infos)
print_test(config, repr(infos['view']))
i+=1

6
python/obitools3/commands/uniq.pyx
View File

@ -19,6 +19,7 @@ from obitools3.apps.config import logger
from obitools3.utils cimport tobytes, tostr
import sys
from cpython.exc cimport PyErr_CheckSignals
__title__="Group sequence records together"
@ -133,7 +134,8 @@ cdef merge_taxonomy_classification(View_NUC_SEQS o_view, Taxonomy taxonomy) :
OBI_STR
)
for seq in o_view:
for seq in o_view:
PyErr_CheckSignals()
if MERGED_TAXID_COLUMN in seq :
m_taxids = []
m_taxids_dict = seq[MERGED_TAXID_COLUMN]
@ -271,6 +273,7 @@ cdef uniq_sequences(View_NUC_SEQS view, View_NUC_SEQS o_view, ProgressBar pb, li
merged_infos = {}
iter_view = iter(view)
for i_seq in iter_view :
PyErr_CheckSignals()
pb(i)
# This can't be done in the same line as the unique_id tuple creation because it generates a bug
@ -386,6 +389,7 @@ cdef uniq_sequences(View_NUC_SEQS view, View_NUC_SEQS o_view, ProgressBar pb, li
o_idx = 0
for unique_id in uniques :
PyErr_CheckSignals()
pb(o_idx)
merged_sequences = uniques[unique_id]