diff --git a/python/obitools3/commands/alignpairedend.pyx b/python/obitools3/commands/alignpairedend.pyx index 6a1bd97..c9af464 100755 --- a/python/obitools3/commands/alignpairedend.pyx +++ b/python/obitools3/commands/alignpairedend.pyx @@ -19,6 +19,7 @@ from obitools3.commands.ngsfilter import REVERSE_SEQ_COLUMN_NAME, REVERSE_QUALIT import sys import os +from cpython.exc cimport PyErr_CheckSignals __title__="Aligns paired-ended reads" @@ -95,6 +96,7 @@ def alignmentIterator(entries, aligner): entries_len = len(entries) for i in range(entries_len): + if two_views: seqF = forward[i] seqR = reverse[i] @@ -180,24 +182,24 @@ def run(config): # Initialize the progress bar pb = ProgressBar(entries_len, config, seconde=5) - if config['alignpairedend']['trueali']: - kmer_ali = False - aligner = buildAlignment - else : - kmer_ali = True - if type(entries) == list: - forward = entries[0] - reverse = entries[1] - aligner = Kmer_similarity(forward, \ - view2=reverse, \ - kmer_size=config['alignpairedend']['kmersize'], \ - reversed_column=None) - else: - aligner = Kmer_similarity(entries, \ - column2=entries[REVERSE_SEQ_COLUMN_NAME], \ - qual_column2=entries[REVERSE_QUALITY_COLUMN_NAME], \ - kmer_size=config['alignpairedend']['kmersize'], \ - reversed_column=entries[b'reversed']) # column created by the ngsfilter tool + #if config['alignpairedend']['trueali']: + # kmer_ali = False + # aligner = buildAlignment + #else : + kmer_ali = True + if type(entries) == list: + forward = entries[0] + reverse = entries[1] + aligner = Kmer_similarity(forward, \ + view2=reverse, \ + kmer_size=config['alignpairedend']['kmersize'], \ + reversed_column=None) + else: + aligner = Kmer_similarity(entries, \ + column2=entries[REVERSE_SEQ_COLUMN_NAME], \ + qual_column2=entries[REVERSE_QUALITY_COLUMN_NAME], \ + kmer_size=config['alignpairedend']['kmersize'], \ + reversed_column=entries[b'reversed']) # column created by the ngsfilter tool ba = alignmentIterator(entries, aligner) @@ -206,6 +208,8 @@ def run(config): pb(i) + PyErr_CheckSignals() + consensus = view[i] if not two_views: diff --git a/python/obitools3/commands/annotate.pyx b/python/obitools3/commands/annotate.pyx index 2f76449..ba7c75e 100755 --- a/python/obitools3/commands/annotate.pyx +++ b/python/obitools3/commands/annotate.pyx @@ -19,6 +19,8 @@ import time import math import sys +from cpython.exc cimport PyErr_CheckSignals + __title__="Annotate views with new tags and edit existing annotations" @@ -351,6 +353,7 @@ def run(config): # Editions at line level sequenceTagger = sequenceTaggerGenerator(config, taxo=taxo) for i in range(len(o_view)): + PyErr_CheckSignals() pb(i) sequenceTagger(o_view[i]) diff --git a/python/obitools3/commands/count.pyx b/python/obitools3/commands/count.pyx index 44b6b30..9713d67 100755 --- a/python/obitools3/commands/count.pyx +++ b/python/obitools3/commands/count.pyx @@ -7,6 +7,8 @@ from obitools3.dms import DMS from obitools3.apps.optiongroups import addMinimalInputOption from obitools3.dms.capi.obiview cimport COUNT_COLUMN +from cpython.exc cimport PyErr_CheckSignals + __title__="Counts sequence records" @@ -45,6 +47,7 @@ def run(config): if COUNT_COLUMN in entries and ((config['count']['sequence'] == config['count']['all']) or (config['count']['all'])) : for e in entries: + PyErr_CheckSignals() count2+=e[COUNT_COLUMN] if COUNT_COLUMN in entries and (config['count']['sequence'] == config['count']['all']): diff --git a/python/obitools3/commands/export.pyx b/python/obitools3/commands/export.pyx index 2f5e354..71ec36a 100755 --- a/python/obitools3/commands/export.pyx +++ b/python/obitools3/commands/export.pyx @@ -14,6 +14,8 @@ from obitools3.apps.optiongroups import addMinimalInputOption, \ import sys import io +from cpython.exc cimport PyErr_CheckSignals + __title__="Export a view to a different file format" @@ -66,6 +68,7 @@ def run(config): i=0 for seq in iview : + PyErr_CheckSignals() if pb is not None: pb(i) try: diff --git a/python/obitools3/commands/grep.pyx b/python/obitools3/commands/grep.pyx index beb92bd..dd92c2c 100644 --- a/python/obitools3/commands/grep.pyx +++ b/python/obitools3/commands/grep.pyx @@ -13,6 +13,7 @@ from functools import reduce import time import re import sys +from cpython.exc cimport PyErr_CheckSignals __title__="Grep view lines that match the given predicates" @@ -308,6 +309,7 @@ def run(config): filter = Filter_generator(config["grep"], tax_filter) selection = Line_selection(i_view) for i in range(len(i_view)): + PyErr_CheckSignals() pb(i) line = i_view[i] diff --git a/python/obitools3/commands/head.pyx b/python/obitools3/commands/head.pyx index 27cd46c..e588943 100755 --- a/python/obitools3/commands/head.pyx +++ b/python/obitools3/commands/head.pyx @@ -12,6 +12,8 @@ from obitools3.utils cimport str2bytes import time import sys +from cpython.exc cimport PyErr_CheckSignals + __title__="Keep the N first lines of a view." @@ -70,6 +72,7 @@ def run(config): selection = Line_selection(i_view) for i in range(n): + PyErr_CheckSignals() pb(i) selection.append(i) diff --git a/python/obitools3/commands/import.pyx b/python/obitools3/commands/import.pyx index 6e09cee..9217473 100755 --- a/python/obitools3/commands/import.pyx +++ b/python/obitools3/commands/import.pyx @@ -43,6 +43,9 @@ from obitools3.uri.decode import open_uri from obitools3.apps.config import logger +from cpython.exc cimport PyErr_CheckSignals + + __title__="Imports sequences from different formats into a DMS" @@ -189,6 +192,8 @@ def run(config): i = 0 for entry in entries : + PyErr_CheckSignals() + if entry is None: # error or exception handled at lower level, not raised because Python generators can't resume after any exception is raised if config['obi']['skiperror']: i-=1 @@ -200,8 +205,7 @@ def run(config): pb(i) elif not i%50000: logger("info", "Imported %d entries", i) - - + if NUC_SEQS_view: id_col[i] = entry.id def_col[i] = entry.definition diff --git a/python/obitools3/commands/ngsfilter.pyx b/python/obitools3/commands/ngsfilter.pyx index 67e69ab..64137dd 100755 --- a/python/obitools3/commands/ngsfilter.pyx +++ b/python/obitools3/commands/ngsfilter.pyx @@ -19,6 +19,7 @@ from libc.stdint cimport INT32_MAX from functools import reduce import math import sys +from cpython.exc cimport PyErr_CheckSignals REVERSE_SEQ_COLUMN_NAME = b"REVERSE_SEQUENCE" # used by alignpairedend tool @@ -573,6 +574,7 @@ def run(config): u = 0 try: for i in range(entries_len): + PyErr_CheckSignals() pb(i) if not_aligned: modseq = [Nuc_Seq.new_from_stored(forward[i]), Nuc_Seq.new_from_stored(reverse[i])] diff --git a/python/obitools3/commands/sort.pyx b/python/obitools3/commands/sort.pyx index 7569847..0577c38 100755 --- a/python/obitools3/commands/sort.pyx +++ b/python/obitools3/commands/sort.pyx @@ -23,6 +23,7 @@ from obitools3.dms.capi.obitypes cimport OBI_BOOL, \ import time import sys +from cpython.exc cimport PyErr_CheckSignals NULL_VALUE = {OBI_BOOL: OBIBool_NA, @@ -104,9 +105,11 @@ def run(config): selection = Line_selection(i_view) for i in range(len(i_view)): # TODO special function? + PyErr_CheckSignals() selection.append(i) for k in keys: # TODO order? + PyErr_CheckSignals() selection.sort(key=lambda line_idx: line_cmp(i_view[line_idx], k, pb(line_idx)), reverse=config['sort']['reverse']) pb(len(i_view), force=True) diff --git a/python/obitools3/commands/stats.pyx b/python/obitools3/commands/stats.pyx index f40d714..6e1718a 100755 --- a/python/obitools3/commands/stats.pyx +++ b/python/obitools3/commands/stats.pyx @@ -13,7 +13,8 @@ from functools import reduce import math import time import sys - +from cpython.exc cimport PyErr_CheckSignals + __title__="Compute basic statistics for attribute values." @@ -164,6 +165,7 @@ def run(config): pb = ProgressBar(len(i_view), config, seconde=5) for i in range(len(i_view)): + PyErr_CheckSignals() pb(i) line = i_view[i] diff --git a/python/obitools3/commands/tail.pyx b/python/obitools3/commands/tail.pyx index c1d2d28..26d9d84 100755 --- a/python/obitools3/commands/tail.pyx +++ b/python/obitools3/commands/tail.pyx @@ -11,6 +11,7 @@ from obitools3.utils cimport str2bytes import time import sys +from cpython.exc cimport PyErr_CheckSignals __title__="Keep the N last lines of a view." @@ -70,6 +71,7 @@ def run(config): selection = Line_selection(i_view) for i in range(start, len(i_view)): + PyErr_CheckSignals() pb(i) selection.append(i) diff --git a/python/obitools3/commands/test.pyx b/python/obitools3/commands/test.pyx index 5bc6de7..aa23c32 100755 --- a/python/obitools3/commands/test.pyx +++ b/python/obitools3/commands/test.pyx @@ -23,7 +23,7 @@ from obitools3.dms.capi.obiview cimport NUC_SEQUENCE_COLUMN, \ import shutil import string import random -#import subprocess +from cpython.exc cimport PyErr_CheckSignals VIEW_TYPES = [b"", b"NUC_SEQS_VIEW"] @@ -512,6 +512,7 @@ def run(config): i = 0 for t in range(config['test']['nbtests']): + PyErr_CheckSignals() random_test(config, infos) print_test(config, repr(infos['view'])) i+=1 diff --git a/python/obitools3/commands/uniq.pyx b/python/obitools3/commands/uniq.pyx index c0c437a..36ff184 100644 --- a/python/obitools3/commands/uniq.pyx +++ b/python/obitools3/commands/uniq.pyx @@ -19,6 +19,7 @@ from obitools3.apps.config import logger from obitools3.utils cimport tobytes, tostr import sys +from cpython.exc cimport PyErr_CheckSignals __title__="Group sequence records together" @@ -133,7 +134,8 @@ cdef merge_taxonomy_classification(View_NUC_SEQS o_view, Taxonomy taxonomy) : OBI_STR ) - for seq in o_view: + for seq in o_view: + PyErr_CheckSignals() if MERGED_TAXID_COLUMN in seq : m_taxids = [] m_taxids_dict = seq[MERGED_TAXID_COLUMN] @@ -271,6 +273,7 @@ cdef uniq_sequences(View_NUC_SEQS view, View_NUC_SEQS o_view, ProgressBar pb, li merged_infos = {} iter_view = iter(view) for i_seq in iter_view : + PyErr_CheckSignals() pb(i) # This can't be done in the same line as the unique_id tuple creation because it generates a bug @@ -386,6 +389,7 @@ cdef uniq_sequences(View_NUC_SEQS view, View_NUC_SEQS o_view, ProgressBar pb, li o_idx = 0 for unique_id in uniques : + PyErr_CheckSignals() pb(o_idx) merged_sequences = uniques[unique_id] diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 63cd484..2439f3b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -14,7 +14,8 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}) file(GLOB_RECURSE CSRC "*.c") -add_library(cobitools3 SHARED obierrno.c +add_library(cobitools3 SHARED obisig.c + obierrno.c obidms_taxonomy.c obiblob_indexer.c obi_lcs.c diff --git a/src/build_reference_db.c b/src/build_reference_db.c index 0771489..14f3ca5 100755 --- a/src/build_reference_db.c +++ b/src/build_reference_db.c @@ -24,6 +24,7 @@ #include "obidms.h" #include "obidebug.h" #include "obierrno.h" +#include "obisig.h" #include "obitypes.h" #include "obiview.h" #include "obi_lcs.h" @@ -171,6 +172,8 @@ int build_reference_db(const char* dms_name, char threshold_str[5]; char* new_comments; + signal(SIGINT, sig_handler); + // Discuss keeping the matrix view or not matrix_view_name = calloc((strlen(o_view_name)+strlen("_matrix")+1), sizeof(char)); if (matrix_view_name == NULL) @@ -321,6 +324,9 @@ int build_reference_db(const char* dms_name, // For each pair for (i=0; i<(matrix_with_lca_view->infos)->line_count; i++) { + if (! keep_running) + return -1; + // Read all taxids associated with the first sequence and compute their LCA // Read line index idx1 = obi_get_int_with_elt_idx_and_col_p_in_view(matrix_with_lca_view, matrix_idx1_column, i, 0); @@ -440,6 +446,9 @@ int build_reference_db(const char* dms_name, // Going through matrix once, filling refs arrays on the go for efficiency for (i=0; i<(matrix_with_lca_view->infos)->line_count; i++) { + if (! keep_running) + return -1; + // Read ref line indexes idx1 = obi_get_int_with_elt_idx_and_col_p_in_view(matrix_with_lca_view, matrix_idx1_column, i, 0); idx2 = obi_get_int_with_elt_idx_and_col_p_in_view(matrix_with_lca_view, matrix_idx2_column, i, 0); diff --git a/src/obi_clean.c b/src/obi_clean.c index 0c367a3..2851cc0 100755 --- a/src/obi_clean.c +++ b/src/obi_clean.c @@ -24,6 +24,7 @@ #include "obidms.h" #include "obidebug.h" #include "obierrno.h" +#include "obisig.h" #include "obitypes.h" #include "obiview.h" #include "sse_banded_LCS_alignment.h" @@ -202,6 +203,8 @@ int obi_clean(const char* dms_name, int max_threads = 1; + signal(SIGINT, sig_handler); + #ifdef _OPENMP max_threads = omp_get_max_threads(); if ((thread_count == -1) || (thread_count > max_threads)) @@ -408,7 +411,7 @@ int obi_clean(const char* dms_name, #pragma omp parallel default(none) \ shared(thread_count, seq_count, blob_array, complete_sample_count_array, alignment_result_array, \ - stop, blob1, i, obi_errno, stderr, max_ratio, iseq_column, i_view, \ + stop, blob1, i, obi_errno, keep_running, stderr, max_ratio, iseq_column, i_view, \ similarity_mode, reference, normalize, threshold, ktable, status_column, o_view, sample_count) { index_t j; @@ -430,6 +433,9 @@ int obi_clean(const char* dms_name, #pragma omp for schedule(dynamic, sample_count/thread_count + (sample_count % thread_count != 0)) // Avoid 0 which blocks the program for (sample=0; sample < sample_count; sample++) { + if (! keep_running) + stop = true; + sample_count_array = complete_sample_count_array+(sample*seq_count); // Get count for this sample diff --git a/src/obi_ecopcr.c b/src/obi_ecopcr.c index b1e004b..f7eb809 100755 --- a/src/obi_ecopcr.c +++ b/src/obi_ecopcr.c @@ -25,6 +25,7 @@ #include "obiview.h" #include "obidebug.h" #include "obierrno.h" +#include "obisig.h" #include "obitypes.h" #include "obiview.h" @@ -718,6 +719,8 @@ int obi_ecopcr(const char* i_dms_name, int32_t erri; int32_t errj; + signal(SIGINT, sig_handler); + if (circular) { circular = strlen(primer1); @@ -999,6 +1002,9 @@ int obi_ecopcr(const char* i_dms_name, fprintf(stderr,"\rDone : %f %% ", p); } + if (! keep_running) + return -1; + checkedSequence++; // Get the taxid diff --git a/src/obi_ecotag.c b/src/obi_ecotag.c index 8e29a71..1c73c68 100755 --- a/src/obi_ecotag.c +++ b/src/obi_ecotag.c @@ -25,6 +25,7 @@ #include "obidms.h" #include "obidebug.h" #include "obierrno.h" +#include "obisig.h" #include "obitypes.h" #include "obiview.h" #include "obidmscolumn.h" @@ -273,6 +274,8 @@ int obi_ecotag(const char* dms_name, buffer_size = 1024; best_match_ids_buffer_size = 1024; + signal(SIGINT, sig_handler); + // Open main DMS containing the query sequences and where the output will be written dms = obi_open_dms(dms_name, false); if (dms == NULL) @@ -471,6 +474,9 @@ int obi_ecotag(const char* dms_name, for (j=0; j < ref_count; j++) { + if (! keep_running) + return -1; + // Get reference sequence and its index ref_seq_idx = obi_get_index_with_elt_idx_and_col_p_in_view(ref_view, ref_seq_column, j, 0); blob2 = obi_get_blob_with_elt_idx_and_col_p_in_view(ref_view, ref_seq_column, j, 0); diff --git a/src/obi_lcs.c b/src/obi_lcs.c index 3a3ca60..41a2d21 100755 --- a/src/obi_lcs.c +++ b/src/obi_lcs.c @@ -21,6 +21,7 @@ #include "obi_lcs.h" #include "obidebug.h" #include "obierrno.h" +#include "obisig.h" #include "obitypes.h" #include "obiview.h" #include "sse_banded_LCS_alignment.h" @@ -428,6 +429,8 @@ int obi_lcs_align_one_column(const char* dms_name, OBIDMS_column_p ali_length_column = NULL; OBIDMS_column_p score_column = NULL; + signal(SIGINT, sig_handler); + k = 0; // Open DMS @@ -574,6 +577,9 @@ int obi_lcs_align_one_column(const char* dms_name, if (i%100 == 0) fprintf(stderr,"\rDone : %f %% ", (i / (float) seq_count)*100); + if (! keep_running) + return -1; + // Get first id idx id1_idx = obi_get_index_with_elt_idx_and_col_p_in_view(seq_view, id_column, i, 0); // TODO Could there be multiple IDs per line? // Get first sequence and its index @@ -725,6 +731,8 @@ int obi_lcs_align_two_columns(const char* dms_name, OBIDMS_column_p ali_length_column = NULL; OBIDMS_column_p score_column = NULL; + signal(SIGINT, sig_handler); + k = 0; // Open DMS @@ -979,6 +987,9 @@ int obi_lcs_align_two_columns(const char* dms_name, for (j=0; j < seq2_count; j++) { + if (! keep_running) + return -1; + // Get second sequence and its index seq2_idx = obi_get_index_with_elt_idx_and_col_p_in_view(seq2_view, i_seq2_column, j, seq2_elt_idx); blob2 = obi_get_blob_with_elt_idx_and_col_p_in_view(seq2_view, i_seq2_column, j, seq2_elt_idx); diff --git a/src/obidms.c b/src/obidms.c index 28f6309..8a5963f 100755 --- a/src/obidms.c +++ b/src/obidms.c @@ -32,6 +32,7 @@ #include "utils.h" #include "obilittlebigman.h" #include "libjson/json_utils.h" +#include "obisig.h" #define DEBUG_LEVEL 0 // TODO has to be defined somewhere else (cython compil flag?) @@ -1627,6 +1628,8 @@ int obi_import_view(const char* dms_path_1, const char* dms_path_2, const char* OBIDMS_column_header_p header = NULL; OBIDMS_column_header_p header_2 = NULL; + signal(SIGINT, sig_handler); + dms_1 = obi_open_dms(dms_path_1, false); if (dms_1 == NULL) { @@ -1676,6 +1679,9 @@ int obi_import_view(const char* dms_path_1, const char* dms_path_2, const char* // Import each column and update with the new version number for (i=0; i < (view_1->infos->column_count); i++) { + if (! keep_running) + return -1; + new_version = obi_import_column(dms_path_1, dms_path_2, ((((view_1->infos)->column_references)[i]).column_refs).column_name, ((((view_1->infos)->column_references)[i]).column_refs).version); if (new_version == -1) { @@ -1708,6 +1714,9 @@ int obi_import_view(const char* dms_path_1, const char* dms_path_2, const char* // Go through columns again to update associated columns for (i=0; i < (view_1->infos->column_count); i++) { + if (! keep_running) + return -1; + header = obi_column_get_header_from_name(dms_1, ((((view_1->infos)->column_references)[i]).column_refs).column_name, ((((view_1->infos)->column_references)[i]).column_refs).version); if (header == NULL) { @@ -1747,6 +1756,9 @@ int obi_import_view(const char* dms_path_1, const char* dms_path_2, const char* } } + if (! keep_running) + return -1; + // Close the views if (obi_save_and_close_view(view_1) < 0) { diff --git a/src/obierrno.h b/src/obierrno.h index 0e2ec93..dc80002 100755 --- a/src/obierrno.h +++ b/src/obierrno.h @@ -136,6 +136,9 @@ extern int obi_errno; */ #define OBIDMS_WORKING (38) /** OBIDMS is tagged as being currently used by a process. */ +#define OBI_SIGNAL_CAUGHT (39) /** Caught an interrupting signal. + */ /**@}*/ + #endif /* OBIERRNO_H_ */ diff --git a/src/obisig.c b/src/obisig.c new file mode 100755 index 0000000..d2667ff --- /dev/null +++ b/src/obisig.c @@ -0,0 +1,31 @@ +/**************************************************************************** + * Functions for signal catching and handling * + ****************************************************************************/ + +/** + * @file obisig.c + * @author Celine Mercier (celine.mercier@metabarcoding.org) + * @date September 2019 + * @brief Functions for signal catching and handling. + */ + +#include "obidebug.h" +#include "obierrno.h" +#include "obisig.h" + +#include +#include + + +#define DEBUG_LEVEL 0 // TODO has to be defined somewhere else (cython compil flag?) + + +bool volatile keep_running = true; + + +void sig_handler(int signum) +{ + obi_set_errno(OBI_SIGNAL_CAUGHT); + obidebug(1, "\nCaught signal: %s\n", strsignal(signum)); + keep_running = false; +} diff --git a/src/obisig.h b/src/obisig.h new file mode 100755 index 0000000..4be47db --- /dev/null +++ b/src/obisig.h @@ -0,0 +1,32 @@ +/**************************************************************************** + * Header file for obisig * + ****************************************************************************/ + +/** + * @file obisig.h + * @author Celine Mercier (celine.mercier@metabarcoding.org) + * @date September 2019 + * @brief Header file for signal catching and handling. + */ + + +#ifndef OBISIG_H_ +#define OBISIG_H_ + +#include +#include +#include + + +/** + * @brief Signal handling. + * + * @since September 2019 + * @author Celine Mercier (celine.mercier@metabarcoding.org) + * + */ +bool volatile keep_running; +void sig_handler(int signum); + + +#endif /* OBISIG_H_ */