Added signal catching and handling in C and Cython

python/obitools3/commands/alignpairedend.pyx

@@ -19,6 +19,7 @@ from obitools3.commands.ngsfilter import REVERSE_SEQ_COLUMN_NAME, REVERSE_QUALIT
 import sys
 import os
 
+from cpython.exc cimport PyErr_CheckSignals
 
 __title__="Aligns paired-ended reads"
 
@@ -95,6 +96,7 @@ def alignmentIterator(entries, aligner):
         entries_len = len(entries)
     
     for i in range(entries_len):
+                
         if two_views:
             seqF = forward[i]
             seqR = reverse[i]
@@ -180,24 +182,24 @@ def run(config):
     # Initialize the progress bar
     pb = ProgressBar(entries_len, config, seconde=5)
     
-    if config['alignpairedend']['trueali']:
-        kmer_ali = False
-        aligner = buildAlignment
-    else :
-        kmer_ali = True
-        if type(entries) == list:
-            forward = entries[0]
-            reverse = entries[1]
-            aligner = Kmer_similarity(forward, \
-                                      view2=reverse, \
-                                      kmer_size=config['alignpairedend']['kmersize'], \
-                                      reversed_column=None)
-        else:
-            aligner = Kmer_similarity(entries, \
-                                      column2=entries[REVERSE_SEQ_COLUMN_NAME], \
-                                      qual_column2=entries[REVERSE_QUALITY_COLUMN_NAME], \
-                                      kmer_size=config['alignpairedend']['kmersize'], \
-                                      reversed_column=entries[b'reversed'])  # column created by the ngsfilter tool
+    #if config['alignpairedend']['trueali']:
+    #    kmer_ali = False
+    #    aligner = buildAlignment
+    #else :
+    kmer_ali = True
+    if type(entries) == list:
+        forward = entries[0]
+        reverse = entries[1]
+        aligner = Kmer_similarity(forward, \
+                                  view2=reverse, \
+                                  kmer_size=config['alignpairedend']['kmersize'], \
+                                  reversed_column=None)
+    else:
+        aligner = Kmer_similarity(entries, \
+                                  column2=entries[REVERSE_SEQ_COLUMN_NAME], \
+                                  qual_column2=entries[REVERSE_QUALITY_COLUMN_NAME], \
+                                  kmer_size=config['alignpairedend']['kmersize'], \
+                                  reversed_column=entries[b'reversed'])  # column created by the ngsfilter tool
         
     ba = alignmentIterator(entries, aligner)
 
@@ -206,6 +208,8 @@ def run(config):
         
         pb(i)
         
+        PyErr_CheckSignals()
+
         consensus = view[i]
                
         if not two_views:

python/obitools3/commands/annotate.pyx

@@ -19,6 +19,8 @@ import time
 import math 
 import sys
 
+from cpython.exc cimport PyErr_CheckSignals
+
  
 __title__="Annotate views with new tags and edit existing annotations"
 
@@ -351,6 +353,7 @@ def run(config):
         # Editions at line level
         sequenceTagger = sequenceTaggerGenerator(config, taxo=taxo)
         for i in range(len(o_view)):
+            PyErr_CheckSignals()
             pb(i)
             sequenceTagger(o_view[i])
 

python/obitools3/commands/count.pyx

@@ -7,6 +7,8 @@ from obitools3.dms import DMS
 from obitools3.apps.optiongroups import addMinimalInputOption
 from obitools3.dms.capi.obiview cimport COUNT_COLUMN
 
+from cpython.exc cimport PyErr_CheckSignals
+
 
 __title__="Counts sequence records"
 
@@ -45,6 +47,7 @@ def run(config):
     
     if COUNT_COLUMN in entries and ((config['count']['sequence'] == config['count']['all']) or (config['count']['all'])) :
         for e in entries:
+            PyErr_CheckSignals()
             count2+=e[COUNT_COLUMN]
 
     if COUNT_COLUMN in entries and (config['count']['sequence'] == config['count']['all']):

python/obitools3/commands/export.pyx

@@ -14,6 +14,8 @@ from obitools3.apps.optiongroups import addMinimalInputOption, \
 import sys
 import io
 
+from cpython.exc cimport PyErr_CheckSignals
+
 __title__="Export a view to a different file format"
 
 
@@ -66,6 +68,7 @@ def run(config):
     
     i=0
     for seq in iview :
+        PyErr_CheckSignals()
         if pb is not None:
             pb(i)
         try:

python/obitools3/commands/grep.pyx

@@ -13,6 +13,7 @@ from functools import reduce
 import time
 import re
 import sys
+from cpython.exc cimport PyErr_CheckSignals
 
  
 __title__="Grep view lines that match the given predicates"
@@ -308,6 +309,7 @@ def run(config):
     filter = Filter_generator(config["grep"], tax_filter)
     selection = Line_selection(i_view)
     for i in range(len(i_view)):
+        PyErr_CheckSignals()
         pb(i)
         line = i_view[i]
                  

python/obitools3/commands/head.pyx

@@ -12,6 +12,8 @@ from obitools3.utils cimport str2bytes
 import time
 import sys
  
+from cpython.exc cimport PyErr_CheckSignals
+
 
 __title__="Keep the N first lines of a view."
 
@@ -70,6 +72,7 @@ def run(config):
     selection = Line_selection(i_view)
     
     for i in range(n):
+        PyErr_CheckSignals()
         pb(i)
         selection.append(i)
 

python/obitools3/commands/import.pyx

@@ -43,6 +43,9 @@ from obitools3.uri.decode import open_uri
 
 from obitools3.apps.config import logger
 
+from cpython.exc cimport PyErr_CheckSignals
+
+
 __title__="Imports sequences from different formats into a DMS"
  
  
@@ -189,6 +192,8 @@ def run(config):
     i = 0
     for entry in entries :
         
+        PyErr_CheckSignals()
+
         if entry is None:  # error or exception handled at lower level, not raised because Python generators can't resume after any exception is raised
             if config['obi']['skiperror']:
                 i-=1
@@ -201,7 +206,6 @@ def run(config):
         elif not i%50000:
             logger("info", "Imported %d entries", i)
               
-                        
         if NUC_SEQS_view: 
             id_col[i] = entry.id
             def_col[i] = entry.definition

python/obitools3/commands/ngsfilter.pyx

@@ -19,6 +19,7 @@ from libc.stdint cimport INT32_MAX
 from functools import reduce
 import math
 import sys
+from cpython.exc cimport PyErr_CheckSignals
 
 
 REVERSE_SEQ_COLUMN_NAME = b"REVERSE_SEQUENCE"      # used by alignpairedend tool
@@ -573,6 +574,7 @@ def run(config):
     u = 0
     try:
         for i in range(entries_len):
+            PyErr_CheckSignals()
             pb(i)
             if not_aligned:
                 modseq = [Nuc_Seq.new_from_stored(forward[i]), Nuc_Seq.new_from_stored(reverse[i])]

python/obitools3/commands/sort.pyx

@@ -23,6 +23,7 @@ from obitools3.dms.capi.obitypes cimport OBI_BOOL, \
 
 import time
 import sys
+from cpython.exc cimport PyErr_CheckSignals
 
  
 NULL_VALUE = {OBI_BOOL: OBIBool_NA, 
@@ -104,9 +105,11 @@ def run(config):
     selection = Line_selection(i_view)
     
     for i in range(len(i_view)):  # TODO special function?
+        PyErr_CheckSignals()
         selection.append(i)
     
     for k in keys:  # TODO order?
+        PyErr_CheckSignals()
         selection.sort(key=lambda line_idx: line_cmp(i_view[line_idx], k, pb(line_idx)), reverse=config['sort']['reverse'])
 
     pb(len(i_view), force=True)

python/obitools3/commands/stats.pyx

@@ -13,6 +13,7 @@ from functools import reduce
 import math
 import time
 import sys
+from cpython.exc cimport PyErr_CheckSignals
 
 
 __title__="Compute basic statistics for attribute values."
@@ -164,6 +165,7 @@ def run(config):
     pb = ProgressBar(len(i_view), config, seconde=5)
     
     for i in range(len(i_view)):
+        PyErr_CheckSignals()
         pb(i)
         line = i_view[i]
         

python/obitools3/commands/tail.pyx

@@ -11,6 +11,7 @@ from obitools3.utils cimport str2bytes
 
 import time
 import sys
+from cpython.exc cimport PyErr_CheckSignals
 
 
 __title__="Keep the N last lines of a view."
@@ -70,6 +71,7 @@ def run(config):
     selection = Line_selection(i_view)
     
     for i in range(start, len(i_view)):
+        PyErr_CheckSignals()
         pb(i)
         selection.append(i)
 

python/obitools3/commands/test.pyx

@@ -23,7 +23,7 @@ from obitools3.dms.capi.obiview cimport NUC_SEQUENCE_COLUMN, \
 import shutil
 import string
 import random
-#import subprocess
+from cpython.exc cimport PyErr_CheckSignals
 
 
 VIEW_TYPES = [b"", b"NUC_SEQS_VIEW"]
@@ -512,6 +512,7 @@ def run(config):
     
     i = 0
     for t in range(config['test']['nbtests']):
+        PyErr_CheckSignals()
         random_test(config, infos)
         print_test(config, repr(infos['view']))
         i+=1

python/obitools3/commands/uniq.pyx

@@ -19,6 +19,7 @@ from obitools3.apps.config import logger
 from obitools3.utils cimport tobytes, tostr
 
 import sys
+from cpython.exc cimport PyErr_CheckSignals
 
 
 __title__="Group sequence records together"
@@ -134,6 +135,7 @@ cdef merge_taxonomy_classification(View_NUC_SEQS o_view, Taxonomy taxonomy) :
                          )
         
     for seq in o_view:     
+        PyErr_CheckSignals()   
         if MERGED_TAXID_COLUMN in seq :
             m_taxids = []            
             m_taxids_dict = seq[MERGED_TAXID_COLUMN]
@@ -271,6 +273,7 @@ cdef uniq_sequences(View_NUC_SEQS view, View_NUC_SEQS o_view, ProgressBar pb, li
     merged_infos = {}
     iter_view = iter(view)
     for i_seq in iter_view :
+        PyErr_CheckSignals()
         pb(i)
          
         # This can't be done in the same line as the unique_id tuple creation because it generates a bug
@@ -386,6 +389,7 @@ cdef uniq_sequences(View_NUC_SEQS view, View_NUC_SEQS o_view, ProgressBar pb, li
     o_idx = 0
     
     for unique_id in uniques :
+        PyErr_CheckSignals()
         pb(o_idx)
         
         merged_sequences = uniques[unique_id]

src/CMakeLists.txt

@@ -14,7 +14,8 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR})
 
 file(GLOB_RECURSE CSRC "*.c")
 
-add_library(cobitools3 SHARED   obierrno.c
+add_library(cobitools3 SHARED   obisig.c
+                                obierrno.c
 								obidms_taxonomy.c
 								obiblob_indexer.c
 								obi_lcs.c

src/build_reference_db.c

@@ -24,6 +24,7 @@
 #include "obidms.h"
 #include "obidebug.h"
 #include "obierrno.h"
+#include "obisig.h"
 #include "obitypes.h"
 #include "obiview.h"
 #include "obi_lcs.h"
@@ -171,6 +172,8 @@ int build_reference_db(const char* dms_name,
 	char threshold_str[5];
 	char* new_comments;
 
+	signal(SIGINT, sig_handler);
+
 	// Discuss keeping the matrix view or not
 	matrix_view_name = calloc((strlen(o_view_name)+strlen("_matrix")+1), sizeof(char));
 	if (matrix_view_name == NULL)
@@ -321,6 +324,9 @@ int build_reference_db(const char* dms_name,
 		// For each pair
 	for (i=0; i<(matrix_with_lca_view->infos)->line_count; i++)
 	{
+		if (! keep_running)
+			return -1;
+
 		// Read all taxids associated with the first sequence and compute their LCA
 		// Read line index
 		idx1 = obi_get_int_with_elt_idx_and_col_p_in_view(matrix_with_lca_view, matrix_idx1_column, i, 0);
@@ -440,6 +446,9 @@ int build_reference_db(const char* dms_name,
 	// Going through matrix once, filling refs arrays on the go for efficiency
 	for (i=0; i<(matrix_with_lca_view->infos)->line_count; i++)
 	{
+		if (! keep_running)
+			return -1;
+
 		// Read ref line indexes
 		idx1 = obi_get_int_with_elt_idx_and_col_p_in_view(matrix_with_lca_view, matrix_idx1_column, i, 0);
 		idx2 = obi_get_int_with_elt_idx_and_col_p_in_view(matrix_with_lca_view, matrix_idx2_column, i, 0);

src/obi_clean.c

@@ -24,6 +24,7 @@
 #include "obidms.h"
 #include "obidebug.h"
 #include "obierrno.h"
+#include "obisig.h"
 #include "obitypes.h"
 #include "obiview.h"
 #include "sse_banded_LCS_alignment.h"
@@ -202,6 +203,8 @@ int obi_clean(const char* dms_name,
 
 	int max_threads = 1;
 
+	signal(SIGINT, sig_handler);
+
 	#ifdef _OPENMP
 	max_threads = omp_get_max_threads();
 	if ((thread_count == -1) || (thread_count > max_threads))
@@ -408,7 +411,7 @@ int obi_clean(const char* dms_name,
 
 		#pragma omp parallel default(none) \
 					 	 	 shared(thread_count, seq_count, blob_array, complete_sample_count_array, alignment_result_array, \
-					 	 			 stop, blob1, i, obi_errno, stderr, max_ratio, iseq_column, i_view, \
+					 	 			 stop, blob1, i, obi_errno, keep_running, stderr, max_ratio, iseq_column, i_view, \
 									 similarity_mode, reference, normalize, threshold, ktable, status_column, o_view, sample_count)
 		{
 			index_t    j;
@@ -430,6 +433,9 @@ int obi_clean(const char* dms_name,
 			#pragma omp for schedule(dynamic, sample_count/thread_count + (sample_count % thread_count != 0))  // Avoid 0 which blocks the program
 			for (sample=0; sample < sample_count; sample++)
 			{
+				if (! keep_running)
+					stop = true;
+
 				sample_count_array = complete_sample_count_array+(sample*seq_count);
 
 				// Get count for this sample

src/obi_ecopcr.c

@@ -25,6 +25,7 @@
 #include "obiview.h"
 #include "obidebug.h"
 #include "obierrno.h"
+#include "obisig.h"
 #include "obitypes.h"
 #include "obiview.h"
 
@@ -718,6 +719,8 @@ int obi_ecopcr(const char* i_dms_name,
 	int32_t       erri;
 	int32_t		  errj;
 
+	signal(SIGINT, sig_handler);
+
 	if (circular)
 	{
 		circular = strlen(primer1);
@@ -999,6 +1002,9 @@ int obi_ecopcr(const char* i_dms_name,
 		    fprintf(stderr,"\rDone : %f %%          ", p);
 		}
 
+		if (! keep_running)
+			return -1;
+
 		checkedSequence++;
 
 		// Get the taxid

src/obi_ecotag.c

@@ -25,6 +25,7 @@
 #include "obidms.h"
 #include "obidebug.h"
 #include "obierrno.h"
+#include "obisig.h"
 #include "obitypes.h"
 #include "obiview.h"
 #include "obidmscolumn.h"
@@ -273,6 +274,8 @@ int obi_ecotag(const char* dms_name,
 	buffer_size = 1024;
 	best_match_ids_buffer_size = 1024;
 
+	signal(SIGINT, sig_handler);
+
 	// Open main DMS containing the query sequences and where the output will be written
 	dms = obi_open_dms(dms_name, false);
 	if (dms == NULL)
@@ -471,6 +474,9 @@ int obi_ecotag(const char* dms_name,
 
 		for (j=0; j < ref_count; j++)
 		{
+			if (! keep_running)
+				return -1;
+
 			// Get reference sequence and its index
 			ref_seq_idx = obi_get_index_with_elt_idx_and_col_p_in_view(ref_view, ref_seq_column, j, 0);
 			blob2 = obi_get_blob_with_elt_idx_and_col_p_in_view(ref_view, ref_seq_column, j, 0);

src/obi_lcs.c

@@ -21,6 +21,7 @@
 #include "obi_lcs.h"
 #include "obidebug.h"
 #include "obierrno.h"
+#include "obisig.h"
 #include "obitypes.h"
 #include "obiview.h"
 #include "sse_banded_LCS_alignment.h"
@@ -428,6 +429,8 @@ int obi_lcs_align_one_column(const char* dms_name,
 	OBIDMS_column_p ali_length_column = NULL;
 	OBIDMS_column_p score_column = NULL;
 
+	signal(SIGINT, sig_handler);
+
 	k = 0;
 
 	// Open DMS
@@ -574,6 +577,9 @@ int obi_lcs_align_one_column(const char* dms_name,
 		if (i%100 == 0)
 			fprintf(stderr,"\rDone : %f %%       ", (i / (float) seq_count)*100);
 
+		if (! keep_running)
+			return -1;
+
 		// Get first id idx
 		id1_idx = obi_get_index_with_elt_idx_and_col_p_in_view(seq_view, id_column, i, 0);	// TODO Could there be multiple IDs per line?
 		// Get first sequence and its index
@@ -725,6 +731,8 @@ int obi_lcs_align_two_columns(const char* dms_name,
 	OBIDMS_column_p ali_length_column = NULL;
 	OBIDMS_column_p score_column = NULL;
 
+	signal(SIGINT, sig_handler);
+
 	k = 0;
 
 	// Open DMS
@@ -979,6 +987,9 @@ int obi_lcs_align_two_columns(const char* dms_name,
 
 		for (j=0; j < seq2_count; j++)
 		{
+			if (! keep_running)
+				return -1;
+
 			// Get second sequence and its index
 			seq2_idx = obi_get_index_with_elt_idx_and_col_p_in_view(seq2_view, i_seq2_column, j, seq2_elt_idx);
 			blob2 = obi_get_blob_with_elt_idx_and_col_p_in_view(seq2_view, i_seq2_column, j, seq2_elt_idx);

src/obidms.c

@@ -32,6 +32,7 @@
 #include "utils.h"
 #include "obilittlebigman.h"
 #include "libjson/json_utils.h"
+#include "obisig.h"
 
 
 #define DEBUG_LEVEL 0	// TODO has to be defined somewhere else (cython compil flag?)
@@ -1627,6 +1628,8 @@ int obi_import_view(const char* dms_path_1, const char* dms_path_2, const char*
 	OBIDMS_column_header_p header = NULL;
 	OBIDMS_column_header_p header_2 = NULL;
 
+	signal(SIGINT, sig_handler);
+
 	dms_1 = obi_open_dms(dms_path_1, false);
 	if (dms_1 == NULL)
 	{
@@ -1676,6 +1679,9 @@ int obi_import_view(const char* dms_path_1, const char* dms_path_2, const char*
 	// Import each column and update with the new version number
 	for (i=0; i < (view_1->infos->column_count); i++)
 	{
+		if (! keep_running)
+			return -1;
+
 		new_version = obi_import_column(dms_path_1, dms_path_2, ((((view_1->infos)->column_references)[i]).column_refs).column_name, ((((view_1->infos)->column_references)[i]).column_refs).version);
 		if (new_version == -1)
 		{
@@ -1708,6 +1714,9 @@ int obi_import_view(const char* dms_path_1, const char* dms_path_2, const char*
 	// Go through columns again to update associated columns
 	for (i=0; i < (view_1->infos->column_count); i++)
 	{
+		if (! keep_running)
+			return -1;
+
 		header = obi_column_get_header_from_name(dms_1, ((((view_1->infos)->column_references)[i]).column_refs).column_name, ((((view_1->infos)->column_references)[i]).column_refs).version);
 		if (header == NULL)
 		{
@@ -1747,6 +1756,9 @@ int obi_import_view(const char* dms_path_1, const char* dms_path_2, const char*
 		}
 	}
 
+	if (! keep_running)
+		return -1;
+
 	// Close the views
 	if (obi_save_and_close_view(view_1) < 0)
 	{

src/obierrno.h

@@ -136,6 +136,9 @@ extern int obi_errno;
  	 	 	 	 	 	 	 	 	 	 	 */
 #define OBIDMS_WORKING (38) 				/** OBIDMS is tagged as being currently used by a process.
  	 	 	 	 	 	 	 	 	 	 	 */
+#define OBI_SIGNAL_CAUGHT (39)				/** Caught an interrupting signal.
+	 	 	 	 	 	 	 	 	 	 	 */
 /**@}*/
 
+
 #endif /* OBIERRNO_H_ */

src/obisig.c

@@ -0,0 +1,31 @@
+/****************************************************************************
+ *  Functions for signal catching and handling                              *
+ ****************************************************************************/
+
+/**
+ * @file obisig.c
+ * @author Celine Mercier (celine.mercier@metabarcoding.org)
+ * @date September 2019
+ * @brief Functions for signal catching and handling.
+ */
+
+#include "obidebug.h"
+#include "obierrno.h"
+#include "obisig.h"
+
+#include <stdbool.h>
+#include <signal.h>
+
+
+#define DEBUG_LEVEL 0	// TODO has to be defined somewhere else (cython compil flag?)
+
+
+bool volatile keep_running = true;
+
+
+void sig_handler(int signum)
+{
+	obi_set_errno(OBI_SIGNAL_CAUGHT);
+	obidebug(1, "\nCaught signal: %s\n", strsignal(signum));
+	keep_running = false;
+}

src/obisig.h

@@ -0,0 +1,32 @@
+/****************************************************************************
+ * Header file for obisig                                                   *
+ ****************************************************************************/
+
+/**
+ * @file obisig.h
+ * @author Celine Mercier (celine.mercier@metabarcoding.org)
+ * @date September 2019
+ * @brief Header file for signal catching and handling.
+ */
+
+
+#ifndef OBISIG_H_
+#define OBISIG_H_
+
+#include <stdint.h>
+#include <signal.h>
+#include <stdbool.h>
+
+
+/**
+ * @brief Signal handling.
+ *
+ * @since September 2019
+ * @author Celine Mercier (celine.mercier@metabarcoding.org)
+ *
+ */
+bool volatile keep_running;
+void sig_handler(int signum);
+
+
+#endif /* OBISIG_H_ */