switch to version 3.0.1b7

should not have

python/obitools3/apps/optiongroups/__init__.py

@@ -1,3 +1,9 @@
+import codecs
+
+def unescaped_str(arg_str):
+    return arg_str.encode('latin-1', 'backslashreplace').decode('unicode-escape')
+
+
 def __addInputOption(optionManager):
     
     optionManager.add_argument(
@@ -133,7 +139,7 @@ def __addTabularOption(optionManager):
     group.add_argument('--sep',
                      action="store", dest="obi:sep",
                      default="\t",
-                     type=str,
+                     type=unescaped_str,
                      help="Column separator")
 
 

python/obitools3/commands/addtaxids.pyx

@@ -26,7 +26,7 @@ import sys
 from cpython.exc cimport PyErr_CheckSignals
 
  
-__title__="Annotate sequences with their corresponding NCBI taxid found from the taxon scientific name."
+__title__="Annotate sequences with their corresponding NCBI taxid found from the taxon scientific name"
 
 
  

python/obitools3/commands/align.pyx

@@ -19,7 +19,7 @@ import time
 import sys
 
 
-__title__="Aligns one sequence column with itself or two sequence columns"
+__title__="Align one sequence column with itself or two sequence columns"
 
 
 def addOptions(parser):

python/obitools3/commands/alignpairedend.pyx

@@ -23,7 +23,7 @@ import os
 
 from cpython.exc cimport PyErr_CheckSignals
 
-__title__="Aligns paired-ended reads"
+__title__="Align paired-ended reads"
 
 
 

python/obitools3/commands/build_ref_db.pyx

@@ -16,7 +16,7 @@ import sys
 from cpython.exc cimport PyErr_CheckSignals
 
 
-__title__="Tag a set of sequences for PCR and sequencing errors identification"
+__title__="Build a reference database for ecotag"
 
 
 def addOptions(parser):

python/obitools3/commands/cat.pyx

@@ -22,7 +22,7 @@ import sys
 from cpython.exc cimport PyErr_CheckSignals
 
 
-__title__="Concatenate views."
+__title__="Concatenate views"
 
  
 def addOptions(parser):

python/obitools3/commands/count.pyx

@@ -10,7 +10,7 @@ from obitools3.dms.capi.obiview cimport COUNT_COLUMN
 from cpython.exc cimport PyErr_CheckSignals
 
 
-__title__="Counts sequence records"
+__title__="Count sequence records"
 
  
 def addOptions(parser):

python/obitools3/commands/head.pyx

@@ -16,7 +16,7 @@ from io import BufferedWriter
 from cpython.exc cimport PyErr_CheckSignals
 
 
-__title__="Keep the N first lines of a view."
+__title__="Keep the N first lines of a view"
 
  
 def addOptions(parser):

python/obitools3/commands/import.pyx

@@ -52,7 +52,7 @@ from cpython.exc cimport PyErr_CheckSignals
 from io import BufferedWriter
 
 
-__title__="Imports sequences from different formats into a DMS"
+__title__="Import sequences from different formats into a DMS"
  
  
 default_config = {   'destview'     : None,
@@ -371,7 +371,7 @@ def run(config):
                             # Fill value
                             dcols[tag][0][i] = value
                          
-                        except IndexError :
+                        except (IndexError, OverflowError):
                                                                                  
                             value_type = type(value)
                             old_column = dcols[tag][0]
@@ -419,7 +419,7 @@ def run(config):
                                 dcols[tag][0][i] = value
         
         except Exception as e:
-            print("\nCould not import sequence id:", entry.id, "(error raised:", e, ")")
+            print("\nCould not import sequence:", entry, "(error raised:", e, ")")
             if 'skiperror' in config['obi'] and not config['obi']['skiperror']:
                 raise e
             else:
@@ -430,7 +430,7 @@ def run(config):
     if pb is not None:
         pb(i, force=True)
         print("", file=sys.stderr)
-    logger("info", "Imported %d entries", i)
+    logger("info", "Imported %d entries", len(view))
     
     # Save command config in View and DMS comments
     command_line = " ".join(sys.argv[1:])

python/obitools3/commands/ngsfilter.pyx

@@ -24,7 +24,7 @@ from cpython.exc cimport PyErr_CheckSignals
 from io import BufferedWriter
 
 
-__title__="Assigns sequence records to the corresponding experiment/sample based on DNA tags and primers"
+__title__="Assign sequence records to the corresponding experiment/sample based on DNA tags and primers"
 
 
 def addOptions(parser):
@@ -394,7 +394,7 @@ cdef tuple annotate(sequences, infos, no_tags, verbose=False):
             seq_to_match = sequences[0]
         reversematch = []
         # Compute begin
-        begin=directmatch[1][2]+1  # end of match + 1 on the same sequence
+        #begin=directmatch[1][2]+1  # end of match + 1 on the same sequence -- No, already cut out forward primer        
         # Try reverse matching on the other sequence:
         new_seq = True
         pattern = 0
@@ -408,7 +408,7 @@ cdef tuple annotate(sequences, infos, no_tags, verbose=False):
                 primer=p
             # Saving original primer as 4th member of the tuple to serve as correct key in infos dict even if it might have been reversed complemented
             # (3rd member already used by directmatch)
-            reversematch.append((primer, primer(seq_to_match, same_sequence=not new_seq, pattern=pattern, begin=begin), None, p))
+            reversematch.append((primer, primer(seq_to_match, same_sequence=not new_seq, pattern=pattern, begin=0), None, p))
             new_seq = False
             pattern+=1
         # Choose match closer to the end of the sequence

python/obitools3/commands/rm.pyx

@@ -0,0 +1,44 @@
+#cython: language_level=3
+
+from obitools3.uri.decode import open_uri
+from obitools3.apps.config import logger
+from obitools3.dms import DMS
+from obitools3.apps.optiongroups import addMinimalInputOption
+from obitools3.dms.view.view cimport View
+import os
+ 
+__title__="Delete a view"
+
+
+def addOptions(parser):    
+    addMinimalInputOption(parser)
+
+def run(config):
+
+    DMS.obi_atexit()
+    
+    logger("info", "obi rm")
+
+    # Open the input
+    input = open_uri(config['obi']['inputURI'])
+    if input is None:
+        raise Exception("Could not read input")
+    
+    # Check that it's a view
+    if isinstance(input[1], View) :
+        view = input[1]
+    else: 
+        raise NotImplementedError()
+    
+    # Get the path to the view file to remove
+    path = input[0].full_path  # dms path
+    path+=b"/VIEWS/"
+    path+=view.name
+    path+=b".obiview"
+    
+    # Close the view and the DMS
+    view.close()
+    input[0].close(force=True)
+    
+    # Rm
+    os.remove(path)

python/obitools3/commands/sort.pyx

@@ -36,7 +36,7 @@ NULL_VALUE = {OBI_BOOL: OBIBool_NA,
               OBI_STR: b""}
  
 
-__title__="Sort view lines according to the value of a given attribute."
+__title__="Sort view lines according to the value of a given attribute"
 
  
 def addOptions(parser):

python/obitools3/commands/stats.pyx

@@ -16,7 +16,7 @@ import sys
 from cpython.exc cimport PyErr_CheckSignals
 
 
-__title__="Compute basic statistics for attribute values."
+__title__="Compute basic statistics for attribute values"
 
 '''
 `obi stats` computes basic statistics for attribute values of sequence records.
@@ -238,14 +238,14 @@ def run(config):
     else:
         sdvar= "%s"
         
-    hcat = "\t".join([pcat % x for x in config['stats']['categories']]) + "\t" +\
+    hcat = "\t".join([pcat % x for x in config['stats']['categories']]) + \
-           "\t".join([minvar % x for x in config['stats']['minimum']])  + "\t" +\
+           "\t".join([minvar % x for x in config['stats']['minimum']])  + \
-           "\t".join([maxvar % x for x in config['stats']['maximum']])  + "\t" +\
+           "\t".join([maxvar % x for x in config['stats']['maximum']])  + \
-           "\t".join([meanvar % x for x in config['stats']['mean']])  + "\t" +\
+           "\t".join([meanvar % x for x in config['stats']['mean']])  + \
-           "\t".join([varvar % x for x in config['stats']['var']])  + "\t" +\
+           "\t".join([varvar % x for x in config['stats']['var']])  + \
            "\t".join([sdvar % x for x in config['stats']['sd']]) + \
-           "\t   count" + \
+           "count\t" + \
-           "\t   total" 
+           "total" 
     print(hcat)
     sorted_stats = sorted(catcount.items(), key = lambda kv:(totcount[kv[0]]), reverse=True)
     for i in range(len(sorted_stats)):

python/obitools3/commands/tail.pyx

@@ -15,7 +15,7 @@ from cpython.exc cimport PyErr_CheckSignals
 from io import BufferedWriter
 
 
-__title__="Keep the N last lines of a view."
+__title__="Keep the N last lines of a view"
 
  
 def addOptions(parser):

python/obitools3/commands/test.pyx

@@ -39,7 +39,7 @@ COL_COMMENTS_MAX_LEN = 2048
 MAX_INT = 2147483647    # used to generate random float values
  
  
-__title__="Tests if the obitools are working properly"
+__title__="Test if the obitools are working properly"
 
 
 default_config = {

python/obitools3/dms/capi/obitypes.pxd

@@ -54,6 +54,8 @@ cdef extern from "obitypes.h" nogil:
     extern uint8_t*      OBIQual_int_NA
     extern void*         OBITuple_NA
     
+    extern obiint_t      OBI_INT_MAX
+
     const_char_p name_data_type(int data_type)
 
 ctypedef OBIType_t obitype_t

python/obitools3/dms/view/view.pyx

@@ -600,7 +600,8 @@ cdef class View(OBIWrapper) :
                 if element is not None:
                     if element.comments[b"input_dms_name"] is not None :
                         for i in range(len(element.comments[b"input_dms_name"])) :
-                            if element.comments[b"input_dms_name"][i] == element.dms.name and b"/" not in element.comments[b"input_view_name"][i]:  # Same DMS and not a special element like a taxonomy
+                            if b"/" not in element.comments[b"input_view_name"][i] and element.comments[b"input_view_name"][i] in element.dms \
+                            and element.comments[b"input_dms_name"][i] == element.dms.name :  # Same DMS and not a special element like a taxonomy and view was not deleted
                                 top_level.append(element.dms[element.comments[b"input_view_name"][i]])
                             else:
                                 top_level.append(None)

python/obitools3/parsers/tab.pyx

@@ -8,7 +8,7 @@ Created on feb 20th 2018
 
 import types
 from obitools3.utils cimport __etag__
-
+from obitools3.utils cimport str2bytes
 
 def tabIterator(lineiterator, 
                 bint header = False,
@@ -75,7 +75,7 @@ def tabIterator(lineiterator,
                 continue
             else:
                 # TODO ??? default column names? like R?
-                keys = [i for i in range(len(line.split(sep)))]
+                keys = [str2bytes(str(i)) for i in range(len(line.split(sep)))]
                 
         while skipped < skip :
             line = next(iterator)

python/obitools3/uri/decode.pyx

@@ -565,6 +565,6 @@ def open_uri(uri,
         
         entry_count = -1
         if input:
-            entry_count = count_entries(file, format)
+            entry_count = count_entries(file, format, header)
         
         return (file, iseq, objclass, urib, entry_count)

python/obitools3/utils.pxd

@@ -3,7 +3,7 @@
 from obitools3.dms.capi.obitypes cimport obitype_t, index_t
 
 cpdef bytes format_uniq_pattern(bytes format)
-cpdef int count_entries(file, bytes format)
+cpdef int count_entries(file, bytes format, bint header)
 
 cdef obi_errno_to_exception(index_t line_nb=*, object elt_id=*, str error_message=*)
 

python/obitools3/utils.pyx

@@ -9,7 +9,8 @@ from obitools3.dms.capi.obitypes cimport is_a_DNA_seq, \
                                          OBI_QUAL, \
                                          OBI_SEQ, \
                                          OBI_STR, \
-                                         index_t
+                                         index_t, \
+                                         OBI_INT_MAX
 
 from obitools3.dms.capi.obierrno cimport OBI_LINE_IDX_ERROR, \
                                          OBI_ELT_IDX_ERROR, \
@@ -39,7 +40,7 @@ cpdef bytes format_uniq_pattern(bytes format):
         return None
 
 
-cpdef int count_entries(file, bytes format):
+cpdef int count_entries(file, bytes format, bint header):
     
     try:
         sep = format_uniq_pattern(format)
@@ -74,6 +75,8 @@ cpdef int count_entries(file, bytes format):
             total_count += len(re.findall(sep, mmapped_file))
             if format != b"ngsfilter" and format != b"tabular" and format != b"embl" and format != b"genbank" and format != b"fastq":
                 total_count += 1 # adding +1 for 1st entry because separators include \n (ngsfilter and tabular already count one more because of last \n)
+            if format == b"tabular" and header: # not counting header as an entry
+                total_count -= 1
             
     except:
         if len(files) > 1:
@@ -258,7 +261,7 @@ cdef obitype_t update_obitype(obitype_t obitype, object new_value) :
     new_type = type(new_value)
     
     if obitype == OBI_INT :
-        if new_type == float :
+        if new_type == float or new_value > OBI_INT_MAX :
             return OBI_FLOAT
     # TODO BOOL vers INT/FLOAT
     elif new_type == str or new_type == bytes :

python/obitools3/version.py

@@ -1,5 +1,5 @@
 major = 3
 minor = 0
-serial= '1b3'
+serial= '1b7'
 
 version ="%d.%d.%s" % (major,minor,serial)

src/kmer_similarity.c

@@ -77,7 +77,6 @@ Obi_ali_p kmer_similarity(Obiview_p view1, OBIDMS_column_p column1, index_t idx1
 	int32_t*        shift_count_array;
 	Obi_ali_p       ali = NULL;
 	int 			i, j;
-	bool 			switched_seqs;
 	bool			reversed;
 	int 			score = 0;
 	Obi_blob_p      blob1 = NULL;
@@ -124,6 +123,8 @@ Obi_ali_p kmer_similarity(Obiview_p view1, OBIDMS_column_p column1, index_t idx1
 	bool			keep_seq2_start;
 	bool			keep_seq1_end;
 	bool			keep_seq2_end;
+	bool			left_ali;
+	bool 			rev_quals = false;
 
 	// Check kmer size
 	if ((kmer_size < 1) || (kmer_size > 4))
@@ -148,19 +149,8 @@ Obi_ali_p kmer_similarity(Obiview_p view1, OBIDMS_column_p column1, index_t idx1
 	}
 
 	// Choose the shortest sequence to save kmer positions in array
-	switched_seqs = false;
 	len1 = blob1->length_decoded_value;
 	len2 = blob2->length_decoded_value;
-	if (len2 < len1)
-	{
-		switched_seqs = true;
-		temp_len = len1;
-		len1 = len2;
-		len2 = temp_len;
-		temp_blob = blob1;
-		blob1 = blob2;
-		blob2 = temp_blob;
-	}
 
 	// Force encoding on 2 bits by replacing ambiguous nucleotides by 'a's
 	if (blob1->element_size == 4)
@@ -196,7 +186,47 @@ Obi_ali_p kmer_similarity(Obiview_p view1, OBIDMS_column_p column1, index_t idx1
 	else
 		reversed = false;
 	if (reversed)
-		switched_seqs = !switched_seqs;
+		// unreverse to make cases simpler. Costly but rare (direct match is reverse primer match)
+	{
+		if (seq2 == NULL)
+			seq2 = obi_blob_to_seq(blob2);
+		seq2 = reverse_complement_sequence(seq2);
+		blob2 = obi_seq_to_blob(seq2);
+
+		if (seq1 == NULL)
+			seq1 = obi_blob_to_seq(blob1);
+		seq1 = reverse_complement_sequence(seq1);
+		blob1 = obi_seq_to_blob(seq1);
+		free_blob1 = true;
+
+		// Get the quality arrays
+		qual1 = obi_get_qual_int_with_elt_idx_and_col_p_in_view(view1, qual_col1, idx1, 0, &qual1_len);
+		if (qual1 == NULL)
+		{
+			obidebug(1, "\nError getting the quality of the 1st sequence when computing the kmer similarity between two sequences");
+			return NULL;
+		}
+		qual2 = obi_get_qual_int_with_elt_idx_and_col_p_in_view(view2, qual_col2, idx2, 0, &qual2_len);
+		if (qual2 == NULL)
+		{
+			obidebug(1, "\nError getting the quality of the 2nd sequence when computing the kmer similarity between two sequences");
+			return NULL;
+		}
+
+		uint8_t* newqual1 = malloc(qual1_len*sizeof(uint8_t));
+		uint8_t* newqual2 = malloc(qual2_len*sizeof(uint8_t));
+
+		for (i=0;i<qual1_len;i++)
+			newqual1[i] = qual1[qual1_len-1-i];
+
+		for (i=0;i<qual2_len;i++)
+			newqual2[i] = qual2[qual2_len-1-i];
+
+		qual1 = newqual1;
+		qual2 = newqual2;
+
+		rev_quals = true;
+	}
 
 	// Save total length for the shift counts array
 	total_len = len1 + len2 + 1;   // +1 for shift 0
@@ -237,7 +267,7 @@ Obi_ali_p kmer_similarity(Obiview_p view1, OBIDMS_column_p column1, index_t idx1
 		 	return NULL;
 		}
 	}
-	else if (len1 >= shift_array_height)
+	else if (total_len >= shift_array_height)
 	{
 		shift_array_height = total_len;
 		*shift_array_p = (int32_t*) realloc(*shift_array_p, ARRAY_LENGTH * shift_array_height * sizeof(int32_t));
@@ -291,7 +321,7 @@ Obi_ali_p kmer_similarity(Obiview_p view1, OBIDMS_column_p column1, index_t idx1
 	*shift_array_height_p = shift_array_height;
 	*shift_count_array_length_p = shift_count_array_length;
 
-	// Fill array with positions of kmers in the shortest sequence
+	// Fill array with positions of kmers in the first sequence
 	encoding = blob1->element_size;
 	kmer_count = len1 - kmer_size + 1;
 	for (kmer_idx=0; kmer_idx < kmer_count; kmer_idx++)
@@ -310,7 +340,7 @@ Obi_ali_p kmer_similarity(Obiview_p view1, OBIDMS_column_p column1, index_t idx1
 		kmer_pos_array[(kmer*kmer_pos_array_height)+i] = kmer_idx;
 	}
 
-	// Compare positions of kmers between both sequences and store shifts
+	// Compare positions of kmers between both sequences and store shifts (a shift corresponds to pos2 - pos1)
 	kmer_count = blob2->length_decoded_value - kmer_size + 1;
 	for (kmer_idx=0; kmer_idx < kmer_count; kmer_idx++)
 	{
@@ -374,35 +404,42 @@ Obi_ali_p kmer_similarity(Obiview_p view1, OBIDMS_column_p column1, index_t idx1
 	// The 873863 cases of hell
 	if (best_shift > 0)
 	{
+		left_ali = false;
 		overlap_len = len2 - best_shift;
 		if (len1 <= overlap_len)
 		{
 			overlap_len = len1;
-			if (! switched_seqs)
 			keep_seq2_end = true;
-			else
-				keep_seq2_start = true;
-		}
-		else if (switched_seqs)
-		{
-			keep_seq2_start = true;
-			keep_seq1_end = true;
 		}
 	}
 	else if (best_shift < 0)
 	{
+		left_ali = true;
 	    overlap_len = len1 + best_shift;
-	    if (!switched_seqs)
+		if (len2 <= overlap_len)
+		{
+			overlap_len = len2;
+			keep_seq1_start = true;
+		}
+		else
 		{
 			keep_seq1_start = true;
 			keep_seq2_end = true;
 		}
 	}
-	else
+	else // if (best_shift == 0)
+	{
+		if (len2 >= len1)
 		{
 			overlap_len = len1;
-		if ((!switched_seqs) && (len2 > len1))
 			keep_seq2_end = true;
+			left_ali = true;
+		}
+		else
+		{
+			overlap_len = len2;
+			left_ali = false; // discussable
+		}
 	}
 
 	ali = (Obi_ali_p) malloc(sizeof(Obi_ali_t));
@@ -433,7 +470,7 @@ Obi_ali_p kmer_similarity(Obiview_p view1, OBIDMS_column_p column1, index_t idx1
 		ali->direction[0] = '\0';
 	else
 	{
-		if (((best_shift <= 0) && (!switched_seqs)) || ((best_shift > 0) && switched_seqs))
+		if (left_ali)
 			strcpy(ali->direction, "left");
 		else
 			strcpy(ali->direction, "right");
@@ -441,6 +478,8 @@ Obi_ali_p kmer_similarity(Obiview_p view1, OBIDMS_column_p column1, index_t idx1
 
 	// Build the consensus sequence if asked
 	if (build_consensus)
+	{
+		if (! rev_quals)
 		{
 			// Get the quality arrays
 			qual1 = obi_get_qual_int_with_elt_idx_and_col_p_in_view(view1, qual_col1, idx1, 0, &qual1_len);
@@ -455,15 +494,13 @@ Obi_ali_p kmer_similarity(Obiview_p view1, OBIDMS_column_p column1, index_t idx1
 				obidebug(1, "\nError getting the quality of the 2nd sequence when computing the kmer similarity between two sequences");
 				return NULL;
 			}
+		}
 
 		// Decode the first sequence if not already done
 		if (seq1 == NULL)
 			seq1 = obi_blob_to_seq(blob1);
 
-		if (! switched_seqs)
 		consensus_len = len2 - best_shift;
-		else
-		    consensus_len = len1 + best_shift;
 
 		// Allocate memory for consensus sequence
 		consensus_seq = (char*) malloc(consensus_len + 1 * sizeof(char)); // TODO keep malloced too maybe
@@ -557,6 +594,12 @@ Obi_ali_p kmer_similarity(Obiview_p view1, OBIDMS_column_p column1, index_t idx1
 	free(seq2);
 	free(blob2);
 
+	if (rev_quals)
+	{
+		free(qual1);
+		free(qual2);
+	}
+
 	return ali;
 }
 

src/obidms_taxonomy.c

@@ -2463,6 +2463,32 @@ int read_merged_dmp(const char* taxdump, OBIDMS_taxonomy_p tax, int32_t* delnode
 				return -1;
 			}
 
+			// Write the rest of the taxa from the current taxa list
+			while (nT < (tax->taxa)->count)
+			{
+			// Add element from taxa list
+				// Enlarge structure if needed
+				if (n == buffer_size)
+				{
+					buffer_size = buffer_size * 2;
+					tax->merged_idx = (ecomergedidx_t*) realloc(tax->merged_idx, sizeof(ecomergedidx_t) + sizeof(ecomerged_t) * buffer_size);
+					if (tax->merged_idx == NULL)
+					{
+						obi_set_errno(OBI_MALLOC_ERROR);
+						obidebug(1, "\nError reallocating memory for a taxonomy structure");
+						closedir(tax_dir);
+						fclose(file);
+						return -1;
+					}
+				}
+
+				(tax->merged_idx)->merged[n].taxid = (tax->taxa)->taxon[nT].taxid;
+				(tax->merged_idx)->merged[n].idx = nT;
+
+				nT++;
+				n++;
+			}
+
 			// Store count
 			(tax->merged_idx)->count = n;
 

src/obidms_taxonomy.h

@@ -75,7 +75,7 @@ typedef struct {
  	 	 	 	 	 	 	 */
 	int32_t max_taxid;		/**< Maximum taxid existing in the taxon index.
 	 	 	 	 	 	 	 */
-	int32_t buffer_size;	/**< Number of taxa.	// TODO kept this but not sure of its use
+	int32_t buffer_size;	/**< .	// TODO kept this but not sure of its use
  	 	 	 	 	 	 	 */
 	ecotx_t taxon[];		/**< Taxon array.
  	 	 	 	 	 	 	 */

src/obitypes.h

@@ -29,6 +29,8 @@
 #define OBIQual_int_NA (NULL)		/**< NA value for the type OBI_QUAL if the quality is in integer format */
 #define OBITuple_NA (NULL)			/**< NA value for tuples of any type */
 
+#define OBI_INT_MAX (INT32_MAX)     /**< Maximum value for the type OBI_INT */
+
 
 /**
  * @brief enum for the boolean OBIType.