switch to version 3.0.1b6

'\'+'t' when parsing an option value

python/obitools3/commands/alignpairedend.pyx

@@ -205,19 +205,25 @@ def run(config):
     if type(entries) == list:
         forward = entries[0]
         reverse = entries[1]
-        aligner = Kmer_similarity(forward, \
-                                  view2=reverse, \
-                                  kmer_size=config['alignpairedend']['kmersize'], \
-                                  reversed_column=None)
+        if len(forward) == 0 or len(reverse) == 0:
+            aligner = None
+        else:
+            aligner = Kmer_similarity(forward, \
+                                      view2=reverse, \
+                                      kmer_size=config['alignpairedend']['kmersize'], \
+                                      reversed_column=None)
     else:
-        aligner = Kmer_similarity(entries, \
-                                  column2=entries[REVERSE_SEQUENCE_COLUMN], \
-                                  qual_column2=entries[REVERSE_QUALITY_COLUMN], \
-                                  kmer_size=config['alignpairedend']['kmersize'], \
-                                  reversed_column=entries[b'reversed'])  # column created by the ngsfilter tool
+        if len(entries) == 0:
+            aligner = None
+        else:
+            aligner = Kmer_similarity(entries, \
+                                      column2=entries[REVERSE_SEQUENCE_COLUMN], \
+                                      qual_column2=entries[REVERSE_QUALITY_COLUMN], \
+                                      kmer_size=config['alignpairedend']['kmersize'], \
+                                      reversed_column=entries[b'reversed'])  # column created by the ngsfilter tool
         
     ba = alignmentIterator(entries, aligner)
-
+    
     i = 0
     for ali in ba:
         
@@ -251,7 +257,7 @@ def run(config):
         pb(i, force=True)
         print("", file=sys.stderr)
 
-    if kmer_ali :
+    if kmer_ali and aligner is not None:
         aligner.free()
 
     # Save command config in View and DMS comments

python/obitools3/commands/import.pyx

@@ -371,8 +371,8 @@ def run(config):
                             # Fill value
                             dcols[tag][0][i] = value
                          
-                        except IndexError :
-                                                     
+                        except (IndexError, OverflowError):
+                                                                                 
                             value_type = type(value)
                             old_column = dcols[tag][0]
                             old_nb_elements_per_line = old_column.nb_elements_per_line
@@ -419,7 +419,7 @@ def run(config):
                                 dcols[tag][0][i] = value
         
         except Exception as e:
-            print("\nCould not import sequence id:", entry.id, "(error raised:", e, ")")
+            print("\nCould not import sequence:", entry, "(error raised:", e, ")")
             if 'skiperror' in config['obi'] and not config['obi']['skiperror']:
                 raise e
             else:

python/obitools3/commands/ngsfilter.pyx

@@ -322,7 +322,7 @@ cdef tuple annotate(sequences, infos, no_tags, verbose=False):
         sequences[0] = sequences[0][directmatch[1][2]:]
     else:
         sequences[1] = sequences[1][directmatch[1][2]:]
-        sequences[0][REVERSE_SEQUENCE_COLUMN] = sequences[1].seq           # used by alignpairedend tool
+        sequences[0][REVERSE_SEQUENCE_COLUMN] = sequences[1].seq      # used by alignpairedend tool
         sequences[0][REVERSE_QUALITY_COLUMN] = sequences[1].quality   # used by alignpairedend tool
     
     if directmatch[0].forward:
@@ -369,7 +369,7 @@ cdef tuple annotate(sequences, infos, no_tags, verbose=False):
                 sequences[0] = sequences[0][:r[1]]
             else:
                 sequences[1] = sequences[1][:r[1]]
-                sequences[0][REVERSE_SEQUENCE_COLUMN] = sequences[1].seq           # used by alignpairedend tool
+                sequences[0][REVERSE_SEQUENCE_COLUMN] = sequences[1].seq      # used by alignpairedend tool
                 sequences[0][REVERSE_QUALITY_COLUMN] = sequences[1].quality   # used by alignpairedend tool
         # do the same on the other seq
         if first_match_first_seq: 
@@ -394,7 +394,7 @@ cdef tuple annotate(sequences, infos, no_tags, verbose=False):
             seq_to_match = sequences[0]
         reversematch = []
         # Compute begin
-        begin=directmatch[1][2]+1  # end of match + 1 on the same sequence
+        #begin=directmatch[1][2]+1  # end of match + 1 on the same sequence -- No, already cut out forward primer        
         # Try reverse matching on the other sequence:
         new_seq = True
         pattern = 0
@@ -408,7 +408,7 @@ cdef tuple annotate(sequences, infos, no_tags, verbose=False):
                 primer=p
             # Saving original primer as 4th member of the tuple to serve as correct key in infos dict even if it might have been reversed complemented
             # (3rd member already used by directmatch)
-            reversematch.append((primer, primer(seq_to_match, same_sequence=not new_seq, pattern=pattern, begin=begin), None, p))
+            reversematch.append((primer, primer(seq_to_match, same_sequence=not new_seq, pattern=pattern, begin=0), None, p))
             new_seq = False
             pattern+=1
         # Choose match closer to the end of the sequence
@@ -645,6 +645,7 @@ def run(config):
     
     g = 0
     u = 0
+    i = 0
     no_tags = config['ngsfilter']['notags']
     try:
         for i in range(entries_len):

python/obitools3/commands/rm.pyx

@@ -0,0 +1,44 @@
+#cython: language_level=3
+
+from obitools3.uri.decode import open_uri
+from obitools3.apps.config import logger
+from obitools3.dms import DMS
+from obitools3.apps.optiongroups import addMinimalInputOption
+from obitools3.dms.view.view cimport View
+import os
+ 
+__title__="Delete a view"
+
+
+def addOptions(parser):    
+    addMinimalInputOption(parser)
+
+def run(config):
+
+    DMS.obi_atexit()
+    
+    logger("info", "obi rm")
+
+    # Open the input
+    input = open_uri(config['obi']['inputURI'])
+    if input is None:
+        raise Exception("Could not read input")
+    
+    # Check that it's a view
+    if isinstance(input[1], View) :
+        view = input[1]
+    else: 
+        raise NotImplementedError()
+    
+    # Get the path to the view file to remove
+    path = input[0].full_path  # dms path
+    path+=b"/VIEWS/"
+    path+=view.name
+    path+=b".obiview"
+    
+    # Close the view and the DMS
+    view.close()
+    input[0].close(force=True)
+    
+    # Rm
+    os.remove(path)

python/obitools3/commands/stats.pyx

@@ -238,14 +238,14 @@ def run(config):
     else:
         sdvar= "%s"
         
-    hcat = "\t".join([pcat % x for x in config['stats']['categories']]) + "\t" +\
-           "\t".join([minvar % x for x in config['stats']['minimum']])  + "\t" +\
-           "\t".join([maxvar % x for x in config['stats']['maximum']])  + "\t" +\
-           "\t".join([meanvar % x for x in config['stats']['mean']])  + "\t" +\
-           "\t".join([varvar % x for x in config['stats']['var']])  + "\t" +\
+    hcat = "\t".join([pcat % x for x in config['stats']['categories']]) + \
+           "\t".join([minvar % x for x in config['stats']['minimum']])  + \
+           "\t".join([maxvar % x for x in config['stats']['maximum']])  + \
+           "\t".join([meanvar % x for x in config['stats']['mean']])  + \
+           "\t".join([varvar % x for x in config['stats']['var']])  + \
            "\t".join([sdvar % x for x in config['stats']['sd']]) + \
-           "\t   count" + \
-           "\t   total" 
+           "count\t" + \
+           "total" 
     print(hcat)
     sorted_stats = sorted(catcount.items(), key = lambda kv:(totcount[kv[0]]), reverse=True)
     for i in range(len(sorted_stats)):

python/obitools3/commands/uniq.pyx

@@ -354,8 +354,8 @@ cdef uniq_sequences(View_NUC_SEQS view, View_NUC_SEQS o_view, ProgressBar pb, di
         key = mergedKeys[k]
         merged_col_name = mergedKeys_m[k]
         
-        if merged_infos[merged_col_name]['nb_elts'] == 1:
-            raise Exception("Can't merge information from a tag with only one element (e.g. one sample ; don't use -m option)")
+#        if merged_infos[merged_col_name]['nb_elts'] == 1:
+#            raise Exception("Can't merge information from a tag with only one element (e.g. one sample ; don't use -m option)")
         
         if merged_col_name in view:
             i_col = view[merged_col_name]

python/obitools3/dms/capi/obitypes.pxd

@@ -53,6 +53,8 @@ cdef extern from "obitypes.h" nogil:
     extern const_char_p  OBIQual_char_NA
     extern uint8_t*      OBIQual_int_NA
     extern void*         OBITuple_NA
+    
+    extern obiint_t      OBI_INT_MAX
 
     const_char_p name_data_type(int data_type)
 

python/obitools3/dms/view/view.pyx

@@ -600,7 +600,8 @@ cdef class View(OBIWrapper) :
                 if element is not None:
                     if element.comments[b"input_dms_name"] is not None :
                         for i in range(len(element.comments[b"input_dms_name"])) :
-                            if element.comments[b"input_dms_name"][i] == element.dms.name and b"/" not in element.comments[b"input_view_name"][i]:  # Same DMS and not a special element like a taxonomy
+                            if b"/" not in element.comments[b"input_view_name"][i] and element.comments[b"input_view_name"][i] in element.dms \
+                            and element.comments[b"input_dms_name"][i] == element.dms.name :  # Same DMS and not a special element like a taxonomy and view was not deleted
                                 top_level.append(element.dms[element.comments[b"input_view_name"][i]])
                             else:
                                 top_level.append(None)

python/obitools3/parsers/tab.pyx

@@ -8,7 +8,7 @@ Created on feb 20th 2018
 
 import types
 from obitools3.utils cimport __etag__
-
+from obitools3.utils cimport str2bytes
 
 def tabIterator(lineiterator, 
                 bint header = False,
@@ -75,7 +75,7 @@ def tabIterator(lineiterator,
                 continue
             else:
                 # TODO ??? default column names? like R?
-                keys = [i for i in range(len(line.split(sep)))]
+                keys = [str2bytes(str(i)) for i in range(len(line.split(sep)))]
                 
         while skipped < skip :
             line = next(iterator)

python/obitools3/parsers/universal.pyx

@@ -53,7 +53,11 @@ def entryIteratorFactory(lineiterator,
 
     i = iterator
         
-    first=next(i)    
+    try:
+        first=next(i)
+    except StopIteration:
+        first=""
+        pass
 
     format=b"tabular"
     

python/obitools3/uri/decode.pyx

@@ -386,10 +386,13 @@ def open_uri(uri,
             raise MalformedURIException('Malformed header argument in URI')
        
         if b"sep" in qualifiers:
-            sep=tobytes(qualifiers[b"sep"][0][0])
+            sep = tobytes(qualifiers[b"sep"][0][0])
         else:
             try:
-                sep=tobytes(config["obi"]["sep"])
+                sep = config["obi"]["sep"]
+                if sep == '\\t':   # dirty workaround for flake8(?) issue that reads '\t' as '\'+'t' when parsing the option value
+                    sep = '\t'
+                sep = tobytes(sep)
             except KeyError:
                 sep=None
         

python/obitools3/utils.pyx

@@ -9,7 +9,8 @@ from obitools3.dms.capi.obitypes cimport is_a_DNA_seq, \
                                          OBI_QUAL, \
                                          OBI_SEQ, \
                                          OBI_STR, \
-                                         index_t
+                                         index_t, \
+                                         OBI_INT_MAX
 
 from obitools3.dms.capi.obierrno cimport OBI_LINE_IDX_ERROR, \
                                          OBI_ELT_IDX_ERROR, \
@@ -258,7 +259,7 @@ cdef obitype_t update_obitype(obitype_t obitype, object new_value) :
     new_type = type(new_value)
     
     if obitype == OBI_INT :
-        if new_type == float :
+        if new_type == float or new_value > OBI_INT_MAX :
             return OBI_FLOAT
     # TODO BOOL vers INT/FLOAT
     elif new_type == str or new_type == bytes :

python/obitools3/version.py

@@ -1,5 +1,5 @@
 major = 3
 minor = 0
-serial= '0b43'
+serial= '1b6'
 
 version ="%d.%d.%s" % (major,minor,serial)

src/obi_clean.c

@@ -229,6 +229,8 @@ int obi_clean(const char* dms_name,
 		return -1;
 	}
 
+	seq_count = (i_view->infos)->line_count;
+
 	// Open the sequence column
 	if (strcmp((i_view->infos)->view_type, VIEW_TYPE_NUC_SEQS) == 0)
 		iseq_column = obi_view_get_column(i_view, NUC_SEQUENCE_COLUMN);
@@ -245,7 +247,7 @@ int obi_clean(const char* dms_name,
 	}
 
 	// Open the sample column if there is one
-	if ((strcmp(sample_column_name, "") == 0) || (sample_column_name == NULL))
+	if ((strcmp(sample_column_name, "") == 0) || (sample_column_name == NULL) || (seq_count == 0))
 	{
 		fprintf(stderr, "Info: No sample information provided, assuming one sample.\n");
 		sample_column = obi_view_get_column(i_view, COUNT_COLUMN);
@@ -340,66 +342,67 @@ int obi_clean(const char* dms_name,
 		return -1;
 	}
 
-	// Build kmer tables
-	ktable = hash_seq_column(i_view, iseq_column, 0);
-	if (ktable == NULL)
+	if (seq_count > 0)
 	{
-		obi_set_errno(OBI_CLEAN_ERROR);
-		obidebug(1, "\nError building kmer tables before aligning");
-		return -1;
-	}
+		// Build kmer tables
+		ktable = hash_seq_column(i_view, iseq_column, 0);
+		if (ktable == NULL)
+		{
+			obi_set_errno(OBI_CLEAN_ERROR);
+			obidebug(1, "\nError building kmer tables before aligning");
+			return -1;
+		}
 
-	seq_count = (i_view->infos)->line_count;
-
-	// Allocate arrays for sample counts otherwise reading in mapped files takes longer
-	complete_sample_count_array = (int*) malloc(seq_count * sample_count * sizeof(int));
-	if (complete_sample_count_array == NULL)
-	{
-		obi_set_errno(OBI_MALLOC_ERROR);
-		obidebug(1, "\nError allocating memory for the array of sample counts, size: %lld", seq_count * sample_count * sizeof(int));
-		return -1;
-	}
-	for (samp=0; samp < sample_count; samp++)
-	{
-		for (k=0; k<seq_count; k++)
-			complete_sample_count_array[k+(samp*seq_count)] = obi_get_int_with_elt_idx_and_col_p_in_view(i_view, sample_column, k, samp);
-	}
-
-	// Allocate arrays for blobs otherwise reading in mapped files takes longer
-	blob_array = (Obi_blob_p*) malloc(seq_count * sizeof(Obi_blob_p));
-	if (blob_array == NULL)
-	{
-		obi_set_errno(OBI_MALLOC_ERROR);
-		obidebug(1, "\nError allocating memory for the array of blobs");
-		return -1;
-	}
-	for (k=0; k<seq_count; k++)
-	{
-		blob_array[k] = obi_get_blob_with_elt_idx_and_col_p_in_view(i_view, iseq_column, k, 0);
-	}
-
-	// Allocate alignment result array (byte at 0 if not aligned yet,
-	//											1 if sequence at index has a similarity above the threshold with the current sequence,
-	//											2 if sequence at index has a similarity below the threshold with the current sequence)
-	alignment_result_array = (byte_t*) calloc(seq_count, sizeof(byte_t));
-	if (alignment_result_array == NULL)
-	{
-		obi_set_errno(OBI_MALLOC_ERROR);
-		obidebug(1, "\nError allocating memory for alignment result array");
-		return -1;
-	}
-
-	// Initialize all sequences to singletons or NA if no sequences in that sample
-	for (k=0; k<seq_count; k++)
-	{
+		// Allocate arrays for sample counts otherwise reading in mapped files takes longer
+		complete_sample_count_array = (int*) malloc(seq_count * sample_count * sizeof(int));
+		if (complete_sample_count_array == NULL)
+		{
+			obi_set_errno(OBI_MALLOC_ERROR);
+			obidebug(1, "\nError allocating memory for the array of sample counts, size: %lld", seq_count * sample_count * sizeof(int));
+			return -1;
+		}
 		for (samp=0; samp < sample_count; samp++)
 		{
-			if (obi_get_int_with_elt_idx_and_col_p_in_view(i_view, sample_column, k, samp) != OBIInt_NA)  // Only initialize samples where there are some sequences
+			for (k=0; k<seq_count; k++)
+				complete_sample_count_array[k+(samp*seq_count)] = obi_get_int_with_elt_idx_and_col_p_in_view(i_view, sample_column, k, samp);
+		}
+
+		// Allocate arrays for blobs otherwise reading in mapped files takes longer
+		blob_array = (Obi_blob_p*) malloc(seq_count * sizeof(Obi_blob_p));
+		if (blob_array == NULL)
+		{
+			obi_set_errno(OBI_MALLOC_ERROR);
+			obidebug(1, "\nError allocating memory for the array of blobs");
+			return -1;
+		}
+		for (k=0; k<seq_count; k++)
+		{
+			blob_array[k] = obi_get_blob_with_elt_idx_and_col_p_in_view(i_view, iseq_column, k, 0);
+		}
+
+		// Allocate alignment result array (byte at 0 if not aligned yet,
+		//											1 if sequence at index has a similarity above the threshold with the current sequence,
+		//											2 if sequence at index has a similarity below the threshold with the current sequence)
+		alignment_result_array = (byte_t*) calloc(seq_count, sizeof(byte_t));
+		if (alignment_result_array == NULL)
+		{
+			obi_set_errno(OBI_MALLOC_ERROR);
+			obidebug(1, "\nError allocating memory for alignment result array");
+			return -1;
+		}
+
+		// Initialize all sequences to singletons or NA if no sequences in that sample
+		for (k=0; k<seq_count; k++)
+		{
+			for (samp=0; samp < sample_count; samp++)
 			{
-				if (obi_set_char_with_elt_idx_and_col_p_in_view(o_view, status_column, k, samp, 's') < 0)
+				if (obi_get_int_with_elt_idx_and_col_p_in_view(i_view, sample_column, k, samp) != OBIInt_NA)  // Only initialize samples where there are some sequences
 				{
-					obidebug(1, "\nError initializing all sequences to singletons");
-					return -1;
+					if (obi_set_char_with_elt_idx_and_col_p_in_view(o_view, status_column, k, samp, 's') < 0)
+					{
+						obidebug(1, "\nError initializing all sequences to singletons");
+						return -1;
+					}
 				}
 			}
 		}
@@ -551,17 +554,20 @@ int obi_clean(const char* dms_name,
 		}
 	}
 
-	free_kmer_tables(ktable, seq_count);
-	free(complete_sample_count_array);
-	free(blob_array);
-	free(alignment_result_array);
+	if (seq_count > 0)
+	{
+		free_kmer_tables(ktable, seq_count);
+		free(complete_sample_count_array);
+		free(blob_array);
+		free(alignment_result_array);
+	}
 
 	fprintf(stderr, "\n");
 
 	if (stop)
 		return -1;
 
-	if (heads_only)
+	if (heads_only && (seq_count > 0))
 	{
 		line_selection = malloc((((o_view->infos)->line_count) + 1) * sizeof(index_t));
 		if (line_selection == NULL)
@@ -635,7 +641,7 @@ int obi_clean(const char* dms_name,
 	}
 
 	// Flag the end of the line selection
-	if (heads_only)
+	if (heads_only && (seq_count > 0))
 		line_selection[l] = -1;
 
 	// Create new view with line selection if heads only

src/obidms_taxonomy.c

@@ -2463,6 +2463,32 @@ int read_merged_dmp(const char* taxdump, OBIDMS_taxonomy_p tax, int32_t* delnode
 				return -1;
 			}
 
+			// Write the rest of the taxa from the current taxa list
+			while (nT < (tax->taxa)->count)
+			{
+			// Add element from taxa list
+				// Enlarge structure if needed
+				if (n == buffer_size)
+				{
+					buffer_size = buffer_size * 2;
+					tax->merged_idx = (ecomergedidx_t*) realloc(tax->merged_idx, sizeof(ecomergedidx_t) + sizeof(ecomerged_t) * buffer_size);
+					if (tax->merged_idx == NULL)
+					{
+						obi_set_errno(OBI_MALLOC_ERROR);
+						obidebug(1, "\nError reallocating memory for a taxonomy structure");
+						closedir(tax_dir);
+						fclose(file);
+						return -1;
+					}
+				}
+
+				(tax->merged_idx)->merged[n].taxid = (tax->taxa)->taxon[nT].taxid;
+				(tax->merged_idx)->merged[n].idx = nT;
+
+				nT++;
+				n++;
+			}
+
 			// Store count
 			(tax->merged_idx)->count = n;
 

src/obidms_taxonomy.h

@@ -75,7 +75,7 @@ typedef struct {
  	 	 	 	 	 	 	 */
 	int32_t max_taxid;		/**< Maximum taxid existing in the taxon index.
 	 	 	 	 	 	 	 */
-	int32_t buffer_size;	/**< Number of taxa.	// TODO kept this but not sure of its use
+	int32_t buffer_size;	/**< .	// TODO kept this but not sure of its use
  	 	 	 	 	 	 	 */
 	ecotx_t taxon[];		/**< Taxon array.
  	 	 	 	 	 	 	 */

src/obitypes.h

@@ -29,6 +29,8 @@
 #define OBIQual_int_NA (NULL)		/**< NA value for the type OBI_QUAL if the quality is in integer format */
 #define OBITuple_NA (NULL)			/**< NA value for tuples of any type */
 
+#define OBI_INT_MAX (INT32_MAX)     /**< Maximum value for the type OBI_INT */
+
 
 /**
  * @brief enum for the boolean OBIType.