Switch to version 3.0.0b24

not properly closed and imported

python/obitools3/commands/uniq.pyx

@@ -419,12 +419,12 @@ cdef uniq_sequences(View_NUC_SEQS view, View_NUC_SEQS o_view, ProgressBar pb, di
     print("")
     logger("info", "Second browsing through the input")
     # Initialize the progress bar
-    pb = ProgressBar(len(uniques), seconde=5)
+    pb = ProgressBar(len(view), seconde=5)
     o_idx = 0
+    total_treated = 0
     
     for unique_id in uniques :
         PyErr_CheckSignals()
-        pb(o_idx)
         
         merged_sequences = uniques[unique_id]
         
@@ -453,7 +453,8 @@ cdef uniq_sequences(View_NUC_SEQS view, View_NUC_SEQS o_view, ProgressBar pb, di
             merged_dict[mkey] = {}
 
         for i_idx in merged_sequences:
-                        
+            pb(total_treated)
+ 
             i_id = i_id_col[i_idx]
             i_seq = view[i_idx]
 
@@ -504,7 +505,9 @@ cdef uniq_sequences(View_NUC_SEQS view, View_NUC_SEQS o_view, ProgressBar pb, di
                 if key != COUNT_COLUMN and key != ID_COLUMN and key != NUC_SEQUENCE_COLUMN and key in o_seq and o_seq[key] != i_seq[key] \
                     and key not in merged_dict :
                     o_seq[key] = None
-
+            
+            total_treated += 1
+            
         # Write merged dicts
         for mkey in merged_dict: 
             if mkey in str_merged_cols:
@@ -526,7 +529,7 @@ cdef uniq_sequences(View_NUC_SEQS view, View_NUC_SEQS o_view, ProgressBar pb, di
         o_count_col[o_idx] = o_count
         o_idx += 1
     
-    pb(len(uniques), force=True)
+    pb(len(view), force=True)
     
     # Deletes quality columns if there is one because the matching between sequence and quality will be broken (quality set to NA when sequence not)
     if QUALITY_COLUMN in view:

python/obitools3/dms/view/view.pyx

@@ -533,6 +533,7 @@ cdef class View(OBIWrapper) :
             for command in command_list:
                 s+=b"obi "
                 s+=command
+                s+=b"\n"
         return s
 
 

python/obitools3/libalign/_solexapairend.pyx

@@ -188,7 +188,7 @@ def buildConsensus(ali, seq, ref_tags=None):
         seq[b'shift']=ali.shift
     else:
         if len(ali[0])>999:   # TODO why?
-            raise AssertionError,"Too long alignemnt"
+            raise AssertionError,"Too long alignment"
     
         ic=IterOnConsensus(ali)
     
@@ -250,11 +250,21 @@ def buildJoinedSequence(ali, reverse, seq, forward=None):
     quality.extend(reverse.quality)
     seq.set(forward.id +b"_PairedEnd", s, definition=forward.definition, quality=quality)
     seq[b"score"]=ali.score
-    seq[b"ali_direction"]=ali.direction
+    if len(ali.direction) > 0:
+        seq[b"ali_direction"]=ali.direction
+    else:
+        seq[b"ali_direction"]=None
     seq[b"mode"]=b"joined"
-    seq[b"pairedend_limit"]=len(forward)    
+    seq[b"pairedend_limit"]=len(forward)
+    seq[b"ali_length"] = ali.consensus_len
+    if ali.consensus_len > 0:
+        seq[b"score_norm"]=float(ali.score)/ali.consensus_len
+    else:
+        seq[b"score_norm"]=0.0
+  
     for tag in forward:
-        if tag != REVERSE_SEQUENCE_COLUMN and tag != REVERSE_QUALITY_COLUMN:
+        if tag != REVERSE_SEQUENCE_COLUMN and tag != REVERSE_QUALITY_COLUMN and \
+            tag != NUC_SEQUENCE_COLUMN and tag != QUALITY_COLUMN:
             seq[tag] = forward[tag]
     return seq
 

python/obitools3/version.py

@@ -1,5 +1,5 @@
 major = 3
 minor = 0
-serial= '0b23'
+serial= '0b24'
 
 version ="%d.%d.%s" % (major,minor,serial)

src/kmer_similarity.c

@@ -413,7 +413,10 @@ Obi_ali_p kmer_similarity(Obiview_p view1, OBIDMS_column_p column1, index_t idx1
 	 	return NULL;
 	}
 
-	score = max_common_kmers + kmer_size - 1;  // aka the number of nucleotides in the longest stretch of kmers perfectly matching
+	if (max_common_kmers > 0)
+		score = max_common_kmers + kmer_size - 1;  // aka the number of nucleotides in the longest stretch of kmers perfectly matching
+	else
+		score = 0;
 	abs_shift = abs(best_shift);
 
 	// Save result in Obi_ali structure
@@ -423,10 +426,15 @@ Obi_ali_p kmer_similarity(Obiview_p view1, OBIDMS_column_p column1, index_t idx1
 	ali->shift = abs_shift;
 	ali->consensus_seq = NULL;
 	ali->consensus_qual = NULL;
-	if (((best_shift <= 0) && (!switched_seqs)) || ((best_shift > 0) && switched_seqs))
+	if (score == 0)
-		strcpy(ali->direction, "left");
+		ali->direction[0] = '\0';
 	else
-		strcpy(ali->direction, "right");
+	{
+		if (((best_shift <= 0) && (!switched_seqs)) || ((best_shift > 0) && switched_seqs))
+			strcpy(ali->direction, "left");
+		else
+			strcpy(ali->direction, "right");
+	}
 
 	// Build the consensus sequence if asked
 	if (build_consensus)

src/obi_ecotag.c

@@ -71,9 +71,12 @@ static int create_output_columns(Obiview_p o_view);
  * @param name The assigned scientific name.
  * @param assigned_status_column A pointer on the column where the assigned status should be written.
  * @param assigned The assigned status (whether the sequence was assigned to a taxon or not).
- * @param best_match_column A pointer on the column where the list of ids of the best matches should be written.
+ * @param best_match_ids_column A pointer on the column where the list of ids of the best matches should be written.
  * @param best_match_ids The list of ids of the best matches as an array of the concatenated ids separated by '\0'.
  * @param best_match_ids_length The total length of the array of ids of best matches.
+ * @param best_match_taxids_column A pointer on the column where the list of taxids of the best matches should be written.
+ * @param best_match_taxids The list of taxids of the best matches as an array of the taxids.
+ * @param best_match_taxids_length The length of the array of taxids of best matches.
  * @param score_column A pointer on the column where the score should be written.
  * @param score The similarity score of the sequence with its best match(es).
  *
@@ -87,7 +90,8 @@ int print_assignment_result(Obiview_p output_view, index_t line,
 							OBIDMS_column_p assigned_taxid_column, int32_t taxid,
 							OBIDMS_column_p assigned_name_column, const char* name,
 							OBIDMS_column_p assigned_status_column, bool assigned,
-							OBIDMS_column_p best_match_column, const char* best_match_ids, int best_match_ids_length,
+							OBIDMS_column_p best_match_ids_column, const char* best_match_ids, int best_match_ids_length,
+							OBIDMS_column_p best_match_taxids_column, const int32_t* best_match_taxids, int best_match_taxids_length,
 							OBIDMS_column_p score_column, double score);
 
 
@@ -130,7 +134,14 @@ static int create_output_columns(Obiview_p o_view)
 	// Column for array of best match ids
 	if (obi_view_add_column(o_view, ECOTAG_BEST_MATCH_IDS_COLUMN_NAME, -1, NULL, OBI_STR, 0, 1, NULL, false, true, false, NULL, NULL, -1, "{}", true) < 0)
 	{
-		obidebug(1, "\nError creating the column for the array of ids of the best match in ecotag");
+		obidebug(1, "\nError creating the column for the array of ids of best matches in ecotag");
+		return -1;
+	}
+
+	// Column for array of best match taxids
+	if (obi_view_add_column(o_view, ECOTAG_BEST_MATCH_TAXIDS_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, true, false, NULL, NULL, -1, "{}", true) < 0)
+	{
+		obidebug(1, "\nError creating the column for the array of taxids of best matches in ecotag");
 		return -1;
 	}
 
@@ -142,7 +153,8 @@ int print_assignment_result(Obiview_p output_view, index_t line,
 							OBIDMS_column_p assigned_taxid_column, int32_t taxid,
 							OBIDMS_column_p assigned_name_column, const char* name,
 							OBIDMS_column_p assigned_status_column, bool assigned,
-							OBIDMS_column_p best_match_column, const char* best_match_ids, int best_match_ids_length,
+							OBIDMS_column_p best_match_ids_column, const char* best_match_ids, int best_match_ids_length,
+							OBIDMS_column_p best_match_taxids_column, const int32_t* best_match_taxids, int best_match_taxids_length,
 							OBIDMS_column_p score_column, double score)
 {
 	// Write the assigned taxid
@@ -167,9 +179,16 @@ int print_assignment_result(Obiview_p output_view, index_t line,
 	}
 
 	// Write the best match ids
-	if (obi_set_array_with_col_p_in_view(output_view, best_match_column, line, best_match_ids, (uint8_t)(sizeof(char)*8), best_match_ids_length) < 0)
+	if (obi_set_array_with_col_p_in_view(output_view, best_match_ids_column, line, best_match_ids, (uint8_t)(sizeof(char)*8), best_match_ids_length) < 0)
 	{
-		obidebug(1, "\nError writing a assignment status in a column when writing ecotag results");
+		obidebug(1, "\nError writing the array of best match ids in a column when writing ecotag results");
+		return -1;
+	}
+
+	// Write the best match taxids
+	if (obi_set_array_with_col_p_in_view(output_view, best_match_taxids_column, line, best_match_taxids, (uint8_t)(sizeof(OBI_INT)*8), best_match_taxids_length) < 0)
+	{
+		obidebug(1, "\nError writing the array of best match taxids in a column when writing ecotag results");
 		return -1;
 	}
 
@@ -235,6 +254,8 @@ int obi_ecotag(const char* dms_name,
 	char*     		best_match_ids;
 	char*			best_match_ids_to_store;
 	int32_t         best_match_ids_length;
+	int32_t*   		best_match_taxids;
+	int32_t*		best_match_taxids_to_store;
 	int				best_match_count;
 	int             buffer_size;
 	int 			best_match_ids_buffer_size;
@@ -263,7 +284,8 @@ int obi_ecotag(const char* dms_name,
 	OBIDMS_column_p assigned_taxid_column = NULL;
 	OBIDMS_column_p assigned_name_column = NULL;
 	OBIDMS_column_p assigned_status_column = NULL;
-	OBIDMS_column_p best_match_column = NULL;
+	OBIDMS_column_p best_match_ids_column = NULL;
+	OBIDMS_column_p best_match_taxids_column = NULL;
 	OBIDMS_column_p lca_taxid_a_column = NULL;
 	OBIDMS_column_p score_a_column = NULL;
 	OBIDMS_column_p ref_taxid_column = NULL;
@@ -396,7 +418,8 @@ int obi_ecotag(const char* dms_name,
 	assigned_taxid_column = obi_view_get_column(output_view, ECOTAG_TAXID_COLUMN_NAME);
 	assigned_name_column = obi_view_get_column(output_view, ECOTAG_NAME_COLUMN_NAME);
 	assigned_status_column = obi_view_get_column(output_view, ECOTAG_STATUS_COLUMN_NAME);
-	best_match_column = obi_view_get_column(output_view, ECOTAG_BEST_MATCH_IDS_COLUMN_NAME);
+	best_match_ids_column = obi_view_get_column(output_view, ECOTAG_BEST_MATCH_IDS_COLUMN_NAME);
+	best_match_taxids_column = obi_view_get_column(output_view, ECOTAG_BEST_MATCH_TAXIDS_COLUMN_NAME);
 	score_column = obi_view_get_column(output_view, ECOTAG_SCORE_COLUMN_NAME);
 
 	// Open the used reference columns
@@ -453,6 +476,14 @@ int obi_ecotag(const char* dms_name,
 		return -1;
 	}
 
+	best_match_taxids = (int32_t*) malloc(buffer_size* sizeof(int32_t));
+	if (best_match_taxids == NULL)
+	{
+		obi_set_errno(OBI_MALLOC_ERROR);
+		obidebug(1, "\nError allocating memory for the best match taxid array in ecotag");
+		return -1;
+	}
+
 	for (i=0; i < query_count; i++)
 	{
 		if (i%1000 == 0)
@@ -514,7 +545,7 @@ int obi_ecotag(const char* dms_name,
 
 				// Store in best match array
 
-				// Grow match array if needed
+				// Grow match and taxid array if needed
 				if (best_match_count == buffer_size)
 				{
 					buffer_size = buffer_size*2;
@@ -525,6 +556,13 @@ int obi_ecotag(const char* dms_name,
 						obidebug(1, "\nError reallocating match array when assigning");
 						return -1;
 					}
+					best_match_taxids = (int32_t*) realloc(best_match_taxids, buffer_size*sizeof(int32_t));
+					if (best_match_taxids == NULL)
+					{
+						obi_set_errno(OBI_MALLOC_ERROR);
+						obidebug(1, "\nError reallocating match taxids array when assigning");
+						return -1;
+					}
 				}
 
 				id = obi_get_str_with_elt_idx_and_col_p_in_view(ref_view, ref_id_column, j, 0);
@@ -545,6 +583,7 @@ int obi_ecotag(const char* dms_name,
 
 				// Save match
 				best_match_array[best_match_count] = j;
+				best_match_taxids[best_match_count] = obi_get_int_with_elt_idx_and_col_p_in_view(ref_view, ref_taxid_column, j, 0);
 				best_match_count++;
 				strcpy(best_match_ids+best_match_ids_length, id);
 				best_match_ids_length = best_match_ids_length + id_len + 1;
@@ -629,6 +668,7 @@ int obi_ecotag(const char* dms_name,
 			else
 				lca_name = lca->name;
 			best_match_ids_to_store = best_match_ids;
+			best_match_taxids_to_store = best_match_taxids;
 		}
 		else
 		{
@@ -636,6 +676,7 @@ int obi_ecotag(const char* dms_name,
 			lca_name = OBIStr_NA;
 			lca_taxid = OBIInt_NA;
 			best_match_ids_to_store = OBITuple_NA;
+			best_match_taxids_to_store = OBITuple_NA;
 			score = OBIFloat_NA;
 		}
 
@@ -644,7 +685,8 @@ int obi_ecotag(const char* dms_name,
 								    assigned_taxid_column, lca_taxid,
 									assigned_name_column, lca_name,
 									assigned_status_column, assigned,
-									best_match_column, best_match_ids_to_store, best_match_ids_length,
+									best_match_ids_column, best_match_ids_to_store, best_match_ids_length,
+									best_match_taxids_column, best_match_taxids_to_store, best_match_count,
 									score_column, best_score
 									) < 0)
 							return -1;
@@ -652,6 +694,7 @@ int obi_ecotag(const char* dms_name,
 
 	free(best_match_array);
 	free(best_match_ids);
+	free(best_match_taxids);
 
 	obi_close_taxonomy(taxonomy);
 	obi_save_and_close_view(query_view);

src/obi_ecotag.h

@@ -23,7 +23,8 @@
 #define ECOTAG_TAXID_COLUMN_NAME "TAXID"
 #define ECOTAG_NAME_COLUMN_NAME "SCIENTIFIC_NAME"
 #define ECOTAG_STATUS_COLUMN_NAME "ID_STATUS"
-#define ECOTAG_BEST_MATCH_IDS_COLUMN_NAME "BEST_MATCH"
+#define ECOTAG_BEST_MATCH_IDS_COLUMN_NAME "BEST_MATCH_IDS"
+#define ECOTAG_BEST_MATCH_TAXIDS_COLUMN_NAME "BEST_MATCH_TAXIDS"
 #define ECOTAG_SCORE_COLUMN_NAME "BEST_IDENTITY"
 
 

src/obidms.c

@@ -1496,7 +1496,7 @@ obiversion_t obi_import_column(const char* dms_path_1, const char* dms_path_2, c
 	memcpy(column_2->data, column_1->data, header_1->data_size);
 
 	// Copy the AVL files if there are some (overwriting the automatically created files)
-	if ((header_1->returned_data_type == OBI_STR) || (header_1->returned_data_type == OBI_SEQ) || (header_1->returned_data_type == OBI_QUAL))
+	if ((header_1->tuples) || ((header_1->returned_data_type == OBI_STR) || (header_1->returned_data_type == OBI_SEQ) || (header_1->returned_data_type == OBI_QUAL)))
 	{
 		avl_name_1 = (char*) malloc((strlen(header_1->indexer_name) + 1)  * sizeof(char));
 		if (avl_name_1 == NULL)

src/obidmscolumn.c

@@ -1693,8 +1693,8 @@ int obi_close_column(OBIDMS_column_p column)
 		if (obi_dms_unlist_column(column->dms, column) < 0)
 			ret_val = -1;
 
-		// If the data type is OBI_STR, OBI_SEQ or OBI_QUAL, the associated indexer is closed
+		// If it's a tuple column or the data type is OBI_STR, OBI_SEQ or OBI_QUAL, the associated indexer is closed
-		if (((column->header)->returned_data_type == OBI_STR) || ((column->header)->returned_data_type == OBI_SEQ) || ((column->header)->returned_data_type == OBI_QUAL))
+		if (((column->header)->tuples) || (((column->header)->returned_data_type == OBI_STR) || ((column->header)->returned_data_type == OBI_SEQ) || ((column->header)->returned_data_type == OBI_QUAL)))
 			if (obi_close_indexer(column->indexer) < 0)
 				ret_val = -1;