obitools/obitools3
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 66 Wiki Activity

Columns: elements names informations are now kept in a memory arena of

Browse Source 
adapted size in the header, and added a boolean in the header indicating
whether the values should be evaluated (typically character strings to
be evaluated in Python)
This commit is contained in:
Celine Mercier
2017-12-13 22:46:50 +01:00
parent 2df5932b67
commit 1fd3323372
10 changed files with 177 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
python/obitools3/dms/capi/obidmscolumn.pxd
View File

@ -32,6 +32,7 @@ cdef extern from "obidmscolumn.h" nogil:
OBIType_t returned_data_type OBIType_t returned_data_type
OBIType_t stored_data_type OBIType_t stored_data_type
bint tuples bint tuples
bint to_eval
time_t creation_date time_t creation_date
obiversion_t version obiversion_t version
obiversion_t cloned_from obiversion_t cloned_from

2
python/obitools3/dms/capi/obiview.pxd
View File

@ -86,7 +86,9 @@ cdef extern from "obiview.h" nogil:
index_t nb_lines, index_t nb_lines,
index_t nb_elements_per_line, index_t nb_elements_per_line,
char* elements_names, char* elements_names,
bint elt_names_formatted,
bint tuples, bint tuples,
bint to_eval,
const_char_p indexer_name, const_char_p indexer_name,
const_char_p associated_column_name, const_char_p associated_column_name,
obiversion_t associated_column_version, obiversion_t associated_column_version,

3
python/obitools3/dms/column/typed_column/str.pyx
View File

@ -56,6 +56,9 @@ cdef class Column_str(Column_idx):
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem getting a value from a column") obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem getting a value from a column")
if value == OBIStr_NA : if value == OBIStr_NA :
result = None result = None
# For columns containing character strings that should be evaluated:
elif self.to_eval:
result = eval(value)
else : else :
result = <bytes> value # NOTE: value is not freed because the pointer points to a mmapped region in an AVL data file. result = <bytes> value # NOTE: value is not freed because the pointer points to a mmapped region in an AVL data file.
return result return result

3
python/obitools3/dms/view/view.pyx
View File

@ -330,6 +330,7 @@ cdef class View(OBIWrapper) :
line = self[idx] line = self[idx]
for k in item : for k in item :
# If setting from another View Line and the column doesn't exist, create it based on the informations from the other View # If setting from another View Line and the column doesn't exist, create it based on the informations from the other View
# TODO use clone_column
if isinstance(item, Line) and tostr(k) not in self: if isinstance(item, Line) and tostr(k) not in self:
col = item.view[k] col = item.view[k]
Column.new_column(self, Column.new_column(self,
@ -337,6 +338,8 @@ cdef class View(OBIWrapper) :
col.data_type_int, col.data_type_int,
nb_elements_per_line = col.nb_elements_per_line, nb_elements_per_line = col.nb_elements_per_line,
elements_names = col.elements_names, elements_names = col.elements_names,
tuples = col.tuples,
to_eval = col.to_eval,
comments = col.comments, comments = col.comments,
alias=k alias=k
) )

24
src/obi_align.c
View File

@ -154,35 +154,35 @@ static int create_alignment_output_columns(Obiview_p output_view,
bool normalize, int reference, bool similarity_mode) bool normalize, int reference, bool similarity_mode)
{ {
// Create the column for the ids of the 1st sequence aligned // Create the column for the ids of the 1st sequence aligned
if (obi_view_add_column(output_view, ID1_COLUMN_NAME, -1, NULL, OBI_STR, 0, 1, NULL, false, id1_indexer_name, NULL, -1, ID1_COLUMN_COMMENTS, true) < 0) if (obi_view_add_column(output_view, ID1_COLUMN_NAME, -1, NULL, OBI_STR, 0, 1, NULL, false, false, false, id1_indexer_name, NULL, -1, ID1_COLUMN_COMMENTS, true) < 0)
{ {
obidebug(1, "\nError creating the first column for the sequence ids when aligning"); obidebug(1, "\nError creating the first column for the sequence ids when aligning");
return -1; return -1;
} }
// Create the column for the ids of the 2nd sequence aligned // Create the column for the ids of the 2nd sequence aligned
if (obi_view_add_column(output_view, ID2_COLUMN_NAME, -1, NULL, OBI_STR, 0, 1, NULL, false, id2_indexer_name, NULL, -1, ID2_COLUMN_COMMENTS, true) < 0) if (obi_view_add_column(output_view, ID2_COLUMN_NAME, -1, NULL, OBI_STR, 0, 1, NULL, false, false, false, id2_indexer_name, NULL, -1, ID2_COLUMN_COMMENTS, true) < 0)
{ {
obidebug(1, "\nError creating the second column for the sequence ids when aligning"); obidebug(1, "\nError creating the second column for the sequence ids when aligning");
return -1; return -1;
} }
// Create the column for the index (in the input view) of the first sequences aligned // Create the column for the index (in the input view) of the first sequences aligned
if (obi_view_add_column(output_view, IDX1_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, NULL, NULL, -1, IDX1_COLUMN_COMMENTS, true) < 0) if (obi_view_add_column(output_view, IDX1_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, false, false, NULL, NULL, -1, IDX1_COLUMN_COMMENTS, true) < 0)
{ {
obidebug(1, "\nError creating the first column for the sequence indices when aligning"); obidebug(1, "\nError creating the first column for the sequence indices when aligning");
return -1; return -1;
} }
// Create the column for the index (in the input view) of the second sequences aligned // Create the column for the index (in the input view) of the second sequences aligned
if (obi_view_add_column(output_view, IDX2_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, NULL, NULL, -1, IDX2_COLUMN_COMMENTS, true) < 0) if (obi_view_add_column(output_view, IDX2_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, false, false, NULL, NULL, -1, IDX2_COLUMN_COMMENTS, true) < 0)
{ {
obidebug(1, "\nError creating the second column for the sequence indices when aligning"); obidebug(1, "\nError creating the second column for the sequence indices when aligning");
return -1; return -1;
} }
// Create the column for the LCS length // Create the column for the LCS length
if (obi_view_add_column(output_view, LCS_LENGTH_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, NULL, NULL, -1, LCS_LENGTH_COLUMN_COMMENTS, true) < 0) if (obi_view_add_column(output_view, LCS_LENGTH_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, false, false, NULL, NULL, -1, LCS_LENGTH_COLUMN_COMMENTS, true) < 0)
{ {
obidebug(1, "\nError creating the column for the LCS length when aligning"); obidebug(1, "\nError creating the column for the LCS length when aligning");
return -1; return -1;
@ -191,7 +191,7 @@ static int create_alignment_output_columns(Obiview_p output_view,
// Create the column for the alignment length if it is computed // Create the column for the alignment length if it is computed
if ((reference == ALILEN) && (normalize || !similarity_mode)) if ((reference == ALILEN) && (normalize || !similarity_mode))
{ {
if (obi_view_add_column(output_view, ALI_LENGTH_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, NULL, NULL, -1, ALI_LENGTH_COLUMN_COMMENTS, true) < 0) if (obi_view_add_column(output_view, ALI_LENGTH_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, false, false, NULL, NULL, -1, ALI_LENGTH_COLUMN_COMMENTS, true) < 0)
{ {
obidebug(1, "\nError creating the column for the alignment length when aligning"); obidebug(1, "\nError creating the column for the alignment length when aligning");
return -1; return -1;
@ -200,7 +200,7 @@ static int create_alignment_output_columns(Obiview_p output_view,
// Create the column for the alignment score // Create the column for the alignment score
if (normalize) if (normalize)
{ {
if (obi_view_add_column(output_view, SCORE_COLUMN_NAME, -1, NULL, OBI_FLOAT, 0, 1, NULL, false, NULL, NULL, -1, SCORE_COLUMN_NAME, true) < 0) if (obi_view_add_column(output_view, SCORE_COLUMN_NAME, -1, NULL, OBI_FLOAT, 0, 1, NULL, false, false, false, NULL, NULL, -1, SCORE_COLUMN_NAME, true) < 0)
{ {
obidebug(1, "\nError creating the column for the score when aligning"); obidebug(1, "\nError creating the column for the score when aligning");
return -1; return -1;
@ -208,7 +208,7 @@ static int create_alignment_output_columns(Obiview_p output_view,
} }
else else
{ {
if (obi_view_add_column(output_view, SCORE_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, NULL, NULL, -1, SCORE_COLUMN_NAME, true) < 0) if (obi_view_add_column(output_view, SCORE_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, false, false, NULL, NULL, -1, SCORE_COLUMN_NAME, true) < 0)
{ {
obidebug(1, "\nError creating the column for the score when aligning"); obidebug(1, "\nError creating the column for the score when aligning");
return -1; return -1;
@ -218,14 +218,14 @@ static int create_alignment_output_columns(Obiview_p output_view,
if (print_seq) if (print_seq)
{ {
// Create the column for the first sequences aligned // Create the column for the first sequences aligned
if (obi_view_add_column(output_view, SEQ1_COLUMN_NAME, -1, NULL, OBI_SEQ, 0, 1, NULL, false, seq1_indexer_name, NULL, -1, SEQ1_COLUMN_COMMENTS, true) < 0) if (obi_view_add_column(output_view, SEQ1_COLUMN_NAME, -1, NULL, OBI_SEQ, 0, 1, NULL, false, false, false, seq1_indexer_name, NULL, -1, SEQ1_COLUMN_COMMENTS, true) < 0)
{ {
obidebug(1, "\nError creating the first column for the sequences when aligning"); obidebug(1, "\nError creating the first column for the sequences when aligning");
return -1; return -1;
} }
// Create the column for the second sequences aligned // Create the column for the second sequences aligned
if (obi_view_add_column(output_view, SEQ2_COLUMN_NAME, -1, NULL, OBI_SEQ, 0, 1, NULL, false, seq2_indexer_name, NULL, -1, SEQ2_COLUMN_COMMENTS, true) < 0) if (obi_view_add_column(output_view, SEQ2_COLUMN_NAME, -1, NULL, OBI_SEQ, 0, 1, NULL, false, false, false, seq2_indexer_name, NULL, -1, SEQ2_COLUMN_COMMENTS, true) < 0)
{ {
obidebug(1, "\nError creating the second column for the sequences when aligning"); obidebug(1, "\nError creating the second column for the sequences when aligning");
return -1; return -1;
@ -234,14 +234,14 @@ static int create_alignment_output_columns(Obiview_p output_view,
// if (print_count) // TODO count columns not implemented yet // if (print_count) // TODO count columns not implemented yet
// { // {
// // Create the column for the count of the first sequences aligned // // Create the column for the count of the first sequences aligned
// if (obi_view_add_column(output_view, COUNT1_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, NULL, NULL, -1, COUNT1_COLUMN_COMMENTS, true) < 0) // if (obi_view_add_column(output_view, COUNT1_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, false, false, NULL, NULL, -1, COUNT1_COLUMN_COMMENTS, true) < 0)
// { // {
// obidebug(1, "\nError creating the first column for the sequence counts when aligning"); // obidebug(1, "\nError creating the first column for the sequence counts when aligning");
// return -1; // return -1;
// } // }
// //
// // Create the column for the count of the second sequences aligned // // Create the column for the count of the second sequences aligned
// if (obi_view_add_column(output_view, COUNT2_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, NULL, NULL, -1, COUNT2_COLUMN_COMMENTS, true) < 0) // if (obi_view_add_column(output_view, COUNT2_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, false, false, NULL, NULL, -1, COUNT2_COLUMN_COMMENTS, true) < 0)
// { // {
// obidebug(1, "\nError creating the second column for the sequence counts when aligning"); // obidebug(1, "\nError creating the second column for the sequence counts when aligning");
// return -1; // return -1;

6
src/obidms.c
View File

@ -1036,8 +1036,8 @@ obiversion_t obi_import_column(const char* dms_path_1, const char* dms_path_2, c
// Create new column // Create new column
column_2 = obi_create_column(dms_2, column_name, header_1->returned_data_type, header_1->line_count, column_2 = obi_create_column(dms_2, column_name, header_1->returned_data_type, header_1->line_count,
header_1->nb_elements_per_line, header_1->elements_names, true, header_1->tuples, header_1->nb_elements_per_line, header_1->elements_names, true, header_1->tuples,
new_avl_name, (header_1->associated_column).column_name, (header_1->associated_column).version, header_1->to_eval, new_avl_name, (header_1->associated_column).column_name,
header_1->comments); (header_1->associated_column).version, header_1->comments);
if (column_2 == NULL) if (column_2 == NULL)
{ {
@ -1254,6 +1254,8 @@ int obi_import_view(const char* dms_path_1, const char* dms_path_2, const char*
0, 0,
NULL, NULL,
false, false,
false,
false,
NULL, NULL,
NULL, NULL,
-1, -1,

179
src/obidmscolumn.c
View File

@ -139,12 +139,11 @@ static char* build_default_elements_names(index_t nb_elements_per_line);
* column with '\0' as separator (e.g. "0\01\02\0...\0n\0"). * column with '\0' as separator (e.g. "0\01\02\0...\0n\0").
* *
* @param elements_names The character string formatted with ';' as separator (e.g. "0;1;2;...;n\0"). * @param elements_names The character string formatted with ';' as separator (e.g. "0;1;2;...;n\0").
* @param elts_names_length A pointer on an integer where the function will store the length of the character string.
* *
* @since January 2017 * @since January 2017
* @author Celine Mercier (celine.mercier@metabarcoding.org) * @author Celine Mercier (celine.mercier@metabarcoding.org)
*/ */
static void format_elements_names(char* elements_names, int* elts_names_length); static void format_elements_names(char* elements_names);
/** /**
@ -188,7 +187,7 @@ static int cmp_elements_names_with_name_and_idx(const void* name1, const void* n
* *
* @param column A pointer as returned by obi_create_column(). * @param column A pointer as returned by obi_create_column().
* @param elements_names The names of the elements as formatted by format_elements_names(). * @param elements_names The names of the elements as formatted by format_elements_names().
* @param elts_names_length The length of elements_names. * @param elts_names_length The length of elements_names including the two terminal '\0's.
* *
* @retval 0 if the operation was successfully completed. * @retval 0 if the operation was successfully completed.
* @retval -1 if an error occurred. * @retval -1 if an error occurred.
@ -196,7 +195,19 @@ static int cmp_elements_names_with_name_and_idx(const void* name1, const void* n
* @since July 2015 * @since July 2015
* @author Celine Mercier (celine.mercier@metabarcoding.org) * @author Celine Mercier (celine.mercier@metabarcoding.org)
*/ */
static int set_elements_names(OBIDMS_column_p column, char* elements_names, int elts_names_length); static int set_elements_names(OBIDMS_column_p column, char* elements_names, int64_t elts_names_length, index_t nb_elements_per_line);
/**
* @brief Internal function reading the informations related to the elements names
* of the lines of a column in the header of the OBIDMS column structure.
*
* @param header A pointer on the header of the column.
*
* @since December 2017
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
static void read_elt_names_informations(OBIDMS_column_header_p header);
/** /**
@ -224,7 +235,7 @@ static index_t check_elt_names_count(const char* elements_names, bool elt_names_
* @since January 2017 * @since January 2017
* @author Celine Mercier (celine.mercier@metabarcoding.org) * @author Celine Mercier (celine.mercier@metabarcoding.org)
*/ */
static int get_formatted_elt_names_length(const char* elements_names); static int get_formatted_elt_names_length(const char* elements_names, index_t nb_elements);
/** /**
@ -521,7 +532,15 @@ static char* build_default_elements_names(index_t nb_elements_per_line)
int i; int i;
int len; int len;
elements_names = (char*) malloc(ELEMENTS_NAMES_MAX * sizeof(char)); // if (nb_elements_per_line > NB_ELTS_MAX_IF_DEFAULT_NAME)
// {
// obi_set_errno(OBICOL_UNKNOWN_ERROR);
// obidebug(1, "\nError: too many elements per line to use the default names (max = %d elements)", NB_ELTS_MAX_IF_DEFAULT_NAME);
// return NULL;
// }
// TODO
elements_names = (char*) malloc(nb_elements_per_line * 10 * sizeof(char));
if (elements_names == NULL) if (elements_names == NULL)
{ {
obi_set_errno(OBI_MALLOC_ERROR); obi_set_errno(OBI_MALLOC_ERROR);
@ -529,13 +548,6 @@ static char* build_default_elements_names(index_t nb_elements_per_line)
return NULL; return NULL;
} }
if (nb_elements_per_line > NB_ELTS_MAX_IF_DEFAULT_NAME)
{
obi_set_errno(OBICOL_UNKNOWN_ERROR);
obidebug(1, "\nError: too many elements per line to use the default names (max = %d elements)", NB_ELTS_MAX_IF_DEFAULT_NAME);
return NULL;
}
len = 0; len = 0;
for (i = 0; i < nb_elements_per_line; i++) for (i = 0; i < nb_elements_per_line; i++)
len += sprintf(elements_names+len, "%d;", i); len += sprintf(elements_names+len, "%d;", i);
@ -549,14 +561,15 @@ static char* build_default_elements_names(index_t nb_elements_per_line)
static void format_elements_names(char* elements_names, int* elts_names_length) static void format_elements_names(char* elements_names)
{ {
int i; int i;
int64_t elts_names_length;
*elts_names_length = strlen(elements_names); elts_names_length = strlen(elements_names);
// Replace the ';' with '\0' // Replace the ';' with '\0'
for (i=0; i < *elts_names_length; i++) for (i=0; i < elts_names_length; i++)
{ {
if (elements_names[i] == ';') if (elements_names[i] == ';')
elements_names[i] = '\0'; elements_names[i] = '\0';
@ -570,11 +583,11 @@ static int cmp_elements_names_with_idx(const void* n1_sort_idx, const void* n2_s
char* name1=NULL; char* name1=NULL;
char* name2=NULL; char* name2=NULL;
int name1_idx; index_t name1_idx;
int name2_idx; index_t name2_idx;
int name1_sort_idx = *((int*)n1_sort_idx); index_t name1_sort_idx = *((index_t*)n1_sort_idx);
int name2_sort_idx = *((int*)n2_sort_idx); index_t name2_sort_idx = *((index_t*)n2_sort_idx);
OBIDMS_column_header_p header = (OBIDMS_column_header_p) h; OBIDMS_column_header_p header = (OBIDMS_column_header_p) h;
name1_idx = (header->elements_names_idx)[name1_sort_idx]; name1_idx = (header->elements_names_idx)[name1_sort_idx];
@ -590,10 +603,10 @@ static int cmp_elements_names_with_idx(const void* n1_sort_idx, const void* n2_s
static int cmp_elements_names_with_name_and_idx(const void* name1, const void* n2_sort_idx, const void* h) static int cmp_elements_names_with_name_and_idx(const void* name1, const void* n2_sort_idx, const void* h)
{ {
char* name2=NULL; char* name2=NULL;
int name2_idx; index_t name2_idx;
int name2_sort_idx = *((int*)n2_sort_idx); index_t name2_sort_idx = *((index_t*)n2_sort_idx);
OBIDMS_column_header_p header = (OBIDMS_column_header_p) h; OBIDMS_column_header_p header = (OBIDMS_column_header_p) h;
name2_idx = (header->elements_names_idx)[name2_sort_idx]; name2_idx = (header->elements_names_idx)[name2_sort_idx];
@ -603,56 +616,67 @@ static int cmp_elements_names_with_name_and_idx(const void* name1, const void* n
} }
static int set_elements_names(OBIDMS_column_p column, char* elements_names, int64_t elts_names_length, index_t nb_elements)
static int set_elements_names(OBIDMS_column_p column, char* elements_names, int elts_names_length)
{ {
int i, j; OBIDMS_column_header_p header;
index_t i, j;
// Check that the elements names are not too long header = column->header;
if (elts_names_length+2 > ELEMENTS_NAMES_MAX)
{
obi_set_errno(OBICOL_UNKNOWN_ERROR);
obidebug(1, "\nError: element names too long (%d, max: %d)", elts_names_length+2, ELEMENTS_NAMES_MAX);
return -1;
}
// Copy the elements names in the header
memcpy((column->header)->elements_names, elements_names, elts_names_length*sizeof(char));
// Terminal characters
(column->header)->elements_names[elts_names_length] = '\0';
(column->header)->elements_names[elts_names_length + 1] = '\0';
// Store the length of the character array containing the elements names // Store the length of the character array containing the elements names
(column->header)->elements_names_length = elts_names_length; header->elements_names_length = elts_names_length;
// Store the pointers pointing to the different elements stored in the memory arena
header->elements_names = (char*)&(header->mem_arena)[0];
header->elements_names_idx = (index_t*)&((char*)(header->mem_arena) + elts_names_length)[0];
header->sorted_elements_idx = (header->elements_names_idx) + nb_elements;
// Copy the elements names in the header
memcpy(header->elements_names, elements_names, (elts_names_length-2)*sizeof(char));
// Terminal characters
header->elements_names[elts_names_length - 2] = '\0';
header->elements_names[elts_names_length - 1] = '\0';
// Build the elements names index // Build the elements names index
i = 0; i = 0;
j = 0; j = 0;
// Index the first element name // Index the first element name
((column->header)->elements_names_idx)[j] = i; (header->elements_names_idx)[j] = i;
((column->header)->sorted_elements_idx)[j] = j; (header->sorted_elements_idx)[j] = j;
i++; i++;
j++; j++;
while (i < elts_names_length)
while (i < elts_names_length-2)
{ {
if (elements_names[i] == '\0') if (elements_names[i] == '\0')
{ // Index new element name { // Index new element name
((column->header)->elements_names_idx)[j] = i+1; (header->elements_names_idx)[j] = i+1;
((column->header)->sorted_elements_idx)[j] = j; (header->sorted_elements_idx)[j] = j;
j++; j++;
} }
i++; i++;
} }
// Build the sorted index // Build the sorted index
qsort_user_data((column->header)->sorted_elements_idx, j, sizeof(int), column->header, cmp_elements_names_with_idx); qsort_user_data(header->sorted_elements_idx, j, sizeof(index_t), column->header, cmp_elements_names_with_idx);
return 0; return 0;
} }
static void read_elt_names_informations(OBIDMS_column_header_p header)
{
int64_t elts_names_length;
elts_names_length = header->elements_names_length;
header->elements_names = (char*)&(header->mem_arena)[0];
header->elements_names_idx = (index_t*)&((char*)(header->mem_arena) + elts_names_length)[0];
header->sorted_elements_idx = (index_t*)&((header->elements_names_idx) + (header->nb_elements_per_line))[0];
}
static index_t check_elt_names_count(const char* elements_names, bool elt_names_formatted) static index_t check_elt_names_count(const char* elements_names, bool elt_names_formatted)
{ {
char sep; char sep;
@ -680,20 +704,19 @@ static index_t check_elt_names_count(const char* elements_names, bool elt_names_
static int get_formatted_elt_names_length(const char* elements_names) static int get_formatted_elt_names_length(const char* elements_names, index_t nb_elements)
{ {
int i = 0; int i = 0;
bool stop = false; index_t n = 0;
while (! stop) while (n < nb_elements)
{ {
if ((elements_names[i] == '\0') && (elements_names[i+1] == '\0')) if (elements_names[i] == '\0')
stop = true; n++;
else i++;
i++;
} }
return i; return i+1;
} }
@ -866,14 +889,15 @@ obiversion_t obi_column_get_latest_version_from_name(OBIDMS_p dms, const char* c
} }
// TODO make private
size_t obi_get_platform_header_size() size_t obi_calculate_header_size(index_t nb_elements_per_line, int64_t elts_names_length)
{ {
size_t header_size; size_t header_size;
size_t rounded_header_size; size_t rounded_header_size;
double multiple; double multiple;
header_size = sizeof(OBIDMS_column_header_t); header_size = sizeof(OBIDMS_column_header_t);
header_size = header_size + (nb_elements_per_line*2)*sizeof(int64_t) + elts_names_length*sizeof(char);
multiple = ceil((double) header_size / (double) getpagesize()); multiple = ceil((double) header_size / (double) getpagesize());
@ -891,6 +915,7 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms,
char* elements_names, char* elements_names,
bool elt_names_formatted, bool elt_names_formatted,
bool tuples, bool tuples,
bool to_eval,
const char* indexer_name, const char* indexer_name,
const char* associated_column_name, const char* associated_column_name,
obiversion_t associated_column_version, obiversion_t associated_column_version,
@ -911,7 +936,7 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms,
OBIType_t stored_data_type; OBIType_t stored_data_type;
char* final_indexer_name; char* final_indexer_name;
char* built_elements_names = NULL; char* built_elements_names = NULL;
int elts_names_length; int64_t elts_names_length;
new_column = NULL; new_column = NULL;
@ -1012,12 +1037,11 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms,
// Format the elements names string // Format the elements names string
if (! elt_names_formatted) if (! elt_names_formatted)
format_elements_names(elements_names, &elts_names_length); format_elements_names(elements_names);
else elts_names_length = get_formatted_elt_names_length(elements_names, nb_elements_per_line);
elts_names_length = get_formatted_elt_names_length(elements_names);
// Calculate the size needed // Calculate the size needed
header_size = obi_get_platform_header_size(); header_size = obi_calculate_header_size(nb_elements_per_line, elts_names_length);
data_size = obi_array_sizeof(stored_data_type, nb_lines, nb_elements_per_line); data_size = obi_array_sizeof(stored_data_type, nb_lines, nb_elements_per_line);
file_size = header_size + data_size; file_size = header_size + data_size;
@ -1107,12 +1131,14 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms,
header->stored_data_type = stored_data_type; header->stored_data_type = stored_data_type;
header->returned_data_type = returned_data_type; header->returned_data_type = returned_data_type;
header->tuples = tuples; header->tuples = tuples;
header->to_eval = to_eval;
header->creation_date = time(NULL); header->creation_date = time(NULL);
header->version = version_number; header->version = version_number;
header->cloned_from = -1; header->cloned_from = -1;
header->finished = false; header->finished = false;
set_elements_names(new_column, elements_names, elts_names_length); set_elements_names(new_column, elements_names, elts_names_length, nb_elements_per_line);
read_elt_names_informations(header);
// Free the element names if they were built // Free the element names if they were built
if (built_elements_names != NULL) if (built_elements_names != NULL)
@ -1237,7 +1263,7 @@ OBIDMS_column_p obi_open_column(OBIDMS_p dms,
} }
// Open the column file, ALWAYS READ-ONLY // Open the column file, ALWAYS READ-ONLY
column_file_descriptor = openat(column_directory->dir_fd, column_file_name, O_RDONLY); column_file_descriptor = openat(column_directory->dir_fd, column_file_name, O_RDWR);
if (column_file_descriptor < 0) if (column_file_descriptor < 0)
{ {
obi_set_errno(OBICOL_UNKNOWN_ERROR); obi_set_errno(OBICOL_UNKNOWN_ERROR);
@ -1273,7 +1299,7 @@ OBIDMS_column_p obi_open_column(OBIDMS_p dms,
column->column_directory = column_directory; column->column_directory = column_directory;
column->header = mmap(NULL, column->header = mmap(NULL,
header_size, header_size,
PROT_READ, PROT_READ | PROT_WRITE,
MAP_SHARED, MAP_SHARED,
column_file_descriptor, column_file_descriptor,
0 0
@ -1288,6 +1314,9 @@ OBIDMS_column_p obi_open_column(OBIDMS_p dms,
return NULL; return NULL;
} }
// Set the pointers relative to elements names informations in the header
read_elt_names_informations(column->header);
// Map the data // Map the data
column->data = mmap(NULL, column->data = mmap(NULL,
(column->header)->data_size, (column->header)->data_size,
@ -1380,6 +1409,7 @@ OBIDMS_column_p obi_clone_column(OBIDMS_p dms,
(column_to_clone->header)->elements_names, (column_to_clone->header)->elements_names,
true, true,
(column_to_clone->header)->tuples, (column_to_clone->header)->tuples,
(column_to_clone->header)->to_eval,
(column_to_clone->header)->indexer_name, (column_to_clone->header)->indexer_name,
((column_to_clone->header)->associated_column).column_name, ((column_to_clone->header)->associated_column).column_name,
((column_to_clone->header)->associated_column).version, ((column_to_clone->header)->associated_column).version,
@ -1821,8 +1851,8 @@ OBIDMS_column_header_p obi_column_get_header_from_name(OBIDMS_p dms, const char*
return NULL; return NULL;
} }
// Open the column file (READ-ONLY) // Open the column file
column_file_descriptor = openat(column_directory->dir_fd, column_file_name, O_RDONLY); column_file_descriptor = openat(column_directory->dir_fd, column_file_name, O_RDWR);
if (column_file_descriptor < 0) if (column_file_descriptor < 0)
{ {
obidebug(1, "\nError opening a column file"); obidebug(1, "\nError opening a column file");
@ -1842,10 +1872,10 @@ OBIDMS_column_header_p obi_column_get_header_from_name(OBIDMS_p dms, const char*
return NULL; return NULL;
} }
// Fill the header structure // Map the header structure
header = mmap(NULL, header = mmap(NULL,
header_size, header_size,
PROT_READ, PROT_READ | PROT_WRITE,
MAP_SHARED, MAP_SHARED,
column_file_descriptor, column_file_descriptor,
0 0
@ -1859,6 +1889,9 @@ OBIDMS_column_header_p obi_column_get_header_from_name(OBIDMS_p dms, const char*
return NULL; return NULL;
} }
// Read the element names informations (storing pointers on informations)
read_elt_names_informations(header);
if (close(column_file_descriptor) < 0) if (close(column_file_descriptor) < 0)
{ {
obi_set_errno(OBICOL_UNKNOWN_ERROR); obi_set_errno(OBICOL_UNKNOWN_ERROR);
@ -1888,7 +1921,7 @@ index_t obi_column_get_element_index_from_name(OBIDMS_column_p column, const cha
{ {
int* elt_names_idx; int* elt_names_idx;
elt_names_idx = bsearch_user_data(element_name, (column->header)->sorted_elements_idx, (column->header)->nb_elements_per_line, sizeof(int), column->header, cmp_elements_names_with_name_and_idx); elt_names_idx = bsearch_user_data(element_name, (column->header)->sorted_elements_idx, (column->header)->nb_elements_per_line, sizeof(index_t), column->header, cmp_elements_names_with_name_and_idx);
if (elt_names_idx != NULL) if (elt_names_idx != NULL)
return (index_t)(*elt_names_idx); return (index_t)(*elt_names_idx);
@ -1906,7 +1939,7 @@ char* obi_get_elements_names(OBIDMS_column_p column)
int elt_idx; int elt_idx;
int len; int len;
elements_names = (char*) malloc(ELEMENTS_NAMES_MAX * sizeof(char)); elements_names = (char*) malloc((column->header)->elements_names_length * sizeof(char));
if (elements_names == NULL) if (elements_names == NULL)
{ {
obi_set_errno(OBI_MALLOC_ERROR); obi_set_errno(OBI_MALLOC_ERROR);
@ -2148,6 +2181,8 @@ int obi_clean_unfinished_columns(OBIDMS_p dms)
// dir_to_delete[ddir] = column_dir_path; // dir_to_delete[ddir] = column_dir_path;
// ddir++; // ddir++;
// } // }
// else
// free(column_dir_path);
free(col_name); free(col_name);
} }

45
src/obidmscolumn.h
View File

@ -28,9 +28,10 @@
#include "obiblob_indexer.h" #include "obiblob_indexer.h"
#define ELEMENTS_NAMES_MAX (20480) /**< The maximum length of the list of elements names. // TODO Discuss // TODO delete useless ones, add default nb?
#define ELEMENTS_NAMES_MAX (1000000) /**< The maximum length of the list of elements names. // TODO Discuss
*/ */
#define NB_ELTS_MAX_IF_DEFAULT_NAME (20480) /**< The maximum number of elements per line if the default element names #define NB_ELTS_MAX_IF_DEFAULT_NAME (1000000) /**< The maximum number of elements per line if the default element names
* are used ("0\01\02\0...\0n"), considering ELEMENTS_NAMES_MAX. // TODO not up to date * are used ("0\01\02\0...\0n"), considering ELEMENTS_NAMES_MAX. // TODO not up to date
*/ */
#define COLUMN_GROWTH_FACTOR (2) /**< The growth factor when a column is enlarged. #define COLUMN_GROWTH_FACTOR (2) /**< The growth factor when a column is enlarged.
@ -70,16 +71,6 @@ typedef struct OBIDMS_column_header {
*/ */
index_t nb_elements_per_line; /**< Number of elements per line. index_t nb_elements_per_line; /**< Number of elements per line.
*/ */
char elements_names[ELEMENTS_NAMES_MAX+1]; /**< Names of the line elements with '\0' as separator
* and '\0\0' as terminal flag.
* (default are the indices: "0\01\02\0...\0n\0\0").
*/
int elements_names_length; /**< Length of the character array where the elements names are stored.
*/
int elements_names_idx[NB_ELTS_MAX_IF_DEFAULT_NAME]; /**< Index for the start of each element name in elements_names.
*/
int sorted_elements_idx[NB_ELTS_MAX_IF_DEFAULT_NAME]; /**< Index for the sorted element names in elements_names_idx.
*/
OBIType_t returned_data_type; /**< Type of the data that is returned when getting an OBIType_t returned_data_type; /**< Type of the data that is returned when getting an
* element from the column. * element from the column.
*/ */
@ -88,6 +79,9 @@ typedef struct OBIDMS_column_header {
*/ */
bool tuples; /**< A boolean indicating whether the column contains indices referring to indexed tuples. bool tuples; /**< A boolean indicating whether the column contains indices referring to indexed tuples.
*/ */
bool to_eval; /**< A boolean indicating whether the column contains expressions that should be evaluated
* (typically OBI_STR columns containing character strings to be evaluated by Python).
*/
time_t creation_date; /**< Date of creation of the file. time_t creation_date; /**< Date of creation of the file.
*/ */
obiversion_t version; /**< Version of the column. obiversion_t version; /**< Version of the column.
@ -106,6 +100,18 @@ typedef struct OBIDMS_column_header {
*/ */
char comments[COMMENTS_MAX_LENGTH+1]; /**< Comments stored as a classical zero end C string. char comments[COMMENTS_MAX_LENGTH+1]; /**< Comments stored as a classical zero end C string.
*/ */
int64_t elements_names_length; /**< Length of the character array where the elements names are stored.
*/
char* elements_names; /**< Pointer in mem_arnea on the names of the line elements with '\0' as separator
* and '\0\0' as terminal flag.
* (default are the indices: "0\01\02\0...\0n\0\0").
*/
int64_t* elements_names_idx; /**< Pointer in mem_arnea on the index for the start of each element name in elements_names.
*/
int64_t* sorted_elements_idx; /**< Index for the sorted element names in elements_names_idx.
*/
byte_t mem_arena[]; /**< Memory array where the elements names, the elements names index and the sorted elements index are stored.
*/
} OBIDMS_column_header_t, *OBIDMS_column_header_p; } OBIDMS_column_header_t, *OBIDMS_column_header_p;
@ -212,17 +218,19 @@ obiversion_t obi_column_get_latest_version_from_name(OBIDMS_p dms, const char* c
/** /**
* @brief Returns the header size in bytes of a column on this platform. * @brief Returns the header size in bytes of a column.
* *
* The header size is defined as a multiple of the memory page size. * The header size is rounded to a multiple of the memory page size.
* As of now the header size is defined as one time the page size. *
* @param nb_elements_per_line The number of elements per line.
* @param elts_names_length The length of elements_names including the two terminal '\0's.
* *
* @returns The header size in bytes. * @returns The header size in bytes.
* *
* @since May 2015 * @since May 2015
* @author Eric Coissac (eric.coissac@metabarcoding.org) * @author Celine Mercier (celine.mercier@metabarcoding.org)
*/ */
size_t obi_get_platform_header_size(); size_t obi_calculate_header_size(index_t nb_elements_per_line, int64_t elts_names_length);
/** /**
@ -242,6 +250,8 @@ size_t obi_get_platform_header_size();
* NULL or "" if the default names are to be used ("0\01\02\0...\0n"). * NULL or "" if the default names are to be used ("0\01\02\0...\0n").
* @param elt_names_formatted Whether the separator for the elements names is ';' (false), or '\0' (true, as formatted by format_elements_names()). * @param elt_names_formatted Whether the separator for the elements names is ';' (false), or '\0' (true, as formatted by format_elements_names()).
* @param tuples A boolean indicating whether the column should contain indices referring to indexed tuples. * @param tuples A boolean indicating whether the column should contain indices referring to indexed tuples.
* @param to_eval A boolean indicating whether the column contains expressions that should be evaluated
* (typically OBI_STR columns containing character strings to be evaluated by Python).
* @param indexer_name The name of the indexer if there is one associated with the column. * @param indexer_name The name of the indexer if there is one associated with the column.
* If NULL or "", the indexer name is set as the column name. * If NULL or "", the indexer name is set as the column name.
* @param associated_column_name The name of the associated column if there is one (otherwise NULL or ""). * @param associated_column_name The name of the associated column if there is one (otherwise NULL or "").
@ -262,6 +272,7 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms,
char* elements_names, char* elements_names,
bool elt_names_formatted, bool elt_names_formatted,
bool tuples, bool tuples,
bool to_eval,
const char* indexer_name, const char* indexer_name,
const char* associated_column_name, const char* associated_column_name,
obiversion_t associated_column_version, obiversion_t associated_column_version,

20
src/obiview.c
View File

@ -1646,7 +1646,7 @@ Obiview_p obi_new_view(OBIDMS_p dms, const char* view_name, Obiview_p view_to_cl
// If there is a new line selection, build it by combining it with the one from the view to clone if there is one // If there is a new line selection, build it by combining it with the one from the view to clone if there is one
else if (line_selection != NULL) else if (line_selection != NULL)
{ {
view->line_selection = obi_create_column(view->dms, LINES_COLUMN_NAME, OBI_IDX, 0, 1, NULL, false, false, NULL, NULL, -1, NULL); view->line_selection = obi_create_column(view->dms, LINES_COLUMN_NAME, OBI_IDX, 0, 1, NULL, false, false, false, NULL, NULL, -1, NULL);
if ((view->line_selection) == NULL) if ((view->line_selection) == NULL)
{ {
obidebug(1, "\nError creating a column corresponding to a line selection"); obidebug(1, "\nError creating a column corresponding to a line selection");
@ -1799,6 +1799,8 @@ Obiview_p obi_new_view(OBIDMS_p dms, const char* view_name, Obiview_p view_to_cl
0, 0,
NULL, NULL,
false, false,
false,
false,
NULL, NULL,
NULL, NULL,
-1, -1,
@ -1866,19 +1868,19 @@ Obiview_p obi_new_view_nuc_seqs(OBIDMS_p dms, const char* view_name, Obiview_p v
if (view_to_clone == NULL) if (view_to_clone == NULL)
{ {
// Adding sequence column // Adding sequence column
if (obi_view_add_column(view, NUC_SEQUENCE_COLUMN, -1, NULL, OBI_SEQ, 0, 1, NULL, false, NULL, NULL, -1, "Nucleotide sequences", true) < 0) // discuss using same indexer "NUC_SEQ_INDEXER" if (obi_view_add_column(view, NUC_SEQUENCE_COLUMN, -1, NULL, OBI_SEQ, 0, 1, NULL, false, false, false, NULL, NULL, -1, "Nucleotide sequences", true) < 0) // discuss using same indexer "NUC_SEQ_INDEXER"
{ {
obidebug(1, "Error adding an obligatory column in a nucleotide sequences view"); obidebug(1, "Error adding an obligatory column in a nucleotide sequences view");
return NULL; return NULL;
} }
// Adding id column // Adding id column
if (obi_view_add_column(view, ID_COLUMN, -1, NULL, OBI_STR, 0, 1, NULL, false, NULL, NULL, -1, "Sequence identifiers", true) < 0) if (obi_view_add_column(view, ID_COLUMN, -1, NULL, OBI_STR, 0, 1, NULL, false, false, false, NULL, NULL, -1, "Sequence identifiers", true) < 0)
{ {
obidebug(1, "Error adding an obligatory column in a nucleotide sequences view"); obidebug(1, "Error adding an obligatory column in a nucleotide sequences view");
return NULL; return NULL;
} }
// Adding definition column // Adding definition column
if (obi_view_add_column(view, DEFINITION_COLUMN, -1, NULL, OBI_STR, 0, 1, NULL, false, NULL, NULL, -1, "Definitions", true) < 0) if (obi_view_add_column(view, DEFINITION_COLUMN, -1, NULL, OBI_STR, 0, 1, NULL, false, false, false, NULL, NULL, -1, "Definitions", true) < 0)
{ {
obidebug(1, "Error adding an obligatory column in a nucleotide sequences view"); obidebug(1, "Error adding an obligatory column in a nucleotide sequences view");
return NULL; return NULL;
@ -1887,7 +1889,7 @@ Obiview_p obi_new_view_nuc_seqs(OBIDMS_p dms, const char* view_name, Obiview_p v
if (quality_column) if (quality_column)
{ {
associated_nuc_column = obi_view_get_column(view, NUC_SEQUENCE_COLUMN); associated_nuc_column = obi_view_get_column(view, NUC_SEQUENCE_COLUMN);
if (obi_view_add_column(view, QUALITY_COLUMN, -1, NULL, OBI_QUAL, 0, 1, NULL, false, NULL, (associated_nuc_column->header)->name, (associated_nuc_column->header)->version, "Sequence qualities", true) < 0) // TODO discuss automatic association if (obi_view_add_column(view, QUALITY_COLUMN, -1, NULL, OBI_QUAL, 0, 1, NULL, false, false, false, NULL, (associated_nuc_column->header)->name, (associated_nuc_column->header)->version, "Sequence qualities", true) < 0) // TODO discuss automatic association
{ {
obidebug(1, "Error adding an obligatory column in a nucleotide sequences view"); obidebug(1, "Error adding an obligatory column in a nucleotide sequences view");
return NULL; return NULL;
@ -2215,7 +2217,9 @@ int obi_view_add_column(Obiview_p view,
index_t nb_lines, index_t nb_lines,
index_t nb_elements_per_line, index_t nb_elements_per_line,
char* elements_names, char* elements_names,
bool elt_names_formatted,
bool tuples, bool tuples,
bool to_eval,
const char* indexer_name, const char* indexer_name,
const char* associated_column_name, const char* associated_column_name,
obiversion_t associated_column_version, obiversion_t associated_column_version,
@ -2294,7 +2298,7 @@ int obi_view_add_column(Obiview_p view,
// Open or create the column // Open or create the column
if (create) if (create)
{ // Create column { // Create column
column = obi_create_column(view->dms, column_name, data_type, nb_lines, nb_elements_per_line, elements_names, false, tuples, indexer_name, associated_column_name, associated_column_version, comments); column = obi_create_column(view->dms, column_name, data_type, nb_lines, nb_elements_per_line, elements_names, elt_names_formatted, tuples, to_eval, indexer_name, associated_column_name, associated_column_version, comments);
if (column == NULL) if (column == NULL)
{ {
obidebug(1, "\nError creating a column to add to a view"); obidebug(1, "\nError creating a column to add to a view");
@ -2801,7 +2805,7 @@ int obi_create_auto_count_column(Obiview_p view)
return -1; return -1;
} }
if (obi_view_add_column(view, COUNT_COLUMN, -1, NULL, OBI_INT, 0, 1, NULL, false, NULL, NULL, -1, "Sequence counts", true) < 0) if (obi_view_add_column(view, COUNT_COLUMN, -1, NULL, OBI_INT, 0, 1, NULL, false, false, false, NULL, NULL, -1, "Sequence counts", true) < 0)
{ {
obidebug(1, "Error adding an automatic count column in a view"); obidebug(1, "Error adding an automatic count column in a view");
return -1; return -1;
@ -2853,7 +2857,7 @@ int obi_create_auto_id_column(Obiview_p view, const char* prefix)
} }
// Create the new ID column // Create the new ID column
if (obi_view_add_column(view, ID_COLUMN, -1, NULL, OBI_STR, 0, 1, NULL, false, NULL, NULL, -1, "Sequence identifiers", true) < 0) if (obi_view_add_column(view, ID_COLUMN, -1, NULL, OBI_STR, 0, 1, NULL, false, false, false, NULL, NULL, -1, "Sequence identifiers", true) < 0)
{ {
obidebug(1, "Error adding an automatic ID column in a view"); obidebug(1, "Error adding an automatic ID column in a view");
return -1; return -1;

5
src/obiview.h
View File

@ -361,7 +361,10 @@ Obiview_p obi_open_view(OBIDMS_p dms, const char* view_name);
* @param nb_elements_per_line The number of elements per line, if the column is created. * @param nb_elements_per_line The number of elements per line, if the column is created.
* @param elements_names The names of the elements with ';' as separator (no terminal ';'), * @param elements_names The names of the elements with ';' as separator (no terminal ';'),
* if the column is created; NULL or "" if the default names are to be used ("0\01\02\0...\0n"). * if the column is created; NULL or "" if the default names are to be used ("0\01\02\0...\0n").
* @param elt_names_formatted Whether the separator for the elements names is ';' (false), or '\0' (true, as formatted by format_elements_names()).
* @param tuples A boolean indicating whether the column should contain indices referring to indexed tuples. * @param tuples A boolean indicating whether the column should contain indices referring to indexed tuples.
* @param to_eval A boolean indicating whether the column contains expressions that should be evaluated
* (typically OBI_STR columns containing character strings to be evaluated by Python).
* @param indexer_name The name of the indexer if there is one associated with the column, if the column is created. * @param indexer_name The name of the indexer if there is one associated with the column, if the column is created.
* If NULL or "", the indexer name is set as the column name. * If NULL or "", the indexer name is set as the column name.
* @param associated_column_name The name of the associated column if there is one (otherwise NULL or ""), if the column is created. * @param associated_column_name The name of the associated column if there is one (otherwise NULL or ""), if the column is created.
@ -384,7 +387,9 @@ int obi_view_add_column(Obiview_p view,
index_t nb_lines, index_t nb_lines,
index_t nb_elements_per_line, index_t nb_elements_per_line,
char* elements_names, char* elements_names,
bool elt_names_formatted,
bool tuples, bool tuples,
bool to_eval,
const char* indexer_name, const char* indexer_name,
const char* associated_column_name, const char* associated_column_name,
obiversion_t associated_column_version, obiversion_t associated_column_version,