diff --git a/doc/.gitignore b/doc/.gitignore index ccd0596..f4cc153 100644 --- a/doc/.gitignore +++ b/doc/.gitignore @@ -1,3 +1,5 @@ /build/ /doxygen/ /build_dir.txt +/.DS_Store +/.gitignore diff --git a/doc/conf.py b/doc/conf.py index d717aa0..ecb4f68 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -51,7 +51,7 @@ source_suffix = '.rst' #source_encoding = 'utf-8-sig' # The master toctree document. -master_doc = 'index' +master_doc = 'source/index' # General information about the project. project = u'OBITools3' diff --git a/doc/doxygen/xml/.gitignore b/doc/doxygen/xml/.gitignore deleted file mode 100644 index 86d0cb2..0000000 --- a/doc/doxygen/xml/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore \ No newline at end of file diff --git a/doc/source/types.rst b/doc/source/types.rst index 150b669..d50f60d 100644 --- a/doc/source/types.rst +++ b/doc/source/types.rst @@ -7,11 +7,14 @@ OBITypes :download:`html version of the OBITypes UML file ` +.. image:: ./UML/Obicolumn_classes_UML.png + +:download:`html version of the OBIDMS classes UML file ` + + .. toctree:: :maxdepth: 2 The elementary types The containers Special values - - diff --git a/python/obitools3/obidms/_obidms.cfiles b/python/obitools3/obidms/_obidms.cfiles index 90dff1e..aa46c53 100644 --- a/python/obitools3/obidms/_obidms.cfiles +++ b/python/obitools3/obidms/_obidms.cfiles @@ -12,3 +12,5 @@ ../../../src/obitypes.c ../../../src/private_at_functions.h ../../../src/private_at_functions.c +../../../src/obiarray.h +../../../src/obiarray.c diff --git a/python/obitools3/obidms/_obidms.pxd b/python/obitools3/obidms/_obidms.pxd index 52c977b..4699fe5 100644 --- a/python/obitools3/obidms/_obidms.pxd +++ b/python/obitools3/obidms/_obidms.pxd @@ -23,7 +23,8 @@ cdef class OBIDMS: OBIType_t data_type=*, size_t nb_lines=*, size_t nb_elements_per_line=*, - list elements_names=*) + list elements_names=*, + str array_name=*) cdef class OBIDMS_column: diff --git a/python/obitools3/obidms/_obidms.pyx b/python/obitools3/obidms/_obidms.pyx index dbf82c6..36ef4f9 100644 --- a/python/obitools3/obidms/_obidms.pyx +++ b/python/obitools3/obidms/_obidms.pyx @@ -42,10 +42,10 @@ from ._obidmscolumn_char cimport OBIDMS_column_char, \ OBIDMS_column_char_multi_elts, \ OBIDMS_column_char_multi_elts_writable -# from ._obidmscolumn_idx cimport OBIDMS_column_idx, \ -# OBIDMS_column_idx_writable, \ -# OBIDMS_column_idx_multi_elts, \ -# OBIDMS_column_idx_multi_elts_writable +from ._obidmscolumn_str cimport OBIDMS_column_str, \ + OBIDMS_column_str_writable, \ + OBIDMS_column_str_multi_elts, \ + OBIDMS_column_str_multi_elts_writable cdef class OBIDMS : @@ -119,7 +119,8 @@ cdef class OBIDMS : OBIType_t data_type= 0, size_t nb_lines=0, size_t nb_elements_per_line=0, - list elements_names=None): + list elements_names=None, + str array_name="default_obiarray"): # Declarations cdef OBIDMS_column column @@ -202,17 +203,17 @@ cdef class OBIDMS : subclass = OBIDMS_column_char else : subclass = OBIDMS_column_char_multi_elts -# elif data_type == 5 : -# if (create or clone) : -# if nb_elements_per_line == 1 : -# subclass = OBIDMS_column_idx_writable -# else : -# subclass = OBIDMS_column_idx_multi_elts_writable -# else : -# if nb_elements_per_line == 1 : -# subclass = OBIDMS_column_idx -# else : -# subclass = OBIDMS_column_idx_multi_elts + elif data_type == 5 : + if (create or clone) : + if nb_elements_per_line == 1 : + subclass = OBIDMS_column_str_writable + else : + subclass = OBIDMS_column_str_multi_elts_writable + else : + if nb_elements_per_line == 1 : + subclass = OBIDMS_column_str + else : + subclass = OBIDMS_column_str_multi_elts else : raise Exception("Problem with the data type") @@ -220,7 +221,7 @@ cdef class OBIDMS : create, clone, clone_data, version_number, data_type, nb_lines, nb_elements_per_line, - elements_names) + elements_names, array_name) return column @@ -238,11 +239,13 @@ cdef class OBIDMS_column : OBIType_t type, size_t nb_lines, size_t nb_elements_per_line, - list elements_names): + list elements_names, + str array_name): # Declarations cdef bytes column_name_b cdef bytes dms_name_b + cdef bytes array_name_b cdef bytes elements_names_b # Fill structure @@ -255,6 +258,7 @@ cdef class OBIDMS_column : # Format the character strings to send them to C functions column_name_b = str2bytes(column_name) dms_name_b = str2bytes(self.dms.dms_name) + array_name_b = str2bytes(array_name) # Create, clone or open column if create : @@ -262,7 +266,9 @@ cdef class OBIDMS_column : elements_names_b = column_name_b else : elements_names_b = str2bytes(";".join(elements_names)) - self.pointer = obi_create_column(self.dms.pointer, column_name_b, type, nb_lines, nb_elements_per_line, elements_names_b) + self.pointer = obi_create_column(self.dms.pointer, column_name_b, type, + nb_lines, nb_elements_per_line, + elements_names_b, array_name_b) else : if clone : self.pointer = obi_clone_column(self.dms.pointer, column_name_b, version_number, clone_data) diff --git a/python/obitools3/obidms/_obidmscolumn_bool.cfiles b/python/obitools3/obidms/_obidmscolumn_bool.cfiles index fb8a24a..04bda7a 100644 --- a/python/obitools3/obidms/_obidmscolumn_bool.cfiles +++ b/python/obitools3/obidms/_obidmscolumn_bool.cfiles @@ -14,3 +14,5 @@ ../../../src/obitypes.c ../../../src/private_at_functions.h ../../../src/private_at_functions.c +../../../src/obiarray.h +../../../src/obiarray.c diff --git a/python/obitools3/obidms/_obidmscolumn_char.cfiles b/python/obitools3/obidms/_obidmscolumn_char.cfiles index 94eb20a..dc88b30 100644 --- a/python/obitools3/obidms/_obidmscolumn_char.cfiles +++ b/python/obitools3/obidms/_obidmscolumn_char.cfiles @@ -14,3 +14,5 @@ ../../../src/obitypes.c ../../../src/private_at_functions.h ../../../src/private_at_functions.c +../../../src/obiarray.h +../../../src/obiarray.c diff --git a/python/obitools3/obidms/_obidmscolumn_float.cfiles b/python/obitools3/obidms/_obidmscolumn_float.cfiles index 4cf1ea3..7a6df19 100644 --- a/python/obitools3/obidms/_obidmscolumn_float.cfiles +++ b/python/obitools3/obidms/_obidmscolumn_float.cfiles @@ -14,3 +14,5 @@ ../../../src/obitypes.c ../../../src/private_at_functions.h ../../../src/private_at_functions.c +../../../src/obiarray.h +../../../src/obiarray.c diff --git a/python/obitools3/obidms/_obidmscolumn_int.cfiles b/python/obitools3/obidms/_obidmscolumn_int.cfiles index cc10725..ab45f82 100644 --- a/python/obitools3/obidms/_obidmscolumn_int.cfiles +++ b/python/obitools3/obidms/_obidmscolumn_int.cfiles @@ -14,3 +14,5 @@ ../../../src/obitypes.c ../../../src/private_at_functions.h ../../../src/private_at_functions.c +../../../src/obiarray.h +../../../src/obiarray.c diff --git a/python/obitools3/obidms/_obidmscolumn_idx.cfiles b/python/obitools3/obidms/_obidmscolumn_str.cfiles similarity index 77% rename from python/obitools3/obidms/_obidmscolumn_idx.cfiles rename to python/obitools3/obidms/_obidmscolumn_str.cfiles index 5dfff68..7875117 100644 --- a/python/obitools3/obidms/_obidmscolumn_idx.cfiles +++ b/python/obitools3/obidms/_obidmscolumn_str.cfiles @@ -1,5 +1,5 @@ -../../../src/obidmscolumn_idx.c -../../../src/obidmscolumn_idx.h +../../../src/obidmscolumn_str.c +../../../src/obidmscolumn_str.h ../../../src/obidmscolumn.h ../../../src/obidmscolumn.c ../../../src/obidmscolumndir.h @@ -14,3 +14,5 @@ ../../../src/obitypes.c ../../../src/private_at_functions.h ../../../src/private_at_functions.c +../../../src/obiarray.h +../../../src/obiarray.c diff --git a/python/obitools3/obidms/_obidmscolumn_idx.pxd b/python/obitools3/obidms/_obidmscolumn_str.pxd similarity index 54% rename from python/obitools3/obidms/_obidmscolumn_idx.pxd rename to python/obitools3/obidms/_obidmscolumn_str.pxd index da49e46..b3c2488 100644 --- a/python/obitools3/obidms/_obidmscolumn_idx.pxd +++ b/python/obitools3/obidms/_obidmscolumn_str.pxd @@ -1,25 +1,24 @@ #cython: language_level=3 -from .capi.obitypes cimport obiidx_t from ._obidms cimport OBIDMS_column -cdef class OBIDMS_column_idx(OBIDMS_column): +cdef class OBIDMS_column_str(OBIDMS_column): cpdef object get_line(self, size_t line_nb) cpdef set_line(self, size_t line_nb, object value) cpdef close(self) -cdef class OBIDMS_column_idx_writable(OBIDMS_column_idx): +cdef class OBIDMS_column_str_writable(OBIDMS_column_str): cpdef set_line(self, size_t line_nb, object value) cpdef close(self) -cdef class OBIDMS_column_idx_multi_elts(OBIDMS_column_idx): +cdef class OBIDMS_column_str_multi_elts(OBIDMS_column_str): cpdef object get_item(self, size_t line_nb, str element_name) cpdef object get_line(self, size_t line_nb) - cpdef set_item(self, size_t line_nb, str element_name, obiidx_t value) + cpdef set_item(self, size_t line_nb, str element_name, str value) cpdef set_line(self, size_t line_nb, object values) -cdef class OBIDMS_column_idx_multi_elts_writable(OBIDMS_column_idx_multi_elts): - cpdef set_item(self, size_t line_nb, str element_name, obiidx_t value) +cdef class OBIDMS_column_str_multi_elts_writable(OBIDMS_column_str_multi_elts): + cpdef set_item(self, size_t line_nb, str element_name, str value) cpdef set_line(self, size_t line_nb, object values) cpdef close(self) diff --git a/python/obitools3/obidms/_obidmscolumn_idx.pyx b/python/obitools3/obidms/_obidmscolumn_str.pyx similarity index 56% rename from python/obitools3/obidms/_obidmscolumn_idx.pyx rename to python/obitools3/obidms/_obidmscolumn_str.pyx index fa59c01..1a114f2 100644 --- a/python/obitools3/obidms/_obidmscolumn_idx.pyx +++ b/python/obitools3/obidms/_obidmscolumn_str.pyx @@ -2,30 +2,46 @@ from .capi.obidmscolumn cimport obi_close_column,\ obi_truncate_and_close_column, \ - obi_column_get_obiidx_with_elt_name, \ - obi_column_get_obiidx_with_elt_idx, \ - obi_column_set_obiidx_with_elt_name, \ - obi_column_set_obiidx_with_elt_idx + obi_column_get_obistr_with_elt_name, \ + obi_column_get_obistr_with_elt_idx, \ + obi_column_set_obistr_with_elt_name, \ + obi_column_set_obistr_with_elt_idx from .capi.obierrno cimport obi_errno -from .capi.obitypes cimport OBIIdx_NA +from .capi.obitypes cimport OBIIdx_NA, const_char_p -from obitools3.utils cimport str2bytes + +from libc.string cimport strlen + +from obitools3.utils cimport str2bytes, bytes2str from cpython.int cimport PyInt_FromSsize_t -cdef class OBIDMS_column_idx(OBIDMS_column): +cdef class OBIDMS_column_str(OBIDMS_column): cpdef object get_line(self, size_t line_nb): - cdef obiidx_t value + cdef const_char_p cvalue + cdef bytes value cdef object result - value = obi_column_get_obiidx_with_elt_idx(self.pointer, line_nb, 0) + + cvalue = obi_column_get_obistr_with_elt_idx(self.pointer, line_nb, 0) + + print('test 1') + + print(hex( cvalue)) + + + print(strlen(cvalue)) + print("test") + value = cvalue + print(value) if obi_errno > 0 : raise IndexError(line_nb) if value == OBIIdx_NA : result = None else : - result = PyInt_FromSsize_t(value) + print(value) + result = bytes2str(value) return result cpdef set_line(self, size_t line_nb, object value): @@ -36,10 +52,10 @@ cdef class OBIDMS_column_idx(OBIDMS_column): raise Exception("Problem closing a column") -cdef class OBIDMS_column_idx_writable(OBIDMS_column_idx): +cdef class OBIDMS_column_str_writable(OBIDMS_column_str): cpdef set_line(self, size_t line_nb, object value): - if obi_column_set_obiidx_with_elt_idx(self.pointer, line_nb, 0, value) < 0: + if obi_column_set_obistr_with_elt_idx(self.pointer, line_nb, 0, str2bytes(value)) < 0: raise Exception("Problem setting a value in a column") cpdef close(self): @@ -47,55 +63,55 @@ cdef class OBIDMS_column_idx_writable(OBIDMS_column_idx): raise Exception("Problem closing a column") -cdef class OBIDMS_column_idx_multi_elts(OBIDMS_column_idx): +cdef class OBIDMS_column_str_multi_elts(OBIDMS_column_str): cpdef object get_item(self, size_t line_nb, str element_name): - cdef obiidx_t value + cdef bytes value cdef object result - value = obi_column_get_obiidx_with_elt_name(self.pointer, line_nb, str2bytes(element_name)) + value = obi_column_get_obistr_with_elt_name(self.pointer, line_nb, str2bytes(element_name)) if obi_errno > 0 : raise IndexError(line_nb, element_name) if value == OBIIdx_NA : result = None else : - result = PyInt_FromSsize_t(value) + result = bytes2str(value) return result cpdef object get_line(self, size_t line_nb) : - cdef obiidx_t value + cdef bytes value cdef object result cdef size_t i cdef bint all_NA result = {} all_NA = True for i in range(self.nb_elements_per_line) : - value = obi_column_get_obiidx_with_elt_idx(self.pointer, line_nb, i) + value = obi_column_get_obistr_with_elt_idx(self.pointer, line_nb, i) if obi_errno > 0 : raise IndexError(line_nb) - result[self.elements_names[i]] = PyInt_FromSsize_t(value) + result[self.elements_names[i]] = bytes2str(value) if all_NA and (value != OBIIdx_NA) : all_NA = False if all_NA : result = None return result - cpdef set_item(self, size_t line_nb, str element_name, obiidx_t value): + cpdef set_item(self, size_t line_nb, str element_name, str value): raise Exception("Column is read-only") cpdef set_line(self, size_t line_nb, object values): raise Exception("Column is read-only") -cdef class OBIDMS_column_idx_multi_elts_writable(OBIDMS_column_idx_multi_elts): +cdef class OBIDMS_column_str_multi_elts_writable(OBIDMS_column_str_multi_elts): - cpdef set_item(self, size_t line_nb, str element_name, obiidx_t value): - if obi_column_set_obiidx_with_elt_name(self.pointer, line_nb, str2bytes(element_name), value) < 0: + cpdef set_item(self, size_t line_nb, str element_name, str value): + if obi_column_set_obistr_with_elt_name(self.pointer, line_nb, str2bytes(element_name), str2bytes(value)) < 0: raise Exception("Problem setting a value in a column") cpdef set_line(self, size_t line_nb, object values): - cdef obiidx_t value + cdef str value for element_name in values : - value = values[element_name] + value = values[element_name] self.set_item(line_nb, element_name, value) cpdef close(self): diff --git a/python/obitools3/obidms/capi/obidmscolumn.pxd b/python/obitools3/obidms/capi/obidmscolumn.pxd index 29a2f79..b153b28 100644 --- a/python/obitools3/obidms/capi/obidmscolumn.pxd +++ b/python/obitools3/obidms/capi/obidmscolumn.pxd @@ -8,7 +8,6 @@ from ..capi.obitypes cimport const_char_p, \ obibool_t, \ obichar_t, \ obifloat_t, \ - obiidx_t, \ time_t @@ -26,6 +25,7 @@ cdef extern from "obidmscolumn.h" nogil: obiversion_t version obiversion_t cloned_from const_char_p name + const_char_p array_name const_char_p comments ctypedef OBIDMS_column_header_t* OBIDMS_column_header_p @@ -40,7 +40,8 @@ cdef extern from "obidmscolumn.h" nogil: OBIType_t type, size_t nb_lines, size_t nb_elements_per_line, - const_char_p elements_names) + const_char_p elements_names, + const_char_p array_name) OBIDMS_column_p obi_open_column(OBIDMS_p dms, const_char_p column_name, @@ -146,23 +147,23 @@ cdef extern from "obidmscolumn_float.h" nogil: size_t line_nb, size_t element_idx) -cdef extern from "obidmscolumn_idx.h" nogil: +cdef extern from "obidmscolumn_str.h" nogil: - int obi_column_set_obiidx_with_elt_name(OBIDMS_column_p column, + int obi_column_set_obistr_with_elt_name(OBIDMS_column_p column, size_t line_nb, const_char_p element_name, - obiidx_t value) + char* value) - int obi_column_set_obiidx_with_elt_idx(OBIDMS_column_p column, + int obi_column_set_obistr_with_elt_idx(OBIDMS_column_p column, size_t line_nb, size_t element_idx, - obiidx_t value) + char* value) - obiidx_t obi_column_get_obiidx_with_elt_name(OBIDMS_column_p column, - size_t line_nb, - const_char_p element_name) + const_char_p obi_column_get_obistr_with_elt_name(OBIDMS_column_p column, + size_t line_nb, + const_char_p element_name) - obiidx_t obi_column_get_obiidx_with_elt_idx(OBIDMS_column_p column, - size_t line_nb, - size_t element_idx) + const_char_p obi_column_get_obistr_with_elt_idx(OBIDMS_column_p column, + size_t line_nb, + size_t element_idx) diff --git a/src/obiarray.c b/src/obiarray.c new file mode 100644 index 0000000..f5d5ce4 --- /dev/null +++ b/src/obiarray.c @@ -0,0 +1,1173 @@ +/**************************************************************************** + * OBIDMS array functions * + ****************************************************************************/ + +/** + * @file obiarray.c + * @author Celine Mercier + * @date October 26th 2015 + * @brief Functions handling arrays for storing and retrieving bit arrays. + */ + + +#include +#include +#include +#include +#include +#include +#include + +#include "obiarray.h" +#include "obierrno.h" +#include "obidebug.h" + + +#define DEBUG_LEVEL 0 // TODO has to be defined somewhere else (cython compil flag?) + + +/************************************************************************** + * + * D E C L A R A T I O N O F T H E P R I V A T E F U N C T I O N S + * + **************************************************************************/ + +/** + * @brief Internal function building the file name for an array file. + * + * @warning The returned pointer has to be freed by the caller. + * + * @param name The name of the array. + * + * @returns A pointer to the array file name. + * @retval NULL if an error occurred. + * + * @since October 2015 + * @author Celine Mercier (celine.mercier@metabarcoding.org) + */ +static char* build_array_file_name(const char* array_name); + + +/** + * @brief Internal function building the file name for an array data file. + * + * @warning The returned pointer has to be freed by the caller. + * + * @param name The name of the array. + * + * @returns A pointer to the array data file name. + * @retval NULL if an error occurred. + * + * @since October 2015 + * @author Celine Mercier (celine.mercier@metabarcoding.org) + */ +static char* build_array_data_file_name(const char* array_name); + + +/** + * PRIVATE + */ +static char* build_array_file_path(OBIDMS_p dms, const char* array_name); + + +/** + * PRIVATE + */ +size_t get_array_header_size(); + + +/** + * PRIVATE + */ +size_t get_initial_array_size(); + + +/** + * PRIVATE + */ +size_t get_array_data_header_size(); + + +/** + * PRIVATE + */ +size_t get_initial_array_data_size(); + + +/** + * PRIVATE + */ +int close_array_data(OBIDMS_array_data_p array_data); + + +/** + * PRIVATE + */ +int grow_array(OBIDMS_array_p array); + + +/** + * PRIVATE + */ +int grow_array_data(OBIDMS_array_p array); + + +/** + * PRIVATE + */ +int array_compare(byte_t* value_1, byte_t* value_2); + +/** + * PRIVATE + */ +int32_t array_sizeof(byte_t* value); // TODO not sure about type + + +/************************************************************************ + * + * D E F I N I T I O N O F T H E P R I V A T E F U N C T I O N S + * + ************************************************************************/ + +static char* build_array_file_name(const char* array_name) +{ + char* file_name; + + // Build the file name + if (asprintf(&file_name,"%s.oda", array_name) < 0) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError building an array file name"); + return NULL; + } + + // Test if the array name is not too long + if (strlen(file_name) >= ARRAY_MAX_NAME) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError due to array name too long"); + free(file_name); + return NULL; + } + + return file_name; +} + + +static char* build_array_data_file_name(const char* array_name) +{ + char* file_name; + + // Build the file name + if (asprintf(&file_name,"%s.odd", array_name) < 0) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError building an array data file name"); + return NULL; + } + + return file_name; +} + + +static char* build_array_file_path(OBIDMS_p dms, const char* array_name) // TODO store DIR in dms structure +{ + char* array_file_path; + char* array_file_name; + + // Build the array file name + array_file_name = build_array_file_name(array_name); + if (array_file_name == NULL) + return NULL; + + // TODO use get_full_path(int directory_file_descriptor, const char* path_name) + array_file_path = (char*) malloc((strlen(dms->directory_name) + strlen("/arrays/") + strlen(array_file_name) + 1) * sizeof(char)); + if (array_file_path == NULL) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError building an array file path"); + return NULL; + } + + strcpy(array_file_path, dms->directory_name); + strcat(array_file_path, "/arrays/"); + strcat(array_file_path, array_file_name); + + return array_file_path; +} + + +size_t get_array_header_size() +{ + return getpagesize() * 1; +} + + +size_t get_initial_array_size() +{ + return getpagesize() * 1; +} + + +size_t get_array_data_header_size() +{ + return getpagesize() * 1; +} + + +size_t get_initial_array_data_size() +{ + return getpagesize() * 1; +} + + +int close_array_data(OBIDMS_array_data_p array_data) +{ + int ret_val = 0; + + if (munmap(array_data->header, (array_data->header)->header_size) < 0) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError munmapping the header of an array data file"); + ret_val = -1; + } + + if (munmap(array_data->data, (array_data->header)->data_size_max) < 0) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError munmapping the data of an array data file"); + ret_val = -1; + } + + free(array_data); + + return ret_val; +} + + +int grow_array(OBIDMS_array_p array) // TODO Lock when needed +{ + size_t file_size; + size_t old_data_size; + size_t new_data_size; + size_t header_size; + int array_dir_file_descriptor; + int array_file_descriptor; + char* array_file_name; + + // Get the file descriptor associated to the array directory + array_dir_file_descriptor = dirfd(array->directory); + if (array_dir_file_descriptor < 0) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError getting the file descriptor for an array directory"); + return -1; + } + + // Get the array file name + array_file_name = build_array_file_name((array->header)->array_name); + if (array_file_name == NULL) + { + close(array_dir_file_descriptor); + return -1; + } + + // Open the array data file + array_file_descriptor = openat(array_dir_file_descriptor, array_file_name, O_RDWR); + if (array_file_descriptor < 0) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError opening an array file"); + close(array_dir_file_descriptor); + free(array_file_name); + return -1; + } + free(array_file_name); + + // Calculate the new file size + old_data_size = ((array->header)->nb_items_max) * sizeof(index_t); + new_data_size = old_data_size * ARRAY_GROWTH_FACTOR; + header_size = (array->header)->header_size; + file_size = header_size + new_data_size; + + // Enlarge the file // TODO isn't it possible that this makes the file "move"? + if (ftruncate(array_file_descriptor, file_size) < 0) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError enlarging an array file"); + close(array_dir_file_descriptor); + close(array_file_descriptor); + return -1; + } + + // Unmap and re-map the data + + if (munmap(array->first, old_data_size) < 0) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError munmapping the array of an array file before enlarging"); + close(array_dir_file_descriptor); + close(array_file_descriptor); + return -1; + } + + array->first = mmap(NULL, + new_data_size, + PROT_READ | PROT_WRITE, + MAP_SHARED, + array_file_descriptor, + header_size + ); + + if (array->first == MAP_FAILED) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError re-mmapping the array of an array file after enlarging the file"); + close(array_dir_file_descriptor); + close(array_file_descriptor); + return -1; + } + + // Set new maximum number of items + (array->header)->nb_items_max = ceil(((double) new_data_size) / ((double) sizeof(index_t))); + + // Initialize new data to 0 + memset((array->first)+old_data_size, 0, new_data_size - old_data_size); + + close(array_file_descriptor); + + return 0; +} + + +int grow_array_data(OBIDMS_array_p array) // TODO Lock when needed +{ + size_t file_size; + size_t old_data_size; + size_t new_data_size; + size_t header_size; + int array_dir_file_descriptor; + int array_data_file_descriptor; + char* array_data_file_name; + + // Get the file descriptor associated to the array directory + array_dir_file_descriptor = dirfd(array->directory); + if (array_dir_file_descriptor < 0) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError getting the file descriptor for an array directory"); + return -1; + } + + // Get the array data file name + array_data_file_name = build_array_data_file_name((array->header)->array_name); + if (array_data_file_name == NULL) + { + close(array_dir_file_descriptor); + return -1; + } + + // Open the array data file + array_data_file_descriptor = openat(array_dir_file_descriptor, array_data_file_name, O_RDWR); + if (array_data_file_descriptor < 0) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError opening an array data file"); + close(array_dir_file_descriptor); + free(array_data_file_name); + return -1; + } + free(array_data_file_name); + + // Calculate the new file size + old_data_size = ((array->data)->header)->data_size_max; + new_data_size = old_data_size * ARRAY_GROWTH_FACTOR; + header_size = ((array->data)->header)->header_size; + file_size = header_size + new_data_size; + + // Enlarge the file // TODO isn't it possible that this makes the file "move"? + if (ftruncate(array_data_file_descriptor, file_size) < 0) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError enlarging an array data file"); + close(array_dir_file_descriptor); + close(array_data_file_descriptor); + return -1; + } + + // Unmap and re-map the data + + if (munmap((array->data)->data, old_data_size) < 0) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError munmapping the data of an array data file before enlarging"); + close(array_dir_file_descriptor); + close(array_data_file_descriptor); + return -1; + } + + (array->data)->data = mmap(NULL, + new_data_size, + PROT_READ | PROT_WRITE, + MAP_SHARED, + array_data_file_descriptor, + header_size + ); + + if ((array->data)->data == MAP_FAILED) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError re-mmapping the data of an array data file after enlarging the file"); + close(array_dir_file_descriptor); + close(array_data_file_descriptor); + return -1; + } + + // Set new data size + ((array->data)->header)->data_size_max = new_data_size; + + // Initialize new data to 0 + memset(((array->data)->data)+old_data_size, 0, new_data_size - old_data_size); + + close(array_data_file_descriptor); + + return 0; +} + + +int array_compare(byte_t* value_1, byte_t* value_2) +{ + int comp; + uint8_t size_1; + uint8_t size_2; + int32_t len_1; + int32_t len_2; + int32_t b; + + obidebug(1, "\nCOMPARING 1=%d,%.*s; 2=%d,%.*s", (int32_t) *(value_1+1), (int32_t) *(value_1+1), value_1+5, (int32_t) *(value_2+1), (int32_t) *(value_2+1), value_2+5); + + size_1 = (uint8_t) *(value_1); + size_2 = (uint8_t) *(value_2); + + if (size_1 != size_2) + return (size_1 - size_2); + + len_1 = (int32_t) *(value_1+1); + len_2 = (int32_t) *(value_2+1); + + obidebug(1, "\ncomp length = %d - %d = %d", len_1, len_2, (len_1 - len_2)); + + if (len_1 != len_2) + return (len_1 - len_2); + + b = 0; + comp = 0; + while (!comp && (b < len_1)) + { + comp = *(value_1+b) - *(value_2+b); + b++; + } + + obidebug(1, "\ncomp = %d", comp); + + return comp; +} + + +int32_t array_sizeof(byte_t* value) +{ + return (6 + (int32_t) *(value+1)); +} + + +/********************************************************************** + * + * D E F I N I T I O N O F T H E P U B L I C F U N C T I O N S + * + **********************************************************************/ + +int obi_array_exists(OBIDMS_p dms, const char* array_name) +{ + struct stat buffer; + char* array_file_path; + int check_dir; + + // Build the array file path + array_file_path = build_array_file_path(dms, array_name); + + check_dir = stat(array_file_path, &buffer); + + free(array_file_path); + + if (check_dir == 0) + return 1; + else + return 0; +} + + +OBIDMS_array_p obi_array(OBIDMS_p dms, const char* array_name) +{ + int exists; + + exists = obi_array_exists(dms, array_name); + + switch (exists) + { + case 0: + return obi_create_array(dms, array_name); + case 1: + return obi_open_array(dms, array_name); + }; + + obidebug(1, "\nError checking if an array already exists"); + return NULL; +} + + +OBIDMS_array_p obi_create_array(OBIDMS_p dms, const char* array_name) +{ + char* array_file_name; + char* array_data_file_name; + size_t header_size; + size_t data_size; + size_t file_size; + int array_file_descriptor; + int array_data_file_descriptor; + int array_dir_file_descriptor; + OBIDMS_array_data_p array_data; + OBIDMS_array_p array; + + // Create the data file + + // Get the file descriptor of the array directory + array_dir_file_descriptor = dirfd(dms->array_directory); + if (array_dir_file_descriptor < 0) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError getting the file descriptor for an array directory"); + return NULL; + } + + obidebug(1, "\nboom"); + + // Build file name + array_data_file_name = build_array_data_file_name(array_name); + if (array_data_file_name == NULL) + { + close(array_dir_file_descriptor); + return NULL; + } + + obidebug(1, "\nboom"); + + // Create file + array_data_file_descriptor = openat(array_dir_file_descriptor, array_data_file_name, O_RDWR | O_CREAT | O_EXCL, 0777); + if (array_data_file_descriptor < 0) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError creating an array data file"); + close(array_dir_file_descriptor); + free(array_data_file_name); + return NULL; + } + free(array_data_file_name); + + // Calculate the size needed + header_size = get_array_data_header_size(); + data_size = get_initial_array_data_size(); + file_size = header_size + data_size; + + // Truncate the array data file to the right size + if (ftruncate(array_data_file_descriptor, file_size) < 0) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError truncating an array data file to the right size"); + close(array_dir_file_descriptor); + close(array_data_file_descriptor); + return NULL; + } + + // Allocate the memory for the array data structure + array_data = (OBIDMS_array_data_p) malloc(sizeof(OBIDMS_array_data_t)); + if (array_data == NULL) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError allocating the memory for the array data structure"); + close(array_dir_file_descriptor); + close(array_data_file_descriptor); + return NULL; + } + + // Fill the array data structure + array_data->header = mmap(NULL, + header_size, + PROT_READ | PROT_WRITE, + MAP_SHARED, + array_data_file_descriptor, + 0 + ); + if (array_data->header == MAP_FAILED) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError mmapping the header of an array data file"); + close(array_dir_file_descriptor); + close(array_data_file_descriptor); + free(array_data); + return NULL; + } + + array_data->data = mmap(NULL, + data_size, + PROT_READ | PROT_WRITE, + MAP_SHARED, + array_data_file_descriptor, + header_size + ); + if (array_data->data == MAP_FAILED) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError mmapping the data of an array data file"); + munmap(array_data->header, header_size); + close(array_dir_file_descriptor); + close(array_data_file_descriptor); + free(array_data); + return NULL; + } + + (array_data->header)->header_size = header_size; + (array_data->header)->data_size_max = data_size; + (array_data->header)->data_size_used = 0; + (array_data->header)->nb_items = 0; + (array_data->header)->creation_date = time(NULL); + strcpy((array_data->header)->array_name, array_name); + + // Initialize all bits to 0 + memset(array_data->data, 0, (array_data->header)->data_size_max); + + close(array_data_file_descriptor); + + + // Create the array file + + // Build file name + array_file_name = build_array_file_name(array_name); + if (array_file_name == NULL) + { + close(array_dir_file_descriptor); + close_array_data(array_data); + return NULL; + } + + // Calculate the size needed + header_size = get_array_header_size(); + data_size = get_initial_array_size(); + file_size = header_size + data_size; + + // Create file + array_file_descriptor = openat(array_dir_file_descriptor, array_file_name, O_RDWR | O_CREAT | O_EXCL, 0777); + if (array_file_descriptor < 0) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError creating an array file"); + close(array_dir_file_descriptor); + close_array_data(array_data); + free(array_file_name); + return NULL; + } + free(array_file_name); + + // Truncate the array file to the right size + if (ftruncate(array_file_descriptor, file_size) < 0) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError truncating an array file to the right size"); + close(array_dir_file_descriptor); + close_array_data(array_data); + close(array_file_descriptor); + return NULL; + } + + // Allocate the memory for the array structure + array = (OBIDMS_array_p) malloc(sizeof(OBIDMS_array_t)); + if (array == NULL) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError allocating the memory for the array structure"); + close(array_dir_file_descriptor); + close_array_data(array_data); + close(array_file_descriptor); + return NULL; + } + + // Fill the array structure + array->header = mmap(NULL, + header_size, + PROT_READ | PROT_WRITE, + MAP_SHARED, + array_file_descriptor, + 0 + ); + if (array->header == MAP_FAILED) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError mmapping the header of an array file"); + close(array_dir_file_descriptor); + close_array_data(array_data); + close(array_file_descriptor); + free(array); + return NULL; + } + + array->first = mmap(NULL, + data_size, + PROT_READ | PROT_WRITE, + MAP_SHARED, + array_file_descriptor, + header_size + ); + if (array->first == MAP_FAILED) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError mmapping the data of an array file"); + close(array_dir_file_descriptor); + close_array_data(array_data); + munmap(array->header, header_size); + close(array_file_descriptor); + free(array); + return NULL; + } + + array->data = array_data; + array->directory = dms->array_directory; + + (array->header)->header_size = header_size; + (array->header)->nb_items = 0; + (array->header)->nb_items_max = (index_t) ceil(((double) get_initial_array_size()) / ((double) sizeof(index_t))); + (array->header)->creation_date = time(NULL); + strcpy((array->header)->array_name, array_name); + + // Initialize to 0 (TODO: unnecessary?) + memset(array->first, 0, data_size); + + close(array_file_descriptor); + //close(array_dir_file_descriptor); TODO + + return array; +} + + +OBIDMS_array_p obi_open_array(OBIDMS_p dms, const char* array_name) +{ + char* array_file_name; + char* array_data_file_name; + size_t header_size; + int array_file_descriptor; + int array_data_file_descriptor; + int array_dir_file_descriptor; + OBIDMS_array_data_p array_data; + OBIDMS_array_p array; + + // Open the data file + + // Get the file descriptor of the array directory + array_dir_file_descriptor = dirfd(dms->array_directory); + if (array_dir_file_descriptor < 0) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError getting the file descriptor for an array directory"); + return NULL; + } + + // Build file name + array_data_file_name = build_array_data_file_name(array_name); + if (array_data_file_name == NULL) + { + close(array_dir_file_descriptor); + return NULL; + } + + // Open file + array_data_file_descriptor = openat(array_dir_file_descriptor, array_data_file_name, O_RDWR, 0777); + if (array_data_file_descriptor < 0) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError opening an array data file"); + close(array_dir_file_descriptor); + free(array_data_file_name); + return NULL; + } + free(array_data_file_name); + + // Allocate the memory for the array data structure + array_data = (OBIDMS_array_data_p) malloc(sizeof(OBIDMS_array_data_t)); + if (array_data == NULL) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError allocating the memory for the array data structure"); + close(array_dir_file_descriptor); + close(array_data_file_descriptor); + return NULL; + } + + // Fill the array data structure + header_size = get_array_data_header_size(); // TODO should be read? + array_data->header = mmap(NULL, + header_size, + PROT_READ | PROT_WRITE, + MAP_SHARED, + array_data_file_descriptor, + 0 + ); + if (array_data->header == MAP_FAILED) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError mmapping the header of an array data file"); + close(array_dir_file_descriptor); + close(array_data_file_descriptor); + free(array_data); + return NULL; + } + + array_data->data = mmap(NULL, + (array_data->header)->data_size_max, + PROT_READ | PROT_WRITE, + MAP_SHARED, + array_data_file_descriptor, + header_size + ); + if (array_data->data == MAP_FAILED) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError mmapping the data of an array data file"); + close(array_dir_file_descriptor); + munmap(array_data->header, header_size); + close(array_data_file_descriptor); + free(array_data); + return NULL; + } + + close(array_data_file_descriptor); + + + // Open the array file + + // Build file name + array_file_name = build_array_file_name(array_name); + if (array_file_name == NULL) + { + close(array_dir_file_descriptor); + close_array_data(array_data); + return NULL; + } + + // Open file + array_file_descriptor = openat(array_dir_file_descriptor, array_file_name, O_RDWR, 0777); + if (array_file_descriptor < 0) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError opening an array file"); + close(array_dir_file_descriptor); + close_array_data(array_data); + free(array_file_name); + return NULL; + } + free(array_file_name); + + // Allocate the memory for the array structure + array = (OBIDMS_array_p) malloc(sizeof(OBIDMS_array_t)); + if (array == NULL) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError allocating the memory for the array structure"); + close(array_dir_file_descriptor); + close_array_data(array_data); + close(array_file_descriptor); + return NULL; + } + + // Fill the array structure + header_size = get_array_header_size(); // TODO should be read? + array->header = mmap(NULL, + header_size, + PROT_READ | PROT_WRITE, + MAP_SHARED, + array_file_descriptor, + 0 + ); + if (array->header == MAP_FAILED) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError mmapping the header of an array file"); + close(array_dir_file_descriptor); + close_array_data(array_data); + close(array_file_descriptor); + free(array); + return NULL; + } + + array->first = mmap(NULL, + (((array->header)->nb_items_max) * sizeof(index_t)), + PROT_READ | PROT_WRITE, + MAP_SHARED, + array_file_descriptor, + header_size + ); + if (array->first == MAP_FAILED) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError mmapping the data of an array file"); + close(array_dir_file_descriptor); + close_array_data(array_data); + munmap(array->header, header_size); + close(array_file_descriptor); + free(array); + return NULL; + } + + array->data = array_data; + array->directory = dms->array_directory; + + close(array_file_descriptor); + //close(array_dir_file_descriptor); TODO + + return array; +} + + +int obi_close_array(OBIDMS_array_p array) +{ + int ret_val = 0; + + ret_val = close_array_data(array->data); + + if (munmap(array->header, (array->header)->header_size) < 0) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError munmapping the header of an array file"); + ret_val = -1; + } + + if (munmap(array->first, (((array->header)->nb_items_max) * sizeof(index_t))) < 0) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError munmapping the array of an array file"); + ret_val = -1; + } + + free(array); + + return ret_val; +} + + +//int obi_delete_array(const char* array_name) +//{ +// +//} + + +index_t obi_array_add(OBIDMS_array_p array, byte_t* value) +{ + index_t idx; + index_t nb_items; + index_t nb_items_max; + index_t* a; + byte_t* data; + int64_t data_size_used; + int64_t* data_size_max_p; + int32_t value_size; + + obidebug(1, "\n"); + + nb_items = (array->header)->nb_items; + nb_items_max = (array->header)->nb_items_max; + a = array->first; + data = (array->data)->data; + data_size_used = ((array->data)->header)->data_size_used; + data_size_max_p = &(((array->data)->header)->data_size_max); + + // Check if the value is already in the array + if (nb_items > 0) + { + idx = obi_array_search(array, value); + if (idx >= 0) // The value is already in the array + return idx; + else + idx = -(idx+1); + } + else // First item + idx = 0; + + obidebug(1, "\n"); + + // Grow the array if needed + if (nb_items == nb_items_max) + { + if (grow_array(array) < 0) + return -1; + } + + obidebug(1, "\n"); + + // Grow the data if needed + value_size = array_sizeof(value); + while (*data_size_max_p < (data_size_used + value_size)) + { + if (grow_array_data(array) < 0) + return -1; + } + + obidebug(1, "\nidx = %d", idx); + + // Add the index of the value in the array + memmove((a+idx+1), (a+idx), ((nb_items - idx) * sizeof(index_t))); + a[idx] = data_size_used; + + int i; + for (i=0; idata)->header)->data_size_used = data_size_used + value_size; + + // Update the number of items + ((array->header)->nb_items)++; + (((array->data)->header)->nb_items)++; + + //obidebug(1, "\nSHOULD ++ : %d", (array->header)->nb_items); + + for (i=0; i<(array->header)->nb_items; i++) + fprintf(stderr, "\narray[%d] = %d\n", i, a[i]); + + fprintf(stderr, "\nreturning %lld\n", data_size_used); + + return data_size_used; +} + + +byte_t* obi_array_get(OBIDMS_array_p array, index_t idx) // TODO copy? +{ + return (((array->data)->data)+idx); +} + + +index_t obi_array_search(OBIDMS_array_p array, byte_t* value) +{ + index_t idx_min; + index_t idx_max; + index_t idx_mid; + index_t idx; + index_t* a; + byte_t* data; + byte_t* to_compare; + index_t nb_items; + int comp; + + nb_items = (array->header)->nb_items; + data = (array->data)->data; + a = array->first; + + obidebug(1, "\nnb items = %d", nb_items); + + if (nb_items == 0) + return 0; + + idx_max = nb_items; + idx_min = 0; + + obidebug(1, "\n"); + + while (idx_min < idx_max) + { + idx_mid = nb_items/2; + + obidebug(1, "\ncomparing with idx = %d, data idx %d", idx_mid, a[idx_mid]); + //fprintf(stderr, "\ndata+14=%.5s\n", data+14); + + to_compare = obi_array_get(array, a[idx_mid]); + + //fprintf(stderr, "\nto_compare=%.5s\n", to_compare+5); + + comp = array_compare(to_compare, value); + + obidebug(1, "\nCOMP = %d", comp); + + if (!comp) + return a[idx_mid]; + else if (comp < 0) + { + idx = idx_mid + 1; + // Search in upper half + idx_min = idx_mid + 1; + } + else + idx = idx_mid; + // Search in lower half + idx_max = idx_mid - 1; + } + + obidebug(1, "\nNOT FOUND, idx=%d", idx); + + // Not found: return where the value should be + return -(idx+1); +} + + +byte_t* obi_str_to_obibytes(char* value) +{ + byte_t* value_b; + int32_t length; + uint8_t size; + + size = 8; + length = strlen(value); + value_b = (byte_t*) malloc(length + 6); + if (value_b == NULL) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError allocating memory for a byte array"); + return NULL; + } + + *(value_b) = size; + *(value_b+1) = length; + + strcpy(value_b+5, value); + + // TODO 5 as macro + + return value_b; +} + + +const char* obi_obibytes_to_str(byte_t* value_b) +{ + const char* value; + + value = value_b+5; + + obidebug(1, "\nvalue = %s", value); + + // TODO 5 as macro + + return value; +} + + + + + + diff --git a/src/obiarray.h b/src/obiarray.h new file mode 100644 index 0000000..4b8154f --- /dev/null +++ b/src/obiarray.h @@ -0,0 +1,177 @@ +/**************************************************************************** + * OBIDMS array header file * + ****************************************************************************/ + +/** + * @file array.h + * @author Celine Mercier + * @date October 19th 2015 + * @brief Header file for handling arrays for storing and retrieving data arrays (i.e. character strings). + */ + + +#ifndef ARRAY_H_ +#define ARRAY_H_ + + +#include +#include +#include +#include +#include +#include + +#include "obidms.h" +#include "obitypes.h" + + +#define ARRAY_MAX_NAME (2048) /**< The maximum length of an array name. + */ +#define ARRAY_GROWTH_FACTOR (2) /**< The growth factor when an array is enlarged. + */ + +typedef int32_t index_t; /**< Type of the array indices. + */ +typedef char byte_t; /**< Defining byte type since data is stored in bits + * and char (stored on one byte) is the smallest addressable unit. + */ + + +/** + * @brief OBIDMS array data header structure. + */ +typedef struct OBIDMS_array_data_header { + int header_size; /**< Size of the header in bytes. + */ + int64_t data_size_used; /**< Size of the data used in bytes. // TODO not sure about type + */ + int64_t data_size_max; /**< Max size of the data in bytes. // TODO not sure about type + */ + index_t nb_items; /**< Number of items. + */ + char array_name[ARRAY_MAX_NAME+1]; /**< The array name as a NULL terminated string. + */ + time_t creation_date; /**< Date of creation of the file. + */ +} OBIDMS_array_data_header_t, *OBIDMS_array_data_header_p; + + +/** + * @brief OBIDMS array data structure. + */ +typedef struct OBIDMS_array_data { + OBIDMS_array_data_header_p header; /**< A pointer to the header of the array data. + */ + byte_t* data; /**< A pointer to the beginning of the data. + */ +} OBIDMS_array_data_t, *OBIDMS_array_data_p; + + +/** + * @brief OBIDMS array header structure. + */ +typedef struct OBIDMS_array_header { + int header_size; /**< Size of the header in bytes. + */ + index_t nb_items; /**< Number of items in the array. + */ + index_t nb_items_max; /**< Maximum number of items in the array before it has to be enlarged. + */ + char array_name[ARRAY_MAX_NAME+1]; /**< The array name as a NULL terminated string. + */ + time_t creation_date; /**< Date of creation of the file. + */ +} OBIDMS_array_header_t, *OBIDMS_array_header_p; + + +/** + * @brief OBIDMS array structure. + */ +typedef struct OBIDMS_array { + OBIDMS_array_header_p header; /**< A pointer to the header of the array. + */ + index_t* first; /**< A pointer to the beginning of the array itself. + */ + OBIDMS_array_data_p data; /**< A pointer to the structure containing the data + * that the array references. + */ + DIR* directory; /**< A directory entry usable to + * refer and scan the array directory. + */ +} OBIDMS_array_t, *OBIDMS_array_p; + + +/** + * + */ +int obi_array_exists(OBIDMS_p dms, const char* array_name); + + +/** + * + */ +OBIDMS_array_p obi_array(OBIDMS_p dms, const char* array_name); + + +/** + * + */ +OBIDMS_array_p obi_create_array(OBIDMS_p dms, const char* array_name); + + +/** + * + */ +OBIDMS_array_p obi_open_array(OBIDMS_p dms, const char* array_name); + + +/** + * + */ +int obi_close_array(OBIDMS_array_p array); + + +/** + * + */ +//int obi_delete_array(OBIDMS_p dms, const char* array_name); + + +/** + * The header is already in the byte array + */ +index_t obi_array_add(OBIDMS_array_p array, byte_t* value); + + +/** + * Returns the entire array including the header + */ +byte_t* obi_array_get(OBIDMS_array_p array, index_t index); + + +/** + * + */ +index_t obi_array_search(OBIDMS_array_p array, byte_t* value); + + +/** + * + */ +byte_t* obi_str_to_obibytes(char* value); + + +/** + * + */ +const char* obi_obibytes_to_str(byte_t* value_b); + + +/** + * TODO LATER + */ +// OBIDMS_array_p obi_column_arrayify(FILE, encoding); + + +#endif /* ARRAY_H_ */ + diff --git a/src/obidms.c b/src/obidms.c index 06ff2c4..c0a1ced 100644 --- a/src/obidms.c +++ b/src/obidms.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -21,6 +22,7 @@ #include "obierrno.h" #include "obidebug.h" #include "obidmscolumn.h" +#include "private_at_functions.h" #define DEBUG_LEVEL 0 // TODO has to be defined somewhere else (cython compil flag?) @@ -113,7 +115,9 @@ int obi_dms_exists(const char* dms_name) OBIDMS_p obi_create_dms(const char* dms_name) { - char *directory_name; + char* directory_name; + DIR* dms_dir; + int dms_file_descriptor; // Build and check the directory name directory_name = build_directory_name(dms_name); @@ -132,6 +136,35 @@ OBIDMS_p obi_create_dms(const char* dms_name) return NULL; } + // Get file descriptor of DMS directory to create the arrays directory + dms_dir = opendir(directory_name); + if (dms_dir == NULL) + { + obi_set_errno(OBIDMS_UNKNOWN_ERROR); + obidebug(1, "\nProblem opening a newly created OBIDMS directory"); + free(directory_name); + return NULL; + } + dms_file_descriptor = dirfd(dms_dir); + if (dms_file_descriptor < 0) + { + obi_set_errno(OBIDMS_UNKNOWN_ERROR); + obidebug(1, "\nProblem getting the file descriptor of a newly created OBIDMS directory"); + free(directory_name); + return NULL; + } + + // Create the arrays directory + if (mkdirat(dms_file_descriptor, ARRAY_DIR_NAME, 00777) < 0) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nProblem creating an arrays directory"); + free(directory_name); + return NULL; + } + + // TODO close file descriptor? + free(directory_name); return obi_open_dms(dms_name); @@ -143,6 +176,7 @@ OBIDMS_p obi_open_dms(const char* dms_name) OBIDMS_p dms; char* directory_name; DIR* directory; + int dms_file_descriptor; dms = NULL; @@ -191,8 +225,30 @@ OBIDMS_p obi_open_dms(const char* dms_name) strcpy(dms->directory_name, directory_name); dms->directory = directory; + // Get file descriptor of DMS directory to open the arrays directory + dms_file_descriptor = dirfd(directory); + if (dms_file_descriptor < 0) + { + obi_set_errno(OBIDMS_UNKNOWN_ERROR); + obidebug(1, "\nError getting the file descriptor for a newly created OBIDMS directory"); + free(directory_name); + return NULL; + } + + // Open the arrays directory + dms->array_directory = private_opendirat(dms_file_descriptor, ARRAY_DIR_NAME); + if (dms->array_directory == NULL) + { + obi_set_errno(OBIDMS_UNKNOWN_ERROR); + obidebug(1, "\nError opening the arrays directory"); + free(directory_name); + return NULL; + } + free(directory_name); + // TODO test if close file descriptor + return dms; } @@ -223,7 +279,14 @@ int obi_close_dms(OBIDMS_p dms) if (closedir(dms->directory) < 0) { obi_set_errno(OBIDMS_MEMORY_ERROR); - obidebug(1, "\nError closing an OBIDSM directory"); + obidebug(1, "\nError closing an OBIDMS directory"); + free(dms); + return -1; + } + if (closedir(dms->array_directory) < 0) + { + obi_set_errno(OBI_ARRAY_ERROR); + obidebug(1, "\nError closing an array directory"); free(dms); return -1; } diff --git a/src/obidms.h b/src/obidms.h index 4065006..cded60b 100644 --- a/src/obidms.h +++ b/src/obidms.h @@ -24,8 +24,10 @@ #include "obierrno.h" -#define OBIDMS_MAX_NAME (2048) /**< The maximum length of an OBIDMS name - */ +#define OBIDMS_MAX_NAME (2048) /**< The maximum length of an OBIDMS name. + */ +#define ARRAY_DIR_NAME "arrays" /**< The name of the arrays directory. + */ /** @@ -41,6 +43,9 @@ typedef struct OBIDMS { DIR* directory; /**< A directory entry usable to * refer and scan the database directory. */ + DIR* array_directory; /**< A directory entry usable to + * refer and scan the array directory. + */ } OBIDMS_t, *OBIDMS_p; diff --git a/src/obidmscolumn.c b/src/obidmscolumn.c index ac91b23..889a3dc 100644 --- a/src/obidmscolumn.c +++ b/src/obidmscolumn.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -28,6 +29,7 @@ #include "obierrno.h" #include "obidebug.h" #include "obilittlebigman.h" +#include "obiarray.h" #define DEBUG_LEVEL 0 // TODO has to be defined somewhere else (cython compil flag?) @@ -151,35 +153,35 @@ static size_t get_line_count_per_page(OBIType_t data_type, size_t nb_elements_pe ************************************************************************/ -static char *build_column_file_name(const char *column_name, obiversion_t version_number) +static char* build_column_file_name(const char* column_name, obiversion_t version_number) { - char *filename; + char* file_name; - // Build the database directory name - if (asprintf(&filename,"%s@%d.odc", column_name, version_number) < 0) + // Build the file name + if (asprintf(&file_name,"%s@%d.odc", column_name, version_number) < 0) { obi_set_errno(OBICOL_MEMORY_ERROR); obidebug(1, "\nError building a column file name"); return NULL; } - return filename; + return file_name; } -static char *build_version_file_name(const char *column_name) +static char* build_version_file_name(const char* column_name) { - char *filename; + char* file_name; - // Build the database directory name - if (asprintf(&filename,"%s.odv", column_name) < 0) + // Build the file name + if (asprintf(&file_name,"%s.odv", column_name) < 0) { obi_set_errno(OBICOL_MEMORY_ERROR); obidebug(1, "\nError building a version file name"); return NULL; } - return filename; + return file_name; } @@ -602,11 +604,13 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms, OBIType_t data_type, size_t nb_lines, size_t nb_elements_per_line, - const char* elements_names) + const char* elements_names, + const char* array_name) { OBIDMS_column_p new_column; OBIDMS_column_directory_p column_directory; OBIDMS_column_header_p header; + OBIDMS_array_p array; size_t file_size; obiversion_t version_number; char* column_file_name; @@ -629,17 +633,23 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms, obidebug(1, "\nCan't create column because of empty column name"); return NULL; } - if ((data_type < 1) || (data_type > 4)) + if ((data_type < 1) || (data_type > 5)) { obidebug(1, "\nCan't create column because of invalid data type"); return NULL; } + if ((data_type == 5) && (array_name == NULL)) + { + obidebug(1, "\nCan't create column because of empty array name"); + return NULL; + } + // The initial line count should be between the minimum (corresponding to the page size) and the maximum allowed minimum_line_count = get_line_count_per_page(data_type, nb_elements_per_line); if (nb_lines > MAXIMUM_LINE_COUNT) { - obidebug(1, "\nCan't create column because of line count greater than the maximum allowed (%ld)", MAXIMUM_LINE_COUNT); + obidebug(1, "\nCan't create column because of line count greater than the maximum allowed (%d)", MAXIMUM_LINE_COUNT); return NULL; } else if (nb_lines < minimum_line_count) @@ -800,6 +810,23 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms, strncpy(header->name, column_name, OBIDMS_COLUMN_MAX_NAME); + // If the data type is OBI_IDX, the associated obi_array is opened or created + if (data_type == 5) + { + array = obi_array(dms, array_name); + if (array == NULL) + { + obidebug(1, "\nError opening or creating the array associated with a column"); + munmap(new_column->header, header_size); + close(column_file_descriptor); + free(column_file_name); + free(new_column); + return NULL; + } + new_column->array = array; + strncpy(header->array_name, array_name, ARRAY_MAX_NAME); + } + // Fill the data with NA values obi_ini_to_NA_values(new_column, 0, nb_lines); @@ -816,6 +843,7 @@ OBIDMS_column_p obi_open_column(OBIDMS_p dms, { OBIDMS_column_p column; OBIDMS_column_directory_p column_directory; + OBIDMS_array_p array; char* column_file_name; int column_file_descriptor; int column_dir_file_descriptor; @@ -931,6 +959,22 @@ OBIDMS_column_p obi_open_column(OBIDMS_p dms, column->writable = false; + // If the data type is OBI_IDX, the associated obi_array is opened or created + if ((column->header)->data_type == 5) + { + array = obi_array(dms, (column->header)->array_name); + if (array == NULL) + { + obidebug(1, "\nError opening the array associated with a column"); + munmap(column->header, header_size); + close(column_file_descriptor); + free(column_file_name); + free(column); + return NULL; + } + column->array = array; + } + free(column_file_name); close(column_file_descriptor); @@ -972,7 +1016,8 @@ OBIDMS_column_p obi_clone_column(OBIDMS_p dms, data_type, nb_lines, nb_elements_per_line, - (column_to_clone->header)->elements_names); + (column_to_clone->header)->elements_names, + (column_to_clone->header)->array_name); if (new_column == NULL) { @@ -1170,7 +1215,7 @@ int obi_enlarge_column(OBIDMS_column_p column) // Calculate the new file size old_line_count = (column->header)->line_count; - new_line_count = old_line_count * GROWTH_FACTOR; + new_line_count = old_line_count * COLUMN_GROWTH_FACTOR; if (new_line_count > MAXIMUM_LINE_COUNT) { @@ -1181,7 +1226,7 @@ int obi_enlarge_column(OBIDMS_column_p column) return -1; } old_data_size = obi_array_sizeof((column->header)->data_type, old_line_count, (column->header)->nb_elements_per_line); - new_data_size = old_data_size * GROWTH_FACTOR; + new_data_size = old_data_size * COLUMN_GROWTH_FACTOR; header_size = (column->header)->header_size; file_size = header_size + new_data_size; diff --git a/src/obidmscolumn.h b/src/obidmscolumn.h index 102c4b5..8e7ea84 100644 --- a/src/obidmscolumn.h +++ b/src/obidmscolumn.h @@ -25,19 +25,21 @@ #include "obierrno.h" #include "obilittlebigman.h" #include "obidmscolumndir.h" +#include "obiarray.h" #define ONE_IF_ZERO(x) (((x)==0)?1:(x)) /**< If x is equal to 0, x takes the value 1. */ #define ELEMENTS_NAMES_MAX (2048) /**< The maximum length of the list of elements names. */ -#define GROWTH_FACTOR (2) /**< The growth factor when a column is enlarged. +#define COLUMN_GROWTH_FACTOR (2) /**< The growth factor when a column is enlarged. */ #define MAXIMUM_LINE_COUNT (1000000) /**< The maximum line count for the data of a column. //TODO */ #define FORMATTED_TIME_LENGTH (1024) /**< The length allocated for the character string containing a formatted date */ + typedef int32_t obiversion_t; /**< Used to store the column version number */ @@ -74,6 +76,8 @@ typedef struct OBIDMS_column_header { */ char name[OBIDMS_COLUMN_MAX_NAME+1]; /**< The column name as a NULL terminated string. */ + char array_name[ARRAY_MAX_NAME+1]; /**< If there is one, the obi_array name as a NULL terminated string. + */ char comments[1]; /**< Comments stored as a classical zero end C string. * The size of the comment is only limited by the header size. */ @@ -93,6 +97,8 @@ typedef struct OBIDMS_column { */ OBIDMS_column_header_p header; /**< A pointer to the header of the column. */ + OBIDMS_array_p array; /**< A pointer to the array associated with the column if there is one. + */ void* data; /**< A `void` pointer to the beginning of the data. * * @warning Never use this member directly outside of the code of the @@ -172,11 +178,12 @@ size_t obi_get_platform_header_size(); * @author Eric Coissac (eric.coissac@metabarcoding.org) */ OBIDMS_column_p obi_create_column(OBIDMS_p dms, - const char* column_name, + const char* column_name, OBIType_t data_type, size_t nb_lines, size_t nb_elements_per_line, - const char* elements_names); + const char* elements_names, + const char* array_name); /** diff --git a/src/obidmscolumn_idx.c b/src/obidmscolumn_str.c similarity index 50% rename from src/obidmscolumn_idx.c rename to src/obidmscolumn_str.c index fa8484e..1fa4c5a 100644 --- a/src/obidmscolumn_idx.c +++ b/src/obidmscolumn_str.c @@ -1,27 +1,15 @@ /**************************************************************************** - * OBIDMS_column_idx functions * + * OBIDMS_column_str functions * ****************************************************************************/ /** - * @file obidsmcolumn_idx.c + * @file obidsmcolumn_str.c * @author Celine Mercier - * @date August 10th 2015 + * @date October 28th 2015 * @brief Functions handling OBIColumns containing data with the OBIType OBI_IDX. */ -/**************************************************************************** - * OBIDMS_column_idx functions * - ****************************************************************************/ - -/** - * @file obidsmcolumn_idx.c - * @author Celine Mercier - * @date August 10th 2015 - * @brief Functions handling OBIColumns containing data with the OBIType OBI_BOOL. - */ - - #include #include @@ -29,6 +17,7 @@ #include "obitypes.h" #include "obierrno.h" #include "obidebug.h" +#include "obiarray.h" #define DEBUG_LEVEL 0 // TODO has to be defined somewhere else (cython compil flag?) @@ -40,8 +29,11 @@ * **********************************************************************/ -int obi_column_set_obiidx_with_elt_idx(OBIDMS_column_p column, size_t line_nb, size_t element_idx, obiidx_t value) +int obi_column_set_obistr_with_elt_idx(OBIDMS_column_p column, size_t line_nb, size_t element_idx, char* value) { + byte_t* value_b; + index_t idx; + // Check that the line number is not greater than the maximum allowed if (line_nb >= MAXIMUM_LINE_COUNT) { @@ -62,43 +54,84 @@ int obi_column_set_obiidx_with_elt_idx(OBIDMS_column_p column, size_t line_nb, s if ((line_nb+1) > (column->header)->lines_used) (column->header)->lines_used = line_nb+1; - // Set the value - *(((obiidx_t*) (column->data)) + (line_nb * ((column->header)->nb_elements_per_line)) + element_idx) = value; + // Encode the value on a byte array with a header + value_b = obi_str_to_obibytes(value); + if (value_b == NULL) + return -1; + + obidebug(1, "\nvalue=%s", value); + //obidebug(1, "\nbytes=%s", value_b+5); + + // Add in the obiarray + idx = obi_array_add(column->array, value_b); + if (idx == -1) + return -1; + + obidebug(1, "\nidx=%d", idx); + //obidebug(1, "\nbytes 2=%s", obi_array_get(column->array, idx)+5); + + // Add the value's index in the column + *(((obiidx_t*) (column->data)) + (line_nb * ((column->header)->nb_elements_per_line)) + element_idx) = idx; + + // TODO free value_b probably return 0; } -obiidx_t obi_column_get_obiidx_with_elt_idx(OBIDMS_column_p column, size_t line_nb, size_t element_idx) +const char* obi_column_get_obistr_with_elt_idx(OBIDMS_column_p column, size_t line_nb, size_t element_idx) { + index_t idx; + byte_t* value_b; + const char* value; + if ((line_nb+1) > (column->header)->lines_used) { obi_set_errno(OBICOL_UNKNOWN_ERROR); obidebug(1, "\nError trying to get a value that is beyond the current number of lines used"); - return OBIIdx_NA; + return "\0"; // TODO } - return *(((obiidx_t*) (column->data)) + (line_nb * ((column->header)->nb_elements_per_line)) + element_idx); + + idx = *(((obiidx_t*) (column->data)) + (line_nb * ((column->header)->nb_elements_per_line)) + element_idx); + + // Check NA + if (idx == OBIIdx_NA) + return "\0"; // TODO + + obidebug(1, "\nwhy, idx = %d", idx); + + value_b = obi_array_get(column->array, idx); + + obidebug(1, "\nwhyyyy"); + + value = obi_obibytes_to_str(value_b); + + obidebug(1, "\nwhyyyyyyyyyyyy, value=%s %p", value, value); + obidebug(1, "\nwhyyyyyyyyyyyy, len value=%d", strlen(value)); + + return value; } -int obi_column_set_obiidx_with_elt_name(OBIDMS_column_p column, size_t line_nb, const char* element_name, obiidx_t value) +int obi_column_set_obistr_with_elt_name(OBIDMS_column_p column, size_t line_nb, const char* element_name, char* value) { size_t element_idx; element_idx = obi_column_get_element_index_from_name(column, element_name); if (element_idx == SIZE_MAX) //TODO return -1; - obi_column_set_obiidx_with_elt_idx(column, line_nb, element_idx, value); + if (obi_column_set_obistr_with_elt_idx(column, line_nb, element_idx, value) < 0) + return -1; return 0; } -obiidx_t obi_column_get_obiidx_with_elt_name(OBIDMS_column_p column, size_t line_nb, const char* element_name) +const char* obi_column_get_obistr_with_elt_name(OBIDMS_column_p column, size_t line_nb, const char* element_name) { size_t element_idx; element_idx = obi_column_get_element_index_from_name(column, element_name); if (element_idx == SIZE_MAX) //TODO - return OBIIdx_NA; - return obi_column_get_obiidx_with_elt_idx(column, line_nb, element_idx); + return "\0"; + return obi_column_get_obistr_with_elt_idx(column, line_nb, element_idx); } diff --git a/src/obidmscolumn_idx.h b/src/obidmscolumn_str.h similarity index 91% rename from src/obidmscolumn_idx.h rename to src/obidmscolumn_str.h index b18f9b8..984a54a 100644 --- a/src/obidmscolumn_idx.h +++ b/src/obidmscolumn_str.h @@ -38,7 +38,7 @@ * @since July 2015 * @author Celine Mercier (celine.mercier@metabarcoding.org) */ -int obi_column_set_obiidx_with_elt_idx(OBIDMS_column_p column, size_t line_nb, size_t element_idx, obiidx_t value); +int obi_column_set_obiidx_with_elt_idx(OBIDMS_column_p column, size_t line_nb, size_t element_idx, void* value); /** @@ -54,7 +54,7 @@ int obi_column_set_obiidx_with_elt_idx(OBIDMS_column_p column, size_t line_nb, s * @since July 2015 * @author Celine Mercier (celine.mercier@metabarcoding.org) */ -obiidx_t obi_column_get_obiidx_with_elt_idx(OBIDMS_column_p column, size_t line_nb, size_t element_idx); +const char* obi_column_get_obiidx_with_elt_idx(OBIDMS_column_p column, size_t line_nb, size_t element_idx); /** @@ -76,7 +76,7 @@ obiidx_t obi_column_get_obiidx_with_elt_idx(OBIDMS_column_p column, size_t line_ * @since August 2015 * @author Celine Mercier (celine.mercier@metabarcoding.org) */ -int obi_column_set_obiidx_with_elt_name(OBIDMS_column_p column, size_t line_nb, const char* element_name, obiidx_t value); +int obi_column_set_obiidx_with_elt_name(OBIDMS_column_p column, size_t line_nb, const char* element_name, void* value); /** @@ -94,7 +94,7 @@ int obi_column_set_obiidx_with_elt_name(OBIDMS_column_p column, size_t line_nb, * @since August 2015 * @author Celine Mercier (celine.mercier@metabarcoding.org) */ -obiidx_t obi_column_get_obiidx_with_elt_name(OBIDMS_column_p column, size_t line_nb, const char* element_name); +const char* obi_column_get_obiidx_with_elt_name(OBIDMS_column_p column, size_t line_nb, const char* element_name); #endif /* OBIDMSCOLUMN_IDX_H_ */ diff --git a/src/obierrno.h b/src/obierrno.h index d0a5a95..1abcd12 100644 --- a/src/obierrno.h +++ b/src/obierrno.h @@ -98,6 +98,8 @@ extern int obi_errno; */ #define OBICOL_ACCESS_ERROR (19) /**< Permission error trying to access an OBIDSM column directory */ +#define OBI_ARRAY_ERROR (20) /** Error while handling a heap + */ /**@}*/ #endif /* OBIERRNO_H_ */ diff --git a/src/private_at_functions.c b/src/private_at_functions.c index e06bd58..debacab 100644 --- a/src/private_at_functions.c +++ b/src/private_at_functions.c @@ -43,7 +43,7 @@ char* get_full_path(int directory_file_descriptor, const char* path_name) return NULL; } - // TODO check errors + // TODO check errors? strlcat(full_path, "/", MAX_PATH_LEN); strlcat(full_path, path_name, MAX_PATH_LEN);