From 9a50803c001e5762b33a985c9e18c8adf12d8d60 Mon Sep 17 00:00:00 2001 From: Celine Mercier Date: Wed, 15 Nov 2017 13:48:59 +0100 Subject: [PATCH] Added tuple columns containing immutable indexed data arrays of any type --- python/obitools3/commands/test.pyx | 105 ++++++- python/obitools3/dms/capi/obidmscolumn.pxd | 4 + python/obitools3/dms/capi/obitypes.pxd | 1 + python/obitools3/dms/capi/obiview.pxd | 29 +- python/obitools3/dms/column/column.pxd | 3 +- python/obitools3/dms/column/column.pyx | 21 +- .../dms/column/typed_column/bool.pxd | 5 +- .../dms/column/typed_column/bool.pyx | 280 +++++------------- .../dms/column/typed_column/char.pxd | 6 + .../dms/column/typed_column/char.pyx | 82 ++++- .../dms/column/typed_column/float.pxd | 7 +- .../dms/column/typed_column/float.pyx | 78 ++++- .../obitools3/dms/column/typed_column/int.pxd | 7 +- .../obitools3/dms/column/typed_column/int.pyx | 79 ++++- .../dms/column/typed_column/qual.pyx | 9 +- .../obitools3/dms/column/typed_column/seq.pxd | 7 +- .../obitools3/dms/column/typed_column/seq.pyx | 89 +++++- .../obitools3/dms/column/typed_column/str.pxd | 7 +- .../obitools3/dms/column/typed_column/str.pyx | 88 +++++- python/obitools3/dms/dms.cfiles | 2 + python/obitools3/utils.cfiles | 2 + src/array_indexer.c | 73 +++++ src/array_indexer.h | 60 ++++ src/obi_align.c | 24 +- src/obidms.c | 3 +- src/obidmscolumn.c | 9 +- src/obidmscolumn.h | 6 +- src/obidmscolumn_array.c | 85 ++++++ src/obidmscolumn_array.h | 64 ++++ src/obitypes.h | 2 +- src/obiview.c | 58 +++- src/obiview.h | 86 ++++++ 32 files changed, 1097 insertions(+), 284 deletions(-) create mode 100644 src/array_indexer.c create mode 100644 src/array_indexer.h create mode 100644 src/obidmscolumn_array.c create mode 100644 src/obidmscolumn_array.h diff --git a/python/obitools3/commands/test.pyx b/python/obitools3/commands/test.pyx index 89d9122..c434dcc 100644 --- a/python/obitools3/commands/test.pyx +++ b/python/obitools3/commands/test.pyx @@ -1,11 +1,11 @@ #cython: language_level=3 from obitools3.apps.progress cimport ProgressBar # TODO I absolutely don't understand why it doesn't work without that line -from obitools3.dms.view.view import View, Line_selection +from obitools3.dms.view import View, Line_selection from obitools3.dms.view.typed_view.view_NUC_SEQS import View_NUC_SEQS -from obitools3.dms.dms import DMS +from obitools3.dms import DMS from obitools3.dms.column import Column -from obitools3.dms.taxo.taxo import Taxonomy +from obitools3.dms.taxo import Taxonomy from obitools3.utils cimport str2bytes from obitools3.dms.capi.obitypes cimport OBI_INT, \ OBI_FLOAT, \ @@ -69,26 +69,68 @@ def random_bool(config): return random.choice([True, False]) +def random_bool_tuples(config): + l=[] + for i in range(random.randint(1, config['test']['tuplemaxlen'])) : + l.append(random.choice([None, random_bool(config)])) + return tuple(l) + + def random_char(config): return str2bytes(random.choice(string.ascii_lowercase)) +def random_char_tuples(config): + l=[] + for i in range(random.randint(1, config['test']['tuplemaxlen'])) : + l.append(random.choice([None, random_char(config)])) + return tuple(l) + + def random_float(config): return random.randint(0, MAX_INT) + random.random() +def random_float_tuples(config): + l=[] + for i in range(random.randint(1, config['test']['tuplemaxlen'])) : + l.append(random.choice([None, random_float(config)])) + return tuple(l) + + def random_int(config): return random.randint(0, config['test']['maxlinenb']) +def random_int_tuples(config): + l=[] + for i in range(random.randint(1, config['test']['tuplemaxlen'])) : + l.append(random.choice([None, random_int(config)])) + return tuple(l) + + def random_seq(config): return str2bytes(''.join(random.choice(['a','t','g','c']) for i in range(random_length(config['test']['seqmaxlen'])))) +def random_seq_tuples(config): + l=[] + for i in range(random.randint(1, config['test']['tuplemaxlen'])) : + l.append(random.choice([None, random_seq(config)])) + return tuple(l) + + def random_bytes(config): return random_bytes_with_max_len(config['test']['strmaxlen']) +def random_bytes_tuples(config): + l=[] + for i in range(random.randint(1, config['test']['tuplemaxlen'])) : + l.append(random.choice([None, random_bytes(config)])) + return tuple(l) + + def random_str_with_max_len(max_len): return ''.join(random.choice(string.ascii_lowercase) for i in range(random_length(max_len))) @@ -132,11 +174,28 @@ def test_set_and_get(config, infos): print_test(config, "-") return idx = random_int(config) - value = random.choice([None, infos['random_generator'][data_type](config)]) + value = random.choice([None, infos['random_generator'][(data_type, col.tuples)](config)]) if col.nb_elements_per_line > 1 : elt = random.choice(element_names) col[idx][elt] = value assert col[idx][elt] == value, "Column: "+repr(col)+"\nSet value != gotten value "+str(value)+" != "+str(col[idx][elt]) + elif col.tuples: + col[idx] = value + if value is None: + totest = None + else: + totest = [] + for e in value: + if e is not None and e != '': + totest.append(e) + if len(totest) == 0: + totest = None + else: + totest = tuple(totest) + assert col[idx] == totest, "Column: "+repr(col)+"\nSet value != gotten value "+str(totest)+" != "+str(col[idx]) + if totest is not None: + for i in range(len(totest)) : + assert col[idx][i] == totest[i], "Column: "+repr(col)+"\nSet value[i] != gotten value[i] "+str(totest[i])+" != "+str(col[idx][i]) else: col[idx] = value assert col[idx] == value, "Column: "+repr(col)+"\nSet value != gotten value "+str(value)+" != "+str(col[idx]) @@ -210,19 +269,25 @@ def fill_column(config, infos, col) : if len(element_names) > 1 : for i in range(random_int(config)) : for j in range(len(element_names)) : - col[i][element_names[j]] = random.choice([None, infos['random_generator'][data_type](config)]) + col[i][element_names[j]] = random.choice([None, infos['random_generator'][(data_type, col.tuples)](config)]) else : for i in range(random_int(config)) : - col[i] = random.choice([None, infos['random_generator'][data_type](config)]) + r = random.choice([None, infos['random_generator'][(data_type, col.tuples)](config)]) + col[i] = r def create_random_column(config, infos) : alias = random.choice([b'', random_unique_name(infos)]) - nb_elements_per_line=random.randint(1, config['test']['maxelts']) - elements_names = [] - for i in range(nb_elements_per_line) : - elements_names.append(random_unique_element_name(config, infos)) - elements_names = random.choice([None, elements_names]) + tuples = random.choice([True, False]) + if not tuples : + nb_elements_per_line=random.randint(1, config['test']['maxelts']) + elements_names = [] + for i in range(nb_elements_per_line) : + elements_names.append(random_unique_element_name(config, infos)) + elements_names = random.choice([None, elements_names]) + else : + nb_elements_per_line = 1 + elements_names = None name = random_unique_name(infos) data_type = random_col_type() @@ -231,6 +296,7 @@ def create_random_column(config, infos) : data_type, nb_elements_per_line=nb_elements_per_line, elements_names=elements_names, + tuples=tuples, comments=random_str_with_max_len(COL_COMMENTS_MAX_LEN), alias=alias ) @@ -347,6 +413,14 @@ def addOptions(parser): type=int, help="Maximum length of character strings. " "Default: 200") + + group.add_argument('--tuple_max_len','-u', + action="store", dest="test:tuplemaxlen", + metavar='', + default=20, + type=int, + help="Maximum length of tuples. " + "Default: 200") group.add_argument('--comments_max_len','-c', action="store", dest="test:commentsmaxlen", @@ -402,7 +476,14 @@ def run(config): 'view': None, 'view_names': None, 'unique_names': [], - 'random_generator': {b"OBI_BOOL": random_bool, b"OBI_CHAR": random_char, b"OBI_FLOAT": random_float, b"OBI_INT": random_int, b"OBI_SEQ": random_seq, b"OBI_STR": random_bytes}, + 'random_generator': { + (b"OBI_BOOL", False): random_bool, (b"OBI_BOOL", True): random_bool_tuples, + (b"OBI_CHAR", False): random_char, (b"OBI_CHAR", True): random_char_tuples, + (b"OBI_FLOAT", False): random_float, (b"OBI_FLOAT", True): random_float_tuples, + (b"OBI_INT", False): random_int, (b"OBI_INT", True): random_int_tuples, + (b"OBI_SEQ", False): random_seq, (b"OBI_SEQ", True): random_seq_tuples, + (b"OBI_STR", False): random_bytes, (b"OBI_STR", True): random_bytes_tuples + }, 'tests': [test_set_and_get, test_add_col, test_delete_col, test_col_alias, test_new_view] } diff --git a/python/obitools3/dms/capi/obidmscolumn.pxd b/python/obitools3/dms/capi/obidmscolumn.pxd index 581ba2d..88bc563 100644 --- a/python/obitools3/dms/capi/obidmscolumn.pxd +++ b/python/obitools3/dms/capi/obidmscolumn.pxd @@ -31,6 +31,7 @@ cdef extern from "obidmscolumn.h" nogil: const_char_p elements_names OBIType_t returned_data_type OBIType_t stored_data_type + bint tuples time_t creation_date obiversion_t version obiversion_t cloned_from @@ -60,3 +61,6 @@ cdef extern from "obidmscolumn.h" nogil: int obi_close_header(OBIDMS_column_header_p header) char* obi_get_elements_names(OBIDMS_column_p column) + + index_t obi_column_get_element_index_from_name(OBIDMS_column_p column, const char* element_name) + diff --git a/python/obitools3/dms/capi/obitypes.pxd b/python/obitools3/dms/capi/obitypes.pxd index 1f194f8..7b403cb 100644 --- a/python/obitools3/dms/capi/obitypes.pxd +++ b/python/obitools3/dms/capi/obitypes.pxd @@ -52,6 +52,7 @@ cdef extern from "obitypes.h" nogil: extern const_char_p OBIStr_NA extern const_char_p OBIQual_char_NA extern uint8_t* OBIQual_int_NA + extern void* OBITuple_NA const_char_p name_data_type(int data_type) diff --git a/python/obitools3/dms/capi/obiview.pxd b/python/obitools3/dms/capi/obiview.pxd index 4a10c34..9decb64 100644 --- a/python/obitools3/dms/capi/obiview.pxd +++ b/python/obitools3/dms/capi/obiview.pxd @@ -14,7 +14,7 @@ from ..capi.obidmscolumn cimport OBIDMS_column_p, \ Column_reference_t, \ Column_reference_p -from libc.stdint cimport uint8_t +from libc.stdint cimport uint8_t, int32_t cdef extern from "obiview.h" nogil: @@ -86,6 +86,7 @@ cdef extern from "obiview.h" nogil: index_t nb_lines, index_t nb_elements_per_line, char* elements_names, + bint tuples, const_char_p indexer_name, const_char_p associated_column_name, obiversion_t associated_column_version, @@ -327,3 +328,29 @@ cdef extern from "obiview.h" nogil: OBIDMS_column_p column_p, index_t line_nb, index_t element_idx) + + # ARRAY + int obi_set_array_with_col_p_in_view(Obiview_p view, + OBIDMS_column_p column, + index_t line_nb, + const void* value, + uint8_t elt_size, + int32_t value_length) + + const void* obi_get_array_with_col_p_in_view(Obiview_p view, + OBIDMS_column_p column, + index_t line_nb, + int32_t* value_length_p) + + int obi_set_array_with_col_name_in_view(Obiview_p view, + const char* column_name, + index_t line_nb, + const void* value, + uint8_t elt_size, + int32_t value_length) + + const void* obi_get_array_with_col_name_in_view(Obiview_p view, + const char* column_name, + index_t line_nb, + int32_t* value_length_p) + diff --git a/python/obitools3/dms/column/column.pxd b/python/obitools3/dms/column/column.pxd index 81ef1f7..0e1e0b3 100644 --- a/python/obitools3/dms/column/column.pxd +++ b/python/obitools3/dms/column/column.pxd @@ -22,7 +22,7 @@ cdef class Column(OBIWrapper) : cdef inline OBIDMS_column_p pointer(self) @staticmethod - cdef type get_column_class(obitype_t obitype, bint multi_elts) + cdef type get_column_class(obitype_t obitype, bint multi_elts, bint tuples) @staticmethod cdef type get_python_type(obitype_t obitype, bint multi_elts) @@ -44,6 +44,7 @@ cdef class Column_line: cdef register_column_class(obitype_t obitype, bint multi_elts, + bint tuples, type obiclass, type python) diff --git a/python/obitools3/dms/column/column.pyx b/python/obitools3/dms/column/column.pyx index e59a6bc..62bd6df 100644 --- a/python/obitools3/dms/column/column.pyx +++ b/python/obitools3/dms/column/column.pyx @@ -1,6 +1,4 @@ #cython: language_level=3 - -from obitools3.dms.column import typed_column __OBIDMS_COLUMN_CLASS__ = {} @@ -44,12 +42,12 @@ cdef class Column(OBIWrapper) : @staticmethod - cdef type get_column_class(obitype_t obitype, bint multi_elts): + cdef type get_column_class(obitype_t obitype, bint multi_elts, bint tuples): ''' Internal function returning the python class representing a column for a given obitype. ''' - return __OBIDMS_COLUMN_CLASS__[(obitype, multi_elts)][0] + return __OBIDMS_COLUMN_CLASS__[(obitype, multi_elts, tuples)][0] @staticmethod @@ -76,6 +74,7 @@ cdef class Column(OBIWrapper) : obitype_t data_type, index_t nb_elements_per_line=1, list elements_names=None, + bint tuples=False, object comments=b"", object alias=b""): # TODO indexer_name? @@ -115,6 +114,7 @@ cdef class Column(OBIWrapper) : nb_lines = len(view), nb_elements_per_line = nb_elements_per_line, elements_names = elements_names_p, + tuples = tuples, indexer_name = NULL, associated_column_name = NULL, associated_column_version = -1, @@ -150,7 +150,7 @@ cdef class Column(OBIWrapper) : column_p = column_pp[0] column_type = column_p.header.returned_data_type - column_class = Column.get_column_class(column_type, (column_p.header.nb_elements_per_line > 1)) + column_class = Column.get_column_class(column_type, (column_p.header.nb_elements_per_line > 1), column_p.header.tuples) column = OBIWrapper.new_wrapper(column_class, column_pp) column._view = view @@ -184,6 +184,7 @@ cdef class Column(OBIWrapper) : nb_lines = -1, nb_elements_per_line = -1, elements_names = NULL, + tuples = False, indexer_name = NULL, associated_column_name = NULL, associated_column_version = -1, @@ -326,6 +327,13 @@ cdef class Column(OBIWrapper) : raise OBIDeactivatedInstanceError() return self.pointer().header.lines_used + # tuples property getter + @property + def tuples(self): + if not self.active() : + raise OBIDeactivatedInstanceError() + return self.pointer().header.tuples + # comments property getter @property def comments(self): @@ -436,6 +444,7 @@ cdef class Column_line : cdef register_column_class(obitype_t obitype, bint multi_elts, + bint tuples, type obiclass, type python_type): ''' @@ -446,7 +455,7 @@ cdef register_column_class(obitype_t obitype, assert issubclass(obiclass, Column) - __OBIDMS_COLUMN_CLASS__[(obitype, multi_elts)] = (obiclass, python_type) + __OBIDMS_COLUMN_CLASS__[(obitype, multi_elts, tuples)] = (obiclass, python_type) cdef register_all_column_classes() : diff --git a/python/obitools3/dms/column/typed_column/bool.pxd b/python/obitools3/dms/column/typed_column/bool.pxd index 741c639..41209c0 100644 --- a/python/obitools3/dms/column/typed_column/bool.pxd +++ b/python/obitools3/dms/column/typed_column/bool.pxd @@ -20,7 +20,10 @@ cdef class Column_multi_elts_bool(Column_multi_elts) : cpdef set_item(self, index_t line_nb, object elt_id, object value) - +cdef class Column_tuples_bool(Column): + + cpdef object get_line(self, index_t line_nb) + cpdef set_line(self, index_t line_nb, object value) diff --git a/python/obitools3/dms/column/typed_column/bool.pyx b/python/obitools3/dms/column/typed_column/bool.pyx index 08bc9d0..e309ee9 100644 --- a/python/obitools3/dms/column/typed_column/bool.pyx +++ b/python/obitools3/dms/column/typed_column/bool.pyx @@ -13,14 +13,23 @@ from ...capi.obiview cimport obi_get_bool_with_elt_name_and_col_p_in_view, \ obi_get_bool_with_elt_idx_and_col_p_in_view, \ obi_set_bool_with_elt_name_and_col_p_in_view, \ obi_set_bool_with_elt_idx_and_col_p_in_view, \ + obi_get_array_with_col_p_in_view, \ + obi_set_array_with_col_p_in_view, \ Obiview_p from ...capi.obidmscolumn cimport OBIDMS_column_p -from ...capi.obitypes cimport OBI_BOOL, OBIBool_NA, obibool_t +from ...capi.obitypes cimport OBI_BOOL, \ + OBIBool_NA, \ + OBITuple_NA, \ + obibool_t from cpython.bool cimport PyBool_FromLong +from libc.stdint cimport int32_t + +from libc.stdlib cimport malloc, free + cdef class Column_bool(Column): @@ -29,10 +38,12 @@ cdef class Column_bool(Column): object column_name, index_t nb_elements_per_line=1, object elements_names=None, + bint tuples=False, object comments=b""): return Column.new_column(view, column_name, OBI_BOOL, nb_elements_per_line=nb_elements_per_line, elements_names=elements_names, + tuples=tuples, comments=comments) cpdef object get_line(self, index_t line_nb): @@ -115,210 +126,67 @@ cdef class Column_multi_elts_bool(Column_multi_elts): obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=elt_id, error_message="Problem setting a value in a column") +cdef class Column_tuples_bool(Column): + + cpdef object get_line(self, index_t line_nb) : + global obi_errno + cdef obibool_t value + cdef bint value_in_result + cdef object result + cdef int32_t i + cdef obibool_t* array + cdef int32_t value_length + + result = [] + + array = obi_get_array_with_col_p_in_view(self._view.pointer(), self.pointer(), line_nb, &value_length) + obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem getting a value from a column") + + if array == OBITuple_NA : + return None + + for i in range(value_length) : + value = array[i] + value_in_result = PyBool_FromLong(value) + result.append(value_in_result) + + return tuple(result) + + + cpdef set_line(self, index_t line_nb, object value) : + global obi_errno + cdef obibool_t* array + cdef int32_t value_length + cdef int32_t i, j + cdef object e + + value_length = 0 + if value is not None: + for e in value: + if e is not None: + value_length+=1 + if value is None or value_length == 0 : + array = OBITuple_NA + else: + array = malloc(value_length * sizeof(obibool_t)) + if array == NULL: + raise Exception("Problem allocating memory for an array to store a tuple") + #raise RollbackException("Problem allocating memory for an array to store a tuple", self._view) # TODO can't import + j=0 + for i in range(len(value)) : + if value[i] is not None: + array[j] = (value[i]) + j+=1 + + if obi_set_array_with_col_p_in_view(self._view.pointer(), self.pointer(), line_nb, array, sizeof(obibool_t)*8, value_length) < 0 : + obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem setting a value in a column") + + if array != OBITuple_NA: + free(array) + + def register_class() : - register_column_class(OBI_BOOL, False, Column_bool, bool) - register_column_class(OBI_BOOL, True, Column_multi_elts_bool, bool) + register_column_class(OBI_BOOL, False, False, Column_bool, bool) + register_column_class(OBI_BOOL, True, False, Column_multi_elts_bool, bool) + register_column_class(OBI_BOOL, False, True, Column_tuples_bool, bool) - - - - - - - -# cdef class Column_line_bool(Column_line) : -# -# cdef update_pointer(self): -# """ -# Checks if the obicolumn address changed since the last call and update -# if need the `_column_p` and `_data_view` data structure fields. -# """ -# cdef OBIDMS_column_p* column_pp -# column_pp = self._pointer -# cdef OBIDMS_column_p column_p = column_pp[0] -# -# if column_p != self._column_p: -# self._column_p = column_p -# self._data_view = ( (column_p.data)) + \ -# self._index * column_p.header.nb_elements_per_line -# -# @staticmethod -# cdef bool obibool_t2bool(obibool_t value): -# cdef bool result -# -# if value == OBIBool_NA : -# result = None -# else : -# result = PyBool_FromLong(value) -# -# return result -# -# @staticmethod -# cdef bool2obibool_t(bool value): -# cdef obibool_t result -# -# if value is None: -# result=OBIBool_NA -# else: -# result= value -# -# return result -# -# -# def __init__(self, Column column, index_t line_nb) : -# """ -# Creates a new `OBIDMS_column_line_bool` -# -# @param column: an OBIDMS_column instance -# @param line_nb: the line in the column -# """ -# -# Column_line.__init__(self, column, line_nb) -# self.update_pointer() -# -# -# -# cpdef bool get_bool_item_by_name(self, bytes element_name) : -# """ -# Returns the value associated to the name `element_name` of the current line -# -# @param element_name: a `bytes` instance containing the name of the element -# -# @return: the `bool` value corresponding to the name -# """ -# cdef char* cname = element_name -# cdef obibool_t value -# global obi_errno -# -# self.update_pointer() -# -# cdef OBIDMS_column_p* column_pp -# column_pp = self._pointer -# cdef OBIDMS_column_p column_p = column_pp[0] -# -# value = obi_column_get_obibool_with_elt_name(column_p, -# self._index, -# cname) -# -# if obi_errno > 0 : -# obi_errno = 0 -# raise KeyError("Cannot access to key %s" % bytes2str(element_name)) -# -# return Column_line_bool.obibool_t2bool(value) -# -# -# cpdef bool get_bool_item_by_idx(self,index_t index): -# """ -# Returns the value associated to the name `element_name` of the current line -# -# @param index: a `int` instance containing the index of the element -# -# @return: the `bool` value corresponding to the name -# """ -# cdef obibool_t value # @DuplicatedSignature -# global obi_errno -# -# cdef OBIDMS_column_p* column_pp -# column_pp = self._pointer -# cdef OBIDMS_column_p column_p = column_pp[0] -# -# self.update_pointer() -# -# value = obi_column_get_obibool_with_elt_idx(column_p, -# self._index, -# index) -# -# if obi_errno > 0 : -# obi_errno = 0 -# raise IndexError("Cannot access to element %d" % index) -# -# return Column_line_bool.obibool_t2bool(value) -# -# -# def __getitem__(self, object element_name) : -# cdef bytes name -# cdef int cindex -# cdef obibool_t value -# cdef type typearg = type(element_name) -# cdef bool result -# -# -# if typearg == int: -# cindex=element_name -# if cindex < 0: -# cindex = self._len - cindex -# result=self.get_bool_item_by_idx(cindex) -# elif typearg == bytes: -# result=self.get_bool_item_by_name(element_name) -# elif typearg == str: -# name = str2bytes(element_name) -# result=self.get_bool_item_by_name(name) -# -# return result -# -# cpdef set_bool_item_by_name(self,bytes element_name,bool value): -# """ -# Sets the value associated to the name `element_name` of the current line -# -# @param element_name: a `bytes` instance containing the name of the element -# @param value: a `bool` instance of the new value -# -# @return: the `bool` value corresponding to the name -# """ -# cdef char* cname = element_name -# cdef obibool_t cvalue -# -# self.update_pointer() -# cvalue = OBIDMS_column_line_bool.bool2obibool_t(value) -# -# if ( obi_column_set_obibool_with_elt_name((self._pointer)[0], -# self._index, -# cname, -# cvalue) < 0 ): -# raise KeyError("Cannot access to key %s" % bytes2str(element_name)) -# -# cpdef set_bool_item_by_idx(self,index_t index,bool value): -# """ -# Sets the value associated to the name `element_name` of the current line -# -# @param index: a `int` instance containing the index of the element -# @param value: a `bool` instance of the new value -# -# @return: the `bool` value corresponding to the name -# """ -# cdef obibool_t cvalue # @DuplicatedSignature -# -# self.update_pointer() -# cvalue = OBIDMS_column_line_bool.bool2obibool_t(value) -# -# if ( obi_column_set_obibool_with_elt_idx((self._pointer)[0], -# self._index, -# index, -# cvalue) < 0 ): -# raise IndexError("Cannot access to item index %d" % index) -# -# -# -# def __setitem__(self, object element_name, object value): -# cdef bytes name -# cdef int cindex -# cdef type typearg = type(element_name) -# cdef bool result -# -# -# if typearg == int: -# cindex=element_name -# if cindex < 0: -# cindex = self._len - cindex -# self.set_bool_item_by_idx(cindex,value) -# elif typearg == bytes: -# self.set_bool_item_by_name(element_name,value) -# elif typearg == str: -# name = str2bytes(element_name) -# self.set_bool_item_by_name(name,value) -# -# def __repr__(self) : -# return str(self._column.get_line(self._index)) -# -# def __len__(self): -# return self._len diff --git a/python/obitools3/dms/column/typed_column/char.pxd b/python/obitools3/dms/column/typed_column/char.pxd index 1d70e00..c67db85 100644 --- a/python/obitools3/dms/column/typed_column/char.pxd +++ b/python/obitools3/dms/column/typed_column/char.pxd @@ -18,3 +18,9 @@ cdef class Column_multi_elts_char(Column_multi_elts) : cpdef object get_item(self, index_t line_nb, object elt_id) cpdef object get_line(self, index_t line_nb) cpdef set_item(self, index_t line_nb, object elt_id, object value) + + +cdef class Column_tuples_char(Column): + + cpdef object get_line(self, index_t line_nb) + cpdef set_line(self, index_t line_nb, object value) diff --git a/python/obitools3/dms/column/typed_column/char.pyx b/python/obitools3/dms/column/typed_column/char.pyx index c37d8a3..fe06d7e 100644 --- a/python/obitools3/dms/column/typed_column/char.pyx +++ b/python/obitools3/dms/column/typed_column/char.pyx @@ -13,12 +13,20 @@ from ...capi.obiview cimport obi_get_char_with_elt_name_and_col_p_in_view, \ obi_get_char_with_elt_idx_and_col_p_in_view, \ obi_set_char_with_elt_name_and_col_p_in_view, \ obi_set_char_with_elt_idx_and_col_p_in_view, \ + obi_get_array_with_col_p_in_view, \ + obi_set_array_with_col_p_in_view, \ Obiview_p from ...capi.obidmscolumn cimport OBIDMS_column_p -from ...capi.obitypes cimport OBI_CHAR, OBIChar_NA, obichar_t +from ...capi.obitypes cimport OBI_CHAR, \ + OBIChar_NA, \ + OBITuple_NA, \ + obichar_t +from libc.stdint cimport int32_t + +from libc.stdlib cimport malloc, free cdef class Column_char(Column): @@ -28,11 +36,13 @@ cdef class Column_char(Column): object column_name, index_t nb_elements_per_line=1, object elements_names=None, + bint tuples=False, object comments=b""): return Column.new_column(view, column_name, OBI_CHAR, nb_elements_per_line=nb_elements_per_line, elements_names=elements_names, + tuples=tuples, comments=comments) cpdef object get_line(self, index_t line_nb): @@ -106,7 +116,7 @@ cdef class Column_multi_elts_char(Column_multi_elts): cpdef set_item(self, index_t line_nb, object elt_id, object value) : global obi_errno - cdef bytes elt_name + cdef bytes elt_name cdef obichar_t value_b if value is None : value_b = OBIChar_NA @@ -121,7 +131,69 @@ cdef class Column_multi_elts_char(Column_multi_elts): obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=elt_id, error_message="Problem setting a value in a column") -def register_class(): - register_column_class(OBI_CHAR, False, Column_char, bytes) # TODO bytes or str? - register_column_class(OBI_CHAR, True, Column_multi_elts_char, bytes) # TODO bytes or str? +cdef class Column_tuples_char(Column): + + cpdef object get_line(self, index_t line_nb) : + global obi_errno + cdef obichar_t value + cdef bytes value_in_result + cdef object result + cdef int32_t i + cdef obichar_t* array + cdef int32_t value_length + + result = [] + + array = obi_get_array_with_col_p_in_view(self._view.pointer(), self.pointer(), line_nb, &value_length) + obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem getting a value from a column") + + if array == OBITuple_NA : + return None + + for i in range(value_length) : + value = array[i] + value_in_result = value + result.append(value_in_result) + + return tuple(result) + + + cpdef set_line(self, index_t line_nb, object value) : + global obi_errno + cdef obichar_t* array + cdef int32_t value_length + cdef int32_t i, j + cdef object e + cdef obichar_t value_b + + value_length = 0 + if value is not None: + for e in value: + if e is not None: + value_length+=1 + if value is None or value_length == 0 : + array = OBITuple_NA + else: + array = malloc(value_length * sizeof(obichar_t)) + if array == NULL: + raise Exception("Problem allocating memory for an array to store a tuple") + #raise RollbackException("Problem allocating memory for an array to store a tuple", self._view) # TODO can't import + j=0 + for i in range(len(value)) : + if value[i] is not None: + value_b = tobytes(value[i])[0] + array[j] = value_b + j+=1 + + if obi_set_array_with_col_p_in_view(self._view.pointer(), self.pointer(), line_nb, array, sizeof(obichar_t)*8, value_length) < 0 : + obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem setting a value in a column") + + if array != OBITuple_NA: + free(array) + + +def register_class(): + register_column_class(OBI_CHAR, False, False, Column_char, bytes) + register_column_class(OBI_CHAR, True, False, Column_multi_elts_char, bytes) + register_column_class(OBI_CHAR, False, True, Column_tuples_char, bytes) diff --git a/python/obitools3/dms/column/typed_column/float.pxd b/python/obitools3/dms/column/typed_column/float.pxd index 29d22ca..07e9f7b 100644 --- a/python/obitools3/dms/column/typed_column/float.pxd +++ b/python/obitools3/dms/column/typed_column/float.pxd @@ -18,4 +18,9 @@ cdef class Column_multi_elts_float(Column_multi_elts) : cpdef object get_item(self, index_t line_nb, object elt_id) cpdef object get_line(self, index_t line_nb) cpdef set_item(self, index_t line_nb, object elt_id, object value) - \ No newline at end of file + + +cdef class Column_tuples_float(Column): + + cpdef object get_line(self, index_t line_nb) + cpdef set_line(self, index_t line_nb, object value) diff --git a/python/obitools3/dms/column/typed_column/float.pyx b/python/obitools3/dms/column/typed_column/float.pyx index 4eca2c6..160582d 100644 --- a/python/obitools3/dms/column/typed_column/float.pyx +++ b/python/obitools3/dms/column/typed_column/float.pyx @@ -13,12 +13,20 @@ from ...capi.obiview cimport obi_get_float_with_elt_name_and_col_p_in_view, \ obi_get_float_with_elt_idx_and_col_p_in_view, \ obi_set_float_with_elt_name_and_col_p_in_view, \ obi_set_float_with_elt_idx_and_col_p_in_view, \ + obi_get_array_with_col_p_in_view, \ + obi_set_array_with_col_p_in_view, \ Obiview_p from ...capi.obidmscolumn cimport OBIDMS_column_p -from ...capi.obitypes cimport OBI_FLOAT, OBIFloat_NA, obifloat_t +from ...capi.obitypes cimport OBI_FLOAT, \ + OBIFloat_NA, \ + OBITuple_NA, \ + obifloat_t +from libc.stdint cimport int32_t + +from libc.stdlib cimport malloc, free cdef class Column_float(Column): @@ -28,11 +36,13 @@ cdef class Column_float(Column): object column_name, index_t nb_elements_per_line=1, object elements_names=None, + bint tuples=False, object comments=b""): return Column.new_column(view, column_name, OBI_FLOAT, nb_elements_per_line=nb_elements_per_line, elements_names=elements_names, + tuples=tuples, comments=comments) cpdef object get_line(self, index_t line_nb): @@ -115,7 +125,67 @@ cdef class Column_multi_elts_float(Column_multi_elts): obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=elt_id, error_message="Problem setting a value in a column") -def register_class(): - register_column_class(OBI_FLOAT, False, Column_float, float) # TODO why not double? - register_column_class(OBI_FLOAT, True, Column_multi_elts_float, float) # TODO why not double? +cdef class Column_tuples_float(Column): + + cpdef object get_line(self, index_t line_nb) : + global obi_errno + cdef obifloat_t value + cdef double value_in_result + cdef object result + cdef int32_t i + cdef obifloat_t* array + cdef int32_t value_length + + result = [] + + array = obi_get_array_with_col_p_in_view(self._view.pointer(), self.pointer(), line_nb, &value_length) + obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem getting a value from a column") + + if array == OBITuple_NA : + return None + + for i in range(value_length) : + value = array[i] + value_in_result = value + result.append(value_in_result) + + return tuple(result) + + + cpdef set_line(self, index_t line_nb, object value) : + global obi_errno + cdef obifloat_t* array + cdef int32_t value_length + cdef int32_t i, j + cdef object e + + value_length = 0 + if value is not None: + for e in value: + if e is not None: + value_length+=1 + if value is None or value_length == 0 : + array = OBITuple_NA + else: + array = malloc(value_length * sizeof(obifloat_t)) + if array == NULL: + raise Exception("Problem allocating memory for an array to store a tuple") + #raise RollbackException("Problem allocating memory for an array to store a tuple", self._view) # TODO can't import + j=0 + for i in range(len(value)) : + if value[i] is not None: + array[j] = (value[i]) + j+=1 + + if obi_set_array_with_col_p_in_view(self._view.pointer(), self.pointer(), line_nb, array, sizeof(obifloat_t)*8, value_length) < 0 : + obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem setting a value in a column") + + if array != OBITuple_NA: + free(array) + + +def register_class(): + register_column_class(OBI_FLOAT, False, False, Column_float, float) # TODO why not double? + register_column_class(OBI_FLOAT, True, False, Column_multi_elts_float, float) # TODO why not double? + register_column_class(OBI_FLOAT, False, True, Column_tuples_float, float) diff --git a/python/obitools3/dms/column/typed_column/int.pxd b/python/obitools3/dms/column/typed_column/int.pxd index 23a57ff..0cac022 100644 --- a/python/obitools3/dms/column/typed_column/int.pxd +++ b/python/obitools3/dms/column/typed_column/int.pxd @@ -18,4 +18,9 @@ cdef class Column_multi_elts_int(Column_multi_elts) : cpdef object get_item(self, index_t line_nb, object elt_id) cpdef object get_line(self, index_t line_nb) cpdef set_item(self, index_t line_nb, object elt_id, object value) - \ No newline at end of file + + +cdef class Column_tuples_int(Column): + + cpdef object get_line(self, index_t line_nb) + cpdef set_line(self, index_t line_nb, object value) diff --git a/python/obitools3/dms/column/typed_column/int.pyx b/python/obitools3/dms/column/typed_column/int.pyx index 8906aed..41489ba 100644 --- a/python/obitools3/dms/column/typed_column/int.pyx +++ b/python/obitools3/dms/column/typed_column/int.pyx @@ -13,14 +13,23 @@ from ...capi.obiview cimport obi_get_int_with_elt_name_and_col_p_in_view, \ obi_get_int_with_elt_idx_and_col_p_in_view, \ obi_set_int_with_elt_name_and_col_p_in_view, \ obi_set_int_with_elt_idx_and_col_p_in_view, \ + obi_get_array_with_col_p_in_view, \ + obi_set_array_with_col_p_in_view, \ Obiview_p from ...capi.obidmscolumn cimport OBIDMS_column_p -from ...capi.obitypes cimport OBI_INT, OBIInt_NA, obiint_t +from ...capi.obitypes cimport OBI_INT, \ + OBIInt_NA, \ + OBITuple_NA, \ + obiint_t from cpython.int cimport PyInt_FromLong +from libc.stdint cimport int32_t + +from libc.stdlib cimport malloc, free + cdef class Column_int(Column): @@ -29,11 +38,13 @@ cdef class Column_int(Column): object column_name, index_t nb_elements_per_line=1, object elements_names=None, + bint tuples=False, object comments=b""): return Column.new_column(view, column_name, OBI_INT, nb_elements_per_line=nb_elements_per_line, elements_names=elements_names, + tuples=tuples, comments=comments) @@ -118,7 +129,67 @@ cdef class Column_multi_elts_int(Column_multi_elts): obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=elt_id, error_message="Problem setting a value in a column") -def register_class(): - register_column_class(OBI_INT, False, Column_int, int) - register_column_class(OBI_INT, True, Column_multi_elts_int, int) +cdef class Column_tuples_int(Column): + + cpdef object get_line(self, index_t line_nb) : + global obi_errno + cdef obiint_t value + cdef double value_in_result + cdef object result + cdef int32_t i + cdef obiint_t* array + cdef int32_t value_length + + result = [] + + array = obi_get_array_with_col_p_in_view(self._view.pointer(), self.pointer(), line_nb, &value_length) + obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem getting a value from a column") + + if array == OBITuple_NA : + return None + + for i in range(value_length) : + value = array[i] + value_in_result = PyInt_FromLong(value) + result.append(value_in_result) + + return tuple(result) + + + cpdef set_line(self, index_t line_nb, object value) : + global obi_errno + cdef obiint_t* array + cdef int32_t value_length + cdef int32_t i, j + cdef object e + + value_length = 0 + if value is not None: + for e in value: + if e is not None: + value_length+=1 + if value is None or value_length == 0 : + array = OBITuple_NA + else: + array = malloc(value_length * sizeof(obiint_t)) + if array == NULL: + raise Exception("Problem allocating memory for an array to store a tuple") + #raise RollbackException("Problem allocating memory for an array to store a tuple", self._view) # TODO can't import + j=0 + for i in range(len(value)) : + if value[i] is not None: + array[j] = (value[i]) + j+=1 + + if obi_set_array_with_col_p_in_view(self._view.pointer(), self.pointer(), line_nb, array, sizeof(obiint_t)*8, value_length) < 0 : + obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem setting a value in a column") + + if array != OBITuple_NA: + free(array) + + +def register_class(): + register_column_class(OBI_INT, False, False, Column_int, int) + register_column_class(OBI_INT, True, False, Column_multi_elts_int, int) + register_column_class(OBI_INT, False, True, Column_tuples_int, int) diff --git a/python/obitools3/dms/column/typed_column/qual.pyx b/python/obitools3/dms/column/typed_column/qual.pyx index 3ff8ad3..92c630b 100644 --- a/python/obitools3/dms/column/typed_column/qual.pyx +++ b/python/obitools3/dms/column/typed_column/qual.pyx @@ -25,9 +25,8 @@ from ...capi.obidmscolumn cimport OBIDMS_column_p from ...capi.obitypes cimport OBI_QUAL, OBIQual_char_NA, OBIQual_int_NA, const_char_p -from libc.stdlib cimport free +from libc.stdlib cimport malloc, free from libc.stdint cimport uint8_t -from libc.stdlib cimport malloc # TODO detect type of value and call set_item_str if str or bytes @@ -44,6 +43,7 @@ cdef class Column_qual(Column_idx): return Column.new_column(view, column_name, OBI_QUAL, nb_elements_per_line=nb_elements_per_line, elements_names=elements_names, + tuples=False, comments=comments) @@ -255,8 +255,7 @@ cdef class Column_multi_elts_qual(Column_multi_elts_idx): obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=elt_id, error_message="Problem setting a value in a column") - def register_class(): - register_column_class(OBI_QUAL, False, Column_qual, bytes) # TODO str? int? - register_column_class(OBI_QUAL, True, Column_multi_elts_qual, bytes) # TODO str? int? + register_column_class(OBI_QUAL, False, False, Column_qual, int) # TODO bytes? + register_column_class(OBI_QUAL, True, False, Column_multi_elts_qual, int) # bytes? diff --git a/python/obitools3/dms/column/typed_column/seq.pxd b/python/obitools3/dms/column/typed_column/seq.pxd index 854fd1b..fcab9af 100644 --- a/python/obitools3/dms/column/typed_column/seq.pxd +++ b/python/obitools3/dms/column/typed_column/seq.pxd @@ -18,4 +18,9 @@ cdef class Column_multi_elts_seq(Column_multi_elts_idx) : cpdef object get_item(self, index_t line_nb, object elt_id) cpdef object get_line(self, index_t line_nb) cpdef set_item(self, index_t line_nb, object elt_id, object value) - \ No newline at end of file + + +cdef class Column_tuples_seq(Column_idx): + + cpdef object get_line(self, index_t line_nb) + cpdef set_line(self, index_t line_nb, object value) diff --git a/python/obitools3/dms/column/typed_column/seq.pyx b/python/obitools3/dms/column/typed_column/seq.pyx index 1929469..f663871 100644 --- a/python/obitools3/dms/column/typed_column/seq.pyx +++ b/python/obitools3/dms/column/typed_column/seq.pyx @@ -15,13 +15,21 @@ from ...capi.obiview cimport obi_get_seq_with_elt_name_and_col_p_in_view, \ obi_get_seq_with_elt_idx_and_col_p_in_view, \ obi_set_seq_with_elt_name_and_col_p_in_view, \ obi_set_seq_with_elt_idx_and_col_p_in_view, \ + obi_get_array_with_col_p_in_view, \ + obi_set_array_with_col_p_in_view, \ + obi_get_array_with_col_p_in_view, \ + obi_set_array_with_col_p_in_view, \ Obiview_p from ...capi.obidmscolumn cimport OBIDMS_column_p -from ...capi.obitypes cimport OBI_SEQ, OBISeq_NA +from ...capi.obitypes cimport OBI_SEQ, \ + OBISeq_NA, \ + OBITuple_NA -from libc.stdlib cimport free +from libc.stdint cimport int32_t +from libc.stdlib cimport calloc, free +from libc.string cimport strcpy cdef class Column_seq(Column_idx): @@ -31,11 +39,13 @@ cdef class Column_seq(Column_idx): object column_name, index_t nb_elements_per_line=1, object elements_names=None, + bint tuples=False, object comments=b""): return Column.new_column(view, column_name, OBI_SEQ, nb_elements_per_line=nb_elements_per_line, elements_names=elements_names, + tuples=tuples, comments=comments) @@ -145,7 +155,76 @@ cdef class Column_multi_elts_seq(Column_multi_elts_idx): obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=elt_id, error_message="Problem setting a value in a column") -def register_class(): - register_column_class(OBI_SEQ, False, Column_seq, bytes) # TODO str? - register_column_class(OBI_SEQ, True, Column_multi_elts_seq, bytes) # TODO str? +cdef class Column_tuples_seq(Column_idx): + + cpdef object get_line(self, index_t line_nb) : + global obi_errno + cdef const char* value + cdef bytes value_in_result + cdef object result + cdef int32_t i + cdef const char* array + cdef int32_t value_length + + result = [] + + array = obi_get_array_with_col_p_in_view(self._view.pointer(), self.pointer(), line_nb, &value_length) + obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem getting a value from a column") + + if array == OBITuple_NA : + return None + + i = 0 + # First value + value_in_result = array + result.append(value_in_result) + while i+1 < value_length : + if array[i] != b'\0' : + i+=1 + else : + value = array+i+1 + value_in_result = value + result.append(value_in_result) + i+=1 + + return tuple(result) + + cpdef set_line(self, index_t line_nb, object value) : + global obi_errno + cdef char* array + cdef int32_t value_length + cdef int32_t i + cdef object elt + cdef bytes elt_b + + value_length = 0 + if value is not None: + for i in range(len(value)) : + if value[i] is not None and value[i] != '' : + value_length = value_length + len(value[i]) + 1 # Total size of the array with the '\0' + if value is None or value_length == 0 : + array = OBITuple_NA + else: + array = calloc(value_length, sizeof(char)) + if array == NULL: + raise Exception("Problem allocating memory for an array to store a tuple") + #raise RollbackException("Problem allocating memory for an array to store a tuple", self._view) # TODO can't import + i = 0 + for elt in value : + if elt is not None and elt != '': + elt_b = tobytes(elt) + strcpy(array+i, elt_b) + i = i + len(elt_b) + 1 + + if obi_set_array_with_col_p_in_view(self._view.pointer(), self.pointer(), line_nb, array, sizeof(char)*8, value_length) < 0 : + obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem setting a value in a column") + + if array != OBITuple_NA: + free(array) + + +def register_class(): + register_column_class(OBI_SEQ, False, False, Column_seq, bytes) + register_column_class(OBI_SEQ, True, False, Column_multi_elts_seq, bytes) + register_column_class(OBI_SEQ, False, True, Column_tuples_seq, bytes) diff --git a/python/obitools3/dms/column/typed_column/str.pxd b/python/obitools3/dms/column/typed_column/str.pxd index d35ebda..8f72b44 100644 --- a/python/obitools3/dms/column/typed_column/str.pxd +++ b/python/obitools3/dms/column/typed_column/str.pxd @@ -18,4 +18,9 @@ cdef class Column_multi_elts_str(Column_multi_elts_idx) : cpdef object get_item(self, index_t line_nb, object elt_id) cpdef object get_line(self, index_t line_nb) cpdef set_item(self, index_t line_nb, object elt_id, object value) - \ No newline at end of file + + +cdef class Column_tuples_str(Column_idx): + + cpdef object get_line(self, index_t line_nb) + cpdef set_line(self, index_t line_nb, object value) diff --git a/python/obitools3/dms/column/typed_column/str.pyx b/python/obitools3/dms/column/typed_column/str.pyx index f0c78f8..ba672f2 100644 --- a/python/obitools3/dms/column/typed_column/str.pyx +++ b/python/obitools3/dms/column/typed_column/str.pyx @@ -15,11 +15,20 @@ from ...capi.obiview cimport obi_get_str_with_elt_name_and_col_p_in_view, \ obi_get_str_with_elt_idx_and_col_p_in_view, \ obi_set_str_with_elt_name_and_col_p_in_view, \ obi_set_str_with_elt_idx_and_col_p_in_view, \ + obi_get_array_with_col_p_in_view, \ + obi_set_array_with_col_p_in_view, \ Obiview_p from ...capi.obidmscolumn cimport OBIDMS_column_p -from ...capi.obitypes cimport OBI_STR, OBIStr_NA, const_char_p +from ...capi.obitypes cimport OBI_STR, \ + OBIStr_NA, \ + OBITuple_NA, \ + const_char_p + +from libc.stdint cimport int32_t +from libc.stdlib cimport calloc, free +from libc.string cimport strcpy cdef class Column_str(Column_idx): @@ -29,11 +38,13 @@ cdef class Column_str(Column_idx): object column_name, index_t nb_elements_per_line=1, object elements_names=None, + bint tuples=False, object comments=b""): return Column.new_column(view, column_name, OBI_STR, nb_elements_per_line=nb_elements_per_line, elements_names=elements_names, + tuples=tuples, comments=comments) @@ -133,7 +144,76 @@ cdef class Column_multi_elts_str(Column_multi_elts_idx): obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=elt_id, error_message="Problem setting a value in a column") -def register_class(): - register_column_class(OBI_STR, False, Column_str, bytes) # TODO str? - register_column_class(OBI_STR, True, Column_multi_elts_str, bytes) # TODO str? +cdef class Column_tuples_str(Column_idx): + + cpdef object get_line(self, index_t line_nb) : + global obi_errno + cdef const char* value + cdef bytes value_in_result + cdef object result + cdef int32_t i + cdef const char* array + cdef int32_t value_length + + result = [] + + array = obi_get_array_with_col_p_in_view(self._view.pointer(), self.pointer(), line_nb, &value_length) + obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem getting a value from a column") + + if array == OBITuple_NA : + return None + + i = 0 + # First value + value_in_result = array + result.append(value_in_result) + while i+1 < value_length : + if array[i] != b'\0' : + i+=1 + else : + value = array+i+1 + value_in_result = value + result.append(value_in_result) + i+=1 + + return tuple(result) + + cpdef set_line(self, index_t line_nb, object value) : + global obi_errno + cdef char* array + cdef int32_t value_length + cdef int32_t i + cdef object elt + cdef bytes elt_b + + value_length = 0 + if value is not None: + for i in range(len(value)) : + if value[i] is not None and value[i] != '' : + value_length = value_length + len(value[i]) + 1 # Total size of the array with the '\0' + if value is None or value_length == 0 : + array = OBITuple_NA + else: + array = calloc(value_length, sizeof(char)) + if array == NULL: + raise Exception("Problem allocating memory for an array to store a tuple") + #raise RollbackException("Problem allocating memory for an array to store a tuple", self._view) # TODO can't import + i = 0 + for elt in value : + if elt is not None and elt != '': + elt_b = tobytes(elt) + strcpy(array+i, elt_b) + i = i + len(elt_b) + 1 + + if obi_set_array_with_col_p_in_view(self._view.pointer(), self.pointer(), line_nb, array, sizeof(char)*8, value_length) < 0 : + obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem setting a value in a column") + + if array != OBITuple_NA: + free(array) + + +def register_class(): + register_column_class(OBI_STR, False, False, Column_str, bytes) + register_column_class(OBI_STR, True, False, Column_multi_elts_str, bytes) + register_column_class(OBI_STR, False, True, Column_tuples_str, bytes) diff --git a/python/obitools3/dms/dms.cfiles b/python/obitools3/dms/dms.cfiles index 1d280fb..43de89d 100644 --- a/python/obitools3/dms/dms.cfiles +++ b/python/obitools3/dms/dms.cfiles @@ -1,3 +1,4 @@ +../../../src/array_indexer.c ../../../src/bloom.c ../../../src/char_str_indexer.c ../../../src/crc64.c @@ -12,6 +13,7 @@ ../../../src/obiblob.c ../../../src/obidms_taxonomy.c ../../../src/obidms.c +../../../src/obidmscolumn_array.c ../../../src/obidmscolumn_blob.c ../../../src/obidmscolumn_bool.c ../../../src/obidmscolumn_char.c diff --git a/python/obitools3/utils.cfiles b/python/obitools3/utils.cfiles index cbb30fc..a775d3c 100644 --- a/python/obitools3/utils.cfiles +++ b/python/obitools3/utils.cfiles @@ -1,3 +1,4 @@ +../../src/array_indexer.c ../../src/bloom.c ../../src/char_str_indexer.c ../../src/crc64.c @@ -21,6 +22,7 @@ ../../src/obidmscolumn_qual.c ../../src/obidmscolumn_seq.c ../../src/obidmscolumn_str.c +../../src/obidmscolumn_array.c ../../src/obidmscolumn.c ../../src/obidmscolumndir.c ../../src/obierrno.c diff --git a/src/array_indexer.c b/src/array_indexer.c new file mode 100644 index 0000000..ef8488a --- /dev/null +++ b/src/array_indexer.c @@ -0,0 +1,73 @@ +/**************************************************************************** + * Array indexing functions * + ****************************************************************************/ + +/** + * @file array_indexer.c + * @author Celine Mercier + * @date October 5th 2017 + * @brief Functions handling the indexing and retrieval of arrays of any type. + */ + + +#include +#include +#include +#include + +#include "obiblob.h" +#include "obiblob_indexer.h" +#include "obidebug.h" +#include "obitypes.h" + + +#define DEBUG_LEVEL 0 // TODO has to be defined somewhere else (cython compil flag?) + + +index_t obi_index_array(Obi_indexer_p indexer, const void* value, uint8_t elt_size, int value_length) +{ + Obi_blob_p value_b; + index_t idx; + int32_t length_bytes; + +// fprintf(stderr, "\nelt size in C: %u", elt_size); + +// for (int i=0; ivalue))[i]); + + // Add in the indexer + idx = obi_indexer_add(indexer, value_b); + + free(value_b); + + return idx; +} + + +const void* obi_retrieve_array(Obi_indexer_p indexer, index_t idx, int* value_length_p) +{ + Obi_blob_p value_b; + + // Get encoded value + value_b = obi_indexer_get(indexer, idx); + + // Store array length + *value_length_p = (value_b->length_decoded_value) / (value_b->element_size); + +// for (int i=0; i<*value_length_p; i++) +// fprintf(stderr, "\nvalue %d", ((obibool_t*)(value_b->value))[i]); + + // Return pointer on mapped array + return ((void*) (value_b->value)); +} + diff --git a/src/array_indexer.h b/src/array_indexer.h new file mode 100644 index 0000000..c029a50 --- /dev/null +++ b/src/array_indexer.h @@ -0,0 +1,60 @@ +/**************************************************************************** + * Array indexer header file * + ****************************************************************************/ + +/** + * @file array_indexer.h + * @author Celine Mercier + * @date October 5th 2017 + * @brief Header file for the functions handling the indexing of arrays of any type. + */ + + +#ifndef ARRAY_INDEXER_H_ +#define ARRAY_INDEXER_H_ + + +#include +#include + +#include "obidms.h" +#include "obitypes.h" +#include "obiblob.h" +#include "obiblob_indexer.h" + + +/** + * @brief Stores an array of elements of any type in an indexer and returns the index. + * + * @param indexer The indexer structure. + * @param value The array to index. + * @param elt_size The size in bits of one element. + * @param value_length The length (number of elements) of the array to index. + * + * @returns The index referring to the stored array in the indexer. + * + * @since October 2017 + * @author Celine Mercier (celine.mercier@metabarcoding.org) + */ +index_t obi_index_array(Obi_indexer_p indexer, const void* value, uint8_t elt_size, int32_t value_length); + + +/** + * @brief Retrieves an array from an indexer. + * + * @warning The array returned is mapped. + * + * @param indexer The indexer structure. + * @param idx The index referring to the array to retrieve in the indexer. + * @param value_length A pointer on an integer to store the length of the array retrieved. + * + * @returns A pointer on the array. + * + * @since October 2017 + * @author Celine Mercier (celine.mercier@metabarcoding.org) + */ +const void* obi_retrieve_array(Obi_indexer_p indexer, index_t idx, int32_t* value_length_p); + + +#endif /* ARRAY_INDEXER_H_ */ + diff --git a/src/obi_align.c b/src/obi_align.c index db07274..d43c55c 100644 --- a/src/obi_align.c +++ b/src/obi_align.c @@ -154,35 +154,35 @@ static int create_alignment_output_columns(Obiview_p output_view, bool normalize, int reference, bool similarity_mode) { // Create the column for the ids of the 1st sequence aligned - if (obi_view_add_column(output_view, ID1_COLUMN_NAME, -1, NULL, OBI_STR, 0, 1, NULL, id1_indexer_name, NULL, -1, ID1_COLUMN_COMMENTS, true) < 0) + if (obi_view_add_column(output_view, ID1_COLUMN_NAME, -1, NULL, OBI_STR, 0, 1, NULL, false, id1_indexer_name, NULL, -1, ID1_COLUMN_COMMENTS, true) < 0) { obidebug(1, "\nError creating the first column for the sequence ids when aligning"); return -1; } // Create the column for the ids of the 2nd sequence aligned - if (obi_view_add_column(output_view, ID2_COLUMN_NAME, -1, NULL, OBI_STR, 0, 1, NULL, id2_indexer_name, NULL, -1, ID2_COLUMN_COMMENTS, true) < 0) + if (obi_view_add_column(output_view, ID2_COLUMN_NAME, -1, NULL, OBI_STR, 0, 1, NULL, false, id2_indexer_name, NULL, -1, ID2_COLUMN_COMMENTS, true) < 0) { obidebug(1, "\nError creating the second column for the sequence ids when aligning"); return -1; } // Create the column for the index (in the input view) of the first sequences aligned - if (obi_view_add_column(output_view, IDX1_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, NULL, NULL, -1, IDX1_COLUMN_COMMENTS, true) < 0) + if (obi_view_add_column(output_view, IDX1_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, NULL, NULL, -1, IDX1_COLUMN_COMMENTS, true) < 0) { obidebug(1, "\nError creating the first column for the sequence indices when aligning"); return -1; } // Create the column for the index (in the input view) of the second sequences aligned - if (obi_view_add_column(output_view, IDX2_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, NULL, NULL, -1, IDX2_COLUMN_COMMENTS, true) < 0) + if (obi_view_add_column(output_view, IDX2_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, NULL, NULL, -1, IDX2_COLUMN_COMMENTS, true) < 0) { obidebug(1, "\nError creating the second column for the sequence indices when aligning"); return -1; } // Create the column for the LCS length - if (obi_view_add_column(output_view, LCS_LENGTH_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, NULL, NULL, -1, LCS_LENGTH_COLUMN_COMMENTS, true) < 0) + if (obi_view_add_column(output_view, LCS_LENGTH_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, NULL, NULL, -1, LCS_LENGTH_COLUMN_COMMENTS, true) < 0) { obidebug(1, "\nError creating the column for the LCS length when aligning"); return -1; @@ -191,7 +191,7 @@ static int create_alignment_output_columns(Obiview_p output_view, // Create the column for the alignment length if it is computed if ((reference == ALILEN) && (normalize || !similarity_mode)) { - if (obi_view_add_column(output_view, ALI_LENGTH_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, NULL, NULL, -1, ALI_LENGTH_COLUMN_COMMENTS, true) < 0) + if (obi_view_add_column(output_view, ALI_LENGTH_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, NULL, NULL, -1, ALI_LENGTH_COLUMN_COMMENTS, true) < 0) { obidebug(1, "\nError creating the column for the alignment length when aligning"); return -1; @@ -200,7 +200,7 @@ static int create_alignment_output_columns(Obiview_p output_view, // Create the column for the alignment score if (normalize) { - if (obi_view_add_column(output_view, SCORE_COLUMN_NAME, -1, NULL, OBI_FLOAT, 0, 1, NULL, NULL, NULL, -1, SCORE_COLUMN_NAME, true) < 0) + if (obi_view_add_column(output_view, SCORE_COLUMN_NAME, -1, NULL, OBI_FLOAT, 0, 1, NULL, false, NULL, NULL, -1, SCORE_COLUMN_NAME, true) < 0) { obidebug(1, "\nError creating the column for the score when aligning"); return -1; @@ -208,7 +208,7 @@ static int create_alignment_output_columns(Obiview_p output_view, } else { - if (obi_view_add_column(output_view, SCORE_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, NULL, NULL, -1, SCORE_COLUMN_NAME, true) < 0) + if (obi_view_add_column(output_view, SCORE_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, NULL, NULL, -1, SCORE_COLUMN_NAME, true) < 0) { obidebug(1, "\nError creating the column for the score when aligning"); return -1; @@ -218,14 +218,14 @@ static int create_alignment_output_columns(Obiview_p output_view, if (print_seq) { // Create the column for the first sequences aligned - if (obi_view_add_column(output_view, SEQ1_COLUMN_NAME, -1, NULL, OBI_SEQ, 0, 1, NULL, seq1_indexer_name, NULL, -1, SEQ1_COLUMN_COMMENTS, true) < 0) + if (obi_view_add_column(output_view, SEQ1_COLUMN_NAME, -1, NULL, OBI_SEQ, 0, 1, NULL, false, seq1_indexer_name, NULL, -1, SEQ1_COLUMN_COMMENTS, true) < 0) { obidebug(1, "\nError creating the first column for the sequences when aligning"); return -1; } // Create the column for the second sequences aligned - if (obi_view_add_column(output_view, SEQ2_COLUMN_NAME, -1, NULL, OBI_SEQ, 0, 1, NULL, seq2_indexer_name, NULL, -1, SEQ2_COLUMN_COMMENTS, true) < 0) + if (obi_view_add_column(output_view, SEQ2_COLUMN_NAME, -1, NULL, OBI_SEQ, 0, 1, NULL, false, seq2_indexer_name, NULL, -1, SEQ2_COLUMN_COMMENTS, true) < 0) { obidebug(1, "\nError creating the second column for the sequences when aligning"); return -1; @@ -234,14 +234,14 @@ static int create_alignment_output_columns(Obiview_p output_view, // if (print_count) // TODO count columns not implemented yet // { // // Create the column for the count of the first sequences aligned -// if (obi_view_add_column(output_view, COUNT1_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, NULL, NULL, -1, COUNT1_COLUMN_COMMENTS, true) < 0) +// if (obi_view_add_column(output_view, COUNT1_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, NULL, NULL, -1, COUNT1_COLUMN_COMMENTS, true) < 0) // { // obidebug(1, "\nError creating the first column for the sequence counts when aligning"); // return -1; // } // // // Create the column for the count of the second sequences aligned -// if (obi_view_add_column(output_view, COUNT2_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, NULL, NULL, -1, COUNT2_COLUMN_COMMENTS, true) < 0) +// if (obi_view_add_column(output_view, COUNT2_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, NULL, NULL, -1, COUNT2_COLUMN_COMMENTS, true) < 0) // { // obidebug(1, "\nError creating the second column for the sequence counts when aligning"); // return -1; diff --git a/src/obidms.c b/src/obidms.c index 0af9fb2..1fccaa7 100644 --- a/src/obidms.c +++ b/src/obidms.c @@ -924,7 +924,7 @@ obiversion_t obi_import_column(const char* dms_path_1, const char* dms_path_2, c // Create new column column_2 = obi_create_column(dms_2, column_name, header_1->returned_data_type, header_1->line_count, - header_1->nb_elements_per_line, header_1->elements_names, true, + header_1->nb_elements_per_line, header_1->elements_names, true, header_1->tuples, new_avl_name, (header_1->associated_column).column_name, (header_1->associated_column).version, header_1->comments); @@ -1142,6 +1142,7 @@ int obi_import_view(const char* dms_path_1, const char* dms_path_2, const char* 0, 0, NULL, + false, NULL, NULL, -1, diff --git a/src/obidmscolumn.c b/src/obidmscolumn.c index 1f3ad91..4554503 100644 --- a/src/obidmscolumn.c +++ b/src/obidmscolumn.c @@ -890,6 +890,7 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms, index_t nb_elements_per_line, char* elements_names, bool elt_names_formatted, + bool tuples, const char* indexer_name, const char* associated_column_name, obiversion_t associated_column_version, @@ -957,7 +958,7 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms, } // Build the indexer name if needed - if ((data_type == OBI_STR) || (data_type == OBI_SEQ) || (data_type == OBI_QUAL)) + if ((data_type == OBI_STR) || (data_type == OBI_SEQ) || (data_type == OBI_QUAL) || tuples) { if ((indexer_name == NULL) || (*indexer_name == 0)) { @@ -973,7 +974,7 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms, } returned_data_type = data_type; - if ((data_type == OBI_STR) || (data_type == OBI_SEQ) || (data_type == OBI_QUAL)) + if ((data_type == OBI_STR) || (data_type == OBI_SEQ) || (data_type == OBI_QUAL) || tuples) // stored data is indices referring to data stored elsewhere stored_data_type = OBI_IDX; else @@ -1105,6 +1106,7 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms, header->nb_elements_per_line = nb_elements_per_line; header->stored_data_type = stored_data_type; header->returned_data_type = returned_data_type; + header->tuples = tuples; header->creation_date = time(NULL); header->version = version_number; header->cloned_from = -1; @@ -1146,7 +1148,7 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms, } // If the data type is OBI_STR, OBI_SEQ or OBI_QUAL, the associated obi_indexer is opened or created - if ((returned_data_type == OBI_STR) || (returned_data_type == OBI_SEQ) || (returned_data_type == OBI_QUAL)) + if ((returned_data_type == OBI_STR) || (returned_data_type == OBI_SEQ) || (returned_data_type == OBI_QUAL) || tuples) { new_column->indexer = obi_indexer(dms, final_indexer_name); if (new_column->indexer == NULL) @@ -1377,6 +1379,7 @@ OBIDMS_column_p obi_clone_column(OBIDMS_p dms, nb_elements_per_line, (column_to_clone->header)->elements_names, true, + (column_to_clone->header)->tuples, (column_to_clone->header)->indexer_name, ((column_to_clone->header)->associated_column).column_name, ((column_to_clone->header)->associated_column).version, diff --git a/src/obidmscolumn.h b/src/obidmscolumn.h index 939effb..34e27d4 100644 --- a/src/obidmscolumn.h +++ b/src/obidmscolumn.h @@ -86,6 +86,8 @@ typedef struct OBIDMS_column_header { OBIType_t stored_data_type; /**< Type of the data that is actually stored in the data * part of the column. */ + bool tuples; /**< A boolean indicating whether the column contains indices referring to indexed tuples. + */ time_t creation_date; /**< Date of creation of the file. */ obiversion_t version; /**< Version of the column. @@ -100,7 +102,7 @@ typedef struct OBIDMS_column_header { */ Column_reference_t associated_column; /**< If there is one, the reference to the associated column. */ - bool finished; /**< A boolean indicating whether the column was properly closed by the view that created it. TODO + bool finished; /**< A boolean indicating whether the column was properly closed by the view that created it. */ char comments[COMMENTS_MAX_LENGTH+1]; /**< Comments stored as a classical zero end C string. */ @@ -239,6 +241,7 @@ size_t obi_get_platform_header_size(); * @param elements_names The names of the elements with ';' as separator (no terminal ';'), * NULL or "" if the default names are to be used ("0\01\02\0...\0n"). * @param elt_names_formatted Whether the separator for the elements names is ';' (false), or '\0' (true, as formatted by format_elements_names()). + * @param tuples A boolean indicating whether the column should contain indices referring to indexed tuples. * @param indexer_name The name of the indexer if there is one associated with the column. * If NULL or "", the indexer name is set as the column name. * @param associated_column_name The name of the associated column if there is one (otherwise NULL or ""). @@ -258,6 +261,7 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms, index_t nb_elements_per_line, char* elements_names, bool elt_names_formatted, + bool tuples, const char* indexer_name, const char* associated_column_name, obiversion_t associated_column_version, diff --git a/src/obidmscolumn_array.c b/src/obidmscolumn_array.c new file mode 100644 index 0000000..73e0aab --- /dev/null +++ b/src/obidmscolumn_array.c @@ -0,0 +1,85 @@ +/**************************************************************************** + * OBIDMS_column_array functions * + ****************************************************************************/ + +/** + * @file obidsmcolumn_array.c + * @author Celine Mercier + * @date October 27th 2017 + * @brief Functions handling OBIColumns containing data arrays of any type. + */ + + +#include +#include + +#include "obidmscolumn.h" +#include "obitypes.h" +#include "array_indexer.h" + + +/********************************************************************** + * + * D E F I N I T I O N O F T H E P U B L I C F U N C T I O N S + * + **********************************************************************/ + + +int obi_column_set_array(OBIDMS_column_p column, index_t line_nb, const void* value, uint8_t elt_size, int32_t value_length) +{ + index_t idx; + + if (obi_column_prepare_to_set_value(column, line_nb, 0) < 0) + return -1; + + if (value == OBITuple_NA) + { + idx = OBIIdx_NA; + } + else + { + // Add the value in the indexer + idx = obi_index_array(column->indexer, value, elt_size, value_length); + if (idx == -1) // An error occurred + { + if (obi_errno == OBI_READ_ONLY_INDEXER_ERROR) + { + // If the error is that the indexer is read-only, clone it + if (obi_clone_column_indexer(column) < 0) + return -1; + obi_set_errno(0); + + // Add the value in the new indexer + idx = obi_index_array(column->indexer, value, elt_size, value_length); + if (idx == -1) + return -1; + } + else + return -1; + } + } + + // Add the value's index in the column + *(((index_t*) (column->data)) + line_nb) = idx; + + return 0; +} + + +const void* obi_column_get_array(OBIDMS_column_p column, index_t line_nb, int32_t* value_length_p) +{ + index_t idx; + + if (obi_column_prepare_to_get_value(column, line_nb) < 0) + return OBITuple_NA; + + idx = *(((index_t*) (column->data)) + line_nb); + + // Check NA + if (idx == OBIIdx_NA) + return OBITuple_NA; + + return obi_retrieve_array(column->indexer, idx, value_length_p); +} + + diff --git a/src/obidmscolumn_array.h b/src/obidmscolumn_array.h new file mode 100644 index 0000000..2286914 --- /dev/null +++ b/src/obidmscolumn_array.h @@ -0,0 +1,64 @@ +/**************************************************************************** + * Array columns header file * + ****************************************************************************/ + +/** + * @file obidsmcolumn_array.h + * @author Celine Mercier + * @date October 30th 2017 + * @brief Header file for the functions handling OBIColumns containing data in the form of indices referring to data arrays. + */ + + +#ifndef OBIDMSCOLUMN_ARRAY_H_ +#define OBIDMSCOLUMN_ARRAY_H_ + + +#include +#include + +#include "obidmscolumn.h" +#include "obitypes.h" + + +/** + * @brief Sets a value in an OBIDMS column containing data in the form of indices referring + * to arrays handled by an indexer. + * + * @warning Pointers returned by obi_open_column() don't allow writing. + * + * @param column A pointer as returned by obi_create_column() or obi_clone_column(). + * @param line_nb The number of the line where the value should be set. + * @param value A pointer on the array. + * @param elt_size The size in bits of one element. + * @param value_length The length (number of elements) of the array to index. + * + * @returns An integer value indicating the success of the operation. + * @retval 0 on success. + * @retval -1 if an error occurred. + * + * @since October 2017 + * @author Celine Mercier (celine.mercier@metabarcoding.org) + */ +int obi_column_set_array(OBIDMS_column_p column, index_t line_nb, const void* value, uint8_t elt_size, int32_t value_length); + + +/** + * @brief Recovers a value in an OBIDMS column containing data in the form of indices referring + * to arrays handled by an indexer. + * + * @param column A pointer as returned by obi_create_column(). + * @param line_nb The number of the line where the value should be recovered. + * @param value_length A pointer on an integer to store the length of the array retrieved. + * + * @returns The recovered value. + * @retval OBITuple_NA the NA value of the type if an error occurred and obi_errno is set. + * + * @since October 2017 + * @author Celine Mercier (celine.mercier@metabarcoding.org) + */ +const void* obi_column_get_array(OBIDMS_column_p column, index_t line_nb, int32_t* value_length_p); + + +#endif /* OBIDMSCOLUMN_ARRAY_H_ */ + diff --git a/src/obitypes.h b/src/obitypes.h index 8e642ac..c6e9402 100644 --- a/src/obitypes.h +++ b/src/obitypes.h @@ -27,7 +27,7 @@ #define OBIBlob_NA (NULL) /**< NA value for the type Obiblobs */ // TODO discuss #define OBIQual_char_NA (NULL) /**< NA value for the type OBI_QUAL if the quality is in character string format */ #define OBIQual_int_NA (NULL) /**< NA value for the type OBI_QUAL if the quality is in integer format */ - +#define OBITuple_NA (NULL) /**< NA value for tuples of any type */ /** * @brief enum for the boolean OBIType. diff --git a/src/obiview.c b/src/obiview.c index 9859387..d163f1a 100644 --- a/src/obiview.c +++ b/src/obiview.c @@ -28,6 +28,7 @@ #include "obidmscolumn_qual.h" #include "obidmscolumn_seq.h" #include "obidmscolumn_str.h" +#include "obidmscolumn_array.h" #include "obierrno.h" #include "obidebug.h" #include "obilittlebigman.h" @@ -1643,7 +1644,7 @@ Obiview_p obi_new_view(OBIDMS_p dms, const char* view_name, Obiview_p view_to_cl // If there is a new line selection, build it by combining it with the one from the view to clone if there is one else if (line_selection != NULL) { - view->line_selection = obi_create_column(view->dms, LINES_COLUMN_NAME, OBI_IDX, 0, 1, NULL, false, NULL, NULL, -1, NULL); + view->line_selection = obi_create_column(view->dms, LINES_COLUMN_NAME, OBI_IDX, 0, 1, NULL, false, false, NULL, NULL, -1, NULL); if ((view->line_selection) == NULL) { obidebug(1, "\nError creating a column corresponding to a line selection"); @@ -1795,6 +1796,7 @@ Obiview_p obi_new_view(OBIDMS_p dms, const char* view_name, Obiview_p view_to_cl 0, 0, NULL, + false, NULL, NULL, -1, @@ -1862,19 +1864,19 @@ Obiview_p obi_new_view_nuc_seqs(OBIDMS_p dms, const char* view_name, Obiview_p v if (view_to_clone == NULL) { // Adding sequence column - if (obi_view_add_column(view, NUC_SEQUENCE_COLUMN, -1, NULL, OBI_SEQ, 0, 1, NULL, NULL, NULL, -1, "Nucleotide sequences", true) < 0) // discuss using same indexer "NUC_SEQ_INDEXER" + if (obi_view_add_column(view, NUC_SEQUENCE_COLUMN, -1, NULL, OBI_SEQ, 0, 1, NULL, false, NULL, NULL, -1, "Nucleotide sequences", true) < 0) // discuss using same indexer "NUC_SEQ_INDEXER" { obidebug(1, "Error adding an obligatory column in a nucleotide sequences view"); return NULL; } // Adding id column - if (obi_view_add_column(view, ID_COLUMN, -1, NULL, OBI_STR, 0, 1, NULL, NULL, NULL, -1, "Sequence identifiers", true) < 0) + if (obi_view_add_column(view, ID_COLUMN, -1, NULL, OBI_STR, 0, 1, NULL, false, NULL, NULL, -1, "Sequence identifiers", true) < 0) { obidebug(1, "Error adding an obligatory column in a nucleotide sequences view"); return NULL; } // Adding definition column - if (obi_view_add_column(view, DEFINITION_COLUMN, -1, NULL, OBI_STR, 0, 1, NULL, NULL, NULL, -1, "Definitions", true) < 0) + if (obi_view_add_column(view, DEFINITION_COLUMN, -1, NULL, OBI_STR, 0, 1, NULL, false, NULL, NULL, -1, "Definitions", true) < 0) { obidebug(1, "Error adding an obligatory column in a nucleotide sequences view"); return NULL; @@ -1883,7 +1885,7 @@ Obiview_p obi_new_view_nuc_seqs(OBIDMS_p dms, const char* view_name, Obiview_p v if (quality_column) { associated_nuc_column = obi_view_get_column(view, NUC_SEQUENCE_COLUMN); - if (obi_view_add_column(view, QUALITY_COLUMN, -1, NULL, OBI_QUAL, 0, 1, NULL, NULL, (associated_nuc_column->header)->name, (associated_nuc_column->header)->version, "Sequence qualities", true) < 0) // TODO discuss automatic association + if (obi_view_add_column(view, QUALITY_COLUMN, -1, NULL, OBI_QUAL, 0, 1, NULL, false, NULL, (associated_nuc_column->header)->name, (associated_nuc_column->header)->version, "Sequence qualities", true) < 0) // TODO discuss automatic association { obidebug(1, "Error adding an obligatory column in a nucleotide sequences view"); return NULL; @@ -2211,6 +2213,7 @@ int obi_view_add_column(Obiview_p view, index_t nb_lines, index_t nb_elements_per_line, char* elements_names, + bool tuples, const char* indexer_name, const char* associated_column_name, obiversion_t associated_column_version, @@ -2289,7 +2292,7 @@ int obi_view_add_column(Obiview_p view, // Open or create the column if (create) { // Create column - column = obi_create_column(view->dms, column_name, data_type, nb_lines, nb_elements_per_line, elements_names, false, indexer_name, associated_column_name, associated_column_version, comments); + column = obi_create_column(view->dms, column_name, data_type, nb_lines, nb_elements_per_line, elements_names, false, tuples, indexer_name, associated_column_name, associated_column_version, comments); if (column == NULL) { obidebug(1, "\nError creating a column to add to a view"); @@ -2796,7 +2799,7 @@ int obi_create_auto_count_column(Obiview_p view) return -1; } - if (obi_view_add_column(view, COUNT_COLUMN, -1, NULL, OBI_INT, 0, 1, NULL, NULL, NULL, -1, "Sequence counts", true) < 0) + if (obi_view_add_column(view, COUNT_COLUMN, -1, NULL, OBI_INT, 0, 1, NULL, false, NULL, NULL, -1, "Sequence counts", true) < 0) { obidebug(1, "Error adding an automatic count column in a view"); return -1; @@ -2848,7 +2851,7 @@ int obi_create_auto_id_column(Obiview_p view, const char* prefix) } // Create the new ID column - if (obi_view_add_column(view, ID_COLUMN, -1, NULL, OBI_STR, 0, 1, NULL, NULL, NULL, -1, "Sequence identifiers", true) < 0) + if (obi_view_add_column(view, ID_COLUMN, -1, NULL, OBI_STR, 0, 1, NULL, false, NULL, NULL, -1, "Sequence identifiers", true) < 0) { obidebug(1, "Error adding an automatic ID column in a view"); return -1; @@ -3623,3 +3626,42 @@ index_t obi_get_index_with_elt_name_and_col_name_in_view(Obiview_p view, const c /****************************************/ + +/*********** FOR ARRAY COLUMNS ***********/ + +int obi_set_array_with_col_p_in_view(Obiview_p view, OBIDMS_column_p column, index_t line_nb, const void* value, uint8_t elt_size, int32_t value_length) +{ + if (prepare_to_set_value_in_column(view, &column, &line_nb) < 0) + return -1; + return obi_column_set_array(column, line_nb, value, elt_size, value_length); +} + + +const void* obi_get_array_with_col_p_in_view(Obiview_p view, OBIDMS_column_p column, index_t line_nb, int32_t* value_length_p) +{ + if (prepare_to_get_value_from_column(view, &line_nb) < 0) + return OBITuple_NA; + return obi_column_get_array(column, line_nb, value_length_p); +} + + +int obi_set_array_with_col_name_in_view(Obiview_p view, const char* column_name, index_t line_nb, const void* value, uint8_t elt_size, int32_t value_length) +{ + OBIDMS_column_p column_p; + column_p = obi_view_get_column(view, column_name); + if (column_p == NULL) + return -1; + return obi_set_array_with_col_p_in_view(view, column_p, line_nb, value, elt_size, value_length); +} + + +const void* obi_get_array_with_col_name_in_view(Obiview_p view, const char* column_name, index_t line_nb, int32_t* value_length_p) +{ + OBIDMS_column_p column_p; + column_p = obi_view_get_column(view, column_name); + if (column_p == NULL) + return OBITuple_NA; + return obi_get_array_with_col_p_in_view(view, column_p, line_nb, value_length_p); +} + +/****************************************/ diff --git a/src/obiview.h b/src/obiview.h index 162afaa..aac9a58 100644 --- a/src/obiview.h +++ b/src/obiview.h @@ -361,6 +361,7 @@ Obiview_p obi_open_view(OBIDMS_p dms, const char* view_name); * @param nb_elements_per_line The number of elements per line, if the column is created. * @param elements_names The names of the elements with ';' as separator (no terminal ';'), * if the column is created; NULL or "" if the default names are to be used ("0\01\02\0...\0n"). + * @param tuples A boolean indicating whether the column should contain indices referring to indexed tuples. * @param indexer_name The name of the indexer if there is one associated with the column, if the column is created. * If NULL or "", the indexer name is set as the column name. * @param associated_column_name The name of the associated column if there is one (otherwise NULL or ""), if the column is created. @@ -383,6 +384,7 @@ int obi_view_add_column(Obiview_p view, index_t nb_lines, index_t nb_elements_per_line, char* elements_names, + bool tuples, const char* indexer_name, const char* associated_column_name, obiversion_t associated_column_version, @@ -2207,4 +2209,88 @@ index_t obi_get_index_with_elt_name_and_col_p_in_view(Obiview_p view, OBIDMS_col index_t obi_get_index_with_elt_name_and_col_name_in_view(Obiview_p view, const char* column_name, index_t line_nb, const char* element_name); +/*********** FOR ARRAY COLUMNS ***********/ + +/** + * @brief Sets a value in an OBIDMS column containing indices referring to indexed arrays, + * using the column pointer, in the context of a view. + * + * Note: If the column is read-only or if there is a line selection associated with the view (making columns non-writable), it is cloned. + * + * @param view A pointer on the opened writable view. + * @param column_p A pointer on the column. + * @param line_nb The number of the line where the value should be set. + * @param value The value that should be set. + * @param elt_size The size in bits of one element. + * @param value_length The length (number of elements) of the array to index. + * + * @returns An integer value indicating the success of the operation. + * @retval 0 on success. + * @retval -1 if an error occurred. + * + * @since October 2017 + * @author Celine Mercier (celine.mercier@metabarcoding.org) + */ +int obi_set_array_with_col_p_in_view(Obiview_p view, OBIDMS_column_p column, index_t line_nb, const void* value, uint8_t elt_size, int32_t value_length); + + +/** + * @brief Recovers a value in an OBIDMS column containing indices referring to indexed arrays, + * using the column pointer, in the context of a view. + * + * @param view A pointer on the opened view. + * @param column_p A pointer on the column. + * @param line_nb The number of the line where the value should be recovered. + * @param value_length_p A pointer on an int where the length of the value (number of elements in the array) will be stored. + * + * @returns The recovered value. + * @retval OBITuple_NA the NA value of the type if an error occurred and obi_errno is set. + * + * @since October 2017 + * @author Celine Mercier (celine.mercier@metabarcoding.org) + */ +const void* obi_get_array_with_col_p_in_view(Obiview_p view, OBIDMS_column_p column, index_t line_nb, int32_t* value_length_p); + + +/** + * @brief Sets a value in an OBIDMS column containing indices referring to indexed arrays, + * using the column name, in the context of a view. + * + * Note: If the column is read-only or if there is a line selection associated with the view (making columns non-writable), it is cloned. + * + * @param view A pointer on the opened writable view. + * @param column_name The name of the column. + * @param line_nb The number of the line where the value should be set. + * @param value The value that should be set. + * @param elt_size The size in bits of one element. + * @param value_length The length (number of elements) of the array to index. + * + * @returns An integer value indicating the success of the operation. + * @retval 0 on success. + * @retval -1 if an error occurred. + * + * @since October 2017 + * @author Celine Mercier (celine.mercier@metabarcoding.org) + */ +int obi_set_array_with_col_name_in_view(Obiview_p view, const char* column_name, index_t line_nb, const void* value, uint8_t elt_size, int32_t value_length); + + +/** + * @brief Recovers a value in an OBIDMS column containing indices referring to indexed arrays, + * using the column name, in the context of a view. + * + * @param view A pointer on the opened view. + * @param column_name The name of the column. + * @param line_nb The number of the line where the value should be recovered. + * @param value_length_p A pointer on an int where the length of the value (number of elements in the array) will be stored. + * + * @returns The recovered value. + * @retval OBITuple_NA the NA value of the type if an error occurred and obi_errno is set. + * + * @since October 2017 + * @author Celine Mercier (celine.mercier@metabarcoding.org) + */ +const void* obi_get_array_with_col_name_in_view(Obiview_p view, const char* column_name, index_t line_nb, int32_t* value_length_p); + + #endif /* OBIVIEW_H_ */