obitools/obitools3
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 66 Wiki Activity

Added tuple columns containing immutable indexed data arrays of any type

Browse Source
This commit is contained in:
Celine Mercier
2017-11-15 13:48:59 +01:00
parent 1684f96b79
commit 9a50803c00
32 changed files with 1097 additions and 284 deletions
Download Patch File Download Diff File Expand all files Collapse all files

105
python/obitools3/commands/test.pyx
View File

@ -1,11 +1,11 @@
#cython: language_level=3 #cython: language_level=3
from obitools3.apps.progress cimport ProgressBar # TODO I absolutely don't understand why it doesn't work without that line from obitools3.apps.progress cimport ProgressBar # TODO I absolutely don't understand why it doesn't work without that line
from obitools3.dms.view.view import View, Line_selection from obitools3.dms.view import View, Line_selection
from obitools3.dms.view.typed_view.view_NUC_SEQS import View_NUC_SEQS from obitools3.dms.view.typed_view.view_NUC_SEQS import View_NUC_SEQS
from obitools3.dms.dms import DMS from obitools3.dms import DMS
from obitools3.dms.column import Column from obitools3.dms.column import Column
from obitools3.dms.taxo.taxo import Taxonomy from obitools3.dms.taxo import Taxonomy
from obitools3.utils cimport str2bytes from obitools3.utils cimport str2bytes
from obitools3.dms.capi.obitypes cimport OBI_INT, \ from obitools3.dms.capi.obitypes cimport OBI_INT, \
OBI_FLOAT, \ OBI_FLOAT, \
@ -69,26 +69,68 @@ def random_bool(config):
return random.choice([True, False]) return random.choice([True, False])
def random_bool_tuples(config):
l=[]
for i in range(random.randint(1, config['test']['tuplemaxlen'])) :
l.append(random.choice([None, random_bool(config)]))
return tuple(l)
def random_char(config): def random_char(config):
return str2bytes(random.choice(string.ascii_lowercase)) return str2bytes(random.choice(string.ascii_lowercase))
def random_char_tuples(config):
l=[]
for i in range(random.randint(1, config['test']['tuplemaxlen'])) :
l.append(random.choice([None, random_char(config)]))
return tuple(l)
def random_float(config): def random_float(config):
return random.randint(0, MAX_INT) + random.random() return random.randint(0, MAX_INT) + random.random()
def random_float_tuples(config):
l=[]
for i in range(random.randint(1, config['test']['tuplemaxlen'])) :
l.append(random.choice([None, random_float(config)]))
return tuple(l)
def random_int(config): def random_int(config):
return random.randint(0, config['test']['maxlinenb']) return random.randint(0, config['test']['maxlinenb'])
def random_int_tuples(config):
l=[]
for i in range(random.randint(1, config['test']['tuplemaxlen'])) :
l.append(random.choice([None, random_int(config)]))
return tuple(l)
def random_seq(config): def random_seq(config):
return str2bytes(''.join(random.choice(['a','t','g','c']) for i in range(random_length(config['test']['seqmaxlen'])))) return str2bytes(''.join(random.choice(['a','t','g','c']) for i in range(random_length(config['test']['seqmaxlen']))))
def random_seq_tuples(config):
l=[]
for i in range(random.randint(1, config['test']['tuplemaxlen'])) :
l.append(random.choice([None, random_seq(config)]))
return tuple(l)
def random_bytes(config): def random_bytes(config):
return random_bytes_with_max_len(config['test']['strmaxlen']) return random_bytes_with_max_len(config['test']['strmaxlen'])
def random_bytes_tuples(config):
l=[]
for i in range(random.randint(1, config['test']['tuplemaxlen'])) :
l.append(random.choice([None, random_bytes(config)]))
return tuple(l)
def random_str_with_max_len(max_len): def random_str_with_max_len(max_len):
return ''.join(random.choice(string.ascii_lowercase) for i in range(random_length(max_len))) return ''.join(random.choice(string.ascii_lowercase) for i in range(random_length(max_len)))
@ -132,11 +174,28 @@ def test_set_and_get(config, infos):
print_test(config, "-") print_test(config, "-")
return return
idx = random_int(config) idx = random_int(config)
value = random.choice([None, infos['random_generator'][data_type](config)]) value = random.choice([None, infos['random_generator'][(data_type, col.tuples)](config)])
if col.nb_elements_per_line > 1 : if col.nb_elements_per_line > 1 :
elt = random.choice(element_names) elt = random.choice(element_names)
col[idx][elt] = value col[idx][elt] = value
assert col[idx][elt] == value, "Column: "+repr(col)+"\nSet value != gotten value "+str(value)+" != "+str(col[idx][elt]) assert col[idx][elt] == value, "Column: "+repr(col)+"\nSet value != gotten value "+str(value)+" != "+str(col[idx][elt])
elif col.tuples:
col[idx] = value
if value is None:
totest = None
else:
totest = []
for e in value:
if e is not None and e != '':
totest.append(e)
if len(totest) == 0:
totest = None
else:
totest = tuple(totest)
assert col[idx] == totest, "Column: "+repr(col)+"\nSet value != gotten value "+str(totest)+" != "+str(col[idx])
if totest is not None:
for i in range(len(totest)) :
assert col[idx][i] == totest[i], "Column: "+repr(col)+"\nSet value[i] != gotten value[i] "+str(totest[i])+" != "+str(col[idx][i])
else: else:
col[idx] = value col[idx] = value
assert col[idx] == value, "Column: "+repr(col)+"\nSet value != gotten value "+str(value)+" != "+str(col[idx]) assert col[idx] == value, "Column: "+repr(col)+"\nSet value != gotten value "+str(value)+" != "+str(col[idx])
@ -210,19 +269,25 @@ def fill_column(config, infos, col) :
if len(element_names) > 1 : if len(element_names) > 1 :
for i in range(random_int(config)) : for i in range(random_int(config)) :
for j in range(len(element_names)) : for j in range(len(element_names)) :
col[i][element_names[j]] = random.choice([None, infos['random_generator'][data_type](config)]) col[i][element_names[j]] = random.choice([None, infos['random_generator'][(data_type, col.tuples)](config)])
else : else :
for i in range(random_int(config)) : for i in range(random_int(config)) :
col[i] = random.choice([None, infos['random_generator'][data_type](config)]) r = random.choice([None, infos['random_generator'][(data_type, col.tuples)](config)])
col[i] = r
def create_random_column(config, infos) : def create_random_column(config, infos) :
alias = random.choice([b'', random_unique_name(infos)]) alias = random.choice([b'', random_unique_name(infos)])
nb_elements_per_line=random.randint(1, config['test']['maxelts']) tuples = random.choice([True, False])
elements_names = [] if not tuples :
for i in range(nb_elements_per_line) : nb_elements_per_line=random.randint(1, config['test']['maxelts'])
elements_names.append(random_unique_element_name(config, infos)) elements_names = []
elements_names = random.choice([None, elements_names]) for i in range(nb_elements_per_line) :
elements_names.append(random_unique_element_name(config, infos))
elements_names = random.choice([None, elements_names])
else :
nb_elements_per_line = 1
elements_names = None
name = random_unique_name(infos) name = random_unique_name(infos)
data_type = random_col_type() data_type = random_col_type()
@ -231,6 +296,7 @@ def create_random_column(config, infos) :
data_type, data_type,
nb_elements_per_line=nb_elements_per_line, nb_elements_per_line=nb_elements_per_line,
elements_names=elements_names, elements_names=elements_names,
tuples=tuples,
comments=random_str_with_max_len(COL_COMMENTS_MAX_LEN), comments=random_str_with_max_len(COL_COMMENTS_MAX_LEN),
alias=alias alias=alias
) )
@ -348,6 +414,14 @@ def addOptions(parser):
help="Maximum length of character strings. " help="Maximum length of character strings. "
"Default: 200") "Default: 200")
group.add_argument('--tuple_max_len','-u',
action="store", dest="test:tuplemaxlen",
metavar='<TUPLE_MAX_LEN>',
default=20,
type=int,
help="Maximum length of tuples. "
"Default: 200")
group.add_argument('--comments_max_len','-c', group.add_argument('--comments_max_len','-c',
action="store", dest="test:commentsmaxlen", action="store", dest="test:commentsmaxlen",
metavar='<COMMENTS_MAX_LEN>', metavar='<COMMENTS_MAX_LEN>',
@ -402,7 +476,14 @@ def run(config):
'view': None, 'view': None,
'view_names': None, 'view_names': None,
'unique_names': [], 'unique_names': [],
'random_generator': {b"OBI_BOOL": random_bool, b"OBI_CHAR": random_char, b"OBI_FLOAT": random_float, b"OBI_INT": random_int, b"OBI_SEQ": random_seq, b"OBI_STR": random_bytes}, 'random_generator': {
(b"OBI_BOOL", False): random_bool, (b"OBI_BOOL", True): random_bool_tuples,
(b"OBI_CHAR", False): random_char, (b"OBI_CHAR", True): random_char_tuples,
(b"OBI_FLOAT", False): random_float, (b"OBI_FLOAT", True): random_float_tuples,
(b"OBI_INT", False): random_int, (b"OBI_INT", True): random_int_tuples,
(b"OBI_SEQ", False): random_seq, (b"OBI_SEQ", True): random_seq_tuples,
(b"OBI_STR", False): random_bytes, (b"OBI_STR", True): random_bytes_tuples
},
'tests': [test_set_and_get, test_add_col, test_delete_col, test_col_alias, test_new_view] 'tests': [test_set_and_get, test_add_col, test_delete_col, test_col_alias, test_new_view]
} }

4
python/obitools3/dms/capi/obidmscolumn.pxd
View File

@ -31,6 +31,7 @@ cdef extern from "obidmscolumn.h" nogil:
const_char_p elements_names const_char_p elements_names
OBIType_t returned_data_type OBIType_t returned_data_type
OBIType_t stored_data_type OBIType_t stored_data_type
bint tuples
time_t creation_date time_t creation_date
obiversion_t version obiversion_t version
obiversion_t cloned_from obiversion_t cloned_from
@ -60,3 +61,6 @@ cdef extern from "obidmscolumn.h" nogil:
int obi_close_header(OBIDMS_column_header_p header) int obi_close_header(OBIDMS_column_header_p header)
char* obi_get_elements_names(OBIDMS_column_p column) char* obi_get_elements_names(OBIDMS_column_p column)
index_t obi_column_get_element_index_from_name(OBIDMS_column_p column, const char* element_name)

1
python/obitools3/dms/capi/obitypes.pxd
View File

@ -52,6 +52,7 @@ cdef extern from "obitypes.h" nogil:
extern const_char_p OBIStr_NA extern const_char_p OBIStr_NA
extern const_char_p OBIQual_char_NA extern const_char_p OBIQual_char_NA
extern uint8_t* OBIQual_int_NA extern uint8_t* OBIQual_int_NA
extern void* OBITuple_NA
const_char_p name_data_type(int data_type) const_char_p name_data_type(int data_type)

29
python/obitools3/dms/capi/obiview.pxd
View File

@ -14,7 +14,7 @@ from ..capi.obidmscolumn cimport OBIDMS_column_p, \
Column_reference_t, \ Column_reference_t, \
Column_reference_p Column_reference_p
from libc.stdint cimport uint8_t from libc.stdint cimport uint8_t, int32_t
cdef extern from "obiview.h" nogil: cdef extern from "obiview.h" nogil:
@ -86,6 +86,7 @@ cdef extern from "obiview.h" nogil:
index_t nb_lines, index_t nb_lines,
index_t nb_elements_per_line, index_t nb_elements_per_line,
char* elements_names, char* elements_names,
bint tuples,
const_char_p indexer_name, const_char_p indexer_name,
const_char_p associated_column_name, const_char_p associated_column_name,
obiversion_t associated_column_version, obiversion_t associated_column_version,
@ -327,3 +328,29 @@ cdef extern from "obiview.h" nogil:
OBIDMS_column_p column_p, OBIDMS_column_p column_p,
index_t line_nb, index_t line_nb,
index_t element_idx) index_t element_idx)
# ARRAY
int obi_set_array_with_col_p_in_view(Obiview_p view,
OBIDMS_column_p column,
index_t line_nb,
const void* value,
uint8_t elt_size,
int32_t value_length)
const void* obi_get_array_with_col_p_in_view(Obiview_p view,
OBIDMS_column_p column,
index_t line_nb,
int32_t* value_length_p)
int obi_set_array_with_col_name_in_view(Obiview_p view,
const char* column_name,
index_t line_nb,
const void* value,
uint8_t elt_size,
int32_t value_length)
const void* obi_get_array_with_col_name_in_view(Obiview_p view,
const char* column_name,
index_t line_nb,
int32_t* value_length_p)

3
python/obitools3/dms/column/column.pxd
View File

@ -22,7 +22,7 @@ cdef class Column(OBIWrapper) :
cdef inline OBIDMS_column_p pointer(self) cdef inline OBIDMS_column_p pointer(self)
@staticmethod @staticmethod
cdef type get_column_class(obitype_t obitype, bint multi_elts) cdef type get_column_class(obitype_t obitype, bint multi_elts, bint tuples)
@staticmethod @staticmethod
cdef type get_python_type(obitype_t obitype, bint multi_elts) cdef type get_python_type(obitype_t obitype, bint multi_elts)
@ -44,6 +44,7 @@ cdef class Column_line:
cdef register_column_class(obitype_t obitype, cdef register_column_class(obitype_t obitype,
bint multi_elts, bint multi_elts,
bint tuples,
type obiclass, type obiclass,
type python) type python)

21
python/obitools3/dms/column/column.pyx
View File

@ -1,7 +1,5 @@
#cython: language_level=3 #cython: language_level=3
from obitools3.dms.column import typed_column
__OBIDMS_COLUMN_CLASS__ = {} __OBIDMS_COLUMN_CLASS__ = {}
@ -44,12 +42,12 @@ cdef class Column(OBIWrapper) :
@staticmethod @staticmethod
cdef type get_column_class(obitype_t obitype, bint multi_elts): cdef type get_column_class(obitype_t obitype, bint multi_elts, bint tuples):
''' '''
Internal function returning the python class representing Internal function returning the python class representing
a column for a given obitype. a column for a given obitype.
''' '''
return __OBIDMS_COLUMN_CLASS__[(obitype, multi_elts)][0] return __OBIDMS_COLUMN_CLASS__[(obitype, multi_elts, tuples)][0]
@staticmethod @staticmethod
@ -76,6 +74,7 @@ cdef class Column(OBIWrapper) :
obitype_t data_type, obitype_t data_type,
index_t nb_elements_per_line=1, index_t nb_elements_per_line=1,
list elements_names=None, list elements_names=None,
bint tuples=False,
object comments=b"", object comments=b"",
object alias=b""): object alias=b""):
# TODO indexer_name? # TODO indexer_name?
@ -115,6 +114,7 @@ cdef class Column(OBIWrapper) :
nb_lines = len(view), nb_lines = len(view),
nb_elements_per_line = nb_elements_per_line, nb_elements_per_line = nb_elements_per_line,
elements_names = elements_names_p, elements_names = elements_names_p,
tuples = tuples,
indexer_name = NULL, indexer_name = NULL,
associated_column_name = NULL, associated_column_name = NULL,
associated_column_version = -1, associated_column_version = -1,
@ -150,7 +150,7 @@ cdef class Column(OBIWrapper) :
column_p = column_pp[0] column_p = column_pp[0]
column_type = column_p.header.returned_data_type column_type = column_p.header.returned_data_type
column_class = Column.get_column_class(column_type, (column_p.header.nb_elements_per_line > 1)) column_class = Column.get_column_class(column_type, (column_p.header.nb_elements_per_line > 1), column_p.header.tuples)
column = OBIWrapper.new_wrapper(column_class, column_pp) column = OBIWrapper.new_wrapper(column_class, column_pp)
column._view = view column._view = view
@ -184,6 +184,7 @@ cdef class Column(OBIWrapper) :
nb_lines = -1, nb_lines = -1,
nb_elements_per_line = -1, nb_elements_per_line = -1,
elements_names = NULL, elements_names = NULL,
tuples = False,
indexer_name = NULL, indexer_name = NULL,
associated_column_name = NULL, associated_column_name = NULL,
associated_column_version = -1, associated_column_version = -1,
@ -326,6 +327,13 @@ cdef class Column(OBIWrapper) :
raise OBIDeactivatedInstanceError() raise OBIDeactivatedInstanceError()
return self.pointer().header.lines_used return self.pointer().header.lines_used
# tuples property getter
@property
def tuples(self):
if not self.active() :
raise OBIDeactivatedInstanceError()
return self.pointer().header.tuples
# comments property getter # comments property getter
@property @property
def comments(self): def comments(self):
@ -436,6 +444,7 @@ cdef class Column_line :
cdef register_column_class(obitype_t obitype, cdef register_column_class(obitype_t obitype,
bint multi_elts, bint multi_elts,
bint tuples,
type obiclass, type obiclass,
type python_type): type python_type):
''' '''
@ -446,7 +455,7 @@ cdef register_column_class(obitype_t obitype,
assert issubclass(obiclass, Column) assert issubclass(obiclass, Column)
__OBIDMS_COLUMN_CLASS__[(obitype, multi_elts)] = (obiclass, python_type) __OBIDMS_COLUMN_CLASS__[(obitype, multi_elts, tuples)] = (obiclass, python_type)
cdef register_all_column_classes() : cdef register_all_column_classes() :

3
python/obitools3/dms/column/typed_column/bool.pxd
View File

@ -20,7 +20,10 @@ cdef class Column_multi_elts_bool(Column_multi_elts) :
cpdef set_item(self, index_t line_nb, object elt_id, object value) cpdef set_item(self, index_t line_nb, object elt_id, object value)
cdef class Column_tuples_bool(Column):
cpdef object get_line(self, index_t line_nb)
cpdef set_line(self, index_t line_nb, object value)

280
python/obitools3/dms/column/typed_column/bool.pyx
View File

@ -13,14 +13,23 @@ from ...capi.obiview cimport obi_get_bool_with_elt_name_and_col_p_in_view, \
obi_get_bool_with_elt_idx_and_col_p_in_view, \ obi_get_bool_with_elt_idx_and_col_p_in_view, \
obi_set_bool_with_elt_name_and_col_p_in_view, \ obi_set_bool_with_elt_name_and_col_p_in_view, \
obi_set_bool_with_elt_idx_and_col_p_in_view, \ obi_set_bool_with_elt_idx_and_col_p_in_view, \
obi_get_array_with_col_p_in_view, \
obi_set_array_with_col_p_in_view, \
Obiview_p Obiview_p
from ...capi.obidmscolumn cimport OBIDMS_column_p from ...capi.obidmscolumn cimport OBIDMS_column_p
from ...capi.obitypes cimport OBI_BOOL, OBIBool_NA, obibool_t from ...capi.obitypes cimport OBI_BOOL, \
OBIBool_NA, \
OBITuple_NA, \
obibool_t
from cpython.bool cimport PyBool_FromLong from cpython.bool cimport PyBool_FromLong
from libc.stdint cimport int32_t
from libc.stdlib cimport malloc, free
cdef class Column_bool(Column): cdef class Column_bool(Column):
@ -29,10 +38,12 @@ cdef class Column_bool(Column):
object column_name, object column_name,
index_t nb_elements_per_line=1, index_t nb_elements_per_line=1,
object elements_names=None, object elements_names=None,
bint tuples=False,
object comments=b""): object comments=b""):
return Column.new_column(view, column_name, OBI_BOOL, return Column.new_column(view, column_name, OBI_BOOL,
nb_elements_per_line=nb_elements_per_line, nb_elements_per_line=nb_elements_per_line,
elements_names=elements_names, elements_names=elements_names,
tuples=tuples,
comments=comments) comments=comments)
cpdef object get_line(self, index_t line_nb): cpdef object get_line(self, index_t line_nb):
@ -115,210 +126,67 @@ cdef class Column_multi_elts_bool(Column_multi_elts):
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=elt_id, error_message="Problem setting a value in a column") obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=elt_id, error_message="Problem setting a value in a column")
cdef class Column_tuples_bool(Column):
cpdef object get_line(self, index_t line_nb) :
global obi_errno
cdef obibool_t value
cdef bint value_in_result
cdef object result
cdef int32_t i
cdef obibool_t* array
cdef int32_t value_length
result = []
array = <obibool_t*>obi_get_array_with_col_p_in_view(self._view.pointer(), self.pointer(), line_nb, &value_length)
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem getting a value from a column")
if array == OBITuple_NA :
return None
for i in range(value_length) :
value = array[i]
value_in_result = PyBool_FromLong(value)
result.append(value_in_result)
return tuple(result)
cpdef set_line(self, index_t line_nb, object value) :
global obi_errno
cdef obibool_t* array
cdef int32_t value_length
cdef int32_t i, j
cdef object e
value_length = 0
if value is not None:
for e in value:
if e is not None:
value_length+=1
if value is None or value_length == 0 :
array = <obibool_t*>OBITuple_NA
else:
array = <obibool_t*>malloc(value_length * sizeof(obibool_t))
if array == NULL:
raise Exception("Problem allocating memory for an array to store a tuple")
#raise RollbackException("Problem allocating memory for an array to store a tuple", self._view) # TODO can't import
j=0
for i in range(len(value)) :
if value[i] is not None:
array[j] = <obibool_t>(value[i])
j+=1
if obi_set_array_with_col_p_in_view(self._view.pointer(), self.pointer(), line_nb, <obibool_t*> array, sizeof(obibool_t)*8, value_length) < 0 :
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem setting a value in a column")
if array != <obibool_t*>OBITuple_NA:
free(array)
def register_class() : def register_class() :
register_column_class(OBI_BOOL, False, Column_bool, bool) register_column_class(OBI_BOOL, False, False, Column_bool, bool)
register_column_class(OBI_BOOL, True, Column_multi_elts_bool, bool) register_column_class(OBI_BOOL, True, False, Column_multi_elts_bool, bool)
register_column_class(OBI_BOOL, False, True, Column_tuples_bool, bool)
# cdef class Column_line_bool(Column_line) :
#
# cdef update_pointer(self):
# """
# Checks if the obicolumn address changed since the last call and update
# if need the `_column_p` and `_data_view` data structure fields.
# """
# cdef OBIDMS_column_p* column_pp
# column_pp = <OBIDMS_column_p*>self._pointer
# cdef OBIDMS_column_p column_p = column_pp[0]
#
# if column_p != self._column_p:
# self._column_p = column_p
# self._data_view = (<obibool_t*> (column_p.data)) + \
# self._index * column_p.header.nb_elements_per_line
#
# @staticmethod
# cdef bool obibool_t2bool(obibool_t value):
# cdef bool result
#
# if value == OBIBool_NA :
# result = None
# else :
# result = PyBool_FromLong(value)
#
# return result
#
# @staticmethod
# cdef bool2obibool_t(bool value):
# cdef obibool_t result
#
# if value is None:
# result=OBIBool_NA
# else:
# result= <obibool_t> <int> value
#
# return result
#
#
# def __init__(self, Column column, index_t line_nb) :
# """
# Creates a new `OBIDMS_column_line_bool`
#
# @param column: an OBIDMS_column instance
# @param line_nb: the line in the column
# """
#
# Column_line.__init__(self, column, line_nb)
# self.update_pointer()
#
#
#
# cpdef bool get_bool_item_by_name(self, bytes element_name) :
# """
# Returns the value associated to the name `element_name` of the current line
#
# @param element_name: a `bytes` instance containing the name of the element
#
# @return: the `bool` value corresponding to the name
# """
# cdef char* cname = element_name
# cdef obibool_t value
# global obi_errno
#
# self.update_pointer()
#
# cdef OBIDMS_column_p* column_pp
# column_pp = <OBIDMS_column_p*>self._pointer
# cdef OBIDMS_column_p column_p = column_pp[0]
#
# value = obi_column_get_obibool_with_elt_name(column_p,
# self._index,
# cname)
#
# if obi_errno > 0 :
# obi_errno = 0
# raise KeyError("Cannot access to key %s" % bytes2str(element_name))
#
# return Column_line_bool.obibool_t2bool(value)
#
#
# cpdef bool get_bool_item_by_idx(self,index_t index):
# """
# Returns the value associated to the name `element_name` of the current line
#
# @param index: a `int` instance containing the index of the element
#
# @return: the `bool` value corresponding to the name
# """
# cdef obibool_t value # @DuplicatedSignature
# global obi_errno
#
# cdef OBIDMS_column_p* column_pp
# column_pp = <OBIDMS_column_p*>self._pointer
# cdef OBIDMS_column_p column_p = column_pp[0]
#
# self.update_pointer()
#
# value = obi_column_get_obibool_with_elt_idx(column_p,
# self._index,
# index)
#
# if obi_errno > 0 :
# obi_errno = 0
# raise IndexError("Cannot access to element %d" % index)
#
# return Column_line_bool.obibool_t2bool(value)
#
#
# def __getitem__(self, object element_name) :
# cdef bytes name
# cdef int cindex
# cdef obibool_t value
# cdef type typearg = type(element_name)
# cdef bool result
#
#
# if typearg == int:
# cindex=element_name
# if cindex < 0:
# cindex = self._len - cindex
# result=self.get_bool_item_by_idx(cindex)
# elif typearg == bytes:
# result=self.get_bool_item_by_name(element_name)
# elif typearg == str:
# name = str2bytes(element_name)
# result=self.get_bool_item_by_name(name)
#
# return result
#
# cpdef set_bool_item_by_name(self,bytes element_name,bool value):
# """
# Sets the value associated to the name `element_name` of the current line
#
# @param element_name: a `bytes` instance containing the name of the element
# @param value: a `bool` instance of the new value
#
# @return: the `bool` value corresponding to the name
# """
# cdef char* cname = element_name
# cdef obibool_t cvalue
#
# self.update_pointer()
# cvalue = OBIDMS_column_line_bool.bool2obibool_t(value)
#
# if ( obi_column_set_obibool_with_elt_name((<OBIDMS_column_p*>self._pointer)[0],
# self._index,
# cname,
# cvalue) < 0 ):
# raise KeyError("Cannot access to key %s" % bytes2str(element_name))
#
# cpdef set_bool_item_by_idx(self,index_t index,bool value):
# """
# Sets the value associated to the name `element_name` of the current line
#
# @param index: a `int` instance containing the index of the element
# @param value: a `bool` instance of the new value
#
# @return: the `bool` value corresponding to the name
# """
# cdef obibool_t cvalue # @DuplicatedSignature
#
# self.update_pointer()
# cvalue = OBIDMS_column_line_bool.bool2obibool_t(value)
#
# if ( obi_column_set_obibool_with_elt_idx((<OBIDMS_column_p*>self._pointer)[0],
# self._index,
# index,
# cvalue) < 0 ):
# raise IndexError("Cannot access to item index %d" % index)
#
#
#
# def __setitem__(self, object element_name, object value):
# cdef bytes name
# cdef int cindex
# cdef type typearg = type(element_name)
# cdef bool result
#
#
# if typearg == int:
# cindex=element_name
# if cindex < 0:
# cindex = self._len - cindex
# self.set_bool_item_by_idx(cindex,value)
# elif typearg == bytes:
# self.set_bool_item_by_name(element_name,value)
# elif typearg == str:
# name = str2bytes(element_name)
# self.set_bool_item_by_name(name,value)
#
# def __repr__(self) :
# return str(self._column.get_line(self._index))
#
# def __len__(self):
# return self._len

6
python/obitools3/dms/column/typed_column/char.pxd
View File

@ -18,3 +18,9 @@ cdef class Column_multi_elts_char(Column_multi_elts) :
cpdef object get_item(self, index_t line_nb, object elt_id) cpdef object get_item(self, index_t line_nb, object elt_id)
cpdef object get_line(self, index_t line_nb) cpdef object get_line(self, index_t line_nb)
cpdef set_item(self, index_t line_nb, object elt_id, object value) cpdef set_item(self, index_t line_nb, object elt_id, object value)
cdef class Column_tuples_char(Column):
cpdef object get_line(self, index_t line_nb)
cpdef set_line(self, index_t line_nb, object value)

82
python/obitools3/dms/column/typed_column/char.pyx
View File

@ -13,12 +13,20 @@ from ...capi.obiview cimport obi_get_char_with_elt_name_and_col_p_in_view, \
obi_get_char_with_elt_idx_and_col_p_in_view, \ obi_get_char_with_elt_idx_and_col_p_in_view, \
obi_set_char_with_elt_name_and_col_p_in_view, \ obi_set_char_with_elt_name_and_col_p_in_view, \
obi_set_char_with_elt_idx_and_col_p_in_view, \ obi_set_char_with_elt_idx_and_col_p_in_view, \
obi_get_array_with_col_p_in_view, \
obi_set_array_with_col_p_in_view, \
Obiview_p Obiview_p
from ...capi.obidmscolumn cimport OBIDMS_column_p from ...capi.obidmscolumn cimport OBIDMS_column_p
from ...capi.obitypes cimport OBI_CHAR, OBIChar_NA, obichar_t from ...capi.obitypes cimport OBI_CHAR, \
OBIChar_NA, \
OBITuple_NA, \
obichar_t
from libc.stdint cimport int32_t
from libc.stdlib cimport malloc, free
cdef class Column_char(Column): cdef class Column_char(Column):
@ -28,11 +36,13 @@ cdef class Column_char(Column):
object column_name, object column_name,
index_t nb_elements_per_line=1, index_t nb_elements_per_line=1,
object elements_names=None, object elements_names=None,
bint tuples=False,
object comments=b""): object comments=b""):
return Column.new_column(view, column_name, OBI_CHAR, return Column.new_column(view, column_name, OBI_CHAR,
nb_elements_per_line=nb_elements_per_line, nb_elements_per_line=nb_elements_per_line,
elements_names=elements_names, elements_names=elements_names,
tuples=tuples,
comments=comments) comments=comments)
cpdef object get_line(self, index_t line_nb): cpdef object get_line(self, index_t line_nb):
@ -106,7 +116,7 @@ cdef class Column_multi_elts_char(Column_multi_elts):
cpdef set_item(self, index_t line_nb, object elt_id, object value) : cpdef set_item(self, index_t line_nb, object elt_id, object value) :
global obi_errno global obi_errno
cdef bytes elt_name cdef bytes elt_name
cdef obichar_t value_b cdef obichar_t value_b
if value is None : if value is None :
value_b = OBIChar_NA value_b = OBIChar_NA
@ -121,7 +131,69 @@ cdef class Column_multi_elts_char(Column_multi_elts):
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=elt_id, error_message="Problem setting a value in a column") obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=elt_id, error_message="Problem setting a value in a column")
def register_class(): cdef class Column_tuples_char(Column):
register_column_class(OBI_CHAR, False, Column_char, bytes) # TODO bytes or str?
register_column_class(OBI_CHAR, True, Column_multi_elts_char, bytes) # TODO bytes or str? cpdef object get_line(self, index_t line_nb) :
global obi_errno
cdef obichar_t value
cdef bytes value_in_result
cdef object result
cdef int32_t i
cdef obichar_t* array
cdef int32_t value_length
result = []
array = <obichar_t*>obi_get_array_with_col_p_in_view(self._view.pointer(), self.pointer(), line_nb, &value_length)
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem getting a value from a column")
if array == OBITuple_NA :
return None
for i in range(value_length) :
value = array[i]
value_in_result = <bytes> value
result.append(value_in_result)
return tuple(result)
cpdef set_line(self, index_t line_nb, object value) :
global obi_errno
cdef obichar_t* array
cdef int32_t value_length
cdef int32_t i, j
cdef object e
cdef obichar_t value_b
value_length = 0
if value is not None:
for e in value:
if e is not None:
value_length+=1
if value is None or value_length == 0 :
array = <obichar_t*>OBITuple_NA
else:
array = <obichar_t*>malloc(value_length * sizeof(obichar_t))
if array == NULL:
raise Exception("Problem allocating memory for an array to store a tuple")
#raise RollbackException("Problem allocating memory for an array to store a tuple", self._view) # TODO can't import
j=0
for i in range(len(value)) :
if value[i] is not None:
value_b = <obichar_t> tobytes(value[i])[0]
array[j] = value_b
j+=1
if obi_set_array_with_col_p_in_view(self._view.pointer(), self.pointer(), line_nb, <obichar_t*> array, sizeof(obichar_t)*8, value_length) < 0 :
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem setting a value in a column")
if array != <obichar_t*>OBITuple_NA:
free(array)
def register_class():
register_column_class(OBI_CHAR, False, False, Column_char, bytes)
register_column_class(OBI_CHAR, True, False, Column_multi_elts_char, bytes)
register_column_class(OBI_CHAR, False, True, Column_tuples_char, bytes)

5
python/obitools3/dms/column/typed_column/float.pxd
View File

@ -19,3 +19,8 @@ cdef class Column_multi_elts_float(Column_multi_elts) :
cpdef object get_line(self, index_t line_nb) cpdef object get_line(self, index_t line_nb)
cpdef set_item(self, index_t line_nb, object elt_id, object value) cpdef set_item(self, index_t line_nb, object elt_id, object value)
cdef class Column_tuples_float(Column):
cpdef object get_line(self, index_t line_nb)
cpdef set_line(self, index_t line_nb, object value)

78
python/obitools3/dms/column/typed_column/float.pyx
View File

@ -13,12 +13,20 @@ from ...capi.obiview cimport obi_get_float_with_elt_name_and_col_p_in_view, \
obi_get_float_with_elt_idx_and_col_p_in_view, \ obi_get_float_with_elt_idx_and_col_p_in_view, \
obi_set_float_with_elt_name_and_col_p_in_view, \ obi_set_float_with_elt_name_and_col_p_in_view, \
obi_set_float_with_elt_idx_and_col_p_in_view, \ obi_set_float_with_elt_idx_and_col_p_in_view, \
obi_get_array_with_col_p_in_view, \
obi_set_array_with_col_p_in_view, \
Obiview_p Obiview_p
from ...capi.obidmscolumn cimport OBIDMS_column_p from ...capi.obidmscolumn cimport OBIDMS_column_p
from ...capi.obitypes cimport OBI_FLOAT, OBIFloat_NA, obifloat_t from ...capi.obitypes cimport OBI_FLOAT, \
OBIFloat_NA, \
OBITuple_NA, \
obifloat_t
from libc.stdint cimport int32_t
from libc.stdlib cimport malloc, free
cdef class Column_float(Column): cdef class Column_float(Column):
@ -28,11 +36,13 @@ cdef class Column_float(Column):
object column_name, object column_name,
index_t nb_elements_per_line=1, index_t nb_elements_per_line=1,
object elements_names=None, object elements_names=None,
bint tuples=False,
object comments=b""): object comments=b""):
return Column.new_column(view, column_name, OBI_FLOAT, return Column.new_column(view, column_name, OBI_FLOAT,
nb_elements_per_line=nb_elements_per_line, nb_elements_per_line=nb_elements_per_line,
elements_names=elements_names, elements_names=elements_names,
tuples=tuples,
comments=comments) comments=comments)
cpdef object get_line(self, index_t line_nb): cpdef object get_line(self, index_t line_nb):
@ -115,7 +125,67 @@ cdef class Column_multi_elts_float(Column_multi_elts):
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=elt_id, error_message="Problem setting a value in a column") obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=elt_id, error_message="Problem setting a value in a column")
def register_class(): cdef class Column_tuples_float(Column):
register_column_class(OBI_FLOAT, False, Column_float, float) # TODO why not double?
register_column_class(OBI_FLOAT, True, Column_multi_elts_float, float) # TODO why not double? cpdef object get_line(self, index_t line_nb) :
global obi_errno
cdef obifloat_t value
cdef double value_in_result
cdef object result
cdef int32_t i
cdef obifloat_t* array
cdef int32_t value_length
result = []
array = <obifloat_t*>obi_get_array_with_col_p_in_view(self._view.pointer(), self.pointer(), line_nb, &value_length)
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem getting a value from a column")
if array == OBITuple_NA :
return None
for i in range(value_length) :
value = array[i]
value_in_result = <double> value
result.append(value_in_result)
return tuple(result)
cpdef set_line(self, index_t line_nb, object value) :
global obi_errno
cdef obifloat_t* array
cdef int32_t value_length
cdef int32_t i, j
cdef object e
value_length = 0
if value is not None:
for e in value:
if e is not None:
value_length+=1
if value is None or value_length == 0 :
array = <obifloat_t*>OBITuple_NA
else:
array = <obifloat_t*>malloc(value_length * sizeof(obifloat_t))
if array == NULL:
raise Exception("Problem allocating memory for an array to store a tuple")
#raise RollbackException("Problem allocating memory for an array to store a tuple", self._view) # TODO can't import
j=0
for i in range(len(value)) :
if value[i] is not None:
array[j] = <obifloat_t>(value[i])
j+=1
if obi_set_array_with_col_p_in_view(self._view.pointer(), self.pointer(), line_nb, <obifloat_t*> array, sizeof(obifloat_t)*8, value_length) < 0 :
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem setting a value in a column")
if array != <obifloat_t*>OBITuple_NA:
free(array)
def register_class():
register_column_class(OBI_FLOAT, False, False, Column_float, float) # TODO why not double?
register_column_class(OBI_FLOAT, True, False, Column_multi_elts_float, float) # TODO why not double?
register_column_class(OBI_FLOAT, False, True, Column_tuples_float, float)

5
python/obitools3/dms/column/typed_column/int.pxd
View File

@ -19,3 +19,8 @@ cdef class Column_multi_elts_int(Column_multi_elts) :
cpdef object get_line(self, index_t line_nb) cpdef object get_line(self, index_t line_nb)
cpdef set_item(self, index_t line_nb, object elt_id, object value) cpdef set_item(self, index_t line_nb, object elt_id, object value)
cdef class Column_tuples_int(Column):
cpdef object get_line(self, index_t line_nb)
cpdef set_line(self, index_t line_nb, object value)

79
python/obitools3/dms/column/typed_column/int.pyx
View File

@ -13,14 +13,23 @@ from ...capi.obiview cimport obi_get_int_with_elt_name_and_col_p_in_view, \
obi_get_int_with_elt_idx_and_col_p_in_view, \ obi_get_int_with_elt_idx_and_col_p_in_view, \
obi_set_int_with_elt_name_and_col_p_in_view, \ obi_set_int_with_elt_name_and_col_p_in_view, \
obi_set_int_with_elt_idx_and_col_p_in_view, \ obi_set_int_with_elt_idx_and_col_p_in_view, \
obi_get_array_with_col_p_in_view, \
obi_set_array_with_col_p_in_view, \
Obiview_p Obiview_p
from ...capi.obidmscolumn cimport OBIDMS_column_p from ...capi.obidmscolumn cimport OBIDMS_column_p
from ...capi.obitypes cimport OBI_INT, OBIInt_NA, obiint_t from ...capi.obitypes cimport OBI_INT, \
OBIInt_NA, \
OBITuple_NA, \
obiint_t
from cpython.int cimport PyInt_FromLong from cpython.int cimport PyInt_FromLong
from libc.stdint cimport int32_t
from libc.stdlib cimport malloc, free
cdef class Column_int(Column): cdef class Column_int(Column):
@ -29,11 +38,13 @@ cdef class Column_int(Column):
object column_name, object column_name,
index_t nb_elements_per_line=1, index_t nb_elements_per_line=1,
object elements_names=None, object elements_names=None,
bint tuples=False,
object comments=b""): object comments=b""):
return Column.new_column(view, column_name, OBI_INT, return Column.new_column(view, column_name, OBI_INT,
nb_elements_per_line=nb_elements_per_line, nb_elements_per_line=nb_elements_per_line,
elements_names=elements_names, elements_names=elements_names,
tuples=tuples,
comments=comments) comments=comments)
@ -118,7 +129,67 @@ cdef class Column_multi_elts_int(Column_multi_elts):
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=elt_id, error_message="Problem setting a value in a column") obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=elt_id, error_message="Problem setting a value in a column")
def register_class(): cdef class Column_tuples_int(Column):
register_column_class(OBI_INT, False, Column_int, int)
register_column_class(OBI_INT, True, Column_multi_elts_int, int) cpdef object get_line(self, index_t line_nb) :
global obi_errno
cdef obiint_t value
cdef double value_in_result
cdef object result
cdef int32_t i
cdef obiint_t* array
cdef int32_t value_length
result = []
array = <obiint_t*>obi_get_array_with_col_p_in_view(self._view.pointer(), self.pointer(), line_nb, &value_length)
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem getting a value from a column")
if array == OBITuple_NA :
return None
for i in range(value_length) :
value = array[i]
value_in_result = PyInt_FromLong(value)
result.append(value_in_result)
return tuple(result)
cpdef set_line(self, index_t line_nb, object value) :
global obi_errno
cdef obiint_t* array
cdef int32_t value_length
cdef int32_t i, j
cdef object e
value_length = 0
if value is not None:
for e in value:
if e is not None:
value_length+=1
if value is None or value_length == 0 :
array = <obiint_t*>OBITuple_NA
else:
array = <obiint_t*>malloc(value_length * sizeof(obiint_t))
if array == NULL:
raise Exception("Problem allocating memory for an array to store a tuple")
#raise RollbackException("Problem allocating memory for an array to store a tuple", self._view) # TODO can't import
j=0
for i in range(len(value)) :
if value[i] is not None:
array[j] = <obiint_t>(value[i])
j+=1
if obi_set_array_with_col_p_in_view(self._view.pointer(), self.pointer(), line_nb, <obiint_t*> array, sizeof(obiint_t)*8, value_length) < 0 :
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem setting a value in a column")
if array != <obiint_t*>OBITuple_NA:
free(array)
def register_class():
register_column_class(OBI_INT, False, False, Column_int, int)
register_column_class(OBI_INT, True, False, Column_multi_elts_int, int)
register_column_class(OBI_INT, False, True, Column_tuples_int, int)

9
python/obitools3/dms/column/typed_column/qual.pyx
View File

@ -25,9 +25,8 @@ from ...capi.obidmscolumn cimport OBIDMS_column_p
from ...capi.obitypes cimport OBI_QUAL, OBIQual_char_NA, OBIQual_int_NA, const_char_p from ...capi.obitypes cimport OBI_QUAL, OBIQual_char_NA, OBIQual_int_NA, const_char_p
from libc.stdlib cimport free from libc.stdlib cimport malloc, free
from libc.stdint cimport uint8_t from libc.stdint cimport uint8_t
from libc.stdlib cimport malloc
# TODO detect type of value and call set_item_str if str or bytes # TODO detect type of value and call set_item_str if str or bytes
@ -44,6 +43,7 @@ cdef class Column_qual(Column_idx):
return Column.new_column(view, column_name, OBI_QUAL, return Column.new_column(view, column_name, OBI_QUAL,
nb_elements_per_line=nb_elements_per_line, nb_elements_per_line=nb_elements_per_line,
elements_names=elements_names, elements_names=elements_names,
tuples=False,
comments=comments) comments=comments)
@ -255,8 +255,7 @@ cdef class Column_multi_elts_qual(Column_multi_elts_idx):
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=elt_id, error_message="Problem setting a value in a column") obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=elt_id, error_message="Problem setting a value in a column")
def register_class(): def register_class():
register_column_class(OBI_QUAL, False, Column_qual, bytes) # TODO str? int? register_column_class(OBI_QUAL, False, False, Column_qual, int) # TODO bytes?
register_column_class(OBI_QUAL, True, Column_multi_elts_qual, bytes) # TODO str? int? register_column_class(OBI_QUAL, True, False, Column_multi_elts_qual, int) # bytes?

5
python/obitools3/dms/column/typed_column/seq.pxd
View File

@ -19,3 +19,8 @@ cdef class Column_multi_elts_seq(Column_multi_elts_idx) :
cpdef object get_line(self, index_t line_nb) cpdef object get_line(self, index_t line_nb)
cpdef set_item(self, index_t line_nb, object elt_id, object value) cpdef set_item(self, index_t line_nb, object elt_id, object value)
cdef class Column_tuples_seq(Column_idx):
cpdef object get_line(self, index_t line_nb)
cpdef set_line(self, index_t line_nb, object value)

89
python/obitools3/dms/column/typed_column/seq.pyx
View File

@ -15,13 +15,21 @@ from ...capi.obiview cimport obi_get_seq_with_elt_name_and_col_p_in_view, \
obi_get_seq_with_elt_idx_and_col_p_in_view, \ obi_get_seq_with_elt_idx_and_col_p_in_view, \
obi_set_seq_with_elt_name_and_col_p_in_view, \ obi_set_seq_with_elt_name_and_col_p_in_view, \
obi_set_seq_with_elt_idx_and_col_p_in_view, \ obi_set_seq_with_elt_idx_and_col_p_in_view, \
obi_get_array_with_col_p_in_view, \
obi_set_array_with_col_p_in_view, \
obi_get_array_with_col_p_in_view, \
obi_set_array_with_col_p_in_view, \
Obiview_p Obiview_p
from ...capi.obidmscolumn cimport OBIDMS_column_p from ...capi.obidmscolumn cimport OBIDMS_column_p
from ...capi.obitypes cimport OBI_SEQ, OBISeq_NA from ...capi.obitypes cimport OBI_SEQ, \
OBISeq_NA, \
OBITuple_NA
from libc.stdlib cimport free from libc.stdint cimport int32_t
from libc.stdlib cimport calloc, free
from libc.string cimport strcpy
cdef class Column_seq(Column_idx): cdef class Column_seq(Column_idx):
@ -31,11 +39,13 @@ cdef class Column_seq(Column_idx):
object column_name, object column_name,
index_t nb_elements_per_line=1, index_t nb_elements_per_line=1,
object elements_names=None, object elements_names=None,
bint tuples=False,
object comments=b""): object comments=b""):
return Column.new_column(view, column_name, OBI_SEQ, return Column.new_column(view, column_name, OBI_SEQ,
nb_elements_per_line=nb_elements_per_line, nb_elements_per_line=nb_elements_per_line,
elements_names=elements_names, elements_names=elements_names,
tuples=tuples,
comments=comments) comments=comments)
@ -145,7 +155,76 @@ cdef class Column_multi_elts_seq(Column_multi_elts_idx):
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=elt_id, error_message="Problem setting a value in a column") obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=elt_id, error_message="Problem setting a value in a column")
def register_class(): cdef class Column_tuples_seq(Column_idx):
register_column_class(OBI_SEQ, False, Column_seq, bytes) # TODO str?
register_column_class(OBI_SEQ, True, Column_multi_elts_seq, bytes) # TODO str? cpdef object get_line(self, index_t line_nb) :
global obi_errno
cdef const char* value
cdef bytes value_in_result
cdef object result
cdef int32_t i
cdef const char* array
cdef int32_t value_length
result = []
array = <const char*>obi_get_array_with_col_p_in_view(self._view.pointer(), self.pointer(), line_nb, &value_length)
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem getting a value from a column")
if array == OBITuple_NA :
return None
i = 0
# First value
value_in_result = <bytes> array
result.append(value_in_result)
while i+1 < value_length :
if array[i] != b'\0' :
i+=1
else :
value = array+i+1
value_in_result = <bytes> value
result.append(value_in_result)
i+=1
return tuple(result)
cpdef set_line(self, index_t line_nb, object value) :
global obi_errno
cdef char* array
cdef int32_t value_length
cdef int32_t i
cdef object elt
cdef bytes elt_b
value_length = 0
if value is not None:
for i in range(len(value)) :
if value[i] is not None and value[i] != '' :
value_length = value_length + len(value[i]) + 1 # Total size of the array with the '\0'
if value is None or value_length == 0 :
array = <char*>OBITuple_NA
else:
array = <char*>calloc(value_length, sizeof(char))
if array == NULL:
raise Exception("Problem allocating memory for an array to store a tuple")
#raise RollbackException("Problem allocating memory for an array to store a tuple", self._view) # TODO can't import
i = 0
for elt in value :
if elt is not None and elt != '':
elt_b = tobytes(elt)
strcpy(array+i, <char*>elt_b)
i = i + len(elt_b) + 1
if obi_set_array_with_col_p_in_view(self._view.pointer(), self.pointer(), line_nb, <char*> array, sizeof(char)*8, value_length) < 0 :
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem setting a value in a column")
if array != <char*>OBITuple_NA:
free(array)
def register_class():
register_column_class(OBI_SEQ, False, False, Column_seq, bytes)
register_column_class(OBI_SEQ, True, False, Column_multi_elts_seq, bytes)
register_column_class(OBI_SEQ, False, True, Column_tuples_seq, bytes)

5
python/obitools3/dms/column/typed_column/str.pxd
View File

@ -19,3 +19,8 @@ cdef class Column_multi_elts_str(Column_multi_elts_idx) :
cpdef object get_line(self, index_t line_nb) cpdef object get_line(self, index_t line_nb)
cpdef set_item(self, index_t line_nb, object elt_id, object value) cpdef set_item(self, index_t line_nb, object elt_id, object value)
cdef class Column_tuples_str(Column_idx):
cpdef object get_line(self, index_t line_nb)
cpdef set_line(self, index_t line_nb, object value)

88
python/obitools3/dms/column/typed_column/str.pyx
View File

@ -15,11 +15,20 @@ from ...capi.obiview cimport obi_get_str_with_elt_name_and_col_p_in_view, \
obi_get_str_with_elt_idx_and_col_p_in_view, \ obi_get_str_with_elt_idx_and_col_p_in_view, \
obi_set_str_with_elt_name_and_col_p_in_view, \ obi_set_str_with_elt_name_and_col_p_in_view, \
obi_set_str_with_elt_idx_and_col_p_in_view, \ obi_set_str_with_elt_idx_and_col_p_in_view, \
obi_get_array_with_col_p_in_view, \
obi_set_array_with_col_p_in_view, \
Obiview_p Obiview_p
from ...capi.obidmscolumn cimport OBIDMS_column_p from ...capi.obidmscolumn cimport OBIDMS_column_p
from ...capi.obitypes cimport OBI_STR, OBIStr_NA, const_char_p from ...capi.obitypes cimport OBI_STR, \
OBIStr_NA, \
OBITuple_NA, \
const_char_p
from libc.stdint cimport int32_t
from libc.stdlib cimport calloc, free
from libc.string cimport strcpy
cdef class Column_str(Column_idx): cdef class Column_str(Column_idx):
@ -29,11 +38,13 @@ cdef class Column_str(Column_idx):
object column_name, object column_name,
index_t nb_elements_per_line=1, index_t nb_elements_per_line=1,
object elements_names=None, object elements_names=None,
bint tuples=False,
object comments=b""): object comments=b""):
return Column.new_column(view, column_name, OBI_STR, return Column.new_column(view, column_name, OBI_STR,
nb_elements_per_line=nb_elements_per_line, nb_elements_per_line=nb_elements_per_line,
elements_names=elements_names, elements_names=elements_names,
tuples=tuples,
comments=comments) comments=comments)
@ -133,7 +144,76 @@ cdef class Column_multi_elts_str(Column_multi_elts_idx):
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=elt_id, error_message="Problem setting a value in a column") obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=elt_id, error_message="Problem setting a value in a column")
def register_class(): cdef class Column_tuples_str(Column_idx):
register_column_class(OBI_STR, False, Column_str, bytes) # TODO str?
register_column_class(OBI_STR, True, Column_multi_elts_str, bytes) # TODO str? cpdef object get_line(self, index_t line_nb) :
global obi_errno
cdef const char* value
cdef bytes value_in_result
cdef object result
cdef int32_t i
cdef const char* array
cdef int32_t value_length
result = []
array = <const char*>obi_get_array_with_col_p_in_view(self._view.pointer(), self.pointer(), line_nb, &value_length)
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem getting a value from a column")
if array == OBITuple_NA :
return None
i = 0
# First value
value_in_result = <bytes> array
result.append(value_in_result)
while i+1 < value_length :
if array[i] != b'\0' :
i+=1
else :
value = array+i+1
value_in_result = <bytes> value
result.append(value_in_result)
i+=1
return tuple(result)
cpdef set_line(self, index_t line_nb, object value) :
global obi_errno
cdef char* array
cdef int32_t value_length
cdef int32_t i
cdef object elt
cdef bytes elt_b
value_length = 0
if value is not None:
for i in range(len(value)) :
if value[i] is not None and value[i] != '' :
value_length = value_length + len(value[i]) + 1 # Total size of the array with the '\0'
if value is None or value_length == 0 :
array = <char*>OBITuple_NA
else:
array = <char*>calloc(value_length, sizeof(char))
if array == NULL:
raise Exception("Problem allocating memory for an array to store a tuple")
#raise RollbackException("Problem allocating memory for an array to store a tuple", self._view) # TODO can't import
i = 0
for elt in value :
if elt is not None and elt != '':
elt_b = tobytes(elt)
strcpy(array+i, <char*>elt_b)
i = i + len(elt_b) + 1
if obi_set_array_with_col_p_in_view(self._view.pointer(), self.pointer(), line_nb, <char*> array, sizeof(char)*8, value_length) < 0 :
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem setting a value in a column")
if array != <char*>OBITuple_NA:
free(array)
def register_class():
register_column_class(OBI_STR, False, False, Column_str, bytes)
register_column_class(OBI_STR, True, False, Column_multi_elts_str, bytes)
register_column_class(OBI_STR, False, True, Column_tuples_str, bytes)

2
python/obitools3/dms/dms.cfiles
View File

@ -1,3 +1,4 @@
../../../src/array_indexer.c
../../../src/bloom.c ../../../src/bloom.c
../../../src/char_str_indexer.c ../../../src/char_str_indexer.c
../../../src/crc64.c ../../../src/crc64.c
@ -12,6 +13,7 @@
../../../src/obiblob.c ../../../src/obiblob.c
../../../src/obidms_taxonomy.c ../../../src/obidms_taxonomy.c
../../../src/obidms.c ../../../src/obidms.c
../../../src/obidmscolumn_array.c
../../../src/obidmscolumn_blob.c ../../../src/obidmscolumn_blob.c
../../../src/obidmscolumn_bool.c ../../../src/obidmscolumn_bool.c
../../../src/obidmscolumn_char.c ../../../src/obidmscolumn_char.c

2
python/obitools3/utils.cfiles
View File

@ -1,3 +1,4 @@
../../src/array_indexer.c
../../src/bloom.c ../../src/bloom.c
../../src/char_str_indexer.c ../../src/char_str_indexer.c
../../src/crc64.c ../../src/crc64.c
@ -21,6 +22,7 @@
../../src/obidmscolumn_qual.c ../../src/obidmscolumn_qual.c
../../src/obidmscolumn_seq.c ../../src/obidmscolumn_seq.c
../../src/obidmscolumn_str.c ../../src/obidmscolumn_str.c
../../src/obidmscolumn_array.c
../../src/obidmscolumn.c ../../src/obidmscolumn.c
../../src/obidmscolumndir.c ../../src/obidmscolumndir.c
../../src/obierrno.c ../../src/obierrno.c

73
src/array_indexer.c Normal file
View File

@ -0,0 +1,73 @@
/****************************************************************************
* Array indexing functions *
****************************************************************************/
/**
* @file array_indexer.c
* @author Celine Mercier
* @date October 5th 2017
* @brief Functions handling the indexing and retrieval of arrays of any type.
*/
#include <stdlib.h>
#include <stdio.h>
#include <stdint.h>
#include <math.h>
#include "obiblob.h"
#include "obiblob_indexer.h"
#include "obidebug.h"
#include "obitypes.h"
#define DEBUG_LEVEL 0 // TODO has to be defined somewhere else (cython compil flag?)
index_t obi_index_array(Obi_indexer_p indexer, const void* value, uint8_t elt_size, int value_length)
{
Obi_blob_p value_b;
index_t idx;
int32_t length_bytes;
// fprintf(stderr, "\nelt size in C: %u", elt_size);
// for (int i=0; i<value_length; i++)
// fprintf(stderr, "\nvalue %d", ((obibool_t*)value)[i]);
length_bytes = value_length * elt_size;
// Encode value
value_b = obi_blob((byte_t*)value, elt_size, length_bytes, length_bytes);
if (value_b == NULL)
return -1;
// for (int i=0; i<value_length; i++)
// fprintf(stderr, "\nin blob: value %d", ((obibool_t*)(value_b->value))[i]);
// Add in the indexer
idx = obi_indexer_add(indexer, value_b);
free(value_b);
return idx;
}
const void* obi_retrieve_array(Obi_indexer_p indexer, index_t idx, int* value_length_p)
{
Obi_blob_p value_b;
// Get encoded value
value_b = obi_indexer_get(indexer, idx);
// Store array length
*value_length_p = (value_b->length_decoded_value) / (value_b->element_size);
// for (int i=0; i<*value_length_p; i++)
// fprintf(stderr, "\nvalue %d", ((obibool_t*)(value_b->value))[i]);
// Return pointer on mapped array
return ((void*) (value_b->value));
}

60
src/array_indexer.h Normal file
View File

@ -0,0 +1,60 @@
/****************************************************************************
* Array indexer header file *
****************************************************************************/
/**
* @file array_indexer.h
* @author Celine Mercier
* @date October 5th 2017
* @brief Header file for the functions handling the indexing of arrays of any type.
*/
#ifndef ARRAY_INDEXER_H_
#define ARRAY_INDEXER_H_
#include <stdlib.h>
#include <stdio.h>
#include "obidms.h"
#include "obitypes.h"
#include "obiblob.h"
#include "obiblob_indexer.h"
/**
* @brief Stores an array of elements of any type in an indexer and returns the index.
*
* @param indexer The indexer structure.
* @param value The array to index.
* @param elt_size The size in bits of one element.
* @param value_length The length (number of elements) of the array to index.
*
* @returns The index referring to the stored array in the indexer.
*
* @since October 2017
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
index_t obi_index_array(Obi_indexer_p indexer, const void* value, uint8_t elt_size, int32_t value_length);
/**
* @brief Retrieves an array from an indexer.
*
* @warning The array returned is mapped.
*
* @param indexer The indexer structure.
* @param idx The index referring to the array to retrieve in the indexer.
* @param value_length A pointer on an integer to store the length of the array retrieved.
*
* @returns A pointer on the array.
*
* @since October 2017
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
const void* obi_retrieve_array(Obi_indexer_p indexer, index_t idx, int32_t* value_length_p);
#endif /* ARRAY_INDEXER_H_ */

24
src/obi_align.c
View File

@ -154,35 +154,35 @@ static int create_alignment_output_columns(Obiview_p output_view,
bool normalize, int reference, bool similarity_mode) bool normalize, int reference, bool similarity_mode)
{ {
// Create the column for the ids of the 1st sequence aligned // Create the column for the ids of the 1st sequence aligned
if (obi_view_add_column(output_view, ID1_COLUMN_NAME, -1, NULL, OBI_STR, 0, 1, NULL, id1_indexer_name, NULL, -1, ID1_COLUMN_COMMENTS, true) < 0) if (obi_view_add_column(output_view, ID1_COLUMN_NAME, -1, NULL, OBI_STR, 0, 1, NULL, false, id1_indexer_name, NULL, -1, ID1_COLUMN_COMMENTS, true) < 0)
{ {
obidebug(1, "\nError creating the first column for the sequence ids when aligning"); obidebug(1, "\nError creating the first column for the sequence ids when aligning");
return -1; return -1;
} }
// Create the column for the ids of the 2nd sequence aligned // Create the column for the ids of the 2nd sequence aligned
if (obi_view_add_column(output_view, ID2_COLUMN_NAME, -1, NULL, OBI_STR, 0, 1, NULL, id2_indexer_name, NULL, -1, ID2_COLUMN_COMMENTS, true) < 0) if (obi_view_add_column(output_view, ID2_COLUMN_NAME, -1, NULL, OBI_STR, 0, 1, NULL, false, id2_indexer_name, NULL, -1, ID2_COLUMN_COMMENTS, true) < 0)
{ {
obidebug(1, "\nError creating the second column for the sequence ids when aligning"); obidebug(1, "\nError creating the second column for the sequence ids when aligning");
return -1; return -1;
} }
// Create the column for the index (in the input view) of the first sequences aligned // Create the column for the index (in the input view) of the first sequences aligned
if (obi_view_add_column(output_view, IDX1_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, NULL, NULL, -1, IDX1_COLUMN_COMMENTS, true) < 0) if (obi_view_add_column(output_view, IDX1_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, NULL, NULL, -1, IDX1_COLUMN_COMMENTS, true) < 0)
{ {
obidebug(1, "\nError creating the first column for the sequence indices when aligning"); obidebug(1, "\nError creating the first column for the sequence indices when aligning");
return -1; return -1;
} }
// Create the column for the index (in the input view) of the second sequences aligned // Create the column for the index (in the input view) of the second sequences aligned
if (obi_view_add_column(output_view, IDX2_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, NULL, NULL, -1, IDX2_COLUMN_COMMENTS, true) < 0) if (obi_view_add_column(output_view, IDX2_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, NULL, NULL, -1, IDX2_COLUMN_COMMENTS, true) < 0)
{ {
obidebug(1, "\nError creating the second column for the sequence indices when aligning"); obidebug(1, "\nError creating the second column for the sequence indices when aligning");
return -1; return -1;
} }
// Create the column for the LCS length // Create the column for the LCS length
if (obi_view_add_column(output_view, LCS_LENGTH_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, NULL, NULL, -1, LCS_LENGTH_COLUMN_COMMENTS, true) < 0) if (obi_view_add_column(output_view, LCS_LENGTH_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, NULL, NULL, -1, LCS_LENGTH_COLUMN_COMMENTS, true) < 0)
{ {
obidebug(1, "\nError creating the column for the LCS length when aligning"); obidebug(1, "\nError creating the column for the LCS length when aligning");
return -1; return -1;
@ -191,7 +191,7 @@ static int create_alignment_output_columns(Obiview_p output_view,
// Create the column for the alignment length if it is computed // Create the column for the alignment length if it is computed
if ((reference == ALILEN) && (normalize || !similarity_mode)) if ((reference == ALILEN) && (normalize || !similarity_mode))
{ {
if (obi_view_add_column(output_view, ALI_LENGTH_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, NULL, NULL, -1, ALI_LENGTH_COLUMN_COMMENTS, true) < 0) if (obi_view_add_column(output_view, ALI_LENGTH_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, NULL, NULL, -1, ALI_LENGTH_COLUMN_COMMENTS, true) < 0)
{ {
obidebug(1, "\nError creating the column for the alignment length when aligning"); obidebug(1, "\nError creating the column for the alignment length when aligning");
return -1; return -1;
@ -200,7 +200,7 @@ static int create_alignment_output_columns(Obiview_p output_view,
// Create the column for the alignment score // Create the column for the alignment score
if (normalize) if (normalize)
{ {
if (obi_view_add_column(output_view, SCORE_COLUMN_NAME, -1, NULL, OBI_FLOAT, 0, 1, NULL, NULL, NULL, -1, SCORE_COLUMN_NAME, true) < 0) if (obi_view_add_column(output_view, SCORE_COLUMN_NAME, -1, NULL, OBI_FLOAT, 0, 1, NULL, false, NULL, NULL, -1, SCORE_COLUMN_NAME, true) < 0)
{ {
obidebug(1, "\nError creating the column for the score when aligning"); obidebug(1, "\nError creating the column for the score when aligning");
return -1; return -1;
@ -208,7 +208,7 @@ static int create_alignment_output_columns(Obiview_p output_view,
} }
else else
{ {
if (obi_view_add_column(output_view, SCORE_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, NULL, NULL, -1, SCORE_COLUMN_NAME, true) < 0) if (obi_view_add_column(output_view, SCORE_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, NULL, NULL, -1, SCORE_COLUMN_NAME, true) < 0)
{ {
obidebug(1, "\nError creating the column for the score when aligning"); obidebug(1, "\nError creating the column for the score when aligning");
return -1; return -1;
@ -218,14 +218,14 @@ static int create_alignment_output_columns(Obiview_p output_view,
if (print_seq) if (print_seq)
{ {
// Create the column for the first sequences aligned // Create the column for the first sequences aligned
if (obi_view_add_column(output_view, SEQ1_COLUMN_NAME, -1, NULL, OBI_SEQ, 0, 1, NULL, seq1_indexer_name, NULL, -1, SEQ1_COLUMN_COMMENTS, true) < 0) if (obi_view_add_column(output_view, SEQ1_COLUMN_NAME, -1, NULL, OBI_SEQ, 0, 1, NULL, false, seq1_indexer_name, NULL, -1, SEQ1_COLUMN_COMMENTS, true) < 0)
{ {
obidebug(1, "\nError creating the first column for the sequences when aligning"); obidebug(1, "\nError creating the first column for the sequences when aligning");
return -1; return -1;
} }
// Create the column for the second sequences aligned // Create the column for the second sequences aligned
if (obi_view_add_column(output_view, SEQ2_COLUMN_NAME, -1, NULL, OBI_SEQ, 0, 1, NULL, seq2_indexer_name, NULL, -1, SEQ2_COLUMN_COMMENTS, true) < 0) if (obi_view_add_column(output_view, SEQ2_COLUMN_NAME, -1, NULL, OBI_SEQ, 0, 1, NULL, false, seq2_indexer_name, NULL, -1, SEQ2_COLUMN_COMMENTS, true) < 0)
{ {
obidebug(1, "\nError creating the second column for the sequences when aligning"); obidebug(1, "\nError creating the second column for the sequences when aligning");
return -1; return -1;
@ -234,14 +234,14 @@ static int create_alignment_output_columns(Obiview_p output_view,
// if (print_count) // TODO count columns not implemented yet // if (print_count) // TODO count columns not implemented yet
// { // {
// // Create the column for the count of the first sequences aligned // // Create the column for the count of the first sequences aligned
// if (obi_view_add_column(output_view, COUNT1_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, NULL, NULL, -1, COUNT1_COLUMN_COMMENTS, true) < 0) // if (obi_view_add_column(output_view, COUNT1_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, NULL, NULL, -1, COUNT1_COLUMN_COMMENTS, true) < 0)
// { // {
// obidebug(1, "\nError creating the first column for the sequence counts when aligning"); // obidebug(1, "\nError creating the first column for the sequence counts when aligning");
// return -1; // return -1;
// } // }
// //
// // Create the column for the count of the second sequences aligned // // Create the column for the count of the second sequences aligned
// if (obi_view_add_column(output_view, COUNT2_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, NULL, NULL, -1, COUNT2_COLUMN_COMMENTS, true) < 0) // if (obi_view_add_column(output_view, COUNT2_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, NULL, NULL, -1, COUNT2_COLUMN_COMMENTS, true) < 0)
// { // {
// obidebug(1, "\nError creating the second column for the sequence counts when aligning"); // obidebug(1, "\nError creating the second column for the sequence counts when aligning");
// return -1; // return -1;

3
src/obidms.c
View File

@ -924,7 +924,7 @@ obiversion_t obi_import_column(const char* dms_path_1, const char* dms_path_2, c
// Create new column // Create new column
column_2 = obi_create_column(dms_2, column_name, header_1->returned_data_type, header_1->line_count, column_2 = obi_create_column(dms_2, column_name, header_1->returned_data_type, header_1->line_count,
header_1->nb_elements_per_line, header_1->elements_names, true, header_1->nb_elements_per_line, header_1->elements_names, true, header_1->tuples,
new_avl_name, (header_1->associated_column).column_name, (header_1->associated_column).version, new_avl_name, (header_1->associated_column).column_name, (header_1->associated_column).version,
header_1->comments); header_1->comments);
@ -1142,6 +1142,7 @@ int obi_import_view(const char* dms_path_1, const char* dms_path_2, const char*
0, 0,
0, 0,
NULL, NULL,
false,
NULL, NULL,
NULL, NULL,
-1, -1,

9
src/obidmscolumn.c
View File

@ -890,6 +890,7 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms,
index_t nb_elements_per_line, index_t nb_elements_per_line,
char* elements_names, char* elements_names,
bool elt_names_formatted, bool elt_names_formatted,
bool tuples,
const char* indexer_name, const char* indexer_name,
const char* associated_column_name, const char* associated_column_name,
obiversion_t associated_column_version, obiversion_t associated_column_version,
@ -957,7 +958,7 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms,
} }
// Build the indexer name if needed // Build the indexer name if needed
if ((data_type == OBI_STR) || (data_type == OBI_SEQ) || (data_type == OBI_QUAL)) if ((data_type == OBI_STR) || (data_type == OBI_SEQ) || (data_type == OBI_QUAL) || tuples)
{ {
if ((indexer_name == NULL) || (*indexer_name == 0)) if ((indexer_name == NULL) || (*indexer_name == 0))
{ {
@ -973,7 +974,7 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms,
} }
returned_data_type = data_type; returned_data_type = data_type;
if ((data_type == OBI_STR) || (data_type == OBI_SEQ) || (data_type == OBI_QUAL)) if ((data_type == OBI_STR) || (data_type == OBI_SEQ) || (data_type == OBI_QUAL) || tuples)
// stored data is indices referring to data stored elsewhere // stored data is indices referring to data stored elsewhere
stored_data_type = OBI_IDX; stored_data_type = OBI_IDX;
else else
@ -1105,6 +1106,7 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms,
header->nb_elements_per_line = nb_elements_per_line; header->nb_elements_per_line = nb_elements_per_line;
header->stored_data_type = stored_data_type; header->stored_data_type = stored_data_type;
header->returned_data_type = returned_data_type; header->returned_data_type = returned_data_type;
header->tuples = tuples;
header->creation_date = time(NULL); header->creation_date = time(NULL);
header->version = version_number; header->version = version_number;
header->cloned_from = -1; header->cloned_from = -1;
@ -1146,7 +1148,7 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms,
} }
// If the data type is OBI_STR, OBI_SEQ or OBI_QUAL, the associated obi_indexer is opened or created // If the data type is OBI_STR, OBI_SEQ or OBI_QUAL, the associated obi_indexer is opened or created
if ((returned_data_type == OBI_STR) || (returned_data_type == OBI_SEQ) || (returned_data_type == OBI_QUAL)) if ((returned_data_type == OBI_STR) || (returned_data_type == OBI_SEQ) || (returned_data_type == OBI_QUAL) || tuples)
{ {
new_column->indexer = obi_indexer(dms, final_indexer_name); new_column->indexer = obi_indexer(dms, final_indexer_name);
if (new_column->indexer == NULL) if (new_column->indexer == NULL)
@ -1377,6 +1379,7 @@ OBIDMS_column_p obi_clone_column(OBIDMS_p dms,
nb_elements_per_line, nb_elements_per_line,
(column_to_clone->header)->elements_names, (column_to_clone->header)->elements_names,
true, true,
(column_to_clone->header)->tuples,
(column_to_clone->header)->indexer_name, (column_to_clone->header)->indexer_name,
((column_to_clone->header)->associated_column).column_name, ((column_to_clone->header)->associated_column).column_name,
((column_to_clone->header)->associated_column).version, ((column_to_clone->header)->associated_column).version,

6
src/obidmscolumn.h
View File

@ -86,6 +86,8 @@ typedef struct OBIDMS_column_header {
OBIType_t stored_data_type; /**< Type of the data that is actually stored in the data OBIType_t stored_data_type; /**< Type of the data that is actually stored in the data
* part of the column. * part of the column.
*/ */
bool tuples; /**< A boolean indicating whether the column contains indices referring to indexed tuples.
*/
time_t creation_date; /**< Date of creation of the file. time_t creation_date; /**< Date of creation of the file.
*/ */
obiversion_t version; /**< Version of the column. obiversion_t version; /**< Version of the column.
@ -100,7 +102,7 @@ typedef struct OBIDMS_column_header {
*/ */
Column_reference_t associated_column; /**< If there is one, the reference to the associated column. Column_reference_t associated_column; /**< If there is one, the reference to the associated column.
*/ */
bool finished; /**< A boolean indicating whether the column was properly closed by the view that created it. TODO bool finished; /**< A boolean indicating whether the column was properly closed by the view that created it.
*/ */
char comments[COMMENTS_MAX_LENGTH+1]; /**< Comments stored as a classical zero end C string. char comments[COMMENTS_MAX_LENGTH+1]; /**< Comments stored as a classical zero end C string.
*/ */
@ -239,6 +241,7 @@ size_t obi_get_platform_header_size();
* @param elements_names The names of the elements with ';' as separator (no terminal ';'), * @param elements_names The names of the elements with ';' as separator (no terminal ';'),
* NULL or "" if the default names are to be used ("0\01\02\0...\0n"). * NULL or "" if the default names are to be used ("0\01\02\0...\0n").
* @param elt_names_formatted Whether the separator for the elements names is ';' (false), or '\0' (true, as formatted by format_elements_names()). * @param elt_names_formatted Whether the separator for the elements names is ';' (false), or '\0' (true, as formatted by format_elements_names()).
* @param tuples A boolean indicating whether the column should contain indices referring to indexed tuples.
* @param indexer_name The name of the indexer if there is one associated with the column. * @param indexer_name The name of the indexer if there is one associated with the column.
* If NULL or "", the indexer name is set as the column name. * If NULL or "", the indexer name is set as the column name.
* @param associated_column_name The name of the associated column if there is one (otherwise NULL or ""). * @param associated_column_name The name of the associated column if there is one (otherwise NULL or "").
@ -258,6 +261,7 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms,
index_t nb_elements_per_line, index_t nb_elements_per_line,
char* elements_names, char* elements_names,
bool elt_names_formatted, bool elt_names_formatted,
bool tuples,
const char* indexer_name, const char* indexer_name,
const char* associated_column_name, const char* associated_column_name,
obiversion_t associated_column_version, obiversion_t associated_column_version,

85
src/obidmscolumn_array.c Normal file
View File

@ -0,0 +1,85 @@
/****************************************************************************
* OBIDMS_column_array functions *
****************************************************************************/
/**
* @file obidsmcolumn_array.c
* @author Celine Mercier
* @date October 27th 2017
* @brief Functions handling OBIColumns containing data arrays of any type.
*/
#include <stdlib.h>
#include <stdio.h>
#include "obidmscolumn.h"
#include "obitypes.h"
#include "array_indexer.h"
/**********************************************************************
*
* D E F I N I T I O N O F T H E P U B L I C F U N C T I O N S
*
**********************************************************************/
int obi_column_set_array(OBIDMS_column_p column, index_t line_nb, const void* value, uint8_t elt_size, int32_t value_length)
{
index_t idx;
if (obi_column_prepare_to_set_value(column, line_nb, 0) < 0)
return -1;
if (value == OBITuple_NA)
{
idx = OBIIdx_NA;
}
else
{
// Add the value in the indexer
idx = obi_index_array(column->indexer, value, elt_size, value_length);
if (idx == -1) // An error occurred
{
if (obi_errno == OBI_READ_ONLY_INDEXER_ERROR)
{
// If the error is that the indexer is read-only, clone it
if (obi_clone_column_indexer(column) < 0)
return -1;
obi_set_errno(0);
// Add the value in the new indexer
idx = obi_index_array(column->indexer, value, elt_size, value_length);
if (idx == -1)
return -1;
}
else
return -1;
}
}
// Add the value's index in the column
*(((index_t*) (column->data)) + line_nb) = idx;
return 0;
}
const void* obi_column_get_array(OBIDMS_column_p column, index_t line_nb, int32_t* value_length_p)
{
index_t idx;
if (obi_column_prepare_to_get_value(column, line_nb) < 0)
return OBITuple_NA;
idx = *(((index_t*) (column->data)) + line_nb);
// Check NA
if (idx == OBIIdx_NA)
return OBITuple_NA;
return obi_retrieve_array(column->indexer, idx, value_length_p);
}

64
src/obidmscolumn_array.h Normal file
View File

@ -0,0 +1,64 @@
/****************************************************************************
* Array columns header file *
****************************************************************************/
/**
* @file obidsmcolumn_array.h
* @author Celine Mercier
* @date October 30th 2017
* @brief Header file for the functions handling OBIColumns containing data in the form of indices referring to data arrays.
*/
#ifndef OBIDMSCOLUMN_ARRAY_H_
#define OBIDMSCOLUMN_ARRAY_H_
#include <stdlib.h>
#include <stdio.h>
#include "obidmscolumn.h"
#include "obitypes.h"
/**
* @brief Sets a value in an OBIDMS column containing data in the form of indices referring
* to arrays handled by an indexer.
*
* @warning Pointers returned by obi_open_column() don't allow writing.
*
* @param column A pointer as returned by obi_create_column() or obi_clone_column().
* @param line_nb The number of the line where the value should be set.
* @param value A pointer on the array.
* @param elt_size The size in bits of one element.
* @param value_length The length (number of elements) of the array to index.
*
* @returns An integer value indicating the success of the operation.
* @retval 0 on success.
* @retval -1 if an error occurred.
*
* @since October 2017
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
int obi_column_set_array(OBIDMS_column_p column, index_t line_nb, const void* value, uint8_t elt_size, int32_t value_length);
/**
* @brief Recovers a value in an OBIDMS column containing data in the form of indices referring
* to arrays handled by an indexer.
*
* @param column A pointer as returned by obi_create_column().
* @param line_nb The number of the line where the value should be recovered.
* @param value_length A pointer on an integer to store the length of the array retrieved.
*
* @returns The recovered value.
* @retval OBITuple_NA the NA value of the type if an error occurred and obi_errno is set.
*
* @since October 2017
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
const void* obi_column_get_array(OBIDMS_column_p column, index_t line_nb, int32_t* value_length_p);
#endif /* OBIDMSCOLUMN_ARRAY_H_ */

2
src/obitypes.h
View File

@ -27,7 +27,7 @@
#define OBIBlob_NA (NULL) /**< NA value for the type Obiblobs */ // TODO discuss #define OBIBlob_NA (NULL) /**< NA value for the type Obiblobs */ // TODO discuss
#define OBIQual_char_NA (NULL) /**< NA value for the type OBI_QUAL if the quality is in character string format */ #define OBIQual_char_NA (NULL) /**< NA value for the type OBI_QUAL if the quality is in character string format */
#define OBIQual_int_NA (NULL) /**< NA value for the type OBI_QUAL if the quality is in integer format */ #define OBIQual_int_NA (NULL) /**< NA value for the type OBI_QUAL if the quality is in integer format */
#define OBITuple_NA (NULL) /**< NA value for tuples of any type */
/** /**
* @brief enum for the boolean OBIType. * @brief enum for the boolean OBIType.

58
src/obiview.c
View File

@ -28,6 +28,7 @@
#include "obidmscolumn_qual.h" #include "obidmscolumn_qual.h"
#include "obidmscolumn_seq.h" #include "obidmscolumn_seq.h"
#include "obidmscolumn_str.h" #include "obidmscolumn_str.h"
#include "obidmscolumn_array.h"
#include "obierrno.h" #include "obierrno.h"
#include "obidebug.h" #include "obidebug.h"
#include "obilittlebigman.h" #include "obilittlebigman.h"
@ -1643,7 +1644,7 @@ Obiview_p obi_new_view(OBIDMS_p dms, const char* view_name, Obiview_p view_to_cl
// If there is a new line selection, build it by combining it with the one from the view to clone if there is one // If there is a new line selection, build it by combining it with the one from the view to clone if there is one
else if (line_selection != NULL) else if (line_selection != NULL)
{ {
view->line_selection = obi_create_column(view->dms, LINES_COLUMN_NAME, OBI_IDX, 0, 1, NULL, false, NULL, NULL, -1, NULL); view->line_selection = obi_create_column(view->dms, LINES_COLUMN_NAME, OBI_IDX, 0, 1, NULL, false, false, NULL, NULL, -1, NULL);
if ((view->line_selection) == NULL) if ((view->line_selection) == NULL)
{ {
obidebug(1, "\nError creating a column corresponding to a line selection"); obidebug(1, "\nError creating a column corresponding to a line selection");
@ -1795,6 +1796,7 @@ Obiview_p obi_new_view(OBIDMS_p dms, const char* view_name, Obiview_p view_to_cl
0, 0,
0, 0,
NULL, NULL,
false,
NULL, NULL,
NULL, NULL,
-1, -1,
@ -1862,19 +1864,19 @@ Obiview_p obi_new_view_nuc_seqs(OBIDMS_p dms, const char* view_name, Obiview_p v
if (view_to_clone == NULL) if (view_to_clone == NULL)
{ {
// Adding sequence column // Adding sequence column
if (obi_view_add_column(view, NUC_SEQUENCE_COLUMN, -1, NULL, OBI_SEQ, 0, 1, NULL, NULL, NULL, -1, "Nucleotide sequences", true) < 0) // discuss using same indexer "NUC_SEQ_INDEXER" if (obi_view_add_column(view, NUC_SEQUENCE_COLUMN, -1, NULL, OBI_SEQ, 0, 1, NULL, false, NULL, NULL, -1, "Nucleotide sequences", true) < 0) // discuss using same indexer "NUC_SEQ_INDEXER"
{ {
obidebug(1, "Error adding an obligatory column in a nucleotide sequences view"); obidebug(1, "Error adding an obligatory column in a nucleotide sequences view");
return NULL; return NULL;
} }
// Adding id column // Adding id column
if (obi_view_add_column(view, ID_COLUMN, -1, NULL, OBI_STR, 0, 1, NULL, NULL, NULL, -1, "Sequence identifiers", true) < 0) if (obi_view_add_column(view, ID_COLUMN, -1, NULL, OBI_STR, 0, 1, NULL, false, NULL, NULL, -1, "Sequence identifiers", true) < 0)
{ {
obidebug(1, "Error adding an obligatory column in a nucleotide sequences view"); obidebug(1, "Error adding an obligatory column in a nucleotide sequences view");
return NULL; return NULL;
} }
// Adding definition column // Adding definition column
if (obi_view_add_column(view, DEFINITION_COLUMN, -1, NULL, OBI_STR, 0, 1, NULL, NULL, NULL, -1, "Definitions", true) < 0) if (obi_view_add_column(view, DEFINITION_COLUMN, -1, NULL, OBI_STR, 0, 1, NULL, false, NULL, NULL, -1, "Definitions", true) < 0)
{ {
obidebug(1, "Error adding an obligatory column in a nucleotide sequences view"); obidebug(1, "Error adding an obligatory column in a nucleotide sequences view");
return NULL; return NULL;
@ -1883,7 +1885,7 @@ Obiview_p obi_new_view_nuc_seqs(OBIDMS_p dms, const char* view_name, Obiview_p v
if (quality_column) if (quality_column)
{ {
associated_nuc_column = obi_view_get_column(view, NUC_SEQUENCE_COLUMN); associated_nuc_column = obi_view_get_column(view, NUC_SEQUENCE_COLUMN);
if (obi_view_add_column(view, QUALITY_COLUMN, -1, NULL, OBI_QUAL, 0, 1, NULL, NULL, (associated_nuc_column->header)->name, (associated_nuc_column->header)->version, "Sequence qualities", true) < 0) // TODO discuss automatic association if (obi_view_add_column(view, QUALITY_COLUMN, -1, NULL, OBI_QUAL, 0, 1, NULL, false, NULL, (associated_nuc_column->header)->name, (associated_nuc_column->header)->version, "Sequence qualities", true) < 0) // TODO discuss automatic association
{ {
obidebug(1, "Error adding an obligatory column in a nucleotide sequences view"); obidebug(1, "Error adding an obligatory column in a nucleotide sequences view");
return NULL; return NULL;
@ -2211,6 +2213,7 @@ int obi_view_add_column(Obiview_p view,
index_t nb_lines, index_t nb_lines,
index_t nb_elements_per_line, index_t nb_elements_per_line,
char* elements_names, char* elements_names,
bool tuples,
const char* indexer_name, const char* indexer_name,
const char* associated_column_name, const char* associated_column_name,
obiversion_t associated_column_version, obiversion_t associated_column_version,
@ -2289,7 +2292,7 @@ int obi_view_add_column(Obiview_p view,
// Open or create the column // Open or create the column
if (create) if (create)
{ // Create column { // Create column
column = obi_create_column(view->dms, column_name, data_type, nb_lines, nb_elements_per_line, elements_names, false, indexer_name, associated_column_name, associated_column_version, comments); column = obi_create_column(view->dms, column_name, data_type, nb_lines, nb_elements_per_line, elements_names, false, tuples, indexer_name, associated_column_name, associated_column_version, comments);
if (column == NULL) if (column == NULL)
{ {
obidebug(1, "\nError creating a column to add to a view"); obidebug(1, "\nError creating a column to add to a view");
@ -2796,7 +2799,7 @@ int obi_create_auto_count_column(Obiview_p view)
return -1; return -1;
} }
if (obi_view_add_column(view, COUNT_COLUMN, -1, NULL, OBI_INT, 0, 1, NULL, NULL, NULL, -1, "Sequence counts", true) < 0) if (obi_view_add_column(view, COUNT_COLUMN, -1, NULL, OBI_INT, 0, 1, NULL, false, NULL, NULL, -1, "Sequence counts", true) < 0)
{ {
obidebug(1, "Error adding an automatic count column in a view"); obidebug(1, "Error adding an automatic count column in a view");
return -1; return -1;
@ -2848,7 +2851,7 @@ int obi_create_auto_id_column(Obiview_p view, const char* prefix)
} }
// Create the new ID column // Create the new ID column
if (obi_view_add_column(view, ID_COLUMN, -1, NULL, OBI_STR, 0, 1, NULL, NULL, NULL, -1, "Sequence identifiers", true) < 0) if (obi_view_add_column(view, ID_COLUMN, -1, NULL, OBI_STR, 0, 1, NULL, false, NULL, NULL, -1, "Sequence identifiers", true) < 0)
{ {
obidebug(1, "Error adding an automatic ID column in a view"); obidebug(1, "Error adding an automatic ID column in a view");
return -1; return -1;
@ -3623,3 +3626,42 @@ index_t obi_get_index_with_elt_name_and_col_name_in_view(Obiview_p view, const c
/****************************************/ /****************************************/
/*********** FOR ARRAY COLUMNS ***********/
int obi_set_array_with_col_p_in_view(Obiview_p view, OBIDMS_column_p column, index_t line_nb, const void* value, uint8_t elt_size, int32_t value_length)
{
if (prepare_to_set_value_in_column(view, &column, &line_nb) < 0)
return -1;
return obi_column_set_array(column, line_nb, value, elt_size, value_length);
}
const void* obi_get_array_with_col_p_in_view(Obiview_p view, OBIDMS_column_p column, index_t line_nb, int32_t* value_length_p)
{
if (prepare_to_get_value_from_column(view, &line_nb) < 0)
return OBITuple_NA;
return obi_column_get_array(column, line_nb, value_length_p);
}
int obi_set_array_with_col_name_in_view(Obiview_p view, const char* column_name, index_t line_nb, const void* value, uint8_t elt_size, int32_t value_length)
{
OBIDMS_column_p column_p;
column_p = obi_view_get_column(view, column_name);
if (column_p == NULL)
return -1;
return obi_set_array_with_col_p_in_view(view, column_p, line_nb, value, elt_size, value_length);
}
const void* obi_get_array_with_col_name_in_view(Obiview_p view, const char* column_name, index_t line_nb, int32_t* value_length_p)
{
OBIDMS_column_p column_p;
column_p = obi_view_get_column(view, column_name);
if (column_p == NULL)
return OBITuple_NA;
return obi_get_array_with_col_p_in_view(view, column_p, line_nb, value_length_p);
}
/****************************************/

86
src/obiview.h
View File

@ -361,6 +361,7 @@ Obiview_p obi_open_view(OBIDMS_p dms, const char* view_name);
* @param nb_elements_per_line The number of elements per line, if the column is created. * @param nb_elements_per_line The number of elements per line, if the column is created.
* @param elements_names The names of the elements with ';' as separator (no terminal ';'), * @param elements_names The names of the elements with ';' as separator (no terminal ';'),
* if the column is created; NULL or "" if the default names are to be used ("0\01\02\0...\0n"). * if the column is created; NULL or "" if the default names are to be used ("0\01\02\0...\0n").
* @param tuples A boolean indicating whether the column should contain indices referring to indexed tuples.
* @param indexer_name The name of the indexer if there is one associated with the column, if the column is created. * @param indexer_name The name of the indexer if there is one associated with the column, if the column is created.
* If NULL or "", the indexer name is set as the column name. * If NULL or "", the indexer name is set as the column name.
* @param associated_column_name The name of the associated column if there is one (otherwise NULL or ""), if the column is created. * @param associated_column_name The name of the associated column if there is one (otherwise NULL or ""), if the column is created.
@ -383,6 +384,7 @@ int obi_view_add_column(Obiview_p view,
index_t nb_lines, index_t nb_lines,
index_t nb_elements_per_line, index_t nb_elements_per_line,
char* elements_names, char* elements_names,
bool tuples,
const char* indexer_name, const char* indexer_name,
const char* associated_column_name, const char* associated_column_name,
obiversion_t associated_column_version, obiversion_t associated_column_version,
@ -2207,4 +2209,88 @@ index_t obi_get_index_with_elt_name_and_col_p_in_view(Obiview_p view, OBIDMS_col
index_t obi_get_index_with_elt_name_and_col_name_in_view(Obiview_p view, const char* column_name, index_t line_nb, const char* element_name); index_t obi_get_index_with_elt_name_and_col_name_in_view(Obiview_p view, const char* column_name, index_t line_nb, const char* element_name);
/*********** FOR ARRAY COLUMNS ***********/
/**
* @brief Sets a value in an OBIDMS column containing indices referring to indexed arrays,
* using the column pointer, in the context of a view.
*
* Note: If the column is read-only or if there is a line selection associated with the view (making columns non-writable), it is cloned.
*
* @param view A pointer on the opened writable view.
* @param column_p A pointer on the column.
* @param line_nb The number of the line where the value should be set.
* @param value The value that should be set.
* @param elt_size The size in bits of one element.
* @param value_length The length (number of elements) of the array to index.
*
* @returns An integer value indicating the success of the operation.
* @retval 0 on success.
* @retval -1 if an error occurred.
*
* @since October 2017
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
int obi_set_array_with_col_p_in_view(Obiview_p view, OBIDMS_column_p column, index_t line_nb, const void* value, uint8_t elt_size, int32_t value_length);
/**
* @brief Recovers a value in an OBIDMS column containing indices referring to indexed arrays,
* using the column pointer, in the context of a view.
*
* @param view A pointer on the opened view.
* @param column_p A pointer on the column.
* @param line_nb The number of the line where the value should be recovered.
* @param value_length_p A pointer on an int where the length of the value (number of elements in the array) will be stored.
*
* @returns The recovered value.
* @retval OBITuple_NA the NA value of the type if an error occurred and obi_errno is set.
*
* @since October 2017
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
const void* obi_get_array_with_col_p_in_view(Obiview_p view, OBIDMS_column_p column, index_t line_nb, int32_t* value_length_p);
/**
* @brief Sets a value in an OBIDMS column containing indices referring to indexed arrays,
* using the column name, in the context of a view.
*
* Note: If the column is read-only or if there is a line selection associated with the view (making columns non-writable), it is cloned.
*
* @param view A pointer on the opened writable view.
* @param column_name The name of the column.
* @param line_nb The number of the line where the value should be set.
* @param value The value that should be set.
* @param elt_size The size in bits of one element.
* @param value_length The length (number of elements) of the array to index.
*
* @returns An integer value indicating the success of the operation.
* @retval 0 on success.
* @retval -1 if an error occurred.
*
* @since October 2017
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
int obi_set_array_with_col_name_in_view(Obiview_p view, const char* column_name, index_t line_nb, const void* value, uint8_t elt_size, int32_t value_length);
/**
* @brief Recovers a value in an OBIDMS column containing indices referring to indexed arrays,
* using the column name, in the context of a view.
*
* @param view A pointer on the opened view.
* @param column_name The name of the column.
* @param line_nb The number of the line where the value should be recovered.
* @param value_length_p A pointer on an int where the length of the value (number of elements in the array) will be stored.
*
* @returns The recovered value.
* @retval OBITuple_NA the NA value of the type if an error occurred and obi_errno is set.
*
* @since October 2017
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
const void* obi_get_array_with_col_name_in_view(Obiview_p view, const char* column_name, index_t line_nb, int32_t* value_length_p);
#endif /* OBIVIEW_H_ */ #endif /* OBIVIEW_H_ */