obitools/obitools3
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 66 Wiki Activity

Added tuple columns containing immutable indexed data arrays of any type

Browse Source
This commit is contained in:
Celine Mercier
2017-11-15 13:48:59 +01:00
parent 1684f96b79
commit 9a50803c00
32 changed files with 1097 additions and 284 deletions
Download Patch File Download Diff File Expand all files Collapse all files

105
python/obitools3/commands/test.pyx
View File

@ -1,11 +1,11 @@
#cython: language_level=3
from obitools3.apps.progress cimport ProgressBar # TODO I absolutely don't understand why it doesn't work without that line
from obitools3.dms.view.view import View, Line_selection
from obitools3.dms.view import View, Line_selection
from obitools3.dms.view.typed_view.view_NUC_SEQS import View_NUC_SEQS
from obitools3.dms.dms import DMS
from obitools3.dms import DMS
from obitools3.dms.column import Column
from obitools3.dms.taxo.taxo import Taxonomy
from obitools3.dms.taxo import Taxonomy
from obitools3.utils cimport str2bytes
from obitools3.dms.capi.obitypes cimport OBI_INT, \
OBI_FLOAT, \
@ -69,26 +69,68 @@ def random_bool(config):
return random.choice([True, False])
def random_bool_tuples(config):
l=[]
for i in range(random.randint(1, config['test']['tuplemaxlen'])) :
l.append(random.choice([None, random_bool(config)]))
return tuple(l)
def random_char(config):
return str2bytes(random.choice(string.ascii_lowercase))
def random_char_tuples(config):
l=[]
for i in range(random.randint(1, config['test']['tuplemaxlen'])) :
l.append(random.choice([None, random_char(config)]))
return tuple(l)
def random_float(config):
return random.randint(0, MAX_INT) + random.random()
def random_float_tuples(config):
l=[]
for i in range(random.randint(1, config['test']['tuplemaxlen'])) :
l.append(random.choice([None, random_float(config)]))
return tuple(l)
def random_int(config):
return random.randint(0, config['test']['maxlinenb'])
def random_int_tuples(config):
l=[]
for i in range(random.randint(1, config['test']['tuplemaxlen'])) :
l.append(random.choice([None, random_int(config)]))
return tuple(l)
def random_seq(config):
return str2bytes(''.join(random.choice(['a','t','g','c']) for i in range(random_length(config['test']['seqmaxlen']))))
def random_seq_tuples(config):
l=[]
for i in range(random.randint(1, config['test']['tuplemaxlen'])) :
l.append(random.choice([None, random_seq(config)]))
return tuple(l)
def random_bytes(config):
return random_bytes_with_max_len(config['test']['strmaxlen'])
def random_bytes_tuples(config):
l=[]
for i in range(random.randint(1, config['test']['tuplemaxlen'])) :
l.append(random.choice([None, random_bytes(config)]))
return tuple(l)
def random_str_with_max_len(max_len):
return ''.join(random.choice(string.ascii_lowercase) for i in range(random_length(max_len)))
@ -132,11 +174,28 @@ def test_set_and_get(config, infos):
print_test(config, "-")
return
idx = random_int(config)
value = random.choice([None, infos['random_generator'][data_type](config)])
value = random.choice([None, infos['random_generator'][(data_type, col.tuples)](config)])
if col.nb_elements_per_line > 1 :
elt = random.choice(element_names)
col[idx][elt] = value
assert col[idx][elt] == value, "Column: "+repr(col)+"\nSet value != gotten value "+str(value)+" != "+str(col[idx][elt])
elif col.tuples:
col[idx] = value
if value is None:
totest = None
else:
totest = []
for e in value:
if e is not None and e != '':
totest.append(e)
if len(totest) == 0:
totest = None
else:
totest = tuple(totest)
assert col[idx] == totest, "Column: "+repr(col)+"\nSet value != gotten value "+str(totest)+" != "+str(col[idx])
if totest is not None:
for i in range(len(totest)) :
assert col[idx][i] == totest[i], "Column: "+repr(col)+"\nSet value[i] != gotten value[i] "+str(totest[i])+" != "+str(col[idx][i])
else:
col[idx] = value
assert col[idx] == value, "Column: "+repr(col)+"\nSet value != gotten value "+str(value)+" != "+str(col[idx])
@ -210,19 +269,25 @@ def fill_column(config, infos, col) :
if len(element_names) > 1 :
for i in range(random_int(config)) :
for j in range(len(element_names)) :
col[i][element_names[j]] = random.choice([None, infos['random_generator'][data_type](config)])
col[i][element_names[j]] = random.choice([None, infos['random_generator'][(data_type, col.tuples)](config)])
else :
for i in range(random_int(config)) :
col[i] = random.choice([None, infos['random_generator'][data_type](config)])
r = random.choice([None, infos['random_generator'][(data_type, col.tuples)](config)])
col[i] = r
def create_random_column(config, infos) :
alias = random.choice([b'', random_unique_name(infos)])
nb_elements_per_line=random.randint(1, config['test']['maxelts'])
elements_names = []
for i in range(nb_elements_per_line) :
elements_names.append(random_unique_element_name(config, infos))
elements_names = random.choice([None, elements_names])
tuples = random.choice([True, False])
if not tuples :
nb_elements_per_line=random.randint(1, config['test']['maxelts'])
elements_names = []
for i in range(nb_elements_per_line) :
elements_names.append(random_unique_element_name(config, infos))
elements_names = random.choice([None, elements_names])
else :
nb_elements_per_line = 1
elements_names = None
name = random_unique_name(infos)
data_type = random_col_type()
@ -231,6 +296,7 @@ def create_random_column(config, infos) :
data_type,
nb_elements_per_line=nb_elements_per_line,
elements_names=elements_names,
tuples=tuples,
comments=random_str_with_max_len(COL_COMMENTS_MAX_LEN),
alias=alias
)
@ -347,6 +413,14 @@ def addOptions(parser):
type=int,
help="Maximum length of character strings. "
"Default: 200")
group.add_argument('--tuple_max_len','-u',
action="store", dest="test:tuplemaxlen",
metavar='<TUPLE_MAX_LEN>',
default=20,
type=int,
help="Maximum length of tuples. "
"Default: 200")
group.add_argument('--comments_max_len','-c',
action="store", dest="test:commentsmaxlen",
@ -402,7 +476,14 @@ def run(config):
'view': None,
'view_names': None,
'unique_names': [],
'random_generator': {b"OBI_BOOL": random_bool, b"OBI_CHAR": random_char, b"OBI_FLOAT": random_float, b"OBI_INT": random_int, b"OBI_SEQ": random_seq, b"OBI_STR": random_bytes},
'random_generator': {
(b"OBI_BOOL", False): random_bool, (b"OBI_BOOL", True): random_bool_tuples,
(b"OBI_CHAR", False): random_char, (b"OBI_CHAR", True): random_char_tuples,
(b"OBI_FLOAT", False): random_float, (b"OBI_FLOAT", True): random_float_tuples,
(b"OBI_INT", False): random_int, (b"OBI_INT", True): random_int_tuples,
(b"OBI_SEQ", False): random_seq, (b"OBI_SEQ", True): random_seq_tuples,
(b"OBI_STR", False): random_bytes, (b"OBI_STR", True): random_bytes_tuples
},
'tests': [test_set_and_get, test_add_col, test_delete_col, test_col_alias, test_new_view]
}

4
python/obitools3/dms/capi/obidmscolumn.pxd
View File

@ -31,6 +31,7 @@ cdef extern from "obidmscolumn.h" nogil:
const_char_p elements_names
OBIType_t returned_data_type
OBIType_t stored_data_type
bint tuples
time_t creation_date
obiversion_t version
obiversion_t cloned_from
@ -60,3 +61,6 @@ cdef extern from "obidmscolumn.h" nogil:
int obi_close_header(OBIDMS_column_header_p header)
char* obi_get_elements_names(OBIDMS_column_p column)
index_t obi_column_get_element_index_from_name(OBIDMS_column_p column, const char* element_name)

1
python/obitools3/dms/capi/obitypes.pxd
View File

@ -52,6 +52,7 @@ cdef extern from "obitypes.h" nogil:
extern const_char_p OBIStr_NA
extern const_char_p OBIQual_char_NA
extern uint8_t* OBIQual_int_NA
extern void* OBITuple_NA
const_char_p name_data_type(int data_type)

29
python/obitools3/dms/capi/obiview.pxd
View File

@ -14,7 +14,7 @@ from ..capi.obidmscolumn cimport OBIDMS_column_p, \
Column_reference_t, \
Column_reference_p
from libc.stdint cimport uint8_t
from libc.stdint cimport uint8_t, int32_t
cdef extern from "obiview.h" nogil:
@ -86,6 +86,7 @@ cdef extern from "obiview.h" nogil:
index_t nb_lines,
index_t nb_elements_per_line,
char* elements_names,
bint tuples,
const_char_p indexer_name,
const_char_p associated_column_name,
obiversion_t associated_column_version,
@ -327,3 +328,29 @@ cdef extern from "obiview.h" nogil:
OBIDMS_column_p column_p,
index_t line_nb,
index_t element_idx)
# ARRAY
int obi_set_array_with_col_p_in_view(Obiview_p view,
OBIDMS_column_p column,
index_t line_nb,
const void* value,
uint8_t elt_size,
int32_t value_length)
const void* obi_get_array_with_col_p_in_view(Obiview_p view,
OBIDMS_column_p column,
index_t line_nb,
int32_t* value_length_p)
int obi_set_array_with_col_name_in_view(Obiview_p view,
const char* column_name,
index_t line_nb,
const void* value,
uint8_t elt_size,
int32_t value_length)
const void* obi_get_array_with_col_name_in_view(Obiview_p view,
const char* column_name,
index_t line_nb,
int32_t* value_length_p)

3
python/obitools3/dms/column/column.pxd
View File

@ -22,7 +22,7 @@ cdef class Column(OBIWrapper) :
cdef inline OBIDMS_column_p pointer(self)
@staticmethod
cdef type get_column_class(obitype_t obitype, bint multi_elts)
cdef type get_column_class(obitype_t obitype, bint multi_elts, bint tuples)
@staticmethod
cdef type get_python_type(obitype_t obitype, bint multi_elts)
@ -44,6 +44,7 @@ cdef class Column_line:
cdef register_column_class(obitype_t obitype,
bint multi_elts,
bint tuples,
type obiclass,
type python)

21
python/obitools3/dms/column/column.pyx
View File

@ -1,6 +1,4 @@
#cython: language_level=3
from obitools3.dms.column import typed_column
__OBIDMS_COLUMN_CLASS__ = {}
@ -44,12 +42,12 @@ cdef class Column(OBIWrapper) :
@staticmethod
cdef type get_column_class(obitype_t obitype, bint multi_elts):
cdef type get_column_class(obitype_t obitype, bint multi_elts, bint tuples):
'''
Internal function returning the python class representing
a column for a given obitype.
'''
return __OBIDMS_COLUMN_CLASS__[(obitype, multi_elts)][0]
return __OBIDMS_COLUMN_CLASS__[(obitype, multi_elts, tuples)][0]
@staticmethod
@ -76,6 +74,7 @@ cdef class Column(OBIWrapper) :
obitype_t data_type,
index_t nb_elements_per_line=1,
list elements_names=None,
bint tuples=False,
object comments=b"",
object alias=b""):
# TODO indexer_name?
@ -115,6 +114,7 @@ cdef class Column(OBIWrapper) :
nb_lines = len(view),
nb_elements_per_line = nb_elements_per_line,
elements_names = elements_names_p,
tuples = tuples,
indexer_name = NULL,
associated_column_name = NULL,
associated_column_version = -1,
@ -150,7 +150,7 @@ cdef class Column(OBIWrapper) :
column_p = column_pp[0]
column_type = column_p.header.returned_data_type
column_class = Column.get_column_class(column_type, (column_p.header.nb_elements_per_line > 1))
column_class = Column.get_column_class(column_type, (column_p.header.nb_elements_per_line > 1), column_p.header.tuples)
column = OBIWrapper.new_wrapper(column_class, column_pp)
column._view = view
@ -184,6 +184,7 @@ cdef class Column(OBIWrapper) :
nb_lines = -1,
nb_elements_per_line = -1,
elements_names = NULL,
tuples = False,
indexer_name = NULL,
associated_column_name = NULL,
associated_column_version = -1,
@ -326,6 +327,13 @@ cdef class Column(OBIWrapper) :
raise OBIDeactivatedInstanceError()
return self.pointer().header.lines_used
# tuples property getter
@property
def tuples(self):
if not self.active() :
raise OBIDeactivatedInstanceError()
return self.pointer().header.tuples
# comments property getter
@property
def comments(self):
@ -436,6 +444,7 @@ cdef class Column_line :
cdef register_column_class(obitype_t obitype,
bint multi_elts,
bint tuples,
type obiclass,
type python_type):
'''
@ -446,7 +455,7 @@ cdef register_column_class(obitype_t obitype,
assert issubclass(obiclass, Column)
__OBIDMS_COLUMN_CLASS__[(obitype, multi_elts)] = (obiclass, python_type)
__OBIDMS_COLUMN_CLASS__[(obitype, multi_elts, tuples)] = (obiclass, python_type)
cdef register_all_column_classes() :

5
python/obitools3/dms/column/typed_column/bool.pxd
View File

@ -20,7 +20,10 @@ cdef class Column_multi_elts_bool(Column_multi_elts) :
cpdef set_item(self, index_t line_nb, object elt_id, object value)
cdef class Column_tuples_bool(Column):
cpdef object get_line(self, index_t line_nb)
cpdef set_line(self, index_t line_nb, object value)

280
python/obitools3/dms/column/typed_column/bool.pyx
View File

@ -13,14 +13,23 @@ from ...capi.obiview cimport obi_get_bool_with_elt_name_and_col_p_in_view, \
obi_get_bool_with_elt_idx_and_col_p_in_view, \
obi_set_bool_with_elt_name_and_col_p_in_view, \
obi_set_bool_with_elt_idx_and_col_p_in_view, \
obi_get_array_with_col_p_in_view, \
obi_set_array_with_col_p_in_view, \
Obiview_p
from ...capi.obidmscolumn cimport OBIDMS_column_p
from ...capi.obitypes cimport OBI_BOOL, OBIBool_NA, obibool_t
from ...capi.obitypes cimport OBI_BOOL, \
OBIBool_NA, \
OBITuple_NA, \
obibool_t
from cpython.bool cimport PyBool_FromLong
from libc.stdint cimport int32_t
from libc.stdlib cimport malloc, free
cdef class Column_bool(Column):
@ -29,10 +38,12 @@ cdef class Column_bool(Column):
object column_name,
index_t nb_elements_per_line=1,
object elements_names=None,
bint tuples=False,
object comments=b""):
return Column.new_column(view, column_name, OBI_BOOL,
nb_elements_per_line=nb_elements_per_line,
elements_names=elements_names,
tuples=tuples,
comments=comments)
cpdef object get_line(self, index_t line_nb):
@ -115,210 +126,67 @@ cdef class Column_multi_elts_bool(Column_multi_elts):
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=elt_id, error_message="Problem setting a value in a column")
cdef class Column_tuples_bool(Column):
cpdef object get_line(self, index_t line_nb) :
global obi_errno
cdef obibool_t value
cdef bint value_in_result
cdef object result
cdef int32_t i
cdef obibool_t* array
cdef int32_t value_length
result = []
array = <obibool_t*>obi_get_array_with_col_p_in_view(self._view.pointer(), self.pointer(), line_nb, &value_length)
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem getting a value from a column")
if array == OBITuple_NA :
return None
for i in range(value_length) :
value = array[i]
value_in_result = PyBool_FromLong(value)
result.append(value_in_result)
return tuple(result)
cpdef set_line(self, index_t line_nb, object value) :
global obi_errno
cdef obibool_t* array
cdef int32_t value_length
cdef int32_t i, j
cdef object e
value_length = 0
if value is not None:
for e in value:
if e is not None:
value_length+=1
if value is None or value_length == 0 :
array = <obibool_t*>OBITuple_NA
else:
array = <obibool_t*>malloc(value_length * sizeof(obibool_t))
if array == NULL:
raise Exception("Problem allocating memory for an array to store a tuple")
#raise RollbackException("Problem allocating memory for an array to store a tuple", self._view) # TODO can't import
j=0
for i in range(len(value)) :
if value[i] is not None:
array[j] = <obibool_t>(value[i])
j+=1
if obi_set_array_with_col_p_in_view(self._view.pointer(), self.pointer(), line_nb, <obibool_t*> array, sizeof(obibool_t)*8, value_length) < 0 :
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem setting a value in a column")
if array != <obibool_t*>OBITuple_NA:
free(array)
def register_class() :
register_column_class(OBI_BOOL, False, Column_bool, bool)
register_column_class(OBI_BOOL, True, Column_multi_elts_bool, bool)
register_column_class(OBI_BOOL, False, False, Column_bool, bool)
register_column_class(OBI_BOOL, True, False, Column_multi_elts_bool, bool)
register_column_class(OBI_BOOL, False, True, Column_tuples_bool, bool)
# cdef class Column_line_bool(Column_line) :
#
# cdef update_pointer(self):
# """
# Checks if the obicolumn address changed since the last call and update
# if need the `_column_p` and `_data_view` data structure fields.
# """
# cdef OBIDMS_column_p* column_pp
# column_pp = <OBIDMS_column_p*>self._pointer
# cdef OBIDMS_column_p column_p = column_pp[0]
#
# if column_p != self._column_p:
# self._column_p = column_p
# self._data_view = (<obibool_t*> (column_p.data)) + \
# self._index * column_p.header.nb_elements_per_line
#
# @staticmethod
# cdef bool obibool_t2bool(obibool_t value):
# cdef bool result
#
# if value == OBIBool_NA :
# result = None
# else :
# result = PyBool_FromLong(value)
#
# return result
#
# @staticmethod
# cdef bool2obibool_t(bool value):
# cdef obibool_t result
#
# if value is None:
# result=OBIBool_NA
# else:
# result= <obibool_t> <int> value
#
# return result
#
#
# def __init__(self, Column column, index_t line_nb) :
# """
# Creates a new `OBIDMS_column_line_bool`
#
# @param column: an OBIDMS_column instance
# @param line_nb: the line in the column
# """
#
# Column_line.__init__(self, column, line_nb)
# self.update_pointer()
#
#
#
# cpdef bool get_bool_item_by_name(self, bytes element_name) :
# """
# Returns the value associated to the name `element_name` of the current line
#
# @param element_name: a `bytes` instance containing the name of the element
#
# @return: the `bool` value corresponding to the name
# """
# cdef char* cname = element_name
# cdef obibool_t value
# global obi_errno
#
# self.update_pointer()
#
# cdef OBIDMS_column_p* column_pp
# column_pp = <OBIDMS_column_p*>self._pointer
# cdef OBIDMS_column_p column_p = column_pp[0]
#
# value = obi_column_get_obibool_with_elt_name(column_p,
# self._index,
# cname)
#
# if obi_errno > 0 :
# obi_errno = 0
# raise KeyError("Cannot access to key %s" % bytes2str(element_name))
#
# return Column_line_bool.obibool_t2bool(value)
#
#
# cpdef bool get_bool_item_by_idx(self,index_t index):
# """
# Returns the value associated to the name `element_name` of the current line
#
# @param index: a `int` instance containing the index of the element
#
# @return: the `bool` value corresponding to the name
# """
# cdef obibool_t value # @DuplicatedSignature
# global obi_errno
#
# cdef OBIDMS_column_p* column_pp
# column_pp = <OBIDMS_column_p*>self._pointer
# cdef OBIDMS_column_p column_p = column_pp[0]
#
# self.update_pointer()
#
# value = obi_column_get_obibool_with_elt_idx(column_p,
# self._index,
# index)
#
# if obi_errno > 0 :
# obi_errno = 0
# raise IndexError("Cannot access to element %d" % index)
#
# return Column_line_bool.obibool_t2bool(value)
#
#
# def __getitem__(self, object element_name) :
# cdef bytes name
# cdef int cindex
# cdef obibool_t value
# cdef type typearg = type(element_name)
# cdef bool result
#
#
# if typearg == int:
# cindex=element_name
# if cindex < 0:
# cindex = self._len - cindex
# result=self.get_bool_item_by_idx(cindex)
# elif typearg == bytes:
# result=self.get_bool_item_by_name(element_name)
# elif typearg == str:
# name = str2bytes(element_name)
# result=self.get_bool_item_by_name(name)
#
# return result
#
# cpdef set_bool_item_by_name(self,bytes element_name,bool value):
# """
# Sets the value associated to the name `element_name` of the current line
#
# @param element_name: a `bytes` instance containing the name of the element
# @param value: a `bool` instance of the new value
#
# @return: the `bool` value corresponding to the name
# """
# cdef char* cname = element_name
# cdef obibool_t cvalue
#
# self.update_pointer()
# cvalue = OBIDMS_column_line_bool.bool2obibool_t(value)
#
# if ( obi_column_set_obibool_with_elt_name((<OBIDMS_column_p*>self._pointer)[0],
# self._index,
# cname,
# cvalue) < 0 ):
# raise KeyError("Cannot access to key %s" % bytes2str(element_name))
#
# cpdef set_bool_item_by_idx(self,index_t index,bool value):
# """
# Sets the value associated to the name `element_name` of the current line
#
# @param index: a `int` instance containing the index of the element
# @param value: a `bool` instance of the new value
#
# @return: the `bool` value corresponding to the name
# """
# cdef obibool_t cvalue # @DuplicatedSignature
#
# self.update_pointer()
# cvalue = OBIDMS_column_line_bool.bool2obibool_t(value)
#
# if ( obi_column_set_obibool_with_elt_idx((<OBIDMS_column_p*>self._pointer)[0],
# self._index,
# index,
# cvalue) < 0 ):
# raise IndexError("Cannot access to item index %d" % index)
#
#
#
# def __setitem__(self, object element_name, object value):
# cdef bytes name
# cdef int cindex
# cdef type typearg = type(element_name)
# cdef bool result
#
#
# if typearg == int:
# cindex=element_name
# if cindex < 0:
# cindex = self._len - cindex
# self.set_bool_item_by_idx(cindex,value)
# elif typearg == bytes:
# self.set_bool_item_by_name(element_name,value)
# elif typearg == str:
# name = str2bytes(element_name)
# self.set_bool_item_by_name(name,value)
#
# def __repr__(self) :
# return str(self._column.get_line(self._index))
#
# def __len__(self):
# return self._len

6
python/obitools3/dms/column/typed_column/char.pxd
View File

@ -18,3 +18,9 @@ cdef class Column_multi_elts_char(Column_multi_elts) :
cpdef object get_item(self, index_t line_nb, object elt_id)
cpdef object get_line(self, index_t line_nb)
cpdef set_item(self, index_t line_nb, object elt_id, object value)
cdef class Column_tuples_char(Column):
cpdef object get_line(self, index_t line_nb)
cpdef set_line(self, index_t line_nb, object value)

82
python/obitools3/dms/column/typed_column/char.pyx
View File

@ -13,12 +13,20 @@ from ...capi.obiview cimport obi_get_char_with_elt_name_and_col_p_in_view, \
obi_get_char_with_elt_idx_and_col_p_in_view, \
obi_set_char_with_elt_name_and_col_p_in_view, \
obi_set_char_with_elt_idx_and_col_p_in_view, \
obi_get_array_with_col_p_in_view, \
obi_set_array_with_col_p_in_view, \
Obiview_p
from ...capi.obidmscolumn cimport OBIDMS_column_p
from ...capi.obitypes cimport OBI_CHAR, OBIChar_NA, obichar_t
from ...capi.obitypes cimport OBI_CHAR, \
OBIChar_NA, \
OBITuple_NA, \
obichar_t
from libc.stdint cimport int32_t
from libc.stdlib cimport malloc, free
cdef class Column_char(Column):
@ -28,11 +36,13 @@ cdef class Column_char(Column):
object column_name,
index_t nb_elements_per_line=1,
object elements_names=None,
bint tuples=False,
object comments=b""):
return Column.new_column(view, column_name, OBI_CHAR,
nb_elements_per_line=nb_elements_per_line,
elements_names=elements_names,
tuples=tuples,
comments=comments)
cpdef object get_line(self, index_t line_nb):
@ -106,7 +116,7 @@ cdef class Column_multi_elts_char(Column_multi_elts):
cpdef set_item(self, index_t line_nb, object elt_id, object value) :
global obi_errno
cdef bytes elt_name
cdef bytes elt_name
cdef obichar_t value_b
if value is None :
value_b = OBIChar_NA
@ -121,7 +131,69 @@ cdef class Column_multi_elts_char(Column_multi_elts):
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=elt_id, error_message="Problem setting a value in a column")
def register_class():
register_column_class(OBI_CHAR, False, Column_char, bytes) # TODO bytes or str?
register_column_class(OBI_CHAR, True, Column_multi_elts_char, bytes) # TODO bytes or str?
cdef class Column_tuples_char(Column):
cpdef object get_line(self, index_t line_nb) :
global obi_errno
cdef obichar_t value
cdef bytes value_in_result
cdef object result
cdef int32_t i
cdef obichar_t* array
cdef int32_t value_length
result = []
array = <obichar_t*>obi_get_array_with_col_p_in_view(self._view.pointer(), self.pointer(), line_nb, &value_length)
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem getting a value from a column")
if array == OBITuple_NA :
return None
for i in range(value_length) :
value = array[i]
value_in_result = <bytes> value
result.append(value_in_result)
return tuple(result)
cpdef set_line(self, index_t line_nb, object value) :
global obi_errno
cdef obichar_t* array
cdef int32_t value_length
cdef int32_t i, j
cdef object e
cdef obichar_t value_b
value_length = 0
if value is not None:
for e in value:
if e is not None:
value_length+=1
if value is None or value_length == 0 :
array = <obichar_t*>OBITuple_NA
else:
array = <obichar_t*>malloc(value_length * sizeof(obichar_t))
if array == NULL:
raise Exception("Problem allocating memory for an array to store a tuple")
#raise RollbackException("Problem allocating memory for an array to store a tuple", self._view) # TODO can't import
j=0
for i in range(len(value)) :
if value[i] is not None:
value_b = <obichar_t> tobytes(value[i])[0]
array[j] = value_b
j+=1
if obi_set_array_with_col_p_in_view(self._view.pointer(), self.pointer(), line_nb, <obichar_t*> array, sizeof(obichar_t)*8, value_length) < 0 :
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem setting a value in a column")
if array != <obichar_t*>OBITuple_NA:
free(array)
def register_class():
register_column_class(OBI_CHAR, False, False, Column_char, bytes)
register_column_class(OBI_CHAR, True, False, Column_multi_elts_char, bytes)
register_column_class(OBI_CHAR, False, True, Column_tuples_char, bytes)

7
python/obitools3/dms/column/typed_column/float.pxd
View File

@ -18,4 +18,9 @@ cdef class Column_multi_elts_float(Column_multi_elts) :
cpdef object get_item(self, index_t line_nb, object elt_id)
cpdef object get_line(self, index_t line_nb)
cpdef set_item(self, index_t line_nb, object elt_id, object value)
cdef class Column_tuples_float(Column):
cpdef object get_line(self, index_t line_nb)
cpdef set_line(self, index_t line_nb, object value)

78
python/obitools3/dms/column/typed_column/float.pyx
View File

@ -13,12 +13,20 @@ from ...capi.obiview cimport obi_get_float_with_elt_name_and_col_p_in_view, \
obi_get_float_with_elt_idx_and_col_p_in_view, \
obi_set_float_with_elt_name_and_col_p_in_view, \
obi_set_float_with_elt_idx_and_col_p_in_view, \
obi_get_array_with_col_p_in_view, \
obi_set_array_with_col_p_in_view, \
Obiview_p
from ...capi.obidmscolumn cimport OBIDMS_column_p
from ...capi.obitypes cimport OBI_FLOAT, OBIFloat_NA, obifloat_t
from ...capi.obitypes cimport OBI_FLOAT, \
OBIFloat_NA, \
OBITuple_NA, \
obifloat_t
from libc.stdint cimport int32_t
from libc.stdlib cimport malloc, free
cdef class Column_float(Column):
@ -28,11 +36,13 @@ cdef class Column_float(Column):
object column_name,
index_t nb_elements_per_line=1,
object elements_names=None,
bint tuples=False,
object comments=b""):
return Column.new_column(view, column_name, OBI_FLOAT,
nb_elements_per_line=nb_elements_per_line,
elements_names=elements_names,
tuples=tuples,
comments=comments)
cpdef object get_line(self, index_t line_nb):
@ -115,7 +125,67 @@ cdef class Column_multi_elts_float(Column_multi_elts):
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=elt_id, error_message="Problem setting a value in a column")
def register_class():
register_column_class(OBI_FLOAT, False, Column_float, float) # TODO why not double?
register_column_class(OBI_FLOAT, True, Column_multi_elts_float, float) # TODO why not double?
cdef class Column_tuples_float(Column):
cpdef object get_line(self, index_t line_nb) :
global obi_errno
cdef obifloat_t value
cdef double value_in_result
cdef object result
cdef int32_t i
cdef obifloat_t* array
cdef int32_t value_length
result = []
array = <obifloat_t*>obi_get_array_with_col_p_in_view(self._view.pointer(), self.pointer(), line_nb, &value_length)
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem getting a value from a column")
if array == OBITuple_NA :
return None
for i in range(value_length) :
value = array[i]
value_in_result = <double> value
result.append(value_in_result)
return tuple(result)
cpdef set_line(self, index_t line_nb, object value) :
global obi_errno
cdef obifloat_t* array
cdef int32_t value_length
cdef int32_t i, j
cdef object e
value_length = 0
if value is not None:
for e in value:
if e is not None:
value_length+=1
if value is None or value_length == 0 :
array = <obifloat_t*>OBITuple_NA
else:
array = <obifloat_t*>malloc(value_length * sizeof(obifloat_t))
if array == NULL:
raise Exception("Problem allocating memory for an array to store a tuple")
#raise RollbackException("Problem allocating memory for an array to store a tuple", self._view) # TODO can't import
j=0
for i in range(len(value)) :
if value[i] is not None:
array[j] = <obifloat_t>(value[i])
j+=1
if obi_set_array_with_col_p_in_view(self._view.pointer(), self.pointer(), line_nb, <obifloat_t*> array, sizeof(obifloat_t)*8, value_length) < 0 :
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem setting a value in a column")
if array != <obifloat_t*>OBITuple_NA:
free(array)
def register_class():
register_column_class(OBI_FLOAT, False, False, Column_float, float) # TODO why not double?
register_column_class(OBI_FLOAT, True, False, Column_multi_elts_float, float) # TODO why not double?
register_column_class(OBI_FLOAT, False, True, Column_tuples_float, float)

7
python/obitools3/dms/column/typed_column/int.pxd
View File

@ -18,4 +18,9 @@ cdef class Column_multi_elts_int(Column_multi_elts) :
cpdef object get_item(self, index_t line_nb, object elt_id)
cpdef object get_line(self, index_t line_nb)
cpdef set_item(self, index_t line_nb, object elt_id, object value)
cdef class Column_tuples_int(Column):
cpdef object get_line(self, index_t line_nb)
cpdef set_line(self, index_t line_nb, object value)

79
python/obitools3/dms/column/typed_column/int.pyx
View File

@ -13,14 +13,23 @@ from ...capi.obiview cimport obi_get_int_with_elt_name_and_col_p_in_view, \
obi_get_int_with_elt_idx_and_col_p_in_view, \
obi_set_int_with_elt_name_and_col_p_in_view, \
obi_set_int_with_elt_idx_and_col_p_in_view, \
obi_get_array_with_col_p_in_view, \
obi_set_array_with_col_p_in_view, \
Obiview_p
from ...capi.obidmscolumn cimport OBIDMS_column_p
from ...capi.obitypes cimport OBI_INT, OBIInt_NA, obiint_t
from ...capi.obitypes cimport OBI_INT, \
OBIInt_NA, \
OBITuple_NA, \
obiint_t
from cpython.int cimport PyInt_FromLong
from libc.stdint cimport int32_t
from libc.stdlib cimport malloc, free
cdef class Column_int(Column):
@ -29,11 +38,13 @@ cdef class Column_int(Column):
object column_name,
index_t nb_elements_per_line=1,
object elements_names=None,
bint tuples=False,
object comments=b""):
return Column.new_column(view, column_name, OBI_INT,
nb_elements_per_line=nb_elements_per_line,
elements_names=elements_names,
tuples=tuples,
comments=comments)
@ -118,7 +129,67 @@ cdef class Column_multi_elts_int(Column_multi_elts):
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=elt_id, error_message="Problem setting a value in a column")
def register_class():
register_column_class(OBI_INT, False, Column_int, int)
register_column_class(OBI_INT, True, Column_multi_elts_int, int)
cdef class Column_tuples_int(Column):
cpdef object get_line(self, index_t line_nb) :
global obi_errno
cdef obiint_t value
cdef double value_in_result
cdef object result
cdef int32_t i
cdef obiint_t* array
cdef int32_t value_length
result = []
array = <obiint_t*>obi_get_array_with_col_p_in_view(self._view.pointer(), self.pointer(), line_nb, &value_length)
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem getting a value from a column")
if array == OBITuple_NA :
return None
for i in range(value_length) :
value = array[i]
value_in_result = PyInt_FromLong(value)
result.append(value_in_result)
return tuple(result)
cpdef set_line(self, index_t line_nb, object value) :
global obi_errno
cdef obiint_t* array
cdef int32_t value_length
cdef int32_t i, j
cdef object e
value_length = 0
if value is not None:
for e in value:
if e is not None:
value_length+=1
if value is None or value_length == 0 :
array = <obiint_t*>OBITuple_NA
else:
array = <obiint_t*>malloc(value_length * sizeof(obiint_t))
if array == NULL:
raise Exception("Problem allocating memory for an array to store a tuple")
#raise RollbackException("Problem allocating memory for an array to store a tuple", self._view) # TODO can't import
j=0
for i in range(len(value)) :
if value[i] is not None:
array[j] = <obiint_t>(value[i])
j+=1
if obi_set_array_with_col_p_in_view(self._view.pointer(), self.pointer(), line_nb, <obiint_t*> array, sizeof(obiint_t)*8, value_length) < 0 :
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem setting a value in a column")
if array != <obiint_t*>OBITuple_NA:
free(array)
def register_class():
register_column_class(OBI_INT, False, False, Column_int, int)
register_column_class(OBI_INT, True, False, Column_multi_elts_int, int)
register_column_class(OBI_INT, False, True, Column_tuples_int, int)

9
python/obitools3/dms/column/typed_column/qual.pyx
View File

@ -25,9 +25,8 @@ from ...capi.obidmscolumn cimport OBIDMS_column_p
from ...capi.obitypes cimport OBI_QUAL, OBIQual_char_NA, OBIQual_int_NA, const_char_p
from libc.stdlib cimport free
from libc.stdlib cimport malloc, free
from libc.stdint cimport uint8_t
from libc.stdlib cimport malloc
# TODO detect type of value and call set_item_str if str or bytes
@ -44,6 +43,7 @@ cdef class Column_qual(Column_idx):
return Column.new_column(view, column_name, OBI_QUAL,
nb_elements_per_line=nb_elements_per_line,
elements_names=elements_names,
tuples=False,
comments=comments)
@ -255,8 +255,7 @@ cdef class Column_multi_elts_qual(Column_multi_elts_idx):
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=elt_id, error_message="Problem setting a value in a column")
def register_class():
register_column_class(OBI_QUAL, False, Column_qual, bytes) # TODO str? int?
register_column_class(OBI_QUAL, True, Column_multi_elts_qual, bytes) # TODO str? int?
register_column_class(OBI_QUAL, False, False, Column_qual, int) # TODO bytes?
register_column_class(OBI_QUAL, True, False, Column_multi_elts_qual, int) # bytes?

7
python/obitools3/dms/column/typed_column/seq.pxd
View File

@ -18,4 +18,9 @@ cdef class Column_multi_elts_seq(Column_multi_elts_idx) :
cpdef object get_item(self, index_t line_nb, object elt_id)
cpdef object get_line(self, index_t line_nb)
cpdef set_item(self, index_t line_nb, object elt_id, object value)
cdef class Column_tuples_seq(Column_idx):
cpdef object get_line(self, index_t line_nb)
cpdef set_line(self, index_t line_nb, object value)

89
python/obitools3/dms/column/typed_column/seq.pyx
View File

@ -15,13 +15,21 @@ from ...capi.obiview cimport obi_get_seq_with_elt_name_and_col_p_in_view, \
obi_get_seq_with_elt_idx_and_col_p_in_view, \
obi_set_seq_with_elt_name_and_col_p_in_view, \
obi_set_seq_with_elt_idx_and_col_p_in_view, \
obi_get_array_with_col_p_in_view, \
obi_set_array_with_col_p_in_view, \
obi_get_array_with_col_p_in_view, \
obi_set_array_with_col_p_in_view, \
Obiview_p
from ...capi.obidmscolumn cimport OBIDMS_column_p
from ...capi.obitypes cimport OBI_SEQ, OBISeq_NA
from ...capi.obitypes cimport OBI_SEQ, \
OBISeq_NA, \
OBITuple_NA
from libc.stdlib cimport free
from libc.stdint cimport int32_t
from libc.stdlib cimport calloc, free
from libc.string cimport strcpy
cdef class Column_seq(Column_idx):
@ -31,11 +39,13 @@ cdef class Column_seq(Column_idx):
object column_name,
index_t nb_elements_per_line=1,
object elements_names=None,
bint tuples=False,
object comments=b""):
return Column.new_column(view, column_name, OBI_SEQ,
nb_elements_per_line=nb_elements_per_line,
elements_names=elements_names,
tuples=tuples,
comments=comments)
@ -145,7 +155,76 @@ cdef class Column_multi_elts_seq(Column_multi_elts_idx):
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=elt_id, error_message="Problem setting a value in a column")
def register_class():
register_column_class(OBI_SEQ, False, Column_seq, bytes) # TODO str?
register_column_class(OBI_SEQ, True, Column_multi_elts_seq, bytes) # TODO str?
cdef class Column_tuples_seq(Column_idx):
cpdef object get_line(self, index_t line_nb) :
global obi_errno
cdef const char* value
cdef bytes value_in_result
cdef object result
cdef int32_t i
cdef const char* array
cdef int32_t value_length
result = []
array = <const char*>obi_get_array_with_col_p_in_view(self._view.pointer(), self.pointer(), line_nb, &value_length)
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem getting a value from a column")
if array == OBITuple_NA :
return None
i = 0
# First value
value_in_result = <bytes> array
result.append(value_in_result)
while i+1 < value_length :
if array[i] != b'\0' :
i+=1
else :
value = array+i+1
value_in_result = <bytes> value
result.append(value_in_result)
i+=1
return tuple(result)
cpdef set_line(self, index_t line_nb, object value) :
global obi_errno
cdef char* array
cdef int32_t value_length
cdef int32_t i
cdef object elt
cdef bytes elt_b
value_length = 0
if value is not None:
for i in range(len(value)) :
if value[i] is not None and value[i] != '' :
value_length = value_length + len(value[i]) + 1 # Total size of the array with the '\0'
if value is None or value_length == 0 :
array = <char*>OBITuple_NA
else:
array = <char*>calloc(value_length, sizeof(char))
if array == NULL:
raise Exception("Problem allocating memory for an array to store a tuple")
#raise RollbackException("Problem allocating memory for an array to store a tuple", self._view) # TODO can't import
i = 0
for elt in value :
if elt is not None and elt != '':
elt_b = tobytes(elt)
strcpy(array+i, <char*>elt_b)
i = i + len(elt_b) + 1
if obi_set_array_with_col_p_in_view(self._view.pointer(), self.pointer(), line_nb, <char*> array, sizeof(char)*8, value_length) < 0 :
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem setting a value in a column")
if array != <char*>OBITuple_NA:
free(array)
def register_class():
register_column_class(OBI_SEQ, False, False, Column_seq, bytes)
register_column_class(OBI_SEQ, True, False, Column_multi_elts_seq, bytes)
register_column_class(OBI_SEQ, False, True, Column_tuples_seq, bytes)

7
python/obitools3/dms/column/typed_column/str.pxd
View File

@ -18,4 +18,9 @@ cdef class Column_multi_elts_str(Column_multi_elts_idx) :
cpdef object get_item(self, index_t line_nb, object elt_id)
cpdef object get_line(self, index_t line_nb)
cpdef set_item(self, index_t line_nb, object elt_id, object value)
cdef class Column_tuples_str(Column_idx):
cpdef object get_line(self, index_t line_nb)
cpdef set_line(self, index_t line_nb, object value)

88
python/obitools3/dms/column/typed_column/str.pyx
View File

@ -15,11 +15,20 @@ from ...capi.obiview cimport obi_get_str_with_elt_name_and_col_p_in_view, \
obi_get_str_with_elt_idx_and_col_p_in_view, \
obi_set_str_with_elt_name_and_col_p_in_view, \
obi_set_str_with_elt_idx_and_col_p_in_view, \
obi_get_array_with_col_p_in_view, \
obi_set_array_with_col_p_in_view, \
Obiview_p
from ...capi.obidmscolumn cimport OBIDMS_column_p
from ...capi.obitypes cimport OBI_STR, OBIStr_NA, const_char_p
from ...capi.obitypes cimport OBI_STR, \
OBIStr_NA, \
OBITuple_NA, \
const_char_p
from libc.stdint cimport int32_t
from libc.stdlib cimport calloc, free
from libc.string cimport strcpy
cdef class Column_str(Column_idx):
@ -29,11 +38,13 @@ cdef class Column_str(Column_idx):
object column_name,
index_t nb_elements_per_line=1,
object elements_names=None,
bint tuples=False,
object comments=b""):
return Column.new_column(view, column_name, OBI_STR,
nb_elements_per_line=nb_elements_per_line,
elements_names=elements_names,
tuples=tuples,
comments=comments)
@ -133,7 +144,76 @@ cdef class Column_multi_elts_str(Column_multi_elts_idx):
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=elt_id, error_message="Problem setting a value in a column")
def register_class():
register_column_class(OBI_STR, False, Column_str, bytes) # TODO str?
register_column_class(OBI_STR, True, Column_multi_elts_str, bytes) # TODO str?
cdef class Column_tuples_str(Column_idx):
cpdef object get_line(self, index_t line_nb) :
global obi_errno
cdef const char* value
cdef bytes value_in_result
cdef object result
cdef int32_t i
cdef const char* array
cdef int32_t value_length
result = []
array = <const char*>obi_get_array_with_col_p_in_view(self._view.pointer(), self.pointer(), line_nb, &value_length)
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem getting a value from a column")
if array == OBITuple_NA :
return None
i = 0
# First value
value_in_result = <bytes> array
result.append(value_in_result)
while i+1 < value_length :
if array[i] != b'\0' :
i+=1
else :
value = array+i+1
value_in_result = <bytes> value
result.append(value_in_result)
i+=1
return tuple(result)
cpdef set_line(self, index_t line_nb, object value) :
global obi_errno
cdef char* array
cdef int32_t value_length
cdef int32_t i
cdef object elt
cdef bytes elt_b
value_length = 0
if value is not None:
for i in range(len(value)) :
if value[i] is not None and value[i] != '' :
value_length = value_length + len(value[i]) + 1 # Total size of the array with the '\0'
if value is None or value_length == 0 :
array = <char*>OBITuple_NA
else:
array = <char*>calloc(value_length, sizeof(char))
if array == NULL:
raise Exception("Problem allocating memory for an array to store a tuple")
#raise RollbackException("Problem allocating memory for an array to store a tuple", self._view) # TODO can't import
i = 0
for elt in value :
if elt is not None and elt != '':
elt_b = tobytes(elt)
strcpy(array+i, <char*>elt_b)
i = i + len(elt_b) + 1
if obi_set_array_with_col_p_in_view(self._view.pointer(), self.pointer(), line_nb, <char*> array, sizeof(char)*8, value_length) < 0 :
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem setting a value in a column")
if array != <char*>OBITuple_NA:
free(array)
def register_class():
register_column_class(OBI_STR, False, False, Column_str, bytes)
register_column_class(OBI_STR, True, False, Column_multi_elts_str, bytes)
register_column_class(OBI_STR, False, True, Column_tuples_str, bytes)

2
python/obitools3/dms/dms.cfiles
View File

@ -1,3 +1,4 @@
../../../src/array_indexer.c
../../../src/bloom.c
../../../src/char_str_indexer.c
../../../src/crc64.c
@ -12,6 +13,7 @@
../../../src/obiblob.c
../../../src/obidms_taxonomy.c
../../../src/obidms.c
../../../src/obidmscolumn_array.c
../../../src/obidmscolumn_blob.c
../../../src/obidmscolumn_bool.c
../../../src/obidmscolumn_char.c

2
python/obitools3/utils.cfiles
View File

@ -1,3 +1,4 @@
../../src/array_indexer.c
../../src/bloom.c
../../src/char_str_indexer.c
../../src/crc64.c
@ -21,6 +22,7 @@
../../src/obidmscolumn_qual.c
../../src/obidmscolumn_seq.c
../../src/obidmscolumn_str.c
../../src/obidmscolumn_array.c
../../src/obidmscolumn.c
../../src/obidmscolumndir.c
../../src/obierrno.c