From 704d9b04748bb6e355c1778b8585cc57a126d85c Mon Sep 17 00:00:00 2001 From: Celine Mercier Date: Sun, 7 Oct 2018 18:59:43 +0200 Subject: [PATCH] Cython: Columns: added support for JSON formatted comments --- python/obitools3/dms/column/column.pxd | 5 +- python/obitools3/dms/column/column.pyx | 117 ++++++++++++++++++++++--- python/obitools3/dms/dms.cfiles | 2 + python/obitools3/utils.cfiles | 2 + 4 files changed, 111 insertions(+), 15 deletions(-) diff --git a/python/obitools3/dms/column/column.pxd b/python/obitools3/dms/column/column.pxd index a636a6a..d182f55 100644 --- a/python/obitools3/dms/column/column.pxd +++ b/python/obitools3/dms/column/column.pxd @@ -22,13 +22,16 @@ cdef class Column(OBIWrapper) : cdef inline OBIDMS_column_p pointer(self) cdef read_elements_names(self) - + @staticmethod cdef type get_column_class(obitype_t obitype, bint multi_elts, bint tuples) @staticmethod cdef type get_python_type(obitype_t obitype, bint multi_elts) +cdef class Column_comments(dict): + cdef Column _column + cdef class Column_multi_elts(Column) : diff --git a/python/obitools3/dms/column/column.pyx b/python/obitools3/dms/column/column.pyx index 2e15201..175616b 100644 --- a/python/obitools3/dms/column/column.pyx +++ b/python/obitools3/dms/column/column.pyx @@ -13,7 +13,8 @@ from ..capi.obidms cimport obi_import_column from ..capi.obidmscolumn cimport OBIDMS_column_header_p, \ obi_close_column, \ - obi_get_elements_names + obi_get_elements_names, \ + obi_column_write_comments from ..capi.obiutils cimport obi_format_date @@ -26,7 +27,10 @@ from ..object cimport OBIDeactivatedInstanceError from obitools3.utils cimport tobytes, \ bytes2str, \ - str2bytes + str2bytes, \ + str2bytes_object, \ + bytes2str_object, \ + clean_empty_values_from_object from obitools3.dms.column import typed_column @@ -35,6 +39,8 @@ from libc.stdlib cimport free import importlib import inspect import pkgutil +import json + cdef class Column(OBIWrapper) : ''' @@ -82,13 +88,13 @@ cdef class Column(OBIWrapper) : bint to_eval=False, object associated_column_name=b"", int associated_column_version=-1, - object comments=b"", + object comments={}, object alias=b""): # TODO indexer_name? cdef bytes column_name_b = tobytes(column_name) cdef bytes alias_b = tobytes(alias) - cdef bytes comments_b = tobytes(comments) + cdef bytes comments_b = str2bytes(json.dumps(bytes2str_object(comments))) cdef bytes associated_column_name_b = tobytes(associated_column_name) cdef list elements_names_s cdef bytes elements_names_b @@ -159,7 +165,7 @@ cdef class Column(OBIWrapper) : column_pp = obi_view_get_pointer_on_column_in_view(view.pointer(), column_name_b) - + if column_pp == NULL: raise KeyError("Cannot access to column %s in view %s" % ( bytes2str(column_name_b), @@ -373,21 +379,104 @@ cdef class Column(OBIWrapper) : raise OBIDeactivatedInstanceError() return self.pointer().header.to_eval - # comments property getter - @property - def comments(self): - if not self.active() : - raise OBIDeactivatedInstanceError() - return self.pointer().header.comments - # creation_date property getter @property def creation_date(self): if not self.active() : raise OBIDeactivatedInstanceError() return obi_format_date(self.pointer().header.creation_date) - - + + # comments property getter + @property + def comments(self): + return Column_comments(self) + @comments.setter + def comments(self, object value): + Column_comments(self, value) + + +cdef class Column_comments(dict): # Not thread safe + def __init__(self, Column column, value=None) : + if not column.active() : + raise OBIDeactivatedInstanceError() + self._column = column + if value is not None: + self.update(value) # TODO test and discuss not overwriting (could use replace bool) + self._update_from_file() + + def _update_from_file(self): + cdef bytes comments_json + cdef str comments_json_str + cdef OBIDMS_column_p column_p + cdef Column column + if not self._column.active() : + raise OBIDeactivatedInstanceError() + column = self._column + column_p = (column.pointer()) + comments_json = column_p.header.comments + comments_json_str = bytes2str(comments_json) + comments_dict = json.loads(comments_json_str) + str2bytes_object(comments_dict) + super(Column_comments, self).update(comments_dict) + + def __getitem__(self, object key): + if not self._column.active() : + raise OBIDeactivatedInstanceError() + if type(key) == str: + key = str2bytes(key) + self._update_from_file() + return super(Column_comments, self).__getitem__(key) + + def __setitem__(self, object key, object value): + cdef OBIDMS_column_p column_p + cdef Column column + + if not self._column.active() : + raise OBIDeactivatedInstanceError() + + column = self._column + column_p = (column.pointer()) + + # Remove virtually empty values from the object # TODO discuss + clean_empty_values_from_object(value) + + # If value is virtually empty, don't add it # TODO discuss + if value is None or len(value) == 0: + return + + # Convert to bytes + if type(key) == str: + key = str2bytes(key) + value_bytes = str2bytes_object(value) + + # Update dict with comments already written in file + self._update_from_file() + + # Add new element # TODO don't overwrite? + super(Column_comments, self).__setitem__(key, value_bytes) + + # Convert to str because json library doens't like bytes + dict_str = {key:item for key,item in self.items()} + dict_str = bytes2str_object(dict_str) + + # Convert to json string + comments_json = json.dumps(dict_str) + + # Write new comments + if obi_column_write_comments(column_p, tobytes(comments_json)) < 0: + raise Exception("Could not write column comments: %s", comments_json) + + def update(self, value): + for k,v in value.items(): + self[k] = v + + def __contains__(self, key): + return super(Column_comments, self).__contains__(tobytes(key)) + + def __str__(self): + return bytes2str(self._column.pointer().header.comments) + + ###################################################################################################### diff --git a/python/obitools3/dms/dms.cfiles b/python/obitools3/dms/dms.cfiles index 04a31d7..6fc9329 100644 --- a/python/obitools3/dms/dms.cfiles +++ b/python/obitools3/dms/dms.cfiles @@ -18,6 +18,8 @@ ../../../src/libecoPCR/ecodna.c ../../../src/libecoPCR/ecoError.c ../../../src/libecoPCR/ecoMalloc.c +../../../src/libjson/cJSON.c +../../../src/libjson/json_utils.c ../../../src/obiavl.c ../../../src/obiblob_indexer.c ../../../src/obiblob.c diff --git a/python/obitools3/utils.cfiles b/python/obitools3/utils.cfiles index 1e6c323..7e738af 100644 --- a/python/obitools3/utils.cfiles +++ b/python/obitools3/utils.cfiles @@ -18,6 +18,8 @@ ../../src/libecoPCR/ecodna.c ../../src/libecoPCR/ecoError.c ../../src/libecoPCR/ecoMalloc.c +../../src/libjson/cJSON.c +../../src/libjson/json_utils.c ../../src/obiavl.c ../../src/obiblob_indexer.c ../../src/obiblob.c