Cython: Views: JSON formatted comments and history handling

python/obitools3/dms/capi/obiview.pxd

@@ -103,6 +103,10 @@ cdef extern from "obiview.h" nogil:
 
     int obi_view_create_column_alias(Obiview_p view, const_char_p current_name, const_char_p alias)    
     
+    int obi_view_write_comments(Obiview_p view, const_char_p comments)
+
+    int obi_view_add_comment(Obiview_p view, const_char_p key, const_char_p value)
+
     int obi_save_and_close_view(Obiview_p view)
 
     int obi_clean_unfinished_views(OBIDMS_p dms)

python/obitools3/dms/view/typed_view/view_NUC_SEQS.pyx

@@ -12,12 +12,13 @@ from obitools3.dms.dms cimport DMS
 
 from obitools3.dms.capi.obitypes cimport index_t
 
-from obitools3.utils cimport tobytes, bytes2str
+from obitools3.utils cimport tobytes, bytes2str, str2bytes
 
 from obitools3.dms.capi.obidms cimport OBIDMS_p
 
 from obitools3.dms.object cimport OBIWrapper                          
 
+import json
 
 
 cdef class View_NUC_SEQS(View):
@@ -25,7 +26,7 @@ cdef class View_NUC_SEQS(View):
     @staticmethod
     def new(DMS dms, 
             object view_name, 
-            object comments=None,
+            dict comments={},
             bint quality=False):
 
         cdef bytes view_name_b = tobytes(view_name)
@@ -35,10 +36,7 @@ cdef class View_NUC_SEQS(View):
 
         cdef View_NUC_SEQS view
 
-        if comments is not None:
-            comments_b = tobytes(comments)
-        else:
-            comments_b = b''
+        comments_b = str2bytes(json.dumps(comments))
 
         pointer = <void*>obi_new_view_nuc_seqs(<OBIDMS_p>dms._pointer, 
                                                view_name_b, 

python/obitools3/dms/view/view.pxd

@@ -38,6 +38,10 @@ cdef class View(OBIWrapper):
                                        list lines=*)
 
 
+cdef class View_comments(dict):
+    cdef View _view
+    
+    
 cdef class Line_selection(list):
     
     cdef View  _view

python/obitools3/dms/view/view.pyx

@@ -14,7 +14,8 @@ from ..capi.obiview cimport Alias_column_pair_p, \
                             obi_save_and_close_view, \
                             obi_view_get_pointer_on_column_in_view, \
                             obi_view_delete_column, \
-                            obi_view_create_column_alias
+                            obi_view_create_column_alias, \
+                            obi_view_write_comments
                            
 from ..capi.obidmscolumn cimport OBIDMS_column_p
 from ..capi.obidms cimport OBIDMS_p
@@ -22,7 +23,10 @@ from ..capi.obidms cimport OBIDMS_p
 from obitools3.utils cimport tobytes, \
                              str2bytes, \
                              bytes2str, \
-                             tostr
+                             tostr, \
+                             bytes2str_object, \
+                             str2bytes_object, \
+                             clean_empty_values_from_object
    
 from ..object cimport OBIDeactivatedInstanceError
 
@@ -43,6 +47,7 @@ from ..capi.obidms cimport obi_import_view
 import importlib
 import inspect
 import pkgutil
+import json
 
 
 cdef class View(OBIWrapper) :
@@ -66,7 +71,7 @@ cdef class View(OBIWrapper) :
     @staticmethod
     def new(DMS dms, 
             object view_name, 
-            object comments=None):
+            object comments={}):
 
         cdef bytes view_name_b = tobytes(view_name)
         cdef bytes comments_b
@@ -75,10 +80,7 @@ cdef class View(OBIWrapper) :
 
         cdef View view  # @DuplicatedSignature
 
-        if comments is not None:
-            comments_b  = tobytes(comments)
-        else:
-            comments_b  = b''
+        comments_b = str2bytes(json.dumps(bytes2str_object(comments)))
 
         pointer = <void*>obi_new_view(<OBIDMS_p>dms._pointer, 
                                       view_name_b, 
@@ -99,7 +101,7 @@ cdef class View(OBIWrapper) :
           
     def clone(self, 
               object view_name, 
-              object comments=None):
+              object comments={}):
                           
         cdef bytes view_name_b = tobytes(view_name)
         cdef bytes comments_b
@@ -109,10 +111,7 @@ cdef class View(OBIWrapper) :
         if not self.active() :
             raise OBIDeactivatedInstanceError()
         
-        if comments is not None:
-            comments_b  = tobytes(comments)
-        else:
-            comments_b  = b''
+        comments_b = str2bytes(json.dumps(bytes2str_object(dict(comments))))  # TODO hmmmmm function in View_comments class probably
                  
         pointer = <void*> obi_clone_view(self._dms.pointer(), 
                                          self.pointer(), 
@@ -366,6 +365,175 @@ cdef class View(OBIWrapper) :
         return to_print
 
     
+    @staticmethod
+    def _config_to_dict(dict config, str command_name, str command_line, list input_str=None, list input_dms_name=None, list input_view_name=None):
+        INVALID_KEYS = ["__root_config__", "module", "nocreatedms", "logger", "defaultdms", "inputview", "outputview", "log", "loglevel", "progress"]
+        comments = {}
+        comments["obi"] = {k: config["obi"][k] for k in config["obi"] if k not in INVALID_KEYS}
+        comments[command_name] = config[command_name]      # TODO or discuss update instead of nested dict
+        comments["command_line"] = command_line
+        if input_str is None and input_dms_name is None and input_view_name is None:
+            raise Exception("Can't build view configuration with None input")  # TODO discuss
+        if (input_dms_name is not None and input_view_name is not None and len(input_dms_name) != len(input_view_name)) or \
+            (input_dms_name is None and input_view_name is not None) or \
+            (input_dms_name is not None and input_view_name is None):
+            raise Exception("Error building view configuration: there must be as many input DMS names as input view names")  # TODO discuss
+        comments["input_dms_name"] = input_dms_name
+        comments["input_view_name"] = input_view_name
+        if input_str is None:
+            input_str = []
+            for i in range(len(input_view_name)):
+                input_str.append(tostr(input_dms_name[i])+"/"+tostr(input_view_name[i]))
+        comments["input_str"] = input_str
+        return bytes2str_object(comments)
+
+    
+    @staticmethod
+    def print_config(dict config, str command_name, str command_line, list input_str=None, list input_dms_name=None, list input_view_name=None):
+        config_dict = View._config_to_dict(config, command_name, command_line, \
+                                           input_str=input_str, input_dms_name=input_dms_name, input_view_name=input_view_name)
+        # Clean virtually empty values
+        config_dict = clean_empty_values_from_object(config_dict, exclude=[View_comments.KEEP_KEYS])
+        # Convert to json string
+        comments_json = json.dumps(config_dict)
+        return str2bytes(comments_json)
+        
+    
+    @OBIWrapper.checkIsActive
+    def write_config(self, dict config, str command_name, str command_line, list input_str=None, list input_dms_name=None, list input_view_name=None):
+        self.comments = View._config_to_dict(config, command_name, command_line, \
+                                           input_str=input_str, input_dms_name=input_dms_name, input_view_name=input_view_name)
+        
+
+    # command and view history DOT graph property getter in the form of a list (to remove duplicate elements afterwards)
+    @property
+    def dot_history_graph_list(self):
+        history = []
+        view_history = self.view_history
+        history.append(b"\tnode [shape=record]\n")
+        history.append(b"\tcompound=true\n")
+        for i in range(len(view_history)):
+            level = view_history[i]
+            for input in level:
+                # Command node
+                command = b"\""+level[input][b"command_line"]+b"\""
+                s = b"\t"
+                s+=command
+                s+=b" [style=filled, color=lightblue]\n"
+                history.append(s)
+                if len(input) > 1:
+                    # Create invisible node
+                    invi_node_name_no_quotes = b"_".join(input)
+                    invi_node_name_quotes = b"\""+ invi_node_name_no_quotes + b"\""
+                    s = b"\t"
+                    s+=invi_node_name_quotes
+                    s+=b" [width=0, style=invis, shape=point]\n"
+                    history.append(s)
+                    for input in level:
+                        # Connect all inputs to the invisible node
+                        for elt in input:
+                            s = b"\t"
+                            s = s+b"\""+elt+b"\""
+                            s+=b" -> "
+                            s+=invi_node_name_quotes
+                            s+=b" [arrowhead=none]\n"
+                            history.append(s)
+                    to_connect_to_command = invi_node_name_no_quotes
+                else:
+                    to_connect_to_command = input[0]
+                # Color node if input element is a taxonomy (to do for output nodes too if taxonomy history is to be recorded)
+                for elt in input:
+                    if b"taxonomy" in elt:
+                        s = b"\t"
+                        s = s+b"\""+elt+b"\""
+                        s+=b" [style=filled, color=navajowhite]\n"
+                        history.append(s)
+                # Connect input to command
+                s = b"\t"
+                s = s+b"\""+to_connect_to_command+b"\""
+                s+=b" -> "
+                s+=command
+                s+=b"\n"
+                history.append(s)
+                # Connect command to output
+                s = b"\t"
+                s+=command
+                s+=b" -> "
+                s = s+b"\""+level[input][b"output"]+b"\""
+                s+=b"\n"
+                history.append(s)
+        return history
+
+
+    # command history DOT graph property getter in the form of a bytes string
+    @property
+    def dot_history_graph(self):
+        uniq_graph = []
+        for elt in self.history_graph_list:
+            if elt not in uniq_graph:
+                uniq_graph.append(elt)
+        uniq_graph.insert(0, b"digraph \""+self.name+b"\" {\n")
+        uniq_graph.append(b"}")
+        return b"".join(uniq_graph)
+
+
+    # ASCII command history graph property getter
+    @property
+    def ascii_history(self):
+        arrow = b"\t|\n\tV\n"
+        s = b""
+        first = True
+        for level in self.view_history:
+            command_list = [level[input][b"command_line"] for input in level.keys()]
+            if not first:
+                s+=arrow
+            else:
+                first=False
+            for command in command_list:
+                s+=command
+                s+=b"\n"
+        return s
+
+
+    # bash command history property getter
+    @property
+    def bash_history(self):
+        s = b"#!/bin/bash\n\n"
+        first = True
+        for level in self.view_history:
+            command_list = [level[input][b"command_line"] for input in level.keys()]
+            for command in command_list:
+                s+=command
+                s+=b"\n"
+        return s
+
+
+    # View and command history property getter
+    @property
+    def view_history(self):
+        if not self.active() :
+            raise OBIDeactivatedInstanceError()
+        current_level = [self]
+        history = []
+        while current_level[0] is not None:   # TODO not sure about robustness
+            top_level = []
+            level_dict = {}
+            for element in current_level:
+                if element is not None:
+                    if element.comments[b"input_dms_name"] is not None :
+                        for i in range(len(element.comments[b"input_dms_name"])) :
+                            if element.comments[b"input_dms_name"][i] == element.dms.name and b"/" not in element.comments[b"input_view_name"][i]:  # Same DMS and not a special element like a taxonomy
+                                top_level.append(element.dms[element.comments[b"input_view_name"][i]])
+                    else:
+                        top_level.append(None)
+                    level_dict[tuple(element.comments[b"input_str"])] = {}
+                    level_dict[tuple(element.comments[b"input_str"])][b"output"] = element.dms.name+b"/"+element.name
+                    level_dict[tuple(element.comments[b"input_str"])][b"command_line"] = element.comments[b"command_line"]
+            history.insert(0, level_dict)
+            current_level = top_level
+        return history
+
+
     # Width (column count) property getter
     @property
     def width(self):
@@ -389,7 +557,7 @@ cdef class View(OBIWrapper) :
             raise OBIDeactivatedInstanceError()
         return self.pointer().infos.line_count
 
-    # line_count property getter
+    # read_only state property getter
     @property
     def read_only(self):
         if not self.active() :
@@ -403,7 +571,6 @@ cdef class View(OBIWrapper) :
             raise OBIDeactivatedInstanceError()
         return <bytes> self.pointer().infos.name
 
-
     # view type property getter
     @property
     def type(self):  # @ReservedAssignment
@@ -411,14 +578,98 @@ cdef class View(OBIWrapper) :
             raise OBIDeactivatedInstanceError()
         return <bytes> self.pointer().infos.view_type
 
-
     # comments property getter
     @property
     def comments(self):
-        if not self.active() :
+        return View_comments(self)
+    @comments.setter
+    def comments(self, object value):
+        View_comments(self, value)
+        
+                  
+cdef class View_comments(dict):   # Not thread safe
+
+    KEEP_KEYS = [b"input_dms_name", b"input_view_name", b"input_str", "input_dms_name", "input_view_name", "input_str"]
+    
+    def __init__(self, View view, value=None) :
+        if not view.active() :
             raise OBIDeactivatedInstanceError()
-        return <bytes> self.pointer().infos.comments
-    # TODO setter that concatenates new comments?
+        self._view = view
+        if value is not None:
+            self.update(value)     # TODO test and discuss not overwriting (could use replace bool)
+        self._update_from_file()
+    
+    def _update_from_file(self):
+        cdef bytes comments_json
+        cdef str comments_json_str
+        cdef Obiview_p view_p
+        cdef View view
+        if not self._view.active() :
+            raise OBIDeactivatedInstanceError()
+        view = self._view
+        view_p = <Obiview_p>(view.pointer())
+        comments_json = <bytes> view_p.infos.comments
+        comments_json_str = bytes2str(comments_json)
+        comments_dict = json.loads(comments_json_str)
+        str2bytes_object(comments_dict)
+        super(View_comments, self).update(comments_dict)
+        
+    def __getitem__(self, object key):
+        if not self._view.active() :
+            raise OBIDeactivatedInstanceError()
+        if type(key) == str:
+            key = str2bytes(key)
+        self._update_from_file()
+        return super(View_comments, self).__getitem__(key)
+
+    def __setitem__(self, object key, object value):
+        cdef Obiview_p view_p
+        cdef View view
+
+        if not self._view.active() :
+            raise OBIDeactivatedInstanceError()
+
+        view = self._view
+        view_p = <Obiview_p>(view.pointer())
+         
+        # Remove virtually empty values from the object    # TODO discuss
+        clean_empty_values_from_object(value, exclude=[self.KEEP_KEYS])
+        
+        # If value is virtually empty, don't add it    # TODO discuss
+        if (key not in self.KEEP_KEYS) and (value is None or len(value) == 0):
+            return
+        
+        # Convert to bytes
+        if type(key) == str:
+            key = str2bytes(key)
+        value_bytes = str2bytes_object(value)
+
+        # Update dict with comments already written in file
+        self._update_from_file()
+       
+        # Add new element  # TODO don't overwrite?
+        super(View_comments, self).__setitem__(key, value_bytes)
+                
+        # Convert to str because json library doens't like bytes
+        dict_str = {key:item for key,item in self.items()}
+        dict_str = bytes2str_object(dict_str)
+        
+        # Convert to json string
+        comments_json = json.dumps(dict_str)
+
+        # Write new comments
+        if obi_view_write_comments(view_p, tobytes(comments_json)) < 0:
+            raise Exception("Could not write view comments, view:", view.name, "comments:", comments_json)
+
+    def update(self, value):
+        for k,v in value.items():
+            self[k] = v
+
+    def __contains__(self, key):
+        return super(View_comments, self).__contains__(tobytes(key))
+    
+    def __str__(self):
+        return bytes2str(self._view.pointer().infos.comments)
 
 
 cdef class Line :
@@ -601,7 +852,7 @@ cdef class Line_selection(list):
         
     cpdef View materialize(self,
                            object view_name,
-                           object comments=b""):
+                           object comments={}):
 
         cdef bytes view_name_b = tobytes(view_name)
         cdef bytes comments_b
@@ -611,10 +862,7 @@ cdef class Line_selection(list):
         if not self._view.active() :
             raise OBIDeactivatedInstanceError()
 
-        if comments is not None:
-            comments_b  = tobytes(comments)
-        else:
-            comments_b  = b''
+        comments_b = str2bytes(json.dumps(bytes2str_object(comments)))
     
         pointer = obi_clone_view(self._view._dms.pointer(), 
                                  self._view.pointer(),