Commands: updated for JSON formatted comments with history

python/obitools3/commands/alignpairedend.pyx

@@ -16,6 +16,9 @@ from obitools3.align._qsrassemble import QSolexaRightReverseAssemble
 from obitools3.align._solexapairend import buildConsensus, buildJoinedSequence
 from obitools3.dms.obiseq cimport Nuc_Seq_Stored, Nuc_Seq
 
+import sys
+import os
+
 
 REVERSE_SEQ_COLUMN_NAME = b"REVERSE_SEQUENCE"
 REVERSE_QUALITY_COLUMN_NAME = b"REVERSE_QUALITY"
@@ -112,7 +115,7 @@ def run(config):
     DMS.obi_atexit()
     
     logger("info", "obi alignpairedend")
-    
+       
     # Open the input
     
     two_views = False
@@ -124,8 +127,8 @@ def run(config):
     if input is None:
         raise Exception("Could not open input reads")
     if input[2] != View_NUC_SEQS:
-        raise NotImplementedError('obi alignpairedend only works on NUC_SEQS views')
+        raise NotImplementedError('obi alignpairedend only works on NUC_SEQS views')    
-
+    
     if "reverse" in config["alignpairedend"]:
         
         two_views = True
@@ -144,10 +147,14 @@ def run(config):
             raise Exception("Error: the number of forward and reverse reads are different")
         
         entries = [forward, reverse]
+        input_dms_name = [forward.dms.name, reverse.dms.name]
+        input_view_name = [forward.name, reverse.name]
     
     else:
         entries = input[1]
-    
+        input_dms_name = [entries.dms.name]
+        input_view_name = [entries.name]
+
     if two_views:
         entries_len = len(forward)
     else:
@@ -166,6 +173,7 @@ def run(config):
         raise Exception("Could not create output view")
 
     view = output[1]
+    
     Column.new_column(view, b"QUALITY", OBI_QUAL)   #TODO output URI quality option
 
     if 'smin' in config['alignpairedend']:
@@ -213,7 +221,12 @@ def run(config):
 #            consensus[b"illumina_index"] = idx
     
         i+=1
-    
+
+    # Save command config in View and DMS comments
+    command_line = " ".join(sys.argv[1:])
+    view.write_config(config, "alignpairedend", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name)
+    output[0].record_command_line(command_line)
+
     print("\n")
     print(repr(view))
     

python/obitools3/commands/annotate.pyx

@@ -9,16 +9,16 @@ from obitools3.dms.view import RollbackException
 from functools import reduce
 from obitools3.apps.config import logger
 from obitools3.utils cimport tobytes
-from obitools3.dms.capi.obiview cimport QUALITY_COLUMN
-import time
-import math 
-
 from obitools3.dms.capi.obiview cimport NUC_SEQUENCE_COLUMN, \
                                         ID_COLUMN, \
                                         DEFINITION_COLUMN, \
                                         QUALITY_COLUMN, \
                                         COUNT_COLUMN
 
+import time
+import math 
+import sys
+
  
 __title__="Annotate views with new tags and edit existing annotations"
 
@@ -285,10 +285,10 @@ def run(config):
         output_view_name = uri[0]
     
     # Clone output view from input view
-    o_view = i_view.clone(output_view_name, comments=i_view.comments+b"\nobi annotate")  # TODO comments
+    o_view = i_view.clone(output_view_name, comments=i_view.comments)  # TODO comments
     if o_view is None:
         raise Exception("Couldn't create output view")
-    
+
     if 'taxoURI' in config['obi'] :   # TODO default None problem
         taxo_uri = open_uri(config['obi']['taxoURI'])
         if taxo_uri is None:
@@ -341,6 +341,16 @@ def run(config):
 
     except Exception, e:
         raise RollbackException("obi annotate error, rollbacking view: "+str(e), o_view)
+    
+    # Save command config in View and DMS comments
+    command_line = " ".join(sys.argv[1:])
+    input_dms_name=[input[0].name]
+    input_view_name=[input[1].name]
+    if 'taxoURI' in config['obi'] and config['obi']['taxoURI'] is not None:
+        input_dms_name.append(config['obi']['taxoURI'].split("/", 1)[0])
+        input_view_name.append(config['obi']['taxoURI'].split("/", 1)[1])
+    o_view.write_config(config, "annotate", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name)
+    input[0].record_command_line(command_line)   # TODO assuming same dms
 
     print("\n")
     print(repr(o_view))

python/obitools3/commands/clean.pyx

@@ -9,9 +9,11 @@ from obitools3.apps.optiongroups import addSequenceInputOption, addMinimalOutput
 from obitools3.uri.decode import open_uri
 from obitools3.apps.config import logger
 from obitools3.utils cimport tobytes
-
+from obitools3.dms.view.view cimport View
 from obitools3.dms.view.typed_view.view_NUC_SEQS cimport View_NUC_SEQS
 
+import sys
+
 
 __title__="Tag a set of sequences for PCR and sequencing errors identification"
 
@@ -82,11 +84,18 @@ def run(config):
         o_view_name = uri_o[1]
     else:
         o_view_name = uri_o[0]
-    
+
-    if obi_clean(tobytes(dms_name), tobytes(i_view_name), tobytes(config['clean']['sample-tag-name']), tobytes(o_view_name), b"obiclean", \
+    # Save command config in View comments
+    command_line = " ".join(sys.argv[1:])
+    comments = View.print_config(config, "clean", command_line, input_dms_name=[dms_name], input_view_name=[i_view_name])
+
+    if obi_clean(tobytes(dms_name), tobytes(i_view_name), tobytes(config['clean']['sample-tag-name']), tobytes(o_view_name), comments, \
               config['clean']['distance'], config['clean']['ratio'], config['clean']['heads-only'], 1) < 0:
         raise Exception("Error running obiclean")
 
+    # Save command config in DMS comments
+    dms.record_command_line(command_line)
+
     print("\n")
     print(repr(dms[o_view_name]))
 

python/obitools3/commands/ecopcr.pyx

@@ -10,10 +10,13 @@ from obitools3.uri.decode import open_uri
 from obitools3.apps.config import logger
 from obitools3.utils cimport tobytes
 from obitools3.dms.view.typed_view.view_NUC_SEQS cimport View_NUC_SEQS
+from obitools3.dms.view import View
 
 from libc.stdlib  cimport malloc, free
 from libc.stdint  cimport int32_t
 
+import sys
+
 
 __title__="in silico PCR"
 
@@ -166,10 +169,14 @@ def run(config):
     # Read taxonomy name    
     taxonomy_name = config['obi']['taxoURI'].split('/')[2]
 
-    # TODO: input DMS, taxonomy and primers in comments
+    # Save command config in View comments
+    command_line = " ".join(sys.argv[1:])
+    comments = View.print_config(config, "ecopcr", command_line, input_dms_name=[i_dms_name], input_view_name=[i_view_name, config['obi']['taxoURI']])
+
+    # TODO: primers in comments?
 
     if obi_ecopcr(tobytes(i_dms_name), tobytes(i_view_name), tobytes(taxonomy_name), \
-                  tobytes(o_dms_name), tobytes(o_view_name), b"ecopcr", \
+                  tobytes(o_dms_name), tobytes(o_view_name), comments, \
                   tobytes(config['ecopcr']['primer1']), tobytes(config['ecopcr']['primer2']), \
                   config['ecopcr']['error'], \
                   config['ecopcr']['min-length'], config['ecopcr']['max-length'], \
@@ -177,7 +184,10 @@ def run(config):
                   config['ecopcr']['circular'], config['ecopcr']['salt-concentration'], config['ecopcr']['salt-correction-method'], \
                   config['ecopcr']['keep-nucs'], config['ecopcr']['kingdom-mode']) < 0:
         raise Exception("Error running ecopcr")
-    
+
+    # Save command config in DMS comments
+    o_dms.record_command_line(command_line)
+
     free(restrict_to_taxids_p)
     free(ignore_taxids_p)
     

python/obitools3/commands/grep.pyx

@@ -12,7 +12,8 @@ from obitools3.utils cimport tobytes
 from functools import reduce
 import time
 import re
- 
+import sys
+
  
 __title__="Grep view lines that match the given predicates"
 
@@ -29,7 +30,7 @@ def addOptions(parser):
     group=parser.add_argument_group("obi grep specific options")
  
     group.add_argument("--predicate", "-p",
-                       action="append", dest="grep:predicates",
+                       action="append", dest="grep:grep_predicates",
                        metavar="<PREDICATE>",
                        default=None,
                        type=str,
@@ -306,12 +307,22 @@ def run(config):
 
     # Create output view with the line selection
     try:
-        o_view = selection.materialize(output_view_name, comments="obi grep: "+str(config["grep"])+"\n")
+        o_view = selection.materialize(output_view_name)
     except Exception, e:
         raise RollbackException("obi grep error, rollbacking view: "+str(e), o_view)
 
     # TODO DISCUSS if output URI to different DMS, copy view?
 
+    # Save command config in View and DMS comments
+    command_line = " ".join(sys.argv[1:])
+    input_dms_name=[input[0].name]
+    input_view_name=[input[1].name]
+    if 'taxoURI' in config['obi'] and config['obi']['taxoURI'] is not None:
+        input_dms_name.append(config['obi']['taxoURI'].split("/", 1)[0])
+        input_view_name.append(config['obi']['taxoURI'].split("/", 1)[1])
+    o_view.write_config(config, "grep", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name)
+    input[0].record_command_line(command_line)  # TODO assuming input and output dms are the same
+
     print("\n")
     print(repr(o_view))
 

python/obitools3/commands/head.pyx

@@ -9,6 +9,7 @@ from obitools3.dms.view import RollbackException
 from obitools3.apps.config import logger
 
 import time
+import sys
  
 
 __title__="Keep the N first lines of a view."
@@ -39,6 +40,7 @@ def run(config):
     input = open_uri(config['obi']['inputURI'])
     if input is None:
         raise Exception("Could not read input view")
+    i_dms = input[0]
     i_view = input[1]
 
     # Read the name of the output view
@@ -60,15 +62,22 @@ def run(config):
     
     for i in range(n):
         selection.append(i)
-    
+
+    # Save command config in View comments
+    command_line = " ".join(sys.argv[1:])
+    comments = View.print_config(config, "head", command_line, input_dms_name=[i_dms.name], input_view_name=[i_view.name])
+
     # Create output view with the line selection
     try:
-        o_view = selection.materialize(output_view_name, comments="obi head: "+str(len(selection))+"\n")
+        o_view = selection.materialize(output_view_name, comments=comments)
     except Exception, e:
         raise RollbackException("obi head error, rollbacking view: "+str(e), o_view)
 
     # TODO DISCUSS if output URI to different DMS, copy view?
 
+    # Save command config in DMS comments
+    i_dms.record_command_line(command_line)
+
     print("\n")
     print(repr(o_view))
 

python/obitools3/commands/import.pyx

@@ -1,6 +1,7 @@
 #cython: language_level=3
 
 import sys
+import os
 
 from obitools3.apps.progress cimport ProgressBar  # @UnresolvedImport
 from obitools3.dms.view.view cimport View
@@ -89,7 +90,7 @@ def run(config):
     
     logger("info", "obi import : imports file into a DMS")
     
-    if not config['obi']['taxdump']:   # TODO discuss
+    if not config['obi']['taxdump']:
         input = open_uri(config['obi']['inputURI'])
         if input is None:  # TODO check for bytes instead now?
             raise Exception("Could not open input URI")
@@ -108,12 +109,13 @@ def run(config):
                       #quality=get_quality)  # TODO
     if output is None:
         raise Exception("Could not create output view")
-
+        
     # Read taxdump
     if config['obi']['taxdump']:  # The input is a taxdump to import in a DMS
         taxo = Taxonomy.open_taxdump(output[0], config['obi']['inputURI'])
         taxo.write(output[1])
         taxo.close()
+        output[0].record_command_line(" ".join(sys.argv[1:]))
         output[0].close()
         return
         
@@ -132,16 +134,16 @@ def run(config):
         raise NotImplementedError()
     
     # Save basic columns in variables for optimization
-    if NUC_SEQS_view :      
+    if NUC_SEQS_view :
         id_col = view[b"ID"]            # TODO use macros or globals for column names
         def_col = view[b"DEFINITION"]
         seq_col = view[b"NUC_SEQ"]
-     
+    
     dcols = {}
-     
+    
     i = 0
     for entry in entries :
-            
+        
         pb(i)
                 
         if NUC_SEQS_view :
@@ -258,6 +260,11 @@ def run(config):
                                     
         i+=1
  
+    # Save command config in View and DMS comments
+    command_line = " ".join(sys.argv[1:])
+    view.write_config(config, "import", command_line, input_str=[os.path.abspath(config['obi']['inputURI'])])
+    output[0].record_command_line(command_line)
+
     print("\n")
     print(view.__repr__())
   

python/obitools3/commands/ngsfilter.pyx

@@ -14,6 +14,7 @@ from obitools3.dms.capi.obitypes cimport OBI_SEQ, OBI_QUAL
 
 from functools import reduce, cmp_to_key
 import math
+import sys
 
 
 REVERSE_SEQ_COLUMN_NAME = b"REVERSE_SEQUENCE"
@@ -436,9 +437,15 @@ def run(config):
         
         entries = [forward, reverse]
         not_aligned = True
+
+        input_dms_name = [forward.dms.name, reverse.dms.name]
+        input_view_name = [forward.name, reverse.name]
     
     else:
         entries = input[1]
+        input_dms_name = [entries.dms.name]
+        input_view_name = [entries.name]
+
     
     if not_aligned:
         entries_len = len(forward)
@@ -460,6 +467,8 @@ def run(config):
     if info_input is None:
         raise Exception("Could not read the view containing the informations about the tags and the primers")
     info_view = info_input[1]
+    input_dms_name.append(info_input[0].name)   
+    input_view_name.append(info_input[1].name)
 
     # Open the unidentified view
     if 'unidentified' in config['ngsfilter'] and config['ngsfilter']['unidentified'] is not None:  # TODO keyError if undefined problem
@@ -504,6 +513,13 @@ def run(config):
     except Exception, e:
         raise RollbackException("obi ngsfilter error, rollbacking views: "+str(e), o_view, unidentified)
     
+    # Save command config in View and DMS comments
+    command_line = " ".join(sys.argv[1:])
+    o_view.write_config(config, "ngsfilter", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name)
+    unidentified.write_config(config, "ngsfilter", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name)
+    # TODO add comment about unidentified seqs
+    output[0].record_command_line(command_line)   # TODO if same dms...
+    
     print("\n")
     print(repr(o_view))
 

python/obitools3/commands/sort.pyx

@@ -20,7 +20,8 @@ from obitools3.dms.capi.obitypes cimport OBI_BOOL, \
                                          OBIInt_NA
 
 import time
- 
+import sys
+
  
 NULL_VALUE = {OBI_BOOL: OBIBool_NA, 
               OBI_CHAR: OBIChar_NA, 
@@ -105,6 +106,11 @@ def run(config):
 
     # TODO DISCUSS if output URI to different DMS, copy view?
 
+    # Save command config in View and DMS comments
+    command_line = " ".join(sys.argv[1:])
+    o_view.write_config(config, "sort", command_line, input_dms_name=[input[0].name], input_view_name=[input[1].name])
+    input[0].record_command_line(command_line)     # TODO assuming same dms...
+
     print("\n")
     print(repr(o_view))
 

python/obitools3/commands/tail.pyx

@@ -9,7 +9,8 @@ from obitools3.dms.view import RollbackException
 from obitools3.apps.config import logger
 
 import time
- 
+import sys
+
 
 __title__="Keep the N last lines of a view."
 
@@ -39,6 +40,7 @@ def run(config):
     input = open_uri(config['obi']['inputURI'])
     if input is None:
         raise Exception("Could not read input view")
+    i_dms = input[0]
     i_view = input[1]
 
     # Read the name of the output view
@@ -60,7 +62,11 @@ def run(config):
     
     for i in range(start, len(i_view)):
         selection.append(i)
-    
+
+    # Save command config in View comments
+    command_line = " ".join(sys.argv[1:])
+    comments = View.print_config(config, "tail", command_line, input_dms_name=[i_dms.name], input_view_name=[i_view.name])
+
     # Create output view with the line selection
     try:
         o_view = selection.materialize(output_view_name, comments="obi tail: "+str(len(selection))+"\n")
@@ -69,6 +75,9 @@ def run(config):
 
     # TODO DISCUSS if output URI to different DMS, copy view?
 
+    # Save command config in DMS comments
+    i_dms.record_command_line(command_line)
+
     print("\n")
     print(repr(o_view))
 

python/obitools3/commands/uniq.pyx

@@ -14,6 +14,8 @@ from obitools3.uri.decode import open_uri
 from obitools3.apps.config import logger
 from obitools3.utils cimport tobytes
 
+import sys
+
 
 __title__="Group sequence records together"
  
@@ -214,8 +216,6 @@ cdef uniq_sequences(View_NUC_SEQS view, View_NUC_SEQS o_view, ProgressBar pb, li
     cdef object         mcol
     cdef object         to_merge
 
-    #print(categories)
-
     uniques = {}
     
     mergedKeys_list_b = []
@@ -521,7 +521,17 @@ def run(config):
         uniq_sequences(entries, o_view, pb, mergedKeys_list=config['uniq']['merge'], taxonomy=taxo, mergeIds=config['uniq']['mergeids'], categories=config['uniq']['categories'], max_elts=config['obi']['maxelts'])       
     except Exception, e:
         raise RollbackException("obi uniq error, rollbacking view: "+str(e), o_view)
-        
+    
+    # Save command config in View and DMS comments
+    command_line = " ".join(sys.argv[1:])
+    input_dms_name=[input[0].name]
+    input_view_name=[input[1].name]
+    if 'taxoURI' in config['obi'] and config['obi']['taxoURI'] is not None:
+        input_dms_name.append(config['obi']['taxoURI'].split("/", 1)[0])
+        input_view_name.append(config['obi']['taxoURI'].split("/", 1)[1])
+    o_view.write_config(config, "uniq", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name)
+    output[0].record_command_line(command_line)
+
     print("\n")
     print(repr(o_view))