diff --git a/python/obitools3/commands/alignpairedend.pyx b/python/obitools3/commands/alignpairedend.pyx index eab56f7..8e09ea1 100644 --- a/python/obitools3/commands/alignpairedend.pyx +++ b/python/obitools3/commands/alignpairedend.pyx @@ -16,6 +16,9 @@ from obitools3.align._qsrassemble import QSolexaRightReverseAssemble from obitools3.align._solexapairend import buildConsensus, buildJoinedSequence from obitools3.dms.obiseq cimport Nuc_Seq_Stored, Nuc_Seq +import sys +import os + REVERSE_SEQ_COLUMN_NAME = b"REVERSE_SEQUENCE" REVERSE_QUALITY_COLUMN_NAME = b"REVERSE_QUALITY" @@ -112,7 +115,7 @@ def run(config): DMS.obi_atexit() logger("info", "obi alignpairedend") - + # Open the input two_views = False @@ -124,8 +127,8 @@ def run(config): if input is None: raise Exception("Could not open input reads") if input[2] != View_NUC_SEQS: - raise NotImplementedError('obi alignpairedend only works on NUC_SEQS views') - + raise NotImplementedError('obi alignpairedend only works on NUC_SEQS views') + if "reverse" in config["alignpairedend"]: two_views = True @@ -144,10 +147,14 @@ def run(config): raise Exception("Error: the number of forward and reverse reads are different") entries = [forward, reverse] + input_dms_name = [forward.dms.name, reverse.dms.name] + input_view_name = [forward.name, reverse.name] else: entries = input[1] - + input_dms_name = [entries.dms.name] + input_view_name = [entries.name] + if two_views: entries_len = len(forward) else: @@ -166,6 +173,7 @@ def run(config): raise Exception("Could not create output view") view = output[1] + Column.new_column(view, b"QUALITY", OBI_QUAL) #TODO output URI quality option if 'smin' in config['alignpairedend']: @@ -213,7 +221,12 @@ def run(config): # consensus[b"illumina_index"] = idx i+=1 - + + # Save command config in View and DMS comments + command_line = " ".join(sys.argv[1:]) + view.write_config(config, "alignpairedend", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name) + output[0].record_command_line(command_line) + print("\n") print(repr(view)) diff --git a/python/obitools3/commands/annotate.pyx b/python/obitools3/commands/annotate.pyx index 365165d..3b4e5da 100644 --- a/python/obitools3/commands/annotate.pyx +++ b/python/obitools3/commands/annotate.pyx @@ -9,16 +9,16 @@ from obitools3.dms.view import RollbackException from functools import reduce from obitools3.apps.config import logger from obitools3.utils cimport tobytes -from obitools3.dms.capi.obiview cimport QUALITY_COLUMN -import time -import math - from obitools3.dms.capi.obiview cimport NUC_SEQUENCE_COLUMN, \ ID_COLUMN, \ DEFINITION_COLUMN, \ QUALITY_COLUMN, \ COUNT_COLUMN +import time +import math +import sys + __title__="Annotate views with new tags and edit existing annotations" @@ -285,10 +285,10 @@ def run(config): output_view_name = uri[0] # Clone output view from input view - o_view = i_view.clone(output_view_name, comments=i_view.comments+b"\nobi annotate") # TODO comments + o_view = i_view.clone(output_view_name, comments=i_view.comments) # TODO comments if o_view is None: raise Exception("Couldn't create output view") - + if 'taxoURI' in config['obi'] : # TODO default None problem taxo_uri = open_uri(config['obi']['taxoURI']) if taxo_uri is None: @@ -341,6 +341,16 @@ def run(config): except Exception, e: raise RollbackException("obi annotate error, rollbacking view: "+str(e), o_view) + + # Save command config in View and DMS comments + command_line = " ".join(sys.argv[1:]) + input_dms_name=[input[0].name] + input_view_name=[input[1].name] + if 'taxoURI' in config['obi'] and config['obi']['taxoURI'] is not None: + input_dms_name.append(config['obi']['taxoURI'].split("/", 1)[0]) + input_view_name.append(config['obi']['taxoURI'].split("/", 1)[1]) + o_view.write_config(config, "annotate", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name) + input[0].record_command_line(command_line) # TODO assuming same dms print("\n") print(repr(o_view)) diff --git a/python/obitools3/commands/clean.pyx b/python/obitools3/commands/clean.pyx index 1cd6195..41df108 100644 --- a/python/obitools3/commands/clean.pyx +++ b/python/obitools3/commands/clean.pyx @@ -9,9 +9,11 @@ from obitools3.apps.optiongroups import addSequenceInputOption, addMinimalOutput from obitools3.uri.decode import open_uri from obitools3.apps.config import logger from obitools3.utils cimport tobytes - +from obitools3.dms.view.view cimport View from obitools3.dms.view.typed_view.view_NUC_SEQS cimport View_NUC_SEQS +import sys + __title__="Tag a set of sequences for PCR and sequencing errors identification" @@ -82,11 +84,18 @@ def run(config): o_view_name = uri_o[1] else: o_view_name = uri_o[0] - - if obi_clean(tobytes(dms_name), tobytes(i_view_name), tobytes(config['clean']['sample-tag-name']), tobytes(o_view_name), b"obiclean", \ + + # Save command config in View comments + command_line = " ".join(sys.argv[1:]) + comments = View.print_config(config, "clean", command_line, input_dms_name=[dms_name], input_view_name=[i_view_name]) + + if obi_clean(tobytes(dms_name), tobytes(i_view_name), tobytes(config['clean']['sample-tag-name']), tobytes(o_view_name), comments, \ config['clean']['distance'], config['clean']['ratio'], config['clean']['heads-only'], 1) < 0: raise Exception("Error running obiclean") + # Save command config in DMS comments + dms.record_command_line(command_line) + print("\n") print(repr(dms[o_view_name])) diff --git a/python/obitools3/commands/ecopcr.pyx b/python/obitools3/commands/ecopcr.pyx index 9329d4a..cd05623 100644 --- a/python/obitools3/commands/ecopcr.pyx +++ b/python/obitools3/commands/ecopcr.pyx @@ -10,10 +10,13 @@ from obitools3.uri.decode import open_uri from obitools3.apps.config import logger from obitools3.utils cimport tobytes from obitools3.dms.view.typed_view.view_NUC_SEQS cimport View_NUC_SEQS +from obitools3.dms.view import View from libc.stdlib cimport malloc, free from libc.stdint cimport int32_t +import sys + __title__="in silico PCR" @@ -166,10 +169,14 @@ def run(config): # Read taxonomy name taxonomy_name = config['obi']['taxoURI'].split('/')[2] - # TODO: input DMS, taxonomy and primers in comments + # Save command config in View comments + command_line = " ".join(sys.argv[1:]) + comments = View.print_config(config, "ecopcr", command_line, input_dms_name=[i_dms_name], input_view_name=[i_view_name, config['obi']['taxoURI']]) + + # TODO: primers in comments? if obi_ecopcr(tobytes(i_dms_name), tobytes(i_view_name), tobytes(taxonomy_name), \ - tobytes(o_dms_name), tobytes(o_view_name), b"ecopcr", \ + tobytes(o_dms_name), tobytes(o_view_name), comments, \ tobytes(config['ecopcr']['primer1']), tobytes(config['ecopcr']['primer2']), \ config['ecopcr']['error'], \ config['ecopcr']['min-length'], config['ecopcr']['max-length'], \ @@ -177,7 +184,10 @@ def run(config): config['ecopcr']['circular'], config['ecopcr']['salt-concentration'], config['ecopcr']['salt-correction-method'], \ config['ecopcr']['keep-nucs'], config['ecopcr']['kingdom-mode']) < 0: raise Exception("Error running ecopcr") - + + # Save command config in DMS comments + o_dms.record_command_line(command_line) + free(restrict_to_taxids_p) free(ignore_taxids_p) diff --git a/python/obitools3/commands/grep.pyx b/python/obitools3/commands/grep.pyx index 338ff0d..75115b4 100644 --- a/python/obitools3/commands/grep.pyx +++ b/python/obitools3/commands/grep.pyx @@ -12,7 +12,8 @@ from obitools3.utils cimport tobytes from functools import reduce import time import re - +import sys + __title__="Grep view lines that match the given predicates" @@ -29,7 +30,7 @@ def addOptions(parser): group=parser.add_argument_group("obi grep specific options") group.add_argument("--predicate", "-p", - action="append", dest="grep:predicates", + action="append", dest="grep:grep_predicates", metavar="", default=None, type=str, @@ -306,12 +307,22 @@ def run(config): # Create output view with the line selection try: - o_view = selection.materialize(output_view_name, comments="obi grep: "+str(config["grep"])+"\n") + o_view = selection.materialize(output_view_name) except Exception, e: raise RollbackException("obi grep error, rollbacking view: "+str(e), o_view) # TODO DISCUSS if output URI to different DMS, copy view? + # Save command config in View and DMS comments + command_line = " ".join(sys.argv[1:]) + input_dms_name=[input[0].name] + input_view_name=[input[1].name] + if 'taxoURI' in config['obi'] and config['obi']['taxoURI'] is not None: + input_dms_name.append(config['obi']['taxoURI'].split("/", 1)[0]) + input_view_name.append(config['obi']['taxoURI'].split("/", 1)[1]) + o_view.write_config(config, "grep", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name) + input[0].record_command_line(command_line) # TODO assuming input and output dms are the same + print("\n") print(repr(o_view)) diff --git a/python/obitools3/commands/head.pyx b/python/obitools3/commands/head.pyx index 9eb40e2..088e8e6 100644 --- a/python/obitools3/commands/head.pyx +++ b/python/obitools3/commands/head.pyx @@ -9,6 +9,7 @@ from obitools3.dms.view import RollbackException from obitools3.apps.config import logger import time +import sys __title__="Keep the N first lines of a view." @@ -39,6 +40,7 @@ def run(config): input = open_uri(config['obi']['inputURI']) if input is None: raise Exception("Could not read input view") + i_dms = input[0] i_view = input[1] # Read the name of the output view @@ -60,15 +62,22 @@ def run(config): for i in range(n): selection.append(i) - + + # Save command config in View comments + command_line = " ".join(sys.argv[1:]) + comments = View.print_config(config, "head", command_line, input_dms_name=[i_dms.name], input_view_name=[i_view.name]) + # Create output view with the line selection try: - o_view = selection.materialize(output_view_name, comments="obi head: "+str(len(selection))+"\n") + o_view = selection.materialize(output_view_name, comments=comments) except Exception, e: raise RollbackException("obi head error, rollbacking view: "+str(e), o_view) # TODO DISCUSS if output URI to different DMS, copy view? + # Save command config in DMS comments + i_dms.record_command_line(command_line) + print("\n") print(repr(o_view)) diff --git a/python/obitools3/commands/import.pyx b/python/obitools3/commands/import.pyx index 19953b6..d9406c3 100644 --- a/python/obitools3/commands/import.pyx +++ b/python/obitools3/commands/import.pyx @@ -1,6 +1,7 @@ #cython: language_level=3 import sys +import os from obitools3.apps.progress cimport ProgressBar # @UnresolvedImport from obitools3.dms.view.view cimport View @@ -89,7 +90,7 @@ def run(config): logger("info", "obi import : imports file into a DMS") - if not config['obi']['taxdump']: # TODO discuss + if not config['obi']['taxdump']: input = open_uri(config['obi']['inputURI']) if input is None: # TODO check for bytes instead now? raise Exception("Could not open input URI") @@ -108,12 +109,13 @@ def run(config): #quality=get_quality) # TODO if output is None: raise Exception("Could not create output view") - + # Read taxdump if config['obi']['taxdump']: # The input is a taxdump to import in a DMS taxo = Taxonomy.open_taxdump(output[0], config['obi']['inputURI']) taxo.write(output[1]) taxo.close() + output[0].record_command_line(" ".join(sys.argv[1:])) output[0].close() return @@ -132,16 +134,16 @@ def run(config): raise NotImplementedError() # Save basic columns in variables for optimization - if NUC_SEQS_view : + if NUC_SEQS_view : id_col = view[b"ID"] # TODO use macros or globals for column names def_col = view[b"DEFINITION"] seq_col = view[b"NUC_SEQ"] - + dcols = {} - + i = 0 for entry in entries : - + pb(i) if NUC_SEQS_view : @@ -258,6 +260,11 @@ def run(config): i+=1 + # Save command config in View and DMS comments + command_line = " ".join(sys.argv[1:]) + view.write_config(config, "import", command_line, input_str=[os.path.abspath(config['obi']['inputURI'])]) + output[0].record_command_line(command_line) + print("\n") print(view.__repr__()) diff --git a/python/obitools3/commands/ngsfilter.pyx b/python/obitools3/commands/ngsfilter.pyx index 3693a99..bb94136 100644 --- a/python/obitools3/commands/ngsfilter.pyx +++ b/python/obitools3/commands/ngsfilter.pyx @@ -14,6 +14,7 @@ from obitools3.dms.capi.obitypes cimport OBI_SEQ, OBI_QUAL from functools import reduce, cmp_to_key import math +import sys REVERSE_SEQ_COLUMN_NAME = b"REVERSE_SEQUENCE" @@ -436,9 +437,15 @@ def run(config): entries = [forward, reverse] not_aligned = True + + input_dms_name = [forward.dms.name, reverse.dms.name] + input_view_name = [forward.name, reverse.name] else: entries = input[1] + input_dms_name = [entries.dms.name] + input_view_name = [entries.name] + if not_aligned: entries_len = len(forward) @@ -460,6 +467,8 @@ def run(config): if info_input is None: raise Exception("Could not read the view containing the informations about the tags and the primers") info_view = info_input[1] + input_dms_name.append(info_input[0].name) + input_view_name.append(info_input[1].name) # Open the unidentified view if 'unidentified' in config['ngsfilter'] and config['ngsfilter']['unidentified'] is not None: # TODO keyError if undefined problem @@ -504,6 +513,13 @@ def run(config): except Exception, e: raise RollbackException("obi ngsfilter error, rollbacking views: "+str(e), o_view, unidentified) + # Save command config in View and DMS comments + command_line = " ".join(sys.argv[1:]) + o_view.write_config(config, "ngsfilter", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name) + unidentified.write_config(config, "ngsfilter", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name) + # TODO add comment about unidentified seqs + output[0].record_command_line(command_line) # TODO if same dms... + print("\n") print(repr(o_view)) diff --git a/python/obitools3/commands/sort.pyx b/python/obitools3/commands/sort.pyx index 1dc4a13..9f9d281 100644 --- a/python/obitools3/commands/sort.pyx +++ b/python/obitools3/commands/sort.pyx @@ -20,7 +20,8 @@ from obitools3.dms.capi.obitypes cimport OBI_BOOL, \ OBIInt_NA import time - +import sys + NULL_VALUE = {OBI_BOOL: OBIBool_NA, OBI_CHAR: OBIChar_NA, @@ -105,6 +106,11 @@ def run(config): # TODO DISCUSS if output URI to different DMS, copy view? + # Save command config in View and DMS comments + command_line = " ".join(sys.argv[1:]) + o_view.write_config(config, "sort", command_line, input_dms_name=[input[0].name], input_view_name=[input[1].name]) + input[0].record_command_line(command_line) # TODO assuming same dms... + print("\n") print(repr(o_view)) diff --git a/python/obitools3/commands/tail.pyx b/python/obitools3/commands/tail.pyx index c5b0d4e..25af7af 100644 --- a/python/obitools3/commands/tail.pyx +++ b/python/obitools3/commands/tail.pyx @@ -9,7 +9,8 @@ from obitools3.dms.view import RollbackException from obitools3.apps.config import logger import time - +import sys + __title__="Keep the N last lines of a view." @@ -39,6 +40,7 @@ def run(config): input = open_uri(config['obi']['inputURI']) if input is None: raise Exception("Could not read input view") + i_dms = input[0] i_view = input[1] # Read the name of the output view @@ -60,7 +62,11 @@ def run(config): for i in range(start, len(i_view)): selection.append(i) - + + # Save command config in View comments + command_line = " ".join(sys.argv[1:]) + comments = View.print_config(config, "tail", command_line, input_dms_name=[i_dms.name], input_view_name=[i_view.name]) + # Create output view with the line selection try: o_view = selection.materialize(output_view_name, comments="obi tail: "+str(len(selection))+"\n") @@ -69,6 +75,9 @@ def run(config): # TODO DISCUSS if output URI to different DMS, copy view? + # Save command config in DMS comments + i_dms.record_command_line(command_line) + print("\n") print(repr(o_view)) diff --git a/python/obitools3/commands/uniq.pyx b/python/obitools3/commands/uniq.pyx index 379394f..9073ea8 100644 --- a/python/obitools3/commands/uniq.pyx +++ b/python/obitools3/commands/uniq.pyx @@ -14,6 +14,8 @@ from obitools3.uri.decode import open_uri from obitools3.apps.config import logger from obitools3.utils cimport tobytes +import sys + __title__="Group sequence records together" @@ -214,8 +216,6 @@ cdef uniq_sequences(View_NUC_SEQS view, View_NUC_SEQS o_view, ProgressBar pb, li cdef object mcol cdef object to_merge - #print(categories) - uniques = {} mergedKeys_list_b = [] @@ -521,7 +521,17 @@ def run(config): uniq_sequences(entries, o_view, pb, mergedKeys_list=config['uniq']['merge'], taxonomy=taxo, mergeIds=config['uniq']['mergeids'], categories=config['uniq']['categories'], max_elts=config['obi']['maxelts']) except Exception, e: raise RollbackException("obi uniq error, rollbacking view: "+str(e), o_view) - + + # Save command config in View and DMS comments + command_line = " ".join(sys.argv[1:]) + input_dms_name=[input[0].name] + input_view_name=[input[1].name] + if 'taxoURI' in config['obi'] and config['obi']['taxoURI'] is not None: + input_dms_name.append(config['obi']['taxoURI'].split("/", 1)[0]) + input_view_name.append(config['obi']['taxoURI'].split("/", 1)[1]) + o_view.write_config(config, "uniq", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name) + output[0].record_command_line(command_line) + print("\n") print(repr(o_view))