From 8aa455ad8a4c15c6e638fd7b66f65669e77649f6 Mon Sep 17 00:00:00 2001 From: Celine Mercier Date: Fri, 24 Jul 2020 16:09:48 +0200 Subject: [PATCH] Python: made all commands handle output to buffer object (e.g. stdout) --- python/obitools3/commands/align.pyx | 33 +++++--- python/obitools3/commands/alignpairedend.pyx | 59 +++++++++++--- python/obitools3/commands/annotate.pyx | 44 +++++++--- python/obitools3/commands/build_ref_db.pyx | 31 +++++--- python/obitools3/commands/cat.pyx | 84 ++++++++++++-------- python/obitools3/commands/clean.pyx | 30 ++++--- python/obitools3/commands/ecopcr.pyx | 28 ++++++- python/obitools3/commands/ecotag.pyx | 27 +++++-- python/obitools3/commands/export.pyx | 2 +- python/obitools3/commands/grep.pyx | 59 +++++++++----- python/obitools3/commands/head.pyx | 54 ++++++++----- python/obitools3/commands/import.pyx | 8 +- python/obitools3/commands/ngsfilter.pyx | 53 +++++++++--- python/obitools3/commands/sort.pyx | 61 +++++++++----- python/obitools3/commands/stats.pyx | 2 +- python/obitools3/commands/tail.pyx | 52 ++++++++---- python/obitools3/commands/uniq.pyx | 82 ++++++++++++++----- python/obitools3/uri/decode.pyx | 15 ++-- 18 files changed, 511 insertions(+), 213 deletions(-) diff --git a/python/obitools3/commands/align.pyx b/python/obitools3/commands/align.pyx index 68c4ed9..06ef8b2 100644 --- a/python/obitools3/commands/align.pyx +++ b/python/obitools3/commands/align.pyx @@ -4,7 +4,7 @@ from obitools3.apps.progress cimport ProgressBar # @UnresolvedImport from obitools3.dms import DMS from obitools3.dms.view.view cimport View from obitools3.uri.decode import open_uri -from obitools3.apps.optiongroups import addMinimalInputOption, addMinimalOutputOption +from obitools3.apps.optiongroups import addMinimalInputOption, addMinimalOutputOption, addNoProgressBarOption from obitools3.dms.view import RollbackException from obitools3.apps.config import logger from obitools3.utils cimport tobytes, str2bytes @@ -12,6 +12,9 @@ from obitools3.utils cimport tobytes, str2bytes from obitools3.dms.capi.obilcsalign cimport obi_lcs_align_one_column, \ obi_lcs_align_two_columns +from io import BufferedWriter +from cpython.exc cimport PyErr_CheckSignals + import time import sys @@ -23,6 +26,7 @@ def addOptions(parser): addMinimalInputOption(parser) addMinimalOutputOption(parser) + addNoProgressBarOption(parser) group=parser.add_argument_group('obi align specific options') @@ -201,20 +205,20 @@ def run(config): if output is None: raise Exception("Could not create output") o_dms = output[0] + output_0 = output[0] o_dms_name = o_dms.name final_o_view_name = output[1] + o_view_name = final_o_view_name - # If the input and output DMS are not the same, align creating a temporary view in the input dms that will be exported to + # If stdout output or the input and output DMS are not the same, align creating a temporary view in the input dms that will be exported to # the right DMS and deleted in the other afterwards. - if i_dms != o_dms: - temporary_view_name = final_o_view_name - i=0 - while temporary_view_name in i_dms: # Making sure view name is unique in input DMS - temporary_view_name = final_o_view_name+b"_"+str2bytes(str(i)) + if i_dms != o_dms or type(output_0)==BufferedWriter: + if type(output_0)==BufferedWriter: + o_dms = i_dms + o_view_name = b"temp" + while o_view_name in i_dms: # Making sure view name is unique in input DMS + o_view_name = o_view_name+b"_"+str2bytes(str(i)) i+=1 - o_view_name = temporary_view_name - else: - o_view_name = final_o_view_name # Save command config in View comments command_line = " ".join(sys.argv[1:]) @@ -263,8 +267,15 @@ def run(config): View.delete_view(i_dms, i_view_name_2) i_dms_2.close() + # stdout output: write to buffer + if type(output_0)==BufferedWriter: + logger("info", "Printing to output...") + o_view = o_dms[o_view_name] + o_view.print_to_output(output_0, noprogressbar=config['obi']['noprogressbar']) + o_view.close() + # If the input and the output DMS are different, delete the temporary result view in the input DMS - if i_dms != o_dms: + if i_dms != o_dms or type(output_0)==BufferedWriter: View.delete_view(i_dms, o_view_name) o_dms.close(force=True) diff --git a/python/obitools3/commands/alignpairedend.pyx b/python/obitools3/commands/alignpairedend.pyx index c7fd0d2..e012c99 100755 --- a/python/obitools3/commands/alignpairedend.pyx +++ b/python/obitools3/commands/alignpairedend.pyx @@ -6,7 +6,7 @@ from obitools3.dms.view.typed_view.view_NUC_SEQS cimport View_NUC_SEQS from obitools3.dms.column.column cimport Column from obitools3.dms.capi.obiview cimport QUALITY_COLUMN from obitools3.dms.capi.obitypes cimport OBI_QUAL -from obitools3.apps.optiongroups import addMinimalInputOption, addMinimalOutputOption +from obitools3.apps.optiongroups import addMinimalInputOption, addMinimalOutputOption, addNoProgressBarOption from obitools3.uri.decode import open_uri from obitools3.apps.config import logger from obitools3.libalign._qsassemble import QSolexaReverseAssemble @@ -15,7 +15,9 @@ from obitools3.libalign._solexapairend import buildConsensus, buildJoinedSequenc from obitools3.dms.obiseq cimport Nuc_Seq from obitools3.libalign.shifted_ali cimport Kmer_similarity, Ali_shifted from obitools3.dms.capi.obiview cimport REVERSE_SEQUENCE_COLUMN, REVERSE_QUALITY_COLUMN +from obitools3.utils cimport str2bytes +from io import BufferedWriter import sys import os @@ -29,6 +31,7 @@ def addOptions(parser): addMinimalInputOption(parser) addMinimalOutputOption(parser) + addNoProgressBarOption(parser) group = parser.add_argument_group('obi alignpairedend specific options') @@ -171,13 +174,29 @@ def run(config): if output is None: raise Exception("Could not create output view") - view = output[1] + output_0 = output[0] + o_dms = output[0] - Column.new_column(view, QUALITY_COLUMN, OBI_QUAL) #TODO output URI quality option? + # stdout output: create temporary view + if type(output_0)==BufferedWriter: + i_dms = forward.dms # using any dms + o_dms = i_dms + i=0 + o_view_name = b"temp" + while o_view_name in i_dms: # Making sure view name is unique in input DMS + o_view_name = o_view_name+b"_"+str2bytes(str(i)) + i+=1 + o_view = View_NUC_SEQS.new(o_dms, o_view_name, quality=True) + else: + o_view = output[1] + Column.new_column(o_view, QUALITY_COLUMN, OBI_QUAL) # Initialize the progress bar - pb = ProgressBar(entries_len, config, seconde=5) - + if config['obi']['noprogressbar'] == False: + pb = ProgressBar(entries_len, config) + else: + pb = None + #if config['alignpairedend']['trueali']: # kmer_ali = False # aligner = buildAlignment @@ -202,11 +221,12 @@ def run(config): i = 0 for ali in ba: - pb(i) + if pb is not None: + pb(i) PyErr_CheckSignals() - consensus = view[i] + consensus = o_view[i] if not two_views: seqF = entries[i] @@ -227,24 +247,37 @@ def run(config): i+=1 - pb(i, force=True) - print("", file=sys.stderr) + if pb is not None: + pb(i, force=True) + print("", file=sys.stderr) if kmer_ali : aligner.free() # Save command config in View and DMS comments command_line = " ".join(sys.argv[1:]) - view.write_config(config, "alignpairedend", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name) - output[0].record_command_line(command_line) + o_view.write_config(config, "alignpairedend", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name) + o_dms.record_command_line(command_line) #print("\n\nOutput view:\n````````````", file=sys.stderr) #print(repr(view), file=sys.stderr) - + + # stdout output: write to buffer + if type(output_0)==BufferedWriter: + logger("info", "Printing to output...") + o_view.print_to_output(output_0, noprogressbar=config['obi']['noprogressbar']) + o_view.close() + + # If stdout output, delete the temporary imported view used to create the final file + if type(output_0)==BufferedWriter: + View_NUC_SEQS.delete_view(o_dms, o_view_name) + output_0.close() + + # Close all DMS input[0].close(force=True) if two_views: rinput[0].close(force=True) - output[0].close(force=True) + o_dms.close(force=True) logger("info", "Done.") \ No newline at end of file diff --git a/python/obitools3/commands/annotate.pyx b/python/obitools3/commands/annotate.pyx index 4bc613e..8d572ce 100755 --- a/python/obitools3/commands/annotate.pyx +++ b/python/obitools3/commands/annotate.pyx @@ -4,11 +4,12 @@ from obitools3.apps.progress cimport ProgressBar # @UnresolvedImport from obitools3.dms import DMS from obitools3.dms.view.view cimport View, Line_selection from obitools3.uri.decode import open_uri -from obitools3.apps.optiongroups import addMinimalInputOption, addTaxonomyOption, addMinimalOutputOption +from obitools3.apps.optiongroups import addMinimalInputOption, addTaxonomyOption, addMinimalOutputOption, addNoProgressBarOption from obitools3.dms.view import RollbackException from functools import reduce from obitools3.apps.config import logger from obitools3.utils cimport tobytes, str2bytes +from io import BufferedWriter from obitools3.dms.capi.obiview cimport NUC_SEQUENCE_COLUMN, \ ID_COLUMN, \ DEFINITION_COLUMN, \ @@ -34,6 +35,7 @@ def addOptions(parser): addMinimalInputOption(parser) addTaxonomyOption(parser) addMinimalOutputOption(parser) + addNoProgressBarOption(parser) group=parser.add_argument_group('obi annotate specific options') @@ -278,8 +280,19 @@ def run(config): if output is None: raise Exception("Could not create output view") o_dms = output[0] + output_0 = output[0] o_view_name = output[1] - + + # stdout output: create temporary view + if type(output_0)==BufferedWriter: + o_dms = i_dms + i=0 + o_view_name = b"temp" + while o_view_name in i_dms: # Making sure view name is unique in output DMS + o_view_name = o_view_name+b"_"+str2bytes(str(i)) + i+=1 + imported_view_name = o_view_name + # If the input and output DMS are not the same, import the input view in the output DMS before cloning it to modify it # (could be the other way around: clone and modify in the input DMS then import the new view in the output DMS) if i_dms != o_dms: @@ -290,7 +303,7 @@ def run(config): i+=1 View.import_view(i_dms.full_path[:-7], o_dms.full_path[:-7], i_view_name, imported_view_name) i_view = o_dms[imported_view_name] - + # Clone output view from input view o_view = i_view.clone(o_view_name) if o_view is None: @@ -307,7 +320,10 @@ def run(config): taxo = None # Initialize the progress bar - pb = ProgressBar(len(o_view), config, seconde=5) + if config['obi']['noprogressbar'] == False: + pb = ProgressBar(len(o_view), config) + else: + pb = None try: @@ -346,14 +362,16 @@ def run(config): sequenceTagger = sequenceTaggerGenerator(config, taxo=taxo) for i in range(len(o_view)): PyErr_CheckSignals() - pb(i) + if pb is not None: + pb(i) sequenceTagger(o_view[i]) except Exception, e: raise RollbackException("obi annotate error, rollbacking view: "+str(e), o_view) - pb(i, force=True) - print("", file=sys.stderr) + if pb is not None: + pb(i, force=True) + print("", file=sys.stderr) # Save command config in View and DMS comments command_line = " ".join(sys.argv[1:]) @@ -363,13 +381,19 @@ def run(config): input_dms_name.append(config['obi']['taxoURI'].split("/")[-3]) input_view_name.append("taxonomy/"+config['obi']['taxoURI'].split("/")[-1]) o_view.write_config(config, "annotate", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name) - output[0].record_command_line(command_line) + o_dms.record_command_line(command_line) #print("\n\nOutput view:\n````````````", file=sys.stderr) #print(repr(o_view), file=sys.stderr) - # If the input and the output DMS are different, delete the temporary imported view used to create the final view - if i_dms != o_dms: + # stdout output: write to buffer + if type(output_0)==BufferedWriter: + logger("info", "Printing to output...") + o_view.print_to_output(output_0, noprogressbar=config['obi']['noprogressbar']) + o_view.close() + + # If the input and the output DMS are different or if stdout output, delete the temporary imported view used to create the final view + if i_dms != o_dms or type(output_0)==BufferedWriter: View.delete_view(o_dms, imported_view_name) o_dms.close(force=True) i_dms.close(force=True) diff --git a/python/obitools3/commands/build_ref_db.pyx b/python/obitools3/commands/build_ref_db.pyx index ad8e5bc..d6f37c3 100755 --- a/python/obitools3/commands/build_ref_db.pyx +++ b/python/obitools3/commands/build_ref_db.pyx @@ -4,14 +4,16 @@ from obitools3.apps.progress cimport ProgressBar # @UnresolvedImport from obitools3.dms.dms cimport DMS from obitools3.dms.view import RollbackException from obitools3.dms.capi.build_reference_db cimport build_reference_db -from obitools3.apps.optiongroups import addMinimalInputOption, addTaxonomyOption, addMinimalOutputOption +from obitools3.apps.optiongroups import addMinimalInputOption, addTaxonomyOption, addMinimalOutputOption, addNoProgressBarOption from obitools3.uri.decode import open_uri from obitools3.apps.config import logger from obitools3.utils cimport tobytes, str2bytes from obitools3.dms.view.view cimport View from obitools3.dms.view.typed_view.view_NUC_SEQS cimport View_NUC_SEQS +from io import BufferedWriter import sys +from cpython.exc cimport PyErr_CheckSignals __title__="Tag a set of sequences for PCR and sequencing errors identification" @@ -22,6 +24,7 @@ def addOptions(parser): addMinimalInputOption(parser) addTaxonomyOption(parser) addMinimalOutputOption(parser) + addNoProgressBarOption(parser) group = parser.add_argument_group('obi build_ref_db specific options') @@ -56,17 +59,20 @@ def run(config): if output is None: raise Exception("Could not create output") o_dms = output[0] + output_0 = output[0] final_o_view_name = output[1] - # If the input and output DMS are not the same, build the database creating a temporary view that will be exported to + # If stdout output or the input and output DMS are not the same, build the database creating a temporary view that will be exported to # the right DMS and deleted in the other afterwards. - if i_dms != o_dms: - temporary_view_name = final_o_view_name + if i_dms != o_dms or type(output_0)==BufferedWriter: + temporary_view_name = b"temp" i=0 while temporary_view_name in i_dms: # Making sure view name is unique in input DMS - temporary_view_name = final_o_view_name+b"_"+str2bytes(str(i)) + temporary_view_name = temporary_view_name+b"_"+str2bytes(str(i)) i+=1 o_view_name = temporary_view_name + if type(output_0)==BufferedWriter: + o_dms = i_dms else: o_view_name = final_o_view_name @@ -80,22 +86,29 @@ def run(config): input_dms_name.append(config['obi']['taxoURI'].split("/")[-3]) input_view_name.append("taxonomy/"+config['obi']['taxoURI'].split("/")[-1]) comments = View.print_config(config, "build_ref_db", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name) - + if build_reference_db(i_dms.name_with_full_path, tobytes(i_view_name), tobytes(taxonomy_name), tobytes(o_view_name), comments, config['build_ref_db']['threshold']) < 0: raise Exception("Error building a reference database") # If the input and output DMS are not the same, export result view to output DMS if i_dms != o_dms: View.import_view(i_dms.full_path[:-7], o_dms.full_path[:-7], o_view_name, final_o_view_name) - + + # stdout output: write to buffer + if type(output_0)==BufferedWriter: + logger("info", "Printing to output...") + o_view = o_dms[o_view_name] + o_view.print_to_output(output_0, noprogressbar=config['obi']['noprogressbar']) + o_view.close() + # Save command config in DMS comments o_dms.record_command_line(command_line) #print("\n\nOutput view:\n````````````", file=sys.stderr) #print(repr(o_dms[final_o_view_name]), file=sys.stderr) - # If the input and the output DMS are different, delete the temporary result view in the input DMS - if i_dms != o_dms: + # If the input and the output DMS are different or if stdout output, delete the temporary imported view used to create the final view + if i_dms != o_dms or type(output_0)==BufferedWriter: View.delete_view(i_dms, o_view_name) o_dms.close(force=True) diff --git a/python/obitools3/commands/cat.pyx b/python/obitools3/commands/cat.pyx index 803158f..2df9023 100755 --- a/python/obitools3/commands/cat.pyx +++ b/python/obitools3/commands/cat.pyx @@ -15,6 +15,7 @@ from obitools3.dms.capi.obiview cimport NUC_SEQUENCE_COLUMN, REVERSE_SEQUENCE_CO from obitools3.dms.capi.obitypes cimport OBI_SEQ, OBI_QUAL from obitools3.dms.column.column cimport Column +from io import BufferedWriter import time import sys @@ -76,53 +77,70 @@ def run(config): if output is None: raise Exception("Could not create output view") o_dms = output[0] + output_0 = output[0] o_view = output[1] + # stdout output + if type(output_0)==BufferedWriter: + o_dms = i_dms + # Initialize quality columns and their associated sequence columns if needed - if not remove_qual: - if NUC_SEQUENCE_COLUMN not in o_view: - Column.new_column(o_view, NUC_SEQUENCE_COLUMN, OBI_SEQ) - Column.new_column(o_view, QUALITY_COLUMN, OBI_QUAL, associated_column_name=NUC_SEQUENCE_COLUMN, associated_column_version=o_view[NUC_SEQUENCE_COLUMN].version) - if not remove_rev_qual: - Column.new_column(o_view, REVERSE_SEQUENCE_COLUMN, OBI_SEQ) - Column.new_column(o_view, REVERSE_QUALITY_COLUMN, OBI_QUAL, associated_column_name=REVERSE_SEQUENCE_COLUMN, associated_column_version=o_view[REVERSE_SEQUENCE_COLUMN].version) + if type(output_0) != BufferedWriter: + if not remove_qual: + if NUC_SEQUENCE_COLUMN not in o_view: + Column.new_column(o_view, NUC_SEQUENCE_COLUMN, OBI_SEQ) + Column.new_column(o_view, QUALITY_COLUMN, OBI_QUAL, associated_column_name=NUC_SEQUENCE_COLUMN, associated_column_version=o_view[NUC_SEQUENCE_COLUMN].version) + if not remove_rev_qual: + Column.new_column(o_view, REVERSE_SEQUENCE_COLUMN, OBI_SEQ) + Column.new_column(o_view, REVERSE_QUALITY_COLUMN, OBI_QUAL, associated_column_name=REVERSE_SEQUENCE_COLUMN, associated_column_version=o_view[REVERSE_SEQUENCE_COLUMN].version) # Initialize multiple elements columns - dict_cols = {} - for v in iview_list: - for coln in v.keys(): - if v[coln].nb_elements_per_line > 1: - if coln not in dict_cols: - dict_cols[coln] = {} - dict_cols[coln]['eltnames'] = set(v[coln].elements_names) - dict_cols[coln]['nbelts'] = v[coln].nb_elements_per_line - dict_cols[coln]['obitype'] = v[coln].data_type_int - else: - dict_cols[coln]['eltnames'] = set(v[coln].elements_names + list(dict_cols[coln]['eltnames'])) - dict_cols[coln]['nbelts'] = len(dict_cols[coln]['eltnames']) - for coln in dict_cols: - Column.new_column(o_view, coln, dict_cols[coln]['obitype'], - nb_elements_per_line=dict_cols[coln]['nbelts'], elements_names=list(dict_cols[coln]['eltnames'])) + if type(output_0)==BufferedWriter: + dict_cols = {} + for v in iview_list: + for coln in v.keys(): + if v[coln].nb_elements_per_line > 1: + if coln not in dict_cols: + dict_cols[coln] = {} + dict_cols[coln]['eltnames'] = set(v[coln].elements_names) + dict_cols[coln]['nbelts'] = v[coln].nb_elements_per_line + dict_cols[coln]['obitype'] = v[coln].data_type_int + else: + dict_cols[coln]['eltnames'] = set(v[coln].elements_names + list(dict_cols[coln]['eltnames'])) + dict_cols[coln]['nbelts'] = len(dict_cols[coln]['eltnames']) + for coln in dict_cols: + Column.new_column(o_view, coln, dict_cols[coln]['obitype'], + nb_elements_per_line=dict_cols[coln]['nbelts'], elements_names=list(dict_cols[coln]['eltnames'])) # Initialize the progress bar - pb = ProgressBar(total_len, config, seconde=5) + if not config['obi']['noprogressbar']: + pb = ProgressBar(total_len, config) + else: + pb = None i = 0 for v in iview_list: - for l in v: + for entry in v: PyErr_CheckSignals() - pb(i) - o_view[i] = l + if pb is not None: + pb(i) + if type(output_0)==BufferedWriter: + rep = repr(entry) + output_0.write(str2bytes(rep)+b"\n") + else: + o_view[i] = entry i+=1 # Deletes quality columns if needed - if QUALITY_COLUMN in o_view and remove_qual : - o_view.delete_column(QUALITY_COLUMN) - if REVERSE_QUALITY_COLUMN in o_view and remove_rev_qual : - o_view.delete_column(REVERSE_QUALITY_COLUMN) - - pb(i, force=True) - print("", file=sys.stderr) + if type(output_0)!=BufferedWriter: + if QUALITY_COLUMN in o_view and remove_qual : + o_view.delete_column(QUALITY_COLUMN) + if REVERSE_QUALITY_COLUMN in o_view and remove_rev_qual : + o_view.delete_column(REVERSE_QUALITY_COLUMN) + + if pb is not None: + pb(i, force=True) + print("", file=sys.stderr) # Save command config in DMS comments command_line = " ".join(sys.argv[1:]) diff --git a/python/obitools3/commands/clean.pyx b/python/obitools3/commands/clean.pyx index 72aa918..c817259 100755 --- a/python/obitools3/commands/clean.pyx +++ b/python/obitools3/commands/clean.pyx @@ -4,13 +4,14 @@ from obitools3.apps.progress cimport ProgressBar # @UnresolvedImport from obitools3.dms.dms cimport DMS from obitools3.dms.view import RollbackException from obitools3.dms.capi.obiclean cimport obi_clean -from obitools3.apps.optiongroups import addMinimalInputOption, addMinimalOutputOption +from obitools3.apps.optiongroups import addMinimalInputOption, addMinimalOutputOption, addNoProgressBarOption from obitools3.uri.decode import open_uri from obitools3.apps.config import logger from obitools3.utils cimport tobytes, str2bytes from obitools3.dms.view.view cimport View from obitools3.dms.view.typed_view.view_NUC_SEQS cimport View_NUC_SEQS +from io import BufferedWriter import sys @@ -21,7 +22,8 @@ def addOptions(parser): addMinimalInputOption(parser) addMinimalOutputOption(parser) - + addNoProgressBarOption(parser) + group = parser.add_argument_group('obi clean specific options') group.add_argument('--distance', '-d', @@ -88,17 +90,20 @@ def run(config): if output is None: raise Exception("Could not create output") o_dms = output[0] + output_0 = output[0] final_o_view_name = output[1] - # If the input and output DMS are not the same, run obiclean creating a temporary view that will be exported to + # If stdout output or the input and output DMS are not the same, create a temporary view that will be exported to # the right DMS and deleted in the other afterwards. - if i_dms != o_dms: - temporary_view_name = final_o_view_name + if i_dms != o_dms or type(output_0)==BufferedWriter: + temporary_view_name = b"temp" i=0 while temporary_view_name in i_dms: # Making sure view name is unique in input DMS - temporary_view_name = final_o_view_name+b"_"+str2bytes(str(i)) + temporary_view_name = temporary_view_name+b"_"+str2bytes(str(i)) i+=1 o_view_name = temporary_view_name + if type(output_0)==BufferedWriter: + o_dms = i_dms else: o_view_name = final_o_view_name @@ -116,15 +121,22 @@ def run(config): # If the input and output DMS are not the same, export result view to output DMS if i_dms != o_dms: View.import_view(i_dms.full_path[:-7], o_dms.full_path[:-7], o_view_name, final_o_view_name) - + + # stdout output: write to buffer + if type(output_0)==BufferedWriter: + logger("info", "Printing to output...") + o_view = o_dms[o_view_name] + o_view.print_to_output(output_0, noprogressbar=config['obi']['noprogressbar']) + o_view.close() + # Save command config in DMS comments o_dms.record_command_line(command_line) #print("\n\nOutput view:\n````````````", file=sys.stderr) #print(repr(o_dms[final_o_view_name]), file=sys.stderr) - # If the input and the output DMS are different, delete the temporary result view in the input DMS - if i_dms != o_dms: + # If the input and the output DMS are different or if stdout output, delete the temporary imported view used to create the final view + if i_dms != o_dms or type(output_0)==BufferedWriter: View.delete_view(i_dms, o_view_name) o_dms.close(force=True) diff --git a/python/obitools3/commands/ecopcr.pyx b/python/obitools3/commands/ecopcr.pyx index ed7c8b6..d33bc6b 100755 --- a/python/obitools3/commands/ecopcr.pyx +++ b/python/obitools3/commands/ecopcr.pyx @@ -5,10 +5,10 @@ from obitools3.dms.dms cimport DMS from obitools3.dms.capi.obidms cimport OBIDMS_p from obitools3.dms.view import RollbackException from obitools3.dms.capi.obiecopcr cimport obi_ecopcr -from obitools3.apps.optiongroups import addMinimalInputOption, addMinimalOutputOption, addTaxonomyOption +from obitools3.apps.optiongroups import addMinimalInputOption, addMinimalOutputOption, addTaxonomyOption, addNoProgressBarOption from obitools3.uri.decode import open_uri from obitools3.apps.config import logger -from obitools3.utils cimport tobytes +from obitools3.utils cimport tobytes, str2bytes from obitools3.dms.view.typed_view.view_NUC_SEQS cimport View_NUC_SEQS from obitools3.dms.view import View @@ -16,6 +16,7 @@ from libc.stdlib cimport malloc, free from libc.stdint cimport int32_t import sys +from io import BufferedWriter __title__="in silico PCR" @@ -27,6 +28,7 @@ def addOptions(parser): addMinimalInputOption(parser) addTaxonomyOption(parser) addMinimalOutputOption(parser) + addNoProgressBarOption(parser) group = parser.add_argument_group('obi ecopcr specific options') @@ -169,11 +171,20 @@ def run(config): if output is None: raise Exception("Could not create output") o_dms = output[0] + output_0 = output[0] o_dms_name = output[0].name o_view_name = output[1] # Read taxonomy name taxonomy_name = config['obi']['taxoURI'].split("/")[-1] # Robust in theory + + # If stdout output create a temporary view in the input dms that will be deleted afterwards. + if type(output_0)==BufferedWriter: + o_dms = i_dms + o_view_name = b"temp" + while o_view_name in i_dms: # Making sure view name is unique in input DMS + o_view_name = o_view_name+b"_"+str2bytes(str(i)) + i+=1 # Save command config in View comments command_line = " ".join(sys.argv[1:]) @@ -201,10 +212,21 @@ def run(config): free(restrict_to_taxids_p) free(ignore_taxids_p) - + + # stdout output: write to buffer + if type(output_0)==BufferedWriter: + logger("info", "Printing to output...") + o_view = o_dms[o_view_name] + o_view.print_to_output(output_0, noprogressbar=config['obi']['noprogressbar']) + o_view.close() + #print("\n\nOutput view:\n````````````", file=sys.stderr) #print(repr(o_dms[o_view_name]), file=sys.stderr) + # If stdout output, delete the temporary result view in the input DMS + if type(output_0)==BufferedWriter: + View.delete_view(i_dms, o_view_name) + i_dms.close(force=True) o_dms.close(force=True) diff --git a/python/obitools3/commands/ecotag.pyx b/python/obitools3/commands/ecotag.pyx index c86344c..d916087 100755 --- a/python/obitools3/commands/ecotag.pyx +++ b/python/obitools3/commands/ecotag.pyx @@ -4,7 +4,7 @@ from obitools3.apps.progress cimport ProgressBar # @UnresolvedImport from obitools3.dms.dms cimport DMS from obitools3.dms.view import RollbackException from obitools3.dms.capi.obiecotag cimport obi_ecotag -from obitools3.apps.optiongroups import addMinimalInputOption, addTaxonomyOption, addMinimalOutputOption +from obitools3.apps.optiongroups import addMinimalInputOption, addTaxonomyOption, addMinimalOutputOption, addNoProgressBarOption from obitools3.uri.decode import open_uri from obitools3.apps.config import logger from obitools3.utils cimport tobytes, str2bytes @@ -12,6 +12,7 @@ from obitools3.dms.view.view cimport View from obitools3.dms.view.typed_view.view_NUC_SEQS cimport View_NUC_SEQS import sys +from io import BufferedWriter __title__="Taxonomic assignment of sequences" @@ -22,6 +23,7 @@ def addOptions(parser): addMinimalInputOption(parser) addTaxonomyOption(parser) addMinimalOutputOption(parser) + addNoProgressBarOption(parser) group = parser.add_argument_group('obi ecotag specific options') @@ -75,17 +77,19 @@ def run(config): if output is None: raise Exception("Could not create output") o_dms = output[0] + output_0 = output[0] final_o_view_name = output[1] - # If the input and output DMS are not the same, run ecotag creating a temporary view that will be exported to - # the right DMS and deleted in the other afterwards. - if i_dms != o_dms: - temporary_view_name = final_o_view_name + # If stdout output or the input and output DMS are not the same, create a temporary view that will be exported and deleted. + if i_dms != o_dms or type(output_0)==BufferedWriter: + temporary_view_name = b"temp" i=0 while temporary_view_name in i_dms: # Making sure view name is unique in input DMS - temporary_view_name = final_o_view_name+b"_"+str2bytes(str(i)) + temporary_view_name = temporary_view_name+b"_"+str2bytes(str(i)) i+=1 o_view_name = temporary_view_name + if type(output_0)==BufferedWriter: + o_dms = i_dms else: o_view_name = final_o_view_name @@ -120,11 +124,18 @@ def run(config): # Save command config in DMS comments o_dms.record_command_line(command_line) + # stdout output: write to buffer + if type(output_0)==BufferedWriter: + logger("info", "Printing to output...") + o_view = o_dms[o_view_name] + o_view.print_to_output(output_0, noprogressbar=config['obi']['noprogressbar']) + o_view.close() + #print("\n\nOutput view:\n````````````", file=sys.stderr) #print(repr(o_dms[final_o_view_name]), file=sys.stderr) - # If the input and the output DMS are different, delete the temporary result view in the input DMS - if i_dms != o_dms: + # If the input and the output DMS are different or if stdout output, delete the temporary imported view used to create the final view + if i_dms != o_dms or type(output_0)==BufferedWriter: View.delete_view(i_dms, o_view_name) o_dms.close(force=True) diff --git a/python/obitools3/commands/export.pyx b/python/obitools3/commands/export.pyx index 9cd1a50..1ff6159 100755 --- a/python/obitools3/commands/export.pyx +++ b/python/obitools3/commands/export.pyx @@ -74,7 +74,7 @@ def run(config): if config['obi']['noprogressbar']: pb = None else: - pb = ProgressBar(withoutskip - skip, config, seconde=5) + pb = ProgressBar(withoutskip - skip, config) i=0 for seq in iview : diff --git a/python/obitools3/commands/grep.pyx b/python/obitools3/commands/grep.pyx index fcce3d4..4039e28 100644 --- a/python/obitools3/commands/grep.pyx +++ b/python/obitools3/commands/grep.pyx @@ -4,7 +4,7 @@ from obitools3.apps.progress cimport ProgressBar # @UnresolvedImport from obitools3.dms import DMS from obitools3.dms.view.view cimport View, Line_selection from obitools3.uri.decode import open_uri -from obitools3.apps.optiongroups import addMinimalInputOption, addTaxonomyOption, addMinimalOutputOption +from obitools3.apps.optiongroups import addMinimalInputOption, addTaxonomyOption, addMinimalOutputOption, addNoProgressBarOption from obitools3.dms.view import RollbackException from obitools3.apps.config import logger from obitools3.utils cimport tobytes, str2bytes @@ -14,6 +14,7 @@ import time import re import sys import ast +from io import BufferedWriter from cpython.exc cimport PyErr_CheckSignals @@ -28,6 +29,7 @@ def addOptions(parser): addMinimalInputOption(parser) addTaxonomyOption(parser) addMinimalOutputOption(parser) + addNoProgressBarOption(parser) group=parser.add_argument_group("obi grep specific options") @@ -304,16 +306,21 @@ def run(config): if output is None: raise Exception("Could not create output view") o_dms = output[0] - o_view_name_final = output[1] - o_view_name = o_view_name_final - - # If the input and output DMS are not the same, create output view in input DMS first, then export it - # to output DMS, making sure the temporary view name is unique in the input DMS - if i_dms != o_dms: + output_0 = output[0] + final_o_view_name = output[1] + + # If stdout output or the input and output DMS are not the same, create a temporary view that will be exported and deleted afterwards. + if i_dms != o_dms or type(output_0)==BufferedWriter: + temporary_view_name = b"temp" i=0 - while o_view_name in i_dms: - o_view_name = o_view_name_final+b"_"+str2bytes(str(i)) - i+=1 + while temporary_view_name in i_dms: # Making sure view name is unique in input DMS + temporary_view_name = temporary_view_name+b"_"+str2bytes(str(i)) + i+=1 + o_view_name = temporary_view_name + if type(output_0)==BufferedWriter: + o_dms = i_dms + else: + o_view_name = final_o_view_name if 'taxoURI' in config['obi'] and config['obi']['taxoURI'] is not None: taxo_uri = open_uri(config["obi"]["taxoURI"]) @@ -324,7 +331,10 @@ def run(config): taxo = None # Initialize the progress bar - pb = ProgressBar(len(i_view), config, seconde=5) + if config['obi']['noprogressbar'] == False: + pb = ProgressBar(len(i_view), config) + else: + pb = None # Apply filter tax_filter = Taxonomy_filter_generator(taxo, config["grep"]) @@ -334,13 +344,15 @@ def run(config): if filter is None and config["grep"]["invert_selection"]: # all sequences are selected: filter is None if no line will be selected because some columns don't exist for i in range(len(i_view)): PyErr_CheckSignals() - pb(i) + if pb is not None: + pb(i) selection.append(i) elif filter is not None : # filter is None if no line will be selected because some columns don't exist for i in range(len(i_view)): PyErr_CheckSignals() - pb(i) + if pb is not None: + pb(i) line = i_view[i] loc_env = {"sequence": line, "line": line, "taxonomy": taxo, "obi_eval_result": False} @@ -349,9 +361,10 @@ def run(config): if good : selection.append(i) - - pb(len(i_view), force=True) - print("", file=sys.stderr) + + if pb is not None: + pb(len(i_view), force=True) + print("", file=sys.stderr) # Create output view with the line selection try: @@ -375,14 +388,20 @@ def run(config): # and delete the temporary view in the input DMS if i_dms != o_dms: o_view.close() - View.import_view(i_dms.full_path[:-7], o_dms.full_path[:-7], o_view_name, o_view_name_final) - o_view = o_dms[o_view_name_final] + View.import_view(i_dms.full_path[:-7], o_dms.full_path[:-7], o_view_name, final_o_view_name) + o_view = o_dms[final_o_view_name] + + # stdout output: write to buffer + if type(output_0)==BufferedWriter: + logger("info", "Printing to output...") + o_view.print_to_output(output_0, noprogressbar=config['obi']['noprogressbar']) + o_view.close() #print("\n\nOutput view:\n````````````", file=sys.stderr) #print(repr(o_view), file=sys.stderr) - # If the input and the output DMS are different, delete the temporary imported view used to create the final view - if i_dms != o_dms: + # If the input and the output DMS are different or if stdout output, delete the temporary imported view used to create the final view + if i_dms != o_dms or type(output_0)==BufferedWriter: View.delete_view(i_dms, o_view_name) o_dms.close(force=True) i_dms.close(force=True) diff --git a/python/obitools3/commands/head.pyx b/python/obitools3/commands/head.pyx index 4ac2231..4a44067 100755 --- a/python/obitools3/commands/head.pyx +++ b/python/obitools3/commands/head.pyx @@ -4,14 +4,15 @@ from obitools3.apps.progress cimport ProgressBar # @UnresolvedImport from obitools3.dms import DMS from obitools3.dms.view.view cimport View, Line_selection from obitools3.uri.decode import open_uri -from obitools3.apps.optiongroups import addMinimalInputOption, addMinimalOutputOption +from obitools3.apps.optiongroups import addMinimalInputOption, addMinimalOutputOption, addNoProgressBarOption from obitools3.dms.view import RollbackException from obitools3.apps.config import logger from obitools3.utils cimport str2bytes import time import sys - +from io import BufferedWriter + from cpython.exc cimport PyErr_CheckSignals @@ -22,6 +23,7 @@ def addOptions(parser): addMinimalInputOption(parser) addMinimalOutputOption(parser) + addNoProgressBarOption(parser) group=parser.add_argument_group('obi head specific options') @@ -53,31 +55,41 @@ def run(config): if output is None: raise Exception("Could not create output view") o_dms = output[0] - o_view_name_final = output[1] - o_view_name = o_view_name_final + output_0 = output[0] + final_o_view_name = output[1] - # If the input and output DMS are not the same, create output view in input DMS first, then export it - # to output DMS, making sure the temporary view name is unique in the input DMS - if i_dms != o_dms: + # If stdout output or the input and output DMS are not the same, create a temporary view that will be exported and deleted. + if i_dms != o_dms or type(output_0)==BufferedWriter: + temporary_view_name = b"temp" i=0 - while o_view_name in i_dms: - o_view_name = o_view_name_final+b"_"+str2bytes(str(i)) - i+=1 + while temporary_view_name in i_dms: # Making sure view name is unique in input DMS + temporary_view_name = temporary_view_name+b"_"+str2bytes(str(i)) + i+=1 + o_view_name = temporary_view_name + if type(output_0)==BufferedWriter: + o_dms = i_dms + else: + o_view_name = final_o_view_name n = min(config['head']['count'], len(i_view)) # Initialize the progress bar - pb = ProgressBar(n, config, seconde=5) + if config['obi']['noprogressbar'] == False: + pb = ProgressBar(n, config) + else: + pb = None selection = Line_selection(i_view) for i in range(n): PyErr_CheckSignals() - pb(i) + if pb is not None: + pb(i) selection.append(i) - pb(i, force=True) - print("", file=sys.stderr) + if pb is not None: + pb(i, force=True) + print("", file=sys.stderr) # Create output view with the line selection try: @@ -94,14 +106,20 @@ def run(config): # and delete the temporary view in the input DMS if i_dms != o_dms: o_view.close() - View.import_view(i_dms.full_path[:-7], o_dms.full_path[:-7], o_view_name, o_view_name_final) - o_view = o_dms[o_view_name_final] + View.import_view(i_dms.full_path[:-7], o_dms.full_path[:-7], o_view_name, final_o_view_name) + o_view = o_dms[final_o_view_name] + + # stdout output: write to buffer + if type(output_0)==BufferedWriter: + logger("info", "Printing to output...") + o_view.print_to_output(output_0, noprogressbar=config['obi']['noprogressbar']) + o_view.close() #print("\n\nOutput view:\n````````````", file=sys.stderr) #print(repr(view), file=sys.stderr) - # If the input and the output DMS are different, delete the temporary imported view used to create the final view - if i_dms != o_dms: + # If the input and the output DMS are different or if stdout output, delete the temporary imported view used to create the final view + if i_dms != o_dms or type(output_0)==BufferedWriter: View.delete_view(i_dms, o_view_name) o_dms.close(force=True) i_dms.close(force=True) diff --git a/python/obitools3/commands/import.pyx b/python/obitools3/commands/import.pyx index 01ec794..60a7267 100755 --- a/python/obitools3/commands/import.pyx +++ b/python/obitools3/commands/import.pyx @@ -47,6 +47,8 @@ from obitools3.apps.config import logger from cpython.exc cimport PyErr_CheckSignals +from io import BufferedWriter + __title__="Imports sequences from different formats into a DMS" @@ -130,7 +132,7 @@ def run(config): if entry_count > 0: logger("info", "Importing %d entries", entry_count) else: - logger("info", "Importing an unknow number of entries") + logger("info", "Importing an unknown number of entries") # TODO a bit dirty? if input[2]==Nuc_Seq or input[2]==View_NUC_SEQS: @@ -178,7 +180,7 @@ def run(config): return if entry_count >= 0: - pb = ProgressBar(entry_count, config, seconde=5) + pb = ProgressBar(entry_count, config) NUC_SEQS_view = False if isinstance(output[1], View) : @@ -243,7 +245,7 @@ def run(config): if isinstance(input[0], CompressedFile): input_is_file = True if entry_count >= 0: - pb = ProgressBar(entry_count, config, seconde=5) + pb = ProgressBar(entry_count, config) try: input[0].close() except AttributeError: diff --git a/python/obitools3/commands/ngsfilter.pyx b/python/obitools3/commands/ngsfilter.pyx index 89371e0..22ee76e 100644 --- a/python/obitools3/commands/ngsfilter.pyx +++ b/python/obitools3/commands/ngsfilter.pyx @@ -2,10 +2,10 @@ from obitools3.apps.progress cimport ProgressBar # @UnresolvedImport from obitools3.dms import DMS -from obitools3.dms.view import RollbackException +from obitools3.dms.view import RollbackException, View from obitools3.dms.view.typed_view.view_NUC_SEQS cimport View_NUC_SEQS from obitools3.dms.column.column cimport Column, Column_line -from obitools3.apps.optiongroups import addMinimalInputOption, addMinimalOutputOption +from obitools3.apps.optiongroups import addMinimalInputOption, addMinimalOutputOption, addNoProgressBarOption from obitools3.uri.decode import open_uri from obitools3.apps.config import logger from obitools3.libalign._freeendgapfm import FreeEndGapFullMatch @@ -14,13 +14,14 @@ from obitools3.dms.obiseq cimport Nuc_Seq from obitools3.dms.capi.obitypes cimport OBI_SEQ, OBI_QUAL from obitools3.dms.capi.apat cimport MAX_PATTERN from obitools3.dms.capi.obiview cimport REVERSE_SEQUENCE_COLUMN, REVERSE_QUALITY_COLUMN -from obitools3.utils cimport tobytes +from obitools3.utils cimport tobytes, str2bytes from libc.stdint cimport INT32_MAX from functools import reduce import math import sys from cpython.exc cimport PyErr_CheckSignals +from io import BufferedWriter #REVERSE_SEQ_COLUMN_NAME = b"REVERSE_SEQUENCE" # used by alignpairedend tool @@ -34,7 +35,8 @@ def addOptions(parser): addMinimalInputOption(parser) addMinimalOutputOption(parser) - + addNoProgressBarOption(parser) + group = parser.add_argument_group('obi ngsfilter specific options') group.add_argument('-t','--info-view', @@ -539,7 +541,8 @@ def run(config): raise Exception("Could not open input reads") if input[2] != View_NUC_SEQS: raise NotImplementedError('obi ngsfilter only works on NUC_SEQS views') - + i_dms = input[0] + if "reverse" in config["ngsfilter"]: forward = input[1] @@ -580,8 +583,19 @@ def run(config): if output is None: raise Exception("Could not create output view") + o_dms = output[0] + output_0 = output[0] o_view = output[1] + # If stdout output, create a temporary view in the input dms that will be deleted afterwards. + if type(output_0)==BufferedWriter: + o_dms = i_dms + o_view_name = b"temp" + while o_view_name in i_dms: # Making sure view name is unique in input DMS + o_view_name = o_view_name+b"_"+str2bytes(str(i)) + i+=1 + o_view = View_NUC_SEQS.new(i_dms, o_view_name) + # Open the view containing the informations about the tags and the primers info_input = open_uri(config['ngsfilter']['info_view']) if info_input is None: @@ -602,7 +616,10 @@ def run(config): unidentified = None # Initialize the progress bar - pb = ProgressBar(entries_len, config, seconde=5) + if config['obi']['noprogressbar'] == False: + pb = ProgressBar(entries_len, config) + else: + pb = None # Check and store primers and tags try: @@ -636,7 +653,8 @@ def run(config): try: for i in range(entries_len): PyErr_CheckSignals() - pb(i) + if pb is not None: + pb(i) if not_aligned: modseq = [Nuc_Seq.new_from_stored(forward[i]), Nuc_Seq.new_from_stored(reverse[i])] else: @@ -660,8 +678,9 @@ def run(config): else: raise RollbackException("obi ngsfilter error, rollbacking view: "+str(e), o_view) - pb(i, force=True) - print("", file=sys.stderr) + if pb is not None: + pb(i, force=True) + print("", file=sys.stderr) # Save command config in View and DMS comments command_line = " ".join(sys.argv[1:]) @@ -670,13 +689,23 @@ def run(config): unidentified.write_config(config, "ngsfilter", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name) # Add comment about unidentified seqs unidentified.comments["info"] = "View containing sequences categorized as unidentified by the ngsfilter command" - output[0].record_command_line(command_line) + o_dms.record_command_line(command_line) + # stdout output: write to buffer + if type(output_0)==BufferedWriter: + logger("info", "Printing to output...") + o_view.print_to_output(output_0, noprogressbar=config['obi']['noprogressbar']) + o_view.close() + #print("\n\nOutput view:\n````````````", file=sys.stderr) #print(repr(o_view), file=sys.stderr) - input[0].close(force=True) - output[0].close(force=True) + # If stdout output, delete the temporary result view in the input DMS + if type(output_0)==BufferedWriter: + View.delete_view(i_dms, o_view_name) + + i_dms.close(force=True) + o_dms.close(force=True) info_input[0].close(force=True) if unidentified is not None: unidentified_input[0].close(force=True) diff --git a/python/obitools3/commands/sort.pyx b/python/obitools3/commands/sort.pyx index ff0929e..163bc66 100755 --- a/python/obitools3/commands/sort.pyx +++ b/python/obitools3/commands/sort.pyx @@ -4,7 +4,7 @@ from obitools3.apps.progress cimport ProgressBar # @UnresolvedImport from obitools3.dms import DMS from obitools3.dms.view.view cimport View, Line_selection from obitools3.uri.decode import open_uri -from obitools3.apps.optiongroups import addMinimalInputOption, addMinimalOutputOption +from obitools3.apps.optiongroups import addMinimalInputOption, addMinimalOutputOption, addNoProgressBarOption from obitools3.dms.view import RollbackException from obitools3.apps.config import logger from obitools3.utils cimport str2bytes @@ -24,6 +24,7 @@ from obitools3.dms.capi.obitypes cimport OBI_BOOL, \ import time import sys from cpython.exc cimport PyErr_CheckSignals +from io import BufferedWriter NULL_VALUE = {OBI_BOOL: OBIBool_NA, @@ -42,6 +43,7 @@ def addOptions(parser): addMinimalInputOption(parser) addMinimalOutputOption(parser) + addNoProgressBarOption(parser) group=parser.add_argument_group('obi sort specific options') @@ -59,7 +61,7 @@ def addOptions(parser): def line_cmp(line, key, pb): - pb + pb if line[key] is None: return NULL_VALUE[line.view[key].data_type_int] else: @@ -86,20 +88,28 @@ def run(config): if output is None: raise Exception("Could not create output view") o_dms = output[0] - o_view_name_final = output[1] - o_view_name = o_view_name_final + output_0 = output[0] + final_o_view_name = output[1] - # If the input and output DMS are not the same, create output view in input DMS first, then export it - # to output DMS, making sure the temporary view name is unique in the input DMS - if i_dms != o_dms: + # If stdout output or the input and output DMS are not the same, create a temporary view that will be exported and deleted. + if i_dms != o_dms or type(output_0)==BufferedWriter: + temporary_view_name = b"temp" i=0 - while o_view_name in i_dms: - o_view_name = o_view_name_final+b"_"+str2bytes(str(i)) - i+=1 + while temporary_view_name in i_dms: # Making sure view name is unique in input DMS + temporary_view_name = temporary_view_name+b"_"+str2bytes(str(i)) + i+=1 + o_view_name = temporary_view_name + if type(output_0)==BufferedWriter: + o_dms = i_dms + else: + o_view_name = final_o_view_name # Initialize the progress bar - pb = ProgressBar(len(i_view), config, seconde=5) - + if config['obi']['noprogressbar'] == False: + pb = ProgressBar(len(i_view), config) + else: + pb = None + keys = config['sort']['keys'] selection = Line_selection(i_view) @@ -110,10 +120,14 @@ def run(config): for k in keys: # TODO order? PyErr_CheckSignals() - selection.sort(key=lambda line_idx: line_cmp(i_view[line_idx], k, pb(line_idx)), reverse=config['sort']['reverse']) - - pb(len(i_view), force=True) - print("", file=sys.stderr) + if pb is not None: + selection.sort(key=lambda line_idx: line_cmp(i_view[line_idx], k, pb(line_idx)), reverse=config['sort']['reverse']) + else: + selection.sort(key=lambda line_idx: line_cmp(i_view[line_idx], k, None), reverse=config['sort']['reverse']) + + if pb is not None: + pb(len(i_view), force=True) + print("", file=sys.stderr) # Create output view with the sorted line selection try: @@ -132,16 +146,23 @@ def run(config): # and delete the temporary view in the input DMS if i_dms != o_dms: o_view.close() - View.import_view(i_dms.full_path[:-7], o_dms.full_path[:-7], o_view_name, o_view_name_final) - o_view = o_dms[o_view_name_final] + View.import_view(i_dms.full_path[:-7], o_dms.full_path[:-7], o_view_name, final_o_view_name) + o_view = o_dms[final_o_view_name] + + # stdout output: write to buffer + if type(output_0)==BufferedWriter: + logger("info", "Printing to output...") + o_view.print_to_output(output_0, noprogressbar=config['obi']['noprogressbar']) + o_view.close() #print("\n\nOutput view:\n````````````", file=sys.stderr) #print(repr(o_view), file=sys.stderr) - # If the input and the output DMS are different, delete the temporary imported view used to create the final view - if i_dms != o_dms: + # If the input and the output DMS are different or if stdout output, delete the temporary imported view used to create the final view + if i_dms != o_dms or type(output_0)==BufferedWriter: View.delete_view(i_dms, o_view_name) o_dms.close(force=True) + i_dms.close(force=True) logger("info", "Done.") diff --git a/python/obitools3/commands/stats.pyx b/python/obitools3/commands/stats.pyx index 97dbc53..f2b8274 100755 --- a/python/obitools3/commands/stats.pyx +++ b/python/obitools3/commands/stats.pyx @@ -162,7 +162,7 @@ def run(config): lcat=0 # Initialize the progress bar - pb = ProgressBar(len(i_view), config, seconde=5) + pb = ProgressBar(len(i_view), config) for i in range(len(i_view)): PyErr_CheckSignals() diff --git a/python/obitools3/commands/tail.pyx b/python/obitools3/commands/tail.pyx index aa9fd95..7024fa0 100755 --- a/python/obitools3/commands/tail.pyx +++ b/python/obitools3/commands/tail.pyx @@ -4,7 +4,7 @@ from obitools3.apps.progress cimport ProgressBar # @UnresolvedImport from obitools3.dms import DMS from obitools3.dms.view.view cimport View, Line_selection from obitools3.uri.decode import open_uri -from obitools3.apps.optiongroups import addMinimalInputOption, addMinimalOutputOption +from obitools3.apps.optiongroups import addMinimalInputOption, addMinimalOutputOption, addNoProgressBarOption from obitools3.dms.view import RollbackException from obitools3.apps.config import logger from obitools3.utils cimport str2bytes @@ -12,6 +12,7 @@ from obitools3.utils cimport str2bytes import time import sys from cpython.exc cimport PyErr_CheckSignals +from io import BufferedWriter __title__="Keep the N last lines of a view." @@ -21,6 +22,7 @@ def addOptions(parser): addMinimalInputOption(parser) addMinimalOutputOption(parser) + addNoProgressBarOption(parser) group=parser.add_argument_group('obi tail specific options') @@ -52,31 +54,41 @@ def run(config): if output is None: raise Exception("Could not create output view") o_dms = output[0] - o_view_name_final = output[1] - o_view_name = o_view_name_final + output_0 = output[0] + final_o_view_name = output[1] - # If the input and output DMS are not the same, create output view in input DMS first, then export it - # to output DMS, making sure the temporary view name is unique in the input DMS - if i_dms != o_dms: + # If stdout output or the input and output DMS are not the same, create a temporary view that will be exported and deleted. + if i_dms != o_dms or type(output_0)==BufferedWriter: + temporary_view_name = b"temp" i=0 - while o_view_name in i_dms: - o_view_name = o_view_name_final+b"_"+str2bytes(str(i)) - i+=1 + while temporary_view_name in i_dms: # Making sure view name is unique in input DMS + temporary_view_name = temporary_view_name+b"_"+str2bytes(str(i)) + i+=1 + o_view_name = temporary_view_name + if type(output_0)==BufferedWriter: + o_dms = i_dms + else: + o_view_name = final_o_view_name start = max(len(i_view) - config['tail']['count'], 0) # Initialize the progress bar - pb = ProgressBar(len(i_view) - start, config, seconde=5) + if config['obi']['noprogressbar'] == False: + pb = ProgressBar(len(i_view) - start, config) + else: + pb = None selection = Line_selection(i_view) for i in range(start, len(i_view)): PyErr_CheckSignals() - pb(i) + if pb is not None: + pb(i) selection.append(i) - pb(i, force=True) - print("", file=sys.stderr) + if pb is not None: + pb(i, force=True) + print("", file=sys.stderr) # Save command config in View comments command_line = " ".join(sys.argv[1:]) @@ -97,14 +109,20 @@ def run(config): # and delete the temporary view in the input DMS if i_dms != o_dms: o_view.close() - View.import_view(i_dms.full_path[:-7], o_dms.full_path[:-7], o_view_name, o_view_name_final) - o_view = o_dms[o_view_name_final] + View.import_view(i_dms.full_path[:-7], o_dms.full_path[:-7], o_view_name, final_o_view_name) + o_view = o_dms[final_o_view_name] + + # stdout output: write to buffer + if type(output_0)==BufferedWriter: + logger("info", "Printing to output...") + o_view.print_to_output(output_0, noprogressbar=config['obi']['noprogressbar']) + o_view.close() #print("\n\nOutput view:\n````````````", file=sys.stderr) #print(repr(o_view), file=sys.stderr) - # If the input and the output DMS are different, delete the temporary imported view used to create the final view - if i_dms != o_dms: + # If the input and the output DMS are different or if stdout output, delete the temporary imported view used to create the final view + if i_dms != o_dms or type(output_0)==BufferedWriter: View.delete_view(i_dms, o_view_name) o_dms.close(force=True) i_dms.close(force=True) diff --git a/python/obitools3/commands/uniq.pyx b/python/obitools3/commands/uniq.pyx index a9c4f62..15ff5d3 100644 --- a/python/obitools3/commands/uniq.pyx +++ b/python/obitools3/commands/uniq.pyx @@ -14,13 +14,15 @@ from obitools3.dms.capi.obitypes cimport OBI_INT, OBI_STR, index_t from obitools3.apps.optiongroups import addMinimalInputOption, \ addMinimalOutputOption, \ addTaxonomyOption, \ - addEltLimitOption + addEltLimitOption, \ + addNoProgressBarOption from obitools3.uri.decode import open_uri from obitools3.apps.config import logger -from obitools3.utils cimport tobytes, tostr +from obitools3.utils cimport tobytes, tostr, str2bytes import sys from cpython.exc cimport PyErr_CheckSignals +from io import BufferedWriter __title__="Group sequence records together" @@ -32,6 +34,7 @@ def addOptions(parser): addTaxonomyOption(parser) addMinimalOutputOption(parser) addEltLimitOption(parser) + addNoProgressBarOption(parser) group = parser.add_argument_group('obi uniq specific options') @@ -143,12 +146,16 @@ cdef merge_taxonomy_classification(View_NUC_SEQS o_view, Taxonomy taxonomy, dict scientific_name_column = o_view[b"scientific_name"] # Initialize the progress bar - pb = ProgressBar(len(o_view), config, seconde=5) + if config['obi']['noprogressbar'] == False: + pb = ProgressBar(len(o_view), config) + else: + pb = None i=0 for seq in o_view: PyErr_CheckSignals() - pb(i) + if pb is not None: + pb(i) if MERGED_TAXID_COLUMN in seq : m_taxids = [] m_taxids_dict = seq[MERGED_TAXID_COLUMN] @@ -191,7 +198,8 @@ cdef merge_taxonomy_classification(View_NUC_SEQS o_view, Taxonomy taxonomy, dict scientific_name_column[i] = taxonomy.get_scientific_name(taxid) i+=1 - pb(len(o_view), force=True) + if pb is not None: + pb(len(o_view), force=True) cdef uniq_sequences(View_NUC_SEQS view, View_NUC_SEQS o_view, ProgressBar pb, dict config, list mergedKeys_list=None, Taxonomy taxonomy=None, bint mergeIds=False, list categories=None, int max_elts=1000000) : @@ -297,7 +305,8 @@ cdef uniq_sequences(View_NUC_SEQS view, View_NUC_SEQS o_view, ProgressBar pb, di iter_view = iter(view) for i_seq in iter_view : PyErr_CheckSignals() - pb(i) + if pb is not None: + pb(i) # This can't be done in the same line as the unique_id tuple creation because it generates a bug # where Cython (version 0.25.2) does not detect the reference to the categs_list variable and deallocates @@ -415,12 +424,17 @@ cdef uniq_sequences(View_NUC_SEQS view, View_NUC_SEQS o_view, ProgressBar pb, di o_count_col = o_view[COUNT_COLUMN] if COUNT_COLUMN in view: i_count_col = view[COUNT_COLUMN] + + if pb is not None: + pb(len(view), force=True) + print("") - pb(len(view), force=True) - print("") logger("info", "Second browsing through the input") + # Initialize the progress bar - pb = ProgressBar(len(view), seconde=5) + if pb is not None: + pb = ProgressBar(len(view)) + o_idx = 0 total_treated = 0 @@ -455,7 +469,9 @@ cdef uniq_sequences(View_NUC_SEQS view, View_NUC_SEQS o_view, ProgressBar pb, di for i_idx in merged_sequences: PyErr_CheckSignals() - pb(total_treated) + + if pb is not None: + pb(total_treated) i_id = i_id_col[i_idx] i_seq = view[i_idx] @@ -531,7 +547,8 @@ cdef uniq_sequences(View_NUC_SEQS view, View_NUC_SEQS o_view, ProgressBar pb, di o_count_col[o_idx] = o_count o_idx += 1 - pb(len(view), force=True) + if pb is not None: + pb(len(view), force=True) # Deletes quality columns if there is one because the matching between sequence and quality will be broken (quality set to NA when sequence not) if QUALITY_COLUMN in view: @@ -579,8 +596,23 @@ def run(config): if output is None: raise Exception("Could not create output view") + i_dms = input[0] entries = input[1] - o_view = output[1] + o_dms = output[0] + output_0 = output[0] + + # If stdout output create a temporary view that will be exported and deleted. + if type(output_0)==BufferedWriter: + temporary_view_name = b"temp" + i=0 + while temporary_view_name in i_dms: # Making sure view name is unique in input DMS + temporary_view_name = temporary_view_name+b"_"+str2bytes(str(i)) + i+=1 + o_view_name = temporary_view_name + o_dms = i_dms + o_view = View_NUC_SEQS.new(i_dms, o_view_name) + else: + o_view = output[1] if 'taxoURI' in config['obi'] and config['obi']['taxoURI'] is not None: taxo_uri = open_uri(config['obi']['taxoURI']) @@ -591,7 +623,10 @@ def run(config): taxo = None # Initialize the progress bar - pb = ProgressBar(len(entries), config, seconde=5) + if config['obi']['noprogressbar'] == False: + pb = ProgressBar(len(entries), config) + else: + pb = None if len(entries) > 0: try: @@ -599,7 +634,8 @@ def run(config): except Exception, e: raise RollbackException("obi uniq error, rollbacking view: "+str(e), o_view) - print("", file=sys.stderr) + if pb is not None: + print("", file=sys.stderr) # Save command config in View and DMS comments command_line = " ".join(sys.argv[1:]) @@ -609,13 +645,23 @@ def run(config): input_dms_name.append(config['obi']['taxoURI'].split("/")[-3]) input_view_name.append("taxonomy/"+config['obi']['taxoURI'].split("/")[-1]) o_view.write_config(config, "uniq", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name) - output[0].record_command_line(command_line) + o_dms.record_command_line(command_line) + + # stdout output: write to buffer + if type(output_0)==BufferedWriter: + logger("info", "Printing to output...") + o_view.print_to_output(output_0, noprogressbar=config['obi']['noprogressbar']) + o_view.close() #print("\n\nOutput view:\n````````````", file=sys.stderr) #print(repr(o_view), file=sys.stderr) - - input[0].close(force=True) - output[0].close(force=True) + + # If stdout output, delete the temporary result view in the input DMS + if type(output_0)==BufferedWriter: + View.delete_view(i_dms, o_view_name) + + i_dms.close(force=True) + o_dms.close(force=True) logger("info", "Done.") diff --git a/python/obitools3/uri/decode.pyx b/python/obitools3/uri/decode.pyx index 4451503..c2b756d 100644 --- a/python/obitools3/uri/decode.pyx +++ b/python/obitools3/uri/decode.pyx @@ -210,10 +210,11 @@ def open_uri(uri, error = None - if scheme==b"dms" or \ - (scheme==b"" and \ - (((not input) and "outputformat" not in config["obi"]) or \ - (input and "inputformat" not in config["obi"]))): # TODO maybe not best way + if urib != b"-" and \ + (scheme==b"dms" or \ + (scheme==b"" and \ + (((not input) and "outputformat" not in config["obi"]) or \ + (input and "inputformat" not in config["obi"])))): # TODO maybe not best way if default_dms is not None and b"/" not in urip.path: # assuming view to open in default DMS (TODO discuss) dms=(default_dms, urip.path) @@ -275,10 +276,10 @@ def open_uri(uri, iseq = urib objclass = bytes else: # TODO update uopen to be able to write? - if urip.path: - file = open(urip.path, 'wb') - else: + if urip.path == b'-': file = sys.stdout.buffer + elif urip.path : + file = open(urip.path, 'wb') if file is not None: qualifiers=parse_qs(urip.query)