obitools/obitools3
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 66 Wiki Activity

Commands: updated for JSON formatted comments with history

Browse Source
This commit is contained in:
Celine Mercier
2018-10-07 19:10:34 +02:00
parent 35cf2962cc
commit e9a41c5b97
11 changed files with 144 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
python/obitools3/commands/alignpairedend.pyx
View File

@ -16,6 +16,9 @@ from obitools3.align._qsrassemble import QSolexaRightReverseAssemble
from obitools3.align._solexapairend import buildConsensus, buildJoinedSequence
from obitools3.dms.obiseq cimport Nuc_Seq_Stored, Nuc_Seq
import sys
import os
REVERSE_SEQ_COLUMN_NAME = b"REVERSE_SEQUENCE"
REVERSE_QUALITY_COLUMN_NAME = b"REVERSE_QUALITY"
@ -144,9 +147,13 @@ def run(config):
raise Exception("Error: the number of forward and reverse reads are different")
entries = [forward, reverse]
input_dms_name = [forward.dms.name, reverse.dms.name]
input_view_name = [forward.name, reverse.name]
else:
entries = input[1]
input_dms_name = [entries.dms.name]
input_view_name = [entries.name]
if two_views:
entries_len = len(forward)
@ -166,6 +173,7 @@ def run(config):
raise Exception("Could not create output view")
view = output[1]
Column.new_column(view, b"QUALITY", OBI_QUAL) #TODO output URI quality option
if 'smin' in config['alignpairedend']:
@ -214,6 +222,11 @@ def run(config):
i+=1
# Save command config in View and DMS comments
command_line = " ".join(sys.argv[1:])
view.write_config(config, "alignpairedend", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name)
output[0].record_command_line(command_line)
print("\n")
print(repr(view))

20
python/obitools3/commands/annotate.pyx
View File

@ -9,16 +9,16 @@ from obitools3.dms.view import RollbackException
from functools import reduce
from obitools3.apps.config import logger
from obitools3.utils cimport tobytes
from obitools3.dms.capi.obiview cimport QUALITY_COLUMN
import time
import math
from obitools3.dms.capi.obiview cimport NUC_SEQUENCE_COLUMN, \
ID_COLUMN, \
DEFINITION_COLUMN, \
QUALITY_COLUMN, \
COUNT_COLUMN
import time
import math
import sys
__title__="Annotate views with new tags and edit existing annotations"
@ -285,7 +285,7 @@ def run(config):
output_view_name = uri[0]
# Clone output view from input view
o_view = i_view.clone(output_view_name, comments=i_view.comments+b"\nobi annotate") # TODO comments
o_view = i_view.clone(output_view_name, comments=i_view.comments) # TODO comments
if o_view is None:
raise Exception("Couldn't create output view")
@ -342,6 +342,16 @@ def run(config):
except Exception, e:
raise RollbackException("obi annotate error, rollbacking view: "+str(e), o_view)
# Save command config in View and DMS comments
command_line = " ".join(sys.argv[1:])
input_dms_name=[input[0].name]
input_view_name=[input[1].name]
if 'taxoURI' in config['obi'] and config['obi']['taxoURI'] is not None:
input_dms_name.append(config['obi']['taxoURI'].split("/", 1)[0])
input_view_name.append(config['obi']['taxoURI'].split("/", 1)[1])
o_view.write_config(config, "annotate", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name)
input[0].record_command_line(command_line) # TODO assuming same dms
print("\n")
print(repr(o_view))

13
python/obitools3/commands/clean.pyx
View File

@ -9,9 +9,11 @@ from obitools3.apps.optiongroups import addSequenceInputOption, addMinimalOutput
from obitools3.uri.decode import open_uri
from obitools3.apps.config import logger
from obitools3.utils cimport tobytes
from obitools3.dms.view.view cimport View
from obitools3.dms.view.typed_view.view_NUC_SEQS cimport View_NUC_SEQS
import sys
__title__="Tag a set of sequences for PCR and sequencing errors identification"
@ -83,10 +85,17 @@ def run(config):
else:
o_view_name = uri_o[0]
if obi_clean(tobytes(dms_name), tobytes(i_view_name), tobytes(config['clean']['sample-tag-name']), tobytes(o_view_name), b"obiclean", \
# Save command config in View comments
command_line = " ".join(sys.argv[1:])
comments = View.print_config(config, "clean", command_line, input_dms_name=[dms_name], input_view_name=[i_view_name])
if obi_clean(tobytes(dms_name), tobytes(i_view_name), tobytes(config['clean']['sample-tag-name']), tobytes(o_view_name), comments, \
config['clean']['distance'], config['clean']['ratio'], config['clean']['heads-only'], 1) < 0:
raise Exception("Error running obiclean")
# Save command config in DMS comments
dms.record_command_line(command_line)
print("\n")
print(repr(dms[o_view_name]))

14
python/obitools3/commands/ecopcr.pyx
View File

@ -10,10 +10,13 @@ from obitools3.uri.decode import open_uri
from obitools3.apps.config import logger
from obitools3.utils cimport tobytes
from obitools3.dms.view.typed_view.view_NUC_SEQS cimport View_NUC_SEQS
from obitools3.dms.view import View
from libc.stdlib cimport malloc, free
from libc.stdint cimport int32_t
import sys
__title__="in silico PCR"
@ -166,10 +169,14 @@ def run(config):
# Read taxonomy name
taxonomy_name = config['obi']['taxoURI'].split('/')[2]
# TODO: input DMS, taxonomy and primers in comments
# Save command config in View comments
command_line = " ".join(sys.argv[1:])
comments = View.print_config(config, "ecopcr", command_line, input_dms_name=[i_dms_name], input_view_name=[i_view_name, config['obi']['taxoURI']])
# TODO: primers in comments?
if obi_ecopcr(tobytes(i_dms_name), tobytes(i_view_name), tobytes(taxonomy_name), \
tobytes(o_dms_name), tobytes(o_view_name), b"ecopcr", \
tobytes(o_dms_name), tobytes(o_view_name), comments, \
tobytes(config['ecopcr']['primer1']), tobytes(config['ecopcr']['primer2']), \
config['ecopcr']['error'], \
config['ecopcr']['min-length'], config['ecopcr']['max-length'], \
@ -178,6 +185,9 @@ def run(config):
config['ecopcr']['keep-nucs'], config['ecopcr']['kingdom-mode']) < 0:
raise Exception("Error running ecopcr")
# Save command config in DMS comments
o_dms.record_command_line(command_line)
free(restrict_to_taxids_p)
free(ignore_taxids_p)

15
python/obitools3/commands/grep.pyx
View File

@ -12,6 +12,7 @@ from obitools3.utils cimport tobytes
from functools import reduce
import time
import re
import sys
__title__="Grep view lines that match the given predicates"
@ -29,7 +30,7 @@ def addOptions(parser):
group=parser.add_argument_group("obi grep specific options")
group.add_argument("--predicate", "-p",
action="append", dest="grep:predicates",
action="append", dest="grep:grep_predicates",
metavar="<PREDICATE>",
default=None,
type=str,
@ -306,12 +307,22 @@ def run(config):
# Create output view with the line selection
try:
o_view = selection.materialize(output_view_name, comments="obi grep: "+str(config["grep"])+"\n")
o_view = selection.materialize(output_view_name)
except Exception, e:
raise RollbackException("obi grep error, rollbacking view: "+str(e), o_view)
# TODO DISCUSS if output URI to different DMS, copy view?
# Save command config in View and DMS comments
command_line = " ".join(sys.argv[1:])
input_dms_name=[input[0].name]
input_view_name=[input[1].name]
if 'taxoURI' in config['obi'] and config['obi']['taxoURI'] is not None:
input_dms_name.append(config['obi']['taxoURI'].split("/", 1)[0])
input_view_name.append(config['obi']['taxoURI'].split("/", 1)[1])
o_view.write_config(config, "grep", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name)
input[0].record_command_line(command_line) # TODO assuming input and output dms are the same
print("\n")
print(repr(o_view))

11
python/obitools3/commands/head.pyx
View File

@ -9,6 +9,7 @@ from obitools3.dms.view import RollbackException
from obitools3.apps.config import logger
import time
import sys
__title__="Keep the N first lines of a view."
@ -39,6 +40,7 @@ def run(config):
input = open_uri(config['obi']['inputURI'])
if input is None:
raise Exception("Could not read input view")
i_dms = input[0]
i_view = input[1]
# Read the name of the output view
@ -61,14 +63,21 @@ def run(config):
for i in range(n):
selection.append(i)
# Save command config in View comments
command_line = " ".join(sys.argv[1:])
comments = View.print_config(config, "head", command_line, input_dms_name=[i_dms.name], input_view_name=[i_view.name])
# Create output view with the line selection
try:
o_view = selection.materialize(output_view_name, comments="obi head: "+str(len(selection))+"\n")
o_view = selection.materialize(output_view_name, comments=comments)
except Exception, e:
raise RollbackException("obi head error, rollbacking view: "+str(e), o_view)
# TODO DISCUSS if output URI to different DMS, copy view?
# Save command config in DMS comments
i_dms.record_command_line(command_line)
print("\n")
print(repr(o_view))

9
python/obitools3/commands/import.pyx
View File

@ -1,6 +1,7 @@
#cython: language_level=3
import sys
import os
from obitools3.apps.progress cimport ProgressBar # @UnresolvedImport
from obitools3.dms.view.view cimport View
@ -89,7 +90,7 @@ def run(config):
logger("info", "obi import : imports file into a DMS")
if not config['obi']['taxdump']: # TODO discuss
if not config['obi']['taxdump']:
input = open_uri(config['obi']['inputURI'])
if input is None: # TODO check for bytes instead now?
raise Exception("Could not open input URI")
@ -114,6 +115,7 @@ def run(config):
taxo = Taxonomy.open_taxdump(output[0], config['obi']['inputURI'])
taxo.write(output[1])
taxo.close()
output[0].record_command_line(" ".join(sys.argv[1:]))
output[0].close()
return
@ -258,6 +260,11 @@ def run(config):
i+=1
# Save command config in View and DMS comments
command_line = " ".join(sys.argv[1:])
view.write_config(config, "import", command_line, input_str=[os.path.abspath(config['obi']['inputURI'])])
output[0].record_command_line(command_line)
print("\n")
print(view.__repr__())

16
python/obitools3/commands/ngsfilter.pyx
View File

@ -14,6 +14,7 @@ from obitools3.dms.capi.obitypes cimport OBI_SEQ, OBI_QUAL
from functools import reduce, cmp_to_key
import math
import sys
REVERSE_SEQ_COLUMN_NAME = b"REVERSE_SEQUENCE"
@ -437,8 +438,14 @@ def run(config):
entries = [forward, reverse]
not_aligned = True
input_dms_name = [forward.dms.name, reverse.dms.name]
input_view_name = [forward.name, reverse.name]
else:
entries = input[1]
input_dms_name = [entries.dms.name]
input_view_name = [entries.name]
if not_aligned:
entries_len = len(forward)
@ -460,6 +467,8 @@ def run(config):
if info_input is None:
raise Exception("Could not read the view containing the informations about the tags and the primers")
info_view = info_input[1]
input_dms_name.append(info_input[0].name)
input_view_name.append(info_input[1].name)
# Open the unidentified view
if 'unidentified' in config['ngsfilter'] and config['ngsfilter']['unidentified'] is not None: # TODO keyError if undefined problem
@ -504,6 +513,13 @@ def run(config):
except Exception, e:
raise RollbackException("obi ngsfilter error, rollbacking views: "+str(e), o_view, unidentified)
# Save command config in View and DMS comments
command_line = " ".join(sys.argv[1:])
o_view.write_config(config, "ngsfilter", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name)
unidentified.write_config(config, "ngsfilter", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name)
# TODO add comment about unidentified seqs
output[0].record_command_line(command_line) # TODO if same dms...
print("\n")
print(repr(o_view))

6
python/obitools3/commands/sort.pyx
View File

@ -20,6 +20,7 @@ from obitools3.dms.capi.obitypes cimport OBI_BOOL, \
OBIInt_NA
import time
import sys
NULL_VALUE = {OBI_BOOL: OBIBool_NA,
@ -105,6 +106,11 @@ def run(config):
# TODO DISCUSS if output URI to different DMS, copy view?
# Save command config in View and DMS comments
command_line = " ".join(sys.argv[1:])
o_view.write_config(config, "sort", command_line, input_dms_name=[input[0].name], input_view_name=[input[1].name])
input[0].record_command_line(command_line) # TODO assuming same dms...
print("\n")
print(repr(o_view))

9
python/obitools3/commands/tail.pyx
View File

@ -9,6 +9,7 @@ from obitools3.dms.view import RollbackException
from obitools3.apps.config import logger
import time
import sys
__title__="Keep the N last lines of a view."
@ -39,6 +40,7 @@ def run(config):
input = open_uri(config['obi']['inputURI'])
if input is None:
raise Exception("Could not read input view")
i_dms = input[0]
i_view = input[1]
# Read the name of the output view
@ -61,6 +63,10 @@ def run(config):
for i in range(start, len(i_view)):
selection.append(i)
# Save command config in View comments
command_line = " ".join(sys.argv[1:])
comments = View.print_config(config, "tail", command_line, input_dms_name=[i_dms.name], input_view_name=[i_view.name])
# Create output view with the line selection
try:
o_view = selection.materialize(output_view_name, comments="obi tail: "+str(len(selection))+"\n")
@ -69,6 +75,9 @@ def run(config):
# TODO DISCUSS if output URI to different DMS, copy view?
# Save command config in DMS comments
i_dms.record_command_line(command_line)
print("\n")
print(repr(o_view))

14
python/obitools3/commands/uniq.pyx
View File

@ -14,6 +14,8 @@ from obitools3.uri.decode import open_uri
from obitools3.apps.config import logger
from obitools3.utils cimport tobytes
import sys
__title__="Group sequence records together"
@ -214,8 +216,6 @@ cdef uniq_sequences(View_NUC_SEQS view, View_NUC_SEQS o_view, ProgressBar pb, li
cdef object mcol
cdef object to_merge
#print(categories)
uniques = {}
mergedKeys_list_b = []
@ -522,6 +522,16 @@ def run(config):
except Exception, e:
raise RollbackException("obi uniq error, rollbacking view: "+str(e), o_view)
# Save command config in View and DMS comments
command_line = " ".join(sys.argv[1:])
input_dms_name=[input[0].name]
input_view_name=[input[1].name]
if 'taxoURI' in config['obi'] and config['obi']['taxoURI'] is not None:
input_dms_name.append(config['obi']['taxoURI'].split("/", 1)[0])
input_view_name.append(config['obi']['taxoURI'].split("/", 1)[1])
o_view.write_config(config, "uniq", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name)
output[0].record_command_line(command_line)
print("\n")
print(repr(o_view))