obitools/obitools3
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 66 Wiki Activity

Commands: updated for JSON formatted comments with history

Browse Source
This commit is contained in:
Celine Mercier
2018-10-07 19:10:34 +02:00
parent 35cf2962cc
commit e9a41c5b97
11 changed files with 144 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
python/obitools3/commands/alignpairedend.pyx
View File

@ -16,6 +16,9 @@ from obitools3.align._qsrassemble import QSolexaRightReverseAssemble
from obitools3.align._solexapairend import buildConsensus, buildJoinedSequence
from obitools3.dms.obiseq cimport Nuc_Seq_Stored, Nuc_Seq
import sys
import os
REVERSE_SEQ_COLUMN_NAME = b"REVERSE_SEQUENCE"
REVERSE_QUALITY_COLUMN_NAME = b"REVERSE_QUALITY"
@ -112,7 +115,7 @@ def run(config):
DMS.obi_atexit()
logger("info", "obi alignpairedend")
# Open the input
two_views = False
@ -124,8 +127,8 @@ def run(config):
if input is None:
raise Exception("Could not open input reads")
if input[2] != View_NUC_SEQS:
raise NotImplementedError('obi alignpairedend only works on NUC_SEQS views')
raise NotImplementedError('obi alignpairedend only works on NUC_SEQS views')
if "reverse" in config["alignpairedend"]:
two_views = True
@ -144,10 +147,14 @@ def run(config):
raise Exception("Error: the number of forward and reverse reads are different")
entries = [forward, reverse]
input_dms_name = [forward.dms.name, reverse.dms.name]
input_view_name = [forward.name, reverse.name]
else:
entries = input[1]
input_dms_name = [entries.dms.name]
input_view_name = [entries.name]
if two_views:
entries_len = len(forward)
else:
@ -166,6 +173,7 @@ def run(config):
raise Exception("Could not create output view")
view = output[1]
Column.new_column(view, b"QUALITY", OBI_QUAL) #TODO output URI quality option
if 'smin' in config['alignpairedend']:
@ -213,7 +221,12 @@ def run(config):
# consensus[b"illumina_index"] = idx
i+=1
# Save command config in View and DMS comments
command_line = " ".join(sys.argv[1:])
view.write_config(config, "alignpairedend", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name)
output[0].record_command_line(command_line)
print("\n")
print(repr(view))

22
python/obitools3/commands/annotate.pyx
View File

@ -9,16 +9,16 @@ from obitools3.dms.view import RollbackException
from functools import reduce
from obitools3.apps.config import logger
from obitools3.utils cimport tobytes
from obitools3.dms.capi.obiview cimport QUALITY_COLUMN
import time
import math
from obitools3.dms.capi.obiview cimport NUC_SEQUENCE_COLUMN, \
ID_COLUMN, \
DEFINITION_COLUMN, \
QUALITY_COLUMN, \
COUNT_COLUMN
import time
import math
import sys
__title__="Annotate views with new tags and edit existing annotations"
@ -285,10 +285,10 @@ def run(config):
output_view_name = uri[0]
# Clone output view from input view
o_view = i_view.clone(output_view_name, comments=i_view.comments+b"\nobi annotate") # TODO comments
o_view = i_view.clone(output_view_name, comments=i_view.comments) # TODO comments
if o_view is None:
raise Exception("Couldn't create output view")
if 'taxoURI' in config['obi'] : # TODO default None problem
taxo_uri = open_uri(config['obi']['taxoURI'])
if taxo_uri is None:
@ -341,6 +341,16 @@ def run(config):
except Exception, e:
raise RollbackException("obi annotate error, rollbacking view: "+str(e), o_view)
# Save command config in View and DMS comments
command_line = " ".join(sys.argv[1:])
input_dms_name=[input[0].name]
input_view_name=[input[1].name]
if 'taxoURI' in config['obi'] and config['obi']['taxoURI'] is not None:
input_dms_name.append(config['obi']['taxoURI'].split("/", 1)[0])
input_view_name.append(config['obi']['taxoURI'].split("/", 1)[1])
o_view.write_config(config, "annotate", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name)
input[0].record_command_line(command_line) # TODO assuming same dms
print("\n")
print(repr(o_view))

15
python/obitools3/commands/clean.pyx
View File

@ -9,9 +9,11 @@ from obitools3.apps.optiongroups import addSequenceInputOption, addMinimalOutput
from obitools3.uri.decode import open_uri
from obitools3.apps.config import logger
from obitools3.utils cimport tobytes
from obitools3.dms.view.view cimport View
from obitools3.dms.view.typed_view.view_NUC_SEQS cimport View_NUC_SEQS
import sys
__title__="Tag a set of sequences for PCR and sequencing errors identification"
@ -82,11 +84,18 @@ def run(config):
o_view_name = uri_o[1]
else:
o_view_name = uri_o[0]
if obi_clean(tobytes(dms_name), tobytes(i_view_name), tobytes(config['clean']['sample-tag-name']), tobytes(o_view_name), b"obiclean", \
# Save command config in View comments
command_line = " ".join(sys.argv[1:])
comments = View.print_config(config, "clean", command_line, input_dms_name=[dms_name], input_view_name=[i_view_name])
if obi_clean(tobytes(dms_name), tobytes(i_view_name), tobytes(config['clean']['sample-tag-name']), tobytes(o_view_name), comments, \
config['clean']['distance'], config['clean']['ratio'], config['clean']['heads-only'], 1) < 0:
raise Exception("Error running obiclean")
# Save command config in DMS comments
dms.record_command_line(command_line)
print("\n")
print(repr(dms[o_view_name]))

16
python/obitools3/commands/ecopcr.pyx
View File

@ -10,10 +10,13 @@ from obitools3.uri.decode import open_uri
from obitools3.apps.config import logger
from obitools3.utils cimport tobytes
from obitools3.dms.view.typed_view.view_NUC_SEQS cimport View_NUC_SEQS
from obitools3.dms.view import View
from libc.stdlib cimport malloc, free
from libc.stdint cimport int32_t
import sys
__title__="in silico PCR"
@ -166,10 +169,14 @@ def run(config):
# Read taxonomy name
taxonomy_name = config['obi']['taxoURI'].split('/')[2]
# TODO: input DMS, taxonomy and primers in comments
# Save command config in View comments
command_line = " ".join(sys.argv[1:])
comments = View.print_config(config, "ecopcr", command_line, input_dms_name=[i_dms_name], input_view_name=[i_view_name, config['obi']['taxoURI']])
# TODO: primers in comments?
if obi_ecopcr(tobytes(i_dms_name), tobytes(i_view_name), tobytes(taxonomy_name), \
tobytes(o_dms_name), tobytes(o_view_name), b"ecopcr", \
tobytes(o_dms_name), tobytes(o_view_name), comments, \
tobytes(config['ecopcr']['primer1']), tobytes(config['ecopcr']['primer2']), \
config['ecopcr']['error'], \
config['ecopcr']['min-length'], config['ecopcr']['max-length'], \
@ -177,7 +184,10 @@ def run(config):
config['ecopcr']['circular'], config['ecopcr']['salt-concentration'], config['ecopcr']['salt-correction-method'], \
config['ecopcr']['keep-nucs'], config['ecopcr']['kingdom-mode']) < 0:
raise Exception("Error running ecopcr")
# Save command config in DMS comments
o_dms.record_command_line(command_line)
free(restrict_to_taxids_p)
free(ignore_taxids_p)

17
python/obitools3/commands/grep.pyx
View File

@ -12,7 +12,8 @@ from obitools3.utils cimport tobytes
from functools import reduce
import time
import re
import sys
__title__="Grep view lines that match the given predicates"
@ -29,7 +30,7 @@ def addOptions(parser):
group=parser.add_argument_group("obi grep specific options")
group.add_argument("--predicate", "-p",
action="append", dest="grep:predicates",
action="append", dest="grep:grep_predicates",
metavar="<PREDICATE>",
default=None,
type=str,
@ -306,12 +307,22 @@ def run(config):
# Create output view with the line selection
try:
o_view = selection.materialize(output_view_name, comments="obi grep: "+str(config["grep"])+"\n")
o_view = selection.materialize(output_view_name)
except Exception, e:
raise RollbackException("obi grep error, rollbacking view: "+str(e), o_view)
# TODO DISCUSS if output URI to different DMS, copy view?
# Save command config in View and DMS comments
command_line = " ".join(sys.argv[1:])
input_dms_name=[input[0].name]
input_view_name=[input[1].name]
if 'taxoURI' in config['obi'] and config['obi']['taxoURI'] is not None:
input_dms_name.append(config['obi']['taxoURI'].split("/", 1)[0])
input_view_name.append(config['obi']['taxoURI'].split("/", 1)[1])
o_view.write_config(config, "grep", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name)
input[0].record_command_line(command_line) # TODO assuming input and output dms are the same
print("\n")
print(repr(o_view))

13
python/obitools3/commands/head.pyx
View File

@ -9,6 +9,7 @@ from obitools3.dms.view import RollbackException
from obitools3.apps.config import logger
import time
import sys
__title__="Keep the N first lines of a view."
@ -39,6 +40,7 @@ def run(config):
input = open_uri(config['obi']['inputURI'])
if input is None:
raise Exception("Could not read input view")
i_dms = input[0]
i_view = input[1]
# Read the name of the output view
@ -60,15 +62,22 @@ def run(config):
for i in range(n):
selection.append(i)
# Save command config in View comments
command_line = " ".join(sys.argv[1:])
comments = View.print_config(config, "head", command_line, input_dms_name=[i_dms.name], input_view_name=[i_view.name])
# Create output view with the line selection
try:
o_view = selection.materialize(output_view_name, comments="obi head: "+str(len(selection))+"\n")
o_view = selection.materialize(output_view_name, comments=comments)
except Exception, e:
raise RollbackException("obi head error, rollbacking view: "+str(e), o_view)
# TODO DISCUSS if output URI to different DMS, copy view?
# Save command config in DMS comments
i_dms.record_command_line(command_line)
print("\n")
print(repr(o_view))

19
python/obitools3/commands/import.pyx
View File

@ -1,6 +1,7 @@
#cython: language_level=3
import sys
import os
from obitools3.apps.progress cimport ProgressBar # @UnresolvedImport
from obitools3.dms.view.view cimport View
@ -89,7 +90,7 @@ def run(config):
logger("info", "obi import : imports file into a DMS")
if not config['obi']['taxdump']: # TODO discuss
if not config['obi']['taxdump']:
input = open_uri(config['obi']['inputURI'])
if input is None: # TODO check for bytes instead now?
raise Exception("Could not open input URI")
@ -108,12 +109,13 @@ def run(config):
#quality=get_quality) # TODO
if output is None:
raise Exception("Could not create output view")
# Read taxdump
if config['obi']['taxdump']: # The input is a taxdump to import in a DMS
taxo = Taxonomy.open_taxdump(output[0], config['obi']['inputURI'])
taxo.write(output[1])
taxo.close()
output[0].record_command_line(" ".join(sys.argv[1:]))
output[0].close()
return
@ -132,16 +134,16 @@ def run(config):
raise NotImplementedError()
# Save basic columns in variables for optimization
if NUC_SEQS_view :
if NUC_SEQS_view :
id_col = view[b"ID"] # TODO use macros or globals for column names
def_col = view[b"DEFINITION"]
seq_col = view[b"NUC_SEQ"]
dcols = {}
i = 0
for entry in entries :
pb(i)
if NUC_SEQS_view :
@ -258,6 +260,11 @@ def run(config):
i+=1
# Save command config in View and DMS comments
command_line = " ".join(sys.argv[1:])
view.write_config(config, "import", command_line, input_str=[os.path.abspath(config['obi']['inputURI'])])
output[0].record_command_line(command_line)
print("\n")
print(view.__repr__())

16
python/obitools3/commands/ngsfilter.pyx
View File

@ -14,6 +14,7 @@ from obitools3.dms.capi.obitypes cimport OBI_SEQ, OBI_QUAL
from functools import reduce, cmp_to_key
import math
import sys
REVERSE_SEQ_COLUMN_NAME = b"REVERSE_SEQUENCE"
@ -436,9 +437,15 @@ def run(config):
entries = [forward, reverse]
not_aligned = True
input_dms_name = [forward.dms.name, reverse.dms.name]
input_view_name = [forward.name, reverse.name]
else:
entries = input[1]
input_dms_name = [entries.dms.name]
input_view_name = [entries.name]
if not_aligned:
entries_len = len(forward)
@ -460,6 +467,8 @@ def run(config):
if info_input is None:
raise Exception("Could not read the view containing the informations about the tags and the primers")
info_view = info_input[1]
input_dms_name.append(info_input[0].name)
input_view_name.append(info_input[1].name)
# Open the unidentified view
if 'unidentified' in config['ngsfilter'] and config['ngsfilter']['unidentified'] is not None: # TODO keyError if undefined problem
@ -504,6 +513,13 @@ def run(config):
except Exception, e:
raise RollbackException("obi ngsfilter error, rollbacking views: "+str(e), o_view, unidentified)
# Save command config in View and DMS comments
command_line = " ".join(sys.argv[1:])
o_view.write_config(config, "ngsfilter", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name)
unidentified.write_config(config, "ngsfilter", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name)
# TODO add comment about unidentified seqs
output[0].record_command_line(command_line) # TODO if same dms...
print("\n")
print(repr(o_view))

8
python/obitools3/commands/sort.pyx
View File

@ -20,7 +20,8 @@ from obitools3.dms.capi.obitypes cimport OBI_BOOL, \
OBIInt_NA
import time
import sys
NULL_VALUE = {OBI_BOOL: OBIBool_NA,
OBI_CHAR: OBIChar_NA,
@ -105,6 +106,11 @@ def run(config):
# TODO DISCUSS if output URI to different DMS, copy view?
# Save command config in View and DMS comments
command_line = " ".join(sys.argv[1:])
o_view.write_config(config, "sort", command_line, input_dms_name=[input[0].name], input_view_name=[input[1].name])
input[0].record_command_line(command_line) # TODO assuming same dms...
print("\n")
print(repr(o_view))

13
python/obitools3/commands/tail.pyx
View File

@ -9,7 +9,8 @@ from obitools3.dms.view import RollbackException
from obitools3.apps.config import logger
import time
import sys
__title__="Keep the N last lines of a view."
@ -39,6 +40,7 @@ def run(config):
input = open_uri(config['obi']['inputURI'])
if input is None:
raise Exception("Could not read input view")
i_dms = input[0]
i_view = input[1]
# Read the name of the output view
@ -60,7 +62,11 @@ def run(config):
for i in range(start, len(i_view)):
selection.append(i)
# Save command config in View comments
command_line = " ".join(sys.argv[1:])
comments = View.print_config(config, "tail", command_line, input_dms_name=[i_dms.name], input_view_name=[i_view.name])
# Create output view with the line selection
try:
o_view = selection.materialize(output_view_name, comments="obi tail: "+str(len(selection))+"\n")
@ -69,6 +75,9 @@ def run(config):
# TODO DISCUSS if output URI to different DMS, copy view?
# Save command config in DMS comments
i_dms.record_command_line(command_line)
print("\n")
print(repr(o_view))

16
python/obitools3/commands/uniq.pyx
View File

@ -14,6 +14,8 @@ from obitools3.uri.decode import open_uri
from obitools3.apps.config import logger
from obitools3.utils cimport tobytes
import sys
__title__="Group sequence records together"
@ -214,8 +216,6 @@ cdef uniq_sequences(View_NUC_SEQS view, View_NUC_SEQS o_view, ProgressBar pb, li
cdef object mcol
cdef object to_merge
#print(categories)
uniques = {}
mergedKeys_list_b = []
@ -521,7 +521,17 @@ def run(config):
uniq_sequences(entries, o_view, pb, mergedKeys_list=config['uniq']['merge'], taxonomy=taxo, mergeIds=config['uniq']['mergeids'], categories=config['uniq']['categories'], max_elts=config['obi']['maxelts'])
except Exception, e:
raise RollbackException("obi uniq error, rollbacking view: "+str(e), o_view)
# Save command config in View and DMS comments
command_line = " ".join(sys.argv[1:])
input_dms_name=[input[0].name]
input_view_name=[input[1].name]
if 'taxoURI' in config['obi'] and config['obi']['taxoURI'] is not None:
input_dms_name.append(config['obi']['taxoURI'].split("/", 1)[0])
input_view_name.append(config['obi']['taxoURI'].split("/", 1)[1])
o_view.write_config(config, "uniq", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name)
output[0].record_command_line(command_line)
print("\n")
print(repr(o_view))