obitools/obitools3
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 66 Wiki Activity

All commands now handle outputing to another DMS + small fixes

Browse Source
This commit is contained in:
Celine Mercier
2018-11-02 19:03:09 +01:00
parent 8a8e9e50b2
commit 35f3e7c30b
10 changed files with 266 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
python/obitools3/commands/annotate.pyx
View File

@ -8,7 +8,7 @@ from obitools3.apps.optiongroups import addMinimalInputOption, addTaxonomyOption
from obitools3.dms.view import RollbackException
from functools import reduce
from obitools3.apps.config import logger
from obitools3.utils cimport tobytes
from obitools3.utils cimport tobytes, str2bytes
from obitools3.dms.capi.obiview cimport NUC_SEQUENCE_COLUMN, \
ID_COLUMN, \
DEFINITION_COLUMN, \
@ -272,24 +272,39 @@ def run(config):
input = open_uri(config['obi']['inputURI'])
if input is None:
raise Exception("Could not read input view")
i_dms = input[0]
i_view = input[1]
i_view_name = input[1].name
# Read the name of the output view
uri = config['obi']['outputURI'].split('/')
if len(uri)==2:
# Check that input and output DMS are the same (predicate, to discuss)
if config['obi']['inputURI'].split('/')[0] != uri[0]:
raise Exception("Input and output DMS must be the same")
output_view_name = uri[1]
else:
output_view_name = uri[0]
# Open the output: only the DMS, as the output view is going to be created by cloning the input view
# (could eventually be done via an open_uri() argument)
output = open_uri(config['obi']['outputURI'],
input=False,
dms_only=True)
if output is None:
raise Exception("Could not create output view")
o_dms = output[0]
o_view_name = output[1]
# If the input and output DMS are not the same, import the input view in the output DMS before cloning it to modify it
# (could be the other way around: clone and modify in the input DMS then import the new view in the output DMS)
if i_dms != o_dms:
imported_view_name = i_view_name
i=0
while imported_view_name in o_dms: # Making sure view name is unique in output DMS
imported_view_name = i_view_name+b"_"+str2bytes(str(i))
i+=1
View.import_view(i_dms.full_path[:-7], o_dms.full_path[:-7], i_view_name, imported_view_name)
i_view = o_dms[imported_view_name]
# Clone output view from input view
o_view = i_view.clone(output_view_name, comments=i_view.comments) # TODO comments
o_view = i_view.clone(o_view_name)
if o_view is None:
raise Exception("Couldn't create output view")
i_view.close()
if 'taxoURI' in config['obi'] : # TODO default None problem
# Open taxonomy if there is one
if 'taxoURI' in config['obi'] and config['obi']['taxoURI'] is not None:
taxo_uri = open_uri(config['obi']['taxoURI'])
if taxo_uri is None:
raise Exception("Couldn't open taxonomy")
@ -345,16 +360,19 @@ def run(config):
# Save command config in View and DMS comments
command_line = " ".join(sys.argv[1:])
input_dms_name=[input[0].name]
input_view_name=[input[1].name]
input_view_name=[i_view_name]
if 'taxoURI' in config['obi'] and config['obi']['taxoURI'] is not None:
input_dms_name.append(config['obi']['taxoURI'].split("/", 1)[0])
input_view_name.append(config['obi']['taxoURI'].split("/", 1)[1])
input_dms_name.append(config['obi']['taxoURI'].split("/")[-3])
input_view_name.append("taxonomy/"+config['obi']['taxoURI'].split("/")[-1])
o_view.write_config(config, "annotate", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name)
input[0].record_command_line(command_line) # TODO assuming same dms
output[0].record_command_line(command_line)
print("\n")
print(repr(o_view))
input[0].close()
# output[0].close()
# If the input and the output DMS are different, delete the temporary imported view used to create the final view
if i_dms != o_dms:
View.delete_view(o_dms, imported_view_name)
o_dms.close()
i_dms.close()

70
python/obitools3/commands/clean.pyx Normal file → Executable file
View File

@ -8,7 +8,7 @@ from obitools3.dms.capi.obiclean cimport obi_clean
from obitools3.apps.optiongroups import addMinimalInputOption, addMinimalOutputOption
from obitools3.uri.decode import open_uri
from obitools3.apps.config import logger
from obitools3.utils cimport tobytes
from obitools3.utils cimport tobytes, str2bytes
from obitools3.dms.view.view cimport View
from obitools3.dms.view.typed_view.view_NUC_SEQS cimport View_NUC_SEQS
@ -66,38 +66,58 @@ def run(config):
DMS.obi_atexit()
logger("info", "obi clean")
# Open DMS
dms_name = config['obi']['inputURI'].split('/')[0]
dms = open_uri(dms_name)[0]
# Read the name of the input view
uri_i = config['obi']['inputURI'].split('/')
i_view_name = uri_i[1]
# Read the name of the output view
uri_o = config['obi']['outputURI'].split('/')
if len(uri_o)==2:
# Check that input and output DMS are the same (predicate, to discuss)
if dms_name != uri_o[0]:
raise Exception("Input and output DMS must be the same")
o_view_name = uri_o[1]
# Open the input: only the DMS
input = open_uri(config['obi']['inputURI'],
dms_only=True)
if input is None:
raise Exception("Could not read input")
i_dms = input[0]
i_dms_name = input[0].name
i_view_name = input[1]
# Open the output: only the DMS
output = open_uri(config['obi']['outputURI'],
input=False,
dms_only=True)
if output is None:
raise Exception("Could not create output")
o_dms = output[0]
final_o_view_name = output[1]
# If the input and output DMS are not the same, run obiclean creating a temporary view that will be exported to
# the right DMS and deleted in the other afterwards.
if i_dms != o_dms:
temporary_view_name = final_o_view_name
i=0
while temporary_view_name in i_dms: # Making sure view name is unique in input DMS
temporary_view_name = final_o_view_name+b"_"+str2bytes(str(i))
i+=1
o_view_name = temporary_view_name
else:
o_view_name = uri_o[0]
o_view_name = final_o_view_name
# Save command config in View comments
command_line = " ".join(sys.argv[1:])
comments = View.print_config(config, "clean", command_line, input_dms_name=[dms_name], input_view_name=[i_view_name])
comments = View.print_config(config, "clean", command_line, input_dms_name=[i_dms_name], input_view_name=[i_view_name])
if obi_clean(tobytes(dms_name), tobytes(i_view_name), tobytes(config['clean']['sample-tag-name']), tobytes(o_view_name), comments, \
if obi_clean(tobytes(i_dms_name), tobytes(i_view_name), tobytes(config['clean']['sample-tag-name']), tobytes(o_view_name), comments, \
config['clean']['distance'], config['clean']['ratio'], config['clean']['heads-only'], 1) < 0:
raise Exception("Error running obiclean")
# If the input and output DMS are not the same, export result view to output DMS
if i_dms != o_dms:
View.import_view(i_dms.full_path[:-7], o_dms.full_path[:-7], o_view_name, final_o_view_name)
# Save command config in DMS comments
dms.record_command_line(command_line)
o_dms.record_command_line(command_line)
print("\n")
print(repr(dms[o_view_name]))
dms.close()
print(repr(o_dms[final_o_view_name]))
# If the input and the output DMS are different, delete the temporary result view in the input DMS
if i_dms != o_dms:
View.delete_view(i_dms, o_view_name)
o_dms.close()
i_dms.close()

51
python/obitools3/commands/ecopcr.pyx
View File

@ -144,34 +144,37 @@ def run(config):
DMS.obi_atexit()
logger("info", "obi ecopcr")
# TODO Bad URI reading because current one is not adapted
# Get input DMS path
i_dms_name = config['obi']['inputURI'].split('/')[0]
# Read the name of the input view
i_uri = config['obi']['inputURI'].split('/')
i_view_name = i_uri[1]
# Read the name of the output view
o_uri = config['obi']['outputURI'].split('/')
if len(o_uri)==2:
# Get output DMS path
o_dms_name = o_uri[0]
o_view_name = o_uri[1]
else:
o_dms_name = i_dms_name
o_view_name = o_uri[0]
o_dms = open_uri(o_dms_name, input=False)[0]
# Open the input: only the DMS
input = open_uri(config['obi']['inputURI'],
dms_only=True)
if input is None:
raise Exception("Could not read input")
i_dms = input[0]
i_dms_name = input[0].name
i_view_name = input[1]
# Open the output: only the DMS
output = open_uri(config['obi']['outputURI'],
input=False,
dms_only=True)
if output is None:
raise Exception("Could not create output")
o_dms = output[0]
o_dms_name = output[0].name
o_view_name = output[1]
# Read taxonomy name
taxonomy_name = config['obi']['taxoURI'].split('/')[2]
taxonomy_name = config['obi']['taxoURI'].split("/")[-1] # Robust in theory
# Save command config in View comments
command_line = " ".join(sys.argv[1:])
comments = View.print_config(config, "ecopcr", command_line, input_dms_name=[i_dms_name], input_view_name=[i_view_name, config['obi']['taxoURI']])
input_dms_name=[i_dms_name]
input_view_name= [i_view_name]
input_dms_name.append(config['obi']['taxoURI'].split("/")[-3])
input_view_name.append("taxonomy/"+config['obi']['taxoURI'].split("/")[-1])
comments = View.print_config(config, "ecopcr", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name)
# TODO: primers in comments?

58
python/obitools3/commands/grep.pyx
View File

@ -7,7 +7,7 @@ from obitools3.uri.decode import open_uri
from obitools3.apps.optiongroups import addMinimalInputOption, addTaxonomyOption, addMinimalOutputOption
from obitools3.dms.view import RollbackException
from obitools3.apps.config import logger
from obitools3.utils cimport tobytes
from obitools3.utils cimport tobytes, str2bytes
from functools import reduce
import time
@ -267,19 +267,28 @@ def run(config):
input = open_uri(config["obi"]["inputURI"])
if input is None:
raise Exception("Could not read input view")
i_dms = input[0]
i_view = input[1]
# Read the name of the output view
uri = config["obi"]["outputURI"].split("/")
if len(uri)==2:
# Check that input and output DMS are the same (predicate, to discuss)
if config["obi"]["inputURI"].split("/")[0] != uri[0]:
raise Exception("Input and output DMS must be the same")
output_view_name = uri[1]
else:
output_view_name = uri[0]
# Open the output: only the DMS
output = open_uri(config['obi']['outputURI'],
input=False,
dms_only=True)
if output is None:
raise Exception("Could not create output view")
o_dms = output[0]
o_view_name_final = output[1]
o_view_name = o_view_name_final
# If the input and output DMS are not the same, create output view in input DMS first, then export it
# to output DMS, making sure the temporary view name is unique in the input DMS
if i_dms != o_dms:
i=0
while o_view_name in i_dms:
o_view_name = o_view_name_final+b"_"+str2bytes(str(i))
i+=1
if "taxoURI" in config["obi"] : # TODO default None problem
if 'taxoURI' in config['obi'] and config['obi']['taxoURI'] is not None:
taxo_uri = open_uri(config["obi"]["taxoURI"])
if taxo_uri is None:
raise Exception("Couldn't open taxonomy")
@ -307,25 +316,32 @@ def run(config):
# Create output view with the line selection
try:
o_view = selection.materialize(output_view_name)
o_view = selection.materialize(o_view_name)
except Exception, e:
raise RollbackException("obi grep error, rollbacking view: "+str(e), o_view)
# TODO DISCUSS if output URI to different DMS, copy view?
# Save command config in View and DMS comments
command_line = " ".join(sys.argv[1:])
input_dms_name=[input[0].name]
input_view_name=[input[1].name]
if 'taxoURI' in config['obi'] and config['obi']['taxoURI'] is not None:
input_dms_name.append(config['obi']['taxoURI'].split("/", 1)[0])
input_view_name.append(config['obi']['taxoURI'].split("/", 1)[1])
input_dms_name.append(config['obi']['taxoURI'].split("/")[-3])
input_view_name.append("taxonomy/"+config['obi']['taxoURI'].split("/")[-1])
o_view.write_config(config, "grep", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name)
input[0].record_command_line(command_line) # TODO assuming input and output dms are the same
o_dms.record_command_line(command_line)
# If input and output DMS are not the same, export the temporary view to the output DMS
# and delete the temporary view in the input DMS
if i_dms != o_dms:
o_view.close()
View.import_view(i_dms.full_path[:-7], o_dms.full_path[:-7], o_view_name, o_view_name_final)
o_view = o_dms[o_view_name_final]
print("\n")
print(repr(o_view))
input[0].close()
#output[0].close()
# If the input and the output DMS are different, delete the temporary imported view used to create the final view
if i_dms != o_dms:
View.delete_view(i_dms, o_view_name)
o_dms.close()
i_dms.close()

62
python/obitools3/commands/head.pyx
View File

@ -7,6 +7,7 @@ from obitools3.uri.decode import open_uri
from obitools3.apps.optiongroups import addMinimalInputOption, addMinimalOutputOption
from obitools3.dms.view import RollbackException
from obitools3.apps.config import logger
from obitools3.utils cimport str2bytes
import time
import sys
@ -37,50 +38,65 @@ def run(config):
logger("info", "obi head")
# Open the input
input = open_uri(config['obi']['inputURI'])
input = open_uri(config["obi"]["inputURI"])
if input is None:
raise Exception("Could not read input view")
i_dms = input[0]
i_view = input[1]
# Read the name of the output view
uri = config['obi']['outputURI'].split('/')
if len(uri)==2:
# Check that input and output DMS are the same (predicate, to discuss)
if config['obi']['inputURI'].split('/')[0] != uri[0]:
raise Exception("Input and output DMS must be the same")
output_view_name = uri[1]
else:
output_view_name = uri[0]
# Initialize the progress bar
pb = ProgressBar(len(i_view), config, seconde=5)
# Open the output: only the DMS
output = open_uri(config['obi']['outputURI'],
input=False,
dms_only=True)
if output is None:
raise Exception("Could not create output view")
o_dms = output[0]
o_view_name_final = output[1]
o_view_name = o_view_name_final
# If the input and output DMS are not the same, create output view in input DMS first, then export it
# to output DMS, making sure the temporary view name is unique in the input DMS
if i_dms != o_dms:
i=0
while o_view_name in i_dms:
o_view_name = o_view_name_final+b"_"+str2bytes(str(i))
i+=1
n = min(config['head']['count'], len(i_view))
# Initialize the progress bar
pb = ProgressBar(n, config, seconde=5)
selection = Line_selection(i_view)
for i in range(n):
pb(i)
selection.append(i)
# Save command config in View comments
command_line = " ".join(sys.argv[1:])
comments = View.get_config_dict(config, "head", command_line, input_dms_name=[i_dms.name], input_view_name=[i_view.name])
# Create output view with the line selection
try:
o_view = selection.materialize(output_view_name, comments=comments)
o_view = selection.materialize(o_view_name)
except Exception, e:
raise RollbackException("obi head error, rollbacking view: "+str(e), o_view)
# TODO DISCUSS if output URI to different DMS, copy view?
# Save command config in DMS comments
i_dms.record_command_line(command_line)
command_line = " ".join(sys.argv[1:])
o_view.write_config(config, "head", command_line, input_dms_name=[i_dms.name], input_view_name=[i_view.name])
o_dms.record_command_line(command_line)
# If input and output DMS are not the same, export the temporary view to the output DMS
# and delete the temporary view in the input DMS
if i_dms != o_dms:
o_view.close()
View.import_view(i_dms.full_path[:-7], o_dms.full_path[:-7], o_view_name, o_view_name_final)
o_view = o_dms[o_view_name_final]
print("\n")
print(repr(o_view))
input[0].close()
#output[0].close()
# If the input and the output DMS are different, delete the temporary imported view used to create the final view
if i_dms != o_dms:
View.delete_view(i_dms, o_view_name)
o_dms.close()
i_dms.close()

2
python/obitools3/commands/ngsfilter.pyx
View File

@ -511,7 +511,7 @@ def run(config):
o_view.write_config(config, "ngsfilter", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name)
unidentified.write_config(config, "ngsfilter", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name)
# TODO add comment about unidentified seqs
output[0].record_command_line(command_line) # TODO if same dms...
output[0].record_command_line(command_line)
print("\n")
print(repr(o_view))

56
python/obitools3/commands/sort.pyx
View File

@ -7,6 +7,8 @@ from obitools3.uri.decode import open_uri
from obitools3.apps.optiongroups import addMinimalInputOption, addMinimalOutputOption
from obitools3.dms.view import RollbackException
from obitools3.apps.config import logger
from obitools3.utils cimport str2bytes
from obitools3.dms.capi.obitypes cimport OBI_BOOL, \
OBI_CHAR, \
OBI_FLOAT, \
@ -70,20 +72,29 @@ def run(config):
logger("info", "obi sort")
# Open the input
input = open_uri(config['obi']['inputURI'])
input = open_uri(config["obi"]["inputURI"])
if input is None:
raise Exception("Could not read input view")
i_dms = input[0]
i_view = input[1]
# Read the name of the output view
uri = config['obi']['outputURI'].split('/')
if len(uri)==2:
# Check that input and output DMS are the same (predicate, to discuss)
if config['obi']['inputURI'].split('/')[0] != uri[0]:
raise Exception("Input and output DMS must be the same")
output_view_name = uri[1]
else:
output_view_name = uri[0]
# Open the output: only the DMS
output = open_uri(config['obi']['outputURI'],
input=False,
dms_only=True)
if output is None:
raise Exception("Could not create output view")
o_dms = output[0]
o_view_name_final = output[1]
o_view_name = o_view_name_final
# If the input and output DMS are not the same, create output view in input DMS first, then export it
# to output DMS, making sure the temporary view name is unique in the input DMS
if i_dms != o_dms:
i=0
while o_view_name in i_dms:
o_view_name = o_view_name_final+b"_"+str2bytes(str(i))
i+=1
# Initialize the progress bar
pb = ProgressBar(len(i_view), config, seconde=5)
@ -100,20 +111,29 @@ def run(config):
# Create output view with the sorted line selection
try:
o_view = selection.materialize(output_view_name, comments="obi sort: "+str(config['sort']['keys'])+"\n")
o_view = selection.materialize(o_view_name)
except Exception, e:
raise RollbackException("obi sort error, rollbacking view: "+str(e), o_view)
# TODO DISCUSS if output URI to different DMS, copy view?
# Save command config in View and DMS comments
command_line = " ".join(sys.argv[1:])
o_view.write_config(config, "sort", command_line, input_dms_name=[input[0].name], input_view_name=[input[1].name])
input[0].record_command_line(command_line) # TODO assuming same dms...
input_dms_name=[input[0].name]
input_view_name=[input[1].name]
o_view.write_config(config, "sort", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name)
o_dms.record_command_line(command_line)
# If input and output DMS are not the same, export the temporary view to the output DMS
# and delete the temporary view in the input DMS
if i_dms != o_dms:
o_view.close()
View.import_view(i_dms.full_path[:-7], o_dms.full_path[:-7], o_view_name, o_view_name_final)
o_view = o_dms[o_view_name_final]
print("\n")
print(repr(o_view))
input[0].close()
#output[0].close()
# If the input and the output DMS are different, delete the temporary imported view used to create the final view
if i_dms != o_dms:
View.delete_view(i_dms, o_view_name)
o_dms.close()
i_dms.close()

2
python/obitools3/commands/stats.pyx
View File

@ -143,7 +143,7 @@ def run(config):
raise Exception("Could not read input view")
i_view = input[1]
if 'taxoURI' in config['obi'] : # TODO default None problem
if 'taxoURI' in config['obi'] and config['obi']['taxoURI'] is not None:
taxo_uri = open_uri(config['obi']['taxoURI'])
if taxo_uri is None:
raise Exception("Couldn't open taxonomy")

58
python/obitools3/commands/tail.pyx
View File

@ -7,6 +7,7 @@ from obitools3.uri.decode import open_uri
from obitools3.apps.optiongroups import addMinimalInputOption, addMinimalOutputOption
from obitools3.dms.view import RollbackException
from obitools3.apps.config import logger
from obitools3.utils cimport str2bytes
import time
import sys
@ -37,30 +38,39 @@ def run(config):
logger("info", "obi tail")
# Open the input
input = open_uri(config['obi']['inputURI'])
input = open_uri(config["obi"]["inputURI"])
if input is None:
raise Exception("Could not read input view")
i_dms = input[0]
i_view = input[1]
# Read the name of the output view
uri = config['obi']['outputURI'].split('/')
if len(uri)==2:
# Check that input and output DMS are the same (predicate, to discuss)
if config['obi']['inputURI'].split('/')[0] != uri[0]:
raise Exception("Input and output DMS must be the same")
output_view_name = uri[1]
else:
output_view_name = uri[0]
# Initialize the progress bar
pb = ProgressBar(len(i_view), config, seconde=5)
# Open the output: only the DMS
output = open_uri(config['obi']['outputURI'],
input=False,
dms_only=True)
if output is None:
raise Exception("Could not create output view")
o_dms = output[0]
o_view_name_final = output[1]
o_view_name = o_view_name_final
# If the input and output DMS are not the same, create output view in input DMS first, then export it
# to output DMS, making sure the temporary view name is unique in the input DMS
if i_dms != o_dms:
i=0
while o_view_name in i_dms:
o_view_name = o_view_name_final+b"_"+str2bytes(str(i))
i+=1
start = max(len(i_view) - config['tail']['count'], 0)
# Initialize the progress bar
pb = ProgressBar(len(i_view) - start, config, seconde=5)
selection = Line_selection(i_view)
for i in range(start, len(i_view)):
pb(i)
selection.append(i)
# Save command config in View comments
@ -69,18 +79,28 @@ def run(config):
# Create output view with the line selection
try:
o_view = selection.materialize(output_view_name, comments="obi tail: "+str(len(selection))+"\n")
o_view = selection.materialize(o_view_name)
except Exception, e:
raise RollbackException("obi tail error, rollbacking view: "+str(e), o_view)
# TODO DISCUSS if output URI to different DMS, copy view?
# Save command config in DMS comments
i_dms.record_command_line(command_line)
command_line = " ".join(sys.argv[1:])
o_view.write_config(config, "tail", command_line, input_dms_name=[i_dms.name], input_view_name=[i_view.name])
o_dms.record_command_line(command_line)
# If input and output DMS are not the same, export the temporary view to the output DMS
# and delete the temporary view in the input DMS
if i_dms != o_dms:
o_view.close()
View.import_view(i_dms.full_path[:-7], o_dms.full_path[:-7], o_view_name, o_view_name_final)
o_view = o_dms[o_view_name_final]
print("\n")
print(repr(o_view))
input[0].close()
#output[0].close()
# If the input and the output DMS are different, delete the temporary imported view used to create the final view
if i_dms != o_dms:
View.delete_view(i_dms, o_view_name)
o_dms.close()
i_dms.close()

6
python/obitools3/commands/uniq.pyx
View File

@ -508,7 +508,7 @@ def run(config):
entries = input[1]
o_view = output[1]
if 'taxoURI' in config['obi'] : # TODO default None problem
if 'taxoURI' in config['obi'] and config['obi']['taxoURI'] is not None:
taxo_uri = open_uri(config['obi']['taxoURI'])
if taxo_uri is None:
raise RollbackException("Couldn't open taxonomy, rollbacking view", o_view)
@ -529,8 +529,8 @@ def run(config):
input_dms_name=[input[0].name]
input_view_name=[input[1].name]
if 'taxoURI' in config['obi'] and config['obi']['taxoURI'] is not None:
input_dms_name.append(config['obi']['taxoURI'].split("/", 1)[0])
input_view_name.append(config['obi']['taxoURI'].split("/", 1)[1])
input_dms_name.append(config['obi']['taxoURI'].split("/")[-3])
input_view_name.append("taxonomy/"+config['obi']['taxoURI'].split("/")[-1])
o_view.write_config(config, "uniq", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name)
output[0].record_command_line(command_line)