diff --git a/python/obi.py b/python/obi.py index 8c7cd75..de84b7d 100644 --- a/python/obi.py +++ b/python/obi.py @@ -41,7 +41,8 @@ default_config = { 'software' : "The OBITools", "nastring" : b"NA", "stripwhite" : True, "blanklineskip" : True, - "commentchar" : b"#" + "commentchar" : b"#", + "nocreatedms" : False } root_config_name='obi' diff --git a/python/obitools3/apps/optiongroups/__init__.py b/python/obitools3/apps/optiongroups/__init__.py index 4428c1b..486e1f6 100644 --- a/python/obitools3/apps/optiongroups/__init__.py +++ b/python/obitools3/apps/optiongroups/__init__.py @@ -168,6 +168,14 @@ def __addOutputOption(optionManager): dest='obi:outputURI', metavar='OUTPUT', help='Data destination URI') + + group = optionManager.add_argument_group("Management of DMS for output") + + group.add_argument('--no-create-dms', + action="store_true", dest="obi:nocreatedms", + default=False, + help="Don't create an output DMS is it is not existing") + def addMinimalOutputOption(optionManager): __addOutputOption(optionManager) diff --git a/python/obitools3/apps/temp.pxd b/python/obitools3/apps/temp.pxd new file mode 100644 index 0000000..252e41d --- /dev/null +++ b/python/obitools3/apps/temp.pxd @@ -0,0 +1,10 @@ +#cython: language_level=3 + +''' +Created on 28 juillet 2017 + +@author: coissac +''' + +from obitools3.dms.dms cimport DMS +from obitools3.utils cimport tobytes,tostr \ No newline at end of file diff --git a/python/obitools3/apps/temp.pyx b/python/obitools3/apps/temp.pyx new file mode 100644 index 0000000..eb01da8 --- /dev/null +++ b/python/obitools3/apps/temp.pyx @@ -0,0 +1,96 @@ +#cython: language_level=3 + +''' +Created on 28 juillet 2017 + +@author: coissac +''' + +from os import environb,getpid +from os.path import join, isdir +from tempfile import TemporaryDirectory, _get_candidate_names +from shutil import rmtree +from atexit import register + +from obitools3.dms.dms import DMS + +from obitools3.apps.config import getConfiguration +from obitools3.apps.config import logger + +cpdef get_temp_dir(): + """ + Returns a temporary directory object specific of this instance of obitools. + + This is an application function. It cannot be called out of an obi command. + It requires a valid configuration. + + If the function is called several time from the same obi session, the same + directory is returned. + + If the OBITMP environment variable exist, the temporary directory is created + inside this directory. + + The directory is automatically destroyed at the end of the end of the process. + + @return: a temporary python directory object. + """ + cdef bytes tmpdirname + cdef dict config = getConfiguration() + + root = config["__root_config__"] + + try: + return config[root]["tempdir"].name + except KeyError: + pass + + try: + basedir=environb[b'OBITMP'] + except KeyError: + basedir=None + + tmp = TemporaryDirectory(dir=basedir) + + config[root]["tempdir"]=tmp + + return tmp.name + +cpdef get_temp_dir_name(): + """ + Returns the name of the temporary directory object + specific of this instance of obitools. + + @return: the name of the temporary directory. + + @see get_temp_dir + """ + return get_temp_dir_name().name + + +cpdef get_temp_dms(): + + cdef bytes tmpdirname # @DuplicatedSignature + cdef dict config = getConfiguration() # @DuplicatedSignature + cdef DMS tmpdms + + root = config["__root_config__"] + + try: + return config[root]["tempdms"] + except KeyError: + pass + + tmpdirname=get_temp_dir() + + tempname = join(tmpdirname, + b"obi.%d.%s" % (getpid(), + tobytes(next(_get_candidate_names()))) + ) + + tmpdms = DMS.new(tempname) + + config[root]["tempdms"]=tmpdms + + return tmpdms + + diff --git a/python/obitools3/commands/import.pyx b/python/obitools3/commands/import.pyx index bc7c790..bd8da66 100644 --- a/python/obitools3/commands/import.pyx +++ b/python/obitools3/commands/import.pyx @@ -13,6 +13,8 @@ from obitools3.dms.view.view cimport View from obitools3.dms.view.typed_view.view_NUC_SEQS import View_NUC_SEQS # TODO cimport doesn't work from obitools3.dms.column.column cimport Column +from obitools3.dms.obiseq import Nuc_Seq + from obitools3.utils cimport tobytes, \ get_obitype, \ update_obitype @@ -25,6 +27,8 @@ from obitools3.dms.capi.obierrno cimport obi_errno from obitools3.apps.optiongroups import addSequenceInputOption, addMinimalOutputOption from obitools3.uri.decode import open_uri +from obitools3.apps.config import logger + __title__="Imports sequences from different formats into a DMS" @@ -76,17 +80,26 @@ def run(config): cdef list new_elements_names cdef ProgressBar pb global obi_errno + - - logger=config['obi']['logger'] - - - logger.info("obi import : imports file into an DMS") + logger("info","obi import : imports file into an DMS") inputs = open_uri(config['obi']['inputURI']) - print(inputs) + if inputs[2]==Nuc_Seq: + v = View_NUC_SEQS + else: + v= View + + print(v) + output = open_uri(config['obi']['outputURI'], + input=False, + newviewtype=v) + + print(input) + print(output) + sys.exit() # pb = ProgressBar(1000000, config, seconde=5) # TODO should be number of records in file diff --git a/python/obitools3/uri/decode.pxd b/python/obitools3/uri/decode.pxd index 573415c..573e0d8 100644 --- a/python/obitools3/uri/decode.pxd +++ b/python/obitools3/uri/decode.pxd @@ -8,3 +8,4 @@ from obitools3.dms.taxo.taxo cimport Taxonomy from obitools3.utils cimport tobytes, tostr from obitools3.files.universalopener cimport uopen +cdef open_dms(bytes path, bint create=?) diff --git a/python/obitools3/uri/decode.pyx b/python/obitools3/uri/decode.pyx index 093bfd7..b3b254a 100644 --- a/python/obitools3/uri/decode.pyx +++ b/python/obitools3/uri/decode.pyx @@ -11,11 +11,12 @@ from obitools3.parsers.universal import entryIteratorFactory from obitools3.dms.obiseq import Nuc_Seq from obitools3.apps.config import getConfiguration,logger +from obitools3.apps.temp import get_temp_dms class MalformedURIException(RuntimeError): pass -cdef open_dms(bytes path,create=False): +cdef open_dms(bytes path, bint create=False): """ Opens a DMS from the path part of an URI """ @@ -50,13 +51,15 @@ cdef open_dms(bytes path,create=False): pos=pos+1 return None -def open_dms_element(DMS dms, bytes path): +def open_dms_element(DMS dms, bytes path, + bint create=False, + type newviewtype=View): """ """ cdef list path_parts = path.split(b'/') - + # The URI is only composed of a DMS - if not path_parts: + if not path: return (dms,dms) # The URI is target a taxonomy @@ -74,7 +77,10 @@ def open_dms_element(DMS dms, bytes path): # The URI is target a view # dms:dmspath/viewname[/columnname|#line|*[/#line|columnname|*[/subcolumn]]] - view = View.open(dms,path_parts[0]) + if create: + view = newviewtype.new(dms,path_parts[0]) + else: + view = newviewtype.open(dms,path_parts[0]) if len(path_parts) > 1: if path_parts[1]==b'*': @@ -128,7 +134,9 @@ def open_dms_element(DMS dms, bytes path): return (dms,subsubpart) -def open_uri(uri,bint input=True): +def open_uri(uri, + bint input=True, + type newviewtype=View): cdef bytes urib = tobytes(uri) cdef bytes scheme cdef tuple dms @@ -146,19 +154,27 @@ def open_uri(uri,bint input=True): except KeyError: default_dms=None + try: + create=(not input) and (not config["obi"]["nocreatedms"]) + except KeyError: + create=not input + scheme = urip.scheme error = None - if scheme==b"" : - dms = open_dms(urip.path) + if scheme==b"" or scheme==b"dms" : + dms = open_dms(urip.path,create) if dms is None and default_dms is not None: dms=(default_dms,urip.path) if dms is not None: try: - resource=open_dms_element(*dms) + resource=open_dms_element(dms[0],dms[1], + create, + newviewtype + ) scheme=b"dms" urip = ParseResultBytes(scheme=b"dms", netloc=urip.netloc, @@ -169,17 +185,24 @@ def open_uri(uri,bint input=True): if default_dms is None: config["obi"]["defaultdms"]=resource[0] - - return (resource[0],resource[1],urlunparse(urip)) + + return (resource[0], + resource[1], + type(resource[1]), + urlunparse(urip)) except Exception as e: error=e + + if scheme==b"dms" : + logger('error','cannot open DMS: %s', uri) + raise FileNotFoundError('uri') if not urip.scheme: urib=b"file:"+urib try: - logger('info','Trying to open file : %s', tostr(urib)) file = uopen(tostr(urib)) + logger('info','Opened file : %s', tostr(urib)) except Exception as e: file = None error=e @@ -366,7 +389,10 @@ def open_uri(uri,bint input=True): blanklineskip, commentchar) - return (file,iseq,objclass) + + tmpdms = get_temp_dms() + + return (file,iseq,objclass,urib)