diff --git a/distutils.ext/obidistutils/command/build_ext.py b/distutils.ext/obidistutils/command/build_ext.py index fc31d96..7562083 100644 --- a/distutils.ext/obidistutils/command/build_ext.py +++ b/distutils.ext/obidistutils/command/build_ext.py @@ -7,107 +7,109 @@ Created on 13 fevr. 2014 from distutils import log import os -from Cython.Distutils import build_ext as ori_build_ext # @UnresolvedImport - -from Cython.Compiler import Options as cython_options # @UnresolvedImport from distutils.errors import DistutilsSetupError -class build_ext(ori_build_ext): - +try: + from Cython.Distutils import build_ext as ori_build_ext # @UnresolvedImport + from Cython.Compiler import Options as cython_options # @UnresolvedImport + class build_ext(ori_build_ext): - def modifyDocScripts(self): - build_dir_file=open("doc/sphinx/build_dir.txt","w") - print(self.build_lib,file=build_dir_file) - build_dir_file.close() - def initialize_options(self): - ori_build_ext.initialize_options(self) # @UndefinedVariable - self.littlebigman = None - self.built_files = None - - - def finalize_options(self): - ori_build_ext.finalize_options(self) # @UndefinedVariable - - self.set_undefined_options('littlebigman', - ('littlebigman', 'littlebigman')) - - self.set_undefined_options('build_files', - ('files', 'built_files')) - - self.cython_c_in_temp = 1 - - if self.littlebigman =='-DLITTLE_END': - if self.define is None: - self.define=[('LITTLE_END',None)] - else: - self.define.append('LITTLE_END',None) - - def substitute_sources(self,exe_name,sources): - """ - Substitutes source file name starting by an @ by the actual - name of the built file (see --> build_files) - """ - sources = list(sources) - for i in range(len(sources)): - message = "%s :-> %s" % (exe_name,sources[i]) - if sources[i][0]=='@': - try: - filename = self.built_files[sources[i][1:]] - except KeyError: - tmpfilename = os.path.join(self.build_temp,sources[i][1:]) - if os.path.isfile (tmpfilename): - filename = tmpfilename - else: - raise DistutilsSetupError( - 'The %s filename declared in the source ' - 'files of the program %s have not been ' - 'built by the installation process' % (sources[i], - exe_name)) - sources[i]=filename - log.info("%s changed to %s",message,filename) - else: - log.info("%s ok",message) - - return sources - - def build_extensions(self): - # First, sanity-check the 'extensions' list - - for ext in self.extensions: - ext.sources = self.substitute_sources(ext.name,ext.sources) + def modifyDocScripts(self): + build_dir_file=open("doc/sphinx/build_dir.txt","w") + print(self.build_lib,file=build_dir_file) + build_dir_file.close() - self.check_extensions_list(self.extensions) - - for ext in self.extensions: - log.info("%s :-> %s",ext.name,ext.sources) - ext.sources = self.cython_sources(ext.sources, ext) - self.build_extension(ext) - - - def run(self): - self.modifyDocScripts() - - for cmd_name in self.get_sub_commands(): - self.run_command(cmd_name) - - cython_options.annotate = True - ori_build_ext.run(self) # @UndefinedVariable - - - def has_files(self): - return self.distribution.has_files() - - def has_executables(self): - return self.distribution.has_executables() + def initialize_options(self): + ori_build_ext.initialize_options(self) # @UndefinedVariable + self.littlebigman = None + self.built_files = None - sub_commands = [('build_files',has_files), - ('build_cexe', has_executables) - ] + \ - ori_build_ext.sub_commands - + def finalize_options(self): + ori_build_ext.finalize_options(self) # @UndefinedVariable + + self.set_undefined_options('littlebigman', + ('littlebigman', 'littlebigman')) + + self.set_undefined_options('build_files', + ('files', 'built_files')) + + self.cython_c_in_temp = 1 + + if self.littlebigman =='-DLITTLE_END': + if self.define is None: + self.define=[('LITTLE_END',None)] + else: + self.define.append('LITTLE_END',None) + + def substitute_sources(self,exe_name,sources): + """ + Substitutes source file name starting by an @ by the actual + name of the built file (see --> build_files) + """ + sources = list(sources) + for i in range(len(sources)): + message = "%s :-> %s" % (exe_name,sources[i]) + if sources[i][0]=='@': + try: + filename = self.built_files[sources[i][1:]] + except KeyError: + tmpfilename = os.path.join(self.build_temp,sources[i][1:]) + if os.path.isfile (tmpfilename): + filename = tmpfilename + else: + raise DistutilsSetupError( + 'The %s filename declared in the source ' + 'files of the program %s have not been ' + 'built by the installation process' % (sources[i], + exe_name)) + sources[i]=filename + log.info("%s changed to %s",message,filename) + else: + log.info("%s ok",message) + + return sources + + def build_extensions(self): + # First, sanity-check the 'extensions' list + + for ext in self.extensions: + ext.sources = self.substitute_sources(ext.name,ext.sources) + + self.check_extensions_list(self.extensions) + + for ext in self.extensions: + log.info("%s :-> %s",ext.name,ext.sources) + ext.sources = self.cython_sources(ext.sources, ext) + self.build_extension(ext) + + + def run(self): + self.modifyDocScripts() + + for cmd_name in self.get_sub_commands(): + self.run_command(cmd_name) + + cython_options.annotate = True + ori_build_ext.run(self) # @UndefinedVariable + + + def has_files(self): + return self.distribution.has_files() + + def has_executables(self): + return self.distribution.has_executables() + + sub_commands = [('build_files',has_files), + ('build_cexe', has_executables) + ] + \ + ori_build_ext.sub_commands + +except ImportError: + from distutils.command import build_ext # @UnusedImport + \ No newline at end of file diff --git a/distutils.ext/obidistutils/serenity/bootstrappip.py b/distutils.ext/obidistutils/serenity/bootstrappip.py new file mode 100644 index 0000000..01aebdf --- /dev/null +++ b/distutils.ext/obidistutils/serenity/bootstrappip.py @@ -0,0 +1,36 @@ +''' +Created on 22 janv. 2016 + +@author: coissac +''' + +import sys +from urllib import request +import os.path + +from obidistutils.serenity.util import get_serenity_dir +from obidistutils.serenity.rerun import rerun_with_anothe_python +from obidistutils.serenity.checkpython import is_a_virtualenv_python + +getpipurl="https://bootstrap.pypa.io/get-pip.py" + +def bootstrap(): + + getpipfile=os.path.join(get_serenity_dir(),"get-pip.py") + + with request.urlopen(getpipurl) as getpip: + with open(getpipfile,"wb") as out: + for l in getpip: + out.write(l) + + python = sys.executable + + if is_a_virtualenv_python(): + command= "%s %s" % (python,getpipfile) + else: + command= "%s %s --user" % (python,getpipfile) + + os.system(command) + + rerun_with_anothe_python(python) + \ No newline at end of file diff --git a/distutils.ext/obidistutils/serenity/checkpackage.py b/distutils.ext/obidistutils/serenity/checkpackage.py index 91a2411..7bfdc9c 100644 --- a/distutils.ext/obidistutils/serenity/checkpackage.py +++ b/distutils.ext/obidistutils/serenity/checkpackage.py @@ -7,8 +7,13 @@ Created on 2 oct. 2014 import re import os -import pip # @UnresolvedImport -from pip.utils import get_installed_distributions # @UnresolvedImport +try: + import pip # @UnresolvedImport + from pip.utils import get_installed_distributions # @UnresolvedImport +except ImportError: + from .bootstrappip import bootstrap + bootstrap() + from distutils.version import StrictVersion # @UnusedImport from distutils.errors import DistutilsError from distutils import log diff --git a/distutils.ext/obidistutils/serenity/virtual.py b/distutils.ext/obidistutils/serenity/virtual.py index 6e8e2c9..27ee67c 100644 --- a/distutils.ext/obidistutils/serenity/virtual.py +++ b/distutils.ext/obidistutils/serenity/virtual.py @@ -59,7 +59,7 @@ def serenity_virtualenv(envname,package,version,minversion='3.4',maxversion=None clear=True, symlinks=False, with_pip=True) - + # check the newly created virtualenv return serenity_virtualenv(envname,package,version) diff --git a/python/obi.py b/python/obi.py index 355527a..9c05a1c 100644 --- a/python/obi.py +++ b/python/obi.py @@ -16,30 +16,21 @@ It defines classes_and_methods @deffield updated: Updated ''' -import sys -import pkgutil -import argparse -import logging -import json -default_config = { +default_config = { 'software' : "The OBITools", + 'log' : False, + 'loglevel' : 'INFO', + 'progress' : True + } - 'obi' : { 'log' : True, - 'loglevel' : 'INFO', - 'version' : False, - 'progress' : True - } - -} +root_config_name='obi' - - -from obitools3 import command +from obitools3.apps.config import getConfiguration # @UnresolvedImport from obitools3.version import version -__all__ = [] +__all__ = [] __version__ = version -__date__ = '2014-09-28' +__date__ = '2014-09-28' __updated__ = '2014-09-28' DEBUG = 1 @@ -47,182 +38,12 @@ TESTRUN = 0 PROFILE = 0 - -def loadCommand(name,loader): - ''' - Load a command module from its name and an ImpLoader - - This function is for internal use - - @param name: name of the module - @type name: str - @param loader: the module loader - @type loader: ImpLoader - - @return the loaded module - @rtype: module - ''' - - module = loader.find_module(name).load_module(name) - return module - -def getCommandsList(): - ''' - Returns the list of sub-commands available to the main `obi` command - - @return: a dict instance with key corresponding to each command and - value corresponding to the module - - @rtype: dict - ''' - cmds = dict((x[1],loadCommand(x[1],x[0])) - for x in pkgutil.iter_modules(command.__path__) - if not x[2]) - return cmds - -def getLogger(config): - ''' - Returns the logger as defined by the command line option - or by the config file - :param config: - ''' - - output = config['obi']['outputfilename'] - level = config['obi']['loglevel'] - logfile= config['obi']['log'] - - rootlogger = logging.getLogger() - logFormatter = logging.Formatter("%(asctime)s [%(levelname)-5.5s] %(message)s") - - stderrHandler = logging.StreamHandler(sys.stderr) - stderrHandler.setFormatter(logFormatter) - - rootlogger.addHandler(stderrHandler) - - if logfile: - fileHandler = logging.FileHandler("%s.log" % output) - fileHandler.setFormatter(logFormatter) - rootlogger.addHandler(fileHandler) - - try: - loglevel = getattr(logging, level) - except: - loglevel = logging.INFO - - rootlogger.setLevel(loglevel) - - config['obi']['logger']=rootlogger - - return rootlogger - - -class ObiParser(argparse.ArgumentParser): - def error(self, message): - sys.stderr.write('error: %s\n' % message) - self.print_help() - sys.exit(2) - -def buildArgumentParser(): - parser = ObiParser() - - parser.add_argument('--version', dest='obi:version', - action='store_true', - default=False, - help='Print the version of the OBITools') - - parser.add_argument('--no-log', dest='obi:log', - action='store_false', - default=None, - help='Do not create a logfile for the data analyze') - - parser.add_argument('--no-progress', dest='obi:progress', - action='store_false', - default=None, - help='Do not print the progress bar during analyzes') - - subparsers = parser.add_subparsers(title='subcommands', - description='valid subcommands', - help='additional help') - - commands = getCommandsList() - - for c in commands: - module = commands[c] - - if hasattr(module, "run"): - if hasattr(module, "__title__"): - sub = subparsers.add_parser(c,help=module.__title__) - else: - sub = subparsers.add_parser(c) - - if hasattr(module, "addOptions"): - module.addOptions(sub) - - sub.set_defaults(**{'obi:module' : module}) - - return parser - -def buildDefaultConfiguration(): - global default_config - - commands = getCommandsList() - - for c in commands: - module = commands[c] - - assert hasattr(module, "run") - - if hasattr(module, 'default_config'): - default_config[c]=module.default_config - else: - default_config[c]={} - - return default_config - - -def getConfiguration(): - global default_config - - if '__done__' in default_config: - return default_config - - parser = buildArgumentParser() - options = vars(parser.parse_args()) - - config = buildDefaultConfiguration() - - - for k in options: - section,key = k.split(':') - s = config[section] - if options[k] is not None: - s[key]=options[k] - - if config['obi']['version']: - print("The OBITools - Version %s" % __version__) - sys.exit(0) - - if not 'module' in config['obi']: - print('\nError: No obi command specified',file=sys.stderr) - parser.print_help() - sys.exit(2) - - - if config['obi']['outputfilename'] is None: - config['obi']['outputfilename']=config['obi']['indexfilename'] - - getLogger(config) - - config['__done__']=True - - return config - - if __name__ =="__main__": - config = getConfiguration() + config = getConfiguration(root_config_name, + default_config) - config['obi']['module'].run(config) + config[root_config_name]['module'].run(config) \ No newline at end of file diff --git a/python/obitools3/apps/__init__.py b/python/obitools3/apps/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python/obitools3/apps/arguments.pxd b/python/obitools3/apps/arguments.pxd new file mode 100644 index 0000000..b4713b7 --- /dev/null +++ b/python/obitools3/apps/arguments.pxd @@ -0,0 +1,3 @@ +#cython: language_level=3 + +cpdef buildArgumentParser(str configname, str softname) \ No newline at end of file diff --git a/python/obitools3/apps/arguments.pyx b/python/obitools3/apps/arguments.pyx new file mode 100644 index 0000000..c521fac --- /dev/null +++ b/python/obitools3/apps/arguments.pyx @@ -0,0 +1,61 @@ +#cython: language_level=3 + +''' +Created on 27 mars 2016 + +@author: coissac +''' + +import argparse +import sys + + +from .command import getCommandsList + +class ObiParser(argparse.ArgumentParser): + def error(self, message): + sys.stderr.write('error: %s\n' % message) + self.print_help() + sys.exit(2) + +cpdef buildArgumentParser(str configname, + str softname): + parser = ObiParser() + + parser.add_argument('--version', dest='%s:version' % configname, + action='store_true', + default=False, + help='Print the version of %s' % softname) + + parser.add_argument('--log', dest='%s:log' % configname, + action='store', + type=str, + default=None, + help='Create a logfile') + + parser.add_argument('--no-progress', dest='%s:progress' % configname, + action='store_false', + default=None, + help='Do not print the progress bar during analyzes') + + subparsers = parser.add_subparsers(title='subcommands', + description='valid subcommands', + help='additional help') + + commands = getCommandsList() + + for c in commands: + module = commands[c] + + if hasattr(module, "run"): + if hasattr(module, "__title__"): + sub = subparsers.add_parser(c,help=module.__title__) + else: + sub = subparsers.add_parser(c) + + if hasattr(module, "addOptions"): + module.addOptions(sub) + + sub.set_defaults(**{'%s:module' % configname : module}) + + return parser diff --git a/python/obitools3/apps/command.pxd b/python/obitools3/apps/command.pxd new file mode 100644 index 0000000..8c034a4 --- /dev/null +++ b/python/obitools3/apps/command.pxd @@ -0,0 +1,3 @@ +#cython: language_level=3 + +cdef object loadCommand(str name,loader) diff --git a/python/obitools3/apps/command.pyx b/python/obitools3/apps/command.pyx new file mode 100644 index 0000000..1234188 --- /dev/null +++ b/python/obitools3/apps/command.pyx @@ -0,0 +1,44 @@ +#cython: language_level=3 + +''' +Created on 27 mars 2016 + +@author: coissac +''' + +import pkgutil + +from obitools3 import commands + +cdef object loadCommand(str name,loader): + ''' + Load a command module from its name and an ImpLoader + + This function is for internal use + + @param name: name of the module + @type name: str + @param loader: the module loader + @type loader: ImpLoader + + @return the loaded module + @rtype: module + ''' + + module = loader.find_module(name).load_module(name) + return module + +def getCommandsList(): + ''' + Returns the list of sub-commands available to the main `obi` command + + @return: a dict instance with key corresponding to each command and + value corresponding to the module + + @rtype: dict + ''' + + cdef dict cmds = dict((x[1],loadCommand(x[1],x[0])) + for x in pkgutil.iter_modules(commands.__path__) + if not x[2]) + return cmds diff --git a/python/obitools3/apps/config.pxd b/python/obitools3/apps/config.pxd new file mode 100644 index 0000000..74c595f --- /dev/null +++ b/python/obitools3/apps/config.pxd @@ -0,0 +1,10 @@ +#cython: language_level=3 + +cpdef str setRootConfigName(str rootname) +cpdef str getRootConfigName() + +cdef dict buildDefaultConfiguration(str root_config_name, + dict config) + +cpdef dict getConfiguration(str root_config_name=?, + dict config=?) \ No newline at end of file diff --git a/python/obitools3/apps/config.pyx b/python/obitools3/apps/config.pyx new file mode 100644 index 0000000..c8e3baa --- /dev/null +++ b/python/obitools3/apps/config.pyx @@ -0,0 +1,103 @@ +#cython: language_level=3 + +''' +Created on 27 mars 2016 + +@author: coissac +''' + +import sys + +from .command import getCommandsList +from .logging cimport getLogger +from .arguments cimport buildArgumentParser + +from ..version import version +from _curses import version + +cdef dict __default_config__ = {} + + +cpdef str setRootConfigName(str rootname): + global __default_config__ + if '__root_config__' in __default_config__: + if __default_config__["__root_config__"] in __default_config__: + __default_config__[rootname]=__default_config__[__default_config__["__root_config__"]] + del __default_config__[__default_config__["__root_config__"]] + __default_config__['__root_config__']=rootname + return rootname + +cpdef str getRootConfigName(): + global __default_config__ + return __default_config__.get('__root_config__',None) + +cdef dict buildDefaultConfiguration(str root_config_name, + dict config): + global __default_config__ + + __default_config__.clear() + setRootConfigName(root_config_name) + + __default_config__[root_config_name]=config + + config['version']=version + + commands = getCommandsList() + + for c in commands: + module = commands[c] + + assert hasattr(module, "run") + + if hasattr(module, 'default_config'): + __default_config__[c]=module.default_config + else: + __default_config__[c]={} + + return __default_config__ + + +cpdef dict getConfiguration(str root_config_name="__default__", + dict config={}): + global __default_config__ + + if '__done__' in __default_config__: + return __default_config__ + + if root_config_name=="__default__": + raise RuntimeError("No root_config_name specified") + + if not config: + raise RuntimeError("Base configuration is empty") + + + + config = buildDefaultConfiguration(root_config_name, + config) + + parser = buildArgumentParser(root_config_name, + config[root_config_name]['software']) + + options = vars(parser.parse_args()) + + if options['%s:version' % root_config_name]: + print("%s - Version %s" % (config[root_config_name]['software'], + config[root_config_name]['version'])) + sys.exit(0) + + for k in options: + section,key = k.split(':') + s = config[section] + if options[k] is not None: + s[key]=options[k] + + if not 'module' in config[root_config_name]: + print('\nError: No command specified',file=sys.stderr) + parser.print_help() + sys.exit(2) + + getLogger(config) + + config['__done__']=True + + return config diff --git a/python/obitools3/apps/logging.pxd b/python/obitools3/apps/logging.pxd new file mode 100644 index 0000000..5ead8bb --- /dev/null +++ b/python/obitools3/apps/logging.pxd @@ -0,0 +1,3 @@ +#cython: language_level=3 + +cpdef getLogger(dict config) diff --git a/python/obitools3/apps/logging.pyx b/python/obitools3/apps/logging.pyx new file mode 100644 index 0000000..559977d --- /dev/null +++ b/python/obitools3/apps/logging.pyx @@ -0,0 +1,46 @@ +#cython: language_level=3 + +''' +Created on 27 mars 2016 + +@author: coissac +''' + +import logging +import sys + +cpdef getLogger(dict config): + ''' + Returns the logger as defined by the command line option + or by the config file + :param config: + ''' + + root = config["__root_config__"] + + level = config[root]['loglevel'] + logfile= config[root]['log'] + + rootlogger = logging.getLogger() + logFormatter = logging.Formatter("%(asctime)s [%(levelname)-5.5s] %(message)s") + + stderrHandler = logging.StreamHandler(sys.stderr) + stderrHandler.setFormatter(logFormatter) + + rootlogger.addHandler(stderrHandler) + + if logfile: + fileHandler = logging.FileHandler(logfile) + fileHandler.setFormatter(logFormatter) + rootlogger.addHandler(fileHandler) + + try: + loglevel = getattr(logging, level) + except: + loglevel = logging.INFO + + rootlogger.setLevel(loglevel) + + config[root]['logger']=rootlogger + + return rootlogger diff --git a/python/obitools3/apps/progress.pxd b/python/obitools3/apps/progress.pxd new file mode 100644 index 0000000..979ab80 --- /dev/null +++ b/python/obitools3/apps/progress.pxd @@ -0,0 +1,65 @@ +#cython: language_level=3 + +from ..utils cimport str2bytes + +cdef extern from "stdio.h": + struct FILE + int fprintf(FILE *stream, char *format, ...) + FILE* stderr + ctypedef unsigned int off_t "unsigned long long" + +cdef extern from "unistd.h": + int fsync(int fd); + +cdef extern from "time.h": + struct tm : + int tm_yday + int tm_hour + int tm_min + int tm_sec + + enum: CLOCKS_PER_SEC + + ctypedef int time_t + ctypedef int clock_t + ctypedef int suseconds_t + + struct timeval: + time_t tv_sec # seconds */ + suseconds_t tv_usec # microseconds */ + + + struct timezone : + int tz_minuteswest; # minutes west of Greenwich + int tz_dsttime; # type of DST correction + + + int gettimeofday(timeval *tv, timezone *tz) + + + tm *gmtime_r(time_t *clock, tm *result) + time_t time(time_t *tloc) + clock_t clock() + +cdef class ProgressBar: + cdef off_t maxi + cdef clock_t starttime + cdef clock_t lasttime + cdef clock_t tickcount + cdef int freq + cdef int cycle + cdef int arrow + cdef int lastlog + cdef bint ontty + cdef int fd + + cdef bytes head + cdef char *chead + + cdef object logger + + cdef char *wheel + cdef char *spaces + cdef char* diese + + cdef clock_t clock(self) diff --git a/python/obitools3/apps/progress.pyx b/python/obitools3/apps/progress.pyx new file mode 100644 index 0000000..48bffe4 --- /dev/null +++ b/python/obitools3/apps/progress.pyx @@ -0,0 +1,138 @@ +#cython: language_level=3 + +''' +Created on 27 mars 2016 + +@author: coissac +''' + +import sys +from ..utils cimport bytes2str + +cdef class ProgressBar: + cdef clock_t clock(self): + cdef clock_t t + cdef timeval tp + cdef clock_t s + + gettimeofday(&tp,NULL) + s = ( tp.tv_usec * 1.e-6 * CLOCKS_PER_SEC) + t = tp.tv_sec * CLOCKS_PER_SEC + s + + return t + + def __init__(self, + off_t maxi, + dict config, + str head="", + double seconde=0.1): + self.starttime = self.clock() + self.lasttime = self.starttime + self.tickcount = (seconde * CLOCKS_PER_SEC) + self.freq = 1 + self.cycle = 0 + self.arrow = 0 + self.lastlog = 0 + + self.ontty = sys.stderr.isatty() + + if (maxi<=0): + maxi=1 + + self.maxi = maxi + self.head = str2bytes(head) + self.chead= self.head + + + self.logger=config[config["__root_config__"]]["logger"] + self.wheel = '|/-\\' + self.spaces=' ' \ + ' ' \ + ' ' \ + ' ' \ + ' ' + self.diese ='##########' \ + '##########' \ + '##########' \ + '##########' \ + '##########' + + def __call__(self,object pos): + cdef off_t ipos + cdef clock_t elapsed + cdef clock_t newtime + cdef clock_t delta + cdef clock_t more + cdef double percent + cdef tm remain + cdef int days,hour,minu,sec + cdef off_t fraction + cdef int twentyth + + self.cycle+=1 + + if self.cycle % self.freq == 0: + self.cycle=1 + newtime = self.clock() + delta = newtime - self.lasttime + self.lasttime = newtime + elapsed = newtime - self.starttime +# print(" ",delta,elapsed,elapsed/CLOCKS_PER_SEC,self.tickcount) + + if delta < self.tickcount / 5 : + self.freq*=2 + elif delta > self.tickcount * 5 and self.freq>1: + self.freq/=2 + + + if callable(pos): + ipos=pos() + else: + ipos=pos + + if ipos==0: + ipos=1 + + percent = ipos/self.maxi + more = ((elapsed / percent * (1. - percent))/CLOCKS_PER_SEC) + gmtime_r(&more, &remain) + days = remain.tm_yday + hour = remain.tm_hour + minu = remain.tm_min + sec = remain.tm_sec + + if self.ontty: + fraction=(percent * 50.) + self.arrow=(self.arrow+1) % 4 + self.diese[fraction]=0 + self.spaces[50 - fraction]=0 + + if days: + fprintf(stderr,b'\r%s %5.1f %% |%s%c%s] remain : %d days %02d:%02d:%02d', + self.chead, + percent*100, + self.diese,self.wheel[self.arrow],self.spaces, + days,hour,minu,sec) + else: + fprintf(stderr,b'\r%s %5.1f %% |%s%c%s] remain : %02d:%02d:%02d', + self.chead, + percent*100., + self.diese,self.wheel[self.arrow],self.spaces, + hour,minu,sec) + self.diese[fraction]=b'#' + self.spaces[50 - fraction]=b' ' + + twentyth = int(percent * 20) + if twentyth != self.lastlog: + + if self.ontty: + fprintf(stderr,b'\n') + + self.logger.info('%s %5.1f %% remain : %02d:%02d:%02d' % ( + bytes2str(self.head), + percent*100., + hour,minu,sec)) + self.lastlog=twentyth + else: + self.cycle+=1 + diff --git a/python/obitools3/commands/__init__.py b/python/obitools3/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python/obitools3/commands/count.py b/python/obitools3/commands/count.py new file mode 100644 index 0000000..28bd996 --- /dev/null +++ b/python/obitools3/commands/count.py @@ -0,0 +1,44 @@ +''' +Created on 8 mars 2016 + +@author: coissac +''' + +from obitools3.apps.progress import ProgressBar # @UnresolvedImport +import time + +__title__="Counts sequences in a sequence set" + + +default_config = { 'countmode' : None + } + +def addOptions(parser): + parser.add_argument(dest='obi:input', metavar='obi:input', + nargs='?', + default=None, + help='input data set' ) + + group=parser.add_argument_group('Obicount specific options') + group.add_argument('-s','--sequence', + action="store_true", dest="count:sequence", + default=False, + help="Prints only the number of sequence records." + ) + + group.add_argument('-a','--all', + action="store_true", dest="count:all", + default=False, + help="Prints only the total count of sequence records (if a sequence has no `count` attribute, its default count is 1) (default: False)." + ) + + + +def run(config): + # The code of my command + pb = ProgressBar(1000,config,seconde=1) + + for i in range(1,1001): + pb(i) + time.sleep(0.01) + diff --git a/python/obitools3/files/__init__.py b/python/obitools3/files/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python/obitools3/files/linebuffer.pxd b/python/obitools3/files/linebuffer.pxd new file mode 100644 index 0000000..a867d3e --- /dev/null +++ b/python/obitools3/files/linebuffer.pxd @@ -0,0 +1,6 @@ +#cython: language_level=3 + + +cdef class LineBuffer: + cdef object fileobj + cdef int size diff --git a/python/obitools3/files/linebuffer.pyx b/python/obitools3/files/linebuffer.pyx new file mode 100644 index 0000000..b0dbaf1 --- /dev/null +++ b/python/obitools3/files/linebuffer.pyx @@ -0,0 +1,23 @@ +#cython: language_level=3 + +''' +Created on 30 mars 2016 + +@author: coissac +''' + +cdef class LineBuffer: + + def __init__(self,object fileobj,int size=100000000): + self.fileobj=fileobj + self.size=size + + def __iter__(self): + cdef list buff = self.fileobj.readlines(self.size) + cdef str l + + while buff: + for l in buff: + yield l + buff = self.fileobj.readlines(self.size) + diff --git a/python/obitools3/files/uncompress.pxd b/python/obitools3/files/uncompress.pxd new file mode 100644 index 0000000..80c5b5d --- /dev/null +++ b/python/obitools3/files/uncompress.pxd @@ -0,0 +1,17 @@ +#cython: language_level=3 + +cdef class MagicKeyFile: + cdef object stream + cdef str stream_mode + cdef object binary + cdef bytes key + cdef int keylength + cdef int pos + + cpdef bytes read(self,int size=?) + cpdef int tell(self) + + +cdef class CompressedFile: + cdef object accessor + \ No newline at end of file diff --git a/python/obitools3/files/uncompress.pyx b/python/obitools3/files/uncompress.pyx new file mode 100644 index 0000000..ecd5d77 --- /dev/null +++ b/python/obitools3/files/uncompress.pyx @@ -0,0 +1,114 @@ +#cython: language_level=3 + +''' +Created on 28 mars 2016 + +@author: coissac +''' + +import zipfile +import bz2 +import gzip + +import io + +cdef class MagicKeyFile: + def __init__(self,stream,length=2): + + binary=stream + self.stream = stream + self.stream_mode = None + if hasattr(stream, "mode"): + self.stream_mode = stream.mode + if (not 'b' in stream.mode and + hasattr(stream, "buffer") and + 'b' in stream.buffer.mode): + binary=stream.buffer + + if (self.stream_mode is None and + not (hasattr(stream, 'headers') and + hasattr(stream.headers, "keys") and + 'Content-type' in stream.headers)): + raise TypeError("stream does not present the good interface") + + self.binary=binary + self.key=binary.read(length) + self.keylength=length + self.pos=0 + + cpdef bytes read(self,int size=-1): + cdef bytes r + + if self.pos < self.keylength: + if size > (self.keylength - self.pos): + size = size - self.keylength + self.pos + r = self.key[self.pos:] + self.binary.read(size) + self.pos=self.keylength + 1 + elif size >=0 : + r = self.key[self.pos:(self.pos+size)] + self.pos+=size + else: + r = self.key[self.pos:] + self.binary.read(size) + self.pos=self.keylength + 1 + else: + r = self.binary.read(size) + + return r + + cpdef int tell(self): + cdef int p + + if self.pos < self.keylength: + p = self.pos + else: + p = self.tell() + + return p + + def __getattr__(self,name): + return getattr(self.binary, name) + + + +cdef class CompressedFile: + + + def __init__(self,stream): + cdef int keylength + cdef MagicKeyFile magic + cdef str compressor + cdef bytes k + cdef object c + + cdef dict compress = { 'zip' : (b'\x50\x4b\x03\x04',zipfile.ZipFile), + 'bz2' : (b'\x42\x5a\x68',bz2.BZ2File), + 'gz' : (b'\x1f\x8b\x08',gzip.open) + } + + keylength = max([len(x[0]) for x in compress.values()]) + magic=MagicKeyFile(stream,keylength) + + self.accessor = None + + for compressor in compress: + k,c = compress[compressor] + if magic.key.startswith(k): + self.accessor = c(magic) + + if self.accessor is None: + self.accessor = magic + + if ((hasattr(stream, 'headers') and + hasattr(stream.headers, "keys") and + 'Content-type' in stream.headers and + stream.headers['Content-type'].startswith('text/')) or + 'b' not in magic.stream_mode): + self.accessor = io.TextIOWrapper(self.accessor) + + + def __getattr__(self,name): + return getattr(self.accessor, name) + + def __iter__(self): + for x in self.accessor: + yield x \ No newline at end of file diff --git a/python/obitools3/files/universalopener.pxd b/python/obitools3/files/universalopener.pxd new file mode 100644 index 0000000..7743db6 --- /dev/null +++ b/python/obitools3/files/universalopener.pxd @@ -0,0 +1,5 @@ +#cython: language_level=3 + +from .uncompress cimport CompressedFile + +cpdef CompressedFile uopen(str name, mode=?) \ No newline at end of file diff --git a/python/obitools3/files/universalopener.pyx b/python/obitools3/files/universalopener.pyx new file mode 100644 index 0000000..3aefb92 --- /dev/null +++ b/python/obitools3/files/universalopener.pyx @@ -0,0 +1,23 @@ +#cython: language_level=3 + +''' +Created on 25 mars 2016 + +@author: coissac +''' + +from urllib.request import urlopen + + +cpdef CompressedFile uopen(str name, mode='r'): + cdef CompressedFile c + + try: + f = urlopen(name) + except ValueError: + f = open(name,mode) + + c = CompressedFile(f) + + return c + \ No newline at end of file diff --git a/python/obitools3/parsers/__init__.py b/python/obitools3/parsers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python/obitools3/parsers/fasta.pxd b/python/obitools3/parsers/fasta.pxd new file mode 100644 index 0000000..d1546d1 --- /dev/null +++ b/python/obitools3/parsers/fasta.pxd @@ -0,0 +1,8 @@ +#cython: language_level=3 + +from .header cimport parseHeader +from ..files.universalopener cimport uopen +from ..files.linebuffer cimport LineBuffer + + + \ No newline at end of file diff --git a/python/obitools3/parsers/fasta.pyx b/python/obitools3/parsers/fasta.pyx new file mode 100644 index 0000000..2a262bc --- /dev/null +++ b/python/obitools3/parsers/fasta.pyx @@ -0,0 +1,46 @@ +#cython: language_level=3 + +''' +Created on 30 mars 2016 + +@author: coissac +''' + +def fastaIterator(lineiterator, int buffersize=100000000): + cdef LineBuffer lb + cdef str ident + cdef str definition + cdef dict tags + cdef list s + + if isinstance(lineiterator,(str,bytes)): + lineiterator=uopen(lineiterator) + + if isinstance(lineiterator, LineBuffer): + lb=lineiterator + else: + lb=LineBuffer(lineiterator,buffersize) + + i = iter(lb) + line = next(i) + + while True: + ident,tags,definition = parseHeader(line) + s = [] + line = next(i) + while line[0]!='>': + s.append(line[0:-1]) + line = next(i) + sequence = "".join(s) + quality = None + + yield { "id" : ident, + "definition" : definition, + "sequence" : sequence, + "quality" : quality, + "tags" : tags, + "annotation" : {} + } + + + \ No newline at end of file diff --git a/python/obitools3/parsers/fastq.pxd b/python/obitools3/parsers/fastq.pxd new file mode 100644 index 0000000..d1546d1 --- /dev/null +++ b/python/obitools3/parsers/fastq.pxd @@ -0,0 +1,8 @@ +#cython: language_level=3 + +from .header cimport parseHeader +from ..files.universalopener cimport uopen +from ..files.linebuffer cimport LineBuffer + + + \ No newline at end of file diff --git a/python/obitools3/parsers/fastq.pyx b/python/obitools3/parsers/fastq.pyx new file mode 100644 index 0000000..0e44cf8 --- /dev/null +++ b/python/obitools3/parsers/fastq.pyx @@ -0,0 +1,41 @@ +#cython: language_level=3 + +''' +Created on 30 mars 2016 + +@author: coissac +''' + + + +def fastqIterator(lineiterator, int buffersize=100000000): + cdef LineBuffer lb + cdef str ident + cdef str definition + cdef dict tags + + if isinstance(lineiterator,(str,bytes)): + lineiterator=uopen(lineiterator) + + if isinstance(lineiterator, LineBuffer): + lb=lineiterator + else: + lb=LineBuffer(lineiterator,buffersize) + + i = iter(lb) + for line in i: + ident,tags,definition = parseHeader(line) + sequence = next(i)[0:-1] + next(i) + quality = next(i)[0:-1] + + yield { "id" : ident, + "definition" : definition, + "sequence" : sequence, + "quality" : quality, + "tags" : tags, + "annotation" : {} + } + + + \ No newline at end of file diff --git a/python/obitools3/parsers/header.pxd b/python/obitools3/parsers/header.pxd new file mode 100644 index 0000000..b09a418 --- /dev/null +++ b/python/obitools3/parsers/header.pxd @@ -0,0 +1,5 @@ +#cython: language_level=3 + +cdef object __etag__(str x) + +cpdef tuple parseHeader(str header) diff --git a/python/obitools3/parsers/header.pyx b/python/obitools3/parsers/header.pyx new file mode 100644 index 0000000..5cd7e82 --- /dev/null +++ b/python/obitools3/parsers/header.pyx @@ -0,0 +1,78 @@ +#cython: language_level=3 + +''' +Created on 25 mars 2016 + +@author: coissac +''' + +import re + +__ret__ = re.compile('''(([^ ]+)=('[^']*'|"[^"]*"|[^;]+); *)+?''') +__re_int__ = re.compile("^[+-]?[0-9]+$") +__re_float__ = re.compile("^[+-]?[0-9]+(\.[0-9]*)?([eE][+-]?[0-9]+)?$") +__re_str__ = re.compile("""^"[^"]*"|'[^']*'$""") +__re_dict__ = re.compile("""^\{\ * + ( + ("[^"]*"|'[^']*') + \ *:\ * + ([^,}]+| + "[^"]*"| + '[^']*' + ) + )? + (\ *,\ * + ("[^"]*"|'[^']*') + \ *:\ * + ([^,}]+| + "[^"]*"| + '[^']*' + ) + )*\ *\}$""", re.VERBOSE) + + +cdef object __etag__(str x): + if __re_int__.match(x): + v=int(x) + elif __re_float__.match(x): + v=float(x) + elif __re_str__.match(x): + v=x[1:-1] + elif x=='False': + v=False + elif x=='True': + v=True + elif __re_dict__.match(x): + v=eval(x) + else: + v=x + return v + +cpdef tuple parseHeader(str header): + cdef list m + cdef dict tags + cdef str definition + cdef str ident + cdef str second + + m=header[1:-1].split(maxsplit=1) + + ident=m[0] + + if len(m)==1: + tags={} + definition='' + else: + second=m[1] + m = __ret__.findall(second) + + if m: + tags = dict([(a[1],__etag__(a[2])) for a in m]) + definition = second.split(m[-1][0],1)[1].strip() + else: + tags = {} + definition = second.strip() + + return ident,tags,definition + + diff --git a/python/obitools3/utils.pxd b/python/obitools3/utils.pxd index 0eafa4d..2a1b144 100644 --- a/python/obitools3/utils.pxd +++ b/python/obitools3/utils.pxd @@ -1,5 +1,6 @@ #cython: language_level=3 + cdef bytes str2bytes(str string) cdef str bytes2str(bytes string) \ No newline at end of file diff --git a/python/obitools3/utils.pyx b/python/obitools3/utils.pyx index 183b373..793e7f1 100644 --- a/python/obitools3/utils.pyx +++ b/python/obitools3/utils.pyx @@ -1,7 +1,28 @@ #cython: language_level=3 +import sys +import io + cdef bytes str2bytes(str string): + """ + Short cut to convert ascii encoded python string (str) to bytes + which can be easily converted to C-strings. + + @param string: the python string to be converted. + @type string: str + @return a transcoded string + @rtype: bytes + """ return string.encode('ascii') cdef str bytes2str(bytes string): - return string.decode('ascii') \ No newline at end of file + """ + Short cut to convert bytes (C-strings) to ascii encoded python string (str). + + @param string: the binary (C-string) string to be converted. + @type string: bytes + @return an ascii transcoded string + @rtype: str + """ + return string.decode('ascii') +