Patch the way of installing the obi main command

previous commit

.gitignore

@@ -0,0 +1,13 @@
+.DS_Store
+/obitools.test/
+/OBITools3- 1.01.16/
+/OBITools3-0.00.0/
+/build/
+/testpip/
+/toto.obidms/
+/unittests/
+/setup.py.old
+/3_Hornung_testseq.fastq
+/sample/
+/save.temp/
+/obitmp/

.project

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>obitools3</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.python.pydev.PyDevBuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.python.pydev.pythonNature</nature>
+	</natures>
+</projectDescription>

.pydevproject

@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?eclipse-pydev version="1.0"?><pydev_project>
+        
+    
+    <pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">python3.6-obitool3</pydev_property>
+        
+    
+    <pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python interpreter</pydev_property>
+        
+    
+    <pydev_pathproperty name="org.python.pydev.PROJECT_SOURCE_PATH">
+                
+        
+        <path>/${PROJECT_DIR_NAME}/python</path>
+                
+    
+    </pydev_pathproperty>
+        
+
+</pydev_project>

distutils.ext/obidistutils/command/build.py

@@ -1,38 +0,0 @@
-'''
-Created on 20 oct. 2012
-
-@author: coissac
-'''
-
-from distutils.command.build import build as ori_build
-from obidistutils.serenity.checksystem import is_mac_system
-
-
-class build(ori_build):
-    
-    def has_ext_modules(self):
-        return self.distribution.has_ext_modules()
-    
-    def has_pidname(self):
-        return is_mac_system()
-
-    def has_doc(self):
-        return True
-    
-    def has_littlebigman(self):
-        return True
-    
-    try:
-        from obidistutils.command.build_sphinx import build_sphinx  # @UnusedImport
-
-        sub_commands = [("littlebigman",has_littlebigman),
-                         ('pidname',has_pidname)
-                       ] \
-                       + ori_build.sub_commands + \
-                       [('build_sphinx',has_doc)]
-    except ImportError:
-        sub_commands = [("littlebigman",has_littlebigman),
-                         ('pidname',has_pidname)
-                       ] \
-                       + ori_build.sub_commands
-        

distutils.ext/obidistutils/command/build_cexe.py

@@ -1,85 +0,0 @@
-'''
-Created on 20 oct. 2012
-
-@author: coissac
-'''
-
-from .build_ctools import build_ctools
-from .build_exe import build_exe
-from distutils.errors import DistutilsSetupError
-from distutils import log
-import os
-
-class build_cexe(build_ctools):
-
-    description = "build C/C++ executable distributed with Python extensions"
-
-
-    def initialize_options(self):
-        build_ctools.initialize_options(self)
-        self.built_files = None
-
-
-    def finalize_options(self):
-        # This might be confusing: both build-cexe and build-temp default
-        # to build-temp as defined by the "build" command.  This is because
-        # I think that C libraries are really just temporary build
-        # by-products, at least from the point of view of building Python
-        # extensions -- but I want to keep my options open.
-
-        build_cexe_dir = self.build_cexe
-        build_ctools.finalize_options(self)
-
-        if build_cexe_dir is None:
-            self.build_cexe=None
-            
-        self.set_undefined_options('build',
-                                   ('build_scripts',  'build_cexe'))
-
-        self.set_undefined_options('build_files',
-                                   ('files',  'built_files'))
-                   
-        self.executables   = self.distribution.executables
-#         self.build_cexe = os.path.join(os.path.dirname(self.build_cexe),'cbinaries') 
-#         self.mkpath(self.build_cexe)
-
-        if self.executables:
-            self.check_executable_list(self.executables)
-                            
-
-        # XXX same as for build_ext -- what about 'self.define' and
-        # 'self.undef' ?
-
-    def substitute_sources(self,exe_name,sources):
-        """
-        Substitutes source file name starting by an @ by the actual
-        name of the built file (see --> build_files)
-        """
-        sources = list(sources)
-        for i in range(len(sources)):
-            message = "%s :-> %s" % (exe_name,sources[i])
-            if sources[i][0]=='@':
-                try:
-                    filename = self.built_files[sources[i][1:]]
-                except KeyError:
-                    raise DistutilsSetupError(
-                         'The %s filename declared in the source '
-                         'files of the program %s have not been '
-                         'built by the installation process' % (sources[i],
-                                                                exe_name))
-                sources[i]=filename
-                log.info("%s changed to %s",message,filename)
-            else:
-                log.info("%s ok",message)
-
-        return sources
- 
-    def run(self):
-
-        for cmd_name in self.get_sub_commands():
-            self.run_command(cmd_name)
-
-        build_exe.run(self)
-        
-
-   

distutils.ext/obidistutils/command/build_ctools.py

@@ -1,63 +0,0 @@
-'''
-Created on 20 oct. 2012
-
-@author: coissac
-'''
-
-
-from .build_exe import build_exe
-from distutils import log
-
-class build_ctools(build_exe):
-    description = "build C/C++ executable not distributed with Python extensions"
-
-    def initialize_options(self):
-        build_exe.initialize_options(self)
-        
-        # List of built tools
-        self.ctools = None
-        self.littlebigman = None
-
-
-    def finalize_options(self):
-        # This might be confusing: both build-cexe and build-temp default
-        # to build-temp as defined by the "build" command.  This is because
-        # I think that C libraries are really just temporary build
-        # by-products, at least from the point of view of building Python
-        # extensions -- but I want to keep my options open.
-            
-        build_exe.finalize_options(self)
-        
-        
-        self.set_undefined_options('build',
-                                    ('build_temp', 'build_cexe'))
-
-        self.set_undefined_options('littlebigman',
-                                   ('littlebigman',  'littlebigman'))
-                   
-        self.executables = self.distribution.ctools
-        self.check_executable_list(self.executables)
-        
-
-        if self.littlebigman =='-DLITTLE_END':
-            if self.define is None:
-                self.define=[('LITTLE_END',None)]
-            else:
-                self.define.append('LITTLE_END',None)
-                
-        log.info('Look for CPU architecture... %s',self.define)
-
-        self.ctools = set()
-
-    def run(self):
-
-        for cmd_name in self.get_sub_commands():
-            self.run_command(cmd_name)
-
-        
-        build_exe.run(self)
-        
-        for e,p in self.executables:                                  # @UnusedVariable
-            self.ctools.add(e)
-            
-

distutils.ext/obidistutils/command/build_exe.py

@@ -1,211 +0,0 @@
-'''
-Created on 20 oct. 2012
-
-@author: coissac
-'''
-
-import os
-
-from distutils.core import Command
-from distutils.sysconfig import customize_compiler
-from distutils.errors import DistutilsSetupError
-from distutils import log
-from distutils.ccompiler import show_compilers
-
-class build_exe(Command):
-    
-    description = "build an executable -- Abstract command "
-
-    user_options = [
-        ('build-cexe', 'x',
-         "directory to build C/C++ libraries to"),
-        ('build-temp', 't',
-         "directory to put temporary build by-products"),
-        ('debug', 'g',
-         "compile with debugging information"),
-        ('force', 'f',
-         "forcibly build everything (ignore file timestamps)"),
-        ('compiler=', 'c',
-         "specify the compiler type"),
-        ]
-
-    boolean_options = ['debug', 'force']
-
-    help_options = [
-        ('help-compiler', None,
-         "list available compilers", show_compilers),
-        ]
-
-    def initialize_options(self):
-        self.build_cexe = None
-        self.build_temp = None
-
-        # List of executables to build
-        self.executables = None
-
-        # Compilation options for all libraries
-        self.include_dirs = None
-        self.define = None
-        self.undef = None
-        self.extra_compile_args = None
-        self.debug = None
-        self.force = 0
-        self.compiler = None
-        self.sse = None
-        self.built_files=None
-
-    def finalize_options(self):
-        # This might be confusing: both build-cexe and build-temp default
-        # to build-temp as defined by the "build" command.  This is because
-        # I think that C libraries are really just temporary build
-        # by-products, at least from the point of view of building Python
-        # extensions -- but I want to keep my options open.
-        self.set_undefined_options('build',
-                                   ('build_temp', 'build_temp'),
-                                   ('compiler', 'compiler'),
-                                   ('debug', 'debug'),
-                                   ('force', 'force'))
-
-        if self.include_dirs is None:
-            self.include_dirs = self.distribution.include_dirs or []
-            
-        if isinstance(self.include_dirs, str):
-            self.include_dirs = self.include_dirs.split(os.pathsep)
-
-        self.sse = self.distribution.sse
-        
-        if self.sse is not None:
-            if self.extra_compile_args is None:
-                self.extra_compile_args=['-m%s' % self.sse]
-            else:
-                self.extra_compile_args.append('-m%s' % self.sse)
-
-        # XXX same as for build_ext -- what about 'self.define' and
-        # 'self.undef' ?
-
-    def run(self):
-
-        if not self.executables:
-            return
-
-        self.mkpath(self.build_cexe)
-
-        # Yech -- this is cut 'n pasted from build_ext.py!
-        from distutils.ccompiler import new_compiler
-        self.compiler = new_compiler(compiler=self.compiler,
-                                     dry_run=self.dry_run,
-                                     force=self.force)
-        customize_compiler(self.compiler)
-
-        if self.include_dirs is not None:
-            self.compiler.set_include_dirs(self.include_dirs)
-        if self.define is not None:
-            # 'define' option is a list of (name,value) tuples
-            for (name,value) in self.define:
-                self.compiler.define_macro(name, value)
-
-        if self.undef is not None:
-            for macro in self.undef:
-                self.compiler.undefine_macro(macro)
-                
-        self.build_executables(self.executables)
-
-
-    def check_executable_list(self, executables):
-        """Ensure that the list of executables is valid.
-
-        `executable` is presumably provided as a command option 'executables'.
-        This method checks that it is a list of 2-tuples, where the tuples
-        are (executable_name, build_info_dict).
-
-        Raise DistutilsSetupError if the structure is invalid anywhere;
-        just returns otherwise.
-        """
-        if not isinstance(executables, list):
-            raise DistutilsSetupError("'executables' option must be a list of tuples")
-
-        for exe in executables:
-            if not isinstance(exe, tuple) and len(exe) != 2:
-                raise DistutilsSetupError("each element of 'executables' must a 2-tuple")
-
-            name, build_info = exe
-
-            if not isinstance(name, str):
-                raise DistutilsSetupError(
-                      "first element of each tuple in 'executables' " 
-                      "must be a string (the executables name)")
-                
-            if '/' in name or (os.sep != '/' and os.sep in name):
-                raise DistutilsSetupError(
-                       "bad executable name '%s': " 
-                       "may not contain directory separators" % exe[0])
-
-            if not isinstance(build_info, dict):
-                raise DistutilsSetupError(
-                      "second element of each tuple in 'executables' " 
-                      "must be a dictionary (build info)")
-
-    def get_executable_names(self):
-        # Assume the executables list is valid -- 'check_executable_list()' is
-        # called from 'finalize_options()', so it should be!
-        if not self.executables:
-            return None
-
-        exe_names = []
-        for (exe_name, build_info) in self.executables:                     # @UnusedVariable
-            exe_names.append(exe_name)
-        return exe_names
-
-
-    def get_source_files(self):
-        self.check_executable_list(self.executables)
-        filenames = []
-        for (exe_name, build_info) in self.executables:                     # @UnusedVariable
-            sources = build_info.get('sources')
-            if sources is None or not isinstance(sources, (list, tuple)):
-                raise DistutilsSetupError(
-                       "in 'executables' option (library '%s'), "
-                       "'sources' must be present and must be "
-                       "a list of source filenames" % exe_name)
-
-            filenames.extend(sources)
-        return filenames
-    
-    def substitute_sources(self,exe_name,sources):
-        return list(sources)
-
-    def build_executables(self, executables):
-        for (exe_name, build_info) in executables:
-            sources = build_info.get('sources')
-            if sources is None or not isinstance(sources, (list, tuple)):
-                raise DistutilsSetupError(
-                       "in 'executables' option (library '%s'), " 
-                       "'sources' must be present and must be " 
-                       "a list of source filenames" % exe_name)
-                
-            sources = self.substitute_sources(exe_name,sources)
-
-            log.info("building '%s' program", exe_name)
-
-            # First, compile the source code to object files in the library
-            # directory.  (This should probably change to putting object
-            # files in a temporary build directory.)
-            macros = build_info.get('macros')
-            include_dirs = build_info.get('include_dirs')
-            extra_args = self.extra_compile_args or []
-            
-            objects = self.compiler.compile(sources,
-                                            output_dir=self.build_temp,
-                                            macros=macros,
-                                            include_dirs=include_dirs,
-                                            extra_postargs=extra_args,
-                                            debug=self.debug)
-
-            # Now "link" the object files together into a static library.
-            # (On Unix at least, this isn't really linking -- it just
-            # builds an archive.  Whatever.)
-            self.compiler.link_executable(objects, exe_name,
-                                            output_dir=self.build_cexe,
-                                            debug=self.debug)
-
-        

distutils.ext/obidistutils/command/build_ext.py

@@ -1,115 +0,0 @@
-'''
-Created on 13 fevr. 2014
-
-@author: coissac
-'''
-
-from distutils import log
-import os
-
-
-from distutils.errors import DistutilsSetupError
-
-try:
-    from Cython.Distutils import build_ext  as ori_build_ext  # @UnresolvedImport
-    from Cython.Compiler import Options as cython_options  # @UnresolvedImport
-    class build_ext(ori_build_ext):
-    
-        
-        def modifyDocScripts(self):
-            build_dir_file=open("doc/sphinx/build_dir.txt","w")
-            print(self.build_lib,file=build_dir_file)
-            build_dir_file.close()
-            
-        def initialize_options(self):
-            ori_build_ext.initialize_options(self)  # @UndefinedVariable
-            self.littlebigman = None
-            self.built_files = None
-    
-        
-        def finalize_options(self):
-            ori_build_ext.finalize_options(self)  # @UndefinedVariable
-    
-            self.set_undefined_options('littlebigman',
-                                       ('littlebigman',  'littlebigman'))
-            
-            self.set_undefined_options('build_files',
-                                       ('files',  'built_files'))
-    
-            self.cython_c_in_temp = 1
-                       
-            if self.littlebigman =='-DLITTLE_END':
-                if self.define is None:
-                    self.define=[('LITTLE_END',None)]
-                else:
-                    self.define.append('LITTLE_END',None)
-            
-        def substitute_sources(self,exe_name,sources):
-            """
-            Substitutes source file name starting by an @ by the actual
-            name of the built file (see --> build_files)
-            """
-            sources = list(sources)
-            for i in range(len(sources)):
-                message = "%s :-> %s" % (exe_name,sources[i])
-                if sources[i][0]=='@':
-                    try:
-                        filename = self.built_files[sources[i][1:]]
-                    except KeyError:
-                        tmpfilename = os.path.join(self.build_temp,sources[i][1:])
-                        if os.path.isfile       (tmpfilename):
-                            filename = tmpfilename
-                        else:
-                            raise DistutilsSetupError(
-                                 'The %s filename declared in the source '
-                                 'files of the program %s have not been '
-                                 'built by the installation process' % (sources[i],
-                                                                         exe_name))
-                    sources[i]=filename
-                    log.info("%s changed to %s",message,filename)
-                else:
-                    log.info("%s ok",message)
-    
-            return sources
-    
-        def build_extensions(self):
-            # First, sanity-check the 'extensions' list
-            
-            for ext in self.extensions:
-                ext.sources = self.substitute_sources(ext.name,ext.sources)
-                
-            self.check_extensions_list(self.extensions)
-    
-            for ext in self.extensions:
-                log.info("%s :-> %s",ext.name,ext.sources)
-                ext.sources = self.cython_sources(ext.sources, ext)
-                self.build_extension(ext)
-    
-            
-        def run(self):
-            self.modifyDocScripts()
-    
-            for cmd_name in self.get_sub_commands():
-                self.run_command(cmd_name)
-    
-            cython_options.annotate = True
-            ori_build_ext.run(self)  # @UndefinedVariable
-            
-    
-        def has_files(self):
-            return self.distribution.has_files()
-    
-        def has_executables(self):
-            return self.distribution.has_executables()
-        
-        sub_commands = [('build_files',has_files),
-                        ('build_cexe', has_executables)
-                        ] + \
-                       ori_build_ext.sub_commands
-
-except ImportError:
-    from distutils.command import build_ext  # @UnusedImport
-           
-
-
-        

distutils.ext/obidistutils/command/build_files.py

@@ -1,63 +0,0 @@
-'''
-Created on 20 oct. 2012
-
-@author: coissac
-'''
-
-import os.path
-
-from distutils.core import Command
-from distutils import log
-
-class build_files(Command):
-            
-    def initialize_options(self):
-        self.files=None
-        self.ctools=None
-        self.build_temp=None
-        self.build_cexe=None
-
-    def finalize_options(self):
-        
-        self.set_undefined_options('build_ctools',
-                                   ('ctools',  'ctools'),
-                                   ('build_temp','build_temp'),
-                                   ('build_cexe','build_cexe'),
-                                   )
-        
-        self.files = {}
-        
-    def run(self):
-
-        for cmd_name in self.get_sub_commands():
-            self.run_command(cmd_name)
-
-        
-        for dest,prog,command in self.distribution.files:
-            destfile = os.path.join(self.build_temp,dest)
-            if prog in self.ctools:
-                progfile = os.path.join(self.build_cexe,prog)
-            else:
-                progfile = prog 
-                
-            log.info("Building file : %s" % dest)
-            
-            commandline = command % {'prog' : progfile,
-                                     'dest' : destfile}
-
-            log.info("    --> %s" % commandline)
-            
-            os.system(commandline)
-            self.files[dest]=destfile
-            
-            log.info("Done.\n")
-            
-    def has_ctools(self):
-        return self.distribution.has_ctools()
-
-
-    sub_commands = [('build_ctools', has_ctools)] + \
-                   Command.sub_commands
-            
-            
-        

distutils.ext/obidistutils/command/build_scripts.py

@@ -1,104 +0,0 @@
-'''
-Created on 20 oct. 2012
-
-@author: coissac
-'''
-
-import os.path
-
-from distutils.command.build_scripts import build_scripts as ori_build_scripts
-from distutils.util import convert_path
-from distutils import log, sysconfig
-from distutils.dep_util import newer
-from stat import ST_MODE
-import re
-
-first_line_re = re.compile('^#!.*python[0-9.]*([ \t].*)?$')
-
-class build_scripts(ori_build_scripts):
-            
-    def copy_scripts (self):
-        """Copy each script listed in 'self.scripts'; if it's marked as a
-        Python script in the Unix way (first line matches 'first_line_re',
-        ie. starts with "\#!" and contains "python"), then adjust the first
-        line to refer to the current Python interpreter as we copy.
-        """        
-        self.mkpath(self.build_dir)
-        rawbuild_dir = os.path.join(os.path.dirname(self.build_dir),'raw_scripts')
-        self.mkpath(rawbuild_dir)
-        
-        outfiles = []
-        for script in self.scripts:
-            adjust = 0
-            script = convert_path(script)
-            outfile = os.path.join(self.build_dir, os.path.splitext(os.path.basename(script))[0])
-            rawoutfile = os.path.join(rawbuild_dir, os.path.basename(script))
-            outfiles.append(outfile)
-
-            if not self.force and not newer(script, outfile):
-                log.debug("not copying %s (up-to-date)", script)
-                continue
-
-            # Always open the file but ignore failures in dry-run mode --
-            # that way, we'll get accurate feedback if we can read the
-            # script.
-            try:
-                f = open(script, "r")
-            except IOError:
-                if not self.dry_run:
-                    raise
-                f = None
-            else:
-                first_line = f.readline()
-                if not first_line:
-                    self.warn("%s is an empty file (skipping)" % script)
-                    continue
-
-                match = first_line_re.match(first_line)
-                if match:
-                    adjust = 1
-                    post_interp = match.group(1) or ''
-            
-            log.info("Store the raw script %s -> %s", script,rawoutfile)        
-            self.copy_file(script, rawoutfile)
-
-
-            if adjust:
-                log.info("copying and adjusting %s -> %s", script,
-                         self.build_dir)
-                if not self.dry_run:
-                    outf = open(outfile, "w")
-                    if not sysconfig.python_build:
-                        outf.write("#!%s%s\n" %
-                                   (self.executable,
-                                    post_interp))
-                    else:
-                        outf.write("#!%s%s\n" %
-                                   (os.path.join(
-                            sysconfig.get_config_var("BINDIR"),
-                           "python%s%s" % (sysconfig.get_config_var("VERSION"),
-                                           sysconfig.get_config_var("EXE"))),
-                                    post_interp))
-                    outf.writelines(f.readlines())
-                    outf.close()
-                if f:
-                    f.close()
-            else:
-                if f:
-                    f.close()
-                self.copy_file(script, outfile)
-
-        if os.name == 'posix':
-            for F in outfiles:
-                if self.dry_run:
-                    log.info("changing mode of %s", F)
-                else:
-                    oldmode = os.stat(F)[ST_MODE]
-                    oldmode = oldmode & 0o7777
-                    
-                    newmode = (oldmode | 0o555) & 0o7777
-                    if newmode != oldmode:
-                        log.info("changing mode of %s from %o to %o",
-                                 F, oldmode, newmode)
-                        os.chmod(F, newmode)
-

distutils.ext/obidistutils/command/build_sphinx.py

@@ -1,27 +0,0 @@
-'''
-Created on 10 mars 2015
-
-@author: coissac
-'''
-
-try:
-    from sphinx.setup_command import BuildDoc as ori_build_sphinx  # @UnresolvedImport
-        
-    class build_sphinx(ori_build_sphinx):
-        '''Build Sphinx documentation in html, epub and man formats 
-        '''
-    
-        description = __doc__
-    
-        def run(self):
-            self.builder='html'
-            self.finalize_options()
-            ori_build_sphinx.run(self)
-            self.builder='epub'
-            self.finalize_options()
-            ori_build_sphinx.run(self)
-            self.builder='man'
-            self.finalize_options()
-            ori_build_sphinx.run(self)
-except ImportError:
-    pass  

distutils.ext/obidistutils/command/install.py

@@ -1,19 +0,0 @@
-'''
-Created on 6 oct. 2014
-
-@author: coissac
-'''
-
-# try:
-#     from setuptools.command.install import install as install_ori
-# except ImportError:
-#     from distutils.command.install import install as install_ori
-
-from distutils.command.install import install as install_ori
-
-class install(install_ori):
-    
-    def __init__(self,dist):
-        install_ori.__init__(self, dist)
-#        self.sub_commands.insert(0, ('build',lambda self: True))
-        self.sub_commands.append(('install_sphinx',lambda self: self.distribution.serenity))

distutils.ext/obidistutils/command/install_scripts.py

@@ -1,47 +0,0 @@
-'''
-Created on 20 oct. 2012
-
-@author: coissac
-'''
-
-# try:
-#     from setuptools.command.install_scripts import install_scripts as ori_install_scripts
-# except ImportError:
-#     from distutils.command.install_scripts import install_scripts as ori_install_scripts
-
-from distutils.command.install_scripts import install_scripts as ori_install_scripts
-
-import os.path
-from distutils import log
-
-class install_scripts(ori_install_scripts):
-
-    def initialize_options(self):
-        ori_install_scripts.initialize_options(self)
-        self.public_dir = None
-                        
-                    
-    def install_public_link(self):
-        self.mkpath(self.public_dir)
-        for file in self.get_outputs():
-            log.info("exporting file %s -> %s", file,os.path.join(self.public_dir,
-                                    os.path.split(file)[1]
-                                    ))
-            if not self.dry_run:
-                dest = os.path.join(self.public_dir,
-                                    os.path.split(file)[1]
-                                   )
-                if os.path.exists(dest):
-                    os.unlink(dest)
-                os.symlink(file,dest)
-                    
-
-
-    def run(self):
-        ori_install_scripts.run(self)
-        if self.distribution.serenity:
-            self.public_dir=os.path.join(self.install_dir,"../export/bin")
-            self.public_dir=os.path.abspath(self.public_dir)
-            self.install_public_link()
-
-

distutils.ext/obidistutils/command/install_sphinx.py

@@ -1,61 +0,0 @@
-'''
-Created on 10 mars 2015
-
-@author: coissac
-'''
-from distutils.core import Command
-import os.path
-import glob
-
-class install_sphinx(Command):
-    '''
-    Install the sphinx documentation
-    '''
-    
-    description = "Install the sphinx documentation in serenity mode"
-
-    boolean_options = ['force', 'skip-build']
-
-
-    def initialize_options (self):
-        self.install_doc = None
-        self.build_dir = None
-
-    def finalize_options (self):
-        self.set_undefined_options('build_sphinx', ('build_dir', 'build_dir'))
-        self.set_undefined_options('install',
-                                   ('install_scripts', 'install_doc'))
-
-    def run (self):
-        if self.distribution.serenity:
-            self.install_doc = os.path.join(self.install_doc,"../export/share")
-            self.install_doc=os.path.abspath(self.install_doc)
-            self.mkpath(self.install_doc)
-            self.mkpath(os.path.join(self.install_doc,'html'))
-            outfiles = self.copy_tree(os.path.join(self.build_dir,'html'),  # @UnusedVariable
-                                      os.path.join(self.install_doc,'html'))
-                
-            self.mkpath(os.path.join(self.install_doc,'man','man1'))
-            outfiles = self.copy_tree(os.path.join(self.build_dir,'man'),  # @UnusedVariable
-                                      os.path.join(self.install_doc,'man','man1'))
-
-            for epub in glob.glob(os.path.join(self.build_dir,'epub/*.epub')):
-                self.copy_file(os.path.join(epub), 
-                               os.path.join(self.install_doc,os.path.split(epub)[1]))
-            
-    def get_outputs(self):
-        directory=os.path.join(self.install_doc,'html')
-        files = [os.path.join(self.install_doc,'html', f) 
-                 for dp, dn, filenames in os.walk(directory) for f in filenames]  # @UnusedVariable
-        
-        directory=os.path.join(self.build_dir,'man')
-        files.append(os.path.join(self.install_doc,'man','man1', f) 
-                 for dp, dn, filenames in os.walk(directory) for f in filenames)  # @UnusedVariable
-
-        directory=os.path.join(self.build_dir,'epub')
-        files.append(os.path.join(self.install_doc, f) 
-                 for dp, dn, filenames in os.walk(directory)  # @UnusedVariable
-                 for f in glob.glob(os.path.join(dp, '*.epub')) )
-        
-        return files
-        

distutils.ext/obidistutils/command/littlebigman.py

@@ -1,59 +0,0 @@
-'''
-Created on 20 oct. 2012
-
-@author: coissac
-'''
-
-import os
-
-from obidistutils.command.build_exe import build_exe
-from distutils import log
-
-import subprocess
-
-class littlebigman(build_exe):
-    
-    description = "build the littlebigman executable testing endianness of the CPU"
-
-
-    def initialize_options(self):
-        build_exe.initialize_options(self)
-
-        self.littlebigman = None
-
-
-    def finalize_options(self):
-        # This might be confusing: both build-cexe and build-temp default
-        # to build-temp as defined by the "build" command.  This is because
-        # I think that C libraries are really just temporary build
-        # by-products, at least from the point of view of building Python
-        # extensions -- but I want to keep my options open.
-        
-        build_exe.finalize_options(self)
-        
-        self.set_undefined_options('build',
-                                   ('build_temp', 'build_cexe'))
-
-        # self.ctools = self.distribution.ctools
-        
-        if os.path.isfile("distutils.ext/src/littlebigman.c"):
-            self.executables = [('littlebigman',{"sources":["distutils.ext/src/littlebigman.c"]})]
-            self.check_executable_list(self.executables)
-        else:
-            self.executables = []
-
-
-    def run_littlebigman(self):
-        p = subprocess.Popen("'%s'" % os.path.join(self.build_temp,
-                                                   'littlebigman'), 
-                             shell=True, 
-                             stdout=subprocess.PIPE)
-        little = p.communicate()[0]
-        return little.decode('latin1')
-
-    def run(self):
-        build_exe.run(self)
-        self.littlebigman=self.run_littlebigman()
-        log.info("Your CPU is in mode : %s" % self.littlebigman)
-        
-        

distutils.ext/obidistutils/command/pidname.py

@@ -1,55 +0,0 @@
-'''
-Created on 20 oct. 2012
-
-@author: coissac
-'''
-
-import os
-
-from obidistutils.command.build_exe import build_exe
-from obidistutils.serenity.checksystem import is_mac_system
-
-
-class pidname(build_exe):
-    
-    description = "build the pidname executable returning the executable path from a PID on a mac"
-
-
-    def initialize_options(self):
-        build_exe.initialize_options(self)
-
-        self.pidname = False
-
-
-    def finalize_options(self):
-        # This might be confusing: both build-cexe and build-temp default
-        # to build-temp as defined by the "build" command.  This is because
-        # I think that C libraries are really just temporary build
-        # by-products, at least from the point of view of building Python
-        # extensions -- but I want to keep my options open.
-        
-        build_exe.finalize_options(self)
-        
-        self.set_undefined_options('build',
-                                   ('build_scripts', 'build_cexe'))
-
-        # self.ctools = self.distribution.ctools
-        
-        if os.path.isfile("distutils.ext/src/pidname.c"):
-            self.executables = [('pidname',{"sources":["distutils.ext/src/pidname.c"]})]
-            self.check_executable_list(self.executables)
-        else:
-            self.executables = []
-
-#         self.build_cexe = os.path.join(os.path.dirname(self.build_cexe),'cbinaries') 
-#         self.mkpath(self.build_cexe)
-
-
-
-    def run(self):
-        if is_mac_system():
-            build_exe.run(self)
-            self.pidname=True
-        else:
-            self.pidname=False
-        

distutils.ext/obidistutils/command/sdist.py

@@ -1,42 +0,0 @@
-'''
-Created on 10 mars 2015
-
-@author: coissac
-'''
-
-import os.path
-
-from distutils.command.sdist import sdist as orig_sdist
-from distutils import dir_util
-
-class sdist(orig_sdist):
-
-    def make_distribution(self):
-        """Create the source distribution(s).  First, we create the release
-        tree with 'make_release_tree()'; then, we create all required
-        archive files (according to 'self.formats') from the release tree.
-        Finally, we clean up by blowing away the release tree (unless
-        'self.keep_temp' is true).  The list of archive files created is
-        stored so it can be retrieved later by 'get_archive_files()'.
-        """
-        # Don't warn about missing meta-data here -- should be (and is!)
-        # done elsewhere.
-        base_dir = self.distribution.get_fullname()
-        base_name = os.path.join(self.dist_dir,base_dir)
-
-        self.make_release_tree(os.path.join('tmp',base_dir), self.filelist.files)
-        archive_files = []              # remember names of files we create
-        # tar archive must be created last to avoid overwrite and remove
-        if 'tar' in self.formats:
-            self.formats.append(self.formats.pop(self.formats.index('tar')))
-
-        for fmt in self.formats:
-            file = self.make_archive(base_name, fmt, root_dir='tmp',base_dir=base_dir,
-                                     owner=self.owner, group=self.group)
-            archive_files.append(file)
-            self.distribution.dist_files.append(('sdist', '', file))
-
-        self.archive_files = archive_files
-
-        if not self.keep_temp:
-            dir_util.remove_tree(os.path.join('tmp',base_dir), dry_run=self.dry_run)

distutils.ext/obidistutils/core.py

@@ -1,228 +0,0 @@
-'''
-Created on 20 oct. 2012
-
-@author: coissac
-'''
-
-from os import path
-import os.path
-import glob
-import sys
-
-# try:
-#     from setuptools.extension import Extension
-# except ImportError:
-#     from distutils.extension import Extension
-
-from distutils.extension import Extension
-
-from obidistutils.serenity.checkpackage import  install_requirements,\
-                                                check_requirements, \
-                                                RequirementError
-                                                
-from obidistutils.serenity.rerun import enforce_good_python
-from obidistutils.serenity.rerun import rerun_with_anothe_python
-from distutils import log
-
-from obidistutils.dist import Distribution
-from obidistutils.serenity import is_serenity
- 
- 
-def findPackage(root,base=None):
-    modules=[]
-    if base is None:
-        base=[]
-    for module in (path.basename(path.dirname(x)) 
-                   for x in glob.glob(path.join(root,'*','__init__.py'))):
-        modules.append('.'.join(base+[module]))
-        modules.extend(findPackage(path.join(root,module),base+[module]))
-    return modules
-    
-def findCython(root,base=None,pyrexs=None):
-    setupdir = os.path.dirname(sys.argv[0])
-    csourcedir = os.path.join(setupdir,"src")
-    pyrexs=[]
-
-    if base is None:
-        base=[]
-    for module in (path.basename(path.dirname(x)) 
-                   for x in glob.glob(path.join(root,'*','__init__.py'))):
-                       
-                
-        for pyrex in glob.glob(path.join(root,module,'*.pyx')):
-            pyrexs.append(Extension('.'.join(base+[module,path.splitext(path.basename(pyrex))[0]]),
-                                    [pyrex]
-                                    )
-                          )
-            pyrexs[-1].include_dirs.append(csourcedir)
-            try:
-                cfiles = os.path.splitext(pyrex)[0]+".cfiles"
-                cfilesdir = os.path.dirname(cfiles)
-                cfiles = open(cfiles)
-                cfiles = [os.path.relpath(os.path.join(cfilesdir,y),setupdir).strip() 
-                          if y[0] !='@' else y.strip()
-                          for y in cfiles]
-                
-                log.info("Cython module : %s",cfiles)   
-                incdir = set(os.path.dirname(x) for x in cfiles if x[-2:]==".h")
-                cfiles = [x for x in cfiles if x[-2:]==".c"]                
-                pyrexs[-1].sources.extend(cfiles)
-                pyrexs[-1].include_dirs.extend(incdir)
-                pyrexs[-1].extra_compile_args.extend(['-msse2',
-                                                      '-Wno-unused-function',
-                                                      '-Wmissing-braces',
-                                                      '-Wchar-subscripts'])
-                
-            except IOError:
-                pass
-            
-        pyrexs.extend(findCython(path.join(root,module),base+[module]))
-    return pyrexs
-    
-
-def rootname(x):
-    return os.path.splitext(x.sources[0])[0]
-
-
-def prepare_commands():
-    from obidistutils.command.build import build
-    from obidistutils.command.littlebigman import littlebigman
-#    from obidistutils.command.serenity import serenity
-    from obidistutils.command.build_cexe import build_cexe
-    from obidistutils.command.build_ext import build_ext
-    from obidistutils.command.build_ctools import build_ctools
-    from obidistutils.command.build_files import build_files
-    from obidistutils.command.build_scripts import build_scripts
-    from obidistutils.command.install_scripts import install_scripts
-    from obidistutils.command.install_sphinx import install_sphinx
-    from obidistutils.command.install import install
-    from obidistutils.command.pidname import pidname
-    from obidistutils.command.sdist import sdist
-    
-        
-    
-    COMMANDS = {'build':build,
-#                'serenity':serenity,
-                'littlebigman':littlebigman,
-                'pidname':pidname,
-                'build_ctools':build_ctools, 
-                'build_files':build_files,
-                'build_cexe':build_cexe, 
-                'build_ext': build_ext,
-                'build_scripts':build_scripts, 
-                'install_scripts':install_scripts,
-                'install_sphinx':install_sphinx,
-                'install':install,
-                'sdist':sdist}
-    
-#     try:
-#         from setuptools.commands import egg_info
-#         COMMANDS['egg_info']=egg_info
-#     except ImportError:
-#         pass
-    try:
-        from obidistutils.command.build_sphinx import build_sphinx
-        COMMANDS['build_sphinx']=build_sphinx
-    except ImportError:
-        pass
-
-    return COMMANDS
-
-
-CTOOLS =[]
-CEXES  =[]
-FILES  =[]
-
-def setup(**attrs):
-    
-    log.set_threshold(log.INFO)
-    
-    minversion      = attrs.get("pythonmin",'3.4')
-    maxversion      = attrs.get('pythonmax',None)    
-    fork            = attrs.get('fork',False)
-    requirementfile = attrs.get('requirements','requirements.txt')
-
-    try:
-        del attrs['pythonmin']
-    except KeyError:
-        pass
-
-    try:
-        del attrs['pythonmax']
-    except KeyError:
-        pass
-
-    try:
-        del attrs['fork']
-    except KeyError:
-        pass
-
-    try:
-        del attrs['requirements']
-    except KeyError:
-        pass
-    
-    if is_serenity():
-    
-        
-        enforce_good_python(minversion, maxversion, fork)
-        
-        if (install_requirements(requirementfile)):
-            rerun_with_anothe_python(sys.executable,minversion,maxversion,fork)
-        
-    
-        try:
-            check_requirements(requirementfile)
-        except RequirementError as e :
-            log.error(e)                                   
-            sys.exit(1)
-        
-    if 'distclass' not in attrs:
-        attrs['distclass']=Distribution
-
-    if 'python_src' not in attrs:
-        SRC = 'python'
-    else:
-        SRC = attrs['python_src']
-        del(attrs['python_src'])
-    
-    if 'scripts' not in attrs:
-        attrs['scripts'] = glob.glob('%s/*.py' % SRC)
-
-    if 'package_dir' not in attrs:
-        attrs['package_dir'] = {'': SRC}
-
-    if 'packages' not in attrs:
-        attrs['packages'] = findPackage(SRC)
-    
-    if 'cmdclass' not in attrs:
-        attrs['cmdclass'] = prepare_commands()
-
-    if 'ctools' not in attrs:
-        attrs['ctools'] = CTOOLS
-    
-    if 'executables' not in attrs:
-        attrs['executables'] = CEXES
-        
-    if 'files' not in attrs:
-        attrs['files'] = FILES
-        
-    if 'sse' not in attrs:
-        attrs['sse']=None
-        
-    if 'serenity' not in attrs:
-        attrs['serenity']=False
-    
-    EXTENTION=findCython(SRC)
-    
-    if 'ext_modules' not in attrs:
-        attrs['ext_modules'] = EXTENTION
-        
-#     try:
-#         from setuptools.core import setup as ori_setup
-#     except ImportError:
-#         from distutils.core import setup as ori_setup
-
-    from distutils.core import setup as ori_setup
-    
-    ori_setup(**attrs)

distutils.ext/obidistutils/dist.py

@@ -1,58 +0,0 @@
-'''
-Created on 20 oct. 2012
-
-@author: coissac
-'''
-
-# try:
-#     from setuptools.dist import Distribution as ori_Distribution
-# except ImportError:
-#     from distutils.dist import Distribution as ori_Distribution
-
-from distutils.dist import Distribution as ori_Distribution
-
-class Distribution(ori_Distribution):
-    
-    def __init__(self,attrs=None):
-        self.executables = None
-        self.ctools = None
-        self.files = None
-        self.build_cexe  = None
-        self.deprecated_scripts = None
-        self.zip_safe=False
-        self.sse = None
-        self.serenity=attrs['serenity']
-        
-        ori_Distribution.__init__(self, attrs)
-        
-        self.global_options.insert(0,('serenity', None, "install or build the package in a python virtualenv "
-                                                       "without polluting the installed python and with many "
-                                                       "checks during the installation process"
-                                     ))
-        self.global_options.insert(0,('virtualenv', None, "if the installation is done using the serenity mode "
-                                                        "this option allows for specifying the virtualenv name. "
-                                                        "By default the name is PACKAGE-VERSION"
-                                     ))
-        
-    def run_commands(self):
-        """Run each command that was seen on the setup script command line.
-        Uses the list of commands found and cache of command objects
-        created by 'get_command_obj()'.
-        """
-#        self.run_command('littlebigman')
-        ori_Distribution.run_commands(self)
-        
-        
-    def has_executables(self):
-        return self.executables is not None and self.executables
-        
-    def has_ctools(self):
-        return self.ctools is not None and self.ctools
-        
-    def has_files(self):
-        return self.files is not None and self.files
-        
-    def has_deprecated_scripts(self):
-        return self.deprecated_scripts is not None and self.deprecated_scripts
-
-    

distutils.ext/obidistutils/serenity/__init__.py

@@ -1,112 +0,0 @@
-import sys
-
-from distutils import util
-from distutils import sysconfig
-from distutils import log
-from distutils.version import LooseVersion, StrictVersion
-import glob
-import os
-import subprocess
-import re 
-from distutils.errors import DistutilsError
-import tempfile
-
-from importlib.util import spec_from_file_location  # @UnresolvedImport
-import zipimport
-
-import argparse
-
-import base64
-
-from .checkpython import is_python_version
-                        
-                        
-from obidistutils.serenity.rerun import  enforce_good_python
-from obidistutils.serenity.rerun import rerun_with_anothe_python
-
-from obidistutils.serenity.virtual import serenity_virtualenv
-                        
-from obidistutils.serenity.checksystem import is_mac_system, \
-                                              is_windows_system
-                        
-from obidistutils.serenity.checkpackage import install_requirements
-from obidistutils.serenity.checkpackage import check_requirements
-
-from obidistutils.serenity.util import save_argv
-                        
-from obidistutils.serenity.snake import snake
-                            
-    
-def serenity_snake(envname,package,version):
-    old = log.set_threshold(log.INFO)
-
-    log.info("Installing %s (%s) in serenity mode" % (package,version))
-
-    enforce_good_python()
-
-    virtualpython=serenity_virtualenv(envname,package,version)
-    
-    if virtualpython!=os.path.realpath(sys.executable):
-        log.info("Restarting installation within the %s virtualenv" % (envname))
-        rerun_with_anothe_python(virtualpython)
-        
-    log.info("%s will be installed with python : %s" % (package,virtualpython))
-        
-    if install_requirements():    
-        log.info("Restarting installation with all dependencies ok")
-        rerun_with_anothe_python(virtualpython)
-    
-    log.set_threshold(old)
-    
-def serenity_assert(version):
-    check_requirements()
-
-
-def is_serenity():
-    from obidistutils.serenity.globals import local_serenity
-    return local_serenity and local_serenity[0]
-
-def serenity_mode(package,version):
-    
-    save_argv()
-
-    
-    from obidistutils.serenity.globals import saved_args
-    from obidistutils.serenity.globals import local_serenity
-    
-
-    old = log.set_threshold(log.INFO)
-    
-    argparser = argparse.ArgumentParser(add_help=False)
-    argparser.add_argument('--serenity',
-                           dest='serenity', 
-                           action='store_true',
-                           default=False, 
-                           help='Switch the installer in serenity mode. Everythings are installed in a virtualenv')
-
-    argparser.add_argument('--virtualenv',
-                           dest='virtual', 
-                           type=str,
-                           action='store',
-                           default="%s-%s" % (package,version), 
-                           help='Specify the name of the virtualenv used by the serenity mode [default: %s-%s]' % (package,version))    
-    
-    args, unknown = argparser.parse_known_args()
-    sys.argv = [sys.argv[0]] + unknown
-    
-    if args.serenity:
-        local_serenity.append(True)
-        serenity_snake(args.virtual,package,version)
-    else:
-        local_serenity.append(False)       
-    
-    log.set_threshold(old)
-    
-    return args.serenity
-    
-    
-def getVersion(source,main,version):
-    path  = os.path.join(source,main,'%s.py' % version)
-    spec = spec_from_file_location('version',path)
-    return spec.loader.load_module().version.strip()
-    

distutils.ext/obidistutils/serenity/bootstrappip.py

@@ -1,36 +0,0 @@
-'''
-Created on 22 janv. 2016
-
-@author: coissac
-'''
-
-import sys
-from urllib import request
-import os.path
-
-from obidistutils.serenity.util import get_serenity_dir
-from obidistutils.serenity.rerun import rerun_with_anothe_python
-from obidistutils.serenity.checkpython import is_a_virtualenv_python
-
-getpipurl="https://bootstrap.pypa.io/get-pip.py"
-
-def bootstrap():
-
-    getpipfile=os.path.join(get_serenity_dir(),"get-pip.py")
-    
-    with request.urlopen(getpipurl) as getpip:
-        with open(getpipfile,"wb") as out:
-            for l in getpip:
-                out.write(l)
-                
-    python = sys.executable
-    
-    if is_a_virtualenv_python():
-        command= "%s %s" % (python,getpipfile)        
-    else:
-        command= "%s %s --user" % (python,getpipfile)
-    
-    os.system(command)
-    
-    rerun_with_anothe_python(python)
-    

distutils.ext/obidistutils/serenity/checkpackage.py

@@ -1,165 +0,0 @@
-'''
-Created on 2 oct. 2014
-
-@author: coissac
-'''
-
-import re
-import os
-
-try:
-    import pip                                              # @UnresolvedImport
-    from pip.utils import get_installed_distributions       # @UnresolvedImport
-except ImportError:
-    from .bootstrappip import bootstrap
-    bootstrap()
-
-from distutils.version import StrictVersion             # @UnusedImport
-from distutils.errors import DistutilsError
-from distutils import log
-
-class RequirementError(Exception):
-    pass
-
-def is_installed(requirement):
-    
-    
-    requirement_project,requirement_relation,requirement_version = parse_package_requirement(requirement)
-    
-    package = [x for x in get_installed_distributions() if x.project_name==requirement_project]
-    
-    if len(package)==1:
-        if requirement_version is not None and requirement_relation is not None:    
-            rep = (len(package)==1) and eval("StrictVersion('%s') %s StrictVersion('%s')" % (package[0].version,
-                                                                                           requirement_relation,
-                                                                                           requirement_version)
-                                             )
-        else:
-            rep=True
-    else:
-        rep=False
-    
-    if rep:
-        if requirement_version is not None and requirement_relation is not None:        
-            log.info("Look for package %s (%s%s) : ok version %s installed" % (requirement_project,
-                                                                               requirement_relation,
-                                                                               requirement_version,
-                                                                               package[0].version))
-        else:
-            log.info("Look for package %s : ok version %s installed" % (requirement_project,
-                                                                        package[0].version))
-    else:
-        if len(package)!=1:
-            log.info("Look for package %s (%s%s) : not installed" % (requirement_project,
-                                                                     requirement_relation,
-                                                                     requirement_version))
-        else:
-            log.info("Look for package %s (%s%s) : failed only version %s installed" % (requirement_project,
-                                                                                        requirement_relation,
-                                                                                        requirement_version,
-                                                                                        package[0].version))
-        
-    return rep
-
-
-def get_requirements(requirementfile='requirements.txt'):
-    
-    try:
-        requirements = open(requirementfile).readlines()
-        requirements = [x.strip() for x in requirements]
-        requirements = [x for x in requirements if x[0]!='-']
-    
-    except IOError:
-        requirements = []
-        
-    return requirements
-    
-    
-def install_requirements(requirementfile='requirements.txt'):
-    
-    install_something=False
-
-    requirements = get_requirements(requirementfile)
-
-    log.info("Required packages for the installation :")
-    for x in requirements:
-        ok = is_installed(x)
-        if not ok:
-            log.info("  Installing requirement : %s" % x)
-            pip_install_package(x)
-            install_something=True
-            if x[0:3]=='pip':
-                return True
-                
-    return install_something
- 
-
-def check_requirements(requirementfile='requirements.txt'):
-    
-    requirements = get_requirements(requirementfile)
-    
-    log.info("Required packages for the installation :")
-    for x in requirements:
-        ok = is_installed(x)
-        if not ok:
-            raise RequirementError("  Missing requirement : %s -- Package installation stopped" % x)
-                
- 
-
-
-def parse_package_requirement(requirement):
-    
-    version_pattern = re.compile('[=><]+(.*)$')
-    project_pattern  = re.compile('[^=><]+')
-    relationship_pattern = re.compile('[=><]+')
-    
-    try:
-        requirement_project = project_pattern.search(requirement).group(0)
-        requirement_version = version_pattern.search(requirement)
-        if requirement_version is not None:
-            requirement_version=requirement_version.group(1)
-        requirement_relation= relationship_pattern.search(requirement)
-        if requirement_relation is not None:
-            requirement_relation=requirement_relation.group(0)
-    except:
-        raise DistutilsError("Requirement : %s not correctly formated" % requirement)
-    
-    return requirement_project,requirement_relation,requirement_version
-    
-
-def get_package_requirement(package,requirementfile='requirements.txt'):            
-    requirements = get_requirements(requirementfile)
-    req = [x for x in requirements
-             if x[0:len(package)]==package
-          ]
-    
-    if len(req)==1:
-        return req[0]
-    else:
-        return None
-        
-        
-def pip_install_package(package,directory=None,upgrade=True):
-
-    if directory is not None:
-        log.info('    installing %s in directory %s' % (package,str(directory)))
-        
-
-    if 'http_proxy' in os.environ and 'https_proxy' not in os.environ:
-        os.environ['https_proxy']=os.environ['http_proxy']
-
-    args = ['install']
-    
-    if upgrade:
-        args.append('--upgrade')
-    
-    if 'https_proxy' in os.environ:
-        args.append('--proxy=%s' % os.environ['https_proxy'])
-        
-    if directory is not None:
-        args.append('--target=%s' % directory)
-    
-    args.append(package)
-        
-    return pip.main(args)
-

distutils.ext/obidistutils/serenity/checkpython.py

@@ -1,138 +0,0 @@
-'''
-Created on 24 mai 2015
-
-@author: coissac
-'''
-from distutils.version import StrictVersion
-from distutils import sysconfig
-import subprocess
-import os
-import glob
-import re
-
-from obidistutils.serenity.checksystem import is_windows_system
-import sys
-
-
-def is_python_version(path=None,minversion='3.4',maxversion=None):
-    '''
-    Checks that the python version is in the range {minversion,maxversion[
-    
-        @param path: if None consider the running python
-                     otherwise the python pointed by the path
-        @param minversion: the minimum version to consider
-        @param maxversion: the maximum version to consider (strictly inferior to)
-                     
-        @return: True if the python version match
-        @rtype: bool
-    '''
-    if path is None:
-        pythonversion = StrictVersion(sysconfig.get_python_version())
-    else:
-        command = """'%s' -c 'from distutils import sysconfig; """ \
-                  """print(sysconfig.get_python_version())'""" % path
-                  
-        p = subprocess.Popen(command, 
-                             shell=True, 
-                             stdout=subprocess.PIPE)
-        pythonversion=str(p.communicate()[0],'utf8').strip()
-        pythonversion = StrictVersion(pythonversion)
-                        
-    return  (        pythonversion >=StrictVersion(minversion) 
-             and (   maxversion is None    
-                  or pythonversion < StrictVersion(maxversion))
-            )
-           
-
-def lookfor_good_python(minversion='3.4',maxversion=None,followLink=False):
-    '''
-    Look for all python interpreters present in the system path that
-    match the version constraints.
-    
-    @param minversion: the minimum version to consider
-    @param maxversion: the maximum version to consider (strictly inferior to)
-    @param followLink: a boolean value indicating if link must be substituted 
-                       by their real path.
-                       
-    @return: a list of path to interpreters
-    '''
-    exe = []
-    if not is_windows_system():
-        paths = os.environ['PATH'].split(os.pathsep)
-        for p in paths:
-            candidates = glob.glob(os.path.join(p,'python*')) 
-            pexe = []
-            pythonpat=re.compile('python([0-9]|[0-9]\.[0-9])?$')
-            for e in candidates:
-                print(e)
-                if pythonpat.search(e) is not None:
-                    if followLink and os.path.islink(e):
-                        e = os.path.realpath(e)
-                    if (os.path.isfile(e) and 
-                       os.access(e, os.X_OK) and 
-                       is_python_version(e,minversion,maxversion)):
-                        pexe.append(e)
-            exe.extend(set(pexe))
-        
-    return exe
-
-def is_a_virtualenv_python(path=None):
-    '''
-    Check if the python is belonging a virtualenv
-    
-        @param path: the path pointing to the python executable.
-                     if path is None then the running python is
-                     considered.
-        @param path: str or None 
-                     
-        @return: True if the python belongs a virtualenv
-                 False otherwise
-        @rtype: bool
-                 
-    '''
-    if path is None:
-        rep = sys.base_exec_prefix != sys.exec_prefix
-    else:
-        command = """'%s' -c 'import sys; print(sys.base_exec_prefix != sys.exec_prefix)'""" % path
-        p = subprocess.Popen(command, 
-                             shell=True, 
-                             stdout=subprocess.PIPE)
-        rep = eval(str(p.communicate()[0],'utf8'))
-        
-    return rep
-
-
-def which_virtualenv(path=None,full=False):
-    '''
-    Returns the name of the virtualenv.
-        @param path: the path to a python binary or None
-                     if you want to consider the running python
-        @type path: str or None
-                     
-        @param full: if set to True, returns the absolute path,
-                     otherwise only return a simple directory name
-        @type full: bool
-                 
-        @return: the virtual environment name or None if the
-                 path does not belong a virtualenv
-        @rtype: str or None
-    '''
-    if path is None:
-        path = sys.executable
-    
-    if is_a_virtualenv_python(path):
-        parts = path.split(os.sep)
-        try:
-            if full:
-                rep = os.sep.join(parts[0:parts.index('bin')])
-                rep = os.path.realpath(rep)
-            else:
-                rep = parts[parts.index('bin')-1]
-        except ValueError:
-            rep = None
-    else:
-        rep=None
-        
-    return rep
-        
-

distutils.ext/obidistutils/serenity/checksystem.py

@@ -1,18 +0,0 @@
-'''
-Created on 24 mai 2015
-
-@author: coissac
-'''
-
-from distutils import util
-
-def is_mac_system():
-    platform = util.get_platform().split('-')[0]
-    
-    return platform=='macosx'
-
-def is_windows_system():
-    platform = util.get_platform().split('-')[0]
-    
-    return platform=='Windows'
-        

distutils.ext/obidistutils/serenity/globals.py

@@ -1,14 +0,0 @@
-'''
-Created on 24 mai 2015
-
-@author: coissac
-'''
-
-
-saved_args=[]
-tmpdir=[]
-local_pip=[]
-local_virtualenv=[]
-local_cython=[]
-local_serenity=[]
-        

distutils.ext/obidistutils/serenity/rerun.py

@@ -1,61 +0,0 @@
-'''
-Created on 24 mai 2015
-
-@author: coissac
-'''
-
-import sys
-import os
-from distutils import log
-from distutils.errors import DistutilsError
-
-
-from obidistutils.serenity.globals import saved_args
-from obidistutils.serenity.checkpython import is_python_version,\
-    lookfor_good_python
-
-
-def rerun_with_anothe_python(path, minversion='3.4',maxversion=None, fork=False):
-        
-    if saved_args:
-        args = saved_args
-    else:
-        args = list(sys.argv)
-        
-          
-    assert is_python_version(path,minversion,maxversion), \
-           'the selected python is not adapted to the installation of this package'
-                   
-    args.insert(0, path)
-        
-    sys.stderr.flush()
-    sys.stdout.flush()
-    
-    if fork:
-        log.info('Forking a new install process')
-        os.system(' '.join(list(args)))
-        log.info('External process ended')
-        sys.exit(0)
-    else:
-        log.info('Install script restarting...')
-        os.execv(path,list(args))
-
-def enforce_good_python(minversion='3.4',maxversion=None, fork=False):
-    if is_python_version(minversion=minversion,maxversion=maxversion):
-        log.info('You are running the good python')
-        return True
-    
-    goodpython = lookfor_good_python(minversion,maxversion)
-    
-    if not goodpython:
-        raise DistutilsError('No good python identified on your system')
-
-    goodpython=goodpython[0]
-    
-    log.warn("========================================")    
-    log.warn("")
-    log.warn("    Switching to python : %s" % goodpython)
-    log.warn("")
-    log.warn("========================================")    
-
-    rerun_with_anothe_python(goodpython)

distutils.ext/obidistutils/serenity/snake.py

@@ -1,35 +0,0 @@
-'''
-Created on 2 oct. 2014
-
-@author: coissac
-'''
-
-snake ="""
-
-        ___
-      ,'._,`.
-     (-.___.-)
-     (-.___.-)
-     `-.___.-'
-      ((  @ @|              .            __
-       \   ` |         ,\   |`.    @|   |  |      _.-._
-      __`.`=-=mm===mm:: |   | |`.   |   |  |    ,'=` '=`.
-     (    `-'|:/  /:/  `/  @| | |   |, @| @|   /---)W(---\ 
-      \ \   / /  / /         @| |   '         (----| |----) ,~
-      |\ \ / /| / /            @|              \---| |---/  |
-      | \ V /||/ /                              `.-| |-,'   |
-      |  `-' |V /                                 \| |/    @'
-      |    , |-'                                 __| |__
-      |    .;: _,-.                         ,--""..| |..""--.
-      ;;:::' "    )                        (`--::__|_|__::--')
-    ,-"      _,  /                          \`--...___...--'/
-   (    -:--'/  /                           /`--...___...--'\  
-    "-._  `"'._/                           /`---...___...---'\  
-        "-._   "---.                      (`---....___....---')
-         .' ",._ ,' )                     |`---....___....---'|
-         /`._|  `|  |                     (`---....___....---')
-        (   \    |  /                      \`---...___...---'/
-         `.  `,  ^""                        `:--...___...--;'
-           `.,'               hh              `-._______.-'
-
-"""

distutils.ext/obidistutils/serenity/util.py

@@ -1,27 +0,0 @@
-'''
-Created on 2 oct. 2014
-
-@author: coissac
-'''
-
-import sys 
-import tempfile
-
-
-from obidistutils.serenity.globals import tmpdir         # @UnusedImport
-from obidistutils.serenity.globals import saved_args     # @UnusedImport
-
-def get_serenity_dir():
-    global tmpdir
-
-    if not tmpdir:
-        tmpdir.append(tempfile.mkdtemp())
-    return tmpdir[0]
-        
-def save_argv():
-    global saved_args
-
-    del saved_args[:]
-    saved_args.extend(list(sys.argv))
-
-           

distutils.ext/obidistutils/serenity/virtual.py

@@ -1,69 +0,0 @@
-'''
-Created on 2 oct. 2014
-
-@author: coissac
-'''
-
-import os
-import sys
-import venv
-
-from distutils.errors import DistutilsError
-from .globals import local_virtualenv  # @UnusedImport
-from .checkpython import   which_virtualenv,\
-                           is_python_version, \
-                           is_a_virtualenv_python
-                                                
-
-    
-    
-def serenity_virtualenv(envname,package,version,minversion='3.4',maxversion=None):
-        
-    #
-    # Checks if we are already running under the good virtualenv
-    #
-    ve = which_virtualenv(full=True)
-    if ve == os.path.realpath(envname) and is_python_version(minversion=minversion,maxversion=maxversion):
-        return sys.executable
-        
-    #
-    # Check if the virtualenv exist
-    # 
-        
-    python = None
-    
-    if os.path.isdir(envname):
-        python = os.path.join(envname,'bin','python')
-        ok = (is_python_version(python,
-                                minversion=minversion,
-                                maxversion=maxversion) and 
-              is_a_virtualenv_python(python))
-        
-        
-        #
-        # The virtualenv already exist but it is not ok
-        #
-        if not ok:
-            raise DistutilsError("A virtualenv %s already exists but not with the required python")
-                 
-    else:
-        ok = False
-                
-        
-    #
-    # Creates a new virtualenv
-    #
-    if not ok:
-        venv.create(envname, 
-                    system_site_packages=False, 
-                    clear=True, 
-                    symlinks=False, 
-                    with_pip=True)
-                
-        # check the newly created virtualenv
-        return serenity_virtualenv(envname,package,version)
-    
-    return os.path.realpath(python)
-    
-    
-    

distutils.ext/src/littlebigman.c

@@ -1,24 +0,0 @@
-/*
- * littlebigman.c
- *
- *  Created on: 11 juil. 2012
- *      Author: coissac
- */
-
-#include<stdio.h>
-
-int main(int argc, char *argv[])
-{
-    union { int entier;
-            char caractere[4] ;
-    } test;
-
-    test.entier=0x01020304;
-
-    if (test.caractere[3] == 1)
-       printf("-DLITTLE_END");
-    else
-        printf("-DBIG_END");
-
-	return 0;
-}

distutils.ext/src/pidname.c

@@ -1,24 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <errno.h>
-#include <libproc.h>
-
-int main (int argc, char* argv[])
-{
-	pid_t pid; int ret;
-	char pathbuf[PROC_PIDPATHINFO_MAXSIZE];
-
-	if ( argc > 1 ) {
-		pid = (pid_t) atoi(argv[1]);
-		ret = proc_pidpath (pid, pathbuf, sizeof(pathbuf));
-		if ( ret <= 0 ) {
-			fprintf(stderr, "PID %d: proc_pidpath ();\n", pid);
-			fprintf(stderr,	"    %s\n", strerror(errno));
-		} else {
-			printf("proc %d: %s\n", pid, pathbuf);
-		}
-	}
-
-	return 0;
-}

doc/conf.py

@@ -33,7 +33,7 @@ extensions = [
     'sphinx.ext.autodoc',
     'sphinx.ext.todo',
     'sphinx.ext.coverage',
-    'sphinx.ext.pngmath',
+    'sphinx.ext.imgmath',
     'sphinx.ext.ifconfig',
     'sphinx.ext.viewcode',
     'breathe',

doc/sphinx/.gitignore

@@ -0,0 +1,2 @@
+.DS_Store
+/build_dir.txt

doc/sphinx/build_dir.txt

@@ -1 +0,0 @@
-build/lib.macosx-10.6-intel-3.4

python/.gitignore

@@ -0,0 +1 @@
+/OBITools3.egg-info/

python/obitools3/apps/arguments.cfiles

@@ -0,0 +1,110 @@
+../../../src/obi_lcs.h
+../../../src/obi_lcs.c
+../../../src/obierrno.h
+../../../src/obierrno.c
+../../../src/upperband.h
+../../../src/upperband.c
+../../../src/sse_banded_LCS_alignment.h
+../../../src/sse_banded_LCS_alignment.c
+../../../src/obiblob.h
+../../../src/obiblob.c
+../../../src/utils.h
+../../../src/utils.c
+../../../src/obidms.h
+../../../src/obidms.c
+../../../src/libjson/json_utils.h
+../../../src/libjson/json_utils.c
+../../../src/libjson/cJSON.h
+../../../src/libjson/cJSON.c
+../../../src/obiavl.h
+../../../src/obiavl.c
+../../../src/bloom.h
+../../../src/bloom.c
+../../../src/crc64.h
+../../../src/crc64.c
+../../../src/murmurhash2.h
+../../../src/murmurhash2.c
+../../../src/obidmscolumn.h
+../../../src/obidmscolumn.c
+../../../src/obitypes.h
+../../../src/obitypes.c
+../../../src/obidmscolumndir.h
+../../../src/obidmscolumndir.c
+../../../src/obiblob_indexer.h
+../../../src/obiblob_indexer.c
+../../../src/obiview.h
+../../../src/obiview.c
+../../../src/hashtable.h
+../../../src/hashtable.c
+../../../src/linked_list.h
+../../../src/linked_list.c
+../../../src/obidmscolumn_array.h
+../../../src/obidmscolumn_array.c
+../../../src/obidmscolumn_blob.h
+../../../src/obidmscolumn_blob.c
+../../../src/obidmscolumn_idx.h
+../../../src/obidmscolumn_idx.c
+../../../src/obidmscolumn_bool.h
+../../../src/obidmscolumn_bool.c
+../../../src/obidmscolumn_char.h
+../../../src/obidmscolumn_char.c
+../../../src/obidmscolumn_float.h
+../../../src/obidmscolumn_float.c
+../../../src/obidmscolumn_int.h
+../../../src/obidmscolumn_int.c
+../../../src/obidmscolumn_qual.h
+../../../src/obidmscolumn_qual.c
+../../../src/obidmscolumn_seq.h
+../../../src/obidmscolumn_seq.c
+../../../src/obidmscolumn_str.h
+../../../src/obidmscolumn_str.c
+../../../src/array_indexer.h
+../../../src/array_indexer.c
+../../../src/char_str_indexer.h
+../../../src/char_str_indexer.c
+../../../src/dna_seq_indexer.h
+../../../src/dna_seq_indexer.c
+../../../src/encode.c
+../../../src/encode.h
+../../../src/uint8_indexer.c
+../../../src/uint8_indexer.h
+../../../src/build_reference_db.c
+../../../src/build_reference_db.h
+../../../src/kmer_similarity.c
+../../../src/kmer_similarity.h
+../../../src/obi_clean.c
+../../../src/obi_clean.h
+../../../src/obi_ecopcr.c
+../../../src/obi_ecopcr.h
+../../../src/obi_ecotag.c
+../../../src/obi_ecotag.h
+../../../src/obidms_taxonomy.c
+../../../src/obidms_taxonomy.h
+../../../src/obilittlebigman.c
+../../../src/obilittlebigman.h
+../../../src/_sse.h
+../../../src/obidebug.h
+../../../src/libecoPCR/libapat/CODES/dft_code.h
+../../../src/libecoPCR/libapat/CODES/dna_code.h
+../../../src/libecoPCR/libapat/CODES/prot_code.h
+../../../src/libecoPCR/libapat/apat_parse.c
+../../../src/libecoPCR/libapat/apat_search.c
+../../../src/libecoPCR/libapat/apat.h
+../../../src/libecoPCR/libapat/Gmach.h
+../../../src/libecoPCR/libapat/Gtypes.h
+../../../src/libecoPCR/libapat/libstki.c
+../../../src/libecoPCR/libapat/libstki.h
+../../../src/libecoPCR/libthermo/nnparams.h
+../../../src/libecoPCR/libthermo/nnparams.c
+../../../src/libecoPCR/ecoapat.c
+../../../src/libecoPCR/ecodna.c
+../../../src/libecoPCR/ecoError.c
+../../../src/libecoPCR/ecoMalloc.c
+../../../src/libecoPCR/ecoPCR.h
+
+
+
+
+
+
+

python/obitools3/apps/arguments.pyx

@@ -57,5 +57,6 @@ cpdef buildArgumentParser(str configname,
                 module.addOptions(sub)
             
             sub.set_defaults(**{'%s:module'  % configname : module})
-                        
+            sub.set_defaults(**{'%s:modulename'  % configname : c})
+              
     return parser

python/obitools3/apps/command.cfiles

@@ -0,0 +1,110 @@
+../../../src/obi_lcs.h
+../../../src/obi_lcs.c
+../../../src/obierrno.h
+../../../src/obierrno.c
+../../../src/upperband.h
+../../../src/upperband.c
+../../../src/sse_banded_LCS_alignment.h
+../../../src/sse_banded_LCS_alignment.c
+../../../src/obiblob.h
+../../../src/obiblob.c
+../../../src/utils.h
+../../../src/utils.c
+../../../src/obidms.h
+../../../src/obidms.c
+../../../src/libjson/json_utils.h
+../../../src/libjson/json_utils.c
+../../../src/libjson/cJSON.h
+../../../src/libjson/cJSON.c
+../../../src/obiavl.h
+../../../src/obiavl.c
+../../../src/bloom.h
+../../../src/bloom.c
+../../../src/crc64.h
+../../../src/crc64.c
+../../../src/murmurhash2.h
+../../../src/murmurhash2.c
+../../../src/obidmscolumn.h
+../../../src/obidmscolumn.c
+../../../src/obitypes.h
+../../../src/obitypes.c
+../../../src/obidmscolumndir.h
+../../../src/obidmscolumndir.c
+../../../src/obiblob_indexer.h
+../../../src/obiblob_indexer.c
+../../../src/obiview.h
+../../../src/obiview.c
+../../../src/hashtable.h
+../../../src/hashtable.c
+../../../src/linked_list.h
+../../../src/linked_list.c
+../../../src/obidmscolumn_array.h
+../../../src/obidmscolumn_array.c
+../../../src/obidmscolumn_blob.h
+../../../src/obidmscolumn_blob.c
+../../../src/obidmscolumn_idx.h
+../../../src/obidmscolumn_idx.c
+../../../src/obidmscolumn_bool.h
+../../../src/obidmscolumn_bool.c
+../../../src/obidmscolumn_char.h
+../../../src/obidmscolumn_char.c
+../../../src/obidmscolumn_float.h
+../../../src/obidmscolumn_float.c
+../../../src/obidmscolumn_int.h
+../../../src/obidmscolumn_int.c
+../../../src/obidmscolumn_qual.h
+../../../src/obidmscolumn_qual.c
+../../../src/obidmscolumn_seq.h
+../../../src/obidmscolumn_seq.c
+../../../src/obidmscolumn_str.h
+../../../src/obidmscolumn_str.c
+../../../src/array_indexer.h
+../../../src/array_indexer.c
+../../../src/char_str_indexer.h
+../../../src/char_str_indexer.c
+../../../src/dna_seq_indexer.h
+../../../src/dna_seq_indexer.c
+../../../src/encode.c
+../../../src/encode.h
+../../../src/uint8_indexer.c
+../../../src/uint8_indexer.h
+../../../src/build_reference_db.c
+../../../src/build_reference_db.h
+../../../src/kmer_similarity.c
+../../../src/kmer_similarity.h
+../../../src/obi_clean.c
+../../../src/obi_clean.h
+../../../src/obi_ecopcr.c
+../../../src/obi_ecopcr.h
+../../../src/obi_ecotag.c
+../../../src/obi_ecotag.h
+../../../src/obidms_taxonomy.c
+../../../src/obidms_taxonomy.h
+../../../src/obilittlebigman.c
+../../../src/obilittlebigman.h
+../../../src/_sse.h
+../../../src/obidebug.h
+../../../src/libecoPCR/libapat/CODES/dft_code.h
+../../../src/libecoPCR/libapat/CODES/dna_code.h
+../../../src/libecoPCR/libapat/CODES/prot_code.h
+../../../src/libecoPCR/libapat/apat_parse.c
+../../../src/libecoPCR/libapat/apat_search.c
+../../../src/libecoPCR/libapat/apat.h
+../../../src/libecoPCR/libapat/Gmach.h
+../../../src/libecoPCR/libapat/Gtypes.h
+../../../src/libecoPCR/libapat/libstki.c
+../../../src/libecoPCR/libapat/libstki.h
+../../../src/libecoPCR/libthermo/nnparams.h
+../../../src/libecoPCR/libthermo/nnparams.c
+../../../src/libecoPCR/ecoapat.c
+../../../src/libecoPCR/ecodna.c
+../../../src/libecoPCR/ecoError.c
+../../../src/libecoPCR/ecoMalloc.c
+../../../src/libecoPCR/ecoPCR.h
+
+
+
+
+
+
+

python/obitools3/apps/config.cfiles

@@ -0,0 +1,110 @@
+../../../src/obi_lcs.h
+../../../src/obi_lcs.c
+../../../src/obierrno.h
+../../../src/obierrno.c
+../../../src/upperband.h
+../../../src/upperband.c
+../../../src/sse_banded_LCS_alignment.h
+../../../src/sse_banded_LCS_alignment.c
+../../../src/obiblob.h
+../../../src/obiblob.c
+../../../src/utils.h
+../../../src/utils.c
+../../../src/obidms.h
+../../../src/obidms.c
+../../../src/libjson/json_utils.h
+../../../src/libjson/json_utils.c
+../../../src/libjson/cJSON.h
+../../../src/libjson/cJSON.c
+../../../src/obiavl.h
+../../../src/obiavl.c
+../../../src/bloom.h
+../../../src/bloom.c
+../../../src/crc64.h
+../../../src/crc64.c
+../../../src/murmurhash2.h
+../../../src/murmurhash2.c
+../../../src/obidmscolumn.h
+../../../src/obidmscolumn.c
+../../../src/obitypes.h
+../../../src/obitypes.c
+../../../src/obidmscolumndir.h
+../../../src/obidmscolumndir.c
+../../../src/obiblob_indexer.h
+../../../src/obiblob_indexer.c
+../../../src/obiview.h
+../../../src/obiview.c
+../../../src/hashtable.h
+../../../src/hashtable.c
+../../../src/linked_list.h
+../../../src/linked_list.c
+../../../src/obidmscolumn_array.h
+../../../src/obidmscolumn_array.c
+../../../src/obidmscolumn_blob.h
+../../../src/obidmscolumn_blob.c
+../../../src/obidmscolumn_idx.h
+../../../src/obidmscolumn_idx.c
+../../../src/obidmscolumn_bool.h
+../../../src/obidmscolumn_bool.c
+../../../src/obidmscolumn_char.h
+../../../src/obidmscolumn_char.c
+../../../src/obidmscolumn_float.h
+../../../src/obidmscolumn_float.c
+../../../src/obidmscolumn_int.h
+../../../src/obidmscolumn_int.c
+../../../src/obidmscolumn_qual.h
+../../../src/obidmscolumn_qual.c
+../../../src/obidmscolumn_seq.h
+../../../src/obidmscolumn_seq.c
+../../../src/obidmscolumn_str.h
+../../../src/obidmscolumn_str.c
+../../../src/array_indexer.h
+../../../src/array_indexer.c
+../../../src/char_str_indexer.h
+../../../src/char_str_indexer.c
+../../../src/dna_seq_indexer.h
+../../../src/dna_seq_indexer.c
+../../../src/encode.c
+../../../src/encode.h
+../../../src/uint8_indexer.c
+../../../src/uint8_indexer.h
+../../../src/build_reference_db.c
+../../../src/build_reference_db.h
+../../../src/kmer_similarity.c
+../../../src/kmer_similarity.h
+../../../src/obi_clean.c
+../../../src/obi_clean.h
+../../../src/obi_ecopcr.c
+../../../src/obi_ecopcr.h
+../../../src/obi_ecotag.c
+../../../src/obi_ecotag.h
+../../../src/obidms_taxonomy.c
+../../../src/obidms_taxonomy.h
+../../../src/obilittlebigman.c
+../../../src/obilittlebigman.h
+../../../src/_sse.h
+../../../src/obidebug.h
+../../../src/libecoPCR/libapat/CODES/dft_code.h
+../../../src/libecoPCR/libapat/CODES/dna_code.h
+../../../src/libecoPCR/libapat/CODES/prot_code.h
+../../../src/libecoPCR/libapat/apat_parse.c
+../../../src/libecoPCR/libapat/apat_search.c
+../../../src/libecoPCR/libapat/apat.h
+../../../src/libecoPCR/libapat/Gmach.h
+../../../src/libecoPCR/libapat/Gtypes.h
+../../../src/libecoPCR/libapat/libstki.c
+../../../src/libecoPCR/libapat/libstki.h
+../../../src/libecoPCR/libthermo/nnparams.h
+../../../src/libecoPCR/libthermo/nnparams.c
+../../../src/libecoPCR/ecoapat.c
+../../../src/libecoPCR/ecodna.c
+../../../src/libecoPCR/ecoError.c
+../../../src/libecoPCR/ecoMalloc.c
+../../../src/libecoPCR/ecoPCR.h
+
+
+
+
+
+
+

python/obitools3/apps/config.pxd

@@ -7,4 +7,4 @@ cdef dict buildDefaultConfiguration(str root_config_name,
                                     dict  config)
 
 cpdef dict getConfiguration(str root_config_name=?,
-                                    dict  config=?)
+                                    dict  config=?)

python/obitools3/apps/config.pyx

@@ -101,3 +101,14 @@ cpdef dict getConfiguration(str root_config_name="__default__",
     config['__done__']=True
             
     return config
+
+def logger(level, *messages):
+    try:
+        config=getConfiguration()
+        root = config["__root_config__"]
+        l = config[root]['logger']
+        if config[root]['verbose']:
+            getattr(l, level)(*messages)
+    except:
+        print(*messages,file=sys.stderr)
+

python/obitools3/apps/logging.cfiles

@@ -0,0 +1,110 @@
+../../../src/obi_lcs.h
+../../../src/obi_lcs.c
+../../../src/obierrno.h
+../../../src/obierrno.c
+../../../src/upperband.h
+../../../src/upperband.c
+../../../src/sse_banded_LCS_alignment.h
+../../../src/sse_banded_LCS_alignment.c
+../../../src/obiblob.h
+../../../src/obiblob.c
+../../../src/utils.h
+../../../src/utils.c
+../../../src/obidms.h
+../../../src/obidms.c
+../../../src/libjson/json_utils.h
+../../../src/libjson/json_utils.c
+../../../src/libjson/cJSON.h
+../../../src/libjson/cJSON.c
+../../../src/obiavl.h
+../../../src/obiavl.c
+../../../src/bloom.h
+../../../src/bloom.c
+../../../src/crc64.h
+../../../src/crc64.c
+../../../src/murmurhash2.h
+../../../src/murmurhash2.c
+../../../src/obidmscolumn.h
+../../../src/obidmscolumn.c
+../../../src/obitypes.h
+../../../src/obitypes.c
+../../../src/obidmscolumndir.h
+../../../src/obidmscolumndir.c
+../../../src/obiblob_indexer.h
+../../../src/obiblob_indexer.c
+../../../src/obiview.h
+../../../src/obiview.c
+../../../src/hashtable.h
+../../../src/hashtable.c
+../../../src/linked_list.h
+../../../src/linked_list.c
+../../../src/obidmscolumn_array.h
+../../../src/obidmscolumn_array.c
+../../../src/obidmscolumn_blob.h
+../../../src/obidmscolumn_blob.c
+../../../src/obidmscolumn_idx.h
+../../../src/obidmscolumn_idx.c
+../../../src/obidmscolumn_bool.h
+../../../src/obidmscolumn_bool.c
+../../../src/obidmscolumn_char.h
+../../../src/obidmscolumn_char.c
+../../../src/obidmscolumn_float.h
+../../../src/obidmscolumn_float.c
+../../../src/obidmscolumn_int.h
+../../../src/obidmscolumn_int.c
+../../../src/obidmscolumn_qual.h
+../../../src/obidmscolumn_qual.c
+../../../src/obidmscolumn_seq.h
+../../../src/obidmscolumn_seq.c
+../../../src/obidmscolumn_str.h
+../../../src/obidmscolumn_str.c
+../../../src/array_indexer.h
+../../../src/array_indexer.c
+../../../src/char_str_indexer.h
+../../../src/char_str_indexer.c
+../../../src/dna_seq_indexer.h
+../../../src/dna_seq_indexer.c
+../../../src/encode.c
+../../../src/encode.h
+../../../src/uint8_indexer.c
+../../../src/uint8_indexer.h
+../../../src/build_reference_db.c
+../../../src/build_reference_db.h
+../../../src/kmer_similarity.c
+../../../src/kmer_similarity.h
+../../../src/obi_clean.c
+../../../src/obi_clean.h
+../../../src/obi_ecopcr.c
+../../../src/obi_ecopcr.h
+../../../src/obi_ecotag.c
+../../../src/obi_ecotag.h
+../../../src/obidms_taxonomy.c
+../../../src/obidms_taxonomy.h
+../../../src/obilittlebigman.c
+../../../src/obilittlebigman.h
+../../../src/_sse.h
+../../../src/obidebug.h
+../../../src/libecoPCR/libapat/CODES/dft_code.h
+../../../src/libecoPCR/libapat/CODES/dna_code.h
+../../../src/libecoPCR/libapat/CODES/prot_code.h
+../../../src/libecoPCR/libapat/apat_parse.c
+../../../src/libecoPCR/libapat/apat_search.c
+../../../src/libecoPCR/libapat/apat.h
+../../../src/libecoPCR/libapat/Gmach.h
+../../../src/libecoPCR/libapat/Gtypes.h
+../../../src/libecoPCR/libapat/libstki.c
+../../../src/libecoPCR/libapat/libstki.h
+../../../src/libecoPCR/libthermo/nnparams.h
+../../../src/libecoPCR/libthermo/nnparams.c
+../../../src/libecoPCR/ecoapat.c
+../../../src/libecoPCR/ecodna.c
+../../../src/libecoPCR/ecoError.c
+../../../src/libecoPCR/ecoMalloc.c
+../../../src/libecoPCR/ecoPCR.h
+
+
+
+
+
+
+

python/obitools3/apps/logging.pyx

@@ -22,7 +22,7 @@ cpdef getLogger(dict config):
     logfile= config[root]['log'] 
     
     rootlogger   = logging.getLogger()
-    logFormatter = logging.Formatter("%(asctime)s [%(levelname)-5.5s]  %(message)s")
+    logFormatter = logging.Formatter("%%(asctime)s [%s : %%(levelname)-5.5s]  %%(message)s" % config[root]['modulename'])
 
     stderrHandler = logging.StreamHandler(sys.stderr)
     stderrHandler.setFormatter(logFormatter)
@@ -42,5 +42,7 @@ cpdef getLogger(dict config):
     rootlogger.setLevel(loglevel)
     
     config[root]['logger']=rootlogger
-    
+    config[root]['verbose']=True
+        
     return rootlogger
+

python/obitools3/apps/optiongroups/__init__.py

@@ -0,0 +1,272 @@
+def __addInputOption(optionManager):
+    
+    optionManager.add_argument(
+                    dest='obi:inputURI',  
+                    metavar='INPUT', 
+                    help='Data source URI')
+    
+
+    group = optionManager.add_argument_group("Restriction to a sub-part options",
+                    "Allows to limit analysis to a sub-part of the input")
+
+    group.add_argument('--skip',
+                     action="store", dest="obi:skip",
+                     metavar='<N>',
+                     default=None,
+                     type=int,
+                     help="skip the N first sequences")
+
+    group.add_argument('--only',
+                     action="store", dest="obi:only",
+                     metavar='<N>',
+                     default=None,
+                     type=int,
+                     help="treat only N sequences")
+    
+
+def __addImportInputOption(optionManager):
+    group = optionManager.add_argument_group("Input format options for imported files")
+
+    group.add_argument('--fasta-input',
+                     action="store_const", dest="obi:inputformat",
+                     default=None,
+                     const=b'fasta',
+                     help="Input file is in sanger fasta format")
+
+    group.add_argument('--fastq-input',
+                     action="store_const", dest="obi:inputformat",
+                     default=None,
+                     const=b'fastq',
+                     help="Input file is in fastq format")
+
+    group.add_argument('--embl-input',
+                     action="store_const", dest="obi:inputformat",
+                     default=None,
+                     const=b'embl',
+                     help="Input file is in embl nucleic format")
+
+    group.add_argument('--genbank-input',
+                     action="store_const", dest="obi:inputformat",
+                     default=None,
+                     const=b'genbank',
+                     help="Input file is in genbank nucleic format")
+
+    group.add_argument('--ngsfilter-input',
+                     action="store_const", dest="obi:inputformat",
+                     default=None,
+                     const=b'ngsfilter',
+                     help="Input file is an ngsfilter file")
+
+    group.add_argument('--ecopcr-result-input',
+                     action="store_const", dest="obi:inputformat",
+                     default=None,
+                     const=b'ecopcr',
+                     help="Input file is the result of an ecoPCR (version 2)")
+
+    group.add_argument('--ecoprimers-result-input',
+                     action="store_const", dest="obi:inputformat",
+                     default=None,
+                     const=b'ecoprimers',
+                     help="Input file is the result of an ecoprimers")
+
+    group.add_argument('--tabular-input',
+                     action="store_const", dest="obi:inputformat",
+                     default=None,
+                     const=b'tabular',
+                     help="Input file is a tabular file")
+
+    group.add_argument('--no-skip-on-error',
+                     action="store_false", dest="obi:skiperror",
+                     default=True,
+                     help="Don't skip sequence entries with parsing errors (default: they are skipped)")
+
+    group.add_argument('--no-quality',
+                     action="store_true", dest="obi:noquality",
+                     default=False,
+                     help="Do not import fastQ quality")
+
+    group.add_argument('--quality-sanger',
+                     action="store_const", dest="obi:qualityformat",
+                     default=None,
+                     const=b'sanger',
+                     help="Fastq quality is encoded following sanger format (standard fastq)")
+
+    group.add_argument('--quality-solexa',
+                     action="store_const", dest="obi:qualityformat",
+                     default=None,
+                     const=b'solexa',
+                     help="Fastq quality is encoded following solexa sequencer format")
+
+    group.add_argument('--nuc',
+                     action="store_const", dest="obi:moltype",
+                     default=None,
+                     const=b'nuc',
+                     help="Input file contains nucleic sequences")
+    
+    group.add_argument('--prot',
+                     action="store_const", dest="obi:moltype",
+                     default=None,
+                     const=b'pep',
+                     help="Input file contains protein sequences")
+
+    group.add_argument('--input-na-string',
+                     action="store", dest="obi:inputnastring",
+                     default="NA",
+                     type=str,
+                     help="String associated with Non Available (NA) values in the input")
+
+  
+def __addTabularInputOption(optionManager):
+    group = optionManager.add_argument_group("Input format options for tabular files")
+
+    group.add_argument('--header',
+                     action="store_true", dest="obi:header",
+                     default=False,
+                     help="First line of tabular file contains column names")
+
+    group.add_argument('--sep',
+                     action="store", dest="obi:sep",
+                     default=None,
+                     type=str,
+                     help="Column separator")
+    
+    group.add_argument('--dec',
+                     action="store", dest="obi:dec",
+                     default=".",
+                     type=str,
+                     help="Decimal separator")
+    
+    group.add_argument('--strip-white',
+                     action="store_false", dest="obi:stripwhite",
+                     default=True,
+                     help="Remove white chars at the beginning and the end of values")
+
+    group.add_argument('--blank-line-skip',
+                     action="store_false", dest="obi:blanklineskip",
+                     default=True,
+                     help="Skip empty lines")
+
+    group.add_argument('--comment-char',
+                     action="store", dest="obi:commentchar",
+                     default="#",
+                     type=str,
+                     help="Lines starting by this char are considered as comment")
+
+
+def __addTaxdumpInputOption(optionManager):  # TODO maybe not the best way to do it
+    group = optionManager.add_argument_group("Input format options for taxdump")
+
+    group.add_argument('--taxdump',
+                     action="store_true", dest="obi:taxdump",
+                     default=False,
+                     help="Whether the input is a taxdump")
+
+
+def __addTaxonomyOption(optionManager):
+    group = optionManager.add_argument_group("Input format options for taxonomy")
+
+    group.add_argument('--taxonomy',
+                     action="store", dest="obi:taxoURI",
+                     default=None,
+                     help="Taxonomy URI")
+    
+    #TODO option bool to download taxo if URI doesn't exist
+
+
+def addMinimalInputOption(optionManager):
+    __addInputOption(optionManager)
+
+    
+def addImportInputOption(optionManager):
+    __addInputOption(optionManager)
+    __addImportInputOption(optionManager)
+
+
+def addTabularInputOption(optionManager):
+    __addTabularInputOption(optionManager)
+
+
+def addTaxonomyOption(optionManager):
+    __addTaxonomyOption(optionManager)
+
+
+def addTaxdumpInputOption(optionManager):
+    __addTaxdumpInputOption(optionManager)
+
+
+def addAllInputOption(optionManager):
+    __addInputOption(optionManager)
+    __addImportInputOption(optionManager)
+    __addTabularInputOption(optionManager)
+    __addTaxonomyOption(optionManager)
+    __addTaxdumpInputOption(optionManager)    
+
+
+def __addOutputOption(optionManager):
+    
+    optionManager.add_argument(
+                    dest='obi:outputURI',  
+                    metavar='OUTPUT', 
+                    help='Data destination URI')
+
+
+def __addDMSOutputOption(optionManager):
+    group = optionManager.add_argument_group("Output options for DMS data")
+
+    group.add_argument('--no-create-dms',
+                 action="store_true", dest="obi:nocreatedms",
+                 default=False,
+                 help="Don't create an output DMS is it is not existing")
+
+    group.add_argument('--max-elts',
+                 action="store", dest="obi:maxelts",
+                 metavar='<N>',
+                 default=1000,
+                 type=int,
+                 help="Maximum number of elements per line in a column "
+                      "(e.g. the number of different keys in a dictionary-type "
+                      "key from sequence headers). If the number of different keys "
+                      "is greater than N, the values are stored as character strings")
+
+
+def __addExportOutputOption(optionManager):
+    group = optionManager.add_argument_group("Output format options for exported files")
+
+    group.add_argument('--fasta-output',
+                     action="store_const", dest="obi:outputformat",
+                     default=None,
+                     const=b'fasta',
+                     help="Output file is in sanger fasta format")
+
+    group.add_argument('--fastq-output',
+                     action="store_const", dest="obi:outputformat",
+                     default=None,
+                     const=b'fastq',
+                     help="Output file is in fastq format")
+
+    group.add_argument('--print-na',
+                     action="store_true", dest="obi:printna",
+                     default=False,
+                     help="Print Non Available (NA) values in the output")
+
+    group.add_argument('--output-na-string',
+                     action="store", dest="obi:outputnastring",
+                     default="NA",
+                     type=str,
+                     help="String associated with Non Available (NA) values in the output")
+
+
+def addMinimalOutputOption(optionManager):
+    __addOutputOption(optionManager)
+    __addDMSOutputOption(optionManager)
+
+def addExportOutputOption(optionManager):
+    __addOutputOption(optionManager)
+    __addExportOutputOption(optionManager)
+
+
+def addAllOutputOption(optionManager):
+    __addOutputOption(optionManager)
+    __addDMSOutputOption(optionManager)
+    __addExportOutputOption(optionManager)
+

python/obitools3/apps/progress.cfiles

@@ -0,0 +1,110 @@
+../../../src/obi_lcs.h
+../../../src/obi_lcs.c
+../../../src/obierrno.h
+../../../src/obierrno.c
+../../../src/upperband.h
+../../../src/upperband.c
+../../../src/sse_banded_LCS_alignment.h
+../../../src/sse_banded_LCS_alignment.c
+../../../src/obiblob.h
+../../../src/obiblob.c
+../../../src/utils.h
+../../../src/utils.c
+../../../src/obidms.h
+../../../src/obidms.c
+../../../src/libjson/json_utils.h
+../../../src/libjson/json_utils.c
+../../../src/libjson/cJSON.h
+../../../src/libjson/cJSON.c
+../../../src/obiavl.h
+../../../src/obiavl.c
+../../../src/bloom.h
+../../../src/bloom.c
+../../../src/crc64.h
+../../../src/crc64.c
+../../../src/murmurhash2.h
+../../../src/murmurhash2.c
+../../../src/obidmscolumn.h
+../../../src/obidmscolumn.c
+../../../src/obitypes.h
+../../../src/obitypes.c
+../../../src/obidmscolumndir.h
+../../../src/obidmscolumndir.c
+../../../src/obiblob_indexer.h
+../../../src/obiblob_indexer.c
+../../../src/obiview.h
+../../../src/obiview.c
+../../../src/hashtable.h
+../../../src/hashtable.c
+../../../src/linked_list.h
+../../../src/linked_list.c
+../../../src/obidmscolumn_array.h
+../../../src/obidmscolumn_array.c
+../../../src/obidmscolumn_blob.h
+../../../src/obidmscolumn_blob.c
+../../../src/obidmscolumn_idx.h
+../../../src/obidmscolumn_idx.c
+../../../src/obidmscolumn_bool.h
+../../../src/obidmscolumn_bool.c
+../../../src/obidmscolumn_char.h
+../../../src/obidmscolumn_char.c
+../../../src/obidmscolumn_float.h
+../../../src/obidmscolumn_float.c
+../../../src/obidmscolumn_int.h
+../../../src/obidmscolumn_int.c
+../../../src/obidmscolumn_qual.h
+../../../src/obidmscolumn_qual.c
+../../../src/obidmscolumn_seq.h
+../../../src/obidmscolumn_seq.c
+../../../src/obidmscolumn_str.h
+../../../src/obidmscolumn_str.c
+../../../src/array_indexer.h
+../../../src/array_indexer.c
+../../../src/char_str_indexer.h
+../../../src/char_str_indexer.c
+../../../src/dna_seq_indexer.h
+../../../src/dna_seq_indexer.c
+../../../src/encode.c
+../../../src/encode.h
+../../../src/uint8_indexer.c
+../../../src/uint8_indexer.h
+../../../src/build_reference_db.c
+../../../src/build_reference_db.h
+../../../src/kmer_similarity.c
+../../../src/kmer_similarity.h
+../../../src/obi_clean.c
+../../../src/obi_clean.h
+../../../src/obi_ecopcr.c
+../../../src/obi_ecopcr.h
+../../../src/obi_ecotag.c
+../../../src/obi_ecotag.h
+../../../src/obidms_taxonomy.c
+../../../src/obidms_taxonomy.h
+../../../src/obilittlebigman.c
+../../../src/obilittlebigman.h
+../../../src/_sse.h
+../../../src/obidebug.h
+../../../src/libecoPCR/libapat/CODES/dft_code.h
+../../../src/libecoPCR/libapat/CODES/dna_code.h
+../../../src/libecoPCR/libapat/CODES/prot_code.h
+../../../src/libecoPCR/libapat/apat_parse.c
+../../../src/libecoPCR/libapat/apat_search.c
+../../../src/libecoPCR/libapat/apat.h
+../../../src/libecoPCR/libapat/Gmach.h
+../../../src/libecoPCR/libapat/Gtypes.h
+../../../src/libecoPCR/libapat/libstki.c
+../../../src/libecoPCR/libapat/libstki.h
+../../../src/libecoPCR/libthermo/nnparams.h
+../../../src/libecoPCR/libthermo/nnparams.c
+../../../src/libecoPCR/ecoapat.c
+../../../src/libecoPCR/ecodna.c
+../../../src/libecoPCR/ecoError.c
+../../../src/libecoPCR/ecoMalloc.c
+../../../src/libecoPCR/ecoPCR.h
+
+
+
+
+
+
+

python/obitools3/apps/progress.pxd

@@ -51,6 +51,7 @@ cdef class ProgressBar:
     cdef int lastlog
     cdef bint ontty
     cdef int fd
+    cdef bint cut
     
     cdef bytes _head
     cdef char *chead

python/obitools3/apps/progress.pyx

@@ -6,11 +6,14 @@ Created on 27 mars 2016
 @author: coissac
 '''
 
+
 from ..utils cimport str2bytes, bytes2str
 from .config cimport getConfiguration 
 import sys
 
+
 cdef class ProgressBar:
+    
     cdef clock_t clock(self):
         cdef clock_t t
         cdef timeval tp
@@ -22,11 +25,14 @@ cdef class ProgressBar:
         
         return t
 
+
     def __init__(self,
                  off_t maxi,
                  dict  config={},
                  str head="",
-                 double seconde=0.1):
+                 double seconde=0.1,
+                 cut=False):
+        
         self.starttime = self.clock()
         self.lasttime  = self.starttime
         self.tickcount = <clock_t> (seconde * CLOCKS_PER_SEC)
@@ -37,17 +43,16 @@ cdef class ProgressBar:
         
         if not config:
             config=getConfiguration()
-            
         
         self.ontty = sys.stderr.isatty()
         
         if (maxi<=0):
             maxi=1
             
-        self.maxi = maxi
-        self.head = head
-        self.chead= self._head 
-        
+        self.maxi  = maxi
+        self.head  = head
+        self.chead = self._head 
+        self.cut   = cut
         
         self.logger=config[config["__root_config__"]]["logger"]
         self.wheel =  '|/-\\'
@@ -61,8 +66,9 @@ cdef class ProgressBar:
                     '##########' \
                     '##########' \
                     '##########'  
-                           
-    def __call__(self,object pos):
+                          
+             
+    def __call__(self, object pos, bint force=False):
         cdef off_t    ipos
         cdef clock_t  elapsed
         cdef clock_t  newtime
@@ -76,7 +82,7 @@ cdef class ProgressBar:
         
         self.cycle+=1
     
-        if self.cycle % self.freq == 0:
+        if self.cycle % self.freq == 0 or force:
             self.cycle=1
             newtime  = self.clock()
             delta         = newtime - self.lasttime
@@ -111,7 +117,7 @@ cdef class ProgressBar:
                 self.arrow=(self.arrow+1) % 4
             
                 if days:
-                    <void>fprintf(stderr,b'\r%s %5.1f %% |%.*s%c%.*s] remain : %d days %02d:%02d:%02d',
+                    <void>fprintf(stderr,b'\r%s %5.1f %% |%.*s%c%.*s] remain : %d days %02d:%02d:%02d\033[K',
                                     self.chead,
                                     percent*100,
                                     fraction,self.diese,
@@ -119,7 +125,7 @@ cdef class ProgressBar:
                                     50-fraction,self.spaces,
                                     days,hour,minu,sec)
                 else:
-                    <void>fprintf(stderr,b'\r%s %5.1f %% |%.*s%c%.*s] remain : %02d:%02d:%02d',
+                    <void>fprintf(stderr,b'\r%s %5.1f %% |%.*s%c%.*s] remain : %02d:%02d:%02d\033[K',
                                     self.chead,
                                     percent*100.,
                                     fraction,self.diese,
@@ -127,25 +133,25 @@ cdef class ProgressBar:
                                     50-fraction,self.spaces,
                                     hour,minu,sec)
 
-            tenth = int(percent * 10)
-            if tenth != self.lastlog:
-                
-                if self.ontty:
-                    <void>fputs(b'\n',stderr)
+            if self.cut:
+                tenth = int(percent * 10)
+                if tenth != self.lastlog:
                     
-                self.logger.info('%s %5.1f %% remain : %02d:%02d:%02d' % (
-                                        bytes2str(self._head),
-                                        percent*100.,
-                                        hour,minu,sec))
-                self.lastlog=tenth
+                    if self.ontty:
+                        <void>fputs(b'\n',stderr)
+                    
+                    self.logger.info('%s %5.1f %% remain : %02d:%02d:%02d\033[K' % (
+                                            bytes2str(self._head),
+                                            percent*100.,
+                                            hour,minu,sec))
+                    self.lastlog=tenth
         else:
             self.cycle+=1
 
-    property head:
-    
+
+    property head:    
         def __get__(self):
             return self._head
-        
         def __set__(self,str value):
             self._head=str2bytes(value)
             self.chead=self._head

python/obitools3/apps/temp.cfiles

@@ -0,0 +1,110 @@
+../../../src/obi_lcs.h
+../../../src/obi_lcs.c
+../../../src/obierrno.h
+../../../src/obierrno.c
+../../../src/upperband.h
+../../../src/upperband.c
+../../../src/sse_banded_LCS_alignment.h
+../../../src/sse_banded_LCS_alignment.c
+../../../src/obiblob.h
+../../../src/obiblob.c
+../../../src/utils.h
+../../../src/utils.c
+../../../src/obidms.h
+../../../src/obidms.c
+../../../src/libjson/json_utils.h
+../../../src/libjson/json_utils.c
+../../../src/libjson/cJSON.h
+../../../src/libjson/cJSON.c
+../../../src/obiavl.h
+../../../src/obiavl.c
+../../../src/bloom.h
+../../../src/bloom.c
+../../../src/crc64.h
+../../../src/crc64.c
+../../../src/murmurhash2.h
+../../../src/murmurhash2.c
+../../../src/obidmscolumn.h
+../../../src/obidmscolumn.c
+../../../src/obitypes.h
+../../../src/obitypes.c
+../../../src/obidmscolumndir.h
+../../../src/obidmscolumndir.c
+../../../src/obiblob_indexer.h
+../../../src/obiblob_indexer.c
+../../../src/obiview.h
+../../../src/obiview.c
+../../../src/hashtable.h
+../../../src/hashtable.c
+../../../src/linked_list.h
+../../../src/linked_list.c
+../../../src/obidmscolumn_array.h
+../../../src/obidmscolumn_array.c
+../../../src/obidmscolumn_blob.h
+../../../src/obidmscolumn_blob.c
+../../../src/obidmscolumn_idx.h
+../../../src/obidmscolumn_idx.c
+../../../src/obidmscolumn_bool.h
+../../../src/obidmscolumn_bool.c
+../../../src/obidmscolumn_char.h
+../../../src/obidmscolumn_char.c
+../../../src/obidmscolumn_float.h
+../../../src/obidmscolumn_float.c
+../../../src/obidmscolumn_int.h
+../../../src/obidmscolumn_int.c
+../../../src/obidmscolumn_qual.h
+../../../src/obidmscolumn_qual.c
+../../../src/obidmscolumn_seq.h
+../../../src/obidmscolumn_seq.c
+../../../src/obidmscolumn_str.h
+../../../src/obidmscolumn_str.c
+../../../src/array_indexer.h
+../../../src/array_indexer.c
+../../../src/char_str_indexer.h
+../../../src/char_str_indexer.c
+../../../src/dna_seq_indexer.h
+../../../src/dna_seq_indexer.c
+../../../src/encode.c
+../../../src/encode.h
+../../../src/uint8_indexer.c
+../../../src/uint8_indexer.h
+../../../src/build_reference_db.c
+../../../src/build_reference_db.h
+../../../src/kmer_similarity.c
+../../../src/kmer_similarity.h
+../../../src/obi_clean.c
+../../../src/obi_clean.h
+../../../src/obi_ecopcr.c
+../../../src/obi_ecopcr.h
+../../../src/obi_ecotag.c
+../../../src/obi_ecotag.h
+../../../src/obidms_taxonomy.c
+../../../src/obidms_taxonomy.h
+../../../src/obilittlebigman.c
+../../../src/obilittlebigman.h
+../../../src/_sse.h
+../../../src/obidebug.h
+../../../src/libecoPCR/libapat/CODES/dft_code.h
+../../../src/libecoPCR/libapat/CODES/dna_code.h
+../../../src/libecoPCR/libapat/CODES/prot_code.h
+../../../src/libecoPCR/libapat/apat_parse.c
+../../../src/libecoPCR/libapat/apat_search.c
+../../../src/libecoPCR/libapat/apat.h
+../../../src/libecoPCR/libapat/Gmach.h
+../../../src/libecoPCR/libapat/Gtypes.h
+../../../src/libecoPCR/libapat/libstki.c
+../../../src/libecoPCR/libapat/libstki.h
+../../../src/libecoPCR/libthermo/nnparams.h
+../../../src/libecoPCR/libthermo/nnparams.c
+../../../src/libecoPCR/ecoapat.c
+../../../src/libecoPCR/ecodna.c
+../../../src/libecoPCR/ecoError.c
+../../../src/libecoPCR/ecoMalloc.c
+../../../src/libecoPCR/ecoPCR.h
+
+
+
+
+
+
+

python/obitools3/apps/temp.pxd

@@ -0,0 +1,10 @@
+#cython: language_level=3
+
+'''
+Created on 28 juillet 2017
+
+@author: coissac
+'''
+
+from obitools3.dms.dms cimport DMS
+from obitools3.utils cimport tobytes,tostr

python/obitools3/apps/temp.pyx

@@ -0,0 +1,96 @@
+#cython: language_level=3
+
+'''
+Created on 28 juillet 2017
+
+@author: coissac
+'''
+
+from os import environb,getpid
+from os.path import join, isdir
+from tempfile import TemporaryDirectory, _get_candidate_names
+from shutil import rmtree
+from atexit import register
+
+from obitools3.dms.dms import DMS
+
+from obitools3.apps.config import getConfiguration
+from obitools3.apps.config import logger
+
+cpdef get_temp_dir():
+    """
+    Returns a temporary directory object specific of this instance of obitools.
+    
+    This is an application function. It cannot be called out of an obi command.
+    It requires a valid configuration.
+    
+    If the function is called several time from the same obi session, the same
+    directory is returned.
+    
+    If the OBITMP environment variable exist, the temporary directory is created
+    inside this directory.
+    
+    The directory is automatically destroyed at the end of the end of the process.
+    
+        @return: a temporary python directory object.
+    """
+    cdef bytes tmpdirname
+    cdef dict config = getConfiguration()
+
+    root = config["__root_config__"]
+    
+    try:
+        return config[root]["tempdir"].name
+    except KeyError:
+        pass
+    
+    try:
+        basedir=environb[b'OBITMP']
+    except KeyError:
+        basedir=None
+    
+    tmp = TemporaryDirectory(dir=basedir)
+        
+    config[root]["tempdir"]=tmp
+    
+    return tmp.name
+ 
+cpdef get_temp_dir_name():
+    """
+    Returns the name of the  temporary directory object 
+    specific of this instance of obitools.
+    
+        @return: the name of the temporary directory.
+        
+        @see get_temp_dir
+    """
+    return get_temp_dir_name().name
+          
+               
+cpdef get_temp_dms():
+
+    cdef bytes tmpdirname                   # @DuplicatedSignature
+    cdef dict config = getConfiguration()   # @DuplicatedSignature
+    cdef DMS tmpdms
+    
+    root = config["__root_config__"]
+    
+    try:
+        return config[root]["tempdms"]
+    except KeyError:
+        pass
+
+    tmpdirname=get_temp_dir()
+        
+    tempname = join(tmpdirname,
+                    b"obi.%d.%s" % (getpid(),
+                                    tobytes(next(_get_candidate_names())))
+                   )
+        
+    tmpdms = DMS.new(tempname)
+    
+    config[root]["tempdms"]=tmpdms
+    
+    return tmpdms
+    
+    

python/obitools3/commands/align.cfiles

@@ -1,65 +1,103 @@
-../../../src/bloom.h
-../../../src/bloom.c
-../../../src/char_str_indexer.h
-../../../src/char_str_indexer.c
-../../../src/crc64.h
-../../../src/crc64.c
-../../../src/dna_seq_indexer.h
-../../../src/dna_seq_indexer.c
-../../../src/encode.h
-../../../src/encode.c
-../../../src/hashtable.h
-../../../src/hashtable.c
-../../../src/murmurhash2.h
-../../../src/murmurhash2.c
-../../../src/obi_align.h
-../../../src/obi_align.c
-../../../src/obiavl.h
-../../../src/obiavl.c
-../../../src/obiblob_indexer.h
-../../../src/obiblob_indexer.c
-../../../src/obiblob.h
-../../../src/obiblob.c
-../../../src/obidebug.h
-../../../src/obidms_taxonomy.h
-../../../src/obidms_taxonomy.c
-../../../src/obidms.h
-../../../src/obidms.c
-../../../src/obidmscolumn_blob.c
-../../../src/obidmscolumn_blob.h
-../../../src/obidmscolumn_bool.c
-../../../src/obidmscolumn_bool.h
-../../../src/obidmscolumn_char.c
-../../../src/obidmscolumn_char.h
-../../../src/obidmscolumn_float.c
-../../../src/obidmscolumn_float.h
-../../../src/obidmscolumn_idx.h
-../../../src/obidmscolumn_idx.c
-../../../src/obidmscolumn_int.c
-../../../src/obidmscolumn_int.h
-../../../src/obidmscolumn_qual.h
-../../../src/obidmscolumn_qual.c
-../../../src/obidmscolumn_seq.c
-../../../src/obidmscolumn_seq.h
-../../../src/obidmscolumn_str.c
-../../../src/obidmscolumn_str.h
-../../../src/obidmscolumn.h
-../../../src/obidmscolumn.c
-../../../src/obidmscolumndir.h
-../../../src/obidmscolumndir.c
+../../../src/obi_lcs.h
+../../../src/obi_lcs.c
 ../../../src/obierrno.h
 ../../../src/obierrno.c
-../../../src/obilittlebigman.h
-../../../src/obilittlebigman.c
-../../../src/obitypes.h
-../../../src/obitypes.c
-../../../src/obiview.h
-../../../src/obiview.c
-../../../src/sse_banded_LCS_alignment.h
-../../../src/sse_banded_LCS_alignment.c
-../../../src/uint8_indexer.h
-../../../src/uint8_indexer.c
 ../../../src/upperband.h
 ../../../src/upperband.c
+../../../src/sse_banded_LCS_alignment.h
+../../../src/sse_banded_LCS_alignment.c
+../../../src/obiblob.h
+../../../src/obiblob.c
 ../../../src/utils.h
 ../../../src/utils.c
+../../../src/obidms.h
+../../../src/obidms.c
+../../../src/libjson/json_utils.h
+../../../src/libjson/json_utils.c
+../../../src/libjson/cJSON.h
+../../../src/libjson/cJSON.c
+../../../src/obiavl.h
+../../../src/obiavl.c
+../../../src/bloom.h
+../../../src/bloom.c
+../../../src/crc64.h
+../../../src/crc64.c
+../../../src/murmurhash2.h
+../../../src/murmurhash2.c
+../../../src/obidmscolumn.h
+../../../src/obidmscolumn.c
+../../../src/obitypes.h
+../../../src/obitypes.c
+../../../src/obidmscolumndir.h
+../../../src/obidmscolumndir.c
+../../../src/obiblob_indexer.h
+../../../src/obiblob_indexer.c
+../../../src/obiview.h
+../../../src/obiview.c
+../../../src/hashtable.h
+../../../src/hashtable.c
+../../../src/linked_list.h
+../../../src/linked_list.c
+../../../src/obidmscolumn_array.h
+../../../src/obidmscolumn_array.c
+../../../src/obidmscolumn_blob.h
+../../../src/obidmscolumn_blob.c
+../../../src/obidmscolumn_idx.h
+../../../src/obidmscolumn_idx.c
+../../../src/obidmscolumn_bool.h
+../../../src/obidmscolumn_bool.c
+../../../src/obidmscolumn_char.h
+../../../src/obidmscolumn_char.c
+../../../src/obidmscolumn_float.h
+../../../src/obidmscolumn_float.c
+../../../src/obidmscolumn_int.h
+../../../src/obidmscolumn_int.c
+../../../src/obidmscolumn_qual.h
+../../../src/obidmscolumn_qual.c
+../../../src/obidmscolumn_seq.h
+../../../src/obidmscolumn_seq.c
+../../../src/obidmscolumn_str.h
+../../../src/obidmscolumn_str.c
+../../../src/array_indexer.h
+../../../src/array_indexer.c
+../../../src/char_str_indexer.h
+../../../src/char_str_indexer.c
+../../../src/dna_seq_indexer.h
+../../../src/dna_seq_indexer.c
+../../../src/encode.c
+../../../src/encode.h
+../../../src/uint8_indexer.c
+../../../src/uint8_indexer.h
+../../../src/build_reference_db.c
+../../../src/build_reference_db.h
+../../../src/kmer_similarity.c
+../../../src/kmer_similarity.h
+../../../src/obi_clean.c
+../../../src/obi_clean.h
+../../../src/obi_ecopcr.c
+../../../src/obi_ecopcr.h
+../../../src/obi_ecotag.c
+../../../src/obi_ecotag.h
+../../../src/obidms_taxonomy.c
+../../../src/obidms_taxonomy.h
+../../../src/obilittlebigman.c
+../../../src/obilittlebigman.h
+../../../src/_sse.h
+../../../src/obidebug.h
+../../../src/libecoPCR/libapat/CODES/dft_code.h
+../../../src/libecoPCR/libapat/CODES/dna_code.h
+../../../src/libecoPCR/libapat/CODES/prot_code.h
+../../../src/libecoPCR/libapat/apat_parse.c
+../../../src/libecoPCR/libapat/apat_search.c
+../../../src/libecoPCR/libapat/apat.h
+../../../src/libecoPCR/libapat/Gmach.h
+../../../src/libecoPCR/libapat/Gtypes.h
+../../../src/libecoPCR/libapat/libstki.c
+../../../src/libecoPCR/libapat/libstki.h
+../../../src/libecoPCR/libthermo/nnparams.h
+../../../src/libecoPCR/libthermo/nnparams.c
+../../../src/libecoPCR/ecoapat.c
+../../../src/libecoPCR/ecodna.c
+../../../src/libecoPCR/ecoError.c
+../../../src/libecoPCR/ecoMalloc.c
+../../../src/libecoPCR/ecoPCR.h

python/obitools3/commands/align.pxd

@@ -0,0 +1,18 @@
+#cython: language_level=3
+
+
+cpdef align_columns(bytes dms_n, 
+                    bytes input_view_1_n, 
+                    bytes output_view_n,
+                    bytes input_view_2_n=*,
+                    bytes input_column_1_n=*, 
+                    bytes input_column_2_n=*,
+                    bytes input_elt_1_n=*, 
+                    bytes input_elt_2_n=*,
+                    bytes id_column_1_n=*, 
+                    bytes id_column_2_n=*,
+                    double threshold=*, bint normalize=*, 
+                    int reference=*, bint similarity_mode=*,
+                    bint print_seq=*, bint print_count=*,
+                    bytes comments=*,
+                    int thread_count=*)

python/obitools3/commands/align.pyx

@@ -0,0 +1,274 @@
+#cython: language_level=3
+
+from obitools3.apps.progress cimport ProgressBar  # @UnresolvedImport
+from obitools3.dms import DMS
+from obitools3.dms.view.view cimport View
+from obitools3.uri.decode import open_uri
+from obitools3.apps.optiongroups import addMinimalInputOption, addMinimalOutputOption
+from obitools3.dms.view import RollbackException
+from obitools3.apps.config import logger
+from obitools3.utils cimport tobytes, str2bytes
+
+from obitools3.dms.capi.obilcsalign cimport obi_lcs_align_one_column, \
+                                            obi_lcs_align_two_columns
+
+import time
+import sys
+
+
+__title__="Aligns one sequence column with itself or two sequence columns"
+
+
+def addOptions(parser):
+   
+   addMinimalInputOption(parser)
+   addMinimalOutputOption(parser)
+
+   group=parser.add_argument_group('obi align specific options')
+
+   group.add_argument('--input-2', '-I',
+                      action="store", dest="align:inputuri2",
+                      metavar='<INPUT URI>',
+                      default="",
+                      type=str,
+                      help="Eventually, the URI of the second input to align with the first one.")
+      
+   group.add_argument('--threshold','-t',
+                      action="store", dest="align:threshold",
+                      metavar='<THRESHOLD>',
+                      default=0.0,
+                      type=float,
+                      help="Score threshold. If the score is normalized and expressed in similarity (default),"
+                           " it is an identity, e.g. 0.95 for an identity of 95%%. If the score is normalized"
+                           " and expressed in distance, it is (1.0 - identity), e.g. 0.05 for an identity of 95%%."
+                           " If the score is not normalized and expressed in similarity, it is the length of the"
+                           " Longest Common Subsequence. If the score is not normalized and expressed in distance,"
+                           " it is (reference length - LCS length)."
+                           " Only sequence pairs with a similarity above <THRESHOLD> are printed. Default: 0.00"
+                           " (no threshold).")
+   
+   group.add_argument('--longest-length','-L',
+                      action="store_const", dest="align:reflength",
+                      default=0,
+                      const=1,
+                      help="The reference length is the length of the longest sequence."
+                           " Default: the reference length is the length of the alignment.")
+   
+   group.add_argument('--shortest-length','-l',
+                      action="store_const", dest="align:reflength",
+                      default=0,
+                      const=2,
+                      help="The reference length is the length of the shortest sequence."
+                           " Default: the reference length is the length of the alignment.")
+   
+   group.add_argument('--raw','-r',
+                      action="store_false", dest="align:normalize",
+                      default=True,
+                      help="Raw score, not normalized. Default: score is normalized with the reference sequence length.")
+   
+   group.add_argument('--distance','-D',
+                      action="store_false", dest="align:similarity",
+                      default=True,
+                      help="Score is expressed in distance. Default: score is expressed in similarity.")
+   
+   group.add_argument('--print-seq','-s',
+                      action="store_true", dest="align:printseq",
+                      default=False,
+                      help="The nucleotide sequences are written in the output view. Default: they are not written.")
+   
+   group.add_argument('--print-count','-n',
+                      action="store_true", dest="align:printcount",
+                      default=False,
+                      help="Sequence counts are written in the output view. Default: they are not written.")
+   
+   group.add_argument('--thread-count','-p',   # TODO should probably be in a specific option group
+                      action="store", dest="align:threadcount",
+                      metavar='<THREAD COUNT>',
+                      default=1,
+                      type=int,
+                      help="Number of threads to use for the computation. Default: one.")
+
+
+cpdef align_columns(bytes dms_n, 
+                    bytes input_view_1_n, 
+                    bytes output_view_n,
+                    bytes input_view_2_n=b"",
+                    bytes input_column_1_n=b"", 
+                    bytes input_column_2_n=b"",
+                    bytes input_elt_1_n=b"", 
+                    bytes input_elt_2_n=b"",
+                    bytes id_column_1_n=b"", 
+                    bytes id_column_2_n=b"",
+                    double threshold=0.0, bint normalize=True, 
+                    int reference=0, bint similarity_mode=True,
+                    bint print_seq=False, bint print_count=False,
+                    bytes comments=b"{}",
+                    int thread_count=1) : 
+
+    if input_view_2_n == b"" and input_column_2_n == b"" :
+        if obi_lcs_align_one_column(dms_n, \
+                                    input_view_1_n, \
+                                    input_column_1_n, \
+                                    input_elt_1_n, \
+                                    id_column_1_n, \
+                                    output_view_n, \
+                                    comments, \
+                                    print_seq, \
+                                    print_count, \
+                                    threshold, normalize, reference, similarity_mode,
+                                    thread_count) < 0 :
+            raise Exception("Error aligning sequences")        
+            
+    else:
+        if obi_lcs_align_two_columns(dms_n, \
+                                     input_view_1_n, \
+                                     input_view_2_n, \
+                                     input_column_1_n, \
+                                     input_column_2_n, \
+                                     input_elt_1_n, \
+                                     input_elt_2_n, \
+                                     id_column_1_n, \
+                                     id_column_2_n, \
+                                     output_view_n, \
+                                     comments, \
+                                     print_seq, \
+                                     print_count, \
+                                     threshold, normalize, reference, similarity_mode) < 0 :
+            raise Exception("Error aligning sequences")        
+ 
+ 
+def run(config):
+    
+    DMS.obi_atexit()
+    
+    logger("info", "obi align")
+
+    # Open the input: only the DMS
+    input = open_uri(config['obi']['inputURI'],
+                     dms_only=True)
+    if input is None:
+        raise Exception("Could not read input")
+    i_dms = input[0]
+    i_dms_name = input[0].name
+    i_uri = input[1]
+    i_view_name = i_uri.split(b"/")[0]
+    i_column_name = b""
+    i_element_name = b""
+    if len(i_uri.split(b"/")) == 2:
+        i_column_name = i_uri.split(b"/")[1]
+    if len(i_uri.split(b"/")) == 3:
+        i_element_name = i_uri.split(b"/")[2]
+    if len(i_uri.split(b"/")) > 3:
+        raise Exception("Input URI contains too many elements:", config['obi']['inputURI'])
+
+    # Open the second input if there is one
+    i_dms_2 = None
+    i_dms_name_2 = b""
+    original_i_view_name_2 = b""
+    i_view_name_2 = b""
+    i_column_name_2 = b""
+    i_element_name_2 = b""
+    if config['align']['inputuri2']:
+        input_2 = open_uri(config['align']['inputuri2'],
+                           dms_only=True)
+        if input_2 is None:
+            raise Exception("Could not read second input")
+        i_dms_2 = input_2[0]
+        i_dms_name_2 = i_dms_2.name
+        i_uri_2 = input_2[1]
+        original_i_view_name_2 = i_uri_2.split(b"/")[0]
+        if len(i_uri_2.split(b"/")) == 2:
+            i_column_name_2 = i_uri_2.split(b"/")[1]
+        if len(i_uri_2.split(b"/")) == 3:
+            i_element_name_2 = i_uri_2.split(b"/")[2]
+        if len(i_uri_2.split(b"/")) > 3:
+            raise Exception("Input URI contains too many elements:", config['align']['inputuri2'])
+
+        # If the 2 input DMS are not the same, temporarily import 2nd input view in first input DMS
+        if i_dms != i_dms_2:
+            temp_i_view_name_2 = original_i_view_name_2
+            i=0
+            while temp_i_view_name_2 in i_dms:  # Making sure view name is unique in input DMS
+                temp_i_view_name_2 = original_i_view_name_2+b"_"+str2bytes(str(i))
+                i+=1
+            i_view_name_2 = temp_i_view_name_2
+            View.import_view(i_dms_2.full_path[:-7], i_dms.full_path[:-7], original_i_view_name_2, i_view_name_2)
+
+    # Open the output: only the DMS
+    output = open_uri(config['obi']['outputURI'],
+                      input=False,
+                      dms_only=True)
+    if output is None:
+        raise Exception("Could not create output")
+    o_dms = output[0]
+    o_dms_name = o_dms.name 
+    final_o_view_name = output[1]
+
+    # If the input and output DMS are not the same, align creating a temporary view in the input dms that will be exported to 
+    # the right DMS and deleted in the other afterwards.
+    if i_dms != o_dms:
+        temporary_view_name = final_o_view_name
+        i=0
+        while temporary_view_name in i_dms:  # Making sure view name is unique in input DMS
+            temporary_view_name = final_o_view_name+b"_"+str2bytes(str(i))
+            i+=1
+        o_view_name = temporary_view_name
+    else:
+        o_view_name = final_o_view_name
+        
+    # Save command config in View comments
+    command_line = " ".join(sys.argv[1:])
+    
+    i_dms_list = [i_dms_name]
+    if i_dms_name_2:
+        i_dms_list.append(i_dms_name_2)
+    i_view_list = [i_view_name]
+    if original_i_view_name_2:
+        i_view_list.append(original_i_view_name_2)
+    comments = View.print_config(config, "align", command_line, input_dms_name=i_dms_list, input_view_name=i_view_list)
+    
+    # Call cython alignment function
+      # Using default ID columns of the view. TODO discuss adding option
+    align_columns(i_dms_name,  \
+                  i_view_name,  \
+                  o_view_name,  \
+                  input_view_2_n   = i_view_name_2,  \
+                  input_column_1_n = i_column_name,  \
+                  input_column_2_n = i_column_name_2, \
+                  input_elt_1_n    = i_element_name,  \
+                  input_elt_2_n    = i_element_name_2, \
+                  id_column_1_n    = b"",  \
+                  id_column_2_n    = b"", \
+                  threshold        = config['align']['threshold'], \
+                  normalize        = config['align']['normalize'],  \
+                  reference        = config['align']['reflength'],  \
+                  similarity_mode  = config['align']['similarity'],  \
+                  print_seq        = config['align']['printseq'],  \
+                  print_count      = config['align']['printcount'], \
+                  comments         = comments, \
+                  thread_count     = config['align']['threadcount'])
+
+    # If the input and output DMS are not the same, export result view to output DMS
+    if i_dms != o_dms:
+        View.import_view(i_dms.full_path[:-7], o_dms.full_path[:-7], o_view_name, final_o_view_name)
+
+    # Save command config in output DMS comments
+    o_dms.record_command_line(command_line)
+
+    #print("\n\nOutput view:\n````````````", file=sys.stderr)
+    #print(repr(o_dms[final_o_view_name]), file=sys.stderr)
+
+    # If the two input DMS are different, delete the temporary input view in the first input DMS
+    if i_dms_2 and i_dms != i_dms_2:
+        View.delete_view(i_dms, i_view_name_2)
+        i_dms_2.close()
+
+    # If the input and the output DMS are different, delete the temporary result view in the input DMS
+    if i_dms != o_dms:
+        View.delete_view(i_dms, o_view_name)
+        o_dms.close()
+    
+    i_dms.close()
+    
+    logger("info", "Done.")
+  

python/obitools3/commands/alignpairedend.cfiles

@@ -0,0 +1,103 @@
+../../../src/obi_lcs.h
+../../../src/obi_lcs.c
+../../../src/obierrno.h
+../../../src/obierrno.c
+../../../src/upperband.h
+../../../src/upperband.c
+../../../src/sse_banded_LCS_alignment.h
+../../../src/sse_banded_LCS_alignment.c
+../../../src/obiblob.h
+../../../src/obiblob.c
+../../../src/utils.h
+../../../src/utils.c
+../../../src/obidms.h
+../../../src/obidms.c
+../../../src/libjson/json_utils.h
+../../../src/libjson/json_utils.c
+../../../src/libjson/cJSON.h
+../../../src/libjson/cJSON.c
+../../../src/obiavl.h
+../../../src/obiavl.c
+../../../src/bloom.h
+../../../src/bloom.c
+../../../src/crc64.h
+../../../src/crc64.c
+../../../src/murmurhash2.h
+../../../src/murmurhash2.c
+../../../src/obidmscolumn.h
+../../../src/obidmscolumn.c
+../../../src/obitypes.h
+../../../src/obitypes.c
+../../../src/obidmscolumndir.h
+../../../src/obidmscolumndir.c
+../../../src/obiblob_indexer.h
+../../../src/obiblob_indexer.c
+../../../src/obiview.h
+../../../src/obiview.c
+../../../src/hashtable.h
+../../../src/hashtable.c
+../../../src/linked_list.h
+../../../src/linked_list.c
+../../../src/obidmscolumn_array.h
+../../../src/obidmscolumn_array.c
+../../../src/obidmscolumn_blob.h
+../../../src/obidmscolumn_blob.c
+../../../src/obidmscolumn_idx.h
+../../../src/obidmscolumn_idx.c
+../../../src/obidmscolumn_bool.h
+../../../src/obidmscolumn_bool.c
+../../../src/obidmscolumn_char.h
+../../../src/obidmscolumn_char.c
+../../../src/obidmscolumn_float.h
+../../../src/obidmscolumn_float.c
+../../../src/obidmscolumn_int.h
+../../../src/obidmscolumn_int.c
+../../../src/obidmscolumn_qual.h
+../../../src/obidmscolumn_qual.c
+../../../src/obidmscolumn_seq.h
+../../../src/obidmscolumn_seq.c
+../../../src/obidmscolumn_str.h
+../../../src/obidmscolumn_str.c
+../../../src/array_indexer.h
+../../../src/array_indexer.c
+../../../src/char_str_indexer.h
+../../../src/char_str_indexer.c
+../../../src/dna_seq_indexer.h
+../../../src/dna_seq_indexer.c
+../../../src/encode.c
+../../../src/encode.h
+../../../src/uint8_indexer.c
+../../../src/uint8_indexer.h
+../../../src/build_reference_db.c
+../../../src/build_reference_db.h
+../../../src/kmer_similarity.c
+../../../src/kmer_similarity.h
+../../../src/obi_clean.c
+../../../src/obi_clean.h
+../../../src/obi_ecopcr.c
+../../../src/obi_ecopcr.h
+../../../src/obi_ecotag.c
+../../../src/obi_ecotag.h
+../../../src/obidms_taxonomy.c
+../../../src/obidms_taxonomy.h
+../../../src/obilittlebigman.c
+../../../src/obilittlebigman.h
+../../../src/_sse.h
+../../../src/obidebug.h
+../../../src/libecoPCR/libapat/CODES/dft_code.h
+../../../src/libecoPCR/libapat/CODES/dna_code.h
+../../../src/libecoPCR/libapat/CODES/prot_code.h
+../../../src/libecoPCR/libapat/apat_parse.c
+../../../src/libecoPCR/libapat/apat_search.c
+../../../src/libecoPCR/libapat/apat.h
+../../../src/libecoPCR/libapat/Gmach.h
+../../../src/libecoPCR/libapat/Gtypes.h
+../../../src/libecoPCR/libapat/libstki.c
+../../../src/libecoPCR/libapat/libstki.h
+../../../src/libecoPCR/libthermo/nnparams.h
+../../../src/libecoPCR/libthermo/nnparams.c
+../../../src/libecoPCR/ecoapat.c
+../../../src/libecoPCR/ecodna.c
+../../../src/libecoPCR/ecoError.c
+../../../src/libecoPCR/ecoMalloc.c
+../../../src/libecoPCR/ecoPCR.h

python/obitools3/commands/alignpairedend.pxd

@@ -0,0 +1,4 @@
+#cython: language_level=3
+
+
+cdef object buildAlignment(object direct, object reverse)

python/obitools3/commands/alignpairedend.pyx

@@ -0,0 +1,249 @@
+#cython: language_level=3
+
+from obitools3.apps.progress cimport ProgressBar  # @UnresolvedImport
+from obitools3.dms import DMS
+from obitools3.dms.view import RollbackException
+from obitools3.dms.view.typed_view.view_NUC_SEQS cimport View_NUC_SEQS
+from obitools3.dms.column.column cimport Column
+from obitools3.dms.capi.obiview cimport QUALITY_COLUMN
+from obitools3.dms.capi.obitypes cimport OBI_QUAL
+from obitools3.apps.optiongroups import addMinimalInputOption, addMinimalOutputOption
+from obitools3.uri.decode import open_uri
+from obitools3.apps.config import logger
+from obitools3.libalign._qsassemble import QSolexaReverseAssemble
+from obitools3.libalign._qsrassemble import QSolexaRightReverseAssemble
+from obitools3.libalign._solexapairend import buildConsensus, buildJoinedSequence
+from obitools3.dms.obiseq cimport Nuc_Seq
+from obitools3.libalign.shifted_ali cimport Kmer_similarity, Ali_shifted
+from obitools3.commands.ngsfilter import REVERSE_SEQ_COLUMN_NAME, REVERSE_QUALITY_COLUMN_NAME
+
+import sys
+import os
+
+
+__title__="Aligns paired-ended reads"
+
+
+
+def addOptions(parser):
+
+    addMinimalInputOption(parser)
+    addMinimalOutputOption(parser)
+ 
+    group = parser.add_argument_group('obi alignpairedend specific options')
+
+    group.add_argument('-R', '--reverse-reads',
+                     action="store", dest="alignpairedend:reverse",
+                     metavar="<URI>",
+                     default=None,
+                     type=str,
+                     help="URI to the reverse reads if they are in a different view than the forward reads")
+
+    group.add_argument('--score-min',
+                     action="store", dest="alignpairedend:smin",
+                     metavar="#.###",
+                     default=None,
+                     type=float,
+                     help="Minimum score for keeping alignments")
+
+    group.add_argument('-A', '--true-ali',
+                       action="store_true", dest="alignpairedend:trueali",
+                       default=False,
+                       help="Performs gap free end alignment of sequences instead of using kmers to compute alignments (slower).")
+
+    group.add_argument('-k', '--kmer-size',
+                       action="store", dest="alignpairedend:kmersize",
+                       metavar="#",
+                       default=3,
+                       type=int,
+                       help="K-mer size for kmer comparisons, between 1 and 4 (not when using -A option; default: 3)")
+
+
+la = QSolexaReverseAssemble()
+ra = QSolexaRightReverseAssemble()
+cdef object buildAlignment(object direct, object reverse):
+    
+    if len(direct)==0 or len(reverse)==0:
+        return None
+    
+    la.seqA = direct
+    la.seqB = reverse
+    
+    ali=la()
+    ali.direction='left'
+
+    ra.seqA = direct
+    ra.seqB = reverse
+
+    rali=ra()
+    rali.direction='right'
+     
+    if ali.score < rali.score:
+        ali = rali
+    
+    return ali
+    
+
+def alignmentIterator(entries, aligner): 
+    
+    if type(entries) == list:
+        two_views = True
+        forward = entries[0]
+        reverse = entries[1]
+        entries_len = len(forward)
+    else:
+        two_views = False
+        entries_len = len(entries)
+    
+    for i in range(entries_len):
+        if two_views:
+            seqF = forward[i]
+            seqR = reverse[i]
+        else:
+            seqF = Nuc_Seq.new_from_stored(entries[i])
+            seqR = Nuc_Seq(seqF.id, seqF[REVERSE_SEQ_COLUMN_NAME], quality=seqF[REVERSE_QUALITY_COLUMN_NAME])
+            seqR.index = i
+        
+        ali = aligner(seqF, seqR)
+        
+        if ali is None:
+            continue
+        
+        yield ali
+
+
+def run(config):
+        
+    DMS.obi_atexit()
+    
+    logger("info", "obi alignpairedend")
+       
+    # Open the input
+    
+    two_views = False
+    forward = None
+    reverse = None
+    input = None
+    
+    input = open_uri(config['obi']['inputURI'])
+    if input is None:
+        raise Exception("Could not open input reads")
+    if input[2] != View_NUC_SEQS:
+        raise NotImplementedError('obi alignpairedend only works on NUC_SEQS views')    
+    
+    if "reverse" in config["alignpairedend"]:
+        
+        two_views = True
+        
+        forward = input[1]        
+        
+        rinput = open_uri(config["alignpairedend"]["reverse"])
+        if rinput is None:
+            raise Exception("Could not open reverse reads")
+        if rinput[2] != View_NUC_SEQS:
+            raise NotImplementedError('obi alignpairedend only works on NUC_SEQS views')
+        
+        reverse = rinput[1]
+    
+        if len(forward) != len(reverse):
+            raise Exception("Error: the number of forward and reverse reads are different")
+        
+        entries = [forward, reverse]
+        input_dms_name = [forward.dms.name, reverse.dms.name]
+        input_view_name = [forward.name, reverse.name]
+    
+    else:
+        entries = input[1]
+        input_dms_name = [entries.dms.name]
+        input_view_name = [entries.name]
+
+    if two_views:
+        entries_len = len(forward)
+    else:
+        entries_len = len(entries)
+    
+    # Open the output
+    output = open_uri(config['obi']['outputURI'],
+                      input=False,
+                      newviewtype=View_NUC_SEQS)
+    if output is None:
+        raise Exception("Could not create output view")
+
+    view = output[1]
+    
+    Column.new_column(view, QUALITY_COLUMN, OBI_QUAL)   #TODO output URI quality option?
+
+    if 'smin' in config['alignpairedend']:
+        smin = config['alignpairedend']['smin']
+    else:
+        smin = 0
+
+    # Initialize the progress bar
+    pb = ProgressBar(entries_len, config, seconde=5)
+    
+    if config['alignpairedend']['trueali']:
+        kmer_ali = False
+        aligner = buildAlignment
+    else :
+        kmer_ali = True
+        if type(entries) == list:
+            forward = entries[0]
+            reverse = entries[1]
+            aligner = Kmer_similarity(forward, view2=reverse, kmer_size=config['alignpairedend']['kmersize'])
+        else:
+            aligner = Kmer_similarity(entries, column2=entries[REVERSE_SEQ_COLUMN_NAME], qual_column2=entries[REVERSE_QUALITY_COLUMN_NAME], kmer_size=config['alignpairedend']['kmersize'])
+        
+    ba = alignmentIterator(entries, aligner)
+
+    i = 0
+    for ali in ba:
+        
+        pb(i)
+        
+        consensus = view[i]
+        
+        if not two_views:
+            seqF = entries[i]
+        else:
+            seqF = forward[i]
+                
+        if smin > 0:
+            if (ali.score > smin) :
+                buildConsensus(ali, consensus, seqF)
+            else:
+                if not two_views:
+                    seqR = Nuc_Seq(seqF.id, seqF[REVERSE_SEQ_COLUMN_NAME], quality = seqF[REVERSE_QUALITY_COLUMN_NAME])
+                else:
+                    seqR = reverse[i]
+                buildJoinedSequence(ali, seqR, consensus, forward=seqF)
+                
+            consensus[b"smin"] = smin
+        else:
+            buildConsensus(ali, consensus, seqF)
+                
+        if kmer_ali :
+            ali.free()
+    
+        i+=1
+
+    pb(i, force=True)
+    print("", file=sys.stderr)
+
+    if kmer_ali :
+        aligner.free()
+
+    # Save command config in View and DMS comments
+    command_line = " ".join(sys.argv[1:])
+    view.write_config(config, "alignpairedend", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name)
+    output[0].record_command_line(command_line)
+
+    #print("\n\nOutput view:\n````````````", file=sys.stderr)
+    #print(repr(view), file=sys.stderr)
+    
+    input[0].close()
+    if two_views:
+        rinput[0].close()
+    output[0].close()
+
+    logger("info", "Done.")
+    

python/obitools3/commands/annotate.cfiles

@@ -0,0 +1,103 @@
+../../../src/obi_lcs.h
+../../../src/obi_lcs.c
+../../../src/obierrno.h
+../../../src/obierrno.c
+../../../src/upperband.h
+../../../src/upperband.c
+../../../src/sse_banded_LCS_alignment.h
+../../../src/sse_banded_LCS_alignment.c
+../../../src/obiblob.h
+../../../src/obiblob.c
+../../../src/utils.h
+../../../src/utils.c
+../../../src/obidms.h
+../../../src/obidms.c
+../../../src/libjson/json_utils.h
+../../../src/libjson/json_utils.c
+../../../src/libjson/cJSON.h
+../../../src/libjson/cJSON.c
+../../../src/obiavl.h
+../../../src/obiavl.c
+../../../src/bloom.h
+../../../src/bloom.c
+../../../src/crc64.h
+../../../src/crc64.c
+../../../src/murmurhash2.h
+../../../src/murmurhash2.c
+../../../src/obidmscolumn.h
+../../../src/obidmscolumn.c
+../../../src/obitypes.h
+../../../src/obitypes.c
+../../../src/obidmscolumndir.h
+../../../src/obidmscolumndir.c
+../../../src/obiblob_indexer.h
+../../../src/obiblob_indexer.c
+../../../src/obiview.h
+../../../src/obiview.c
+../../../src/hashtable.h
+../../../src/hashtable.c
+../../../src/linked_list.h
+../../../src/linked_list.c
+../../../src/obidmscolumn_array.h
+../../../src/obidmscolumn_array.c
+../../../src/obidmscolumn_blob.h
+../../../src/obidmscolumn_blob.c
+../../../src/obidmscolumn_idx.h
+../../../src/obidmscolumn_idx.c
+../../../src/obidmscolumn_bool.h
+../../../src/obidmscolumn_bool.c
+../../../src/obidmscolumn_char.h
+../../../src/obidmscolumn_char.c
+../../../src/obidmscolumn_float.h
+../../../src/obidmscolumn_float.c
+../../../src/obidmscolumn_int.h
+../../../src/obidmscolumn_int.c
+../../../src/obidmscolumn_qual.h
+../../../src/obidmscolumn_qual.c
+../../../src/obidmscolumn_seq.h
+../../../src/obidmscolumn_seq.c
+../../../src/obidmscolumn_str.h
+../../../src/obidmscolumn_str.c
+../../../src/array_indexer.h
+../../../src/array_indexer.c
+../../../src/char_str_indexer.h
+../../../src/char_str_indexer.c
+../../../src/dna_seq_indexer.h
+../../../src/dna_seq_indexer.c
+../../../src/encode.c
+../../../src/encode.h
+../../../src/uint8_indexer.c
+../../../src/uint8_indexer.h
+../../../src/build_reference_db.c
+../../../src/build_reference_db.h
+../../../src/kmer_similarity.c
+../../../src/kmer_similarity.h
+../../../src/obi_clean.c
+../../../src/obi_clean.h
+../../../src/obi_ecopcr.c
+../../../src/obi_ecopcr.h
+../../../src/obi_ecotag.c
+../../../src/obi_ecotag.h
+../../../src/obidms_taxonomy.c
+../../../src/obidms_taxonomy.h
+../../../src/obilittlebigman.c
+../../../src/obilittlebigman.h
+../../../src/_sse.h
+../../../src/obidebug.h
+../../../src/libecoPCR/libapat/CODES/dft_code.h
+../../../src/libecoPCR/libapat/CODES/dna_code.h
+../../../src/libecoPCR/libapat/CODES/prot_code.h
+../../../src/libecoPCR/libapat/apat_parse.c
+../../../src/libecoPCR/libapat/apat_search.c
+../../../src/libecoPCR/libapat/apat.h
+../../../src/libecoPCR/libapat/Gmach.h
+../../../src/libecoPCR/libapat/Gtypes.h
+../../../src/libecoPCR/libapat/libstki.c
+../../../src/libecoPCR/libapat/libstki.h
+../../../src/libecoPCR/libthermo/nnparams.h
+../../../src/libecoPCR/libthermo/nnparams.c
+../../../src/libecoPCR/ecoapat.c
+../../../src/libecoPCR/ecodna.c
+../../../src/libecoPCR/ecoError.c
+../../../src/libecoPCR/ecoMalloc.c
+../../../src/libecoPCR/ecoPCR.h

python/obitools3/commands/annotate.pyx

@@ -0,0 +1,382 @@
+#cython: language_level=3
+
+from obitools3.apps.progress cimport ProgressBar  # @UnresolvedImport
+from obitools3.dms import DMS
+from obitools3.dms.view.view cimport View, Line_selection
+from obitools3.uri.decode import open_uri
+from obitools3.apps.optiongroups import addMinimalInputOption, addTaxonomyOption, addMinimalOutputOption
+from obitools3.dms.view import RollbackException
+from functools import reduce
+from obitools3.apps.config import logger
+from obitools3.utils cimport tobytes, str2bytes
+from obitools3.dms.capi.obiview cimport NUC_SEQUENCE_COLUMN, \
+                                        ID_COLUMN, \
+                                        DEFINITION_COLUMN, \
+                                        QUALITY_COLUMN, \
+                                        COUNT_COLUMN
+
+import time
+import math 
+import sys
+
+ 
+__title__="Annotate views with new tags and edit existing annotations"
+
+
+SPECIAL_COLUMNS = [NUC_SEQUENCE_COLUMN, ID_COLUMN, DEFINITION_COLUMN, QUALITY_COLUMN]
+
+ 
+def addOptions(parser):
+    
+    addMinimalInputOption(parser)
+    addTaxonomyOption(parser)
+    addMinimalOutputOption(parser)
+
+    group=parser.add_argument_group('obi annotate specific options')
+ 
+    group.add_argument('--seq-rank',   # TODO seq/elt/line???
+                       action="store_true", 
+                       dest="annotate:add_rank",
+                       default=False,
+                       help="Add a rank attribute to the sequence "
+                            "indicating the sequence position in the data.")
+ 
+    group.add_argument('-R', '--rename-tag',
+                       action="append", 
+                       dest="annotate:rename_tags",
+                       metavar="<OLD_NAME:NEW_NAME>",
+                       type=str,
+                       default=[],
+                       help="Change tag name from OLD_NAME to NEW_NAME.")
+
+    group.add_argument('-D', '--delete-tag',
+                       action="append", 
+                       dest="annotate:delete_tags",
+                       metavar="<TAG_NAME>",
+                       type=str,
+                       default=[],
+                       help="Delete tag TAG_NAME.")
+
+    group.add_argument('-S', '--set-tag',
+                       action="append", 
+                       dest="annotate:set_tags",
+                       metavar="<TAG_NAME:PYTHON_EXPRESSION>",
+                       type=str,
+                       default=[],
+                       help="Add a new tag named TAG_NAME with "
+                            "a value computed from PYTHON_EXPRESSION.")
+
+    group.add_argument('--set-identifier',
+                       action="store", 
+                       dest="annotate:set_identifier",
+                       metavar="<PYTHON_EXPRESSION>",
+                       type=str,
+                       default=None,
+                       help="Set sequence identifier with "
+                            "a value computed from PYTHON_EXPRESSION.")
+    
+    group.add_argument('--set-sequence',
+                       action="store", 
+                       dest="annotate:set_sequence",
+                       metavar="<PYTHON_EXPRESSION>",
+                       type=str,
+                       default=None,
+                       help="Change the sequence itself with "
+                            "a value computed from PYTHON_EXPRESSION.")
+
+    group.add_argument('--set-definition',
+                       action="store", 
+                       dest="annotate:set_definition",
+                       metavar="<PYTHON_EXPRESSION>",
+                       type=str,
+                       default=None,
+                       help="Set sequence definition with "
+                            "a value computed from PYTHON_EXPRESSION.")
+
+    group.add_argument('--run',
+                       action="store", 
+                       dest="annotate:run",
+                       metavar="<PYTHON_EXPRESSION>",
+                       type=str,
+                       default=None,
+                       help="Run a python expression on each element.")
+
+    group.add_argument('-C', '--clear',
+                       action="store_true", 
+                       dest="annotate:clear",
+                       default=False,
+                       help="Clear all tags except the obligatory ones.")
+    
+    group.add_argument('-k','--keep',
+                       action='append',
+                       dest="annotate:keep",
+                       metavar="<TAG>",
+                       default=[],
+                       type=str,
+                       help="Only keep this tag. (Can be specified several times.)")
+    
+    group.add_argument('--length',
+                       action="store_true", 
+                       dest="annotate:length",
+                       default=False,
+                       help="Add 'seq_length' tag with sequence length.")
+    
+    group.add_argument('--with-taxon-at-rank',
+                       action='append',
+                       dest="annotate:taxon_at_rank",
+                       metavar="<RANK_NAME>",
+                       default=[],
+                       type=str,
+                       help="Add taxonomy annotation at the specified rank level RANK_NAME.")
+
+
+def sequenceTaggerGenerator(config, taxo=None):
+    
+    toSet=None
+    newId=None
+    newDef=None
+    newSeq=None
+    length=None
+    add_rank=None
+    run=None
+    
+    if 'set_tags' in config['annotate']:   # TODO default option problem, to fix
+        toSet = [x.split(':',1) for x in config['annotate']['set_tags'] if len(x.split(':',1))==2]
+    if 'set_identifier' in config['annotate']:
+        newId = config['annotate']['set_identifier']
+    if 'set_definition' in config['annotate']:
+        newDef = config['annotate']['set_definition']
+    if 'set_sequence' in config['annotate']:
+        newSeq = config['annotate']['set_sequence']
+    if 'length' in config['annotate']:
+        length = config['annotate']['length']
+    if 'add_rank' in config["annotate"]:
+        add_rank = config["annotate"]["add_rank"]
+    if 'run' in config['annotate']:
+        run = config['annotate']['run']
+    counter = [0]
+    
+    for i in range(len(toSet)):
+        for j in range(len(toSet[i])):
+            toSet[i][j] = tobytes(toSet[i][j])
+
+    annoteRank=[]
+    if config['annotate']['taxon_at_rank']:
+        if taxo is not None:
+            annoteRank = config['annotate']['taxon_at_rank']
+        else:
+            raise Exception("A taxonomy must be provided to annotate taxon ranks")
+    
+    def sequenceTagger(seq):
+                
+        if counter[0]>=0:
+            counter[0]+=1
+                            
+        for rank in annoteRank:
+            if 'taxid' in seq:
+                taxid = seq['taxid']
+                if taxid is not None:
+                    rtaxid = taxo.get_taxon_at_rank(taxid, rank)
+                    if rtaxid is not None:
+                        scn = taxo.get_scientific_name(rtaxid)
+                    else:
+                        scn=None
+                    seq[rank]=rtaxid
+                    seq["%s_name"%rank]=scn
+             
+        if add_rank:
+            seq['seq_rank']=counter[0]
+    
+        for i,v in toSet:
+            #try:
+            if taxo is not None:
+                environ = {'taxonomy' : taxo, 'sequence':seq, 'counter':counter[0], 'math':math}
+            else:
+                environ = {'sequence':seq, 'counter':counter[0], 'math':math}
+            val = eval(v, environ, seq)
+            #except Exception,e:       # TODO discuss usefulness of this
+            #    if options.onlyValid:
+            #        raise e
+            #    val = v
+            seq[i]=val
+
+        if length:
+            seq['seq_length']=len(seq)
+
+        if newId is not None:
+#            try:
+            if taxo is not None:
+                environ = {'taxonomy' : taxo, 'sequence':seq, 'counter':counter[0], 'math':math}
+            else:
+                environ = {'sequence':seq, 'counter':counter[0], 'math':math}     
+            val = eval(newId, environ, seq)
+#            except Exception,e:
+#                if options.onlyValid:
+#                    raise e
+#                val = newId
+            seq.id=val
+
+        if newDef is not None:
+#            try:
+            if taxo is not None:
+                environ = {'taxonomy' : taxo, 'sequence':seq, 'counter':counter[0], 'math':math}
+            else:
+                environ = {'sequence':seq, 'counter':counter[0], 'math':math}     
+            val = eval(newDef, environ, seq)
+#            except Exception,e:
+#                if options.onlyValid:
+#                    raise e
+#                val = newDef
+            seq.definition=val
+#             
+        if newSeq is not None:
+#            try:
+            if taxo is not None:
+                environ = {'taxonomy' : taxo, 'sequence':seq, 'counter':counter[0], 'math':math}
+            else:
+                environ = {'sequence':seq, 'counter':counter[0], 'math':math}     
+            val = eval(newSeq, environ, seq)
+#            except Exception,e:
+#                if options.onlyValid:
+#                    raise e
+#                val = newSeq
+            seq.seq=val
+            if 'seq_length' in seq:
+                seq['seq_length']=len(seq)
+            # Delete quality since it must match the sequence.
+            # TODO discuss deleting for each sequence separately
+            if QUALITY_COLUMN in seq:
+                seq.view.delete_column(QUALITY_COLUMN)
+                    
+        if run is not None:
+#            try:
+            if taxo is not None:
+                environ = {'taxonomy' : taxo, 'sequence':seq, 'counter':counter[0], 'math':math}
+            else:
+                environ = {'sequence':seq, 'counter':counter[0], 'math':math}     
+            eval(run, environ, seq)
+#            except Exception,e:
+#                if options.onlyValid:
+#                    raise e
+
+    return sequenceTagger
+
+
+def run(config):
+     
+    DMS.obi_atexit()
+    
+    logger("info", "obi annotate")
+
+    # Open the input
+    input = open_uri(config['obi']['inputURI'])
+    if input is None:
+        raise Exception("Could not read input view")
+    i_dms = input[0]
+    i_view = input[1]
+    i_view_name = input[1].name
+
+    # Open the output: only the DMS, as the output view is going to be created by cloning the input view
+    # (could eventually be done via an open_uri() argument)
+    output = open_uri(config['obi']['outputURI'],
+                      input=False,
+                      dms_only=True)
+    if output is None:
+        raise Exception("Could not create output view")
+    o_dms = output[0]
+    o_view_name = output[1]
+    
+    # If the input and output DMS are not the same, import the input view in the output DMS before cloning it to modify it
+    # (could be the other way around: clone and modify in the input DMS then import the new view in the output DMS)
+    if i_dms != o_dms:
+        imported_view_name = i_view_name
+        i=0
+        while imported_view_name in o_dms:  # Making sure view name is unique in output DMS
+            imported_view_name = i_view_name+b"_"+str2bytes(str(i))
+            i+=1
+        View.import_view(i_dms.full_path[:-7], o_dms.full_path[:-7], i_view_name, imported_view_name)
+        i_view = o_dms[imported_view_name]
+        
+    # Clone output view from input view
+    o_view = i_view.clone(o_view_name)
+    if o_view is None:
+        raise Exception("Couldn't create output view")
+    i_view.close()
+
+    # Open taxonomy if there is one
+    if 'taxoURI' in config['obi'] and config['obi']['taxoURI'] is not None:
+        taxo_uri = open_uri(config['obi']['taxoURI'])
+        if taxo_uri is None:
+            raise Exception("Couldn't open taxonomy")
+        taxo = taxo_uri[1]
+    else :
+        taxo = None
+
+    # Initialize the progress bar
+    pb = ProgressBar(len(o_view), config, seconde=5)
+
+    try:
+
+        # Apply editions
+        # Editions at view level
+        if 'delete_tags' in config['annotate']:
+            toDelete = config['annotate']['delete_tags'][:]
+        if 'rename_tags' in config['annotate']:
+            toRename = [x.split(':',1) for x in config['annotate']['rename_tags'] if len(x.split(':',1))==2]
+        if 'clear' in config['annotate']:
+            clear = config['annotate']['clear']
+        if 'keep' in config['annotate']:
+            keep = config['annotate']['keep']
+        for i in range(len(toDelete)):
+            toDelete[i] = tobytes(toDelete[i])
+        for i in range(len(toRename)):
+            for j in range(len(toRename[i])):
+                toRename[i][j] = tobytes(toRename[i][j])
+        for i in range(len(keep)):
+            keep[i] = tobytes(keep[i])
+        keep = set(keep)
+    
+        if clear or keep:
+            keys = [k for k in o_view.keys()]
+            for k in keys:
+                if k not in keep and k not in SPECIAL_COLUMNS:
+                    o_view.delete_column(k)
+        else:
+            for k in toDelete:
+                o_view.delete_column(k)
+            for old_name, new_name in toRename:
+                if old_name in o_view:
+                    o_view.rename_column(old_name, new_name)
+        
+        # Editions at line level
+        sequenceTagger = sequenceTaggerGenerator(config, taxo=taxo)
+        for i in range(len(o_view)):
+            pb(i)
+            sequenceTagger(o_view[i])
+
+    except Exception, e:
+        raise RollbackException("obi annotate error, rollbacking view: "+str(e), o_view)
+
+    pb(i, force=True)
+    print("", file=sys.stderr)
+    
+    # Save command config in View and DMS comments
+    command_line = " ".join(sys.argv[1:])
+    input_dms_name=[input[0].name]
+    input_view_name=[i_view_name]
+    if 'taxoURI' in config['obi'] and config['obi']['taxoURI'] is not None:
+        input_dms_name.append(config['obi']['taxoURI'].split("/")[-3])
+        input_view_name.append("taxonomy/"+config['obi']['taxoURI'].split("/")[-1])
+    o_view.write_config(config, "annotate", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name)
+    output[0].record_command_line(command_line)
+
+    #print("\n\nOutput view:\n````````````", file=sys.stderr)
+    #print(repr(o_view), file=sys.stderr)
+
+    # If the input and the output DMS are different, delete the temporary imported view used to create the final view
+    if i_dms != o_dms:
+        View.delete_view(o_dms, imported_view_name)
+        o_dms.close()
+    i_dms.close()
+
+    logger("info", "Done.")

python/obitools3/commands/build_ref_db.cfiles

@@ -0,0 +1,103 @@
+../../../src/obi_lcs.h
+../../../src/obi_lcs.c
+../../../src/obierrno.h
+../../../src/obierrno.c
+../../../src/upperband.h
+../../../src/upperband.c
+../../../src/sse_banded_LCS_alignment.h
+../../../src/sse_banded_LCS_alignment.c
+../../../src/obiblob.h
+../../../src/obiblob.c
+../../../src/utils.h
+../../../src/utils.c
+../../../src/obidms.h
+../../../src/obidms.c
+../../../src/libjson/json_utils.h
+../../../src/libjson/json_utils.c
+../../../src/libjson/cJSON.h
+../../../src/libjson/cJSON.c
+../../../src/obiavl.h
+../../../src/obiavl.c
+../../../src/bloom.h
+../../../src/bloom.c
+../../../src/crc64.h
+../../../src/crc64.c
+../../../src/murmurhash2.h
+../../../src/murmurhash2.c
+../../../src/obidmscolumn.h
+../../../src/obidmscolumn.c
+../../../src/obitypes.h
+../../../src/obitypes.c
+../../../src/obidmscolumndir.h
+../../../src/obidmscolumndir.c
+../../../src/obiblob_indexer.h
+../../../src/obiblob_indexer.c
+../../../src/obiview.h
+../../../src/obiview.c
+../../../src/hashtable.h
+../../../src/hashtable.c
+../../../src/linked_list.h
+../../../src/linked_list.c
+../../../src/obidmscolumn_array.h
+../../../src/obidmscolumn_array.c
+../../../src/obidmscolumn_blob.h
+../../../src/obidmscolumn_blob.c
+../../../src/obidmscolumn_idx.h
+../../../src/obidmscolumn_idx.c
+../../../src/obidmscolumn_bool.h
+../../../src/obidmscolumn_bool.c
+../../../src/obidmscolumn_char.h
+../../../src/obidmscolumn_char.c
+../../../src/obidmscolumn_float.h
+../../../src/obidmscolumn_float.c
+../../../src/obidmscolumn_int.h
+../../../src/obidmscolumn_int.c
+../../../src/obidmscolumn_qual.h
+../../../src/obidmscolumn_qual.c
+../../../src/obidmscolumn_seq.h
+../../../src/obidmscolumn_seq.c
+../../../src/obidmscolumn_str.h
+../../../src/obidmscolumn_str.c
+../../../src/array_indexer.h
+../../../src/array_indexer.c
+../../../src/char_str_indexer.h
+../../../src/char_str_indexer.c
+../../../src/dna_seq_indexer.h
+../../../src/dna_seq_indexer.c
+../../../src/encode.c
+../../../src/encode.h
+../../../src/uint8_indexer.c
+../../../src/uint8_indexer.h
+../../../src/build_reference_db.c
+../../../src/build_reference_db.h
+../../../src/kmer_similarity.c
+../../../src/kmer_similarity.h
+../../../src/obi_clean.c
+../../../src/obi_clean.h
+../../../src/obi_ecopcr.c
+../../../src/obi_ecopcr.h
+../../../src/obi_ecotag.c
+../../../src/obi_ecotag.h
+../../../src/obidms_taxonomy.c
+../../../src/obidms_taxonomy.h
+../../../src/obilittlebigman.c
+../../../src/obilittlebigman.h
+../../../src/_sse.h
+../../../src/obidebug.h
+../../../src/libecoPCR/libapat/CODES/dft_code.h
+../../../src/libecoPCR/libapat/CODES/dna_code.h
+../../../src/libecoPCR/libapat/CODES/prot_code.h
+../../../src/libecoPCR/libapat/apat_parse.c
+../../../src/libecoPCR/libapat/apat_search.c
+../../../src/libecoPCR/libapat/apat.h
+../../../src/libecoPCR/libapat/Gmach.h
+../../../src/libecoPCR/libapat/Gtypes.h
+../../../src/libecoPCR/libapat/libstki.c
+../../../src/libecoPCR/libapat/libstki.h
+../../../src/libecoPCR/libthermo/nnparams.h
+../../../src/libecoPCR/libthermo/nnparams.c
+../../../src/libecoPCR/ecoapat.c
+../../../src/libecoPCR/ecodna.c
+../../../src/libecoPCR/ecoError.c
+../../../src/libecoPCR/ecoMalloc.c
+../../../src/libecoPCR/ecoPCR.h

python/obitools3/commands/build_ref_db.pyx

@@ -0,0 +1,105 @@
+#cython: language_level=3
+
+from obitools3.apps.progress cimport ProgressBar  # @UnresolvedImport
+from obitools3.dms.dms cimport DMS
+from obitools3.dms.view import RollbackException
+from obitools3.dms.capi.build_reference_db cimport build_reference_db
+from obitools3.apps.optiongroups import addMinimalInputOption, addTaxonomyOption, addMinimalOutputOption
+from obitools3.uri.decode import open_uri
+from obitools3.apps.config import logger
+from obitools3.utils cimport tobytes, str2bytes
+from obitools3.dms.view.view cimport View
+from obitools3.dms.view.typed_view.view_NUC_SEQS cimport View_NUC_SEQS
+
+import sys
+
+
+__title__="Tag a set of sequences for PCR and sequencing errors identification"
+
+
+def addOptions(parser):
+
+    addMinimalInputOption(parser)
+    addTaxonomyOption(parser)
+    addMinimalOutputOption(parser)
+ 
+    group = parser.add_argument_group('obi build_ref_db specific options')
+
+    group.add_argument('--threshold','-t',
+                      action="store", dest="build_ref_db:threshold",
+                      metavar='<THRESHOLD>',
+                      default=0.0,
+                      type=float,
+                      help="Score threshold as a normalized identity, e.g. 0.95 for an identity of 95%%. Default: 0.00"
+                           " (no threshold).")
+
+
+def run(config):
+        
+    DMS.obi_atexit()
+    
+    logger("info", "obi build_ref_db")
+
+    # Open the input: only the DMS
+    input = open_uri(config['obi']['inputURI'],
+                     dms_only=True)
+    if input is None:
+        raise Exception("Could not read input")
+    i_dms = input[0]
+    i_dms_name = input[0].name
+    i_view_name = input[1]
+
+    # Open the output: only the DMS
+    output = open_uri(config['obi']['outputURI'],
+                      input=False,
+                      dms_only=True)
+    if output is None:
+        raise Exception("Could not create output")
+    o_dms = output[0]
+    final_o_view_name = output[1]
+    
+    # If the input and output DMS are not the same, build the database creating a temporary view that will be exported to 
+    # the right DMS and deleted in the other afterwards.
+    if i_dms != o_dms:
+        temporary_view_name = final_o_view_name
+        i=0
+        while temporary_view_name in i_dms:  # Making sure view name is unique in input DMS
+            temporary_view_name = final_o_view_name+b"_"+str2bytes(str(i))
+            i+=1
+        o_view_name = temporary_view_name
+    else:
+        o_view_name = final_o_view_name
+
+    # Read taxonomy name
+    taxonomy_name = config['obi']['taxoURI'].split("/")[-1]   # Robust in theory
+
+    # Save command config in View comments
+    command_line = " ".join(sys.argv[1:])
+    input_dms_name=[i_dms_name]
+    input_view_name= [i_view_name]
+    input_dms_name.append(config['obi']['taxoURI'].split("/")[-3])
+    input_view_name.append("taxonomy/"+config['obi']['taxoURI'].split("/")[-1])
+    comments = View.print_config(config, "build_ref_db", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name)
+
+    if build_reference_db(tobytes(i_dms_name), tobytes(i_view_name), tobytes(taxonomy_name), tobytes(o_view_name), comments, config['build_ref_db']['threshold']) < 0:
+        raise Exception("Error building a reference database")
+    
+    # If the input and output DMS are not the same, export result view to output DMS
+    if i_dms != o_dms:
+        View.import_view(i_dms.full_path[:-7], o_dms.full_path[:-7], o_view_name, final_o_view_name)
+    
+    # Save command config in DMS comments
+    o_dms.record_command_line(command_line)
+
+    #print("\n\nOutput view:\n````````````", file=sys.stderr)
+    #print(repr(o_dms[final_o_view_name]), file=sys.stderr)
+
+    # If the input and the output DMS are different, delete the temporary result view in the input DMS
+    if i_dms != o_dms:
+        View.delete_view(i_dms, o_view_name)
+        o_dms.close()
+    
+    i_dms.close()
+
+    logger("info", "Done.")
+    

python/obitools3/commands/check.pyx

@@ -1,68 +0,0 @@
-#cython: language_level=3
-
-from obitools3.dms.dms import DMS       # TODO cimport doesn't work
-from obitools3.dms.view.view import View    # TODO cimport doesn't work
- 
- 
-__title__="Print a preview of a DMS, view, column...."
- 
-default_config = {   'inputview'    : None,
-                 }
- 
-
-# TODO make it work with URIs
- 
-def addOptions(parser):
- 
-    # TODO put this common group somewhere else but I don't know where
-    group=parser.add_argument_group('DMS and view options')
- 
-    group.add_argument('--default-dms','-d', 
-                       action="store", dest="obi:defaultdms",
-                       metavar='<DMS NAME>',
-                       default=None,
-                       type=str,
-                       help="Name of the default DMS for reading and writing data.")
- 
-    group.add_argument('--view','-v',
-                       action="store", dest="obi:view",
-                       metavar='<VIEW NAME>',
-                       default=None,
-                       type=str,
-                       help="Name of the view.")
-  
- 
-#    group=parser.add_argument_group('obi check specific options')
- 
-#     group.add_argument('--print',
-#                      action="store", dest="less:print",
-#                      metavar='<N>',
-#                      default=None,
-#                      type=int,
-#                      help="Print N sequences (default: 10)")
- 
-     
-def run(config):
-     
-    # Open DMS
-    d = DMS.open(config['obi']['defaultdms'])
-     
-    # Open input view uif there is one 
-    if config['obi']['inputview'] is not None :
-        iview = View.open(d, config['obi']['inputview'])
-        print(repr(iview))
-    
-    else :
-        for v in d :
-            print(repr(v))
-     
-    d.close()
-     
-     
-     
- 
-     
-     
-     
-     
-         

python/obitools3/commands/clean.cfiles

@@ -0,0 +1,103 @@
+../../../src/obi_lcs.h
+../../../src/obi_lcs.c
+../../../src/obierrno.h
+../../../src/obierrno.c
+../../../src/upperband.h
+../../../src/upperband.c
+../../../src/sse_banded_LCS_alignment.h
+../../../src/sse_banded_LCS_alignment.c
+../../../src/obiblob.h
+../../../src/obiblob.c
+../../../src/utils.h
+../../../src/utils.c
+../../../src/obidms.h
+../../../src/obidms.c
+../../../src/libjson/json_utils.h
+../../../src/libjson/json_utils.c
+../../../src/libjson/cJSON.h
+../../../src/libjson/cJSON.c
+../../../src/obiavl.h
+../../../src/obiavl.c
+../../../src/bloom.h
+../../../src/bloom.c
+../../../src/crc64.h
+../../../src/crc64.c
+../../../src/murmurhash2.h
+../../../src/murmurhash2.c
+../../../src/obidmscolumn.h
+../../../src/obidmscolumn.c
+../../../src/obitypes.h
+../../../src/obitypes.c
+../../../src/obidmscolumndir.h
+../../../src/obidmscolumndir.c
+../../../src/obiblob_indexer.h
+../../../src/obiblob_indexer.c
+../../../src/obiview.h
+../../../src/obiview.c
+../../../src/hashtable.h
+../../../src/hashtable.c
+../../../src/linked_list.h
+../../../src/linked_list.c
+../../../src/obidmscolumn_array.h
+../../../src/obidmscolumn_array.c
+../../../src/obidmscolumn_blob.h
+../../../src/obidmscolumn_blob.c
+../../../src/obidmscolumn_idx.h
+../../../src/obidmscolumn_idx.c
+../../../src/obidmscolumn_bool.h
+../../../src/obidmscolumn_bool.c
+../../../src/obidmscolumn_char.h
+../../../src/obidmscolumn_char.c
+../../../src/obidmscolumn_float.h
+../../../src/obidmscolumn_float.c
+../../../src/obidmscolumn_int.h
+../../../src/obidmscolumn_int.c
+../../../src/obidmscolumn_qual.h
+../../../src/obidmscolumn_qual.c
+../../../src/obidmscolumn_seq.h
+../../../src/obidmscolumn_seq.c
+../../../src/obidmscolumn_str.h
+../../../src/obidmscolumn_str.c
+../../../src/array_indexer.h
+../../../src/array_indexer.c
+../../../src/char_str_indexer.h
+../../../src/char_str_indexer.c
+../../../src/dna_seq_indexer.h
+../../../src/dna_seq_indexer.c
+../../../src/encode.c
+../../../src/encode.h
+../../../src/uint8_indexer.c
+../../../src/uint8_indexer.h
+../../../src/build_reference_db.c
+../../../src/build_reference_db.h
+../../../src/kmer_similarity.c
+../../../src/kmer_similarity.h
+../../../src/obi_clean.c
+../../../src/obi_clean.h
+../../../src/obi_ecopcr.c
+../../../src/obi_ecopcr.h
+../../../src/obi_ecotag.c
+../../../src/obi_ecotag.h
+../../../src/obidms_taxonomy.c
+../../../src/obidms_taxonomy.h
+../../../src/obilittlebigman.c
+../../../src/obilittlebigman.h
+../../../src/_sse.h
+../../../src/obidebug.h
+../../../src/libecoPCR/libapat/CODES/dft_code.h
+../../../src/libecoPCR/libapat/CODES/dna_code.h
+../../../src/libecoPCR/libapat/CODES/prot_code.h
+../../../src/libecoPCR/libapat/apat_parse.c
+../../../src/libecoPCR/libapat/apat_search.c
+../../../src/libecoPCR/libapat/apat.h
+../../../src/libecoPCR/libapat/Gmach.h
+../../../src/libecoPCR/libapat/Gtypes.h
+../../../src/libecoPCR/libapat/libstki.c
+../../../src/libecoPCR/libapat/libstki.h
+../../../src/libecoPCR/libthermo/nnparams.h
+../../../src/libecoPCR/libthermo/nnparams.c
+../../../src/libecoPCR/ecoapat.c
+../../../src/libecoPCR/ecodna.c
+../../../src/libecoPCR/ecoError.c
+../../../src/libecoPCR/ecoMalloc.c
+../../../src/libecoPCR/ecoPCR.h

python/obitools3/commands/clean.pyx

@@ -0,0 +1,124 @@
+#cython: language_level=3
+
+from obitools3.apps.progress cimport ProgressBar  # @UnresolvedImport
+from obitools3.dms.dms cimport DMS
+from obitools3.dms.view import RollbackException
+from obitools3.dms.capi.obiclean cimport obi_clean
+from obitools3.apps.optiongroups import addMinimalInputOption, addMinimalOutputOption
+from obitools3.uri.decode import open_uri
+from obitools3.apps.config import logger
+from obitools3.utils cimport tobytes, str2bytes
+from obitools3.dms.view.view cimport View
+from obitools3.dms.view.typed_view.view_NUC_SEQS cimport View_NUC_SEQS
+
+import sys
+
+
+__title__="Tag a set of sequences for PCR and sequencing errors identification"
+
+
+def addOptions(parser):
+
+    addMinimalInputOption(parser)
+    addMinimalOutputOption(parser)
+ 
+    group = parser.add_argument_group('obi clean specific options')
+
+    group.add_argument('--distance', '-d',
+                       action="store", dest="clean:distance",
+                       metavar='<DISTANCE>',
+                       default=1.0,
+                       type=float,
+                       help="Maximum numbers of errors between two variant sequences. Default: 1.")
+
+    group.add_argument('--sample-tag', '-s',
+                       action="store", 
+                       dest="clean:sample-tag-name",
+                       metavar="<SAMPLE TAG NAME>",
+                       type=str,
+                       default="merged_sample",
+                       help="Name of the tag where sample counts are kept.")
+    
+    group.add_argument('--ratio', '-r',
+                       action="store", dest="clean:ratio",
+                       metavar='<RATIO>',
+                       default=0.5,
+                       type=float,
+                       help="Maximum ratio between the counts of two sequences so that the less abundant one can be considered"
+                            " a variant of the more abundant one. Default: 0.5.")
+ 
+    group.add_argument('--heads-only', '-H',
+                       action="store_true", 
+                       dest="clean:heads-only",
+                       default=False,
+                       help="Only sequences labeled as heads are kept in the output. Default: False")
+
+    group.add_argument('--cluster-tags', '-C',
+                       action="store_true", 
+                       dest="clean:cluster-tags",
+                       default=False,
+                       help="Adds tags for each sequence giving its cluster's head and weight for each sample.")
+
+
+def run(config):
+        
+    DMS.obi_atexit()
+    
+    logger("info", "obi clean")
+
+    # Open the input: only the DMS
+    input = open_uri(config['obi']['inputURI'],
+                     dms_only=True)
+    if input is None:
+        raise Exception("Could not read input")
+    i_dms = input[0]
+    i_dms_name = input[0].name
+    i_view_name = input[1]
+
+    # Open the output: only the DMS
+    output = open_uri(config['obi']['outputURI'],
+                      input=False,
+                      dms_only=True)
+    if output is None:
+        raise Exception("Could not create output")
+    o_dms = output[0]
+    final_o_view_name = output[1]
+    
+    # If the input and output DMS are not the same, run obiclean creating a temporary view that will be exported to 
+    # the right DMS and deleted in the other afterwards.
+    if i_dms != o_dms:
+        temporary_view_name = final_o_view_name
+        i=0
+        while temporary_view_name in i_dms:  # Making sure view name is unique in input DMS
+            temporary_view_name = final_o_view_name+b"_"+str2bytes(str(i))
+            i+=1
+        o_view_name = temporary_view_name
+    else:
+        o_view_name = final_o_view_name
+        
+    # Save command config in View comments
+    command_line = " ".join(sys.argv[1:])
+    comments = View.print_config(config, "clean", command_line, input_dms_name=[i_dms_name], input_view_name=[i_view_name])
+
+    if obi_clean(tobytes(i_dms_name), tobytes(i_view_name), tobytes(config['clean']['sample-tag-name']), tobytes(o_view_name), comments, \
+              config['clean']['distance'], config['clean']['ratio'], config['clean']['heads-only'], 1) < 0:
+        raise Exception("Error running obiclean")
+    
+    # If the input and output DMS are not the same, export result view to output DMS
+    if i_dms != o_dms:
+        View.import_view(i_dms.full_path[:-7], o_dms.full_path[:-7], o_view_name, final_o_view_name)
+    
+    # Save command config in DMS comments
+    o_dms.record_command_line(command_line)
+
+    #print("\n\nOutput view:\n````````````", file=sys.stderr)
+    #print(repr(o_dms[final_o_view_name]), file=sys.stderr)
+
+    # If the input and the output DMS are different, delete the temporary result view in the input DMS
+    if i_dms != o_dms:
+        View.delete_view(i_dms, o_view_name)
+        o_dms.close()
+    
+    i_dms.close()
+
+    logger("info", "Done.")

python/obitools3/commands/count.cfiles

@@ -0,0 +1,103 @@
+../../../src/obi_lcs.h
+../../../src/obi_lcs.c
+../../../src/obierrno.h
+../../../src/obierrno.c
+../../../src/upperband.h
+../../../src/upperband.c
+../../../src/sse_banded_LCS_alignment.h
+../../../src/sse_banded_LCS_alignment.c
+../../../src/obiblob.h
+../../../src/obiblob.c
+../../../src/utils.h
+../../../src/utils.c
+../../../src/obidms.h
+../../../src/obidms.c
+../../../src/libjson/json_utils.h
+../../../src/libjson/json_utils.c
+../../../src/libjson/cJSON.h
+../../../src/libjson/cJSON.c
+../../../src/obiavl.h
+../../../src/obiavl.c
+../../../src/bloom.h
+../../../src/bloom.c
+../../../src/crc64.h
+../../../src/crc64.c
+../../../src/murmurhash2.h
+../../../src/murmurhash2.c
+../../../src/obidmscolumn.h
+../../../src/obidmscolumn.c
+../../../src/obitypes.h
+../../../src/obitypes.c
+../../../src/obidmscolumndir.h
+../../../src/obidmscolumndir.c
+../../../src/obiblob_indexer.h
+../../../src/obiblob_indexer.c
+../../../src/obiview.h
+../../../src/obiview.c
+../../../src/hashtable.h
+../../../src/hashtable.c
+../../../src/linked_list.h
+../../../src/linked_list.c
+../../../src/obidmscolumn_array.h
+../../../src/obidmscolumn_array.c
+../../../src/obidmscolumn_blob.h
+../../../src/obidmscolumn_blob.c
+../../../src/obidmscolumn_idx.h
+../../../src/obidmscolumn_idx.c
+../../../src/obidmscolumn_bool.h
+../../../src/obidmscolumn_bool.c
+../../../src/obidmscolumn_char.h
+../../../src/obidmscolumn_char.c
+../../../src/obidmscolumn_float.h
+../../../src/obidmscolumn_float.c
+../../../src/obidmscolumn_int.h
+../../../src/obidmscolumn_int.c
+../../../src/obidmscolumn_qual.h
+../../../src/obidmscolumn_qual.c
+../../../src/obidmscolumn_seq.h
+../../../src/obidmscolumn_seq.c
+../../../src/obidmscolumn_str.h
+../../../src/obidmscolumn_str.c
+../../../src/array_indexer.h
+../../../src/array_indexer.c
+../../../src/char_str_indexer.h
+../../../src/char_str_indexer.c
+../../../src/dna_seq_indexer.h
+../../../src/dna_seq_indexer.c
+../../../src/encode.c
+../../../src/encode.h
+../../../src/uint8_indexer.c
+../../../src/uint8_indexer.h
+../../../src/build_reference_db.c
+../../../src/build_reference_db.h
+../../../src/kmer_similarity.c
+../../../src/kmer_similarity.h
+../../../src/obi_clean.c
+../../../src/obi_clean.h
+../../../src/obi_ecopcr.c
+../../../src/obi_ecopcr.h
+../../../src/obi_ecotag.c
+../../../src/obi_ecotag.h
+../../../src/obidms_taxonomy.c
+../../../src/obidms_taxonomy.h
+../../../src/obilittlebigman.c
+../../../src/obilittlebigman.h
+../../../src/_sse.h
+../../../src/obidebug.h
+../../../src/libecoPCR/libapat/CODES/dft_code.h
+../../../src/libecoPCR/libapat/CODES/dna_code.h
+../../../src/libecoPCR/libapat/CODES/prot_code.h
+../../../src/libecoPCR/libapat/apat_parse.c
+../../../src/libecoPCR/libapat/apat_search.c
+../../../src/libecoPCR/libapat/apat.h
+../../../src/libecoPCR/libapat/Gmach.h
+../../../src/libecoPCR/libapat/Gtypes.h
+../../../src/libecoPCR/libapat/libstki.c
+../../../src/libecoPCR/libapat/libstki.h
+../../../src/libecoPCR/libthermo/nnparams.h
+../../../src/libecoPCR/libthermo/nnparams.c
+../../../src/libecoPCR/ecoapat.c
+../../../src/libecoPCR/ecodna.c
+../../../src/libecoPCR/ecoError.c
+../../../src/libecoPCR/ecoMalloc.c
+../../../src/libecoPCR/ecoPCR.h

python/obitools3/commands/count.py

@@ -1,44 +0,0 @@
-'''
-Created on 8 mars 2016
-
-@author: coissac
-'''
-
-from obitools3.apps.progress import ProgressBar  # @UnresolvedImport
-import time
-
-__title__="Counts sequences in a sequence set"
-
-
-default_config = {   'countmode'       : None
-                 }
-
-def addOptions(parser):
-    parser.add_argument(dest='obi:input',     metavar='obi:input', 
-                                              nargs='?', 
-                                              default=None,
-                        help='input data set' )
-
-    group=parser.add_argument_group('Obicount specific options')
-    group.add_argument('-s','--sequence',
-                             action="store_true", dest="count:sequence",
-                             default=False,
-                             help="Prints only the number of sequence records."
-                             )
- 
-    group.add_argument('-a','--all',
-                             action="store_true", dest="count:all",
-                             default=False,
-                             help="Prints only the total count of sequence records (if a sequence has no `count` attribute, its default count is 1) (default: False)."
-                             )
-
-    
-    
-def run(config):
-    # The code of my command
-    pb = ProgressBar(1000,config,seconde=1)
-    
-    for i in range(1,1001):
-        pb(i)
-        time.sleep(0.01)
-        

python/obitools3/commands/count.pyx

@@ -0,0 +1,55 @@
+#cython: language_level=3
+
+
+from obitools3.uri.decode import open_uri
+from obitools3.apps.config import logger
+from obitools3.dms import DMS
+from obitools3.apps.optiongroups import addMinimalInputOption
+from obitools3.dms.capi.obiview cimport COUNT_COLUMN
+
+
+__title__="Counts sequence records"
+
+ 
+def addOptions(parser):
+ 
+    addMinimalInputOption(parser)
+ 
+    group = parser.add_argument_group('obi count specific options')
+ 
+    group.add_argument('-s','--sequence',
+                        action="store_true", dest="count:sequence",
+                        default=False,
+                        help="Prints only the number of sequence records.")
+ 
+    group.add_argument('-a','--all',
+                        action="store_true", dest="count:all",
+                        default=False,
+                        help="Prints only the total count of sequence records (if a sequence has no `count` attribute, its default count is 1) (default: False).")
+
+    
+def run(config):
+    
+    DMS.obi_atexit()
+    
+    logger("info", "obi count")
+
+    # Open the input
+    input = open_uri(config['obi']['inputURI'])
+    if input is None:
+        raise Exception("Could not read input")
+    entries = input[1]
+
+    count1 = len(entries)
+    count2 = 0
+    
+    if COUNT_COLUMN in entries and ((config['count']['sequence'] == config['count']['all']) or (config['count']['all'])) :
+        for e in entries:
+            count2+=e[COUNT_COLUMN]
+
+    if COUNT_COLUMN in entries and (config['count']['sequence'] == config['count']['all']):
+        print(count1,count2)
+    elif COUNT_COLUMN in entries and config['count']['all']:
+        print(count2)
+    else:
+        print(count1)

python/obitools3/commands/ecopcr.cfiles

@@ -0,0 +1,103 @@
+../../../src/obi_lcs.h
+../../../src/obi_lcs.c
+../../../src/obierrno.h
+../../../src/obierrno.c
+../../../src/upperband.h
+../../../src/upperband.c
+../../../src/sse_banded_LCS_alignment.h
+../../../src/sse_banded_LCS_alignment.c
+../../../src/obiblob.h
+../../../src/obiblob.c
+../../../src/utils.h
+../../../src/utils.c
+../../../src/obidms.h
+../../../src/obidms.c
+../../../src/libjson/json_utils.h
+../../../src/libjson/json_utils.c
+../../../src/libjson/cJSON.h
+../../../src/libjson/cJSON.c
+../../../src/obiavl.h
+../../../src/obiavl.c
+../../../src/bloom.h
+../../../src/bloom.c
+../../../src/crc64.h
+../../../src/crc64.c
+../../../src/murmurhash2.h
+../../../src/murmurhash2.c
+../../../src/obidmscolumn.h
+../../../src/obidmscolumn.c
+../../../src/obitypes.h
+../../../src/obitypes.c
+../../../src/obidmscolumndir.h
+../../../src/obidmscolumndir.c
+../../../src/obiblob_indexer.h
+../../../src/obiblob_indexer.c
+../../../src/obiview.h
+../../../src/obiview.c
+../../../src/hashtable.h
+../../../src/hashtable.c
+../../../src/linked_list.h
+../../../src/linked_list.c
+../../../src/obidmscolumn_array.h
+../../../src/obidmscolumn_array.c
+../../../src/obidmscolumn_blob.h
+../../../src/obidmscolumn_blob.c
+../../../src/obidmscolumn_idx.h
+../../../src/obidmscolumn_idx.c
+../../../src/obidmscolumn_bool.h
+../../../src/obidmscolumn_bool.c
+../../../src/obidmscolumn_char.h
+../../../src/obidmscolumn_char.c
+../../../src/obidmscolumn_float.h
+../../../src/obidmscolumn_float.c
+../../../src/obidmscolumn_int.h
+../../../src/obidmscolumn_int.c
+../../../src/obidmscolumn_qual.h
+../../../src/obidmscolumn_qual.c
+../../../src/obidmscolumn_seq.h
+../../../src/obidmscolumn_seq.c
+../../../src/obidmscolumn_str.h
+../../../src/obidmscolumn_str.c
+../../../src/array_indexer.h
+../../../src/array_indexer.c
+../../../src/char_str_indexer.h
+../../../src/char_str_indexer.c
+../../../src/dna_seq_indexer.h
+../../../src/dna_seq_indexer.c
+../../../src/encode.c
+../../../src/encode.h
+../../../src/uint8_indexer.c
+../../../src/uint8_indexer.h
+../../../src/build_reference_db.c
+../../../src/build_reference_db.h
+../../../src/kmer_similarity.c
+../../../src/kmer_similarity.h
+../../../src/obi_clean.c
+../../../src/obi_clean.h
+../../../src/obi_ecopcr.c
+../../../src/obi_ecopcr.h
+../../../src/obi_ecotag.c
+../../../src/obi_ecotag.h
+../../../src/obidms_taxonomy.c
+../../../src/obidms_taxonomy.h
+../../../src/obilittlebigman.c
+../../../src/obilittlebigman.h
+../../../src/_sse.h
+../../../src/obidebug.h
+../../../src/libecoPCR/libapat/CODES/dft_code.h
+../../../src/libecoPCR/libapat/CODES/dna_code.h
+../../../src/libecoPCR/libapat/CODES/prot_code.h
+../../../src/libecoPCR/libapat/apat_parse.c
+../../../src/libecoPCR/libapat/apat_search.c
+../../../src/libecoPCR/libapat/apat.h
+../../../src/libecoPCR/libapat/Gmach.h
+../../../src/libecoPCR/libapat/Gtypes.h
+../../../src/libecoPCR/libapat/libstki.c
+../../../src/libecoPCR/libapat/libstki.h
+../../../src/libecoPCR/libthermo/nnparams.h
+../../../src/libecoPCR/libthermo/nnparams.c
+../../../src/libecoPCR/ecoapat.c
+../../../src/libecoPCR/ecodna.c
+../../../src/libecoPCR/ecoError.c
+../../../src/libecoPCR/ecoMalloc.c
+../../../src/libecoPCR/ecoPCR.h

python/obitools3/commands/ecopcr.pyx

@@ -0,0 +1,202 @@
+#cython: language_level=3
+
+from obitools3.apps.progress cimport ProgressBar  # @UnresolvedImport
+from obitools3.dms.dms cimport DMS
+from obitools3.dms.capi.obidms cimport OBIDMS_p
+from obitools3.dms.view import RollbackException
+from obitools3.dms.capi.obiecopcr cimport obi_ecopcr
+from obitools3.apps.optiongroups import addMinimalInputOption, addMinimalOutputOption, addTaxonomyOption
+from obitools3.uri.decode import open_uri
+from obitools3.apps.config import logger
+from obitools3.utils cimport tobytes
+from obitools3.dms.view.typed_view.view_NUC_SEQS cimport View_NUC_SEQS
+from obitools3.dms.view import View
+
+from libc.stdlib  cimport malloc, free
+from libc.stdint  cimport int32_t
+
+import sys
+
+
+__title__="in silico PCR"
+
+
+# TODO: add option to output unique ids
+def addOptions(parser):
+
+    addMinimalInputOption(parser)
+    addTaxonomyOption(parser)
+    addMinimalOutputOption(parser)
+
+ 
+    group = parser.add_argument_group('obi ecopcr specific options')
+
+    group.add_argument('--primer1', '-F',
+                       action="store", dest="ecopcr:primer1",
+                       metavar='<PRIMER>',
+                       type=str,
+                       help="Forward primer.")
+
+    group.add_argument('--primer2', '-R',
+                       action="store", dest="ecopcr:primer2",
+                       metavar='<PRIMER>',
+                       type=str,
+                       help="Reverse primer.")
+
+    group.add_argument('--error', '-e',
+                       action="store", dest="ecopcr:error",
+                       metavar='<ERROR>',
+                       default=0,
+                       type=int,
+                       help="Maximum number of errors (mismatches) allowed per primer. Default: 0.")
+
+    group.add_argument('--min-length', '-l',
+                       action="store", 
+                       dest="ecopcr:min-length",
+                       metavar="<MINIMUM LENGTH>",
+                       type=int,
+                       default=0,
+                       help="Minimum length of the in silico amplified DNA fragment, excluding primers.")
+    
+    group.add_argument('--max-length', '-L',
+                       action="store", 
+                       dest="ecopcr:max-length",
+                       metavar="<MAXIMUM LENGTH>",
+                       type=int,
+                       default=0,
+                       help="Maximum length of the in silico amplified DNA fragment, excluding primers.")
+ 
+    group.add_argument('--restrict-to-taxid', '-r',
+                       action="append", 
+                       dest="ecopcr:restrict-to-taxid",
+                       metavar="<TAXID>",
+                       type=int,
+                       default=[],
+                       help="Only the sequence records corresponding to the taxonomic group identified "
+                            "by TAXID are considered for the in silico PCR. The TAXID is an integer "
+                            "that can be found in the NCBI taxonomic database.")
+ 
+    group.add_argument('--ignore-taxid', '-i',
+                       action="append", 
+                       dest="ecopcr:ignore-taxid",
+                       metavar="<TAXID>",
+                       type=int,
+                       default=[],
+                       help="The sequences of the taxonomic group identified by TAXID are not considered for the in silico PCR.")
+
+    group.add_argument('--circular', '-c',
+                       action="store_true", 
+                       dest="ecopcr:circular",
+                       default=False,
+                       help="Considers that the input sequences are circular (e.g. mitochondrial or chloroplastic DNA).")
+
+    group.add_argument('--salt-concentration', '-a',
+                       action="store", 
+                       dest="ecopcr:salt-concentration",
+                       metavar="<FLOAT>",
+                       type=float,
+                       default=0.05,
+                       help="Salt concentration used for estimating the Tm. Default: 0.05.")
+
+    group.add_argument('--salt-correction-method', '-m',
+                       action="store", 
+                       dest="ecopcr:salt-correction-method",
+                       metavar="<1|2>",
+                       type=int,
+                       default=1,
+                       help="Defines the method used for estimating the Tm (melting temperature) between the primers and their corresponding "
+                            "target sequences. SANTALUCIA: 1, or OWCZARZY: 2. Default: 1.")
+
+    group.add_argument('--keep-nucs', '-D',
+                       action="store", 
+                       dest="ecopcr:keep-nucs",
+                       metavar="<INTEGER>",
+                       type=int,
+                       default=0,
+                       help="Keeps the specified number of nucleotides on each side of the in silico amplified sequences, "
+                            "(already including the amplified DNA fragment plus the two target sequences of the primers).")
+
+    group.add_argument('--kingdom-mode', '-k',
+                       action="store_true", 
+                       dest="ecopcr:kingdom-mode",
+                       default=False,
+                       help="Print in the output the kingdom of the in silico amplified sequences (default: print the superkingdom).")
+
+ 
+
+def run(config):
+
+    cdef int32_t* restrict_to_taxids_p = NULL
+    cdef int32_t* ignore_taxids_p = NULL
+    
+    restrict_to_taxids_len = len(config['ecopcr']['restrict-to-taxid'])
+    restrict_to_taxids_p = <int32_t*> malloc((restrict_to_taxids_len + 1) * sizeof(int32_t))   # +1 for the -1 flagging the end of the array
+    for i in range(restrict_to_taxids_len) :
+        restrict_to_taxids_p[i] = config['ecopcr']['restrict-to-taxid'][i]
+    restrict_to_taxids_p[restrict_to_taxids_len] = -1
+
+    ignore_taxids_len = len(config['ecopcr']['ignore-taxid'])
+    ignore_taxids_p = <int32_t*> malloc((ignore_taxids_len + 1) * sizeof(int32_t))   # +1 for the -1 flagging the end of the array
+    for i in range(ignore_taxids_len) :
+        ignore_taxids_p[i] = config['ecopcr']['ignore-taxid'][i]
+    ignore_taxids_p[ignore_taxids_len] = -1
+    
+    DMS.obi_atexit()
+    
+    logger("info", "obi ecopcr")
+
+    # Open the input: only the DMS
+    input = open_uri(config['obi']['inputURI'],
+                     dms_only=True)
+    if input is None:
+        raise Exception("Could not read input")
+    i_dms = input[0]
+    i_dms_name = input[0].name
+    i_view_name = input[1]
+
+    # Open the output: only the DMS
+    output = open_uri(config['obi']['outputURI'],
+                      input=False,
+                      dms_only=True)
+    if output is None:
+        raise Exception("Could not create output")
+    o_dms = output[0]
+    o_dms_name = output[0].name
+    o_view_name = output[1]
+
+    # Read taxonomy name    
+    taxonomy_name = config['obi']['taxoURI'].split("/")[-1]   # Robust in theory
+    
+    # Save command config in View comments
+    command_line = " ".join(sys.argv[1:])
+    input_dms_name=[i_dms_name]
+    input_view_name= [i_view_name]
+    input_dms_name.append(config['obi']['taxoURI'].split("/")[-3])
+    input_view_name.append("taxonomy/"+config['obi']['taxoURI'].split("/")[-1])
+
+    comments = View.print_config(config, "ecopcr", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name)
+
+    # TODO: primers in comments?
+
+    if obi_ecopcr(tobytes(i_dms_name), tobytes(i_view_name), tobytes(taxonomy_name), \
+                  tobytes(o_dms_name), tobytes(o_view_name), comments, \
+                  tobytes(config['ecopcr']['primer1']), tobytes(config['ecopcr']['primer2']), \
+                  config['ecopcr']['error'], \
+                  config['ecopcr']['min-length'], config['ecopcr']['max-length'], \
+                  restrict_to_taxids_p, ignore_taxids_p, \
+                  config['ecopcr']['circular'], config['ecopcr']['salt-concentration'], config['ecopcr']['salt-correction-method'], \
+                  config['ecopcr']['keep-nucs'], config['ecopcr']['kingdom-mode']) < 0:
+        raise Exception("Error running ecopcr")
+
+    # Save command config in DMS comments
+    o_dms.record_command_line(command_line)
+
+    free(restrict_to_taxids_p)
+    free(ignore_taxids_p)
+    
+    #print("\n\nOutput view:\n````````````", file=sys.stderr)
+    #print(repr(o_dms[o_view_name]), file=sys.stderr)
+
+    o_dms.close()
+
+    logger("info", "Done.")

python/obitools3/commands/ecotag.cfiles

@@ -0,0 +1,103 @@
+../../../src/obi_lcs.h
+../../../src/obi_lcs.c
+../../../src/obierrno.h
+../../../src/obierrno.c
+../../../src/upperband.h
+../../../src/upperband.c
+../../../src/sse_banded_LCS_alignment.h
+../../../src/sse_banded_LCS_alignment.c
+../../../src/obiblob.h
+../../../src/obiblob.c
+../../../src/utils.h
+../../../src/utils.c
+../../../src/obidms.h
+../../../src/obidms.c
+../../../src/libjson/json_utils.h
+../../../src/libjson/json_utils.c
+../../../src/libjson/cJSON.h
+../../../src/libjson/cJSON.c
+../../../src/obiavl.h
+../../../src/obiavl.c
+../../../src/bloom.h
+../../../src/bloom.c
+../../../src/crc64.h
+../../../src/crc64.c
+../../../src/murmurhash2.h
+../../../src/murmurhash2.c
+../../../src/obidmscolumn.h
+../../../src/obidmscolumn.c
+../../../src/obitypes.h
+../../../src/obitypes.c
+../../../src/obidmscolumndir.h
+../../../src/obidmscolumndir.c
+../../../src/obiblob_indexer.h
+../../../src/obiblob_indexer.c
+../../../src/obiview.h
+../../../src/obiview.c
+../../../src/hashtable.h
+../../../src/hashtable.c
+../../../src/linked_list.h
+../../../src/linked_list.c
+../../../src/obidmscolumn_array.h
+../../../src/obidmscolumn_array.c
+../../../src/obidmscolumn_blob.h
+../../../src/obidmscolumn_blob.c
+../../../src/obidmscolumn_idx.h
+../../../src/obidmscolumn_idx.c
+../../../src/obidmscolumn_bool.h
+../../../src/obidmscolumn_bool.c
+../../../src/obidmscolumn_char.h
+../../../src/obidmscolumn_char.c
+../../../src/obidmscolumn_float.h
+../../../src/obidmscolumn_float.c
+../../../src/obidmscolumn_int.h
+../../../src/obidmscolumn_int.c
+../../../src/obidmscolumn_qual.h
+../../../src/obidmscolumn_qual.c
+../../../src/obidmscolumn_seq.h
+../../../src/obidmscolumn_seq.c
+../../../src/obidmscolumn_str.h
+../../../src/obidmscolumn_str.c
+../../../src/array_indexer.h
+../../../src/array_indexer.c
+../../../src/char_str_indexer.h
+../../../src/char_str_indexer.c
+../../../src/dna_seq_indexer.h
+../../../src/dna_seq_indexer.c
+../../../src/encode.c
+../../../src/encode.h
+../../../src/uint8_indexer.c
+../../../src/uint8_indexer.h
+../../../src/build_reference_db.c
+../../../src/build_reference_db.h
+../../../src/kmer_similarity.c
+../../../src/kmer_similarity.h
+../../../src/obi_clean.c
+../../../src/obi_clean.h
+../../../src/obi_ecopcr.c
+../../../src/obi_ecopcr.h
+../../../src/obi_ecotag.c
+../../../src/obi_ecotag.h
+../../../src/obidms_taxonomy.c
+../../../src/obidms_taxonomy.h
+../../../src/obilittlebigman.c
+../../../src/obilittlebigman.h
+../../../src/_sse.h
+../../../src/obidebug.h
+../../../src/libecoPCR/libapat/CODES/dft_code.h
+../../../src/libecoPCR/libapat/CODES/dna_code.h
+../../../src/libecoPCR/libapat/CODES/prot_code.h
+../../../src/libecoPCR/libapat/apat_parse.c
+../../../src/libecoPCR/libapat/apat_search.c
+../../../src/libecoPCR/libapat/apat.h
+../../../src/libecoPCR/libapat/Gmach.h
+../../../src/libecoPCR/libapat/Gtypes.h
+../../../src/libecoPCR/libapat/libstki.c
+../../../src/libecoPCR/libapat/libstki.h
+../../../src/libecoPCR/libthermo/nnparams.h
+../../../src/libecoPCR/libthermo/nnparams.c
+../../../src/libecoPCR/ecoapat.c
+../../../src/libecoPCR/ecodna.c
+../../../src/libecoPCR/ecoError.c
+../../../src/libecoPCR/ecoMalloc.c
+../../../src/libecoPCR/ecoPCR.h