Switch to version 3.0.0b23

triggering error

MANIFEST.in

@@ -6,7 +6,7 @@ recursive-include doc/sphinx/source *.txt *.rst *.py
 recursive-include doc/sphinx/sphinxext *.py
 include doc/sphinx/Makefile
 include doc/sphinx/Doxyfile
-include README.txt
+include README.md
 include requirements.txt
 include scripts/obi
 

python/obitools3/commands/ecopcr.pyx

@@ -35,12 +35,14 @@ def addOptions(parser):
                        action="store", dest="ecopcr:primer1",
                        metavar='<PRIMER>',
                        type=str,
+                       required=True,
                        help="Forward primer, length must be less than or equal to 32")
 
     group.add_argument('--primer2', '-R',
                        action="store", dest="ecopcr:primer2",
                        metavar='<PRIMER>',
                        type=str,
+                       required=True,
                        help="Reverse primer, length must be less than or equal to 32")
 
     group.add_argument('--error', '-e',

python/obitools3/commands/import.pyx

@@ -73,7 +73,7 @@ def addOptions(parser):
                      action="store_true", dest="import:preread",
                      default=False,
                      help="Do a first readthrough of the dataset if it contains huge dictionaries (more than 100 keys) for "
-                          "a much faster import.")
+                          "a much faster import. This option is not recommended and will slow down the import in any other case.")
 
 
 def run(config):
@@ -236,7 +236,7 @@ def run(config):
             dcols[tag] = (Column.new_column(view, tag, dict_dict[tag][1], \
                               nb_elements_per_line=len(dict_dict[tag][0]), \
                               elements_names=list(dict_dict[tag][0])), \
-                          value_obitype)
+                          dict_dict[tag][1])
     
         
         # Reinitialize the input
@@ -260,7 +260,6 @@ def run(config):
 
         if entry is None:  # error or exception handled at lower level, not raised because Python generators can't resume after any exception is raised
             if config['obi']['skiperror']:
-                i-=1
                 continue
             else:
                 raise RollbackException("obi import error, rollbacking view", view)
@@ -270,6 +269,8 @@ def run(config):
         elif not i%50000:
             logger("info", "Imported %d entries", i)
         
+        try:
+             
             if NUC_SEQS_view: 
                 id_col[i] = entry.id
                 def_col[i] = entry.definition
@@ -388,6 +389,13 @@ def run(config):
                                 # Fill value
                                 dcols[tag][0][i] = value
         
+        except Exception as e:
+            print("\nCould not import sequence id:", entry.id, "(error raised:", e, ")")
+            if 'skiperror' in config['obi'] and not config['obi']['skiperror']:
+                raise e
+            else:
+                pass
+        
         i+=1
  
     if pb is not None:

python/obitools3/commands/ngsfilter.pyx

@@ -42,6 +42,7 @@ def addOptions(parser):
                      metavar="<URI>",
                      type=str,
                      default=None,
+                     required=True,
                      help="URI to the view containing the samples definition (with tags, primers, sample names,...).\n"
                           "\nWarning: primer lengths must be less than or equal to 32")
 

python/obitools3/dms/capi/obidmscolumn.pxd

@@ -63,6 +63,8 @@ cdef extern from "obidmscolumn.h" nogil:
     
     char* obi_get_elements_names(OBIDMS_column_p column)
     
+    char* obi_column_formatted_infos(OBIDMS_column_p column)
+
     index_t obi_column_get_element_index_from_name(OBIDMS_column_p column, const char* element_name)
     
     int obi_column_write_comments(OBIDMS_column_p column, const char* comments)

python/obitools3/dms/column/column.pyx

@@ -14,6 +14,7 @@ from ..capi.obidms cimport obi_import_column
 from ..capi.obidmscolumn cimport OBIDMS_column_header_p, \
                                  obi_close_column, \
                                  obi_get_elements_names, \
+                                 obi_column_formatted_infos, \
                                  obi_column_write_comments
                                      
 from ..capi.obiutils cimport obi_format_date
@@ -288,7 +289,13 @@ cdef class Column(OBIWrapper) :
     @OBIWrapper.checkIsActive
     def __repr__(self) :
         cdef bytes s
+        #cdef char* s_b
+        #cdef str s_str
+        #s_b = obi_column_formatted_infos(self.pointer())
+        #s_str = bytes2str(s_b)
+        #free(s_b)
         s = self._alias + b", data type: " + self.data_type
+        #return s_str
         return bytes2str(s)
     
          

python/obitools3/parsers/embl.pyx

@@ -177,7 +177,7 @@ def emblIterator_dir(dir_path,
     for filename in files:
         if read==only:
             return
-        print("Parsing file %s (%d/%d)" % (tostr(filename), read_files, len(files)))
+        print("Parsing file %s (%d/%d)" % (tostr(filename), read_files+1, len(files)))
         f = uopen(filename)
         if only is not None:
             only_f = only-read

python/obitools3/parsers/fasta.pyx

@@ -104,6 +104,7 @@ def fastaNucIterator(lineiterator,
     cdef bytes      sequence
     cdef int        skipped, ionly, read
     cdef Nuc_Seq    seq
+    cdef bint       stop
     
     if only is None:
         ionly = -1
@@ -130,7 +131,8 @@ def fastaNucIterator(lineiterator,
     else:
         line = firstline       
     
-    while True:
+    stop=False
+    while not stop:
                 
         if ionly >= 0 and read >= ionly:
             break
@@ -153,7 +155,7 @@ def fastaNucIterator(lineiterator,
                 s.append(line[0:-1])
                 line = next(iterator)
         except StopIteration:
-            pass
+            stop=True
         
         sequence  = b"".join(s)        
         

python/obitools3/parsers/genbank.pyx

@@ -25,8 +25,9 @@ from libc.string cimport strcpy, strlen
 _featureMatcher = re.compile(b'^FEATURES.+\n(?=ORIGIN)',re.DOTALL + re.M)
 
 _headerMatcher = re.compile(b'^LOCUS.+(?=\nFEATURES)', re.DOTALL + re.M)
-_seqMatcher    = re.compile(b'(?<=ORIGIN).+(?=//\n)', re.DOTALL + re.M)
+_seqMatcher    = re.compile(b'ORIGIN.+(?=//\n)', re.DOTALL + re.M)
-_cleanSeq      = re.compile(b'[ \n0-9]+')
+_cleanSeq1     = re.compile(b'ORIGIN.+\n')
+_cleanSeq2     = re.compile(b'[ \n0-9]+')
 _acMatcher     = re.compile(b'(?<=^ACCESSION   ).+',re.M)
 _deMatcher     = re.compile(b'(?<=^DEFINITION  ).+\n( .+\n)*',re.M)
 _cleanDe       = re.compile(b'\n *')
@@ -42,7 +43,8 @@ def genbankParser(bytes text):
         ft     = _featureMatcher.search(text).group()
         
         s      = _seqMatcher.search(text).group()
-        s      = _cleanSeq.sub(b'', s).upper()
+        s      = _cleanSeq1.sub(b'', s)
+        s      = _cleanSeq2.sub(b'', s)
         
         acs    = _acMatcher.search(text).group()
         acs    = acs.split()
@@ -52,12 +54,6 @@ def genbankParser(bytes text):
         de     = _deMatcher.search(header).group()
         de     = _cleanDe.sub(b' ',de).strip().strip(b'.')
     
-    except Exception as e:
-        print("\nCould not import sequence id:", text.split()[1], "(error raised:", e, ")")
-        # Do not raise any Exception if you need the possibility to resume the generator
-        # (Python generators can't resume after any exception is raised)
-        return None
-    
         tags = {}
         extractTaxon(ft, tags)
             
@@ -68,6 +64,12 @@ def genbankParser(bytes text):
                       offset=-1,
                       tags=tags)
     
+    except Exception as e:
+        print("\nCould not import sequence id:", text.split()[1], "(error raised:", e, ")")
+        # Do not raise any Exception if you need the possibility to resume the generator
+        # (Python generators can't resume after any exception is raised)
+        return None
+
     return seq
 
 
@@ -171,10 +173,12 @@ def genbankIterator_dir(dir_path,
     read = 0
     read_files = 0
     files = [filename for filename in glob.glob(os.path.join(path, b'*.gbff*'))]
+    files.extend([filename for filename in glob.glob(os.path.join(path, b'*.seq*'))])  # new genbank extension
+    files = list(set(files))
     for filename in files:
         if read==only:
             return
-        print("Parsing file %s (%d/%d)" % (tostr(filename), read_files, len(files)))
+        print("Parsing file %s (%d/%d)" % (tostr(filename), read_files+1, len(files)))
         f = uopen(filename)
         if only is not None:
             only_f = only-read

python/obitools3/version.py

@@ -1,5 +1,5 @@
 major = 3
 minor = 0
-serial= '0-beta16'
+serial= '0b23'
 
-version ="%d.%02d.%s" % (major,minor,serial)
+version ="%d.%d.%s" % (major,minor,serial)

requirements.txt

@@ -0,0 +1,5 @@
+--extra-index-url https://pypi.python.org/simple/
+Cython>=0.24
+Sphinx>=1.2.0
+ipython>=3.0.0
+breathe>=4.0.0

setup.py

@@ -5,7 +5,8 @@ import re
 import subprocess
 
 from distutils import log
-from distutils.core import setup
+#from distutils.core import setup
+from setuptools import setup    # to work with pip
 
 from distutils.core import Extension
 from distutils.sysconfig import get_python_lib
@@ -26,10 +27,11 @@ class Distribution(ori_Distribution):
         
         ori_Distribution.__init__(self, attrs)
         
-        self.global_options.insert(0,('cobitools3', None, "intall location of the C library"
+        self.global_options.insert(0,('cobitools3', None, "install location of the C library"
                                      ))
 
 from distutils.command.build import build as build_ori
+from setuptools.command.bdist_egg import bdist_egg as bdist_egg_ori
 from distutils.core import Command
 
 
@@ -70,6 +72,12 @@ class build(build_ori):
         build_ori.run(self)
 
 
+class bdist_egg(bdist_egg_ori):
+    def run(self):
+        self.run_command('build_clib')
+        bdist_egg_ori.run(self)
+
+
 sys.path.append(os.path.abspath("python"))
 
 
@@ -88,9 +96,10 @@ PACKAGE     = "OBITools3"
 VERSION     = version
 AUTHOR      = 'Celine Mercier'
 EMAIL       = 'celine.mercier@metabarcoding.org'
-URL         = "http://metabarcoding.org/obitools3"
+URL         = "https://metabarcoding.org/obitools3"
+PLATFORMS   = "posix"
 LICENSE     = "CeCILL-V2"
-DESCRIPTION = "Tools and library for DNA metabarcoding",
+DESCRIPTION = "A package for the management of analyses and data in DNA metabarcoding."
 PYTHONMIN   = '3.5'
 
 SRC       = 'python'
@@ -147,17 +156,24 @@ classifiers=['Development Status :: 4 - Beta',
              'Topic :: Utilities',
              ]
 
+with open("README.md", "r") as fh:
+    long_description = fh.read()
+
 setup(name=PACKAGE,
       description=DESCRIPTION,
+      long_description=long_description,
+      long_description_content_type="text/markdown",
       classifiers=classifiers,
       version=VERSION,
       author=AUTHOR,
       author_email=EMAIL,
+      platforms=PLATFORMS,
       license=LICENSE,
       url=URL,
       ext_modules=xx,
       distclass=Distribution,
       cmdclass={'build': build,
+                'bdist_egg': bdist_egg,
                 'build_clib': build_clib},
       cobitools3=get_python_lib(),
       packages = findPackage('python'),