Abandoned beginning of functions to apply comparisons to whole columns

(not worth it for now as comparisons are already fast enough)

.gitignore

@@ -1,13 +0,0 @@
-.DS_Store
-/obitools.test/
-/OBITools3- 1.01.16/
-/OBITools3-0.00.0/
-/build/
-/testpip/
-/toto.obidms/
-/unittests/
-/setup.py.old
-/3_Hornung_testseq.fastq
-/sample/
-/save.temp/
-/obitmp/

.project

@@ -1,17 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
-	<name>obitools3</name>
-	<comment></comment>
-	<projects>
-	</projects>
-	<buildSpec>
-		<buildCommand>
-			<name>org.python.pydev.PyDevBuilder</name>
-			<arguments>
-			</arguments>
-		</buildCommand>
-	</buildSpec>
-	<natures>
-		<nature>org.python.pydev.pythonNature</nature>
-	</natures>
-</projectDescription>

.pydevproject

@@ -1,20 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<?eclipse-pydev version="1.0"?><pydev_project>
-        
-    
-    <pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">python3.6-obitool3</pydev_property>
-        
-    
-    <pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python interpreter</pydev_property>
-        
-    
-    <pydev_pathproperty name="org.python.pydev.PROJECT_SOURCE_PATH">
-                
-        
-        <path>/${PROJECT_DIR_NAME}/python</path>
-                
-    
-    </pydev_pathproperty>
-        
-
-</pydev_project>

Licence_CeCILL_V2-en.txt

@@ -0,0 +1,506 @@
+
+CeCILL FREE SOFTWARE LICENSE AGREEMENT
+
+
+    Notice
+
+This Agreement is a Free Software license agreement that is the result
+of discussions between its authors in order to ensure compliance with
+the two main principles guiding its drafting:
+
+    * firstly, compliance with the principles governing the distribution
+      of Free Software: access to source code, broad rights granted to
+      users,
+    * secondly, the election of a governing law, French law, with which
+      it is conformant, both as regards the law of torts and
+      intellectual property law, and the protection that it offers to
+      both authors and holders of the economic rights over software.
+
+The authors of the CeCILL (for Ce[a] C[nrs] I[nria] L[ogiciel] L[ibre])
+license are:
+
+Commissariat à l'Energie Atomique - CEA, a public scientific, technical
+and industrial research establishment, having its principal place of
+business at 25 rue Leblanc, immeuble Le Ponant D, 75015 Paris, France.
+
+Centre National de la Recherche Scientifique - CNRS, a public scientific
+and technological establishment, having its principal place of business
+at 3 rue Michel-Ange, 75794 Paris cedex 16, France.
+
+Institut National de Recherche en Informatique et en Automatique -
+INRIA, a public scientific and technological establishment, having its
+principal place of business at Domaine de Voluceau, Rocquencourt, BP
+105, 78153 Le Chesnay cedex, France.
+
+
+    Preamble
+
+The purpose of this Free Software license agreement is to grant users
+the right to modify and redistribute the software governed by this
+license within the framework of an open source distribution model.
+
+The exercising of these rights is conditional upon certain obligations
+for users so as to preserve this status for all subsequent redistributions.
+
+In consideration of access to the source code and the rights to copy,
+modify and redistribute granted by the license, users are provided only
+with a limited warranty and the software's author, the holder of the
+economic rights, and the successive licensors only have limited liability.
+
+In this respect, the risks associated with loading, using, modifying
+and/or developing or reproducing the software by the user are brought to
+the user's attention, given its Free Software status, which may make it
+complicated to use, with the result that its use is reserved for
+developers and experienced professionals having in-depth computer
+knowledge. Users are therefore encouraged to load and test the
+suitability of the software as regards their requirements in conditions
+enabling the security of their systems and/or data to be ensured and,
+more generally, to use and operate it in the same conditions of
+security. This Agreement may be freely reproduced and published,
+provided it is not altered, and that no provisions are either added or
+removed herefrom.
+
+This Agreement may apply to any or all software for which the holder of
+the economic rights decides to submit the use thereof to its provisions.
+
+
+    Article 1 - DEFINITIONS
+
+For the purpose of this Agreement, when the following expressions
+commence with a capital letter, they shall have the following meaning:
+
+Agreement: means this license agreement, and its possible subsequent
+versions and annexes.
+
+Software: means the software in its Object Code and/or Source Code form
+and, where applicable, its documentation, "as is" when the Licensee
+accepts the Agreement.
+
+Initial Software: means the Software in its Source Code and possibly its
+Object Code form and, where applicable, its documentation, "as is" when
+it is first distributed under the terms and conditions of the Agreement.
+
+Modified Software: means the Software modified by at least one
+Contribution.
+
+Source Code: means all the Software's instructions and program lines to
+which access is required so as to modify the Software.
+
+Object Code: means the binary files originating from the compilation of
+the Source Code.
+
+Holder: means the holder(s) of the economic rights over the Initial
+Software.
+
+Licensee: means the Software user(s) having accepted the Agreement.
+
+Contributor: means a Licensee having made at least one Contribution.
+
+Licensor: means the Holder, or any other individual or legal entity, who
+distributes the Software under the Agreement.
+
+Contribution: means any or all modifications, corrections, translations,
+adaptations and/or new functions integrated into the Software by any or
+all Contributors, as well as any or all Internal Modules.
+
+Module: means a set of sources files including their documentation that
+enables supplementary functions or services in addition to those offered
+by the Software.
+
+External Module: means any or all Modules, not derived from the
+Software, so that this Module and the Software run in separate address
+spaces, with one calling the other when they are run.
+
+Internal Module: means any or all Module, connected to the Software so
+that they both execute in the same address space.
+
+GNU GPL: means the GNU General Public License version 2 or any
+subsequent version, as published by the Free Software Foundation Inc.
+
+Parties: mean both the Licensee and the Licensor.
+
+These expressions may be used both in singular and plural form.
+
+
+    Article 2 - PURPOSE
+
+The purpose of the Agreement is the grant by the Licensor to the
+Licensee of a non-exclusive, transferable and worldwide license for the
+Software as set forth in Article 5 hereinafter for the whole term of the
+protection granted by the rights over said Software. 
+
+
+    Article 3 - ACCEPTANCE
+
+3.1 The Licensee shall be deemed as having accepted the terms and
+conditions of this Agreement upon the occurrence of the first of the
+following events:
+
+    * (i) loading the Software by any or all means, notably, by
+      downloading from a remote server, or by loading from a physical
+      medium;
+    * (ii) the first time the Licensee exercises any of the rights
+      granted hereunder.
+
+3.2 One copy of the Agreement, containing a notice relating to the
+characteristics of the Software, to the limited warranty, and to the
+fact that its use is restricted to experienced users has been provided
+to the Licensee prior to its acceptance as set forth in Article 3.1
+hereinabove, and the Licensee hereby acknowledges that it has read and
+understood it.
+
+
+    Article 4 - EFFECTIVE DATE AND TERM
+
+
+      4.1 EFFECTIVE DATE
+
+The Agreement shall become effective on the date when it is accepted by
+the Licensee as set forth in Article 3.1.
+
+
+      4.2 TERM
+
+The Agreement shall remain in force for the entire legal term of
+protection of the economic rights over the Software.
+
+
+    Article 5 - SCOPE OF RIGHTS GRANTED
+
+The Licensor hereby grants to the Licensee, who accepts, the following
+rights over the Software for any or all use, and for the term of the
+Agreement, on the basis of the terms and conditions set forth hereinafter.
+
+Besides, if the Licensor owns or comes to own one or more patents
+protecting all or part of the functions of the Software or of its
+components, the Licensor undertakes not to enforce the rights granted by
+these patents against successive Licensees using, exploiting or
+modifying the Software. If these patents are transferred, the Licensor
+undertakes to have the transferees subscribe to the obligations set
+forth in this paragraph.
+
+
+      5.1 RIGHT OF USE
+
+The Licensee is authorized to use the Software, without any limitation
+as to its fields of application, with it being hereinafter specified
+that this comprises:
+
+   1. permanent or temporary reproduction of all or part of the Software
+      by any or all means and in any or all form.
+
+   2. loading, displaying, running, or storing the Software on any or
+      all medium.
+
+   3. entitlement to observe, study or test its operation so as to
+      determine the ideas and principles behind any or all constituent
+      elements of said Software. This shall apply when the Licensee
+      carries out any or all loading, displaying, running, transmission
+      or storage operation as regards the Software, that it is entitled
+      to carry out hereunder.
+
+
+      5.2 ENTITLEMENT TO MAKE CONTRIBUTIONS
+
+The right to make Contributions includes the right to translate, adapt,
+arrange, or make any or all modifications to the Software, and the right
+to reproduce the resulting software.
+
+The Licensee is authorized to make any or all Contributions to the
+Software provided that it includes an explicit notice that it is the
+author of said Contribution and indicates the date of the creation thereof.
+
+
+      5.3 RIGHT OF DISTRIBUTION
+
+In particular, the right of distribution includes the right to publish,
+transmit and communicate the Software to the general public on any or
+all medium, and by any or all means, and the right to market, either in
+consideration of a fee, or free of charge, one or more copies of the
+Software by any means.
+
+The Licensee is further authorized to distribute copies of the modified
+or unmodified Software to third parties according to the terms and
+conditions set forth hereinafter.
+
+
+        5.3.1 DISTRIBUTION OF SOFTWARE WITHOUT MODIFICATION
+
+The Licensee is authorized to distribute true copies of the Software in
+Source Code or Object Code form, provided that said distribution
+complies with all the provisions of the Agreement and is accompanied by:
+
+   1. a copy of the Agreement,
+
+   2. a notice relating to the limitation of both the Licensor's
+      warranty and liability as set forth in Articles 8 and 9,
+
+and that, in the event that only the Object Code of the Software is
+redistributed, the Licensee allows future Licensees unhindered access to
+the full Source Code of the Software by indicating how to access it, it
+being understood that the additional cost of acquiring the Source Code
+shall not exceed the cost of transferring the data.
+
+
+        5.3.2 DISTRIBUTION OF MODIFIED SOFTWARE
+
+When the Licensee makes a Contribution to the Software, the terms and
+conditions for the distribution of the resulting Modified Software
+become subject to all the provisions of this Agreement.
+
+The Licensee is authorized to distribute the Modified Software, in
+source code or object code form, provided that said distribution
+complies with all the provisions of the Agreement and is accompanied by:
+
+   1. a copy of the Agreement,
+
+   2. a notice relating to the limitation of both the Licensor's
+      warranty and liability as set forth in Articles 8 and 9,
+
+and that, in the event that only the object code of the Modified
+Software is redistributed, the Licensee allows future Licensees
+unhindered access to the full source code of the Modified Software by
+indicating how to access it, it being understood that the additional
+cost of acquiring the source code shall not exceed the cost of
+transferring the data.
+
+
+        5.3.3 DISTRIBUTION OF EXTERNAL MODULES
+
+When the Licensee has developed an External Module, the terms and
+conditions of this Agreement do not apply to said External Module, that
+may be distributed under a separate license agreement.
+
+
+        5.3.4 COMPATIBILITY WITH THE GNU GPL
+
+The Licensee can include a code that is subject to the provisions of one
+of the versions of the GNU GPL in the Modified or unmodified Software,
+and distribute that entire code under the terms of the same version of
+the GNU GPL.
+
+The Licensee can include the Modified or unmodified Software in a code
+that is subject to the provisions of one of the versions of the GNU GPL,
+and distribute that entire code under the terms of the same version of
+the GNU GPL.
+
+
+    Article 6 - INTELLECTUAL PROPERTY
+
+
+      6.1 OVER THE INITIAL SOFTWARE
+
+The Holder owns the economic rights over the Initial Software. Any or
+all use of the Initial Software is subject to compliance with the terms
+and conditions under which the Holder has elected to distribute its work
+and no one shall be entitled to modify the terms and conditions for the
+distribution of said Initial Software.
+
+The Holder undertakes that the Initial Software will remain ruled at
+least by this Agreement, for the duration set forth in Article 4.2.
+
+
+      6.2 OVER THE CONTRIBUTIONS
+
+The Licensee who develops a Contribution is the owner of the
+intellectual property rights over this Contribution as defined by
+applicable law.
+
+
+      6.3 OVER THE EXTERNAL MODULES
+
+The Licensee who develops an External Module is the owner of the
+intellectual property rights over this External Module as defined by
+applicable law and is free to choose the type of agreement that shall
+govern its distribution.
+
+
+      6.4 JOINT PROVISIONS
+
+The Licensee expressly undertakes:
+
+   1. not to remove, or modify, in any manner, the intellectual property
+      notices attached to the Software;
+
+   2. to reproduce said notices, in an identical manner, in the copies
+      of the Software modified or not.
+
+The Licensee undertakes not to directly or indirectly infringe the
+intellectual property rights of the Holder and/or Contributors on the
+Software and to take, where applicable, vis-à-vis its staff, any and all
+measures required to ensure respect of said intellectual property rights
+of the Holder and/or Contributors.
+
+
+    Article 7 - RELATED SERVICES
+
+7.1 Under no circumstances shall the Agreement oblige the Licensor to
+provide technical assistance or maintenance services for the Software.
+
+However, the Licensor is entitled to offer this type of services. The
+terms and conditions of such technical assistance, and/or such
+maintenance, shall be set forth in a separate instrument. Only the
+Licensor offering said maintenance and/or technical assistance services
+shall incur liability therefor.
+
+7.2 Similarly, any Licensor is entitled to offer to its licensees, under
+its sole responsibility, a warranty, that shall only be binding upon
+itself, for the redistribution of the Software and/or the Modified
+Software, under terms and conditions that it is free to decide. Said
+warranty, and the financial terms and conditions of its application,
+shall be subject of a separate instrument executed between the Licensor
+and the Licensee.
+
+
+    Article 8 - LIABILITY
+
+8.1 Subject to the provisions of Article 8.2, the Licensee shall be
+entitled to claim compensation for any direct loss it may have suffered
+from the Software as a result of a fault on the part of the relevant
+Licensor, subject to providing evidence thereof.
+
+8.2 The Licensor's liability is limited to the commitments made under
+this Agreement and shall not be incurred as a result of in particular:
+(i) loss due the Licensee's total or partial failure to fulfill its
+obligations, (ii) direct or consequential loss that is suffered by the
+Licensee due to the use or performance of the Software, and (iii) more
+generally, any consequential loss. In particular the Parties expressly
+agree that any or all pecuniary or business loss (i.e. loss of data,
+loss of profits, operating loss, loss of customers or orders,
+opportunity cost, any disturbance to business activities) or any or all
+legal proceedings instituted against the Licensee by a third party,
+shall constitute consequential loss and shall not provide entitlement to
+any or all compensation from the Licensor.
+
+
+    Article 9 - WARRANTY
+
+9.1 The Licensee acknowledges that the scientific and technical
+state-of-the-art when the Software was distributed did not enable all
+possible uses to be tested and verified, nor for the presence of
+possible defects to be detected. In this respect, the Licensee's
+attention has been drawn to the risks associated with loading, using,
+modifying and/or developing and reproducing the Software which are
+reserved for experienced users.
+
+The Licensee shall be responsible for verifying, by any or all means,
+the suitability of the product for its requirements, its good working
+order, and for ensuring that it shall not cause damage to either persons
+or properties.
+
+9.2 The Licensor hereby represents, in good faith, that it is entitled
+to grant all the rights over the Software (including in particular the
+rights set forth in Article 5).
+
+9.3 The Licensee acknowledges that the Software is supplied "as is" by
+the Licensor without any other express or tacit warranty, other than
+that provided for in Article 9.2 and, in particular, without any warranty 
+as to its commercial value, its secured, safe, innovative or relevant
+nature.
+
+Specifically, the Licensor does not warrant that the Software is free
+from any error, that it will operate without interruption, that it will
+be compatible with the Licensee's own equipment and software
+configuration, nor that it will meet the Licensee's requirements.
+
+9.4 The Licensor does not either expressly or tacitly warrant that the
+Software does not infringe any third party intellectual property right
+relating to a patent, software or any other property right. Therefore,
+the Licensor disclaims any and all liability towards the Licensee
+arising out of any or all proceedings for infringement that may be
+instituted in respect of the use, modification and redistribution of the
+Software. Nevertheless, should such proceedings be instituted against
+the Licensee, the Licensor shall provide it with technical and legal
+assistance for its defense. Such technical and legal assistance shall be
+decided on a case-by-case basis between the relevant Licensor and the
+Licensee pursuant to a memorandum of understanding. The Licensor
+disclaims any and all liability as regards the Licensee's use of the
+name of the Software. No warranty is given as regards the existence of
+prior rights over the name of the Software or as regards the existence
+of a trademark.
+
+
+    Article 10 - TERMINATION
+
+10.1 In the event of a breach by the Licensee of its obligations
+hereunder, the Licensor may automatically terminate this Agreement
+thirty (30) days after notice has been sent to the Licensee and has
+remained ineffective.
+
+10.2 A Licensee whose Agreement is terminated shall no longer be
+authorized to use, modify or distribute the Software. However, any
+licenses that it may have granted prior to termination of the Agreement
+shall remain valid subject to their having been granted in compliance
+with the terms and conditions hereof.
+
+
+    Article 11 - MISCELLANEOUS
+
+
+      11.1 EXCUSABLE EVENTS
+
+Neither Party shall be liable for any or all delay, or failure to
+perform the Agreement, that may be attributable to an event of force
+majeure, an act of God or an outside cause, such as defective
+functioning or interruptions of the electricity or telecommunications
+networks, network paralysis following a virus attack, intervention by
+government authorities, natural disasters, water damage, earthquakes,
+fire, explosions, strikes and labor unrest, war, etc.
+
+11.2 Any failure by either Party, on one or more occasions, to invoke
+one or more of the provisions hereof, shall under no circumstances be
+interpreted as being a waiver by the interested Party of its right to
+invoke said provision(s) subsequently.
+
+11.3 The Agreement cancels and replaces any or all previous agreements,
+whether written or oral, between the Parties and having the same
+purpose, and constitutes the entirety of the agreement between said
+Parties concerning said purpose. No supplement or modification to the
+terms and conditions hereof shall be effective as between the Parties
+unless it is made in writing and signed by their duly authorized
+representatives.
+
+11.4 In the event that one or more of the provisions hereof were to
+conflict with a current or future applicable act or legislative text,
+said act or legislative text shall prevail, and the Parties shall make
+the necessary amendments so as to comply with said act or legislative
+text. All other provisions shall remain effective. Similarly, invalidity
+of a provision of the Agreement, for any reason whatsoever, shall not
+cause the Agreement as a whole to be invalid.
+
+
+      11.5 LANGUAGE
+
+The Agreement is drafted in both French and English and both versions
+are deemed authentic.
+
+
+    Article 12 - NEW VERSIONS OF THE AGREEMENT
+
+12.1 Any person is authorized to duplicate and distribute copies of this
+Agreement.
+
+12.2 So as to ensure coherence, the wording of this Agreement is
+protected and may only be modified by the authors of the License, who
+reserve the right to periodically publish updates or new versions of the
+Agreement, each with a separate number. These subsequent versions may
+address new issues encountered by Free Software.
+
+12.3 Any Software distributed under a given version of the Agreement may
+only be subsequently distributed under the same version of the Agreement
+or a subsequent version, subject to the provisions of Article 5.3.4.
+
+
+    Article 13 - GOVERNING LAW AND JURISDICTION
+
+13.1 The Agreement is governed by French law. The Parties agree to
+endeavor to seek an amicable solution to any disagreements or disputes
+that may arise during the performance of the Agreement.
+
+13.2 Failing an amicable solution within two (2) months as from their
+occurrence, and unless emergency proceedings are necessary, the
+disagreements or disputes shall be referred to the Paris Courts having
+jurisdiction, by the more diligent Party.
+
+
+Version 2.0 dated 2006-09-05.

Licence_CeCILL_V2-fr.txt

@@ -0,0 +1,512 @@
+
+CONTRAT DE LICENCE DE LOGICIEL LIBRE CeCILL
+
+
+    Avertissement
+
+Ce contrat est une licence de logiciel libre issue d'une concertation
+entre ses auteurs afin que le respect de deux grands principes préside à
+sa rédaction:
+
+    * d'une part, le respect des principes de diffusion des logiciels
+      libres: accès au code source, droits étendus conférés aux
+      utilisateurs,
+    * d'autre part, la désignation d'un droit applicable, le droit
+      français, auquel elle est conforme, tant au regard du droit de la
+      responsabilité civile que du droit de la propriété intellectuelle
+      et de la protection qu'il offre aux auteurs et titulaires des
+      droits patrimoniaux sur un logiciel.
+
+Les auteurs de la licence CeCILL (pour Ce[a] C[nrs] I[nria] L[ogiciel]
+L[ibre]) sont:
+
+Commissariat à l'Energie Atomique - CEA, établissement public de
+recherche à caractère scientifique, technique et industriel, dont le
+siège est situé 25 rue Leblanc, immeuble Le Ponant D, 75015 Paris.
+
+Centre National de la Recherche Scientifique - CNRS, établissement
+public à caractère scientifique et technologique, dont le siège est
+situé 3 rue Michel-Ange, 75794 Paris cedex 16.
+
+Institut National de Recherche en Informatique et en Automatique -
+INRIA, établissement public à caractère scientifique et technologique,
+dont le siège est situé Domaine de Voluceau, Rocquencourt, BP 105, 78153
+Le Chesnay cedex.
+
+
+    Préambule
+
+Ce contrat est une licence de logiciel libre dont l'objectif est de
+conférer aux utilisateurs la liberté de modification et de
+redistribution du logiciel régi par cette licence dans le cadre d'un
+modèle de diffusion en logiciel libre.
+
+L'exercice de ces libertés est assorti de certains devoirs à la charge
+des utilisateurs afin de préserver ce statut au cours des
+redistributions ultérieures.
+
+L'accessibilité au code source et les droits de copie, de modification
+et de redistribution qui en découlent ont pour contrepartie de n'offrir
+aux utilisateurs qu'une garantie limitée et de ne faire peser sur
+l'auteur du logiciel, le titulaire des droits patrimoniaux et les
+concédants successifs qu'une responsabilité restreinte.
+
+A cet égard l'attention de l'utilisateur est attirée sur les risques
+associés au chargement, à l'utilisation, à la modification et/ou au
+développement et à la reproduction du logiciel par l'utilisateur étant
+donné sa spécificité de logiciel libre, qui peut le rendre complexe à
+manipuler et qui le réserve donc à des développeurs ou des
+professionnels avertis possédant des connaissances informatiques
+approfondies. Les utilisateurs sont donc invités à charger et tester
+l'adéquation du logiciel à leurs besoins dans des conditions permettant
+d'assurer la sécurité de leurs systèmes et/ou de leurs données et, plus
+généralement, à l'utiliser et l'exploiter dans les mêmes conditions de
+sécurité. Ce contrat peut être reproduit et diffusé librement, sous
+réserve de le conserver en l'état, sans ajout ni suppression de clauses.
+
+Ce contrat est susceptible de s'appliquer à tout logiciel dont le
+titulaire des droits patrimoniaux décide de soumettre l'exploitation aux
+dispositions qu'il contient.
+
+
+    Article 1 - DEFINITIONS
+
+Dans ce contrat, les termes suivants, lorsqu'ils seront écrits avec une
+lettre capitale, auront la signification suivante:
+
+Contrat: désigne le présent contrat de licence, ses éventuelles versions
+postérieures et annexes.
+
+Logiciel: désigne le logiciel sous sa forme de Code Objet et/ou de Code
+Source et le cas échéant sa documentation, dans leur état au moment de
+l'acceptation du Contrat par le Licencié.
+
+Logiciel Initial: désigne le Logiciel sous sa forme de Code Source et
+éventuellement de Code Objet et le cas échéant sa documentation, dans
+leur état au moment de leur première diffusion sous les termes du Contrat.
+
+Logiciel Modifié: désigne le Logiciel modifié par au moins une
+Contribution.
+
+Code Source: désigne l'ensemble des instructions et des lignes de
+programme du Logiciel et auquel l'accès est nécessaire en vue de
+modifier le Logiciel.
+
+Code Objet: désigne les fichiers binaires issus de la compilation du
+Code Source.
+
+Titulaire: désigne le ou les détenteurs des droits patrimoniaux d'auteur
+sur le Logiciel Initial.
+
+Licencié: désigne le ou les utilisateurs du Logiciel ayant accepté le
+Contrat.
+
+Contributeur: désigne le Licencié auteur d'au moins une Contribution.
+
+Concédant: désigne le Titulaire ou toute personne physique ou morale
+distribuant le Logiciel sous le Contrat.
+
+Contribution: désigne l'ensemble des modifications, corrections,
+traductions, adaptations et/ou nouvelles fonctionnalités intégrées dans
+le Logiciel par tout Contributeur, ainsi que tout Module Interne.
+
+Module: désigne un ensemble de fichiers sources y compris leur
+documentation qui permet de réaliser des fonctionnalités ou services
+supplémentaires à ceux fournis par le Logiciel.
+
+Module Externe: désigne tout Module, non dérivé du Logiciel, tel que ce
+Module et le Logiciel s'exécutent dans des espaces d'adressage
+différents, l'un appelant l'autre au moment de leur exécution.
+
+Module Interne: désigne tout Module lié au Logiciel de telle sorte
+qu'ils s'exécutent dans le même espace d'adressage.
+
+GNU GPL: désigne la GNU General Public License dans sa version 2 ou
+toute version ultérieure, telle que publiée par Free Software Foundation
+Inc.
+
+Parties: désigne collectivement le Licencié et le Concédant.
+
+Ces termes s'entendent au singulier comme au pluriel.
+
+
+    Article 2 - OBJET
+
+Le Contrat a pour objet la concession par le Concédant au Licencié d'une
+licence non exclusive, cessible et mondiale du Logiciel telle que
+définie ci-après à l'article 5 pour toute la durée de protection des droits
+portant sur ce Logiciel.
+
+
+    Article 3 - ACCEPTATION
+
+3.1 L'acceptation par le Licencié des termes du Contrat est réputée
+acquise du fait du premier des faits suivants:
+
+    * (i) le chargement du Logiciel par tout moyen notamment par
+      téléchargement à partir d'un serveur distant ou par chargement à
+      partir d'un support physique;
+    * (ii) le premier exercice par le Licencié de l'un quelconque des
+      droits concédés par le Contrat.
+
+3.2 Un exemplaire du Contrat, contenant notamment un avertissement
+relatif aux spécificités du Logiciel, à la restriction de garantie et à
+la limitation à un usage par des utilisateurs expérimentés a été mis à
+disposition du Licencié préalablement à son acceptation telle que
+définie à l'article 3.1 ci dessus et le Licencié reconnaît en avoir pris
+connaissance.
+
+
+    Article 4 - ENTREE EN VIGUEUR ET DUREE
+
+
+      4.1 ENTREE EN VIGUEUR
+
+Le Contrat entre en vigueur à la date de son acceptation par le Licencié
+telle que définie en 3.1.
+
+
+      4.2 DUREE
+
+Le Contrat produira ses effets pendant toute la durée légale de
+protection des droits patrimoniaux portant sur le Logiciel.
+
+
+    Article 5 - ETENDUE DES DROITS CONCEDES
+
+Le Concédant concède au Licencié, qui accepte, les droits suivants sur
+le Logiciel pour toutes destinations et pour la durée du Contrat dans
+les conditions ci-après détaillées.
+
+Par ailleurs, si le Concédant détient ou venait à détenir un ou
+plusieurs brevets d'invention protégeant tout ou partie des
+fonctionnalités du Logiciel ou de ses composants, il s'engage à ne pas
+opposer les éventuels droits conférés par ces brevets aux Licenciés
+successifs qui utiliseraient, exploiteraient ou modifieraient le
+Logiciel. En cas de cession de ces brevets, le Concédant s'engage à
+faire reprendre les obligations du présent alinéa aux cessionnaires.
+
+
+      5.1 DROIT D'UTILISATION
+
+Le Licencié est autorisé à utiliser le Logiciel, sans restriction quant
+aux domaines d'application, étant ci-après précisé que cela comporte:
+
+   1. la reproduction permanente ou provisoire du Logiciel en tout ou
+      partie par tout moyen et sous toute forme.
+
+   2. le chargement, l'affichage, l'exécution, ou le stockage du
+      Logiciel sur tout support.
+
+   3. la possibilité d'en observer, d'en étudier, ou d'en tester le
+      fonctionnement afin de déterminer les idées et principes qui sont
+      à la base de n'importe quel élément de ce Logiciel; et ceci,
+      lorsque le Licencié effectue toute opération de chargement,
+      d'affichage, d'exécution, de transmission ou de stockage du
+      Logiciel qu'il est en droit d'effectuer en vertu du Contrat.
+
+
+      5.2 DROIT D'APPORTER DES CONTRIBUTIONS
+
+Le droit d'apporter des Contributions comporte le droit de traduire,
+d'adapter, d'arranger ou d'apporter toute autre modification au Logiciel
+et le droit de reproduire le logiciel en résultant.
+
+Le Licencié est autorisé à apporter toute Contribution au Logiciel sous
+réserve de mentionner, de façon explicite, son nom en tant qu'auteur de
+cette Contribution et la date de création de celle-ci.
+
+
+      5.3 DROIT DE DISTRIBUTION
+
+Le droit de distribution comporte notamment le droit de diffuser, de
+transmettre et de communiquer le Logiciel au public sur tout support et
+par tout moyen ainsi que le droit de mettre sur le marché à titre
+onéreux ou gratuit, un ou des exemplaires du Logiciel par tout procédé.
+
+Le Licencié est autorisé à distribuer des copies du Logiciel, modifié ou
+non, à des tiers dans les conditions ci-après détaillées.
+
+
+        5.3.1 DISTRIBUTION DU LOGICIEL SANS MODIFICATION
+
+Le Licencié est autorisé à distribuer des copies conformes du Logiciel,
+sous forme de Code Source ou de Code Objet, à condition que cette
+distribution respecte les dispositions du Contrat dans leur totalité et
+soit accompagnée:
+
+   1. d'un exemplaire du Contrat,
+
+   2. d'un avertissement relatif à la restriction de garantie et de
+      responsabilité du Concédant telle que prévue aux articles 8
+      et 9,
+
+et que, dans le cas où seul le Code Objet du Logiciel est redistribué,
+le Licencié permette aux futurs Licenciés d'accéder facilement au Code
+Source complet du Logiciel en indiquant les modalités d'accès, étant
+entendu que le coût additionnel d'acquisition du Code Source ne devra
+pas excéder le simple coût de transfert des données.
+
+
+        5.3.2 DISTRIBUTION DU LOGICIEL MODIFIE
+
+Lorsque le Licencié apporte une Contribution au Logiciel, les conditions
+de distribution du Logiciel Modifié en résultant sont alors soumises à
+l'intégralité des dispositions du Contrat.
+
+Le Licencié est autorisé à distribuer le Logiciel Modifié, sous forme de
+code source ou de code objet, à condition que cette distribution
+respecte les dispositions du Contrat dans leur totalité et soit
+accompagnée:
+
+   1. d'un exemplaire du Contrat,
+
+   2. d'un avertissement relatif à la restriction de garantie et de
+      responsabilité du Concédant telle que prévue aux articles 8
+      et 9,
+
+et que, dans le cas où seul le code objet du Logiciel Modifié est
+redistribué, le Licencié permette aux futurs Licenciés d'accéder
+facilement au code source complet du Logiciel Modifié en indiquant les
+modalités d'accès, étant entendu que le coût additionnel d'acquisition
+du code source ne devra pas excéder le simple coût de transfert des données.
+
+
+        5.3.3 DISTRIBUTION DES MODULES EXTERNES
+
+Lorsque le Licencié a développé un Module Externe les conditions du
+Contrat ne s'appliquent pas à ce Module Externe, qui peut être distribué
+sous un contrat de licence différent.
+
+
+        5.3.4 COMPATIBILITE AVEC LA LICENCE GNU GPL
+
+Le Licencié peut inclure un code soumis aux dispositions d'une des
+versions de la licence GNU GPL dans le Logiciel modifié ou non et
+distribuer l'ensemble sous les conditions de la même version de la
+licence GNU GPL.
+
+Le Licencié peut inclure le Logiciel modifié ou non dans un code soumis
+aux dispositions d'une des versions de la licence GNU GPL et distribuer
+l'ensemble sous les conditions de la même version de la licence GNU GPL.
+
+
+    Article 6 - PROPRIETE INTELLECTUELLE
+
+
+      6.1 SUR LE LOGICIEL INITIAL
+
+Le Titulaire est détenteur des droits patrimoniaux sur le Logiciel
+Initial. Toute utilisation du Logiciel Initial est soumise au respect
+des conditions dans lesquelles le Titulaire a choisi de diffuser son
+oeuvre et nul autre n'a la faculté de modifier les conditions de
+diffusion de ce Logiciel Initial.
+
+Le Titulaire s'engage à ce que le Logiciel Initial reste au moins régi
+par le Contrat et ce, pour la durée visée à l'article 4.2.
+
+
+      6.2 SUR LES CONTRIBUTIONS
+
+Le Licencié qui a développé une Contribution est titulaire sur celle-ci
+des droits de propriété intellectuelle dans les conditions définies par
+la législation applicable.
+
+
+      6.3 SUR LES MODULES EXTERNES
+
+Le Licencié qui a développé un Module Externe est titulaire sur celui-ci
+des droits de propriété intellectuelle dans les conditions définies par
+la législation applicable et reste libre du choix du contrat régissant
+sa diffusion.
+
+
+      6.4 DISPOSITIONS COMMUNES
+
+Le Licencié s'engage expressément:
+
+   1. à ne pas supprimer ou modifier de quelque manière que ce soit les
+      mentions de propriété intellectuelle apposées sur le Logiciel;
+
+   2. à reproduire à l'identique lesdites mentions de propriété
+      intellectuelle sur les copies du Logiciel modifié ou non.
+
+Le Licencié s'engage à ne pas porter atteinte, directement ou
+indirectement, aux droits de propriété intellectuelle du Titulaire et/ou
+des Contributeurs sur le Logiciel et à prendre, le cas échéant, à
+l'égard de son personnel toutes les mesures nécessaires pour assurer le
+respect des dits droits de propriété intellectuelle du Titulaire et/ou
+des Contributeurs.
+
+
+    Article 7 - SERVICES ASSOCIES
+
+7.1 Le Contrat n'oblige en aucun cas le Concédant à la réalisation de
+prestations d'assistance technique ou de maintenance du Logiciel.
+
+Cependant le Concédant reste libre de proposer ce type de services. Les
+termes et conditions d'une telle assistance technique et/ou d'une telle
+maintenance seront alors déterminés dans un acte séparé. Ces actes de
+maintenance et/ou assistance technique n'engageront que la seule
+responsabilité du Concédant qui les propose.
+
+7.2 De même, tout Concédant est libre de proposer, sous sa seule
+responsabilité, à ses licenciés une garantie, qui n'engagera que lui,
+lors de la redistribution du Logiciel et/ou du Logiciel Modifié et ce,
+dans les conditions qu'il souhaite. Cette garantie et les modalités
+financières de son application feront l'objet d'un acte séparé entre le
+Concédant et le Licencié.
+
+
+    Article 8 - RESPONSABILITE
+
+8.1 Sous réserve des dispositions de l'article 8.2, le Licencié a la 
+faculté, sous réserve de prouver la faute du Concédant concerné, de
+solliciter la réparation du préjudice direct qu'il subirait du fait du
+Logiciel et dont il apportera la preuve.
+
+8.2 La responsabilité du Concédant est limitée aux engagements pris en
+application du Contrat et ne saurait être engagée en raison notamment:
+(i) des dommages dus à l'inexécution, totale ou partielle, de ses
+obligations par le Licencié, (ii) des dommages directs ou indirects
+découlant de l'utilisation ou des performances du Logiciel subis par le
+Licencié et (iii) plus généralement d'un quelconque dommage indirect. En
+particulier, les Parties conviennent expressément que tout préjudice
+financier ou commercial (par exemple perte de données, perte de
+bénéfices, perte d'exploitation, perte de clientèle ou de commandes,
+manque à gagner, trouble commercial quelconque) ou toute action dirigée
+contre le Licencié par un tiers, constitue un dommage indirect et
+n'ouvre pas droit à réparation par le Concédant.
+
+
+    Article 9 - GARANTIE
+
+9.1 Le Licencié reconnaît que l'état actuel des connaissances
+scientifiques et techniques au moment de la mise en circulation du
+Logiciel ne permet pas d'en tester et d'en vérifier toutes les
+utilisations ni de détecter l'existence d'éventuels défauts. L'attention
+du Licencié a été attirée sur ce point sur les risques associés au
+chargement, à l'utilisation, la modification et/ou au développement et à
+la reproduction du Logiciel qui sont réservés à des utilisateurs avertis.
+
+Il relève de la responsabilité du Licencié de contrôler, par tous
+moyens, l'adéquation du produit à ses besoins, son bon fonctionnement et
+de s'assurer qu'il ne causera pas de dommages aux personnes et aux biens.
+
+9.2 Le Concédant déclare de bonne foi être en droit de concéder
+l'ensemble des droits attachés au Logiciel (comprenant notamment les
+droits visés à l'article 5).
+
+9.3 Le Licencié reconnaît que le Logiciel est fourni "en l'état" par le
+Concédant sans autre garantie, expresse ou tacite, que celle prévue à
+l'article 9.2 et notamment sans aucune garantie sur sa valeur commerciale, 
+son caractère sécurisé, innovant ou pertinent.
+
+En particulier, le Concédant ne garantit pas que le Logiciel est exempt
+d'erreur, qu'il fonctionnera sans interruption, qu'il sera compatible
+avec l'équipement du Licencié et sa configuration logicielle ni qu'il
+remplira les besoins du Licencié.
+
+9.4 Le Concédant ne garantit pas, de manière expresse ou tacite, que le
+Logiciel ne porte pas atteinte à un quelconque droit de propriété
+intellectuelle d'un tiers portant sur un brevet, un logiciel ou sur tout
+autre droit de propriété. Ainsi, le Concédant exclut toute garantie au
+profit du Licencié contre les actions en contrefaçon qui pourraient être
+diligentées au titre de l'utilisation, de la modification, et de la
+redistribution du Logiciel. Néanmoins, si de telles actions sont
+exercées contre le Licencié, le Concédant lui apportera son aide
+technique et juridique pour sa défense. Cette aide technique et
+juridique est déterminée au cas par cas entre le Concédant concerné et
+le Licencié dans le cadre d'un protocole d'accord. Le Concédant dégage
+toute responsabilité quant à l'utilisation de la dénomination du
+Logiciel par le Licencié. Aucune garantie n'est apportée quant à
+l'existence de droits antérieurs sur le nom du Logiciel et sur
+l'existence d'une marque.
+
+
+    Article 10 - RESILIATION
+
+10.1 En cas de manquement par le Licencié aux obligations mises à sa
+charge par le Contrat, le Concédant pourra résilier de plein droit le
+Contrat trente (30) jours après notification adressée au Licencié et
+restée sans effet.
+
+10.2 Le Licencié dont le Contrat est résilié n'est plus autorisé à
+utiliser, modifier ou distribuer le Logiciel. Cependant, toutes les
+licences qu'il aura concédées antérieurement à la résiliation du Contrat
+resteront valides sous réserve qu'elles aient été effectuées en
+conformité avec le Contrat.
+
+
+    Article 11 - DISPOSITIONS DIVERSES
+
+
+      11.1 CAUSE EXTERIEURE
+
+Aucune des Parties ne sera responsable d'un retard ou d'une défaillance
+d'exécution du Contrat qui serait dû à un cas de force majeure, un cas
+fortuit ou une cause extérieure, telle que, notamment, le mauvais
+fonctionnement ou les interruptions du réseau électrique ou de
+télécommunication, la paralysie du réseau liée à une attaque
+informatique, l'intervention des autorités gouvernementales, les
+catastrophes naturelles, les dégâts des eaux, les tremblements de terre,
+le feu, les explosions, les grèves et les conflits sociaux, l'état de
+guerre...
+
+11.2 Le fait, par l'une ou l'autre des Parties, d'omettre en une ou
+plusieurs occasions de se prévaloir d'une ou plusieurs dispositions du
+Contrat, ne pourra en aucun cas impliquer renonciation par la Partie
+intéressée à s'en prévaloir ultérieurement.
+
+11.3 Le Contrat annule et remplace toute convention antérieure, écrite
+ou orale, entre les Parties sur le même objet et constitue l'accord
+entier entre les Parties sur cet objet. Aucune addition ou modification
+aux termes du Contrat n'aura d'effet à l'égard des Parties à moins
+d'être faite par écrit et signée par leurs représentants dûment habilités.
+
+11.4 Dans l'hypothèse où une ou plusieurs des dispositions du Contrat
+s'avèrerait contraire à une loi ou à un texte applicable, existants ou
+futurs, cette loi ou ce texte prévaudrait, et les Parties feraient les
+amendements nécessaires pour se conformer à cette loi ou à ce texte.
+Toutes les autres dispositions resteront en vigueur. De même, la
+nullité, pour quelque raison que ce soit, d'une des dispositions du
+Contrat ne saurait entraîner la nullité de l'ensemble du Contrat.
+
+
+      11.5 LANGUE
+
+Le Contrat est rédigé en langue française et en langue anglaise, ces
+deux versions faisant également foi.
+
+
+    Article 12 - NOUVELLES VERSIONS DU CONTRAT
+
+12.1 Toute personne est autorisée à copier et distribuer des copies de
+ce Contrat.
+
+12.2 Afin d'en préserver la cohérence, le texte du Contrat est protégé
+et ne peut être modifié que par les auteurs de la licence, lesquels se
+réservent le droit de publier périodiquement des mises à jour ou de
+nouvelles versions du Contrat, qui posséderont chacune un numéro
+distinct. Ces versions ultérieures seront susceptibles de prendre en
+compte de nouvelles problématiques rencontrées par les logiciels libres.
+
+12.3 Tout Logiciel diffusé sous une version donnée du Contrat ne pourra
+faire l'objet d'une diffusion ultérieure que sous la même version du
+Contrat ou une version postérieure, sous réserve des dispositions de
+l'article 5.3.4.
+
+
+    Article 13 - LOI APPLICABLE ET COMPETENCE TERRITORIALE
+
+13.1 Le Contrat est régi par la loi française. Les Parties conviennent
+de tenter de régler à l'amiable les différends ou litiges qui
+viendraient à se produire par suite ou à l'occasion du Contrat.
+
+13.2 A défaut d'accord amiable dans un délai de deux (2) mois à compter
+de leur survenance et sauf situation relevant d'une procédure d'urgence,
+les différends ou litiges seront portés par la Partie la plus diligente
+devant les Tribunaux compétents de Paris.
+
+
+Version 2.0 du 2006-09-05.

distutils.ext/obidistutils/command/build.py

@@ -0,0 +1,38 @@
+'''
+Created on 20 oct. 2012
+
+@author: coissac
+'''
+
+from distutils.command.build import build as ori_build
+from obidistutils.serenity.checksystem import is_mac_system
+
+
+class build(ori_build):
+    
+    def has_ext_modules(self):
+        return self.distribution.has_ext_modules()
+    
+    def has_pidname(self):
+        return is_mac_system()
+
+    def has_doc(self):
+        return True
+    
+    def has_littlebigman(self):
+        return True
+    
+    try:
+        from obidistutils.command.build_sphinx import build_sphinx  # @UnusedImport
+
+        sub_commands = [("littlebigman",has_littlebigman),
+                         ('pidname',has_pidname)
+                       ] \
+                       + ori_build.sub_commands + \
+                       [('build_sphinx',has_doc)]
+    except ImportError:
+        sub_commands = [("littlebigman",has_littlebigman),
+                         ('pidname',has_pidname)
+                       ] \
+                       + ori_build.sub_commands
+        

distutils.ext/obidistutils/command/build_cexe.py

@@ -0,0 +1,85 @@
+'''
+Created on 20 oct. 2012
+
+@author: coissac
+'''
+
+from .build_ctools import build_ctools
+from .build_exe import build_exe
+from distutils.errors import DistutilsSetupError
+from distutils import log
+import os
+
+class build_cexe(build_ctools):
+
+    description = "build C/C++ executable distributed with Python extensions"
+
+
+    def initialize_options(self):
+        build_ctools.initialize_options(self)
+        self.built_files = None
+
+
+    def finalize_options(self):
+        # This might be confusing: both build-cexe and build-temp default
+        # to build-temp as defined by the "build" command.  This is because
+        # I think that C libraries are really just temporary build
+        # by-products, at least from the point of view of building Python
+        # extensions -- but I want to keep my options open.
+
+        build_cexe_dir = self.build_cexe
+        build_ctools.finalize_options(self)
+
+        if build_cexe_dir is None:
+            self.build_cexe=None
+            
+        self.set_undefined_options('build',
+                                   ('build_scripts',  'build_cexe'))
+
+        self.set_undefined_options('build_files',
+                                   ('files',  'built_files'))
+                   
+        self.executables   = self.distribution.executables
+#         self.build_cexe = os.path.join(os.path.dirname(self.build_cexe),'cbinaries') 
+#         self.mkpath(self.build_cexe)
+
+        if self.executables:
+            self.check_executable_list(self.executables)
+                            
+
+        # XXX same as for build_ext -- what about 'self.define' and
+        # 'self.undef' ?
+
+    def substitute_sources(self,exe_name,sources):
+        """
+        Substitutes source file name starting by an @ by the actual
+        name of the built file (see --> build_files)
+        """
+        sources = list(sources)
+        for i in range(len(sources)):
+            message = "%s :-> %s" % (exe_name,sources[i])
+            if sources[i][0]=='@':
+                try:
+                    filename = self.built_files[sources[i][1:]]
+                except KeyError:
+                    raise DistutilsSetupError(
+                         'The %s filename declared in the source '
+                         'files of the program %s have not been '
+                         'built by the installation process' % (sources[i],
+                                                                exe_name))
+                sources[i]=filename
+                log.info("%s changed to %s",message,filename)
+            else:
+                log.info("%s ok",message)
+
+        return sources
+ 
+    def run(self):
+
+        for cmd_name in self.get_sub_commands():
+            self.run_command(cmd_name)
+
+        build_exe.run(self)
+        
+
+   

distutils.ext/obidistutils/command/build_ctools.py

@@ -0,0 +1,63 @@
+'''
+Created on 20 oct. 2012
+
+@author: coissac
+'''
+
+
+from .build_exe import build_exe
+from distutils import log
+
+class build_ctools(build_exe):
+    description = "build C/C++ executable not distributed with Python extensions"
+
+    def initialize_options(self):
+        build_exe.initialize_options(self)
+        
+        # List of built tools
+        self.ctools = None
+        self.littlebigman = None
+
+
+    def finalize_options(self):
+        # This might be confusing: both build-cexe and build-temp default
+        # to build-temp as defined by the "build" command.  This is because
+        # I think that C libraries are really just temporary build
+        # by-products, at least from the point of view of building Python
+        # extensions -- but I want to keep my options open.
+            
+        build_exe.finalize_options(self)
+        
+        
+        self.set_undefined_options('build',
+                                    ('build_temp', 'build_cexe'))
+
+        self.set_undefined_options('littlebigman',
+                                   ('littlebigman',  'littlebigman'))
+                   
+        self.executables = self.distribution.ctools
+        self.check_executable_list(self.executables)
+        
+
+        if self.littlebigman =='-DLITTLE_END':
+            if self.define is None:
+                self.define=[('LITTLE_END',None)]
+            else:
+                self.define.append('LITTLE_END',None)
+                
+        log.info('Look for CPU architecture... %s',self.define)
+
+        self.ctools = set()
+
+    def run(self):
+
+        for cmd_name in self.get_sub_commands():
+            self.run_command(cmd_name)
+
+        
+        build_exe.run(self)
+        
+        for e,p in self.executables:                                  # @UnusedVariable
+            self.ctools.add(e)
+            
+

distutils.ext/obidistutils/command/build_exe.py

@@ -0,0 +1,211 @@
+'''
+Created on 20 oct. 2012
+
+@author: coissac
+'''
+
+import os
+
+from distutils.core import Command
+from distutils.sysconfig import customize_compiler
+from distutils.errors import DistutilsSetupError
+from distutils import log
+from distutils.ccompiler import show_compilers
+
+class build_exe(Command):
+    
+    description = "build an executable -- Abstract command "
+
+    user_options = [
+        ('build-cexe', 'x',
+         "directory to build C/C++ libraries to"),
+        ('build-temp', 't',
+         "directory to put temporary build by-products"),
+        ('debug', 'g',
+         "compile with debugging information"),
+        ('force', 'f',
+         "forcibly build everything (ignore file timestamps)"),
+        ('compiler=', 'c',
+         "specify the compiler type"),
+        ]
+
+    boolean_options = ['debug', 'force']
+
+    help_options = [
+        ('help-compiler', None,
+         "list available compilers", show_compilers),
+        ]
+
+    def initialize_options(self):
+        self.build_cexe = None
+        self.build_temp = None
+
+        # List of executables to build
+        self.executables = None
+
+        # Compilation options for all libraries
+        self.include_dirs = None
+        self.define = None
+        self.undef = None
+        self.extra_compile_args = None
+        self.debug = None
+        self.force = 0
+        self.compiler = None
+        self.sse = None
+        self.built_files=None
+
+    def finalize_options(self):
+        # This might be confusing: both build-cexe and build-temp default
+        # to build-temp as defined by the "build" command.  This is because
+        # I think that C libraries are really just temporary build
+        # by-products, at least from the point of view of building Python
+        # extensions -- but I want to keep my options open.
+        self.set_undefined_options('build',
+                                   ('build_temp', 'build_temp'),
+                                   ('compiler', 'compiler'),
+                                   ('debug', 'debug'),
+                                   ('force', 'force'))
+
+        if self.include_dirs is None:
+            self.include_dirs = self.distribution.include_dirs or []
+            
+        if isinstance(self.include_dirs, str):
+            self.include_dirs = self.include_dirs.split(os.pathsep)
+
+        self.sse = self.distribution.sse
+        
+        if self.sse is not None:
+            if self.extra_compile_args is None:
+                self.extra_compile_args=['-m%s' % self.sse]
+            else:
+                self.extra_compile_args.append('-m%s' % self.sse)
+
+        # XXX same as for build_ext -- what about 'self.define' and
+        # 'self.undef' ?
+
+    def run(self):
+
+        if not self.executables:
+            return
+
+        self.mkpath(self.build_cexe)
+
+        # Yech -- this is cut 'n pasted from build_ext.py!
+        from distutils.ccompiler import new_compiler
+        self.compiler = new_compiler(compiler=self.compiler,
+                                     dry_run=self.dry_run,
+                                     force=self.force)
+        customize_compiler(self.compiler)
+
+        if self.include_dirs is not None:
+            self.compiler.set_include_dirs(self.include_dirs)
+        if self.define is not None:
+            # 'define' option is a list of (name,value) tuples
+            for (name,value) in self.define:
+                self.compiler.define_macro(name, value)
+
+        if self.undef is not None:
+            for macro in self.undef:
+                self.compiler.undefine_macro(macro)
+                
+        self.build_executables(self.executables)
+
+
+    def check_executable_list(self, executables):
+        """Ensure that the list of executables is valid.
+
+        `executable` is presumably provided as a command option 'executables'.
+        This method checks that it is a list of 2-tuples, where the tuples
+        are (executable_name, build_info_dict).
+
+        Raise DistutilsSetupError if the structure is invalid anywhere;
+        just returns otherwise.
+        """
+        if not isinstance(executables, list):
+            raise DistutilsSetupError("'executables' option must be a list of tuples")
+
+        for exe in executables:
+            if not isinstance(exe, tuple) and len(exe) != 2:
+                raise DistutilsSetupError("each element of 'executables' must a 2-tuple")
+
+            name, build_info = exe
+
+            if not isinstance(name, str):
+                raise DistutilsSetupError(
+                      "first element of each tuple in 'executables' " 
+                      "must be a string (the executables name)")
+                
+            if '/' in name or (os.sep != '/' and os.sep in name):
+                raise DistutilsSetupError(
+                       "bad executable name '%s': " 
+                       "may not contain directory separators" % exe[0])
+
+            if not isinstance(build_info, dict):
+                raise DistutilsSetupError(
+                      "second element of each tuple in 'executables' " 
+                      "must be a dictionary (build info)")
+
+    def get_executable_names(self):
+        # Assume the executables list is valid -- 'check_executable_list()' is
+        # called from 'finalize_options()', so it should be!
+        if not self.executables:
+            return None
+
+        exe_names = []
+        for (exe_name, build_info) in self.executables:                     # @UnusedVariable
+            exe_names.append(exe_name)
+        return exe_names
+
+
+    def get_source_files(self):
+        self.check_executable_list(self.executables)
+        filenames = []
+        for (exe_name, build_info) in self.executables:                     # @UnusedVariable
+            sources = build_info.get('sources')
+            if sources is None or not isinstance(sources, (list, tuple)):
+                raise DistutilsSetupError(
+                       "in 'executables' option (library '%s'), "
+                       "'sources' must be present and must be "
+                       "a list of source filenames" % exe_name)
+
+            filenames.extend(sources)
+        return filenames
+    
+    def substitute_sources(self,exe_name,sources):
+        return list(sources)
+
+    def build_executables(self, executables):
+        for (exe_name, build_info) in executables:
+            sources = build_info.get('sources')
+            if sources is None or not isinstance(sources, (list, tuple)):
+                raise DistutilsSetupError(
+                       "in 'executables' option (library '%s'), " 
+                       "'sources' must be present and must be " 
+                       "a list of source filenames" % exe_name)
+                
+            sources = self.substitute_sources(exe_name,sources)
+
+            log.info("building '%s' program", exe_name)
+
+            # First, compile the source code to object files in the library
+            # directory.  (This should probably change to putting object
+            # files in a temporary build directory.)
+            macros = build_info.get('macros')
+            include_dirs = build_info.get('include_dirs')
+            extra_args = self.extra_compile_args or []
+            
+            objects = self.compiler.compile(sources,
+                                            output_dir=self.build_temp,
+                                            macros=macros,
+                                            include_dirs=include_dirs,
+                                            extra_postargs=extra_args,
+                                            debug=self.debug)
+
+            # Now "link" the object files together into a static library.
+            # (On Unix at least, this isn't really linking -- it just
+            # builds an archive.  Whatever.)
+            self.compiler.link_executable(objects, exe_name,
+                                            output_dir=self.build_cexe,
+                                            debug=self.debug)
+
+        

distutils.ext/obidistutils/command/build_ext.py

@@ -0,0 +1,115 @@
+'''
+Created on 13 fevr. 2014
+
+@author: coissac
+'''
+
+from distutils import log
+import os
+
+
+from distutils.errors import DistutilsSetupError
+
+try:
+    from Cython.Distutils import build_ext  as ori_build_ext  # @UnresolvedImport
+    from Cython.Compiler import Options as cython_options  # @UnresolvedImport
+    class build_ext(ori_build_ext):
+    
+        
+        def modifyDocScripts(self):
+            build_dir_file=open("doc/sphinx/build_dir.txt","w")
+            print(self.build_lib,file=build_dir_file)
+            build_dir_file.close()
+            
+        def initialize_options(self):
+            ori_build_ext.initialize_options(self)  # @UndefinedVariable
+            self.littlebigman = None
+            self.built_files = None
+    
+        
+        def finalize_options(self):
+            ori_build_ext.finalize_options(self)  # @UndefinedVariable
+    
+            self.set_undefined_options('littlebigman',
+                                       ('littlebigman',  'littlebigman'))
+            
+            self.set_undefined_options('build_files',
+                                       ('files',  'built_files'))
+    
+            self.cython_c_in_temp = 1
+                       
+            if self.littlebigman =='-DLITTLE_END':
+                if self.define is None:
+                    self.define=[('LITTLE_END',None)]
+                else:
+                    self.define.append('LITTLE_END',None)
+            
+        def substitute_sources(self,exe_name,sources):
+            """
+            Substitutes source file name starting by an @ by the actual
+            name of the built file (see --> build_files)
+            """
+            sources = list(sources)
+            for i in range(len(sources)):
+                message = "%s :-> %s" % (exe_name,sources[i])
+                if sources[i][0]=='@':
+                    try:
+                        filename = self.built_files[sources[i][1:]]
+                    except KeyError:
+                        tmpfilename = os.path.join(self.build_temp,sources[i][1:])
+                        if os.path.isfile       (tmpfilename):
+                            filename = tmpfilename
+                        else:
+                            raise DistutilsSetupError(
+                                 'The %s filename declared in the source '
+                                 'files of the program %s have not been '
+                                 'built by the installation process' % (sources[i],
+                                                                         exe_name))
+                    sources[i]=filename
+                    log.info("%s changed to %s",message,filename)
+                else:
+                    log.info("%s ok",message)
+    
+            return sources
+    
+        def build_extensions(self):
+            # First, sanity-check the 'extensions' list
+            
+            for ext in self.extensions:
+                ext.sources = self.substitute_sources(ext.name,ext.sources)
+                
+            self.check_extensions_list(self.extensions)
+    
+            for ext in self.extensions:
+                log.info("%s :-> %s",ext.name,ext.sources)
+                ext.sources = self.cython_sources(ext.sources, ext)
+                self.build_extension(ext)
+    
+            
+        def run(self):
+            self.modifyDocScripts()
+    
+            for cmd_name in self.get_sub_commands():
+                self.run_command(cmd_name)
+    
+            cython_options.annotate = True
+            ori_build_ext.run(self)  # @UndefinedVariable
+            
+    
+        def has_files(self):
+            return self.distribution.has_files()
+    
+        def has_executables(self):
+            return self.distribution.has_executables()
+        
+        sub_commands = [('build_files',has_files),
+                        ('build_cexe', has_executables)
+                        ] + \
+                       ori_build_ext.sub_commands
+
+except ImportError:
+    from distutils.command import build_ext  # @UnusedImport
+           
+
+
+        

distutils.ext/obidistutils/command/build_files.py

@@ -0,0 +1,63 @@
+'''
+Created on 20 oct. 2012
+
+@author: coissac
+'''
+
+import os.path
+
+from distutils.core import Command
+from distutils import log
+
+class build_files(Command):
+            
+    def initialize_options(self):
+        self.files=None
+        self.ctools=None
+        self.build_temp=None
+        self.build_cexe=None
+
+    def finalize_options(self):
+        
+        self.set_undefined_options('build_ctools',
+                                   ('ctools',  'ctools'),
+                                   ('build_temp','build_temp'),
+                                   ('build_cexe','build_cexe'),
+                                   )
+        
+        self.files = {}
+        
+    def run(self):
+
+        for cmd_name in self.get_sub_commands():
+            self.run_command(cmd_name)
+
+        
+        for dest,prog,command in self.distribution.files:
+            destfile = os.path.join(self.build_temp,dest)
+            if prog in self.ctools:
+                progfile = os.path.join(self.build_cexe,prog)
+            else:
+                progfile = prog 
+                
+            log.info("Building file : %s" % dest)
+            
+            commandline = command % {'prog' : progfile,
+                                     'dest' : destfile}
+
+            log.info("    --> %s" % commandline)
+            
+            os.system(commandline)
+            self.files[dest]=destfile
+            
+            log.info("Done.\n")
+            
+    def has_ctools(self):
+        return self.distribution.has_ctools()
+
+
+    sub_commands = [('build_ctools', has_ctools)] + \
+                   Command.sub_commands
+            
+            
+        

distutils.ext/obidistutils/command/build_scripts.py

@@ -0,0 +1,104 @@
+'''
+Created on 20 oct. 2012
+
+@author: coissac
+'''
+
+import os.path
+
+from distutils.command.build_scripts import build_scripts as ori_build_scripts
+from distutils.util import convert_path
+from distutils import log, sysconfig
+from distutils.dep_util import newer
+from stat import ST_MODE
+import re
+
+first_line_re = re.compile('^#!.*python[0-9.]*([ \t].*)?$')
+
+class build_scripts(ori_build_scripts):
+            
+    def copy_scripts (self):
+        """Copy each script listed in 'self.scripts'; if it's marked as a
+        Python script in the Unix way (first line matches 'first_line_re',
+        ie. starts with "\#!" and contains "python"), then adjust the first
+        line to refer to the current Python interpreter as we copy.
+        """        
+        self.mkpath(self.build_dir)
+        rawbuild_dir = os.path.join(os.path.dirname(self.build_dir),'raw_scripts')
+        self.mkpath(rawbuild_dir)
+        
+        outfiles = []
+        for script in self.scripts:
+            adjust = 0
+            script = convert_path(script)
+            outfile = os.path.join(self.build_dir, os.path.splitext(os.path.basename(script))[0])
+            rawoutfile = os.path.join(rawbuild_dir, os.path.basename(script))
+            outfiles.append(outfile)
+
+            if not self.force and not newer(script, outfile):
+                log.debug("not copying %s (up-to-date)", script)
+                continue
+
+            # Always open the file but ignore failures in dry-run mode --
+            # that way, we'll get accurate feedback if we can read the
+            # script.
+            try:
+                f = open(script, "r")
+            except IOError:
+                if not self.dry_run:
+                    raise
+                f = None
+            else:
+                first_line = f.readline()
+                if not first_line:
+                    self.warn("%s is an empty file (skipping)" % script)
+                    continue
+
+                match = first_line_re.match(first_line)
+                if match:
+                    adjust = 1
+                    post_interp = match.group(1) or ''
+            
+            log.info("Store the raw script %s -> %s", script,rawoutfile)        
+            self.copy_file(script, rawoutfile)
+
+
+            if adjust:
+                log.info("copying and adjusting %s -> %s", script,
+                         self.build_dir)
+                if not self.dry_run:
+                    outf = open(outfile, "w")
+                    if not sysconfig.python_build:
+                        outf.write("#!%s%s\n" %
+                                   (self.executable,
+                                    post_interp))
+                    else:
+                        outf.write("#!%s%s\n" %
+                                   (os.path.join(
+                            sysconfig.get_config_var("BINDIR"),
+                           "python%s%s" % (sysconfig.get_config_var("VERSION"),
+                                           sysconfig.get_config_var("EXE"))),
+                                    post_interp))
+                    outf.writelines(f.readlines())
+                    outf.close()
+                if f:
+                    f.close()
+            else:
+                if f:
+                    f.close()
+                self.copy_file(script, outfile)
+
+        if os.name == 'posix':
+            for F in outfiles:
+                if self.dry_run:
+                    log.info("changing mode of %s", F)
+                else:
+                    oldmode = os.stat(F)[ST_MODE]
+                    oldmode = oldmode & 0o7777
+                    
+                    newmode = (oldmode | 0o555) & 0o7777
+                    if newmode != oldmode:
+                        log.info("changing mode of %s from %o to %o",
+                                 F, oldmode, newmode)
+                        os.chmod(F, newmode)
+

distutils.ext/obidistutils/command/build_sphinx.py

@@ -0,0 +1,27 @@
+'''
+Created on 10 mars 2015
+
+@author: coissac
+'''
+
+try:
+    from sphinx.setup_command import BuildDoc as ori_build_sphinx  # @UnresolvedImport
+        
+    class build_sphinx(ori_build_sphinx):
+        '''Build Sphinx documentation in html, epub and man formats 
+        '''
+    
+        description = __doc__
+    
+        def run(self):
+            self.builder='html'
+            self.finalize_options()
+            ori_build_sphinx.run(self)
+            self.builder='epub'
+            self.finalize_options()
+            ori_build_sphinx.run(self)
+            self.builder='man'
+            self.finalize_options()
+            ori_build_sphinx.run(self)
+except ImportError:
+    pass  

distutils.ext/obidistutils/command/install.py

@@ -0,0 +1,19 @@
+'''
+Created on 6 oct. 2014
+
+@author: coissac
+'''
+
+# try:
+#     from setuptools.command.install import install as install_ori
+# except ImportError:
+#     from distutils.command.install import install as install_ori
+
+from distutils.command.install import install as install_ori
+
+class install(install_ori):
+    
+    def __init__(self,dist):
+        install_ori.__init__(self, dist)
+#        self.sub_commands.insert(0, ('build',lambda self: True))
+        self.sub_commands.append(('install_sphinx',lambda self: self.distribution.serenity))

distutils.ext/obidistutils/command/install_scripts.py

@@ -0,0 +1,47 @@
+'''
+Created on 20 oct. 2012
+
+@author: coissac
+'''
+
+# try:
+#     from setuptools.command.install_scripts import install_scripts as ori_install_scripts
+# except ImportError:
+#     from distutils.command.install_scripts import install_scripts as ori_install_scripts
+
+from distutils.command.install_scripts import install_scripts as ori_install_scripts
+
+import os.path
+from distutils import log
+
+class install_scripts(ori_install_scripts):
+
+    def initialize_options(self):
+        ori_install_scripts.initialize_options(self)
+        self.public_dir = None
+                        
+                    
+    def install_public_link(self):
+        self.mkpath(self.public_dir)
+        for file in self.get_outputs():
+            log.info("exporting file %s -> %s", file,os.path.join(self.public_dir,
+                                    os.path.split(file)[1]
+                                    ))
+            if not self.dry_run:
+                dest = os.path.join(self.public_dir,
+                                    os.path.split(file)[1]
+                                   )
+                if os.path.exists(dest):
+                    os.unlink(dest)
+                os.symlink(file,dest)
+                    
+
+
+    def run(self):
+        ori_install_scripts.run(self)
+        if self.distribution.serenity:
+            self.public_dir=os.path.join(self.install_dir,"../export/bin")
+            self.public_dir=os.path.abspath(self.public_dir)
+            self.install_public_link()
+
+

distutils.ext/obidistutils/command/install_sphinx.py

@@ -0,0 +1,61 @@
+'''
+Created on 10 mars 2015
+
+@author: coissac
+'''
+from distutils.core import Command
+import os.path
+import glob
+
+class install_sphinx(Command):
+    '''
+    Install the sphinx documentation
+    '''
+    
+    description = "Install the sphinx documentation in serenity mode"
+
+    boolean_options = ['force', 'skip-build']
+
+
+    def initialize_options (self):
+        self.install_doc = None
+        self.build_dir = None
+
+    def finalize_options (self):
+        self.set_undefined_options('build_sphinx', ('build_dir', 'build_dir'))
+        self.set_undefined_options('install',
+                                   ('install_scripts', 'install_doc'))
+
+    def run (self):
+        if self.distribution.serenity:
+            self.install_doc = os.path.join(self.install_doc,"../export/share")
+            self.install_doc=os.path.abspath(self.install_doc)
+            self.mkpath(self.install_doc)
+            self.mkpath(os.path.join(self.install_doc,'html'))
+            outfiles = self.copy_tree(os.path.join(self.build_dir,'html'),  # @UnusedVariable
+                                      os.path.join(self.install_doc,'html'))
+                
+            self.mkpath(os.path.join(self.install_doc,'man','man1'))
+            outfiles = self.copy_tree(os.path.join(self.build_dir,'man'),  # @UnusedVariable
+                                      os.path.join(self.install_doc,'man','man1'))
+
+            for epub in glob.glob(os.path.join(self.build_dir,'epub/*.epub')):
+                self.copy_file(os.path.join(epub), 
+                               os.path.join(self.install_doc,os.path.split(epub)[1]))
+            
+    def get_outputs(self):
+        directory=os.path.join(self.install_doc,'html')
+        files = [os.path.join(self.install_doc,'html', f) 
+                 for dp, dn, filenames in os.walk(directory) for f in filenames]  # @UnusedVariable
+        
+        directory=os.path.join(self.build_dir,'man')
+        files.append(os.path.join(self.install_doc,'man','man1', f) 
+                 for dp, dn, filenames in os.walk(directory) for f in filenames)  # @UnusedVariable
+
+        directory=os.path.join(self.build_dir,'epub')
+        files.append(os.path.join(self.install_doc, f) 
+                 for dp, dn, filenames in os.walk(directory)  # @UnusedVariable
+                 for f in glob.glob(os.path.join(dp, '*.epub')) )
+        
+        return files
+        

distutils.ext/obidistutils/command/littlebigman.py

@@ -0,0 +1,59 @@
+'''
+Created on 20 oct. 2012
+
+@author: coissac
+'''
+
+import os
+
+from obidistutils.command.build_exe import build_exe
+from distutils import log
+
+import subprocess
+
+class littlebigman(build_exe):
+    
+    description = "build the littlebigman executable testing endianness of the CPU"
+
+
+    def initialize_options(self):
+        build_exe.initialize_options(self)
+
+        self.littlebigman = None
+
+
+    def finalize_options(self):
+        # This might be confusing: both build-cexe and build-temp default
+        # to build-temp as defined by the "build" command.  This is because
+        # I think that C libraries are really just temporary build
+        # by-products, at least from the point of view of building Python
+        # extensions -- but I want to keep my options open.
+        
+        build_exe.finalize_options(self)
+        
+        self.set_undefined_options('build',
+                                   ('build_temp', 'build_cexe'))
+
+        # self.ctools = self.distribution.ctools
+        
+        if os.path.isfile("distutils.ext/src/littlebigman.c"):
+            self.executables = [('littlebigman',{"sources":["distutils.ext/src/littlebigman.c"]})]
+            self.check_executable_list(self.executables)
+        else:
+            self.executables = []
+
+
+    def run_littlebigman(self):
+        p = subprocess.Popen("'%s'" % os.path.join(self.build_temp,
+                                                   'littlebigman'), 
+                             shell=True, 
+                             stdout=subprocess.PIPE)
+        little = p.communicate()[0]
+        return little.decode('latin1')
+
+    def run(self):
+        build_exe.run(self)
+        self.littlebigman=self.run_littlebigman()
+        log.info("Your CPU is in mode : %s" % self.littlebigman)
+        
+        

distutils.ext/obidistutils/command/pidname.py

@@ -0,0 +1,55 @@
+'''
+Created on 20 oct. 2012
+
+@author: coissac
+'''
+
+import os
+
+from obidistutils.command.build_exe import build_exe
+from obidistutils.serenity.checksystem import is_mac_system
+
+
+class pidname(build_exe):
+    
+    description = "build the pidname executable returning the executable path from a PID on a mac"
+
+
+    def initialize_options(self):
+        build_exe.initialize_options(self)
+
+        self.pidname = False
+
+
+    def finalize_options(self):
+        # This might be confusing: both build-cexe and build-temp default
+        # to build-temp as defined by the "build" command.  This is because
+        # I think that C libraries are really just temporary build
+        # by-products, at least from the point of view of building Python
+        # extensions -- but I want to keep my options open.
+        
+        build_exe.finalize_options(self)
+        
+        self.set_undefined_options('build',
+                                   ('build_scripts', 'build_cexe'))
+
+        # self.ctools = self.distribution.ctools
+        
+        if os.path.isfile("distutils.ext/src/pidname.c"):
+            self.executables = [('pidname',{"sources":["distutils.ext/src/pidname.c"]})]
+            self.check_executable_list(self.executables)
+        else:
+            self.executables = []
+
+#         self.build_cexe = os.path.join(os.path.dirname(self.build_cexe),'cbinaries') 
+#         self.mkpath(self.build_cexe)
+
+
+
+    def run(self):
+        if is_mac_system():
+            build_exe.run(self)
+            self.pidname=True
+        else:
+            self.pidname=False
+        

distutils.ext/obidistutils/command/sdist.py

@@ -0,0 +1,42 @@
+'''
+Created on 10 mars 2015
+
+@author: coissac
+'''
+
+import os.path
+
+from distutils.command.sdist import sdist as orig_sdist
+from distutils import dir_util
+
+class sdist(orig_sdist):
+
+    def make_distribution(self):
+        """Create the source distribution(s).  First, we create the release
+        tree with 'make_release_tree()'; then, we create all required
+        archive files (according to 'self.formats') from the release tree.
+        Finally, we clean up by blowing away the release tree (unless
+        'self.keep_temp' is true).  The list of archive files created is
+        stored so it can be retrieved later by 'get_archive_files()'.
+        """
+        # Don't warn about missing meta-data here -- should be (and is!)
+        # done elsewhere.
+        base_dir = self.distribution.get_fullname()
+        base_name = os.path.join(self.dist_dir,base_dir)
+
+        self.make_release_tree(os.path.join('tmp',base_dir), self.filelist.files)
+        archive_files = []              # remember names of files we create
+        # tar archive must be created last to avoid overwrite and remove
+        if 'tar' in self.formats:
+            self.formats.append(self.formats.pop(self.formats.index('tar')))
+
+        for fmt in self.formats:
+            file = self.make_archive(base_name, fmt, root_dir='tmp',base_dir=base_dir,
+                                     owner=self.owner, group=self.group)
+            archive_files.append(file)
+            self.distribution.dist_files.append(('sdist', '', file))
+
+        self.archive_files = archive_files
+
+        if not self.keep_temp:
+            dir_util.remove_tree(os.path.join('tmp',base_dir), dry_run=self.dry_run)

distutils.ext/obidistutils/core.py

@@ -0,0 +1,226 @@
+'''
+Created on 20 oct. 2012
+
+@author: coissac
+'''
+
+from os import path
+import os.path
+import glob
+import sys
+
+# try:
+#     from setuptools.extension import Extension
+# except ImportError:
+#     from distutils.extension import Extension
+
+from distutils.extension import Extension
+
+from obidistutils.serenity.checkpackage import  install_requirements,\
+                                                check_requirements, \
+                                                RequirementError
+                                                
+from obidistutils.serenity.rerun import enforce_good_python
+from obidistutils.serenity.rerun import rerun_with_anothe_python
+from distutils import log
+
+from obidistutils.dist import Distribution
+from obidistutils.serenity import is_serenity
+ 
+ 
+def findPackage(root,base=None):
+    modules=[]
+    if base is None:
+        base=[]
+    for module in (path.basename(path.dirname(x)) 
+                   for x in glob.glob(path.join(root,'*','__init__.py'))):
+        modules.append('.'.join(base+[module]))
+        modules.extend(findPackage(path.join(root,module),base+[module]))
+    return modules
+    
+def findCython(root,base=None,pyrexs=None):
+    setupdir = os.path.dirname(sys.argv[0])
+    pyrexs=[]
+
+    if base is None:
+        base=[]
+    for module in (path.basename(path.dirname(x)) 
+                   for x in glob.glob(path.join(root,'*','__init__.py'))):
+                       
+                
+        for pyrex in glob.glob(path.join(root,module,'*.pyx')):
+            pyrexs.append(Extension('.'.join(base+[module,path.splitext(path.basename(pyrex))[0]]),
+                                    [pyrex]
+                                    )
+                          )
+            try:
+                cfiles = os.path.splitext(pyrex)[0]+".cfiles"
+                cfilesdir = os.path.dirname(cfiles)
+                cfiles = open(cfiles)
+                cfiles = [os.path.relpath(os.path.join(cfilesdir,y),setupdir).strip() 
+                          if y[0] !='@' else y.strip()
+                          for y in cfiles]
+                
+                log.info("Cython module : %s",cfiles)   
+                incdir = set(os.path.dirname(x) for x in cfiles if x[-2:]==".h")
+                cfiles = [x for x in cfiles if x[-2:]==".c"]                
+                pyrexs[-1].sources.extend(cfiles)
+                pyrexs[-1].include_dirs.extend(incdir)
+                pyrexs[-1].extra_compile_args.extend(['-msse2',
+                                                      '-Wno-unused-function',
+                                                      '-Wmissing-braces',
+                                                      '-Wchar-subscripts'])
+                
+            except IOError:
+                pass
+            
+        pyrexs.extend(findCython(path.join(root,module),base+[module]))
+    return pyrexs
+    
+
+def rootname(x):
+    return os.path.splitext(x.sources[0])[0]
+
+
+def prepare_commands():
+    from obidistutils.command.build import build
+    from obidistutils.command.littlebigman import littlebigman
+#    from obidistutils.command.serenity import serenity
+    from obidistutils.command.build_cexe import build_cexe
+    from obidistutils.command.build_ext import build_ext
+    from obidistutils.command.build_ctools import build_ctools
+    from obidistutils.command.build_files import build_files
+    from obidistutils.command.build_scripts import build_scripts
+    from obidistutils.command.install_scripts import install_scripts
+    from obidistutils.command.install_sphinx import install_sphinx
+    from obidistutils.command.install import install
+    from obidistutils.command.pidname import pidname
+    from obidistutils.command.sdist import sdist
+    
+        
+    
+    COMMANDS = {'build':build,
+#                'serenity':serenity,
+                'littlebigman':littlebigman,
+                'pidname':pidname,
+                'build_ctools':build_ctools, 
+                'build_files':build_files,
+                'build_cexe':build_cexe, 
+                'build_ext': build_ext,
+                'build_scripts':build_scripts, 
+                'install_scripts':install_scripts,
+                'install_sphinx':install_sphinx,
+                'install':install,
+                'sdist':sdist}
+    
+#     try:
+#         from setuptools.commands import egg_info
+#         COMMANDS['egg_info']=egg_info
+#     except ImportError:
+#         pass
+    try:
+        from obidistutils.command.build_sphinx import build_sphinx
+        COMMANDS['build_sphinx']=build_sphinx
+    except ImportError:
+        pass
+
+    return COMMANDS
+
+
+CTOOLS =[]
+CEXES  =[]
+FILES  =[]
+
+def setup(**attrs):
+    
+    log.set_threshold(log.INFO)
+    
+    minversion      = attrs.get("pythonmin",'3.4')
+    maxversion      = attrs.get('pythonmax',None)    
+    fork            = attrs.get('fork',False)
+    requirementfile = attrs.get('requirements','requirements.txt')
+
+    try:
+        del attrs['pythonmin']
+    except KeyError:
+        pass
+
+    try:
+        del attrs['pythonmax']
+    except KeyError:
+        pass
+
+    try:
+        del attrs['fork']
+    except KeyError:
+        pass
+
+    try:
+        del attrs['requirements']
+    except KeyError:
+        pass
+    
+    if is_serenity():
+    
+        
+        enforce_good_python(minversion, maxversion, fork)
+        
+        if (install_requirements(requirementfile)):
+            rerun_with_anothe_python(sys.executable,minversion,maxversion,fork)
+        
+    
+        try:
+            check_requirements(requirementfile)
+        except RequirementError as e :
+            log.error(e)                                   
+            sys.exit(1)
+        
+    if 'distclass' not in attrs:
+        attrs['distclass']=Distribution
+
+    if 'python_src' not in attrs:
+        SRC = 'python'
+    else:
+        SRC = attrs['python_src']
+        del(attrs['python_src'])
+    
+    if 'scripts' not in attrs:
+        attrs['scripts'] = glob.glob('%s/*.py' % SRC)
+
+    if 'package_dir' not in attrs:
+        attrs['package_dir'] = {'': SRC}
+
+    if 'packages' not in attrs:
+        attrs['packages'] = findPackage(SRC)
+    
+    if 'cmdclass' not in attrs:
+        attrs['cmdclass'] = prepare_commands()
+
+    if 'ctools' not in attrs:
+        attrs['ctools'] = CTOOLS
+    
+    if 'executables' not in attrs:
+        attrs['executables'] = CEXES
+        
+    if 'files' not in attrs:
+        attrs['files'] = FILES
+        
+    if 'sse' not in attrs:
+        attrs['sse']=None
+        
+    if 'serenity' not in attrs:
+        attrs['serenity']=False
+    
+    EXTENTION=findCython(SRC)
+    
+    if 'ext_modules' not in attrs:
+        attrs['ext_modules'] = EXTENTION
+        
+#     try:
+#         from setuptools.core import setup as ori_setup
+#     except ImportError:
+#         from distutils.core import setup as ori_setup
+
+    from distutils.core import setup as ori_setup
+    
+    ori_setup(**attrs)

distutils.ext/obidistutils/dist.py

@@ -0,0 +1,58 @@
+'''
+Created on 20 oct. 2012
+
+@author: coissac
+'''
+
+# try:
+#     from setuptools.dist import Distribution as ori_Distribution
+# except ImportError:
+#     from distutils.dist import Distribution as ori_Distribution
+
+from distutils.dist import Distribution as ori_Distribution
+
+class Distribution(ori_Distribution):
+    
+    def __init__(self,attrs=None):
+        self.executables = None
+        self.ctools = None
+        self.files = None
+        self.build_cexe  = None
+        self.deprecated_scripts = None
+        self.zip_safe=False
+        self.sse = None
+        self.serenity=attrs['serenity']
+        
+        ori_Distribution.__init__(self, attrs)
+        
+        self.global_options.insert(0,('serenity', None, "install or build the package in a python virtualenv "
+                                                       "without polluting the installed python and with many "
+                                                       "checks during the installation process"
+                                     ))
+        self.global_options.insert(0,('virtualenv', None, "if the installation is done using the serenity mode "
+                                                        "this option allows for specifying the virtualenv name. "
+                                                        "By default the name is PACKAGE-VERSION"
+                                     ))
+        
+    def run_commands(self):
+        """Run each command that was seen on the setup script command line.
+        Uses the list of commands found and cache of command objects
+        created by 'get_command_obj()'.
+        """
+#        self.run_command('littlebigman')
+        ori_Distribution.run_commands(self)
+        
+        
+    def has_executables(self):
+        return self.executables is not None and self.executables
+        
+    def has_ctools(self):
+        return self.ctools is not None and self.ctools
+        
+    def has_files(self):
+        return self.files is not None and self.files
+        
+    def has_deprecated_scripts(self):
+        return self.deprecated_scripts is not None and self.deprecated_scripts
+
+    

distutils.ext/obidistutils/serenity/__init__.py

@@ -0,0 +1,112 @@
+import sys
+
+from distutils import util
+from distutils import sysconfig
+from distutils import log
+from distutils.version import LooseVersion, StrictVersion
+import glob
+import os
+import subprocess
+import re 
+from distutils.errors import DistutilsError
+import tempfile
+
+from importlib.util import spec_from_file_location  # @UnresolvedImport
+import zipimport
+
+import argparse
+
+import base64
+
+from .checkpython import is_python_version
+                        
+                        
+from obidistutils.serenity.rerun import  enforce_good_python
+from obidistutils.serenity.rerun import rerun_with_anothe_python
+
+from obidistutils.serenity.virtual import serenity_virtualenv
+                        
+from obidistutils.serenity.checksystem import is_mac_system, \
+                                              is_windows_system
+                        
+from obidistutils.serenity.checkpackage import install_requirements
+from obidistutils.serenity.checkpackage import check_requirements
+
+from obidistutils.serenity.util import save_argv
+                        
+from obidistutils.serenity.snake import snake
+                            
+    
+def serenity_snake(envname,package,version):
+    old = log.set_threshold(log.INFO)
+
+    log.info("Installing %s (%s) in serenity mode" % (package,version))
+
+    enforce_good_python()
+
+    virtualpython=serenity_virtualenv(envname,package,version)
+    
+    if virtualpython!=os.path.realpath(sys.executable):
+        log.info("Restarting installation within the %s virtualenv" % (envname))
+        rerun_with_anothe_python(virtualpython)
+        
+    log.info("%s will be installed with python : %s" % (package,virtualpython))
+        
+    if install_requirements():    
+        log.info("Restarting installation with all dependencies ok")
+        rerun_with_anothe_python(virtualpython)
+    
+    log.set_threshold(old)
+    
+def serenity_assert(version):
+    check_requirements()
+
+
+def is_serenity():
+    from obidistutils.serenity.globals import local_serenity
+    return local_serenity and local_serenity[0]
+
+def serenity_mode(package,version):
+    
+    save_argv()
+
+    
+    from obidistutils.serenity.globals import saved_args
+    from obidistutils.serenity.globals import local_serenity
+    
+
+    old = log.set_threshold(log.INFO)
+    
+    argparser = argparse.ArgumentParser(add_help=False)
+    argparser.add_argument('--serenity',
+                           dest='serenity', 
+                           action='store_true',
+                           default=False, 
+                           help='Switch the installer in serenity mode. Everythings are installed in a virtualenv')
+
+    argparser.add_argument('--virtualenv',
+                           dest='virtual', 
+                           type=str,
+                           action='store',
+                           default="%s-%s" % (package,version), 
+                           help='Specify the name of the virtualenv used by the serenity mode [default: %s-%s]' % (package,version))    
+    
+    args, unknown = argparser.parse_known_args()
+    sys.argv = [sys.argv[0]] + unknown
+    
+    if args.serenity:
+        local_serenity.append(True)
+        serenity_snake(args.virtual,package,version)
+    else:
+        local_serenity.append(False)       
+    
+    log.set_threshold(old)
+    
+    return args.serenity
+    
+    
+def getVersion(source,main,version):
+    path  = os.path.join(source,main,'%s.py' % version)
+    spec = spec_from_file_location('version',path)
+    return spec.loader.load_module().version.strip()
+    

distutils.ext/obidistutils/serenity/bootstrappip.py

@@ -0,0 +1,36 @@
+'''
+Created on 22 janv. 2016
+
+@author: coissac
+'''
+
+import sys
+from urllib import request
+import os.path
+
+from obidistutils.serenity.util import get_serenity_dir
+from obidistutils.serenity.rerun import rerun_with_anothe_python
+from obidistutils.serenity.checkpython import is_a_virtualenv_python
+
+getpipurl="https://bootstrap.pypa.io/get-pip.py"
+
+def bootstrap():
+
+    getpipfile=os.path.join(get_serenity_dir(),"get-pip.py")
+    
+    with request.urlopen(getpipurl) as getpip:
+        with open(getpipfile,"wb") as out:
+            for l in getpip:
+                out.write(l)
+                
+    python = sys.executable
+    
+    if is_a_virtualenv_python():
+        command= "%s %s" % (python,getpipfile)        
+    else:
+        command= "%s %s --user" % (python,getpipfile)
+    
+    os.system(command)
+    
+    rerun_with_anothe_python(python)
+    

distutils.ext/obidistutils/serenity/checkpackage.py

@@ -0,0 +1,165 @@
+'''
+Created on 2 oct. 2014
+
+@author: coissac
+'''
+
+import re
+import os
+
+try:
+    import pip                                              # @UnresolvedImport
+    from pip.utils import get_installed_distributions       # @UnresolvedImport
+except ImportError:
+    from .bootstrappip import bootstrap
+    bootstrap()
+
+from distutils.version import StrictVersion             # @UnusedImport
+from distutils.errors import DistutilsError
+from distutils import log
+
+class RequirementError(Exception):
+    pass
+
+def is_installed(requirement):
+    
+    
+    requirement_project,requirement_relation,requirement_version = parse_package_requirement(requirement)
+    
+    package = [x for x in get_installed_distributions() if x.project_name==requirement_project]
+    
+    if len(package)==1:
+        if requirement_version is not None and requirement_relation is not None:    
+            rep = (len(package)==1) and eval("StrictVersion('%s') %s StrictVersion('%s')" % (package[0].version,
+                                                                                           requirement_relation,
+                                                                                           requirement_version)
+                                             )
+        else:
+            rep=True
+    else:
+        rep=False
+    
+    if rep:
+        if requirement_version is not None and requirement_relation is not None:        
+            log.info("Look for package %s (%s%s) : ok version %s installed" % (requirement_project,
+                                                                               requirement_relation,
+                                                                               requirement_version,
+                                                                               package[0].version))
+        else:
+            log.info("Look for package %s : ok version %s installed" % (requirement_project,
+                                                                        package[0].version))
+    else:
+        if len(package)!=1:
+            log.info("Look for package %s (%s%s) : not installed" % (requirement_project,
+                                                                     requirement_relation,
+                                                                     requirement_version))
+        else:
+            log.info("Look for package %s (%s%s) : failed only version %s installed" % (requirement_project,
+                                                                                        requirement_relation,
+                                                                                        requirement_version,
+                                                                                        package[0].version))
+        
+    return rep
+
+
+def get_requirements(requirementfile='requirements.txt'):
+    
+    try:
+        requirements = open(requirementfile).readlines()
+        requirements = [x.strip() for x in requirements]
+        requirements = [x for x in requirements if x[0]!='-']
+    
+    except IOError:
+        requirements = []
+        
+    return requirements
+    
+    
+def install_requirements(requirementfile='requirements.txt'):
+    
+    install_something=False
+
+    requirements = get_requirements(requirementfile)
+
+    log.info("Required packages for the installation :")
+    for x in requirements:
+        ok = is_installed(x)
+        if not ok:
+            log.info("  Installing requirement : %s" % x)
+            pip_install_package(x)
+            install_something=True
+            if x[0:3]=='pip':
+                return True
+                
+    return install_something
+ 
+
+def check_requirements(requirementfile='requirements.txt'):
+    
+    requirements = get_requirements(requirementfile)
+    
+    log.info("Required packages for the installation :")
+    for x in requirements:
+        ok = is_installed(x)
+        if not ok:
+            raise RequirementError("  Missing requirement : %s -- Package installation stopped" % x)
+                
+ 
+
+
+def parse_package_requirement(requirement):
+    
+    version_pattern = re.compile('[=><]+(.*)$')
+    project_pattern  = re.compile('[^=><]+')
+    relationship_pattern = re.compile('[=><]+')
+    
+    try:
+        requirement_project = project_pattern.search(requirement).group(0)
+        requirement_version = version_pattern.search(requirement)
+        if requirement_version is not None:
+            requirement_version=requirement_version.group(1)
+        requirement_relation= relationship_pattern.search(requirement)
+        if requirement_relation is not None:
+            requirement_relation=requirement_relation.group(0)
+    except:
+        raise DistutilsError("Requirement : %s not correctly formated" % requirement)
+    
+    return requirement_project,requirement_relation,requirement_version
+    
+
+def get_package_requirement(package,requirementfile='requirements.txt'):            
+    requirements = get_requirements(requirementfile)
+    req = [x for x in requirements
+             if x[0:len(package)]==package
+          ]
+    
+    if len(req)==1:
+        return req[0]
+    else:
+        return None
+        
+        
+def pip_install_package(package,directory=None,upgrade=True):
+
+    if directory is not None:
+        log.info('    installing %s in directory %s' % (package,str(directory)))
+        
+
+    if 'http_proxy' in os.environ and 'https_proxy' not in os.environ:
+        os.environ['https_proxy']=os.environ['http_proxy']
+
+    args = ['install']
+    
+    if upgrade:
+        args.append('--upgrade')
+    
+    if 'https_proxy' in os.environ:
+        args.append('--proxy=%s' % os.environ['https_proxy'])
+        
+    if directory is not None:
+        args.append('--target=%s' % directory)
+    
+    args.append(package)
+        
+    return pip.main(args)
+

distutils.ext/obidistutils/serenity/checkpython.py

@@ -0,0 +1,138 @@
+'''
+Created on 24 mai 2015
+
+@author: coissac
+'''
+from distutils.version import StrictVersion
+from distutils import sysconfig
+import subprocess
+import os
+import glob
+import re
+
+from obidistutils.serenity.checksystem import is_windows_system
+import sys
+
+
+def is_python_version(path=None,minversion='3.4',maxversion=None):
+    '''
+    Checks that the python version is in the range {minversion,maxversion[
+    
+        @param path: if None consider the running python
+                     otherwise the python pointed by the path
+        @param minversion: the minimum version to consider
+        @param maxversion: the maximum version to consider (strictly inferior to)
+                     
+        @return: True if the python version match
+        @rtype: bool
+    '''
+    if path is None:
+        pythonversion = StrictVersion(sysconfig.get_python_version())
+    else:
+        command = """'%s' -c 'from distutils import sysconfig; """ \
+                  """print(sysconfig.get_python_version())'""" % path
+                  
+        p = subprocess.Popen(command, 
+                             shell=True, 
+                             stdout=subprocess.PIPE)
+        pythonversion=str(p.communicate()[0],'utf8').strip()
+        pythonversion = StrictVersion(pythonversion)
+                        
+    return  (        pythonversion >=StrictVersion(minversion) 
+             and (   maxversion is None    
+                  or pythonversion < StrictVersion(maxversion))
+            )
+           
+
+def lookfor_good_python(minversion='3.4',maxversion=None,followLink=False):
+    '''
+    Look for all python interpreters present in the system path that
+    match the version constraints.
+    
+    @param minversion: the minimum version to consider
+    @param maxversion: the maximum version to consider (strictly inferior to)
+    @param followLink: a boolean value indicating if link must be substituted 
+                       by their real path.
+                       
+    @return: a list of path to interpreters
+    '''
+    exe = []
+    if not is_windows_system():
+        paths = os.environ['PATH'].split(os.pathsep)
+        for p in paths:
+            candidates = glob.glob(os.path.join(p,'python*')) 
+            pexe = []
+            pythonpat=re.compile('python([0-9]|[0-9]\.[0-9])?$')
+            for e in candidates:
+                print(e)
+                if pythonpat.search(e) is not None:
+                    if followLink and os.path.islink(e):
+                        e = os.path.realpath(e)
+                    if (os.path.isfile(e) and 
+                       os.access(e, os.X_OK) and 
+                       is_python_version(e,minversion,maxversion)):
+                        pexe.append(e)
+            exe.extend(set(pexe))
+        
+    return exe
+
+def is_a_virtualenv_python(path=None):
+    '''
+    Check if the python is belonging a virtualenv
+    
+        @param path: the path pointing to the python executable.
+                     if path is None then the running python is
+                     considered.
+        @param path: str or None 
+                     
+        @return: True if the python belongs a virtualenv
+                 False otherwise
+        @rtype: bool
+                 
+    '''
+    if path is None:
+        rep = sys.base_exec_prefix != sys.exec_prefix
+    else:
+        command = """'%s' -c 'import sys; print(sys.base_exec_prefix != sys.exec_prefix)'""" % path
+        p = subprocess.Popen(command, 
+                             shell=True, 
+                             stdout=subprocess.PIPE)
+        rep = eval(str(p.communicate()[0],'utf8'))
+        
+    return rep
+
+
+def which_virtualenv(path=None,full=False):
+    '''
+    Returns the name of the virtualenv.
+        @param path: the path to a python binary or None
+                     if you want to consider the running python
+        @type path: str or None
+                     
+        @param full: if set to True, returns the absolute path,
+                     otherwise only return a simple directory name
+        @type full: bool
+                 
+        @return: the virtual environment name or None if the
+                 path does not belong a virtualenv
+        @rtype: str or None
+    '''
+    if path is None:
+        path = sys.executable
+    
+    if is_a_virtualenv_python(path):
+        parts = path.split(os.sep)
+        try:
+            if full:
+                rep = os.sep.join(parts[0:parts.index('bin')])
+                rep = os.path.realpath(rep)
+            else:
+                rep = parts[parts.index('bin')-1]
+        except ValueError:
+            rep = None
+    else:
+        rep=None
+        
+    return rep
+        
+

distutils.ext/obidistutils/serenity/checksystem.py

@@ -0,0 +1,18 @@
+'''
+Created on 24 mai 2015
+
+@author: coissac
+'''
+
+from distutils import util
+
+def is_mac_system():
+    platform = util.get_platform().split('-')[0]
+    
+    return platform=='macosx'
+
+def is_windows_system():
+    platform = util.get_platform().split('-')[0]
+    
+    return platform=='Windows'
+        

distutils.ext/obidistutils/serenity/globals.py

@@ -0,0 +1,14 @@
+'''
+Created on 24 mai 2015
+
+@author: coissac
+'''
+
+
+saved_args=[]
+tmpdir=[]
+local_pip=[]
+local_virtualenv=[]
+local_cython=[]
+local_serenity=[]
+        

distutils.ext/obidistutils/serenity/rerun.py

@@ -0,0 +1,61 @@
+'''
+Created on 24 mai 2015
+
+@author: coissac
+'''
+
+import sys
+import os
+from distutils import log
+from distutils.errors import DistutilsError
+
+
+from obidistutils.serenity.globals import saved_args
+from obidistutils.serenity.checkpython import is_python_version,\
+    lookfor_good_python
+
+
+def rerun_with_anothe_python(path, minversion='3.4',maxversion=None, fork=False):
+        
+    if saved_args:
+        args = saved_args
+    else:
+        args = list(sys.argv)
+        
+          
+    assert is_python_version(path,minversion,maxversion), \
+           'the selected python is not adapted to the installation of this package'
+                   
+    args.insert(0, path)
+        
+    sys.stderr.flush()
+    sys.stdout.flush()
+    
+    if fork:
+        log.info('Forking a new install process')
+        os.system(' '.join(list(args)))
+        log.info('External process ended')
+        sys.exit(0)
+    else:
+        log.info('Install script restarting...')
+        os.execv(path,list(args))
+
+def enforce_good_python(minversion='3.4',maxversion=None, fork=False):
+    if is_python_version(minversion=minversion,maxversion=maxversion):
+        log.info('You are running the good python')
+        return True
+    
+    goodpython = lookfor_good_python(minversion,maxversion)
+    
+    if not goodpython:
+        raise DistutilsError('No good python identified on your system')
+
+    goodpython=goodpython[0]
+    
+    log.warn("========================================")    
+    log.warn("")
+    log.warn("    Switching to python : %s" % goodpython)
+    log.warn("")
+    log.warn("========================================")    
+
+    rerun_with_anothe_python(goodpython)

distutils.ext/obidistutils/serenity/snake.py

@@ -0,0 +1,35 @@
+'''
+Created on 2 oct. 2014
+
+@author: coissac
+'''
+
+snake ="""
+
+        ___
+      ,'._,`.
+     (-.___.-)
+     (-.___.-)
+     `-.___.-'
+      ((  @ @|              .            __
+       \   ` |         ,\   |`.    @|   |  |      _.-._
+      __`.`=-=mm===mm:: |   | |`.   |   |  |    ,'=` '=`.
+     (    `-'|:/  /:/  `/  @| | |   |, @| @|   /---)W(---\ 
+      \ \   / /  / /         @| |   '         (----| |----) ,~
+      |\ \ / /| / /            @|              \---| |---/  |
+      | \ V /||/ /                              `.-| |-,'   |
+      |  `-' |V /                                 \| |/    @'
+      |    , |-'                                 __| |__
+      |    .;: _,-.                         ,--""..| |..""--.
+      ;;:::' "    )                        (`--::__|_|__::--')
+    ,-"      _,  /                          \`--...___...--'/
+   (    -:--'/  /                           /`--...___...--'\  
+    "-._  `"'._/                           /`---...___...---'\  
+        "-._   "---.                      (`---....___....---')
+         .' ",._ ,' )                     |`---....___....---'|
+         /`._|  `|  |                     (`---....___....---')
+        (   \    |  /                      \`---...___...---'/
+         `.  `,  ^""                        `:--...___...--;'
+           `.,'               hh              `-._______.-'
+
+"""

distutils.ext/obidistutils/serenity/util.py

@@ -0,0 +1,27 @@
+'''
+Created on 2 oct. 2014
+
+@author: coissac
+'''
+
+import sys 
+import tempfile
+
+
+from obidistutils.serenity.globals import tmpdir         # @UnusedImport
+from obidistutils.serenity.globals import saved_args     # @UnusedImport
+
+def get_serenity_dir():
+    global tmpdir
+
+    if not tmpdir:
+        tmpdir.append(tempfile.mkdtemp())
+    return tmpdir[0]
+        
+def save_argv():
+    global saved_args
+
+    del saved_args[:]
+    saved_args.extend(list(sys.argv))
+
+           

distutils.ext/obidistutils/serenity/virtual.py

@@ -0,0 +1,69 @@
+'''
+Created on 2 oct. 2014
+
+@author: coissac
+'''
+
+import os
+import sys
+import venv
+
+from distutils.errors import DistutilsError
+from .globals import local_virtualenv  # @UnusedImport
+from .checkpython import   which_virtualenv,\
+                           is_python_version, \
+                           is_a_virtualenv_python
+                                                
+
+    
+    
+def serenity_virtualenv(envname,package,version,minversion='3.4',maxversion=None):
+        
+    #
+    # Checks if we are already running under the good virtualenv
+    #
+    ve = which_virtualenv(full=True)
+    if ve == os.path.realpath(envname) and is_python_version(minversion=minversion,maxversion=maxversion):
+        return sys.executable
+        
+    #
+    # Check if the virtualenv exist
+    # 
+        
+    python = None
+    
+    if os.path.isdir(envname):
+        python = os.path.join(envname,'bin','python')
+        ok = (is_python_version(python,
+                                minversion=minversion,
+                                maxversion=maxversion) and 
+              is_a_virtualenv_python(python))
+        
+        
+        #
+        # The virtualenv already exist but it is not ok
+        #
+        if not ok:
+            raise DistutilsError("A virtualenv %s already exists but not with the required python")
+                 
+    else:
+        ok = False
+                
+        
+    #
+    # Creates a new virtualenv
+    #
+    if not ok:
+        venv.create(envname, 
+                    system_site_packages=False, 
+                    clear=True, 
+                    symlinks=False, 
+                    with_pip=True)
+                
+        # check the newly created virtualenv
+        return serenity_virtualenv(envname,package,version)
+    
+    return os.path.realpath(python)
+    
+    
+    

distutils.ext/src/littlebigman.c

@@ -0,0 +1,24 @@
+/*
+ * littlebigman.c
+ *
+ *  Created on: 11 juil. 2012
+ *      Author: coissac
+ */
+
+#include<stdio.h>
+
+int main(int argc, char *argv[])
+{
+    union { int entier;
+            char caractere[4] ;
+    } test;
+
+    test.entier=0x01020304;
+
+    if (test.caractere[3] == 1)
+       printf("-DLITTLE_END");
+    else
+        printf("-DBIG_END");
+
+	return 0;
+}

distutils.ext/src/pidname.c

@@ -0,0 +1,24 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <libproc.h>
+
+int main (int argc, char* argv[])
+{
+	pid_t pid; int ret;
+	char pathbuf[PROC_PIDPATHINFO_MAXSIZE];
+
+	if ( argc > 1 ) {
+		pid = (pid_t) atoi(argv[1]);
+		ret = proc_pidpath (pid, pathbuf, sizeof(pathbuf));
+		if ( ret <= 0 ) {
+			fprintf(stderr, "PID %d: proc_pidpath ();\n", pid);
+			fprintf(stderr,	"    %s\n", strerror(errno));
+		} else {
+			printf("proc %d: %s\n", pid, pathbuf);
+		}
+	}
+
+	return 0;
+}

doc/conf.py

@@ -33,10 +33,10 @@ extensions = [
     'sphinx.ext.autodoc',
     'sphinx.ext.todo',
     'sphinx.ext.coverage',
-    'sphinx.ext.imgmath',
+    'sphinx.ext.pngmath',
     'sphinx.ext.ifconfig',
     'sphinx.ext.viewcode',
-    'breathe',
+#    'breathe',
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -295,6 +295,4 @@ texinfo_documents = [
 sys.path.append( "breathe/" )
 breathe_projects = { "OBITools3": "doxygen/xml/" }
 breathe_default_project = "OBITools3"
-#breathe_projects_source = {
-#     "auto" : ( "../src", ["obidms.h", "obiavl.h"] )
-#     }
+

doc/source/index.rst

@@ -11,7 +11,7 @@ OBITools3 documentation
 
    Programming guidelines <guidelines>
    Data structures <data>
-   Code documentation <code_doc/codedoc>
+
 
 Indices and tables
 ------------------

doc/source/types.rst

@@ -4,7 +4,6 @@ OBITypes
 
     
 .. image:: ./UML/OBITypes_UML.png
-
 :download:`html version of the OBITypes UML file <UML/OBITypes_UML.class.violet.html>`
 
 

doc/sphinx/.gitignore

@@ -1,2 +0,0 @@
-.DS_Store
-/build_dir.txt

doc/sphinx/build_dir.txt

@@ -0,0 +1 @@
+build/lib.macosx-10.6-intel-3.5

python/.gitignore

@@ -1 +0,0 @@
-/OBITools3.egg-info/

python/obi.py

@@ -0,0 +1,50 @@
+#!/usr/local/bin/python3.4
+'''
+obi -- shortdesc
+
+obi is a description
+
+It defines classes_and_methods
+
+@author:     user_name
+
+@copyright:  2014 organization_name. All rights reserved.
+
+@license:    license
+
+@contact:    user_email
+@deffield    updated: Updated
+'''
+
+
+default_config = { 'software'       : "The OBITools",
+                   'log'            : False,
+                   'loglevel'       : 'INFO',
+                   'progress'       : True,
+                   'defaultdms'     : None
+                  }
+
+root_config_name='obi'
+
+from obitools3.apps.config import getConfiguration     # @UnresolvedImport
+from obitools3.version import version
+
+__all__     = []
+__version__ = version
+__date__    = '2014-09-28'
+__updated__ = '2014-09-28'
+
+DEBUG = 1
+TESTRUN = 0
+PROFILE = 0
+
+
+if __name__ =="__main__":
+    
+    config = getConfiguration(root_config_name,
+                              default_config)    
+                
+    config[root_config_name]['module'].run(config)
+    
+    
+    

python/obitools3/apps/arguments.cfiles

@@ -1,110 +0,0 @@
-../../../src/obi_lcs.h
-../../../src/obi_lcs.c
-../../../src/obierrno.h
-../../../src/obierrno.c
-../../../src/upperband.h
-../../../src/upperband.c
-../../../src/sse_banded_LCS_alignment.h
-../../../src/sse_banded_LCS_alignment.c
-../../../src/obiblob.h
-../../../src/obiblob.c
-../../../src/utils.h
-../../../src/utils.c
-../../../src/obidms.h
-../../../src/obidms.c
-../../../src/libjson/json_utils.h
-../../../src/libjson/json_utils.c
-../../../src/libjson/cJSON.h
-../../../src/libjson/cJSON.c
-../../../src/obiavl.h
-../../../src/obiavl.c
-../../../src/bloom.h
-../../../src/bloom.c
-../../../src/crc64.h
-../../../src/crc64.c
-../../../src/murmurhash2.h
-../../../src/murmurhash2.c
-../../../src/obidmscolumn.h
-../../../src/obidmscolumn.c
-../../../src/obitypes.h
-../../../src/obitypes.c
-../../../src/obidmscolumndir.h
-../../../src/obidmscolumndir.c
-../../../src/obiblob_indexer.h
-../../../src/obiblob_indexer.c
-../../../src/obiview.h
-../../../src/obiview.c
-../../../src/hashtable.h
-../../../src/hashtable.c
-../../../src/linked_list.h
-../../../src/linked_list.c
-../../../src/obidmscolumn_array.h
-../../../src/obidmscolumn_array.c
-../../../src/obidmscolumn_blob.h
-../../../src/obidmscolumn_blob.c
-../../../src/obidmscolumn_idx.h
-../../../src/obidmscolumn_idx.c
-../../../src/obidmscolumn_bool.h
-../../../src/obidmscolumn_bool.c
-../../../src/obidmscolumn_char.h
-../../../src/obidmscolumn_char.c
-../../../src/obidmscolumn_float.h
-../../../src/obidmscolumn_float.c
-../../../src/obidmscolumn_int.h
-../../../src/obidmscolumn_int.c
-../../../src/obidmscolumn_qual.h
-../../../src/obidmscolumn_qual.c
-../../../src/obidmscolumn_seq.h
-../../../src/obidmscolumn_seq.c
-../../../src/obidmscolumn_str.h
-../../../src/obidmscolumn_str.c
-../../../src/array_indexer.h
-../../../src/array_indexer.c
-../../../src/char_str_indexer.h
-../../../src/char_str_indexer.c
-../../../src/dna_seq_indexer.h
-../../../src/dna_seq_indexer.c
-../../../src/encode.c
-../../../src/encode.h
-../../../src/uint8_indexer.c
-../../../src/uint8_indexer.h
-../../../src/build_reference_db.c
-../../../src/build_reference_db.h
-../../../src/kmer_similarity.c
-../../../src/kmer_similarity.h
-../../../src/obi_clean.c
-../../../src/obi_clean.h
-../../../src/obi_ecopcr.c
-../../../src/obi_ecopcr.h
-../../../src/obi_ecotag.c
-../../../src/obi_ecotag.h
-../../../src/obidms_taxonomy.c
-../../../src/obidms_taxonomy.h
-../../../src/obilittlebigman.c
-../../../src/obilittlebigman.h
-../../../src/_sse.h
-../../../src/obidebug.h
-../../../src/libecoPCR/libapat/CODES/dft_code.h
-../../../src/libecoPCR/libapat/CODES/dna_code.h
-../../../src/libecoPCR/libapat/CODES/prot_code.h
-../../../src/libecoPCR/libapat/apat_parse.c
-../../../src/libecoPCR/libapat/apat_search.c
-../../../src/libecoPCR/libapat/apat.h
-../../../src/libecoPCR/libapat/Gmach.h
-../../../src/libecoPCR/libapat/Gtypes.h
-../../../src/libecoPCR/libapat/libstki.c
-../../../src/libecoPCR/libapat/libstki.h
-../../../src/libecoPCR/libthermo/nnparams.h
-../../../src/libecoPCR/libthermo/nnparams.c
-../../../src/libecoPCR/ecoapat.c
-../../../src/libecoPCR/ecodna.c
-../../../src/libecoPCR/ecoError.c
-../../../src/libecoPCR/ecoMalloc.c
-../../../src/libecoPCR/ecoPCR.h
-
-
-
-
-
-
-

python/obitools3/apps/arguments.pyx

@@ -57,6 +57,5 @@ cpdef buildArgumentParser(str configname,
                 module.addOptions(sub)
             
             sub.set_defaults(**{'%s:module'  % configname : module})
-            sub.set_defaults(**{'%s:modulename'  % configname : c})
-              
+                        
     return parser

python/obitools3/apps/command.cfiles

@@ -1,110 +0,0 @@
-../../../src/obi_lcs.h
-../../../src/obi_lcs.c
-../../../src/obierrno.h
-../../../src/obierrno.c
-../../../src/upperband.h
-../../../src/upperband.c
-../../../src/sse_banded_LCS_alignment.h
-../../../src/sse_banded_LCS_alignment.c
-../../../src/obiblob.h
-../../../src/obiblob.c
-../../../src/utils.h
-../../../src/utils.c
-../../../src/obidms.h
-../../../src/obidms.c
-../../../src/libjson/json_utils.h
-../../../src/libjson/json_utils.c
-../../../src/libjson/cJSON.h
-../../../src/libjson/cJSON.c
-../../../src/obiavl.h
-../../../src/obiavl.c
-../../../src/bloom.h
-../../../src/bloom.c
-../../../src/crc64.h
-../../../src/crc64.c
-../../../src/murmurhash2.h
-../../../src/murmurhash2.c
-../../../src/obidmscolumn.h
-../../../src/obidmscolumn.c
-../../../src/obitypes.h
-../../../src/obitypes.c
-../../../src/obidmscolumndir.h
-../../../src/obidmscolumndir.c
-../../../src/obiblob_indexer.h
-../../../src/obiblob_indexer.c
-../../../src/obiview.h
-../../../src/obiview.c
-../../../src/hashtable.h
-../../../src/hashtable.c
-../../../src/linked_list.h
-../../../src/linked_list.c
-../../../src/obidmscolumn_array.h
-../../../src/obidmscolumn_array.c
-../../../src/obidmscolumn_blob.h
-../../../src/obidmscolumn_blob.c
-../../../src/obidmscolumn_idx.h
-../../../src/obidmscolumn_idx.c
-../../../src/obidmscolumn_bool.h
-../../../src/obidmscolumn_bool.c
-../../../src/obidmscolumn_char.h
-../../../src/obidmscolumn_char.c
-../../../src/obidmscolumn_float.h
-../../../src/obidmscolumn_float.c
-../../../src/obidmscolumn_int.h
-../../../src/obidmscolumn_int.c
-../../../src/obidmscolumn_qual.h
-../../../src/obidmscolumn_qual.c
-../../../src/obidmscolumn_seq.h
-../../../src/obidmscolumn_seq.c
-../../../src/obidmscolumn_str.h
-../../../src/obidmscolumn_str.c
-../../../src/array_indexer.h
-../../../src/array_indexer.c
-../../../src/char_str_indexer.h
-../../../src/char_str_indexer.c
-../../../src/dna_seq_indexer.h
-../../../src/dna_seq_indexer.c
-../../../src/encode.c
-../../../src/encode.h
-../../../src/uint8_indexer.c
-../../../src/uint8_indexer.h
-../../../src/build_reference_db.c
-../../../src/build_reference_db.h
-../../../src/kmer_similarity.c
-../../../src/kmer_similarity.h
-../../../src/obi_clean.c
-../../../src/obi_clean.h
-../../../src/obi_ecopcr.c
-../../../src/obi_ecopcr.h
-../../../src/obi_ecotag.c
-../../../src/obi_ecotag.h
-../../../src/obidms_taxonomy.c
-../../../src/obidms_taxonomy.h
-../../../src/obilittlebigman.c
-../../../src/obilittlebigman.h
-../../../src/_sse.h
-../../../src/obidebug.h
-../../../src/libecoPCR/libapat/CODES/dft_code.h
-../../../src/libecoPCR/libapat/CODES/dna_code.h
-../../../src/libecoPCR/libapat/CODES/prot_code.h
-../../../src/libecoPCR/libapat/apat_parse.c
-../../../src/libecoPCR/libapat/apat_search.c
-../../../src/libecoPCR/libapat/apat.h
-../../../src/libecoPCR/libapat/Gmach.h
-../../../src/libecoPCR/libapat/Gtypes.h
-../../../src/libecoPCR/libapat/libstki.c
-../../../src/libecoPCR/libapat/libstki.h
-../../../src/libecoPCR/libthermo/nnparams.h
-../../../src/libecoPCR/libthermo/nnparams.c
-../../../src/libecoPCR/ecoapat.c
-../../../src/libecoPCR/ecodna.c
-../../../src/libecoPCR/ecoError.c
-../../../src/libecoPCR/ecoMalloc.c
-../../../src/libecoPCR/ecoPCR.h
-
-
-
-
-
-
-

python/obitools3/apps/config.cfiles

@@ -1,110 +0,0 @@
-../../../src/obi_lcs.h
-../../../src/obi_lcs.c
-../../../src/obierrno.h
-../../../src/obierrno.c
-../../../src/upperband.h
-../../../src/upperband.c
-../../../src/sse_banded_LCS_alignment.h
-../../../src/sse_banded_LCS_alignment.c
-../../../src/obiblob.h
-../../../src/obiblob.c
-../../../src/utils.h
-../../../src/utils.c
-../../../src/obidms.h
-../../../src/obidms.c
-../../../src/libjson/json_utils.h
-../../../src/libjson/json_utils.c
-../../../src/libjson/cJSON.h
-../../../src/libjson/cJSON.c
-../../../src/obiavl.h
-../../../src/obiavl.c
-../../../src/bloom.h
-../../../src/bloom.c
-../../../src/crc64.h
-../../../src/crc64.c
-../../../src/murmurhash2.h
-../../../src/murmurhash2.c
-../../../src/obidmscolumn.h
-../../../src/obidmscolumn.c
-../../../src/obitypes.h
-../../../src/obitypes.c
-../../../src/obidmscolumndir.h
-../../../src/obidmscolumndir.c
-../../../src/obiblob_indexer.h
-../../../src/obiblob_indexer.c
-../../../src/obiview.h
-../../../src/obiview.c
-../../../src/hashtable.h
-../../../src/hashtable.c
-../../../src/linked_list.h
-../../../src/linked_list.c
-../../../src/obidmscolumn_array.h
-../../../src/obidmscolumn_array.c
-../../../src/obidmscolumn_blob.h
-../../../src/obidmscolumn_blob.c
-../../../src/obidmscolumn_idx.h
-../../../src/obidmscolumn_idx.c
-../../../src/obidmscolumn_bool.h
-../../../src/obidmscolumn_bool.c
-../../../src/obidmscolumn_char.h
-../../../src/obidmscolumn_char.c
-../../../src/obidmscolumn_float.h
-../../../src/obidmscolumn_float.c
-../../../src/obidmscolumn_int.h
-../../../src/obidmscolumn_int.c
-../../../src/obidmscolumn_qual.h
-../../../src/obidmscolumn_qual.c
-../../../src/obidmscolumn_seq.h
-../../../src/obidmscolumn_seq.c
-../../../src/obidmscolumn_str.h
-../../../src/obidmscolumn_str.c
-../../../src/array_indexer.h
-../../../src/array_indexer.c
-../../../src/char_str_indexer.h
-../../../src/char_str_indexer.c
-../../../src/dna_seq_indexer.h
-../../../src/dna_seq_indexer.c
-../../../src/encode.c
-../../../src/encode.h
-../../../src/uint8_indexer.c
-../../../src/uint8_indexer.h
-../../../src/build_reference_db.c
-../../../src/build_reference_db.h
-../../../src/kmer_similarity.c
-../../../src/kmer_similarity.h
-../../../src/obi_clean.c
-../../../src/obi_clean.h
-../../../src/obi_ecopcr.c
-../../../src/obi_ecopcr.h
-../../../src/obi_ecotag.c
-../../../src/obi_ecotag.h
-../../../src/obidms_taxonomy.c
-../../../src/obidms_taxonomy.h
-../../../src/obilittlebigman.c
-../../../src/obilittlebigman.h
-../../../src/_sse.h
-../../../src/obidebug.h
-../../../src/libecoPCR/libapat/CODES/dft_code.h
-../../../src/libecoPCR/libapat/CODES/dna_code.h
-../../../src/libecoPCR/libapat/CODES/prot_code.h
-../../../src/libecoPCR/libapat/apat_parse.c
-../../../src/libecoPCR/libapat/apat_search.c
-../../../src/libecoPCR/libapat/apat.h
-../../../src/libecoPCR/libapat/Gmach.h
-../../../src/libecoPCR/libapat/Gtypes.h
-../../../src/libecoPCR/libapat/libstki.c
-../../../src/libecoPCR/libapat/libstki.h
-../../../src/libecoPCR/libthermo/nnparams.h
-../../../src/libecoPCR/libthermo/nnparams.c
-../../../src/libecoPCR/ecoapat.c
-../../../src/libecoPCR/ecodna.c
-../../../src/libecoPCR/ecoError.c
-../../../src/libecoPCR/ecoMalloc.c
-../../../src/libecoPCR/ecoPCR.h
-
-
-
-
-
-
-

python/obitools3/apps/config.pxd

@@ -7,4 +7,4 @@ cdef dict buildDefaultConfiguration(str root_config_name,
                                     dict  config)
 
 cpdef dict getConfiguration(str root_config_name=?,
-                                    dict  config=?)
+                                    dict  config=?)

python/obitools3/apps/config.pyx

@@ -101,14 +101,3 @@ cpdef dict getConfiguration(str root_config_name="__default__",
     config['__done__']=True
             
     return config
-
-def logger(level, *messages):
-    try:
-        config=getConfiguration()
-        root = config["__root_config__"]
-        l = config[root]['logger']
-        if config[root]['verbose']:
-            getattr(l, level)(*messages)
-    except:
-        print(*messages,file=sys.stderr)
-

python/obitools3/apps/logging.cfiles

@@ -1,110 +0,0 @@
-../../../src/obi_lcs.h
-../../../src/obi_lcs.c
-../../../src/obierrno.h
-../../../src/obierrno.c
-../../../src/upperband.h
-../../../src/upperband.c
-../../../src/sse_banded_LCS_alignment.h
-../../../src/sse_banded_LCS_alignment.c
-../../../src/obiblob.h
-../../../src/obiblob.c
-../../../src/utils.h
-../../../src/utils.c
-../../../src/obidms.h
-../../../src/obidms.c
-../../../src/libjson/json_utils.h
-../../../src/libjson/json_utils.c
-../../../src/libjson/cJSON.h
-../../../src/libjson/cJSON.c
-../../../src/obiavl.h
-../../../src/obiavl.c
-../../../src/bloom.h
-../../../src/bloom.c
-../../../src/crc64.h
-../../../src/crc64.c
-../../../src/murmurhash2.h
-../../../src/murmurhash2.c
-../../../src/obidmscolumn.h
-../../../src/obidmscolumn.c
-../../../src/obitypes.h
-../../../src/obitypes.c
-../../../src/obidmscolumndir.h
-../../../src/obidmscolumndir.c
-../../../src/obiblob_indexer.h
-../../../src/obiblob_indexer.c
-../../../src/obiview.h
-../../../src/obiview.c
-../../../src/hashtable.h
-../../../src/hashtable.c
-../../../src/linked_list.h
-../../../src/linked_list.c
-../../../src/obidmscolumn_array.h
-../../../src/obidmscolumn_array.c
-../../../src/obidmscolumn_blob.h
-../../../src/obidmscolumn_blob.c
-../../../src/obidmscolumn_idx.h
-../../../src/obidmscolumn_idx.c
-../../../src/obidmscolumn_bool.h
-../../../src/obidmscolumn_bool.c
-../../../src/obidmscolumn_char.h
-../../../src/obidmscolumn_char.c
-../../../src/obidmscolumn_float.h
-../../../src/obidmscolumn_float.c
-../../../src/obidmscolumn_int.h
-../../../src/obidmscolumn_int.c
-../../../src/obidmscolumn_qual.h
-../../../src/obidmscolumn_qual.c
-../../../src/obidmscolumn_seq.h
-../../../src/obidmscolumn_seq.c
-../../../src/obidmscolumn_str.h
-../../../src/obidmscolumn_str.c
-../../../src/array_indexer.h
-../../../src/array_indexer.c
-../../../src/char_str_indexer.h
-../../../src/char_str_indexer.c
-../../../src/dna_seq_indexer.h
-../../../src/dna_seq_indexer.c
-../../../src/encode.c
-../../../src/encode.h
-../../../src/uint8_indexer.c
-../../../src/uint8_indexer.h
-../../../src/build_reference_db.c
-../../../src/build_reference_db.h
-../../../src/kmer_similarity.c
-../../../src/kmer_similarity.h
-../../../src/obi_clean.c
-../../../src/obi_clean.h
-../../../src/obi_ecopcr.c
-../../../src/obi_ecopcr.h
-../../../src/obi_ecotag.c
-../../../src/obi_ecotag.h
-../../../src/obidms_taxonomy.c
-../../../src/obidms_taxonomy.h
-../../../src/obilittlebigman.c
-../../../src/obilittlebigman.h
-../../../src/_sse.h
-../../../src/obidebug.h
-../../../src/libecoPCR/libapat/CODES/dft_code.h
-../../../src/libecoPCR/libapat/CODES/dna_code.h
-../../../src/libecoPCR/libapat/CODES/prot_code.h
-../../../src/libecoPCR/libapat/apat_parse.c
-../../../src/libecoPCR/libapat/apat_search.c
-../../../src/libecoPCR/libapat/apat.h
-../../../src/libecoPCR/libapat/Gmach.h
-../../../src/libecoPCR/libapat/Gtypes.h
-../../../src/libecoPCR/libapat/libstki.c
-../../../src/libecoPCR/libapat/libstki.h
-../../../src/libecoPCR/libthermo/nnparams.h
-../../../src/libecoPCR/libthermo/nnparams.c
-../../../src/libecoPCR/ecoapat.c
-../../../src/libecoPCR/ecodna.c
-../../../src/libecoPCR/ecoError.c
-../../../src/libecoPCR/ecoMalloc.c
-../../../src/libecoPCR/ecoPCR.h
-
-
-
-
-
-
-

python/obitools3/apps/logging.pyx

@@ -22,7 +22,7 @@ cpdef getLogger(dict config):
     logfile= config[root]['log'] 
     
     rootlogger   = logging.getLogger()
-    logFormatter = logging.Formatter("%%(asctime)s [%s : %%(levelname)-5.5s]  %%(message)s" % config[root]['modulename'])
+    logFormatter = logging.Formatter("%(asctime)s [%(levelname)-5.5s]  %(message)s")
 
     stderrHandler = logging.StreamHandler(sys.stderr)
     stderrHandler.setFormatter(logFormatter)
@@ -42,7 +42,5 @@ cpdef getLogger(dict config):
     rootlogger.setLevel(loglevel)
     
     config[root]['logger']=rootlogger
-    config[root]['verbose']=True
-        
+    
     return rootlogger
-

python/obitools3/apps/optiongroups/__init__.py

@@ -1,272 +0,0 @@
-def __addInputOption(optionManager):
-    
-    optionManager.add_argument(
-                    dest='obi:inputURI',  
-                    metavar='INPUT', 
-                    help='Data source URI')
-    
-
-    group = optionManager.add_argument_group("Restriction to a sub-part options",
-                    "Allows to limit analysis to a sub-part of the input")
-
-    group.add_argument('--skip',
-                     action="store", dest="obi:skip",
-                     metavar='<N>',
-                     default=None,
-                     type=int,
-                     help="skip the N first sequences")
-
-    group.add_argument('--only',
-                     action="store", dest="obi:only",
-                     metavar='<N>',
-                     default=None,
-                     type=int,
-                     help="treat only N sequences")
-    
-
-def __addImportInputOption(optionManager):
-    group = optionManager.add_argument_group("Input format options for imported files")
-
-    group.add_argument('--fasta-input',
-                     action="store_const", dest="obi:inputformat",
-                     default=None,
-                     const=b'fasta',
-                     help="Input file is in sanger fasta format")
-
-    group.add_argument('--fastq-input',
-                     action="store_const", dest="obi:inputformat",
-                     default=None,
-                     const=b'fastq',
-                     help="Input file is in fastq format")
-
-    group.add_argument('--embl-input',
-                     action="store_const", dest="obi:inputformat",
-                     default=None,
-                     const=b'embl',
-                     help="Input file is in embl nucleic format")
-
-    group.add_argument('--genbank-input',
-                     action="store_const", dest="obi:inputformat",
-                     default=None,
-                     const=b'genbank',
-                     help="Input file is in genbank nucleic format")
-
-    group.add_argument('--ngsfilter-input',
-                     action="store_const", dest="obi:inputformat",
-                     default=None,
-                     const=b'ngsfilter',
-                     help="Input file is an ngsfilter file")
-
-    group.add_argument('--ecopcr-result-input',
-                     action="store_const", dest="obi:inputformat",
-                     default=None,
-                     const=b'ecopcr',
-                     help="Input file is the result of an ecoPCR (version 2)")
-
-    group.add_argument('--ecoprimers-result-input',
-                     action="store_const", dest="obi:inputformat",
-                     default=None,
-                     const=b'ecoprimers',
-                     help="Input file is the result of an ecoprimers")
-
-    group.add_argument('--tabular-input',
-                     action="store_const", dest="obi:inputformat",
-                     default=None,
-                     const=b'tabular',
-                     help="Input file is a tabular file")
-
-    group.add_argument('--no-skip-on-error',
-                     action="store_false", dest="obi:skiperror",
-                     default=True,
-                     help="Don't skip sequence entries with parsing errors (default: they are skipped)")
-
-    group.add_argument('--no-quality',
-                     action="store_true", dest="obi:noquality",
-                     default=False,
-                     help="Do not import fastQ quality")
-
-    group.add_argument('--quality-sanger',
-                     action="store_const", dest="obi:qualityformat",
-                     default=None,
-                     const=b'sanger',
-                     help="Fastq quality is encoded following sanger format (standard fastq)")
-
-    group.add_argument('--quality-solexa',
-                     action="store_const", dest="obi:qualityformat",
-                     default=None,
-                     const=b'solexa',
-                     help="Fastq quality is encoded following solexa sequencer format")
-
-    group.add_argument('--nuc',
-                     action="store_const", dest="obi:moltype",
-                     default=None,
-                     const=b'nuc',
-                     help="Input file contains nucleic sequences")
-    
-    group.add_argument('--prot',
-                     action="store_const", dest="obi:moltype",
-                     default=None,
-                     const=b'pep',
-                     help="Input file contains protein sequences")
-
-    group.add_argument('--input-na-string',
-                     action="store", dest="obi:inputnastring",
-                     default="NA",
-                     type=str,
-                     help="String associated with Non Available (NA) values in the input")
-
-  
-def __addTabularInputOption(optionManager):
-    group = optionManager.add_argument_group("Input format options for tabular files")
-
-    group.add_argument('--header',
-                     action="store_true", dest="obi:header",
-                     default=False,
-                     help="First line of tabular file contains column names")
-
-    group.add_argument('--sep',
-                     action="store", dest="obi:sep",
-                     default=None,
-                     type=str,
-                     help="Column separator")
-    
-    group.add_argument('--dec',
-                     action="store", dest="obi:dec",
-                     default=".",
-                     type=str,
-                     help="Decimal separator")
-    
-    group.add_argument('--strip-white',
-                     action="store_false", dest="obi:stripwhite",
-                     default=True,
-                     help="Remove white chars at the beginning and the end of values")
-
-    group.add_argument('--blank-line-skip',
-                     action="store_false", dest="obi:blanklineskip",
-                     default=True,
-                     help="Skip empty lines")
-
-    group.add_argument('--comment-char',
-                     action="store", dest="obi:commentchar",
-                     default="#",
-                     type=str,
-                     help="Lines starting by this char are considered as comment")
-
-
-def __addTaxdumpInputOption(optionManager):  # TODO maybe not the best way to do it
-    group = optionManager.add_argument_group("Input format options for taxdump")
-
-    group.add_argument('--taxdump',
-                     action="store_true", dest="obi:taxdump",
-                     default=False,
-                     help="Whether the input is a taxdump")
-
-
-def __addTaxonomyOption(optionManager):
-    group = optionManager.add_argument_group("Input format options for taxonomy")
-
-    group.add_argument('--taxonomy',
-                     action="store", dest="obi:taxoURI",
-                     default=None,
-                     help="Taxonomy URI")
-    
-    #TODO option bool to download taxo if URI doesn't exist
-
-
-def addMinimalInputOption(optionManager):
-    __addInputOption(optionManager)
-
-    
-def addImportInputOption(optionManager):
-    __addInputOption(optionManager)
-    __addImportInputOption(optionManager)
-
-
-def addTabularInputOption(optionManager):
-    __addTabularInputOption(optionManager)
-
-
-def addTaxonomyOption(optionManager):
-    __addTaxonomyOption(optionManager)
-
-
-def addTaxdumpInputOption(optionManager):
-    __addTaxdumpInputOption(optionManager)
-
-
-def addAllInputOption(optionManager):
-    __addInputOption(optionManager)
-    __addImportInputOption(optionManager)
-    __addTabularInputOption(optionManager)
-    __addTaxonomyOption(optionManager)
-    __addTaxdumpInputOption(optionManager)    
-
-
-def __addOutputOption(optionManager):
-    
-    optionManager.add_argument(
-                    dest='obi:outputURI',  
-                    metavar='OUTPUT', 
-                    help='Data destination URI')
-
-
-def __addDMSOutputOption(optionManager):
-    group = optionManager.add_argument_group("Output options for DMS data")
-
-    group.add_argument('--no-create-dms',
-                 action="store_true", dest="obi:nocreatedms",
-                 default=False,
-                 help="Don't create an output DMS is it is not existing")
-
-    group.add_argument('--max-elts',
-                 action="store", dest="obi:maxelts",
-                 metavar='<N>',
-                 default=1000,
-                 type=int,
-                 help="Maximum number of elements per line in a column "
-                      "(e.g. the number of different keys in a dictionary-type "
-                      "key from sequence headers). If the number of different keys "
-                      "is greater than N, the values are stored as character strings")
-
-
-def __addExportOutputOption(optionManager):
-    group = optionManager.add_argument_group("Output format options for exported files")
-
-    group.add_argument('--fasta-output',
-                     action="store_const", dest="obi:outputformat",
-                     default=None,
-                     const=b'fasta',
-                     help="Output file is in sanger fasta format")
-
-    group.add_argument('--fastq-output',
-                     action="store_const", dest="obi:outputformat",
-                     default=None,
-                     const=b'fastq',
-                     help="Output file is in fastq format")
-
-    group.add_argument('--print-na',
-                     action="store_true", dest="obi:printna",
-                     default=False,
-                     help="Print Non Available (NA) values in the output")
-
-    group.add_argument('--output-na-string',
-                     action="store", dest="obi:outputnastring",
-                     default="NA",
-                     type=str,
-                     help="String associated with Non Available (NA) values in the output")
-
-
-def addMinimalOutputOption(optionManager):
-    __addOutputOption(optionManager)
-    __addDMSOutputOption(optionManager)
-
-def addExportOutputOption(optionManager):
-    __addOutputOption(optionManager)
-    __addExportOutputOption(optionManager)
-
-
-def addAllOutputOption(optionManager):
-    __addOutputOption(optionManager)
-    __addDMSOutputOption(optionManager)
-    __addExportOutputOption(optionManager)
-

python/obitools3/apps/progress.cfiles

@@ -1,110 +0,0 @@
-../../../src/obi_lcs.h
-../../../src/obi_lcs.c
-../../../src/obierrno.h
-../../../src/obierrno.c
-../../../src/upperband.h
-../../../src/upperband.c
-../../../src/sse_banded_LCS_alignment.h
-../../../src/sse_banded_LCS_alignment.c
-../../../src/obiblob.h
-../../../src/obiblob.c
-../../../src/utils.h
-../../../src/utils.c
-../../../src/obidms.h
-../../../src/obidms.c
-../../../src/libjson/json_utils.h
-../../../src/libjson/json_utils.c
-../../../src/libjson/cJSON.h
-../../../src/libjson/cJSON.c
-../../../src/obiavl.h
-../../../src/obiavl.c
-../../../src/bloom.h
-../../../src/bloom.c
-../../../src/crc64.h
-../../../src/crc64.c
-../../../src/murmurhash2.h
-../../../src/murmurhash2.c
-../../../src/obidmscolumn.h
-../../../src/obidmscolumn.c
-../../../src/obitypes.h
-../../../src/obitypes.c
-../../../src/obidmscolumndir.h
-../../../src/obidmscolumndir.c
-../../../src/obiblob_indexer.h
-../../../src/obiblob_indexer.c
-../../../src/obiview.h
-../../../src/obiview.c
-../../../src/hashtable.h
-../../../src/hashtable.c
-../../../src/linked_list.h
-../../../src/linked_list.c
-../../../src/obidmscolumn_array.h
-../../../src/obidmscolumn_array.c
-../../../src/obidmscolumn_blob.h
-../../../src/obidmscolumn_blob.c
-../../../src/obidmscolumn_idx.h
-../../../src/obidmscolumn_idx.c
-../../../src/obidmscolumn_bool.h
-../../../src/obidmscolumn_bool.c
-../../../src/obidmscolumn_char.h
-../../../src/obidmscolumn_char.c
-../../../src/obidmscolumn_float.h
-../../../src/obidmscolumn_float.c
-../../../src/obidmscolumn_int.h
-../../../src/obidmscolumn_int.c
-../../../src/obidmscolumn_qual.h
-../../../src/obidmscolumn_qual.c
-../../../src/obidmscolumn_seq.h
-../../../src/obidmscolumn_seq.c
-../../../src/obidmscolumn_str.h
-../../../src/obidmscolumn_str.c
-../../../src/array_indexer.h
-../../../src/array_indexer.c
-../../../src/char_str_indexer.h
-../../../src/char_str_indexer.c
-../../../src/dna_seq_indexer.h
-../../../src/dna_seq_indexer.c
-../../../src/encode.c
-../../../src/encode.h
-../../../src/uint8_indexer.c
-../../../src/uint8_indexer.h
-../../../src/build_reference_db.c
-../../../src/build_reference_db.h
-../../../src/kmer_similarity.c
-../../../src/kmer_similarity.h
-../../../src/obi_clean.c
-../../../src/obi_clean.h
-../../../src/obi_ecopcr.c
-../../../src/obi_ecopcr.h
-../../../src/obi_ecotag.c
-../../../src/obi_ecotag.h
-../../../src/obidms_taxonomy.c
-../../../src/obidms_taxonomy.h
-../../../src/obilittlebigman.c
-../../../src/obilittlebigman.h
-../../../src/_sse.h
-../../../src/obidebug.h
-../../../src/libecoPCR/libapat/CODES/dft_code.h
-../../../src/libecoPCR/libapat/CODES/dna_code.h
-../../../src/libecoPCR/libapat/CODES/prot_code.h
-../../../src/libecoPCR/libapat/apat_parse.c
-../../../src/libecoPCR/libapat/apat_search.c
-../../../src/libecoPCR/libapat/apat.h
-../../../src/libecoPCR/libapat/Gmach.h
-../../../src/libecoPCR/libapat/Gtypes.h
-../../../src/libecoPCR/libapat/libstki.c
-../../../src/libecoPCR/libapat/libstki.h
-../../../src/libecoPCR/libthermo/nnparams.h
-../../../src/libecoPCR/libthermo/nnparams.c
-../../../src/libecoPCR/ecoapat.c
-../../../src/libecoPCR/ecodna.c
-../../../src/libecoPCR/ecoError.c
-../../../src/libecoPCR/ecoMalloc.c
-../../../src/libecoPCR/ecoPCR.h
-
-
-
-
-
-
-

python/obitools3/apps/progress.pxd

@@ -1,9 +1,10 @@
 #cython: language_level=3
 
+from ..utils cimport str2bytes
+
 cdef extern from "stdio.h":
     struct FILE
     int fprintf(FILE *stream, char *format, ...)
-    int fputs(char *string, FILE *stream)
     FILE* stderr
     ctypedef unsigned int off_t "unsigned long long"
     
@@ -51,9 +52,8 @@ cdef class ProgressBar:
     cdef int lastlog
     cdef bint ontty
     cdef int fd
-    cdef bint cut
     
-    cdef bytes _head
+    cdef bytes head
     cdef char *chead
     
     cdef object logger

python/obitools3/apps/progress.pyx

@@ -6,14 +6,10 @@ Created on 27 mars 2016
 @author: coissac
 '''
 
-
-from ..utils cimport str2bytes, bytes2str
-from .config cimport getConfiguration 
 import sys
-
+from ..utils cimport bytes2str
 
 cdef class ProgressBar:
-    
     cdef clock_t clock(self):
         cdef clock_t t
         cdef timeval tp
@@ -25,14 +21,11 @@ cdef class ProgressBar:
         
         return t
 
-
     def __init__(self,
                  off_t maxi,
-                 dict  config={},
+                 dict  config,
                  str head="",
-                 double seconde=0.1,
-                 cut=False):
-        
+                 double seconde=0.1):
         self.starttime = self.clock()
         self.lasttime  = self.starttime
         self.tickcount = <clock_t> (seconde * CLOCKS_PER_SEC)
@@ -41,18 +34,15 @@ cdef class ProgressBar:
         self.arrow     = 0
         self.lastlog   = 0
         
-        if not config:
-            config=getConfiguration()
-        
         self.ontty = sys.stderr.isatty()
         
         if (maxi<=0):
             maxi=1
             
-        self.maxi  = maxi
-        self.head  = head
-        self.chead = self._head 
-        self.cut   = cut
+        self.maxi = maxi
+        self.head = str2bytes(head)
+        self.chead= self.head 
+        
         
         self.logger=config[config["__root_config__"]]["logger"]
         self.wheel =  '|/-\\'
@@ -66,9 +56,8 @@ cdef class ProgressBar:
                     '##########' \
                     '##########' \
                     '##########'  
-                          
-             
-    def __call__(self, object pos, bint force=False):
+                           
+    def __call__(self,object pos):
         cdef off_t    ipos
         cdef clock_t  elapsed
         cdef clock_t  newtime
@@ -82,7 +71,7 @@ cdef class ProgressBar:
         
         self.cycle+=1
     
-        if self.cycle % self.freq == 0 or force:
+        if self.cycle % self.freq == 0:
             self.cycle=1
             newtime  = self.clock()
             delta         = newtime - self.lasttime
@@ -115,43 +104,35 @@ cdef class ProgressBar:
             if self.ontty:
                 fraction=<int>(percent * 50.)
                 self.arrow=(self.arrow+1) % 4
+                self.diese[fraction]=0
+                self.spaces[50 - fraction]=0
             
                 if days:
-                    <void>fprintf(stderr,b'\r%s %5.1f %% |%.*s%c%.*s] remain : %d days %02d:%02d:%02d\033[K',
+                    <void>fprintf(stderr,b'\r%s %5.1f %% |%s%c%s] remain : %d days %02d:%02d:%02d',
                                     self.chead,
                                     percent*100,
-                                    fraction,self.diese,
-                                    self.wheel[self.arrow],
-                                    50-fraction,self.spaces,
+                                    self.diese,self.wheel[self.arrow],self.spaces,
                                     days,hour,minu,sec)
                 else:
-                    <void>fprintf(stderr,b'\r%s %5.1f %% |%.*s%c%.*s] remain : %02d:%02d:%02d\033[K',
+                    <void>fprintf(stderr,b'\r%s %5.1f %% |%s%c%s] remain : %02d:%02d:%02d',
                                     self.chead,
                                     percent*100.,
-                                    fraction,self.diese,
-                                    self.wheel[self.arrow],
-                                    50-fraction,self.spaces,
+                                    self.diese,self.wheel[self.arrow],self.spaces,
                                     hour,minu,sec)
+                self.diese[fraction]=b'#'
+                self.spaces[50 - fraction]=b' '
 
-            if self.cut:
-                tenth = int(percent * 10)
-                if tenth != self.lastlog:
+            twentyth = int(percent * 20)
+            if twentyth != self.lastlog:
+                
+                if self.ontty:
+                    <void>fprintf(stderr,b'\n')
                     
-                    if self.ontty:
-                        <void>fputs(b'\n',stderr)
-                    
-                    self.logger.info('%s %5.1f %% remain : %02d:%02d:%02d\033[K' % (
-                                            bytes2str(self._head),
-                                            percent*100.,
-                                            hour,minu,sec))
-                    self.lastlog=tenth
+                self.logger.info('%s %5.1f %% remain : %02d:%02d:%02d' % (
+                                        bytes2str(self.head),
+                                        percent*100.,
+                                        hour,minu,sec))
+                self.lastlog=twentyth
         else:
             self.cycle+=1
 
-
-    property head:    
-        def __get__(self):
-            return self._head
-        def __set__(self,str value):
-            self._head=str2bytes(value)
-            self.chead=self._head

python/obitools3/apps/temp.cfiles

@@ -1,110 +0,0 @@
-../../../src/obi_lcs.h
-../../../src/obi_lcs.c
-../../../src/obierrno.h
-../../../src/obierrno.c
-../../../src/upperband.h
-../../../src/upperband.c
-../../../src/sse_banded_LCS_alignment.h
-../../../src/sse_banded_LCS_alignment.c
-../../../src/obiblob.h
-../../../src/obiblob.c
-../../../src/utils.h
-../../../src/utils.c
-../../../src/obidms.h
-../../../src/obidms.c
-../../../src/libjson/json_utils.h
-../../../src/libjson/json_utils.c
-../../../src/libjson/cJSON.h
-../../../src/libjson/cJSON.c
-../../../src/obiavl.h
-../../../src/obiavl.c
-../../../src/bloom.h
-../../../src/bloom.c
-../../../src/crc64.h
-../../../src/crc64.c
-../../../src/murmurhash2.h
-../../../src/murmurhash2.c
-../../../src/obidmscolumn.h
-../../../src/obidmscolumn.c
-../../../src/obitypes.h
-../../../src/obitypes.c
-../../../src/obidmscolumndir.h
-../../../src/obidmscolumndir.c
-../../../src/obiblob_indexer.h
-../../../src/obiblob_indexer.c
-../../../src/obiview.h
-../../../src/obiview.c
-../../../src/hashtable.h
-../../../src/hashtable.c
-../../../src/linked_list.h
-../../../src/linked_list.c
-../../../src/obidmscolumn_array.h
-../../../src/obidmscolumn_array.c
-../../../src/obidmscolumn_blob.h
-../../../src/obidmscolumn_blob.c
-../../../src/obidmscolumn_idx.h
-../../../src/obidmscolumn_idx.c
-../../../src/obidmscolumn_bool.h
-../../../src/obidmscolumn_bool.c
-../../../src/obidmscolumn_char.h
-../../../src/obidmscolumn_char.c
-../../../src/obidmscolumn_float.h
-../../../src/obidmscolumn_float.c
-../../../src/obidmscolumn_int.h
-../../../src/obidmscolumn_int.c
-../../../src/obidmscolumn_qual.h
-../../../src/obidmscolumn_qual.c
-../../../src/obidmscolumn_seq.h
-../../../src/obidmscolumn_seq.c
-../../../src/obidmscolumn_str.h
-../../../src/obidmscolumn_str.c
-../../../src/array_indexer.h
-../../../src/array_indexer.c
-../../../src/char_str_indexer.h
-../../../src/char_str_indexer.c
-../../../src/dna_seq_indexer.h
-../../../src/dna_seq_indexer.c
-../../../src/encode.c
-../../../src/encode.h
-../../../src/uint8_indexer.c
-../../../src/uint8_indexer.h
-../../../src/build_reference_db.c
-../../../src/build_reference_db.h
-../../../src/kmer_similarity.c
-../../../src/kmer_similarity.h
-../../../src/obi_clean.c
-../../../src/obi_clean.h
-../../../src/obi_ecopcr.c
-../../../src/obi_ecopcr.h
-../../../src/obi_ecotag.c
-../../../src/obi_ecotag.h
-../../../src/obidms_taxonomy.c
-../../../src/obidms_taxonomy.h
-../../../src/obilittlebigman.c
-../../../src/obilittlebigman.h
-../../../src/_sse.h
-../../../src/obidebug.h
-../../../src/libecoPCR/libapat/CODES/dft_code.h
-../../../src/libecoPCR/libapat/CODES/dna_code.h
-../../../src/libecoPCR/libapat/CODES/prot_code.h
-../../../src/libecoPCR/libapat/apat_parse.c
-../../../src/libecoPCR/libapat/apat_search.c
-../../../src/libecoPCR/libapat/apat.h
-../../../src/libecoPCR/libapat/Gmach.h
-../../../src/libecoPCR/libapat/Gtypes.h
-../../../src/libecoPCR/libapat/libstki.c
-../../../src/libecoPCR/libapat/libstki.h
-../../../src/libecoPCR/libthermo/nnparams.h
-../../../src/libecoPCR/libthermo/nnparams.c
-../../../src/libecoPCR/ecoapat.c
-../../../src/libecoPCR/ecodna.c
-../../../src/libecoPCR/ecoError.c
-../../../src/libecoPCR/ecoMalloc.c
-../../../src/libecoPCR/ecoPCR.h
-
-
-
-
-
-
-

python/obitools3/apps/temp.pxd

@@ -1,10 +0,0 @@
-#cython: language_level=3
-
-'''
-Created on 28 juillet 2017
-
-@author: coissac
-'''
-
-from obitools3.dms.dms cimport DMS
-from obitools3.utils cimport tobytes,tostr

python/obitools3/apps/temp.pyx

@@ -1,96 +0,0 @@
-#cython: language_level=3
-
-'''
-Created on 28 juillet 2017
-
-@author: coissac
-'''
-
-from os import environb,getpid
-from os.path import join, isdir
-from tempfile import TemporaryDirectory, _get_candidate_names
-from shutil import rmtree
-from atexit import register
-
-from obitools3.dms.dms import DMS
-
-from obitools3.apps.config import getConfiguration
-from obitools3.apps.config import logger
-
-cpdef get_temp_dir():
-    """
-    Returns a temporary directory object specific of this instance of obitools.
-    
-    This is an application function. It cannot be called out of an obi command.
-    It requires a valid configuration.
-    
-    If the function is called several time from the same obi session, the same
-    directory is returned.
-    
-    If the OBITMP environment variable exist, the temporary directory is created
-    inside this directory.
-    
-    The directory is automatically destroyed at the end of the end of the process.
-    
-        @return: a temporary python directory object.
-    """
-    cdef bytes tmpdirname
-    cdef dict config = getConfiguration()
-
-    root = config["__root_config__"]
-    
-    try:
-        return config[root]["tempdir"].name
-    except KeyError:
-        pass
-    
-    try:
-        basedir=environb[b'OBITMP']
-    except KeyError:
-        basedir=None
-    
-    tmp = TemporaryDirectory(dir=basedir)
-        
-    config[root]["tempdir"]=tmp
-    
-    return tmp.name
- 
-cpdef get_temp_dir_name():
-    """
-    Returns the name of the  temporary directory object 
-    specific of this instance of obitools.
-    
-        @return: the name of the temporary directory.
-        
-        @see get_temp_dir
-    """
-    return get_temp_dir_name().name
-          
-               
-cpdef get_temp_dms():
-
-    cdef bytes tmpdirname                   # @DuplicatedSignature
-    cdef dict config = getConfiguration()   # @DuplicatedSignature
-    cdef DMS tmpdms
-    
-    root = config["__root_config__"]
-    
-    try:
-        return config[root]["tempdms"]
-    except KeyError:
-        pass
-
-    tmpdirname=get_temp_dir()
-        
-    tempname = join(tmpdirname,
-                    b"obi.%d.%s" % (getpid(),
-                                    tobytes(next(_get_candidate_names())))
-                   )
-        
-    tmpdms = DMS.new(tempname)
-    
-    config[root]["tempdms"]=tmpdms
-    
-    return tmpdms
-    
-    

python/obitools3/commands/align.cfiles

@@ -1,103 +0,0 @@
-../../../src/obi_lcs.h
-../../../src/obi_lcs.c
-../../../src/obierrno.h
-../../../src/obierrno.c
-../../../src/upperband.h
-../../../src/upperband.c
-../../../src/sse_banded_LCS_alignment.h
-../../../src/sse_banded_LCS_alignment.c
-../../../src/obiblob.h
-../../../src/obiblob.c
-../../../src/utils.h
-../../../src/utils.c
-../../../src/obidms.h
-../../../src/obidms.c
-../../../src/libjson/json_utils.h
-../../../src/libjson/json_utils.c
-../../../src/libjson/cJSON.h
-../../../src/libjson/cJSON.c
-../../../src/obiavl.h
-../../../src/obiavl.c
-../../../src/bloom.h
-../../../src/bloom.c
-../../../src/crc64.h
-../../../src/crc64.c
-../../../src/murmurhash2.h
-../../../src/murmurhash2.c
-../../../src/obidmscolumn.h
-../../../src/obidmscolumn.c
-../../../src/obitypes.h
-../../../src/obitypes.c
-../../../src/obidmscolumndir.h
-../../../src/obidmscolumndir.c
-../../../src/obiblob_indexer.h
-../../../src/obiblob_indexer.c
-../../../src/obiview.h
-../../../src/obiview.c
-../../../src/hashtable.h
-../../../src/hashtable.c
-../../../src/linked_list.h
-../../../src/linked_list.c
-../../../src/obidmscolumn_array.h
-../../../src/obidmscolumn_array.c
-../../../src/obidmscolumn_blob.h
-../../../src/obidmscolumn_blob.c
-../../../src/obidmscolumn_idx.h
-../../../src/obidmscolumn_idx.c
-../../../src/obidmscolumn_bool.h
-../../../src/obidmscolumn_bool.c
-../../../src/obidmscolumn_char.h
-../../../src/obidmscolumn_char.c
-../../../src/obidmscolumn_float.h
-../../../src/obidmscolumn_float.c
-../../../src/obidmscolumn_int.h
-../../../src/obidmscolumn_int.c
-../../../src/obidmscolumn_qual.h
-../../../src/obidmscolumn_qual.c
-../../../src/obidmscolumn_seq.h
-../../../src/obidmscolumn_seq.c
-../../../src/obidmscolumn_str.h
-../../../src/obidmscolumn_str.c
-../../../src/array_indexer.h
-../../../src/array_indexer.c
-../../../src/char_str_indexer.h
-../../../src/char_str_indexer.c
-../../../src/dna_seq_indexer.h
-../../../src/dna_seq_indexer.c
-../../../src/encode.c
-../../../src/encode.h
-../../../src/uint8_indexer.c
-../../../src/uint8_indexer.h
-../../../src/build_reference_db.c
-../../../src/build_reference_db.h
-../../../src/kmer_similarity.c
-../../../src/kmer_similarity.h
-../../../src/obi_clean.c
-../../../src/obi_clean.h
-../../../src/obi_ecopcr.c
-../../../src/obi_ecopcr.h
-../../../src/obi_ecotag.c
-../../../src/obi_ecotag.h
-../../../src/obidms_taxonomy.c
-../../../src/obidms_taxonomy.h
-../../../src/obilittlebigman.c
-../../../src/obilittlebigman.h
-../../../src/_sse.h
-../../../src/obidebug.h
-../../../src/libecoPCR/libapat/CODES/dft_code.h
-../../../src/libecoPCR/libapat/CODES/dna_code.h
-../../../src/libecoPCR/libapat/CODES/prot_code.h
-../../../src/libecoPCR/libapat/apat_parse.c
-../../../src/libecoPCR/libapat/apat_search.c
-../../../src/libecoPCR/libapat/apat.h
-../../../src/libecoPCR/libapat/Gmach.h
-../../../src/libecoPCR/libapat/Gtypes.h
-../../../src/libecoPCR/libapat/libstki.c
-../../../src/libecoPCR/libapat/libstki.h
-../../../src/libecoPCR/libthermo/nnparams.h
-../../../src/libecoPCR/libthermo/nnparams.c
-../../../src/libecoPCR/ecoapat.c
-../../../src/libecoPCR/ecodna.c
-../../../src/libecoPCR/ecoError.c
-../../../src/libecoPCR/ecoMalloc.c
-../../../src/libecoPCR/ecoPCR.h

python/obitools3/commands/align.pxd

@@ -1,18 +0,0 @@
-#cython: language_level=3
-
-
-cpdef align_columns(bytes dms_n, 
-                    bytes input_view_1_n, 
-                    bytes output_view_n,
-                    bytes input_view_2_n=*,
-                    bytes input_column_1_n=*, 
-                    bytes input_column_2_n=*,
-                    bytes input_elt_1_n=*, 
-                    bytes input_elt_2_n=*,
-                    bytes id_column_1_n=*, 
-                    bytes id_column_2_n=*,
-                    double threshold=*, bint normalize=*, 
-                    int reference=*, bint similarity_mode=*,
-                    bint print_seq=*, bint print_count=*,
-                    bytes comments=*,
-                    int thread_count=*)

python/obitools3/commands/align.pyx

@@ -1,274 +0,0 @@
-#cython: language_level=3
-
-from obitools3.apps.progress cimport ProgressBar  # @UnresolvedImport
-from obitools3.dms import DMS
-from obitools3.dms.view.view cimport View
-from obitools3.uri.decode import open_uri
-from obitools3.apps.optiongroups import addMinimalInputOption, addMinimalOutputOption
-from obitools3.dms.view import RollbackException
-from obitools3.apps.config import logger
-from obitools3.utils cimport tobytes, str2bytes
-
-from obitools3.dms.capi.obilcsalign cimport obi_lcs_align_one_column, \
-                                            obi_lcs_align_two_columns
-
-import time
-import sys
-
-
-__title__="Aligns one sequence column with itself or two sequence columns"
-
-
-def addOptions(parser):
-   
-   addMinimalInputOption(parser)
-   addMinimalOutputOption(parser)
-
-   group=parser.add_argument_group('obi align specific options')
-
-   group.add_argument('--input-2', '-I',
-                      action="store", dest="align:inputuri2",
-                      metavar='<INPUT URI>',
-                      default="",
-                      type=str,
-                      help="Eventually, the URI of the second input to align with the first one.")
-      
-   group.add_argument('--threshold','-t',
-                      action="store", dest="align:threshold",
-                      metavar='<THRESHOLD>',
-                      default=0.0,
-                      type=float,
-                      help="Score threshold. If the score is normalized and expressed in similarity (default),"
-                           " it is an identity, e.g. 0.95 for an identity of 95%%. If the score is normalized"
-                           " and expressed in distance, it is (1.0 - identity), e.g. 0.05 for an identity of 95%%."
-                           " If the score is not normalized and expressed in similarity, it is the length of the"
-                           " Longest Common Subsequence. If the score is not normalized and expressed in distance,"
-                           " it is (reference length - LCS length)."
-                           " Only sequence pairs with a similarity above <THRESHOLD> are printed. Default: 0.00"
-                           " (no threshold).")
-   
-   group.add_argument('--longest-length','-L',
-                      action="store_const", dest="align:reflength",
-                      default=0,
-                      const=1,
-                      help="The reference length is the length of the longest sequence."
-                           " Default: the reference length is the length of the alignment.")
-   
-   group.add_argument('--shortest-length','-l',
-                      action="store_const", dest="align:reflength",
-                      default=0,
-                      const=2,
-                      help="The reference length is the length of the shortest sequence."
-                           " Default: the reference length is the length of the alignment.")
-   
-   group.add_argument('--raw','-r',
-                      action="store_false", dest="align:normalize",
-                      default=True,
-                      help="Raw score, not normalized. Default: score is normalized with the reference sequence length.")
-   
-   group.add_argument('--distance','-D',
-                      action="store_false", dest="align:similarity",
-                      default=True,
-                      help="Score is expressed in distance. Default: score is expressed in similarity.")
-   
-   group.add_argument('--print-seq','-s',
-                      action="store_true", dest="align:printseq",
-                      default=False,
-                      help="The nucleotide sequences are written in the output view. Default: they are not written.")
-   
-   group.add_argument('--print-count','-n',
-                      action="store_true", dest="align:printcount",
-                      default=False,
-                      help="Sequence counts are written in the output view. Default: they are not written.")
-   
-   group.add_argument('--thread-count','-p',   # TODO should probably be in a specific option group
-                      action="store", dest="align:threadcount",
-                      metavar='<THREAD COUNT>',
-                      default=1,
-                      type=int,
-                      help="Number of threads to use for the computation. Default: one.")
-
-
-cpdef align_columns(bytes dms_n, 
-                    bytes input_view_1_n, 
-                    bytes output_view_n,
-                    bytes input_view_2_n=b"",
-                    bytes input_column_1_n=b"", 
-                    bytes input_column_2_n=b"",
-                    bytes input_elt_1_n=b"", 
-                    bytes input_elt_2_n=b"",
-                    bytes id_column_1_n=b"", 
-                    bytes id_column_2_n=b"",
-                    double threshold=0.0, bint normalize=True, 
-                    int reference=0, bint similarity_mode=True,
-                    bint print_seq=False, bint print_count=False,
-                    bytes comments=b"{}",
-                    int thread_count=1) : 
-
-    if input_view_2_n == b"" and input_column_2_n == b"" :
-        if obi_lcs_align_one_column(dms_n, \
-                                    input_view_1_n, \
-                                    input_column_1_n, \
-                                    input_elt_1_n, \
-                                    id_column_1_n, \
-                                    output_view_n, \
-                                    comments, \
-                                    print_seq, \
-                                    print_count, \
-                                    threshold, normalize, reference, similarity_mode,
-                                    thread_count) < 0 :
-            raise Exception("Error aligning sequences")        
-            
-    else:
-        if obi_lcs_align_two_columns(dms_n, \
-                                     input_view_1_n, \
-                                     input_view_2_n, \
-                                     input_column_1_n, \
-                                     input_column_2_n, \
-                                     input_elt_1_n, \
-                                     input_elt_2_n, \
-                                     id_column_1_n, \
-                                     id_column_2_n, \
-                                     output_view_n, \
-                                     comments, \
-                                     print_seq, \
-                                     print_count, \
-                                     threshold, normalize, reference, similarity_mode) < 0 :
-            raise Exception("Error aligning sequences")        
- 
- 
-def run(config):
-    
-    DMS.obi_atexit()
-    
-    logger("info", "obi align")
-
-    # Open the input: only the DMS
-    input = open_uri(config['obi']['inputURI'],
-                     dms_only=True)
-    if input is None:
-        raise Exception("Could not read input")
-    i_dms = input[0]
-    i_dms_name = input[0].name
-    i_uri = input[1]
-    i_view_name = i_uri.split(b"/")[0]
-    i_column_name = b""
-    i_element_name = b""
-    if len(i_uri.split(b"/")) == 2:
-        i_column_name = i_uri.split(b"/")[1]
-    if len(i_uri.split(b"/")) == 3:
-        i_element_name = i_uri.split(b"/")[2]
-    if len(i_uri.split(b"/")) > 3:
-        raise Exception("Input URI contains too many elements:", config['obi']['inputURI'])
-
-    # Open the second input if there is one
-    i_dms_2 = None
-    i_dms_name_2 = b""
-    original_i_view_name_2 = b""
-    i_view_name_2 = b""
-    i_column_name_2 = b""
-    i_element_name_2 = b""
-    if config['align']['inputuri2']:
-        input_2 = open_uri(config['align']['inputuri2'],
-                           dms_only=True)
-        if input_2 is None:
-            raise Exception("Could not read second input")
-        i_dms_2 = input_2[0]
-        i_dms_name_2 = i_dms_2.name
-        i_uri_2 = input_2[1]
-        original_i_view_name_2 = i_uri_2.split(b"/")[0]
-        if len(i_uri_2.split(b"/")) == 2:
-            i_column_name_2 = i_uri_2.split(b"/")[1]
-        if len(i_uri_2.split(b"/")) == 3:
-            i_element_name_2 = i_uri_2.split(b"/")[2]
-        if len(i_uri_2.split(b"/")) > 3:
-            raise Exception("Input URI contains too many elements:", config['align']['inputuri2'])
-
-        # If the 2 input DMS are not the same, temporarily import 2nd input view in first input DMS
-        if i_dms != i_dms_2:
-            temp_i_view_name_2 = original_i_view_name_2
-            i=0
-            while temp_i_view_name_2 in i_dms:  # Making sure view name is unique in input DMS
-                temp_i_view_name_2 = original_i_view_name_2+b"_"+str2bytes(str(i))
-                i+=1
-            i_view_name_2 = temp_i_view_name_2
-            View.import_view(i_dms_2.full_path[:-7], i_dms.full_path[:-7], original_i_view_name_2, i_view_name_2)
-
-    # Open the output: only the DMS
-    output = open_uri(config['obi']['outputURI'],
-                      input=False,
-                      dms_only=True)
-    if output is None:
-        raise Exception("Could not create output")
-    o_dms = output[0]
-    o_dms_name = o_dms.name 
-    final_o_view_name = output[1]
-
-    # If the input and output DMS are not the same, align creating a temporary view in the input dms that will be exported to 
-    # the right DMS and deleted in the other afterwards.
-    if i_dms != o_dms:
-        temporary_view_name = final_o_view_name
-        i=0
-        while temporary_view_name in i_dms:  # Making sure view name is unique in input DMS
-            temporary_view_name = final_o_view_name+b"_"+str2bytes(str(i))
-            i+=1
-        o_view_name = temporary_view_name
-    else:
-        o_view_name = final_o_view_name
-        
-    # Save command config in View comments
-    command_line = " ".join(sys.argv[1:])
-    
-    i_dms_list = [i_dms_name]
-    if i_dms_name_2:
-        i_dms_list.append(i_dms_name_2)
-    i_view_list = [i_view_name]
-    if original_i_view_name_2:
-        i_view_list.append(original_i_view_name_2)
-    comments = View.print_config(config, "align", command_line, input_dms_name=i_dms_list, input_view_name=i_view_list)
-    
-    # Call cython alignment function
-      # Using default ID columns of the view. TODO discuss adding option
-    align_columns(i_dms_name,  \
-                  i_view_name,  \
-                  o_view_name,  \
-                  input_view_2_n   = i_view_name_2,  \
-                  input_column_1_n = i_column_name,  \
-                  input_column_2_n = i_column_name_2, \
-                  input_elt_1_n    = i_element_name,  \
-                  input_elt_2_n    = i_element_name_2, \
-                  id_column_1_n    = b"",  \
-                  id_column_2_n    = b"", \
-                  threshold        = config['align']['threshold'], \
-                  normalize        = config['align']['normalize'],  \
-                  reference        = config['align']['reflength'],  \
-                  similarity_mode  = config['align']['similarity'],  \
-                  print_seq        = config['align']['printseq'],  \
-                  print_count      = config['align']['printcount'], \
-                  comments         = comments, \
-                  thread_count     = config['align']['threadcount'])
-
-    # If the input and output DMS are not the same, export result view to output DMS
-    if i_dms != o_dms:
-        View.import_view(i_dms.full_path[:-7], o_dms.full_path[:-7], o_view_name, final_o_view_name)
-
-    # Save command config in output DMS comments
-    o_dms.record_command_line(command_line)
-
-    #print("\n\nOutput view:\n````````````", file=sys.stderr)
-    #print(repr(o_dms[final_o_view_name]), file=sys.stderr)
-
-    # If the two input DMS are different, delete the temporary input view in the first input DMS
-    if i_dms_2 and i_dms != i_dms_2:
-        View.delete_view(i_dms, i_view_name_2)
-        i_dms_2.close()
-
-    # If the input and the output DMS are different, delete the temporary result view in the input DMS
-    if i_dms != o_dms:
-        View.delete_view(i_dms, o_view_name)
-        o_dms.close()
-    
-    i_dms.close()
-    
-    logger("info", "Done.")
-  

python/obitools3/commands/alignpairedend.cfiles

@@ -1,103 +0,0 @@
-../../../src/obi_lcs.h
-../../../src/obi_lcs.c
-../../../src/obierrno.h
-../../../src/obierrno.c
-../../../src/upperband.h
-../../../src/upperband.c
-../../../src/sse_banded_LCS_alignment.h
-../../../src/sse_banded_LCS_alignment.c
-../../../src/obiblob.h
-../../../src/obiblob.c
-../../../src/utils.h
-../../../src/utils.c
-../../../src/obidms.h
-../../../src/obidms.c
-../../../src/libjson/json_utils.h
-../../../src/libjson/json_utils.c
-../../../src/libjson/cJSON.h
-../../../src/libjson/cJSON.c
-../../../src/obiavl.h
-../../../src/obiavl.c
-../../../src/bloom.h
-../../../src/bloom.c
-../../../src/crc64.h
-../../../src/crc64.c
-../../../src/murmurhash2.h
-../../../src/murmurhash2.c
-../../../src/obidmscolumn.h
-../../../src/obidmscolumn.c
-../../../src/obitypes.h
-../../../src/obitypes.c
-../../../src/obidmscolumndir.h
-../../../src/obidmscolumndir.c
-../../../src/obiblob_indexer.h
-../../../src/obiblob_indexer.c
-../../../src/obiview.h
-../../../src/obiview.c
-../../../src/hashtable.h
-../../../src/hashtable.c
-../../../src/linked_list.h
-../../../src/linked_list.c
-../../../src/obidmscolumn_array.h
-../../../src/obidmscolumn_array.c
-../../../src/obidmscolumn_blob.h
-../../../src/obidmscolumn_blob.c
-../../../src/obidmscolumn_idx.h
-../../../src/obidmscolumn_idx.c
-../../../src/obidmscolumn_bool.h
-../../../src/obidmscolumn_bool.c
-../../../src/obidmscolumn_char.h
-../../../src/obidmscolumn_char.c
-../../../src/obidmscolumn_float.h
-../../../src/obidmscolumn_float.c
-../../../src/obidmscolumn_int.h
-../../../src/obidmscolumn_int.c
-../../../src/obidmscolumn_qual.h
-../../../src/obidmscolumn_qual.c
-../../../src/obidmscolumn_seq.h
-../../../src/obidmscolumn_seq.c
-../../../src/obidmscolumn_str.h
-../../../src/obidmscolumn_str.c
-../../../src/array_indexer.h
-../../../src/array_indexer.c
-../../../src/char_str_indexer.h
-../../../src/char_str_indexer.c
-../../../src/dna_seq_indexer.h
-../../../src/dna_seq_indexer.c
-../../../src/encode.c
-../../../src/encode.h
-../../../src/uint8_indexer.c
-../../../src/uint8_indexer.h
-../../../src/build_reference_db.c
-../../../src/build_reference_db.h
-../../../src/kmer_similarity.c
-../../../src/kmer_similarity.h
-../../../src/obi_clean.c
-../../../src/obi_clean.h
-../../../src/obi_ecopcr.c
-../../../src/obi_ecopcr.h
-../../../src/obi_ecotag.c
-../../../src/obi_ecotag.h
-../../../src/obidms_taxonomy.c
-../../../src/obidms_taxonomy.h
-../../../src/obilittlebigman.c
-../../../src/obilittlebigman.h
-../../../src/_sse.h
-../../../src/obidebug.h
-../../../src/libecoPCR/libapat/CODES/dft_code.h
-../../../src/libecoPCR/libapat/CODES/dna_code.h
-../../../src/libecoPCR/libapat/CODES/prot_code.h
-../../../src/libecoPCR/libapat/apat_parse.c
-../../../src/libecoPCR/libapat/apat_search.c
-../../../src/libecoPCR/libapat/apat.h
-../../../src/libecoPCR/libapat/Gmach.h
-../../../src/libecoPCR/libapat/Gtypes.h
-../../../src/libecoPCR/libapat/libstki.c
-../../../src/libecoPCR/libapat/libstki.h
-../../../src/libecoPCR/libthermo/nnparams.h
-../../../src/libecoPCR/libthermo/nnparams.c
-../../../src/libecoPCR/ecoapat.c
-../../../src/libecoPCR/ecodna.c
-../../../src/libecoPCR/ecoError.c
-../../../src/libecoPCR/ecoMalloc.c
-../../../src/libecoPCR/ecoPCR.h

python/obitools3/commands/alignpairedend.pxd

@@ -1,4 +0,0 @@
-#cython: language_level=3
-
-
-cdef object buildAlignment(object direct, object reverse)

python/obitools3/commands/alignpairedend.pyx

@@ -1,249 +0,0 @@
-#cython: language_level=3
-
-from obitools3.apps.progress cimport ProgressBar  # @UnresolvedImport
-from obitools3.dms import DMS
-from obitools3.dms.view import RollbackException
-from obitools3.dms.view.typed_view.view_NUC_SEQS cimport View_NUC_SEQS
-from obitools3.dms.column.column cimport Column
-from obitools3.dms.capi.obiview cimport QUALITY_COLUMN
-from obitools3.dms.capi.obitypes cimport OBI_QUAL
-from obitools3.apps.optiongroups import addMinimalInputOption, addMinimalOutputOption
-from obitools3.uri.decode import open_uri
-from obitools3.apps.config import logger
-from obitools3.libalign._qsassemble import QSolexaReverseAssemble
-from obitools3.libalign._qsrassemble import QSolexaRightReverseAssemble
-from obitools3.libalign._solexapairend import buildConsensus, buildJoinedSequence
-from obitools3.dms.obiseq cimport Nuc_Seq
-from obitools3.libalign.shifted_ali cimport Kmer_similarity, Ali_shifted
-from obitools3.commands.ngsfilter import REVERSE_SEQ_COLUMN_NAME, REVERSE_QUALITY_COLUMN_NAME
-
-import sys
-import os
-
-
-__title__="Aligns paired-ended reads"
-
-
-
-def addOptions(parser):
-
-    addMinimalInputOption(parser)
-    addMinimalOutputOption(parser)
- 
-    group = parser.add_argument_group('obi alignpairedend specific options')
-
-    group.add_argument('-R', '--reverse-reads',
-                     action="store", dest="alignpairedend:reverse",
-                     metavar="<URI>",
-                     default=None,
-                     type=str,
-                     help="URI to the reverse reads if they are in a different view than the forward reads")
-
-    group.add_argument('--score-min',
-                     action="store", dest="alignpairedend:smin",
-                     metavar="#.###",
-                     default=None,
-                     type=float,
-                     help="Minimum score for keeping alignments")
-
-    group.add_argument('-A', '--true-ali',
-                       action="store_true", dest="alignpairedend:trueali",
-                       default=False,
-                       help="Performs gap free end alignment of sequences instead of using kmers to compute alignments (slower).")
-
-    group.add_argument('-k', '--kmer-size',
-                       action="store", dest="alignpairedend:kmersize",
-                       metavar="#",
-                       default=3,
-                       type=int,
-                       help="K-mer size for kmer comparisons, between 1 and 4 (not when using -A option; default: 3)")
-
-
-la = QSolexaReverseAssemble()
-ra = QSolexaRightReverseAssemble()
-cdef object buildAlignment(object direct, object reverse):
-    
-    if len(direct)==0 or len(reverse)==0:
-        return None
-    
-    la.seqA = direct
-    la.seqB = reverse
-    
-    ali=la()
-    ali.direction='left'
-
-    ra.seqA = direct
-    ra.seqB = reverse
-
-    rali=ra()
-    rali.direction='right'
-     
-    if ali.score < rali.score:
-        ali = rali
-    
-    return ali
-    
-
-def alignmentIterator(entries, aligner): 
-    
-    if type(entries) == list:
-        two_views = True
-        forward = entries[0]
-        reverse = entries[1]
-        entries_len = len(forward)
-    else:
-        two_views = False
-        entries_len = len(entries)
-    
-    for i in range(entries_len):
-        if two_views:
-            seqF = forward[i]
-            seqR = reverse[i]
-        else:
-            seqF = Nuc_Seq.new_from_stored(entries[i])
-            seqR = Nuc_Seq(seqF.id, seqF[REVERSE_SEQ_COLUMN_NAME], quality=seqF[REVERSE_QUALITY_COLUMN_NAME])
-            seqR.index = i
-        
-        ali = aligner(seqF, seqR)
-        
-        if ali is None:
-            continue
-        
-        yield ali
-
-
-def run(config):
-        
-    DMS.obi_atexit()
-    
-    logger("info", "obi alignpairedend")
-       
-    # Open the input
-    
-    two_views = False
-    forward = None
-    reverse = None
-    input = None
-    
-    input = open_uri(config['obi']['inputURI'])
-    if input is None:
-        raise Exception("Could not open input reads")
-    if input[2] != View_NUC_SEQS:
-        raise NotImplementedError('obi alignpairedend only works on NUC_SEQS views')    
-    
-    if "reverse" in config["alignpairedend"]:
-        
-        two_views = True
-        
-        forward = input[1]        
-        
-        rinput = open_uri(config["alignpairedend"]["reverse"])
-        if rinput is None:
-            raise Exception("Could not open reverse reads")
-        if rinput[2] != View_NUC_SEQS:
-            raise NotImplementedError('obi alignpairedend only works on NUC_SEQS views')
-        
-        reverse = rinput[1]
-    
-        if len(forward) != len(reverse):
-            raise Exception("Error: the number of forward and reverse reads are different")
-        
-        entries = [forward, reverse]
-        input_dms_name = [forward.dms.name, reverse.dms.name]
-        input_view_name = [forward.name, reverse.name]
-    
-    else:
-        entries = input[1]
-        input_dms_name = [entries.dms.name]
-        input_view_name = [entries.name]
-
-    if two_views:
-        entries_len = len(forward)
-    else:
-        entries_len = len(entries)
-    
-    # Open the output
-    output = open_uri(config['obi']['outputURI'],
-                      input=False,
-                      newviewtype=View_NUC_SEQS)
-    if output is None:
-        raise Exception("Could not create output view")
-
-    view = output[1]
-    
-    Column.new_column(view, QUALITY_COLUMN, OBI_QUAL)   #TODO output URI quality option?
-
-    if 'smin' in config['alignpairedend']:
-        smin = config['alignpairedend']['smin']
-    else:
-        smin = 0
-
-    # Initialize the progress bar
-    pb = ProgressBar(entries_len, config, seconde=5)
-    
-    if config['alignpairedend']['trueali']:
-        kmer_ali = False
-        aligner = buildAlignment
-    else :
-        kmer_ali = True
-        if type(entries) == list:
-            forward = entries[0]
-            reverse = entries[1]
-            aligner = Kmer_similarity(forward, view2=reverse, kmer_size=config['alignpairedend']['kmersize'])
-        else:
-            aligner = Kmer_similarity(entries, column2=entries[REVERSE_SEQ_COLUMN_NAME], qual_column2=entries[REVERSE_QUALITY_COLUMN_NAME], kmer_size=config['alignpairedend']['kmersize'])
-        
-    ba = alignmentIterator(entries, aligner)
-
-    i = 0
-    for ali in ba:
-        
-        pb(i)
-        
-        consensus = view[i]
-        
-        if not two_views:
-            seqF = entries[i]
-        else:
-            seqF = forward[i]
-                
-        if smin > 0:
-            if (ali.score > smin) :
-                buildConsensus(ali, consensus, seqF)
-            else:
-                if not two_views:
-                    seqR = Nuc_Seq(seqF.id, seqF[REVERSE_SEQ_COLUMN_NAME], quality = seqF[REVERSE_QUALITY_COLUMN_NAME])
-                else:
-                    seqR = reverse[i]
-                buildJoinedSequence(ali, seqR, consensus, forward=seqF)
-                
-            consensus[b"smin"] = smin
-        else:
-            buildConsensus(ali, consensus, seqF)
-                
-        if kmer_ali :
-            ali.free()
-    
-        i+=1
-
-    pb(i, force=True)
-    print("", file=sys.stderr)
-
-    if kmer_ali :
-        aligner.free()
-
-    # Save command config in View and DMS comments
-    command_line = " ".join(sys.argv[1:])
-    view.write_config(config, "alignpairedend", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name)
-    output[0].record_command_line(command_line)
-
-    #print("\n\nOutput view:\n````````````", file=sys.stderr)
-    #print(repr(view), file=sys.stderr)
-    
-    input[0].close()
-    if two_views:
-        rinput[0].close()
-    output[0].close()
-
-    logger("info", "Done.")
-    

python/obitools3/commands/annotate.cfiles

@@ -1,103 +0,0 @@
-../../../src/obi_lcs.h
-../../../src/obi_lcs.c
-../../../src/obierrno.h
-../../../src/obierrno.c
-../../../src/upperband.h
-../../../src/upperband.c
-../../../src/sse_banded_LCS_alignment.h
-../../../src/sse_banded_LCS_alignment.c
-../../../src/obiblob.h
-../../../src/obiblob.c
-../../../src/utils.h
-../../../src/utils.c
-../../../src/obidms.h
-../../../src/obidms.c
-../../../src/libjson/json_utils.h
-../../../src/libjson/json_utils.c
-../../../src/libjson/cJSON.h
-../../../src/libjson/cJSON.c
-../../../src/obiavl.h
-../../../src/obiavl.c
-../../../src/bloom.h
-../../../src/bloom.c
-../../../src/crc64.h
-../../../src/crc64.c
-../../../src/murmurhash2.h
-../../../src/murmurhash2.c
-../../../src/obidmscolumn.h
-../../../src/obidmscolumn.c
-../../../src/obitypes.h
-../../../src/obitypes.c
-../../../src/obidmscolumndir.h
-../../../src/obidmscolumndir.c
-../../../src/obiblob_indexer.h
-../../../src/obiblob_indexer.c
-../../../src/obiview.h
-../../../src/obiview.c
-../../../src/hashtable.h
-../../../src/hashtable.c
-../../../src/linked_list.h
-../../../src/linked_list.c
-../../../src/obidmscolumn_array.h
-../../../src/obidmscolumn_array.c
-../../../src/obidmscolumn_blob.h
-../../../src/obidmscolumn_blob.c
-../../../src/obidmscolumn_idx.h
-../../../src/obidmscolumn_idx.c
-../../../src/obidmscolumn_bool.h
-../../../src/obidmscolumn_bool.c
-../../../src/obidmscolumn_char.h
-../../../src/obidmscolumn_char.c
-../../../src/obidmscolumn_float.h
-../../../src/obidmscolumn_float.c
-../../../src/obidmscolumn_int.h
-../../../src/obidmscolumn_int.c
-../../../src/obidmscolumn_qual.h
-../../../src/obidmscolumn_qual.c
-../../../src/obidmscolumn_seq.h
-../../../src/obidmscolumn_seq.c
-../../../src/obidmscolumn_str.h
-../../../src/obidmscolumn_str.c
-../../../src/array_indexer.h
-../../../src/array_indexer.c
-../../../src/char_str_indexer.h
-../../../src/char_str_indexer.c
-../../../src/dna_seq_indexer.h
-../../../src/dna_seq_indexer.c
-../../../src/encode.c
-../../../src/encode.h
-../../../src/uint8_indexer.c
-../../../src/uint8_indexer.h
-../../../src/build_reference_db.c
-../../../src/build_reference_db.h
-../../../src/kmer_similarity.c
-../../../src/kmer_similarity.h
-../../../src/obi_clean.c
-../../../src/obi_clean.h
-../../../src/obi_ecopcr.c
-../../../src/obi_ecopcr.h
-../../../src/obi_ecotag.c
-../../../src/obi_ecotag.h
-../../../src/obidms_taxonomy.c
-../../../src/obidms_taxonomy.h
-../../../src/obilittlebigman.c
-../../../src/obilittlebigman.h
-../../../src/_sse.h
-../../../src/obidebug.h
-../../../src/libecoPCR/libapat/CODES/dft_code.h
-../../../src/libecoPCR/libapat/CODES/dna_code.h
-../../../src/libecoPCR/libapat/CODES/prot_code.h
-../../../src/libecoPCR/libapat/apat_parse.c
-../../../src/libecoPCR/libapat/apat_search.c
-../../../src/libecoPCR/libapat/apat.h
-../../../src/libecoPCR/libapat/Gmach.h
-../../../src/libecoPCR/libapat/Gtypes.h
-../../../src/libecoPCR/libapat/libstki.c
-../../../src/libecoPCR/libapat/libstki.h
-../../../src/libecoPCR/libthermo/nnparams.h
-../../../src/libecoPCR/libthermo/nnparams.c
-../../../src/libecoPCR/ecoapat.c
-../../../src/libecoPCR/ecodna.c
-../../../src/libecoPCR/ecoError.c
-../../../src/libecoPCR/ecoMalloc.c
-../../../src/libecoPCR/ecoPCR.h

python/obitools3/commands/annotate.pyx

@@ -1,382 +0,0 @@
-#cython: language_level=3
-
-from obitools3.apps.progress cimport ProgressBar  # @UnresolvedImport
-from obitools3.dms import DMS
-from obitools3.dms.view.view cimport View, Line_selection
-from obitools3.uri.decode import open_uri
-from obitools3.apps.optiongroups import addMinimalInputOption, addTaxonomyOption, addMinimalOutputOption
-from obitools3.dms.view import RollbackException
-from functools import reduce
-from obitools3.apps.config import logger
-from obitools3.utils cimport tobytes, str2bytes
-from obitools3.dms.capi.obiview cimport NUC_SEQUENCE_COLUMN, \
-                                        ID_COLUMN, \
-                                        DEFINITION_COLUMN, \
-                                        QUALITY_COLUMN, \
-                                        COUNT_COLUMN
-
-import time
-import math 
-import sys
-
- 
-__title__="Annotate views with new tags and edit existing annotations"
-
-
-SPECIAL_COLUMNS = [NUC_SEQUENCE_COLUMN, ID_COLUMN, DEFINITION_COLUMN, QUALITY_COLUMN]
-
- 
-def addOptions(parser):
-    
-    addMinimalInputOption(parser)
-    addTaxonomyOption(parser)
-    addMinimalOutputOption(parser)
-
-    group=parser.add_argument_group('obi annotate specific options')
- 
-    group.add_argument('--seq-rank',   # TODO seq/elt/line???
-                       action="store_true", 
-                       dest="annotate:add_rank",
-                       default=False,
-                       help="Add a rank attribute to the sequence "
-                            "indicating the sequence position in the data.")
- 
-    group.add_argument('-R', '--rename-tag',
-                       action="append", 
-                       dest="annotate:rename_tags",
-                       metavar="<OLD_NAME:NEW_NAME>",
-                       type=str,
-                       default=[],
-                       help="Change tag name from OLD_NAME to NEW_NAME.")
-
-    group.add_argument('-D', '--delete-tag',
-                       action="append", 
-                       dest="annotate:delete_tags",
-                       metavar="<TAG_NAME>",
-                       type=str,
-                       default=[],
-                       help="Delete tag TAG_NAME.")
-
-    group.add_argument('-S', '--set-tag',
-                       action="append", 
-                       dest="annotate:set_tags",
-                       metavar="<TAG_NAME:PYTHON_EXPRESSION>",
-                       type=str,
-                       default=[],
-                       help="Add a new tag named TAG_NAME with "
-                            "a value computed from PYTHON_EXPRESSION.")
-
-    group.add_argument('--set-identifier',
-                       action="store", 
-                       dest="annotate:set_identifier",
-                       metavar="<PYTHON_EXPRESSION>",
-                       type=str,
-                       default=None,
-                       help="Set sequence identifier with "
-                            "a value computed from PYTHON_EXPRESSION.")
-    
-    group.add_argument('--set-sequence',
-                       action="store", 
-                       dest="annotate:set_sequence",
-                       metavar="<PYTHON_EXPRESSION>",
-                       type=str,
-                       default=None,
-                       help="Change the sequence itself with "
-                            "a value computed from PYTHON_EXPRESSION.")
-
-    group.add_argument('--set-definition',
-                       action="store", 
-                       dest="annotate:set_definition",
-                       metavar="<PYTHON_EXPRESSION>",
-                       type=str,
-                       default=None,
-                       help="Set sequence definition with "
-                            "a value computed from PYTHON_EXPRESSION.")
-
-    group.add_argument('--run',
-                       action="store", 
-                       dest="annotate:run",
-                       metavar="<PYTHON_EXPRESSION>",
-                       type=str,
-                       default=None,
-                       help="Run a python expression on each element.")
-
-    group.add_argument('-C', '--clear',
-                       action="store_true", 
-                       dest="annotate:clear",
-                       default=False,
-                       help="Clear all tags except the obligatory ones.")
-    
-    group.add_argument('-k','--keep',
-                       action='append',
-                       dest="annotate:keep",
-                       metavar="<TAG>",
-                       default=[],
-                       type=str,
-                       help="Only keep this tag. (Can be specified several times.)")
-    
-    group.add_argument('--length',
-                       action="store_true", 
-                       dest="annotate:length",
-                       default=False,
-                       help="Add 'seq_length' tag with sequence length.")
-    
-    group.add_argument('--with-taxon-at-rank',
-                       action='append',
-                       dest="annotate:taxon_at_rank",
-                       metavar="<RANK_NAME>",
-                       default=[],
-                       type=str,
-                       help="Add taxonomy annotation at the specified rank level RANK_NAME.")
-
-
-def sequenceTaggerGenerator(config, taxo=None):
-    
-    toSet=None
-    newId=None
-    newDef=None
-    newSeq=None
-    length=None
-    add_rank=None
-    run=None
-    
-    if 'set_tags' in config['annotate']:   # TODO default option problem, to fix
-        toSet = [x.split(':',1) for x in config['annotate']['set_tags'] if len(x.split(':',1))==2]
-    if 'set_identifier' in config['annotate']:
-        newId = config['annotate']['set_identifier']
-    if 'set_definition' in config['annotate']:
-        newDef = config['annotate']['set_definition']
-    if 'set_sequence' in config['annotate']:
-        newSeq = config['annotate']['set_sequence']
-    if 'length' in config['annotate']:
-        length = config['annotate']['length']
-    if 'add_rank' in config["annotate"]:
-        add_rank = config["annotate"]["add_rank"]
-    if 'run' in config['annotate']:
-        run = config['annotate']['run']
-    counter = [0]
-    
-    for i in range(len(toSet)):
-        for j in range(len(toSet[i])):
-            toSet[i][j] = tobytes(toSet[i][j])
-
-    annoteRank=[]
-    if config['annotate']['taxon_at_rank']:
-        if taxo is not None:
-            annoteRank = config['annotate']['taxon_at_rank']
-        else:
-            raise Exception("A taxonomy must be provided to annotate taxon ranks")
-    
-    def sequenceTagger(seq):
-                
-        if counter[0]>=0:
-            counter[0]+=1
-                            
-        for rank in annoteRank:
-            if 'taxid' in seq:
-                taxid = seq['taxid']
-                if taxid is not None:
-                    rtaxid = taxo.get_taxon_at_rank(taxid, rank)
-                    if rtaxid is not None:
-                        scn = taxo.get_scientific_name(rtaxid)
-                    else:
-                        scn=None
-                    seq[rank]=rtaxid
-                    seq["%s_name"%rank]=scn
-             
-        if add_rank:
-            seq['seq_rank']=counter[0]
-    
-        for i,v in toSet:
-            #try:
-            if taxo is not None:
-                environ = {'taxonomy' : taxo, 'sequence':seq, 'counter':counter[0], 'math':math}
-            else:
-                environ = {'sequence':seq, 'counter':counter[0], 'math':math}
-            val = eval(v, environ, seq)
-            #except Exception,e:       # TODO discuss usefulness of this
-            #    if options.onlyValid:
-            #        raise e
-            #    val = v
-            seq[i]=val
-
-        if length:
-            seq['seq_length']=len(seq)
-
-        if newId is not None:
-#            try:
-            if taxo is not None:
-                environ = {'taxonomy' : taxo, 'sequence':seq, 'counter':counter[0], 'math':math}
-            else:
-                environ = {'sequence':seq, 'counter':counter[0], 'math':math}     
-            val = eval(newId, environ, seq)
-#            except Exception,e:
-#                if options.onlyValid:
-#                    raise e
-#                val = newId
-            seq.id=val
-
-        if newDef is not None:
-#            try:
-            if taxo is not None:
-                environ = {'taxonomy' : taxo, 'sequence':seq, 'counter':counter[0], 'math':math}
-            else:
-                environ = {'sequence':seq, 'counter':counter[0], 'math':math}     
-            val = eval(newDef, environ, seq)
-#            except Exception,e:
-#                if options.onlyValid:
-#                    raise e
-#                val = newDef
-            seq.definition=val
-#             
-        if newSeq is not None:
-#            try:
-            if taxo is not None:
-                environ = {'taxonomy' : taxo, 'sequence':seq, 'counter':counter[0], 'math':math}
-            else:
-                environ = {'sequence':seq, 'counter':counter[0], 'math':math}     
-            val = eval(newSeq, environ, seq)
-#            except Exception,e:
-#                if options.onlyValid:
-#                    raise e
-#                val = newSeq
-            seq.seq=val
-            if 'seq_length' in seq:
-                seq['seq_length']=len(seq)
-            # Delete quality since it must match the sequence.
-            # TODO discuss deleting for each sequence separately
-            if QUALITY_COLUMN in seq:
-                seq.view.delete_column(QUALITY_COLUMN)
-                    
-        if run is not None:
-#            try:
-            if taxo is not None:
-                environ = {'taxonomy' : taxo, 'sequence':seq, 'counter':counter[0], 'math':math}
-            else:
-                environ = {'sequence':seq, 'counter':counter[0], 'math':math}     
-            eval(run, environ, seq)
-#            except Exception,e:
-#                if options.onlyValid:
-#                    raise e
-
-    return sequenceTagger
-
-
-def run(config):
-     
-    DMS.obi_atexit()
-    
-    logger("info", "obi annotate")
-
-    # Open the input
-    input = open_uri(config['obi']['inputURI'])
-    if input is None:
-        raise Exception("Could not read input view")
-    i_dms = input[0]
-    i_view = input[1]
-    i_view_name = input[1].name
-
-    # Open the output: only the DMS, as the output view is going to be created by cloning the input view
-    # (could eventually be done via an open_uri() argument)
-    output = open_uri(config['obi']['outputURI'],
-                      input=False,
-                      dms_only=True)
-    if output is None:
-        raise Exception("Could not create output view")
-    o_dms = output[0]
-    o_view_name = output[1]
-    
-    # If the input and output DMS are not the same, import the input view in the output DMS before cloning it to modify it
-    # (could be the other way around: clone and modify in the input DMS then import the new view in the output DMS)
-    if i_dms != o_dms:
-        imported_view_name = i_view_name
-        i=0
-        while imported_view_name in o_dms:  # Making sure view name is unique in output DMS
-            imported_view_name = i_view_name+b"_"+str2bytes(str(i))
-            i+=1
-        View.import_view(i_dms.full_path[:-7], o_dms.full_path[:-7], i_view_name, imported_view_name)
-        i_view = o_dms[imported_view_name]
-        
-    # Clone output view from input view
-    o_view = i_view.clone(o_view_name)
-    if o_view is None:
-        raise Exception("Couldn't create output view")
-    i_view.close()
-
-    # Open taxonomy if there is one
-    if 'taxoURI' in config['obi'] and config['obi']['taxoURI'] is not None:
-        taxo_uri = open_uri(config['obi']['taxoURI'])
-        if taxo_uri is None:
-            raise Exception("Couldn't open taxonomy")
-        taxo = taxo_uri[1]
-    else :
-        taxo = None
-
-    # Initialize the progress bar
-    pb = ProgressBar(len(o_view), config, seconde=5)
-
-    try:
-
-        # Apply editions
-        # Editions at view level
-        if 'delete_tags' in config['annotate']:
-            toDelete = config['annotate']['delete_tags'][:]
-        if 'rename_tags' in config['annotate']:
-            toRename = [x.split(':',1) for x in config['annotate']['rename_tags'] if len(x.split(':',1))==2]
-        if 'clear' in config['annotate']:
-            clear = config['annotate']['clear']
-        if 'keep' in config['annotate']:
-            keep = config['annotate']['keep']
-        for i in range(len(toDelete)):
-            toDelete[i] = tobytes(toDelete[i])
-        for i in range(len(toRename)):
-            for j in range(len(toRename[i])):
-                toRename[i][j] = tobytes(toRename[i][j])
-        for i in range(len(keep)):
-            keep[i] = tobytes(keep[i])
-        keep = set(keep)
-    
-        if clear or keep:
-            keys = [k for k in o_view.keys()]
-            for k in keys:
-                if k not in keep and k not in SPECIAL_COLUMNS:
-                    o_view.delete_column(k)
-        else:
-            for k in toDelete:
-                o_view.delete_column(k)
-            for old_name, new_name in toRename:
-                if old_name in o_view:
-                    o_view.rename_column(old_name, new_name)
-        
-        # Editions at line level
-        sequenceTagger = sequenceTaggerGenerator(config, taxo=taxo)
-        for i in range(len(o_view)):
-            pb(i)
-            sequenceTagger(o_view[i])
-
-    except Exception, e:
-        raise RollbackException("obi annotate error, rollbacking view: "+str(e), o_view)
-
-    pb(i, force=True)
-    print("", file=sys.stderr)
-    
-    # Save command config in View and DMS comments
-    command_line = " ".join(sys.argv[1:])
-    input_dms_name=[input[0].name]
-    input_view_name=[i_view_name]
-    if 'taxoURI' in config['obi'] and config['obi']['taxoURI'] is not None:
-        input_dms_name.append(config['obi']['taxoURI'].split("/")[-3])
-        input_view_name.append("taxonomy/"+config['obi']['taxoURI'].split("/")[-1])
-    o_view.write_config(config, "annotate", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name)
-    output[0].record_command_line(command_line)
-
-    #print("\n\nOutput view:\n````````````", file=sys.stderr)
-    #print(repr(o_view), file=sys.stderr)
-
-    # If the input and the output DMS are different, delete the temporary imported view used to create the final view
-    if i_dms != o_dms:
-        View.delete_view(o_dms, imported_view_name)
-        o_dms.close()
-    i_dms.close()
-
-    logger("info", "Done.")

python/obitools3/commands/build_ref_db.cfiles

@@ -1,103 +0,0 @@
-../../../src/obi_lcs.h
-../../../src/obi_lcs.c
-../../../src/obierrno.h
-../../../src/obierrno.c
-../../../src/upperband.h
-../../../src/upperband.c
-../../../src/sse_banded_LCS_alignment.h
-../../../src/sse_banded_LCS_alignment.c
-../../../src/obiblob.h
-../../../src/obiblob.c
-../../../src/utils.h
-../../../src/utils.c
-../../../src/obidms.h
-../../../src/obidms.c
-../../../src/libjson/json_utils.h
-../../../src/libjson/json_utils.c
-../../../src/libjson/cJSON.h
-../../../src/libjson/cJSON.c
-../../../src/obiavl.h
-../../../src/obiavl.c
-../../../src/bloom.h
-../../../src/bloom.c
-../../../src/crc64.h
-../../../src/crc64.c
-../../../src/murmurhash2.h
-../../../src/murmurhash2.c
-../../../src/obidmscolumn.h
-../../../src/obidmscolumn.c
-../../../src/obitypes.h
-../../../src/obitypes.c
-../../../src/obidmscolumndir.h
-../../../src/obidmscolumndir.c
-../../../src/obiblob_indexer.h
-../../../src/obiblob_indexer.c
-../../../src/obiview.h
-../../../src/obiview.c
-../../../src/hashtable.h
-../../../src/hashtable.c
-../../../src/linked_list.h
-../../../src/linked_list.c
-../../../src/obidmscolumn_array.h
-../../../src/obidmscolumn_array.c
-../../../src/obidmscolumn_blob.h
-../../../src/obidmscolumn_blob.c
-../../../src/obidmscolumn_idx.h
-../../../src/obidmscolumn_idx.c
-../../../src/obidmscolumn_bool.h
-../../../src/obidmscolumn_bool.c
-../../../src/obidmscolumn_char.h
-../../../src/obidmscolumn_char.c
-../../../src/obidmscolumn_float.h
-../../../src/obidmscolumn_float.c
-../../../src/obidmscolumn_int.h
-../../../src/obidmscolumn_int.c
-../../../src/obidmscolumn_qual.h
-../../../src/obidmscolumn_qual.c
-../../../src/obidmscolumn_seq.h
-../../../src/obidmscolumn_seq.c
-../../../src/obidmscolumn_str.h
-../../../src/obidmscolumn_str.c
-../../../src/array_indexer.h
-../../../src/array_indexer.c
-../../../src/char_str_indexer.h
-../../../src/char_str_indexer.c
-../../../src/dna_seq_indexer.h
-../../../src/dna_seq_indexer.c
-../../../src/encode.c
-../../../src/encode.h
-../../../src/uint8_indexer.c
-../../../src/uint8_indexer.h
-../../../src/build_reference_db.c
-../../../src/build_reference_db.h
-../../../src/kmer_similarity.c
-../../../src/kmer_similarity.h
-../../../src/obi_clean.c
-../../../src/obi_clean.h
-../../../src/obi_ecopcr.c
-../../../src/obi_ecopcr.h
-../../../src/obi_ecotag.c
-../../../src/obi_ecotag.h
-../../../src/obidms_taxonomy.c
-../../../src/obidms_taxonomy.h
-../../../src/obilittlebigman.c
-../../../src/obilittlebigman.h
-../../../src/_sse.h
-../../../src/obidebug.h
-../../../src/libecoPCR/libapat/CODES/dft_code.h
-../../../src/libecoPCR/libapat/CODES/dna_code.h
-../../../src/libecoPCR/libapat/CODES/prot_code.h
-../../../src/libecoPCR/libapat/apat_parse.c
-../../../src/libecoPCR/libapat/apat_search.c
-../../../src/libecoPCR/libapat/apat.h
-../../../src/libecoPCR/libapat/Gmach.h
-../../../src/libecoPCR/libapat/Gtypes.h
-../../../src/libecoPCR/libapat/libstki.c
-../../../src/libecoPCR/libapat/libstki.h
-../../../src/libecoPCR/libthermo/nnparams.h
-../../../src/libecoPCR/libthermo/nnparams.c
-../../../src/libecoPCR/ecoapat.c
-../../../src/libecoPCR/ecodna.c
-../../../src/libecoPCR/ecoError.c
-../../../src/libecoPCR/ecoMalloc.c
-../../../src/libecoPCR/ecoPCR.h

python/obitools3/commands/build_ref_db.pyx

@@ -1,105 +0,0 @@
-#cython: language_level=3
-
-from obitools3.apps.progress cimport ProgressBar  # @UnresolvedImport
-from obitools3.dms.dms cimport DMS
-from obitools3.dms.view import RollbackException
-from obitools3.dms.capi.build_reference_db cimport build_reference_db
-from obitools3.apps.optiongroups import addMinimalInputOption, addTaxonomyOption, addMinimalOutputOption
-from obitools3.uri.decode import open_uri
-from obitools3.apps.config import logger
-from obitools3.utils cimport tobytes, str2bytes
-from obitools3.dms.view.view cimport View
-from obitools3.dms.view.typed_view.view_NUC_SEQS cimport View_NUC_SEQS
-
-import sys
-
-
-__title__="Tag a set of sequences for PCR and sequencing errors identification"
-
-
-def addOptions(parser):
-
-    addMinimalInputOption(parser)
-    addTaxonomyOption(parser)
-    addMinimalOutputOption(parser)
- 
-    group = parser.add_argument_group('obi build_ref_db specific options')
-
-    group.add_argument('--threshold','-t',
-                      action="store", dest="build_ref_db:threshold",
-                      metavar='<THRESHOLD>',
-                      default=0.0,
-                      type=float,
-                      help="Score threshold as a normalized identity, e.g. 0.95 for an identity of 95%%. Default: 0.00"
-                           " (no threshold).")
-
-
-def run(config):
-        
-    DMS.obi_atexit()
-    
-    logger("info", "obi build_ref_db")
-
-    # Open the input: only the DMS
-    input = open_uri(config['obi']['inputURI'],
-                     dms_only=True)
-    if input is None:
-        raise Exception("Could not read input")
-    i_dms = input[0]
-    i_dms_name = input[0].name
-    i_view_name = input[1]
-
-    # Open the output: only the DMS
-    output = open_uri(config['obi']['outputURI'],
-                      input=False,
-                      dms_only=True)
-    if output is None:
-        raise Exception("Could not create output")
-    o_dms = output[0]
-    final_o_view_name = output[1]
-    
-    # If the input and output DMS are not the same, build the database creating a temporary view that will be exported to 
-    # the right DMS and deleted in the other afterwards.
-    if i_dms != o_dms:
-        temporary_view_name = final_o_view_name
-        i=0
-        while temporary_view_name in i_dms:  # Making sure view name is unique in input DMS
-            temporary_view_name = final_o_view_name+b"_"+str2bytes(str(i))
-            i+=1
-        o_view_name = temporary_view_name
-    else:
-        o_view_name = final_o_view_name
-
-    # Read taxonomy name
-    taxonomy_name = config['obi']['taxoURI'].split("/")[-1]   # Robust in theory
-
-    # Save command config in View comments
-    command_line = " ".join(sys.argv[1:])
-    input_dms_name=[i_dms_name]
-    input_view_name= [i_view_name]
-    input_dms_name.append(config['obi']['taxoURI'].split("/")[-3])
-    input_view_name.append("taxonomy/"+config['obi']['taxoURI'].split("/")[-1])
-    comments = View.print_config(config, "build_ref_db", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name)
-
-    if build_reference_db(tobytes(i_dms_name), tobytes(i_view_name), tobytes(taxonomy_name), tobytes(o_view_name), comments, config['build_ref_db']['threshold']) < 0:
-        raise Exception("Error building a reference database")
-    
-    # If the input and output DMS are not the same, export result view to output DMS
-    if i_dms != o_dms:
-        View.import_view(i_dms.full_path[:-7], o_dms.full_path[:-7], o_view_name, final_o_view_name)
-    
-    # Save command config in DMS comments
-    o_dms.record_command_line(command_line)
-
-    #print("\n\nOutput view:\n````````````", file=sys.stderr)
-    #print(repr(o_dms[final_o_view_name]), file=sys.stderr)
-
-    # If the input and the output DMS are different, delete the temporary result view in the input DMS
-    if i_dms != o_dms:
-        View.delete_view(i_dms, o_view_name)
-        o_dms.close()
-    
-    i_dms.close()
-
-    logger("info", "Done.")
-    

python/obitools3/commands/clean.cfiles

@@ -1,103 +0,0 @@
-../../../src/obi_lcs.h
-../../../src/obi_lcs.c
-../../../src/obierrno.h
-../../../src/obierrno.c
-../../../src/upperband.h
-../../../src/upperband.c
-../../../src/sse_banded_LCS_alignment.h
-../../../src/sse_banded_LCS_alignment.c
-../../../src/obiblob.h
-../../../src/obiblob.c
-../../../src/utils.h
-../../../src/utils.c
-../../../src/obidms.h
-../../../src/obidms.c
-../../../src/libjson/json_utils.h
-../../../src/libjson/json_utils.c
-../../../src/libjson/cJSON.h
-../../../src/libjson/cJSON.c
-../../../src/obiavl.h
-../../../src/obiavl.c
-../../../src/bloom.h
-../../../src/bloom.c
-../../../src/crc64.h
-../../../src/crc64.c
-../../../src/murmurhash2.h
-../../../src/murmurhash2.c
-../../../src/obidmscolumn.h
-../../../src/obidmscolumn.c
-../../../src/obitypes.h
-../../../src/obitypes.c
-../../../src/obidmscolumndir.h
-../../../src/obidmscolumndir.c
-../../../src/obiblob_indexer.h
-../../../src/obiblob_indexer.c
-../../../src/obiview.h
-../../../src/obiview.c
-../../../src/hashtable.h
-../../../src/hashtable.c
-../../../src/linked_list.h
-../../../src/linked_list.c
-../../../src/obidmscolumn_array.h
-../../../src/obidmscolumn_array.c
-../../../src/obidmscolumn_blob.h
-../../../src/obidmscolumn_blob.c
-../../../src/obidmscolumn_idx.h
-../../../src/obidmscolumn_idx.c
-../../../src/obidmscolumn_bool.h
-../../../src/obidmscolumn_bool.c
-../../../src/obidmscolumn_char.h
-../../../src/obidmscolumn_char.c
-../../../src/obidmscolumn_float.h
-../../../src/obidmscolumn_float.c
-../../../src/obidmscolumn_int.h
-../../../src/obidmscolumn_int.c
-../../../src/obidmscolumn_qual.h
-../../../src/obidmscolumn_qual.c
-../../../src/obidmscolumn_seq.h
-../../../src/obidmscolumn_seq.c
-../../../src/obidmscolumn_str.h
-../../../src/obidmscolumn_str.c
-../../../src/array_indexer.h
-../../../src/array_indexer.c
-../../../src/char_str_indexer.h
-../../../src/char_str_indexer.c
-../../../src/dna_seq_indexer.h
-../../../src/dna_seq_indexer.c
-../../../src/encode.c
-../../../src/encode.h
-../../../src/uint8_indexer.c
-../../../src/uint8_indexer.h
-../../../src/build_reference_db.c
-../../../src/build_reference_db.h
-../../../src/kmer_similarity.c
-../../../src/kmer_similarity.h
-../../../src/obi_clean.c
-../../../src/obi_clean.h
-../../../src/obi_ecopcr.c
-../../../src/obi_ecopcr.h
-../../../src/obi_ecotag.c
-../../../src/obi_ecotag.h
-../../../src/obidms_taxonomy.c
-../../../src/obidms_taxonomy.h
-../../../src/obilittlebigman.c
-../../../src/obilittlebigman.h
-../../../src/_sse.h
-../../../src/obidebug.h
-../../../src/libecoPCR/libapat/CODES/dft_code.h
-../../../src/libecoPCR/libapat/CODES/dna_code.h
-../../../src/libecoPCR/libapat/CODES/prot_code.h
-../../../src/libecoPCR/libapat/apat_parse.c
-../../../src/libecoPCR/libapat/apat_search.c
-../../../src/libecoPCR/libapat/apat.h
-../../../src/libecoPCR/libapat/Gmach.h
-../../../src/libecoPCR/libapat/Gtypes.h
-../../../src/libecoPCR/libapat/libstki.c
-../../../src/libecoPCR/libapat/libstki.h
-../../../src/libecoPCR/libthermo/nnparams.h
-../../../src/libecoPCR/libthermo/nnparams.c
-../../../src/libecoPCR/ecoapat.c
-../../../src/libecoPCR/ecodna.c
-../../../src/libecoPCR/ecoError.c
-../../../src/libecoPCR/ecoMalloc.c
-../../../src/libecoPCR/ecoPCR.h

python/obitools3/commands/clean.pyx

@@ -1,124 +0,0 @@
-#cython: language_level=3
-
-from obitools3.apps.progress cimport ProgressBar  # @UnresolvedImport
-from obitools3.dms.dms cimport DMS
-from obitools3.dms.view import RollbackException
-from obitools3.dms.capi.obiclean cimport obi_clean
-from obitools3.apps.optiongroups import addMinimalInputOption, addMinimalOutputOption
-from obitools3.uri.decode import open_uri
-from obitools3.apps.config import logger
-from obitools3.utils cimport tobytes, str2bytes
-from obitools3.dms.view.view cimport View
-from obitools3.dms.view.typed_view.view_NUC_SEQS cimport View_NUC_SEQS
-
-import sys
-
-
-__title__="Tag a set of sequences for PCR and sequencing errors identification"
-
-
-def addOptions(parser):
-
-    addMinimalInputOption(parser)
-    addMinimalOutputOption(parser)
- 
-    group = parser.add_argument_group('obi clean specific options')
-
-    group.add_argument('--distance', '-d',
-                       action="store", dest="clean:distance",
-                       metavar='<DISTANCE>',
-                       default=1.0,
-                       type=float,
-                       help="Maximum numbers of errors between two variant sequences. Default: 1.")
-
-    group.add_argument('--sample-tag', '-s',
-                       action="store", 
-                       dest="clean:sample-tag-name",
-                       metavar="<SAMPLE TAG NAME>",
-                       type=str,
-                       default="merged_sample",
-                       help="Name of the tag where sample counts are kept.")
-    
-    group.add_argument('--ratio', '-r',
-                       action="store", dest="clean:ratio",
-                       metavar='<RATIO>',
-                       default=0.5,
-                       type=float,
-                       help="Maximum ratio between the counts of two sequences so that the less abundant one can be considered"
-                            " a variant of the more abundant one. Default: 0.5.")
- 
-    group.add_argument('--heads-only', '-H',
-                       action="store_true", 
-                       dest="clean:heads-only",
-                       default=False,
-                       help="Only sequences labeled as heads are kept in the output. Default: False")
-
-    group.add_argument('--cluster-tags', '-C',
-                       action="store_true", 
-                       dest="clean:cluster-tags",
-                       default=False,
-                       help="Adds tags for each sequence giving its cluster's head and weight for each sample.")
-
-
-def run(config):
-        
-    DMS.obi_atexit()
-    
-    logger("info", "obi clean")
-
-    # Open the input: only the DMS
-    input = open_uri(config['obi']['inputURI'],
-                     dms_only=True)
-    if input is None:
-        raise Exception("Could not read input")
-    i_dms = input[0]
-    i_dms_name = input[0].name
-    i_view_name = input[1]
-
-    # Open the output: only the DMS
-    output = open_uri(config['obi']['outputURI'],
-                      input=False,
-                      dms_only=True)
-    if output is None:
-        raise Exception("Could not create output")
-    o_dms = output[0]
-    final_o_view_name = output[1]
-    
-    # If the input and output DMS are not the same, run obiclean creating a temporary view that will be exported to 
-    # the right DMS and deleted in the other afterwards.
-    if i_dms != o_dms:
-        temporary_view_name = final_o_view_name
-        i=0
-        while temporary_view_name in i_dms:  # Making sure view name is unique in input DMS
-            temporary_view_name = final_o_view_name+b"_"+str2bytes(str(i))
-            i+=1
-        o_view_name = temporary_view_name
-    else:
-        o_view_name = final_o_view_name
-        
-    # Save command config in View comments
-    command_line = " ".join(sys.argv[1:])
-    comments = View.print_config(config, "clean", command_line, input_dms_name=[i_dms_name], input_view_name=[i_view_name])
-
-    if obi_clean(tobytes(i_dms_name), tobytes(i_view_name), tobytes(config['clean']['sample-tag-name']), tobytes(o_view_name), comments, \
-              config['clean']['distance'], config['clean']['ratio'], config['clean']['heads-only'], 1) < 0:
-        raise Exception("Error running obiclean")
-    
-    # If the input and output DMS are not the same, export result view to output DMS
-    if i_dms != o_dms:
-        View.import_view(i_dms.full_path[:-7], o_dms.full_path[:-7], o_view_name, final_o_view_name)
-    
-    # Save command config in DMS comments
-    o_dms.record_command_line(command_line)
-
-    #print("\n\nOutput view:\n````````````", file=sys.stderr)
-    #print(repr(o_dms[final_o_view_name]), file=sys.stderr)
-
-    # If the input and the output DMS are different, delete the temporary result view in the input DMS
-    if i_dms != o_dms:
-        View.delete_view(i_dms, o_view_name)
-        o_dms.close()
-    
-    i_dms.close()
-
-    logger("info", "Done.")

python/obitools3/commands/count.cfiles

@@ -1,103 +0,0 @@
-../../../src/obi_lcs.h
-../../../src/obi_lcs.c
-../../../src/obierrno.h
-../../../src/obierrno.c
-../../../src/upperband.h
-../../../src/upperband.c
-../../../src/sse_banded_LCS_alignment.h
-../../../src/sse_banded_LCS_alignment.c
-../../../src/obiblob.h
-../../../src/obiblob.c
-../../../src/utils.h
-../../../src/utils.c
-../../../src/obidms.h
-../../../src/obidms.c
-../../../src/libjson/json_utils.h
-../../../src/libjson/json_utils.c
-../../../src/libjson/cJSON.h
-../../../src/libjson/cJSON.c
-../../../src/obiavl.h
-../../../src/obiavl.c
-../../../src/bloom.h
-../../../src/bloom.c
-../../../src/crc64.h
-../../../src/crc64.c
-../../../src/murmurhash2.h
-../../../src/murmurhash2.c
-../../../src/obidmscolumn.h
-../../../src/obidmscolumn.c
-../../../src/obitypes.h
-../../../src/obitypes.c
-../../../src/obidmscolumndir.h
-../../../src/obidmscolumndir.c
-../../../src/obiblob_indexer.h
-../../../src/obiblob_indexer.c
-../../../src/obiview.h
-../../../src/obiview.c
-../../../src/hashtable.h
-../../../src/hashtable.c
-../../../src/linked_list.h
-../../../src/linked_list.c
-../../../src/obidmscolumn_array.h
-../../../src/obidmscolumn_array.c
-../../../src/obidmscolumn_blob.h
-../../../src/obidmscolumn_blob.c
-../../../src/obidmscolumn_idx.h
-../../../src/obidmscolumn_idx.c
-../../../src/obidmscolumn_bool.h
-../../../src/obidmscolumn_bool.c
-../../../src/obidmscolumn_char.h
-../../../src/obidmscolumn_char.c
-../../../src/obidmscolumn_float.h
-../../../src/obidmscolumn_float.c
-../../../src/obidmscolumn_int.h
-../../../src/obidmscolumn_int.c
-../../../src/obidmscolumn_qual.h
-../../../src/obidmscolumn_qual.c
-../../../src/obidmscolumn_seq.h
-../../../src/obidmscolumn_seq.c
-../../../src/obidmscolumn_str.h
-../../../src/obidmscolumn_str.c
-../../../src/array_indexer.h
-../../../src/array_indexer.c
-../../../src/char_str_indexer.h
-../../../src/char_str_indexer.c
-../../../src/dna_seq_indexer.h
-../../../src/dna_seq_indexer.c
-../../../src/encode.c
-../../../src/encode.h
-../../../src/uint8_indexer.c
-../../../src/uint8_indexer.h
-../../../src/build_reference_db.c
-../../../src/build_reference_db.h
-../../../src/kmer_similarity.c
-../../../src/kmer_similarity.h
-../../../src/obi_clean.c
-../../../src/obi_clean.h
-../../../src/obi_ecopcr.c
-../../../src/obi_ecopcr.h
-../../../src/obi_ecotag.c
-../../../src/obi_ecotag.h
-../../../src/obidms_taxonomy.c
-../../../src/obidms_taxonomy.h
-../../../src/obilittlebigman.c
-../../../src/obilittlebigman.h
-../../../src/_sse.h
-../../../src/obidebug.h
-../../../src/libecoPCR/libapat/CODES/dft_code.h
-../../../src/libecoPCR/libapat/CODES/dna_code.h
-../../../src/libecoPCR/libapat/CODES/prot_code.h
-../../../src/libecoPCR/libapat/apat_parse.c
-../../../src/libecoPCR/libapat/apat_search.c
-../../../src/libecoPCR/libapat/apat.h
-../../../src/libecoPCR/libapat/Gmach.h
-../../../src/libecoPCR/libapat/Gtypes.h
-../../../src/libecoPCR/libapat/libstki.c
-../../../src/libecoPCR/libapat/libstki.h
-../../../src/libecoPCR/libthermo/nnparams.h
-../../../src/libecoPCR/libthermo/nnparams.c
-../../../src/libecoPCR/ecoapat.c
-../../../src/libecoPCR/ecodna.c
-../../../src/libecoPCR/ecoError.c
-../../../src/libecoPCR/ecoMalloc.c
-../../../src/libecoPCR/ecoPCR.h

python/obitools3/commands/count.py

@@ -0,0 +1,44 @@
+'''
+Created on 8 mars 2016
+
+@author: coissac
+'''
+
+from obitools3.apps.progress import ProgressBar  # @UnresolvedImport
+import time
+
+__title__="Counts sequences in a sequence set"
+
+
+default_config = {   'countmode'       : None
+                 }
+
+def addOptions(parser):
+    parser.add_argument(dest='obi:input',     metavar='obi:input', 
+                                              nargs='?', 
+                                              default=None,
+                        help='input data set' )
+
+    group=parser.add_argument_group('Obicount specific options')
+    group.add_argument('-s','--sequence',
+                             action="store_true", dest="count:sequence",
+                             default=False,
+                             help="Prints only the number of sequence records."
+                             )
+ 
+    group.add_argument('-a','--all',
+                             action="store_true", dest="count:all",
+                             default=False,
+                             help="Prints only the total count of sequence records (if a sequence has no `count` attribute, its default count is 1) (default: False)."
+                             )
+
+    
+    
+def run(config):
+    # The code of my command
+    pb = ProgressBar(1000,config,seconde=1)
+    
+    for i in range(1,1001):
+        pb(i)
+        time.sleep(0.01)
+        

python/obitools3/commands/count.pyx

@@ -1,55 +0,0 @@
-#cython: language_level=3
-
-
-from obitools3.uri.decode import open_uri
-from obitools3.apps.config import logger
-from obitools3.dms import DMS
-from obitools3.apps.optiongroups import addMinimalInputOption
-from obitools3.dms.capi.obiview cimport COUNT_COLUMN
-
-
-__title__="Counts sequence records"
-
- 
-def addOptions(parser):
- 
-    addMinimalInputOption(parser)
- 
-    group = parser.add_argument_group('obi count specific options')
- 
-    group.add_argument('-s','--sequence',
-                        action="store_true", dest="count:sequence",
-                        default=False,
-                        help="Prints only the number of sequence records.")
- 
-    group.add_argument('-a','--all',
-                        action="store_true", dest="count:all",
-                        default=False,
-                        help="Prints only the total count of sequence records (if a sequence has no `count` attribute, its default count is 1) (default: False).")
-
-    
-def run(config):
-    
-    DMS.obi_atexit()
-    
-    logger("info", "obi count")
-
-    # Open the input
-    input = open_uri(config['obi']['inputURI'])
-    if input is None:
-        raise Exception("Could not read input")
-    entries = input[1]
-
-    count1 = len(entries)
-    count2 = 0
-    
-    if COUNT_COLUMN in entries and ((config['count']['sequence'] == config['count']['all']) or (config['count']['all'])) :
-        for e in entries:
-            count2+=e[COUNT_COLUMN]
-
-    if COUNT_COLUMN in entries and (config['count']['sequence'] == config['count']['all']):
-        print(count1,count2)
-    elif COUNT_COLUMN in entries and config['count']['all']:
-        print(count2)
-    else:
-        print(count1)