diff --git a/doc/conf.py b/doc/conf.py index ecb4f68..b7274d9 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -36,7 +36,7 @@ extensions = [ 'sphinx.ext.pngmath', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode', -# 'breathe', + 'breathe', ] # Add any paths that contain templates here, relative to this directory. @@ -295,4 +295,6 @@ texinfo_documents = [ sys.path.append( "breathe/" ) breathe_projects = { "OBITools3": "doxygen/xml/" } breathe_default_project = "OBITools3" - +#breathe_projects_source = { +# "auto" : ( "../src", ["obidms.h", "obiavl.h"] ) +# } \ No newline at end of file diff --git a/doc/source/index.rst b/doc/source/index.rst index 1f8b861..a5f46ab 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -11,7 +11,7 @@ OBITools3 documentation Programming guidelines Data structures - + Code documentation Indices and tables ------------------ diff --git a/doc/source/types.rst b/doc/source/types.rst index d50f60d..6895949 100644 --- a/doc/source/types.rst +++ b/doc/source/types.rst @@ -4,6 +4,7 @@ OBITypes .. image:: ./UML/OBITypes_UML.png + :download:`html version of the OBITypes UML file ` diff --git a/doc/sphinx/build_dir.txt b/doc/sphinx/build_dir.txt index 9431bd8..6d44141 100644 --- a/doc/sphinx/build_dir.txt +++ b/doc/sphinx/build_dir.txt @@ -1 +1 @@ -build/lib.macosx-10.6-intel-3.5 +build/lib.macosx-10.6-intel-3.4 diff --git a/python/obitools3/__init__.pyc b/python/obitools3/__init__.pyc new file mode 100644 index 0000000..418f2f1 Binary files /dev/null and b/python/obitools3/__init__.pyc differ diff --git a/python/obitools3/commands/uniq.pyx b/python/obitools3/commands/uniq.pyx new file mode 100644 index 0000000..2a75e86 --- /dev/null +++ b/python/obitools3/commands/uniq.pyx @@ -0,0 +1,96 @@ +from obitools3.apps.progress cimport ProgressBar # @UnresolvedImport +from obitools3.obidms._obidms import OBIDMS, OBIView, OBIView_line_selection # TODO cimport doesn't work + +from functools import reduce +import time + +__title__="Grep view lines that match the given predicates" + + +default_config = { 'inputview' : None, + 'outputview' : None + } + +def addOptions(parser): + + # TODO put this common group somewhere else but I don't know where + group=parser.add_argument_group('DMS and view options') + + group.add_argument('--default-dms','-d', + action="store", dest="obi:defaultdms", + metavar='', + default=None, + type=str, + help="Name of the default DMS for reading and writing data.") + + group.add_argument('--input-view','-i', + action="store", dest="obi:inputview", + metavar='', + default=None, + type=str, + help="Name of the input view, either raw if the view is in the default DMS," + " or in the form 'dms:view' if it is in another DMS.") + + group.add_argument('--output-view','-o', + action="store", dest="obi:outputview", + metavar='', + default=None, + type=str, + help="Name of the output view, either raw if the view is in the default DMS," + " or in the form 'dms:view' if it is in another DMS.") + + + group=parser.add_argument_group('obi grep specific options') + + group.add_argument('--predicate','-p', + action="append", dest="grep:predicates", + metavar='', + default=None, + type=str, + help="Grep lines that match the given python expression on or .") + + +def run(config): + + # Open DMS + d = OBIDMS(config['obi']['defaultdms']) + + # Open input view 1 + iview = d.open_view(config['obi']['inputview']) + + # Initialize the progress bar + pb = ProgressBar(len(iview), config, seconde=5) + + # Apply filter + selection = OBIView_line_selection(iview) + for i in range(len(iview)) : + pb(i) + line = iview[i] + + loc_env = {'sequence': line, 'line': line} # TODO add taxonomy + + good = (reduce(lambda bint x, bint y: x and y, + (bool(eval(p, loc_env, line)) + for p in config['grep']['predicates']), True)) + + if good : + selection.append(i) + + # Create output view with the line selection + oview = d.new_view(config['obi']['outputview'], line_selection=selection, comments="obi grep: "+str(config['grep']['predicates'])+"\n") + + #print("\n") + #print(repr(oview)) + + iview.close() + oview.close() + d.close() + + + + + + + + + \ No newline at end of file diff --git a/python/obitools3/dms/__init__.py b/python/obitools3/dms/__init__.py new file mode 100644 index 0000000..60bfaf1 --- /dev/null +++ b/python/obitools3/dms/__init__.py @@ -0,0 +1,2 @@ +from .dms import DMS # @UnresolvedImport + diff --git a/python/obitools3/dms/capi/__init__.py b/python/obitools3/dms/capi/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python/obitools3/dms/capi/obialign.pxd b/python/obitools3/dms/capi/obialign.pxd new file mode 100644 index 0000000..4ab5dba --- /dev/null +++ b/python/obitools3/dms/capi/obialign.pxd @@ -0,0 +1,41 @@ +#cython: language_level=3 + +from obitools3.obidms.capi.obidms cimport OBIDMS_p +from obitools3.obidms.capi.obitypes cimport const_char_p + + +cdef extern from "obi_align.h" nogil: + + int obi_lcs_align_one_column(OBIDMS_p dms, + const_char_p seq_view_name, + const_char_p seq_column_name, + const_char_p seq_elt_name, + const_char_p id_column_name, + const_char_p output_view_name, + const_char_p output_view_comments, + bint print_seq, + bint print_count, + double threshold, + bint normalize, + int reference, + bint similarity_mode, + int thread_count) + + + int obi_lcs_align_two_columns(OBIDMS_p dms, + const_char_p seq1_view_name, + const_char_p seq2_view_name, + const_char_p seq1_column_name, + const_char_p seq2_column_name, + const_char_p seq1_elt_name, + const_char_p seq2_elt_name, + const_char_p id1_column_name, + const_char_p id2_column_name, + const_char_p output_view_name, + const_char_p output_view_comments, + bint print_seq, + bint print_count, + double threshold, + bint normalize, + int reference, + bint similarity_mode) diff --git a/python/obitools3/dms/capi/obidms.pxd b/python/obitools3/dms/capi/obidms.pxd new file mode 100644 index 0000000..5f30088 --- /dev/null +++ b/python/obitools3/dms/capi/obidms.pxd @@ -0,0 +1,16 @@ +#cython: language_level=3 + +from .obitypes cimport const_char_p + +cdef extern from "obidms.h" nogil: + struct OBIDMS_t: + const_char_p dms_name + + ctypedef OBIDMS_t* OBIDMS_p + + OBIDMS_p obi_dms(const_char_p dms_name) + int obi_close_dms(OBIDMS_p dms) + char* obi_dms_get_dms_path(OBIDMS_p dms) + char* obi_dms_get_full_path(OBIDMS_p dms, const_char_p path_name) + + diff --git a/python/obitools3/dms/capi/obidmscolumn.pxd b/python/obitools3/dms/capi/obidmscolumn.pxd new file mode 100644 index 0000000..9589421 --- /dev/null +++ b/python/obitools3/dms/capi/obidmscolumn.pxd @@ -0,0 +1,227 @@ +#cython: language_level=3 + +from ..capi.obidms cimport OBIDMS_p +from ..capi.obitypes cimport const_char_p, \ + OBIType_t, \ + obiversion_t, \ + obiint_t, \ + obibool_t, \ + obichar_t, \ + obifloat_t, \ + index_t, \ + time_t + +from libc.stdint cimport uint8_t + + +cdef extern from "obidmscolumn.h" nogil: + + struct Column_reference_t : + const_char_p column_name + obiversion_t version + + ctypedef Column_reference_t* Column_reference_p + + struct OBIDMS_column_header_t: + size_t header_size + size_t data_size + index_t line_count + index_t lines_used + index_t nb_elements_per_line + const_char_p elements_names + OBIType_t returned_data_type + OBIType_t stored_data_type + time_t creation_date + obiversion_t version + obiversion_t cloned_from + const_char_p name + const_char_p indexer_name + Column_reference_t associated_column + const_char_p comments + + ctypedef OBIDMS_column_header_t* OBIDMS_column_header_p + + struct OBIDMS_column_t: + OBIDMS_p dms + OBIDMS_column_header_p header + bint writable + + ctypedef OBIDMS_column_t* OBIDMS_column_p + + int obi_close_column(OBIDMS_column_p column) + + obiversion_t obi_column_get_latest_version_from_name(OBIDMS_p dms, + const_char_p column_name) + + OBIDMS_column_header_p obi_column_get_header_from_name(OBIDMS_p dms, + const_char_p column_name, + obiversion_t version_number) + + int obi_close_header(OBIDMS_column_header_p header) + + char* obi_get_elements_names(OBIDMS_column_p column) + + +cdef extern from "obidmscolumn_int.h" nogil: + + int obi_column_set_obiint_with_elt_name(OBIDMS_column_p column, + index_t line_nb, + const_char_p element_name, + obiint_t value) + + int obi_column_set_obiint_with_elt_idx(OBIDMS_column_p column, + index_t line_nb, + index_t element_idx, + obiint_t value) + + obiint_t obi_column_get_obiint_with_elt_name(OBIDMS_column_p column, + index_t line_nb, + const_char_p element_name) + + obiint_t obi_column_get_obiint_with_elt_idx(OBIDMS_column_p column, + index_t line_nb, + index_t element_idx) + +cdef extern from "obidmscolumn_bool.h" nogil: + + int obi_column_set_obibool_with_elt_name(OBIDMS_column_p column, + index_t line_nb, + const_char_p element_name, + obibool_t value) + + int obi_column_set_obibool_with_elt_idx(OBIDMS_column_p column, + index_t line_nb, + index_t element_idx, + obibool_t value) + + obibool_t obi_column_get_obibool_with_elt_name(OBIDMS_column_p column, + index_t line_nb, + const_char_p element_name) + + obibool_t obi_column_get_obibool_with_elt_idx(OBIDMS_column_p column, + index_t line_nb, + index_t element_idx) + +cdef extern from "obidmscolumn_char.h" nogil: + + int obi_column_set_obichar_with_elt_name(OBIDMS_column_p column, + index_t line_nb, + const_char_p element_name, + obichar_t value) + + int obi_column_set_obichar_with_elt_idx(OBIDMS_column_p column, + index_t line_nb, + index_t element_idx, + obichar_t value) + + obichar_t obi_column_get_obichar_with_elt_name(OBIDMS_column_p column, + index_t line_nb, + const_char_p element_name) + + obichar_t obi_column_get_obichar_with_elt_idx(OBIDMS_column_p column, + index_t line_nb, + index_t element_idx) + +cdef extern from "obidmscolumn_float.h" nogil: + + int obi_column_set_obifloat_with_elt_name(OBIDMS_column_p column, + index_t line_nb, + const_char_p element_name, + obifloat_t value) + + int obi_column_set_obifloat_with_elt_idx(OBIDMS_column_p column, + index_t line_nb, + index_t element_idx, + obifloat_t value) + + obifloat_t obi_column_get_obifloat_with_elt_name(OBIDMS_column_p column, + index_t line_nb, + const_char_p element_name) + + obifloat_t obi_column_get_obifloat_with_elt_idx(OBIDMS_column_p column, + index_t line_nb, + index_t element_idx) + +cdef extern from "obidmscolumn_str.h" nogil: + + int obi_column_set_obistr_with_elt_name(OBIDMS_column_p column, + index_t line_nb, + const_char_p element_name, + const_char_p value) + + int obi_column_set_obistr_with_elt_idx(OBIDMS_column_p column, + index_t line_nb, + index_t element_idx, + const_char_p value) + + const_char_p obi_column_get_obistr_with_elt_name(OBIDMS_column_p column, + index_t line_nb, + const_char_p element_name) + + const_char_p obi_column_get_obistr_with_elt_idx(OBIDMS_column_p column, + index_t line_nb, + index_t element_idx) + +cdef extern from "obidmscolumn_seq.h" nogil: + + int obi_column_set_obiseq_with_elt_name(OBIDMS_column_p column, + index_t line_nb, + const_char_p element_name, + const_char_p value) + + int obi_column_set_obiseq_with_elt_idx(OBIDMS_column_p column, + index_t line_nb, + index_t element_idx, + const_char_p value) + + char* obi_column_get_obiseq_with_elt_name(OBIDMS_column_p column, + index_t line_nb, + const_char_p element_name) + + char* obi_column_get_obiseq_with_elt_idx(OBIDMS_column_p column, + index_t line_nb, + index_t element_idx) + + +cdef extern from "obidmscolumn_qual.h" nogil: + + int obi_column_set_obiqual_char_with_elt_name(OBIDMS_column_p column, + index_t line_nb, + const_char_p element_name, + const_char_p value) + + int obi_column_set_obiqual_char_with_elt_idx(OBIDMS_column_p column, + index_t line_nb, + index_t element_idx, + const_char_p value) + + int obi_column_set_obiqual_int_with_elt_name(OBIDMS_column_p column, + index_t line_nb, + const_char_p element_name, + const uint8_t* value, + int value_length) + + int obi_column_set_obiqual_int_with_elt_idx(OBIDMS_column_p column, + index_t line_nb, + index_t element_idx, + const uint8_t* value, + int value_length) + + char* obi_column_get_obiqual_char_with_elt_name(OBIDMS_column_p column, + index_t line_nb, + const_char_p element_name) + + char* obi_column_get_obiqual_char_with_elt_idx(OBIDMS_column_p column, + index_t line_nb, + index_t element_idx) + + const uint8_t* obi_column_get_obiqual_int_with_elt_name(OBIDMS_column_p column, + index_t line_nb, + const_char_p element_name, + int* value_length) + + const uint8_t* obi_column_get_obiqual_int_with_elt_idx(OBIDMS_column_p column, + index_t line_nb, + index_t element_idx, + int* value_length) + diff --git a/python/obitools3/dms/capi/obierrno.pxd b/python/obitools3/dms/capi/obierrno.pxd new file mode 100644 index 0000000..7ba7f1e --- /dev/null +++ b/python/obitools3/dms/capi/obierrno.pxd @@ -0,0 +1,5 @@ +#cython: language_level=3 + + +cdef extern from "obierrno.h": + int obi_errno diff --git a/python/obitools3/dms/capi/obitaxonomy.pxd b/python/obitools3/dms/capi/obitaxonomy.pxd new file mode 100644 index 0000000..d90693c --- /dev/null +++ b/python/obitools3/dms/capi/obitaxonomy.pxd @@ -0,0 +1,65 @@ +#cython: language_level=3 + +from .obitypes cimport const_char_p +from .obidms cimport OBIDMS_p +from libc.stdint cimport int32_t + + +cdef extern from "obidms_taxonomy.h" nogil: + + struct ecotxnode : + int32_t taxid + int32_t rank + int32_t farest + ecotxnode* parent + char* name + char* preferred_name + + ctypedef ecotxnode ecotx_t + + + struct ecotxidx_t : + int32_t count + int32_t max_taxid + int32_t buffer_size + ecotx_t* taxon + + + struct OBIDMS_taxonomy_t : +# ecorankidx_t* ranks +# econameidx_t* names + ecotxidx_t* taxa + + ctypedef OBIDMS_taxonomy_t* OBIDMS_taxonomy_p + + + OBIDMS_taxonomy_p obi_read_taxonomy(OBIDMS_p dms, const_char_p taxonomy_name, bint read_alternative_names) + + OBIDMS_taxonomy_p obi_read_taxdump(const_char_p taxdump) + + int obi_write_taxonomy(OBIDMS_p dms, OBIDMS_taxonomy_p tax, const_char_p tax_name) + + int obi_close_taxonomy(OBIDMS_taxonomy_p taxonomy) + + ecotx_t* obi_taxo_get_parent_at_rank(ecotx_t* taxon, int32_t rankidx) + + ecotx_t* obi_taxo_get_taxon_with_taxid(OBIDMS_taxonomy_p taxonomy, int32_t taxid) + + bint obi_taxo_is_taxon_under_taxid(ecotx_t* taxon, int32_t other_taxid) + + ecotx_t* obi_taxo_get_species(ecotx_t* taxon, OBIDMS_taxonomy_p taxonomy) + + ecotx_t* obi_taxo_get_genus(ecotx_t* taxon, OBIDMS_taxonomy_p taxonomy) + + ecotx_t* obi_taxo_get_family(ecotx_t* taxon, OBIDMS_taxonomy_p taxonomy) + + ecotx_t* obi_taxo_get_kingdom(ecotx_t* taxon, OBIDMS_taxonomy_p taxonomy) + + ecotx_t* obi_taxo_get_superkingdom(ecotx_t* taxon, OBIDMS_taxonomy_p taxonomy) + + int obi_taxo_add_local_taxon(OBIDMS_taxonomy_p tax, const char* name, const char* rank_name, int32_t parent_taxid, int32_t min_taxid) + + int obi_taxo_add_preferred_name_with_taxid(OBIDMS_taxonomy_p tax, int32_t taxid, const char* preferred_name) + + int obi_taxo_add_preferred_name_with_taxon(OBIDMS_taxonomy_p tax, ecotx_t* taxon, const char* preferred_name) + diff --git a/python/obitools3/dms/capi/obitypes.pxd b/python/obitools3/dms/capi/obitypes.pxd new file mode 100644 index 0000000..efc2fb5 --- /dev/null +++ b/python/obitools3/dms/capi/obitypes.pxd @@ -0,0 +1,56 @@ +#cython: language_level=3 + + +from libc.stdint cimport int32_t, int64_t, uint8_t + +from posix.types cimport time_t + + +cdef extern from *: + ctypedef char* const_char_p "const char*" + + +cdef extern from "encode.h" nogil: + bint only_ATGC(const_char_p seq) + + +cdef extern from "obitypes.h" nogil: + + enum OBIType: + OBI_VOID, + OBI_INT, + OBI_FLOAT, + OBI_BOOL, + OBI_CHAR, + OBI_QUAL, + OBI_STR, + OBI_SEQ, + OBI_IDX + + + ctypedef OBIType OBIType_t + + enum OBIBool: + pass + + ctypedef OBIBool obibool_t + ctypedef int32_t obiint_t + ctypedef double obifloat_t + ctypedef char obichar_t + ctypedef int64_t index_t + + ctypedef int32_t obiversion_t + + extern obiint_t OBIInt_NA + extern index_t OBIIdx_NA + extern obifloat_t OBIFloat_NA + extern obichar_t OBIChar_NA + extern obibool_t OBIBool_NA + extern const_char_p OBISeq_NA + extern const_char_p OBIStr_NA + extern const_char_p OBIQual_char_NA + extern uint8_t* OBIQual_int_NA + + const_char_p name_data_type(int data_type) + +ctypedef OBIType_t obitype_t diff --git a/python/obitools3/dms/capi/obiutils.pxd b/python/obitools3/dms/capi/obiutils.pxd new file mode 100644 index 0000000..726ad0e --- /dev/null +++ b/python/obitools3/dms/capi/obiutils.pxd @@ -0,0 +1,12 @@ +#cython: language_level=3 + + +from posix.types cimport time_t + +from ..capi.obitypes cimport const_char_p + + +cdef extern from "utils.h" nogil: + + const_char_p obi_format_date(time_t date) + diff --git a/python/obitools3/dms/capi/obiview.pxd b/python/obitools3/dms/capi/obiview.pxd new file mode 100644 index 0000000..792441b --- /dev/null +++ b/python/obitools3/dms/capi/obiview.pxd @@ -0,0 +1,294 @@ +#cython: language_level=3 + +from .obitypes cimport const_char_p, \ + OBIType_t, \ + obiversion_t, \ + obiint_t, \ + obibool_t, \ + obichar_t, \ + obifloat_t, \ + index_t, \ + time_t +from ..capi.obidms cimport OBIDMS_p +from ..capi.obidmscolumn cimport OBIDMS_column_p, \ + Column_reference_t, \ + Column_reference_p + +from libc.stdint cimport uint8_t + + +cdef extern from "obiview.h" nogil: + + extern const_char_p VIEW_TYPE_NUC_SEQS + extern const_char_p NUC_SEQUENCE_COLUMN + extern const_char_p ID_COLUMN + extern const_char_p DEFINITION_COLUMN + extern const_char_p QUALITY_COLUMN + + + struct Alias_column_pair_t : + Column_reference_t column_refs + const_char_p alias + + ctypedef Alias_column_pair_t* Alias_column_pair_p + + + struct Obiview_infos_t : + time_t creation_date + const_char_p name + const_char_p created_from + const_char_p view_type + bint all_lines + Column_reference_t line_selection + index_t line_count + int column_count + Alias_column_pair_p column_references + const_char_p comments + + ctypedef Obiview_infos_t* Obiview_infos_p + + + struct Obiview_t : + Obiview_infos_p infos + OBIDMS_p dms + bint read_only + OBIDMS_column_p line_selection + OBIDMS_column_p columns + int nb_predicates + # TODO declarations for column dictionary and predicate function array? + + ctypedef Obiview_t* Obiview_p + + + Obiview_p obi_new_view_nuc_seqs(OBIDMS_p dms, const_char_p view_name, Obiview_p view_to_clone, index_t* line_selection, const_char_p comments, bint quality_column) + + Obiview_p obi_new_view(OBIDMS_p dms, const_char_p view_name, Obiview_p view_to_clone, index_t* line_selection, const_char_p comments) + + Obiview_p obi_new_view_cloned_from_name(OBIDMS_p dms, const_char_p view_name, const_char_p view_to_clone_name, index_t* line_selection, const_char_p comments) + + Obiview_p obi_new_view_nuc_seqs_cloned_from_name(OBIDMS_p dms, const_char_p view_name, const_char_p view_to_clone_name, index_t* line_selection, const_char_p comments, bint quality_column) + + Obiview_infos_p obi_view_map_file(OBIDMS_p dms, const char* view_name, bint finished) + + int obi_view_unmap_file(OBIDMS_p dms, Obiview_infos_p view_infos) + + Obiview_p obi_open_view(OBIDMS_p dms, const_char_p view_name) + + int obi_view_add_column(Obiview_p view, + const_char_p column_name, + obiversion_t version_number, + const_char_p alias, + OBIType_t data_type, + index_t nb_lines, + index_t nb_elements_per_line, + char* elements_names, + const_char_p indexer_name, + const_char_p associated_column_name, + obiversion_t associated_column_version, + const_char_p comments, + bint create) + + int obi_view_delete_column(Obiview_p view, const_char_p column_name) + + OBIDMS_column_p obi_view_get_column(Obiview_p view, const_char_p column_name) + + OBIDMS_column_p* obi_view_get_pointer_on_column_in_view(Obiview_p view, const_char_p column_name) + + int obi_view_create_column_alias(Obiview_p view, const_char_p current_name, const_char_p alias) + + int obi_save_and_close_view(Obiview_p view) + + + # OBI_INT + int obi_set_int_with_elt_name_and_col_p_in_view(Obiview_p view, + OBIDMS_column_p column_p, + index_t line_nb, + const_char_p element_name, + obiint_t value) + + int obi_set_int_with_elt_idx_and_col_p_in_view(Obiview_p view, + OBIDMS_column_p column_p, + index_t line_nb, + index_t element_idx, + obiint_t value) + + obiint_t obi_get_int_with_elt_name_and_col_p_in_view(Obiview_p view, + OBIDMS_column_p column_p, + index_t line_nb, + const_char_p element_name) + + obiint_t obi_get_int_with_elt_idx_and_col_p_in_view(Obiview_p view, + OBIDMS_column_p column_p, + index_t line_nb, + index_t element_idx) + + + # OBI_BOOL + int obi_set_bool_with_elt_name_and_col_p_in_view(Obiview_p view, + OBIDMS_column_p column_p, + index_t line_nb, + const_char_p element_name, + obibool_t value) + + int obi_set_bool_with_elt_idx_and_col_p_in_view(Obiview_p view, + OBIDMS_column_p column_p, + index_t line_nb, + index_t element_idx, + obibool_t value) + + obibool_t obi_get_bool_with_elt_name_and_col_p_in_view(Obiview_p view, + OBIDMS_column_p column_p, + index_t line_nb, + const_char_p element_name) + + obibool_t obi_get_bool_with_elt_idx_and_col_p_in_view(Obiview_p view, + OBIDMS_column_p column_p, + index_t line_nb, + index_t element_idx) + + + # OBI_CHAR + int obi_set_char_with_elt_name_and_col_p_in_view(Obiview_p view, + OBIDMS_column_p column_p, + index_t line_nb, + const_char_p element_name, + obichar_t value) + + int obi_set_char_with_elt_idx_and_col_p_in_view(Obiview_p view, + OBIDMS_column_p column_p, + index_t line_nb, + index_t element_idx, + obichar_t value) + + obichar_t obi_get_char_with_elt_name_and_col_p_in_view(Obiview_p view, + OBIDMS_column_p column_p, + index_t line_nb, + const_char_p element_name) + + obichar_t obi_get_char_with_elt_idx_and_col_p_in_view(Obiview_p view, + OBIDMS_column_p column_p, + index_t line_nb, + index_t element_idx) + + + # OBI_FLOAT + int obi_set_float_with_elt_name_and_col_p_in_view(Obiview_p view, + OBIDMS_column_p column_p, + index_t line_nb, + const_char_p element_name, + obifloat_t value) + + int obi_set_float_with_elt_idx_and_col_p_in_view(Obiview_p view, + OBIDMS_column_p column_p, + index_t line_nb, + index_t element_idx, + obifloat_t value) + + obifloat_t obi_get_float_with_elt_name_and_col_p_in_view(Obiview_p view, + OBIDMS_column_p column_p, + index_t line_nb, + const_char_p element_name) + + obifloat_t obi_get_float_with_elt_idx_and_col_p_in_view(Obiview_p view, + OBIDMS_column_p column_p, + index_t line_nb, + index_t element_idx) + + + # OBI_QUAL + int obi_set_qual_char_with_elt_idx_and_col_p_in_view(Obiview_p view, + OBIDMS_column_p column_p, + index_t line_nb, + index_t element_idx, + const char* value) + + int obi_set_qual_int_with_elt_idx_and_col_p_in_view(Obiview_p view, + OBIDMS_column_p column_p, + index_t line_nb, + index_t element_idx, + const uint8_t* value, + int value_length) + + char* obi_get_qual_char_with_elt_idx_and_col_p_in_view(Obiview_p view, + OBIDMS_column_p column_p, + index_t line_nb, + index_t element_idx) + + const uint8_t* obi_get_qual_int_with_elt_idx_and_col_p_in_view(Obiview_p view, + OBIDMS_column_p column_p, + index_t line_nb, + index_t element_idx, + int* value_length) + + int obi_set_qual_char_with_elt_name_and_col_p_in_view(Obiview_p view, + OBIDMS_column_p column_p, + index_t line_nb, + const char* element_name, + const char* value) + + int obi_set_qual_int_with_elt_name_and_col_p_in_view(Obiview_p view, + OBIDMS_column_p column_p, + index_t line_nb, + const char* element_name, + const uint8_t* value, + int value_length) + + char* obi_get_qual_char_with_elt_name_and_col_p_in_view(Obiview_p view, + OBIDMS_column_p column_p, + index_t line_nb, + const char* element_name) + + const uint8_t* obi_get_qual_int_with_elt_name_and_col_p_in_view(Obiview_p view, + OBIDMS_column_p column_p, + index_t line_nb, + const char* element_name, + int* value_length) + + + # OBI_STR + int obi_set_str_with_elt_name_and_col_p_in_view(Obiview_p view, + OBIDMS_column_p column_p, + index_t line_nb, + const_char_p element_name, + const_char_p value) + + int obi_set_str_with_elt_idx_and_col_p_in_view(Obiview_p view, + OBIDMS_column_p column_p, + index_t line_nb, + index_t element_idx, + const_char_p value) + + const_char_p obi_get_str_with_elt_name_and_col_p_in_view(Obiview_p view, + OBIDMS_column_p column_p, + index_t line_nb, + const_char_p element_name) + + const_char_p obi_get_str_with_elt_idx_and_col_p_in_view(Obiview_p view, + OBIDMS_column_p column_p, + index_t line_nb, + index_t element_idx) + + + # OBI_SEQ + int obi_set_seq_with_elt_name_and_col_p_in_view(Obiview_p view, + OBIDMS_column_p column_p, + index_t line_nb, + const_char_p element_name, + const_char_p value) + + int obi_set_seq_with_elt_idx_and_col_p_in_view(Obiview_p view, + OBIDMS_column_p column_p, + index_t line_nb, + index_t element_idx, + const_char_p value) + + char* obi_get_seq_with_elt_name_and_col_p_in_view(Obiview_p view, + OBIDMS_column_p column_p, + index_t line_nb, + const_char_p element_name) + + char* obi_get_seq_with_elt_idx_and_col_p_in_view(Obiview_p view, + OBIDMS_column_p column_p, + index_t line_nb, + index_t element_idx) + diff --git a/python/obitools3/dms/column.pxd b/python/obitools3/dms/column.pxd new file mode 100644 index 0000000..cbdd86b --- /dev/null +++ b/python/obitools3/dms/column.pxd @@ -0,0 +1,25 @@ +#cython: language_level=3 + +from .capi.obitypes cimport index_t, \ + obitype_t +from .capi.obidmscolumn cimport OBIDMS_column_p + +from .view cimport View + +cdef class Column: + + cdef OBIDMS_column_p* _pointer + cdef View _view + + cpdef close(self) + +cdef class Column_line: + + cdef Column _column + cdef index_t _index + + cpdef update(self, data) + +cdef register_column_class(obitype_t obitype, + type classe, + type python) diff --git a/python/obitools3/dms/column.pyx b/python/obitools3/dms/column.pyx new file mode 100644 index 0000000..f849861 --- /dev/null +++ b/python/obitools3/dms/column.pyx @@ -0,0 +1,180 @@ +#cython: language_level=3 + +from .capi.obitypes cimport name_data_type + +from .capi.obidmscolumn cimport OBIDMS_column_header_p, \ + obi_close_column, \ + obi_column_prepare_to_get_value + +from .capi.obiutils cimport obi_format_date + +from .dms cimport __OBIDMS_COLUMN_CLASS__ + +from obitools3.utils cimport bytes2str + + +cdef class Column : + """ + The obitools3.dms.column.Column class wraps a C instance of a column in the context of a View + """ + + # Note: should only be initialized through a subclass + def __init__(self, + View view, + int __internalCall__): + ''' + Creates a new OBDMS column objected referring to a already created column + in the context of a view. + + This constructor is normally only called by subclass constructor. + + @param view: The view object containing the column. + @type view: OBIView + ''' + + cdef OBIDMS_column_p* column_pp + + if __internalCall__!=987654: + raise RuntimeError('OBIView constructor cannot be called directly') + + # Check that the class is only created as a subclass instance + if type(self)==Column or not isinstance(self, Column): + raise RuntimeError('OBIDMS.Column constructor cannot be called directly') + + # Fill structure + self._pointer = NULL + self._view = view + + def __len__(self): + ''' + Implements the len() function for the Column class + + @rtype: `int` + ''' + return self.lines_used + + def __sizeof__(self): + ''' + returns the size of the C object wrapped by the Column instance + ''' + cdef OBIDMS_column_header_p header = self._pointer[0].header + return header.header_size + header.data_size + + def __iter__(self): + cdef index_t line_nb + + for line_nb in range(self.lines_used): + yield self[line_nb] + + def __str__(self) : + cdef str to_print + to_print = '' + for line in self : + to_print = to_print + str(line) + "\n" + return to_print + + def __repr__(self) : + return b"%s, original name: %s, version %d, data type: %d" % ( + self._alias, + self.original_name, + self.version, + self.data_type + ) + + cpdef close(self): + if self._pointer != NULL: + if obi_close_column(self._pointer[0]) < 0 : + raise Exception("Problem closing column %s" % bytes2str(self.name)) + + # Column alias property getter and setter + @property + def name(self): + return self._alias + @name.setter + def name(self, new_alias): # @DuplicatedSignature + self._view.change_column_alias(self._alias, new_alias) + + # elements_names property getter + @property + def elements_names(self): + return (((self._pointer)[0].header).elements_names).split(b';') + + # nb_elements_per_line property getter + @property + def nb_elements_per_line(self): + return ((self._pointer)[0].header).nb_elements_per_line + + # data_type property getter + @property + def data_type(self): + return name_data_type(((self._pointer)[0].header).returned_data_type) + + # original_name property getter + @property + def original_name(self): + return ((self._pointer)[0].header).name + + # version property getter + @property + def version(self): + return ((self._pointer)[0].header).version + + # lines_used property getter + @property + def lines_used(self): + return (self._pointer)[0].header.lines_used + + # comments property getter + @property + def comments(self): + return (self._pointer)[0].header.comments + + # creation_date property getter + @property + def creation_date(self): + return obi_format_date((self._pointer)[0].header.creation_date) + + +###################################################################################################### + + +cdef class Column_line : + + def __init__(self, Column column, index_t line_nb) : + self._index = line_nb + self._column = column + + if obi_column_prepare_to_get_value(self._column._pointer[0],line_nb) < 0: + raise IndexError("Cannot access to the line %d" % line_nb) + + + def __contains__(self, str element_name): + pass + #return (element_name in self._column.elements_names) + + def __repr__(self) : + return str(self._column.get_line(self._index)) + + cpdef update(self, data): + if isinstance(data, dict): + data=data.items() + for key,value in data: + if key in self: + self[key]=value + + +###################################################################################################### + + +cdef register_column_class(obitype_t obitype, + type classe, + type python): + """ + Each sub class of `OBIDMS_column` needs to be registered after its declaration + to declare its relationship with an `OBIType_t` + """ + global __OBIDMS_COLUMN_CLASS__ + + assert issubclass(classe,Column) + + __OBIDMS_COLUMN_CLASS__[obitype]=(classe,python) diff --git a/python/obitools3/dms/column/__init__.py b/python/obitools3/dms/column/__init__.py new file mode 100644 index 0000000..e975aa6 --- /dev/null +++ b/python/obitools3/dms/column/__init__.py @@ -0,0 +1 @@ +from .column import column # @UnresolvedImport \ No newline at end of file diff --git a/python/obitools3/dms/column/bool.pxd b/python/obitools3/dms/column/bool.pxd new file mode 100644 index 0000000..8e5fdc3 --- /dev/null +++ b/python/obitools3/dms/column/bool.pxd @@ -0,0 +1,56 @@ +#cython: language_level=3 + +from cpython.bool cimport bool, PyBool_FromLong + +from ..capi.obitypes cimport index_t, \ + const_char_p, \ + OBIType_t, \ + obibool_t, \ + OBI_BOOL, \ + OBIBool_NA + +from ..capi.obiview cimport obi_get_bool_with_elt_name_and_col_p_in_view, \ + obi_get_bool_with_elt_idx_and_col_p_in_view, \ + obi_set_bool_with_elt_name_and_col_p_in_view, \ + obi_set_bool_with_elt_idx_and_col_p_in_view + +from ..capi.obidmscolumn cimport obi_column_get_obibool_with_elt_name, \ + obi_column_get_obibool_with_elt_idx, \ + obi_column_set_obibool_with_elt_name, \ + obi_column_set_obibool_with_elt_idx, \ + OBIDMS_column_p + +from ..capi.obierrno cimport obi_errno + +from .column cimport Column, \ + Column_line, \ + register_column_class + +from obitools3.utils cimport str2bytes, bytes2str + +cdef class Column_line_bool(OBIDMS_column_line) : + + @staticmethod + cdef bool obibool_t2bool(obibool_t value) + + @staticmethod + cdef bool2obibool_t(bool value) + + cpdef bool get_bool_item_by_name(self,bytes element_name) + cpdef bool get_bool_item_by_idx(self,index_t index) + cpdef set_bool_item_by_name(self,bytes element_name,bool value) + cpdef set_bool_item_by_idx(self,index_t index,bool value) + + +# cdef obibool_t [:] _data_view + +cdef class Column_bool(OBIDMS_column): + cdef Column _new(OBIView view, + bytes column_name, + index_t nb_elements_per_line=1, + object elements_names=None, + bytes comments=b""): + + cpdef object get_line(self, index_t line_nb) + cpdef set_line(self, index_t line_nb, object value) + diff --git a/python/obitools3/dms/column/bool.pyx b/python/obitools3/dms/column/bool.pyx new file mode 100644 index 0000000..5556ff0 --- /dev/null +++ b/python/obitools3/dms/column/bool.pyx @@ -0,0 +1,294 @@ +#cython: language_level=3 + + +cdef class OBIDMS_column_line_bool(OBIDMS_column_line) : + + cdef update_pointer(self): + """ + Checks if the obicolumn address changed since the last call and update + if need the `_column_p` and `_data_view` data structure fields. + """ + cdef OBIDMS_column_p column_p = self._column_pp[0] + + if column_p != self._column_p: + self._column_p = column_p +# self._data_view = ( (column_p.data)) + \ +# self._index * column_p.header.nb_elements_per_line + + @staticmethod + cdef bool obibool_t2bool(obibool_t value): + cdef bool result + + if value == OBIBool_NA : + result = None + else : + result = PyBool_FromLong(value) + + return result + + @staticmethod + cdef bool2obibool_t(bool value): + cdef obibool_t result + + if value is None: + result=OBIBool_NA + else: + result= value + + return result + + + def __init__(self, OBIDMS_column column, index_t line_nb) : + """ + Creates a new `OBIDMS_column_line_bool` + + @param column: an OBIDMS_column instance + @param line_nb: the line in the column + """ + + OBIDMS_column_line.__init__(self,column,line_nb) + self.update_pointer() + + + + cpdef bool get_bool_item_by_name(self,bytes element_name): + """ + Returns the value associated to the name `element_name` of the current line + + @param element_name: a `bytes` instance containing the name of the element + + @return: the `bool` value corresponding to the name + """ + cdef char* cname = element_name + cdef obibool_t value + global obi_errno + + self.update_pointer() + + value = obi_column_get_obibool_with_elt_name(self._column_p, + self._index, + cname) + + if obi_errno > 0 : + obi_errno = 0 + raise KeyError("Cannot access to key %s" % bytes2str(element_name)) + + return OBIDMS_column_line_bool.obibool_t2bool(value) + + cpdef bool get_bool_item_by_idx(self,index_t index): + """ + Returns the value associated to the name `element_name` of the current line + + @param index: a `int` instance containing the index of the element + + @return: the `bool` value corresponding to the name + """ + cdef obibool_t value # @DuplicatedSignature + global obi_errno + + + self.update_pointer() + + value = obi_column_get_obibool_with_elt_idx(self._column_p, + self._index, + index) + + if obi_errno > 0 : + obi_errno = 0 + raise IndexError("Cannot access to element %d" % index) + + return OBIDMS_column_line_bool.obibool_t2bool(value) + + + def __getitem__(self, object element_name) : + cdef bytes name + cdef int cindex + cdef obibool_t value + cdef type typearg = type(element_name) + cdef bool result + + + if typearg == int: + cindex=element_name + if cindex < 0: + cindex = self._len - cindex + result=self.get_bool_item_by_idx(cindex) + elif typearg == bytes: + result=self.get_bool_item_by_name(element_name) + elif typearg == str: + name = str2bytes(element_name) + result=self.get_bool_item_by_name(name) + + return result + + cpdef set_bool_item_by_name(self,bytes element_name,bool value): + """ + Sets the value associated to the name `element_name` of the current line + + @param element_name: a `bytes` instance containing the name of the element + @param value: a `bool` instance of the new value + + @return: the `bool` value corresponding to the name + """ + cdef char* cname = element_name + cdef obibool_t cvalue + + self.update_pointer() + cvalue = OBIDMS_column_line_bool.bool2obibool_t(value) + + if ( obi_column_set_obibool_with_elt_name(self._column_p, + self._index, + cname, + cvalue) < 0 ): + raise KeyError("Cannot access to key %s" % bytes2str(element_name)) + + cpdef set_bool_item_by_idx(self,index_t index,bool value): + """ + Sets the value associated to the name `element_name` of the current line + + @param index: a `int` instance containing the index of the element + @param value: a `bool` instance of the new value + + @return: the `bool` value corresponding to the name + """ + cdef obibool_t cvalue # @DuplicatedSignature + + self.update_pointer() + cvalue = OBIDMS_column_line_bool.bool2obibool_t(value) + + if ( obi_column_set_obibool_with_elt_idx(self._column_p, + self._index, + index, + cvalue) < 0 ): + raise IndexError("Cannot access to item index %d" % index) + + + + def __setitem__(self, object element_name, object value): + cdef bytes name + cdef int cindex + cdef type typearg = type(element_name) + cdef bool result + + + if typearg == int: + cindex=element_name + if cindex < 0: + cindex = self._len - cindex + self.set_bool_item_by_idx(cindex,value) + elif typearg == bytes: + self.set_bool_item_by_name(element_name,value) + elif typearg == str: + name = str2bytes(element_name) + self.set_bool_item_by_name(name,value) + + def __repr__(self) : + return str(self._column.get_line(self._index)) + + def __len__(self): + return self._len + + + +cdef class Column_bool(Column): + + @staticmethod + def new(OBIView view, + object column_name, + index_t nb_elements_per_line=1, + object elements_names=None, + object comments=b""): + + cdef bytes column_name_b = tobytes(column_name) + cdef bytes comments_b + cdef bytes elements_names_b + cdef char* elements_names_p + cdef OBIDMS_column new_column + + if comments is not None: + comments_b = tobytes(comments) + else: + comments_b = b'' + + if elements_names is not None: + elements_names_b = b''.join([tobytes(x) for x in elements_names]) + elements_names_p = elements_names_b + else: + elements_names_p = NULL + + if (obi_view_add_column(view = view._pointer, + column_name = column_name_b, + version_number = -1, + alias = NULL, + data_type = OBI_BOOL, + nb_lines = len(view), + nb_elements_per_line = nb_elements_per_line, + elements_names = elements_names_p, + indexer_name = NULL, + associated_column_name = NULL, + associated_column_version = -1, + comments = comments_b, + create = True)<0): + raise RuntimeError("Cannot create column %s in view %s" % (bytes2str(column_name), + bytes2str(view.name))) + + view.__init_columns__() + new_column = self._columns[column_name] + + return new_column + + def add_to_view(self, + OBIView view, + object column_name=None, + object comments=b""): + + cdef OBIDMS_column_p column_p = self._column_pp[0] + cdef bytes alias + + if (column_name is None): + alias = self._alias + else: + alias = tobytes(column_name) + + if (obi_view_add_column(view = view._ponter, + column_name = column_p.header.name, + version_number = column_p.header.version, + alias = alias, + data_type = OBI_BOOL, + nb_lines = column_p.header.lines_used, + nb_elements_per_line = column_p.header.nb_elements_per_line, + elements_names = column_p.header.elements_names, + indexer_name = NULL, + associated_column_name = NULL, + associated_column_version = NULL, + comments = tobytes(comments), + create = False) < 0): + raise RuntimeError("Cannot insert column %s (%s@%d) into view %s" % + ( bytes2str(alias), + bytes2str(column_p.header.name), + column_p.header.version, + bytes2str(view.name) + )) + + cpdef object get_line(self, index_t line_nb): + cdef obibool_t value + cdef object result + global obi_errno + + value = obi_get_bool_with_elt_idx_and_col_p_in_view(self._view._pointer, (self._pointer)[0], line_nb, 0) + if obi_errno > 0 : + raise IndexError(line_nb) + if value == OBIBool_NA : + result = None + else : + result = PyBool_FromLong(value) + return result + + cpdef set_line(self, index_t line_nb, object value): + if value is None : + value = OBIBool_NA + if obi_set_bool_with_elt_idx_and_col_p_in_view(self._view._pointer, (self._pointer)[0], line_nb, 0, value) < 0: + raise Exception("Problem setting a value in a column") + + +register_column_class(OBI_BOOL,Column_bool,bool) \ No newline at end of file diff --git a/python/obitools3/dms/column/column.pxd b/python/obitools3/dms/column/column.pxd new file mode 100644 index 0000000..4c6047e --- /dev/null +++ b/python/obitools3/dms/column/column.pxd @@ -0,0 +1,26 @@ +#cython: language_level=3 + +from ..capi.obitypes cimport index_t, \ + obitype_t +from ..capi.obidmscolumn cimport OBIDMS_column_p + +from ..view cimport View + +cdef class Column: + + cdef OBIDMS_column_p* _pointer + cdef View _view + + cpdef close(self) + +cdef class Column_line: + + cdef Column _column + cdef index_t _index + + cpdef update(self, data) + +cdef register_column_class(obitype_t obitype, + type classe, + type matrix, + type python) diff --git a/python/obitools3/dms/column/column.pyx b/python/obitools3/dms/column/column.pyx new file mode 100644 index 0000000..dfb57c2 --- /dev/null +++ b/python/obitools3/dms/column/column.pyx @@ -0,0 +1,225 @@ +#cython: language_level=3 + +from .capi.obitypes cimport name_data_type + +from .capi.obidmscolumn cimport OBIDMS_column_header_p, \ + obi_close_column, \ + obi_column_prepare_to_get_value + +from .capi.obiutils cimport obi_format_date + +from .dms cimport __OBIDMS_COLUMN_CLASS__ + +from obitools3.utils cimport bytes2str + + +cdef class Column : + """ + The obitools3.dms.column.Column class wraps a c instance of a column in the context of a View + """ + + # Note: should only be initialized through a subclass + def __init__(self, + View view, + int __internalCall__): + ''' + Create a new OBDMS column objected referring to a already created column + in the context of a view. + + This constructor is normally only called by subclass constructor. + + @param view: The view object containing the column. + @type view: OBIView + ''' + + cdef OBIDMS_column_p* column_pp + + if __internalCall__!=987654: + raise RuntimeError('OBIView constructor cannot be called directly') + + # Check that the class is only created as a subclass instance + if type(self)==Column or not isinstance(self, Column): + raise RuntimeError('OBIDMS.Column constructor cannot be called directly') + + # Fill structure + self._pointer = NULL + self._view = view + + @staticmethod + def new(OBIView view, + object column_name, + index_t nb_elements_per_line=1, + object elements_names=None, + object comments=b""): + + cdef bytes column_name_b = tobytes(column_name) + cdef bytes comments_b + cdef bytes elements_names_b + cdef char* elements_names_p + cdef OBIDMS_column new_column + + if comments is not None: + comments_b = tobytes(comments) + else: + comments_b = b'' + + if elements_names is not None: + elements_names_b = b''.join([tobytes(x) for x in elements_names]) + elements_names_p = elements_names_b + else: + elements_names_p = NULL + + if (obi_view_add_column(view = view._pointer, + column_name = column_name_b, + version_number = -1, + alias = NULL, + data_type = self.pointer.header.returned_data_type, + nb_lines = len(view), + nb_elements_per_line = nb_elements_per_line, + elements_names = elements_names_p, + indexer_name = NULL, + associated_column_name = NULL, + associated_column_version = -1, + comments = comments_b, + create = True)<0): + raise RuntimeError("Cannot create column %s in view %s" % (bytes2str(column_name), + bytes2str(view.name))) + + view.__init_columns__() + new_column = self._columns[column_name] + + return new_column + + def __len__(self): + ''' + implements the len() function for the Column class + + @rtype: `int` + ''' + return self.lines_used + + def __sizeof__(self): + ''' + returns the size of the C object wrapped by the Column instance + ''' + cdef OBIDMS_column_header_p header = self._pointer[0].header + return header.header_size + header.data_size + + def __iter__(self): + cdef index_t line_nb + + for line_nb in range(self.lines_used): + yield self[line_nb] + + def __str__(self) : + cdef str to_print + to_print = '' + for line in self : + to_print = to_print + str(line) + "\n" + return to_print + + def __repr__(self) : + return b"%s, original name: %s, version %d, data type: %d" % ( + self._alias, + self.original_name, + self.version, + self.data_type + ) + + cpdef close(self): + if self._pointer != NULL: + if obi_close_column(self._pointer[0]) < 0 : + raise Exception("Problem closing column %s" % bytes2str(self.name)) + + # Column alias property getter and setter + @property + def name(self): + return self._alias + @name.setter + def name(self, new_alias): # @DuplicatedSignature + self._view.change_column_alias(self._alias, new_alias) + + # elements_names property getter + @property + def elements_names(self): + return (((self._pointer)[0].header).elements_names).split(b';') + + # nb_elements_per_line property getter + @property + def nb_elements_per_line(self): + return ((self._pointer)[0].header).nb_elements_per_line + + # data_type property getter + @property + def data_type(self): + return name_data_type(((self._pointer)[0].header).returned_data_type) + + # original_name property getter + @property + def original_name(self): + return ((self._pointer)[0].header).name + + # version property getter + @property + def version(self): + return ((self._pointer)[0].header).version + + # lines_used property getter + @property + def lines_used(self): + return (self._pointer)[0].header.lines_used + + # comments property getter + @property + def comments(self): + return (self._pointer)[0].header.comments + + # creation_date property getter + @property + def creation_date(self): + return obi_format_date((self._pointer)[0].header.creation_date) + + +###################################################################################################### + + +cdef class Column_line : + + def __init__(self, Column column, index_t line_nb) : + self._index = line_nb + self._column = column + + if obi_column_prepare_to_get_value(self._column._pointer[0],line_nb) < 0: + raise IndexError("Cannot access to the line %d" % line_nb) + + + def __contains__(self, str element_name): + pass + #return (element_name in self._column.elements_names) + + def __repr__(self) : + return str(self._column.get_line(self._index)) + + cpdef update(self, data): + if isinstance(data, dict): + data=data.items() + for key,value in data: + if key in self: + self[key]=value + + +###################################################################################################### + + +cdef register_column_class(obitype_t obitype, + type classe, + type python): + """ + Each sub class of `OBIDMS_column` needs to be registered after its declaration + to declare its relationship with an `OBIType_t` + """ + global __OBIDMS_COLUMN_CLASS__ + + assert issubclass(classe,Column) + + __OBIDMS_COLUMN_CLASS__[obitype]=(classe,python) diff --git a/python/obitools3/dms/dms.cfiles b/python/obitools3/dms/dms.cfiles new file mode 100644 index 0000000..f541dda --- /dev/null +++ b/python/obitools3/dms/dms.cfiles @@ -0,0 +1,32 @@ +../../../src/bloom.c +../../../src/char_str_indexer.c +../../../src/crc64.c +../../../src/dna_seq_indexer.c +../../../src/encode.c +../../../src/hashtable.c +../../../src/murmurhash2.c +../../../src/obi_align.c +../../../src/obiavl.c +../../../src/obiblob_indexer.c +../../../src/obiblob.c +../../../src/obidms_taxonomy.c +../../../src/obidms.c +../../../src/obidmscolumn_blob.c +../../../src/obidmscolumn_bool.c +../../../src/obidmscolumn_char.c +../../../src/obidmscolumn_float.c +../../../src/obidmscolumn_idx.c +../../../src/obidmscolumn_int.c +../../../src/obidmscolumn_qual.c +../../../src/obidmscolumn_seq.c +../../../src/obidmscolumn_str.c +../../../src/obidmscolumn.c +../../../src/obidmscolumndir.c +../../../src/obierrno.c +../../../src/obilittlebigman.c +../../../src/obitypes.c +../../../src/obiview.c +../../../src/sse_banded_LCS_alignment.c +../../../src/uint8_indexer.c +../../../src/upperband.c +../../../src/utils.c diff --git a/python/obitools3/dms/dms.pxd b/python/obitools3/dms/dms.pxd new file mode 100644 index 0000000..e4d589c --- /dev/null +++ b/python/obitools3/dms/dms.pxd @@ -0,0 +1,25 @@ +#cython: language_level=3 + +from .capi.obidms cimport OBIDMS_p +from .capi.obitypes cimport obiversion_t, \ + obitype_t, \ + index_t + +from .object cimport OBIObject + +cdef dict __OBIDMS_COLUMN_CLASS__ +cdef dict __OBIDMS_VIEW_CLASS__ + +cdef class DMS(OBIObject): + + cdef OBIDMS_p _pointer + + @staticmethod + cdef type get_column_class(obitype_t obitype) + + @staticmethod + cdef type get_python_type(obitype_t obitype) + + cpdef close(self) + cpdef int view_count(self) + diff --git a/python/obitools3/dms/dms.pyx b/python/obitools3/dms/dms.pyx new file mode 100644 index 0000000..b64ecc6 --- /dev/null +++ b/python/obitools3/dms/dms.pyx @@ -0,0 +1,152 @@ +#cython: language_level=3 + +from libc.stdlib cimport free +from cpython.list cimport PyList_Size + +from .capi.obidms cimport obi_dms, \ + obi_close_dms, \ + obi_dms_get_full_path + +from .capi.obitypes cimport const_char_p + +from obitools3.utils cimport bytes2str, \ + str2bytes, \ + tobytes, \ + tostr + +from .object cimport OBIObjectClosedInstance + +from pathlib import Path + + +__OBIDMS_COLUMN_CLASS__ = {} +__OBIDMS_VIEW_CLASS__= {} + + +cdef class DMS(OBIObject): + + @staticmethod + cdef type get_column_class(obitype_t obitype): + """ + Internal function returning the python class representing + a column for a given obitype. + """ + return __OBIDMS_COLUMN_CLASS__[obitype][0] + + @staticmethod + cdef type get_python_type(obitype_t obitype): + """ + Internal function returning the python type representing + an instance for a given obitype. + """ + return __OBIDMS_COLUMN_CLASS__[obitype][1] + + @staticmethod + cdef type get_view_class(bytes viewtype): + return __OBIDMS_VIEW_CLASS__[viewtype] + + + def __init__(self, object dms_name) : + ''' + Constructor of a obitools3.dms.DMS instance. + + @param dms_name: The name of the DMS + @type dms_name: a `str` or a `bytes` instance + ''' + + OBIObject.__init__(self) + + # Declarations + cdef bytes dms_name_b = tobytes(dms_name) + + # Fill structure and create or open the DMS + self._pointer = obi_dms( dms_name_b) + + if self._pointer == NULL : + raise Exception("Failed opening or creating an OBIDMS") + + # name property getter + @property + def name(self): + """ + Returns the name of the DMS instance + + @rtype: bytes + """ + return self._pointer.dms_name + + def close(self) : + """ + Closes the DNS instance and free the associated memory + + the `close` method is automatically called by the object destructor. + """ + cdef OBIDMS_p pointer = self._pointer + + OBIObject.close(self) + self._pointer=NULL + + if pointer!=NULL: + if (obi_close_dms(pointer)) < 0 : + raise Exception("Problem closing an OBIDMS") + else: + raise OBIObjectClosedInstance() + + def keys(self): + cdef const_char_p path = obi_dms_get_full_path(self._pointer, + b"VIEWS" + ) + + if path==NULL: + raise RuntimeError("Cannot retreive the Dataabase path") + + p = Path(bytes2str(path)) + + free(path) + + for v in p.glob("*.obiview"): + yield str2bytes(v.stem) + + def values(self): + cdef bytes view_name + + for view_name in self.keys(): + yield self.get_view(view_name) + + def items(self): + cdef bytes view_name + + for view_name in self.keys(): + yield (view_name,self.get_view(view_name)) + + def __contains__(self,key): + cdef str key_s = tostr(key) + cdef const_char_p path = obi_dms_get_full_path(self._pointer, + b"VIEWS" + ) + p = Path(bytes2str(path),key_s) + + free(path) + + return p.with_suffix(".obiview").is_file() + + cpdef int view_count(self): + return PyList_Size(list(self.keys())) + + def get_view(self): + raise NotImplemented + + def __len__(self): + return self.view_count() + + def __getitem__(self, object view_name): + return self.get_view(view_name) + + def __iter__(self): + return self.keys() + + cpdef bint is_view_writable(self, object view_name): + raise NotImplemented + + + \ No newline at end of file diff --git a/python/obitools3/dms/object.pxd b/python/obitools3/dms/object.pxd new file mode 100644 index 0000000..8a34923 --- /dev/null +++ b/python/obitools3/dms/object.pxd @@ -0,0 +1,13 @@ +#cython: language_level=3 + +cdef class OBIObject: + + cdef dict _dependent_object + + cpdef register(self, OBIObject object) + cpdef unregister(self, OBIObject object) + cpdef close(self) + + +cdef class OBIObjectClosedInstance(Exception): + pass diff --git a/python/obitools3/dms/object.pyx b/python/obitools3/dms/object.pyx new file mode 100644 index 0000000..b03eaad --- /dev/null +++ b/python/obitools3/dms/object.pyx @@ -0,0 +1,39 @@ +#cython: language_level=3 + +cdef class OBIObject: + + cpdef register(self, OBIObject object): + self._dependent_object[id(object)]=object + + + cpdef unregister(self, OBIObject object): + del self._dependent_object[id(object)] + + + def close(self): + cdef OBIObject object + cdef list toclose = list(self._dependent_object.values()) + + for object in toclose: + object.close() + + assert len(dependent_object.values)==0 + + + def __init__(self): + self._dependent_object={} + + + def __dealloc__(self): + """ + Destructor of any OBI instance. + + The destructor automatically calls the `close` method and + therefore frees all the associated memory. + """ + + self.close() + + +cdef class OBIObjectClosedInstance(Exception): + pass diff --git a/python/obitools3/dms/view.pxd b/python/obitools3/dms/view.pxd new file mode 100644 index 0000000..59dcff3 --- /dev/null +++ b/python/obitools3/dms/view.pxd @@ -0,0 +1,60 @@ +#cython: language_level=3 + +from .capi.obiview cimport Obiview_p +from .capi.obitypes cimport index_t, \ + obitype_t + +from .dms cimport DMS +from .column cimport Column + + + +cdef class View: + + cdef DMS _dms + cdef Obiview_p _pointer + + cpdef View clone(self, + object view_name, + object comments=*) + + @staticmethod + cdef View _new(DMS dms, + bytes view_name, + bytes comments=*) + + @staticmethod + cdef View _open(DMS dms, + bytes view_name) + + cpdef close(self) + + cpdef get_column(self, + object column_name) + + cpdef delete_column(self, + object column_name) + + cpdef rename_column(self, + object current_name, + object new_name) + + cpdef View_line_selection new_selection(self, + list lines=*) + +cdef class View_line_selection(list): + + cdef View _view + + cdef index_t* __build_binary_list__(self) + + cpdef View materialize(self, + object view_name, + object comments=*) + +cdef class View_line : + + cdef index_t _index + cdef View _view + + diff --git a/python/obitools3/dms/view.pyx b/python/obitools3/dms/view.pyx new file mode 100644 index 0000000..aa860d8 --- /dev/null +++ b/python/obitools3/dms/view.pyx @@ -0,0 +1,417 @@ +#cython: language_level=3 + +from libc.stdlib cimport malloc + +from .capi.obiview cimport obi_new_view, \ + obi_open_view, \ + obi_save_and_close_view, \ + obi_view_get_pointer_on_column_in_view, \ + obi_view_delete_column, \ + obi_view_create_column_alias + +from .capi.obidmscolumn cimport OBIDMS_column_p + +from obitools3.utils cimport tobytes, \ + bytes2str + + +cdef class View : + + + def __init__(self,dms,int __internalCall__): + + if __internalCall__!=987654: + raise RuntimeError('OBIView constructor cannot be called directly') + + self._dms = dms + self._pointer = NULL + + cpdef View clone(self, + object view_name, + object comments=None): + + + cdef bytes view_name_b = tobytes(view_name) + cdef bytes comments_b + cdef View view = View(self._dms, + 987654) + + if comments is not None: + comments_b = tobytes(comments) + else: + comments_b = b'' + + view._pointer = obi_new_view(self._pointer.dms, + view_name_b, + self._pointer, + NULL, + comments_b) + + if view._pointer == NULL : + raise RuntimeError("Error : Cannot clone view %s into view %s" + % (str(self.name), + bytes2str(view_name_b)) + ) + + return view + + @staticmethod + cdef View _new(DMS dms, + bytes view_name, + bytes comments=b""): + + cdef View view = View(dms, + 987654) # @DuplicatedSignature + + view._pointer = obi_new_view(dms._pointer, + view_name, + NULL, + NULL, + comments) + + if view._pointer == NULL : + raise RuntimeError("Error : Cannot create view %s" % view_name) + + return view + + @staticmethod + def new(DMS dms, + object view_name, + object comments=None): + + cdef bytes view_name_b = tobytes(view_name) + cdef bytes comments_b + + if comments is not None: + comments_b = tobytes(comments) + else: + comments_b = b'' + + + return View._new(dms, + view_name_b, + comments_b) + + + @staticmethod + cdef View _open(DMS dms, + bytes view_name): + + cdef View view = View(dms, + 987654) # @DuplicatedSignature + + view._pointer = obi_open_view(dms._pointer, + view_name) + + if view._pointer == NULL : + raise RuntimeError("Error : Cannot open view %s" % bytes2str(view_name)) + + return view + + @staticmethod + def open(DMS dms, # @ReservedAssignment + object view_name): + + cdef bytes view_name_b = tobytes(view_name) + + return View._open(dms,view_name_b) + + cpdef close(self): + if (obi_save_and_close_view(self._pointer) < 0) : + raise Exception("Problem closing view %s" % + bytes2str(self.name)) + + def __dealloc__(self): + self.close() + + def __repr__(self) : + cdef str s = "{name:s}\n{comments:s}\n{line_count:d} lines\n".format(name = str(self.name), + comments = str(self.comments), + line_count = self.line_count) + + #for column_name in self._columns : + # s = s + repr(self._columns[column_name]) + '\n' + + return s + + cpdef get_column(self, + object column_name): + cdef bytes column_name_b = tobytes(column_name) + cdef OBIDMS_column_p* column_pp + cdef OBIDMS_column_p column_p + cdef Column column + cdef obitype_t column_type + + column_pp = obi_view_get_pointer_on_column_in_view(self._pointer, + column_name_b) + + if column_pp == NULL: + raise KeyError("Cannot access to column %s in view %s" % ( + bytes2str(column_name_b), + bytes2str(self.name) + )) + + column_p = column_pp[0] + column_type = column_p.header.returned_data_type + + column = DMS.get_column_class(column_type)(self) + column._pointer = column_pp + + return column + + cpdef delete_column(self, + object column_name) : + + cdef bytes column_name_b = tobytes(column_name) + + if obi_view_delete_column(self._pointer, column_name_b) < 0 : + raise Exception("Problem deleting column %s from a view", + bytes2str(column_name_b)) + + + + cpdef rename_column(self, + object current_name, + object new_name): + + cdef Column column + cdef bytes current_name_b = tobytes(current_name) + cdef bytes new_name_b = tobytes(new_name) + + if (obi_view_create_column_alias(self._pointer, + tobytes(current_name_b), + tobytes(new_name_b)) < 0) : + raise Exception("Problem in renaming column %s to %s" % ( + bytes2str(current_name_b), + bytes2str(new_name_b))) + + + cpdef View_line_selection new_selection(self,list lines=None): + return View_line_selection(self,lines) + + def __iter__(self): + # Iteration on each line of all columns + + # Declarations + cdef index_t line_nb + cdef View_line line + + # Yield each line + for line_nb in range(self.line_count) : + line = self[line_nb] + yield line + + + def __getitem__(self, object item) : + if type(item) == str : + return (self._columns)[item] + elif type(item) == int : + return View_line(self, item) + + + def __contains__(self, str column_name): + return (column_name in self._columns) + + + def __len__(self): + return(self.line_count) + + + def __str__(self) : + cdef View_line line + cdef str to_print + to_print = "" + for line in self : + to_print = to_print + str(line) + "\n" + return to_print + + + @property + def dms(self): + return self._dms + + # line_count property getter + @property + def line_count(self): + return self._pointer.infos.line_count + + # name property getter + @property + def name(self): + return self._pointer.infos.name + + # view type property getter + @property + def type(self): # @ReservedAssignment + return bytes2str(self._pointer.infos.view_type) + + # comments property getter + @property + def comments(self): + return self._pointer.infos.comments + # TODO setter that concatenates new comments? + + +cdef class View_line_selection(list): + + def __init__(self, View view, lines=None) : + if view._pointer == NULL: + raise Exception("Error: trying to create a line selection with an invalidated view") + self._view = view + self._view_name = view.name + + if lines is not None: + self.extend(lines) + + def extend(self, iterable): + cdef index_t i + cdef index_t max_i = self._view.line_count + + for i in iterable: + if i > max_i: + raise RuntimeError("Error: trying to select line %d beyond the line count %d of view %s" % + (i, + max_i, + self._view_name) + ) + list.append(self,i) + + def append(self, index_t idx) : + if idx >= self._view.line_count : + raise IndexError("Error: trying to select line %d beyond the line count %d of view %s" % + (idx, + self._view.line_count, + bytes2str(self.name)) + ) + list.append(self,idx) + + cdef index_t* __build_binary_list__(self): + cdef index_t* line_selection_p = NULL + cdef int i + cdef size_t l_selection = len(self) + + line_selection_p = malloc((l_selection + 1) * sizeof(index_t)) # +1 for the -1 flagging the end of the array + for i in range(l_selection) : + line_selection_p[i] = self[i] + line_selection_p[l_selection] = -1 # flagging the end of the array + + return line_selection_p + + cpdef View materialize(self, + object view_name, + object comments=""): + + cdef View view = View(987654) + cdef bytes view_name_b=tobytes(view_name) + + view._pointer = obi_new_view(self._view._pointer.dms, + view_name_b, + self._view._pointer, + self.__build_binary_list__(), + tobytes(comments)) + + if view._pointer == NULL : + raise RuntimeError("Error : Cannot clone view %s into view %s" + % (str(self.name), + view_name) + ) + + return view + + +cdef class View_line : + + def __init__(self, View view, index_t line_nb) : + self._index = line_nb + self._view = view + + def __getitem__(self, str column_name) : + return ((self._view)._columns)[column_name][self._index] + +# def __setitem__(self, str column_name, object value): +# # TODO detect multiple elements (dict type)? put somewhere else? but more risky (in get) +# # TODO OBI_QUAL ? +# cdef type value_type +# cdef str value_obitype +# cdef bytes value_b +# +# if column_name not in self._view : +# if value == None : +# raise Exception("Trying to create a column from a None value (can't guess type)") +# value_type = type(value) +# if value_type == int : +# value_obitype = 'OBI_INT' +# elif value_type == float : +# value_obitype = 'OBI_FLOAT' +# elif value_type == bool : +# value_obitype = 'OBI_BOOL' +# elif value_type == str or value_type == bytes : +# if value_type == str : +# value_b = str2bytes(value) +# else : +# value_b = value +# if only_ATGC(value_b) : # TODO detect IUPAC +# value_obitype = 'OBI_SEQ' +# elif len(value) == 1 : +# value_obitype = 'OBI_CHAR' +# elif (len(value) > 1) : +# value_obitype = 'OBI_STR' +# else : +# raise Exception("Could not guess the type of a value to create a new column") +# self._view.add_column(column_name, type=value_obitype) +# +# (((self._view)._columns)[column_name]).set_line(self._index, value) +# +# def __iter__(self): +# for column_name in ((self._view)._columns) : +# yield column_name +# +# def __contains__(self, str column_name): +# return (column_name in self._view._columns) + + def __repr__(self): + cdef dict line + cdef str column_name + line = {} +# for column_name in self._view._columns : +# line[column_name] = self[column_name] + return str(line) + + +# cpdef dict get_view_infos(self, str view_name) : +# +# cdef Obiview_infos_p view_infos_p +# cdef dict view_infos_d +# cdef Alias_column_pair_p column_refs +# cdef int i, j +# cdef str column_name +# +# view_infos_p = obi_view_map_file(self._pointer, +# tobytes(view_name)) +# view_infos_d = {} +# view_infos_d["name"] = bytes2str(view_infos_p.name) +# view_infos_d["comments"] = bytes2str(view_infos_p.comments) +# view_infos_d["view_type"] = bytes2str(view_infos_p.view_type) +# view_infos_d["column_count"] = view_infos_p.column_count +# view_infos_d["line_count"] = view_infos_p.line_count +# view_infos_d["created_from"] = bytes2str(view_infos_p.created_from) +# view_infos_d["creation_date"] = bytes2str(obi_format_date(view_infos_p.creation_date)) +# if (view_infos_p.all_lines) : +# view_infos_d["line_selection"] = None +# else : +# view_infos_d["line_selection"] = {} +# view_infos_d["line_selection"]["column_name"] = bytes2str((view_infos_p.line_selection).column_name) +# view_infos_d["line_selection"]["version"] = (view_infos_p.line_selection).version +# view_infos_d["column_references"] = {} +# column_references = view_infos_p.column_references +# for j in range(view_infos_d["column_count"]) : +# column_name = bytes2str((column_references[j]).alias) +# view_infos_d["column_references"][column_name] = {} +# view_infos_d["column_references"][column_name]["original_name"] = bytes2str((column_references[j]).column_refs.column_name) +# view_infos_d["column_references"][column_name]["version"] = (column_references[j]).column_refs.version +# +# obi_view_unmap_file(self._pointer, view_infos_p) +# +# return view_infos_d + + diff --git a/python/obitools3/dms/view/__init__.py b/python/obitools3/dms/view/__init__.py new file mode 100644 index 0000000..154f29e --- /dev/null +++ b/python/obitools3/dms/view/__init__.py @@ -0,0 +1,3 @@ +from .view import View # @UnresolvedImport +from .view import Line_selection # @UnresolvedImport +from .view import Line # @UnresolvedImport \ No newline at end of file diff --git a/python/obitools3/dms/view/view.pxd b/python/obitools3/dms/view/view.pxd new file mode 100644 index 0000000..8963744 --- /dev/null +++ b/python/obitools3/dms/view/view.pxd @@ -0,0 +1,44 @@ +#cython: language_level=3 + +from ..capi.obiview cimport Obiview_p +from ..capi.obitypes cimport index_t, \ + obitype_t + +from ..object cimport OBIObject +from ..dms cimport DMS + +from ..column.column cimport Column + + + +cdef class View(OBIObject): + + cdef DMS _dms + cdef Obiview_p _pointer + + cpdef delete_column(self, + object column_name) + + cpdef rename_column(self, + object current_name, + object new_name) + + cpdef View_line_selection new_selection(self, + list lines=*) + +cdef class Line_selection(list): + + cdef View _view + + cdef index_t* __build_binary_list__(self) + + cpdef View materialize(self, + object view_name, + object comments=*) + +cdef class Line : + + cdef index_t _index + cdef View _view + + diff --git a/python/obitools3/dms/view/view.pyx b/python/obitools3/dms/view/view.pyx new file mode 100644 index 0000000..7715386 --- /dev/null +++ b/python/obitools3/dms/view/view.pyx @@ -0,0 +1,443 @@ +#cython: language_level=3 + +from libc.stdlib cimport malloc + +from ..capi.obiview cimport Alias_column_pair_p, \ + obi_new_view, \ + obi_open_view, \ + obi_save_and_close_view, \ + obi_view_get_pointer_on_column_in_view, \ + obi_view_delete_column, \ + obi_view_create_column_alias + +from ..capi.obidmscolumn cimport OBIDMS_column_p + +from .dms cimport __OBIDMS_VIEW_CLASS__ + +from obitools3.utils cimport tobytes, \ + bytes2str + +from ..object cimport OBIObjectClosedInstance + + +cdef class View(OBIObject) : + + + def __init__(self,dms,int __internalCall__): + + OBIObject.__init__(self) + + if __internalCall__!=987654: + raise RuntimeError('OBIView constructor cannot be called directly') + + self._dms = dms + self._pointer = NULL + + def clone(self, + object view_name, + object comments=None): + + + cdef bytes view_name_b = tobytes(view_name) + cdef bytes comments_b + cdef View view = View(self._dms, + 987654) + + if comments is not None: + comments_b = tobytes(comments) + else: + comments_b = b'' + + view._pointer = obi_new_view(self._pointer.dms, + view_name_b, + self._pointer, + NULL, + comments_b) + + if view._pointer == NULL : + raise RuntimeError("Error : Cannot clone view %s into view %s" + % (str(self.name), + bytes2str(view_name_b)) + ) + + self._dms.register(view) + + return view + + @staticmethod + def new(DMS dms, + object view_name, + object comments=None): + + cdef bytes view_name_b = tobytes(view_name) + cdef bytes comments_b + cdef str message + + cdef View view = View(dms, + 987654) # @DuplicatedSignature + + if comments is not None: + comments_b = tobytes(comments) + else: + comments_b = b'' + + view._pointer = obi_new_view(dms._pointer, + view_name_b, + NULL, + NULL, + comments_b) + + if view._pointer == NULL : + message = "Error : Cannot create view %s" % bytes2str(view_name_b) + raise RuntimeError(message) + + dms.register(view) + + return view + + @staticmethod + def open(DMS dms, # @ReservedAssignment + object view_name): + + cdef bytes view_name_b = tobytes(view_name) + cdef View view = View(dms, + 987654) # @DuplicatedSignature + + view._pointer = obi_open_view(dms._pointer, + view_name_b) + + if view._pointer == NULL : + raise RuntimeError("Error : Cannot open view %s" % bytes2str(view_name_b)) + + dms.register(view) + + return view + + def close(self): + cdef Obiview_p pointer = self._pointer + + if (pointer != NULL): + self._dms.unregister(self) + OBIObject.close(self) + + self._pointer = NULL + + if obi_save_and_close_view(self._pointer) < 0 : + raise Exception("Problem closing view %s" % + bytes2str(self.name)) + else: + raise OBIObjectClosedInstance() + + + def __repr__(self) : + cdef str s = "{name:s}\n{comments:s}\n{line_count:d} lines\n".format(name = str(self.name), + comments = str(self.comments), + line_count = self.line_count) + + #for column_name in self._columns : + # s = s + repr(self._columns[column_name]) + '\n' + + return s + + def keys(self): + cdef int i + cdef int nb_column = self._pointer.infos.column_count + cdef Alias_column_pair_p column_p = self._pointer.infos.column_references + + for i in range(nb_column) : + col_alias = bytes2str(self._pointer.infos.column_references[i].alias) + yield col_alias + + def get_column(self, + object column_name): + cdef bytes column_name_b = tobytes(column_name) + cdef OBIDMS_column_p* column_pp + cdef OBIDMS_column_p column_p + cdef Column column + cdef obitype_t column_type + + column_pp = obi_view_get_pointer_on_column_in_view(self._pointer, + column_name_b) + + if column_pp == NULL: + raise KeyError("Cannot access to column %s in view %s" % ( + bytes2str(column_name_b), + bytes2str(self.name) + )) + + column_p = column_pp[0] + column_type = column_p.header.returned_data_type + + column = DMS.get_column_class(column_type)(self) + column._pointer = column_pp + + return column + + cpdef delete_column(self, + object column_name) : + + cdef bytes column_name_b = tobytes(column_name) + + if obi_view_delete_column(self._pointer, column_name_b) < 0 : + raise Exception("Problem deleting column %s from a view", + bytes2str(column_name_b)) + + + + cpdef rename_column(self, + object current_name, + object new_name): + + cdef Column column + cdef bytes current_name_b = tobytes(current_name) + cdef bytes new_name_b = tobytes(new_name) + + if (obi_view_create_column_alias(self._pointer, + tobytes(current_name_b), + tobytes(new_name_b)) < 0) : + raise Exception("Problem in renaming column %s to %s" % ( + bytes2str(current_name_b), + bytes2str(new_name_b))) + + + cpdef View_line_selection new_selection(self,list lines=None): + return View_line_selection(self,lines) + + def __iter__(self): + # Iteration on each line of all columns + + # Declarations + cdef index_t line_nb + cdef View_line line + + # Yield each line + for line_nb in range(self.line_count) : + line = self[line_nb] + yield line + + + def __getitem__(self, object item) : + if type(item) == str : + return (self._columns)[item] + elif type(item) == int : + return View_line(self, item) + + + def __contains__(self, str column_name): + return (column_name in self._columns) + + + def __len__(self): + return(self.line_count) + + + def __str__(self) : + cdef View_line line + cdef str to_print + to_print = "" + for line in self : + to_print = to_print + str(line) + "\n" + return to_print + + + @property + def dms(self): + return self._dms + + # line_count property getter + @property + def line_count(self): + return self._pointer.infos.line_count + + # name property getter + @property + def name(self): + return self._pointer.infos.name + + # view type property getter + @property + def type(self): # @ReservedAssignment + return self._pointer.infos.view_type + + # comments property getter + @property + def comments(self): + return self._pointer.infos.comments + # TODO setter that concatenates new comments? + + +cdef class Line_selection(list): + + def __init__(self, View view, lines=None) : + if view._pointer == NULL: + raise Exception("Error: trying to create a line selection with an invalidated view") + self._view = view + self._view_name = view.name + + if lines is not None: + self.extend(lines) + + def extend(self, iterable): + cdef index_t i + cdef index_t max_i = self._view.line_count + + for i in iterable: + if i > max_i: + raise RuntimeError("Error: trying to select line %d beyond the line count %d of view %s" % + (i, + max_i, + self._view_name) + ) + list.append(self,i) + + def append(self, index_t idx) : + if idx >= self._view.line_count : + raise IndexError("Error: trying to select line %d beyond the line count %d of view %s" % + (idx, + self._view.line_count, + bytes2str(self.name)) + ) + list.append(self,idx) + + cdef index_t* __build_binary_list__(self): + cdef index_t* line_selection_p = NULL + cdef int i + cdef size_t l_selection = len(self) + + line_selection_p = malloc((l_selection + 1) * sizeof(index_t)) # +1 for the -1 flagging the end of the array + for i in range(l_selection) : + line_selection_p[i] = self[i] + line_selection_p[l_selection] = -1 # flagging the end of the array + + return line_selection_p + + cpdef View materialize(self, + object view_name, + object comments=""): + + cdef View view = View(987654) + cdef bytes view_name_b=tobytes(view_name) + + view._pointer = obi_new_view(self._view._pointer.dms, + view_name_b, + self._view._pointer, + self.__build_binary_list__(), + tobytes(comments)) + + if view._pointer == NULL : + raise RuntimeError("Error : Cannot clone view %s into view %s" + % (str(self.name), + view_name) + ) + + return view + + +cdef class Line : + + def __init__(self, View view, index_t line_nb) : + self._index = line_nb + self._view = view + + def __getitem__(self, str column_name) : + return ((self._view)._columns)[column_name][self._index] + +# def __setitem__(self, str column_name, object value): +# # TODO detect multiple elements (dict type)? put somewhere else? but more risky (in get) +# # TODO OBI_QUAL ? +# cdef type value_type +# cdef str value_obitype +# cdef bytes value_b +# +# if column_name not in self._view : +# if value == None : +# raise Exception("Trying to create a column from a None value (can't guess type)") +# value_type = type(value) +# if value_type == int : +# value_obitype = 'OBI_INT' +# elif value_type == float : +# value_obitype = 'OBI_FLOAT' +# elif value_type == bool : +# value_obitype = 'OBI_BOOL' +# elif value_type == str or value_type == bytes : +# if value_type == str : +# value_b = str2bytes(value) +# else : +# value_b = value +# if only_ATGC(value_b) : # TODO detect IUPAC +# value_obitype = 'OBI_SEQ' +# elif len(value) == 1 : +# value_obitype = 'OBI_CHAR' +# elif (len(value) > 1) : +# value_obitype = 'OBI_STR' +# else : +# raise Exception("Could not guess the type of a value to create a new column") +# self._view.add_column(column_name, type=value_obitype) +# +# (((self._view)._columns)[column_name]).set_line(self._index, value) +# +# def __iter__(self): +# for column_name in ((self._view)._columns) : +# yield column_name +# +# def __contains__(self, str column_name): +# return (column_name in self._view._columns) + + def __repr__(self): + cdef dict line + cdef str column_name + line = {} +# for column_name in self._view._columns : +# line[column_name] = self[column_name] + return str(line) + + +# cpdef dict get_view_infos(self, str view_name) : +# +# cdef Obiview_infos_p view_infos_p +# cdef dict view_infos_d +# cdef Alias_column_pair_p column_refs +# cdef int i, j +# cdef str column_name +# +# view_infos_p = obi_view_map_file(self._pointer, +# tobytes(view_name)) +# view_infos_d = {} +# view_infos_d["name"] = bytes2str(view_infos_p.name) +# view_infos_d["comments"] = bytes2str(view_infos_p.comments) +# view_infos_d["view_type"] = bytes2str(view_infos_p.view_type) +# view_infos_d["column_count"] = view_infos_p.column_count +# view_infos_d["line_count"] = view_infos_p.line_count +# view_infos_d["created_from"] = bytes2str(view_infos_p.created_from) +# view_infos_d["creation_date"] = bytes2str(obi_format_date(view_infos_p.creation_date)) +# if (view_infos_p.all_lines) : +# view_infos_d["line_selection"] = None +# else : +# view_infos_d["line_selection"] = {} +# view_infos_d["line_selection"]["column_name"] = bytes2str((view_infos_p.line_selection).column_name) +# view_infos_d["line_selection"]["version"] = (view_infos_p.line_selection).version +# view_infos_d["column_references"] = {} +# column_references = view_infos_p.column_references +# for j in range(view_infos_d["column_count"]) : +# column_name = bytes2str((column_references[j]).alias) +# view_infos_d["column_references"][column_name] = {} +# view_infos_d["column_references"][column_name]["original_name"] = bytes2str((column_references[j]).column_refs.column_name) +# view_infos_d["column_references"][column_name]["version"] = (column_references[j]).column_refs.version +# +# obi_view_unmap_file(self._pointer, view_infos_p) +# +# return view_infos_d + + +cdef register_view_class(bytes viewtype, + type classe): + """ + Each sub class of `dms.View` needs to be registered after its declaration + to declare its relationship with an `OBIType_t` + """ + global __OBIDMS_VIEW_CLASS__ + + assert issubclass(classe,View) + + __OBIDMS_VIEW_CLASS__[viewtype]=classe + diff --git a/python/obitools3/parsers/fasta.pyx b/python/obitools3/parsers/fasta.pyx index c788414..685b865 100644 --- a/python/obitools3/parsers/fasta.pyx +++ b/python/obitools3/parsers/fasta.pyx @@ -6,8 +6,12 @@ Created on 30 mars 2016 @author: coissac ''' +#from obitools3.dms._obiseq cimport OBI_Seq -def fastaIterator(lineiterator, int buffersize=100000000): + +def fastaIterator(lineiterator, + int buffersize=100000000 + ): cdef LineBuffer lb cdef str ident cdef str definition @@ -15,6 +19,7 @@ def fastaIterator(lineiterator, int buffersize=100000000): cdef list s cdef bytes sequence cdef bytes quality +# cdef OBI_Seq seq if isinstance(lineiterator,(str,bytes)): lineiterator=uopen(lineiterator) @@ -41,7 +46,60 @@ def fastaIterator(lineiterator, int buffersize=100000000): sequence = b"".join(s) quality = None - + + +# seq = OBI_Seq(id, +# sequence, +# definition, +# tags=tags, +# ) + yield { "id" : ident, + "definition" : definition, + "sequence" : sequence, + "quality" : quality, + "tags" : tags, + "annotation" : {} + } + + +def fastaNucIterator(lineiterator, int buffersize=100000000): + cdef LineBuffer lb + cdef str ident + cdef str definition + cdef dict tags + cdef list s + cdef bytes sequence + cdef bytes quality +# cdef OBI_Seq seq + + if isinstance(lineiterator,(str,bytes)): + lineiterator=uopen(lineiterator) + + if isinstance(lineiterator, LineBuffer): + lb=lineiterator + else: + lb=LineBuffer(lineiterator,buffersize) + + i = iter(lb) + line = next(i) + + while True: + ident,tags,definition = parseHeader(line) + s = [] + line = next(i) + + try: + while line[0]!='>': + s.append(str2bytes(line)[0:-1]) + line = next(i) + except StopIteration: + pass + + sequence = b"".join(s) + quality = None + + +# seq = yield { "id" : ident, "definition" : definition, "sequence" : sequence, diff --git a/python/obitools3/utils.pxd b/python/obitools3/utils.pxd index 2a1b144..725a9fe 100644 --- a/python/obitools3/utils.pxd +++ b/python/obitools3/utils.pxd @@ -2,5 +2,6 @@ cdef bytes str2bytes(str string) - -cdef str bytes2str(bytes string) \ No newline at end of file +cdef str bytes2str(bytes string) +cdef bytes tobytes(object string) +cdef str tostr(object string) \ No newline at end of file diff --git a/python/obitools3/utils.pyx b/python/obitools3/utils.pyx index 76c2493..b7e4399 100644 --- a/python/obitools3/utils.pyx +++ b/python/obitools3/utils.pyx @@ -1,6 +1,5 @@ #cython: language_level=3 - cdef bytes str2bytes(str string): """ Short cut to convert ascii encoded python string (str) to bytes @@ -24,3 +23,32 @@ cdef str bytes2str(bytes string): """ return string.decode('ascii') +cdef bytes tobytes(object string): + """ + Short cut to convert ascii encoded string (str or bytes) to bytes + which can be easily converted to C-strings. + + @param string: the python string to be converted. + @type string: bytes or str + @return a transcoded string + @rtype: bytes + """ + if isinstance(string, bytes): + return string + return str2bytes(string) + + +cdef str tostr(object string): + """ + Short cut to convert ascii encoded string (str or bytes) to bytes + which can be easily converted to C-strings. + + @param string: the python string to be converted. + @type string: bytes or str + @return a transcoded string + @rtype: bytes + """ + if isinstance(string, str): + return string + return bytes2str(string) + diff --git a/python/obitools3/version.pyc b/python/obitools3/version.pyc new file mode 100644 index 0000000..03e614f Binary files /dev/null and b/python/obitools3/version.pyc differ diff --git a/src/obidmscolumn.c b/src/obidmscolumn.c index 00fbae1..5ce1f3c 100644 --- a/src/obidmscolumn.c +++ b/src/obidmscolumn.c @@ -865,7 +865,7 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms, obidebug(1, "\nCan't create column because of empty column name"); return NULL; } - if ((data_type < 1) || (data_type > 8)) // TODO check in more robust way + if ((data_type < 1) || (data_type > 8)) // TODO check in more robust way and use macro define somewhere { obi_set_errno(OBICOL_UNKNOWN_ERROR); obidebug(1, "\nCan't create column because of invalid data type"); @@ -891,7 +891,7 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms, // Build the indexer name if needed if ((data_type == OBI_STR) || (data_type == OBI_SEQ) || (data_type == OBI_QUAL)) { - if ((indexer_name == NULL) || (strcmp(indexer_name, "") == 0)) + if ((indexer_name == NULL) || (*indexer_name == 0)) { final_indexer_name = obi_build_indexer_name(column_name, version_number); if (final_indexer_name == NULL) diff --git a/src/obidmscolumn.h b/src/obidmscolumn.h index cef1f59..96f28d8 100644 --- a/src/obidmscolumn.h +++ b/src/obidmscolumn.h @@ -112,18 +112,22 @@ typedef struct OBIDMS_column_header { * creating, opening or cloning an OBIDMS column. */ typedef struct OBIDMS_column { - OBIDMS_p dms; /**< A pointer to the OBIDMS structure to which the column belongs. + OBIDMS_p dms; /**< A pointer to the OBIDMS structure to + * which the column belongs. */ - OBIDMS_column_directory_p column_directory; /**< A pointer to the OBIDMS column directory structure to which the column belongs. + OBIDMS_column_directory_p column_directory; /**< A pointer to the OBIDMS column directory + * structure to which the column belongs. */ OBIDMS_column_header_p header; /**< A pointer to the header of the column. */ - Obi_indexer_p indexer; /**< A pointer to the blob indexer associated with the column if there is one. + Obi_indexer_p indexer; /**< A pointer to the blob indexer associated + * with the column if there is one. */ void* data; /**< A `void` pointer to the beginning of the data. * - * @warning Never use this member directly outside of the code of the - * low level functions of the OBIDMS. + * @warning Never use this member directly outside + * of the code of the low level functions + * of the OBIDMS. */ bool writable; /**< Indicates if the column is writable or not. * - `true` the column is writable @@ -132,7 +136,8 @@ typedef struct OBIDMS_column { * A column is writable only by its creator * until it closes it. */ - size_t counter; /**< Indicates by how many threads/programs (TODO) the column is used. + size_t counter; /**< Indicates by how many threads/programs + * (TODO) the column is used. */ } OBIDMS_column_t, *OBIDMS_column_p; diff --git a/src/obilittlebigman.c b/src/obilittlebigman.c index 3931dd8..c8f6caa 100644 --- a/src/obilittlebigman.c +++ b/src/obilittlebigman.c @@ -12,6 +12,7 @@ #include "obilittlebigman.h" +// TODO this function does not seem to work properly bool obi_is_little_endian() { union { int entier; char caractere[4] ;