obitools/obitools3
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 66 Wiki Activity

Merge branch 'master' into Eric_version_for_sequence

Browse Source 
Conflicts:
	python/obitools3/obidms/_obidmscolumn_seq.pyx
This commit is contained in:
Eric Coissac
2016-08-03 10:09:20 +02:00
parent 448fa8d325 000b9999ad
commit 8f0462c407
31 changed files with 1142 additions and 953 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
python/obitools3/format/fasta.pxd
View File

@ -1,3 +1,4 @@
from ..utils cimport bytes2str
from .header cimport HeaderFormat from .header cimport HeaderFormat
from cython.view cimport array as cvarray from cython.view cimport array as cvarray

55
python/obitools3/format/fasta.pyx
View File

@ -10,38 +10,47 @@ cdef class FastaFormat:
printNAKeys) printNAKeys)
def __cinit(self): # def __cinit(self):
self.sequenceBufferLength=1000 # self.sequenceBufferLength=1000
self.sequenceBuffer = malloc(self.sequenceBufferLength) # self.sequenceBuffer = malloc(self.sequenceBufferLength)
#
def __dealloc__(self) # def __dealloc__(self)
free(self.sequenceBuffer) # free(self.sequenceBuffer)
@cython.boundscheck(False) @cython.boundscheck(False)
def __call__(self, dict data): def __call__(self, dict data):
cdef bytes brawseq = <bytes> data['sequence'] cdef bytes brawseq = data['sequence']
cdef char* crowseq = <char*> brawseq
cdef size_t lseq = len(brawseq) cdef size_t lseq = len(brawseq)
cdef size_t needed_size = lseq + 1
cdef size_t k=0 cdef size_t k=0
cdef list lines = []
needed_size += needed_size/ 50 for k in range(0,lseq,60):
lines.append(brawseq[k:(k+60)])
if needed_size > self.sequenceBufferLength: brawseq = b'\n'.join(lines)
self.sequenceBufferLength=needed_size
self.sequenceBuffer = realloc(self.sequenceBuffer,
self.sequenceBufferLength
)
return "%s\n%s" % (self.headerFormater(data),bytes2str(brawseq))
for i in range(0,lseq,60): # cdef char* crowseq = <char*> brawseq
if i+60 <= lseq: # cdef size_t needed_size = lseq + 1
fasta[k:(k+60)]=seq[i:(i+60)] #
fasta[k+60]='\n' # needed_size += needed_size/ 50
k+=61 #
else: # if needed_size > self.sequenceBufferLength:
fasta[k:(k+lseq-i)]=seq[i:lseq] # self.sequenceBufferLength=needed_size
k+=lseq-i # self.sequenceBuffer = realloc(self.sequenceBuffer,
# self.sequenceBufferLength
# )
#
#
# for i in range(0,lseq,60):
# if i+60 <= lseq:
# fasta[k:(k+60)]=seq[i:(i+60)]
# fasta[k+60]='\n'
# k+=61
# else:
# fasta[k:(k+lseq-i)]=seq[i:lseq]
# k+=lseq-i

2
python/obitools3/format/header.pxd
View File

@ -1,5 +1,7 @@
cdef class HeaderFormat: cdef class HeaderFormat:
cdef str start cdef str start
cdef set tags
cdef bint printNaKeys
cdef size_t headerBufferLength cdef size_t headerBufferLength

43
python/obitools3/format/header.pyx
View File

@ -2,8 +2,19 @@
cdef class HeaderFormat: cdef class HeaderFormat:
def __init__(self, bint fastaHeader=True, list tags=[], bint printNAKeys=False): def __init__(self, bint fastaHeader=True, list tags=[], bint printNAKeys=False):
'''
self.tags = tags @param fastaHeader:
@type fastaHeader: `bool`
@param tags:
@type tags: `list` of `bytes`
@param printNAKeys:
@type printNAKeys: `bool`
'''
self.tags = set(tags)
self.printNaKeys = printNAKeys self.printNaKeys = printNAKeys
if fastaHeader: if fastaHeader:
@ -12,16 +23,38 @@ cdef class HeaderFormat:
self.start="@" self.start="@"
self.headerBufferLength = 1000 self.headerBufferLength = 1000
self.headerBuffer = [] #self.headerBuffer = []
def __call__(self, dict data): def __call__(self, dict data):
cdef str header cdef str header
cdef dict tags = data['tags']
cdef set ktags
cdef list lines = [""]
cdef str tagline
if self.tags is not None and self.tags:
ktags = self.tags
else:
ktags = set(tags.keys())
for k in ktags:
if k in tags:
value = tags[k]
if value is not None or self.printNaKeys:
lines.append("%s=%s;" % (k,tags[k]))
if len(lines) > 1:
tagline=" ".join(lines)
else:
tagline=""
if data['definition'] is not None: if data['definition'] is not None:
header = "%s%s %s" % (self.start,data['id'], header = "%s%s%s %s" % (self.start,data['id'],
data['definition']) tagline,
data['definition'])
else: else:
header = "%s%s" % (self.start,data['id']) header = "%s%s%s" % (self.start,data['id'],
tagline)
return header return header

2
python/obitools3/obidms/_obidms.pxd
View File

@ -91,5 +91,5 @@ cdef class OBIDMS:
cpdef OBIView open_view(self, str view_name) cpdef OBIView open_view(self, str view_name)
cpdef OBIView new_view(self, str view_name, object view_to_clone=*, list line_selection=*, str view_type=*, str comments=*) cpdef OBIView new_view(self, str view_name, object view_to_clone=*, list line_selection=*, str view_type=*, str comments=*)
cpdef dict read_view_infos(self, str view_name) cpdef dict read_view_infos(self, str view_name)
cpdef dict read_views(self) # cpdef dict read_views(self) TODO

165
python/obitools3/obidms/_obidms.pyx
View File

@ -44,7 +44,8 @@ from ._obidmscolumn_bool cimport OBIDMS_column_bool, \
from ._obidmscolumn_char cimport OBIDMS_column_char, \ from ._obidmscolumn_char cimport OBIDMS_column_char, \
OBIDMS_column_multi_elts_char OBIDMS_column_multi_elts_char
from ._obidmscolumn_qual cimport OBIDMS_column_qual from ._obidmscolumn_qual cimport OBIDMS_column_qual, \
OBIDMS_column_multi_elts_qual
from ._obidmscolumn_str cimport OBIDMS_column_str, \ from ._obidmscolumn_str cimport OBIDMS_column_str, \
OBIDMS_column_multi_elts_str OBIDMS_column_multi_elts_str
@ -53,16 +54,15 @@ from ._obidmscolumn_seq cimport OBIDMS_column_seq, \
OBIDMS_column_multi_elts_seq OBIDMS_column_multi_elts_seq
from .capi.obiview cimport Obiview_p, \ from .capi.obiview cimport Obiview_p, \
Obiviews_infos_all_p, \
Obiview_infos_p, \ Obiview_infos_p, \
Column_reference_p, \ Column_reference_p, \
obi_new_view_nuc_seqs, \ obi_new_view_nuc_seqs, \
obi_new_view, \ obi_new_view, \
obi_new_view_cloned_from_name, \ obi_new_view_cloned_from_name, \
obi_new_view_nuc_seqs_cloned_from_name, \ obi_new_view_nuc_seqs_cloned_from_name, \
obi_view_map_file, \
obi_view_unmap_file, \
obi_open_view, \ obi_open_view, \
obi_read_view_infos, \
obi_close_view_infos, \
obi_view_delete_column, \ obi_view_delete_column, \
obi_view_add_column, \ obi_view_add_column, \
obi_view_get_column, \ obi_view_get_column, \
@ -194,8 +194,8 @@ cdef class OBIDMS_column :
elif col_type == OBI_QUAL : elif col_type == OBI_QUAL :
if col_one_element_per_line : if col_one_element_per_line :
subclass = OBIDMS_column_qual subclass = OBIDMS_column_qual
# else : # TODO else :
# subclass = OBIDMS_column_multi_elts_qual subclass = OBIDMS_column_multi_elts_qual
elif col_type == OBI_STR : elif col_type == OBI_STR :
if col_one_element_per_line : if col_one_element_per_line :
subclass = OBIDMS_column_str subclass = OBIDMS_column_str
@ -290,11 +290,11 @@ cdef class OBIView :
raise Exception("Error creating/opening a view") raise Exception("Error creating/opening a view")
self.pointer = view self.pointer = view
self.name = bytes2str(view.name) self.name = bytes2str(view.infos.name)
# Go through columns to build list of corresponding python instances # Go through columns to build list of corresponding python instances
self.columns = {} self.columns = {}
for i in range(view.column_count) : for i in range(view.infos.column_count) :
column_p = <OBIDMS_column_p> (view.columns)[i] column_p = <OBIDMS_column_p> (view.columns)[i]
header = (column_p).header header = (column_p).header
col_name = bytes2str(header.name) col_name = bytes2str(header.name)
@ -304,7 +304,7 @@ cdef class OBIView :
def __repr__(self) : def __repr__(self) :
cdef str s cdef str s
s = str(self.name) + ", " + str(self.comments) + ", " + str(self.pointer.line_count) + " lines\n" s = str(self.name) + ", " + str(self.comments) + ", " + str(self.pointer.infos.line_count) + " lines\n"
for column_name in self.columns : for column_name in self.columns :
s = s + self.columns[column_name].__repr__() + '\n' s = s + self.columns[column_name].__repr__() + '\n'
return s return s
@ -313,7 +313,7 @@ cdef class OBIView :
cpdef delete_column(self, str column_name) : cpdef delete_column(self, str column_name) :
cdef int i cdef int i
cdef Obiview_p view cdef Obiview_p view_p
cdef OBIDMS_column column cdef OBIDMS_column column
cdef OBIDMS_column_p column_p cdef OBIDMS_column_p column_p
cdef OBIDMS_column_header_p header cdef OBIDMS_column_header_p header
@ -321,7 +321,7 @@ cdef class OBIView :
view = self.pointer view = self.pointer
if obi_view_delete_column(view, str2bytes(column_name)) < 0 : if obi_view_delete_column(view_p, str2bytes(column_name)) < 0 :
raise Exception("Problem deleting a column from a view") raise Exception("Problem deleting a column from a view")
# Update the dictionaries of column pointers and column objects, and update pointers in column objects (make function?): # Update the dictionaries of column pointers and column objects, and update pointers in column objects (make function?):
@ -401,7 +401,7 @@ cdef class OBIView :
cdef OBIView_line line # TODO Check that this works for NUC SEQ views cdef OBIView_line line # TODO Check that this works for NUC SEQ views
# Yield each line # Yield each line
lines_used = (self.pointer).line_count lines_used = self.pointer.infos.line_count
for line_nb in range(lines_used) : for line_nb in range(lines_used) :
line = self[line_nb] line = self[line_nb]
@ -485,12 +485,12 @@ cdef class OBIView_NUC_SEQS(OBIView):
raise Exception("Error creating/opening view") raise Exception("Error creating/opening view")
self.pointer = view self.pointer = view
self.name = bytes2str(view.name) self.name = bytes2str(view.infos.name)
self.comments = bytes2str(view.comments) self.comments = bytes2str(view.infos.comments)
# Go through columns to build list of corresponding python instances # Go through columns to build list of corresponding python instances
self.columns = {} self.columns = {}
for i in range(view.column_count) : for i in range(view.infos.column_count) :
column_p = <OBIDMS_column_p> (view.columns)[i] column_p = <OBIDMS_column_p> (view.columns)[i]
header = (column_p).header header = (column_p).header
col_name = bytes2str(header.name) col_name = bytes2str(header.name)
@ -502,35 +502,12 @@ cdef class OBIView_NUC_SEQS(OBIView):
self.definitions = self.columns[bytes2str(DEFINITION_COLUMN)] self.definitions = self.columns[bytes2str(DEFINITION_COLUMN)]
self.qualities = self.columns[bytes2str(QUALITY_COLUMN)] self.qualities = self.columns[bytes2str(QUALITY_COLUMN)]
cpdef delete_column(self, str column_name) :
cdef int i
cdef Obiview_p view_p
cdef OBIDMS_column column
cdef OBIDMS_column_p column_p
cdef OBIDMS_column_header_p header
cdef str column_n
view_p = self.pointer
if obi_view_delete_column(view_p, str2bytes(column_name)) < 0 :
raise Exception("Problem deleting a column from a view")
# Remove instance from the dictionary
(self.columns).pop(column_name)
for column_n in self.columns :
(self.columns[column_n]).update_pointer()
def __getitem__(self, object item) : def __getitem__(self, object item) :
if type(item) == str : if type(item) == str :
return (self.columns)[item] return (self.columns)[item]
elif type(item) == int : elif type(item) == int :
return OBI_Nuc_Seq_Stored(self, item) return OBI_Nuc_Seq_Stored(self, item)
def __setitem__(self, index_t line_idx, OBI_Nuc_Seq sequence_obj) : def __setitem__(self, index_t line_idx, OBI_Nuc_Seq sequence_obj) :
for key in sequence_obj : for key in sequence_obj :
self[line_idx][key] = sequence_obj[key] self[line_idx][key] = sequence_obj[key]
@ -643,54 +620,84 @@ cdef class OBIDMS :
cpdef dict read_view_infos(self, str view_name) : cpdef dict read_view_infos(self, str view_name) :
all_views = self.read_views()
return all_views[view_name]
cdef Obiview_infos_p view_infos_p
cpdef dict read_views(self) : # TODO function that prints the dic nicely and function that prints 1 view nicely. Add column type in col ref cdef dict view_infos_d
cdef Obiviews_infos_all_p all_views_p
cdef Obiview_infos_p view_p
cdef Column_reference_p column_refs cdef Column_reference_p column_refs
cdef int nb_views
cdef int i, j cdef int i, j
cdef str view_name
cdef str column_name cdef str column_name
cdef dict views
cdef bytes name_b
views = {} view_infos_p = obi_view_map_file(self.pointer, str2bytes(view_name))
all_views_p = obi_read_view_infos(self.pointer) view_infos_d = {}
if all_views_p == NULL : view_infos_d["name"] = bytes2str(view_infos_p.name)
raise Exception("No views to read") view_infos_d["comments"] = bytes2str(view_infos_p.comments)
nb_views = <int> (all_views_p.header).view_count view_infos_d["view_type"] = bytes2str(view_infos_p.view_type)
for i in range(nb_views) : view_infos_d["column_count"] = <int> view_infos_p.column_count
view_p = (<Obiview_infos_p> (all_views_p.view_infos)) + i view_infos_d["line_count"] = <int> view_infos_p.line_count
view_name = bytes2str(view_p.name) view_infos_d["created_from"] = bytes2str(view_infos_p.created_from)
views[view_name] = {} view_infos_d["creation_date"] = bytes2str(obi_format_date(view_infos_p.creation_date))
views[view_name]["comments"] = bytes2str(view_p.comments) if (view_infos_p.all_lines) :
views[view_name]["view_type"] = bytes2str(view_p.view_type) view_infos_d["line_selection"] = None
views[view_name]["column_count"] = <int> view_p.column_count else :
views[view_name]["line_count"] = <int> view_p.line_count view_infos_d["line_selection"] = {}
views[view_name]["view_number"] = <int> view_p.view_number view_infos_d["line_selection"]["column_name"] = bytes2str((view_infos_p.line_selection).column_name)
views[view_name]["created_from"] = bytes2str(view_p.created_from) view_infos_d["line_selection"]["version"] = <int> (view_infos_p.line_selection).version
views[view_name]["creation_date"] = bytes2str(obi_format_date(view_p.creation_date)) view_infos_d["column_references"] = {}
if (view_p.all_lines) : column_refs = view_infos_p.column_references
views[view_name]["line_selection"] = None for j in range(view_infos_d["column_count"]) :
else : column_name = bytes2str((column_refs[j]).column_name)
views[view_name]["line_selection"] = {} view_infos_d["column_references"][column_name] = {}
views[view_name]["line_selection"]["column_name"] = bytes2str((view_p.line_selection).column_name) view_infos_d["column_references"][column_name]["version"] = column_refs[j].version
views[view_name]["line_selection"]["version"] = <int> (view_p.line_selection).version
views[view_name]["column_references"] = {}
column_refs = view_p.column_references
for j in range(views[view_name]["column_count"]) :
column_name = bytes2str((column_refs[j]).column_name)
views[view_name]["column_references"][column_name] = {}
views[view_name]["column_references"][column_name]["version"] = column_refs[j].version
obi_close_view_infos(all_views_p); obi_view_unmap_file(self.pointer, view_infos_p)
return views return view_infos_d
# cpdef dict read_views(self) : # TODO function that prints the dic nicely and function that prints 1 view nicely. Add column type in col ref
#
# cdef Obiviews_infos_all_p all_views_p
# cdef Obiview_infos_p view_p
# cdef Column_reference_p column_refs
# cdef int nb_views
# cdef int i, j
# cdef str view_name
# cdef str column_name
# cdef dict views
# cdef bytes name_b
#
# views = {}
# all_views_p = obi_read_view_infos(self.pointer)
# if all_views_p == NULL :
# raise Exception("No views to read")
# nb_views = <int> (all_views_p.header).view_count
# for i in range(nb_views) :
# view_p = (<Obiview_infos_p> (all_views_p.view_infos)) + i
# view_name = bytes2str(view_p.name)
# views[view_name] = {}
# views[view_name]["comments"] = bytes2str(view_p.comments)
# views[view_name]["view_type"] = bytes2str(view_p.view_type)
# views[view_name]["column_count"] = <int> view_p.column_count
# views[view_name]["line_count"] = <int> view_p.line_count
# views[view_name]["view_number"] = <int> view_p.view_number
# views[view_name]["created_from"] = bytes2str(view_p.created_from)
# views[view_name]["creation_date"] = bytes2str(obi_format_date(view_p.creation_date))
# if (view_p.all_lines) :
# views[view_name]["line_selection"] = None
# else :
# views[view_name]["line_selection"] = {}
# views[view_name]["line_selection"]["column_name"] = bytes2str((view_p.line_selection).column_name)
# views[view_name]["line_selection"]["version"] = <int> (view_p.line_selection).version
# views[view_name]["column_references"] = {}
# column_refs = view_p.column_references
# for j in range(views[view_name]["column_count"]) :
# column_name = bytes2str((column_refs[j]).column_name)
# views[view_name]["column_references"][column_name] = {}
# views[view_name]["column_references"][column_name]["version"] = column_refs[j].version
#
# obi_close_view_infos(all_views_p);
#
# return views

13
python/obitools3/obidms/_obidmscolumn_qual.pxd
View File

@ -1,7 +1,7 @@
#cython: language_level=3 #cython: language_level=3
from .capi.obitypes cimport index_t from .capi.obitypes cimport index_t
from ._obidms cimport OBIDMS_column #, OBIDMS_column_multi_elts from ._obidms cimport OBIDMS_column , OBIDMS_column_multi_elts
cdef class OBIDMS_column_qual(OBIDMS_column): cdef class OBIDMS_column_qual(OBIDMS_column):
@ -11,7 +11,10 @@ cdef class OBIDMS_column_qual(OBIDMS_column):
cpdef set_str_line(self, index_t line_nb, object value) cpdef set_str_line(self, index_t line_nb, object value)
# cdef class OBIDMS_column_multi_elts_qual(OBIDMS_column_multi_elts): cdef class OBIDMS_column_multi_elts_qual(OBIDMS_column_multi_elts):
# cpdef object get_item(self, index_t line_nb, str element_name) cpdef object get_item(self, index_t line_nb, str element_name)
# cpdef object get_line(self, index_t line_nb) cpdef object get_str_item(self, index_t line_nb, str element_name)
# cpdef set_item(self, index_t line_nb, str element_name, object value) cpdef object get_line(self, index_t line_nb)
cpdef object get_str_line(self, index_t line_nb)
cpdef set_item(self, index_t line_nb, str element_name, object value)
cpdef set_str_item(self, index_t line_nb, str element_name, object value)

195
python/obitools3/obidms/_obidmscolumn_qual.pyx
View File

@ -32,7 +32,7 @@ cdef class OBIDMS_column_qual(OBIDMS_column):
value = obi_column_get_obiqual_int_with_elt_idx_in_view(self.view.pointer, (self.pointer)[0], line_nb, 0, &value_length) value = obi_column_get_obiqual_int_with_elt_idx_in_view(self.view.pointer, (self.pointer)[0], line_nb, 0, &value_length)
if obi_errno > 0 : if obi_errno > 0 :
raise IndexError(line_nb) raise IndexError(line_nb)
if value == NULL : # TODO if value == OBIQual_int_NA :
result = None result = None
else : else :
result = [] result = []
@ -47,7 +47,7 @@ cdef class OBIDMS_column_qual(OBIDMS_column):
value = obi_column_get_obiqual_char_with_elt_idx_in_view(self.view.pointer, (self.pointer)[0], line_nb, 0) value = obi_column_get_obiqual_char_with_elt_idx_in_view(self.view.pointer, (self.pointer)[0], line_nb, 0)
if obi_errno > 0 : if obi_errno > 0 :
raise IndexError(line_nb) raise IndexError(line_nb)
if strcmp(value, OBIQual_char_NA) == 0 : if value == OBIQual_char_NA :
result = None result = None
else : else :
result = bytes2str(value) result = bytes2str(value)
@ -58,94 +58,127 @@ cdef class OBIDMS_column_qual(OBIDMS_column):
cdef uint8_t* value_b cdef uint8_t* value_b
cdef int value_length cdef int value_length
if value is None : if value is None :
value_b = NULL # TODO if obi_column_set_obiqual_int_with_elt_idx_in_view(self.view.pointer, (self.pointer)[0], line_nb, 0, OBIQual_int_NA, 0) < 0:
value_length = 0 raise Exception("Problem setting a value in a column")
else : else :
value_length = len(value) value_length = len(value)
value_b = <uint8_t*> malloc(value_length * sizeof(uint8_t)) value_b = <uint8_t*> malloc(value_length * sizeof(uint8_t))
for i in range(value_length) : for i in range(value_length) :
value_b[i] = <uint8_t>value[i] value_b[i] = <uint8_t>value[i]
if obi_column_set_obiqual_int_with_elt_idx_in_view(self.view.pointer, (self.pointer)[0], line_nb, 0, value_b, value_length) < 0: if obi_column_set_obiqual_int_with_elt_idx_in_view(self.view.pointer, (self.pointer)[0], line_nb, 0, value_b, value_length) < 0:
raise Exception("Problem setting a value in a column") raise Exception("Problem setting a value in a column")
if value is not None :
free(value_b) free(value_b)
cpdef set_str_line(self, index_t line_nb, object value): cpdef set_str_line(self, index_t line_nb, object value):
cdef bytes value_b
if value is None : if value is None :
value_b = OBIQual_char_NA if obi_column_set_obiqual_char_with_elt_idx_in_view(self.view.pointer, (self.pointer)[0], line_nb, 0, OBIQual_char_NA) < 0:
raise Exception("Problem setting a value in a column")
else : else :
value_b = str2bytes(value) if obi_column_set_obiqual_char_with_elt_idx_in_view(self.view.pointer, (self.pointer)[0], line_nb, 0, str2bytes(value)) < 0:
if obi_column_set_obiqual_char_with_elt_idx_in_view(self.view.pointer, (self.pointer)[0], line_nb, 0, value_b) < 0: raise Exception("Problem setting a value in a column")
raise Exception("Problem setting a value in a column")
# TODO OR NOT? cdef class OBIDMS_column_multi_elts_qual(OBIDMS_column_multi_elts):
# cdef class OBIDMS_column_multi_elts_qual(OBIDMS_column_multi_elts):
# cpdef object get_item(self, index_t line_nb, str element_name):
# cdef const uint8_t* value
# cpdef object get_item(self, index_t line_nb, str element_name): cdef int value_length
# cdef const uint8_t* value cdef object result
# cdef int value_length cdef int i
# cdef object result value = obi_column_get_obiqual_int_with_elt_name_in_view(self.view.pointer, (self.pointer)[0], line_nb, str2bytes(element_name), &value_length)
# cdef int i if obi_errno > 0 :
# value = obi_column_get_obiqual_int_with_elt_name_in_view(self.view.pointer, (self.pointer)[0], line_nb, str2bytes(element_name), &value_length) raise IndexError(line_nb, element_name)
# if obi_errno > 0 : if value == OBIQual_int_NA :
# raise IndexError(line_nb, element_name) result = None
# if value == NULL: # TODO else :
# result = None result = []
# else : for i in range(value_length) :
# result = [] result.append(<int>value[i])
# for i in range(value_length) : return result
# result.append(<int>value[i])
# return result cpdef object get_str_item(self, index_t line_nb, str element_name):
# cdef char* value
# # cpdef object get_str_item(self, index_t line_nb, str element_name): cdef object result
# # pass value = obi_column_get_obiqual_char_with_elt_name_in_view(self.view.pointer, (self.pointer)[0], line_nb, str2bytes(element_name))
# # cdef char* value if obi_errno > 0 :
# # cdef object result raise IndexError(line_nb, element_name)
# # value = obi_column_get_obiseq_with_elt_name_in_view(self.view.pointer, (self.pointer)[0], line_nb, str2bytes(element_name)) if value == OBIQual_char_NA :
# # if obi_errno > 0 : result = None
# # raise IndexError(line_nb, element_name) else :
# # if strcmp(value, OBISeq_NA) == 0 : result = bytes2str(value)
# # result = None free(value)
# # else : return result
# # result = bytes2str(value)
# # free(value) cpdef object get_line(self, index_t line_nb) :
# # return result cdef const uint8_t* value
# cdef int value_length
# cpdef object get_line(self, index_t line_nb) : cdef object value_in_result
# pass cdef dict result
# # cdef char* value cdef index_t i
# # cdef object value_in_result cdef int j
# # cdef dict result cdef bint all_NA
# # cdef index_t i result = {}
# # cdef bint all_NA all_NA = True
# # result = {} for i in range(self.nb_elements_per_line) :
# # all_NA = True value = obi_column_get_obiqual_int_with_elt_idx_in_view(self.view.pointer, (self.pointer)[0], line_nb, i, &value_length)
# # for i in range(self.nb_elements_per_line) : if obi_errno > 0 :
# # value = obi_column_get_obiseq_with_elt_idx_in_view(self.view.pointer, (self.pointer)[0], line_nb, i) raise IndexError(line_nb)
# # if obi_errno > 0 : if value == OBIQual_int_NA :
# # raise IndexError(line_nb) value_in_result = None
# # if strcmp(value, OBISeq_NA) == 0 : else :
# # value_in_result = None value_in_result = []
# # else : for j in range(value_length) :
# # value_in_result = bytes2str(value) value_in_result.append(<int>value[j])
# # free(value) result[self.elements_names[i]] = value_in_result
# # result[self.elements_names[i]] = value_in_result if all_NA and (value_in_result is not None) :
# # if all_NA and (value_in_result is not None) : all_NA = False
# # all_NA = False if all_NA :
# # if all_NA : result = None
# # result = None return result
# # return result
# cpdef object get_str_line(self, index_t line_nb) :
# cpdef set_item(self, index_t line_nb, str element_name, object value): cdef char* value
# pass cdef object value_in_result
# # cdef bytes value_b cdef dict result
# # if value is None : cdef index_t i
# # value_b = OBISeq_NA cdef bint all_NA
# # else : result = {}
# # value_b = str2bytes(value) all_NA = True
# # if obi_column_set_obiseq_with_elt_name_in_view(self.view.pointer, (self.pointer)[0], line_nb, str2bytes(element_name), value_b) < 0: for i in range(self.nb_elements_per_line) :
# # raise Exception("Problem setting a value in a column") value = obi_column_get_obiqual_char_with_elt_idx_in_view(self.view.pointer, (self.pointer)[0], line_nb, i)
# # if obi_errno > 0 :
raise IndexError(line_nb)
if value == OBIQual_char_NA :
value_in_result = None
else :
value_in_result = bytes2str(value)
free(value)
result[self.elements_names[i]] = value_in_result
if all_NA and (value_in_result is not None) :
all_NA = False
if all_NA :
result = None
return result
cpdef set_item(self, index_t line_nb, str element_name, object value):
cdef uint8_t* value_b
cdef int value_length
if value is None :
if obi_column_set_obiqual_int_with_elt_name_in_view(self.view.pointer, (self.pointer)[0], line_nb, str2bytes(element_name), OBIQual_int_NA, 0) < 0:
raise Exception("Problem setting a value in a column")
else :
value_length = len(value)
value_b = <uint8_t*> malloc(value_length * sizeof(uint8_t))
for i in range(value_length) :
value_b[i] = <uint8_t>value[i]
if obi_column_set_obiqual_int_with_elt_name_in_view(self.view.pointer, (self.pointer)[0], line_nb, str2bytes(element_name), value_b, value_length) < 0:
raise Exception("Problem setting a value in a column")
free(value_b)
cpdef set_str_item(self, index_t line_nb, str element_name, object value):
if value is None :
if obi_column_set_obiqual_char_with_elt_name_in_view(self.view.pointer, (self.pointer)[0], line_nb, str2bytes(element_name), OBIQual_char_NA) < 0:
raise Exception("Problem setting a value in a column")
else :
if obi_column_set_obiqual_char_with_elt_name_in_view(self.view.pointer, (self.pointer)[0], line_nb, str2bytes(element_name), str2bytes(value)) < 0:
raise Exception("Problem setting a value in a column")

10
python/obitools3/obidms/_obidmscolumn_seq.pyx
View File

@ -13,7 +13,6 @@ from ._obidms cimport OBIView
from obitools3.utils cimport str2bytes, bytes2str from obitools3.utils cimport str2bytes, bytes2str
from libc.stdlib cimport free from libc.stdlib cimport free
from libc.string cimport strcmp
cdef class OBIDMS_column_seq(OBIDMS_column): cdef class OBIDMS_column_seq(OBIDMS_column):
@ -24,7 +23,7 @@ cdef class OBIDMS_column_seq(OBIDMS_column):
value = obi_column_get_obiseq_with_elt_idx_in_view(self.view.pointer, (self.pointer)[0], line_nb, 0) value = obi_column_get_obiseq_with_elt_idx_in_view(self.view.pointer, (self.pointer)[0], line_nb, 0)
if obi_errno > 0 : if obi_errno > 0 :
raise IndexError(line_nb) raise IndexError(line_nb)
if strcmp(value, OBISeq_NA) == 0 : if value == OBISeq_NA :
result = None result = None
else : else :
try: try:
@ -47,6 +46,9 @@ cdef class OBIDMS_column_seq(OBIDMS_column):
if obi_column_set_obiseq_with_elt_idx_in_view(self.view.pointer, (self.pointer)[0], line_nb, 0, value_b) < 0: if obi_column_set_obiseq_with_elt_idx_in_view(self.view.pointer, (self.pointer)[0], line_nb, 0, value_b) < 0:
raise Exception("Problem setting a value in a column") raise Exception("Problem setting a value in a column")
else :
if obi_column_set_obiseq_with_elt_idx_in_view(self.view.pointer, (self.pointer)[0], line_nb, 0, str2bytes(value)) < 0:
raise Exception("Problem setting a value in a column")
# TODO choose alignment type (lcs or other) with supplementary argument # TODO choose alignment type (lcs or other) with supplementary argument
cpdef align(self, cpdef align(self,
@ -69,7 +71,7 @@ cdef class OBIDMS_column_multi_elts_seq(OBIDMS_column_multi_elts):
value = obi_column_get_obiseq_with_elt_name_in_view(self.view.pointer, (self.pointer)[0], line_nb, str2bytes(element_name)) value = obi_column_get_obiseq_with_elt_name_in_view(self.view.pointer, (self.pointer)[0], line_nb, str2bytes(element_name))
if obi_errno > 0 : if obi_errno > 0 :
raise IndexError(line_nb, element_name) raise IndexError(line_nb, element_name)
if strcmp(value, OBISeq_NA) == 0 : if value == OBISeq_NA :
result = None result = None
else : else :
try: try:
@ -91,7 +93,7 @@ cdef class OBIDMS_column_multi_elts_seq(OBIDMS_column_multi_elts):
value = obi_column_get_obiseq_with_elt_idx_in_view(self.view.pointer, (self.pointer)[0], line_nb, i) value = obi_column_get_obiseq_with_elt_idx_in_view(self.view.pointer, (self.pointer)[0], line_nb, i)
if obi_errno > 0 : if obi_errno > 0 :
raise IndexError(line_nb) raise IndexError(line_nb)
if strcmp(value, OBISeq_NA) == 0 : if value == OBISeq_NA :
value_in_result = None value_in_result = None
else : else :
try: try:

17
python/obitools3/obidms/_obidmscolumn_str.pyx
View File

@ -9,8 +9,6 @@ from .capi.obitypes cimport OBIStr_NA, const_char_p
from obitools3.utils cimport str2bytes, bytes2str from obitools3.utils cimport str2bytes, bytes2str
from libc.string cimport strcmp
cdef class OBIDMS_column_str(OBIDMS_column): cdef class OBIDMS_column_str(OBIDMS_column):
@ -20,7 +18,7 @@ cdef class OBIDMS_column_str(OBIDMS_column):
value = obi_column_get_obistr_with_elt_idx_in_view(self.view.pointer, (self.pointer)[0], line_nb, 0) value = obi_column_get_obistr_with_elt_idx_in_view(self.view.pointer, (self.pointer)[0], line_nb, 0)
if obi_errno > 0 : if obi_errno > 0 :
raise IndexError(line_nb) raise IndexError(line_nb)
if strcmp(value, OBIStr_NA) == 0 : if value == OBIStr_NA :
result = None result = None
else : else :
result = bytes2str(value) result = bytes2str(value)
@ -28,13 +26,12 @@ cdef class OBIDMS_column_str(OBIDMS_column):
return result return result
cpdef set_line(self, index_t line_nb, object value): cpdef set_line(self, index_t line_nb, object value):
cdef bytes value_b
if value is None : if value is None :
value_b = OBIStr_NA if obi_column_set_obistr_with_elt_idx_in_view(self.view.pointer, (self.pointer)[0], line_nb, 0, OBIStr_NA) < 0:
raise Exception("Problem setting a value in a column")
else : else :
value_b = str2bytes(value) if obi_column_set_obistr_with_elt_idx_in_view(self.view.pointer, (self.pointer)[0], line_nb, 0, str2bytes(value)) < 0:
if obi_column_set_obistr_with_elt_idx_in_view(self.view.pointer, (self.pointer)[0], line_nb, 0, value_b) < 0: raise Exception("Problem setting a value in a column")
raise Exception("Problem setting a value in a column")
cdef class OBIDMS_column_multi_elts_str(OBIDMS_column_multi_elts): cdef class OBIDMS_column_multi_elts_str(OBIDMS_column_multi_elts):
@ -45,7 +42,7 @@ cdef class OBIDMS_column_multi_elts_str(OBIDMS_column_multi_elts):
value = obi_column_get_obistr_with_elt_name_in_view(self.view.pointer, (self.pointer)[0], line_nb, str2bytes(element_name)) value = obi_column_get_obistr_with_elt_name_in_view(self.view.pointer, (self.pointer)[0], line_nb, str2bytes(element_name))
if obi_errno > 0 : if obi_errno > 0 :
raise IndexError(line_nb, element_name) raise IndexError(line_nb, element_name)
if strcmp(value, OBIStr_NA) == 0 : if value == OBIStr_NA :
result = None result = None
else : else :
result = bytes2str(value) result = bytes2str(value)
@ -64,7 +61,7 @@ cdef class OBIDMS_column_multi_elts_str(OBIDMS_column_multi_elts):
value = obi_column_get_obistr_with_elt_idx_in_view(self.view.pointer, (self.pointer)[0], line_nb, i) value = obi_column_get_obistr_with_elt_idx_in_view(self.view.pointer, (self.pointer)[0], line_nb, i)
if obi_errno > 0 : if obi_errno > 0 :
raise IndexError(line_nb) raise IndexError(line_nb)
if strcmp(value, OBIStr_NA) == 0 : if value == OBIStr_NA :
value_in_result = None value_in_result = None
else : else :
value_in_result = bytes2str(value) value_in_result = bytes2str(value)

40
python/obitools3/obidms/_obiseq.pxd
View File

@ -4,35 +4,35 @@ from ._obidms cimport OBIView_line
cdef class OBI_Seq(dict) : cdef class OBI_Seq(dict) :
cdef str id cdef object id
cdef str definition cdef object definition
cdef str sequence cdef object sequence
cpdef set_id(self, str id) cpdef set_id(self, object id)
cpdef get_id(self) cpdef object get_id(self)
cpdef set_definition(self, str definition) cpdef set_definition(self, object definition)
cpdef get_definition(self) cpdef object get_definition(self)
cpdef get_sequence(self) cpdef object get_sequence(self)
cdef class OBI_Nuc_Seq(OBI_Seq) : cdef class OBI_Nuc_Seq(OBI_Seq) :
cdef object quality cdef object quality
#cpdef str reverse_complement(self) #cpdef object reverse_complement(self)
cpdef set_sequence(self, str sequence) cpdef set_sequence(self, object sequence)
cpdef set_quality(self, object quality) cpdef set_quality(self, object quality)
cpdef get_quality(self) cpdef object get_quality(self)
cdef class OBI_Nuc_Seq_Stored(OBIView_line) : cdef class OBI_Nuc_Seq_Stored(OBIView_line) :
cpdef set_id(self, str id) cpdef set_id(self, object id)
cpdef get_id(self) cpdef object get_id(self)
cpdef set_definition(self, str definition) cpdef set_definition(self, object definition)
cpdef get_definition(self) cpdef object get_definition(self)
cpdef set_sequence(self, str sequence) cpdef set_sequence(self, object sequence)
cpdef get_sequence(self) cpdef object get_sequence(self)
cpdef set_quality(self, object quality) cpdef set_quality(self, object quality)
cpdef get_quality(self) cpdef object get_quality(self)
cpdef get_str_quality(self) cpdef object get_str_quality(self)
# cpdef str reverse_complement(self) # cpdef object reverse_complement(self)

26
python/obitools3/obidms/_obiseq.pyx
View File

@ -9,20 +9,20 @@ from .capi.obiview cimport NUC_SEQUENCE_COLUMN, \
cdef class OBI_Seq(dict) : cdef class OBI_Seq(dict) :
def __init__(self, str id, str seq, str definition=None) : def __init__(self, object id, object seq, object definition=None) :
self.set_id(id) self.set_id(id)
self.set_sequence(seq) self.set_sequence(seq)
if definition is not None : if definition is not None :
self.set_definition(definition) self.set_definition(definition)
cpdef set_id(self, str id) : cpdef set_id(self, object id) :
self.id = id self.id = id
self[bytes2str(ID_COLUMN)] = id self[bytes2str(ID_COLUMN)] = id
cpdef get_id(self) : cpdef get_id(self) :
return self.id return self.id
cpdef set_definition(self, str definition) : cpdef set_definition(self, object definition) :
self.definition = definition self.definition = definition
self[bytes2str(DEFINITION_COLUMN)] = definition self[bytes2str(DEFINITION_COLUMN)] = definition
@ -38,7 +38,7 @@ cdef class OBI_Seq(dict) :
cdef class OBI_Nuc_Seq(OBI_Seq) : cdef class OBI_Nuc_Seq(OBI_Seq) :
cpdef set_sequence(self, str sequence) : cpdef set_sequence(self, object sequence) :
self.sequence = sequence self.sequence = sequence
self[bytes2str(NUC_SEQUENCE_COLUMN)] = sequence self[bytes2str(NUC_SEQUENCE_COLUMN)] = sequence
@ -57,34 +57,34 @@ cdef class OBI_Nuc_Seq_Stored(OBIView_line) :
# TODO store the str version of column name macros # TODO store the str version of column name macros
cpdef set_id(self, str id) : cpdef set_id(self, object id) :
self[bytes2str(ID_COLUMN)] = id self[bytes2str(ID_COLUMN)] = id
cpdef get_id(self) : cpdef object get_id(self) :
return self[bytes2str(ID_COLUMN)] return self[bytes2str(ID_COLUMN)]
cpdef set_definition(self, str definition) : cpdef set_definition(self, object definition) :
self[bytes2str(DEFINITION_COLUMN)] = definition self[bytes2str(DEFINITION_COLUMN)] = definition
cpdef get_definition(self) : cpdef object get_definition(self) :
return self[bytes2str(DEFINITION_COLUMN)] return self[bytes2str(DEFINITION_COLUMN)]
cpdef set_sequence(self, str sequence) : cpdef set_sequence(self, object sequence) :
self[bytes2str(NUC_SEQUENCE_COLUMN)] = sequence self[bytes2str(NUC_SEQUENCE_COLUMN)] = sequence
cpdef get_sequence(self) : cpdef object get_sequence(self) :
return self[bytes2str(NUC_SEQUENCE_COLUMN)] return self[bytes2str(NUC_SEQUENCE_COLUMN)]
cpdef set_quality(self, object quality) : cpdef set_quality(self, object quality) :
if type(quality) == list : if (type(quality) == list) or (quality is None) :
self[bytes2str(QUALITY_COLUMN)] = quality self[bytes2str(QUALITY_COLUMN)] = quality
else : # Quality is in str form else : # Quality is in str form
(((self.view).columns)[bytes2str(QUALITY_COLUMN)]).set_str_line(self.index, quality) (((self.view).columns)[bytes2str(QUALITY_COLUMN)]).set_str_line(self.index, quality)
cpdef get_quality(self) : cpdef object get_quality(self) :
return self[bytes2str(QUALITY_COLUMN)] return self[bytes2str(QUALITY_COLUMN)]
cpdef get_str_quality(self) : cpdef object get_str_quality(self) :
return ((self.view).columns)[bytes2str(QUALITY_COLUMN)].get_str_line(self.index) return ((self.view).columns)[bytes2str(QUALITY_COLUMN)].get_str_line(self.index)
# def __str__(self) : # def __str__(self) :

5
python/obitools3/obidms/capi/obitypes.pxd
View File

@ -1,7 +1,8 @@
#cython: language_level=3 #cython: language_level=3
from libc.stdint cimport int32_t, int64_t from libc.stdint cimport int32_t, int64_t, uint8_t
from posix.types cimport time_t from posix.types cimport time_t
@ -47,8 +48,8 @@ cdef extern from "obitypes.h" nogil:
extern obibool_t OBIBool_NA extern obibool_t OBIBool_NA
extern const_char_p OBISeq_NA extern const_char_p OBISeq_NA
extern const_char_p OBIStr_NA extern const_char_p OBIStr_NA
extern const_char_p OBIQual_int_NA
extern const_char_p OBIQual_char_NA extern const_char_p OBIQual_char_NA
extern uint8_t* OBIQual_int_NA
const_char_p name_data_type(int data_type) const_char_p name_data_type(int data_type)

44
python/obitools3/obidms/capi/obiview.pxd
View File

@ -23,21 +23,6 @@ cdef extern from "obiview.h" nogil:
extern const_char_p DEFINITION_COLUMN extern const_char_p DEFINITION_COLUMN
extern const_char_p QUALITY_COLUMN extern const_char_p QUALITY_COLUMN
struct Obiview_t :
OBIDMS_p dms
const_char_p name
const_char_p created_from
const_char_p view_type
bint read_only
OBIDMS_column_p line_selection
OBIDMS_column_p new_line_selection
index_t line_count
int column_count
OBIDMS_column_p columns
const_char_p comments
ctypedef Obiview_t* Obiview_p
struct Column_reference_t : struct Column_reference_t :
const_char_p column_name const_char_p column_name
@ -47,7 +32,6 @@ cdef extern from "obiview.h" nogil:
struct Obiview_infos_t : struct Obiview_infos_t :
int view_number
time_t creation_date time_t creation_date
const_char_p name const_char_p name
const_char_p created_from const_char_p created_from
@ -62,19 +46,15 @@ cdef extern from "obiview.h" nogil:
ctypedef Obiview_infos_t* Obiview_infos_p ctypedef Obiview_infos_t* Obiview_infos_p
struct Obiviews_header_t : struct Obiview_t :
size_t header_size Obiview_infos_p infos
size_t views_size OBIDMS_p dms
int view_count bint read_only
OBIDMS_column_p line_selection
OBIDMS_column_p new_line_selection
OBIDMS_column_p columns
ctypedef Obiviews_header_t* Obiviews_header_p ctypedef Obiview_t* Obiview_p
struct Obiviews_infos_all_t :
Obiviews_header_p header
Obiview_infos_p view_infos
ctypedef Obiviews_infos_all_t* Obiviews_infos_all_p
Obiview_p obi_new_view_nuc_seqs(OBIDMS_p dms, const_char_p view_name, Obiview_p view_to_clone, index_t* line_selection, const_char_p comments) Obiview_p obi_new_view_nuc_seqs(OBIDMS_p dms, const_char_p view_name, Obiview_p view_to_clone, index_t* line_selection, const_char_p comments)
@ -85,6 +65,10 @@ cdef extern from "obiview.h" nogil:
Obiview_p obi_new_view_nuc_seqs_cloned_from_name(OBIDMS_p dms, const_char_p view_name, const_char_p view_to_clone_name, index_t* line_selection, const_char_p comments) Obiview_p obi_new_view_nuc_seqs_cloned_from_name(OBIDMS_p dms, const_char_p view_name, const_char_p view_to_clone_name, index_t* line_selection, const_char_p comments)
Obiview_infos_p obi_view_map_file(OBIDMS_p dms, const char* view_name)
int obi_view_unmap_file(OBIDMS_p dms, Obiview_infos_p view_infos)
Obiview_p obi_open_view(OBIDMS_p dms, const_char_p view_name) Obiview_p obi_open_view(OBIDMS_p dms, const_char_p view_name)
int obi_view_add_column(Obiview_p view, int obi_view_add_column(Obiview_p view,
@ -114,10 +98,6 @@ cdef extern from "obiview.h" nogil:
int obi_save_and_close_view(Obiview_p view) int obi_save_and_close_view(Obiview_p view)
Obiviews_infos_all_p obi_read_view_infos(OBIDMS_p dms)
int obi_close_view_infos(Obiviews_infos_all_p views)
int obi_column_set_obiint_with_elt_name_in_view(Obiview_p view, int obi_column_set_obiint_with_elt_name_in_view(Obiview_p view,
OBIDMS_column_p column, OBIDMS_column_p column,
index_t line_nb, index_t line_nb,

1
python/obitools3/parsers/fasta.pxd
View File

@ -1,5 +1,6 @@
#cython: language_level=3 #cython: language_level=3
from ..utils cimport str2bytes
from .header cimport parseHeader from .header cimport parseHeader
from ..files.universalopener cimport uopen from ..files.universalopener cimport uopen
from ..files.linebuffer cimport LineBuffer from ..files.linebuffer cimport LineBuffer

7
python/obitools3/parsers/fasta.pyx
View File

@ -6,12 +6,15 @@ Created on 30 mars 2016
@author: coissac @author: coissac
''' '''
def fastaIterator(lineiterator, int buffersize=100000000): def fastaIterator(lineiterator, int buffersize=100000000):
cdef LineBuffer lb cdef LineBuffer lb
cdef str ident cdef str ident
cdef str definition cdef str definition
cdef dict tags cdef dict tags
cdef list s cdef list s
cdef bytes sequence
cdef bytes quality
if isinstance(lineiterator,(str,bytes)): if isinstance(lineiterator,(str,bytes)):
lineiterator=uopen(lineiterator) lineiterator=uopen(lineiterator)
@ -31,12 +34,12 @@ def fastaIterator(lineiterator, int buffersize=100000000):
try: try:
while line[0]!='>': while line[0]!='>':
s.append(line[0:-1]) s.append(str2bytes(line)[0:-1])
line = next(i) line = next(i)
except StopIteration: except StopIteration:
pass pass
sequence = "".join(s) sequence = b"".join(s)
quality = None quality = None
yield { "id" : ident, yield { "id" : ident,

2
python/obitools3/parsers/fastq.pxd
View File

@ -1,5 +1,7 @@
#cython: language_level=3 #cython: language_level=3
from ..utils cimport str2bytes
from .header cimport parseHeader from .header cimport parseHeader
from ..files.universalopener cimport uopen from ..files.universalopener cimport uopen
from ..files.linebuffer cimport LineBuffer from ..files.linebuffer cimport LineBuffer

10
python/obitools3/parsers/fastq.pyx
View File

@ -6,15 +6,13 @@ Created on 30 mars 2016
@author: coissac @author: coissac
''' '''
def fastqIterator(lineiterator, int buffersize=100000000): def fastqIterator(lineiterator, int buffersize=100000000):
cdef LineBuffer lb cdef LineBuffer lb
cdef str ident cdef str ident
cdef str definition cdef str definition
cdef dict tags cdef dict tags
cdef bytes sequence
cdef bytes quality
if isinstance(lineiterator,(str,bytes)): if isinstance(lineiterator,(str,bytes)):
lineiterator=uopen(lineiterator) lineiterator=uopen(lineiterator)
@ -27,9 +25,9 @@ def fastqIterator(lineiterator, int buffersize=100000000):
i = iter(lb) i = iter(lb)
for line in i: for line in i:
ident,tags,definition = parseHeader(line) ident,tags,definition = parseHeader(line)
sequence = next(i)[0:-1] sequence = str2bytes(next(i)[0:-1])
next(i) next(i)
quality = next(i)[0:-1] quality = str2bytes(next(i)[0:-1])
yield { "id" : ident, yield { "id" : ident,
"definition" : definition, "definition" : definition,

2
src/obi_align.c
View File

@ -25,7 +25,7 @@
// TODO // TODO
// use openMP pragmas : garder scores en memoire et ecrire a la fin? // use openMP pragmas : garder scores en memoire et ecrire a la fin? (normalement c bon avec index)
// tout ecrire en stdint? // tout ecrire en stdint?
// check NUC_SEQS and score type (int or float if normalize) // check NUC_SEQS and score type (int or float if normalize)
// what's with multiple sequence/line columns? // what's with multiple sequence/line columns?

256
src/obiavl.c
View File

@ -107,6 +107,42 @@ static char* build_avl_file_name(const char* avl_name);
static char* build_avl_data_file_name(const char* avl_name); static char* build_avl_data_file_name(const char* avl_name);
/**
* @brief Internal function building the full path of an AVL tree file.
*
* @warning The returned pointer has to be freed by the caller.
*
* @param dms A pointer to the OBIDMS to which the AVL tree belongs.
* @param avl_name The name of the AVL tree.
* @param avl_idx The index of the AVL if it's part of an AVL group, or -1 if not.
*
* @returns A pointer to the full path of the file where the AVL tree is stored.
* @retval NULL if an error occurred.
*
* @since May 2016
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
static char* get_full_path_of_avl_file_name(OBIDMS_p dms, const char* avl_name, int avl_idx);
/**
* @brief Internal function building the file name for an AVL data file.
*
* @warning The returned pointer has to be freed by the caller.
*
* @param dms A pointer to the OBIDMS to which the AVL tree belongs.
* @param avl_name The name of the AVL tree.
* @param avl_idx The index of the AVL if it's part of an AVL group, or -1 if not.
*
* @returns A pointer to the full path of the file where the data referred to by the AVL tree is stored.
* @retval NULL if an error occurred.
*
* @since May 2016
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
static char* get_full_path_of_avl_data_file_name(OBIDMS_p dms, const char* avl_name, int avl_idx);
/** /**
* @brief Internal function returning the size of an AVL tree header on this platform, * @brief Internal function returning the size of an AVL tree header on this platform,
* including the size of the bloom filter associated with the AVL tree. * including the size of the bloom filter associated with the AVL tree.
@ -253,9 +289,12 @@ int remap_an_avl(OBIDMS_avl_p avl);
/** /**
* @brief Internal function (re)mapping the tree and data parts of an AVL tree structure. * @brief Internal function creating and adding a new AVL in an AVL group.
* *
* @param avl A pointer to the AVL tree group structure. * @warning The previous AVL in the list of the group is unmapped,
* if it's not the 1st AVL being added.
*
* @param avl A pointer on the AVL tree group structure.
* *
* @retval 0 if the operation was successfully completed. * @retval 0 if the operation was successfully completed.
* @retval -1 if an error occurred. * @retval -1 if an error occurred.
@ -547,6 +586,102 @@ static char* build_avl_data_file_name(const char* avl_name)
} }
static char* get_full_path_of_avl_file_name(OBIDMS_p dms, const char* avl_name, int avl_idx)
{
char* complete_avl_name;
char* full_path;
char* avl_file_name;
if (avl_idx >= 0)
{
complete_avl_name = build_avl_name_with_idx(avl_name, avl_idx);
if (complete_avl_name == NULL)
return NULL;
}
else
{
complete_avl_name = (char*) malloc((strlen(avl_name)+1)*sizeof(char));
if (complete_avl_name == NULL)
{
obi_set_errno(OBI_MALLOC_ERROR);
obidebug(1, "\nError allocating memory for an AVL name");
return NULL;
}
strcpy(complete_avl_name, avl_name);
}
avl_file_name = build_avl_file_name(complete_avl_name);
if (avl_file_name == NULL)
{
free(complete_avl_name);
return NULL;
}
full_path = get_full_path_of_avl_dir(dms, avl_name);
if (full_path == NULL)
{
free(complete_avl_name);
free(avl_file_name);
return NULL;
}
strcat(full_path, "/");
strcat(full_path, avl_file_name);
free(complete_avl_name);
return full_path;
}
static char* get_full_path_of_avl_data_file_name(OBIDMS_p dms, const char* avl_name, int avl_idx)
{
char* complete_avl_name;
char* full_path;
char* avl_data_file_name;
if (avl_idx >= 0)
{
complete_avl_name = build_avl_name_with_idx(avl_name, avl_idx);
if (complete_avl_name == NULL)
return NULL;
}
else
{
complete_avl_name = (char*) malloc((strlen(avl_name)+1)*sizeof(char));
if (complete_avl_name == NULL)
{
obi_set_errno(OBI_MALLOC_ERROR);
obidebug(1, "\nError allocating memory for an AVL name");
return NULL;
}
strcpy(complete_avl_name, avl_name);
}
avl_data_file_name = build_avl_data_file_name(complete_avl_name);
if (avl_data_file_name == NULL)
{
free(complete_avl_name);
return NULL;
}
full_path = get_full_path_of_avl_dir(dms, avl_name);
if (full_path == NULL)
{
free(complete_avl_name);
free(avl_data_file_name);
return NULL;
}
strcat(full_path, "/");
strcat(full_path, avl_data_file_name);
free(complete_avl_name);
return full_path;
}
size_t get_avl_header_size() size_t get_avl_header_size()
{ {
size_t header_size; size_t header_size;
@ -646,7 +781,6 @@ int truncate_avl_to_size_used(OBIDMS_avl_p avl) // TODO is it necessary to unmap
file_descriptor, file_descriptor,
(avl->header)->header_size (avl->header)->header_size
); );
if (avl->tree == MAP_FAILED) if (avl->tree == MAP_FAILED)
{ {
obi_set_errno(OBI_AVL_ERROR); obi_set_errno(OBI_AVL_ERROR);
@ -930,9 +1064,10 @@ int add_new_avl_in_group(OBIDMS_avl_group_p avl_group)
return -1; return -1;
} }
// Unmap the previous AVL // Unmap the previous AVL if it's not the 1st
if (unmap_an_avl((avl_group->sub_avls)[avl_group->last_avl_idx]) < 0) if (avl_group->last_avl_idx > 0)
return -1; if (unmap_an_avl((avl_group->sub_avls)[avl_group->last_avl_idx]) < 0)
return -1;
// Increment current AVL index // Increment current AVL index
(avl_group->last_avl_idx)++; (avl_group->last_avl_idx)++;
@ -949,6 +1084,36 @@ int add_new_avl_in_group(OBIDMS_avl_group_p avl_group)
} }
// TODO doc
int add_existing_avl_in_group(OBIDMS_avl_group_p avl_group_dest, OBIDMS_avl_group_p avl_group_source, int avl_idx)
{
if (link(get_full_path_of_avl_file_name(avl_group_source->dms, avl_group_source->name, avl_idx), get_full_path_of_avl_file_name(avl_group_dest->dms, avl_group_dest->name, avl_idx)) < 0)
{
obi_set_errno(OBI_AVL_ERROR);
obidebug(1, "\nError creating a hard link to an existing AVL tree file");
return -1;
}
if (link(get_full_path_of_avl_data_file_name(avl_group_source->dms, avl_group_source->name, avl_idx), get_full_path_of_avl_data_file_name(avl_group_dest->dms, avl_group_dest->name, avl_idx)) < 0)
{
obi_set_errno(OBI_AVL_ERROR);
obidebug(1, "\nError creating a hard link to an existing AVL data file");
return -1;
}
// Increment current AVL index
(avl_group_dest->last_avl_idx)++;
// Open AVL for that group TODO ideally not needed, but needed for now
avl_group_dest->sub_avls[avl_group_dest->last_avl_idx] = obi_open_avl(avl_group_source->dms, avl_group_source->name, avl_idx);
if ((avl_group_dest->sub_avls)[avl_group_dest->last_avl_idx] == NULL)
{
obidebug(1, "\nError opening an AVL to add in an AVL group");
return -1;
}
return 0;
}
int maybe_in_avl(OBIDMS_avl_p avl, Obi_blob_p value) int maybe_in_avl(OBIDMS_avl_p avl, Obi_blob_p value)
{ {
return (bloom_check(&((avl->header)->bloom_filter), value, obi_blob_sizeof(value))); return (bloom_check(&((avl->header)->bloom_filter), value, obi_blob_sizeof(value)));
@ -1529,8 +1694,7 @@ OBIDMS_avl_p obi_create_avl(OBIDMS_p dms, const char* avl_name, int avl_idx)
// Bloom filter // Bloom filter
bloom_init(&((avl->header)->bloom_filter), MAX_NODE_COUNT_PER_AVL); bloom_init(&((avl->header)->bloom_filter), MAX_NODE_COUNT_PER_AVL);
if (avl_idx >= 0) free(complete_avl_name);
free(complete_avl_name);
return avl; return avl;
} }
@ -1777,8 +1941,7 @@ OBIDMS_avl_p obi_open_avl(OBIDMS_p dms, const char* avl_name, int avl_idx)
avl->dir_fd = avl_dir_file_descriptor; avl->dir_fd = avl_dir_file_descriptor;
avl->avl_fd = avl_file_descriptor; avl->avl_fd = avl_file_descriptor;
if (avl_idx >= 0) free(complete_avl_name);
free(complete_avl_name);
return avl; return avl;
} }
@ -1806,6 +1969,7 @@ OBIDMS_avl_group_p obi_avl_group(OBIDMS_p dms, const char* avl_name)
OBIDMS_avl_group_p obi_create_avl_group(OBIDMS_p dms, const char* avl_name) OBIDMS_avl_group_p obi_create_avl_group(OBIDMS_p dms, const char* avl_name)
{ {
OBIDMS_avl_group_p avl_group; OBIDMS_avl_group_p avl_group;
char* avl_dir_name;
avl_group = (OBIDMS_avl_group_p) malloc(sizeof(OBIDMS_avl_group_t)); avl_group = (OBIDMS_avl_group_p) malloc(sizeof(OBIDMS_avl_group_t));
if (avl_group == NULL) if (avl_group == NULL)
@ -1815,18 +1979,22 @@ OBIDMS_avl_group_p obi_create_avl_group(OBIDMS_p dms, const char* avl_name)
return NULL; return NULL;
} }
// Create 1st avl avl_group->last_avl_idx = -1;
(avl_group->sub_avls)[0] = obi_create_avl(dms, avl_name, 0); avl_group->dms = dms;
if ((avl_group->sub_avls)[0] == NULL)
{
obidebug(1, "\nError creating the first AVL of an AVL group");
return NULL;
}
avl_group->last_avl_idx = 0;
strcpy(avl_group->name, avl_name); strcpy(avl_group->name, avl_name);
avl_group->dms = dms; // Create the directory for that AVL group
avl_dir_name = get_full_path_of_avl_dir(dms, avl_name);
if (avl_dir_name == NULL)
return NULL;
if (mkdirat(dms->indexer_dir_fd, avl_dir_name, 00777) < 0)
{
obi_set_errno(OBI_AVL_ERROR);
obidebug(1, "\nError creating an AVL directory");
free(avl_dir_name);
return NULL;
}
// Add in the list of open indexers // Add in the list of open indexers
obi_dms_list_indexer(dms, avl_group); obi_dms_list_indexer(dms, avl_group);
@ -1926,20 +2094,23 @@ OBIDMS_avl_group_p obi_clone_avl_group(OBIDMS_avl_group_p avl_group, const char*
// Create the new AVL group // Create the new AVL group
new_avl_group = obi_create_avl_group(avl_group->dms, new_avl_name); new_avl_group = obi_create_avl_group(avl_group->dms, new_avl_name);
// Copy the data from each old AVL to the new ones // Create hard links to all the full AVLs that won't be modified: all but the last one
for (i=0; i<=(avl_group->last_avl_idx); i++) for (i=0; i<(avl_group->last_avl_idx); i++)
{ {
if (i > 0) // Don't need to create the 1st AVL if (add_existing_avl_in_group(new_avl_group, avl_group, i) < 0)
{ return NULL;
if (add_new_avl_in_group(new_avl_group) < 0)
{
obi_close_avl_group(new_avl_group);
return NULL;
}
}
obi_clone_avl((avl_group->sub_avls)[i], (new_avl_group->sub_avls)[i]);
} }
// Create the last AVL to copy data in it
if (add_new_avl_in_group(new_avl_group) < 0)
{
obi_close_avl_group(new_avl_group);
return NULL;
}
// Copy the data from the last AVL to a new one that can be modified
obi_clone_avl((avl_group->sub_avls)[avl_group->last_avl_idx], (new_avl_group->sub_avls)[new_avl_group->last_avl_idx]);
// Close old AVL group // Close old AVL group
if (obi_close_avl_group(avl_group) < 0) if (obi_close_avl_group(avl_group) < 0)
{ {
@ -1959,7 +2130,7 @@ int obi_close_avl(OBIDMS_avl_p avl)
ret_val = truncate_avl_to_size_used(avl); ret_val = truncate_avl_to_size_used(avl);
if (munmap(avl->tree, (((avl->header)->nb_items_max) * sizeof(AVL_node_t))) < 0) if (munmap(avl->tree, (avl->header)->avl_size) < 0)
{ {
obi_set_errno(OBI_AVL_ERROR); obi_set_errno(OBI_AVL_ERROR);
obidebug(1, "\nError munmapping the tree of an AVL tree file"); obidebug(1, "\nError munmapping the tree of an AVL tree file");
@ -1996,9 +2167,17 @@ int obi_close_avl_group(OBIDMS_avl_group_p avl_group)
ret_val = obi_dms_unlist_indexer(avl_group->dms, avl_group); ret_val = obi_dms_unlist_indexer(avl_group->dms, avl_group);
// Close each AVL of the group // Close each AVL of the group
for (i=0; i < (avl_group->last_avl_idx); i++) for (i=0; i <= (avl_group->last_avl_idx); i++)
{
// Remap all but the last AVL (already mapped) before closing to truncate and close properly
if (i < (avl_group->last_avl_idx))
{
if (remap_an_avl((avl_group->sub_avls)[i]) < 0)
ret_val = -1;
}
if (obi_close_avl((avl_group->sub_avls)[i]) < 0) if (obi_close_avl((avl_group->sub_avls)[i]) < 0)
ret_val = -1; ret_val = -1;
}
free(avl_group); free(avl_group);
} }
@ -2207,6 +2386,16 @@ index_t obi_avl_group_add(OBIDMS_avl_group_p avl_group, Obi_blob_p value)
index_t index_with_avl; index_t index_with_avl;
int i; int i;
// Create 1st AVL if group is empty
if (avl_group->last_avl_idx == -1)
{
if (add_new_avl_in_group(avl_group) < 0)
{
obidebug(1, "\nError creating the first AVL of an AVL group");
return -1;
}
}
if (maybe_in_avl((avl_group->sub_avls)[avl_group->last_avl_idx], value)) if (maybe_in_avl((avl_group->sub_avls)[avl_group->last_avl_idx], value))
{ {
index_in_avl = (int32_t) obi_avl_find((avl_group->sub_avls)[avl_group->last_avl_idx], value); index_in_avl = (int32_t) obi_avl_find((avl_group->sub_avls)[avl_group->last_avl_idx], value);
@ -2218,6 +2407,7 @@ index_t obi_avl_group_add(OBIDMS_avl_group_p avl_group, Obi_blob_p value)
return index_with_avl; return index_with_avl;
} }
} }
for (i=0; i < (avl_group->last_avl_idx); i++) for (i=0; i < (avl_group->last_avl_idx); i++)
{ {
if (maybe_in_avl((avl_group->sub_avls)[i], value)) if (maybe_in_avl((avl_group->sub_avls)[i], value))

6
src/obiavl.h
View File

@ -73,7 +73,7 @@ typedef struct AVL_node {
* @brief OBIDMS AVL tree data header structure. * @brief OBIDMS AVL tree data header structure.
*/ */
typedef struct OBIDMS_avl_data_header { typedef struct OBIDMS_avl_data_header {
int header_size; /**< Size of the header in bytes. size_t header_size; /**< Size of the header in bytes.
*/ */
index_t data_size_used; /**< Size of the data used in bytes. index_t data_size_used; /**< Size of the data used in bytes.
*/ */
@ -105,7 +105,7 @@ typedef struct OBIDMS_avl_data {
* @brief OBIDMS AVL tree header structure. * @brief OBIDMS AVL tree header structure.
*/ */
typedef struct OBIDMS_avl_header { typedef struct OBIDMS_avl_header {
int header_size; /**< Size of the header in bytes. size_t header_size; /**< Size of the header in bytes.
*/ */
size_t avl_size; /**< Size of the AVL tree in bytes. size_t avl_size; /**< Size of the AVL tree in bytes.
*/ */
@ -160,7 +160,7 @@ typedef struct OBIDMS_avl {
typedef struct OBIDMS_avl_group { typedef struct OBIDMS_avl_group {
OBIDMS_avl_p sub_avls[MAX_NB_OF_AVLS_IN_GROUP]; /**< Array containing the pointers to the AVL trees of the group. OBIDMS_avl_p sub_avls[MAX_NB_OF_AVLS_IN_GROUP]; /**< Array containing the pointers to the AVL trees of the group.
*/ */
int last_avl_idx; /**< Index in the sub_avls array of the AVL tree currently being filled. int last_avl_idx; /**< Index in the sub_avls array of the AVL tree currently being filled.
*/ */
char name[AVL_MAX_NAME+1]; /**< Base name of the AVL group. The AVL trees in it have names of the form basename_idx. char name[AVL_MAX_NAME+1]; /**< Base name of the AVL group. The AVL trees in it have names of the form basename_idx.
*/ */

43
src/obidms.c
View File

@ -252,7 +252,7 @@ OBIDMS_p obi_create_dms(const char* dms_path)
return NULL; return NULL;
} }
// Get file descriptor of DMS directory to create the indexer directory // Get file descriptor of DMS directory to create other directories
dms_dir = opendir(directory_name); dms_dir = opendir(directory_name);
if (dms_dir == NULL) if (dms_dir == NULL)
{ {
@ -280,6 +280,14 @@ OBIDMS_p obi_create_dms(const char* dms_path)
return NULL; return NULL;
} }
// Create the view directory
if (mkdirat(dms_file_descriptor, VIEW_DIR_NAME, 00777) < 0)
{
obi_set_errno(OBIVIEW_ERROR);
obidebug(1, "\nProblem creating a view directory");
return NULL;
}
// Isolate the dms name // Isolate the dms name
j = 0; j = 0;
for (i=0; i<strlen(dms_path); i++) for (i=0; i<strlen(dms_path); i++)
@ -447,6 +455,30 @@ OBIDMS_p obi_open_dms(const char* dms_path)
return NULL; return NULL;
} }
// Open the view directory
dms->view_directory = opendir_in_dms(dms, VIEW_DIR_NAME);
if (dms->view_directory == NULL)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nError opening the view directory");
closedir(dms->indexer_directory);
closedir(dms->directory);
free(dms);
return NULL;
}
// Store the view directory's file descriptor
dms->view_dir_fd = dirfd(dms->view_directory);
if (dms->view_dir_fd < 0)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nError getting the file descriptor of the view directory");
closedir(dms->view_directory);
closedir(dms->directory);
free(dms);
return NULL;
}
// Initialize the list of opened columns // Initialize the list of opened columns
dms->opened_columns = (Opened_columns_list_p) malloc(sizeof(Opened_columns_list_t)); dms->opened_columns = (Opened_columns_list_p) malloc(sizeof(Opened_columns_list_t));
(dms->opened_columns)->nb_opened_columns = 0; (dms->opened_columns)->nb_opened_columns = 0;
@ -486,7 +518,7 @@ int obi_close_dms(OBIDMS_p dms)
while ((dms->opened_columns)->nb_opened_columns > 0) while ((dms->opened_columns)->nb_opened_columns > 0)
obi_close_column(*((dms->opened_columns)->columns)); obi_close_column(*((dms->opened_columns)->columns));
// Close dms and indexer directories // Close dms, and view and indexer directories
if (closedir(dms->directory) < 0) if (closedir(dms->directory) < 0)
{ {
obi_set_errno(OBIDMS_MEMORY_ERROR); obi_set_errno(OBIDMS_MEMORY_ERROR);
@ -501,6 +533,13 @@ int obi_close_dms(OBIDMS_p dms)
free(dms); free(dms);
return -1; return -1;
} }
if (closedir(dms->view_directory) < 0)
{
obi_set_errno(OBIVIEW_ERROR);
obidebug(1, "\nError closing a view directory");
free(dms);
return -1;
}
free(dms); free(dms);
} }

8
src/obidms.h
View File

@ -30,6 +30,8 @@
*/ */
#define INDEXER_DIR_NAME "OBIBLOB_INDEXERS" /**< The name of the Obiblob indexer directory. #define INDEXER_DIR_NAME "OBIBLOB_INDEXERS" /**< The name of the Obiblob indexer directory.
*/ */
#define VIEW_DIR_NAME "VIEWS" /**< The name of the view directory.
*/
#define TAXONOMY_DIR_NAME "TAXONOMY" /**< The name of the taxonomy directory. #define TAXONOMY_DIR_NAME "TAXONOMY" /**< The name of the taxonomy directory.
*/ */
#define MAX_NB_OPENED_COLUMNS (100) /**< The maximum number of columns open at the same time. #define MAX_NB_OPENED_COLUMNS (100) /**< The maximum number of columns open at the same time.
@ -98,6 +100,12 @@ typedef struct OBIDMS {
int indexer_dir_fd; /**< The file descriptor of the directory entry int indexer_dir_fd; /**< The file descriptor of the directory entry
* usable to refer and scan the indexer directory. * usable to refer and scan the indexer directory.
*/ */
DIR* view_directory; /**< A directory entry usable to
* refer and scan the view directory.
*/
int view_dir_fd; /**< The file descriptor of the directory entry
* usable to refer and scan the view directory.
*/
bool little_endian; /**< Endianness of the database. bool little_endian; /**< Endianness of the database.
*/ */
Opened_columns_list_p opened_columns; /**< List of opened columns. Opened_columns_list_p opened_columns; /**< List of opened columns.

2
src/obidmscolumn.c
View File

@ -940,8 +940,6 @@ OBIDMS_column_p obi_clone_column(OBIDMS_p dms,
size_t line_size; size_t line_size;
index_t i, index; index_t i, index;
obidebug(1, "\nline_selection == NULL = %d\n", (line_selection == NULL));
column_to_clone = obi_open_column(dms, column_name, version_number); column_to_clone = obi_open_column(dms, column_name, version_number);
if (column_to_clone == NULL) if (column_to_clone == NULL)
{ {

69
src/obidmscolumn_qual.c
View File

@ -33,16 +33,22 @@ int obi_column_set_obiqual_char_with_elt_idx(OBIDMS_column_p column, index_t lin
int i; int i;
int ret_value; int ret_value;
int_value_length = strlen(value); // Check NA value
int_value = (uint8_t*) malloc(int_value_length * sizeof(uint8_t)); if (value == OBIQual_char_NA)
{
ret_value = obi_column_set_obiqual_int_with_elt_idx(column, line_nb, element_idx, OBIQual_int_NA, 0);
}
else
{
int_value_length = strlen(value);
int_value = (uint8_t*) malloc(int_value_length * sizeof(uint8_t));
// Convert in uint8_t array to index in that format // Convert in uint8_t array to index in that format
for (i=0; i<int_value_length; i++) for (i=0; i<int_value_length; i++)
int_value[i] = ((uint8_t)(value[i])) - QUALITY_ASCII_BASE; int_value[i] = ((uint8_t)(value[i])) - QUALITY_ASCII_BASE;
ret_value = obi_column_set_obiqual_int_with_elt_idx(column, line_nb, element_idx, int_value, int_value_length);
ret_value = obi_column_set_obiqual_int_with_elt_idx(column, line_nb, element_idx, int_value, int_value_length); free(int_value);
}
free(int_value);
return ret_value; return ret_value;
} }
@ -56,25 +62,38 @@ int obi_column_set_obiqual_int_with_elt_idx(OBIDMS_column_p column, index_t line
if (obi_column_prepare_to_set_value(column, line_nb) < 0) if (obi_column_prepare_to_set_value(column, line_nb) < 0)
return -1; return -1;
// Add the value in the indexer if (value == OBIQual_int_NA)
idx = obi_index_uint8(column->indexer, value, value_length);
if (idx == -1) // An error occurred
{ {
if (obi_errno == OBI_READ_ONLY_INDEXER_ERROR) idx = OBIIdx_NA;
}
else
{
// Add the value in the indexer
idx = obi_index_uint8(column->indexer, value, value_length);
if (idx == -1) // An error occurred
{ {
// If the error is that the indexer is read-only, clone it if (obi_errno == OBI_READ_ONLY_INDEXER_ERROR)
new_indexer_name = obi_build_indexer_name((column->header)->name, (column->header)->version); {
if (new_indexer_name == NULL) // TODO PUT IN A COLUMN FUNCTION
return -1; // If the error is that the indexer is read-only, clone it
column->indexer = obi_clone_indexer(column->indexer, new_indexer_name); // TODO Need to lock this somehow? new_indexer_name = obi_build_indexer_name((column->header)->name, (column->header)->version);
// Add the value in the new indexer if (new_indexer_name == NULL)
idx = obi_index_uint8(column->indexer, value, value_length); return -1;
if (idx == -1) column->indexer = obi_clone_indexer(column->indexer, new_indexer_name); // TODO Need to lock this somehow?
strcpy((column->header)->indexer_name, new_indexer_name);
free(new_indexer_name);
obi_set_errno(0);
// Add the value in the new indexer
idx = obi_index_uint8(column->indexer, value, value_length);
if (idx == -1)
return -1;
}
else
return -1; return -1;
} }
else
return -1;
} }
// Add the value's index in the column // Add the value's index in the column
*(((index_t*) (column->data)) + (line_nb * ((column->header)->nb_elements_per_line)) + element_idx) = idx; *(((index_t*) (column->data)) + (line_nb * ((column->header)->nb_elements_per_line)) + element_idx) = idx;
@ -91,6 +110,10 @@ char* obi_column_get_obiqual_char_with_elt_idx(OBIDMS_column_p column, index_t l
int_value = obi_column_get_obiqual_int_with_elt_idx(column, line_nb, element_idx, &int_value_length); int_value = obi_column_get_obiqual_int_with_elt_idx(column, line_nb, element_idx, &int_value_length);
// Check NA
if (int_value == OBIQual_int_NA)
return OBIQual_char_NA;
value = (char*) malloc((int_value_length + 1) * sizeof(char)); value = (char*) malloc((int_value_length + 1) * sizeof(char));
// Encode int quality to char quality // Encode int quality to char quality

4
src/obidmscolumn_qual.h
View File

@ -86,7 +86,7 @@ int obi_column_set_obiqual_int_with_elt_idx(OBIDMS_column_p column, index_t line
* @param element_idx The index of the element that should be recovered in the line. * @param element_idx The index of the element that should be recovered in the line.
* *
* @returns The recovered value, in the character string format. * @returns The recovered value, in the character string format.
* @retval OBIQual_str_NA the NA value of the type if an error occurred and obi_errno is set. * @retval OBIQual_char_NA the NA value of the type if an error occurred and obi_errno is set.
* *
* @since May 2016 * @since May 2016
* @author Celine Mercier (celine.mercier@metabarcoding.org) * @author Celine Mercier (celine.mercier@metabarcoding.org)
@ -172,7 +172,7 @@ int obi_column_set_obiqual_int_with_elt_name(OBIDMS_column_p column, index_t lin
* @param element_name The name of the element that should be recovered in the line. * @param element_name The name of the element that should be recovered in the line.
* *
* @returns The recovered value, in the character string format. * @returns The recovered value, in the character string format.
* @retval OBIQual_str_NA the NA value of the type if an error occurred and obi_errno is set. * @retval OBIQual_char_NA the NA value of the type if an error occurred and obi_errno is set.
* *
* @since May 2016 * @since May 2016
* @author Celine Mercier (celine.mercier@metabarcoding.org) * @author Celine Mercier (celine.mercier@metabarcoding.org)

41
src/obidmscolumn_seq.c
View File

@ -32,25 +32,38 @@ int obi_column_set_obiseq_with_elt_idx(OBIDMS_column_p column, index_t line_nb,
if (obi_column_prepare_to_set_value(column, line_nb) < 0) if (obi_column_prepare_to_set_value(column, line_nb) < 0)
return -1; return -1;
// Add the value in the indexer if (value == OBISeq_NA)
idx = obi_index_dna_seq(column->indexer, value);
if (idx == -1) // An error occurred
{ {
if (obi_errno == OBI_READ_ONLY_INDEXER_ERROR) idx = OBIIdx_NA;
}
else
{
// Add the value in the indexer
idx = obi_index_dna_seq(column->indexer, value);
if (idx == -1) // An error occurred
{ {
// If the error is that the indexer is read-only, clone it if (obi_errno == OBI_READ_ONLY_INDEXER_ERROR)
new_indexer_name = obi_build_indexer_name((column->header)->name, (column->header)->version); {
if (new_indexer_name == NULL) // TODO PUT IN A COLUMN FUNCTION
return -1; // If the error is that the indexer is read-only, clone it
column->indexer = obi_clone_indexer(column->indexer, new_indexer_name); // TODO Need to lock this somehow? new_indexer_name = obi_build_indexer_name((column->header)->name, (column->header)->version);
// Add the value in the new indexer if (new_indexer_name == NULL)
idx = obi_index_dna_seq(column->indexer, value); return -1;
if (idx == -1) column->indexer = obi_clone_indexer(column->indexer, new_indexer_name); // TODO Need to lock this somehow?
strcpy((column->header)->indexer_name, new_indexer_name);
free(new_indexer_name);
obi_set_errno(0);
// Add the value in the new indexer
idx = obi_index_dna_seq(column->indexer, value);
if (idx == -1)
return -1;
}
else
return -1; return -1;
} }
else
return -1;
} }
// Add the value's index in the column // Add the value's index in the column
*(((index_t*) (column->data)) + (line_nb * ((column->header)->nb_elements_per_line)) + element_idx) = idx; *(((index_t*) (column->data)) + (line_nb * ((column->header)->nb_elements_per_line)) + element_idx) = idx;

42
src/obidmscolumn_str.c
View File

@ -32,24 +32,36 @@ int obi_column_set_obistr_with_elt_idx(OBIDMS_column_p column, index_t line_nb,
if (obi_column_prepare_to_set_value(column, line_nb) < 0) if (obi_column_prepare_to_set_value(column, line_nb) < 0)
return -1; return -1;
// Add the value in the indexer if (value == OBIStr_NA)
idx = obi_index_char_str(column->indexer, value);
if (idx == -1) // An error occurred
{ {
if (obi_errno == OBI_READ_ONLY_INDEXER_ERROR) idx = OBIIdx_NA;
}
else
{
// Add the value in the indexer
idx = obi_index_char_str(column->indexer, value);
if (idx == -1) // An error occurred
{ {
// If the error is that the indexer is read-only, clone it if (obi_errno == OBI_READ_ONLY_INDEXER_ERROR)
new_indexer_name = obi_build_indexer_name((column->header)->name, (column->header)->version); {
if (new_indexer_name == NULL) // TODO PUT IN A COLUMN FUNCTION
return -1; // If the error is that the indexer is read-only, clone it
column->indexer = obi_clone_indexer(column->indexer, new_indexer_name); // TODO Need to lock this somehow? new_indexer_name = obi_build_indexer_name((column->header)->name, (column->header)->version);
// Add the value in the new indexer if (new_indexer_name == NULL)
idx = obi_index_char_str(column->indexer, value); return -1;
if (idx == -1) column->indexer = obi_clone_indexer(column->indexer, new_indexer_name); // TODO Need to lock this somehow?
strcpy((column->header)->indexer_name, new_indexer_name);
free(new_indexer_name);
obi_set_errno(0);
// Add the value in the new indexer
idx = obi_index_char_str(column->indexer, value);
if (idx == -1)
return -1;
}
else
return -1; return -1;
} }
else
return -1;
} }
// Add the value's index in the column // Add the value's index in the column
@ -64,7 +76,7 @@ const char* obi_column_get_obistr_with_elt_idx(OBIDMS_column_p column, index_t l
index_t idx; index_t idx;
if (obi_column_prepare_to_get_value(column, line_nb) < 0) if (obi_column_prepare_to_get_value(column, line_nb) < 0)
return OBISeq_NA; return OBIStr_NA;
idx = *(((index_t*) (column->data)) + (line_nb * ((column->header)->nb_elements_per_line)) + element_idx); idx = *(((index_t*) (column->data)) + (line_nb * ((column->header)->nb_elements_per_line)) + element_idx);

8
src/obitypes.h
View File

@ -22,10 +22,10 @@
#define OBIIdx_NA (INT64_MIN) /**< NA value for indices */ #define OBIIdx_NA (INT64_MIN) /**< NA value for indices */
#define OBIFloat_NA (float_NA()) /**< NA value for the type OBI_FLOAT */ #define OBIFloat_NA (float_NA()) /**< NA value for the type OBI_FLOAT */
#define OBIChar_NA (0) /**< NA value for the type OBI_CHAR */ // TODO not sure about this one as it can be impossible to distinguish from uninitialized values #define OBIChar_NA (0) /**< NA value for the type OBI_CHAR */ // TODO not sure about this one as it can be impossible to distinguish from uninitialized values
#define OBISeq_NA ("\0") /**< NA value for the type OBI_SEQ */ // TODO discuss #define OBISeq_NA (NULL) /**< NA value for the type OBI_SEQ */ // TODO discuss
#define OBIStr_NA ("\0") /**< NA value for the type OBI_STR */ // TODO discuss #define OBIStr_NA (NULL) /**< NA value for the type OBI_STR */ // TODO discuss
#define OBIQual_char_NA ("\0") /**< NA value for the type OBI_QUAL if the quality is in character string format */ // TODO test and discuss #define OBIQual_char_NA (NULL) /**< NA value for the type OBI_QUAL if the quality is in character string format */
#define OBIQual_int_NA (NULL) /**< NA value for the type OBI_QUAL if the quality is in integer format */ // TODO test and discuss #define OBIQual_int_NA (NULL) /**< NA value for the type OBI_QUAL if the quality is in integer format */
/** /**

862
src/obiview.c
View File

File diff suppressed because it is too large Load Diff

102
src/obiview.h
View File

@ -34,8 +34,6 @@
*/ */
#define VIEW_TYPE_MAX_LENGTH (1024) /**< The maximum length of the type name of a view. #define VIEW_TYPE_MAX_LENGTH (1024) /**< The maximum length of the type name of a view.
*/ */
#define OBIVIEW_FILE_NAME "obiviews" /**< The default name of a view file.
*/
#define LINES_COLUMN_NAME "LINES" /**< The name of the column containing the line selections #define LINES_COLUMN_NAME "LINES" /**< The name of the column containing the line selections
* in all views. * in all views.
*/ */
@ -56,45 +54,6 @@
*/ */
/**
* @brief Structure for an opened view.
*/
typedef struct Obiview {
OBIDMS_p dms; /**< A pointer on the DMS to which the view belongs.
*/
char name[OBIVIEW_NAME_MAX_LENGTH+1]; /**< Name of the view.
*/
char created_from[OBIVIEW_NAME_MAX_LENGTH+1]; /**< Name of the view from which that view was cloned if the view was cloned.
*/
char view_type[VIEW_TYPE_MAX_LENGTH+1]; /**< Type of the view if there is one.
* Types existing: NUC_SEQS_VIEW.
*/
bool read_only; /**< Whether the view is read-only or can be modified.
*/
OBIDMS_column_p line_selection; /**< A pointer on the column containing the line selection
* associated with the view if there is one.
* This line selection is read-only, and when a line from the view is read,
* it is this line selection that is used.
*/
OBIDMS_column_p new_line_selection; /**< A pointer on the column containing the new line selection being built
* to associate with the view, if there is one.
* When a line is selected with obi_select_line() or obi_select_lines(),
* it is recorded in this line selection.
*/
index_t line_count; /**< The number of lines in the view. Refers to the number of lines in each
* column of the view if line_selection is NULL, or to the line count of
* line_selection if it is not NULL.
*/
int column_count; /**< The number of columns in the view.
*/
OBIDMS_column_p columns[MAX_NB_OPENED_COLUMNS]; /**< Array of pointers on all the columns of the view.
*/
char comments[OBIVIEW_COMMENTS_MAX_LENGTH+1]; /**< Comments, additional informations on the view.
*/
} Obiview_t, *Obiview_p;
/** /**
* @brief Structure referencing a column by its name and its version. * @brief Structure referencing a column by its name and its version.
*/ */
@ -112,8 +71,6 @@ typedef struct Column_reference {
* Once a view has been written in the view file, it can not be modified and can only be read. * Once a view has been written in the view file, it can not be modified and can only be read.
*/ */
typedef struct Obiview_infos { typedef struct Obiview_infos {
int view_number; /**< Number of the view in the view file.
*/
time_t creation_date; /**< Time at which the view was written in the view file. time_t creation_date; /**< Time at which the view was written in the view file.
*/ */
char name[OBIVIEW_NAME_MAX_LENGTH+1]; /**< Name of the view, used to identify it. char name[OBIVIEW_NAME_MAX_LENGTH+1]; /**< Name of the view, used to identify it.
@ -138,7 +95,29 @@ typedef struct Obiview_infos {
} Obiview_infos_t, *Obiview_infos_p; } Obiview_infos_t, *Obiview_infos_p;
// TODO : Combine the common elements of the Obiview_infos and Obiview structures in one structure used by both? /**
* @brief Structure for an opened view.
*/
typedef struct Obiview {
Obiview_infos_p infos; /**< A pointer on the mapped view informations.
*/
OBIDMS_p dms; /**< A pointer on the DMS to which the view belongs.
*/
bool read_only; /**< Whether the view is read-only or can be modified.
*/
OBIDMS_column_p line_selection; /**< A pointer on the column containing the line selection
* associated with the view if there is one.
* This line selection is read-only, and when a line from the view is read,
* it is this line selection that is used.
*/
OBIDMS_column_p new_line_selection; /**< A pointer on the column containing the new line selection being built
* to associate with the view, if there is one.
* When a line is selected with obi_select_line() or obi_select_lines(),
* it is recorded in this line selection.
*/
OBIDMS_column_p columns[MAX_NB_OPENED_COLUMNS]; /**< Array of pointers on all the columns of the view.
*/
} Obiview_t, *Obiview_p;
/** /**
@ -269,6 +248,37 @@ Obiview_p obi_new_view_nuc_seqs(OBIDMS_p dms, const char* view_name, Obiview_p v
Obiview_p obi_new_view_nuc_seqs_cloned_from_name(OBIDMS_p dms, const char* view_name, const char* view_to_clone_name, index_t* line_selection, const char* comments); Obiview_p obi_new_view_nuc_seqs_cloned_from_name(OBIDMS_p dms, const char* view_name, const char* view_to_clone_name, index_t* line_selection, const char* comments);
/**
* @brief Maps a view file and returns the mapped structure stored in it.
*
* @param dms A pointer on the OBIDMS.
* @param view_name The unique name identifying the view.
*
* @returns A pointer on the mapped view infos structure.
* @retval NULL if an error occurred.
*
* @since June 2016
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
Obiview_infos_p obi_view_map_file(OBIDMS_p dms, const char* view_name);
/**
* @brief Unmaps a view file.
*
* @param dms A pointer on the OBIDMS.
* @param view_infos A pointer on the mapped view infos structure.
*
* @returns A value indicating the success of the operation.
* @retval 0 if the operation was successfully completed.
* @retval -1 if an error occurred.
*
* @since June 2016
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
int obi_view_unmap_file(OBIDMS_p dms, Obiview_infos_p view_infos);
/** /**
* @brief Opens a view identified by its name stored in the view file. * @brief Opens a view identified by its name stored in the view file.
* *
@ -873,7 +883,7 @@ int obi_column_set_obiqual_int_with_elt_idx_in_view(Obiview_p view, OBIDMS_colum
* @param element_idx The index of the element that should be recovered in the line. * @param element_idx The index of the element that should be recovered in the line.
* *
* @returns The recovered value, in the character string format. * @returns The recovered value, in the character string format.
* @retval OBIQual_str_NA the NA value of the type if an error occurred and obi_errno is set. * @retval OBIQual_char_NA the NA value of the type if an error occurred and obi_errno is set.
* *
* @since May 2016 * @since May 2016
* @author Celine Mercier (celine.mercier@metabarcoding.org) * @author Celine Mercier (celine.mercier@metabarcoding.org)
@ -967,7 +977,7 @@ int obi_column_set_obiqual_int_with_elt_name_in_view(Obiview_p view, OBIDMS_colu
* @param element_name The name of the element that should be recovered in the line. * @param element_name The name of the element that should be recovered in the line.
* *
* @returns The recovered value, in the character string format. * @returns The recovered value, in the character string format.
* @retval OBIQual_str_NA the NA value of the type if an error occurred and obi_errno is set. * @retval OBIQual_char_NA the NA value of the type if an error occurred and obi_errno is set.
* *
* @since May 2016 * @since May 2016
* @author Celine Mercier (celine.mercier@metabarcoding.org) * @author Celine Mercier (celine.mercier@metabarcoding.org)