obitools/obitools3
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 66 Wiki Activity

Added the lists of opened columns and arrays in the OBIDMS structure,

Browse Source 
and a counter in the OBIDMS column structure; fixed some bugs and
created tests for referring columns that are bound to disappear anyway.
This commit is contained in:
Celine Mercier
2015-12-02 17:32:07 +01:00
parent b45b496b0e
commit 1586956d57
8 changed files with 331 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
python/obitools3/obidms/_obidms.pyx
View File

@ -79,7 +79,6 @@ cdef class OBIDMS :
cpdef close(self) : cpdef close(self) :
#TODO close all columns (needs to be discussed)
if (obi_close_dms(self.pointer)) < 0 : if (obi_close_dms(self.pointer)) < 0 :
raise Exception("Problem closing an OBIDMS") raise Exception("Problem closing an OBIDMS")

153
python/obitools3/unit_tests.py
View File

@ -12,7 +12,6 @@ LINE_COUNT_FOR_TEST_COLUMN = 10000 # TODO randomize?
SMALLER_LINE_COUNT_FOR_TEST_COLUMN = 1000 # TODO randomize? SMALLER_LINE_COUNT_FOR_TEST_COLUMN = 1000 # TODO randomize?
NB_ELEMENTS_PER_LINE = 10 # TODO randomize? NB_ELEMENTS_PER_LINE = 10 # TODO randomize?
DMS_NAME = "unit_test_dms" DMS_NAME = "unit_test_dms"
DATA_TYPES = ['OBI_INT', 'OBI_FLOAT', 'OBI_BOOL', 'OBI_CHAR', 'OBI_STR', 'OBI_SEQ']
def create_test_obidms(): def create_test_obidms():
@ -22,26 +21,24 @@ def create_test_obidms():
return (dms, dms_name, dms_dir_name) return (dms, dms_name, dms_dir_name)
def create_test_column(dms, data_type_code, multiple_elements_per_line=False): def create_test_column(dms, data_type, multiple_elements_per_line=False):
data_types = DATA_TYPES
data_type_code = data_type_code col_name = "unit_test_"+data_type
data_type_str = data_types[data_type_code-1]
col_name = "unit_test_"+data_type_str
if multiple_elements_per_line : if multiple_elements_per_line :
elts_names = elements_names() elts_names = elements_names()
col = dms.open_column(col_name, col = dms.open_column(col_name,
create=True, create=True,
data_type=data_type_code, type=data_type,
nb_elements_per_line=NB_ELEMENTS_PER_LINE, nb_elements_per_line=NB_ELEMENTS_PER_LINE,
elements_names=elts_names) elements_names=elts_names)
return (col, col_name, elts_names, data_type_str) return (col, col_name, elts_names)
else : else :
col = dms.open_column(col_name, col = dms.open_column(col_name,
create=True, create=True,
data_type=data_type_code) type=data_type)
return (col, col_name, data_type_str) return (col, col_name)
def elements_names(): def elements_names():
@ -74,10 +71,10 @@ class OBIDMS_Column_TestCase(unittest.TestCase):
self.dms.close() self.dms.close()
shutil.rmtree(self.dms_dir_name, ignore_errors=True) shutil.rmtree(self.dms_dir_name, ignore_errors=True)
def test_OBIDMS_column_type(self): def test_OBIDMS_column_type(self):
assert self.col.get_data_type() == self.data_type_str, 'Wrong data type associated with column' assert self.col.get_data_type() == self.data_type, 'Wrong data type associated with column'
def test_OBIDMS_column_cloning(self): def test_OBIDMS_column_cloning(self):
for i in range(LINE_COUNT_FOR_TEST_COLUMN): for i in range(LINE_COUNT_FOR_TEST_COLUMN):
self.col[i]= random_obivalue(self.data_type_str) self.col[i]= random_obivalue(self.data_type)
self.col.close() self.col.close()
clone = self.dms.open_column(self.col_name, clone=True) clone = self.dms.open_column(self.col_name, clone=True)
self.col = self.dms.open_column(self.col_name) self.col = self.dms.open_column(self.col_name)
@ -89,10 +86,21 @@ class OBIDMS_Column_TestCase(unittest.TestCase):
clone.close() clone.close()
def test_OBIDMS_column_set_and_get(self): def test_OBIDMS_column_set_and_get(self):
for i in range(LINE_COUNT_FOR_TEST_COLUMN): for i in range(LINE_COUNT_FOR_TEST_COLUMN):
v = random_obivalue(self.data_type_str) v = random_obivalue(self.data_type)
self.col[i] = v self.col[i] = v
assert self.col[i] == v, "Different value than the set value" assert self.col[i] == v, "Different value than the set value"
assert self.col[i] is not None, "None value" assert self.col[i] is not None, "None value"
def test_OBIDMS_referring_column(self):
for i in range(LINE_COUNT_FOR_TEST_COLUMN):
self.col[i] = random_obivalue(self.data_type)
ref_col = self.dms.open_column(self.col_name, referring=True)
j = 0
for i in range(LINE_COUNT_FOR_TEST_COLUMN):
if i%2 : # TODO randomize
ref_col.grep_line(i)
assert ref_col[j] == self.col[i], "Different value in original column and returned by referring column"
assert ref_col[j] is not None, "None value"
j+=1
class OBIDMS_Column_multiple_elements_TestCase(OBIDMS_Column_TestCase): class OBIDMS_Column_multiple_elements_TestCase(OBIDMS_Column_TestCase):
@ -101,7 +109,7 @@ class OBIDMS_Column_multiple_elements_TestCase(OBIDMS_Column_TestCase):
for i in range(SMALLER_LINE_COUNT_FOR_TEST_COLUMN): for i in range(SMALLER_LINE_COUNT_FOR_TEST_COLUMN):
v = {} v = {}
for e in self.elts_names : for e in self.elts_names :
v[e] = random_obivalue(self.data_type_str) v[e] = random_obivalue(self.data_type)
self.col[i] = v self.col[i] = v
self.col.close() self.col.close()
clone = self.dms.open_column(self.col_name, clone=True) clone = self.dms.open_column(self.col_name, clone=True)
@ -115,7 +123,7 @@ class OBIDMS_Column_multiple_elements_TestCase(OBIDMS_Column_TestCase):
def test_OBIDMS_column_set_and_get_with_elements_names(self): def test_OBIDMS_column_set_and_get_with_elements_names(self):
for i in range(SMALLER_LINE_COUNT_FOR_TEST_COLUMN): for i in range(SMALLER_LINE_COUNT_FOR_TEST_COLUMN):
for e in range(NB_ELEMENTS_PER_LINE) : for e in range(NB_ELEMENTS_PER_LINE) :
v = random_obivalue(self.data_type_str) v = random_obivalue(self.data_type)
self.col.set_item(i, self.elts_names[e], v) self.col.set_item(i, self.elts_names[e], v)
assert self.col.get_item(i, self.elts_names[e]) == v, "Different value than the set value" assert self.col.get_item(i, self.elts_names[e]) == v, "Different value than the set value"
assert self.col.get_item(i, self.elts_names[e]) is not None, "None value" assert self.col.get_item(i, self.elts_names[e]) is not None, "None value"
@ -123,165 +131,168 @@ class OBIDMS_Column_multiple_elements_TestCase(OBIDMS_Column_TestCase):
for i in range(SMALLER_LINE_COUNT_FOR_TEST_COLUMN): for i in range(SMALLER_LINE_COUNT_FOR_TEST_COLUMN):
v = {} v = {}
for e in self.elts_names : for e in self.elts_names :
v[e] = random_obivalue(self.data_type_str) v[e] = random_obivalue(self.data_type)
self.col[i] = v self.col[i] = v
assert self.col[i] == v, "Different value than the set value" assert self.col[i] == v, "Different value than the set value"
assert self.col[i] is not None, "None value" assert self.col[i] is not None, "None value"
def test_OBIDMS_referring_column(self):
for i in range(SMALLER_LINE_COUNT_FOR_TEST_COLUMN):
v = {}
for e in self.elts_names :
v[e] = random_obivalue(self.data_type)
self.col[i] = v
ref_col = self.dms.open_column(self.col_name, referring=True)
j = 0
for i in range(SMALLER_LINE_COUNT_FOR_TEST_COLUMN):
if i%2 : # TODO randomize
ref_col.grep_line(i)
assert ref_col[j] == self.col[i], "Different value in original column and returned by referring column"
assert ref_col[j] is not None, "None value"
j+=1
ref_col.close()
class OBIDMS_Column_OBI_INT_TestCase(OBIDMS_Column_TestCase): class OBIDMS_Column_OBI_INT_TestCase(OBIDMS_Column_TestCase):
def setUp(self): def setUp(self):
self.data_type_code = 1 self.data_type = 'OBI_INT'
self.dms, \ self.dms, \
self.dms_name, \ self.dms_name, \
self.dms_dir_name = create_test_obidms() self.dms_dir_name = create_test_obidms()
self.col, \ self.col, \
self.col_name, \ self.col_name = create_test_column(self.dms,
self.data_type_str = create_test_column(self.dms, self.data_type)
self.data_type_code)
class OBIDMS_Column_OBI_INT_multiple_elements_TestCase(OBIDMS_Column_multiple_elements_TestCase): class OBIDMS_Column_OBI_INT_multiple_elements_TestCase(OBIDMS_Column_multiple_elements_TestCase):
def setUp(self): def setUp(self):
self.data_type_code = 1 self.data_type = 'OBI_INT'
self.dms, \ self.dms, \
self.dms_name, \ self.dms_name, \
self.dms_dir_name = create_test_obidms() self.dms_dir_name = create_test_obidms()
self.col, \ self.col, \
self.col_name, \ self.col_name, \
self.elts_names, \ self.elts_names = create_test_column(self.dms,
self.data_type_str = create_test_column(self.dms, self.data_type,
self.data_type_code, multiple_elements_per_line=True)
multiple_elements_per_line=True)
class OBIDMS_Column_OBI_FLOAT_TestCase(OBIDMS_Column_TestCase): class OBIDMS_Column_OBI_FLOAT_TestCase(OBIDMS_Column_TestCase):
def setUp(self): def setUp(self):
self.data_type_code = 2 self.data_type = 'OBI_FLOAT'
self.dms, \ self.dms, \
self.dms_name, \ self.dms_name, \
self.dms_dir_name = create_test_obidms() self.dms_dir_name = create_test_obidms()
self.col, \ self.col, \
self.col_name, \ self.col_name = create_test_column(self.dms,
self.data_type_str = create_test_column(self.dms, self.data_type)
self.data_type_code)
class OBIDMS_Column_OBI_FLOAT_multiple_elements_TestCase(OBIDMS_Column_multiple_elements_TestCase): class OBIDMS_Column_OBI_FLOAT_multiple_elements_TestCase(OBIDMS_Column_multiple_elements_TestCase):
def setUp(self): def setUp(self):
self.data_type_code = 2 self.data_type = 'OBI_FLOAT'
self.dms, \ self.dms, \
self.dms_name, \ self.dms_name, \
self.dms_dir_name = create_test_obidms() self.dms_dir_name = create_test_obidms()
self.col, \ self.col, \
self.col_name, \ self.col_name, \
self.elts_names, \ self.elts_names = create_test_column(self.dms,
self.data_type_str = create_test_column(self.dms, self.data_type,
self.data_type_code, multiple_elements_per_line=True)
multiple_elements_per_line=True)
class OBIDMS_Column_OBI_BOOL_TestCase(OBIDMS_Column_TestCase): class OBIDMS_Column_OBI_BOOL_TestCase(OBIDMS_Column_TestCase):
def setUp(self): def setUp(self):
self.data_type_code = 3 self.data_type = 'OBI_BOOL'
self.dms, \ self.dms, \
self.dms_name, \ self.dms_name, \
self.dms_dir_name = create_test_obidms() self.dms_dir_name = create_test_obidms()
self.col, \ self.col, \
self.col_name, \ self.col_name = create_test_column(self.dms,
self.data_type_str = create_test_column(self.dms, self.data_type)
self.data_type_code)
class OBIDMS_Column_OBI_BOOL_multiple_elements_TestCase(OBIDMS_Column_multiple_elements_TestCase): class OBIDMS_Column_OBI_BOOL_multiple_elements_TestCase(OBIDMS_Column_multiple_elements_TestCase):
def setUp(self): def setUp(self):
self.data_type_code = 3 self.data_type = 'OBI_BOOL'
self.dms, \ self.dms, \
self.dms_name, \ self.dms_name, \
self.dms_dir_name = create_test_obidms() self.dms_dir_name = create_test_obidms()
self.col, \ self.col, \
self.col_name, \ self.col_name, \
self.elts_names, \ self.elts_names = create_test_column(self.dms,
self.data_type_str = create_test_column(self.dms, self.data_type,
self.data_type_code, multiple_elements_per_line=True)
multiple_elements_per_line=True)
class OBIDMS_Column_OBI_CHAR_TestCase(OBIDMS_Column_TestCase): class OBIDMS_Column_OBI_CHAR_TestCase(OBIDMS_Column_TestCase):
def setUp(self): def setUp(self):
self.data_type_code = 4 self.data_type = 'OBI_CHAR'
self.dms, \ self.dms, \
self.dms_name, \ self.dms_name, \
self.dms_dir_name = create_test_obidms() self.dms_dir_name = create_test_obidms()
self.col, \ self.col, \
self.col_name, \ self.col_name = create_test_column(self.dms,
self.data_type_str = create_test_column(self.dms, self.data_type)
self.data_type_code)
class OBIDMS_Column_OBI_CHAR_multiple_elements_TestCase(OBIDMS_Column_multiple_elements_TestCase): class OBIDMS_Column_OBI_CHAR_multiple_elements_TestCase(OBIDMS_Column_multiple_elements_TestCase):
def setUp(self): def setUp(self):
self.data_type_code = 4 self.data_type = 'OBI_CHAR'
self.dms, \ self.dms, \
self.dms_name, \ self.dms_name, \
self.dms_dir_name = create_test_obidms() self.dms_dir_name = create_test_obidms()
self.col, \ self.col, \
self.col_name, \ self.col_name, \
self.elts_names, \ self.elts_names = create_test_column(self.dms,
self.data_type_str = create_test_column(self.dms, self.data_type,
self.data_type_code, multiple_elements_per_line=True)
multiple_elements_per_line=True)
class OBIDMS_Column_OBI_STR_TestCase(OBIDMS_Column_TestCase): class OBIDMS_Column_OBI_STR_TestCase(OBIDMS_Column_TestCase):
def setUp(self): def setUp(self):
self.data_type_code = 5 self.data_type = 'OBI_STR'
self.dms, \ self.dms, \
self.dms_name, \ self.dms_name, \
self.dms_dir_name = create_test_obidms() self.dms_dir_name = create_test_obidms()
self.col, \ self.col, \
self.col_name, \ self.col_name = create_test_column(self.dms,
self.data_type_str = create_test_column(self.dms, self.data_type)
self.data_type_code)
class OBIDMS_Column_OBI_STR_multiple_elements_TestCase(OBIDMS_Column_multiple_elements_TestCase): class OBIDMS_Column_OBI_STR_multiple_elements_TestCase(OBIDMS_Column_multiple_elements_TestCase):
def setUp(self): def setUp(self):
self.data_type_code = 5 self.data_type = 'OBI_STR'
self.dms, \ self.dms, \
self.dms_name, \ self.dms_name, \
self.dms_dir_name = create_test_obidms() self.dms_dir_name = create_test_obidms()
self.col, \ self.col, \
self.col_name, \ self.col_name, \
self.elts_names, \ self.elts_names = create_test_column(self.dms,
self.data_type_str = create_test_column(self.dms, self.data_type,
self.data_type_code, multiple_elements_per_line=True)
multiple_elements_per_line=True)
class OBIDMS_Column_OBI_SEQ_TestCase(OBIDMS_Column_TestCase): class OBIDMS_Column_OBI_SEQ_TestCase(OBIDMS_Column_TestCase):
def setUp(self): def setUp(self):
self.data_type_code = 6 self.data_type = 'OBI_SEQ'
self.dms, \ self.dms, \
self.dms_name, \ self.dms_name, \
self.dms_dir_name = create_test_obidms() self.dms_dir_name = create_test_obidms()
self.col, \ self.col, \
self.col_name, \ self.col_name = create_test_column(self.dms,
self.data_type_str = create_test_column(self.dms, self.data_type)
self.data_type_code)
class OBIDMS_Column_OBI_SEQ_multiple_elements_TestCase(OBIDMS_Column_multiple_elements_TestCase): class OBIDMS_Column_OBI_SEQ_multiple_elements_TestCase(OBIDMS_Column_multiple_elements_TestCase):
def setUp(self): def setUp(self):
self.data_type_code = 6 self.data_type = 'OBI_SEQ'
self.dms, \ self.dms, \
self.dms_name, \ self.dms_name, \
self.dms_dir_name = create_test_obidms() self.dms_dir_name = create_test_obidms()
self.col, \ self.col, \
self.col_name, \ self.col_name, \
self.elts_names, \ self.elts_names = create_test_column(self.dms,
self.data_type_str = create_test_column(self.dms, self.data_type,
self.data_type_code, multiple_elements_per_line=True)
multiple_elements_per_line=True)
if __name__ == '__main__': if __name__ == '__main__':
unittest.main(verbosity=2, defaultTest=["OBIDMS_Column_OBI_INT_TestCase", unittest.main(verbosity=2, defaultTest=["OBIDMS_Column_OBI_INT_TestCase",

112
src/obiarray.c
View File

@ -233,9 +233,20 @@ static char* build_array_data_file_name(const char* array_name)
} }
size_t get_array_header_size() size_t get_array_header_size()
{ {
return getpagesize() * 1; size_t header_size;
size_t rounded_header_size;
double multiple;
header_size = sizeof(OBIDMS_array_header_t);
multiple = ceil((double) header_size / (double) getpagesize());
rounded_header_size = multiple * getpagesize();
return rounded_header_size;
} }
@ -247,7 +258,17 @@ size_t get_initial_array_size()
size_t get_array_data_header_size() size_t get_array_data_header_size()
{ {
return getpagesize() * 1; size_t header_size;
size_t rounded_header_size;
double multiple;
header_size = sizeof(OBIDMS_array_data_header_t);
multiple = ceil((double) header_size / (double) getpagesize());
rounded_header_size = multiple * getpagesize();
return rounded_header_size;
} }
@ -261,13 +282,6 @@ int close_array_data(OBIDMS_array_data_p array_data)
{ {
int ret_val = 0; int ret_val = 0;
if (munmap(array_data->header, (array_data->header)->header_size) < 0)
{
obi_set_errno(OBI_ARRAY_ERROR);
obidebug(1, "\nError munmapping the header of an array data file");
ret_val = -1;
}
if (munmap(array_data->data, (array_data->header)->data_size_max) < 0) if (munmap(array_data->data, (array_data->header)->data_size_max) < 0)
{ {
obi_set_errno(OBI_ARRAY_ERROR); obi_set_errno(OBI_ARRAY_ERROR);
@ -275,6 +289,13 @@ int close_array_data(OBIDMS_array_data_p array_data)
ret_val = -1; ret_val = -1;
} }
if (munmap(array_data->header, (array_data->header)->header_size) < 0)
{
obi_set_errno(OBI_ARRAY_ERROR);
obidebug(1, "\nError munmapping the header of an array data file");
ret_val = -1;
}
free(array_data); free(array_data);
return ret_val; return ret_val;
@ -737,6 +758,7 @@ OBIDMS_array_p obi_create_array(OBIDMS_p dms, const char* array_name)
return NULL; return NULL;
} }
array->dms = dms;
array->data = array_data; array->data = array_data;
array->directory = dms->array_directory; array->directory = dms->array_directory;
array->dir_fd = array_dir_file_descriptor; array->dir_fd = array_dir_file_descriptor;
@ -753,6 +775,11 @@ OBIDMS_array_p obi_create_array(OBIDMS_p dms, const char* array_name)
close(array_file_descriptor); close(array_file_descriptor);
// Add in the list of opened arrays
*(((dms->opened_arrays)->arrays)+((dms->opened_arrays)->nb_opened_arrays)) = array;
((dms->opened_arrays)->nb_opened_arrays)++;
array->counter = 1;
return array; return array;
} }
@ -767,6 +794,17 @@ OBIDMS_array_p obi_open_array(OBIDMS_p dms, const char* array_name)
int array_dir_file_descriptor; int array_dir_file_descriptor;
OBIDMS_array_data_p array_data; OBIDMS_array_data_p array_data;
OBIDMS_array_p array; OBIDMS_array_p array;
size_t i;
// Check if the array is already in the list of opened arrays
for (i=0; i < ((dms->opened_arrays)->nb_opened_arrays); i++)
{
if (!strcmp(((*(((dms->opened_arrays)->arrays)+i))->header)->array_name, array_name))
{ // Found the array already opened
((*(((dms->opened_arrays)->arrays)+i))->counter)++;
return *(((dms->opened_arrays)->arrays)+i);
}
}
// Open the data file // Open the data file
@ -923,37 +961,65 @@ OBIDMS_array_p obi_open_array(OBIDMS_p dms, const char* array_name)
return NULL; return NULL;
} }
array->dms = dms;
array->data = array_data; array->data = array_data;
array->directory = dms->array_directory; array->directory = dms->array_directory;
array->dir_fd = array_dir_file_descriptor; array->dir_fd = array_dir_file_descriptor;
close(array_file_descriptor); close(array_file_descriptor);
// Add in the list of opened arrays
*(((dms->opened_arrays)->arrays)+((dms->opened_arrays)->nb_opened_arrays)) = array;
((dms->opened_arrays)->nb_opened_arrays)++;
array->counter = 1;
return array; return array;
} }
int obi_close_array(OBIDMS_array_p array) // TODO when do we call it? (as multiple threads can be using the same array) int obi_close_array(OBIDMS_array_p array)
{ {
int ret_val = 0; int ret_val = 0;
size_t i;
Opened_arrays_list_p arrays_list;
OBIDMS_p dms;
ret_val = close_array_data(array->data); dms = array->dms;
if (munmap(array->first, (((array->header)->nb_items_max) * sizeof(index_t))) < 0) arrays_list = dms->opened_arrays;
(array->counter)--;
if (array->counter == 0)
{ {
obi_set_errno(OBI_ARRAY_ERROR); // Delete from the list of opened arrays
obidebug(1, "\nError munmapping the array of an array file"); for (i=0; i < (arrays_list->nb_opened_arrays); i++)
ret_val = -1; {
} if (!strcmp(((*((arrays_list->arrays)+i))->header)->array_name, (array->header)->array_name))
{ // Found the array. Rearrange list
(arrays_list->nb_opened_arrays)--;
(arrays_list->arrays)[i] = (arrays_list->arrays)[arrays_list->nb_opened_arrays];
}
}
if (munmap(array->header, (array->header)->header_size) < 0) ret_val = close_array_data(array->data);
{
obi_set_errno(OBI_ARRAY_ERROR);
obidebug(1, "\nError munmapping the header of an array file");
ret_val = -1;
}
free(array); if (munmap(array->first, (((array->header)->nb_items_max) * sizeof(index_t))) < 0)
{
obi_set_errno(OBI_ARRAY_ERROR);
obidebug(1, "\nError munmapping the array of an array file");
ret_val = -1;
}
if (munmap(array->header, (array->header)->header_size) < 0)
{
obi_set_errno(OBI_ARRAY_ERROR);
obidebug(1, "\nError munmapping the header of an array file");
ret_val = -1;
}
free(array);
}
return ret_val; return ret_val;
} }

4
src/obiarray.h
View File

@ -91,6 +91,8 @@ typedef struct OBIDMS_array_header {
* @brief OBIDMS array structure. * @brief OBIDMS array structure.
*/ */
typedef struct OBIDMS_array { typedef struct OBIDMS_array {
OBIDMS_p dms; /**< A pointer to the OBIDMS structure to which the array belongs.
*/
OBIDMS_array_header_p header; /**< A pointer to the header of the array. OBIDMS_array_header_p header; /**< A pointer to the header of the array.
*/ */
index_t* first; /**< A pointer to the beginning of the array itself. index_t* first; /**< A pointer to the beginning of the array itself.
@ -104,6 +106,8 @@ typedef struct OBIDMS_array {
int dir_fd; /**< The file descriptor of the directory entry int dir_fd; /**< The file descriptor of the directory entry
* usable to refer and scan the array directory. * usable to refer and scan the array directory.
*/ */
size_t counter; /**< Indicates by how many threads/programs (TODO) the array is used.
*/
} OBIDMS_array_t, *OBIDMS_array_p; } OBIDMS_array_t, *OBIDMS_array_p;

15
src/obidms.c
View File

@ -413,6 +413,16 @@ OBIDMS_p obi_open_dms(const char* dms_name)
return NULL; return NULL;
} }
// Initialize the list of opened columns
dms->opened_columns = (Opened_columns_list_p) malloc(sizeof(Opened_columns_list_t));
(dms->opened_columns)->columns = (OBIDMS_column_p*) malloc(MAX_NB_OPENED_COLUMNS*sizeof(OBIDMS_column_p));
(dms->opened_columns)->nb_opened_columns = 0;
// Initialize the list of opened arrays
dms->opened_arrays = (Opened_arrays_list_p) malloc(sizeof(Opened_arrays_list_t));
(dms->opened_arrays)->arrays = (OBIDMS_array_p*) malloc(MAX_NB_OPENED_ARRAYS*sizeof(OBIDMS_array_p));
(dms->opened_arrays)->nb_opened_arrays = 0;
return dms; return dms;
} }
@ -440,6 +450,11 @@ int obi_close_dms(OBIDMS_p dms)
{ {
if (dms != NULL) if (dms != NULL)
{ {
// Close all columns
while ((dms->opened_columns)->nb_opened_columns > 0)
obi_close_column(*((dms->opened_columns)->columns));
// Close dms and array directories
if (closedir(dms->directory) < 0) if (closedir(dms->directory) < 0)
{ {
obi_set_errno(OBIDMS_MEMORY_ERROR); obi_set_errno(OBIDMS_MEMORY_ERROR);

66
src/obidms.h
View File

@ -25,10 +25,30 @@
#include "obierrno.h" #include "obierrno.h"
#define OBIDMS_MAX_NAME (2048) /**< The maximum length of an OBIDMS name. #define OBIDMS_MAX_NAME (2048) /**< The maximum length of an OBIDMS name.
*/ */
#define ARRAYS_DIR_NAME "arrays" /**< The name of the arrays directory. #define ARRAYS_DIR_NAME "arrays" /**< The name of the arrays directory.
*/ */
#define MAX_NB_OPENED_COLUMNS (100) /**< The maximum number of columns open at the same time.
*/
#define MAX_NB_OPENED_ARRAYS (100) /**< The maximum number of arrays open at the same time.
*/
struct OBIDMS_column; // TODO
typedef struct Opened_columns_list {
size_t nb_opened_columns;
struct OBIDMS_column** columns;
} Opened_columns_list_t, *Opened_columns_list_p;
struct OBIDMS_array; // TODO
typedef struct Opened_arrays_list {
size_t nb_opened_arrays;
struct OBIDMS_array** arrays;
} Opened_arrays_list_t, *Opened_arrays_list_p;
/** /**
@ -38,23 +58,27 @@
* and opening of an OBITools Data Management System (DMS) * and opening of an OBITools Data Management System (DMS)
*/ */
typedef struct OBIDMS { typedef struct OBIDMS {
char directory_name[OBIDMS_MAX_NAME+1]; /**< The name of the directory char directory_name[OBIDMS_MAX_NAME+1]; /**< The name of the directory
* containing the DMS. * containing the DMS.
*/ */
DIR* directory; /**< A directory entry usable to DIR* directory; /**< A directory entry usable to
* refer and scan the database directory. * refer and scan the database directory.
*/ */
int dir_fd; /**< The file descriptor of the directory entry int dir_fd; /**< The file descriptor of the directory entry
* usable to refer and scan the database directory. * usable to refer and scan the database directory.
*/ */
DIR* array_directory; /**< A directory entry usable to DIR* array_directory; /**< A directory entry usable to
* refer and scan the array directory. * refer and scan the array directory.
*/ */
int array_dir_fd; /**< The file descriptor of the directory entry int array_dir_fd; /**< The file descriptor of the directory entry
* usable to refer and scan the array directory. * usable to refer and scan the array directory.
*/ */
bool little_endian; /**< Endianness of the database. bool little_endian; /**< Endianness of the database.
*/ */
Opened_columns_list_p opened_columns; /**< List of opened columns.
*/
Opened_arrays_list_p opened_arrays; /**< List of opened arrays.
*/
} OBIDMS_t, *OBIDMS_p; } OBIDMS_t, *OBIDMS_p;

120
src/obidmscolumn.c
View File

@ -732,9 +732,8 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms,
if (comments != NULL) if (comments != NULL)
strncpy(header->comments, comments, COMMENTS_MAX_LENGTH); strncpy(header->comments, comments, COMMENTS_MAX_LENGTH);
// If the data type is OBI_STR or OBI_SEQ, and the column is not referring another, // If the data type is OBI_STR or OBI_SEQ, the associated obi_array is opened or created
// the associated obi_array is opened or created if ((returned_data_type == OBI_STR) || (returned_data_type == OBI_SEQ))
if ((stored_data_type == OBI_STR) || (stored_data_type == OBI_SEQ))
{ {
array = obi_array(dms, array_name); array = obi_array(dms, array_name);
if (array == NULL) if (array == NULL)
@ -754,6 +753,11 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms,
close(column_file_descriptor); close(column_file_descriptor);
// Add in the list of opened columns
*(((dms->opened_columns)->columns)+((dms->opened_columns)->nb_opened_columns)) = new_column;
((dms->opened_columns)->nb_opened_columns)++;
new_column->counter = 1;
return new_column; return new_column;
} }
@ -768,6 +772,7 @@ OBIDMS_column_p obi_open_column(OBIDMS_p dms,
char* column_file_name; char* column_file_name;
int column_file_descriptor; int column_file_descriptor;
size_t header_size; size_t header_size;
size_t i;
column = NULL; column = NULL;
@ -790,6 +795,17 @@ OBIDMS_column_p obi_open_column(OBIDMS_p dms,
} }
} }
// Check if the column is already in the list of opened columns
for (i=0; i < ((dms->opened_columns)->nb_opened_columns); i++)
{
if (!strcmp(((*(((dms->opened_columns)->columns)+i))->header)->name, column_name)
&& (((*(((dms->opened_columns)->columns)+i))->header)->version == version_number))
{ // Found the column already opened, increase its counter and return the column
((*(((dms->opened_columns)->columns)+i))->counter)++;
return *(((dms->opened_columns)->columns)+i);
}
}
// Get the column file name // Get the column file name
column_file_name = build_column_file_name(column_name, version_number); column_file_name = build_column_file_name(column_name, version_number);
if (column_file_name == NULL) if (column_file_name == NULL)
@ -870,9 +886,8 @@ OBIDMS_column_p obi_open_column(OBIDMS_p dms,
column->writable = false; column->writable = false;
// If the data type is OBI_STR or OBI_SEQ, and the column is not referring, // If the data type is OBI_STR or OBI_SEQ, the associated obi_array is opened
// the associated obi_array is opened if (((column->header)->returned_data_type == OBI_STR) || ((column->header)->returned_data_type == OBI_SEQ))
if (((column->header)->stored_data_type == OBI_STR) || ((column->header)->stored_data_type == OBI_SEQ))
{ {
array = obi_array(dms, (column->header)->array_name); array = obi_array(dms, (column->header)->array_name);
if (array == NULL) if (array == NULL)
@ -900,6 +915,11 @@ OBIDMS_column_p obi_open_column(OBIDMS_p dms,
close(column_file_descriptor); close(column_file_descriptor);
// Add in the list of opened columns and set column counter to 1
*(((dms->opened_columns)->columns)+((dms->opened_columns)->nb_opened_columns)) = column;
((dms->opened_columns)->nb_opened_columns)++;
column->counter = 1;
return column; return column;
} }
@ -924,6 +944,14 @@ OBIDMS_column_p obi_clone_column(OBIDMS_p dms,
return NULL; return NULL;
} }
// A referring column can't be cloned
if ((column_to_clone->header)->referring)
{
obi_set_errno(OBICOL_UNKNOWN_ERROR);
obidebug(1, "\nError: can't clone a referring column");
return NULL;
}
data_type = (column_to_clone->header)->returned_data_type; data_type = (column_to_clone->header)->returned_data_type;
nb_elements_per_line = (column_to_clone->header)->returned_nb_elements_per_line; nb_elements_per_line = (column_to_clone->header)->returned_nb_elements_per_line;
@ -990,29 +1018,69 @@ OBIDMS_column_p obi_clone_column(OBIDMS_p dms,
int obi_close_column(OBIDMS_column_p column) int obi_close_column(OBIDMS_column_p column)
{ {
if ((column->header)->referring) size_t i;
obi_close_column(column->referred_column); bool close_dir;
Opened_columns_list_p columns_list;
// Munmap data columns_list = (column->dms)->opened_columns;
if (munmap(column->data, (column->header)->data_size) < 0)
(column->counter)--;
if (column->counter == 0)
{ {
obi_set_errno(OBICOL_UNKNOWN_ERROR); // Delete from the list of opened columns
obidebug(1, "\nError munmapping column data"); for (i=0; i < (columns_list)->nb_opened_columns; i++)
return -1; {
if (!strcmp(((*((columns_list->columns)+i))->header)->name, (column->header)->name)
&& (((*((columns_list->columns)+i))->header)->version == (column->header)->version))
{ // Found the column. Rearrange list
(columns_list->nb_opened_columns)--;
(columns_list->columns)[i] = (columns_list->columns)[columns_list->nb_opened_columns];
}
}
// Close column directory if it was the last column opened from that directory
close_dir = 1;
for (i=0; i < (columns_list->nb_opened_columns); i++)
{
if (!strcmp(((*((columns_list->columns)+i))->header)->name, (column->header)->name))
{ // Not the last column from that directory
close_dir = 0;
}
}
if ((column->header)->referring)
obi_close_column(column->referred_column);
// If the data type is OBI_STR or OBI_SEQ, the associated obi_array is closed
if (((column->header)->returned_data_type == OBI_STR) || ((column->header)->returned_data_type == OBI_SEQ))
{
if (obi_close_array(column->array) < 0)
return -1;
}
// Munmap data
if (munmap(column->data, (column->header)->data_size) < 0)
{
obi_set_errno(OBICOL_UNKNOWN_ERROR);
obidebug(1, "\nError munmapping column data");
return -1;
}
// Munmap header
if (munmap(column->header, (column->header)->header_size) < 0)
{
obi_set_errno(OBICOL_UNKNOWN_ERROR);
obidebug(1, "\nError munmapping a column header");
return -1;
}
free(column);
if (close_dir)
obi_close_column_directory(column->column_directory);
} }
// Munmap header
if (munmap(column->header, (column->header)->header_size) < 0)
{
obi_set_errno(OBICOL_UNKNOWN_ERROR);
obidebug(1, "\nError munmapping a column header");
return -1;
}
//obi_close_column_directory(column->column_directory); // TODO or not
free(column);
return 0; return 0;
} }
@ -1409,7 +1477,7 @@ char* obi_column_format_date(time_t date)
} }
// TODO put in separate file and needs to lock the dependency with the referred column but... // TODO put in separate file ?
// warning for the dependency and for the fact that it's always added at the next line (or not cuz might not be a good idea?) // warning for the dependency and for the fact that it's always added at the next line (or not cuz might not be a good idea?)
int obi_grep_line(OBIDMS_column_p referring_column, index_t line_to_grep) int obi_grep_line(OBIDMS_column_p referring_column, index_t line_to_grep)
{ {

2
src/obidmscolumn.h
View File

@ -122,6 +122,8 @@ typedef struct OBIDMS_column {
* A column is writable only by its creator * A column is writable only by its creator
* until it closes it. * until it closes it.
*/ */
size_t counter; /**< Indicates by how many threads/programs (TODO) the column is used.
*/
} OBIDMS_column_t, *OBIDMS_column_p; } OBIDMS_column_t, *OBIDMS_column_p;