obitools/obitools3
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 66 Wiki Activity

AVLs: AVLs in a group are not unmapped and remapped constantly anymore

Browse Source 
when adding new values, fixed a bug when calculating if an AVL data file
has reached the maximum size, fixed a casting bug, and added a boolean
so read-only AVLs files are not truncated
This commit is contained in:
Celine Mercier
2018-11-19 11:19:07 +01:00
parent 31053591b5
commit 7fc1b578cf
1 changed files with 48 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
src/obiavl.c
View File

@ -197,6 +197,7 @@ int grow_avl_data(OBIDMS_avl_data_p avl_data);
* @brief Internal function closing an AVL data structure where the data referred to by an AVL tree is stored. * @brief Internal function closing an AVL data structure where the data referred to by an AVL tree is stored.
* *
* @param avl_data A pointer to the data structure referred to by an AVL tree. * @param avl_data A pointer to the data structure referred to by an AVL tree.
* @param writable Whether the AVL is writable or not (and therefore if the files can and should be truncated to the used size).
* *
* @retval 0 if the operation was successfully completed. * @retval 0 if the operation was successfully completed.
* @retval -1 if an error occurred. * @retval -1 if an error occurred.
@ -204,7 +205,7 @@ int grow_avl_data(OBIDMS_avl_data_p avl_data);
* @since December 2015 * @since December 2015
* @author Celine Mercier (celine.mercier@metabarcoding.org) * @author Celine Mercier (celine.mercier@metabarcoding.org)
*/ */
int close_avl_data(OBIDMS_avl_data_p avl_data); int close_avl_data(OBIDMS_avl_data_p avl_data, bool writable);
/** /**
@ -586,7 +587,7 @@ int truncate_avl_to_size_used(OBIDMS_avl_p avl) // TODO is it necessary to unmap
// Compute the new size: used size rounded to the nearest greater multiple of page size greater than 0 // Compute the new size: used size rounded to the nearest greater multiple of page size greater than 0
multiple = ceil((double) (ONE_IF_ZERO((avl->header)->nb_items * sizeof(AVL_node_t))) / (double) getpagesize()); multiple = ceil((double) (ONE_IF_ZERO((avl->header)->nb_items * sizeof(AVL_node_t))) / (double) getpagesize());
new_data_size = ((int) multiple) * getpagesize(); new_data_size = ((size_t) multiple) * getpagesize();
// Check that it is actually greater than the current size of the file, otherwise no need to truncate // Check that it is actually greater than the current size of the file, otherwise no need to truncate
if ((avl->header)->avl_size == new_data_size) if ((avl->header)->avl_size == new_data_size)
@ -644,7 +645,7 @@ int truncate_avl_data_to_size_used(OBIDMS_avl_data_p avl_data) // TODO is it nec
// Compute the new size: used size rounded to the nearest greater multiple of page size greater than 0 // Compute the new size: used size rounded to the nearest greater multiple of page size greater than 0
multiple = ceil((double) (ONE_IF_ZERO((avl_data->header)->data_size_used)) / (double) getpagesize()); multiple = ceil((double) (ONE_IF_ZERO((avl_data->header)->data_size_used)) / (double) getpagesize());
new_data_size = ((int) multiple) * getpagesize(); new_data_size = ((index_t) multiple) * getpagesize();
// Check that it is actually greater than the current size of the file, otherwise no need to truncate // Check that it is actually greater than the current size of the file, otherwise no need to truncate
if ((avl_data->header)->data_size_max == new_data_size) if ((avl_data->header)->data_size_max == new_data_size)
@ -809,10 +810,11 @@ int grow_avl_data(OBIDMS_avl_data_p avl_data) // TODO Lock when needed
} }
int close_avl_data(OBIDMS_avl_data_p avl_data) int close_avl_data(OBIDMS_avl_data_p avl_data, bool writable)
{ {
int ret_val = 0; int ret_val = 0;
if (writable)
ret_val = truncate_avl_data_to_size_used(avl_data); ret_val = truncate_avl_data_to_size_used(avl_data);
if (munmap(avl_data->data, (avl_data->header)->data_size_max) < 0) if (munmap(avl_data->data, (avl_data->header)->data_size_max) < 0)
@ -903,9 +905,10 @@ int add_new_avl_in_group(OBIDMS_avl_group_p avl_group)
} }
// Unmap the previous AVL if it's not the 1st // Unmap the previous AVL if it's not the 1st
if (avl_group->last_avl_idx > 0) // Not done anymore, currently keeping all mapped for efficiency reasons
if (unmap_an_avl((avl_group->sub_avls)[avl_group->last_avl_idx]) < 0) // if (avl_group->last_avl_idx > 0)
return -1; // if (unmap_an_avl((avl_group->sub_avls)[avl_group->last_avl_idx]) < 0)
// return -1;
// Increment current AVL index // Increment current AVL index
(avl_group->last_avl_idx)++; (avl_group->last_avl_idx)++;
@ -1556,7 +1559,7 @@ OBIDMS_avl_p obi_create_avl(OBIDMS_p dms, const char* avl_name, int avl_idx)
avl_file_name = build_avl_file_name(complete_avl_name); avl_file_name = build_avl_file_name(complete_avl_name);
if (avl_file_name == NULL) if (avl_file_name == NULL)
{ {
close_avl_data(avl_data); close_avl_data(avl_data, true);
return NULL; return NULL;
} }
@ -1573,7 +1576,7 @@ OBIDMS_avl_p obi_create_avl(OBIDMS_p dms, const char* avl_name, int avl_idx)
obidebug(1, "\nError creating an AVL tree file"); obidebug(1, "\nError creating an AVL tree file");
if (avl_idx >= 0) if (avl_idx >= 0)
free(complete_avl_name); free(complete_avl_name);
close_avl_data(avl_data); close_avl_data(avl_data, true);
free(avl_file_name); free(avl_file_name);
return NULL; return NULL;
} }
@ -1586,7 +1589,7 @@ OBIDMS_avl_p obi_create_avl(OBIDMS_p dms, const char* avl_name, int avl_idx)
obidebug(1, "\nError truncating an AVL tree file to the right size"); obidebug(1, "\nError truncating an AVL tree file to the right size");
if (avl_idx >= 0) if (avl_idx >= 0)
free(complete_avl_name); free(complete_avl_name);
close_avl_data(avl_data); close_avl_data(avl_data, true);
close(avl_file_descriptor); close(avl_file_descriptor);
return NULL; return NULL;
} }
@ -1599,7 +1602,7 @@ OBIDMS_avl_p obi_create_avl(OBIDMS_p dms, const char* avl_name, int avl_idx)
obidebug(1, "\nError allocating the memory for the AVL tree structure"); obidebug(1, "\nError allocating the memory for the AVL tree structure");
if (avl_idx >= 0) if (avl_idx >= 0)
free(complete_avl_name); free(complete_avl_name);
close_avl_data(avl_data); close_avl_data(avl_data, true);
close(avl_file_descriptor); close(avl_file_descriptor);
return NULL; return NULL;
} }
@ -1618,7 +1621,7 @@ OBIDMS_avl_p obi_create_avl(OBIDMS_p dms, const char* avl_name, int avl_idx)
obidebug(1, "\nError mmapping the header of an AVL tree file"); obidebug(1, "\nError mmapping the header of an AVL tree file");
if (avl_idx >= 0) if (avl_idx >= 0)
free(complete_avl_name); free(complete_avl_name);
close_avl_data(avl_data); close_avl_data(avl_data, true);
close(avl_file_descriptor); close(avl_file_descriptor);
free(avl); free(avl);
return NULL; return NULL;
@ -1637,7 +1640,7 @@ OBIDMS_avl_p obi_create_avl(OBIDMS_p dms, const char* avl_name, int avl_idx)
obidebug(1, "\nError mmapping the data of an AVL tree file"); obidebug(1, "\nError mmapping the data of an AVL tree file");
if (avl_idx >= 0) if (avl_idx >= 0)
free(complete_avl_name); free(complete_avl_name);
close_avl_data(avl_data); close_avl_data(avl_data, true);
munmap(avl->header, header_size); munmap(avl->header, header_size);
close(avl_file_descriptor); close(avl_file_descriptor);
free(avl); free(avl);
@ -1709,7 +1712,6 @@ OBIDMS_avl_p obi_open_avl(OBIDMS_p dms, const char* avl_name, int avl_idx)
avl_dir_name = get_full_path_of_avl_dir(dms, avl_name); avl_dir_name = get_full_path_of_avl_dir(dms, avl_name);
if (avl_dir_name == NULL) if (avl_dir_name == NULL)
{ {
if (avl_idx >= 0)
free(complete_avl_name); free(complete_avl_name);
return NULL; return NULL;
} }
@ -1718,7 +1720,6 @@ OBIDMS_avl_p obi_open_avl(OBIDMS_p dms, const char* avl_name, int avl_idx)
{ {
obi_set_errno(OBI_AVL_ERROR); obi_set_errno(OBI_AVL_ERROR);
obidebug(1, "\nError opening an AVL directory"); obidebug(1, "\nError opening an AVL directory");
if (avl_idx >= 0)
free(complete_avl_name); free(complete_avl_name);
free(avl_dir_name); free(avl_dir_name);
return NULL; return NULL;
@ -1729,7 +1730,6 @@ OBIDMS_avl_p obi_open_avl(OBIDMS_p dms, const char* avl_name, int avl_idx)
{ {
obi_set_errno(OBI_AVL_ERROR); obi_set_errno(OBI_AVL_ERROR);
obidebug(1, "\nError getting the file descriptor of an AVL directory"); obidebug(1, "\nError getting the file descriptor of an AVL directory");
if (avl_idx >= 0)
free(complete_avl_name); free(complete_avl_name);
return NULL; return NULL;
} }
@ -1748,7 +1748,6 @@ OBIDMS_avl_p obi_open_avl(OBIDMS_p dms, const char* avl_name, int avl_idx)
{ {
obi_set_errno(OBI_AVL_ERROR); obi_set_errno(OBI_AVL_ERROR);
obidebug(1, "\nError opening an AVL tree data file"); obidebug(1, "\nError opening an AVL tree data file");
if (avl_idx >= 0)
free(complete_avl_name); free(complete_avl_name);
free(avl_data_file_name); free(avl_data_file_name);
return NULL; return NULL;
@ -1761,7 +1760,6 @@ OBIDMS_avl_p obi_open_avl(OBIDMS_p dms, const char* avl_name, int avl_idx)
{ {
obi_set_errno(OBI_MALLOC_ERROR); obi_set_errno(OBI_MALLOC_ERROR);
obidebug(1, "\nError allocating the memory for the AVL tree data structure"); obidebug(1, "\nError allocating the memory for the AVL tree data structure");
if (avl_idx >= 0)
free(complete_avl_name); free(complete_avl_name);
close(avl_data_file_descriptor); close(avl_data_file_descriptor);
return NULL; return NULL;
@ -1772,7 +1770,6 @@ OBIDMS_avl_p obi_open_avl(OBIDMS_p dms, const char* avl_name, int avl_idx)
{ {
obi_set_errno(OBI_AVL_ERROR); obi_set_errno(OBI_AVL_ERROR);
obidebug(1, "\nError reading the header size to open an AVL tree data file"); obidebug(1, "\nError reading the header size to open an AVL tree data file");
if (avl_idx >= 0)
free(complete_avl_name); free(complete_avl_name);
close(avl_data_file_descriptor); close(avl_data_file_descriptor);
return NULL; return NULL;
@ -1790,7 +1787,6 @@ OBIDMS_avl_p obi_open_avl(OBIDMS_p dms, const char* avl_name, int avl_idx)
{ {
obi_set_errno(OBI_AVL_ERROR); obi_set_errno(OBI_AVL_ERROR);
obidebug(1, "\nError mmapping the header of an AVL tree data file"); obidebug(1, "\nError mmapping the header of an AVL tree data file");
if (avl_idx >= 0)
free(complete_avl_name); free(complete_avl_name);
close(avl_data_file_descriptor); close(avl_data_file_descriptor);
free(avl_data); free(avl_data);
@ -1808,7 +1804,6 @@ OBIDMS_avl_p obi_open_avl(OBIDMS_p dms, const char* avl_name, int avl_idx)
{ {
obi_set_errno(OBI_AVL_ERROR); obi_set_errno(OBI_AVL_ERROR);
obidebug(1, "\nError mmapping the data of an AVL tree data file"); obidebug(1, "\nError mmapping the data of an AVL tree data file");
if (avl_idx >= 0)
free(complete_avl_name); free(complete_avl_name);
munmap(avl_data->header, header_size); munmap(avl_data->header, header_size);
close(avl_data_file_descriptor); close(avl_data_file_descriptor);
@ -1825,7 +1820,7 @@ OBIDMS_avl_p obi_open_avl(OBIDMS_p dms, const char* avl_name, int avl_idx)
avl_file_name = build_avl_file_name(complete_avl_name); avl_file_name = build_avl_file_name(complete_avl_name);
if (avl_file_name == NULL) if (avl_file_name == NULL)
{ {
close_avl_data(avl_data); close_avl_data(avl_data, false);
return NULL; return NULL;
} }
@ -1835,9 +1830,8 @@ OBIDMS_avl_p obi_open_avl(OBIDMS_p dms, const char* avl_name, int avl_idx)
{ {
obi_set_errno(OBI_AVL_ERROR); obi_set_errno(OBI_AVL_ERROR);
obidebug(1, "\nError opening an AVL tree file"); obidebug(1, "\nError opening an AVL tree file");
if (avl_idx >= 0)
free(complete_avl_name); free(complete_avl_name);
close_avl_data(avl_data); close_avl_data(avl_data, false);
free(avl_file_name); free(avl_file_name);
return NULL; return NULL;
} }
@ -1849,9 +1843,8 @@ OBIDMS_avl_p obi_open_avl(OBIDMS_p dms, const char* avl_name, int avl_idx)
{ {
obi_set_errno(OBI_MALLOC_ERROR); obi_set_errno(OBI_MALLOC_ERROR);
obidebug(1, "\nError allocating the memory for the AVL tree structure"); obidebug(1, "\nError allocating the memory for the AVL tree structure");
if (avl_idx >= 0)
free(complete_avl_name); free(complete_avl_name);
close_avl_data(avl_data); close_avl_data(avl_data, false);
close(avl_file_descriptor); close(avl_file_descriptor);
return NULL; return NULL;
} }
@ -1861,8 +1854,8 @@ OBIDMS_avl_p obi_open_avl(OBIDMS_p dms, const char* avl_name, int avl_idx)
{ {
obi_set_errno(OBI_AVL_ERROR); obi_set_errno(OBI_AVL_ERROR);
obidebug(1, "\nError reading the header size to open an AVL tree"); obidebug(1, "\nError reading the header size to open an AVL tree");
if (avl_idx >= 0)
free(complete_avl_name); free(complete_avl_name);
close_avl_data(avl_data, false);
close(avl_file_descriptor); close(avl_file_descriptor);
return NULL; return NULL;
} }
@ -1879,9 +1872,8 @@ OBIDMS_avl_p obi_open_avl(OBIDMS_p dms, const char* avl_name, int avl_idx)
{ {
obi_set_errno(OBI_AVL_ERROR); obi_set_errno(OBI_AVL_ERROR);
obidebug(1, "\nError mmapping the header of an AVL tree file"); obidebug(1, "\nError mmapping the header of an AVL tree file");
if (avl_idx >= 0)
free(complete_avl_name); free(complete_avl_name);
close_avl_data(avl_data); close_avl_data(avl_data, false);
close(avl_file_descriptor); close(avl_file_descriptor);
free(avl); free(avl);
return NULL; return NULL;
@ -1898,9 +1890,8 @@ OBIDMS_avl_p obi_open_avl(OBIDMS_p dms, const char* avl_name, int avl_idx)
{ {
obi_set_errno(OBI_AVL_ERROR); obi_set_errno(OBI_AVL_ERROR);
obidebug(1, "\nError mmapping the data of an AVL tree file"); obidebug(1, "\nError mmapping the data of an AVL tree file");
if (avl_idx >= 0)
free(complete_avl_name); free(complete_avl_name);
close_avl_data(avl_data); close_avl_data(avl_data, false);
munmap(avl->header, header_size); munmap(avl->header, header_size);
close(avl_file_descriptor); close(avl_file_descriptor);
free(avl); free(avl);
@ -2141,12 +2132,13 @@ OBIDMS_avl_group_p obi_clone_avl_group(OBIDMS_avl_group_p avl_group, const char*
} }
int obi_close_avl(OBIDMS_avl_p avl) int obi_close_avl(OBIDMS_avl_p avl, bool writable)
{ {
int ret_val = 0; int ret_val = 0;
ret_val = close_avl_data(avl->data); ret_val = close_avl_data(avl->data, writable);
if (writable)
ret_val = truncate_avl_to_size_used(avl); ret_val = truncate_avl_to_size_used(avl);
if (munmap(avl->tree, (avl->header)->avl_size) < 0) if (munmap(avl->tree, (avl->header)->avl_size) < 0)
@ -2199,7 +2191,7 @@ int obi_close_avl_group(OBIDMS_avl_group_p avl_group)
if (remap_an_avl((avl_group->sub_avls)[i]) < 0) if (remap_an_avl((avl_group->sub_avls)[i]) < 0)
ret_val = -1; ret_val = -1;
} }
if (obi_close_avl((avl_group->sub_avls)[i]) < 0) if (obi_close_avl((avl_group->sub_avls)[i], avl_group->writable) < 0)
ret_val = -1; ret_val = -1;
} }
@ -2427,6 +2419,7 @@ index_t obi_avl_group_add(OBIDMS_avl_group_p avl_group, Obi_blob_p value)
} }
} }
// Check if already in current AVL
if (maybe_in_avl((avl_group->sub_avls)[avl_group->last_avl_idx], value)) if (maybe_in_avl((avl_group->sub_avls)[avl_group->last_avl_idx], value))
{ {
index_in_avl = (int32_t) obi_avl_find((avl_group->sub_avls)[avl_group->last_avl_idx], value); index_in_avl = (int32_t) obi_avl_find((avl_group->sub_avls)[avl_group->last_avl_idx], value);
@ -2442,12 +2435,12 @@ index_t obi_avl_group_add(OBIDMS_avl_group_p avl_group, Obi_blob_p value)
for (i=0; i < (avl_group->last_avl_idx); i++) for (i=0; i < (avl_group->last_avl_idx); i++)
{ {
if (maybe_in_avl((avl_group->sub_avls)[i], value)) if (maybe_in_avl((avl_group->sub_avls)[i], value))
{ { // AVLS are not unmapped and remapped anymore as it is very costly and keeping all mapped seems to be handled well
if (remap_an_avl((avl_group->sub_avls)[i]) < 0) //if (remap_an_avl((avl_group->sub_avls)[i]) < 0)
return -1; // return -1;
index_in_avl = (int32_t) obi_avl_find((avl_group->sub_avls)[i], value); index_in_avl = (int32_t) obi_avl_find((avl_group->sub_avls)[i], value);
if (unmap_an_avl((avl_group->sub_avls)[i]) < 0) //if (unmap_an_avl((avl_group->sub_avls)[i]) < 0)
return -1; // return -1;
if (index_in_avl >= 0) if (index_in_avl >= 0)
{ {
index_with_avl = i; index_with_avl = i;
@ -2468,7 +2461,7 @@ index_t obi_avl_group_add(OBIDMS_avl_group_p avl_group, Obi_blob_p value)
} }
// Check if need to make new AVL // Check if need to make new AVL
if (((((avl_group->sub_avls)[avl_group->last_avl_idx])->header)->nb_items == MAX_NODE_COUNT_PER_AVL) || (((((avl_group->sub_avls)[avl_group->last_avl_idx])->data)->header)->data_size_used >= MAX_DATA_SIZE_PER_AVL)) if (((((avl_group->sub_avls)[avl_group->last_avl_idx])->header)->nb_items == MAX_NODE_COUNT_PER_AVL) || ((((((avl_group->sub_avls)[avl_group->last_avl_idx])->data)->header)->data_size_used + obi_blob_sizeof(value)) >= MAX_DATA_SIZE_PER_AVL))
{ {
if (add_new_avl_in_group(avl_group) < 0) if (add_new_avl_in_group(avl_group) < 0)
return -1; return -1;