From 7f0f63cf2694f0a8b34f1e11abad56c6244bf40d Mon Sep 17 00:00:00 2001 From: MercierC Date: Thu, 11 Mar 2021 15:12:40 +1300 Subject: [PATCH] C: now completely unmapping files before truncating them to a smaller size (#68) --- src/obiavl.c | 117 ++++++++++++++++++++++++++++++++++++++++----- src/obidms.c | 19 ++++---- src/obidmscolumn.c | 35 ++++++++++++-- src/obiview.c | 19 ++++---- 4 files changed, 156 insertions(+), 34 deletions(-) diff --git a/src/obiavl.c b/src/obiavl.c index 16ef574..1ee9a8e 100755 --- a/src/obiavl.c +++ b/src/obiavl.c @@ -582,6 +582,7 @@ int truncate_avl_to_size_used(OBIDMS_avl_p avl) // TODO is it necessary to unmap { size_t file_size; size_t new_data_size; + size_t header_size; double multiple; int file_descriptor; @@ -589,6 +590,8 @@ int truncate_avl_to_size_used(OBIDMS_avl_p avl) // TODO is it necessary to unmap multiple = ceil((double) (ONE_IF_ZERO((avl->header)->nb_items * sizeof(AVL_node_t))) / (double) getpagesize()); new_data_size = ((size_t) multiple) * getpagesize(); + header_size = (avl->header)->header_size; + // Check that it is actually greater than the current size of the file, otherwise no need to truncate if ((avl->header)->avl_size == new_data_size) return 0; @@ -596,16 +599,22 @@ int truncate_avl_to_size_used(OBIDMS_avl_p avl) // TODO is it necessary to unmap // Get the file descriptor file_descriptor = avl->avl_fd; - // Unmap the tree before truncating the file + // Unmap the entire file before truncating it (WSL requirement) if (munmap(avl->tree, (avl->header)->avl_size) < 0) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError munmapping the tree of an AVL before truncating"); return -1; } + if (munmap(avl->header, header_size) < 0) + { + obi_set_errno(OBI_AVL_ERROR); + obidebug(1, "\nError munmapping the tree of an AVL before truncating"); + return -1; + } // Truncate the file - file_size = (avl->header)->header_size + new_data_size; + file_size = header_size + new_data_size; if (ftruncate(file_descriptor, file_size) < 0) { obi_set_errno(OBI_AVL_ERROR); @@ -613,7 +622,22 @@ int truncate_avl_to_size_used(OBIDMS_avl_p avl) // TODO is it necessary to unmap return -1; } - // Remap the data + // Remap the header and the data + + avl->header = mmap(NULL, + header_size, + PROT_READ | PROT_WRITE, + MAP_SHARED, + file_descriptor, + 0 + ); + if (avl->header == MAP_FAILED) + { + obi_set_errno(OBI_AVL_ERROR); + obidebug(1, "\nError re-mmapping the header of an AVL after truncating"); + return -1; + } + avl->tree = mmap(NULL, new_data_size, PROT_READ | PROT_WRITE, @@ -640,6 +664,7 @@ int truncate_avl_data_to_size_used(OBIDMS_avl_data_p avl_data) // TODO is it nec { size_t file_size; index_t new_data_size; + size_t header_size; double multiple; int file_descriptor; @@ -647,6 +672,8 @@ int truncate_avl_data_to_size_used(OBIDMS_avl_data_p avl_data) // TODO is it nec multiple = ceil((double) (ONE_IF_ZERO((avl_data->header)->data_size_used)) / (double) getpagesize()); new_data_size = ((index_t) multiple) * getpagesize(); + header_size = (avl_data->header)->header_size; + // Check that it is actually greater than the current size of the file, otherwise no need to truncate if ((avl_data->header)->data_size_max >= new_data_size) return 0; @@ -654,7 +681,8 @@ int truncate_avl_data_to_size_used(OBIDMS_avl_data_p avl_data) // TODO is it nec // Get the file descriptor file_descriptor = avl_data->data_fd; - // Unmap the data before truncating the file + // Unmap the entire file before truncating it (WSL requirement) + if (munmap(avl_data->data, (avl_data->header)->data_size_max) < 0) { obi_set_errno(OBI_AVL_ERROR); @@ -662,8 +690,15 @@ int truncate_avl_data_to_size_used(OBIDMS_avl_data_p avl_data) // TODO is it nec return -1; } + if (munmap(avl_data->header, header_size) < 0) + { + obi_set_errno(OBI_AVL_ERROR); + obidebug(1, "\nError munmapping the header of an AVL before truncating"); + return -1; + } + // Truncate the file - file_size = (avl_data->header)->header_size + new_data_size; + file_size = header_size + new_data_size; if (ftruncate(file_descriptor, file_size) < 0) { obi_set_errno(OBI_AVL_ERROR); @@ -672,6 +707,22 @@ int truncate_avl_data_to_size_used(OBIDMS_avl_data_p avl_data) // TODO is it nec } // Remap the data + + avl_data->header = mmap(NULL, + header_size, + PROT_READ | PROT_WRITE, + MAP_SHARED, + file_descriptor, + 0 + ); + + if (avl_data->header == MAP_FAILED) + { + obi_set_errno(OBI_AVL_ERROR); + obidebug(1, "\nError re-mmapping the header of an AVL after truncating"); + return -1; + } + avl_data->data = mmap(NULL, new_data_size, PROT_READ | PROT_WRITE, @@ -710,6 +761,20 @@ int grow_avl(OBIDMS_avl_p avl) // TODO Lock when needed header_size = (avl->header)->header_size; file_size = header_size + new_data_size; + // Unmap the entire file before truncating it (WSL requirement) + if (munmap(avl->tree, old_data_size) < 0) + { + obi_set_errno(OBI_AVL_ERROR); + obidebug(1, "\nError munmapping the tree of an AVL tree file before enlarging"); + return -1; + } + if (munmap(avl->header, header_size) < 0) + { + obi_set_errno(OBI_AVL_ERROR); + obidebug(1, "\nError munmapping the header of an AVL tree file before enlarging"); + return -1; + } + // Enlarge the file if (ftruncate(avl_file_descriptor, file_size) < 0) { @@ -718,12 +783,20 @@ int grow_avl(OBIDMS_avl_p avl) // TODO Lock when needed return -1; } - // Unmap and re-map the data + // Re-map - if (munmap(avl->tree, old_data_size) < 0) + avl->header = mmap(NULL, + header_size, + PROT_READ | PROT_WRITE, + MAP_SHARED, + avl_file_descriptor, + 0 + ); + + if (avl->header == MAP_FAILED) { obi_set_errno(OBI_AVL_ERROR); - obidebug(1, "\nError munmapping the tree of an AVL tree file before enlarging"); + obidebug(1, "\nError re-mmapping the header of an AVL tree file after enlarging the file"); return -1; } @@ -768,6 +841,20 @@ int grow_avl_data(OBIDMS_avl_data_p avl_data) // TODO Lock when needed header_size = (avl_data->header)->header_size; file_size = header_size + new_data_size; + // Unmap the entire file before truncating it (WSL requirement) + if (munmap(avl_data->data, old_data_size) < 0) + { + obi_set_errno(OBI_AVL_ERROR); + obidebug(1, "\nError munmapping the data of an AVL tree data file before enlarging"); + return -1; + } + if (munmap(avl_data->header, header_size) < 0) + { + obi_set_errno(OBI_AVL_ERROR); + obidebug(1, "\nError munmapping the header of an AVL tree data file before enlarging"); + return -1; + } + // Enlarge the file if (ftruncate(avl_data_file_descriptor, file_size) < 0) { @@ -776,12 +863,19 @@ int grow_avl_data(OBIDMS_avl_data_p avl_data) // TODO Lock when needed return -1; } - // Unmap and re-map the data + // Re-map - if (munmap(avl_data->data, old_data_size) < 0) + avl_data->header = mmap(NULL, + header_size, + PROT_READ | PROT_WRITE, + MAP_SHARED, + avl_data_file_descriptor, + 0 + ); + if (avl_data->header == MAP_FAILED) { obi_set_errno(OBI_AVL_ERROR); - obidebug(1, "\nError munmapping the data of an AVL tree data file before enlarging"); + obidebug(1, "\nError re-mmapping the header of an AVL tree data file after enlarging the file"); return -1; } @@ -792,7 +886,6 @@ int grow_avl_data(OBIDMS_avl_data_p avl_data) // TODO Lock when needed avl_data_file_descriptor, header_size ); - if (avl_data->data == MAP_FAILED) { obi_set_errno(OBI_AVL_ERROR); diff --git a/src/obidms.c b/src/obidms.c index c3b5fbb..ad66431 100755 --- a/src/obidms.c +++ b/src/obidms.c @@ -316,6 +316,15 @@ static int enlarge_infos_file(OBIDMS_p dms, size_t new_size) multiple = ceil((double) new_size / (double) getpagesize()); rounded_new_size = multiple * getpagesize(); + // Unmap the entire file before truncating it (WSL requirement) + if (munmap(dms->infos, (dms->infos)->file_size) < 0) + { + obi_set_errno(OBIDMS_UNKNOWN_ERROR); + obidebug(1, "\nError munmapping a DMS information file when enlarging"); + close(infos_file_descriptor); + return -1; + } + // Enlarge the file if (ftruncate(infos_file_descriptor, rounded_new_size) < 0) { @@ -325,15 +334,7 @@ static int enlarge_infos_file(OBIDMS_p dms, size_t new_size) return -1; } - // Unmap and remap the file - if (munmap(dms->infos, (dms->infos)->file_size) < 0) - { - obi_set_errno(OBIDMS_UNKNOWN_ERROR); - obidebug(1, "\nError munmapping a DMS information file when enlarging"); - close(infos_file_descriptor); - return -1; - } - + // Remap the file dms->infos = mmap(NULL, rounded_new_size, PROT_READ | PROT_WRITE, diff --git a/src/obidmscolumn.c b/src/obidmscolumn.c index 93200b0..14894eb 100644 --- a/src/obidmscolumn.c +++ b/src/obidmscolumn.c @@ -1769,6 +1769,7 @@ int obi_truncate_column(OBIDMS_column_p column) // TODO is it necessary to unmap { size_t file_size; size_t data_size; + size_t header_size; index_t new_line_count; double multiple; int column_file_descriptor; @@ -1791,6 +1792,8 @@ int obi_truncate_column(OBIDMS_column_p column) // TODO is it necessary to unmap data_size = obi_array_sizeof((column->header)->stored_data_type, new_line_count, (column->header)->nb_elements_per_line); + header_size = (column->header)->header_size; + // Check that it is actually greater than the current data size, otherwise no need to truncate if ((column->header)->data_size == data_size) return 0; @@ -1855,7 +1858,7 @@ int obi_truncate_column(OBIDMS_column_p column) // TODO is it necessary to unmap return -1; } - // Unmap the data before truncating the file + // Unmap the entire file before truncating it (WSL requirement) if (munmap(column->data, (column->header)->data_size) < 0) { obi_set_errno(OBICOL_UNKNOWN_ERROR); @@ -1863,9 +1866,16 @@ int obi_truncate_column(OBIDMS_column_p column) // TODO is it necessary to unmap close(column_file_descriptor); return -1; } + if (munmap(column->header, header_size) < 0) + { + obi_set_errno(OBICOL_UNKNOWN_ERROR); + obidebug(1, "\nError munmapping the header of a column before truncating"); + close(column_file_descriptor); + return -1; + } // Truncate the column file - file_size = (column->header)->header_size + data_size; + file_size = header_size + data_size; if (ftruncate(column_file_descriptor, file_size) < 0) { obi_set_errno(OBICOL_UNKNOWN_ERROR); @@ -1874,13 +1884,30 @@ int obi_truncate_column(OBIDMS_column_p column) // TODO is it necessary to unmap return -1; } - // Remap the data + // Remap the header and the data + + column->header = mmap(NULL, + header_size, + PROT_READ | PROT_WRITE, + MAP_SHARED, + column_file_descriptor, + 0 + ); + + if (column->header == MAP_FAILED) + { + obi_set_errno(OBICOL_UNKNOWN_ERROR); + obidebug(1, "\nError re-mmapping the header of a column after truncating"); + close(column_file_descriptor); + return -1; + } + column->data = mmap(NULL, data_size, PROT_READ | PROT_WRITE, MAP_SHARED, column_file_descriptor, - (column->header)->header_size + header_size ); if (column->data == MAP_FAILED) diff --git a/src/obiview.c b/src/obiview.c index e259fca..f2c741f 100755 --- a/src/obiview.c +++ b/src/obiview.c @@ -638,6 +638,15 @@ static int enlarge_view_file(Obiview_p view, size_t new_size) multiple = ceil((double) new_size / (double) getpagesize()); rounded_new_size = multiple * getpagesize(); + // Unmap the entire file before truncating it (WSL requirement) + if (munmap(view->infos, (view->infos)->file_size) < 0) + { + obi_set_errno(OBIVIEW_ERROR); + obidebug(1, "\nError munmapping a view file when enlarging"); + close(obiview_file_descriptor); + return -1; + } + // Enlarge the file if (ftruncate(obiview_file_descriptor, rounded_new_size) < 0) { @@ -647,15 +656,7 @@ static int enlarge_view_file(Obiview_p view, size_t new_size) return -1; } - // Unmap and remap the file - if (munmap(view->infos, (view->infos)->file_size) < 0) - { - obi_set_errno(OBIVIEW_ERROR); - obidebug(1, "\nError munmapping a view file when enlarging"); - close(obiview_file_descriptor); - return -1; - } - + // Remap the file view->infos = mmap(NULL, rounded_new_size, PROT_READ | PROT_WRITE,