From dfd51939a03a13b6b81beaa34668f3967dd76cbe Mon Sep 17 00:00:00 2001 From: Celine Mercier Date: Thu, 26 Oct 2017 18:58:48 +0200 Subject: [PATCH] Views are now rollbacked if an error occurs, and unfinished views and columns are deleted when an OBIDMS is opened. --- python/obitools3/dms/capi/obiview.pxd | 4 + python/obitools3/dms/dms.pyx | 7 +- python/obitools3/dms/view/__init__.py | 1 + python/obitools3/dms/view/view.pyx | 30 ++- src/obidms.c | 24 +++ src/obidmscolumn.c | 250 ++++++++++++++++++++++- src/obidmscolumn.h | 57 ++++++ src/obidmscolumndir.c | 46 +---- src/obidmscolumndir.h | 19 ++ src/obiview.c | 282 +++++++++++++++++++++++++- src/obiview.h | 37 ++++ 11 files changed, 702 insertions(+), 55 deletions(-) diff --git a/python/obitools3/dms/capi/obiview.pxd b/python/obitools3/dms/capi/obiview.pxd index ca34466..4a10c34 100644 --- a/python/obitools3/dms/capi/obiview.pxd +++ b/python/obitools3/dms/capi/obiview.pxd @@ -102,6 +102,10 @@ cdef extern from "obiview.h" nogil: int obi_save_and_close_view(Obiview_p view) + int obi_clean_unfinished_views(OBIDMS_p dms) + + int obi_rollback_view(Obiview_p view) + # OBI_INT int obi_set_int_with_elt_name_and_col_p_in_view(Obiview_p view, diff --git a/python/obitools3/dms/dms.pyx b/python/obitools3/dms/dms.pyx index 4e3de64..b39c6ca 100644 --- a/python/obitools3/dms/dms.pyx +++ b/python/obitools3/dms/dms.pyx @@ -28,11 +28,11 @@ from .object import OBIWrapper cdef class DMS(OBIWrapper): - cdef inline OBIDMS_p pointer(self): + cdef inline OBIDMS_p pointer(self) : return (self._pointer) @staticmethod - def obi_atexit(): + def obi_atexit() : atexit(obi_close_atexit) @staticmethod @@ -50,7 +50,7 @@ cdef class DMS(OBIWrapper): return dms @staticmethod - def exists(object dms_name): + def exists(object dms_name) : cdef bytes dms_name_b = tobytes(dms_name) cdef int rep rep = obi_dms_exists(dms_name_b) @@ -90,7 +90,6 @@ cdef class DMS(OBIWrapper): The `close` method is automatically called by the object destructor. ''' cdef OBIDMS_p pointer = self.pointer() - if self.active() : OBIWrapper.close(self) if (obi_close_dms(pointer)) < 0 : diff --git a/python/obitools3/dms/view/__init__.py b/python/obitools3/dms/view/__init__.py index c1c23ad..ab7f83d 100644 --- a/python/obitools3/dms/view/__init__.py +++ b/python/obitools3/dms/view/__init__.py @@ -1,2 +1,3 @@ from .view import View # @UnresolvedImport from .view import Line_selection # @UnresolvedImport +from .view import RollbackException # @UnresolvedImport \ No newline at end of file diff --git a/python/obitools3/dms/view/view.pyx b/python/obitools3/dms/view/view.pyx index 643c563..1370d92 100644 --- a/python/obitools3/dms/view/view.pyx +++ b/python/obitools3/dms/view/view.pyx @@ -10,6 +10,7 @@ from ..capi.obiview cimport Alias_column_pair_p, \ obi_new_view, \ obi_open_view, \ obi_clone_view, \ + obi_rollback_view, \ obi_save_and_close_view, \ obi_view_get_pointer_on_column_in_view, \ obi_view_delete_column, \ @@ -69,7 +70,7 @@ cdef class View(OBIWrapper) : cdef bytes view_name_b = tobytes(view_name) cdef bytes comments_b - cdef str message + cdef str message cdef void* pointer cdef View view # @DuplicatedSignature @@ -103,7 +104,7 @@ cdef class View(OBIWrapper) : cdef bytes view_name_b = tobytes(view_name) cdef bytes comments_b cdef void* pointer - cdef View view + cdef View view if not self.active() : raise OBIDeactivatedInstanceError() @@ -231,8 +232,8 @@ cdef class View(OBIWrapper) : # Remove the column from the view which closes the C structure if obi_view_delete_column(self.pointer(), column_name_b) < 0 : - raise Exception("Problem deleting column %s from a view", - bytes2str(column_name_b)) + raise RollbackException("Problem deleting column %s from a view", + bytes2str(column_name_b), self) cpdef rename_column(self, @@ -249,9 +250,9 @@ cdef class View(OBIWrapper) : if (obi_view_create_column_alias(self.pointer(), tobytes(current_name_b), tobytes(new_name_b)) < 0) : - raise Exception("Problem in renaming column %s to %s" % ( + raise RollbackException("Problem in renaming column %s to %s" % ( bytes2str(current_name_b), - bytes2str(new_name_b))) + bytes2str(new_name_b)), self) # TODO warning, not multithreading compliant @@ -434,7 +435,7 @@ cdef class Line : if column_name_b not in self._view : if value == None : - raise Exception("Trying to create a column from a None value (can't guess type)") + raise RollbackException("Trying to create a column from a None value (can't guess type)", self) value_type = type(value) if value_type == int : value_obitype = OBI_INT @@ -454,7 +455,7 @@ cdef class Line : elif (len(value) > 1) : value_obitype = OBI_STR else : - raise Exception("Could not guess the type of a value to create a new column") + raise RollbackException("Could not guess the type of a value to create a new column", self) Column.new_column(self._view, column_name_b, value_obitype) @@ -622,6 +623,19 @@ cdef class Line_selection(list): ############################################################# +class RollbackException(Exception): + def __init__(self, message, View view): + super(RollbackException, self).__init__(message) + if obi_rollback_view((view.pointer())) < 0 : + raise Exception("Error rollbacking view") + if view.active() : + view._dms.unregister(view) + OBIWrapper.close(view) + + +############################################################# + + cdef register_view_class(bytes view_type_name, type view_class): ''' diff --git a/src/obidms.c b/src/obidms.c index 9228820..0af9fb2 100644 --- a/src/obidms.c +++ b/src/obidms.c @@ -587,6 +587,30 @@ OBIDMS_p obi_open_dms(const char* dms_path) return NULL; } + // Clean unfinished views + if (obi_clean_unfinished_views(dms) < 0) + { + obidebug(1, "\nError cleaning unfinished views when opening an OBIDMS"); + closedir(dms->indexer_directory); + closedir(dms->tax_directory); + closedir(dms->view_directory); + closedir(dms->directory); + free(dms); + return NULL; + } + + // Clean unfinished columns + if (obi_clean_unfinished_columns(dms) < 0) + { + obidebug(1, "\nError cleaning unfinished columns when opening an OBIDMS"); + closedir(dms->indexer_directory); + closedir(dms->tax_directory); + closedir(dms->view_directory); + closedir(dms->directory); + free(dms); + return NULL; + } + // Initialize the list of opened columns dms->opened_columns = (Opened_columns_list_p) malloc(sizeof(Opened_columns_list_t)); (dms->opened_columns)->nb_opened_columns = 0; diff --git a/src/obidmscolumn.c b/src/obidmscolumn.c index d01c1ce..1f3ad91 100644 --- a/src/obidmscolumn.c +++ b/src/obidmscolumn.c @@ -275,7 +275,6 @@ static char* build_column_file_name(const char* column_name, obiversion_t versio } - static char* build_version_file_name(const char* column_name) { char* file_name; @@ -300,7 +299,6 @@ static char* build_version_file_name(const char* column_name) } - static obiversion_t obi_get_new_version_number(OBIDMS_column_directory_p column_directory, bool block) { off_t loc_size; @@ -425,7 +423,6 @@ static obiversion_t obi_get_new_version_number(OBIDMS_column_directory_p column_ } - static obiversion_t create_version_file(OBIDMS_column_directory_p column_directory) { off_t loc_size; @@ -714,6 +711,71 @@ static index_t get_line_count_per_page(OBIType_t data_type, index_t nb_elements_ **********************************************************************/ +char* obi_version_file_full_path(OBIDMS_p dms, const char* column_name) +{ + char* version_file_name; + char* column_dir_name; + char* relative_path; + char* full_path; + + version_file_name = build_version_file_name(column_name); + if (version_file_name == NULL) + return NULL; + + column_dir_name = obi_build_column_directory_name(column_name); + if (column_dir_name == NULL) + return NULL; + + relative_path = (char*) malloc(strlen(version_file_name) + strlen(column_dir_name) + 2); + + strcpy(relative_path, column_dir_name); + strcat(relative_path, "/"); + strcat(relative_path, version_file_name); + + // Build path relative to DMS + full_path = obi_dms_get_full_path(dms, relative_path); + + free(version_file_name); + free(column_dir_name); + free(relative_path); + + return full_path; +} + + +char* obi_column_full_path(OBIDMS_p dms, const char* column_name, obiversion_t version_number) +{ + char* column_file_name; + char* column_dir_name; + char* relative_path; + char* full_path; + + + column_file_name = build_column_file_name(column_name, version_number); + if (column_file_name == NULL) + return NULL; + + column_dir_name = obi_build_column_directory_name(column_name); + if (column_dir_name == NULL) + return NULL; + + relative_path = (char*) malloc(strlen(column_file_name) + strlen(column_dir_name) + 2); + + strcpy(relative_path, column_dir_name); + strcat(relative_path, "/"); + strcat(relative_path, column_file_name); + + // Build path relative to DMS + full_path = obi_dms_get_full_path(dms, relative_path); + + free(column_file_name); + free(column_dir_name); + free(relative_path); + + return full_path; +} + + obiversion_t obi_get_latest_version_number(OBIDMS_column_directory_p column_directory) { off_t loc_size; @@ -1046,6 +1108,7 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms, header->creation_date = time(NULL); header->version = version_number; header->cloned_from = -1; + header->finished = false; set_elements_names(new_column, elements_names, elts_names_length); @@ -1833,7 +1896,6 @@ index_t obi_column_get_element_index_from_name(OBIDMS_column_p column, const cha } -// TODO doc, returns elements names with ; as separator (discuss maybe char**) char* obi_get_elements_names(OBIDMS_column_p column) { char* elements_names; @@ -1921,3 +1983,183 @@ int obi_column_prepare_to_get_value(OBIDMS_column_p column, index_t line_nb) return 0; } + +int obi_clean_unfinished_columns(OBIDMS_p dms) +{ + struct dirent* dms_dirent; + struct dirent* col_dirent; + DIR* col_dir; + int i,j; + char* column_file_path; + char* column_dir_path; + char* col_name; + char* col_version_str; + char* version_file; + obiversion_t col_version; + OBIDMS_column_header_p col_header; + int n; + char* col_to_delete[1000]; + char* dir_to_delete[1000]; + int ddir; + int dcol; + int d; + int ret_value; + + ret_value = 0; + + // Find column directories + ddir = 0; + while ((dms_dirent = readdir(dms->directory)) != NULL) + { + if ((dms_dirent->d_name)[0] == '.') + continue; + i=0; + while (((dms_dirent->d_name)[i] != '.') && (i < strlen(dms_dirent->d_name))) + i++; + if ((i != strlen(dms_dirent->d_name)) && (strcmp((dms_dirent->d_name)+i, ".obicol") == 0)) // Found a column directory + { + column_dir_path = obi_dms_get_full_path(dms, dms_dirent->d_name); + if (column_dir_path == NULL) + { + obidebug(1, "\nError getting a column directory path when deleting unfinished columns"); + ret_value = -1; + } + col_name = (char*) malloc(strlen(dms_dirent->d_name) * sizeof(char)); + if (col_name == NULL) + { + obi_set_errno(OBI_MALLOC_ERROR); + obidebug(1, "\nError allocating memory for a column name when deleting unfinished columns: directory %s", dms_dirent->d_name); + ret_value = -1; + continue; + } + strncpy(col_name, dms_dirent->d_name, i); + col_name[i] = '\0'; + col_dir = opendir_in_dms(dms, dms_dirent->d_name); + if (col_dir == NULL) + { + obidebug(1, "\nError opening a column directory when deleting unfinished columns"); + ret_value = -1; + continue; + } + + // Iteration on files of this column directory + dcol = 0; + while ((col_dirent = readdir(col_dir)) != NULL) + { + if ((col_dirent->d_name)[0] == '.') + continue; + i=0; + j=0; + while (((col_dirent->d_name)[i] != '@') && ((col_dirent->d_name)[i] != '.')) + i++; + if ((col_dirent->d_name)[i] == '@') // Found a column file + { + j=i; + while ((col_dirent->d_name)[j] != '.') + j++; + col_version_str = (char*) malloc(strlen(col_dirent->d_name) * sizeof(char)); + if (col_version_str == NULL) + { + obi_set_errno(OBI_MALLOC_ERROR); + obidebug(1, "\nError allocating memory for a column version when deleting unfinished columns: directory %s", dms_dirent->d_name); + ret_value = -1; + continue; + } + strncpy(col_version_str, (col_dirent->d_name)+i, j-i); + col_version_str[j-i] = '\0'; + col_version = atoi(col_version_str); + free(col_version_str); + col_header = obi_column_get_header_from_name(dms, col_name, col_version); + if (col_header == NULL) // TODO discuss if delete file or not + { + obidebug(1, "\nError reading a column header when deleting unfinished columns: file %s", col_dirent->d_name); + ret_value = -1; + continue; + } + + // Check if the column is finished and delete it if not + if (col_header->finished == false) + { + // Build file and dir paths + column_file_path = obi_column_full_path(dms, col_name, col_version); + if (column_file_path == NULL) + { + obidebug(1, "\nError getting a column file path when deleting unfinished columns"); + ret_value = -1; + continue; + } + + // Add the column path in the list of files to delete (can't delete while in loop) + col_to_delete[dcol] = column_file_path; + dcol++; + } + // Close the header + if (obi_close_header(col_header) < 0) + ret_value = -1; + } + } + + // Delete all column files in to_delete list + for (d=0; dread_only) @@ -1111,6 +1113,19 @@ static int finish_view(Obiview_p view) if (rename_finished_view(view) < 0) return -1; + // Flag the columns as finished + for (i=0; i < ((view->infos)->column_count); i++) + { + column = *((OBIDMS_column_p*)ll_get(view->columns, i)); + if (column == NULL) + { + obi_set_errno(OBIVIEW_ERROR); + obidebug(1, "\nError getting a column to flag it as finished when finishing a view"); + return -1; + } + (column->header)->finished = true; + } + // Flag the view as finished (view->infos)->finished = true; @@ -2502,6 +2517,271 @@ int obi_save_and_close_view(Obiview_p view) } +int obi_clean_unfinished_views(OBIDMS_p dms) +{ + struct dirent* dp; + int i; + char* full_path; + char* relative_path; + Obiview_infos_p view_infos; + char* view_name; + int ret_value; + char* to_delete[1000]; + int d; + + ret_value = 0; + d = 0; + + // Look for unfinished views and delete them + while ((dp = readdir(dms->view_directory)) != NULL) + { + if ((dp->d_name)[0] == '.') + continue; + i=0; + while ((dp->d_name)[i] != '.') + i++; + relative_path = (char*) malloc(strlen(VIEW_DIR_NAME) + strlen(dp->d_name) + 2); + strcpy(relative_path, VIEW_DIR_NAME); + strcat(relative_path, "/"); + strcat(relative_path, dp->d_name); + full_path = obi_dms_get_full_path(dms, relative_path); + free(relative_path); + if (full_path == NULL) + { + obidebug(1, "\nError getting the full path to a view file when cleaning unfinished views"); + ret_value = -1; + continue; + } + if (strcmp((dp->d_name)+i, ".obiview_unfinished") == 0) + { + // Add to the list of files to delete (deleting in loop not safe) + to_delete[d] = full_path; + d++; + } + else if (strcmp((dp->d_name)+i, ".obiview") == 0) + { // Check if the view was properly flagged as finished + view_name = (char*) malloc((i+1) * sizeof(char)); + if (view_name == NULL) + { + obi_set_errno(OBI_MALLOC_ERROR); + obidebug(1, "\nError allocating memory for a view name when deleting unfinished views: file %s", dp->d_name); + ret_value = -1; + continue; + } + strncpy(view_name, dp->d_name, i); + view_name[i] = '\0'; + view_infos = obi_view_map_file(dms, view_name, true); + if (view_infos == NULL) + { + obidebug(1, "\nError reading a view file when deleting unfinished views: file %s", dp->d_name); + ret_value = -1; + continue; + } + if (view_infos->finished == false) + { + // Add to the list of files to delete (deleting in loop not safe) + to_delete[d] = full_path; + d++; + } + } + + for (i=0; iread_only) + return ret_value; + + for (i=0; i<((view->infos)->column_count); i++) + { + column = *((OBIDMS_column_p*)ll_get(view->columns, i)); + if (column == NULL) + { + obi_set_errno(OBIVIEW_ERROR); + obidebug(1, "\nError getting a column from the linked list of column pointers of a view when rollbacking the view"); + ret_value = -1; + continue; + } + + // Delete the column file if it was created by the view (it was if it is writable) + if (column->writable) + { + // Build file and dir paths + column_file_path = obi_column_full_path(view->dms, (column->header)->name, (column->header)->version); + if (column_file_path == NULL) + { + obidebug(1, "\nError getting a column file path when rollbacking a view"); + ret_value = -1; + continue; + } + column_dir_name = obi_build_column_directory_name((column->header)->name); + if (column_dir_name == NULL) + { + obidebug(1, "\nError getting a column directory name when rollbacking a view"); + ret_value = -1; + } + column_dir_path = obi_dms_get_full_path(view->dms, column_dir_name); + if (column_dir_path == NULL) + { + obidebug(1, "\nError getting a column directory path when rollbacking a view"); + ret_value = -1; + } + + // Try to close the column (?) + if (obi_close_column(column) < 0) + ret_value = -1; + + // Delete the column file + if (remove(column_file_path) < 0) + { + obi_set_errno(OBIVIEW_ERROR); + obidebug(1, "\nError deleting a column file when rollbacking a view"); + ret_value = -1; + } + + // Delete column dir if it's empty + n = count_dir(column_dir_path); + if (n == 0) + { + if (remove(column_dir_path) < 0) + { + obi_set_errno(OBIVIEW_ERROR); + obidebug(1, "\nError deleting a column directory when rollbacking a view"); + ret_value = -1; + } + } + free(column_file_path); + free(column_dir_name); + free(column_dir_path); + } + } + + // Delete line selection if there is one + if (view->line_selection != NULL) + { + column = view->line_selection; + if (column->writable) + { + // Build file and dir paths + column_file_path = obi_column_full_path(view->dms, (column->header)->name, (column->header)->version); + if (column_file_path == NULL) + { + obidebug(1, "\nError getting a column file path when rollbacking a view"); + ret_value = -1; + } + column_dir_name = obi_build_column_directory_name((column->header)->name); + if (column_dir_name == NULL) + { + obidebug(1, "\nError getting a column directory name when rollbacking a view"); + ret_value = -1; + } + column_dir_path = obi_dms_get_full_path(view->dms, column_dir_name); + if (column_dir_path == NULL) + { + obidebug(1, "\nError getting a column directory path when rollbacking a view"); + ret_value = -1; + } + + // Try to close the column (?) + if (obi_close_column(column) < 0) + ret_value = -1; + + // Delete the column file + if (remove(column_file_path) < 0) + { + obi_set_errno(OBIVIEW_ERROR); + obidebug(1, "\nError deleting a column file when rollbacking a view"); + ret_value = -1; + } + + // Delete column dir if it's empty + n = count_dir(column_dir_path); + if (n == 0) + { + if (remove(column_dir_path) < 0) + { + obi_set_errno(OBIVIEW_ERROR); + obidebug(1, "\nError deleting a column directory when rollbacking a view"); + ret_value = -1; + } + } + free(column_file_path); + free(column_dir_name); + free(column_dir_path); + } + } + + // Delete view file + view_file_name = (char*) malloc(strlen((view->infos)->name) + strlen(".obiview_unfinished") + 1); + if (view_file_name == NULL) + { + obi_set_errno(OBI_MALLOC_ERROR); + obidebug(1, "\nError allocating memory for a view file name"); + ret_value = -1; + } + else + { + strcpy(view_file_name, (view->infos)->name); + strcat(view_file_name, ".obiview_unfinished"); + while ((dp = readdir((view->dms)->view_directory)) != NULL) + { + if ((dp->d_name)[0] == '.') + continue; + if (strcmp(dp->d_name, view_file_name) == 0) + { + view_relative_path = (char*) malloc(strlen(VIEW_DIR_NAME) + strlen(view_file_name) + 2); + strcpy(view_relative_path, VIEW_DIR_NAME); + strcat(view_relative_path, "/"); + strcat(view_relative_path, view_file_name); + view_full_path = obi_dms_get_full_path(view->dms, view_relative_path); + remove(view_full_path); + } + } + free(view_file_name); + free(view_relative_path); + free(view_full_path); + } + + // Free the linked list of column pointers + ll_free(view->columns); + + // Free the column dictionary + ht_free(view->column_dict); + + free(view); + + return ret_value; +} + + int obi_create_auto_count_column(Obiview_p view) { index_t i; diff --git a/src/obiview.h b/src/obiview.h index 93297a4..162afaa 100644 --- a/src/obiview.h +++ b/src/obiview.h @@ -538,6 +538,43 @@ int obi_select_lines(Obiview_p view, index_t* line_nbs); int obi_save_and_close_view(Obiview_p view); +/** + * @brief Goes through all the view files of a DMS and deletes views that have + * not been flagged as finished (file extension renamed from '.obiview_unfinished' + * to '.obiview' and finished boolean set to true in the file, done by finish_view()). + * + * @param dms A pointer on an OBIDMS. + * + * @returns A value indicating the success of the operation. + * @retval 0 if the operation was successfully completed. + * @retval -1 if an error occurred. + * + * @since October 2017 + * @author Celine Mercier (celine.mercier@metabarcoding.org) + */ +int obi_clean_unfinished_views(OBIDMS_p dms); + + +/** + * @brief Frees and deletes an opened, writable view and the columns it created. + * + * The view and column files are deleted and the structures are freed. + * + * @warning The view pointer becomes invalid, as well as the pointers on + * the columns created by the view. + * + * @param view A pointer on the writable view to rollback. + * + * @returns A value indicating the success of the operation. + * @retval 0 if the operation was successfully completed. + * @retval -1 if an error occurred. + * + * @since October 2017 + * @author Celine Mercier (celine.mercier@metabarcoding.org) + */ +int obi_rollback_view(Obiview_p view); + + /** * @brief Creates an OBI_INT column with the line count of the view it belongs to, and sets all lines to 1. *