From 783a1343c437bd60cbe53a5acaff2f39d0c58314 Mon Sep 17 00:00:00 2001 From: Celine Mercier Date: Fri, 20 Sep 2019 20:37:19 +0200 Subject: [PATCH] DMS are now locked when used by a command. Added checks and changed cleaning mechanisms. --- python/obitools3/dms/capi/obidms.pxd | 6 +- python/obitools3/dms/capi/obierrno.pxd | 2 + python/obitools3/dms/dms.pyx | 4 +- src/build_reference_db.c | 2 +- src/obi_ecopcr.c | 4 +- src/obi_ecotag.c | 6 +- src/obi_lcs.c | 4 +- src/obidms.c | 117 +++++++++++++++++++++--- src/obidms.h | 36 +++++++- src/obidmscolumn.c | 121 +++++++++++++++++++++++++ src/obidmscolumn.h | 15 +++ src/obierrno.h | 6 +- src/obiview.c | 68 ++++++++++++++ src/obiview.h | 15 +++ 14 files changed, 378 insertions(+), 28 deletions(-) diff --git a/python/obitools3/dms/capi/obidms.pxd b/python/obitools3/dms/capi/obidms.pxd index 516c596..cedfa70 100755 --- a/python/obitools3/dms/capi/obidms.pxd +++ b/python/obitools3/dms/capi/obidms.pxd @@ -9,6 +9,7 @@ cdef extern from "obidms.h" nogil: bint little_endian size_t file_size size_t used_size + bint working const_char_p comments ctypedef OBIDMS_infos_t* OBIDMS_infos_p @@ -21,9 +22,10 @@ cdef extern from "obidms.h" nogil: ctypedef OBIDMS_t* OBIDMS_p - + int obi_dms_is_clean(OBIDMS_p dms) + int obi_clean_dms(const_char_p dms_path) OBIDMS_p obi_dms(const_char_p dms_name) - OBIDMS_p obi_open_dms(const_char_p dms_path) + OBIDMS_p obi_open_dms(const_char_p dms_path, bint cleaning) OBIDMS_p obi_test_open_dms(const_char_p dms_path) OBIDMS_p obi_create_dms(const_char_p dms_path) int obi_dms_exists(const char* dms_path) diff --git a/python/obitools3/dms/capi/obierrno.pxd b/python/obitools3/dms/capi/obierrno.pxd index b67da92..906eb00 100755 --- a/python/obitools3/dms/capi/obierrno.pxd +++ b/python/obitools3/dms/capi/obierrno.pxd @@ -7,3 +7,5 @@ cdef extern from "obierrno.h": extern int OBI_LINE_IDX_ERROR extern int OBI_ELT_IDX_ERROR extern int OBIVIEW_ALREADY_EXISTS_ERROR + extern int OBIDMS_NOT_CLEAN + extern int OBIDMS_WORKING \ No newline at end of file diff --git a/python/obitools3/dms/dms.pyx b/python/obitools3/dms/dms.pyx index dab267f..06b93c9 100755 --- a/python/obitools3/dms/dms.pyx +++ b/python/obitools3/dms/dms.pyx @@ -50,7 +50,7 @@ cdef class DMS(OBIWrapper): cdef DMS dms cdef bytes dms_name_b = tobytes(dms_name) if DMS.exists(dms_name_b) : - pointer = obi_open_dms( dms_name_b) + pointer = obi_open_dms( dms_name_b, False) else : pointer = obi_create_dms( dms_name_b) if pointer == NULL : @@ -87,7 +87,7 @@ cdef class DMS(OBIWrapper): cdef OBIDMS_p pointer cdef DMS dms cdef bytes dms_name_b = tobytes(dms_name) - pointer = obi_open_dms( dms_name_b) + pointer = obi_open_dms( dms_name_b, False) if pointer == NULL : raise Exception("Failed opening an OBIDMS") dms = OBIWrapper.new_wrapper(DMS, pointer) diff --git a/src/build_reference_db.c b/src/build_reference_db.c index 7a048c0..0771489 100755 --- a/src/build_reference_db.c +++ b/src/build_reference_db.c @@ -212,7 +212,7 @@ int build_reference_db(const char* dms_name, // Clone the matrix view // Open the DMS - dms = obi_open_dms(dms_name); + dms = obi_open_dms(dms_name, false); if (dms == NULL) { obidebug(1, "\nError opening the DMS when building a reference database"); diff --git a/src/obi_ecopcr.c b/src/obi_ecopcr.c index c732361..b1e004b 100755 --- a/src/obi_ecopcr.c +++ b/src/obi_ecopcr.c @@ -749,7 +749,7 @@ int obi_ecopcr(const char* i_dms_name, tm = (tm1 < tm2) ? tm1:tm2; // Open input DMS - i_dms = obi_open_dms(i_dms_name); + i_dms = obi_open_dms(i_dms_name, false); if (i_dms == NULL) { obidebug(1, "\nError opening the input DMS"); @@ -765,7 +765,7 @@ int obi_ecopcr(const char* i_dms_name, } // Open output DMS - o_dms = obi_open_dms(o_dms_name); + o_dms = obi_open_dms(o_dms_name, false); if (o_dms == NULL) { obidebug(1, "\nError opening the output DMS"); diff --git a/src/obi_ecotag.c b/src/obi_ecotag.c index 407a44c..8e29a71 100755 --- a/src/obi_ecotag.c +++ b/src/obi_ecotag.c @@ -274,7 +274,7 @@ int obi_ecotag(const char* dms_name, best_match_ids_buffer_size = 1024; // Open main DMS containing the query sequences and where the output will be written - dms = obi_open_dms(dms_name); + dms = obi_open_dms(dms_name, false); if (dms == NULL) { obidebug(1, "\nError opening the DMS containing the query sequences for ecotag"); @@ -282,7 +282,7 @@ int obi_ecotag(const char* dms_name, } // Open the DMS containing the reference database (can be the same or not) - ref_dms = obi_open_dms(ref_dms_name); + ref_dms = obi_open_dms(ref_dms_name, false); if (ref_dms == NULL) { obidebug(1, "\nError opening the DMS containing the reference database for ecotag"); @@ -290,7 +290,7 @@ int obi_ecotag(const char* dms_name, } // Open the DMS containing the taxonomy (can be the same or not) - taxo_dms = obi_open_dms(taxo_dms_name); + taxo_dms = obi_open_dms(taxo_dms_name, false); if (taxo_dms == NULL) { obidebug(1, "\nError opening the DMS containing the taxonomy for ecotag"); diff --git a/src/obi_lcs.c b/src/obi_lcs.c index 8174850..3a3ca60 100755 --- a/src/obi_lcs.c +++ b/src/obi_lcs.c @@ -431,7 +431,7 @@ int obi_lcs_align_one_column(const char* dms_name, k = 0; // Open DMS - dms = obi_open_dms(dms_name); + dms = obi_open_dms(dms_name, false); if (dms == NULL) { obidebug(1, "\nError opening the DMS"); @@ -728,7 +728,7 @@ int obi_lcs_align_two_columns(const char* dms_name, k = 0; // Open DMS - dms = obi_open_dms(dms_name); + dms = obi_open_dms(dms_name, false); if (dms == NULL) { obidebug(1, "\nError opening the DMS to align"); diff --git a/src/obidms.c b/src/obidms.c index b7725d3..28f6309 100755 --- a/src/obidms.c +++ b/src/obidms.c @@ -554,6 +554,7 @@ static int create_dms_infos_file(int dms_file_descriptor, const char* dms_name) infos->little_endian = obi_is_little_endian(); infos->file_size = file_size; infos->used_size = 0; + infos->working = false; infos->comments[0] = '\0'; // Unmap the infos file @@ -652,6 +653,52 @@ static int dms_count_in_list(OBIDMS_p dms) * **********************************************************************/ +int obi_dms_is_clean(OBIDMS_p dms) +{ + int ret_val1; + int ret_val2; + + ret_val1 = obi_dms_has_unfinished_views(dms); + if (ret_val1 < 0) + return -1; + ret_val2 = obi_dms_has_unfinished_columns(dms); + if (ret_val2 < 0) + return -1; + + return !(ret_val1 || ret_val2); +} + + +int obi_clean_dms(const char* dms_path) +{ + OBIDMS_p dms; + + dms = obi_open_dms(dms_path, true); + if (dms == NULL) + { + obidebug(1, "\nError opening a DMS before cleaning unfinished views and columns"); + return -1; + } + + // Currently done in obi_open_dms +// // Clean unfinished views +// if (obi_clean_unfinished_views(dms) < 0) +// { +// obidebug(1, "\nError cleaning unfinished views"); +// return -1; +// } +// +// // Clean unfinished columns +// if (obi_clean_unfinished_columns(dms) < 0) +// { +// obidebug(1, "\nError cleaning unfinished columns"); +// return -1; +// } + + return 0; +} + + int obi_dms_exists(const char* dms_path) { struct stat buffer; @@ -750,7 +797,7 @@ OBIDMS_p obi_create_dms(const char* dms_path) return NULL; // Open DMS - dms = obi_open_dms(dms_path); + dms = obi_open_dms(dms_path, false); if (dms == NULL) { obidebug(1, "\nProblem opening a DMS"); @@ -775,11 +822,12 @@ OBIDMS_p obi_create_dms(const char* dms_path) } -OBIDMS_p obi_open_dms(const char* dms_path) +OBIDMS_p obi_open_dms(const char* dms_path, bool cleaning) { OBIDMS_p dms; char* relative_dms_path; char* absolute_dms_path; + int clean_dms; dms = NULL; @@ -868,6 +916,26 @@ OBIDMS_p obi_open_dms(const char* dms_path) return NULL; } + // Reset the working variable to false if cleaning the DMS + if (cleaning) + (dms->infos)->working = false; + + // Check that the DMS is not already working (being used by a process) + if ((dms->infos)->working) + { + obidebug(1, "\n\nERROR:\nThe DMS '%s' contains unfinished views or columns. Either another command is currently running, " + "in which case you have to wait for it to finish, or a previous command was interrupted, " + "in which case you can run 'obi clean_dms [your_dms]' to clean the DMS.\n", dms->dms_name); + obi_set_errno(OBIDMS_WORKING); + unmap_infos_file(dms); + closedir(dms->directory); + free(dms); + return NULL; + } + + // Set the working variable to true + (dms->infos)->working = true; + // Open the indexer directory dms->indexer_directory = opendir_in_dms(dms, INDEXER_DIR_NAME); if (dms->indexer_directory == NULL) @@ -948,6 +1016,23 @@ OBIDMS_p obi_open_dms(const char* dms_path) return NULL; } + // // Check for unfinished views and columns + // clean_dms = obi_dms_is_clean(dms); + // if (clean_dms < 0) + // { + // obi_set_errno(OBIDMS_UNKNOWN_ERROR); + // obidebug(1, "\nError checking if a DMS has unfinished views or columns when opening it"); + // unmap_infos_file(dms); + // closedir(dms->indexer_directory); + // closedir(dms->tax_directory); + // closedir(dms->view_directory); + // closedir(dms->directory); + // free(dms); + // return NULL; + // } + // if (! clean_dms) + // obi_set_errno(OBIDMS_NOT_CLEAN); + // Clean unfinished views if (obi_clean_unfinished_views(dms) < 0) { @@ -1012,7 +1097,7 @@ OBIDMS_p obi_test_open_dms(const char* dms_name) case 0: return NULL; case 1: - return obi_open_dms(dms_name); + return obi_open_dms(dms_name, false); }; obidebug(1, "\nError checking if an OBIDMS directory exists"); @@ -1031,7 +1116,7 @@ OBIDMS_p obi_dms(const char* dms_name) case 0: return obi_create_dms(dms_name); case 1: - return obi_open_dms(dms_name); + return obi_open_dms(dms_name, false); }; obidebug(1, "\nError checking if an OBIDMS directory exists"); @@ -1062,14 +1147,6 @@ int obi_close_dms(OBIDMS_p dms, bool force) if (dms != NULL) { - // Unmap information file - if (unmap_infos_file(dms) < 0) - { - obidebug(1, "\nError unmaping a DMS information file while closing a DMS"); - free(dms); - return -1; - } - // Close all columns while ((dms->opened_columns)->nb_opened_columns > 0) obi_close_column(*((dms->opened_columns)->columns)); @@ -1096,6 +1173,18 @@ int obi_close_dms(OBIDMS_p dms, bool force) free(dms); return -1; } + + // Set the working variable as false + (dms->infos)->working = false; + + // Unmap information file + if (unmap_infos_file(dms) < 0) + { + obidebug(1, "\nError unmaping a DMS information file while closing a DMS"); + free(dms); + return -1; + } + if (closedir(dms->directory) < 0) { obi_set_errno(OBIDMS_MEMORY_ERROR); @@ -1338,7 +1427,7 @@ obiversion_t obi_import_column(const char* dms_path_1, const char* dms_path_2, c char* complete_avl_name; Obi_indexer_p avl_group; - dms_1 = obi_open_dms(dms_path_1); + dms_1 = obi_open_dms(dms_path_1, false); if (dms_1 == NULL) { obidebug(1, "\nError opening a DMS to import a column from it"); @@ -1538,7 +1627,7 @@ int obi_import_view(const char* dms_path_1, const char* dms_path_2, const char* OBIDMS_column_header_p header = NULL; OBIDMS_column_header_p header_2 = NULL; - dms_1 = obi_open_dms(dms_path_1); + dms_1 = obi_open_dms(dms_path_1, false); if (dms_1 == NULL) { obidebug(1, "\nError opening a DMS to import a view from it"); diff --git a/src/obidms.h b/src/obidms.h index 3de8672..d8c5bac 100755 --- a/src/obidms.h +++ b/src/obidms.h @@ -88,6 +88,8 @@ typedef struct OBIDMS_infos { */ size_t used_size; /**< Used size in bytes. */ + bool working; /**< If the DMS is currently working (being used by a process). + */ char comments[]; /**< Comments, additional informations on the DMS including * the command line history. */ @@ -146,6 +148,37 @@ extern OBIDMS_p global_opened_dms_list[MAX_NB_OPENED_DMS+1]; extern int global_opened_dms_counter_list[MAX_NB_OPENED_DMS+1]; +/** + * @brief Checks if an OBIDMS contains unfinished views or columns. + * + * @param dms A pointer on the DMS. + * + * @returns An integer value indicating whether the DMS is clean or not. + * @retval 1 if the DMS is clean. + * @retval 0 if the DMS contains unfinished views or columns. + * @retval -1 if an error occurred. + * + * @since September 2019 + * @author Celine Mercier (celine.mercier@metabarcoding.org) + */ +int obi_dms_is_clean(OBIDMS_p dms); + + +/** + * @brief Cleans an OBIDMS that contains unfinished views or columns and tags it as not working + * (not being used by a process). + * + * @param dms_path A pointer to a C string containing the path to the DMS. + * + * @retval 0 on success. + * @retval -1 if an error occurred. + * + * @since September 2019 + * @author Celine Mercier (celine.mercier@metabarcoding.org) + */ +int obi_clean_dms(const char* dms_path); + + /** * @brief Checks if an OBIDMS exists. * @@ -189,6 +222,7 @@ OBIDMS_p obi_create_dms(const char* dms_name); * @brief Opens an existing OBITools Data Management instance (OBIDMS). * * @param dms_path A pointer to a C string containing the path to the database. + * @param cleaning A boolean indicating whether the DMS is being opened for cleaning unfinished views and columns. * * @returns A pointer to an OBIDMS structure describing the opened DMS. * @retval NULL if an error occurred. @@ -197,7 +231,7 @@ OBIDMS_p obi_create_dms(const char* dms_name); * @since May 2015 * @author Eric Coissac (eric.coissac@metabarcoding.org) */ -OBIDMS_p obi_open_dms(const char* dms_path); +OBIDMS_p obi_open_dms(const char* dms_path, bool cleaning); /** diff --git a/src/obidmscolumn.c b/src/obidmscolumn.c index 6edcf21..b75c813 100644 --- a/src/obidmscolumn.c +++ b/src/obidmscolumn.c @@ -2437,6 +2437,127 @@ int obi_column_prepare_to_get_value(OBIDMS_column_p column, index_t line_nb) } +int obi_dms_has_unfinished_columns(OBIDMS_p dms) +{ + struct dirent* dms_dirent; + struct dirent* col_dirent; + DIR* col_dir; + size_t i,j; + char* column_file_path; + char* column_dir_path; + char* col_name; + char* col_version_str; +// char* version_file; + obiversion_t col_version; + OBIDMS_column_header_p col_header; +// int n; + char* col_to_delete[1000]; + char* dir_to_delete[1000]; + int ddir; + int dcol; + int d; + int ret_value; + + ret_value = 0; + + // Find column directories + ddir = 0; + rewinddir(dms->directory); + while ((dms_dirent = readdir(dms->directory)) != NULL) + { + if ((dms_dirent->d_name)[0] == '.') + continue; + i=0; + while (((dms_dirent->d_name)[i] != '.') && (i < strlen(dms_dirent->d_name))) + i++; + if ((i != strlen(dms_dirent->d_name)) && (strcmp((dms_dirent->d_name)+i, ".obicol") == 0)) // Found a column directory + { + column_dir_path = obi_dms_get_full_path(dms, dms_dirent->d_name); + if (column_dir_path == NULL) + { + obidebug(1, "\nError getting a column directory path when deleting unfinished columns"); + ret_value = -1; + } + col_name = (char*) malloc(strlen(dms_dirent->d_name) * sizeof(char)); + if (col_name == NULL) + { + obi_set_errno(OBI_MALLOC_ERROR); + obidebug(1, "\nError allocating memory for a column name when deleting unfinished columns: directory %s", dms_dirent->d_name); + ret_value = -1; + continue; + } + strncpy(col_name, dms_dirent->d_name, i); + col_name[i] = '\0'; + col_dir = opendir_in_dms(dms, dms_dirent->d_name); + if (col_dir == NULL) + { + obidebug(1, "\nError opening a column directory when deleting unfinished columns"); + ret_value = -1; + continue; + } + + // Iteration on files of this column directory + dcol = 0; + while ((col_dirent = readdir(col_dir)) != NULL) + { + if ((col_dirent->d_name)[0] == '.') + continue; + i=0; + j=0; + while (((col_dirent->d_name)[i] != '@') && ((col_dirent->d_name)[i] != '.')) + i++; + if ((col_dirent->d_name)[i] == '@') // Found a column file + { + i++; + j=i; + while ((col_dirent->d_name)[j] != '.') + j++; + col_version_str = (char*) malloc(strlen(col_dirent->d_name) * sizeof(char)); + if (col_version_str == NULL) + { + obi_set_errno(OBI_MALLOC_ERROR); + obidebug(1, "\nError allocating memory for a column version when deleting unfinished columns: directory %s", dms_dirent->d_name); + ret_value = -1; + continue; + } + strncpy(col_version_str, (col_dirent->d_name)+i, j-i); + col_version_str[j-i] = '\0'; + col_version = atoi(col_version_str); + free(col_version_str); + col_header = obi_column_get_header_from_name(dms, col_name, col_version); + if (col_header == NULL) // TODO discuss if delete file or not + { + obidebug(1, "\nError reading a column header when deleting unfinished columns: file %s", col_dirent->d_name); + ret_value = -1; + continue; + } + + // Check if the column is finished and delete it if not + if (col_header->finished == false) + ret_value = 1; + // Close the header + if (obi_close_header(col_header) < 0) + ret_value = -1; + } + } + + // Close column directory + if (closedir(col_dir) < 0) + { + obi_set_errno(OBICOL_UNKNOWN_ERROR); + obidebug(1, "\nError closing a column directory when deleting unfinished columns"); + ret_value = -1; + continue; + } + + free(col_name); + } + } + + return ret_value; +} + + int obi_clean_unfinished_columns(OBIDMS_p dms) { struct dirent* dms_dirent; diff --git a/src/obidmscolumn.h b/src/obidmscolumn.h index f1d06bd..d9599e0 100755 --- a/src/obidmscolumn.h +++ b/src/obidmscolumn.h @@ -535,6 +535,21 @@ int obi_column_prepare_to_set_value(OBIDMS_column_p column, index_t line_nb, ind int obi_column_prepare_to_get_value(OBIDMS_column_p column, index_t line_nb); +/** + * @brief Goes through all the column files of a DMS and checks for views that have + * not been flagged as finished (done by the finish_view() function in the + * obiview source file). + * + * @param dms A pointer on an OBIDMS. + * + * @returns A boolean indicating whether unfinished views were found. + * + * @since September 2019 + * @author Celine Mercier (celine.mercier@metabarcoding.org) + */ +int obi_dms_has_unfinished_columns(OBIDMS_p dms); + + /** * @brief Goes through all the column files of a DMS and deletes columns that have * not been flagged as finished (done by the finish_view() function in the diff --git a/src/obierrno.h b/src/obierrno.h index 46a86e3..0e2ec93 100755 --- a/src/obierrno.h +++ b/src/obierrno.h @@ -130,8 +130,12 @@ extern int obi_errno; */ #define OBIVIEW_ALREADY_EXISTS_ERROR (35) /** Tried to create a new view with a name already existing in the DMS. */ -#define OBI_ECOTAG_ERROR (36) /** Tried to create a new view with a name already existing in the DMS. +#define OBI_ECOTAG_ERROR (36) /** Error while running ecotag. */ +#define OBIDMS_NOT_CLEAN (37) /** OBIDMS has unfinished views or columns. + */ +#define OBIDMS_WORKING (38) /** OBIDMS is tagged as being currently used by a process. + */ /**@}*/ #endif /* OBIERRNO_H_ */ diff --git a/src/obiview.c b/src/obiview.c index 4f74a54..e195894 100755 --- a/src/obiview.c +++ b/src/obiview.c @@ -2594,6 +2594,74 @@ int obi_save_and_close_view(Obiview_p view) } +int obi_dms_has_unfinished_views(OBIDMS_p dms) +{ + struct dirent* dp; + int i; + char* full_path; + char* relative_path; + Obiview_infos_p view_infos; + char* view_name; + int ret_value; + char* to_delete[1000]; + int d; + + ret_value = 0; + d = 0; + + // Look for unfinished views and delete them + rewinddir(dms->view_directory); + while ((dp = readdir(dms->view_directory)) != NULL) + { + if ((dp->d_name)[0] == '.') + continue; + i=0; + while ((dp->d_name)[i] != '.') + i++; + relative_path = (char*) malloc(strlen(VIEW_DIR_NAME) + strlen(dp->d_name) + 2); + strcpy(relative_path, VIEW_DIR_NAME); + strcat(relative_path, "/"); + strcat(relative_path, dp->d_name); + full_path = obi_dms_get_full_path(dms, relative_path); + free(relative_path); + if (full_path == NULL) + { + obidebug(1, "\nError getting the full path to a view file when cleaning unfinished views"); + ret_value = -1; + continue; + } + + if (strcmp((dp->d_name)+i, ".obiview_unfinished") == 0) + ret_value = 1; + + else if (strcmp((dp->d_name)+i, ".obiview") == 0) + { // Check if the view was properly flagged as finished + view_name = (char*) malloc((i+1) * sizeof(char)); + if (view_name == NULL) + { + obi_set_errno(OBI_MALLOC_ERROR); + obidebug(1, "\nError allocating memory for a view name when deleting unfinished views: file %s", dp->d_name); + ret_value = -1; + continue; + } + strncpy(view_name, dp->d_name, i); + view_name[i] = '\0'; + view_infos = obi_view_map_file(dms, view_name, true); + if (view_infos == NULL) + { + obidebug(1, "\nError reading a view file when deleting unfinished views: file %s", dp->d_name); + ret_value = -1; + continue; + } + if (view_infos->finished == false) + ret_value = 1; + } + } + + return ret_value; +} + + int obi_clean_unfinished_views(OBIDMS_p dms) { struct dirent* dp; diff --git a/src/obiview.h b/src/obiview.h index b7e4f5f..7acc767 100755 --- a/src/obiview.h +++ b/src/obiview.h @@ -565,6 +565,21 @@ int obi_view_add_comment(Obiview_p view, const char* key, const char* value); int obi_save_and_close_view(Obiview_p view); +/** + * @brief Goes through all the view files of a DMS and checks for views that have + * not been flagged as finished (file extension renamed from '.obiview_unfinished' + * to '.obiview' and finished boolean set to true in the file, done by finish_view()). + * + * @param dms A pointer on an OBIDMS. + * + * @returns A boolean indicating whether unfinished views were found. + * + * @since September 2019 + * @author Celine Mercier (celine.mercier@metabarcoding.org) + */ +int obi_dms_has_unfinished_views(OBIDMS_p dms); + + /** * @brief Goes through all the view files of a DMS and deletes views that have * not been flagged as finished (file extension renamed from '.obiview_unfinished'