From 9e3ac477eb0e108b7369eb98b2a01fc36f9d3d4c Mon Sep 17 00:00:00 2001 From: Celine Mercier Date: Fri, 24 Nov 2017 17:58:47 +0100 Subject: [PATCH] OBIDMS: Opened DMS now have a counter associated so that DMS are not actually opened several times by the same program, which triggers the cleaning of unfinished views and columns (to discuss) --- python/obitools3/dms/capi/obidms.pxd | 2 +- python/obitools3/dms/dms.pyx | 2 +- src/obidms.c | 207 ++++++++++++++++++++------- src/obidms.h | 10 +- 4 files changed, 167 insertions(+), 54 deletions(-) diff --git a/python/obitools3/dms/capi/obidms.pxd b/python/obitools3/dms/capi/obidms.pxd index f62a61a..905064e 100644 --- a/python/obitools3/dms/capi/obidms.pxd +++ b/python/obitools3/dms/capi/obidms.pxd @@ -14,7 +14,7 @@ cdef extern from "obidms.h" nogil: OBIDMS_p obi_test_open_dms(const_char_p dms_path) OBIDMS_p obi_create_dms(const_char_p dms_path) int obi_dms_exists(const char* dms_path) - int obi_close_dms(OBIDMS_p dms) + int obi_close_dms(OBIDMS_p dms, bint force) char* obi_dms_get_dms_path(OBIDMS_p dms) char* obi_dms_get_full_path(OBIDMS_p dms, const_char_p path_name) void obi_close_atexit() diff --git a/python/obitools3/dms/dms.pyx b/python/obitools3/dms/dms.pyx index b39c6ca..fdbc690 100644 --- a/python/obitools3/dms/dms.pyx +++ b/python/obitools3/dms/dms.pyx @@ -92,7 +92,7 @@ cdef class DMS(OBIWrapper): cdef OBIDMS_p pointer = self.pointer() if self.active() : OBIWrapper.close(self) - if (obi_close_dms(pointer)) < 0 : + if (obi_close_dms(pointer, False)) < 0 : raise Exception("Problem closing an OBIDMS") diff --git a/src/obidms.c b/src/obidms.c index 1fccaa7..540004f 100644 --- a/src/obidms.c +++ b/src/obidms.c @@ -34,8 +34,10 @@ #define DEBUG_LEVEL 0 // TODO has to be defined somewhere else (cython compil flag?) -// Initialize global list of opened DMS +// Initialize global list of opened DMS and their associated counters OBIDMS_p global_opened_dms_list[MAX_NB_OPENED_DMS+1] = { 0 }; +int global_opened_dms_counter_list[MAX_NB_OPENED_DMS+1] = { 0 }; + /************************************************************************** @@ -120,11 +122,44 @@ static int list_dms(OBIDMS_p dms); * Internal function removing a DMS from the global list of DMS opened by a program. * * @param dms A pointer on the DMS to remove. + * @param force Whether the DMS should be unlisted even if it is opened more than once. + * + * @retval 0 if the operation was successfully completed. + * @retval -1 if the DMS was not found. * * @since October 2017 * @author Celine Mercier (celine.mercier@metabarcoding.org) */ -static void unlist_dms(OBIDMS_p dms); +static int unlist_dms(OBIDMS_p dms, bool force); + + +/** + * Internal function checking if a DMS is already in the global list of DMS opened by the program. + * + * @param full_dms_path The absolute path to the DMS directory. + * + * @returns A pointer on the DMS if it is already opened, or NULL if it was not opened. + * + * @since November 2017 + * @author Celine Mercier (celine.mercier@metabarcoding.org) + */ +static OBIDMS_p check_if_dms_in_list(const char* full_dms_path); + + +/** + * Internal function returning the count for an opened DMS (how many times the DMS is opened) in the global list of DMS opened by the program. + * + * @param dms A pointer on the DMS. + * + * @returns The count indicating how many times the DMS has been opened by the program. + * + * @retval -1 if the DMS was not found. + * + * @since November 2017 + * @author Celine Mercier (celine.mercier@metabarcoding.org) + */ +static int dms_count_in_list(OBIDMS_p dms); + /************************************************************************ @@ -238,7 +273,7 @@ int create_dms_infos_file(int dms_file_descriptor, const char* dms_name) static int list_dms(OBIDMS_p dms) { int i = 0; - while (global_opened_dms_list[i] != NULL) + while ((global_opened_dms_list[i] != NULL) && (global_opened_dms_list[i] != dms)) i++; if (i == (MAX_NB_OPENED_DMS-1)) { @@ -246,21 +281,63 @@ static int list_dms(OBIDMS_p dms) obidebug(1, "\nError opening a DMS: maximum number of DMS opened by a program reached"); return -1; } - global_opened_dms_list[i] = dms; + if (global_opened_dms_list[i] == NULL) + { // new dms + global_opened_dms_list[i] = dms; + global_opened_dms_counter_list[i] = 0; + } + else + { // already opened dms + (global_opened_dms_counter_list[i])++; + } + return 0; } -static void unlist_dms(OBIDMS_p dms) +static int unlist_dms(OBIDMS_p dms, bool force) { int i = 0; - while (global_opened_dms_list[i] != dms) - i++; - while (global_opened_dms_list[i] != NULL) - { - global_opened_dms_list[i] = global_opened_dms_list[i+1]; + while ((global_opened_dms_list[i] != dms) && (i <= MAX_NB_OPENED_DMS)) i++; + if (i == MAX_NB_OPENED_DMS) + return -1; // DMS not found + // If opened more than once, and the unlisting is not forced, decrement counter + if ((global_opened_dms_counter_list[i] > 1) && (! force)) + (global_opened_dms_counter_list[i])--; + else + { // If opened once or forced unlisting, delete and unlist by shifting the list + while (global_opened_dms_list[i] != NULL) + { + global_opened_dms_list[i] = global_opened_dms_list[i+1]; + i++; + } } + return 0; +} + + +static OBIDMS_p check_if_dms_in_list(const char* full_dms_path) +{ + int i = 0; + while ((i <= MAX_NB_OPENED_DMS) && (global_opened_dms_list[i] != NULL) && (strcmp(global_opened_dms_list[i]->directory_path, full_dms_path) != 0)) + i++; + if (i == MAX_NB_OPENED_DMS) + return NULL; // DMS not found + if (global_opened_dms_list[i] != NULL) + return global_opened_dms_list[i]; + return NULL; +} + + +static int dms_count_in_list(OBIDMS_p dms) +{ + int i = 0; + while ((i <= MAX_NB_OPENED_DMS) && (global_opened_dms_list[i] != dms)) + i++; + if (i == MAX_NB_OPENED_DMS) + return -1; // DMS not found + return global_opened_dms_counter_list[i]; } @@ -384,7 +461,8 @@ OBIDMS_p obi_create_dms(const char* dms_path) OBIDMS_p obi_open_dms(const char* dms_path) { OBIDMS_p dms; - char* complete_dms_path; + char* relative_dms_path; + char* absolute_dms_path; char* infos_file_name; int infos_file_descriptor; bool little_endian_dms; @@ -392,6 +470,32 @@ OBIDMS_p obi_open_dms(const char* dms_path) dms = NULL; + // Build and check the directory name including the relative path + relative_dms_path = build_directory_name(dms_path); + if (relative_dms_path == NULL) + return NULL; + + // Get and store the absolute path to the DMS directory + absolute_dms_path = realpath(relative_dms_path, NULL); + if (absolute_dms_path == NULL) + { + obi_set_errno(OBIDMS_UNKNOWN_ERROR); + obidebug(1, "\nError getting the absolute path to the DMS directory (DMS does not exist)"); + free(relative_dms_path); + return NULL; + } + + free(relative_dms_path); + + // Check if the DMS is already opened + dms = check_if_dms_in_list(absolute_dms_path); + if (dms != NULL) + { + list_dms(dms); + free(absolute_dms_path); + return dms; + } + // Allocate the data structure dms = (OBIDMS_p) malloc(sizeof(OBIDMS_t)); if (dms == NULL) @@ -401,36 +505,8 @@ OBIDMS_p obi_open_dms(const char* dms_path) return NULL; } -/* - // Isolate and store the dms name - j = 0; - for (i=0; idms_name, basename((char*)dms_path)); - - // Build and check the directory name including the relative path - complete_dms_path = build_directory_name(dms_path); - if (complete_dms_path == NULL) - { - free(dms); - return NULL; - } - - // Get and store the absolute path to the DMS directory - if (realpath(complete_dms_path, dms->directory_path) == NULL) - { - obi_set_errno(OBIDMS_UNKNOWN_ERROR); - obidebug(1, "\nError getting the absolute path to the DMS directory (DMS does not exist)"); - free(complete_dms_path); - return NULL; - } - - free(complete_dms_path); + strcpy(dms->directory_path, absolute_dms_path); // Try to open the directory dms->directory = opendir(dms->directory_path); @@ -620,7 +696,18 @@ OBIDMS_p obi_open_dms(const char* dms_path) (dms->opened_indexers)->nb_opened_indexers = 0; // Add in the global list of opened DMS - list_dms(dms); + if (list_dms(dms) < 0) + { + obidebug(1, "\nError cleaning unfinished columns when opening an OBIDMS"); + closedir(dms->indexer_directory); + closedir(dms->tax_directory); + closedir(dms->view_directory); + closedir(dms->directory); + free(dms->opened_columns); + free(dms->opened_indexers); + free(dms); + return NULL; + } return dms; } @@ -664,8 +751,27 @@ OBIDMS_p obi_dms(const char* dms_name) } -int obi_close_dms(OBIDMS_p dms) +int obi_close_dms(OBIDMS_p dms, bool force) { + int dms_counter; + + if (!force) + { + dms_counter = dms_count_in_list(dms); + if (dms_counter < 0) + obidebug(1, "\nError checking the counter of an OBIDMS in the global list of opened OBIDMS"); + + if (dms_counter > 1) // Don't close if the DMS is opened more than once + { + if (unlist_dms(dms, force) < 0) + { + obidebug(1, "\nError decrementing the counter of an OBIDMS in the global list of opened OBIDMS"); + return -1; + } + return 0; + } + } + if (dms != NULL) { // Close all columns @@ -703,7 +809,12 @@ int obi_close_dms(OBIDMS_p dms) } // Remove DMS from global list of opened DMS - unlist_dms(dms); + if (unlist_dms(dms, force) < 0) + { + obidebug(1, "\nError removing an OBIDMS from the global list of opened OBIDMS when closing it"); + free(dms); + return -1; + } free(dms); } @@ -1060,8 +1171,8 @@ obiversion_t obi_import_column(const char* dms_path_1, const char* dms_path_2, c // TODO no? because if iterating over all columns in a view etc.... iterate and change associated columns version refs afterwards? // Close the DMS - obi_close_dms(dms_1); - obi_close_dms(dms_2); + obi_close_dms(dms_1, false); + obi_close_dms(dms_2, false); return new_version; } @@ -1167,8 +1278,8 @@ int obi_import_view(const char* dms_path_1, const char* dms_path_2, const char* } // Close the DMS - obi_close_dms(dms_1); - obi_close_dms(dms_2); + obi_close_dms(dms_1, false); + obi_close_dms(dms_2, false); return 0; } @@ -1180,7 +1291,7 @@ void obi_close_atexit() while (global_opened_dms_list[i] != NULL) { - obi_close_dms(global_opened_dms_list[i]); + obi_close_dms(global_opened_dms_list[i], true); i++; } } diff --git a/src/obidms.h b/src/obidms.h index 0e44213..88a59f3 100644 --- a/src/obidms.h +++ b/src/obidms.h @@ -54,7 +54,7 @@ typedef struct OBIDMS_column* OBIDMS_column_p; /**< Declarations to avoid circu typedef struct Opened_columns_list { int nb_opened_columns; /**< Number of opened columns. */ - OBIDMS_column_p columns[MAX_NB_OPENED_COLUMNS]; /**< Array of pointers on the opened columns. + OBIDMS_column_p columns[MAX_NB_OPENED_COLUMNS+1]; /**< Array of pointers on the opened columns. */ } Opened_columns_list_t, *Opened_columns_list_p; @@ -70,7 +70,7 @@ typedef OBIDMS_avl_group_p Obi_indexer_p; /**< Declarations to avoid circular typedef struct Opened_indexers_list { int nb_opened_indexers; /**< Number of opened indexers. */ - Obi_indexer_p indexers[MAX_NB_OPENED_INDEXERS]; /**< Array of pointers on the opened indexers. + Obi_indexer_p indexers[MAX_NB_OPENED_INDEXERS+1]; /**< Array of pointers on the opened indexers. */ } Opened_indexers_list_t, *Opened_indexers_list_p; @@ -87,7 +87,7 @@ typedef struct OBIDMS { char directory_name[OBIDMS_MAX_NAME+1]; /**< The name of the directory * containing the DMS. */ - char directory_path[MAX_PATH_LEN]; /**< The absolute path of the directory + char directory_path[MAX_PATH_LEN+1]; /**< The absolute path of the directory * containing the DMS. */ DIR* directory; /**< A directory entry usable to @@ -127,6 +127,7 @@ typedef struct OBIDMS { * @brief Global Array of DMS pointers listing all the DMS opened by a program. */ extern OBIDMS_p global_opened_dms_list[MAX_NB_OPENED_DMS+1]; +extern int global_opened_dms_counter_list[MAX_NB_OPENED_DMS+1]; /** @@ -223,6 +224,7 @@ OBIDMS_p obi_dms(const char* dms_path); * @brief Closes an opened OBITools Data Management instance (OBIDMS). * * @param dms A pointer as returned by obi_create_dms() or obi_open_dms(). + * @param force Whether the DMS should be closed even if it is opened more than once. * * @returns An integer value indicating the success of the operation. Even on * error, the `OBIDMS` structure is freed. @@ -234,7 +236,7 @@ OBIDMS_p obi_dms(const char* dms_path); * @since May 2015 * @author Eric Coissac (eric.coissac@metabarcoding.org) */ -int obi_close_dms(OBIDMS_p dms); +int obi_close_dms(OBIDMS_p dms, bool force); /**