diff --git a/src/obidms.c b/src/obidms.c index e8f6eb7..6d426e6 100644 --- a/src/obidms.c +++ b/src/obidms.c @@ -20,6 +20,7 @@ #include #include #include /* : Added July 28th 2017 to include basename */ +#include #include "obidms.h" #include "obierrno.h" @@ -29,6 +30,7 @@ #include "obiblob_indexer.h" #include "utils.h" #include "obilittlebigman.h" +#include "libjson/json_utils.h" #define DEBUG_LEVEL 0 // TODO has to be defined somewhere else (cython compil flag?) @@ -75,7 +77,7 @@ static char* build_directory_name(const char* dms_name); * * @param dms_name The name of the OBIDMS. * - * @returns A pointer to the file name. + * @returns A pointer on the file name. * @retval NULL if an error occurred. * * @since November 2015 @@ -84,16 +86,73 @@ static char* build_directory_name(const char* dms_name); static char* build_infos_file_name(const char* dms_name); +/** + * Internal function calculating the initial size of the file where the informations about a DMS are stored. + * + * @returns The initial size of the file in bytes, rounded to a multiple of page size. + * + * @since September 2018 + * @author Celine Mercier (celine.mercier@metabarcoding.org) + */ +static size_t get_platform_infos_file_size(); + + +/** + * @brief Internal function enlarging a DMS information file. + * + * @param dms A pointer on the DMS. + * @param new_size The new size needed, in bytes (not necessarily rounded to a page size multiple). + * + * @retval 0 if the operation was successfully completed. + * @retval -1 if an error occurred. + * + * @since September 2018 + * @author Celine Mercier (celine.mercier@metabarcoding.org) + */ +static int enlarge_infos_file(OBIDMS_p dms, size_t new_size); + + +/** + * @brief Internal function mapping a DMS information file and returning the mapped structure stored in it. + * + * The function checks that endianness of the platform and of the DMS match. + * + * @param dms_file_descriptor The file descriptor of the DMS directory. + * @param dms_name The base name of the DMS. + * + * @returns A pointer on the mapped DMS infos structure. + * @retval NULL if an error occurred. + * + * @since September 2018 + * @author Celine Mercier (celine.mercier@metabarcoding.org) + */ +static OBIDMS_infos_p map_infos_file(int dms_file_descriptor, const char* dms_name); + + +/** + * @brief Unmaps a DMS information file. + * + * @param dms A pointer on the OBIDMS. + * + * @returns A value indicating the success of the operation. + * @retval 0 if the operation was successfully completed. + * @retval -1 if an error occurred. + * + * @since October 2018 + * @author Celine Mercier (celine.mercier@metabarcoding.org) + */ +static int unmap_infos_file(OBIDMS_p dms); + + /** * Internal function creating the file containing basic informations on the OBIDMS. * - * This file contains: - * - The endianness of the platform + * This file contains an OBIDMS_infos structure. * * @warning The returned pointer has to be freed by the caller. * - * @param dms_file_descriptor The file descriptor for the OBIDMS directory. - * @param dms_name The name of the OBIDMS. + * @param dms_file_descriptor The file descriptor of the DMS directory. + * @param dms_name The base name of the DMS. * * @retval 0 if the operation was successfully completed. * @retval -1 if an error occurred. @@ -101,7 +160,7 @@ static char* build_infos_file_name(const char* dms_name); * @since November 2015 * @author Celine Mercier (celine.mercier@metabarcoding.org) */ -int create_dms_infos_file(int dms_file_descriptor, const char* dms_name); +static int create_dms_infos_file(int dms_file_descriptor, const char* dms_name); /** @@ -211,16 +270,246 @@ static char* build_infos_file_name(const char* dms_name) } -int create_dms_infos_file(int dms_file_descriptor, const char* dms_name) +static size_t get_platform_infos_file_size() { - char* file_name; - int infos_file_descriptor; - off_t file_size; - bool little_endian; + size_t infos_size; + size_t rounded_infos_size; + double multiple; - file_size = sizeof(bool); + infos_size = sizeof(OBIDMS_infos_t); + + multiple = ceil((double) (infos_size) / (double) getpagesize()); + + rounded_infos_size = multiple * getpagesize(); + + return rounded_infos_size; +} + + +static int enlarge_infos_file(OBIDMS_p dms, size_t new_size) +{ + int infos_file_descriptor; + double multiple; + size_t rounded_new_size; + char* file_name; // Create file name + file_name = build_infos_file_name(dms->dms_name); + if (file_name == NULL) + return -1; + + // Open infos file + infos_file_descriptor = openat(dms->dir_fd, file_name, O_RDWR, 0777); + if (infos_file_descriptor < 0) + { + obi_set_errno(OBIDMS_UNKNOWN_ERROR); + obidebug(1, "\nError opening a DMS information file"); + free(file_name); + return -1; + } + + free(file_name); + + // Round new size to a multiple of page size // TODO make function in utils + multiple = ceil((double) new_size / (double) getpagesize()); + rounded_new_size = multiple * getpagesize(); + + // Enlarge the file + if (ftruncate(infos_file_descriptor, rounded_new_size) < 0) + { + obi_set_errno(OBIDMS_UNKNOWN_ERROR); + obidebug(1, "\nError enlarging a DMS information file"); + close(infos_file_descriptor); + return -1; + } + + // Unmap and remap the file + if (munmap(dms->infos, (dms->infos)->file_size) < 0) + { + obi_set_errno(OBIDMS_UNKNOWN_ERROR); + obidebug(1, "\nError munmapping a DMS information file when enlarging"); + close(infos_file_descriptor); + return -1; + } + + dms->infos = mmap(NULL, + rounded_new_size, + PROT_READ | PROT_WRITE, + MAP_SHARED, + infos_file_descriptor, + 0 + ); + if (dms->infos == MAP_FAILED) + { + obi_set_errno(OBIDMS_UNKNOWN_ERROR); + obidebug(1, "\nError re-mmapping a DMS information file after enlarging the file"); + close(infos_file_descriptor); + return -1; + } + + // Set new size + (dms->infos)->file_size = rounded_new_size; + + if (close(infos_file_descriptor) < 0) + { + obi_set_errno(OBIDMS_UNKNOWN_ERROR); + obidebug(1, "\nError closing a DMS information file"); + return -1; + } + + return 0; +} + + +static OBIDMS_infos_p map_infos_file(int dms_file_descriptor, const char* dms_name) +{ + char* file_name; + OBIDMS_infos_p dms_infos; + int file_descriptor; + size_t min_size; + size_t full_size; + bool little_endian_platform; + + // Build file name + file_name = build_infos_file_name(dms_name); + if (file_name == NULL) + return NULL; + + // Open file + file_descriptor = openat(dms_file_descriptor, + file_name, + O_RDWR, 0777); + if (file_descriptor < 0) + { + obi_set_errno(OBIDMS_UNKNOWN_ERROR); + obidebug(1, "\nError opening a DMS information file (DMS %s)", dms_name); + free(file_name); + return NULL; + } + + free(file_name); + + // Map minimum size to read endianness and file size + // (fread() fails to read properly from the structure in the file because of padding) + min_size = get_platform_infos_file_size(); + + dms_infos = mmap(NULL, + min_size, + PROT_READ | PROT_WRITE, + MAP_SHARED, + file_descriptor, + 0 + ); + if (dms_infos == NULL) + { + obi_set_errno(OBIDMS_UNKNOWN_ERROR); + obidebug(1, "\nError mapping a DMS information file"); + return NULL; + } + + // Check endianness of the platform and DMS + little_endian_platform = obi_is_little_endian(); + if (little_endian_platform != dms_infos->little_endian) + { + obi_set_errno(OBIDMS_BAD_ENDIAN_ERROR); + obidebug(1, "\nError: The DMS and the platform have different endianness"); + close(file_descriptor); + return NULL; + } + + // Read actual file size + full_size = dms_infos->file_size; + + // Unmap the minimum size and remap the file with the full size + if (munmap(dms_infos, min_size) < 0) + { + obi_set_errno(OBIDMS_UNKNOWN_ERROR); + obidebug(1, "\nError munmapping a DMS information file"); + close(file_descriptor); + return NULL; + } + + dms_infos = mmap(NULL, + full_size, + PROT_READ | PROT_WRITE, + MAP_SHARED, + file_descriptor, + 0 + ); + if (dms_infos == NULL) + { + obi_set_errno(OBIDMS_UNKNOWN_ERROR); + obidebug(1, "\nError mapping a DMS information file"); + return NULL; + } + + if (close(file_descriptor) < 0) + { + obi_set_errno(OBIDMS_UNKNOWN_ERROR); + obidebug(1, "\nError closing a DMS information file"); + return NULL; + } + + return dms_infos; +} + + +static int unmap_infos_file(OBIDMS_p dms) +{ + OBIDMS_infos_p dms_infos; + char* file_name; + int file_descriptor; + size_t file_size; + + dms_infos = dms->infos; + + // Build file name + file_name = build_infos_file_name(dms->dms_name); + if (file_name == NULL) + return -1; + + // Open file + file_descriptor = openat(dms->dir_fd, + file_name, + O_RDWR, 0777); + if (file_descriptor < 0) + { + obi_set_errno(OBIDMS_UNKNOWN_ERROR); + obidebug(1, "\nError opening a DMS information file (DMS %s) to unmap it", dms->dms_name); + free(file_name); + return -1; + } + + free(file_name); + + // Unmap the DMS infos structure + file_size = dms_infos->file_size; + if (munmap(dms_infos, file_size) < 0) + { + obi_set_errno(OBIDMS_UNKNOWN_ERROR); + obidebug(1, "\nError unmapping a DMS information file"); + close(file_descriptor); + return -1; + } + + if (close(file_descriptor) < 0) + { + obi_set_errno(OBIDMS_UNKNOWN_ERROR); + obidebug(1, "\nError closing a DMS information file"); + return -1; + } + + return 0; +} + + +static int create_dms_infos_file(int dms_file_descriptor, const char* dms_name) +{ + char* file_name; + int infos_file_descriptor; + size_t file_size; + + // Build file name file_name = build_infos_file_name(dms_name); if (file_name == NULL) return -1; @@ -228,41 +517,56 @@ int create_dms_infos_file(int dms_file_descriptor, const char* dms_name) // Create file infos_file_descriptor = openat(dms_file_descriptor, file_name, - O_RDWR | O_CREAT | O_EXCL, 0777); + O_RDWR | O_CREAT | O_EXCL, + 0777); if (infos_file_descriptor < 0) { obi_set_errno(OBIDMS_UNKNOWN_ERROR); - obidebug(1, "\nError creating an informations file"); + obidebug(1, "\nError creating a DMS information file (DMS %s)", dms_name); free(file_name); return -1; } free(file_name); - // Truncate the infos file to the right size + // Truncate file to the initial size + file_size = get_platform_infos_file_size(); + if (ftruncate(infos_file_descriptor, file_size) < 0) { obi_set_errno(OBIDMS_UNKNOWN_ERROR); - obidebug(1, "\nError truncating an informations file"); + obidebug(1, "\nError truncating a DMS information file to the right size"); close(infos_file_descriptor); return -1; } - // Write endianness - little_endian = obi_is_little_endian(); - if (write(infos_file_descriptor, &little_endian, sizeof(bool)) < ((ssize_t) sizeof(bool))) + // Map the DMS infos structure + OBIDMS_infos_p infos = mmap(NULL, + file_size, + PROT_READ | PROT_WRITE, + MAP_SHARED, + infos_file_descriptor, + 0 + ); + + // Initialize values + infos->little_endian = obi_is_little_endian(); + infos->file_size = file_size; + infos->used_size = 0; + infos->comments[0] = '\0'; + + // Unmap the infos file + if (munmap(infos, file_size) < 0) { - obi_set_errno(OBIDMS_UNKNOWN_ERROR); - obidebug(1, "\nError writing the endianness in an informations file"); - close(infos_file_descriptor); + obi_set_errno(OBIVIEW_ERROR); + obidebug(1, "\nError munmapping a DMS information file"); return -1; } - // Close file if (close(infos_file_descriptor) < 0) { obi_set_errno(OBIDMS_UNKNOWN_ERROR); - obidebug(1, "\nError closing a DMS information file"); + obidebug(1, "\nError closing a view file"); return -1; } @@ -371,9 +675,10 @@ int obi_dms_exists(const char* dms_path) OBIDMS_p obi_create_dms(const char* dms_path) { - char* directory_name; - DIR* dms_dir; - int dms_file_descriptor; + char* directory_name; + DIR* dms_dir; + int dms_file_descriptor; + OBIDMS_p dms; // Build and check the directory name directory_name = build_directory_name(dms_path); @@ -439,22 +744,26 @@ OBIDMS_p obi_create_dms(const char* dms_path) return NULL; } -/* - // Isolate the dms name - j = 0; - for (i=0; idir_fd = dirfd(dms->directory); if (dms->dir_fd < 0) { @@ -545,56 +850,23 @@ OBIDMS_p obi_open_dms(const char* dms_path) return NULL; } - // Open informations file to check endianness - infos_file_name = build_infos_file_name(dms->dms_name); - infos_file_descriptor = openat(dms->dir_fd, infos_file_name, O_RDONLY, 0777); - if (infos_file_descriptor < 0) + // Open the information file + dms->infos = map_infos_file(dms->dir_fd, dms->dms_name); + if (dms->infos == NULL) { - obi_set_errno(OBIDMS_UNKNOWN_ERROR); - obidebug(1, "\nError opening an informations file"); + obidebug(1, "\nError opening a DMS information file"); closedir(dms->directory); free(dms); return NULL; } - free(infos_file_name); - - // Check endianness of the platform and DMS - little_endian_platform = obi_is_little_endian(); - if (read(infos_file_descriptor, &little_endian_dms, sizeof(bool)) < ((ssize_t) sizeof(bool))) - { - obi_set_errno(OBIDMS_UNKNOWN_ERROR); - obidebug(1, "\nError reading the endianness in an informations file"); - close(infos_file_descriptor); - closedir(dms->directory); - free(dms); - return NULL; - } - if (little_endian_platform != little_endian_dms) - { - obi_set_errno(OBIDMS_BAD_ENDIAN_ERROR); - obidebug(1, "\nError: The DMS and the platform have different endianness"); - close(infos_file_descriptor); - closedir(dms->directory); - free(dms); - return NULL; - } - - if (close(infos_file_descriptor) < 0) - { - obi_set_errno(OBIDMS_UNKNOWN_ERROR); - obidebug(1, "\nError closing a DMS information file"); - return NULL; - } - - dms->little_endian = little_endian_dms; - // Open the indexer directory dms->indexer_directory = opendir_in_dms(dms, INDEXER_DIR_NAME); if (dms->indexer_directory == NULL) { obi_set_errno(OBIDMS_UNKNOWN_ERROR); obidebug(1, "\nError opening the indexer directory"); + unmap_infos_file(dms); closedir(dms->directory); free(dms); return NULL; @@ -606,6 +878,7 @@ OBIDMS_p obi_open_dms(const char* dms_path) { obi_set_errno(OBIDMS_UNKNOWN_ERROR); obidebug(1, "\nError getting the file descriptor of the indexer directory"); + unmap_infos_file(dms); closedir(dms->indexer_directory); closedir(dms->directory); free(dms); @@ -618,6 +891,7 @@ OBIDMS_p obi_open_dms(const char* dms_path) { obi_set_errno(OBIDMS_UNKNOWN_ERROR); obidebug(1, "\nError opening the view directory"); + unmap_infos_file(dms); closedir(dms->indexer_directory); closedir(dms->directory); free(dms); @@ -630,6 +904,7 @@ OBIDMS_p obi_open_dms(const char* dms_path) { obi_set_errno(OBIDMS_UNKNOWN_ERROR); obidebug(1, "\nError getting the file descriptor of the view directory"); + unmap_infos_file(dms); closedir(dms->view_directory); closedir(dms->directory); free(dms); @@ -642,6 +917,7 @@ OBIDMS_p obi_open_dms(const char* dms_path) { obi_set_errno(OBIDMS_UNKNOWN_ERROR); obidebug(1, "\nError opening the taxonomy directory"); + unmap_infos_file(dms); closedir(dms->indexer_directory); closedir(dms->view_directory); closedir(dms->directory); @@ -655,6 +931,7 @@ OBIDMS_p obi_open_dms(const char* dms_path) { obi_set_errno(OBIDMS_UNKNOWN_ERROR); obidebug(1, "\nError getting the file descriptor of the taxonomy directory"); + unmap_infos_file(dms); closedir(dms->indexer_directory); closedir(dms->tax_directory); closedir(dms->view_directory); @@ -667,6 +944,7 @@ OBIDMS_p obi_open_dms(const char* dms_path) if (obi_clean_unfinished_views(dms) < 0) { obidebug(1, "\nError cleaning unfinished views when opening an OBIDMS"); + unmap_infos_file(dms); closedir(dms->indexer_directory); closedir(dms->tax_directory); closedir(dms->view_directory); @@ -679,6 +957,7 @@ OBIDMS_p obi_open_dms(const char* dms_path) if (obi_clean_unfinished_columns(dms) < 0) { obidebug(1, "\nError cleaning unfinished columns when opening an OBIDMS"); + unmap_infos_file(dms); closedir(dms->indexer_directory); closedir(dms->tax_directory); closedir(dms->view_directory); @@ -699,6 +978,7 @@ OBIDMS_p obi_open_dms(const char* dms_path) if (list_dms(dms) < 0) { obidebug(1, "\nError cleaning unfinished columns when opening an OBIDMS"); + unmap_infos_file(dms); closedir(dms->indexer_directory); closedir(dms->tax_directory); closedir(dms->view_directory); @@ -774,6 +1054,14 @@ int obi_close_dms(OBIDMS_p dms, bool force) if (dms != NULL) { + // Unmap information file + if (unmap_infos_file(dms) < 0) + { + obidebug(1, "\nError unmaping a DMS information file while closing a DMS"); + free(dms); + return -1; + } + // Close all columns while ((dms->opened_columns)->nb_opened_columns > 0) obi_close_column(*((dms->opened_columns)->columns)); @@ -823,6 +1111,53 @@ int obi_close_dms(OBIDMS_p dms, bool force) } +int obi_dms_write_comments(OBIDMS_p dms, const char* comments) +{ + size_t new_size; + + if (comments == NULL) + return 0; // TODO or error? discuss + + new_size = sizeof(OBIDMS_infos_t) + strlen(comments) + 1; + + // Check if the file has to be enlarged + if (new_size >= (dms->infos)->file_size) + { + if (enlarge_infos_file(dms, new_size) < 0) + return -1; + } + + strcpy((dms->infos)->comments, comments); + + (dms->infos)->used_size = new_size; + + return 0; +} + + +int obi_dms_add_comment(OBIDMS_p dms, const char* key, const char* value) +{ + char* new_comments = NULL; + + new_comments = obi_add_comment((dms->infos)->comments, key, value); + if (new_comments == NULL) + { + obidebug(1, "\nError adding a comment to a dms, key: %s, value: %s", key, value); + return -1; + } + + if (obi_dms_write_comments(dms, new_comments) < 0) + { + obidebug(1, "\nError adding a comment to a dms, key: %s, value: %s", key, value); + return -1; + } + + free(new_comments); + + return 0; +} + + OBIDMS_column_p obi_dms_get_column_from_list(OBIDMS_p dms, const char* column_name, obiversion_t version) { int i; diff --git a/src/obidms.h b/src/obidms.h index 88a59f3..cc92196 100644 --- a/src/obidms.h +++ b/src/obidms.h @@ -75,6 +75,25 @@ typedef struct Opened_indexers_list { } Opened_indexers_list_t, *Opened_indexers_list_p; +/** + * @brief A structure stored in an information file and containing comments and additional informations on the DMS + * including the command line history. + * + * A pointer on the comments is kept in the OBIDMS structure when a DMS is opened. + */ +typedef struct OBIDMS_infos { + bool little_endian; /** Whether the DMS is in little endian. + */ + size_t file_size; /** The size of the file in bytes. + */ + size_t used_size; /**< Used size in bytes. + */ + char comments[]; /**< Comments, additional informations on the DMS including + * the command line history. + */ +} OBIDMS_infos_t, *OBIDMS_infos_p; + + /** * @brief A structure describing an OBIDMS instance * @@ -114,12 +133,12 @@ typedef struct OBIDMS { int tax_dir_fd; /**< The file descriptor of the directory entry * usable to refer and scan the taxonomy directory. */ - bool little_endian; /**< Endianness of the database. - */ Opened_columns_list_p opened_columns; /**< List of opened columns. */ Opened_indexers_list_p opened_indexers; /**< List of opened indexers. */ + OBIDMS_infos_p infos; /**< A pointer on the mapped DMS information file. + */ } OBIDMS_t, *OBIDMS_p; @@ -239,6 +258,44 @@ OBIDMS_p obi_dms(const char* dms_path); int obi_close_dms(OBIDMS_p dms, bool force); +/** + * @brief Internal function writing new comments in a DMS informations file. + * + * The new comments replace the pre-existing ones. + * The informations file is enlarged if necessary. + * + * @param dms A pointer on the DMS. + * @param comments The new comments that should be written. + * + * @retval 0 if the operation was successfully completed. + * @retval -1 if an error occurred. + * + * @since September 2018 + * @author Celine Mercier (celine.mercier@metabarcoding.org) + */ +int obi_dms_write_comments(OBIDMS_p dms, const char* comments); + + +/** + * @brief Adds comments to a DMS informations file. + * + * This reads the comments in the JSON format and adds the key value pair. + * If the key already exists, the value format is turned into an array and the new value is appended + * if it is not already in the array. + * + * @param column A pointer on an OBIDMS column. + * @param key The key. + * @param value The value associated with the key. + * + * @retval 0 if the operation was successfully completed. + * @retval -1 if an error occurred. + * + * @since September 2018 + * @author Celine Mercier (celine.mercier@metabarcoding.org) + */ +int obi_dms_add_comment(OBIDMS_p dms, const char* key, const char* value); + + /** * @brief Returns a column identified by its name and its version number from the list of opened columns. *