/**************************************************************************** * OBIDMS AVL tree functions * ****************************************************************************/ /** * @file obiavl.c * @author Celine Mercier * @date December 3rd 2015 * @brief Functions handling AVL trees for storing and retrieving bit arrays. */ #include #include #include #include #include #include #include #include #include "bloom.h" #include "crc64.h" #include "obiavl.h" #include "obierrno.h" #include "obitypes.h" #include "obidebug.h" #include "private_at_functions.h" #include "encode.h" #define DEBUG_LEVEL 0 // TODO has to be defined somewhere else (cython compil flag?) /************************************************************************** * * D E C L A R A T I O N O F T H E P R I V A T E F U N C T I O N S * **************************************************************************/ /** * @brief Internal function building the file name for an AVL tree file. * * @warning The returned pointer has to be freed by the caller. * * @param avl_name The name of the AVL tree. * * @returns A pointer to the name of the file where the AVL tree is stored. * @retval NULL if an error occurred. * * @since December 2015 * @author Celine Mercier (celine.mercier@metabarcoding.org) */ static char* build_avl_file_name(const char* avl_name); /** * @brief Internal function building the file name for an AVL tree file. * * @warning The returned pointer has to be freed by the caller. * * @param avl_name The name of the AVL tree. * * @returns A pointer to the name of the file where the data referred to by the AVL tree is stored. * @retval NULL if an error occurred. * * @since December 2015 * @author Celine Mercier (celine.mercier@metabarcoding.org) */ static char* build_avl_data_file_name(const char* avl_name); /** * @brief Internal function returning the size of an AVL tree header on this platform. * * @returns The size of an AVL tree header in bytes. * * @since December 2015 * @author Celine Mercier (celine.mercier@metabarcoding.org) */ size_t get_avl_header_size(); /** * @brief Internal function returning the initial size of an AVL tree on this platform. * * @returns The initial size of an AVL tree in bytes. * * @since December 2015 * @author Celine Mercier (celine.mercier@metabarcoding.org) */ size_t get_initial_avl_size(); /** * @brief Internal function returning the size, on this platform, of the header of the data * referred to by an AVL tree. * * @returns The size of an AVL data header in bytes. * * @since December 2015 * @author Celine Mercier (celine.mercier@metabarcoding.org) */ size_t get_avl_data_header_size(); /** * @brief Internal function returning the initial size, on this platform, of the data * referred to by an AVL tree. * * @returns The initial size of an AVL data array in bytes. * * @since December 2015 * @author Celine Mercier (celine.mercier@metabarcoding.org) */ size_t get_initial_avl_data_size(); /** * @brief Internal function closing an AVL data structure where the data referred to by an AVL tree is stored. * * @param avl_data A pointer to the data structure referred to by an AVL tree. * * @retval 0 if the operation was successfully completed. * @retval -1 if an error occurred. * * @since December 2015 * @author Celine Mercier (celine.mercier@metabarcoding.org) */ int close_avl_data(OBIDMS_avl_data_p avl_data); /** * @brief Internal function enlarging an AVL tree. * * @param avl A pointer to the AVL tree structure. * * @retval 0 if the operation was successfully completed. * @retval -1 if an error occurred. * * @since December 2015 * @author Celine Mercier (celine.mercier@metabarcoding.org) */ int grow_avl(OBIDMS_avl_p avl); /** * @brief Internal function enlarging the data array referred to by an AVL tree. * * @param avl A pointer to the AVL tree structure. * * @retval 0 if the operation was successfully completed. * @retval -1 if an error occurred. * * @since December 2015 * @author Celine Mercier (celine.mercier@metabarcoding.org) */ int grow_avl_data(OBIDMS_avl_p avl); /** * @brief Internal function storing a value (byte array) in the data array referred to by an AVL tree. * * @param avl A pointer to the AVL tree structure. * @param value A pointer to the value (byte array). * * @returns The index of the stored value. * @retval -1 if an error occurred. * * @since December 2015 * @author Celine Mercier (celine.mercier@metabarcoding.org) */ index_t avl_add_value_in_data_array(OBIDMS_avl_p avl, byte_t* value); /** * @brief Internal function comparing two byte arrays. * * The encoding is compared first, then the length of the * values, then the values themselves. * * @param value_1 A pointer to the first byte array. * @param value_2 A pointer to the second byte array. * * @returns A value < 0 if value_1 < value_2, * a value > 0 if value_1 > value_2, * and 0 if value_1 == value_2. * * @since October 2015 * @author Celine Mercier (celine.mercier@metabarcoding.org) */ int byte_array_compare(byte_t* value_1, byte_t* value_2); /** * @brief Internal function calculating the size in bytes of a byte array. * * @param value A pointer to the byte array. * * @returns The size of the byte array in bytes. * * @since October 2015 * @author Celine Mercier (celine.mercier@metabarcoding.org) */ size_t byte_array_sizeof(byte_t* value); /** * @brief Internal function initializing a node in an AVL tree. * * @param avl A pointer to the AVL tree structure. * @param node_idx The index of the node to initialize in the mmapped AVL tree. * * @returns The node structure initialized. * * @since December 2015 * @author Celine Mercier (celine.mercier@metabarcoding.org) */ AVL_node_p avl_create_node(OBIDMS_avl_p avl, index_t node_idx); /** * @brief Internal function updating the balance factors in an AVL tree * after adding a node, only in the subtree that will have to be balanced. * That subtree is found using the avl->path_idx array and the directions taken * down the tree to add the new node are stored in the avl->path_dir array. * * @param avl A pointer to the AVL tree structure. * * @since December 2015 * @author Celine Mercier (celine.mercier@metabarcoding.org) */ void avl_update_balance_factors(OBIDMS_avl_p avl); /** * @brief Internal function rotating a node with a "left left rotation". * * @param avl A pointer to the AVL tree structure. * @param node A pointer to the node that has to be rotated. * @param node_idx The index of the node that has to be rotated. * * @returns The new root of the subtree. * * @since December 2015 * @author Celine Mercier (celine.mercier@metabarcoding.org) */ index_t avl_rotate_leftleft(OBIDMS_avl_p avl, AVL_node_p node, index_t node_idx); /** * @brief Internal function rotating a node with a "left right rotation". * * @param avl A pointer to the AVL tree structure. * @param node A pointer to the node that has to be rotated. * @param node_idx The index of the node that has to be rotated. * * @returns The new root of the subtree. * * @since December 2015 * @author Celine Mercier (celine.mercier@metabarcoding.org) */ index_t avl_rotate_leftright(OBIDMS_avl_p avl, AVL_node_p node, index_t node_idx); /** * @brief Internal function rotating a node with a "right left rotation". * * @param avl A pointer to the AVL tree structure. * @param node A pointer to the node that has to be rotated. * @param node_idx The index of the node that has to be rotated. * * @returns The new root of the subtree. * * @since December 2015 * @author Celine Mercier (celine.mercier@metabarcoding.org) */ index_t avl_rotate_rightleft(OBIDMS_avl_p avl, AVL_node_p node, index_t node_idx); /** * @brief Internal function rotating a node with a "right right rotation". * * @param avl A pointer to the AVL tree structure. * @param node A pointer to the node that has to be rotated. * @param node_idx The index of the node that has to be rotated. * * @returns The new root of the subtree. * * @since December 2015 * @author Celine Mercier (celine.mercier@metabarcoding.org) */ index_t avl_rotate_rightright(OBIDMS_avl_p avl, AVL_node_p node, index_t node_idx); /** * @brief Internal function balancing one node. * * @param avl A pointer to the AVL tree structure. * @param node A pointer to the node that has to be balanced. * @param node_idx The index of the node that has to be balanced. * * @returns The new root of the subtree. * * @since December 2015 * @author Celine Mercier (celine.mercier@metabarcoding.org) */ index_t avl_balance_node(OBIDMS_avl_p avl, AVL_node_p node, index_t node_idx); /** * @brief Internal function balancing the nodes of an AVL tree after adding a node, * only in the subtree that eventually has to be balanced. * That subtree is found using the avl->path_idx array. * * @param avl A pointer to the AVL tree structure. * * @since December 2015 * @author Celine Mercier (celine.mercier@metabarcoding.org) */ void avl_balance(OBIDMS_avl_p avl); /** * @brief Internal function printing a depth first traverse of a node. * * @param avl A pointer to the AVL tree structure. * @param node A pointer to the node. * @param node_idx The index of the node. * @param depth The depth of the node. * * @since December 2015 * @author Celine Mercier (celine.mercier@metabarcoding.org) */ void avl_print_node(OBIDMS_avl_p avl, AVL_node_p node, index_t node_idx, int depth); /** * @brief Internal function printing a depth first traverse of an AVL tree. * * @param avl A pointer to the AVL tree structure. * * @since December 2015 * @author Celine Mercier (celine.mercier@metabarcoding.org) */ void avl_print(OBIDMS_avl_p avl); /************************************************************************ * * D E F I N I T I O N O F T H E P R I V A T E F U N C T I O N S * ************************************************************************/ static char* build_avl_file_name(const char* avl_name) { char* file_name; // Build the file name file_name = (char*) malloc((strlen(avl_name) + 5)*sizeof(char)); if (sprintf(file_name,"%s.oda", avl_name) < 0) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError building an AVL tree file name"); return NULL; } // Test if the avl name is not too long if (strlen(file_name) >= AVL_MAX_NAME) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError due to AVL tree name too long"); free(file_name); return NULL; } return file_name; } static char* build_avl_data_file_name(const char* avl_name) { char* file_name; // Build the file name file_name = (char*) malloc((strlen(avl_name) + 5)*sizeof(char)); if (sprintf(file_name,"%s.odd", avl_name) < 0) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError building an AVL tree data file name"); return NULL; } return file_name; } size_t get_avl_header_size() { size_t header_size; size_t rounded_header_size; double multiple; header_size = sizeof(OBIDMS_avl_header_t); multiple = ceil((double) header_size / (double) getpagesize()); rounded_header_size = multiple * getpagesize(); return rounded_header_size; } size_t get_initial_avl_size() { size_t s; size_t m; m = 1; s = getpagesize() * m; return s; } size_t get_avl_data_header_size() { size_t header_size; size_t rounded_header_size; double multiple; header_size = sizeof(OBIDMS_avl_data_header_t); multiple = ceil((double) header_size / (double) getpagesize()); rounded_header_size = multiple * getpagesize(); return rounded_header_size; } size_t get_initial_avl_data_size() { size_t s; size_t m; m = 1; s = getpagesize() * m; return s; } int close_avl_data(OBIDMS_avl_data_p avl_data) { int ret_val = 0; if (munmap(avl_data->data, (avl_data->header)->data_size_max) < 0) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError munmapping the data of an AVL tree data file"); ret_val = -1; } if (munmap(avl_data->header, (avl_data->header)->header_size) < 0) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError munmapping the header of an AVL tree data file"); ret_val = -1; } free(avl_data); return ret_val; } int grow_avl(OBIDMS_avl_p avl) // TODO Lock when needed { size_t file_size; size_t old_data_size; size_t new_data_size; size_t header_size; int avl_file_descriptor; avl_file_descriptor = avl->avl_fd; // Calculate the new file size old_data_size = (avl->header)->avl_size; new_data_size = old_data_size * AVL_GROWTH_FACTOR; header_size = (avl->header)->header_size; file_size = header_size + new_data_size; // Enlarge the file if (ftruncate(avl_file_descriptor, file_size) < 0) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError enlarging an AVL tree file"); close(avl_file_descriptor); return -1; } // Unmap and re-map the data if (munmap(avl->tree, old_data_size) < 0) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError munmapping the tree of an AVL tree file before enlarging"); close(avl_file_descriptor); return -1; } avl->tree = mmap(NULL, new_data_size, PROT_READ | PROT_WRITE, MAP_SHARED, avl_file_descriptor, header_size ); if (avl->tree == MAP_FAILED) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError re-mmapping the tree of an AVL tree file after enlarging the file"); close(avl_file_descriptor); return -1; } // Set new maximum number of items (avl->header)->nb_items_max = floor(((double) new_data_size) / ((double) sizeof(AVL_node_t))); // Set the new avl size (avl->header)->avl_size = new_data_size; //close(avl_file_descriptor); return 0; } int grow_avl_data(OBIDMS_avl_p avl) // TODO Lock when needed { size_t file_size; index_t old_data_size; index_t new_data_size; size_t header_size; int avl_data_file_descriptor; avl_data_file_descriptor = avl->data_fd; // Calculate the new file size old_data_size = ((avl->data)->header)->data_size_max; new_data_size = old_data_size * AVL_GROWTH_FACTOR; header_size = ((avl->data)->header)->header_size; file_size = header_size + new_data_size; // Enlarge the file if (ftruncate(avl_data_file_descriptor, file_size) < 0) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError enlarging an AVL tree data file"); close(avl_data_file_descriptor); return -1; } // Unmap and re-map the data if (munmap((avl->data)->data, old_data_size) < 0) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError munmapping the data of an AVL tree data file before enlarging"); close(avl_data_file_descriptor); return -1; } (avl->data)->data = mmap(NULL, new_data_size, PROT_READ | PROT_WRITE, MAP_SHARED, avl_data_file_descriptor, header_size ); if ((avl->data)->data == MAP_FAILED) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError re-mmapping the data of an AVL tree data file after enlarging the file"); close(avl_data_file_descriptor); return -1; } // Set new data size ((avl->data)->header)->data_size_max = new_data_size; //fprintf(stderr, "\nGrowing AVL, new data size = %lld, count = %ld\n", new_data_size, (avl->header)->nb_items); // Initialize new data to 0 memset(((avl->data)->data)+old_data_size, 0, new_data_size - old_data_size); //close(avl_data_file_descriptor); return 0; } index_t avl_add_value_in_data_array(OBIDMS_avl_p avl, byte_t* value) { index_t value_idx; size_t value_size; value_idx = ((avl->data)->header)->data_size_used; // Grow the data if needed value_size = byte_array_sizeof(value); while (((avl->data)->header)->data_size_max < (value_idx + (int64_t) value_size)) { if (grow_avl_data(avl) < 0) return -1; } // Store the value itself at the end of the data memcpy((((avl->data)->data)+value_idx), value, value_size); // Update the data size ((avl->data)->header)->data_size_used = value_idx + value_size; // Update the number of items (((avl->data)->header)->nb_items)++; return value_idx; } int byte_array_compare(byte_t* value_1, byte_t* value_2) { int comp; uint8_t size_1; uint8_t size_2; int32_t len_1; int32_t len_2; int32_t ini_len_1; int32_t ini_len_2; int32_t b; size_1 = (uint8_t) *(value_1); size_2 = (uint8_t) *(value_2); if (size_1 != size_2) return (size_1 - size_2); len_1 = *((int32_t*)(value_1+1)); len_2 = *((int32_t*)(value_2+1)); if (len_1 != len_2) return (len_1 - len_2); if (size_1 != 8) { ini_len_1 = *((int32_t*)(value_1+5)); ini_len_2 = *((int32_t*)(value_2+5)); if (ini_len_1 != ini_len_2) return (ini_len_1 - ini_len_2); } b = BYTE_ARRAY_HEADER_SIZE; comp = 0; while (!comp && (b < len_1+BYTE_ARRAY_HEADER_SIZE)) { comp = *(value_1+b) - *(value_2+b); b++; } return comp; } size_t byte_array_sizeof(byte_t* value) { return (BYTE_ARRAY_HEADER_SIZE + *((int32_t*)(value+1))); } // Initialize a new node AVL_node_p avl_create_node(OBIDMS_avl_p avl, index_t node_idx) { AVL_node_p node; node = (avl->tree)+node_idx; node->left_child = -1; node->right_child = -1; node->balance_factor = 0; node->value = -1; node->crc64 = 0; // TODO return node; } // Update the balance factors of the nodes from the node that will need balancing void avl_update_balance_factors(OBIDMS_avl_p avl) { uint8_t n; AVL_node_p node; // Update balance factors from the node where balancing might be needed node=(avl->tree)+((avl->path_idx)[1]); for (n=1; (avl->path_dir)[n] != -1; n++) { if ((avl->path_dir)[n]) // Went right { (node->balance_factor)--; node=RIGHT_CHILD(node); } else // Went left { (node->balance_factor)++; node=LEFT_CHILD(node); } } } // Left Left Rotate index_t avl_rotate_leftleft(OBIDMS_avl_p avl, AVL_node_p node, index_t node_idx) { AVL_node_p left_child = LEFT_CHILD(node); index_t left_child_idx = node->left_child; node->left_child = left_child->right_child; left_child->right_child = node_idx; node->balance_factor = 0; left_child->balance_factor = 0; return left_child_idx; } // Left Right Rotate index_t avl_rotate_leftright(OBIDMS_avl_p avl, AVL_node_p node, index_t node_idx) { AVL_node_p left_child = LEFT_CHILD(node); index_t left_child_idx = node->left_child; AVL_node_p rc_of_lc = RIGHT_CHILD(left_child); index_t rc_of_lc_idx = left_child->right_child; node->left_child = rc_of_lc->right_child; left_child->right_child = rc_of_lc->left_child; rc_of_lc->left_child = left_child_idx; rc_of_lc->right_child = node_idx; if (rc_of_lc->balance_factor == -1) { left_child->balance_factor = 1; node->balance_factor = 0; } else if (rc_of_lc->balance_factor == 0) { left_child->balance_factor = 0; node->balance_factor = 0; } else // if (rc_of_lc->balance_factor == 1) { left_child->balance_factor = 0; node->balance_factor = -1; } rc_of_lc->balance_factor = 0; return rc_of_lc_idx; } // Right Left Rotate index_t avl_rotate_rightleft(OBIDMS_avl_p avl, AVL_node_p node, index_t node_idx) { AVL_node_p right_child = RIGHT_CHILD(node); index_t right_child_idx = node->right_child;; AVL_node_p lc_of_rc = LEFT_CHILD(right_child); index_t lc_of_rc_idx = right_child->left_child; node->right_child = lc_of_rc->left_child; right_child->left_child = lc_of_rc->right_child; lc_of_rc->right_child = right_child_idx; lc_of_rc->left_child = node_idx; if (lc_of_rc->balance_factor == 1) { right_child->balance_factor = 1; node->balance_factor = 0; } else if (lc_of_rc->balance_factor == 0) { right_child->balance_factor = 0; node->balance_factor = 0; } else // if (lc_of_rc->balance_factor == -1) { right_child->balance_factor = 0; node->balance_factor = 1; } lc_of_rc->balance_factor = 0; return lc_of_rc_idx; } // Right Right Rotate index_t avl_rotate_rightright(OBIDMS_avl_p avl, AVL_node_p node, index_t node_idx) { AVL_node_p right_child = RIGHT_CHILD(node); index_t right_child_idx = node->right_child; node->right_child = right_child->left_child; right_child->left_child = node_idx; node->balance_factor = 0; right_child->balance_factor = 0; return right_child_idx; } // Balance a given node index_t avl_balance_node(OBIDMS_avl_p avl, AVL_node_p node, index_t node_idx) { index_t new_root = 0; if (node->balance_factor == 2) { // Left Heavy if ((LEFT_CHILD(node))->balance_factor == -1) new_root = avl_rotate_leftright(avl, node, node_idx); else new_root = avl_rotate_leftleft(avl, node, node_idx); } else if (node->balance_factor == -2) { // Right Heavy if ((RIGHT_CHILD(node))->balance_factor == 1) new_root = avl_rotate_rightleft(avl, node, node_idx); else new_root = avl_rotate_rightright(avl, node, node_idx); } else // Node is balanced new_root = node_idx; return new_root; } // Balance a given tree void avl_balance(OBIDMS_avl_p avl) { index_t new_root; index_t node_index; AVL_node_p node_to_balance; AVL_node_p parent_of_node_to_balance; node_index = (avl->path_idx)[1]; node_to_balance = (avl->tree)+node_index; parent_of_node_to_balance = (avl->tree)+((avl->path_idx)[0]); // Balance the 2nd node stored in the path (the first is only kept to connect the new root // of the subtree if needed). new_root = avl_balance_node(avl, node_to_balance, node_index); if (new_root != node_index) // If the root of the subtree has changed { // If the subtree's root is the tree's root, store the new root if (node_index == (avl->header)->root_idx) (avl->header)->root_idx = new_root; // Else, connect the new subtree's root to the parent of the subtree else if ((avl->path_dir)[0]) // Subtree is the right child of its parent parent_of_node_to_balance->right_child = new_root; else // Subtree is the left child of its parent parent_of_node_to_balance->left_child = new_root; } } // Print a depth first traverse of a node void avl_print_node(OBIDMS_avl_p avl, AVL_node_p node, index_t node_idx, int depth) { int i = 0; if (node->left_child != -1) avl_print_node(avl, LEFT_CHILD(node), node->left_child, depth+2); for (i = 0; i < depth; i++) putchar(' '); fprintf(stderr, "Node idx: %lld, Value idx: %lld, Left child: %lld, Right child: %lld, " "Balance factor: %d\n", node_idx, node->value, node->left_child, node->right_child, node->balance_factor); if (node->right_child != -1) avl_print_node(avl, RIGHT_CHILD(node), node->right_child, depth+2); } // Print a depth first traverse of a tree void avl_print(OBIDMS_avl_p avl) { fprintf(stderr, "\nRoot index: %lld\n", (avl->header)->root_idx); avl_print_node(avl, (avl->tree)+((avl->header)->root_idx), (avl->header)->root_idx, 0); } /********************************************************************** * * D E F I N I T I O N O F T H E P U B L I C F U N C T I O N S * **********************************************************************/ int obi_avl_exists(OBIDMS_p dms, const char* avl_name) { struct stat buffer; char* avl_file_path; char* avl_file_name; char* avl_file_relative_path; int relative_path_size; int check_dir; // Build the AVL tree file path avl_file_name = build_avl_file_name(avl_name); if (avl_file_name == NULL) return -1; relative_path_size = strlen(avl_file_name) + strlen(AVL_TREES_DIR_NAME) + 2; avl_file_relative_path = (char*) malloc(relative_path_size*sizeof(char)); strcpy(avl_file_relative_path, AVL_TREES_DIR_NAME); strcat(avl_file_relative_path, "/"); strcat(avl_file_relative_path, avl_file_name); avl_file_path = get_full_path(dms, avl_file_relative_path); if (avl_file_path == NULL) { obidebug(1, "\nError getting the file path for an AVL tree file"); return -1; } check_dir = stat(avl_file_path, &buffer); free(avl_file_path); free(avl_file_name); free(avl_file_relative_path); if (check_dir == 0) return 1; else return 0; } OBIDMS_avl_p obi_avl(OBIDMS_p dms, const char* avl_name) { int exists; exists = obi_avl_exists(dms, avl_name); switch (exists) { case 0: return obi_create_avl(dms, avl_name); case 1: return obi_open_avl(dms, avl_name); }; obidebug(1, "\nError checking if an AVL tree already exists"); return NULL; } OBIDMS_avl_group_p obi_create_avl_group(OBIDMS_p dms, const char* avl_name) { OBIDMS_avl_group_p avl_group; char* avl_name_with_idx; avl_group = (OBIDMS_avl_group_p) malloc(sizeof(OBIDMS_avl_group_t)); // Create 1st avl avl_name_with_idx = malloc((strlen(avl_name) + 3)*sizeof(char)); if (sprintf(avl_name_with_idx, "%s_%u", avl_name, 0) < 0) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError building an AVL tree file name"); return NULL; } (avl_group->sub_avls)[0] = obi_create_avl(dms, avl_name_with_idx); if ((avl_group->sub_avls)[0] == NULL) { obidebug(1, "\nError creating the first AVL of an AVL group"); return NULL; } avl_group->current_avl_idx = 0; strcpy(avl_group->avl_name, avl_name); avl_group->dms = dms; return avl_group; } int unmap_an_avl(OBIDMS_avl_p avl) { if (munmap((avl->data)->data, ((avl->data)->header)->data_size_max) < 0) return -1; if (munmap(avl->tree, (((avl->header)->nb_items_max) * sizeof(AVL_node_t))) < 0) return -1; return 0; } int remap_an_avl(OBIDMS_avl_p avl) { (avl->data)->data = mmap(NULL, ((avl->data)->header)->data_size_max, PROT_READ, MAP_SHARED, // TODO test MAP_PRIVATE? avl->data_fd, ((avl->data)->header)->header_size); if ((avl->data)->data == NULL) return -1; avl->tree = mmap(NULL, ((avl->header)->nb_items_max) * sizeof(AVL_node_t), PROT_READ, MAP_SHARED, // TODO test MAP_PRIVATE? avl->avl_fd, (avl->header)->header_size); if (avl->tree == NULL) return -1; return 0; } int obi_add_new_avl_in_group(OBIDMS_avl_group_p avl_group) // TODO check for errors { char* avl_name_with_idx; int avl_idx_length; // unmap older unmap_an_avl((avl_group->sub_avls)[avl_group->current_avl_idx]); (avl_group->current_avl_idx)++; avl_idx_length = ((avl_group->current_avl_idx) == 0 ? 1 : (int)(log10(avl_group->current_avl_idx)+1)); avl_name_with_idx = malloc((strlen(avl_group->avl_name) + avl_idx_length + 2)*sizeof(char)); if (sprintf(avl_name_with_idx, "%s_%u", avl_group->avl_name, avl_group->current_avl_idx) < 0) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError building an AVL tree file name"); return -1; } (avl_group->sub_avls)[avl_group->current_avl_idx] = obi_create_avl(avl_group->dms, avl_name_with_idx); if ((avl_group->sub_avls)[avl_group->current_avl_idx] == NULL) { obidebug(1, "\nError creating a new AVL tree in a group"); return -1; } return 0; } OBIDMS_avl_p obi_create_avl(OBIDMS_p dms, const char* avl_name) { char* avl_file_name; char* avl_data_file_name; size_t header_size; size_t data_size; size_t file_size; int avl_file_descriptor; int avl_data_file_descriptor; int avl_dir_file_descriptor; OBIDMS_avl_data_p avl_data; OBIDMS_avl_p avl; // Create the data file // Build file name avl_data_file_name = build_avl_data_file_name(avl_name); if (avl_data_file_name == NULL) return NULL; // Get the file descriptor of the avl directory avl_dir_file_descriptor = dms->avl_dir_fd; // Create file avl_data_file_descriptor = openat(avl_dir_file_descriptor, avl_data_file_name, O_RDWR | O_CREAT | O_EXCL, 0777); if (avl_data_file_descriptor < 0) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError creating an AVL tree data file"); free(avl_data_file_name); return NULL; } free(avl_data_file_name); // Calculate the size needed header_size = get_avl_data_header_size(); data_size = get_initial_avl_data_size(); file_size = header_size + data_size; // Truncate the AVL tree data file to the right size if (ftruncate(avl_data_file_descriptor, file_size) < 0) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError truncating an AVL tree data file to the right size"); close(avl_data_file_descriptor); return NULL; } // Allocate the memory for the AVL tree data structure avl_data = (OBIDMS_avl_data_p) malloc(sizeof(OBIDMS_avl_data_t)); if (avl_data == NULL) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError allocating the memory for the AVL tree data structure"); close(avl_data_file_descriptor); return NULL; } // Fill the AVL tree data structure avl_data->header = mmap(NULL, header_size, PROT_READ | PROT_WRITE, MAP_SHARED, avl_data_file_descriptor, 0 ); if (avl_data->header == MAP_FAILED) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError mmapping the header of an AVL tree data file"); close(avl_data_file_descriptor); free(avl_data); return NULL; } avl_data->data = mmap(NULL, data_size, PROT_READ | PROT_WRITE, MAP_SHARED, avl_data_file_descriptor, header_size ); if (avl_data->data == MAP_FAILED) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError mmapping the data of an AVL tree data file"); munmap(avl_data->header, header_size); close(avl_data_file_descriptor); free(avl_data); return NULL; } (avl_data->header)->header_size = header_size; (avl_data->header)->data_size_max = data_size; (avl_data->header)->data_size_used = 0; (avl_data->header)->nb_items = 0; (avl_data->header)->creation_date = time(NULL); strcpy((avl_data->header)->avl_name, avl_name); // Initialize all bits to 0 memset(avl_data->data, 0, (avl_data->header)->data_size_max); //close(avl_data_file_descriptor); // Create the AVL tree file // Build file name avl_file_name = build_avl_file_name(avl_name); if (avl_file_name == NULL) { close_avl_data(avl_data); return NULL; } // Calculate the size needed header_size = get_avl_header_size(); data_size = get_initial_avl_size(); file_size = header_size + data_size; // Create file avl_file_descriptor = openat(avl_dir_file_descriptor, avl_file_name, O_RDWR | O_CREAT | O_EXCL, 0777); if (avl_file_descriptor < 0) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError creating an AVL tree file"); close_avl_data(avl_data); free(avl_file_name); return NULL; } free(avl_file_name); // Truncate the AVL tree file to the right size if (ftruncate(avl_file_descriptor, file_size) < 0) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError truncating an AVL tree file to the right size"); close_avl_data(avl_data); close(avl_file_descriptor); return NULL; } // Allocate the memory for the AVL tree structure avl = (OBIDMS_avl_p) malloc(sizeof(OBIDMS_avl_t)); if (avl == NULL) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError allocating the memory for the AVL tree structure"); close_avl_data(avl_data); close(avl_file_descriptor); return NULL; } // Fill the AVL tree structure avl->header = mmap(NULL, header_size, PROT_READ | PROT_WRITE, MAP_SHARED, avl_file_descriptor, 0 ); if (avl->header == MAP_FAILED) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError mmapping the header of an AVL tree file"); close_avl_data(avl_data); close(avl_file_descriptor); free(avl); return NULL; } avl->tree = mmap(NULL, data_size, PROT_READ | PROT_WRITE, MAP_SHARED, avl_file_descriptor, header_size ); if (avl->tree == MAP_FAILED) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError mmapping the data of an AVL tree file"); close_avl_data(avl_data); munmap(avl->header, header_size); close(avl_file_descriptor); free(avl); return NULL; } avl->dms = dms; avl->data = avl_data; avl->directory = dms->avl_directory; avl->dir_fd = avl_dir_file_descriptor; (avl->header)->header_size = header_size; (avl->header)->avl_size = data_size; (avl->header)->nb_items = 0; (avl->header)->nb_items_max = (index_t) floor(((double) get_initial_avl_size()) / ((double) sizeof(AVL_node_t))); (avl->header)->root_idx = -1; (avl->header)->creation_date = time(NULL); strcpy((avl->header)->avl_name, avl_name); avl->avl_fd = avl_file_descriptor; avl->data_fd = avl_data_file_descriptor; // Bloom filter bloom_init(&((avl->header)->bloom_filter), NODE_COUNT_PER_AVL, BLOOM_FILTER_ERROR_RATE); //close(avl_file_descriptor); // Add in the list of opened AVL trees *(((dms->opened_avls)->avls)+((dms->opened_avls)->nb_opened_avls)) = avl; ((dms->opened_avls)->nb_opened_avls)++; avl->counter = 1; return avl; } OBIDMS_avl_p obi_open_avl(OBIDMS_p dms, const char* avl_name) { char* avl_file_name; char* avl_data_file_name; size_t header_size; int avl_file_descriptor; int avl_data_file_descriptor; int avl_dir_file_descriptor; OBIDMS_avl_data_p avl_data; OBIDMS_avl_p avl; size_t i; // Check if the AVL tree is already in the list of opened AVL trees for (i=0; i < ((dms->opened_avls)->nb_opened_avls); i++) { if (!strcmp(((*(((dms->opened_avls)->avls)+i))->header)->avl_name, avl_name)) { // Found the AVL tree already opened ((*(((dms->opened_avls)->avls)+i))->counter)++; return *(((dms->opened_avls)->avls)+i); } } // Open the data file // Get the file descriptor of the AVL trees directory avl_dir_file_descriptor = dms->avl_dir_fd; // Build file name avl_data_file_name = build_avl_data_file_name(avl_name); if (avl_data_file_name == NULL) return NULL; // Open file avl_data_file_descriptor = openat(avl_dir_file_descriptor, avl_data_file_name, O_RDWR, 0777); if (avl_data_file_descriptor < 0) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError opening an AVL tree data file"); free(avl_data_file_name); return NULL; } free(avl_data_file_name); // Allocate the memory for the AVL tree data structure avl_data = (OBIDMS_avl_data_p) malloc(sizeof(OBIDMS_avl_data_t)); if (avl_data == NULL) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError allocating the memory for the AVL tree data structure"); close(avl_data_file_descriptor); return NULL; } // Read the header size if (read(avl_data_file_descriptor, &header_size, sizeof(size_t)) < ((ssize_t) sizeof(size_t))) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError reading the header size to open an AVL tree data file"); close(avl_data_file_descriptor); return NULL; } // Fill the avl data structure avl_data->header = mmap(NULL, header_size, PROT_READ | PROT_WRITE, MAP_SHARED, avl_data_file_descriptor, 0 ); if (avl_data->header == MAP_FAILED) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError mmapping the header of an AVL tree data file"); close(avl_data_file_descriptor); free(avl_data); return NULL; } avl_data->data = mmap(NULL, (avl_data->header)->data_size_max, PROT_READ | PROT_WRITE, MAP_SHARED, avl_data_file_descriptor, header_size ); if (avl_data->data == MAP_FAILED) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError mmapping the data of an AVL tree data file"); munmap(avl_data->header, header_size); close(avl_data_file_descriptor); free(avl_data); return NULL; } //close(avl_data_file_descriptor); // Open the AVL tree file // Build file name avl_file_name = build_avl_file_name(avl_name); if (avl_file_name == NULL) { close_avl_data(avl_data); return NULL; } // Open file avl_file_descriptor = openat(avl_dir_file_descriptor, avl_file_name, O_RDWR, 0777); if (avl_file_descriptor < 0) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError opening an AVL tree file"); close_avl_data(avl_data); free(avl_file_name); return NULL; } free(avl_file_name); // Allocate the memory for the AVL tree structure avl = (OBIDMS_avl_p) malloc(sizeof(OBIDMS_avl_t)); if (avl == NULL) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError allocating the memory for the AVL tree structure"); close_avl_data(avl_data); close(avl_file_descriptor); return NULL; } // Read the header size if (read(avl_file_descriptor, &header_size, sizeof(size_t)) < ((ssize_t) sizeof(size_t))) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError reading the header size to open an AVL tree"); close(avl_file_descriptor); return NULL; } // Fill the avl structure avl->header = mmap(NULL, header_size, PROT_READ | PROT_WRITE, MAP_SHARED, avl_file_descriptor, 0 ); if (avl->header == MAP_FAILED) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError mmapping the header of an AVL tree file"); close_avl_data(avl_data); close(avl_file_descriptor); free(avl); return NULL; } avl->tree = mmap(NULL, (((avl->header)->nb_items_max) * sizeof(AVL_node_t)), PROT_READ | PROT_WRITE, MAP_SHARED, avl_file_descriptor, header_size ); if (avl->tree == MAP_FAILED) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError mmapping the data of an AVL tree file"); close_avl_data(avl_data); munmap(avl->header, header_size); close(avl_file_descriptor); free(avl); return NULL; } avl->dms = dms; avl->data = avl_data; avl->directory = dms->avl_directory; avl->dir_fd = avl_dir_file_descriptor; avl->avl_fd = avl_file_descriptor; avl->data_fd = avl_data_file_descriptor; //close(avl_file_descriptor); // Add in the list of opened AVL trees *(((dms->opened_avls)->avls)+((dms->opened_avls)->nb_opened_avls)) = avl; ((dms->opened_avls)->nb_opened_avls)++; avl->counter = 1; return avl; } int obi_close_avl(OBIDMS_avl_p avl) { int ret_val = 0; size_t i; Opened_avls_list_p avls_list; OBIDMS_p dms; dms = avl->dms; avls_list = dms->opened_avls; (avl->counter)--; if (avl->counter == 0) { // Delete from the list of opened avls for (i=0; i < (avls_list->nb_opened_avls); i++) { if (!strcmp(((*((avls_list->avls)+i))->header)->avl_name, (avl->header)->avl_name)) { // Found the avl. Rearrange list (avls_list->nb_opened_avls)--; (avls_list->avls)[i] = (avls_list->avls)[avls_list->nb_opened_avls]; } } ret_val = close_avl_data(avl->data); if (munmap(avl->tree, (((avl->header)->nb_items_max) * sizeof(AVL_node_t))) < 0) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError munmapping the tree of an AVL tree file"); ret_val = -1; } if (munmap(avl->header, (avl->header)->header_size) < 0) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError munmapping the header of an AVL tree file"); ret_val = -1; } free(avl); } return ret_val; } byte_t* obi_avl_group_get(OBIDMS_avl_group_p avl_group, index_t idx) { int32_t avl_idx; index_t idx_in_avl; avl_idx = (int32_t) (idx >> 32); idx_in_avl = idx & 0x00000000FFFFFFFF; return obi_avl_get((avl_group->sub_avls)[avl_idx], idx_in_avl); } byte_t* obi_avl_get(OBIDMS_avl_p avl, index_t idx) { return (((avl->data)->data)+idx); } int maybe_in_avl(OBIDMS_avl_p avl, byte_t* value) { return (bloom_check(&((avl->header)->bloom_filter), value, (BYTE_ARRAY_HEADER_SIZE + *((int32_t*)(value+1))))); } int64_t insert_in_avl_group(OBIDMS_avl_group_p avl_group, byte_t* value) // TODO won't be index_t { int32_t index_in_avl; int64_t index_with_avl; int i; if (maybe_in_avl((avl_group->sub_avls)[avl_group->current_avl_idx], value)) { index_in_avl = (int32_t) obi_avl_find((avl_group->sub_avls)[avl_group->current_avl_idx], value); if (index_in_avl >= 0) { index_with_avl = avl_group->current_avl_idx; index_with_avl = index_with_avl << 32; index_with_avl = index_with_avl + index_in_avl; return index_with_avl; } } for (i=0; i < (avl_group->current_avl_idx); i++) { if (maybe_in_avl((avl_group->sub_avls)[i], value)) { if (remap_an_avl((avl_group->sub_avls)[i]) < 0) return -1; index_in_avl = (int32_t) obi_avl_find((avl_group->sub_avls)[i], value); if (unmap_an_avl((avl_group->sub_avls)[i]) < 0) return -1; if (index_in_avl >= 0) { index_with_avl = i; index_with_avl = index_with_avl << 32; index_with_avl = index_with_avl + index_in_avl; return index_with_avl; } } } // Not found in any AVL: add in current // First, check if make new AVL if ((((avl_group->sub_avls)[avl_group->current_avl_idx])->header)->nb_items == NODE_COUNT_PER_AVL) // TODO add condition with data size obi_add_new_avl_in_group(avl_group); // Add in the current AVL bloom_add(&((((avl_group->sub_avls)[avl_group->current_avl_idx])->header)->bloom_filter), value, BYTE_ARRAY_HEADER_SIZE + *((int32_t*)(value+1))); // Build the index containing the AVL index index_in_avl = (int32_t) obi_avl_add((avl_group->sub_avls)[avl_group->current_avl_idx], value); index_with_avl = avl_group->current_avl_idx; index_with_avl = index_with_avl << 32; index_with_avl = index_with_avl + index_in_avl; return index_with_avl; } // Insert a new node index_t obi_avl_add(OBIDMS_avl_p avl, byte_t* value) { AVL_node_p node_to_add = NULL; AVL_node_p current_node; index_t next, parent; index_t value_data_idx; index_t node_idx; byte_t* to_compare; int comp; int n = 0; int depth = 0; uint64_t crc; crc = crc64(value, BYTE_ARRAY_HEADER_SIZE + ((uint64_t) (*((int32_t*)(value+1))))); // TODO warning // Check if first node if (!((avl->header)->nb_items)) { node_to_add = avl_create_node(avl, 0); // Add the value in the data array and store its index value_data_idx = avl_add_value_in_data_array(avl, value); node_to_add->value = value_data_idx; node_to_add->crc64 = crc; // Update the number of items ((avl->header)->nb_items)++; // Set the AVL tree root (avl->header)->root_idx = 0; return 0; } // Not first node next = (avl->header)->root_idx; parent = next; comp = 0; while (next != -1) { current_node = (avl->tree)+next; // Store path from the lowest node with a balance factor different than 0, // as it is the node that will have to be balanced. if (current_node->balance_factor != 0) // New lowest node with a balance factor different than 0 n=0; (avl->path_idx)[n] = parent; // Store parent (avl->path_dir)[n] = comp < 0; // Store direction (0 if left, 1 if right) n++; parent = next; // Compare value with value of current node //to_compare = obi_avl_get(avl, current_node->value); //comp = byte_array_compare(to_compare, value); comp = (current_node->crc64) - crc; if (comp == 0) { // check if really same value to_compare = obi_avl_get(avl, current_node->value); comp = byte_array_compare(to_compare, value); } if (comp > 0) // Go to left child next = current_node->left_child; else if (comp < 0) // Go to right child next = current_node->right_child; else if (comp == 0) // Value already stored { fprintf(stderr, "\n>>>ALREADY IN, %s, %lld\n", obi_obibytes_to_seq(value), (avl->header)->nb_items); return current_node->value; // TODO should trigger error if using bloom filters } depth++; } // Check if the AVL tree has not become too big if (depth == AVL_MAX_DEPTH) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nThis AVL tree has reached the maximum height (%d).", AVL_MAX_DEPTH); return -1; } // Grow the AVL tree if needed if ((avl->header)->nb_items == (avl->header)->nb_items_max) { if (grow_avl(avl) < 0) return -1; } // Initialize node at the end of the tree node_idx = (avl->header)->nb_items; node_to_add = avl_create_node(avl, node_idx); // Add the value in the data array and store its index value_data_idx = avl_add_value_in_data_array(avl, value); node_to_add->value = value_data_idx; node_to_add->crc64 = crc; // Update the number of items ((avl->header)->nb_items)++; // Add either as right or left child if (comp > 0) // Add as left child ((avl->tree)+parent)->left_child = node_idx; else // Add as right child ((avl->tree)+parent)->right_child = node_idx; // End path (avl->path_idx)[n] = parent; (avl->path_dir)[n] = comp < 0; // 0 if went left, 1 if went right n++; (avl->path_idx)[n] = -1; // flag path end (avl->path_dir)[n] = -1; // Update balance factors avl_update_balance_factors(avl); // Balance tree avl_balance(avl); // Print tree //avl_print(avl); return value_data_idx; } // Find if a value is already in an AVL tree index_t obi_avl_find(OBIDMS_avl_p avl, byte_t* value) { int comp; index_t next; byte_t* to_compare; AVL_node_p current_node; uint64_t crc; crc = crc64(value, BYTE_ARRAY_HEADER_SIZE + ((uint64_t) (*((int32_t*)(value+1))))); // TODO warning next = (avl->header)->root_idx; while (next != -1) { current_node = (avl->tree)+next; // Compare value with value of current node //to_compare = obi_avl_get(avl, current_node->value); //comp = byte_array_compare(to_compare, value); comp = (current_node->crc64) - crc; if (comp == 0) { // check if really same value to_compare = obi_avl_get(avl, current_node->value); comp = byte_array_compare(to_compare, value); } if (comp > 0) // Go to left child next = current_node->left_child; else if (comp < 0) // Go to right child next = current_node->right_child; else if (comp == 0) { // Value found fprintf(stderr, "\n>>>ALREADY IN in find, %s, %lld\n", obi_obibytes_to_seq(value), (avl->header)->nb_items); return current_node->value; } } // Value not found return -1; } byte_t* obi_str_to_obibytes(char* value) { byte_t* value_b; int32_t length; // Compute the number of bytes on which the value will be encoded length = strlen(value) + 1; // +1 to store \0 at the end (makes retrieving faster) // Allocate the memory for the encoded value value_b = (byte_t*) malloc(BYTE_ARRAY_HEADER_SIZE + length); if (value_b == NULL) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError allocating memory for a byte array"); return NULL; } // Store the number of bits on which each element is encoded *(value_b) = 8; // Store the length (in bytes) of the encoded value (same as decoded for character strings) *((int32_t*)(value_b+1)) = length; // Store the initial length (in bytes) of the decoded value (same as encoded for character strings) *((int64_t*)(value_b+5)) = length; // Store the character string strcpy(value_b+BYTE_ARRAY_HEADER_SIZE, value); return value_b; } const char* obi_obibytes_to_str(byte_t* value_b) { const char* value; value = value_b+BYTE_ARRAY_HEADER_SIZE; return value; } byte_t* obi_seq_to_obibytes(char* seq) { byte_t* value_b; int32_t length; // length of the value (without the header) in bytes uint8_t size; // size of one element in bits int32_t seq_length; byte_t* encoded_seq; // Check if just ATGC and set size of a nucleotide accordingly (2 bits or 4 bits) if (only_ATGC(seq)) size = 2; else size = 4; // Compute the length (in bytes) of the encoded sequence seq_length = strlen(seq); if (size == 2) length = ceil((double) seq_length / (double) 4.0); else // size == 4 length = ceil((double) seq_length / (double) 2.0); // Encode if (size == 2) encoded_seq = encode_seq_on_2_bits(seq, seq_length); else // size == 4 encoded_seq = encode_seq_on_4_bits(seq, seq_length); if (encoded_seq == NULL) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError encoding a DNA sequence"); return NULL; } // Allocate the memory for the encoded value value_b = (byte_t*) malloc(BYTE_ARRAY_HEADER_SIZE + length); if (value_b == NULL) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError allocating memory for a byte array"); return NULL; } // Store the number of bits on which each nucleotide is encoded *(value_b) = size; // Store the length (in bytes) of the encoded sequence *((int32_t*)(value_b+1)) = length; // Store the length (in bytes) of the initial sequence (necessary for decoding) *((int32_t*)(value_b+5)) = seq_length; // Store the encoded sequence memcpy(value_b+BYTE_ARRAY_HEADER_SIZE, encoded_seq, length); free(encoded_seq); return value_b; } const char* obi_obibytes_to_seq(byte_t* value_b) { const char* value; uint8_t size; // size of one element in bits // Check the encoding (each nucleotide on 2 bits or 4 bits) size = *(value_b); // Decode if (size == 2) value = decode_seq_on_2_bits(value_b+BYTE_ARRAY_HEADER_SIZE, *((int32_t*)(value_b+5))); else value = decode_seq_on_4_bits(value_b+BYTE_ARRAY_HEADER_SIZE, *((int32_t*)(value_b+5))); if (value == NULL) { obi_set_errno(OBI_AVL_ERROR); obidebug(1, "\nError decoding a DNA sequence"); return NULL; } return value; }