obitools/obitools3
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 66 Wiki Activity

Merge branch 'multiple_avls_bloom'

Browse Source
This commit is contained in:
Celine Mercier
2016-03-22 14:14:10 +01:00
parent b04b4b5902 3681cecb4d
commit 4085904362
12 changed files with 704 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

211
src/obiavl.c
View File

@ -19,6 +19,9 @@
#include <fcntl.h>
#include <math.h>
//#include <libbloom.h>
#include "bloom.h"
#include "obiavl.h"
#include "obierrno.h"
#include "obitypes.h"
@ -30,6 +33,7 @@
#define DEBUG_LEVEL 0 // TODO has to be defined somewhere else (cython compil flag?)
/**************************************************************************
*
* D E C L A R A T I O N O F T H E P R I V A T E F U N C T I O N S
@ -481,21 +485,23 @@ int grow_avl(OBIDMS_avl_p avl) // TODO Lock when needed
int avl_file_descriptor;
char* avl_file_name;
// Get the avl file name
avl_file_name = build_avl_file_name((avl->header)->avl_name);
if (avl_file_name == NULL)
return -1;
avl_file_descriptor = avl->avl_fd;
// Open the avl file
avl_file_descriptor = openat(avl->dir_fd, avl_file_name, O_RDWR);
if (avl_file_descriptor < 0)
{
obi_set_errno(OBI_AVL_ERROR);
obidebug(1, "\nError opening an AVL tree file");
free(avl_file_name);
return -1;
}
free(avl_file_name);
// // Get the avl file name
// avl_file_name = build_avl_file_name((avl->header)->avl_name);
// if (avl_file_name == NULL)
// return -1;
//
// // Open the avl file
// avl_file_descriptor = openat(avl->dir_fd, avl_file_name, O_RDWR);
// if (avl_file_descriptor < 0)
// {
// obi_set_errno(OBI_AVL_ERROR);
// obidebug(1, "\nError opening an AVL tree file");
// free(avl_file_name);
// return -1;
// }
// free(avl_file_name);
// Calculate the new file size
old_data_size = (avl->header)->avl_size;
@ -544,7 +550,7 @@ int grow_avl(OBIDMS_avl_p avl) // TODO Lock when needed
// Set the new avl size
(avl->header)->avl_size = new_data_size;
close(avl_file_descriptor);
//close(avl_file_descriptor);
return 0;
}
@ -559,21 +565,23 @@ int grow_avl_data(OBIDMS_avl_p avl) // TODO Lock when needed
int avl_data_file_descriptor;
char* avl_data_file_name;
// Get the avl data file name
avl_data_file_name = build_avl_data_file_name((avl->header)->avl_name);
if (avl_data_file_name == NULL)
return -1;
avl_data_file_descriptor = avl->data_fd;
// Open the avl data file
avl_data_file_descriptor = openat(avl->dir_fd, avl_data_file_name, O_RDWR);
if (avl_data_file_descriptor < 0)
{
obi_set_errno(OBI_AVL_ERROR);
obidebug(1, "\nError opening an AVL tree data file");
free(avl_data_file_name);
return -1;
}
free(avl_data_file_name);
// // Get the avl data file name
// avl_data_file_name = build_avl_data_file_name((avl->header)->avl_name);
// if (avl_data_file_name == NULL)
// return -1;
//
// // Open the avl data file
// avl_data_file_descriptor = openat(avl->dir_fd, avl_data_file_name, O_RDWR);
// if (avl_data_file_descriptor < 0)
// {
// obi_set_errno(OBI_AVL_ERROR);
// obidebug(1, "\nError opening an AVL tree data file");
// free(avl_data_file_name);
// return -1;
// }
// free(avl_data_file_name);
// Calculate the new file size
old_data_size = ((avl->data)->header)->data_size_max;
@ -619,10 +627,12 @@ int grow_avl_data(OBIDMS_avl_p avl) // TODO Lock when needed
// Set new data size
((avl->data)->header)->data_size_max = new_data_size;
//fprintf(stderr, "\nGrowing AVL, new data size = %lld, count = %ld\n", new_data_size, (avl->header)->nb_items);
// Initialize new data to 0
memset(((avl->data)->data)+old_data_size, 0, new_data_size - old_data_size);
close(avl_data_file_descriptor);
//close(avl_data_file_descriptor);
return 0;
}
@ -996,6 +1006,74 @@ OBIDMS_avl_p obi_avl(OBIDMS_p dms, const char* avl_name)
}
OBIDMS_avl_group_p obi_create_avl_group(OBIDMS_p dms, const char* avl_name)
{
OBIDMS_avl_group_p avl_group;
char* avl_name_with_idx;
avl_group = (OBIDMS_avl_group_p) malloc(sizeof(OBIDMS_avl_group_t));
// Create 1st avl
asprintf(&avl_name_with_idx,"%s_%u", avl_name, 0);
(avl_group->sub_avls)[0] = obi_create_avl(dms, avl_name_with_idx);
avl_group->current_avl_idx = 0;
strcpy(avl_group->avl_name, avl_name);
avl_group->dms = dms;
return avl_group;
}
int unmap_an_avl(OBIDMS_avl_p avl)
{
if (munmap((avl->data)->data, ((avl->data)->header)->data_size_max) < 0)
return -1;
if (munmap(avl->tree, (((avl->header)->nb_items_max) * sizeof(AVL_node_t))) < 0)
return -1;
return 0;
}
int remap_an_avl(OBIDMS_avl_p avl)
{
(avl->data)->data = mmap(NULL,
((avl->data)->header)->data_size_max,
PROT_READ | PROT_WRITE,
MAP_SHARED,
avl->data_fd,
((avl->data)->header)->header_size);
if ((avl->data)->data == NULL)
return -1;
avl->tree = mmap(NULL,
((avl->header)->nb_items_max) * sizeof(AVL_node_t),
PROT_READ | PROT_WRITE,
MAP_SHARED,
avl->avl_fd,
(avl->header)->header_size);
if (avl->tree == NULL)
return -1;
return 0;
}
int obi_add_new_avl_in_group(OBIDMS_avl_group_p avl_group) // TODO check for errors
{
char* avl_name_with_idx;
// unmap older
unmap_an_avl((avl_group->sub_avls)[avl_group->current_avl_idx]);
(avl_group->current_avl_idx)++;
asprintf(&avl_name_with_idx,"%s_%u", avl_group->avl_name, avl_group->current_avl_idx);
(avl_group->sub_avls)[avl_group->current_avl_idx] = obi_create_avl(avl_group->dms, avl_name_with_idx);
return 0;
}
OBIDMS_avl_p obi_create_avl(OBIDMS_p dms, const char* avl_name)
{
char* avl_file_name;
@ -1098,7 +1176,7 @@ OBIDMS_avl_p obi_create_avl(OBIDMS_p dms, const char* avl_name)
// Initialize all bits to 0
memset(avl_data->data, 0, (avl_data->header)->data_size_max);
close(avl_data_file_descriptor);
//close(avl_data_file_descriptor);
// Create the AVL tree file
@ -1198,7 +1276,13 @@ OBIDMS_avl_p obi_create_avl(OBIDMS_p dms, const char* avl_name)
(avl->header)->creation_date = time(NULL);
strcpy((avl->header)->avl_name, avl_name);
close(avl_file_descriptor);
avl->avl_fd = avl_file_descriptor;
avl->data_fd = avl_data_file_descriptor;
// Bloom filter
bloom_init(&((avl->header)->bloom_filter), 2000000, 0.001); // TODO use macros
//close(avl_file_descriptor);
// Add in the list of opened AVL trees
*(((dms->opened_avls)->avls)+((dms->opened_avls)->nb_opened_avls)) = avl;
@ -1305,7 +1389,7 @@ OBIDMS_avl_p obi_open_avl(OBIDMS_p dms, const char* avl_name)
return NULL;
}
close(avl_data_file_descriptor);
//close(avl_data_file_descriptor);
// Open the AVL tree file
@ -1391,7 +1475,10 @@ OBIDMS_avl_p obi_open_avl(OBIDMS_p dms, const char* avl_name)
avl->directory = dms->avl_directory;
avl->dir_fd = avl_dir_file_descriptor;
close(avl_file_descriptor);
avl->avl_fd = avl_file_descriptor;
avl->data_fd = avl_data_file_descriptor;
//close(avl_file_descriptor);
// Add in the list of opened AVL trees
*(((dms->opened_avls)->avls)+((dms->opened_avls)->nb_opened_avls)) = avl;
@ -1456,6 +1543,53 @@ byte_t* obi_avl_get(OBIDMS_avl_p avl, index_t idx)
}
int maybe_in_avl(OBIDMS_avl_p avl, byte_t* value)
{
return (bloom_check(&((avl->header)->bloom_filter), value, (BYTE_ARRAY_HEADER_SIZE + *((int32_t*)(value+1)))));
}
index_t insert_in_avl_group(OBIDMS_avl_group_p avl_group, byte_t* value) // TODO won't be index_t
{
index_t index_if_already_in;
int i;
if (maybe_in_avl((avl_group->sub_avls)[avl_group->current_avl_idx], value))
{
//fprintf(stderr, "\nyah maybe");
index_if_already_in = obi_avl_find((avl_group->sub_avls)[avl_group->current_avl_idx], value);
if (index_if_already_in >= 0)
return index_if_already_in;
}
// else
// fprintf(stderr, "\nnah");
for (i=0; i < (avl_group->current_avl_idx); i++)
{
if (maybe_in_avl((avl_group->sub_avls)[i], value))
{
//fprintf(stderr, "\nyah maybe");
if (remap_an_avl((avl_group->sub_avls)[i]) < 0)
return -1;
index_if_already_in = obi_avl_find((avl_group->sub_avls)[i], value);
if (unmap_an_avl((avl_group->sub_avls)[i]) < 0)
return -1;
if (index_if_already_in >= 0)
return index_if_already_in;
}
// else
// fprintf(stderr, "\nnah");
}
// not found in any avl: add in current
// first, check if make new one
if ((((avl_group->sub_avls)[avl_group->current_avl_idx])->header)->nb_items == 2000000) // TODO add condition with data size + use macro
obi_add_new_avl_in_group(avl_group);
bloom_add(&((((avl_group->sub_avls)[avl_group->current_avl_idx])->header)->bloom_filter), value, (BYTE_ARRAY_HEADER_SIZE + *((int32_t*)(value+1))));
return obi_avl_add((avl_group->sub_avls)[avl_group->current_avl_idx], value);
}
// Insert a new node
index_t obi_avl_add(OBIDMS_avl_p avl, byte_t* value)
{
@ -1519,7 +1653,10 @@ index_t obi_avl_add(OBIDMS_avl_p avl, byte_t* value)
next = current_node->right_child;
else if (comp == 0)
// Value already stored
return current_node->value;
{
//fprintf(stderr, "\n>>>ALREADY IN, %s, %lld\n", obi_obibytes_to_seq(value), (avl->header)->nb_items);
return current_node->value; // TODO should trigger error if using bloom filters
}
depth++;
}
@ -1576,7 +1713,7 @@ index_t obi_avl_add(OBIDMS_avl_p avl, byte_t* value)
}
// Find if a value is already in an AVL tree
// Find if a value is already in an AVL tree TODO use bloom
index_t obi_avl_find(OBIDMS_avl_p avl, byte_t* value)
{
int comp;
@ -1632,7 +1769,7 @@ byte_t* obi_str_to_obibytes(char* value)
*((int32_t*)(value_b+1)) = length;
// Store the initial length (in bytes) of the decoded value (same as encoded for character strings)
*((int32_t*)(value_b+5)) = length;
*((int64_t*)(value_b+5)) = length;
// Store the character string
strcpy(value_b+BYTE_ARRAY_HEADER_SIZE, value);