obitools/obitools3
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 66 Wiki Activity

Branch to refactor and debug (AVLs bugged)

Browse Source
This commit is contained in:
Celine Mercier
2016-04-08 15:38:57 +02:00
parent edc4fd7b3e
commit 019dfc01b4
32 changed files with 1553 additions and 812 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
src/bloom.c
View File

@ -122,6 +122,40 @@ static void setup_buckets(struct bloom * bloom, unsigned int cache_size)
}
// TODO
int bloom_filter_size(int entries, double error)
{
int bytes;
double num;
double denom;
double bpe;
int bits;
unsigned bucket_bytes;
int not_even_by;
num = log(error);
denom = 0.480453013918201; // ln(2)^2
bpe = -(num / denom);
bits = (int)(((double)entries) * bpe);
if (bits % 8) {
bytes = (bits / 8) + 1;
}
else {
bytes = bits / 8;
}
bucket_bytes = BLOOM_BUCKET_SIZE_FALLBACK;
not_even_by = bytes % bucket_bytes;
if (not_even_by) {
// adjust bytes
bytes += (bucket_bytes - not_even_by);
}
return bytes;
}
int bloom_init_size(struct bloom * bloom, int entries, double error,
unsigned int cache_size)
{
@ -151,19 +185,21 @@ int bloom_init_size(struct bloom * bloom, int entries, double error,
setup_buckets(bloom, cache_size);
bloom->bf = (unsigned char *)calloc(bloom->bytes, sizeof(unsigned char));
if (bloom->bf == NULL) {
return 1;
}
// TODO comment
memset(bloom->bf, 0, bloom->bytes);
//bloom->bf = (unsigned char *)calloc(bloom->bytes, sizeof(unsigned char));
//if (bloom->bf == NULL) {
// return 1;
//}
bloom->ready = 1;
return 0;
}
int bloom_init(struct bloom * bloom, int entries, double error)
int bloom_init(struct bloom * bloom, int entries) //, double error)
{
return bloom_init_size(bloom, entries, error, 0);
return bloom_init_size(bloom, entries, BLOOM_FILTER_ERROR_RATE, 0);
}

15
src/bloom.h
View File

@ -9,6 +9,10 @@
#define _BLOOM_H
// TODO
#define BLOOM_FILTER_ERROR_RATE (0.001)
/** ***************************************************************************
* On Linux, the code attempts to compute a bucket size based on CPU cache
* size info, if available. If that fails for any reason, this fallback size
@ -60,10 +64,17 @@ struct bloom
unsigned bucket_bits_fast_mod_operand;
double bpe;
unsigned char * bf;
int ready;
unsigned char bf[];
};
typedef struct bloom bloom_t;
// TODO
int bloom_filter_size(int entries, double error);
/** ***************************************************************************
* Initialize the bloom filter for use.
@ -91,7 +102,7 @@ struct bloom
* 1 - on failure
*
*/
int bloom_init(struct bloom * bloom, int entries, double error);
int bloom_init(struct bloom * bloom, int entries); //, double error);
/** ***************************************************************************

133
src/encode.c
View File

@ -64,6 +64,12 @@ byte_t* encode_seq_on_2_bits(char* seq, int32_t length)
length_b = ceil((double) length / (double) 4.0);
seq_b = (byte_t*) malloc(length_b * sizeof(byte_t));
if (seq_b == NULL)
{
obi_set_errno(OBI_MALLOC_ERROR); // TODO
obidebug(1, "\nError allocating memory for an encoded DNA sequence");
return NULL;
}
// Initialize all the bits to 0
memset(seq_b, 0, length_b);
@ -93,6 +99,7 @@ byte_t* encode_seq_on_2_bits(char* seq, int32_t length)
seq_b[i/4] |= NUC_T_2b;
break;
default:
obi_set_errno(OBI_ENCODE_ERROR); // TODO
obidebug(1, "\nInvalid nucleotide base when encoding (not [atgcATGC])");
return NULL;
}
@ -116,6 +123,12 @@ char* decode_seq_on_2_bits(byte_t* seq_b, int32_t length_seq)
uint8_t nuc;
seq = (char*) malloc((length_seq+1) * sizeof(char));
if (seq == NULL)
{
obi_set_errno(OBI_MALLOC_ERROR); // TODO
obidebug(1, "\nError allocating memory for a decoded DNA sequence");
return NULL;
}
for (i=0; i<length_seq; i++)
{
@ -138,6 +151,7 @@ char* decode_seq_on_2_bits(byte_t* seq_b, int32_t length_seq)
seq[i] = 't';
break;
default:
obi_set_errno(OBI_DECODE_ERROR); // TODO
obidebug(1, "\nInvalid nucleotide base when decoding");
return NULL;
}
@ -159,6 +173,12 @@ byte_t* encode_seq_on_4_bits(char* seq, int32_t length)
length_b = ceil((double) length / (double) 2.0);
seq_b = (byte_t*) malloc(length_b * sizeof(byte_t));
if (seq_b == NULL)
{
obi_set_errno(OBI_MALLOC_ERROR); // TODO
obidebug(1, "\nError allocating memory for an encoded DNA sequence");
return NULL;
}
// Initialize all the bits to 0
memset(seq_b, 0, length_b);
@ -232,6 +252,7 @@ byte_t* encode_seq_on_4_bits(char* seq, int32_t length)
seq_b[i/2] |= NUC_N_4b;
break;
default:
obi_set_errno(OBI_ENCODE_ERROR); // TODO
obidebug(1, "\nInvalid nucleotide base when encoding (not IUPAC)");
return NULL;
}
@ -255,6 +276,12 @@ char* decode_seq_on_4_bits(byte_t* seq_b, int32_t length_seq)
uint8_t nuc;
seq = (char*) malloc((length_seq+1) * sizeof(char));
if (seq == NULL)
{
obi_set_errno(OBI_MALLOC_ERROR); // TODO
obidebug(1, "\nError allocating memory for a decoded DNA sequence");
return NULL;
}
for (i=0; i<length_seq; i++)
{
@ -310,6 +337,7 @@ char* decode_seq_on_4_bits(byte_t* seq_b, int32_t length_seq)
seq[i] = 'n';
break;
default:
obi_set_errno(OBI_DECODE_ERROR); // TODO
obidebug(1, "\nInvalid nucleotide base when decoding");
return NULL;
}
@ -321,6 +349,111 @@ char* decode_seq_on_4_bits(byte_t* seq_b, int32_t length_seq)
}
Obi_byte_array_p obi_byte_array(byte_t* encoded_value, uint8_t element_size, int32_t length_encoded_value, int32_t length_decoded_value)
{
Obi_byte_array_p byte_array;
// Allocate the memory for the byte array structure
byte_array = (Obi_byte_array_p) malloc(sizeof(Obi_byte_array_t) + length_encoded_value);
if (byte_array == NULL)
{
obi_set_errno(OBI_MALLOC_ERROR); // TODO
obidebug(1, "\nError allocating memory for a byte array");
return NULL;
}
// Store the number of bits on which each element is encoded
byte_array->element_size = element_size;
// Store the length (in bytes) of the encoded value
byte_array->length_encoded_value = length_encoded_value;
// Store the initial length (in bytes) of the decoded value
byte_array->length_decoded_value = length_decoded_value;
// Store the encoded value
memcpy(byte_array->value, encoded_value, length_encoded_value);
return byte_array;
}
Obi_byte_array_p obi_str_to_obibytes(char* value)
{
Obi_byte_array_p value_b;
int32_t length;
// Compute the number of bytes on which the value will be encoded
length = strlen(value) + 1; // +1 to store \0 at the end (makes retrieving faster)
value_b = obi_byte_array(value, ELEMENT_SIZE_STR, length, length);
if (value_b == NULL)
{
obidebug(1, "\nError encoding a character string in a byte array");
return NULL;
}
return value_b;
}
const char* obi_obibytes_to_str(Obi_byte_array_p value_b)
{
fprintf(stderr, "\n%s", value_b->value);
return value_b->value;
}
Obi_byte_array_p obi_seq_to_obibytes(char* seq)
{
Obi_byte_array_p value_b;
int32_t length_encoded_seq; // length of the encoded sequence in bytes
int32_t seq_length;
byte_t* encoded_seq;
seq_length = strlen(seq);
// Check if just ATGC and encode accordingly
if (only_ATGC(seq))
{
// Compute the length (in bytes) of the encoded sequence
length_encoded_seq = ceil((double) seq_length / (double) 4.0);
// Encode
encoded_seq = encode_seq_on_2_bits(seq, seq_length);
if (encoded_seq == NULL)
return NULL;
value_b = obi_byte_array(encoded_seq, ELEMENT_SIZE_SEQ_2, length_encoded_seq, seq_length);
}
else
{
// Compute the length (in bytes) of the encoded sequence
length_encoded_seq = ceil((double) seq_length / (double) 2.0);
// Encode
encoded_seq = encode_seq_on_4_bits(seq, seq_length);
if (encoded_seq == NULL)
return NULL;
value_b = obi_byte_array(encoded_seq, ELEMENT_SIZE_SEQ_4, length_encoded_seq, seq_length);
}
free(encoded_seq);
return value_b;
}
const char* obi_obibytes_to_seq(Obi_byte_array_p value_b)
{
// Decode
if (value_b->element_size == 2)
return decode_seq_on_2_bits(value_b->value, value_b->length_decoded_value);
else
return decode_seq_on_4_bits(value_b->value, value_b->length_decoded_value);
}
// TODO same for int
///////////////////// FOR DEBUGGING ///////////////////////////
//NOTE: The first byte is printed the first (at the left-most).

93
src/encode.h
View File

@ -10,6 +10,10 @@
*/
#ifndef ENCODE_H_
#define ENCODE_H_
#include <stdlib.h>
#include <stdio.h>
#include <stdint.h>
@ -18,8 +22,31 @@
#include "obitypes.h"
#define NUC_MASK_2B 0x3 /**< Binary: 11 to use when decoding 2 bits sequences */
#define NUC_MASK_4B 0xF /**< Binary: 1111 to use when decoding 4 bits sequences */
#define NUC_MASK_2B 0x3 /**< Binary: 11 to use when decoding 2 bits sequences
*/
#define NUC_MASK_4B 0xF /**< Binary: 1111 to use when decoding 4 bits sequences
*/
#define ELEMENT_SIZE_STR (8) /**< The size of an element from a value of type character string.
*/
#define ELEMENT_SIZE_SEQ_2 (2) /**< The size of an element from a value of type DNA sequence encoded on 2 bits.
*/
#define ELEMENT_SIZE_SEQ_4 (4) /**< The size of an element from a value of type DNA sequence encoded on 4 bits.
*/
/**
* @brief Byte array structure.
*/
typedef struct Obi_byte_array {
uint8_t element_size; /**< Size in bits of one element from the value.
*/
int32_t length_encoded_value; /**< Length in bytes of the encoded value.
*/
int32_t length_decoded_value; /**< Length in bytes of the decoded value.
*/
byte_t value[]; /**< Encoded value.
*/
} Obi_byte_array_t, *Obi_byte_array_p;
/**
@ -174,8 +201,70 @@ byte_t* encode_seq_on_4_bits(char* seq, int32_t length);
char* decode_seq_on_4_bits(byte_t* seq_b, int32_t length_seq);
/**
* @brief Converts a character string to a byte array with a header.
*
* @warning The byte array must be freed by the caller.
*
* @param value The character string to convert.
*
* @returns A pointer to the byte array created.
* @retval NULL if an error occurred.
*
* @since October 2015
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
Obi_byte_array_p obi_str_to_obibytes(char* value);
/**
* @brief Converts a byte array to a character string.
*
* @param value_b The byte array to convert.
*
* @returns A pointer to the character string contained in the byte array.
*
* @since October 2015
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
const char* obi_obibytes_to_str(Obi_byte_array_p value_b);
/**
* @brief Converts a DNA sequence to a byte array with a header.
*
* @warning The byte array must be freed by the caller.
*
* @param value The DNA sequence to convert.
*
* @returns A pointer to the byte array created.
* @retval NULL if an error occurred.
*
* @since November 2015
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
Obi_byte_array_p obi_seq_to_obibytes(char* seq);
/**
* @brief Converts a byte array to a DNA sequence.
*
* @param value_b The byte array to convert.
*
* @returns A pointer to the DNA sequence contained in the byte array.
* @retval NULL if an error occurred.
*
* @since November 2015
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
const char* obi_obibytes_to_seq(Obi_byte_array_p value_b); // TODO move to encode source files
////////// FOR DEBUGGING ///////////
// little endian
void print_bits(void* ptr, int32_t length);
#endif /* ENCODE_H_ */

1421
src/obiavl.c
View File

File diff suppressed because it is too large Load Diff

324
src/obiavl.h
View File

@ -25,41 +25,46 @@
#include "obidms.h"
#include "obitypes.h"
#include "bloom.h"
#include "utils.h"
#include "encode.h"
#define NODE_COUNT_PER_AVL (10000000)
#define BLOOM_FILTER_ERROR_RATE (0.001)
#define MAX_NB_OF_AVLS_IN_GROUP (100) /**< The maximum number of AVL trees in a group. // TODO discuss
*/
#define MAX_NODE_COUNT_PER_AVL (10000000) /**< The maximum number of nodes in an AVL tree.
* Only used to decide when to create a new AVL in a group, and to initialize the bloom filter // TODO discuss.
*/
#define MAX_DATA_SIZE_PER_AVL (1073741824) /**< The maximum size of the data referred to by an AVL tree in a group.
* Only used to decide when to create a new AVL in a group.
* Should not be greater than int32_t max (2,147,483,647), as indexes will have to be stored on 32 bits.
* Here 1073741824 B = 1 GB
*/
#define AVL_MAX_DEPTH (1024) /**< The maximum depth of an AVL tree. Used to save paths through the tree.
*/
#define AVL_MAX_NAME (1024) /**< The maximum length of an AVL tree name.
*/
#define AVL_GROWTH_FACTOR (2) /**< The growth factor when an AVL tree is enlarged.
*/
#define AVL_MAX_DEPTH (1000) /**< The maximum depth of an AVL tree.
*/
#define LEFT_CHILD(node) (avl->tree)+(node->left_child) /**< Pointer to the left child of a node in an AVL tree.
*/
#define RIGHT_CHILD(node) (avl->tree)+(node->right_child) /**< Pointer to the right child of a node in an AVL tree.
*/
#define BYTE_ARRAY_HEADER_SIZE (9) /**< The size of the header of a byte array.
*/
typedef struct bloom bloom_t;
/**
* @brief AVL tree node structure.
*/
typedef struct AVL_node {
index_t left_child; /**< Index of left less child node.
*/
index_t right_child; /**< Index of right greater child node.
*/
int8_t balance_factor; /**< Balance factor of the node.
*/
index_t value; /**< Index of the value associated with the node in the data array.
*/
uint64_t crc64; // TODO
index_t left_child; /**< Index of left less child node.
*/
index_t right_child; /**< Index of right greater child node.
*/
int8_t balance_factor; /**< Balance factor of the node.
*/
index_t value; /**< Index of the value associated with the node in the data array.
*/
uint64_t crc64; /**< Cyclic Redundancy Check code on 64 bits associated with the value.
*/
} AVL_node_t, *AVL_node_p;
@ -88,8 +93,10 @@ typedef struct OBIDMS_avl_data_header {
typedef struct OBIDMS_avl_data {
OBIDMS_avl_data_header_p header; /**< A pointer to the header of the AVL tree data.
*/
byte_t* data; /**< A pointer to the beginning of the data.
byte_t* data; /**< A pointer to the beginning of the data.
*/
int data_fd; /**< File descriptor of the file containing the data.
*/
} OBIDMS_avl_data_t, *OBIDMS_avl_data_p;
@ -111,7 +118,9 @@ typedef struct OBIDMS_avl_header {
*/
time_t creation_date; /**< Date of creation of the file.
*/
bloom_t bloom_filter;
bloom_t bloom_filter; /**< Bloom filter associated with the AVL tree, enabling to know if a value
* might already be stored in the data associated with the tree.
*/
} OBIDMS_avl_header_t, *OBIDMS_avl_header_p;
@ -119,30 +128,30 @@ typedef struct OBIDMS_avl_header {
* @brief OBIDMS AVL tree structure.
*/
typedef struct OBIDMS_avl {
OBIDMS_p dms; /**< A pointer to the OBIDMS structure to which the AVL tree belongs.
*/
OBIDMS_avl_header_p header; /**< A pointer to the header of the AVL tree.
*/
struct AVL_node* tree; /**< A pointer to the root of the AVL tree.
*/
OBIDMS_p dms; /**< A pointer to the OBIDMS structure to which the AVL tree belongs.
*/
OBIDMS_avl_header_p header; /**< A pointer to the header of the AVL tree.
*/
struct AVL_node* tree; /**< A pointer to the root of the AVL tree.
*/
index_t path_idx[AVL_MAX_DEPTH]; /**< The path taken to a node from the root as an array of node indices.
*/
int8_t path_dir[AVL_MAX_DEPTH]; /**< The path taken to a node from the root as an array of directions
* (0 for left, -1 for right).
* (0 for left, -1 for right).
*/
OBIDMS_avl_data_p data; /**< A pointer to the structure containing the data
* that the AVL tree references.
*/
DIR* directory; /**< A directory entry usable to
* refer and scan the AVL tree directory.
*/
int dir_fd; /**< The file descriptor of the directory entry
* usable to refer and scan the AVL tree directory.
*/
size_t counter; /**< Indicates by how many threads/programs (TODO) the AVL tree is used.
*/
int avl_fd;
int data_fd;
OBIDMS_avl_data_p data; /**< A pointer to the structure containing the data
* that the AVL tree references.
*/
DIR* directory; /**< A directory entry usable to
* refer and scan the AVL tree directory.
*/
int dir_fd; /**< The file descriptor of the directory entry
* usable to refer and scan the AVL tree directory.
*/
int avl_fd; /**< The file descriptor of the file containing the AVL tree.
*/
size_t counter; /**< Indicates by how many threads/programs (TODO) the AVL tree is used.
*/
} OBIDMS_avl_t, *OBIDMS_avl_p;
@ -150,28 +159,26 @@ typedef struct OBIDMS_avl {
* @brief OBIDMS AVL tree group structure.
*/
typedef struct OBIDMS_avl_group {
// TODO put each group in a directory later
OBIDMS_avl_p sub_avls[64]; // TODO macro for max
int current_avl_idx;
char avl_name[AVL_MAX_NAME+1];
OBIDMS_p dms;
OBIDMS_avl_p sub_avls[MAX_NB_OF_AVLS_IN_GROUP]; /**< Array containing the pointers to the AVL trees of the group.
*/
int current_avl_idx; /**< Index in the sub_avls array of the AVL tree currently being filled.
*/
char avl_name[AVL_MAX_NAME+1]; /**< Base name of the AVL group. The AVL trees in it have names of the form basename_idx.
*/
OBIDMS_p dms; /**< Pointer to the OBIDMS structure to which the AVL group belongs.
*/
} OBIDMS_avl_group_t, *OBIDMS_avl_group_p;
OBIDMS_avl_group_p obi_create_avl_group(OBIDMS_p dms, const char* avl_name);
index_t insert_in_avl_group(OBIDMS_avl_group_p avl_group, byte_t* value);
/**
* @brief Checks if an AVL tree already exists or not.
* @brief Checks if an AVL tree or AVL tree group already exists or not.
*
* @param dms The OBIDMS to which the AVL tree belongs.
* @param avl_name The name of the AVL tree.
* @param dms The OBIDMS to which the AVL tree or AVL tree group belongs.
* @param avl_name The name of the AVL treeor the base name of the AVL tree group.
*
* @returns A value indicating whether the AVL tree exists or not.
* @retval 1 if the AVL tree exists.
* @retval 0 if the AVL tree does not exist.
* @returns A value indicating whether the AVL tree or AVL tree group exists or not.
* @retval 1 if the AVL tree or AVL tree group exists.
* @retval 0 if the AVL tree or AVL tree group does not exist.
* @retval -1 if an error occurred.
*
* @since December 2015
@ -180,36 +187,19 @@ index_t insert_in_avl_group(OBIDMS_avl_group_p avl_group, byte_t* value);
int obi_avl_exists(OBIDMS_p dms, const char* avl_name);
/**
* @brief Opens an AVL tree and creates it if it does not already exist.
*
* Note: An AVL tree is made of two files (referred to by two structures).
* One file contains the indices referring to the data, and the other
* file contains the data itself. The AVL tree as a whole is referred
* to via the OBIDMS_avl structure.
*
* @param dms The OBIDMS to which the AVL tree belongs.
* @param avl_name The name of the AVL tree.
*
* @returns A pointer to the AVL tree structure.
* @retval NULL if an error occurred.
*
* @since December 2015
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
OBIDMS_avl_p obi_avl(OBIDMS_p dms, const char* avl_name);
/**
* @brief Creates an AVL tree. Fails if it already exists.
*
* Note: An AVL tree is made of two files (referred to by two structures).
* One file contains the indices referring to the data, and the other
* file contains the data itself. The AVL tree as a whole is referred
* to via the OBIDMS_avl structure.
* to via the OBIDMS_avl structure. An AVL tree is stored in a directory
* with the same name, or with the base name of the AVL group if it is
* part of an AVL group.
*
* @param dms The OBIDMS to which the AVL tree belongs.
* @param avl_name The name of the AVL tree.
* @param avl_idx The index of the AVL tree if it is part of an AVL group.
*
* @returns A pointer to the newly created AVL tree structure.
* @retval NULL if an error occurred.
@ -217,7 +207,7 @@ OBIDMS_avl_p obi_avl(OBIDMS_p dms, const char* avl_name);
* @since December 2015
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
OBIDMS_avl_p obi_create_avl(OBIDMS_p dms, const char* avl_name);
OBIDMS_avl_p obi_create_avl(OBIDMS_p dms, const char* avl_name, int avl_idx);
/**
@ -230,6 +220,7 @@ OBIDMS_avl_p obi_create_avl(OBIDMS_p dms, const char* avl_name);
*
* @param dms The OBIDMS to which the AVL tree belongs.
* @param avl_name The name of the AVL tree.
* @param avl_idx The index of the AVL tree if it is part of an AVL group.
*
* @returns A pointer to the AVL tree structure.
* @retval NULL if an error occurred.
@ -237,17 +228,66 @@ OBIDMS_avl_p obi_create_avl(OBIDMS_p dms, const char* avl_name);
* @since December 2015
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
OBIDMS_avl_p obi_open_avl(OBIDMS_p dms, const char* avl_name);
OBIDMS_avl_p obi_open_avl(OBIDMS_p dms, const char* avl_name, int avl_idx);
/**
* @brief Opens an AVL tree group and creates it if it does not already exist.
*
* Note: An AVL tree group is composed of multiple AVL trees that all have the
* same base name, and an index differentiating them.
*
* @param dms The OBIDMS to which the AVL tree belongs.
* @param avl_name The base name of the AVL tree group.
*
* @returns A pointer to the AVL tree group structure.
* @retval NULL if an error occurred.
*
* @since April 2016
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
OBIDMS_avl_group_p obi_avl_group(OBIDMS_p dms, const char* avl_name);
/**
* @brief Creates an AVL tree group.
*
* Note: An AVL tree group is composed of multiple AVL trees that all have the
* same base name, and an index differentiating them.
*
* @param dms The OBIDMS to which the AVL tree belongs.
* @param avl_name The base name of the AVL tree group.
*
* @returns A pointer to the AVL tree group structure.
* @retval NULL if an error occurred.
*
* @since April 2016
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
OBIDMS_avl_group_p obi_create_avl_group(OBIDMS_p dms, const char* avl_name);
/**
* @brief Opens an AVL tree group.
*
* Note: An AVL tree group is composed of multiple AVL trees that all have the
* same base name, and an index differentiating them.
*
* @param dms The OBIDMS to which the AVL tree belongs.
* @param avl_name The base name of the AVL tree group.
*
* @returns A pointer to the AVL tree group structure.
* @retval NULL if an error occurred.
*
* @since April 2016
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
OBIDMS_avl_group_p obi_open_avl_group(OBIDMS_p dms, const char* avl_name);
/**
* @brief Closes an AVL tree.
*
* Note: An AVL tree is made of two files (referred to by two structures).
* One file contains the indices referring to the data, and the other
* file contains the data itself. The AVL tree as a whole is referred
* to via the OBIDMS_avl structure.
*
* @param avl A pointer to the AVL tree structure to close and free.
*
* @retval 0 if the operation was successfully completed.
@ -260,26 +300,56 @@ int obi_close_avl(OBIDMS_avl_p avl);
/**
* @brief Adds a value (byte array) in an AVL tree, checking if it is already in it.
* @brief Closes an AVL tree group.
*
* @warning The byte array to add must already be encoded and contain its header.
* @param avl_group A pointer to the AVL tree group structure to close and free.
*
* @retval 0 if the operation was successfully completed.
* @retval -1 if an error occurred.
*
* @since April 2016
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
int obi_close_avl_group(OBIDMS_avl_group_p avl_group);
/**
* @brief Recovers a value (byte array) in an AVL tree.
*
* @warning The byte array recovered must be decoded to get the original value.
*
* @param avl A pointer to the AVL tree.
* @param index The index of the value in the data array.
*
* @returns A pointer to the byte array recovered.
*
* @since December 2015
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
Obi_byte_array_p obi_avl_get(OBIDMS_avl_p avl, index_t index);
/**
* @brief Adds a value (byte array) in an AVL tree NOT checking first if it is already in it. // TODO to discuss
*
* @warning The value given must be already be encoded into a byte array structure (Obi_byte_array_t).
*
* @param avl A pointer to the AVL tree.
* @param value The byte array to add in the AVL tree.
*
* @returns The index of the value, whether it was added or already in the AVL tree.
* @returns The index of the value newly added in the AVL tree.
* @retval -1 if an error occurred.
*
* @since December 2015
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
index_t obi_avl_add(OBIDMS_avl_p avl, byte_t* value);
index_t obi_avl_add(OBIDMS_avl_p avl, Obi_byte_array_p value);
/**
* @brief Finds a value (byte array) in an AVL tree, checking first if it is already in it.
* @brief Finds a value (byte array) in an AVL tree.
*
* @warning The byte array to add must already be encoded and contain its header.
* @warning The value given must be already be encoded into a byte array structure (Obi_byte_array_t).
*
* @param avl A pointer to the AVL tree.
* @param value The byte array to add in the AVL tree.
@ -290,86 +360,40 @@ index_t obi_avl_add(OBIDMS_avl_p avl, byte_t* value);
* @since December 2015
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
index_t obi_avl_find(OBIDMS_avl_p avl, byte_t* value);
index_t obi_avl_find(OBIDMS_avl_p avl, Obi_byte_array_p value);
/**
* @brief Recovers a value (byte array) in an AVL tree.
*
* @warning The byte array recovered is encoded and contains its header.
* @warning The byte array recovered must be decoded to get the original value.
*
* @param avl A pointer to the AVL tree.
* @param avl_group A pointer to the AVL tree.
* @param index The index of the value in the data array.
*
* @returns A pointer to the byte array recovered.
*
* @since December 2015
* @since April 2016
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
byte_t* obi_avl_get(OBIDMS_avl_p avl, index_t index);
Obi_byte_array_p obi_avl_group_get(OBIDMS_avl_group_p avl_group, index_t idx);
/**
* @brief Converts a character string to a byte array with a header.
* @brief Adds a value (byte array) in an AVL tree group, checking if it is already in it.
*
* @warning The byte array must be freed by the caller.
* @warning The value given must be already be encoded into a byte array structure (Obi_byte_array_t).
*
* @param value The character string to convert.
* @param avl_group A pointer to the AVL tree group.
* @param value The byte array to add in the AVL tree group.
*
* @returns A pointer to the byte array created.
* @retval NULL if an error occurred.
* @returns The index of the value newly added in the AVL tree group.
* @retval -1 if an error occurred.
*
* @since October 2015
* @since April 2016
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
byte_t* obi_str_to_obibytes(char* value);
/**
* @brief Converts a byte array to a character string.
*
* @param value_b The byte array to convert.
*
* @returns A pointer to the character string contained in the byte array.
*
* @since October 2015
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
const char* obi_obibytes_to_str(byte_t* value_b);
/**
* @brief Converts a DNA sequence to a byte array with a header.
*
* @warning The byte array must be freed by the caller.
*
* @param value The DNA sequence to convert.
*
* @returns A pointer to the byte array created.
* @retval NULL if an error occurred.
*
* @since November 2015
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
byte_t* obi_seq_to_obibytes(char* seq);
/**
* @brief Converts a byte array to a DNA sequence.
*
* @param value_b The byte array to convert.
*
* @returns A pointer to the DNA sequence contained in the byte array.
* @retval NULL if an error occurred.
*
* @since November 2015
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
const char* obi_obibytes_to_seq(byte_t* value_b); // TODO move to encode source files
// TODO
byte_t* obi_avl_group_get(OBIDMS_avl_group_p avl_group, index_t idx);
index_t obi_avl_group_add(OBIDMS_avl_group_p avl_group, Obi_byte_array_p value);
#endif /* OBIAVL_H_ */

2
src/obidms.c
View File

@ -23,7 +23,7 @@
#include "obierrno.h"
#include "obidebug.h"
#include "obidmscolumn.h"
#include "private_at_functions.h"
#include "utils.h"
#include "obilittlebigman.h"

2
src/obidms.h
View File

@ -33,7 +33,7 @@
*/
#define MAX_NB_OPENED_COLUMNS (100) /**< The maximum number of columns open at the same time.
*/
#define MAX_NB_OPENED_AVL_TREES (100) /**< The maximum number of AVL trees open at the same time.
#define MAX_NB_OPENED_AVL_TREES (1000) /**< The maximum number of AVL trees open at the same time.
*/

2
src/obidms_taxonomy.c
View File

@ -20,7 +20,7 @@
#include "obidms.h"
#include "obidebug.h"
#include "obierrno.h"
#include "private_at_functions.h"
#include "utils.h"
#define DEBUG_LEVEL 0 // TODO has to be defined somewhere else (cython compil flag?)

48
src/obidmscolumn.c
View File

@ -30,6 +30,7 @@
#include "obidebug.h"
#include "obilittlebigman.h"
#include "obiavl.h"
#include "utils.h"
#define DEBUG_LEVEL 0 // TODO has to be defined somewhere else (cython compil flag?)
@ -726,15 +727,15 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms,
// If the data type is OBI_STR or OBI_SEQ, the associated obi_avl is opened or created
if ((returned_data_type == OBI_STR) || (returned_data_type == OBI_SEQ))
{
new_column->avl = obi_create_avl_group(dms, avl_name);
// if (avl == NULL) TODO
// {
// obidebug(1, "\nError opening or creating the aVL tree associated with a column");
// munmap(new_column->header, header_size);
// close(column_file_descriptor);
// free(new_column);
// return NULL;
// }
new_column->avl = obi_avl_group(dms, avl_name);
if (new_column->avl == NULL)
{
obidebug(1, "\nError opening or creating the AVL group associated with a column");
munmap(new_column->header, header_size);
close(column_file_descriptor);
free(new_column);
return NULL;
}
strncpy(header->avl_name, avl_name, AVL_MAX_NAME);
}
@ -762,7 +763,6 @@ OBIDMS_column_p obi_open_column(OBIDMS_p dms,
int column_file_descriptor;
size_t header_size;
size_t i;
OBIDMS_avl_p avl;
column = NULL;
@ -770,7 +770,7 @@ OBIDMS_column_p obi_open_column(OBIDMS_p dms,
column_directory = obi_open_column_directory(dms, column_name);
if (column_directory == NULL)
{
//obidebug(1, "\nError opening a column directory structure");
//obidebug(1, "\nError opening a column directory structure"); // TODO
return NULL;
}
@ -879,8 +879,8 @@ OBIDMS_column_p obi_open_column(OBIDMS_p dms,
// If the data type is OBI_STR or OBI_SEQ, the associated AVL tree is opened
if (((column->header)->returned_data_type == OBI_STR) || ((column->header)->returned_data_type == OBI_SEQ))
{
avl = obi_avl(dms, (column->header)->avl_name);
if (avl == NULL)
column->avl = obi_open_avl_group(dms, (column->header)->avl_name);
if (column->avl == NULL)
{
obidebug(1, "\nError opening the AVL tree associated with a column");
munmap(column->header, header_size);
@ -888,7 +888,6 @@ OBIDMS_column_p obi_open_column(OBIDMS_p dms,
free(column);
return NULL;
}
//column->avl = avl; TODO
}
close(column_file_descriptor);
@ -964,18 +963,13 @@ OBIDMS_column_p obi_clone_column(OBIDMS_p dms,
}
else if (clone_data && (line_selection != NULL))
{
obidebug(1, "\nCloning data from line selection\n");
line_size = obi_sizeof((new_column->header)->stored_data_type) * (new_column->header)->nb_elements_per_line;
fprintf(stderr, "\nline size = %ld\n", line_size);
for (i=0; i<((line_selection->header)->lines_used); i++)
{
index = *(((index_t*) (line_selection->data)) + i);
fprintf(stderr, "\nindex = %lld, i = %lld\n", index, i);
memcpy((new_column->data)+(i*line_size), (column_to_clone->data)+(index*line_size), line_size);
fprintf(stderr, "\nmemcpied\n");
}
(new_column->header)->lines_used = (line_selection->header)->lines_used;
obidebug(1, "\nCloned data from line selection\n");
}
// Close column_to_clone
@ -1022,12 +1016,12 @@ int obi_close_column(OBIDMS_column_p column)
}
}
// If the data type is OBI_STR or OBI_SEQ, the associated AVL tree is closed TODO
// if (((column->header)->returned_data_type == OBI_STR) || ((column->header)->returned_data_type == OBI_SEQ))
// {
// if (obi_close_avl(column->avl) < 0)
// return -1;
// }
// If the data type is OBI_STR or OBI_SEQ, the associated AVL group is closed
if (((column->header)->returned_data_type == OBI_STR) || ((column->header)->returned_data_type == OBI_SEQ))
{
if (obi_close_avl_group(column->avl) < 0)
return -1;
}
// Munmap data
if (munmap(column->data, (column->header)->data_size) < 0)
@ -1045,10 +1039,10 @@ int obi_close_column(OBIDMS_column_p column)
return -1;
}
free(column);
if (close_dir)
obi_close_column_directory(column->column_directory);
free(column);
}
return 0;

2
src/obidmscolumn.h
View File

@ -28,8 +28,6 @@
#include "obiavl.h"
#define ONE_IF_ZERO(x) (((x)==0)?1:(x)) /**< If x is equal to 0, x takes the value 1.
*/
#define ELEMENTS_NAMES_MAX (2048) /**< The maximum length of the list of elements names.
*/
#define COLUMN_GROWTH_FACTOR (2) /**< The growth factor when a column is enlarged.

15
src/obidmscolumn_seq.c
View File

@ -33,7 +33,7 @@
int obi_column_set_obiseq_with_elt_idx(OBIDMS_column_p column, index_t line_nb, index_t element_idx, char* value)
{
byte_t* value_b;
Obi_byte_array_p value_b;
index_t idx;
// Check that the line number is not greater than the maximum allowed
@ -56,18 +56,13 @@ int obi_column_set_obiseq_with_elt_idx(OBIDMS_column_p column, index_t line_nb,
if ((line_nb+1) > (column->header)->lines_used)
(column->header)->lines_used = line_nb+1;
// Encode the value on a byte array with a header
// Encode the value on a byte array with a header // TODO make function
value_b = obi_seq_to_obibytes(value);
if (value_b == NULL)
return -1;
//if (strlen(value_b) == 0)
// fprintf(stderr, "\nPOUIC");
//fprintf(stderr, "\n>%s||%s", value, obi_obibytes_to_seq(value_b));
// Add in the AVL tree
idx = insert_in_avl_group(column->avl, value_b);
idx = obi_avl_group_add(column->avl, value_b);
if (idx == -1)
return -1;
@ -119,8 +114,8 @@ int obi_column_set_obiseq_with_elt_idx_in_view(Obiview_p view, OBIDMS_column_p c
const char* obi_column_get_obiseq_with_elt_idx(OBIDMS_column_p column, index_t line_nb, index_t element_idx)
{
index_t idx;
byte_t* value_b;
index_t idx;
Obi_byte_array_p value_b;
if ((line_nb+1) > ((column->header)->line_count))
{

10
src/obidmscolumn_str.c
View File

@ -32,8 +32,8 @@
int obi_column_set_obistr_with_elt_idx(OBIDMS_column_p column, index_t line_nb, index_t element_idx, char* value)
{
byte_t* value_b;
index_t idx;
Obi_byte_array_p value_b;
index_t idx;
// Check that the line number is not greater than the maximum allowed
if (line_nb >= MAXIMUM_LINE_COUNT)
@ -61,7 +61,7 @@ int obi_column_set_obistr_with_elt_idx(OBIDMS_column_p column, index_t line_nb,
return -1;
// Add in the AVL tree
idx = insert_in_avl_group(column->avl, value_b);
idx = obi_avl_group_add(column->avl, value_b);
if (idx == -1)
return -1;
@ -113,8 +113,8 @@ int obi_column_set_obistr_with_elt_idx_in_view(Obiview_p view, OBIDMS_column_p c
const char* obi_column_get_obistr_with_elt_idx(OBIDMS_column_p column, index_t line_nb, index_t element_idx)
{
index_t idx;
byte_t* value_b;
index_t idx;
Obi_byte_array_p value_b;
if ((line_nb+1) > ((column->header)->line_count))
{

2
src/obidmscolumndir.c
View File

@ -20,7 +20,7 @@
#include "obidmscolumndir.h"
#include "obidms.h"
#include "private_at_functions.h"
#include "utils.h"
#include "obierrno.h"
#include "obidebug.h"

8
src/obierrno.h
View File

@ -104,6 +104,14 @@ extern int obi_errno;
*/
#define OBI_TAXONOMY_ERROR (22) /** Error while handling binary taxonomy files
*/
#define OBI_MALLOC_ERROR (23) /** Error while allocating memory
*/
#define OBI_ENCODE_ERROR (24) /** Error while encoding a value
*/
#define OBI_DECODE_ERROR (25) /** Error while decoding a value
*/
#define OBI_UTILS_ERROR (26) /** Error in a utils function
*/
/**@}*/
#endif /* OBIERRNO_H_ */

2
src/obiview.c
View File

@ -22,7 +22,7 @@
#include "obierrno.h"
#include "obidebug.h"
#include "obidmscolumn.h"
#include "private_at_functions.h"
#include "utils.h"
#include "obilittlebigman.h"
#include "obidmscolumn_idx.h"

45
src/private_at_functions.c → src/utils.c
View File

@ -1,15 +1,14 @@
/****************************************************************************
* Private *at functions *
* Utility functions *
****************************************************************************/
/**
* @file private_at_functions.c
* @file utils.c
* @author Celine Mercier (celine.mercier@metabarcoding.org)
* @date 15 June 2015
* @brief Private replacement functions for *at functions.
* @date 29 March 2016
* @brief Code for utility functions.
*/
#include <fcntl.h>
#include <string.h>
#include <sys/stat.h>
@ -19,7 +18,7 @@
#include <dirent.h>
#include <unistd.h>
#include "private_at_functions.h"
#include "utils.h"
#include "obidebug.h"
#include "obierrno.h"
#include "obidms.h"
@ -28,6 +27,13 @@
#define DEBUG_LEVEL 0 // TODO has to be defined somewhere else (cython compil flag?)
/**********************************************************************
*
* D E F I N I T I O N O F T H E P U B L I C F U N C T I O N S
*
**********************************************************************/
char* get_full_path(OBIDMS_p dms, const char* path_name)
{
char* full_path;
@ -35,17 +41,18 @@ char* get_full_path(OBIDMS_p dms, const char* path_name)
full_path = (char*) malloc((MAX_PATH_LEN)*sizeof(char));
if (full_path == NULL)
{
obi_set_errno(OBI_MALLOC_ERROR);
obidebug(1, "\nError allocating memory for the char* path to a file or directory");
return NULL;
}
if (getcwd(full_path, MAX_PATH_LEN) == NULL)
{
obi_set_errno(OBI_UTILS_ERROR);
obidebug(1, "\nError getting the path to a file or directory");
return NULL;
}
// TODO check errors?
strcat(full_path, "/");
strcat(full_path, dms->directory_name);
strcat(full_path, "/");
@ -66,7 +73,10 @@ DIR* opendir_in_dms(OBIDMS_p dms, const char* path_name)
directory = opendir(full_path);
if (directory == NULL)
{
obi_set_errno(OBI_UTILS_ERROR);
obidebug(1, "\nError opening a directory");
}
free(full_path);
@ -74,3 +84,24 @@ DIR* opendir_in_dms(OBIDMS_p dms, const char* path_name)
}
int count_dir(char *dir)
{
struct dirent *dp;
DIR *fd;
int count;
count = 0;
if ((fd = opendir(dir)) == NULL)
{
obi_set_errno(OBI_UTILS_ERROR);
obidebug(1, "Error opening a directory: %s\n", dir);
return -1;
}
while ((dp = readdir(fd)) != NULL)
{
if ((dp->d_name)[0] == '.')
continue;
count++;
}
return count;
}

31
src/private_at_functions.h → src/utils.h
View File

@ -1,25 +1,30 @@
/****************************************************************************
* Header file for private *at functions *
* Header file for utility functions *
****************************************************************************/
/**
* @file private_at_functions.h
* @file utils.h
* @author Celine Mercier (celine.mercier@metabarcoding.org)
* @date 15 June 2015
* @brief Header file for the private replacement functions for *at functions.
* @date 29 March 2016
* @brief Header file for utility functions.
*/
#ifndef PRIVATE_OPENAT_H_
#define PRIVATE_OPENAT_H_
#ifndef UTILS_H_
#define UTILS_H_
#include <stdio.h>
#include <sys/stat.h>
#include "obidms.h"
#define MAX_PATH_LEN 4096 /**< Maximum length for the character string defining a
file or directory path */
#define ONE_IF_ZERO(x) (((x)==0)?1:(x)) /**< If x is equal to 0, x takes the value 1.
*/
#define MAX_PATH_LEN 4096 /**< Maximum length for the character string defining a
* file or directory path.
*/
/**
@ -56,4 +61,10 @@ char* get_full_path(OBIDMS_p dms, const char* path_name);
DIR* opendir_in_dms(OBIDMS_p dms, const char* path_name);
#endif /* PRIVATEOPENAT_H_ */
/*
* TODO
*/
int count_dir(char *dir);
#endif /* UTILS_H_ */