obitools/obitools3
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 66 Wiki Activity

obi arrays that don't work because of cython bug passing wrong pointers

Browse Source
This commit is contained in:
Celine Mercier
2015-11-03 14:22:00 +01:00
parent ecb9d97adb
commit 456551ffeb
25 changed files with 1669 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1173
src/obiarray.c Normal file
View File

File diff suppressed because it is too large Load Diff

177
src/obiarray.h Normal file
View File

@ -0,0 +1,177 @@
/****************************************************************************
* OBIDMS array header file *
****************************************************************************/
/**
* @file array.h
* @author Celine Mercier
* @date October 19th 2015
* @brief Header file for handling arrays for storing and retrieving data arrays (i.e. character strings).
*/
#ifndef ARRAY_H_
#define ARRAY_H_
#include <stdlib.h>
#include <stdio.h>
#include <stdint.h>
#include <time.h>
#include <sys/types.h>
#include <dirent.h>
#include "obidms.h"
#include "obitypes.h"
#define ARRAY_MAX_NAME (2048) /**< The maximum length of an array name.
*/
#define ARRAY_GROWTH_FACTOR (2) /**< The growth factor when an array is enlarged.
*/
typedef int32_t index_t; /**< Type of the array indices.
*/
typedef char byte_t; /**< Defining byte type since data is stored in bits
* and char (stored on one byte) is the smallest addressable unit.
*/
/**
* @brief OBIDMS array data header structure.
*/
typedef struct OBIDMS_array_data_header {
int header_size; /**< Size of the header in bytes.
*/
int64_t data_size_used; /**< Size of the data used in bytes. // TODO not sure about type
*/
int64_t data_size_max; /**< Max size of the data in bytes. // TODO not sure about type
*/
index_t nb_items; /**< Number of items.
*/
char array_name[ARRAY_MAX_NAME+1]; /**< The array name as a NULL terminated string.
*/
time_t creation_date; /**< Date of creation of the file.
*/
} OBIDMS_array_data_header_t, *OBIDMS_array_data_header_p;
/**
* @brief OBIDMS array data structure.
*/
typedef struct OBIDMS_array_data {
OBIDMS_array_data_header_p header; /**< A pointer to the header of the array data.
*/
byte_t* data; /**< A pointer to the beginning of the data.
*/
} OBIDMS_array_data_t, *OBIDMS_array_data_p;
/**
* @brief OBIDMS array header structure.
*/
typedef struct OBIDMS_array_header {
int header_size; /**< Size of the header in bytes.
*/
index_t nb_items; /**< Number of items in the array.
*/
index_t nb_items_max; /**< Maximum number of items in the array before it has to be enlarged.
*/
char array_name[ARRAY_MAX_NAME+1]; /**< The array name as a NULL terminated string.
*/
time_t creation_date; /**< Date of creation of the file.
*/
} OBIDMS_array_header_t, *OBIDMS_array_header_p;
/**
* @brief OBIDMS array structure.
*/
typedef struct OBIDMS_array {
OBIDMS_array_header_p header; /**< A pointer to the header of the array.
*/
index_t* first; /**< A pointer to the beginning of the array itself.
*/
OBIDMS_array_data_p data; /**< A pointer to the structure containing the data
* that the array references.
*/
DIR* directory; /**< A directory entry usable to
* refer and scan the array directory.
*/
} OBIDMS_array_t, *OBIDMS_array_p;
/**
*
*/
int obi_array_exists(OBIDMS_p dms, const char* array_name);
/**
*
*/
OBIDMS_array_p obi_array(OBIDMS_p dms, const char* array_name);
/**
*
*/
OBIDMS_array_p obi_create_array(OBIDMS_p dms, const char* array_name);
/**
*
*/
OBIDMS_array_p obi_open_array(OBIDMS_p dms, const char* array_name);
/**
*
*/
int obi_close_array(OBIDMS_array_p array);
/**
*
*/
//int obi_delete_array(OBIDMS_p dms, const char* array_name);
/**
* The header is already in the byte array
*/
index_t obi_array_add(OBIDMS_array_p array, byte_t* value);
/**
* Returns the entire array including the header
*/
byte_t* obi_array_get(OBIDMS_array_p array, index_t index);
/**
*
*/
index_t obi_array_search(OBIDMS_array_p array, byte_t* value);
/**
*
*/
byte_t* obi_str_to_obibytes(char* value);
/**
*
*/
const char* obi_obibytes_to_str(byte_t* value_b);
/**
* TODO LATER
*/
// OBIDMS_array_p obi_column_arrayify(FILE, encoding);
#endif /* ARRAY_H_ */

67
src/obidms.c
View File

@ -14,6 +14,7 @@
#include <stdio.h>
#include <string.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <sys/types.h>
#include <dirent.h>
@ -21,6 +22,7 @@
#include "obierrno.h"
#include "obidebug.h"
#include "obidmscolumn.h"
#include "private_at_functions.h"
#define DEBUG_LEVEL 0 // TODO has to be defined somewhere else (cython compil flag?)
@ -113,7 +115,9 @@ int obi_dms_exists(const char* dms_name)
OBIDMS_p obi_create_dms(const char* dms_name)
{
char *directory_name;
char* directory_name;
DIR* dms_dir;
int dms_file_descriptor;
// Build and check the directory name
directory_name = build_directory_name(dms_name);
@ -132,6 +136,35 @@ OBIDMS_p obi_create_dms(const char* dms_name)
return NULL;
}
// Get file descriptor of DMS directory to create the arrays directory
dms_dir = opendir(directory_name);
if (dms_dir == NULL)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nProblem opening a newly created OBIDMS directory");
free(directory_name);
return NULL;
}
dms_file_descriptor = dirfd(dms_dir);
if (dms_file_descriptor < 0)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nProblem getting the file descriptor of a newly created OBIDMS directory");
free(directory_name);
return NULL;
}
// Create the arrays directory
if (mkdirat(dms_file_descriptor, ARRAY_DIR_NAME, 00777) < 0)
{
obi_set_errno(OBI_ARRAY_ERROR);
obidebug(1, "\nProblem creating an arrays directory");
free(directory_name);
return NULL;
}
// TODO close file descriptor?
free(directory_name);
return obi_open_dms(dms_name);
@ -143,6 +176,7 @@ OBIDMS_p obi_open_dms(const char* dms_name)
OBIDMS_p dms;
char* directory_name;
DIR* directory;
int dms_file_descriptor;
dms = NULL;
@ -191,8 +225,30 @@ OBIDMS_p obi_open_dms(const char* dms_name)
strcpy(dms->directory_name, directory_name);
dms->directory = directory;
// Get file descriptor of DMS directory to open the arrays directory
dms_file_descriptor = dirfd(directory);
if (dms_file_descriptor < 0)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nError getting the file descriptor for a newly created OBIDMS directory");
free(directory_name);
return NULL;
}
// Open the arrays directory
dms->array_directory = private_opendirat(dms_file_descriptor, ARRAY_DIR_NAME);
if (dms->array_directory == NULL)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nError opening the arrays directory");
free(directory_name);
return NULL;
}
free(directory_name);
// TODO test if close file descriptor
return dms;
}
@ -223,7 +279,14 @@ int obi_close_dms(OBIDMS_p dms)
if (closedir(dms->directory) < 0)
{
obi_set_errno(OBIDMS_MEMORY_ERROR);
obidebug(1, "\nError closing an OBIDSM directory");
obidebug(1, "\nError closing an OBIDMS directory");
free(dms);
return -1;
}
if (closedir(dms->array_directory) < 0)
{
obi_set_errno(OBI_ARRAY_ERROR);
obidebug(1, "\nError closing an array directory");
free(dms);
return -1;
}

9
src/obidms.h
View File

@ -24,8 +24,10 @@
#include "obierrno.h"
#define OBIDMS_MAX_NAME (2048) /**< The maximum length of an OBIDMS name
*/
#define OBIDMS_MAX_NAME (2048) /**< The maximum length of an OBIDMS name.
*/
#define ARRAY_DIR_NAME "arrays" /**< The name of the arrays directory.
*/
/**
@ -41,6 +43,9 @@ typedef struct OBIDMS {
DIR* directory; /**< A directory entry usable to
* refer and scan the database directory.
*/
DIR* array_directory; /**< A directory entry usable to
* refer and scan the array directory.
*/
} OBIDMS_t, *OBIDMS_p;

77
src/obidmscolumn.c
View File

@ -16,6 +16,7 @@
#include <sys/types.h>
#include <dirent.h>
#include <unistd.h>
#include <time.h>
#include <fcntl.h>
#include <stdbool.h>
#include <math.h>
@ -28,6 +29,7 @@
#include "obierrno.h"
#include "obidebug.h"
#include "obilittlebigman.h"
#include "obiarray.h"
#define DEBUG_LEVEL 0 // TODO has to be defined somewhere else (cython compil flag?)
@ -151,35 +153,35 @@ static size_t get_line_count_per_page(OBIType_t data_type, size_t nb_elements_pe
************************************************************************/
static char *build_column_file_name(const char *column_name, obiversion_t version_number)
static char* build_column_file_name(const char* column_name, obiversion_t version_number)
{
char *filename;
char* file_name;
// Build the database directory name
if (asprintf(&filename,"%s@%d.odc", column_name, version_number) < 0)
// Build the file name
if (asprintf(&file_name,"%s@%d.odc", column_name, version_number) < 0)
{
obi_set_errno(OBICOL_MEMORY_ERROR);
obidebug(1, "\nError building a column file name");
return NULL;
}
return filename;
return file_name;
}
static char *build_version_file_name(const char *column_name)
static char* build_version_file_name(const char* column_name)
{
char *filename;
char* file_name;
// Build the database directory name
if (asprintf(&filename,"%s.odv", column_name) < 0)
// Build the file name
if (asprintf(&file_name,"%s.odv", column_name) < 0)
{
obi_set_errno(OBICOL_MEMORY_ERROR);
obidebug(1, "\nError building a version file name");
return NULL;
}
return filename;
return file_name;
}
@ -602,11 +604,13 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms,
OBIType_t data_type,
size_t nb_lines,
size_t nb_elements_per_line,
const char* elements_names)
const char* elements_names,
const char* array_name)
{
OBIDMS_column_p new_column;
OBIDMS_column_directory_p column_directory;
OBIDMS_column_header_p header;
OBIDMS_array_p array;
size_t file_size;
obiversion_t version_number;
char* column_file_name;
@ -629,17 +633,23 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms,
obidebug(1, "\nCan't create column because of empty column name");
return NULL;
}
if ((data_type < 1) || (data_type > 4))
if ((data_type < 1) || (data_type > 5))
{
obidebug(1, "\nCan't create column because of invalid data type");
return NULL;
}
if ((data_type == 5) && (array_name == NULL))
{
obidebug(1, "\nCan't create column because of empty array name");
return NULL;
}
// The initial line count should be between the minimum (corresponding to the page size) and the maximum allowed
minimum_line_count = get_line_count_per_page(data_type, nb_elements_per_line);
if (nb_lines > MAXIMUM_LINE_COUNT)
{
obidebug(1, "\nCan't create column because of line count greater than the maximum allowed (%ld)", MAXIMUM_LINE_COUNT);
obidebug(1, "\nCan't create column because of line count greater than the maximum allowed (%d)", MAXIMUM_LINE_COUNT);
return NULL;
}
else if (nb_lines < minimum_line_count)
@ -800,6 +810,23 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms,
strncpy(header->name, column_name, OBIDMS_COLUMN_MAX_NAME);
// If the data type is OBI_IDX, the associated obi_array is opened or created
if (data_type == 5)
{
array = obi_array(dms, array_name);
if (array == NULL)
{
obidebug(1, "\nError opening or creating the array associated with a column");
munmap(new_column->header, header_size);
close(column_file_descriptor);
free(column_file_name);
free(new_column);
return NULL;
}
new_column->array = array;
strncpy(header->array_name, array_name, ARRAY_MAX_NAME);
}
// Fill the data with NA values
obi_ini_to_NA_values(new_column, 0, nb_lines);
@ -816,6 +843,7 @@ OBIDMS_column_p obi_open_column(OBIDMS_p dms,
{
OBIDMS_column_p column;
OBIDMS_column_directory_p column_directory;
OBIDMS_array_p array;
char* column_file_name;
int column_file_descriptor;
int column_dir_file_descriptor;
@ -931,6 +959,22 @@ OBIDMS_column_p obi_open_column(OBIDMS_p dms,
column->writable = false;
// If the data type is OBI_IDX, the associated obi_array is opened or created
if ((column->header)->data_type == 5)
{
array = obi_array(dms, (column->header)->array_name);
if (array == NULL)
{
obidebug(1, "\nError opening the array associated with a column");
munmap(column->header, header_size);
close(column_file_descriptor);
free(column_file_name);
free(column);
return NULL;
}
column->array = array;
}
free(column_file_name);
close(column_file_descriptor);
@ -972,7 +1016,8 @@ OBIDMS_column_p obi_clone_column(OBIDMS_p dms,
data_type,
nb_lines,
nb_elements_per_line,
(column_to_clone->header)->elements_names);
(column_to_clone->header)->elements_names,
(column_to_clone->header)->array_name);
if (new_column == NULL)
{
@ -1170,7 +1215,7 @@ int obi_enlarge_column(OBIDMS_column_p column)
// Calculate the new file size
old_line_count = (column->header)->line_count;
new_line_count = old_line_count * GROWTH_FACTOR;
new_line_count = old_line_count * COLUMN_GROWTH_FACTOR;
if (new_line_count > MAXIMUM_LINE_COUNT)
{
@ -1181,7 +1226,7 @@ int obi_enlarge_column(OBIDMS_column_p column)
return -1;
}
old_data_size = obi_array_sizeof((column->header)->data_type, old_line_count, (column->header)->nb_elements_per_line);
new_data_size = old_data_size * GROWTH_FACTOR;
new_data_size = old_data_size * COLUMN_GROWTH_FACTOR;
header_size = (column->header)->header_size;
file_size = header_size + new_data_size;

13
src/obidmscolumn.h
View File

@ -25,19 +25,21 @@
#include "obierrno.h"
#include "obilittlebigman.h"
#include "obidmscolumndir.h"
#include "obiarray.h"
#define ONE_IF_ZERO(x) (((x)==0)?1:(x)) /**< If x is equal to 0, x takes the value 1.
*/
#define ELEMENTS_NAMES_MAX (2048) /**< The maximum length of the list of elements names.
*/
#define GROWTH_FACTOR (2) /**< The growth factor when a column is enlarged.
#define COLUMN_GROWTH_FACTOR (2) /**< The growth factor when a column is enlarged.
*/
#define MAXIMUM_LINE_COUNT (1000000) /**< The maximum line count for the data of a column. //TODO
*/
#define FORMATTED_TIME_LENGTH (1024) /**< The length allocated for the character string containing a formatted date
*/
typedef int32_t obiversion_t; /**< Used to store the column version number
*/
@ -74,6 +76,8 @@ typedef struct OBIDMS_column_header {
*/
char name[OBIDMS_COLUMN_MAX_NAME+1]; /**< The column name as a NULL terminated string.
*/
char array_name[ARRAY_MAX_NAME+1]; /**< If there is one, the obi_array name as a NULL terminated string.
*/
char comments[1]; /**< Comments stored as a classical zero end C string.
* The size of the comment is only limited by the header size.
*/
@ -93,6 +97,8 @@ typedef struct OBIDMS_column {
*/
OBIDMS_column_header_p header; /**< A pointer to the header of the column.
*/
OBIDMS_array_p array; /**< A pointer to the array associated with the column if there is one.
*/
void* data; /**< A `void` pointer to the beginning of the data.
*
* @warning Never use this member directly outside of the code of the
@ -172,11 +178,12 @@ size_t obi_get_platform_header_size();
* @author Eric Coissac (eric.coissac@metabarcoding.org)
*/
OBIDMS_column_p obi_create_column(OBIDMS_p dms,
const char* column_name,
const char* column_name,
OBIType_t data_type,
size_t nb_lines,
size_t nb_elements_per_line,
const char* elements_names);
const char* elements_names,
const char* array_name);
/**

85
src/obidmscolumn_idx.c → src/obidmscolumn_str.c
View File

@ -1,27 +1,15 @@
/****************************************************************************
* OBIDMS_column_idx functions *
* OBIDMS_column_str functions *
****************************************************************************/
/**
* @file obidsmcolumn_idx.c
* @file obidsmcolumn_str.c
* @author Celine Mercier
* @date August 10th 2015
* @date October 28th 2015
* @brief Functions handling OBIColumns containing data with the OBIType OBI_IDX.
*/
/****************************************************************************
* OBIDMS_column_idx functions *
****************************************************************************/
/**
* @file obidsmcolumn_idx.c
* @author Celine Mercier
* @date August 10th 2015
* @brief Functions handling OBIColumns containing data with the OBIType OBI_BOOL.
*/
#include <stdlib.h>
#include <stdio.h>
@ -29,6 +17,7 @@
#include "obitypes.h"
#include "obierrno.h"
#include "obidebug.h"
#include "obiarray.h"
#define DEBUG_LEVEL 0 // TODO has to be defined somewhere else (cython compil flag?)
@ -40,8 +29,11 @@
*
**********************************************************************/
int obi_column_set_obiidx_with_elt_idx(OBIDMS_column_p column, size_t line_nb, size_t element_idx, obiidx_t value)
int obi_column_set_obistr_with_elt_idx(OBIDMS_column_p column, size_t line_nb, size_t element_idx, char* value)
{
byte_t* value_b;
index_t idx;
// Check that the line number is not greater than the maximum allowed
if (line_nb >= MAXIMUM_LINE_COUNT)
{
@ -62,43 +54,84 @@ int obi_column_set_obiidx_with_elt_idx(OBIDMS_column_p column, size_t line_nb, s
if ((line_nb+1) > (column->header)->lines_used)
(column->header)->lines_used = line_nb+1;
// Set the value
*(((obiidx_t*) (column->data)) + (line_nb * ((column->header)->nb_elements_per_line)) + element_idx) = value;
// Encode the value on a byte array with a header
value_b = obi_str_to_obibytes(value);
if (value_b == NULL)
return -1;
obidebug(1, "\nvalue=%s", value);
//obidebug(1, "\nbytes=%s", value_b+5);
// Add in the obiarray
idx = obi_array_add(column->array, value_b);
if (idx == -1)
return -1;
obidebug(1, "\nidx=%d", idx);
//obidebug(1, "\nbytes 2=%s", obi_array_get(column->array, idx)+5);
// Add the value's index in the column
*(((obiidx_t*) (column->data)) + (line_nb * ((column->header)->nb_elements_per_line)) + element_idx) = idx;
// TODO free value_b probably
return 0;
}
obiidx_t obi_column_get_obiidx_with_elt_idx(OBIDMS_column_p column, size_t line_nb, size_t element_idx)
const char* obi_column_get_obistr_with_elt_idx(OBIDMS_column_p column, size_t line_nb, size_t element_idx)
{
index_t idx;
byte_t* value_b;
const char* value;
if ((line_nb+1) > (column->header)->lines_used)
{
obi_set_errno(OBICOL_UNKNOWN_ERROR);
obidebug(1, "\nError trying to get a value that is beyond the current number of lines used");
return OBIIdx_NA;
return "\0"; // TODO
}
return *(((obiidx_t*) (column->data)) + (line_nb * ((column->header)->nb_elements_per_line)) + element_idx);
idx = *(((obiidx_t*) (column->data)) + (line_nb * ((column->header)->nb_elements_per_line)) + element_idx);
// Check NA
if (idx == OBIIdx_NA)
return "\0"; // TODO
obidebug(1, "\nwhy, idx = %d", idx);
value_b = obi_array_get(column->array, idx);
obidebug(1, "\nwhyyyy");
value = obi_obibytes_to_str(value_b);
obidebug(1, "\nwhyyyyyyyyyyyy, value=%s %p", value, value);
obidebug(1, "\nwhyyyyyyyyyyyy, len value=%d", strlen(value));
return value;
}
int obi_column_set_obiidx_with_elt_name(OBIDMS_column_p column, size_t line_nb, const char* element_name, obiidx_t value)
int obi_column_set_obistr_with_elt_name(OBIDMS_column_p column, size_t line_nb, const char* element_name, char* value)
{
size_t element_idx;
element_idx = obi_column_get_element_index_from_name(column, element_name);
if (element_idx == SIZE_MAX) //TODO
return -1;
obi_column_set_obiidx_with_elt_idx(column, line_nb, element_idx, value);
if (obi_column_set_obistr_with_elt_idx(column, line_nb, element_idx, value) < 0)
return -1;
return 0;
}
obiidx_t obi_column_get_obiidx_with_elt_name(OBIDMS_column_p column, size_t line_nb, const char* element_name)
const char* obi_column_get_obistr_with_elt_name(OBIDMS_column_p column, size_t line_nb, const char* element_name)
{
size_t element_idx;
element_idx = obi_column_get_element_index_from_name(column, element_name);
if (element_idx == SIZE_MAX) //TODO
return OBIIdx_NA;
return obi_column_get_obiidx_with_elt_idx(column, line_nb, element_idx);
return "\0";
return obi_column_get_obistr_with_elt_idx(column, line_nb, element_idx);
}

8
src/obidmscolumn_idx.h → src/obidmscolumn_str.h
View File

@ -38,7 +38,7 @@
* @since July 2015
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
int obi_column_set_obiidx_with_elt_idx(OBIDMS_column_p column, size_t line_nb, size_t element_idx, obiidx_t value);
int obi_column_set_obiidx_with_elt_idx(OBIDMS_column_p column, size_t line_nb, size_t element_idx, void* value);
/**
@ -54,7 +54,7 @@ int obi_column_set_obiidx_with_elt_idx(OBIDMS_column_p column, size_t line_nb, s
* @since July 2015
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
obiidx_t obi_column_get_obiidx_with_elt_idx(OBIDMS_column_p column, size_t line_nb, size_t element_idx);
const char* obi_column_get_obiidx_with_elt_idx(OBIDMS_column_p column, size_t line_nb, size_t element_idx);
/**
@ -76,7 +76,7 @@ obiidx_t obi_column_get_obiidx_with_elt_idx(OBIDMS_column_p column, size_t line_
* @since August 2015
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
int obi_column_set_obiidx_with_elt_name(OBIDMS_column_p column, size_t line_nb, const char* element_name, obiidx_t value);
int obi_column_set_obiidx_with_elt_name(OBIDMS_column_p column, size_t line_nb, const char* element_name, void* value);
/**
@ -94,7 +94,7 @@ int obi_column_set_obiidx_with_elt_name(OBIDMS_column_p column, size_t line_nb,
* @since August 2015
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
obiidx_t obi_column_get_obiidx_with_elt_name(OBIDMS_column_p column, size_t line_nb, const char* element_name);
const char* obi_column_get_obiidx_with_elt_name(OBIDMS_column_p column, size_t line_nb, const char* element_name);
#endif /* OBIDMSCOLUMN_IDX_H_ */

2
src/obierrno.h
View File

@ -98,6 +98,8 @@ extern int obi_errno;
*/
#define OBICOL_ACCESS_ERROR (19) /**< Permission error trying to access an OBIDSM column directory
*/
#define OBI_ARRAY_ERROR (20) /** Error while handling a heap
*/
/**@}*/
#endif /* OBIERRNO_H_ */

2
src/private_at_functions.c
View File

@ -43,7 +43,7 @@ char* get_full_path(int directory_file_descriptor, const char* path_name)
return NULL;
}
// TODO check errors
// TODO check errors?
strlcat(full_path, "/", MAX_PATH_LEN);
strlcat(full_path, path_name, MAX_PATH_LEN);