obitools/obitools3
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 66 Wiki Activity

obi arrays that don't work because of cython bug passing wrong pointers

Browse Source
This commit is contained in:
Celine Mercier
2015-11-03 14:22:00 +01:00
parent ecb9d97adb
commit 456551ffeb
25 changed files with 1669 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
doc/.gitignore vendored
View File

@ -1,3 +1,5 @@
/build/
/doxygen/
/build_dir.txt
/.DS_Store
/.gitignore

2
doc/conf.py
View File

@ -51,7 +51,7 @@ source_suffix = '.rst'
#source_encoding = 'utf-8-sig'
# The master toctree document.
master_doc = 'index'
master_doc = 'source/index'
# General information about the project.
project = u'OBITools3'

4
doc/doxygen/xml/.gitignore vendored
View File

@ -1,4 +0,0 @@
# Ignore everything in this directory
*
# Except this file
!.gitignore

7
doc/source/types.rst
View File

@ -7,11 +7,14 @@ OBITypes
:download:`html version of the OBITypes UML file <UML/OBITypes_UML.class.violet.html>`
.. image:: ./UML/Obicolumn_classes_UML.png
:download:`html version of the OBIDMS classes UML file <UML/Obicolumn_classes_UML.class.violet.html>`
.. toctree::
:maxdepth: 2
The elementary types <elementary>
The containers <containers>
Special values <specialvalues>

2
python/obitools3/obidms/_obidms.cfiles
View File

@ -12,3 +12,5 @@
../../../src/obitypes.c
../../../src/private_at_functions.h
../../../src/private_at_functions.c
../../../src/obiarray.h
../../../src/obiarray.c

3
python/obitools3/obidms/_obidms.pxd
View File

@ -23,7 +23,8 @@ cdef class OBIDMS:
OBIType_t data_type=*,
size_t nb_lines=*,
size_t nb_elements_per_line=*,
list elements_names=*)
list elements_names=*,
str array_name=*)
cdef class OBIDMS_column:

44
python/obitools3/obidms/_obidms.pyx
View File

@ -42,10 +42,10 @@ from ._obidmscolumn_char cimport OBIDMS_column_char, \
OBIDMS_column_char_multi_elts, \
OBIDMS_column_char_multi_elts_writable
# from ._obidmscolumn_idx cimport OBIDMS_column_idx, \
# OBIDMS_column_idx_writable, \
# OBIDMS_column_idx_multi_elts, \
# OBIDMS_column_idx_multi_elts_writable
from ._obidmscolumn_str cimport OBIDMS_column_str, \
OBIDMS_column_str_writable, \
OBIDMS_column_str_multi_elts, \
OBIDMS_column_str_multi_elts_writable
cdef class OBIDMS :
@ -119,7 +119,8 @@ cdef class OBIDMS :
OBIType_t data_type= <OBIType_t> 0,
size_t nb_lines=0,
size_t nb_elements_per_line=0,
list elements_names=None):
list elements_names=None,
str array_name="default_obiarray"):
# Declarations
cdef OBIDMS_column column
@ -202,17 +203,17 @@ cdef class OBIDMS :
subclass = OBIDMS_column_char
else :
subclass = OBIDMS_column_char_multi_elts
# elif data_type == 5 :
# if (create or clone) :
# if nb_elements_per_line == 1 :
# subclass = OBIDMS_column_idx_writable
# else :
# subclass = OBIDMS_column_idx_multi_elts_writable
# else :
# if nb_elements_per_line == 1 :
# subclass = OBIDMS_column_idx
# else :
# subclass = OBIDMS_column_idx_multi_elts
elif data_type == 5 :
if (create or clone) :
if nb_elements_per_line == 1 :
subclass = OBIDMS_column_str_writable
else :
subclass = OBIDMS_column_str_multi_elts_writable
else :
if nb_elements_per_line == 1 :
subclass = OBIDMS_column_str
else :
subclass = OBIDMS_column_str_multi_elts
else :
raise Exception("Problem with the data type")
@ -220,7 +221,7 @@ cdef class OBIDMS :
create, clone, clone_data,
version_number, data_type,
nb_lines, nb_elements_per_line,
elements_names)
elements_names, array_name)
return column
@ -238,11 +239,13 @@ cdef class OBIDMS_column :
OBIType_t type,
size_t nb_lines,
size_t nb_elements_per_line,
list elements_names):
list elements_names,
str array_name):
# Declarations
cdef bytes column_name_b
cdef bytes dms_name_b
cdef bytes array_name_b
cdef bytes elements_names_b
# Fill structure
@ -255,6 +258,7 @@ cdef class OBIDMS_column :
# Format the character strings to send them to C functions
column_name_b = str2bytes(column_name)
dms_name_b = str2bytes(self.dms.dms_name)
array_name_b = str2bytes(array_name)
# Create, clone or open column
if create :
@ -262,7 +266,9 @@ cdef class OBIDMS_column :
elements_names_b = column_name_b
else :
elements_names_b = str2bytes(";".join(elements_names))
self.pointer = obi_create_column(self.dms.pointer, column_name_b, type, nb_lines, nb_elements_per_line, elements_names_b)
self.pointer = obi_create_column(self.dms.pointer, column_name_b, type,
nb_lines, nb_elements_per_line,
elements_names_b, array_name_b)
else :
if clone :
self.pointer = obi_clone_column(self.dms.pointer, column_name_b, version_number, clone_data)

2
python/obitools3/obidms/_obidmscolumn_bool.cfiles
View File

@ -14,3 +14,5 @@
../../../src/obitypes.c
../../../src/private_at_functions.h
../../../src/private_at_functions.c
../../../src/obiarray.h
../../../src/obiarray.c

2
python/obitools3/obidms/_obidmscolumn_char.cfiles
View File

@ -14,3 +14,5 @@
../../../src/obitypes.c
../../../src/private_at_functions.h
../../../src/private_at_functions.c
../../../src/obiarray.h
../../../src/obiarray.c

2
python/obitools3/obidms/_obidmscolumn_float.cfiles
View File

@ -14,3 +14,5 @@
../../../src/obitypes.c
../../../src/private_at_functions.h
../../../src/private_at_functions.c
../../../src/obiarray.h
../../../src/obiarray.c

2
python/obitools3/obidms/_obidmscolumn_int.cfiles
View File

@ -14,3 +14,5 @@
../../../src/obitypes.c
../../../src/private_at_functions.h
../../../src/private_at_functions.c
../../../src/obiarray.h
../../../src/obiarray.c

6
python/obitools3/obidms/_obidmscolumn_idx.cfiles → python/obitools3/obidms/_obidmscolumn_str.cfiles
View File

@ -1,5 +1,5 @@
../../../src/obidmscolumn_idx.c
../../../src/obidmscolumn_idx.h
../../../src/obidmscolumn_str.c
../../../src/obidmscolumn_str.h
../../../src/obidmscolumn.h
../../../src/obidmscolumn.c
../../../src/obidmscolumndir.h
@ -14,3 +14,5 @@
../../../src/obitypes.c
../../../src/private_at_functions.h
../../../src/private_at_functions.c
../../../src/obiarray.h
../../../src/obiarray.c

13
python/obitools3/obidms/_obidmscolumn_idx.pxd → python/obitools3/obidms/_obidmscolumn_str.pxd
View File

@ -1,25 +1,24 @@
#cython: language_level=3
from .capi.obitypes cimport obiidx_t
from ._obidms cimport OBIDMS_column
cdef class OBIDMS_column_idx(OBIDMS_column):
cdef class OBIDMS_column_str(OBIDMS_column):
cpdef object get_line(self, size_t line_nb)
cpdef set_line(self, size_t line_nb, object value)
cpdef close(self)
cdef class OBIDMS_column_idx_writable(OBIDMS_column_idx):
cdef class OBIDMS_column_str_writable(OBIDMS_column_str):
cpdef set_line(self, size_t line_nb, object value)
cpdef close(self)
cdef class OBIDMS_column_idx_multi_elts(OBIDMS_column_idx):
cdef class OBIDMS_column_str_multi_elts(OBIDMS_column_str):
cpdef object get_item(self, size_t line_nb, str element_name)
cpdef object get_line(self, size_t line_nb)
cpdef set_item(self, size_t line_nb, str element_name, obiidx_t value)
cpdef set_item(self, size_t line_nb, str element_name, str value)
cpdef set_line(self, size_t line_nb, object values)
cdef class OBIDMS_column_idx_multi_elts_writable(OBIDMS_column_idx_multi_elts):
cpdef set_item(self, size_t line_nb, str element_name, obiidx_t value)
cdef class OBIDMS_column_str_multi_elts_writable(OBIDMS_column_str_multi_elts):
cpdef set_item(self, size_t line_nb, str element_name, str value)
cpdef set_line(self, size_t line_nb, object values)
cpdef close(self)

66
python/obitools3/obidms/_obidmscolumn_idx.pyx → python/obitools3/obidms/_obidmscolumn_str.pyx
View File

@ -2,30 +2,46 @@
from .capi.obidmscolumn cimport obi_close_column,\
obi_truncate_and_close_column, \
obi_column_get_obiidx_with_elt_name, \
obi_column_get_obiidx_with_elt_idx, \
obi_column_set_obiidx_with_elt_name, \
obi_column_set_obiidx_with_elt_idx
obi_column_get_obistr_with_elt_name, \
obi_column_get_obistr_with_elt_idx, \
obi_column_set_obistr_with_elt_name, \
obi_column_set_obistr_with_elt_idx
from .capi.obierrno cimport obi_errno
from .capi.obitypes cimport OBIIdx_NA
from .capi.obitypes cimport OBIIdx_NA, const_char_p
from obitools3.utils cimport str2bytes
from libc.string cimport strlen
from obitools3.utils cimport str2bytes, bytes2str
from cpython.int cimport PyInt_FromSsize_t
cdef class OBIDMS_column_idx(OBIDMS_column):
cdef class OBIDMS_column_str(OBIDMS_column):
cpdef object get_line(self, size_t line_nb):
cdef obiidx_t value
cdef const_char_p cvalue
cdef bytes value
cdef object result
value = obi_column_get_obiidx_with_elt_idx(self.pointer, line_nb, 0)
cvalue = obi_column_get_obistr_with_elt_idx(self.pointer, line_nb, 0)
print('test 1')
print(hex(<int> cvalue))
print(strlen(cvalue))
print("test")
value = <bytes> cvalue
print(value)
if obi_errno > 0 :
raise IndexError(line_nb)
if value == OBIIdx_NA :
result = None
else :
result = PyInt_FromSsize_t(value)
print(value)
result = bytes2str(value)
return result
cpdef set_line(self, size_t line_nb, object value):
@ -36,10 +52,10 @@ cdef class OBIDMS_column_idx(OBIDMS_column):
raise Exception("Problem closing a column")
cdef class OBIDMS_column_idx_writable(OBIDMS_column_idx):
cdef class OBIDMS_column_str_writable(OBIDMS_column_str):
cpdef set_line(self, size_t line_nb, object value):
if obi_column_set_obiidx_with_elt_idx(self.pointer, line_nb, 0, <obiidx_t> value) < 0:
if obi_column_set_obistr_with_elt_idx(self.pointer, line_nb, 0, str2bytes(value)) < 0:
raise Exception("Problem setting a value in a column")
cpdef close(self):
@ -47,55 +63,55 @@ cdef class OBIDMS_column_idx_writable(OBIDMS_column_idx):
raise Exception("Problem closing a column")
cdef class OBIDMS_column_idx_multi_elts(OBIDMS_column_idx):
cdef class OBIDMS_column_str_multi_elts(OBIDMS_column_str):
cpdef object get_item(self, size_t line_nb, str element_name):
cdef obiidx_t value
cdef bytes value
cdef object result
value = obi_column_get_obiidx_with_elt_name(self.pointer, line_nb, str2bytes(element_name))
value = obi_column_get_obistr_with_elt_name(self.pointer, line_nb, str2bytes(element_name))
if obi_errno > 0 :
raise IndexError(line_nb, element_name)
if value == OBIIdx_NA :
result = None
else :
result = PyInt_FromSsize_t(value)
result = bytes2str(value)
return result
cpdef object get_line(self, size_t line_nb) :
cdef obiidx_t value
cdef bytes value
cdef object result
cdef size_t i
cdef bint all_NA
result = {}
all_NA = True
for i in range(self.nb_elements_per_line) :
value = obi_column_get_obiidx_with_elt_idx(self.pointer, line_nb, i)
value = obi_column_get_obistr_with_elt_idx(self.pointer, line_nb, i)
if obi_errno > 0 :
raise IndexError(line_nb)
result[self.elements_names[i]] = PyInt_FromSsize_t(value)
result[self.elements_names[i]] = bytes2str(value)
if all_NA and (value != OBIIdx_NA) :
all_NA = False
if all_NA :
result = None
return result
cpdef set_item(self, size_t line_nb, str element_name, obiidx_t value):
cpdef set_item(self, size_t line_nb, str element_name, str value):
raise Exception("Column is read-only")
cpdef set_line(self, size_t line_nb, object values):
raise Exception("Column is read-only")
cdef class OBIDMS_column_idx_multi_elts_writable(OBIDMS_column_idx_multi_elts):
cdef class OBIDMS_column_str_multi_elts_writable(OBIDMS_column_str_multi_elts):
cpdef set_item(self, size_t line_nb, str element_name, obiidx_t value):
if obi_column_set_obiidx_with_elt_name(self.pointer, line_nb, str2bytes(element_name), value) < 0:
cpdef set_item(self, size_t line_nb, str element_name, str value):
if obi_column_set_obistr_with_elt_name(self.pointer, line_nb, str2bytes(element_name), str2bytes(value)) < 0:
raise Exception("Problem setting a value in a column")
cpdef set_line(self, size_t line_nb, object values):
cdef obiidx_t value
cdef str value
for element_name in values :
value = <obiidx_t> values[element_name]
value = values[element_name]
self.set_item(line_nb, element_name, value)
cpdef close(self):

27
python/obitools3/obidms/capi/obidmscolumn.pxd
View File

@ -8,7 +8,6 @@ from ..capi.obitypes cimport const_char_p, \
obibool_t, \
obichar_t, \
obifloat_t, \
obiidx_t, \
time_t
@ -26,6 +25,7 @@ cdef extern from "obidmscolumn.h" nogil:
obiversion_t version
obiversion_t cloned_from
const_char_p name
const_char_p array_name
const_char_p comments
ctypedef OBIDMS_column_header_t* OBIDMS_column_header_p
@ -40,7 +40,8 @@ cdef extern from "obidmscolumn.h" nogil:
OBIType_t type,
size_t nb_lines,
size_t nb_elements_per_line,
const_char_p elements_names)
const_char_p elements_names,
const_char_p array_name)
OBIDMS_column_p obi_open_column(OBIDMS_p dms,
const_char_p column_name,
@ -146,23 +147,23 @@ cdef extern from "obidmscolumn_float.h" nogil:
size_t line_nb,
size_t element_idx)
cdef extern from "obidmscolumn_idx.h" nogil:
cdef extern from "obidmscolumn_str.h" nogil:
int obi_column_set_obiidx_with_elt_name(OBIDMS_column_p column,
int obi_column_set_obistr_with_elt_name(OBIDMS_column_p column,
size_t line_nb,
const_char_p element_name,
obiidx_t value)
char* value)
int obi_column_set_obiidx_with_elt_idx(OBIDMS_column_p column,
int obi_column_set_obistr_with_elt_idx(OBIDMS_column_p column,
size_t line_nb,
size_t element_idx,
obiidx_t value)
char* value)
obiidx_t obi_column_get_obiidx_with_elt_name(OBIDMS_column_p column,
size_t line_nb,
const_char_p element_name)
const_char_p obi_column_get_obistr_with_elt_name(OBIDMS_column_p column,
size_t line_nb,
const_char_p element_name)
obiidx_t obi_column_get_obiidx_with_elt_idx(OBIDMS_column_p column,
size_t line_nb,
size_t element_idx)
const_char_p obi_column_get_obistr_with_elt_idx(OBIDMS_column_p column,
size_t line_nb,
size_t element_idx)

1173
src/obiarray.c Normal file
View File

File diff suppressed because it is too large Load Diff

177
src/obiarray.h Normal file
View File

@ -0,0 +1,177 @@
/****************************************************************************
* OBIDMS array header file *
****************************************************************************/
/**
* @file array.h
* @author Celine Mercier
* @date October 19th 2015
* @brief Header file for handling arrays for storing and retrieving data arrays (i.e. character strings).
*/
#ifndef ARRAY_H_
#define ARRAY_H_
#include <stdlib.h>
#include <stdio.h>
#include <stdint.h>
#include <time.h>
#include <sys/types.h>
#include <dirent.h>
#include "obidms.h"
#include "obitypes.h"
#define ARRAY_MAX_NAME (2048) /**< The maximum length of an array name.
*/
#define ARRAY_GROWTH_FACTOR (2) /**< The growth factor when an array is enlarged.
*/
typedef int32_t index_t; /**< Type of the array indices.
*/
typedef char byte_t; /**< Defining byte type since data is stored in bits
* and char (stored on one byte) is the smallest addressable unit.
*/
/**
* @brief OBIDMS array data header structure.
*/
typedef struct OBIDMS_array_data_header {
int header_size; /**< Size of the header in bytes.
*/
int64_t data_size_used; /**< Size of the data used in bytes. // TODO not sure about type
*/
int64_t data_size_max; /**< Max size of the data in bytes. // TODO not sure about type
*/
index_t nb_items; /**< Number of items.
*/
char array_name[ARRAY_MAX_NAME+1]; /**< The array name as a NULL terminated string.
*/
time_t creation_date; /**< Date of creation of the file.
*/
} OBIDMS_array_data_header_t, *OBIDMS_array_data_header_p;
/**
* @brief OBIDMS array data structure.
*/
typedef struct OBIDMS_array_data {
OBIDMS_array_data_header_p header; /**< A pointer to the header of the array data.
*/
byte_t* data; /**< A pointer to the beginning of the data.
*/
} OBIDMS_array_data_t, *OBIDMS_array_data_p;
/**
* @brief OBIDMS array header structure.
*/
typedef struct OBIDMS_array_header {
int header_size; /**< Size of the header in bytes.
*/
index_t nb_items; /**< Number of items in the array.
*/
index_t nb_items_max; /**< Maximum number of items in the array before it has to be enlarged.
*/
char array_name[ARRAY_MAX_NAME+1]; /**< The array name as a NULL terminated string.
*/
time_t creation_date; /**< Date of creation of the file.
*/
} OBIDMS_array_header_t, *OBIDMS_array_header_p;
/**
* @brief OBIDMS array structure.
*/
typedef struct OBIDMS_array {
OBIDMS_array_header_p header; /**< A pointer to the header of the array.
*/
index_t* first; /**< A pointer to the beginning of the array itself.
*/
OBIDMS_array_data_p data; /**< A pointer to the structure containing the data
* that the array references.
*/
DIR* directory; /**< A directory entry usable to
* refer and scan the array directory.
*/
} OBIDMS_array_t, *OBIDMS_array_p;
/**
*
*/
int obi_array_exists(OBIDMS_p dms, const char* array_name);
/**
*
*/
OBIDMS_array_p obi_array(OBIDMS_p dms, const char* array_name);
/**
*
*/
OBIDMS_array_p obi_create_array(OBIDMS_p dms, const char* array_name);
/**
*
*/
OBIDMS_array_p obi_open_array(OBIDMS_p dms, const char* array_name);
/**
*
*/
int obi_close_array(OBIDMS_array_p array);
/**
*
*/
//int obi_delete_array(OBIDMS_p dms, const char* array_name);
/**
* The header is already in the byte array
*/
index_t obi_array_add(OBIDMS_array_p array, byte_t* value);
/**
* Returns the entire array including the header
*/
byte_t* obi_array_get(OBIDMS_array_p array, index_t index);
/**
*
*/
index_t obi_array_search(OBIDMS_array_p array, byte_t* value);
/**
*
*/
byte_t* obi_str_to_obibytes(char* value);
/**
*
*/
const char* obi_obibytes_to_str(byte_t* value_b);
/**
* TODO LATER
*/
// OBIDMS_array_p obi_column_arrayify(FILE, encoding);
#endif /* ARRAY_H_ */

67
src/obidms.c
View File

@ -14,6 +14,7 @@
#include <stdio.h>
#include <string.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <sys/types.h>
#include <dirent.h>
@ -21,6 +22,7 @@
#include "obierrno.h"
#include "obidebug.h"
#include "obidmscolumn.h"
#include "private_at_functions.h"
#define DEBUG_LEVEL 0 // TODO has to be defined somewhere else (cython compil flag?)
@ -113,7 +115,9 @@ int obi_dms_exists(const char* dms_name)
OBIDMS_p obi_create_dms(const char* dms_name)
{
char *directory_name;
char* directory_name;
DIR* dms_dir;
int dms_file_descriptor;
// Build and check the directory name
directory_name = build_directory_name(dms_name);
@ -132,6 +136,35 @@ OBIDMS_p obi_create_dms(const char* dms_name)
return NULL;
}
// Get file descriptor of DMS directory to create the arrays directory
dms_dir = opendir(directory_name);
if (dms_dir == NULL)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nProblem opening a newly created OBIDMS directory");
free(directory_name);
return NULL;
}
dms_file_descriptor = dirfd(dms_dir);
if (dms_file_descriptor < 0)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nProblem getting the file descriptor of a newly created OBIDMS directory");
free(directory_name);
return NULL;
}
// Create the arrays directory
if (mkdirat(dms_file_descriptor, ARRAY_DIR_NAME, 00777) < 0)
{
obi_set_errno(OBI_ARRAY_ERROR);
obidebug(1, "\nProblem creating an arrays directory");
free(directory_name);
return NULL;
}
// TODO close file descriptor?
free(directory_name);
return obi_open_dms(dms_name);
@ -143,6 +176,7 @@ OBIDMS_p obi_open_dms(const char* dms_name)
OBIDMS_p dms;
char* directory_name;
DIR* directory;
int dms_file_descriptor;
dms = NULL;
@ -191,8 +225,30 @@ OBIDMS_p obi_open_dms(const char* dms_name)
strcpy(dms->directory_name, directory_name);
dms->directory = directory;
// Get file descriptor of DMS directory to open the arrays directory
dms_file_descriptor = dirfd(directory);
if (dms_file_descriptor < 0)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nError getting the file descriptor for a newly created OBIDMS directory");
free(directory_name);
return NULL;
}
// Open the arrays directory
dms->array_directory = private_opendirat(dms_file_descriptor, ARRAY_DIR_NAME);
if (dms->array_directory == NULL)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nError opening the arrays directory");
free(directory_name);
return NULL;
}
free(directory_name);
// TODO test if close file descriptor
return dms;
}
@ -223,7 +279,14 @@ int obi_close_dms(OBIDMS_p dms)
if (closedir(dms->directory) < 0)
{
obi_set_errno(OBIDMS_MEMORY_ERROR);
obidebug(1, "\nError closing an OBIDSM directory");
obidebug(1, "\nError closing an OBIDMS directory");
free(dms);
return -1;
}
if (closedir(dms->array_directory) < 0)
{
obi_set_errno(OBI_ARRAY_ERROR);
obidebug(1, "\nError closing an array directory");
free(dms);
return -1;
}

9
src/obidms.h
View File

@ -24,8 +24,10 @@
#include "obierrno.h"
#define OBIDMS_MAX_NAME (2048) /**< The maximum length of an OBIDMS name
*/
#define OBIDMS_MAX_NAME (2048) /**< The maximum length of an OBIDMS name.
*/
#define ARRAY_DIR_NAME "arrays" /**< The name of the arrays directory.
*/
/**
@ -41,6 +43,9 @@ typedef struct OBIDMS {
DIR* directory; /**< A directory entry usable to
* refer and scan the database directory.
*/
DIR* array_directory; /**< A directory entry usable to
* refer and scan the array directory.
*/
} OBIDMS_t, *OBIDMS_p;

77
src/obidmscolumn.c
View File

@ -16,6 +16,7 @@
#include <sys/types.h>
#include <dirent.h>
#include <unistd.h>
#include <time.h>
#include <fcntl.h>
#include <stdbool.h>
#include <math.h>
@ -28,6 +29,7 @@
#include "obierrno.h"
#include "obidebug.h"
#include "obilittlebigman.h"
#include "obiarray.h"
#define DEBUG_LEVEL 0 // TODO has to be defined somewhere else (cython compil flag?)
@ -151,35 +153,35 @@ static size_t get_line_count_per_page(OBIType_t data_type, size_t nb_elements_pe
************************************************************************/
static char *build_column_file_name(const char *column_name, obiversion_t version_number)
static char* build_column_file_name(const char* column_name, obiversion_t version_number)
{
char *filename;
char* file_name;
// Build the database directory name
if (asprintf(&filename,"%s@%d.odc", column_name, version_number) < 0)
// Build the file name
if (asprintf(&file_name,"%s@%d.odc", column_name, version_number) < 0)
{
obi_set_errno(OBICOL_MEMORY_ERROR);
obidebug(1, "\nError building a column file name");
return NULL;
}
return filename;
return file_name;
}
static char *build_version_file_name(const char *column_name)
static char* build_version_file_name(const char* column_name)
{
char *filename;
char* file_name;
// Build the database directory name
if (asprintf(&filename,"%s.odv", column_name) < 0)
// Build the file name
if (asprintf(&file_name,"%s.odv", column_name) < 0)
{
obi_set_errno(OBICOL_MEMORY_ERROR);
obidebug(1, "\nError building a version file name");
return NULL;
}
return filename;
return file_name;
}
@ -602,11 +604,13 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms,
OBIType_t data_type,
size_t nb_lines,
size_t nb_elements_per_line,
const char* elements_names)
const char* elements_names,
const char* array_name)
{
OBIDMS_column_p new_column;
OBIDMS_column_directory_p column_directory;
OBIDMS_column_header_p header;
OBIDMS_array_p array;
size_t file_size;
obiversion_t version_number;
char* column_file_name;
@ -629,17 +633,23 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms,
obidebug(1, "\nCan't create column because of empty column name");
return NULL;
}
if ((data_type < 1) || (data_type > 4))
if ((data_type < 1) || (data_type > 5))
{
obidebug(1, "\nCan't create column because of invalid data type");
return NULL;
}
if ((data_type == 5) && (array_name == NULL))
{
obidebug(1, "\nCan't create column because of empty array name");
return NULL;
}
// The initial line count should be between the minimum (corresponding to the page size) and the maximum allowed
minimum_line_count = get_line_count_per_page(data_type, nb_elements_per_line);
if (nb_lines > MAXIMUM_LINE_COUNT)
{
obidebug(1, "\nCan't create column because of line count greater than the maximum allowed (%ld)", MAXIMUM_LINE_COUNT);
obidebug(1, "\nCan't create column because of line count greater than the maximum allowed (%d)", MAXIMUM_LINE_COUNT);
return NULL;
}
else if (nb_lines < minimum_line_count)
@ -800,6 +810,23 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms,
strncpy(header->name, column_name, OBIDMS_COLUMN_MAX_NAME);
// If the data type is OBI_IDX, the associated obi_array is opened or created
if (data_type == 5)
{
array = obi_array(dms, array_name);
if (array == NULL)
{
obidebug(1, "\nError opening or creating the array associated with a column");
munmap(new_column->header, header_size);
close(column_file_descriptor);
free(column_file_name);
free(new_column);
return NULL;
}
new_column->array = array;
strncpy(header->array_name, array_name, ARRAY_MAX_NAME);
}
// Fill the data with NA values
obi_ini_to_NA_values(new_column, 0, nb_lines);
@ -816,6 +843,7 @@ OBIDMS_column_p obi_open_column(OBIDMS_p dms,
{
OBIDMS_column_p column;
OBIDMS_column_directory_p column_directory;
OBIDMS_array_p array;
char* column_file_name;
int column_file_descriptor;
int column_dir_file_descriptor;
@ -931,6 +959,22 @@ OBIDMS_column_p obi_open_column(OBIDMS_p dms,
column->writable = false;
// If the data type is OBI_IDX, the associated obi_array is opened or created
if ((column->header)->data_type == 5)
{
array = obi_array(dms, (column->header)->array_name);
if (array == NULL)
{
obidebug(1, "\nError opening the array associated with a column");
munmap(column->header, header_size);
close(column_file_descriptor);
free(column_file_name);
free(column);
return NULL;
}
column->array = array;
}
free(column_file_name);
close(column_file_descriptor);
@ -972,7 +1016,8 @@ OBIDMS_column_p obi_clone_column(OBIDMS_p dms,
data_type,
nb_lines,
nb_elements_per_line,
(column_to_clone->header)->elements_names);
(column_to_clone->header)->elements_names,
(column_to_clone->header)->array_name);
if (new_column == NULL)
{
@ -1170,7 +1215,7 @@ int obi_enlarge_column(OBIDMS_column_p column)
// Calculate the new file size
old_line_count = (column->header)->line_count;
new_line_count = old_line_count * GROWTH_FACTOR;
new_line_count = old_line_count * COLUMN_GROWTH_FACTOR;
if (new_line_count > MAXIMUM_LINE_COUNT)
{
@ -1181,7 +1226,7 @@ int obi_enlarge_column(OBIDMS_column_p column)
return -1;
}
old_data_size = obi_array_sizeof((column->header)->data_type, old_line_count, (column->header)->nb_elements_per_line);
new_data_size = old_data_size * GROWTH_FACTOR;
new_data_size = old_data_size * COLUMN_GROWTH_FACTOR;
header_size = (column->header)->header_size;
file_size = header_size + new_data_size;

13
src/obidmscolumn.h
View File

@ -25,19 +25,21 @@
#include "obierrno.h"
#include "obilittlebigman.h"
#include "obidmscolumndir.h"
#include "obiarray.h"
#define ONE_IF_ZERO(x) (((x)==0)?1:(x)) /**< If x is equal to 0, x takes the value 1.
*/
#define ELEMENTS_NAMES_MAX (2048) /**< The maximum length of the list of elements names.
*/
#define GROWTH_FACTOR (2) /**< The growth factor when a column is enlarged.
#define COLUMN_GROWTH_FACTOR (2) /**< The growth factor when a column is enlarged.
*/
#define MAXIMUM_LINE_COUNT (1000000) /**< The maximum line count for the data of a column. //TODO
*/
#define FORMATTED_TIME_LENGTH (1024) /**< The length allocated for the character string containing a formatted date
*/
typedef int32_t obiversion_t; /**< Used to store the column version number
*/
@ -74,6 +76,8 @@ typedef struct OBIDMS_column_header {
*/
char name[OBIDMS_COLUMN_MAX_NAME+1]; /**< The column name as a NULL terminated string.
*/
char array_name[ARRAY_MAX_NAME+1]; /**< If there is one, the obi_array name as a NULL terminated string.
*/
char comments[1]; /**< Comments stored as a classical zero end C string.
* The size of the comment is only limited by the header size.
*/
@ -93,6 +97,8 @@ typedef struct OBIDMS_column {
*/
OBIDMS_column_header_p header; /**< A pointer to the header of the column.
*/
OBIDMS_array_p array; /**< A pointer to the array associated with the column if there is one.
*/
void* data; /**< A `void` pointer to the beginning of the data.
*
* @warning Never use this member directly outside of the code of the
@ -172,11 +178,12 @@ size_t obi_get_platform_header_size();
* @author Eric Coissac (eric.coissac@metabarcoding.org)
*/
OBIDMS_column_p obi_create_column(OBIDMS_p dms,
const char* column_name,
const char* column_name,
OBIType_t data_type,
size_t nb_lines,
size_t nb_elements_per_line,
const char* elements_names);
const char* elements_names,
const char* array_name);
/**

85
src/obidmscolumn_idx.c → src/obidmscolumn_str.c
View File

@ -1,27 +1,15 @@
/****************************************************************************
* OBIDMS_column_idx functions *
* OBIDMS_column_str functions *
****************************************************************************/
/**
* @file obidsmcolumn_idx.c
* @file obidsmcolumn_str.c
* @author Celine Mercier
* @date August 10th 2015
* @date October 28th 2015
* @brief Functions handling OBIColumns containing data with the OBIType OBI_IDX.
*/
/****************************************************************************
* OBIDMS_column_idx functions *
****************************************************************************/
/**
* @file obidsmcolumn_idx.c
* @author Celine Mercier
* @date August 10th 2015
* @brief Functions handling OBIColumns containing data with the OBIType OBI_BOOL.
*/
#include <stdlib.h>
#include <stdio.h>
@ -29,6 +17,7 @@
#include "obitypes.h"
#include "obierrno.h"
#include "obidebug.h"
#include "obiarray.h"
#define DEBUG_LEVEL 0 // TODO has to be defined somewhere else (cython compil flag?)
@ -40,8 +29,11 @@
*
**********************************************************************/
int obi_column_set_obiidx_with_elt_idx(OBIDMS_column_p column, size_t line_nb, size_t element_idx, obiidx_t value)
int obi_column_set_obistr_with_elt_idx(OBIDMS_column_p column, size_t line_nb, size_t element_idx, char* value)
{
byte_t* value_b;
index_t idx;
// Check that the line number is not greater than the maximum allowed
if (line_nb >= MAXIMUM_LINE_COUNT)
{
@ -62,43 +54,84 @@ int obi_column_set_obiidx_with_elt_idx(OBIDMS_column_p column, size_t line_nb, s
if ((line_nb+1) > (column->header)->lines_used)
(column->header)->lines_used = line_nb+1;
// Set the value
*(((obiidx_t*) (column->data)) + (line_nb * ((column->header)->nb_elements_per_line)) + element_idx) = value;
// Encode the value on a byte array with a header
value_b = obi_str_to_obibytes(value);
if (value_b == NULL)
return -1;
obidebug(1, "\nvalue=%s", value);
//obidebug(1, "\nbytes=%s", value_b+5);
// Add in the obiarray
idx = obi_array_add(column->array, value_b);
if (idx == -1)
return -1;
obidebug(1, "\nidx=%d", idx);
//obidebug(1, "\nbytes 2=%s", obi_array_get(column->array, idx)+5);
// Add the value's index in the column
*(((obiidx_t*) (column->data)) + (line_nb * ((column->header)->nb_elements_per_line)) + element_idx) = idx;
// TODO free value_b probably
return 0;
}
obiidx_t obi_column_get_obiidx_with_elt_idx(OBIDMS_column_p column, size_t line_nb, size_t element_idx)
const char* obi_column_get_obistr_with_elt_idx(OBIDMS_column_p column, size_t line_nb, size_t element_idx)
{
index_t idx;
byte_t* value_b;
const char* value;
if ((line_nb+1) > (column->header)->lines_used)
{
obi_set_errno(OBICOL_UNKNOWN_ERROR);
obidebug(1, "\nError trying to get a value that is beyond the current number of lines used");
return OBIIdx_NA;
return "\0"; // TODO
}
return *(((obiidx_t*) (column->data)) + (line_nb * ((column->header)->nb_elements_per_line)) + element_idx);
idx = *(((obiidx_t*) (column->data)) + (line_nb * ((column->header)->nb_elements_per_line)) + element_idx);
// Check NA
if (idx == OBIIdx_NA)
return "\0"; // TODO
obidebug(1, "\nwhy, idx = %d", idx);
value_b = obi_array_get(column->array, idx);
obidebug(1, "\nwhyyyy");
value = obi_obibytes_to_str(value_b);
obidebug(1, "\nwhyyyyyyyyyyyy, value=%s %p", value, value);
obidebug(1, "\nwhyyyyyyyyyyyy, len value=%d", strlen(value));
return value;
}
int obi_column_set_obiidx_with_elt_name(OBIDMS_column_p column, size_t line_nb, const char* element_name, obiidx_t value)
int obi_column_set_obistr_with_elt_name(OBIDMS_column_p column, size_t line_nb, const char* element_name, char* value)
{
size_t element_idx;
element_idx = obi_column_get_element_index_from_name(column, element_name);
if (element_idx == SIZE_MAX) //TODO
return -1;
obi_column_set_obiidx_with_elt_idx(column, line_nb, element_idx, value);
if (obi_column_set_obistr_with_elt_idx(column, line_nb, element_idx, value) < 0)
return -1;
return 0;
}
obiidx_t obi_column_get_obiidx_with_elt_name(OBIDMS_column_p column, size_t line_nb, const char* element_name)
const char* obi_column_get_obistr_with_elt_name(OBIDMS_column_p column, size_t line_nb, const char* element_name)
{
size_t element_idx;
element_idx = obi_column_get_element_index_from_name(column, element_name);
if (element_idx == SIZE_MAX) //TODO
return OBIIdx_NA;
return obi_column_get_obiidx_with_elt_idx(column, line_nb, element_idx);
return "\0";
return obi_column_get_obistr_with_elt_idx(column, line_nb, element_idx);
}

8
src/obidmscolumn_idx.h → src/obidmscolumn_str.h
View File

@ -38,7 +38,7 @@
* @since July 2015
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
int obi_column_set_obiidx_with_elt_idx(OBIDMS_column_p column, size_t line_nb, size_t element_idx, obiidx_t value);
int obi_column_set_obiidx_with_elt_idx(OBIDMS_column_p column, size_t line_nb, size_t element_idx, void* value);
/**
@ -54,7 +54,7 @@ int obi_column_set_obiidx_with_elt_idx(OBIDMS_column_p column, size_t line_nb, s
* @since July 2015
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
obiidx_t obi_column_get_obiidx_with_elt_idx(OBIDMS_column_p column, size_t line_nb, size_t element_idx);
const char* obi_column_get_obiidx_with_elt_idx(OBIDMS_column_p column, size_t line_nb, size_t element_idx);
/**
@ -76,7 +76,7 @@ obiidx_t obi_column_get_obiidx_with_elt_idx(OBIDMS_column_p column, size_t line_
* @since August 2015
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
int obi_column_set_obiidx_with_elt_name(OBIDMS_column_p column, size_t line_nb, const char* element_name, obiidx_t value);
int obi_column_set_obiidx_with_elt_name(OBIDMS_column_p column, size_t line_nb, const char* element_name, void* value);
/**
@ -94,7 +94,7 @@ int obi_column_set_obiidx_with_elt_name(OBIDMS_column_p column, size_t line_nb,
* @since August 2015
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
obiidx_t obi_column_get_obiidx_with_elt_name(OBIDMS_column_p column, size_t line_nb, const char* element_name);
const char* obi_column_get_obiidx_with_elt_name(OBIDMS_column_p column, size_t line_nb, const char* element_name);
#endif /* OBIDMSCOLUMN_IDX_H_ */

2
src/obierrno.h
View File

@ -98,6 +98,8 @@ extern int obi_errno;
*/
#define OBICOL_ACCESS_ERROR (19) /**< Permission error trying to access an OBIDSM column directory
*/
#define OBI_ARRAY_ERROR (20) /** Error while handling a heap
*/
/**@}*/
#endif /* OBIERRNO_H_ */

2
src/private_at_functions.c
View File

@ -43,7 +43,7 @@ char* get_full_path(int directory_file_descriptor, const char* path_name)
return NULL;
}
// TODO check errors
// TODO check errors?
strlcat(full_path, "/", MAX_PATH_LEN);
strlcat(full_path, path_name, MAX_PATH_LEN);