obi arrays that don't work because of cython bug passing wrong pointers

doc/.gitignore

@@ -1,3 +1,5 @@
 /build/
 /doxygen/
 /build_dir.txt
+/.DS_Store
+/.gitignore

doc/conf.py

@@ -51,7 +51,7 @@ source_suffix = '.rst'
 #source_encoding = 'utf-8-sig'
 
 # The master toctree document.
-master_doc = 'index'
+master_doc = 'source/index'
 
 # General information about the project.
 project = u'OBITools3'

doc/doxygen/xml/.gitignore

@@ -1,4 +0,0 @@
-# Ignore everything in this directory
-*
-# Except this file
-!.gitignore

doc/source/types.rst

@@ -7,11 +7,14 @@ OBITypes
 :download:`html version of the OBITypes UML file <UML/OBITypes_UML.class.violet.html>`
 
 
+.. image:: ./UML/Obicolumn_classes_UML.png
+
+:download:`html version of the OBIDMS classes UML file <UML/Obicolumn_classes_UML.class.violet.html>`
+
+
 .. toctree::
    :maxdepth: 2
 
    The elementary types <elementary>
    The containers <containers>
    Special values <specialvalues>
-
-

python/obitools3/obidms/_obidms.cfiles

@@ -12,3 +12,5 @@
 ../../../src/obitypes.c
 ../../../src/private_at_functions.h
 ../../../src/private_at_functions.c
+../../../src/obiarray.h
+../../../src/obiarray.c

python/obitools3/obidms/_obidms.pxd

@@ -23,7 +23,8 @@ cdef class OBIDMS:
                                     OBIType_t data_type=*,
                                     size_t nb_lines=*, 
                                     size_t nb_elements_per_line=*, 
-                                    list elements_names=*)
+                                    list elements_names=*,
+                                    str array_name=*)
 
 
 cdef class OBIDMS_column:

python/obitools3/obidms/_obidms.pyx

@@ -42,10 +42,10 @@ from ._obidmscolumn_char cimport OBIDMS_column_char, \
                                  OBIDMS_column_char_multi_elts, \
                                  OBIDMS_column_char_multi_elts_writable
 
-# from ._obidmscolumn_idx cimport OBIDMS_column_idx, \
+from ._obidmscolumn_str cimport OBIDMS_column_str, \
-#                                 OBIDMS_column_idx_writable, \
+                                OBIDMS_column_str_writable, \
-#                                 OBIDMS_column_idx_multi_elts, \
+                                OBIDMS_column_str_multi_elts, \
-#                                 OBIDMS_column_idx_multi_elts_writable
+                                OBIDMS_column_str_multi_elts_writable
 
 
 cdef class OBIDMS :
@@ -119,7 +119,8 @@ cdef class OBIDMS :
                                     OBIType_t data_type= <OBIType_t> 0,
                                     size_t nb_lines=0, 
                                     size_t nb_elements_per_line=0, 
-                                    list elements_names=None):
+                                    list elements_names=None,
+                                    str array_name="default_obiarray"):
         
         # Declarations
         cdef OBIDMS_column column
@@ -202,17 +203,17 @@ cdef class OBIDMS :
                     subclass = OBIDMS_column_char
                 else :
                     subclass = OBIDMS_column_char_multi_elts
-#         elif data_type == 5 :
+        elif data_type == 5 :
-#             if (create or clone) :
+            if (create or clone) :
-#                 if nb_elements_per_line == 1 :
+                if nb_elements_per_line == 1 :
-#                     subclass = OBIDMS_column_idx_writable
+                    subclass = OBIDMS_column_str_writable
-#                 else :
+                else :
-#                     subclass = OBIDMS_column_idx_multi_elts_writable
+                    subclass = OBIDMS_column_str_multi_elts_writable
-#             else :
+            else :
-#                 if nb_elements_per_line == 1 :
+                if nb_elements_per_line == 1 :
-#                     subclass = OBIDMS_column_idx
+                    subclass = OBIDMS_column_str
-#                 else :
+                else :
-#                     subclass = OBIDMS_column_idx_multi_elts
+                    subclass = OBIDMS_column_str_multi_elts
         else :
             raise Exception("Problem with the data type")
             
@@ -220,7 +221,7 @@ cdef class OBIDMS :
                           create, clone, clone_data, 
                           version_number, data_type, 
                           nb_lines, nb_elements_per_line, 
-                          elements_names)
+                          elements_names, array_name)
         
         return column
 
@@ -238,11 +239,13 @@ cdef class OBIDMS_column :
                  OBIType_t type,
                  size_t nb_lines, 
                  size_t nb_elements_per_line, 
-                 list elements_names):
+                 list elements_names,
+                 str array_name):
         
         # Declarations
         cdef bytes column_name_b
         cdef bytes dms_name_b
+        cdef bytes array_name_b
         cdef bytes elements_names_b
         
         # Fill structure
@@ -255,6 +258,7 @@ cdef class OBIDMS_column :
         # Format the character strings to send them to C functions
         column_name_b = str2bytes(column_name)
         dms_name_b = str2bytes(self.dms.dms_name)
+        array_name_b = str2bytes(array_name)
         
         # Create, clone or open column
         if create :
@@ -262,7 +266,9 @@ cdef class OBIDMS_column :
                 elements_names_b = column_name_b
             else :
                 elements_names_b = str2bytes(";".join(elements_names))
-            self.pointer = obi_create_column(self.dms.pointer, column_name_b, type, nb_lines, nb_elements_per_line, elements_names_b)
+            self.pointer = obi_create_column(self.dms.pointer, column_name_b, type, 
+                                             nb_lines, nb_elements_per_line, 
+                                             elements_names_b, array_name_b)
         else :
             if clone :
                 self.pointer = obi_clone_column(self.dms.pointer, column_name_b, version_number, clone_data)

python/obitools3/obidms/_obidmscolumn_bool.cfiles

@@ -14,3 +14,5 @@
 ../../../src/obitypes.c
 ../../../src/private_at_functions.h
 ../../../src/private_at_functions.c
+../../../src/obiarray.h
+../../../src/obiarray.c

python/obitools3/obidms/_obidmscolumn_char.cfiles

@@ -14,3 +14,5 @@
 ../../../src/obitypes.c
 ../../../src/private_at_functions.h
 ../../../src/private_at_functions.c
+../../../src/obiarray.h
+../../../src/obiarray.c

python/obitools3/obidms/_obidmscolumn_float.cfiles

@@ -14,3 +14,5 @@
 ../../../src/obitypes.c
 ../../../src/private_at_functions.h
 ../../../src/private_at_functions.c
+../../../src/obiarray.h
+../../../src/obiarray.c

python/obitools3/obidms/_obidmscolumn_int.cfiles

@@ -14,3 +14,5 @@
 ../../../src/obitypes.c
 ../../../src/private_at_functions.h
 ../../../src/private_at_functions.c
+../../../src/obiarray.h
+../../../src/obiarray.c

python/obitools3/obidms/_obidmscolumn_str.cfiles

@@ -1,5 +1,5 @@
-../../../src/obidmscolumn_idx.c
+../../../src/obidmscolumn_str.c
-../../../src/obidmscolumn_idx.h
+../../../src/obidmscolumn_str.h
 ../../../src/obidmscolumn.h
 ../../../src/obidmscolumn.c
 ../../../src/obidmscolumndir.h
@@ -14,3 +14,5 @@
 ../../../src/obitypes.c
 ../../../src/private_at_functions.h
 ../../../src/private_at_functions.c
+../../../src/obiarray.h
+../../../src/obiarray.c

python/obitools3/obidms/_obidmscolumn_str.pxd

@@ -1,25 +1,24 @@
 #cython: language_level=3
 
-from .capi.obitypes cimport obiidx_t
 from ._obidms       cimport OBIDMS_column
 
 
-cdef class OBIDMS_column_idx(OBIDMS_column):
+cdef class OBIDMS_column_str(OBIDMS_column):
     cpdef object get_line(self, size_t line_nb)
     cpdef set_line(self, size_t line_nb, object value)
     cpdef close(self)
 
-cdef class OBIDMS_column_idx_writable(OBIDMS_column_idx):     
+cdef class OBIDMS_column_str_writable(OBIDMS_column_str):     
     cpdef set_line(self, size_t line_nb, object value)
     cpdef close(self)
 
-cdef class OBIDMS_column_idx_multi_elts(OBIDMS_column_idx):
+cdef class OBIDMS_column_str_multi_elts(OBIDMS_column_str):
     cpdef object get_item(self, size_t line_nb, str element_name)
     cpdef object get_line(self, size_t line_nb)
-    cpdef set_item(self, size_t line_nb, str element_name, obiidx_t value)
+    cpdef set_item(self, size_t line_nb, str element_name, str value)
     cpdef set_line(self, size_t line_nb, object values)
 
-cdef class OBIDMS_column_idx_multi_elts_writable(OBIDMS_column_idx_multi_elts):
+cdef class OBIDMS_column_str_multi_elts_writable(OBIDMS_column_str_multi_elts):
-    cpdef set_item(self, size_t line_nb, str element_name, obiidx_t value)
+    cpdef set_item(self, size_t line_nb, str element_name, str value)
     cpdef set_line(self, size_t line_nb, object values)
     cpdef close(self)

python/obitools3/obidms/_obidmscolumn_str.pyx

@@ -2,30 +2,46 @@
  
 from .capi.obidmscolumn cimport obi_close_column,\
                                 obi_truncate_and_close_column, \
-                                obi_column_get_obiidx_with_elt_name, \
+                                obi_column_get_obistr_with_elt_name, \
-                                obi_column_get_obiidx_with_elt_idx, \
+                                obi_column_get_obistr_with_elt_idx, \
-                                obi_column_set_obiidx_with_elt_name, \
+                                obi_column_set_obistr_with_elt_name, \
-                                obi_column_set_obiidx_with_elt_idx
+                                obi_column_set_obistr_with_elt_idx
 from .capi.obierrno     cimport obi_errno
-from .capi.obitypes     cimport OBIIdx_NA
+from .capi.obitypes     cimport OBIIdx_NA, const_char_p
 
-from obitools3.utils cimport str2bytes
+
+from libc.string cimport strlen
+
+from obitools3.utils cimport str2bytes, bytes2str
 
 from cpython.int cimport PyInt_FromSsize_t
 
 
-cdef class OBIDMS_column_idx(OBIDMS_column):
+cdef class OBIDMS_column_str(OBIDMS_column):
 
     cpdef object get_line(self, size_t line_nb):
-        cdef obiidx_t value
+        cdef const_char_p cvalue 
+        cdef bytes value
         cdef object result
-        value = obi_column_get_obiidx_with_elt_idx(self.pointer, line_nb, 0)
+        
+        cvalue = obi_column_get_obistr_with_elt_idx(self.pointer, line_nb, 0)
+        
+        print('test 1')
+        
+        print(hex(<int> cvalue))
+        
+        
+        print(strlen(cvalue))
+        print("test")
+        value = <bytes> cvalue
+        print(value)
         if obi_errno > 0 :
             raise IndexError(line_nb)
         if value == OBIIdx_NA :
             result = None
         else :
-            result = PyInt_FromSsize_t(value)
+            print(value)
+            result = bytes2str(value)
         return result
 
     cpdef set_line(self, size_t line_nb, object value):
@@ -36,10 +52,10 @@ cdef class OBIDMS_column_idx(OBIDMS_column):
             raise Exception("Problem closing a column")
 
 
-cdef class OBIDMS_column_idx_writable(OBIDMS_column_idx):     
+cdef class OBIDMS_column_str_writable(OBIDMS_column_str):     
 
     cpdef set_line(self, size_t line_nb, object value):
-        if obi_column_set_obiidx_with_elt_idx(self.pointer, line_nb, 0, <obiidx_t> value) < 0:
+        if obi_column_set_obistr_with_elt_idx(self.pointer, line_nb, 0, str2bytes(value)) < 0:
             raise Exception("Problem setting a value in a column")
 
     cpdef close(self):
@@ -47,55 +63,55 @@ cdef class OBIDMS_column_idx_writable(OBIDMS_column_idx):
             raise Exception("Problem closing a column")
 
 
-cdef class OBIDMS_column_idx_multi_elts(OBIDMS_column_idx):
+cdef class OBIDMS_column_str_multi_elts(OBIDMS_column_str):
 
     cpdef object get_item(self, size_t line_nb, str element_name):
-        cdef obiidx_t value
+        cdef bytes value
         cdef object result
-        value = obi_column_get_obiidx_with_elt_name(self.pointer, line_nb, str2bytes(element_name))
+        value = obi_column_get_obistr_with_elt_name(self.pointer, line_nb, str2bytes(element_name))
         if obi_errno > 0 :
             raise IndexError(line_nb, element_name)
         if value == OBIIdx_NA :
             result = None
         else :
-            result = PyInt_FromSsize_t(value)
+            result = bytes2str(value)
         return result
     
     cpdef object get_line(self, size_t line_nb) :
-        cdef obiidx_t value
+        cdef bytes value
         cdef object result
         cdef size_t i
         cdef bint all_NA
         result = {}
         all_NA = True
         for i in range(self.nb_elements_per_line) :
-            value = obi_column_get_obiidx_with_elt_idx(self.pointer, line_nb, i)
+            value = obi_column_get_obistr_with_elt_idx(self.pointer, line_nb, i)
             if obi_errno > 0 :
                 raise IndexError(line_nb)           
-            result[self.elements_names[i]] = PyInt_FromSsize_t(value)
+            result[self.elements_names[i]] = bytes2str(value)
             if all_NA and (value != OBIIdx_NA) :
                 all_NA = False
         if all_NA :
             result = None
         return result
 
-    cpdef set_item(self, size_t line_nb, str element_name, obiidx_t value):
+    cpdef set_item(self, size_t line_nb, str element_name, str value):
         raise Exception("Column is read-only")
 
     cpdef set_line(self, size_t line_nb, object values):
         raise Exception("Column is read-only")
 
 
-cdef class OBIDMS_column_idx_multi_elts_writable(OBIDMS_column_idx_multi_elts):     
+cdef class OBIDMS_column_str_multi_elts_writable(OBIDMS_column_str_multi_elts):     
 
-    cpdef set_item(self, size_t line_nb, str element_name, obiidx_t value):
+    cpdef set_item(self, size_t line_nb, str element_name, str value):
-        if obi_column_set_obiidx_with_elt_name(self.pointer, line_nb, str2bytes(element_name), value) < 0:
+        if obi_column_set_obistr_with_elt_name(self.pointer, line_nb, str2bytes(element_name), str2bytes(value)) < 0:
             raise Exception("Problem setting a value in a column")
     
     cpdef set_line(self, size_t line_nb, object values):
-        cdef obiidx_t value
+        cdef str value
         for element_name in values :
-            value = <obiidx_t> values[element_name]
+            value = values[element_name]
             self.set_item(line_nb, element_name, value)
 
     cpdef close(self):

python/obitools3/obidms/capi/obidmscolumn.pxd

@@ -8,7 +8,6 @@ from ..capi.obitypes cimport const_char_p, \
                              obibool_t, \
                              obichar_t, \
                              obifloat_t, \
-                             obiidx_t, \
                              time_t
 
 
@@ -26,6 +25,7 @@ cdef extern from "obidmscolumn.h" nogil:
         obiversion_t    version
         obiversion_t    cloned_from
         const_char_p    name
+        const_char_p    array_name
         const_char_p    comments
     
     ctypedef OBIDMS_column_header_t* OBIDMS_column_header_p
@@ -40,7 +40,8 @@ cdef extern from "obidmscolumn.h" nogil:
                                       OBIType_t type, 
                                       size_t nb_lines, 
                                       size_t nb_elements_per_line, 
-                                      const_char_p elements_names)
+                                      const_char_p elements_names,
+                                      const_char_p array_name)
                 
     OBIDMS_column_p obi_open_column(OBIDMS_p dms, 
                                     const_char_p column_name, 
@@ -146,23 +147,23 @@ cdef extern from "obidmscolumn_float.h" nogil:
                                                     size_t line_nb, 
                                                     size_t element_idx)
 
-cdef extern from "obidmscolumn_idx.h" nogil:
+cdef extern from "obidmscolumn_str.h" nogil:
 
-    int obi_column_set_obiidx_with_elt_name(OBIDMS_column_p column, 
+    int obi_column_set_obistr_with_elt_name(OBIDMS_column_p column, 
                                             size_t line_nb, 
                                             const_char_p element_name, 
-                                            obiidx_t value)
+                                            char* value)
     
-    int obi_column_set_obiidx_with_elt_idx(OBIDMS_column_p column, 
+    int obi_column_set_obistr_with_elt_idx(OBIDMS_column_p column, 
                                            size_t line_nb, 
                                            size_t element_idx, 
-                                           obiidx_t value)
+                                           char* value)
     
-    obiidx_t obi_column_get_obiidx_with_elt_name(OBIDMS_column_p column, 
+    const_char_p obi_column_get_obistr_with_elt_name(OBIDMS_column_p column, 
                                               size_t line_nb, 
                                               const_char_p element_name)
     
-    obiidx_t obi_column_get_obiidx_with_elt_idx(OBIDMS_column_p column, 
+    const_char_p obi_column_get_obistr_with_elt_idx(OBIDMS_column_p column, 
                                              size_t line_nb, 
                                              size_t element_idx)
 

src/obiarray.h

@@ -0,0 +1,177 @@
+/****************************************************************************
+ * OBIDMS array header file	                                                *
+ ****************************************************************************/
+
+/**
+ * @file array.h
+ * @author Celine Mercier
+ * @date October 19th 2015
+ * @brief Header file for handling arrays for storing and retrieving data arrays (i.e. character strings).
+ */
+
+
+#ifndef ARRAY_H_
+#define ARRAY_H_
+
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <stdint.h>
+#include <time.h>
+#include <sys/types.h>
+#include <dirent.h>
+
+#include "obidms.h"
+#include "obitypes.h"
+
+
+#define ARRAY_MAX_NAME (2048) 		/**< The maximum length of an array name.
+                              	  	 */
+#define ARRAY_GROWTH_FACTOR (2)		/**< The growth factor when an array is enlarged.
+                                	 */
+
+typedef int32_t index_t;	/**< Type of the array indices.
+ 	 	 	 	 	 	 	 */
+typedef char byte_t;		/**< Defining byte type since data is stored in bits
+							 *	 and char (stored on one byte) is the smallest addressable unit.
+ 	 	 	 	 	 	 	 */
+
+
+/**
+ * @brief OBIDMS array data header structure.
+ */
+typedef struct OBIDMS_array_data_header {
+	int			header_size;						/**< Size of the header in bytes.
+	 	 	 	 	 	 	 	 	 	 	 	 	 */
+	int64_t		data_size_used;						/**< Size of the data used in bytes.	// TODO not sure about type
+	 	 	 	 	 	 	 	 	 	 	 	 	 */
+	int64_t		data_size_max;						/**< Max size of the data in bytes.		// TODO not sure about type
+	 	 	 	 	 	 	 	 	 	 	 	 	 */
+	index_t		nb_items;							/**< Number of items.
+	 	 	 	 	 	 	 	 	 	 	 	 	 */
+	char    	array_name[ARRAY_MAX_NAME+1];		/**< The array name as a NULL terminated string.
+													 */
+	time_t		creation_date;			    	    /**< Date of creation of the file.
+												     */
+} OBIDMS_array_data_header_t, *OBIDMS_array_data_header_p;
+
+
+/**
+ * @brief OBIDMS array data structure.
+ */
+typedef struct OBIDMS_array_data {
+	OBIDMS_array_data_header_p	header; 		 	/**< A pointer to the header of the array data.
+	 	 	 	 	 	 	 	 	 	 	 	 	 */
+	byte_t*                   	data;   		 	/**< A pointer to the beginning of the data.
+													 */
+} OBIDMS_array_data_t, *OBIDMS_array_data_p;
+
+
+/**
+ * @brief OBIDMS array header structure.
+ */
+typedef struct OBIDMS_array_header {
+	int			header_size;						/**< Size of the header in bytes.
+	 	 	 	 	 	 	 	 	 	 	 	 	 */
+	index_t		nb_items;							/**< Number of items in the array.
+	 	 	 	 	 	 	 	 	 	 	 	 	 */
+	index_t		nb_items_max;						/**< Maximum number of items in the array before it has to be enlarged.
+	 	 	 	 	 	 	 	 	 	 	 	 	 */
+	char    	array_name[ARRAY_MAX_NAME+1];		/**< The array name as a NULL terminated string.
+													 */
+	time_t		creation_date;			    	    /**< Date of creation of the file.
+												     */
+} OBIDMS_array_header_t, *OBIDMS_array_header_p;
+
+
+/**
+ * @brief OBIDMS array structure.
+ */
+typedef struct OBIDMS_array {
+	OBIDMS_array_header_p		header; 		 	/**< A pointer to the header of the array.
+	 	 	 	 	 	 	 	 	 	 	 	 	 */
+	index_t*                 	first;   		 	/**< A pointer to the beginning of the array itself.
+													 */
+	OBIDMS_array_data_p			data;				/**< A pointer to the structure containing the data
+													 *   that the array references.
+	 	 	 	 	 	 	 	 	 	 	 	 	 */
+	DIR*						directory;			/**< A directory entry usable to
+										 	 	 	 *    refer and scan the array directory.
+										 	 	 	 */
+} OBIDMS_array_t, *OBIDMS_array_p;
+
+
+/**
+ *
+ */
+int obi_array_exists(OBIDMS_p dms, const char* array_name);
+
+
+/**
+ *
+ */
+OBIDMS_array_p obi_array(OBIDMS_p dms, const char* array_name);
+
+
+/**
+ *
+ */
+OBIDMS_array_p obi_create_array(OBIDMS_p dms, const char* array_name);
+
+
+/**
+ *
+ */
+OBIDMS_array_p obi_open_array(OBIDMS_p dms, const char* array_name);
+
+
+/**
+ *
+ */
+int obi_close_array(OBIDMS_array_p array);
+
+
+/**
+ *
+ */
+//int obi_delete_array(OBIDMS_p dms, const char* array_name);
+
+
+/**
+ * The header is already in the byte array
+ */
+index_t obi_array_add(OBIDMS_array_p array, byte_t* value);
+
+
+/**
+ * Returns the entire array including the header
+ */
+byte_t* obi_array_get(OBIDMS_array_p array, index_t index);
+
+
+/**
+ *
+ */
+index_t obi_array_search(OBIDMS_array_p array, byte_t* value);
+
+
+/**
+ *
+ */
+byte_t* obi_str_to_obibytes(char* value);
+
+
+/**
+ *
+ */
+const char* obi_obibytes_to_str(byte_t* value_b);
+
+
+/**
+ *	TODO LATER
+ */
+// OBIDMS_array_p obi_column_arrayify(FILE, encoding);
+
+
+#endif /* ARRAY_H_ */
+

src/obidms.c

@@ -14,6 +14,7 @@
 #include <stdio.h>
 #include <string.h>
 #include <sys/stat.h>
+#include <fcntl.h>
 #include <sys/types.h>
 #include <dirent.h>
 
@@ -21,6 +22,7 @@
 #include "obierrno.h"
 #include "obidebug.h"
 #include "obidmscolumn.h"
+#include "private_at_functions.h"
 
 
 #define DEBUG_LEVEL 0	// TODO has to be defined somewhere else (cython compil flag?)
@@ -114,6 +116,8 @@ int obi_dms_exists(const char* dms_name)
 OBIDMS_p obi_create_dms(const char* dms_name)
 {
 	char* directory_name;
+	DIR* dms_dir;
+	int dms_file_descriptor;
 
 	// Build and check the directory name
 	directory_name = build_directory_name(dms_name);
@@ -132,6 +136,35 @@ OBIDMS_p obi_create_dms(const char* dms_name)
 		return NULL;
 	}
 
+	// Get file descriptor of DMS directory to create the arrays directory
+	dms_dir = opendir(directory_name);
+	if (dms_dir == NULL)
+	{
+		obi_set_errno(OBIDMS_UNKNOWN_ERROR);
+		obidebug(1, "\nProblem opening a newly created OBIDMS directory");
+		free(directory_name);
+		return NULL;
+	}
+	dms_file_descriptor = dirfd(dms_dir);
+	if (dms_file_descriptor < 0)
+	{
+		obi_set_errno(OBIDMS_UNKNOWN_ERROR);
+		obidebug(1, "\nProblem getting the file descriptor of a newly created OBIDMS directory");
+		free(directory_name);
+		return NULL;
+	}
+
+	// Create the arrays directory
+	if (mkdirat(dms_file_descriptor, ARRAY_DIR_NAME, 00777) < 0)
+	{
+		obi_set_errno(OBI_ARRAY_ERROR);
+		obidebug(1, "\nProblem creating an arrays directory");
+		free(directory_name);
+		return NULL;
+	}
+
+	// TODO close file descriptor?
+
 	free(directory_name);
 
 	return obi_open_dms(dms_name);
@@ -143,6 +176,7 @@ OBIDMS_p obi_open_dms(const char* dms_name)
 	OBIDMS_p	dms;
 	char*	 	directory_name;
 	DIR*		directory;
+	int			dms_file_descriptor;
 
 	dms = NULL;
 
@@ -191,7 +225,29 @@ OBIDMS_p obi_open_dms(const char* dms_name)
 	strcpy(dms->directory_name, directory_name);
 	dms->directory = directory;
 
+	// Get file descriptor of DMS directory to open the arrays directory
+	dms_file_descriptor = dirfd(directory);
+	if (dms_file_descriptor < 0)
+	{
+		obi_set_errno(OBIDMS_UNKNOWN_ERROR);
+		obidebug(1, "\nError getting the file descriptor for a newly created OBIDMS directory");
 		free(directory_name);
+		return NULL;
+	}
+
+	// Open the arrays directory
+	dms->array_directory = private_opendirat(dms_file_descriptor, ARRAY_DIR_NAME);
+	if (dms->array_directory == NULL)
+	{
+		obi_set_errno(OBIDMS_UNKNOWN_ERROR);
+		obidebug(1, "\nError opening the arrays directory");
+		free(directory_name);
+		return NULL;
+	}
+
+	free(directory_name);
+
+	// TODO test if close file descriptor
 
 	return dms;
 }
@@ -223,7 +279,14 @@ int obi_close_dms(OBIDMS_p dms)
 		if (closedir(dms->directory) < 0)
 		{
 			obi_set_errno(OBIDMS_MEMORY_ERROR);
-			obidebug(1, "\nError closing an OBIDSM directory");
+			obidebug(1, "\nError closing an OBIDMS directory");
+			free(dms);
+			return -1;
+		}
+		if (closedir(dms->array_directory) < 0)
+		{
+			obi_set_errno(OBI_ARRAY_ERROR);
+			obidebug(1, "\nError closing an array directory");
 			free(dms);
 			return -1;
 		}

src/obidms.h

@@ -24,7 +24,9 @@
 #include "obierrno.h"
 
 
-#define OBIDMS_MAX_NAME (2048) /**< The maximum length of an OBIDMS name
+#define OBIDMS_MAX_NAME (2048) 	   /**< The maximum length of an OBIDMS name.
+                                	*/
+#define ARRAY_DIR_NAME "arrays"	   /**< The name of the arrays directory.
                                 	*/
 
 
@@ -41,6 +43,9 @@ typedef struct OBIDMS {
 	DIR*	directory;							/**< A directory entry usable to
 										 	 	 *   refer and scan the database directory.
 										 	 	 */
+	DIR*	array_directory;					/**< A directory entry usable to
+										 	 	 *   refer and scan the array directory.
+										 	 	 */
 } OBIDMS_t, *OBIDMS_p;
 
 

src/obidmscolumn.c

@@ -16,6 +16,7 @@
 #include <sys/types.h>
 #include <dirent.h>
 #include <unistd.h>
+#include <time.h>
 #include <fcntl.h>
 #include <stdbool.h>
 #include <math.h>
@@ -28,6 +29,7 @@
 #include "obierrno.h"
 #include "obidebug.h"
 #include "obilittlebigman.h"
+#include "obiarray.h"
 
 
 #define DEBUG_LEVEL 0	// TODO has to be defined somewhere else (cython compil flag?)
@@ -153,33 +155,33 @@ static size_t get_line_count_per_page(OBIType_t data_type, size_t nb_elements_pe
 
 static char* build_column_file_name(const char* column_name, obiversion_t version_number)
 {
-	char *filename;
+	char* file_name;
 
-	// Build the database directory name
+	// Build the file name
-	if (asprintf(&filename,"%s@%d.odc", column_name, version_number) < 0)
+	if (asprintf(&file_name,"%s@%d.odc", column_name, version_number) < 0)
 	{
 		obi_set_errno(OBICOL_MEMORY_ERROR);
 		obidebug(1, "\nError building a column file name");
 		return NULL;
 	}
 
-	return filename;
+	return file_name;
 }
 
 
 static char* build_version_file_name(const char* column_name)
 {
-	char *filename;
+	char* file_name;
 
-	// Build the database directory name
+	// Build the file name
-	if (asprintf(&filename,"%s.odv", column_name) < 0)
+	if (asprintf(&file_name,"%s.odv", column_name) < 0)
 	{
 		obi_set_errno(OBICOL_MEMORY_ERROR);
 		obidebug(1, "\nError building a version file name");
 		return NULL;
 	}
 
-	return filename;
+	return file_name;
 }
 
 
@@ -602,11 +604,13 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms,
 								  OBIType_t data_type,
 								  size_t nb_lines,
 								  size_t nb_elements_per_line,
-								  const char* elements_names)
+								  const char* elements_names,
+								  const char* array_name)
 {
 	OBIDMS_column_p 			new_column;
 	OBIDMS_column_directory_p	column_directory;
 	OBIDMS_column_header_p 		header;
+	OBIDMS_array_p				array;
 	size_t 						file_size;
 	obiversion_t 				version_number;
 	char* 						column_file_name;
@@ -629,17 +633,23 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms,
 		obidebug(1, "\nCan't create column because of empty column name");
 		return NULL;
 	}
-	if ((data_type < 1) || (data_type > 4))
+	if ((data_type < 1) || (data_type > 5))
 	{
 		obidebug(1, "\nCan't create column because of invalid data type");
 		return NULL;
 	}
+	if ((data_type == 5) && (array_name == NULL))
+	{
+		obidebug(1, "\nCan't create column because of empty array name");
+		return NULL;
+	}
+
 
 	// The initial line count should be between the minimum (corresponding to the page size) and the maximum allowed
 	minimum_line_count = get_line_count_per_page(data_type, nb_elements_per_line);
 	if (nb_lines > MAXIMUM_LINE_COUNT)
 	{
-		obidebug(1, "\nCan't create column because of line count greater than the maximum allowed (%ld)", MAXIMUM_LINE_COUNT);
+		obidebug(1, "\nCan't create column because of line count greater than the maximum allowed (%d)", MAXIMUM_LINE_COUNT);
 		return NULL;
 	}
 	else if (nb_lines < minimum_line_count)
@@ -800,6 +810,23 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms,
 
 	strncpy(header->name, column_name, OBIDMS_COLUMN_MAX_NAME);
 
+	// If the data type is OBI_IDX, the associated obi_array is opened or created
+	if (data_type == 5)
+	{
+		array = obi_array(dms, array_name);
+		if (array == NULL)
+		{
+			obidebug(1, "\nError opening or creating the array associated with a column");
+			munmap(new_column->header, header_size);
+			close(column_file_descriptor);
+			free(column_file_name);
+			free(new_column);
+			return NULL;
+		}
+		new_column->array = array;
+		strncpy(header->array_name, array_name, ARRAY_MAX_NAME);
+	}
+
 	// Fill the data with NA values
 	obi_ini_to_NA_values(new_column, 0, nb_lines);
 
@@ -816,6 +843,7 @@ OBIDMS_column_p obi_open_column(OBIDMS_p dms,
 {
 	OBIDMS_column_p 		    column;
 	OBIDMS_column_directory_p	column_directory;
+	OBIDMS_array_p				array;
 	char* 						column_file_name;
 	int 						column_file_descriptor;
 	int							column_dir_file_descriptor;
@@ -931,6 +959,22 @@ OBIDMS_column_p obi_open_column(OBIDMS_p dms,
 
 	column->writable  = false;
 
+	// If the data type is OBI_IDX, the associated obi_array is opened or created
+	if ((column->header)->data_type == 5)
+	{
+		array = obi_array(dms, (column->header)->array_name);
+		if (array == NULL)
+		{
+			obidebug(1, "\nError opening the array associated with a column");
+			munmap(column->header, header_size);
+			close(column_file_descriptor);
+			free(column_file_name);
+			free(column);
+			return NULL;
+		}
+		column->array = array;
+	}
+
 	free(column_file_name);
 	close(column_file_descriptor);
 
@@ -972,7 +1016,8 @@ OBIDMS_column_p obi_clone_column(OBIDMS_p dms,
 								   data_type,
 								   nb_lines,
 								   nb_elements_per_line,
-								   (column_to_clone->header)->elements_names);
+								   (column_to_clone->header)->elements_names,
+								   (column_to_clone->header)->array_name);
 
 	if (new_column == NULL)
 	{
@@ -1170,7 +1215,7 @@ int obi_enlarge_column(OBIDMS_column_p column)
 
 	// Calculate the new file size
 	old_line_count = (column->header)->line_count;
-	new_line_count = old_line_count * GROWTH_FACTOR;
+	new_line_count = old_line_count * COLUMN_GROWTH_FACTOR;
 
 	if (new_line_count > MAXIMUM_LINE_COUNT)
 	{
@@ -1181,7 +1226,7 @@ int obi_enlarge_column(OBIDMS_column_p column)
 		return -1;
 	}
 	old_data_size = obi_array_sizeof((column->header)->data_type, old_line_count, (column->header)->nb_elements_per_line);
-	new_data_size = old_data_size * GROWTH_FACTOR;
+	new_data_size = old_data_size * COLUMN_GROWTH_FACTOR;
 	header_size = (column->header)->header_size;
 	file_size = header_size + new_data_size;
 

src/obidmscolumn.h

@@ -25,19 +25,21 @@
 #include "obierrno.h"
 #include "obilittlebigman.h"
 #include "obidmscolumndir.h"
+#include "obiarray.h"
 
 
 #define ONE_IF_ZERO(x) (((x)==0)?1:(x))	/**< If x is equal to 0, x takes the value 1.
                                 	   	 */
 #define ELEMENTS_NAMES_MAX (2048)     	/**< The maximum length of the list of elements names.
                                 	     */
-#define GROWTH_FACTOR (2)			  	/**< The growth factor when a column is enlarged.
+#define COLUMN_GROWTH_FACTOR (2)	 	/**< The growth factor when a column is enlarged.
                                 	   	 */
 #define MAXIMUM_LINE_COUNT (1000000)    /**< The maximum line count for the data of a column. //TODO
                                 	     */
 #define FORMATTED_TIME_LENGTH (1024)    /**< The length allocated for the character string containing a formatted date
  	 	 	 	 	 	 	 	 	     */
 
+
 typedef int32_t obiversion_t; 	/**< Used to store the column version number
                                  */
 
@@ -74,6 +76,8 @@ typedef struct OBIDMS_column_header {
 															 */
 	char            name[OBIDMS_COLUMN_MAX_NAME+1]; 	    /**< The column name as a NULL terminated string.
 	                                             	 	 	 */
+	char            array_name[ARRAY_MAX_NAME+1]; 	    	/**< If there is one, the obi_array name as a NULL terminated string.
+	                                             	 	 	 */
 	char 			comments[1];							/**< Comments stored as a classical zero end C string.
 												 	 	 	 *	 The size of the comment is only limited by the header size.
 												 	 	 	 */
@@ -93,6 +97,8 @@ typedef struct OBIDMS_column {
 	 	 	 	 	 	 	 	 	 	 	 	 	 */
 	OBIDMS_column_header_p		header; 		 	/**< A pointer to the header of the column.
 	 	 	 	 	 	 	 	 	 	 	 	 	 */
+	OBIDMS_array_p 				array;				/**< A pointer to the array associated with the column if there is one.
+	 	 	 	 	 	 	 	 	 	 	 	 	 */
 	void*                   	data;   		 	/**< A `void` pointer to the beginning of the data.
 	                                 	 	         *
 													 *   @warning Never use this member directly outside of the code of the
@@ -176,7 +182,8 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms,
 								  OBIType_t data_type,
 								  size_t nb_lines,
 								  size_t nb_elements_per_line,
-								  const char* elements_names);
+								  const char* elements_names,
+								  const char* array_name);
 
 
 /**

src/obidmscolumn_str.c

@@ -1,27 +1,15 @@
 /****************************************************************************
- * OBIDMS_column_idx functions                                              *
+ * OBIDMS_column_str functions                                              *
  ****************************************************************************/
 
 /**
- * @file obidsmcolumn_idx.c
+ * @file obidsmcolumn_str.c
  * @author Celine Mercier
- * @date August 10th 2015
+ * @date October 28th 2015
  * @brief Functions handling OBIColumns containing data with the OBIType OBI_IDX.
  */
 
 
-/****************************************************************************
- * OBIDMS_column_idx functions                                             *
- ****************************************************************************/
-
-/**
- * @file obidsmcolumn_idx.c
- * @author Celine Mercier
- * @date August 10th 2015
- * @brief Functions handling OBIColumns containing data with the OBIType OBI_BOOL.
- */
-
-
 #include <stdlib.h>
 #include <stdio.h>
 
@@ -29,6 +17,7 @@
 #include "obitypes.h"
 #include "obierrno.h"
 #include "obidebug.h"
+#include "obiarray.h"
 
 
 #define DEBUG_LEVEL 0	// TODO has to be defined somewhere else (cython compil flag?)
@@ -40,8 +29,11 @@
  *
  **********************************************************************/
 
-int obi_column_set_obiidx_with_elt_idx(OBIDMS_column_p column, size_t line_nb, size_t element_idx, obiidx_t value)
+int obi_column_set_obistr_with_elt_idx(OBIDMS_column_p column, size_t line_nb, size_t element_idx, char* value)
 {
+	byte_t* value_b;
+	index_t idx;
+
 	// Check that the line number is not greater than the maximum allowed
 	if (line_nb >= MAXIMUM_LINE_COUNT)
 	{
@@ -62,43 +54,84 @@ int obi_column_set_obiidx_with_elt_idx(OBIDMS_column_p column, size_t line_nb, s
 	if ((line_nb+1) > (column->header)->lines_used)
 		(column->header)->lines_used = line_nb+1;
 
-	// Set the value
+	// Encode the value on a byte array with a header
-	*(((obiidx_t*) (column->data)) + (line_nb * ((column->header)->nb_elements_per_line)) + element_idx) = value;
+	value_b = obi_str_to_obibytes(value);
+	if (value_b == NULL)
+		return -1;
+
+	obidebug(1, "\nvalue=%s", value);
+	//obidebug(1, "\nbytes=%s", value_b+5);
+
+	// Add in the obiarray
+	idx = obi_array_add(column->array, value_b);
+	if (idx == -1)
+		return -1;
+
+	obidebug(1, "\nidx=%d", idx);
+	//obidebug(1, "\nbytes 2=%s", obi_array_get(column->array, idx)+5);
+
+	// Add the value's index in the column
+	*(((obiidx_t*) (column->data)) + (line_nb * ((column->header)->nb_elements_per_line)) + element_idx) = idx;
+
+	// TODO free value_b probably
 
 	return 0;
 }
 
 
-obiidx_t obi_column_get_obiidx_with_elt_idx(OBIDMS_column_p column, size_t line_nb, size_t element_idx)
+const char* obi_column_get_obistr_with_elt_idx(OBIDMS_column_p column, size_t line_nb, size_t element_idx)
 {
+	index_t idx;
+	byte_t* value_b;
+	const char* value;
+
 	if ((line_nb+1) > (column->header)->lines_used)
 	{
 		obi_set_errno(OBICOL_UNKNOWN_ERROR);
 		obidebug(1, "\nError trying to get a value that is beyond the current number of lines used");
-		return OBIIdx_NA;
+		return "\0";		// TODO
 	}
-	return *(((obiidx_t*) (column->data)) + (line_nb * ((column->header)->nb_elements_per_line)) + element_idx);
+
+	idx = *(((obiidx_t*) (column->data)) + (line_nb * ((column->header)->nb_elements_per_line)) + element_idx);
+
+	// Check NA
+	if (idx == OBIIdx_NA)
+		return "\0";		// TODO
+
+	obidebug(1, "\nwhy, idx = %d", idx);
+
+	value_b = obi_array_get(column->array, idx);
+
+	obidebug(1, "\nwhyyyy");
+
+	value = obi_obibytes_to_str(value_b);
+
+	obidebug(1, "\nwhyyyyyyyyyyyy, value=%s %p", value, value);
+	obidebug(1, "\nwhyyyyyyyyyyyy, len value=%d", strlen(value));
+
+	return value;
 }
 
 
-int obi_column_set_obiidx_with_elt_name(OBIDMS_column_p column, size_t line_nb, const char* element_name, obiidx_t value)
+int obi_column_set_obistr_with_elt_name(OBIDMS_column_p column, size_t line_nb, const char* element_name, char* value)
 {
 	size_t element_idx;
 	element_idx = obi_column_get_element_index_from_name(column, element_name);
 	if (element_idx == SIZE_MAX)	//TODO
 		return -1;
-	obi_column_set_obiidx_with_elt_idx(column, line_nb, element_idx, value);
+	if (obi_column_set_obistr_with_elt_idx(column, line_nb, element_idx, value) < 0)
+		return -1;
 	return 0;
 }
 
 
-obiidx_t obi_column_get_obiidx_with_elt_name(OBIDMS_column_p column, size_t line_nb, const char* element_name)
+const char* obi_column_get_obistr_with_elt_name(OBIDMS_column_p column, size_t line_nb, const char* element_name)
 {
 	size_t element_idx;
 
 	element_idx = obi_column_get_element_index_from_name(column, element_name);
 	if (element_idx == SIZE_MAX)	//TODO
-		return OBIIdx_NA;
+		return "\0";
-	return obi_column_get_obiidx_with_elt_idx(column, line_nb, element_idx);
+	return obi_column_get_obistr_with_elt_idx(column, line_nb, element_idx);
 }
 

src/obidmscolumn_str.h

@@ -38,7 +38,7 @@
  * @since July 2015
  * @author Celine Mercier (celine.mercier@metabarcoding.org)
  */
-int obi_column_set_obiidx_with_elt_idx(OBIDMS_column_p column, size_t line_nb, size_t element_idx, obiidx_t value);
+int obi_column_set_obiidx_with_elt_idx(OBIDMS_column_p column, size_t line_nb, size_t element_idx, void* value);
 
 
 /**
@@ -54,7 +54,7 @@ int obi_column_set_obiidx_with_elt_idx(OBIDMS_column_p column, size_t line_nb, s
  * @since July 2015
  * @author Celine Mercier (celine.mercier@metabarcoding.org)
  */
-obiidx_t obi_column_get_obiidx_with_elt_idx(OBIDMS_column_p column, size_t line_nb, size_t element_idx);
+const char* obi_column_get_obiidx_with_elt_idx(OBIDMS_column_p column, size_t line_nb, size_t element_idx);
 
 
 /**
@@ -76,7 +76,7 @@ obiidx_t obi_column_get_obiidx_with_elt_idx(OBIDMS_column_p column, size_t line_
  * @since August 2015
  * @author Celine Mercier (celine.mercier@metabarcoding.org)
  */
-int obi_column_set_obiidx_with_elt_name(OBIDMS_column_p column, size_t line_nb, const char* element_name, obiidx_t value);
+int obi_column_set_obiidx_with_elt_name(OBIDMS_column_p column, size_t line_nb, const char* element_name, void* value);
 
 
 /**
@@ -94,7 +94,7 @@ int obi_column_set_obiidx_with_elt_name(OBIDMS_column_p column, size_t line_nb,
  * @since August 2015
  * @author Celine Mercier (celine.mercier@metabarcoding.org)
  */
-obiidx_t obi_column_get_obiidx_with_elt_name(OBIDMS_column_p column, size_t line_nb, const char* element_name);
+const char* obi_column_get_obiidx_with_elt_name(OBIDMS_column_p column, size_t line_nb, const char* element_name);
 
 
 #endif /* OBIDMSCOLUMN_IDX_H_ */

src/obierrno.h

@@ -98,6 +98,8 @@ extern int obi_errno;
                                     	 	 */
 #define OBICOL_ACCESS_ERROR    		(19) 	/**< Permission error trying to access an OBIDSM column directory
                                     		 */
+#define OBI_ARRAY_ERROR				(20)    /** Error while handling a heap
+											 */
 /**@}*/
 
 #endif /* OBIERRNO_H_ */

src/private_at_functions.c

@@ -43,7 +43,7 @@ char* get_full_path(int directory_file_descriptor, const char* path_name)
 		return NULL;
 	}
 
-	// TODO check errors
+	// TODO check errors?
 	strlcat(full_path, "/", MAX_PATH_LEN);
 	strlcat(full_path, path_name, MAX_PATH_LEN);