From 48d10ea17c4a94ab3df04eaa2eeffe2cf3c886c3 Mon Sep 17 00:00:00 2001 From: Celine Mercier Date: Wed, 26 Aug 2015 17:01:54 +0200 Subject: [PATCH] Functions to truncate and/or close a column --- .../obidms/obidmscolumn/capidmscolumn.pxd | 1 + .../obidms/obidmscolumn/capidmscolumn.pyx | 6 +- .../obidmscolumn_bool/capidmscolumn_bool.pyx | 11 +- .../obidmscolumn_char/capidmscolumn_char.pyx | 9 ++ .../capidmscolumn_float.pyx | 9 ++ .../obidmscolumn_idx/capidmscolumn_idx.pyx | 9 ++ .../obidmscolumn_int/capidmscolumn_int.pyx | 10 +- src/obidmscolumn.c | 119 +++++++++++++++++- src/obidmscolumn.h | 28 +++++ 9 files changed, 193 insertions(+), 9 deletions(-) diff --git a/python/obitools3/obidms/obidmscolumn/capidmscolumn.pxd b/python/obitools3/obidms/obidmscolumn/capidmscolumn.pxd index b6b80c7..e45f94b 100644 --- a/python/obitools3/obidms/obidmscolumn/capidmscolumn.pxd +++ b/python/obitools3/obidms/obidmscolumn/capidmscolumn.pxd @@ -34,6 +34,7 @@ cdef extern from "obidmscolumn.h" nogil: int obi_close_column(OBIDMS_column_p column) OBIType_t obi_column_get_data_type_from_name(OBIDMS_p dms, const char* column_name) OBIDMS_column_p obi_clone_column(OBIDMS_p dms, const char* column_name, obiversion_t version_number, bint clone_data) + int obi_truncate_and_close_column(OBIDMS_column_p column) cdef class OBIDMS_column: diff --git a/python/obitools3/obidms/obidmscolumn/capidmscolumn.pyx b/python/obitools3/obidms/obidmscolumn/capidmscolumn.pyx index ffba429..d7a2cec 100644 --- a/python/obitools3/obidms/obidmscolumn/capidmscolumn.pyx +++ b/python/obitools3/obidms/obidmscolumn/capidmscolumn.pyx @@ -65,11 +65,7 @@ cdef class OBIDMS_column: def get_data_type(self): return self.data_type - -# def __del__(self): -# if obi_close_column(self.pointer) != 0 : -# raise Exception("Problem closing a column") - + @staticmethod def open(dms_name, column_name, create=False, clone=False, clone_data=True, diff --git a/python/obitools3/obidms/obidmscolumn/obidmscolumn_bool/capidmscolumn_bool.pyx b/python/obitools3/obidms/obidmscolumn/obidmscolumn_bool/capidmscolumn_bool.pyx index f38b903..929523f 100644 --- a/python/obitools3/obidms/obidmscolumn/obidmscolumn_bool/capidmscolumn_bool.pyx +++ b/python/obitools3/obidms/obidmscolumn/obidmscolumn_bool/capidmscolumn_bool.pyx @@ -13,11 +13,20 @@ cdef class OBIDMS_column_bool_read(OBIDMS_column_bool) : def set_item(self, line_nb, element_name, value): raise Exception('Column is read-only') + + def close(self): + if obi_close_column(self.pointer) < 0 : + raise Exception("Problem closing a column") cdef class OBIDMS_column_bool_writable(OBIDMS_column_bool) : def set_item(self, line_nb, element_name, value): return obi_column_set_bool_with_elt_name(self.pointer, line_nb, element_name.encode('utf-8'), value) - + + def close(self): + if obi_truncate_and_close_column(self.pointer) < 0 : + raise Exception("Problem closing a column") + + \ No newline at end of file diff --git a/python/obitools3/obidms/obidmscolumn/obidmscolumn_char/capidmscolumn_char.pyx b/python/obitools3/obidms/obidmscolumn/obidmscolumn_char/capidmscolumn_char.pyx index 76a1cbc..6e9bf52 100644 --- a/python/obitools3/obidms/obidmscolumn/obidmscolumn_char/capidmscolumn_char.pyx +++ b/python/obitools3/obidms/obidmscolumn/obidmscolumn_char/capidmscolumn_char.pyx @@ -8,16 +8,25 @@ cdef class OBIDMS_column_char(OBIDMS_column) : def get_item(self, line_nb, element_name): return (obi_column_get_char_with_elt_name(self.pointer, line_nb, element_name.encode('utf-8'))).decode(encoding='utf-8')[:1] + cdef class OBIDMS_column_char_read(OBIDMS_column_char) : def set_item(self, line_nb, element_name, value): raise Exception('Column is read-only') + + def close(self): + if obi_close_column(self.pointer) < 0 : + raise Exception("Problem closing a column") cdef class OBIDMS_column_char_writable(OBIDMS_column_char) : def set_item(self, line_nb, element_name, value): return obi_column_set_char_with_elt_name(self.pointer, line_nb, element_name.encode('utf-8'), value.encode('utf-8')) + + def close(self): + if obi_truncate_and_close_column(self.pointer) < 0 : + raise Exception("Problem closing a column") \ No newline at end of file diff --git a/python/obitools3/obidms/obidmscolumn/obidmscolumn_float/capidmscolumn_float.pyx b/python/obitools3/obidms/obidmscolumn/obidmscolumn_float/capidmscolumn_float.pyx index b48a616..273d75a 100644 --- a/python/obitools3/obidms/obidmscolumn/obidmscolumn_float/capidmscolumn_float.pyx +++ b/python/obitools3/obidms/obidmscolumn/obidmscolumn_float/capidmscolumn_float.pyx @@ -8,15 +8,24 @@ cdef class OBIDMS_column_float(OBIDMS_column) : def get_item(self, line_nb, element_name): return obi_column_get_float_with_elt_name(self.pointer, line_nb, element_name.encode('utf-8')) + cdef class OBIDMS_column_float_read(OBIDMS_column_float) : def set_item(self, line_nb, element_name, value): raise Exception('Column is read-only') + + def close(self): + if obi_close_column(self.pointer) < 0 : + raise Exception("Problem closing a column") cdef class OBIDMS_column_float_writable(OBIDMS_column_float) : def set_item(self, line_nb, element_name, value): return obi_column_set_float_with_elt_name(self.pointer, line_nb, element_name.encode('utf-8'), value) + + def close(self): + if obi_truncate_and_close_column(self.pointer) < 0 : + raise Exception("Problem closing a column") \ No newline at end of file diff --git a/python/obitools3/obidms/obidmscolumn/obidmscolumn_idx/capidmscolumn_idx.pyx b/python/obitools3/obidms/obidmscolumn/obidmscolumn_idx/capidmscolumn_idx.pyx index e93b267..bfafb0b 100644 --- a/python/obitools3/obidms/obidmscolumn/obidmscolumn_idx/capidmscolumn_idx.pyx +++ b/python/obitools3/obidms/obidmscolumn/obidmscolumn_idx/capidmscolumn_idx.pyx @@ -7,15 +7,24 @@ cdef class OBIDMS_column_idx(OBIDMS_column) : def get_item(self, line_nb, element_name): return obi_column_get_idx_with_elt_name(self.pointer, line_nb, element_name.encode('utf-8')) + cdef class OBIDMS_column_idx_read(OBIDMS_column_idx) : def set_item(self, line_nb, element_name, value): raise Exception('Column is read-only') + + def close(self): + if obi_close_column(self.pointer) < 0 : + raise Exception("Problem closing a column") cdef class OBIDMS_column_idx_writable(OBIDMS_column_idx) : def set_item(self, line_nb, element_name, value): return obi_column_set_idx_with_elt_name(self.pointer, line_nb, element_name.encode('utf-8'), value) + + def close(self): + if obi_truncate_and_close_column(self.pointer) < 0 : + raise Exception("Problem closing a column") \ No newline at end of file diff --git a/python/obitools3/obidms/obidmscolumn/obidmscolumn_int/capidmscolumn_int.pyx b/python/obitools3/obidms/obidmscolumn/obidmscolumn_int/capidmscolumn_int.pyx index 447962f..4146f9f 100644 --- a/python/obitools3/obidms/obidmscolumn/obidmscolumn_int/capidmscolumn_int.pyx +++ b/python/obitools3/obidms/obidmscolumn/obidmscolumn_int/capidmscolumn_int.pyx @@ -13,10 +13,18 @@ cdef class OBIDMS_column_int_read(OBIDMS_column_int) : def set_item(self, line_nb, element_name, value): raise Exception('Column is read-only') - + + def close(self): + if obi_close_column(self.pointer) < 0 : + raise Exception("Problem closing a column") + cdef class OBIDMS_column_int_writable(OBIDMS_column_int) : def set_item(self, line_nb, element_name, value): return obi_column_set_int_with_elt_name(self.pointer, line_nb, element_name.encode('utf-8'), value) + + def close(self): + if obi_truncate_and_close_column(self.pointer) < 0 : + raise Exception("Problem closing a column") \ No newline at end of file diff --git a/src/obidmscolumn.c b/src/obidmscolumn.c index 1fb5005..b9300c3 100644 --- a/src/obidmscolumn.c +++ b/src/obidmscolumn.c @@ -948,15 +948,130 @@ OBIDMS_column_p obi_clone_column(OBIDMS_p dms, const char* column_name, obiversi (new_column->header)->lines_used = nb_lines; } + // close column_to_clone TODO + return new_column; } int obi_close_column(OBIDMS_column_p column) { - //munmap? TODO - //truncate to lines used TODO + size_t data_size; + + // Munmap data + data_size = (column->header)->line_count * (column->header)->nb_elements_per_line * sizeof((column->header)->data_type); + if (munmap(column->data, data_size) < 0) + { + obi_set_errno(OBICOL_UNKNOWN_ERROR); + obidebug(1, "\nError munmapping column data"); + return -1; + } + + // Munmap header + if (munmap(column->header, (column->header)->header_size) < 0) + { + obi_set_errno(OBICOL_UNKNOWN_ERROR); + obidebug(1, "\nError munmapping a column header"); + return -1; + } + free(column); + + return 0; +} + + +int obi_truncate_column_to_lines_used(OBIDMS_column_p column) +{ + size_t file_size; + size_t data_size; + int column_dir_file_descriptor; + int column_file_descriptor; + char* column_file_name; + + // Get the file descriptor associated to the column directory + column_dir_file_descriptor = dirfd((column->column_directory)->directory); + if (column_dir_file_descriptor < 0) + { + obi_set_errno(OBICOLDIR_UNKNOWN_ERROR); + obidebug(1, "\nError getting the file descriptor for a column directory"); + return -1; + } + + // Get the column file name + column_file_name = build_column_file_name((column->header)->name, (column->header)->version); + if (column_file_name == NULL) + { + return -1; + } + + // Open the column file + column_file_descriptor = openat(column_dir_file_descriptor, column_file_name, O_RDWR | O_CREAT); + if (column_file_descriptor < 0) + { + obi_set_errno(OBICOL_UNKNOWN_ERROR); + free(column_file_name); + return -1; + } + + // Unmap the data before truncating the file + data_size = (column->header)->line_count * (column->header)->nb_elements_per_line * sizeof((column->header)->data_type); + if (munmap(column->data, data_size) < 0) + { + obi_set_errno(OBICOL_UNKNOWN_ERROR); + obidebug(1, "\nError munmapping the data of a column before truncating"); + free(column_file_name); + close(column_file_descriptor); + return -1; + } + + // Truncate the column file at the number of lines used + data_size = (column->header)->lines_used * (column->header)->nb_elements_per_line * sizeof((column->header)->data_type); + file_size = (column->header)->header_size + data_size; + if (ftruncate(column_file_descriptor, file_size) < 0) + { + obi_set_errno(OBICOL_UNKNOWN_ERROR); + obidebug(1, "\nError truncating a column file at the number of lines used"); + free(column_file_name); + close(column_file_descriptor); + return -1; + } + + // Remap the data + column->data = mmap(NULL, + data_size, + PROT_READ | PROT_WRITE, + MAP_SHARED, + column_file_descriptor, + (column->header)->header_size + ); + + if (column->data == MAP_FAILED) + { + obi_set_errno(OBICOL_UNKNOWN_ERROR); + obidebug(1, "\nError re-mmapping the data of a column after truncating"); + free(column_file_name); + close(column_file_descriptor); + return -1; + } + + // Set line_count to lines_used + (column->header)->line_count = (column->header)->lines_used; + + free(column_file_name); + close(column_file_descriptor); + + return 0; +} + + +int obi_truncate_and_close_column(OBIDMS_column_p column) +{ + if (obi_truncate_column_to_lines_used(column) < 0) + return -1; + if (obi_close_column(column) < 0) + return -1; + return 0; } diff --git a/src/obidmscolumn.h b/src/obidmscolumn.h index 11f6f37..b431e85 100644 --- a/src/obidmscolumn.h +++ b/src/obidmscolumn.h @@ -191,6 +191,34 @@ OBIDMS_column_p obi_clone_column(OBIDMS_p dms, const char* column_name, obiversi int obi_close_column(OBIDMS_column_p column); +/** + * @brief Truncates a column file to the number of lines used. + * + * @param column a pointer on an OBIDMS column + * + * @return 0 if the operation was successfully completed + * @return -1 if an error occurred + * + * @since August 2015 + * @author Celine Mercier (celine.mercier@metabarcoding.org) + */ +int obi_truncate_column_to_lines_used(OBIDMS_column_p column); + + +/** + * @brief Truncates a column file to the number of lines used and closes it. + * + * @param column a pointer on an OBIDMS column + * + * @return 0 if the operation was successfully completed + * @return -1 if an error occurred + * + * @since August 2015 + * @author Celine Mercier (celine.mercier@metabarcoding.org) + */ +int obi_truncate_and_close_column(OBIDMS_column_p column); + + /** * @brief Sets the 'writable' header attribute of an OBIDMS column to False. *