C: made it so column DIR* are not kept open to handle very large DMS

src/obidmscolumn.c

@@ -318,6 +318,9 @@ static obiversion_t obi_get_new_version_number(OBIDMS_column_directory_p column_
 	char* 			version_file_name;
 	int    			version_file_descriptor;
 	int 			lock_mode;
+	char* 			column_directory_name;
+	DIR* 			col_directory;
+	int 			col_dir_fd;
 
 	new_version_number = 0;
 	loc_size = sizeof(obiversion_t);
@@ -334,7 +337,34 @@ static obiversion_t obi_get_new_version_number(OBIDMS_column_directory_p column_
 		return -1;
 
 	// Open the version file
-	version_file_descriptor = openat(column_directory->dir_fd, version_file_name, O_RDWR);
+	column_directory_name = obi_build_column_directory_name(column_directory->column_name);
+	if (column_directory_name == NULL)
+	{
+		obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
+		obidebug(1, "\nError opening a version file");
+		free(version_file_name);
+		return -1;
+	}
+	col_directory = opendir_in_dms(column_directory->dms, column_directory_name);
+	if (col_directory == NULL)
+	{
+		obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
+		obidebug(1, "\nError opening a version file");
+		free(version_file_name);
+		free(column_directory_name);
+		return -1;
+	}
+	col_dir_fd = dirfd(col_directory);
+	if (col_dir_fd < 0)
+	{
+		obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
+		obidebug(1, "\nError opening a version file");
+		free(version_file_name);
+		free(column_directory_name);
+		closedir(col_directory);
+		return -1;
+	}
+	version_file_descriptor = openat(col_dir_fd, version_file_name, O_RDWR);
 	if (version_file_descriptor < 0)
 	{
 		if (errno == ENOENT)
@@ -349,6 +379,8 @@ static obiversion_t obi_get_new_version_number(OBIDMS_column_directory_p column_
 	}
 
 	free(version_file_name);
+	free(column_directory_name);
+	closedir(col_directory);
 
 	// Test if the version file size is ok
 	if (lseek(version_file_descriptor, 0, SEEK_END) < loc_size)
@@ -441,6 +473,9 @@ static obiversion_t create_version_file(OBIDMS_column_directory_p column_directo
 	obiversion_t 	version_number;
 	char* 			version_file_name;
 	int    			version_file_descriptor;
+	char* 			column_directory_name;
+	DIR* 			col_directory;
+	int 			col_dir_fd;
 
 	loc_size = sizeof(obiversion_t);
 	version_number = 0;
@@ -450,7 +485,34 @@ static obiversion_t create_version_file(OBIDMS_column_directory_p column_directo
 		return -1;
 
 	// Get the file descriptor associated to the version file
-	version_file_descriptor = openat(column_directory->dir_fd, version_file_name, O_RDWR | O_CREAT | O_EXCL, 0777);
+	column_directory_name = obi_build_column_directory_name(column_directory->column_name);
+	if (column_directory_name == NULL)
+	{
+		obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
+		obidebug(1, "\nError opening a version file");
+		free(version_file_name);
+		return -1;
+	}
+	col_directory = opendir_in_dms(column_directory->dms, column_directory_name);
+	if (col_directory == NULL)
+	{
+		obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
+		obidebug(1, "\nError opening a version file");
+		free(version_file_name);
+		free(column_directory_name);
+		return -1;
+	}
+	col_dir_fd = dirfd(col_directory);
+	if (col_dir_fd < 0)
+	{
+		obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
+		obidebug(1, "\nError opening a version file");
+		free(version_file_name);
+		free(column_directory_name);
+		closedir(col_directory);
+		return -1;
+	}
+	version_file_descriptor = openat(col_dir_fd, version_file_name, O_RDWR | O_CREAT | O_EXCL, 0777);
 	if (version_file_descriptor < 0)
 	{
 		obi_set_errno(OBICOL_UNKNOWN_ERROR);
@@ -460,6 +522,8 @@ static obiversion_t create_version_file(OBIDMS_column_directory_p column_directo
 	}
 
 	free(version_file_name);
+	free(column_directory_name);
+	closedir(col_directory);
 
 	// Lock the file
 	if (lockf(version_file_descriptor, F_LOCK, loc_size) < 0)
@@ -806,6 +870,9 @@ obiversion_t obi_get_latest_version_number(OBIDMS_column_directory_p column_dire
 	obiversion_t 	latest_version_number;
 	char * 			version_file_name;
 	int    			version_file_descriptor;
+	char*			column_directory_name;
+	DIR*			col_directory;
+	int				col_dir_fd;
 
 	loc_size = sizeof(obiversion_t);
 	latest_version_number = 0;
@@ -815,7 +882,34 @@ obiversion_t obi_get_latest_version_number(OBIDMS_column_directory_p column_dire
 		return -1;
 
 	// Get the file descriptor associated to the version file
-	version_file_descriptor = openat(column_directory->dir_fd, version_file_name, O_RDONLY);
+	column_directory_name = obi_build_column_directory_name(column_directory->column_name);
+	if (column_directory_name == NULL)
+	{
+		obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
+		obidebug(1, "\nError opening a version file");
+		free(version_file_name);
+		return -1;
+	}
+	col_directory = opendir_in_dms(column_directory->dms, column_directory_name);
+	if (col_directory == NULL)
+	{
+		obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
+		obidebug(1, "\nError opening a version file");
+		free(version_file_name);
+		free(column_directory_name);
+		return -1;
+	}
+	col_dir_fd = dirfd(col_directory);
+	if (col_dir_fd < 0)
+	{
+		obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
+		obidebug(1, "\nError opening a version file");
+		free(version_file_name);
+		free(column_directory_name);
+		closedir(col_directory);
+		return -1;
+	}
+	version_file_descriptor = openat(col_dir_fd, version_file_name, O_RDONLY);
 	if (version_file_descriptor < 0)
 	{
 		obi_set_errno(OBICOL_UNKNOWN_ERROR);
@@ -825,6 +919,8 @@ obiversion_t obi_get_latest_version_number(OBIDMS_column_directory_p column_dire
 	}
 
 	free(version_file_name);
+	free(column_directory_name);
+	closedir(col_directory);
 
 	// Check that the version file size is ok
 	if (lseek(version_file_descriptor, 0, SEEK_END) < loc_size)
@@ -939,6 +1035,9 @@ OBIDMS_column_p obi_create_column(OBIDMS_p     dms,
 	char*			    		final_indexer_name;
 	char*						built_elements_names = NULL;
 	int64_t						elts_names_length;
+	char*                       column_directory_name;
+	DIR*						col_dir;
+	int							col_dir_fd;
 
 	new_column = NULL;
 
@@ -1053,16 +1152,48 @@ OBIDMS_column_p obi_create_column(OBIDMS_p     dms,
 		return NULL;
 
 	// Open the column file
-	column_file_descriptor = openat(column_directory->dir_fd, column_file_name, O_RDWR | O_CREAT | O_EXCL, 0777);
+	column_directory_name = obi_build_column_directory_name(column_directory->column_name);
+	if (column_directory_name == NULL)
+	{
+		obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
+		obidebug(1, "\nError opening a column file");
+		free(column_file_name);
+		return NULL;
+	}
+	col_dir = opendir_in_dms(dms, column_directory_name);
+	if (col_dir == NULL)
+	{
+		obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
+		obidebug(1, "\nError opening a version file");
+		free(column_file_name);
+		free(column_directory_name);
+		return NULL;
+	}
+	col_dir_fd = dirfd(col_dir);
+	if (col_dir_fd < 0)
+	{
+		obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
+		obidebug(1, "\nError opening a version file");
+		free(column_file_name);
+		free(column_directory_name);
+		closedir(col_dir);
+		return NULL;
+	}
+
+	column_file_descriptor = openat(col_dir_fd, column_file_name, O_RDWR | O_CREAT | O_EXCL, 0777);
 	if (column_file_descriptor < 0)
 	{
 		obi_set_errno(OBICOL_UNKNOWN_ERROR);
 		obidebug(1, "\nError opening a column file %s", column_file_name);
 		free(column_file_name);
+		free(column_directory_name);
+		closedir(col_dir);
 		return NULL;
 	}
 
 	free(column_file_name);
+	free(column_directory_name);
+	closedir(col_dir);
 
 	// Truncate the column file to the right size
 	if (ftruncate(column_file_descriptor, file_size) < 0)
@@ -1231,6 +1362,9 @@ OBIDMS_column_p obi_open_column(OBIDMS_p     dms,
 	char* 						column_file_name;
 	int 						column_file_descriptor;
 	size_t 						header_size;
+	char*						column_directory_name;
+	DIR*						col_dir;
+	int							col_dir_fd;
 
 	column = NULL;
 
@@ -1276,16 +1410,50 @@ OBIDMS_column_p obi_open_column(OBIDMS_p     dms,
 	}
 
 	// Open the column file, ALWAYS READ-ONLY
-	column_file_descriptor = openat(column_directory->dir_fd, column_file_name, O_RDWR);
+	column_directory_name = obi_build_column_directory_name(column_directory->column_name);
+	if (column_directory_name == NULL)
+	{
+		obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
+		obidebug(1, "\nError opening a column file");
+		free(column_file_name);
+		return NULL;
+	}
+
+	col_dir = opendir_in_dms(dms, column_directory_name);
+	if (col_dir == NULL)
+	{
+		obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
+		obidebug(1, "\nError opening a version file");
+		free(column_file_name);
+		free(column_directory_name);
+		return NULL;
+	}
+
+	col_dir_fd = dirfd(col_dir);
+	if (col_dir_fd < 0)
+	{
+		obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
+		obidebug(1, "\nError opening a version file");
+		free(column_file_name);
+		free(column_directory_name);
+		closedir(col_dir);
+		return NULL;
+	}
+
+	column_file_descriptor = openat(col_dir_fd, column_file_name, O_RDWR);
 	if (column_file_descriptor < 0)
 	{
 		obi_set_errno(OBICOL_UNKNOWN_ERROR);
 		obidebug(1, "\nError opening column file %s", column_file_name);
 		free(column_file_name);
+		free(column_directory_name);
+		closedir(col_dir);
 		return NULL;
 	}
 
 	free(column_file_name);
+	free(column_directory_name);
+	closedir(col_dir);
 
 	// Allocate the memory for the column structure
 	column = (OBIDMS_column_p) malloc(sizeof(OBIDMS_column_t));
@@ -1557,7 +1725,7 @@ int obi_clone_column_indexer(OBIDMS_column_p column)
 
 
 
-int obi_truncate_column(OBIDMS_column_p column)	// TODO is it necessary to unmap/remap?
+int obi_truncate_column(OBIDMS_column_p column)	// TODO is it necessary to unmap/remap? -> yes the whole file for windows WLS
 {
 	size_t  file_size;
 	size_t  data_size;
@@ -1565,6 +1733,9 @@ int obi_truncate_column(OBIDMS_column_p column)	// TODO is it necessary to unmap
 	double  multiple;
 	int 	column_file_descriptor;
 	char* 	column_file_name;
+	char*   column_directory_name;
+	DIR*	col_dir;
+	int		col_dir_fd;
 
 	// Check if the column is read-only
 	if (!(column->writable))
@@ -1596,16 +1767,47 @@ int obi_truncate_column(OBIDMS_column_p column)	// TODO is it necessary to unmap
 		return -1;
 
 	// Open the column file
-	column_file_descriptor = openat((column->column_directory)->dir_fd, column_file_name, O_RDWR);
+	column_directory_name = obi_build_column_directory_name(column->column_directory->column_name);
+	if (column_directory_name == NULL)
+	{
+		obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
+		obidebug(1, "\nError opening a column file");
+		free(column_file_name);
+		return -1;
+	}
+	col_dir = opendir_in_dms(column->dms, column_directory_name);
+	if (col_dir == NULL)
+	{
+		obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
+		obidebug(1, "\nError opening a column file");
+		free(column_file_name);
+		free(column_directory_name);
+		return -1;
+	}
+	col_dir_fd = dirfd(col_dir);
+	if (col_dir_fd < 0)
+	{
+		obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
+		obidebug(1, "\nError opening a column file");
+		free(column_file_name);
+		free(column_directory_name);
+		closedir(col_dir);
+		return -1;
+	}
+	column_file_descriptor = openat(col_dir_fd, column_file_name, O_RDWR);
 	if (column_file_descriptor < 0)
 	{
 		obi_set_errno(OBICOL_UNKNOWN_ERROR);
 		obidebug(1, "\nError getting the file descriptor of a column file");
 		free(column_file_name);
+		free(column_directory_name);
+		closedir(col_dir);
 		return -1;
 	}
 
 	free(column_file_name);
+	free(column_directory_name);
+	closedir(col_dir);
 
 	// Unmap the data before truncating the file
 	if (munmap(column->data, (column->header)->data_size) < 0)
@@ -1669,6 +1871,9 @@ int obi_enlarge_column(OBIDMS_column_p column)
 	index_t new_line_count;
 	int 	column_file_descriptor;
 	char* 	column_file_name;
+	char*	column_directory_name;
+	DIR*	col_dir;
+	int		col_dir_fd;
 
 	// Check if the column is read-only
 	if (!(column->writable))
@@ -1686,16 +1891,48 @@ int obi_enlarge_column(OBIDMS_column_p column)
 	}
 
 	// Open the column file
-	column_file_descriptor = openat((column->column_directory)->dir_fd, column_file_name, O_RDWR);
+	column_directory_name = obi_build_column_directory_name(column->column_directory->column_name);
+	if (column_directory_name == NULL)
+	{
+		obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
+		obidebug(1, "\nError opening a column file");
+		free(column_file_name);
+		return -1;
+	}
+	col_dir = opendir_in_dms(column->dms, column_directory_name);
+	if (col_dir == NULL)
+	{
+		obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
+		obidebug(1, "\nError opening a column file");
+		free(column_file_name);
+		free(column_directory_name);
+		return -1;
+	}
+	col_dir_fd = dirfd(col_dir);
+	if (col_dir_fd < 0)
+	{
+		obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
+		obidebug(1, "\nError opening a column file");
+		free(column_file_name);
+		free(column_directory_name);
+		closedir(col_dir);
+		return -1;
+	}
+
+	column_file_descriptor = openat(col_dir_fd, column_file_name, O_RDWR);
 	if (column_file_descriptor < 0)
 	{
 		obi_set_errno(OBICOL_UNKNOWN_ERROR);
 		obidebug(1, "\nError getting the file descriptor of a column file");
 		free(column_file_name);
+		free(column_directory_name);
+		closedir(col_dir);
 		return -1;
 	}
 
 	free(column_file_name);
+	free(column_directory_name);
+	closedir(col_dir);
 
 	// Calculate the new file size
 	old_line_count = (column->header)->line_count;
@@ -1920,6 +2157,9 @@ OBIDMS_column_header_p obi_column_get_header_from_name(OBIDMS_p dms, const char*
 	char* 						column_file_name;
 	int 						column_file_descriptor;
 	size_t 						header_size;
+	char*						column_directory_name;
+	DIR*						col_dir;
+	int							col_dir_fd;
 
 	// Get the column directory structure associated to the column
 	column_directory = obi_open_column_directory(dms, column_name);
@@ -1948,7 +2188,35 @@ OBIDMS_column_header_p obi_column_get_header_from_name(OBIDMS_p dms, const char*
 	}
 
 	// Open the column file
-	column_file_descriptor = openat(column_directory->dir_fd, column_file_name, O_RDWR);
+	column_directory_name = obi_build_column_directory_name(column_name);
+	if (column_directory_name == NULL)
+	{
+		obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
+		obidebug(1, "\nError opening a column file");
+		free(column_file_name);
+		return NULL;
+	}
+
+	col_dir = opendir_in_dms(dms, column_directory_name);
+	if (col_dir == NULL)
+	{
+		obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
+		obidebug(1, "\nError opening a column file");
+		free(column_file_name);
+		free(column_directory_name);
+		return NULL;
+	}
+	col_dir_fd = dirfd(col_dir);
+	if (col_dir_fd < 0)
+	{
+		obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
+		obidebug(1, "\nError opening a column file");
+		free(column_file_name);
+		free(column_directory_name);
+		closedir(col_dir);
+		return NULL;
+	}
+	column_file_descriptor = openat(col_dir_fd, column_file_name, O_RDWR);
 	if (column_file_descriptor < 0)
 	{
 		obidebug(1, "\nError opening a column file");
@@ -1958,6 +2226,8 @@ OBIDMS_column_header_p obi_column_get_header_from_name(OBIDMS_p dms, const char*
 	}
 
 	free(column_file_name);
+	free(column_directory_name);
+	closedir(col_dir);
 
 	// Read the header size
 	if (read(column_file_descriptor, &header_size, sizeof(size_t)) < ((ssize_t) sizeof(size_t)))

src/obidmscolumndir.c

@@ -177,15 +177,13 @@ OBIDMS_column_directory_p obi_open_column_directory(OBIDMS_p dms, const char* co
 	// Initialize the data structure
 	strcpy(column_directory->directory_name, column_directory_name);
 	strcpy(column_directory->column_name, column_name);
-	column_directory->directory = directory;
 
-	column_directory->dir_fd = dirfd(directory);
+	if (closedir(directory) < 0)
-	if (column_directory->dir_fd < 0)
 	{
 		obi_set_errno(OBICOLDIR_MEMORY_ERROR);
-		obidebug(1, "\nError allocating the memory for an OBIDMS column directory structure");
+		obidebug(1, "\nError closing a DIR after opening a column directory");
 		free(column_directory_name);
-		free(column_directory);
+		return NULL;
 	}
 
 	free(column_directory_name);
@@ -215,17 +213,7 @@ OBIDMS_column_directory_p obi_column_directory(OBIDMS_p dms, const char* column_
 int obi_close_column_directory(OBIDMS_column_directory_p column_directory)
 {
 	if (column_directory != NULL)
-	{
-		// Close the column directory
-		if (closedir(column_directory->directory) < 0)
-		{
-			obi_set_errno(OBICOLDIR_MEMORY_ERROR);
-			obidebug(1, "\nError closing a column directory");
-			free(column_directory);
-			return -1;
-		}
 		free(column_directory);
-	}
 
 	return 0;
 }

src/obidmscolumndir.h

@@ -40,12 +40,6 @@ typedef struct OBIDMS_column_directory {
 	char		directory_name[OBIDMS_COLUMN_MAX_NAME+1];	     /**< The name of the directory
 	                                     	 	 	 	 	 	  *    containing the column.
 	                                     	 	 	 	 	 	  */
-	DIR*		directory;										 /**< A directory entry usable to
-										 	 	 	 	 	 	  *    refer and scan the column directory.
-										 	 	 	 	 	 	  */
-	int			dir_fd;											 /**< The file descriptor of the directory entry
-																  *   usable to refer and scan the column directory.
-																  */
 } OBIDMS_column_directory_t, *OBIDMS_column_directory_p;