obitools/obitools3
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 66 Wiki Activity

C: made it so column DIR* are not kept open to handle very large DMS

Browse Source
This commit is contained in:
Celine Mercier
2019-09-04 12:55:21 +02:00
parent f296517716
commit a7fabff1c7
3 changed files with 282 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

288
src/obidmscolumn.c
View File

@ -318,6 +318,9 @@ static obiversion_t obi_get_new_version_number(OBIDMS_column_directory_p column_
char* version_file_name;
int version_file_descriptor;
int lock_mode;
char* column_directory_name;
DIR* col_directory;
int col_dir_fd;
new_version_number = 0;
loc_size = sizeof(obiversion_t);
@ -334,7 +337,34 @@ static obiversion_t obi_get_new_version_number(OBIDMS_column_directory_p column_
return -1;
// Open the version file
version_file_descriptor = openat(column_directory->dir_fd, version_file_name, O_RDWR);
column_directory_name = obi_build_column_directory_name(column_directory->column_name);
if (column_directory_name == NULL)
{
obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
obidebug(1, "\nError opening a version file");
free(version_file_name);
return -1;
}
col_directory = opendir_in_dms(column_directory->dms, column_directory_name);
if (col_directory == NULL)
{
obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
obidebug(1, "\nError opening a version file");
free(version_file_name);
free(column_directory_name);
return -1;
}
col_dir_fd = dirfd(col_directory);
if (col_dir_fd < 0)
{
obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
obidebug(1, "\nError opening a version file");
free(version_file_name);
free(column_directory_name);
closedir(col_directory);
return -1;
}
version_file_descriptor = openat(col_dir_fd, version_file_name, O_RDWR);
if (version_file_descriptor < 0)
{
if (errno == ENOENT)
@ -349,6 +379,8 @@ static obiversion_t obi_get_new_version_number(OBIDMS_column_directory_p column_
}
free(version_file_name);
free(column_directory_name);
closedir(col_directory);
// Test if the version file size is ok
if (lseek(version_file_descriptor, 0, SEEK_END) < loc_size)
@ -441,6 +473,9 @@ static obiversion_t create_version_file(OBIDMS_column_directory_p column_directo
obiversion_t version_number;
char* version_file_name;
int version_file_descriptor;
char* column_directory_name;
DIR* col_directory;
int col_dir_fd;
loc_size = sizeof(obiversion_t);
version_number = 0;
@ -450,7 +485,34 @@ static obiversion_t create_version_file(OBIDMS_column_directory_p column_directo
return -1;
// Get the file descriptor associated to the version file
version_file_descriptor = openat(column_directory->dir_fd, version_file_name, O_RDWR | O_CREAT | O_EXCL, 0777);
column_directory_name = obi_build_column_directory_name(column_directory->column_name);
if (column_directory_name == NULL)
{
obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
obidebug(1, "\nError opening a version file");
free(version_file_name);
return -1;
}
col_directory = opendir_in_dms(column_directory->dms, column_directory_name);
if (col_directory == NULL)
{
obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
obidebug(1, "\nError opening a version file");
free(version_file_name);
free(column_directory_name);
return -1;
}
col_dir_fd = dirfd(col_directory);
if (col_dir_fd < 0)
{
obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
obidebug(1, "\nError opening a version file");
free(version_file_name);
free(column_directory_name);
closedir(col_directory);
return -1;
}
version_file_descriptor = openat(col_dir_fd, version_file_name, O_RDWR | O_CREAT | O_EXCL, 0777);
if (version_file_descriptor < 0)
{
obi_set_errno(OBICOL_UNKNOWN_ERROR);
@ -460,6 +522,8 @@ static obiversion_t create_version_file(OBIDMS_column_directory_p column_directo
}
free(version_file_name);
free(column_directory_name);
closedir(col_directory);
// Lock the file
if (lockf(version_file_descriptor, F_LOCK, loc_size) < 0)
@ -806,6 +870,9 @@ obiversion_t obi_get_latest_version_number(OBIDMS_column_directory_p column_dire
obiversion_t latest_version_number;
char * version_file_name;
int version_file_descriptor;
char* column_directory_name;
DIR* col_directory;
int col_dir_fd;
loc_size = sizeof(obiversion_t);
latest_version_number = 0;
@ -815,7 +882,34 @@ obiversion_t obi_get_latest_version_number(OBIDMS_column_directory_p column_dire
return -1;
// Get the file descriptor associated to the version file
version_file_descriptor = openat(column_directory->dir_fd, version_file_name, O_RDONLY);
column_directory_name = obi_build_column_directory_name(column_directory->column_name);
if (column_directory_name == NULL)
{
obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
obidebug(1, "\nError opening a version file");
free(version_file_name);
return -1;
}
col_directory = opendir_in_dms(column_directory->dms, column_directory_name);
if (col_directory == NULL)
{
obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
obidebug(1, "\nError opening a version file");
free(version_file_name);
free(column_directory_name);
return -1;
}
col_dir_fd = dirfd(col_directory);
if (col_dir_fd < 0)
{
obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
obidebug(1, "\nError opening a version file");
free(version_file_name);
free(column_directory_name);
closedir(col_directory);
return -1;
}
version_file_descriptor = openat(col_dir_fd, version_file_name, O_RDONLY);
if (version_file_descriptor < 0)
{
obi_set_errno(OBICOL_UNKNOWN_ERROR);
@ -825,6 +919,8 @@ obiversion_t obi_get_latest_version_number(OBIDMS_column_directory_p column_dire
}
free(version_file_name);
free(column_directory_name);
closedir(col_directory);
// Check that the version file size is ok
if (lseek(version_file_descriptor, 0, SEEK_END) < loc_size)
@ -939,6 +1035,9 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms,
char* final_indexer_name;
char* built_elements_names = NULL;
int64_t elts_names_length;
char* column_directory_name;
DIR* col_dir;
int col_dir_fd;
new_column = NULL;
@ -1053,16 +1152,48 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms,
return NULL;
// Open the column file
column_file_descriptor = openat(column_directory->dir_fd, column_file_name, O_RDWR | O_CREAT | O_EXCL, 0777);
column_directory_name = obi_build_column_directory_name(column_directory->column_name);
if (column_directory_name == NULL)
{
obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
obidebug(1, "\nError opening a column file");
free(column_file_name);
return NULL;
}
col_dir = opendir_in_dms(dms, column_directory_name);
if (col_dir == NULL)
{
obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
obidebug(1, "\nError opening a version file");
free(column_file_name);
free(column_directory_name);
return NULL;
}
col_dir_fd = dirfd(col_dir);
if (col_dir_fd < 0)
{
obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
obidebug(1, "\nError opening a version file");
free(column_file_name);
free(column_directory_name);
closedir(col_dir);
return NULL;
}
column_file_descriptor = openat(col_dir_fd, column_file_name, O_RDWR | O_CREAT | O_EXCL, 0777);
if (column_file_descriptor < 0)
{
obi_set_errno(OBICOL_UNKNOWN_ERROR);
obidebug(1, "\nError opening a column file %s", column_file_name);
free(column_file_name);
free(column_directory_name);
closedir(col_dir);
return NULL;
}
free(column_file_name);
free(column_directory_name);
closedir(col_dir);
// Truncate the column file to the right size
if (ftruncate(column_file_descriptor, file_size) < 0)
@ -1231,6 +1362,9 @@ OBIDMS_column_p obi_open_column(OBIDMS_p dms,
char* column_file_name;
int column_file_descriptor;
size_t header_size;
char* column_directory_name;
DIR* col_dir;
int col_dir_fd;
column = NULL;
@ -1276,16 +1410,50 @@ OBIDMS_column_p obi_open_column(OBIDMS_p dms,
}
// Open the column file, ALWAYS READ-ONLY
column_file_descriptor = openat(column_directory->dir_fd, column_file_name, O_RDWR);
column_directory_name = obi_build_column_directory_name(column_directory->column_name);
if (column_directory_name == NULL)
{
obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
obidebug(1, "\nError opening a column file");
free(column_file_name);
return NULL;
}
col_dir = opendir_in_dms(dms, column_directory_name);
if (col_dir == NULL)
{
obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
obidebug(1, "\nError opening a version file");
free(column_file_name);
free(column_directory_name);
return NULL;
}
col_dir_fd = dirfd(col_dir);
if (col_dir_fd < 0)
{
obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
obidebug(1, "\nError opening a version file");
free(column_file_name);
free(column_directory_name);
closedir(col_dir);
return NULL;
}
column_file_descriptor = openat(col_dir_fd, column_file_name, O_RDWR);
if (column_file_descriptor < 0)
{
obi_set_errno(OBICOL_UNKNOWN_ERROR);
obidebug(1, "\nError opening column file %s", column_file_name);
free(column_file_name);
free(column_directory_name);
closedir(col_dir);
return NULL;
}
free(column_file_name);
free(column_directory_name);
closedir(col_dir);
// Allocate the memory for the column structure
column = (OBIDMS_column_p) malloc(sizeof(OBIDMS_column_t));
@ -1557,7 +1725,7 @@ int obi_clone_column_indexer(OBIDMS_column_p column)
int obi_truncate_column(OBIDMS_column_p column) // TODO is it necessary to unmap/remap?
int obi_truncate_column(OBIDMS_column_p column) // TODO is it necessary to unmap/remap? -> yes the whole file for windows WLS
{
size_t file_size;
size_t data_size;
@ -1565,6 +1733,9 @@ int obi_truncate_column(OBIDMS_column_p column) // TODO is it necessary to unmap
double multiple;
int column_file_descriptor;
char* column_file_name;
char* column_directory_name;
DIR* col_dir;
int col_dir_fd;
// Check if the column is read-only
if (!(column->writable))
@ -1596,16 +1767,47 @@ int obi_truncate_column(OBIDMS_column_p column) // TODO is it necessary to unmap
return -1;
// Open the column file
column_file_descriptor = openat((column->column_directory)->dir_fd, column_file_name, O_RDWR);
column_directory_name = obi_build_column_directory_name(column->column_directory->column_name);
if (column_directory_name == NULL)
{
obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
obidebug(1, "\nError opening a column file");
free(column_file_name);
return -1;
}
col_dir = opendir_in_dms(column->dms, column_directory_name);
if (col_dir == NULL)
{
obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
obidebug(1, "\nError opening a column file");
free(column_file_name);
free(column_directory_name);
return -1;
}
col_dir_fd = dirfd(col_dir);
if (col_dir_fd < 0)
{
obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
obidebug(1, "\nError opening a column file");
free(column_file_name);
free(column_directory_name);
closedir(col_dir);
return -1;
}
column_file_descriptor = openat(col_dir_fd, column_file_name, O_RDWR);
if (column_file_descriptor < 0)
{
obi_set_errno(OBICOL_UNKNOWN_ERROR);
obidebug(1, "\nError getting the file descriptor of a column file");
free(column_file_name);
free(column_directory_name);
closedir(col_dir);
return -1;
}
free(column_file_name);
free(column_directory_name);
closedir(col_dir);
// Unmap the data before truncating the file
if (munmap(column->data, (column->header)->data_size) < 0)
@ -1669,6 +1871,9 @@ int obi_enlarge_column(OBIDMS_column_p column)
index_t new_line_count;
int column_file_descriptor;
char* column_file_name;
char* column_directory_name;
DIR* col_dir;
int col_dir_fd;
// Check if the column is read-only
if (!(column->writable))
@ -1686,16 +1891,48 @@ int obi_enlarge_column(OBIDMS_column_p column)
}
// Open the column file
column_file_descriptor = openat((column->column_directory)->dir_fd, column_file_name, O_RDWR);
column_directory_name = obi_build_column_directory_name(column->column_directory->column_name);
if (column_directory_name == NULL)
{
obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
obidebug(1, "\nError opening a column file");
free(column_file_name);
return -1;
}
col_dir = opendir_in_dms(column->dms, column_directory_name);
if (col_dir == NULL)
{
obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
obidebug(1, "\nError opening a column file");
free(column_file_name);
free(column_directory_name);
return -1;
}
col_dir_fd = dirfd(col_dir);
if (col_dir_fd < 0)
{
obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
obidebug(1, "\nError opening a column file");
free(column_file_name);
free(column_directory_name);
closedir(col_dir);
return -1;
}
column_file_descriptor = openat(col_dir_fd, column_file_name, O_RDWR);
if (column_file_descriptor < 0)
{
obi_set_errno(OBICOL_UNKNOWN_ERROR);
obidebug(1, "\nError getting the file descriptor of a column file");
free(column_file_name);
free(column_directory_name);
closedir(col_dir);
return -1;
}
free(column_file_name);
free(column_directory_name);
closedir(col_dir);
// Calculate the new file size
old_line_count = (column->header)->line_count;
@ -1920,6 +2157,9 @@ OBIDMS_column_header_p obi_column_get_header_from_name(OBIDMS_p dms, const char*
char* column_file_name;
int column_file_descriptor;
size_t header_size;
char* column_directory_name;
DIR* col_dir;
int col_dir_fd;
// Get the column directory structure associated to the column
column_directory = obi_open_column_directory(dms, column_name);
@ -1948,7 +2188,35 @@ OBIDMS_column_header_p obi_column_get_header_from_name(OBIDMS_p dms, const char*
}
// Open the column file
column_file_descriptor = openat(column_directory->dir_fd, column_file_name, O_RDWR);
column_directory_name = obi_build_column_directory_name(column_name);
if (column_directory_name == NULL)
{
obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
obidebug(1, "\nError opening a column file");
free(column_file_name);
return NULL;
}
col_dir = opendir_in_dms(dms, column_directory_name);
if (col_dir == NULL)
{
obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
obidebug(1, "\nError opening a column file");
free(column_file_name);
free(column_directory_name);
return NULL;
}
col_dir_fd = dirfd(col_dir);
if (col_dir_fd < 0)
{
obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
obidebug(1, "\nError opening a column file");
free(column_file_name);
free(column_directory_name);
closedir(col_dir);
return NULL;
}
column_file_descriptor = openat(col_dir_fd, column_file_name, O_RDWR);
if (column_file_descriptor < 0)
{
obidebug(1, "\nError opening a column file");
@ -1958,6 +2226,8 @@ OBIDMS_column_header_p obi_column_get_header_from_name(OBIDMS_p dms, const char*
}
free(column_file_name);
free(column_directory_name);
closedir(col_dir);
// Read the header size
if (read(column_file_descriptor, &header_size, sizeof(size_t)) < ((ssize_t) sizeof(size_t)))

18
src/obidmscolumndir.c
View File

@ -177,15 +177,13 @@ OBIDMS_column_directory_p obi_open_column_directory(OBIDMS_p dms, const char* co
// Initialize the data structure
strcpy(column_directory->directory_name, column_directory_name);
strcpy(column_directory->column_name, column_name);
column_directory->directory = directory;
column_directory->dir_fd = dirfd(directory);
if (column_directory->dir_fd < 0)
if (closedir(directory) < 0)
{
obi_set_errno(OBICOLDIR_MEMORY_ERROR);
obidebug(1, "\nError allocating the memory for an OBIDMS column directory structure");
obidebug(1, "\nError closing a DIR after opening a column directory");
free(column_directory_name);
free(column_directory);
return NULL;
}
free(column_directory_name);
@ -215,17 +213,7 @@ OBIDMS_column_directory_p obi_column_directory(OBIDMS_p dms, const char* column_
int obi_close_column_directory(OBIDMS_column_directory_p column_directory)
{
if (column_directory != NULL)
{
// Close the column directory
if (closedir(column_directory->directory) < 0)
{
obi_set_errno(OBICOLDIR_MEMORY_ERROR);
obidebug(1, "\nError closing a column directory");
free(column_directory);
return -1;
}
free(column_directory);
}
return 0;
}

6
src/obidmscolumndir.h
View File

@ -40,12 +40,6 @@ typedef struct OBIDMS_column_directory {
char directory_name[OBIDMS_COLUMN_MAX_NAME+1]; /**< The name of the directory
* containing the column.
*/
DIR* directory; /**< A directory entry usable to
* refer and scan the column directory.
*/
int dir_fd; /**< The file descriptor of the directory entry
* usable to refer and scan the column directory.
*/
} OBIDMS_column_directory_t, *OBIDMS_column_directory_p;