obitools/obitools3
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 66 Wiki Activity

Closes #10 : column groups stored in directories

Browse Source
This commit is contained in:
Celine Mercier
2015-06-23 18:35:34 +02:00
parent 61b6c3ce83
commit 152b34b5f4
10 changed files with 745 additions and 223 deletions
Download Patch File Download Diff File Expand all files Collapse all files

357
src/obidmscolumn.c
View File

@ -1,5 +1,5 @@
/****************************************************************************
* OBIDMS_column functions *
* OBIDMS_column functions *
****************************************************************************/
/**
@ -13,22 +13,25 @@
#include <stdlib.h>
#include <stdio.h>
#include <sys/types.h>
#include <dirent.h>
#include <unistd.h>
#include <fcntl.h>
#include <stdbool.h>
#include <sys/mman.h> /* mmap() is defined in this header */
#include "obidmscolumn.h"
#include "obidmscolumngroup.h"
#include "obidms.h"
#include "obitypes.h"
#include "obierrno.h"
#include "obilittlebigman.h"
#include "private_openat.h"
#include "private_at_functions.h"
/*************************************************************************
/**************************************************************************
*
* D E C L A R A T I O N O F T H E P R I V A T E F U N C T I O N S
*
*************************************************************************/
**************************************************************************/
/**
@ -44,7 +47,7 @@
* @retvalue NULL if an error occurs
*
* ###Error values
* - OBIDMS_MEMORY_ERROR : something wrong occured during memory allocation.
* - OBIDMS_MEMORY_ERROR : something wrong occurred during memory allocation.
*
* @since May 2015
* @author Eric Coissac (eric.coissac@metabarcoding.org)
@ -90,7 +93,7 @@ static char *build_version_file_name(const char *column_name);
* @since May 2015
* @author Eric Coissac (eric.coissac@metabarcoding.org)
*/
static obiversion_t obi_get_new_version_number(OBIDMS_p dms, char *column_name, bool block);
static obiversion_t obi_get_new_version_number(OBIDMS_column_group_p column_group, bool block);
/**
@ -108,7 +111,7 @@ static obiversion_t obi_get_new_version_number(OBIDMS_p dms, char *column_name,
* @since May 2015
* @author Eric Coissac (eric.coissac@metabarcoding.org)
*/
static int create_version_file(OBIDMS_p dms, char *column_name);
static int create_version_file(OBIDMS_column_group_p column_group);
/************************************************************************
@ -124,7 +127,7 @@ static char *build_column_file_name(const char *column_name, obiversion_t versio
// Build the database directory name
if (asprintf(&filename,"%s@%d.odc", column_name, version_number) < 0)
{
obi_set_errno(OBIDMS_MEMORY_ERROR);
obi_set_errno(OBICOL_MEMORY_ERROR);
return NULL;
}
@ -139,7 +142,7 @@ static char *build_version_file_name(const char *column_name)
// Build the database directory name
if (asprintf(&filename,"%s.odv", column_name) < 0)
{
obi_set_errno(OBIDMS_MEMORY_ERROR);
obi_set_errno(OBICOL_MEMORY_ERROR);
return NULL;
}
@ -147,12 +150,12 @@ static char *build_version_file_name(const char *column_name)
}
static obiversion_t obi_get_new_version_number(OBIDMS_p dms, char *column_name, bool block)
static obiversion_t obi_get_new_version_number(OBIDMS_column_group_p column_group, bool block)
{
off_t loc_size;
obiversion_t new_version_number;
char* version_file_name;
int directory_file_descriptor;
int column_dir_file_descriptor;
int version_file_descriptor;
bool little_endian;
int lock_mode;
@ -166,34 +169,31 @@ static obiversion_t obi_get_new_version_number(OBIDMS_p dms, char *column_name,
else
lock_mode=F_TLOCK;
// build the version file name
version_file_name = build_version_file_name(column_name);
// Build the version file name
version_file_name = build_version_file_name(column_group->column_name);
if (version_file_name == NULL)
return -1;
// Get the file descriptor associated to the database directory
directory_file_descriptor = dirfd(dms->directory);
if (directory_file_descriptor < 0)
// Get the file descriptor associated to the column group directory
column_dir_file_descriptor = dirfd(column_group->directory);
if (column_dir_file_descriptor < 0)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obi_set_errno(OBICOL_UNKNOWN_ERROR);
free(version_file_name);
return -1;
}
//open the version file
version_file_descriptor = private_openat(directory_file_descriptor, version_file_name, O_RDWR);
// Open the version file
version_file_descriptor = private_openat(column_dir_file_descriptor, version_file_name, O_RDWR);
if (version_file_descriptor < 0)
{
free(version_file_name);
//close(column_dir_file_descriptor);
if (errno == ENOENT)
{
return create_version_file(dms, column_name);
}
return create_version_file(column_group);
else
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obi_set_errno(OBICOL_UNKNOWN_ERROR);
return -1;
}
}
@ -201,199 +201,223 @@ static obiversion_t obi_get_new_version_number(OBIDMS_p dms, char *column_name,
// Test if the version file size is ok
if (lseek(version_file_descriptor, 0, SEEK_END) < loc_size)
{
obi_set_errno(OBICOL_UNKNOWN_ERROR);
close(version_file_descriptor);
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
close(column_dir_file_descriptor);
free(version_file_name);
return -1;
}
// prepare the file for locking
// Prepare the file for locking
if (lseek(version_file_descriptor, 0, SEEK_SET) != 0)
{
obi_set_errno(OBICOL_UNKNOWN_ERROR);
close(version_file_descriptor);
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
close(column_dir_file_descriptor);
free(version_file_name);
return -1;
}
// Lock the file
if (lockf(version_file_descriptor, lock_mode, loc_size) < -1)
if (lockf(version_file_descriptor, lock_mode, loc_size) < 0)
{
obi_set_errno(OBICOL_UNKNOWN_ERROR);
close(version_file_descriptor);
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
close(column_dir_file_descriptor);
free(version_file_name);
return -1;
}
// read the endianess of the file
// Read the endianness of the file
if (read(version_file_descriptor, &little_endian, sizeof(bool)) < sizeof(bool))
{
obi_set_errno(OBICOL_UNKNOWN_ERROR);
close(version_file_descriptor);
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
close(column_dir_file_descriptor);
free(version_file_name);
return -1;
}
// check if endianess is correct
// Check if endianness is correct
if (little_endian != obi_is_little_endian())
{
obi_set_errno(OBICOL_BAD_ENDIAN_ERROR);
close(version_file_descriptor);
obi_set_errno(OBIDMS_BAD_ENDIAN_ERROR);
close(column_dir_file_descriptor);
free(version_file_name);
return -1;
}
// read the current version number
// Read the current version number
if (read(version_file_descriptor, &new_version_number, sizeof(obiversion_t)) < sizeof(obiversion_t))
{
obi_set_errno(OBICOL_UNKNOWN_ERROR);
close(version_file_descriptor);
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
close(column_dir_file_descriptor);
free(version_file_name);
return -1;
}
new_version_number++;
// write the new version number
// Write the new version number
if (lseek(version_file_descriptor, sizeof(bool), SEEK_SET) != sizeof(bool))
{
obi_set_errno(OBICOL_UNKNOWN_ERROR);
close(version_file_descriptor);
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
close(column_dir_file_descriptor);
free(version_file_name);
return -1;
}
if (write(version_file_descriptor, &new_version_number, sizeof(obiversion_t)) < sizeof(obiversion_t))
{
obi_set_errno(OBICOL_UNKNOWN_ERROR);
close(version_file_descriptor);
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
close(column_dir_file_descriptor);
free(version_file_name);
return -1;
}
// prepare for the unlocking
// Prepare for unlocking
if (lseek(version_file_descriptor, 0, SEEK_SET) != 0)
{
obi_set_errno(OBICOL_UNKNOWN_ERROR);
close(version_file_descriptor);
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
close(column_dir_file_descriptor);
free(version_file_name);
return -1;
}
// unlock the file
if (lockf(version_file_descriptor, F_ULOCK, loc_size) < -1)
// Unlock the file
if (lockf(version_file_descriptor, F_ULOCK, loc_size) < 0)
{
obi_set_errno(OBICOL_UNKNOWN_ERROR);
close(version_file_descriptor);
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
close(column_dir_file_descriptor);
free(version_file_name);
return -1;
}
close(version_file_descriptor);
//close(version_file_descriptor);
//close(column_dir_file_descriptor);
free(version_file_name);
return new_version_number;
}
static int create_version_file(OBIDMS_p dms, char *column_name)
static int create_version_file(OBIDMS_column_group_p column_group)
{
off_t loc_size;
obiversion_t version_number;
char* version_file_name;
int directory_file_descriptor;
int column_dir_file_descriptor;
int version_file_descriptor;
bool little_endian;
loc_size = sizeof(bool) + sizeof(obiversion_t);
version_number = 0;
version_file_name = build_version_file_name(column_name);
version_file_name = build_version_file_name(column_group->column_name);
if (version_file_name == NULL)
return -1;
directory_file_descriptor = dirfd(dms->directory);
if (directory_file_descriptor < 0)
// Get the file descriptor associated to the column group directory
column_dir_file_descriptor = dirfd(column_group->directory);
if (column_dir_file_descriptor < 0)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obi_set_errno(OBICOL_UNKNOWN_ERROR);
free(version_file_name);
return -1;
}
version_file_descriptor = private_openat(directory_file_descriptor,
version_file_name,
O_RDWR | O_CREAT);
// Get the file descriptor associated to the version file
version_file_descriptor = private_openat(column_dir_file_descriptor, version_file_name, O_RDWR | O_CREAT);
if (version_file_descriptor < 0)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obi_set_errno(OBICOL_UNKNOWN_ERROR);
free(version_file_name);
close(column_dir_file_descriptor);
return -1;
}
// Lock the file
if (lockf(version_file_descriptor, F_LOCK, loc_size) < -1)
if (lockf(version_file_descriptor, F_LOCK, loc_size) < 0)
{
close(version_file_descriptor);
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obi_set_errno(OBICOL_UNKNOWN_ERROR);
free(version_file_name);
close(version_file_descriptor);
close(column_dir_file_descriptor);
return -1;
}
// Truncate the version file to the right size
if (ftruncate(version_file_descriptor, loc_size) < 0)
{
close(version_file_descriptor);
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obi_set_errno(OBICOL_UNKNOWN_ERROR);
free(version_file_name);
close(version_file_descriptor);
close(column_dir_file_descriptor);
return -1;
}
// Position offset to 0 to prepare for writing
if (lseek(version_file_descriptor, 0, SEEK_SET) != 0)
{
close(version_file_descriptor);
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obi_set_errno(OBICOL_UNKNOWN_ERROR);
free(version_file_name);
close(version_file_descriptor);
close(column_dir_file_descriptor);
return -1;
}
little_endian = obi_is_little_endian();
// Write endianness
if (write(version_file_descriptor, &little_endian, sizeof(bool)) < sizeof(bool))
{
close(version_file_descriptor);
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obi_set_errno(OBICOL_UNKNOWN_ERROR);
free(version_file_name);
close(version_file_descriptor);
close(column_dir_file_descriptor);
return -1;
}
// Write version number
if (write(version_file_descriptor, &version_number, sizeof(obiversion_t)) < sizeof(obiversion_t))
{
close(version_file_descriptor);
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obi_set_errno(OBICOL_UNKNOWN_ERROR);
free(version_file_name);
close(version_file_descriptor);
close(column_dir_file_descriptor);
return -1;
}
// prepare for the unlocking
// Prepare for unlocking
if (lseek(version_file_descriptor, 0, SEEK_SET) != 0)
{
close(version_file_descriptor);
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obi_set_errno(OBICOL_UNKNOWN_ERROR);
free(version_file_name);
close(version_file_descriptor);
close(column_dir_file_descriptor);
return -1;
}
// unlock the file
if (lockf(version_file_descriptor, F_ULOCK, loc_size) < -1)
// Unlock the file
if (lockf(version_file_descriptor, F_ULOCK, loc_size) < 0)
{
close(version_file_descriptor);
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obi_set_errno(OBICOL_UNKNOWN_ERROR);
free(version_file_name);
close(version_file_descriptor);
close(column_dir_file_descriptor);
return -1;
}
close(version_file_descriptor);
//close(version_file_descriptor);
//close(column_dir_file_descriptor);
free(version_file_name);
return version_number;
}
@ -404,80 +428,95 @@ static int create_version_file(OBIDMS_p dms, char *column_name)
*
**********************************************************************/
obiversion_t obi_get_latest_version_number(OBIDMS_p dms, char *column_name)
obiversion_t obi_get_latest_version_number(OBIDMS_column_group_p column_group)
{
off_t loc_size;
obiversion_t latest_version_number;
char * version_file_name;
int directory_file_descriptor;
int column_dir_file_descriptor;
int version_file_descriptor;
bool little_endian;
loc_size = sizeof(bool) + sizeof(obiversion_t);
latest_version_number = 0;
version_file_name = build_version_file_name(column_name);
version_file_name = build_version_file_name(column_group->column_name);
if (version_file_name==NULL)
return -1;
directory_file_descriptor = dirfd(dms->directory);
if (directory_file_descriptor < 0) {
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
// Get the file descriptor associated to the column group directory
column_dir_file_descriptor = dirfd(column_group->directory);
if (column_dir_file_descriptor < 0)
{
obi_set_errno(OBICOL_UNKNOWN_ERROR);
free(version_file_name);
return -1;
}
version_file_descriptor = private_openat(directory_file_descriptor, version_file_name, O_RDONLY);
if (version_file_descriptor < 0) {
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
// Get the file descriptor associated to the version file
version_file_descriptor = private_openat(column_dir_file_descriptor, version_file_name, O_RDONLY);
if (version_file_descriptor < 0)
{
obi_set_errno(OBICOL_UNKNOWN_ERROR);
free(version_file_name);
close(column_dir_file_descriptor);
return -1;
}
// Check that the version file size is ok
if (lseek(version_file_descriptor, 0, SEEK_END) < loc_size)
{
close(version_file_descriptor);
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obi_set_errno(OBICOL_UNKNOWN_ERROR);
free(version_file_name);
close(version_file_descriptor);
close(column_dir_file_descriptor);
return -1;
}
// Set the offset to 0 in the version file
if (lseek(version_file_descriptor, 0, SEEK_SET) != 0)
{
close(version_file_descriptor);
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obi_set_errno(OBICOL_UNKNOWN_ERROR);
free(version_file_name);
close(version_file_descriptor);
close(column_dir_file_descriptor);
return -1;
}
// Read the endianness
if (read(version_file_descriptor, &little_endian, sizeof(bool)) < sizeof(bool))
{
close(version_file_descriptor);
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obi_set_errno(OBICOL_UNKNOWN_ERROR);
free(version_file_name);
close(version_file_descriptor);
close(column_dir_file_descriptor);
return -1;
}
// Verify the endianness
if (little_endian != obi_is_little_endian())
{
close(version_file_descriptor);
obi_set_errno(OBIDMS_BAD_ENDIAN_ERROR);
obi_set_errno(OBICOL_BAD_ENDIAN_ERROR);
free(version_file_name);
close(version_file_descriptor);
close(column_dir_file_descriptor);
return -1;
}
// Read the latest version number
if (read(version_file_descriptor, &latest_version_number, sizeof(obiversion_t)) < sizeof(obiversion_t))
{
close(version_file_descriptor);
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obi_set_errno(OBICOL_UNKNOWN_ERROR);
free(version_file_name);
close(version_file_descriptor);
close(column_dir_file_descriptor);
return -1;
}
close(version_file_descriptor);
free(version_file_name);
//close(version_file_descriptor);
//close(column_dir_file_descriptor);
return latest_version_number;
}
@ -489,104 +528,139 @@ size_t obi_get_platform_header_size()
OBIDMS_column_p obi_create_column(OBIDMS_p dms,
char *column_name,
OBIType_t type,
size_t nb_elements)
const char *column_name,
OBIType_t type,
size_t nb_elements)
{
OBIDMS_column_p new_column;
OBIDMS_column_group_p column_group;
OBIDMS_column_header_p header;
size_t file_size;
obiversion_t version_number;
char* column_file_name;
int column_file_descriptor;
int directory_file_descriptor;
int column_dir_file_descriptor;
size_t header_size;
size_t data_size;
new_column = NULL;
directory_file_descriptor = dirfd(dms->directory);
if (directory_file_descriptor < 0) {
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
// Get the column group structure associated to the column
column_group = obi_column_group(dms, column_name);
if (column_group == NULL)
return NULL;
// Get the file descriptor associated to the column group directory
column_dir_file_descriptor = dirfd(column_group->directory);
if (column_dir_file_descriptor < 0)
{
obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
obi_close_column_group(column_group);
return NULL;
}
// Calculate the size needed
header_size = obi_get_platform_header_size();
data_size = obi_array_sizeof(type, nb_elements);
file_size = header_size + data_size;
version_number = obi_get_new_version_number(dms, column_name, true);
// Get the latest version number
version_number = obi_get_new_version_number(column_group, true);
if (version_number < 0)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obi_close_column_group(column_group);
close(column_dir_file_descriptor);
return NULL;
}
// Get the column file name
column_file_name = build_column_file_name(column_name, version_number);
if (column_file_name == NULL)
{
obi_close_column_group(column_group);
close(column_dir_file_descriptor);
return NULL;
}
column_file_descriptor = private_openat(directory_file_descriptor,
column_file_name,
O_RDWR | O_CREAT);
// Open the column file
column_file_descriptor = private_openat(column_dir_file_descriptor, column_file_name, O_RDWR | O_CREAT);
if (column_file_descriptor < 0)
{
obi_set_errno(OBICOL_UNKNOWN_ERROR);
obi_close_column_group(column_group);
close(column_dir_file_descriptor);
free(column_file_name);
return NULL;
}
// Truncate the column file to the right size
if (ftruncate(column_file_descriptor, file_size) < 0)
{
obi_set_errno(OBICOL_UNKNOWN_ERROR);
obi_close_column_group(column_group);
close(column_dir_file_descriptor);
close(column_file_descriptor);
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
free(column_file_name);
return NULL;
}
// Allocate the memory for the column structure
new_column = (OBIDMS_column_p) malloc(sizeof(OBIDMS_column_t));
if (new_column)
if (new_column == NULL)
{
obi_set_errno(OBICOL_UNKNOWN_ERROR);
obi_close_column_group(column_group);
close(column_dir_file_descriptor);
close(column_file_descriptor);
obi_set_errno(OBIDMS_MEMORY_ERROR);
free(column_file_name);
return NULL;
}
new_column->dms = dms;
new_column->header = mmap(NULL,
header_size,
PROT_READ | PROT_WRITE,
MAP_SHARED,
column_file_descriptor,
0
);
// Fill the column structure
new_column->dms = dms;
new_column->column_group = column_group;
new_column->header = mmap(NULL,
header_size,
PROT_READ | PROT_WRITE,
MAP_SHARED,
column_file_descriptor,
0
);
if (new_column->header == MAP_FAILED)
{
obi_set_errno(OBICOL_UNKNOWN_ERROR);
obi_close_column_group(column_group);
close(column_dir_file_descriptor);
close(column_file_descriptor);
obi_set_errno(OBIDMS_MEMORY_ERROR);
free(column_file_name);
free(new_column);
return NULL;
}
new_column->data = mmap(NULL,
data_size,
PROT_READ | PROT_WRITE,
MAP_SHARED,
column_file_descriptor,
header_size
);
new_column->data = mmap(NULL,
data_size,
PROT_READ | PROT_WRITE,
MAP_SHARED,
column_file_descriptor,
header_size
);
if (new_column->data == MAP_FAILED)
{
munmap(new_column->header, header_size);
obi_set_errno(OBICOL_UNKNOWN_ERROR);
obi_close_column_group(column_group);
close(column_dir_file_descriptor);
close(column_file_descriptor);
obi_set_errno(OBIDMS_MEMORY_ERROR);
free(column_file_name);
free(new_column);
return NULL;
}
new_column->writable= true;
new_column->writable = true;
header = new_column->header;
header = new_column->header;
header->little_endian = obi_is_little_endian();
header->header_size = header_size;
header->line_count = nb_elements;
@ -598,14 +672,9 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms,
strncpy(header->name, column_name, OBIDMS_MAX_COLNAME);
free(column_file_name);
//obi_close_column_group(column_group);
//close(column_dir_file_descriptor);
//close(column_file_descriptor);
return new_column;
}
// if( access( fname, F_OK ) != -1 ) {
// // file exists
// } else {
// // file doesn't exist
// }