obitools3/src/obidms.c

/********************************************************************
 * OBIDMS functions                                                 *
 ********************************************************************/

/**
 * @file obidms.c
 * @author Eric Coissac (eric.coissac@metabarcoding.org)
 * @date 23 May 2015
 * @brief OBIDMS functions.
 */


#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <stdbool.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <sys/types.h>
#include <dirent.h>
#include <unistd.h>

#include "obidms.h"
#include "obierrno.h"
#include "obidebug.h"
#include "obidmscolumn.h"
#include "obiblob_indexer.h"
#include "utils.h"
#include "obilittlebigman.h"


#define DEBUG_LEVEL 0	// TODO has to be defined somewhere else (cython compil flag?)


/**************************************************************************
 *
 * D E C L A R A T I O N   O F   T H E   P R I V A T E   F U N C T I O N S
 *
 **************************************************************************/


/**
 * Internal function building the OBIDMS directory name from an OBIDMS name.
 *
 * The function builds the directory name corresponding to an OBIDMS.
 * It also checks that the name is not too long.
 *
 * @warning The returned pointer has to be freed by the caller.
 *
 * @param dms_name The name of the OBIDMS.
 *
 * @returns A pointer to the directory name.
 * @retval NULL if an error occurred.
 *
 * @since May 2015
 * @author Eric Coissac (eric.coissac@metabarcoding.org)
 */
static char* build_directory_name(const char* dms_name);


/**
 * Internal function building the informations file name from an OBIDMS name.
 *
 * The function builds the file name for the informations file of an OBIDMS.
 *
 * @warning The returned pointer has to be freed by the caller.
 *
 * @param dms_name The name of the OBIDMS.
 *
 * @returns A pointer to the file name.
 * @retval NULL if an error occurred.
 *
 * @since November 2015
 * @author Celine Mercier (celine.mercier@metabarcoding.org)
 */
static char* build_infos_file_name(const char* dms_name);


/**
 * Internal function creating the file containing basic informations on the OBIDMS.
 *
 * This file contains:
 * 		- The endianness of the platform
 *
 * @warning The returned pointer has to be freed by the caller.
 *
 * @param dms_file_descriptor The file descriptor for the OBIDMS directory.
 * @param dms_name The name of the OBIDMS.
 *
 * @retval 0 if the operation was successfully completed.
 * @retval -1 if an error occurred.
 *
 * @since November 2015
 * @author Celine Mercier (celine.mercier@metabarcoding.org)
 */
int create_dms_infos_file(int dms_file_descriptor, const char* dms_name);


/************************************************************************
 *
 * D E F I N I T I O N   O F   T H E   P R I V A T E   F U N C T I O N S
 *
 ************************************************************************/

static char* build_directory_name(const char* dms_name)
{
	char* directory_name;

	// Build the database directory name
	directory_name = (char*) malloc((strlen(dms_name) + 8)*sizeof(char));
	if (sprintf(directory_name, "%s.obidms", dms_name) < 0)
	{
		obi_set_errno(OBIDMS_MEMORY_ERROR);
		obidebug(1, "\nProblem building an OBIDMS directory name");
		return NULL;
	}

	// Test if the database name is not too long
	if (strlen(directory_name) >= OBIDMS_MAX_NAME)
	{
		obi_set_errno(OBIDMS_LONG_NAME_ERROR);
		obidebug(1, "\nProblem building an OBIDMS directory name");
		free(directory_name);
		return NULL;
	}

	return directory_name;
}


static char* build_infos_file_name(const char* dms_name)
{
	char* file_name;

	// Build file name
	file_name = (char*) malloc((strlen(dms_name) + 7)*sizeof(char));
	if (sprintf(file_name, "%s_infos", dms_name) < 0)
	{
		obi_set_errno(OBIDMS_MEMORY_ERROR);
		obidebug(1, "\nProblem building an informations file name");
		return NULL;
	}

	return file_name;
}


int create_dms_infos_file(int dms_file_descriptor, const char* dms_name)
{
	char* 	file_name;
	int 	infos_file_descriptor;
	off_t 	file_size;
	bool	little_endian;

	file_size = sizeof(bool);

	// Create file name
	file_name = build_infos_file_name(dms_name);
	if (file_name == NULL)
		return -1;

    // Create file
	infos_file_descriptor = openat(dms_file_descriptor, file_name, O_RDWR | O_CREAT | O_EXCL, 0777);
	if (infos_file_descriptor < 0)
	{
		obi_set_errno(OBIDMS_UNKNOWN_ERROR);
		obidebug(1, "\nError creating an informations file");
		free(file_name);
		return -1;
	}

	free(file_name);

	// Truncate the infos file to the right size
	if (ftruncate(infos_file_descriptor, file_size) < 0)
	{
		obi_set_errno(OBIDMS_UNKNOWN_ERROR);
		obidebug(1, "\nError truncating an informations file");
		close(infos_file_descriptor);
		return -1;
	}

	// Write endianness
	little_endian = obi_is_little_endian();
	if (write(infos_file_descriptor, &little_endian, sizeof(bool)) < ((ssize_t) sizeof(bool)))
	{
		obi_set_errno(OBIDMS_UNKNOWN_ERROR);
		obidebug(1, "\nError writing the endianness in an informations file");
		close(infos_file_descriptor);
		return -1;
	}

	// Close file
	if (close(infos_file_descriptor) < 0)
	{
		obi_set_errno(OBIDMS_UNKNOWN_ERROR);
		obidebug(1, "\nError closing a DMS information file");
		return -1;
	}


	return 0;
}


/**********************************************************************
 *
 * D E F I N I T I O N   O F   T H E   P U B L I C   F U N C T I O N S
 *
 **********************************************************************/

int obi_dms_exists(const char* dms_path)
{
    struct stat buffer;
	char*       directory_name;
    int 		check_dir;

	// Build and check the directory name
    directory_name = build_directory_name(dms_path);
	if (directory_name == NULL)
		return -1;

	check_dir = stat(directory_name, &buffer);

	free(directory_name);

    if (check_dir == 0)
        return 1;
    else
    	return 0;
}


OBIDMS_p obi_create_dms(const char* dms_path)
{
	char*  directory_name;
	DIR*   dms_dir;
	int    dms_file_descriptor;
	size_t i, j;

	// Build and check the directory name
	directory_name = build_directory_name(dms_path);
	if (directory_name == NULL)
		return NULL;

	// Try to create the directory
	if (mkdir(directory_name, 00777) < 0)
	{
		if (errno == EEXIST)
		{
			obi_set_errno(OBIDMS_EXIST_ERROR);
			obidebug(1, "\nAn OBIDMS directory with the same name already exists in this directory.");
		}
		else
			obi_set_errno(OBIDMS_UNKNOWN_ERROR);
		obidebug(1, "\nProblem creating an OBIDMS directory");
		free(directory_name);
		return NULL;
	}

	// Get file descriptor of DMS directory to create other directories
	dms_dir = opendir(directory_name);
	if (dms_dir == NULL)
	{
		obi_set_errno(OBIDMS_UNKNOWN_ERROR);
		obidebug(1, "\nProblem opening a newly created OBIDMS directory");
		free(directory_name);
		return NULL;
	}

	free(directory_name);

	dms_file_descriptor = dirfd(dms_dir);
	if (dms_file_descriptor < 0)
	{
		obi_set_errno(OBIDMS_UNKNOWN_ERROR);
		obidebug(1, "\nProblem getting the file descriptor of a newly created OBIDMS directory");
		return NULL;
	}

	// Create the indexer directory
	if (mkdirat(dms_file_descriptor, INDEXER_DIR_NAME, 00777) < 0)
	{
		obi_set_errno(OBI_INDEXER_ERROR);
		obidebug(1, "\nProblem creating the indexer directory");
		return NULL;
	}

	// Create the view directory
	if (mkdirat(dms_file_descriptor, VIEW_DIR_NAME, 00777) < 0)
	{
		obi_set_errno(OBIVIEW_ERROR);
		obidebug(1, "\nProblem creating the view directory");
		return NULL;
	}

	// Create the taxonomy directory
	if (mkdirat(dms_file_descriptor, TAXONOMY_DIR_NAME, 00777) < 0)
	{
		obi_set_errno(OBIVIEW_ERROR);
		obidebug(1, "\nProblem creating the taxonomy directory");
		return NULL;
	}

	// Isolate the dms name
	j = 0;
	for (i=0; i<strlen(dms_path); i++)
	{
		if (dms_path[i] == '/')
			j = i+1;
		i++;
	}

	// Create the informations file
	if (create_dms_infos_file(dms_file_descriptor, dms_path+j) < 0)
		return NULL;

	return obi_open_dms(dms_path);
}


OBIDMS_p obi_open_dms(const char* dms_path)
{
	OBIDMS_p	dms;
	char*	 	complete_dms_path;
	char* 		infos_file_name;
	int			infos_file_descriptor;
	bool		little_endian_dms;
	bool 		little_endian_platform;
	size_t		i, j;

	dms = NULL;

	// Allocate the data structure
	dms = (OBIDMS_p) malloc(sizeof(OBIDMS_t));
	if (dms == NULL)
	{
		obi_set_errno(OBIDMS_MEMORY_ERROR);
		obidebug(1, "\nError allocating the memory for the OBIDMS structure");
		return NULL;
	}

	// Isolate and store the dms name
	j = 0;
	for (i=0; i<strlen(dms_path); i++)
	{
		if (dms_path[i] == '/')
			j = i+1;
		i++;
	}
	strcpy(dms->dms_name, dms_path+j);

	// Build and check the directory name including the relative path
	complete_dms_path = build_directory_name(dms_path);
	if (complete_dms_path == NULL)
	{
		free(dms);
		return NULL;
	}

	// Get and store the absolute path to the DMS directory
	if (realpath(complete_dms_path, dms->directory_path) == NULL)
	{
		obi_set_errno(OBIDMS_UNKNOWN_ERROR);
		obidebug(1, "\nError getting the absolute path to the DMS directory");
		free(complete_dms_path);
		return NULL;
	}

	free(complete_dms_path);

	// Try to open the directory
	dms->directory = opendir(dms->directory_path);
	if (dms->directory == NULL)
	{
		switch (errno)
		{
			case ENOENT:
				obi_set_errno(OBIDMS_NOT_EXIST_ERROR);
				break;

			case EACCES:
				obi_set_errno(OBIDMS_ACCESS_ERROR);
				break;

			case ENOMEM:
				obi_set_errno(OBIDMS_MEMORY_ERROR);
				break;

			default:
				obi_set_errno(OBIDMS_UNKNOWN_ERROR);
		}
		obidebug(1, "\nCan't open OBIDMS directory");
		free(dms);
		return NULL;
	}

	// Get and store file descriptor of DMS directory to open the informations file
	dms->dir_fd = dirfd(dms->directory);
	if (dms->dir_fd < 0)
	{
		obi_set_errno(OBIDMS_UNKNOWN_ERROR);
		obidebug(1, "\nError getting the file descriptor for a newly created OBIDMS directory");
		closedir(dms->directory);
		free(dms);
		return NULL;
	}

	// Open informations file to check endianness
	infos_file_name = build_infos_file_name(dms->dms_name);
	infos_file_descriptor = openat(dms->dir_fd, infos_file_name, O_RDONLY, 0777);
	if (infos_file_descriptor < 0)
	{
		obi_set_errno(OBIDMS_UNKNOWN_ERROR);
		obidebug(1, "\nError opening an informations file");
		closedir(dms->directory);
		free(dms);
		return NULL;
	}

	free(infos_file_name);

	// Check endianness of the platform and DMS
	little_endian_platform = obi_is_little_endian();
	if (read(infos_file_descriptor, &little_endian_dms, sizeof(bool)) < ((ssize_t) sizeof(bool)))
	{
		obi_set_errno(OBIDMS_UNKNOWN_ERROR);
		obidebug(1, "\nError reading the endianness in an informations file");
		close(infos_file_descriptor);
		closedir(dms->directory);
		free(dms);
		return NULL;
	}
	if (little_endian_platform != little_endian_dms)
	{
		obi_set_errno(OBIDMS_BAD_ENDIAN_ERROR);
		obidebug(1, "\nError: The DMS and the platform have different endianness");
		close(infos_file_descriptor);
		closedir(dms->directory);
		free(dms);
		return NULL;
	}

	if (close(infos_file_descriptor) < 0)
	{
		obi_set_errno(OBIDMS_UNKNOWN_ERROR);
		obidebug(1, "\nError closing a DMS information file");
		return NULL;
	}

	dms->little_endian = little_endian_dms;

	// Open the indexer directory
	dms->indexer_directory = opendir_in_dms(dms, INDEXER_DIR_NAME);
	if (dms->indexer_directory == NULL)
	{
		obi_set_errno(OBIDMS_UNKNOWN_ERROR);
		obidebug(1, "\nError opening the indexer directory");
		closedir(dms->directory);
		free(dms);
		return NULL;
	}

	// Store the indexer directory's file descriptor
	dms->indexer_dir_fd = dirfd(dms->indexer_directory);
	if (dms->indexer_dir_fd < 0)
	{
		obi_set_errno(OBIDMS_UNKNOWN_ERROR);
		obidebug(1, "\nError getting the file descriptor of the indexer directory");
		closedir(dms->indexer_directory);
		closedir(dms->directory);
		free(dms);
		return NULL;
	}

	// Open the view directory
	dms->view_directory = opendir_in_dms(dms, VIEW_DIR_NAME);
	if (dms->view_directory == NULL)
	{
		obi_set_errno(OBIDMS_UNKNOWN_ERROR);
		obidebug(1, "\nError opening the view directory");
		closedir(dms->indexer_directory);
		closedir(dms->directory);
		free(dms);
		return NULL;
	}

	// Store the view directory's file descriptor
	dms->view_dir_fd = dirfd(dms->view_directory);
	if (dms->view_dir_fd < 0)
	{
		obi_set_errno(OBIDMS_UNKNOWN_ERROR);
		obidebug(1, "\nError getting the file descriptor of the view directory");
		closedir(dms->view_directory);
		closedir(dms->directory);
		free(dms);
		return NULL;
	}

	// Open the taxonomy directory
	dms->tax_directory = opendir_in_dms(dms, TAXONOMY_DIR_NAME);
	if (dms->tax_directory == NULL)
	{
		obi_set_errno(OBIDMS_UNKNOWN_ERROR);
		obidebug(1, "\nError opening the taxonomy directory");
		closedir(dms->indexer_directory);
		closedir(dms->view_directory);
		closedir(dms->directory);
		free(dms);
		return NULL;
	}

	// Store the taxonomy directory's file descriptor
	dms->tax_dir_fd = dirfd(dms->tax_directory);
	if (dms->tax_dir_fd < 0)
	{
		obi_set_errno(OBIDMS_UNKNOWN_ERROR);
		obidebug(1, "\nError getting the file descriptor of the taxonomy directory");
		closedir(dms->indexer_directory);
		closedir(dms->tax_directory);
		closedir(dms->view_directory);
		closedir(dms->directory);
		free(dms);
		return NULL;
	}

	// Initialize the list of opened columns
	dms->opened_columns = (Opened_columns_list_p) malloc(sizeof(Opened_columns_list_t));
	(dms->opened_columns)->nb_opened_columns = 0;

	// Initialize the list of opened indexers
	dms->opened_indexers = (Opened_indexers_list_p) malloc(sizeof(Opened_indexers_list_t));
	(dms->opened_indexers)->nb_opened_indexers = 0;

	return dms;
}


OBIDMS_p obi_dms(const char* dms_name)
{
	int exists;

	exists = obi_dms_exists(dms_name);

	switch (exists)
	{
		case 0:
			return obi_create_dms(dms_name);
		case 1:
			return obi_open_dms(dms_name);
	};

	obidebug(1, "\nError checking if an OBIDMS directory exists");
	return NULL;
}


int obi_close_dms(OBIDMS_p dms)
{
	if (dms != NULL)
	{
		// Close all columns
		while ((dms->opened_columns)->nb_opened_columns > 0)
			obi_close_column(*((dms->opened_columns)->columns));

		// Close dms, and view, indexer and taxonomy directories
		if (closedir(dms->indexer_directory) < 0)
		{
			obi_set_errno(OBI_INDEXER_ERROR);
			obidebug(1, "\nError closing an indexer directory");
			free(dms);
			return -1;
		}
		if (closedir(dms->view_directory) < 0)
		{
			obi_set_errno(OBIVIEW_ERROR);
			obidebug(1, "\nError closing a view directory");
			free(dms);
			return -1;
		}
		if (closedir(dms->tax_directory) < 0)
		{
			obi_set_errno(OBIVIEW_ERROR);
			obidebug(1, "\nError closing a taxonomy directory");
			free(dms);
			return -1;
		}
		if (closedir(dms->directory) < 0)
		{
			obi_set_errno(OBIDMS_MEMORY_ERROR);
			obidebug(1, "\nError closing an OBIDMS directory");
			free(dms);
			return -1;
		}
		free(dms);
	}

	return 0;
}


OBIDMS_column_p obi_dms_get_column_from_list(OBIDMS_p dms, const char* column_name, obiversion_t version)
{
	int i;

	for (i=0; i < ((dms->opened_columns)->nb_opened_columns); i++)
	{
		if (!strcmp(((*(((dms->opened_columns)->columns)+i))->header)->name, column_name)
				&& (((*(((dms->opened_columns)->columns)+i))->header)->version == version))
		{	// Found the column already opened, return it
			return *(((dms->opened_columns)->columns)+i);
		}
	}
	// Didn't find the column
	return NULL;
}


void obi_dms_list_column(OBIDMS_p dms, OBIDMS_column_p column)	// TODO add check if column already in list?
{
	*(((dms->opened_columns)->columns)+((dms->opened_columns)->nb_opened_columns)) = column;
	((dms->opened_columns)->nb_opened_columns)++;
}


int obi_dms_unlist_column(OBIDMS_p dms, OBIDMS_column_p column)
{
	int i;
	Opened_columns_list_p columns_list;

	columns_list = dms->opened_columns;

	for (i=0; i < columns_list->nb_opened_columns; i++)
	{
		if (!strcmp(((*((columns_list->columns)+i))->header)->name, (column->header)->name)
				&& (((*((columns_list->columns)+i))->header)->version == (column->header)->version))
		{	// Found the column. Rearrange list
			(columns_list->nb_opened_columns)--;
			(columns_list->columns)[i] = (columns_list->columns)[columns_list->nb_opened_columns];
			return 0;
		}
	}

	obidebug(1, "\nCould not find the column to delete from list of open columns");
	return -1;
}


Obi_indexer_p obi_dms_get_indexer_from_list(OBIDMS_p dms, const char* indexer_name)
{
	int i;
	Opened_indexers_list_p indexers_list;

	indexers_list = dms->opened_indexers;

	for (i=0; i < (indexers_list->nb_opened_indexers); i++)
	{
		if (!strcmp(obi_indexer_get_name((indexers_list->indexers)[i]), indexer_name))
		{	// Found the indexer already opened, return it
			return (indexers_list->indexers)[i];
		}
	}
	// Didn't find the indexer
	return NULL;
}


void obi_dms_list_indexer(OBIDMS_p dms, Obi_indexer_p indexer)  	// TODO add check if indexer already in list?
{
	*(((dms->opened_indexers)->indexers)+((dms->opened_indexers)->nb_opened_indexers)) = indexer;
	((dms->opened_indexers)->nb_opened_indexers)++;
}


int obi_dms_unlist_indexer(OBIDMS_p dms, Obi_indexer_p indexer)
{
	int i;
	Opened_indexers_list_p indexers_list;

	indexers_list = dms->opened_indexers;

	for (i=0; i < indexers_list->nb_opened_indexers; i++)
	{
		if (!strcmp(obi_indexer_get_name((indexers_list->indexers)[i]), indexer->name))
		{	// Found the indexer. Rearrange list
			(indexers_list->nb_opened_indexers)--;
			(indexers_list->indexers)[i] = (indexers_list->indexers)[indexers_list->nb_opened_indexers];
			return 0;
		}
	}

	obidebug(1, "\nCould not find the indexer to delete from list of open indexers");
	return -1;
}


char* obi_dms_get_dms_path(OBIDMS_p dms)
{
	char* full_path;

	full_path = (char*) malloc((MAX_PATH_LEN)*sizeof(char));
	if (full_path == NULL)
	{
		obi_set_errno(OBI_MALLOC_ERROR);
		obidebug(1, "\nError allocating memory for the char* path to a file or directory");
		return NULL;
	}

	strcpy(full_path, dms->directory_path);

	return full_path;
}


char* obi_dms_get_full_path(OBIDMS_p dms, const char* path_name)
{
	char* full_path;

	full_path = obi_dms_get_dms_path(dms);
	strcat(full_path, "/");
	strcat(full_path, path_name);

	return full_path;
}


DIR* opendir_in_dms(OBIDMS_p dms, const char* path_name)
{
	char* full_path;
	DIR* directory;

	full_path = obi_dms_get_full_path(dms, path_name);
	if (full_path == NULL)
		return NULL;

	directory = opendir(full_path);
	if (directory == NULL)
	{
		obi_set_errno(OBI_UTILS_ERROR);
		obidebug(1, "\nError opening a directory");
	}

	free(full_path);

	return directory;
}