obitools/obitools3
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 66 Wiki Activity

DMS are now locked when used by a command. Added checks and changed

Browse Source 
cleaning mechanisms.
This commit is contained in:
Celine Mercier
2019-09-20 20:37:19 +02:00
parent eb6c59dc1e
commit 783a1343c4
14 changed files with 378 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
python/obitools3/dms/capi/obidms.pxd
View File

@ -9,6 +9,7 @@ cdef extern from "obidms.h" nogil:
bint little_endian
size_t file_size
size_t used_size
bint working
const_char_p comments
ctypedef OBIDMS_infos_t* OBIDMS_infos_p
@ -21,9 +22,10 @@ cdef extern from "obidms.h" nogil:
ctypedef OBIDMS_t* OBIDMS_p
int obi_dms_is_clean(OBIDMS_p dms)
int obi_clean_dms(const_char_p dms_path)
OBIDMS_p obi_dms(const_char_p dms_name)
OBIDMS_p obi_open_dms(const_char_p dms_path)
OBIDMS_p obi_open_dms(const_char_p dms_path, bint cleaning)
OBIDMS_p obi_test_open_dms(const_char_p dms_path)
OBIDMS_p obi_create_dms(const_char_p dms_path)
int obi_dms_exists(const char* dms_path)

2
python/obitools3/dms/capi/obierrno.pxd
View File

@ -7,3 +7,5 @@ cdef extern from "obierrno.h":
extern int OBI_LINE_IDX_ERROR
extern int OBI_ELT_IDX_ERROR
extern int OBIVIEW_ALREADY_EXISTS_ERROR
extern int OBIDMS_NOT_CLEAN
extern int OBIDMS_WORKING

4
python/obitools3/dms/dms.pyx
View File

@ -50,7 +50,7 @@ cdef class DMS(OBIWrapper):
cdef DMS dms
cdef bytes dms_name_b = tobytes(dms_name)
if DMS.exists(dms_name_b) :
pointer = obi_open_dms(<const_char_p> dms_name_b)
pointer = obi_open_dms(<const_char_p> dms_name_b, False)
else :
pointer = obi_create_dms(<const_char_p> dms_name_b)
if pointer == NULL :
@ -87,7 +87,7 @@ cdef class DMS(OBIWrapper):
cdef OBIDMS_p pointer
cdef DMS dms
cdef bytes dms_name_b = tobytes(dms_name)
pointer = obi_open_dms(<const_char_p> dms_name_b)
pointer = obi_open_dms(<const_char_p> dms_name_b, False)
if pointer == NULL :
raise Exception("Failed opening an OBIDMS")
dms = OBIWrapper.new_wrapper(DMS, pointer)

2
src/build_reference_db.c
View File

@ -212,7 +212,7 @@ int build_reference_db(const char* dms_name,
// Clone the matrix view
// Open the DMS
dms = obi_open_dms(dms_name);
dms = obi_open_dms(dms_name, false);
if (dms == NULL)
{
obidebug(1, "\nError opening the DMS when building a reference database");

4
src/obi_ecopcr.c
View File

@ -749,7 +749,7 @@ int obi_ecopcr(const char* i_dms_name,
tm = (tm1 < tm2) ? tm1:tm2;
// Open input DMS
i_dms = obi_open_dms(i_dms_name);
i_dms = obi_open_dms(i_dms_name, false);
if (i_dms == NULL)
{
obidebug(1, "\nError opening the input DMS");
@ -765,7 +765,7 @@ int obi_ecopcr(const char* i_dms_name,
}
// Open output DMS
o_dms = obi_open_dms(o_dms_name);
o_dms = obi_open_dms(o_dms_name, false);
if (o_dms == NULL)
{
obidebug(1, "\nError opening the output DMS");

6
src/obi_ecotag.c
View File

@ -274,7 +274,7 @@ int obi_ecotag(const char* dms_name,
best_match_ids_buffer_size = 1024;
// Open main DMS containing the query sequences and where the output will be written
dms = obi_open_dms(dms_name);
dms = obi_open_dms(dms_name, false);
if (dms == NULL)
{
obidebug(1, "\nError opening the DMS containing the query sequences for ecotag");
@ -282,7 +282,7 @@ int obi_ecotag(const char* dms_name,
}
// Open the DMS containing the reference database (can be the same or not)
ref_dms = obi_open_dms(ref_dms_name);
ref_dms = obi_open_dms(ref_dms_name, false);
if (ref_dms == NULL)
{
obidebug(1, "\nError opening the DMS containing the reference database for ecotag");
@ -290,7 +290,7 @@ int obi_ecotag(const char* dms_name,
}
// Open the DMS containing the taxonomy (can be the same or not)
taxo_dms = obi_open_dms(taxo_dms_name);
taxo_dms = obi_open_dms(taxo_dms_name, false);
if (taxo_dms == NULL)
{
obidebug(1, "\nError opening the DMS containing the taxonomy for ecotag");

4
src/obi_lcs.c
View File

@ -431,7 +431,7 @@ int obi_lcs_align_one_column(const char* dms_name,
k = 0;
// Open DMS
dms = obi_open_dms(dms_name);
dms = obi_open_dms(dms_name, false);
if (dms == NULL)
{
obidebug(1, "\nError opening the DMS");
@ -728,7 +728,7 @@ int obi_lcs_align_two_columns(const char* dms_name,
k = 0;
// Open DMS
dms = obi_open_dms(dms_name);
dms = obi_open_dms(dms_name, false);
if (dms == NULL)
{
obidebug(1, "\nError opening the DMS to align");

117
src/obidms.c
View File

@ -554,6 +554,7 @@ static int create_dms_infos_file(int dms_file_descriptor, const char* dms_name)
infos->little_endian = obi_is_little_endian();
infos->file_size = file_size;
infos->used_size = 0;
infos->working = false;
infos->comments[0] = '\0';
// Unmap the infos file
@ -652,6 +653,52 @@ static int dms_count_in_list(OBIDMS_p dms)
*
**********************************************************************/
int obi_dms_is_clean(OBIDMS_p dms)
{
int ret_val1;
int ret_val2;
ret_val1 = obi_dms_has_unfinished_views(dms);
if (ret_val1 < 0)
return -1;
ret_val2 = obi_dms_has_unfinished_columns(dms);
if (ret_val2 < 0)
return -1;
return !(ret_val1 || ret_val2);
}
int obi_clean_dms(const char* dms_path)
{
OBIDMS_p dms;
dms = obi_open_dms(dms_path, true);
if (dms == NULL)
{
obidebug(1, "\nError opening a DMS before cleaning unfinished views and columns");
return -1;
}
// Currently done in obi_open_dms
// // Clean unfinished views
// if (obi_clean_unfinished_views(dms) < 0)
// {
// obidebug(1, "\nError cleaning unfinished views");
// return -1;
// }
//
// // Clean unfinished columns
// if (obi_clean_unfinished_columns(dms) < 0)
// {
// obidebug(1, "\nError cleaning unfinished columns");
// return -1;
// }
return 0;
}
int obi_dms_exists(const char* dms_path)
{
struct stat buffer;
@ -750,7 +797,7 @@ OBIDMS_p obi_create_dms(const char* dms_path)
return NULL;
// Open DMS
dms = obi_open_dms(dms_path);
dms = obi_open_dms(dms_path, false);
if (dms == NULL)
{
obidebug(1, "\nProblem opening a DMS");
@ -775,11 +822,12 @@ OBIDMS_p obi_create_dms(const char* dms_path)
}
OBIDMS_p obi_open_dms(const char* dms_path)
OBIDMS_p obi_open_dms(const char* dms_path, bool cleaning)
{
OBIDMS_p dms;
char* relative_dms_path;
char* absolute_dms_path;
int clean_dms;
dms = NULL;
@ -868,6 +916,26 @@ OBIDMS_p obi_open_dms(const char* dms_path)
return NULL;
}
// Reset the working variable to false if cleaning the DMS
if (cleaning)
(dms->infos)->working = false;
// Check that the DMS is not already working (being used by a process)
if ((dms->infos)->working)
{
obidebug(1, "\n\nERROR:\nThe DMS '%s' contains unfinished views or columns. Either another command is currently running, "
"in which case you have to wait for it to finish, or a previous command was interrupted, "
"in which case you can run 'obi clean_dms [your_dms]' to clean the DMS.\n", dms->dms_name);
obi_set_errno(OBIDMS_WORKING);
unmap_infos_file(dms);
closedir(dms->directory);
free(dms);
return NULL;
}
// Set the working variable to true
(dms->infos)->working = true;
// Open the indexer directory
dms->indexer_directory = opendir_in_dms(dms, INDEXER_DIR_NAME);
if (dms->indexer_directory == NULL)
@ -948,6 +1016,23 @@ OBIDMS_p obi_open_dms(const char* dms_path)
return NULL;
}
// // Check for unfinished views and columns
// clean_dms = obi_dms_is_clean(dms);
// if (clean_dms < 0)
// {
// obi_set_errno(OBIDMS_UNKNOWN_ERROR);
// obidebug(1, "\nError checking if a DMS has unfinished views or columns when opening it");
// unmap_infos_file(dms);
// closedir(dms->indexer_directory);
// closedir(dms->tax_directory);
// closedir(dms->view_directory);
// closedir(dms->directory);
// free(dms);
// return NULL;
// }
// if (! clean_dms)
// obi_set_errno(OBIDMS_NOT_CLEAN);
// Clean unfinished views
if (obi_clean_unfinished_views(dms) < 0)
{
@ -1012,7 +1097,7 @@ OBIDMS_p obi_test_open_dms(const char* dms_name)
case 0:
return NULL;
case 1:
return obi_open_dms(dms_name);
return obi_open_dms(dms_name, false);
};
obidebug(1, "\nError checking if an OBIDMS directory exists");
@ -1031,7 +1116,7 @@ OBIDMS_p obi_dms(const char* dms_name)
case 0:
return obi_create_dms(dms_name);
case 1:
return obi_open_dms(dms_name);
return obi_open_dms(dms_name, false);
};
obidebug(1, "\nError checking if an OBIDMS directory exists");
@ -1062,14 +1147,6 @@ int obi_close_dms(OBIDMS_p dms, bool force)
if (dms != NULL)
{
// Unmap information file
if (unmap_infos_file(dms) < 0)
{
obidebug(1, "\nError unmaping a DMS information file while closing a DMS");
free(dms);
return -1;
}
// Close all columns
while ((dms->opened_columns)->nb_opened_columns > 0)
obi_close_column(*((dms->opened_columns)->columns));
@ -1096,6 +1173,18 @@ int obi_close_dms(OBIDMS_p dms, bool force)
free(dms);
return -1;
}
// Set the working variable as false
(dms->infos)->working = false;
// Unmap information file
if (unmap_infos_file(dms) < 0)
{
obidebug(1, "\nError unmaping a DMS information file while closing a DMS");
free(dms);
return -1;
}
if (closedir(dms->directory) < 0)
{
obi_set_errno(OBIDMS_MEMORY_ERROR);
@ -1338,7 +1427,7 @@ obiversion_t obi_import_column(const char* dms_path_1, const char* dms_path_2, c
char* complete_avl_name;
Obi_indexer_p avl_group;
dms_1 = obi_open_dms(dms_path_1);
dms_1 = obi_open_dms(dms_path_1, false);
if (dms_1 == NULL)
{
obidebug(1, "\nError opening a DMS to import a column from it");
@ -1538,7 +1627,7 @@ int obi_import_view(const char* dms_path_1, const char* dms_path_2, const char*
OBIDMS_column_header_p header = NULL;
OBIDMS_column_header_p header_2 = NULL;
dms_1 = obi_open_dms(dms_path_1);
dms_1 = obi_open_dms(dms_path_1, false);
if (dms_1 == NULL)
{
obidebug(1, "\nError opening a DMS to import a view from it");

36
src/obidms.h
View File

@ -88,6 +88,8 @@ typedef struct OBIDMS_infos {
*/
size_t used_size; /**< Used size in bytes.
*/
bool working; /**< If the DMS is currently working (being used by a process).
*/
char comments[]; /**< Comments, additional informations on the DMS including
* the command line history.
*/
@ -146,6 +148,37 @@ extern OBIDMS_p global_opened_dms_list[MAX_NB_OPENED_DMS+1];
extern int global_opened_dms_counter_list[MAX_NB_OPENED_DMS+1];
/**
* @brief Checks if an OBIDMS contains unfinished views or columns.
*
* @param dms A pointer on the DMS.
*
* @returns An integer value indicating whether the DMS is clean or not.
* @retval 1 if the DMS is clean.
* @retval 0 if the DMS contains unfinished views or columns.
* @retval -1 if an error occurred.
*
* @since September 2019
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
int obi_dms_is_clean(OBIDMS_p dms);
/**
* @brief Cleans an OBIDMS that contains unfinished views or columns and tags it as not working
* (not being used by a process).
*
* @param dms_path A pointer to a C string containing the path to the DMS.
*
* @retval 0 on success.
* @retval -1 if an error occurred.
*
* @since September 2019
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
int obi_clean_dms(const char* dms_path);
/**
* @brief Checks if an OBIDMS exists.
*
@ -189,6 +222,7 @@ OBIDMS_p obi_create_dms(const char* dms_name);
* @brief Opens an existing OBITools Data Management instance (OBIDMS).
*
* @param dms_path A pointer to a C string containing the path to the database.
* @param cleaning A boolean indicating whether the DMS is being opened for cleaning unfinished views and columns.
*
* @returns A pointer to an OBIDMS structure describing the opened DMS.
* @retval NULL if an error occurred.
@ -197,7 +231,7 @@ OBIDMS_p obi_create_dms(const char* dms_name);
* @since May 2015
* @author Eric Coissac (eric.coissac@metabarcoding.org)
*/
OBIDMS_p obi_open_dms(const char* dms_path);
OBIDMS_p obi_open_dms(const char* dms_path, bool cleaning);
/**

121
src/obidmscolumn.c
View File

@ -2437,6 +2437,127 @@ int obi_column_prepare_to_get_value(OBIDMS_column_p column, index_t line_nb)
}
int obi_dms_has_unfinished_columns(OBIDMS_p dms)
{
struct dirent* dms_dirent;
struct dirent* col_dirent;
DIR* col_dir;
size_t i,j;
char* column_file_path;
char* column_dir_path;
char* col_name;
char* col_version_str;
// char* version_file;
obiversion_t col_version;
OBIDMS_column_header_p col_header;
// int n;
char* col_to_delete[1000];
char* dir_to_delete[1000];
int ddir;
int dcol;
int d;
int ret_value;
ret_value = 0;
// Find column directories
ddir = 0;
rewinddir(dms->directory);
while ((dms_dirent = readdir(dms->directory)) != NULL)
{
if ((dms_dirent->d_name)[0] == '.')
continue;
i=0;
while (((dms_dirent->d_name)[i] != '.') && (i < strlen(dms_dirent->d_name)))
i++;
if ((i != strlen(dms_dirent->d_name)) && (strcmp((dms_dirent->d_name)+i, ".obicol") == 0)) // Found a column directory
{
column_dir_path = obi_dms_get_full_path(dms, dms_dirent->d_name);
if (column_dir_path == NULL)
{
obidebug(1, "\nError getting a column directory path when deleting unfinished columns");
ret_value = -1;
}
col_name = (char*) malloc(strlen(dms_dirent->d_name) * sizeof(char));
if (col_name == NULL)
{
obi_set_errno(OBI_MALLOC_ERROR);
obidebug(1, "\nError allocating memory for a column name when deleting unfinished columns: directory %s", dms_dirent->d_name);
ret_value = -1;
continue;
}
strncpy(col_name, dms_dirent->d_name, i);
col_name[i] = '\0';
col_dir = opendir_in_dms(dms, dms_dirent->d_name);
if (col_dir == NULL)
{
obidebug(1, "\nError opening a column directory when deleting unfinished columns");
ret_value = -1;
continue;
}
// Iteration on files of this column directory
dcol = 0;
while ((col_dirent = readdir(col_dir)) != NULL)
{
if ((col_dirent->d_name)[0] == '.')
continue;
i=0;
j=0;
while (((col_dirent->d_name)[i] != '@') && ((col_dirent->d_name)[i] != '.'))
i++;
if ((col_dirent->d_name)[i] == '@') // Found a column file
{
i++;
j=i;
while ((col_dirent->d_name)[j] != '.')
j++;
col_version_str = (char*) malloc(strlen(col_dirent->d_name) * sizeof(char));
if (col_version_str == NULL)
{
obi_set_errno(OBI_MALLOC_ERROR);
obidebug(1, "\nError allocating memory for a column version when deleting unfinished columns: directory %s", dms_dirent->d_name);
ret_value = -1;
continue;
}
strncpy(col_version_str, (col_dirent->d_name)+i, j-i);
col_version_str[j-i] = '\0';
col_version = atoi(col_version_str);
free(col_version_str);
col_header = obi_column_get_header_from_name(dms, col_name, col_version);
if (col_header == NULL) // TODO discuss if delete file or not
{
obidebug(1, "\nError reading a column header when deleting unfinished columns: file %s", col_dirent->d_name);
ret_value = -1;
continue;
}
// Check if the column is finished and delete it if not
if (col_header->finished == false)
ret_value = 1;
// Close the header
if (obi_close_header(col_header) < 0)
ret_value = -1;
}
}
// Close column directory
if (closedir(col_dir) < 0)
{
obi_set_errno(OBICOL_UNKNOWN_ERROR);
obidebug(1, "\nError closing a column directory when deleting unfinished columns");
ret_value = -1;
continue;
}
free(col_name);
}
}
return ret_value;
}
int obi_clean_unfinished_columns(OBIDMS_p dms)
{
struct dirent* dms_dirent;

15
src/obidmscolumn.h
View File

@ -535,6 +535,21 @@ int obi_column_prepare_to_set_value(OBIDMS_column_p column, index_t line_nb, ind
int obi_column_prepare_to_get_value(OBIDMS_column_p column, index_t line_nb);
/**
* @brief Goes through all the column files of a DMS and checks for views that have
* not been flagged as finished (done by the finish_view() function in the
* obiview source file).
*
* @param dms A pointer on an OBIDMS.
*
* @returns A boolean indicating whether unfinished views were found.
*
* @since September 2019
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
int obi_dms_has_unfinished_columns(OBIDMS_p dms);
/**
* @brief Goes through all the column files of a DMS and deletes columns that have
* not been flagged as finished (done by the finish_view() function in the

6
src/obierrno.h
View File

@ -130,8 +130,12 @@ extern int obi_errno;
*/
#define OBIVIEW_ALREADY_EXISTS_ERROR (35) /** Tried to create a new view with a name already existing in the DMS.
*/
#define OBI_ECOTAG_ERROR (36) /** Tried to create a new view with a name already existing in the DMS.
#define OBI_ECOTAG_ERROR (36) /** Error while running ecotag.
*/
#define OBIDMS_NOT_CLEAN (37) /** OBIDMS has unfinished views or columns.
*/
#define OBIDMS_WORKING (38) /** OBIDMS is tagged as being currently used by a process.
*/
/**@}*/
#endif /* OBIERRNO_H_ */

68
src/obiview.c
View File

@ -2594,6 +2594,74 @@ int obi_save_and_close_view(Obiview_p view)
}
int obi_dms_has_unfinished_views(OBIDMS_p dms)
{
struct dirent* dp;
int i;
char* full_path;
char* relative_path;
Obiview_infos_p view_infos;
char* view_name;
int ret_value;
char* to_delete[1000];
int d;
ret_value = 0;
d = 0;
// Look for unfinished views and delete them
rewinddir(dms->view_directory);
while ((dp = readdir(dms->view_directory)) != NULL)
{
if ((dp->d_name)[0] == '.')
continue;
i=0;
while ((dp->d_name)[i] != '.')
i++;
relative_path = (char*) malloc(strlen(VIEW_DIR_NAME) + strlen(dp->d_name) + 2);
strcpy(relative_path, VIEW_DIR_NAME);
strcat(relative_path, "/");
strcat(relative_path, dp->d_name);
full_path = obi_dms_get_full_path(dms, relative_path);
free(relative_path);
if (full_path == NULL)
{
obidebug(1, "\nError getting the full path to a view file when cleaning unfinished views");
ret_value = -1;
continue;
}
if (strcmp((dp->d_name)+i, ".obiview_unfinished") == 0)
ret_value = 1;
else if (strcmp((dp->d_name)+i, ".obiview") == 0)
{ // Check if the view was properly flagged as finished
view_name = (char*) malloc((i+1) * sizeof(char));
if (view_name == NULL)
{
obi_set_errno(OBI_MALLOC_ERROR);
obidebug(1, "\nError allocating memory for a view name when deleting unfinished views: file %s", dp->d_name);
ret_value = -1;
continue;
}
strncpy(view_name, dp->d_name, i);
view_name[i] = '\0';
view_infos = obi_view_map_file(dms, view_name, true);
if (view_infos == NULL)
{
obidebug(1, "\nError reading a view file when deleting unfinished views: file %s", dp->d_name);
ret_value = -1;
continue;
}
if (view_infos->finished == false)
ret_value = 1;
}
}
return ret_value;
}
int obi_clean_unfinished_views(OBIDMS_p dms)
{
struct dirent* dp;

15
src/obiview.h
View File

@ -565,6 +565,21 @@ int obi_view_add_comment(Obiview_p view, const char* key, const char* value);
int obi_save_and_close_view(Obiview_p view);
/**
* @brief Goes through all the view files of a DMS and checks for views that have
* not been flagged as finished (file extension renamed from '.obiview_unfinished'
* to '.obiview' and finished boolean set to true in the file, done by finish_view()).
*
* @param dms A pointer on an OBIDMS.
*
* @returns A boolean indicating whether unfinished views were found.
*
* @since September 2019
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
int obi_dms_has_unfinished_views(OBIDMS_p dms);
/**
* @brief Goes through all the view files of a DMS and deletes views that have
* not been flagged as finished (file extension renamed from '.obiview_unfinished'