obitools/obitools3
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 66 Wiki Activity

Closes issue #47 by storing each view in a separate file named with the

Browse Source 
view's name and created upon view creation.
This commit is contained in:
Celine Mercier
2016-06-30 11:41:30 +02:00
parent ad2af0b512
commit 0869b9ba3f
7 changed files with 526 additions and 649 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
python/obitools3/obidms/_obidms.pxd
View File

@ -91,5 +91,5 @@ cdef class OBIDMS:
cpdef OBIView open_view(self, str view_name)
cpdef OBIView new_view(self, str view_name, object view_to_clone=*, list line_selection=*, str view_type=*, str comments=*)
cpdef dict read_view_infos(self, str view_name)
cpdef dict read_views(self)
# cpdef dict read_views(self) TODO

127
python/obitools3/obidms/_obidms.pyx
View File

@ -54,16 +54,15 @@ from ._obidmscolumn_seq cimport OBIDMS_column_seq, \
OBIDMS_column_multi_elts_seq
from .capi.obiview cimport Obiview_p, \
Obiviews_infos_all_p, \
Obiview_infos_p, \
Column_reference_p, \
obi_new_view_nuc_seqs, \
obi_new_view, \
obi_new_view_cloned_from_name, \
obi_new_view_nuc_seqs_cloned_from_name, \
obi_view_map_file, \
obi_view_unmap_file, \
obi_open_view, \
obi_read_view_infos, \
obi_close_view_infos, \
obi_view_delete_column, \
obi_view_add_column, \
obi_view_get_column, \
@ -291,11 +290,11 @@ cdef class OBIView :
raise Exception("Error creating/opening a view")
self.pointer = view
self.name = bytes2str(view.name)
self.name = bytes2str(view.infos.name)
# Go through columns to build list of corresponding python instances
self.columns = {}
for i in range(view.column_count) :
for i in range(view.infos.column_count) :
column_p = <OBIDMS_column_p> (view.columns)[i]
header = (column_p).header
col_name = bytes2str(header.name)
@ -305,7 +304,7 @@ cdef class OBIView :
def __repr__(self) :
cdef str s
s = str(self.name) + ", " + str(self.comments) + ", " + str(self.pointer.line_count) + " lines\n"
s = str(self.name) + ", " + str(self.comments) + ", " + str(self.pointer.infos.line_count) + " lines\n"
for column_name in self.columns :
s = s + self.columns[column_name].__repr__() + '\n'
return s
@ -402,7 +401,7 @@ cdef class OBIView :
cdef OBIView_line line # TODO Check that this works for NUC SEQ views
# Yield each line
lines_used = (self.pointer).line_count
lines_used = self.pointer.infos.line_count
for line_nb in range(lines_used) :
line = self[line_nb]
@ -486,12 +485,12 @@ cdef class OBIView_NUC_SEQS(OBIView):
raise Exception("Error creating/opening view")
self.pointer = view
self.name = bytes2str(view.name)
self.comments = bytes2str(view.comments)
self.name = bytes2str(view.infos.name)
self.comments = bytes2str(view.infos.comments)
# Go through columns to build list of corresponding python instances
self.columns = {}
for i in range(view.column_count) :
for i in range(view.infos.column_count) :
column_p = <OBIDMS_column_p> (view.columns)[i]
header = (column_p).header
col_name = bytes2str(header.name)
@ -621,54 +620,84 @@ cdef class OBIDMS :
cpdef dict read_view_infos(self, str view_name) :
all_views = self.read_views()
return all_views[view_name]
cpdef dict read_views(self) : # TODO function that prints the dic nicely and function that prints 1 view nicely. Add column type in col ref
cdef Obiviews_infos_all_p all_views_p
cdef Obiview_infos_p view_p
cdef Obiview_infos_p view_infos_p
cdef dict view_infos_d
cdef Column_reference_p column_refs
cdef int nb_views
cdef int i, j
cdef str view_name
cdef str column_name
cdef dict views
cdef bytes name_b
views = {}
all_views_p = obi_read_view_infos(self.pointer)
if all_views_p == NULL :
raise Exception("No views to read")
nb_views = <int> (all_views_p.header).view_count
for i in range(nb_views) :
view_p = (<Obiview_infos_p> (all_views_p.view_infos)) + i
view_name = bytes2str(view_p.name)
views[view_name] = {}
views[view_name]["comments"] = bytes2str(view_p.comments)
views[view_name]["view_type"] = bytes2str(view_p.view_type)
views[view_name]["column_count"] = <int> view_p.column_count
views[view_name]["line_count"] = <int> view_p.line_count
views[view_name]["view_number"] = <int> view_p.view_number
views[view_name]["created_from"] = bytes2str(view_p.created_from)
views[view_name]["creation_date"] = bytes2str(obi_format_date(view_p.creation_date))
if (view_p.all_lines) :
views[view_name]["line_selection"] = None
view_infos_p = obi_view_map_file(self.pointer, str2bytes(view_name))
view_infos_d = {}
view_infos_d["name"] = bytes2str(view_infos_p.name)
view_infos_d["comments"] = bytes2str(view_infos_p.comments)
view_infos_d["view_type"] = bytes2str(view_infos_p.view_type)
view_infos_d["column_count"] = <int> view_infos_p.column_count
view_infos_d["line_count"] = <int> view_infos_p.line_count
view_infos_d["created_from"] = bytes2str(view_infos_p.created_from)
view_infos_d["creation_date"] = bytes2str(obi_format_date(view_infos_p.creation_date))
if (view_infos_p.all_lines) :
view_infos_d["line_selection"] = None
else :
views[view_name]["line_selection"] = {}
views[view_name]["line_selection"]["column_name"] = bytes2str((view_p.line_selection).column_name)
views[view_name]["line_selection"]["version"] = <int> (view_p.line_selection).version
views[view_name]["column_references"] = {}
column_refs = view_p.column_references
for j in range(views[view_name]["column_count"]) :
view_infos_d["line_selection"] = {}
view_infos_d["line_selection"]["column_name"] = bytes2str((view_infos_p.line_selection).column_name)
view_infos_d["line_selection"]["version"] = <int> (view_infos_p.line_selection).version
view_infos_d["column_references"] = {}
column_refs = view_infos_p.column_references
for j in range(view_infos_d["column_count"]) :
column_name = bytes2str((column_refs[j]).column_name)
views[view_name]["column_references"][column_name] = {}
views[view_name]["column_references"][column_name]["version"] = column_refs[j].version
view_infos_d["column_references"][column_name] = {}
view_infos_d["column_references"][column_name]["version"] = column_refs[j].version
obi_close_view_infos(all_views_p);
obi_view_unmap_file(self.pointer, view_infos_p)
return views
return view_infos_d
# cpdef dict read_views(self) : # TODO function that prints the dic nicely and function that prints 1 view nicely. Add column type in col ref
#
# cdef Obiviews_infos_all_p all_views_p
# cdef Obiview_infos_p view_p
# cdef Column_reference_p column_refs
# cdef int nb_views
# cdef int i, j
# cdef str view_name
# cdef str column_name
# cdef dict views
# cdef bytes name_b
#
# views = {}
# all_views_p = obi_read_view_infos(self.pointer)
# if all_views_p == NULL :
# raise Exception("No views to read")
# nb_views = <int> (all_views_p.header).view_count
# for i in range(nb_views) :
# view_p = (<Obiview_infos_p> (all_views_p.view_infos)) + i
# view_name = bytes2str(view_p.name)
# views[view_name] = {}
# views[view_name]["comments"] = bytes2str(view_p.comments)
# views[view_name]["view_type"] = bytes2str(view_p.view_type)
# views[view_name]["column_count"] = <int> view_p.column_count
# views[view_name]["line_count"] = <int> view_p.line_count
# views[view_name]["view_number"] = <int> view_p.view_number
# views[view_name]["created_from"] = bytes2str(view_p.created_from)
# views[view_name]["creation_date"] = bytes2str(obi_format_date(view_p.creation_date))
# if (view_p.all_lines) :
# views[view_name]["line_selection"] = None
# else :
# views[view_name]["line_selection"] = {}
# views[view_name]["line_selection"]["column_name"] = bytes2str((view_p.line_selection).column_name)
# views[view_name]["line_selection"]["version"] = <int> (view_p.line_selection).version
# views[view_name]["column_references"] = {}
# column_refs = view_p.column_references
# for j in range(views[view_name]["column_count"]) :
# column_name = bytes2str((column_refs[j]).column_name)
# views[view_name]["column_references"][column_name] = {}
# views[view_name]["column_references"][column_name]["version"] = column_refs[j].version
#
# obi_close_view_infos(all_views_p);
#
# return views

44
python/obitools3/obidms/capi/obiview.pxd
View File

@ -23,21 +23,6 @@ cdef extern from "obiview.h" nogil:
extern const_char_p DEFINITION_COLUMN
extern const_char_p QUALITY_COLUMN
struct Obiview_t :
OBIDMS_p dms
const_char_p name
const_char_p created_from
const_char_p view_type
bint read_only
OBIDMS_column_p line_selection
OBIDMS_column_p new_line_selection
index_t line_count
int column_count
OBIDMS_column_p columns
const_char_p comments
ctypedef Obiview_t* Obiview_p
struct Column_reference_t :
const_char_p column_name
@ -47,7 +32,6 @@ cdef extern from "obiview.h" nogil:
struct Obiview_infos_t :
int view_number
time_t creation_date
const_char_p name
const_char_p created_from
@ -62,19 +46,15 @@ cdef extern from "obiview.h" nogil:
ctypedef Obiview_infos_t* Obiview_infos_p
struct Obiviews_header_t :
size_t header_size
size_t views_size
int view_count
struct Obiview_t :
Obiview_infos_p infos
OBIDMS_p dms
bint read_only
OBIDMS_column_p line_selection
OBIDMS_column_p new_line_selection
OBIDMS_column_p columns
ctypedef Obiviews_header_t* Obiviews_header_p
struct Obiviews_infos_all_t :
Obiviews_header_p header
Obiview_infos_p view_infos
ctypedef Obiviews_infos_all_t* Obiviews_infos_all_p
ctypedef Obiview_t* Obiview_p
Obiview_p obi_new_view_nuc_seqs(OBIDMS_p dms, const_char_p view_name, Obiview_p view_to_clone, index_t* line_selection, const_char_p comments)
@ -85,6 +65,10 @@ cdef extern from "obiview.h" nogil:
Obiview_p obi_new_view_nuc_seqs_cloned_from_name(OBIDMS_p dms, const_char_p view_name, const_char_p view_to_clone_name, index_t* line_selection, const_char_p comments)
Obiview_infos_p obi_view_map_file(OBIDMS_p dms, const char* view_name)
int obi_view_unmap_file(OBIDMS_p dms, Obiview_infos_p view_infos)
Obiview_p obi_open_view(OBIDMS_p dms, const_char_p view_name)
int obi_view_add_column(Obiview_p view,
@ -114,10 +98,6 @@ cdef extern from "obiview.h" nogil:
int obi_save_and_close_view(Obiview_p view)
Obiviews_infos_all_p obi_read_view_infos(OBIDMS_p dms)
int obi_close_view_infos(Obiviews_infos_all_p views)
int obi_column_set_obiint_with_elt_name_in_view(Obiview_p view,
OBIDMS_column_p column,
index_t line_nb,

43
src/obidms.c
View File

@ -252,7 +252,7 @@ OBIDMS_p obi_create_dms(const char* dms_path)
return NULL;
}
// Get file descriptor of DMS directory to create the indexer directory
// Get file descriptor of DMS directory to create other directories
dms_dir = opendir(directory_name);
if (dms_dir == NULL)
{
@ -280,6 +280,14 @@ OBIDMS_p obi_create_dms(const char* dms_path)
return NULL;
}
// Create the view directory
if (mkdirat(dms_file_descriptor, VIEW_DIR_NAME, 00777) < 0)
{
obi_set_errno(OBIVIEW_ERROR);
obidebug(1, "\nProblem creating a view directory");
return NULL;
}
// Isolate the dms name
j = 0;
for (i=0; i<strlen(dms_path); i++)
@ -447,6 +455,30 @@ OBIDMS_p obi_open_dms(const char* dms_path)
return NULL;
}
// Open the view directory
dms->view_directory = opendir_in_dms(dms, VIEW_DIR_NAME);
if (dms->view_directory == NULL)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nError opening the view directory");
closedir(dms->indexer_directory);
closedir(dms->directory);
free(dms);
return NULL;
}
// Store the view directory's file descriptor
dms->view_dir_fd = dirfd(dms->view_directory);
if (dms->view_dir_fd < 0)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nError getting the file descriptor of the view directory");
closedir(dms->view_directory);
closedir(dms->directory);
free(dms);
return NULL;
}
// Initialize the list of opened columns
dms->opened_columns = (Opened_columns_list_p) malloc(sizeof(Opened_columns_list_t));
(dms->opened_columns)->nb_opened_columns = 0;
@ -486,7 +518,7 @@ int obi_close_dms(OBIDMS_p dms)
while ((dms->opened_columns)->nb_opened_columns > 0)
obi_close_column(*((dms->opened_columns)->columns));
// Close dms and indexer directories
// Close dms, and view and indexer directories
if (closedir(dms->directory) < 0)
{
obi_set_errno(OBIDMS_MEMORY_ERROR);
@ -501,6 +533,13 @@ int obi_close_dms(OBIDMS_p dms)
free(dms);
return -1;
}
if (closedir(dms->view_directory) < 0)
{
obi_set_errno(OBIVIEW_ERROR);
obidebug(1, "\nError closing a view directory");
free(dms);
return -1;
}
free(dms);
}

8
src/obidms.h
View File

@ -30,6 +30,8 @@
*/
#define INDEXER_DIR_NAME "OBIBLOB_INDEXERS" /**< The name of the Obiblob indexer directory.
*/
#define VIEW_DIR_NAME "VIEWS" /**< The name of the view directory.
*/
#define TAXONOMY_DIR_NAME "TAXONOMY" /**< The name of the taxonomy directory.
*/
#define MAX_NB_OPENED_COLUMNS (100) /**< The maximum number of columns open at the same time.
@ -98,6 +100,12 @@ typedef struct OBIDMS {
int indexer_dir_fd; /**< The file descriptor of the directory entry
* usable to refer and scan the indexer directory.
*/
DIR* view_directory; /**< A directory entry usable to
* refer and scan the view directory.
*/
int view_dir_fd; /**< The file descriptor of the directory entry
* usable to refer and scan the view directory.
*/
bool little_endian; /**< Endianness of the database.
*/
Opened_columns_list_p opened_columns; /**< List of opened columns.

813
src/obiview.c
View File

File diff suppressed because it is too large Load Diff

98
src/obiview.h
View File

@ -34,8 +34,6 @@
*/
#define VIEW_TYPE_MAX_LENGTH (1024) /**< The maximum length of the type name of a view.
*/
#define OBIVIEW_FILE_NAME "obiviews" /**< The default name of a view file.
*/
#define LINES_COLUMN_NAME "LINES" /**< The name of the column containing the line selections
* in all views.
*/
@ -56,45 +54,6 @@
*/
/**
* @brief Structure for an opened view.
*/
typedef struct Obiview {
OBIDMS_p dms; /**< A pointer on the DMS to which the view belongs.
*/
char name[OBIVIEW_NAME_MAX_LENGTH+1]; /**< Name of the view.
*/
char created_from[OBIVIEW_NAME_MAX_LENGTH+1]; /**< Name of the view from which that view was cloned if the view was cloned.
*/
char view_type[VIEW_TYPE_MAX_LENGTH+1]; /**< Type of the view if there is one.
* Types existing: NUC_SEQS_VIEW.
*/
bool read_only; /**< Whether the view is read-only or can be modified.
*/
OBIDMS_column_p line_selection; /**< A pointer on the column containing the line selection
* associated with the view if there is one.
* This line selection is read-only, and when a line from the view is read,
* it is this line selection that is used.
*/
OBIDMS_column_p new_line_selection; /**< A pointer on the column containing the new line selection being built
* to associate with the view, if there is one.
* When a line is selected with obi_select_line() or obi_select_lines(),
* it is recorded in this line selection.
*/
index_t line_count; /**< The number of lines in the view. Refers to the number of lines in each
* column of the view if line_selection is NULL, or to the line count of
* line_selection if it is not NULL.
*/
int column_count; /**< The number of columns in the view.
*/
OBIDMS_column_p columns[MAX_NB_OPENED_COLUMNS]; /**< Array of pointers on all the columns of the view.
*/
char comments[OBIVIEW_COMMENTS_MAX_LENGTH+1]; /**< Comments, additional informations on the view.
*/
} Obiview_t, *Obiview_p;
/**
* @brief Structure referencing a column by its name and its version.
*/
@ -112,8 +71,6 @@ typedef struct Column_reference {
* Once a view has been written in the view file, it can not be modified and can only be read.
*/
typedef struct Obiview_infos {
int view_number; /**< Number of the view in the view file.
*/
time_t creation_date; /**< Time at which the view was written in the view file.
*/
char name[OBIVIEW_NAME_MAX_LENGTH+1]; /**< Name of the view, used to identify it.
@ -138,7 +95,29 @@ typedef struct Obiview_infos {
} Obiview_infos_t, *Obiview_infos_p;
// TODO : Combine the common elements of the Obiview_infos and Obiview structures in one structure used by both?
/**
* @brief Structure for an opened view.
*/
typedef struct Obiview {
Obiview_infos_p infos; /**< A pointer on the mapped view informations.
*/
OBIDMS_p dms; /**< A pointer on the DMS to which the view belongs.
*/
bool read_only; /**< Whether the view is read-only or can be modified.
*/
OBIDMS_column_p line_selection; /**< A pointer on the column containing the line selection
* associated with the view if there is one.
* This line selection is read-only, and when a line from the view is read,
* it is this line selection that is used.
*/
OBIDMS_column_p new_line_selection; /**< A pointer on the column containing the new line selection being built
* to associate with the view, if there is one.
* When a line is selected with obi_select_line() or obi_select_lines(),
* it is recorded in this line selection.
*/
OBIDMS_column_p columns[MAX_NB_OPENED_COLUMNS]; /**< Array of pointers on all the columns of the view.
*/
} Obiview_t, *Obiview_p;
/**
@ -269,6 +248,37 @@ Obiview_p obi_new_view_nuc_seqs(OBIDMS_p dms, const char* view_name, Obiview_p v
Obiview_p obi_new_view_nuc_seqs_cloned_from_name(OBIDMS_p dms, const char* view_name, const char* view_to_clone_name, index_t* line_selection, const char* comments);
/**
* @brief Maps a view file and returns the mapped structure stored in it.
*
* @param dms A pointer on the OBIDMS.
* @param view_name The unique name identifying the view.
*
* @returns A pointer on the mapped view infos structure.
* @retval NULL if an error occurred.
*
* @since June 2016
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
Obiview_infos_p obi_view_map_file(OBIDMS_p dms, const char* view_name);
/**
* @brief Unmaps a view file.
*
* @param dms A pointer on the OBIDMS.
* @param view_infos A pointer on the mapped view infos structure.
*
* @returns A value indicating the success of the operation.
* @retval 0 if the operation was successfully completed.
* @retval -1 if an error occurred.
*
* @since June 2016
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
int obi_view_unmap_file(OBIDMS_p dms, Obiview_infos_p view_infos);
/**
* @brief Opens a view identified by its name stored in the view file.
*