obitools/obitools3
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 66 Wiki Activity

obi ls: now done in C (preparing things for R packages to read DMS) and

Browse Source 
switch to version 3.0.0b36
This commit is contained in:
Celine Mercier
2020-09-29 17:51:39 +02:00
parent b6648ae81e
commit f879b6bb52
14 changed files with 460 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
python/obitools3/commands/ls.pyx
View File

@ -31,27 +31,11 @@ def run(config):
input = open_uri(config['obi']['inputURI'])
if input is None:
raise Exception("Could not read input")
if input[2] == DMS and not config['ls']['longformat']:
dms = input[0]
l = []
for viewname in input[0]:
view = dms[viewname]
l.append(tostr(viewname) + "\t(Date created: " + str(bytes2str_object(view.comments["Date created"]))+")")
view.close()
l.sort()
for v in l:
print(v)
# Print representation
if config['ls']['longformat']:
print(input[1].repr_longformat())
else:
print(repr(input[1]))
if input[2] == DMS:
taxolist = ["\n### Taxonomies:"]
for t in Taxonomy.list_taxos(input[0]):
taxolist.append("\t"+tostr(t))
if len(taxolist) > 1:
for t in taxolist:
print(t)
if config['ls']['longformat'] and len(input[1].comments) > 0:
print("\n### Comments:")
print(str(input[1].comments))
input[0].close(force=True)

1
python/obitools3/dms/capi/obidms.pxd
View File

@ -34,6 +34,7 @@ cdef extern from "obidms.h" nogil:
int obi_close_dms(OBIDMS_p dms, bint force)
char* obi_dms_get_dms_path(OBIDMS_p dms)
char* obi_dms_get_full_path(OBIDMS_p dms, const_char_p path_name)
char* obi_dms_formatted_infos(OBIDMS_p dms, bint detailed)
void obi_close_atexit()
obiversion_t obi_import_column(const char* dms_path_1, const char* dms_path_2, const char* column_name, obiversion_t version_number)

3
python/obitools3/dms/capi/obidmscolumn.pxd
View File

@ -70,3 +70,6 @@ cdef extern from "obidmscolumn.h" nogil:
int obi_column_write_comments(OBIDMS_column_p column, const char* comments)
int obi_column_add_comment(OBIDMS_column_p column, const char* key, const char* value)
char* obi_column_formatted_infos(OBIDMS_column_p column, bint detailed)

8
python/obitools3/dms/capi/obiview.pxd
View File

@ -103,13 +103,17 @@ cdef extern from "obiview.h" nogil:
bint create)
int obi_view_delete_column(Obiview_p view, const_char_p column_name, bint delete_file)
OBIDMS_column_p obi_view_get_column(Obiview_p view, const_char_p column_name)
OBIDMS_column_p* obi_view_get_pointer_on_column_in_view(Obiview_p view, const_char_p column_name)
int obi_view_create_column_alias(Obiview_p view, const_char_p current_name, const_char_p alias)
char* obi_view_formatted_infos(Obiview_p view, bint detailed)
char* obi_view_formatted_infos_one_line(Obiview_p view)
int obi_view_write_comments(Obiview_p view, const_char_p comments)
int obi_view_add_comment(Obiview_p view, const_char_p key, const_char_p value)

27
python/obitools3/dms/column/column.pyx
View File

@ -302,15 +302,24 @@ cdef class Column(OBIWrapper) :
@OBIWrapper.checkIsActive
def __repr__(self) :
cdef bytes s
#cdef char* s_b
#cdef str s_str
#s_b = obi_column_formatted_infos(self.pointer())
#s_str = bytes2str(s_b)
#free(s_b)
s = self._alias + b", data type: " + self.data_type
#return s_str
return bytes2str(s)
cdef str s
cdef char* sc
cdef OBIDMS_column_p pointer = self.pointer()
sc = obi_column_formatted_infos(pointer, False)
s = bytes2str(sc)
free(sc)
return s
@OBIWrapper.checkIsActive
def repr_longformat(self) :
cdef str s
cdef char* sc
cdef OBIDMS_column_p pointer = self.pointer()
sc = obi_column_formatted_infos(pointer, True)
s = bytes2str(sc)
free(sc)
return s
def close(self): # TODO discuss, can't be called bc then bug when closing view that tries to close it in C

30
python/obitools3/dms/dms.pyx
View File

@ -10,7 +10,8 @@ from .capi.obidms cimport obi_open_dms, \
obi_dms_exists, \
obi_dms_get_full_path, \
obi_close_atexit, \
obi_dms_write_comments
obi_dms_write_comments, \
obi_dms_formatted_infos
from .capi.obitypes cimport const_char_p
@ -32,6 +33,8 @@ from .object import OBIWrapper
import json
import time
from libc.stdlib cimport free
cdef class DMS(OBIWrapper):
@ -223,13 +226,24 @@ cdef class DMS(OBIWrapper):
@OBIWrapper.checkIsActive
def __repr__(self):
cdef str s
s=""
for view_name in self.keys():
view = self.get_view(view_name)
s = s + repr(view) + "\n"
view.close()
def __repr__(self) :
cdef str s
cdef char* sc
cdef OBIDMS_p pointer = self.pointer()
sc = obi_dms_formatted_infos(pointer, False)
s = bytes2str(sc)
free(sc)
return s
@OBIWrapper.checkIsActive
def repr_longformat(self) :
cdef str s
cdef char* sc
cdef OBIDMS_p pointer = self.pointer()
sc = obi_dms_formatted_infos(pointer, True)
s = bytes2str(sc)
free(sc)
return s

31
python/obitools3/dms/view/view.pyx
View File

@ -19,7 +19,9 @@ from ..capi.obiview cimport Alias_column_pair_p, \
obi_view_delete_column, \
obi_view_create_column_alias, \
obi_view_write_comments, \
obi_delete_view
obi_delete_view, \
obi_view_formatted_infos, \
obi_view_formatted_infos_one_line
from ..capi.obidmscolumn cimport OBIDMS_column_p
from ..capi.obidms cimport OBIDMS_p
@ -59,6 +61,8 @@ import pkgutil
import json
import sys
from libc.stdlib cimport free
cdef class View(OBIWrapper) :
@ -186,15 +190,22 @@ cdef class View(OBIWrapper) :
@OBIWrapper.checkIsActive
def __repr__(self) :
cdef str s
if self.read_only: # can read date
s = "#View name:\n{name:s}\n#Date created:\n{date:s}\n#Line count:\n{line_count:d}\n#Columns:\n".format(name = bytes2str(self.name),
line_count = self.line_count,
date = str(bytes2str_object(self.comments["Date created"])))
else:
s = "#View name:\n{name:s}\n#Line count:\n{line_count:d}\n#Columns:\n".format(name = bytes2str(self.name),
line_count = self.line_count)
for column_name in self.keys() :
s = s + repr(self[column_name]) + '\n'
cdef char* sc
cdef Obiview_p pointer = self.pointer()
sc = obi_view_formatted_infos(pointer, False)
s = bytes2str(sc)
free(sc)
return s
@OBIWrapper.checkIsActive
def repr_longformat(self) :
cdef str s
cdef char* sc
cdef Obiview_p pointer = self.pointer()
sc = obi_view_formatted_infos(pointer, True)
s = bytes2str(sc)
free(sc)
return s

2
python/obitools3/version.py
View File

@ -1,5 +1,5 @@
major = 3
minor = 0
serial= '0b35'
serial= '0b36'
version ="%d.%d.%s" % (major,minor,serial)

101
src/obidms.c
View File

@ -1409,6 +1409,107 @@ DIR* opendir_in_dms(OBIDMS_p dms, const char* path_name)
}
char* obi_dms_formatted_infos(OBIDMS_p dms, bool detailed)
{
char* dms_infos = NULL;
char* view_infos = NULL;
char* view_name = NULL;
char* tax_name = NULL;
char* all_tax_dir_path = NULL;
int i;
struct dirent* dp;
Obiview_p view;
// DMS name
dms_infos = (char*) malloc((strlen("# DMS name: ")+strlen(dms->dms_name)+strlen("\n# Views:\n")+1) * sizeof(char));
if (dms_infos == NULL)
{
obidebug(1, "\nError allocating memory for DMS formatted infos");
return NULL;
}
strcpy(dms_infos, "# DMS name: ");
strcat(dms_infos, dms->dms_name);
strcat(dms_infos, "\n# Views:\n");
// Go through views and get their infos
rewinddir(dms->view_directory);
while ((dp = readdir(dms->view_directory)) != NULL)
{
if ((dp->d_name)[0] == '.')
continue;
i=0;
while ((dp->d_name)[i] != '.')
i++;
view_name = (char*) malloc((i+1) * sizeof(char));
if (view_name == NULL)
{
obi_set_errno(OBI_MALLOC_ERROR);
obidebug(1, "\nError allocating memory for a view name when getting formatted DMS infos: file %s", dp->d_name);
return NULL;
}
strncpy(view_name, dp->d_name, i);
view_name[i] = '\0';
view = obi_open_view(dms, view_name);
if (view == NULL)
{
obidebug(1, "\nError opening a view to get DMS formatted infos");
return NULL;
}
if (detailed)
view_infos = obi_view_formatted_infos(view, detailed);
else
view_infos = obi_view_formatted_infos_one_line(view);
if (view_infos == NULL)
{
obidebug(1, "\nError getting a view infos to get DMS formatted infos");
return NULL;
}
dms_infos = realloc(dms_infos, (strlen(dms_infos)+strlen(view_infos)+1) * sizeof(char));
if (dms_infos == NULL)
{
obidebug(1, "\nError reallocating memory for DMS formatted infos");
return NULL;
}
strcat(dms_infos, view_infos);
if (obi_save_and_close_view(view) < 0)
{
obidebug(1, "\nError closing view while getting DMS formatted infos");
return NULL;
}
if (detailed)
{
dms_infos = realloc(dms_infos, (strlen(dms_infos)+2) * sizeof(char));
strcat(dms_infos, "\n");
}
}
// Add taxonomies
dms_infos = realloc(dms_infos, (strlen(dms_infos)+strlen("\n# Taxonomies:\n")+1) * sizeof(char));
if (dms_infos == NULL)
{
obidebug(1, "\nError reallocating memory for DMS formatted infos");
return NULL;
}
strcat(dms_infos, "# Taxonomies:\n");
rewinddir(dms->tax_directory);
while ((dp = readdir(dms->tax_directory)) != NULL)
{
if ((dp->d_name)[0] == '.')
continue;
tax_name = dp->d_name;
dms_infos = realloc(dms_infos, (strlen(dms_infos)+strlen(" # ")+strlen(view_infos)+1) * sizeof(char));
if (dms_infos == NULL)
{
obidebug(1, "\nError reallocating memory for DMS formatted infos");
return NULL;
}
strcat(dms_infos, " # ");
strcat(dms_infos, tax_name);
}
return dms_infos;
}
// TODO move somewhere else maybe
// TODO discuss arguments
obiversion_t obi_import_column(const char* dms_path_1, const char* dms_path_2, const char* column_name, obiversion_t version_number)

17
src/obidms.h
View File

@ -459,6 +459,23 @@ char* obi_dms_get_full_path(OBIDMS_p dms, const char* path_name);
DIR* opendir_in_dms(OBIDMS_p dms, const char* path_name);
/**
* @brief Returns the informations of a DMS with a human readable format (dms name, taxonomies and view infos).
*
* @warning The returned pointer has to be freed by the caller.
*
* @param column A pointer on a DMS.
* @param detailed Whether the informations should contain detailed view infos.
*
* @returns A pointer on a character array where the formatted DMS informations are stored.
* @retval NULL if an error occurred.
*
* @since September 2020
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
char* obi_dms_formatted_infos(OBIDMS_p dms, bool detailed);
/**
* @brief Imports a column, copying it from a DMS to another DMS, and returns the version of the column in the destination DMS.
*

69
src/obidmscolumn.c
View File

@ -2435,17 +2435,77 @@ char* obi_column_formatted_infos(OBIDMS_column_p column, bool detailed)
{
char* column_infos = NULL;
char* elt_names = NULL;
char* column_name = NULL;
// should be in view.c because alias exists in the context of view
column_infos = malloc(2048 * sizeof(char)); // TODO
char* data_type_str = NULL;
char* comments = NULL;
// Get element names informations
elt_names = obi_get_formatted_elements_names(column);
if (elt_names == NULL)
{
obidebug(1, "\nError getting formatted elements names for formatted columns infos");
return NULL;
}
// Get data type informations
data_type_str = name_data_type((column->header)->returned_data_type);
if (data_type_str == NULL)
{
obidebug(1, "\nError getting formatted data type for formatted columns infos");
return NULL;
}
// "column_name, data type: OBI_TYPE, element names: [formatted element names](, all comments)"
// Get commments if detailed informations required
if (detailed)
comments = (column->header)->comments;
// Build the string of formatted infos, allocating memory as needed
// Data type
column_infos = (char*) malloc((strlen("data type: ")+strlen(data_type_str)+1) * sizeof(char));
if (column_infos == NULL)
{
obi_set_errno(OBI_MALLOC_ERROR);
obidebug(1, "\nError allocating memory for formatted column infos");
return NULL;
}
strcpy(column_infos, "data type: ");
strcat(column_infos, data_type_str);
// Element names if more than 1
if ((column->header)->nb_elements_per_line > 1)
{
column_infos = realloc(column_infos, (strlen(column_infos)+strlen(", elements: ")+strlen(elt_names)+1) * sizeof(char));
if (column_infos == NULL)
{
obi_set_errno(OBI_MALLOC_ERROR);
obidebug(1, "\nError allocating memory for formatted column infos");
return NULL;
}
strcat(column_infos, ", elements: ");
strcat(column_infos, elt_names);
}
if (detailed && (strlen(comments)>2)) // Add all comments if required and not empty
{
column_infos = realloc(column_infos, (strlen(column_infos)+strlen("\nComments:\n")+strlen(comments)+1) * sizeof(char));
if (column_infos == NULL)
{
obi_set_errno(OBI_MALLOC_ERROR);
obidebug(1, "\nError allocating memory for formatted column infos");
return NULL;
}
strcat(column_infos, "\nComments:\n");
strcat(column_infos, comments);
}
// "data type: OBI_TYPE, element names: [formatted element names](, all comments)"
free(elt_names);
free(data_type_str);
return column_infos;
}
@ -2492,7 +2552,6 @@ int obi_column_prepare_to_set_value(OBIDMS_column_p column, index_t line_nb, ind
}
int obi_column_prepare_to_get_value(OBIDMS_column_p column, index_t line_nb)
{
if ((line_nb+1) > ((column->header)->line_count))

33
src/obidmscolumn.h
View File

@ -505,12 +505,37 @@ index_t obi_column_get_element_index_from_name(OBIDMS_column_p column, const cha
char* obi_get_elements_names(OBIDMS_column_p column);
// TODO
//char* obi_get_formatted_elements_names(OBIDMS_column_p column);
/**
* @brief Recovers the elements names of the lines of a column with a human readable format ("0; 1; 2; ...; n\0").
*
* @warning The returned pointer has to be freed by the caller.
*
* @param column A pointer on an OBIDMS column.
*
* @returns A pointer on a character array where the elements names are stored.
* @retval NULL if an error occurred.
*
* @since September 2020
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
char* obi_get_formatted_elements_names(OBIDMS_column_p column);
// TODO
//char* obi_column_formatted_infos(OBIDMS_column_p column);
/**
* @brief Returns the informations of a column with a human readable format (data type, element names, comments).
*
* @warning The returned pointer has to be freed by the caller.
*
* @param column A pointer on an OBIDMS column.
* @param detailed Whether the informations should contain column comments or just data type and element names.
*
* @returns A pointer on a character array where the formatted column informations are stored.
* @retval NULL if an error occurred.
*
* @since September 2020
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
char* obi_column_formatted_infos(OBIDMS_column_p column, bool detailed);
/**

140
src/obiview.c
View File

@ -17,6 +17,7 @@
#include <sys/mman.h>
#include <inttypes.h>
#include <math.h>
#include <time.h>
//#include <ctype.h>
#include "obiview.h"
@ -1185,6 +1186,7 @@ static int close_view(Obiview_p view)
obidebug(1, "\nError getting a column to close from the linked list of column pointers of a view");
return -1;
}
if (obi_close_column(column) < 0)
{
obidebug(1, "\nError closing a column while closing a view");
@ -2603,6 +2605,144 @@ int obi_view_create_column_alias(Obiview_p view, const char* current_name, const
}
char* obi_view_formatted_infos(Obiview_p view, bool detailed)
{
int i;
char* view_infos = NULL;
char* view_name = NULL;
time_t creation_date = NULL;
char* creation_date_str = NULL;
index_t line_count;
char line_count_str[256];
OBIDMS_column_p column;
char* column_alias = NULL;
char* column_infos = NULL;
char* comments = NULL;
// View name
view_name = (view->infos)->name;
view_infos = (char*) malloc((strlen("# View name:\n")+strlen(view_name)+1) * sizeof(char));
strcpy(view_infos, "# View name:\n");
strcat(view_infos, view_name);
// Date created
if (view->read_only) // Date not saved until view is finished writing
{
creation_date = (view->infos)->creation_date;
creation_date_str = ctime(&creation_date);
view_infos = realloc(view_infos, (strlen(view_infos)+strlen("\n# Date created:\n")+strlen(creation_date_str)+1) * sizeof(char));
strcat(view_infos, "\n# Date created:\n");
strcat(view_infos, creation_date_str);
}
// Line count
line_count = (view->infos)->line_count;
snprintf(line_count_str, sizeof line_count_str, "%zu", line_count);
view_infos = realloc(view_infos, (strlen(view_infos)+strlen("\n# Line count:\n")+strlen(line_count_str)+1) * sizeof(char));
strcat(view_infos, "# Line count:\n");
strcat(view_infos, line_count_str);
// Columns: go through each, print their alias then their infos
view_infos = realloc(view_infos, (strlen(view_infos)+strlen("\n# Columns:")+1) * sizeof(char));
strcat(view_infos, "\n# Columns:");
for (i=0; i<((view->infos)->column_count); i++)
{
column = *((OBIDMS_column_p*)ll_get(view->columns, i));
if (column == NULL)
{
obidebug(1, "\nError getting a column from the linked list of column pointers of a view to format view infos");
return NULL;
}
// Column alias
column_alias = (((view->infos)->column_references)[i]).alias;
view_infos = realloc(view_infos, (strlen(view_infos)+strlen("\n")+strlen(column_alias)+strlen(", ")+1) * sizeof(char));
strcat(view_infos, "\n");
strcat(view_infos, column_alias);
strcat(view_infos, ", ");
// Column infos
column_infos = obi_column_formatted_infos(column, detailed);
if (column_infos == NULL)
{
obidebug(1, "\nError getting column infos to format view infos");
return NULL;
}
view_infos = realloc(view_infos, (strlen(view_infos)+strlen(column_infos)+1) * sizeof(char));
strcat(view_infos, column_infos);
free(column_infos);
}
// Get commments if detailed informations required
if (detailed)
{
comments = (view->infos)->comments;
if (strlen(comments)>2) // Add all comments if not empty
{
view_infos = realloc(view_infos, (strlen(view_infos)+strlen("\n# Comments:\n")+strlen(comments)+1) * sizeof(char));
if (view_infos == NULL)
{
obi_set_errno(OBI_MALLOC_ERROR);
obidebug(1, "\nError allocating memory for formatted view infos");
return NULL;
}
strcat(view_infos, "\n# Comments:\n");
strcat(view_infos, comments);
}
}
view_infos = realloc(view_infos, (strlen(view_infos)+2) * sizeof(char));
strcat(view_infos, "\n");
return view_infos;
}
char* obi_view_formatted_infos_one_line(Obiview_p view)
{
int i;
char* view_infos = NULL;
char* view_name = NULL;
time_t creation_date = NULL;
char* creation_date_str = NULL;
index_t line_count;
char line_count_str[256];
// View name
view_name = (view->infos)->name;
view_infos = (char*) malloc((strlen(" # ")+strlen(view_name)+2) * sizeof(char));
strcpy(view_infos, " # ");
strcat(view_infos, view_name);
strcat(view_infos, ":");
// Date created
if (view->read_only) // Date not saved until view is finished writing
{
creation_date = (view->infos)->creation_date;
creation_date_str = ctime(&creation_date);
// Delete \n added by ctime
creation_date_str[strlen(creation_date_str)-1] = '\0';
view_infos = realloc(view_infos, (strlen(view_infos)+strlen(" Date created: ")+strlen(creation_date_str)+1) * sizeof(char));
strcat(view_infos, " Date created: ");
strcat(view_infos, creation_date_str);
}
// Line count
line_count = (view->infos)->line_count;
snprintf(line_count_str, sizeof line_count_str, "%zu", line_count);
view_infos = realloc(view_infos, (strlen(view_infos)+strlen(" ; Line count: ")+strlen(line_count_str)+1) * sizeof(char));
strcat(view_infos, " ; Line count: ");
strcat(view_infos, line_count_str);
view_infos = realloc(view_infos, (strlen(view_infos)+2) * sizeof(char));
strcat(view_infos, "\n");
return view_infos;
}
int obi_view_write_comments(Obiview_p view, const char* comments)
{
size_t new_size;

33
src/obiview.h
View File

@ -519,6 +519,39 @@ OBIDMS_column_p* obi_view_get_pointer_on_column_in_view(Obiview_p view, const ch
int obi_view_create_column_alias(Obiview_p view, const char* current_name, const char* alias);
/**
* @brief Returns the informations of a view with a human readable format (view name, date created, line count, column informations, comments).
*
* @warning The returned pointer has to be freed by the caller.
*
* @param column A pointer on a view.
* @param detailed Whether the informations should contain view comments.
*
* @returns A pointer on a character array where the formatted view informations are stored.
* @retval NULL if an error occurred.
*
* @since September 2020
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
char* obi_view_formatted_infos(Obiview_p view, bool detailed);
/**
* @brief Returns the informations of a view with a human readable format on one line (view name, date created, line count).
*
* @warning The returned pointer has to be freed by the caller.
*
* @param column A pointer on a view.
*
* @returns A pointer on a character array where the formatted view informations are stored.
* @retval NULL if an error occurred.
*
* @since September 2020
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
char* obi_view_formatted_infos_one_line(Obiview_p view);
/**
* @brief Internal function writing new comments in a view file.
*