obitools/obitools3
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 66 Wiki Activity

Merge branch 'master' of

Browse Source 
git@git.metabarcoding.org:obitools/obitools3.git

Conflicts:
	doc/source/formats.rst
	src/obicolumn.h
This commit is contained in:
Eric Coissac
2015-05-23 10:23:33 +03:00
parent 76d714f10a 0b031da28c
commit 41d8453e56
11 changed files with 293 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
doc/Makefile
View File

@ -6,6 +6,7 @@ SPHINXOPTS =
SPHINXBUILD = sphinx-build
PAPER =
BUILDDIR = build
DOXYGENDIR = doxygen
# User-friendly check for sphinx-build
ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
@ -50,8 +51,12 @@ help:
clean:
rm -rf $(BUILDDIR)/*
rm -rf $(DOXYGENDIR)/*
html:
@echo "Generating Doxygen documentation..."
doxygen Doxyfile
@echo "Doxygen documentation generated. \n"
$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
@echo
@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
@ -114,6 +119,9 @@ epub:
@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
latex:
@echo "Generating Doxygen documentation..."
doxygen Doxyfile
@echo "Doxygen documentation generated. \n"
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
@echo
@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
@ -121,6 +129,9 @@ latex:
"(use \`make latexpdf' here to do that automatically)."
latexpdf:
@echo "Generating Doxygen documentation..."
doxygen Doxyfile
@echo "Doxygen documentation generated. \n"
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
@echo "Running LaTeX files through pdflatex..."
$(MAKE) -C $(BUILDDIR)/latex all-pdf

8
doc/source/DMS.rst
View File

@ -98,7 +98,7 @@ or more (if there are several versions), and that there is one file per version.
Versioning
----------
The first version of a column file is numbered 1, and each new version increments that
The first version of a column file is numbered 0, and each new version increments that
number by 1.
The number of the latest version of an OBIDMS column is stored in an `OBIDMS release file <formats.html#obidms-release-files>`_.
@ -133,13 +133,17 @@ OBIDMS history file
An OBIDMS history file consists of an ordered list of views and commands, those commands leading
from one view to the next one.
This history can be represented in the form of a --- showing all the
This history can be represented in the form of a ?? showing all the
operations ever done in the OBIDMS directory and the views in between them :
.. image:: ./images/history.png
:width: 150 px
:align: center
OBIType header file
========================
.. doxygenfile:: obitypes.h
OBIIntColumn header file

5
doc/source/pistes.rst
View File

@ -9,7 +9,8 @@ Ce que l'on veut pouvoir faire
* Gerer les valeurs manquantes
* Modifier une colonne en cours d'ecriture (mmap)
* Ajouter des valeurs a la fin d'une colonne en cours d'ecriture (mmap)
* Ajouter des valeurs a la fin du fichier d'une colonne en cours d'ecriture (mmap)
*
******
@ -17,4 +18,4 @@ Divers
******
* Si l'ordre d'une colonne est change, elle est reecrite (pas d'index).
* Truc pour verrouiller l'acces en lecture a un programme a la fois...
* Utilisation de semaphores pour la lecture

2
src/obicolumn.h
View File

@ -36,7 +36,7 @@ typedef enum OBIDataType {
typedef struct OBIColumnHeader {
bool little_endian_order; /**< endian byte order :
- True : little endian
- False: big endian
- False: big endianx
*/
int header_size_value; /**< size of the header: a multiple of PAGESIZE */
int line_count; /**< number of lines of data */

18
src/obicount/Makefile Normal file
View File

@ -0,0 +1,18 @@
CC = gcc
CFLAGS = -c -Wall
LDFLAGS =
SOURCES = obicount.c ../obidmscolumn.c
OBJECTS = $(SOURCES:.c=.o)
EXECUTABLE = obicount
all: $(SOURCES) $(EXECUTABLE)
$(EXECUTABLE): $(OBJECTS)
$(CC) $(LDFLAGS) $(OBJECTS) -o $@
.c.o:
$(CC) $(CFLAGS) $< -o $@
clean:
rm *o
rm $(EXECUTABLE)

87
src/obicount/obicount.c Normal file
View File

@ -0,0 +1,87 @@
#include <stdio.h>
#include <stdlib.h>
#include <sys/mman.h> /* mmap() is defined in this header */
#include <stdint.h>
#include <fcntl.h>
#include <sys/stat.h>
#include "../obitypes.h"
#include "../obidmscolumn.h"
/**
* @brief Computes the size to map.
*
* * @param OBIDMSColumn_file The file to map.
* @return The size to map.
*/
int get_size_to_map(int OBIDMSColumn_file)
// compute size to map : file size minus size of the header
{
int size;
struct stat s;
fstat(OBIDMSColumn_file, &s);
size = (s.st_size) - HEADER_SIZE;
return(size);
}
/**
* @brief Computes and prints the total number of sequences by summing their counts.
*
* * @param The count file.
*/
int main(int argc, char const *argv[])
{
char* map;
int size;
int OBIDMSColumn_file;
int count;
char c;
char num_str[10] = "";
int num_int;
int i,j;
// initialize variables
OBIDMSColumn_file = open(argv[1], O_RDONLY); //read only
count = 0;
j = 0;
// compute size to map
size = get_size_to_map(OBIDMSColumn_file);
// map the data
map = obi_map_read_only(OBIDMSColumn_file, HEADER_SIZE, size);
// sum the counts
for (i=0; i<size; i++)
{
c = map[i];
if (c != SEPARATOR) // reading lines
{
num_str[j] = c;
j++;
}
else if (c == SEPARATOR) // end of a line
{
num_int = atoi(num_str); // turn number from character string to int
count = count + num_int; // add the number to the sum
j = 0;
num_str[j] = '\0';
}
}
// print the final count of sequences
fprintf(stderr, "Sequence count = %d\n", count);
// unmap
obi_unmap(size);
// close file
close(OBIDMSColumn_file);
return(0);
}

46
src/obidmscolumn.c Normal file
View File

@ -0,0 +1,46 @@
/****************************************************************************
* OBIDMSColumn functions *
****************************************************************************/
/**
* @file obidmscolumn.h
* @author Celine Mercier
* @date 22 May 2015
* @brief Functions for the shared elements of all the OBIColumn structures.
*/
#include <stdio.h>
#include <stdlib.h>
#include <sys/mman.h> /* mmap() is defined in this header */
/**
* @brief Reads a memory block from a file.
*
* * @param file The file that should be mapped.
* * @param size The number of bytes that should be mapped.
* * @param start The position at which the mapping should start.
* @return A pointer on the first element of the block.
*/
char* obi_map_read_only(int file, int start, int size)
{
char* map;
map = (char*) mmap(0, size, PROT_READ, MAP_PRIVATE, file, start);
if (map == MAP_FAILED)
{
fprintf(stderr, "\nmmap read-only error\n");
exit(1);
}
return(map);
}
/**
* @brief Unmaps a memory block.
*
* * @param size The size of the block that should be unmapped.
*/
void obi_unmap(int size)
{
munmap(0, size);
}

63
src/obidmscolumn.h Normal file
View File

@ -0,0 +1,63 @@
/****************************************************************************
* OBIDMSColumn header file *
****************************************************************************/
/**
* @file obidmscolumn.h
* @author Celine Mercier
* @date 12 May 2015
* @brief Header file for the shared elements of all the OBIColumn structures.
*/
#ifndef OBIDMSCOLUMN_H_
#define OBIDMSCOLUMN_H_
#include <stdio.h>
#include <sys/types.h>
#include <unistd.h>
#include <stdbool.h>
#include "obitypes.h"
/**
* @brief Value separator in OBIDMSColumn files.
*/
static const char SEPARATOR = '\n';
/**
* @brief Header size in OBIDMSColumn files.
*/
static const int HEADER_SIZE = 4096;
/**
* @brief Number of bytes to map
*/
static const int BUFFER_SIZE = 4;
/**
* @brief OBIColumnHeader structure.
*/
typedef struct OBIDMSColumnHeader {
bool endyan_byte_order; /**< endyan byte order */
int header_size_value; /**< size of the header: a multiple of PAGESIZE */
bool file_status; /**< file status : 0 is Open and 1 is Closed */
pid_t PID; /**< PID of the process that created the file and is
the only one allowed to modify it if it is open */
int line_count; /**< number of lines of data */
OBIType_t data_type; /**< type of the data */
char* creation_date; /**< date of creation of the file */
int version; /**< version of the OBIColumn */
char* comments; /**< comments */
} OBIDMSColumnHeader_t, *OBIDMSColumnHeader_p;
char* obi_map_read_only(int file, int start, int size);
void obi_unmap(int size);
#endif /* OBIDMSCOLUMN_H_ */

81
src/obiintcolumn.h
View File

@ -1,81 +0,0 @@
/****************************************************************************
* OBIIntColumn header file *
****************************************************************************/
/**
* @file obiintcolumn.h
* @author Celine Mercier
* @date 11 May 2015
* @brief Header file for the OBIIntColumn structure.
*/
#ifndef OBIINTCOLUMN_H_
#define OBIINTCOLUMN_H_
#include <stdio.h>
#include <stdlib.h>
#include "obicolumn.h"
/**
* @brief OBIIntColumn stucture.
*/
typedef struct OBIIntcolumn {
} OBIIntColumn_t, *OBIIntColumn_p;
/**
* @brief Reads a memory block from an OBIIntColumn file.
*
* @param OBIIntColumn The OBIColumn that should be read.
* @return A pointer on the first element of the block.
*/
int* readMemoryBlock(int* OBIIntcolumn, int startingPosition, int memoryBlockSize);
/**
* @brief Writes a memory block line by line at the end of a file.
*/
void writeMemoryBlock(int* OBIIntcolumn, int memoryBlockSize, char* fileName);
/**
* @brief Returns one line of an OBIColumn.
*/
int readLine(int line);
/**
* @brief Creates a new file for a new version of the OBIColumn.
*/
void newColumn(OBIIntcolumn* column);
/**
* @brief malloc for OBIIntColumn.
*/
int* OBIIntMalloc();
/**
* @brief realloc for OBIIntColumn.
*/
int* OBIIntRealloc();
/**
* @brief free for OBIIntColumn.
*/
void OBIIntFree();
#endif /* OBIINTCOLUMN_H_ */

50
src/obitypes.h Normal file
View File

@ -0,0 +1,50 @@
/****************************************************************************
* OBITypes header file *
****************************************************************************/
/**
* @file obitypes.h
* @author Celine Mercier
* @date 18 May 2015
* @brief Header file for the OBITypes.
*/
#ifndef OBITYPES_H_
#define OBITYPES_H_
#include <stdio.h>
#define OBIInt_NA (INT32_MIN)
#define OBIFloat_NA (NAN)
#define OBIIdx_NA (SIZE_MAX)
/**
* @brief enum for the boolean OBIType.
*/
typedef enum {
FALSE = 0,
TRUE = 1,
OBIBool_NA = 2
} OBIBool_t, *OBIBool_p; /**< a boolean true/false value */
/**
* @brief enum OBITypes for the data type of the OBIColumns.
*/
typedef enum OBIType {
OBI_VOID = 0, /**< data type not specified */
OBIInt_t, /**< a signed integer value (C type : int32_t) */
OBIFloat_t, /**< a floating value (C type : double) */
OBIChar_t, /**< a character (C type : char) */
OBIIdx_t /**< an index in a data structure (C type : size_t) */
} OBIType_t, *OBIType_p;
typedef int32_t obiint_t;
typedef double obifloat_t;
typedef char obichar_t;
typedef size_t obiidx_t;
#endif /* OBITYPES_H_ */

8
test_data/count@0.odc Normal file
View File

@ -0,0 +1,8 @@
...............................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................
2
3
1
8
4
4
2