obitools/obitools3
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 66 Wiki Activity

Made the OBITools3 more 'empty file friendly'

Browse Source
This commit is contained in:
MercierC
2021-03-15 16:25:41 +13:00
parent d48aed38d4
commit 425fe25bd2
4 changed files with 90 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
python/obitools3/commands/alignpairedend.pyx
View File

@ -205,19 +205,25 @@ def run(config):
if type(entries) == list:
forward = entries[0]
reverse = entries[1]
aligner = Kmer_similarity(forward, \
view2=reverse, \
kmer_size=config['alignpairedend']['kmersize'], \
reversed_column=None)
if len(forward) == 0 or len(reverse) == 0:
aligner = None
else:
aligner = Kmer_similarity(forward, \
view2=reverse, \
kmer_size=config['alignpairedend']['kmersize'], \
reversed_column=None)
else:
aligner = Kmer_similarity(entries, \
column2=entries[REVERSE_SEQUENCE_COLUMN], \
qual_column2=entries[REVERSE_QUALITY_COLUMN], \
kmer_size=config['alignpairedend']['kmersize'], \
reversed_column=entries[b'reversed']) # column created by the ngsfilter tool
if len(entries) == 0:
aligner = None
else:
aligner = Kmer_similarity(entries, \
column2=entries[REVERSE_SEQUENCE_COLUMN], \
qual_column2=entries[REVERSE_QUALITY_COLUMN], \
kmer_size=config['alignpairedend']['kmersize'], \
reversed_column=entries[b'reversed']) # column created by the ngsfilter tool
ba = alignmentIterator(entries, aligner)
i = 0
for ali in ba:
@ -251,7 +257,7 @@ def run(config):
pb(i, force=True)
print("", file=sys.stderr)
if kmer_ali :
if kmer_ali and aligner is not None:
aligner.free()
# Save command config in View and DMS comments

1
python/obitools3/commands/ngsfilter.pyx
View File

@ -645,6 +645,7 @@ def run(config):
g = 0
u = 0
i = 0
no_tags = config['ngsfilter']['notags']
try:
for i in range(entries_len):

6
python/obitools3/parsers/universal.pyx
View File

@ -53,7 +53,11 @@ def entryIteratorFactory(lineiterator,
i = iterator
first=next(i)
try:
first=next(i)
except StopIteration:
first=""
pass
format=b"tabular"

128
src/obi_clean.c
View File

@ -229,6 +229,8 @@ int obi_clean(const char* dms_name,
return -1;
}
seq_count = (i_view->infos)->line_count;
// Open the sequence column
if (strcmp((i_view->infos)->view_type, VIEW_TYPE_NUC_SEQS) == 0)
iseq_column = obi_view_get_column(i_view, NUC_SEQUENCE_COLUMN);
@ -245,7 +247,7 @@ int obi_clean(const char* dms_name,
}
// Open the sample column if there is one
if ((strcmp(sample_column_name, "") == 0) || (sample_column_name == NULL))
if ((strcmp(sample_column_name, "") == 0) || (sample_column_name == NULL) || (seq_count == 0))
{
fprintf(stderr, "Info: No sample information provided, assuming one sample.\n");
sample_column = obi_view_get_column(i_view, COUNT_COLUMN);
@ -340,66 +342,67 @@ int obi_clean(const char* dms_name,
return -1;
}
// Build kmer tables
ktable = hash_seq_column(i_view, iseq_column, 0);
if (ktable == NULL)
if (seq_count > 0)
{
obi_set_errno(OBI_CLEAN_ERROR);
obidebug(1, "\nError building kmer tables before aligning");
return -1;
}
// Build kmer tables
ktable = hash_seq_column(i_view, iseq_column, 0);
if (ktable == NULL)
{
obi_set_errno(OBI_CLEAN_ERROR);
obidebug(1, "\nError building kmer tables before aligning");
return -1;
}
seq_count = (i_view->infos)->line_count;
// Allocate arrays for sample counts otherwise reading in mapped files takes longer
complete_sample_count_array = (int*) malloc(seq_count * sample_count * sizeof(int));
if (complete_sample_count_array == NULL)
{
obi_set_errno(OBI_MALLOC_ERROR);
obidebug(1, "\nError allocating memory for the array of sample counts, size: %lld", seq_count * sample_count * sizeof(int));
return -1;
}
for (samp=0; samp < sample_count; samp++)
{
for (k=0; k<seq_count; k++)
complete_sample_count_array[k+(samp*seq_count)] = obi_get_int_with_elt_idx_and_col_p_in_view(i_view, sample_column, k, samp);
}
// Allocate arrays for blobs otherwise reading in mapped files takes longer
blob_array = (Obi_blob_p*) malloc(seq_count * sizeof(Obi_blob_p));
if (blob_array == NULL)
{
obi_set_errno(OBI_MALLOC_ERROR);
obidebug(1, "\nError allocating memory for the array of blobs");
return -1;
}
for (k=0; k<seq_count; k++)
{
blob_array[k] = obi_get_blob_with_elt_idx_and_col_p_in_view(i_view, iseq_column, k, 0);
}
// Allocate alignment result array (byte at 0 if not aligned yet,
// 1 if sequence at index has a similarity above the threshold with the current sequence,
// 2 if sequence at index has a similarity below the threshold with the current sequence)
alignment_result_array = (byte_t*) calloc(seq_count, sizeof(byte_t));
if (alignment_result_array == NULL)
{
obi_set_errno(OBI_MALLOC_ERROR);
obidebug(1, "\nError allocating memory for alignment result array");
return -1;
}
// Initialize all sequences to singletons or NA if no sequences in that sample
for (k=0; k<seq_count; k++)
{
// Allocate arrays for sample counts otherwise reading in mapped files takes longer
complete_sample_count_array = (int*) malloc(seq_count * sample_count * sizeof(int));
if (complete_sample_count_array == NULL)
{
obi_set_errno(OBI_MALLOC_ERROR);
obidebug(1, "\nError allocating memory for the array of sample counts, size: %lld", seq_count * sample_count * sizeof(int));
return -1;
}
for (samp=0; samp < sample_count; samp++)
{
if (obi_get_int_with_elt_idx_and_col_p_in_view(i_view, sample_column, k, samp) != OBIInt_NA) // Only initialize samples where there are some sequences
for (k=0; k<seq_count; k++)
complete_sample_count_array[k+(samp*seq_count)] = obi_get_int_with_elt_idx_and_col_p_in_view(i_view, sample_column, k, samp);
}
// Allocate arrays for blobs otherwise reading in mapped files takes longer
blob_array = (Obi_blob_p*) malloc(seq_count * sizeof(Obi_blob_p));
if (blob_array == NULL)
{
obi_set_errno(OBI_MALLOC_ERROR);
obidebug(1, "\nError allocating memory for the array of blobs");
return -1;
}
for (k=0; k<seq_count; k++)
{
blob_array[k] = obi_get_blob_with_elt_idx_and_col_p_in_view(i_view, iseq_column, k, 0);
}
// Allocate alignment result array (byte at 0 if not aligned yet,
// 1 if sequence at index has a similarity above the threshold with the current sequence,
// 2 if sequence at index has a similarity below the threshold with the current sequence)
alignment_result_array = (byte_t*) calloc(seq_count, sizeof(byte_t));
if (alignment_result_array == NULL)
{
obi_set_errno(OBI_MALLOC_ERROR);
obidebug(1, "\nError allocating memory for alignment result array");
return -1;
}
// Initialize all sequences to singletons or NA if no sequences in that sample
for (k=0; k<seq_count; k++)
{
for (samp=0; samp < sample_count; samp++)
{
if (obi_set_char_with_elt_idx_and_col_p_in_view(o_view, status_column, k, samp, 's') < 0)
if (obi_get_int_with_elt_idx_and_col_p_in_view(i_view, sample_column, k, samp) != OBIInt_NA) // Only initialize samples where there are some sequences
{
obidebug(1, "\nError initializing all sequences to singletons");
return -1;
if (obi_set_char_with_elt_idx_and_col_p_in_view(o_view, status_column, k, samp, 's') < 0)
{
obidebug(1, "\nError initializing all sequences to singletons");
return -1;
}
}
}
}
@ -551,17 +554,20 @@ int obi_clean(const char* dms_name,
}
}
free_kmer_tables(ktable, seq_count);
free(complete_sample_count_array);
free(blob_array);
free(alignment_result_array);
if (seq_count > 0)
{
free_kmer_tables(ktable, seq_count);
free(complete_sample_count_array);
free(blob_array);
free(alignment_result_array);
}
fprintf(stderr, "\n");
if (stop)
return -1;
if (heads_only)
if (heads_only && (seq_count > 0))
{
line_selection = malloc((((o_view->infos)->line_count) + 1) * sizeof(index_t));
if (line_selection == NULL)
@ -635,7 +641,7 @@ int obi_clean(const char* dms_name,
}
// Flag the end of the line selection
if (heads_only)
if (heads_only && (seq_count > 0))
line_selection[l] = -1;
// Create new view with line selection if heads only