obitools/obitools3
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 66 Wiki Activity

LCS alignment: it is now checked that sequences are not longer than what

Browse Source 
a 16 bits integer can code for (as the LCS and alignment lengths are
kept in 16 bits registers)
This commit is contained in:
Celine Mercier
2016-12-22 17:06:23 +01:00
parent 30e4359c85
commit 8e92bf6dac
1 changed files with 16 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
src/sse_banded_LCS_alignment.c
View File

@ -900,6 +900,14 @@ double generic_sse_banded_lcs_align(char* seq1, char* seq2, double threshold, bo
lmin = l1; lmin = l1;
} }
// Check that the sequences are not greater than what can be aligned using the 16 bits registers (as the LCS and alignment lengths are kept on 16 bits)
if (lmax > SHRT_MAX)
{
obi_set_errno(OBI_ALIGN_ERROR);
obidebug(1, "\nError: can not align sequences longer than %d (as the LCS and alignment lengths are kept on 16 bits)", SHRT_MAX);
return 0; // TODO DOUBLE_MIN to flag error
}
// If the score is expressed as a normalized distance, get the corresponding identity // If the score is expressed as a normalized distance, get the corresponding identity
if (!similarity_mode && normalize) if (!similarity_mode && normalize)
threshold = 1.0 - threshold; threshold = 1.0 - threshold;
@ -995,6 +1003,14 @@ double obiblob_sse_banded_lcs_align(Obi_blob_p seq1, Obi_blob_p seq2, double thr
lmin = l1; lmin = l1;
} }
// Check that the sequences are not greater than what can be aligned using the 16 bits registers (as the LCS and alignment lengths are kept on 16 bits)
if (lmax > SHRT_MAX)
{
obi_set_errno(OBI_ALIGN_ERROR);
obidebug(1, "\nError: can not align sequences longer than %d (as the LCS and alignment lengths are kept on 16 bits)", SHRT_MAX);
return 0; // TODO DOUBLE_MIN to flag error
}
// If the score is expressed as a normalized distance, get the corresponding identity // If the score is expressed as a normalized distance, get the corresponding identity
if (!similarity_mode && normalize) if (!similarity_mode && normalize)
threshold = 1.0 - threshold; threshold = 1.0 - threshold;