Refactor indexing, sampling, and entropy documentation
This commit introduces new theoretical frameworks for k-mer indexing, pair coverage sampling, and sibling partition unification. It also implements new algorithms for calculating family entropy and statistical tests for overlap analysis. Documentation is updated across the board to reflect these new features and theoretical concepts.
This commit is contained in:
1 parent
95bbc0b2ae
commit
0d12494f1a
12 files changed
+1182
-56
No files matched your search
@@ -71,6 +71,54 @@
|
||||
</else>
|
||||
</choose>
|
||||
</macro>
|
||||
<macro name="access-detail">
|
||||
<group delimiter=" ">
|
||||
<text variable="URL" prefix="Available at: "/>
|
||||
<group prefix="(" suffix=")" delimiter=" ">
|
||||
<text term="accessed"/>
|
||||
<date variable="accessed">
|
||||
<date-part name="day" suffix=" "/>
|
||||
<date-part name="month" form="long" suffix=" "/>
|
||||
<date-part name="year"/>
|
||||
</date>
|
||||
</group>
|
||||
</group>
|
||||
</macro>
|
||||
<macro name="access">
|
||||
<choose>
|
||||
<if type="webpage post post-weblog entry entry-encyclopedia dataset software report manuscript document" match="any">
|
||||
<text macro="access-detail"/>
|
||||
</if>
|
||||
<else-if variable="accessed">
|
||||
<text macro="access-detail"/>
|
||||
</else-if>
|
||||
</choose>
|
||||
</macro>
|
||||
<macro name="access-fallback">
|
||||
<group delimiter=" ">
|
||||
<choose>
|
||||
<if variable="URL">
|
||||
<text macro="access-detail" suffix="."/>
|
||||
</if>
|
||||
<else-if variable="publisher">
|
||||
<text variable="publisher" prefix="Available at: " suffix="."/>
|
||||
</else-if>
|
||||
</choose>
|
||||
<text variable="note" suffix="."/>
|
||||
</group>
|
||||
</macro>
|
||||
<macro name="issued-year">
|
||||
<choose>
|
||||
<if variable="issued">
|
||||
<date variable="issued">
|
||||
<date-part name="year"/>
|
||||
</date>
|
||||
</if>
|
||||
<else>
|
||||
<text term="no date" form="short"/>
|
||||
</else>
|
||||
</choose>
|
||||
</macro>
|
||||
<macro name="publisher">
|
||||
<group delimiter=", ">
|
||||
<text variable="publisher"/>
|
||||
@@ -85,9 +133,7 @@
|
||||
<layout prefix="(" suffix=")" delimiter="; ">
|
||||
<group delimiter=" ">
|
||||
<text macro="author-short"/>
|
||||
<date variable="issued">
|
||||
<date-part name="year"/>
|
||||
</date>
|
||||
<text macro="issued-year"/>
|
||||
<group>
|
||||
<label variable="locator" form="short"/>
|
||||
<text variable="locator" prefix=" "/>
|
||||
@@ -102,9 +148,7 @@
|
||||
</sort>
|
||||
<layout>
|
||||
<text macro="author" suffix=" "/>
|
||||
<date variable="issued" prefix="(" suffix=")">
|
||||
<date-part name="year"/>
|
||||
</date>
|
||||
<text macro="issued-year" prefix="(" suffix=")"/>
|
||||
<choose>
|
||||
<if type="bill book graphic legal_case legislation motion_picture report song" match="any">
|
||||
<group suffix=".">
|
||||
@@ -112,6 +156,7 @@
|
||||
<text macro="editor-translator" prefix=" "/>
|
||||
</group>
|
||||
<text prefix=" " suffix="." macro="publisher"/>
|
||||
<text macro="access" prefix=" " suffix="."/>
|
||||
</if>
|
||||
<else-if type="thesis" match="any">
|
||||
<group suffix=".">
|
||||
@@ -119,6 +164,7 @@
|
||||
<text variable="genre" prefix=" " suffix=" Thesis."/>
|
||||
<text prefix=" " macro="publisher"/>
|
||||
</group>
|
||||
<text macro="access" prefix=" " suffix="."/>
|
||||
</else-if>
|
||||
<else-if type="chapter paper-conference" match="any">
|
||||
<text macro="title" prefix=" "/>
|
||||
@@ -138,6 +184,18 @@
|
||||
</group>
|
||||
</group>
|
||||
</else-if>
|
||||
<else-if type="webpage post post-weblog entry entry-encyclopedia dataset software" match="any">
|
||||
<group suffix=".">
|
||||
<text macro="title" prefix=" "/>
|
||||
<text macro="editor-translator" prefix=" "/>
|
||||
</group>
|
||||
<group prefix=" " suffix=".">
|
||||
<text variable="container-title" text-case="title" font-style="italic"/>
|
||||
<text variable="genre" prefix=", "/>
|
||||
<text macro="publisher" prefix=". "/>
|
||||
</group>
|
||||
<text macro="access" prefix=" " suffix="."/>
|
||||
</else-if>
|
||||
<else>
|
||||
<group suffix=".">
|
||||
<text macro="title" prefix=" "/>
|
||||
@@ -150,6 +208,14 @@
|
||||
</group>
|
||||
<text variable="page" prefix=", "/>
|
||||
</group>
|
||||
<choose>
|
||||
<if variable="container-title">
|
||||
<text macro="access" prefix=" " suffix="."/>
|
||||
</if>
|
||||
<else>
|
||||
<text macro="access-fallback" prefix=" "/>
|
||||
</else>
|
||||
</choose>
|
||||
</else>
|
||||
</choose>
|
||||
</layout>
|
||||
|
||||
@@ -768,3 +768,92 @@
|
||||
pages = {535--563},
|
||||
year = {1988}
|
||||
}
|
||||
@ARTICLE{Le-Cam1960-pe,
|
||||
title = "{An approximation theorem for the Poisson binomial distribution}",
|
||||
author = "Le Cam, Lucien",
|
||||
journal = "Pacific Journal of Mathematics",
|
||||
publisher = "Mathematical Sciences Publishers",
|
||||
volume = 10,
|
||||
number = 4,
|
||||
pages = "1181--1197",
|
||||
month = dec,
|
||||
year = 1960,
|
||||
url = "http://dx.doi.org/10.2140/pjm.1960.10.1181",
|
||||
doi = "10.2140/pjm.1960.10.1181",
|
||||
issn = "0030-8730,1945-5844"
|
||||
}
|
||||
|
||||
@ARTICLE{Hoeffding1956-bx,
|
||||
title = "{On the distribution of the number of successes in independent
|
||||
trials}",
|
||||
author = "Hoeffding, Wassily",
|
||||
journal = "The Annals of Mathematical Statistics",
|
||||
publisher = "Institute of Mathematical Statistics",
|
||||
volume = 27,
|
||||
number = 3,
|
||||
pages = "713--721",
|
||||
abstract = "Let $S$ be the number of successes in $n$ independent trials, and
|
||||
let $p_j$ denote the probability of success in the $j$th trial,
|
||||
$j = 1, 2, \cdots, n$ (Poisson trials). We consider the problem
|
||||
of finding the maximum and the minimum of $Eg(S),$ the expected
|
||||
value of a given real-valued function of $S,$ when $ES = np$ is
|
||||
fixed. It is well known that the maximum of the variance of $S$
|
||||
is attained when $p_1 = p_2 = \cdots = p_n = p.$ This can be
|
||||
interpreted as showing that the variability in the number of
|
||||
successes is highest when the successes are equally probable
|
||||
(Bernoulli trials). This interpretation is further supported by
|
||||
the following two theorems, proved in this paper. If $b$ and $c$
|
||||
are two integers, $0 \leqq b \leqq np \leqq c \leqq n,$ the
|
||||
probability $P(b \leqq S \leqq c)$ attains its minimum if and
|
||||
only if $p_1 = p_2 = \cdots = p_n = p,$ unless $b = 0$ and $c =
|
||||
n$ (Theorem 5, a corollary of Theorem 4, which gives the maximum
|
||||
and the minimum of $P(S \leqq c)$). If $g$ is a strictly convex
|
||||
function, $Eg(S)$ attains its maximum if and only if $p_1 = p_2 =
|
||||
\cdots = p_n = p$ (Theorem 3). These results are obtained with
|
||||
the help of two theorems concerning the extrema of the expected
|
||||
value of an arbitrary function $g(S)$ under the condition $ES =
|
||||
np.$ Theorem 1 gives necessary conditions for the maximum and the
|
||||
minimum of $Eg(S).$ Theorem 2 gives a partial characterization of
|
||||
the set of points at which an extremum is attained. Corollary 2.1
|
||||
states that the maximum and the minimum are attained when $p_1,
|
||||
p_2, \cdots, p_n$ take on, at most, three different values, only
|
||||
one of which is distinct from 0 and 1. Applications of Theorems 3
|
||||
and 5 to problems of estimation and testing are pointed out in
|
||||
Section 5.",
|
||||
month = sep,
|
||||
year = 1956,
|
||||
url = "http://dx.doi.org/10.1214/aoms/1177728178",
|
||||
doi = "10.1214/aoms/1177728178",
|
||||
issn = "0003-4851,2168-8990",
|
||||
language = "en"
|
||||
}
|
||||
@ARTICLE{Gleser1975-yu,
|
||||
title = "{On the distribution of the number of successes in independent
|
||||
trials}",
|
||||
author = "Gleser, Leon Jay",
|
||||
journal = "Annals of Probability",
|
||||
publisher = "Institute of Mathematical Statistics",
|
||||
volume = 3,
|
||||
number = 1,
|
||||
pages = "182--188",
|
||||
abstract = "Let $S$ be the number of successes in $n$ independent Bernoulli
|
||||
trials, where $p_j$ is the probability of success on the $j$th
|
||||
trial. Let $\mathbf{p} = (p_1, p_2, \cdots, p_n)$, and for any
|
||||
integer $c, 0 \leqq c \leqq n$, let $H(c \mid \mathbf{p}) = P\{S
|
||||
\leqq c\}$. Let $\mathbf{p}^{(1)}$ be one possible choice of
|
||||
$\mathbf{p}$ for which $E(S) = \lambda$. For any $n \times n$
|
||||
doubly stochastic matrix $\Pi$, let $\mathbf{p}^{(2)} =
|
||||
\mathbf{p}^{(1)}\Pi$. Then in the present paper it is shown that
|
||||
$H(c \mid \mathbf{p}^{(1)}) \leqq H(c \mid \mathbf{p}^{(2)})$ for
|
||||
$0 \leqq c \leqq \lbrack\lambda - 2\rbrack$, and $H(c \mid
|
||||
\mathbf{p}^{(1)}) \geqq H(c \mid \mathbf{p}^{(2)})$ for
|
||||
$\lbrack\lambda + 2\rbrack \leqq c \leqq n$. These results
|
||||
provide a refinement of inequalities for $H(c \mid \mathbf{p})$
|
||||
obtained by Hoeffding [3]. Their derivation is achieved by
|
||||
applying consequences of the partial ordering of majorization.",
|
||||
month = feb,
|
||||
year = 1975,
|
||||
url = "http://dx.doi.org/10.2307/2959277",
|
||||
doi = "10.1214/aop/1176996461",
|
||||
issn = "0091-1798,2168-894X"
|
||||
}
|
||||
@@ -22,24 +22,102 @@ This gives the hard representation limit: $k \leq 32.$ Furthermore, OBIKmer only
|
||||
|
||||
### K-mer specificity
|
||||
|
||||
A simple way to illustrate the effect of k-mer size is to consider a random DNA sequence in which the four nucleotides are independent and equally probable. There are
|
||||
One simple way to illustrate the effect of k-mer size is to consider a random DNA sequence in which the four nucleotides are independent and equally probable. There are $4^k$ possible k-mers, so a particular k-mer is expected to occur once every $4^k$ positions on average. In a sequence of length $G$, the expected number of occurrences of a particular k-mer is approximately $G \cdot 4^{-k}$. When k is large enough to ensure $4^k >> G$, the chance of a second occurrence of the same k-mer becomes close to zero. Consequently, observing the same k-mer twice can only be explained by a homology relationship, except for low-complexity k-mers, which are overrepresented in genomes.
|
||||
|
||||
### Estimating False discovery rate on homology inference
|
||||
|
||||
OBIKmer uses k-mers long enough that their occurrence in multiple indexed genomes provides evidence of homology. The corresponding false discovery rate (FDR) can be expressed in terms of the probability of observing a k-mer in at least $R$ of the $S$ indexed genomes under a purely random model.
|
||||
|
||||
#### The random model
|
||||
|
||||
The random model, or null hypothesis, assumes that the $S$ genomes are unrelated and that each genome is an independent random sequence in which each nucleotide occurs with equal probability. Rejecting this hypothesis for a given k-mer indicates that its presence in multiple genomes is unlikely to be explained by chance alone, providing evidence for homology.
|
||||
|
||||
For a genome of size $G$, the number of occurrences of a given canonical k-mer $K$ is approximated by a Poisson distribution with parameter
|
||||
|
||||
$$ \lambda = 2\frac{G}{4^k}, $$
|
||||
|
||||
which is also the expected number of occurrences under the random model. The factor of two accounts for k-mer canonicalisation, which halves the k-mer space when $k$ is odd.
|
||||
|
||||
The probability of observing at least one occurrence of this k-mer is therefore
|
||||
|
||||
$$
|
||||
4^k
|
||||
\begin{aligned}
|
||||
P(n_K > 0) &= 1-e^{-\lambda} \\
|
||||
&= 1-e^{-2G/4^k}.
|
||||
\end{aligned}
|
||||
$$
|
||||
|
||||
possible k-mers, and a particular k-mer is therefore expected to occur once every $4^k$ positions on average. In a sequence of length $G$, the expected number of occurrences of a particular k-mer is approximately
|
||||
Assuming equal genome lengths, the number of genomes containing at least one occurrence of $K$ among the $S$ sequences follows a binomial distribution. The probability of observing $K$ in at least $r$ genomes is therefore
|
||||
|
||||
|
||||
$$
|
||||
E[n_K] \simeq \frac{G}{4^k}.
|
||||
P(X \geq r)=\sum_{j=r}^{S}\binom{S}{j}\left(1-e^{-2G/4^k}\right)^j\left(e^{-2G/4^k}\right){S-j}.
|
||||
$$
|
||||
|
||||
When $4^k\gg G$, this quantity is also a good approximation of the probability that an observed k-mer will occur at least once by chance elsewhere in a sequence of length $G$. It therefore provides a simple measure of the risk of a chance recurrence of an observed k-mer:
|
||||
This sum is the regularized incomplete beta function $I_p(r,\,S-r+1)$, where
|
||||
|
||||
$$
|
||||
P_{\mathrm{chance}}(k,G) \simeq \frac{G}{4^k}.
|
||||
p = 1-e^{-2G/4^k}.
|
||||
$$
|
||||
|
||||
Different genome sizes
|
||||
|
||||
When genome sizes differ, the probability of observing a given k-mer also differs between genomes. A simple approximation is to replace the individual genome sizes $G_i$ by their mean
|
||||
|
||||
$$ \bar G = \frac{1}{S}\sum_{i=1}^{S}G_i. $$
|
||||
|
||||
This gives a common probability
|
||||
|
||||
$$ \bar p = 1-e^{-2\bar G/4^k}, $$
|
||||
|
||||
and the number of genomes containing the k-mer can then be approximated by a binomial distribution:
|
||||
|
||||
$$ P(X\geq r)=\sum_{j=r}^{S}\binom{S}{j}\bar p^j(1-\bar p)^{S-j}. $$
|
||||
|
||||
This approximation is conservative for the upper tail. Hoeffding [-@Hoeffding1956-bx, p. 717, Eq. 26] shows that, for a fixed mean, the binomial upper tail is at least as large as the corresponding Poisson-binomial tail for
|
||||
|
||||
$$ r\geq S\bar p+1. $$
|
||||
|
||||
Using the mean genome size therefore provides a conservative estimate of the FDR.
|
||||
|
||||
When genome sizes differ substantially, however, this approximation can become overly conservative. In that case, the exact Poisson-binomial distribution can be computed directly from the individual genome-specific probabilities $p_i$.
|
||||
|
||||
### Exact tail probability
|
||||
|
||||
With genome-specific probabilities $p_i$, the number of genomes containing the k-mer follows a Poisson-binomial distribution. Writing the probability of exactly $j$ presences as a sum over all subsets of $j$ genomes would be impractical, since there can be up to $2^S$ subsets. The distribution can instead be obtained by adding the genomes one at a time.
|
||||
|
||||
Let $d_j$ be the probability that exactly $j$ of the genomes processed so far contain the k-mer. Adding genome $i$ leaves $j$ unchanged if the genome does not contain the k-mer, and moves $j-1$ to $j$ if it does:
|
||||
|
||||
$$
|
||||
\left \{
|
||||
\begin{aligned}
|
||||
d'_j &= d_j(1-p_i) + d_{j-1}p_i, \\
|
||||
d_0&=1
|
||||
\end{aligned}
|
||||
\right .
|
||||
$$
|
||||
|
||||
before processing the first genome.
|
||||
|
||||
The table $d$ only needs to record how many genomes contain the k-mer, not which ones, reducing the computational cost to $O(S^2)$. The result is independent of the order in which the genomes are processed and of the k-mer itself, so it can be computed once per index. The upper tail is then
|
||||
|
||||
$$
|
||||
P(X\geq r)=1-\sum_{j=0}^{r-1}d_j.
|
||||
$$
|
||||
|
||||
Only $d_0,\ldots,d_{r-1}$ are needed, reducing the cost to $O(Sr)$.
|
||||
|
||||
When all $p_i$ are small, as is typically the case for large $k$, the Poisson-binomial distribution can also be approximated by a Poisson distribution with mean
|
||||
|
||||
$$
|
||||
\lambda=\sum_{i=1}^{S}p_i.
|
||||
$$
|
||||
|
||||
The approximation error is bounded in terms of $\sum_i p_i^2$ by Le Cam's inequality [@LeCam1960-pe].
|
||||
|
||||
|
||||
*******
|
||||
|
||||
This probability decreases by a factor of four for every additional nucleotide in the k-mer. Conversely, it increases linearly with the amount of sequence searched.
|
||||
|
||||
This simple model already shows why very short k-mers provide limited specificity for genome-scale analyses. With $k=11$,
|
||||
@@ -189,34 +267,150 @@ Le rapport est exactement $S_1 / P(R=2\mid w) = 4^{k}$ — c'est-à-dire le nomb
|
||||
Une seule réserve sur le texte proposé tel quel : $p_i\simeq1-\exp(-G_i/4^k)$ souffre du même biais que celui identifié plus tôt dans notre discussion — un mot de longueur $k$ n'est pas indépendant à chaque position (chevauchement), donc ce $p_i$ est un peu optimiste (légèrement trop haut) pour les mêmes raisons que $E[C]$ l'était. Négligeable pour $k\geq21$, plus discutable à $k=11$.
|
||||
|
||||
|
||||
## FDR
|
||||
## Chance sharing across genomes and the FDR of homology
|
||||
|
||||
C'est exactement la bonne question à se poser — ni la p-value par hit, ni la probabilité globale « y a-t-il au moins une collision quelque part », mais **la fraction de tes appels d'homologie qui sont du bruit**. C'est un problème de type FDR (*false discovery rate*), et il se construit directement à partir des deux briques qu'on a déjà :
|
||||
The formulas above ask how likely a match is between two sequences. The question OBIKmer actually has to answer is different: a k-mer is found in $r$ of the $S$ indexed genomes, and one wants to know whether this sharing can be explained by chance or whether it is evidence of homology. This section builds a null model for that question and turns it into a false discovery rate (FDR) for the call "a k-mer shared by at least $r$ genomes is homologous".
|
||||
|
||||
### The null model
|
||||
|
||||
The null hypothesis is that the $S$ genomes are unrelated, that is, that each one is an independent random sequence. Rejecting it for a k-mer means that chance alone cannot explain why the k-mer is shared, which is the signal of homology one is looking for.
|
||||
|
||||
The model only records whether a k-mer is present in a genome, not how many times. Two consequences follow. First, repeats inside a genome are irrelevant, and the size $G_i$ of genome $i$ is its number of **distinct canonical k-mers**. Second, a repeat family shared between genomes is paralogy, which is a form of homology; it is a true positive and not a false one. The only k-mers that could be shared by composition rather than by descent are low-complexity ones, which OBIKmer removes beforehand with its entropy filter.
|
||||
|
||||
Everything takes place in canonical space. Because $k$ is odd, no k-mer equals its own reverse complement, so the $4^k$ k-mers pair up perfectly and the space contains exactly
|
||||
|
||||
$$
|
||||
\widehat{FDR}(r) = \frac{\text{nombre de k-mers *attendus par hasard* avec }R\geq r}{\text{nombre de k-mers *observés* avec }R\geq r}
|
||||
N=\frac{4^k}{2}
|
||||
$$
|
||||
|
||||
**Le numérateur** (bruit attendu) : c'est la généralisation, au seuil $\geq r$ (pas seulement « partagé par exactement ces $r$ génomes »), de la quantité pont qu'on vient d'identifier — le nombre de k-mers testés $M$ multiplié par la probabilité par-mot sous le null :
|
||||
canonical k-mers. A given canonical k-mer is present in genome $i$ by chance with probability
|
||||
|
||||
$$
|
||||
E[\text{faux positifs}] \simeq M \cdot P_0(R\geq r) = M\sum_{j=r}^{S}\binom{S}{j}p^{j}(1-p)^{S-j}
|
||||
p_i = 1-\left(1-\frac1N\right)^{G_i}\simeq \frac{G_i}{N}=\frac{2\,G_i}{4^k}.
|
||||
$$
|
||||
|
||||
avec $p\simeq G/4^k$ (ou la version Poisson-binomiale à $G_i$ inégaux) et $M$ le nombre de k-mers canoniques réellement testés (pas $4^k/2$ théorique, mais le nombre de k-mers **effectivement présents** dans ton index — c'est cette correction, plutôt que le $4^k$ théorique, qui rend l'estimation réaliste, puisque la plupart des $4^k$ mots possibles n'apparaissent jamais dans de vrais génomes).
|
||||
This is twice the $G/4^k$ used earlier in this page, because there $G$ is a number of positions on one strand, whereas $G_i$ here counts canonical k-mers, for which both strands are already merged.
|
||||
|
||||
**Le dénominateur** (bruit + vrai signal, observé) : le nombre de k-mers de ton index dont le masque de présence a une cardinalité $\geq r$ — une donnée que `FamilyMask`/`SiblingAnnex` dans `obikphylo::siblings` te donne déjà directement, sans aucun calcul supplémentaire.
|
||||
Let $R$ be the number of genomes, out of $S$, that contain a given canonical k-mer by chance. Genomes are independent under the null, so $R$ is a sum of independent presence indicators, each with its own probability $p_i$.
|
||||
|
||||
**Ce que ça donne concrètement.** Tu n'as besoin que de trois nombres, tous déjà disponibles ou calculables :
|
||||
1. $M$ = cardinalité totale de ton set de k-mers indexés (mesure directe de l'index)
|
||||
2. $N_{obs}(r)$ = nombre de familles/k-mers avec présence $\geq r$ (mesure directe, déjà dans `FamilyMask`)
|
||||
3. $P_0(R\geq r)$ = calculé depuis le modèle nul (le $p$ de l'approche « naïve » du tour précédent)
|
||||
### Exact tail probability
|
||||
|
||||
et $\widehat{FDR}(r) = M\cdot P_0(R\geq r) / N_{obs}(r)$ te dit directement : « sur les k-mers que j'ai retenus comme preuve d'homologie au seuil $r$, telle fraction (%) est attendue être du pur hasard ».
|
||||
When genome sizes differ, the $p_i$ differ, and $R$ follows a Poisson-binomial distribution. Writing the probability of exactly $j$ presences as a sum over all subsets of $j$ genomes would be hopeless, since there are up to $2^S$ subsets. The distribution is instead obtained by adding the genomes one at a time. Let $d_j$ be the probability that exactly $j$ of the genomes processed so far contain the k-mer. Adding genome $i$ leaves $j$ unchanged if the genome does not contain the k-mer, and moves $j-1$ to $j$ if it does:
|
||||
|
||||
**Pourquoi c'est mieux que ce qu'on avait avant** :
|
||||
- Le $P(K_{r,s}\geq k)$ de Karlin-Ost (tour précédent) répond à « y a-t-il *au moins un* faux positif quelque part » — binaire, ne dit rien sur combien.
|
||||
- La p-value par hit répond à « ce hit précis est-il suspect » — mot par mot, ne donne pas de vue d'ensemble.
|
||||
- Le $\widehat{FDR}$ répond directement à ta question : **quelle proportion** de l'ensemble retenu est due au hasard — c'est la quantité qu'on utiliserait pour dire « à ce seuil $r$, mon jeu de zones d'homologie candidates est fiable à 95 % » ou pour choisir le $r$ minimal qui ramène le $\widehat{FDR}$ sous un seuil acceptable, à $k$ fixé par la contrainte de représentation (64 bits).
|
||||
$$
|
||||
d'_j = d_j\,(1-p_i) + d_{j-1}\,p_i, \qquad d_0=1 \text{ before the first genome.}
|
||||
$$
|
||||
|
||||
Un point à trancher avec toi avant d'aller plus loin : pour $M$, préfères-tu le nombre théorique de k-mers canoniques possibles ($4^k/2$) ou le nombre réel de k-mers distincts dans ton index (qui donnerait un $\widehat{FDR}$ beaucoup plus conservateur et réaliste, mais nécessite de le lire depuis l'index au moment du calcul plutôt que de le dériver de $k$ seul) ?
|
||||
The table $d$ only needs to remember how many genomes were hit, not which ones, and this brings the cost down to $O(S^2)$. The result does not depend on the order in which the genomes are processed. It does not depend on the k-mer either, so it is computed once per index. The upper tail is then
|
||||
|
||||
$$
|
||||
P_0(R\geq r) = 1-\sum_{j=0}^{r-1} d_j .
|
||||
$$
|
||||
|
||||
Only $d_0,\dots,d_{r-1}$ are needed, so the cost is $O(S\,r)$.
|
||||
|
||||
### Approximation with equal genome sizes
|
||||
|
||||
If all genomes are given the same size, taken as the mean
|
||||
|
||||
$$
|
||||
\bar G=\frac1S\sum_{i=1}^{S}G_i, \qquad \bar p=\frac{2\bar G}{4^k},
|
||||
$$
|
||||
|
||||
then $R$ follows a plain binomial distribution and the tail is
|
||||
|
||||
$$
|
||||
P_0(R\geq r)\simeq\sum_{j=r}^{S}\binom{S}{j}\,\bar p^{\,j}(1-\bar p)^{S-j}.
|
||||
$$
|
||||
|
||||
This sum is the regularized incomplete beta function $I_{\bar p}(r,\,S-r+1)$, available in standard statistical libraries. If it is summed by hand, the terms are obtained one from the next by $t_{j+1}=t_j\,\frac{S-j}{j+1}\,\frac{\bar p}{1-\bar p}$.
|
||||
|
||||
A theorem of Hoeffding [@Hoeffding1956-bx] shows that this approximation errs on the safe side. For a fixed mean, the Poisson-binomial distribution is more concentrated than the binomial one, so for $r\geq S\bar p+1$ the binomial tail is never smaller than the exact tail. Using the mean genome size therefore overestimates the chance of sharing and can only make the FDR more pessimistic. Gleser [@Gleser1975-yu] refines this result: replacing the $p_i$ by any partial average of themselves, not only by their full mean, also moves the upper tail towards the binomial one, provided $r-1\geq S\bar p+2$. Grouping genomes of similar size is therefore safe in the same sense. What it loses is power when genome sizes differ by orders of magnitude, since one large genome then weighs as much as several small ones in $\bar p$.
|
||||
|
||||
When all $p_i$ are small, which is the case for large $k$, the number of hits is also well approximated by a Poisson distribution of mean $\lambda=\sum_i p_i$, with an error bounded by $\sum_i p_i^2$ [@Le-Cam1960-pe].
|
||||
|
||||
For the numerical evaluation, the tail that does not contain the mode $S\bar p$ should be summed directly. When $r>S\bar p$, computing the sum from $j=r$ upwards is accurate even for very small probabilities, whereas computing $1$ minus the sum of the lower terms loses all precision once the tail falls below the machine epsilon.
|
||||
|
||||
### False discovery rate
|
||||
|
||||
The FDR of the call "present in at least $r$ genomes" is the fraction of retained k-mers expected to be pure chance:
|
||||
|
||||
$$
|
||||
\widehat{\mathrm{FDR}}(r)=\min\!\left(1,\ \frac{N\,P_0(R\geq r)}{n_{\geq r}}\right).
|
||||
$$
|
||||
|
||||
The numerator is the number of k-mers expected to reach $r$ genomes by chance if the genomes were unrelated. Each of the $N$ canonical k-mers has probability $P_0(R\geq r)$ of doing so, and no further correction for multiple testing is needed because the number of tests is already in the product. The denominator $n_{\geq r}$ is the number of distinct k-mers actually observed in at least $r$ genomes, which is read directly from the index. The observed count mixes chance and true homology, so the ratio is an upper bound on the true FDR: it implicitly assumes that all k-mers are null, and it never underestimates the noise. The minimal $r$ for which $\widehat{\mathrm{FDR}}(r)$ falls below the accepted level, for example 5 %, is the sharing threshold to use.
|
||||
|
||||
As an illustration, take $k=21$ (so $N=2.2\times10^{12}$), $S=10$ genomes and $G_i=10^9$ distinct canonical k-mers each, hence $p=4.5\times10^{-4}$. The number of k-mers expected to be shared by chance is:
|
||||
|
||||
| $r$ | $N\,P_0(R\geq r)$ |
|
||||
| ---: | ---: |
|
||||
| 2 | $2\times10^{7}$ |
|
||||
| 3 | $2.5\times10^{4}$ |
|
||||
| 4 | $\approx 20$ |
|
||||
| 5 | $\approx 0.01$ |
|
||||
|
||||
Chance alone produces millions of pairwise shared 21-mers at this genome size, a few tens of thousands shared by three genomes, and almost none shared by five or more. Whether a given $r$ is acceptable then depends on how many k-mers the index actually reports at that level.
|
||||
|
||||
### Limits of the model
|
||||
|
||||
- The FDR is computed per k-mer. Neighbouring k-mers of a shared region overlap and are not independent, so a single homologous region contributes many correlated k-mers. A per-region FDR would count regions instead of k-mers and is not covered here.
|
||||
- The probability $p_i$ assumes equiprobable, independent nucleotides. The composition biases discussed earlier in this page violate this assumption, and the resulting error is not accounted for.
|
||||
|
||||
---
|
||||
|
||||
Je n'ai rien modifié. Voici la formalisation des deux points, à valider avant de l'écrire dans `kmers.md`.
|
||||
|
||||
## 1. FDR d'un ensemble de k-mers sélectionnés
|
||||
|
||||
**Estimation de l'espérance du nombre de faux positifs.** Soit $M$ le nombre de k-mers canoniques testés, $M = 4^k/2$ pour k impair. Soit $q(r)=P(X\ge r)$ la queue du modèle nul. Le nombre attendu de k-mers présents par hasard dans au moins $r$ génomes vaut
|
||||
|
||||
$$
|
||||
E[\mathrm{FP}(r)] = M\,q(r).
|
||||
$$
|
||||
|
||||
Cette égalité est exacte par linéarité de l'espérance, même si les k-mers chevauchants sont dépendants. La dépendance n'affecte que la variance, pas l'espérance. Seule l'approximation de Poisson par génome reste une approximation.
|
||||
|
||||
**FDR.** Soit $N(r)=\#\{i : r_i\ge r\}$ le nombre observé de k-mers couvrant au moins $r$ génomes. Alors
|
||||
|
||||
$$
|
||||
\mathrm{FDR}(r)=\min\!\Big(1,\ \frac{M\,q(r)}{N(r)}\Big).
|
||||
$$
|
||||
|
||||
Cette estimation est légèrement conservatrice. Elle suppose $\pi_0=1$, c'est-à-dire que presque tout l'espace $M$ est nul, ce qui est vrai pour k grand.
|
||||
|
||||
**Ensemble hétérogène en $r_i$.** Si le jeu n'est pas défini par un seuil unique, on utilise le local fdr par classe de couverture :
|
||||
|
||||
$$
|
||||
\mathrm{fdr}(r)=\frac{M\,P(X=r)}{N_r},\qquad N_r=\#\{i : r_i=r\}.
|
||||
$$
|
||||
|
||||
Le FDR de l'ensemble est alors la moyenne $\frac1N\sum_i \mathrm{fdr}(r_i)$. Pour un ensemble défini par seuil $r_{\min}$, cela redonne exactement la formule précédente.
|
||||
|
||||
**Remarque.** $M\,p\approx G$, d'où $E[\mathrm{FP}(r)]\approx\binom{S}{r}\,G^r/M^{r-1}$ en régime de petits $p$. Cela donne une lecture directe de la quantité.
|
||||
|
||||
## 2. Markov d'ordre 0 avec fréquences inégales
|
||||
|
||||
Soit $\pi_b$ la fréquence du nucléotide $b$. Pour un k-mer $w$ de composition $n=(n_A,n_C,n_G,n_T)$ avec $\sum n_b=k$, on a $P(w)=\prod_b \pi_b^{n_b}$.
|
||||
|
||||
**Attendu par génome.** Le k-mer canonique $K=\{w,\overline w\}$ est vu quand le brin direct porte $w$ ou $\overline w$, donc
|
||||
|
||||
$$
|
||||
\lambda_K = G\,\big(P(w)+P(\overline w)\big)=G\Big(\prod_b\pi_b^{n_b}+\pi_A^{n_T}\pi_T^{n_A}\pi_C^{n_G}\pi_G^{n_C}\Big).
|
||||
$$
|
||||
|
||||
Cela ne dépend que de la composition. Si $\pi_A=\pi_T$ et $\pi_C=\pi_G$, on retrouve $2P(w)$.
|
||||
|
||||
**Faux positifs.** Le test n'est plus homogène : chaque composition a sa propre $p(n)=1-e^{-\lambda(n)}$. Il faut donc sommer sur les classes de composition (un k-mer et son complément inverse partagent le même $\lambda$, d'où le facteur ½) :
|
||||
|
||||
$$
|
||||
E[\mathrm{FP}(r)]=\frac12\sum_{n}\binom{k}{n_A,n_C,n_G,n_T}\,P\big(X\ge r\mid p(n)\big).
|
||||
$$
|
||||
|
||||
Cette somme porte sur $\binom{k+3}{3}$ compositions, soit 5 984 pour k = 31. Elle est donc légère avec la binomiale de la moyenne, ou avec une Poisson-binomiale par composition si les $G_i$ diffèrent. Si les $\pi_b$ varient d'un génome à l'autre (GC différent), les $p_i(n)$ se construisent par génome.
|
||||
|
||||
**Effet attendu.** La moyenne de $\lambda_w$ sur tous les k-mers reste $2G/4^k$, car $\sum_w P(w)=1$. Dans le régime de petits $p$, le terme dominant $\binom{S}{r}p^r$ est convexe en $\lambda$. Par Jensen, le modèle équiprobable est donc le minimum de $E[\mathrm{FP}]$, et la déviation de composition ne peut qu'augmenter le FDR. Les k-mers riches en AT ou GC dominent la somme, ce qui rejoint ton remarque sur la faible complexité. C'est un résultat sur le terme dominant, pas une preuve pour la queue exacte.
|
||||
|
||||
**Limites à signaler dans le texte.** Markov 0 ignore les dinucléotides, comme les CpG ou les répétitions, et le $M$ du modèle nul reste celui de l'espace total.
|
||||
|
||||
Veux-tu que j'écrive ces deux sections, avec les corrections de numérique évoquées plus haut (`expm1`, somme directe de la queue) ?
|
||||
@@ -111,6 +111,7 @@ Without `--subsample`, every variable family (family size ≥ 2) is used. With `
|
||||
|
||||
| Column | Meaning |
|
||||
|---|---|
|
||||
| `partition` | the index partition that holds the family |
|
||||
| `layer` | an internal index-layer identifier — stable within one run, not meaningful across indexes |
|
||||
| `family_idx` | the family's position within that layer |
|
||||
| `entropy15` | Shannon entropy (bits) over the 16 possible states (the 15 non-empty subsets of `{A,C,G,T}`), genomes absent from the family excluded from the count |
|
||||
@@ -118,7 +119,7 @@ Without `--subsample`, every variable family (family size ≥ 2) is used. With `
|
||||
| `family_size` | number of distinct central bases observed anywhere in the index for this family (2-4) |
|
||||
| `n_genomes_present` | how many genomes the entropy was computed over |
|
||||
|
||||
Run with `--subsample N --shannon` to get a bounded diagnostic sample instead of a full-index pass — useful for choosing `--entropy`/`--entropy-sd` values before a full run.
|
||||
`--shannon` always scans every non-monomorphic family of the whole index; `--subsample`, `--entropy` and `--entropy-sd` have no effect on it. The output therefore grows with the index, with one row per family, and can reach hundreds of millions of rows (tens of gigabytes) on a large index. Analyse it with a tool that reads the file as a stream (for example DuckDB or Miller) rather than loading it whole. It is nevertheless the right basis for choosing `--entropy`/`--entropy-sd` values, since it gives the complete entropy distribution, including how it varies with `n_genomes_present`, rather than an estimate from a sample.
|
||||
|
||||
### `--entropy MU` / `--entropy-sd SIGMA`: biasing the sample toward informative families
|
||||
|
||||
|
||||
Reference in new issue
Block a user