From 33d7a59dd9f8089a6dc27fea10fad6df77fd9922 Mon Sep 17 00:00:00 2001 From: Eric Coissac Date: Thu, 19 Nov 2026 15:58:05 +0100 Subject: [PATCH] docs: restructure documentation for kmer indexing and phylogeny theory Reorganize the documentation structure to better separate kmer indexing logic from phylogenetic analysis methods. This involves moving several existing pages into new directories under `theory/kmer_indexing` and `theory/phylogeny`, updating navigation links across the site, and adding new content for VCF pseudo-alignment and specific phylogenetic models. - Move `doc/index.html` to `doc/theory/kmer_indexing/minimizer_selection/index.html` to document minimizer selection logic. - Move `doc/installation/index.html` to `doc/theory/phylogeny/kmer_based/kmer_set_distances/index.html` for whole k-mer set distance documentation. - Move `doc/usage/dump/index.html` to `doc/theory/kmer_indexing/superkmers/index.html` to cover canonical super-kmers. - Move `doc/usage/query/index.html` to `doc/theory/kmer_indexing/entropy_filter/index.html` for low-complexity kmer filtering details. - Move `doc/theory/encoding/index.html` to `doc/theory/kmer_indexing/encoding/index.html` for DNA and kmer encoding specifics. - Move `doc/theory/indexing_architecture/index.html` to `doc/theory/kmer_indexing/indexing_architecture/index.html` for partitioning and routing architecture. - Move `doc/usage/query/index.html` (duplicate move noted in facts, likely referring to the query command usage or similar) to `doc/theory/phylogeny/snp_based/sankoff_model/index.html` for the 16-state Sankoff model. - Move `doc/theory/minimizer_selection/index.html` to `doc/theory/phylogeny/snp_based/central_snp_model/index.html` for central-position SNP models and substitution corrections. - Move `doc/theory/indexing_architecture/index.html` (duplicate move noted) to `doc/theory/phylogeny/snp_based/gamma_rate_heterogeneity/index.html` for gamma rate heterogeneity and automatic alpha estimation. - Move `doc/theory/entropy_filter/index.html` to `doc/theory/phylogeny/snp_based/sampling_design/index.html` for entropy-biased sampling design. - Move `doc/formats/index_layout/index.html` to `doc/formats/vcf/index.html` to document VCF pseudo-alignment output and structure. - Add `doc/molecular-ecology-modified.csl` with a new XML style definition for Molecular Ecology Modified citations. - Update `UserDocMD/formats/vcf.md` to clarify that `--vcf` implies `--pseudo-alignment`, requires `--n-sites`, and generates files alongside `--fasta`. - Add three new BibTeX entries in `UserDocMD/references.bib` for Karlin (1997, 1995) and Bennett & Moran (2013). - Update `UserDocMD/theory/kmer_indexing/kmers.md` to use binomial distribution for probability calculations and define k-mer sharing deviation. - Clarify in `UserDocMD/theory/phylogeny/kmer_based/kmer_set_distances.md` that counts are used instead of presence/absence for $L^1$ distance. - Update `UserDocMD/theory/phylogeny/snp_based/gamma_rate_heterogeneity.md` to attribute method-of-moments $\alpha$ estimation to obikmer, not Jin & Nei (1990). - Update `UserDocMD/theory/phylogeny/snp_based/sampling_design.md` to redirect entropy sampling references to the correct phylo section. - Add new options `--cpu-max`, `--output`, `--pair-balance`, `--sibling-stats`, `--family-overlap`, and `--shannon` to `UserDocMD/usage/phylo.md`. - Mark SNP-based distance saving and session reuse as unavailable in `src/obikmer/src/cmd/phylo/args/distance.rs`. --- UserDocMD/formats/vcf.md | 2 +- UserDocMD/references.bib | 232 ++ UserDocMD/theory/kmer_indexing/kmers.md | 93 +- .../kmer_based/kmer_set_distances.md | 2 +- .../snp_based/gamma_rate_heterogeneity.md | 2 +- .../phylogeny/snp_based/sampling_design.md | 2 +- UserDocMD/usage/phylo.md | 203 +- doc/404.html | 438 +++- doc/architecture/index.html | 444 +++- doc/formats/index_layout/index.html | 444 +++- doc/formats/vcf/index.html | 1904 ++++++++++++++ doc/index.html | 446 +++- doc/installation/index.html | 440 +++- doc/molecular-ecology-modified.csl | 223 ++ doc/references.bib | 598 +++++ doc/sitemap.xml.gz | Bin 127 -> 127 bytes .../{ => kmer_indexing}/encoding/index.html | 496 +++- .../entropy_filter/index.html | 490 +++- .../indexing_architecture/index.html | 542 +++- doc/theory/kmer_indexing/kmers/index.html | 2250 +++++++++++++++++ .../minimizer_selection/index.html | 1756 +++++++++++++ .../superkmers}/index.html | 675 +++-- .../kmer_based/kmer_set_distances/index.html | 1715 +++++++++++++ .../snp_based/central_snp_model/index.html | 1659 ++++++++++++ .../gamma_rate_heterogeneity/index.html | 1589 ++++++++++++ .../snp_based/sampling_design/index.html | 1696 +++++++++++++ .../snp_based/sankoff_model}/index.html | 629 +++-- doc/usage/annotate/index.html | 438 +++- doc/usage/convert/index.html | 438 +++- doc/usage/dump/index.html | 438 +++- doc/usage/estimate/index.html | 438 +++- doc/usage/filter/index.html | 438 +++- doc/usage/index_command/index.html | 464 +++- doc/usage/merge/index.html | 496 +++- doc/usage/pack/index.html | 438 +++- doc/usage/phylo/index.html | 1226 +++++++-- doc/usage/predicates/index.html | 438 +++- doc/usage/query/index.html | 438 +++- doc/usage/select/index.html | 492 +++- doc/usage/superkmer/index.html | 440 +++- doc/usage/unitig/index.html | 438 +++- doc/usage/utils/index.html | 438 +++- src/obikmer/src/cmd/phylo/args/distance.rs | 24 +- 43 files changed, 25375 insertions(+), 1117 deletions(-) create mode 100644 doc/formats/vcf/index.html create mode 100644 doc/molecular-ecology-modified.csl rename doc/theory/{ => kmer_indexing}/encoding/index.html (65%) rename doc/theory/{ => kmer_indexing}/entropy_filter/index.html (70%) rename doc/theory/{ => kmer_indexing}/indexing_architecture/index.html (65%) create mode 100644 doc/theory/kmer_indexing/kmers/index.html create mode 100644 doc/theory/kmer_indexing/minimizer_selection/index.html rename doc/theory/{kmers_and_superkmers => kmer_indexing/superkmers}/index.html (62%) create mode 100644 doc/theory/phylogeny/kmer_based/kmer_set_distances/index.html create mode 100644 doc/theory/phylogeny/snp_based/central_snp_model/index.html create mode 100644 doc/theory/phylogeny/snp_based/gamma_rate_heterogeneity/index.html create mode 100644 doc/theory/phylogeny/snp_based/sampling_design/index.html rename doc/theory/{minimizer_selection => phylogeny/snp_based/sankoff_model}/index.html (53%) diff --git a/UserDocMD/formats/vcf.md b/UserDocMD/formats/vcf.md index b9aa3a98..bbdf6fe8 100644 --- a/UserDocMD/formats/vcf.md +++ b/UserDocMD/formats/vcf.md @@ -14,7 +14,7 @@ Nothing here comes from a reference genome. The file has no real coordinates, an obikmer phylo --vcf --n-sites 5000000 -o run1 my_index ``` -`--vcf` implies `--pseudo-alignment`, so `--n-sites N|all` is required, together with an already-built sibling annex (see [`phylo`](../usage/phylo.md)). `--fasta` and `--vcf` can be given together to get both files from the same draw; with neither, only the FASTA is written. Every option that shapes the draw (`--free-loss`, `--no-ambiguity`, `--only-diploid`, `--quartet`, `--exclude-genome`, ...) applies to the VCF exactly as to the FASTA, since both describe the same sites. +`--vcf` implies `--pseudo-alignment`, so `--n-sites N|all` is required, (the sibling annex it reads is built automatically if missing; see [`phylo`](../usage/phylo.md)). `--fasta` and `--vcf` can be given together to get both files from the same draw; with neither, only the FASTA is written. Every option that shapes the draw (`--free-loss`, `--no-ambiguity`, `--only-diploid`, `--quartet`, `--exclude-genome`, ...) applies to the VCF exactly as to the FASTA, since both describe the same sites. ## Layout diff --git a/UserDocMD/references.bib b/UserDocMD/references.bib index 23ca6fe4..cf8e4f6b 100644 --- a/UserDocMD/references.bib +++ b/UserDocMD/references.bib @@ -857,3 +857,235 @@ doi = "10.1214/aop/1176996461", issn = "0091-1798,2168-894X" } + + @ARTICLE{Karlin1997-lm, + title = "{Compositional differences within and between eukaryotic genomes}", + author = "Karlin, S and Mrázek, J", + journal = "Proceedings of the National Academy of Sciences of the United + States of America", + publisher = "Proceedings of the National Academy of Sciences", + volume = 94, + number = 19, + pages = "10227--10232", + abstract = "Eukaryotic genome similarity relationships are inferred using + sequence information derived from large aggregates of genomic + sequences. Comparisons within and between species sample + sequences are based on the profile of dinucleotide relative + abundance values (The profile is ρ* XY = f * XY / f * X f * Y for + all XY , where f * X denotes the frequency of the nucleotide X + and f * XY denotes the frequency of the dinucleotide XY , both + computed from the sequence concatenated with its inverted + complement). Previous studies with respect to prokaryotes and + this study document that profiles of different DNA sequence + samples (sample size ≥50 kb) from the same organism are generally + much more similar to each other than they are to profiles from + other organisms, and that closely related organisms generally + have more similar profiles than do distantly related organisms. + On this basis we refer to the collection {ρ* XY } as the genome + signature . This paper identifies ρ* XY extremes and compares + genome signature differences for a diverse range of eukaryotic + species. Interpretations on the mechanisms maintaining these + profile differences center on genome-wide replication, repair, + DNA structures, and context-dependent mutational biases. It is + also observed that mitochondrial genome signature differences + between species parallel the corresponding nuclear genome + signature differences despite large differences between + corresponding mitochondrial and nuclear signatures. The genome + signature differences also have implications for contrasts + between rodents and other mammals, and between monocot and dicot + plants, as well as providing evidence for similarities among + fungi and the diversity of protists.", + month = sep, + year = 1997, + url = "http://dx.doi.org/10.1073/pnas.94.19.10227", + doi = "10.1073/pnas.94.19.10227", + pmc = "PMC23344", + pmid = 9294192, + issn = "0027-8424,1091-6490", + language = "en" + } + + @ARTICLE{Karlin1997-vm, + title = "{Compositional biases of bacterial genomes and evolutionary + implications}", + author = "Karlin, S and Mrázek, J and Campbell, A M", + journal = "Journal of Bacteriology", + publisher = "American Society for Microbiology", + volume = 179, + number = 12, + pages = "3899--3913", + abstract = "We compare and contrast genome-wide compositional biases and + distributions of short oligonucleotides across 15 diverse + prokaryotes that have substantial genomic sequence collections. + These include seven complete genomes (Escherichia coli, + Haemophilus influenzae, Mycoplasma genitalium, Mycoplasma + pneumoniae, Synechocystis sp. strain PCC6803, Methanococcus + jannaschii, and Pyrobaculum aerophilum). A key observation + concerns the constancy of the dinucleotide relative abundance + profiles over multiple 50-kb disjoint contigs within the same + genome. (The profile is rhoXY* = fXY*/fX*fY* for all XY, where + fX* denotes the frequency of the nucleotide X and fY* denotes the + frequency of the dinucleotide XY, both computed from the sequence + concatenated with its inverted complementary sequence.) On the + basis of this constancy, we refer to the collection [rhoXY*] as + the genome signature. We establish that the differences between + [rhoXY*] vectors of 50-kb sample contigs of different genomes + virtually always exceed the differences between those of the same + genomes. Various di- and tetranucleotide biases are identified. + In particular, we find that the dinucleotide CpG=CG is + underrepresented in many thermophiles (e.g., M. jannaschii, + Sulfolobus sp., and M. thermoautotrophicum) but overrepresented + in halobacteria. TA is broadly underrepresented in prokaryotes + and eukaryotes, but normal counts appear in Sulfolobus and P. + aerophilum sequences. More than for any other bacterial genome, + palindromic tetranucleotides are underrepresented in H. + influenzae. The M. jannaschii sequence is unprecedented in its + extreme underrepresentation of CTAG tetranucleotides and in the + anomalous distribution of CTAG sites around the genome. + Comparative analysis of numbers of long tetranucleotide + microsatellites distinguishes H. influenzae. Dinucleotide + relative abundance differences between bacterial sequences are + compared. For example, in these assessments of differences, the + cyanobacteria Synechocystis, Synechococcus, and Anabaena do not + form a coherent group and are as far from each other as general + gram-negative sequences are from general gram-positive sequences. + The difference of M. jannaschii from low-G+C gram-positive + proteobacteria is one-half of the difference from gram-negative + proteobacteria. Interpretations and hypotheses center on the role + of the genome signature in highlighting similarities and + dissimilarities across different classes of prokaryotic species, + possible mechanisms underlying the genome signature, the form and + level of genome compositional flux, the use of the genome + signature as a chronometer of molecular phylogeny, and + implications with respect to the three putative eubacterial, + archaeal, and eukaryote domains of life and to the origin and + early evolution of eukaryotes.", + month = jun, + year = 1997, + url = "http://dx.doi.org/10.1128/jb.179.12.3899-3913.1997", + doi = "10.1128/jb.179.12.3899-3913.1997", + pmc = "PMC179198", + pmid = 9190805, + issn = "0021-9193,1098-5530", + language = "en" + } + @ARTICLE{Karlin1995-lt, + title = "{Dinucleotide relative abundance extremes: a genomic signature}", + author = "Karlin, S and Burge, C", + journal = "Trends in Genetics: TIG", + publisher = "Elsevier BV", + volume = 11, + number = 7, + pages = "283--290", + abstract = "Early biochemical experiments established that the set of + dinucleotide odds ratios or 'general design' is a remarkably + stable property of the DNA of an organism, which is essentially + the same in protein-coding DNA, bulk genomic DNA, and in + different renaturation rate and density gradient fractions of + genomic DNA in many organisms. Analysis of currently available + genomic sequence data has extended these earlier results, showing + that the general designs of disjoint samples of a genome are + substantially more similar to each other than to those of + sequences from other organisms and that closely related organisms + have similar general designs. From this perspective, the set of + dinucleotide odds ratio (relative abundance) values constitute a + signature of each DNA genome, which can discriminate between + sequences from different organisms. Dinucleotide-odds ratio + values appear to reflect not only the chemistry of dinucleotide + stacking energies and base-step conformational preferences, but + also the species-specific properties of DNA modification, + replication and repair mechanisms.", + month = jul, + year = 1995, + url = "http://dx.doi.org/10.1016/s0168-9525(00)89076-9", + doi = "10.1016/s0168-9525(00)89076-9", + pmid = 7482779, + issn = "0168-9525,1362-4555", + language = "en" + } + + @ARTICLE{Karlin1994-on, + title = "{Heterogeneity of genomes: measures and values}", + author = "Karlin, S and Ladunga, I and Blaisdell, B E", + journal = "Proceedings of the National Academy of Sciences of the United + States of America", + publisher = "Proceedings of the National Academy of Sciences", + volume = 91, + number = 26, + pages = "12837--12841", + abstract = "Genomic homogeneity is investigated for a broad base of DNA + sequences in terms of dinucleotide relative abundance distances + (abbreviated delta-distances) and of oligonucleotide + compositional extremes. It is shown that delta-distances between + different genomic sequences in the same species are low, only + about 2 or 3 times the distance found in random DNA, and are + generally smaller than the between-species delta-distances. + Extremes in short oligonucleotides include underrepresentation of + TpA and overrepresentation of GpC in most temperate bacteriophage + sequences; underrepresentation of CTAG in most eubacterial + genomes; underrepresentation of GATC in most bacteriophage; CpG + suppression in vertebrates, in all animal mitochondrial genomes, + and in many thermophilic bacterial sequences; and + overrepresentation of GpG/CpC in all animal mitochondrial sets + and chloroplast genomes. Interpretations center on DNA structures + (dinucleotide stacking energies, DNA curvature and superhelicity, + nucleosome organization), context-dependent mutational events, + methylation effects, and processes of replication and repair.", + month = dec, + year = 1994, + url = "http://dx.doi.org/10.1073/pnas.91.26.12837", + doi = "10.1073/pnas.91.26.12837", + pmc = "PMC45535", + pmid = 7809131, + issn = "0027-8424,1091-6490", + language = "en" + } + @ARTICLE{Bennett2013-cl, + title = "{Small, smaller, smallest: the origins and evolution of ancient + dual symbioses in a Phloem-feeding insect}", + author = "Bennett, Gordon M and Moran, Nancy A", + journal = "Genome Biology and Evolution", + publisher = "Oxford University Press (OUP)", + volume = 5, + number = 9, + pages = "1675--1688", + abstract = "Many insects rely on bacterial symbionts with tiny genomes + specialized for provisioning nutrients lacking in host diets. + Xylem sap and phloem sap are both deficient as insect diets, but + differ dramatically in nutrient content, potentially affecting + symbiont genome evolution. For sap-feeding insects, sequenced + symbiont genomes are available only for phloem-feeding examples + from the suborder Sternorrhyncha and xylem-feeding examples from + the suborder Auchenorrhyncha, confounding comparisons. We + sequenced genomes of the obligate symbionts, Sulcia muelleri and + Nasuia deltocephalinicola, of the phloem-feeding pest insect, + Macrosteles quadrilineatus (Auchenorrhyncha: Cicadellidae). Our + results reveal that Nasuia-ALF has the smallest bacterial genome + yet sequenced (112 kb), and that the Sulcia-ALF genome (190 kb) + is smaller than that of Sulcia in other insect lineages. + Together, these symbionts retain the capability to synthesize the + 10 essential amino acids, as observed for several symbiont pairs + from xylem-feeding Auchenorrhyncha. Nasuia retains genes enabling + synthesis of two amino acids, DNA replication, transcription, and + translation. Both symbionts have lost genes underlying ATP + synthesis through oxidative phosphorylation, possibly as a + consequence of the enriched sugar content of phloem. Shared + genomic features, including reassignment of the UGA codon from + Stop to tryptophan, and phylogenetic results suggest that + Nasuia-ALF is most closely related to Zinderia, the + betaproteobacterial symbiont of spittlebugs. Thus, + Nasuia/Zinderia and Sulcia likely represent ancient associates + that have co-resided in hosts since the divergence of leafhoppers + and spittlebugs >200 Ma, and possibly since the origin of the + Auchenorrhyncha, >260 Ma.", + month = aug, + year = 2013, + url = "http://dx.doi.org/10.1093/gbe/evt118", + keywords = "Nasuia deltocephalinicola; Sulcia muelleri; gene loss; genome + evolution; leafhopper; nutritional symbioses", + doi = "10.1093/gbe/evt118", + pmc = "PMC3787670", + pmid = 23918810, + issn = "1759-6653", + language = "en" + } diff --git a/UserDocMD/theory/kmer_indexing/kmers.md b/UserDocMD/theory/kmer_indexing/kmers.md index 9fd3033b..c2e73210 100644 --- a/UserDocMD/theory/kmer_indexing/kmers.md +++ b/UserDocMD/theory/kmer_indexing/kmers.md @@ -47,7 +47,9 @@ P(n_K > 0) &= 1-e^{-\lambda} \\ \end{aligned} $$ -Assuming equal genome lengths, the number of genomes containing at least one occurrence of $K$ among the $S$ sequences follows a binomial distribution. The probability of observing $K$ in at least $r$ genomes is therefore +##### Assuming equal genome lengths + +The number of genomes containing at least one occurrence of $K$ among the $S$ sequences follows a binomial distribution. The probability of observing $K$ in at least $r$ genomes is therefore $$ @@ -56,11 +58,22 @@ $$ This sum is the regularized incomplete beta function $I_p(r,\,S-r+1)$, where +$$ +I_p(r,S-r+1) += +\frac{1}{B(r,S-r+1)} +\int_0^p t^{r-1}(1-t)^{S-r}\,dt +$$ + +and + $$ p = 1-e^{-2G/4^k}. $$ -Different genome sizes +the regularized incomplete beta function is available in standard statistical libraries. + +##### Different genome sizes When genome sizes differ, the probability of observing a given k-mer also differs between genomes. A simple approximation is to replace the individual genome sizes $G_i$ by their mean @@ -82,39 +95,63 @@ Using the mean genome size therefore provides a conservative estimate of the FDR When genome sizes differ substantially, however, this approximation can become overly conservative. In that case, the exact Poisson-binomial distribution can be computed directly from the individual genome-specific probabilities $p_i$. -### Exact tail probability +##### Exact tail probability With genome-specific probabilities $p_i$, the number of genomes containing the k-mer follows a Poisson-binomial distribution. Writing the probability of exactly $j$ presences as a sum over all subsets of $j$ genomes would be impractical, since there can be up to $2^S$ subsets. The distribution can instead be obtained by adding the genomes one at a time. -Let $d_j$ be the probability that exactly $j$ of the genomes processed so far contain the k-mer. Adding genome $i$ leaves $j$ unchanged if the genome does not contain the k-mer, and moves $j-1$ to $j$ if it does: +Let $P_i(j)$ denote the probability that the k-mer occurs in exactly $j$ of the first $i$ genomes. Initially, + +$$ +P_0(0)=1,\qquad P_0(j)=0 \quad (j>0). +$$ + +When genome $i$ is added, the k-mer is either absent, with probability $1-p_i$, or present, with probability $p_i$. The probability of observing exactly $j$ presences among the first $i$ genomes is therefore + +$$ +P_i(j)=(1-p_i)P_{i-1}(j)+p_iP_{i-1}(j-1). +$$ + +This dynamic programming algorithm requires $O(S^2)$ operations and $O(S)$ memory. However, to compute $P(X\geq r)$, only the probabilities for fewer than $r$ presences need to be tracked, since + +$$ +P(X\geq r)=1-\sum_{j=0}^{r-1}P_S(j). +$$ + +This reduces the computational cost to $O(Sr)$ operations and $O(r)$ memory. + +#### Distinguishing random from non-random k-mer sharing + +The probability $P(X\geq r)$ describes the occurrence of a given k-mer in at least $r$ genomes under the random-sequence model. However, the index contains only k-mers observed in at least one genome. Therefore, when considering the k-mers recorded in the index, the relevant probability is the conditional probability: + +$$ +q_r = P(X\geq r\mid X\geq 1) + = \frac{P(X\geq r)}{P(X\geq 1)}. +$$ + +Let $k_r$ be the number of distinct k-mers observed in at least $r$ genomes, among the $k_1$ distinct k-mers recorded in the index. The observed proportion of k-mers shared by at least $r$ genomes is: + +$$ +O_r = \frac{k_r}{k_1}. +$$ + +Under the random-sequence model, the expected proportion is $q_r$. We can therefore quantify the deviation from random expectations using the difference: $$ -\left \{ \begin{aligned} -d'_j &= d_j(1-p_i) + d_{j-1}p_i, \\ -d_0&=1 +D_r &= O_r-q_r \\ + &= \frac{k_r}{k_1}-q_r. \end{aligned} -\right . $$ -before processing the first genome. +Positive values of $D_r$ indicate that k-mers are shared across genomes more frequently than expected by chance, whereas negative values indicate less frequent sharing. A value of zero indicates agreement with the random-sequence model. -The table $d$ only needs to record how many genomes contain the k-mer, not which ones, reducing the computational cost to $O(S^2)$. The result is independent of the order in which the genomes are processed and of the k-mer itself, so it can be computed once per index. The upper tail is then +Since both $O_r$ and $q_r$ are proportions between zero and one, $D_r$ is bounded: $$ -P(X\geq r)=1-\sum_{j=0}^{r-1}d_j. +-1\leq D_r\leq 1. $$ -Only $d_0,\ldots,d_{r-1}$ are needed, reducing the cost to $O(Sr)$. - -When all $p_i$ are small, as is typically the case for large $k$, the Poisson-binomial distribution can also be approximated by a Poisson distribution with mean - -$$ -\lambda=\sum_{i=1}^{S}p_i. -$$ - -The approximation error is bounded in terms of $\sum_i p_i^2$ by Le Cam's inequality [@LeCam1960-pe]. - +More precisely, for a given $q_r$, its bounds are $-q_r$ and $1-q_r$. By construction, $D_1=0$, since all k-mers recorded in the index occur in at least one genome. Unlike a ratio-based measure, this difference remains defined even when no k-mers are observed in at least $r$ genomes. It thus provides a bounded measure of the excess or deficit of k-mer sharing relative to the random-sequence model. ******* @@ -150,11 +187,12 @@ Consequently, the appropriate value of $k$ depends on the total amount of sequen -Considering that bacterial genomes are typically a few megabases long, with the smallesr plublished today *Mycoplasma genitalium* having a size $G=580\,Mb$ [@Fraser1995-jc], and the largest $13\,Mb$ in *Sorangium cellulosum* [@Schneiker2007-gs]. For such genomes, the space of 11-mers is already to small to get a reasonable false positive rate: +Considering that bacterial genomes are typically a few megabases long, with the smallesr plublished today *Nasuia deltocephalinicola* having a size $G=112\,Kb$ [@Bennett2013-cl], and the largest $13\,Mb$ in *Sorangium cellulosum* [@Schneiker2007-gs]. For such genomes, the space of 11-mers is already to small to get a reasonable false positive rate: $$ \begin{aligned} -P(C\geq1) &= 1- e^{-\frac{580.000^2}{2 \cdot 4^{11}}} \\ +P(C\geq1) &= 1- e^{-\frac{112\,000^2}{2 \cdot 4^{11}}} \\ +&= 1- e^{-1.5\times10^{3}} \\ &\simeq 1 \end{aligned} $$ @@ -182,7 +220,7 @@ The distinction between the simple $4^k$ model and the Karlin--Ost framework is These scales have very different implications for different genome sizes. -The random-sequence calculation is only a null model. Real genomes exhibit strong and reproducible oligonucleotide composition biases. Karlin and colleagues showed that relative abundances of short oligonucleotides, in particular dinucleotides, form characteristic genomic signatures that remain relatively stable within genomes and differ between organisms [@Karlin1995; @Karlin1997]. More generally, short k-mer frequencies can therefore capture compositional properties of genomes rather than provide highly specific sequence identifiers. +The random-sequence calculation is only a null model. Real genomes exhibit strong and reproducible oligonucleotide composition biases. Karlin and colleagues showed that relative abundances of short oligonucleotides, in particular dinucleotides, form characteristic genomic signatures that remain relatively stable within genomes and differ between organisms [@Karlin1995-lt; @Karlin1994-on]. More generally, short k-mer frequencies can therefore capture compositional properties of genomes rather than provide highly specific sequence identifiers. This distinction is important for whole-genome k-mer comparison. Increasing $k$ increases the theoretical diversity of the k-mer space exponentially and generally increases the proportion of k-mers that occur at unique genomic positions. Conversely, shorter k-mers occur more frequently and are more strongly influenced by genomic composition and repeated sequence. Empirical studies of k-mer spectra show this transition clearly: for example, k-mers of length $k\leq11$ can behave primarily as genomic-composition signatures, whereas longer k-mers progressively provide greater sequence specificity. @@ -232,7 +270,7 @@ $$ The factor $(1-\lambda^{*})$ multiplying $S_1(N,k)$ is the key correction over the naive pairwise-collision count: it de-clumps overlapping candidate matches, which are not independent events, into an effective Poisson rate. This theorem is stated for $s\geq2$ independent sequences (e.g. comparing two distinct genomes); the intra-sequence self-repeat case (a single genome compared to itself) is treated separately, via the reverse-complement match length $R(N)$, in [@Karlin1985]. - + ## Chance sharing across genomes and the FDR of homology @@ -359,6 +397,7 @@ Chance alone produces millions of pairwise shared 21-mers at this genome size, a --- + diff --git a/UserDocMD/theory/phylogeny/kmer_based/kmer_set_distances.md b/UserDocMD/theory/phylogeny/kmer_based/kmer_set_distances.md index 21c1c3c3..c6188b9e 100644 --- a/UserDocMD/theory/phylogeny/kmer_based/kmer_set_distances.md +++ b/UserDocMD/theory/phylogeny/kmer_based/kmer_set_distances.md @@ -34,7 +34,7 @@ $$ D = \sum_i \mathbb{1}[a_i \ne b_i], $$ -where $a_i,b_i\in\{0,1\}$ indicate the presence of k-mer $i$ in the two genomes. It counts the number of k-mers for which the two incidence vectors differ, without normalization by the size of the k-mer space. As an $L^1$ distance, Hamming distance is a metric. Unlike abundance-based measures, it cannot be recovered from k-mer counts alone once the presence/absence representation has been discarded. +where $a_i,b_i\in\{0,1\}$ indicate the presence of k-mer $i$ in the two genomes. It counts the number of k-mers for which the two incidence vectors differ, without normalization by the size of the k-mer space. As an $L^1$ distance, Hamming distance is a metric. Unlike abundance-based measures, it cannot be recovered from k-mer counts alone once the presence/absence representation has been discarded. On an index that stores counts, a k-mer is taken as present when its count reaches the presence threshold (`--presence-threshold`, 1 by default). **Mash** is not an independent set dissimilarity but a nonlinear transformation of the Jaccard similarity [@Ondov2016-dq]. If diff --git a/UserDocMD/theory/phylogeny/snp_based/gamma_rate_heterogeneity.md b/UserDocMD/theory/phylogeny/snp_based/gamma_rate_heterogeneity.md index 7ab77c39..7accd4ca 100644 --- a/UserDocMD/theory/phylogeny/snp_based/gamma_rate_heterogeneity.md +++ b/UserDocMD/theory/phylogeny/snp_based/gamma_rate_heterogeneity.md @@ -24,4 +24,4 @@ If the measured variance $V$ doesn't exceed the Poisson floor $\hat\mu/\bar L$ ( **This is not Jin & Nei's (1990) original estimator.** Their publication defines the $+\Gamma$ distance formula itself and, absent an estimate, recommends the fixed default $\alpha = 1$ — it does not propose a way to estimate $\alpha$ from data. The method-of-moments estimator above is a standard consequence of the Poisson–Gamma relationship between substitution counts and gamma-distributed rate variation, applied per-partition; it is `obikmer`'s own addition, not part of the cited correction. -See [`phylo`](../../../usage/phylo.md#distance-matrix---distance) for how to select a fixed $\alpha$ or request automatic estimation. +See [`phylo`](../../../usage/phylo.md#distance-estimation-and-tree-reconstruction) for how to select a fixed $\alpha$ or request automatic estimation. diff --git a/UserDocMD/theory/phylogeny/snp_based/sampling_design.md b/UserDocMD/theory/phylogeny/snp_based/sampling_design.md index 66f67f85..e72111e9 100644 --- a/UserDocMD/theory/phylogeny/snp_based/sampling_design.md +++ b/UserDocMD/theory/phylogeny/snp_based/sampling_design.md @@ -31,4 +31,4 @@ How well a pair of genomes is represented in a sample depends on how many famili By default, sampling draws families uniformly. Weighting instead by informativeness biases the draw toward a target entropy $\mu$ with a Gaussian curve of width $\sigma$: a family's probability of being kept scales with $\exp\!\left(-\dfrac{(\text{entropy15} - \mu)^2}{2\sigma^2}\right)$ — a soft preference, not a hard cutoff, so no family is categorically excluded purely for having low or high entropy. -See [`phylo`](../../../usage/phylo.md#sampling-at-scale---n-sites---shannon---entropy) for how to bound and bias the sample, and for session/checkpointing behavior. +See [`phylo`](../../../usage/phylo.md#snp-character-selection-and-sampling) for how to bound and bias the sample. diff --git a/UserDocMD/usage/phylo.md b/UserDocMD/usage/phylo.md index 6f4a7297..cdbcda8f 100644 --- a/UserDocMD/usage/phylo.md +++ b/UserDocMD/usage/phylo.md @@ -1,6 +1,13 @@ # phylo -Compute pairwise distances between the genomes stored in an index, optionally build trees (NJ/UPGMA) from them, and optionally calibrate a 16-state parsimony model for a central-position SNP character with exports for external phylogenetic tools (TNT, PhyG, IQ-TREE). +Compare the genomes stored in an index and build phylogenetic trees from them. Two complementary approaches are available: + +- **Whole-genome distances** compare the k-mer content of the genomes, without any alignment. +- **SNP-based distances and analyses** compare the nucleotide variants found at the central position of related k-mers. + +Distance matrices can be used to build Neighbor-Joining or UPGMA trees. SNP-based analyses can also produce a pseudo-alignment and calibrated substitution costs, with exports for external phylogenetic tools (TNT, PhyG, IQ-TREE). + +## Usage ```bash obikmer phylo INDEX [OPTIONS] @@ -12,28 +19,43 @@ obikmer phylo INDEX [OPTIONS] |---|---| | `INDEX` | Index directory | -## Distance matrix (`--distance`) +## General options | Option | Default | Description | |---|---|---| -| `--distance` | none | Compute and write a distance matrix. See the two tables below for the full list of accepted values. No matrix is computed unless this is given, or `--nj`, `--upgma` or `--shared-kmers` is (they need a matrix and then use `jaccard`) | +| `--cpu-max N` | all available cores | Maximum number of CPU cores to use. It caps every thread pool of the process and can only lower the number of cores, never raise it above those available. Common to all subcommands | +| `-o, --output PREFIX` | none | Prefix of the output files. Without it, the distance matrix goes to standard output and every other file is written in the current directory under its name without the prefix. See "Output files" below | + +## SNP families + +SNP-based analyses use groups of k-mers that differ only at their central nucleotide. These groups are called families (see the [central-position SNP model](../theory/phylogeny/snp_based/central_snp_model.md) for the exact definitions and eligibility rules). A variable family is one character of the analysis, and the variants a genome carries give its state for that character. + +## Distance estimation and tree reconstruction + +| Option | Default | Description | +|---|---|---| +| `--distance` | none | Compute and write a distance matrix. The accepted values are listed just below the table. No matrix is computed unless this is given, or `--nj` or `--upgma` is (they need a matrix and then use `jaccard`) | | `--gamma-shape ALPHA\|auto` | none | Rate-heterogeneity correction, for `snp-*` values that support it (see [Gamma rate heterogeneity](../theory/phylogeny/snp_based/gamma_rate_heterogeneity.md)). Either a fixed $\alpha$ or `auto`/`estimate` to fit it from the data. No effect on the other values; rejected if given together with a value that doesn't support it | -| `--presence-threshold` | `1` | Minimum count for a kmer to be considered present, for `jaccard`/`mash` on a count index | +| `--presence-threshold` | `1` | How counts are turned into presence/absence on an index that stores counts: a kmer is present in a genome only when its count reaches this value. Used by `jaccard`, `mash` and `hamming`, and by the shared-kmer counts of `--shared-kmers`. No effect on an index without counts, nor on the distances that use the counts themselves | +| `--session DIR`, `--session-force` | none | Save and reuse the sample of a `snp-*` distance. Currently unavailable, see `--session` below | | `--csv` | off | Write the matrix as plain CSV instead of the default relaxed-PHYLIP format | -| `--shared-kmers` | off | Also write the shared-kmer count matrix. Only valid with a whole-index distance, not a `snp-*` value | +| `--shared-kmers` | off | Write the shared-kmer count matrix (`_shared.csv`). It can be used alone, without any distance, or together with a whole-index distance; it is refused with a `snp-*` value | | `--nj` | off | Compute and write a Neighbor-Joining tree (Newick) | | `--upgma` | off | Compute and write a UPGMA tree (Newick) | -| `-o, --output` | none (stdout) | Output file prefix | -Every value routes to one of two independent computations: **whole-index distances** (`jaccard`, `mash`, `hamming`, `bray-curtis`, `relfreq-bray-curtis`, `euclidean`, `relfreq-euclidean`, `hellinger`, `hellinger-euclidean` — see [Whole k-mer set distances](../theory/phylogeny/kmer_based/kmer_set_distances.md) for the definitions; `hamming` requires a presence/absence index, the others work on either index type), or **`snp-*` corrections** (`snp-raw`, `snp-jc`, `snp-k2p`, `snp-k81`, `snp-f81`, `snp-t92`, `snp-tn93`, `snp-tv` — see the [central-position SNP model](../theory/phylogeny/snp_based/central_snp_model.md) for the definitions). +Every value routes to one of two independent computations: **whole-index distances** (`jaccard`, `mash`, `hamming`, `bray-curtis`, `relfreq-bray-curtis`, `euclidean`, `relfreq-euclidean`, `hellinger`, `hellinger-euclidean` — see [Whole k-mer set distances](../theory/phylogeny/kmer_based/kmer_set_distances.md) for the definitions; `jaccard`, `mash` and `hamming` compare which kmers are present and work on either index type; the six others (`bray-curtis`, `relfreq-bray-curtis`, `euclidean`, `relfreq-euclidean`, `hellinger`, `hellinger-euclidean`) use the counts and need an index that stores them), or **`snp-*` corrections** (`snp-raw`, `snp-jc`, `snp-k2p`, `snp-k81`, `snp-f81`, `snp-t92`, `snp-tn93`, `snp-tv` — see the [central-position SNP model](../theory/phylogeny/snp_based/central_snp_model.md) for the definitions). -`snp-*` values require the sibling annex (`--sibling-annex`, below) and are, by default, computed exhaustively over every non-monomorphic family in the index; add `--n-sites N` to bound the computation to approximately `N` families instead (see "Sampling at scale" below — the same option `--pseudo-alignment`/`--sankoff` use, but optional here). +`snp-*` values read the sibling annex (below), which is built automatically the first time it is needed, and are, by default, computed exhaustively over every non-monomorphic family in the index; add `--n-sites N` to bound the computation to approximately `N` families instead (see "SNP character selection and sampling" below — the same option `--pseudo-alignment`/`--sankoff` use, but optional here). -### Output +### Matrix format When a matrix is computed and there is no `-o`, it goes to stdout in relaxed-PHYLIP format (`n` on the first line, then one `labelvalue...` row per genome). With `--csv`, the format is instead a header row `genome,,,...` followed by one ` + + + + + + + + + + + + + + + + + + + +
  • + + + + + + + + + + +
  • + + + + @@ -944,6 +1332,34 @@ + + + + + + +
  • + + + + + + + + VCF pseudo-alignment + + + + + + + + +
  • + + + + @@ -1157,13 +1573,13 @@

    Architecture notes for advanced use

    -

    This page describes execution-level behavior relevant to sizing and running obikmer on large datasets or multi-socket machines. It complements the index format and theory pages.

    +

    This page describes execution-level behavior relevant to sizing and running obikmer on large datasets or multi-socket machines. It complements the index format and theory pages.

    Sequence invariant

    Every input sequence is treated purely as a compact representation of a set of overlapping kmers:

    • Only the A/C/G/T alphabet (case-insensitive) is recognized; a sequence is cut at any other character (including IUPAC ambiguity codes), so runs containing them are not represented in the index.
    • Sequences are internally processed in chunks of at most 256 nucleotides; a chunk shorter than k is dropped. This is invisible to the user beyond the ACGT-only, minimum-length-k constraints above.
    • -
    • Kmers are always handled in canonical form (see DNA encoding), so the tool is strand-agnostic throughout: a kmer and its reverse complement are always the same entry.
    • +
    • Kmers are always handled in canonical form (see DNA encoding), so the tool is strand-agnostic throughout: a kmer and its reverse complement are always the same entry.

    Index dimensioning

    An index directory is organized as KmerIndex → partitions → layers, with a canonical kmer belonging to exactly one (partition, layer) pair. This is what makes set operations (merge, filter, distance) parallel and coordination-free across partitions.

    diff --git a/doc/formats/index_layout/index.html b/doc/formats/index_layout/index.html index 9ad40494..cbc5bbb1 100644 --- a/doc/formats/index_layout/index.html +++ b/doc/formats/index_layout/index.html @@ -12,7 +12,7 @@ - + @@ -256,15 +256,26 @@ -
  • - + + + + + +
  • + + + + + + -
  • + + + + + + + + + + + + + + + + + + + + + +
  • + + + + + + + + + + +
  • + + + + @@ -1039,6 +1427,34 @@ + + + + + + +
  • + + + + + + + + VCF pseudo-alignment + + + + + + + + +
  • + + + + @@ -1162,9 +1578,9 @@

    Index construction and on-disk layout

    Construction pipeline

    -

    Building an index (index) proceeds through a fixed sequence of phases, each operating independently per partition (see Partitioning and indexing architecture):

    +

    Building an index (index) proceeds through a fixed sequence of phases, each operating independently per partition (see Partitioning and indexing architecture):

      -
    1. Scatter. A single streaming pass over the input. Each sequence fragment is cut at non-ACGT bases, passed through the low-complexity entropy filter (see Low-complexity kmer filter), and any resulting segment shorter than k is dropped. Surviving segments are decomposed into super-kmers, canonicalized, and routed by hash(minimizer) mod n_partitions into one file per partition.
    2. +
    3. Scatter. A single streaming pass over the input. Each sequence fragment is cut at non-ACGT bases, passed through the low-complexity entropy filter (see Low-complexity kmer filter), and any resulting segment shorter than k is dropped. Surviving segments are decomposed into super-kmers, canonicalized, and routed by hash(minimizer) mod n_partitions into one file per partition.
    4. Dereplication. Within each partition, identical super-kmer sequences are merged and their occurrence counts summed. This count is per super-kmer, not per kmer — a kmer's true abundance is the sum of the counts of every super-kmer containing it.
    5. Exact counting. Every kmer in every dereplicated super-kmer is enumerated and its exact total count computed. A per-genome kmer frequency spectrum is produced at this stage.
    6. Quorum filtering. Kmers outside the --min-abundance/--max-abundance range are dropped, and super-kmers are recompacted around the surviving kmer set.
    7. diff --git a/doc/formats/vcf/index.html b/doc/formats/vcf/index.html new file mode 100644 index 00000000..339d6f62 --- /dev/null +++ b/doc/formats/vcf/index.html @@ -0,0 +1,1904 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + VCF pseudo-alignment - obikmer — User Guide + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      + +
      + + + + + + +
      + + +
      + +
      + + + + + + +
      +
      + + + +
      +
      +
      + + + + + +
      +
      +
      + + + + + + + +
      + +
      + + + + + +

      VCF pseudo-alignment

      +

      phylo --vcf writes the sites of a SNP pseudo-alignment as a VCF file, <prefix>_alignment.vcf. This page describes exactly what that file contains, because the k-mer families it is built from are not ordinary genomic variants and a few conventions follow from that.

      +

      Why a VCF, and what a site is

      +

      The pseudo-alignment (see Central-position SNP model) has one column per family: the up to four k-mers that share all their bases except the central one. A genome that carries one of these k-mers shows a particular base at the centre, and the columns of the alignment are those central bases, genome by genome. The FASTA output keeps this as a matrix of letters. The VCF output lists the same columns as sites with alleles and genotypes, which is the form that variant-oriented tools (bcftools, population-genetics software, genotype viewers) expect.

      +

      Nothing here comes from a reference genome. The file has no real coordinates, and the sites are not located on a chromosome: the positions below are labels that make the file valid and reproducible, not map positions.

      +

      Producing the file

      +
      obikmer phylo --vcf --n-sites 5000000 -o run1 my_index
      +
      +

      --vcf implies --pseudo-alignment, so --n-sites N|all is required, (the sibling annex it reads is built automatically if missing; see phylo). --fasta and --vcf can be given together to get both files from the same draw; with neither, only the FASTA is written. Every option that shapes the draw (--free-loss, --no-ambiguity, --only-diploid, --quartet, --exclude-genome, ...) applies to the VCF exactly as to the FASTA, since both describe the same sites.

      +

      Layout

      +

      An illustrative excerpt (columns are tab-separated in the file):

      +
      ##fileformat=VCFv4.5
      +##source=obikmer
      +##contig=<ID=obikmer_0_0>
      +##contig=<ID=obikmer_0_1>
      +##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype (diploid)">
      +#CHROM       POS   ID           REF  ALT  QUAL  FILTER  INFO  FORMAT  idx0  idx1  idx2  idx3
      +obikmer_0_0  440   CGATTNGTTAA  A    G    .     .       .     GT      0/0   1/1   0/0   0/0
      +obikmer_0_0  1511  CACGANTGTAG  A    G,T  .     .       .     GT      0/0   0/0   1/1   2/2
      +obikmer_0_1  9     TTACGNAGCTA  C    *    .     .       .     GT      0/0   0/0   1/1   0/0
      +
      +

      There is one sample column per genome that takes part in the draw, named after the genome label, in the index's own order. A genome removed with --exclude-genome or --min-shared-family has no column at all. QUAL, FILTER and INFO are always .: obikmer applies no variant filter and attaches no annotation, and leaving them empty keeps a file of millions of sites compact.

      +

      Coordinates: CHROM and POS

      +

      An index is organised in partitions, and each partition holds one or more layers of k-mers (see Index construction and on-disk layout). A family always belongs to exactly one layer of one partition, and the families of a layer are visited in a fixed order. The VCF uses that structure as coordinates:

      +
        +
      • CHROM is obikmer_<partition>_<layer>, one virtual contig per layer of each partition in which at least one site was drawn;
      • +
      • POS is the rank of the family inside that layer, starting at 1.
      • +
      +

      For a given index and a given sibling annex, a family therefore always receives the same CHROM and POS, whatever the draw, the options or the number of sites requested. Two runs on the same index can be compared position by position. This stability is lost if the sibling annex is rebuilt, or if the index is rebuilt, merged or filtered, because the families are then enumerated again. Positions are never comparable between two different indexes; use ID for that. Even rebuilding an index from the very same genomes may enumerate the families in another order: on a small test index built twice from identical input, the family CGATTNGTTAA had POS 440 the first time and 371 the second, with the same ID.

      +

      The file is sorted by partition, then layer, then position, which is also the order of the ##contig lines, so tools that require a sorted file accept it directly. Positions are sparse: only the drawn families are written.

      +

      Identifier: ID

      +

      The ID column holds a label for the family that does not depend on the index. It is built from one of the family's k-mers by replacing its central base with N (for example CGATTNGTTAA for k = 11), after choosing, among the family's possible orientations, the one that sorts first. All the members of a family, whichever strand they were read on, give the same label.

      +

      This is what makes two files comparable even when they come from different indexes: the same family has the same ID in both, while its CHROM and POS will generally differ. It is also unique within a file, since two different families never share a label.

      +

      Alleles: REF and ALT

      +

      The alleles are the bases carried at the centre of the family's k-mers, as in the FASTA. Two points are worth knowing.

      +

      First, the base is the central base of the member's canonical form (the smaller of the k-mer and its reverse complement). That is not necessarily the strand of the ID label, so the letters should not be matched to the ID sequence by position; they are labels for the allele, not a reading of that sequence.

      +

      Second, REF carries no meaning of reference: it is simply the most frequent allele among the genomes of the draw, each allele copy counted once, with the alphabetical order A < C < G < T deciding ties. ALT lists the remaining observed bases in the order A, C, G, T, separated by commas. There is no option to choose a reference genome.

      +

      Absence: the <ABS> allele

      +

      A genome may carry none of the family's k-mers. Without --free-loss this absence is treated as a real state (the family was lost, or is absent from that genome), which the FASTA shows as -. In the VCF it becomes one more allele, the symbolic allele <ABS>, listed last in ALT and declared in the header by ##ALT=<ID=ABS,Description="Family absent from the genome">. A genome without the family receives the genotype of <ABS>; for a site with ALT equal to G,T,<ABS> that is 3/3. A site that differs between genomes only by presence versus absence has ALT equal to <ABS>.

      +

      The VCF specification gives the plain character * a meaning of its own (an allele missing because of an overlapping deletion described by another record), which is not the situation here, so it is deliberately not used. <ABS> is a free-form symbolic identifier, outside the reserved structural-variant names (DEL, INS, DUP, INV, CNV) and outside <*>, which belongs to reference blocks. Tools that only look at sequence alleles treat the site as having an alternative that is not a base.

      +

      With --free-loss, absence is declared missing data instead: <ABS> never appears, and the genomes concerned get a missing genotype.

      +

      Genotypes

      +

      Every genome gets one GT value per site. The ploidy of these values is a convention chosen from the options, and the header says which (Genotype (diploid) or Genotype (haploid)).

      + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      Cell in the alignmentDiploid (default, --only-diploid)Haploid (--no-ambiguity)
      one base XX/X, for example 0/0 or 1/1X, for example 0
      two bases (IUPAC R Y S W K M)the heterozygote, for example 0/1does not occur
      three or four bases (B D H V N)./. (not expressible)does not occur
      absent family (<ABS>)<ABS>/<ABS>, for example 3/3the index of <ABS>
      missing (?)./..
      +

      The two allele indices of a diploid genotype are always written in increasing order (0/1, never 1/0), and the separator is / because no phasing information exists.

      +

      Why diploid by default? A genome can carry several forms of the same family, which the alignment records as an IUPAC ambiguity code. A diploid genotype can express up to two forms, and --only-diploid removes the genomes that would need more (three or four forms) before the sites are selected, so that no ./. comes from them. --no-ambiguity goes further and treats every such genome as missing, which makes all genotypes haploid.

      +

      This is a convention, not a measurement. The file records which forms a genome carries, not how many copies of each: A/A/G and A/G/G both give the code R and the genotype 0/1. Likewise, a genome with two forms may be heterozygous at one locus, or it may carry two similar regions of its genome (paralogues, repeats); the file cannot tell these apart. Do not read the genotypes as calls from sequencing reads.

      +

      Which sites are written

      +

      Only the families that survive the draw and its masking appear. Within those:

      +
        +
      • a site needs at least one genome with a base or (in diploid mode) a two-base code, because REF is a base; a site where every genome is missing, absent or has an ambiguity code too wide for the mode is left out of the VCF while remaining in the FASTA;
      • +
      • the log line at the end of the run gives the number of sites written out of the number drawn, so the difference is visible.
      • +
      +

      A site is selected because its genomes show at least two distinct states, and absence counts as a state. A site that differs between genomes only by presence versus absence is therefore a legitimate site, written with ALT equal to <ABS>.

      +

      Using the file with other tools

      +
        +
      • The contigs are virtual and have no declared length, and the file refers to no reference. Tools that need one (annotation, liftover) cannot use it; tools that only read sites and genotypes (bcftools view, query, stats, population-genetics software, PLINK-style converters) can.
      • +
      • If a tool rejects multiallelic sites or the <ABS> allele, bcftools view -m2 -M2 keeps biallelic sites only, and bcftools view -e 'ALT="<ABS>"' removes the sites involving absence. The file as written was read by bcftools and indexed by tabix without complaint on a small test index, in both diploid and haploid modes. bcftools stats counts the sites with <ABS> among the "others" (symbolic alleles) rather than as SNPs.
      • +
      • If a tool expects haploid data, run phylo with --no-ambiguity rather than converting the genotypes afterwards.
      • +
      +

      See also

      + + + + + + + + + + + + + + +
      +
      + + + +
      + +
      + + + +
      +
      +
      +
      + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/doc/index.html b/doc/index.html index 01928602..c3e3c189 100644 --- a/doc/index.html +++ b/doc/index.html @@ -367,15 +367,26 @@ -
    8. - + + + + + +
    9. + + + + + + -
    10. + + + + + + + + + + + + + + + + + + + + + +
    11. + + + + + + + + + + +
    12. + + + + @@ -1057,6 +1445,34 @@ + + + + + + +
    13. + + + + + + + + VCF pseudo-alignment + + + + + + + + +
    14. + + + + @@ -1205,11 +1621,11 @@

      All functionality is exposed through a single binary, obikmer, organized as subcommands.

      Core principles

        -
      • Kmers are of fixed, odd length \(k\), chosen at index-construction time in the range \([11, 31]\) (see Kmers and super-kmers).
      • -
      • Each kmer fits in a 64-bit word using a 2-bit-per-base encoding (see DNA encoding).
      • +
      • Kmers are of fixed, odd length \(k\), chosen at index-construction time in the range \([11, 31]\) (see Kmers).
      • +
      • Each kmer fits in a 64-bit word using a 2-bit-per-base encoding (see DNA encoding).
      • Kmers are handled in canonical form (\(\text{canonical}(kmer) = \min(kmer, \text{revcomp}(kmer))\)), making counting strand-independent.
      • -
      • Sequences are decomposed into super-kmers before storage, anchored on a hash-selected minimizer (see Minimizer selection), then routed to one of several partitions for parallel, memory-bounded processing (see Partitioning and indexing architecture).
      • -
      • Low-complexity kmers can be filtered out at index-construction time using an entropy-based score (see Low-complexity kmer filter).
      • +
      • Sequences are decomposed into super-kmers before storage, anchored on a hash-selected minimizer (see Minimizer selection), then routed to one of several partitions for parallel, memory-bounded processing (see Partitioning and indexing architecture).
      • +
      • Low-complexity kmers can be filtered out at index-construction time using an entropy-based score (see Low-complexity kmer filter).

      Commands

      diff --git a/doc/installation/index.html b/doc/installation/index.html index f4f93c3b..64de88ce 100644 --- a/doc/installation/index.html +++ b/doc/installation/index.html @@ -12,7 +12,7 @@ - + @@ -442,15 +442,26 @@ -
    15. - + + + + + +
    16. + + + + + + -
    17. + + + + + + + + + + + + + + + + + + + + + +
    18. + + + + + + + + + + +
    19. + + + + @@ -1132,6 +1520,34 @@ + + + + + + +
    20. + + + + + + + + VCF pseudo-alignment + + + + + + + + +
    21. + + + + diff --git a/doc/molecular-ecology-modified.csl b/doc/molecular-ecology-modified.csl new file mode 100644 index 00000000..d8551650 --- /dev/null +++ b/doc/molecular-ecology-modified.csl @@ -0,0 +1,223 @@ + + diff --git a/doc/references.bib b/doc/references.bib index fca4c8bc..23ca6fe4 100644 --- a/doc/references.bib +++ b/doc/references.bib @@ -259,3 +259,601 @@ url = {https://doi.org/10.1186/s12864-015-1647-5}, volume = 16, year = 2015} + +@article{Roberts2004-rz, + title = "{Reducing storage requirements for biological sequence + comparison}", + author = "Roberts, Michael and Hayes, Wayne and Hunt, Brian R and Mount, + Stephen M and Yorke, James A", + journal = "Bioinformatics (Oxford, England)", + publisher = "Oxford University Press (OUP)", + volume = 20, + number = 18, + pages = "3363--3369", + abstract = "MOTIVATION: Comparison of nucleic acid and protein sequences is a + fundamental tool of modern bioinformatics. A dominant method of + such string matching is the 'seed-and-extend' approach, in which + occurrences of short subsequences called 'seeds' are used to + search for potentially longer matches in a large database of + sequences. Each such potential match is then checked to see if it + extends beyond the seed. To be effective, the seed-and-extend + approach needs to catalogue seeds from virtually every substring + in the database of search strings. Projects such as mammalian + genome assemblies and large-scale protein matching, however, have + such large sequence databases that the resulting list of seeds + cannot be stored in RAM on a single computer. This significantly + slows the matching process. RESULTS: We present a simple and + elegant method in which only a small fraction of seeds, called + 'minimizers', needs to be stored. Using minimizers can speed up + string-matching computations by a large factor while missing only + a small fraction of the matches found using all seeds.", + month = dec, + year = 2004, + url = "https://dx.doi.org/10.1093/bioinformatics/bth408", + doi = "10.1093/bioinformatics/bth408", + pmid = 15256412, + issn = "1367-4803,1367-4811", + language = "en" + } + +@INPROCEEDINGS{Steele2014-cz, + title = "{Fast splittable pseudorandom number generators}", + author = "Steele, Jr, Guy L and Lea, Doug and Flood, Christine H", + booktitle = "{Proceedings of the 2014 ACM International Conference on Object + Oriented Programming Systems Languages \& Applications}", + publisher = "ACM", + address = "New York, NY, USA", + abstract = "We describe a new algorithm SplitMix for an object-oriented and + splittable pseudorandom number generator (PRNG) that is quite + fast: 9 64-bit arithmetic/logical operations per 64 bits + generated. A conventional linear PRNG object provides a generate + method that returns one pseudorandom value and updates the state + of the PRNG, but a splittable PRNG object also has a second + operation, split, that replaces the original PRNG object with two + (seemingly) independent PRNG objects, by creating and returning a + new such object and updating the state of the original object. + Splittable PRNG objects make it easy to organize the use of + pseudorandom numbers in multithreaded programs structured using + fork-join parallelism. No locking or synchronization is required + (other than the usual memory fence immediately after object + creation). Because the generate method has no loops or + conditionals, it is suitable for SIMD or GPU implementation. We + derive SplitMix from the DotMix algorithm of Leiserson, Schardl, + and Sukha by making a series of program transformations and + engineering improvements. The end result is an object-oriented + version of the purely functional API used in the Haskell library + for over a decade, but SplitMix is faster and produces + pseudorandom sequences of higher quality; it is also far superior + in quality and speed to java.util.Random, and has been included + in Java JDK8 as the class java.util.SplittableRandom. We have + tested the pseudorandom sequences produced by SplitMix using two + standard statistical test suites (DieHarder and TestU01) and they + appear to be adequate for ``everyday'' use, such as in Monte + Carlo algorithms and randomized data structures where speed is + important.", + month = oct, + year = 2014, + url = "http://dx.doi.org/10.1145/2660193.2660195", + doi = "10.1145/2660193.2660195", + isbn = 9781450325851 + } + +@online{Stafford2011, + author = {Stafford, David}, + title = {Better Bit Mixing - Improving on MurmurHash3's 64-bit Finalizer}, + date = {2011-09-28}, + url = {https://zimbry.blogspot.com/2011/09/better-bit-mixing-improving-on.html}, + urldate = {2026-09-12} + } + + @ARTICLE{Hamming1950-cw, + title = "{Error detecting and error correcting codes}", + author = "Hamming, R W", + journal = "The Bell System Technical Journal", + publisher = "Institute of Electrical and Electronics Engineers (IEEE)", + volume = 29, + number = 2, + pages = "147--160", + abstract = "The author was led to the study given in this paper from a + consideration of large scale computing machines in which a large + number of operations must be performed without a single error in + the end result. This problem of “doing things right” on a large + scale is not essentially new; in a telephone central office, for + example, a very large number of operations are performed while + the errors leading to wrong numbers are kept well under control, + though they have not been completely eliminated. This has been + achieved, in part, through the use of self-checking circuits. The + occasional failure that escapes routine checking is still + detected by the customer and will, if it persists, result in + customer complaint, while if it is transient it will produce only + occasional wrong numbers. At the same time the rest of the + central office functions satisfactorily. In a digital computer, + on the other hand, a single failure usually means the complete + failure, in the sense that if it is detected no more computing + can be done until the failure is located and corrected, while if + it escapes detection then it invalidates all subsequent + operations of the machine. Put in other words, in a telephone + central office there are a number of parallel paths which are + more or less independent of each other; in a digital machine + there is usually a single long path which passes through the same + piece of equipment many, many times before the answer is + obtained.", + month = apr, + year = 1950, + url = "http://dx.doi.org/10.1002/j.1538-7305.1950.tb00463.x", + doi = "10.1002/j.1538-7305.1950.tb00463.x", + issn = "0005-8580" + } + + @ARTICLE{Ondov2016-dq, + title = "{Mash: fast genome and metagenome distance estimation using + {MinHash}}", + author = "Ondov, Brian D and Treangen, Todd J and Melsted, Páll and + Mallonee, Adam B and Bergman, Nicholas H and Koren, Sergey and + Phillippy, Adam M", + journal = "Genome biology", + volume = 17, + number = 1, + pages = 132, + abstract = "Mash extends the MinHash dimensionality-reduction technique to + include a pairwise mutation distance and P value significance + test, enabling the efficient clustering and search of massive + sequence collections. Mash reduces large sequences and sequence + sets to small, representative sketches, from which global mutation + distances can be rapidly estimated. We demonstrate several use + cases, including the clustering of all 54,118 NCBI RefSeq genomes + in 33 CPU h; real-time database search using assembled or + unassembled Illumina, Pacific Biosciences, and Oxford Nanopore + data; and the scalable clustering of hundreds of metagenomic + samples by composition. Mash is freely released under a BSD + license ( https://github.com/marbl/mash ).", + month = jun, + year = 2016, + url = "http://dx.doi.org/10.1186/s13059-016-0997-x", + keywords = "Alignment; Comparative genomics; Genomic distance; Metagenomics; + Nanopore; Sequencing", + doi = "10.1186/s13059-016-0997-x", + pmc = "PMC4915045", + pmid = 27323842, + issn = "1465-6906", + language = "en" + } + @ARTICLE{Bray1957-vv, + title = "{An ordination of the upland forest communities of southern + Wisconsin}", + author = "Bray, J Roger and Curtis, J T", + journal = "Ecological Monographs", + publisher = "Wiley", + volume = 27, + number = 4, + pages = "325--349", + month = oct, + year = 1957, + url = "http://dx.doi.org/10.2307/1942268", + doi = "10.2307/1942268", + issn = "0012-9615,1557-7015", + language = "en" + } + @ARTICLE{Gower1986-ri, + title = "{Metric and Euclidean properties of dissimilarity coefficients}", + author = "Gower, J C and Legendre, P", + journal = "Journal of Classification", + publisher = "Springer Science and Business Media LLC", + volume = 3, + number = 1, + pages = "5--48", + abstract = "We assemble here properties of certain dissimilarity coefficients + and are specially concerned with their metric and Euclidean + status. No attempt is made to be exhaustive as far as + coefficients are concerned, but certain mathematical results that + we have found useful are presented and should help establish + similar properties for other coefficients. The response to + different types of data is investigated, leading to guidance on + the choice of an appropriate coefficient.", + month = mar, + year = 1986, + url = "http://dx.doi.org/10.1007/bf01896809", + doi = "10.1007/bf01896809", + issn = "0176-4268,1432-1343", + language = "en" + } + @ARTICLE{Legendre2001-cq, + title = "{Ecologically meaningful transformations for ordination of species + data}", + author = "Legendre, Pierre and Gallagher, Eugene D", + journal = "Oecologia", + volume = 129, + number = 2, + pages = "271--280", + abstract = "This paper examines how to obtain species biplots in unconstrained + or constrained ordination without resorting to the Euclidean + distance [used in principal-component analysis (PCA) and + redundancy analysis (RDA)] or the chi-square distance [preserved + in correspondence analysis (CA) and canonical correspondence + analysis (CCA)] which are not always appropriate for the analysis + of community composition data. To achieve this goal, + transformations are proposed for species data tables. They allow + ecologists to use ordination methods such as PCA and RDA, which + are Euclidean-based, for the analysis of community data, while + circumventing the problems associated with the Euclidean distance, + and avoiding CA and CCA which present problems of their own in + some cases. This allows the use of the original (transformed) + species data in RDA carried out to test for relationships with + explanatory variables (i.e. environmental variables, or factors of + a multifactorial analysis-of-variance model); ecologists can then + draw biplots displaying the relationships of the species to the + explanatory variables. Another application allows the use of + species data in other methods of multivariate data analysis which + optimize a least-squares loss function; an example is K-means + partitioning.", + month = oct, + year = 2001, + url = "http://dx.doi.org/10.1007/s004420100716", + keywords = "Biplot diagram; Canonical correspondence analysis; Correspondence + analysis; Principal-component analysis; Redundancy analysis", + doi = "10.1007/s004420100716", + pmid = 28547606, + issn = "0029-8549,1432-1939", + language = "en" + } + @ARTICLE{Anderson2011-rq, + title = "{Navigating the multiple meanings of β diversity: a roadmap for + the practicing ecologist}", + author = "Anderson, Marti J and Crist, Thomas O and Chase, Jonathan M and + Vellend, Mark and Inouye, Brian D and Freestone, Amy L and + Sanders, Nathan J and Cornell, Howard V and Comita, Liza S and + Davies, Kendi F and Harrison, Susan P and Kraft, Nathan J B and + Stegen, James C and Swenson, Nathan G", + journal = "Ecology Letters", + publisher = "Wiley", + volume = 14, + number = 1, + pages = "19--28", + abstract = "A recent increase in studies of β diversity has yielded a + confusing array of concepts, measures and methods. Here, we + provide a roadmap of the most widely used and ecologically + relevant approaches for analysis through a series of mission + statements. We distinguish two types of β diversity: directional + turnover along a gradient vs. non-directional variation. + Different measures emphasize different properties of ecological + data. Such properties include the degree of emphasis on + presence/absence vs. relative abundance information and the + inclusion vs. exclusion of joint absences. Judicious use of + multiple measures in concert can uncover the underlying nature of + patterns in β diversity for a given dataset. A case study of + Indonesian coral assemblages shows the utility of a multi-faceted + approach. We advocate careful consideration of relevant + questions, matched by appropriate analyses. The rigorous + application of null models will also help to reveal potential + processes driving observed patterns in β diversity.", + month = jan, + year = 2011, + url = "http://dx.doi.org/10.1111/j.1461-0248.2010.01552.x", + doi = "10.1111/j.1461-0248.2010.01552.x", + pmid = 21070562, + issn = "1461-023X,1461-0248", + language = "en" + } + @ARTICLE{Jaccard1901-mx, + title = "{Étude comparative de la distribution florale dans une portion + des Alpes et du Jura}", + author = "Jaccard, Paul", + journal = "Bulletin de la Société vaudoise des sciences naturelles", + publisher = "Imprimerie Corbaz \& Comp.", + volume = 37, + number = 142, + pages = "547--579", + abstract = "Schweizer Zeitschriften online - Ein Service der ETH-Bibliothek. + Revues suisses en ligne - Un service de ETH Library. Swiss + journals online - A service by ETH Library.", + year = 1901, + url = "http://dx.doi.org/10.5169/seals-266450", + doi = "10.5169/seals-266450", + issn = "0037-9603", + language = "de" + } + @ARTICLE{Kosub2019-xt, + title = "{A note on the triangle inequality for the Jaccard distance}", + author = "Kosub, Sven", + journal = "Pattern Recognition Letters", + publisher = "Elsevier BV", + volume = 120, + pages = "36--38", + month = apr, + year = 2019, + url = "http://dx.doi.org/10.1016/j.patrec.2018.12.007", + doi = "10.1016/j.patrec.2018.12.007", + issn = "0167-8655,1872-7344", + language = "en" + } + @ARTICLE{Fleischmann2014-ed, + title = {Evolution of genome size and chromosome number in the + carnivorous plant genus Genlisea (Lentibulariaceae), with a new + estimate of the minimum genome size in angiosperms}, + author = "Fleischmann, Andreas and Michael, Todd P and Rivadavia, Fernando + and Sousa, Aretuza and Wang, Wenqin and Temsch, Eva M and + Greilhuber, Johann and Müller, Kai F and Heubl, Günther", + journal = "Annals of Botany", + publisher = "Oxford University Press (OUP)", + volume = 114, + number = 8, + pages = "1651--1663", + abstract = "BACKGROUND AND AIMS: Some species of Genlisea possess ultrasmall + nuclear genomes, the smallest known among angiosperms, and some + have been found to have chromosomes of diminutive size, which may + explain why chromosome numbers and karyotypes are not known for + the majority of species of the genus. However, other members of + the genus do not possess ultrasmall genomes, nor do most taxa + studied in related genera of the family or order. This study + therefore examined the evolution of genome sizes and chromosome + numbers in Genlisea in a phylogenetic context. The correlations + of genome size with chromosome number and size, with the + phylogeny of the group and with growth forms and habitats were + also examined. METHODS: Nuclear genome sizes were measured from + cultivated plant material for a comprehensive sampling of taxa, + including nearly half of all species of Genlisea and representing + all major lineages. Flow cytometric measurements were conducted + in parallel in two laboratories in order to compare the + consistency of different methods and controls. Chromosome counts + were performed for the majority of taxa, comparing different + staining techniques for the ultrasmall chromosomes. KEY RESULTS: + Genome sizes of 15 taxa of Genlisea are presented and interpreted + in a phylogenetic context. A high degree of congruence was found + between genome size distribution and the major phylogenetic + lineages. Ultrasmall genomes with 1C values of <100 Mbp were + almost exclusively found in a derived lineage of South American + species. The ancestral haploid chromosome number was inferred to + be n = 8. Chromosome numbers in Genlisea ranged from 2n = 2x = 16 + to 2n = 4x = 32. Ascendant dysploid series (2n = 36, 38) are + documented for three derived taxa. The different ploidy levels + corresponded to the two subgenera, but were not directly + correlated to differences in genome size; the three different + karyotype ranges mirrored the different sections of the genus. + The smallest known plant genomes were not found in G. margaretae, + as previously reported, but in G. tuberosa (1C ≈ 61 Mbp) and some + strains of G. aurea (1C ≈ 64 Mbp). CONCLUSIONS: Genlisea is an + ideal candidate model organism for the understanding of genome + reduction as the genus includes species with both relatively + large (∼1700 Mbp) and ultrasmall (∼61 Mbp) genomes. This + comparative, phylogeny-based analysis of genome sizes and + karyotypes in Genlisea provides essential data for selection of + suitable species for comparative whole-genome analyses, as well + as for further studies on both the molecular and cytogenetic + basis of genome reduction in plants.", + month = dec, + year = 2014, + url = "http://dx.doi.org/10.1093/aob/mcu189", + keywords = "Bladderwort; Genlisea; Lamiales; Lentibulariaceae; carnivorous + plant; chromosome number; flow cytometry; genome miniaturization; + genome size", + doi = "10.1093/aob/mcu189", + pmc = "PMC4649684", + pmid = 25274549, + issn = "0305-7364,1095-8290", + language = "en" + } + @ARTICLE{Fernandez2024-kd, + title = "A 160 Gbp fork fern genome shatters size record for eukaryotes", + author = "Fernández, Pol and Amice, Rémy and Bruy, David and Christenhusz, + Maarten J M and Leitch, Ilia J and Leitch, Andrew L and Pokorny, + Lisa and Hidalgo, Oriane and Pellicer, Jaume", + journal = "iScience", + publisher = "Elsevier BV", + volume = 27, + number = 6, + pages = 109889, + abstract = "Vascular plants are exceptional among eukaryotes due to their + outstanding genome size diversity which ranges ∼2,400-fold, + including the largest genome so far recorded in the angiosperm + Paris japonica (148.89 Gbp/1C). Despite available data showing + that giant genomes are restricted across the Tree of Life, the + biological limits to genome size expansion remain to be + established. Here, we report the discovery of an even larger + eukaryotic genome in Tmesipteris oblanceolata, a New Caledonian + fork fern. At 160.45 Gbp/1C, this record-breaking genome + challenges current understanding and opens new avenues to explore + the evolutionary dynamics of genomic gigantism.", + month = jun, + year = 2024, + url = "http://dx.doi.org/10.1016/j.isci.2024.109889", + keywords = "Plant evolution; Plant genetics", + doi = "10.1016/j.isci.2024.109889", + pmc = "PMC11270024", + pmid = 39055604, + issn = "2589-0042", + language = "en" + } + @ARTICLE{Fraser1995-jc, + title = "The minimal gene complement of Mycoplasma genitalium", + author = "Fraser, C M and Gocayne, J D and White, O and Adams, M D and + Clayton, R A and Fleischmann, R D and Bult, C J and Kerlavage, A + R and Sutton, G and Kelley, J M and Fritchman, R D and Weidman, J + F and Small, K V and Sandusky, M and Fuhrmann, J and Nguyen, D + and Utterback, T R and Saudek, D M and Phillips, C A and Merrick, + J M and Tomb, J F and Dougherty, B A and Bott, K F and Hu, P C + and Lucier, T S and Peterson, S N and Smith, H O and Hutchison, + 3rd, C A and Venter, J C", + journal = "Science (New York, N.Y.)", + publisher = "American Association for the Advancement of Science (AAAS)", + volume = 270, + number = 5235, + pages = "397--403", + abstract = "The complete nucleotide sequence (580,070 base pairs) of the + Mycoplasma genitalium genome, the smallest known genome of any + free-living organism, has been determined by whole-genome random + sequencing and assembly. A total of only 470 predicted coding + regions were identified that include genes required for DNA + replication, transcription and translation, DNA repair, cellular + transport, and energy metabolism. Comparison of this genome to + that of Haemophilus influenzae suggests that differences in + genome content are reflected as profound differences in + physiology and metabolic capacity between these two organisms.", + month = oct, + year = 1995, + url = "http://dx.doi.org/10.1126/science.270.5235.397", + doi = "10.1126/science.270.5235.397", + pmid = 7569993, + issn = "0036-8075,1095-9203", + language = "en" + } + @ARTICLE{Schneiker2007-gs, + title = "Complete genome sequence of the myxobacterium Sorangium + cellulosum", + author = "Schneiker, Susanne and Perlova, Olena and Kaiser, Olaf and Gerth, + Klaus and Alici, Aysel and Altmeyer, Matthias O and Bartels, + Daniela and Bekel, Thomas and Beyer, Stefan and Bode, Edna and + Bode, Helge B and Bolten, Christoph J and Choudhuri, Jomuna V and + Doss, Sabrina and Elnakady, Yasser A and Frank, Bettina and + Gaigalat, Lars and Goesmann, Alexander and Groeger, Carolin and + Gross, Frank and Jelsbak, Lars and Jelsbak, Lotte and Kalinowski, + Jörn and Kegler, Carsten and Knauber, Tina and Konietzny, + Sebastian and Kopp, Maren and Krause, Lutz and Krug, Daniel and + Linke, Bukhard and Mahmud, Taifo and Martinez-Arias, Rosa and + McHardy, Alice C and Merai, Michelle and Meyer, Folker and + Mormann, Sascha and Muñoz-Dorado, Jose and Perez, Juana and + Pradella, Silke and Rachid, Shwan and Raddatz, Günter and + Rosenau, Frank and Rückert, Christian and Sasse, Florenz and + Scharfe, Maren and Schuster, Stephan C and Suen, Garret and + Treuner-Lange, Anke and Velicer, Gregory J and Vorhölter, + Frank-Jörg and Weissman, Kira J and Welch, Roy D and Wenzel, + Silke C and Whitworth, David E and Wilhelm, Susanne and Wittmann, + Christoph and Blöcker, Helmut and Pühler, Alfred and Müller, Rolf", + journal = "Nature Biotechnology", + publisher = "Springer Science and Business Media LLC", + volume = 25, + number = 11, + pages = "1281--1289", + abstract = "The genus Sorangium synthesizes approximately half of the + secondary metabolites isolated from myxobacteria, including the + anti-cancer metabolite epothilone. We report the complete genome + sequence of the model Sorangium strain S. cellulosum So ce56, + which produces several natural products and has morphological and + physiological properties typical of the genus. The circular + genome, comprising 13,033,779 base pairs, is the largest + bacterial genome sequenced to date. No global synteny with the + genome of Myxococcus xanthus is apparent, revealing an + unanticipated level of divergence between these myxobacteria. A + large percentage of the genome is devoted to regulation, + particularly post-translational phosphorylation, which probably + supports the strain's complex, social lifestyle. This regulatory + network includes the highest number of eukaryotic protein + kinase-like kinases discovered in any organism. Seventeen + secondary metabolite loci are encoded in the genome, as well as + many enzymes with potential utility in industry.", + month = nov, + year = 2007, + url = "http://dx.doi.org/10.1038/nbt1354", + doi = "10.1038/nbt1354", + pmid = 17965706, + issn = "1087-0156,1546-1696", + language = "en" + } + @incollection{Karlin1985, + author = {Karlin, Samuel and Ost, Fr{\'e}d{\'e}ric}, + title = {Maximal segmental match length among random sequences from a finite alphabet}, + booktitle = {Proceedings of the Berkeley Conference in Honor of Jerzy Neyman and Jack Kiefer}, + editor = {Le Cam, Lucien M. and Olshen, Richard A.}, + volume = {1}, + pages = {225--243}, + year = {1985}, + publisher = {Wadsworth}, + address = {Monterey, CA} + } + @article{Karlin1988, + author = {Karlin, Samuel and Ost, Fr{\'e}d{\'e}ric}, + title = {Maximal length of common words among random letter sequences}, + journal = {The Annals of Probability}, + volume = {16}, + number = {2}, + pages = {535--563}, + year = {1988} + } + @ARTICLE{Le-Cam1960-pe, + title = "{An approximation theorem for the Poisson binomial distribution}", + author = "Le Cam, Lucien", + journal = "Pacific Journal of Mathematics", + publisher = "Mathematical Sciences Publishers", + volume = 10, + number = 4, + pages = "1181--1197", + month = dec, + year = 1960, + url = "http://dx.doi.org/10.2140/pjm.1960.10.1181", + doi = "10.2140/pjm.1960.10.1181", + issn = "0030-8730,1945-5844" + } + + @ARTICLE{Hoeffding1956-bx, + title = "{On the distribution of the number of successes in independent + trials}", + author = "Hoeffding, Wassily", + journal = "The Annals of Mathematical Statistics", + publisher = "Institute of Mathematical Statistics", + volume = 27, + number = 3, + pages = "713--721", + abstract = "Let $S$ be the number of successes in $n$ independent trials, and + let $p_j$ denote the probability of success in the $j$th trial, + $j = 1, 2, \cdots, n$ (Poisson trials). We consider the problem + of finding the maximum and the minimum of $Eg(S),$ the expected + value of a given real-valued function of $S,$ when $ES = np$ is + fixed. It is well known that the maximum of the variance of $S$ + is attained when $p_1 = p_2 = \cdots = p_n = p.$ This can be + interpreted as showing that the variability in the number of + successes is highest when the successes are equally probable + (Bernoulli trials). This interpretation is further supported by + the following two theorems, proved in this paper. If $b$ and $c$ + are two integers, $0 \leqq b \leqq np \leqq c \leqq n,$ the + probability $P(b \leqq S \leqq c)$ attains its minimum if and + only if $p_1 = p_2 = \cdots = p_n = p,$ unless $b = 0$ and $c = + n$ (Theorem 5, a corollary of Theorem 4, which gives the maximum + and the minimum of $P(S \leqq c)$). If $g$ is a strictly convex + function, $Eg(S)$ attains its maximum if and only if $p_1 = p_2 = + \cdots = p_n = p$ (Theorem 3). These results are obtained with + the help of two theorems concerning the extrema of the expected + value of an arbitrary function $g(S)$ under the condition $ES = + np.$ Theorem 1 gives necessary conditions for the maximum and the + minimum of $Eg(S).$ Theorem 2 gives a partial characterization of + the set of points at which an extremum is attained. Corollary 2.1 + states that the maximum and the minimum are attained when $p_1, + p_2, \cdots, p_n$ take on, at most, three different values, only + one of which is distinct from 0 and 1. Applications of Theorems 3 + and 5 to problems of estimation and testing are pointed out in + Section 5.", + month = sep, + year = 1956, + url = "http://dx.doi.org/10.1214/aoms/1177728178", + doi = "10.1214/aoms/1177728178", + issn = "0003-4851,2168-8990", + language = "en" + } + @ARTICLE{Gleser1975-yu, + title = "{On the distribution of the number of successes in independent + trials}", + author = "Gleser, Leon Jay", + journal = "Annals of Probability", + publisher = "Institute of Mathematical Statistics", + volume = 3, + number = 1, + pages = "182--188", + abstract = "Let $S$ be the number of successes in $n$ independent Bernoulli + trials, where $p_j$ is the probability of success on the $j$th + trial. Let $\mathbf{p} = (p_1, p_2, \cdots, p_n)$, and for any + integer $c, 0 \leqq c \leqq n$, let $H(c \mid \mathbf{p}) = P\{S + \leqq c\}$. Let $\mathbf{p}^{(1)}$ be one possible choice of + $\mathbf{p}$ for which $E(S) = \lambda$. For any $n \times n$ + doubly stochastic matrix $\Pi$, let $\mathbf{p}^{(2)} = + \mathbf{p}^{(1)}\Pi$. Then in the present paper it is shown that + $H(c \mid \mathbf{p}^{(1)}) \leqq H(c \mid \mathbf{p}^{(2)})$ for + $0 \leqq c \leqq \lbrack\lambda - 2\rbrack$, and $H(c \mid + \mathbf{p}^{(1)}) \geqq H(c \mid \mathbf{p}^{(2)})$ for + $\lbrack\lambda + 2\rbrack \leqq c \leqq n$. These results + provide a refinement of inequalities for $H(c \mid \mathbf{p})$ + obtained by Hoeffding [3]. Their derivation is achieved by + applying consequences of the partial ordering of majorization.", + month = feb, + year = 1975, + url = "http://dx.doi.org/10.2307/2959277", + doi = "10.1214/aop/1176996461", + issn = "0091-1798,2168-894X" + } diff --git a/doc/sitemap.xml.gz b/doc/sitemap.xml.gz index 03a78e939757e3344acaf2aa3115fcca5d967fa9..ea8ab0db33f5aabffd81b1a154172da3e59897d5 100644 GIT binary patch delta 13 Ucmb=gXP58h;9#)%H<7&p02;3Z3IG5A delta 13 Ucmb=gXP58h;Al8MVIq4403Ox^uK)l5 diff --git a/doc/theory/encoding/index.html b/doc/theory/kmer_indexing/encoding/index.html similarity index 65% rename from doc/theory/encoding/index.html rename to doc/theory/kmer_indexing/encoding/index.html index 533eb4dd..a116192b 100644 --- a/doc/theory/encoding/index.html +++ b/doc/theory/kmer_indexing/encoding/index.html @@ -9,16 +9,16 @@ - + - + - + @@ -27,7 +27,7 @@ - + @@ -46,7 +46,7 @@ - + @@ -80,7 +80,7 @@
      @@ -1042,7 +1460,7 @@
    22. - + @@ -1172,9 +1590,9 @@

      A kmer of length \(k\) (\(k \le 31\)) fits in a single 64-bit word. The first nucleotide occupies the two most significant bits, each following nucleotide occupies the next two bits, and unused low-order bits are zero. Extracting nucleotide i (0-indexed from the 5′ end) is a shift-and-mask operation.

      Reverse complement is computed by bit manipulation directly on the packed word, without any lookup table: complement every base, reverse the byte order, then reverse the order of 2-bit groups within each byte in two more passes, and finally realign the result to the most-significant bits.

      Canonical form

      -

      The canonical form of a kmer is the lexicographic minimum of the kmer and its reverse complement:

      -
      \[\text{canonical}(kmer) = \min\big(kmer,\ \text{revcomp}(kmer)\big)\]
      -

      Using the canonical form halves the kmer space and makes counting strand-independent: a kmer and its reverse complement are always treated as the same entity, regardless of which DNA strand was sequenced.

      +

      The canonical form of a packed 2-bit sequence is the lexicographic minimum of the sequence and its reverse complement:

      +
      \[\text{canonical}(x) = \min\big(x,\ \text{revcomp}(x)\big)\]
      +

      This is a single operation on the packed representation, so it applies identically regardless of what x represents — a kmer (see Kmers), an m-mer (see Minimizer selection), or a super-kmer (see Super-kmers). Using the canonical form halves the relevant space and makes counting/selection strand-independent: a sequence and its reverse complement are always treated as the same entity, regardless of which DNA strand was sequenced.

      @@ -1224,10 +1642,10 @@ - + - + diff --git a/doc/theory/entropy_filter/index.html b/doc/theory/kmer_indexing/entropy_filter/index.html similarity index 70% rename from doc/theory/entropy_filter/index.html rename to doc/theory/kmer_indexing/entropy_filter/index.html index bc0f568a..a7908340 100644 --- a/doc/theory/entropy_filter/index.html +++ b/doc/theory/kmer_indexing/entropy_filter/index.html @@ -9,16 +9,16 @@ - + - + - + @@ -27,7 +27,7 @@ - + @@ -46,7 +46,7 @@ - + @@ -80,7 +80,7 @@
      @@ -1064,7 +1482,7 @@
    23. - + @@ -1252,10 +1670,10 @@ - + - + diff --git a/doc/theory/indexing_architecture/index.html b/doc/theory/kmer_indexing/indexing_architecture/index.html similarity index 65% rename from doc/theory/indexing_architecture/index.html rename to doc/theory/kmer_indexing/indexing_architecture/index.html index b7fbba06..96663f3a 100644 --- a/doc/theory/indexing_architecture/index.html +++ b/doc/theory/kmer_indexing/indexing_architecture/index.html @@ -9,16 +9,16 @@ - + - + - + @@ -27,7 +27,7 @@ - + @@ -46,7 +46,7 @@ - + @@ -80,7 +80,7 @@
    24. + + + + + + + + + + + + + + + +
    25. + + + + + + + + + + +
    26. + + + + @@ -533,7 +912,7 @@
    27. - + @@ -561,7 +940,7 @@
    28. - + @@ -589,7 +968,7 @@
    29. - + @@ -617,7 +996,7 @@
    30. - + @@ -645,7 +1024,7 @@
    31. - + @@ -673,7 +1052,7 @@
    32. - + @@ -701,7 +1080,7 @@
    33. - + @@ -729,7 +1108,7 @@
    34. - + @@ -757,7 +1136,7 @@
    35. - + @@ -785,7 +1164,7 @@
    36. - + @@ -813,7 +1192,7 @@
    37. - + @@ -841,7 +1220,7 @@
    38. - + @@ -869,7 +1248,7 @@
    39. - + @@ -897,7 +1276,7 @@
    40. - + @@ -925,7 +1304,7 @@
    41. - + @@ -1007,7 +1386,7 @@
    42. - + @@ -1028,6 +1407,34 @@ + + + + + + +
    43. + + + + + + + + VCF pseudo-alignment + + + + + + + + +
    44. + + + + @@ -1042,7 +1449,7 @@
    45. - + @@ -1097,17 +1504,6 @@ -
    46. - -
    47. - - - - Why hashing is necessary - - - -
    48. @@ -1141,15 +1537,13 @@

      Partitioning and indexing architecture

      An index is split into a fixed number of partitions, each handling an independent, disjoint slice of the kmer space. Partitioning keeps the working set of each stage small enough to process efficiently and enables parallel construction and querying.

      Routing

      -

      The canonical minimizer of a super-kmer (see Minimizer selection) is hashed to produce a \(p\)-bit routing value that selects the destination partition:

      +

      The canonical minimizer of a super-kmer (see Minimizer selection) is hashed to produce a \(p\)-bit routing value that selects the destination partition — the low \(p\) bits of the minimizer's hash:

      canonical minimizer → hash(minimizer) → p-bit value → partition index
       
      -

      The routing value is recomputed whenever it is needed (during construction and again at query time) rather than stored — it is not part of the on-disk super-kmer representation.

      -

      Within a partition, kmers are indexed as plain values via a minimal perfect hash function (see On-disk storage); the minimizer plays no further role once a super-kmer has reached its partition.

      -

      Why hashing is necessary

      -

      A canonical minimizer is an m-mer (\(m \in \{9, 11, 13, 15\}\)), and its distribution over all possible m-mer values is not uniform — as the lexicographic minimum of a window, small values are systematically over-represented (Golan & Shur 2025; Kille et al. 2023; Pan & Reinert 2024; Zheng et al. 2020, 2021)1 2 3 4 5. Routing directly on the raw minimizer value would therefore produce badly unbalanced partitions.

      -

      Hashing the minimizer before routing redistributes this skewed distribution uniformly across partitions. This works reliably because the number of partition-index bits \(p\) is chosen well below the number of bits available in the minimizer (\(2m\)): even with strong bias in the minimizer distribution, the hash has enough entropy margin to absorb it, provided the number of distinct minimizers actually observed is much larger than the number of partitions.

      +
      \[\text{partition} = H(\text{minimizer}) \bmod 2^p\]
      +

      Within a partition, kmers are indexed as plain values via a minimal perfect hash function (see On-disk storage); the minimizer plays no further role once a super-kmer has reached its partition.

      Parameter guidance

      +

      Even though \(H\) already makes minimizer values well-distributed (see Minimizer selection), choosing \(p\) well below the number of bits available in the minimizer (\(2m\)) leaves a comfortable entropy margin, provided the number of distinct minimizers actually observed is much larger than the number of partitions.

    49. @@ -1187,26 +1581,6 @@

      The number of partitions must satisfy \(p \le 2m\), and in practice \(p\) is chosen well below that bound to leave a comfortable entropy margin. For \(k=31\), \(m=13\), \(p=10\) (1024 partitions), partition load is well balanced on real genomic data.

      -
      -
      -
        -
      1. -

        Zheng, H., Kingsford, C. & Marçais, G. (2020). Improved design and analysis of practical minimizers. Bioinformatics (Oxford, England), 36, i119--i127. ↩

        -
      2. -
      3. -

        Zheng, H., Kingsford, C. & Marçais, G. (2021). Sequence-specific minimizers via polar sets. Bioinformatics (Oxford, England), 37, i187--i195. ↩

        -
      4. -
      5. -

        Pan, C. & Reinert, K. (2024). A simple refined DNA minimizer operator enables 2-fold faster computation. Bioinformatics (Oxford, England), 40. ↩

        -
      6. -
      7. -

        Kille, B., Garrison, E., Treangen, T.J. & Phillippy, A.M. (2023). Minmers are a generalization of minimizers that enable unbiased local jaccard estimation. Bioinformatics (Oxford, England), 39. ↩

        -
      8. -
      9. -

        Golan, S. & Shur, A.M. (2025). Expected density of random minimizers. In: Lecture notes in computer science, Lecture notes in computer science. Springer Nature Switzerland, Cham, pp. 347--360. ↩

        -
      10. -
      -
      @@ -1256,10 +1630,10 @@ - + - + diff --git a/doc/theory/kmer_indexing/kmers/index.html b/doc/theory/kmer_indexing/kmers/index.html new file mode 100644 index 00000000..1fcf8f89 --- /dev/null +++ b/doc/theory/kmer_indexing/kmers/index.html @@ -0,0 +1,2250 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + Kmers - obikmer — User Guide + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      + +
      + + + + + + +
      + + +
      + +
      + + + + + + +
      +
      + + + +
      +
      +
      + + + + + +
      +
      +
      + + + + + + + +
      + +
      + + + + + +

      K-mer size

      +

      A k-mer is a DNA sequence of length \(k\). OBIKmer is designed to index large k-mers. Here, large refers to a k-mer length for which the probability of observing the same k-mer in several genomes by chance alone is sufficiently small. The k-mer length must therefore be large enough that, when a k-mer is observed in several genomes of the dataset, the probability that this shared occurrence is due to chance alone is sufficiently low for the match to be considered evidence of homology.

      +

      OBIKmer restricts \(k\) to odd values in the range

      +
      \[ +11 \leq k \leq 31. +\]
      +

      These limits result from two different considerations. The upper bound is imposed by the representation used by OBIKmer, whereas the lower bound defines a practical range of k-mer lengths for genome-scale sequence comparisons. This range covers most practical applications: the lower limit is well below the k-mer size that can be considered large, even for the smallest bacterial genomes, whereas the upper limit provides sufficient specificity for most applications, including those involving very large genomes.

      +

      Representation limit

      +

      OBIKmer uses two bits to encode each nucleotide. Therefore, a k-mer requires \(2k\) bits. To ensure computational efficiency, OBIKmer limits the representation of k-mers to a single 64-bit word, thereby restricting their length.

      +
      \[ +2k \leq 64, +\]
      +

      This gives the hard representation limit: \(k \leq 32.\) Furthermore, OBIKmer only considers odd values of \(k\), meaning the largest supported k-mer size is \(31\).

      +

      K-mer specificity

      +

      One simple way to illustrate the effect of k-mer size is to consider a random DNA sequence in which the four nucleotides are independent and equally probable. There are \(4^k\) possible k-mers, so a particular k-mer is expected to occur once every \(4^k\) positions on average. In a sequence of length \(G\), the expected number of occurrences of a particular k-mer is approximately \(G \cdot 4^{-k}\). When k is large enough to ensure \(4^k >> G\), the chance of a second occurrence of the same k-mer becomes close to zero. Consequently, observing the same k-mer twice can only be explained by a homology relationship, except for low-complexity k-mers, which are overrepresented in genomes.

      +

      Estimating False discovery rate on homology inference

      +

      OBIKmer uses k-mers long enough that their occurrence in multiple indexed genomes provides evidence of homology. The corresponding false discovery rate (FDR) can be expressed in terms of the probability of observing a k-mer in at least \(R\) of the \(S\) indexed genomes under a purely random model.

      +

      The random model

      +

      The random model, or null hypothesis, assumes that the \(S\) genomes are unrelated and that each genome is an independent random sequence in which each nucleotide occurs with equal probability. Rejecting this hypothesis for a given k-mer indicates that its presence in multiple genomes is unlikely to be explained by chance alone, providing evidence for homology.

      +

      For a genome of size \(G\), the number of occurrences of a given canonical k-mer \(K\) is approximated by a Poisson distribution with parameter

      +
      \[ \lambda = 2\frac{G}{4^k}, \]
      +

      which is also the expected number of occurrences under the random model. The factor of two accounts for k-mer canonicalisation, which halves the k-mer space when \(k\) is odd.

      +

      The probability of observing at least one occurrence of this k-mer is therefore

      +
      \[ +\begin{aligned} +P(n_K > 0) &= 1-e^{-\lambda} \\ +&= 1-e^{-2G/4^k}. +\end{aligned} +\]
      +

      Assuming equal genome lengths, the number of genomes containing at least one occurrence of \(K\) among the \(S\) sequences follows a binomial distribution. The probability of observing \(K\) in at least \(r\) genomes is therefore

      +
      \[ +P(X \geq r)=\sum_{j=r}^{S}\binom{S}{j}\left(1-e^{-2G/4^k}\right)^j\left(e^{-2G/4^k}\right)^{S-j}. +\]
      +

      This sum is the regularized incomplete beta function \(I_p(r,\,S-r+1)\), where

      +
      \[ +p = 1-e^{-2G/4^k}. +\]
      +

      Different genome sizes

      +

      When genome sizes differ, the probability of observing a given k-mer also differs between genomes. A simple approximation is to replace the individual genome sizes \(G_i\) by their mean

      +
      \[ \bar G = \frac{1}{S}\sum_{i=1}^{S}G_i. \]
      +

      This gives a common probability

      +
      \[ \bar p = 1-e^{-2\bar G/4^k}, \]
      +

      and the number of genomes containing the k-mer can then be approximated by a binomial distribution:

      +
      \[ P(X\geq r)=\sum_{j=r}^{S}\binom{S}{j}\bar p^j(1-\bar p)^{S-j}. \]
      +

      This approximation is conservative for the upper tail. Hoeffding (1956 p. 717, Eq. 26)1 shows that, for a fixed mean, the binomial upper tail is at least as large as the corresponding Poisson-binomial tail for

      +
      \[ r\geq S\bar p+1. \]
      +

      Using the mean genome size therefore provides a conservative estimate of the FDR.

      +

      When genome sizes differ substantially, however, this approximation can become overly conservative. In that case, the exact Poisson-binomial distribution can be computed directly from the individual genome-specific probabilities \(p_i\).

      +

      Exact tail probability

      +

      With genome-specific probabilities \(p_i\), the number of genomes containing the k-mer follows a Poisson-binomial distribution. Writing the probability of exactly \(j\) presences as a sum over all subsets of \(j\) genomes would be impractical, since there can be up to \(2^S\) subsets. The distribution can instead be obtained by adding the genomes one at a time.

      +

      Let \(d_j\) be the probability that exactly \(j\) of the genomes processed so far contain the k-mer. Adding genome \(i\) leaves \(j\) unchanged if the genome does not contain the k-mer, and moves \(j-1\) to \(j\) if it does:

      +
      \[ +\left \{ +\begin{aligned} +d'_j &= d_j(1-p_i) + d_{j-1}p_i, \\ +d_0&=1 +\end{aligned} +\right . +\]
      +

      before processing the first genome.

      +

      The table \(d\) only needs to record how many genomes contain the k-mer, not which ones, reducing the computational cost to \(O(S^2)\). The result is independent of the order in which the genomes are processed and of the k-mer itself, so it can be computed once per index. The upper tail is then

      +
      \[ +P(X\geq r)=1-\sum_{j=0}^{r-1}d_j. +\]
      +

      Only \(d_0,\ldots,d_{r-1}\) are needed, reducing the cost to \(O(Sr)\).

      +

      When all \(p_i\) are small, as is typically the case for large \(k\), the Poisson-binomial distribution can also be approximated by a Poisson distribution with mean

      +
      \[ +\lambda=\sum_{i=1}^{S}p_i. +\]
      +

      The approximation error is bounded in terms of \(\sum_i p_i^2\) by Le Cam's inequality (LeCam1960-pe?).

      +
      +

      This probability decreases by a factor of four for every additional nucleotide in the k-mer. Conversely, it increases linearly with the amount of sequence searched.

      +

      This simple model already shows why very short k-mers provide limited specificity for genome-scale analyses. With \(k=11\),

      +
      \[ +4^{11}\simeq4.2\times10^6, +\]
      +

      which is a small combinatorial space even compared with the size of very small bacterial genomes. In contrast,

      +
      \[ +4^{31}\simeq4.6\times10^{18}, +\]
      +

      which is vastly larger than the number of k-mer positions even in very large genomes. Thus, increasing \(k\) from 11 to 31 provides an enormous increase in sequence specificity.

      +

      The estimate above is intentionally simplistic. Real genomes do not consist of independent, equiprobable nucleotides: nucleotide composition and sequence dependencies affect the probability of observing particular words. More accurate estimates of the probability of observing matches by chance therefore take sequence length and nucleotide composition into account.

      +

      Karlin and Ost provide an analytical framework for this problem based on the distribution of the longest exact match expected between random sequences. If \(L_{\max}\) denotes the length of the longest seed observed in random sequences of given size and nucleotide composition, the corresponding seed-level probability is

      +
      \[ +P_{\mathrm{seeds}}(L)=P(L_{\max}\geq L). +\]
      +

      Conversely, for a chosen false-positive probability \(P\), one can determine the minimum seed length \(L_{\min}\) such that matches of length \(L_{\min}\) or greater are not expected to occur by chance. This is the approach used by Repseek to determine statistically significant seed lengths.

      +

      The simple \(G/4^k\) estimate and the Karlin–Ost statistics therefore describe the same fundamental effect at different levels of approximation: the probability of a chance match decreases rapidly with k-mer length, while the amount of sequence being searched increases the opportunity for such matches.

      +

      Consequently, the appropriate value of \(k\) depends on the total amount of sequence being compared and its nucleotide composition. Choosing \(k\) appropriately allows the false-positive rate of homology inference to be controlled.

      +

      Considering that bacterial genomes are typically a few megabases long, with the smallesr plublished today Mycoplasma genitalium having a size \(G=580\,Mb\) (Fraser et al. 1995)2, and the largest \(13\,Mb\) in Sorangium cellulosum (Schneiker et al. 2007)3. For such genomes, the space of 11-mers is already to small to get a reasonable false positive rate:

      +
      \[ +\begin{aligned} +P(C\geq1) &= 1- e^{-\frac{580.000^2}{2 \cdot 4^{11}}} \\ +&\simeq 1 +\end{aligned} +\]
      +

      , whereas 21-mers already provide a space of several trillion possible sequences.

      +

      Plant genomes illustrate the opposite situation. Plant genome sizes span several orders of magnitude, from the smallest angiosperm genomes present in the carnivorous plans genus (Fleischmann et al. 2014)4 about 60 Mb to the largest one 160 Gb in Tmesipteris oblanceolata, a New Caledonian fork fern (Fernández et al. 2024)5, and large genomes often contain substantial amounts of repeated sequence. Even at \(k=31\), however, the theoretical k-mer space remains vastly larger than any individual genome. At these lengths, the limiting factor is therefore not exhaustion of the theoretical k-mer space but the biological repetitiveness and non-random composition of the genome.

      +

      For example, the number of possible k-mers grows as follows:

      + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      \(k\)\(4^k\)\(4^k/2\) after canonicalization
      11\(4.19\times10^6\)\(2.10\times10^6\)
      15\(1.07\times10^9\)\(5.37\times10^8\)
      21\(4.40\times10^{12}\)\(2.20\times10^{12}\)
      25\(1.13\times10^{15}\)\(5.63\times10^{14}\)
      31\(4.61\times10^{18}\)\(2.31\times10^{18}\)
      +

      Canonicalization approximately halves the number of possible representations because a k-mer and its reverse complement are represented by the same canonical form.

      +

      The distinction between the simple \(4^k\) model and the Karlin--Ost framework is important. The former describes the size of the combinatorial k-mer space; the latter describes the probability of observing long matches in sequences whose composition may depart from the uniform random model. Consequently, the value of \(k\) cannot be selected from genome size alone.

      +

      These scales have very different implications for different genome sizes.

      +

      The random-sequence calculation is only a null model. Real genomes exhibit strong and reproducible oligonucleotide composition biases. Karlin and colleagues showed that relative abundances of short oligonucleotides, in particular dinucleotides, form characteristic genomic signatures that remain relatively stable within genomes and differ between organisms (Karlin1995?; Karlin1997?). More generally, short k-mer frequencies can therefore capture compositional properties of genomes rather than provide highly specific sequence identifiers.

      +

      This distinction is important for whole-genome k-mer comparison. Increasing \(k\) increases the theoretical diversity of the k-mer space exponentially and generally increases the proportion of k-mers that occur at unique genomic positions. Conversely, shorter k-mers occur more frequently and are more strongly influenced by genomic composition and repeated sequence. Empirical studies of k-mer spectra show this transition clearly: for example, k-mers of length \(k\leq11\) can behave primarily as genomic-composition signatures, whereas longer k-mers progressively provide greater sequence specificity.

      +

      The supported range \(11\leq k\leq31\) should therefore be understood as an engineering compromise rather than as a universal biological optimum. Smaller values increase the probability that unrelated genomic regions share the same k-mers, while larger values increase specificity but also make the representation more sensitive to substitutions, indels, and incomplete homology. The appropriate value within this range consequently depends on the biological scale and comparison task.

      +

      Odd k-mer lengths

      +

      OBIKmer additionally requires \(k\) to be odd. This constraint is independent of the resolution argument above.

      +

      For an odd-length DNA sequence, a k-mer cannot be identical to its own reverse complement. Consequently, every k-mer has two distinct orientations, which can be unambiguously collapsed into a canonical representation. For even \(k\), self-reverse-complementary sequences exist, so a k-mer can coincide with its own reverse complement.

      +

      Odd \(k\) is therefore not required for canonicalization itself: the canonical form

      +
      \[ +s^c=\min_{\mathrm{lex}}\left(s,\operatorname{RC}(s)\right) +\]
      +

      remains well defined for even \(k\). The odd-\(k\) restriction instead guarantees that the two orientations are always distinct, which simplifies the representation and orientation handling used by OBIKmer.

      +

      Karlin--Ost (1988) formula

      +

      (Karlin & Ost 1988)6 considers \(s\) independent stationary letter sequences over a finite alphabet and studies \(K_{r,s}(N)\), the length of the longest word common to \(r\) of the \(s\) sequences, each of length \(N\). For i.i.d. equiprobable letters over an alphabet of size \(m\) (here \(m=4\) for DNA), the characteristic local match parameter reduces to

      +
      \[ +\lambda^{[r]} = m^{1-r}, +\]
      +

      so for \(r=2\) (a word shared by two occurrences), \(\lambda = m^{-1} = 1/4\).

      +

      The expected number of common words of length \(k\) among the \(\binom{s}{r}\) relevant groups of sequences is

      +
      \[ +S_1(N,k) = N^{r}\binom{s}{r}\gamma\,\lambda^{k}\,(1+o(1)), +\]
      +

      with \(\gamma=1\) exactly in the i.i.d. equiprobable case. Theorem 2.2 gives the limiting distribution of \(K_{r,s}(N)\), with \(k\) set on the scale \(k=\left[\dfrac{\log N^{r}}{-\log\lambda^{*}}+x+1\right]\):

      +
      \[ +\lim_{N\to\infty}\left[\Pr\{K_{r,s}(N)<k\} - \exp\{-(1-\lambda^{*})\,S_1(N,k)\}\right] = 0, +\]
      +

      equivalently

      +
      \[ +\frac{-\log\Pr\{K_{r,s}(N)<k\}}{(\lambda^{*})^{\rho(N,x)}} \xrightarrow[N\to\infty]{} (1-\lambda^{*})\,\gamma^{*}(\lambda^{*})^{x}. +\]
      +

      The factor \((1-\lambda^{*})\) multiplying \(S_1(N,k)\) is the key correction over the naive pairwise-collision count: it de-clumps overlapping candidate matches, which are not independent events, into an effective Poisson rate. This theorem is stated for \(s\geq2\) independent sequences (e.g. comparing two distinct genomes); the intra-sequence self-repeat case (a single genome compared to itself) is treated separately, via the reverse-complement match length \(R(N)\), in (Karlin & Ost 1985)7.

      +

      \(o(1)\) (« petit o de 1 ») désigne une quantité qui tend vers 0 quand \(N\to\infty\), mais dont on ne précise pas la vitesse exacte ni la formule exacte. C'est une façon d'écrire « il reste un terme d'erreur, mais il devient négligeable ».

      +

      Concrètement dans \(S_1(N,k) = N^{r}\binom{s}{r}\gamma\,\lambda^{k}\,(1+o(1))\) :

      +
        +
      • \(N^{r}\binom{s}{r}\gamma\,\lambda^{k}\) est la valeur exacte à la limite, celle qu'on utilise en pratique (c'est elle qui redonne \(E[C]\) de notre discussion).
      • +
      • \((1+o(1))\) dit : « la vraie valeur de \(S_1(N,k)\) n'est pas exactement égale à ce produit pour \(N\) fini, mais le rapport entre les deux tend vers 1 quand \(N\to\infty\) ». Autrement dit \(S_1(N,k) / \left[N^{r}\binom{s}{r}\gamma\lambda^{k}\right] \to 1\).
      • +
      +

      Pourquoi ce terme existe : la formule \(F_\sigma(k)/\lambda^k \to \gamma\) (éq. 2.8 du papier) est elle-même une limite quand \(k\to\infty\) — pour \(k\) fini, il y a un petit écart entre \(F_\sigma(k)\) et \(\gamma\lambda^k\). C'est cet écart-là qui devient le \(o(1)\).

      +

      Ce que ça implique pour l'usage pratique : dans le cas iid équiprobable que tu utilises (ADN, \(\gamma=1\) exactement, pas seulement à la limite — je l'ai vérifié directement sur la dérivation : \(F_\sigma(k)=m^{-k}=\lambda^k\) pour tout \(k\), pas juste asymptotiquement), le \(o(1)\) est en réalité nul dans ce cas précis, pas juste négligeable. La formule \(S_1(N,k)=N^r\binom{s}{r}\lambda^k\) est donc exacte (à l'asymptotique en \(N\) près, qui elle vient d'ailleurs — de l'approximation de Poisson elle-même, pas de ce \(\gamma\)).

      +

      Les deux approches sont légitimes, mais elles répondent à des questions différentes — et il y a une relation algébrique exacte entre les deux qui vaut la peine d'être explicitée.

      +

      Ce que calcule l'approche naïve. Pour un k-mer \(w\) déjà observé et fixé, \(p_i\simeq1-\exp(-G_i/4^k)\) est la probabilité que ce mot précis apparaisse quelque part dans le génome \(i\) par hasard, et \(P(R\geq r)\) (Poisson-binomiale ou binomiale) est la probabilité que ce mot précis soit partagé par au moins \(r\) des \(S\) génomes par hasard. C'est une p-value par observation : « j'ai vu ce k-mer particulier dans 5 génomes sur 10, est-ce surprenant pour lui ? »

      +

      Ce que calcule Karlin-Ost. \(S_1(k)\) et \(P(K_{r,s}\geq k)\) répondent à : « parmi tous les \(4^k\) mots possibles, quelle est la probabilité qu'au moins un seul d'entre eux atteigne ce seuil de partage par hasard, n'importe où dans les génomes ? » C'est une question globale sur tout l'espace des k-mers, pas sur un mot déjà identifié.

      +

      La relation exacte entre les deux. Reprenons le cas \(r=s=2\), génomes égaux de taille \(G\) :

      +
        +
      • Naïf (mot \(w\) fixé) : \(P(R=2\mid w)\simeq p^2=(G\cdot4^{-k})^2=G^2\cdot4^{-2k}\)
      • +
      • Karlin-Ost : \(S_1=G^2\cdot4^{-k}\)
      • +
      +

      Le rapport est exactement \(S_1 / P(R=2\mid w) = 4^{k}\) — c'est-à-dire le nombre de mots possibles. Ce n'est pas une coïncidence : \(S_1=N^r\binom{s}{r}\lambda^k\) vient de \(\lambda^k=\sum_w \Pr\{w\}^r\), une somme explicite sur les \(4^k\) mots, alors que le calcul naïf tel que présenté ne somme sur rien — il traite \(w\) comme donné.

      +

      Donc l'approche naïve, formellement, c'est Karlin-Ost auquel il manque le facteur « nombre de k-mers testés ». Pour la rendre comparable à la question « dois-je m'attendre à une collision fortuite n'importe où dans mon jeu de données », il faudrait une correction à la Bonferroni : multiplier \(P(R\geq r\mid w)\) par le nombre effectif de k-mers canoniques testables (\(\approx4^k/2\)). Karlin-Ost fait cette correction analytiquement et correctement (avec en prime le facteur de déclumping \((1-\lambda)\) qu'on a discuté, qui corrige le sur-comptage des matches chevauchants — une union bound brute comme le Bonferroni décrit ci-dessus le surestimerait légèrement).

      +

      Ce qui veut dire, en pratique, que les deux outils ne sont pas concurrents mais complémentaires : +- Choix a priori de \(k\) / FPR global du dataset (la question de kmers.md depuis le début de cette conversation) → Karlin-Ost, parce qu'il intègre nativement la multiplicité sur tous les mots possibles. +- Score de significativité a posteriori pour un k-mer effectivement observé dans l'index (« ce hit précis, présent dans \(r\) génomes sur \(S\), est-il compatible avec le hasard ? ») → l'approche Poisson-binomiale est exactement le bon outil, et même préférable à Karlin-Ost pour cet usage : elle gère nativement des \(G_i\) inégaux sans les subtilités de convergence du théorème 2.2, et elle donne directement une p-value par hit exploitable pour filtrer les résultats d'un index présence/absence — ce qui correspond bien à l'usage qu'a en tête ton correspondant.

      +

      Une seule réserve sur le texte proposé tel quel : \(p_i\simeq1-\exp(-G_i/4^k)\) souffre du même biais que celui identifié plus tôt dans notre discussion — un mot de longueur \(k\) n'est pas indépendant à chaque position (chevauchement), donc ce \(p_i\) est un peu optimiste (légèrement trop haut) pour les mêmes raisons que \(E[C]\) l'était. Négligeable pour \(k\geq21\), plus discutable à \(k=11\).

      +

      Chance sharing across genomes and the FDR of homology

      +

      The formulas above ask how likely a match is between two sequences. The question OBIKmer actually has to answer is different: a k-mer is found in \(r\) of the \(S\) indexed genomes, and one wants to know whether this sharing can be explained by chance or whether it is evidence of homology. This section builds a null model for that question and turns it into a false discovery rate (FDR) for the call "a k-mer shared by at least \(r\) genomes is homologous".

      +

      The null model

      +

      The null hypothesis is that the \(S\) genomes are unrelated, that is, that each one is an independent random sequence. Rejecting it for a k-mer means that chance alone cannot explain why the k-mer is shared, which is the signal of homology one is looking for.

      +

      The model only records whether a k-mer is present in a genome, not how many times. Two consequences follow. First, repeats inside a genome are irrelevant, and the size \(G_i\) of genome \(i\) is its number of distinct canonical k-mers. Second, a repeat family shared between genomes is paralogy, which is a form of homology; it is a true positive and not a false one. The only k-mers that could be shared by composition rather than by descent are low-complexity ones, which OBIKmer removes beforehand with its entropy filter.

      +

      Everything takes place in canonical space. Because \(k\) is odd, no k-mer equals its own reverse complement, so the \(4^k\) k-mers pair up perfectly and the space contains exactly

      +
      \[ +N=\frac{4^k}{2} +\]
      +

      canonical k-mers. A given canonical k-mer is present in genome \(i\) by chance with probability

      +
      \[ +p_i = 1-\left(1-\frac1N\right)^{G_i}\simeq \frac{G_i}{N}=\frac{2\,G_i}{4^k}. +\]
      +

      This is twice the \(G/4^k\) used earlier in this page, because there \(G\) is a number of positions on one strand, whereas \(G_i\) here counts canonical k-mers, for which both strands are already merged.

      +

      Let \(R\) be the number of genomes, out of \(S\), that contain a given canonical k-mer by chance. Genomes are independent under the null, so \(R\) is a sum of independent presence indicators, each with its own probability \(p_i\).

      +

      Exact tail probability

      +

      When genome sizes differ, the \(p_i\) differ, and \(R\) follows a Poisson-binomial distribution. Writing the probability of exactly \(j\) presences as a sum over all subsets of \(j\) genomes would be hopeless, since there are up to \(2^S\) subsets. The distribution is instead obtained by adding the genomes one at a time. Let \(d_j\) be the probability that exactly \(j\) of the genomes processed so far contain the k-mer. Adding genome \(i\) leaves \(j\) unchanged if the genome does not contain the k-mer, and moves \(j-1\) to \(j\) if it does:

      +
      \[ +d'_j = d_j\,(1-p_i) + d_{j-1}\,p_i, \qquad d_0=1 \text{ before the first genome.} +\]
      +

      The table \(d\) only needs to remember how many genomes were hit, not which ones, and this brings the cost down to \(O(S^2)\). The result does not depend on the order in which the genomes are processed. It does not depend on the k-mer either, so it is computed once per index. The upper tail is then

      +
      \[ +P_0(R\geq r) = 1-\sum_{j=0}^{r-1} d_j . +\]
      +

      Only \(d_0,\dots,d_{r-1}\) are needed, so the cost is \(O(S\,r)\).

      +

      Approximation with equal genome sizes

      +

      If all genomes are given the same size, taken as the mean

      +
      \[ +\bar G=\frac1S\sum_{i=1}^{S}G_i, \qquad \bar p=\frac{2\bar G}{4^k}, +\]
      +

      then \(R\) follows a plain binomial distribution and the tail is

      +
      \[ +P_0(R\geq r)\simeq\sum_{j=r}^{S}\binom{S}{j}\,\bar p^{\,j}(1-\bar p)^{S-j}. +\]
      +

      This sum is the regularized incomplete beta function \(I_{\bar p}(r,\,S-r+1)\), available in standard statistical libraries. If it is summed by hand, the terms are obtained one from the next by \(t_{j+1}=t_j\,\frac{S-j}{j+1}\,\frac{\bar p}{1-\bar p}\).

      +

      A theorem of Hoeffding (Hoeffding 1956)1 shows that this approximation errs on the safe side. For a fixed mean, the Poisson-binomial distribution is more concentrated than the binomial one, so for \(r\geq S\bar p+1\) the binomial tail is never smaller than the exact tail. Using the mean genome size therefore overestimates the chance of sharing and can only make the FDR more pessimistic. Gleser (Gleser 1975)8 refines this result: replacing the \(p_i\) by any partial average of themselves, not only by their full mean, also moves the upper tail towards the binomial one, provided \(r-1\geq S\bar p+2\). Grouping genomes of similar size is therefore safe in the same sense. What it loses is power when genome sizes differ by orders of magnitude, since one large genome then weighs as much as several small ones in \(\bar p\).

      +

      When all \(p_i\) are small, which is the case for large \(k\), the number of hits is also well approximated by a Poisson distribution of mean \(\lambda=\sum_i p_i\), with an error bounded by \(\sum_i p_i^2\) (Le Cam 1960)9.

      +

      For the numerical evaluation, the tail that does not contain the mode \(S\bar p\) should be summed directly. When \(r>S\bar p\), computing the sum from \(j=r\) upwards is accurate even for very small probabilities, whereas computing \(1\) minus the sum of the lower terms loses all precision once the tail falls below the machine epsilon.

      +

      False discovery rate

      +

      The FDR of the call "present in at least \(r\) genomes" is the fraction of retained k-mers expected to be pure chance:

      +
      \[ +\widehat{\mathrm{FDR}}(r)=\min\!\left(1,\ \frac{N\,P_0(R\geq r)}{n_{\geq r}}\right). +\]
      +

      The numerator is the number of k-mers expected to reach \(r\) genomes by chance if the genomes were unrelated. Each of the \(N\) canonical k-mers has probability \(P_0(R\geq r)\) of doing so, and no further correction for multiple testing is needed because the number of tests is already in the product. The denominator \(n_{\geq r}\) is the number of distinct k-mers actually observed in at least \(r\) genomes, which is read directly from the index. The observed count mixes chance and true homology, so the ratio is an upper bound on the true FDR: it implicitly assumes that all k-mers are null, and it never underestimates the noise. The minimal \(r\) for which \(\widehat{\mathrm{FDR}}(r)\) falls below the accepted level, for example 5 %, is the sharing threshold to use.

      +

      As an illustration, take \(k=21\) (so \(N=2.2\times10^{12}\)), \(S=10\) genomes and \(G_i=10^9\) distinct canonical k-mers each, hence \(p=4.5\times10^{-4}\). The number of k-mers expected to be shared by chance is:

      + + + + + + + + + + + + + + + + + + + + + + + + + +
      \(r\)\(N\,P_0(R\geq r)\)
      2\(2\times10^{7}\)
      3\(2.5\times10^{4}\)
      4\(\approx 20\)
      5\(\approx 0.01\)
      +

      Chance alone produces millions of pairwise shared 21-mers at this genome size, a few tens of thousands shared by three genomes, and almost none shared by five or more. Whether a given \(r\) is acceptable then depends on how many k-mers the index actually reports at that level.

      +

      Limits of the model

      +
        +
      • The FDR is computed per k-mer. Neighbouring k-mers of a shared region overlap and are not independent, so a single homologous region contributes many correlated k-mers. A per-region FDR would count regions instead of k-mers and is not covered here.
      • +
      • The probability \(p_i\) assumes equiprobable, independent nucleotides. The composition biases discussed earlier in this page violate this assumption, and the resulting error is not accounted for.
      • +
      +
      +

      Je n'ai rien modifié. Voici la formalisation des deux points, à valider avant de l'écrire dans kmers.md.

      +

      1. FDR d'un ensemble de k-mers sélectionnés

      +

      Estimation de l'espérance du nombre de faux positifs. Soit \(M\) le nombre de k-mers canoniques testés, \(M = 4^k/2\) pour k impair. Soit \(q(r)=P(X\ge r)\) la queue du modèle nul. Le nombre attendu de k-mers présents par hasard dans au moins \(r\) génomes vaut

      +
      \[ +E[\mathrm{FP}(r)] = M\,q(r). +\]
      +

      Cette égalité est exacte par linéarité de l'espérance, même si les k-mers chevauchants sont dépendants. La dépendance n'affecte que la variance, pas l'espérance. Seule l'approximation de Poisson par génome reste une approximation.

      +

      FDR. Soit \(N(r)=\#\{i : r_i\ge r\}\) le nombre observé de k-mers couvrant au moins \(r\) génomes. Alors

      +
      \[ +\mathrm{FDR}(r)=\min\!\Big(1,\ \frac{M\,q(r)}{N(r)}\Big). +\]
      +

      Cette estimation est légèrement conservatrice. Elle suppose \(\pi_0=1\), c'est-à-dire que presque tout l'espace \(M\) est nul, ce qui est vrai pour k grand.

      +

      Ensemble hétérogène en \(r_i\). Si le jeu n'est pas défini par un seuil unique, on utilise le local fdr par classe de couverture :

      +
      \[ +\mathrm{fdr}(r)=\frac{M\,P(X=r)}{N_r},\qquad N_r=\#\{i : r_i=r\}. +\]
      +

      Le FDR de l'ensemble est alors la moyenne \(\frac1N\sum_i \mathrm{fdr}(r_i)\). Pour un ensemble défini par seuil \(r_{\min}\), cela redonne exactement la formule précédente.

      +

      Remarque. \(M\,p\approx G\), d'où \(E[\mathrm{FP}(r)]\approx\binom{S}{r}\,G^r/M^{r-1}\) en régime de petits \(p\). Cela donne une lecture directe de la quantité.

      +

      2. Markov d'ordre 0 avec fréquences inégales

      +

      Soit \(\pi_b\) la fréquence du nucléotide \(b\). Pour un k-mer \(w\) de composition \(n=(n_A,n_C,n_G,n_T)\) avec \(\sum n_b=k\), on a \(P(w)=\prod_b \pi_b^{n_b}\).

      +

      Attendu par génome. Le k-mer canonique \(K=\{w,\overline w\}\) est vu quand le brin direct porte \(w\) ou \(\overline w\), donc

      +
      \[ +\lambda_K = G\,\big(P(w)+P(\overline w)\big)=G\Big(\prod_b\pi_b^{n_b}+\pi_A^{n_T}\pi_T^{n_A}\pi_C^{n_G}\pi_G^{n_C}\Big). +\]
      +

      Cela ne dépend que de la composition. Si \(\pi_A=\pi_T\) et \(\pi_C=\pi_G\), on retrouve \(2P(w)\).

      +

      Faux positifs. Le test n'est plus homogène : chaque composition a sa propre \(p(n)=1-e^{-\lambda(n)}\). Il faut donc sommer sur les classes de composition (un k-mer et son complément inverse partagent le même \(\lambda\), d'où le facteur ½) :

      +
      \[ +E[\mathrm{FP}(r)]=\frac12\sum_{n}\binom{k}{n_A,n_C,n_G,n_T}\,P\big(X\ge r\mid p(n)\big). +\]
      +

      Cette somme porte sur \(\binom{k+3}{3}\) compositions, soit 5 984 pour k = 31. Elle est donc légère avec la binomiale de la moyenne, ou avec une Poisson-binomiale par composition si les \(G_i\) diffèrent. Si les \(\pi_b\) varient d'un génome à l'autre (GC différent), les \(p_i(n)\) se construisent par génome.

      +

      Effet attendu. La moyenne de \(\lambda_w\) sur tous les k-mers reste \(2G/4^k\), car \(\sum_w P(w)=1\). Dans le régime de petits \(p\), le terme dominant \(\binom{S}{r}p^r\) est convexe en \(\lambda\). Par Jensen, le modèle équiprobable est donc le minimum de \(E[\mathrm{FP}]\), et la déviation de composition ne peut qu'augmenter le FDR. Les k-mers riches en AT ou GC dominent la somme, ce qui rejoint ton remarque sur la faible complexité. C'est un résultat sur le terme dominant, pas une preuve pour la queue exacte.

      +

      Limites à signaler dans le texte. Markov 0 ignore les dinucléotides, comme les CpG ou les répétitions, et le \(M\) du modèle nul reste celui de l'espace total.

      +

      Veux-tu que j'écrive ces deux sections, avec les corrections de numérique évoquées plus haut (expm1, somme directe de la queue) ?

      +
      +
      +
        +
      1. +

        Hoeffding W (1956) [On the distribution of the number of successes in independent trials]{.nocase}. The Annals of Mathematical Statistics, 27, 713--721. ↩↩

        +
      2. +
      3. +

        Fraser CM, Gocayne JD, White O, Adams MD, Clayton RA, Fleischmann RD, Bult CJ, Kerlavage AR, Sutton G, Kelley JM, Fritchman RD, Weidman JF, Small KV, Sandusky M, Fuhrmann J, Nguyen D, Utterback TR, Saudek DM, Phillips CA, Merrick JM, Tomb JF, Dougherty BA, Bott KF, Hu PC, Lucier TS, Peterson SN, Smith HO, Hutchison CA 3rd, Venter JC (1995) The minimal gene complement of mycoplasma genitalium. Science (New York, N.Y.), 270, 397--403. ↩

        +
      4. +
      5. +

        Schneiker S, Perlova O, Kaiser O, Gerth K, Alici A, Altmeyer MO, Bartels D, Bekel T, Beyer S, Bode E, Bode HB, Bolten CJ, Choudhuri JV, Doss S, Elnakady YA, Frank B, Gaigalat L, Goesmann A, Groeger C, Gross F, Jelsbak L, Jelsbak L, Kalinowski J, Kegler C, Knauber T, Konietzny S, Kopp M, Krause L, Krug D, Linke B, Mahmud T, Martinez-Arias R, McHardy AC, Merai M, Meyer F, Mormann S, Muñoz-Dorado J, Perez J, Pradella S, Rachid S, Raddatz G, Rosenau F, Rückert C, Sasse F, Scharfe M, Schuster SC, Suen G, Treuner-Lange A, Velicer GJ, Vorhölter F-J, Weissman KJ, Welch RD, Wenzel SC, Whitworth DE, Wilhelm S, Wittmann C, Blöcker H, Pühler A, Müller R (2007) Complete genome sequence of the myxobacterium sorangium cellulosum. Nature Biotechnology, 25, 1281--1289. ↩

        +
      6. +
      7. +

        Fleischmann A, Michael TP, Rivadavia F, Sousa A, Wang W, Temsch EM, Greilhuber J, Müller KF, Heubl G (2014) Evolution of genome size and chromosome number in the carnivorous plant genus genlisea (lentibulariaceae), with a new estimate of the minimum genome size in angiosperms. Annals of Botany, 114, 1651--1663. ↩

        +
      8. +
      9. +

        Fernández P, Amice R, Bruy D, Christenhusz MJM, Leitch IJ, Leitch AL, Pokorny L, Hidalgo O, Pellicer J (2024) A 160 gbp fork fern genome shatters size record for eukaryotes. iScience, 27, 109889. ↩

        +
      10. +
      11. +

        Karlin S, Ost F (1988) Maximal length of common words among random letter sequences. The Annals of Probability, 16, 535--563. ↩

        +
      12. +
      13. +

        Karlin S, Ost F (1985) Maximal segmental match length among random sequences from a finite alphabet. In: Proceedings of the berkeley conference in honor of jerzy neyman and jack kiefer (eds Le Cam LM, Olshen RA), pp. 225--243. Wadsworth, Monterey, CA. ↩

        +
      14. +
      15. +

        Gleser LJ (1975) [On the distribution of the number of successes in independent trials]{.nocase}. Annals of Probability, 3, 182--188. ↩

        +
      16. +
      17. +

        Le Cam L (1960) [An approximation theorem for the Poisson binomial distribution]{.nocase}. Pacific Journal of Mathematics, 10, 1181--1197. ↩

        +
      18. +
      +
      + + + + + + + + + + + + + +
      +
      + + + +
      + +
      + + + +
      +
      +
      +
      + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/doc/theory/kmer_indexing/minimizer_selection/index.html b/doc/theory/kmer_indexing/minimizer_selection/index.html new file mode 100644 index 00000000..698ceb26 --- /dev/null +++ b/doc/theory/kmer_indexing/minimizer_selection/index.html @@ -0,0 +1,1756 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + Minimizer selection - obikmer — User Guide + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      + +
      + + + + + + +
      + + +
      + +
      + + + + + + +
      +
      + + + +
      +
      +
      + + + + + +
      +
      +
      + + + +
      +
      +
      + + + +
      +
      +
      + + + +
      + +
      + + + + + +

      Minimizer selection

      +

      Definition

      +

      The minimizer of a k-mer is the canonical form of the m-mer that is smallest according to a chosen ordering among the \(k-m+1\) overlapping m-mers contained in the k-mer, with \(m<k\) (Roberts et al. 2004)1. Canonical form is the same operation defined for a full kmer (see DNA encoding), applied here to an m-mer \(s\) instead: \(s^c = \min_{\mathrm{lex}}\left(s,\operatorname{RC}(s)\right)\).

      +

      In OBIkmer, canonical m-mers are ordered according to a deterministic hash function \(H\) applied to their canonical form:

      +
      \[ +s_1^c <_h s_2^c +\quad\Longleftrightarrow\quad +H(s_1^c) < H(s_2^c). +\]
      +

      Thus, for a k-mer \(K\), let

      +
      \[ +M(K)=(s_1,\ldots,s_{k-m+1}) +\]
      +

      be its sequence of overlapping m-mers. The minimizer is

      +
      \[ +\operatorname{minimizer}(K) = s_j^c, +\qquad +j=\underset{i}{\operatorname{argmin}}\;H(s_i^c). +\]
      +

      In other words, the hash function defines the ordering of canonical m-mers, while the minimizer itself is the canonical form of the m-mer selected by that ordering.

      +

      Applying the minimizer to consecutive k-mers partitions a sequence into super-kmers (see Super-kmers).

      +

      Hash-based minimizer ordering

      +

      obikmer selects minimizers by hash order rather than plain lexicographic order. Ordering m-mers lexicographically on their 2-bit encoding systematically favors AT-rich m-mers (an all-A m-mer always encodes to 0), which causes low-complexity regions to dominate as minimizers and produces unbalanced partitions (Zheng et al. 2020, 2021; Kille et al. 2023; Pan & Reinert 2024; Golan & Shur 2025)2 3 4 5 6.

      +

      Instead, a well-distributed hash function \(H\) is applied to the canonical (lexicographically minimal) form of each m-mer, and the m-mer with the smallest \(H\) value wins. Because \(H\) is a bijective mixing function with good avalanche properties (Steele et al. 2014)7, its output behaves approximately as a composition-independent pseudorandom ordering of distinct m-mers. Under the usual random-hash assumption, each distinct m-mer in a window has approximately the same probability of being the minimum, independently of nucleotide composition.

      +

      The canonical form used as input to \(H\) is still the lexicographic minimum of forward/reverse-complement — hashing is applied on top of it, not used to redefine it. Defining canonicity by hash value instead would bias the distribution of hash values themselves toward small values (the minimum of two independent hashes is not uniformly distributed), reintroducing a bias one layer down.

      +

      Hash function

      +

      The hash function \(H\) applies Stafford's 64-bit mixing function, variant 13 (Stafford 2011)8, to the m-mer encoding XORed with a seed \(\sigma\):

      +
      H(x):
      +    x ← x  ⊕  σ
      +    x ← x  ⊕  (x >> 30)
      +    x ← x  ×  0xbf58476d1ce4e5b9
      +    x ← x  ⊕  (x >> 27)
      +    x ← x  ×  0x94d049bb133111eb
      +    return x ⊕ (x >> 31)
      +
      +

      The hash uses a fixed non-zero seed

      +
      \[ \sigma=\texttt{0x9e3779b97f4a7c15}. \]
      +

      The choice of \(\sigma\) is not arbitrary. Low-complexity m-mers (homopolymers and short tandem repeats) are disproportionately abundant in real genomes. If one of them happened to be the global argmin of \(H\), it could therefore occur as the minimizer in far more windows than expected from the composition-uniform behavior described above. This is not a bias of the mixing function itself, but a consequence of the repeated occurrence of the same input in real sequence data.

      +

      In particular, with \(\sigma=0\), the all-A m-mer is encoded as \(0\), and

      +
      \[ \operatorname{Mix13}(0)=0, \]
      +

      This makes it the global argmin of the hash function. The non-zero seed eliminates the particular pathological correspondence between the all-A m-mer and the zero output of the mixer.

      +

      Exhaustive enumeration of all \(4^m\) m-mers confirms that, with \(\sigma=\texttt{0x9e3779b97f4a7c15}\), the global argmin is neither a homopolymer nor a periodic repeat for any tested \(m\):

      + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      \(m\)argmin (canonical)decoded sequenceminimal period
      316CAA3
      578ACATG5
      75512CCCGAGA7
      9108760CGGGATCGA9
      11179014AAGGTGTCACG11
      1333759044GAAATACTTCACA13
      1529869313AACTACTTACCAAAC15
      +

      When the minimal period of the sequence is \(m\), as observed in the table above, the sequence is primitive: it cannot be represented as repetitions of a shorter sequence.

      +

      See Partitioning and indexing architecture for how this hash routes a super-kmer to a partition.

      +
      +
      +
        +
      1. +

        Roberts M, Hayes W, Hunt BR, Mount SM, Yorke JA (2004) [Reducing storage requirements for biological sequence comparison]{.nocase}. Bioinformatics (Oxford, England), 20, 3363--3369. ↩

        +
      2. +
      3. +

        Zheng H, Kingsford C, Marçais G (2020) Improved design and analysis of practical minimizers. Bioinformatics (Oxford, England), 36, i119--i127. ↩

        +
      4. +
      5. +

        Zheng H, Kingsford C, Marçais G (2021) Sequence-specific minimizers via polar sets. Bioinformatics (Oxford, England), 37, i187--i195. ↩

        +
      6. +
      7. +

        Pan C, Reinert K (2024) A simple refined DNA minimizer operator enables 2-fold faster computation. Bioinformatics (Oxford, England), 40. ↩

        +
      8. +
      9. +

        Kille B, Garrison E, Treangen TJ, Phillippy AM (2023) Minmers are a generalization of minimizers that enable unbiased local jaccard estimation. Bioinformatics (Oxford, England), 39. ↩

        +
      10. +
      11. +

        Golan S, Shur AM (2025) Expected density of random minimizers. In: Lecture notes in computer science Lecture notes in computer science., pp. 347--360. Springer Nature Switzerland, Cham. ↩

        +
      12. +
      13. +

        Steele GL Jr, Lea D, Flood CH (2014) [Fast splittable pseudorandom number generators]{.nocase}. In: [Proceedings of the 2014 ACM International Conference on Object Oriented Programming Systems Languages\ & Applications]{.nocase}. ACM, New York, NY, USA. ↩

        +
      14. +
      15. +

        Stafford D (2011) Better bit mixing - improving on MurmurHash3's 64-bit finalizer. Available at: https://zimbry.blogspot.com/2011/09/better-bit-mixing-improving-on.html (accessed 12 September 2026). ↩

        +
      16. +
      +
      + + + + + + + + + + + + + +
      +
      + + + +
      + +
      + + + +
      +
      +
      +
      + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/doc/theory/kmers_and_superkmers/index.html b/doc/theory/kmer_indexing/superkmers/index.html similarity index 62% rename from doc/theory/kmers_and_superkmers/index.html rename to doc/theory/kmer_indexing/superkmers/index.html index 3f9aa0a6..c140e564 100644 --- a/doc/theory/kmers_and_superkmers/index.html +++ b/doc/theory/kmer_indexing/superkmers/index.html @@ -9,25 +9,25 @@ - + - + - + - Kmers and super-kmers - obikmer — User Guide + Super-kmers - obikmer — User Guide - + @@ -46,7 +46,7 @@ - + @@ -64,7 +64,7 @@
      - + Skip to content @@ -80,7 +80,7 @@
      @@ -539,7 +901,7 @@
    50. - + @@ -567,7 +929,7 @@
    51. - + @@ -595,7 +957,7 @@
    52. - + @@ -623,7 +985,7 @@
    53. - + @@ -651,7 +1013,7 @@
    54. - + @@ -679,7 +1041,7 @@
    55. - + @@ -707,7 +1069,7 @@
    56. - + @@ -735,7 +1097,7 @@
    57. - + @@ -763,7 +1125,7 @@
    58. - + @@ -791,7 +1153,7 @@
    59. - + @@ -819,7 +1181,7 @@
    60. - + @@ -847,7 +1209,7 @@
    61. - + @@ -875,7 +1237,7 @@
    62. - + @@ -903,7 +1265,7 @@
    63. - + @@ -931,7 +1293,7 @@
    64. - + @@ -1013,7 +1375,7 @@
    65. - + @@ -1034,6 +1396,34 @@ + + + + + + +
    66. + + + + + + + + VCF pseudo-alignment + + + + + + + + +
    67. + + + + @@ -1048,7 +1438,7 @@
    68. - + @@ -1095,29 +1485,6 @@ @@ -1150,29 +1512,22 @@ -

      Kmers and super-kmers

      -

      Kmers

      -

      A kmer is a DNA subsequence of fixed length \(k\). Two constraints apply to \(k\), both enforced when a command starts (an invalid value exits immediately with an error):

      -
        -
      • \(k \in [11, 31]\): long enough to be specific, short enough to fit in a single 64-bit word at 2 bits/base (\(k \le 32\) is the hard limit; \(k < 11\) gives insufficient specificity).
      • -
      • \(k\) is odd: an odd-length sequence can never equal its own reverse complement, so the two orientations of any kmer are always distinct. This is required for the canonical form (see DNA encoding) to be well defined.
      • -
      -

      Super-kmers

      +

      Super-kmers

      A super-kmer is a maximal run of consecutive, overlapping kmers from a read that share the same canonical minimizer (see Minimizer selection). Each kmer in the run overlaps the next by \(k-1\) nucleotides. A super-kmer is capped at 256 nucleotides; a longer run is split at that boundary.

      -

      For a random minimizer of length \(m\) over kmers of length \(k\), the expected length of a super-kmer is approximately (Golan & Shur 2025; Zheng et al. 2020)1 2:

      +

      For a random minimizer of length \(m\) over kmers of length \(k\), the expected length of a super-kmer is approximately (Zheng et al. 2020; Golan & Shur 2025)1 2:

      \[L_{\text{nt}} \approx \frac{k-m+2}{2} + k - 1\]

      For \(k=31\), \(m=13\) this is about 40 nucleotides; in practice super-kmers rarely exceed a few dozen nucleotides.

      -

      Canonical super-kmers

      -

      A canonical super-kmer is the lexicographic minimum of a super-kmer and its reverse complement. When a read and its reverse complement are both encountered, they produce super-kmers that are reverse complements of each other; both reduce to the same canonical super-kmer, so a genomic region is represented once regardless of which strand was read.

      +

      Canonical super-kmers

      +

      A canonical super-kmer is the canonical form of a super-kmer (see DNA encoding): the lexicographic minimum of the super-kmer and its reverse complement. When a read and its reverse complement are both encountered, they produce super-kmers that are reverse complements of each other; both reduce to the same canonical super-kmer, so a genomic region is represented once regardless of which strand was read.

      Super-kmers are the unit of work used throughout construction and querying: sequences are decomposed into super-kmers first, and every downstream step (partition routing, deduplication, counting) operates on them rather than on individual kmers.


      1. -

        Zheng, H., Kingsford, C. & Marçais, G. (2020). Improved design and analysis of practical minimizers. Bioinformatics (Oxford, England), 36, i119--i127. ↩

        +

        Zheng H, Kingsford C, Marçais G (2020) Improved design and analysis of practical minimizers. Bioinformatics (Oxford, England), 36, i119--i127. ↩

      2. -

        Golan, S. & Shur, A.M. (2025). Expected density of random minimizers. In: Lecture notes in computer science, Lecture notes in computer science. Springer Nature Switzerland, Cham, pp. 347--360. ↩

        +

        Golan S, Shur AM (2025) Expected density of random minimizers. In: Lecture notes in computer science Lecture notes in computer science., pp. 347--360. Springer Nature Switzerland, Cham. ↩

      @@ -1225,10 +1580,10 @@ - + - + diff --git a/doc/theory/phylogeny/kmer_based/kmer_set_distances/index.html b/doc/theory/phylogeny/kmer_based/kmer_set_distances/index.html new file mode 100644 index 00000000..ad173635 --- /dev/null +++ b/doc/theory/phylogeny/kmer_based/kmer_set_distances/index.html @@ -0,0 +1,1715 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + Whole k-mer set distances - obikmer — User Guide + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      + +
      + + + + + + +
      + + +
      + +
      + + + + + + +
      +
      + + + +
      +
      +
      + + + + + +
      +
      +
      + + + +
      +
      +
      + + + +
      +
      +
      + + + +
      + +
      + + + + + +

      Whole k-mer set distances

      +

      The k-mer composition of a genome provides a representation of its sequence content that can be compared directly between genomes. The distances considered here quantify dissimilarity between genomes from their complete k-mer composition, independently of any particular representation, storage scheme, or indexing strategy.

      +

      Two complementary classes of comparison arise according to the information retained about k-mer occurrences (Anderson et al. 2011)1. Presence/absence-based distances consider only whether each distinct k-mer occurs in a genome, whereas abundance-based distances also incorporate its observed frequency. Abundance-based measures may be computed either from raw k-mer counts or from the corresponding relative frequencies.

      +

      Let \(A\) and \(B\) denote the k-mer sets of two genomes. For each distinct k-mer \(i\), let \(c_i^A\) and \(c_i^B\) denote its counts in the two genomes, and let

      +
      \[ +p_i^A=\frac{c_i^A}{\sum_j c_j^A}, +\qquad +p_i^B=\frac{c_i^B}{\sum_j c_j^B} +\]
      +

      denote the corresponding relative frequencies.

      +

      The resulting measures can be classified along two independent dimensions: the information retained about k-mer occurrences (presence/absence versus abundance) and the metric properties of the resulting dissimilarity, in particular whether the triangle inequality is satisfied.

      +

      Presence/absence-based distances

      +

      When only occurrence is retained, each genome is represented by its incidence vector over the k-mer space. The comparison therefore depends exclusively on the presence or absence of each distinct k-mer and is insensitive to its multiplicity.

      +

      The Jaccard distance compares two k-mer sets from the cardinalities of their intersection and union (Jaccard 1901)2:

      +
      \[ +D = 1 - \frac{\lvert A \cap B \rvert}{\lvert A \cup B \rvert}. +\]
      +

      Jaccard distance is a metric and therefore satisfies the triangle inequality (Kosub 2019)3.

      +

      The Hamming distance is the \(L^1\) distance between the corresponding binary incidence vectors:

      +
      \[ +D = \sum_i \mathbb{1}[a_i \ne b_i], +\]
      +

      where \(a_i,b_i\in\{0,1\}\) indicate the presence of k-mer \(i\) in the two genomes. It counts the number of k-mers for which the two incidence vectors differ, without normalization by the size of the k-mer space. As an \(L^1\) distance, Hamming distance is a metric. Unlike abundance-based measures, it cannot be recovered from k-mer counts alone once the presence/absence representation has been discarded. On an index that stores counts, a k-mer is taken as present when its count reaches the presence threshold (--presence-threshold, 1 by default).

      +

      Mash is not an independent set dissimilarity but a nonlinear transformation of the Jaccard similarity (Ondov et al. 2016)4. If

      +
      \[ +J = 1-D_{\mathrm{Jaccard}}, +\]
      +

      the Mash distance is defined as

      +
      \[ +D = -\frac{1}{k}\ln\left(\frac{2J}{1+J}\right) + = -\frac{1}{k}\ln\left(\frac{2(1-J_{\mathrm{Jaccard}})} + {2-J_{\mathrm{Jaccard}}}\right), +\]
      +

      with the result clamped to \(1.0\) when the Jaccard similarity reaches zero. This transformation is monotone increasing with Jaccard distance and therefore preserves the ordering of pairwise dissimilarities. Monotonicity alone, however, does not guarantee preservation of the triangle inequality. Mash should consequently be regarded as an evolutionary distance estimator derived from k-mer similarity rather than assumed to be a metric.

      +

      Abundance-based distances

      +

      When k-mer abundances are retained, genomes are represented by non-negative vectors of k-mer counts or frequencies. Distances can then distinguish genomes that contain the same k-mers but differ in their relative abundances.

      +

      The Euclidean distance is the ordinary \(L^2\) distance between the corresponding vectors. On raw counts it is

      +
      \[ +D = \sqrt{\sum_i (c_i^A-c_i^B)^2}, +\]
      +

      and the same definition applied to relative frequencies gives relfreq-euclidean. In both cases, Euclidean distance is a metric.

      +

      The Hellinger distance applies the Euclidean distance to the square-root-transformed relative frequencies (Legendre & Gallagher 2001)5:

      +
      \[ +D = +\frac{1}{\sqrt{2}}\sqrt{\sum_i\left(\sqrt{p_i^A}-\sqrt{p_i^B}\right)^2}. +\]
      +

      It is bounded between 0 and 1 and is a metric because it is a positive rescaling of the Euclidean distance between the transformed vectors \(\sqrt{p^A}\) and \(\sqrt{p^B}\). Its unnormalized form, hellinger-euclidean,

      +
      \[ +\begin{aligned} +D &=\sqrt{\sum_i\left(\sqrt{p_i^A}-\sqrt{p_i^B}\right)^2} \\ + &=\sqrt{2}\,D_{\mathrm{Hellinger}}, +\end{aligned} +\]
      +

      is likewise a metric, since multiplication by a positive constant preserves the triangle inequality.

      +

      The Bray–Curtis dissimilarity (Bray & Curtis 1957)6 is defined from the amount of abundance shared between two vectors:

      +
      \[ +D = 1-\frac{2\sum_i\min(c_i^A,c_i^B)}{\sum_i c_i^A+\sum_i c_i^B}. +\]
      +

      Using

      +
      \[ +a+b-2\min(a,b)=|a-b|, +\]
      +

      this can equivalently be written as

      +
      \[ +D = +\frac{\sum_i|c_i^A-c_i^B|} +{\sum_i c_i^A+\sum_i c_i^B}. +\]
      +

      For arbitrary count vectors, Bray–Curtis is not a metric (Gower & Legendre 1986)7: the pair-dependent normalization by the combined abundance can lead to violations of the triangle inequality. This distinguishes it from the \(L^1\)-based distances above, whose normalization factor is fixed independently of the pair being compared.

      +

      The metric property is recovered when all vectors under comparison have the same total abundance \(S\). In that case,

      +
      \[ +\sum_i c_i^A=\sum_i c_i^B=S +\]
      +

      for every pair, and Bray–Curtis reduces to

      +
      \[ +D = +\frac{1}{2S} +\sum_i|c_i^A-c_i^B|. +\]
      +

      It is therefore a positive rescaling of the \(L^1\) distance and is a metric on this fixed-total space.

      +

      Relative frequencies satisfy this condition by construction, since

      +
      \[ +\sum_i p_i^A=\sum_i p_i^B=1. +\]
      +

      Consequently, relfreq-bray-curtis reduces to

      +
      \[ +D = +\frac12\sum_i|p_i^A-p_i^B|, +\]
      +

      which is the total variation distance between the two frequency distributions. It is therefore a metric.

      +

      Thus, bray-curtis and relfreq-bray-curtis have the same algebraic form but operate on different domains: the former compares count vectors whose total abundance may vary between genomes, whereas the latter compares normalized frequency vectors with a fixed total. This difference in normalization is sufficient to change the metric properties of the measure.

      +

      See phylo for how to select a distance and the resulting output formats.

      +
      +
      +
        +
      1. +

        Anderson MJ, Crist TO, Chase JM, Vellend M, Inouye BD, Freestone AL, Sanders NJ, Cornell HV, Comita LS, Davies KF, Harrison SP, Kraft NJB, Stegen JC, Swenson NG (2011) [Navigating the multiple meanings of β diversity: a roadmap for the practicing ecologist]{.nocase}. Ecology Letters, 14, 19--28. ↩

        +
      2. +
      3. +

        Jaccard P (1901) [Étude comparative de la distribution florale dans une portion des Alpes et du Jura]{.nocase}. Bulletin de La Société Vaudoise Des Sciences Naturelles, 37, 547--579. ↩

        +
      4. +
      5. +

        Kosub S (2019) [A note on the triangle inequality for the Jaccard distance]{.nocase}. Pattern Recognition Letters, 120, 36--38. ↩

        +
      6. +
      7. +

        Ondov BD, Treangen TJ, Melsted P, Mallonee AB, Bergman NH, Koren S, Phillippy AM (2016) [Mash: fast genome and metagenome distance estimation using MinHash]{.nocase}. Genome Biology, 17, 132. ↩

        +
      8. +
      9. +

        Legendre P, Gallagher ED (2001) [Ecologically meaningful transformations for ordination of species data]{.nocase}. Oecologia, 129, 271--280. ↩

        +
      10. +
      11. +

        Bray JR, Curtis JT (1957) [An ordination of the upland forest communities of southern Wisconsin]{.nocase}. Ecological Monographs, 27, 325--349. ↩

        +
      12. +
      13. +

        Gower JC, Legendre P (1986) [Metric and Euclidean properties of dissimilarity coefficients]{.nocase}. Journal of Classification, 3, 5--48. ↩

        +
      14. +
      +
      + + + + + + + + + + + + + +
      +
      + + + +
      + +
      + + + +
      +
      +
      +
      + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/doc/theory/phylogeny/snp_based/central_snp_model/index.html b/doc/theory/phylogeny/snp_based/central_snp_model/index.html new file mode 100644 index 00000000..37c28b4a --- /dev/null +++ b/doc/theory/phylogeny/snp_based/central_snp_model/index.html @@ -0,0 +1,1659 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + Central-position SNP model - obikmer — User Guide + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      + +
      + + + + + + +
      + + +
      + +
      + + + + + + +
      +
      + + + +
      +
      +
      + + + + + +
      +
      +
      + + + +
      +
      +
      + + + +
      +
      +
      + + + +
      + +
      + + + + + +

      Central-position SNP model

      +

      Family

      +

      A family is the set of up to 4 kmers sharing identical flanking sequence and differing only at the central base — the odd kmer length guarantees a single, well-defined central position (see Kmers). A family is eligible for a genome pair \((i,j)\) only if both genomes carry exactly one of its observed forms (single-copy, unambiguous) — this excludes multi-copy and absent loci from the comparison rather than folding them into an undifferentiated "not identical" bucket the way a whole-index distance would.

      +

      Conditioning on eligibility this way restricts every comparison to loci that are directly, positively confirmed comparable in both genomes: a locus never enters the statistic because of a genuinely absent homologous region, a diverged paralogous copy, or a genome-size asymmetry — only because a real single-copy substitution (or lack of one) was observed at flanks confirmed intact in both genomes.

      +

      Substitution corrections (snp-*)

      +

      For a genome pair, let \(L\) be its total number of eligible loci, \(p\) the raw proportion of substitutions among those loci, \(P\)/\(Q\) the transition/transversion proportions, \(Q_1\)/\(Q_2\) Kimura's two transversion categories (A↔C & G↔T vs. A↔T & C↔G), \(P_1\)/\(P_2\) the purine (A↔G) / pyrimidine (C↔T) transition proportions, and \(\pi_A,\pi_C,\pi_G,\pi_T\) the pair's pooled base frequencies.

      +

      snp-raw

      +
      \[d = p\]
      +

      snp-jc

      +
      \[d = -\frac{3}{4}\ln\!\left(1-\frac{4p}{3}\right)\]
      +

      snp-k2p

      +
      \[ +\begin{aligned} +a_1 &= 1-2P-Q \\ +a_2 &= 1-2Q \\ +d &= -\frac{1}{2}\ln a_1-\frac{1}{4}\ln a_2 +\end{aligned} +\]
      +

      snp-k81

      +
      \[ +\begin{aligned} +a_1 &= 1-2P-2Q_1 \\ +a_2 &= 1-2P-2Q_2 \\ +a_3 &= 1-2Q_1-2Q_2 \\ +d &= -\frac{1}{4}\left(\ln a_1+\ln a_2+\ln a_3\right) +\end{aligned} +\]
      +

      snp-f81

      +
      \[ +\begin{aligned} +E &= 1-\left(\pi_A^2+\pi_C^2+\pi_G^2+\pi_T^2\right) \\ +d &= -E\ln\!\left(1-\frac{p}{E}\right) +\end{aligned} +\]
      +

      snp-t92

      +
      \[ +\begin{aligned} +g &= \pi_C+\pi_G \\ +w &= 2g(1-g) \\ +a_1 &= 1-\frac{P}{w}-Q \\ +a_2 &= 1-2Q \\ +d &= -w\ln a_1-\frac{1}{2}(1-w)\ln a_2 +\end{aligned} +\]
      +

      snp-tn93

      +
      \[ +\begin{aligned} +g_R &= \pi_A+\pi_G \\ +g_Y &= \pi_C+\pi_T \\ +k_1 &= \frac{2\pi_A\pi_G}{g_R} \\ +k_2 &= \frac{2\pi_C\pi_T}{g_Y} \\ +k_3 &= 2\left(g_Rg_Y-\frac{\pi_A\pi_G\,g_Y}{g_R}-\frac{\pi_C\pi_T\,g_R}{g_Y}\right) \\ +w_1 &= 1-\frac{P_1}{k_1}-\frac{Q}{2g_R} \\ +w_2 &= 1-\frac{P_2}{k_2}-\frac{Q}{2g_Y} \\ +w_3 &= 1-\frac{Q}{2g_Rg_Y} \\ +d &= -k_1\ln w_1-k_2\ln w_2-k_3\ln w_3 +\end{aligned} +\]
      +

      snp-tv — transversions only, deliberately uncorrected:

      +
      \[d = Q\]
      +

      A rate-heterogeneity correction applies to every formula above except snp-raw and snp-tv: each \(-\ln(x)\) term is replaced by \(\alpha\left(x^{-1/\alpha}-1\right)\) (same weight, same \(x\)) — see Gamma rate heterogeneity for the correction itself and how \(\alpha\) is estimated.

      +

      See phylo for how to select a snp-* value and the prerequisite index annex it requires.

      + + + + + + + + + + + + + +
      +
      + + + +
      + +
      + + + +
      +
      +
      +
      + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/doc/theory/phylogeny/snp_based/gamma_rate_heterogeneity/index.html b/doc/theory/phylogeny/snp_based/gamma_rate_heterogeneity/index.html new file mode 100644 index 00000000..8c38dd8d --- /dev/null +++ b/doc/theory/phylogeny/snp_based/gamma_rate_heterogeneity/index.html @@ -0,0 +1,1589 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + Gamma rate heterogeneity - obikmer — User Guide + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      + +
      + + + + + + +
      + + +
      + +
      + + + + + + +
      +
      + + + +
      +
      +
      + + + + + +
      +
      +
      + + + +
      +
      +
      + + + +
      +
      +
      + + + +
      + +
      + + + + + +

      Gamma rate heterogeneity

      +

      Real substitution rates vary across sites rather than being uniform, which a plain substitution correction (see Central-position SNP model) ignores. The standard \(+\Gamma\) correction models this by replacing each \(-\ln(x)\) term in a correction formula with \(\alpha\left(x^{-1/\alpha}-1\right)\) — the same \(x\), the same weight, under the assumption that the per-site rate is \(\mathrm{Gamma}(\alpha,\alpha)\)-distributed (mean 1) rather than fixed. A small \(\alpha\) indicates strong among-site rate heterogeneity (many near-invariant sites, a few fast ones); as \(\alpha\) grows large the correction converges to the uncorrected formula.

      +

      Automatic \(\alpha\) estimation

      +

      \(\alpha\) can be estimated automatically from the index itself, using a method-of-moments estimator computed once, from the same sampling pass that builds the pairwise substitution tally — no extra scan of the index.

      +

      The estimator pools substitution counts by partition rather than by genome pair: for partition \(i\), let \(n_i\) be the total number of substitutions observed across every genome pair, and \(L_i\) the total number of eligible loci across every genome pair, in that partition. Define the partition's observed substitution rate:

      +
      \[R_i = \frac{n_i}{L_i}\]
      +

      Under a single shared substitution rate with no among-site heterogeneity, each \(R_i\) would vary only by Poisson sampling noise. Rate heterogeneity is modeled, as in the correction itself, by a \(\mathrm{Gamma}(\alpha,\alpha)\)-distributed multiplicative rate (mean 1) shared by every locus in a partition — the classical Poisson–Gamma (negative-binomial) mixture. Under that model:

      +
      \[\mathbb{E}[R_i] = \mu \qquad \mathrm{Var}[R_i] = \frac{\mu}{L_i} + \frac{\mu^2}{\alpha}\]
      +

      where \(\mu\) is the pooled substitution rate across every partition. Weighting each partition's squared deviation by its own \(L_i\) removes the first (Poisson) term before attributing what's left to genuine rate heterogeneity:

      +
      \[\hat\mu = \frac{\sum_i n_i}{\sum_i L_i} \qquad V = \frac{\sum_i L_i\,(R_i-\hat\mu)^2}{\sum_i L_i} \qquad \bar L = \frac{\sum_i L_i}{\text{number of partitions}}\]
      +
      \[\hat\alpha = \frac{\hat\mu^2}{V - \hat\mu/\bar L}\]
      +

      If the measured variance \(V\) doesn't exceed the Poisson floor \(\hat\mu/\bar L\) (no detectable over-dispersion across partitions — the data are consistent with a single shared rate), \(\alpha\) is left undefined: the correction is silently disabled for that run rather than applying a fabricated value, and a warning is logged.

      +

      This is not Jin & Nei's (1990) original estimator. Their publication defines the \(+\Gamma\) distance formula itself and, absent an estimate, recommends the fixed default \(\alpha = 1\) — it does not propose a way to estimate \(\alpha\) from data. The method-of-moments estimator above is a standard consequence of the Poisson–Gamma relationship between substitution counts and gamma-distributed rate variation, applied per-partition; it is obikmer's own addition, not part of the cited correction.

      +

      See phylo for how to select a fixed \(\alpha\) or request automatic estimation.

      + + + + + + + + + + + + + +
      +
      + + + +
      + +
      + + + +
      +
      +
      +
      + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/doc/theory/phylogeny/snp_based/sampling_design/index.html b/doc/theory/phylogeny/snp_based/sampling_design/index.html new file mode 100644 index 00000000..0af96af3 --- /dev/null +++ b/doc/theory/phylogeny/snp_based/sampling_design/index.html @@ -0,0 +1,1696 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + Sampling design - obikmer — User Guide + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      + +
      + + + + + + +
      + + +
      + +
      + + + + + + +
      +
      + + + +
      +
      +
      + + + + + +
      +
      +
      + + + + + + + +
      + +
      + + + + + +

      Sampling design

      +

      Exhaustive computation over every non-monomorphic family in an index (see Central-position SNP model) is the default, but can be bounded to approximately N families instead. Sampling is designed around two properties: staying representative of the whole index, and letting the user favor families that actually carry signal.

      +

      Proportional sampling

      +

      Families are drawn in proportion to how many candidate families each part of the index actually holds, rather than uniformly across index partitions — this keeps the sample's composition representative of the whole index regardless of how unevenly candidate families happen to be distributed across partitions.

      +

      Per-family informativeness: Shannon entropy

      +

      A family's informativeness is measured directly rather than assumed uniform. For a family, entropy is computed over its observed states across genomes:

      + + + + + + + + + + + + + + + + + +
      QuantityDefinition
      entropy15Shannon entropy (bits) over the 16 possible states (the 15 non-empty subsets of {A,C,G,T}), genomes absent from the family excluded from the count
      entropy4the same over the 4 plain bases, restricted to the genomes carrying exactly one member of the family (those that stay informative when multi-member genomes are treated as missing data)
      +

      With --no-ambiguity the entropy is entropy4, computed over the genomes carrying exactly one member of the family, so that the weighting sees the same genomes as the final alignment; without it, entropy15 is used. Each is stored in its own annex.

      +

      A family where every genome carries the same single state has entropy 0 (uninformative); one where genomes are spread evenly across several states has higher entropy (more informative for distinguishing genomes).

      +

      Quartet restriction

      +

      A family only constrains the shape of a tree if its carrier genomes show a bipartition: two distinct states, each carried by at least two genomes. With fewer carriers, or with a single variant against a uniform background, every topology explains the family equally well. In a skimming dataset many families have only two or three carriers, so a uniform draw spends a large share of its quota on them. --quartet removes them before the draw. The test counts the states of the alignment: without --free-loss, absence is a state like any other and can form one of the two pairs; with --free-loss it is missing data and never does.

      +

      Pair balance

      +

      How well a pair of genomes is represented in a sample depends on how many families the pair shares: pairs of distant genomes share few, so a uniform draw gives them few sites, and the deep branches of a tree — which rest on exactly those pairs — are the least supported. --pair-balance weights each family by the mean, over the pairs of genomes carrying it, of 1 / (families shared by the pair), computed on the whole index. Averaging rather than summing keeps the weight from growing with the number of carriers, which would favour the families carried by most genomes regardless of which pairs they inform. Layers are allotted their share of the draw in proportion to the total weight of their families.

      +

      Entropy-biased sampling

      +

      By default, sampling draws families uniformly. Weighting instead by informativeness biases the draw toward a target entropy \(\mu\) with a Gaussian curve of width \(\sigma\): a family's probability of being kept scales with \(\exp\!\left(-\dfrac{(\text{entropy15} - \mu)^2}{2\sigma^2}\right)\) — a soft preference, not a hard cutoff, so no family is categorically excluded purely for having low or high entropy.

      +

      See phylo for how to bound and bias the sample.

      + + + + + + + + + + + + + +
      +
      + + + +
      + +
      + + + +
      +
      +
      +
      + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/doc/theory/minimizer_selection/index.html b/doc/theory/phylogeny/snp_based/sankoff_model/index.html similarity index 53% rename from doc/theory/minimizer_selection/index.html rename to doc/theory/phylogeny/snp_based/sankoff_model/index.html index 947cd9f8..9d378a07 100644 --- a/doc/theory/minimizer_selection/index.html +++ b/doc/theory/phylogeny/snp_based/sankoff_model/index.html @@ -9,25 +9,25 @@ - + - + - + - Minimizer selection - obikmer — User Guide + Sankoff model - obikmer — User Guide - + @@ -46,7 +46,7 @@ - + @@ -64,7 +64,7 @@
    69. + + + + + + + + + + + + + + + + + +
    70. + + + + + + + + + + +
    71. + + @@ -550,7 +903,7 @@
    72. - + @@ -578,7 +931,7 @@
    73. - + @@ -606,7 +959,7 @@
    74. - + @@ -634,7 +987,7 @@
    75. - + @@ -662,7 +1015,7 @@
    76. - + @@ -690,7 +1043,7 @@
    77. - + @@ -718,7 +1071,7 @@
    78. - + @@ -746,7 +1099,7 @@
    79. - + @@ -774,7 +1127,7 @@
    80. - + @@ -802,7 +1155,7 @@
    81. - + @@ -830,7 +1183,7 @@
    82. - + @@ -858,7 +1211,7 @@
    83. - + @@ -886,7 +1239,7 @@
    84. - + @@ -914,7 +1267,7 @@
    85. - + @@ -942,7 +1295,7 @@
    86. - + @@ -1024,7 +1377,7 @@
    87. - + @@ -1045,6 +1398,34 @@ + + + + + + +
    88. + + + + + + + + VCF pseudo-alignment + + + + + + + + +
    89. + + + + @@ -1059,7 +1440,7 @@
    90. - + @@ -1106,49 +1487,10 @@
      • - + - Definition - - - - -
      • - -
      • - - - - Hash-based ("random") minimizer - - - - - - -
      • - -
      • - - - - Partition routing is independent of minimizer selection + Combining into a 16×16 cost matrix @@ -1172,33 +1514,20 @@ -

        Minimizer selection

        -

        Definition

        -

        A minimizer of a kmer window is the m-mer (\(m < k\)) that is smallest, among all \(k - m + 1\) overlapping m-mers in the window, under a chosen ordering. The minimizer is always taken in canonical form (lexicographic minimum of forward and reverse complement) so that selection is strand-independent.

        -

        The minimizer partitions a sequence into super-kmers: maximal runs of overlapping kmers that share the same minimizer (see Kmers and super-kmers).

        -

        Hash-based ("random") minimizer

        -

        obikmer selects minimizers by hash order rather than plain lexicographic order. Ordering m-mers lexicographically on their 2-bit encoding systematically favors AT-rich m-mers (an all-A m-mer always encodes to 0), which causes low-complexity regions to dominate as minimizers and produces unbalanced partitions.

        -

        Instead, a well-distributed hash function \(H\) is applied to the canonical (lexicographically minimal) form of each m-mer, and the m-mer with the smallest \(H\) value wins. Because \(H\) is a bijection with good avalanche properties, every distinct m-mer in a window has an equal chance of holding the minimum hash value, independent of its nucleotide composition.

        -

        The canonical form used as input to \(H\) is still the lexicographic minimum of forward/reverse-complement — hashing is applied on top of it, not used to redefine it. Defining canonicity by hash value instead would bias the distribution of hash values themselves toward small values (the minimum of two independent hashes is not uniformly distributed), reintroducing a bias one layer down.

        -

        Hash function

        -

        The hash function is a 64-bit mixing function (splitmix64-style finalizer) applied to the m-mer XORed with a fixed non-zero seed:

        -
        \[H(x) = \text{mix64}(x \oplus s), \quad s = \lfloor 2^{64}/\varphi \rfloor = \texttt{0x9e3779b97f4a7c15}\]
        -
        H(x):
        -    x ← x  ⊕  0x9e3779b97f4a7c15
        -    x ← x  ⊕  (x >> 30)
        -    x ← x  ×  0xbf58476d1ce4e5b9
        -    x ← x  ⊕  (x >> 27)
        -    x ← x  ×  0x94d049bb133111eb
        -    return x ⊕ (x >> 31)
        -
        -

        The XOR seed avoids the finalizer's fixed point at 0 (\(\text{mix64}(0) = 0\)), which would otherwise make an all-A m-mer (canonical value 0) win every window comparison.

        -

        Partition routing is independent of minimizer selection

        -

        The hash used to select a minimizer within a window (the minimum of several hash values) and the hash used to route a super-kmer to a storage partition are computed separately:

        +

        The 16-state Sankoff model

        +

        Each family (see Central-position SNP model) is treated as a character with 16 possible states: one per subset of the 4 possible central bases, including the empty subset (no observed form). A genome pair's calibration reduces to two separately-estimated first-order Markov models, both restricted to genome pairs below a configurable SNP-ratio ceiling (excluding saturated pairs from skewing the calibration):

          -
        • Selection uses \(H\) applied to every candidate m-mer in the window, keeping the minimum.
        • -
        • Partition routing recomputes \(H\) on the single selected minimizer only, once its position is fixed. This is a hash of one specific value, not the minimum of several, so it is uniformly distributed and safe to use directly for routing.
        • +
        • Cardinality transitions (\(5 \times 5\), row-stochastic): the probability that a family observed with \(i\) forms in one genome (\(i \in \{0,\dots,4\}\)) is observed with \(j\) forms in the other.
        • +
        • Composition transitions (\(4 \times 4\), row-stochastic): the probability that an unambiguous, single-copy base observed as one of A/C/G/T in one genome is observed as another base in the other genome.
        -

        See Partitioning and indexing architecture for how the routing value is turned into a partition index.

        +

        Combining into a 16×16 cost matrix

        +

        The two matrices are not combined as a plain tensor product. For a pair of states \((A, B)\) — each a subset of \(\{A,C,G,T\}\) — let shared \(= A \cap B\), lost \(= A \setminus B\), gained \(= B \setminus A\). The log-probability of the transition \(A \to B\) is:

        +
        \[\log P(A,B) = \underbrace{\log P_{\text{card}}(|A|,|B|)}_{\text{cardinality term, omitted when loss events are treated as free}} \;+\; \underbrace{\sum_{i \,\in\, \text{shared}} \log P_{\text{comp}}(i,i)}_{\text{bases conserved on both sides}} \;-\; \underbrace{\text{best\_pairing\_cost}(\text{lost}, \text{gained})}_{\text{substitutions among the differing bases}}\]
        +

        best_pairing_cost enumerates every injection pairing elements of lost with elements of gained and keeps the one minimizing \(-\sum \log P_{\text{comp}}\) over the pairs — the discrete analogue of "prefer a substitution over an independent loss and gain": if a base is lost from one side and a different base is gained on the other, that is scored as a single substitution between them (cheaper, under any sane calibration, than treating them as two unrelated cardinality-changing events) whenever such a pairing is possible.

        +

        The resulting \(16 \times 16\) log-probability matrix is row-normalized in log-space (log-sum-exp, not a direct exp()/divide, for numerical stability), giving a genuine row-stochastic transition matrix \(P\). The cost matrix is then \(\text{cost}(A,B) = -\ln P(A,B)\), symmetrized by simple averaging:

        +
        \[\text{sym}(A,B) = \frac{\text{cost}(A,B) + \text{cost}(B,A)}{2}\]
        +

        This calibrated cost matrix is what every downstream export (for TNT, PhyG, IQ-TREE) reuses, recoded into each tool's own format.

        +

        See phylo for how to run the calibration and produce these exports.

        @@ -1248,10 +1577,10 @@ - + - + diff --git a/doc/usage/annotate/index.html b/doc/usage/annotate/index.html index e8f31b7e..6d7bce23 100644 --- a/doc/usage/annotate/index.html +++ b/doc/usage/annotate/index.html @@ -256,15 +256,26 @@ -
      • - + + + + + +
      • + + + + + + -
      • + + + + + + + + + + + + + + + + + + + + + +
      • + + + + + + + + + + +
      • + + + +
      @@ -1017,6 +1405,34 @@ + + + + + + +
    91. + + + + + + + + VCF pseudo-alignment + + + + + + + + +
    92. + + + + diff --git a/doc/usage/convert/index.html b/doc/usage/convert/index.html index c83e1aa1..7f81abee 100644 --- a/doc/usage/convert/index.html +++ b/doc/usage/convert/index.html @@ -256,15 +256,26 @@ -
    93. - + + + + + +
    94. + + + + + + -
    95. + + + + + + + + + + + + + + + + + + + + + +
    96. + + + + + + + + + + +
    97. + + + + @@ -1017,6 +1405,34 @@ + + + + + + +
    98. + + + + + + + + VCF pseudo-alignment + + + + + + + + +
    99. + + + + diff --git a/doc/usage/dump/index.html b/doc/usage/dump/index.html index 631611aa..039f2e37 100644 --- a/doc/usage/dump/index.html +++ b/doc/usage/dump/index.html @@ -256,15 +256,26 @@ -
    100. - + + + + + +
    101. + + + + + + -
    102. + + + + + + + + + + + + + + + + + + + + + +
    103. + + + + + + + + + + +
    104. + + + + @@ -1017,6 +1405,34 @@ + + + + + + +
    105. + + + + + + + + VCF pseudo-alignment + + + + + + + + +
    106. + + + + diff --git a/doc/usage/estimate/index.html b/doc/usage/estimate/index.html index 849c5939..d73dd447 100644 --- a/doc/usage/estimate/index.html +++ b/doc/usage/estimate/index.html @@ -256,15 +256,26 @@ -
    107. - + + + + + +
    108. + + + + + + -
    109. + + + + + + + + + + + + + + + + + + + + + +
    110. + + + + + + + + + + +
    111. + + + + @@ -1006,6 +1394,34 @@ + + + + + + +
    112. + + + + + + + + VCF pseudo-alignment + + + + + + + + +
    113. + + + + diff --git a/doc/usage/filter/index.html b/doc/usage/filter/index.html index 438b5ace..7fbe318d 100644 --- a/doc/usage/filter/index.html +++ b/doc/usage/filter/index.html @@ -256,15 +256,26 @@ -
    114. - + + + + + +
    115. + + + + + + -
    116. + + + + + + + + + + + + + + + + + + + + + +
    117. + + + + + + + + + + +
    118. + + + + @@ -1028,6 +1416,34 @@ + + + + + + +
    119. + + + + + + + + VCF pseudo-alignment + + + + + + + + +
    120. + + + + diff --git a/doc/usage/index_command/index.html b/doc/usage/index_command/index.html index 4d339687..9327e9d4 100644 --- a/doc/usage/index_command/index.html +++ b/doc/usage/index_command/index.html @@ -256,15 +256,26 @@ -
    121. - + + + + + +
    122. + + + + + + -
    123. + + + + + + + + + + + + + + + + + + + + + +
    124. + + + + + + + + + + +
    125. + + + + @@ -549,6 +937,17 @@ + + +
    126. + + + + Which files are read + + + +
    127. @@ -1028,6 +1427,34 @@ + + + + + + +
    128. + + + + + + + + VCF pseudo-alignment + + + + + + + + +
    129. + + + + @@ -1097,6 +1524,17 @@ + + +
    130. + + + + Which files are read + + + +
    131. @@ -1157,6 +1595,10 @@ +

      Which files are read

      +

      A file named explicitly on the command line is always read, whatever its extension: the format (FASTA, FASTQ or GenBank) and the compression (gzip, bzip2, xz or zstd) are detected from the content. If the content is not a sequence file, the command reports an error rather than skipping the file.

      +

      When an input is a directory, the files inside it are selected by extension: .fasta, .fa, .fna (the extension of NCBI genome downloads), .ffn, .fas, .fsa, .fastq, .fq, .gbff, .gbk and .gb, each optionally followed by .gz.

      +

      A small input spread over many partitions leaves most partitions empty. This is valid: the index is built normally and empty partitions simply hold no k-mers.

      Options

      diff --git a/doc/usage/merge/index.html b/doc/usage/merge/index.html index 61006d48..e8a7df7c 100644 --- a/doc/usage/merge/index.html +++ b/doc/usage/merge/index.html @@ -256,15 +256,26 @@ -
    132. - + + + + + +
    133. + + + + + + -
    134. + + + + + + + + + + + + + + + + + + + + + +
    135. + + + + + + + + + + +
    136. + + + + @@ -599,6 +987,28 @@ + + +
    137. + + + + Resource usage + + + + +
    138. + +
    139. + + + + Merging an index that was already merged + + + +
    140. @@ -1028,6 +1438,34 @@ + + + + + + +
    141. + + + + + + + + VCF pseudo-alignment + + + + + + + + +
    142. + + + + @@ -1119,6 +1557,28 @@ + + +
    143. + + + + Resource usage + + + + +
    144. + +
    145. + + + + Merging an index that was already merged + + + +
    146. @@ -1188,15 +1648,23 @@ - - - + + +
      Disambiguate duplicate genome labels (.1, .2, …) instead of failing
      --budget-fraction0.5Fraction of available RAM reserved as the memory budget for parallel partition merging--denseoffStore the merged index's presence/absence matrices in the dense packed format instead of the default sparse one (see pack)

      Behaviour

      The output mode is chosen automatically: if every source index stores counts, the merged index stores counts too; otherwise it is presence/absence. --force-presence forces presence/absence regardless of the sources.

      By default, merging two indexes that share a genome label fails with an error; --rename-duplicates instead appends a numeric suffix to keep both copies.

      +

      The merged index is written in the sparse packed format by default, because it is usually several times smaller on disk; --dense keeps the dense format, which is faster when whole genome columns are read at once, for example for distance matrices. The pack page explains the trade-off between the two formats.

      +

      Resource usage

      +

      The merge works partition by partition, and processes several partitions in parallel. Within a partition, the source indexes are read two at a time rather than all at once. The memory and the number of files mapped into memory for one partition therefore depend on how many k-mers the partition holds, not on how many indexes are being merged: merging hundreds of genomes maps about as many files per partition as merging three. The partitions do grow as more diverse genomes are added, so the total memory still rises with the size of the merged collection.

      +

      Each genome column being filled is held in memory and written to disk in one go once it is complete. A full disk or an exhausted quota is therefore reported as an ordinary error rather than crashing the program. The largest memory cost comes from the k-mers that are new to the merged index, which are assembled in memory, partition by partition, before being written out.

      +

      Merging an index that was already merged

      +

      An index produced by merge can be used as one of the sources of a later merge, which is how a collection is extended without starting again from the original genomes. There is one rule: among the sources, at least one must not have been through a merge. A merged index is stored in a compact packed format (see pack) that is read very efficiently but cannot be extended in place. The merge therefore takes a freshly built index as its starting point and brings the packed index in as a source. A typical case is a merged index of many genomes plus one or a few newly built single-genome indexes, and this works.

      +

      If every source has already been merged or packed, the command stops before doing any work and says so. In that case, merge the original single-genome indexes together instead.

      +

      The genome columns of the result follow the order of the starting index first, then the other sources, so they may not come out in the order the sources were listed. The genome labels always identify each column unambiguously.

      diff --git a/doc/usage/pack/index.html b/doc/usage/pack/index.html index 0db80671..b25836fa 100644 --- a/doc/usage/pack/index.html +++ b/doc/usage/pack/index.html @@ -256,15 +256,26 @@ -
    147. - + + + + + +
    148. + + + + + + -
    149. + + + + + + + + + + + + + + + + + + + + + +
    150. + + + + + + + + + + +
    151. + + + + @@ -1028,6 +1416,34 @@ + + + + + + +
    152. + + + + + + + + VCF pseudo-alignment + + + + + + + + +
    153. + + + + diff --git a/doc/usage/phylo/index.html b/doc/usage/phylo/index.html index 9d9c30dd..0f554def 100644 --- a/doc/usage/phylo/index.html +++ b/doc/usage/phylo/index.html @@ -256,15 +256,26 @@ -
    154. - + + + + + +
    155. + + + + + + -
    156. + + + + + + + + + + + + + + + + + + + + + +
    157. + + + + + + + + + + +
    158. + + + + @@ -736,6 +1124,17 @@
        +
      • + + + + Usage + + + + +
      • +
      • @@ -748,22 +1147,44 @@
      • - + - Distance matrix (--distance) + General options -
      • + +
      • + + + + SNP families + + + + +
      • + +
      • + + + + Distance estimation and tree reconstruction + + + + + @@ -655,6 +1043,17 @@ +
      • + +
      • + + + + Disk usage + + + +
      @@ -1028,6 +1427,34 @@ + + + + + + +
    159. + + + + + + + + VCF pseudo-alignment + + + + + + + + +
    160. + + + + @@ -1119,6 +1546,17 @@ + + +
    161. + + + + Disk usage + + + +
    162. @@ -1139,8 +1577,8 @@

      select

      -

      Project and/or aggregate the genome columns of an index into a new (or in-place) index. Where filter selects rows (kmers), select operates on columns (genomes): grouping several genomes into one aggregated column, reordering columns, or dropping some.

      -
      obikmer select SOURCE (--output OUTPUT | --in-place) [OPTIONS]
      +

      Project and/or aggregate the genome columns of an index into a new index. Where filter selects rows (kmers), select operates on columns (genomes): grouping several genomes into one aggregated column, reordering columns, or dropping some.

      +
      obikmer select SOURCE --output OUTPUT [OPTIONS]
       

      Arguments

      @@ -1168,14 +1606,9 @@ - + - - - - - - + @@ -1212,12 +1645,29 @@ + + + + + + + + + +
      --output-o, --output —Output index directory (mutually exclusive with --in-place)
      --in-placeoffRewrite the source index in place (mutually exclusive with --output)Output index directory (required)
      -f, --force0 Minimum count for a genome to be considered a carrier (logical operators only)
      --denseoffPack the output's presence matrices in the dense format instead of the default sparse one
      --force-copyoffCopy each layer's unchanged kmer-identity files (mphf/unitigs/evidence/fingerprint) instead of hard-linking them

      Aggregation operators

      any, all, none (logical, evaluated against --presence-threshold), sum, min, max (numeric, count index only). If a group's operator is left unspecified, it defaults to any when the source is a presence/absence index and sum when it stores counts.

      A select never changes the underlying kmer set — only the per-genome data (counts or presence) is rewritten, so an unaggregated pass-through column (a plain genome label in --select) is a cheap copy.

      -

      At least one of --output/--in-place is required, and at least one output column must be defined; every name listed in --select must resolve to either a defined group or an existing genome label. See Genome predicates and taxonomy paths for the predicate syntax used by --group.

      +

      At least one output column must be defined; every name listed in --select must resolve to either a defined group or an existing genome label. See Genome predicates and taxonomy paths for the predicate syntax used by --group.

      +

      Disk usage

      +

      select always writes to a new output directory — there is no in-place mode. Each layer's kmer-identity files (MPHF, unitigs, evidence, fingerprint) never change under a column projection/aggregation, so they are hard-linked into the output rather than copied: no extra disk is used for them, even on a very large index. Linking falls back to a real copy automatically if it fails (e.g. SOURCE/OUTPUT on different filesystems). Use --force-copy to always copy instead — needed when the output must be able to survive independently of the source on disk (a hard link shares the same underlying data, so overwriting one path outside select itself would affect the other).

      +

      To replace an index with a selected version of itself, select to a temporary directory and swap it in:

      +
      obikmer select INDEX --output INDEX.tmp --group ... --group-op ... --select ...
      +rm -rf INDEX
      +mv INDEX.tmp INDEX
      +
      diff --git a/doc/usage/superkmer/index.html b/doc/usage/superkmer/index.html index 25a96334..dcf8e597 100644 --- a/doc/usage/superkmer/index.html +++ b/doc/usage/superkmer/index.html @@ -9,7 +9,7 @@ - + @@ -256,15 +256,26 @@ -
    163. - + + + + + +
    164. + + + + + + -
    165. + + + + + + + + + + + + + + + + + + + + + +
    166. + + + + + + + + + + +
    167. + + + + @@ -1017,6 +1405,34 @@ + + + + + + +
    168. + + + + + + + + VCF pseudo-alignment + + + + + + + + +
    169. + + + + diff --git a/doc/usage/unitig/index.html b/doc/usage/unitig/index.html index f07efbaa..fdb8920b 100644 --- a/doc/usage/unitig/index.html +++ b/doc/usage/unitig/index.html @@ -256,15 +256,26 @@ -
    170. - + + + + + +
    171. + + + + + + -
    172. + + + + + + + + + + + + + + + + + + + + + +
    173. + + + + + + + + + + +
    174. + + + + @@ -1017,6 +1405,34 @@ + + + + + + +
    175. + + + + + + + + VCF pseudo-alignment + + + + + + + + +
    176. + + + + diff --git a/doc/usage/utils/index.html b/doc/usage/utils/index.html index cc372a21..9e668a8d 100644 --- a/doc/usage/utils/index.html +++ b/doc/usage/utils/index.html @@ -256,15 +256,26 @@ -
    177. - + + + + + +
    178. + + + + + + -
    179. + + + + + + + + + + + + + + + + + + + + + +
    180. + + + + + + + + + + +
    181. + + + + @@ -1017,6 +1405,34 @@ + + + + + + +
    182. + + + + + + + + VCF pseudo-alignment + + + + + + + + +
    183. + + + + diff --git a/src/obikmer/src/cmd/phylo/args/distance.rs b/src/obikmer/src/cmd/phylo/args/distance.rs index 659355b3..7370ffea 100644 --- a/src/obikmer/src/cmd/phylo/args/distance.rs +++ b/src/obikmer/src/cmd/phylo/args/distance.rs @@ -77,27 +77,21 @@ pub struct DistanceArgs { #[arg(long, value_name = "ALPHA|auto", value_parser = parse_gamma_shape)] pub gamma_shape: Option, - /// Save and reuse the sample of a SNP-based distance + /// Save and reuse a SNP distance sample (unavailable for now) /// - /// Saves, in this directory, the sample's tally of pairwise differences - /// between genomes, so that a later run on the same index can reuse it - /// instead of drawing again. The directory is created if needed. + /// Meant to save, in this directory, the sample of a SNP-based distance, + /// so that a later run with identical options reuses it instead of + /// drawing again. /// - /// The saved sample is reused only if these options are identical: - /// --n-sites, --free-loss, --no-ambiguity, --quartet, --exclude-genome, - /// --min-shared-family, --entropy and --entropy-sd. Otherwise the run - /// stops and lists the differences, unless --session-force is given. - /// - /// For now only SNP-based distances use sessions; Sankoff analyses and - /// the TNT, PhyG and IQ-TREE exports neither read nor write them. + /// This feature is being rebuilt and is not available for now: giving + /// this option, or --session-force, stops the command with an error. #[arg(long, value_name = "DIR")] pub session: Option, - /// Replace a saved sample made with other parameters + /// With --session: replace a saved sample (unavailable for now) /// - /// With --session, discards the saved sample when its parameters differ - /// from this run's, and draws a new one. Without this option, a - /// difference is an error. + /// Meant to discard a saved sample made with other parameters and draw a + /// new one. Unavailable for now, like --session. #[arg(long, requires = "session")] pub session_force: bool,