Refactor k-mer indexing, enhance session features, and update documentation

This commit introduces significant refinements across the codebase. Core indexing logic is updated with hard constraints on k-mer length, statistical modeling clarifications, and the implementation of the Karlin & Ost theory for chance-collision statistics. Session management features are enhanced with robust memoization capabilities, atomic file operations, per-key locking, and memory mapping support. Additionally, documentation is updated, including a new citation style, and new mathematical notebook support is added.
This commit is contained in:
Eric Coissac committed 2026-10-13 08:41:42 +02:00
1 parent e715528d34
commit 0d1ecac233
13 files changed
+1683 -18

No files matched your search

+157
View File
@@ -0,0 +1,157 @@
<?xml version="1.0" encoding="utf-8"?>
<style xmlns="http://purl.org/net/xbiblio/csl" class="in-text" version="1.0" default-locale="en-US" demote-non-dropping-particle="sort-only">
<info>
<title>Molecular Ecology Modified</title>
<id>http://www.zotero.org/styles/molecular-ecology</id>
<link href="http://www.zotero.org/styles/molecular-ecology" rel="self"/>
<link href="http://www.zotero.org/styles/ecology-letters" rel="template"/>
<link href="http://onlinelibrary.wiley.com/journal/10.1111/%28ISSN%291365-294X/homepage/ForAuthors.html" rel="documentation"/>
<author>
<name>Xiaodong Dang</name>
<email>dangxdong@gmail.com</email>
</author>
<contributor>
<name>Johan Asplund</name>
<email>asplundj@gmail.com</email>
</contributor>
<contributor>
<name>David Kaplan</name>
<email>david.kaplan@ird.fr</email>
</contributor>
<category citation-format="author-date"/>
<category field="biology"/>
<issn>0962-1083</issn>
<eissn>1365-294X</eissn>
<updated>2012-09-09T22:24:43+00:00</updated>
<rights license="http://creativecommons.org/licenses/by-sa/3.0/">This work is licensed under a Creative Commons Attribution-ShareAlike 3.0 License</rights>
</info>
<macro name="editor-translator">
<names variable="editor translator" prefix=" (" delimiter=", " suffix=")">
<name initialize-with="" delimiter=", "/>
<label form="short" prefix=", " text-case="capitalize-first" suffix="," strip-periods="true"/>
</names>
</macro>
<macro name="author">
<names variable="author">
<name name-as-sort-order="all" sort-separator=" " initialize-with="" delimiter=", " delimiter-precedes-et-al="never"/>
<label form="short" prefix=" (" suffix=")" text-case="capitalize-first"/>
<et-al font-style="italic"/>
<substitute>
<names variable="editor"/>
<names variable="translator"/>
<text macro="title"/>
</substitute>
</names>
</macro>
<macro name="author-short">
<names variable="author">
<name form="short" and="symbol" delimiter=", " initialize-with=". "/>
<et-al font-style="italic"/>
<substitute>
<names variable="editor"/>
<names variable="translator"/>
<choose>
<if type="bill book graphic legal_case legislation motion_picture report song" match="any">
<text variable="title" form="short" font-style="italic"/>
</if>
<else>
<text variable="title" form="short" quotes="true"/>
</else>
</choose>
</substitute>
</names>
</macro>
<macro name="title">
<choose>
<if type="bill book graphic legal_case legislation motion_picture report song" match="any">
<text variable="title" font-style="italic"/>
</if>
<else>
<text variable="title"/>
</else>
</choose>
</macro>
<macro name="publisher">
<group delimiter=", ">
<text variable="publisher"/>
<text variable="publisher-place"/>
</group>
</macro>
<citation collapse="year-suffix" et-al-min="3" et-al-use-first="1" disambiguate-add-year-suffix="true">
<sort>
<key variable="issued"/>
<key variable="author"/>
</sort>
<layout prefix="(" suffix=")" delimiter="; ">
<group delimiter=" ">
<text macro="author-short"/>
<date variable="issued">
<date-part name="year"/>
</date>
<group>
<label variable="locator" form="short"/>
<text variable="locator" prefix=" "/>
</group>
</group>
</layout>
</citation>
<bibliography hanging-indent="true" et-al-min="6" et-al-use-first="1000" entry-spacing="0.5" line-spacing="1">
<sort>
<key macro="author-short"/>
<key variable="issued"/>
</sort>
<layout>
<text macro="author" suffix=" "/>
<date variable="issued" prefix="(" suffix=")">
<date-part name="year"/>
</date>
<choose>
<if type="bill book graphic legal_case legislation motion_picture report song" match="any">
<group suffix=".">
<text macro="title" prefix=" "/>
<text macro="editor-translator" prefix=" "/>
</group>
<text prefix=" " suffix="." macro="publisher"/>
</if>
<else-if type="thesis" match="any">
<group suffix=".">
<text macro="title" prefix=" " suffix="."/>
<text variable="genre" prefix=" " suffix=" Thesis."/>
<text prefix=" " macro="publisher"/>
</group>
</else-if>
<else-if type="chapter paper-conference" match="any">
<text macro="title" prefix=" "/>
<group prefix=".">
<group prefix=" In:">
<text variable="container-title" font-style="italic" prefix=" "/>
<text variable="collection-title" prefix=" " suffix="."/>
<names variable="editor translator" prefix=" (" delimiter=", " suffix=")">
<label form="short" suffix=" " strip-periods="true"/>
<name name-as-sort-order="all" sort-separator=" " initialize-with="" delimiter=", " delimiter-precedes-last="never"/>
</names>
<group suffix=".">
<label variable="page" form="short" prefix=", " suffix=" "/>
<text variable="page"/>
<text macro="publisher" prefix=". " suffix="."/>
</group>
</group>
</group>
</else-if>
<else>
<group suffix=".">
<text macro="title" prefix=" "/>
<text macro="editor-translator" prefix=" "/>
</group>
<group prefix=" " suffix=".">
<text variable="container-title" text-case="title" font-style="italic"/>
<group prefix=", ">
<text variable="volume" font-weight="bold"/>
</group>
<text variable="page" prefix=", "/>
</group>
</else>
</choose>
</layout>
</bibliography>
</style>
+202
View File
@@ -566,3 +566,205 @@
issn = "0167-8655,1872-7344",
language = "en"
}
@ARTICLE{Fleischmann2014-ed,
title = {Evolution of genome size and chromosome number in the
carnivorous plant genus Genlisea (Lentibulariaceae), with a new
estimate of the minimum genome size in angiosperms},
author = "Fleischmann, Andreas and Michael, Todd P and Rivadavia, Fernando
and Sousa, Aretuza and Wang, Wenqin and Temsch, Eva M and
Greilhuber, Johann and Müller, Kai F and Heubl, Günther",
journal = "Annals of Botany",
publisher = "Oxford University Press (OUP)",
volume = 114,
number = 8,
pages = "1651--1663",
abstract = "BACKGROUND AND AIMS: Some species of Genlisea possess ultrasmall
nuclear genomes, the smallest known among angiosperms, and some
have been found to have chromosomes of diminutive size, which may
explain why chromosome numbers and karyotypes are not known for
the majority of species of the genus. However, other members of
the genus do not possess ultrasmall genomes, nor do most taxa
studied in related genera of the family or order. This study
therefore examined the evolution of genome sizes and chromosome
numbers in Genlisea in a phylogenetic context. The correlations
of genome size with chromosome number and size, with the
phylogeny of the group and with growth forms and habitats were
also examined. METHODS: Nuclear genome sizes were measured from
cultivated plant material for a comprehensive sampling of taxa,
including nearly half of all species of Genlisea and representing
all major lineages. Flow cytometric measurements were conducted
in parallel in two laboratories in order to compare the
consistency of different methods and controls. Chromosome counts
were performed for the majority of taxa, comparing different
staining techniques for the ultrasmall chromosomes. KEY RESULTS:
Genome sizes of 15 taxa of Genlisea are presented and interpreted
in a phylogenetic context. A high degree of congruence was found
between genome size distribution and the major phylogenetic
lineages. Ultrasmall genomes with 1C values of <100 Mbp were
almost exclusively found in a derived lineage of South American
species. The ancestral haploid chromosome number was inferred to
be n = 8. Chromosome numbers in Genlisea ranged from 2n = 2x = 16
to 2n = 4x = 32. Ascendant dysploid series (2n = 36, 38) are
documented for three derived taxa. The different ploidy levels
corresponded to the two subgenera, but were not directly
correlated to differences in genome size; the three different
karyotype ranges mirrored the different sections of the genus.
The smallest known plant genomes were not found in G. margaretae,
as previously reported, but in G. tuberosa (1C ≈ 61 Mbp) and some
strains of G. aurea (1C ≈ 64 Mbp). CONCLUSIONS: Genlisea is an
ideal candidate model organism for the understanding of genome
reduction as the genus includes species with both relatively
large (∼1700 Mbp) and ultrasmall (∼61 Mbp) genomes. This
comparative, phylogeny-based analysis of genome sizes and
karyotypes in Genlisea provides essential data for selection of
suitable species for comparative whole-genome analyses, as well
as for further studies on both the molecular and cytogenetic
basis of genome reduction in plants.",
month = dec,
year = 2014,
url = "http://dx.doi.org/10.1093/aob/mcu189",
keywords = "Bladderwort; Genlisea; Lamiales; Lentibulariaceae; carnivorous
plant; chromosome number; flow cytometry; genome miniaturization;
genome size",
doi = "10.1093/aob/mcu189",
pmc = "PMC4649684",
pmid = 25274549,
issn = "0305-7364,1095-8290",
language = "en"
}
@ARTICLE{Fernandez2024-kd,
title = "A 160 Gbp fork fern genome shatters size record for eukaryotes",
author = "Fernández, Pol and Amice, Rémy and Bruy, David and Christenhusz,
Maarten J M and Leitch, Ilia J and Leitch, Andrew L and Pokorny,
Lisa and Hidalgo, Oriane and Pellicer, Jaume",
journal = "iScience",
publisher = "Elsevier BV",
volume = 27,
number = 6,
pages = 109889,
abstract = "Vascular plants are exceptional among eukaryotes due to their
outstanding genome size diversity which ranges ∼2,400-fold,
including the largest genome so far recorded in the angiosperm
Paris japonica (148.89 Gbp/1C). Despite available data showing
that giant genomes are restricted across the Tree of Life, the
biological limits to genome size expansion remain to be
established. Here, we report the discovery of an even larger
eukaryotic genome in Tmesipteris oblanceolata, a New Caledonian
fork fern. At 160.45 Gbp/1C, this record-breaking genome
challenges current understanding and opens new avenues to explore
the evolutionary dynamics of genomic gigantism.",
month = jun,
year = 2024,
url = "http://dx.doi.org/10.1016/j.isci.2024.109889",
keywords = "Plant evolution; Plant genetics",
doi = "10.1016/j.isci.2024.109889",
pmc = "PMC11270024",
pmid = 39055604,
issn = "2589-0042",
language = "en"
}
@ARTICLE{Fraser1995-jc,
title = "The minimal gene complement of Mycoplasma genitalium",
author = "Fraser, C M and Gocayne, J D and White, O and Adams, M D and
Clayton, R A and Fleischmann, R D and Bult, C J and Kerlavage, A
R and Sutton, G and Kelley, J M and Fritchman, R D and Weidman, J
F and Small, K V and Sandusky, M and Fuhrmann, J and Nguyen, D
and Utterback, T R and Saudek, D M and Phillips, C A and Merrick,
J M and Tomb, J F and Dougherty, B A and Bott, K F and Hu, P C
and Lucier, T S and Peterson, S N and Smith, H O and Hutchison,
3rd, C A and Venter, J C",
journal = "Science (New York, N.Y.)",
publisher = "American Association for the Advancement of Science (AAAS)",
volume = 270,
number = 5235,
pages = "397--403",
abstract = "The complete nucleotide sequence (580,070 base pairs) of the
Mycoplasma genitalium genome, the smallest known genome of any
free-living organism, has been determined by whole-genome random
sequencing and assembly. A total of only 470 predicted coding
regions were identified that include genes required for DNA
replication, transcription and translation, DNA repair, cellular
transport, and energy metabolism. Comparison of this genome to
that of Haemophilus influenzae suggests that differences in
genome content are reflected as profound differences in
physiology and metabolic capacity between these two organisms.",
month = oct,
year = 1995,
url = "http://dx.doi.org/10.1126/science.270.5235.397",
doi = "10.1126/science.270.5235.397",
pmid = 7569993,
issn = "0036-8075,1095-9203",
language = "en"
}
@ARTICLE{Schneiker2007-gs,
title = "Complete genome sequence of the myxobacterium Sorangium
cellulosum",
author = "Schneiker, Susanne and Perlova, Olena and Kaiser, Olaf and Gerth,
Klaus and Alici, Aysel and Altmeyer, Matthias O and Bartels,
Daniela and Bekel, Thomas and Beyer, Stefan and Bode, Edna and
Bode, Helge B and Bolten, Christoph J and Choudhuri, Jomuna V and
Doss, Sabrina and Elnakady, Yasser A and Frank, Bettina and
Gaigalat, Lars and Goesmann, Alexander and Groeger, Carolin and
Gross, Frank and Jelsbak, Lars and Jelsbak, Lotte and Kalinowski,
Jörn and Kegler, Carsten and Knauber, Tina and Konietzny,
Sebastian and Kopp, Maren and Krause, Lutz and Krug, Daniel and
Linke, Bukhard and Mahmud, Taifo and Martinez-Arias, Rosa and
McHardy, Alice C and Merai, Michelle and Meyer, Folker and
Mormann, Sascha and Muñoz-Dorado, Jose and Perez, Juana and
Pradella, Silke and Rachid, Shwan and Raddatz, Günter and
Rosenau, Frank and Rückert, Christian and Sasse, Florenz and
Scharfe, Maren and Schuster, Stephan C and Suen, Garret and
Treuner-Lange, Anke and Velicer, Gregory J and Vorhölter,
Frank-Jörg and Weissman, Kira J and Welch, Roy D and Wenzel,
Silke C and Whitworth, David E and Wilhelm, Susanne and Wittmann,
Christoph and Blöcker, Helmut and Pühler, Alfred and Müller, Rolf",
journal = "Nature Biotechnology",
publisher = "Springer Science and Business Media LLC",
volume = 25,
number = 11,
pages = "1281--1289",
abstract = "The genus Sorangium synthesizes approximately half of the
secondary metabolites isolated from myxobacteria, including the
anti-cancer metabolite epothilone. We report the complete genome
sequence of the model Sorangium strain S. cellulosum So ce56,
which produces several natural products and has morphological and
physiological properties typical of the genus. The circular
genome, comprising 13,033,779 base pairs, is the largest
bacterial genome sequenced to date. No global synteny with the
genome of Myxococcus xanthus is apparent, revealing an
unanticipated level of divergence between these myxobacteria. A
large percentage of the genome is devoted to regulation,
particularly post-translational phosphorylation, which probably
supports the strain's complex, social lifestyle. This regulatory
network includes the highest number of eukaryotic protein
kinase-like kinases discovered in any organism. Seventeen
secondary metabolite loci are encoded in the genome, as well as
many enzymes with potential utility in industry.",
month = nov,
year = 2007,
url = "http://dx.doi.org/10.1038/nbt1354",
doi = "10.1038/nbt1354",
pmid = 17965706,
issn = "1087-0156,1546-1696",
language = "en"
}
@incollection{Karlin1985,
author = {Karlin, Samuel and Ost, Fr{\'e}d{\'e}ric},
title = {Maximal segmental match length among random sequences from a finite alphabet},
booktitle = {Proceedings of the Berkeley Conference in Honor of Jerzy Neyman and Jack Kiefer},
editor = {Le Cam, Lucien M. and Olshen, Richard A.},
volume = {1},
pages = {225--243},
year = {1985},
publisher = {Wadsworth},
address = {Monterey, CA}
}
@article{Karlin1988,
author = {Karlin, Samuel and Ost, Fr{\'e}d{\'e}ric},
title = {Maximal length of common words among random letter sequences},
journal = {The Annals of Probability},
volume = {16},
number = {2},
pages = {535--563},
year = {1988}
}
+220 -4
View File
@@ -1,6 +1,222 @@
# Kmers
# K-mer size
A **kmer** is a DNA subsequence of fixed length $k$. Two constraints apply to $k$, both enforced when a command starts (an invalid value exits immediately with an error):
A k-mer is a DNA sequence of length $k$. OBIKmer is designed to index large k-mers. Here, *large* refers to a k-mer length for which the probability of observing the same k-mer in several genomes by chance alone is sufficiently small. The k-mer length must therefore be large enough that, when a k-mer is observed in several genomes of the dataset, the probability that this shared occurrence is due to chance alone is sufficiently low for the match to be considered evidence of homology.
- $k \in [11, 31]$: long enough to be specific, short enough to fit in a single 64-bit word at 2 bits/base ($k \le 32$ is the hard limit; $k < 11$ gives insufficient specificity).
- $k$ **is odd**: an odd-length sequence can never equal its own reverse complement, so the two orientations of any kmer are always distinct. This is required for the canonical form (see [DNA encoding](encoding.md)) to be well defined.
OBIKmer restricts $k$ to odd values in the range
$$
11 \leq k \leq 31.
$$
These limits result from two different considerations. The upper bound is imposed by the representation used by OBIKmer, whereas the lower bound defines a practical range of k-mer lengths for genome-scale sequence comparisons. This range covers most practical applications: the lower limit is well below the k-mer size that can be considered large, even for the smallest bacterial genomes, whereas the upper limit provides sufficient specificity for most applications, including those involving very large genomes.
## Representation limit
OBIKmer uses two bits to encode each nucleotide. Therefore, a k-mer requires $2k$ bits. To ensure computational efficiency, OBIKmer limits the representation of k-mers to a single 64-bit word, thereby restricting their length.
$$
2k \leq 64,
$$
This gives the hard representation limit: $k \leq 32.$ Furthermore, OBIKmer only considers odd values of $k$, meaning the largest supported k-mer size is $31$.
### K-mer specificity
A simple way to illustrate the effect of k-mer size is to consider a random DNA sequence in which the four nucleotides are independent and equally probable. There are
$$
4^k
$$
possible k-mers, and a particular k-mer is therefore expected to occur once every $4^k$ positions on average. In a sequence of length $G$, the expected number of occurrences of a particular k-mer is approximately
$$
E[n_K] \simeq \frac{G}{4^k}.
$$
When $4^k\gg G$, this quantity is also a good approximation of the probability that an observed k-mer will occur at least once by chance elsewhere in a sequence of length $G$. It therefore provides a simple measure of the risk of a chance recurrence of an observed k-mer:
$$
P_{\mathrm{chance}}(k,G) \simeq \frac{G}{4^k}.
$$
This probability decreases by a factor of four for every additional nucleotide in the k-mer. Conversely, it increases linearly with the amount of sequence searched.
This simple model already shows why very short k-mers provide limited specificity for genome-scale analyses. With $k=11$,
$$
4^{11}\simeq4.2\times10^6,
$$
which is a small combinatorial space even compared with the size of very small bacterial genomes. In contrast,
$$
4^{31}\simeq4.6\times10^{18},
$$
which is vastly larger than the number of k-mer positions even in very large genomes. Thus, increasing $k$ from 11 to 31 provides an enormous increase in sequence specificity.
The estimate above is intentionally simplistic. Real genomes do not consist of independent, equiprobable nucleotides: nucleotide composition and sequence dependencies affect the probability of observing particular words. More accurate estimates of the probability of observing matches by chance therefore take sequence length and nucleotide composition into account.
Karlin and Ost provide an analytical framework for this problem based on the distribution of the longest exact match expected between random sequences. If $L_{\max}$ denotes the length of the longest seed observed in random sequences of given size and nucleotide composition, the corresponding seed-level probability is
$$
P_{\mathrm{seeds}}(L)=P(L_{\max}\geq L).
$$
Conversely, for a chosen false-positive probability $P$, one can determine the minimum seed length $L_{\min}$ such that matches of length $L_{\min}$ or greater are not expected to occur by chance. This is the approach used by Repseek to determine statistically significant seed lengths.
The simple $G/4^k$ estimate and the Karlin–Ost statistics therefore describe the same fundamental effect at different levels of approximation: the probability of a chance match decreases rapidly with k-mer length, while the amount of sequence being searched increases the opportunity for such matches.
Consequently, the appropriate value of $k$ depends on the total amount of sequence being compared and its nucleotide composition. Choosing $k$ appropriately allows the false-positive rate of homology inference to be controlled.
Considering that bacterial genomes are typically a few megabases long, with the smallesr plublished today *Mycoplasma genitalium* having a size $G=580\,Mb$ [@Fraser1995-jc], and the largest $13\,Mb$ in *Sorangium cellulosum* [@Schneiker2007-gs]. For such genomes, the space of 11-mers is already to small to get a reasonable false positive rate:
$$
\begin{aligned}
P(C\geq1) &= 1- e^{-\frac{580.000^2}{2 \cdot 4^{11}}} \\
&\simeq 1
\end{aligned}
$$
, whereas 21-mers already provide a space of several trillion possible sequences.
Plant genomes illustrate the opposite situation. Plant genome sizes span several orders of magnitude, from the smallest angiosperm genomes present in the carnivorous plans genus [@Fleischmann2014-ed] about 60 Mb to the largest one 160 Gb in Tmesipteris oblanceolata, a New Caledonian fork fern [@Fernandez2024-kd], and large genomes often contain substantial amounts of repeated sequence. Even at $k=31$, however, the theoretical k-mer space remains vastly larger than any individual genome. At these lengths, the limiting factor is therefore not exhaustion of the theoretical k-mer space but the biological repetitiveness and non-random composition of the genome.
For example, the number of possible k-mers grows as follows:
| $k$ | $4^k$ | $4^k/2$ after canonicalization |
| ----: | --------------------: | -------------------------------: |
| 11 | $4.19\times10^6$ | $2.10\times10^6$ |
| 15 | $1.07\times10^9$ | $5.37\times10^8$ |
| 21 | $4.40\times10^{12}$ | $2.20\times10^{12}$ |
| 25 | $1.13\times10^{15}$ | $5.63\times10^{14}$ |
| 31 | $4.61\times10^{18}$ | $2.31\times10^{18}$ |
Canonicalization approximately halves the number of possible representations because a k-mer and its reverse complement are represented by the same canonical form.
The distinction between the simple $4^k$ model and the Karlin--Ost framework is important. The former describes the size of the combinatorial k-mer space; the latter describes the probability of observing long matches in sequences whose composition may depart from the uniform random model. Consequently, the value of $k$ cannot be selected from genome size alone.
These scales have very different implications for different genome sizes.
The random-sequence calculation is only a null model. Real genomes exhibit strong and reproducible oligonucleotide composition biases. Karlin and colleagues showed that relative abundances of short oligonucleotides, in particular dinucleotides, form characteristic genomic signatures that remain relatively stable within genomes and differ between organisms [@Karlin1995; @Karlin1997]. More generally, short k-mer frequencies can therefore capture compositional properties of genomes rather than provide highly specific sequence identifiers.
This distinction is important for whole-genome k-mer comparison. Increasing $k$ increases the theoretical diversity of the k-mer space exponentially and generally increases the proportion of k-mers that occur at unique genomic positions. Conversely, shorter k-mers occur more frequently and are more strongly influenced by genomic composition and repeated sequence. Empirical studies of k-mer spectra show this transition clearly: for example, k-mers of length $k\leq11$ can behave primarily as genomic-composition signatures, whereas longer k-mers progressively provide greater sequence specificity.
The supported range $11\leq k\leq31$ should therefore be understood as an engineering compromise rather than as a universal biological optimum. Smaller values increase the probability that unrelated genomic regions share the same k-mers, while larger values increase specificity but also make the representation more sensitive to substitutions, indels, and incomplete homology. The appropriate value within this range consequently depends on the biological scale and comparison task.
## Odd k-mer lengths
OBIKmer additionally requires $k$ to be odd. This constraint is independent of the resolution argument above.
For an odd-length DNA sequence, a k-mer cannot be identical to its own reverse complement. Consequently, every k-mer has two distinct orientations, which can be unambiguously collapsed into a canonical representation. For even $k$, self-reverse-complementary sequences exist, so a k-mer can coincide with its own reverse complement.
Odd $k$ is therefore **not required for canonicalization itself**: the canonical form
$$
s^c=\min_{\mathrm{lex}}\left(s,\operatorname{RC}(s)\right)
$$
remains well defined for even $k$. The odd-$k$ restriction instead guarantees that the two orientations are always distinct, which simplifies the representation and orientation handling used by OBIKmer.
## Karlin--Ost (1988) formula
[@Karlin1988] considers $s$ independent stationary letter sequences over a finite alphabet and studies $K_{r,s}(N)$, the length of the longest word common to $r$ of the $s$ sequences, each of length $N$. For i.i.d. equiprobable letters over an alphabet of size $m$ (here $m=4$ for DNA), the characteristic local match parameter reduces to
$$
\lambda^{[r]} = m^{1-r},
$$
so for $r=2$ (a word shared by two occurrences), $\lambda = m^{-1} = 1/4$.
The expected number of common words of length $k$ among the $\binom{s}{r}$ relevant groups of sequences is
$$
S_1(N,k) = N^{r}\binom{s}{r}\gamma\,\lambda^{k}\,(1+o(1)),
$$
with $\gamma=1$ exactly in the i.i.d. equiprobable case. Theorem 2.2 gives the limiting distribution of $K_{r,s}(N)$, with $k$ set on the scale $k=\left[\dfrac{\log N^{r}}{-\log\lambda^{*}}+x+1\right]$:
$$
\lim_{N\to\infty}\left[\Pr\{K_{r,s}(N)<k\} - \exp\{-(1-\lambda^{*})\,S_1(N,k)\}\right] = 0,
$$
equivalently
$$
\frac{-\log\Pr\{K_{r,s}(N)<k\}}{(\lambda^{*})^{\rho(N,x)}} \xrightarrow[N\to\infty]{} (1-\lambda^{*})\,\gamma^{*}(\lambda^{*})^{x}.
$$
The factor $(1-\lambda^{*})$ multiplying $S_1(N,k)$ is the key correction over the naive pairwise-collision count: it de-clumps overlapping candidate matches, which are not independent events, into an effective Poisson rate. This theorem is stated for $s\geq2$ independent sequences (e.g. comparing two distinct genomes); the intra-sequence self-repeat case (a single genome compared to itself) is treated separately, via the reverse-complement match length $R(N)$, in [@Karlin1985].
$o(1)$ (« petit o de 1 ») désigne une quantité qui **tend vers 0 quand $N\to\infty$**, mais dont on ne précise pas la vitesse exacte ni la formule exacte. C'est une façon d'écrire « il reste un terme d'erreur, mais il devient négligeable ».
Concrètement dans $S_1(N,k) = N^{r}\binom{s}{r}\gamma\,\lambda^{k}\,(1+o(1))$ :
- $N^{r}\binom{s}{r}\gamma\,\lambda^{k}$ est la valeur **exacte à la limite**, celle qu'on utilise en pratique (c'est elle qui redonne $E[C]$ de notre discussion).
- $(1+o(1))$ dit : « la vraie valeur de $S_1(N,k)$ n'est pas *exactement* égale à ce produit pour $N$ fini, mais le rapport entre les deux tend vers 1 quand $N\to\infty$ ». Autrement dit $S_1(N,k) / \left[N^{r}\binom{s}{r}\gamma\lambda^{k}\right] \to 1$.
Pourquoi ce terme existe : la formule $F_\sigma(k)/\lambda^k \to \gamma$ (éq. 2.8 du papier) est elle-même une limite quand $k\to\infty$ — pour $k$ fini, il y a un petit écart entre $F_\sigma(k)$ et $\gamma\lambda^k$. C'est cet écart-là qui devient le $o(1)$.
Ce que ça implique pour l'usage pratique : dans le cas iid équiprobable que tu utilises (ADN, $\gamma=1$ exactement, pas seulement à la limite — je l'ai vérifié directement sur la dérivation : $F_\sigma(k)=m^{-k}=\lambda^k$ pour tout $k$, pas juste asymptotiquement), le $o(1)$ est en réalité **nul** dans ce cas précis, pas juste négligeable. La formule $S_1(N,k)=N^r\binom{s}{r}\lambda^k$ est donc exacte (à l'asymptotique en $N$ près, qui elle vient d'ailleurs — de l'approximation de Poisson elle-même, pas de ce $\gamma$).
Les deux approches sont légitimes, mais elles répondent à des questions différentes — et il y a une relation algébrique exacte entre les deux qui vaut la peine d'être explicitée.
**Ce que calcule l'approche naïve.** Pour **un k-mer $w$ déjà observé et fixé**, $p_i\simeq1-\exp(-G_i/4^k)$ est la probabilité que *ce mot précis* apparaisse quelque part dans le génome $i$ par hasard, et $P(R\geq r)$ (Poisson-binomiale ou binomiale) est la probabilité que *ce mot précis* soit partagé par au moins $r$ des $S$ génomes par hasard. C'est une **p-value par observation** : « j'ai vu ce k-mer particulier dans 5 génomes sur 10, est-ce surprenant pour lui ? »
**Ce que calcule Karlin-Ost.** $S_1(k)$ et $P(K_{r,s}\geq k)$ répondent à : « parmi **tous les $4^k$ mots possibles**, quelle est la probabilité qu'au moins un seul d'entre eux atteigne ce seuil de partage par hasard, n'importe où dans les génomes ? » C'est une question **globale sur tout l'espace des k-mers**, pas sur un mot déjà identifié.
**La relation exacte entre les deux.** Reprenons le cas $r=s=2$, génomes égaux de taille $G$ :
- Naïf (mot $w$ fixé) : $P(R=2\mid w)\simeq p^2=(G\cdot4^{-k})^2=G^2\cdot4^{-2k}$
- Karlin-Ost : $S_1=G^2\cdot4^{-k}$
Le rapport est exactement $S_1 / P(R=2\mid w) = 4^{k}$ — c'est-à-dire le nombre de mots possibles. Ce n'est pas une coïncidence : $S_1=N^r\binom{s}{r}\lambda^k$ vient de $\lambda^k=\sum_w \Pr\{w\}^r$, une somme explicite sur les $4^k$ mots, alors que le calcul naïf tel que présenté ne somme sur rien — il traite $w$ comme donné.
**Donc l'approche naïve, formellement, c'est Karlin-Ost auquel il manque le facteur « nombre de k-mers testés ».** Pour la rendre comparable à la question « dois-je m'attendre à *une* collision fortuite n'importe où dans mon jeu de données », il faudrait une correction à la Bonferroni : multiplier $P(R\geq r\mid w)$ par le nombre effectif de k-mers canoniques testables ($\approx4^k/2$). Karlin-Ost fait cette correction analytiquement et correctement (avec en prime le facteur de déclumping $(1-\lambda)$ qu'on a discuté, qui corrige le sur-comptage des matches chevauchants — une union bound brute comme le Bonferroni décrit ci-dessus le surestimerait légèrement).
**Ce qui veut dire, en pratique, que les deux outils ne sont pas concurrents mais complémentaires :**
- **Choix a priori de $k$ / FPR global du dataset** (la question de `kmers.md` depuis le début de cette conversation) → Karlin-Ost, parce qu'il intègre nativement la multiplicité sur tous les mots possibles.
- **Score de significativité a posteriori pour un k-mer effectivement observé dans l'index** (« ce hit précis, présent dans $r$ génomes sur $S$, est-il compatible avec le hasard ? ») → l'approche Poisson-binomiale est exactement le bon outil, et même préférable à Karlin-Ost pour cet usage : elle gère nativement des $G_i$ inégaux sans les subtilités de convergence du théorème 2.2, et elle donne directement une p-value par hit exploitable pour filtrer les résultats d'un index présence/absence — ce qui correspond bien à l'usage qu'a en tête ton correspondant.
Une seule réserve sur le texte proposé tel quel : $p_i\simeq1-\exp(-G_i/4^k)$ souffre du même biais que celui identifié plus tôt dans notre discussion — un mot de longueur $k$ n'est pas indépendant à chaque position (chevauchement), donc ce $p_i$ est un peu optimiste (légèrement trop haut) pour les mêmes raisons que $E[C]$ l'était. Négligeable pour $k\geq21$, plus discutable à $k=11$.
## FDR
C'est exactement la bonne question à se poser — ni la p-value par hit, ni la probabilité globale « y a-t-il au moins une collision quelque part », mais **la fraction de tes appels d'homologie qui sont du bruit**. C'est un problème de type FDR (*false discovery rate*), et il se construit directement à partir des deux briques qu'on a déjà :
$$
\widehat{FDR}(r) = \frac{\text{nombre de k-mers *attendus par hasard* avec }R\geq r}{\text{nombre de k-mers *observés* avec }R\geq r}
$$
**Le numérateur** (bruit attendu) : c'est la généralisation, au seuil $\geq r$ (pas seulement « partagé par exactement ces $r$ génomes »), de la quantité pont qu'on vient d'identifier — le nombre de k-mers testés $M$ multiplié par la probabilité par-mot sous le null :
$$
E[\text{faux positifs}] \simeq M \cdot P_0(R\geq r) = M\sum_{j=r}^{S}\binom{S}{j}p^{j}(1-p)^{S-j}
$$
avec $p\simeq G/4^k$ (ou la version Poisson-binomiale à $G_i$ inégaux) et $M$ le nombre de k-mers canoniques réellement testés (pas $4^k/2$ théorique, mais le nombre de k-mers **effectivement présents** dans ton index — c'est cette correction, plutôt que le $4^k$ théorique, qui rend l'estimation réaliste, puisque la plupart des $4^k$ mots possibles n'apparaissent jamais dans de vrais génomes).
**Le dénominateur** (bruit + vrai signal, observé) : le nombre de k-mers de ton index dont le masque de présence a une cardinalité $\geq r$ — une donnée que `FamilyMask`/`SiblingAnnex` dans `obikphylo::siblings` te donne déjà directement, sans aucun calcul supplémentaire.
**Ce que ça donne concrètement.** Tu n'as besoin que de trois nombres, tous déjà disponibles ou calculables :
1. $M$ = cardinalité totale de ton set de k-mers indexés (mesure directe de l'index)
2. $N_{obs}(r)$ = nombre de familles/k-mers avec présence $\geq r$ (mesure directe, déjà dans `FamilyMask`)
3. $P_0(R\geq r)$ = calculé depuis le modèle nul (le $p$ de l'approche « naïve » du tour précédent)
et $\widehat{FDR}(r) = M\cdot P_0(R\geq r) / N_{obs}(r)$ te dit directement : « sur les k-mers que j'ai retenus comme preuve d'homologie au seuil $r$, telle fraction (%) est attendue être du pur hasard ».
**Pourquoi c'est mieux que ce qu'on avait avant** :
- Le $P(K_{r,s}\geq k)$ de Karlin-Ost (tour précédent) répond à « y a-t-il *au moins un* faux positif quelque part » — binaire, ne dit rien sur combien.
- La p-value par hit répond à « ce hit précis est-il suspect » — mot par mot, ne donne pas de vue d'ensemble.
- Le $\widehat{FDR}$ répond directement à ta question : **quelle proportion** de l'ensemble retenu est due au hasard — c'est la quantité qu'on utiliserait pour dire « à ce seuil $r$, mon jeu de zones d'homologie candidates est fiable à 95 % » ou pour choisir le $r$ minimal qui ramène le $\widehat{FDR}$ sous un seuil acceptable, à $k$ fixé par la contrainte de représentation (64 bits).
Un point à trancher avec toi avant d'aller plus loin : pour $M$, préfères-tu le nombre théorique de k-mers canoniques possibles ($4^k/2$) ou le nombre réel de k-mers distincts dans ton index (qui donnerait un $\widehat{FDR}$ beaucoup plus conservateur et réaliste, mais nécessite de le lire depuis l'index au moment du calcul plutôt que de le dériver de $k$ seul) ?
@@ -0,0 +1,6 @@
{
"cells": [],
"metadata": {},
"nbformat": 4,
"nbformat_minor": 5
}
+126
View File
@@ -0,0 +1,126 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "12264929-a50f-435a-a893-070b3f9573d9",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(G, k)"
]
},
"execution_count": 1,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"var('G k')"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "30d5efd3-8bdc-4249-ad9d-abf7cad3481a",
"metadata": {},
"outputs": [],
"source": [
"q = 1 / 4^k"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "588770fe-ddf2-49c4-9f1f-df177543c69c",
"metadata": {},
"outputs": [],
"source": [
"P0 = (1-q)^G"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "4d52ce3d-88e5-4c10-ae8e-dba268185f7e",
"metadata": {},
"outputs": [],
"source": [
"P1 = 1 - P0"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "91c8cba6-bf05-4694-9598-eae9663e5c82",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"-(-1/4^k + 1)^G + 1"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"P1"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "c29cfeae-9a3c-483e-a007-81df52eef62d",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"Lambda"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"var('Lambda')"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "53028547-57d4-4375-8a26-8ae92309a3c0",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "SageMath 10.9",
"language": "sage",
"name": "sagemath-10.9"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.14.3"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
+1 -1
View File
@@ -9,7 +9,7 @@ plugins:
- mermaid2:
- bibtex:
bib_file: UserDocMD/references.bib
csl_file: UserDocMD/ecology-letters.csl
csl_file: UserDocMD/molecular-ecology-modified.csl
enable_inline_citations: false
markdown_extensions:
+11
View File
@@ -98,6 +98,15 @@ version = "1.0.104"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
[[package]]
name = "approx"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6"
dependencies = [
"num-traits",
]
[[package]]
name = "arbitrary-chunks"
version = "0.4.1"
@@ -1665,8 +1674,10 @@ dependencies = [
name = "obiksession"
version = "0.1.0"
dependencies = [
"approx",
"memmap2",
"obisys",
"serde",
"tempfile",
"tracing",
"xxhash-rust",
+331
View File
@@ -0,0 +1,331 @@
//! Chance-collision statistics for exact k-mer matches shared by `r` of `s`
//! independent random sequences, after Karlin & Ost's asymptotic theory of
//! the longest common word among random letter sequences.
//!
//! # The question this answers
//!
//! Given `s` sequences of length `N` (e.g. `s` genomes), each drawn from an
//! i.i.d. equiprobable `m`-letter alphabet (`m = 4` for DNA), what is the
//! probability that a k-mer of length `k` is shared, purely by chance, by
//! at least `r` of those `s` sequences at unrelated, unaligned positions?
//! This is the null-model baseline against which an *observed* shared
//! k-mer (e.g. a [`crate::siblings`] family mask spanning `r` layers) can be
//! judged significant rather than a statistical accident.
//!
//! # Why not just `G / 4^k`
//!
//! The naive estimate used elsewhere for a single pair of sequences,
//! `P_chance(k, G) ~= G / 4^k` (see `UserDocMD/theory/kmer_indexing/kmers.md`),
//! treats every pair of positions as an independent trial. It ignores that
//! overlapping candidate matches (a match of length `L` contains a match of
//! every shorter length at the same start) are not independent events, and
//! it has no notion of "shared by `r` of `s` sequences" at all — only
//! pairwise. Karlin & Ost's asymptotic theory answers exactly this
//! generalized question, with a rigorous de-clumping correction instead of
//! the naive independence assumption.
//!
//! # Derivation
//!
//! For `s` independent i.i.d. equiprobable sequences over an alphabet of
//! size `m`, define the characteristic local match parameter for an
//! `r`-fold match (Karlin & Ost 1988, eq. 2.5-2.6):
//!
//! ```text
//! lambda^[r] = m^(1-r)
//! ```
//!
//! (`r = 2`: a plain pairwise match, `lambda = 1/m`; larger `r` shrinks
//! `lambda` geometrically, since all `r` sequences must agree on the same
//! word simultaneously).
//!
//! The expected number of `r`-fold common k-words among the `C(s, r)`
//! relevant groups of sequences is (eq. 2.9, with the i.i.d. equiprobable
//! simplification `gamma = 1` derived below):
//!
//! ```text
//! S_1(N, k) = N^r * C(s, r) * lambda^k
//! ```
//!
//! `gamma` (eq. 2.8) is defined as `lim_{k->oo} F(k) / lambda^k`, where
//! `F(k) = sum_w Pr{w}^r` over all `m^k` possible words `w`. For the i.i.d.
//! equiprobable case, `Pr{w} = m^-k` for every word, so `F(k) = m^k *
//! m^(-k*r) = m^(k*(1-r)) = lambda^k` **exactly**, for every `k` — not just
//! asymptotically. Hence `gamma = 1` exactly and the `(1 + o(1))` error term
//! in the general formula vanishes identically in this case; no correction
//! is dropped.
//!
//! Theorem 2.2 (eq. 2.12) gives the limiting distribution of the longest
//! `r`-fold common word length `K_{r,s}(N)` as `N -> oo`:
//!
//! ```text
//! Pr{K_{r,s}(N) < k} ~= exp{ -(1 - lambda) * S_1(N, k) }
//! ```
//!
//! so the probability of at least one chance `r`-fold match of length `k`
//! is
//!
//! ```text
//! P(k) = Pr{K_{r,s}(N) >= k} ~= 1 - exp{ -(1 - lambda) * S_1(N, k) }
//! ```
//!
//! The `(1 - lambda)` factor is the de-clumping correction: it accounts for
//! the fact that a single long chance match of length `L` is otherwise
//! counted once for every `k <= L` it contains, which would overstate the
//! rate of genuinely distinct matches.
//!
//! # Unequal genome lengths
//!
//! The `N^r` factor in `S_1(N, k)` is a shortcut for a more basic fact: for
//! one fixed group of `r` sequences, the number of candidate position
//! combinations is the *product* of that group's individual lengths (one
//! independent position choice per sequence in the group), not `N^r` — the
//! latter only holds because all `s` sequences share the same length `N`.
//! Dropping that assumption and summing over all `C(s, r)` groups of `s`
//! sequences with individual lengths `N_1, ..., N_s` gives
//!
//! ```text
//! S_1(k) = lambda^k * e_r(N_1, ..., N_s)
//! ```
//!
//! where `e_r` is the degree-`r` [elementary symmetric polynomial] in the
//! `s` lengths, `e_r(N_1,...,N_s) = sum_{i_1 < ... < i_r} N_i1 * N_i2 * ...
//! * N_ir`. Setting all `N_i = N` collapses `e_r` to `C(s, r) * N^r`,
//! recovering the equal-length formula above exactly — this is a strict
//! generalization, not a different approximation.
//!
//! [elementary symmetric polynomial]: https://en.wikipedia.org/wiki/Elementary_symmetric_polynomial
//!
//! The Remark following Theorem 2.2 in Karlin & Ost (1988) notes that for
//! the *limiting distribution* itself (not just its `S_1` parameter) to
//! converge, the `N_i` must stay of comparable order as they all grow
//! (formally, `(N_{sigma_1} * ... * N_{sigma_r}) / N^r` must converge to
//! some `alpha(sigma)` for a common reference scale `N`) — in practice this
//! is a non-issue unless genome sizes differ by several orders of
//! magnitude.
//!
//! `N_i` here is the sequence length proper; as a first approximation it
//! can be taken as the genome's k-mer set cardinality (or total k-mer
//! occurrence count), which is already what OBIKmer computes per genome
//! and tracks the true sequence length closely enough for this purpose.
//!
//! # Validity
//!
//! - The `exp{-(1-lambda) S_1}` form (Poisson approximation) is valid
//! whenever the per-position match probability `lambda^k` is small, which
//! holds throughout OBIKmer's supported range (`k in [11, 31]`) — it does
//! **not** require `S_1` itself to be small.
//! - The further linearisation `1 - exp(-x) ~= x` used by callers that want
//! `P(k) ~= (1 - lambda) * S_1(N, k)` **does** require `(1-lambda) *
//! S_1(N, k) << 1`; callers must check this themselves (or just use
//! [`p_at_least_one`], which always uses the exponential form and is
//! valid regardless of scale).
//! - This module only covers the *unrestricted-position* case (`K_{r,s}`,
//! matches anywhere in each sequence, not aligned at a common start) and
//! *distinct* sequences (`s` independent realizations). The single
//! sequence self-repeat case (a genome compared to itself, positions
//! drawn from the same realization) needs the reverse-complement variant
//! `R(N)` treated separately in Karlin & Ost (1987); it is not
//! implemented here.
//!
//! # References
//!
//! - Karlin, S. and Ost, F. (1988). "Maximal length of common words among
//! random letter sequences." *The Annals of Probability*, 16(2):535-563.
//! [`Theorem 2.2`] — the unrestricted-position `K_{r,s}(N)` result
//! implemented here. DOI: 10.1214/aop/1176991772.
//! - Karlin, S. and Ost, F. (1987). "Counts of long aligned word matches
//! among random letter sequences." *Advances in Applied Probability*,
//! 19(2):293-351. Companion paper for the *aligned* variant `C_{r,s}(N)`
//! (fixed common start position across sequences, relevant to local
//! alignment significance, not to unrestricted repeat-finding) — not
//! implemented here.
//! - Karlin, S. and Ost, F. (1985). "Maximal segmental match length among
//! random sequences from a finite alphabet." In *Proceedings of the
//! Berkeley Conference in Honor of Jerzy Neyman and Jack Kiefer*, vol. 1,
//! pp. 225-243. Wadsworth. No DOI found (pre-DOI conference proceedings).
//! Evaluates the reverse-complement match parameter needed for the
//! self-repeat case `R(N)` mentioned above.
/// DNA alphabet size (`A`, `C`, `G`, `T`), assumed equiprobable and
/// independent — the same null-model assumption used throughout
/// `UserDocMD/theory/kmer_indexing/kmers.md`.
const DNA_ALPHABET_SIZE: f64 = 4.0;
/// `C(n, k)`, the binomial coefficient, computed as `f64` since `S_1` is
/// itself only ever an asymptotic approximation — exact integer precision
/// is not meaningful here.
fn binomial(n: u32, k: u32) -> f64 {
if k > n {
return 0.0;
}
let k = k.min(n - k);
let mut result = 1.0f64;
for i in 0..k {
result *= (n - i) as f64 / (i + 1) as f64;
}
result
}
/// The Karlin-Ost characteristic local match parameter `lambda^[r] = m^(1-r)`
/// for `r` i.i.d. equiprobable sequences over an alphabet of size `m`
/// (eq. 2.5-2.6 of Karlin & Ost 1988).
///
/// `alphabet_size` defaults to [`DNA_ALPHABET_SIZE`] for the DNA case via
/// [`dna_lambda`]; this general form is kept for documentation fidelity to
/// the source formula and for non-DNA alphabets (e.g. amino acids).
fn lambda(alphabet_size: f64, r: u32) -> f64 {
alphabet_size.powi(1 - r as i32)
}
/// [`lambda`] specialized to the DNA alphabet (`m = 4`).
pub fn dna_lambda(r: u32) -> f64 {
lambda(DNA_ALPHABET_SIZE, r)
}
/// Degree-`r` elementary symmetric polynomial `e_r(lengths)` — see module
/// docs, "Unequal genome lengths". Computed by the standard `O(r * s)` DP:
/// `e[j]` accumulates the sum over all `j`-subsets of the lengths seen so
/// far, updated one length at a time from high `j` to low so each length is
/// used at most once per subset.
///
/// Returns `0.0` if `r > lengths.len()`.
fn elementary_symmetric(lengths: &[f64], r: u32) -> f64 {
let r = r as usize;
if r > lengths.len() {
return 0.0;
}
let mut e = vec![0.0f64; r + 1];
e[0] = 1.0;
for &length in lengths {
for j in (1..=r).rev() {
e[j] += length * e[j - 1];
}
}
e[r]
}
/// Expected number of chance `r`-fold common k-words among `C(s, r)` groups
/// of `s` independent i.i.d. equiprobable DNA sequences of individual
/// lengths `lengths` (Karlin & Ost 1988, eq. 2.9, generalized to unequal
/// lengths via the degree-`r` elementary symmetric polynomial — see module
/// docs, "Unequal genome lengths" — with the i.i.d. `gamma = 1`
/// simplification derived in the module docs).
///
/// `lengths` can be exact sequence lengths, or, as a first approximation,
/// each genome's k-mer set cardinality (or total k-mer occurrence count).
///
/// # Panics
///
/// Does not panic; returns `0.0` if `r > lengths.len()`.
pub fn expected_shared_kmers_varying(lengths: &[f64], k: u32, r: u32) -> f64 {
elementary_symmetric(lengths, r) * dna_lambda(r).powi(k as i32)
}
/// Expected number of chance `r`-fold common k-words among `C(s, r)` groups
/// of `s` independent i.i.d. equiprobable DNA sequences of length `n`
/// (Karlin & Ost 1988, eq. 2.9, with the i.i.d. `gamma = 1` simplification
/// derived in the module docs).
///
/// `r = 2, s = 2` recovers the plain pairwise chance-collision count for
/// two sequences; `r = s` recovers Karlin-Ost's within-a-single-genome
/// growth rate scaled by the number of independent sequences considered.
///
/// Closed-form fast path for the equal-length case — equivalent to
/// [`expected_shared_kmers_varying`] called with `s` copies of `n`, since
/// `e_r(n,...,n) = C(s, r) * n^r` (see module docs, "Unequal genome
/// lengths").
///
/// # Panics
///
/// Does not panic; returns `0.0` if `r > s`.
pub fn expected_shared_kmers(n: f64, k: u32, r: u32, s: u32) -> f64 {
n.powi(r as i32) * binomial(s, r) * dna_lambda(r).powi(k as i32)
}
/// Probability that at least one k-mer of length `k` is shared, purely by
/// chance, by at least `r` of `s` independent i.i.d. equiprobable DNA
/// sequences of individual lengths `lengths` (Karlin & Ost 1988, Theorem
/// 2.2 / eq. 2.12, exponential form, generalized to unequal lengths).
///
/// Valid across the full range relevant to OBIKmer (`k` in `[11, 31]`)
/// regardless of whether [`expected_shared_kmers_varying`] itself is small
/// or large — unlike the linear approximation `P ~= (1 - lambda) *
/// expected_shared_kmers_varying(..)`, which additionally requires the
/// latter to be `<< 1` (see module docs, "Validity").
pub fn p_at_least_one_varying(lengths: &[f64], k: u32, r: u32) -> f64 {
let s1 = expected_shared_kmers_varying(lengths, k, r);
let lambda = dna_lambda(r);
-f64::exp_m1(-(1.0 - lambda) * s1)
}
/// Probability that at least one k-mer of length `k` is shared, purely by
/// chance, by at least `r` of `s` independent i.i.d. equiprobable DNA
/// sequences of length `n` (Karlin & Ost 1988, Theorem 2.2 / eq. 2.12,
/// exponential form).
///
/// Valid across the full range relevant to OBIKmer (`k` in `[11, 31]`)
/// regardless of whether [`expected_shared_kmers`] itself is small or
/// large — unlike the linear approximation `P ~= (1 - lambda) *
/// expected_shared_kmers(..)`, which additionally requires the latter to
/// be `<< 1` (see module docs, "Validity").
pub fn p_at_least_one(n: f64, k: u32, r: u32, s: u32) -> f64 {
let s1 = expected_shared_kmers(n, k, r, s);
let lambda = dna_lambda(r);
-f64::exp_m1(-(1.0 - lambda) * s1)
}
#[cfg(test)]
mod tests {
use super::*;
/// Reproduces the r=2, s=2, N=1Gb, k=31 example worked out by hand:
/// S_1 ~= 0.217, P ~= 0.150.
#[test]
fn two_genomes_one_gb_k31() {
let s1 = expected_shared_kmers(1e9, 31, 2, 2);
assert!((s1 - 0.217).abs() < 0.01, "S_1 = {s1}");
let p = p_at_least_one(1e9, 31, 2, 2);
assert!((p - 0.150).abs() < 0.01, "P = {p}");
}
/// Reproduces the r=5, s=10, N=10Gb, k=31 example: S_1 ~= 5.6e-23.
#[test]
fn five_of_ten_genomes_ten_gb_k31() {
let s1 = expected_shared_kmers(1e10, 31, 5, 10);
assert!(s1 < 1e-20, "S_1 = {s1}");
assert!(s1 > 1e-25, "S_1 = {s1}");
}
/// Reproduces the r=3, s=10, N=10Gb, k=31 example: S_1 ~= 5.6e-6.
#[test]
fn three_of_ten_genomes_ten_gb_k31() {
let s1 = expected_shared_kmers(1e10, 31, 3, 10);
assert!((s1 - 5.6e-6).abs() < 1e-6, "S_1 = {s1}");
}
/// `expected_shared_kmers_varying` with `s` equal lengths must agree
/// with the closed-form `expected_shared_kmers`, since `e_r(n,...,n) =
/// C(s, r) * n^r`.
#[test]
fn varying_matches_closed_form_when_lengths_equal() {
let lengths = vec![1e10; 10];
let general = expected_shared_kmers_varying(&lengths, 31, 3);
let closed_form = expected_shared_kmers(1e10, 31, 3, 10);
assert!(
(general - closed_form).abs() / closed_form < 1e-9,
"general = {general}, closed_form = {closed_form}"
);
}
/// `e_2(a, b) = a * b` for two sequences — direct check of the
/// unequal-length case against hand computation, using an 8 Gb and a
/// 12 Gb genome at k=21.
#[test]
fn two_genomes_unequal_lengths_k21() {
let lengths = [8e9, 12e9];
let s1 = expected_shared_kmers_varying(&lengths, 21, 2);
let expected = 8e9 * 12e9 * dna_lambda(2).powi(21);
assert!((s1 - expected).abs() / expected < 1e-9, "S_1 = {s1}");
}
}
+8
View File
@@ -7,6 +7,13 @@
//! cardinality, pseudo-alignment); further phylo-domain functionality
//! (currently [`distance`]) moves here incrementally.
//!
//! [`karlin`] holds standalone chance-collision statistics (Karlin & Ost)
//! for judging whether a k-mer shared by several genomes/layers is
//! statistically significant; kept here for now as it has no other
//! consumer yet, but it depends on nothing else in this crate and may move
//! to a lower-level crate if an indexing-side consumer (e.g. default-`k`
//! guidance) appears.
//!
//! `siblings` is being reconnected piece by piece while `obikindex`'s
//! stateless audit is caught up with — see the project memory on this. So
//! far it only carries the annex-construction path
@@ -15,6 +22,7 @@
//! still disconnected.
mod distance;
pub mod karlin;
mod tree;
pub mod siblings;
+2
View File
@@ -8,6 +8,8 @@ obisys = { path = "../obisys", default-features = false }
memmap2 = "0.9.11"
xxhash-rust = { version = "0.8.18", features = ["xxh3"] }
tracing = "0.1"
serde = { version = "1", optional = true }
[dev-dependencies]
approx = "0.5.1"
tempfile = "3"
+15
View File
@@ -13,7 +13,22 @@
//! already used elsewhere in this project between a generic mechanism
//! crate and the domain crate that plugs into it (`obisys`/`obikindex`,
//! `obicompactvec`/`obikindexer`).
//!
//! ## Memoization
//!
//! [`Session::memoize`] returns a [`Memoize`] handle that lets callers
//! memoize expensive computations on top of the same checksummed,
//! persist-on-disk artifact store. Two API styles are provided:
//!
//! * **Manual** — [`Memoize::get_or_compute_with_key`] with an explicit
//! string key, or [`Memoize::get_or_compute`] with auto-derived keys
//! via the [`MemoKey`] trait.
//! * **Declarative** — the `session_memoize!` macro wraps a function so
//! that recursive calls transparently consult the cache, similar to
//! `kash`/`cached`.
mod memoize;
mod session;
pub use memoize::{MemoKey, Memoize, MemoizeValue};
pub use session::{ParamsConflict, Session, SessionOutcome};
+511
View File
@@ -0,0 +1,511 @@
use std::fmt;
use std::io;
use std::path::PathBuf;
use memmap2::Mmap;
use crate::Session;
use crate::session::{MEMO_DIR, checksum_path_for, verify_mmap, write_checked};
/// Trait for types that can be used as memoization keys.
///
/// The default implementation uses `Debug` + `xxh3` to produce a
/// stable string key from any `fmt::Debug` type. Callers can
/// override this to supply a custom key (e.g. a pre-hashed value
/// or a semantic key).
pub trait MemoKey: fmt::Debug {
fn memo_key(&self, function_name: &str) -> String;
}
impl<T: fmt::Debug> MemoKey for T {
fn memo_key(&self, function_name: &str) -> String {
let payload = format!("{}:{:?}", function_name, self);
xxhash_rust::xxh3::xxh3_64(payload.as_bytes()).to_string()
}
}
/// Marker trait for types that can be stored in the memoization
/// cache. The default blanket impl covers all `Vec<u8>` values.
pub trait MemoizeValue: Sized {
fn to_bytes(&self) -> Vec<u8>;
fn from_bytes(bytes: &[u8]) -> io::Result<Self>;
}
impl MemoizeValue for Vec<u8> {
fn to_bytes(&self) -> Vec<u8> {
self.clone()
}
fn from_bytes(bytes: &[u8]) -> io::Result<Self> {
Ok(bytes.to_vec())
}
}
// Blanket impl for serde types when the serde feature is enabled
#[cfg(feature = "serde")]
impl<T> MemoizeValue for T
where
T: serde::Serialize + serde::de::DeserializeOwned,
{
fn to_bytes(&self) -> Vec<u8> {
bincode::serialize(self).expect("serialization failed")
}
fn from_bytes(bytes: &[u8]) -> io::Result<Self> {
bincode::deserialize(bytes).map_err(io::Error::other)
}
}
// Blanket impls for common primitive types (no serde needed)
impl MemoizeValue for u8 {
fn to_bytes(&self) -> Vec<u8> {
vec![*self]
}
fn from_bytes(bytes: &[u8]) -> io::Result<Self> {
bytes.first().copied().ok_or_else(|| {
io::Error::new(io::ErrorKind::InvalidData, "invalid u8 bytes")
})
}
}
impl MemoizeValue for u16 {
fn to_bytes(&self) -> Vec<u8> {
self.to_le_bytes().to_vec()
}
fn from_bytes(bytes: &[u8]) -> io::Result<Self> {
let arr = <[u8; 2]>::try_from(bytes)
.map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid u16 bytes"))?;
Ok(u16::from_le_bytes(arr))
}
}
impl MemoizeValue for u32 {
fn to_bytes(&self) -> Vec<u8> {
self.to_le_bytes().to_vec()
}
fn from_bytes(bytes: &[u8]) -> io::Result<Self> {
let arr = <[u8; 4]>::try_from(bytes)
.map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid u32 bytes"))?;
Ok(u32::from_le_bytes(arr))
}
}
impl MemoizeValue for u64 {
fn to_bytes(&self) -> Vec<u8> {
self.to_le_bytes().to_vec()
}
fn from_bytes(bytes: &[u8]) -> io::Result<Self> {
let arr = <[u8; 8]>::try_from(bytes)
.map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid u64 bytes"))?;
Ok(u64::from_le_bytes(arr))
}
}
impl MemoizeValue for i32 {
fn to_bytes(&self) -> Vec<u8> {
self.to_le_bytes().to_vec()
}
fn from_bytes(bytes: &[u8]) -> io::Result<Self> {
let arr = <[u8; 4]>::try_from(bytes)
.map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid i32 bytes"))?;
Ok(i32::from_le_bytes(arr))
}
}
impl MemoizeValue for i64 {
fn to_bytes(&self) -> Vec<u8> {
self.to_le_bytes().to_vec()
}
fn from_bytes(bytes: &[u8]) -> io::Result<Self> {
let arr = <[u8; 8]>::try_from(bytes)
.map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid i64 bytes"))?;
Ok(i64::from_le_bytes(arr))
}
}
impl MemoizeValue for f64 {
fn to_bytes(&self) -> Vec<u8> {
self.to_le_bytes().to_vec()
}
fn from_bytes(bytes: &[u8]) -> io::Result<Self> {
let arr = <[u8; 8]>::try_from(bytes)
.map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid f64 bytes"))?;
Ok(f64::from_le_bytes(arr))
}
}
/// A memoization handle backed by a `Session` directory.
#[derive(Debug)]
pub struct Memoize<'a> {
session: &'a Session,
}
impl<'a> Memoize<'a> {
pub(crate) fn new(session: &'a Session) -> Self {
Self { session }
}
/// Retrieve a cached value by its explicit key, or compute and
/// store it via `compute_fn`.
///
/// Concurrent calls for the *same* `key` (from different threads
/// sharing this session) don't race to recompute: the first caller
/// runs `compute_fn` while the others block on
/// [`Session::with_key_lock`], then each re-checks the cache — which
/// the first caller has by then populated — instead of recomputing.
pub fn get_or_compute_with_key(
&self,
key: &str,
compute_fn: impl FnOnce() -> io::Result<Vec<u8>>,
) -> io::Result<Vec<u8>> {
if let Some(cached) = self.restore_raw(key)? {
return Ok(cached.to_vec());
}
self.session.with_key_lock(key, || {
if let Some(cached) = self.restore_raw(key)? {
return Ok(cached.to_vec());
}
let value = compute_fn()?;
self.store_raw(key, &value)?;
Ok(value)
})
}
/// Retrieve a cached value whose key is auto-derived from
/// `args` via [`MemoKey`], or compute and store it.
pub fn get_or_compute<K, V, F>(
&self,
function_name: &str,
args: &K,
compute_fn: F,
) -> io::Result<Vec<u8>>
where
K: MemoKey,
V: MemoizeValue,
F: FnOnce() -> io::Result<V>,
{
let key = args.memo_key(function_name);
let raw = self.get_or_compute_with_key(&key, || {
compute_fn().map(|v| v.to_bytes())
})?;
Ok(raw)
}
/// Restore a raw (byte) artifact by name, `mmap`ed read-only.
fn restore_raw(&self, name: &str) -> io::Result<Option<Mmap>> {
let path = self.memo_path(name);
verify_mmap(&path)
}
/// Store raw bytes under `name`, atomically with a checksum sidecar.
fn store_raw(&self, name: &str, bytes: &[u8]) -> io::Result<()> {
let path = self.memo_path(name);
write_checked(&path, bytes)
}
/// Remove a single memoized entry by key. Returns `true` if the
/// entry existed and was deleted.
pub fn invalidate(&self, key: &str) -> io::Result<bool> {
let path = self.memo_path(key);
let checksum_path = checksum_path_for(&path);
let existed = path.exists();
if path.exists() {
std::fs::remove_file(&path)?;
}
if checksum_path.exists() {
let _ = std::fs::remove_file(&checksum_path);
}
Ok(existed)
}
pub fn cached_keys(&self) -> io::Result<Vec<String>> {
let dir = self.memo_dir();
if !dir.exists() {
return Ok(Vec::new());
}
let mut keys = Vec::new();
for entry in std::fs::read_dir(dir)? {
let entry = entry?;
let name = entry.file_name();
let name = name.to_string_lossy().to_string();
if name.ends_with(".xxh3") || name.ends_with(".tmp") {
continue;
}
keys.push(name);
}
Ok(keys)
}
fn memo_dir(&self) -> PathBuf {
self.session.dir().join(MEMO_DIR)
}
fn memo_path(&self, name: &str) -> PathBuf {
self.memo_dir().join(name)
}
}
/// Declarative memoization macro, similar in spirit to `kash`/`cached`.
///
/// Wraps a function so that every call transparently consults the
/// session's persistent cache. On a cache miss the function body
/// is executed and its return value is stored; on a hit the stored
/// value is returned without re-executing the body.
///
/// # Syntax
///
/// ```ignore
/// session_memoize! {
/// fn my_func(session: &Session, a: u64, b: &str) -> MyResult {
/// // expensive computation
/// }
/// }
/// ```
///
/// After expansion, call `my_func(&session, 5, "hello")` — the function
/// returns `io::Result<MyResult>`. Check the result with `.unwrap()` or
/// `.expect()`.
///
/// Recursive calls within the body to `my_func` also pass through the
/// cache (they must also include the session as the first argument).
///
/// # Requirements
///
/// * The first parameter must be `session: &Session`.
/// * All argument types must implement [`MemoKey`] (provided for `Debug` types).
/// * The return type must implement [`MemoizeValue`] (provided for
/// `Vec<u8>`, primitive numeric types, and, when the `serde`
/// feature is enabled, for `Serialize + DeserializeOwned` types).
#[macro_export]
macro_rules! session_memoize {
(
fn $name:ident(session: &Session, $($arg:ident : $arg_ty:ty),*) -> $ret:ty $body:block
) => {
fn $name(session: &Session, $($arg: $arg_ty),*) -> ::std::io::Result<$ret> {
let memo = session.memoize();
let args = (&($($arg),*),);
let raw = memo.get_or_compute(
stringify!($name),
&args,
|| -> ::std::io::Result<$ret> {
let __result: $ret = $body;
Ok(__result)
},
)?;
<$ret as $crate::MemoizeValue>::from_bytes(&raw)
.map_err(|e| ::std::io::Error::new(::std::io::ErrorKind::Other, e))
}
};
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn memoize_get_or_compute_miss_then_hit() {
let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path().join("session");
let session = Session::open(&dir, b"params", false).unwrap().unwrap();
let memo = Memoize::new(&session);
let computed = memo.get_or_compute_with_key("key1", || Ok(b"value1".to_vec())).unwrap();
assert_eq!(&computed[..], b"value1");
let cached = memo.get_or_compute_with_key("key1", || panic!("should not be called")).unwrap();
assert_eq!(&cached[..], b"value1");
}
#[test]
fn memoize_invalidate() {
let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path().join("session");
let session = Session::open(&dir, b"params", false).unwrap().unwrap();
let memo = Memoize::new(&session);
memo.get_or_compute_with_key("key1", || Ok(b"value1".to_vec())).unwrap();
assert!(memo.invalidate("key1").unwrap());
assert!(memo.restore_raw("key1").unwrap().is_none());
assert!(!memo.invalidate("key1").unwrap());
}
#[test]
fn memoize_cached_keys() {
let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path().join("session");
let session = Session::open(&dir, b"params", false).unwrap().unwrap();
let memo = Memoize::new(&session);
assert!(memo.cached_keys().unwrap().is_empty());
memo.get_or_compute_with_key("a", || Ok(vec![1])).unwrap();
memo.get_or_compute_with_key("b", || Ok(vec![2])).unwrap();
let keys = memo.cached_keys().unwrap();
assert!(keys.contains(&"a".to_string()));
assert!(keys.contains(&"b".to_string()));
assert_eq!(keys.len(), 2);
}
#[test]
fn memo_key_from_debug() {
#[derive(Debug)]
struct Args {
x: u64,
y: String,
}
let args = Args { x: 42, y: "hello".into() };
let key1 = args.memo_key("my_fn");
let key2 = args.memo_key("my_fn");
assert_eq!(key1, key2);
let key_other = args.memo_key("other_fn");
assert_ne!(key1, key_other);
}
#[test]
fn session_memoize_macro_u64() {
let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path().join("session");
let session = Session::open(&dir, b"params", false).unwrap().unwrap();
session_memoize! {
fn double(session: &Session, x: u64) -> u64 {
x * 2
}
}
let result = double(&session, 7).unwrap();
assert_eq!(result, 14);
let result2 = double(&session, 7).unwrap();
assert_eq!(result2, 14);
}
#[test]
fn session_memoize_macro_i32() {
let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path().join("session");
let session = Session::open(&dir, b"params", false).unwrap().unwrap();
session_memoize! {
fn add(session: &Session, a: i32, b: i32) -> i32 {
a + b
}
}
let result = add(&session, 3, 4).unwrap();
assert_eq!(result, 7);
let result2 = add(&session, 3, 4).unwrap();
assert_eq!(result2, 7);
}
#[test]
fn session_memoize_macro_f64() {
let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path().join("session");
let session = Session::open(&dir, b"params", false).unwrap().unwrap();
session_memoize! {
fn square(session: &Session, x: f64) -> f64 {
x * x
}
}
let result = square(&session, 3.0).unwrap();
approx::assert_relative_eq!(result, 9.0);
let result2 = square(&session, 3.0).unwrap();
approx::assert_relative_eq!(result2, 9.0);
}
#[test]
fn session_memoize_macro_persists_across_sessions() {
let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path().join("session");
// Define the memoized function once
session_memoize! {
fn cached_add(session: &Session, a: u64, b: u64) -> u64 {
a + b
}
}
// First session: compute and cache
{
let session = Session::open(&dir, b"params", false).unwrap().unwrap();
assert_eq!(cached_add(&session, 3, 4).unwrap(), 7);
}
// Second session with same params: should hit cache
{
let session = Session::open(&dir, b"params", false).unwrap().unwrap();
assert_eq!(cached_add(&session, 3, 4).unwrap(), 7);
}
}
#[test]
fn session_memoize_macro_invalidated_by_force_reset() {
let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path().join("session");
session_memoize! {
fn tagged(session: &Session, x: u64) -> u64 {
x
}
}
// First session: compute and cache under "params-v1".
{
let session = Session::open(&dir, b"params-v1", false).unwrap().unwrap();
assert_eq!(tagged(&session, 1).unwrap(), 1);
assert_eq!(session.memoize().cached_keys().unwrap().len(), 1);
}
// Reopen with different params and force: true — the whole
// session, including the memo cache, is reset, so the call
// below must recompute rather than reuse the stale entry.
{
let session = Session::open(&dir, b"params-v2", true).unwrap().unwrap();
assert!(session.memoize().cached_keys().unwrap().is_empty());
assert_eq!(tagged(&session, 1).unwrap(), 1);
}
}
#[test]
fn concurrent_calls_for_same_key_compute_once() {
let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path().join("session");
let session = std::sync::Arc::new(Session::open(&dir, b"params", false).unwrap().unwrap());
let calls = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
// Forces both threads to reach `get_or_compute_with_key` at
// essentially the same instant, so the second one is guaranteed
// to hit the "wait, don't recompute" path instead of just
// happening to run after the first one already finished.
let barrier = std::sync::Arc::new(std::sync::Barrier::new(2));
let handles: Vec<_> = (0..2)
.map(|_| {
let session = session.clone();
let calls = calls.clone();
let barrier = barrier.clone();
std::thread::spawn(move || {
barrier.wait();
session
.memoize()
.get_or_compute_with_key("shared-key", || {
calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
std::thread::sleep(std::time::Duration::from_millis(50));
Ok(b"computed".to_vec())
})
.unwrap()
})
})
.collect();
for handle in handles {
assert_eq!(&handle.join().unwrap()[..], b"computed");
}
assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 1);
// The per-key lock must have been cleaned up once both threads
// are done with it, not left behind forever.
assert_eq!(session.memo_lock_table_len(), 0);
}
}
+93 -13
View File
@@ -1,12 +1,15 @@
use std::collections::HashMap;
use std::fs;
use std::io;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
use memmap2::Mmap;
use obisys::DirLock;
const PARAMS_FILE: &str = "params.bin";
const ARTIFACTS_DIR: &str = "artifacts";
pub(crate) const MEMO_DIR: &str = "memo";
const CHECKSUM_EXT: &str = "xxh3";
/// Whether [`Session::open`] found a fresh (just-created, or just-reset by
@@ -48,6 +51,9 @@ pub struct Session {
dir: PathBuf,
outcome: SessionOutcome,
_lock: DirLock,
/// Per-key in-process locks backing [`Memoize`](crate::Memoize)'s
/// single-flight behavior — see [`Session::with_key_lock`].
memo_locks: Mutex<HashMap<String, Arc<Mutex<()>>>>,
}
impl Session {
@@ -66,24 +72,31 @@ impl Session {
dir: dir.to_path_buf(),
outcome: SessionOutcome::Reused,
_lock: lock,
memo_locks: Mutex::new(HashMap::new()),
})),
Some(saved) if !force => Ok(Err(ParamsConflict { saved, requested: params.to_vec() })),
_ => {
// No saved params yet, or a mismatch with `force: true`:
// start clean. Any artifact cached under different params
// is stale by construction — deleting the whole
// `artifacts/` directory is simpler and safer than trying
// to figure out which entries are still valid.
// start clean. Any artifact or memoized entry cached under
// different params is stale by construction — deleting the
// whole `artifacts/` and `memo/` directories is simpler and
// safer than trying to figure out which entries are still
// valid.
let artifacts_dir = dir.join(ARTIFACTS_DIR);
if artifacts_dir.exists() {
fs::remove_dir_all(&artifacts_dir)?;
}
fs::create_dir_all(&artifacts_dir)?;
let memo_dir = dir.join(MEMO_DIR);
if memo_dir.exists() {
fs::remove_dir_all(&memo_dir)?;
}
write_checked(&params_path, params)?;
Ok(Ok(Session {
dir: dir.to_path_buf(),
outcome: SessionOutcome::Created,
_lock: lock,
memo_locks: Mutex::new(HashMap::new()),
}))
}
}
@@ -123,20 +136,87 @@ impl Session {
pub fn store(&self, name: &str, bytes: &[u8]) -> io::Result<()> {
write_checked(&self.artifact_path(name), bytes)
}
pub fn memoize(&self) -> crate::Memoize<'_> {
crate::Memoize::new(self)
}
fn checksum_path_for(path: &Path) -> PathBuf {
/// Serialize concurrent calls that share the same `key`, within this
/// process: the first caller for a given `key` runs `f` while any
/// other caller for that *same* `key` blocks; once `f` returns, each
/// blocked caller runs in turn. Backs [`Memoize`](crate::Memoize)'s
/// single-flight behavior — combined with a cache re-check inside `f`,
/// it turns "N threads race to recompute and overwrite the same
/// artifact" into "one thread computes, the rest wait and then read
/// the cache".
///
/// Purely an in-process mechanism: cross-process exclusivity is
/// already provided by the `DirLock` held for the `Session`'s entire
/// lifetime, so contention here only ever happens between threads of
/// this one process sharing the same `Session`.
///
/// The per-key lock is garbage-collected: its entry is removed as
/// soon as no other thread is (or could still be) waiting on it, so a
/// long-running process doesn't accumulate one entry per distinct key
/// ever requested — only per key currently or recently contended.
pub(crate) fn with_key_lock<T>(&self, key: &str, f: impl FnOnce() -> T) -> T {
let entry = {
let mut table = self.memo_locks.lock().unwrap();
table.entry(key.to_string()).or_insert_with(|| Arc::new(Mutex::new(()))).clone()
};
let guard = entry.lock().unwrap();
let result = f();
// Still holding `guard`: any other thread that already cloned
// `entry` out of the table is necessarily still blocked trying to
// acquire it below — it cannot have run `f` and returned yet — so
// `entry`'s strong count here is exact. `table` holds one
// reference and this call holds the other (`entry` itself); if
// that is still all of them, nobody else is or ever will be
// waiting on this particular entry, so it is safe to drop it from
// the table. A later call for the same `key` just starts a fresh
// entry.
{
let mut table = self.memo_locks.lock().unwrap();
if Arc::strong_count(&entry) == 2 {
table.remove(key);
}
}
drop(guard);
result
}
/// Number of currently-contended per-key locks — exposed only for
/// tests, to check that [`with_key_lock`](Self::with_key_lock)'s
/// garbage collection actually removes entries once uncontended.
#[cfg(test)]
pub(crate) fn memo_lock_table_len(&self) -> usize {
self.memo_locks.lock().unwrap().len()
}
}
pub(crate) fn checksum_path_for(path: &Path) -> PathBuf {
let mut os_str = path.as_os_str().to_owned();
os_str.push(".");
os_str.push(CHECKSUM_EXT);
PathBuf::from(os_str)
}
/// `mmap` `path` and verify it against its checksum sidecar. `Ok(None)`
/// when `path` doesn't exist, its checksum sidecar doesn't exist/parse, or
/// the checksum doesn't match — every case here means "nothing usable is
/// cached," never an error the caller has to handle specially.
fn verify_mmap(path: &Path) -> io::Result<Option<Mmap>> {
/// Append `.tmp` to `path`'s full file name (never replaces an existing
/// extension the way [`Path::with_extension`] would). Used to derive the
/// staging path for both the data file and the checksum sidecar in
/// [`write_checked`] — appending rather than replacing guarantees the two
/// staging paths can never collide with each other, nor with an unrelated
/// artifact whose name happens to contain a `.`.
fn tmp_path_for(path: &Path) -> PathBuf {
let mut os_str = path.as_os_str().to_owned();
os_str.push(".tmp");
PathBuf::from(os_str)
}
pub(crate) fn verify_mmap(path: &Path) -> io::Result<Option<Mmap>> {
if !path.exists() {
return Ok(None);
}
@@ -169,17 +249,17 @@ fn read_checked(path: &Path) -> io::Result<Option<Vec<u8>>> {
Ok(verify_mmap(path)?.map(|mmap| mmap[..].to_vec()))
}
fn write_checked(path: &Path, bytes: &[u8]) -> io::Result<()> {
pub(crate) fn write_checked(path: &Path, bytes: &[u8]) -> io::Result<()> {
if let Some(parent) = path.parent() {
fs::create_dir_all(parent)?;
}
let tmp = path.with_extension("tmp");
let tmp = tmp_path_for(path);
fs::write(&tmp, bytes)?;
fs::rename(&tmp, path)?;
let checksum = xxhash_rust::xxh3::xxh3_64(bytes);
let checksum_path = checksum_path_for(path);
let checksum_tmp = checksum_path.with_extension("tmp");
let checksum_tmp = tmp_path_for(&checksum_path);
fs::write(&checksum_tmp, checksum.to_string())?;
fs::rename(&checksum_tmp, checksum_path)?;
Ok(())