Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1f0233d033 |
No files matched your search
@@ -1,8 +1,9 @@
|
||||
pname: CI
|
||||
name: CI
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches: ['main']
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
build:
|
||||
@@ -25,8 +26,8 @@ jobs:
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
src/target
|
||||
key: ${{ runner.os }}-cargo-v2-${{ hashFiles('src/Cargo.lock') }}
|
||||
restore-keys: ${{ runner.os }}-cargo-v2-
|
||||
key: ${{ runner.os }}-cargo-${{ hashFiles('src/Cargo.lock') }}
|
||||
restore-keys: ${{ runner.os }}-cargo-
|
||||
|
||||
- name: Build
|
||||
run: cargo build --release
|
||||
|
||||
@@ -11,10 +11,6 @@ jobs:
|
||||
outputs:
|
||||
release_id: ${{ steps.create.outputs.release_id }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Create Gitea release
|
||||
id: create
|
||||
env:
|
||||
@@ -22,12 +18,11 @@ jobs:
|
||||
TAG: ${{ github.ref_name }}
|
||||
run: |
|
||||
sudo apt-get update -qq && sudo apt-get install -y -qq jq
|
||||
body=$(git for-each-ref --format='%(contents)' "refs/tags/$TAG")
|
||||
release_id=$(curl -s -X POST \
|
||||
"${{ github.server_url }}/api/v1/repos/${{ github.repository }}/releases" \
|
||||
-H "Authorization: token $GITEA_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"tag_name\":\"$TAG\",\"name\":\"$TAG\",\"body\":$(echo "$body" | jq -Rs .)}" | jq -r '.id')
|
||||
-d "{\"tag_name\":\"$TAG\",\"name\":\"$TAG\"}" | jq -r '.id')
|
||||
echo "release_id=$release_id" >> $GITHUB_OUTPUT
|
||||
|
||||
build-linux-x86_64:
|
||||
@@ -62,8 +57,7 @@ jobs:
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
src/target
|
||||
tools/migrate-mphf/target
|
||||
key: linux-musl-cargo-${{ hashFiles('src/Cargo.lock', 'tools/migrate-mphf/Cargo.lock') }}
|
||||
key: linux-musl-cargo-${{ hashFiles('src/Cargo.lock') }}
|
||||
restore-keys: linux-musl-cargo-
|
||||
|
||||
- name: Build static binary
|
||||
@@ -84,36 +78,23 @@ jobs:
|
||||
-H "Authorization: token $GITEA_TOKEN" \
|
||||
-F "attachment=@/tmp/dist/obikmer-linux-x86_64"
|
||||
|
||||
# One-shot converter for indexes built with ptr_hash 1.x. A standalone
|
||||
# crate (own Cargo.lock/target, outside src/'s workspace), built after
|
||||
# obikmer's own asset is already uploaded: it never blocks that one.
|
||||
- name: Build migrate-mphf (static)
|
||||
working-directory: tools/migrate-mphf
|
||||
env:
|
||||
PKG_CONFIG_ALLOW_CROSS: "1"
|
||||
run: cargo zigbuild --release --target x86_64-unknown-linux-musl
|
||||
|
||||
- name: Upload migrate-mphf artifact
|
||||
working-directory: tools/migrate-mphf
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITEATOKEN }}
|
||||
RELEASE_ID: ${{ needs.create-release.outputs.release_id }}
|
||||
run: |
|
||||
cp target/x86_64-unknown-linux-musl/release/migrate-mphf /tmp/dist/migrate-mphf-linux-x86_64
|
||||
strip /tmp/dist/migrate-mphf-linux-x86_64
|
||||
curl -s -X POST \
|
||||
"${{ github.server_url }}/api/v1/repos/${{ github.repository }}/releases/$RELEASE_ID/assets" \
|
||||
-H "Authorization: token $GITEA_TOKEN" \
|
||||
-F "attachment=@/tmp/dist/migrate-mphf-linux-x86_64"
|
||||
|
||||
build-macos-arm64:
|
||||
needs: create-release
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: src
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Login to registry
|
||||
run: echo "${{ secrets.REGISTRYTOKEN }}" | docker login registry.metabarcoding.org -u ${{ secrets.REGISTRYUSER }} --password-stdin
|
||||
- name: Install Rust + zigbuild
|
||||
run: |
|
||||
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable
|
||||
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
|
||||
sudo apt-get update -qq && sudo apt-get install -y -qq jq
|
||||
pip install ziglang --quiet --break-system-packages
|
||||
$HOME/.cargo/bin/cargo install cargo-zigbuild
|
||||
$HOME/.cargo/bin/rustup target add aarch64-apple-darwin
|
||||
|
||||
- name: Cache cargo registry
|
||||
uses: actions/cache@v4
|
||||
@@ -126,24 +107,15 @@ jobs:
|
||||
restore-keys: macos-arm64-cargo-
|
||||
|
||||
- name: Build macOS binary
|
||||
run: |
|
||||
CID=$(docker create \
|
||||
-w /src/src \
|
||||
registry.metabarcoding.org/cibuilder/rustcrossosx:latest \
|
||||
cargo build --release --target aarch64-apple-darwin --no-default-features)
|
||||
docker cp . "$CID:/src"
|
||||
docker start -a "$CID"
|
||||
STATUS=$(docker wait "$CID")
|
||||
mkdir -p /tmp/dist
|
||||
docker cp "$CID:/src/src/target/aarch64-apple-darwin/release/obikmer" /tmp/dist/obikmer-macos-arm64
|
||||
docker rm "$CID" > /dev/null
|
||||
[ "$STATUS" -eq 0 ]
|
||||
run: cargo zigbuild --release --target aarch64-apple-darwin --no-default-features
|
||||
|
||||
- name: Prepare and upload artifact
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITEATOKEN }}
|
||||
RELEASE_ID: ${{ needs.create-release.outputs.release_id }}
|
||||
run: |
|
||||
mkdir -p /tmp/dist
|
||||
cp target/aarch64-apple-darwin/release/obikmer /tmp/dist/obikmer-macos-arm64
|
||||
curl -s -X POST \
|
||||
"${{ github.server_url }}/api/v1/repos/${{ github.repository }}/releases/$RELEASE_ID/assets" \
|
||||
-H "Authorization: token $GITEA_TOKEN" \
|
||||
|
||||
+13
-38
@@ -1,48 +1,23 @@
|
||||
.venv/
|
||||
.DS_Store
|
||||
.claude/
|
||||
.kilo/
|
||||
.serena/
|
||||
.zed/
|
||||
.ast-cache/
|
||||
CLAUDE.md
|
||||
memory/
|
||||
sandbox/
|
||||
src/target
|
||||
tools/migrate-mphf/target
|
||||
data-stress
|
||||
*.fasta
|
||||
*.fasta.gz
|
||||
*.fastq
|
||||
*.fastq.gz
|
||||
*.vcf
|
||||
*.zst
|
||||
*.zst.meta
|
||||
*.pb
|
||||
**/*.json
|
||||
./**/*.json
|
||||
*.bin
|
||||
*.log
|
||||
*.csv
|
||||
*.meta
|
||||
*.pfiv
|
||||
*.done
|
||||
*.efh
|
||||
*.efl
|
||||
*.bin.idx
|
||||
*.prsb
|
||||
Betula_exilis--IGA-24-33
|
||||
benchmark/run/
|
||||
TNT
|
||||
phyg
|
||||
biblio
|
||||
*.tnt
|
||||
*.tre
|
||||
*.phy
|
||||
*.treefile
|
||||
*.bionj
|
||||
*.iqtree
|
||||
*.mldist
|
||||
*.parstree
|
||||
*.ckp.gz
|
||||
*.model
|
||||
scripts/__pycache__/mk_wiki.cpython-314.pyc
|
||||
benchmark/genomes
|
||||
benchmark/simulated_data
|
||||
benchmark/specimen_index_presence
|
||||
benchmark/specimen_index_count
|
||||
benchmark/global_index_presence
|
||||
benchmark/global_index_count
|
||||
benchmark/stats
|
||||
benchmark/reference_index
|
||||
benchmark/reference_dist
|
||||
benchmark/obikmer_dist
|
||||
benchmark/specific_index_count
|
||||
benchmark/specific_index_presence
|
||||
@@ -0,0 +1,87 @@
|
||||
# Plan d'amélioration technique - obiskio
|
||||
|
||||
## 1. Contexte et objectifs
|
||||
- **Objectif** : Renforcer la robustesse, la maintenabilité et les performances de la crate `obiskio`.
|
||||
- **Priorités** :
|
||||
1. Gestion des erreurs
|
||||
2. Optimisation de la mémoire du pool
|
||||
3. Robustesse concurrente
|
||||
4. Couverture de tests
|
||||
5. Documentation
|
||||
|
||||
---
|
||||
|
||||
## 2. Axes d'amélioration détaillés
|
||||
|
||||
### 2.1 Gestion des erreurs
|
||||
- **Problème** : `SKError` ne couvre pas tous les cas (format invalide, taille maximale, CRC)
|
||||
- **Actions** :
|
||||
- Ajouter variante `ParseError(String)` dans `src/error.rs`
|
||||
- Valider les tailles de SuperKmer avant parsing
|
||||
- Remplacer `expect()` par `unwrap_or_else` avec messages explicites
|
||||
- Documenter chaque variante d’erreur dans le README
|
||||
|
||||
### 2.2 Optimisation du pool de fichiers
|
||||
- **Problème** : `SKFilePool` utilise un `Vec<WriteEntry>` non contraint et n’effectue pas de nettoyage en cas d’erreur
|
||||
- **Actions** :
|
||||
- Implémenter un `LimitedVec` avec limite stricte à `MAX_POOL_SIZE`
|
||||
- Créer `clear_memory()` qui supprime les entrées orphelines
|
||||
- Ajouter `evict_lru_threshold()` pour éviction proactive
|
||||
- Introduire un `RwLock` pour les opérations de lecture massives
|
||||
|
||||
### 2.3 Robustesse concurrente
|
||||
- **Problème** : Risque de deadlocks dans `SKFileWriter::write_batch()` et `SKFileReader::reopen_and_seek()`
|
||||
- **Actions** :
|
||||
- Remplacer `Mutex` par `RwLock` pour les accès en lecture
|
||||
- Ajouter un compteur de blocage et logs de timeout
|
||||
- Utiliser `std::thread::park_timeout` pour débloquer
|
||||
- Insérer `debug_assert!` sur les états invariants
|
||||
|
||||
### 2.4 Couverture de tests
|
||||
- **Problème** : Absence de benchmarks, de tests de migration, de résilience de fichiers corrompus
|
||||
- **Actions** :
|
||||
- Benchmarks I/O sur 10k+ SuperKmer avec `criterion`
|
||||
- Tests de migration de version de fichier `.meta` → `.v2.meta`
|
||||
- Tests de corruption volontaire (truncature, inversion de bits)
|
||||
- Tests de stress sur pool saturation (100 threads)
|
||||
|
||||
### 2.5 Documentation & exemples
|
||||
- **Actions** :
|
||||
- Ajouter des examples dans chaque module (`# Examples`)
|
||||
- Documenter la logique LRU avec diagrammes Mermaid
|
||||
- Créer un guide « How to recover from eviction »
|
||||
- Mettre à jour le `README.md` avec tableau des variantes d’erreur
|
||||
|
||||
---
|
||||
|
||||
## 3. Plan d'exécution (Roadmap)
|
||||
|
||||
| Sprint | Durée | Livrables clés |
|
||||
|--------|-------|----------------|
|
||||
| **S1** | 2 jours | Refactorisation `SKError`, ajout de tests unitaires |
|
||||
| **S2** | 3 jours | Implémentation `clear_memory()` + `LimitedVec` |
|
||||
| **S3** | 2 jours | Passage à `RwLock`, ajout de compteurs de blocage |
|
||||
| **S4** | 2 jours | Benchmarks + tests de migration |
|
||||
| **S5** | 1 jour | Documentation finale & mise à jour du README |
|
||||
|
||||
---
|
||||
|
||||
## 4. Dépendances externes
|
||||
- Mettre à jour `niffler` vers la version 2.0 (performance compression)
|
||||
- Évaluer `bincode` vs `serde_json` pour les métas (I/O)
|
||||
- Ajouter dépendance `criterion` (dev‑dependencies)
|
||||
|
||||
---
|
||||
|
||||
## 5. KPI de suivi
|
||||
- **Couverture de tests** : ≥85 % des chemins critiques
|
||||
- **Latence moyenne d’écriture** : ↓15 % après optimisation du pool
|
||||
- **Taux d’erreurs résolues** : 100 % des nouvelles variantes couvertes
|
||||
- **Temps de build CI** : ≤5 min pour l’ensemble des benchmarks
|
||||
|
||||
---
|
||||
|
||||
## 6. Validation finale
|
||||
- Revue de code avec `cargo clippy -- -D warnings`
|
||||
- Analyse de toxicité avec `cargo deny open-source-licenses`
|
||||
- Vérification de la conformité aux standards de naming du projet
|
||||
@@ -0,0 +1,2 @@
|
||||
/cache
|
||||
/project.local.yml
|
||||
@@ -0,0 +1,133 @@
|
||||
# the name by which the project can be referenced within Serena
|
||||
project_name: "obikmer"
|
||||
|
||||
|
||||
# list of languages for which language servers are started; choose from:
|
||||
# al angular ansible bash clojure
|
||||
# cpp cpp_ccls crystal csharp csharp_omnisharp
|
||||
# dart elixir elm erlang fortran
|
||||
# fsharp go groovy haskell haxe
|
||||
# hlsl html java json julia
|
||||
# kotlin lean4 lua luau markdown
|
||||
# matlab msl nix ocaml pascal
|
||||
# perl php php_phpactor powershell python
|
||||
# python_jedi python_ty r rego ruby
|
||||
# ruby_solargraph rust scala scss solidity
|
||||
# svelte swift systemverilog terraform toml
|
||||
# typescript typescript_vts vue yaml zig
|
||||
# (This list may be outdated. For the current list, see values of Language enum here:
|
||||
# https://github.com/oraios/serena/blob/main/src/solidlsp/ls_config.py
|
||||
# For some languages, there are alternative language servers, e.g. csharp_omnisharp, ruby_solargraph.)
|
||||
# Note:
|
||||
# - For C, use cpp
|
||||
# - For JavaScript, use typescript
|
||||
# - For Angular projects, use angular (subsumes typescript+html; requires `npm install` in the project root)
|
||||
# - For Svelte projects, use svelte (subsumes typescript/javascript for .svelte projects; requires npm)
|
||||
# - For SCSS / Sass / plain CSS, use scss (some-sass-language-server handles all three)
|
||||
# - For Free Pascal/Lazarus, use pascal
|
||||
# Special requirements:
|
||||
# Some languages require additional setup/installations.
|
||||
# See here for details: https://oraios.github.io/serena/01-about/020_programming-languages.html#language-servers
|
||||
# When using multiple languages, the first language server that supports a given file will be used for that file.
|
||||
# The first language is the default language and the respective language server will be used as a fallback.
|
||||
# Note that when using the JetBrains backend, language servers are not used and this list is correspondingly ignored.
|
||||
languages:
|
||||
- rust
|
||||
|
||||
# the encoding used by text files in the project
|
||||
# For a list of possible encodings, see https://docs.python.org/3.11/library/codecs.html#standard-encodings
|
||||
encoding: "utf-8"
|
||||
|
||||
# line ending convention to use when writing source files.
|
||||
# Possible values: unset (use global setting), "lf", "crlf", or "native" (platform default)
|
||||
# This does not affect Serena's own files (e.g. memories and configuration files), which always use native line endings.
|
||||
line_ending:
|
||||
|
||||
# The language backend to use for this project.
|
||||
# If not set, the global setting from serena_config.yml is used.
|
||||
# Valid values: LSP, JetBrains
|
||||
# Note: the backend is fixed at startup. If a project with a different backend
|
||||
# is activated post-init, an error will be returned.
|
||||
language_backend:
|
||||
|
||||
# whether to use project's .gitignore files to ignore files
|
||||
ignore_all_files_in_gitignore: true
|
||||
|
||||
# advanced configuration option allowing to configure language server-specific options.
|
||||
# Maps the language key to the options.
|
||||
# Have a look at the docstring of the constructors of the LS implementations within solidlsp (e.g., for C# or PHP) to see which options are available.
|
||||
# No documentation on options means no options are available.
|
||||
ls_specific_settings: {}
|
||||
|
||||
# list of additional workspace folder paths for cross-package reference support (e.g. in monorepos).
|
||||
# Paths can be absolute or relative to the project root.
|
||||
# Each folder is registered as an LSP workspace folder, enabling language servers to discover
|
||||
# symbols and references across package boundaries.
|
||||
# Currently supported for: TypeScript.
|
||||
# Example:
|
||||
# additional_workspace_folders:
|
||||
# - ../sibling-package
|
||||
# - ../shared-lib
|
||||
additional_workspace_folders: []
|
||||
|
||||
# list of additional paths to ignore in this project.
|
||||
# Same syntax as gitignore, so you can use * and **.
|
||||
# Note: global ignored_paths from serena_config.yml are also applied additively.
|
||||
ignored_paths: []
|
||||
|
||||
# whether the project is in read-only mode
|
||||
# If set to true, all editing tools will be disabled and attempts to use them will result in an error
|
||||
# Added on 2025-04-18
|
||||
read_only: false
|
||||
|
||||
# list of tool names to exclude.
|
||||
# This extends the existing exclusions (e.g. from the global configuration)
|
||||
# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html
|
||||
excluded_tools: []
|
||||
|
||||
# list of tools to include that would otherwise be disabled (particularly optional tools that are disabled by default).
|
||||
# This extends the existing inclusions (e.g. from the global configuration).
|
||||
# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html
|
||||
included_optional_tools: []
|
||||
|
||||
# fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools.
|
||||
# This cannot be combined with non-empty excluded_tools or included_optional_tools.
|
||||
# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html
|
||||
fixed_tools: []
|
||||
|
||||
# list of mode names that are to be activated by default, overriding the setting in the global configuration.
|
||||
# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes.
|
||||
# If the setting is undefined/empty, the default_modes from the global configuration (serena_config.yml) apply.
|
||||
# Otherwise, this overrides the setting from the global configuration (serena_config.yml).
|
||||
# Therefore, you can set this to [] if you do not want the default modes defined in the global config to apply
|
||||
# for this project.
|
||||
# This setting can, in turn, be overridden by CLI parameters (--mode).
|
||||
# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes
|
||||
default_modes:
|
||||
|
||||
# list of mode names to be activated additionally for this project, e.g. ["query-projects"]
|
||||
# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes.
|
||||
# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes
|
||||
added_modes:
|
||||
|
||||
# initial prompt for the project. It will always be given to the LLM upon activating the project
|
||||
# (contrary to the memories, which are loaded on demand).
|
||||
initial_prompt: ""
|
||||
|
||||
# time budget (seconds) per tool call for the retrieval of additional symbol information
|
||||
# such as docstrings or parameter information.
|
||||
# This overrides the corresponding setting in the global configuration; see the documentation there.
|
||||
# If null or missing, use the setting from the global configuration.
|
||||
symbol_info_budget:
|
||||
|
||||
# list of regex patterns which, when matched, mark a memory entry as read‑only.
|
||||
# Extends the list from the global configuration, merging the two lists.
|
||||
read_only_memory_patterns: []
|
||||
|
||||
# list of regex patterns for memories to completely ignore.
|
||||
# Matching memories will not appear in list_memories or activate_project output
|
||||
# and cannot be accessed via read_memory or write_memory.
|
||||
# To access ignored memory files, use the read_file tool on the raw file path.
|
||||
# Extends the list from the global configuration, merging the two lists.
|
||||
# Example: ["_archive/.*", "_episodes/.*"]
|
||||
ignored_memory_patterns: []
|
||||
@@ -0,0 +1,59 @@
|
||||
// Project tasks configuration. See https://zed.dev/docs/tasks for documentation.
|
||||
//
|
||||
// Example:
|
||||
[
|
||||
{
|
||||
"label": "Example task",
|
||||
"command": "for i in {1..5}; do echo \"Hello $i/5\"; sleep 1; done",
|
||||
//"args": [],
|
||||
// Env overrides for the command, will be appended to the terminal's environment from the settings.
|
||||
"env": { "foo": "bar" },
|
||||
// Current working directory to spawn the command into, defaults to current project root.
|
||||
//"cwd": "/path/to/working/directory",
|
||||
// Whether to use a new terminal tab or reuse the existing one to spawn the process, defaults to `false`.
|
||||
"use_new_terminal": false,
|
||||
// Whether to allow multiple instances of the same task to be run, or rather wait for the existing ones to finish, defaults to `false`.
|
||||
"allow_concurrent_runs": false,
|
||||
// What to do with the terminal pane and tab, after the command was started:
|
||||
// * `always` — always show the task's pane, and focus the corresponding tab in it (default)
|
||||
// * `no_focus` — always show the task's pane, add the task's tab in it, but don't focus it
|
||||
// * `never` — do not alter focus, but still add/reuse the task's tab in its pane
|
||||
"reveal": "always",
|
||||
// Where to place the task's terminal item after starting the task:
|
||||
// * `dock` — in the terminal dock, "regular" terminal items' place (default)
|
||||
// * `center` — in the central pane group, "main" editor area
|
||||
"reveal_target": "dock",
|
||||
// What to do with the terminal pane and tab, after the command had finished:
|
||||
// * `never` — Do nothing when the command finishes (default)
|
||||
// * `always` — always hide the terminal tab, hide the pane also if it was the last tab in it
|
||||
// * `on_success` — hide the terminal tab on task success only, otherwise behaves similar to `always`
|
||||
"hide": "never",
|
||||
// Which shell to use when running a task inside the terminal.
|
||||
// May take 3 values:
|
||||
// 1. (default) Use the system's default terminal configuration in /etc/passwd
|
||||
// "shell": "system"
|
||||
// 2. A program:
|
||||
// "shell": {
|
||||
// "program": "sh"
|
||||
// }
|
||||
// 3. A program with arguments:
|
||||
// "shell": {
|
||||
// "with_arguments": {
|
||||
// "program": "/bin/bash",
|
||||
// "args": ["--login"]
|
||||
// }
|
||||
// }
|
||||
"shell": "system",
|
||||
// Whether to show the task line in the output of the spawned task, defaults to `true`.
|
||||
"show_summary": true,
|
||||
// Whether to show the command line in the output of the spawned task, defaults to `true`.
|
||||
"show_command": true,
|
||||
// Which edited buffers to save before running the task:
|
||||
// * `all` — save all edited buffers
|
||||
// * `current` — save currently active buffer only
|
||||
// * `none` — don't save any buffers
|
||||
"save": "none",
|
||||
// Represents the tags for inline runnable indicators, or spawning multiple tasks at once.
|
||||
// "tags": []
|
||||
},
|
||||
]
|
||||
@@ -0,0 +1,101 @@
|
||||
**PROMPT**
|
||||
|
||||
Tu es ma base de connaissance et mon bloc-notes intelligent sur le projet **obikmer**. Tu ne proposes pas, tu ne codes pas spontanément — tu réponds à mes questions et tu structures mes idées au fur et à mesure que je les exprime.
|
||||
|
||||
**Règle absolue : une question appelle une réponse, pas une action.**
|
||||
Ne modifier aucun fichier à moins d'une demande explicite de modification. En particulier : observer un bug ou une incohérence dans le code montré ne constitue pas un mandat pour le corriger. Le code montré peut refléter une intention en cours — modifier sans mandat risque d'introduire un vrai bug là où tu croyais corriger.
|
||||
|
||||
**Règle absolue : ne jamais substituer une dépendance ou une bibliothèque sans validation explicite.**
|
||||
Si une dépendance demandée pose problème (erreur de compilation, bug, API manquante), exposer le problème et proposer des alternatives — ne jamais switcher silencieusement vers une autre bibliothèque. Le choix des dépendances est une décision d'architecture qui appartient au développeur.
|
||||
|
||||
**Règle absolue : le code existant est une hypothèse, pas une vérité.**
|
||||
Quand une nouvelle construction (type, itérateur, abstraction) rend du code historique injustifié, le signaler immédiatement et proposer de le supprimer — ne pas conserver les deux en parallèle par inertie. Le développeur demande explicitement de remettre en cause le code base : ne pas attendre qu'il insiste.
|
||||
|
||||
Tu maintiens en **anglais**, dense et sans remplissage, les documents suivants :
|
||||
- `docmd/index.md` — document de discussion de base, enrichi progressivement au fil de nos échanges ; il reflète l'état courant de la réflexion sur le projet
|
||||
- les autres fichiers Markdown dans `docmd/` selon leur thème respectif
|
||||
|
||||
Les snippets de code y sont courts et illustrent uniquement des principes architecturaux. Nos échanges se font en **français**.
|
||||
|
||||
---
|
||||
|
||||
## Contexte du projet
|
||||
|
||||
`obikmer` est un outil Rust de manipulation, comptage, indexation et opérations ensemblistes sur des séquences ADN représentées comme des ensembles de kmers.
|
||||
|
||||
**Contraintes fondamentales**
|
||||
- Efficacité maximale en calcul, mémoire et disque
|
||||
- Données métagénomiques : plusieurs dizaines de Gbases, milliards de kmers
|
||||
- k impair, k ∈ [11, 31], fixé à l'exécution
|
||||
- Formats d'entrée : FASTA, FASTQ, gzip, streaming stdin
|
||||
|
||||
**Opérations prioritaires**
|
||||
- Comptage de kmers (fréquences)
|
||||
- Recherche / requête rapide
|
||||
- Opérations ensemblistes (union, intersection, diff)
|
||||
|
||||
**Ce qui a déjà été discuté**
|
||||
- Encodage 2 bits/base → kmer tient dans un `u64`
|
||||
- Forme canonique : `min(kmer, revcomp)` pour réduire l'espace de moitié
|
||||
|
||||
---
|
||||
|
||||
## Infrastructure de documentation
|
||||
|
||||
La documentation est gérée via **MkDocs + thème Material**, avec publication sur **GitHub Pages**.
|
||||
|
||||
**Structure des répertoires**
|
||||
|
||||
```
|
||||
docmd/ ← sources Markdown + mkdocs.yml
|
||||
docmd/mkdocs.yml
|
||||
doc/ ← site HTML généré (servi par GitHub Pages)
|
||||
.venv/ ← environnement Python (ignoré par git)
|
||||
```
|
||||
|
||||
**Configuration `docmd/mkdocs.yml`**
|
||||
- `docs_dir: .` (sources = `docmd/` lui-même)
|
||||
- `site_dir: ../doc` (sortie = `doc/`)
|
||||
|
||||
**Commandes Makefile**
|
||||
|
||||
| Commande | Effet |
|
||||
|---|---|
|
||||
| `make doc` | Construit le HTML dans `doc/` |
|
||||
| `make doc-serve` | Serveur local avec rechargement automatique |
|
||||
| `make clean-doc` | Supprime `doc/` |
|
||||
| `make clean` | Supprime `doc/` et `.venv/` |
|
||||
|
||||
Le `.venv/` est dans `.gitignore`. Le répertoire `doc/` (sortie HTML) est versionné pour GitHub Pages.
|
||||
|
||||
Lors de l'ajout de nouveaux fichiers Markdown dans `docmd/`, mettre à jour la section `nav:` de `docmd/mkdocs.yml`.
|
||||
|
||||
---
|
||||
|
||||
Je continue à poser mes questions et à guider la discussion.
|
||||
|
||||
---
|
||||
|
||||
## MCP Tools
|
||||
|
||||
**Règle absolue : avant tout travail de code, appeler `mcp__serena__initial_instructions` pour charger les instructions Serena.**
|
||||
|
||||
### Hiérarchie des outils pour ce projet Rust
|
||||
|
||||
**Navigation et édition de code → serena en priorité**
|
||||
- Trouver un symbole, une déclaration, les implémentations d'un trait : `mcp__serena__find_symbol`, `mcp__serena__find_declaration`, `mcp__serena__find_implementations`
|
||||
- Trouver les usages d'un symbole : `mcp__serena__find_referencing_symbols`
|
||||
- Diagnostics LSP (erreurs de compilation) : `mcp__serena__get_diagnostics_for_file`
|
||||
- Vue d'ensemble d'un fichier : `mcp__serena__get_symbols_overview`
|
||||
- Modifier le corps d'une fonction/impl : `mcp__serena__replace_symbol_body`
|
||||
- Ne pas utiliser `cclsp` quand serena couvre le besoin
|
||||
|
||||
**Analyse architecturale → jcodemunch**
|
||||
- Hotspots, couplage, dead code, dépendances entre modules
|
||||
- Utiliser avant de refactorer une zone critique
|
||||
|
||||
**Raisonnement complexe → sequential-thinking**
|
||||
- Décisions d'architecture, choix d'algorithme, trade-offs non triviaux
|
||||
|
||||
**Documentation de crates → context7**
|
||||
- Toujours consulter avant d'utiliser une API de bibliothèque externe
|
||||
-1275
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
Binary file not shown.
|
Before Width: | Height: | Size: 1.8 KiB |
@@ -1,16 +0,0 @@
|
||||
"use strict";(()=>{var Zi=Object.create;var _r=Object.defineProperty;var ea=Object.getOwnPropertyDescriptor;var ta=Object.getOwnPropertyNames,Bt=Object.getOwnPropertySymbols,ra=Object.getPrototypeOf,Ar=Object.prototype.hasOwnProperty,bo=Object.prototype.propertyIsEnumerable;var ho=(e,t,r)=>t in e?_r(e,t,{enumerable:!0,configurable:!0,writable:!0,value:r}):e[t]=r,P=(e,t)=>{for(var r in t||(t={}))Ar.call(t,r)&&ho(e,r,t[r]);if(Bt)for(var r of Bt(t))bo.call(t,r)&&ho(e,r,t[r]);return e};var vo=(e,t)=>{var r={};for(var o in e)Ar.call(e,o)&&t.indexOf(o)<0&&(r[o]=e[o]);if(e!=null&&Bt)for(var o of Bt(e))t.indexOf(o)<0&&bo.call(e,o)&&(r[o]=e[o]);return r};var Cr=(e,t)=>()=>(t||e((t={exports:{}}).exports,t),t.exports);var oa=(e,t,r,o)=>{if(t&&typeof t=="object"||typeof t=="function")for(let n of ta(t))!Ar.call(e,n)&&n!==r&&_r(e,n,{get:()=>t[n],enumerable:!(o=ea(t,n))||o.enumerable});return e};var $t=(e,t,r)=>(r=e!=null?Zi(ra(e)):{},oa(t||!e||!e.__esModule?_r(r,"default",{value:e,enumerable:!0}):r,e));var go=(e,t,r)=>new Promise((o,n)=>{var i=c=>{try{a(r.next(c))}catch(p){n(p)}},s=c=>{try{a(r.throw(c))}catch(p){n(p)}},a=c=>c.done?o(c.value):Promise.resolve(c.value).then(i,s);a((r=r.apply(e,t)).next())});var xo=Cr((kr,yo)=>{(function(e,t){typeof kr=="object"&&typeof yo!="undefined"?t():typeof define=="function"&&define.amd?define(t):t()})(kr,(function(){"use strict";function e(r){var o=!0,n=!1,i=null,s={text:!0,search:!0,url:!0,tel:!0,email:!0,password:!0,number:!0,date:!0,month:!0,week:!0,time:!0,datetime:!0,"datetime-local":!0};function a(k){return!!(k&&k!==document&&k.nodeName!=="HTML"&&k.nodeName!=="BODY"&&"classList"in k&&"contains"in k.classList)}function c(k){var ut=k.type,je=k.tagName;return!!(je==="INPUT"&&s[ut]&&!k.readOnly||je==="TEXTAREA"&&!k.readOnly||k.isContentEditable)}function p(k){k.classList.contains("focus-visible")||(k.classList.add("focus-visible"),k.setAttribute("data-focus-visible-added",""))}function l(k){k.hasAttribute("data-focus-visible-added")&&(k.classList.remove("focus-visible"),k.removeAttribute("data-focus-visible-added"))}function f(k){k.metaKey||k.altKey||k.ctrlKey||(a(r.activeElement)&&p(r.activeElement),o=!0)}function u(k){o=!1}function d(k){a(k.target)&&(o||c(k.target))&&p(k.target)}function v(k){a(k.target)&&(k.target.classList.contains("focus-visible")||k.target.hasAttribute("data-focus-visible-added"))&&(n=!0,window.clearTimeout(i),i=window.setTimeout(function(){n=!1},100),l(k.target))}function S(k){document.visibilityState==="hidden"&&(n&&(o=!0),X())}function X(){document.addEventListener("mousemove",ee),document.addEventListener("mousedown",ee),document.addEventListener("mouseup",ee),document.addEventListener("pointermove",ee),document.addEventListener("pointerdown",ee),document.addEventListener("pointerup",ee),document.addEventListener("touchmove",ee),document.addEventListener("touchstart",ee),document.addEventListener("touchend",ee)}function re(){document.removeEventListener("mousemove",ee),document.removeEventListener("mousedown",ee),document.removeEventListener("mouseup",ee),document.removeEventListener("pointermove",ee),document.removeEventListener("pointerdown",ee),document.removeEventListener("pointerup",ee),document.removeEventListener("touchmove",ee),document.removeEventListener("touchstart",ee),document.removeEventListener("touchend",ee)}function ee(k){k.target.nodeName&&k.target.nodeName.toLowerCase()==="html"||(o=!1,re())}document.addEventListener("keydown",f,!0),document.addEventListener("mousedown",u,!0),document.addEventListener("pointerdown",u,!0),document.addEventListener("touchstart",u,!0),document.addEventListener("visibilitychange",S,!0),X(),r.addEventListener("focus",d,!0),r.addEventListener("blur",v,!0),r.nodeType===Node.DOCUMENT_FRAGMENT_NODE&&r.host?r.host.setAttribute("data-js-focus-visible",""):r.nodeType===Node.DOCUMENT_NODE&&(document.documentElement.classList.add("js-focus-visible"),document.documentElement.setAttribute("data-js-focus-visible",""))}if(typeof window!="undefined"&&typeof document!="undefined"){window.applyFocusVisiblePolyfill=e;var t;try{t=new CustomEvent("focus-visible-polyfill-ready")}catch(r){t=document.createEvent("CustomEvent"),t.initCustomEvent("focus-visible-polyfill-ready",!1,!1,{})}window.dispatchEvent(t)}typeof document!="undefined"&&e(document)}))});var ro=Cr((jy,Rn)=>{"use strict";/*!
|
||||
* escape-html
|
||||
* Copyright(c) 2012-2013 TJ Holowaychuk
|
||||
* Copyright(c) 2015 Andreas Lubbe
|
||||
* Copyright(c) 2015 Tiancheng "Timothy" Gu
|
||||
* MIT Licensed
|
||||
*/var qa=/["'&<>]/;Rn.exports=Ka;function Ka(e){var t=""+e,r=qa.exec(t);if(!r)return t;var o,n="",i=0,s=0;for(i=r.index;i<t.length;i++){switch(t.charCodeAt(i)){case 34:o=""";break;case 38:o="&";break;case 39:o="'";break;case 60:o="<";break;case 62:o=">";break;default:continue}s!==i&&(n+=t.substring(s,i)),s=i+1,n+=o}return s!==i?n+t.substring(s,i):n}});var ao=Cr((Nt,io)=>{/*!
|
||||
* clipboard.js v2.0.11
|
||||
* https://clipboardjs.com/
|
||||
*
|
||||
* Licensed MIT © Zeno Rocha
|
||||
*/(function(t,r){typeof Nt=="object"&&typeof io=="object"?io.exports=r():typeof define=="function"&&define.amd?define([],r):typeof Nt=="object"?Nt.ClipboardJS=r():t.ClipboardJS=r()})(Nt,function(){return(function(){var e={686:(function(o,n,i){"use strict";i.d(n,{default:function(){return Xi}});var s=i(279),a=i.n(s),c=i(370),p=i.n(c),l=i(817),f=i.n(l);function u(q){try{return document.execCommand(q)}catch(C){return!1}}var d=function(C){var _=f()(C);return u("cut"),_},v=d;function S(q){var C=document.documentElement.getAttribute("dir")==="rtl",_=document.createElement("textarea");_.style.fontSize="12pt",_.style.border="0",_.style.padding="0",_.style.margin="0",_.style.position="absolute",_.style[C?"right":"left"]="-9999px";var D=window.pageYOffset||document.documentElement.scrollTop;return _.style.top="".concat(D,"px"),_.setAttribute("readonly",""),_.value=q,_}var X=function(C,_){var D=S(C);_.container.appendChild(D);var N=f()(D);return u("copy"),D.remove(),N},re=function(C){var _=arguments.length>1&&arguments[1]!==void 0?arguments[1]:{container:document.body},D="";return typeof C=="string"?D=X(C,_):C instanceof HTMLInputElement&&!["text","search","url","tel","password"].includes(C==null?void 0:C.type)?D=X(C.value,_):(D=f()(C),u("copy")),D},ee=re;function k(q){"@babel/helpers - typeof";return typeof Symbol=="function"&&typeof Symbol.iterator=="symbol"?k=function(_){return typeof _}:k=function(_){return _&&typeof Symbol=="function"&&_.constructor===Symbol&&_!==Symbol.prototype?"symbol":typeof _},k(q)}var ut=function(){var C=arguments.length>0&&arguments[0]!==void 0?arguments[0]:{},_=C.action,D=_===void 0?"copy":_,N=C.container,G=C.target,We=C.text;if(D!=="copy"&&D!=="cut")throw new Error('Invalid "action" value, use either "copy" or "cut"');if(G!==void 0)if(G&&k(G)==="object"&&G.nodeType===1){if(D==="copy"&&G.hasAttribute("disabled"))throw new Error('Invalid "target" attribute. Please use "readonly" instead of "disabled" attribute');if(D==="cut"&&(G.hasAttribute("readonly")||G.hasAttribute("disabled")))throw new Error(`Invalid "target" attribute. You can't cut text from elements with "readonly" or "disabled" attributes`)}else throw new Error('Invalid "target" value, use a valid Element');if(We)return ee(We,{container:N});if(G)return D==="cut"?v(G):ee(G,{container:N})},je=ut;function R(q){"@babel/helpers - typeof";return typeof Symbol=="function"&&typeof Symbol.iterator=="symbol"?R=function(_){return typeof _}:R=function(_){return _&&typeof Symbol=="function"&&_.constructor===Symbol&&_!==Symbol.prototype?"symbol":typeof _},R(q)}function se(q,C){if(!(q instanceof C))throw new TypeError("Cannot call a class as a function")}function ce(q,C){for(var _=0;_<C.length;_++){var D=C[_];D.enumerable=D.enumerable||!1,D.configurable=!0,"value"in D&&(D.writable=!0),Object.defineProperty(q,D.key,D)}}function he(q,C,_){return C&&ce(q.prototype,C),_&&ce(q,_),q}function Se(q,C){if(typeof C!="function"&&C!==null)throw new TypeError("Super expression must either be null or a function");q.prototype=Object.create(C&&C.prototype,{constructor:{value:q,writable:!0,configurable:!0}}),C&&Ue(q,C)}function Ue(q,C){return Ue=Object.setPrototypeOf||function(D,N){return D.__proto__=N,D},Ue(q,C)}function Qi(q){var C=Gi();return function(){var D=Qt(q),N;if(C){var G=Qt(this).constructor;N=Reflect.construct(D,arguments,G)}else N=D.apply(this,arguments);return Yi(this,N)}}function Yi(q,C){return C&&(R(C)==="object"||typeof C=="function")?C:Bi(q)}function Bi(q){if(q===void 0)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return q}function Gi(){if(typeof Reflect=="undefined"||!Reflect.construct||Reflect.construct.sham)return!1;if(typeof Proxy=="function")return!0;try{return Date.prototype.toString.call(Reflect.construct(Date,[],function(){})),!0}catch(q){return!1}}function Qt(q){return Qt=Object.setPrototypeOf?Object.getPrototypeOf:function(_){return _.__proto__||Object.getPrototypeOf(_)},Qt(q)}function Mr(q,C){var _="data-clipboard-".concat(q);if(C.hasAttribute(_))return C.getAttribute(_)}var Ji=(function(q){Se(_,q);var C=Qi(_);function _(D,N){var G;return se(this,_),G=C.call(this),G.resolveOptions(N),G.listenClick(D),G}return he(_,[{key:"resolveOptions",value:function(){var N=arguments.length>0&&arguments[0]!==void 0?arguments[0]:{};this.action=typeof N.action=="function"?N.action:this.defaultAction,this.target=typeof N.target=="function"?N.target:this.defaultTarget,this.text=typeof N.text=="function"?N.text:this.defaultText,this.container=R(N.container)==="object"?N.container:document.body}},{key:"listenClick",value:function(N){var G=this;this.listener=p()(N,"click",function(We){return G.onClick(We)})}},{key:"onClick",value:function(N){var G=N.delegateTarget||N.currentTarget,We=this.action(G)||"copy",Yt=je({action:We,container:this.container,target:this.target(G),text:this.text(G)});this.emit(Yt?"success":"error",{action:We,text:Yt,trigger:G,clearSelection:function(){G&&G.focus(),window.getSelection().remoLine truncated
|
||||
`+r.map(function(o,n){return n+1+") "+o.toString()}).join(`
|
||||
`):"",this.name="UnsubscriptionError",this.errors=r}});function Ze(e,t){if(e){var r=e.indexOf(t);0<=r&&e.splice(r,1)}}var qe=(function(){function e(t){this.initialTeardown=t,this.closed=!1,this._parentage=null,this._finalizers=null}return e.prototype.unsubscribe=function(){var t,r,o,n,i;if(!this.closed){this.closed=!0;var s=this._parentage;if(s)if(this._parentage=null,Array.isArray(s))try{for(var a=Oe(s),c=a.next();!c.done;c=a.next()){var p=c.value;p.remove(this)}}catch(S){t={error:S}}finally{try{c&&!c.done&&(r=a.return)&&r.call(a)}finally{if(t)throw t.error}}else s.remove(this);var l=this.initialTeardown;if(I(l))try{l()}catch(S){i=S instanceof Jt?S.errors:[S]}var f=this._finalizers;if(f){this._finalizers=null;try{for(var u=Oe(f),d=u.next();!d.done;d=u.next()){var v=d.value;try{So(v)}catch(S){i=i!=null?i:[],S instanceof Jt?i=B(B([],K(i)),K(S.errors)):i.push(S)}}}catch(S){o={error:S}}finally{try{d&&!d.done&&(n=u.return)&&n.call(u)}finally{if(o)throw o.error}}}if(i)throw new Jt(i)}},e.prototype.add=function(t){var r;if(t&&t!==this)if(this.closed)So(t);else{if(t instanceof e){if(t.closed||t._hasParent(this))return;t._addParent(this)}(this._finalizers=(r=this._finalizers)!==null&&r!==void 0?r:[]).push(t)}},e.prototype._hasParent=function(t){var r=this._parentage;return r===t||Array.isArray(r)&&r.includes(t)},e.prototype._addParent=function(t){var r=this._parentage;this._parentage=Array.isArray(r)?(r.push(t),r):r?[r,t]:t},e.prototype._removeParent=function(t){var r=this._parentage;r===t?this._parentage=null:Array.isArray(r)&&Ze(r,t)},e.prototype.remove=function(t){var r=this._finalizers;r&&Ze(r,t),t instanceof e&&t._removeParent(this)},e.EMPTY=(function(){var t=new e;return t.closed=!0,t})(),e})();var $r=qe.EMPTY;function Xt(e){return e instanceof qe||e&&"closed"in e&&I(e.remove)&&I(e.add)&&I(e.unsubscribe)}function So(e){I(e)?e():e.unsubscribe()}var De={onUnhandledError:null,onStoppedNotification:null,Promise:void 0,useDeprecatedSynchronousErrorHandling:!1,useDeprecatedNextContext:!1};var xt={setTimeout:function(e,t){for(var r=[],o=2;o<arguments.length;o++)r[o-2]=arguments[o];var n=xt.delegate;return n!=null&&n.setTimeout?n.setTimeout.apply(n,B([e,t],K(r))):setTimeout.apply(void 0,B([e,t],K(r)))},clearTimeout:function(e){var t=xt.delegate;return((t==null?void 0:t.clearTimeout)||clearTimeout)(e)},delegate:void 0};function Zt(e){xt.setTimeout(function(){var t=De.onUnhandledError;if(t)t(e);else throw e})}function ge(){}var Oo=(function(){return Pr("C",void 0,void 0)})();function Lo(e){return Pr("E",void 0,e)}function Mo(e){return Pr("N",e,void 0)}function Pr(e,t,r){return{kind:e,value:t,error:r}}var ht=null;function Et(e){if(De.useDeprecatedSynchronousErrorHandling){var t=!ht;if(t&&(ht={errorThrown:!1,error:null}),e(),t){var r=ht,o=r.errorThrown,n=r.error;if(ht=null,o)throw n}}else e()}function _o(e){De.useDeprecatedSynchronousErrorHandling&&ht&&(ht.errorThrown=!0,ht.error=e)}var Pt=(function(e){ie(t,e);function t(r){var o=e.call(this)||this;return o.isStopped=!1,r?(o.destination=r,Xt(r)&&r.add(o)):o.destination=sa,o}return t.create=function(r,o,n){return new bt(r,o,n)},t.prototype.next=function(r){this.isStopped?Ir(Mo(r),this):this._next(r)},t.prototype.error=function(r){this.isStopped?Ir(Lo(r),this):(this.isStopped=!0,this._error(r))},t.prototype.complete=function(){this.isStopped?Ir(Oo,this):(this.isStopped=!0,this._complete())},t.prototype.unsubscribe=function(){this.closed||(this.isStopped=!0,e.prototype.unsubscribe.call(this),this.destination=null)},t.prototype._next=function(r){this.destination.next(r)},t.prototype._error=function(r){try{this.destination.error(r)}finally{this.unsubscribe()}},t.prototype._complete=function(){try{this.destination.complete()}finally{this.unsubscribe()}},t})(qe);var na=Function.prototype.bind;function Rr(e,t){return na.call(e,t)}var ia=(function(){function e(t){this.partialObserver=t}return e.prototype.next=function(t){var r=this.partialObserver;if(r.next)try{r.next(t)}catch(o){er(o)}},e.prototype.error=function(t){var r=this.partialObserver;if(r.error)try{r.error(t)}catch(o){er(o)}else er(t)},e.prototype.complete=function(){var t=this.partialObserver;if(t.complete)try{t.complete()}catch(r){er(r)}},e})(),bt=(function(e){ie(t,e);function t(r,o,n){var i=e.call(this)||this,s;if(I(r)||!r)s={next:r!=null?r:void 0,error:o!=null?o:void 0,complete:n!=null?n:void 0};else{var a;i&&De.useDeprecatedNextContext?(a=Object.create(r),a.unsubscribe=function(){return i.unsubscribe()},s={next:r.next&&Rr(r.next,a),error:r.error&&Rr(r.error,a),complete:r.complete&&Rr(r.complete,a)}):s=r}return i.destination=new ia(s),i}return t})(Pt);function er(e){De.useDeprecatedSynchronousErrorHandling?_o(e):Zt(e)}function aa(e){throw e}function Ir(e,t){var r=De.onStoppedNotification;r&&xt.setTimeout(function(){return r(e,t)})}var sa={closed:!0,next:ge,error:aa,complete:ge};var wt=(function(){return typeof Symbol=="function"&&Symbol.observable||"@@observable"})();function be(e){return e}function Ao(){for(var
Line truncated
|
||||
//# sourceMappingURL=bundle.79ae519e.min.js.map
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
{
|
||||
"version": 3,
|
||||
"sources": ["node_modules/focus-visible/dist/focus-visible.js", "node_modules/escape-html/index.js", "node_modules/clipboard/dist/clipboard.js", "src/templates/assets/javascripts/bundle.ts", "node_modules/tslib/tslib.es6.mjs", "node_modules/rxjs/src/internal/util/isFunction.ts", "node_modules/rxjs/src/internal/util/createErrorClass.ts", "node_modules/rxjs/src/internal/util/UnsubscriptionError.ts", "node_modules/rxjs/src/internal/util/arrRemove.ts", "node_modules/rxjs/src/internal/Subscription.ts", "node_modules/rxjs/src/internal/config.ts", "node_modules/rxjs/src/internal/scheduler/timeoutProvider.ts", "node_modules/rxjs/src/internal/util/reportUnhandledError.ts", "node_modules/rxjs/src/internal/util/noop.ts", "node_modules/rxjs/src/internal/NotificationFactories.ts", "node_modules/rxjs/src/internal/util/errorContext.ts", "node_modules/rxjs/src/internal/Subscriber.ts", "node_modules/rxjs/src/internal/symbol/observable.ts", "node_modules/rxjs/src/internal/util/identity.ts", "node_modules/rxjs/src/internal/util/pipe.ts", "node_modules/rxjs/src/internal/Observable.ts", "node_modules/rxjs/src/internal/util/lift.ts", "node_modules/rxjs/src/internal/operators/OperatorSubscriber.ts", "node_modules/rxjs/src/internal/scheduler/animationFrameProvider.ts", "node_modules/rxjs/src/internal/util/ObjectUnsubscribedError.ts", "node_modules/rxjs/src/internal/Subject.ts", "node_modules/rxjs/src/internal/BehaviorSubject.ts", "node_modules/rxjs/src/internal/scheduler/dateTimestampProvider.ts", "node_modules/rxjs/src/internal/ReplaySubject.ts", "node_modules/rxjs/src/internal/scheduler/Action.ts", "node_modules/rxjs/src/internal/scheduler/intervalProvider.ts", "node_modules/rxjs/src/internal/scheduler/AsyncAction.ts", "node_modules/rxjs/src/internal/Scheduler.ts", "node_modules/rxjs/src/internal/scheduler/AsyncScheduler.ts", "node_modules/rxjs/src/internal/scheduler/async.ts", "node_modules/rxjs/src/internal/scheduler/QueueAction.ts", "node_modules/rxjs/src/internal/scheduler/QueueScheduler.ts", "node_modules/rxjs/src/internal/scheduler/queue.ts", "node_modules/rxjs/src/internal/scheduler/AnimationFrameAction.ts", "node_modules/rxjs/src/internal/scheduler/AnimationFrameScheduler.ts", "node_modules/rxjs/src/internal/scheduler/animationFrame.ts", "node_modules/rxjs/src/internal/observable/empty.ts", "node_modules/rxjs/src/internal/util/isScheduler.ts", "node_modules/rxjs/src/internal/util/args.ts", "node_modules/rxjs/src/internal/util/isArrayLike.ts", "node_modules/rxjs/src/internal/util/isPromise.ts", "node_modules/rxjs/src/internal/util/isInteropObservable.ts", "node_modules/rxjs/src/internal/util/isAsyncIterable.ts", "node_modules/rxjs/src/internal/util/throwUnobservableError.ts", "node_modules/rxjs/src/internal/symbol/iterator.ts", "node_modules/rxjs/src/internal/util/isIterable.ts", "node_modules/rxjs/src/internal/util/isReadableStreamLike.ts", "node_modules/rxjs/src/internal/observable/innerFrom.ts", "node_modules/rxjs/src/internal/util/executeSchedule.ts", "node_modules/rxjs/src/internal/operators/observeOn.ts", "node_modules/rxjs/src/internal/operators/subscribeOn.ts", "node_modules/rxjs/src/internal/scheduled/scheduleObservable.ts", "node_modules/rxjs/src/internal/scheduled/schedulePromise.ts", "node_modules/rxjs/src/internal/scheduled/scheduleArray.ts", "node_modules/rxjs/src/internal/scheduled/scheduleIterable.ts", "node_modules/rxjs/src/internal/scheduled/scheduleAsyncIterable.ts", "node_modules/rxjs/src/internal/scheduled/scheduleReadableStreamLike.ts", "node_modules/rxjs/src/internal/scheduled/scheduled.ts", "node_modules/rxjs/src/internal/observable/from.ts", "node_modules/rxjs/src/internal/observable/of.ts", "node_modules/rxjs/src/internal/observable/throwError.ts", "node_modules/rxjs/src/internal/util/EmptyError.ts", "node_modules/rxjs/src/internal/util/isDate.ts", "node_modules/rxjs/src/internal/operators/map.ts", "node_modules/rxjs/src/internal/util/mapOneOrManyArgs.ts", "node_modules/rxjs/src/internal/util/argsArgArrayOrObject.ts", "node_modules/rxjs/src/internal/util/createObject.ts", "node_modules/rxjs/src/internal/observable/combineLatest.ts", "node_modules/rxjs/src/internal/operators/mergeInternals.ts", "node_modules/rxjs/src/internal/operators/mergeMap.ts", "node_modules/rxjs/src/internal/operators/mergeAll.ts", "node_modules/rxjs/src/internal/operators/concatAll.ts", "node_modules/rxjs/src/internal/observable/concat.ts", "node_modules/rxjs/src/internal/observable/defer.ts", "node_modules/rxjs/src/internal/observable/fromEvent.ts", "node_modules/rxjs/src/internal/observable/fromEventPattern.ts", "node_modules/rxjs/src/internal/observable/timer.ts", "node_modules/rxjs/src/internal/observable/merge.ts", "node_modules/rxjs/src/internal/observable/never.ts", "node_modules/rxjs/src/internal/util/argsOrArgArray.ts", "node_modules/rxjs/src/internal/operators/filter.ts", "node_modules/rxjs/src/internal/observable/zip.ts", "node_modules/rxjs/src/internal/operators/audit.ts", "node_modules/rxjs/src/internal/operLine truncated
|
||||
"sourcesContent": ["(function (global, factory) {\n typeof exports === 'object' && typeof module !== 'undefined' ? factory() :\n typeof define === 'function' && define.amd ? define(factory) :\n (factory());\n}(this, (function () { 'use strict';\n\n /**\n * Applies the :focus-visible polyfill at the given scope.\n * A scope in this case is either the top-level Document or a Shadow Root.\n *\n * @param {(Document|ShadowRoot)} scope\n * @see https://github.com/WICG/focus-visible\n */\n function applyFocusVisiblePolyfill(scope) {\n var hadKeyboardEvent = true;\n var hadFocusVisibleRecently = false;\n var hadFocusVisibleRecentlyTimeout = null;\n\n var inputTypesAllowlist = {\n text: true,\n search: true,\n url: true,\n tel: true,\n email: true,\n password: true,\n number: true,\n date: true,\n month: true,\n week: true,\n time: true,\n datetime: true,\n 'datetime-local': true\n };\n\n /**\n * Helper function for legacy browsers and iframes which sometimes focus\n * elements like document, body, and non-interactive SVG.\n * @param {Element} el\n */\n function isValidFocusTarget(el) {\n if (\n el &&\n el !== document &&\n el.nodeName !== 'HTML' &&\n el.nodeName !== 'BODY' &&\n 'classList' in el &&\n 'contains' in el.classList\n ) {\n return true;\n }\n return false;\n }\n\n /**\n * Computes whether the given element should automatically trigger the\n * `focus-visible` class being added, i.e. whether it should always match\n * `:focus-visible` when focused.\n * @param {Element} el\n * @return {boolean}\n */\n function focusTriggersKeyboardModality(el) {\n var type = el.type;\n var tagName = el.tagName;\n\n if (tagName === 'INPUT' && inputTypesAllowlist[type] && !el.readOnly) {\n return true;\n }\n\n if (tagName === 'TEXTAREA' && !el.readOnly) {\n return true;\n }\n\n if (el.isContentEditable) {\n return true;\n }\n\n return false;\n }\n\n /**\n * Add the `focus-visible` class to the given element if it was not added by\n * the author.\n * @param {Element} el\n */\n function addFocusVisibleClass(el) {\n if (el.classList.contains('focus-visible')) {\n return;\n }\n el.classList.add('focus-visible');\n el.setAttribute('data-focus-visible-added', '');\n }\n\n /**\n * Remove the `focus-visible` class from the given element if it was not\n * originally added by the author.\n * @param {Element} el\n */\n function removeFocusVisibleClass(el) {\n if (!el.hasAttribute('data-focus-visible-added')) {\n return;\n }\n el.classList.remove('focus-visible');\n el.removeAttribute('data-focus-visible-added');\n }\n\n /**\n * If the most recent user interaction was via the keyboard;\n * and the key press did not include a meta, alt/option, or control key;\n * then the modality is keyboard. Otherwise, the modality is not keyboard.\n * Apply `focus-visible` to any current active element and keep track\n * of our keyboard modality state with `hadKeyboardEvent`.\n * @param {KeyboardEvent} e\n */\n function onKeyDown(e) {\n if (e.metaKey || e.altKey || e.ctrlKey) {\n return;\n }\n\n if (isValidFocusTarget(scope.activeElement)) {\n addFocusVisibleClass(scope.activeElement);\n }\n\n hadKeyboardEvent = true;\n }\n\n /**\n * If at any point a user clicks with a pointing device, ensure that we change\n * the modality away from keyboard.\n * This avoids the situation where a user presses a key on an already focused\n * element, and then clicks on a different element, focusing it with a\n * pointing device, while we still think we're in keyboard modality.\n * @param {Event} e\n */\n function onPointerDown(e) {\n hadKeyboardEvent = false;\n }\n\n /**\n * On `focus`, add the `focus-visible` class to the target if:\n * - the target received focus as a result of keyboard navigation, or\n * - the event target is an element that will likely require interaction\n * via the keyboard (e.g. a text box)\n * @param {Event} e\n */\n function onFocus(e) {\n // Prevent IE from focusing the document or HTML element.\n if (!isValidFocusTarget(e.target)) {\n return;\n }\n\n if (hadKeyboardEvent || focusTriggersKeyboardModality(e.target)) {\n addFocusVisibleClass(e.target);\n }\n }\n\n /**\n * On `blur`, remove the `focus-visible` class from the target.\n * @param {Event} e\n */\n function onBlur(e) {\n if (!isValidFocusTarget(e.target)) {\n return;\n }\n\n if (\n e.target.classList.contains('focus-visible') ||\n e.target.hasAttributeLine truncated
|
||||
"mappings": "2rCAAA,IAAAA,GAAAC,GAAA,CAAAC,GAAAC,KAAA,EAAC,SAAUC,EAAQC,EAAS,CAC1B,OAAOH,IAAY,UAAY,OAAOC,IAAW,YAAcE,EAAQ,EACvE,OAAO,QAAW,YAAc,OAAO,IAAM,OAAOA,CAAO,EAC1DA,EAAQ,CACX,GAAEH,IAAO,UAAY,CAAE,aASrB,SAASI,EAA0BC,EAAO,CACxC,IAAIC,EAAmB,GACnBC,EAA0B,GAC1BC,EAAiC,KAEjCC,EAAsB,CACxB,KAAM,GACN,OAAQ,GACR,IAAK,GACL,IAAK,GACL,MAAO,GACP,SAAU,GACV,OAAQ,GACR,KAAM,GACN,MAAO,GACP,KAAM,GACN,KAAM,GACN,SAAU,GACV,iBAAkB,EACpB,EAOA,SAASC,EAAmBC,EAAI,CAC9B,MACE,GAAAA,GACAA,IAAO,UACPA,EAAG,WAAa,QAChBA,EAAG,WAAa,QAChB,cAAeA,GACf,aAAcA,EAAG,UAKrB,CASA,SAASC,EAA8BD,EAAI,CACzC,IAAIE,GAAOF,EAAG,KACVG,GAAUH,EAAG,QAUjB,MARI,GAAAG,KAAY,SAAWL,EAAoBI,EAAI,GAAK,CAACF,EAAG,UAIxDG,KAAY,YAAc,CAACH,EAAG,UAI9BA,EAAG,kBAKT,CAOA,SAASI,EAAqBJ,EAAI,CAC5BA,EAAG,UAAU,SAAS,eAAe,IAGzCA,EAAG,UAAU,IAAI,eAAe,EAChCA,EAAG,aAAa,2BAA4B,EAAE,EAChD,CAOA,SAASK,EAAwBL,EAAI,CAC9BA,EAAG,aAAa,0BAA0B,IAG/CA,EAAG,UAAU,OAAO,eAAe,EACnCA,EAAG,gBAAgB,0BAA0B,EAC/C,CAUA,SAASM,EAAUC,EAAG,CAChBA,EAAE,SAAWA,EAAE,QAAUA,EAAE,UAI3BR,EAAmBL,EAAM,aAAa,GACxCU,EAAqBV,EAAM,aAAa,EAG1CC,EAAmB,GACrB,CAUA,SAASa,EAAcD,EAAG,CACxBZ,EAAmB,EACrB,CASA,SAASc,EAAQF,EAAG,CAEbR,EAAmBQ,EAAE,MAAM,IAI5BZ,GAAoBM,EAA8BM,EAAE,MAAM,IAC5DH,EAAqBG,EAAE,MAAM,CAEjC,CAMA,SAASG,EAAOH,EAAG,CACZR,EAAmBQ,EAAE,MAAM,IAK9BA,EAAE,OAAO,UAAU,SAAS,eAAe,GAC3CA,EAAE,OAAO,aAAa,0BAA0B,KAMhDX,EAA0B,GAC1B,OAAO,aAAaC,CAA8B,EAClDA,EAAiC,OAAO,WAAW,UAAW,CAC5DD,EAA0B,EAC5B,EAAG,GAAG,EACNS,EAAwBE,EAAE,MAAM,EAEpC,CAOA,SAASI,EAAmBJ,EAAG,CACzB,SAAS,kBAAoB,WAK3BX,IACFD,EAAmB,IAErBiB,EAA+B,EAEnC,CAQA,SAASA,GAAiC,CACxC,SAAS,iBAAiB,YAAaC,EAAoB,EAC3D,SAAS,iBAAiB,YAAaA,EAAoB,EAC3D,SAAS,iBAAiB,UAAWA,EAAoB,EACzD,SAAS,iBAAiB,cAAeA,EAAoB,EAC7D,SAAS,iBAAiB,cAAeA,EAAoB,EAC7D,SAAS,iBAAiB,YAAaA,EAAoB,EAC3D,SAAS,iBAAiB,YAAaA,EAAoB,EAC3D,SAAS,iBAAiB,aAAcA,EAAoB,EAC5D,SAAS,iBAAiB,WAAYA,EAAoB,CAC5D,CAEA,SAASC,IAAoC,CAC3C,SAAS,oBAAoB,YAAaD,EAAoB,EAC9D,SAAS,oBAAoB,YAAaA,EAAoB,EAC9D,SAAS,oBAAoB,UAAWA,EAAoB,EAC5D,SAAS,oBAAoB,cAAeA,EAAoB,EAChE,SAAS,oBAAoB,cAAeA,EAAoB,EAChE,SAAS,oBAAoB,YAAaA,EAAoB,EAC9D,SAAS,oBAAoB,YAAaA,EAAoB,EAC9D,SAAS,oBAAoB,aAAcA,EAAoB,EAC/D,SAAS,oBAAoB,WAAYA,EAAoB,CAC/D,CASA,SAASA,GAAqBN,EAAG,CAG3BA,EAAE,OAAO,UAAYA,EAAE,OAAO,SAAS,YAAY,IAAM,SAI7DZ,EAAmB,GACnBmB,GAAkC,EACpC,CAKA,SAAS,iBAAiB,UAAWR,EAAW,EAAI,EACpD,SAAS,iBAAiB,YAAaE,EAAe,EAAI,EAC1D,SAAS,iBAAiB,cAAeA,EAAe,EAAI,EAC5D,SAAS,iBAAiB,aAAcA,EAAe,EAAI,EAC3D,SAAS,iBAAiB,mBAAoBG,EAAoB,EAAI,EAEtEC,EAA+B,EAM/BlB,EAAM,iBAAiB,QAASe,EAAS,EAAI,EAC7Cf,EAAM,iBAAiB,OAAQgB,EAAQ,EAAI,EAOvChB,EAAM,WAAa,KAAK,wBAA0BA,EAAM,KAI1DA,EAAM,KAAK,aAAa,wBAAyB,EAAE,EAC1CA,EAAM,WAAa,KAAK,gBACjC,SAAS,gBAAgB,UAAU,IAAI,kBAAkB,EACzD,SAAS,gBAAgB,aAAa,wBAAyB,EAAE,EAErE,CAKA,GAAI,OAAO,QAAW,aAAe,OAAO,UAAa,YAAa,CAIpE,OAAO,0BAA4BD,EAInC,IAAIsB,EAEJ,GAAI,CACFA,EAAQ,IAAI,YAAY,8BAA8B,CACxD,OAASC,EAAO,CAEdD,EAAQ,SAAS,YAAY,aAAa,EAC1CA,EAAM,gBAAgB,+BAAgC,GAAO,GAAO,CAAC,CAAC,CACxE,CAEA,OAAO,cAAcA,CAAK,CAC5B,CAEI,OAAO,UAAa,aAGtBtB,EAA0B,QAAQ,CAGtC,EAAE,ICvTF,IAAAwB,GAAAC,GAAA,CAAAC,GAAAC,KAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAeA,IAAIC,GAAkB,UAOtBD,GAAO,QAAUE,GAUjB,SAASA,GAAWC,EAAQ,CAC1B,IAAIC,EAAM,GAAKD,EACXE,EAAQJ,GAAgB,KAAKG,CAAG,EAEpC,GAAI,CAACC,EACH,OAAOD,EAGT,IAAIE,EACAC,EAAO,GACPC,EAAQ,EACRC,EAAY,EAEhB,IAAKD,EAAQH,EAAM,MAAOG,EAAQJ,EAAI,OAAQI,IAAS,CACrD,OAAQJ,EAAI,WAAWI,CAAK,EAAG,CAC7B,IAAK,IACHF,EAAS,SACT,MACF,IAAK,IACHA,EAAS,QACT,MACF,IAAK,IACHA,EAAS,QACT,MACF,IAAK,IACHA,EAAS,OACT,MACF,IAAK,IACHA,EAAS,OACT,MACF,QACE,QACJ,CAEIG,IAAcD,IAChBD,GAAQH,EAAI,UAAUK,EAAWD,CAAK,GAGxCC,EAAYD,EAAQ,EACpBD,GAAQD,CACV,CAEA,OAAOG,IAAcD,EACjBD,EAAOH,EAAI,UAAUK,EAAWD,CAAK,EACrCD,CACN,IC7EA,IAAAG,GAAAC,GAAA,CAAAC,GAAAC,KAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAMC,SAA0CC,EAAMC,EAAS,CACtD,OAAOH,IAAY,UAAY,OAAOC,IAAW,SACnDA,GAAO,QAAUE,EAAQ,EAClB,OAAO,QAAW,YAAc,OAAO,IAC9C,OAAO,CAAC,EAAGA,CAAO,EACX,OAAOH,IAAY,SAC1BA,GAAQ,YAAiBG,EAAQ,EAEjCD,EAAK,YAAiBC,EAAQ,CAChC,GAAGH,GAAM,UAAW,CACpB,OAAiB,UAAW,CAClB,IAAII,EAAuB,CAE/B,KACC,SAASC,EAAyBC,EAAqBC,EAAqB,CAEnF,aAGAA,EAAoB,EAAED,EAAqB,CACzC,QAAW,UAAW,CAAE,OAAqBE,EAAW,CAC1D,CAAC,EAGD,IAAIC,EAAeF,EAAoB,GAAG,EACtCG,EAAoCH,EAAoB,EAAEE,CAAY,EAEtEE,EAASJ,EAAoB,GAAG,EAChCK,EAA8BL,EAAoB,EAAEI,CAAM,EAE1DE,EAAaN,EAAoB,GAAG,EACpCO,EAA8BP,EAAoB,EAAEM,CAAU,EAOlE,SAASE,EAAQC,EAAM,CACrB,GAAI,CACF,OAAO,SAAS,YAAYA,CAAI,CAClC,OAASC,EAAK,CACZ,MAAO,EACT,CACF,CAUA,IAAIC,EAAqB,SAA4BC,EAAQ,CAC3D,IAAIC,EAAeN,EAAe,EAAEK,CAAM,EAC1C,OAAAJ,EAAQ,KAAK,EACNK,CACT,EAEiCC,EAAeH,EAOhD,SAASI,EAAkBC,EAAO,CAChC,IAAIC,EAAQ,SAAS,gBAAgB,aAAa,KAAK,IAAM,MACzDC,EAAc,SAAS,cAAc,UAAU,EAEnDA,EAAY,MAAM,SAAW,OAE7BA,EAAY,MAAM,OAAS,IAC3BA,EAAY,MAAM,QAAU,IAC5BA,EAAY,MAAM,OAAS,IAE3BA,EAAY,MAAM,SAAW,WAC7BA,EAAY,MAAMD,EAAQ,QAAU,MAAM,EAAI,UAE9C,IAAIE,EAAY,OAAO,aAAe,SAAS,gBAAgB,UAC/D,OAAAD,EAAY,MAAM,IAAM,GAAG,OAAOC,EAAW,IAAI,EACjDD,EAAY,aAAa,WAAY,EAAE,EACvCA,EAAY,MAAQF,EACbE,CACT,CAYA,IAAIE,EAAiB,SAAwBJ,EAAOK,EAAS,CAC3D,IAAIH,EAAcH,EAAkBC,CAAK,EACzCK,EAAQ,UAAU,YAAYH,CAAW,EACzC,IAAIL,EAAeN,EAAe,EAAEW,CAAW,EAC/C,OAAAV,EAAQ,MAAM,EACdU,EAAY,OAAO,EACZL,CACT,EASIS,GAAsB,SAA6BV,EAAQ,CAC7D,IAAIS,EAAU,UAAU,OAAS,GAAK,UAAU,CAAC,IAAM,OAAY,UAAU,CAAC,EAAI,CAChF,UAAW,SAAS,IALine truncated
|
||||
"names": ["require_focus_visible", "__commonJSMin", "exports", "module", "global", "factory", "applyFocusVisiblePolyfill", "scope", "hadKeyboardEvent", "hadFocusVisibleRecently", "hadFocusVisibleRecentlyTimeout", "inputTypesAllowlist", "isValidFocusTarget", "el", "focusTriggersKeyboardModality", "type", "tagName", "addFocusVisibleClass", "removeFocusVisibleClass", "onKeyDown", "e", "onPointerDown", "onFocus", "onBlur", "onVisibilityChange", "addInitialPointerMoveListeners", "onInitialPointerMove", "removeInitialPointerMoveListeners", "event", "error", "require_escape_html", "__commonJSMin", "exports", "module", "matchHtmlRegExp", "escapeHtml", "string", "str", "match", "escape", "html", "index", "lastIndex", "require_clipboard", "__commonJSMin", "exports", "module", "root", "factory", "__webpack_modules__", "__unused_webpack_module", "__webpack_exports__", "__webpack_require__", "clipboard", "tiny_emitter", "tiny_emitter_default", "listen", "listen_default", "src_select", "select_default", "command", "type", "err", "ClipboardActionCut", "target", "selectedText", "actions_cut", "createFakeElement", "value", "isRTL", "fakeElement", "yPosition", "fakeCopyAction", "options", "ClipboardActionCopy", "actions_copy", "_typeof", "obj", "ClipboardActionDefault", "_options$action", "action", "container", "text", "actions_default", "clipboard_typeof", "_classCallCheck", "instance", "Constructor", "_defineProperties", "props", "i", "descriptor", "_createClass", "protoProps", "staticProps", "_inherits", "subClass", "superClass", "_setPrototypeOf", "o", "p", "_createSuper", "Derived", "hasNativeReflectConstruct", "_isNativeReflectConstruct", "Super", "_getPrototypeOf", "result", "NewTarget", "_possibleConstructorReturn", "self", "call", "_assertThisInitialized", "e", "getAttributeValue", "suffix", "element", "attribute", "Clipboard", "_Emitter", "_super", "trigger", "_this", "_this2", "selector", "actions", "support", "DOCUMENT_NODE_TYPE", "proto", "closest", "__unused_webpack_exports", "_delegate", "callback", "useCapture", "listenerFn", "listener", "delegate", "elements", "is", "listenNode", "listenNodeList", "listenSelector", "node", "nodeList", "select", "isReadOnly", "selection", "range", "E", "name", "ctx", "data", "evtArr", "len", "evts", "liveEvents", "__webpack_module_cache__", "moduleId", "getter", "definition", "key", "prop", "import_focus_visible", "extendStatics", "d", "b", "p", "__extends", "__", "__awaiter", "thisArg", "_arguments", "P", "generator", "adopt", "value", "resolve", "reject", "fulfilled", "step", "e", "rejected", "result", "__generator", "body", "_", "t", "f", "y", "g", "verb", "n", "v", "op", "__values", "o", "s", "m", "i", "__read", "n", "r", "ar", "e", "error", "__spreadArray", "to", "from", "pack", "i", "l", "ar", "__await", "v", "__asyncGenerator", "thisArg", "_arguments", "generator", "g", "q", "verb", "awaitReturn", "f", "reject", "n", "a", "b", "resume", "step", "e", "settle", "r", "fulfill", "value", "__asyncValues", "o", "m", "i", "__values", "verb", "n", "v", "resolve", "reject", "settle", "d", "isFunction", "value", "createErrorClass", "createImpl", "_super", "instance", "ctorFunc", "UnsubscriptionError", "createErrorClass", "_super", "errors", "err", "i", "arrRemove", "arr", "item", "index", "Subscription", "initialTeardown", "errors", "_parentage", "_parentage_1", "__values", "_parentage_1_1", "parent_1", "initialFinalizer", "isFunction", "e", "UnsubscriptionError", "_finalizers", "_finalizers_1", "_finalizers_1_1", "finalizer", "execFinalizer", "err", "__spreadArray", "__read", "teardown", "_a", "parent", "arrRemove", "empty", "EMPTY_SUBSCRIPTION", "Subscription", "isSubscription", "value", "isFunction", "execFinalizer", "finalizer", "config", "timeoutProvider", "handler", "timeout", "args", "_i", "delegate", "__spreadArray", "__read", "handle", "reportUnhandledError", "err", "timeoutProvider", "onUnhandledError", "config", "noop", "COMPLETE_NOTIFICATION", "createNotification", "errorNotification", "error", "nextNotification", "value", "kind", "context", "errorContext", "cb", "config", "isRoot", "_a", "errorThrown", "error", "captureError", "err", "Subscriber", "_super", "__extends", "destination", "_this", "isSubscription", "EMPTY_OBSERVER", "next", "error", "complete", "SafeSubscriber", "value", "handleStoppedNotification", "nextNotification", "err", "errorNotification", "COMPLETE_NOTIFICATION", "Subscription", "_bind", "bind", "fn", "thisArg", "ConsumerObserver", "partialObserver", "value", "error", "handleUnhandledError", "err", "SafeSubscriber", "_super", "__extends", "observerOrNext", "complete", "_this", "isFunction", "context_1", "config", "Subscriber", "handleUnhandledError", "error", "config", "captureError", "reportUnhandledError", "defaultErrorHandler", "err", "handleStoppedNotification", "notification", "subscriber", "onStoppedNotification", "timeoutProvider", "EMPTY_OBSERVER", "noop", "observable", "identity", "x", "pipe", "fns", "_i", "pipeFromArray", "identity"Line truncated
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.ar=function(){this.pipeline.reset(),this.pipeline.add(e.ar.trimmer,e.ar.stopWordFilter,e.ar.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.ar.stemmer))},e.ar.wordCharacters="ء-ٛٱـ",e.ar.trimmer=e.trimmerSupport.generateTrimmer(e.ar.wordCharacters),e.Pipeline.registerFunction(e.ar.trimmer,"trimmer-ar"),e.ar.stemmer=function(){var e=this;return e.result=!1,e.preRemoved=!1,e.sufRemoved=!1,e.pre={pre1:"ف ك ب و س ل ن ا ي ت",pre2:"ال لل",pre3:"بال وال فال تال كال ولل",pre4:"فبال كبال وبال وكال"},e.suf={suf1:"ه ك ت ن ا ي",suf2:"نك نه ها وك يا اه ون ين تن تم نا وا ان كم كن ني نن ما هم هن تك ته ات يه",suf3:"تين كهم نيه نهم ونه وها يهم ونا ونك وني وهم تكم تنا تها تني تهم كما كها ناه نكم هنا تان يها",suf4:"كموه ناها ونني ونهم تكما تموه تكاه كماه ناكم ناهم نيها وننا"},e.patterns=JSON.parse('{"pt43":[{"pt":[{"c":"ا","l":1}]},{"pt":[{"c":"ا,ت,ن,ي","l":0}],"mPt":[{"c":"ف","l":0,"m":1},{"c":"ع","l":1,"m":2},{"c":"ل","l":2,"m":3}]},{"pt":[{"c":"و","l":2}],"mPt":[{"c":"ف","l":0,"m":0},{"c":"ع","l":1,"m":1},{"c":"ل","l":2,"m":3}]},{"pt":[{"c":"ا","l":2}]},{"pt":[{"c":"ي","l":2}],"mPt":[{"c":"ف","l":0,"m":0},{"c":"ع","l":1,"m":1},{"c":"ا","l":2},{"c":"ل","l":3,"m":3}]},{"pt":[{"c":"م","l":0}]}],"pt53":[{"pt":[{"c":"ت","l":0},{"c":"ا","l":2}]},{"pt":[{"c":"ا,ن,ت,ي","l":0},{"c":"ت","l":2}],"mPt":[{"c":"ا","l":0},{"c":"ف","l":1,"m":1},{"c":"ت","l":2},{"c":"ع","l":3,"m":3},{"c":"ا","l":4},{"c":"ل","l":5,"m":4}]},{"pt":[{"c":"ا","l":0},{"c":"ا","l":2}],"mPt":[{"c":"ا","l":0},{"c":"ف","l":1,"m":1},{"c":"ع","l":2,"m":3},{"c":"ل","l":3,"m":4},{"c":"ا","l":4},{"c":"ل","l":5,"m":4}]},{"pt":[{"c":"ا","l":0},{"c":"ا","l":3}],"mPt":[{"c":"ف","l":0,"m":1},{"c":"ع","l":1,"m":2},{"c":"ل","l":2,"m":4}]},{"pt":[{"c":"ا","l":3},{"c":"ن","l":4}]},{"pt":[{"c":"ت","l":0},{"c":"ي","l":3}]},{"pt":[{"c":"م","l":0},{"c":"و","l":3}]},{"pt":[{"c":"ا","l":1},{"c":"و","l":3}]},{"pt":[{"c":"و","l":1},{"c":"ا","l":2}]},{"pt":[{"c":"م","l":0},{"c":"ا","l":3}]},{"pt":[{"c":"م","l":0},{"c":"ي","l":3}]},{"pt":[{"c":"ا","l":2},{"c":"ن","l":3}]},{"pt":[{"c":"م","l":0},{"c":"ن","l":1}],"mPt":[{"c":"ا","l":0},{"c":"ن","l":1},{"c":"ف","l":2,"m":2},{"c":"ع","l":3,"m":3},{"c":"ا","l":4},{"c":"ل","l":5,"m":4}]},{"pt":[{"c":"م","l":0},{"c":"ت","l":2}],"mPt":[{"c":"ا","l":0},{"c":"ف","l":1,"m":1},{"c":"ت","l":2},{"c":"ع","l":3,"m":3},{"c":"ا","l":4},{"c":"ل","l":5,"m":4}]},{"pt":[{"c":"م","l":0},{"c":"ا","l":2}]},{"pt":[{"c":"م","l":1},{"c":"ا","l":3}]},{"pt":[{"c":"ي,ت,ا,ن","l":0},{"c":"ت","l":1}],"mPt":[{"c":"ف","l":0,"m":2},{"c":"ع","l":1,"m":3},{"c":"ا","l":2},{"c":"ل","l":3,"m":4}]},{"pt":[{"c":"ت,ي,ا,ن","l":0},{"c":"ت","l":2}],"mPt":[{"c":"ا","l":0},{"c":"ف","l":1,"m":1},{"c":"ت","l":2},{"c":"ع","l":3,"m":3},{"c":"ا","l":4},{"c":"ل","l":5,"m":4}]},{"pt":[{"c":"ا","l":2},{"c":"ي","l":3}]},{"pt":[{"c":"ا,ي,ت,ن","l":0},{"c":"ن","l":1}],"mPt":[{"c":"ا","l":0},{"c":"ن","l":1},{"c":"ف","l":2,"m":2},{"c":"ع","l":3,"m":3},{"c":"ا","l":4},{"c":"ل","l":5,"m":4}]},{"pt":[{"c":"ا","l":3},{"c":"ء","l":4}]}],"pt63":[{"pt":[{"c":"ا","l":0},{"c":"ت","l":2},{"c":"ا","l":4}]},{"pt":[{"c":"ا,ت,ن,ي","l":0},{"c":"س","l":1},{"c":"ت","l":2}],"mPt":[{"c":"ا","l":0},{"c":"س","l":1},{"c":"ت","l":2},{"c":"ف","l":3,"m":3},{"c":"ع","l":4,"m":4},{"c":"ا","l":5},{"c":"ل","l":6,"m":5}]},{"pt":[{"c":"ا,ن,ت,ي","l":0},{"c":"و","l":3}]},{"pt":[{"c":"م","l":0},{"c":"س","l":1},{"c":"ت","l":2}],"mPt":[{"c":"ا","l":0},{"c":"س","l":1},{"c":"ت","l":2},{"c":"ف","l":3,"m":3},{"c":"ع","l":4,"m":4},{"c":"ا","l":5},{"c":"ل","l":6,"m":5}]},{"pt":[{"c":"ي","l":1},{"c":"ي","l":3},{"c":"ا","l":4},{"c":"ء","l":5}]},{"pt":[{"c":"ا","l":0},{"c":"ن","l":1},{"c":"ا","l":4}]}],"pt54":[{"pt":[{"c":"ت","l":0}]},{"pt":[{"c":"ا,ي,ت,ن","l":0}],"mPt":[{"c":"ا","l":0},{"c":"ف","l":1,"m":1},{"c":"ع","l":2,"m":2},{"c":"ل","l":3,"m":3},{"c":"ر","l":4,"m":4},{"c":"ا","l":5},{"c":"ر","l":6,"m":4}]},{"pt":[{"c":"م","l":0}],"mPt":[{"c":"ا","l":0},{"c":"ف","l":1,"m":1},{"c":"ع","l":2,"m":2},{"c":"ل","l":3,"m":3},{"c":"ر","l":4,"m":4},{"c":"ا","l":5},{"c":"ر","l":6,"m":4}]},{"pt":[{"c":"ا","l":2}]},{"pt":[{"c":"ا","l":0},{"c":"ن","l":2}]}],"pt64":[{"pt":[{"c":"ا","l":0},{"c":"ا","l":4}]},{"pt":[{"c":"Line truncated
|
||||
@@ -1,18 +0,0 @@
|
||||
/*!
|
||||
* Lunr languages, `Danish` language
|
||||
* https://github.com/MihaiValentin/lunr-languages
|
||||
*
|
||||
* Copyright 2014, Mihai Valentin
|
||||
* http://www.mozilla.org/MPL/
|
||||
*/
|
||||
/*!
|
||||
* based on
|
||||
* Snowball JavaScript Library v0.3
|
||||
* http://code.google.com/p/urim/
|
||||
* http://snowball.tartarus.org/
|
||||
*
|
||||
* Copyright 2010, Oleg Mazko
|
||||
* http://www.mozilla.org/MPL/
|
||||
*/
|
||||
|
||||
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.da=function(){this.pipeline.reset(),this.pipeline.add(e.da.trimmer,e.da.stopWordFilter,e.da.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.da.stemmer))},e.da.wordCharacters="A-Za-zªºÀ-ÖØ-öø-ʸˠ-ˤᴀ-ᴥᴬ-ᵜᵢ-ᵥᵫ-ᵷᵹ-ᶾḀ-ỿⁱⁿₐ-ₜKÅℲⅎⅠ-ↈⱠ-ⱿꜢ-ꞇꞋ-ꞭꞰ-ꞷꟷ-ꟿꬰ-ꭚꭜ-ꭤff-stA-Za-z",e.da.trimmer=e.trimmerSupport.generateTrimmer(e.da.wordCharacters),e.Pipeline.registerFunction(e.da.trimmer,"trimmer-da"),e.da.stemmer=function(){var r=e.stemmerSupport.Among,i=e.stemmerSupport.SnowballProgram,n=new function(){function e(){var e,r=f.cursor+3;if(d=f.limit,0<=r&&r<=f.limit){for(a=r;;){if(e=f.cursor,f.in_grouping(w,97,248)){f.cursor=e;break}if(f.cursor=e,e>=f.limit)return;f.cursor++}for(;!f.out_grouping(w,97,248);){if(f.cursor>=f.limit)return;f.cursor++}d=f.cursor,d<a&&(d=a)}}function n(){var e,r;if(f.cursor>=d&&(r=f.limit_backward,f.limit_backward=d,f.ket=f.cursor,e=f.find_among_b(c,32),f.limit_backward=r,e))switch(f.bra=f.cursor,e){case 1:f.slice_del();break;case 2:f.in_grouping_b(p,97,229)&&f.slice_del()}}function t(){var e,r=f.limit-f.cursor;f.cursor>=d&&(e=f.limit_backward,f.limit_backward=d,f.ket=f.cursor,f.find_among_b(l,4)?(f.bra=f.cursor,f.limit_backward=e,f.cursor=f.limit-r,f.cursor>f.limit_backward&&(f.cursor--,f.bra=f.cursor,f.slice_del())):f.limit_backward=e)}function s(){var e,r,i,n=f.limit-f.cursor;if(f.ket=f.cursor,f.eq_s_b(2,"st")&&(f.bra=f.cursor,f.eq_s_b(2,"ig")&&f.slice_del()),f.cursor=f.limit-n,f.cursor>=d&&(r=f.limit_backward,f.limit_backward=d,f.ket=f.cursor,e=f.find_among_b(m,5),f.limit_backward=r,e))switch(f.bra=f.cursor,e){case 1:f.slice_del(),i=f.limit-f.cursor,t(),f.cursor=f.limit-i;break;case 2:f.slice_from("løs")}}function o(){var e;f.cursor>=d&&(e=f.limit_backward,f.limit_backward=d,f.ket=f.cursor,f.out_grouping_b(w,97,248)?(f.bra=f.cursor,u=f.slice_to(u),f.limit_backward=e,f.eq_v_b(u)&&f.slice_del()):f.limit_backward=e)}var a,d,u,c=[new r("hed",-1,1),new r("ethed",0,1),new r("ered",-1,1),new r("e",-1,1),new r("erede",3,1),new r("ende",3,1),new r("erende",5,1),new r("ene",3,1),new r("erne",3,1),new r("ere",3,1),new r("en",-1,1),new r("heden",10,1),new r("eren",10,1),new r("er",-1,1),new r("heder",13,1),new r("erer",13,1),new r("s",-1,2),new r("heds",16,1),new r("es",16,1),new r("endes",18,1),new r("erendes",19,1),new r("enes",18,1),new r("ernes",18,1),new r("eres",18,1),new r("ens",16,1),new r("hedens",24,1),new r("erens",24,1),new r("ers",16,1),new r("ets",16,1),new r("erets",28,1),new r("et",-1,1),new r("eret",30,1)],l=[new r("gd",-1,-1),new r("dt",-1,-1),new r("gt",-1,-1),new r("kt",-1,-1)],m=[new r("ig",-1,1),new r("lig",0,1),new r("elig",1,1),new r("els",-1,1),new r("løst",-1,2)],w=[17,65,16,1,0,0,0,0,0,0,0,0,0,0,0,0,48,0,128],p=[239,254,42,3,0,0,0,0,0,0,0,0,0,0,0,0,16],f=new i;this.setCurrent=function(e){f.setCurrent(e)},this.getCurrent=function(){return f.getCurrent()},this.stem=function(){var r=f.cursor;return e(),f.limit_backward=r,f.cursor=f.limit,n(),f.cursor=f.limit,t(),f.cursor=f.limit,s(),f.cursor=f.limit,o(),!0}};return function(e){return"function"==typeof e.update?e.update(function(e){return n.setCurrent(e),n.stem(),n.getCurrent()}):(n.setCurrent(e),n.stem(),n.getCurrent())}}(),e.Pipeline.registerFunction(e.da.stemmer,"stemmer-da"),e.da.stopWordFilter=e.generateStopWordFilter("ad af alle alt anden at blev blive bliver da de dem den denne der deres det dette dig din disse dog du efter eller en end er et for fra ham han hans har havde have hende hendes her hos hun hvad hvis hvor i ikke ind jeg jer jo kunne man mange med meget men mig min mine mit mod ned noget nogle nu når og også om op os over på selv sig sin sine sit skal skulle som sådan thi til ud under var vi vil ville vor være været".split(" ")),e.Pipeline.registerFunction(e.da.stopWordFilter,"stopWordFilter-da")}});
|
||||
@@ -1,18 +0,0 @@
|
||||
/*!
|
||||
* Lunr languages, `German` language
|
||||
* https://github.com/MihaiValentin/lunr-languages
|
||||
*
|
||||
* Copyright 2014, Mihai Valentin
|
||||
* http://www.mozilla.org/MPL/
|
||||
*/
|
||||
/*!
|
||||
* based on
|
||||
* Snowball JavaScript Library v0.3
|
||||
* http://code.google.com/p/urim/
|
||||
* http://snowball.tartarus.org/
|
||||
*
|
||||
* Copyright 2010, Oleg Mazko
|
||||
* http://www.mozilla.org/MPL/
|
||||
*/
|
||||
|
||||
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.de=function(){this.pipeline.reset(),this.pipeline.add(e.de.trimmer,e.de.stopWordFilter,e.de.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.de.stemmer))},e.de.wordCharacters="A-Za-zªºÀ-ÖØ-öø-ʸˠ-ˤᴀ-ᴥᴬ-ᵜᵢ-ᵥᵫ-ᵷᵹ-ᶾḀ-ỿⁱⁿₐ-ₜKÅℲⅎⅠ-ↈⱠ-ⱿꜢ-ꞇꞋ-ꞭꞰ-ꞷꟷ-ꟿꬰ-ꭚꭜ-ꭤff-stA-Za-z",e.de.trimmer=e.trimmerSupport.generateTrimmer(e.de.wordCharacters),e.Pipeline.registerFunction(e.de.trimmer,"trimmer-de"),e.de.stemmer=function(){var r=e.stemmerSupport.Among,n=e.stemmerSupport.SnowballProgram,i=new function(){function e(e,r,n){return!(!v.eq_s(1,e)||(v.ket=v.cursor,!v.in_grouping(p,97,252)))&&(v.slice_from(r),v.cursor=n,!0)}function i(){for(var r,n,i,s,t=v.cursor;;)if(r=v.cursor,v.bra=r,v.eq_s(1,"ß"))v.ket=v.cursor,v.slice_from("ss");else{if(r>=v.limit)break;v.cursor=r+1}for(v.cursor=t;;)for(n=v.cursor;;){if(i=v.cursor,v.in_grouping(p,97,252)){if(s=v.cursor,v.bra=s,e("u","U",i))break;if(v.cursor=s,e("y","Y",i))break}if(i>=v.limit)return void(v.cursor=n);v.cursor=i+1}}function s(){for(;!v.in_grouping(p,97,252);){if(v.cursor>=v.limit)return!0;v.cursor++}for(;!v.out_grouping(p,97,252);){if(v.cursor>=v.limit)return!0;v.cursor++}return!1}function t(){m=v.limit,l=m;var e=v.cursor+3;0<=e&&e<=v.limit&&(d=e,s()||(m=v.cursor,m<d&&(m=d),s()||(l=v.cursor)))}function o(){for(var e,r;;){if(r=v.cursor,v.bra=r,!(e=v.find_among(h,6)))return;switch(v.ket=v.cursor,e){case 1:v.slice_from("y");break;case 2:case 5:v.slice_from("u");break;case 3:v.slice_from("a");break;case 4:v.slice_from("o");break;case 6:if(v.cursor>=v.limit)return;v.cursor++}}}function c(){return m<=v.cursor}function u(){return l<=v.cursor}function a(){var e,r,n,i,s=v.limit-v.cursor;if(v.ket=v.cursor,(e=v.find_among_b(w,7))&&(v.bra=v.cursor,c()))switch(e){case 1:v.slice_del();break;case 2:v.slice_del(),v.ket=v.cursor,v.eq_s_b(1,"s")&&(v.bra=v.cursor,v.eq_s_b(3,"nis")&&v.slice_del());break;case 3:v.in_grouping_b(g,98,116)&&v.slice_del()}if(v.cursor=v.limit-s,v.ket=v.cursor,(e=v.find_among_b(f,4))&&(v.bra=v.cursor,c()))switch(e){case 1:v.slice_del();break;case 2:if(v.in_grouping_b(k,98,116)){var t=v.cursor-3;v.limit_backward<=t&&t<=v.limit&&(v.cursor=t,v.slice_del())}}if(v.cursor=v.limit-s,v.ket=v.cursor,(e=v.find_among_b(_,8))&&(v.bra=v.cursor,u()))switch(e){case 1:v.slice_del(),v.ket=v.cursor,v.eq_s_b(2,"ig")&&(v.bra=v.cursor,r=v.limit-v.cursor,v.eq_s_b(1,"e")||(v.cursor=v.limit-r,u()&&v.slice_del()));break;case 2:n=v.limit-v.cursor,v.eq_s_b(1,"e")||(v.cursor=v.limit-n,v.slice_del());break;case 3:if(v.slice_del(),v.ket=v.cursor,i=v.limit-v.cursor,!v.eq_s_b(2,"er")&&(v.cursor=v.limit-i,!v.eq_s_b(2,"en")))break;v.bra=v.cursor,c()&&v.slice_del();break;case 4:v.slice_del(),v.ket=v.cursor,e=v.find_among_b(b,2),e&&(v.bra=v.cursor,u()&&1==e&&v.slice_del())}}var d,l,m,h=[new r("",-1,6),new r("U",0,2),new r("Y",0,1),new r("ä",0,3),new r("ö",0,4),new r("ü",0,5)],w=[new r("e",-1,2),new r("em",-1,1),new r("en",-1,2),new r("ern",-1,1),new r("er",-1,1),new r("s",-1,3),new r("es",5,2)],f=[new r("en",-1,1),new r("er",-1,1),new r("st",-1,2),new r("est",2,1)],b=[new r("ig",-1,1),new r("lich",-1,1)],_=[new r("end",-1,1),new r("ig",-1,2),new r("ung",-1,1),new r("lich",-1,3),new r("isch",-1,2),new r("ik",-1,2),new r("heit",-1,3),new r("keit",-1,4)],p=[17,65,16,1,0,0,0,0,0,0,0,0,0,0,0,0,8,0,32,8],g=[117,30,5],k=[117,30,4],v=new n;this.setCurrent=function(e){v.setCurrent(e)},this.getCurrent=function(){return v.getCurrent()},this.stem=function(){var e=v.cursor;return i(),v.cursor=e,t(),v.limit_backward=e,v.cursor=v.limit,a(),v.cursor=v.limit_backward,o(),!0}};return function(e){return"function"==typeof e.update?e.update(function(e){return i.setCurrent(e),i.stem(),i.getCurrent()}):(i.setCurrent(e),i.stem(),i.getCurrent())}}(),e.Pipeline.registerFunction(e.de.stemmer,"stemmer-de"),e.de.stopWordFilter=e.generateStopWordFilter("aber alle allem allen aller alles als also am an ander andere anderem anderen anderer anderes anderm andern anderr anders auch auf aus bei bin bis bist da damit dann das dasselbe dazu daß dein deine deinem deinen deiner deines dem demselben den denn denselben der derer derselbe derselben des desselben dessen dich die dies diese dieselbe dieselben diesem diesen dieser dieses dir doch dort du durch ein eine einem einen einer eines einig einige einigem einigen einiger einiges einmal er es etwas euch euer eure eurem euren eurer eures für gegen gewesen hab habe haben hat hatte hatten hier hin hinter ich ihm ihn ihnen ihr ihre ihrem ihren ihrer ihres im in indemLine truncated
|
||||
@@ -1,18 +0,0 @@
|
||||
/*!
|
||||
* Lunr languages, `Dutch` language
|
||||
* https://github.com/MihaiValentin/lunr-languages
|
||||
*
|
||||
* Copyright 2014, Mihai Valentin
|
||||
* http://www.mozilla.org/MPL/
|
||||
*/
|
||||
/*!
|
||||
* based on
|
||||
* Snowball JavaScript Library v0.3
|
||||
* http://code.google.com/p/urim/
|
||||
* http://snowball.tartarus.org/
|
||||
*
|
||||
* Copyright 2010, Oleg Mazko
|
||||
* http://www.mozilla.org/MPL/
|
||||
*/
|
||||
|
||||
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");console.warn('[Lunr Languages] Please use the "nl" instead of the "du". The "nl" code is the standard code for Dutch language, and "du" will be removed in the next major versions.'),e.du=function(){this.pipeline.reset(),this.pipeline.add(e.du.trimmer,e.du.stopWordFilter,e.du.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.du.stemmer))},e.du.wordCharacters="A-Za-zªºÀ-ÖØ-öø-ʸˠ-ˤᴀ-ᴥᴬ-ᵜᵢ-ᵥᵫ-ᵷᵹ-ᶾḀ-ỿⁱⁿₐ-ₜKÅℲⅎⅠ-ↈⱠ-ⱿꜢ-ꞇꞋ-ꞭꞰ-ꞷꟷ-ꟿꬰ-ꭚꭜ-ꭤff-stA-Za-z",e.du.trimmer=e.trimmerSupport.generateTrimmer(e.du.wordCharacters),e.Pipeline.registerFunction(e.du.trimmer,"trimmer-du"),e.du.stemmer=function(){var r=e.stemmerSupport.Among,i=e.stemmerSupport.SnowballProgram,n=new function(){function e(){for(var e,r,i,o=C.cursor;;){if(C.bra=C.cursor,e=C.find_among(b,11))switch(C.ket=C.cursor,e){case 1:C.slice_from("a");continue;case 2:C.slice_from("e");continue;case 3:C.slice_from("i");continue;case 4:C.slice_from("o");continue;case 5:C.slice_from("u");continue;case 6:if(C.cursor>=C.limit)break;C.cursor++;continue}break}for(C.cursor=o,C.bra=o,C.eq_s(1,"y")?(C.ket=C.cursor,C.slice_from("Y")):C.cursor=o;;)if(r=C.cursor,C.in_grouping(q,97,232)){if(i=C.cursor,C.bra=i,C.eq_s(1,"i"))C.ket=C.cursor,C.in_grouping(q,97,232)&&(C.slice_from("I"),C.cursor=r);else if(C.cursor=i,C.eq_s(1,"y"))C.ket=C.cursor,C.slice_from("Y"),C.cursor=r;else if(n(r))break}else if(n(r))break}function n(e){return C.cursor=e,e>=C.limit||(C.cursor++,!1)}function o(){_=C.limit,f=_,t()||(_=C.cursor,_<3&&(_=3),t()||(f=C.cursor))}function t(){for(;!C.in_grouping(q,97,232);){if(C.cursor>=C.limit)return!0;C.cursor++}for(;!C.out_grouping(q,97,232);){if(C.cursor>=C.limit)return!0;C.cursor++}return!1}function s(){for(var e;;)if(C.bra=C.cursor,e=C.find_among(p,3))switch(C.ket=C.cursor,e){case 1:C.slice_from("y");break;case 2:C.slice_from("i");break;case 3:if(C.cursor>=C.limit)return;C.cursor++}}function u(){return _<=C.cursor}function c(){return f<=C.cursor}function a(){var e=C.limit-C.cursor;C.find_among_b(g,3)&&(C.cursor=C.limit-e,C.ket=C.cursor,C.cursor>C.limit_backward&&(C.cursor--,C.bra=C.cursor,C.slice_del()))}function l(){var e;w=!1,C.ket=C.cursor,C.eq_s_b(1,"e")&&(C.bra=C.cursor,u()&&(e=C.limit-C.cursor,C.out_grouping_b(q,97,232)&&(C.cursor=C.limit-e,C.slice_del(),w=!0,a())))}function m(){var e;u()&&(e=C.limit-C.cursor,C.out_grouping_b(q,97,232)&&(C.cursor=C.limit-e,C.eq_s_b(3,"gem")||(C.cursor=C.limit-e,C.slice_del(),a())))}function d(){var e,r,i,n,o,t,s=C.limit-C.cursor;if(C.ket=C.cursor,e=C.find_among_b(h,5))switch(C.bra=C.cursor,e){case 1:u()&&C.slice_from("heid");break;case 2:m();break;case 3:u()&&C.out_grouping_b(z,97,232)&&C.slice_del()}if(C.cursor=C.limit-s,l(),C.cursor=C.limit-s,C.ket=C.cursor,C.eq_s_b(4,"heid")&&(C.bra=C.cursor,c()&&(r=C.limit-C.cursor,C.eq_s_b(1,"c")||(C.cursor=C.limit-r,C.slice_del(),C.ket=C.cursor,C.eq_s_b(2,"en")&&(C.bra=C.cursor,m())))),C.cursor=C.limit-s,C.ket=C.cursor,e=C.find_among_b(k,6))switch(C.bra=C.cursor,e){case 1:if(c()){if(C.slice_del(),i=C.limit-C.cursor,C.ket=C.cursor,C.eq_s_b(2,"ig")&&(C.bra=C.cursor,c()&&(n=C.limit-C.cursor,!C.eq_s_b(1,"e")))){C.cursor=C.limit-n,C.slice_del();break}C.cursor=C.limit-i,a()}break;case 2:c()&&(o=C.limit-C.cursor,C.eq_s_b(1,"e")||(C.cursor=C.limit-o,C.slice_del()));break;case 3:c()&&(C.slice_del(),l());break;case 4:c()&&C.slice_del();break;case 5:c()&&w&&C.slice_del()}C.cursor=C.limit-s,C.out_grouping_b(j,73,232)&&(t=C.limit-C.cursor,C.find_among_b(v,4)&&C.out_grouping_b(q,97,232)&&(C.cursor=C.limit-t,C.ket=C.cursor,C.cursor>C.limit_backward&&(C.cursor--,C.bra=C.cursor,C.slice_del())))}var f,_,w,b=[new r("",-1,6),new r("á",0,1),new r("ä",0,1),new r("é",0,2),new r("ë",0,2),new r("í",0,3),new r("ï",0,3),new r("ó",0,4),new r("ö",0,4),new r("ú",0,5),new r("ü",0,5)],p=[new r("",-1,3),new r("I",0,2),new r("Y",0,1)],g=[new r("dd",-1,-1),new r("kk",-1,-1),new r("tt",-1,-1)],h=[new r("ene",-1,2),new r("se",-1,3),new r("en",-1,2),new r("heden",2,1),new r("s",-1,3)],k=[new r("end",-1,1),new r("ig",-1,2),new r("ing",-1,1),new r("lijk",-1,3),new r("baar",-1,4),new r("bar",-1,5)],v=[new r("aa",-1,-1),new r("ee",-1,-1),new r("oo",-1,-1),new r("uu",-1,-1)],q=[17,65,16,1,0,0,0,0,0,0,0,0,0,0,0,0,128],j=[1,0,0,17,65,16,1,0,0,0,0,0,0,0,0,0,0,0,0,128],z=[17,67,16,1,0,0,0,0,0,0,0,0,0,0,0,0,128],C=new i;this.setCurrent=function(e){C.setCurrent(e)},this.getCurrent=function(){return C.getCurrent()},this.stem=function(){var r=C.cursor;return e(),C.cursor=r,o(),C.limit_backward=r,C.cursoLine truncated
|
||||
@@ -1 +0,0 @@
|
||||
!function(e,t){"function"==typeof define&&define.amd?define(t):"object"==typeof exports?module.exports=t():t()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.el=function(){this.pipeline.reset(),void 0===this.searchPipeline&&this.pipeline.add(e.el.trimmer,e.el.normilizer),this.pipeline.add(e.el.stopWordFilter,e.el.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.el.stemmer))},e.el.wordCharacters="A-Za-zΑαΒβΓγΔδΕεΖζΗηΘθΙιΚκΛλΜμΝνΞξΟοΠπΡρΣσςΤτΥυΦφΧχΨψΩωΆάΈέΉήΊίΌόΎύΏώΪΐΫΰΐΰ",e.el.trimmer=e.trimmerSupport.generateTrimmer(e.el.wordCharacters),e.Pipeline.registerFunction(e.el.trimmer,"trimmer-el"),e.el.stemmer=function(){function e(e){return s.test(e)}function t(e){return/[ΑΕΗΙΟΥΩ]$/.test(e)}function r(e){return/[ΑΕΗΙΟΩ]$/.test(e)}function n(n){var s=n;if(n.length<3)return s;if(!e(n))return s;if(i.indexOf(n)>=0)return s;var u=new RegExp("(.*)("+Object.keys(l).join("|")+")$"),o=u.exec(s);return null!==o&&(s=o[1]+l[o[2]]),null!==(o=/^(.+?)(ΑΔΕΣ|ΑΔΩΝ)$/.exec(s))&&(s=o[1],/(ΟΚ|ΜΑΜ|ΜΑΝ|ΜΠΑΜΠ|ΠΑΤΕΡ|ΓΙΑΓΙ|ΝΤΑΝΤ|ΚΥΡ|ΘΕΙ|ΠΕΘΕΡ|ΜΟΥΣΑΜ|ΚΑΠΛΑΜ|ΠΑΡ|ΨΑΡ|ΤΖΟΥΡ|ΤΑΜΠΟΥΡ|ΓΑΛΑΤ|ΦΑΦΛΑΤ)$/.test(o[1])||(s+="ΑΔ")),null!==(o=/^(.+?)(ΕΔΕΣ|ΕΔΩΝ)$/.exec(s))&&(s=o[1],/(ΟΠ|ΙΠ|ΕΜΠ|ΥΠ|ΓΗΠ|ΔΑΠ|ΚΡΑΣΠ|ΜΙΛ)$/.test(o[1])&&(s+="ΕΔ")),null!==(o=/^(.+?)(ΟΥΔΕΣ|ΟΥΔΩΝ)$/.exec(s))&&(s=o[1],/(ΑΡΚ|ΚΑΛΙΑΚ|ΠΕΤΑΛ|ΛΙΧ|ΠΛΕΞ|ΣΚ|Σ|ΦΛ|ΦΡ|ΒΕΛ|ΛΟΥΛ|ΧΝ|ΣΠ|ΤΡΑΓ|ΦΕ)$/.test(o[1])&&(s+="ΟΥΔ")),null!==(o=/^(.+?)(ΕΩΣ|ΕΩΝ|ΕΑΣ|ΕΑ)$/.exec(s))&&(s=o[1],/^(Θ|Δ|ΕΛ|ΓΑΛ|Ν|Π|ΙΔ|ΠΑΡ|ΣΤΕΡ|ΟΡΦ|ΑΝΔΡ|ΑΝΤΡ)$/.test(o[1])&&(s+="Ε")),null!==(o=/^(.+?)(ΕΙΟ|ΕΙΟΣ|ΕΙΟΙ|ΕΙΑ|ΕΙΑΣ|ΕΙΕΣ|ΕΙΟΥ|ΕΙΟΥΣ|ΕΙΩΝ)$/.exec(s))&&o[1].length>4&&(s=o[1]),null!==(o=/^(.+?)(ΙΟΥΣ|ΙΑΣ|ΙΕΣ|ΙΟΣ|ΙΟΥ|ΙΟΙ|ΙΩΝ|ΙΟΝ|ΙΑ|ΙΟ)$/.exec(s))&&(s=o[1],(t(s)||s.length<2||/^(ΑΓ|ΑΓΓΕΛ|ΑΓΡ|ΑΕΡ|ΑΘΛ|ΑΚΟΥΣ|ΑΞ|ΑΣ|Β|ΒΙΒΛ|ΒΥΤ|Γ|ΓΙΑΓ|ΓΩΝ|Δ|ΔΑΝ|ΔΗΛ|ΔΗΜ|ΔΟΚΙΜ|ΕΛ|ΖΑΧΑΡ|ΗΛ|ΗΠ|ΙΔ|ΙΣΚ|ΙΣΤ|ΙΟΝ|ΙΩΝ|ΚΙΜΩΛ|ΚΟΛΟΝ|ΚΟΡ|ΚΤΗΡ|ΚΥΡ|ΛΑΓ|ΛΟΓ|ΜΑΓ|ΜΠΑΝ|ΜΠΡ|ΝΑΥΤ|ΝΟΤ|ΟΠΑΛ|ΟΞ|ΟΡ|ΟΣ|ΠΑΝΑΓ|ΠΑΤΡ|ΠΗΛ|ΠΗΝ|ΠΛΑΙΣ|ΠΟΝΤ|ΡΑΔ|ΡΟΔ|ΣΚ|ΣΚΟΡΠ|ΣΟΥΝ|ΣΠΑΝ|ΣΤΑΔ|ΣΥΡ|ΤΗΛ|ΤΙΜ|ΤΟΚ|ΤΟΠ|ΤΡΟΧ|ΦΙΛ|ΦΩΤ|Χ|ΧΙΛ|ΧΡΩΜ|ΧΩΡ)$/.test(o[1]))&&(s+="Ι"),/^(ΠΑΛ)$/.test(o[1])&&(s+="ΑΙ")),null!==(o=/^(.+?)(ΙΚΟΣ|ΙΚΟΝ|ΙΚΕΙΣ|ΙΚΟΙ|ΙΚΕΣ|ΙΚΟΥΣ|ΙΚΗ|ΙΚΗΣ|ΙΚΟ|ΙΚΑ|ΙΚΟΥ|ΙΚΩΝ|ΙΚΩΣ)$/.exec(s))&&(s=o[1],(t(s)||/^(ΑΔ|ΑΛ|ΑΜΑΝ|ΑΜΕΡ|ΑΜΜΟΧΑΛ|ΑΝΗΘ|ΑΝΤΙΔ|ΑΠΛ|ΑΤΤ|ΑΦΡ|ΒΑΣ|ΒΡΩΜ|ΓΕΝ|ΓΕΡ|Δ|ΔΙΚΑΝ|ΔΥΤ|ΕΙΔ|ΕΝΔ|ΕΞΩΔ|ΗΘ|ΘΕΤ|ΚΑΛΛΙΝ|ΚΑΛΠ|ΚΑΤΑΔ|ΚΟΥΖΙΝ|ΚΡ|ΚΩΔ|ΛΟΓ|Μ|ΜΕΡ|ΜΟΝΑΔ|ΜΟΥΛ|ΜΟΥΣ|ΜΠΑΓΙΑΤ|ΜΠΑΝ|ΜΠΟΛ|ΜΠΟΣ|ΜΥΣΤ|Ν|ΝΙΤ|ΞΙΚ|ΟΠΤ|ΠΑΝ|ΠΕΤΣ|ΠΙΚΑΝΤ|ΠΙΤΣ|ΠΛΑΣΤ|ΠΛΙΑΤΣ|ΠΟΝΤ|ΠΟΣΤΕΛΝ|ΠΡΩΤΟΔ|ΣΕΡΤ|ΣΗΜΑΝΤ|ΣΤΑΤ|ΣΥΝΑΔ|ΣΥΝΟΜΗΛ|ΤΕΛ|ΤΕΧΝ|ΤΡΟΠ|ΤΣΑΜ|ΥΠΟΔ|Φ|ΦΙΛΟΝ|ΦΥΛΟΔ|ΦΥΣ|ΧΑΣ)$/.test(o[1])||/(ΦΟΙΝ)$/.test(o[1]))&&(s+="ΙΚ")),"ΑΓΑΜΕ"===s&&(s="ΑΓΑΜ"),null!==(o=/^(.+?)(ΑΓΑΜΕ|ΗΣΑΜΕ|ΟΥΣΑΜΕ|ΗΚΑΜΕ|ΗΘΗΚΑΜΕ)$/.exec(s))&&(s=o[1]),null!==(o=/^(.+?)(ΑΜΕ)$/.exec(s))&&(s=o[1],/^(ΑΝΑΠ|ΑΠΟΘ|ΑΠΟΚ|ΑΠΟΣΤ|ΒΟΥΒ|ΞΕΘ|ΟΥΛ|ΠΕΘ|ΠΙΚΡ|ΠΟΤ|ΣΙΧ|Χ)$/.test(o[1])&&(s+="ΑΜ")),null!==(o=/^(.+?)(ΑΓΑΝΕ|ΗΣΑΝΕ|ΟΥΣΑΝΕ|ΙΟΝΤΑΝΕ|ΙΟΤΑΝΕ|ΙΟΥΝΤΑΝΕ|ΟΝΤΑΝΕ|ΟΤΑΝΕ|ΟΥΝΤΑΝΕ|ΗΚΑΝΕ|ΗΘΗΚΑΝΕ)$/.exec(s))&&(s=o[1],/^(ΤΡ|ΤΣ)$/.test(o[1])&&(s+="ΑΓΑΝ")),null!==(o=/^(.+?)(ΑΝΕ)$/.exec(s))&&(s=o[1],(r(s)||/^(ΒΕΤΕΡ|ΒΟΥΛΚ|ΒΡΑΧΜ|Γ|ΔΡΑΔΟΥΜ|Θ|ΚΑΛΠΟΥΖ|ΚΑΣΤΕΛ|ΚΟΡΜΟΡ|ΛΑΟΠΛ|ΜΩΑΜΕΘ|Μ|ΜΟΥΣΟΥΛΜΑΝ|ΟΥΛ|Π|ΠΕΛΕΚ|ΠΛ|ΠΟΛΙΣ|ΠΟΡΤΟΛ|ΣΑΡΑΚΑΤΣ|ΣΟΥΛΤ|ΤΣΑΡΛΑΤ|ΟΡΦ|ΤΣΙΓΓ|ΤΣΟΠ|ΦΩΤΟΣΤΕΦ|Χ|ΨΥΧΟΠΛ|ΑΓ|ΟΡΦ|ΓΑΛ|ΓΕΡ|ΔΕΚ|ΔΙΠΛ|ΑΜΕΡΙΚΑΝ|ΟΥΡ|ΠΙΘ|ΠΟΥΡΙΤ|Σ|ΖΩΝΤ|ΙΚ|ΚΑΣΤ|ΚΟΠ|ΛΙΧ|ΛΟΥΘΗΡ|ΜΑΙΝΤ|ΜΕΛ|ΣΙΓ|ΣΠ|ΣΤΕΓ|ΤΡΑΓ|ΤΣΑΓ|Φ|ΕΡ|ΑΔΑΠ|ΑΘΙΓΓ|ΑΜΗΧ|ΑΝΙΚ|ΑΝΟΡΓ|ΑΠΗΓ|ΑΠΙΘ|ΑΤΣΙΓΓ|ΒΑΣ|ΒΑΣΚ|ΒΑΘΥΓΑΛ|ΒΙΟΜΗΧ|ΒΡΑΧΥΚ|ΔΙΑΤ|ΔΙΑΦ|ΕΝΟΡΓ|ΘΥΣ|ΚΑΠΝΟΒΙΟΜΗΧ|ΚΑΤLine truncated
|
||||
@@ -1,18 +0,0 @@
|
||||
/*!
|
||||
* Lunr languages, `Spanish` language
|
||||
* https://github.com/MihaiValentin/lunr-languages
|
||||
*
|
||||
* Copyright 2014, Mihai Valentin
|
||||
* http://www.mozilla.org/MPL/
|
||||
*/
|
||||
/*!
|
||||
* based on
|
||||
* Snowball JavaScript Library v0.3
|
||||
* http://code.google.com/p/urim/
|
||||
* http://snowball.tartarus.org/
|
||||
*
|
||||
* Copyright 2010, Oleg Mazko
|
||||
* http://www.mozilla.org/MPL/
|
||||
*/
|
||||
|
||||
!function(e,s){"function"==typeof define&&define.amd?define(s):"object"==typeof exports?module.exports=s():s()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.es=function(){this.pipeline.reset(),this.pipeline.add(e.es.trimmer,e.es.stopWordFilter,e.es.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.es.stemmer))},e.es.wordCharacters="A-Za-zªºÀ-ÖØ-öø-ʸˠ-ˤᴀ-ᴥᴬ-ᵜᵢ-ᵥᵫ-ᵷᵹ-ᶾḀ-ỿⁱⁿₐ-ₜKÅℲⅎⅠ-ↈⱠ-ⱿꜢ-ꞇꞋ-ꞭꞰ-ꞷꟷ-ꟿꬰ-ꭚꭜ-ꭤff-stA-Za-z",e.es.trimmer=e.trimmerSupport.generateTrimmer(e.es.wordCharacters),e.Pipeline.registerFunction(e.es.trimmer,"trimmer-es"),e.es.stemmer=function(){var s=e.stemmerSupport.Among,r=e.stemmerSupport.SnowballProgram,n=new function(){function e(){if(A.out_grouping(x,97,252)){for(;!A.in_grouping(x,97,252);){if(A.cursor>=A.limit)return!0;A.cursor++}return!1}return!0}function n(){if(A.in_grouping(x,97,252)){var s=A.cursor;if(e()){if(A.cursor=s,!A.in_grouping(x,97,252))return!0;for(;!A.out_grouping(x,97,252);){if(A.cursor>=A.limit)return!0;A.cursor++}}return!1}return!0}function i(){var s,r=A.cursor;if(n()){if(A.cursor=r,!A.out_grouping(x,97,252))return;if(s=A.cursor,e()){if(A.cursor=s,!A.in_grouping(x,97,252)||A.cursor>=A.limit)return;A.cursor++}}g=A.cursor}function a(){for(;!A.in_grouping(x,97,252);){if(A.cursor>=A.limit)return!1;A.cursor++}for(;!A.out_grouping(x,97,252);){if(A.cursor>=A.limit)return!1;A.cursor++}return!0}function t(){var e=A.cursor;g=A.limit,p=g,v=g,i(),A.cursor=e,a()&&(p=A.cursor,a()&&(v=A.cursor))}function o(){for(var e;;){if(A.bra=A.cursor,e=A.find_among(k,6))switch(A.ket=A.cursor,e){case 1:A.slice_from("a");continue;case 2:A.slice_from("e");continue;case 3:A.slice_from("i");continue;case 4:A.slice_from("o");continue;case 5:A.slice_from("u");continue;case 6:if(A.cursor>=A.limit)break;A.cursor++;continue}break}}function u(){return g<=A.cursor}function w(){return p<=A.cursor}function c(){return v<=A.cursor}function m(){var e;if(A.ket=A.cursor,A.find_among_b(y,13)&&(A.bra=A.cursor,(e=A.find_among_b(q,11))&&u()))switch(e){case 1:A.bra=A.cursor,A.slice_from("iendo");break;case 2:A.bra=A.cursor,A.slice_from("ando");break;case 3:A.bra=A.cursor,A.slice_from("ar");break;case 4:A.bra=A.cursor,A.slice_from("er");break;case 5:A.bra=A.cursor,A.slice_from("ir");break;case 6:A.slice_del();break;case 7:A.eq_s_b(1,"u")&&A.slice_del()}}function l(e,s){if(!c())return!0;A.slice_del(),A.ket=A.cursor;var r=A.find_among_b(e,s);return r&&(A.bra=A.cursor,1==r&&c()&&A.slice_del()),!1}function d(e){return!c()||(A.slice_del(),A.ket=A.cursor,A.eq_s_b(2,e)&&(A.bra=A.cursor,c()&&A.slice_del()),!1)}function b(){var e;if(A.ket=A.cursor,e=A.find_among_b(S,46)){switch(A.bra=A.cursor,e){case 1:if(!c())return!1;A.slice_del();break;case 2:if(d("ic"))return!1;break;case 3:if(!c())return!1;A.slice_from("log");break;case 4:if(!c())return!1;A.slice_from("u");break;case 5:if(!c())return!1;A.slice_from("ente");break;case 6:if(!w())return!1;A.slice_del(),A.ket=A.cursor,e=A.find_among_b(C,4),e&&(A.bra=A.cursor,c()&&(A.slice_del(),1==e&&(A.ket=A.cursor,A.eq_s_b(2,"at")&&(A.bra=A.cursor,c()&&A.slice_del()))));break;case 7:if(l(P,3))return!1;break;case 8:if(l(F,3))return!1;break;case 9:if(d("at"))return!1}return!0}return!1}function f(){var e,s;if(A.cursor>=g&&(s=A.limit_backward,A.limit_backward=g,A.ket=A.cursor,e=A.find_among_b(W,12),A.limit_backward=s,e)){if(A.bra=A.cursor,1==e){if(!A.eq_s_b(1,"u"))return!1;A.slice_del()}return!0}return!1}function _(){var e,s,r,n;if(A.cursor>=g&&(s=A.limit_backward,A.limit_backward=g,A.ket=A.cursor,e=A.find_among_b(L,96),A.limit_backward=s,e))switch(A.bra=A.cursor,e){case 1:r=A.limit-A.cursor,A.eq_s_b(1,"u")?(n=A.limit-A.cursor,A.eq_s_b(1,"g")?A.cursor=A.limit-n:A.cursor=A.limit-r):A.cursor=A.limit-r,A.bra=A.cursor;case 2:A.slice_del()}}function h(){var e,s;if(A.ket=A.cursor,e=A.find_among_b(z,8))switch(A.bra=A.cursor,e){case 1:u()&&A.slice_del();break;case 2:u()&&(A.slice_del(),A.ket=A.cursor,A.eq_s_b(1,"u")&&(A.bra=A.cursor,s=A.limit-A.cursor,A.eq_s_b(1,"g")&&(A.cursor=A.limit-s,u()&&A.slice_del())))}}var v,p,g,k=[new s("",-1,6),new s("á",0,1),new s("é",0,2),new s("í",0,3),new s("ó",0,4),new s("ú",0,5)],y=[new s("la",-1,-1),new s("sela",0,-1),new s("le",-1,-1),new s("me",-1,-1),new s("se",-1,-1),new s("lo",-1,-1),new s("selo",5,-1),new s("las",-1,-1),new s("selas",7,-1),new s("les",-1,-1),new s("los",-1,-1),new s("selos",10,-1),new s("nos",-1,-1)],q=[new s("ando",-1,6),new s("iendo",-1,6),new s("yendo",-1,7),new s("ándo",-1,2),new s("iéndo",-1,1),new s("ar",-1,6),new s("er",-1,6),new s("ir",-1,6),new s("ár",-1,3),new s("ér",-1,4),new s("ír",-1,5)],C=[new s("ic",-1,-1),new s("ad",-1,-1),new s("os",-1,-1),new s("iLine truncated
|
||||
@@ -1,18 +0,0 @@
|
||||
/*!
|
||||
* Lunr languages, `Finnish` language
|
||||
* https://github.com/MihaiValentin/lunr-languages
|
||||
*
|
||||
* Copyright 2014, Mihai Valentin
|
||||
* http://www.mozilla.org/MPL/
|
||||
*/
|
||||
/*!
|
||||
* based on
|
||||
* Snowball JavaScript Library v0.3
|
||||
* http://code.google.com/p/urim/
|
||||
* http://snowball.tartarus.org/
|
||||
*
|
||||
* Copyright 2010, Oleg Mazko
|
||||
* http://www.mozilla.org/MPL/
|
||||
*/
|
||||
|
||||
!function(i,e){"function"==typeof define&&define.amd?define(e):"object"==typeof exports?module.exports=e():e()(i.lunr)}(this,function(){return function(i){if(void 0===i)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===i.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");i.fi=function(){this.pipeline.reset(),this.pipeline.add(i.fi.trimmer,i.fi.stopWordFilter,i.fi.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(i.fi.stemmer))},i.fi.wordCharacters="A-Za-zªºÀ-ÖØ-öø-ʸˠ-ˤᴀ-ᴥᴬ-ᵜᵢ-ᵥᵫ-ᵷᵹ-ᶾḀ-ỿⁱⁿₐ-ₜKÅℲⅎⅠ-ↈⱠ-ⱿꜢ-ꞇꞋ-ꞭꞰ-ꞷꟷ-ꟿꬰ-ꭚꭜ-ꭤff-stA-Za-z",i.fi.trimmer=i.trimmerSupport.generateTrimmer(i.fi.wordCharacters),i.Pipeline.registerFunction(i.fi.trimmer,"trimmer-fi"),i.fi.stemmer=function(){var e=i.stemmerSupport.Among,r=i.stemmerSupport.SnowballProgram,n=new function(){function i(){f=A.limit,d=f,n()||(f=A.cursor,n()||(d=A.cursor))}function n(){for(var i;;){if(i=A.cursor,A.in_grouping(W,97,246))break;if(A.cursor=i,i>=A.limit)return!0;A.cursor++}for(A.cursor=i;!A.out_grouping(W,97,246);){if(A.cursor>=A.limit)return!0;A.cursor++}return!1}function t(){return d<=A.cursor}function s(){var i,e;if(A.cursor>=f)if(e=A.limit_backward,A.limit_backward=f,A.ket=A.cursor,i=A.find_among_b(h,10)){switch(A.bra=A.cursor,A.limit_backward=e,i){case 1:if(!A.in_grouping_b(x,97,246))return;break;case 2:if(!t())return}A.slice_del()}else A.limit_backward=e}function o(){var i,e,r;if(A.cursor>=f)if(e=A.limit_backward,A.limit_backward=f,A.ket=A.cursor,i=A.find_among_b(v,9))switch(A.bra=A.cursor,A.limit_backward=e,i){case 1:r=A.limit-A.cursor,A.eq_s_b(1,"k")||(A.cursor=A.limit-r,A.slice_del());break;case 2:A.slice_del(),A.ket=A.cursor,A.eq_s_b(3,"kse")&&(A.bra=A.cursor,A.slice_from("ksi"));break;case 3:A.slice_del();break;case 4:A.find_among_b(p,6)&&A.slice_del();break;case 5:A.find_among_b(g,6)&&A.slice_del();break;case 6:A.find_among_b(j,2)&&A.slice_del()}else A.limit_backward=e}function l(){return A.find_among_b(q,7)}function a(){return A.eq_s_b(1,"i")&&A.in_grouping_b(L,97,246)}function u(){var i,e,r;if(A.cursor>=f)if(e=A.limit_backward,A.limit_backward=f,A.ket=A.cursor,i=A.find_among_b(C,30)){switch(A.bra=A.cursor,A.limit_backward=e,i){case 1:if(!A.eq_s_b(1,"a"))return;break;case 2:case 9:if(!A.eq_s_b(1,"e"))return;break;case 3:if(!A.eq_s_b(1,"i"))return;break;case 4:if(!A.eq_s_b(1,"o"))return;break;case 5:if(!A.eq_s_b(1,"ä"))return;break;case 6:if(!A.eq_s_b(1,"ö"))return;break;case 7:if(r=A.limit-A.cursor,!l()&&(A.cursor=A.limit-r,!A.eq_s_b(2,"ie"))){A.cursor=A.limit-r;break}if(A.cursor=A.limit-r,A.cursor<=A.limit_backward){A.cursor=A.limit-r;break}A.cursor--,A.bra=A.cursor;break;case 8:if(!A.in_grouping_b(W,97,246)||!A.out_grouping_b(W,97,246))return}A.slice_del(),k=!0}else A.limit_backward=e}function c(){var i,e,r;if(A.cursor>=d)if(e=A.limit_backward,A.limit_backward=d,A.ket=A.cursor,i=A.find_among_b(P,14)){if(A.bra=A.cursor,A.limit_backward=e,1==i){if(r=A.limit-A.cursor,A.eq_s_b(2,"po"))return;A.cursor=A.limit-r}A.slice_del()}else A.limit_backward=e}function m(){var i;A.cursor>=f&&(i=A.limit_backward,A.limit_backward=f,A.ket=A.cursor,A.find_among_b(F,2)?(A.bra=A.cursor,A.limit_backward=i,A.slice_del()):A.limit_backward=i)}function w(){var i,e,r,n,t,s;if(A.cursor>=f){if(e=A.limit_backward,A.limit_backward=f,A.ket=A.cursor,A.eq_s_b(1,"t")&&(A.bra=A.cursor,r=A.limit-A.cursor,A.in_grouping_b(W,97,246)&&(A.cursor=A.limit-r,A.slice_del(),A.limit_backward=e,n=A.limit-A.cursor,A.cursor>=d&&(A.cursor=d,t=A.limit_backward,A.limit_backward=A.cursor,A.cursor=A.limit-n,A.ket=A.cursor,i=A.find_among_b(S,2))))){if(A.bra=A.cursor,A.limit_backward=t,1==i){if(s=A.limit-A.cursor,A.eq_s_b(2,"po"))return;A.cursor=A.limit-s}return void A.slice_del()}A.limit_backward=e}}function _(){var i,e,r,n;if(A.cursor>=f){for(i=A.limit_backward,A.limit_backward=f,e=A.limit-A.cursor,l()&&(A.cursor=A.limit-e,A.ket=A.cursor,A.cursor>A.limit_backward&&(A.cursor--,A.bra=A.cursor,A.slice_del())),A.cursor=A.limit-e,A.ket=A.cursor,A.in_grouping_b(y,97,228)&&(A.bra=A.cursor,A.out_grouping_b(W,97,246)&&A.slice_del()),A.cursor=A.limit-e,A.ket=A.cursor,A.eq_s_b(1,"j")&&(A.bra=A.cursor,r=A.limit-A.cursor,A.eq_s_b(1,"o")?A.slice_del():(A.cursor=A.limit-r,A.eq_s_b(1,"u")&&A.slice_del())),A.cursor=A.limit-e,A.ket=A.cursor,A.eq_s_b(1,"o")&&(A.bra=A.cursor,A.eq_s_b(1,"j")&&A.slice_del()),A.cursor=A.limit-e,A.limit_backward=i;;){if(n=A.limit-A.cursor,A.out_grouping_b(W,97,246)){A.cursor=A.limit-n;break}if(A.cursor=A.limit-n,A.cursor<=A.limit_backward)return;A.cursor--}A.ket=A.cursor,A.cursor>A.limit_backward&&(A.cursor--,A.bra=A.cursor,b=A.slice_to(),A.eq_v_b(b)&&A.slice_del())}}var k,b,d,f,h=[new e("pa",-1,1),new e("sti",-1,2),new e("kaan",-1,1),new e("han",-1,1),new e("kin",-1,1),new e("hän",-1,1),new e("kään",-1,1),new e("ko",-1,Line truncated
|
||||
@@ -1,18 +0,0 @@
|
||||
/*!
|
||||
* Lunr languages, `French` language
|
||||
* https://github.com/MihaiValentin/lunr-languages
|
||||
*
|
||||
* Copyright 2014, Mihai Valentin
|
||||
* http://www.mozilla.org/MPL/
|
||||
*/
|
||||
/*!
|
||||
* based on
|
||||
* Snowball JavaScript Library v0.3
|
||||
* http://code.google.com/p/urim/
|
||||
* http://snowball.tartarus.org/
|
||||
*
|
||||
* Copyright 2010, Oleg Mazko
|
||||
* http://www.mozilla.org/MPL/
|
||||
*/
|
||||
|
||||
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.fr=function(){this.pipeline.reset(),this.pipeline.add(e.fr.trimmer,e.fr.stopWordFilter,e.fr.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.fr.stemmer))},e.fr.wordCharacters="A-Za-zªºÀ-ÖØ-öø-ʸˠ-ˤᴀ-ᴥᴬ-ᵜᵢ-ᵥᵫ-ᵷᵹ-ᶾḀ-ỿⁱⁿₐ-ₜKÅℲⅎⅠ-ↈⱠ-ⱿꜢ-ꞇꞋ-ꞭꞰ-ꞷꟷ-ꟿꬰ-ꭚꭜ-ꭤff-stA-Za-z",e.fr.trimmer=e.trimmerSupport.generateTrimmer(e.fr.wordCharacters),e.Pipeline.registerFunction(e.fr.trimmer,"trimmer-fr"),e.fr.stemmer=function(){var r=e.stemmerSupport.Among,s=e.stemmerSupport.SnowballProgram,i=new function(){function e(e,r,s){return!(!W.eq_s(1,e)||(W.ket=W.cursor,!W.in_grouping(F,97,251)))&&(W.slice_from(r),W.cursor=s,!0)}function i(e,r,s){return!!W.eq_s(1,e)&&(W.ket=W.cursor,W.slice_from(r),W.cursor=s,!0)}function n(){for(var r,s;;){if(r=W.cursor,W.in_grouping(F,97,251)){if(W.bra=W.cursor,s=W.cursor,e("u","U",r))continue;if(W.cursor=s,e("i","I",r))continue;if(W.cursor=s,i("y","Y",r))continue}if(W.cursor=r,W.bra=r,!e("y","Y",r)){if(W.cursor=r,W.eq_s(1,"q")&&(W.bra=W.cursor,i("u","U",r)))continue;if(W.cursor=r,r>=W.limit)return;W.cursor++}}}function t(){for(;!W.in_grouping(F,97,251);){if(W.cursor>=W.limit)return!0;W.cursor++}for(;!W.out_grouping(F,97,251);){if(W.cursor>=W.limit)return!0;W.cursor++}return!1}function u(){var e=W.cursor;if(q=W.limit,g=q,p=q,W.in_grouping(F,97,251)&&W.in_grouping(F,97,251)&&W.cursor<W.limit)W.cursor++;else if(W.cursor=e,!W.find_among(v,3)){W.cursor=e;do{if(W.cursor>=W.limit){W.cursor=q;break}W.cursor++}while(!W.in_grouping(F,97,251))}q=W.cursor,W.cursor=e,t()||(g=W.cursor,t()||(p=W.cursor))}function o(){for(var e,r;;){if(r=W.cursor,W.bra=r,!(e=W.find_among(h,4)))break;switch(W.ket=W.cursor,e){case 1:W.slice_from("i");break;case 2:W.slice_from("u");break;case 3:W.slice_from("y");break;case 4:if(W.cursor>=W.limit)return;W.cursor++}}}function c(){return q<=W.cursor}function a(){return g<=W.cursor}function l(){return p<=W.cursor}function w(){var e,r;if(W.ket=W.cursor,e=W.find_among_b(C,43)){switch(W.bra=W.cursor,e){case 1:if(!l())return!1;W.slice_del();break;case 2:if(!l())return!1;W.slice_del(),W.ket=W.cursor,W.eq_s_b(2,"ic")&&(W.bra=W.cursor,l()?W.slice_del():W.slice_from("iqU"));break;case 3:if(!l())return!1;W.slice_from("log");break;case 4:if(!l())return!1;W.slice_from("u");break;case 5:if(!l())return!1;W.slice_from("ent");break;case 6:if(!c())return!1;if(W.slice_del(),W.ket=W.cursor,e=W.find_among_b(z,6))switch(W.bra=W.cursor,e){case 1:l()&&(W.slice_del(),W.ket=W.cursor,W.eq_s_b(2,"at")&&(W.bra=W.cursor,l()&&W.slice_del()));break;case 2:l()?W.slice_del():a()&&W.slice_from("eux");break;case 3:l()&&W.slice_del();break;case 4:c()&&W.slice_from("i")}break;case 7:if(!l())return!1;if(W.slice_del(),W.ket=W.cursor,e=W.find_among_b(y,3))switch(W.bra=W.cursor,e){case 1:l()?W.slice_del():W.slice_from("abl");break;case 2:l()?W.slice_del():W.slice_from("iqU");break;case 3:l()&&W.slice_del()}break;case 8:if(!l())return!1;if(W.slice_del(),W.ket=W.cursor,W.eq_s_b(2,"at")&&(W.bra=W.cursor,l()&&(W.slice_del(),W.ket=W.cursor,W.eq_s_b(2,"ic")))){W.bra=W.cursor,l()?W.slice_del():W.slice_from("iqU");break}break;case 9:W.slice_from("eau");break;case 10:if(!a())return!1;W.slice_from("al");break;case 11:if(l())W.slice_del();else{if(!a())return!1;W.slice_from("eux")}break;case 12:if(!a()||!W.out_grouping_b(F,97,251))return!1;W.slice_del();break;case 13:return c()&&W.slice_from("ant"),!1;case 14:return c()&&W.slice_from("ent"),!1;case 15:return r=W.limit-W.cursor,W.in_grouping_b(F,97,251)&&c()&&(W.cursor=W.limit-r,W.slice_del()),!1}return!0}return!1}function f(){var e,r;if(W.cursor<q)return!1;if(r=W.limit_backward,W.limit_backward=q,W.ket=W.cursor,!(e=W.find_among_b(x,35)))return W.limit_backward=r,!1;if(W.bra=W.cursor,1==e){if(!W.out_grouping_b(F,97,251))return W.limit_backward=r,!1;W.slice_del()}return W.limit_backward=r,!0}function m(){var e,r,s;if(W.cursor<q)return!1;if(r=W.limit_backward,W.limit_backward=q,W.ket=W.cursor,!(e=W.find_among_b(I,38)))return W.limit_backward=r,!1;switch(W.bra=W.cursor,e){case 1:if(!l())return W.limit_backward=r,!1;W.slice_del();break;case 2:W.slice_del();break;case 3:W.slice_del(),s=W.limit-W.cursor,W.ket=W.cursor,W.eq_s_b(1,"e")?(W.bra=W.cursor,W.slice_del()):W.cursor=W.limit-s}return W.limit_backward=r,!0}function _(){var e,r,s,i,n=W.limit-W.cursor;if(W.ket=W.cursor,W.eq_s_b(1,"s")?(W.bra=W.cursor,r=W.limit-W.cursor,W.out_grouping_b(S,97,232)?(W.cursor=W.limit-r,W.slice_del()):W.cursor=W.limit-n):W.cursor=W.limit-n,W.cursor>=q){if(s=W.limit_backward,W.limit_backwarLine truncated
|
||||
@@ -1 +0,0 @@
|
||||
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.he=function(){this.pipeline.reset(),this.pipeline.add(e.he.trimmer,e.he.stopWordFilter,e.he.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.he.stemmer))},e.he.wordCharacters="֑-״א-תa-zA-Za-zA-Z0-90-9",e.he.trimmer=e.trimmerSupport.generateTrimmer(e.he.wordCharacters),e.Pipeline.registerFunction(e.he.trimmer,"trimmer-he"),e.he.stemmer=function(){var e=this;return e.result=!1,e.preRemoved=!1,e.sufRemoved=!1,e.pre={pre1:"ה ו י ת",pre2:"ב כ ל מ ש כש",pre3:"הב הכ הל המ הש בש לכ",pre4:"וב וכ ול ומ וש",pre5:"מה שה כל",pre6:"מב מכ מל ממ מש",pre7:"בה בו בי בת כה כו כי כת לה לו לי לת",pre8:"ובה ובו ובי ובת וכה וכו וכי וכת ולה ולו ולי ולת"},e.suf={suf1:"ך כ ם ן נ",suf2:"ים ות וך וכ ום ון ונ הם הן יכ יך ינ ים",suf3:"תי תך תכ תם תן תנ",suf4:"ותי ותך ותכ ותם ותן ותנ",suf5:"נו כם כן הם הן",suf6:"ונו וכם וכן והם והן",suf7:"תכם תכן תנו תהם תהן",suf8:"הוא היא הם הן אני אתה את אנו אתם אתן",suf9:"ני נו כי כו כם כן תי תך תכ תם תן",suf10:"י ך כ ם ן נ ת"},e.patterns=JSON.parse('{"hebrewPatterns": [{"pt1": [{"c": "ה", "l": 0}]}, {"pt2": [{"c": "ו", "l": 0}]}, {"pt3": [{"c": "י", "l": 0}]}, {"pt4": [{"c": "ת", "l": 0}]}, {"pt5": [{"c": "מ", "l": 0}]}, {"pt6": [{"c": "ל", "l": 0}]}, {"pt7": [{"c": "ב", "l": 0}]}, {"pt8": [{"c": "כ", "l": 0}]}, {"pt9": [{"c": "ש", "l": 0}]}, {"pt10": [{"c": "כש", "l": 0}]}, {"pt11": [{"c": "בה", "l": 0}]}, {"pt12": [{"c": "וב", "l": 0}]}, {"pt13": [{"c": "וכ", "l": 0}]}, {"pt14": [{"c": "ול", "l": 0}]}, {"pt15": [{"c": "ומ", "l": 0}]}, {"pt16": [{"c": "וש", "l": 0}]}, {"pt17": [{"c": "הב", "l": 0}]}, {"pt18": [{"c": "הכ", "l": 0}]}, {"pt19": [{"c": "הל", "l": 0}]}, {"pt20": [{"c": "המ", "l": 0}]}, {"pt21": [{"c": "הש", "l": 0}]}, {"pt22": [{"c": "מה", "l": 0}]}, {"pt23": [{"c": "שה", "l": 0}]}, {"pt24": [{"c": "כל", "l": 0}]}]}'),e.execArray=["cleanWord","removeDiacritics","removeStopWords","normalizeHebrewCharacters"],e.stem=function(){var r=0;for(e.result=!1,e.preRemoved=!1,e.sufRemoved=!1;r<e.execArray.length&&1!=e.result;)e.result=e[e.execArray[r]](),r++},e.setCurrent=function(r){e.word=r},e.getCurrent=function(){return e.word},e.cleanWord=function(){return!!new RegExp("[^֑-״א-ת]").test("")},e.removeDiacritics=function(){var r=new RegExp("[ְ-ֿ]","g");return e.word=e.word.replace(r,""),!1},e.removeStopWords=function(){if("אבל או אולי אותו אותי אותך אותם אותן אותנו אז אחר אחרות אחרי אחריכן אחרים אחרת אי איזה איך אין איפה אל אלה אלו אם אנחנו אני אף אפשר את אתה אתכם אתכן אתם אתן באיזה באיזו בגלל בין בלבד בעבור בעזרת בכל בכן בלי במידה במקום שבו ברוב בשביל בשעה ש בתוך גם דרך הוא היא היה היי היכן היתה היתי הם הן הנה הסיבה שבגללה הרי ואילו ואת זאת זה זות יהיה יוכל יוכלו יותר מדי יכול יכולה יכולות יכולים יכל יכלה יכלו יש כאן כאשר כולם כולן כזה כי כיצד כך כל כלל כמו כן כפי כש לא לאו לאיזותך לאן לבין לה להיות להם להן לו לזה לזות לי לך לכם לכן למה למעלה למעלה מ למטה למטה מ למעט למקום שבו למרות לנו לעבר לעיכן לפיכך לפני מאד מאחורי מאיזו סיבה מאין מאיפה מבלי מבעד מדוע מה מהיכן מול מחוץ מי מידע מכאן מכל מכן מלבד מן מנין מסוגל מעט מעטים מעל מצד מקום בו מתחת מתי נגד נגר נו עד עז על עלי עליו עליה עליהם עליך עלינו עם עצמה עצמהם עצמהן עצמו עצמי עצמם עצמן עצמנו פה רק שוב של שלה שלהם שלהן שלו שלי שלך שלכה שלכם שלכן שלנו שם תהיה תחת".split(" ").indexOf(e.word)>=0)return!0},e.normalizeHebrewCharacters=function(){return e.word=e.word.replace("ך","כ"),e.word=e.word.replace("ם","מ"),e.word=e.word.replace("ן","נ"),e.word=e.word.replace("ף","פ"),e.word=e.word.replace("ץ","צ"),!1},function(r){return"function"==typeof r.update?r.update(function(r){return e.setLine truncated
|
||||
@@ -1 +0,0 @@
|
||||
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.hi=function(){this.pipeline.reset(),this.pipeline.add(e.hi.trimmer,e.hi.stopWordFilter,e.hi.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.hi.stemmer))},e.hi.wordCharacters="ऀ-ःऄ-एऐ-टठ-यर-िी-ॏॐ-य़ॠ-९॰-ॿa-zA-Za-zA-Z0-90-9",e.hi.trimmer=e.trimmerSupport.generateTrimmer(e.hi.wordCharacters),e.Pipeline.registerFunction(e.hi.trimmer,"trimmer-hi"),e.hi.stopWordFilter=e.generateStopWordFilter("अत अपना अपनी अपने अभी अंदर आदि आप इत्यादि इन इनका इन्हीं इन्हें इन्हों इस इसका इसकी इसके इसमें इसी इसे उन उनका उनकी उनके उनको उन्हीं उन्हें उन्हों उस उसके उसी उसे एक एवं एस ऐसे और कई कर करता करते करना करने करें कहते कहा का काफ़ी कि कितना किन्हें किन्हों किया किर किस किसी किसे की कुछ कुल के को कोई कौन कौनसा गया घर जब जहाँ जा जितना जिन जिन्हें जिन्हों जिस जिसे जीधर जैसा जैसे जो तक तब तरह तिन तिन्हें तिन्हों तिस तिसे तो था थी थे दबारा दिया दुसरा दूसरे दो द्वारा न नके नहीं ना निहायत नीचे ने पर पहले पूरा पे फिर बनी बही बहुत बाद बाला बिलकुल भी भीतर मगर मानो मे में यदि यह यहाँ यही या यिह ये रखें रहा रहे ऱ्वासा लिए लिये लेकिन व वग़ैरह वर्ग वह वहाँ वहीं वाले वुह वे वो सकता सकते सबसे सभी साथ साबुत साभ सारा से सो संग ही हुआ हुई हुए है हैं हो होता होती होते होना होने".split(" ")),e.hi.stemmer=function(){return function(e){return"function"==typeof e.update?e.update(function(e){return e}):e}}();var r=e.wordcut;r.init(),e.hi.tokenizer=function(i){if(!arguments.length||null==i||void 0==i)return[];if(Array.isArray(i))return i.map(function(r){return isLunr2?new e.Token(r.toLowerCase()):r.toLowerCase()});var t=i.toString().toLowerCase().replace(/^\s+/,"");return r.cut(t).split("|")},e.Pipeline.registerFunction(e.hi.stemmer,"stemmer-hi"),e.Pipeline.registerFunction(e.hi.stopWordFilter,"stopWordFilter-hi")}});
|
||||
@@ -1,18 +0,0 @@
|
||||
/*!
|
||||
* Lunr languages, `Hungarian` language
|
||||
* https://github.com/MihaiValentin/lunr-languages
|
||||
*
|
||||
* Copyright 2014, Mihai Valentin
|
||||
* http://www.mozilla.org/MPL/
|
||||
*/
|
||||
/*!
|
||||
* based on
|
||||
* Snowball JavaScript Library v0.3
|
||||
* http://code.google.com/p/urim/
|
||||
* http://snowball.tartarus.org/
|
||||
*
|
||||
* Copyright 2010, Oleg Mazko
|
||||
* http://www.mozilla.org/MPL/
|
||||
*/
|
||||
|
||||
!function(e,n){"function"==typeof define&&define.amd?define(n):"object"==typeof exports?module.exports=n():n()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.hu=function(){this.pipeline.reset(),this.pipeline.add(e.hu.trimmer,e.hu.stopWordFilter,e.hu.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.hu.stemmer))},e.hu.wordCharacters="A-Za-zªºÀ-ÖØ-öø-ʸˠ-ˤᴀ-ᴥᴬ-ᵜᵢ-ᵥᵫ-ᵷᵹ-ᶾḀ-ỿⁱⁿₐ-ₜKÅℲⅎⅠ-ↈⱠ-ⱿꜢ-ꞇꞋ-ꞭꞰ-ꞷꟷ-ꟿꬰ-ꭚꭜ-ꭤff-stA-Za-z",e.hu.trimmer=e.trimmerSupport.generateTrimmer(e.hu.wordCharacters),e.Pipeline.registerFunction(e.hu.trimmer,"trimmer-hu"),e.hu.stemmer=function(){var n=e.stemmerSupport.Among,r=e.stemmerSupport.SnowballProgram,i=new function(){function e(){var e,n=L.cursor;if(d=L.limit,L.in_grouping(W,97,252))for(;;){if(e=L.cursor,L.out_grouping(W,97,252))return L.cursor=e,L.find_among(g,8)||(L.cursor=e,e<L.limit&&L.cursor++),void(d=L.cursor);if(L.cursor=e,e>=L.limit)return void(d=e);L.cursor++}if(L.cursor=n,L.out_grouping(W,97,252)){for(;!L.in_grouping(W,97,252);){if(L.cursor>=L.limit)return;L.cursor++}d=L.cursor}}function i(){return d<=L.cursor}function a(){var e;if(L.ket=L.cursor,(e=L.find_among_b(h,2))&&(L.bra=L.cursor,i()))switch(e){case 1:L.slice_from("a");break;case 2:L.slice_from("e")}}function t(){var e=L.limit-L.cursor;return!!L.find_among_b(p,23)&&(L.cursor=L.limit-e,!0)}function s(){if(L.cursor>L.limit_backward){L.cursor--,L.ket=L.cursor;var e=L.cursor-1;L.limit_backward<=e&&e<=L.limit&&(L.cursor=e,L.bra=e,L.slice_del())}}function c(){var e;if(L.ket=L.cursor,(e=L.find_among_b(_,2))&&(L.bra=L.cursor,i())){if((1==e||2==e)&&!t())return;L.slice_del(),s()}}function o(){L.ket=L.cursor,L.find_among_b(v,44)&&(L.bra=L.cursor,i()&&(L.slice_del(),a()))}function w(){var e;if(L.ket=L.cursor,(e=L.find_among_b(z,3))&&(L.bra=L.cursor,i()))switch(e){case 1:L.slice_from("e");break;case 2:case 3:L.slice_from("a")}}function l(){var e;if(L.ket=L.cursor,(e=L.find_among_b(y,6))&&(L.bra=L.cursor,i()))switch(e){case 1:case 2:L.slice_del();break;case 3:L.slice_from("a");break;case 4:L.slice_from("e")}}function u(){var e;if(L.ket=L.cursor,(e=L.find_among_b(j,2))&&(L.bra=L.cursor,i())){if((1==e||2==e)&&!t())return;L.slice_del(),s()}}function m(){var e;if(L.ket=L.cursor,(e=L.find_among_b(C,7))&&(L.bra=L.cursor,i()))switch(e){case 1:L.slice_from("a");break;case 2:L.slice_from("e");break;case 3:case 4:case 5:case 6:case 7:L.slice_del()}}function k(){var e;if(L.ket=L.cursor,(e=L.find_among_b(P,12))&&(L.bra=L.cursor,i()))switch(e){case 1:case 4:case 7:case 9:L.slice_del();break;case 2:case 5:case 8:L.slice_from("e");break;case 3:case 6:L.slice_from("a")}}function f(){var e;if(L.ket=L.cursor,(e=L.find_among_b(F,31))&&(L.bra=L.cursor,i()))switch(e){case 1:case 4:case 7:case 8:case 9:case 12:case 13:case 16:case 17:case 18:L.slice_del();break;case 2:case 5:case 10:case 14:case 19:L.slice_from("a");break;case 3:case 6:case 11:case 15:case 20:L.slice_from("e")}}function b(){var e;if(L.ket=L.cursor,(e=L.find_among_b(S,42))&&(L.bra=L.cursor,i()))switch(e){case 1:case 4:case 5:case 6:case 9:case 10:case 11:case 14:case 15:case 16:case 17:case 20:case 21:case 24:case 25:case 26:case 29:L.slice_del();break;case 2:case 7:case 12:case 18:case 22:case 27:L.slice_from("a");break;case 3:case 8:case 13:case 19:case 23:case 28:L.slice_from("e")}}var d,g=[new n("cs",-1,-1),new n("dzs",-1,-1),new n("gy",-1,-1),new n("ly",-1,-1),new n("ny",-1,-1),new n("sz",-1,-1),new n("ty",-1,-1),new n("zs",-1,-1)],h=[new n("á",-1,1),new n("é",-1,2)],p=[new n("bb",-1,-1),new n("cc",-1,-1),new n("dd",-1,-1),new n("ff",-1,-1),new n("gg",-1,-1),new n("jj",-1,-1),new n("kk",-1,-1),new n("ll",-1,-1),new n("mm",-1,-1),new n("nn",-1,-1),new n("pp",-1,-1),new n("rr",-1,-1),new n("ccs",-1,-1),new n("ss",-1,-1),new n("zzs",-1,-1),new n("tt",-1,-1),new n("vv",-1,-1),new n("ggy",-1,-1),new n("lly",-1,-1),new n("nny",-1,-1),new n("tty",-1,-1),new n("ssz",-1,-1),new n("zz",-1,-1)],_=[new n("al",-1,1),new n("el",-1,2)],v=[new n("ba",-1,-1),new n("ra",-1,-1),new n("be",-1,-1),new n("re",-1,-1),new n("ig",-1,-1),new n("nak",-1,-1),new n("nek",-1,-1),new n("val",-1,-1),new n("vel",-1,-1),new n("ul",-1,-1),new n("nál",-1,-1),new n("nél",-1,-1),new n("ból",-1,-1),new n("ról",-1,-1),new n("tól",-1,-1),new n("bõl",-1,-1),new n("rõl",-1,-1),new n("tõl",-1,-1),new n("ül",-1,-1),new n("n",-1,-1),new n("an",19,-1),new n("ban",20,-1),new n("en",19,-1),new n("ben",22,-1),new n("képpen",22,-1),new n("on",19,-1),new n("ön",19,-1),new n("képp",-1,-1),new n("kor",-1,-1),new n("t",-1,-1),new n("at",29,-1),new n("et",29,-1),new n("ként",29,-1),new n("anként",32,-1),new n("enként",32,-1),new n("onként",32,-1),Line truncated
|
||||
@@ -1 +0,0 @@
|
||||
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.hy=function(){this.pipeline.reset(),this.pipeline.add(e.hy.trimmer,e.hy.stopWordFilter)},e.hy.wordCharacters="[A-Za-z-֏ff-ﭏ]",e.hy.trimmer=e.trimmerSupport.generateTrimmer(e.hy.wordCharacters),e.Pipeline.registerFunction(e.hy.trimmer,"trimmer-hy"),e.hy.stopWordFilter=e.generateStopWordFilter("դու և եք էիր էիք հետո նաև նրանք որը վրա է որ պիտի են այս մեջ ն իր ու ի այդ որոնք այն կամ էր մի ես համար այլ իսկ էին ենք հետ ին թ էինք մենք նրա նա դուք եմ էի ըստ որպես ում".split(" ")),e.Pipeline.registerFunction(e.hy.stopWordFilter,"stopWordFilter-hy"),e.hy.stemmer=function(){return function(e){return"function"==typeof e.update?e.update(function(e){return e}):e}}(),e.Pipeline.registerFunction(e.hy.stemmer,"stemmer-hy")}});
|
||||
@@ -1,18 +0,0 @@
|
||||
/*!
|
||||
* Lunr languages, `Italian` language
|
||||
* https://github.com/MihaiValentin/lunr-languages
|
||||
*
|
||||
* Copyright 2014, Mihai Valentin
|
||||
* http://www.mozilla.org/MPL/
|
||||
*/
|
||||
/*!
|
||||
* based on
|
||||
* Snowball JavaScript Library v0.3
|
||||
* http://code.google.com/p/urim/
|
||||
* http://snowball.tartarus.org/
|
||||
*
|
||||
* Copyright 2010, Oleg Mazko
|
||||
* http://www.mozilla.org/MPL/
|
||||
*/
|
||||
|
||||
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.it=function(){this.pipeline.reset(),this.pipeline.add(e.it.trimmer,e.it.stopWordFilter,e.it.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.it.stemmer))},e.it.wordCharacters="A-Za-zªºÀ-ÖØ-öø-ʸˠ-ˤᴀ-ᴥᴬ-ᵜᵢ-ᵥᵫ-ᵷᵹ-ᶾḀ-ỿⁱⁿₐ-ₜKÅℲⅎⅠ-ↈⱠ-ⱿꜢ-ꞇꞋ-ꞭꞰ-ꞷꟷ-ꟿꬰ-ꭚꭜ-ꭤff-stA-Za-z",e.it.trimmer=e.trimmerSupport.generateTrimmer(e.it.wordCharacters),e.Pipeline.registerFunction(e.it.trimmer,"trimmer-it"),e.it.stemmer=function(){var r=e.stemmerSupport.Among,n=e.stemmerSupport.SnowballProgram,i=new function(){function e(e,r,n){return!(!x.eq_s(1,e)||(x.ket=x.cursor,!x.in_grouping(L,97,249)))&&(x.slice_from(r),x.cursor=n,!0)}function i(){for(var r,n,i,o,t=x.cursor;;){if(x.bra=x.cursor,r=x.find_among(h,7))switch(x.ket=x.cursor,r){case 1:x.slice_from("à");continue;case 2:x.slice_from("è");continue;case 3:x.slice_from("ì");continue;case 4:x.slice_from("ò");continue;case 5:x.slice_from("ù");continue;case 6:x.slice_from("qU");continue;case 7:if(x.cursor>=x.limit)break;x.cursor++;continue}break}for(x.cursor=t;;)for(n=x.cursor;;){if(i=x.cursor,x.in_grouping(L,97,249)){if(x.bra=x.cursor,o=x.cursor,e("u","U",i))break;if(x.cursor=o,e("i","I",i))break}if(x.cursor=i,x.cursor>=x.limit)return void(x.cursor=n);x.cursor++}}function o(e){if(x.cursor=e,!x.in_grouping(L,97,249))return!1;for(;!x.out_grouping(L,97,249);){if(x.cursor>=x.limit)return!1;x.cursor++}return!0}function t(){if(x.in_grouping(L,97,249)){var e=x.cursor;if(x.out_grouping(L,97,249)){for(;!x.in_grouping(L,97,249);){if(x.cursor>=x.limit)return o(e);x.cursor++}return!0}return o(e)}return!1}function s(){var e,r=x.cursor;if(!t()){if(x.cursor=r,!x.out_grouping(L,97,249))return;if(e=x.cursor,x.out_grouping(L,97,249)){for(;!x.in_grouping(L,97,249);){if(x.cursor>=x.limit)return x.cursor=e,void(x.in_grouping(L,97,249)&&x.cursor<x.limit&&x.cursor++);x.cursor++}return void(k=x.cursor)}if(x.cursor=e,!x.in_grouping(L,97,249)||x.cursor>=x.limit)return;x.cursor++}k=x.cursor}function a(){for(;!x.in_grouping(L,97,249);){if(x.cursor>=x.limit)return!1;x.cursor++}for(;!x.out_grouping(L,97,249);){if(x.cursor>=x.limit)return!1;x.cursor++}return!0}function u(){var e=x.cursor;k=x.limit,p=k,g=k,s(),x.cursor=e,a()&&(p=x.cursor,a()&&(g=x.cursor))}function c(){for(var e;;){if(x.bra=x.cursor,!(e=x.find_among(q,3)))break;switch(x.ket=x.cursor,e){case 1:x.slice_from("i");break;case 2:x.slice_from("u");break;case 3:if(x.cursor>=x.limit)return;x.cursor++}}}function w(){return k<=x.cursor}function l(){return p<=x.cursor}function m(){return g<=x.cursor}function f(){var e;if(x.ket=x.cursor,x.find_among_b(C,37)&&(x.bra=x.cursor,(e=x.find_among_b(z,5))&&w()))switch(e){case 1:x.slice_del();break;case 2:x.slice_from("e")}}function v(){var e;if(x.ket=x.cursor,!(e=x.find_among_b(S,51)))return!1;switch(x.bra=x.cursor,e){case 1:if(!m())return!1;x.slice_del();break;case 2:if(!m())return!1;x.slice_del(),x.ket=x.cursor,x.eq_s_b(2,"ic")&&(x.bra=x.cursor,m()&&x.slice_del());break;case 3:if(!m())return!1;x.slice_from("log");break;case 4:if(!m())return!1;x.slice_from("u");break;case 5:if(!m())return!1;x.slice_from("ente");break;case 6:if(!w())return!1;x.slice_del();break;case 7:if(!l())return!1;x.slice_del(),x.ket=x.cursor,e=x.find_among_b(P,4),e&&(x.bra=x.cursor,m()&&(x.slice_del(),1==e&&(x.ket=x.cursor,x.eq_s_b(2,"at")&&(x.bra=x.cursor,m()&&x.slice_del()))));break;case 8:if(!m())return!1;x.slice_del(),x.ket=x.cursor,e=x.find_among_b(F,3),e&&(x.bra=x.cursor,1==e&&m()&&x.slice_del());break;case 9:if(!m())return!1;x.slice_del(),x.ket=x.cursor,x.eq_s_b(2,"at")&&(x.bra=x.cursor,m()&&(x.slice_del(),x.ket=x.cursor,x.eq_s_b(2,"ic")&&(x.bra=x.cursor,m()&&x.slice_del())))}return!0}function b(){var e,r;x.cursor>=k&&(r=x.limit_backward,x.limit_backward=k,x.ket=x.cursor,e=x.find_among_b(W,87),e&&(x.bra=x.cursor,1==e&&x.slice_del()),x.limit_backward=r)}function d(){var e=x.limit-x.cursor;if(x.ket=x.cursor,x.in_grouping_b(y,97,242)&&(x.bra=x.cursor,w()&&(x.slice_del(),x.ket=x.cursor,x.eq_s_b(1,"i")&&(x.bra=x.cursor,w()))))return void x.slice_del();x.cursor=x.limit-e}function _(){d(),x.ket=x.cursor,x.eq_s_b(1,"h")&&(x.bra=x.cursor,x.in_grouping_b(U,99,103)&&w()&&x.slice_del())}var g,p,k,h=[new r("",-1,7),new r("qu",0,6),new r("á",0,1),new r("é",0,2),new r("í",0,3),new r("ó",0,4),new r("ú",0,5)],q=[new r("",-1,3),new r("I",0,1),new r("U",0,2)],C=[new r("la",-1,-1),new r("cela",0,-1),new r("gliela",0,-1),new r("mela",0,-1),new r("tela",0,-1),new r("vela",0,-1),new r("le",-1,-1),new r("cele",6,-1),new r("gliele",6Line truncated
|
||||
@@ -1 +0,0 @@
|
||||
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");var r="2"==e.version[0];e.ja=function(){this.pipeline.reset(),this.pipeline.add(e.ja.trimmer,e.ja.stopWordFilter,e.ja.stemmer),r?this.tokenizer=e.ja.tokenizer:(e.tokenizer&&(e.tokenizer=e.ja.tokenizer),this.tokenizerFn&&(this.tokenizerFn=e.ja.tokenizer))};var t=new e.TinySegmenter;e.ja.tokenizer=function(i){var n,o,s,p,a,u,m,l,c,f;if(!arguments.length||null==i||void 0==i)return[];if(Array.isArray(i))return i.map(function(t){return r?new e.Token(t.toLowerCase()):t.toLowerCase()});for(o=i.toString().toLowerCase().replace(/^\s+/,""),n=o.length-1;n>=0;n--)if(/\S/.test(o.charAt(n))){o=o.substring(0,n+1);break}for(a=[],s=o.length,c=0,l=0;c<=s;c++)if(u=o.charAt(c),m=c-l,u.match(/\s/)||c==s){if(m>0)for(p=t.segment(o.slice(l,c)).filter(function(e){return!!e}),f=l,n=0;n<p.length;n++)r?a.push(new e.Token(p[n],{position:[f,p[n].length],index:a.length})):a.push(p[n]),f+=p[n].length;l=c+1}return a},e.ja.stemmer=function(){return function(e){return e}}(),e.Pipeline.registerFunction(e.ja.stemmer,"stemmer-ja"),e.ja.wordCharacters="一二三四五六七八九十百千万億兆一-龠々〆ヵヶぁ-んァ-ヴーア-ン゙a-zA-Za-zA-Z0-90-9",e.ja.trimmer=e.trimmerSupport.generateTrimmer(e.ja.wordCharacters),e.Pipeline.registerFunction(e.ja.trimmer,"trimmer-ja"),e.ja.stopWordFilter=e.generateStopWordFilter("これ それ あれ この その あの ここ そこ あそこ こちら どこ だれ なに なん 何 私 貴方 貴方方 我々 私達 あの人 あのかた 彼女 彼 です あります おります います は が の に を で え から まで より も どの と し それで しかし".split(" ")),e.Pipeline.registerFunction(e.ja.stopWordFilter,"stopWordFilter-ja"),e.jp=e.ja,e.Pipeline.registerFunction(e.jp.stemmer,"stemmer-jp"),e.Pipeline.registerFunction(e.jp.trimmer,"trimmer-jp"),e.Pipeline.registerFunction(e.jp.stopWordFilter,"stopWordFilter-jp")}});
|
||||
@@ -1 +0,0 @@
|
||||
module.exports=require("./lunr.ja");
|
||||
@@ -1 +0,0 @@
|
||||
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.kn=function(){this.pipeline.reset(),this.pipeline.add(e.kn.trimmer,e.kn.stopWordFilter,e.kn.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.kn.stemmer))},e.kn.wordCharacters="ಀ-಄ಅ-ಔಕ-ಹಾ-ೌ಼-ಽೕ-ೖೝ-ೞೠ-ೡೢ-ೣ೦-೯ೱ-ೳ",e.kn.trimmer=e.trimmerSupport.generateTrimmer(e.kn.wordCharacters),e.Pipeline.registerFunction(e.kn.trimmer,"trimmer-kn"),e.kn.stopWordFilter=e.generateStopWordFilter("ಮತ್ತು ಈ ಒಂದು ರಲ್ಲಿ ಹಾಗೂ ಎಂದು ಅಥವಾ ಇದು ರ ಅವರು ಎಂಬ ಮೇಲೆ ಅವರ ತನ್ನ ಆದರೆ ತಮ್ಮ ನಂತರ ಮೂಲಕ ಹೆಚ್ಚು ನ ಆ ಕೆಲವು ಅನೇಕ ಎರಡು ಹಾಗು ಪ್ರಮುಖ ಇದನ್ನು ಇದರ ಸುಮಾರು ಅದರ ಅದು ಮೊದಲ ಬಗ್ಗೆ ನಲ್ಲಿ ರಂದು ಇತರ ಅತ್ಯಂತ ಹೆಚ್ಚಿನ ಸಹ ಸಾಮಾನ್ಯವಾಗಿ ನೇ ಹಲವಾರು ಹೊಸ ದಿ ಕಡಿಮೆ ಯಾವುದೇ ಹೊಂದಿದೆ ದೊಡ್ಡ ಅನ್ನು ಇವರು ಪ್ರಕಾರ ಇದೆ ಮಾತ್ರ ಕೂಡ ಇಲ್ಲಿ ಎಲ್ಲಾ ವಿವಿಧ ಅದನ್ನು ಹಲವು ರಿಂದ ಕೇವಲ ದ ದಕ್ಷಿಣ ಗೆ ಅವನ ಅತಿ ನೆಯ ಬಹಳ ಕೆಲಸ ಎಲ್ಲ ಪ್ರತಿ ಇತ್ಯಾದಿ ಇವು ಬೇರೆ ಹೀಗೆ ನಡುವೆ ಇದಕ್ಕೆ ಎಸ್ ಇವರ ಮೊದಲು ಶ್ರೀ ಮಾಡುವ ಇದರಲ್ಲಿ ರೀತಿಯ ಮಾಡಿದ ಕಾಲ ಅಲ್ಲಿ ಮಾಡಲು ಅದೇ ಈಗ ಅವು ಗಳು ಎ ಎಂಬುದು ಅವನು ಅಂದರೆ ಅವರಿಗೆ ಇರುವ ವಿಶೇಷ ಮುಂದೆ ಅವುಗಳ ಮುಂತಾದ ಮೂಲ ಬಿ ಮೀ ಒಂದೇ ಇನ್ನೂ ಹೆಚ್ಚಾಗಿ ಮಾಡಿ ಅವರನ್ನು ಇದೇ ಯ ರೀತಿಯಲ್ಲಿ ಜೊತೆ ಅದರಲ್ಲಿ ಮಾಡಿದರು ನಡೆದ ಆಗ ಮತ್ತೆ ಪೂರ್ವ ಆತ ಬಂದ ಯಾವ ಒಟ್ಟು ಇತರೆ ಹಿಂದೆ ಪ್ರಮಾಣದ ಗಳನ್ನು ಕುರಿತು ಯು ಆದ್ದರಿಂದ ಅಲ್ಲದೆ ನಗರದ ಮೇಲಿನ ಏಕೆಂದರೆ ರಷ್ಟು ಎಂಬುದನ್ನು ಬಾರಿ ಎಂದರೆ ಹಿಂದಿನ ಆದರೂ ಆದ ಸಂಬಂಧಿಸಿದ ಮತ್ತೊಂದು ಸಿ ಆತನ ".split(" ")),e.kn.stemmer=function(){return function(e){return"function"==typeof e.update?e.update(function(e){return e}):e}}();var r=e.wordcut;r.init(),e.kn.tokenizer=function(t){if(!arguments.length||null==t||void 0==t)return[];if(Array.isArray(t))return t.map(function(r){return isLunr2?new e.Token(r.toLowerCase()):r.toLowerCase()});var n=t.toString().toLowerCase().replace(/^\s+/,"");return r.cut(n).split("|")},e.Pipeline.registerFunction(e.kn.stemmer,"stemmer-kn"),e.Pipeline.registerFunction(e.kn.stopWordFilter,"stopWordFilter-kn")}});
|
||||
@@ -1 +0,0 @@
|
||||
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.ko=function(){this.pipeline.reset(),this.pipeline.add(e.ko.trimmer,e.ko.stopWordFilter)},e.ko.wordCharacters="[A-Za-z가-힣]",e.ko.trimmer=e.trimmerSupport.generateTrimmer(e.ko.wordCharacters),e.Pipeline.registerFunction(e.ko.trimmer,"trimmer-ko"),e.ko.stopWordFilter=e.generateStopWordFilter("아 휴 아이구 아이쿠 아이고 어 나 우리 저희 따라 의해 을 를 에 의 가 으로 로 에게 뿐이다 의거하여 근거하여 입각하여 기준으로 예하면 예를 들면 예를 들자면 저 소인 소생 저희 지말고 하지마 하지마라 다른 물론 또한 그리고 비길수 없다 해서는 안된다 뿐만 아니라 만이 아니다 만은 아니다 막론하고 관계없이 그치지 않다 그러나 그런데 하지만 든간에 논하지 않다 따지지 않다 설사 비록 더라도 아니면 만 못하다 하는 편이 낫다 불문하고 향하여 향해서 향하다 쪽으로 틈타 이용하여 타다 오르다 제외하고 이 외에 이 밖에 하여야 비로소 한다면 몰라도 외에도 이곳 여기 부터 기점으로 따라서 할 생각이다 하려고하다 이리하여 그리하여 그렇게 함으로써 하지만 일때 할때 앞에서 중에서 보는데서 으로써 로써 까지 해야한다 일것이다 반드시 할줄알다 할수있다 할수있어 임에 틀림없다 한다면 등 등등 제 겨우 단지 다만 할뿐 딩동 댕그 대해서 대하여 대하면 훨씬 얼마나 얼마만큼 얼마큼 남짓 여 얼마간 약간 다소 좀 조금 다수 몇 얼마 지만 하물며 또한 그러나 그렇지만 하지만 이외에도 대해 말하자면 뿐이다 다음에 반대로 반대로 말하자면 이와 반대로 바꾸어서 말하면 바꾸어서 한다면 만약 그렇지않으면 까악 툭 딱 삐걱거리다 보드득 비걱거리다 꽈당 응당 해야한다 에 가서 각 각각 여러분 각종 각자 제각기 하도록하다 와 과 그러므로 그래서 고로 한 까닭에 하기 때문에 거니와 이지만 대하여 관하여 관한 과연 실로 아니나다를가 생각한대로 진짜로 한적이있다 하곤하였다 하 하하 허허 아하 거바 와 오 왜 어째서 무엇때문에 어찌 하겠는가 무슨 어디 어느곳 더군다나 하물며 더욱이는 어느때 언제 야 이봐 어이 여보시오 흐흐 흥 휴 헉헉 헐떡헐떡 영차 여차 어기여차 끙끙 아야 앗 아야 콸콸 졸졸 좍좍 뚝뚝 주룩주룩 솨 우르르 그래도 또 그리고 바꾸어말하면 바꾸어말하자면 혹은 혹시 답다 및 그에 따르는 때가 되어 즉 지든지 설령 가령 하더라도 할지라도 일지라도 지든지 몇 거의 하마터면 인젠 이젠 된바에야 된이상 만큼\t어찌됏든 그위에 게다가 점에서 보아 비추어 보아 고려하면 하게될것이다 일것이다 비교적 좀 보다더 비하면 시키다 하게하다 할만하다 의해서 연이서 이어서 잇따라 뒤따라 뒤이어 결국 의지하여 기대여 통하여 자마자 더욱더 불구하고 얼마든지 마음대로 주저하지 않고 곧 즉시 바로 당장 하자마자 밖에 안된다 하면된다 그래 그렇지 요컨대 다시 말하자면 바꿔 말하면 즉 구체적으로 말하자면 시작하여 시초에 이상 허 헉 허걱 바와같이 해도좋다 해도된다 게다가 더구나 하물며 와르르 팍 퍽 펄렁 동안 이래 하고있었다 이었다 에서 로부터 까지 예하면 했어요 해요 함께 같이 더불어 마저 마저도 양자 모두 습니다 가까스로 하려고하다 즈음하여 다른 다른 방면으로 해봐요 습니까 했어요 말할것도 없고 무릎쓰고 개의치않고 하는것만 못하다 하는것이 낫다 매 매번 들 모 어느것 어느 로써 갖고말하자면 어디 어느쪽 어느것 어느해 어느 년도 라 해도 언젠가 어떤것 어느것 저기 저쪽 저것 그때 그럼 그러면 요만한걸 그래 그때 저것만큼 그저 이르기까지 할 줄 안다 할 힘이 있다 너 너희 당신 어찌 설마 차라리 할지언정 할지라도 할망정 할지언정 구토하다 게우다 토하다 메쓰겁다 옆사람 퉤 쳇 의거하여 근거하여 의해 따라 힘입어 그 다음 버금 두번째로 기타 첫번째로 나머지는 그중에서 견지에서 형식으로 쓰여 입장에서 위해서 단지 의해되다 하도록시키다 뿐만아니라 반대로 전후 전자 앞의것 잠시 잠깐 하면서 그렇지만 다음에 그러한즉 그런즉 남들 아무거나 어찌하Line truncated
|
||||
@@ -1 +0,0 @@
|
||||
!function(e,t){"function"==typeof define&&define.amd?define(t):"object"==typeof exports?module.exports=t():t()(e.lunr)}(this,function(){return function(e){e.multiLanguage=function(){for(var t=Array.prototype.slice.call(arguments),i=t.join("-"),r="",n=[],s=[],p=0;p<t.length;++p)"en"==t[p]?(r+="\\w",n.unshift(e.stopWordFilter),n.push(e.stemmer),s.push(e.stemmer)):(r+=e[t[p]].wordCharacters,e[t[p]].stopWordFilter&&n.unshift(e[t[p]].stopWordFilter),e[t[p]].stemmer&&(n.push(e[t[p]].stemmer),s.push(e[t[p]].stemmer)));var o=e.trimmerSupport.generateTrimmer(r);return e.Pipeline.registerFunction(o,"lunr-multi-trimmer-"+i),n.unshift(o),function(){this.pipeline.reset(),this.pipeline.add.apply(this.pipeline,n),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add.apply(this.searchPipeline,s))}}}});
|
||||
@@ -1,18 +0,0 @@
|
||||
/*!
|
||||
* Lunr languages, `Dutch` language
|
||||
* https://github.com/MihaiValentin/lunr-languages
|
||||
*
|
||||
* Copyright 2014, Mihai Valentin
|
||||
* http://www.mozilla.org/MPL/
|
||||
*/
|
||||
/*!
|
||||
* based on
|
||||
* Snowball JavaScript Library v0.3
|
||||
* http://code.google.com/p/urim/
|
||||
* http://snowball.tartarus.org/
|
||||
*
|
||||
* Copyright 2010, Oleg Mazko
|
||||
* http://www.mozilla.org/MPL/
|
||||
*/
|
||||
|
||||
!function(r,e){"function"==typeof define&&define.amd?define(e):"object"==typeof exports?module.exports=e():e()(r.lunr)}(this,function(){return function(r){if(void 0===r)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===r.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");r.nl=function(){this.pipeline.reset(),this.pipeline.add(r.nl.trimmer,r.nl.stopWordFilter,r.nl.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(r.nl.stemmer))},r.nl.wordCharacters="A-Za-zªºÀ-ÖØ-öø-ʸˠ-ˤᴀ-ᴥᴬ-ᵜᵢ-ᵥᵫ-ᵷᵹ-ᶾḀ-ỿⁱⁿₐ-ₜKÅℲⅎⅠ-ↈⱠ-ⱿꜢ-ꞇꞋ-ꞭꞰ-ꞷꟷ-ꟿꬰ-ꭚꭜ-ꭤff-stA-Za-z",r.nl.trimmer=r.trimmerSupport.generateTrimmer(r.nl.wordCharacters),r.Pipeline.registerFunction(r.nl.trimmer,"trimmer-nl"),r.nl.stemmer=function(){var e=r.stemmerSupport.Among,i=r.stemmerSupport.SnowballProgram,n=new function(){function r(){for(var r,e,i,o=C.cursor;;){if(C.bra=C.cursor,r=C.find_among(b,11))switch(C.ket=C.cursor,r){case 1:C.slice_from("a");continue;case 2:C.slice_from("e");continue;case 3:C.slice_from("i");continue;case 4:C.slice_from("o");continue;case 5:C.slice_from("u");continue;case 6:if(C.cursor>=C.limit)break;C.cursor++;continue}break}for(C.cursor=o,C.bra=o,C.eq_s(1,"y")?(C.ket=C.cursor,C.slice_from("Y")):C.cursor=o;;)if(e=C.cursor,C.in_grouping(q,97,232)){if(i=C.cursor,C.bra=i,C.eq_s(1,"i"))C.ket=C.cursor,C.in_grouping(q,97,232)&&(C.slice_from("I"),C.cursor=e);else if(C.cursor=i,C.eq_s(1,"y"))C.ket=C.cursor,C.slice_from("Y"),C.cursor=e;else if(n(e))break}else if(n(e))break}function n(r){return C.cursor=r,r>=C.limit||(C.cursor++,!1)}function o(){_=C.limit,d=_,t()||(_=C.cursor,_<3&&(_=3),t()||(d=C.cursor))}function t(){for(;!C.in_grouping(q,97,232);){if(C.cursor>=C.limit)return!0;C.cursor++}for(;!C.out_grouping(q,97,232);){if(C.cursor>=C.limit)return!0;C.cursor++}return!1}function s(){for(var r;;)if(C.bra=C.cursor,r=C.find_among(p,3))switch(C.ket=C.cursor,r){case 1:C.slice_from("y");break;case 2:C.slice_from("i");break;case 3:if(C.cursor>=C.limit)return;C.cursor++}}function u(){return _<=C.cursor}function c(){return d<=C.cursor}function a(){var r=C.limit-C.cursor;C.find_among_b(g,3)&&(C.cursor=C.limit-r,C.ket=C.cursor,C.cursor>C.limit_backward&&(C.cursor--,C.bra=C.cursor,C.slice_del()))}function l(){var r;w=!1,C.ket=C.cursor,C.eq_s_b(1,"e")&&(C.bra=C.cursor,u()&&(r=C.limit-C.cursor,C.out_grouping_b(q,97,232)&&(C.cursor=C.limit-r,C.slice_del(),w=!0,a())))}function m(){var r;u()&&(r=C.limit-C.cursor,C.out_grouping_b(q,97,232)&&(C.cursor=C.limit-r,C.eq_s_b(3,"gem")||(C.cursor=C.limit-r,C.slice_del(),a())))}function f(){var r,e,i,n,o,t,s=C.limit-C.cursor;if(C.ket=C.cursor,r=C.find_among_b(h,5))switch(C.bra=C.cursor,r){case 1:u()&&C.slice_from("heid");break;case 2:m();break;case 3:u()&&C.out_grouping_b(j,97,232)&&C.slice_del()}if(C.cursor=C.limit-s,l(),C.cursor=C.limit-s,C.ket=C.cursor,C.eq_s_b(4,"heid")&&(C.bra=C.cursor,c()&&(e=C.limit-C.cursor,C.eq_s_b(1,"c")||(C.cursor=C.limit-e,C.slice_del(),C.ket=C.cursor,C.eq_s_b(2,"en")&&(C.bra=C.cursor,m())))),C.cursor=C.limit-s,C.ket=C.cursor,r=C.find_among_b(k,6))switch(C.bra=C.cursor,r){case 1:if(c()){if(C.slice_del(),i=C.limit-C.cursor,C.ket=C.cursor,C.eq_s_b(2,"ig")&&(C.bra=C.cursor,c()&&(n=C.limit-C.cursor,!C.eq_s_b(1,"e")))){C.cursor=C.limit-n,C.slice_del();break}C.cursor=C.limit-i,a()}break;case 2:c()&&(o=C.limit-C.cursor,C.eq_s_b(1,"e")||(C.cursor=C.limit-o,C.slice_del()));break;case 3:c()&&(C.slice_del(),l());break;case 4:c()&&C.slice_del();break;case 5:c()&&w&&C.slice_del()}C.cursor=C.limit-s,C.out_grouping_b(z,73,232)&&(t=C.limit-C.cursor,C.find_among_b(v,4)&&C.out_grouping_b(q,97,232)&&(C.cursor=C.limit-t,C.ket=C.cursor,C.cursor>C.limit_backward&&(C.cursor--,C.bra=C.cursor,C.slice_del())))}var d,_,w,b=[new e("",-1,6),new e("á",0,1),new e("ä",0,1),new e("é",0,2),new e("ë",0,2),new e("í",0,3),new e("ï",0,3),new e("ó",0,4),new e("ö",0,4),new e("ú",0,5),new e("ü",0,5)],p=[new e("",-1,3),new e("I",0,2),new e("Y",0,1)],g=[new e("dd",-1,-1),new e("kk",-1,-1),new e("tt",-1,-1)],h=[new e("ene",-1,2),new e("se",-1,3),new e("en",-1,2),new e("heden",2,1),new e("s",-1,3)],k=[new e("end",-1,1),new e("ig",-1,2),new e("ing",-1,1),new e("lijk",-1,3),new e("baar",-1,4),new e("bar",-1,5)],v=[new e("aa",-1,-1),new e("ee",-1,-1),new e("oo",-1,-1),new e("uu",-1,-1)],q=[17,65,16,1,0,0,0,0,0,0,0,0,0,0,0,0,128],z=[1,0,0,17,65,16,1,0,0,0,0,0,0,0,0,0,0,0,0,128],j=[17,67,16,1,0,0,0,0,0,0,0,0,0,0,0,0,128],C=new i;this.setCurrent=function(r){C.setCurrent(r)},this.getCurrent=function(){return C.getCurrent()},this.stem=function(){var e=C.cursor;return r(),C.cursor=e,o(),C.limit_backward=e,C.cursor=C.limit,f(),C.cursor=C.limit_backward,s(),!0}};return function(r){return"function"==typeof r.update?r.update(function(r){return n.setCurrent(r),n.stem(),n.getCurrent()}):(n.setCurrLine truncated
|
||||
@@ -1,18 +0,0 @@
|
||||
/*!
|
||||
* Lunr languages, `Norwegian` language
|
||||
* https://github.com/MihaiValentin/lunr-languages
|
||||
*
|
||||
* Copyright 2014, Mihai Valentin
|
||||
* http://www.mozilla.org/MPL/
|
||||
*/
|
||||
/*!
|
||||
* based on
|
||||
* Snowball JavaScript Library v0.3
|
||||
* http://code.google.com/p/urim/
|
||||
* http://snowball.tartarus.org/
|
||||
*
|
||||
* Copyright 2010, Oleg Mazko
|
||||
* http://www.mozilla.org/MPL/
|
||||
*/
|
||||
|
||||
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.no=function(){this.pipeline.reset(),this.pipeline.add(e.no.trimmer,e.no.stopWordFilter,e.no.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.no.stemmer))},e.no.wordCharacters="A-Za-zªºÀ-ÖØ-öø-ʸˠ-ˤᴀ-ᴥᴬ-ᵜᵢ-ᵥᵫ-ᵷᵹ-ᶾḀ-ỿⁱⁿₐ-ₜKÅℲⅎⅠ-ↈⱠ-ⱿꜢ-ꞇꞋ-ꞭꞰ-ꞷꟷ-ꟿꬰ-ꭚꭜ-ꭤff-stA-Za-z",e.no.trimmer=e.trimmerSupport.generateTrimmer(e.no.wordCharacters),e.Pipeline.registerFunction(e.no.trimmer,"trimmer-no"),e.no.stemmer=function(){var r=e.stemmerSupport.Among,n=e.stemmerSupport.SnowballProgram,i=new function(){function e(){var e,r=w.cursor+3;if(a=w.limit,0<=r||r<=w.limit){for(s=r;;){if(e=w.cursor,w.in_grouping(d,97,248)){w.cursor=e;break}if(e>=w.limit)return;w.cursor=e+1}for(;!w.out_grouping(d,97,248);){if(w.cursor>=w.limit)return;w.cursor++}a=w.cursor,a<s&&(a=s)}}function i(){var e,r,n;if(w.cursor>=a&&(r=w.limit_backward,w.limit_backward=a,w.ket=w.cursor,e=w.find_among_b(m,29),w.limit_backward=r,e))switch(w.bra=w.cursor,e){case 1:w.slice_del();break;case 2:n=w.limit-w.cursor,w.in_grouping_b(c,98,122)?w.slice_del():(w.cursor=w.limit-n,w.eq_s_b(1,"k")&&w.out_grouping_b(d,97,248)&&w.slice_del());break;case 3:w.slice_from("er")}}function t(){var e,r=w.limit-w.cursor;w.cursor>=a&&(e=w.limit_backward,w.limit_backward=a,w.ket=w.cursor,w.find_among_b(u,2)?(w.bra=w.cursor,w.limit_backward=e,w.cursor=w.limit-r,w.cursor>w.limit_backward&&(w.cursor--,w.bra=w.cursor,w.slice_del())):w.limit_backward=e)}function o(){var e,r;w.cursor>=a&&(r=w.limit_backward,w.limit_backward=a,w.ket=w.cursor,e=w.find_among_b(l,11),e?(w.bra=w.cursor,w.limit_backward=r,1==e&&w.slice_del()):w.limit_backward=r)}var s,a,m=[new r("a",-1,1),new r("e",-1,1),new r("ede",1,1),new r("ande",1,1),new r("ende",1,1),new r("ane",1,1),new r("ene",1,1),new r("hetene",6,1),new r("erte",1,3),new r("en",-1,1),new r("heten",9,1),new r("ar",-1,1),new r("er",-1,1),new r("heter",12,1),new r("s",-1,2),new r("as",14,1),new r("es",14,1),new r("edes",16,1),new r("endes",16,1),new r("enes",16,1),new r("hetenes",19,1),new r("ens",14,1),new r("hetens",21,1),new r("ers",14,1),new r("ets",14,1),new r("et",-1,1),new r("het",25,1),new r("ert",-1,3),new r("ast",-1,1)],u=[new r("dt",-1,-1),new r("vt",-1,-1)],l=[new r("leg",-1,1),new r("eleg",0,1),new r("ig",-1,1),new r("eig",2,1),new r("lig",2,1),new r("elig",4,1),new r("els",-1,1),new r("lov",-1,1),new r("elov",7,1),new r("slov",7,1),new r("hetslov",9,1)],d=[17,65,16,1,0,0,0,0,0,0,0,0,0,0,0,0,48,0,128],c=[119,125,149,1],w=new n;this.setCurrent=function(e){w.setCurrent(e)},this.getCurrent=function(){return w.getCurrent()},this.stem=function(){var r=w.cursor;return e(),w.limit_backward=r,w.cursor=w.limit,i(),w.cursor=w.limit,t(),w.cursor=w.limit,o(),!0}};return function(e){return"function"==typeof e.update?e.update(function(e){return i.setCurrent(e),i.stem(),i.getCurrent()}):(i.setCurrent(e),i.stem(),i.getCurrent())}}(),e.Pipeline.registerFunction(e.no.stemmer,"stemmer-no"),e.no.stopWordFilter=e.generateStopWordFilter("alle at av bare begge ble blei bli blir blitt både båe da de deg dei deim deira deires dem den denne der dere deres det dette di din disse ditt du dykk dykkar då eg ein eit eitt eller elles en enn er et ett etter for fordi fra før ha hadde han hans har hennar henne hennes her hjå ho hoe honom hoss hossen hun hva hvem hver hvilke hvilken hvis hvor hvordan hvorfor i ikke ikkje ikkje ingen ingi inkje inn inni ja jeg kan kom korleis korso kun kunne kva kvar kvarhelst kven kvi kvifor man mange me med medan meg meget mellom men mi min mine mitt mot mykje ned no noe noen noka noko nokon nokor nokre nå når og også om opp oss over på samme seg selv si si sia sidan siden sin sine sitt sjøl skal skulle slik so som som somme somt så sånn til um upp ut uten var vart varte ved vere verte vi vil ville vore vors vort vår være være vært å".split(" ")),e.Pipeline.registerFunction(e.no.stopWordFilter,"stopWordFilter-no")}});
|
||||
@@ -1,18 +0,0 @@
|
||||
/*!
|
||||
* Lunr languages, `Portuguese` language
|
||||
* https://github.com/MihaiValentin/lunr-languages
|
||||
*
|
||||
* Copyright 2014, Mihai Valentin
|
||||
* http://www.mozilla.org/MPL/
|
||||
*/
|
||||
/*!
|
||||
* based on
|
||||
* Snowball JavaScript Library v0.3
|
||||
* http://code.google.com/p/urim/
|
||||
* http://snowball.tartarus.org/
|
||||
*
|
||||
* Copyright 2010, Oleg Mazko
|
||||
* http://www.mozilla.org/MPL/
|
||||
*/
|
||||
|
||||
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.pt=function(){this.pipeline.reset(),this.pipeline.add(e.pt.trimmer,e.pt.stopWordFilter,e.pt.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.pt.stemmer))},e.pt.wordCharacters="A-Za-zªºÀ-ÖØ-öø-ʸˠ-ˤᴀ-ᴥᴬ-ᵜᵢ-ᵥᵫ-ᵷᵹ-ᶾḀ-ỿⁱⁿₐ-ₜKÅℲⅎⅠ-ↈⱠ-ⱿꜢ-ꞇꞋ-ꞭꞰ-ꞷꟷ-ꟿꬰ-ꭚꭜ-ꭤff-stA-Za-z",e.pt.trimmer=e.trimmerSupport.generateTrimmer(e.pt.wordCharacters),e.Pipeline.registerFunction(e.pt.trimmer,"trimmer-pt"),e.pt.stemmer=function(){var r=e.stemmerSupport.Among,s=e.stemmerSupport.SnowballProgram,n=new function(){function e(){for(var e;;){if(z.bra=z.cursor,e=z.find_among(k,3))switch(z.ket=z.cursor,e){case 1:z.slice_from("a~");continue;case 2:z.slice_from("o~");continue;case 3:if(z.cursor>=z.limit)break;z.cursor++;continue}break}}function n(){if(z.out_grouping(y,97,250)){for(;!z.in_grouping(y,97,250);){if(z.cursor>=z.limit)return!0;z.cursor++}return!1}return!0}function i(){if(z.in_grouping(y,97,250))for(;!z.out_grouping(y,97,250);){if(z.cursor>=z.limit)return!1;z.cursor++}return g=z.cursor,!0}function o(){var e,r,s=z.cursor;if(z.in_grouping(y,97,250))if(e=z.cursor,n()){if(z.cursor=e,i())return}else g=z.cursor;if(z.cursor=s,z.out_grouping(y,97,250)){if(r=z.cursor,n()){if(z.cursor=r,!z.in_grouping(y,97,250)||z.cursor>=z.limit)return;z.cursor++}g=z.cursor}}function t(){for(;!z.in_grouping(y,97,250);){if(z.cursor>=z.limit)return!1;z.cursor++}for(;!z.out_grouping(y,97,250);){if(z.cursor>=z.limit)return!1;z.cursor++}return!0}function a(){var e=z.cursor;g=z.limit,b=g,h=g,o(),z.cursor=e,t()&&(b=z.cursor,t()&&(h=z.cursor))}function u(){for(var e;;){if(z.bra=z.cursor,e=z.find_among(q,3))switch(z.ket=z.cursor,e){case 1:z.slice_from("ã");continue;case 2:z.slice_from("õ");continue;case 3:if(z.cursor>=z.limit)break;z.cursor++;continue}break}}function w(){return g<=z.cursor}function m(){return b<=z.cursor}function c(){return h<=z.cursor}function l(){var e;if(z.ket=z.cursor,!(e=z.find_among_b(F,45)))return!1;switch(z.bra=z.cursor,e){case 1:if(!c())return!1;z.slice_del();break;case 2:if(!c())return!1;z.slice_from("log");break;case 3:if(!c())return!1;z.slice_from("u");break;case 4:if(!c())return!1;z.slice_from("ente");break;case 5:if(!m())return!1;z.slice_del(),z.ket=z.cursor,e=z.find_among_b(j,4),e&&(z.bra=z.cursor,c()&&(z.slice_del(),1==e&&(z.ket=z.cursor,z.eq_s_b(2,"at")&&(z.bra=z.cursor,c()&&z.slice_del()))));break;case 6:if(!c())return!1;z.slice_del(),z.ket=z.cursor,e=z.find_among_b(C,3),e&&(z.bra=z.cursor,1==e&&c()&&z.slice_del());break;case 7:if(!c())return!1;z.slice_del(),z.ket=z.cursor,e=z.find_among_b(P,3),e&&(z.bra=z.cursor,1==e&&c()&&z.slice_del());break;case 8:if(!c())return!1;z.slice_del(),z.ket=z.cursor,z.eq_s_b(2,"at")&&(z.bra=z.cursor,c()&&z.slice_del());break;case 9:if(!w()||!z.eq_s_b(1,"e"))return!1;z.slice_from("ir")}return!0}function f(){var e,r;if(z.cursor>=g){if(r=z.limit_backward,z.limit_backward=g,z.ket=z.cursor,e=z.find_among_b(S,120))return z.bra=z.cursor,1==e&&z.slice_del(),z.limit_backward=r,!0;z.limit_backward=r}return!1}function d(){var e;z.ket=z.cursor,(e=z.find_among_b(W,7))&&(z.bra=z.cursor,1==e&&w()&&z.slice_del())}function v(e,r){if(z.eq_s_b(1,e)){z.bra=z.cursor;var s=z.limit-z.cursor;if(z.eq_s_b(1,r))return z.cursor=z.limit-s,w()&&z.slice_del(),!1}return!0}function p(){var e;if(z.ket=z.cursor,e=z.find_among_b(L,4))switch(z.bra=z.cursor,e){case 1:w()&&(z.slice_del(),z.ket=z.cursor,z.limit-z.cursor,v("u","g")&&v("i","c"));break;case 2:z.slice_from("c")}}function _(){if(!l()&&(z.cursor=z.limit,!f()))return z.cursor=z.limit,void d();z.cursor=z.limit,z.ket=z.cursor,z.eq_s_b(1,"i")&&(z.bra=z.cursor,z.eq_s_b(1,"c")&&(z.cursor=z.limit,w()&&z.slice_del()))}var h,b,g,k=[new r("",-1,3),new r("ã",0,1),new r("õ",0,2)],q=[new r("",-1,3),new r("a~",0,1),new r("o~",0,2)],j=[new r("ic",-1,-1),new r("ad",-1,-1),new r("os",-1,-1),new r("iv",-1,1)],C=[new r("ante",-1,1),new r("avel",-1,1),new r("ível",-1,1)],P=[new r("ic",-1,1),new r("abil",-1,1),new r("iv",-1,1)],F=[new r("ica",-1,1),new r("ância",-1,1),new r("ência",-1,4),new r("ira",-1,9),new r("adora",-1,1),new r("osa",-1,1),new r("ista",-1,1),new r("iva",-1,8),new r("eza",-1,1),new r("logía",-1,2),new r("idade",-1,7),new r("ante",-1,1),new r("mente",-1,6),new r("amente",12,5),new r("ável",-1,1),new r("ível",-1,1),new r("ución",-1,3),new r("ico",-1,1),new r("ismo",-1,1),new r("oso",-1,1),new r("amento",-1,1),new r("imento",-1,1),new r("ivo",-1,8),new r("aça~o",-1,1),new r("ador",-1,1),new r("icas",-1,1),new r("ências",-1,4),new r("iras",-1,9),Line truncated
|
||||
@@ -1,18 +0,0 @@
|
||||
/*!
|
||||
* Lunr languages, `Romanian` language
|
||||
* https://github.com/MihaiValentin/lunr-languages
|
||||
*
|
||||
* Copyright 2014, Mihai Valentin
|
||||
* http://www.mozilla.org/MPL/
|
||||
*/
|
||||
/*!
|
||||
* based on
|
||||
* Snowball JavaScript Library v0.3
|
||||
* http://code.google.com/p/urim/
|
||||
* http://snowball.tartarus.org/
|
||||
*
|
||||
* Copyright 2010, Oleg Mazko
|
||||
* http://www.mozilla.org/MPL/
|
||||
*/
|
||||
|
||||
!function(e,i){"function"==typeof define&&define.amd?define(i):"object"==typeof exports?module.exports=i():i()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.ro=function(){this.pipeline.reset(),this.pipeline.add(e.ro.trimmer,e.ro.stopWordFilter,e.ro.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.ro.stemmer))},e.ro.wordCharacters="A-Za-zªºÀ-ÖØ-öø-ʸˠ-ˤᴀ-ᴥᴬ-ᵜᵢ-ᵥᵫ-ᵷᵹ-ᶾḀ-ỿⁱⁿₐ-ₜKÅℲⅎⅠ-ↈⱠ-ⱿꜢ-ꞇꞋ-ꞭꞰ-ꞷꟷ-ꟿꬰ-ꭚꭜ-ꭤff-stA-Za-z",e.ro.trimmer=e.trimmerSupport.generateTrimmer(e.ro.wordCharacters),e.Pipeline.registerFunction(e.ro.trimmer,"trimmer-ro"),e.ro.stemmer=function(){var i=e.stemmerSupport.Among,r=e.stemmerSupport.SnowballProgram,n=new function(){function e(e,i){L.eq_s(1,e)&&(L.ket=L.cursor,L.in_grouping(W,97,259)&&L.slice_from(i))}function n(){for(var i,r;;){if(i=L.cursor,L.in_grouping(W,97,259)&&(r=L.cursor,L.bra=r,e("u","U"),L.cursor=r,e("i","I")),L.cursor=i,L.cursor>=L.limit)break;L.cursor++}}function t(){if(L.out_grouping(W,97,259)){for(;!L.in_grouping(W,97,259);){if(L.cursor>=L.limit)return!0;L.cursor++}return!1}return!0}function a(){if(L.in_grouping(W,97,259))for(;!L.out_grouping(W,97,259);){if(L.cursor>=L.limit)return!0;L.cursor++}return!1}function o(){var e,i,r=L.cursor;if(L.in_grouping(W,97,259)){if(e=L.cursor,!t())return void(h=L.cursor);if(L.cursor=e,!a())return void(h=L.cursor)}L.cursor=r,L.out_grouping(W,97,259)&&(i=L.cursor,t()&&(L.cursor=i,L.in_grouping(W,97,259)&&L.cursor<L.limit&&L.cursor++),h=L.cursor)}function u(){for(;!L.in_grouping(W,97,259);){if(L.cursor>=L.limit)return!1;L.cursor++}for(;!L.out_grouping(W,97,259);){if(L.cursor>=L.limit)return!1;L.cursor++}return!0}function c(){var e=L.cursor;h=L.limit,k=h,g=h,o(),L.cursor=e,u()&&(k=L.cursor,u()&&(g=L.cursor))}function s(){for(var e;;){if(L.bra=L.cursor,e=L.find_among(z,3))switch(L.ket=L.cursor,e){case 1:L.slice_from("i");continue;case 2:L.slice_from("u");continue;case 3:if(L.cursor>=L.limit)break;L.cursor++;continue}break}}function w(){return h<=L.cursor}function m(){return k<=L.cursor}function l(){return g<=L.cursor}function f(){var e,i;if(L.ket=L.cursor,(e=L.find_among_b(C,16))&&(L.bra=L.cursor,m()))switch(e){case 1:L.slice_del();break;case 2:L.slice_from("a");break;case 3:L.slice_from("e");break;case 4:L.slice_from("i");break;case 5:i=L.limit-L.cursor,L.eq_s_b(2,"ab")||(L.cursor=L.limit-i,L.slice_from("i"));break;case 6:L.slice_from("at");break;case 7:L.slice_from("aţi")}}function p(){var e,i=L.limit-L.cursor;if(L.ket=L.cursor,(e=L.find_among_b(P,46))&&(L.bra=L.cursor,m())){switch(e){case 1:L.slice_from("abil");break;case 2:L.slice_from("ibil");break;case 3:L.slice_from("iv");break;case 4:L.slice_from("ic");break;case 5:L.slice_from("at");break;case 6:L.slice_from("it")}return _=!0,L.cursor=L.limit-i,!0}return!1}function d(){var e,i;for(_=!1;;)if(i=L.limit-L.cursor,!p()){L.cursor=L.limit-i;break}if(L.ket=L.cursor,(e=L.find_among_b(F,62))&&(L.bra=L.cursor,l())){switch(e){case 1:L.slice_del();break;case 2:L.eq_s_b(1,"ţ")&&(L.bra=L.cursor,L.slice_from("t"));break;case 3:L.slice_from("ist")}_=!0}}function b(){var e,i,r;if(L.cursor>=h){if(i=L.limit_backward,L.limit_backward=h,L.ket=L.cursor,e=L.find_among_b(q,94))switch(L.bra=L.cursor,e){case 1:if(r=L.limit-L.cursor,!L.out_grouping_b(W,97,259)&&(L.cursor=L.limit-r,!L.eq_s_b(1,"u")))break;case 2:L.slice_del()}L.limit_backward=i}}function v(){var e;L.ket=L.cursor,(e=L.find_among_b(S,5))&&(L.bra=L.cursor,w()&&1==e&&L.slice_del())}var _,g,k,h,z=[new i("",-1,3),new i("I",0,1),new i("U",0,2)],C=[new i("ea",-1,3),new i("aţia",-1,7),new i("aua",-1,2),new i("iua",-1,4),new i("aţie",-1,7),new i("ele",-1,3),new i("ile",-1,5),new i("iile",6,4),new i("iei",-1,4),new i("atei",-1,6),new i("ii",-1,4),new i("ului",-1,1),new i("ul",-1,1),new i("elor",-1,3),new i("ilor",-1,4),new i("iilor",14,4)],P=[new i("icala",-1,4),new i("iciva",-1,4),new i("ativa",-1,5),new i("itiva",-1,6),new i("icale",-1,4),new i("aţiune",-1,5),new i("iţiune",-1,6),new i("atoare",-1,5),new i("itoare",-1,6),new i("ătoare",-1,5),new i("icitate",-1,4),new i("abilitate",-1,1),new i("ibilitate",-1,2),new i("ivitate",-1,3),new i("icive",-1,4),new i("ative",-1,5),new i("itive",-1,6),new i("icali",-1,4),new i("atori",-1,5),new i("icatori",18,4),new i("itori",-1,6),new i("ători",-1,5),new i("icitati",-1,4),new i("abilitati",-1,1),new i("ivitati",-1,3),new i("icivi",-1,4),new i("ativi",-1,5),new i("itivi",-1,6),new i("icităi",-1,4),new i("abilităi",-1,1),new i("ivităi",-1,3),new i("icităţi",-1,4),new i("abilităţi",-1,1),new i("ivităţi",-1,3),new i("ical",-1,4),new i("ator",-1,5),new i("icator",35,4),new i("itor",-1,6),new i("ător",-1,5),new i("iciv",-1,4),neLine truncated
|
||||
@@ -1,18 +0,0 @@
|
||||
/*!
|
||||
* Lunr languages, `Russian` language
|
||||
* https://github.com/MihaiValentin/lunr-languages
|
||||
*
|
||||
* Copyright 2014, Mihai Valentin
|
||||
* http://www.mozilla.org/MPL/
|
||||
*/
|
||||
/*!
|
||||
* based on
|
||||
* Snowball JavaScript Library v0.3
|
||||
* http://code.google.com/p/urim/
|
||||
* http://snowball.tartarus.org/
|
||||
*
|
||||
* Copyright 2010, Oleg Mazko
|
||||
* http://www.mozilla.org/MPL/
|
||||
*/
|
||||
|
||||
!function(e,n){"function"==typeof define&&define.amd?define(n):"object"==typeof exports?module.exports=n():n()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.ru=function(){this.pipeline.reset(),this.pipeline.add(e.ru.trimmer,e.ru.stopWordFilter,e.ru.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.ru.stemmer))},e.ru.wordCharacters="Ѐ-҄҇-ԯᴫᵸⷠ-ⷿꙀ-ꚟ︮︯",e.ru.trimmer=e.trimmerSupport.generateTrimmer(e.ru.wordCharacters),e.Pipeline.registerFunction(e.ru.trimmer,"trimmer-ru"),e.ru.stemmer=function(){var n=e.stemmerSupport.Among,r=e.stemmerSupport.SnowballProgram,t=new function(){function e(){for(;!W.in_grouping(S,1072,1103);){if(W.cursor>=W.limit)return!1;W.cursor++}return!0}function t(){for(;!W.out_grouping(S,1072,1103);){if(W.cursor>=W.limit)return!1;W.cursor++}return!0}function w(){b=W.limit,_=b,e()&&(b=W.cursor,t()&&e()&&t()&&(_=W.cursor))}function i(){return _<=W.cursor}function u(e,n){var r,t;if(W.ket=W.cursor,r=W.find_among_b(e,n)){switch(W.bra=W.cursor,r){case 1:if(t=W.limit-W.cursor,!W.eq_s_b(1,"а")&&(W.cursor=W.limit-t,!W.eq_s_b(1,"я")))return!1;case 2:W.slice_del()}return!0}return!1}function o(){return u(h,9)}function s(e,n){var r;return W.ket=W.cursor,!!(r=W.find_among_b(e,n))&&(W.bra=W.cursor,1==r&&W.slice_del(),!0)}function c(){return s(g,26)}function m(){return!!c()&&(u(C,8),!0)}function f(){return s(k,2)}function l(){return u(P,46)}function a(){s(v,36)}function p(){var e;W.ket=W.cursor,(e=W.find_among_b(F,2))&&(W.bra=W.cursor,i()&&1==e&&W.slice_del())}function d(){var e;if(W.ket=W.cursor,e=W.find_among_b(q,4))switch(W.bra=W.cursor,e){case 1:if(W.slice_del(),W.ket=W.cursor,!W.eq_s_b(1,"н"))break;W.bra=W.cursor;case 2:if(!W.eq_s_b(1,"н"))break;case 3:W.slice_del()}}var _,b,h=[new n("в",-1,1),new n("ив",0,2),new n("ыв",0,2),new n("вши",-1,1),new n("ивши",3,2),new n("ывши",3,2),new n("вшись",-1,1),new n("ившись",6,2),new n("ывшись",6,2)],g=[new n("ее",-1,1),new n("ие",-1,1),new n("ое",-1,1),new n("ые",-1,1),new n("ими",-1,1),new n("ыми",-1,1),new n("ей",-1,1),new n("ий",-1,1),new n("ой",-1,1),new n("ый",-1,1),new n("ем",-1,1),new n("им",-1,1),new n("ом",-1,1),new n("ым",-1,1),new n("его",-1,1),new n("ого",-1,1),new n("ему",-1,1),new n("ому",-1,1),new n("их",-1,1),new n("ых",-1,1),new n("ею",-1,1),new n("ою",-1,1),new n("ую",-1,1),new n("юю",-1,1),new n("ая",-1,1),new n("яя",-1,1)],C=[new n("ем",-1,1),new n("нн",-1,1),new n("вш",-1,1),new n("ивш",2,2),new n("ывш",2,2),new n("щ",-1,1),new n("ющ",5,1),new n("ующ",6,2)],k=[new n("сь",-1,1),new n("ся",-1,1)],P=[new n("ла",-1,1),new n("ила",0,2),new n("ыла",0,2),new n("на",-1,1),new n("ена",3,2),new n("ете",-1,1),new n("ите",-1,2),new n("йте",-1,1),new n("ейте",7,2),new n("уйте",7,2),new n("ли",-1,1),new n("или",10,2),new n("ыли",10,2),new n("й",-1,1),new n("ей",13,2),new n("уй",13,2),new n("л",-1,1),new n("ил",16,2),new n("ыл",16,2),new n("ем",-1,1),new n("им",-1,2),new n("ым",-1,2),new n("н",-1,1),new n("ен",22,2),new n("ло",-1,1),new n("ило",24,2),new n("ыло",24,2),new n("но",-1,1),new n("ено",27,2),new n("нно",27,1),new n("ет",-1,1),new n("ует",30,2),new n("ит",-1,2),new n("ыт",-1,2),new n("ют",-1,1),new n("уют",34,2),new n("ят",-1,2),new n("ны",-1,1),new n("ены",37,2),new n("ть",-1,1),new n("ить",39,2),new n("ыть",39,2),new n("ешь",-1,1),new n("ишь",-1,2),new n("ю",-1,2),new n("ую",44,2)],v=[new n("а",-1,1),new n("ев",-1,1),new n("ов",-1,1),new n("е",-1,1),new n("ие",3,1),new n("ье",3,1),new n("и",-1,1),new n("еи",6,1),new n("ии",6,1),new n("ами",6,1),new n("ями",6,1),new n("иями",10,1),new n("й",-1,1),new n("ей",12,1),new n("ией",13,1),new n("ий",12,1),new n("ой",12,1),new n("ам",-1,1),new n("ем",-1,1),new n("ием",18,1),new n("ом",-1,1),new n("ям",-1,1),new n("иям",21,1),new n("о",-1,1),new n("у",-1,1),new n("ах",-1,1),new n("ях",-1,1),new n("иях",26,1),new n("ы",-1,1),new n("ь",-1,1),new n("ю",-1,1),new n("ию",30,1),new n("ью",30,1),new n("я",-1,1),new n("ия",33,1),new n("ья",33,1)],F=[new n("ост",-1,1),new n("ость",-1,1)],q=[new n("ейше",-1,1),new n("н",-1,2),new n("ейш",-1,1),new n("ь",-1,3)],S=[33,65,8,232],W=new r;this.setCurrent=function(e){W.setCurrent(e)},this.getCurrent=function(){return W.getCurrent()},this.stem=function(){return w(),W.cursor=W.limit,!(W.cursor<b)&&(W.limit_backward=b,o()||(W.cursor=W.limit,f()||(W.cursor=W.limit),m()||(W.cursor=W.limit,l()||(W.cursor=W.limit,a()))),W.cursor=W.limit,W.ket=W.cursor,W.eq_s_b(1,"и"Line truncated
|
||||
@@ -1 +0,0 @@
|
||||
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.sa=function(){this.pipeline.reset(),this.pipeline.add(e.sa.trimmer,e.sa.stopWordFilter,e.sa.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.sa.stemmer))},e.sa.wordCharacters="ऀ-ःऄ-एऐ-टठ-यर-िी-ॏॐ-य़ॠ-९॰-ॿ꣠-꣱ꣲ-ꣷ꣸-ꣻ꣼-ꣽꣾ-ꣿᆰ0-ᆰ9",e.sa.trimmer=e.trimmerSupport.generateTrimmer(e.sa.wordCharacters),e.Pipeline.registerFunction(e.sa.trimmer,"trimmer-sa"),e.sa.stopWordFilter=e.generateStopWordFilter('तथा अयम् एकम् इत्यस्मिन् तथा तत् वा अयम् इत्यस्य ते आहूत उपरि तेषाम् किन्तु तेषाम् तदा इत्यनेन अधिकः इत्यस्य तत् केचन बहवः द्वि तथा महत्वपूर्णः अयम् अस्य विषये अयं अस्ति तत् प्रथमः विषये इत्युपरि इत्युपरि इतर अधिकतमः अधिकः अपि सामान्यतया ठ इतरेतर नूतनम् द न्यूनम् कश्चित् वा विशालः द सः अस्ति तदनुसारम् तत्र अस्ति केवलम् अपि अत्र सर्वे विविधाः तत् बहवः यतः इदानीम् द दक्षिण इत्यस्मै तस्य उपरि नथ अतीव कार्यम् सर्वे एकैकम् इत्यादि। एते सन्ति उत इत्थम् मध्ये एतदर्थं . स कस्य प्रथमः श्री. करोति अस्मिन् प्रकारः निर्मिता कालः तत्र कर्तुं समान अधुना ते सन्ति स एकः अस्ति सः अर्थात् तेषां कृते . स्थितम् विशेषः अग्रिम तेषाम् समान स्रोतः ख म समान इदानीमपि अधिकतया करोतु ते समान इत्यस्य वीथी सह यस्मिन् कृतवान् धृतः तदा पुनः पूर्वं सः आगतः किम् कुल इतर पुरा मात्रा स विषये उ अतएव अपि नगरस्य उपरि यतः प्रतिशतं कतरः कालः साधनानि भूत तथापि जात सम्बन्धि अन्यत् ग अतः अस्माकं स्वकीयाः अस्माकं इदानीं अन्तः इत्यादयः भवन्तः इत्यादयः एते एताः तस्य अस्य इदम् एते तेषां तेषां तेषां तान् तेषां तेषां तेषां समानः सः एकः च तादृशाः बहवः अन्ये च वदन्ति यत् कियत् कस्मै कस्मै यस्मै यस्मै यस्मै यस्मै न अतिनीचः किन्तु प्रथमं सम्पूर्णतया ततः चिरकालानन्तरं पुस्तकं सम्पूर्णतया अन्तः किन्तु अत्र वा इह इव श्रद्धाय अवशिष्यते परन्तु अन्ये वर्गाः सन्ति ते सन्ति शक्नुवन्ति सर्वे मिलित्वा सर्वे एकत्र"'.split(" ")),e.sa.stemmer=function(){return function(e){return"function"==typeof e.update?e.update(function(e){return e}):e}}();var r=e.wordcut;r.init(),e.sa.tokenizer=function(t){if(!arguments.length||null==t||void 0==t)return[];if(Array.isArray(t))return t.map(function(r){return isLunr2?new e.Token(r.toLowerCase()):r.toLowerCase()});var i=t.toString().toLowerCase().replace(/^\s+/,"");return r.cut(i).split("|")},e.Pipeline.registerFunction(e.sa.stemmer,"stemmer-sa"),e.Pipeline.registerFunction(e.sa.stopWordFilter,"stopWordFilter-sa")}});
|
||||
@@ -1 +0,0 @@
|
||||
!function(r,t){"function"==typeof define&&define.amd?define(t):"object"==typeof exports?module.exports=t():t()(r.lunr)}(this,function(){return function(r){r.stemmerSupport={Among:function(r,t,i,s){if(this.toCharArray=function(r){for(var t=r.length,i=new Array(t),s=0;s<t;s++)i[s]=r.charCodeAt(s);return i},!r&&""!=r||!t&&0!=t||!i)throw"Bad Among initialisation: s:"+r+", substring_i: "+t+", result: "+i;this.s_size=r.length,this.s=this.toCharArray(r),this.substring_i=t,this.result=i,this.method=s},SnowballProgram:function(){var r;return{bra:0,ket:0,limit:0,cursor:0,limit_backward:0,setCurrent:function(t){r=t,this.cursor=0,this.limit=t.length,this.limit_backward=0,this.bra=this.cursor,this.ket=this.limit},getCurrent:function(){var t=r;return r=null,t},in_grouping:function(t,i,s){if(this.cursor<this.limit){var e=r.charCodeAt(this.cursor);if(e<=s&&e>=i&&(e-=i,t[e>>3]&1<<(7&e)))return this.cursor++,!0}return!1},in_grouping_b:function(t,i,s){if(this.cursor>this.limit_backward){var e=r.charCodeAt(this.cursor-1);if(e<=s&&e>=i&&(e-=i,t[e>>3]&1<<(7&e)))return this.cursor--,!0}return!1},out_grouping:function(t,i,s){if(this.cursor<this.limit){var e=r.charCodeAt(this.cursor);if(e>s||e<i)return this.cursor++,!0;if(e-=i,!(t[e>>3]&1<<(7&e)))return this.cursor++,!0}return!1},out_grouping_b:function(t,i,s){if(this.cursor>this.limit_backward){var e=r.charCodeAt(this.cursor-1);if(e>s||e<i)return this.cursor--,!0;if(e-=i,!(t[e>>3]&1<<(7&e)))return this.cursor--,!0}return!1},eq_s:function(t,i){if(this.limit-this.cursor<t)return!1;for(var s=0;s<t;s++)if(r.charCodeAt(this.cursor+s)!=i.charCodeAt(s))return!1;return this.cursor+=t,!0},eq_s_b:function(t,i){if(this.cursor-this.limit_backward<t)return!1;for(var s=0;s<t;s++)if(r.charCodeAt(this.cursor-t+s)!=i.charCodeAt(s))return!1;return this.cursor-=t,!0},find_among:function(t,i){for(var s=0,e=i,n=this.cursor,u=this.limit,o=0,h=0,c=!1;;){for(var a=s+(e-s>>1),f=0,l=o<h?o:h,_=t[a],m=l;m<_.s_size;m++){if(n+l==u){f=-1;break}if(f=r.charCodeAt(n+l)-_.s[m])break;l++}if(f<0?(e=a,h=l):(s=a,o=l),e-s<=1){if(s>0||e==s||c)break;c=!0}}for(;;){var _=t[s];if(o>=_.s_size){if(this.cursor=n+_.s_size,!_.method)return _.result;var b=_.method();if(this.cursor=n+_.s_size,b)return _.result}if((s=_.substring_i)<0)return 0}},find_among_b:function(t,i){for(var s=0,e=i,n=this.cursor,u=this.limit_backward,o=0,h=0,c=!1;;){for(var a=s+(e-s>>1),f=0,l=o<h?o:h,_=t[a],m=_.s_size-1-l;m>=0;m--){if(n-l==u){f=-1;break}if(f=r.charCodeAt(n-1-l)-_.s[m])break;l++}if(f<0?(e=a,h=l):(s=a,o=l),e-s<=1){if(s>0||e==s||c)break;c=!0}}for(;;){var _=t[s];if(o>=_.s_size){if(this.cursor=n-_.s_size,!_.method)return _.result;var b=_.method();if(this.cursor=n-_.s_size,b)return _.result}if((s=_.substring_i)<0)return 0}},replace_s:function(t,i,s){var e=s.length-(i-t),n=r.substring(0,t),u=r.substring(i);return r=n+s+u,this.limit+=e,this.cursor>=i?this.cursor+=e:this.cursor>t&&(this.cursor=t),e},slice_check:function(){if(this.bra<0||this.bra>this.ket||this.ket>this.limit||this.limit>r.length)throw"faulty slice operation"},slice_from:function(r){this.slice_check(),this.replace_s(this.bra,this.ket,r)},slice_del:function(){this.slice_from("")},insert:function(r,t,i){var s=this.replace_s(r,t,i);r<=this.bra&&(this.bra+=s),r<=this.ket&&(this.ket+=s)},slice_to:function(){return this.slice_check(),r.substring(this.bra,this.ket)},eq_v_b:function(r){return this.eq_s_b(r.length,r)}}}},r.trimmerSupport={generateTrimmer:function(r){var t=new RegExp("^[^"+r+"]+"),i=new RegExp("[^"+r+"]+$");return function(r){return"function"==typeof r.update?r.update(function(r){return r.replace(t,"").replace(i,"")}):r.replace(t,"").replace(i,"")}}}}});
|
||||
@@ -1,18 +0,0 @@
|
||||
/*!
|
||||
* Lunr languages, `Swedish` language
|
||||
* https://github.com/MihaiValentin/lunr-languages
|
||||
*
|
||||
* Copyright 2014, Mihai Valentin
|
||||
* http://www.mozilla.org/MPL/
|
||||
*/
|
||||
/*!
|
||||
* based on
|
||||
* Snowball JavaScript Library v0.3
|
||||
* http://code.google.com/p/urim/
|
||||
* http://snowball.tartarus.org/
|
||||
*
|
||||
* Copyright 2010, Oleg Mazko
|
||||
* http://www.mozilla.org/MPL/
|
||||
*/
|
||||
|
||||
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.sv=function(){this.pipeline.reset(),this.pipeline.add(e.sv.trimmer,e.sv.stopWordFilter,e.sv.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.sv.stemmer))},e.sv.wordCharacters="A-Za-zªºÀ-ÖØ-öø-ʸˠ-ˤᴀ-ᴥᴬ-ᵜᵢ-ᵥᵫ-ᵷᵹ-ᶾḀ-ỿⁱⁿₐ-ₜKÅℲⅎⅠ-ↈⱠ-ⱿꜢ-ꞇꞋ-ꞭꞰ-ꞷꟷ-ꟿꬰ-ꭚꭜ-ꭤff-stA-Za-z",e.sv.trimmer=e.trimmerSupport.generateTrimmer(e.sv.wordCharacters),e.Pipeline.registerFunction(e.sv.trimmer,"trimmer-sv"),e.sv.stemmer=function(){var r=e.stemmerSupport.Among,n=e.stemmerSupport.SnowballProgram,t=new function(){function e(){var e,r=w.cursor+3;if(o=w.limit,0<=r||r<=w.limit){for(a=r;;){if(e=w.cursor,w.in_grouping(l,97,246)){w.cursor=e;break}if(w.cursor=e,w.cursor>=w.limit)return;w.cursor++}for(;!w.out_grouping(l,97,246);){if(w.cursor>=w.limit)return;w.cursor++}o=w.cursor,o<a&&(o=a)}}function t(){var e,r=w.limit_backward;if(w.cursor>=o&&(w.limit_backward=o,w.cursor=w.limit,w.ket=w.cursor,e=w.find_among_b(u,37),w.limit_backward=r,e))switch(w.bra=w.cursor,e){case 1:w.slice_del();break;case 2:w.in_grouping_b(d,98,121)&&w.slice_del()}}function i(){var e=w.limit_backward;w.cursor>=o&&(w.limit_backward=o,w.cursor=w.limit,w.find_among_b(c,7)&&(w.cursor=w.limit,w.ket=w.cursor,w.cursor>w.limit_backward&&(w.bra=--w.cursor,w.slice_del())),w.limit_backward=e)}function s(){var e,r;if(w.cursor>=o){if(r=w.limit_backward,w.limit_backward=o,w.cursor=w.limit,w.ket=w.cursor,e=w.find_among_b(m,5))switch(w.bra=w.cursor,e){case 1:w.slice_del();break;case 2:w.slice_from("lös");break;case 3:w.slice_from("full")}w.limit_backward=r}}var a,o,u=[new r("a",-1,1),new r("arna",0,1),new r("erna",0,1),new r("heterna",2,1),new r("orna",0,1),new r("ad",-1,1),new r("e",-1,1),new r("ade",6,1),new r("ande",6,1),new r("arne",6,1),new r("are",6,1),new r("aste",6,1),new r("en",-1,1),new r("anden",12,1),new r("aren",12,1),new r("heten",12,1),new r("ern",-1,1),new r("ar",-1,1),new r("er",-1,1),new r("heter",18,1),new r("or",-1,1),new r("s",-1,2),new r("as",21,1),new r("arnas",22,1),new r("ernas",22,1),new r("ornas",22,1),new r("es",21,1),new r("ades",26,1),new r("andes",26,1),new r("ens",21,1),new r("arens",29,1),new r("hetens",29,1),new r("erns",21,1),new r("at",-1,1),new r("andet",-1,1),new r("het",-1,1),new r("ast",-1,1)],c=[new r("dd",-1,-1),new r("gd",-1,-1),new r("nn",-1,-1),new r("dt",-1,-1),new r("gt",-1,-1),new r("kt",-1,-1),new r("tt",-1,-1)],m=[new r("ig",-1,1),new r("lig",0,1),new r("els",-1,1),new r("fullt",-1,3),new r("löst",-1,2)],l=[17,65,16,1,0,0,0,0,0,0,0,0,0,0,0,0,24,0,32],d=[119,127,149],w=new n;this.setCurrent=function(e){w.setCurrent(e)},this.getCurrent=function(){return w.getCurrent()},this.stem=function(){var r=w.cursor;return e(),w.limit_backward=r,w.cursor=w.limit,t(),w.cursor=w.limit,i(),w.cursor=w.limit,s(),!0}};return function(e){return"function"==typeof e.update?e.update(function(e){return t.setCurrent(e),t.stem(),t.getCurrent()}):(t.setCurrent(e),t.stem(),t.getCurrent())}}(),e.Pipeline.registerFunction(e.sv.stemmer,"stemmer-sv"),e.sv.stopWordFilter=e.generateStopWordFilter("alla allt att av blev bli blir blivit de dem den denna deras dess dessa det detta dig din dina ditt du där då efter ej eller en er era ert ett från för ha hade han hans har henne hennes hon honom hur här i icke ingen inom inte jag ju kan kunde man med mellan men mig min mina mitt mot mycket ni nu när någon något några och om oss på samma sedan sig sin sina sitta själv skulle som så sådan sådana sådant till under upp ut utan vad var vara varför varit varje vars vart vem vi vid vilka vilkas vilken vilket vår våra vårt än är åt över".split(" ")),e.Pipeline.registerFunction(e.sv.stopWordFilter,"stopWordFilter-sv")}});
|
||||
@@ -1 +0,0 @@
|
||||
!function(e,t){"function"==typeof define&&define.amd?define(t):"object"==typeof exports?module.exports=t():t()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.ta=function(){this.pipeline.reset(),this.pipeline.add(e.ta.trimmer,e.ta.stopWordFilter,e.ta.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.ta.stemmer))},e.ta.wordCharacters="-உஊ-ஏஐ-ஙச-ட-னப-யர-ஹ-ிீ-ொ-ௐ---௩௪-௯௰-௹௺-a-zA-Za-zA-Z0-90-9",e.ta.trimmer=e.trimmerSupport.generateTrimmer(e.ta.wordCharacters),e.Pipeline.registerFunction(e.ta.trimmer,"trimmer-ta"),e.ta.stopWordFilter=e.generateStopWordFilter("அங்கு அங்கே அது அதை அந்த அவர் அவர்கள் அவள் அவன் அவை ஆக ஆகவே ஆகையால் ஆதலால் ஆதலினால் ஆனாலும் ஆனால் இங்கு இங்கே இது இதை இந்த இப்படி இவர் இவர்கள் இவள் இவன் இவை இவ்வளவு உனக்கு உனது உன் உன்னால் எங்கு எங்கே எது எதை எந்த எப்படி எவர் எவர்கள் எவள் எவன் எவை எவ்வளவு எனக்கு எனது எனவே என் என்ன என்னால் ஏது ஏன் தனது தன்னால் தானே தான் நாங்கள் நாம் நான் நீ நீங்கள்".split(" ")),e.ta.stemmer=function(){return function(e){return"function"==typeof e.update?e.update(function(e){return e}):e}}();var t=e.wordcut;t.init(),e.ta.tokenizer=function(r){if(!arguments.length||null==r||void 0==r)return[];if(Array.isArray(r))return r.map(function(t){return isLunr2?new e.Token(t.toLowerCase()):t.toLowerCase()});var i=r.toString().toLowerCase().replace(/^\s+/,"");return t.cut(i).split("|")},e.Pipeline.registerFunction(e.ta.stemmer,"stemmer-ta"),e.Pipeline.registerFunction(e.ta.stopWordFilter,"stopWordFilter-ta")}});
|
||||
@@ -1 +0,0 @@
|
||||
!function(e,t){"function"==typeof define&&define.amd?define(t):"object"==typeof exports?module.exports=t():t()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.te=function(){this.pipeline.reset(),this.pipeline.add(e.te.trimmer,e.te.stopWordFilter,e.te.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.te.stemmer))},e.te.wordCharacters="ఀ-ఄఅ-ఔక-హా-ౌౕ-ౖౘ-ౚౠ-ౡౢ-ౣ౦-౯౸-౿఼ఽ్ౝ౷",e.te.trimmer=e.trimmerSupport.generateTrimmer(e.te.wordCharacters),e.Pipeline.registerFunction(e.te.trimmer,"trimmer-te"),e.te.stopWordFilter=e.generateStopWordFilter("అందరూ అందుబాటులో అడగండి అడగడం అడ్డంగా అనుగుణంగా అనుమతించు అనుమతిస్తుంది అయితే ఇప్పటికే ఉన్నారు ఎక్కడైనా ఎప్పుడు ఎవరైనా ఎవరో ఏ ఏదైనా ఏమైనప్పటికి ఒక ఒకరు కనిపిస్తాయి కాదు కూడా గా గురించి చుట్టూ చేయగలిగింది తగిన తర్వాత దాదాపు దూరంగా నిజంగా పై ప్రకారం ప్రక్కన మధ్య మరియు మరొక మళ్ళీ మాత్రమే మెచ్చుకో వద్ద వెంట వేరుగా వ్యతిరేకంగా సంబంధం".split(" ")),e.te.stemmer=function(){return function(e){return"function"==typeof e.update?e.update(function(e){return e}):e}}();var t=e.wordcut;t.init(),e.te.tokenizer=function(r){if(!arguments.length||null==r||void 0==r)return[];if(Array.isArray(r))return r.map(function(t){return isLunr2?new e.Token(t.toLowerCase()):t.toLowerCase()});var i=r.toString().toLowerCase().replace(/^\s+/,"");return t.cut(i).split("|")},e.Pipeline.registerFunction(e.te.stemmer,"stemmer-te"),e.Pipeline.registerFunction(e.te.stopWordFilter,"stopWordFilter-te")}});
|
||||
@@ -1 +0,0 @@
|
||||
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");var r="2"==e.version[0];e.th=function(){this.pipeline.reset(),this.pipeline.add(e.th.trimmer),r?this.tokenizer=e.th.tokenizer:(e.tokenizer&&(e.tokenizer=e.th.tokenizer),this.tokenizerFn&&(this.tokenizerFn=e.th.tokenizer))},e.th.wordCharacters="[-]",e.th.trimmer=e.trimmerSupport.generateTrimmer(e.th.wordCharacters),e.Pipeline.registerFunction(e.th.trimmer,"trimmer-th");var t=e.wordcut;t.init(),e.th.tokenizer=function(i){if(!arguments.length||null==i||void 0==i)return[];if(Array.isArray(i))return i.map(function(t){return r?new e.Token(t):t});var n=i.toString().replace(/^\s+/,"");return t.cut(n).split("|")}}});
|
||||
@@ -1,18 +0,0 @@
|
||||
/*!
|
||||
* Lunr languages, `Turkish` language
|
||||
* https://github.com/MihaiValentin/lunr-languages
|
||||
*
|
||||
* Copyright 2014, Mihai Valentin
|
||||
* http://www.mozilla.org/MPL/
|
||||
*/
|
||||
/*!
|
||||
* based on
|
||||
* Snowball JavaScript Library v0.3
|
||||
* http://code.google.com/p/urim/
|
||||
* http://snowball.tartarus.org/
|
||||
*
|
||||
* Copyright 2010, Oleg Mazko
|
||||
* http://www.mozilla.org/MPL/
|
||||
*/
|
||||
|
||||
!function(r,i){"function"==typeof define&&define.amd?define(i):"object"==typeof exports?module.exports=i():i()(r.lunr)}(this,function(){return function(r){if(void 0===r)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===r.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");r.tr=function(){this.pipeline.reset(),this.pipeline.add(r.tr.trimmer,r.tr.stopWordFilter,r.tr.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(r.tr.stemmer))},r.tr.wordCharacters="A-Za-zªºÀ-ÖØ-öø-ʸˠ-ˤᴀ-ᴥᴬ-ᵜᵢ-ᵥᵫ-ᵷᵹ-ᶾḀ-ỿⁱⁿₐ-ₜKÅℲⅎⅠ-ↈⱠ-ⱿꜢ-ꞇꞋ-ꞭꞰ-ꞷꟷ-ꟿꬰ-ꭚꭜ-ꭤff-stA-Za-z",r.tr.trimmer=r.trimmerSupport.generateTrimmer(r.tr.wordCharacters),r.Pipeline.registerFunction(r.tr.trimmer,"trimmer-tr"),r.tr.stemmer=function(){var i=r.stemmerSupport.Among,e=r.stemmerSupport.SnowballProgram,n=new function(){function r(r,i,e){for(;;){var n=Dr.limit-Dr.cursor;if(Dr.in_grouping_b(r,i,e)){Dr.cursor=Dr.limit-n;break}if(Dr.cursor=Dr.limit-n,Dr.cursor<=Dr.limit_backward)return!1;Dr.cursor--}return!0}function n(){var i,e;i=Dr.limit-Dr.cursor,r(Wr,97,305);for(var n=0;n<Br.length;n++){e=Dr.limit-Dr.cursor;var t=Br[n];if(Dr.eq_s_b(1,t[0])&&r(t[1],t[2],t[3]))return Dr.cursor=Dr.limit-i,!0;Dr.cursor=Dr.limit-e}return Dr.cursor=Dr.limit-e,!(!Dr.eq_s_b(1,"ü")||!r(Zr,246,252))&&(Dr.cursor=Dr.limit-i,!0)}function t(r,i){var e,n=Dr.limit-Dr.cursor;return r()&&(Dr.cursor=Dr.limit-n,Dr.cursor>Dr.limit_backward&&(Dr.cursor--,e=Dr.limit-Dr.cursor,i()))?(Dr.cursor=Dr.limit-e,!0):(Dr.cursor=Dr.limit-n,r()?(Dr.cursor=Dr.limit-n,!1):(Dr.cursor=Dr.limit-n,!(Dr.cursor<=Dr.limit_backward)&&(Dr.cursor--,!!i()&&(Dr.cursor=Dr.limit-n,!0))))}function u(r){return t(r,function(){return Dr.in_grouping_b(Wr,97,305)})}function o(){return u(function(){return Dr.eq_s_b(1,"n")})}function s(){return u(function(){return Dr.eq_s_b(1,"s")})}function c(){return u(function(){return Dr.eq_s_b(1,"y")})}function l(){return t(function(){return Dr.in_grouping_b(Lr,105,305)},function(){return Dr.out_grouping_b(Wr,97,305)})}function a(){return Dr.find_among_b(ur,10)&&l()}function m(){return n()&&Dr.in_grouping_b(Lr,105,305)&&s()}function d(){return Dr.find_among_b(or,2)}function f(){return n()&&Dr.in_grouping_b(Lr,105,305)&&c()}function b(){return n()&&Dr.find_among_b(sr,4)}function w(){return n()&&Dr.find_among_b(cr,4)&&o()}function _(){return n()&&Dr.find_among_b(lr,2)&&c()}function k(){return n()&&Dr.find_among_b(ar,2)}function p(){return n()&&Dr.find_among_b(mr,4)}function g(){return n()&&Dr.find_among_b(dr,2)}function y(){return n()&&Dr.find_among_b(fr,4)}function z(){return n()&&Dr.find_among_b(br,2)}function v(){return n()&&Dr.find_among_b(wr,2)&&c()}function h(){return Dr.eq_s_b(2,"ki")}function q(){return n()&&Dr.find_among_b(_r,2)&&o()}function C(){return n()&&Dr.find_among_b(kr,4)&&c()}function P(){return n()&&Dr.find_among_b(pr,4)}function F(){return n()&&Dr.find_among_b(gr,4)&&c()}function S(){return Dr.find_among_b(yr,4)}function W(){return n()&&Dr.find_among_b(zr,2)}function L(){return n()&&Dr.find_among_b(vr,4)}function x(){return n()&&Dr.find_among_b(hr,8)}function A(){return Dr.find_among_b(qr,2)}function E(){return n()&&Dr.find_among_b(Cr,32)&&c()}function j(){return Dr.find_among_b(Pr,8)&&c()}function T(){return n()&&Dr.find_among_b(Fr,4)&&c()}function Z(){return Dr.eq_s_b(3,"ken")&&c()}function B(){var r=Dr.limit-Dr.cursor;return!(T()||(Dr.cursor=Dr.limit-r,E()||(Dr.cursor=Dr.limit-r,j()||(Dr.cursor=Dr.limit-r,Z()))))}function D(){if(A()){var r=Dr.limit-Dr.cursor;if(S()||(Dr.cursor=Dr.limit-r,W()||(Dr.cursor=Dr.limit-r,C()||(Dr.cursor=Dr.limit-r,P()||(Dr.cursor=Dr.limit-r,F()||(Dr.cursor=Dr.limit-r))))),T())return!1}return!0}function G(){if(W()){Dr.bra=Dr.cursor,Dr.slice_del();var r=Dr.limit-Dr.cursor;return Dr.ket=Dr.cursor,x()||(Dr.cursor=Dr.limit-r,E()||(Dr.cursor=Dr.limit-r,j()||(Dr.cursor=Dr.limit-r,T()||(Dr.cursor=Dr.limit-r)))),nr=!1,!1}return!0}function H(){if(!L())return!0;var r=Dr.limit-Dr.cursor;return!E()&&(Dr.cursor=Dr.limit-r,!j())}function I(){var r,i=Dr.limit-Dr.cursor;return!(S()||(Dr.cursor=Dr.limit-i,F()||(Dr.cursor=Dr.limit-i,P()||(Dr.cursor=Dr.limit-i,C()))))||(Dr.bra=Dr.cursor,Dr.slice_del(),r=Dr.limit-Dr.cursor,Dr.ket=Dr.cursor,T()||(Dr.cursor=Dr.limit-r),!1)}function J(){var r,i=Dr.limit-Dr.cursor;if(Dr.ket=Dr.cursor,nr=!0,B()&&(Dr.cursor=Dr.limit-i,D()&&(Dr.cursor=Dr.limit-i,G()&&(Dr.cursor=Dr.limit-i,H()&&(Dr.cursor=Dr.limit-i,I()))))){if(Dr.cursor=Dr.limit-i,!x())return;Dr.bra=Dr.cursor,Dr.slice_del(),Dr.ket=Dr.cursor,r=Dr.limit-Dr.cursor,S()||(Dr.cursor=Dr.limit-r,W()||(Dr.cursor=Dr.limit-r,C()||(Dr.cursor=Dr.limit-r,P()||(Dr.cursor=Dr.limit-r,F()||(Dr.cursor=Dr.limit-r))))),T()||(Dr.cursor=Dr.limit-r)}Dr.bra=Dr.cursor,Dr.slice_del()}function K(){var r,i,e,n;if(Dr.ket=Dr.cursor,h()){if(r=Dr.limit-Dr.cursorLine truncated
|
||||
@@ -1 +0,0 @@
|
||||
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.vi=function(){this.pipeline.reset(),this.pipeline.add(e.vi.stopWordFilter,e.vi.trimmer)},e.vi.wordCharacters="[A-Za-ẓ̀͐́͑̉̃̓ÂâÊêÔôĂ-ăĐ-đƠ-ơƯ-ư]",e.vi.trimmer=e.trimmerSupport.generateTrimmer(e.vi.wordCharacters),e.Pipeline.registerFunction(e.vi.trimmer,"trimmer-vi"),e.vi.stopWordFilter=e.generateStopWordFilter("là cái nhưng mà".split(" "))}});
|
||||
@@ -1 +0,0 @@
|
||||
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r(require("@node-rs/jieba")):r()(e.lunr)}(this,function(e){return function(r,t){if(void 0===r)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===r.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");var i="2"==r.version[0];r.zh=function(){this.pipeline.reset(),this.pipeline.add(r.zh.trimmer,r.zh.stopWordFilter,r.zh.stemmer),i?this.tokenizer=r.zh.tokenizer:(r.tokenizer&&(r.tokenizer=r.zh.tokenizer),this.tokenizerFn&&(this.tokenizerFn=r.zh.tokenizer))},r.zh.tokenizer=function(n){if(!arguments.length||null==n||void 0==n)return[];if(Array.isArray(n))return n.map(function(e){return i?new r.Token(e.toLowerCase()):e.toLowerCase()});t&&e.load(t);var o=n.toString().trim().toLowerCase(),s=[];e.cut(o,!0).forEach(function(e){s=s.concat(e.split(" "))}),s=s.filter(function(e){return!!e});var u=0;return s.map(function(e,t){if(i){var n=o.indexOf(e,u),s={};return s.position=[n,e.length],s.index=t,u=n,new r.Token(e,s)}return e})},r.zh.wordCharacters="\\w一-龥",r.zh.trimmer=r.trimmerSupport.generateTrimmer(r.zh.wordCharacters),r.Pipeline.registerFunction(r.zh.trimmer,"trimmer-zh"),r.zh.stemmer=function(){return function(e){return e}}(),r.Pipeline.registerFunction(r.zh.stemmer,"stemmer-zh"),r.zh.stopWordFilter=r.generateStopWordFilter("的 一 不 在 人 有 是 为 為 以 于 於 上 他 而 后 後 之 来 來 及 了 因 下 可 到 由 这 這 与 與 也 此 但 并 並 个 個 其 已 无 無 小 我 们 們 起 最 再 今 去 好 只 又 或 很 亦 某 把 那 你 乃 它 吧 被 比 别 趁 当 當 从 從 得 打 凡 儿 兒 尔 爾 该 該 各 给 給 跟 和 何 还 還 即 几 幾 既 看 据 據 距 靠 啦 另 么 麽 每 嘛 拿 哪 您 凭 憑 且 却 卻 让 讓 仍 啥 如 若 使 谁 誰 虽 雖 随 隨 同 所 她 哇 嗡 往 些 向 沿 哟 喲 用 咱 则 則 怎 曾 至 致 着 著 诸 諸 自".split(" ")),r.Pipeline.registerFunction(r.zh.stopWordFilter,"stopWordFilter-zh")}});
|
||||
@@ -1,206 +0,0 @@
|
||||
/**
|
||||
* export the module via AMD, CommonJS or as a browser global
|
||||
* Export code from https://github.com/umdjs/umd/blob/master/returnExports.js
|
||||
*/
|
||||
;(function (root, factory) {
|
||||
if (typeof define === 'function' && define.amd) {
|
||||
// AMD. Register as an anonymous module.
|
||||
define(factory)
|
||||
} else if (typeof exports === 'object') {
|
||||
/**
|
||||
* Node. Does not work with strict CommonJS, but
|
||||
* only CommonJS-like environments that support module.exports,
|
||||
* like Node.
|
||||
*/
|
||||
module.exports = factory()
|
||||
} else {
|
||||
// Browser globals (root is window)
|
||||
factory()(root.lunr);
|
||||
}
|
||||
}(this, function () {
|
||||
/**
|
||||
* Just return a value to define the module export.
|
||||
* This example returns an object, but the module
|
||||
* can return a function as the exported value.
|
||||
*/
|
||||
|
||||
return function(lunr) {
|
||||
// TinySegmenter 0.1 -- Super compact Japanese tokenizer in Javascript
|
||||
// (c) 2008 Taku Kudo <taku@chasen.org>
|
||||
// TinySegmenter is freely distributable under the terms of a new BSD licence.
|
||||
// For details, see http://chasen.org/~taku/software/TinySegmenter/LICENCE.txt
|
||||
|
||||
function TinySegmenter() {
|
||||
var patterns = {
|
||||
"[一二三四五六七八九十百千万億兆]":"M",
|
||||
"[一-龠々〆ヵヶ]":"H",
|
||||
"[ぁ-ん]":"I",
|
||||
"[ァ-ヴーア-ン゙ー]":"K",
|
||||
"[a-zA-Za-zA-Z]":"A",
|
||||
"[0-90-9]":"N"
|
||||
}
|
||||
this.chartype_ = [];
|
||||
for (var i in patterns) {
|
||||
var regexp = new RegExp(i);
|
||||
this.chartype_.push([regexp, patterns[i]]);
|
||||
}
|
||||
|
||||
this.BIAS__ = -332
|
||||
this.BC1__ = {"HH":6,"II":2461,"KH":406,"OH":-1378};
|
||||
this.BC2__ = {"AA":-3267,"AI":2744,"AN":-878,"HH":-4070,"HM":-1711,"HN":4012,"HO":3761,"IA":1327,"IH":-1184,"II":-1332,"IK":1721,"IO":5492,"KI":3831,"KK":-8741,"MH":-3132,"MK":3334,"OO":-2920};
|
||||
this.BC3__ = {"HH":996,"HI":626,"HK":-721,"HN":-1307,"HO":-836,"IH":-301,"KK":2762,"MK":1079,"MM":4034,"OA":-1652,"OH":266};
|
||||
this.BP1__ = {"BB":295,"OB":304,"OO":-125,"UB":352};
|
||||
this.BP2__ = {"BO":60,"OO":-1762};
|
||||
this.BQ1__ = {"BHH":1150,"BHM":1521,"BII":-1158,"BIM":886,"BMH":1208,"BNH":449,"BOH":-91,"BOO":-2597,"OHI":451,"OIH":-296,"OKA":1851,"OKH":-1020,"OKK":904,"OOO":2965};
|
||||
this.BQ2__ = {"BHH":118,"BHI":-1159,"BHM":466,"BIH":-919,"BKK":-1720,"BKO":864,"OHH":-1139,"OHM":-181,"OIH":153,"UHI":-1146};
|
||||
this.BQ3__ = {"BHH":-792,"BHI":2664,"BII":-299,"BKI":419,"BMH":937,"BMM":8335,"BNN":998,"BOH":775,"OHH":2174,"OHM":439,"OII":280,"OKH":1798,"OKI":-793,"OKO":-2242,"OMH":-2402,"OOO":11699};
|
||||
this.BQ4__ = {"BHH":-3895,"BIH":3761,"BII":-4654,"BIK":1348,"BKK":-1806,"BMI":-3385,"BOO":-12396,"OAH":926,"OHH":266,"OHK":-2036,"ONN":-973};
|
||||
this.BW1__ = {",と":660,",同":727,"B1あ":1404,"B1同":542,"、と":660,"、同":727,"」と":1682,"あっ":1505,"いう":1743,"いっ":-2055,"いる":672,"うし":-4817,"うん":665,"から":3472,"がら":600,"こう":-790,"こと":2083,"こん":-1262,"さら":-4143,"さん":4573,"した":2641,"して":1104,"すで":-3399,"そこ":1977,"それ":-871,"たち":1122,"ため":601,"った":3463,"つい":-802,"てい":805,"てき":1249,"でき":1127,"です":3445,"では":844,"とい":-4915,"とみ":1922,"どこ":3887,"ない":5713,"なっ":3015,"など":7379,"なん":-1113,"にし":2468,"には":1498,"にも":1671,"に対":-912,"の一":-501,"の中":741,"ませ":2448,"まで":1711,"まま":2600,"まる":-2155,"やむ":-1947,"よっ":-2565,"れた":2369,"れで":-913,"をし":1860,"を見":731,"亡く":-1886,"京都":2558,"取り":-2784,"大き":-2604,"大阪":1497,"平方":-2314,"引き":-1336,"日本":-195,"本当":-2423,"毎日":-2113,"目指":-724,"B1あ":1404,"B1同":542,"」と":1682};
|
||||
this.BW2__ = {"..":-11822,"11":-669,"――":-5730,"−−":-13175,"いう":-1609,"うか":2490,"かし":-1350,"かも":-602,"から":-7194,"かれ":4612,"がい":853,"がら":-3198,"きた":1941,"くな":-1597,"こと":-8392,"この":-4193,"させ":4533,"され":13168,"さん":-3977,"しい":-1819,"しか":-545,"した":5078,"して":972,"しな":939,"その":-3744,"たい":-1253,"たた":-662,"ただ":-3857,"たち":-786,"たと":1224,"たは":-939,"った":4589,"って":1647,"っと":-2094,"てい":6144,"てき":3640,"てく":2551,"ては":-3110,"ても":-3065,"でい":2666,"でき":-1528,"でし":-3828,"です":-4761,"でも":-4203,"とい":1890,"とこ":-1746,"とと":-2279,"との":720,"とみ":5168,"とも":-3941,"ない":-2488,"なが":-1313,"など":-6509,"なの":2614,"なん":3099,"にお":-1615,"にし":2748,"にな":2454,"によ":-7236,"に対":-14943,"に従":-4688,"に関":-11388,"のか":2093,"ので":-7059,"のに":-6041,"のの":-6125,"はい":1073,"はが":-1033,"はず":-2532,"ばれ":1813,"まし":-1316,"まで":-6621,"まれ":5409,"めて":-3153,"もい":2230,"もの":-10713,"らか":-944,"らし":-1611,"らに":-1897,"りし":651,"りま":1620,"れた":4270,"れて":849,"れば":4114,"ろう":6067,"われ":7901,"を通":-11877,"んだ":728,"んな":-4115,"一人":602,"一方":-1375,"一日":970,"一部":-1051,"上が":-4479,"会社":-1116,"出て":2163,"分の":-7758,"同党":970,"同日":-913,"大阪":-2471,"委員":-1250,"少な":-1050,"年度":-8669,"年間":-1626,"府県":-2363,"手権":-1982,"新聞":-4066,"日新":-722,"日本":-7068,"日米":3372,"曜日":-601,"朝鮮":-2355,"本人":-2697,"東京":-1543,"然と":-1384,"社会":-1276,"立て":-990,"第に":-1612,"米国":-4268,"11":-669};
|
||||
this.BW3__ = {"あた":-2194,"あり":719,"ある":3846,"い.":-1185,"い。":-1185,"いい":5308,"いえ":2079,"いく":3029,"いた":2056,"いっ":1883,"いる":5600,"いわ":1527,"うち":1117,"うと":4798,"えと":1454,"か.":2857,"か。":2857,"かけ":-743,"かっ":-4098,"かに":-669,"から":6520,"かり":-2670,"が,":1816,"が、":1816,"がき":-4855,"がけ":-1127,"がっ":-913,"がら":-4977,"がり":-2064,"きた":1645,"けど":1374,"こと":7397,"この":1542,"ころ":-2757,"さい":-714,"さを":976,"し,":1557,"し、":1557,"しい":-3714,"した":3562,"して":1449,"しな":2608,"しま":1200,"す.":-1310,"す。":-1310,"する":6521,"ず,":3426,"ず、":3426,"ずに":841,"そう":428,"た.":8875,"た。":8875,"たい":-594,"たの":812,"たり":-1183,"たる":-853,"だ.":4098,"だ。":4098,"だっ":1004,"った":-4748,"って":300,"てい":6240,"てお":855,"ても":302,"です":1437,"でに":-1482,"では":2295,"とう":-1387,"とし":2266,"との":541,"とも":-3543,"どう":4664,"ない":1796,"なく":-903,"など":2135,"に,":-1021,"に、":-1021,"にし":1771,"にな":1906,"には":2644,"の,":-724,"の、":-724,"の子":-1000,"は,":1337,"は、":1337,"べき":2181,"まし":1113,"ます":6943,"まっ":-1549,"まで":6154,"まれ":-793,"らし":1479,"られ":6820,"るる":3818,"れ,":854,"れ、":854,"れた":1850,"れて":1375,"れば":-3246,"れる":1091,"われ":-605,"んだ":606,"んで":798,"カ月":990,"会議":860,"入り":1232,"大会":2217,"始め":1681,"市":965,"新聞":-5055,"日,":974,"日、":974,"社会":2024,"カ月":990};
|
||||
this.TC1__ = {"AAA":1093,"HHH":1029,"HHM":580,"HII":998,"HOH":-390,"HOM":-331,"IHI":1169,"IOH":-142,"IOI":-1015,"IOM":467,"MMH":187,"OOI":-1832};
|
||||
this.TC2__ = {"HHO":2088,"HII":-1023,"HMM":-1154,"IHI":-1965,"KKH":703,"OII":-2649};
|
||||
this.TC3__ = {"AAA":-294,"HHH":346,"HHI":-341,"HII":-1088,"HIK":731,"HOH":-1486,"IHH":128,"IHI":-3041,"IHO":-1935,"IIH":-825,"IIM":-1035,"IOI":-542,"KHH":-1216,"KKA":491,"KKH":-1217,"KOK":-1009,"MHH":-2694,"MHM":-457,"MHO":123,"MMH":-471,"NNH":-1689,"NNO":662,"OHO":-3393};
|
||||
this.TC4__ = {"HHH":-203,"HHI":1344,"HHK":365,"HHM":-122,"HHN":182,"HHO":669,"HIH":804,"HII":679,"HOH":446,"IHH":695,"IHO":-2324,"IIH":321,"III":1497,"IIO":656,"IOO":54,"KAK":4845,"KKA":3386,"KKK":3065,"MHH":-405,"MHI":201,"MMH":-241,"MMM":661,"MOM":841};
|
||||
this.TQ1__ = {"BHHH":-227,"BHHI":316,"BHIH":-132,"BIHH":60,"BIII":1595,"BNHH":-744,"BOHH":225,"BOOO":-908,"OAKK":482,"OHHH":281,"OHIH":249,"OIHI":200,"OIIH":-68};
|
||||
this.TQ2__ = {"BIHH":-1401,"BIII":-1033,"BKAK":-543,"BOOO":-5591};
|
||||
this.TQ3__ = {"BHHH":478,"BHHM":-1073,"BHIH":222,"BHII":-504,"BIIH":-116,"BIII":-105,"BMHI":-863,"BMHM":-464,"BOMH":620,"OHHH":346,"OHHI":1729,"OHII":997,"OHMH":481,"OIHH":623,"OIIH":1344,"OKAK":2792,"OKHH":587,"OKKA":679,"OOHH":110,"OOII":-685};
|
||||
this.TQ4__ = {"BHHH":-721,"BHHM":-3604,"BHII":-966,"BIIH":-607,"BIII":-2181,"OAAA":-2763,"OAKK":180,"OHHH":-294,"OHHI":2446,"OHHO":480,"OHIH":-1573,"OIHH":1935,"OIHI":-493,"OIIH":626,"OIII":-4007,"OKAK":-8156};
|
||||
this.TW1__ = {"につい":-4681,"東京都":2026};
|
||||
this.TW2__ = {"ある程":-2049,"いった":-1256,"ころが":-2434,"しょう":3873,"その後":-4430,"だって":-1049,"ていた":1833,"として":-4657,"ともに":-4517,"もので":1882,"一気に":-792,"初めて":-1512,"同時に":-8097,"大きな":-1255,"対して":-2721,"社会党":-3216};
|
||||
this.TW3__ = {"いただ":-1734,"してい":1314,"として":-4314,"につい":-5483,"にとっ":-5989,"に当た":-6247,"ので,":-727,"ので、":-727,"のもの":-600,"れから":-3752,"十二月":-2287};
|
||||
this.TW4__ = {"いう.":8576,"いう。":8576,"からな":-2348,"してい":2958,"たが,":1516,"たが、":1516,"ている":1538,"という":1349,"ました":5543,"ません":1097,"ようと":-4258,"よると":5865};
|
||||
this.UC1__ = {"A":484,"K":93,"M":645,"O":-505};
|
||||
this.UC2__ = {"A":819,"H":1059,"I":409,"M":3987,"N":5775,"O":646};
|
||||
this.UC3__ = {"A":-1370,"I":2311};
|
||||
this.UC4__ = {"A":-2643,"H":1809,"I":-1032,"K":-3450,"M":3565,"N":3876,"O":6646};
|
||||
this.UC5__ = {"H":313,"I":-1238,"K":-799,"M":539,"O":-831};
|
||||
this.UC6__ = {"H":-506,"I":-253,"K":87,"M":247,"O":-387};
|
||||
this.UP1__ = {"O":-214};
|
||||
this.UP2__ = {"B":69,"O":935};
|
||||
this.UP3__ = {"B":189};
|
||||
this.UQ1__ = {"BH":21,"BI":-12,"BK":-99,"BN":142,"BO":-56,"OH":-95,"OI":477,"OK":410,"OO":-2422};
|
||||
this.UQ2__ = {"BH":216,"BI":113,"OK":1759};
|
||||
this.UQ3__ = {"BA":-479,"BH":42,"BI":1913,"BK":-7198,"BM":3160,"BN":6427,"BO":14761,"OI":-827,"ON":-3212};
|
||||
this.UW1__ = {",":156,"、":156,"「":-463,"あ":-941,"う":-127,"が":-553,"き":121,"こ":505,"で":-201,"と":-547,"ど":-123,"に":-789,"の":-185,"は":-847,"も":-466,"や":-470,"よ":182,"ら":-292,"り":208,"れ":169,"を":-446,"ん":-137,"・":-135,"主":-402,"京":-268,"区":-912,"午":871,"国":-460,"大":561,"委":729,"市":-411,"日":-141,"理":361,"生":-408,"県":-386,"都":-718,"「":-463,"・":-135};
|
||||
this.UW2__ = {",":-829,"、":-829,"〇":892,"「":-645,"」":3145,"あ":-538,"い":505,"う":134,"お":-502,"か":1454,"が":-856,"く":-412,"こ":1141,"さ":878,"ざ":540,"し":1529,"す":-675,"せ":300,"そ":-1011,"た":188,"だ":1837,"つ":-949,"て":-291,"で":-268,"と":-981,"ど":1273,"な":1063,"に":-1764,"の":130,"は":-409,"ひ":-1273,"べ":1261,"ま":600,"も":-1263,"や":-402,"よ":1639,"り":-579,"る":-694,"れ":571,"を":-2516,"ん":2095,"ア":-587,"カ":306,"キ":568,"ッ":831,"三":-758,"不":-2150,"世":-302,"中":-968,"主":-861,"事":492,"人":-123,"会":978,"保":362,"入":548,"初":-3025,"副":-1566,"北":-3414,"区":-422,"大":-1769,"天":-865,"太":-483,"子":-1519,"学":760,"実":1023,"小":-2009,"市":-813,"年":-1060,"強":1067,"手":-1519,"揺":-1033,"政":1522,"文":-1355,"新":-1682,"日":-1815,"明":-1462,"最":-630,"朝":-1843,"本":-1650,"東":-931,"果":-665,"次":-2378,"民":-180,"気":-1740,"理":752,"発":529,"目":-1584,"相":-242,"県":-1165,"立":-763,"第":810,"米":509,"自":-1353,"行":838,"西":-744,"見":-3874,"調":1010,"議":1198,"込":3041,"開":1758,"間":-1257,"「":-645,"」":3145,"ッ":831,"ア":-587,"カ":306,"キ":568};
|
||||
this.UW3__ = {",":4889,"1":-800,"−":-1723,"、":4889,"々":-2311,"〇":5827,"」":2670,"〓":-3573,"あ":-2696,"い":1006,"う":2342,"え":1983,"お":-4864,"か":-1163,"が":3271,"く":1004,"け":388,"げ":401,"こ":-3552,"ご":-3116,"さ":-1058,"し":-395,"す":584,"せ":3685,"そ":-5228,"た":842,"ち":-521,"っ":-1444,"つ":-1081,"て":6167,"で":2318,"と":1691,"ど":-899,"な":-2788,"に":2745,"の":4056,"は":4555,"ひ":-2171,"ふ":-1798,"へ":1199,"ほ":-5516,"ま":-4384,"み":-120,"め":1205,"も":2323,"や":-788,"よ":-202,"ら":727,"り":649,"る":5905,"れ":2773,"わ":-1207,"を":6620,"ん":-518,"ア":551,"グ":1319,"ス":874,"ッ":-1350,"ト":521,"ム":1109,"ル":1591,"ロ":2201,"ン":278,"・":-3794,"一":-1619,"下":-1759,"世":-2087,"両":3815,"中":653,"主":-758,"予":-1193,"二":974,"人":2742,"今":792,"他":1889,"以":-1368,"低":811,"何":4265,"作":-361,"保":-2439,"元":4858,"党":3593,"全":1574,"公":-3030,"六":755,"共":-1880,"円":5807,"再":3095,"分":457,"初":2475,"別":1129,"前":2286,"副":4437,"力":365,"動":-949,"務":-1872,"化":1327,"北":-1038,"区":4646,"千":-2309,"午":-783,"協":-1006,"口":483,"右":1233,"各":3588,"合":-241,"同":3906,"和":-837,"員":4513,"国":642,"型":1389,"場":1219,"外":-241,"妻":2016,"学":-1356,"安":-423,"実":-1008,"家":1078,"小":-513,"少":-3102,"州":1155,"市":3197,"平":-1804,"年":2416,"広":-1030,"府":1605,"度":1452,"建":-2352,"当":-3885,"得":1905,"思":-1291,"性":1822,"戸":-488,"指":-3973,"政":-2013,"教":-1479,"数":3222,"文":-1489,"新":1764,"日":2099,"旧":5792,"昨":-661,"時":-1248,"曜":-951,"最":-937,"月":4125,"期":360,"李":3094,"村":364,"東":-805,"核":5156,"森":2438,"業":484,"氏":2613,"民":-1694,"決":-1073,"法":1868,"海":-495,"無":979,"物":461,"特":-3850,"生":-273,"用":914,"町":1215,"的":7313,"直":-1835,"省":792,"県":6293,"知":-1528,"私":4231,"税":401,"立":-960,"第":1201,"米":7767,"系":3066,"約":3663,"級":1384,"統":-4229,"総":1163,"線":1255,"者":6457,"能":725,"自":-2869,"英":785,"見":1044,"調":-562,"財":-733,"費":1777,"車":1835,"軍":1375,"込":-1504,"通":-1136,"選":-681,"郎":1026,"郡":4404,"部":1200,"金":2163,"長":421,"開":-1432,"間":1302,"関":-1282,"雨":2009,"電":-1045,"非":2066,"駅":1620,"1":-800,"」":2670,"・":-3794,"ッ":-1350,"ア":551,"グ":1319,"ス":874,"ト":521,"ム":1109,"ル":1591,"ロ":2201,"ン":278};
|
||||
this.UW4__ = {",":3930,".":3508,"―":-4841,"、":3930,"。":3508,"〇":4999,"「":1895,"」":3798,"〓":-5156,"あ":4752,"い":-3435,"う":-640,"え":-2514,"お":2405,"か":530,"が":6006,"き":-4482,"ぎ":-3821,"く":-3788,"け":-4376,"げ":-4734,"こ":2255,"ご":1979,"さ":2864,"し":-843,"じ":-2506,"す":-731,"ず":1251,"せ":181,"そ":4091,"た":5034,"だ":5408,"ち":-3654,"っ":-5882,"つ":-1659,"て":3994,"で":7410,"と":4547,"な":5433,"に":6499,"ぬ":1853,"ね":1413,"の":7396,"は":8578,"ば":1940,"ひ":4249,"び":-4134,"ふ":1345,"へ":6665,"べ":-744,"ほ":1464,"ま":1051,"み":-2082,"む":-882,"め":-5046,"も":4169,"ゃ":-2666,"や":2795,"ょ":-1544,"よ":3351,"ら":-2922,"り":-9726,"る":-14896,"れ":-2613,"ろ":-4570,"わ":-1783,"を":13150,"ん":-2352,"カ":2145,"コ":1789,"セ":1287,"ッ":-724,"ト":-403,"メ":-1635,"ラ":-881,"リ":-541,"ル":-856,"ン":-3637,"・":-4371,"ー":-11870,"一":-2069,"中":2210,"予":782,"事":-190,"井":-1768,"人":1036,"以":544,"会":950,"体":-1286,"作":530,"側":4292,"先":601,"党":-2006,"共":-1212,"内":584,"円":788,"初":1347,"前":1623,"副":3879,"力":-302,"動":-740,"務":-2715,"化":776,"区":4517,"協":1013,"参":1555,"合":-1834,"和":-681,"員":-910,"器":-851,"回":1500,"国":-619,"園":-1200,"地":866,"場":-1410,"塁":-2094,"士":-1413,"多":1067,"大":571,"子":-4802,"学":-1397,"定":-1057,"寺":-809,"小":1910,"屋":-1328,"山":-1500,"島":-2056,"川":-2667,"市":2771,"年":374,"庁":-4556,"後":456,"性":553,"感":916,"所":-1566,"支":856,"改":787,"政":2182,"教":704,"文":522,"方":-856,"日":1798,"時":1829,"最":845,"月":-9066,"木":-485,"来":-442,"校":-360,"業":-1043,"氏":5388,"民":-2716,"気":-910,"沢":-939,"済":-543,"物":-735,"率":672,"球":-1267,"生":-1286,"産":-1101,"田":-2900,"町":1826,"的":2586,"目":922,"省":-3485,"県":2997,"空":-867,"立":-2112,"第":788,"米":2937,"系":786,"約":2171,"経":1146,"統":-1169,"総":940,"線":-994,"署":749,"者":2145,"能":-730,"般":-852,"行":-792,"規":792,"警":-1184,"議":-244,"谷":-1000,"賞":730,"車":-1481,"軍":1158,"輪":-1433,"込":-3370,"近":929,"道":-1291,"選":2596,"郎":-4866,"都":1192,"野":-1100,"銀":-2213,"長":357,"間":-2344,"院":-2297,"際":-2604,"電":-878,"領":-1659,"題":-792,"館":-1984,"首":1749,"高":2120,"「":1895,"」":3798,"・":-4371,"ッ":-724,"ー":-11870,"カ":2145,"コ":1789,"セ":1287,"ト":-403,"メ":-1635,"ラ":-881,"リ":-541,"ル":-856,"ン":-3637};
|
||||
this.UW5__ = {",":465,".":-299,"1":-514,"E2":-32768,"]":-2762,"、":465,"。":-299,"「":363,"あ":1655,"い":331,"う":-503,"え":1199,"お":527,"か":647,"が":-421,"き":1624,"ぎ":1971,"く":312,"げ":-983,"さ":-1537,"し":-1371,"す":-852,"だ":-1186,"ち":1093,"っ":52,"つ":921,"て":-18,"で":-850,"と":-127,"ど":1682,"な":-787,"に":-1224,"の":-635,"は":-578,"べ":1001,"み":502,"め":865,"ゃ":3350,"ょ":854,"り":-208,"る":429,"れ":504,"わ":419,"を":-1264,"ん":327,"イ":241,"ル":451,"ン":-343,"中":-871,"京":722,"会":-1153,"党":-654,"務":3519,"区":-901,"告":848,"員":2104,"大":-1296,"学":-548,"定":1785,"嵐":-1304,"市":-2991,"席":921,"年":1763,"思":872,"所":-814,"挙":1618,"新":-1682,"日":218,"月":-4353,"査":932,"格":1356,"機":-1508,"氏":-1347,"田":240,"町":-3912,"的":-3149,"相":1319,"省":-1052,"県":-4003,"研":-997,"社":-278,"空":-813,"統":1955,"者":-2233,"表":663,"語":-1073,"議":1219,"選":-1018,"郎":-368,"長":786,"間":1191,"題":2368,"館":-689,"1":-514,"E2":-32768,"「":363,"イ":241,"ル":451,"ン":-343};
|
||||
this.UW6__ = {",":227,".":808,"1":-270,"E1":306,"、":227,"。":808,"あ":-307,"う":189,"か":241,"が":-73,"く":-121,"こ":-200,"じ":1782,"す":383,"た":-428,"っ":573,"て":-1014,"で":101,"と":-105,"な":-253,"に":-149,"の":-417,"は":-236,"も":-206,"り":187,"る":-135,"を":195,"ル":-673,"ン":-496,"一":-277,"中":201,"件":-800,"会":624,"前":302,"区":1792,"員":-1212,"委":798,"学":-960,"市":887,"広":-695,"後":535,"業":-697,"相":753,"社":-507,"福":974,"空":-822,"者":1811,"連":463,"郎":1082,"1":-270,"E1":306,"ル":-673,"ン":-496};
|
||||
|
||||
return this;
|
||||
}
|
||||
TinySegmenter.prototype.ctype_ = function(str) {
|
||||
for (var i in this.chartype_) {
|
||||
if (str.match(this.chartype_[i][0])) {
|
||||
return this.chartype_[i][1];
|
||||
}
|
||||
}
|
||||
return "O";
|
||||
}
|
||||
|
||||
TinySegmenter.prototype.ts_ = function(v) {
|
||||
if (v) { return v; }
|
||||
return 0;
|
||||
}
|
||||
|
||||
TinySegmenter.prototype.segment = function(input) {
|
||||
if (input == null || input == undefined || input == "") {
|
||||
return [];
|
||||
}
|
||||
var result = [];
|
||||
var seg = ["B3","B2","B1"];
|
||||
var ctype = ["O","O","O"];
|
||||
var o = input.split("");
|
||||
for (i = 0; i < o.length; ++i) {
|
||||
seg.push(o[i]);
|
||||
ctype.push(this.ctype_(o[i]))
|
||||
}
|
||||
seg.push("E1");
|
||||
seg.push("E2");
|
||||
seg.push("E3");
|
||||
ctype.push("O");
|
||||
ctype.push("O");
|
||||
ctype.push("O");
|
||||
var word = seg[3];
|
||||
var p1 = "U";
|
||||
var p2 = "U";
|
||||
var p3 = "U";
|
||||
for (var i = 4; i < seg.length - 3; ++i) {
|
||||
var score = this.BIAS__;
|
||||
var w1 = seg[i-3];
|
||||
var w2 = seg[i-2];
|
||||
var w3 = seg[i-1];
|
||||
var w4 = seg[i];
|
||||
var w5 = seg[i+1];
|
||||
var w6 = seg[i+2];
|
||||
var c1 = ctype[i-3];
|
||||
var c2 = ctype[i-2];
|
||||
var c3 = ctype[i-1];
|
||||
var c4 = ctype[i];
|
||||
var c5 = ctype[i+1];
|
||||
var c6 = ctype[i+2];
|
||||
score += this.ts_(this.UP1__[p1]);
|
||||
score += this.ts_(this.UP2__[p2]);
|
||||
score += this.ts_(this.UP3__[p3]);
|
||||
score += this.ts_(this.BP1__[p1 + p2]);
|
||||
score += this.ts_(this.BP2__[p2 + p3]);
|
||||
score += this.ts_(this.UW1__[w1]);
|
||||
score += this.ts_(this.UW2__[w2]);
|
||||
score += this.ts_(this.UW3__[w3]);
|
||||
score += this.ts_(this.UW4__[w4]);
|
||||
score += this.ts_(this.UW5__[w5]);
|
||||
score += this.ts_(this.UW6__[w6]);
|
||||
score += this.ts_(this.BW1__[w2 + w3]);
|
||||
score += this.ts_(this.BW2__[w3 + w4]);
|
||||
score += this.ts_(this.BW3__[w4 + w5]);
|
||||
score += this.ts_(this.TW1__[w1 + w2 + w3]);
|
||||
score += this.ts_(this.TW2__[w2 + w3 + w4]);
|
||||
score += this.ts_(this.TW3__[w3 + w4 + w5]);
|
||||
score += this.ts_(this.TW4__[w4 + w5 + w6]);
|
||||
score += this.ts_(this.UC1__[c1]);
|
||||
score += this.ts_(this.UC2__[c2]);
|
||||
score += this.ts_(this.UC3__[c3]);
|
||||
score += this.ts_(this.UC4__[c4]);
|
||||
score += this.ts_(this.UC5__[c5]);
|
||||
score += this.ts_(this.UC6__[c6]);
|
||||
score += this.ts_(this.BC1__[c2 + c3]);
|
||||
score += this.ts_(this.BC2__[c3 + c4]);
|
||||
score += this.ts_(this.BC3__[c4 + c5]);
|
||||
score += this.ts_(this.TC1__[c1 + c2 + c3]);
|
||||
score += this.ts_(this.TC2__[c2 + c3 + c4]);
|
||||
score += this.ts_(this.TC3__[c3 + c4 + c5]);
|
||||
score += this.ts_(this.TC4__[c4 + c5 + c6]);
|
||||
// score += this.ts_(this.TC5__[c4 + c5 + c6]);
|
||||
score += this.ts_(this.UQ1__[p1 + c1]);
|
||||
score += this.ts_(this.UQ2__[p2 + c2]);
|
||||
score += this.ts_(this.UQ3__[p3 + c3]);
|
||||
score += this.ts_(this.BQ1__[p2 + c2 + c3]);
|
||||
score += this.ts_(this.BQ2__[p2 + c3 + c4]);
|
||||
score += this.ts_(this.BQ3__[p3 + c2 + c3]);
|
||||
score += this.ts_(this.BQ4__[p3 + c3 + c4]);
|
||||
score += this.ts_(this.TQ1__[p2 + c1 + c2 + c3]);
|
||||
score += this.ts_(this.TQ2__[p2 + c2 + c3 + c4]);
|
||||
score += this.ts_(this.TQ3__[p3 + c1 + c2 + c3]);
|
||||
score += this.ts_(this.TQ4__[p3 + c2 + c3 + c4]);
|
||||
var p = "O";
|
||||
if (score > 0) {
|
||||
result.push(word);
|
||||
word = "";
|
||||
p = "B";
|
||||
}
|
||||
p1 = p2;
|
||||
p2 = p3;
|
||||
p3 = p;
|
||||
word += seg[i];
|
||||
}
|
||||
result.push(word);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
lunr.TinySegmenter = TinySegmenter;
|
||||
};
|
||||
|
||||
}));
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,42 +0,0 @@
|
||||
"use strict";(()=>{var xe=Object.create;var G=Object.defineProperty,ve=Object.defineProperties,Se=Object.getOwnPropertyDescriptor,Te=Object.getOwnPropertyDescriptors,Qe=Object.getOwnPropertyNames,J=Object.getOwnPropertySymbols,Ee=Object.getPrototypeOf,Z=Object.prototype.hasOwnProperty,be=Object.prototype.propertyIsEnumerable;var K=Math.pow,X=(t,e,r)=>e in t?G(t,e,{enumerable:!0,configurable:!0,writable:!0,value:r}):t[e]=r,_=(t,e)=>{for(var r in e||(e={}))Z.call(e,r)&&X(t,r,e[r]);if(J)for(var r of J(e))be.call(e,r)&&X(t,r,e[r]);return t},B=(t,e)=>ve(t,Te(e));var Le=(t,e)=>()=>(e||t((e={exports:{}}).exports,e),e.exports);var we=(t,e,r,n)=>{if(e&&typeof e=="object"||typeof e=="function")for(let i of Qe(e))!Z.call(t,i)&&i!==r&&G(t,i,{get:()=>e[i],enumerable:!(n=Se(e,i))||n.enumerable});return t};var Pe=(t,e,r)=>(r=t!=null?xe(Ee(t)):{},we(e||!t||!t.__esModule?G(r,"default",{value:t,enumerable:!0}):r,t));var W=(t,e,r)=>new Promise((n,i)=>{var s=u=>{try{a(r.next(u))}catch(c){i(c)}},o=u=>{try{a(r.throw(u))}catch(c){i(c)}},a=u=>u.done?n(u.value):Promise.resolve(u.value).then(s,o);a((r=r.apply(t,e)).next())});var re=Le((ee,te)=>{/**
|
||||
* lunr - http://lunrjs.com - A bit like Solr, but much smaller and not as bright - 2.3.9
|
||||
* Copyright (C) 2020 Oliver Nightingale
|
||||
* @license MIT
|
||||
*/(function(){var t=function(e){var r=new t.Builder;return r.pipeline.add(t.trimmer,t.stopWordFilter,t.stemmer),r.searchPipeline.add(t.stemmer),e.call(r,r),r.build()};t.version="2.3.9";/*!
|
||||
* lunr.utils
|
||||
* Copyright (C) 2020 Oliver Nightingale
|
||||
*/t.utils={},t.utils.warn=(function(e){return function(r){e.console&&console.warn&&console.warn(r)}})(this),t.utils.asString=function(e){return e==null?"":e.toString()},t.utils.clone=function(e){if(e==null)return e;for(var r=Object.create(null),n=Object.keys(e),i=0;i<n.length;i++){var s=n[i],o=e[s];if(Array.isArray(o)){r[s]=o.slice();continue}if(typeof o=="string"||typeof o=="number"||typeof o=="boolean"){r[s]=o;continue}throw new TypeError("clone is not deep and does not support nested objects")}return r},t.FieldRef=function(e,r,n){this.docRef=e,this.fieldName=r,this._stringValue=n},t.FieldRef.joiner="/",t.FieldRef.fromString=function(e){var r=e.indexOf(t.FieldRef.joiner);if(r===-1)throw"malformed field ref string";var n=e.slice(0,r),i=e.slice(r+1);return new t.FieldRef(i,n,e)},t.FieldRef.prototype.toString=function(){return this._stringValue==null&&(this._stringValue=this.fieldName+t.FieldRef.joiner+this.docRef),this._stringValue};/*!
|
||||
* lunr.Set
|
||||
* Copyright (C) 2020 Oliver Nightingale
|
||||
*/t.Set=function(e){if(this.elements=Object.create(null),e){this.length=e.length;for(var r=0;r<this.length;r++)this.elements[e[r]]=!0}else this.length=0},t.Set.complete={intersect:function(e){return e},union:function(){return this},contains:function(){return!0}},t.Set.empty={intersect:function(){return this},union:function(e){return e},contains:function(){return!1}},t.Set.prototype.contains=function(e){return!!this.elements[e]},t.Set.prototype.intersect=function(e){var r,n,i,s=[];if(e===t.Set.complete)return this;if(e===t.Set.empty)return e;this.length<e.length?(r=this,n=e):(r=e,n=this),i=Object.keys(r.elements);for(var o=0;o<i.length;o++){var a=i[o];a in n.elements&&s.push(a)}return new t.Set(s)},t.Set.prototype.union=function(e){return e===t.Set.complete?t.Set.complete:e===t.Set.empty?this:new t.Set(Object.keys(this.elements).concat(Object.keys(e.elements)))},t.idf=function(e,r){var n=0;for(var i in e)i!="_index"&&(n+=Object.keys(e[i]).length);var s=(r-n+.5)/(n+.5);return Math.log(1+Math.abs(s))},t.Token=function(e,r){this.str=e||"",this.metadata=r||{}},t.Token.prototype.toString=function(){return this.str},t.Token.prototype.update=function(e){return this.str=e(this.str,this.metadata),this},t.Token.prototype.clone=function(e){return e=e||function(r){return r},new t.Token(e(this.str,this.metadata),this.metadata)};/*!
|
||||
* lunr.tokenizer
|
||||
* Copyright (C) 2020 Oliver Nightingale
|
||||
*/t.tokenizer=function(e,r){if(e==null||e==null)return[];if(Array.isArray(e))return e.map(function(g){return new t.Token(t.utils.asString(g).toLowerCase(),t.utils.clone(r))});for(var n=e.toString().toLowerCase(),i=n.length,s=[],o=0,a=0;o<=i;o++){var u=n.charAt(o),c=o-a;if(u.match(t.tokenizer.separator)||o==i){if(c>0){var f=t.utils.clone(r)||{};f.position=[a,c],f.index=s.length,s.push(new t.Token(n.slice(a,o),f))}a=o+1}}return s},t.tokenizer.separator=/[\s\-]+/;/*!
|
||||
* lunr.Pipeline
|
||||
* Copyright (C) 2020 Oliver Nightingale
|
||||
*/t.Pipeline=function(){this._stack=[]},t.Pipeline.registeredFunctions=Object.create(null),t.Pipeline.registerFunction=function(e,r){r in this.registeredFunctions&&t.utils.warn("Overwriting existing registered function: "+r),e.label=r,t.Pipeline.registeredFunctions[e.label]=e},t.Pipeline.warnIfFunctionNotRegistered=function(e){var r=e.label&&e.label in this.registeredFunctions;r||t.utils.warn(`Function is not registered with pipeline. This may cause problems when serialising the index.
|
||||
`,e)},t.Pipeline.load=function(e){var r=new t.Pipeline;return e.forEach(function(n){var i=t.Pipeline.registeredFunctions[n];if(i)r.add(i);else throw new Error("Cannot load unregistered function: "+n)}),r},t.Pipeline.prototype.add=function(){var e=Array.prototype.slice.call(arguments);e.forEach(function(r){t.Pipeline.warnIfFunctionNotRegistered(r),this._stack.push(r)},this)},t.Pipeline.prototype.after=function(e,r){t.Pipeline.warnIfFunctionNotRegistered(r);var n=this._stack.indexOf(e);if(n==-1)throw new Error("Cannot find existingFn");n=n+1,this._stack.splice(n,0,r)},t.Pipeline.prototype.before=function(e,r){t.Pipeline.warnIfFunctionNotRegistered(r);var n=this._stack.indexOf(e);if(n==-1)throw new Error("Cannot find existingFn");this._stack.splice(n,0,r)},t.Pipeline.prototype.remove=function(e){var r=this._stack.indexOf(e);r!=-1&&this._stack.splice(r,1)},t.Pipeline.prototype.run=function(e){for(var r=this._stack.length,n=0;n<r;n++){for(var i=this._stack[n],s=[],o=0;o<e.length;o++){var a=i(e[o],o,e);if(!(a==null||a===""))if(Array.isArray(a))for(var u=0;u<a.length;u++)s.push(a[u]);else s.push(a)}e=s}return e},t.Pipeline.prototype.runString=function(e,r){var n=new t.Token(e,r);return this.run([n]).map(function(i){return i.toString()})},t.Pipeline.prototype.reset=function(){this._stack=[]},t.Pipeline.prototype.toJSON=function(){return this._stack.map(function(e){return t.Pipeline.warnIfFunctionNotRegistered(e),e.label})};/*!
|
||||
* lunr.Vector
|
||||
* Copyright (C) 2020 Oliver Nightingale
|
||||
*/t.Vector=function(e){this._magnitude=0,this.elements=e||[]},t.Vector.prototype.positionForIndex=function(e){if(this.elements.length==0)return 0;for(var r=0,n=this.elements.length/2,i=n-r,s=Math.floor(i/2),o=this.elements[s*2];i>1&&(o<e&&(r=s),o>e&&(n=s),o!=e);)i=n-r,s=r+Math.floor(i/2),o=this.elements[s*2];if(o==e||o>e)return s*2;if(o<e)return(s+1)*2},t.Vector.prototype.insert=function(e,r){this.upsert(e,r,function(){throw"duplicate index"})},t.Vector.prototype.upsert=function(e,r,n){this._magnitude=0;var i=this.positionForIndex(e);this.elements[i]==e?this.elements[i+1]=n(this.elements[i+1],r):this.elements.splice(i,0,e,r)},t.Vector.prototype.magnitude=function(){if(this._magnitude)return this._magnitude;for(var e=0,r=this.elements.length,n=1;n<r;n+=2){var i=this.elements[n];e+=i*i}return this._magnitude=Math.sqrt(e)},t.Vector.prototype.dot=function(e){for(var r=0,n=this.elements,i=e.elements,s=n.length,o=i.length,a=0,u=0,c=0,f=0;c<s&&f<o;)a=n[c],u=i[f],a<u?c+=2:a>u?f+=2:a==u&&(r+=n[c+1]*i[f+1],c+=2,f+=2);return r},t.Vector.prototype.similarity=function(e){return this.dot(e)/this.magnitude()||0},t.Vector.prototype.toArray=function(){for(var e=new Array(this.elements.length/2),r=1,n=0;r<this.elements.length;r+=2,n++)e[n]=this.elements[r];return e},t.Vector.prototype.toJSON=function(){return this.elements};/*!
|
||||
* lunr.stemmer
|
||||
* Copyright (C) 2020 Oliver Nightingale
|
||||
* Includes code from - http://tartarus.org/~martin/PorterStemmer/js.txt
|
||||
*/t.stemmer=(function(){var e={ational:"ate",tional:"tion",enci:"ence",anci:"ance",izer:"ize",bli:"ble",alli:"al",entli:"ent",eli:"e",ousli:"ous",ization:"ize",ation:"ate",ator:"ate",alism:"al",iveness:"ive",fulness:"ful",ousness:"ous",aliti:"al",iviti:"ive",biliti:"ble",logi:"log"},r={icate:"ic",ative:"",alize:"al",iciti:"ic",ical:"ic",ful:"",ness:""},n="[^aeiou]",i="[aeiouy]",s=n+"[^aeiouy]*",o=i+"[aeiou]*",a="^("+s+")?"+o+s,u="^("+s+")?"+o+s+"("+o+")?$",c="^("+s+")?"+o+s+o+s,f="^("+s+")?"+i,g=new RegExp(a),l=new RegExp(c),m=new RegExp(u),x=new RegExp(f),v=/^(.+?)(ss|i)es$/,d=/^(.+?)([^s])s$/,y=/^(.+?)eed$/,b=/^(.+?)(ed|ing)$/,E=/.$/,w=/(at|bl|iz)$/,R=new RegExp("([^aeiouylsz])\\1$"),j=new RegExp("^"+s+i+"[^aeiouwxy]$"),C=/^(.+?[^aeiou])y$/,D=/^(.+?)(ational|tional|enci|anci|izer|bli|alli|entli|eli|ousli|ization|ation|ator|alism|iveness|fulness|ousness|aliti|iviti|biliti|logi)$/,N=/^(.+?)(icate|ative|alize|iciti|ical|ful|ness)$/,M=/^(.+?)(al|ance|ence|er|ic|able|ible|ant|ement|ment|ent|ou|ism|ate|iti|ous|ive|ize)$/,V=/^(.+?)(s|t)(ion)$/,P=/^(.+?)e$/,z=/ll$/,$=new RegExp("^"+s+i+"[^aeiouwxy]$"),A=function(h){var S,k,L,p,T,O,F;if(h.length<3)return h;if(L=h.substr(0,1),L=="y"&&(h=L.toUpperCase()+h.substr(1)),p=v,T=d,p.test(h)?h=h.replace(p,"$1$2"):T.test(h)&&(h=h.replace(T,"$1$2")),p=y,T=b,p.test(h)){var Q=p.exec(h);p=g,p.test(Q[1])&&(p=E,h=h.replace(p,""))}else if(T.test(h)){var Q=T.exec(h);S=Q[1],T=x,T.test(S)&&(h=S,T=w,O=R,F=j,T.test(h)?h=h+"e":O.test(h)?(p=E,h=h.replace(p,"")):F.test(h)&&(h=h+"e"))}if(p=C,p.test(h)){var Q=p.exec(h);S=Q[1],h=S+"i"}if(p=D,p.test(h)){var Q=p.exec(h);S=Q[1],k=Q[2],p=g,p.test(S)&&(h=S+e[k])}if(p=N,p.test(h)){var Q=p.exec(h);S=Q[1],k=Q[2],p=g,p.test(S)&&(h=S+r[k])}if(p=M,T=V,p.test(h)){var Q=p.exec(h);S=Q[1],p=l,p.test(S)&&(h=S)}else if(T.test(h)){var Q=T.exec(h);S=Q[1]+Q[2],T=l,T.test(S)&&(h=S)}if(p=P,p.test(h)){var Q=p.exec(h);S=Q[1],p=l,T=m,O=$,(p.test(S)||T.test(S)&&!O.test(S))&&(h=S)}return p=z,T=l,p.test(h)&&T.test(h)&&(p=E,h=h.replace(p,"")),L=="y"&&(h=L.toLowerCase()+h.substr(1)),h};return function(I){return I.update(A)}})(),t.Pipeline.registerFunction(t.stemmer,"stemmer");/*!
|
||||
* lunr.stopWordFilter
|
||||
* Copyright (C) 2020 Oliver Nightingale
|
||||
*/t.generateStopWordFilter=function(e){var r=e.reduce(function(n,i){return n[i]=i,n},{});return function(n){if(n&&r[n.toString()]!==n.toString())return n}},t.stopWordFilter=t.generateStopWordFilter(["a","able","about","across","after","all","almost","also","am","among","an","and","any","are","as","at","be","because","been","but","by","can","cannot","could","dear","did","do","does","either","else","ever","every","for","from","get","got","had","has","have","he","her","hers","him","his","how","however","i","if","in","into","is","it","its","just","least","let","like","likely","may","me","might","most","must","my","neither","no","nor","not","of","off","often","on","only","or","other","our","own","rather","said","say","says","she","should","since","so","some","than","that","the","their","them","then","there","these","they","this","tis","to","too","twas","us","wants","was","we","were","what","when","where","which","while","who","whom","why","will","with","would","yet","you","your"]),t.Pipeline.registerFunction(t.stopWordFilter,"stopWordFilter");/*!
|
||||
* lunr.trimmer
|
||||
* Copyright (C) 2020 Oliver Nightingale
|
||||
*/t.trimmer=function(e){return e.update(function(r){return r.replace(/^\W+/,"").replace(/\W+$/,"")})},t.Pipeline.registerFunction(t.trimmer,"trimmer");/*!
|
||||
* lunr.TokenSet
|
||||
* Copyright (C) 2020 Oliver Nightingale
|
||||
*/t.TokenSet=function(){this.final=!1,this.edges={},this.id=t.TokenSet._nextId,t.TokenSet._nextId+=1},t.TokenSet._nextId=1,t.TokenSet.fromArray=function(e){for(var r=new t.TokenSet.Builder,n=0,i=e.length;n<i;n++)r.insert(e[n]);return r.finish(),r.root},t.TokenSet.fromClause=function(e){return"editDistance"in e?t.TokenSet.fromFuzzyString(e.term,e.editDistance):t.TokenSet.fromString(e.term)},t.TokenSet.fromFuzzyString=function(e,r){for(var n=new t.TokenSet,i=[{node:n,editsRemaining:r,str:e}];i.length;){var s=i.pop();if(s.str.length>0){var o=s.str.charAt(0),a;o in s.node.edges?a=s.node.edges[o]:(a=new t.TokenSet,s.node.edges[o]=a),s.str.length==1&&(a.final=!0),i.push({node:a,editsRemaining:s.editsRemaining,str:s.str.slice(1)})}if(s.editsRemaining!=0){if("*"in s.node.edges)var u=s.node.edges["*"];else{var u=new t.TokenSet;s.node.edges["*"]=u}if(s.str.length==0&&(u.final=!0),i.push({node:u,editsRemaining:s.editsRemaining-1,str:s.str}),s.str.length>1&&i.push({node:s.node,editsRemaining:s.editsRemaining-1,str:s.str.slice(1)}),s.str.length==1&&(s.node.final=!0),s.str.length>=1){if("*"in s.node.edges)var c=s.node.edges["*"];else{var c=new t.TokenSet;s.node.edges["*"]=c}s.str.length==1&&(c.final=!0),i.push({node:c,editsRemaining:s.editsRemaining-1,str:s.str.slice(1)})}if(s.str.length>1){var f=s.str.charAt(0),g=s.str.charAt(1),l;g in s.node.edges?l=s.node.edges[g]:(l=new t.TokenSet,s.node.edges[g]=l),s.str.length==1&&(l.final=!0),i.push({node:l,editsRemaining:s.editsRemaining-1,str:f+s.str.slice(2)})}}}return n},t.TokenSet.fromString=function(e){for(var r=new t.TokenSet,n=r,i=0,s=e.length;i<s;i++){var o=e[i],a=i==s-1;if(o=="*")r.edges[o]=r,r.final=a;else{var u=new t.TokenSet;u.final=a,r.edges[o]=u,r=u}}return n},t.TokenSet.prototype.toArray=function(){for(var e=[],r=[{prefix:"",node:this}];r.length;){var n=r.pop(),i=Object.keys(n.node.edges),s=i.length;n.node.final&&(n.prefix.charAt(0),e.push(n.prefix));for(var o=0;o<s;o++){var a=i[o];r.push({prefix:n.prefix.concat(a),node:n.node.edges[a]})}}return e},t.TokenSet.prototype.toString=function(){if(this._str)return this._str;for(var e=this.final?"1":"0",r=Object.keys(this.edges).sort(),n=r.length,i=0;i<n;i++){var s=r[i],o=this.edges[s];e=e+s+o.id}return e},t.TokenSet.prototype.intersect=function(e){for(var r=new t.TokenSet,n=void 0,i=[{qNode:e,output:r,node:this}];i.length;){n=i.pop();for(var s=Object.keys(n.qNode.edges),o=s.length,a=Object.keys(n.node.edges),u=a.length,c=0;c<o;c++)for(var f=s[c],g=0;g<u;g++){var l=a[g];if(l==f||f=="*"){var m=n.node.edges[l],x=n.qNode.edges[f],v=m.final&&x.final,d=void 0;l in n.output.edges?(d=n.output.edges[l],d.final=d.final||v):(d=new t.TokenSet,d.final=v,n.output.edges[l]=d),i.push({qNode:x,output:d,node:m})}}}return r},t.TokenSet.Builder=function(){this.previousWord="",this.root=new t.TokenSet,this.uncheckedNodes=[],this.minimizedNodes={}},t.TokenSet.Builder.prototype.insert=function(e){var r,n=0;if(e<this.previousWord)throw new Error("Out of order word insertion");for(var i=0;i<e.length&&i<this.previousWord.length&&e[i]==this.previousWord[i];i++)n++;this.minimize(n),this.uncheckedNodes.length==0?r=this.root:r=this.uncheckedNodes[this.uncheckedNodes.length-1].child;for(var i=n;i<e.length;i++){var s=new t.TokenSet,o=e[i];r.edges[o]=s,this.uncheckedNodes.push({parent:r,char:o,child:s}),r=s}r.final=!0,this.previousWord=e},t.TokenSet.Builder.prototype.finish=function(){this.minimize(0)},t.TokenSet.Builder.prototype.minimize=function(e){for(var r=this.uncheckedNodes.length-1;r>=e;r--){var n=this.uncheckedNodes[r],i=n.child.toString();i in this.minimizedNodes?n.parent.edges[n.char]=this.minimizedNodes[i]:(n.child._str=i,this.minimizedNodes[i]=n.child),this.uncheckedNodes.pop()}};/*!
|
||||
* lunr.Index
|
||||
* Copyright (C) 2020 Oliver Nightingale
|
||||
*/t.Index=function(e){this.invertedIndex=e.invertedIndex,this.fieldVectors=e.fieldVectors,this.tokenSet=e.tokenSet,this.fields=e.fields,this.pipeline=e.pipeline},t.Index.prototype.search=function(e){return this.query(function(r){var n=new t.QueryParser(e,r);n.parse()})},t.Index.prototype.query=function(e){for(var r=new t.Query(this.fields),n=Object.create(null),i=Object.create(null),s=Object.create(null),o=Object.create(null),a=Object.create(null),u=0;u<this.fields.length;u++)i[this.fields[u]]=new t.Vector;e.call(r,r);for(var u=0;u<r.clauses.length;u++){var c=r.clauses[u],f=null,g=t.Set.empty;c.usePipeline?f=this.pipeline.runString(c.term,{fields:c.fields}):f=[c.term];for(var l=0;l<f.length;l++){var m=f[l];c.term=m;var x=t.TokenSet.fromClause(c),v=this.tokenSet.intersect(x).toArray();if(v.length===0&&c.presence===t.Query.presence.REQUIRED){for(var d=0;d<c.fields.length;d++){var y=c.fields[d];o[y]=t.Set.empty}break}for(var b=0;b<v.length;b++)for(var E=v[b],w=this.invertedIndex[E],R=w._index,d=0;d<c.fields.length;d++){var y=c.fields[d],j=w[y],C=Object.keys(j),D=E+"/"+y,N=new t.Set(C);if(c.presence==t.Query.presence.REQUIRED&&(g=g.union(N),o[y]===void 0&&(o[y]=t.Set.complete)),c.presence==t.Query.presence.PROHIBITED){a[y]===void 0&&(a[y]=t.Set.empty),a[y]=a[y].union(N);continue}if(i[y].upsert(R,c.boost,function(ye,me){return ye+me}),!s[D]){for(var M=0;M<C.length;M++){var V=C[M],P=new t.FieldRef(V,y),z=j[V],$;($=n[P])===void 0?n[P]=new t.MatchData(E,y,z):$.add(E,y,z)}s[D]=!0}}}if(c.presence===t.Query.presence.REQUIRED)for(var d=0;d<c.fields.length;d++){var y=c.fields[d];o[y]=o[y].intersect(g)}}for(var A=t.Set.complete,I=t.Set.empty,u=0;u<this.fields.length;u++){var y=this.fields[u];o[y]&&(A=A.intersect(o[y])),a[y]&&(I=I.union(a[y]))}var h=Object.keys(n),S=[],k=Object.create(null);if(r.isNegated()){h=Object.keys(this.fieldVectors);for(var u=0;u<h.length;u++){var P=h[u],L=t.FieldRef.fromString(P);n[P]=new t.MatchData}}for(var u=0;u<h.length;u++){var L=t.FieldRef.fromString(h[u]),p=L.docRef;if(A.contains(p)&&!I.contains(p)){var T=this.fieldVectors[L],O=i[L.fieldName].similarity(T),F;if((F=k[p])!==void 0)F.score+=O,F.matchData.combine(n[L]);else{var Q={ref:p,score:O,matchData:n[L]};k[p]=Q,S.push(Q)}}}return S.sort(function(pe,ge){return ge.score-pe.score})},t.Index.prototype.toJSON=function(){var e=Object.keys(this.invertedIndex).sort().map(function(n){return[n,this.invertedIndex[n]]},this),r=Object.keys(this.fieldVectors).map(function(n){return[n,this.fieldVectors[n].toJSON()]},this);return{version:t.version,fields:this.fields,fieldVectors:r,invertedIndex:e,pipeline:this.pipeline.toJSON()}},t.Index.load=function(e){var r={},n={},i=e.fieldVectors,s=Object.create(null),o=e.invertedIndex,a=new t.TokenSet.Builder,u=t.Pipeline.load(e.pipeline);e.version!=t.version&&t.utils.warn("Version mismatch when loading serialised index. Current version of lunr '"+t.version+"' does not match serialized index '"+e.version+"'");for(var c=0;c<i.length;c++){var f=i[c],g=f[0],l=f[1];n[g]=new t.Vector(l)}for(var c=0;c<o.length;c++){var f=o[c],m=f[0],x=f[1];a.insert(m),s[m]=x}return a.finish(),r.fields=e.fields,r.fieldVectors=n,r.invertedIndex=s,r.tokenSet=a.root,r.pipeline=u,new t.Index(r)};/*!
|
||||
* lunr.Builder
|
||||
* Copyright (C) 2020 Oliver Nightingale
|
||||
*/t.Builder=function(){this._ref="id",this._fields=Object.create(null),this._documents=Object.create(null),this.invertedIndex=Object.create(null),this.fieldTermFrequencies={},this.fieldLengths={},this.tokenizer=t.tokenizer,this.pipeline=new t.Pipeline,this.searchPipeline=new t.Pipeline,this.documentCount=0,this._b=.75,this._k1=1.2,this.termIndex=0,this.metadataWhitelist=[]},t.Builder.prototype.ref=function(e){this._ref=e},t.Builder.prototype.field=function(e,r){if(/\//.test(e))throw new RangeError("Field '"+e+"' contains illegal character '/'");this._fields[e]=r||{}},t.Builder.prototype.b=function(e){e<0?this._b=0:e>1?this._b=1:this._b=e},t.Builder.prototype.k1=function(e){this._k1=e},t.Builder.prototype.add=function(e,r){var n=e[this._ref],i=Object.keys(this._fields);this._documents[n]=r||{},this.documentCount+=1;for(var s=0;s<i.length;s++){var o=i[s],a=this._fields[o].extractor,u=a?a(e):e[o],c=this.tokenizer(u,{fields:[o]}),f=this.pipeline.run(c),g=new t.FieldRef(n,o),l=Object.create(null);this.fieldTermFrequencies[g]=l,this.fieldLengths[g]=0,this.fieldLengths[g]+=f.length;for(var m=0;m<f.length;m++){var x=f[m];if(l[x]==null&&(l[x]=0),l[x]+=1,this.invertedIndex[x]==null){var v=Object.create(null);v._index=this.termIndex,this.termIndex+=1;for(var d=0;d<i.length;d++)v[i[d]]=Object.create(null);this.invertedIndex[x]=v}this.invertedIndex[x][o][n]==null&&(this.invertedIndex[x][o][n]=Object.create(null));for(var y=0;y<this.metadataWhitelist.length;y++){var b=this.metadataWhitelist[y],E=x.metadata[b];this.invertedIndex[x][o][n][b]==null&&(this.invertedIndex[x][o][n][b]=[]),this.invertedIndex[x][o][n][b].push(E)}}}},t.Builder.prototype.calculateAverageFieldLengths=function(){for(var e=Object.keys(this.fieldLengths),r=e.length,n={},i={},s=0;s<r;s++){var o=t.FieldRef.fromString(e[s]),a=o.fieldName;i[a]||(i[a]=0),i[a]+=1,n[a]||(n[a]=0),n[a]+=this.fieldLengths[o]}for(var u=Object.keys(this._fields),s=0;s<u.length;s++){var c=u[s];n[c]=n[c]/i[c]}this.averageFieldLength=n},t.Builder.prototype.createFieldVectors=function(){for(var e={},r=Object.keys(this.fieldTermFrequencies),n=r.length,i=Object.create(null),s=0;s<n;s++){for(var o=t.FieldRef.fromString(r[s]),a=o.fieldName,u=this.fieldLengths[o],c=new t.Vector,f=this.fieldTermFrequencies[o],g=Object.keys(f),l=g.length,m=this._fields[a].boost||1,x=this._documents[o.docRef].boost||1,v=0;v<l;v++){var d=g[v],y=f[d],b=this.invertedIndex[d]._index,E,w,R;i[d]===void 0?(E=t.idf(this.invertedIndex[d],this.documentCount),i[d]=E):E=i[d],w=E*((this._k1+1)*y)/(this._k1*(1-this._b+this._b*(u/this.averageFieldLength[a]))+y),w*=m,w*=x,R=Math.round(w*1e3)/1e3,c.insert(b,R)}e[o]=c}this.fieldVectors=e},t.Builder.prototype.createTokenSet=function(){this.tokenSet=t.TokenSet.fromArray(Object.keys(this.invertedIndex).sort())},t.Builder.prototype.build=function(){return this.calculateAverageFieldLengths(),this.createFieldVectors(),this.createTokenSet(),new t.Index({invertedIndex:this.invertedIndex,fieldVectors:this.fieldVectors,tokenSet:this.tokenSet,fields:Object.keys(this._fields),pipeline:this.searchPipeline})},t.Builder.prototype.use=function(e){var r=Array.prototype.slice.call(arguments,1);r.unshift(this),e.apply(this,r)},t.MatchData=function(e,r,n){for(var i=Object.create(null),s=Object.keys(n||{}),o=0;o<s.length;o++){var a=s[o];i[a]=n[a].slice()}this.metadata=Object.create(null),e!==void 0&&(this.metadata[e]=Object.create(null),this.metadata[e][r]=i)},t.MatchData.prototype.combine=function(e){for(var r=Object.keys(e.metadata),n=0;n<r.length;n++){var i=r[n],s=Object.keys(e.metadata[i]);this.metadata[i]==null&&(this.metadata[i]=Object.create(null));for(var o=0;o<s.length;o++){var a=s[o],u=Object.keys(e.metadata[i][a]);this.metadata[i][a]==null&&(this.metadata[i][a]=Object.create(null));for(var c=0;c<u.length;c++){var f=u[c];this.metadata[i][a][f]==null?this.metadata[i][a][f]=e.metadata[i][a][f]:this.metadata[i][a][f]=this.metadata[i][a][f].concat(e.metadata[i][a][f])}}}},t.MatchData.prototype.add=function(e,r,n){if(!(e in this.metadata)){this.metadata[e]=Object.create(null),this.metadata[e][r]=n;return}if(!(r in this.metadata[e])){this.metadata[e][r]=n;return}for(var i=Object.keys(n),s=0;s<i.length;s++){var o=i[s];o in this.metadata[e][r]?this.metadata[e][r][o]=this.metadata[e][r][o].concat(n[o]):this.metadata[e][r][o]=n[o]}},t.Query=function(e){this.clauses=[],this.allFields=e},t.Query.wildcard=new String("*"),t.Query.wildcard.NONE=0,t.Query.wildcard.LEADING=1,t.Query.wildcard.TRAILING=2,t.Query.presence={OPTIONAL:1,REQUIRED:2,PROHIBITED:3},t.Query.prototype.clause=function(e){return"fields"in e||(e.fields=this.allFields),"boost"in e||(e.boost=1),"usePipeline"in e||(e.usePipeline=!0),"wildcard"in e||(e.wildcard=t.Query.wildcard.NONE),e.wildcard&t.Query.wildcard.LEADING&&e.term.charAt(0)!=t.Query.wildcard&&(e.term="*"+e.term),e.wildcard&t.Query.wildcard.TRAILING&&e.term.slice(-1)!=t.Query.wildcard&&(e.term=""+e.term+"*"),"presence"in e||(e.presence=t.Query.presence.OPTIONAL),this.clauses.
Line truncated
|
||||
//# sourceMappingURL=search.2c215733.min.js.map
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
{
|
||||
"version": 3,
|
||||
"sources": ["node_modules/lunr/lunr.js", "src/templates/assets/javascripts/integrations/search/worker/main/index.ts", "src/templates/assets/javascripts/browser/element/_/index.ts", "src/templates/assets/javascripts/polyfills/index.ts", "src/templates/assets/javascripts/integrations/search/config/index.ts", "src/templates/assets/javascripts/integrations/search/internal/_/index.ts", "src/templates/assets/javascripts/integrations/search/internal/extract/index.ts", "src/templates/assets/javascripts/integrations/search/internal/highlight/index.ts", "src/templates/assets/javascripts/integrations/search/internal/tokenize/index.ts", "src/templates/assets/javascripts/integrations/search/query/transform/index.ts", "src/templates/assets/javascripts/integrations/search/query/_/index.ts", "src/templates/assets/javascripts/integrations/search/query/segment/index.ts", "src/templates/assets/javascripts/integrations/search/_/index.ts"],
|
||||
"sourcesContent": ["/**\n * lunr - http://lunrjs.com - A bit like Solr, but much smaller and not as bright - 2.3.9\n * Copyright (C) 2020 Oliver Nightingale\n * @license MIT\n */\n\n;(function(){\n\n/**\n * A convenience function for configuring and constructing\n * a new lunr Index.\n *\n * A lunr.Builder instance is created and the pipeline setup\n * with a trimmer, stop word filter and stemmer.\n *\n * This builder object is yielded to the configuration function\n * that is passed as a parameter, allowing the list of fields\n * and other builder parameters to be customised.\n *\n * All documents _must_ be added within the passed config function.\n *\n * @example\n * var idx = lunr(function () {\n * this.field('title')\n * this.field('body')\n * this.ref('id')\n *\n * documents.forEach(function (doc) {\n * this.add(doc)\n * }, this)\n * })\n *\n * @see {@link lunr.Builder}\n * @see {@link lunr.Pipeline}\n * @see {@link lunr.trimmer}\n * @see {@link lunr.stopWordFilter}\n * @see {@link lunr.stemmer}\n * @namespace {function} lunr\n */\nvar lunr = function (config) {\n var builder = new lunr.Builder\n\n builder.pipeline.add(\n lunr.trimmer,\n lunr.stopWordFilter,\n lunr.stemmer\n )\n\n builder.searchPipeline.add(\n lunr.stemmer\n )\n\n config.call(builder, builder)\n return builder.build()\n}\n\nlunr.version = \"2.3.9\"\n/*!\n * lunr.utils\n * Copyright (C) 2020 Oliver Nightingale\n */\n\n/**\n * A namespace containing utils for the rest of the lunr library\n * @namespace lunr.utils\n */\nlunr.utils = {}\n\n/**\n * Print a warning message to the console.\n *\n * @param {String} message The message to be printed.\n * @memberOf lunr.utils\n * @function\n */\nlunr.utils.warn = (function (global) {\n /* eslint-disable no-console */\n return function (message) {\n if (global.console && console.warn) {\n console.warn(message)\n }\n }\n /* eslint-enable no-console */\n})(this)\n\n/**\n * Convert an object to a string.\n *\n * In the case of `null` and `undefined` the function returns\n * the empty string, in all other cases the result of calling\n * `toString` on the passed object is returned.\n *\n * @param {Any} obj The object to convert to a string.\n * @return {String} string representation of the passed object.\n * @memberOf lunr.utils\n */\nlunr.utils.asString = function (obj) {\n if (obj === void 0 || obj === null) {\n return \"\"\n } else {\n return obj.toString()\n }\n}\n\n/**\n * Clones an object.\n *\n * Will create a copy of an existing object such that any mutations\n * on the copy cannot affect the original.\n *\n * Only shallow objects are supported, passing a nested object to this\n * function will cause a TypeError.\n *\n * Objects with primitives, and arrays of primitives are supported.\n *\n * @param {Object} obj The object to clone.\n * @return {Object} a clone of the passed object.\n * @throws {TypeError} when a nested object is passed.\n * @memberOf Utils\n */\nlunr.utils.clone = function (obj) {\n if (obj === null || obj === undefined) {\n return obj\n }\n\n var clone = Object.create(null),\n keys = Object.keys(obj)\n\n for (var i = 0; i < keys.length; i++) {\n var key = keys[i],\n val = obj[key]\n\n if (Array.isArray(val)) {\n clone[key] = val.slice()\n continue\n }\n\n if (typeof val === 'string' ||\n typeof val === 'number' ||\n typeof val === 'boolean') {\n clone[key] = val\n continue\n }\n\n throw new TypeError(\"clone is not deep and does not support nested objects\")\n }\n\n return clone\n}\nlunr.FieldRef = function (docRef, fieldName, stringValue) {\n this.docRef = docRef\n this.fieldName = fieldName\n this._stringValue = stringValue\n}\n\nlunr.FieldRef.joiner = \"/\"\n\nlunr.FieldRef.fromString = function (s) {\n var n = s.indexOf(lunr.FieldRef.joiner)\n\n if (n === -1) {\n throw \"malformed field ref string\"\n }\n\n var fieldRef = s.slice(0, n),\n docRef = s.slice(n + 1)\n\n return new lunr.FieldRef (docRef, fieldRef, s)\n}\n\nlunr.FieldRef.prototype.toString = function () {\n if (this._stringValue == undefined) {\n this._stringValue = this.fieldName + lunr.FieldRef.joiner + this.docRef\n }\n\n return this._stringValue\n}\n/*!\n * lunr.Set\n * Copyright (C) 2020 Oliver Nightingale\n */\n\n/**\n * A lunr set.\n *\n * @constructor\n */\nlunr.Set = function (elements) {\n this.elements = Object.create(null)\n\n if (elements) {\n this.length = elements.length\n\n for (var i = 0; i < this.length; i++) {\n this.elements[elements[i]] = true\n }\n } else {\n this.length = 0\n }\n}\n\n/**\n * A complete set that contains all elements.\n *\n * @static\n * @readonly\n * @type {lunr.Set}\n */\nlunr.Set.complete = {\n intersect: function (other) {\n return other\n },\n\n union: function () {\n return this\n },\n\n contains: function () {\n return true\n }\n}\n\n/**\n * An empty set that contains no elementLine truncated
|
||||
"mappings": "6lCAAA,IAAAA,GAAAC,GAAA,CAAAC,GAAAC,KAAA;AAAA;AAAA;AAAA;AAAA,IAME,UAAU,CAiCZ,IAAIC,EAAO,SAAUC,EAAQ,CAC3B,IAAIC,EAAU,IAAIF,EAAK,QAEvB,OAAAE,EAAQ,SAAS,IACfF,EAAK,QACLA,EAAK,eACLA,EAAK,OACP,EAEAE,EAAQ,eAAe,IACrBF,EAAK,OACP,EAEAC,EAAO,KAAKC,EAASA,CAAO,EACrBA,EAAQ,MAAM,CACvB,EAEAF,EAAK,QAAU,QACf;AAAA;AAAA;AAAA,GASAA,EAAK,MAAQ,CAAC,EASdA,EAAK,MAAM,MAAQ,SAAUG,EAAQ,CAEnC,OAAO,SAAUC,EAAS,CACpBD,EAAO,SAAW,QAAQ,MAC5B,QAAQ,KAAKC,CAAO,CAExB,CAEF,GAAG,IAAI,EAaPJ,EAAK,MAAM,SAAW,SAAUK,EAAK,CACnC,OAAsBA,GAAQ,KACrB,GAEAA,EAAI,SAAS,CAExB,EAkBAL,EAAK,MAAM,MAAQ,SAAUK,EAAK,CAChC,GAAIA,GAAQ,KACV,OAAOA,EAMT,QAHIC,EAAQ,OAAO,OAAO,IAAI,EAC1BC,EAAO,OAAO,KAAKF,CAAG,EAEjB,EAAI,EAAG,EAAIE,EAAK,OAAQ,IAAK,CACpC,IAAIC,EAAMD,EAAK,CAAC,EACZE,EAAMJ,EAAIG,CAAG,EAEjB,GAAI,MAAM,QAAQC,CAAG,EAAG,CACtBH,EAAME,CAAG,EAAIC,EAAI,MAAM,EACvB,QACF,CAEA,GAAI,OAAOA,GAAQ,UACf,OAAOA,GAAQ,UACf,OAAOA,GAAQ,UAAW,CAC5BH,EAAME,CAAG,EAAIC,EACb,QACF,CAEA,MAAM,IAAI,UAAU,uDAAuD,CAC7E,CAEA,OAAOH,CACT,EACAN,EAAK,SAAW,SAAUU,EAAQC,EAAWC,EAAa,CACxD,KAAK,OAASF,EACd,KAAK,UAAYC,EACjB,KAAK,aAAeC,CACtB,EAEAZ,EAAK,SAAS,OAAS,IAEvBA,EAAK,SAAS,WAAa,SAAUa,EAAG,CACtC,IAAIC,EAAID,EAAE,QAAQb,EAAK,SAAS,MAAM,EAEtC,GAAIc,IAAM,GACR,KAAM,6BAGR,IAAIC,EAAWF,EAAE,MAAM,EAAGC,CAAC,EACvBJ,EAASG,EAAE,MAAMC,EAAI,CAAC,EAE1B,OAAO,IAAId,EAAK,SAAUU,EAAQK,EAAUF,CAAC,CAC/C,EAEAb,EAAK,SAAS,UAAU,SAAW,UAAY,CAC7C,OAAI,KAAK,cAAgB,OACvB,KAAK,aAAe,KAAK,UAAYA,EAAK,SAAS,OAAS,KAAK,QAG5D,KAAK,YACd,EACA;AAAA;AAAA;AAAA,GAUAA,EAAK,IAAM,SAAUgB,EAAU,CAG7B,GAFA,KAAK,SAAW,OAAO,OAAO,IAAI,EAE9BA,EAAU,CACZ,KAAK,OAASA,EAAS,OAEvB,QAASC,EAAI,EAAGA,EAAI,KAAK,OAAQA,IAC/B,KAAK,SAASD,EAASC,CAAC,CAAC,EAAI,EAEjC,MACE,KAAK,OAAS,CAElB,EASAjB,EAAK,IAAI,SAAW,CAClB,UAAW,SAAUkB,EAAO,CAC1B,OAAOA,CACT,EAEA,MAAO,UAAY,CACjB,OAAO,IACT,EAEA,SAAU,UAAY,CACpB,MAAO,EACT,CACF,EASAlB,EAAK,IAAI,MAAQ,CACf,UAAW,UAAY,CACrB,OAAO,IACT,EAEA,MAAO,SAAUkB,EAAO,CACtB,OAAOA,CACT,EAEA,SAAU,UAAY,CACpB,MAAO,EACT,CACF,EAQAlB,EAAK,IAAI,UAAU,SAAW,SAAUmB,EAAQ,CAC9C,MAAO,CAAC,CAAC,KAAK,SAASA,CAAM,CAC/B,EAUAnB,EAAK,IAAI,UAAU,UAAY,SAAUkB,EAAO,CAC9C,IAAIE,EAAGC,EAAGL,EAAUM,EAAe,CAAC,EAEpC,GAAIJ,IAAUlB,EAAK,IAAI,SACrB,OAAO,KAGT,GAAIkB,IAAUlB,EAAK,IAAI,MACrB,OAAOkB,EAGL,KAAK,OAASA,EAAM,QACtBE,EAAI,KACJC,EAAIH,IAEJE,EAAIF,EACJG,EAAI,MAGNL,EAAW,OAAO,KAAKI,EAAE,QAAQ,EAEjC,QAASH,EAAI,EAAGA,EAAID,EAAS,OAAQC,IAAK,CACxC,IAAIM,EAAUP,EAASC,CAAC,EACpBM,KAAWF,EAAE,UACfC,EAAa,KAAKC,CAAO,CAE7B,CAEA,OAAO,IAAIvB,EAAK,IAAKsB,CAAY,CACnC,EASAtB,EAAK,IAAI,UAAU,MAAQ,SAAUkB,EAAO,CAC1C,OAAIA,IAAUlB,EAAK,IAAI,SACdA,EAAK,IAAI,SAGdkB,IAAUlB,EAAK,IAAI,MACd,KAGF,IAAIA,EAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,OAAO,OAAO,KAAKkB,EAAM,QAAQ,CAAC,CAAC,CACpF,EASAlB,EAAK,IAAM,SAAUwB,EAASC,EAAe,CAC3C,IAAIC,EAAoB,EAExB,QAASf,KAAaa,EAChBb,GAAa,WACjBe,GAAqB,OAAO,KAAKF,EAAQb,CAAS,CAAC,EAAE,QAGvD,IAAIgB,GAAKF,EAAgBC,EAAoB,KAAQA,EAAoB,IAEzE,OAAO,KAAK,IAAI,EAAI,KAAK,IAAIC,CAAC,CAAC,CACjC,EAUA3B,EAAK,MAAQ,SAAU4B,EAAKC,EAAU,CACpC,KAAK,IAAMD,GAAO,GAClB,KAAK,SAAWC,GAAY,CAAC,CAC/B,EAOA7B,EAAK,MAAM,UAAU,SAAW,UAAY,CAC1C,OAAO,KAAK,GACd,EAsBAA,EAAK,MAAM,UAAU,OAAS,SAAU8B,EAAI,CAC1C,YAAK,IAAMA,EAAG,KAAK,IAAK,KAAK,QAAQ,EAC9B,IACT,EASA9B,EAAK,MAAM,UAAU,MAAQ,SAAU8B,EAAI,CACzC,OAAAA,EAAKA,GAAM,SAAUjB,EAAG,CAAE,OAAOA,CAAE,EAC5B,IAAIb,EAAK,MAAO8B,EAAG,KAAK,IAAK,KAAK,QAAQ,EAAG,KAAK,QAAQ,CACnE,EACA;AAAA;AAAA;AAAA,GAuBA9B,EAAK,UAAY,SAAUK,EAAKwB,EAAU,CACxC,GAAIxB,GAAO,MAAQA,GAAO,KACxB,MAAO,CAAC,EAGV,GAAI,MAAM,QAAQA,CAAG,EACnB,OAAOA,EAAI,IAAI,SAAU0B,EAAG,CAC1B,OAAO,IAAI/B,EAAK,MACdA,EAAK,MAAM,SAAS+B,CAAC,EAAE,YAAY,EACnC/B,EAAK,MAAM,MAAM6B,CAAQ,CAC3B,CACF,CAAC,EAOH,QAJID,EAAMvB,EAAI,SAAS,EAAE,YAAY,EACjC2B,EAAMJ,EAAI,OACVK,EAAS,CAAC,EAELC,EAAW,EAAGC,EAAa,EAAGD,GAAYF,EAAKE,IAAY,CAClE,IAAIE,EAAOR,EAAI,OAAOM,CAAQ,EAC1BG,EAAcH,EAAWC,EAE7B,GAAKC,EAAK,MAAMpC,EAAK,UAAU,SAAS,GAAKkC,GAAYF,EAAM,CAE7D,GAAIK,EAAc,EAAG,CACnB,IAAIC,EAAgBtC,EAAK,MAAM,MAAM6B,CAAQ,GAAK,CAAC,EACnDS,EAAc,SAAc,CAACH,EAAYE,CAAW,EACpDC,EAAc,MAAWL,EAAO,OAEhCA,EAAO,KACL,IAAIjC,EAAK,MACP4B,EAAI,MAAMO,EAAYD,CAAQ,EAC9BI,CACF,CACF,CACF,CAEAH,EAAaD,EAAW,CAC1B,CAEF,CAEA,OAAOD,CACT,EASAjC,EAAK,UAAU,UAAY,UAC3B;AAAA;AAAA;AAAA,GAkCAA,EAAK,SAAW,UAAY,CAC1B,KAAK,OAAS,CAAC,CACjB,EAEAA,EAAK,SAAS,oBAAsB,OAAO,OAAO,IAAI,EAmCtDA,EAAK,SAAS,iBAAmB,SAAU8B,EAAIS,EAAO,CAChDA,KAAS,KAAK,qBAChBvC,EAAK,MAAM,KAAK,6CAA+CuC,CAAK,EAGtET,EAAG,MAAQS,EACXvC,EAAK,SAAS,oBAAoB8B,EAAG,KAAK,EAAIA,CAChD,EAQA9B,EAAK,SAAS,4BAA8B,SAAU8B,EAAI,CACxD,IAAIU,EAAeV,EAAG,OAAUA,EAAG,SAAS,KAAK,oBAE5CU,GACHxC,EAAK,MAAM,KAAK;AAAA,EAAmG8B,CAAE,CAEzH,EAYA9B,EAAK,SAAS,KAAO,SAAUyC,EAAY,CACzC,IAAIC,EAAW,IAAI1C,EAAK,SAExB,OAAAyC,EAAW,QAAQ,SAAUE,EAAQ,CACnC,IAAIb,EAAK9B,EAAK,SAAS,oBAAoB2C,CAAM,EAEjD,GAAIb,EACFY,EAAS,IAAIZ,CAAE,MAEf,OAAM,IAAI,MAAM,sCAAwCa,CAAM,CAElE,CAAC,EAEMD,CACT,EASA1C,EAAK,SAAS,UAAU,IAAM,UAAY,CACxC,IAAI4C,EAAM,MAAM,UAAU,MAAM,KAAK,SAAS,EAE9CA,EAAI,QAAQ,SAAUd,EAAI,CACxB9B,EAAK,SAAS,4BAA4B8B,CAAE,EAC5C,KAAK,OAAO,KAAKA,CAAE,CACrB,EAAG,IAAI,CACT,EAWA9B,EAAK,SAAS,UAAU,MAAQ,SAAU6C,EAAYC,EAAO,CAC3D9C,EAAK,SAAS,4BAA4B8C,CAAK,EAE/C,IAAIC,EAAM,KAAK,OAAO,QAAQF,CAAU,EACxC,GAAIE,GAAO,GACT,MAAM,IAAI,MAAM,wBAAwB,EAG1CA,EAAMA,EAAM,EALine truncated
|
||||
"names": ["require_lunr", "__commonJSMin", "exports", "module", "lunr", "config", "builder", "global", "message", "obj", "clone", "keys", "key", "val", "docRef", "fieldName", "stringValue", "s", "n", "fieldRef", "elements", "i", "other", "object", "a", "b", "intersection", "element", "posting", "documentCount", "documentsWithTerm", "x", "str", "metadata", "fn", "t", "len", "tokens", "sliceEnd", "sliceStart", "char", "sliceLength", "tokenMetadata", "label", "isRegistered", "serialised", "pipeline", "fnName", "fns", "existingFn", "newFn", "pos", "stackLength", "memo", "j", "result", "k", "token", "index", "start", "end", "pivotPoint", "pivotIndex", "insertIdx", "position", "sumOfSquares", "elementsLength", "otherVector", "dotProduct", "aLen", "bLen", "aVal", "bVal", "output", "step2list", "step3list", "c", "v", "C", "V", "mgr0", "meq1", "mgr1", "s_v", "re_mgr0", "re_mgr1", "re_meq1", "re_s_v", "re_1a", "re2_1a", "re_1b", "re2_1b", "re_1b_2", "re2_1b_2", "re3_1b_2", "re4_1b_2", "re_1c", "re_2", "re_3", "re_4", "re2_4", "re_5", "re_5_1", "re3_5", "porterStemmer", "w", "stem", "suffix", "firstch", "re", "re2", "re3", "re4", "fp", "stopWords", "words", "stopWord", "arr", "clause", "editDistance", "root", "stack", "frame", "noEditNode", "insertionNode", "substitutionNode", "charA", "charB", "transposeNode", "node", "final", "next", "edges", "edge", "labels", "qEdges", "qLen", "nEdges", "nLen", "q", "qEdge", "nEdge", "qNode", "word", "commonPrefix", "nextNode", "downTo", "childKey", "attrs", "queryString", "query", "parser", "matchingFields", "queryVectors", "termFieldCache", "requiredMatches", "prohibitedMatches", "terms", "clauseMatches", "m", "term", "termTokenSet", "expandedTerms", "field", "expandedTerm", "termIndex", "fieldPosting", "matchingDocumentRefs", "termField", "matchingDocumentsSet", "l", "matchingDocumentRef", "matchingFieldRef", "fieldMatch", "allRequiredMatches", "allProhibitedMatches", "matchingFieldRefs", "results", "matches", "fieldVector", "score", "docMatch", "match", "invertedIndex", "fieldVectors", "ref", "serializedIndex", "serializedVectors", "serializedInvertedIndex", "tokenSetBuilder", "tuple", "attributes", "number", "doc", "fields", "extractor", "fieldTerms", "metadataKey", "fieldRefs", "numberOfFields", "accumulator", "documentsWithField", "fieldRefsLength", "termIdfCache", "fieldLength", "termFrequencies", "termsLength", "fieldBoost", "docBoost", "tf", "idf", "scoreWithPrecision", "args", "clonedMetadata", "metadataKeys", "otherMatchData", "allFields", "options", "state", "subSlices", "type", "charCode", "lexer", "lexeme", "completedClause", "errorMessage", "nextLexeme", "possibleFields", "f", "boost", "factory", "import_lunr", "getElement", "selector", "node", "el", "getOptionalElement", "obj", "data", "key", "x", "y", "nodes", "parent", "i", "node", "setupSearchDocumentMap", "docs", "map", "doc", "path", "article", "split", "input", "separator", "fn", "_a", "match", "index", "until", "term", "extract", "input", "fn", "block", "start", "end", "stack", "highlight", "input", "table", "positions", "full", "highlightAll", "inputs", "mapping", "t", "prev", "next", "p", "q", "i", "cursor", "blocks", "a", "b", "index", "block", "group", "slices", "indexes", "start", "end", "length", "slice", "j", "tokenize", "input", "tokens", "inputs", "i", "table", "total", "extract", "block", "type", "start", "end", "_a", "section", "split", "index", "until", "subsection", "segments", "s", "l", "transform", "query", "fn", "term", "parts", "index", "prev", "next", "transformSearchQuery", "query", "transform", "part", "terms", "lexer", "type", "term", "start", "end", "split", "range", "parseSearchQuery", "value", "clause", "getSearchQueryTerms", "_a", "clauses", "result", "t", "segment", "query", "index", "segments", "wordcuts", "i", "j", "stack", "p", "q", "extractor", "table", "name", "doc", "id", "difference", "a", "b", "x", "y", "value", "Search", "config", "docs", "options", "field", "setupSearchDocumentMap", "tokenize", "fns", "lang", "language", "fn", "spec", "__spreadProps", "__spreadValues", "query", "segment", "transformSearchQuery", "clauses", "parseSearchQuery", "clause", "groups", "item", "ref", "score", "matchData", "terms", "getSearchQueryTerms", "positions", "match", "highlightAll", "highlight", "boost", "t", "__pow", "items", "result", "suggest", "titles", "builder", "index", "setupSearchLanguages", "config", "__async", "base", "worker", "getElement", "path", "scripts", "lang", "handler", "message", "Search", "query", "err", "lunr", "ev"]
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
@charset "UTF-8";html{-webkit-text-size-adjust:none;-moz-text-size-adjust:none;text-size-adjust:none;box-sizing:border-box}*,:after,:before{box-sizing:inherit}@media (prefers-reduced-motion){*,:after,:before{transition:none!important}}body{margin:0}a,button,input,label{-webkit-tap-highlight-color:transparent}a{color:inherit;text-decoration:none}hr{border:0;box-sizing:initial;display:block;height:.05rem;overflow:visible;padding:0}small{font-size:80%}sub,sup{line-height:1em}img{border-style:none}table{border-collapse:initial;border-spacing:0}td,th{font-weight:400;vertical-align:top}button{background:#0000;border:0;font-family:inherit;font-size:inherit;margin:0;padding:0}input{border:0;outline:none}:root{--md-primary-fg-color:#4051b5;--md-primary-fg-color--light:#5d6cc0;--md-primary-fg-color--dark:#303fa1;--md-primary-bg-color:#fff;--md-primary-bg-color--light:#ffffffb3;--md-accent-fg-color:#526cfe;--md-accent-fg-color--transparent:#526cfe1a;--md-accent-bg-color:#fff;--md-accent-bg-color--light:#ffffffb3}[data-md-color-scheme=default]{color-scheme:light}[data-md-color-scheme=default] img[src$="#gh-dark-mode-only"],[data-md-color-scheme=default] img[src$="#only-dark"]{display:none}:root,[data-md-color-scheme=default]{--md-hue:225deg;--md-default-fg-color:#000000de;--md-default-fg-color--light:#0000008a;--md-default-fg-color--lighter:#00000052;--md-default-fg-color--lightest:#00000012;--md-default-bg-color:#fff;--md-default-bg-color--light:#ffffffb3;--md-default-bg-color--lighter:#ffffff4d;--md-default-bg-color--lightest:#ffffff1f;--md-code-fg-color:#36464e;--md-code-bg-color:#f5f5f5;--md-code-bg-color--light:#f5f5f5b3;--md-code-bg-color--lighter:#f5f5f54d;--md-code-hl-color:#4287ff;--md-code-hl-color--light:#4287ff1a;--md-code-hl-number-color:#d52a2a;--md-code-hl-special-color:#db1457;--md-code-hl-function-color:#a846b9;--md-code-hl-constant-color:#6e59d9;--md-code-hl-keyword-color:#3f6ec6;--md-code-hl-string-color:#1c7d4d;--md-code-hl-name-color:var(--md-code-fg-color);--md-code-hl-operator-color:var(--md-default-fg-color--light);--md-code-hl-punctuation-color:var(--md-default-fg-color--light);--md-code-hl-comment-color:var(--md-default-fg-color--light);--md-code-hl-generic-color:var(--md-default-fg-color--light);--md-code-hl-variable-color:var(--md-default-fg-color--light);--md-typeset-color:var(--md-default-fg-color);--md-typeset-a-color:var(--md-primary-fg-color);--md-typeset-del-color:#f5503d26;--md-typeset-ins-color:#0bd57026;--md-typeset-kbd-color:#fafafa;--md-typeset-kbd-accent-color:#fff;--md-typeset-kbd-border-color:#b8b8b8;--md-typeset-mark-color:#ffff0080;--md-typeset-table-color:#0000001f;--md-typeset-table-color--light:rgba(0,0,0,.035);--md-admonition-fg-color:var(--md-default-fg-color);--md-admonition-bg-color:var(--md-default-bg-color);--md-warning-fg-color:#000000de;--md-warning-bg-color:#ff9;--md-footer-fg-color:#fff;--md-footer-fg-color--light:#ffffffb3;--md-footer-fg-color--lighter:#ffffff73;--md-footer-bg-color:#000000de;--md-footer-bg-color--dark:#00000052;--md-shadow-z1:0 0.2rem 0.5rem #0000000d,0 0 0.05rem #0000001a;--md-shadow-z2:0 0.2rem 0.5rem #0000001a,0 0 0.05rem #00000040;--md-shadow-z3:0 0.2rem 0.5rem #0003,0 0 0.05rem #00000059}.md-icon svg{fill:currentcolor;display:block;height:1.2rem;width:1.2rem}body{-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;--md-text-font-family:var(--md-text-font,_),-apple-system,BlinkMacSystemFont,Helvetica,Arial,sans-serif;--md-code-font-family:var(--md-code-font,_),SFMono-Regular,Consolas,Menlo,monospace}aside,body,input{font-feature-settings:"kern","liga";color:var(--md-typeset-color);font-family:var(--md-text-font-family)}code,kbd,pre{font-feature-settings:"kern";font-family:var(--md-code-font-family)}:root{--md-typeset-table-sort-icon:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="m18 21-4-4h3V7h-3l4-4 4 4h-3v10h3M2 19v-2h10v2M2 13v-2h7v2M2 7V5h4v2z"/></svg>');--md-typeset-table-sort-icon--asc:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 17h3l-4 4-4-4h3V3h2M2 17h10v2H2M6 5v2H2V5m0 6h7v2H2z"/></svg>');--md-typeset-table-sort-icon--desc:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 7h3l-4-4-4 4h3v14h2M2 17h10v2H2M6 5v2H2V5m0 6h7v2H2z"/></svg>')}.md-typeset{-webkit-print-color-adjust:exact;color-adjust:exact;font-size:.8rem;line-height:1.6;overflow-wrap:break-word}@media print{.md-typeset{font-size:.68rem}}.md-typeset blockquote,.md-typeset dl,.md-typeset figure,.md-typeset ol,.md-typeset pre,.md-typeset ul{margin-bottom:1em;margin-top:1em}.md-typeset h1{color:var(--md-default-fg-color--light);font-size:2em;line-height:1.3;margin:0 0 1.25em}.md-typeset h1,.md-typeset h2{font-weight:300;letter-spacing:-.01em}.md-typeset h2{font-size:1.5625em;line-height:1.4;margin:1.6em 0 .64em}.md-typeset h3{font-size:1.25em;font-weight:400;letter-spacing:-Line truncated
|
||||
@@ -1 +0,0 @@
|
||||
{"version":3,"sources":["src/templates/assets/stylesheets/main/components/_meta.scss","../../../../src/templates/assets/stylesheets/main.scss","src/templates/assets/stylesheets/main/_resets.scss","src/templates/assets/stylesheets/main/_colors.scss","src/templates/assets/stylesheets/main/_icons.scss","src/templates/assets/stylesheets/main/_typeset.scss","src/templates/assets/stylesheets/utilities/_break.scss","src/templates/assets/stylesheets/main/components/_author.scss","src/templates/assets/stylesheets/main/components/_banner.scss","src/templates/assets/stylesheets/main/components/_base.scss","src/templates/assets/stylesheets/main/components/_clipboard.scss","src/templates/assets/stylesheets/main/components/_code.scss","src/templates/assets/stylesheets/main/components/_consent.scss","src/templates/assets/stylesheets/main/components/_content.scss","src/templates/assets/stylesheets/main/components/_dialog.scss","src/templates/assets/stylesheets/main/components/_feedback.scss","src/templates/assets/stylesheets/main/components/_footer.scss","src/templates/assets/stylesheets/main/components/_form.scss","src/templates/assets/stylesheets/main/components/_header.scss","node_modules/material-design-color/material-color.scss","src/templates/assets/stylesheets/main/components/_nav.scss","src/templates/assets/stylesheets/main/components/_pagination.scss","src/templates/assets/stylesheets/main/components/_path.scss","src/templates/assets/stylesheets/main/components/_post.scss","src/templates/assets/stylesheets/main/components/_progress.scss","src/templates/assets/stylesheets/main/components/_search.scss","src/templates/assets/stylesheets/main/components/_select.scss","src/templates/assets/stylesheets/main/components/_sidebar.scss","src/templates/assets/stylesheets/main/components/_source.scss","src/templates/assets/stylesheets/main/components/_status.scss","src/templates/assets/stylesheets/main/components/_tabs.scss","src/templates/assets/stylesheets/main/components/_tag.scss","src/templates/assets/stylesheets/main/components/_tooltip.scss","src/templates/assets/stylesheets/main/components/_tooltip2.scss","src/templates/assets/stylesheets/main/components/_top.scss","src/templates/assets/stylesheets/main/components/_version.scss","src/templates/assets/stylesheets/main/extensions/markdown/_admonition.scss","src/templates/assets/stylesheets/main/extensions/markdown/_footnotes.scss","src/templates/assets/stylesheets/main/extensions/markdown/_toc.scss","src/templates/assets/stylesheets/main/extensions/pymdownx/_arithmatex.scss","src/templates/assets/stylesheets/main/extensions/pymdownx/_critic.scss","src/templates/assets/stylesheets/main/extensions/pymdownx/_details.scss","src/templates/assets/stylesheets/main/extensions/pymdownx/_emoji.scss","src/templates/assets/stylesheets/main/extensions/pymdownx/_highlight.scss","src/templates/assets/stylesheets/main/extensions/pymdownx/_keys.scss","src/templates/assets/stylesheets/main/extensions/pymdownx/_tabbed.scss","src/templates/assets/stylesheets/main/extensions/pymdownx/_tasklist.scss","src/templates/assets/stylesheets/main/integrations/_giscus.scss","src/templates/assets/stylesheets/main/integrations/_mermaid.scss","src/templates/assets/stylesheets/main/modifiers/_grid.scss","src/templates/assets/stylesheets/main/modifiers/_inline.scss"],"names":[],"mappings":"AA0CE,gBCy2CF,CCv3CA,KAEE,6BAAA,CAAA,0BAAA,CAAA,qBAAA,CADA,qBDzBF,CC8BA,iBAGE,kBD3BF,CC8BE,gCANF,iBAOI,yBDzBF,CACF,CC6BA,KACE,QD1BF,CC8BA,qBAIE,uCD3BF,CC+BA,EACE,aAAA,CACA,oBD5BF,CCgCA,GAME,QAAA,CALA,kBAAA,CACA,aAAA,CACA,aAAA,CAEA,gBAAA,CADA,SD3BF,CCiCA,MACE,aD9BF,CCkCA,QAEE,eD/BF,CCmCA,IACE,iBDhCF,CCoCA,MAEE,uBAAA,CADA,gBDhCF,CCqCA,MAEE,eAAA,CACA,kBDlCF,CCsCA,OAKE,gBAAA,CACA,QAAA,CAHA,mBAAA,CACA,iBAAA,CAFA,QAAA,CADA,SD9BF,CCuCA,MACE,QAAA,CACA,YDpCF,CErDA,MAIE,6BAAA,CACA,oCAAA,CACA,mCAAA,CACA,0BAAA,CACA,sCAAA,CAGA,4BAAA,CACA,2CAAA,CACA,yBAAA,CACA,qCFmDF,CE7CA,+BAIE,kBF6CF,CE1CE,oHAEE,YF4CJ,CEnCA,qCAIE,eAAA,CAGA,+BAAA,CACA,sCAAA,CACA,wCAAA,CACA,yCAAA,CACA,0BAAA,CACA,sCAAA,CACA,wCAAA,CACA,yCAAA,CAGA,0BAAA,CACA,0BAAA,CACA,mCAAA,CACA,qCAAA,CAGA,0BAAA,CACA,mCAAA,CAGA,iCAAA,CACA,kCAAA,CACA,mCAAA,CACA,mCAAA,CACA,kCAAA,CACA,iCAAA,CACA,+CAAA,CACA,6DAAA,CACA,gEAAA,CACA,4DAAA,CACA,4DAAA,CACA,6DAAA,CAGA,6CAAA,CAGA,+CAAA,CAGA,gCAAA,CACA,gCAAA,CAGA,8BAAA,CACA,kCAAA,CACA,qCAAA,CAGA,iCAAA,CAGA,kCAAA,CACA,gDAAA,CAGA,mDAAA,CACA,mDAAA,CAGA,+BAAA,CACA,0BAAA,CAGA,yBAAA,CACA,qCAAA,CACA,uCAAA,CACA,8BAAA,CACA,oCAAA,CAGA,8DAAA,CAKA,8DAAA,CAKA,0DFKF,CGhIE,aAIE,iBAAA,CAHA,aAAA,CAEA,aAAA,CADA,YHqIJ,CI1IA,KACE,kCAAA,CACA,iCAAA,CAGA,uGAAA,CAKA,mFJ2IF,CIrIA,iBAIE,mCAAA,CACA,6BAAA,CAFA,sCJ0IF,CIpIA,aAIE,4BAAA,CADA,sCJwIF,CI/HA,MACE,wNAAA,CACA,gNAAA,CACA,iNJkIF,CI3HA,YAIE,gCAAA,CAAA,kBAAA,CAHA,eAAA,CACA,eAAA,CACA,wBJ+HF,CI1HE,aARF,YASI,gBJ6HF,CACF,CI1HE,uGAME,iBAAA,CAAA,cJ4HJ,CIxHE,eAKE,uCAAA,CAHA,aAAA,CAEA,eAAA,CAHA,iBJ+HJ,CItHE,8BAPE,eAAA,CAGA,qBJiIJ,CI7HE,eAEE,kBAAA,CAEA,eAAA,CAHA,oBJ4HJ,CIpHE,eAEE,gBAAA,CACA,eAAA,CAEA,qBAAA,CADA,eAAA,CAHA,mBJ0HJ,CLine truncated
|
||||
@@ -1 +0,0 @@
|
||||
@media screen{[data-md-color-scheme=slate]{--md-default-fg-color:hsla(var(--md-hue),15%,90%,0.82);--md-default-fg-color--light:hsla(var(--md-hue),15%,90%,0.56);--md-default-fg-color--lighter:hsla(var(--md-hue),15%,90%,0.32);--md-default-fg-color--lightest:hsla(var(--md-hue),15%,90%,0.12);--md-default-bg-color:hsla(var(--md-hue),15%,14%,1);--md-default-bg-color--light:hsla(var(--md-hue),15%,14%,0.54);--md-default-bg-color--lighter:hsla(var(--md-hue),15%,14%,0.26);--md-default-bg-color--lightest:hsla(var(--md-hue),15%,14%,0.07);--md-code-fg-color:hsla(var(--md-hue),18%,86%,0.82);--md-code-bg-color:hsla(var(--md-hue),15%,18%,1);--md-code-bg-color--light:hsla(var(--md-hue),15%,18%,0.9);--md-code-bg-color--lighter:hsla(var(--md-hue),15%,18%,0.54);--md-code-hl-color:#2977ff;--md-code-hl-color--light:#2977ff1a;--md-code-hl-number-color:#e6695b;--md-code-hl-special-color:#f06090;--md-code-hl-function-color:#c973d9;--md-code-hl-constant-color:#9383e2;--md-code-hl-keyword-color:#6791e0;--md-code-hl-string-color:#2fb170;--md-code-hl-name-color:var(--md-code-fg-color);--md-code-hl-operator-color:var(--md-default-fg-color--light);--md-code-hl-punctuation-color:var(--md-default-fg-color--light);--md-code-hl-comment-color:var(--md-default-fg-color--light);--md-code-hl-generic-color:var(--md-default-fg-color--light);--md-code-hl-variable-color:var(--md-default-fg-color--light);--md-typeset-color:var(--md-default-fg-color);--md-typeset-a-color:var(--md-primary-fg-color);--md-typeset-kbd-color:hsla(var(--md-hue),15%,90%,0.12);--md-typeset-kbd-accent-color:hsla(var(--md-hue),15%,90%,0.2);--md-typeset-kbd-border-color:hsla(var(--md-hue),15%,14%,1);--md-typeset-mark-color:#4287ff4d;--md-typeset-table-color:hsla(var(--md-hue),15%,95%,0.12);--md-typeset-table-color--light:hsla(var(--md-hue),15%,95%,0.035);--md-admonition-fg-color:var(--md-default-fg-color);--md-admonition-bg-color:var(--md-default-bg-color);--md-footer-bg-color:hsla(var(--md-hue),15%,10%,0.87);--md-footer-bg-color--dark:hsla(var(--md-hue),15%,8%,1);--md-shadow-z1:0 0.2rem 0.5rem #0000000d,0 0 0.05rem #0000001a;--md-shadow-z2:0 0.2rem 0.5rem #00000040,0 0 0.05rem #00000040;--md-shadow-z3:0 0.2rem 0.5rem #0006,0 0 0.05rem #00000059;color-scheme:dark}[data-md-color-scheme=slate] img[src$="#gh-light-mode-only"],[data-md-color-scheme=slate] img[src$="#only-light"]{display:none}[data-md-color-scheme=slate][data-md-color-primary=pink]{--md-typeset-a-color:#ed5487}[data-md-color-scheme=slate][data-md-color-primary=purple]{--md-typeset-a-color:#c46fd3}[data-md-color-scheme=slate][data-md-color-primary=deep-purple]{--md-typeset-a-color:#a47bea}[data-md-color-scheme=slate][data-md-color-primary=indigo]{--md-typeset-a-color:#5488e8}[data-md-color-scheme=slate][data-md-color-primary=teal]{--md-typeset-a-color:#00ccb8}[data-md-color-scheme=slate][data-md-color-primary=green]{--md-typeset-a-color:#71c174}[data-md-color-scheme=slate][data-md-color-primary=deep-orange]{--md-typeset-a-color:#ff764d}[data-md-color-scheme=slate][data-md-color-primary=brown]{--md-typeset-a-color:#c1775c}[data-md-color-scheme=slate][data-md-color-primary=black],[data-md-color-scheme=slate][data-md-color-primary=blue-grey],[data-md-color-scheme=slate][data-md-color-primary=grey],[data-md-color-scheme=slate][data-md-color-primary=white]{--md-typeset-a-color:#5e8bde}[data-md-color-switching] *,[data-md-color-switching] :after,[data-md-color-switching] :before{transition-duration:0ms!important}}[data-md-color-accent=red]{--md-accent-fg-color:#ff1947;--md-accent-fg-color--transparent:#ff19471a;--md-accent-bg-color:#fff;--md-accent-bg-color--light:#ffffffb3}[data-md-color-accent=pink]{--md-accent-fg-color:#f50056;--md-accent-fg-color--transparent:#f500561a;--md-accent-bg-color:#fff;--md-accent-bg-color--light:#ffffffb3}[data-md-color-accent=purple]{--md-accent-fg-color:#df41fb;--md-accent-fg-color--transparent:#df41fb1a;--md-accent-bg-color:#fff;--md-accent-bg-color--light:#ffffffb3}[data-md-color-accent=deep-purple]{--md-accent-fg-color:#7c4dff;--md-accent-fg-color--transparent:#7c4dff1a;--md-accent-bg-color:#fff;--md-accent-bg-color--light:#ffffffb3}[data-md-color-accent=indigo]{--md-accent-fg-color:#526cfe;--md-accent-fg-color--transparent:#526cfe1a;--md-accent-bg-color:#fff;--md-accent-bg-color--light:#ffffffb3}[data-md-color-accent=blue]{--md-accent-fg-color:#4287ff;--md-accent-fg-color--transparent:#4287ff1a;--md-accent-bg-color:#fff;--md-accent-bg-color--light:#ffffffb3}[data-md-color-accent=light-blue]{--md-accent-fg-color:#0091eb;--md-accent-fg-color--transparent:#0091eb1a;--md-accent-bg-color:#fff;--md-accent-bg-color--light:#ffffffb3}[data-md-color-accent=cyan]{--md-accent-fg-color:#00bad6;--md-accent-fg-color--transparent:#00bad61a;--md-accent-bg-color:#fff;--md-accent-bg-color--light:#ffffffb3}[data-md-color-accent=teal]{--md-accent-fg-color:#00bda4;--md-accent-fg-color--transparent:#00bda41a;--md-accent-bg-color:#fff;--md-accent-bg-color--light:#ffffffb3}[data-md-color-accent=green]{--md-acceLine truncated
|
||||
@@ -1 +0,0 @@
|
||||
{"version":3,"sources":["src/templates/assets/stylesheets/palette/_scheme.scss","../../../../src/templates/assets/stylesheets/palette.scss","src/templates/assets/stylesheets/palette/_accent.scss","src/templates/assets/stylesheets/palette/_primary.scss","src/templates/assets/stylesheets/utilities/_break.scss"],"names":[],"mappings":"AA2BA,cAGE,6BAME,sDAAA,CACA,6DAAA,CACA,+DAAA,CACA,gEAAA,CACA,mDAAA,CACA,6DAAA,CACA,+DAAA,CACA,gEAAA,CAGA,mDAAA,CACA,gDAAA,CACA,yDAAA,CACA,4DAAA,CAGA,0BAAA,CACA,mCAAA,CAGA,iCAAA,CACA,kCAAA,CACA,mCAAA,CACA,mCAAA,CACA,kCAAA,CACA,iCAAA,CACA,+CAAA,CACA,6DAAA,CACA,gEAAA,CACA,4DAAA,CACA,4DAAA,CACA,6DAAA,CAGA,6CAAA,CAGA,+CAAA,CAGA,uDAAA,CACA,6DAAA,CACA,2DAAA,CAGA,iCAAA,CAGA,yDAAA,CACA,iEAAA,CAGA,mDAAA,CACA,mDAAA,CAGA,qDAAA,CACA,uDAAA,CAGA,8DAAA,CAKA,8DAAA,CAKA,0DAAA,CAzEA,iBCiBF,CD6DE,kHAEE,YC3DJ,CDkFE,yDACE,4BChFJ,CD+EE,2DACE,4BC7EJ,CD4EE,gEACE,4BC1EJ,CDyEE,2DACE,4BCvEJ,CDsEE,yDACE,4BCpEJ,CDmEE,0DACE,4BCjEJ,CDgEE,gEACE,4BC9DJ,CD6DE,0DACE,4BC3DJ,CD0DE,2OACE,4BC/CJ,CDsDA,+FAGE,iCCpDF,CACF,CCjDE,2BACE,4BAAA,CACA,2CAAA,CAOE,yBAAA,CACA,qCD6CN,CCvDE,4BACE,4BAAA,CACA,2CAAA,CAOE,yBAAA,CACA,qCDoDN,CC9DE,8BACE,4BAAA,CACA,2CAAA,CAOE,yBAAA,CACA,qCD2DN,CCrEE,mCACE,4BAAA,CACA,2CAAA,CAOE,yBAAA,CACA,qCDkEN,CC5EE,8BACE,4BAAA,CACA,2CAAA,CAOE,yBAAA,CACA,qCDyEN,CCnFE,4BACE,4BAAA,CACA,2CAAA,CAOE,yBAAA,CACA,qCDgFN,CC1FE,kCACE,4BAAA,CACA,2CAAA,CAOE,yBAAA,CACA,qCDuFN,CCjGE,4BACE,4BAAA,CACA,2CAAA,CAOE,yBAAA,CACA,qCD8FN,CCxGE,4BACE,4BAAA,CACA,2CAAA,CAOE,yBAAA,CACA,qCDqGN,CC/GE,6BACE,4BAAA,CACA,2CAAA,CAOE,yBAAA,CACA,qCD4GN,CCtHE,mCACE,4BAAA,CACA,2CAAA,CAOE,yBAAA,CACA,qCDmHN,CC7HE,4BACE,4BAAA,CACA,2CAAA,CAIE,8BAAA,CACA,qCD6HN,CCpIE,8BACE,4BAAA,CACA,2CAAA,CAIE,8BAAA,CACA,qCDoIN,CC3IE,6BACE,yBAAA,CACA,2CAAA,CAIE,8BAAA,CACA,qCD2IN,CClJE,8BACE,4BAAA,CACA,2CAAA,CAIE,8BAAA,CACA,qCDkJN,CCzJE,mCACE,4BAAA,CACA,2CAAA,CAOE,yBAAA,CACA,qCDsJN,CE3JE,4BACE,6BAAA,CACA,oCAAA,CACA,mCAAA,CAOE,0BAAA,CACA,sCFwJN,CEnKE,6BACE,6BAAA,CACA,oCAAA,CACA,mCAAA,CAOE,0BAAA,CACA,sCFgKN,CE3KE,+BACE,6BAAA,CACA,oCAAA,CACA,mCAAA,CAOE,0BAAA,CACA,sCFwKN,CEnLE,oCACE,6BAAA,CACA,oCAAA,CACA,mCAAA,CAOE,0BAAA,CACA,sCFgLN,CE3LE,+BACE,6BAAA,CACA,oCAAA,CACA,mCAAA,CAOE,0BAAA,CACA,sCFwLN,CEnME,6BACE,6BAAA,CACA,oCAAA,CACA,mCAAA,CAOE,0BAAA,CACA,sCFgMN,CE3ME,mCACE,6BAAA,CACA,oCAAA,CACA,mCAAA,CAOE,0BAAA,CACA,sCFwMN,CEnNE,6BACE,6BAAA,CACA,oCAAA,CACA,mCAAA,CAOE,0BAAA,CACA,sCFgNN,CE3NE,6BACE,6BAAA,CACA,oCAAA,CACA,mCAAA,CAOE,0BAAA,CACA,sCFwNN,CEnOE,8BACE,6BAAA,CACA,oCAAA,CACA,mCAAA,CAOE,0BAAA,CACA,sCFgON,CE3OE,oCACE,6BAAA,CACA,oCAAA,CACA,mCAAA,CAOE,0BAAA,CACA,sCFwON,CEnPE,6BACE,6BAAA,CACA,oCAAA,CACA,mCAAA,CAIE,+BAAA,CACA,sCFmPN,CE3PE,+BACE,6BAAA,CACA,oCAAA,CACA,mCAAA,CAIE,+BAAA,CACA,sCF2PN,CEnQE,8BACE,6BAAA,CACA,oCAAA,CACA,mCAAA,CAIE,+BAAA,CACA,sCFmQN,CE3QE,+BACE,6BAAA,CACA,oCAAA,CACA,mCAAA,CAIE,+BAAA,CACA,sCF2QN,CEnRE,oCACE,6BAAA,CACA,oCAAA,CACA,mCAAA,CAOE,0BAAA,CACA,sCFgRN,CE3RE,8BACE,6BAAA,CACA,oCAAA,CACA,mCAAA,CAOE,0BAAA,CACA,sCFwRN,CEnSE,6BACE,6BAAA,CACA,oCAAA,CACA,mCAAA,CAOE,0BAAA,CACA,sCAAA,CAKA,4BF4RN,CE5SE,kCACE,6BAAA,CACA,oCAAA,CACA,mCAAA,CAOE,0BAAA,CACA,sCAAA,CAKA,4BFqSN,CEtRE,sEACE,4BFyRJ,CE1RE,+DACE,4BF6RJ,CE9RE,iEACE,4BFiSJ,CElSE,gEACE,4BFqSJ,CEtSE,iEACE,4BFySJ,CEhSA,8BACE,mDAAA,CACA,4DAAA,CACA,0DAAA,CACA,oDAAA,CACA,2DAAA,CAGA,4BFiSF,CE9RE,yCACE,+BFgSJ,CE7RI,kDAEE,0CAAA,CACA,sCAAA,CAFA,mCFiSN,CG7MI,mCD1EA,+CACE,8CF0RJ,CEvRI,qDACE,8CFyRN,CEpRE,iEACE,mCFsRJ,CACF,CGxNI,sCDvDA,uCACE,oCFkRJ,CACF,CEzQA,8BACE,kDAAA,CACA,4DAAA,CACA,wDAAA,CACA,oDAAA,CACA,6DAAA,CAGA,4BF0QF,CEvQE,yCACE,+BFyQJ,CEtQI,kDAEE,0CAAA,CACA,sCAAA,CAFA,mCF0QN,CEnQE,yCACE,6CFqQJ,CG9NI,0CDhCA,8CACE,gDFiQJ,CACF,CGnOI,0CDvBA,iFACE,6CF6PJ,CACF,CG3PI,sCDKA,uCACE,6CFyPJ,CACF","file":"palette.css"}
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
-1625
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -1,261 +0,0 @@
|
||||
%% This BibTeX bibliography file was created using BibDesk.
|
||||
%% https://bibdesk.sourceforge.io/
|
||||
|
||||
%% Created for Eric Coissac at 2026-04-18 08:19:36 +0200
|
||||
|
||||
|
||||
%% Saved with string encoding Unicode (UTF-8)
|
||||
|
||||
|
||||
|
||||
@article{Zheng2020-ji,
|
||||
abstract = {MOTIVATION: Minimizers are methods to sample k-mers from a
|
||||
string, with the guarantee that similar set of k-mers will be
|
||||
chosen on similar strings. It is parameterized by the k-mer
|
||||
length k, a window length w and an order on the k-mers.
|
||||
Minimizers are used in a large number of softwares and pipelines
|
||||
to improve computation efficiency and decrease memory usage.
|
||||
Despite the method's popularity, many theoretical questions
|
||||
regarding its performance remain open. The core metric for
|
||||
measuring performance of a minimizer is the density, which
|
||||
measures the sparsity of sampled k-mers. The theoretical optimal
|
||||
density for a minimizer is 1/w, provably not achievable in
|
||||
general. For given k and w, little is known about asymptotically
|
||||
optimal minimizers, that is minimizers with density O(1/w).
|
||||
RESULTS: We derive a necessary and sufficient condition for
|
||||
existence of asymptotically optimal minimizers. We also provide a
|
||||
randomized algorithm, called the Miniception, to design
|
||||
minimizers with the best theoretical guarantee to date on density
|
||||
in practical scenarios. Constructing and using the Miniception is
|
||||
as easy as constructing and using a random minimizer, which
|
||||
allows the design of efficient minimizers that scale to the
|
||||
values of k and w used in current bioinformatics software
|
||||
programs. AVAILABILITY AND IMPLEMENTATION: Reference
|
||||
implementation of the Miniception and the codes for analysis can
|
||||
be found at https://github.com/kingsford-group/miniception.
|
||||
SUPPLEMENTARY INFORMATION: Supplementary data are available at
|
||||
Bioinformatics online.},
|
||||
author = {Zheng, Hongyu and Kingsford, Carl and Mar{\c c}ais, Guillaume},
|
||||
doi = {10.1093/bioinformatics/btaa472},
|
||||
issn = {1367-4803,1367-4811},
|
||||
journal = {Bioinformatics (Oxford, England)},
|
||||
language = {en},
|
||||
month = jul,
|
||||
number = {Suppl_1},
|
||||
pages = {i119--i127},
|
||||
pmc = {PMC8248892},
|
||||
pmid = 32657376,
|
||||
publisher = {Oxford University Press (OUP)},
|
||||
title = {Improved design and analysis of practical minimizers},
|
||||
url = {http://dx.doi.org/10.1093/bioinformatics/btaa472},
|
||||
volume = 36,
|
||||
year = 2020,
|
||||
bdsk-url-1 = {http://dx.doi.org/10.1093/bioinformatics/btaa472}}
|
||||
|
||||
@article{Zheng2021-cc,
|
||||
abstract = {MOTIVATION: Minimizers are efficient methods to sample k-mers
|
||||
from genomic sequences that unconditionally preserve sufficiently
|
||||
long matches between sequences. Well-established methods to
|
||||
construct efficient minimizers focus on sampling fewer k-mers on
|
||||
a random sequence and use universal hitting sets (sets of k-mers
|
||||
that appear frequently enough) to upper bound the sketch size. In
|
||||
contrast, the problem of sequence-specific minimizers, which is
|
||||
to construct efficient minimizers to sample fewer k-mers on a
|
||||
specific sequence such as the reference genome, is less studied.
|
||||
Currently, the theoretical understanding of this problem is
|
||||
lacking, and existing methods do not specialize well to sketch
|
||||
specific sequences. RESULTS: We propose the concept of polar
|
||||
sets, complementary to the existing idea of universal hitting
|
||||
sets. Polar sets are k-mer sets that are spread out enough on the
|
||||
reference, and provably specialize well to specific sequences.
|
||||
Link energy measures how well spread out a polar set is, and with
|
||||
it, the sketch size can be bounded from above and below in a
|
||||
theoretically sound way. This allows for direct optimization of
|
||||
sketch size. We propose efficient heuristics to construct polar
|
||||
sets, and via experiments on the human reference genome, show
|
||||
their practical superiority in designing efficient
|
||||
sequence-specific minimizers. AVAILABILITY AND IMPLEMENTATION: A
|
||||
reference implementation and code for analyses under an
|
||||
open-source license are at
|
||||
https://github.com/kingsford-group/polarset. SUPPLEMENTARY
|
||||
INFORMATION: Supplementary data are available at Bioinformatics
|
||||
online.},
|
||||
author = {Zheng, Hongyu and Kingsford, Carl and Mar{\c c}ais, Guillaume},
|
||||
doi = {10.1093/bioinformatics/btab313},
|
||||
issn = {1367-4803,1367-4811},
|
||||
journal = {Bioinformatics (Oxford, England)},
|
||||
language = {en},
|
||||
month = jul,
|
||||
number = {Suppl\_1},
|
||||
pages = {i187--i195},
|
||||
pmc = {PMC8686682},
|
||||
pmid = 34252928,
|
||||
publisher = {Oxford University Press (OUP)},
|
||||
title = {Sequence-specific minimizers via polar sets},
|
||||
url = {http://dx.doi.org/10.1093/bioinformatics/btab313},
|
||||
volume = 37,
|
||||
year = 2021,
|
||||
bdsk-url-1 = {http://dx.doi.org/10.1093/bioinformatics/btab313}}
|
||||
|
||||
@article{Pan2024-hb,
|
||||
abstract = {MOTIVATION: The minimizer concept is a data structure for
|
||||
sequence sketching. The standard canonical minimizer selects a
|
||||
subset of k-mers from the given DNA sequence by comparing the
|
||||
forward and reverse k-mers in a window simultaneously according
|
||||
to a predefined selection scheme. It is widely employed by
|
||||
sequence analysis such as read mapping and assembly. k-mer
|
||||
density, k-mer repetitiveness (e.g. k-mer bias), and
|
||||
computational efficiency are three critical measurements for
|
||||
minimizer selection schemes. However, there exist trade-offs
|
||||
between kinds of minimizer variants. Generic, effective, and
|
||||
efficient are always the requirements for high-performance
|
||||
minimizer algorithms. RESULTS: We propose a simple minimizer
|
||||
operator as a refinement of the standard canonical minimizer. It
|
||||
takes only a few operations to compute. However, it can improve
|
||||
the k-mer repetitiveness, especially for the lexicographic order.
|
||||
It applies to other selection schemes of total orders (e.g.
|
||||
random orders). Moreover, it is computationally efficient and the
|
||||
density is close to that of the standard minimizer. The refined
|
||||
minimizer may benefit high-performance applications like binning
|
||||
and read mapping. AVAILABILITY AND IMPLEMENTATION: The source
|
||||
code of the benchmark in this work is available at the github
|
||||
repository https://github.com/xp3i4/mini\_benchmark.},
|
||||
author = {Pan, Chenxu and Reinert, Knut},
|
||||
doi = {10.1093/bioinformatics/btae045},
|
||||
issn = {1367-4803,1367-4811},
|
||||
journal = {Bioinformatics (Oxford, England)},
|
||||
language = {en},
|
||||
month = feb,
|
||||
number = 2,
|
||||
pmc = {PMC10868324},
|
||||
pmid = 38269626,
|
||||
publisher = {Oxford University Press (OUP)},
|
||||
title = {A simple refined DNA minimizer operator enables 2-fold faster computation},
|
||||
url = {http://dx.doi.org/10.1093/bioinformatics/btae045},
|
||||
volume = 40,
|
||||
year = 2024,
|
||||
bdsk-url-1 = {http://dx.doi.org/10.1093/bioinformatics/btae045}}
|
||||
|
||||
@article{Kille2023-px,
|
||||
abstract = {MOTIVATION: The Jaccard similarity on k-mer sets has shown to be
|
||||
a convenient proxy for sequence identity. By avoiding expensive
|
||||
base-level alignments and comparing reduced sequence
|
||||
representations, tools such as MashMap can scale to massive
|
||||
numbers of pairwise comparisons while still providing useful
|
||||
similarity estimates. However, due to their reliance on minimizer
|
||||
winnowing, previous versions of MashMap were shown to be biased
|
||||
and inconsistent estimators of Jaccard similarity. This directly
|
||||
impacts downstream tools that rely on the accuracy of these
|
||||
estimates. RESULTS: To address this, we propose the minmer
|
||||
winnowing scheme, which generalizes the minimizer scheme by use
|
||||
of a rolling minhash with multiple sampled k-mers per window. We
|
||||
show both theoretically and empirically that minmers yield an
|
||||
unbiased estimator of local Jaccard similarity, and we implement
|
||||
this scheme in an updated version of MashMap. The minmer-based
|
||||
implementation is over 10 times faster than the minimizer-based
|
||||
version under the default ANI threshold, making it well-suited
|
||||
for large-scale comparative genomics applications. AVAILABILITY
|
||||
AND IMPLEMENTATION: MashMap3 is available at
|
||||
https://github.com/marbl/MashMap.},
|
||||
author = {Kille, Bryce and Garrison, Erik and Treangen, Todd J and Phillippy, Adam M},
|
||||
doi = {10.1093/bioinformatics/btad512},
|
||||
issn = {1367-4803,1367-4811},
|
||||
journal = {Bioinformatics (Oxford, England)},
|
||||
language = {en},
|
||||
month = sep,
|
||||
number = 9,
|
||||
pmc = {PMC10505501},
|
||||
pmid = 37603771,
|
||||
publisher = {Oxford University Press (OUP)},
|
||||
title = {Minmers are a generalization of minimizers that enable unbiased local Jaccard estimation},
|
||||
url = {http://dx.doi.org/10.1093/bioinformatics/btad512},
|
||||
volume = 39,
|
||||
year = 2023,
|
||||
bdsk-url-1 = {http://dx.doi.org/10.1093/bioinformatics/btad512}}
|
||||
|
||||
@incollection{Golan2025-xf,
|
||||
address = {Cham},
|
||||
author = {Golan, Shay and Shur, Arseny M},
|
||||
booktitle = {Lecture Notes in Computer Science},
|
||||
doi = {10.1007/978-3-031-82670-2\_25},
|
||||
isbn = {9783031826696,9783031826702},
|
||||
issn = {0302-9743,1611-3349},
|
||||
language = {en},
|
||||
pages = {347--360},
|
||||
publisher = {Springer Nature Switzerland},
|
||||
series = {Lecture Notes in Computer Science},
|
||||
title = {Expected density of random minimizers},
|
||||
url = {http://dx.doi.org/10.1007/978-3-031-82670-2_25},
|
||||
year = 2025,
|
||||
bdsk-url-1 = {http://dx.doi.org/10.1007/978-3-031-82670-2_25},
|
||||
bdsk-url-2 = {http://dx.doi.org/10.1007/978-3-031-82670-2%5C_25}}
|
||||
|
||||
@article{Mohamadi2017-ok,
|
||||
abstract = {Motivation: Many bioinformatics algorithms are designed for the
|
||||
analysis of sequences of some uniform length, conventionally
|
||||
referred to as k -mers. These include de Bruijn graph assembly
|
||||
methods and sequence alignment tools. An efficient algorithm to
|
||||
enumerate the number of unique k -mers, or even better, to build
|
||||
a histogram of k -mer frequencies would be desirable for these
|
||||
tools and their downstream analysis pipelines. Among other
|
||||
applications, estimated frequencies can be used to predict genome
|
||||
sizes, measure sequencing error rates, and tune runtime
|
||||
parameters for analysis tools. However, calculating a k -mer
|
||||
histogram from large volumes of sequencing data is a challenging
|
||||
task. Results: Here, we present ntCard, a streaming algorithm for
|
||||
estimating the frequencies of k -mers in genomics datasets. At
|
||||
its core, ntCard uses the ntHash algorithm to efficiently compute
|
||||
hash values for streamed sequences. It then samples the
|
||||
calculated hash values to build a reduced representation
|
||||
multiplicity table describing the sample distribution. Finally,
|
||||
it uses a statistical model to reconstruct the population
|
||||
distribution from the sample distribution. We have compared the
|
||||
performance of ntCard and other cardinality estimation
|
||||
algorithms. We used three datasets of 480 GB, 500 GB and 2.4 TB
|
||||
in size, where the first two representing whole genome shotgun
|
||||
sequencing experiments on the human genome and the last one on
|
||||
the white spruce genome. Results show ntCard estimates k -mer
|
||||
coverage frequencies >15× faster than the state-of-the-art
|
||||
algorithms, using similar amount of memory, and with higher
|
||||
accuracy rates. Thus, our benchmarks demonstrate ntCard as a
|
||||
potentially enabling technology for large-scale genomics
|
||||
applications. Availability and Implementation: ntCard is written
|
||||
in C ++ and is released under the GPL license. It is freely
|
||||
available at https://github.com/bcgsc/ntCard. Contact:
|
||||
hmohamadi@bcgsc.ca or ibirol@bcgsc.ca. Supplementary information:
|
||||
Supplementary data are available at Bioinformatics online.},
|
||||
author = {Mohamadi, Hamid and Khan, Hamza and Birol, Inanc},
|
||||
date-modified = {2026-04-18 08:19:36 +0200},
|
||||
doi = {10.1093/bioinformatics/btw832},
|
||||
issn = {1367-4803,1367-4811},
|
||||
journal = {Bioinformatics (Oxford, England)},
|
||||
language = {en},
|
||||
month = may,
|
||||
number = 9,
|
||||
pages = {1324--1330},
|
||||
pmc = {PMC5408799},
|
||||
pmid = 28453674,
|
||||
publisher = {Oxford University Press (OUP)},
|
||||
title = {ntCard: a streaming algorithm for cardinality estimation in genomics data},
|
||||
url = {http://dx.doi.org/10.1093/bioinformatics/btw832},
|
||||
volume = 33,
|
||||
year = 2017,
|
||||
bdsk-url-1 = {http://dx.doi.org/10.1093/bioinformatics/btw832}}
|
||||
|
||||
@misc{Mash-distances-doc,
|
||||
author = {{Marbl Lab}},
|
||||
howpublished = {Mash documentation},
|
||||
title = {Mash Distance},
|
||||
url = {https://mash.readthedocs.io/en/latest/distances.html},
|
||||
urldate = {2026-07-09},
|
||||
year = 2026}
|
||||
|
||||
@article{Fan2015-mash-formula,
|
||||
author = {Fan, Huan and Ives, Anthony R and Surget-Groba, Yann and Cannon, Charles H},
|
||||
doi = {10.1186/s12864-015-1647-5},
|
||||
journal = {BMC Genomics},
|
||||
number = 1,
|
||||
title = {An assembly and alignment-free method of phylogeny reconstruction from next-generation sequencing data},
|
||||
url = {https://doi.org/10.1186/s12864-015-1647-5},
|
||||
volume = 16,
|
||||
year = 2015}
|
||||
@@ -1,3 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
|
||||
</urlset>
|
||||
Binary file not shown.
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -1,61 +0,0 @@
|
||||
# Hot resume and memoization of long computations (discussion, 2026-10-14)
|
||||
|
||||
## Goal (user)
|
||||
|
||||
1. **Hot resume**: a run killed midway (crash, `Ctrl-C`, scheduler walltime) restarts from the last finished unit of work, not from zero.
|
||||
2. **No recomputation**: a long function called again with the same parameters (same command rerun, or another command needing the same intermediate) reuses its stored result.
|
||||
|
||||
Not a goal: bit-identical resumed samples. Sampling uses an OS-seeded RNG on purpose, repeated runs must stay independent draws to measure variance; a resumed run being a *different* valid draw is not a defect.
|
||||
|
||||
## Why it matters: one real run
|
||||
|
||||
`phylo --entropy 1 --pseudo-alignment --n-sites 5000000 --free-loss --no-ambiguity --quartet --pair-balance --family-overlap --sibling-stats` on the 180-genome Primulaceae index, from the stderr of the OAR job (2026-10-13), 256 partitions:
|
||||
|
||||
| Stage | Wall |
|
||||
|---|---|
|
||||
| `sibling_stats` (full scan, `Selection::All`) | 8 235 s |
|
||||
| `family_overlap` (sweep 1: overlap annex) | 5 582 s |
|
||||
| `sampled_scan`: sweep 1 again (entropy + quartet annexes) | ≈ 5 347 s |
|
||||
| `sampled_scan`: sweep 2 (pair-weight annex) | ≈ 5 296 s |
|
||||
| `sampled_scan`: the draw itself | ≈ 9 573 s |
|
||||
| `distance` (Jaccard, not asked for; older binary) | 53 s |
|
||||
| Total | 34 085 s (9 h 28) |
|
||||
|
||||
The three annex sweeps cost about the same although their per-family work differs, so the cost is the layer traversal. Part of it was addressed by restricting the matrix pass to selected families (`family_scan.rs`, same session); the sweeps were not fused. Nothing in this run could be resumed or reused: every stage restarts from its beginning.
|
||||
|
||||
## What exists today (verified in the code, 2026-10-14)
|
||||
|
||||
**Persisted per-layer annexes.** Entropy (`entropy*.pent`), quartet (`quartet*.pbiv`), pair weight, sibling annex: one file per layer, built by `ensure_layer_*` and skipped when `path.exists()` (`family_annexes.rs`). A rerun with the same conventions skips every sweep. This is already memoization, keyed by *file name only* (convention in the name, no fingerprint of the inputs; a rebuilt sibling annex deletes everything derived from it, see `pair_coverage_sampling.md`).
|
||||
|
||||
**Not persisted.** The family-overlap accumulator (`FamilyOverlapAccumulator`) is written once at the end of sweep 1 (`sibling_builder.rs`). The draw (`sample_index`) and the `sibling_stats` scan keep their state in memory only.
|
||||
|
||||
**Defect found, then fixed (2026-10-14): a killed annex build left a file that looked finished.** `create_mmap_file` (`obicompactvec/src/mmap_file.rs`) created the file at its *final path* with magic, header and the full length (`set_len`) before any data was written. A kill during a layer left a valid-looking file with unwritten data; the next run saw `path.exists()` and treated the annex as built, silently.
|
||||
|
||||
Reproduced on the 1x testbed (20 genomes, 512 layers): derived annexes deleted, `phylo --family-overlap --quartet --pair-balance --entropy 1 …` killed by `timeout 6`, then rerun. Old binary: 141 finals after the kill, 2 differing from a reference build; the rerun exited 0 and kept both. Fixed binary: kill at 6 s left 499 finals, all identical to the reference, plus 2 `.building` files; the rerun completed 1 537 files identical to the reference, no `.building` left. (Kill at 3 s: nothing published, nothing broken.)
|
||||
|
||||
**Fix.** `obicompactvec::building_path` / `publish_built`: a file is built under `<name>.building` and moved to its final name (same-directory `rename`, atomic) by the builder's `close()`, after the flush. Applied to the builders that go through `create_mmap_file` or write their own mmap on the annex path: `PersistentBitVecBuilder` (all constructors; quartet annex), `PersistentFixedIntVecBuilder`, `PersistentRankSelectBitVecBuilder`, `FamilyScalarAnnexBuilder` (entropy, pair weight), `SiblingAnnexBuilder` (`siblings.psib`) and `FamilyOverlapAccumulator::write` (overlap matrix). A leftover `.building` is truncated by the next build of that file. A build that is dropped without `close()` no longer produces the final file (no caller did; workspace tests pass). Also reordered in `--sibling-annex`: derived annexes are now removed *before* the new sibling annex is built and published, so a kill between the two leaves the old annex with no derived files, never a new annex next to files computed against the old numbering.
|
||||
|
||||
**Not covered.** The builders outside that path still write at their final name: `PersistentCompactIntVecBuilder`, the int/sparse matrix builders, `EliasFanoBuilder`, `TempBitVecBuilder`/`TempCompactIntVecBuilder` (temporary by nature) and the index-construction files that use them. The same reasoning applies to them as soon as a stage built on them has to be resumed; not audited. The in-memory overlap accumulator is still lost on a kill (a sweep restarts), now without leaving a corrupt file.
|
||||
|
||||
**`obiksession` crate** (`src/obiksession/`): `Session` (named directory, opaque parameter blob checked on open, whole-lifetime `DirLock`, artifacts checksummed with xxh3 and mmap-restored) and `Memoize` (`get_or_compute[_with_key]`, `invalidate`, `cached_keys`, `MemoKey` from `Debug` + xxh3, the `session_memoize!` macro). Compiles, 15 tests pass, **no caller**: `obikmer` lists the dependency without using it. `phylo --session` exits with an error (`cmd/phylo/mod.rs:271`) since `session_cache.rs` (the consumer tied to `PairwiseTally`/`PartitionDispersion`/`SnpAlignment`) was removed. `sample_index(skip_layers)` remains as a seam, never called with a non-zero value.
|
||||
|
||||
**Previous design and why it missed the goal** (`theory/evolutionary_distances.md`, section "Explicit `--session`"): one session per *exact* parameter tuple (changing `--n-sites` made another session), artifacts specific to the sampling result, resume only inside the draw (checkpoint every 8 layers or 30 s). Reuse across commands existed only for the sample. An exhaustive-sample cache from which smaller samples would be drawn was proposed and never done. User judgement: this version did not answer the objective.
|
||||
|
||||
## Proposed direction (not implemented; to be validated)
|
||||
|
||||
Start from the goal, not from the existing crate.
|
||||
|
||||
1. **Unit of work = one `(partition, layer)` step that already produces files.** The annex builds, the draw per layer and the scans all iterate that way (`for part … for layer …`). Resume means "skip the units whose output is complete", exactly what `exists()` approximates today.
|
||||
2. **Complete-or-absent outputs.** Done for the annex path (see the fix above); the remaining builders are to be covered as their stages become resumable. Prerequisite for everything below. The mechanism lives in `obicompactvec` (shared lower-level API), not in each consumer.
|
||||
3. **Key = function identity + parameters + fingerprint of the inputs.** A result is reusable iff the function, every parameter that influences it and the identity of its inputs (index content, upstream artifacts) match. Today the key is a file name plus an invalidation convention. The fingerprint requirement is the cost: what identifies an index version cheaply is to be decided (meta file, rebuild counter, hash of annex headers).
|
||||
4. **The cache holds results of long functions, not domain objects.** The crate must not know `PairwiseTally`. Callers serialize their own values (`rkyv` was already chosen for that). Whether `Memoize`'s API fits is unproven: it has no caller, no documentation and was written against the earlier design.
|
||||
5. **Granularity matters more than the API.** A cached 9 h result helps only the exact rerun; a per-layer cache also helps a changed parameter that leaves earlier layers untouched. The overlap accumulator (end-of-sweep write) would need a per-layer partial (a sum of per-layer matrices, mergeable) to be resumable.
|
||||
6. **Fuse before caching.** Three sweeps over the same layers where two suffice (overlap first, then entropy + quartet + pair weight together, since the pair weight needs only the finished overlap matrix) is a cost no cache removes on a first run; to do independently.
|
||||
|
||||
## Open questions
|
||||
|
||||
- Fingerprint of an index version: which cheap, stable identity?
|
||||
- Overlap accumulator resume: per-layer partial matrices on disk (size n² × layers) or a checkpoint every N layers?
|
||||
- `Memoize`/`Session`: keep (adapt to the points above), rewrite, or delete the crate until a caller exists (the "existing code is a hypothesis" rule applies; today it has no user).
|
||||
- Scope of the next step: atomic writes are done for the annex path; extend to the other builders, or go to the cache.
|
||||
- Draw resume: a layer's draw is independent of the others except through the quotas, which depend on every layer's mass; quotas can be recomputed from annexes on resume (cheap), so only the per-layer survivors need storing.
|
||||
@@ -1,342 +0,0 @@
|
||||
# NUMA-aware partition runner
|
||||
|
||||
## Problem
|
||||
|
||||
All partition-level parallel loops in obikindex currently fall into two
|
||||
categories:
|
||||
|
||||
**Naive Rayon** — used in `build_layers`, `pack_matrices`, `dump`, `select`,
|
||||
`stats`, `rebuild`, `reindex`:
|
||||
|
||||
```rust
|
||||
(0..n).into_par_iter().for_each(|i| work(i));
|
||||
```
|
||||
|
||||
Threads come from the global Rayon pool with no NUMA awareness. On
|
||||
multi-socket machines this produces cross-socket memory traffic and degrades
|
||||
performance super-linearly (see [NUMA-aware worker pools](numa_worker_pools.md)).
|
||||
|
||||
**Ad-hoc adaptive pool** — used in `merge`:
|
||||
|
||||
A bespoke implementation with pre-spawned workers, channel-based dispatch, and
|
||||
activation control. It handles NUMA correctly but is not reusable.
|
||||
|
||||
Both cases should be replaced by a single generic mechanism.
|
||||
|
||||
## Unified model
|
||||
|
||||
The key insight is that **UMA is just the NUMA case with a single node**. The
|
||||
runner always works the same way: one controller thread per node, each
|
||||
independently managing its own workers with the same adaptive logic. The only
|
||||
difference between UMA and NUMA is the number of nodes and whether workers are
|
||||
pinned.
|
||||
|
||||
```
|
||||
NUMA (k nodes) UMA (1 node)
|
||||
|
||||
controller-0 controller-1 … controller-0
|
||||
│ │ │
|
||||
workers[0] workers[1] workers[0]
|
||||
(pinned) (pinned) (global pool)
|
||||
└───────────────┴──────────────────┘
|
||||
shared work queue
|
||||
```
|
||||
|
||||
On each node, the Rayon `ThreadPool` is pinned to that node's CPUs.
|
||||
`pool.install()` ensures all internal Rayon calls (inside the work function)
|
||||
use the node-local pool. Linux first-touch then places heap allocations in
|
||||
local DRAM automatically.
|
||||
|
||||
On UMA the global Rayon pool is used directly — no pinning, no overhead.
|
||||
|
||||
## Adaptive mechanism
|
||||
|
||||
Each controller follows the same logic regardless of node count:
|
||||
|
||||
1. Pre-spawn `workers_per_node` dormant worker threads (blocked on `activate_rx`).
|
||||
2. Activate the first worker immediately.
|
||||
3. Loop on result channel with a `SPAWN_POLL` timeout:
|
||||
- On result: call `on_done`; check whether to activate the next worker.
|
||||
- On timeout: same check.
|
||||
- Activation criterion: `should_spawn_worker(active, global_efficiency, prev_efficiency)`.
|
||||
4. Drop `activate_tx` when done — dormant workers exit cleanly.
|
||||
|
||||
**Global CPU efficiency** (`CpuSample`, reads `/proc/stat` on Linux) is used by
|
||||
all controllers — no per-node measurement needed. The signal is coarser than
|
||||
per-node efficiency but correct in practice: if any node saturates memory
|
||||
bandwidth, the global efficiency drops and all controllers stop activating
|
||||
workers. Using a standard portable primitive avoids platform-specific CPU
|
||||
accounting and keeps the implementation clean.
|
||||
|
||||
## Proposed API
|
||||
|
||||
```rust
|
||||
pub struct PartitionRunner {
|
||||
// One entry per NUMA node; one entry total on UMA.
|
||||
nodes: Vec<NodeConfig>,
|
||||
}
|
||||
|
||||
struct NodeConfig {
|
||||
pool: Option<Arc<rayon::ThreadPool>>, // None = global Rayon pool (UMA)
|
||||
cpu_ids: Vec<usize>, // empty = no pinning (UMA)
|
||||
max_workers: usize,
|
||||
}
|
||||
|
||||
impl PartitionRunner {
|
||||
/// Detect topology and build the runner.
|
||||
/// Returns a single-node runner on UMA / macOS / hwloc failure.
|
||||
pub fn new() -> Self;
|
||||
|
||||
/// Run `f(i)` for every index in `order`, collecting results.
|
||||
///
|
||||
/// `on_done(i, result, elapsed)` is called under an internal mutex as
|
||||
/// each partition completes — use it for progress bars and aggregation.
|
||||
/// The runner serialises all calls to `on_done` via an internal
|
||||
/// `Arc<Mutex<C>>`, so no `Sync` bound is required on the callback.
|
||||
/// `Send` is required because the Arc clone crosses thread boundaries.
|
||||
///
|
||||
/// Serialisation is free in practice: a partition takes seconds to
|
||||
/// minutes; the callback takes microseconds. Contention is negligible.
|
||||
///
|
||||
/// Returns the first error from `f`, if any.
|
||||
pub fn run<F, R, E, C>(
|
||||
&self,
|
||||
order: &[usize],
|
||||
f: F,
|
||||
on_done: C,
|
||||
) -> Result<(), E>
|
||||
where
|
||||
F: Fn(usize) -> Result<R, E> + Send + Sync,
|
||||
R: Send,
|
||||
E: Send,
|
||||
C: FnMut(usize, R, Duration) + Send; // Send required, Sync is not
|
||||
}
|
||||
```
|
||||
|
||||
`order` is caller-supplied so each command chooses its scheduling strategy:
|
||||
largest-first for `merge`, sequential for `build_layers`, etc.
|
||||
|
||||
## Migration examples
|
||||
|
||||
### merge.rs (before: ~180 lines of bespoke machinery)
|
||||
|
||||
```rust
|
||||
let runner = PartitionRunner::new();
|
||||
runner.run(
|
||||
&order,
|
||||
|i| dst_partition.merge_partition(i, srcs, mode, n_dst_genomes, block_bits, evidence)
|
||||
.map_err(OKIError::Partition),
|
||||
|i, g_len, dur| {
|
||||
pb.inc(1);
|
||||
debug!("partition {i}: done in {:.1}s — {g_len} new kmers", dur.as_secs_f64());
|
||||
part_stats.push(PartStat { id: i, unitig_bytes: partition_sizes[i], g_len });
|
||||
},
|
||||
)?;
|
||||
```
|
||||
|
||||
### index.rs build_layers (before: naive into_par_iter)
|
||||
|
||||
```rust
|
||||
let order: Vec<usize> = (0..n).collect();
|
||||
let runner = PartitionRunner::new();
|
||||
runner.run(
|
||||
&order,
|
||||
|i| self.partition.build_index_layer(i, min_ab, max_ab, with_counts, &evidence, block_bits)
|
||||
.map_err(OKIError::Partition),
|
||||
|_, n_kmers, _| {
|
||||
total_kmers.fetch_add(n_kmers, Ordering::Relaxed);
|
||||
pb.inc(1);
|
||||
},
|
||||
)?;
|
||||
```
|
||||
|
||||
All other sites (`pack_matrices`, `dump`, `select`, etc.) follow the same
|
||||
pattern.
|
||||
|
||||
## Placement
|
||||
|
||||
`PartitionRunner` lives in `obikindex/src/numa.rs` alongside `NumaSetup`.
|
||||
It depends only on standard library primitives and Rayon — no new dependencies.
|
||||
|
||||
A single `PartitionRunner` instance can be built once per command invocation
|
||||
and reused across multiple `run()` calls (e.g. `merge` runs
|
||||
`merge_partitions` then `pack_matrices`).
|
||||
|
||||
## Known issue: CPU-only activation signal stalls on I/O-bound stages
|
||||
|
||||
Observed on a real `filter` run (109 genomes, 256 partitions, 8×24-core NUMA):
|
||||
`rebuild` (CPU-bound — k-mer construction) scales cleanly from 9 to 43 active
|
||||
workers as `CpuSample::do_i_activate` (`obisys::lib.rs`) sees efficiency climb.
|
||||
`pack_matrices` (I/O-bound — reopens and recomposes per-genome column files
|
||||
into `.pbmx`/`.pcmx`) activates one extra worker then flatlines at 10/192 for
|
||||
the rest of the stage, even though 256 partitions keep completing over several
|
||||
minutes. This matches the documented intent (§ Adaptive mechanism — "avoids
|
||||
over-provisioning ... I/O-bound ... workloads") but conflates two different
|
||||
things: *"CPU is not the bottleneck"* and *"more workers would not help"*. On
|
||||
storage with real queue depth (NVMe, RAID, parallel FS) the second stage could
|
||||
still benefit from more concurrent workers even with flat CPU usage — a signal
|
||||
the current mechanism cannot see.
|
||||
|
||||
A one-off artefact was also found in the same log: right after a stage
|
||||
transition, `do_i_activate` produced a physically impossible spike (efficiency
|
||||
~94 cores on a 192-core box) because it has no minimum-window guard — unlike
|
||||
its sibling `cpu_efficiency`, which returns `0.0` if `wall < 0.1s`
|
||||
(`obisys::lib.rs:260`). `do_i_activate` unconditionally overwrites
|
||||
`self.wall`/`self.user_secs`/`self.sys_secs` even when the elapsed window is
|
||||
too short to be meaningful, so a burst of rapid completions right after
|
||||
activating a worker can divide a real CPU delta by a near-zero wall delta.
|
||||
|
||||
### Implemented: I/O signal + shared debounce guard
|
||||
|
||||
`IoSample` (`obisys::lib.rs`, alongside `CpuSample`) is fed by
|
||||
`read_bytes`/`write_bytes` from `/proc/self/io` on Linux (actual bytes
|
||||
submitted to the block layer — not `rchar`/`wchar`, which also count
|
||||
page-cache hits, and not `ru_inblock`/`ru_oublock`, unreliable on macOS), with
|
||||
a `proc_pid_rusage(RUSAGE_INFO_V4)` fallback on macOS
|
||||
(`ri_diskio_bytesread`/`ri_diskio_byteswritten`, FFI only via `libc`, no new
|
||||
dependency — same pattern as the existing `getrusage` bindings). Any other
|
||||
target degrades gracefully to a signal that never triggers (falls back to
|
||||
CPU-only activation), same pattern as `cgroup_v2_available`.
|
||||
|
||||
`maybe_activate` (`numa.rs`) activates a worker if *either* signal still shows
|
||||
headroom, making `PartitionRunner` adapt to whichever resource is actually the
|
||||
bottleneck without per-call configuration. Both samplers are called
|
||||
unconditionally — no `||` short-circuit — so neither window starves behind
|
||||
whichever signal fires first:
|
||||
|
||||
```rust
|
||||
let cpu_threshold = CPU_SPAWN_THRESHOLD * activation.last_step() as f64;
|
||||
let cpu_wants_more = cpu_sample.do_i_activate(cpu_threshold);
|
||||
let io_wants_more = io_sample.do_i_activate(IO_SPAWN_THRESHOLD);
|
||||
if cpu_wants_more || io_wants_more {
|
||||
activation.grow(GROWTH_DIVISOR, n_total);
|
||||
}
|
||||
```
|
||||
|
||||
The CPU threshold is *not* the flat absolute delta it started as: it scales
|
||||
with `activation.last_step()` — the number of workers activated in the last
|
||||
growth step, tracked by `NodeActivation` (`numa.rs`) and updated every time
|
||||
`grow()` actually grows something. Growing by 8 workers should add ~8 cores of
|
||||
efficiency if the workload is truly CPU-bound; requiring only
|
||||
`CPU_SPAWN_THRESHOLD` (20 %) of that expected gain confirms the growth was
|
||||
useful without demanding perfect linear scaling. Scaling by the *last step's
|
||||
size* rather than the cumulative total keeps the bar equally meaningful
|
||||
whether it's the 2nd growth step or the 20th — a flat absolute threshold
|
||||
(0.2 core) is a strong signal at 8 active workers but pure noise at 150; a
|
||||
threshold scaled by the *cumulative* total instead (considered and rejected)
|
||||
would have made the bar essentially impossible to clear late in the ramp,
|
||||
strangling exactly the CPU-bound saturation the mechanism exists to allow.
|
||||
|
||||
Unlike the CPU signal (an absolute delta in cores — a bounded, portable unit),
|
||||
raw I/O throughput has no natural scale across devices, so `IoSample` uses a
|
||||
**relative** growth threshold instead of an absolute one:
|
||||
|
||||
```rust
|
||||
pub fn do_i_activate(&mut self, threshold: f64) -> bool {
|
||||
let elapsed = self.wall.elapsed().as_secs_f64();
|
||||
if elapsed < 0.1 { return false; } // state untouched — window keeps accumulating
|
||||
|
||||
let n = Self::read_bytes();
|
||||
let rate = n.saturating_sub(self.bytes) as f64 / elapsed;
|
||||
let activate = if self.previous_rate == 0.0 {
|
||||
rate > 0.0 // bootstrap: any measured throughput is signal
|
||||
} else {
|
||||
(rate - self.previous_rate) / self.previous_rate >= threshold
|
||||
};
|
||||
|
||||
self.bytes = n;
|
||||
self.wall = Instant::now(); // reset only on a real sample
|
||||
activate
|
||||
}
|
||||
```
|
||||
|
||||
The `elapsed < 0.1s → return false without mutating state` guard was also
|
||||
back-ported into `CpuSample::do_i_activate` (previously missing — source of
|
||||
the ~94-core artefact above) — one fix for both problems, and it removes the
|
||||
need for any arbitrary I/O-rate floor: a short/noisy window is rejected
|
||||
outright rather than papered over with a hardware-dependent constant.
|
||||
|
||||
Both spawn thresholds (`CPU_SPAWN_THRESHOLD`, `IO_SPAWN_THRESHOLD`, module-level
|
||||
`const` in `numa.rs`, both `0.2`) are a starting point, not a derived value:
|
||||
`0.2` (20 % relative growth) for `IoSample` was chosen to match the CPU
|
||||
threshold's *implicit* relative sensitivity (in the observed log, an 8→9
|
||||
worker step raised efficiency by ~12 %) — but I/O throughput is lumpier than
|
||||
CPU time (buffered writes flush in bursts), so it needs empirical validation
|
||||
against a real `pack` run before being considered final.
|
||||
|
||||
## Known issue: ramp-up too slow, and confused with node count
|
||||
|
||||
The original design started `n_nodes` workers (one per node) and grew one
|
||||
worker at a time. On a real `filter` run this took ~10 minutes to climb from
|
||||
9 to ~40 active workers even on the CPU-bound `rebuild` stage — most of a
|
||||
35-minute stage spent under-provisioned while waiting for evidence to
|
||||
accumulate one worker at a time. There is no scale-down mechanism (`n_active`
|
||||
only grows), so the original caution was deliberate — but a quarter of
|
||||
available cores is still far from saturation, and the real risk zone (over-provisioning
|
||||
a memory-bandwidth-bound stage) only shows up much later in the ramp, near
|
||||
full occupancy — not at 25 %.
|
||||
|
||||
The fix decouples ramp speed from node *count*: both the initial size and the
|
||||
growth step are a fraction of `workers_per_node` (node *size*), applied
|
||||
identically on every node. A single-NUMA-node (UMA) machine ramps exactly as
|
||||
fast as an 8-node one — growing by `n_nodes` per step, as first considered,
|
||||
would have degenerated to "grow by 1" on UMA, reproducing the original
|
||||
problem for exactly the machines that need the fix most.
|
||||
|
||||
```rust
|
||||
// NodeActivation::grow — called both at startup (activate_initial) and on
|
||||
// every CPU/IO-triggered growth step, with a different divisor each time.
|
||||
let wanted = (self.caps[idx] / divisor).max(1); // INITIAL_DIVISOR=4 at startup, GROWTH_DIVISOR=8 per step
|
||||
let room = self.caps[idx].saturating_sub(self.active[idx]);
|
||||
let grow = wanted.min(room).min(n_total.saturating_sub(self.total));
|
||||
```
|
||||
|
||||
This also fixed a latent correctness gap: the original single shared
|
||||
`activate_tx`/`activate_rx` pair had *no* per-node addressing — sending one
|
||||
activation signal woke up whichever dormant worker (from any node) happened
|
||||
to win the race on that channel. `crossbeam_channel` gives no fairness
|
||||
guarantee across competing receivers, so "round-robin across nodes" was an
|
||||
assumption the code never actually enforced. `PartitionRunner::run` now opens
|
||||
one activation channel per node (`activate_txs`/`activate_rxs`, one pair per
|
||||
`NodeConfig`); `NodeActivation` (`numa.rs`) tracks how many of each node's
|
||||
dormant workers have been woken and grows every node by the same amount per
|
||||
step, capped by that node's remaining dormant workers and by the run's total
|
||||
budget (`n_total`) — balance across nodes is now guaranteed by construction,
|
||||
not incidental to channel implementation details.
|
||||
|
||||
## Panic propagation (fixed, 2026-08-28)
|
||||
|
||||
A worker whose `f(i)` panicked used to never send its `WorkerEvent::Completed`
|
||||
— the controller's `while completed < n_total` loop then waited forever for
|
||||
an event that partition could no longer produce, since other live
|
||||
workers/timer threads kept `event_rx` open. Discovered via `obikselect`
|
||||
panicking on an unimplemented `Sparse` matrix case (see `implementation/select.md`):
|
||||
the process hung indefinitely instead of erroring out.
|
||||
|
||||
Fixed: each worker wraps `f(i)` in `std::panic::catch_unwind` and sends a new
|
||||
`WorkerEvent::Panicked(i, payload)` instead of silently dropping the
|
||||
partition. The controller counts it toward `completed` (unblocking the loop)
|
||||
and keeps the first payload seen; once `run` returns, `panic::resume_unwind`
|
||||
re-raises it on the caller's thread — the original message/backtrace still
|
||||
surfaces, from the right place, instead of a silent deadlock. Takes priority
|
||||
over a plain `Err` (a panic means a bug, not a normal typed failure).
|
||||
|
||||
## Open questions
|
||||
|
||||
- **Error handling**: `run` currently returns the first error; remaining errors
|
||||
are dropped. A `Vec<E>` return would give complete diagnostics. (Panics are
|
||||
now caught and re-raised individually — see above — this only concerns
|
||||
plain `Err` results.)
|
||||
|
||||
- **`INITIAL_DIVISOR` / `GROWTH_DIVISOR` tuning**: currently `4` and `8`
|
||||
(start at 1/4 of a node's cores, grow by 1/8 per step), chosen to fix an
|
||||
observed too-slow ramp — not yet validated against a real `pack` (I/O-bound)
|
||||
run, where over-provisioning risk is different from the CPU-bound `rebuild`
|
||||
case this was tuned against.
|
||||
|
||||
- **`on_done` ordering**: the runner serialises calls to `on_done` via an
|
||||
internal `Arc<Mutex<C>>`. `Send` is required (the Arc clone crosses thread
|
||||
boundaries); `Sync` is not (only one thread holds the lock at a time).
|
||||
Contention is negligible because a partition takes seconds while the callback
|
||||
takes microseconds. The callback is therefore simple to write (plain
|
||||
`Vec::push`, plain `FnMut`) with no measurable performance cost.
|
||||
@@ -1,448 +0,0 @@
|
||||
# Query system
|
||||
|
||||
## Goal
|
||||
|
||||
Given a set of query sequences, determine for each sequence how many of its k-mers are found in the index and, for each indexed genome, how many k-mers match. The query system is the foundation for read classification and sequence-to-genome mapping.
|
||||
|
||||
---
|
||||
|
||||
## Input
|
||||
|
||||
- Query sequences in FASTA or FASTQ format (gzip supported, streaming stdin supported). GenBank flat files are not supported at query time (only at index time).
|
||||
- Sequences shorter than k bases are silently skipped.
|
||||
- Non-ACGT characters are handled by the superkmer decomposition layer: they act as hard breaks, producing shorter superkmers (identical to the behaviour at indexing time).
|
||||
|
||||
---
|
||||
|
||||
## Algorithm
|
||||
|
||||
The query follows the same superkmer-based partitioning strategy used at indexing time. Everything below happens inside `process_chunk` (`query.rs`); there is no separate per-stage function, but the internal data flow is staged: k-mer-level dereplication, a two-part MPHF/column-major matrix lookup (`obikindex::partition::query_partition_with`), and a sparse Findere pass, each producing sparse intermediate structures rather than one dense allocation for the whole chunk.
|
||||
|
||||
```
|
||||
for each chunk of sequences (parallel workers via obipipeline, one call to process_chunk):
|
||||
build QueryBatch (QueryBatch::from_records):
|
||||
decompose all sequences into superkmers (SuperKmerIter) — construction only,
|
||||
not the dedup key
|
||||
deduplicate at k-mer granularity, split by partition in the same pass:
|
||||
by_partition: Vec<PartitionKmers> ← sorted unique k-mers + flat KmerDesc = (seq_idx, pos) slices (2026-10-15, was a HashMap — see "Query memory" below)
|
||||
allocate SmerIndex (SmerIndex::new): in_index: Vec<bool>, sized total_smers —
|
||||
NOT multiplied by n_genomes
|
||||
allocate by_genome: Vec<Vec<(seq_idx, pos, value)>>, one empty Vec per genome —
|
||||
stays empty (zero cost) for every genome this chunk never matches
|
||||
for each partition p:
|
||||
query_partition_with(p, kmers_for_p, on_event):
|
||||
stage 1 (MPHF-only): for each unique k-mer, try each layer's MphfLayer::find
|
||||
in turn, stop at the first hit; bucket confirmed hits by (layer, slot);
|
||||
emit QueryHit::Found(descs) once per hit k-mer
|
||||
stage 2 (column-major fetch): for each layer with ≥1 hit, for each genome
|
||||
column g in 0..layer.n_cols(): scan that layer's bucketed slots, look up
|
||||
col_value(g, slot); emit QueryHit::Value(descs, g, value) on nonzero
|
||||
on_event dispatches: Found → SmerIndex::mark_found for every desc;
|
||||
Value → push (seq_idx, pos, value) into by_genome[g]
|
||||
for each genome g with ≥1 hit (sparse_findere_for_genome):
|
||||
sort by_genome[g] by (seq_idx, pos); detect maximal runs of consecutive pos
|
||||
within one seq_idx; monotone-deque window-minimum scoped to each run →
|
||||
confirmed_by_genome[g]: Vec<(seq_idx, pos_out, value)>
|
||||
accumulate genome_totals per sequence from confirmed_by_genome (per genome, direct)
|
||||
accumulate kmer_count / kmer_missing per (sequence, output position), O(1) each,
|
||||
using only the confirmed-any bitmap and SmerIndex — independent of n_genomes
|
||||
if --detail: densify confirmed_by_genome into per-(seq, genome) coverage arrays
|
||||
emit annotated sequences (emit_batch)
|
||||
```
|
||||
|
||||
Superkmers that appear more than once in the batch (same sequence or across sequences), or different superkmers that happen to share a k-mer (read overlaps, repeats, a SNP splitting an otherwise-identical run), are deduplicated at k-mer granularity: each unique `CanonicalKmer` triggers at most one MPHF lookup and, on hit, one matrix fetch, broadcast to every `KmerDesc` occurrence referencing it.
|
||||
|
||||
**Findere requires full-sequence aggregation.** The sliding window (now per-run, not per-sequence — see [Findere z-window filter](#findere-z-window-filter)) only ever runs after all partitions have contributed their hits to `by_genome`. Applying it per superkmer would produce false negatives at superkmer boundaries, where the z-window spans two superkmers.
|
||||
|
||||
Batches are processed in parallel via `obipipeline` workers; the `--threads` flag controls the number of worker threads.
|
||||
|
||||
---
|
||||
|
||||
## Findere z-window filter
|
||||
|
||||
For approximate index modes, the index physically stores s-mers of size `s = k_user − z + 1`; `idx.kmer_size()` (bound to `k` in `process_chunk`) is this physically-indexed s-mer size, so decomposing the query at `k` naturally produces s-mer results.
|
||||
|
||||
The z-window aggregation is **sparse**, per genome, implemented in `sparse_findere_for_genome` (`query.rs`) — a run-detection pass followed by a monotone-deque sliding-window minimum scoped to each run, not a dense scan over every s-mer position of every sequence:
|
||||
|
||||
```
|
||||
sparse_findere_for_genome(hits, z, presence, threshold):
|
||||
// hits: raw (seq_idx, pos_smer, value) triples for this genome, as delivered
|
||||
// by query_partition_with's QueryHit::Value — only ever nonzero entries;
|
||||
// a position with no hit for this genome simply has no entry at all.
|
||||
sort hits by (seq_idx, pos_smer)
|
||||
|
||||
for each maximal run of consecutive pos_smer values within the same seq_idx:
|
||||
dq: VecDeque<(run-relative index, value)>
|
||||
for k, (_, pos, value) in enumerate(run):
|
||||
maintain dq monotone non-decreasing (pop back while back.value >= value)
|
||||
push (k, value)
|
||||
evict dq entries with run-relative index <= k - z
|
||||
if k + 1 >= z:
|
||||
win_min = dq.front().value
|
||||
if win_min > 0:
|
||||
pos_out = pos + 1 - z
|
||||
confirmed.push((seq_idx, pos_out, adjust(win_min)))
|
||||
return confirmed
|
||||
```
|
||||
|
||||
A window can only be confirmed (`win_min > 0`) when all `z` s-mers in it are present *and* nonzero for this genome — which, by construction, can only happen strictly inside one contiguous run of hits (any gap — an absent or zero-valued s-mer — forces `win_min = 0` for every window spanning it, exactly matching the old dense scan's "not in index counts as 0" rule, just never materialising the zero). The deque logic is otherwise identical to the pre-sparsification version; it's scoped to run-relative indices instead of the whole sequence.
|
||||
|
||||
This runs once per genome that has at least one hit in the chunk (`process_chunk` iterates `by_genome`, one `Vec<(seq_idx, pos_smer, value)>` per genome, built from `QueryHit::Value` during the partition loop — genomes with zero hits in this chunk have an empty `Vec` and cost nothing beyond the iteration itself). Total work is `O(hits log hits)` per genome (the sort) rather than `O(n_smers)` per genome regardless of hit count — a genuine complexity win on top of the memory one, for the common case where most `(chunk, genome)` pairs have no or few hits.
|
||||
|
||||
Output position `pos_out` is confirmed for genome `g` iff its run produced a nonzero `win_min` — equivalent to "all `z` consecutive s-mer values in the window are nonzero for `g`", same semantics as before.
|
||||
|
||||
**The value reported per confirmed position is the window minimum, not the leftmost s-mer's raw value** — unchanged from the dense version. For presence indexes (0/1 values) this is equivalent to a logical AND either way. For count indexes it is not: the accumulated count for genome `g` at position `pos_out` is the minimum across the window, the weakest link — not the leftmost s-mer's own count. The presence/count adjustment (`u32::from(win_min >= threshold)` vs. raw `win_min`) is applied once, inside `sparse_findere_for_genome`, rather than later during accumulation.
|
||||
|
||||
**`kmer_missing` bookkeeping is independent of the per-genome sparse structures**, by design (see roadmap point 9): a lightweight dense `SmerIndex` (`in_index: Vec<bool>`, sized `total_smers` — **not** multiplied by `n_genomes`) is populated from `QueryHit::Found` during the partition loop, one entry per hit k-mer regardless of which genome(s) it matched. A position with no genome confirmed counts as `kmer_missing` iff the leftmost s-mer of that window is absent from `SmerIndex` entirely (see [`kmer_missing` semantics](#kmer_missing-semantics)).
|
||||
|
||||
**Coverage (`--detail`)** is built by re-scanning each genome's confirmed-hit list (already computed, no extra pass over raw data) and densifying into the `[u32; n_kmers_out]` arrays the JSON output format requires — but only when `--detail` is actually requested; the sparse structures cost nothing extra when it isn't.
|
||||
|
||||
**Short sequences**: when a sequence's s-mer count is less than `z`, its run(s) — if any hits exist at all — can never reach length `z`, so no window is ever confirmed for it; no k_user-mer is emitted, same outcome as the dense version's `n_kmers_out == 0` early-skip, reached here as a natural consequence rather than a separate check.
|
||||
|
||||
**Exact indexes**: `z = 1`, every single-hit "run" of length 1 immediately satisfies `k + 1 >= z`, so every hit is its own confirmed window with `win_min` equal to its own value — a passthrough, as before.
|
||||
|
||||
### Effective z at query time
|
||||
|
||||
`effective_z` is resolved at the start of `run()`:
|
||||
|
||||
```rust
|
||||
let effective_z = args.findere_z.unwrap_or_else(|| match idx.meta().config.evidence {
|
||||
IndexMode::Approx { z, .. } | IndexMode::Hybrid { z, .. } => z as usize,
|
||||
IndexMode::Exact => 1,
|
||||
});
|
||||
```
|
||||
|
||||
The `-z` CLI option overrides the index metadata value. A higher z increases stringency (lower FP, some true positives may be discarded at sequence ends); a lower z increases sensitivity.
|
||||
|
||||
---
|
||||
|
||||
## Layer lookup: `MphfLayer::find`
|
||||
|
||||
`MphfLayer::open(dir, mode: &IndexMode)` receives the mode from `PartitionMeta` — no per-layer file is read. The caller (`QueryLayer`) never chooses the dispatch path: it is fixed at open time by `LayerEvidence`. See [obikindex::layer](../implementation/layer_tier.md) for the full `find` / `find_strict` API.
|
||||
|
||||
### `QueryLayer` variant selection
|
||||
|
||||
`QueryLayer::open` (`obikindex/src/partition/query_layer.rs:28-45`) only ever returns two variants — `Presence` or `Count`, checked in this order:
|
||||
|
||||
| Order | Condition | Variant | Data returned per k-mer |
|
||||
|---|---|---|---|
|
||||
| 1 | `with_counts=true` and `counts/` exists | `Count` | raw count per genome |
|
||||
| 2 | (else) `presence/` exists, or `counts/` doesn't exist at all | `Presence` | see below |
|
||||
| 3 | (else — `counts/` exists, `presence/` doesn't, `with_counts=false`) | `Count` | counts used as-is |
|
||||
|
||||
There is no `QueryLayer::SetOnly` variant. The "no on-disk matrix at all" case is handled one level down: `Presence` wraps `PersistentBitMatrix`, whose own `open()` (`obicompactvec/src/bitmatrix.rs:260-288`) auto-detects among **three** internal representations — `Packed` (`presence/matrix.pbmx`), `Columnar` (`presence/meta.json`), or `Implicit { n_rows, n_cols }` when neither file exists (built from `layer_meta.json`, `fill_row` returning all-`1`s without touching disk). This is where "1 for every genome" actually happens — not at the `QueryLayer` level.
|
||||
|
||||
**Worth double-checking, not confirmed as a bug**: `PersistentBitMatrix::open`'s `Implicit` branch constructs `Implicit { n_rows: meta.n, n_cols: 1 }` — `n_cols` is hardcoded to `1`, not to the layer's actual `n_genomes`. `fill_row` for `Implicit` only writes `buf[..1]`, leaving the rest of a longer `n_genomes`-sized buffer untouched (zeroed by the caller beforehand). If this path is ever reached for a layer covering more than one genome, only genome index 0 would read as present. Whether that's reachable in practice (layers might always be single-genome when they fall back to `Implicit`) wasn't verified here — flagging for follow-up, not fixing.
|
||||
|
||||
---
|
||||
|
||||
## Presence / count mode at query time
|
||||
|
||||
The `--force-presence` flag and `--presence-threshold` control how per-genome values are accumulated, independently of what the index stores:
|
||||
|
||||
```
|
||||
genome_totals[g] += if presence { u32::from(v >= threshold) } else { v }
|
||||
```
|
||||
|
||||
`presence` is true when `--force-presence` is set or when the index has no counts (`!with_counts`). The default `presence_threshold` is 1, so any nonzero count counts as a match.
|
||||
|
||||
---
|
||||
|
||||
## Coverage vectors (`--detail`)
|
||||
|
||||
When `--detail` is requested, a 3-D accumulator `cov[seq_idx][genome][kmer_pos]` is allocated after all partitions are queried, with dimensions derived from `n_kmers_out = n_smers − z + 1` (k_user-mer positions, not s-mer positions):
|
||||
|
||||
```
|
||||
cov[seq_idx][g][pos] += contribution
|
||||
where pos is the k_user-mer index in the filtered (post-Findere) vector
|
||||
```
|
||||
|
||||
Coverage reflects confirmed k_user-mers only. The vectors are emitted in the JSON annotation under the key `"coverage"`.
|
||||
|
||||
---
|
||||
|
||||
## `kmer_missing` semantics
|
||||
|
||||
`kmer_missing` counts k_user-mer positions where the leftmost s-mer of the window (`smer_index.is_in_index(seq_idx, pos)`, `SmerIndex`) is `false` — i.e. absent from the index entirely. K-mers where the z-window fails because a later s-mer is absent or zero (but the leftmost one is present) are not counted as missing — the leftmost s-mer being present is used as proxy for index membership.
|
||||
|
||||
---
|
||||
|
||||
## Output format
|
||||
|
||||
Output sequences are written in **OBITools4 format**: the original sequence with a JSON annotation map in the title line.
|
||||
|
||||
```
|
||||
>read_id {"kmer_count":59,"kmer_strict_matches":{"genome_a":42,"genome_b":7}}
|
||||
ATCGATCG...
|
||||
```
|
||||
|
||||
With `--detail`:
|
||||
|
||||
```
|
||||
>read_id {"kmer_count":59,"kmer_strict_matches":{...},"coverage":{"genome_a":[0,1,2,...],...}}
|
||||
ATCGATCG...
|
||||
```
|
||||
|
||||
Genome keys follow the iteration order of `meta.genomes`.
|
||||
|
||||
---
|
||||
|
||||
## Annotation schema
|
||||
|
||||
| Key | Type | Condition | Semantics |
|
||||
|---|---|---|---|
|
||||
| `kmer_count` | int | always | k-mers confirmed (post-Findere) with at least one genome match |
|
||||
| `kmer_missing` | int | `--count-missing` | k-mers absent from the index entirely (leftmost s-mer of the window not found) |
|
||||
| `kmer_strict_matches` | object | always | per-genome accumulated value, non-zero entries only (label → count or 0/1) |
|
||||
| `coverage` | object | `--detail` | per-genome array of per-position contributions (label → [u32]) |
|
||||
|
||||
`kmer_count + kmer_missing` ≤ total k_user-mers in the sequence. The gap corresponds to k_user-mers whose z-window was not fully confirmed (at least one s-mer absent or zero for all genomes) but whose first s-mer was present in the index.
|
||||
|
||||
---
|
||||
|
||||
## CLI
|
||||
|
||||
```
|
||||
obikmer query <index> [--detail] [--mismatch] [--count-missing]
|
||||
[--force-presence] [--presence-threshold <n>]
|
||||
[-z <z>] [-T <threads>] [--chunk-size <MiB>]
|
||||
<query.fa> [<query2.fa> ...]
|
||||
```
|
||||
|
||||
| Option | Default | Semantics |
|
||||
|---|---|---|
|
||||
| `-z` / `--findere-z` | from index metadata | Override Findere z parameter |
|
||||
| `--detail` | off | Emit per-position coverage vectors in JSON |
|
||||
| `--count-missing` | off | Add `kmer_missing` field to JSON |
|
||||
| `--force-presence` | off | Report 0/1 per genome regardless of index counts |
|
||||
| `--presence-threshold` | 1 | Minimum count to declare genome present |
|
||||
| `-T` / `--threads` | all CPUs | Worker threads |
|
||||
| `--chunk-size` | auto (from available RAM and thread count) | I/O chunk size in MiB — see [Future work, point 3](#throughput--parallelism--identified-potential-not-yet-implemented) for why the auto-sizing formula currently under-estimates memory on indexes with many genomes |
|
||||
|
||||
`--mismatch` is accepted but currently ignored with a warning on stderr.
|
||||
|
||||
---
|
||||
|
||||
## Future work
|
||||
|
||||
- **`--mismatch`**: 1-mismatch approximate matching — generate `3·k` single-substitution variants per k-mer, look each up independently.
|
||||
- **Read classification** (`--classify`): assign each read to the genome with the highest match score.
|
||||
- **Whitelist / blacklist filtering**: threshold-based accept/reject on per-genome match scores.
|
||||
|
||||
### Throughput & parallelism — identified potential (not yet implemented)
|
||||
|
||||
Observed on a 192-core (8×24 NUMA) machine: `query` uses ~10 cores or fewer, and the default chunk size gets the process OOM-killed. Root causes and candidate fixes, in dependency order:
|
||||
|
||||
**1. Single-threaded I/O source (main core-utilization bottleneck).**
|
||||
`run()` builds `all_chunks` via `paths.into_iter().flat_map(read_sequence_chunks_sized(...))` and passes it directly as the `input` iterator to `pipe.apply()`. In `obipipeline::Pipe::apply` (`scheduler.rs`), `input.next()` is called exclusively from the dedicated source thread — so file opening, decompression, and FASTA/FASTQ chunk-boundary parsing for *all* input files run serially in one thread, regardless of `--threads`. Compare with `steps::scatter` (used by `index`) and `cmd/superkmer.rs`: there, file opening + streaming is itself a `Flat` pipeline stage (`||?`), executed across the `n_workers` pool, with `obipipeline::throttle(paths, max_open)` bounding concurrently-open files in the source thread. That pattern parallelises I/O across files (and NUMA nodes); `query.rs` cannot.
|
||||
Fix direction: restructure `query`'s pipe with an initial `Flat` stage analogous to `scatter`'s, opening/chunking files across workers instead of in `flat_map`.
|
||||
|
||||
**2. Gzip decompression is inherently single-threaded per file.**
|
||||
`niffler`/`flate2` (used by `xopen`) do standard DEFLATE, which has no parallel-decodable structure for an arbitrary stream. Fix (1) parallelises *across* files but not *within* one large gzip file. Parking a possible fix (`rapidgzip-rs`) is tracked in [chunkreader.md](../implementation/chunkreader.md#future-work--parallel-gzip-decompression-in-xopen).
|
||||
|
||||
**3. Chunk-size memory formula ignores `n_genomes`.**
|
||||
`chunk_bytes = available_memory_bytes() / (n_workers * 16)` (`query.rs:407-414`) assumes a fixed ~8–16× overhead per raw input byte. But `KmerResults::new` (`query.rs:165-179`) allocates `data: Vec<u32>` sized `total_kmers_in_chunk × n_genomes` — dense, **for every k-mer position in the chunk, hit or not** — plus `win_min` and (with `--detail`) `cov`, same scaling. Real per-chunk memory is `O(n_genomes)`, not constant; the formula doesn't know `n_genomes` at all. This is the direct cause of the OOM kill on indexes with many reference genomes.
|
||||
|
||||
**4. MPHF lookup and matrix-row fetch are fused, not staged.**
|
||||
`QueryLayer::find_into` (`obikindex/src/partition/query_layer.rs:48-67`) does the MPHF `find` *and* the `fill_row` matrix read in one call per k-mer, inside a single-threaded loop (`query_partition_with`). There is no separation between "is this k-mer indexed" (cheap, `O(1)`, independent of `n_genomes`) and "what are its per-genome values" (the expensive, `n_genomes`-scaling part).
|
||||
|
||||
**5. Dereplication should happen at k-mer granularity, directly — not via an intermediate superkmer-level dedup.**
|
||||
`QueryBatch::from_records` currently dereplicates at the *superkmer* level (`HashMap<RoutableSuperKmer, Vec<SKDesc>>`, `query.rs:112`). This misses redundancy between k-mers shared by *different* superkmers (read overlaps, repeats, a SNP splitting an otherwise-identical run). Superkmer *construction* (`SuperKmerIter`) stays mandatory — it is the mechanism that computes minimizers/partition routing, not an optional dedup layer — but the dedup structure built on top of it should key directly on `CanonicalKmer`, in the same pass: `HashMap<CanonicalKmer, Vec<(seq_idx, pos)>>`. This also means the MPHF `find` itself runs once per **distinct** k-mer instead of once per occurrence — a win independent of the matrix-fetch cost below.
|
||||
|
||||
**6. Stage 1 output: bucket confirmed hits by layer, keyed by MPHF slot.**
|
||||
For each unique canonical k-mer, MPHF lookup across a partition's layers stops at the first match (`query_partition_with:105-111`) — a k-mer belongs to at most one layer. So stage 1's output can be reshaped directly into:
|
||||
```
|
||||
HashMap<layer_idx, HashMap<slot, Vec<(seq_idx, pos)>>>
|
||||
```
|
||||
replacing the `CanonicalKmer` key by the resolved `slot` (compact integer, and exactly what stage 2 needs to address the matrix). K-mers matching no layer simply have no entry here (they still count toward `in_index`/`kmer_missing` bookkeeping, which stays `O(1)` per position, independent of `n_genomes`).
|
||||
|
||||
**7. Partition-level parallelism is currently absent — and a NUMA-aware mechanism for exactly this already exists, unused, in `obikindex`.**
|
||||
`process_chunk`'s partition loop (`query.rs:250-278`, `for (part_idx, part_sks) in by_part.iter().enumerate()`) processes every partition of a chunk sequentially on the single worker thread that owns that chunk. This is a parallelism axis on its own, independent of the column question below.
|
||||
More importantly: `docmd/architecture/numa_partition_runner.md` and `numa_worker_pools.md` document `PartitionRunner` (`obikindex/src/numa.rs`), **already implemented** and already used by `merge.rs`, `index.rs` (`build_layers`), `select.rs`, `reindex.rs`, `rebuild.rs` — one controller thread per NUMA node, a Rayon pool pinned to that node's CPUs (`hwlocality`, `numa` feature, default-on in `obikindex/Cargo.toml`), adaptive worker activation driven by *both* a CPU-efficiency signal and an I/O-throughput signal (`CpuSample`/`IoSample`, `/proc/self/io` on Linux). It exists precisely because a naive `into_par_iter()` on the global Rayon pool measurably degrades ×60 on this codebase's own 192-core/8-NUMA reference machine (`numa_worker_pools.md`, § Problem) once workers contend for cross-socket memory bandwidth on shared mmap'd/hashed structures — exactly the shape of the matrix-column scan in point 8 below.
|
||||
`obikmer` already depends on `obikindex` (`obikmer/Cargo.toml`, for `KmerIndex`), so `PartitionRunner` is directly reachable from `cmd/query.rs` — no new dependency. Both the partition-level loop and (see point 8) the genome-column scan should be driven through it rather than through ad-hoc `rayon::into_par_iter()`, to avoid reproducing the already-measured-and-fixed contention problem. Also relevant: the "CPU-only signal stalls on I/O-bound stages" issue documented for `pack_matrices` (mmap-heavy, page-fault-bound) applies just as much to a column-major mmap scan over persistent matrices — reuse the existing dual CPU/IO activation signal rather than re-deriving one.
|
||||
>
|
||||
> **Correction from implementation (Phase 4 below)**: this turned out not to be viable as described. `PartitionRunner::run()`'s actual body spawns roughly one OS thread per worker slot across every NUMA node **on every call** (confirmed by reading `numa.rs`, not just its doc comments) — fine for the one-call-per-command-invocation batch usage in `merge`/`build_layers`, but `query_partition_with` runs once per `(chunk, partition)`, far too frequently to absorb that spawn cost. Partition-level parallelism via `PartitionRunner` is deferred, not implemented. See Phase 4's "What did not ship, and why" for the detail.
|
||||
|
||||
**8. Stage 2: column-major matrix fetch, parallel across genome columns — via `PartitionRunner`, not naive `rayon`.**
|
||||
Both persistent matrix formats are column-oriented on disk: `ColumnarCompactIntMatrix`/`ColumnarBitMatrix` (`obicompactvec/src/{intmatrix,bitmatrix}.rs`) mmap one file per genome column; `PackedCompactIntMatrix`/`PackedBitMatrix` mmap one region-offset per column in a single file. `fill_row(slot, buf)` as used today (`query.rs:262-272` via `on_hit`) reads **one slot across all `n_genomes` columns** per hit — the worst possible access pattern for this layout (up to `n_genomes` scattered mmap regions touched per single k-mer).
|
||||
Better: for each layer, walk the matrix **column by column** (genome by genome): for each genome, scan the `slot` keys collected in step 6 for that layer and call `col.get(slot)`, keeping only nonzero results, and broadcast to the associated `(seq_idx, pos)` list. Total `get()` calls are unchanged (`n_hits × n_genomes` in the worst case) — the win is locality (sequential access within one mmap'd column at a time, not scattered across all columns per hit), not fewer operations.
|
||||
Columns are independent (read-only, disjoint mmap regions) → embarrassingly parallel across genomes, *but* — per point 7 — `obicompactvec`'s existing `into_par_iter()` over `0..n_cols` (`sum()`, `count_nonzero()`, pairwise distance matrices) is the **naive, unpinned** pattern the rest of the codebase is actively migrating away from, not a model to copy here. Route this through `PartitionRunner` (or the same NUMA-pool machinery) instead. Two things to settle when this is designed: how the partition axis (point 7), the column axis, and the existing chunk-level `n_workers` `obipipeline` pool compose without oversubscribing the machine (three different concurrency mechanisms — raw-thread pipe workers, `PartitionRunner`'s pinned Rayon pools, and whatever drives the column scan — need a single reconciled thread budget, not three independent ones); and the threshold below which per-column dispatch overhead outweighs the gain (small `n_genomes` or small per-layer hit counts) — to be measured, not assumed.
|
||||
>
|
||||
> **Correction from implementation (Phase 4 below)**: column-major fetch is implemented — but as a plain sequential loop, not parallelised via `PartitionRunner`. Same reason as point 7's correction above. The column-major *locality* win (the actual claim of this point) does not depend on adding parallelism on top of it, and is validated independently. Column-level parallelism is deferred pending a mechanism that fits this call frequency (candidates noted in Phase 4).
|
||||
|
||||
**9. Sparse per-genome representation, fed directly to Findere.**
|
||||
Stage 2's output should be `HashMap<genome_idx, Vec<(seq_idx, position, count)>>`, **sorted by `(seq_idx, position)`** once collected, instead of a dense `KmerResults`-style matrix — the key must carry `seq_idx`, not just `genome_idx`, because a chunk batches many sequences and `position` is only meaningful within one; a plain `Vec<(position, count)>` per genome would silently mix positions from different sequences and corrupt the sliding-window scan. This bounds retained memory by actual nonzero hits on both axes (position sparsity from non-matching k-mers, genome sparsity from a matched k-mer typically belonging to only a handful of genomes out of possibly many). The Findere sliding-window (`process_chunk`, the `win_min`/deque loop) would need reworking to run per `(sequence, genome)` over its sparse, sorted `(position, count)` list — detect runs of ≥`z` consecutive positions, window-min within each run — instead of today's dense `O(total_kmers × n_genomes)` scan. This is also a genuine complexity win (`O(hits log hits)` per genome vs. dense scan), not just memory.
|
||||
**Not covered by this sparsification**: `--detail`'s `cov` accumulator (`query.rs:304-308`) has the identical `n_genomes`-dense scaling problem and wasn't folded into points above. It doesn't need to be retained densely throughout processing, though — only the final JSON serialization (`emit_batch`) requires a dense `[u32]` per `(seq, genome)`, and only for the sequences actually being output with `--detail`. Densification can stay a late, output-time-only step, reconstructed from the sparse per-genome lists.
|
||||
|
||||
**Secondary patterns available from `scatter.rs`/`superkmer.rs`, not yet in `query.rs`:**
|
||||
- `throttle()` + `CommonArgs::effective_max_open()` to bound concurrently-open input files (query.rs defines its own `QueryArgs`, doesn't reuse this).
|
||||
- Progress bar with EMA throughput + live active-worker gauges (`obisys::spinner`, `flat_active`/`transform_active` counters) — diagnostic value for locating the bottleneck.
|
||||
- `obisys::Reporter`/`Stage::start`/`stop` timing per phase (used by `index`, `filter`; absent from `query`).
|
||||
|
||||
None of this is implemented yet — parked here as a coherent roadmap while the design is discussed further. Suggested dependency order: (1) I/O parallelism → (3) genome-aware chunk sizing → (4)–(9) staged/k-mer-deduped/NUMA-aware-partition-and-column-major/sparse query engine (larger refactor, biggest structural payoff — reuses `PartitionRunner` rather than inventing a new parallelism mechanism) → (2) parallel gzip (separate, orthogonal, tracked in chunkreader.md) → secondary diagnostics patterns.
|
||||
|
||||
---
|
||||
|
||||
## Implementation plan
|
||||
|
||||
Concrete, phased translation of the roadmap above. Phases 0–2 are small, independent, low-risk, and each individually testable against current `query` output — land them first, in order, and measure on the reference 192-core/8-NUMA machine before deciding whether phases 3–5 (the staged/sparse engine, the larger structural payoff) are still worth their cost. Phases 3–5 are one coordinated change spanning `obikmer`, `obikindex::partition`, and `obicompactvec` — they should not be split across releases mid-way, because the intermediate state (e.g. k-mer-level dedup feeding the old dense `KmerResults`) has no correctness or performance benefit on its own. Phase 6 is unrelated to phases 0–5 and can happen any time, independently, if `rapidgzip-rs` is validated (see [chunkreader.md](../implementation/chunkreader.md#future-work--parallel-gzip-decompression-in-xopen)).
|
||||
|
||||
Instrumentation is deliberately sequenced *before* the I/O fix (reordering the roadmap's own listed order), because every later phase's justification rests on a measurement ("to be measured, not assumed" appears throughout the roadmap above) — without it, phases 3–5 would be undertaken on faith.
|
||||
|
||||
Performance measurement on the reference 192-core/8-NUMA machine is done by the project owner, not from this development environment (macOS, 16 cores — `PartitionRunner`'s NUMA pinning is Linux-only, so even phase 4's mechanism can't be functionally exercised for its actual purpose here). Each phase below is therefore written to be *self-measuring*: the debug-level logging it adds must be enough, on its own, to judge whether that phase's algorithmic choice paid off from a cluster run's logs, without needing to attach a profiler.
|
||||
|
||||
### Conventions applied to every phase below
|
||||
|
||||
**Debug logging.** Every phase that changes an algorithmic choice (not phase 0, which *is* the logging) adds `tracing::debug!`/`trace!` at points that let a cluster run's logs answer "did this help": counts, ratios, and timings that quantify the specific claim that phase makes — e.g. phase 3 must log how many MPHF `find` calls were saved by k-mer-level dedup (the whole justification for that phase), phase 4 must log per-column scan timings, phase 5 must log actual retained-memory / sparsity ratios achieved. Prefer one structured `debug!` per chunk (fields, not prose) over free-text — the cluster logs will be the only evidence available for judging these choices, so they need to be grep/awk-able, not just readable.
|
||||
|
||||
**Unit tests.** This project's convention (`obiread`, `obikseq`, `obidebruinj`, `obicompactvec`, `obikindex::layer`, `obiskio`, `obifastwrite`) is `#[cfg(test)] #[path = "tests/<name>.rs"] mod tests;` at the bottom of the source file, with the actual test code in a sibling `src/tests/<name>.rs`. Neither `obikmer` nor `obikindex::partition` (the two crates phases 3 and 5 touch most) currently have a `src/tests/` directory at all — this needs creating, following the existing pattern exactly, not inventing a new one.
|
||||
|
||||
**Workflow (`jj`).** Work happens in a fresh `jj` commit, easy to abandon. `jj new` between phases is reasonable where it helps isolate a phase for review, but only when the working copy compiles at that point (project convention) — phase 3's internal sub-steps (batch dedup change, then `query_layer.rs` split, then the new return shape) will likely not each compile independently since they're one coupled change, so treat "commit boundary" and "plan phase boundary" as related but not forced to match 1:1; use judgement per phase rather than mechanically splitting on every bullet.
|
||||
|
||||
### Phase 0 — Instrumentation (prerequisite for measuring every later phase)
|
||||
|
||||
**Goal**: make core utilization, throughput, and per-stage timing visible on a real run, so phases 1–5 can be justified with numbers instead of assumption.
|
||||
|
||||
- `obikmer/src/cmd/query.rs`: wrap `run()`'s main loop with `obisys::Reporter`/`Stage::start("query")`/`.stop()`, printed at the end via `rep.print()` — same pattern as `index.rs`/`filter.rs`.
|
||||
- Add an `obisys::spinner("query")` progress bar around the `pipe.apply(...)` loop, with an EMA throughput readout (bases/s or k-mers/s, mirroring `steps::scatter`'s `ema_rate` computation, `scatter.rs:88-118`) and live gauges for "chunks in flight" / "workers busy" — reuse the `AtomicU32` counter pattern from `scatter.rs` (`flat_active`, `transform_active`) rather than inventing a new one.
|
||||
- Add `max_open_files: Option<usize>` to `QueryArgs` and a `effective_max_open()` method mirroring `CommonArgs::effective_max_open()` (`obikmer/src/cli.rs:90-94`) — needed by phase 1's `throttle()` call. (`QueryArgs` can't just embed `CommonArgs` — it doesn't take `kmer_size`/`minimizer_size`/`partitions`/`level_max`/`theta` from the CLI, those come from the index metadata — so this is a small standalone addition, not a flatten.)
|
||||
- Add one structured `debug!` per `process_chunk` call: chunk byte size, sequence count, s-mer count, wall time, and (once later phases exist) the fields they add — this single log line is the baseline every later phase's own logging gets compared against.
|
||||
- **Validation**: none needed beyond "the numbers appear and look sane" — this phase changes no query logic or output.
|
||||
- **Deliverable used by every phase below**: a before/after throughput and core-utilization measurement on the reference machine.
|
||||
|
||||
### Phase 1 — Parallel per-file I/O (fixes root cause of low core utilization)
|
||||
|
||||
**Goal**: file opening, decompression, and chunk-boundary parsing run across the `n_workers` pool instead of serially in the pipe's dedicated source thread.
|
||||
|
||||
- `obikmer/src/cmd/query.rs`:
|
||||
- Replace the `paths.into_iter().flat_map(read_sequence_chunks_sized(...))` construction (current `run()`, building `all_chunks`) with `obipipeline::throttle(paths.into_iter(), args.effective_max_open())`, passed as the pipe's `input`.
|
||||
- Add a new `QueryData::Path(PathBuf)` variant (alongside `Chunk`/`Output`) to carry the throttled path through the pipe's type-erasure mechanism.
|
||||
- Add a new **first** pipe stage, `Flat`/fallible (`||?`), modeled on `scatter.rs:60-86` and `superkmer.rs:54-65`: given a `Throttled<PathBuf>`, call `read_sequence_chunks_sized(path, chunk_bytes)` and yield each `Rope` chunk, keeping `pw.guard` alive until the file's iterator is exhausted (reuse or adapt `scatter.rs`'s `GuardedIter` wrapper — same lifetime problem, same fix).
|
||||
- The existing `process_chunk` transform stage becomes the pipe's **second** stage, unchanged in its own logic — it still receives one `Rope` chunk at a time, just no longer all coming from one serial source.
|
||||
- `make_pipe!` invocation grows from one stage (`Chunk => Output`) to two (`Path => Chunk => Output`).
|
||||
- Log, per file: time spent waiting on the `throttle()` slot (queueing due to `max_open`), and time spent opening/decompressing/producing the first chunk — this is what directly proves (or disproves) that I/O is now spread across workers instead of serialized.
|
||||
- **Validation**: run `query` on a small multi-file input, diff output against the pre-change version — content must be identical; **record order across files is not guaranteed to be preserved** even before this change (chunk-level dispatch across `n_workers` already reorders completions), so the diff must be order-insensitive (sort by read id, or compare as sets) if it wasn't already.
|
||||
- **Measure**: core utilization on the reference machine with several large input files, compare against phase 0's baseline.
|
||||
|
||||
### Phase 2 — Genome-aware chunk-size formula (fixes OOM)
|
||||
|
||||
**Goal**: `chunk_bytes` reflects actual per-chunk memory (`O(n_genomes)`), not a fixed multiplier.
|
||||
|
||||
- `obikmer/src/cmd/query.rs`, `run()`: `n_genomes` and `args.detail` are already computed above the `chunk_bytes` calculation (`n_genomes` at the top of `run()`, before line 407 in the current file) — reorder if needed, then replace:
|
||||
```rust
|
||||
let computed = avail / (n_workers as u64 * 16);
|
||||
```
|
||||
with a formula that scales the divisor by `n_genomes` (and roughly doubles it when `--detail` is set, since `cov` duplicates the per-genome accumulation): e.g. `per_chunk_multiplier = base_overhead + n_genomes as u64 * BYTES_PER_KMER_PER_GENOME * if detail { 2 } else { 1 }`, replacing the flat `16`. `BYTES_PER_KMER_PER_GENOME` should be derived from `KmerResults`'s actual layout (`4` bytes per `u32` entry in `data`, plus the `bool` in `in_index`, plus `win_min`'s equal-sized buffer) rather than guessed.
|
||||
- `args.chunk_size` (manual `--chunk-size` override) keeps taking priority, unchanged.
|
||||
- Log the resolved `chunk_bytes`, `n_genomes`, and the estimated peak per-chunk memory (`chunk_bytes` × the same multiplier used to derive it) once at startup — lets a cluster run confirm the estimate was actually respected, not just that the process didn't get OOM-killed (which could also happen to be true for the wrong reason).
|
||||
- **Validation**: build a test index with a large `n_genomes` (e.g. hundreds), run `query` with default chunk sizing under a memory limit (`ulimit -v` or a cgroup), confirm it no longer gets OOM-killed and that memory scales as predicted when `n_genomes` grows.
|
||||
- **Note**: this phase is superseded once phase 5 lands (sparse retained memory no longer scales with `n_genomes × total_kmers` at all) — but it's needed immediately regardless, since phases 3–5 are a bigger, riskier change and users need a working `query` in the meantime.
|
||||
|
||||
### Phase 3 — K-mer-level dereplication, staged MPHF/matrix lookup
|
||||
|
||||
**Goal**: replace superkmer-level dedup with k-mer-level dedup (roadmap point 5), and split the fused MPHF-find/matrix-fetch (point 4) so stage 1's output is bucketed by layer and MPHF slot (point 6).
|
||||
|
||||
- `obikmer/src/cmd/query.rs`:
|
||||
- Replace `QueryBatch::from_records`'s dedup map (`HashMap<RoutableSuperKmer, Vec<SKDesc>>`, current `query.rs:112`) with a per-partition `HashMap<CanonicalKmer, Vec<(seq_idx: u32, pos: u32)>>`, built in the same `SuperKmerIter` pass: superkmer construction and partition routing (`part_idx` from the superkmer's minimizer hash) are unchanged, only the granularity of what gets deduplicated changes — each `CanonicalKmer` within a superkmer is inserted individually instead of the whole superkmer being the dedup key.
|
||||
- **Verified**: `CanonicalKmer` (`obikseq/src/kmer.rs:390`, `pub type CanonicalKmer = CanonicalKmerOf<KLen>`) — the underlying `CanonicalKmerOf<L>` derives `Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash` (`kmer.rs:269`). Usable as a `HashMap`/`HashSet` key as-is, no change needed.
|
||||
- `obikindex/src/partition/query_layer.rs`:
|
||||
- Split `QueryLayer::find_into` (`query_layer.rs:48-67`) into two methods: `find_slot(&self, kmer: CanonicalKmer) -> Option<usize>` (MPHF only, no matrix touch) and keep `fill_row` as-is for phase 4 to call later.
|
||||
- Replace `query_partition_with`'s inner loop (`query_layer.rs:103-113`) with a version that, for each unique `CanonicalKmer`, calls `find_slot` across the partition's layers (stopping at first hit, same as today), and instead of immediately filling a row, records `(layer_idx, slot)`.
|
||||
- New return shape for the partition-level query, replacing today's `on_hit(sk_idx, kmer_idx, row)` callback: `HashMap<layer_idx, HashMap<slot, Vec<(seq_idx, pos)>>>` (roadmap point 6) — built directly from the k-mer dedup map's `Vec<(seq_idx,pos)>` values, keyed by the resolved slot instead of the k-mer.
|
||||
- **This phase alone has no throughput benefit yet** (matrix fetch still happens, just deferred) beyond the k-mer-level dedup itself (fewer MPHF calls when queries have overlapping/repeated k-mers) — its purpose is to produce the input phase 4 needs. Land phase 3+4 together, not phase 3 alone, per the "don't split 3–5 across releases" note above.
|
||||
- Log, per chunk: total k-mer occurrences vs. unique `CanonicalKmer` count (the dedup ratio — the entire justification for this phase) and the resulting MPHF `find` call count. If the dedup ratio is close to `1.0` on real query data (little redundancy), that's the cluster run telling us this phase wasn't worth it — the logging needs to be able to say that, not just confirm the happy path.
|
||||
- **Unit tests**: create `obikmer/src/cmd/tests/query.rs` (new `src/tests/` dir for this crate, following the project's `#[cfg(test)] #[path = "tests/query.rs"] mod tests;` convention) and `obikindex/src/partition/tests/query_layer.rs` (likewise new for this crate). Cover: the k-mer-level dedup map construction on synthetic sequences with known repeated/overlapping k-mers (assert unique-kmer count and occurrence lists); the `find_slot`/bucket-by-layer-and-slot construction against a small hand-built `QueryLayer` fixture, asserting the `(layer_idx, slot, seq_idx, pos)` tuples match what the old per-occurrence loop would have produced.
|
||||
|
||||
### Phase 4 — Column-major matrix fetch (roadmap points 7–8) — implemented, NUMA parallelism deferred
|
||||
|
||||
**Goal (revised during implementation)**: replace `fill_row`-per-hit (row-major, worst-case mmap locality) with a column-major scan. `PartitionRunner` turned out to be the wrong mechanism for this at this call granularity — see below; the column-major fetch itself is implemented and validated, without it.
|
||||
|
||||
**What shipped:**
|
||||
- `obicompactvec`: the per-column accessors this phase needed **already existed** — `PersistentCompactIntMatrix::col_view(c)` and `PersistentBitMatrix::col_view(c)` are public, and `IntSliceView::get(slot)`/`BitSliceView::get(slot)` are public — the original plan underestimated how much of this plumbing the pairwise-distance code (`dump`/`select`/`stats`) had already required. The one real gap: `PersistentBitMatrix::col_view()` panics on the `Implicit` variant (the documented mono-genome fast path, `bitmatrix.rs`). Added `PersistentBitMatrix::get(c, slot) -> u32` (`bitmatrix.rs`), a non-panicking column-major point lookup that returns `1` for `Implicit` regardless of `c` — the smallest surface needed, not a new `col_get` API from scratch.
|
||||
- `obikindex/src/partition/query_layer.rs`: `query_partition_with` is now two explicit stages, matching roadmap points 6–8: **stage 1** (MPHF-only, per unique k-mer, bucket hits by `(layer_idx, slot)`, emits `QueryHit::Found`) then **stage 2** (per layer with ≥1 hit, column-major: for each genome column `g` in `0..layer.n_cols().min(n_genomes)`, scan that layer's bucketed slots and call `col_value(g, slot)`, emitting `QueryHit::Value(descs, g, value)` on nonzero). `QueryHit` is a single enum delivered through one `FnMut(QueryHit)` callback — an earlier two-closure design (`on_found` + `on_value`) didn't borrow-check, since the caller's single mutable accumulator (`KmerResults`) can't be captured by two separate `FnMut` closures passed to the same call.
|
||||
- `obikmer/src/cmd/query.rs`: `KmerResults::set` (row-major, whole-row-at-once) replaced by `mark_found` (stage 1: flag a position as indexed, independent of any genome's value) and `set_one` (stage 2: write one genome's value at one position). `QueryStats` extended with `n_columns_scanned`/`n_col_get_calls`, logged per chunk.
|
||||
- Total `get()`-equivalent calls are unchanged from the row-major version (`n_hits × n_cols` in the worst case, confirmed by `n_col_get_calls` in the debug log) — the win is locality (sequential access within one layer's column at a time, across `mmap`'d regions, instead of jumping across all columns per hit), exactly as predicted.
|
||||
|
||||
**What did not ship, and why — `PartitionRunner` is architecturally the wrong tool here:**
|
||||
Reading `obikindex/src/numa.rs`'s actual `run()` body (not just its doc comments) shows every call spawns a timer thread **plus one OS thread per worker slot on every NUMA node** (`std::thread::scope` + one `s.spawn()` per node per `max_workers`) — on the 192-core/8-NUMA reference machine, that's on the order of 190+ fresh OS threads spawned **per call**. This is fine for its actual, established usage in this codebase (`merge.rs`, `index.rs`'s `build_layers`): one `PartitionRunner::new()` + one `run()` call per command invocation, amortised over a batch of ~256 long-running partitions. It is not fine for `query`'s call pattern: `query_partition_with` runs once per `(chunk, partition)`, potentially thousands of times per second — spawning ~190 OS threads that often to scan a handful of genome columns would very likely cost far more than the row-major approach it's meant to replace. This is exactly the "resolve empirically, don't assume" composition risk the roadmap flagged, just resolved by reading the mechanism's actual cost before wiring it in, rather than by measuring a regression on the cluster after the fact.
|
||||
The column-major loop in stage 2 is therefore a **plain sequential loop** for now — it captures the whole, provable locality win (roadmap point 8's actual claim) without adding any parallelism mechanism. Genome-column-level parallelism (point 8's "bonus" axis) and partition-level parallelism (point 7) are both deferred — not abandoned. Candidates for a follow-up, once there's a concrete profiling need: (a) `rayon`'s already-warm global pool (`into_par_iter()`) for the column axis specifically — cheap to invoke repeatedly since it doesn't spawn threads per call, though it's the same "naive rayon" pattern `numa_worker_pools.md` warns about for a *different* workload (random pointer-chasing over large hash maps); a column scan's access pattern (sequential reads within one `mmap`'d region) has a different contention profile and hasn't been shown to have the same problem — needs its own measurement, not an assumption either way; (b) restructuring so `PartitionRunner` is invoked once per whole `query` run (or per large batch of chunks) rather than per `(chunk, partition)`, amortising its spawn cost the way `merge`/`build_layers` do — a bigger structural change than this phase's scope.
|
||||
- Log (implemented): `QueryStats::n_columns_scanned`/`n_col_get_calls`, folded into the existing per-chunk `debug!("k-mer dedup + column-major fetch", ...)` line (`query.rs`) alongside phase 3's dedup counters.
|
||||
- **Unit tests**: extended `obikindex/src/partition/tests/query_layer.rs` (phase 3's file) — `query_partition_with`'s empty/missing-index paths updated for the new `QueryStats` fields and single-callback signature.
|
||||
- **Validation performed**: full workspace build + `cargo test --workspace`, zero failures. Functional validation against real indexes: (1) a single-genome index — output byte-identical to pre-phase-4 (same `kmer_count`/`kmer_strict_matches` on every record); (2) the existing 20-genome `benchmark/global_index_presence` index — runs correctly, `n_hits=0` for an unrelated query (expected: no shared k-mers between a plant read and a bacterial reference set), no panics, confirming the `Implicit`/multi-column bounds logic doesn't crash on a real multi-genome, mixed-format index; (3) **the critical correctness case**: built two single-sequence-pair test genomes, merged into one 2-genome index, queried with reads from both — reads from `genomeA` matched **only** `genomeA` (`kmer_count` identical to the pre-dedup occurrence count, zero leakage into `genomeB`'s column) and vice versa. This is the test that would have caught a column-index mixup, an off-by-one in `n_cols`, or cross-genome bleed from the stage-1/stage-2 split — it passed cleanly.
|
||||
- **Not yet done**: the microbenchmark comparing column-major vs. the old row-major access pattern's wall time / page-fault counters on a large-`n_genomes` layer — needs a realistically large multi-genome index and, for the page-fault counters specifically, Linux (not available from this development environment). Left for cluster validation alongside phases 1–3's own pending measurements.
|
||||
|
||||
### Phase 5 — Sparse Findere rework (roadmap point 9)
|
||||
|
||||
**Goal**: replace the dense `KmerResults`/`win_min` sliding-window scan with one operating on phase 4's sparse per-genome output.
|
||||
|
||||
- `obikmer/src/cmd/query.rs`, `process_chunk`:
|
||||
- Remove `KmerResults` (`query.rs:157-202`) and the dense `win_min` allocation (`query.rs:290-291`, sized `max_n_kmers × n_genomes`).
|
||||
- Keep a lightweight dense `in_index: Vec<bool>` per chunk (sized `total_kmers`, independent of `n_genomes`) from phase 3's stage 1 — still needed for `kmer_missing` bookkeeping (leftmost-s-mer-of-window membership test), which phase 4's sparse structure doesn't carry (a k-mer with no genome hit has no entry there at all).
|
||||
- New per-`(seq_idx, genome)` scan: for each genome's `Vec<(seq_idx, pos, count)>` (sorted, per phase 4), group by `seq_idx` (contiguous after sort), then within each sequence's positions detect runs of `pos, pos+1, pos+2, ...` of length ≥ `z`; within each run, the existing monotone-deque window-minimum logic (`query.rs`'s current `dq` loop, conceptually unchanged) applies — but the deque now only scans real entries in the run, never zero-filled gaps.
|
||||
- Update `SeqAcc` accumulation and `emit_batch` to consume this per-genome sparse iteration instead of `results.val`/`results.is_in_index`.
|
||||
- `--detail`/`cov`: build sparsely during the same scan (only positions with a confirmed contribution get an entry), densify into the `[u32]` JSON array only in `emit_batch`, only for genomes/sequences actually being serialized (per roadmap point 9's note, `query.rs:304-308`'s current dense allocation goes away).
|
||||
- Log, per chunk: total sparse entries retained vs. what the old dense `KmerResults` would have allocated (`total_smers × n_genomes`) — the sparsity ratio is this phase's entire reason for existing, so it must be directly visible in the logs, not inferred from process RSS. Also log the run-detection stats (number of runs found, average run length) — a low average run length relative to `z` would mean most positions still fail to form a full window, worth knowing.
|
||||
- **Unit tests**: `obikmer/src/cmd/tests/query.rs` (extended from phase 3) — the property test described below is the primary deliverable here, not an afterthought; write it as an actual `#[test]` (or a small internal fuzz/property-style loop over randomized fixtures if a property-testing crate isn't already a dependency — check before adding one, per this project's dependency-approval rule) rather than a one-off manual comparison.
|
||||
- **Validation — this is the correctness-critical phase**: property-test comparing old (dense, pre-phase-3) and new (sparse) implementations on the same randomized input/index fixtures, asserting identical `kmer_count`, `kmer_missing`, `kmer_strict_matches`, and (with `--detail`) `coverage` for every sequence. Keep both implementations compiled side by side (behind a debug-only flag or a temporary parallel code path) only for the duration of this validation; delete the dense path once parity is confirmed — per this project's own convention, superseded code is not kept "just in case."
|
||||
- **Update `docmd/architecture/query.md` itself**: once this phase lands, the "Findere z-window filter" section (which currently — correctly — describes the dense deque-over-`0..n_smers` scan) needs another pass to describe the sparse run-detection algorithm instead, as already flagged when this phase was discussed.
|
||||
|
||||
**Implemented as planned, no deviations discovered this time.** What shipped:
|
||||
- `KmerResults` removed entirely, replaced by `SmerIndex` (`in_index: Vec<bool>` + `offsets`, unchanged size/purpose, renamed since it's no longer "results" — just the O(1)-per-position "was this k-mer found at all" bookkeeping) and `by_genome: Vec<Vec<(seq_idx, pos, value)>>` (one empty `Vec` per genome until a hit arrives — genomes with zero hits in a chunk cost nothing beyond the outer `Vec`'s own allocation).
|
||||
- New `sparse_findere_for_genome(hits, z, presence, threshold) -> (Vec<ConfirmedHit>, n_runs, total_run_len)` (`query.rs`): sorts one genome's raw hits by `(seq_idx, pos)`, detects maximal runs of consecutive `pos` within one sequence, runs the same monotone-deque window-minimum as before but scoped to each run (run-relative indices for eviction, absolute `pos` for computing `pos_out`). Presence/count adjustment (`u32::from(win_min >= threshold)` vs. raw) is applied inside this function, once per confirmed hit, rather than later during accumulation.
|
||||
- `process_chunk` restructured into three passes after the partition loop: (1) run `sparse_findere_for_genome` per genome, collecting `confirmed_by_genome` and run-detection stats; (2) accumulate `genome_totals` directly from `confirmed_by_genome` and mark a `confirmed_any: Vec<bool>` (sized `total_kmers_out`, not `× n_genomes`); (3) a position-only pass (`O(total_kmers_out)`, no genome factor) computing `kmer_count`/`kmer_missing` from `confirmed_any` + `SmerIndex`. `cov` (`--detail`) is populated by re-scanning `confirmed_by_genome` — only when `--detail` is actually set, otherwise skipped entirely.
|
||||
- Debug log added (`"sparse Findere"`): `n_dense_would_be` (`n_occurrences × n_genomes` — what the deleted dense path would have allocated), `n_sparse_entries` (what's actually retained), `n_runs`/`avg_run_len` (per the plan's ask, to see whether hits mostly fail to form complete windows).
|
||||
- **Unit tests**: `sparse_findere_matches_dense_reference_on_random_inputs` (`obikmer/src/cmd/tests/query.rs`) — 200 randomized cases (sequence count/length, `z`, presence/count mode, threshold, hit density from sparse to fully-dense) comparing `sparse_findere_for_genome` against `dense_reference_findere`, a faithful reimplementation of the deleted dense algorithm kept only as a test-local correctness oracle (no property-testing crate added — checked first, none was a workspace dependency; a small `std`-only xorshift64 PRNG stands in for one, deterministic and dependency-free). All 200 cases pass.
|
||||
- **Functional validation performed**: full workspace build + `cargo test --workspace`, zero failures. End-to-end against real indexes: baseline output (no flags) unchanged from pre-phase-5 recorded values on the same fixtures; `--count-missing` correct (`kmer_missing: 0` on a self-match); `--detail` correct — coverage array length matches `kmer_count`, and critically, re-ran the two-genome cross-contamination check from phase 4 with `--detail --count-missing`: `genomeA` reads show coverage sum `106` for `genomeA` and `0` for `genomeB` (and vice versa) — confirms the sparse-to-dense `cov` reconstruction doesn't leak across genomes either, not just the scalar `kmer_strict_matches` path.
|
||||
- This phase's roadmap item ("update the Findere z-window filter section") — done, see above; the "Algorithm" section's pseudocode was also updated, since it still named `KmerResults`/`SKDesc` from before phases 3–4.
|
||||
|
||||
### Phase 6 — Parallel gzip decompression (independent, optional)
|
||||
|
||||
Tracked separately in [chunkreader.md](../implementation/chunkreader.md#future-work--parallel-gzip-decompression-in-xopen); parked pending validation of `rapidgzip-rs` on real data. Not a dependency of, or a dependency for, phases 0–5 — `xopen` is shared infrastructure (`obiread`), phase 1 benefits from it but doesn't require it (phase 1 parallelises *across* files; this phase would additionally parallelise *within* one large file).
|
||||
|
||||
### Cross-cutting risks
|
||||
|
||||
- **Thread-budget oversubscription** (phase 4): the single biggest unresolved design question in this whole plan — see phase 4's composition note. Should be settled with real measurements early in phase 4, not assumed from the design alone.
|
||||
- **`obicompactvec` API surface growth** (phase 4): new public per-column accessors are additive (existing `fill_row`/`row` stay for other callers — `dump`, `select`, distance computations) — no breaking change expected, but worth checking `obicompactvec`'s other callers aren't already relying on `fill_row` being the only/cheapest access path in a way that would make maintaining two access patterns (row-major and column-major) a real maintenance cost rather than a one-off addition.
|
||||
- **`PersistentBitMatrix::Implicit`'s hardcoded `n_cols: 1` — resolved, not a bug.** `LayerMeta`'s own doc comment (`obicompactvec/src/layer_meta.rs:1-9`) states it is written "alongside `mphf.bin`" and read by `PersistentBitMatrix::open` "to determine `n_rows` for **the implicit (mono-genome presence/absence) case**" — i.e. `Implicit` is a documented single-genome fast path (no presence matrix needed when there is trivially one genome), not a generic "no matrix built yet" fallback. `n_cols: 1` is correct by design for the case it's meant to handle. Phase 4's column loop is safe as planned — this was worth checking once, doesn't need further action.
|
||||
|
||||
|
||||
## Query memory: `PartitionKmers` replaces the per-partition `HashMap` (2026-10-15)
|
||||
|
||||
**Problem.** A full-size query (two ~450 MB fastq.gz of a skim against a 354 M-kmer index) reached **74–81 GB RSS** with the default 256 MiB chunk. Cause, read in `batch.rs`/`chunk.rs` and measured with the `by_partition memory retained` debug log: `QueryBatch::by_partition` was a `HashMap<CanonicalKmer, Vec<KmerDesc>>` per partition. On genomic reads almost every k-mer is unique (12.5 M unique for 13.5 M occurrences in a 48 MB chunk, a dedup of 8 %), so each k-mer paid a table slot, growth slack and its own heap `Vec`: **889 MB per 48 MB chunk, ~71 bytes per unique k-mer, 18× the chunk size** — the dominant term of the per-chunk memory, several chunks being in flight at once.
|
||||
|
||||
**Change.** `obikquery::PartitionKmers`: occurrences collected as `(kmer, KmerDesc)` per partition, sorted by `(kmer, seq_idx, pos)` and stored as three flat arrays (unique k-mers, u32 offsets, descs). `query_partition_with` takes `&PartitionKmers`; `QueryHit` already carried slices. Occurrence order per k-mer is unchanged (query order); no per-k-mer allocation.
|
||||
|
||||
**Measured** (same inputs, release build, identical sorted output digests before and after):
|
||||
|
||||
| | `by_partition` (48 MB chunk) | RSS, 2×48 MB chunks | RSS, full run (≈ 1 GB reads) | wall, full run |
|
||||
|---|---|---|---|---|
|
||||
| before | 889 MB | 5.6 GB | 73.7–80.7 GB | 135–172 s (one 312 s outlier) |
|
||||
| after | 259 MB | 4.1 GB | **28.2 GB** | **103 s** |
|
||||
|
||||
With `--chunk-size 4` the same small query peaks at 3.6 GB: that floor is the touched pages of the index itself, so after the change the remaining per-chunk anonymous memory is ≈ 6× the chunk size, below the formula's factor 12 (`BASE_OVERHEAD + 8·n_genomes`, still a conservative bound — left unchanged). The sort replaces hashing in `from_records` and was not slower (17.3 s vs 21.2 s on the small test).
|
||||
|
||||
**Side finding.** The 256 MiB cap makes small inputs poorly parallel: the two 48 MB files above are two chunks, so two workers; with `--chunk-size 4` the same query takes 3.7 s instead of 17 s. Not changed here.
|
||||
|
||||
**Not verified.** Prod-scale (192 workers × 256 MiB chunks in flight): the factor is now ≈ 6 instead of ≈ 21, but how many chunks are in flight at once on that machine was not observed.
|
||||
|
||||
### Chunk size sweep on a small input (2026-10-15)
|
||||
|
||||
Input: 2 × 48 MB decompressed fastq (200 000 reads each) against a 354 M-kmer single-genome index (4 partitions), 16 cores on AC power, 3 runs per size, sorted outputs identical (same md5) for every size:
|
||||
|
||||
| `--chunk-size` (MiB) | chunks | wall (s) | user CPU (s) | RSS (GB) |
|
||||
|---|---|---|---|---|
|
||||
| 2 | ~50 | 2.5–2.6 | 25.0–25.3 | 3.2 |
|
||||
| 4 | ~25 | 2.5–2.7 | 24.3–25.5 | 3.5 |
|
||||
| 8 | ~12 | 2.5–2.7 | 23.6–26.1 | 4.1 |
|
||||
| 16 | ~6 | 4.4 | 23.1–23.7 | 4.1 |
|
||||
| 32 | 4 | 8.3 | 22.9–23.0 | 4.0 |
|
||||
| 64 and 256 (default) | 2 | 11.6–11.8 | 22.4–22.9 | 4.1 |
|
||||
|
||||
Wall time follows the number of chunks (one worker per chunk) until it reaches the core count, then plateaus at ≈ total CPU / cores; total CPU is almost constant (≈ +10 % at 2 MiB, per-chunk fixed cost, here with 1 genome and 4 partitions). With the 256 MiB cap, **any input below ≈ 16 × 256 MiB = 4 GiB decompressed leaves cores idle**. A rule such as `chunk ≈ input_bytes / (workers × 2)` with a floor of 4–8 MiB would use all cores on small inputs — **implemented, see below**. Unmeasured: the per-chunk cost on an index with many genomes and 256 partitions, which could move the floor up.
|
||||
|
||||
|
||||
### Automatic chunk size now follows the input size (2026-10-15)
|
||||
|
||||
`cmd/query/mod.rs`: when `--chunk-size` is not given, the memory-derived size (unchanged: `available RAM / (workers × overhead × 2)`, clamped to 4–256 MiB) is further capped by `estimated input bytes / (workers × 2)`, never below 4 MiB. The estimate is `obiread::estimated_decompressed_len`: the file length, ×3 when the magic bytes say gzip (≈ 2.8 measured on FASTQ), ×4 for bzip2/xz/zstd — a deliberately rough figure, detected from the magic bytes, not the extension. For stdin, URLs or unreadable files the size is unknown and the memory-derived size stands. An explicit `--chunk-size` still wins.
|
||||
|
||||
Measured (2 × 48 MB reads, 16 cores, AC power, outputs identical): single-genome index, 354 M kmers — default chunk 256 MiB → 4 MiB: **11.4 s → 2.4 s** (−79 %), CPU 22.3 → 26 s (+17 %), peak footprint 1.44 → 0.75 GB. On the 91-genome index the memory-derived size was already 4 MiB (overhead factor 4 + 8 × 91), so nothing changes there (5.1–5.7 s either way). Not measured: a large input (≈ 1 GB gz, whose estimate gives ≈ 95 MiB chunks instead of 256 MiB: more, smaller chunks, expected lower memory).
|
||||
@@ -1,160 +0,0 @@
|
||||
# Sibling annex & SNP-family pipeline — architecture
|
||||
|
||||
## Data model
|
||||
|
||||
A **family** is the up-to-4 k-mers sharing all flanks and differing only at
|
||||
the central base (well-defined for odd `k`). `FamilyMask` (`siblingannex.rs`)
|
||||
is a 4-bit presence mask (bit `b` set iff the family member whose own
|
||||
canonical central base is `b` is observed anywhere in the index) plus a
|
||||
minorant flag; `family_size()`/`siblings()` derive member count from it.
|
||||
`SiblingAnnex`/`SiblingAnnexBuilder` persist one `FamilyMask` per k-mer,
|
||||
**keyed by iteration order** (the physical layout of `unitigs.bin`), never by
|
||||
MPHF slot — this lets every reader zip-iterate a layer's own k-mer stream
|
||||
against the annex file directly, no MPHF/slot indirection.
|
||||
|
||||
## Construction
|
||||
|
||||
`algorithms::annex::build_layer_sibling_annex` builds one layer's annex:
|
||||
seeds each k-mer's own central base, generates its up-to-3 central-base
|
||||
variants, resolves each cross-partition (`IndexCache::find_in_partition`,
|
||||
grouped and chunked per destination partition for mmap locality), writes the
|
||||
final mask (with the minorant flag) in a second streamed pass.
|
||||
`SiblingExt::build_sibling_annex` drives this for every layer of every
|
||||
partition, then invalidates and rebuilds the entropy and family-overlap
|
||||
annexes (their `family_idx` numbering depends on the sibling annex).
|
||||
|
||||
## Shared traversal
|
||||
|
||||
`algorithms::family_scan::scan_layer_families` is the **only** place that
|
||||
resolves a family's full per-genome presence (`genome_mask: &[u8]`,
|
||||
cross-partition) — every other consumer in this module is built on it, never
|
||||
reimplements the resolution. Two decoupled concurrency mechanisms: batch
|
||||
generation (CPU-only) runs on an `obipipeline` throttle/pipe; per-batch
|
||||
resolution (I/O-bound, mmap page faults) runs `rayon`-parallel across
|
||||
destination partitions, one generated batch at a time (never several
|
||||
concurrently, to keep one partition's pages hot instead of scattering
|
||||
access). `fast_mode` (`is_fast_mode`, `<= 7` layers per partition) picks
|
||||
between trusting a family member's recorded layer index outright or probing
|
||||
every layer of the destination partition.
|
||||
|
||||
`Selection<'a>` bounds which families of a layer are resolved:
|
||||
|
||||
- `All` — every minorant, monomorphic included (`SiblingExt::shannon_entropy_csv`,
|
||||
`sibling_annex_stats`).
|
||||
- `Some(&HashSet<usize>)` — a caller-picked subset, skipping the expensive
|
||||
cross-partition resolution entirely for excluded families
|
||||
(`subsample::sample_layer`'s accepted-candidates batch;
|
||||
`entropy::ensure_layer_entropy_annex`'s non-monomorphic-only restriction).
|
||||
|
||||
`minorant_selection::eligible_bitset_layer`/`non_monomorphic_reindex_layer`
|
||||
provide the cheap, annex-bits-only (no cross-partition resolution) filters
|
||||
these restricted scans are built from: monomorphism (`family_size() < 2`) is
|
||||
knowable straight from the annex, so nothing pays `scan_layer_families`'s
|
||||
resolution cost just to discard a monomorphic result.
|
||||
|
||||
## Masking — computed once, shared everywhere
|
||||
|
||||
`algorithms::masking::masked_state(m, free_loss, no_ambiguity)` is the single
|
||||
function that turns a raw per-genome 4-bit state into the masked view every
|
||||
consumer needs (`None` for absence under `--free-loss` or ambiguity under
|
||||
`--no-ambiguity`). It is called **exactly once per (family, genome)**, in
|
||||
`subsample::sample_layer`, immediately after `scan_layer_families` resolves
|
||||
the raw state — never recomputed downstream. `survives_masking` (does the
|
||||
family still carry ≥2 distinct masked states among included genomes?) and
|
||||
`SurvivingFamily::masked: Vec<Option<u8>>` both consume this same
|
||||
precomputed array, which is also what `AlignmentReducer`/`PairwiseReducer`
|
||||
read (see below) — this is what guarantees the alignment and the pairwise
|
||||
tally can never disagree about a genome's state at a given site.
|
||||
|
||||
## Precomputed annexes (entropy, family overlap)
|
||||
|
||||
`algorithms::entropy::ensure_layer_entropy_annex` builds `EntropyAnnex`
|
||||
(`entropy_annex.rs`, one `f32` Shannon-entropy value per non-monomorphic
|
||||
minorant, mmap-backed) and/or feeds `FamilyOverlapAccumulator`
|
||||
(`family_overlap.rs`) from one shared `Selection::Some` scan — both need the
|
||||
same `genome_mask` resolution, so a missing one never pays for it twice.
|
||||
`family_entropy` (15 non-empty states of `{A,C,G,T}`'s powerset, `∅`
|
||||
excluded from the denominator — the project's settled informativeness
|
||||
definition, matching the calibrated 16-state Sankoff cost matrix) and
|
||||
`family_entropy_4` (plain 4-symbol reduction, kept only for comparison) are
|
||||
pure functions of `genome_mask`. `extensions::SiblingBuilder::ensure_entropy_and_overlap_annexes`
|
||||
drives this for every cached layer, lazily (only what's missing).
|
||||
|
||||
## Sampling and the shared scan trunk
|
||||
|
||||
`algorithms::subsample::sample_index` draws up to `n` non-monomorphic
|
||||
families index-wide, proportionally per layer: for each layer, a target
|
||||
quota sized from that layer's share of the index-wide eligible count, filled
|
||||
by a circular walk with a Bernoulli(`p`) draw per candidate — `p` optionally
|
||||
weighted by an unnormalized Gaussian kernel over the candidate's entropy
|
||||
(`EntropyBias { mu, sigma }`, read from the entropy annex). Accepted
|
||||
candidates are resolved in batches (`scan_layer_families`) and checked
|
||||
against `survives_masking`; rejected-after-masking candidates don't count
|
||||
toward quota, triggering further "topup" rounds (bounded, `MAX_TOPUP_ROUNDS
|
||||
= 5`) rather than silently under-filling. Every `SurvivingFamily` reaching a
|
||||
caller has already survived both the draw and post-masking informativeness —
|
||||
no second filter pass needed downstream.
|
||||
|
||||
`algorithms::scan::sampled_scan` is the single entry point every sampled
|
||||
computation goes through: one `sample_index` pass, fanned out to whichever
|
||||
`LayerReducer` implementations the caller actually needs, each receiving the
|
||||
exact same per-layer `SurvivingFamily` batch (an `Arc`, cloned not copied).
|
||||
This is what guarantees several outputs requested together
|
||||
(`--pseudo-alignment` and `--sankoff` and a `snp-*` `--distance` in one CLI
|
||||
invocation) come from **one** random sample, not independently-drawn ones —
|
||||
there is no other driver of `sample_index` anywhere in this crate.
|
||||
|
||||
Two `LayerReducer` implementations exist today:
|
||||
|
||||
- `AlignmentReducer` (`alignment.rs`) — accumulates `SnpAlignment` (one
|
||||
IUPAC-coded row per included genome), reading `family.masked` directly.
|
||||
- `PairwiseReducer` (`pairwise.rs`) — accumulates `PairwiseTally`/
|
||||
`PartitionDispersion`, also reading `family.masked` directly (substitution/
|
||||
agreement counts and cardinality both derive from the same masked state
|
||||
the alignment uses).
|
||||
|
||||
## From a tally to distances and calibration
|
||||
|
||||
`PairwiseTally` stores per-genome-pair substitution/agreement/cardinality
|
||||
counts, unfiltered, from which every downstream output is cheap `O(n²)`
|
||||
post-processing (never a second scan):
|
||||
|
||||
- `algorithms::snp_distance::distance_matrix` — closed-form `snp-*`
|
||||
corrections (raw p-distance, JC69, K80, K81, F81, T92, TN93, transversions-
|
||||
only), optionally gamma-corrected (`GammaShape`, `PartitionDispersion::estimate_alpha`
|
||||
for `Auto`).
|
||||
- `algorithms::sankoff::SankoffBundle::finish` — assembles the
|
||||
`--sankoff`/`--tnt`/`--phyg`/`--iqtree` pipeline's inputs
|
||||
(`RawSnpDistanceOutput`, `BasePairTally`, `CardinalityTally`) from an
|
||||
already-scanned `AlignmentReducer`/`PairwiseReducer` pair, plus
|
||||
`SankoffBundle::snp_distance` to reuse that exact sample for a `snp-*`
|
||||
`--distance` in the same invocation without a second scan.
|
||||
- `algorithms::cardcomp` — pure functions turning `CardinalityTally`/
|
||||
`BasePairTally` into row-stochastic transition probability matrices, then
|
||||
the calibrated 16-state Sankoff cost matrix (`pairwise_cost_matrix`).
|
||||
|
||||
## Public API surface
|
||||
|
||||
`extensions::SiblingExt` (CLI-facing trait on `IndexCache`) covers only the
|
||||
operations with no cross-invocation sharing concern: `build_sibling_annex`,
|
||||
`shannon_entropy_csv`, `sibling_family_size_histogram`/`sibling_annex_stats`,
|
||||
`family_overlap`. The sampled SNP path (`sampled_scan`, `AlignmentReducer`,
|
||||
`PairwiseReducer`, `distance_matrix`, `SankoffBundle`) is **not** wrapped in
|
||||
trait methods — the CLI (`obikmer::cmd::phylo`) composes them directly,
|
||||
because it alone knows the union of reducers a given invocation needs
|
||||
(`--pseudo-alignment`/Sankoff group/`snp-*` `--distance` can combine in one
|
||||
call). A per-output trait method would have to either duplicate that
|
||||
composition logic or reintroduce the "two independent scans" problem this
|
||||
design replaced.
|
||||
|
||||
## Known gaps
|
||||
|
||||
- `sample_index`'s `skip_layers` parameter (checkpointed-resume support) is
|
||||
never called with a non-zero value by any current caller — `--session`
|
||||
persistence was removed pending a redesign on top of `sampled_scan`
|
||||
(single shared trunk, not per-driver checkpoint plumbing). The parameter
|
||||
stays as the seam a future resume mechanism would use.
|
||||
- No cheap entropy pre-filter exists yet for the very high (saturated) tail —
|
||||
a row-marginal coverage approximation was considered and rejected (cannot
|
||||
represent joint multi-base co-occurrence, which is exactly what the
|
||||
15-state entropy definition exists to capture).
|
||||
@@ -1,230 +0,0 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<style xmlns="http://purl.org/net/xbiblio/csl" class="in-text" version="1.0" demote-non-dropping-particle="sort-only" default-locale="en-US">
|
||||
<info>
|
||||
<title>Ecology Letters</title>
|
||||
<id>http://www.zotero.org/styles/ecology-letters</id>
|
||||
<link href="http://www.zotero.org/styles/ecology-letters" rel="self"/>
|
||||
<link href="http://www.zotero.org/styles/apa" rel="template"/>
|
||||
<link href="http://onlinelibrary.wiley.com/journal/10.1111/%28ISSN%291461-0248/homepage/ForAuthors.html" rel="documentation"/>
|
||||
<author>
|
||||
<name>David Kaplan</name>
|
||||
<email>david.kaplan@ird.fr</email>
|
||||
</author>
|
||||
<contributor>
|
||||
<name>Sebastian Karcher</name>
|
||||
</contributor>
|
||||
<category citation-format="author-date"/>
|
||||
<category field="biology"/>
|
||||
<issn>1461-023X</issn>
|
||||
<eissn>1461-0248</eissn>
|
||||
<updated>2023-10-11T10:45:32+00:00</updated>
|
||||
<rights license="http://creativecommons.org/licenses/by-sa/3.0/">This work is licensed under a Creative Commons Attribution-ShareAlike 3.0 License</rights>
|
||||
</info>
|
||||
<macro name="container">
|
||||
<choose>
|
||||
<if type="chapter paper-conference" match="any">
|
||||
<text term="in" text-case="capitalize-first" suffix=": "/>
|
||||
<text variable="container-title" font-style="italic"/>
|
||||
<text variable="collection-title" prefix=", "/>
|
||||
<names variable="editor translator" prefix=" (" delimiter=", " suffix=")">
|
||||
<label form="short" suffix=" "/>
|
||||
<name name-as-sort-order="all" and="symbol" sort-separator=", " initialize-with="." delimiter=", " delimiter-precedes-last="never"/>
|
||||
</names>
|
||||
</if>
|
||||
<else>
|
||||
<group delimiter=", ">
|
||||
<text variable="container-title" font-style="italic" form="short"/>
|
||||
<text variable="collection-title"/>
|
||||
</group>
|
||||
</else>
|
||||
</choose>
|
||||
</macro>
|
||||
<macro name="author">
|
||||
<names variable="author">
|
||||
<name name-as-sort-order="all" and="symbol" sort-separator=", " initialize-with="." delimiter=", " delimiter-precedes-last="never"/>
|
||||
<label form="short" prefix=" (" suffix=")" text-case="capitalize-first"/>
|
||||
<et-al font-style="italic"/>
|
||||
<substitute>
|
||||
<names variable="editor"/>
|
||||
<names variable="translator"/>
|
||||
<text macro="title"/>
|
||||
</substitute>
|
||||
</names>
|
||||
</macro>
|
||||
<macro name="author-short">
|
||||
<names variable="author">
|
||||
<name form="short" and="symbol" delimiter=", " initialize-with=". "/>
|
||||
<et-al font-style="italic"/>
|
||||
<substitute>
|
||||
<names variable="editor"/>
|
||||
<names variable="translator"/>
|
||||
<choose>
|
||||
<if type="bill book graphic legal_case legislation motion_picture report song" match="any">
|
||||
<text variable="title" form="short" font-style="italic"/>
|
||||
</if>
|
||||
<else>
|
||||
<text variable="title" form="short" quotes="true"/>
|
||||
</else>
|
||||
</choose>
|
||||
</substitute>
|
||||
</names>
|
||||
</macro>
|
||||
<macro name="access">
|
||||
<choose>
|
||||
<if type="webpage">
|
||||
<group>
|
||||
<text term="available at" text-case="capitalize-first" suffix=": "/>
|
||||
<text variable="URL" suffix="."/>
|
||||
</group>
|
||||
<text value="Last accessed" prefix=" " suffix=" "/>
|
||||
<date variable="accessed">
|
||||
<date-part name="day" suffix=" "/>
|
||||
<date-part name="month" suffix=" "/>
|
||||
<date-part name="year"/>
|
||||
</date>
|
||||
</if>
|
||||
</choose>
|
||||
</macro>
|
||||
<macro name="title">
|
||||
<choose>
|
||||
<if type="report" match="any">
|
||||
<text variable="title" font-style="italic"/>
|
||||
<group prefix=" (" suffix=")">
|
||||
<text variable="genre"/>
|
||||
<text variable="number" prefix=" No. "/>
|
||||
</group>
|
||||
</if>
|
||||
<else-if type="bill book graphic legal_case legislation motion_picture report song speech" match="any">
|
||||
<text variable="title" font-style="italic"/>
|
||||
</else-if>
|
||||
<else-if type="webpage">
|
||||
<text variable="title" font-style="italic"/>
|
||||
</else-if>
|
||||
<else>
|
||||
<text variable="title"/>
|
||||
</else>
|
||||
</choose>
|
||||
</macro>
|
||||
<macro name="publisher">
|
||||
<choose>
|
||||
<if type="report" match="any">
|
||||
<group delimiter=", ">
|
||||
<text variable="publisher"/>
|
||||
<text variable="publisher-place"/>
|
||||
</group>
|
||||
</if>
|
||||
<else>
|
||||
<text variable="genre" suffix=". "/>
|
||||
<group delimiter=", ">
|
||||
<text variable="publisher"/>
|
||||
<text variable="publisher-place"/>
|
||||
</group>
|
||||
</else>
|
||||
</choose>
|
||||
</macro>
|
||||
<macro name="event">
|
||||
<choose>
|
||||
<if variable="event">
|
||||
<text term="presented at" text-case="capitalize-first" suffix=" "/>
|
||||
<text variable="event"/>
|
||||
</if>
|
||||
</choose>
|
||||
</macro>
|
||||
<macro name="issued">
|
||||
<choose>
|
||||
<if variable="issued">
|
||||
<date variable="issued">
|
||||
<date-part name="year"/>
|
||||
</date>
|
||||
</if>
|
||||
<else-if variable="accessed">
|
||||
<choose>
|
||||
<if type="webpage">
|
||||
<date variable="accessed">
|
||||
<date-part name="year"/>
|
||||
</date>
|
||||
</if>
|
||||
<else>
|
||||
<text term="no date" form="short"/>
|
||||
</else>
|
||||
</choose>
|
||||
</else-if>
|
||||
<else>
|
||||
<text term="no date" form="short"/>
|
||||
</else>
|
||||
</choose>
|
||||
</macro>
|
||||
<macro name="edition">
|
||||
<choose>
|
||||
<if is-numeric="edition">
|
||||
<group delimiter=" ">
|
||||
<number variable="edition" form="ordinal"/>
|
||||
<text value="edn"/>
|
||||
</group>
|
||||
</if>
|
||||
<else>
|
||||
<text variable="edition" suffix="."/>
|
||||
</else>
|
||||
</choose>
|
||||
</macro>
|
||||
<macro name="locators">
|
||||
<choose>
|
||||
<if type="article-journal article-magazine article-newspaper" match="any">
|
||||
<group prefix=", " delimiter=", ">
|
||||
<group>
|
||||
<text variable="volume"/>
|
||||
</group>
|
||||
<text variable="page"/>
|
||||
</group>
|
||||
</if>
|
||||
<else-if type="bill book graphic legal_case legislation motion_picture report song thesis" match="any">
|
||||
<group delimiter=". " prefix=". ">
|
||||
<text macro="edition"/>
|
||||
<text macro="event"/>
|
||||
<text macro="publisher"/>
|
||||
</group>
|
||||
</else-if>
|
||||
<else-if type="chapter paper-conference" match="any">
|
||||
<group delimiter=", " prefix=". ">
|
||||
<text macro="event"/>
|
||||
<text macro="publisher"/>
|
||||
<group>
|
||||
<label variable="page" form="short" suffix=" "/>
|
||||
<text variable="page"/>
|
||||
</group>
|
||||
</group>
|
||||
</else-if>
|
||||
</choose>
|
||||
</macro>
|
||||
<citation et-al-min="3" et-al-use-first="1" disambiguate-add-year-suffix="true" collapse="year-suffix" year-suffix-delimiter=", ">
|
||||
<sort>
|
||||
<key macro="author"/>
|
||||
<key macro="issued"/>
|
||||
</sort>
|
||||
<layout prefix="(" suffix=")" delimiter="; ">
|
||||
<group delimiter=" ">
|
||||
<text macro="author-short"/>
|
||||
<text macro="issued"/>
|
||||
</group>
|
||||
</layout>
|
||||
</citation>
|
||||
<bibliography et-al-min="7" et-al-use-first="6" entry-spacing="0" hanging-indent="true">
|
||||
<sort>
|
||||
<key macro="author"/>
|
||||
<key macro="issued" sort="ascending"/>
|
||||
<key macro="title"/>
|
||||
</sort>
|
||||
<layout>
|
||||
<group suffix=".">
|
||||
<text macro="author" suffix="."/>
|
||||
<text macro="issued" prefix=" (" suffix="). "/>
|
||||
<group delimiter=". ">
|
||||
<text macro="title"/>
|
||||
<text macro="container"/>
|
||||
</group>
|
||||
<text macro="locators"/>
|
||||
<text macro="access" prefix=". "/>
|
||||
</group>
|
||||
</layout>
|
||||
</bibliography>
|
||||
</style>
|
||||
@@ -1,94 +0,0 @@
|
||||
# Benchmark: query-path testing
|
||||
|
||||
`benchmark/Makefile` exercises indexing, merge, and phylo distance
|
||||
reconstruction against simulated bacterial genomes. It now also covers
|
||||
`obikmer query` — the read-matching path — and the sparse packed
|
||||
presence-matrix format (`obikmer pack --sparse`), previously untested by
|
||||
this pipeline.
|
||||
|
||||
## Motivation
|
||||
|
||||
- `query` had no end-to-end coverage. A regression there would not be caught
|
||||
by `verify_presence`/`verify_merge_presence`, which only check index
|
||||
*content* against the `.npz` truth, never the query API.
|
||||
- `pack --sparse` produces a presence-matrix format documented (see
|
||||
[obicompactvec.md](obicompactvec.md), "Sparse matrix formats") as faster
|
||||
for single-row access (query) and slower for column-oriented access
|
||||
(phylo `--metric`).
|
||||
`global_index_presence/` built by `merge_presence.sh` is always packed
|
||||
dense (packing is a stage inside `merge`, not a separate `pack`
|
||||
invocation) — there was no dense/sparse regression check.
|
||||
|
||||
## Query read source
|
||||
|
||||
Query reads are independent of `simulated_data/` (which is folded into the
|
||||
index being queried): reusing those reads would test against the exact
|
||||
error draw the index was built from. `query_data/<species>/<strain>/` holds
|
||||
a *second*, independent `iss generate` run against the same reference
|
||||
genome, via `simulate_query_one.sh` — unseeded, so a second draw picks up
|
||||
different sequencing errors than `simulate_one.sh`'s draw for the same
|
||||
genome. Fixed at 100,000 read pairs per genome (not coverage-proportional
|
||||
like the 15x used for `simulated_data/`), so wall/RSS numbers stay
|
||||
comparable across genomes of very different sizes.
|
||||
|
||||
Two query-source specimens, hardcoded as `QUERY_SPECIMENS` in
|
||||
`make_deps.py`: `Escherichia_coli--K-12_MG1655` (common, well-represented
|
||||
bacterium) and `Saccharolobus_islandicus--M.16.4` (the only archaeon in
|
||||
`SPECIES` — distant lineage, stresses the query path differently from a
|
||||
close-relative match). Two is enough to catch a dense/sparse regression
|
||||
without duplicating the exhaustive per-specimen coverage
|
||||
`verify_merge_presence` already provides across all `SPECIMENS`.
|
||||
|
||||
## Sparse global index
|
||||
|
||||
`global_index_presence_sparse/` is built by `pack_sparse.sh`: copy
|
||||
`global_index_presence/` wholesale, then `obikmer pack --sparse` in place.
|
||||
This works directly because `merge`'s pack stage (`merge.rs:252`,
|
||||
`pack_matrices(false)`) keeps the per-genome column files on disk after
|
||||
dense-packing — `pack_sparse_bit_matrix` (`obicompactvec/src/bitmatrix/sparse.rs:447`)
|
||||
reads those, is idempotent, and removes `matrix.pbmx` once the sparse form
|
||||
is written, so `Persistent::open` falls through to the sparse format
|
||||
afterward. No separate merge run needed.
|
||||
|
||||
## Query runs
|
||||
|
||||
`query_one.sh dense|sparse SPECIMEN` runs `obikmer query --count-missing`
|
||||
against `global_index_presence` or `global_index_presence_sparse`, output
|
||||
gzipped to `query_{dense,sparse}/SPECIMEN.fasta.gz`, Reporter wall/RSS
|
||||
captured to `stats/query_{dense,sparse}/SPECIMEN.stats` (same
|
||||
stderr-parsing convention as `merge_presence.sh`).
|
||||
|
||||
Flags: `--count-missing` only. `--mismatch` is a no-op today
|
||||
(`query/mod.rs:212-213`, prints "not yet implemented, ignored") — left off
|
||||
rather than tested for a feature that doesn't exist yet.
|
||||
|
||||
## Dense/sparse regression
|
||||
|
||||
`verify_query.py` compares the two query outputs per specimen, matched by
|
||||
read id (not stream position — the query pipeline chunks input across
|
||||
worker threads and doesn't guarantee output order). Compares `kmer_count`,
|
||||
`kmer_missing`, and the full `kmer_strict_matches` map per read. Any
|
||||
mismatch is a real regression: dense and sparse must be content-identical,
|
||||
only I/O access pattern differs. `.stats` → `stats/verify_query/`,
|
||||
aggregated by `aggregate_stats.sh query|verify_query`-style cases
|
||||
(`query_dense`, `query_sparse`, `verify_query`).
|
||||
|
||||
## Performance comparison
|
||||
|
||||
No dedicated script: the wall/RSS columns from the `query_dense` and
|
||||
`query_sparse` aggregated `.stats` CSVs are the dense-vs-sparse performance
|
||||
comparison — the expected win for query on sparse, per the `pack --sparse`
|
||||
help text.
|
||||
|
||||
## Scope
|
||||
|
||||
`count` track excluded from the sparse branch: `pack --sparse` targets
|
||||
presence matrices only (per CLI help); `pack_matrices` leaves count
|
||||
matrices untouched regardless of the `sparse` flag
|
||||
(`obikindex/src/index.rs:308`).
|
||||
|
||||
## New Makefile targets
|
||||
|
||||
`simulate_query`, `pack_sparse`, `query_dense`, `query_sparse`,
|
||||
`aggregate_query_dense`, `aggregate_query_sparse`, `verify_query`,
|
||||
`aggregate_verify_query` — the last three folded into `all`.
|
||||
@@ -1,193 +0,0 @@
|
||||
# Merge command
|
||||
|
||||
## Purpose
|
||||
|
||||
`obikmer merge` combines multiple existing kmer indexes into a single index. The result contains all kmers from all sources, with per-genome presence/absence or count data for every genome across every layer.
|
||||
|
||||
---
|
||||
|
||||
## Modes
|
||||
|
||||
```rust
|
||||
pub enum MergeMode { Presence, Count }
|
||||
```
|
||||
|
||||
Default mode is `Presence`. `Count` mode requires **all** source indexes to have `with_counts=true`; mixing count and non-count sources is rejected at validation.
|
||||
|
||||
| Mode | Column type | Constraint |
|
||||
|---|---|---|
|
||||
| `Presence` | `PersistentBitMatrix` (one bit per genome per slot) | none |
|
||||
| `Count` | `PersistentCompactIntMatrix` (one u32 per genome per slot) | all sources `with_counts=true` |
|
||||
|
||||
---
|
||||
|
||||
## Input / output constraints
|
||||
|
||||
All source indexes must satisfy:
|
||||
|
||||
- `IndexState::Indexed` (fully built — `index.done` sentinel present)
|
||||
- Same `kmer_size`, `minimizer_size`, `n_partitions`
|
||||
- Same evidence kind: all `Exact`, or all `Approx` with identical `(b, z)` parameters
|
||||
- If `Count` mode: all sources must have `with_counts=true`
|
||||
|
||||
`--force`: if the output directory already exists, it is deleted before the merge begins.
|
||||
|
||||
---
|
||||
|
||||
## Evidence compatibility
|
||||
|
||||
`validate_evidence_compat(sources)` is called before any I/O. It compares each source's `EvidenceKind` against `sources[0]`:
|
||||
|
||||
- All `Exact` → accepted, output uses `Exact`
|
||||
- All `Approx { b, z }` with same `(b, z)` → accepted, output uses those parameters
|
||||
- Any other combination → `OKIError::IncompatibleEvidence`, with a message directing the user to run `reindex` first
|
||||
|
||||
Mixed exact/approx is a hard error, not a silent conversion.
|
||||
|
||||
```rust
|
||||
fn validate_evidence_compat(sources: &[&KmerIndex]) -> OKIResult<EvidenceKind>
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Genome label deduplication
|
||||
|
||||
`compute_labels(sources, rename_duplicates)` assigns final genome labels across all sources before any file is written. The first occurrence of a label keeps the original name. Subsequent occurrences receive `.1`, `.2`, … suffixes when `rename_duplicates` is true, or trigger `OKIError::DuplicateGenomeLabel` otherwise.
|
||||
|
||||
---
|
||||
|
||||
## Algorithm
|
||||
|
||||
### 1. Validation
|
||||
|
||||
Check all sources against the constraints above. Abort on any mismatch.
|
||||
|
||||
### 2. Bootstrap output from the base source
|
||||
|
||||
The base is chosen by `choose_base` (`merge.rs`): the source with the largest total `unitigs.bin` size (restricted to approximate-evidence sources when the output must be approximate). It is not necessarily `sources[0]`, and the genome columns follow the base first. Because `obikmer index` is not byte-reproducible (unitig segmentation varies), two near-equal sources can swap roles between runs. Its directory is copied to `output` (hard links for files `merge_partition` never rewrites, see below). Immediately after the copy:
|
||||
|
||||
- `index.meta` is rewritten with the final genome list (all sources, possibly renamed) and the effective evidence kind.
|
||||
- In `Presence` mode, any `counts/` directories inherited from source_0 are removed.
|
||||
- `spectrums/` from source_0 is removed and rebuilt from scratch across all sources, applying the (possibly renamed) labels.
|
||||
|
||||
This establishes the partition layout, all existing MPHFs, unitigs, and evidence files. The base source's genomes occupy columns 0 … `n_dst_genomes - 1` in the destination.
|
||||
|
||||
### 3. For each subsequent source (parallel across partitions)
|
||||
|
||||
`obikmerge::partition_merge::merge_partition(dst, i, sources, mode, n_dst_genomes, block_bits, evidence)` runs once per partition index `i` (scheduled largest first by `PartitionRunner`). Each entry of `sources` is `(&KmerIndex, n_genomes)`, where `n_genomes` is the number of columns that source contributes (> 1 for an already-merged source). Code: `partition_merge/mod.rs` (orchestration, pass 1) and `partition_merge/pass2.rs` (per-group fill pipeline).
|
||||
|
||||
**First merge, Presence mode**: when `n_dst_genomes == 1`, `TypedLayer::<()>::init_presence_matrix` is called on every existing dst layer first. It materialises the implicit single-genome matrix (`presence/col_000000.pbiv`, all true) so that source columns can be appended to it.
|
||||
|
||||
The dst layers are opened once per partition and shared by both passes.
|
||||
|
||||
**Pass 1 — new k-mers into a de Bruijn graph.** Only the sources' `unitigs.bin` *paths* are needed (`KmerLayer::new(..).unitigs_path()`); no source layer is opened. The files are streamed (at most `max_open = 2` open at once), and every k-mer absent from all dst layers is pushed into one `GraphDeBruijn`. All new k-mers of all sources go into a **single** new layer per merge, built by `materialize_layer` (unitigs + MPHF + evidence).
|
||||
|
||||
**New-layer matrix.** If a new layer exists, its matrix is created with `n_dst_genomes` absent columns (all zero/false) and closed immediately. This keeps the column-count invariant, and every source group then appends to it like to the existing layers.
|
||||
|
||||
**Pass 2 — by consecutive groups of `SRC_GROUP = 2` sources, in source order.** Per group:
|
||||
|
||||
1. open the group's source layers for this partition;
|
||||
2. `MatrixBuilder::resume` every dst matrix (existing layers + new layer), check `n_cols == n_dst_genomes + col_base`, append the group's `group_n` columns;
|
||||
3. run the pipeline: stream each source layer's k-mers, `hash_batch` + `fill_sub_matrix` on the source, locate each k-mer in the dst layers (`find_batch_grouped`) or else in the new layer's MPHF, and write the values column-major into the group's builders;
|
||||
4. close the columns, then the matrices; drop the group's layers; `col_base += group_n`.
|
||||
|
||||
Only one group's source layers and destination columns are alive at a time, so the number of live mappings per partition does not depend on the number of sources. Columns are appended in source order, so every genome lands in the same absolute column as in a single-session merge.
|
||||
|
||||
**In-memory columns.** The group's columns are built in RAM (`MatrixBuilder::add_mem_col` → `MemBitVecBuilder` / `MemCompactIntVecBuilder`) and written once, sequentially, at `close` (`.building` file, then `rename`). The files are byte-identical to the former mmap-backed builders' output. There is no sparse file and no `MmapMut` on this path, so a block the FS cannot allocate is an `io::Error` instead of a SIGBUS. Files are not `fsync`ed: one sync per column file made merge ~8× slower. RAM per partition is at most `SRC_GROUP × src_n × N_part × (1/8 | 1)` bytes (presence | counts). History and figures: `merge_mmap_audit.md`; scaling limits: `merge_scaling.md`.
|
||||
|
||||
### 4. Update index metadata
|
||||
|
||||
`index.meta` was already written during bootstrap with the complete genome list and evidence kind. No further update is needed after the partition loop.
|
||||
|
||||
---
|
||||
|
||||
## `append_genome_column`
|
||||
|
||||
Defined on two concrete specialisations of `Layer<D>`:
|
||||
|
||||
```rust
|
||||
impl Layer<PersistentCompactIntMatrix> {
|
||||
pub fn append_genome_column(layer_dir: &Path, value_of: impl Fn(usize) -> u32) -> OLMResult<()>
|
||||
}
|
||||
|
||||
impl Layer<PersistentBitMatrix> {
|
||||
pub fn append_genome_column(layer_dir: &Path, value_of: impl Fn(usize) -> bool) -> OLMResult<()>
|
||||
}
|
||||
```
|
||||
|
||||
Each appends one column file to the matrix subdirectory (`counts/` or `presence/`). In `merge_partition`, columns are written through `MatrixBuilder::add_mem_col` (in-memory builders) rather than through these helpers, but the invariant they enforce is the same.
|
||||
|
||||
---
|
||||
|
||||
## Column count invariant
|
||||
|
||||
After any merge, **every layer in every partition has exactly `n_genomes` columns**, where `n_genomes` is the total genome count in the index at that point.
|
||||
|
||||
Maintained by three mechanisms:
|
||||
|
||||
1. **Existing layers**: `n_src_total` columns appended (one per source genome), group by group.
|
||||
2. **New layers created during merge**: `n_dst_genomes` absent columns prepended before source columns.
|
||||
3. **First merge, Presence mode**: `init_presence_matrix` retroactively creates `presence/col_0` all-true for genome 0.
|
||||
|
||||
The invariant is a precondition of `LayeredStore` aggregation traits: `col_weights()` and all partial distance methods assume every inner store has the same column count.
|
||||
|
||||
---
|
||||
|
||||
## Error variants relevant to merge
|
||||
|
||||
| Variant | Condition |
|
||||
|---|---|
|
||||
| `OKIError::NotIndexed(path)` | Source not in `Indexed` state |
|
||||
| `OKIError::IncompatibleConfig` | Mismatched `kmer_size`, `minimizer_size`, or `n_partitions` |
|
||||
| `OKIError::MismatchedMode` | Count mode but a source has `with_counts=false` |
|
||||
| `OKIError::IncompatibleEvidence(msg)` | Mixed exact/approx or different approx `(b, z)` |
|
||||
| `OKIError::DuplicateGenomeLabel(label)` | Duplicate label and `rename_duplicates=false` |
|
||||
|
||||
---
|
||||
|
||||
## On-disk impact
|
||||
|
||||
After merging `G` genomes (sources_0 contributes `G0`, subsequent sources the rest), *before* `finalize` packs each matrix (sparse by default, `matrix.pbmx` with `--dense`):
|
||||
|
||||
```
|
||||
partitions/
|
||||
part_00000/
|
||||
index/
|
||||
meta.json ← n_layers updated if new layer added
|
||||
layer_0/
|
||||
mphf.bin ← unchanged
|
||||
unitigs.bin ← unchanged
|
||||
evidence.bin ← unchanged
|
||||
presence/ ← created on first merge (Presence mode)
|
||||
meta.json {"n": N, "n_cols": G}
|
||||
col_000000.pbiv ← all-true (genome 0 … G0-1)
|
||||
col_000001.pbiv ← next source
|
||||
...
|
||||
counts/ ← extended (Count mode)
|
||||
meta.json {"n": N, "n_cols": G}
|
||||
col_000000.pciv ← genome 0 counts (from original build)
|
||||
col_000001.pciv ← next source
|
||||
...
|
||||
layer_N/ ← new layer (if new kmers found)
|
||||
mphf.bin
|
||||
unitigs.bin
|
||||
evidence.bin
|
||||
presence/ or counts/
|
||||
meta.json {"n": N1, "n_cols": G}
|
||||
col_000000.pbiv ← all-false (absent for existing genomes)
|
||||
...
|
||||
spectrums/
|
||||
<label>.json ← one file per genome, rebuilt from all sources
|
||||
index.meta ← complete genome list + evidence kind written at bootstrap
|
||||
```
|
||||
|
||||
`mphf.bin`/`unitigs.bin`/`evidence.bin`/`unitigs.bin.idx`/`fingerprint.bin`/`layer_meta.json` marked "unchanged" above are hard-linked from the base source's own files during the bootstrap copy (2026-08-28), not copied — `merge_partition` never rewrites them for pre-existing layers, only the `presence`/`counts` subdirectory gets widened in place, so only that subdirectory is a real, independent copy. Falls back to a real copy per file if linking itself fails (different filesystems). Verified: source files are byte-identical (checksummed) before/after a normal merge.
|
||||
|
||||
## Known issue (not yet fixed, 2026-08-28)
|
||||
|
||||
Merging an index against itself — literally the same directory passed twice as separate source arguments (e.g. `obikmer merge -o out IDX IDX --rename-duplicates`) — panics deep in the MPHF's rank-select structure (`common_traits::select_in_word`, `assertion failed: rank < self.count_ones()`), inside `MphfLayer::find` called from `merge_partition`'s "is this source kmer already in dst" check against the bootstrap-copied `dst_layers`. Root cause not identified; ruled out so far:
|
||||
|
||||
- **Not an empty-new-layer issue**: a source contributing zero new kmers (verified both as the sole additional source and as a third, fully-redundant source in a 3-way merge) is handled correctly — no layer is created for it, no crash, exit 0.
|
||||
- **Not caused by the hard-link change above**: verified via checksum that a normal (two distinct sources) merge leaves every source file byte-identical.
|
||||
|
||||
Only reproduces when `sources[0]` and `sources[1]` are the exact same on-disk path opened as two independent `KmerIndex` handles — an artificial scenario (nobody merges an index with itself intentionally), deprioritized rather than investigated further for now. Does terminate cleanly rather than hang, since `PartitionRunner`'s panic propagation fix (see `architecture/numa_partition_runner.md`) now surfaces it as a normal process panic/exit 101 instead of a silent deadlock.
|
||||
@@ -1,90 +0,0 @@
|
||||
# Merge: mmap audit (2026-11-17)
|
||||
|
||||
Audit of the memory-mapping behaviour of `obikmerge::partition_merge::merge_partition`, triggered by two production failures on a 192-core NUMA server merging 186 freshly built mono-genome presence indexes (256 partitions, data on BeeGFS `/hoyt`). Status: **A + C implemented (2026-11-17)**, B deferred. See [Implementation](#implementation-2026-11-17). Implementation prompt: `prompts/merge_mmap_refactor.md` (repository root, outside the doc tree).
|
||||
|
||||
## Observed failures
|
||||
|
||||
| Run | Flags | Failure | When |
|
||||
|---|---|---|---|
|
||||
| 1 | none | `Bus error` (SIGBUS) | seconds after `merge_partitions` starts |
|
||||
| 2 | `--cpu-max 32` | `memory allocation of 32768 bytes failed` → abort, many threads at once | after ~1 h, 128/256 partitions done |
|
||||
|
||||
Run 2 rules out "one partition too large for RAM": partitions are scheduled largest first (LFD, `merge.rs:289`), so at 50 % every remaining partition is smaller than one already merged. Small `malloc` failures on a 700 GB host are not RAM exhaustion either. Linux would OOM-kill (SIGKILL) instead. A NULL from `malloc` means `mmap`/`brk` returned `ENOMEM`: `vm.max_map_count` (default 65 530), `RLIMIT_AS`, or strict overcommit. Root access is not available on the compute server, so the fix must be in the code.
|
||||
|
||||
## Inventory: mappings held by one `merge_partition` call
|
||||
|
||||
Counts read from the code; totals assume 1 layer per source partition and 1 pre-existing dst layer (estimate, not measured).
|
||||
|
||||
| Resource | Where | Maps | Lifetime | Needed? |
|
||||
|---|---|---|---|---|
|
||||
| Source layers, fully opened | `partition_merge.rs:219-227` → `KmerLayer::open` → `MphfLayer::open` (`mphf_layer.rs:171`) | 4 per layer: MPHF (`MphfOnly`), `unitigs.bin` (direct access), `evidence.bin`, `unitigs.bin` again (`LayerEvidence::open`, `mphf_layer.rs:100`). ≈ **740** | whole partition | Only the MPHF. Pass 2 calls `hash_batch` (MPHF only, `mphf_layer.rs:258`) and `fill_sub_matrix` (matrix: `Implicit` for mono-genome, zero maps). Pass 1 only needs the `unitigs.bin` *path*. |
|
||||
| Writable columns, created up front | `partition_merge.rs:327`, `:349` → `PersistentBitVecBuilder::new` / `PersistentCompactIntVecBuilder::new` | `set_len` (sparse) + `MmapMut`, `n_src_total × (L_dst + 1)` ≈ **370** | whole pass 2 | Only the columns of the ≤ 2 sources being streamed (`max_open = 2`). |
|
||||
| dst layers, new-layer MPHF, sequential readers | various | ≈ 10 | variable | yes |
|
||||
|
||||
≈ **1 100 maps per partition**, > 95 % idle at any instant. × ~32 concurrent partitions (`--cpu-max 32`) ≈ 35 k, plus stacks and glibc arenas of several hundred pipeline threads. That is within reach of 65 530. Without a cap (192 cores) it is several times over. Consistent with run 2, **not measured**. Whether run 1's SIGBUS comes from the same exhaustion or from failure 4 below is **not established**.
|
||||
|
||||
## Weaknesses
|
||||
|
||||
1. **Source layers opened in full** although merge needs MPHF + matrix only. Evidence and both `unitigs.bin` maps are dead weight.
|
||||
2. **All sources opened up front, for the whole partition.** This contradicts the point of partitioning, which is to work on a bounded sub-part of the index. The cost of one partition grows linearly with the number of sources, whatever the partition size.
|
||||
3. **All destination columns created up front**, the same mismatch on the write side.
|
||||
4. **Sparse files written through `MmapMut`.** A page whose block cannot be allocated (quota, full FS, BeeGFS) faults with SIGBUS. That cannot be caught and reports nothing.
|
||||
5. **No bound on live mappings.** `obipipeline::throttle` only bounds open read streams.
|
||||
6. *(minor)* `obicompactvec::mmap_file::open_mmap_file` validates magic and header length, not file length against `n`.
|
||||
|
||||
Side finding: `merge_parallelism.md` documents a `MemoryBudget` semaphore and `--budget-fraction`. `obisys::MemoryBudget` still exists but `merge.rs` no longer uses it (scheduling is `PartitionRunner`). That page is stale on this point.
|
||||
|
||||
## Plan
|
||||
|
||||
**A. Source-group streaming in pass 2** (`obikmerge`). Process sources in consecutive groups of `G` (G = 2, matching `max_open`). Per group: open its layers, resume each dst matrix (existing layers + new layer), append its `src_n` columns, fill them through the existing pipeline, close, drop. Pass 1 opens no source layer: it builds paths through `KmerLayer::new(..).unitigs_path()` (valid on `Empty`). Live maps per partition become independent of the source count. Column order is unchanged (sources visited in order, columns appended in order), so the on-disk result is identical. Fixes 2, 3, 5.
|
||||
|
||||
**C. In-memory column builders** (`obicompactvec`). Under A only `G × src_n × (L_dst+1)` columns are alive. Build them in RAM (bits: n/8 B; counts: n B + overflow map) and write once with `write_all` to the `.building` path, then publish. No sparse file and no `MmapMut` on the merge write path, so `ENOSPC`/quota becomes an `io::Error`. The on-disk format must be byte-identical to the current builders' output. Exposed through `MatrixBuilder`/`ColBuilder` so `obikselect`/`obikfilter` can adopt it later. Fixes 4.
|
||||
|
||||
**B. Slots-only layer opening** (`obikindex`, the authority crate). **Deferred to a separate work item** (decision 2026-11-17). Consumers whose kmers are members by construction need MPHF + matrix, never evidence. The same `hash_batch` + `fill_sub_matrix` pattern without `find` appears in three crates: `obikmerge` (`partition_merge.rs:489`), `obikfilter` (`partition_iter.rs:106`), `obikphylo` (`family_scan.rs:289`). `family_scan.rs:402` hashes `variants`; whether those are guaranteed members is unchecked. Fixes 1.
|
||||
|
||||
- **Rejected: lazy evidence inside `MphfLayer`.** `find`/`confirm` return `Option<usize>`, so an evidence open error at first use cannot be propagated (panic, or a signature change across the query path). It would also add an "initialised?" branch to the hottest path, and hide in runtime state a distinction that is structural: external query (`find`, needs evidence) vs in-layer iteration (`index`/`index_batch`, no evidence).
|
||||
- **Rejected: a standalone parallel type** next to `KmerLayer`/`TypedLayer`. It would duplicate count/presence detection and matrix access, and two opening paths would drift.
|
||||
- **Retained: composition.** A slots layer = `MphfOnly` + matrix, carrying `hash_batch`, `fill_sub_matrix`, `n_cols`. The full layer = slots layer + evidence, keeping `find`/`confirm`. `KmerLayer` exposes both openings (`open()` / e.g. `open_slots()`) over a single count/presence detection. Open point: `MphfLayer::n` currently comes from the evidence (`evidence.len()` / `fingerprint.n()`). A slots layer must take it from the matrix (`n_rows`; `layer_meta.json` for `Implicit`), so the two must be checked to always agree.
|
||||
|
||||
Why deferred: after A, B only takes merge from ~8 to ~2 maps per partition, which is negligible. It touches the `obikindex` query path and needs its own session with `query` performance checks.
|
||||
|
||||
Order: oracle first (end-to-end merge golden output), then A → C. A alone removes the scaling problem; C removes the SIGBUS class.
|
||||
|
||||
## Implementation (2026-11-17)
|
||||
|
||||
**A — source-group streaming** (`obikmerge/src/partition_merge/`, split into `mod.rs` + `pass2.rs`). As planned: pass 1 builds paths only; the new-layer matrix is created with its absent columns and closed; pass 2 iterates `sources.chunks(SRC_GROUP = 2)`, resuming each dst matrix, checking `n_cols == n_dst_genomes + col_base` (`InvalidData` otherwise), and appending the group's columns. Prerequisite: `MatrixBuilder::n_cols` (`obicompactvec`).
|
||||
|
||||
**C — in-memory columns** (`obicompactvec`). New `mem_builder.rs` (`MemBitVecBuilder`, `MemCompactIntVecBuilder`), `add_mem_col` on both matrix builders and on `MatrixBuilder`, and `ColBuilder::MemBit`/`MemInt`. Format writing is factored rather than duplicated: `format::write_pciv_header`/`write_pciv_tail` (also used by `finalize_pciv`), `mmap_file::write_std_header`/`write_built`. Byte identity with the mmap builders is unit-tested for n ∈ {0, 1, 63, 64, 65, 10 000}, with overwrites and > 2 048 overflow entries. Merge uses `add_mem_col` for source columns; `add_absent_col` is unchanged (never written through its mapping).
|
||||
|
||||
Deviations from the plan:
|
||||
|
||||
- **No `sync_all` in `close`.** With one forced sync per column file, `merge_partitions` went from 7.4 s to 63.6 s on the benchmark; without it, 7.0 s (measured, macOS). The mmap builders never synced either. The `rename` still prevents publishing a partial file. Caveat: on a write-back network FS an allocation error may only surface at `close(2)`, which `File`'s drop ignores.
|
||||
- `add_mem_col` returns `ColBuilder`, not `io::Result<ColBuilder>`: it does no I/O.
|
||||
- The mmap count builder (`PersistentCompactIntVecBuilder`) writes in place, without `.building`; the in-memory one goes through `.building` + `rename`. Same final bytes.
|
||||
- `obikmerge`'s `matrix_builder_tests` were removed: they only exercised `obicompactvec::MatrixBuilder` and are covered by `obicompactvec/src/tests/{matrix_builder,mem_builder}.rs`. `tempfile` dev-dependency of `obikmerge` removed with them.
|
||||
|
||||
**Verification.** Two golden-output oracles, recorded on the unmodified code:
|
||||
|
||||
- `scripts/smoke_test_merge.sh`: synthetic, 4 genomes with shared fragments and counts ≥ 255, presence/count/`--dense`; frozen source indexes;
|
||||
- `scripts/bench_merge_oracle.sh`: 20 real specimen presence indexes, sorted `dump --debug` hash, 85 681 700 k-mers.
|
||||
|
||||
Both pass after every step. The existing `global_index_presence` (October binary) gives the same hash.
|
||||
|
||||
`merge_partitions` wall time on the benchmark: 7.40 s (ref) → 7.91 s (A) → 7.07 s (A + C) → 8.05 s (after the module split, same code). Single runs; the spread is noise. RSS 243–256 MB throughout. The benchmark is too small to show either the mapping or the memory effect.
|
||||
|
||||
**Live mappings per partition (estimate, not measured).** Assumptions: 186 mono-genome sources, 1 layer each, 1 pre-existing dst layer.
|
||||
|
||||
| | Source layers | Dst columns | Other | Total |
|
||||
|---|---|---|---|---|
|
||||
| Before | 186 × 4 ≈ 744 | 186 × 2 ≈ 372 (`MmapMut`) | ≈ 10 | **≈ 1 100** |
|
||||
| After A + C | 2 × 4 = 8 | 0 (in RAM) | ≈ 10 | **≈ 20**, independent of the source count |
|
||||
|
||||
B would take the 8 down to ~2.
|
||||
|
||||
**Follow-up — sparse pack (same day).** `pack_sparse_bit_matrix`/`pack_sparse_compact_int_matrix` used to open the columnar matrix directly, which maps every column file of a layer at once: `n_genomes` maps per partition, since layers are packed one after another. Both now call the dense pack first. It concatenates the column files one at a time without mapping them, so the sparse matrix is built from a single `matrix.pbmx`/`.pcmx` mapping. This deliberately reverses an earlier choice ("never materialise a packed copy just to read it back once"). It costs one extra write and read of the dense matrix: `pack` went from 1.11 s to 1.60 s on the benchmark (single run). As a side effect, a stale `matrix.pbmx` that predates newer columnar data is now re-packed instead of being used as is (`pack_bit_matrix` compares `n_cols`). Both oracles pass.
|
||||
|
||||
## Known issue (found while building the oracle, not fixed)
|
||||
|
||||
Re-merging a merge output fails with `I/O error: No such file or directory` in `merge_partitions`, in every mode (presence sparse, presence `--dense`, counts). Cause, verified: `finalize_indexed` packs the output matrices (`sparse_meta.json` + dict/PFIV files, or `matrix.pbmx`), so `presence/`/`counts/` no longer holds `meta.json` + `col_*`. When that index is the merge base, bootstrap copies the packed layout and `MatrixBuilder::resume` → `MatrixMeta::load` finds no `meta.json`. Pre-existing, independent of this refactor; deferred to a separate fix (decision 2026-11-17). The synthetic oracle (`scripts/smoke_test_merge.sh`) therefore omits the "merge AB, then AB C D" scenario.
|
||||
|
||||
Residual risk, accepted: read-only maps still SIGBUS if a file is truncated by another process during a merge. That is inherent to `mmap`.
|
||||
@@ -1,41 +0,0 @@
|
||||
# Merge: scaling to thousands of genomes (open, 2026-11-17)
|
||||
|
||||
Open design question raised during the mmap refactor (`merge_mmap_audit.md`). Target: merges of ~6 000 genomes of a few Gb each. Nothing here is implemented or measured. Figures are estimates, labelled as such.
|
||||
|
||||
## What scales with what (after the mmap refactor)
|
||||
|
||||
| Cost | Depends on | Status |
|
||||
|---|---|---|
|
||||
| Live source layers + dst columns per partition (merge pass 2) | `SRC_GROUP × src_n`, not the source count | fixed by plan A |
|
||||
| Pass-2 column RAM | `SRC_GROUP × src_n × N_part × (1/8 \| 1)` B | bounded, see below |
|
||||
| Pass-1 de Bruijn graph | ≥ ~19 B per *new* k-mer (`HashMap<u64, AtomicU8>`, verified in `obidebruinj/src/debruijn/graph.rs`) | unchanged |
|
||||
| Columnar intermediate files | `n_genomes × n_layers × n_partitions` (6 000 × 2 × 256 ≈ 3 M files, estimate) | **does not scale** |
|
||||
| Sparse `pack` | used to map **all** columns of a layer at once (`ColumnarBitMatrix::open`); layers are packed one after another, so `n_genomes` maps per partition (186 × 192 concurrent ≈ 36 k, estimate) | **fixed 2026-11-17**: now goes columnar → dense `matrix.pbmx` (sequential concatenation, unmapped) → sparse, one mapping per partition |
|
||||
|
||||
`N_part`: k-mers of one dst partition over all layers. Pass-2 column RAM stays below pass-1 graph RAM for one-shot merges. It dominates only for **incremental** merges, where few genomes are added to a large index: the new layer is small, the existing layers are large. Example (estimate): `N_total = 1e12`, counts, 192 concurrent partitions → ~1.5 TB. A memory-budgeted concurrency cap is needed there whatever the column strategy. `obisys::MemoryBudget` exists but is no longer used by merge.
|
||||
|
||||
## Why a columnar intermediate at all
|
||||
|
||||
The sparse format (`bitmatrix/sparse.rs`) is row-major and deduplicated: per slot, either one genome (`singleton.pfiv`) or a `dict_id` (`multi.pfiv`) into a dictionary of distinct genome sets (`dict_offsets` + `dict_values.bin`). Its builder only offers `push_row(genomes)`, i.e. the **complete** genome set of a row, rows in **slot order**.
|
||||
|
||||
Merge discovers values **column by column** (source by source), in **random slot order** (MPHF order). The columnar files are a transposition buffer: they absorb random per-column writes, and `pack` transposes them row by row.
|
||||
|
||||
## Options to build the final format directly
|
||||
|
||||
1. **All sources at once.** That is exactly what plan A removed. No.
|
||||
2. **Whole matrix in RAM.** `N_part × n_genomes / 8` B per partition: ~3 TB at 6 000 genomes and `N_part = 4e9` (estimate). No.
|
||||
3. **External sort of `(slot, genome)` pairs.** Volume ∝ non-zeros ≈ Σ genome k-mers (~6 000 × 3e9). That is huge, while the final format is small precisely because genome sets repeat. No.
|
||||
4. **Incremental sparse update per source group** (most promising). Per group:
|
||||
- fill the group's columns in RAM (as now);
|
||||
- stream the partition's existing sparse matrix **sequentially in slot order**;
|
||||
- for each row: new set = old set ∪ group bits. Through the dictionary this is a transition `(old dict_id, group bit pattern) → new dict_id`, which can be memoised in a table;
|
||||
- write the new sparse matrix sequentially, then publish.
|
||||
|
||||
No columnar files, no `pack`, no per-genome mapping, sequential I/O only. It also removes the re-merge bug: it starts from an existing sparse matrix instead of requiring a columnar one.
|
||||
|
||||
The cost is **one full pass over the partition's sparse matrix per group**. With `SRC_GROUP = 2` and 6 000 genomes that is 3 000 rewrites, far too many. Groups must therefore be large (64–256 sources), and group column RAM grows with them: `group × N_part / 8` B, e.g. 32 GB per partition for 64 sources at `N_part = 4e9` (estimate). One parameter trades passes against RAM, and it should be sized from a memory budget. The count matrix (`sparse_intmatrix.rs`) has its own format and needs the same analysis.
|
||||
|
||||
## Related findings
|
||||
|
||||
- The module header of `bitmatrix/sparse.rs` says "not used by any production code path yet". This is stale: `finalize_indexed` packs to sparse by default.
|
||||
- The new layer of a merge is not byte-reproducible run to run: unitig segmentation varies, so the MPHF, evidence and matrix row order vary with it (measured 2026-11-17, `layer_1/*` differs in all 256 partitions; `layer_0` and metadata are identical). `obikmer index` shows the same behaviour. Merge oracles therefore compare sorted dumps, not file hashes. Where the non-determinism originates (parallel graph fill or unitig walk) is not established.
|
||||
@@ -1,369 +0,0 @@
|
||||
# MPHF batch queries — site inventory and expected gains
|
||||
|
||||
Investigation of 2026-10-14 (`ptr_hash` 2.1.2, `epserde` 0.13). Question: where would batched MPHF queries (software prefetch of the random accesses) save time? Status of each claim is marked **[read]** (verified in source), **[computed]** (derived from stated sizes) or **[estimate]** (reasoning, not measured). Measurements, once made, are in the last section.
|
||||
|
||||
## 1. What `ptr_hash` 2.1.2 actually offers [read]
|
||||
|
||||
`Mphf = PtrHash<u64, CubicEps, F = CachelineEfVec, Xxh3Int, V = Vec<u8>, SINGLE_PART = false, REMAP = true>`.
|
||||
|
||||
- **`V` is the pilot array** (one byte per bucket), **`F` is only the remap storage**. A layer's `mphf.bin` (~2.4 bits/key **[computed]**: 40 MB for ~130 M keys) is therefore almost entirely a byte array indexed by bucket. The batch APIs prefetch `pilots[bucket]` — the right line. The remap (read only when `slot >= n`, a small fraction of keys) is **not** prefetched.
|
||||
- `index_batch<K>(xs: [Q; K]) -> [usize; K]` — stable, not feature-gated; hash K keys, prefetch K pilots, query. Remainder (`len % K`) is the caller's job.
|
||||
- `index_stream<B>(keys: IntoIterator) -> impl Iterator` — ring buffer of B keys; handles head and tail itself; accepts any iterator. **Its `next()` is `unimplemented!`**: only fold-based consumption works (`for_each`, `fold`, `sum`). `collect()`, `zip`, `for x in it` panic. Consume with `for_each` into a pre-sized `Vec`.
|
||||
- `index_batch_exact` needs the nightly `unstable` feature; `index_batch_exact2` is `#[doc(hidden)]`. Neither is usable.
|
||||
- Prefetch is `prefetch-index 0.2.1`: `_mm_prefetch(T0)` on x86_64, asm on aarch64, **no-op elsewhere**.
|
||||
- **`SINGLE_PART`**: neither `index_stream` nor `index_batch` branches on it — both always run the multi-part `bucket()`/`slot()`. Only `index()` has the faster single-part path. So with batched queries the `SINGLE_PART = true` advantage disappears, and `false` (our choice) costs nothing while keeping parallel construction (`true` also panics with sharding).
|
||||
- Old `ptr_hash` 1.1.0 (used by `tools/migrate-mphf`) has the same family with an extra `MINIMAL` const: `index_stream<B, MINIMAL, Q>`, `index_batch<K, MINIMAL, Q>`. No new dependency needed there.
|
||||
|
||||
```rust
|
||||
// consuming index_stream: fold-based only
|
||||
let mut out = Vec::with_capacity(kmers.len());
|
||||
mphf.index_stream::<32, _>(kmers.iter().map(|k| k.raw()))
|
||||
.for_each(|slot| out.push(slot));
|
||||
```
|
||||
|
||||
## 2. Existing "batch" APIs that are not batched [read]
|
||||
|
||||
`MphfLayer::hash_batch` and `MphfOnly::index_batch` loop `.index()` one kmer at a time. Consumers: `obikfilter/partition_iter.rs` (`iter_layer_kmers`), `obikmerge/partition_merge.rs` (pass 2: `hash_batch`, `index_batch`), `obikphylo/siblings/algorithms/family_scan.rs` (two sites).
|
||||
|
||||
## 3. Cost chain of a membership query (`MphfLayer::find`) [read]
|
||||
|
||||
1. `pilots[bucket]` — the only access `ptr_hash` prefetches.
|
||||
2. Exact/Hybrid-exact: `evidence.decode(slot)` — 4 B/slot, ~520 MB per 130 M-slot layer **[computed]**, random mmap read.
|
||||
3. `verify_canonical_kmer` → `block_offsets[...]` (O(1) with the default `--block-size 1`; a forward scan of up to `2^block_bits` records otherwise).
|
||||
4. `extract_kmer_raw` — random read in the `unitigs.bin` mmap.
|
||||
|
||||
Approx/Hybrid-fast: pilots then `fingerprint.get(slot)` — one extra random access.
|
||||
Each stage is independent across kmers, so a staged batch (slots → prefetch evidence → decode, prefetch offsets/unitigs → verify) can overlap all of them. `ptr_hash` only helps with stage 1; stages 2–4 need prefetch support in `obikindex`/`obiskio` (not existing). Prefetch on an unmapped (cold, not paged-in) mmap page is dropped by the CPU, so none of this helps on a cold index.
|
||||
|
||||
Layer sizes: `layer_0` MPHF 2–3 MB (cache-resident), `layer_1` ~40 MB (beyond one CCD's L3 on EPYC-class machines — **hypothesis**, prod CPU not verified, and contended by 192 threads). Gains are expected mainly on `layer_1`. For branch-free independent loops (`hash_batch`) out-of-order execution already overlaps several misses, so the explicit prefetch gains less **[estimate]** than for branchy loops (`find` with early `break`).
|
||||
|
||||
## 4. Call-site inventory
|
||||
|
||||
(a) already batched; (b) batch buildable; (c) hard/pointless.
|
||||
|
||||
| Site | What | Class |
|
||||
|---|---|---|
|
||||
| `mphf_layer.rs` `hash_batch`, `MphfOnly::index_batch` | loop of `.index()` | (a) |
|
||||
| `obikfilter/partition_iter.rs:106` | `hash_batch`, 32768 kmers | (a) |
|
||||
| `obikmerge/partition_merge.rs:489, 521` | `hash_batch`, `index_batch`, 4096 | (a) |
|
||||
| `obikphylo/.../family_scan.rs:285, 397` | `hash_batch` | (a) |
|
||||
| `obikquery/query_layer.rs` stage 1 | `find_in_layer` per kmer, layer by layer, `break` on hit | (b) |
|
||||
| `partition_merge.rs` `find_batch_grouped` | `find_slot` per kmer over dst layers | (b) |
|
||||
| `partition_merge.rs` (`build_graph` filter) | `all(find_slot.is_none())`, every layer, per kmer; `build_graph` takes `Fn(CanonicalKmer)->bool` | (b), API change |
|
||||
| `family_scan.rs` slow path; `annex.rs` | `find_slot` / `find_in_partition` per variant | (b) |
|
||||
| `mphf_layer.rs` `build`, `build_exact_evidence`, `build_approx_evidence` | `mphf.index` then random writes (evidence, fingerprint, `seen`) | (b), chunked |
|
||||
| `obikindexer/counter/count.rs:47` | `mphf.index` then `counts.get/set` | (b), hard (needs (kmer, count) pairs) |
|
||||
| `obikindexer/extensions/private_builder.rs:177` | abundance filter | (b), conditional |
|
||||
| `private_builder.rs:196`, `verify.rs`, `IndexCache::find_in_*` | per-kmer by nature / sampled | (c) |
|
||||
| `tools/migrate-mphf/plan.rs:41-42` | `old.index` + `new.index` per kmer, random writes | (b) |
|
||||
|
||||
Dead or suspect code found along the way (candidates for removal, not removed):
|
||||
|
||||
- `MphfLayer::find_strict` — no caller.
|
||||
- the single-kmer `hash` chain (`MphfLayer::hash` → `TypedLayer::hash` → `KmerLayer::hash` → `IndexCache::hash`) — no caller outside the chain.
|
||||
- `IndexCache::find` — no caller.
|
||||
- `obikmerge/src/merge_layer/` — not declared in `lib.rs` (`mod merge; mod partition_merge;` only); still imports the `LayeredMap` removed in 2026-08. To be confirmed by the compiler.
|
||||
|
||||
## 5. Interactions
|
||||
|
||||
- **Verification chain**: the right shape is one `find_batch(&[CanonicalKmer], &mut Vec<Option<usize>>)` on `MphfLayer`, staged as in §3, so every consumer benefits (rule: enrich the shared API, no local loops).
|
||||
- **Memory**: callers already hold bounded batches (4096 / 32768). A buffer-reusing variant avoids one `Vec<usize>` per call. `index_stream` itself only allocates a B-element ring on the stack.
|
||||
- **Parallelism**: no new parallelism. Batches already come out of `obipipeline` stages; the change lives in the transform body. Construction pass 2 is sequential inside a partition: chunk the streamed iterator (1–4 k kmers), never collect the layer. In `query_partition_with` the unique-kmer `HashMap` is already materialised; iterate it chunk-wise.
|
||||
|
||||
## 6. Benchmark
|
||||
|
||||
`benchmark/bench_mphf.sh LABEL` times, on the 20-genome benchmark data, `index` (2 genomes), `merge` (20 presence indexes), `query` (presence/count × 2 genomes), `filter`, `convert` (approx/exact round trip) and `phylo` (`--sibling-annex`, `snp-jc`), `REPS` times each, and records order-independent output digests that must not change across labels. See the results section below.
|
||||
|
||||
**Limit of this data set**: 20 small genomes → per-partition `layer_1` MPHFs of ~100 KB, fully cache-resident on the dev machine (Apple M3 Max, aarch64). This benchmark can detect regressions and the effect on instruction-level overlap; it **cannot** measure the DRAM-latency gain that motivates prefetch at 40 MB/layer. A faithful test needs a prod-scale layer (see the throwaway benchmark below).
|
||||
|
||||
### Throwaway prod-scale benchmark (not written)
|
||||
|
||||
Open one real partition's `layer_1` (~130 M keys) and `layer_0` (~8 M) with `MphfOnly`/`MphfLayer`; key sets: present (sample of `iter_kmers`, random order and sequential order) and absent (random, checked absent); measure kmers/s for `index`, `index_stream::<B>` (B ∈ 8…128), `index_batch::<K>` (K ∈ 8…64), and the full `find` vs staged `find_batch`; at 1, 8, 32, 192 threads; pinned to one NUMA node, then spread; warm and cold page cache stated; 5 repetitions, min/median; sizes of `mphf.bin`, `evidence.bin`, `unitigs.bin(.idx)` reported; first check `index_stream` slots == `index` slots. The ~8–12 ns/key figure of the PtrHash paper (quoted in `mphf.md`) is a paper claim, not tied to our sizes or machines.
|
||||
|
||||
## 7. Not verified
|
||||
|
||||
No measurement at prod scale; prod CPU architecture and cache sizes; evidence mode and `--block-size` used in prod; whether `migrate-mphf` already ran; share of total index/merge/query time spent in these loops; `count.rs` f0 per partition at prod scale; whether `merge_layer/` is dead (only the head of `lib.rs` read).
|
||||
|
||||
## 8. First improvement and measurements (2026-10-15)
|
||||
|
||||
**Implemented** (query stage 1, the top-ranked site; the same change adds the plumbing the other sites will reuse):
|
||||
|
||||
- `MphfLayer` now holds a `MphfOnly` instead of its own `MemCase<Mphf>` (one mmap-open path, one place for the stream call).
|
||||
- `MphfOnly::for_each_slot(kmers, f)` wraps `index_stream::<32>`; `index_batch`/`hash_batch` now really use it (they previously looped `.index()`). Callback-shaped because `index_stream`'s `next()` is `unimplemented!`.
|
||||
- `MphfLayer::find_batch` / `TypedLayer::find_slot_batch` / `KmerLayer::find_slot_batch`; the mode-dependent half of `find` is factored into `MphfLayer::confirm`, shared by both.
|
||||
- `query_partition_with` stage 1 is layer-major over chunks of 4096 unique kmers: each layer receives the still-unresolved kmers as one batch (first hit wins, `n_mphf_calls` unchanged).
|
||||
- Not done: staged prefetch of evidence/offsets/unitigs (needs a prefetch primitive; `prefetch-index` is already in the tree via `ptr_hash`, but adding it to `obikindex` is a dependency decision), and the other (b) sites.
|
||||
|
||||
**Measurements** (`benchmark/bench_mphf.sh`, `benchmark/summarize_bench.py`; Apple M3 Max, aarch64; 20 genomes at 15x). Interleaved A/B, 8 runs each, E. coli reads queried against the 20-genome presence index:
|
||||
|
||||
| | wall (s) | user CPU (s) |
|
||||
|---|---|---|
|
||||
| before | 5.94–6.02 (median 5.96) | 10.95–11.07 |
|
||||
| after | 5.63–6.00 (median 5.67) | 10.33–10.66 |
|
||||
|
||||
About −5 % wall and −5.5 % CPU, ranges non-overlapping **[measured]**. All output digests (query ×4, merge, filter, convert round trip, phylo) are identical before and after.
|
||||
|
||||
**Reading the number.** Small by construction: this data set's `layer_1` MPHFs are ~100 KB per partition, cache-resident, and query time is dominated by reading and k-merising the reads. The gain seen here is instruction-level overlap, not DRAM-latency hiding. The latter, the reason for the whole exercise, remains **unmeasured** and needs a prod-scale layer (§6).
|
||||
|
||||
**Noise caveat.** With a load average of 16–22 from unrelated processes, single-run walls of `index`, `merge` and `filter` moved ±10–40 % between identical binaries; only interleaved repeated A/B runs were trustworthy. The `index`/`merge`/`filter`/`convert`/`phylo` rows of the first comparison are noise, not effects (`index` pass 2 was not changed at all; a later interleaved `filter` A/B was inconclusive, 13.0 s vs 13.8 s medians with a 10–20 s spread).
|
||||
|
||||
**Stale benchmark indexes.** The indexes already under `benchmark/run/` (`specimen_index_*`, `global_index_*`, built 2026-08-28) carry `ptr_hash` 1.x MPHFs and fail with "Major version mismatch" on current binaries (`tools/migrate-mphf` converts them). `bench_mphf.sh` therefore builds its own copies under `run/bench_mphf/data/` on first use.
|
||||
|
||||
## 9. Benchmark procedure
|
||||
|
||||
Prerequisites: the simulated reads (`benchmark/run/simulated_data/`) and query reads (`benchmark/run/query_data/`) produced by `gmake simulate simulate_query`; macOS BSD `time -l` and `md5 -q` (the script uses both), Python 3.
|
||||
|
||||
1. Build the binary to measure and **copy it** under a name that identifies it (cargo overwrites `target/release/obikmer`): `cp src/target/release/obikmer benchmark/run/bench_mphf/obikmer_<label>`.
|
||||
2. `BINARY=$PWD/run/bench_mphf/obikmer_<label> benchmark/bench_mphf.sh <label>` (from `benchmark/`; `REPS=3` by default, `CPU_MAX=n` to restrict cores). The first call builds the shared data set in `run/bench_mphf/data/` with that binary (≈ 2 min); keep it for every label — the on-disk format must not differ between the binaries compared, otherwise delete `data/` and rebuild.
|
||||
3. Output: `run/bench_mphf/<label>/results.csv` (wall and peak RSS per step and repetition, plus one row `step:stage` per Reporter stage) and `checksums.txt`.
|
||||
4. Compare: `python3 benchmark/summarize_bench.py <ref_label> <new_label>` (medians and ratio) and `diff run/bench_mphf/<ref>/checksums.txt run/bench_mphf/<new>/checksums.txt` — **any difference is a bug**, not a performance matter.
|
||||
5. A ratio is only a candidate effect. Check the spread by running the **same** binary twice under two labels (the script's own noise floor was ±10–40 % per step on a loaded laptop), or confirm with interleaved A/B runs of the single command (alternate the two binaries, ≥ 8 runs, compare ranges of wall and user CPU from `/usr/bin/time -l`).
|
||||
6. Rule of thumb for this data set: only `query` (≈ 5 s, stable to ±1 %) resolves effects of a few percent; `index`, `merge`, `filter`, `phylo` need interleaved runs or a quiet machine.
|
||||
|
||||
Step → code exercised:
|
||||
|
||||
| Step | Code on the MPHF path |
|
||||
|---|---|
|
||||
| `index_*` (`:index` stage) | `MphfLayer::build` pass 2 |
|
||||
| `merge_20_presence` | `find_batch_grouped`, `build_graph` filter, `hash_batch`, `index_batch` |
|
||||
| `query_*` | `query_partition_with` stage 1 |
|
||||
| `filter_species` | `iter_layer_kmers` → `hash_batch` |
|
||||
| `convert_to_*` | `build_exact_evidence`, `build_approx_evidence` |
|
||||
| `phylo_sibling_annex` | `annex.rs` `find_in_partition` per variant |
|
||||
| `phylo_snp_distance` | `family_scan.rs` `hash_batch`, `find_slot` |
|
||||
|
||||
## 10. Second step — construction pass 2 (2026-10-15)
|
||||
|
||||
**Implemented**: `layer/slotted.rs`, `SlottedKmers`, an iterator adapter yielding `(slot, kmer, chunk_id, rank)` for the wrapped `(kmer, chunk_id, rank)` stream, computing slots 4096 kmers at a time through `index_stream::<32>` (bounded memory, never the whole layer). The five loops of `MphfLayer::build` (Exact, Approx, Hybrid), `build_exact_evidence` and `build_approx_evidence` now iterate it instead of calling `mphf.index` per kmer. Output digests unchanged.
|
||||
|
||||
**Measured** (interleaved A/B against the first-step binary, 8 runs each, wall / user CPU):
|
||||
|
||||
| Command | before | after |
|
||||
|---|---|---|
|
||||
| `convert --exact-evidence` (20-genome index) | 0.56–0.68 s / 2.22–2.67 s | 0.59–0.74 s / 2.25–2.52 s |
|
||||
| `index` E. coli reads | 2.45–2.63 s / 13.1–14.3 s | 2.48–2.75 s / 13.2–14.0 s |
|
||||
|
||||
**No measurable effect** on this data set (differences within the run-to-run spread; if anything `convert` is ≈ 4 % slower in wall time). Expected here: the MPHF is cache-resident and `index`/`convert` time is dominated by other stages (scatter, dereplication, counting, unitig decoding, the random evidence/`seen` writes themselves). The construction gain, if any, only exists where the pilot array does not fit in cache, and the random *writes* to `evidence.bin`/`seen`/`fingerprint.bin` (not prefetched by `ptr_hash`) probably dominate there too. Whether to keep this change is therefore a decision to take after the prod-scale benchmark (§6), not on this data.
|
||||
|
||||
## 11. Why the two changes gain so little (profiling, 2026-10-15)
|
||||
|
||||
Measured with macOS `sample` (1 ms stack sampling) on the 20-genome data, one run each — coarse, but the orders of magnitude are unambiguous.
|
||||
|
||||
**Query.** In a 2 s window on the `query` stage (16 threads, ≈ 3 100 active samples against ≈ 55 000 samples of idle waiting):
|
||||
|
||||
- `query_partition_with` ≈ 260 samples (8 % of active CPU), of which `find_batch` 217, of which **`verify_canonical_kmer` 185** and everything else in `find_batch` (the `ptr_hash` stream and `confirm`) **32**.
|
||||
- `verify_canonical_kmer` is the single largest active function of the window (399 top-of-stack samples). The MPHF lookup is ≈ 1 % of active CPU: even a free MPHF could not give more.
|
||||
- Reason: per-partition pilot arrays are ~100 KB (≈ 25 MB over 256 partitions) and stay cached; the evidence/`unitigs.bin`/offset accesses of the verification (§3, stages 2–4) are the ones that miss. This data set already has the problem on the verification side, not on the MPHF side.
|
||||
- The threads are mostly idle: the stage is bounded by reading/parsing/k-merising the reads (`rolling_stat`, entropy tracker, string formatting) and by sequential parts, not by lookups.
|
||||
|
||||
**What produced the −5 %.** Interleaved A/B, 8 runs each, median wall: original 6.4 s; chunked layer-major restructuring with plain per-kmer `index` (stream disabled) 6.0 s; restructuring + `index_stream` 5.85 s. **Most of the gain comes from the restructuring, not from the prefetch** (the prefetch adds ≈ 2 %, at the limit of the noise). Hypothesis, not isolated: the old loop called `IndexCache::get_layer` — an `Arc<KmerLayer>` clone, i.e. an atomic increment/decrement on a refcount shared by all threads — once per kmer per layer; the batched loop does it once per layer per chunk of 4096.
|
||||
|
||||
**Index (pass 2).** In a 1 s window late in the `index` stage, `SlottedKmers` appears with ≈ 7 samples out of 304 per thread (≈ 2 %), against ≈ 200 for the de Bruijn graph work (`obidebruinj`) and ≈ 35 for `ptr_hash::build` (pass 1, MPHF *construction*). Pass 2 is a few percent of `index` and `convert` here: nothing to win even if its lookups were free. `convert` is `build_*_evidence` on 2 MB-scale layers, same conclusion.
|
||||
|
||||
**Consequences for priorities.**
|
||||
|
||||
1. On this data the next lever is the verification (`verify_canonical_kmer`, evidence decode), not the MPHF — consistent with §3, now observed.
|
||||
2. Hot `Arc` clones in per-kmer paths (`get_layer`, `IndexCache::find_in_layer`, `find_slot` loops in merge and phylo) are worth auditing independently of MPHF batching.
|
||||
3. The DRAM-latency gain of prefetch is untested until a prod-scale layer is available (§6).
|
||||
|
||||
## 12. Calibration: a single skim at fewer partitions (2026-10-15)
|
||||
|
||||
`Vaccinium myrtillus` skim (2 × ~440 MB fastq.gz, `theta 0`), indexed with the current binary, to find the partition count that gives prod-sized MPHFs (`--partitions` is a power of two; MPHF size ≈ 2.4 bits/key **[measured]**):
|
||||
|
||||
| `-p` | keys / partition | `mphf.bin` / partition | index wall | peak RSS |
|
||||
|---|---|---|---|---|
|
||||
| 256 (default) | ≈ 1.4 M **[computed from p=16]** | ≈ 0.4 MB **[computed]** | – | – |
|
||||
| 16 | 22.1 M | 6.65 MB | 120 s | 12.3 GB |
|
||||
| 4 | 86–90 M | 25.9–27.0 MB | 171 s | 19.5 GB |
|
||||
|
||||
Total ≈ 354 M distinct kmers for one skim. `-p 4` is already in the 30–45 MB range of the prod `layer_1` MPHFs; `-p 2` would give ≈ 53 MB. Indexes kept in `benchmark/run/bench_mphf/skim_p16` and `skim_p4` (2.3 GB each). Note the index stage itself is dominated by graph and MPHF construction, as in §11.
|
||||
|
||||
## 13. Query on a real-scale index (2026-10-15)
|
||||
|
||||
**Index.** Working index `phyloskims_sal_vac_sav` (91 genomes of Salix/Vaccinium, 256 partitions × 2 layers, exact evidence, 180 GB, ptr_hash 1.x MPHFs). Converted with `tools/migrate-mphf` **on an APFS clone** (`cp -c`, 0.8 s, no space used until written; the tool works in place and sets the annexes aside, so the original was left untouched — verified by mtime): 512 layers in 10 min (1311 % CPU), ≈ 57 GB of new data. Clone at `/Users/coissac/travail/obiskim/data/phyloalps/bench_mphf/phyloskims_sal_vac_sav_v2`; its sibling annex is in `stale_annexes_pre_ptrhash2/` and would need rebuilding to be used. Per partition: `layer_1` ≈ 31 M keys / 9.3 MB MPHF, `layer_0` ≈ 5.6 M keys / 1.7 MB — larger than the 20-genome set (100 KB) but 4× smaller than the production layers (~130 M keys, ~40 MB).
|
||||
|
||||
**Query** (2 × 48 MB of *V. oxycoccos* reads, `--chunk-size 8`, AC power, 3 interleaved runs per binary, identical sorted output for all three):
|
||||
|
||||
| binary | wall (s) | user CPU (s) | `query` stage (s) |
|
||||
|---|---|---|---|
|
||||
| original | 28.2–30.5 | 70.9–74.4 | 14.6–17.1 |
|
||||
| + batched query stage 1 | 26.9–28.0 | 48.1–48.6 | 13.6–14.4 |
|
||||
| + `PartitionKmers` memory fix, build-side `SlottedKmers` | 26.4–26.7 | 41.9–42.5 | 12.9–13.3 |
|
||||
|
||||
The `query` stage is ≈ 50 % of wall time; the rest (≈ 13 s) is opening the index (next paragraph). Stage-only gain ≈ −6 % then −11 %; CPU time −33 % then −41 %. **The CPU drop is much larger than the wall drop** and is not explained by MPHF prefetch alone (§11: restructuring, probably the per-kmer `Arc` clones of `get_layer`, not isolated); the original binary was spending CPU in contention that wall time hid.
|
||||
|
||||
**Index-open cost (not related to the changes above).** A query of a single 64-base sequence on this index takes **15 s and a 27 GB peak footprint**. `vmmap` during the open: 24.5 GB of `MALLOC_LARGE` in 1 481 regions (≈ 3 per layer, 7.3 GB already swapped out), i.e. heap allocations made by `IndexCache::new` for every layer of every partition whatever the query. Known contributors: `UnitigFileReader` reads each `.idx` into a `Vec` (≈ 30 MB for `layer_1`, ≈ 9 GB over 512 layers); the remaining ~15 GB are in the presence-matrix open path (not yet attributed). Opening only the partitions a query touches is not an option for a read set that spans all of them, but these structures could be mmapped instead of copied.
|
||||
|
||||
## 14. Index-open memory: `.idx` offsets and `dict_values` mapped, not copied (2026-10-15)
|
||||
|
||||
**Cause** (§13, now attributed): of the 27 GB opened by a one-sequence query on the 91-genome index, **18.2 GB** were `unitigs.bin.idx` read into a `Vec<u32>` **twice** per exact-evidence layer (`MphfLayer::open` builds two `UnitigFileReader`s of the same file — one for the evidence check, one for `iter_kmers`/`find_strict`; both go through `open` → `open_direct_access` → `read_idx` → `fs::read` + copy) and **7.7 GB** were `dict_values.bin` read with `fs::read` in `PersistentSparseBitMatrix::open`. Computed from the real file sizes: 2 × 9.10 + 7.71 = 25.9 GB, against 27.4 GB measured. Everything else the matrix opens (`PersistentRankSelectBitVec`, `PersistentFixedIntVec`, `EliasFano`) was already mmapped.
|
||||
|
||||
**Change.**
|
||||
- `obiskio::unitig_index::reader`: `block_offsets: Vec<u32>` → `BlockOffsets`, a view into the memory-mapped `.idx` (header parsed once, offsets read little-endian on access, no unsafe, no alignment requirement). Truncation is checked at open, so later accesses cannot run past the map. Both readers of a layer now share the same page-cache pages.
|
||||
- `obicompactvec::bitmatrix::sparse`: `dict_values: Vec<u8>` → `Option<Mmap>` (`None` for an empty file, which cannot be mapped), read through `dict_bytes()`.
|
||||
|
||||
**Measured** (same session, same inputs; sorted query outputs identical, md5 `e9e7c35b`):
|
||||
|
||||
| | open only (1-sequence query) | 2×48 MB reads, `--chunk-size 8` |
|
||||
|---|---|---|
|
||||
| before | 10.6–15.7 s, **27.4 GB** footprint | 27.7 s (51 s cold), 30.9 GB, `query` stage 13.8 s |
|
||||
| after | **0.26–0.81 s, 0.04 GB** | **10.7 s, 3.6 GB**, `query` stage 9.95 s |
|
||||
|
||||
Tests: `cargo test --workspace` green; `scripts/smoke_test_index.sh` PASS; `bench_mphf.sh` (index, merge, query ×4, filter, convert round trip, phylo) completes with all output digests identical to the baseline run, i.e. no failure from files mapped while rewritten (the risk of mapping instead of copying: a reader holding a `.idx` map while the same path is truncated and rewritten would fault). The per-step ratios of that run vs the earlier baseline mix this change with the machine moving from battery to AC power and must not be read as its effect.
|
||||
|
||||
## 15. Unbuffered reads: `index.meta` and superkmer files (2026-10-15)
|
||||
|
||||
Follow-up of §14 (search for the same family of problems: file content handled inefficiently at open). Static search over `src/` for `fs::read`, `read_to_end`, `load_full`, `from_reader`, bare `File` readers and mmap `to_vec`: every persistent structure of `obicompactvec`, `obikindex` and the `obikphylo` annexes is already mmapped; the two real findings were *unbuffered* readers.
|
||||
|
||||
**1. `IndexMeta::read_full` (`obikindex/src/index/meta.rs`)** passed a bare `File` to `serde_json::from_reader`, which issues **one `read` syscall per byte**. `index.meta` of the 91-genome index is 20 KB and is read afresh by every `state()`/`genomes()` call (stateless design, §obikindex audit), so each call cost ≈ 0.35 s here (macOS). Profile of `obikmer dump --head 10` (`sample`): ≈ 20 000 samples in `read`, called from `Bytes<File>::next` ← `serde_json` ← `IndexMeta`. Fix: `fs::read` then `serde_json::from_slice`. **`dump --head 10`: 9.0–10.0 s (91–102 s of system CPU) → 0.16–0.84 s**; the same change applies to `SKFileMeta::read` (`obiskio/src/meta.rs`, per-superkmer-file sidecars).
|
||||
|
||||
**2. `SKFileReader::open` (`obiskio/src/reader.rs`)**: for an uncompressed file `niffler::get_reader` returns the sniffed bytes chained to the bare `File`, and `read_superkmer` reads each field with its own small `read_exact`. Fix: wrap the file in a 64 KiB `BufReader` before `niffler`. Effect on `index` (E. coli, 4 interleaved runs each): `count_kmer` 0.51–0.56 s → 0.42–0.50 s (≈ −12 %), total wall unchanged (≈ 1.95 s). Small, low risk, kept.
|
||||
|
||||
Tests: `cargo test --workspace` 549 passed / 0 failed; `scripts/smoke_test_index.sh` PASS; query output on the 91-genome index identical (md5 `e9e7c35b`).
|
||||
|
||||
**Probes of other commands on the 91-genome index (180 GB, 256 partitions)**: `phylo --distance jaccard` peaks at 155–190 MB of footprint while running and finishes in < 30 s; `utils --stats` 130–150 MB. `filter` (and by the same check `select`, `merge`) refused the index — "source index is not fully built" — with the unmodified baseline binary too: this index's `index.meta` predates the `state` field, so they could not be probed.
|
||||
|
||||
**Observations not investigated**: `index` on E. coli shows ≈ 9 s of *system* CPU in 2 s of wall time (both before and after these changes), consistent with the `swtch_pri` (yield) spinning seen in the `index` profile of §11 rather than with I/O; and wall times of the same query on the same binary varied 4.8 s ↔ 10.7 s ↔ 43 s depending on the state of the page cache, so only same-session comparisons are meaningful.
|
||||
|
||||
## 16. Profile of `index` on E. coli reads (2026-10-15)
|
||||
|
||||
`sample` (1 ms stacks) of `obikmer index` on the E. coli K-12 simulated reads (2 × 15x, 16 threads, one run; the `index` stage lasted 2.4 s under the sampler vs ≈ 0.9 s normally, so proportions are approximate). Stage walls: scatter 0.40 s, dereplicate 0.14, count_kmer 0.60, index 2.36.
|
||||
|
||||
**Threads are mostly idle.** Of ≈ 30 800 thread-samples: 24 900 blocked or yielding, 5 900 active. The ≈ 9 s of *system* CPU per run (§15) is `swtch_pri` (`sched_yield`) — 4 550 samples: ≈ 70 % rayon workers spinning for work (`WorkerThread` wait loop), ≈ 28 % `crossbeam` bounded-channel send/recv spinning, the rest in `obidebruinj` thread-locals. It is waiting, not I/O: no `read` samples remain. It costs no wall time on an otherwise idle machine, but burns CPU (visible in `user + sys` accounting and on a shared 192-core machine).
|
||||
|
||||
**Where the active CPU goes** (share of the 5 900 active samples):
|
||||
|
||||
| | share |
|
||||
|---|---|
|
||||
| xxh3 hashing of `CanonicalKmer` for the de Bruijn graph's `HashMap` (callers: `count_neighbors` 791, `hash_one` 358, `walk` 285, `unitig_iter` 252, `push` 164, `unitig_nucleotides` 75, rehash 68; plus `xxh3_stateful_update` 300) | ≈ 34 % |
|
||||
| `debruijn::count_neighbors` itself (inclusive of its hashing ≈ 2 300 samples: `compute_degrees_and_mark_starts`) | 12.5 % own |
|
||||
| `ptr_hash::build` (MPHF construction) | 4.8 % |
|
||||
| crossbeam channel send | 4.8 % |
|
||||
| `open` / `close` / `write` (many small files over 256 partitions) | ≈ 9 % |
|
||||
| debruijn `unitig_iter` / `walk` own time | 8.3 % |
|
||||
|
||||
So at this scale the cost of `index` is the **de Bruijn graph's hash table** (several lookups of neighbouring k-mers per k-mer, each hashed with xxh3 and probably missing the cache), not the MPHF passes this document started from (`SlottedKmers` ≈ 2 %, §11). Not measured: what fraction of those lookups miss the cache, and the same profile on a prod-size partition.
|
||||
|
||||
## 17. Profile of `index` on a skim: *Vaccinium vitis-idaea*, 2.1 GB, `--partitions 16` (2026-10-15)
|
||||
|
||||
One run, `sample` windows of 5 s every ~17 s over the whole run (the sampler slows the process; walls below include it): scatter 19.7 s, dereplicate 8.0 s, count_kmer 25.2 s, **index 139.1 s**; total 192 s real, 1830 s user, **225 s system** (12 % of user, against ≈ 90 % on E. coli, §16), peak footprint 17.8 GB.
|
||||
|
||||
**Index stage** (samples taken fully inside it, 91 000 active vs 187 000 idle/yielding thread-samples):
|
||||
|
||||
| share of active CPU | |
|
||||
|---|---|
|
||||
| xxh3 hashing of kmers (de Bruijn `HashMap`) | 32.7 % |
|
||||
| `debruijn::walk` | 12.4 % |
|
||||
| `debruijn::count_neighbors` | 10.7 % |
|
||||
| `GraphDeBruijn::push` | 6.9 % |
|
||||
| `crossbeam` channel | 6.6 % |
|
||||
| `hashbrown` table operations | 5.0 % |
|
||||
| `debruijn::unitig_iter` | 4.8 % |
|
||||
| memcpy/memset, malloc/free, file I/O, SipHash | ≈ 7 % |
|
||||
| other | 13.7 % |
|
||||
|
||||
- **The de Bruijn graph is ≈ 72 % of the active CPU of `index`; the MPHF construction does not appear (< 1 %)**, and `SlottedKmers`/evidence filling is negligible. The conclusion of §16 holds at 100× the E. coli scale.
|
||||
- **The stage is poorly parallel**: on average ≈ 5 of 16 threads are active (33 % of thread-samples); `swtch_pri` (spin/yield) is 7 % of all thread-samples. With 16 partitions of unequal size the tail partitions run alone. The 12 % system CPU is the same spinning as §16, at a smaller relative weight.
|
||||
- **count_kmer** (one window): sorting 40 %, file I/O 18 %, malloc/free 9 %, memcpy 6 %. **scatter** (one window, 32 % active): `RollingStat::push` 32 %, `EntropyTracker::push` 23 % + `normalized_entropy` 5 %, super-kmer stream 10 %, gzip decode ≈ 9 %.
|
||||
|
||||
Not established: the cache-miss rate of the graph's table (stalls are attributed to whichever instruction waits, so "hashing 33 %" may include time waiting for the hasher's operands or surrounding loads — a counter-based profile is needed to separate compute from memory stalls), and whether a cheaper hash function for `u64` kmers or fewer lookups per kmer would change the picture.
|
||||
|
||||
## 18. De Bruijn graph: hasher, walk, and an orientation bug that fragmented unitigs (2026-10-15)
|
||||
|
||||
Benchmark harness: `obikindexer/examples/graph_bench.rs` (`graph_bench <unitigs.bin> <k> [repeat]`; env `IDEAL=1` counts maximal unitigs independently of `GraphDeBruijn`, `SYNTH=n` uses synthetic reads, `DUMP`/`IDEALDUMP` print unitigs). Reference input: the 32.6 M kmers of one partition of the *V. vitis-idaea* index (`-p 16`, k = 31). Phase timings, 16 threads: push / `compute_degrees_and_mark_starts` / `for_each_unitig`.
|
||||
|
||||
| | push | degrees | unitigs | unitigs produced | nucleotides |
|
||||
|---|---|---|---|---|---|
|
||||
| original | 4.1 s | 2.0 s | 3.9 s | 6.56 M (varies ±900 per run) | 229.4 M |
|
||||
| `u64` hasher (below) | 1.0 s | 0.7 s | 3.1 s | 6.56 M | 229.4 M |
|
||||
| + orientation fix | 1.0 s | 0.8 s | **1.0 s** | **3.81 M** (stable ±3) | **146.9 M** |
|
||||
|
||||
Independent count of maximal unitigs on the same kmers (nodes − mutually-unique links): 3 809 887. After the fix the algorithm gives 3 809 945 (+0.002 %, cycles).
|
||||
|
||||
**1. Hasher** (`debruijn/kmer_hash.rs`). The node table used xxh3's streaming `Hasher` (`Xxh3Builder`), for a key that is one `u64`: ≈ 35 % of the active CPU of `index` (§17). Replaced by a splitmix64 finaliser `BuildHasher` (one multiply–shift pair, no state). push 4.1 → 1.0 s, degrees 2.0 → 0.7 s. `xxhash-rust` is no longer used by `obidebruinj` (left in its `Cargo.toml`; removing it is a dependency decision).
|
||||
|
||||
**2. Walk**: `WalkState` now carries the node's own `&AtomicU8`, so marking the next node visited does not look the kmer up a second time (−2 % on the unitig phase: not where the time was).
|
||||
|
||||
**3. Unitig fragmentation — a bug, not a tuning.** Starts are detected only for the canonical orientation. A chain whose end node is stored reverse-complemented has no recognised start and falls into the sequential "cycles and tails" pass, where `find_chain_start` returned only the kmer, dropping the orientation, and `unitig_nucleotides` always restarted in the canonical orientation: from the wrong end it extends nowhere and emits a **lone-kmer unitig**, repeatedly, along the chain. The result depended on table iteration order (`degrees_linear_chain_extensions` passed with xxh3's order and failed with any other; counts varied run to run) and was never maximal: **72 % more unitigs than maximal** on the reference partition. A second slip in the reverse branch of `WalkState::walk` (the `can_extend_left`/`can_extend_right` test of the next node was swapped) cost another ~15 %. Fixes: `walk` condition corrected; `find_chain_start` returns `(kmer, direct)` with the orientation to walk forward; `unitig_nucleotides` and `UnitigNucIter` take that orientation (and read the first k nucleotides reverse-complemented when it is false); starts are now flagged in both orientations and the claim picks the right one (`start_direction`). Regression tests: `linear_chain_is_one_unitig_whatever_the_orientations` (fails on the original code, passes now) and `unitigs_partition_the_kmers_of_branching_reads` (every kmer in exactly one unitig).
|
||||
|
||||
Residual: on noisy synthetic reads (k = 11–31, 200–20 000 bases) the algorithm still gives up to 5 % more unitigs than the independent count, always a few short pieces next to chain ends; not explained, absent from real data (+0.002 %).
|
||||
|
||||
**End-to-end** (current binary vs previous one, same inputs):
|
||||
|
||||
| | before | after |
|
||||
|---|---|---|
|
||||
| *V. vitis-idaea* 2.1 GB, `--partitions 16`: total wall | 163.8 s | **116.5 s** (−29 %) |
|
||||
| … `index` stage | 114.6 s | **59.2 s** (−48 %) |
|
||||
| … user CPU | 1 870 s | 1 304 s |
|
||||
| … `unitigs.bin` / `.idx` | 1.05 / 0.42 GB | **0.66 / 0.24 GB** (−37 % / −43 %); index 3.5 → 2.9 GB |
|
||||
| E. coli, `index` stage / wall / user CPU | 0.91 s / 1.97 s / 10.2 s | 0.59 s / 1.63 s / 7.3 s |
|
||||
|
||||
Same k-mers: per-layer kmer counts identical (521 630 625 over the 16 layers), E. coli `dump` and `query` outputs identical (md5), `scripts/smoke_test_index.sh` PASS, `cargo test --workspace` 551 passed / 0 failed, and `bench_mphf.sh` digests (index, merge, query ×4, filter, convert round trip, phylo) identical to the baseline run. Existing indexes are unaffected (any partition of kmers into unitigs is valid; evidence records are per-index); only newly built layers get the shorter, fewer unitigs.
|
||||
|
||||
## 19. Rank field: unitig chunks capped at 128 k-mers (2026-10-15)
|
||||
|
||||
`Evidence` packs `(chunk_id, rank)` into a `u32` with a 7-bit rank (`encode` masks `& 0x7F`), but `MAX_KMERS_PER_CHUNK` was 256 (the width of the unitig record's length byte), so a chunk of 129–256 k-mers had its ranks ≥ 128 silently truncated and those k-mers no longer verified. Observed on E. coli reads, same query: **`--partitions 256`** → max chunk 49–50 k-mers, results correct with the old and the new binary; **`--partitions 1`** → max chunk 256, query output wrong with both the old (md5 `bde0f886a0`) and the post-graph-fix (`8f1c94ef32`) binary versus the correct `f99e971087`. The orientation fix of §18 makes unitigs longer, hence makes this more likely at low partition counts.
|
||||
|
||||
**Change** (`obikseq::MAX_KMERS_PER_CHUNK`: 256 → 128; chosen so, rather than failing at write time: a longer unitig is simply written as consecutive chunks, nothing is lost). The constant is shared with the super-kmer chunking of the scatter files (`SuperKmer::write_to_binary`): super-kmers above 128 k-mers (rare) are now split into 128-k-mer records — readers are unchanged and old files stay readable. Tests that hard-coded 255/256/257/259/260 were made relative to the constant; new test `chunk_ranks_fit_the_7_bit_evidence_field`.
|
||||
|
||||
**After** (E. coli, same query): `--partitions` 256 / 16 / 4 / 1 → max chunk 50 / 70 / 118 / 128 k-mers, query md5 `f99e971087` in all four cases. `cargo test --workspace`: 551 passed, 0 failed.
|
||||
|
||||
**Existing indexes.** An index built with 256 partitions has chunks far below the limit (91-genome index, first 4 partitions: max 42–56, none above 128) and is unaffected. The `--partitions 4` skim index used for the benchmarks above (built before the cap) holds 6 chunks over 128 k-mers in its 4 partitions (max 135): a handful of k-mers out of ≈ 354 M verify wrongly. `scripts/check_unitig_chunks.py "<index>/partitions/part_*/index/layer_*/unitigs.bin"` reports chunks over 128 (it reads every unitigs.bin sequentially; k is hard-coded to 31 at its top).
|
||||
|
||||
## 20. Summary of what was gained (2026-10-15)
|
||||
|
||||
Apple M3 Max (aarch64, 16 cores, 128 GB), on AC power; absolute times are not those of the 192-core production machine, ratios measured in one session are what is reported. "Original" = the binary at the start of this work; "now" = current tree. Every output digest checked (query, dump, merge, filter, convert, phylo) is identical before and after, except where stated.
|
||||
|
||||
### Speed
|
||||
|
||||
| Operation | Original | Now | Where |
|
||||
|---|---|---|---|
|
||||
| Open an index (query of one 64-base sequence), 91 genomes, 180 GB | 10.4–13.3 s | **0.18–0.19 s** | §14, same-session A/B |
|
||||
| Query, 2 × 48 MB reads on that index (`--chunk-size 8`) | 30.7–32.0 s (61.8 s cold), 70–76 s CPU | **11.6–13.2 s**, 36–39 s CPU (−62 % wall, −50 % CPU) | §14, same-session A/B |
|
||||
| `dump --head 10` on that index | 9.0–10.0 s (91–102 s system CPU) | **0.16–0.84 s** | §15 |
|
||||
| `index` of a 2.1 GB skim (*V. vitis-idaea*, `--partitions 16`) | 163.8 s (stage `index` 114.6 s) | **116.5 s** (stage `index` 59.2 s): −29 % wall, −48 % on the stage, −30 % CPU | §18 |
|
||||
| `index` of E. coli reads | 1.97 s (stage 0.91 s), 10.2 s CPU | 1.63 s (stage 0.59 s), 7.3 s CPU | §18 |
|
||||
| De Bruijn graph, 32.6 M kmers (push / degrees / unitigs) | 4.1 s / 2.0 s / 3.9 s | 1.0 s / 0.8 s / 1.0 s (×3.5 overall) | §18 |
|
||||
| `index` stage `count_kmer` (E. coli) | 0.51–0.56 s | 0.42–0.50 s | §15 |
|
||||
| Query on a small input (2 × 48 MB reads), single-genome index | 11.4 s (2 chunks = 2 busy cores) | **2.4 s** (−79 %) — automatic chunk size now follows the input size (`architecture/query.md`); on the 91-genome index the size was already 4 MiB | §20 follow-up |
|
||||
|
||||
### Memory (peak RSS / footprint)
|
||||
|
||||
| Operation | Original | Now |
|
||||
|---|---|---|
|
||||
| Opening the 91-genome index (any command that builds an `IndexCache`) | **27.4 GB** | **0.04 GB** (§14: `.idx` offsets and `dict_values.bin` mapped instead of copied; they were read twice and once) |
|
||||
| Query, 2 × 48 MB reads on that index | 32.2–32.5 GB | **3.5–3.6 GB** |
|
||||
| Full query, ≈ 1 GB gz of reads, 354 M-kmer index, default chunk size | **73.7–80.7 GB** | **28.2 GB** (§12 of `architecture/query.md`: `PartitionKmers` instead of a `HashMap<kmer, Vec>` per partition: 71 → ~20 bytes per unique kmer) |
|
||||
| `index` of the 2.1 GB skim | 19.7 GB | 18.3 GB |
|
||||
|
||||
### Index size (newly built indexes; existing ones are unchanged)
|
||||
|
||||
| | Original | Now |
|
||||
|---|---|---|
|
||||
| `unitigs.bin` / `unitigs.bin.idx` (2.1 GB skim, 521.6 M kmers, 16 partitions) | 1.05 / 0.42 GB | **0.66 / 0.24 GB** (−37 % / −43 %: 3.81 M unitigs instead of 6.56 M — the orientation fix of §18) |
|
||||
| Whole index of that skim | 3.5 GB | **2.9 GB** (−17 %) |
|
||||
| `evidence.bin` | 4 bytes per kmer | unchanged (still ≈ 40 % of the 91-genome index, 2.1 GB of 3.5 GB for the skim): reducing it is a separate design question (§ answer on exact evidence: ≈ −22 to −31 % of it by dropping the rank field, far more with a minimizer-indexed layout; neither implemented) |
|
||||
|
||||
### Correctness fixes found on the way
|
||||
|
||||
- Unitigs were not maximal (72 % too many) and depended on hash-table iteration order (§18).
|
||||
- Chunks of 129–256 k-mers had their ranks silently truncated in the 7-bit evidence field and failed verification, visible with few partitions (§19, `scripts/check_unitig_chunks.py` to audit an index).
|
||||
- `index.meta` and super-kmer files were parsed byte by byte (§15).
|
||||
|
||||
### What is **not** established
|
||||
|
||||
- The DRAM-latency gain of MPHF prefetch itself, the original motivation (§1–§6): the measured MPHF share of the profiles is ≈ 1 % (§11), and the layers here are 4× to 1 000× smaller than production ones. The −5 % seen on the first query change came mostly from batching per layer (probably fewer shared `Arc` clones, §11), not from prefetch. A prod-scale benchmark (§6) remains to be run on the production machine.
|
||||
- Speed-ups on the 192-core / 1 TB production machine, on `merge`, `filter`, `select` and `phylo` at the 180-genome scale (the 91-genome index could only be used for `query`, `dump`, `phylo --distance jaccard`, `utils --stats`; `filter`/`select`/`merge` refused its pre-`state` `index.meta` until it was patched in the clone).
|
||||
- The size of the Primula index composition (§ calculated: ≈ half evidence from 35–40 G kmers × 4 bytes; not measured).
|
||||
|
||||
## 21. Dead code removed (2026-10-15)
|
||||
|
||||
The items listed as unused in §4 were removed after Serena's `safe_delete_symbol` and the compiler confirmed no reference: `MphfLayer::find_strict`; the single-kmer `hash` chain (`MphfLayer::hash`, `TypedLayer::hash`, `KmerLayer::hash`, `IndexCache::hash`); `IndexCache::find`; the uncompiled `obikmerge/src/merge_layer/` directory (689 lines; `lib.rs` only declares `merge` and `partition_merge`); and `xxhash-rust` from `obidebruinj/Cargo.toml` (it still serves `obikseq`, `obifastwrite` and `obiksession`).
|
||||
|
||||
Side effect: `find_strict` was the only reader of the `unitigs` handle that `Hybrid` layers opened, so that field (and the extra `.idx` map and mmap per Hybrid layer) was dropped too. `find` on a Hybrid layer was already a fingerprint check; the exact evidence stays on disk (it gives the slot count and is what `convert` works from) but no API consults it any more. Restoring an exact lookup for Hybrid layers means re-adding `find_strict` and that field.
|
||||
|
||||
`ahash` was not referenced by any source file of `obidebruinj` either and was removed from its `Cargo.toml` too.
|
||||
|
||||
Checks after the removal: `cargo build --release` without warnings, `cargo test --workspace` 555 passed / 0 failed, `scripts/smoke_test_index.sh` PASS.
|
||||
|
||||
Scratch artifacts of this work were deleted: the intermediate binaries, the two skim indexes (`skim_p4`, `skim_p16`) and the shared data set `benchmark/run/bench_mphf/data` (rebuilt automatically, ≈ 2 min, by the next `bench_mphf.sh`). Kept in `benchmark/run/bench_mphf/`: the original binary `obikmer_baseline` (reference for A/B runs) and the per-label results and checksums (18 MB in all). The 91-genome index converted to the current MPHF format (`/Users/coissac/travail/obiskim/data/phyloalps/bench_mphf/phyloskims_sal_vac_sav_v2`, with its `index.meta` patched, original `index.meta.before_state_fix` beside it) was left in place.
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,69 +0,0 @@
|
||||
# Chance sharing of a kmer across genomes (discussion)
|
||||
|
||||
Null model for "a kmer is present in $r$ of $S$ genomes": how likely is this by chance alone? Extends the single-pair estimate $P_{\mathrm{chance}}\simeq G/4^k$ (see user doc, *K-mer specificity*).
|
||||
|
||||
## Homogeneous case: binomial tail
|
||||
|
||||
If every genome has the same per-genome chance probability $p$, the number $R$ of genomes containing a given kmer is $\mathrm{Bin}(S,p)$:
|
||||
|
||||
$$P(R\geq r)=\sum_{j=r}^{S}\binom{S}{j}p^j(1-p)^{S-j}=I_p(r,\,S-r+1)$$
|
||||
|
||||
where $I_p$ is the regularized incomplete beta function. No elementary closed form; use $I_p$ directly.
|
||||
|
||||
### Numerical evaluation of the sum
|
||||
|
||||
- Term recurrence, one `lgamma` start: $t_{j+1}=t_j\cdot\frac{S-j}{j+1}\cdot\frac{p}{1-p}$.
|
||||
- Sum the tail **not containing the mode** ($Sp$, not $S/2$). If $r>Sp$, sum $\sum_{j\ge r}$ directly and stop when terms are negligible. The complement $1-\sum_{j=0}^{r-1}$ (note: from $j=0$) suffers catastrophic cancellation when the p-value is tiny.
|
||||
- Large $S$: normal approximation with continuity correction, or Poisson($Sp$) when $p$ is small. Unreliable in extreme tails.
|
||||
|
||||
## Heterogeneous case: $p_i = f(G_i)$
|
||||
|
||||
The $G_i$ (number of kmers of genome $i$) are known and differ. With $p_i\simeq G_i/4^k$, the indicators "kmer present in genome $i$ by chance" are independent Bernoulli($p_i$), so $R$ follows a **Poisson-binomial** law, not a binomial:
|
||||
|
||||
$$P(R=j)=\sum_{|A|=j}\prod_{i\in A}p_i\prod_{i\notin A}(1-p_i).$$
|
||||
|
||||
Everything lives in canonical space: $G_i$ is the number of distinct canonical kmers of genome $i$, and the kmer space has size
|
||||
|
||||
$$N=\frac{4^k}{2}$$
|
||||
|
||||
exactly, since $k$ is odd (no kmer equals its own reverse complement, so canonicalization pairs the $4^k$ kmers perfectly). Exact per-genome probability: $p_i=1-(1-1/N)^{G_i}\simeq 2G_i/4^k$. This is twice the $G/4^k$ of the user doc, where $G$ is a number of positions on one strand.
|
||||
|
||||
### Exact computation
|
||||
|
||||
Dynamic programming over genomes, $d_j$ = probability of $j$ hits so far:
|
||||
|
||||
$$d'_j=d_j(1-p_i)+d_{j-1}p_i.$$
|
||||
|
||||
For the upper tail only $d_0\ldots d_{r-1}$ are needed: $P(R\ge r)=1-\sum_{j<r}d_j$, cost $O(S\,r)$, per distinct $(p_1..p_S)$ vector — independent of the kmer, so computed once per index (table of $P(R\ge r)$ for $r=1..S$), not per kmer. Cancellation caveat as above: for tiny tails compute $\Pr(R\ge r)$ by the DP on the upper side (track $d_j$ for $j\ge r$ only from a top-down recurrence) or in log space. Alternative: DFT of the characteristic function (Hong 2013), $O(S^2)$ exact.
|
||||
|
||||
### Approximations
|
||||
|
||||
- **Poisson**: $R\approx\mathrm{Poisson}(\lambda=\sum_i p_i)$ with total variation error $\le\sum_i p_i^2$ (Le Cam). Excellent when all $p_i\ll1$, i.e. $4^k\gg G_i$ (large $k$).
|
||||
- **Binomial with mean $\bar p=\frac1S\sum_i p_i$**: Hoeffding (1956) shows the Poisson-binomial is more concentrated than $\mathrm{Bin}(S,\bar p)$. Hence for $r\ge S\bar p+1$, $P_{\mathrm{PB}}(R\ge r)\le P_{\mathrm{Bin}}(R\ge r)$: using the mean $\bar G$ overestimates the chance probability, i.e. is **conservative** in the upper tail. Wrong direction for $r\le S\bar p$ (lower tail), irrelevant for significance testing.
|
||||
|
||||
Consequence: assuming a common $G$ is safe (conservative) if $G=\bar G$ (arithmetic mean), but loses power when genome sizes span orders of magnitude; the exact DP is cheap enough (once per index) to make the approximation unnecessary. Small genomes with $k=11$ saturate ($p_i\to1$) and break the Poisson approximation, not the DP.
|
||||
|
||||
## Open points
|
||||
|
||||
- Repeats are not a problem. The model is presence/absence per genome, so copy number within a genome is irrelevant, and $G_i$ is the number of **distinct** kmers of genome $i$. A repeat shared between genomes is paralogy, i.e. homology: a true positive, not a false one. Low-complexity kmers (which would share by composition, not descent) are removed upstream by the [entropy filter](entropy.md).
|
||||
- Residual violation of the equiprobability behind $p_i$: nucleotide composition bias (see Karlin–Ost in the user doc).
|
||||
- Test dependence between overlapping kmers (see FDR section).
|
||||
|
||||
## Goal: an FDR of homology
|
||||
|
||||
$H_0$: the $S$ genomes are unrelated (independent random sequences of sizes $G_i$). Rejecting $H_0$ for a kmer, i.e. sharing that chance cannot explain, is the evidence of homology. The quantity wanted is the false discovery rate of the call "shared kmer $\Rightarrow$ homologous".
|
||||
|
||||
### Empirical FDR from the count histogram
|
||||
|
||||
Let $N$ be the size of the kmer space and $n_{\ge r}$ the observed number of distinct kmers present in at least $r$ genomes (a by-product of the index). Under $H_0$ the expected number of kmers with $R\ge r$ over the whole space is exactly $N\,P(R\ge r)$, with $P$ the Poisson-binomial tail above. Hence
|
||||
|
||||
$$\widehat{\mathrm{FDR}}(r)=\min\!\left(1,\ \frac{N\,P(R\ge r)}{n_{\ge r}}\right).$$
|
||||
|
||||
- No per-kmer multiple-testing correction: the number of tests ($N$) is already in the numerator.
|
||||
- Conservative: the observed $n_{\ge r}$ mixes true homology and chance, and the true-null proportion $\pi_0\le1$ is taken as $1$ (Storey-type upper bound).
|
||||
- Take the smallest $r$ with $\widehat{\mathrm{FDR}}(r)\le\alpha$ as the sharing threshold.
|
||||
- Equivalent per-kmer view: BH on $p=P(R\ge r)$ with $N$ tests. The zero-truncation matters if one tests only kmers present in the index ($R\ge1$ by selection): use $P(R\ge r)/P(R\ge1)$, not $P(R\ge r)$, otherwise the test is too conservative by a factor $\sim1/\lambda$.
|
||||
|
||||
### Dependence
|
||||
|
||||
Kmers of one shared region overlap and are not independent tests. The FDR above holds per kmer; for a per-region FDR, count regions (unitigs / super-kmers, with the appropriate clumping of $N$ and $n_{\ge r}$) rather than kmers. Not settled.
|
||||
@@ -1,84 +0,0 @@
|
||||
# Kmer entropy filter
|
||||
|
||||
Low-complexity kmers (polyA, polyT, tandem repeats) are detected and excluded during phase 1. The filter computes a **normalized Shannon entropy** over sub-words of multiple sizes, corrected for one source of bias: the small number of observations within a single kmer relative to the number of possible sub-words.
|
||||
|
||||
## Sub-word frequencies
|
||||
|
||||
For a kmer of length k and a sub-word size ws (1 ≤ ws ≤ ws_max, typically ws_max = 6), extract the $n_{\text{words}} = k - ws + 1$ overlapping sub-words by sliding a window of length ws:
|
||||
|
||||
$$w_i = \text{kmer}[i \mathinner{..} i+ws-1], \quad i = 0, \ldots, n_{\text{words}}-1$$
|
||||
|
||||
Each sub-word is tallied under its own raw 2-bit-packed value — **no canonicalization**. Let $f_j$ be the count of raw word $j$ among the $n_{\text{words}}$ sub-words ($\sum_j f_j = n_{\text{words}}$), over the $4^{ws}$ possible raw words.
|
||||
|
||||
An earlier version of this filter first folded each sub-word into a circular+reverse-complement equivalence class, then "unfolded" the observed class frequency back onto its members to correct for unequal class sizes. That machinery bought nothing it was claimed for — see *Why no equivalence classes* below — while measurably weakening detection of the very sequences the filter exists to catch, so it was removed.
|
||||
|
||||
## Corrected Shannon entropy
|
||||
|
||||
$$H_{\text{corr}} = \log(n_{\text{words}}) - \frac{1}{n_{\text{words}}} \sum_j f_j \log f_j$$
|
||||
|
||||
This is a plain Shannon entropy over the observed raw-word frequencies.
|
||||
|
||||
## Maximum entropy correction for small samples
|
||||
|
||||
With only $n_{\text{words}}$ observations over $4^{ws}$ possible raw words, the achievable maximum entropy is bounded by the most uniform integer distribution over $4^{ws}$ categories.
|
||||
|
||||
Let $c = \lfloor n_{\text{words}} / 4^{ws} \rfloor$ and $r = n_{\text{words}} \bmod 4^{ws}$. The most uniform integer distribution assigns frequency $c+1$ to $r$ categories and $c$ to the remaining $4^{ws} - r$, with the convention $0 \log 0 = 0$:
|
||||
|
||||
$$H_{\max} = -\left[(4^{ws} - r)\,\frac{c}{n_{\text{words}}}\log\frac{c}{n_{\text{words}}} + r\,\frac{c+1}{n_{\text{words}}}\log\frac{c+1}{n_{\text{words}}}\right]$$
|
||||
|
||||
When $n_{\text{words}} < 4^{ws}$: $c=0$, $r=n_{\text{words}}$, and the formula reduces to $H_{\max} = \log(n_{\text{words}})$ — a single unified expression covers both regimes. A truly random sequence achieves $H_{\text{corr}} \approx H_{\max}$.
|
||||
|
||||
## Normalized entropy
|
||||
|
||||
$$\hat{H}(ws) = \frac{H_{\text{corr}}}{H_{\max}} \in [0, 1]$$
|
||||
|
||||
## Final score
|
||||
|
||||
The filter computes $\hat{H}(ws)$ for each word size ws from 1 to ws_max and returns the **minimum**:
|
||||
|
||||
$$\text{entropy}(kmer) = \min_{ws=1}^{ws_{\max}} \hat{H}(ws)$$
|
||||
|
||||
A value near 0 indicates low complexity (e.g. AAAA…); near 1 indicates high complexity. A kmer is rejected if $\text{entropy}(kmer) < \theta$, where $\theta$ is a collection parameter (default 0.7). The minimum across word sizes ensures that any scale of repetition is detected independently: polyA is caught at ws=1, dinucleotide repeats at ws=2, etc.
|
||||
|
||||
## Why no equivalence classes
|
||||
|
||||
A prior design folded each sub-word into the canonical form of its circular-rotation + reverse-complement equivalence class before tallying, on the reasoning that (a) it guarantees $\text{entropy}(K) = \text{entropy}(\text{revcomp}(K))$, and (b) collapsing phase-shifted repeats (e.g. `ATG` ≡ `TGA` ≡ `GAT`) into one class better reflects that they are "the same" low-complexity pattern.
|
||||
|
||||
Both properties already hold for the raw, unfolded entropy above, without any class machinery:
|
||||
|
||||
- **Reverse complement**: for any K of length n, window $j$ of $\text{revcomp}(K)$ equals $\text{revcomp}$ of window $(n{-}ws{-}j)$ of K. This is a bijection between the window sets under which each window maps to its own revcomp — and revcomp is itself a bijection (involution) on the space of raw ws-mers. So the multiset of raw-word frequencies for $\text{revcomp}(K)$ is exactly a relabeling of the multiset for K, and Shannon entropy — a function of the frequency multiset alone — is exactly invariant. No folding required, for any K.
|
||||
- **Tandem repeats**: a period-p repeat sampled by a stride-1 sliding window naturally cycles through its own rotations as raw tokens (e.g. `ATGATGATG…` yields the raw words `ATG`, `TGA`, `GAT` in rotation as the window slides). The low diversity this represents (few distinct raw words out of $4^{ws}$ possible) is already visible in the raw frequency distribution — no folding needed to detect it.
|
||||
|
||||
What the fold-then-unfold step actually did was credit each observed class with the frequency of equivalence-class members that were **never observed on the read strand**, inflating $H_{\text{corr}}$ for genuine repeats. Worked example: k=31, ws=3, kmer = `ATG` repeated ($n_{\text{words}}=29$, all 29 windows fall into one class of size 6 under the old scheme — 3 rotations × forward/revcomp):
|
||||
|
||||
| | $H_{\text{corr}}$ | normalized |
|
||||
|---|---|---|
|
||||
| old (folded, class size 6) | $\log 6 \approx 1.79$ | $\approx 0.53$ |
|
||||
| current (raw, unfolded) | $\log 3 \approx 1.10$ | $\approx 0.33$ |
|
||||
|
||||
The gap is not a rounding artifact: per sub-word order, the folded score for this same repeat swings from 0.53 (ws=3, aligned with the period) up to **1.03** (ws=5, misaligned with the period) — i.e. a period-3 repeat could score *above* the theoretical maximum for a random sequence, depending on which ws happens to divide the repeat's period. The raw formula stays flat at ≈0.33–0.40 across ws=2..6 regardless of alignment, which is the robustness the "minimum across ws" design was meant to provide in the first place.
|
||||
|
||||
## Interpretation as an effective number of classes
|
||||
|
||||
$H_{\text{corr}}$ is a standard Shannon entropy over raw words, so the classical perplexity interpretation holds directly: $N_{\text{eff}} = e^{H_{\text{corr}}}$ is the number of equiprobable raw words that would yield the same entropy.
|
||||
|
||||
For the normalised score $\hat{H}$, dividing by $H_{\max}$ changes the logarithm base:
|
||||
|
||||
$$\hat{H} = \frac{\log N_{\text{eff}}}{\log N_{\max}} = \log_{N_{\max}} N_{\text{eff}} \quad \Longleftrightarrow \quad N_{\text{eff}} = N_{\max}^{\,\hat{H}}$$
|
||||
|
||||
The property is preserved: $\hat{H}$ is the logarithm (in base $N_{\max}$) of the effective number of equi-represented raw words.
|
||||
|
||||
In the large-sample limit ($n_{\text{words}} \gg 4^{ws}$), $N_{\max} \approx 4^{ws}$, giving:
|
||||
|
||||
$$N_{\text{eff}} \approx 4^{ws \cdot \hat{H}}$$
|
||||
|
||||
This has a clean interpretation: $ws \cdot \hat{H}$ is the **effective word length** (in bases) of a perfectly uniform distribution that would produce the same entropy. At $\hat{H} = 1$ the full space of $4^{ws}$ words is used; at $\hat{H} = 0.5$ with ws=2, only $4^1 = 4$ effective words out of 16 are occupied.
|
||||
|
||||
In our actual regime, $n_{\text{words}}$ is small and $4^{ws}$ can exceed $n_{\text{words}}$, so $H_{\max} < \log(4^{ws})$ due to the small-sample correction. The exact effective count is $N_{\max}^{\hat{H}}$, not $4^{ws \cdot \hat{H}}$.
|
||||
|
||||
## Properties
|
||||
|
||||
The entropy score is a function of the kmer sequence alone — it does not depend on the surrounding context or on the position within any genome. Two consequences:
|
||||
|
||||
- **Orientation invariance**: $\text{entropy}(K) = \text{entropy}(\text{revcomp}(K))$ — see *Why no equivalence classes* above for why this holds without any explicit strand-folding step.
|
||||
- **Context independence**: the same kmer is always rejected or always kept, regardless of which genome it occurs in, where in that genome it appears, or which strand is considered. The filter defines a fixed partition of the kmer space into low-complexity and valid kmers.
|
||||
@@ -1,16 +0,0 @@
|
||||
<!-- coverage sidecar — ne pas ajouter au nav mkdocs -->
|
||||
# Coverage: theory/entropy.md
|
||||
|
||||
## Code couvert
|
||||
|
||||
- `obikentropy/src/table.rs`, `obikentropy/src/tracker.rs` — formule d'entropie et tables de correction petits effectifs
|
||||
- `obikentropy/src/kmer_entropy.rs` — entropie d'un kmer isolé (`KmerEntropy`)
|
||||
- `obiskbuilder/src/rolling_stat.rs` — composition de `obikentropy::EntropyTracker` dans le suivi streaming (sélection de minimiseur + entropie)
|
||||
- `obiskbuilder/src/iter.rs`, `obiskbuilder/src/stream_iter.rs` — application du filtre lors du scatter (phase 1)
|
||||
|
||||
## Notes
|
||||
|
||||
Le repli en classes d'équivalence circulaires + brin inverse (décrit dans une version antérieure de ce document) a été supprimé : voir la section « Why no equivalence classes » de `entropy.md` pour la justification théorique et numérique.
|
||||
|
||||
Vérifier que les paramètres `theta` et `level_max` dans le CLI
|
||||
(`obikmer/src/cli.rs` → `CommonArgs`) correspondent bien à ce qui est décrit.
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,366 +0,0 @@
|
||||
&@")--# Pair-coverage-biased family sampling (discussion, 2026-10-07)
|
||||
|
||||
Idea: bias the family sample used by `--pseudo-alignment`/`--sankoff`/`snp-*` so that the `family_overlap` matrix of the *sample* is as uniform as possible, compensating the scarcity of shared families between distant species. Complements, does not replace, the existing proportional-by-partition and entropy-biased draws (see user doc, *Sampling design*).
|
||||
|
||||
## Motivation: observed on Primulaceae (180 genomes)
|
||||
|
||||
Scatter of patristic distance (IQ-TREE on the pseudo-alignment) vs. normalized `family_overlap` shows three pair classes:
|
||||
|
||||
- close intra-genus pairs: high overlap, well resolved;
|
||||
- inter-genus plus distant intra-genus pairs: overlap collapses by ~2 orders of magnitude, one large noisy cloud;
|
||||
- a small low-overlap cluster at the largest distances, plausibly a few poorly sequenced genomes (user hypothesis, unverified; the tree shows one tip, `Androsace_saussurei--PHA010690`, with a 2.9 terminal branch).
|
||||
|
||||
Uniform family draw reproduces this imbalance in the pseudo-alignment: distant pairs share few informative columns, close pairs many. The tree quality for deep splits is then limited by sampled-column count on those pairs, not by the total number of families.
|
||||
|
||||
## What defines a target
|
||||
|
||||
A family $f$ is carried by a genome set $S_f$ and contributes to every pair in $S_f \times S_f$. For a sample $\mathcal{F}$ with inclusion probabilities $\pi_f$, the expected sample overlap of pair $(i,j)$ is
|
||||
|
||||
$$\mathbb{E}[O^{\mathcal{F}}_{ij}] = \sum_{f:\, i,j\in S_f} \pi_f$$
|
||||
|
||||
The goal is a $\pi$ making this approximately constant over pairs. The full-index $O_{ij}$ (already computed by `--family-overlap`) is the pool size per pair; $\min_{ij} O_{ij}$ is a hard ceiling on the flat level reachable.
|
||||
|
||||
## Nesting hypothesis (user, unverified)
|
||||
|
||||
Families covering the poorest pairs also cover the richest ones: absent genome-skimming sampling bias (locus dropout), a family conserved across distant species is not expected to be missing between closer ones. If it holds, $S_f$ is approximately a clade closure and the family pool is nested by pair distance, which changes the design:
|
||||
|
||||
- no sequential/stateful draw needed: a family's weight depends only on its own reach (farthest pair covered, or equivalently its coverage of the poorest pairs), computable in one pass;
|
||||
- flattening reduces to a monotone reweighting by reach: keep (nearly) all long-reach families, thin the short-reach ones, until the sample overlap of close pairs drops to the level of distant ones.
|
||||
|
||||
My earlier remark that the weight is stateful (depends on what other families already covered) holds only without nesting.
|
||||
|
||||
## Measuring nesting: `scripts/pseudo_alignment_nesting.py`
|
||||
|
||||
Works on a pseudo-alignment FASTA plus the IQ-TREE treefile (columns = families; a call is any character other than `-`/`?`). Outputs the sample's `family_overlap` matrix and, per clade-size stratum, the *completeness* $|S_f|/|C(S_f)|$ where $C(S_f)$ is the smallest clade containing $S_f$. Tree rooted as written in the Newick unless `--outgroup`. Uses the already-existing families × genomes presence structure only; no `S_f` is stored (clade of a column = LCA of its first and last carrier in DFS leaf order, precomputed 180×180 matrix).
|
||||
|
||||
### Result, Primulaceae pseudo-alignment (180 genomes, 5.0 M columns, uniform draw, root as written)
|
||||
|
||||
| Clade size | Families | Mean completeness | Clade-closed |
|
||||
|---|---|---|---|
|
||||
| 2 | 145 047 | 1.00 | 100 % |
|
||||
| 3-4 | 272 918 | 0.71 | 20 % |
|
||||
| 5-8 | 503 114 | 0.45 | 0.6 % |
|
||||
| 9-16 | 637 896 | 0.35 | 0.1 % |
|
||||
| 17-32 | 1 656 229 | 0.21 | 0 % |
|
||||
| 33-64 | 299 632 | 0.18 | 0 % |
|
||||
| 65-128 | 181 126 | 0.08 | 0 % |
|
||||
| 129-180 | 1 307 452 | 0.12 | 0 % |
|
||||
|
||||
Overall 4.1 % clade-closed, mean completeness 0.275; off-diagonal overlap min 4, median 1 665, max 387 057, CV 1.57. **Strict nesting does not hold on this sample.** Caveats: unrooted tree, so the 129-180 stratum is inflated by families spanning the arbitrary root (rerun with `--outgroup` pending); this is the sample fed to IQ-TREE, not the full index; absence symbols `-`/`?` both count as absent.
|
||||
|
||||
### Lift test (`--lift-test`): margin-preserving null
|
||||
|
||||
Question: do families covering a rare pair $(A,B)$ also cover an abundant pair $(A,C)$ more often than margins imply? For each $A$, $B$ = its 10 % rarest partners, $C$ = its 10 % most abundant (ranked on pair overlap). Null: curveball swap randomisation of the genome × family matrix (exact row and column sums), re-selecting $B$/$C$ in every replicate with the same procedure. 200 754 sampled columns, 20 null replicates, 5 000 trades between replicates.
|
||||
|
||||
| Statistic | Observed | Null |
|
||||
|---|---|---|
|
||||
| $T=P(C\mid A,B)=O_{ABC}/O_{AB}$ | 0.559 | 0.498 ± 0.003 (no replicate ≥ observed, p at floor 1/21) |
|
||||
| $L=\sum n_A O_{ABC}/\sum O_{AB}O_{AC}$ | 1.18 | 1.694 ± 0.004 |
|
||||
|
||||
The two statistics disagree, not contradictorily: $T$ above null supports the "positive association beyond margins" statement; $L$ below null says that, relative to $P(C\mid A)$, knowing $B$ adds less than under the null — the observed abundant partners of $A$ are far more abundant than null ones (phylogenetic structure inflates the $L$ denominator). Unverified: mixing of the chain (is 5 000 trades enough? the tiny null sd may reflect correlation), sensitivity to the 10 % threshold and to the 200 k-column subsample.
|
||||
|
||||
Combined reading: association exists but is patchy; a reach-only weight is not enough, weighting must be per presence pattern $S_f$.
|
||||
|
||||
## Candidate weighting strategy: pair raking (proposal, not implemented)
|
||||
|
||||
Goal: inclusion probabilities $\pi_f$ with $\sum_{f\ni i,j}\pi_f\approx c$ for all pairs.
|
||||
|
||||
1. First-order weight: $w_f=\sum_{i,j\in S_f}1/O_{ij}$ (draw a pair uniformly, then a family uniformly among those covering it). With the presence matrix $P$ and $R=1/O$, $w=\tfrac12\,\mathrm{rowsum}(P\odot RP)$, one pass in the existing chunked loop, no $S_f$ storage.
|
||||
2. Iterate (raking/IPF with one multiplier per pair): update $R_{ij}$ from the overlap the current $\pi$ yields, recompute $w_f$; corrects the fact that large families are drawn for many pairs at once. $\pi_f=\min(1,c\,w_f)$, $c$ set to hit the target sample size; independent Poisson draw per family.
|
||||
3. Control: recompute the sample's `family_overlap` and its CV with the script above.
|
||||
|
||||
Pre-conditions: exclude anecdotal genomes first (e.g. `Androsace_saussurei--PHA010690`, terminal branch 2.9), else their tiny $O_{ij}$ dominate $R$. Overlap uniformity is only a proxy: information for a deep split is informative sites across it, not mere co-presence; to be checked on node support of the resulting tree. Unknown: whether long-reach families are rare because conserved-single-copy or because repeats/misassemblies (the latter would be amplified by up-weighting).
|
||||
|
||||
## Alternative / complement: site classes with their own models
|
||||
|
||||
Analogy with multigene supermatrices: loci differ in rate *and* in taxon occupancy. Families covering rare pairs (long reach, conserved, slow) vs. families covering close pairs only could be two (or a few) site classes, each with its own substitution model / branch-length scale (IQ-TREE partitions; edge-proportional vs edge-linked options to be checked in the manual). Independent of the weighting: sampling fixes scarcity, partitioning fixes rate heterogeneity; with class-specific scales, oversampling a class would not distort that class's branch lengths, removing the need for $1/\pi_f$ reweighting (IQ-TREE support for per-site weights unverified).
|
||||
|
||||
Caveats: reach is a continuum, two classes is a simplification and there is no criterion for the cut; class membership is defined from presence patterns, hence correlated with the data the model sees (assignment bias, worst with `?` missing data; magnitude not assessed); partitioning does not fix repeats/misassembly, it only gives them their own rate.
|
||||
|
||||
The pseudo-alignment already keeps only variable families, so branch lengths are not substitutions/site anyway (ascertainment bias, handled by `+ASC`, see the constant-column fix section in [evolutionary_distances.md](evolutionary_distances.md)). With partitions the ascertainment correction must be per class: each class excludes a different number of constant sites. Whether the existing fix applies per partition is unchecked.
|
||||
|
||||
Literature: see the next section.
|
||||
|
||||
## Literature on informative-site selection (2026-10-07; read from search extracts, not full papers)
|
||||
|
||||
- **Phylogenetic informativeness.** Townsend 2007 (*Syst. Biol.* 56:222-231) links each site's rate of change to its power to resolve splits at a given depth: invariant sites are useless, saturated sites misleading, with an optimum in between that depends on the age of the split. Implemented online as PhyDesign (requires an ultrametric tree). Relevance: supports an entropy centre $\mu$ that depends on depth (pair distance) rather than a single value; cf. the $\mu(n)$ discussion.
|
||||
- **Entropy-based trimming.** BMGE (Criscuolo & Gribaldo 2010, *BMC Evol. Biol.* 10:210) computes a per-character entropy-like score weighted by a similarity matrix and removes regions above a threshold. Uses entropy to *discard* over-variable characters, the opposite direction of the Gaussian band here.
|
||||
- **Saturation / fast sites.** Classical removal methods: Slow-Fast, LQP, TIGER, Observed Variability. Counter-evidence (Tree-of-Life protein datasets, PMC10609509): even the fastest sites carry reliable signal, trimming them can lower accuracy, slow sites recover fewer true bipartitions than the fastest. Argument against penalizing high entropy ($H\ge2$) without checking that those families do not carry the deep signal; domain (conserved proteins) differs from central-position SNPs, transfer unverified.
|
||||
- **Taxon-coverage theory.** Sanderson, McMahon & Steel 2010 (*BMC Evol. Biol.*, "Phylogenomics with incomplete taxon coverage: the limits to inference") characterise phylogenetically decisive collections of taxon sets via the four-way partition property: when the coverage pattern suffices to determine the tree uniquely irrespective of signal. Formal frame for pair-coverage sampling; applicability at 180 genomes not assessed.
|
||||
|
||||
Takeaways to check before weighting: (i) whether high-entropy families are the deep-signal ones (entropy vs carrier count measured, see below; the cross with reach is still open); (ii) depth-dependent entropy centre; (iii) whether a decisiveness criterion can serve as a sufficiency test for the sample's pair coverage.
|
||||
|
||||
## Simpler weight: poorest pair of the family (user preference, 2026-10-07)
|
||||
|
||||
$w_f=1/\min_{i,j\in S_f}O_{ij}$, $\pi_f=\min(1,c\,w_f)$. Equivalent in spirit to a reach weight but measured on actual overlaps (no tree needed), same $O(|S_f|^2)$ cost per family as the sum form. Limit: it ignores that large families also feed already-abundant pairs; flattening from above then relies only on dropping short-reach families, which is exact only under nesting, and nesting holds partially (see measurements). Untested.
|
||||
|
||||
### Estimating $O$ locally (per partition / per layer)
|
||||
|
||||
- **Per layer: not representative.** A layer is an increment of kmer content added to a partition (typically one `merge`), not a batch of genomes; all genome columns exist in every layer (`UserDocMD/formats/index_layout.md`). Layer content is dominated by kmers novel at merge time, so a layer's $O_{ij}$ reflects merge order, not the index-wide distribution; not an unbiased estimator of the global $O_{ij}$.
|
||||
- **Per partition: valid only if partitions are exchangeable samples of families** (hash-based; not verified, minimizer distribution may be skewed). Even then, the pairs that matter are the rare ones, with near-zero count per partition; a `min` over noisy estimates is biased downward (winner's curse), weights would go to pairs that were unlucky in that partition rather than globally poorest.
|
||||
- **Recommended:** compute $O$ globally once (the `--family-overlap` scan already exists), apply the weight locally per partition, keeping partitions independent. Without an extra pass: pilot estimate on a random subset of partitions.
|
||||
- **Sequential variant:** update the $O$ estimate as partitions are processed in random order, boosting still-undersampled pairs. Self-correcting but breaks partition independence and makes the result order-dependent.
|
||||
- **Quota consequence:** the current per-partition quota is proportional to candidate-family count; with weights it must be proportional to the partition's weight mass $\sum_f w_f$, otherwise partitions rich in high-weight families are undersampled.
|
||||
|
||||
## Does the entropy bias distort pair representation? (measured 2026-10-07)
|
||||
|
||||
Sample drawn with `--entropy 1 --entropy-sd 0.5` (μ=1 explicit, σ default 0.5): family weight $\exp(-2(H-1)^2)$, $H$ = `entropy15` in bits; weight 1 at $H=1$, 0.135 at $H=0$ or $2$, $\approx 3\!\times\!10^{-4}$ at $H=3$.
|
||||
|
||||
Comparison, per genome pair, of the full-index `family_overlap` $O^{\text{full}}$ (index scan) to the sample's overlap $O^{\text{samp}}$ (5.0 M columns), sampling fraction $r_{ij}=O^{\text{samp}}_{ij}/O^{\text{full}}_{ij}$ (uniform draw would give constant $r$):
|
||||
|
||||
| Quantity | Value |
|
||||
|---|---|
|
||||
| Spearman $O^{\text{full}}$ vs $O^{\text{samp}}$ | 0.994 (Pearson on logs 0.998): ranking preserved |
|
||||
| global fraction $\sum O^{\text{samp}}/\sum O^{\text{full}}$ | 0.52 % |
|
||||
| $r$ quantiles 5 / 50 / 95 % | 0.0025 / 0.0037 / 0.0055 |
|
||||
| slope $\log r$ vs $\log O^{\text{full}}$ | +0.12 |
|
||||
| Spearman($r$, patristic distance) | −0.71 |
|
||||
|
||||
Per decile of $O^{\text{full}}$, median $r$ rises from 0.0027 (poorest decile) to 0.0053 (three richest, plateau): the poorest pairs are represented about half as much as the richest, so the draw amplifies the imbalance. $r$ climbs slowly over deciles 1-6 (0.0027 to 0.0037), jumps at deciles 7-8 (0.0045-0.0052), where the $O^{\text{full}}$ distribution has a gap (689 k to 11.6 M, presumably close intra-clade pairs, unchecked).
|
||||
|
||||
Not established: (a) causality — a dependence of $r$ on pair richness is compatible with the entropy filter but no uniform control draw of the same size exists; (b) noise — the poorest sample pairs have tiny counts (min 4).
|
||||
|
||||
**Candidate mechanism: first hypothesis refuted, see the next section.** I first suggested that long-reach families, carried by many divergent genomes, spread over several states ($H\approx2\text{-}3$) and are downweighted from above by the Gaussian. The full entropy table (next section) shows the opposite: families with many carriers are mostly *low*-entropy, and are downweighted from below.
|
||||
|
||||
**Consequence for the pair-weight proposal:** entropy and reach are likely correlated, not independent criteria. Since the entropy weight of each family is known, the pair weight can compensate it exactly (divide by it) rather than compose with it; order of the two filters must then be defined.
|
||||
|
||||
## Entropy vs number of carrier genomes, full index (measured 2026-10-09)
|
||||
|
||||
Source: `phylo --shannon`, which is **always a full, unsampled scan** of every non-monomorphic minorant of every layer (`SiblingExt::shannon_entropy_csv(path)` takes no `subsample`/`entropy_bias` argument; `--subsample`/`--entropy` apply only to `--pseudo-alignment`/`--sankoff`/`snp-*`). Here: 19.7 GB CSV, 535 257 857 families, read in 11 s with DuckDB (no in-memory load). `UserDocMD/usage/phylo.md` line 121 ("`--subsample N --shannon` gives a bounded diagnostic sample") is stale and contradicts line 55 and the code; to fix.
|
||||
|
||||
| Carriers $n$ | Families | mean $H$ | median $H$ | $P(H<0.5)$ | $P(H\ge1.5)$ | mean weight ($\mu=1,\sigma=0.5$) |
|
||||
|---|---|---|---|---|---|---|
|
||||
| 2 | 163.3 M | 0.66 | 1.00 | 34.5 % | 0 % | 0.70 |
|
||||
| 3-4 | 104.6 M | 0.94 | 0.92 | 0.5 % | 7.9 % | 0.93 |
|
||||
| 5-8 | 91.8 M | 0.83 | 0.72 | 0.1 % | 4.6 % | 0.80 |
|
||||
| 9-16 | 62.7 M | 0.70 | 0.50 | 43.6 % | 4.9 % | 0.66 |
|
||||
| 17-32 | 45.4 M | 0.59 | 0.35 | 58.9 % | 5.9 % | 0.53 |
|
||||
| 33-64 | 48.1 M | 0.34 | 0.19 | 79.7 % | 1.1 % | 0.40 |
|
||||
| 65-128 | 19.1 M | 0.37 | 0.18 | 77.4 % | 3.8 % | 0.37 |
|
||||
| 129-180 | 0.19 M | 0.83 | 0.47 | 53.0 % | 21.4 % | 0.46 |
|
||||
|
||||
Findings:
|
||||
|
||||
- Families with many carriers are mostly one dominant state plus rare variants ($H<0.5$ for ~77-80 % at $n\ge33$); only 1-4 % have $H\ge1.5$. The Gaussian at $\mu=1$ therefore penalizes them from the **low** side (weight 0.135 at $H=0$), not the high side.
|
||||
- Relative to a uniform draw, expected inclusion is about ×1.32 at 3-4 carriers and ×0.53-0.56 at 33-128 carriers: a factor ~2.4 between small and large $n$. Same order as the ~2× pair-level undersampling of poor pairs measured above; compatible, but a family-level ratio is not a pair-level one, so causality remains unproven (no uniform control draw).
|
||||
- The filter does what it is designed for (drop near-autapomorphic variants), but those low-entropy high-$n$ families are also the ones covering distant pairs, hence the side effect.
|
||||
- Informative high-$n$ families are scarce: at $n>32$ only ~19 % have $H\in[0.5,1.5)$, and the 129-180 class is 0.04 % of all families. Flattening deep-pair coverage works on little informative material whatever the weights.
|
||||
- Consequence for a $\mu(n)$ that grows with $n$: it would not recover these families (80 % sit below 0.5 bit); what matters at large $n$ is the share of families in the band around 1 bit, not the position of the centre. Conditional weight $w_H(f)/E[w_H\mid n_f]$ remains an option to make the entropy filter neutral to $n$; $E[w_H\mid n]$ is now available from this table.
|
||||
|
||||
Limit: the table has `n_genomes_present` but not the reach of the family, so the link to specific pairs is still not measured.
|
||||
|
||||
## Site concordance factors on the Primulaceae tree (measured 2026-10-12)
|
||||
|
||||
Command (IQ-TREE 3.1.3): `iqtree3 -t <treefile> -s <alignment> -m GTR+G4+ASC --scfl 100 --cf-verbose -T 5 --prefix concord`. With `-t` plus `-m`, IQ-TREE did **not** just annotate the reference tree: it ran a full ML search started from it (102 iterations, ~59 h wall-clock). `concord.treefile` differs from the input treefile by one bipartition (Robinson-Foulds 2), so sCF refer to the re-inferred tree. Without `-m`, ModelFinder is launched and required 157 GB (first attempt failed). The log also warns that all 180 sequences contain more than 50 % gaps/ambiguity. `sN` below = informative sites averaged over the 100 sampled quartets of a branch (alignment: 5 003 414 columns).
|
||||
|
||||
**Deep Primula/Androsace interface: too few informative sites (data scarcity).**
|
||||
|
||||
| Node | Tips (Primula / Androsace) | sCF | sDF1 | sDF2 | `sN` |
|
||||
|---|---|---|---|---|---|
|
||||
| 209 | 67 (43 / 24) | 27 | 47 | 26 | 17 |
|
||||
| 210 | 66 (43 / 23) | 37 | 33 | 30 | 43 |
|
||||
| 208 | 74 (50 / 24) | 54 | 16 | 29 | 190 |
|
||||
| 211 | 24 (2 / 22) | 64 | 18 | 18 | 226 |
|
||||
| 212 | 21 (2 / 19) | 30 | 38 | 32 | 184 |
|
||||
| 216 | 10 (2 / 8) | 21 | 47 | 32 | 7 |
|
||||
| 217 | 9 (1 / 8) | 43 | 20 | 36 | 7 |
|
||||
|
||||
Quartets crossing these nodes have tens to a few hundred informative sites out of 5 M columns, the quantitative counterpart of the pair-overlap scarcity. Primula tips are spread over several nodes mixed with Androsace (e.g. 43 + 24 at node 209); no monophyletic Primula in this tree.
|
||||
|
||||
**Within Androsace: conflict, not scarcity.** Nodes 283 (72 Androsace, `sN` 17 127), 292 (49 tips, 10 335), 305 (22, 11 980), 342 (11, 6 566) have thousands of informative sites but sCF 33-37 % with both alternatives nearly as supported: sites split almost evenly among the three quartet topologies. More sites would not fix this; cause not established (a poor fit of a tree model to intra-species divergence is one hypothesis, untested). Node 185 (23 Androsace): sDF1 53.6 % > sCF 30.6 %, a majority of the 893 sites prefer an alternative (real conflict).
|
||||
|
||||
**Well-supported nodes have long branches:** node 296 (45 tips, length 0.23, sCF 90 %), 341 (0.21, 91 %), 330 (0.09, 85.5 %), 181 (0.099, 89.5 %); most branches of length 0.01-0.05 have sCF 33-45 %. Degenerate: node 186 (length 0.001, `sN` 0.37) and 187 (`sN` 2.2) carry no informative site.
|
||||
|
||||
Median sCF shows no gradient with clade size (about 39-48 % in every class), so the clade-size class hides the effect; the scarcity is specific to the interface nodes above.
|
||||
|
||||
**Implication for sampling.** The part a sampling design can address is the interface scarcity: select families carried by genomes from the four groups around each target node (a quartet-targeted selection) rather than flattening the whole pair matrix. Not assessed: how many such families the index holds (the `sN` of nodes 209 and 216 suggests few). For the intra-Androsace conflict no sampling strategy is suggested by these measurements alone.
|
||||
|
||||
## Quartet coverage (measured 2026-10-12, `--quartet-test`)
|
||||
|
||||
`scripts/pseudo_alignment_nesting.py --quartet-test --quartets N` draws random genome quartets and counts exactly the columns where all four carry a call, stratified by the size of the smallest tree clade containing the four. Cost: 20 000 quartets over 5.0 M columns in about 35 s total. Exact per-family quartet weighting is out of reach ($\binom{180}{4}\approx 42.3$ M quartets; one family carried by 100 genomes covers $\binom{100}{4}\approx 3.9$ M), so quartets serve as a *diagnostic* of a sample, not as a weight.
|
||||
|
||||
Result on the current sample: 4-way count quantiles 0 / 1 / 10 / 50 / 90 / 99 / 100 % = 0, 3, 68, 200, 2 329, 66 504, 194 460 (CV 3.5, 0.1 % uncovered). By depth: median 131-209 for quartets inside clades ≥ 65 genomes versus 11 580-57 387 for clades 17-64. The ratio of the 4-way count to the smallest of the six pair overlaps has median 0.20-0.29 for deep quartets (0.53 for clades 33-64; 1 would be perfect nesting): pair overlap is only a loose upper bound of 4-way coverage, so flattening pairs does not by itself flatten quartets. 88 % of random quartets fall in the deepest class (root as written; strata with < 5 quartets are noise). Co-presence is not informativeness (a column must also differ among the four). Next measurement: repeat on a sample drawn with the pair weight and check whether the ratio moves toward 1.
|
||||
|
||||
## Two kinds of family loss: sequencing dropout vs divergence (2026-10-12)
|
||||
|
||||
- **Sequencing dropout (genome skimming, ~1× coverage).** About $e^{-1}\approx 37\%$ of each genome is uncovered, independently per family. Predicted for a family truly present in a clade of $m$ genomes: completeness ≈ 0.63-0.70 and clade-closed fraction $c^m$, independent of clade depth. Consistent with small clades (3-4 tips: completeness 0.71, 20 % closed vs $0.65^3=27\%$, $0.65^4=18\%$). Does not predict the observed decay at larger clades (0.45 at 5-8 tips, 0.21 at 17-32, 0.08-0.12 beyond).
|
||||
- **Loss by divergence (user observation).** A family is a kmer whose 30 flanking positions (15 each side for $k=31$) are identical, with a variable central base; with per-site divergence $d$ the flanks survive with probability ≈ $(1-d)^{30}\approx e^{-30d}$ (0.74, 0.21, 0.042, 0.0012 for $d$ = 0.01, 0.05, 0.10, 0.20). Phylogeny-dependent and tied to local conservation, unlike sequencing dropout. Explains the loss of completeness with clade size, and sets a floor: no weighting can sample families that no longer exist for pairs whose $e^{-30d}$ is negligible. Surviving deep families are selected for conserved flanks, hence plausibly slow-evolving (fewer variable sites for deep splits; hypothesis, untested).
|
||||
- The patristic distances of the IQ-TREE tree (1 to 3.5) come from an alignment with variable sites only, hence strongly inflated; the true $d$ is unknown and these numbers cannot be plugged into $e^{-30d}$.
|
||||
- **Test (user's plan), done 2026-10-12: see the next section.**
|
||||
|
||||
## Overlap vs Mash distance on the full index (measured 2026-10-12)
|
||||
|
||||
Inputs: `phylo --distance mash --csv --shared-kmers --nj` on the 180-genome index (distance matrix, shared-kmer matrix, NJ tree) against the full-index `family_overlap` $O_{ij}$. Mash $D$: range 0.011-0.337, median 0.20.
|
||||
|
||||
| Regression | Slope | $R^2$ |
|
||||
|---|---|---|
|
||||
| $\log O_{ij}$ vs $D$, all pairs | -26.95 | 0.979 |
|
||||
| $\log$(shared kmers) vs $D$ | -31.87 | 0.984 |
|
||||
| $\log O_{ij}$ vs $D$ + additive genome effect | -25.3 | 0.9999 |
|
||||
|
||||
Spearman($D$, $O$) = -0.95; Spearman($D$, shared kmers) = -0.95; Spearman($D$, IQ-TREE patristic) = 0.81.
|
||||
|
||||
Reading:
|
||||
|
||||
- Overlap decays log-linearly with distance, slope about -27 versus -30 predicted by flank survival (previous section). **Not an independent confirmation of the flank model**: Mash $D$ is derived from kmer sharing and $O$ counts families defined by kmers, so both measure the same shared content. The kmer slope of -32 is $-k$ by construction of Mash (implementation check only). The ratio of the two slopes, 27/32 ≈ 0.85, is below the 30/31 ≈ 0.97 expected from flank survival alone; cause unknown, the gap is modest.
|
||||
- Local slopes vary: -21 for $D\in[0.011,0.07]$ ($R^2$ 0.45), -28 for [0.07, 0.20] (0.96), -35 for $D>0.23$ (0.83). Half the pairs lie in $D\in[0.20,0.23]$, where the fit explains almost nothing ($R^2$ 0.26 and 0.07): a plateau where Mash has little resolution.
|
||||
- Genome effects (log scale, sd 0.21): lowest `Androsace_saussurei--PHA010690` (factor ≈ 0.39), `Androsace_triflora--PHA000558` (≈ 0.62), `Androsace_vesulensis--PHA010698` and `Primula_stricta_var_obesior--TROM_V_149128` (≈ 0.65); highest `Androsace_pubescens--PHA010697` (≈ 2.8). Three of the four lowest are also among the genomes with the fewest calls in the pseudo-alignment (saussurei, triflora, stricta), pubescens has the most; consistent with a coverage effect at the extremes, correlation over all genomes not computed.
|
||||
- NJ on Mash vs the IQ-TREE tree: Robinson-Foulds 182 of 354 (0.51 normalized); the Mash plateau near 0.2 probably limits NJ resolution for most pairs.
|
||||
- Consequence for normalization: the residual $\log O_{ij}-(a+bD_{ij})$ measures coverage deviation at a given distance, a normalization of $O$ that needs no per-genome call counts. Untested on a weighted sample.
|
||||
|
||||
## Data-poor genomes (measured 2026-10-12)
|
||||
|
||||
Per-genome share of the 5 003 414 sampled columns with a call: median 5.2 %, max 14.3 %; 8 genomes below 1 % and 23 below 2 %. Fewest calls: `Androsace_triflora--PHA000558` 1 936, `Primula_stricta_var_obesior--TROM_V_149128` 2 403, `Primula_carniolica--PHA007185` 4 937, `Androsace_saussurei--PHA010690` 6 986, `Primula_pumila--TROM_V_135924` 17 393, `Androsace_maxima--PHA000534` 17 939, `Primula_auricula_subsp_serratifolia--PHA007184` 30 621, `Androsace_pyrenaica--PHA000543` 37 725. Three of them (saussurei, serratifolia, pyrenaica PHA000543) are among the tips whose placement looked aberrant, previously attributed to possible mislabelling or contamination. A simpler alternative is too little data to place them reliably; testable by removing them (`--exclude-genome`) and watching deep-node support.
|
||||
|
||||
Consequence for flattening: raw $O_{ij}$ mixes evolutionary distance and per-genome coverage; flattening on it chases pairs involving data-poor genomes, for which no family exists. Normalizing, $O_{ij}/(c_ic_j)$ with $c_i$ the number of families carried by genome $i$ (per-genome counts from `--sibling-stats`), targets distance instead of coverage; the best normalization (smoothing of $c_i$, etc.) is not established.
|
||||
|
||||
## Implemented: entropy annex parametrised by `--no-ambiguity` (2026-10-13)
|
||||
|
||||
Design decision (user): the way genomes are counted depends on `--no-ambiguity`, so that flag, not an "alphabet" option, is the annex parameter. Over single-member genomes the 15-state entropy only has the four single-base states, hence `entropy4` (redefined: 4 bases over single-member genomes) equals `entropy15` under `--no-ambiguity`; a unit test checks it.
|
||||
|
||||
- `entropy_annex.rs`: header `flags` (the former padding), bit 0 = `no_ambiguity`; old files have 0 and stay valid. One file per convention (`entropy.pent`, `entropy.noamb.pent`), so switching the flag builds the missing one without discarding the other. `EntropyAnnex::no_ambiguity()`; `iter_full_entropy` rejects a header that disagrees with the requested convention.
|
||||
- `ensure_layer_entropy_annex` takes the list of conventions to build; one scan feeds every missing annex and the overlap accumulator. `sample_index` builds and reads the convention matching its `no_ambiguity` argument, removing the mismatch between the draw (previously always raw masks) and the alignment (masked).
|
||||
- `--sibling-annex` builds both conventions and invalidates both on rebuild. `--shannon` no longer builds any annex as a prerequisite (it recomputes everything from its own scan).
|
||||
- Checked on the 1x benchmark index: a `--no-ambiguity` run creates `entropy.noamb.pent` beside the existing `entropy.pent` in all 512 layers; workspace tests pass.
|
||||
- Still to do, in this order: `--quartet` (per-family bit annex, same parameter), `--pair-balance` (mean of 1/shared-families over the carrier pairs; second overlap matrix for the single-member convention, annex header carrying a fingerprint of the matrix it came from).
|
||||
|
||||
## Implemented: `--quartet` (2026-10-13)
|
||||
|
||||
`--quartet` keeps only the families with at least two called states each carried by at least two genomes (`masking::is_quartet_informative` for the raw masks, `survives_quartet` for the exact check on masked states and `--exclude-genome`). Absence never counts as a state; with `--no-ambiguity`, multi-member genomes are left out of the count. The persisted annex is a plain `PersistentBitVec` (one bit per non-monomorphic family, compact ordinal) — no ad hoc format; the `--no-ambiguity` convention is carried by the file name (`quartet.pbiv`, `quartet.noamb.pbiv`) because that header has no room for a flag. It is a safe pre-filter: excluding genomes can only destroy a bipartition, never create one.
|
||||
|
||||
Plumbing: `AnnexRequest` (which annexes, per convention) and `ensure_layer_family_annexes` (one shared scan for entropy, quartet and overlap) replace the entropy-only builder; the sampling parameters travel as one `SamplingSpec { free_loss, no_ambiguity, quartet, entropy_bias }` instead of a growing positional list.
|
||||
|
||||
First test on the 1x benchmark testbed (3 seeds, exhaustive alignment, `GTR+G4+ASC`, species monophyly out of 3 per seed): without filter 1 / 0 / 0 correct monophylies; carriers >= 4 only: 3 / 1 / 2; `--quartet`: 2 / 2 / 3 (370, 148, 369 columns, all informative by construction). *Klebsiella* stays fragile (wrong in 2 seeds out of 3); two seeds still show a branch at the IQ-TREE ceiling of 10 with so few columns. Small testbed, three seeds: a direction, not a measurement.
|
||||
|
||||
## Implemented: `--subsample` renamed `--n-sites N|all` (2026-10-13)
|
||||
|
||||
`--subsample` read as "draw a subset", while `--pseudo-alignment`/`--sankoff` made it mandatory and accepted no way to say "everything" (the workaround was a number larger than the family count, e.g. `--subsample 100000000`). The option is now `--n-sites <N|all>`: `all` is an explicit value, parsed into its own type (`NSitesArg::{All, N}`, like `GammaShapeArg`) rather than an in-band sentinel (negative number, `0`); `all` gives the exhaustive scan the `snp-*` path already ran when the option was omitted. Still mandatory for `--pseudo-alignment`/`--sankoff`; still optional for `snp-*` `--distance`. No alias is kept for the old name (the software is in active development). The benchmark commands and measurements recorded in the earlier sections of this file were run with the old name.
|
||||
|
||||
## Implemented: `--pair-balance` (2026-10-13)
|
||||
|
||||
Weight of a family = **mean** over the pairs of its carriers of 1/O (O = index-wide shared variable families of the pair, same `--no-ambiguity` convention): the simulation on the *Primulaceae* sample (above) had the mean flatten best (expected-overlap range 23 000× to ~600×, slope 0.37) without chasing high-carrier families, unlike the sum (65 % of the draw at >= 33 carriers).
|
||||
|
||||
- `algorithms/pair_weight.rs`: `inverse_overlap_matrix` (dense n×n of 1/O from the persisted overlap annex), `pair_balance_weight` (mean over carrier pairs; 0 below two carriers). Carriers are defined once, `helpers::is_carrier(mask, no_ambiguity)`, shared with the overlap accumulator and the quartet test.
|
||||
- Storage: no new format. The entropy annex is generalised to `FamilyScalarAnnex` with a `ScalarKind` (`Entropy` magic `PENT`, `PairWeight` magic `PPWT`, one file per kind and convention, header flag `no_ambiguity`); `entropy_annex.rs` became `family_scalar_annex.rs`. Family overlap is now one file per convention too (`family_overlap.povl`, `family_overlap.noamb.povl`, header flag); `--family-overlap` and `--min-shared-family` follow `--no-ambiguity`.
|
||||
- Build order: sweep 1 builds entropy, quartet and the missing overlap annexes together; sweep 2 builds the pair weights, which need the finished matrix (`ensure_layer_pair_weight_annexes`). `AnnexRequest` now lists `entropy`, `quartet`, `overlap`, `pair_weight`; a pair weight implies its overlap. A rebuilt sibling annex removes every annex derived from it (entropy, quartet, pair weight, overlap), so a weight can never outlive the matrix it came from; there is no fingerprint, only that single invalidation point.
|
||||
- Sampler: `LayerPlan` (eligible bits, minorant count, mass) replaces the positional tuple; a layer's quota is `n × mass / Σ mass` (mass = eligible count without the option, Σ weights with it); a family is accepted with probability `quota/mass × weight × entropy kernel`, capped at 1 (the cap, and kernels below 1, undershoot the quota, absorbed by the existing top-up rounds).
|
||||
- Cost note: weights need O(carriers²) per family at build time; not measured on a large index.
|
||||
|
||||
First test, 1x benchmark testbed (20 genomes, 46 706 variable families, `--n-sites 5000 --free-loss --no-ambiguity`, 8 replicates each; sample overlap computed from the alignment, compared with the index `--family-overlap --no-ambiguity`):
|
||||
|
||||
| | slope of log(sample overlap) on log(index overlap) | pairs (index overlap > 0) covered by >= 1 column | columns with >= 10 carriers |
|
||||
|---|---|---|---|
|
||||
| uniform | 1.02 ± 0.14 | 64 % | 0.18 % |
|
||||
| `--pair-balance` | 0.84 ± 0.001 | 97 % | 1.1 % |
|
||||
|
||||
The weight flattens the representation (slope toward 0) and, above all, covers most of the pairs a uniform draw misses. Not measured: effect on the tree (sCF / monophyly), interaction with `--entropy`, behaviour on the *Primulaceae* index. 20 genomes only.
|
||||
|
||||
## Quota rounding fix, and the combined test on the 1x testbed (2026-10-13)
|
||||
|
||||
**Fix.** `sample_index` rounded each layer's quota down on its own, losing up to one site per layer: with 512 layers a budget of 100 sites gave 0 to 3 columns. Quotas now use cumulative rounding (`floor(n × cumulative mass / total)` minus what earlier layers got), so they sum to exactly `n`; a 100-site budget now yields 50-80 usable columns. Not a result of the options below, a defect the small testbed exposed.
|
||||
|
||||
**Test.** 1x testbed, 3 seeds × 10 replicates per variant, 11 genomes (the 9 genomes with fewest calls excluded, so no all-missing sequence), `--free-loss --no-ambiguity`, IQ-TREE `GTR+G4+ASC` on each sampled alignment; score = number of the three species (*E. coli*, *Salmonella*, *Klebsiella*) recovered monophyletic (0-3), mean over the 30 runs; "informative" = columns with two bases each carried by >= 2 genomes.
|
||||
|
||||
| `--n-sites` | variant | trees obtained | columns | informative columns | mean species monophyletic (of 3) | all 3 |
|
||||
|---|---|---|---|---|---|---|
|
||||
| 100 | uniform | 77 % | 49 | 0.5 | 0.04 | 0 % |
|
||||
| 100 | `--quartet` | 100 % | 82 | 82 | 2.30 | 50 % |
|
||||
| 100 | `--pair-balance` | 100 % | 52 | 2.1 | 0.20 | 0 % |
|
||||
| 100 | `--quartet --pair-balance` | 100 % | 67 | 67 | 2.53 | 53 % |
|
||||
| 100 | `... --entropy 1` | 100 % | 67 | 67 | 2.17 | 17 % |
|
||||
| 300 | uniform | 93 % | 163 | 2.1 | 0.04 | 0 % |
|
||||
| 300 | `--quartet` | 100 % | 239 | 239 | 2.33 | 33 % |
|
||||
| 300 | `--pair-balance` | 100 % | 179 | 6.2 | 0.57 | 0 % |
|
||||
| 300 | `--quartet --pair-balance` | 100 % | 197 | 197 | 2.67 | 67 % |
|
||||
| 300 | `... --entropy 1` | 100 % | 196 | 196 | 2.67 | 67 % |
|
||||
|
||||
(Mean over all runs counting a failed tree as 0 is identical to the "mean species" column wherever trees were obtained; the uniform rows lose runs because IQ-TREE refuses alignments where a genome has no data.)
|
||||
|
||||
Reading: `--quartet` is the decisive option here — a uniform draw of 100-300 sites contains almost no informative column (0.5-2), the restricted draw contains only informative ones. `--pair-balance` alone barely helps (it favours rare pairs but does not remove the uninformative columns); added to `--quartet` it raises the share of replicates recovering all three species (50→53 % and 33→67 %). The entropy kernel (centre 1 bit, default width) adds nothing at 300 sites and costs at 100. Limits: 11 genomes, three simulated 1x datasets, small budgets, a single criterion (species monophyly, not deep branches), and the differences between the `--quartet` variants (a few runs out of 30) are small relative to the replicate noise — a direction, not a ranking.
|
||||
|
||||
## Implemented: no distance matrix unless one is wanted (2026-10-13)
|
||||
|
||||
`--distance` had a default (`jaccard`) and `phylo` always computed and wrote a matrix, even for `--sibling-annex`, `--family-overlap` or `--pseudo-alignment` alone. `--distance` is now optional: a matrix is computed only if `--distance` is given, or `--nj`/`--upgma`/`--shared-kmers` needs one (then `jaccard`, `args::distance_to_compute`, unit-tested); `--csv` alone changes nothing. When nothing is wanted the command stops before the distance step. Checked on the 1x testbed: `--family-overlap` and `--pseudo-alignment` runs write no `_dist` file and run no distance stage; `--nj` writes `_dist.phy` and `_nj.nwk`; `--distance mash --csv` writes `_dist.csv`. Cost saved on a large index: not measured.
|
||||
|
||||
## Implemented: diagnostic outputs (2026-10-13)
|
||||
|
||||
Added so the measurements this file reports no longer need ad hoc scripts, and so a draw can be diagnosed from the files of the run that produced it:
|
||||
|
||||
- `--sample-stats`: a `SampleStatsReducer` (`algorithms/sample_stats.rs`), a `LayerReducer` run beside the alignment over the same surviving families; writes `_sample_genomes.csv`, `_sample_carriers.csv`, `_sample_overlap.csv`. A call is a masked state other than `0`; informative = `has_bipartition` (the quartet rule, now shared). Checked on the 1x testbed: per-genome calls and the overlap matrix equal those recomputed from the alignment.
|
||||
- `--sample-plan`: `sampled_scan` now returns a `SamplePlan` (one `LayerReport` per layer: eligible, mass, quota, resolved, kept); `_sample_plan.csv` has the layer lines only (quotas sum to the target by cumulative rounding; on the testbed 3000 asked, 2695 kept).
|
||||
- `--shannon-summary`: `--shannon` and the summary share one traversal (`scan_family_entropies`, which replaces the loop that lived in `shannon_entropy_csv`); the summary keeps counts per (entropy kind, genomes present, genomes without ambiguity, 0.05-bit bin) instead of one line per family: 156 lines for the 46 706 families of the testbed (a few hundred thousand lines at most on any index, by construction: the bins are bounded). Checked: its `entropy15` counts sum to the number of variable families.
|
||||
- Writers are in `cmd/phylo/stats.rs`; the library computes, the command layer formats and writes.
|
||||
|
||||
## Combined draw on Primulaceae: `--quartet --pair-balance --entropy 1` (measured 2026-10-13)
|
||||
|
||||
Command: `phylo --entropy 1 --pseudo-alignment --n-sites 5000000 --free-loss --no-ambiguity --quartet --pair-balance -o primulaceae_quartet_balance --family-overlap --sibling-stats primulaceae`. One draw, 180 genomes, 5 000 000 columns requested and obtained (no shortfall; no `--sample-plan`, so per-layer quotas are not inspectable). The three options are on together: **effects of entropy, quartet and pair balance are not separable here**; the testbed result that the entropy kernel adds nothing at small budgets was not rechecked.
|
||||
|
||||
**Baseline.** The same statistics were recomputed with one script on the alignment fed to IQ-TREE (`primulaceae-gamma_alignment.fasta`, 5 003 414 columns). It reproduces the figures of the earlier sections (sample overlap min 4 / median 1 665 / max 387 057 / CV 1.57; sampling fraction $r$ quantiles 0.0025 / 0.0037 / 0.0055; 5.2 % median calls per genome), hence **is the entropy-drawn sample** (`--entropy 1 --entropy-sd 0.5`, section on entropy bias), not a uniform draw. The "uniform draw" label of the nesting section is inconsistent with this (to settle). So the comparison below is *entropy-only vs entropy + quartet + pair balance*, never against a uniform control. A call is any character other than `?`/`-`. With `--free-loss` absence is written `?` (`masking::masked_state`); `-` only occurs without it. Neither file contains IUPAC characters.
|
||||
|
||||
| | Baseline (entropy only) | Combined |
|
||||
|---|---|---|
|
||||
| Informative columns (≥ 2 bases, each in ≥ 2 genomes) | 21.4 % | 100 % |
|
||||
| Carriers per column: median / share ≤ 4 / share ≥ 33 | 4 / 58.1 % / 8.0 % | 8 / 12.4 % / 11.3 % |
|
||||
| Calls per genome, median; genomes < 1 % / < 2 % | 5.2 %; 8 / 23 | 8.1 %; 4 / 6 |
|
||||
| Sample pair overlap min / median / max | 4 / 1 665 / 387 057 | 282 / 27 380 / 491 038 |
|
||||
| Sample overlap CV; max/min | 1.57; 96 764 | 1.10; 1 741 |
|
||||
| 4-way count (20 000 random quartets, 2 seeds): median | 200-202 | 3 738-3 797 |
|
||||
| 4-way p1 / p10 | 3 / 66-67 | 93-95 / 1 092-1 104 |
|
||||
| 4-way p90 / p99 | 3 000-3 644 / 65 700-67 300 | 14 500-15 600 / 82 900-84 600 |
|
||||
| 4-way CV; share of quartets with 0 | 3.4-3.5; 0.11-0.14 % | 1.83-1.87; 0 % |
|
||||
|
||||
(p90/p99 move by 20 % between the two seeds: read them as orders of magnitude only.) The gain is at the bottom of the 4-way distribution (median ×19, p1 ×30); the top is unchanged. 4-way coverage is not stratified by clade depth here (needs the tree). "Informative" is measured over the 180 genomes: a column informative overall is not necessarily informative among the four genomes of a given quartet.
|
||||
|
||||
**Pair representation.** Index-wide overlap used for the comparison: baseline, the raw matrix (`primulaceae_phylo_family_overlap.csv`); combined, the `--no-ambiguity` matrix of the same run. Different conventions, so the two slopes are indicative.
|
||||
|
||||
| | Baseline | Combined |
|
||||
|---|---|---|
|
||||
| Slope of log(sample overlap) on log(index overlap) | 1.12 ($R^2$ 0.997) | 0.54 ($R^2$ 0.91) |
|
||||
| Pairs with index overlap > 0 and no sample column | 0 | 0 |
|
||||
| Median $r$ = sample/index, poorest decile of index overlap | 0.0027 | 0.059 |
|
||||
| Median $r$, richest decile | 0.0053 | 0.0067 |
|
||||
| Median $r$ by decile 1 → 10 | 0.0027 … 0.0053, rising | 0.059, 0.065, 0.067, 0.067, 0.069, 0.074, 0.023, 0.0082, 0.0077, 0.0067 |
|
||||
|
||||
The weight does not merely flatten: the poorest pairs are now about 9× better represented (relative to their pool) than the richest, where the baseline gave them about half. Deciles 1-6 are flat; the drop sits at deciles 7-10, around the gap of the index-overlap distribution already noted in the entropy-bias section (cause of that gap still unchecked). **Caveat on $r$:** the numerator only holds quartet-informative families while the denominator counts every variable family, and the matrices differ in convention, so $r$ is no longer a sampling fraction of one pool; the decile pattern is a measurement of the sample, not of an inclusion probability.
|
||||
|
||||
**Per-genome richness.** `--sibling-stats` now gives the variable families (size ≥ 2) per genome: 0.72 M to 93.9 M, median 34.5 M. Spearman between a genome's calls in the sample and its variable-family count: **0.974 in the baseline, 0.435 in the combined draw**: the baseline mirrored genome richness, the combined draw largely decouples from it (on the same genomes the correlation with *all* families is 0.66 and 0.32). Genomes poorest in variable families: *A. saussurei* 0.72 M (0.30 % of its families), *A. triflora* 1.26 M (0.72 %), *P. stricta var. obesior* 2.39 M (1.33 %), then *P. pumila* 5.2 M, *P. auricula* subsp. *serratifolia* 5.9 M (1.92 %), *P. carniolica* 6.8 M (1.77 %), *A. pyrenaica* PHA000543 8.4 M (1.46 %). Their calls in the sample went from 6 986 / 1 936 / 2 403 (saussurei, triflora, stricta) to 15 725 / 22 988 / 25 392, and carniolica from 4 937 to 52 310; *saussurei* stays the poorest genome in both. These genomes are data-poor, not empty: their families are mostly of size 1. This is the per-genome $c_i$ the normalisation of the data-poor section calls for.
|
||||
|
||||
Global family-size histogram (`global` row): 34 047 616 087 families; sizes 1 / 2 / 3 / 4 = 33 512 358 231 (98.43 %) / 479 910 992 (1.41 %) / 45 621 663 (0.13 %) / 9 725 201 (0.03 %). Sizes ≥ 2 sum to 535 257 856, matching the 535 257 857 of the entropy table within 1.
|
||||
|
||||
**`_dist.phy` (not asked for).** 180 × 180, symmetric, values 0.456-0.99999, median 0.999; Spearman with the Mash matrix 0.9999998 and with the index overlap −0.95, so it carries Mash-like information, consistent with the `jaccard` default (not verified). `args::distance_to_compute` at HEAD computes no matrix for this command line; why the file exists is unknown (binary predating the change, or a leftover; not checked). Not used above.
|
||||
|
||||
**Not measured:** any tree-level effect (sCF, species monophyly, support at the interface nodes 209/216 where `sN` was 17 and 7); the separate contribution of entropy, quartet and pair balance (needs ablation runs: without `--entropy`, without `--pair-balance`, `--quartet` alone); variance between draws (one draw per setting); clade-stratified 4-way coverage; the new alignment had not been given to IQ-TREE (done 2026-10-16: see the last measured section).
|
||||
|
||||
## Tree-level effect of the combined draw: sCF on three trees (measured 2026-10-16)
|
||||
|
||||
Closes the "not measured" item of the previous section (the combined alignment given to IQ-TREE). Three trees, all sCF by `--scfl 100 --cf-verbose`, model `GTR+G4+ASC`:
|
||||
|
||||
- **A** = `concord` (12/10): entropy-only draw, 180 genomes, 5 003 414 columns; `-t` + `-m` re-ran a full ML search (59 h).
|
||||
- **B** = `concord_quartet_balance`: combined draw (`--entropy 1 --quartet --pair-balance`), 180 genomes, 5 000 000 columns. Its log stops during the tree search (14/10), so the command that produced its `cf.*` (16/10) is not in the files; presumably `-te` on the 15/10 treefile as for G (unverified).
|
||||
- **G** = `scf_good`: same draw as B without the 3 data-poor genomes (*A. pyrenaica* PHA000543, *A. saussurei*, *A. triflora*), 177 genomes. 72 sites became invariant and were removed to satisfy ASC; the search (`concord_quartet_balance_good`, 107 iterations, 6 h 47 on 120 threads) crashed in the sCF step on the FASTA (`Assertion all_prob_const < 1.0`; cause untested, ASC with constant sites is a suspect). sCF were recomputed with `-te <treefile> -s <varsites.phy> -m GTR+G4+ASC --scfl 100`: **no search** (0 iterations, lnL −41 053 676.53 vs −41 053 676.41 from the search), 4 999 729 columns. A used `-t`, B/G a fixed tree: the protocols differ.
|
||||
|
||||
| | A | B | G |
|
||||
|---|---|---|---|
|
||||
| Branches | 177 | 177 | 174 |
|
||||
| Mean `sN` | 3 219 | 13 163 | 13 396 |
|
||||
| Branches with `sN` < 100 / < 1 000 | 13 / 50 | 0 / 5 | 0 / 2 |
|
||||
| Median sCF | 44.2 | 42.9 | 42.8 |
|
||||
| Branches with sCF < 40 / ≥ 70 | 74 / 28 | 67 / 26 | 69 / 25 |
|
||||
| *Primula* and *Androsace* monophyletic | no / no | no / no | yes / yes |
|
||||
|
||||
**Sampling effect (A → B, same 180 genomes).** `sN` ×4.5 and the scarcity is gone. Sites do not become more concordant: on the 113 shared bipartitions (of 177) median sCF 48.0 → 47.3 (44 up, 69 down). Robinson-Foulds 128: the draw rearranged the tree deeply. Consistent with the earlier reading that the intra-*Androsace* conflict is not a matter of site count; the number of branches with sCF < 40 % stays around 70.
|
||||
|
||||
**Interface nodes.** Of the old nodes 209/211/212/216/217 (`sN` 17 / 226 / 184 / 7 / 7) none exists in G. Node 208 survives with `sN` 190 → 5 218 (sCF 54.5 → 37.5). Six mixed clades remain in G (83 to 72 tips, 50 *Primula* + 22-33 *Androsace*), all with `sN` > 3 000.
|
||||
|
||||
**Effect of the 3 genomes (B → G, same draw).** RF 16 (166 of 174 shared), median sCF 43.2 → 43.6, median `sN` unchanged. In B, *A. pyrenaica* and *A. triflora* sit inside *Primula* (a clade of 52 tips = 50 *Primula* + these 2; each has a *Primula* sister, sCF 92 % and 52 %, `sN` 556 and 153); *A. saussurei* stays with *Androsace*. Removing them restores both monophylies and changes nothing else measurably: support for "too little data to place them" (data-poor section), not for contamination, though the latter is not excluded.
|
||||
|
||||
**Where A and G disagree (2026-10-16).** On the 177 common tips: 124 shared bipartitions, 50 only in A, 50 only in G. Differing bipartitions are weakly supported in both trees (median sCF 35.7 in A-only, 37.7 in G-only, versus 47.5-48.1 on the shared ones), so the disagreement sits where support was already low. Differences are mostly local: the closest bipartition of the other tree is 1 tip away for 25 of 50 A-only and 21 of 50 G-only splits (≤ 2 tips: 37 and 30); only 8 and 9 are 5+ tips apart. Greedy removal of the tip that lowers RF most: the first (*P. auricula* subsp. *serratifolia*) takes RF from 100 to 86, then about 2 per tip (*A. maxima*, *P. stricta* var. *obesior*, then ordinary tips); the top three are data-poor genomes. A bipartition separates a data-poor genome (the 5 left after the 3 removals) in 1 of 124 shared splits but in 23 of 50 A-only and 10 of 50 G-only splits; removing those 5 genomes gives RF 72, against 96.6 (range 90-100, 40 draws) for 5 random tips. So about 28 of the 100 differences trace to data-poor genomes, and the remaining 72 do not. Backbone: A has a clade of 50 *Primula* + 25 *Androsace* (sCF 76, `sN` 453), G one of 50 + 26 (sCF 73, `sN` 6 677): one tip apart. Genuine conflict: a clade of 72 *Androsace* exists in both trees (A: sCF 34, `sN` 17 127; G: sCF 38, `sN` 49 473) with 16 different tips, i.e. a topological disagreement carried by tens of thousands of sites in each, not a scarcity effect; consistent with the intra-*Androsace* conflict above.
|
||||
|
||||
**Not established.** Separate contribution of entropy / quartet / pair balance (no ablation); stability of the tree between two independent draws (one draw per setting; A → B shows the topology moves with the draw); whether the quartet-informative restriction biases branch lengths (ASC corrects for variable-only ascertainment, not for requiring two states each in two genomes; tree length 22.5 is not interpretable as substitutions/site).
|
||||
|
||||
## Open questions
|
||||
|
||||
- Per-family $S_f$ without materializing the full matrix: a first pass storing only a per-family summary (reach, $|S_f|$), as `--shannon` already does a full unsampled scan for entropy.
|
||||
- Combination with entropy bias: not independent criteria if entropy correlates with reach (see section above); options: compensate the known entropy weight inside the pair weight, or apply the Gaussian inside reach strata.
|
||||
- Pairs with near-zero $O_{ij}$ (suspected badly sequenced genomes) set the ceiling for everyone; `--min-shared-family`/`--exclude-genome` should be applied first so the floor is not dictated by anecdotal genomes.
|
||||
- Effect on the SNP-distance estimators: a non-uniform draw over pairs biases per-pair eligible-family counts, but each pair's $p$-distance is a ratio over its own eligible families, so unbiasedness per pair is expected to hold. To confirm before relying on it.
|
||||
@@ -1,60 +0,0 @@
|
||||
# Sibling partition unification and masked canonical form (discussion)
|
||||
|
||||
Status: **open, nothing implemented**. Records the state of the reflection on (a) routing the 4 members of a sibling family to one partition and (b) a canonical form that makes siblings directly comparable. Measurements are throwaway (not in the repo); figures are for $k=31$, $m=11$, 256 partitions, 20 bacterial genomes (98.4 M kmer positions), unless stated.
|
||||
|
||||
## Problem
|
||||
|
||||
A sibling family = the 4 kmers differing only at the central base (position $c=(k-1)/2$, $k$ odd). Routing is **not** a hash of all bases: a kmer goes to `minimizer().seq_hash() & mask`, and consecutive kmers of a superkmer share their minimizer. A family splits across partitions whenever the central base falls in (or changes) the selected m-mer.
|
||||
|
||||
Fraction of families whose 4 variants are not all in one partition, standard minimizer: $4m/(k+2m+1)$ (fixed candidates $k-2m+1$ versus $4m$ variable ones). Measured: 81.4 % at $m=11$ (60.2 % at $m=7$, 46.8 % at $m=5$). This is the *theoretical* rate assuming all 4 members present; for two present members it is ≈ 68 %. Real families are overwhelmingly of size 1, so it only becomes cost where something probes variants.
|
||||
|
||||
## Masked canonical form (Can3)
|
||||
|
||||
Notation: `CentreA(K)` = K with its central base set to A; `RC(K)` its reverse complement.
|
||||
|
||||
- Can1 = `min(K, RC(K))` (current). Each sibling canonicalised alone, so some may be flipped relative to the others.
|
||||
- Can2 = orient by `CentreA(K)` vs `RC(CentreA(K))`. **Not a valid canonical form**: when the flanks are reverse complements of each other ($L=\mathrm{rc}(R)$, frequency ≈ $4^{-c}$, 1/1024 at $k=11$), $K$ and $\mathrm{RC}(K)$ get different canonicals (k=5: `ACAGT` and `ACTGT` both kept). Cause: `CentreA(RC(K)) ≠ RC(CentreA(K))` (the latter has centre T).
|
||||
- **Can3** (retained):
|
||||
|
||||
```
|
||||
Can3(K) = K if CentreA(K) < CentreA(RC(K))
|
||||
= RC(K) if CentreA(K) > CentreA(RC(K))
|
||||
= Can1(K) otherwise // self-revcomp family
|
||||
```
|
||||
|
||||
Orientation is decided on the flanks only (centre ignored), tie broken on the full kmer. Gain: the 4 canonical siblings have identical flanks in the same orientation, so they compare directly with no reverse complement. Self-revcomp families have only 2 distinct canonical members (centres A/C; T/G fold onto them). The A↔T (C↔G) substitution in that context is undetectable by any double-strand canonical set, same as under Can1 — not a loss.
|
||||
|
||||
Gain is simplification only: `central_canonical_neighbors` already canonicalises each variant independently and is correct (see `evolutionary_distances.md`, per-partition sweep). The earlier "masked-flank grouping" there is the Can2 idea; Can3 is its repaired form. **No performance gain by itself**, and it breaks existing indexes (canonical orientation changes for some kmers → MPHF hashes change; rebuild).
|
||||
|
||||
Prerequisite if adopted: `Minimizer = CanonicalKmerOf<MLen>` shares the canonical type with index kmers; an m-mer has no sibling family and must keep Can1. Decouple with a distinct `Minimizer` type (6 files outside `kmer.rs`: `routable.rs`, `packed_seq.rs`, `obiskbuilder/{rolling_stat,stream_iter,iter}.rs`, `obifastwrite`).
|
||||
|
||||
## Routing that excludes the centre (scheme B) — rejected
|
||||
|
||||
Candidates restricted to m-mers not overlapping the centre ($m\le c$): $S(j)=\min$ hash over m-mers $p\in[j,j+c-m]$, minimizer$(i)=\min(S(i),S(i+c+1))$. Family- and strand-invariant by construction (checked). But unitigs are partition-local (`pipeline.md`), and the hole moves with each kmer, so runs shrink:
|
||||
|
||||
| $m$ | mean run, current | mean run, B | bases/kmer, current → B |
|
||||
|---|---|---|---|
|
||||
| 5 | 14.9 | 6.3 | 3.02 → 5.76 |
|
||||
| 11 | 11.0 | 3.0 | 3.72 → 10.96 |
|
||||
| 15 | 9.0 | 1.0 | 4.32 → 30.9 |
|
||||
|
||||
Proxy: run = unitig (upper bound; real unitigs may merge). Independent of partition count. In `benchmark/run/global_index_count` (786 MB) `unitigs.bin` + idx ≈ 241 MB (31 %), so tripling them ≈ +60 % total index. Hashing the masked kmer per kmer (no minimizer) would cut every path at every kmer: worse.
|
||||
|
||||
Structural tension: path coherence (long unitigs) needs a key shared by consecutive kmers, family coherence needs a key independent of each kmer's own centre, which is an ordinary base for its neighbour. Decoupling sequence storage from MPHF partitioning fails because a partition can only enumerate its kmers from its own unitigs (MPHF not invertible). Redefining the family on a terminal base would cut the split rate to ≈ 17 % but is not closed under revcomp and changes the SNP semantics. Lower $m$ reduces the split rate for free but deviates from the literature $k/m\approx 2.5$ optimum (balance, bucket size), unmeasured here — $m=11$ kept.
|
||||
|
||||
## What the cross-partition cost actually is
|
||||
|
||||
- Phylo-time: only families of size > 1 trigger remote lookups (a small share).
|
||||
- Annex construction probes the 3 variants of **every** kmer (size-1 families included), ≈ 2/3 of probes crossing partitions; batch bounded to `#kmers(p)*3` per partition (not the whole index).
|
||||
- Real remaining benefit of unification: the outer sweep over source partitions could run in parallel (it is sequential today for memory and for the `q >= p` dedup rule); plus code simplification in `family_scan.rs` / `annex.rs`. Not quantified.
|
||||
- Unification would also allow an MPHF keyed on the family (one lookup, 4-bit member mask, union/intersection as OR/AND); needs the family-size histogram (1–4 members) to judge storage.
|
||||
|
||||
## Open measurements (index not on this machine)
|
||||
|
||||
1. Family-size histogram of the real index: `obikmer phylo --sibling-hist` (needs a built annex).
|
||||
2. Time and peak RSS of `--sibling-annex` on the Vaccinium/Salix index (`Stage` "sibling_annex" traces time; peak RSS from outside).
|
||||
3. Only if those show the sequential sweep or the probe volume is a real bottleneck: real-index test of scheme B or of a smaller $m$.
|
||||
|
||||
## Decision state
|
||||
|
||||
No code change decided. Can3 + `Minimizer` decoupling is the only self-contained step and is independent of the measurements above; it forces an index rebuild, so it is worth doing only if its simplification is wanted.
|
||||
@@ -5,16 +5,11 @@ MKDOCS := $(VENV)/bin/mkdocs
|
||||
|
||||
CARGO_DIR := src
|
||||
|
||||
DOC_DIR := DevDocMD
|
||||
DOC_DIR := docmd
|
||||
DOC_FILE := mkdocs.yml
|
||||
DOC_SITE := DevDoc
|
||||
DOC_SITE := doc
|
||||
DOC_PORT := 8001
|
||||
|
||||
DOC_USER_DIR := UserDocMD
|
||||
DOC_USER_FILE := mkdocs-user.yml
|
||||
DOC_USER_SITE := doc
|
||||
DOC_USER_PORT := 8002
|
||||
|
||||
# ── virtualenv ────────────────────────────────────────────────────────────────
|
||||
|
||||
$(VENV)/bin/activate:
|
||||
@@ -42,18 +37,6 @@ obikmer:
|
||||
debug:
|
||||
cargo build --manifest-path $(CARGO_DIR)/Cargo.toml --bin obikmer
|
||||
|
||||
# One-shot converter of indexes built with ptr_hash 1.x (see tools/migrate-mphf).
|
||||
# Standalone crate, outside the src/ workspace, with its own Cargo.lock/target.
|
||||
MIGRATE_MPHF_DIR := tools/migrate-mphf
|
||||
|
||||
.PHONY: migrate-mphf
|
||||
migrate-mphf:
|
||||
cargo build --release --manifest-path $(MIGRATE_MPHF_DIR)/Cargo.toml
|
||||
|
||||
.PHONY: clean-migrate-mphf
|
||||
clean-migrate-mphf:
|
||||
rm -rf $(MIGRATE_MPHF_DIR)/target
|
||||
|
||||
.PHONY: profile-build
|
||||
profile-build:
|
||||
cargo build --release --manifest-path $(CARGO_DIR)/Cargo.toml \
|
||||
@@ -77,22 +60,8 @@ doc-serve: $(MKDOCS)
|
||||
clean-doc:
|
||||
rm -rf $(DOC_SITE)/
|
||||
|
||||
.PHONY: doc-user
|
||||
doc-user: $(MKDOCS)
|
||||
$(MKDOCS) build -f $(DOC_USER_FILE)
|
||||
|
||||
.PHONY: doc-user-serve
|
||||
doc-user-serve: $(MKDOCS)
|
||||
$(MKDOCS) serve -f $(DOC_USER_FILE) \
|
||||
--dev-addr=127.0.0.1:$(DOC_USER_PORT) \
|
||||
--livereload
|
||||
|
||||
.PHONY: clean-doc-user
|
||||
clean-doc-user:
|
||||
rm -rf $(DOC_USER_SITE)/
|
||||
|
||||
.PHONY: clean
|
||||
clean: clean-doc clean-doc-user
|
||||
clean: clean-doc
|
||||
rm -rf $(VENV)
|
||||
|
||||
# ── release ───────────────────────────────────────────────────────────────────
|
||||
@@ -117,13 +86,9 @@ bump-version:
|
||||
|
||||
.PHONY: release
|
||||
release: bump-version
|
||||
@jj auto-doc
|
||||
@jj auto-describe
|
||||
@jj git push --change @
|
||||
@new_version=$$(grep '^version = ' $(CARGO_TOML) | head -n 1 | sed 's/version = "\(.*\)"/\1/'); \
|
||||
git_hash=$$(jj log -r @ --no-graph -T 'commit_id'); \
|
||||
commits=$$(jj log -r 'latest(tags())..@' --no-graph -T 'description ++ "\n"' 2>/dev/null || \
|
||||
jj log --no-graph -T 'description ++ "\n"' --limit 30); \
|
||||
notes=$$(printf 'Write concise markdown release notes for obikmer (a Rust kmer genomics tool). Be technical and direct. Base them strictly on these commit messages:\n\n%s' "$$commits" | aichat 2>/dev/null); \
|
||||
tag_msg="$${notes:-Release v$$new_version}"; \
|
||||
git tag -a "v$$new_version" -m "$$tag_msg" "$$git_hash" && \
|
||||
git tag "v$$new_version" "$$git_hash" && \
|
||||
git push origin "v$$new_version"
|
||||
@@ -66,11 +66,9 @@ Non-ACGT characters act as hard breaks between k-mer segments in all formats.
|
||||
Annotates each sequence with per-genome k-mer match counts
|
||||
and optional per-position coverage vectors (--detail).
|
||||
Parallel over sequence chunks.
|
||||
phylo Compute pairwise evolutionary-distance proxies between all
|
||||
indexed genomes (Bray-Curtis, Jaccard, etc.), optionally
|
||||
a Newick NJ/UPGMA tree, and optionally a central-position
|
||||
SNP/Sankoff calibration with exports for external
|
||||
phylogenetic tools (TNT, PhyG, IQ-TREE).
|
||||
distance Compute a pairwise Bray-Curtis or Jaccard distance matrix
|
||||
between all indexed genomes.
|
||||
Optionally outputs a Newick NJ or UPGMA tree.
|
||||
annotate Add or update genome metadata (taxonomy, etc.) from a CSV
|
||||
file; or dump the current metadata as CSV.
|
||||
estimate Dry-run: resolve and print approximate-index parameters
|
||||
@@ -108,7 +106,7 @@ obikmer reindex --approx -z 5 --evidence-bits 8 index/
|
||||
obikmer query index/ reads.fq.gz > annotated.fa
|
||||
|
||||
# Pairwise distances
|
||||
obikmer phylo index/ > distances.tsv
|
||||
obikmer distance index/ > distances.tsv
|
||||
```
|
||||
|
||||
## Parameter constraints
|
||||
@@ -121,6 +119,5 @@ obikmer phylo index/ > distances.tsv
|
||||
|
||||
## Documentation
|
||||
|
||||
Extended architecture and implementation notes are in `DevDocMD/`. Build with
|
||||
`make doc` (requires Python + MkDocs Material). User-facing documentation is
|
||||
in `UserDocMD/`, built with `make doc-user`.
|
||||
Extended architecture and implementation notes are in `docmd/`. Build with
|
||||
`make doc` (requires Python + MkDocs Material).
|
||||
@@ -1,33 +0,0 @@
|
||||
# Architecture notes for advanced use
|
||||
|
||||
This page describes execution-level behavior relevant to sizing and running `obikmer` on large datasets or multi-socket machines. It complements the [index format](formats/index_layout.md) and [theory](theory/kmer_indexing/indexing_architecture.md) pages.
|
||||
|
||||
## Sequence invariant
|
||||
|
||||
Every input sequence is treated purely as a compact representation of a set of overlapping kmers:
|
||||
|
||||
- Only the `A`/`C`/`G`/`T` alphabet (case-insensitive) is recognized; a sequence is cut at any other character (including IUPAC ambiguity codes), so runs containing them are not represented in the index.
|
||||
- Sequences are internally processed in chunks of at most 256 nucleotides; a chunk shorter than k is dropped. This is invisible to the user beyond the ACGT-only, minimum-length-k constraints above.
|
||||
- Kmers are always handled in canonical form (see [DNA encoding](theory/kmer_indexing/encoding.md)), so the tool is strand-agnostic throughout: a kmer and its reverse complement are always the same entry.
|
||||
|
||||
## Index dimensioning
|
||||
|
||||
An index directory is organized as `KmerIndex → partitions → layers`, with a canonical kmer belonging to exactly one (partition, layer) pair. This is what makes set operations (merge, filter, distance) parallel and coordination-free across partitions.
|
||||
|
||||
- **Partition count** (`-p`/`--partitions`, rounded up to a power of 2) is the main dimensioning knob: more partitions means more independent parallel units and a smaller working set per partition, at the cost of more open files during construction.
|
||||
- **Layers** accumulate as an index grows through successive merges; per-partition query cost grows with the number of layers (worst case linear, expected constant since most kmer lookups resolve in the first layer they could plausibly be in).
|
||||
- Genome columns (count or presence data) are kept at a consistent width across every layer and partition after a merge, which is what allows whole-index aggregate distances (Jaccard, Bray-Curtis, Euclidean, Hellinger, …) to be computed as a two-pass cascade (local partial sums per partition, then a global combination) with no double counting.
|
||||
|
||||
## Parallel execution and NUMA awareness
|
||||
|
||||
Partition-level work (index construction, `merge`, `filter`, `convert`, `select`, `phylo`'s sibling-annex/Sankoff computations) is dispatched by a partition runner that adapts to the machine's memory topology, detected automatically at startup via hwloc:
|
||||
|
||||
- On a multi-socket / multi-NUMA-node machine, one thread pool is pinned per NUMA node, and each partition is processed entirely by threads pinned to one node — keeping the memory a partition touches local to that node's DRAM. This matters because touching kmer data across NUMA nodes without pinning can degrade throughput by an order of magnitude or more on large multi-socket machines.
|
||||
- On a single-socket machine, Apple Silicon, or if hwloc cannot report NUMA topology, all cores are treated as one node with no pinning and negligible overhead — this is the default behavior on macOS.
|
||||
- Within a node, the number of active worker threads ramps up progressively rather than being fixed up front: it starts conservatively and grows in steps, but only as long as measured CPU efficiency or disk I/O throughput keeps improving. If neither improves after a growth step, the runner stops adding workers — avoiding oversubscription on stages that are memory-bandwidth-bound rather than CPU- or I/O-bound. Ramp speed scales with the number of cores per node, so a single-node machine ramps just as fast as a large multi-node one.
|
||||
|
||||
No CLI flag controls this directly; it is fully automatic at runtime. NUMA-aware pinning can be compiled out (Cargo feature `numa`, on by default), in which case a plain global thread pool is used instead.
|
||||
|
||||
## Kmer filtering (`filter`)
|
||||
|
||||
[`filter`](usage/filter.md) evaluates predicates against the genome metadata matrix directly whenever every active filter can be expressed as a column-level test (e.g. "any outgroup column non-zero"), producing a per-slot keep/drop decision without touching kmer sequence data at all. If any active filter cannot be expressed this way, evaluation falls back to a per-kmer, row-level check. Either way, the result is always written as a single, freshly compacted layer (`unitigs.bin` and the MPHF are rebuilt from the surviving kmers), never as an additional layer on top of the source index.
|
||||
@@ -1,230 +0,0 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<style xmlns="http://purl.org/net/xbiblio/csl" class="in-text" version="1.0" demote-non-dropping-particle="sort-only" default-locale="en-US">
|
||||
<info>
|
||||
<title>Ecology Letters</title>
|
||||
<id>http://www.zotero.org/styles/ecology-letters</id>
|
||||
<link href="http://www.zotero.org/styles/ecology-letters" rel="self"/>
|
||||
<link href="http://www.zotero.org/styles/apa" rel="template"/>
|
||||
<link href="http://onlinelibrary.wiley.com/journal/10.1111/%28ISSN%291461-0248/homepage/ForAuthors.html" rel="documentation"/>
|
||||
<author>
|
||||
<name>David Kaplan</name>
|
||||
<email>david.kaplan@ird.fr</email>
|
||||
</author>
|
||||
<contributor>
|
||||
<name>Sebastian Karcher</name>
|
||||
</contributor>
|
||||
<category citation-format="author-date"/>
|
||||
<category field="biology"/>
|
||||
<issn>1461-023X</issn>
|
||||
<eissn>1461-0248</eissn>
|
||||
<updated>2023-10-11T10:45:32+00:00</updated>
|
||||
<rights license="http://creativecommons.org/licenses/by-sa/3.0/">This work is licensed under a Creative Commons Attribution-ShareAlike 3.0 License</rights>
|
||||
</info>
|
||||
<macro name="container">
|
||||
<choose>
|
||||
<if type="chapter paper-conference" match="any">
|
||||
<text term="in" text-case="capitalize-first" suffix=": "/>
|
||||
<text variable="container-title" font-style="italic"/>
|
||||
<text variable="collection-title" prefix=", "/>
|
||||
<names variable="editor translator" prefix=" (" delimiter=", " suffix=")">
|
||||
<label form="short" suffix=" "/>
|
||||
<name name-as-sort-order="all" and="symbol" sort-separator=", " initialize-with="." delimiter=", " delimiter-precedes-last="never"/>
|
||||
</names>
|
||||
</if>
|
||||
<else>
|
||||
<group delimiter=", ">
|
||||
<text variable="container-title" font-style="italic" form="short"/>
|
||||
<text variable="collection-title"/>
|
||||
</group>
|
||||
</else>
|
||||
</choose>
|
||||
</macro>
|
||||
<macro name="author">
|
||||
<names variable="author">
|
||||
<name name-as-sort-order="all" and="symbol" sort-separator=", " initialize-with="." delimiter=", " delimiter-precedes-last="never"/>
|
||||
<label form="short" prefix=" (" suffix=")" text-case="capitalize-first"/>
|
||||
<et-al font-style="italic"/>
|
||||
<substitute>
|
||||
<names variable="editor"/>
|
||||
<names variable="translator"/>
|
||||
<text macro="title"/>
|
||||
</substitute>
|
||||
</names>
|
||||
</macro>
|
||||
<macro name="author-short">
|
||||
<names variable="author">
|
||||
<name form="short" and="symbol" delimiter=", " initialize-with=". "/>
|
||||
<et-al font-style="italic"/>
|
||||
<substitute>
|
||||
<names variable="editor"/>
|
||||
<names variable="translator"/>
|
||||
<choose>
|
||||
<if type="bill book graphic legal_case legislation motion_picture report song" match="any">
|
||||
<text variable="title" form="short" font-style="italic"/>
|
||||
</if>
|
||||
<else>
|
||||
<text variable="title" form="short" quotes="true"/>
|
||||
</else>
|
||||
</choose>
|
||||
</substitute>
|
||||
</names>
|
||||
</macro>
|
||||
<macro name="access">
|
||||
<choose>
|
||||
<if type="webpage">
|
||||
<group>
|
||||
<text term="available at" text-case="capitalize-first" suffix=": "/>
|
||||
<text variable="URL" suffix="."/>
|
||||
</group>
|
||||
<text value="Last accessed" prefix=" " suffix=" "/>
|
||||
<date variable="accessed">
|
||||
<date-part name="day" suffix=" "/>
|
||||
<date-part name="month" suffix=" "/>
|
||||
<date-part name="year"/>
|
||||
</date>
|
||||
</if>
|
||||
</choose>
|
||||
</macro>
|
||||
<macro name="title">
|
||||
<choose>
|
||||
<if type="report" match="any">
|
||||
<text variable="title" font-style="italic"/>
|
||||
<group prefix=" (" suffix=")">
|
||||
<text variable="genre"/>
|
||||
<text variable="number" prefix=" No. "/>
|
||||
</group>
|
||||
</if>
|
||||
<else-if type="bill book graphic legal_case legislation motion_picture report song speech" match="any">
|
||||
<text variable="title" font-style="italic"/>
|
||||
</else-if>
|
||||
<else-if type="webpage">
|
||||
<text variable="title" font-style="italic"/>
|
||||
</else-if>
|
||||
<else>
|
||||
<text variable="title"/>
|
||||
</else>
|
||||
</choose>
|
||||
</macro>
|
||||
<macro name="publisher">
|
||||
<choose>
|
||||
<if type="report" match="any">
|
||||
<group delimiter=", ">
|
||||
<text variable="publisher"/>
|
||||
<text variable="publisher-place"/>
|
||||
</group>
|
||||
</if>
|
||||
<else>
|
||||
<text variable="genre" suffix=". "/>
|
||||
<group delimiter=", ">
|
||||
<text variable="publisher"/>
|
||||
<text variable="publisher-place"/>
|
||||
</group>
|
||||
</else>
|
||||
</choose>
|
||||
</macro>
|
||||
<macro name="event">
|
||||
<choose>
|
||||
<if variable="event">
|
||||
<text term="presented at" text-case="capitalize-first" suffix=" "/>
|
||||
<text variable="event"/>
|
||||
</if>
|
||||
</choose>
|
||||
</macro>
|
||||
<macro name="issued">
|
||||
<choose>
|
||||
<if variable="issued">
|
||||
<date variable="issued">
|
||||
<date-part name="year"/>
|
||||
</date>
|
||||
</if>
|
||||
<else-if variable="accessed">
|
||||
<choose>
|
||||
<if type="webpage">
|
||||
<date variable="accessed">
|
||||
<date-part name="year"/>
|
||||
</date>
|
||||
</if>
|
||||
<else>
|
||||
<text term="no date" form="short"/>
|
||||
</else>
|
||||
</choose>
|
||||
</else-if>
|
||||
<else>
|
||||
<text term="no date" form="short"/>
|
||||
</else>
|
||||
</choose>
|
||||
</macro>
|
||||
<macro name="edition">
|
||||
<choose>
|
||||
<if is-numeric="edition">
|
||||
<group delimiter=" ">
|
||||
<number variable="edition" form="ordinal"/>
|
||||
<text value="edn"/>
|
||||
</group>
|
||||
</if>
|
||||
<else>
|
||||
<text variable="edition" suffix="."/>
|
||||
</else>
|
||||
</choose>
|
||||
</macro>
|
||||
<macro name="locators">
|
||||
<choose>
|
||||
<if type="article-journal article-magazine article-newspaper" match="any">
|
||||
<group prefix=", " delimiter=", ">
|
||||
<group>
|
||||
<text variable="volume"/>
|
||||
</group>
|
||||
<text variable="page"/>
|
||||
</group>
|
||||
</if>
|
||||
<else-if type="bill book graphic legal_case legislation motion_picture report song thesis" match="any">
|
||||
<group delimiter=". " prefix=". ">
|
||||
<text macro="edition"/>
|
||||
<text macro="event"/>
|
||||
<text macro="publisher"/>
|
||||
</group>
|
||||
</else-if>
|
||||
<else-if type="chapter paper-conference" match="any">
|
||||
<group delimiter=", " prefix=". ">
|
||||
<text macro="event"/>
|
||||
<text macro="publisher"/>
|
||||
<group>
|
||||
<label variable="page" form="short" suffix=" "/>
|
||||
<text variable="page"/>
|
||||
</group>
|
||||
</group>
|
||||
</else-if>
|
||||
</choose>
|
||||
</macro>
|
||||
<citation et-al-min="3" et-al-use-first="1" disambiguate-add-year-suffix="true" collapse="year-suffix" year-suffix-delimiter=", ">
|
||||
<sort>
|
||||
<key macro="author"/>
|
||||
<key macro="issued"/>
|
||||
</sort>
|
||||
<layout prefix="(" suffix=")" delimiter="; ">
|
||||
<group delimiter=" ">
|
||||
<text macro="author-short"/>
|
||||
<text macro="issued"/>
|
||||
</group>
|
||||
</layout>
|
||||
</citation>
|
||||
<bibliography et-al-min="7" et-al-use-first="6" entry-spacing="0" hanging-indent="true">
|
||||
<sort>
|
||||
<key macro="author"/>
|
||||
<key macro="issued" sort="ascending"/>
|
||||
<key macro="title"/>
|
||||
</sort>
|
||||
<layout>
|
||||
<group suffix=".">
|
||||
<text macro="author" suffix="."/>
|
||||
<text macro="issued" prefix=" (" suffix="). "/>
|
||||
<group delimiter=". ">
|
||||
<text macro="title"/>
|
||||
<text macro="container"/>
|
||||
</group>
|
||||
<text macro="locators"/>
|
||||
<text macro="access" prefix=". "/>
|
||||
</group>
|
||||
</layout>
|
||||
</bibliography>
|
||||
</style>
|
||||
@@ -1,56 +0,0 @@
|
||||
# Index construction and on-disk layout
|
||||
|
||||
## Construction pipeline
|
||||
|
||||
Building an index ([`index`](../usage/index_command.md)) proceeds through a fixed sequence of phases, each operating independently per partition (see [Partitioning and indexing architecture](../theory/kmer_indexing/indexing_architecture.md)):
|
||||
|
||||
1. **Scatter.** A single streaming pass over the input. Each sequence fragment is cut at non-ACGT bases, passed through the low-complexity entropy filter (see [Low-complexity kmer filter](../theory/kmer_indexing/entropy_filter.md)), and any resulting segment shorter than k is dropped. Surviving segments are decomposed into super-kmers, canonicalized, and routed by `hash(minimizer) mod n_partitions` into one file per partition.
|
||||
2. **Dereplication.** Within each partition, identical super-kmer sequences are merged and their occurrence counts summed. This count is per super-kmer, not per kmer — a kmer's true abundance is the sum of the counts of every super-kmer containing it.
|
||||
3. **Exact counting.** Every kmer in every dereplicated super-kmer is enumerated and its exact total count computed. A per-genome kmer frequency spectrum is produced at this stage.
|
||||
4. **Quorum filtering.** Kmers outside the `--min-abundance`/`--max-abundance` range are dropped, and super-kmers are recompacted around the surviving kmer set.
|
||||
5. **Local assembly.** The surviving kmers of each partition are assembled into unitigs — maximal non-branching runs of a local de Bruijn graph — such that every kmer appears exactly once, at one (unitig, offset) location.
|
||||
6. **MPHF and evidence construction.** A minimal perfect hash function is built over the canonical kmers of each partition, together with the evidence structure needed to verify that a queried kmer was genuinely indexed (see below). Per-genome counts or presence bits are recorded alongside if requested.
|
||||
|
||||
Phases 1–5 are independent per partition and run in parallel; phase 6 finalizes each partition once its kmer set is fixed.
|
||||
|
||||
## Minimal perfect hash function (MPHF)
|
||||
|
||||
Each partition's surviving kmers are mapped to a dense range of integer slots by a minimal perfect hash function: no collisions, near-optimal space (a few bits per key), O(1) lookup. Because an MPHF maps *any* input to some slot — including kmers that were never indexed — a lookup alone cannot distinguish a genuinely indexed kmer from an arbitrary one; every lookup is followed by an evidence check.
|
||||
|
||||
## Evidence: exact vs. approximate
|
||||
|
||||
Two verification modes are available, selected at build time (`index --approx`) and convertible afterwards ([`convert`](../usage/convert.md)):
|
||||
|
||||
- **Exact** (default): the hashed slot stores a pointer back into the partition's unitig data. At query time the kmer is reconstructed from that location and compared directly to the query. Zero false positives, at the cost of one extra random read per lookup.
|
||||
- **Approximate** (`--approx`): the slot stores a short fingerprint (`--evidence-bits` bits) instead of a pointer; verification is a single fingerprint comparison. This trades a small, bounded false-positive rate ($1/2^b$ per kmer, reduced further to about $1/2^{b \cdot z}$ for a read requiring $z$ consecutive matching kmers via the `-z`/`--findere-z` parameter) for lower memory and disk usage, since no reconstruction index is needed. See [`estimate`](../usage/estimate.md) to explore this trade-off before building.
|
||||
|
||||
## On-disk layout
|
||||
|
||||
```
|
||||
<index_root>/
|
||||
index.meta global configuration (k, minimizer size, partition count,
|
||||
evidence mode, whether counts are stored) and genome list/metadata
|
||||
scatter.done / count.done / index.done build-progress sentinels
|
||||
spectrums/<label>.json per-genome kmer frequency histogram
|
||||
partitions/
|
||||
part_00000/ ... part_NNNNN/
|
||||
index/
|
||||
meta.json number of layers in this partition
|
||||
layer_0/
|
||||
unitigs.bin reconstructible kmer sequence data — always kept
|
||||
unitigs.bin.idx random-access index into unitigs.bin (exact evidence only)
|
||||
mphf.bin the minimal perfect hash function
|
||||
evidence.bin exact evidence (exact mode only)
|
||||
fingerprint.bin approximate evidence (approximate mode only)
|
||||
counts/ per-genome kmer counts (if counts were requested)
|
||||
presence/ per-genome presence/absence bits
|
||||
layer_1/, layer_2/, ... added by later merges, same internal structure
|
||||
```
|
||||
|
||||
`unitigs.bin` is the only file from which the indexed kmer content can be fully recovered; it is always retained. Every other file (MPHF, evidence, counts) is derived from it.
|
||||
|
||||
A **layer** corresponds to one increment of kmer content added to a partition — most commonly, one [`merge`](../usage/merge.md) operation that introduces kmers not already present in the index. Genomes already present in the index simply gain new columns in the existing layers' count/presence data; only genuinely new kmer content is assembled into a new layer. Because of this, merging cost scales with the novel kmer content being added, not with the accumulated size of the index. A query against an index with several layers checks each layer's MPHF in turn.
|
||||
|
||||
Sources merged together must share the same kmer size, minimizer size, partition count, and evidence mode (including matching approximate-mode parameters); mismatches are rejected rather than silently reconciled — [`convert`](../usage/convert.md) one of the sources first if needed.
|
||||
|
||||
`obikmer pack` consolidates a partition's per-column files (counts/presence) into a single file, reducing the number of file opens needed at query time.
|
||||
@@ -1,107 +0,0 @@
|
||||
# VCF pseudo-alignment
|
||||
|
||||
[`phylo --vcf`](../usage/phylo.md) writes the sites of a SNP pseudo-alignment as a VCF file, `<prefix>_alignment.vcf`. This page describes exactly what that file contains, because the k-mer families it is built from are not ordinary genomic variants and a few conventions follow from that.
|
||||
|
||||
## Why a VCF, and what a site is
|
||||
|
||||
The pseudo-alignment (see [Central-position SNP model](../theory/phylogeny/snp_based/central_snp_model.md)) has one column per *family*: the up to four k-mers that share all their bases except the central one. A genome that carries one of these k-mers shows a particular base at the centre, and the columns of the alignment are those central bases, genome by genome. The FASTA output keeps this as a matrix of letters. The VCF output lists the same columns as sites with alleles and genotypes, which is the form that variant-oriented tools (`bcftools`, population-genetics software, genotype viewers) expect.
|
||||
|
||||
Nothing here comes from a reference genome. The file has no real coordinates, and the sites are not located on a chromosome: the positions below are labels that make the file valid and reproducible, not map positions.
|
||||
|
||||
## Producing the file
|
||||
|
||||
```
|
||||
obikmer phylo --vcf --n-sites 5000000 -o run1 my_index
|
||||
```
|
||||
|
||||
`--vcf` implies `--pseudo-alignment`, so `--n-sites N|all` is required, together with an already-built sibling annex (see [`phylo`](../usage/phylo.md)). `--fasta` and `--vcf` can be given together to get both files from the same draw; with neither, only the FASTA is written. Every option that shapes the draw (`--free-loss`, `--no-ambiguity`, `--only-diploid`, `--quartet`, `--exclude-genome`, ...) applies to the VCF exactly as to the FASTA, since both describe the same sites.
|
||||
|
||||
## Layout
|
||||
|
||||
An illustrative excerpt (columns are tab-separated in the file):
|
||||
|
||||
```
|
||||
##fileformat=VCFv4.5
|
||||
##source=obikmer
|
||||
##contig=<ID=obikmer_0_0>
|
||||
##contig=<ID=obikmer_0_1>
|
||||
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype (diploid)">
|
||||
#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT idx0 idx1 idx2 idx3
|
||||
obikmer_0_0 440 CGATTNGTTAA A G . . . GT 0/0 1/1 0/0 0/0
|
||||
obikmer_0_0 1511 CACGANTGTAG A G,T . . . GT 0/0 0/0 1/1 2/2
|
||||
obikmer_0_1 9 TTACGNAGCTA C * . . . GT 0/0 0/0 1/1 0/0
|
||||
```
|
||||
|
||||
There is one sample column per genome that takes part in the draw, named after the genome label, in the index's own order. A genome removed with `--exclude-genome` or `--min-shared-family` has no column at all. `QUAL`, `FILTER` and `INFO` are always `.`: obikmer applies no variant filter and attaches no annotation, and leaving them empty keeps a file of millions of sites compact.
|
||||
|
||||
## Coordinates: `CHROM` and `POS`
|
||||
|
||||
An index is organised in partitions, and each partition holds one or more layers of k-mers (see [Index construction and on-disk layout](index_layout.md)). A family always belongs to exactly one layer of one partition, and the families of a layer are visited in a fixed order. The VCF uses that structure as coordinates:
|
||||
|
||||
- `CHROM` is `obikmer_<partition>_<layer>`, one virtual contig per layer of each partition in which at least one site was drawn;
|
||||
- `POS` is the rank of the family inside that layer, starting at 1.
|
||||
|
||||
For a given index and a given sibling annex, a family therefore always receives the same `CHROM` and `POS`, whatever the draw, the options or the number of sites requested. Two runs on the same index can be compared position by position. This stability is lost if the sibling annex is rebuilt, or if the index is rebuilt, merged or filtered, because the families are then enumerated again. Positions are never comparable between two different indexes; use `ID` for that. Even rebuilding an index from the very same genomes may enumerate the families in another order: on a small test index built twice from identical input, the family `CGATTNGTTAA` had `POS` 440 the first time and 371 the second, with the same `ID`.
|
||||
|
||||
The file is sorted by partition, then layer, then position, which is also the order of the `##contig` lines, so tools that require a sorted file accept it directly. Positions are sparse: only the drawn families are written.
|
||||
|
||||
## Identifier: `ID`
|
||||
|
||||
The `ID` column holds a label for the family that does not depend on the index. It is built from one of the family's k-mers by replacing its central base with `N` (for example `CGATTNGTTAA` for k = 11), after choosing, among the family's possible orientations, the one that sorts first. All the members of a family, whichever strand they were read on, give the same label.
|
||||
|
||||
This is what makes two files comparable even when they come from different indexes: the same family has the same `ID` in both, while its `CHROM` and `POS` will generally differ. It is also unique within a file, since two different families never share a label.
|
||||
|
||||
## Alleles: `REF` and `ALT`
|
||||
|
||||
The alleles are the bases carried at the centre of the family's k-mers, as in the FASTA. Two points are worth knowing.
|
||||
|
||||
First, the base is the central base of the member's *canonical* form (the smaller of the k-mer and its reverse complement). That is not necessarily the strand of the `ID` label, so the letters should not be matched to the `ID` sequence by position; they are labels for the allele, not a reading of that sequence.
|
||||
|
||||
Second, `REF` carries no meaning of reference: it is simply the most frequent allele among the genomes of the draw, each allele copy counted once, with the alphabetical order A < C < G < T deciding ties. `ALT` lists the remaining observed bases in the order A, C, G, T, separated by commas. There is no option to choose a reference genome.
|
||||
|
||||
### Absence: the `<ABS>` allele
|
||||
|
||||
A genome may carry none of the family's k-mers. Without `--free-loss` this absence is treated as a real state (the family was lost, or is absent from that genome), which the FASTA shows as `-`. In the VCF it becomes one more allele, the symbolic allele `<ABS>`, listed last in `ALT` and declared in the header by `##ALT=<ID=ABS,Description="Family absent from the genome">`. A genome without the family receives the genotype of `<ABS>`; for a site with `ALT` equal to `G,T,<ABS>` that is `3/3`. A site that differs between genomes only by presence versus absence has `ALT` equal to `<ABS>`.
|
||||
|
||||
The VCF specification gives the plain character `*` a meaning of its own (an allele missing because of an overlapping deletion described by another record), which is not the situation here, so it is deliberately not used. `<ABS>` is a free-form symbolic identifier, outside the reserved structural-variant names (`DEL`, `INS`, `DUP`, `INV`, `CNV`) and outside `<*>`, which belongs to reference blocks. Tools that only look at sequence alleles treat the site as having an alternative that is not a base.
|
||||
|
||||
With `--free-loss`, absence is declared missing data instead: `<ABS>` never appears, and the genomes concerned get a missing genotype.
|
||||
|
||||
## Genotypes
|
||||
|
||||
Every genome gets one `GT` value per site. The ploidy of these values is a convention chosen from the options, and the header says which (`Genotype (diploid)` or `Genotype (haploid)`).
|
||||
|
||||
| Cell in the alignment | Diploid (default, `--only-diploid`) | Haploid (`--no-ambiguity`) |
|
||||
|---|---|---|
|
||||
| one base `X` | `X/X`, for example `0/0` or `1/1` | `X`, for example `0` |
|
||||
| two bases (IUPAC `R Y S W K M`) | the heterozygote, for example `0/1` | does not occur |
|
||||
| three or four bases (`B D H V N`) | `./.` (not expressible) | does not occur |
|
||||
| absent family (`<ABS>`) | `<ABS>/<ABS>`, for example `3/3` | the index of `<ABS>` |
|
||||
| missing (`?`) | `./.` | `.` |
|
||||
|
||||
The two allele indices of a diploid genotype are always written in increasing order (`0/1`, never `1/0`), and the separator is `/` because no phasing information exists.
|
||||
|
||||
Why diploid by default? A genome can carry several forms of the same family, which the alignment records as an IUPAC ambiguity code. A diploid genotype can express up to two forms, and `--only-diploid` removes the genomes that would need more (three or four forms) before the sites are selected, so that no `./.` comes from them. `--no-ambiguity` goes further and treats every such genome as missing, which makes all genotypes haploid.
|
||||
|
||||
This is a convention, not a measurement. The file records which forms a genome carries, not how many copies of each: `A/A/G` and `A/G/G` both give the code `R` and the genotype `0/1`. Likewise, a genome with two forms may be heterozygous at one locus, or it may carry two similar regions of its genome (paralogues, repeats); the file cannot tell these apart. Do not read the genotypes as calls from sequencing reads.
|
||||
|
||||
## Which sites are written
|
||||
|
||||
Only the families that survive the draw and its masking appear. Within those:
|
||||
|
||||
- a site needs at least one genome with a base or (in diploid mode) a two-base code, because `REF` is a base; a site where every genome is missing, absent or has an ambiguity code too wide for the mode is left out of the VCF while remaining in the FASTA;
|
||||
- the log line at the end of the run gives the number of sites written out of the number drawn, so the difference is visible.
|
||||
|
||||
A site is selected because its genomes show at least two distinct states, and absence counts as a state. A site that differs between genomes only by presence versus absence is therefore a legitimate site, written with `ALT` equal to `<ABS>`.
|
||||
|
||||
## Using the file with other tools
|
||||
|
||||
- The contigs are virtual and have no declared length, and the file refers to no reference. Tools that need one (annotation, liftover) cannot use it; tools that only read sites and genotypes (`bcftools view`, `query`, `stats`, population-genetics software, PLINK-style converters) can.
|
||||
- If a tool rejects multiallelic sites or the `<ABS>` allele, `bcftools view -m2 -M2` keeps biallelic sites only, and `bcftools view -e 'ALT="<ABS>"'` removes the sites involving absence. The file as written was read by `bcftools` and indexed by `tabix` without complaint on a small test index, in both diploid and haploid modes. `bcftools stats` counts the sites with `<ABS>` among the "others" (symbolic alleles) rather than as SNPs.
|
||||
- If a tool expects haploid data, run `phylo` with `--no-ambiguity` rather than converting the genotypes afterwards.
|
||||
|
||||
## See also
|
||||
|
||||
- [`phylo`](../usage/phylo.md): all the options that shape the draw.
|
||||
- [Sampling design](../theory/phylogeny/snp_based/sampling_design.md): how the sites are drawn.
|
||||
- [Central-position SNP model](../theory/phylogeny/snp_based/central_snp_model.md): what a family is.
|
||||
@@ -1,68 +0,0 @@
|
||||
# obikmer
|
||||
|
||||
`obikmer` is a command-line tool for counting, indexing, querying and comparing DNA sequences represented as kmer sets. It targets individual genome datasets of tens of gigabases, with an emphasis on computational, memory, and disk efficiency.
|
||||
|
||||
All functionality is exposed through a single binary, `obikmer`, organized as subcommands.
|
||||
|
||||
## Core principles
|
||||
|
||||
- Kmers are of fixed, odd length $k$, chosen at index-construction time in the range $[11, 31]$ (see [Kmers](theory/kmer_indexing/kmers.md)).
|
||||
- Each kmer fits in a 64-bit word using a 2-bit-per-base encoding (see [DNA encoding](theory/kmer_indexing/encoding.md)).
|
||||
- Kmers are handled in **canonical form** ($\text{canonical}(kmer) = \min(kmer, \text{revcomp}(kmer))$), making counting strand-independent.
|
||||
- Sequences are decomposed into **super-kmers** before storage, anchored on a hash-selected **minimizer** (see [Minimizer selection](theory/kmer_indexing/minimizer_selection.md)), then routed to one of several **partitions** for parallel, memory-bounded processing (see [Partitioning and indexing architecture](theory/kmer_indexing/indexing_architecture.md)).
|
||||
- Low-complexity kmers can be filtered out at index-construction time using an entropy-based score (see [Low-complexity kmer filter](theory/kmer_indexing/entropy_filter.md)).
|
||||
|
||||
## Commands
|
||||
|
||||
| Command | Purpose |
|
||||
|---|---|
|
||||
| [`superkmer`](usage/superkmer.md) | Extract super-kmers from a sequence file and write them to stdout |
|
||||
| [`index`](usage/index_command.md) | Build a genome index |
|
||||
| [`merge`](usage/merge.md) | Merge multiple indexes into one |
|
||||
| [`filter`](usage/filter.md) | Retain only kmers matching ingroup/outgroup predicates |
|
||||
| [`select`](usage/select.md) | Project and/or aggregate genome columns of an index |
|
||||
| [`query`](usage/query.md) | Query an index with sequences and annotate matches |
|
||||
| [`dump`](usage/dump.md) | Dump indexed kmers as CSV |
|
||||
| [`annotate`](usage/annotate.md) | Add, update, or dump genome metadata |
|
||||
| [`phylo`](usage/phylo.md) | Compute pairwise genome distances, trees, and phylogenetic exports |
|
||||
| [`unitig`](usage/unitig.md) | Dump the unitigs of an index as FASTA |
|
||||
| [`estimate`](usage/estimate.md) | Estimate approximate-index parameters before indexing |
|
||||
| [`convert`](usage/convert.md) | Convert an index's evidence representation (exact/approximate/hybrid), in place |
|
||||
| [`utils`](usage/utils.md) | Miscellaneous index maintenance and inspection utilities |
|
||||
| [`pack`](usage/pack.md) | Pack per-column matrix files into a single-file format |
|
||||
|
||||
See [Genome predicates and taxonomy paths](usage/predicates.md) for the selection language shared by `filter`, `select`, `dump`, and `unitig`.
|
||||
|
||||
## Further reading
|
||||
|
||||
- [Index construction and on-disk layout](formats/index_layout.md)
|
||||
- [Architecture notes for advanced use](architecture.md) — parallel execution, NUMA awareness, index dimensioning
|
||||
|
||||
## Input formats
|
||||
|
||||
- `superkmer` and `index`: FASTA (`.fa`, `.fasta`), FASTQ (`.fq`, `.fastq`), GenBank flat file (`.gb`, `.gbk`, `.gbff`), all optionally gzip-compressed; directories are expanded recursively; streaming stdin via `-` or when no input path is given.
|
||||
- `query`: FASTA or FASTQ, optionally gzip-compressed; streaming stdin the same way.
|
||||
|
||||
## Parameter constraints
|
||||
|
||||
These constraints are checked at startup; an invalid value exits immediately with an error.
|
||||
|
||||
| Parameter | Constraint | Reason |
|
||||
|---|---|---|
|
||||
| $k$ (`--kmer-size`) | odd, $k \in [11, 31]$ | odd length guarantees the canonical form is always well defined; the range keeps a kmer within a 64-bit word while retaining specificity |
|
||||
| $m$ (`--minimizer-size`) | odd, $3 \le m \le k-1$ | same palindrome argument as $k$; must be strictly shorter than the kmer |
|
||||
| $z$ (`-z`, approximate evidence only) | $z \le k-1$ | the effective indexed kmer size is $k-z+1$ |
|
||||
|
||||
## Genome label constraints
|
||||
|
||||
Genome labels are arbitrary Unicode strings, with the following restrictions:
|
||||
|
||||
| Character | Forbidden | Reason |
|
||||
|---|---|---|
|
||||
| `/` | yes | filesystem path separator |
|
||||
| `=` | yes | separator used by `--new-label` |
|
||||
| `\0` | yes | null byte |
|
||||
| `\n`, `\r`, `\t` | yes | would break CSV output |
|
||||
| spaces | allowed | quote in the shell, e.g. `--new-label 'new label=old label'` |
|
||||
|
||||
Empty labels are rejected. A label derived automatically from the input file name (when `--label` is omitted) is not validated, since it is already filesystem-safe.
|
||||
@@ -1,84 +0,0 @@
|
||||
# Installation
|
||||
|
||||
## Prerequisites
|
||||
|
||||
### Rust toolchain
|
||||
|
||||
`obikmer` requires **Rust 1.85 or later** (edition 2024). Install or update via [rustup](https://rustup.rs):
|
||||
|
||||
```bash
|
||||
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
|
||||
rustup update stable
|
||||
```
|
||||
|
||||
### C build environment (required for hwloc)
|
||||
|
||||
`obikmer` embeds [hwloc](https://www.open-mpi.org/projects/hwloc/) (Hardware Locality) for NUMA-aware thread placement on multi-socket machines. hwloc is built from source at compile time, which requires a standard C build environment.
|
||||
|
||||
#### Linux (Debian/Ubuntu)
|
||||
|
||||
```bash
|
||||
apt install build-essential automake libtool autoconf pkg-config
|
||||
```
|
||||
|
||||
#### Linux (RHEL/Rocky/AlmaLinux)
|
||||
|
||||
```bash
|
||||
dnf install gcc make automake libtool autoconf pkgconfig
|
||||
```
|
||||
|
||||
#### HPC clusters
|
||||
|
||||
Most HPC clusters provide these tools via the module system:
|
||||
|
||||
```bash
|
||||
module load gcc automake libtool autoconf
|
||||
```
|
||||
|
||||
If in doubt, check that `autoreconf --version` and `libtool --version` return successfully.
|
||||
|
||||
#### macOS
|
||||
|
||||
```bash
|
||||
brew install automake libtool autoconf pkg-config
|
||||
```
|
||||
|
||||
## Building
|
||||
|
||||
```bash
|
||||
git clone <repository-url>
|
||||
cd obikmer/src
|
||||
cargo build --release
|
||||
```
|
||||
|
||||
The compiled binary is at `target/release/obikmer`.
|
||||
|
||||
### Building on HPC clusters (network filesystems)
|
||||
|
||||
HPC home directories are typically on a network filesystem (Lustre, NFS) optimized for large sequential reads, not for the many small file operations Cargo generates during compilation. Building directly on such a filesystem can be extremely slow.
|
||||
|
||||
Redirect the build directory to a local scratch disk:
|
||||
|
||||
```bash
|
||||
CARGO_TARGET_DIR=/scratch/$USER/cargo-target cargo build --release
|
||||
```
|
||||
|
||||
Adapt the path to the scratch space available on your cluster (`/var/tmp`, `/tmp`, `/scratch/local`, etc.). Once built, copy the binary to a permanent location:
|
||||
|
||||
```bash
|
||||
cp /scratch/$USER/cargo-target/release/obikmer ~/bin/
|
||||
```
|
||||
|
||||
## NUMA support
|
||||
|
||||
NUMA-aware thread placement is active automatically on multi-socket Linux machines, detected at runtime via hwloc. No build flag is required — it falls back gracefully to a single-pool strategy on:
|
||||
|
||||
- macOS (Apple Silicon, unified memory)
|
||||
- single-socket Linux machines
|
||||
- any system where hwloc reports only one NUMA node
|
||||
|
||||
## Verifying the installation
|
||||
|
||||
```bash
|
||||
obikmer --help
|
||||
```
|
||||
@@ -1,223 +0,0 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<style xmlns="http://purl.org/net/xbiblio/csl" class="in-text" version="1.0" default-locale="en-US" demote-non-dropping-particle="sort-only">
|
||||
<info>
|
||||
<title>Molecular Ecology Modified</title>
|
||||
<id>http://www.zotero.org/styles/molecular-ecology</id>
|
||||
<link href="http://www.zotero.org/styles/molecular-ecology" rel="self"/>
|
||||
<link href="http://www.zotero.org/styles/ecology-letters" rel="template"/>
|
||||
<link href="http://onlinelibrary.wiley.com/journal/10.1111/%28ISSN%291365-294X/homepage/ForAuthors.html" rel="documentation"/>
|
||||
<author>
|
||||
<name>Xiaodong Dang</name>
|
||||
<email>dangxdong@gmail.com</email>
|
||||
</author>
|
||||
<contributor>
|
||||
<name>Johan Asplund</name>
|
||||
<email>asplundj@gmail.com</email>
|
||||
</contributor>
|
||||
<contributor>
|
||||
<name>David Kaplan</name>
|
||||
<email>david.kaplan@ird.fr</email>
|
||||
</contributor>
|
||||
<category citation-format="author-date"/>
|
||||
<category field="biology"/>
|
||||
<issn>0962-1083</issn>
|
||||
<eissn>1365-294X</eissn>
|
||||
<updated>2012-09-09T22:24:43+00:00</updated>
|
||||
<rights license="http://creativecommons.org/licenses/by-sa/3.0/">This work is licensed under a Creative Commons Attribution-ShareAlike 3.0 License</rights>
|
||||
</info>
|
||||
<macro name="editor-translator">
|
||||
<names variable="editor translator" prefix=" (" delimiter=", " suffix=")">
|
||||
<name initialize-with="" delimiter=", "/>
|
||||
<label form="short" prefix=", " text-case="capitalize-first" suffix="," strip-periods="true"/>
|
||||
</names>
|
||||
</macro>
|
||||
<macro name="author">
|
||||
<names variable="author">
|
||||
<name name-as-sort-order="all" sort-separator=" " initialize-with="" delimiter=", " delimiter-precedes-et-al="never"/>
|
||||
<label form="short" prefix=" (" suffix=")" text-case="capitalize-first"/>
|
||||
<et-al font-style="italic"/>
|
||||
<substitute>
|
||||
<names variable="editor"/>
|
||||
<names variable="translator"/>
|
||||
<text macro="title"/>
|
||||
</substitute>
|
||||
</names>
|
||||
</macro>
|
||||
<macro name="author-short">
|
||||
<names variable="author">
|
||||
<name form="short" and="symbol" delimiter=", " initialize-with=". "/>
|
||||
<et-al font-style="italic"/>
|
||||
<substitute>
|
||||
<names variable="editor"/>
|
||||
<names variable="translator"/>
|
||||
<choose>
|
||||
<if type="bill book graphic legal_case legislation motion_picture report song" match="any">
|
||||
<text variable="title" form="short" font-style="italic"/>
|
||||
</if>
|
||||
<else>
|
||||
<text variable="title" form="short" quotes="true"/>
|
||||
</else>
|
||||
</choose>
|
||||
</substitute>
|
||||
</names>
|
||||
</macro>
|
||||
<macro name="title">
|
||||
<choose>
|
||||
<if type="bill book graphic legal_case legislation motion_picture report song" match="any">
|
||||
<text variable="title" font-style="italic"/>
|
||||
</if>
|
||||
<else>
|
||||
<text variable="title"/>
|
||||
</else>
|
||||
</choose>
|
||||
</macro>
|
||||
<macro name="access-detail">
|
||||
<group delimiter=" ">
|
||||
<text variable="URL" prefix="Available at: "/>
|
||||
<group prefix="(" suffix=")" delimiter=" ">
|
||||
<text term="accessed"/>
|
||||
<date variable="accessed">
|
||||
<date-part name="day" suffix=" "/>
|
||||
<date-part name="month" form="long" suffix=" "/>
|
||||
<date-part name="year"/>
|
||||
</date>
|
||||
</group>
|
||||
</group>
|
||||
</macro>
|
||||
<macro name="access">
|
||||
<choose>
|
||||
<if type="webpage post post-weblog entry entry-encyclopedia dataset software report manuscript document" match="any">
|
||||
<text macro="access-detail"/>
|
||||
</if>
|
||||
<else-if variable="accessed">
|
||||
<text macro="access-detail"/>
|
||||
</else-if>
|
||||
</choose>
|
||||
</macro>
|
||||
<macro name="access-fallback">
|
||||
<group delimiter=" ">
|
||||
<choose>
|
||||
<if variable="URL">
|
||||
<text macro="access-detail" suffix="."/>
|
||||
</if>
|
||||
<else-if variable="publisher">
|
||||
<text variable="publisher" prefix="Available at: " suffix="."/>
|
||||
</else-if>
|
||||
</choose>
|
||||
<text variable="note" suffix="."/>
|
||||
</group>
|
||||
</macro>
|
||||
<macro name="issued-year">
|
||||
<choose>
|
||||
<if variable="issued">
|
||||
<date variable="issued">
|
||||
<date-part name="year"/>
|
||||
</date>
|
||||
</if>
|
||||
<else>
|
||||
<text term="no date" form="short"/>
|
||||
</else>
|
||||
</choose>
|
||||
</macro>
|
||||
<macro name="publisher">
|
||||
<group delimiter=", ">
|
||||
<text variable="publisher"/>
|
||||
<text variable="publisher-place"/>
|
||||
</group>
|
||||
</macro>
|
||||
<citation collapse="year-suffix" et-al-min="3" et-al-use-first="1" disambiguate-add-year-suffix="true">
|
||||
<sort>
|
||||
<key variable="issued"/>
|
||||
<key variable="author"/>
|
||||
</sort>
|
||||
<layout prefix="(" suffix=")" delimiter="; ">
|
||||
<group delimiter=" ">
|
||||
<text macro="author-short"/>
|
||||
<text macro="issued-year"/>
|
||||
<group>
|
||||
<label variable="locator" form="short"/>
|
||||
<text variable="locator" prefix=" "/>
|
||||
</group>
|
||||
</group>
|
||||
</layout>
|
||||
</citation>
|
||||
<bibliography hanging-indent="true" et-al-min="6" et-al-use-first="1000" entry-spacing="0.5" line-spacing="1">
|
||||
<sort>
|
||||
<key macro="author-short"/>
|
||||
<key variable="issued"/>
|
||||
</sort>
|
||||
<layout>
|
||||
<text macro="author" suffix=" "/>
|
||||
<text macro="issued-year" prefix="(" suffix=")"/>
|
||||
<choose>
|
||||
<if type="bill book graphic legal_case legislation motion_picture report song" match="any">
|
||||
<group suffix=".">
|
||||
<text macro="title" prefix=" "/>
|
||||
<text macro="editor-translator" prefix=" "/>
|
||||
</group>
|
||||
<text prefix=" " suffix="." macro="publisher"/>
|
||||
<text macro="access" prefix=" " suffix="."/>
|
||||
</if>
|
||||
<else-if type="thesis" match="any">
|
||||
<group suffix=".">
|
||||
<text macro="title" prefix=" " suffix="."/>
|
||||
<text variable="genre" prefix=" " suffix=" Thesis."/>
|
||||
<text prefix=" " macro="publisher"/>
|
||||
</group>
|
||||
<text macro="access" prefix=" " suffix="."/>
|
||||
</else-if>
|
||||
<else-if type="chapter paper-conference" match="any">
|
||||
<text macro="title" prefix=" "/>
|
||||
<group prefix=".">
|
||||
<group prefix=" In:">
|
||||
<text variable="container-title" font-style="italic" prefix=" "/>
|
||||
<text variable="collection-title" prefix=" " suffix="."/>
|
||||
<names variable="editor translator" prefix=" (" delimiter=", " suffix=")">
|
||||
<label form="short" suffix=" " strip-periods="true"/>
|
||||
<name name-as-sort-order="all" sort-separator=" " initialize-with="" delimiter=", " delimiter-precedes-last="never"/>
|
||||
</names>
|
||||
<group suffix=".">
|
||||
<label variable="page" form="short" prefix=", " suffix=" "/>
|
||||
<text variable="page"/>
|
||||
<text macro="publisher" prefix=". " suffix="."/>
|
||||
</group>
|
||||
</group>
|
||||
</group>
|
||||
</else-if>
|
||||
<else-if type="webpage post post-weblog entry entry-encyclopedia dataset software" match="any">
|
||||
<group suffix=".">
|
||||
<text macro="title" prefix=" "/>
|
||||
<text macro="editor-translator" prefix=" "/>
|
||||
</group>
|
||||
<group prefix=" " suffix=".">
|
||||
<text variable="container-title" text-case="title" font-style="italic"/>
|
||||
<text variable="genre" prefix=", "/>
|
||||
<text macro="publisher" prefix=". "/>
|
||||
</group>
|
||||
<text macro="access" prefix=" " suffix="."/>
|
||||
</else-if>
|
||||
<else>
|
||||
<group suffix=".">
|
||||
<text macro="title" prefix=" "/>
|
||||
<text macro="editor-translator" prefix=" "/>
|
||||
</group>
|
||||
<group prefix=" " suffix=".">
|
||||
<text variable="container-title" text-case="title" font-style="italic"/>
|
||||
<group prefix=", ">
|
||||
<text variable="volume" font-weight="bold"/>
|
||||
</group>
|
||||
<text variable="page" prefix=", "/>
|
||||
</group>
|
||||
<choose>
|
||||
<if variable="container-title">
|
||||
<text macro="access" prefix=" " suffix="."/>
|
||||
</if>
|
||||
<else>
|
||||
<text macro="access-fallback" prefix=" "/>
|
||||
</else>
|
||||
</choose>
|
||||
</else>
|
||||
</choose>
|
||||
</layout>
|
||||
</bibliography>
|
||||
</style>
|
||||
@@ -1,859 +0,0 @@
|
||||
%% This BibTeX bibliography file was created using BibDesk.
|
||||
%% https://bibdesk.sourceforge.io/
|
||||
|
||||
%% Created for Eric Coissac at 2026-04-18 08:19:36 +0200
|
||||
|
||||
|
||||
%% Saved with string encoding Unicode (UTF-8)
|
||||
|
||||
|
||||
|
||||
@article{Zheng2020-ji,
|
||||
abstract = {MOTIVATION: Minimizers are methods to sample k-mers from a
|
||||
string, with the guarantee that similar set of k-mers will be
|
||||
chosen on similar strings. It is parameterized by the k-mer
|
||||
length k, a window length w and an order on the k-mers.
|
||||
Minimizers are used in a large number of softwares and pipelines
|
||||
to improve computation efficiency and decrease memory usage.
|
||||
Despite the method's popularity, many theoretical questions
|
||||
regarding its performance remain open. The core metric for
|
||||
measuring performance of a minimizer is the density, which
|
||||
measures the sparsity of sampled k-mers. The theoretical optimal
|
||||
density for a minimizer is 1/w, provably not achievable in
|
||||
general. For given k and w, little is known about asymptotically
|
||||
optimal minimizers, that is minimizers with density O(1/w).
|
||||
RESULTS: We derive a necessary and sufficient condition for
|
||||
existence of asymptotically optimal minimizers. We also provide a
|
||||
randomized algorithm, called the Miniception, to design
|
||||
minimizers with the best theoretical guarantee to date on density
|
||||
in practical scenarios. Constructing and using the Miniception is
|
||||
as easy as constructing and using a random minimizer, which
|
||||
allows the design of efficient minimizers that scale to the
|
||||
values of k and w used in current bioinformatics software
|
||||
programs. AVAILABILITY AND IMPLEMENTATION: Reference
|
||||
implementation of the Miniception and the codes for analysis can
|
||||
be found at https://github.com/kingsford-group/miniception.
|
||||
SUPPLEMENTARY INFORMATION: Supplementary data are available at
|
||||
Bioinformatics online.},
|
||||
author = {Zheng, Hongyu and Kingsford, Carl and Mar{\c c}ais, Guillaume},
|
||||
doi = {10.1093/bioinformatics/btaa472},
|
||||
issn = {1367-4803,1367-4811},
|
||||
journal = {Bioinformatics (Oxford, England)},
|
||||
language = {en},
|
||||
month = jul,
|
||||
number = {Suppl_1},
|
||||
pages = {i119--i127},
|
||||
pmc = {PMC8248892},
|
||||
pmid = 32657376,
|
||||
publisher = {Oxford University Press (OUP)},
|
||||
title = {Improved design and analysis of practical minimizers},
|
||||
url = {http://dx.doi.org/10.1093/bioinformatics/btaa472},
|
||||
volume = 36,
|
||||
year = 2020,
|
||||
bdsk-url-1 = {http://dx.doi.org/10.1093/bioinformatics/btaa472}}
|
||||
|
||||
@article{Zheng2021-cc,
|
||||
abstract = {MOTIVATION: Minimizers are efficient methods to sample k-mers
|
||||
from genomic sequences that unconditionally preserve sufficiently
|
||||
long matches between sequences. Well-established methods to
|
||||
construct efficient minimizers focus on sampling fewer k-mers on
|
||||
a random sequence and use universal hitting sets (sets of k-mers
|
||||
that appear frequently enough) to upper bound the sketch size. In
|
||||
contrast, the problem of sequence-specific minimizers, which is
|
||||
to construct efficient minimizers to sample fewer k-mers on a
|
||||
specific sequence such as the reference genome, is less studied.
|
||||
Currently, the theoretical understanding of this problem is
|
||||
lacking, and existing methods do not specialize well to sketch
|
||||
specific sequences. RESULTS: We propose the concept of polar
|
||||
sets, complementary to the existing idea of universal hitting
|
||||
sets. Polar sets are k-mer sets that are spread out enough on the
|
||||
reference, and provably specialize well to specific sequences.
|
||||
Link energy measures how well spread out a polar set is, and with
|
||||
it, the sketch size can be bounded from above and below in a
|
||||
theoretically sound way. This allows for direct optimization of
|
||||
sketch size. We propose efficient heuristics to construct polar
|
||||
sets, and via experiments on the human reference genome, show
|
||||
their practical superiority in designing efficient
|
||||
sequence-specific minimizers. AVAILABILITY AND IMPLEMENTATION: A
|
||||
reference implementation and code for analyses under an
|
||||
open-source license are at
|
||||
https://github.com/kingsford-group/polarset. SUPPLEMENTARY
|
||||
INFORMATION: Supplementary data are available at Bioinformatics
|
||||
online.},
|
||||
author = {Zheng, Hongyu and Kingsford, Carl and Mar{\c c}ais, Guillaume},
|
||||
doi = {10.1093/bioinformatics/btab313},
|
||||
issn = {1367-4803,1367-4811},
|
||||
journal = {Bioinformatics (Oxford, England)},
|
||||
language = {en},
|
||||
month = jul,
|
||||
number = {Suppl\_1},
|
||||
pages = {i187--i195},
|
||||
pmc = {PMC8686682},
|
||||
pmid = 34252928,
|
||||
publisher = {Oxford University Press (OUP)},
|
||||
title = {Sequence-specific minimizers via polar sets},
|
||||
url = {http://dx.doi.org/10.1093/bioinformatics/btab313},
|
||||
volume = 37,
|
||||
year = 2021,
|
||||
bdsk-url-1 = {http://dx.doi.org/10.1093/bioinformatics/btab313}}
|
||||
|
||||
@article{Pan2024-hb,
|
||||
abstract = {MOTIVATION: The minimizer concept is a data structure for
|
||||
sequence sketching. The standard canonical minimizer selects a
|
||||
subset of k-mers from the given DNA sequence by comparing the
|
||||
forward and reverse k-mers in a window simultaneously according
|
||||
to a predefined selection scheme. It is widely employed by
|
||||
sequence analysis such as read mapping and assembly. k-mer
|
||||
density, k-mer repetitiveness (e.g. k-mer bias), and
|
||||
computational efficiency are three critical measurements for
|
||||
minimizer selection schemes. However, there exist trade-offs
|
||||
between kinds of minimizer variants. Generic, effective, and
|
||||
efficient are always the requirements for high-performance
|
||||
minimizer algorithms. RESULTS: We propose a simple minimizer
|
||||
operator as a refinement of the standard canonical minimizer. It
|
||||
takes only a few operations to compute. However, it can improve
|
||||
the k-mer repetitiveness, especially for the lexicographic order.
|
||||
It applies to other selection schemes of total orders (e.g.
|
||||
random orders). Moreover, it is computationally efficient and the
|
||||
density is close to that of the standard minimizer. The refined
|
||||
minimizer may benefit high-performance applications like binning
|
||||
and read mapping. AVAILABILITY AND IMPLEMENTATION: The source
|
||||
code of the benchmark in this work is available at the github
|
||||
repository https://github.com/xp3i4/mini\_benchmark.},
|
||||
author = {Pan, Chenxu and Reinert, Knut},
|
||||
doi = {10.1093/bioinformatics/btae045},
|
||||
issn = {1367-4803,1367-4811},
|
||||
journal = {Bioinformatics (Oxford, England)},
|
||||
language = {en},
|
||||
month = feb,
|
||||
number = 2,
|
||||
pmc = {PMC10868324},
|
||||
pmid = 38269626,
|
||||
publisher = {Oxford University Press (OUP)},
|
||||
title = {A simple refined DNA minimizer operator enables 2-fold faster computation},
|
||||
url = {http://dx.doi.org/10.1093/bioinformatics/btae045},
|
||||
volume = 40,
|
||||
year = 2024,
|
||||
bdsk-url-1 = {http://dx.doi.org/10.1093/bioinformatics/btae045}}
|
||||
|
||||
@article{Kille2023-px,
|
||||
abstract = {MOTIVATION: The Jaccard similarity on k-mer sets has shown to be
|
||||
a convenient proxy for sequence identity. By avoiding expensive
|
||||
base-level alignments and comparing reduced sequence
|
||||
representations, tools such as MashMap can scale to massive
|
||||
numbers of pairwise comparisons while still providing useful
|
||||
similarity estimates. However, due to their reliance on minimizer
|
||||
winnowing, previous versions of MashMap were shown to be biased
|
||||
and inconsistent estimators of Jaccard similarity. This directly
|
||||
impacts downstream tools that rely on the accuracy of these
|
||||
estimates. RESULTS: To address this, we propose the minmer
|
||||
winnowing scheme, which generalizes the minimizer scheme by use
|
||||
of a rolling minhash with multiple sampled k-mers per window. We
|
||||
show both theoretically and empirically that minmers yield an
|
||||
unbiased estimator of local Jaccard similarity, and we implement
|
||||
this scheme in an updated version of MashMap. The minmer-based
|
||||
implementation is over 10 times faster than the minimizer-based
|
||||
version under the default ANI threshold, making it well-suited
|
||||
for large-scale comparative genomics applications. AVAILABILITY
|
||||
AND IMPLEMENTATION: MashMap3 is available at
|
||||
https://github.com/marbl/MashMap.},
|
||||
author = {Kille, Bryce and Garrison, Erik and Treangen, Todd J and Phillippy, Adam M},
|
||||
doi = {10.1093/bioinformatics/btad512},
|
||||
issn = {1367-4803,1367-4811},
|
||||
journal = {Bioinformatics (Oxford, England)},
|
||||
language = {en},
|
||||
month = sep,
|
||||
number = 9,
|
||||
pmc = {PMC10505501},
|
||||
pmid = 37603771,
|
||||
publisher = {Oxford University Press (OUP)},
|
||||
title = {Minmers are a generalization of minimizers that enable unbiased local Jaccard estimation},
|
||||
url = {http://dx.doi.org/10.1093/bioinformatics/btad512},
|
||||
volume = 39,
|
||||
year = 2023,
|
||||
bdsk-url-1 = {http://dx.doi.org/10.1093/bioinformatics/btad512}}
|
||||
|
||||
@incollection{Golan2025-xf,
|
||||
address = {Cham},
|
||||
author = {Golan, Shay and Shur, Arseny M},
|
||||
booktitle = {Lecture Notes in Computer Science},
|
||||
doi = {10.1007/978-3-031-82670-2\_25},
|
||||
isbn = {9783031826696,9783031826702},
|
||||
issn = {0302-9743,1611-3349},
|
||||
language = {en},
|
||||
pages = {347--360},
|
||||
publisher = {Springer Nature Switzerland},
|
||||
series = {Lecture Notes in Computer Science},
|
||||
title = {Expected density of random minimizers},
|
||||
url = {http://dx.doi.org/10.1007/978-3-031-82670-2_25},
|
||||
year = 2025,
|
||||
bdsk-url-1 = {http://dx.doi.org/10.1007/978-3-031-82670-2_25},
|
||||
bdsk-url-2 = {http://dx.doi.org/10.1007/978-3-031-82670-2%5C_25}}
|
||||
|
||||
@article{Mohamadi2017-ok,
|
||||
abstract = {Motivation: Many bioinformatics algorithms are designed for the
|
||||
analysis of sequences of some uniform length, conventionally
|
||||
referred to as k -mers. These include de Bruijn graph assembly
|
||||
methods and sequence alignment tools. An efficient algorithm to
|
||||
enumerate the number of unique k -mers, or even better, to build
|
||||
a histogram of k -mer frequencies would be desirable for these
|
||||
tools and their downstream analysis pipelines. Among other
|
||||
applications, estimated frequencies can be used to predict genome
|
||||
sizes, measure sequencing error rates, and tune runtime
|
||||
parameters for analysis tools. However, calculating a k -mer
|
||||
histogram from large volumes of sequencing data is a challenging
|
||||
task. Results: Here, we present ntCard, a streaming algorithm for
|
||||
estimating the frequencies of k -mers in genomics datasets. At
|
||||
its core, ntCard uses the ntHash algorithm to efficiently compute
|
||||
hash values for streamed sequences. It then samples the
|
||||
calculated hash values to build a reduced representation
|
||||
multiplicity table describing the sample distribution. Finally,
|
||||
it uses a statistical model to reconstruct the population
|
||||
distribution from the sample distribution. We have compared the
|
||||
performance of ntCard and other cardinality estimation
|
||||
algorithms. We used three datasets of 480 GB, 500 GB and 2.4 TB
|
||||
in size, where the first two representing whole genome shotgun
|
||||
sequencing experiments on the human genome and the last one on
|
||||
the white spruce genome. Results show ntCard estimates k -mer
|
||||
coverage frequencies >15× faster than the state-of-the-art
|
||||
algorithms, using similar amount of memory, and with higher
|
||||
accuracy rates. Thus, our benchmarks demonstrate ntCard as a
|
||||
potentially enabling technology for large-scale genomics
|
||||
applications. Availability and Implementation: ntCard is written
|
||||
in C ++ and is released under the GPL license. It is freely
|
||||
available at https://github.com/bcgsc/ntCard. Contact:
|
||||
hmohamadi@bcgsc.ca or ibirol@bcgsc.ca. Supplementary information:
|
||||
Supplementary data are available at Bioinformatics online.},
|
||||
author = {Mohamadi, Hamid and Khan, Hamza and Birol, Inanc},
|
||||
date-modified = {2026-04-18 08:19:36 +0200},
|
||||
doi = {10.1093/bioinformatics/btw832},
|
||||
issn = {1367-4803,1367-4811},
|
||||
journal = {Bioinformatics (Oxford, England)},
|
||||
language = {en},
|
||||
month = may,
|
||||
number = 9,
|
||||
pages = {1324--1330},
|
||||
pmc = {PMC5408799},
|
||||
pmid = 28453674,
|
||||
publisher = {Oxford University Press (OUP)},
|
||||
title = {ntCard: a streaming algorithm for cardinality estimation in genomics data},
|
||||
url = {http://dx.doi.org/10.1093/bioinformatics/btw832},
|
||||
volume = 33,
|
||||
year = 2017,
|
||||
bdsk-url-1 = {http://dx.doi.org/10.1093/bioinformatics/btw832}}
|
||||
|
||||
@misc{Mash-distances-doc,
|
||||
author = {{Marbl Lab}},
|
||||
howpublished = {Mash documentation},
|
||||
title = {Mash Distance},
|
||||
url = {https://mash.readthedocs.io/en/latest/distances.html},
|
||||
urldate = {2026-07-09},
|
||||
year = 2026}
|
||||
|
||||
@article{Fan2015-mash-formula,
|
||||
author = {Fan, Huan and Ives, Anthony R and Surget-Groba, Yann and Cannon, Charles H},
|
||||
doi = {10.1186/s12864-015-1647-5},
|
||||
journal = {BMC Genomics},
|
||||
number = 1,
|
||||
title = {An assembly and alignment-free method of phylogeny reconstruction from next-generation sequencing data},
|
||||
url = {https://doi.org/10.1186/s12864-015-1647-5},
|
||||
volume = 16,
|
||||
year = 2015}
|
||||
|
||||
@article{Roberts2004-rz,
|
||||
title = "{Reducing storage requirements for biological sequence
|
||||
comparison}",
|
||||
author = "Roberts, Michael and Hayes, Wayne and Hunt, Brian R and Mount,
|
||||
Stephen M and Yorke, James A",
|
||||
journal = "Bioinformatics (Oxford, England)",
|
||||
publisher = "Oxford University Press (OUP)",
|
||||
volume = 20,
|
||||
number = 18,
|
||||
pages = "3363--3369",
|
||||
abstract = "MOTIVATION: Comparison of nucleic acid and protein sequences is a
|
||||
fundamental tool of modern bioinformatics. A dominant method of
|
||||
such string matching is the 'seed-and-extend' approach, in which
|
||||
occurrences of short subsequences called 'seeds' are used to
|
||||
search for potentially longer matches in a large database of
|
||||
sequences. Each such potential match is then checked to see if it
|
||||
extends beyond the seed. To be effective, the seed-and-extend
|
||||
approach needs to catalogue seeds from virtually every substring
|
||||
in the database of search strings. Projects such as mammalian
|
||||
genome assemblies and large-scale protein matching, however, have
|
||||
such large sequence databases that the resulting list of seeds
|
||||
cannot be stored in RAM on a single computer. This significantly
|
||||
slows the matching process. RESULTS: We present a simple and
|
||||
elegant method in which only a small fraction of seeds, called
|
||||
'minimizers', needs to be stored. Using minimizers can speed up
|
||||
string-matching computations by a large factor while missing only
|
||||
a small fraction of the matches found using all seeds.",
|
||||
month = dec,
|
||||
year = 2004,
|
||||
url = "https://dx.doi.org/10.1093/bioinformatics/bth408",
|
||||
doi = "10.1093/bioinformatics/bth408",
|
||||
pmid = 15256412,
|
||||
issn = "1367-4803,1367-4811",
|
||||
language = "en"
|
||||
}
|
||||
|
||||
@INPROCEEDINGS{Steele2014-cz,
|
||||
title = "{Fast splittable pseudorandom number generators}",
|
||||
author = "Steele, Jr, Guy L and Lea, Doug and Flood, Christine H",
|
||||
booktitle = "{Proceedings of the 2014 ACM International Conference on Object
|
||||
Oriented Programming Systems Languages \& Applications}",
|
||||
publisher = "ACM",
|
||||
address = "New York, NY, USA",
|
||||
abstract = "We describe a new algorithm SplitMix for an object-oriented and
|
||||
splittable pseudorandom number generator (PRNG) that is quite
|
||||
fast: 9 64-bit arithmetic/logical operations per 64 bits
|
||||
generated. A conventional linear PRNG object provides a generate
|
||||
method that returns one pseudorandom value and updates the state
|
||||
of the PRNG, but a splittable PRNG object also has a second
|
||||
operation, split, that replaces the original PRNG object with two
|
||||
(seemingly) independent PRNG objects, by creating and returning a
|
||||
new such object and updating the state of the original object.
|
||||
Splittable PRNG objects make it easy to organize the use of
|
||||
pseudorandom numbers in multithreaded programs structured using
|
||||
fork-join parallelism. No locking or synchronization is required
|
||||
(other than the usual memory fence immediately after object
|
||||
creation). Because the generate method has no loops or
|
||||
conditionals, it is suitable for SIMD or GPU implementation. We
|
||||
derive SplitMix from the DotMix algorithm of Leiserson, Schardl,
|
||||
and Sukha by making a series of program transformations and
|
||||
engineering improvements. The end result is an object-oriented
|
||||
version of the purely functional API used in the Haskell library
|
||||
for over a decade, but SplitMix is faster and produces
|
||||
pseudorandom sequences of higher quality; it is also far superior
|
||||
in quality and speed to java.util.Random, and has been included
|
||||
in Java JDK8 as the class java.util.SplittableRandom. We have
|
||||
tested the pseudorandom sequences produced by SplitMix using two
|
||||
standard statistical test suites (DieHarder and TestU01) and they
|
||||
appear to be adequate for ``everyday'' use, such as in Monte
|
||||
Carlo algorithms and randomized data structures where speed is
|
||||
important.",
|
||||
month = oct,
|
||||
year = 2014,
|
||||
url = "http://dx.doi.org/10.1145/2660193.2660195",
|
||||
doi = "10.1145/2660193.2660195",
|
||||
isbn = 9781450325851
|
||||
}
|
||||
|
||||
@online{Stafford2011,
|
||||
author = {Stafford, David},
|
||||
title = {Better Bit Mixing - Improving on MurmurHash3's 64-bit Finalizer},
|
||||
date = {2011-09-28},
|
||||
url = {https://zimbry.blogspot.com/2011/09/better-bit-mixing-improving-on.html},
|
||||
urldate = {2026-09-12}
|
||||
}
|
||||
|
||||
@ARTICLE{Hamming1950-cw,
|
||||
title = "{Error detecting and error correcting codes}",
|
||||
author = "Hamming, R W",
|
||||
journal = "The Bell System Technical Journal",
|
||||
publisher = "Institute of Electrical and Electronics Engineers (IEEE)",
|
||||
volume = 29,
|
||||
number = 2,
|
||||
pages = "147--160",
|
||||
abstract = "The author was led to the study given in this paper from a
|
||||
consideration of large scale computing machines in which a large
|
||||
number of operations must be performed without a single error in
|
||||
the end result. This problem of “doing things right” on a large
|
||||
scale is not essentially new; in a telephone central office, for
|
||||
example, a very large number of operations are performed while
|
||||
the errors leading to wrong numbers are kept well under control,
|
||||
though they have not been completely eliminated. This has been
|
||||
achieved, in part, through the use of self-checking circuits. The
|
||||
occasional failure that escapes routine checking is still
|
||||
detected by the customer and will, if it persists, result in
|
||||
customer complaint, while if it is transient it will produce only
|
||||
occasional wrong numbers. At the same time the rest of the
|
||||
central office functions satisfactorily. In a digital computer,
|
||||
on the other hand, a single failure usually means the complete
|
||||
failure, in the sense that if it is detected no more computing
|
||||
can be done until the failure is located and corrected, while if
|
||||
it escapes detection then it invalidates all subsequent
|
||||
operations of the machine. Put in other words, in a telephone
|
||||
central office there are a number of parallel paths which are
|
||||
more or less independent of each other; in a digital machine
|
||||
there is usually a single long path which passes through the same
|
||||
piece of equipment many, many times before the answer is
|
||||
obtained.",
|
||||
month = apr,
|
||||
year = 1950,
|
||||
url = "http://dx.doi.org/10.1002/j.1538-7305.1950.tb00463.x",
|
||||
doi = "10.1002/j.1538-7305.1950.tb00463.x",
|
||||
issn = "0005-8580"
|
||||
}
|
||||
|
||||
@ARTICLE{Ondov2016-dq,
|
||||
title = "{Mash: fast genome and metagenome distance estimation using
|
||||
{MinHash}}",
|
||||
author = "Ondov, Brian D and Treangen, Todd J and Melsted, Páll and
|
||||
Mallonee, Adam B and Bergman, Nicholas H and Koren, Sergey and
|
||||
Phillippy, Adam M",
|
||||
journal = "Genome biology",
|
||||
volume = 17,
|
||||
number = 1,
|
||||
pages = 132,
|
||||
abstract = "Mash extends the MinHash dimensionality-reduction technique to
|
||||
include a pairwise mutation distance and P value significance
|
||||
test, enabling the efficient clustering and search of massive
|
||||
sequence collections. Mash reduces large sequences and sequence
|
||||
sets to small, representative sketches, from which global mutation
|
||||
distances can be rapidly estimated. We demonstrate several use
|
||||
cases, including the clustering of all 54,118 NCBI RefSeq genomes
|
||||
in 33 CPU h; real-time database search using assembled or
|
||||
unassembled Illumina, Pacific Biosciences, and Oxford Nanopore
|
||||
data; and the scalable clustering of hundreds of metagenomic
|
||||
samples by composition. Mash is freely released under a BSD
|
||||
license ( https://github.com/marbl/mash ).",
|
||||
month = jun,
|
||||
year = 2016,
|
||||
url = "http://dx.doi.org/10.1186/s13059-016-0997-x",
|
||||
keywords = "Alignment; Comparative genomics; Genomic distance; Metagenomics;
|
||||
Nanopore; Sequencing",
|
||||
doi = "10.1186/s13059-016-0997-x",
|
||||
pmc = "PMC4915045",
|
||||
pmid = 27323842,
|
||||
issn = "1465-6906",
|
||||
language = "en"
|
||||
}
|
||||
@ARTICLE{Bray1957-vv,
|
||||
title = "{An ordination of the upland forest communities of southern
|
||||
Wisconsin}",
|
||||
author = "Bray, J Roger and Curtis, J T",
|
||||
journal = "Ecological Monographs",
|
||||
publisher = "Wiley",
|
||||
volume = 27,
|
||||
number = 4,
|
||||
pages = "325--349",
|
||||
month = oct,
|
||||
year = 1957,
|
||||
url = "http://dx.doi.org/10.2307/1942268",
|
||||
doi = "10.2307/1942268",
|
||||
issn = "0012-9615,1557-7015",
|
||||
language = "en"
|
||||
}
|
||||
@ARTICLE{Gower1986-ri,
|
||||
title = "{Metric and Euclidean properties of dissimilarity coefficients}",
|
||||
author = "Gower, J C and Legendre, P",
|
||||
journal = "Journal of Classification",
|
||||
publisher = "Springer Science and Business Media LLC",
|
||||
volume = 3,
|
||||
number = 1,
|
||||
pages = "5--48",
|
||||
abstract = "We assemble here properties of certain dissimilarity coefficients
|
||||
and are specially concerned with their metric and Euclidean
|
||||
status. No attempt is made to be exhaustive as far as
|
||||
coefficients are concerned, but certain mathematical results that
|
||||
we have found useful are presented and should help establish
|
||||
similar properties for other coefficients. The response to
|
||||
different types of data is investigated, leading to guidance on
|
||||
the choice of an appropriate coefficient.",
|
||||
month = mar,
|
||||
year = 1986,
|
||||
url = "http://dx.doi.org/10.1007/bf01896809",
|
||||
doi = "10.1007/bf01896809",
|
||||
issn = "0176-4268,1432-1343",
|
||||
language = "en"
|
||||
}
|
||||
@ARTICLE{Legendre2001-cq,
|
||||
title = "{Ecologically meaningful transformations for ordination of species
|
||||
data}",
|
||||
author = "Legendre, Pierre and Gallagher, Eugene D",
|
||||
journal = "Oecologia",
|
||||
volume = 129,
|
||||
number = 2,
|
||||
pages = "271--280",
|
||||
abstract = "This paper examines how to obtain species biplots in unconstrained
|
||||
or constrained ordination without resorting to the Euclidean
|
||||
distance [used in principal-component analysis (PCA) and
|
||||
redundancy analysis (RDA)] or the chi-square distance [preserved
|
||||
in correspondence analysis (CA) and canonical correspondence
|
||||
analysis (CCA)] which are not always appropriate for the analysis
|
||||
of community composition data. To achieve this goal,
|
||||
transformations are proposed for species data tables. They allow
|
||||
ecologists to use ordination methods such as PCA and RDA, which
|
||||
are Euclidean-based, for the analysis of community data, while
|
||||
circumventing the problems associated with the Euclidean distance,
|
||||
and avoiding CA and CCA which present problems of their own in
|
||||
some cases. This allows the use of the original (transformed)
|
||||
species data in RDA carried out to test for relationships with
|
||||
explanatory variables (i.e. environmental variables, or factors of
|
||||
a multifactorial analysis-of-variance model); ecologists can then
|
||||
draw biplots displaying the relationships of the species to the
|
||||
explanatory variables. Another application allows the use of
|
||||
species data in other methods of multivariate data analysis which
|
||||
optimize a least-squares loss function; an example is K-means
|
||||
partitioning.",
|
||||
month = oct,
|
||||
year = 2001,
|
||||
url = "http://dx.doi.org/10.1007/s004420100716",
|
||||
keywords = "Biplot diagram; Canonical correspondence analysis; Correspondence
|
||||
analysis; Principal-component analysis; Redundancy analysis",
|
||||
doi = "10.1007/s004420100716",
|
||||
pmid = 28547606,
|
||||
issn = "0029-8549,1432-1939",
|
||||
language = "en"
|
||||
}
|
||||
@ARTICLE{Anderson2011-rq,
|
||||
title = "{Navigating the multiple meanings of β diversity: a roadmap for
|
||||
the practicing ecologist}",
|
||||
author = "Anderson, Marti J and Crist, Thomas O and Chase, Jonathan M and
|
||||
Vellend, Mark and Inouye, Brian D and Freestone, Amy L and
|
||||
Sanders, Nathan J and Cornell, Howard V and Comita, Liza S and
|
||||
Davies, Kendi F and Harrison, Susan P and Kraft, Nathan J B and
|
||||
Stegen, James C and Swenson, Nathan G",
|
||||
journal = "Ecology Letters",
|
||||
publisher = "Wiley",
|
||||
volume = 14,
|
||||
number = 1,
|
||||
pages = "19--28",
|
||||
abstract = "A recent increase in studies of β diversity has yielded a
|
||||
confusing array of concepts, measures and methods. Here, we
|
||||
provide a roadmap of the most widely used and ecologically
|
||||
relevant approaches for analysis through a series of mission
|
||||
statements. We distinguish two types of β diversity: directional
|
||||
turnover along a gradient vs. non-directional variation.
|
||||
Different measures emphasize different properties of ecological
|
||||
data. Such properties include the degree of emphasis on
|
||||
presence/absence vs. relative abundance information and the
|
||||
inclusion vs. exclusion of joint absences. Judicious use of
|
||||
multiple measures in concert can uncover the underlying nature of
|
||||
patterns in β diversity for a given dataset. A case study of
|
||||
Indonesian coral assemblages shows the utility of a multi-faceted
|
||||
approach. We advocate careful consideration of relevant
|
||||
questions, matched by appropriate analyses. The rigorous
|
||||
application of null models will also help to reveal potential
|
||||
processes driving observed patterns in β diversity.",
|
||||
month = jan,
|
||||
year = 2011,
|
||||
url = "http://dx.doi.org/10.1111/j.1461-0248.2010.01552.x",
|
||||
doi = "10.1111/j.1461-0248.2010.01552.x",
|
||||
pmid = 21070562,
|
||||
issn = "1461-023X,1461-0248",
|
||||
language = "en"
|
||||
}
|
||||
@ARTICLE{Jaccard1901-mx,
|
||||
title = "{Étude comparative de la distribution florale dans une portion
|
||||
des Alpes et du Jura}",
|
||||
author = "Jaccard, Paul",
|
||||
journal = "Bulletin de la Société vaudoise des sciences naturelles",
|
||||
publisher = "Imprimerie Corbaz \& Comp.",
|
||||
volume = 37,
|
||||
number = 142,
|
||||
pages = "547--579",
|
||||
abstract = "Schweizer Zeitschriften online - Ein Service der ETH-Bibliothek.
|
||||
Revues suisses en ligne - Un service de ETH Library. Swiss
|
||||
journals online - A service by ETH Library.",
|
||||
year = 1901,
|
||||
url = "http://dx.doi.org/10.5169/seals-266450",
|
||||
doi = "10.5169/seals-266450",
|
||||
issn = "0037-9603",
|
||||
language = "de"
|
||||
}
|
||||
@ARTICLE{Kosub2019-xt,
|
||||
title = "{A note on the triangle inequality for the Jaccard distance}",
|
||||
author = "Kosub, Sven",
|
||||
journal = "Pattern Recognition Letters",
|
||||
publisher = "Elsevier BV",
|
||||
volume = 120,
|
||||
pages = "36--38",
|
||||
month = apr,
|
||||
year = 2019,
|
||||
url = "http://dx.doi.org/10.1016/j.patrec.2018.12.007",
|
||||
doi = "10.1016/j.patrec.2018.12.007",
|
||||
issn = "0167-8655,1872-7344",
|
||||
language = "en"
|
||||
}
|
||||
@ARTICLE{Fleischmann2014-ed,
|
||||
title = {Evolution of genome size and chromosome number in the
|
||||
carnivorous plant genus Genlisea (Lentibulariaceae), with a new
|
||||
estimate of the minimum genome size in angiosperms},
|
||||
author = "Fleischmann, Andreas and Michael, Todd P and Rivadavia, Fernando
|
||||
and Sousa, Aretuza and Wang, Wenqin and Temsch, Eva M and
|
||||
Greilhuber, Johann and Müller, Kai F and Heubl, Günther",
|
||||
journal = "Annals of Botany",
|
||||
publisher = "Oxford University Press (OUP)",
|
||||
volume = 114,
|
||||
number = 8,
|
||||
pages = "1651--1663",
|
||||
abstract = "BACKGROUND AND AIMS: Some species of Genlisea possess ultrasmall
|
||||
nuclear genomes, the smallest known among angiosperms, and some
|
||||
have been found to have chromosomes of diminutive size, which may
|
||||
explain why chromosome numbers and karyotypes are not known for
|
||||
the majority of species of the genus. However, other members of
|
||||
the genus do not possess ultrasmall genomes, nor do most taxa
|
||||
studied in related genera of the family or order. This study
|
||||
therefore examined the evolution of genome sizes and chromosome
|
||||
numbers in Genlisea in a phylogenetic context. The correlations
|
||||
of genome size with chromosome number and size, with the
|
||||
phylogeny of the group and with growth forms and habitats were
|
||||
also examined. METHODS: Nuclear genome sizes were measured from
|
||||
cultivated plant material for a comprehensive sampling of taxa,
|
||||
including nearly half of all species of Genlisea and representing
|
||||
all major lineages. Flow cytometric measurements were conducted
|
||||
in parallel in two laboratories in order to compare the
|
||||
consistency of different methods and controls. Chromosome counts
|
||||
were performed for the majority of taxa, comparing different
|
||||
staining techniques for the ultrasmall chromosomes. KEY RESULTS:
|
||||
Genome sizes of 15 taxa of Genlisea are presented and interpreted
|
||||
in a phylogenetic context. A high degree of congruence was found
|
||||
between genome size distribution and the major phylogenetic
|
||||
lineages. Ultrasmall genomes with 1C values of <100 Mbp were
|
||||
almost exclusively found in a derived lineage of South American
|
||||
species. The ancestral haploid chromosome number was inferred to
|
||||
be n = 8. Chromosome numbers in Genlisea ranged from 2n = 2x = 16
|
||||
to 2n = 4x = 32. Ascendant dysploid series (2n = 36, 38) are
|
||||
documented for three derived taxa. The different ploidy levels
|
||||
corresponded to the two subgenera, but were not directly
|
||||
correlated to differences in genome size; the three different
|
||||
karyotype ranges mirrored the different sections of the genus.
|
||||
The smallest known plant genomes were not found in G. margaretae,
|
||||
as previously reported, but in G. tuberosa (1C ≈ 61 Mbp) and some
|
||||
strains of G. aurea (1C ≈ 64 Mbp). CONCLUSIONS: Genlisea is an
|
||||
ideal candidate model organism for the understanding of genome
|
||||
reduction as the genus includes species with both relatively
|
||||
large (∼1700 Mbp) and ultrasmall (∼61 Mbp) genomes. This
|
||||
comparative, phylogeny-based analysis of genome sizes and
|
||||
karyotypes in Genlisea provides essential data for selection of
|
||||
suitable species for comparative whole-genome analyses, as well
|
||||
as for further studies on both the molecular and cytogenetic
|
||||
basis of genome reduction in plants.",
|
||||
month = dec,
|
||||
year = 2014,
|
||||
url = "http://dx.doi.org/10.1093/aob/mcu189",
|
||||
keywords = "Bladderwort; Genlisea; Lamiales; Lentibulariaceae; carnivorous
|
||||
plant; chromosome number; flow cytometry; genome miniaturization;
|
||||
genome size",
|
||||
doi = "10.1093/aob/mcu189",
|
||||
pmc = "PMC4649684",
|
||||
pmid = 25274549,
|
||||
issn = "0305-7364,1095-8290",
|
||||
language = "en"
|
||||
}
|
||||
@ARTICLE{Fernandez2024-kd,
|
||||
title = "A 160 Gbp fork fern genome shatters size record for eukaryotes",
|
||||
author = "Fernández, Pol and Amice, Rémy and Bruy, David and Christenhusz,
|
||||
Maarten J M and Leitch, Ilia J and Leitch, Andrew L and Pokorny,
|
||||
Lisa and Hidalgo, Oriane and Pellicer, Jaume",
|
||||
journal = "iScience",
|
||||
publisher = "Elsevier BV",
|
||||
volume = 27,
|
||||
number = 6,
|
||||
pages = 109889,
|
||||
abstract = "Vascular plants are exceptional among eukaryotes due to their
|
||||
outstanding genome size diversity which ranges ∼2,400-fold,
|
||||
including the largest genome so far recorded in the angiosperm
|
||||
Paris japonica (148.89 Gbp/1C). Despite available data showing
|
||||
that giant genomes are restricted across the Tree of Life, the
|
||||
biological limits to genome size expansion remain to be
|
||||
established. Here, we report the discovery of an even larger
|
||||
eukaryotic genome in Tmesipteris oblanceolata, a New Caledonian
|
||||
fork fern. At 160.45 Gbp/1C, this record-breaking genome
|
||||
challenges current understanding and opens new avenues to explore
|
||||
the evolutionary dynamics of genomic gigantism.",
|
||||
month = jun,
|
||||
year = 2024,
|
||||
url = "http://dx.doi.org/10.1016/j.isci.2024.109889",
|
||||
keywords = "Plant evolution; Plant genetics",
|
||||
doi = "10.1016/j.isci.2024.109889",
|
||||
pmc = "PMC11270024",
|
||||
pmid = 39055604,
|
||||
issn = "2589-0042",
|
||||
language = "en"
|
||||
}
|
||||
@ARTICLE{Fraser1995-jc,
|
||||
title = "The minimal gene complement of Mycoplasma genitalium",
|
||||
author = "Fraser, C M and Gocayne, J D and White, O and Adams, M D and
|
||||
Clayton, R A and Fleischmann, R D and Bult, C J and Kerlavage, A
|
||||
R and Sutton, G and Kelley, J M and Fritchman, R D and Weidman, J
|
||||
F and Small, K V and Sandusky, M and Fuhrmann, J and Nguyen, D
|
||||
and Utterback, T R and Saudek, D M and Phillips, C A and Merrick,
|
||||
J M and Tomb, J F and Dougherty, B A and Bott, K F and Hu, P C
|
||||
and Lucier, T S and Peterson, S N and Smith, H O and Hutchison,
|
||||
3rd, C A and Venter, J C",
|
||||
journal = "Science (New York, N.Y.)",
|
||||
publisher = "American Association for the Advancement of Science (AAAS)",
|
||||
volume = 270,
|
||||
number = 5235,
|
||||
pages = "397--403",
|
||||
abstract = "The complete nucleotide sequence (580,070 base pairs) of the
|
||||
Mycoplasma genitalium genome, the smallest known genome of any
|
||||
free-living organism, has been determined by whole-genome random
|
||||
sequencing and assembly. A total of only 470 predicted coding
|
||||
regions were identified that include genes required for DNA
|
||||
replication, transcription and translation, DNA repair, cellular
|
||||
transport, and energy metabolism. Comparison of this genome to
|
||||
that of Haemophilus influenzae suggests that differences in
|
||||
genome content are reflected as profound differences in
|
||||
physiology and metabolic capacity between these two organisms.",
|
||||
month = oct,
|
||||
year = 1995,
|
||||
url = "http://dx.doi.org/10.1126/science.270.5235.397",
|
||||
doi = "10.1126/science.270.5235.397",
|
||||
pmid = 7569993,
|
||||
issn = "0036-8075,1095-9203",
|
||||
language = "en"
|
||||
}
|
||||
@ARTICLE{Schneiker2007-gs,
|
||||
title = "Complete genome sequence of the myxobacterium Sorangium
|
||||
cellulosum",
|
||||
author = "Schneiker, Susanne and Perlova, Olena and Kaiser, Olaf and Gerth,
|
||||
Klaus and Alici, Aysel and Altmeyer, Matthias O and Bartels,
|
||||
Daniela and Bekel, Thomas and Beyer, Stefan and Bode, Edna and
|
||||
Bode, Helge B and Bolten, Christoph J and Choudhuri, Jomuna V and
|
||||
Doss, Sabrina and Elnakady, Yasser A and Frank, Bettina and
|
||||
Gaigalat, Lars and Goesmann, Alexander and Groeger, Carolin and
|
||||
Gross, Frank and Jelsbak, Lars and Jelsbak, Lotte and Kalinowski,
|
||||
Jörn and Kegler, Carsten and Knauber, Tina and Konietzny,
|
||||
Sebastian and Kopp, Maren and Krause, Lutz and Krug, Daniel and
|
||||
Linke, Bukhard and Mahmud, Taifo and Martinez-Arias, Rosa and
|
||||
McHardy, Alice C and Merai, Michelle and Meyer, Folker and
|
||||
Mormann, Sascha and Muñoz-Dorado, Jose and Perez, Juana and
|
||||
Pradella, Silke and Rachid, Shwan and Raddatz, Günter and
|
||||
Rosenau, Frank and Rückert, Christian and Sasse, Florenz and
|
||||
Scharfe, Maren and Schuster, Stephan C and Suen, Garret and
|
||||
Treuner-Lange, Anke and Velicer, Gregory J and Vorhölter,
|
||||
Frank-Jörg and Weissman, Kira J and Welch, Roy D and Wenzel,
|
||||
Silke C and Whitworth, David E and Wilhelm, Susanne and Wittmann,
|
||||
Christoph and Blöcker, Helmut and Pühler, Alfred and Müller, Rolf",
|
||||
journal = "Nature Biotechnology",
|
||||
publisher = "Springer Science and Business Media LLC",
|
||||
volume = 25,
|
||||
number = 11,
|
||||
pages = "1281--1289",
|
||||
abstract = "The genus Sorangium synthesizes approximately half of the
|
||||
secondary metabolites isolated from myxobacteria, including the
|
||||
anti-cancer metabolite epothilone. We report the complete genome
|
||||
sequence of the model Sorangium strain S. cellulosum So ce56,
|
||||
which produces several natural products and has morphological and
|
||||
physiological properties typical of the genus. The circular
|
||||
genome, comprising 13,033,779 base pairs, is the largest
|
||||
bacterial genome sequenced to date. No global synteny with the
|
||||
genome of Myxococcus xanthus is apparent, revealing an
|
||||
unanticipated level of divergence between these myxobacteria. A
|
||||
large percentage of the genome is devoted to regulation,
|
||||
particularly post-translational phosphorylation, which probably
|
||||
supports the strain's complex, social lifestyle. This regulatory
|
||||
network includes the highest number of eukaryotic protein
|
||||
kinase-like kinases discovered in any organism. Seventeen
|
||||
secondary metabolite loci are encoded in the genome, as well as
|
||||
many enzymes with potential utility in industry.",
|
||||
month = nov,
|
||||
year = 2007,
|
||||
url = "http://dx.doi.org/10.1038/nbt1354",
|
||||
doi = "10.1038/nbt1354",
|
||||
pmid = 17965706,
|
||||
issn = "1087-0156,1546-1696",
|
||||
language = "en"
|
||||
}
|
||||
@incollection{Karlin1985,
|
||||
author = {Karlin, Samuel and Ost, Fr{\'e}d{\'e}ric},
|
||||
title = {Maximal segmental match length among random sequences from a finite alphabet},
|
||||
booktitle = {Proceedings of the Berkeley Conference in Honor of Jerzy Neyman and Jack Kiefer},
|
||||
editor = {Le Cam, Lucien M. and Olshen, Richard A.},
|
||||
volume = {1},
|
||||
pages = {225--243},
|
||||
year = {1985},
|
||||
publisher = {Wadsworth},
|
||||
address = {Monterey, CA}
|
||||
}
|
||||
@article{Karlin1988,
|
||||
author = {Karlin, Samuel and Ost, Fr{\'e}d{\'e}ric},
|
||||
title = {Maximal length of common words among random letter sequences},
|
||||
journal = {The Annals of Probability},
|
||||
volume = {16},
|
||||
number = {2},
|
||||
pages = {535--563},
|
||||
year = {1988}
|
||||
}
|
||||
@ARTICLE{Le-Cam1960-pe,
|
||||
title = "{An approximation theorem for the Poisson binomial distribution}",
|
||||
author = "Le Cam, Lucien",
|
||||
journal = "Pacific Journal of Mathematics",
|
||||
publisher = "Mathematical Sciences Publishers",
|
||||
volume = 10,
|
||||
number = 4,
|
||||
pages = "1181--1197",
|
||||
month = dec,
|
||||
year = 1960,
|
||||
url = "http://dx.doi.org/10.2140/pjm.1960.10.1181",
|
||||
doi = "10.2140/pjm.1960.10.1181",
|
||||
issn = "0030-8730,1945-5844"
|
||||
}
|
||||
|
||||
@ARTICLE{Hoeffding1956-bx,
|
||||
title = "{On the distribution of the number of successes in independent
|
||||
trials}",
|
||||
author = "Hoeffding, Wassily",
|
||||
journal = "The Annals of Mathematical Statistics",
|
||||
publisher = "Institute of Mathematical Statistics",
|
||||
volume = 27,
|
||||
number = 3,
|
||||
pages = "713--721",
|
||||
abstract = "Let $S$ be the number of successes in $n$ independent trials, and
|
||||
let $p_j$ denote the probability of success in the $j$th trial,
|
||||
$j = 1, 2, \cdots, n$ (Poisson trials). We consider the problem
|
||||
of finding the maximum and the minimum of $Eg(S),$ the expected
|
||||
value of a given real-valued function of $S,$ when $ES = np$ is
|
||||
fixed. It is well known that the maximum of the variance of $S$
|
||||
is attained when $p_1 = p_2 = \cdots = p_n = p.$ This can be
|
||||
interpreted as showing that the variability in the number of
|
||||
successes is highest when the successes are equally probable
|
||||
(Bernoulli trials). This interpretation is further supported by
|
||||
the following two theorems, proved in this paper. If $b$ and $c$
|
||||
are two integers, $0 \leqq b \leqq np \leqq c \leqq n,$ the
|
||||
probability $P(b \leqq S \leqq c)$ attains its minimum if and
|
||||
only if $p_1 = p_2 = \cdots = p_n = p,$ unless $b = 0$ and $c =
|
||||
n$ (Theorem 5, a corollary of Theorem 4, which gives the maximum
|
||||
and the minimum of $P(S \leqq c)$). If $g$ is a strictly convex
|
||||
function, $Eg(S)$ attains its maximum if and only if $p_1 = p_2 =
|
||||
\cdots = p_n = p$ (Theorem 3). These results are obtained with
|
||||
the help of two theorems concerning the extrema of the expected
|
||||
value of an arbitrary function $g(S)$ under the condition $ES =
|
||||
np.$ Theorem 1 gives necessary conditions for the maximum and the
|
||||
minimum of $Eg(S).$ Theorem 2 gives a partial characterization of
|
||||
the set of points at which an extremum is attained. Corollary 2.1
|
||||
states that the maximum and the minimum are attained when $p_1,
|
||||
p_2, \cdots, p_n$ take on, at most, three different values, only
|
||||
one of which is distinct from 0 and 1. Applications of Theorems 3
|
||||
and 5 to problems of estimation and testing are pointed out in
|
||||
Section 5.",
|
||||
month = sep,
|
||||
year = 1956,
|
||||
url = "http://dx.doi.org/10.1214/aoms/1177728178",
|
||||
doi = "10.1214/aoms/1177728178",
|
||||
issn = "0003-4851,2168-8990",
|
||||
language = "en"
|
||||
}
|
||||
@ARTICLE{Gleser1975-yu,
|
||||
title = "{On the distribution of the number of successes in independent
|
||||
trials}",
|
||||
author = "Gleser, Leon Jay",
|
||||
journal = "Annals of Probability",
|
||||
publisher = "Institute of Mathematical Statistics",
|
||||
volume = 3,
|
||||
number = 1,
|
||||
pages = "182--188",
|
||||
abstract = "Let $S$ be the number of successes in $n$ independent Bernoulli
|
||||
trials, where $p_j$ is the probability of success on the $j$th
|
||||
trial. Let $\mathbf{p} = (p_1, p_2, \cdots, p_n)$, and for any
|
||||
integer $c, 0 \leqq c \leqq n$, let $H(c \mid \mathbf{p}) = P\{S
|
||||
\leqq c\}$. Let $\mathbf{p}^{(1)}$ be one possible choice of
|
||||
$\mathbf{p}$ for which $E(S) = \lambda$. For any $n \times n$
|
||||
doubly stochastic matrix $\Pi$, let $\mathbf{p}^{(2)} =
|
||||
\mathbf{p}^{(1)}\Pi$. Then in the present paper it is shown that
|
||||
$H(c \mid \mathbf{p}^{(1)}) \leqq H(c \mid \mathbf{p}^{(2)})$ for
|
||||
$0 \leqq c \leqq \lbrack\lambda - 2\rbrack$, and $H(c \mid
|
||||
\mathbf{p}^{(1)}) \geqq H(c \mid \mathbf{p}^{(2)})$ for
|
||||
$\lbrack\lambda + 2\rbrack \leqq c \leqq n$. These results
|
||||
provide a refinement of inequalities for $H(c \mid \mathbf{p})$
|
||||
obtained by Hoeffding [3]. Their derivation is achieved by
|
||||
applying consequences of the partial ordering of majorization.",
|
||||
month = feb,
|
||||
year = 1975,
|
||||
url = "http://dx.doi.org/10.2307/2959277",
|
||||
doi = "10.1214/aop/1176996461",
|
||||
issn = "0091-1798,2168-894X"
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
# DNA encoding
|
||||
|
||||
## 2-bit nucleotide encoding
|
||||
|
||||
Every nucleotide is encoded on 2 bits, most-significant-bit first within each word:
|
||||
|
||||
| Base | Encoding |
|
||||
|------|----------|
|
||||
| A | `00` |
|
||||
| C | `01` |
|
||||
| G | `10` |
|
||||
| T | `11` |
|
||||
|
||||
The Watson-Crick complement of a base is its bitwise NOT on 2 bits: $\text{complement}(base) = \lnot base \mathbin{\&} \texttt{0b11}$.
|
||||
|
||||
## Kmer encoding
|
||||
|
||||
A kmer of length $k$ ($k \le 31$) fits in a single 64-bit word. The first nucleotide occupies the two most significant bits, each following nucleotide occupies the next two bits, and unused low-order bits are zero. Extracting nucleotide i (0-indexed from the 5′ end) is a shift-and-mask operation.
|
||||
|
||||
Reverse complement is computed by bit manipulation directly on the packed word, without any lookup table: complement every base, reverse the byte order, then reverse the order of 2-bit groups within each byte in two more passes, and finally realign the result to the most-significant bits.
|
||||
|
||||
## Canonical form
|
||||
|
||||
The canonical form of a packed 2-bit sequence is the lexicographic minimum of the sequence and its reverse complement:
|
||||
|
||||
$$\text{canonical}(x) = \min\big(x,\ \text{revcomp}(x)\big)$$
|
||||
|
||||
This is a single operation on the packed representation, so it applies identically regardless of what `x` represents — a kmer (see [Kmers](kmers.md)), an m-mer (see [Minimizer selection](minimizer_selection.md)), or a super-kmer (see [Super-kmers](superkmers.md)). Using the canonical form halves the relevant space and makes counting/selection strand-independent: a sequence and its reverse complement are always treated as the same entity, regardless of which DNA strand was sequenced.
|
||||
@@ -1,36 +0,0 @@
|
||||
# Low-complexity kmer filter
|
||||
|
||||
Low-complexity kmers (homopolymer runs, tandem repeats) can dominate an index without carrying useful information. `obikmer` detects and excludes them during index construction using a normalized Shannon entropy score.
|
||||
|
||||
## Sub-word frequencies
|
||||
|
||||
For a kmer of length $k$ and a sub-word size $ws$ ($1 \le ws \le ws_{\max}$, default $ws_{\max} = 6$), the kmer is decomposed into its $k - ws + 1$ overlapping sub-words of length $ws$ by sliding a window across it. Each sub-word is tallied under its raw 2-bit-packed value, with no canonicalization.
|
||||
|
||||
## Corrected Shannon entropy
|
||||
|
||||
Let $f_j$ be the observed count of raw sub-word $j$, and $n_{\text{words}} = k - ws + 1$ the total number of sub-words. The entropy is:
|
||||
|
||||
$$H_{\text{corr}} = \log(n_{\text{words}}) - \frac{1}{n_{\text{words}}} \sum_j f_j \log f_j$$
|
||||
|
||||
## Small-sample correction
|
||||
|
||||
Because only $n_{\text{words}}$ sub-words are observed among up to $4^{ws}$ possible values, the achievable maximum entropy $H_{\max}$ is bounded below $\log(4^{ws})$ for small samples. $H_{\max}$ is computed from the most uniform integer distribution achievable with $n_{\text{words}}$ observations over $4^{ws}$ categories. The normalized entropy is:
|
||||
|
||||
$$\hat{H}(ws) = \frac{H_{\text{corr}}}{H_{\max}} \in [0, 1]$$
|
||||
|
||||
A value near 0 indicates low complexity (e.g. a homopolymer run); near 1 indicates high complexity, characteristic of a random sequence.
|
||||
|
||||
## Final score
|
||||
|
||||
The filter evaluates $\hat{H}(ws)$ for every word size from 1 to ws_max and keeps the minimum:
|
||||
|
||||
$$\text{entropy}(kmer) = \min_{ws=1}^{ws_{\max}} \hat{H}(ws)$$
|
||||
|
||||
Taking the minimum across word sizes ensures that repetition at any scale is detected: a homopolymer is caught at $ws=1$, a dinucleotide repeat at $ws=2$, and so on. A kmer is rejected if its entropy score falls below a threshold $\theta$ (default 0.7), a configurable collection parameter.
|
||||
|
||||
## Properties
|
||||
|
||||
The entropy score depends only on the kmer sequence itself, not on where or how many times it occurs:
|
||||
|
||||
- **Orientation invariance**: a kmer and its reverse complement always receive the same score.
|
||||
- **Context independence**: a given kmer is always accepted or always rejected, regardless of which genome or read it appears in. The filter defines a fixed partition of the kmer space into low-complexity and valid kmers.
|
||||
Loaded 100 of 653 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user