refactor(benchmark): consolidate generated artifacts under run/ directory

Restructure the benchmark pipeline to direct all simulated data, indices, statistics, and query outputs into a unified `run/` directory. Update Makefile targets, shell scripts, and Python utilities to resolve paths relative to this new base. Adjust documentation and dependency tracking to match the revised layout, and remove outdated temporary artifacts.
2026-08-29 00:19:31 +02:00
parent 54e4600120
commit 52f26809af
1057 changed files with 449 additions and 449 deletions
@@ -8,108 +8,124 @@ gmake simulate     # simulation only
 gmake reference    # reference kmer sets only
 ```

+All generated and downloaded artifacts live under `run/` (see
+[Directory layout](#directory-layout)), so the whole tree is gitignored with
+a single `benchmark/run/` entry.
+
 ## Pipeline overview

 ```mermaid
 flowchart TD
-    GENOMES["genomes/*.fna.gz"]
+    GENOMES["run/genomes/*.fna.gz"]
    BIN["obikmer binary"]

    GENOMES --> simulate
-    simulate --> simdata[("simulated_data/")]
+    simulate --> simdata[("run/simulated_data/")]

    simdata --> reference
-    reference --> refnpz[("reference_index/*.npz")]
+    reference --> refnpz[("run/reference_index/*.npz")]

    subgraph presence ["Presence track"]
        simdata  --> index_presence
        BIN      --> index_presence
-        index_presence --> pres_done[("specimen_index_presence/")]
-        index_presence --> pres_istats[("stats/indexing_presence/")]
+        index_presence --> pres_done[("run/specimen_index_presence/")]
+        index_presence --> pres_istats[("run/stats/indexing_presence/")]
        pres_istats --> aggregate_index_presence

        pres_done --> merge_presence
        BIN       --> merge_presence
-        merge_presence --> gpres[("global_index_presence/")]
+        merge_presence --> gpres[("run/global_index_presence/")]

        refnpz    --> verify_presence
        pres_done --> verify_presence
-        verify_presence --> vpres_stats[("stats/verify_presence/")]
+        verify_presence --> vpres_stats[("run/stats/verify_presence/")]
        vpres_stats --> aggregate_verify_presence

        gpres --> filter_presence
        BIN   --> filter_presence
-        filter_presence --> spec_pres[("specific_index_presence/")]
-        filter_presence --> spec_pres_stats[("stats/specific_kmer_presence/")]
+        filter_presence --> spec_pres[("run/specific_index_presence/")]
+        filter_presence --> spec_pres_stats[("run/stats/specific_kmer_presence/")]
        spec_pres_stats --> aggregate_filter_presence

        refnpz --> verify_merge_presence
        gpres  --> verify_merge_presence
-        verify_merge_presence --> vmp[("stats/verify_merge_presence/")]
+        verify_merge_presence --> vmp[("run/stats/verify_merge_presence/")]
    end

    subgraph count ["Count track"]
        simdata --> index_count
        BIN     --> index_count
-        index_count --> count_done[("specimen_index_count/")]
-        index_count --> count_istats[("stats/indexing_count/")]
+        index_count --> count_done[("run/specimen_index_count/")]
+        index_count --> count_istats[("run/stats/indexing_count/")]
        count_istats --> aggregate_index_count

        count_done --> merge_count
        BIN        --> merge_count
-        merge_count --> gcount[("global_index_count/")]
+        merge_count --> gcount[("run/global_index_count/")]

        refnpz     --> verify_count
        count_done --> verify_count
-        verify_count --> vcount_stats[("stats/verify_count/")]
+        verify_count --> vcount_stats[("run/stats/verify_count/")]
        vcount_stats --> aggregate_verify_count

        gcount --> filter_count
        BIN    --> filter_count
-        filter_count --> spec_count[("specific_index_count/")]
-        filter_count --> spec_count_stats[("stats/specific_kmer_count/")]
+        filter_count --> spec_count[("run/specific_index_count/")]
+        filter_count --> spec_count_stats[("run/stats/specific_kmer_count/")]
        spec_count_stats --> aggregate_filter_count

        refnpz --> verify_merge_count
        gcount --> verify_merge_count
-        verify_merge_count --> vmc[("stats/verify_merge_count/")]
+        verify_merge_count --> vmc[("run/stats/verify_merge_count/")]
    end

    subgraph query ["Query track (2 specimens: E. coli + archaeon)"]
        GENOMES --> simulate_query
-        simulate_query --> qdata[("query_data/")]
+        simulate_query --> qdata[("run/query_data/")]

-        gpres --> pack_sparse
-        BIN   --> pack_sparse
-        pack_sparse --> gsparse[("global_index_presence_sparse/")]
+        gpres --> pack_dense_presence
+        BIN   --> pack_dense_presence
+        pack_dense_presence --> gpresd[("run/global_index_presence_dense/")]

-        qdata  --> query_dense
-        gpres  --> query_dense
-        BIN    --> query_dense
-        query_dense --> qd[("query_dense/")]
-        query_dense --> qd_stats[("stats/query_dense/")]
-        qd_stats --> aggregate_query_dense
+        count_done --> pack_dense_count
+        BIN        --> pack_dense_count
+        pack_dense_count --> gcountd[("run/global_index_count_dense/")]

-        qdata   --> query_sparse
-        gsparse --> query_sparse
-        BIN     --> query_sparse
-        query_sparse --> qs[("query_sparse/")]
-        query_sparse --> qs_stats[("stats/query_sparse/")]
-        qs_stats --> aggregate_query_sparse
+        qdata  --> query_presence_dense
+        gpresd --> query_presence_dense
+        query_presence_dense --> qpd[("run/query_presence_dense/")]
+        qpd --> aggregate_query_presence_dense

-        qd --> verify_query
-        qs --> verify_query
-        verify_query --> vq_stats[("stats/verify_query/")]
+        qdata  --> query_presence_sparse
+        gpres  --> query_presence_sparse
+        query_presence_sparse --> qps[("run/query_presence_sparse/")]
+        qps --> aggregate_query_presence_sparse
+
+        qdata   --> query_count_dense
+        gcountd --> query_count_dense
+        query_count_dense --> qcd[("run/query_count_dense/")]
+        qcd --> aggregate_query_count_dense
+
+        qdata  --> query_count_sparse
+        gcount --> query_count_sparse
+        query_count_sparse --> qcs[("run/query_count_sparse/")]
+        qcs --> aggregate_query_count_sparse
+
+        qpd --> verify_query
+        qps --> verify_query
+        verify_query --> vq_stats[("run/stats/verify_query/")]
        vq_stats --> aggregate_verify_query
    end

-    aggregate_verify_presence  --> all
-    aggregate_verify_count     --> all
-    vmp                        --> all
-    vmc                        --> all
-    aggregate_query_dense      --> all
-    aggregate_query_sparse     --> all
-    aggregate_verify_query     --> all
+    aggregate_verify_presence         --> all
+    aggregate_verify_count            --> all
+    vmp                               --> all
+    vmc                               --> all
+    aggregate_query_presence_dense    --> all
+    aggregate_query_presence_sparse   --> all
+    aggregate_query_count_dense       --> all
+    aggregate_query_count_sparse      --> all
+    aggregate_verify_query            --> all
    all -. "$(MAKE) re-eval" .-> aggregate_filter_presence
    all -. "$(MAKE) re-eval" .-> aggregate_filter_count
 ```
@@ -137,44 +153,59 @@ flowchart TD
 | `verify_merge_presence` | `verify_merge_presence.sh` | Verify global presence index against all reference sets |
 | `verify_merge_count` | `verify_merge_count.sh` | Verify global count index against all reference sets |
 | `simulate_query` | `simulate_query_one.sh` | Simulate a fixed-size (100k pairs) read set per query specimen |
-| `pack_sparse` | `pack_sparse.sh` | Build `global_index_presence_sparse/` from `global_index_presence/` |
-| `query_dense` | `query_one.sh dense` | Query each query specimen's reads against the dense global index |
-| `query_sparse` | `query_one.sh sparse` | Query each query specimen's reads against the sparse global index |
-| `aggregate_query_dense` | `aggregate_stats.sh` | Aggregate dense query wall/RSS stats |
-| `aggregate_query_sparse` | `aggregate_stats.sh` | Aggregate sparse query wall/RSS stats |
-| `verify_query` | `verify_query_one.sh` | Diff dense vs sparse query output per specimen (regression check) |
+| `pack_dense_presence` | `pack_dense.sh presence` | Build `global_index_presence_dense/` from `global_index_presence/` |
+| `pack_dense_count` | `pack_dense.sh count` | Build `global_index_count_dense/` from the per-specimen count sources |
+| `query_presence_dense` | `query_one.sh presence dense` | Query against the dense presence global index |
+| `query_presence_sparse` | `query_one.sh presence sparse` | Query against the sparse (as-merged) presence global index |
+| `query_count_dense` | `query_one.sh count dense` | Query against the dense count global index |
+| `query_count_sparse` | `query_one.sh count sparse` | Query against the sparse (as-merged) count global index |
+| `aggregate_query_presence_dense` | `aggregate_stats.sh` | Aggregate dense presence-query wall/RSS stats |
+| `aggregate_query_presence_sparse` | `aggregate_stats.sh` | Aggregate sparse presence-query wall/RSS stats |
+| `aggregate_query_count_dense` | `aggregate_stats.sh` | Aggregate dense count-query wall/RSS stats |
+| `aggregate_query_count_sparse` | `aggregate_stats.sh` | Aggregate sparse count-query wall/RSS stats |
+| `verify_query` | `verify_query_one.sh` | Diff dense vs sparse presence-query output per specimen (regression check) |
 | `aggregate_verify_query` | `aggregate_stats.sh` | Aggregate dense/sparse query regression stats |

 ## Directory layout

 ```
 benchmark/
-├── genomes/                        # input reference genomes (.fna.gz)
-├── simulated_data/                 # generated by simulate
-│   └── <species>/<specimen>/
-├── query_data/                     # generated by simulate_query (2 specimens, fixed 100k pairs)
-│   └── <species>/<specimen>/
-├── reference_index/                # reference kmer sets (.npz)
-├── specimen_index_presence/        # per-specimen presence indexes
-├── specimen_index_count/           # per-specimen count indexes
-├── global_index_presence/          # merged global presence index (dense-packed)
-├── global_index_presence_sparse/   # global presence index, sparse-packed (query benchmark)
-├── global_index_count/             # merged global count index
-├── specific_index_presence/        # species-specific presence indexes
-├── specific_index_count/           # species-specific count indexes
-├── query_dense/                    # query output against global_index_presence
-├── query_sparse/                   # query output against global_index_presence_sparse
-└── stats/                          # all benchmark statistics
-    ├── indexing_presence/
-    ├── indexing_count/
-    ├── verify_presence/
-    ├── verify_count/
-    ├── specific_kmer_presence/
-    ├── specific_kmer_count/
-    ├── verify_merge_presence/
-    ├── verify_merge_count/
-    ├── pack_sparse/
-    ├── query_dense/
-    ├── query_sparse/
-    └── verify_query/
+└── run/                                 # everything generated/downloaded — gitignored as a whole
+    ├── genomes/                         # input reference genomes (.fna.gz), downloaded by downloads.sh
+    ├── simulated_data/                  # generated by simulate
+    │   └── <species>/<specimen>/
+    ├── query_data/                      # generated by simulate_query (2 specimens, fixed 100k pairs)
+    │   └── <species>/<specimen>/
+    ├── reference_index/                 # reference kmer sets (.npz)
+    ├── reference_dist/                  # reference pairwise distance matrices
+    ├── obikmer_dist/                    # obikmer phylo distance matrices (presence/, count/)
+    ├── specimen_index_presence/         # per-specimen presence indexes
+    ├── specimen_index_count/            # per-specimen count indexes
+    ├── global_index_presence/           # merged global presence index (sparse, as merged)
+    ├── global_index_presence_dense/     # global presence index, dense-repacked (query benchmark)
+    ├── global_index_count/              # merged global count index (sparse, as merged)
+    ├── global_index_count_dense/        # global count index, dense-repacked (query benchmark)
+    ├── specific_index_presence/         # species-specific presence indexes
+    ├── specific_index_count/            # species-specific count indexes
+    ├── query_presence_dense/            # query output against global_index_presence_dense
+    ├── query_presence_sparse/           # query output against global_index_presence
+    ├── query_count_dense/               # query output against global_index_count_dense
+    ├── query_count_sparse/              # query output against global_index_count
+    └── stats/                           # all benchmark statistics
+        ├── indexing_presence/
+        ├── indexing_count/
+        ├── verify_presence/
+        ├── verify_count/
+        ├── specific_kmer_presence/
+        ├── specific_kmer_count/
+        ├── verify_merge_presence/
+        ├── verify_merge_count/
+        ├── pack_dense_presence/
+        ├── pack_dense_count/
+        ├── query_presence_dense/
+        ├── query_presence_sparse/
+        ├── query_count_dense/
+        ├── query_count_sparse/
+        ├── dist_comparison/
+        └── verify_query/
 ```