OBIKmers/obikmer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 34 Wiki Activity

refactor(benchmark): consolidate generated artifacts under run/ directory
ci.yml / build (pull_request) Successful in 3m49s
Details

Browse Source 
Restructure the benchmark pipeline to direct all simulated data, indices, statistics, and query outputs into a unified `run/` directory. Update Makefile targets, shell scripts, and Python utilities to resolve paths relative to this new base. Adjust documentation and dependency tracking to match the revised layout, and remove outdated temporary artifacts.
This commit is contained in:
Eric Coissac
2026-08-29 00:19:31 +02:00
parent 54e4600120
commit 52f26809af
1057 changed files with 449 additions and 449 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmark/Makefile
+88 -83
View File
@@ -2,23 +2,27 @@
BINARY := ../src/target/release/obikmer
VENV_PY := ../.venv/bin/python3
GENOMES := $(wildcard genomes/*.fna.gz)
# All generated/downloaded artifacts live under RUN/ so the whole tree can be
# gitignored with a single entry (benchmark/run/) — see benchmark/README.md.
RUN := run
GENOMES := $(wildcard $(RUN)/genomes/*.fna.gz)
# SPECIMENS, SPECIES, and the full dependency graph are generated by
# make_deps.py from the genome FASTA headers — like .d files in C.
# Make rebuilds deps.mk whenever genomes/ changes and restarts.
-include deps.mk
REF_NPZS := $(SPECIMENS:%=reference_index/%.npz)
REF_DIST_CSVS := $(addprefix reference_dist/, \
REF_NPZS := $(SPECIMENS:%=$(RUN)/reference_index/%.npz)
REF_DIST_CSVS := $(addprefix $(RUN)/reference_dist/, \
shared_kmers.csv hamming_dist.csv jaccard_dist.csv \
bray_curtis_dist.csv relfreq_bray_curtis_dist.csv \
euclidean_dist.csv relfreq_euclidean_dist.csv \
hellinger_dist.csv hellinger_euclidean_dist.csv)
OBIKMER_PRESENCE_DIST := $(addprefix obikmer_dist/presence/, \
OBIKMER_PRESENCE_DIST := $(addprefix $(RUN)/obikmer_dist/presence/, \
jaccard_dist.csv jaccard_shared.csv jaccard_nj.nwk \
hamming_dist.csv hamming_nj.nwk)
OBIKMER_COUNT_DIST := $(addprefix obikmer_dist/count/, \
OBIKMER_COUNT_DIST := $(addprefix $(RUN)/obikmer_dist/count/, \
jaccard_dist.csv jaccard_shared.csv jaccard_nj.nwk \
bray_curtis_dist.csv bray_curtis_nj.nwk \
relfreq_bray_curtis_dist.csv relfreq_bray_curtis_nj.nwk \
@@ -26,28 +30,28 @@ OBIKMER_COUNT_DIST := $(addprefix obikmer_dist/count/, \
relfreq_euclidean_dist.csv relfreq_euclidean_nj.nwk \
hellinger_dist.csv hellinger_nj.nwk \
hellinger_euclidean_dist.csv hellinger_euclidean_nj.nwk)
DIST_COMPARISON := stats/dist_comparison/summary.csv
PRESENCE_DONE := $(SPECIMENS:%=specimen_index_presence/%/index.done)
PRESENCE_STATS := $(SPECIMENS:%=stats/indexing_presence/%.stats)
COUNT_DONE := $(SPECIMENS:%=specimen_index_count/%/index.done)
COUNT_STATS := $(SPECIMENS:%=stats/indexing_count/%.stats)
VERIFY_PRESENCE_STATS := $(SPECIMENS:%=stats/verify_presence/%.stats)
VERIFY_COUNT_STATS := $(SPECIMENS:%=stats/verify_count/%.stats)
SPECIFIC_PRESENCE_DONE := $(SPECIES:%=specific_index_presence/%/index.done)
SPECIFIC_PRESENCE_STATS := $(SPECIES:%=stats/specific_kmer_presence/%.stats)
SPECIFIC_COUNT_DONE := $(SPECIES:%=specific_index_count/%/index.done)
SPECIFIC_COUNT_STATS := $(SPECIES:%=stats/specific_kmer_count/%.stats)
SIMULATED_READS := $(foreach s,$(SPECIMENS),simulated_data/$(subst --,/,$s)/reads_R1.fastq.gz)
QUERY_READS := $(foreach s,$(QUERY_SPECIMENS),query_data/$(subst --,/,$s)/reads_R1.fastq.gz)
QUERY_PRESENCE_DENSE_DONE := $(QUERY_SPECIMENS:%=query_presence_dense/%.fasta.gz)
QUERY_PRESENCE_DENSE_STATS := $(QUERY_SPECIMENS:%=stats/query_presence_dense/%.stats)
QUERY_PRESENCE_SPARSE_DONE := $(QUERY_SPECIMENS:%=query_presence_sparse/%.fasta.gz)
QUERY_PRESENCE_SPARSE_STATS := $(QUERY_SPECIMENS:%=stats/query_presence_sparse/%.stats)
QUERY_COUNT_DENSE_DONE := $(QUERY_SPECIMENS:%=query_count_dense/%.fasta.gz)
QUERY_COUNT_DENSE_STATS := $(QUERY_SPECIMENS:%=stats/query_count_dense/%.stats)
QUERY_COUNT_SPARSE_DONE := $(QUERY_SPECIMENS:%=query_count_sparse/%.fasta.gz)
QUERY_COUNT_SPARSE_STATS := $(QUERY_SPECIMENS:%=stats/query_count_sparse/%.stats)
VERIFY_QUERY_STATS := $(QUERY_SPECIMENS:%=stats/verify_query/%.stats)
DIST_COMPARISON := $(RUN)/stats/dist_comparison/summary.csv
PRESENCE_DONE := $(SPECIMENS:%=$(RUN)/specimen_index_presence/%/index.done)
PRESENCE_STATS := $(SPECIMENS:%=$(RUN)/stats/indexing_presence/%.stats)
COUNT_DONE := $(SPECIMENS:%=$(RUN)/specimen_index_count/%/index.done)
COUNT_STATS := $(SPECIMENS:%=$(RUN)/stats/indexing_count/%.stats)
VERIFY_PRESENCE_STATS := $(SPECIMENS:%=$(RUN)/stats/verify_presence/%.stats)
VERIFY_COUNT_STATS := $(SPECIMENS:%=$(RUN)/stats/verify_count/%.stats)
SPECIFIC_PRESENCE_DONE := $(SPECIES:%=$(RUN)/specific_index_presence/%/index.done)
SPECIFIC_PRESENCE_STATS := $(SPECIES:%=$(RUN)/stats/specific_kmer_presence/%.stats)
SPECIFIC_COUNT_DONE := $(SPECIES:%=$(RUN)/specific_index_count/%/index.done)
SPECIFIC_COUNT_STATS := $(SPECIES:%=$(RUN)/stats/specific_kmer_count/%.stats)
SIMULATED_READS := $(foreach s,$(SPECIMENS),$(RUN)/simulated_data/$(subst --,/,$s)/reads_R1.fastq.gz)
QUERY_READS := $(foreach s,$(QUERY_SPECIMENS),$(RUN)/query_data/$(subst --,/,$s)/reads_R1.fastq.gz)
QUERY_PRESENCE_DENSE_DONE := $(QUERY_SPECIMENS:%=$(RUN)/query_presence_dense/%.fasta.gz)
QUERY_PRESENCE_DENSE_STATS := $(QUERY_SPECIMENS:%=$(RUN)/stats/query_presence_dense/%.stats)
QUERY_PRESENCE_SPARSE_DONE := $(QUERY_SPECIMENS:%=$(RUN)/query_presence_sparse/%.fasta.gz)
QUERY_PRESENCE_SPARSE_STATS := $(QUERY_SPECIMENS:%=$(RUN)/stats/query_presence_sparse/%.stats)
QUERY_COUNT_DENSE_DONE := $(QUERY_SPECIMENS:%=$(RUN)/query_count_dense/%.fasta.gz)
QUERY_COUNT_DENSE_STATS := $(QUERY_SPECIMENS:%=$(RUN)/stats/query_count_dense/%.stats)
QUERY_COUNT_SPARSE_DONE := $(QUERY_SPECIMENS:%=$(RUN)/query_count_sparse/%.fasta.gz)
QUERY_COUNT_SPARSE_STATS := $(QUERY_SPECIMENS:%=$(RUN)/stats/query_count_sparse/%.stats)
VERIFY_QUERY_STATS := $(QUERY_SPECIMENS:%=$(RUN)/stats/verify_query/%.stats)
.NOTPARALLEL:
@@ -69,8 +73,8 @@ VERIFY_QUERY_STATS := $(QUERY_SPECIMENS:%=stats/verify_query/%.stats)
aggregate_query_count_dense aggregate_query_count_sparse \
verify_query aggregate_verify_query
verify_merge_presence: stats/verify_merge_presence/current.csv
verify_merge_count: stats/verify_merge_count/current.csv
verify_merge_presence: $(RUN)/stats/verify_merge_presence/current.csv
verify_merge_count: $(RUN)/stats/verify_merge_count/current.csv
all: aggregate_verify_presence aggregate_verify_count \
verify_merge_presence verify_merge_count \
@@ -104,7 +108,7 @@ simulate_query: $(QUERY_READS)
# ── reference kmer sets ───────────────────────────────────────────────────────
# Prerequisites (reads → npz) are in deps.mk.
reference_index/%.npz:
$(RUN)/reference_index/%.npz:
bash build_reference.sh $*
reference: $(REF_NPZS)
@@ -112,57 +116,58 @@ reference: $(REF_NPZS)
# ── reference distance matrices ───────────────────────────────────────────────
$(REF_DIST_CSVS) &: $(REF_NPZS) build_reference_dist.py
$(VENV_PY) build_reference_dist.py
$(VENV_PY) build_reference_dist.py \
--ref-dir $(RUN)/reference_index --out-dir $(RUN)/reference_dist
reference_dist: $(REF_DIST_CSVS)
# ── obikmer phylo (presence index) ──────────────────────────────────────────
$(OBIKMER_PRESENCE_DIST) &: global_index_presence/index.done $(BINARY)
mkdir -p obikmer_dist/presence
$(OBIKMER_PRESENCE_DIST) &: $(RUN)/global_index_presence/index.done $(BINARY)
mkdir -p $(RUN)/obikmer_dist/presence
$(BINARY) phylo \
--output obikmer_dist/presence/jaccard \
--output $(RUN)/obikmer_dist/presence/jaccard \
--distance jaccard --csv --shared-kmers --nj \
global_index_presence
$(RUN)/global_index_presence
$(BINARY) phylo \
--output obikmer_dist/presence/hamming \
--output $(RUN)/obikmer_dist/presence/hamming \
--distance hamming --csv --nj \
global_index_presence
$(RUN)/global_index_presence
obikmer_dist_presence: $(OBIKMER_PRESENCE_DIST)
# ── obikmer phylo (count index) ─────────────────────────────────────────────
$(OBIKMER_COUNT_DIST) &: global_index_count/index.done $(BINARY)
mkdir -p obikmer_dist/count
$(OBIKMER_COUNT_DIST) &: $(RUN)/global_index_count/index.done $(BINARY)
mkdir -p $(RUN)/obikmer_dist/count
$(BINARY) phylo \
--output obikmer_dist/count/jaccard \
--output $(RUN)/obikmer_dist/count/jaccard \
--distance jaccard --csv --shared-kmers --nj \
global_index_count
$(RUN)/global_index_count
$(BINARY) phylo \
--output obikmer_dist/count/bray_curtis \
--output $(RUN)/obikmer_dist/count/bray_curtis \
--distance bray-curtis --csv --nj \
global_index_count
$(RUN)/global_index_count
$(BINARY) phylo \
--output obikmer_dist/count/relfreq_bray_curtis \
--output $(RUN)/obikmer_dist/count/relfreq_bray_curtis \
--distance relfreq-bray-curtis --csv --nj \
global_index_count
$(RUN)/global_index_count
$(BINARY) phylo \
--output obikmer_dist/count/euclidean \
--output $(RUN)/obikmer_dist/count/euclidean \
--distance euclidean --csv --nj \
global_index_count
$(RUN)/global_index_count
$(BINARY) phylo \
--output obikmer_dist/count/relfreq_euclidean \
--output $(RUN)/obikmer_dist/count/relfreq_euclidean \
--distance relfreq-euclidean --csv --nj \
global_index_count
$(RUN)/global_index_count
$(BINARY) phylo \
--output obikmer_dist/count/hellinger \
--output $(RUN)/obikmer_dist/count/hellinger \
--distance hellinger --csv --nj \
global_index_count
$(RUN)/global_index_count
$(BINARY) phylo \
--output obikmer_dist/count/hellinger_euclidean \
--output $(RUN)/obikmer_dist/count/hellinger_euclidean \
--distance hellinger-euclidean --csv --nj \
global_index_count
$(RUN)/global_index_count
obikmer_dist_count: $(OBIKMER_COUNT_DIST)
@@ -171,19 +176,19 @@ obikmer_dist: obikmer_dist_presence obikmer_dist_count
# ── distance comparison ───────────────────────────────────────────────────────
$(DIST_COMPARISON): $(REF_DIST_CSVS) $(OBIKMER_PRESENCE_DIST) $(OBIKMER_COUNT_DIST) compare_all_dist.py
$(VENV_PY) compare_all_dist.py --out $(DIST_COMPARISON)
$(VENV_PY) compare_all_dist.py --run-dir $(RUN) --out $(DIST_COMPARISON)
dist_comparison: $(DIST_COMPARISON)
# ── per-specimen indexing ─────────────────────────────────────────────────────
# Prerequisites (reads → index.done + .stats) are in deps.mk.
specimen_index_presence/%/index.done \
stats/indexing_presence/%.stats &: $(BINARY)
$(RUN)/specimen_index_presence/%/index.done \
$(RUN)/stats/indexing_presence/%.stats &: $(BINARY)
bash index_one_presence.sh $*
specimen_index_count/%/index.done \
stats/indexing_count/%.stats &: $(BINARY)
$(RUN)/specimen_index_count/%/index.done \
$(RUN)/stats/indexing_count/%.stats &: $(BINARY)
bash index_one_count.sh $*
index_presence: $(PRESENCE_DONE)
@@ -199,22 +204,22 @@ aggregate_index_count: $(COUNT_STATS)
# ── global merge ──────────────────────────────────────────────────────────────
global_index_presence/index.done: $(PRESENCE_DONE) $(BINARY)
$(RUN)/global_index_presence/index.done: $(PRESENCE_DONE) $(BINARY)
bash merge_presence.sh
global_index_count/index.done: $(COUNT_DONE) $(BINARY)
$(RUN)/global_index_count/index.done: $(COUNT_DONE) $(BINARY)
bash merge_count.sh
merge_presence: global_index_presence/index.done
merge_count: global_index_count/index.done
merge_presence: $(RUN)/global_index_presence/index.done
merge_count: $(RUN)/global_index_count/index.done
# ── per-specimen verification ─────────────────────────────────────────────────
# Prerequisites (index.done + npz → .stats) are in deps.mk.
stats/verify_presence/%.stats:
$(RUN)/stats/verify_presence/%.stats:
bash verify_one_presence.sh $*
stats/verify_count/%.stats:
$(RUN)/stats/verify_count/%.stats:
bash verify_one_count.sh $*
verify_presence: $(VERIFY_PRESENCE_STATS)
@@ -231,12 +236,12 @@ aggregate_verify_count: $(VERIFY_COUNT_STATS)
# ── species-specific indexes ──────────────────────────────────────────────────
# Prerequisites (global index → specific index) are in deps.mk.
specific_index_presence/%/index.done \
stats/specific_kmer_presence/%.stats &: $(BINARY)
$(RUN)/specific_index_presence/%/index.done \
$(RUN)/stats/specific_kmer_presence/%.stats &: $(BINARY)
bash filter_one_presence.sh $*
specific_index_count/%/index.done \
stats/specific_kmer_count/%.stats &: $(BINARY)
$(RUN)/specific_index_count/%/index.done \
$(RUN)/stats/specific_kmer_count/%.stats &: $(BINARY)
bash filter_one_count.sh $*
filter_presence: $(SPECIFIC_PRESENCE_DONE)
@@ -250,10 +255,10 @@ aggregate_filter_count: $(SPECIFIC_COUNT_STATS)
# ── merged index verification ─────────────────────────────────────────────────
stats/verify_merge_presence/current.csv: $(REF_NPZS) global_index_presence/index.done
$(RUN)/stats/verify_merge_presence/current.csv: $(REF_NPZS) $(RUN)/global_index_presence/index.done
bash verify_merge_presence.sh
stats/verify_merge_count/current.csv: $(REF_NPZS) global_index_count/index.done
$(RUN)/stats/verify_merge_count/current.csv: $(REF_NPZS) $(RUN)/global_index_count/index.done
bash verify_merge_count.sh
# ── dense variants (query benchmark) ────────────────────────────────────────────
@@ -262,34 +267,34 @@ stats/verify_merge_count/current.csv: $(REF_NPZS) global_index_count/index.done
# built explicitly here, from a hard-link-based copy (see
# copy_index_hardlink.sh) rather than a full `cp -r`.
global_index_presence_dense/index.done: global_index_presence/index.done $(BINARY)
$(RUN)/global_index_presence_dense/index.done: $(RUN)/global_index_presence/index.done $(BINARY)
bash pack_dense.sh presence
# Rebuilt from the per-specimen count sources directly (via `merge --dense`),
# not repacked from global_index_count — see pack_dense.sh's own comment.
global_index_count_dense/index.done: $(COUNT_DONE) $(BINARY)
$(RUN)/global_index_count_dense/index.done: $(COUNT_DONE) $(BINARY)
bash pack_dense.sh count
pack_dense_presence: global_index_presence_dense/index.done
pack_dense_count: global_index_count_dense/index.done
pack_dense_presence: $(RUN)/global_index_presence_dense/index.done
pack_dense_count: $(RUN)/global_index_count_dense/index.done
# ── query: dense vs sparse, presence and count ──────────────────────────────────
# Prerequisites (reads + index → output + .stats) are in deps.mk.
query_presence_dense/%.fasta.gz \
stats/query_presence_dense/%.stats &: $(BINARY) global_index_presence_dense/index.done
$(RUN)/query_presence_dense/%.fasta.gz \
$(RUN)/stats/query_presence_dense/%.stats &: $(BINARY) $(RUN)/global_index_presence_dense/index.done
bash query_one.sh presence dense $*
query_presence_sparse/%.fasta.gz \
stats/query_presence_sparse/%.stats &: $(BINARY) global_index_presence/index.done
$(RUN)/query_presence_sparse/%.fasta.gz \
$(RUN)/stats/query_presence_sparse/%.stats &: $(BINARY) $(RUN)/global_index_presence/index.done
bash query_one.sh presence sparse $*
query_count_dense/%.fasta.gz \
stats/query_count_dense/%.stats &: $(BINARY) global_index_count_dense/index.done
$(RUN)/query_count_dense/%.fasta.gz \
$(RUN)/stats/query_count_dense/%.stats &: $(BINARY) $(RUN)/global_index_count_dense/index.done
bash query_one.sh count dense $*
query_count_sparse/%.fasta.gz \
stats/query_count_sparse/%.stats &: $(BINARY) global_index_count/index.done
$(RUN)/query_count_sparse/%.fasta.gz \
$(RUN)/stats/query_count_sparse/%.stats &: $(BINARY) $(RUN)/global_index_count/index.done
bash query_one.sh count sparse $*
query_presence_dense: $(QUERY_PRESENCE_DENSE_DONE)
@@ -311,7 +316,7 @@ aggregate_query_count_sparse: $(QUERY_COUNT_SPARSE_STATS)
# ── query: dense/sparse regression ──────────────────────────────────────────────
stats/verify_query/%.stats:
$(RUN)/stats/verify_query/%.stats:
bash verify_query_one.sh $*
verify_query: $(VERIFY_QUERY_STATS)