feat: add batched int group stats API and expand benchmark variants
Introduces a `batch_int_group_stats` API for computing presence counts, sums, minimums, and maximums across sparse and dense matrix representations. The selection layer now utilizes this batched approach to optimize aggregation semantics for boolean and numeric operations. Additionally, reorganizes the benchmarking infrastructure to support querying across presence and count index variants in both dense and sparse formats, including new packing scripts and updated statistics aggregation.
This commit is contained in:
Eric Coissac
+68
-34
@@ -39,10 +39,14 @@ SPECIFIC_COUNT_DONE := $(SPECIES:%=specific_index_count/%/index.done)
|
||||
SPECIFIC_COUNT_STATS := $(SPECIES:%=stats/specific_kmer_count/%.stats)
|
||||
SIMULATED_READS := $(foreach s,$(SPECIMENS),simulated_data/$(subst --,/,$s)/reads_R1.fastq.gz)
|
||||
QUERY_READS := $(foreach s,$(QUERY_SPECIMENS),query_data/$(subst --,/,$s)/reads_R1.fastq.gz)
|
||||
QUERY_DENSE_DONE := $(QUERY_SPECIMENS:%=query_dense/%.fasta.gz)
|
||||
QUERY_DENSE_STATS := $(QUERY_SPECIMENS:%=stats/query_dense/%.stats)
|
||||
QUERY_SPARSE_DONE := $(QUERY_SPECIMENS:%=query_sparse/%.fasta.gz)
|
||||
QUERY_SPARSE_STATS := $(QUERY_SPECIMENS:%=stats/query_sparse/%.stats)
|
||||
QUERY_PRESENCE_DENSE_DONE := $(QUERY_SPECIMENS:%=query_presence_dense/%.fasta.gz)
|
||||
QUERY_PRESENCE_DENSE_STATS := $(QUERY_SPECIMENS:%=stats/query_presence_dense/%.stats)
|
||||
QUERY_PRESENCE_SPARSE_DONE := $(QUERY_SPECIMENS:%=query_presence_sparse/%.fasta.gz)
|
||||
QUERY_PRESENCE_SPARSE_STATS := $(QUERY_SPECIMENS:%=stats/query_presence_sparse/%.stats)
|
||||
QUERY_COUNT_DENSE_DONE := $(QUERY_SPECIMENS:%=query_count_dense/%.fasta.gz)
|
||||
QUERY_COUNT_DENSE_STATS := $(QUERY_SPECIMENS:%=stats/query_count_dense/%.stats)
|
||||
QUERY_COUNT_SPARSE_DONE := $(QUERY_SPECIMENS:%=query_count_sparse/%.fasta.gz)
|
||||
QUERY_COUNT_SPARSE_STATS := $(QUERY_SPECIMENS:%=stats/query_count_sparse/%.stats)
|
||||
VERIFY_QUERY_STATS := $(QUERY_SPECIMENS:%=stats/verify_query/%.stats)
|
||||
|
||||
.NOTPARALLEL:
|
||||
@@ -58,9 +62,11 @@ VERIFY_QUERY_STATS := $(QUERY_SPECIMENS:%=stats/verify_query/%.stats)
|
||||
verify_merge_presence verify_merge_count \
|
||||
filter_presence filter_count \
|
||||
aggregate_filter_presence aggregate_filter_count \
|
||||
pack_sparse simulate_query \
|
||||
query_dense query_sparse \
|
||||
aggregate_query_dense aggregate_query_sparse \
|
||||
pack_dense_presence pack_dense_count simulate_query \
|
||||
query_presence_dense query_presence_sparse \
|
||||
query_count_dense query_count_sparse \
|
||||
aggregate_query_presence_dense aggregate_query_presence_sparse \
|
||||
aggregate_query_count_dense aggregate_query_count_sparse \
|
||||
verify_query aggregate_verify_query
|
||||
|
||||
verify_merge_presence: stats/verify_merge_presence/current.csv
|
||||
@@ -70,7 +76,9 @@ all: aggregate_verify_presence aggregate_verify_count \
|
||||
verify_merge_presence verify_merge_count \
|
||||
aggregate_filter_presence aggregate_filter_count \
|
||||
dist_comparison \
|
||||
aggregate_query_dense aggregate_query_sparse aggregate_verify_query
|
||||
aggregate_query_presence_dense aggregate_query_presence_sparse \
|
||||
aggregate_query_count_dense aggregate_query_count_sparse \
|
||||
aggregate_verify_query
|
||||
|
||||
# ── dependency file ───────────────────────────────────────────────────────────
|
||||
|
||||
@@ -114,11 +122,11 @@ $(OBIKMER_PRESENCE_DIST) &: global_index_presence/index.done $(BINARY)
|
||||
mkdir -p obikmer_dist/presence
|
||||
$(BINARY) phylo \
|
||||
--output obikmer_dist/presence/jaccard \
|
||||
--metric jaccard --shared-kmers --nj \
|
||||
--distance jaccard --csv --shared-kmers --nj \
|
||||
global_index_presence
|
||||
$(BINARY) phylo \
|
||||
--output obikmer_dist/presence/hamming \
|
||||
--metric hamming --nj \
|
||||
--distance hamming --csv --nj \
|
||||
global_index_presence
|
||||
|
||||
obikmer_dist_presence: $(OBIKMER_PRESENCE_DIST)
|
||||
@@ -129,31 +137,31 @@ $(OBIKMER_COUNT_DIST) &: global_index_count/index.done $(BINARY)
|
||||
mkdir -p obikmer_dist/count
|
||||
$(BINARY) phylo \
|
||||
--output obikmer_dist/count/jaccard \
|
||||
--metric jaccard --shared-kmers --nj \
|
||||
--distance jaccard --csv --shared-kmers --nj \
|
||||
global_index_count
|
||||
$(BINARY) phylo \
|
||||
--output obikmer_dist/count/bray_curtis \
|
||||
--metric bray-curtis --nj \
|
||||
--distance bray-curtis --csv --nj \
|
||||
global_index_count
|
||||
$(BINARY) phylo \
|
||||
--output obikmer_dist/count/relfreq_bray_curtis \
|
||||
--metric relfreq-bray-curtis --nj \
|
||||
--distance relfreq-bray-curtis --csv --nj \
|
||||
global_index_count
|
||||
$(BINARY) phylo \
|
||||
--output obikmer_dist/count/euclidean \
|
||||
--metric euclidean --nj \
|
||||
--distance euclidean --csv --nj \
|
||||
global_index_count
|
||||
$(BINARY) phylo \
|
||||
--output obikmer_dist/count/relfreq_euclidean \
|
||||
--metric relfreq-euclidean --nj \
|
||||
--distance relfreq-euclidean --csv --nj \
|
||||
global_index_count
|
||||
$(BINARY) phylo \
|
||||
--output obikmer_dist/count/hellinger \
|
||||
--metric hellinger --nj \
|
||||
--distance hellinger --csv --nj \
|
||||
global_index_count
|
||||
$(BINARY) phylo \
|
||||
--output obikmer_dist/count/hellinger_euclidean \
|
||||
--metric hellinger-euclidean --nj \
|
||||
--distance hellinger-euclidean --csv --nj \
|
||||
global_index_count
|
||||
|
||||
obikmer_dist_count: $(OBIKMER_COUNT_DIST)
|
||||
@@ -248,32 +256,58 @@ stats/verify_merge_presence/current.csv: $(REF_NPZS) global_index_presence/index
|
||||
stats/verify_merge_count/current.csv: $(REF_NPZS) global_index_count/index.done
|
||||
bash verify_merge_count.sh
|
||||
|
||||
# ── sparse presence index (query benchmark) ─────────────────────────────────────
|
||||
# ── dense variants (query benchmark) ────────────────────────────────────────────
|
||||
# `merge` packs sparse by default (2026-08-28) — global_index_presence/
|
||||
# global_index_count *are* the sparse variants already; the dense ones are
|
||||
# built explicitly here, from a hard-link-based copy (see
|
||||
# copy_index_hardlink.sh) rather than a full `cp -r`.
|
||||
|
||||
global_index_presence_sparse/index.done: global_index_presence/index.done $(BINARY)
|
||||
bash pack_sparse.sh
|
||||
global_index_presence_dense/index.done: global_index_presence/index.done $(BINARY)
|
||||
bash pack_dense.sh presence
|
||||
|
||||
pack_sparse: global_index_presence_sparse/index.done
|
||||
# Rebuilt from the per-specimen count sources directly (via `merge --dense`),
|
||||
# not repacked from global_index_count — see pack_dense.sh's own comment.
|
||||
global_index_count_dense/index.done: $(COUNT_DONE) $(BINARY)
|
||||
bash pack_dense.sh count
|
||||
|
||||
# ── query: dense vs sparse ───────────────────────────────────────────────────────
|
||||
pack_dense_presence: global_index_presence_dense/index.done
|
||||
pack_dense_count: global_index_count_dense/index.done
|
||||
|
||||
# ── query: dense vs sparse, presence and count ──────────────────────────────────
|
||||
# Prerequisites (reads + index → output + .stats) are in deps.mk.
|
||||
|
||||
query_dense/%.fasta.gz \
|
||||
stats/query_dense/%.stats &: $(BINARY)
|
||||
bash query_one.sh dense $*
|
||||
query_presence_dense/%.fasta.gz \
|
||||
stats/query_presence_dense/%.stats &: $(BINARY) global_index_presence_dense/index.done
|
||||
bash query_one.sh presence dense $*
|
||||
|
||||
query_sparse/%.fasta.gz \
|
||||
stats/query_sparse/%.stats &: $(BINARY)
|
||||
bash query_one.sh sparse $*
|
||||
query_presence_sparse/%.fasta.gz \
|
||||
stats/query_presence_sparse/%.stats &: $(BINARY) global_index_presence/index.done
|
||||
bash query_one.sh presence sparse $*
|
||||
|
||||
query_dense: $(QUERY_DENSE_DONE)
|
||||
query_sparse: $(QUERY_SPARSE_DONE)
|
||||
query_count_dense/%.fasta.gz \
|
||||
stats/query_count_dense/%.stats &: $(BINARY) global_index_count_dense/index.done
|
||||
bash query_one.sh count dense $*
|
||||
|
||||
aggregate_query_dense: $(QUERY_DENSE_STATS)
|
||||
bash aggregate_stats.sh query_dense
|
||||
query_count_sparse/%.fasta.gz \
|
||||
stats/query_count_sparse/%.stats &: $(BINARY) global_index_count/index.done
|
||||
bash query_one.sh count sparse $*
|
||||
|
||||
aggregate_query_sparse: $(QUERY_SPARSE_STATS)
|
||||
bash aggregate_stats.sh query_sparse
|
||||
query_presence_dense: $(QUERY_PRESENCE_DENSE_DONE)
|
||||
query_presence_sparse: $(QUERY_PRESENCE_SPARSE_DONE)
|
||||
query_count_dense: $(QUERY_COUNT_DENSE_DONE)
|
||||
query_count_sparse: $(QUERY_COUNT_SPARSE_DONE)
|
||||
|
||||
aggregate_query_presence_dense: $(QUERY_PRESENCE_DENSE_STATS)
|
||||
bash aggregate_stats.sh query_presence_dense
|
||||
|
||||
aggregate_query_presence_sparse: $(QUERY_PRESENCE_SPARSE_STATS)
|
||||
bash aggregate_stats.sh query_presence_sparse
|
||||
|
||||
aggregate_query_count_dense: $(QUERY_COUNT_DENSE_STATS)
|
||||
bash aggregate_stats.sh query_count_dense
|
||||
|
||||
aggregate_query_count_sparse: $(QUERY_COUNT_SPARSE_STATS)
|
||||
bash aggregate_stats.sh query_count_sparse
|
||||
|
||||
# ── query: dense/sparse regression ──────────────────────────────────────────────
|
||||
|
||||
|
||||
Reference in New Issue
Block a user