ci: improve release workflow and bump obikmer to v1.1.37

Replace inline `docker run` with explicit container lifecycle management to improve isolation and verify exit statuses. Bump `obikmer` crate version to 1.1.37. Add debug logging for sparse hit structure memory tracking, update chunk memory documentation to reflect the Findere rework, and clarify `BYTES_PER_KMER_PER_GENOME` as a pathological bound for safety tuning.

.gitea/workflows/release.yml

@@ -104,19 +104,23 @@ jobs:
 
       - name: Build macOS binary
         run: |
-          docker run --rm \
-            -v "${{ github.workspace }}:/src" \
+          CID=$(docker create \
             -w /src/src \
             registry.metabarcoding.org/cibuilder/rustcrossosx:latest \
-            cargo build --release --target aarch64-apple-darwin --no-default-features
+            cargo build --release --target aarch64-apple-darwin --no-default-features)
+          docker cp . "$CID:/src"
+          docker start -a "$CID"
+          STATUS=$(docker wait "$CID")
+          mkdir -p /tmp/dist
+          docker cp "$CID:/src/src/target/aarch64-apple-darwin/release/obikmer" /tmp/dist/obikmer-macos-arm64
+          docker rm "$CID" > /dev/null
+          [ "$STATUS" -eq 0 ]
 
       - name: Prepare and upload artifact
         env:
           GITEA_TOKEN: ${{ secrets.GITEATOKEN }}
           RELEASE_ID: ${{ needs.create-release.outputs.release_id }}
         run: |
-          mkdir -p /tmp/dist
-          cp src/target/aarch64-apple-darwin/release/obikmer /tmp/dist/obikmer-macos-arm64
           curl -s -X POST \
             "${{ github.server_url }}/api/v1/repos/${{ github.repository }}/releases/$RELEASE_ID/assets" \
             -H "Authorization: token $GITEA_TOKEN" \

src/Cargo.lock

@@ -1704,7 +1704,7 @@ dependencies = [
 
 [[package]]
 name = "obikmer"
-version = "1.1.36"
+version = "1.1.37"
 dependencies = [
  "clap",
  "csv",

src/obikmer/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "obikmer"
-version = "1.1.36"
+version = "1.1.37"
 edition = "2024"
 
 [[bin]]

src/obikmer/src/cmd/query.rs

@@ -435,6 +435,28 @@ fn process_chunk(
         "sparse Findere"
     );
 
+    // Actual bytes retained by the sparse hit structures (by_genome +
+    // confirmed_by_genome, both alive simultaneously at this point — see the
+    // chunk-size formula's comment in `run()`), by allocated capacity rather
+    // than logical length so this reflects real memory pressure including
+    // Vec growth slack. `empirical_multiplier` is directly comparable to
+    // BYTES_PER_KMER_PER_GENOME (`run()`) — the ratio a cluster run's logs
+    // need to judge whether that constant is over- or under-conservative for
+    // real data, instead of guessing.
+    const HIT_ENTRY_BYTES: u64 = std::mem::size_of::<(u32, u32, u32)>() as u64;
+    let by_genome_bytes: u64 = by_genome.iter().map(|v| v.capacity() as u64 * HIT_ENTRY_BYTES).sum();
+    let confirmed_bytes: u64 = confirmed_by_genome.iter().map(|v| v.capacity() as u64 * HIT_ENTRY_BYTES).sum();
+    let retained_bytes = by_genome_bytes + confirmed_bytes;
+
+    debug!(
+        by_genome_bytes,
+        confirmed_bytes,
+        retained_bytes,
+        chunk_bytes,
+        empirical_multiplier = retained_bytes as f64 / chunk_bytes.max(1) as f64,
+        "sparse memory retained"
+    );
+
     // ── Accumulate: genome totals (per genome, from confirmed hits) ──────────
     let mut accs: Vec<SeqAcc> = (0..n_seqs).map(|_| SeqAcc::new(n_genomes)).collect();
     let mut confirmed_any = vec![false; total_out];
@@ -542,12 +564,30 @@ pub fn run(args: QueryArgs) {
 
     // Chunk size: each chunk stays in memory for its entire processing lifetime.
     //
-    // Per-chunk memory is dominated by two dense buffers that scale with
-    // n_genomes — KmerResults.data and win_min (process_chunk) — each
-    // roughly total_kmers_in_chunk × n_genomes × 4 bytes (u32), plus `cov`
-    // (roughly) doubling that cost when --detail is set. total_kmers_in_chunk
-    // ≈ chunk_bytes (one s-mer position per raw input byte; the per-sequence
-    // k-1 tail is negligible at realistic chunk sizes).
+    // Per-chunk memory is no longer a dense n_genomes-wide buffer (removed in
+    // the sparse Findere rework, see process_chunk) — it now scales with
+    // *actual hit count*, not with total_kmers_in_chunk × n_genomes
+    // unconditionally. BYTES_PER_KMER_PER_GENOME below is therefore a
+    // pathological-case bound, not a typical-case estimate: it protects
+    // against a fully-dense hit pattern (every k-mer of the query matching
+    // every genome — a degenerate case, e.g. low-complexity input theta-
+    // filtering should mostly reject, or an index of near-duplicate genomes),
+    // where by_genome and confirmed_by_genome (process_chunk) both end up
+    // holding one (seq_idx, pos, value) entry — 3 × u32 = 12 bytes, vs. 4
+    // bytes for the old dense encoding, where position was implicit in the
+    // array index — per (k-mer, genome) pair, and *coexist simultaneously*
+    // (by_genome isn't freed before confirmed_by_genome is built), for a
+    // worst case of ~24 bytes/pair before Vec growth slack. `cov` remains
+    // fully dense when --detail is set (unaffected by the sparse rework),
+    // still roughly doubling the n_genomes-scaled cost.
+    //
+    // For realistic, sparse hit patterns actual memory is far below this
+    // bound — see the "sparse memory retained" debug log in process_chunk,
+    // which reports the empirical bytes-per-raw-byte multiplier actually
+    // observed per chunk, directly comparable to BYTES_PER_KMER_PER_GENOME
+    // below. Tightening this constant for typical-case throughput (at the
+    // cost of pathological-case safety margin) is a deliberate tuning
+    // decision to make from that data, not something to guess at here.
     //
     // BASE_OVERHEAD approximates what scales with chunk_bytes alone,
     // independent of n_genomes: the Rope itself, parsed SeqRecord sequence +
@@ -559,7 +599,7 @@ pub fn run(args: QueryArgs) {
     // We target ≤ 50 % of available RAM across all concurrent workers
     // (SAFETY_FACTOR).
     const BASE_OVERHEAD: u64 = 4;
-    const BYTES_PER_KMER_PER_GENOME: u64 = 8; // KmerResults.data + win_min, one u32 (4B) each
+    const BYTES_PER_KMER_PER_GENOME: u64 = 8; // pathological-case bound — see comment above
     const SAFETY_FACTOR: u64 = 2;
 
     let detail_factor: u64 = if args.detail { 2 } else { 1 };