diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml index 5fb4684..9951ae9 100644 --- a/.gitea/workflows/release.yml +++ b/.gitea/workflows/release.yml @@ -104,19 +104,23 @@ jobs: - name: Build macOS binary run: | - docker run --rm \ - -v "${{ github.workspace }}:/src" \ + CID=$(docker create \ -w /src/src \ registry.metabarcoding.org/cibuilder/rustcrossosx:latest \ - cargo build --release --target aarch64-apple-darwin --no-default-features + cargo build --release --target aarch64-apple-darwin --no-default-features) + docker cp . "$CID:/src" + docker start -a "$CID" + STATUS=$(docker wait "$CID") + mkdir -p /tmp/dist + docker cp "$CID:/src/src/target/aarch64-apple-darwin/release/obikmer" /tmp/dist/obikmer-macos-arm64 + docker rm "$CID" > /dev/null + [ "$STATUS" -eq 0 ] - name: Prepare and upload artifact env: GITEA_TOKEN: ${{ secrets.GITEATOKEN }} RELEASE_ID: ${{ needs.create-release.outputs.release_id }} run: | - mkdir -p /tmp/dist - cp src/target/aarch64-apple-darwin/release/obikmer /tmp/dist/obikmer-macos-arm64 curl -s -X POST \ "${{ github.server_url }}/api/v1/repos/${{ github.repository }}/releases/$RELEASE_ID/assets" \ -H "Authorization: token $GITEA_TOKEN" \ diff --git a/src/Cargo.lock b/src/Cargo.lock index da8654f..28e2eea 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -1704,7 +1704,7 @@ dependencies = [ [[package]] name = "obikmer" -version = "1.1.36" +version = "1.1.37" dependencies = [ "clap", "csv", diff --git a/src/obikmer/Cargo.toml b/src/obikmer/Cargo.toml index c5a3208..ef80137 100644 --- a/src/obikmer/Cargo.toml +++ b/src/obikmer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "obikmer" -version = "1.1.36" +version = "1.1.37" edition = "2024" [[bin]] diff --git a/src/obikmer/src/cmd/query.rs b/src/obikmer/src/cmd/query.rs index 52c905e..a5ab5e7 100644 --- a/src/obikmer/src/cmd/query.rs +++ b/src/obikmer/src/cmd/query.rs @@ -435,6 +435,28 @@ fn process_chunk( "sparse Findere" ); + // Actual bytes retained by the sparse hit structures (by_genome + + // confirmed_by_genome, both alive simultaneously at this point — see the + // chunk-size formula's comment in `run()`), by allocated capacity rather + // than logical length so this reflects real memory pressure including + // Vec growth slack. `empirical_multiplier` is directly comparable to + // BYTES_PER_KMER_PER_GENOME (`run()`) — the ratio a cluster run's logs + // need to judge whether that constant is over- or under-conservative for + // real data, instead of guessing. + const HIT_ENTRY_BYTES: u64 = std::mem::size_of::<(u32, u32, u32)>() as u64; + let by_genome_bytes: u64 = by_genome.iter().map(|v| v.capacity() as u64 * HIT_ENTRY_BYTES).sum(); + let confirmed_bytes: u64 = confirmed_by_genome.iter().map(|v| v.capacity() as u64 * HIT_ENTRY_BYTES).sum(); + let retained_bytes = by_genome_bytes + confirmed_bytes; + + debug!( + by_genome_bytes, + confirmed_bytes, + retained_bytes, + chunk_bytes, + empirical_multiplier = retained_bytes as f64 / chunk_bytes.max(1) as f64, + "sparse memory retained" + ); + // ── Accumulate: genome totals (per genome, from confirmed hits) ────────── let mut accs: Vec = (0..n_seqs).map(|_| SeqAcc::new(n_genomes)).collect(); let mut confirmed_any = vec![false; total_out]; @@ -542,12 +564,30 @@ pub fn run(args: QueryArgs) { // Chunk size: each chunk stays in memory for its entire processing lifetime. // - // Per-chunk memory is dominated by two dense buffers that scale with - // n_genomes — KmerResults.data and win_min (process_chunk) — each - // roughly total_kmers_in_chunk × n_genomes × 4 bytes (u32), plus `cov` - // (roughly) doubling that cost when --detail is set. total_kmers_in_chunk - // ≈ chunk_bytes (one s-mer position per raw input byte; the per-sequence - // k-1 tail is negligible at realistic chunk sizes). + // Per-chunk memory is no longer a dense n_genomes-wide buffer (removed in + // the sparse Findere rework, see process_chunk) — it now scales with + // *actual hit count*, not with total_kmers_in_chunk × n_genomes + // unconditionally. BYTES_PER_KMER_PER_GENOME below is therefore a + // pathological-case bound, not a typical-case estimate: it protects + // against a fully-dense hit pattern (every k-mer of the query matching + // every genome — a degenerate case, e.g. low-complexity input theta- + // filtering should mostly reject, or an index of near-duplicate genomes), + // where by_genome and confirmed_by_genome (process_chunk) both end up + // holding one (seq_idx, pos, value) entry — 3 × u32 = 12 bytes, vs. 4 + // bytes for the old dense encoding, where position was implicit in the + // array index — per (k-mer, genome) pair, and *coexist simultaneously* + // (by_genome isn't freed before confirmed_by_genome is built), for a + // worst case of ~24 bytes/pair before Vec growth slack. `cov` remains + // fully dense when --detail is set (unaffected by the sparse rework), + // still roughly doubling the n_genomes-scaled cost. + // + // For realistic, sparse hit patterns actual memory is far below this + // bound — see the "sparse memory retained" debug log in process_chunk, + // which reports the empirical bytes-per-raw-byte multiplier actually + // observed per chunk, directly comparable to BYTES_PER_KMER_PER_GENOME + // below. Tightening this constant for typical-case throughput (at the + // cost of pathological-case safety margin) is a deliberate tuning + // decision to make from that data, not something to guess at here. // // BASE_OVERHEAD approximates what scales with chunk_bytes alone, // independent of n_genomes: the Rope itself, parsed SeqRecord sequence + @@ -559,7 +599,7 @@ pub fn run(args: QueryArgs) { // We target ≤ 50 % of available RAM across all concurrent workers // (SAFETY_FACTOR). const BASE_OVERHEAD: u64 = 4; - const BYTES_PER_KMER_PER_GENOME: u64 = 8; // KmerResults.data + win_min, one u32 (4B) each + const BYTES_PER_KMER_PER_GENOME: u64 = 8; // pathological-case bound — see comment above const SAFETY_FACTOR: u64 = 2; let detail_factor: u64 = if args.detail { 2 } else { 1 };