docs: update sparse packing strategy and benchmark scripts
Update documentation to reflect that sparse packing now maps all columns of a layer at once and packs layers sequentially, reversing the previous choice to avoid materializing a packed copy. This change introduces an extra write/read cycle for the dense matrix, increasing pack benchmark time from 1.11s to 1.60s. - Modify `pack_sparse_bit_matrix` in `src/obicompactvec/src/bitmatrix/sparse.rs` to always convert via the dense packed format, simplifying logic by removing conditional handling of existing files and ensuring a single mapping. - Update `pack_sparse_compact_int_matrix` in `src/obicompactvec/src/sparse_intmatrix.rs` to exclusively handle conversion from the dense packed format (`matrix.pcmx`) to sparse, removing columnar format handling and simplifying file cleanup. - Adjust `scripts/bench_merge_oracle.sh` to extract `PACK_WALL` metrics via `stage_field` for merge operations, updating grep patterns and output messages to report pack wall time. - Revise `DevDocMD/implementation/merge_mmap_audit.md` and `merge_scaling.md` to document the new sequential packing strategy, noting that `N_part` scaling applies only to incremental merges and that `obisys::MemoryBudget` is no longer used by merge. - Bump package version in `src/obikmer/Cargo.toml` from 1.4.3 to 1.4.4.
This commit is contained in:
1 parent
1f199c69db
commit
07bc5fc634
7 files changed
+47
-62
No files matched your search
@@ -21,7 +21,7 @@
|
||||
# difference there is reported, not judged)
|
||||
# --check DIR: merges into a scratch directory, compares the hash with
|
||||
# DIR/dump.sha256 (full diff against the zstd copy only on mismatch),
|
||||
# and prints merge_partitions wall/RSS before → after
|
||||
# and prints merge_partitions wall/RSS and pack wall before → after
|
||||
#
|
||||
# Usage:
|
||||
# scripts/bench_merge_oracle.sh (--record DIR | --check DIR) [--keep]
|
||||
@@ -81,9 +81,9 @@ sorted_dump() { # sorted_dump INDEX OUT_TXT — header first, rows sorted
|
||||
|
||||
hash_of() { sha256sum "$1" | cut -d' ' -f1; }
|
||||
|
||||
stage_field() { # stage_field LOG FIELD — FIELD of the 'done stage=merge_partitions' line
|
||||
stage_field() { # stage_field LOG FIELD [STAGE] — FIELD of the 'done stage=STAGE' line (default merge_partitions)
|
||||
perl -pe 's/\e\[[0-9;]*m//g' "$1" \
|
||||
| grep 'done stage=merge_partitions' \
|
||||
| grep "done stage=${3:-merge_partitions} " \
|
||||
| sed -n "s/.*$2=\([^ ]*\( [KMG]*B\)\{0,1\}\).*/\1/p" | tail -n 1
|
||||
}
|
||||
|
||||
@@ -107,7 +107,8 @@ if ! "$BIN" merge --output "$MERGED" --force --force-presence "${SOURCES[@]}" \
|
||||
fi
|
||||
WALL="$(stage_field "$LOG" wall_secs)"
|
||||
RSS="$(stage_field "$LOG" rss)"
|
||||
echo "merge_partitions: wall=${WALL}s rss=${RSS}"
|
||||
PACK_WALL="$(stage_field "$LOG" wall_secs pack)"
|
||||
echo "merge_partitions: wall=${WALL}s rss=${RSS} pack: wall=${PACK_WALL}s"
|
||||
|
||||
# ── 3. sorted dump + hash ───────────────────────────────────────────────────
|
||||
DUMP="$WORK/dump.sorted.csv"
|
||||
@@ -140,7 +141,9 @@ fi
|
||||
REF_H="$(cat "$DIR/dump.sha256")"
|
||||
REF_WALL="$(stage_field "$DIR/merge.stderr.log" wall_secs)"
|
||||
REF_RSS="$(stage_field "$DIR/merge.stderr.log" rss)"
|
||||
REF_PACK="$(stage_field "$DIR/merge.stderr.log" wall_secs pack)"
|
||||
echo "merge_partitions wall: ${REF_WALL}s → ${WALL}s rss: ${REF_RSS} → ${RSS}"
|
||||
echo "pack wall: ${REF_PACK}s → ${PACK_WALL}s"
|
||||
python3 -c "import sys; r,n=float(sys.argv[1]),float(sys.argv[2]); print(f'merge_partitions wall ratio: {n/r:.2f}' + (' (> 1.20: SLOWER THAN 20 %)' if n > 1.2*r else ''))" \
|
||||
"$REF_WALL" "$WALL" || true
|
||||
|
||||
|
||||
Reference in new issue
Block a user