docs: clarify merge re-merge behavior and input file handling
Update documentation to explain that merged indices can serve as sources if at least one source is unmerged, and that the resulting column order follows the base index first. Clarify that explicitly named files are always read regardless of extension, with format detection based on content. - refactor: prevent I/O errors when re-merging packed outputs by restricting `choose_base` to valid destinations via new `can_be_destination` check - fix: handle zero-layer partitions in `build_index_layer` and `remove_build_artifacts` to skip unnecessary operations - feat: allow `PathIter` to treat unrecognized extensions as explicit files, ensuring they are not silently dropped during directory expansion - test: add comprehensive tests for `PathIter` including extension recognition, explicit file handling, and stdin sentinel behavior - test: update smoke tests to verify empty partitions, `.fna.gz` indexing, and refusal of merging only packed indexes - chore: remove obsolete cargo build environment variables from `run.sh` script
This commit is contained in:
1 parent
07c0a54e04
commit
7f829ea557
11 files changed
+262
-39
No files matched your search
@@ -18,12 +18,14 @@
|
||||
# 4. runs the merge scenarios:
|
||||
# s1_presence / s1_count merge A B C D
|
||||
# s3_dense merge A B C D --dense (presence)
|
||||
# Scenario 2 (merge A B -> AB, then merge AB C D: multi-genome source,
|
||||
# multi-layer destination) is NOT included: the current code cannot
|
||||
# re-merge a merge output — its matrices are packed (sparse or dense)
|
||||
# at finalize, and `MatrixBuilder::resume` expects the columnar
|
||||
# `meta.json` layout (ENOENT). Known bug, to be fixed separately; see
|
||||
# DevDocMD/implementation/merge_mmap_audit.md, "Known issue".
|
||||
# Scenario 2 (merge A B -> AB, then merge AB C D: a packed merge output
|
||||
# re-merged as a *source*) is checked relatively, not against the
|
||||
# golden dumps: its table (k-mer x genome label) must equal s1_presence's,
|
||||
# whatever the column order and layer placement. A packed index cannot
|
||||
# be a merge *destination* (its matrices are no longer column-by-column,
|
||||
# see `can_be_destination` in obikmerge); `choose_base` therefore picks
|
||||
# an unpacked source, and merging only packed indexes is refused with a
|
||||
# clear error, which this script also checks.
|
||||
# 5. dumps every result with `obikmer dump --debug` (partition + layer
|
||||
# columns included, so a k-mer moving between layers is caught), keeps
|
||||
# the CSV header first and sorts the rows (LC_ALL=C)
|
||||
@@ -147,6 +149,35 @@ run s3_dense merge --dense -o "$WORK/s3_dense.idx" \
|
||||
"$WORK/A.pres.idx" "$WORK/B.pres.idx" "$WORK/C.pres.idx" "$WORK/D.pres.idx"
|
||||
dump s3_dense "$WORK/s3_dense.idx"
|
||||
|
||||
# scenario 2: re-merge a packed merge output as a source (checked against s1)
|
||||
run s2_ab merge -o "$WORK/s2_ab.idx" "$WORK/A.pres.idx" "$WORK/B.pres.idx"
|
||||
run s2 merge -o "$WORK/s2.idx" "$WORK/s2_ab.idx" "$WORK/C.pres.idx" "$WORK/D.pres.idx"
|
||||
for N in s1_pres s2; do
|
||||
"$BIN" dump "$WORK/$N.idx" > "$WORK/plain_$N.csv" 2> "$WORK/dump_plain_$N.log" \
|
||||
|| { cat "$WORK/dump_plain_$N.log" >&2; fail "obikmer dump $N exited non-zero"; }
|
||||
done
|
||||
cat > "$WORK/compare_tables.py" <<'PY'
|
||||
import csv, sys
|
||||
def table(path):
|
||||
rows = list(csv.reader(open(path)))
|
||||
head = rows[0]
|
||||
return {r[0]: {head[i]: r[i] for i in range(1, len(head))} for r in rows[1:]}
|
||||
a, b = table(sys.argv[1]), table(sys.argv[2])
|
||||
if a != b:
|
||||
print(f"k-mers: {len(a)} vs {len(b)}; only in flat: {len(set(a) - set(b))}; "
|
||||
f"only in re-merge: {len(set(b) - set(a))}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
PY
|
||||
python3 "$WORK/compare_tables.py" "$WORK/plain_s1_pres.csv" "$WORK/plain_s2.csv" \
|
||||
|| fail "scenario 2 (re-merge of a packed output) differs from the flat merge"
|
||||
echo "ok s2 (re-merge of a packed output) equals the flat merge"
|
||||
if "$BIN" merge -o "$WORK/s2_bad.idx" "$WORK/s2_ab.idx" "$WORK/s1_pres.idx" > "$WORK/s2_bad.log" 2>&1; then
|
||||
fail "merging only packed indexes should be refused"
|
||||
fi
|
||||
grep -q "none of the sources can be the destination" "$WORK/s2_bad.log" \
|
||||
|| { cat "$WORK/s2_bad.log" >&2; fail "refusal of an all-packed merge lacks the expected message"; }
|
||||
echo "ok merge of only packed indexes is refused with a clear message"
|
||||
|
||||
# sanity: no dump may be empty (header + at least one row)
|
||||
for f in "$OUT"/*.csv; do
|
||||
[ "$(wc -l < "$f")" -gt 1 ] || fail "empty dump: $(basename "$f")"
|
||||
|
||||
Reference in new issue
Block a user