From 7f829ea557f09c02279e20d4642c0635798815a6 Mon Sep 17 00:00:00 2001 From: Eric Coissac Date: Wed, 18 Nov 2026 12:39:00 +0100 Subject: [PATCH] docs: clarify merge re-merge behavior and input file handling Update documentation to explain that merged indices can serve as sources if at least one source is unmerged, and that the resulting column order follows the base index first. Clarify that explicitly named files are always read regardless of extension, with format detection based on content. - refactor: prevent I/O errors when re-merging packed outputs by restricting `choose_base` to valid destinations via new `can_be_destination` check - fix: handle zero-layer partitions in `build_index_layer` and `remove_build_artifacts` to skip unnecessary operations - feat: allow `PathIter` to treat unrecognized extensions as explicit files, ensuring they are not silently dropped during directory expansion - test: add comprehensive tests for `PathIter` including extension recognition, explicit file handling, and stdin sentinel behavior - test: update smoke tests to verify empty partitions, `.fna.gz` indexing, and refusal of merging only packed indexes - chore: remove obsolete cargo build environment variables from `run.sh` script --- DevDocMD/implementation/merge_mmap_audit.md | 10 ++- UserDocMD/usage/index_command.md | 8 +++ UserDocMD/usage/merge.md | 8 ++- scripts/docker_debug/run.sh | 1 - scripts/smoke_test_index.sh | 24 ++++++- scripts/smoke_test_merge.sh | 43 ++++++++++-- src/Cargo.lock | 16 ++--- .../src/extensions/private_builder.rs | 10 +++ src/obikmerge/src/merge.rs | 58 +++++++++++++++- src/obiread/src/path_iterator.rs | 54 ++++++++++----- src/obiread/src/tests/path_iterator.rs | 69 +++++++++++++++++++ 11 files changed, 262 insertions(+), 39 deletions(-) create mode 100644 src/obiread/src/tests/path_iterator.rs diff --git a/DevDocMD/implementation/merge_mmap_audit.md b/DevDocMD/implementation/merge_mmap_audit.md index c78a9d72..92053f82 100644 --- a/DevDocMD/implementation/merge_mmap_audit.md +++ b/DevDocMD/implementation/merge_mmap_audit.md @@ -83,8 +83,14 @@ B would take the 8 down to ~2. **Follow-up — sparse pack (same day).** `pack_sparse_bit_matrix`/`pack_sparse_compact_int_matrix` used to open the columnar matrix directly, which maps every column file of a layer at once: `n_genomes` maps per partition, since layers are packed one after another. Both now call the dense pack first. It concatenates the column files one at a time without mapping them, so the sparse matrix is built from a single `matrix.pbmx`/`.pcmx` mapping. This deliberately reverses an earlier choice ("never materialise a packed copy just to read it back once"). It costs one extra write and read of the dense matrix: `pack` went from 1.11 s to 1.60 s on the benchmark (single run). As a side effect, a stale `matrix.pbmx` that predates newer columnar data is now re-packed instead of being used as is (`pack_bit_matrix` compares `n_cols`). Both oracles pass. -## Known issue (found while building the oracle, not fixed) +## Re-merging a merge output (resolved 2026-11-18) -Re-merging a merge output fails with `I/O error: No such file or directory` in `merge_partitions`, in every mode (presence sparse, presence `--dense`, counts). Cause, verified: `finalize_indexed` packs the output matrices (`sparse_meta.json` + dict/PFIV files, or `matrix.pbmx`), so `presence/`/`counts/` no longer holds `meta.json` + `col_*`. When that index is the merge base, bootstrap copies the packed layout and `MatrixBuilder::resume` → `MatrixMeta::load` finds no `meta.json`. Pre-existing, independent of this refactor; deferred to a separate fix (decision 2026-11-17). The synthetic oracle (`scripts/smoke_test_merge.sh`) therefore omits the "merge AB, then AB C D" scenario. +Symptom: `I/O error: No such file or directory` in `merge_partitions` when a merge output was one of the sources. Cause (verified): `finalize_indexed` packs the output matrices (`sparse_meta.json` + dict/PFIV files, or `matrix.pbmx`), so `presence/`/`counts/` no longer holds `meta.json` + `col_*`. `choose_base` picks the largest source as destination, i.e. the merged one; bootstrap copies the packed layout and `MatrixBuilder::resume` → `MatrixMeta::load` finds no `meta.json`. + +Fix: a packed index can only be a *source* (pass 2 reads it through `PersistentBitMatrix`/`PersistentIntMatrix::open`, which handle every layout). `choose_base` now filters candidates with `can_be_destination` (`obikmerge/src/merge.rs`): one layer is probed with `detect_storage`; `Columnar` qualifies, `Implicit` only for a single genome (materialized on the fly), `Packed`/`Sparse` never. With a single source there is nothing to append and the filter is skipped. When several sources are given and none qualifies, `run` returns an explicit error before any work. Consequence: the result's column order follows the chosen base first, so it can differ from the command-line order. + +Not done: in-place unpack of a packed index to allow it as destination, or a destination-side append on the packed format — the sparse incremental strategy is tracked in `merge_scaling.md`. + +Oracle: `scripts/smoke_test_merge.sh` scenario 2 (A B → AB, then AB C D) is compared against the flat merge by genome label (not against the golden dumps, whose layer placement differs), and merging only packed indexes is checked to be refused. Residual risk, accepted: read-only maps still SIGBUS if a file is truncated by another process during a merge. That is inherent to `mmap`. diff --git a/UserDocMD/usage/index_command.md b/UserDocMD/usage/index_command.md index 5c1861a7..93519074 100644 --- a/UserDocMD/usage/index_command.md +++ b/UserDocMD/usage/index_command.md @@ -12,6 +12,14 @@ obikmer index -o OUTPUT [OPTIONS] [INPUTS...] |---|---| | `INPUTS...` | Input sequence files or directories (FASTA/FASTQ/GenBank, gzip optional). If omitted, reads from stdin. | +## Which files are read + +A file named explicitly on the command line is always read, whatever its extension: the format (FASTA, FASTQ or GenBank) and the compression (gzip, bzip2, xz or zstd) are detected from the content. If the content is not a sequence file, the command reports an error rather than skipping the file. + +When an input is a directory, the files inside it are selected by extension: `.fasta`, `.fa`, `.fna` (the extension of NCBI genome downloads), `.ffn`, `.fas`, `.fsa`, `.fastq`, `.fq`, `.gbff`, `.gbk` and `.gb`, each optionally followed by `.gz`. + +A small input spread over many partitions leaves most partitions empty. This is valid: the index is built normally and empty partitions simply hold no k-mers. + ## Options | Option | Default | Description | diff --git a/UserDocMD/usage/merge.md b/UserDocMD/usage/merge.md index 58199481..9fcaae2b 100644 --- a/UserDocMD/usage/merge.md +++ b/UserDocMD/usage/merge.md @@ -36,6 +36,10 @@ The merge works partition by partition, and processes several partitions in para Each genome column being filled is held in memory and written to disk in one go once it is complete. A full disk or an exhausted quota is therefore reported as an ordinary error rather than crashing the program. The largest memory cost comes from the k-mers that are new to the merged index, which are assembled in memory, partition by partition, before being written out. -## Current limitation +## Merging an index that was already merged -An index produced by `merge` cannot itself be used as a source of a later merge yet: the command stops with a "No such file or directory" error. To add genomes to a collection, merge the original single-genome indexes again, together with the new ones. +An index produced by `merge` can be used as one of the sources of a later merge, which is how a collection is extended without starting again from the original genomes. There is one rule: among the sources, at least one must not have been through a merge. A merged index is stored in a compact packed format (see [pack](pack.md)) that is read very efficiently but cannot be extended in place. The merge therefore takes a freshly built index as its starting point and brings the packed index in as a source. A typical case is a merged index of many genomes plus one or a few newly built single-genome indexes, and this works. + +If every source has already been merged or packed, the command stops before doing any work and says so. In that case, merge the original single-genome indexes together instead. + +The genome columns of the result follow the order of the starting index first, then the other sources, so they may not come out in the order the sources were listed. The genome labels always identify each column unambiguously. diff --git a/scripts/docker_debug/run.sh b/scripts/docker_debug/run.sh index 22fa37ea..e5b255cc 100755 --- a/scripts/docker_debug/run.sh +++ b/scripts/docker_debug/run.sh @@ -41,7 +41,6 @@ docker run --rm --privileged --platform "$PLATFORM" \ -e CARGO_TARGET_DIR=/target -e CARGO_PROFILE_RELEASE_DEBUG=1 \ -e GEN="${GEN:-}" -e LOCKED_FLAG="$([ "${LOCKED:-1}" = 0 ] || echo --locked)" -e SEED="${SEED:-20261118}" -e REAL="${REAL:-}" -e GENOME_LEN="${GENOME_LEN:-150000}" -e K="${K:-21}" -e M="${M:-9}" -e PART="${PART:-256}" -e RUST_LOG="${RUST_LOG:-}" -e NSRC="$NSRC" -e TARGET="$TARGET" -e TOOL="$TOOL" -e MAXMAP="${MAXMAP:-}" \ -e MALLOC_ARENA_MAX="${MALLOC_ARENA_MAX:-}" \ - -e CXXFLAGS="${CXXFLAGS:-}" \ -e MALLOC_MMAP_THRESHOLD_="${MALLOC_MMAP_THRESHOLD_:-}" \ -e MALLOC_TRIM_THRESHOLD_="${MALLOC_TRIM_THRESHOLD_:-}" \ "$IMG" bash -c ' diff --git a/scripts/smoke_test_index.sh b/scripts/smoke_test_index.sh index 9eb47ca3..cc98c964 100755 --- a/scripts/smoke_test_index.sh +++ b/scripts/smoke_test_index.sh @@ -125,4 +125,26 @@ if [ "$FOUND" -ne 1 ]; then fail "no candidate k-mer round-tripped (tried ${#CANDIDATES[@]}) — likely a real regression, not a low-complexity fixture" fi -echo "PASS: index+query round-trip OK — $N_KMERS kmers indexed, query k-mer '$QUERY_KMER' found" +# ── 5. regressions: (a) many partitions against a tiny input leaves most of +# them empty (no layer 0 on disk) — must still build; (b) an NCBI-style +# `.fna.gz` name must be read, not silently skipped (it used to be +# taken for a directory and ignored, yielding an empty index). +count_kmers() { # $1 = output index dir, rest = obikmer index args + local out="$1"; shift + local log="$WORK/regress.log" + ( cd "$REPO_ROOT/src" && cargo run -q -p obikmer --bin obikmer -- \ + index -k "$K" -m "$M" --theta 0 -o "$out" "$@" ) > "$log" 2>&1 \ + || { cat "$log" >&2; fail "obikmer index $* exited non-zero"; } + grep -o '[0-9]* total kmers indexed' "$log" | grep -o '^[0-9]*' || echo 0 +} + +N_EMPTY="$(count_kmers "$WORK/empty_parts.idx" -p 256 "$WORK/test.fasta")" +[ "$N_EMPTY" = "$N_KMERS" ] \ + || fail "-p 256 (mostly empty partitions) indexed $N_EMPTY kmers, expected $N_KMERS" + +gzip -c "$WORK/test.fasta" > "$WORK/test.fna.gz" +N_GZ="$(count_kmers "$WORK/gz.idx" -p "$PARTITIONS" "$WORK/test.fna.gz")" +[ "$N_GZ" = "$N_KMERS" ] \ + || fail ".fna.gz input indexed $N_GZ kmers, expected $N_KMERS (input skipped?)" + +echo "PASS: index+query round-trip OK — $N_KMERS kmers indexed, query k-mer '$QUERY_KMER' found; empty-partition and .fna.gz regressions OK" diff --git a/scripts/smoke_test_merge.sh b/scripts/smoke_test_merge.sh index 49192f1f..3546555f 100755 --- a/scripts/smoke_test_merge.sh +++ b/scripts/smoke_test_merge.sh @@ -18,12 +18,14 @@ # 4. runs the merge scenarios: # s1_presence / s1_count merge A B C D # s3_dense merge A B C D --dense (presence) -# Scenario 2 (merge A B -> AB, then merge AB C D: multi-genome source, -# multi-layer destination) is NOT included: the current code cannot -# re-merge a merge output — its matrices are packed (sparse or dense) -# at finalize, and `MatrixBuilder::resume` expects the columnar -# `meta.json` layout (ENOENT). Known bug, to be fixed separately; see -# DevDocMD/implementation/merge_mmap_audit.md, "Known issue". +# Scenario 2 (merge A B -> AB, then merge AB C D: a packed merge output +# re-merged as a *source*) is checked relatively, not against the +# golden dumps: its table (k-mer x genome label) must equal s1_presence's, +# whatever the column order and layer placement. A packed index cannot +# be a merge *destination* (its matrices are no longer column-by-column, +# see `can_be_destination` in obikmerge); `choose_base` therefore picks +# an unpacked source, and merging only packed indexes is refused with a +# clear error, which this script also checks. # 5. dumps every result with `obikmer dump --debug` (partition + layer # columns included, so a k-mer moving between layers is caught), keeps # the CSV header first and sorts the rows (LC_ALL=C) @@ -147,6 +149,35 @@ run s3_dense merge --dense -o "$WORK/s3_dense.idx" \ "$WORK/A.pres.idx" "$WORK/B.pres.idx" "$WORK/C.pres.idx" "$WORK/D.pres.idx" dump s3_dense "$WORK/s3_dense.idx" +# scenario 2: re-merge a packed merge output as a source (checked against s1) +run s2_ab merge -o "$WORK/s2_ab.idx" "$WORK/A.pres.idx" "$WORK/B.pres.idx" +run s2 merge -o "$WORK/s2.idx" "$WORK/s2_ab.idx" "$WORK/C.pres.idx" "$WORK/D.pres.idx" +for N in s1_pres s2; do + "$BIN" dump "$WORK/$N.idx" > "$WORK/plain_$N.csv" 2> "$WORK/dump_plain_$N.log" \ + || { cat "$WORK/dump_plain_$N.log" >&2; fail "obikmer dump $N exited non-zero"; } +done +cat > "$WORK/compare_tables.py" <<'PY' +import csv, sys +def table(path): + rows = list(csv.reader(open(path))) + head = rows[0] + return {r[0]: {head[i]: r[i] for i in range(1, len(head))} for r in rows[1:]} +a, b = table(sys.argv[1]), table(sys.argv[2]) +if a != b: + print(f"k-mers: {len(a)} vs {len(b)}; only in flat: {len(set(a) - set(b))}; " + f"only in re-merge: {len(set(b) - set(a))}", file=sys.stderr) + sys.exit(1) +PY +python3 "$WORK/compare_tables.py" "$WORK/plain_s1_pres.csv" "$WORK/plain_s2.csv" \ + || fail "scenario 2 (re-merge of a packed output) differs from the flat merge" +echo "ok s2 (re-merge of a packed output) equals the flat merge" +if "$BIN" merge -o "$WORK/s2_bad.idx" "$WORK/s2_ab.idx" "$WORK/s1_pres.idx" > "$WORK/s2_bad.log" 2>&1; then + fail "merging only packed indexes should be refused" +fi +grep -q "none of the sources can be the destination" "$WORK/s2_bad.log" \ + || { cat "$WORK/s2_bad.log" >&2; fail "refusal of an all-packed merge lacks the expected message"; } +echo "ok merge of only packed indexes is refused with a clear message" + # sanity: no dump may be empty (header + at least one row) for f in "$OUT"/*.csv; do [ "$(wc -l < "$f")" -gt 1 ] || fail "empty dump: $(basename "$f")" diff --git a/src/Cargo.lock b/src/Cargo.lock index 030c244c..a2c163ad 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -422,7 +422,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -758,7 +758,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1040,7 +1040,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1975,9 +1975,9 @@ dependencies = [ [[package]] name = "ptr_hash" -version = "2.1.2" +version = "2.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b22cf1be774892680e056d3387db0dcaa4fd46964b270671ef661e127186b969" +checksum = "0d2381547e2e36c72a455c1cc418d5c60ecd6e94f4819d7560456108e99a4efa" dependencies = [ "bitvec", "cacheline-ef", @@ -2269,7 +2269,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2601,7 +2601,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3006,7 +3006,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] diff --git a/src/obikindexer/src/extensions/private_builder.rs b/src/obikindexer/src/extensions/private_builder.rs index 0d655cbc..89199c37 100644 --- a/src/obikindexer/src/extensions/private_builder.rs +++ b/src/obikindexer/src/extensions/private_builder.rs @@ -135,6 +135,12 @@ impl PrivateBuilder for KmerIndex { mode: &IndexMode, block_bits: u8, ) -> Result { + // A partition that received no k-mer has no layer 0 on disk at all + // (small inputs against many partitions): nothing to build, not an + // error. + if self.partition(i).map_err(|e| io::Error::other(e.to_string()))?.n_layers() == 0 { + return Ok(0); + } let layer0 = self .layer0(i) .map_err(|e| io::Error::other(e.to_string()))?; @@ -208,6 +214,10 @@ impl PrivateBuilder for KmerIndex { } fn remove_build_artifacts(&self, i: usize) { + // No layer 0 = empty partition: no build artifacts to remove. + if self.partition(i).map_or(false, |p| p.n_layers() == 0) { + return; + } let layer0 = match self.layer0(i) { Ok(layer) => layer, Err(e) => { diff --git a/src/obikmerge/src/merge.rs b/src/obikmerge/src/merge.rs index 62ff8bef..523a4235 100644 --- a/src/obikmerge/src/merge.rs +++ b/src/obikmerge/src/merge.rs @@ -3,6 +3,7 @@ use std::fs; use std::io; use std::path::{Path, PathBuf}; +use obicompactvec::{PersistentBitMatrix, PersistentIntMatrix, StorageKind}; use obikalgorithm::Algorithm; use obikindex::layer::IndexMode; use obikindex::partition::PARTITIONS_SUBDIR; @@ -176,7 +177,17 @@ impl Algorithm for Merge<'_> { ); } - let base_idx = choose_base(sources, &src_genomes, self.mode); + let Some(base_idx) = choose_base(sources, &src_genomes, self.mode) else { + return Err( + "merge: none of the sources can be the destination. Every candidate has its \ + matrices packed (finalized index): a packed index can be merged in as a \ + source, but at least one source must be unpacked — typically a freshly \ + built mono-genome index. To extend a packed index, merge it with new \ + mono-genome indexes; to combine several packed indexes, merge their \ + original sources instead." + .into(), + ); + }; let needs_approx = sources.iter().enumerate().any(|(i, src)| { !is_trivial(&src_genomes[i], self.mode) && matches!( @@ -490,7 +501,15 @@ fn index_unitig_size(src: &KmerIndex) -> u64 { (0..n).map(|i| partition_unitig_bytes(src, i)).sum() } -fn choose_base(sources: &[&KmerIndex], src_genomes: &[Vec], mode: MergeMode) -> usize { +/// The source whose partition data is copied as the merge destination: the +/// largest one (least data to merge in). `None` when several sources are +/// given and none of them can be resumed as a destination (see +/// [`can_be_destination`]). +fn choose_base( + sources: &[&KmerIndex], + src_genomes: &[Vec], + mode: MergeMode, +) -> Option { let needs_approx = sources.iter().enumerate().any(|(i, src)| { !is_trivial(&src_genomes[i], mode) && matches!( @@ -509,9 +528,42 @@ fn choose_base(sources: &[&KmerIndex], src_genomes: &[Vec], mode: Me IndexMode::Approx { .. } | IndexMode::Hybrid { .. } ) }) + // With a single source there is nothing to append: the bootstrap + // copy is the whole merge, whatever its matrix format. + .filter(|(i, src)| sources.len() == 1 || can_be_destination(src, &src_genomes[*i], mode)) .max_by_key(|(_, src)| index_unitig_size(src)) .map(|(i, _)| i) - .unwrap() +} + +/// Whether `index` can be the destination of a merge: pass 2 appends the +/// other sources' columns to its matrices (`MatrixBuilder::resume`), which +/// needs them in the column-by-column format. A finalized index has its +/// matrices packed (sparse by default, or dense with `--dense`) into +/// single-file formats that cannot be extended — such an index can only +/// come in as a *source*. An implicit matrix (mono-genome presence) is +/// materialized on the fly, so it qualifies only for a single genome. +/// +/// One layer is probed: `pack` rewrites every partition the same way. An +/// index with no k-mer at all has nothing to resume and qualifies. +fn can_be_destination(index: &KmerIndex, genomes: &[GenomeInfo], mode: MergeMode) -> bool { + let layer_dir = (0..index.n_partitions()).find_map(|i| { + let partition = index.partition(i).ok()?; + (partition.n_layers() > 0) + .then(|| partition.layer_dir(0).ok()) + .flatten() + }); + let Some(layer_dir) = layer_dir else { + return true; + }; + let kind = match mode { + MergeMode::Presence => PersistentBitMatrix::detect_storage(&layer_dir), + MergeMode::Count => PersistentIntMatrix::detect_storage(&layer_dir), + }; + match kind { + Ok(StorageKind::Columnar) => true, + Ok(StorageKind::Implicit) => genomes.len() == 1, + _ => false, + } } /// Copies `src` (the base source's whole `partitions/` tree) into `dst` at diff --git a/src/obiread/src/path_iterator.rs b/src/obiread/src/path_iterator.rs index 815f22c9..4ead8e23 100644 --- a/src/obiread/src/path_iterator.rs +++ b/src/obiread/src/path_iterator.rs @@ -1,3 +1,4 @@ +use std::collections::HashSet; use std::fs; use std::path::{Path, PathBuf}; @@ -5,6 +6,10 @@ use std::path::{Path, PathBuf}; pub struct PathIter { dir_stack: Vec, // dossiers qu'il reste à explorer file_buffer: Vec, // fichiers déjà listés (à distribuer) + /// Paths given by the caller whose extension is not a recognised sequence + /// one. They are tried as directories first; when that fails they are + /// handed back as files (see [`PathIter::next`]) rather than dropped. + explicit: HashSet, } impl PathIter { @@ -12,11 +17,18 @@ impl PathIter { /// /// For non-directory paths only the file extension is checked (no stat syscall). /// Directories are left on the stack and expanded lazily in `next()`. - /// Invalid or missing paths are silently skipped when `open_chunks` fails later. + /// + /// A path without a recognised extension is tried as a directory; if it + /// is not one, it is yielded as a file so the reader reports the real + /// problem (missing file, not a FASTA/FASTQ stream) instead of the path + /// being skipped silently. The extension filter only matters when + /// expanding directories: format and compression are detected from the + /// content. pub fn new(paths: Vec) -> Self { let mut iter = PathIter { dir_stack: Vec::new(), file_buffer: Vec::new(), + explicit: HashSet::new(), }; for path in paths { // "-" is the stdin sentinel — pass it through without any extension @@ -28,6 +40,7 @@ impl PathIter { // where metadata operations can be 100s of milliseconds each. iter.file_buffer.push(path); } else { + iter.explicit.insert(path.clone()); iter.dir_stack.push(path); } } @@ -46,8 +59,19 @@ impl Iterator for PathIter { // Lire le dossier let entries = match fs::read_dir(&dir) { Ok(entries) => entries, - Err(_) => continue, // si erreur (perm, etc.), on ignore ce dossier + Err(_) => { + // Not a readable directory. A path the caller named + // explicitly is handed back as a file: the reader will + // then fail with a precise error (no such file, not a + // FASTA/FASTQ stream). Unreadable sub-directories met + // during expansion are still skipped. + if self.explicit.remove(&dir) { + self.file_buffer.push(dir); + } + continue; + } }; + self.explicit.remove(&dir); for entry in entries.flatten() { let path = entry.path(); // Prefer file_type() over path.is_dir()/is_file() — on most filesystems @@ -72,20 +96,18 @@ pub fn path_iter(paths: &[String]) -> PathIter { } /// Returns true if the path ends with a recognised sequence file extension. +/// +/// `.fna`/`.ffn` (NCBI genomes and genes) and `.fas`/`.fsa` are FASTA +/// extensions in common use, plain or gzip-compressed. fn is_fasta_or_fastq(path: &Path) -> bool { + const SEQ_EXTENSIONS: [&str; 10] = [ + ".fasta", ".fa", ".fna", ".ffn", ".fas", ".fsa", ".fastq", ".fq", ".gbff", ".gbk", + ]; let name = path.file_name().and_then(|n| n.to_str()).unwrap_or(""); - name.ends_with(".fasta") - || name.ends_with(".fa") - || name.ends_with(".fastq") - || name.ends_with(".fq") - || name.ends_with(".fasta.gz") - || name.ends_with(".fa.gz") - || name.ends_with(".fastq.gz") - || name.ends_with(".fq.gz") - || name.ends_with(".gbff") - || name.ends_with(".gbk") - || name.ends_with(".gb") - || name.ends_with(".gbff.gz") - || name.ends_with(".gbk.gz") - || name.ends_with(".gb.gz") + let name = name.strip_suffix(".gz").unwrap_or(name); + SEQ_EXTENSIONS.iter().any(|ext| name.ends_with(ext)) || name.ends_with(".gb") } + +#[cfg(test)] +#[path = "tests/path_iterator.rs"] +mod tests; diff --git a/src/obiread/src/tests/path_iterator.rs b/src/obiread/src/tests/path_iterator.rs new file mode 100644 index 00000000..583fd45a --- /dev/null +++ b/src/obiread/src/tests/path_iterator.rs @@ -0,0 +1,69 @@ +use super::*; +use std::fs; + +fn scratch_dir(tag: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("obiread_pathiter_{}_{tag}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + dir +} + +fn names(it: PathIter) -> Vec { + let mut v: Vec = it + .map(|p| p.file_name().unwrap().to_string_lossy().into_owned()) + .collect(); + v.sort(); + v +} + +#[test] +fn ncbi_and_common_fasta_extensions_are_recognised() { + for name in [ + "g.fna", "g.fna.gz", "g.ffn", "g.fas", "g.fsa.gz", "g.fasta", "g.fa.gz", "g.fq", "g.fastq.gz", + "g.gbff", "g.gbk.gz", "g.gb", + ] { + assert!(is_fasta_or_fastq(Path::new(name)), "{name} should be recognised"); + } + for name in ["g.txt", "g.gz", "g.fna.txt", "g", "g.bam"] { + assert!(!is_fasta_or_fastq(Path::new(name)), "{name} should not be recognised"); + } +} + +#[test] +fn explicit_file_with_unknown_extension_is_not_dropped() { + // The extension filter only applies when expanding directories: a file + // named on the command line reaches the reader, which detects the format + // from its content (and reports a precise error if it is not a sequence). + let dir = scratch_dir("explicit"); + let f = dir.join("genome.unknownext"); + fs::write(&f, b">s\nACGT\n").unwrap(); + assert_eq!(names(PathIter::new(vec![f])), vec!["genome.unknownext"]); + fs::remove_dir_all(dir).unwrap(); +} + +#[test] +fn explicit_missing_path_is_handed_to_the_reader() { + // Not silently skipped: opening it later yields the real I/O error. + let missing = std::env::temp_dir().join(format!("obiread_pathiter_{}_missing.xyz", std::process::id())); + assert_eq!(PathIter::new(vec![missing.clone()]).collect::>(), vec![missing]); +} + +#[test] +fn directory_expansion_still_filters_by_extension() { + let dir = scratch_dir("dir"); + for n in ["a.fna.gz", "b.fa", "c.txt", "d.bam"] { + fs::write(dir.join(n), b"x").unwrap(); + } + fs::create_dir(dir.join("sub")).unwrap(); + fs::write(dir.join("sub").join("e.fastq"), b"x").unwrap(); + assert_eq!( + names(PathIter::new(vec![dir.clone()])), + vec!["a.fna.gz", "b.fa", "e.fastq"] + ); + fs::remove_dir_all(dir).unwrap(); +} + +#[test] +fn stdin_sentinel_passes_through() { + assert_eq!(PathIter::new(vec![PathBuf::from("-")]).collect::>(), vec![PathBuf::from("-")]); +}