docs: clarify merge re-merge behavior and input file handling
Update documentation to explain that merged indices can serve as sources if at least one source is unmerged, and that the resulting column order follows the base index first. Clarify that explicitly named files are always read regardless of extension, with format detection based on content. - refactor: prevent I/O errors when re-merging packed outputs by restricting `choose_base` to valid destinations via new `can_be_destination` check - fix: handle zero-layer partitions in `build_index_layer` and `remove_build_artifacts` to skip unnecessary operations - feat: allow `PathIter` to treat unrecognized extensions as explicit files, ensuring they are not silently dropped during directory expansion - test: add comprehensive tests for `PathIter` including extension recognition, explicit file handling, and stdin sentinel behavior - test: update smoke tests to verify empty partitions, `.fna.gz` indexing, and refusal of merging only packed indexes - chore: remove obsolete cargo build environment variables from `run.sh` script
This commit is contained in:
1 parent
07c0a54e04
commit
7f829ea557
11 files changed
+262
-39
No files matched your search
Generated
+8
-8
@@ -422,7 +422,7 @@ version = "3.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -758,7 +758,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1040,7 +1040,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
|
||||
dependencies = [
|
||||
"hermit-abi",
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1975,9 +1975,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ptr_hash"
|
||||
version = "2.1.2"
|
||||
version = "2.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b22cf1be774892680e056d3387db0dcaa4fd46964b270671ef661e127186b969"
|
||||
checksum = "0d2381547e2e36c72a455c1cc418d5c60ecd6e94f4819d7560456108e99a4efa"
|
||||
dependencies = [
|
||||
"bitvec",
|
||||
"cacheline-ef",
|
||||
@@ -2269,7 +2269,7 @@ dependencies = [
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2601,7 +2601,7 @@ dependencies = [
|
||||
"getrandom 0.4.3",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3006,7 +3006,7 @@ version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -135,6 +135,12 @@ impl PrivateBuilder for KmerIndex {
|
||||
mode: &IndexMode,
|
||||
block_bits: u8,
|
||||
) -> Result<usize, SKError> {
|
||||
// A partition that received no k-mer has no layer 0 on disk at all
|
||||
// (small inputs against many partitions): nothing to build, not an
|
||||
// error.
|
||||
if self.partition(i).map_err(|e| io::Error::other(e.to_string()))?.n_layers() == 0 {
|
||||
return Ok(0);
|
||||
}
|
||||
let layer0 = self
|
||||
.layer0(i)
|
||||
.map_err(|e| io::Error::other(e.to_string()))?;
|
||||
@@ -208,6 +214,10 @@ impl PrivateBuilder for KmerIndex {
|
||||
}
|
||||
|
||||
fn remove_build_artifacts(&self, i: usize) {
|
||||
// No layer 0 = empty partition: no build artifacts to remove.
|
||||
if self.partition(i).map_or(false, |p| p.n_layers() == 0) {
|
||||
return;
|
||||
}
|
||||
let layer0 = match self.layer0(i) {
|
||||
Ok(layer) => layer,
|
||||
Err(e) => {
|
||||
|
||||
@@ -3,6 +3,7 @@ use std::fs;
|
||||
use std::io;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use obicompactvec::{PersistentBitMatrix, PersistentIntMatrix, StorageKind};
|
||||
use obikalgorithm::Algorithm;
|
||||
use obikindex::layer::IndexMode;
|
||||
use obikindex::partition::PARTITIONS_SUBDIR;
|
||||
@@ -176,7 +177,17 @@ impl Algorithm for Merge<'_> {
|
||||
);
|
||||
}
|
||||
|
||||
let base_idx = choose_base(sources, &src_genomes, self.mode);
|
||||
let Some(base_idx) = choose_base(sources, &src_genomes, self.mode) else {
|
||||
return Err(
|
||||
"merge: none of the sources can be the destination. Every candidate has its \
|
||||
matrices packed (finalized index): a packed index can be merged in as a \
|
||||
source, but at least one source must be unpacked — typically a freshly \
|
||||
built mono-genome index. To extend a packed index, merge it with new \
|
||||
mono-genome indexes; to combine several packed indexes, merge their \
|
||||
original sources instead."
|
||||
.into(),
|
||||
);
|
||||
};
|
||||
let needs_approx = sources.iter().enumerate().any(|(i, src)| {
|
||||
!is_trivial(&src_genomes[i], self.mode)
|
||||
&& matches!(
|
||||
@@ -490,7 +501,15 @@ fn index_unitig_size(src: &KmerIndex) -> u64 {
|
||||
(0..n).map(|i| partition_unitig_bytes(src, i)).sum()
|
||||
}
|
||||
|
||||
fn choose_base(sources: &[&KmerIndex], src_genomes: &[Vec<GenomeInfo>], mode: MergeMode) -> usize {
|
||||
/// The source whose partition data is copied as the merge destination: the
|
||||
/// largest one (least data to merge in). `None` when several sources are
|
||||
/// given and none of them can be resumed as a destination (see
|
||||
/// [`can_be_destination`]).
|
||||
fn choose_base(
|
||||
sources: &[&KmerIndex],
|
||||
src_genomes: &[Vec<GenomeInfo>],
|
||||
mode: MergeMode,
|
||||
) -> Option<usize> {
|
||||
let needs_approx = sources.iter().enumerate().any(|(i, src)| {
|
||||
!is_trivial(&src_genomes[i], mode)
|
||||
&& matches!(
|
||||
@@ -509,9 +528,42 @@ fn choose_base(sources: &[&KmerIndex], src_genomes: &[Vec<GenomeInfo>], mode: Me
|
||||
IndexMode::Approx { .. } | IndexMode::Hybrid { .. }
|
||||
)
|
||||
})
|
||||
// With a single source there is nothing to append: the bootstrap
|
||||
// copy is the whole merge, whatever its matrix format.
|
||||
.filter(|(i, src)| sources.len() == 1 || can_be_destination(src, &src_genomes[*i], mode))
|
||||
.max_by_key(|(_, src)| index_unitig_size(src))
|
||||
.map(|(i, _)| i)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
/// Whether `index` can be the destination of a merge: pass 2 appends the
|
||||
/// other sources' columns to its matrices (`MatrixBuilder::resume`), which
|
||||
/// needs them in the column-by-column format. A finalized index has its
|
||||
/// matrices packed (sparse by default, or dense with `--dense`) into
|
||||
/// single-file formats that cannot be extended — such an index can only
|
||||
/// come in as a *source*. An implicit matrix (mono-genome presence) is
|
||||
/// materialized on the fly, so it qualifies only for a single genome.
|
||||
///
|
||||
/// One layer is probed: `pack` rewrites every partition the same way. An
|
||||
/// index with no k-mer at all has nothing to resume and qualifies.
|
||||
fn can_be_destination(index: &KmerIndex, genomes: &[GenomeInfo], mode: MergeMode) -> bool {
|
||||
let layer_dir = (0..index.n_partitions()).find_map(|i| {
|
||||
let partition = index.partition(i).ok()?;
|
||||
(partition.n_layers() > 0)
|
||||
.then(|| partition.layer_dir(0).ok())
|
||||
.flatten()
|
||||
});
|
||||
let Some(layer_dir) = layer_dir else {
|
||||
return true;
|
||||
};
|
||||
let kind = match mode {
|
||||
MergeMode::Presence => PersistentBitMatrix::detect_storage(&layer_dir),
|
||||
MergeMode::Count => PersistentIntMatrix::detect_storage(&layer_dir),
|
||||
};
|
||||
match kind {
|
||||
Ok(StorageKind::Columnar) => true,
|
||||
Ok(StorageKind::Implicit) => genomes.len() == 1,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Copies `src` (the base source's whole `partitions/` tree) into `dst` at
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use std::collections::HashSet;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
@@ -5,6 +6,10 @@ use std::path::{Path, PathBuf};
|
||||
pub struct PathIter {
|
||||
dir_stack: Vec<PathBuf>, // dossiers qu'il reste à explorer
|
||||
file_buffer: Vec<PathBuf>, // fichiers déjà listés (à distribuer)
|
||||
/// Paths given by the caller whose extension is not a recognised sequence
|
||||
/// one. They are tried as directories first; when that fails they are
|
||||
/// handed back as files (see [`PathIter::next`]) rather than dropped.
|
||||
explicit: HashSet<PathBuf>,
|
||||
}
|
||||
|
||||
impl PathIter {
|
||||
@@ -12,11 +17,18 @@ impl PathIter {
|
||||
///
|
||||
/// For non-directory paths only the file extension is checked (no stat syscall).
|
||||
/// Directories are left on the stack and expanded lazily in `next()`.
|
||||
/// Invalid or missing paths are silently skipped when `open_chunks` fails later.
|
||||
///
|
||||
/// A path without a recognised extension is tried as a directory; if it
|
||||
/// is not one, it is yielded as a file so the reader reports the real
|
||||
/// problem (missing file, not a FASTA/FASTQ stream) instead of the path
|
||||
/// being skipped silently. The extension filter only matters when
|
||||
/// expanding directories: format and compression are detected from the
|
||||
/// content.
|
||||
pub fn new(paths: Vec<PathBuf>) -> Self {
|
||||
let mut iter = PathIter {
|
||||
dir_stack: Vec::new(),
|
||||
file_buffer: Vec::new(),
|
||||
explicit: HashSet::new(),
|
||||
};
|
||||
for path in paths {
|
||||
// "-" is the stdin sentinel — pass it through without any extension
|
||||
@@ -28,6 +40,7 @@ impl PathIter {
|
||||
// where metadata operations can be 100s of milliseconds each.
|
||||
iter.file_buffer.push(path);
|
||||
} else {
|
||||
iter.explicit.insert(path.clone());
|
||||
iter.dir_stack.push(path);
|
||||
}
|
||||
}
|
||||
@@ -46,8 +59,19 @@ impl Iterator for PathIter {
|
||||
// Lire le dossier
|
||||
let entries = match fs::read_dir(&dir) {
|
||||
Ok(entries) => entries,
|
||||
Err(_) => continue, // si erreur (perm, etc.), on ignore ce dossier
|
||||
Err(_) => {
|
||||
// Not a readable directory. A path the caller named
|
||||
// explicitly is handed back as a file: the reader will
|
||||
// then fail with a precise error (no such file, not a
|
||||
// FASTA/FASTQ stream). Unreadable sub-directories met
|
||||
// during expansion are still skipped.
|
||||
if self.explicit.remove(&dir) {
|
||||
self.file_buffer.push(dir);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
self.explicit.remove(&dir);
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
// Prefer file_type() over path.is_dir()/is_file() — on most filesystems
|
||||
@@ -72,20 +96,18 @@ pub fn path_iter(paths: &[String]) -> PathIter {
|
||||
}
|
||||
|
||||
/// Returns true if the path ends with a recognised sequence file extension.
|
||||
///
|
||||
/// `.fna`/`.ffn` (NCBI genomes and genes) and `.fas`/`.fsa` are FASTA
|
||||
/// extensions in common use, plain or gzip-compressed.
|
||||
fn is_fasta_or_fastq(path: &Path) -> bool {
|
||||
const SEQ_EXTENSIONS: [&str; 10] = [
|
||||
".fasta", ".fa", ".fna", ".ffn", ".fas", ".fsa", ".fastq", ".fq", ".gbff", ".gbk",
|
||||
];
|
||||
let name = path.file_name().and_then(|n| n.to_str()).unwrap_or("");
|
||||
name.ends_with(".fasta")
|
||||
|| name.ends_with(".fa")
|
||||
|| name.ends_with(".fastq")
|
||||
|| name.ends_with(".fq")
|
||||
|| name.ends_with(".fasta.gz")
|
||||
|| name.ends_with(".fa.gz")
|
||||
|| name.ends_with(".fastq.gz")
|
||||
|| name.ends_with(".fq.gz")
|
||||
|| name.ends_with(".gbff")
|
||||
|| name.ends_with(".gbk")
|
||||
|| name.ends_with(".gb")
|
||||
|| name.ends_with(".gbff.gz")
|
||||
|| name.ends_with(".gbk.gz")
|
||||
|| name.ends_with(".gb.gz")
|
||||
let name = name.strip_suffix(".gz").unwrap_or(name);
|
||||
SEQ_EXTENSIONS.iter().any(|ext| name.ends_with(ext)) || name.ends_with(".gb")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "tests/path_iterator.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,69 @@
|
||||
use super::*;
|
||||
use std::fs;
|
||||
|
||||
fn scratch_dir(tag: &str) -> PathBuf {
|
||||
let dir = std::env::temp_dir().join(format!("obiread_pathiter_{}_{tag}", std::process::id()));
|
||||
let _ = fs::remove_dir_all(&dir);
|
||||
fs::create_dir_all(&dir).unwrap();
|
||||
dir
|
||||
}
|
||||
|
||||
fn names(it: PathIter) -> Vec<String> {
|
||||
let mut v: Vec<String> = it
|
||||
.map(|p| p.file_name().unwrap().to_string_lossy().into_owned())
|
||||
.collect();
|
||||
v.sort();
|
||||
v
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ncbi_and_common_fasta_extensions_are_recognised() {
|
||||
for name in [
|
||||
"g.fna", "g.fna.gz", "g.ffn", "g.fas", "g.fsa.gz", "g.fasta", "g.fa.gz", "g.fq", "g.fastq.gz",
|
||||
"g.gbff", "g.gbk.gz", "g.gb",
|
||||
] {
|
||||
assert!(is_fasta_or_fastq(Path::new(name)), "{name} should be recognised");
|
||||
}
|
||||
for name in ["g.txt", "g.gz", "g.fna.txt", "g", "g.bam"] {
|
||||
assert!(!is_fasta_or_fastq(Path::new(name)), "{name} should not be recognised");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_file_with_unknown_extension_is_not_dropped() {
|
||||
// The extension filter only applies when expanding directories: a file
|
||||
// named on the command line reaches the reader, which detects the format
|
||||
// from its content (and reports a precise error if it is not a sequence).
|
||||
let dir = scratch_dir("explicit");
|
||||
let f = dir.join("genome.unknownext");
|
||||
fs::write(&f, b">s\nACGT\n").unwrap();
|
||||
assert_eq!(names(PathIter::new(vec![f])), vec!["genome.unknownext"]);
|
||||
fs::remove_dir_all(dir).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_missing_path_is_handed_to_the_reader() {
|
||||
// Not silently skipped: opening it later yields the real I/O error.
|
||||
let missing = std::env::temp_dir().join(format!("obiread_pathiter_{}_missing.xyz", std::process::id()));
|
||||
assert_eq!(PathIter::new(vec![missing.clone()]).collect::<Vec<_>>(), vec![missing]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn directory_expansion_still_filters_by_extension() {
|
||||
let dir = scratch_dir("dir");
|
||||
for n in ["a.fna.gz", "b.fa", "c.txt", "d.bam"] {
|
||||
fs::write(dir.join(n), b"x").unwrap();
|
||||
}
|
||||
fs::create_dir(dir.join("sub")).unwrap();
|
||||
fs::write(dir.join("sub").join("e.fastq"), b"x").unwrap();
|
||||
assert_eq!(
|
||||
names(PathIter::new(vec![dir.clone()])),
|
||||
vec!["a.fna.gz", "b.fa", "e.fastq"]
|
||||
);
|
||||
fs::remove_dir_all(dir).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stdin_sentinel_passes_through() {
|
||||
assert_eq!(PathIter::new(vec![PathBuf::from("-")]).collect::<Vec<_>>(), vec![PathBuf::from("-")]);
|
||||
}
|
||||
Reference in new issue
Block a user