From 736337006598161e163af0462f79b107bac64117 Mon Sep 17 00:00:00 2001 From: Eric Coissac Date: Fri, 28 Aug 2026 10:42:54 +0200 Subject: [PATCH] feat: add --force-copy flag to copy files instead of hard-linking Introduces the `--force-copy` CLI argument to bypass hard-linking during selection. When enabled, unchanged kmer-identity files are copied instead of hard-linked to ensure output independence from the source filesystem. The default behavior remains unchanged, continuing to use hard-links with automatic fallback to copying on failure. --- src/obikmer/src/cmd/select/mod.rs | 12 ++++++++++++ src/obikselect/src/select.rs | 16 +++++++++++++++- src/obikselect/src/select_layer.rs | 26 +++++++++++++++++++------- 3 files changed, 46 insertions(+), 8 deletions(-) diff --git a/src/obikmer/src/cmd/select/mod.rs b/src/obikmer/src/cmd/select/mod.rs index 28539b4c..5b5474d3 100644 --- a/src/obikmer/src/cmd/select/mod.rs +++ b/src/obikmer/src/cmd/select/mod.rs @@ -70,6 +70,17 @@ pub struct SelectArgs { /// Overwrite existing output directory #[arg(short, long)] pub force: bool, + + /// Copy each layer's unchanged kmer-identity files (mphf/unitigs/ + /// evidence/fingerprint) instead of hard-linking them. By default + /// `select` hard-links these (never rewritten by a column + /// projection/aggregation, so a hard link is exact and avoids + /// transiently doubling disk usage for a large index), falling back to + /// a real copy only if linking itself fails (e.g. source and output on + /// different filesystems). Set this when the output must survive + /// independently of the source on disk. + #[arg(long)] + pub force_copy: bool, } /// Split a repeatable `:` argument. Exits on malformed input. @@ -128,6 +139,7 @@ pub fn run(args: SelectArgs) { let mut alg = Select::new(&src, &args.output, &specs, output_presence) .threshold(args.presence_threshold) .force(args.force) + .force_copy(args.force_copy) .sparse(!args.dense) .on_progress(|_: Progress| pb.inc(1)); diff --git a/src/obikselect/src/select.rs b/src/obikselect/src/select.rs index e26a32ab..7dcecfb1 100644 --- a/src/obikselect/src/select.rs +++ b/src/obikselect/src/select.rs @@ -24,6 +24,7 @@ pub struct Select<'a> { threshold: u32, output_presence: bool, force: bool, + force_copy: bool, sparse: bool, reporter: Reporter, on_progress: Option>, @@ -43,6 +44,7 @@ impl<'a> Select<'a> { threshold: 0, output_presence, force: false, + force_copy: false, sparse: true, reporter: Reporter::new(), on_progress: None, @@ -63,6 +65,18 @@ impl<'a> Select<'a> { self } + /// Always copy each layer's unchanged kmer-identity files + /// (mphf/unitigs/evidence/fingerprint) instead of hard-linking them + /// (default: `false` — hard link, falling back to a copy only if + /// linking itself fails). Set this when `output` must survive + /// independently of `src` on disk — a hard link still shares the same + /// underlying inode, so rewriting one path outside `select` would + /// affect the other. + pub fn force_copy(mut self, v: bool) -> Self { + self.force_copy = v; + self + } + /// Pack the output's presence matrices in the compact sparse format /// rather than dense (default: `true`). pub fn sparse(mut self, v: bool) -> Self { @@ -124,7 +138,7 @@ impl Algorithm for Select<'_> { runner .run( &order, - |i| select_partition(&dst, src, i, self.specs, self.threshold, self.output_presence), + |i| select_partition(&dst, src, i, self.specs, self.threshold, self.output_presence, self.force_copy), |_, _, _| { done += 1; if let Some(cb) = on_progress.as_mut() { diff --git a/src/obikselect/src/select_layer.rs b/src/obikselect/src/select_layer.rs index 3f1648a3..9fda0cea 100644 --- a/src/obikselect/src/select_layer.rs +++ b/src/obikselect/src/select_layer.rs @@ -107,15 +107,26 @@ fn add_result(mb: &mut MatrixBuilder, r: AggResult) -> io::Result<()> { // ── Helpers ─────────────────────────────────────────────────────────────────── -/// Copy all plain files (not subdirectories) from `src_dir` to `dst_dir` — -/// carries a layer's kmer identity (mphf/unitigs/evidence/fingerprint) -/// across unchanged; only the data matrix (a subdirectory) is rebuilt. -fn copy_layer_files(src_dir: &Path, dst_dir: &Path) -> io::Result<()> { +/// Carries a layer's kmer identity (mphf/unitigs/evidence/fingerprint) from +/// `src_dir` to `dst_dir` **unchanged** — these files are never rewritten by +/// `select` (only the data matrix, a subdirectory, is rebuilt), so a hard +/// link is exact and avoids transiently doubling disk usage for the layer's +/// largest files on a big index. Falls back to a real copy per file when +/// linking fails (e.g. `src`/`dst` on different filesystems, `EXDEV`) or +/// when `force_copy` is set (`--force-copy`, for a destination meant to +/// survive independently of the source — a hard link still shares the same +/// underlying inode, so truncating/rewriting one path in place, outside +/// `select` itself, would affect the other). +fn copy_layer_files(src_dir: &Path, dst_dir: &Path, force_copy: bool) -> io::Result<()> { for entry in fs::read_dir(src_dir)? { let entry = entry?; let path = entry.path(); - if path.is_file() { - fs::copy(&path, dst_dir.join(entry.file_name()))?; + if !path.is_file() { + continue; + } + let dst_path = dst_dir.join(entry.file_name()); + if force_copy || fs::hard_link(&path, &dst_path).is_err() { + fs::copy(&path, &dst_path)?; } } Ok(()) @@ -133,6 +144,7 @@ pub(crate) fn select_partition( specs: &[OutputCol], threshold: u32, output_presence: bool, + force_copy: bool, ) -> OKIResult<()> { let src_partition = src.partition(i)?; let dst_partition = dst.partition(i)?; @@ -147,7 +159,7 @@ pub(crate) fn select_partition( .map_err(OKIError::Io)? .dir() .to_path_buf(); - copy_layer_files(src_layer.dir(), &dst_layer_dir).map_err(OKIError::Io)?; + copy_layer_files(src_layer.dir(), &dst_layer_dir, force_copy).map_err(OKIError::Io)?; let group_mat: Box = match src_layer.content() { LayerContent::Count => {