Push zpwxxpnpktps #67
@@ -2,7 +2,7 @@ use std::collections::BTreeMap;
|
|||||||
use std::fs;
|
use std::fs;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
use obikpartitionner::{KmerPartition, KmerSpectrum};
|
use obikpartitionner::{KmerPartition, KmerSpectrum, PARTITIONS_SUBDIR};
|
||||||
use obilayeredmap;
|
use obilayeredmap;
|
||||||
use obisys::{Reporter, Stage, progress_bar};
|
use obisys::{Reporter, Stage, progress_bar};
|
||||||
use rayon::prelude::*;
|
use rayon::prelude::*;
|
||||||
@@ -68,6 +68,65 @@ impl KmerIndex {
|
|||||||
IndexMeta::exists(path.as_ref())
|
IndexMeta::exists(path.as_ref())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Make `output` ready to receive a newly created index.
|
||||||
|
///
|
||||||
|
/// If an index already exists there, remove it when `force` is set,
|
||||||
|
/// otherwise fail. A bare directory with no `index.meta` (e.g. one left
|
||||||
|
/// behind by a `DirLock`) is not considered a pre-existing index.
|
||||||
|
pub fn clear_output_for_create<P: AsRef<Path>>(output: P, force: bool) -> OKIResult<()> {
|
||||||
|
let output = output.as_ref();
|
||||||
|
if Self::exists(output) {
|
||||||
|
if force {
|
||||||
|
fs::remove_dir_all(output).map_err(OKIError::Io)?;
|
||||||
|
} else {
|
||||||
|
return Err(OKIError::Io(std::io::Error::new(
|
||||||
|
std::io::ErrorKind::AlreadyExists,
|
||||||
|
format!("{}: output directory already exists", output.display()),
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Lay out a fresh index skeleton at `output`: the root directory,
|
||||||
|
/// `index.meta` (from `meta`), and an opened, empty partition set.
|
||||||
|
///
|
||||||
|
/// For construction paths that build partitions from scratch (`select`,
|
||||||
|
/// `rebuild`). `merge` bootstraps by copying a source index instead, so
|
||||||
|
/// it does not use this.
|
||||||
|
pub(crate) fn create_skeleton<P: AsRef<Path>>(
|
||||||
|
output: P,
|
||||||
|
meta: &IndexMeta,
|
||||||
|
) -> OKIResult<KmerPartition> {
|
||||||
|
let output = output.as_ref();
|
||||||
|
fs::create_dir_all(output).map_err(OKIError::Io)?;
|
||||||
|
meta.write(output).map_err(OKIError::Io)?;
|
||||||
|
fs::create_dir_all(output.join(PARTITIONS_SUBDIR)).map_err(OKIError::Io)?;
|
||||||
|
Ok(KmerPartition::open_with_config(
|
||||||
|
output,
|
||||||
|
meta.config.kmer_size,
|
||||||
|
meta.config.minimizer_size,
|
||||||
|
meta.config.n_bits,
|
||||||
|
)?)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Mark `output` as fully indexed, pack its column matrices, and reopen it.
|
||||||
|
///
|
||||||
|
/// Shared tail of the `select`/`rebuild` construction paths, once their
|
||||||
|
/// partitions have been written.
|
||||||
|
pub(crate) fn finalize_indexed<P: AsRef<Path>>(
|
||||||
|
output: P,
|
||||||
|
rep: &mut Reporter,
|
||||||
|
) -> OKIResult<Self> {
|
||||||
|
let output = output.as_ref();
|
||||||
|
fs::File::create(output.join(SENTINEL_INDEXED)).map_err(OKIError::Io)?;
|
||||||
|
let idx = KmerIndex::open(output)?;
|
||||||
|
let t_pack = Stage::start("pack");
|
||||||
|
idx.pack_matrices()?;
|
||||||
|
rep.push(t_pack.stop());
|
||||||
|
Ok(idx)
|
||||||
|
}
|
||||||
|
|
||||||
/// Current construction state, as reported by sentinel files on disk.
|
/// Current construction state, as reported by sentinel files on disk.
|
||||||
pub fn state(&self) -> IndexState {
|
pub fn state(&self) -> IndexState {
|
||||||
IndexState::detect(&self.root_path).unwrap_or(IndexState::Empty)
|
IndexState::detect(&self.root_path).unwrap_or(IndexState::Empty)
|
||||||
|
|||||||
@@ -130,16 +130,7 @@ impl KmerIndex {
|
|||||||
let (source_labels, all_genomes) = compute_labels(sources, rename_duplicates)?;
|
let (source_labels, all_genomes) = compute_labels(sources, rename_duplicates)?;
|
||||||
|
|
||||||
// ── Prepare output directory ──────────────────────────────────────────
|
// ── Prepare output directory ──────────────────────────────────────────
|
||||||
if output.exists() {
|
KmerIndex::clear_output_for_create(output, force)?;
|
||||||
if force {
|
|
||||||
fs::remove_dir_all(output)?;
|
|
||||||
} else {
|
|
||||||
return Err(OKIError::Io(io::Error::new(
|
|
||||||
io::ErrorKind::AlreadyExists,
|
|
||||||
format!("{}: output directory already exists", output.display()),
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ── Bootstrap: copy first source to output ────────────────────────────
|
// ── Bootstrap: copy first source to output ────────────────────────────
|
||||||
info!(
|
info!(
|
||||||
|
|||||||
@@ -1,15 +1,13 @@
|
|||||||
use std::fs;
|
|
||||||
use std::io;
|
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
|
||||||
use obikpartitionner::{KmerFilter, KmerPartition, MergeMode};
|
use obikpartitionner::{KmerFilter, MergeMode};
|
||||||
use obisys::{Reporter, Stage, progress_bar};
|
use obisys::{Reporter, Stage, progress_bar};
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
use crate::error::{OKIError, OKIResult};
|
use crate::error::{OKIError, OKIResult};
|
||||||
use crate::index::KmerIndex;
|
use crate::index::KmerIndex;
|
||||||
use crate::meta::IndexMeta;
|
use crate::meta::IndexMeta;
|
||||||
use crate::state::{IndexState, SENTINEL_INDEXED};
|
use crate::state::IndexState;
|
||||||
|
|
||||||
impl KmerIndex {
|
impl KmerIndex {
|
||||||
/// Rebuild `src` into a new compact single-layer index at `output`.
|
/// Rebuild `src` into a new compact single-layer index at `output`.
|
||||||
@@ -40,36 +38,18 @@ impl KmerIndex {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
if output.exists() {
|
KmerIndex::clear_output_for_create(output, force)?;
|
||||||
if force {
|
|
||||||
fs::remove_dir_all(output)?;
|
|
||||||
} else {
|
|
||||||
return Err(OKIError::Io(io::Error::new(
|
|
||||||
io::ErrorKind::AlreadyExists,
|
|
||||||
format!("{}: output directory already exists", output.display()),
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ── Create output directory + metadata ────────────────────────────────
|
// ── Create output directory + metadata ────────────────────────────────
|
||||||
fs::create_dir_all(output)?;
|
|
||||||
let mut meta = IndexMeta::new(src.meta.config.clone());
|
let mut meta = IndexMeta::new(src.meta.config.clone());
|
||||||
meta.config.with_counts = mode == MergeMode::Count;
|
meta.config.with_counts = mode == MergeMode::Count;
|
||||||
meta.genomes = src.meta.genomes.clone();
|
meta.genomes = src.meta.genomes.clone();
|
||||||
meta.write(output)?;
|
|
||||||
|
|
||||||
let n_genomes = src.meta.genomes.len();
|
let n_genomes = src.meta.genomes.len();
|
||||||
let n_partitions = src.partition.n_partitions();
|
let n_partitions = src.partition.n_partitions();
|
||||||
|
|
||||||
// ── Create an empty destination KmerPartition ─────────────────────────
|
// ── Create an empty destination KmerPartition ─────────────────────────
|
||||||
// Create the partitions/ subdirectory so KmerPartition::open_with_config works.
|
let dst_partition = KmerIndex::create_skeleton(output, &meta)?;
|
||||||
fs::create_dir_all(output.join(obikpartitionner::PARTITIONS_SUBDIR))?;
|
|
||||||
let dst_partition = KmerPartition::open_with_config(
|
|
||||||
output,
|
|
||||||
meta.config.kmer_size,
|
|
||||||
meta.config.minimizer_size,
|
|
||||||
meta.config.n_bits,
|
|
||||||
)?;
|
|
||||||
|
|
||||||
info!(
|
info!(
|
||||||
"rebuild: {} partition(s), {} genome(s), mode={:?}",
|
"rebuild: {} partition(s), {} genome(s), mode={:?}",
|
||||||
@@ -94,13 +74,6 @@ impl KmerIndex {
|
|||||||
|
|
||||||
rep.push(t.stop());
|
rep.push(t.stop());
|
||||||
|
|
||||||
// Write SENTINEL_INDEXED — output is ready to use.
|
KmerIndex::finalize_indexed(output, rep)
|
||||||
fs::File::create(output.join(SENTINEL_INDEXED))?;
|
|
||||||
|
|
||||||
let idx = KmerIndex::open(output)?;
|
|
||||||
let t_pack = Stage::start("pack");
|
|
||||||
idx.pack_matrices()?;
|
|
||||||
rep.push(t_pack.stop());
|
|
||||||
Ok(idx)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,15 +1,13 @@
|
|||||||
use std::fs;
|
|
||||||
use std::io;
|
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
|
||||||
use obikpartitionner::{KmerPartition, OutputCol, PARTITIONS_SUBDIR};
|
use obikpartitionner::{KmerPartition, OutputCol};
|
||||||
use obisys::{Reporter, Stage, progress_bar};
|
use obisys::{Reporter, Stage, progress_bar};
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
use crate::error::{OKIError, OKIResult};
|
use crate::error::{OKIError, OKIResult};
|
||||||
use crate::index::KmerIndex;
|
use crate::index::KmerIndex;
|
||||||
use crate::meta::{GenomeInfo, IndexMeta};
|
use crate::meta::{GenomeInfo, IndexMeta};
|
||||||
use crate::state::{IndexState, SENTINEL_INDEXED};
|
use crate::state::IndexState;
|
||||||
|
|
||||||
impl KmerIndex {
|
impl KmerIndex {
|
||||||
/// Create a new index at `output` by projecting/aggregating the genome columns
|
/// Create a new index at `output` by projecting/aggregating the genome columns
|
||||||
@@ -33,35 +31,18 @@ impl KmerIndex {
|
|||||||
return Err(OKIError::NotIndexed(src.root_path.clone()));
|
return Err(OKIError::NotIndexed(src.root_path.clone()));
|
||||||
}
|
}
|
||||||
|
|
||||||
if output.exists() {
|
KmerIndex::clear_output_for_create(output, force)?;
|
||||||
if force {
|
|
||||||
fs::remove_dir_all(output)?;
|
|
||||||
} else {
|
|
||||||
return Err(OKIError::Io(io::Error::new(
|
|
||||||
io::ErrorKind::AlreadyExists,
|
|
||||||
format!("{}: output directory already exists", output.display()),
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fs::create_dir_all(output)?;
|
|
||||||
let mut meta = IndexMeta::new(src.meta.config.clone());
|
let mut meta = IndexMeta::new(src.meta.config.clone());
|
||||||
meta.config.with_counts = !output_presence;
|
meta.config.with_counts = !output_presence;
|
||||||
meta.genomes = specs.iter()
|
meta.genomes = specs.iter()
|
||||||
.map(|s| GenomeInfo::new(s.label.clone()))
|
.map(|s| GenomeInfo::new(s.label.clone()))
|
||||||
.collect();
|
.collect();
|
||||||
meta.write(output)?;
|
|
||||||
|
|
||||||
let n_src_genomes = src.meta.genomes.len();
|
let n_src_genomes = src.meta.genomes.len();
|
||||||
let n_partitions = src.partition.n_partitions();
|
let n_partitions = src.partition.n_partitions();
|
||||||
|
|
||||||
fs::create_dir_all(output.join(PARTITIONS_SUBDIR))?;
|
let dst_partition = KmerIndex::create_skeleton(output, &meta)?;
|
||||||
let dst_partition = KmerPartition::open_with_config(
|
|
||||||
output,
|
|
||||||
meta.config.kmer_size,
|
|
||||||
meta.config.minimizer_size,
|
|
||||||
meta.config.n_bits,
|
|
||||||
)?;
|
|
||||||
|
|
||||||
info!(
|
info!(
|
||||||
"select: {} partition(s), {} source genome(s) → {} output column(s)",
|
"select: {} partition(s), {} source genome(s) → {} output column(s)",
|
||||||
@@ -83,13 +64,7 @@ impl KmerIndex {
|
|||||||
pb.finish_and_clear();
|
pb.finish_and_clear();
|
||||||
rep.push(t.stop());
|
rep.push(t.stop());
|
||||||
|
|
||||||
fs::File::create(output.join(SENTINEL_INDEXED))?;
|
KmerIndex::finalize_indexed(output, rep)
|
||||||
|
|
||||||
let idx = KmerIndex::open(output)?;
|
|
||||||
let t_pack = Stage::start("pack");
|
|
||||||
idx.pack_matrices()?;
|
|
||||||
rep.push(t_pack.stop());
|
|
||||||
Ok(idx)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Rewrite the genome columns of this index in-place according to `specs`.
|
/// Rewrite the genome columns of this index in-place according to `specs`.
|
||||||
|
|||||||
@@ -91,12 +91,12 @@ pub fn spinner(label: &str) -> TracedBar {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Progress bar with the standard project look:
|
/// Progress bar with the standard project look:
|
||||||
/// `⠋ label — [████░░░░] pos/len unit elapsed`.
|
/// `⠋ label — [████░░░░] pos/len unit elapsed (eta: remaining)`.
|
||||||
pub fn progress_bar(label: &str, n: u64, unit: &str) -> TracedBar {
|
pub fn progress_bar(label: &str, n: u64, unit: &str) -> TracedBar {
|
||||||
let pb = ProgressBar::new(n);
|
let pb = ProgressBar::new(n);
|
||||||
pb.set_style(
|
pb.set_style(
|
||||||
ProgressStyle::with_template(&format!(
|
ProgressStyle::with_template(&format!(
|
||||||
"{{spinner}} {label} — {{bar:40.cyan/blue}} {{pos}}/{{len}} {unit} {{elapsed}}"
|
"{{spinner}} {label} — {{bar:40.cyan/blue}} {{pos}}/{{len}} {unit} {{elapsed}} (eta: {{eta}})"
|
||||||
))
|
))
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.tick_strings(BRAILLE),
|
.tick_strings(BRAILLE),
|
||||||
|
|||||||
Reference in New Issue
Block a user