This change replaces unbuffered file reads with buffered I/O operations to improve performance during index metadata loading. This includes implementing new methods for reading index metadata files and wrapping file handles in `BufReader` to optimize subsequent reads.
311 lines
11 KiB
Rust
311 lines
11 KiB
Rust
use std::collections::HashMap;
|
|
use std::fs;
|
|
use std::io;
|
|
use std::path::{Path, PathBuf};
|
|
use std::sync::RwLock;
|
|
|
|
use crate::index::kmer_index::KmerIndex;
|
|
use crate::index::state::IndexState;
|
|
use crate::layer::IndexMode;
|
|
use serde::{Deserialize, Serialize};
|
|
|
|
pub const META_FILENAME: &str = "index.meta";
|
|
const META_VERSION: u32 = 1;
|
|
|
|
/// Per-genome label + categorical metadata.
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct GenomeInfo {
|
|
pub label: String,
|
|
#[serde(default)]
|
|
pub meta: HashMap<String, String>,
|
|
}
|
|
|
|
impl GenomeInfo {
|
|
/// Panics if `label` fails [`validate_label`](Self::validate_label) —
|
|
/// an invalid genome label is a programmer error to surface loudly,
|
|
/// not a recoverable condition for this constructor to paper over.
|
|
/// Callers that need a graceful error (e.g. a CLI parsing user input)
|
|
/// should call [`validate_label`](Self::validate_label) themselves
|
|
/// first and report it their own way.
|
|
pub fn new(label: impl Into<String>) -> Self {
|
|
let label = label.into();
|
|
Self::validate_label(&label).expect("invalid genome label");
|
|
Self {
|
|
label,
|
|
meta: HashMap::new(),
|
|
}
|
|
}
|
|
|
|
/// Validate a user-supplied genome label.
|
|
///
|
|
/// Forbidden: `/` (filesystem separator), `=` (--new-label parser separator),
|
|
/// `\0` (null byte), `\n`, `\r`, `\t` (break CSV output).
|
|
/// Empty labels are also rejected.
|
|
pub fn validate_label(label: &str) -> Result<(), String> {
|
|
if label.is_empty() {
|
|
return Err("genome label must not be empty".into());
|
|
}
|
|
const FORBIDDEN: &[char] = &['/', '=', '\0', '\n', '\r', '\t'];
|
|
if let Some(c) = label.chars().find(|c| FORBIDDEN.contains(c)) {
|
|
let display = match c {
|
|
'\0' => "\\0 (null)".to_string(),
|
|
'\n' => "\\n (newline)".to_string(),
|
|
'\r' => "\\r (carriage return)".to_string(),
|
|
'\t' => "\\t (tab)".to_string(),
|
|
c => format!("'{c}'"),
|
|
};
|
|
return Err(format!(
|
|
"genome label contains forbidden character {display}"
|
|
));
|
|
}
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
/// Fixed at index creation, never changes afterward — freely mutable while
|
|
/// being assembled (before any `IndexMeta::create` call), immutable once an
|
|
/// `IndexMeta` wraps it.
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct IndexConfig {
|
|
pub kmer_size: usize,
|
|
pub minimizer_size: usize,
|
|
pub n_bits: usize,
|
|
pub with_counts: bool,
|
|
#[serde(default)]
|
|
pub evidence: IndexMode,
|
|
/// Block size for the unitig index as a power-of-two exponent.
|
|
/// The `.idx` block covers 2^block_bits consecutive unitigs.
|
|
/// 0 = one entry per unitig (O(1) access, largest `.idx`).
|
|
#[serde(default)]
|
|
pub block_bits: u8,
|
|
}
|
|
|
|
/// On-disk shape of `index.meta` — the whole file, read/written atomically
|
|
/// as one JSON blob by every `IndexMeta` operation below.
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
struct IndexMetadata {
|
|
version: u32,
|
|
config: IndexConfig,
|
|
#[serde(default)]
|
|
genomes: Vec<GenomeInfo>,
|
|
#[serde(default)]
|
|
state: IndexState,
|
|
}
|
|
|
|
/// Stateless accessor for one index's `index.meta` — `config` is the only
|
|
/// part cached in memory (fixed forever once written; re-reading it would
|
|
/// only ever return the same value), everything else (`genomes`, `state`)
|
|
/// is read fresh from disk on every access and written fresh on every
|
|
/// change, since it can legitimately change during a run — see
|
|
/// `DevDocMD/implementation/partition_layer_cache.md`.
|
|
///
|
|
/// One instance per index, held as `Arc<IndexMeta>` by `KmerIndex` and
|
|
/// shared by every borrower — the internal `lock` only serialises access
|
|
/// *through that one shared instance*; it is not a cross-process file lock
|
|
/// (that's `obisys::DirLock`, held by `cmd/index` for the whole build).
|
|
/// Mutating methods hold the write lock across the full read-modify-write
|
|
/// sequence, not just the write, so two concurrent callers can't silently
|
|
/// clobber each other's change.
|
|
pub struct IndexMeta {
|
|
root_path: PathBuf,
|
|
version: u32,
|
|
pub config: IndexConfig,
|
|
lock: RwLock<()>,
|
|
}
|
|
|
|
impl IndexMeta {
|
|
/// Create a brand-new `index.meta` for `index`, from fixed config and
|
|
/// an initial genome list (often empty — genomes are usually added
|
|
/// later via [`push_genome`](Self::push_genome)).
|
|
pub fn create(
|
|
index: &KmerIndex,
|
|
config: IndexConfig,
|
|
genomes: Vec<GenomeInfo>,
|
|
) -> io::Result<Self> {
|
|
Self::create_at(index.dir(), config, genomes)
|
|
}
|
|
|
|
/// Same as [`create`](Self::create), taking a raw path — for
|
|
/// `KmerIndex::create` itself, which doesn't have a complete
|
|
/// `KmerIndex` yet to hand in (it's what's being built).
|
|
pub(crate) fn create_at(
|
|
root_path: &Path,
|
|
config: IndexConfig,
|
|
genomes: Vec<GenomeInfo>,
|
|
) -> io::Result<Self> {
|
|
let meta = Self {
|
|
root_path: root_path.to_owned(),
|
|
version: META_VERSION,
|
|
config,
|
|
lock: RwLock::new(()),
|
|
};
|
|
meta.write_full(&IndexMetadata {
|
|
version: meta.version,
|
|
config: meta.config.clone(),
|
|
genomes,
|
|
state: IndexState::Empty,
|
|
})?;
|
|
Ok(meta)
|
|
}
|
|
|
|
/// Reopen the `index.meta` already on disk for `index`.
|
|
pub fn open(index: &KmerIndex) -> io::Result<Self> {
|
|
Self::open_at(index.dir())
|
|
}
|
|
|
|
pub(crate) fn open_at(root_path: &Path) -> io::Result<Self> {
|
|
let on_disk = Self::read_full(root_path)?;
|
|
Ok(Self {
|
|
root_path: root_path.to_owned(),
|
|
version: on_disk.version,
|
|
config: on_disk.config,
|
|
lock: RwLock::new(()),
|
|
})
|
|
}
|
|
|
|
/// Returns whether the metadata file exists in `root`.
|
|
///
|
|
/// This is a static method (associated function), not an instance method.
|
|
/// It only checks for the presence of the `index.meta` file inside the
|
|
/// given directory; it does not validate its contents.
|
|
pub(crate) fn exists(root: &Path) -> bool {
|
|
root.join(META_FILENAME).exists()
|
|
}
|
|
|
|
// ── immutable, cached — no disk access ──────────────────────────────────
|
|
|
|
pub fn version(&self) -> u32 {
|
|
self.version
|
|
}
|
|
|
|
pub fn config(&self) -> &IndexConfig {
|
|
&self.config
|
|
}
|
|
|
|
// ── variable, stateless — every call reads or writes `index.meta` fresh ─
|
|
|
|
pub fn genomes(&self) -> io::Result<Vec<GenomeInfo>> {
|
|
let _guard = self.lock.read().unwrap();
|
|
Ok(Self::read_full(&self.root_path)?.genomes)
|
|
}
|
|
|
|
pub fn state(&self) -> io::Result<IndexState> {
|
|
let _guard = self.lock.read().unwrap();
|
|
Ok(Self::read_full(&self.root_path)?.state)
|
|
}
|
|
|
|
pub fn push_genome(&self, info: GenomeInfo) -> io::Result<()> {
|
|
let _guard = self.lock.write().unwrap();
|
|
let mut on_disk = Self::read_full(&self.root_path)?;
|
|
on_disk.genomes.push(info);
|
|
self.write_full(&on_disk)
|
|
}
|
|
|
|
pub fn rename_genome(&self, pos: usize, new_label: impl Into<String>) -> io::Result<()> {
|
|
let _guard = self.lock.write().unwrap();
|
|
let mut on_disk = Self::read_full(&self.root_path)?;
|
|
on_disk.genomes[pos].label = new_label.into();
|
|
self.write_full(&on_disk)
|
|
}
|
|
|
|
/// Replace the whole genome list at once — for construction paths
|
|
/// (`select`/`merge`) that compute the full output list up front
|
|
/// rather than pushing one at a time.
|
|
pub fn set_genomes(&self, genomes: Vec<GenomeInfo>) -> io::Result<()> {
|
|
let _guard = self.lock.write().unwrap();
|
|
let mut on_disk = Self::read_full(&self.root_path)?;
|
|
on_disk.genomes = genomes;
|
|
self.write_full(&on_disk)
|
|
}
|
|
|
|
/// Update the evidence representation in place — `evidence`, and (for
|
|
/// `Exact`) its block size. The only `IndexConfig` fields a reindex is
|
|
/// ever allowed to touch: `kmer_size`/`minimizer_size`/`n_bits`/
|
|
/// `with_counts` are baked into the already-built MPHF/unitigs/matrices
|
|
/// and can never change post-build, unlike the evidence bundle, which
|
|
/// is derived from `unitigs.bin` + `mphf.bin` alone and can be
|
|
/// rebuilt in a different form at any time (see
|
|
/// `obikindexer::IndexBuilder`).
|
|
///
|
|
/// The caller must refresh its own cached `Arc<IndexMeta>` afterward
|
|
/// (e.g. `self.meta = Arc::new(IndexMeta::open(self)?)`) — this method
|
|
/// only updates the file, it has no way to reach back into whatever
|
|
/// `KmerIndex` holds it.
|
|
pub fn set_evidence(&self, evidence: IndexMode, block_bits: u8) -> io::Result<()> {
|
|
let _guard = self.lock.write().unwrap();
|
|
let mut on_disk = Self::read_full(&self.root_path)?;
|
|
on_disk.config.evidence = evidence;
|
|
if matches!(on_disk.config.evidence, IndexMode::Exact) {
|
|
on_disk.config.block_bits = block_bits;
|
|
}
|
|
self.write_full(&on_disk)
|
|
}
|
|
|
|
pub fn set_state(&self, state: IndexState) -> io::Result<()> {
|
|
let _guard = self.lock.write().unwrap();
|
|
let mut on_disk = Self::read_full(&self.root_path)?;
|
|
on_disk.state = state;
|
|
self.write_full(&on_disk)
|
|
}
|
|
|
|
/// Mark scatter as complete (`IndexState::Scattered`).
|
|
///
|
|
/// If no genome label was set yet, one is derived from the index root
|
|
/// directory name (stripped of all extensions).
|
|
pub fn mark_scattered(&self) -> io::Result<()> {
|
|
let _guard = self.lock.write().unwrap();
|
|
let mut on_disk = Self::read_full(&self.root_path)?;
|
|
if on_disk.genomes.is_empty() {
|
|
let label = label_from_path(&self.root_path);
|
|
on_disk.genomes.push(GenomeInfo::new(label));
|
|
}
|
|
on_disk.state = IndexState::Scattered;
|
|
self.write_full(&on_disk)
|
|
}
|
|
|
|
pub fn mark_counted(&self) -> io::Result<()> {
|
|
let _guard = self.lock.write().unwrap();
|
|
let mut on_disk = Self::read_full(&self.root_path)?;
|
|
on_disk.state = IndexState::Counted;
|
|
self.write_full(&on_disk)
|
|
}
|
|
|
|
pub fn mark_indexed(&self) -> io::Result<()> {
|
|
let _guard = self.lock.write().unwrap();
|
|
let mut on_disk = Self::read_full(&self.root_path)?;
|
|
on_disk.state = IndexState::Indexed;
|
|
self.write_full(&on_disk)
|
|
}
|
|
|
|
// ── private ──────────────────────────────────────────────────────────────
|
|
|
|
fn read_full(root: &Path) -> io::Result<IndexMetadata> {
|
|
// One `read`, then parse from memory: `from_reader` on a bare `File`
|
|
// issues one syscall per byte (≈ 0.35 s for a 91-genome `index.meta`).
|
|
let bytes = fs::read(root.join(META_FILENAME))?;
|
|
serde_json::from_slice(&bytes).map_err(io::Error::other)
|
|
}
|
|
|
|
fn write_full(&self, on_disk: &IndexMetadata) -> io::Result<()> {
|
|
let file = fs::File::create(self.root_path.join(META_FILENAME))?;
|
|
serde_json::to_writer_pretty(file, on_disk).map_err(io::Error::other)
|
|
}
|
|
}
|
|
|
|
fn label_from_path(path: &Path) -> String {
|
|
let name = path
|
|
.file_name()
|
|
.unwrap_or(path.as_os_str())
|
|
.to_string_lossy()
|
|
.into_owned();
|
|
let mut s = name;
|
|
while let Some(pos) = s.rfind('.') {
|
|
s.truncate(pos);
|
|
}
|
|
if s.is_empty() {
|
|
"unknown".to_string()
|
|
} else {
|
|
s
|
|
}
|
|
}
|