refactor: migrate index metadata to on-disk JSON with fallible access
Migrate index state tracking from filesystem sentinel files to an on-disk JSON schema within `index.meta`. The `IndexMeta` struct is now wrapped in an `Arc` with internal locking, exposing only fallible methods for genome and state access. In-memory mutation capabilities have been removed, requiring callers to handle I/O errors explicitly and pass immutable references to downstream components like `PartitionRouter`. Public sentinel constants have been removed from exports.
This commit is contained in:
1 parent
02dbdd11aa
commit
8d6ba6546b
37 files changed
+573
-251
No files matched your search
@@ -79,7 +79,7 @@ impl Drop for GuardedIter {
|
||||
/// whether) the caller displays them; rendering a spinner/progress bar is
|
||||
/// the caller's decision, not this crate's.
|
||||
pub struct PartitionRouter<'a> {
|
||||
index: &'a mut KmerIndex,
|
||||
index: &'a KmerIndex,
|
||||
partitions_mask: u64,
|
||||
writers: Vec<Option<SKFileWriter>>,
|
||||
level: Level,
|
||||
@@ -94,7 +94,7 @@ impl<'a> PartitionRouter<'a> {
|
||||
/// Configure a router for `index`'s partition layout. Doesn't touch
|
||||
/// disk by itself — partitions and their layer-0 shells are created
|
||||
/// lazily, on the first super-kmer routed to each of them.
|
||||
pub fn new(index: &'a mut KmerIndex) -> Self {
|
||||
pub fn new(index: &'a KmerIndex) -> Self {
|
||||
let n_bits = index.n_bits();
|
||||
let n_partitions = 1usize << n_bits;
|
||||
let workers = obisys::effective_parallelism();
|
||||
|
||||
@@ -35,8 +35,7 @@ use obicompactvec::{PersistentCompactIntMatrix, PersistentCompactIntVec};
|
||||
use obidebruinj::GraphDeBruijn;
|
||||
use obikindex::layer::IndexMode;
|
||||
use obikindex::layer::{TypedLayer, meta::PartitionMeta};
|
||||
use obikindex::{GenomeInfo, KmerIndex, OKIError, OKIResult};
|
||||
use obikindex::{SENTINEL_COUNTED, SENTINEL_INDEXED, SENTINEL_SCATTERED};
|
||||
use obikindex::{KmerIndex, OKIError, OKIResult};
|
||||
use obikindex::{materialize_layer, olm_to_sk, write_graph_as_unitigs};
|
||||
use obiskio::{SKError, SKFileMeta, SKFileReader};
|
||||
use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xx64};
|
||||
@@ -44,16 +43,18 @@ use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xx64};
|
||||
type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xx64, Vec<u8>>;
|
||||
|
||||
pub(crate) trait PrivateBuilder {
|
||||
/// Mark scatter as complete and write `scatter.done`.
|
||||
/// Mark scatter as complete (`IndexState::Scattered`).
|
||||
///
|
||||
/// If no genome label was set at creation time, one is derived from
|
||||
/// the index root directory name (stripped of all extensions).
|
||||
fn mark_scattered(&mut self) -> OKIResult<()>;
|
||||
/// the index root directory name (stripped of all extensions). `&self`
|
||||
/// — not `&mut self` anymore: nothing is cached in memory to mutate,
|
||||
/// see `DevDocMD/implementation/partition_layer_cache.md`, "(11)".
|
||||
fn mark_scattered(&self) -> OKIResult<()>;
|
||||
|
||||
/// Mark dereplicate+count as complete and write `count.done`.
|
||||
/// Mark dereplicate+count as complete (`IndexState::Counted`).
|
||||
fn mark_counted(&self) -> OKIResult<()>;
|
||||
|
||||
/// Mark layer construction as complete and write `index.done`.
|
||||
/// Mark layer construction as complete (`IndexState::Indexed`).
|
||||
fn mark_indexed(&self) -> OKIResult<()>;
|
||||
|
||||
/// Write `spectrums/{label}.json` from an already-computed kmer
|
||||
@@ -89,33 +90,25 @@ pub(crate) trait PrivateBuilder {
|
||||
}
|
||||
|
||||
impl PrivateBuilder for KmerIndex {
|
||||
fn mark_scattered(&mut self) -> OKIResult<()> {
|
||||
if self.meta().genomes.is_empty() {
|
||||
let label = label_from_path(self.root_path());
|
||||
self.meta_mut().genomes.push(GenomeInfo::new(label));
|
||||
self.meta().write(self.root_path())?;
|
||||
}
|
||||
touch(&self.root_path().join(SENTINEL_SCATTERED))?;
|
||||
Ok(())
|
||||
fn mark_scattered(&self) -> OKIResult<()> {
|
||||
self.meta().mark_scattered().map_err(OKIError::Io)
|
||||
}
|
||||
|
||||
fn mark_counted(&self) -> OKIResult<()> {
|
||||
touch(&self.root_path().join(SENTINEL_COUNTED))?;
|
||||
Ok(())
|
||||
self.meta().mark_counted().map_err(OKIError::Io)
|
||||
}
|
||||
|
||||
fn mark_indexed(&self) -> OKIResult<()> {
|
||||
touch(&self.root_path().join(SENTINEL_INDEXED))?;
|
||||
Ok(())
|
||||
self.meta().mark_indexed().map_err(OKIError::Io)
|
||||
}
|
||||
|
||||
fn write_spectrum(&self, f0: u64, f1: u64, counts: &BTreeMap<u32, u64>) -> OKIResult<()> {
|
||||
let label = self
|
||||
.meta()
|
||||
.genomes
|
||||
let genomes = self.meta().genomes().map_err(OKIError::Io)?;
|
||||
let label = genomes
|
||||
.first()
|
||||
.map(|g| g.label.as_str())
|
||||
.unwrap_or("unknown");
|
||||
.unwrap_or("unknown")
|
||||
.to_owned();
|
||||
let spectrums_dir = self.root_path().join("spectrums");
|
||||
fs::create_dir_all(&spectrums_dir)?;
|
||||
let path = spectrums_dir.join(format!("{label}.json"));
|
||||
@@ -231,27 +224,6 @@ impl PrivateBuilder for KmerIndex {
|
||||
|
||||
// ── private helpers ──────────────────────────────────────────────────────
|
||||
|
||||
fn label_from_path(path: &Path) -> String {
|
||||
let name = path
|
||||
.file_name()
|
||||
.unwrap_or(path.as_os_str())
|
||||
.to_string_lossy()
|
||||
.into_owned();
|
||||
let mut s = name;
|
||||
while let Some(pos) = s.rfind('.') {
|
||||
s.truncate(pos);
|
||||
}
|
||||
if s.is_empty() {
|
||||
"unknown".to_string()
|
||||
} else {
|
||||
s
|
||||
}
|
||||
}
|
||||
|
||||
fn touch(path: &Path) -> Result<(), io::Error> {
|
||||
fs::File::create(path).map(|_| ())
|
||||
}
|
||||
|
||||
fn remove_if_exists(path: &Path) {
|
||||
if let Err(e) = fs::remove_file(path) {
|
||||
if e.kind() != io::ErrorKind::NotFound {
|
||||
|
||||
Reference in new issue
Block a user