refactor: migrate index metadata to on-disk JSON with fallible access

Migrate index state tracking from filesystem sentinel files to an on-disk JSON schema within `index.meta`. The `IndexMeta` struct is now wrapped in an `Arc` with internal locking, exposing only fallible methods for genome and state access. In-memory mutation capabilities have been removed, requiring callers to handle I/O errors explicitly and pass immutable references to downstream components like `PartitionRouter`. Public sentinel constants have been removed from exports.
This commit is contained in:
Eric Coissac committed 2026-08-21 21:51:42 +02:00
1 parent 02dbdd11aa
commit 8d6ba6546b
37 files changed
+573 -251

No files matched your search

@@ -79,7 +79,7 @@ impl Drop for GuardedIter {
/// whether) the caller displays them; rendering a spinner/progress bar is
/// the caller's decision, not this crate's.
pub struct PartitionRouter<'a> {
index: &'a mut KmerIndex,
index: &'a KmerIndex,
partitions_mask: u64,
writers: Vec<Option<SKFileWriter>>,
level: Level,
@@ -94,7 +94,7 @@ impl<'a> PartitionRouter<'a> {
/// Configure a router for `index`'s partition layout. Doesn't touch
/// disk by itself — partitions and their layer-0 shells are created
/// lazily, on the first super-kmer routed to each of them.
pub fn new(index: &'a mut KmerIndex) -> Self {
pub fn new(index: &'a KmerIndex) -> Self {
let n_bits = index.n_bits();
let n_partitions = 1usize << n_bits;
let workers = obisys::effective_parallelism();
+16 -44
View File
@@ -35,8 +35,7 @@ use obicompactvec::{PersistentCompactIntMatrix, PersistentCompactIntVec};
use obidebruinj::GraphDeBruijn;
use obikindex::layer::IndexMode;
use obikindex::layer::{TypedLayer, meta::PartitionMeta};
use obikindex::{GenomeInfo, KmerIndex, OKIError, OKIResult};
use obikindex::{SENTINEL_COUNTED, SENTINEL_INDEXED, SENTINEL_SCATTERED};
use obikindex::{KmerIndex, OKIError, OKIResult};
use obikindex::{materialize_layer, olm_to_sk, write_graph_as_unitigs};
use obiskio::{SKError, SKFileMeta, SKFileReader};
use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xx64};
@@ -44,16 +43,18 @@ use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xx64};
type Mphf = PtrHash<u64, CubicEps, CachelineEfVec<Vec<CachelineEf>>, Xx64, Vec<u8>>;
pub(crate) trait PrivateBuilder {
/// Mark scatter as complete and write `scatter.done`.
/// Mark scatter as complete (`IndexState::Scattered`).
///
/// If no genome label was set at creation time, one is derived from
/// the index root directory name (stripped of all extensions).
fn mark_scattered(&mut self) -> OKIResult<()>;
/// the index root directory name (stripped of all extensions). `&self`
/// — not `&mut self` anymore: nothing is cached in memory to mutate,
/// see `DevDocMD/implementation/partition_layer_cache.md`, "(11)".
fn mark_scattered(&self) -> OKIResult<()>;
/// Mark dereplicate+count as complete and write `count.done`.
/// Mark dereplicate+count as complete (`IndexState::Counted`).
fn mark_counted(&self) -> OKIResult<()>;
/// Mark layer construction as complete and write `index.done`.
/// Mark layer construction as complete (`IndexState::Indexed`).
fn mark_indexed(&self) -> OKIResult<()>;
/// Write `spectrums/{label}.json` from an already-computed kmer
@@ -89,33 +90,25 @@ pub(crate) trait PrivateBuilder {
}
impl PrivateBuilder for KmerIndex {
fn mark_scattered(&mut self) -> OKIResult<()> {
if self.meta().genomes.is_empty() {
let label = label_from_path(self.root_path());
self.meta_mut().genomes.push(GenomeInfo::new(label));
self.meta().write(self.root_path())?;
}
touch(&self.root_path().join(SENTINEL_SCATTERED))?;
Ok(())
fn mark_scattered(&self) -> OKIResult<()> {
self.meta().mark_scattered().map_err(OKIError::Io)
}
fn mark_counted(&self) -> OKIResult<()> {
touch(&self.root_path().join(SENTINEL_COUNTED))?;
Ok(())
self.meta().mark_counted().map_err(OKIError::Io)
}
fn mark_indexed(&self) -> OKIResult<()> {
touch(&self.root_path().join(SENTINEL_INDEXED))?;
Ok(())
self.meta().mark_indexed().map_err(OKIError::Io)
}
fn write_spectrum(&self, f0: u64, f1: u64, counts: &BTreeMap<u32, u64>) -> OKIResult<()> {
let label = self
.meta()
.genomes
let genomes = self.meta().genomes().map_err(OKIError::Io)?;
let label = genomes
.first()
.map(|g| g.label.as_str())
.unwrap_or("unknown");
.unwrap_or("unknown")
.to_owned();
let spectrums_dir = self.root_path().join("spectrums");
fs::create_dir_all(&spectrums_dir)?;
let path = spectrums_dir.join(format!("{label}.json"));
@@ -231,27 +224,6 @@ impl PrivateBuilder for KmerIndex {
// ── private helpers ──────────────────────────────────────────────────────
fn label_from_path(path: &Path) -> String {
let name = path
.file_name()
.unwrap_or(path.as_os_str())
.to_string_lossy()
.into_owned();
let mut s = name;
while let Some(pos) = s.rfind('.') {
s.truncate(pos);
}
if s.is_empty() {
"unknown".to_string()
} else {
s
}
}
fn touch(path: &Path) -> Result<(), io::Error> {
fs::File::create(path).map(|_| ())
}
fn remove_if_exists(path: &Path) {
if let Err(e) = fs::remove_file(path) {
if e.kind() != io::ErrorKind::NotFound {