refactor: migrate index metadata to on-disk JSON with fallible access

Migrate index state tracking from filesystem sentinel files to an on-disk JSON schema within `index.meta`. The `IndexMeta` struct is now wrapped in an `Arc` with internal locking, exposing only fallible methods for genome and state access. In-memory mutation capabilities have been removed, requiring callers to handle I/O errors explicitly and pass immutable references to downstream components like `PartitionRouter`. Public sentinel constants have been removed from exports.

src/obikindex/src/index/builder.rs

@@ -15,18 +15,19 @@
 //! there and covers a different set of methods — the ones exclusive to
 //! the four-stage build pipeline, never needed by `merge`/`select`/
 //! `rebuild`/`reindex`. See
-//! `DevDocMD/implementation/partition_layer_cache.md`, "(8)"/"(9)".
+//! `DevDocMD/implementation/partition_layer_cache.md`, "(8)"/"(9)"/"(11)".
 
 use std::fs;
 use std::path::Path;
+use std::sync::Arc;
 
 use obisys::Reporter;
 use obisys::Stage;
 
 use crate::index::error::{OKIError, OKIResult};
 use crate::index::kmer_index::KmerIndex;
-use crate::index::meta::IndexMeta;
-use crate::index::state::{IndexState, SENTINEL_INDEXED};
+use crate::index::meta::{GenomeInfo, IndexConfig, IndexMeta};
+use crate::index::state::IndexState;
 
 pub trait IndexBuilder: Sized {
     /// Make `output` ready to receive a newly created index.
@@ -37,12 +38,13 @@ pub trait IndexBuilder: Sized {
     fn clear_output_for_create<P: AsRef<Path>>(output: P, force: bool) -> OKIResult<()>;
 
     /// Lay out a fresh index skeleton at `output`: the root directory,
-    /// `index.meta` (from `meta`), and an empty partition layout.
+    /// `index.meta` (from `config` + an initial `genomes` list — often
+    /// empty), and an empty partition layout.
     ///
     /// For construction paths that build partitions from scratch (`select`,
     /// `rebuild`). `merge` bootstraps by copying a source index instead, so
     /// it does not use this.
-    fn create_skeleton<P: AsRef<Path>>(output: P, meta: &IndexMeta) -> OKIResult<Self>;
+    fn create_skeleton<P: AsRef<Path>>(output: P, config: IndexConfig, genomes: Vec<GenomeInfo>) -> OKIResult<Self>;
 
     /// Mark `output` as fully indexed, pack its column matrices, and reopen it.
     ///
@@ -50,8 +52,10 @@ pub trait IndexBuilder: Sized {
     /// partitions have been written.
     fn finalize_indexed<P: AsRef<Path>>(output: P, rep: &mut Reporter) -> OKIResult<Self>;
 
-    /// Current construction state, as reported by sentinel files on disk.
-    fn state(&self) -> IndexState;
+    /// Current construction state, read fresh from `index.meta` (see
+    /// `DevDocMD/implementation/partition_layer_cache.md`, "(11)" — no
+    /// longer detected from sentinel files, so this is now fallible).
+    fn state(&self) -> OKIResult<IndexState>;
 }
 
 impl IndexBuilder for KmerIndex {
@@ -70,24 +74,24 @@ impl IndexBuilder for KmerIndex {
         Ok(())
     }
 
-    fn create_skeleton<P: AsRef<Path>>(output: P, meta: &IndexMeta) -> OKIResult<Self> {
+    fn create_skeleton<P: AsRef<Path>>(output: P, config: IndexConfig, genomes: Vec<GenomeInfo>) -> OKIResult<Self> {
         let output = output.as_ref();
         fs::create_dir_all(output).map_err(OKIError::Io)?;
-        meta.write(output).map_err(OKIError::Io)?;
-        Ok(KmerIndex { root_path: output.to_owned(), meta: meta.clone() })
+        let meta = IndexMeta::create_at(output, config, genomes).map_err(OKIError::Io)?;
+        Ok(KmerIndex { root_path: output.to_owned(), meta: Arc::new(meta) })
     }
 
     fn finalize_indexed<P: AsRef<Path>>(output: P, rep: &mut Reporter) -> OKIResult<Self> {
         let output = output.as_ref();
-        fs::File::create(output.join(SENTINEL_INDEXED)).map_err(OKIError::Io)?;
         let idx = KmerIndex::open(output)?;
+        idx.meta.mark_indexed().map_err(OKIError::Io)?;
         let t_pack = Stage::start("pack");
         idx.pack_matrices(false)?;
         rep.push(t_pack.stop());
         Ok(idx)
     }
 
-    fn state(&self) -> IndexState {
-        IndexState::detect(&self.root_path).unwrap_or(IndexState::Empty)
+    fn state(&self) -> OKIResult<IndexState> {
+        Ok(self.meta.state().map_err(OKIError::Io)?)
     }
 }

src/obikindex/src/index/distance.rs

@@ -55,7 +55,7 @@ impl DistanceMetric {
 
 impl KmerIndex {
     pub fn distance(&self, metric: DistanceMetric, shared_kmers: bool, presence_threshold: u32) -> OKIResult<DistanceOutput> {
-        let n_genomes = self.meta.genomes.len();
+        let n_genomes = self.meta.genomes().map_err(OKIError::Io)?.len();
         if n_genomes < 2 {
             return Err(OKIError::InvalidInput(
                 "distance requires at least 2 genomes in the index".into(),

src/obikindex/src/index/dump.rs

@@ -30,7 +30,7 @@ impl KmerIndex {
         filters: &[Box<dyn KmerFilter>],
         on_partition: F,
     ) -> OKIResult<()> {
-        let genomes    = &self.meta.genomes;
+        let genomes    = self.meta.genomes().map_err(OKIError::Io)?;
         let use_counts = self.meta.config.with_counts && !force_presence;
         let n_genomes  = genomes.len().max(1);
         let kmer_size  = self.kmer_size();
@@ -40,7 +40,7 @@ impl KmerIndex {
             write!(out, "partition,layer,")?;
         }
         write!(out, "kmer")?;
-        for g in genomes {
+        for g in &genomes {
             write!(out, ",{}", g.label)?;
         }
         writeln!(out)?;

src/obikindex/src/index/kmer_index.rs

@@ -1,5 +1,6 @@
 use std::fs;
 use std::path::{Path, PathBuf};
+use std::sync::Arc;
 
 use crate::layer::meta::PartitionMeta;
 use obisys::progress_bar;
@@ -13,7 +14,7 @@ use crate::index::meta::{GenomeInfo, IndexConfig, IndexMeta};
 
 pub struct KmerIndex {
     pub(crate) root_path: PathBuf,
-    pub(crate) meta: IndexMeta,
+    pub(crate) meta: Arc<IndexMeta>,
 }
 
 impl KmerIndex {
@@ -30,20 +31,17 @@ impl KmerIndex {
         fs::create_dir_all(&root_path).map_err(OKIError::Io)?;
         set_k(config.kmer_size);
         set_m(config.minimizer_size);
-        let mut meta = IndexMeta::new(config);
-        if let Some(info) = genome_info {
-            meta.genomes.push(info);
-        }
-        meta.write(&root_path)?;
-        Ok(Self { root_path, meta })
+        let genomes = genome_info.into_iter().collect();
+        let meta = IndexMeta::create_at(&root_path, config, genomes).map_err(OKIError::Io)?;
+        Ok(Self { root_path, meta: Arc::new(meta) })
     }
 
     pub fn open<P: AsRef<Path>>(path: P) -> OKIResult<Self> {
         let root_path = path.as_ref().to_owned();
-        let meta = IndexMeta::read(&root_path).map_err(OKIError::Io)?;
+        let meta = IndexMeta::open_at(&root_path).map_err(OKIError::Io)?;
         set_k(meta.config.kmer_size);
         set_m(meta.config.minimizer_size);
-        Ok(Self { root_path, meta })
+        Ok(Self { root_path, meta: Arc::new(meta) })
     }
 
     /// Return `true` if `path` contains an `index.meta` file.
@@ -55,11 +53,8 @@ impl KmerIndex {
     pub fn root_path(&self) -> &Path {
         &self.root_path
     }
-    pub fn meta(&self) -> &IndexMeta {
-        &self.meta
-    }
-    pub fn meta_mut(&mut self) -> &mut IndexMeta {
-        &mut self.meta
+    pub fn meta(&self) -> Arc<IndexMeta> {
+        Arc::clone(&self.meta)
     }
     pub fn kmer_size(&self) -> usize {
         self.meta.config.kmer_size
@@ -91,8 +86,10 @@ impl KmerIndex {
     pub fn block_bits(&self) -> u8 {
         self.meta.config.block_bits
     }
-    pub fn genomes(&self) -> &[GenomeInfo] {
-        &self.meta.genomes
+    /// Genome list — reads `index.meta` fresh (see `IndexMeta::genomes`,
+    /// this can legitimately change during a run, unlike `config`).
+    pub fn genomes(&self) -> OKIResult<Vec<GenomeInfo>> {
+        Ok(self.meta.genomes().map_err(OKIError::Io)?)
     }
     pub fn n_partitions(&self) -> usize {
         1usize << self.meta.config.n_bits
@@ -256,5 +253,3 @@ impl KmerIndex {
         Ok(())
     }
 }
-
-

src/obikindex/src/index/merge.rs

@@ -13,7 +13,7 @@ use crate::layer::IndexMode;
 use crate::index::error::{OKIError, OKIResult};
 use crate::index::kmer_index::KmerIndex;
 use crate::index::meta::{GenomeInfo, IndexMeta};
-use crate::index::state::{IndexState, SENTINEL_INDEXED};
+use crate::index::state::IndexState;
 
 pub use crate::index::merge_layer::MergeMode;
 
@@ -50,7 +50,7 @@ impl KmerIndex {
         // ── Validate config compatibility ─────────────────────────────────────
         let ref0 = sources[0];
         for src in sources {
-            if src.state() != IndexState::Indexed {
+            if src.state()? != IndexState::Indexed {
                 return Err(OKIError::NotIndexed(src.root_path.clone()));
             }
             if src.kmer_size() != ref0.kmer_size()
@@ -64,6 +64,12 @@ impl KmerIndex {
             }
         }
 
+        // Read each source's genome list once — `IndexMeta::genomes` is a
+        // fresh disk read every call, so cache it rather than re-reading it
+        // repeatedly through the rest of this function.
+        let src_genomes: Vec<Vec<GenomeInfo>> =
+            sources.iter().map(|s| s.genomes()).collect::<OKIResult<_>>()?;
+
         // ── Log source characteristics and choose base ────────────────────────
         let mode_str = if mode == MergeMode::Presence {
             "presence"
@@ -77,12 +83,12 @@ impl KmerIndex {
             mode_str,
         );
         for (i, src) in sources.iter().enumerate() {
-            let genome_str = if src.meta.genomes.len() == 1 {
+            let genome_str = if src_genomes[i].len() == 1 {
                 "mono-genome".to_string()
             } else {
-                format!("{} genomes", src.meta.genomes.len())
+                format!("{} genomes", src_genomes[i].len())
             };
-            let trivial_str = if is_trivial(src, mode) {
+            let trivial_str = if is_trivial(&src_genomes[i], mode) {
                 " [trivial: no data approximation]"
             } else {
                 ""
@@ -98,9 +104,9 @@ impl KmerIndex {
             );
         }
 
-        let base_idx = choose_base(sources, mode);
-        let needs_approx = sources.iter().any(|src| {
-            !is_trivial(src, mode)
+        let base_idx = choose_base(sources, &src_genomes, mode);
+        let needs_approx = sources.iter().enumerate().any(|(i, src)| {
+            !is_trivial(&src_genomes[i], mode)
                 && matches!(
                     src.meta.config.evidence,
                     IndexMode::Approx { .. } | IndexMode::Hybrid { .. }
@@ -119,17 +125,21 @@ impl KmerIndex {
         );
 
         let mut ordered: Vec<&KmerIndex> = Vec::with_capacity(sources.len());
+        let mut ordered_genomes: Vec<Vec<GenomeInfo>> = Vec::with_capacity(sources.len());
         ordered.push(sources[base_idx]);
+        ordered_genomes.push(src_genomes[base_idx].clone());
         for (i, &src) in sources.iter().enumerate() {
             if i != base_idx {
                 ordered.push(src);
+                ordered_genomes.push(src_genomes[i].clone());
             }
         }
         let sources: &[&KmerIndex] = &ordered;
+        let src_genomes = ordered_genomes;
         let evidence = sources[0].meta.config.evidence.clone();
 
         // ── Compute final genome labels ────────────────────────────────────────
-        let (source_labels, all_genomes) = compute_labels(sources, rename_duplicates)?;
+        let (source_labels, all_genomes) = compute_labels(&src_genomes, rename_duplicates)?;
 
         // ── Prepare output directory ──────────────────────────────────────────
         KmerIndex::clear_output_for_create(output, force)?;
@@ -139,18 +149,18 @@ impl KmerIndex {
             "bootstrap: copying {} → {} ({} genome(s))",
             sources[0].root_path.display(),
             output.display(),
-            sources[0].meta.genomes.len(),
+            src_genomes[0].len(),
         );
         let t = Stage::start("bootstrap");
         let pb = spinner("bootstrap");
         pb.set_message("copying index …");
         copy_dir_all(&sources[0].root_path, output)?;
 
-        let mut meta = IndexMeta::read(output).map_err(OKIError::Io)?;
-        meta.genomes = all_genomes;
-        meta.config.with_counts = mode == MergeMode::Count;
-        meta.config.evidence = evidence.clone();
-        meta.write(output)?;
+        let dst_meta = IndexMeta::open_at(output).map_err(OKIError::Io)?;
+        let mut config = dst_meta.config.clone();
+        config.with_counts = mode == MergeMode::Count;
+        config.evidence = evidence.clone();
+        dst_meta.rewrite_config(config, all_genomes).map_err(OKIError::Io)?;
 
         if mode == MergeMode::Presence {
             remove_dirs_named(output, "counts")?;
@@ -167,9 +177,8 @@ impl KmerIndex {
         if spectrums_dir.exists() {
             fs::remove_dir_all(&spectrums_dir)?;
         }
-        for (src, new_labels) in sources.iter().zip(&source_labels) {
-            let old_labels: Vec<String> =
-                src.meta.genomes.iter().map(|g| g.label.clone()).collect();
+        for ((src, new_labels), genomes) in sources.iter().zip(&source_labels).zip(&src_genomes) {
+            let old_labels: Vec<String> = genomes.iter().map(|g| g.label.clone()).collect();
             copy_spectrums(&src.root_path, output, &old_labels, new_labels)?;
         }
         pb.finish_and_clear();
@@ -178,12 +187,12 @@ impl KmerIndex {
         // ── Open destination ──────────────────────────────────────────────────
         let dst = KmerIndex::open(output)?;
         let n_partitions = dst.n_partitions();
-        let n_dst_genomes = sources[0].meta.genomes.len();
+        let n_dst_genomes = src_genomes[0].len();
 
         // ── Merge partitions ──────────────────────────────────────────────────
         let remaining_sources: Vec<&KmerIndex> = sources[1..].to_vec();
         if !remaining_sources.is_empty() {
-            let n_src_genomes: usize = remaining_sources.iter().map(|s| s.meta.genomes.len()).sum();
+            let n_src_genomes: usize = src_genomes[1..].iter().map(|g| g.len()).sum();
             info!(
                 "merging {} partition(s) × {} additional source genome(s) into {} destination genome(s)",
                 n_partitions, n_src_genomes, n_dst_genomes,
@@ -196,7 +205,8 @@ impl KmerIndex {
             // Pre-build source list once (avoid rebuilding per partition)
             let srcs: Vec<(&KmerIndex, usize)> = remaining_sources
                 .iter()
-                .map(|s| (*s, s.meta.genomes.len()))
+                .zip(&src_genomes[1..])
+                .map(|(s, g)| (*s, g.len()))
                 .collect();
 
             // Per-partition unitig byte sizes across remaining sources (stat() only)
@@ -266,12 +276,11 @@ impl KmerIndex {
             pb.set_message("consolidating column files …");
             let dst2 = KmerIndex::open(output)?;
             dst2.pack_matrices(false)?;
+            dst2.meta.mark_indexed().map_err(OKIError::Io)?;
             pb.finish_and_clear();
             rep.push(t.stop());
         }
 
-        fs::File::create(output.join(SENTINEL_INDEXED)).map_err(OKIError::Io)?;
-
         KmerIndex::open(output)
     }
 }
@@ -341,16 +350,16 @@ fn partition_unitig_bytes(src: &KmerIndex, i: usize) -> u64 {
 }
 
 fn compute_labels(
-    sources: &[&KmerIndex],
+    src_genomes: &[Vec<GenomeInfo>],
     rename_duplicates: bool,
 ) -> OKIResult<(Vec<Vec<String>>, Vec<GenomeInfo>)> {
     let mut seen: HashMap<String, usize> = HashMap::new();
-    let mut source_labels: Vec<Vec<String>> = Vec::with_capacity(sources.len());
+    let mut source_labels: Vec<Vec<String>> = Vec::with_capacity(src_genomes.len());
     let mut all_genomes: Vec<GenomeInfo> = Vec::new();
 
-    for src in sources {
-        let mut labels = Vec::with_capacity(src.meta.genomes.len());
-        for genome in &src.meta.genomes {
+    for genomes in src_genomes {
+        let mut labels = Vec::with_capacity(genomes.len());
+        for genome in genomes {
             let label = &genome.label;
             let count = seen.entry(label.clone()).or_insert(0);
             let new_label = if *count == 0 {
@@ -414,8 +423,8 @@ fn format_evidence(ev: &IndexMode) -> String {
     }
 }
 
-fn is_trivial(src: &KmerIndex, mode: MergeMode) -> bool {
-    src.meta.genomes.len() == 1 && mode == MergeMode::Presence
+fn is_trivial(genomes: &[GenomeInfo], mode: MergeMode) -> bool {
+    genomes.len() == 1 && mode == MergeMode::Presence
 }
 
 fn index_unitig_size(src: &KmerIndex) -> u64 {
@@ -423,9 +432,9 @@ fn index_unitig_size(src: &KmerIndex) -> u64 {
     (0..n).map(|i| partition_unitig_bytes(src, i)).sum()
 }
 
-fn choose_base(sources: &[&KmerIndex], mode: MergeMode) -> usize {
-    let needs_approx = sources.iter().any(|src| {
-        !is_trivial(src, mode)
+fn choose_base(sources: &[&KmerIndex], src_genomes: &[Vec<GenomeInfo>], mode: MergeMode) -> usize {
+    let needs_approx = sources.iter().enumerate().any(|(i, src)| {
+        !is_trivial(&src_genomes[i], mode)
             && matches!(
                 src.meta.config.evidence,
                 IndexMode::Approx { .. } | IndexMode::Hybrid { .. }

src/obikindex/src/index/meta.rs

@@ -1,8 +1,11 @@
 use std::collections::HashMap;
 use std::fs;
 use std::io;
-use std::path::Path;
+use std::path::{Path, PathBuf};
+use std::sync::RwLock;
 
+use crate::index::kmer_index::KmerIndex;
+use crate::index::state::IndexState;
 use crate::layer::IndexMode;
 use serde::{Deserialize, Serialize};
 
@@ -23,6 +26,9 @@ impl GenomeInfo {
     }
 }
 
+/// Fixed at index creation, never changes afterward — freely mutable while
+/// being assembled (before any `IndexMeta::create` call), immutable once an
+/// `IndexMeta` wraps it.
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct IndexConfig {
     pub kmer_size: usize,
@@ -38,36 +44,208 @@ pub struct IndexConfig {
     pub block_bits: u8,
 }
 
+/// On-disk shape of `index.meta` — the whole file, read/written atomically
+/// as one JSON blob by every `IndexMeta` operation below.
 #[derive(Debug, Clone, Serialize, Deserialize)]
+struct OnDiskMeta {
+    version: u32,
+    config: IndexConfig,
+    #[serde(default)]
+    genomes: Vec<GenomeInfo>,
+    #[serde(default)]
+    state: IndexState,
+}
+
+/// Stateless accessor for one index's `index.meta` — `config` is the only
+/// part cached in memory (fixed forever once written; re-reading it would
+/// only ever return the same value), everything else (`genomes`, `state`)
+/// is read fresh from disk on every access and written fresh on every
+/// change, since it can legitimately change during a run — see
+/// `DevDocMD/implementation/partition_layer_cache.md`.
+///
+/// One instance per index, held as `Arc<IndexMeta>` by `KmerIndex` and
+/// shared by every borrower — the internal `lock` only serialises access
+/// *through that one shared instance*; it is not a cross-process file lock
+/// (that's `obisys::DirLock`, held by `cmd/index` for the whole build).
+/// Mutating methods hold the write lock across the full read-modify-write
+/// sequence, not just the write, so two concurrent callers can't silently
+/// clobber each other's change.
 pub struct IndexMeta {
-    pub version: u32,
+    root_path: PathBuf,
+    version: u32,
     pub config: IndexConfig,
-    /// Ordered list of genomes indexed here (label + optional categorical metadata).
-    pub genomes: Vec<GenomeInfo>,
+    lock: RwLock<()>,
 }
 
 impl IndexMeta {
-    pub fn new(config: IndexConfig) -> Self {
-        Self { version: META_VERSION, config, genomes: Vec::new() }
+    /// Create a brand-new `index.meta` for `index`, from fixed config and
+    /// an initial genome list (often empty — genomes are usually added
+    /// later via [`push_genome`](Self::push_genome)).
+    pub fn create(index: &KmerIndex, config: IndexConfig, genomes: Vec<GenomeInfo>) -> io::Result<Self> {
+        Self::create_at(index.root_path(), config, genomes)
     }
 
-    pub fn write(&self, root: &Path) -> io::Result<()> {
-        let file = fs::File::create(root.join(META_FILENAME))?;
-        serde_json::to_writer_pretty(file, self).map_err(io::Error::other)
+    /// Same as [`create`](Self::create), taking a raw path — for
+    /// `KmerIndex::create` itself, which doesn't have a complete
+    /// `KmerIndex` yet to hand in (it's what's being built).
+    pub(crate) fn create_at(root_path: &Path, config: IndexConfig, genomes: Vec<GenomeInfo>) -> io::Result<Self> {
+        let meta = Self {
+            root_path: root_path.to_owned(),
+            version: META_VERSION,
+            config,
+            lock: RwLock::new(()),
+        };
+        meta.write_full(&OnDiskMeta {
+            version: meta.version,
+            config: meta.config.clone(),
+            genomes,
+            state: IndexState::Empty,
+        })?;
+        Ok(meta)
     }
 
-    pub fn read(root: &Path) -> io::Result<Self> {
-        let file = fs::File::open(root.join(META_FILENAME))?;
-        serde_json::from_reader(file).map_err(io::Error::other)
+    /// Reopen the `index.meta` already on disk for `index`.
+    pub fn open(index: &KmerIndex) -> io::Result<Self> {
+        Self::open_at(index.root_path())
+    }
+
+    pub(crate) fn open_at(root_path: &Path) -> io::Result<Self> {
+        let on_disk = Self::read_full(root_path)?;
+        Ok(Self {
+            root_path: root_path.to_owned(),
+            version: on_disk.version,
+            config: on_disk.config,
+            lock: RwLock::new(()),
+        })
     }
 
     pub fn exists(root: &Path) -> bool {
         root.join(META_FILENAME).exists()
     }
 
-    /// Iterate over genome labels only.
-    pub fn genome_labels(&self) -> impl Iterator<Item = &str> {
-        self.genomes.iter().map(|g| g.label.as_str())
+    // ── immutable, cached — no disk access ──────────────────────────────────
+
+    pub fn version(&self) -> u32 {
+        self.version
+    }
+
+    pub fn config(&self) -> &IndexConfig {
+        &self.config
+    }
+
+    // ── variable, stateless — every call reads or writes `index.meta` fresh ─
+
+    pub fn genomes(&self) -> io::Result<Vec<GenomeInfo>> {
+        let _guard = self.lock.read().unwrap();
+        Ok(Self::read_full(&self.root_path)?.genomes)
+    }
+
+    pub fn state(&self) -> io::Result<IndexState> {
+        let _guard = self.lock.read().unwrap();
+        Ok(Self::read_full(&self.root_path)?.state)
+    }
+
+    pub fn push_genome(&self, info: GenomeInfo) -> io::Result<()> {
+        let _guard = self.lock.write().unwrap();
+        let mut on_disk = Self::read_full(&self.root_path)?;
+        on_disk.genomes.push(info);
+        self.write_full(&on_disk)
+    }
+
+    pub fn rename_genome(&self, pos: usize, new_label: impl Into<String>) -> io::Result<()> {
+        let _guard = self.lock.write().unwrap();
+        let mut on_disk = Self::read_full(&self.root_path)?;
+        on_disk.genomes[pos].label = new_label.into();
+        self.write_full(&on_disk)
+    }
+
+    /// Replace the whole genome list at once — for construction paths
+    /// (`select`/`merge`) that compute the full output list up front
+    /// rather than pushing one at a time.
+    pub fn set_genomes(&self, genomes: Vec<GenomeInfo>) -> io::Result<()> {
+        let _guard = self.lock.write().unwrap();
+        let mut on_disk = Self::read_full(&self.root_path)?;
+        on_disk.genomes = genomes;
+        self.write_full(&on_disk)
+    }
+
+    /// Overwrite `config` and the whole `genomes` list at once, preserving
+    /// `state`. `config` otherwise never changes once an index exists —
+    /// this is the deliberate, rare exception for construction paths that
+    /// legitimately rewrite it in place (`select_in_place`, `reindex`).
+    /// The caller must refresh its own cached `Arc<IndexMeta>` afterward
+    /// (e.g. `self.meta = Arc::new(IndexMeta::open(self)?)`) — this method
+    /// only updates the file, it has no way to reach back into whatever
+    /// `KmerIndex` holds it.
+    pub fn rewrite_config(&self, config: IndexConfig, genomes: Vec<GenomeInfo>) -> io::Result<()> {
+        let _guard = self.lock.write().unwrap();
+        let state = Self::read_full(&self.root_path)?.state;
+        self.write_full(&OnDiskMeta { version: self.version, config, genomes, state })
+    }
+
+    pub fn set_state(&self, state: IndexState) -> io::Result<()> {
+        let _guard = self.lock.write().unwrap();
+        let mut on_disk = Self::read_full(&self.root_path)?;
+        on_disk.state = state;
+        self.write_full(&on_disk)
+    }
+
+    /// Mark scatter as complete (`IndexState::Scattered`).
+    ///
+    /// If no genome label was set yet, one is derived from the index root
+    /// directory name (stripped of all extensions).
+    pub fn mark_scattered(&self) -> io::Result<()> {
+        let _guard = self.lock.write().unwrap();
+        let mut on_disk = Self::read_full(&self.root_path)?;
+        if on_disk.genomes.is_empty() {
+            let label = label_from_path(&self.root_path);
+            on_disk.genomes.push(GenomeInfo::new(label));
+        }
+        on_disk.state = IndexState::Scattered;
+        self.write_full(&on_disk)
+    }
+
+    pub fn mark_counted(&self) -> io::Result<()> {
+        let _guard = self.lock.write().unwrap();
+        let mut on_disk = Self::read_full(&self.root_path)?;
+        on_disk.state = IndexState::Counted;
+        self.write_full(&on_disk)
+    }
+
+    pub fn mark_indexed(&self) -> io::Result<()> {
+        let _guard = self.lock.write().unwrap();
+        let mut on_disk = Self::read_full(&self.root_path)?;
+        on_disk.state = IndexState::Indexed;
+        self.write_full(&on_disk)
+    }
+
+    // ── private ──────────────────────────────────────────────────────────────
+
+    fn read_full(root: &Path) -> io::Result<OnDiskMeta> {
+        let file = fs::File::open(root.join(META_FILENAME))?;
+        serde_json::from_reader(file).map_err(io::Error::other)
+    }
+
+    fn write_full(&self, on_disk: &OnDiskMeta) -> io::Result<()> {
+        let file = fs::File::create(self.root_path.join(META_FILENAME))?;
+        serde_json::to_writer_pretty(file, on_disk).map_err(io::Error::other)
+    }
+}
+
+fn label_from_path(path: &Path) -> String {
+    let name = path
+        .file_name()
+        .unwrap_or(path.as_os_str())
+        .to_string_lossy()
+        .into_owned();
+    let mut s = name;
+    while let Some(pos) = s.rfind('.') {
+        s.truncate(pos);
+    }
+    if s.is_empty() {
+        "unknown".to_string()
+    } else {
+        s
     }
 }
 

src/obikindex/src/index/mod.rs

@@ -34,6 +34,6 @@ pub use meta::{validate_label, GenomeInfo, IndexConfig, IndexMeta, META_FILENAME
 pub use predicate::{GroupFilterParams, MetaPred};
 pub use query_layer::{KmerDesc, QueryHit, QueryStats};
 pub use select_layer::{AggOp, OutputCol};
-pub use state::{IndexState, SENTINEL_COUNTED, SENTINEL_INDEXED, SENTINEL_SCATTERED};
+pub use state::IndexState;
 pub use stats::IndexBitsPerKmer;
 pub use numa::PartitionRunner;

src/obikindex/src/index/predicate.rs

@@ -157,7 +157,8 @@ impl IndexMeta {
     /// Returns indices of genomes matching `pred_str` (single predicate).
     pub fn matching_genome_indices(&self, pred_str: &str) -> Result<Vec<usize>, String> {
         let pred = MetaPred::parse(pred_str)?;
-        Ok(self.genomes.iter().enumerate()
+        let genomes = self.genomes().map_err(|e| e.to_string())?;
+        Ok(genomes.iter().enumerate()
             .filter_map(|(i, g)| {
                 if g.matches(&pred) == Some(true) { Some(i) } else { std::option::Option::None }
             })
@@ -176,10 +177,11 @@ impl IndexMeta {
         outgroup_preds: &[MetaPred],
         p:              GroupFilterParams,
     ) -> Result<GroupQuorumFilter, String> {
+        let genomes = self.genomes().map_err(|e| e.to_string())?;
         let (ingroup_idx, outgroup_idx) = if ingroup_preds.is_empty() && outgroup_preds.is_empty() {
-            ((0..self.genomes.len()).collect(), vec![])
+            ((0..genomes.len()).collect(), vec![])
         } else {
-            let members = classify(&self.genomes, ingroup_preds, outgroup_preds);
+            let members = classify(&genomes, ingroup_preds, outgroup_preds);
             let in_idx: Vec<usize>  = members.iter().enumerate()
                 .filter(|(_, m)| matches!(m, Membership::Ingroup))
                 .map(|(i, _)| i).collect();

src/obikindex/src/index/rebuild.rs

@@ -7,7 +7,6 @@ use tracing::info;
 
 use crate::index::error::{OKIError, OKIResult};
 use crate::index::kmer_index::KmerIndex;
-use crate::index::meta::IndexMeta;
 use crate::index::state::IndexState;
 
 impl KmerIndex {
@@ -29,7 +28,7 @@ impl KmerIndex {
     ) -> OKIResult<Self> {
         let output = output.as_ref();
 
-        if src.state() != IndexState::Indexed {
+        if src.state()? != IndexState::Indexed {
             return Err(OKIError::NotIndexed(src.root_path.clone()));
         }
 
@@ -42,15 +41,16 @@ impl KmerIndex {
         KmerIndex::clear_output_for_create(output, force)?;
 
         // ── Create output directory + metadata ────────────────────────────────
-        let mut meta = IndexMeta::new(src.meta.config.clone());
-        meta.config.with_counts = mode == MergeMode::Count;
-        meta.genomes = src.meta.genomes.clone();
+        let mut config = src.meta.config.clone();
+        config.with_counts = mode == MergeMode::Count;
+        let genomes = src.genomes()?;
 
-        let n_genomes = src.meta.genomes.len();
+        let n_genomes = genomes.len();
         let n_partitions = src.n_partitions();
+        let block_bits = config.block_bits;
 
         // ── Create an empty destination KmerPartition ─────────────────────────
-        let dst_partition = KmerIndex::create_skeleton(output, &meta)?;
+        let dst_partition = KmerIndex::create_skeleton(output, config, genomes)?;
 
         info!(
             "rebuild: {} partition(s), {} genome(s), mode={:?}",
@@ -60,8 +60,6 @@ impl KmerIndex {
         let t = Stage::start("rebuild");
         let pb = progress_bar("rebuild", n_partitions as u64, "partitions");
 
-        let block_bits = meta.config.block_bits;
-
         let order: Vec<usize> = (0..n_partitions).collect();
         let runner = crate::index::numa::PartitionRunner::new();
         runner.run(

src/obikindex/src/index/reindex.rs

@@ -1,8 +1,10 @@
 use crate::index::builder::IndexBuilder;
+use crate::index::meta::IndexMeta;
 use crate::layer::{IndexMode, TypedLayer};
 use obisys::{Reporter, Stage, progress_bar};
 use std::fs;
 use std::path::Path;
+use std::sync::Arc;
 use tracing::info;
 
 use crate::index::error::{OKIError, OKIResult};
@@ -31,7 +33,7 @@ impl KmerIndex {
         block_bits: u8,
         rep: &mut Reporter,
     ) -> OKIResult<()> {
-        if self.state() != IndexState::Indexed {
+        if self.state()? != IndexState::Indexed {
             return Err(OKIError::NotIndexed(self.root_path.clone()));
         }
 
@@ -59,11 +61,14 @@ impl KmerIndex {
 
         pb.finish_and_clear();
 
-        self.meta.config.evidence = target;
-        if matches!(self.meta.config.evidence, IndexMode::Exact) {
-            self.meta.config.block_bits = block_bits;
+        let mut config = self.meta.config.clone();
+        config.evidence = target;
+        if matches!(config.evidence, IndexMode::Exact) {
+            config.block_bits = block_bits;
         }
-        self.meta.write(&self.root_path)?;
+        let genomes = self.meta.genomes().map_err(OKIError::Io)?;
+        self.meta.rewrite_config(config, genomes).map_err(OKIError::Io)?;
+        self.meta = Arc::new(IndexMeta::open(self).map_err(OKIError::Io)?);
         rep.push(t.stop());
         Ok(())
     }

src/obikindex/src/index/select.rs

@@ -1,4 +1,5 @@
 use std::path::Path;
+use std::sync::Arc;
 
 use crate::index::builder::IndexBuilder;
 use crate::index::OutputCol;
@@ -28,23 +29,23 @@ impl KmerIndex {
     ) -> OKIResult<Self> {
         let output = output.as_ref();
 
-        if src.state() != IndexState::Indexed {
+        if src.state()? != IndexState::Indexed {
             return Err(OKIError::NotIndexed(src.root_path.clone()));
         }
 
         KmerIndex::clear_output_for_create(output, force)?;
 
-        let mut meta = IndexMeta::new(src.meta.config.clone());
-        meta.config.with_counts = !output_presence;
-        meta.genomes = specs
+        let mut config = src.meta.config.clone();
+        config.with_counts = !output_presence;
+        let genomes: Vec<GenomeInfo> = specs
             .iter()
             .map(|s| GenomeInfo::new(s.label.clone()))
             .collect();
 
-        let n_src_genomes = src.meta.genomes.len();
+        let n_src_genomes = src.meta.genomes().map_err(OKIError::Io)?.len();
         let n_partitions = src.n_partitions();
 
-        let dst_partition = KmerIndex::create_skeleton(output, &meta)?;
+        let dst_partition = KmerIndex::create_skeleton(output, config, genomes)?;
 
         info!(
             "select: {} partition(s), {} source genome(s) → {} output column(s)",
@@ -94,11 +95,11 @@ impl KmerIndex {
         output_presence: bool,
         rep: &mut Reporter,
     ) -> OKIResult<()> {
-        if self.state() != IndexState::Indexed {
+        if self.state()? != IndexState::Indexed {
             return Err(OKIError::NotIndexed(self.root_path.clone()));
         }
 
-        let n_src_genomes = self.meta.genomes.len();
+        let n_src_genomes = self.meta.genomes().map_err(OKIError::Io)?.len();
         let n_partitions = self.n_partitions();
 
         info!(
@@ -136,12 +137,14 @@ impl KmerIndex {
         pb.finish_and_clear();
         rep.push(t.stop());
 
-        self.meta.config.with_counts = !output_presence;
-        self.meta.genomes = specs
+        let mut config = self.meta.config.clone();
+        config.with_counts = !output_presence;
+        let genomes: Vec<GenomeInfo> = specs
             .iter()
             .map(|s| GenomeInfo::new(s.label.clone()))
             .collect();
-        self.meta.write(&self.root_path)?;
+        self.meta.rewrite_config(config, genomes).map_err(OKIError::Io)?;
+        self.meta = Arc::new(IndexMeta::open(self).map_err(OKIError::Io)?);
 
         let t_pack = Stage::start("pack");
         self.pack_matrices(false)?;

src/obikindex/src/index/state.rs

@@ -1,45 +1,19 @@
-use std::path::Path;
+use serde::{Deserialize, Serialize};
 
-use crate::index::meta::META_FILENAME;
-
-pub const SENTINEL_SCATTERED: &str = "scatter.done";
-pub const SENTINEL_COUNTED:   &str = "count.done";
-pub const SENTINEL_INDEXED:   &str = "index.done";
-
-/// Progression state of a `KmerIndex`.
+/// Progression state of a `KmerIndex` — a field of `index.meta`
+/// (`IndexMeta`'s `state`), not detected from sentinel files anymore (see
+/// `DevDocMD/implementation/partition_layer_cache.md`).
 ///
 /// Variants are ordered: `Empty < Scattered < Counted < Indexed`.
-/// A state is reported only when its sentinel file is fully present —
-/// partial states (e.g. scatter interrupted mid-way) are not accepted.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Default, Serialize, Deserialize)]
 pub enum IndexState {
     /// `index.meta` present; scatter not yet completed.
+    #[default]
     Empty,
-    /// `scatter.done` sentinel present — all super-kmers have been routed.
+    /// All super-kmers have been routed.
     Scattered,
-    /// `count.done` sentinel present — dereplicate + count complete.
+    /// Dereplicate + count complete.
     Counted,
-    /// `index.done` sentinel present — layered MPHF index fully built.
+    /// Layered MPHF index fully built.
     Indexed,
 }
-
-impl IndexState {
-    /// Detect the state of the index at `root`.
-    ///
-    /// Returns `None` if `index.meta` is absent (not an obikindex directory).
-    pub fn detect(root: &Path) -> Option<Self> {
-        if !root.join(META_FILENAME).exists() {
-            return None;
-        }
-        if root.join(SENTINEL_INDEXED).exists() {
-            return Some(Self::Indexed);
-        }
-        if root.join(SENTINEL_COUNTED).exists() {
-            return Some(Self::Counted);
-        }
-        if root.join(SENTINEL_SCATTERED).exists() {
-            return Some(Self::Scattered);
-        }
-        Some(Self::Empty)
-    }
-}

src/obikindex/src/index/stats.rs

@@ -5,7 +5,7 @@ use obicompactvec::{LayerMeta, PersistentBitMatrix, PersistentCompactIntMatrix};
 use obicompactvec::traits::ColumnWeights;
 use rayon::prelude::*;
 
-use crate::index::error::OKIResult;
+use crate::index::error::{OKIError, OKIResult};
 use crate::index::kmer_index::KmerIndex;
 
 /// Bits per kmer broken down by index component.
@@ -84,7 +84,7 @@ impl KmerIndex {
     /// Computation is parallelised across partitions.
     pub fn bits_per_kmer(&self) -> OKIResult<IndexBitsPerKmer> {
         let n          = self.n_partitions();
-        let n_genomes  = self.meta().genomes.len().max(1);
+        let n_genomes  = self.meta().genomes().map_err(OKIError::Io)?.len().max(1);
 
         let (n_kmers, mphf_b, evidence_b, matrix_b) = (0..n)
             .into_par_iter()
@@ -130,7 +130,7 @@ impl KmerIndex {
     /// Partitions are scanned in parallel; results are summed across partitions.
     pub fn genome_kmer_counts(&self) -> OKIResult<(usize, Vec<u64>)> {
         let n          = self.n_partitions();
-        let n_genomes  = self.meta.genomes.len();
+        let n_genomes  = self.meta.genomes().map_err(OKIError::Io)?.len();
 
         let partials: Vec<(usize, Vec<u64>)> = (0..n)
             .into_par_iter()

src/obikindex/src/lib.rs

@@ -16,7 +16,7 @@ pub use index::{
     validate_label, AggOp, DistanceMetric, DistanceOutput, GenomeInfo, GroupFilterParams,
     GroupQuorumFilter, IndexBitsPerKmer, IndexBuilder, IndexConfig, IndexMeta, IndexState,
     KmerDesc, KmerFilter, KmerIndex, MergeMode, MetaPred, OKIError, OKIResult, OutputCol,
-    PartitionRunner, QueryHit, QueryStats, META_FILENAME, SENTINEL_COUNTED, SENTINEL_INDEXED,
-    SENTINEL_SCATTERED, materialize_layer, olm_to_sk, write_graph_as_unitigs, passes_all,
+    PartitionRunner, QueryHit, QueryStats, META_FILENAME,
+    materialize_layer, olm_to_sk, write_graph_as_unitigs, passes_all,
 };
 pub use index::{filter, meta};