feat: introduce preloaded index cache and thread-safe progress tracker

Introduce `PreloadedIndex` to cache partition indices and eliminate redundant I/O during repeated queries. Refactor the query pipeline to route through this pre-loaded index, and expose it publicly in `obikpartitionner`. Additionally, add a thread-safe, lazily-initialized `MultiProgress` singleton for improved progress tracking.

src/obikpartitionner/src/lib.rs

@@ -11,3 +11,4 @@ mod rebuild_layer;
 pub use filter::KmerFilter;
 pub use merge_layer::MergeMode;
 pub use partition::{KmerPartition, KmerSpectrum, PARTITIONS_SUBDIR};
+pub use query_layer::PreloadedIndex;

src/obikpartitionner/src/query_layer.rs

@@ -68,17 +68,111 @@ impl QueryLayer {
     }
 }
 
-// ── KmerPartition::query_partition ───────────────────────────────────────────
+// ── PreloadedIndex ────────────────────────────────────────────────────────────
 
-impl KmerPartition {
-    /// Query a single partition for a slice of (already-routed) super-kmers.
+/// All query layers for every partition, opened once at startup.
+///
+/// Wrap in `Arc` and share across worker threads — all access is read-only.
+pub struct PreloadedIndex {
+    /// `layers[part_idx]` — ordered vec of query layers for that partition.
+    /// Empty vec when the partition has no index directory yet.
+    layers: Vec<Vec<QueryLayer>>,
+}
+
+// SAFETY: QueryLayer and its contents are opened read-only (mmap + in-memory
+// data structures).  No mutation occurs after construction.
+unsafe impl Sync for PreloadedIndex {}
+unsafe impl Send for PreloadedIndex {}
+
+impl PreloadedIndex {
+    /// Open all partition index directories and deserialise every MPHF once.
+    ///
+    /// This is the expensive call — do it once before spawning query workers.
+    pub fn new(
+        partition: &KmerPartition,
+        n_partitions: usize,
+        with_counts: bool,
+    ) -> SKResult<Self> {
+        let active: Vec<usize> = (0..n_partitions).collect();
+        Self::new_subset(partition, n_partitions, &active, with_counts)
+    }
+
+    /// Open only the listed partition indices.
+    ///
+    /// Keeps file-descriptor and memory usage bounded to the active set.
+    /// Unlisted partitions have an empty layer vec and return all-None on query.
+    pub fn new_subset(
+        partition: &KmerPartition,
+        n_partitions: usize,
+        active: &[usize],
+        with_counts: bool,
+    ) -> SKResult<Self> {
+        let mut layers: Vec<Vec<QueryLayer>> = (0..n_partitions).map(|_| Vec::new()).collect();
+        for &i in active {
+            let index_dir = partition.part_dir(i).join(INDEX_SUBDIR);
+            if !index_dir.exists() {
+                continue;
+            }
+            let meta = PartitionMeta::load(&index_dir).map_err(olm_to_sk)?;
+            layers[i] = (0..meta.n_layers)
+                .map(|l| QueryLayer::open(
+                    &index_dir.join(format!("layer_{l}")),
+                    with_counts,
+                    &meta.mode,
+                ))
+                .collect::<SKResult<_>>()?;
+        }
+        Ok(Self { layers })
+    }
+
+    /// Query one partition for a slice of already-routed super-kmers.
     ///
     /// Returns one entry per input super-kmer; each entry is a `Vec` with one
     /// `Option<Box<[u32]>>` per k-mer inside that super-kmer:
     /// - `None`        — k-mer absent from the index
-    /// - `Some(row)`   — per-genome count (count index) or 0/1 (presence index)
+    /// - `Some(row)`   — per-genome count or 0/1 presence
+    pub fn query_partition(
+        &self,
+        part_idx: usize,
+        superkmers: &[&RoutableSuperKmer],
+        k: usize,
+        n_genomes: usize,
+    ) -> SKResult<Vec<Vec<Option<Box<[u32]>>>>> {
+        if superkmers.is_empty() {
+            return Ok(Vec::new());
+        }
+
+        let layers = &self.layers[part_idx];
+
+        if layers.is_empty() {
+            return Ok(superkmers
+                .iter()
+                .map(|rsk| vec![None; rsk.seql() - k + 1])
+                .collect());
+        }
+
+        Ok(superkmers
+            .iter()
+            .map(|rsk| {
+                rsk.superkmer()
+                    .iter_canonical_kmers()
+                    .map(|kmer| {
+                        layers.iter().find_map(|layer| layer.find(kmer, n_genomes))
+                    })
+                    .collect()
+            })
+            .collect())
+    }
+}
+
+// ── KmerPartition::query_partition (kept for backward compatibility) ──────────
+
+impl KmerPartition {
+    /// Query a single partition for a slice of (already-routed) super-kmers.
     ///
-    /// All `superkmers` must belong to this partition (same minimizer bucket).
+    /// **Prefer [`PreloadedIndex`] for repeated queries** — this method
+    /// re-opens and deserialises the MPHF on every call.
+    #[deprecated(note = "use PreloadedIndex::query_partition to avoid repeated MPHF I/O")]
     pub fn query_partition(
         &self,
         part_idx: usize,