Switch sibling modules to sequential layer directory processing

The sibling calculation modules now process layer directories sequentially instead of in parallel. This eliminates concurrent processing overhead and prevents interleaved cache sweeps, improving disk I/O and page-cache locality for partition-grouped data access. Progress bar updates and result accumulation have been adapted to the sequential control flow, while core filtering logic and output structures remain unchanged.

src/obikindex/src/siblings/alignment.rs

@@ -1,5 +1,3 @@
-use rayon::prelude::*;
-
 use obikpartitionner::KmerPartition;
 use obisys::progress_bar;
 
@@ -74,24 +72,26 @@ impl KmerIndex {
         let layer_dirs = self.sibling_layer_dirs()?;
 
         let pb = progress_bar("snp_pseudo_alignment", layer_dirs.len() as u64, "layers");
-        // `Vec<Vec<u8>>` per layer, one entry (column) per variable family;
-        // `par_iter().map(...).collect()` on this indexed source preserves
-        // input order, so concatenating the results below in order gives a
-        // single deterministic column order across the whole index.
-        let partials: Vec<Vec<Vec<u8>>> = layer_dirs
-            .par_iter()
-            .map(|layer_dir| -> OKIResult<Vec<Vec<u8>>> {
+        // One layer at a time, not `par_iter()` over layers — same
+        // rationale as `build_sibling_annex`: running many layers'
+        // `scan_layer_families` concurrently would each group their own
+        // lookups by partition internally, but interleave those sweeps
+        // across layers at the OS level, scattering page-cache access over
+        // every partition at once again and defeating the whole point of
+        // the grouping. `Vec<Vec<u8>>` per layer, one entry (column) per
+        // variable family, appended in layer order for a single
+        // deterministic column order across the whole index.
+        let mut partials: Vec<Vec<Vec<u8>>> = Vec::with_capacity(layer_dirs.len());
+        for layer_dir in &layer_dirs {
             let families = scan_layer_families(layer_dir, n_parts, n_genomes, with_counts, k, &cache)?;
             let columns = families
                 .into_iter()
                 .filter(|f| f.mask.family_size() >= 2) // monomorphic family — no signal, skip
                 .map(|f| f.genome_mask.iter().map(|&m| iupac_code(m)).collect())
                 .collect();
-
+            partials.push(columns);
             pb.inc(1);
-                Ok(columns)
-            })
-            .collect::<OKIResult<Vec<_>>>()?;
+        }
         pb.finish_and_clear();
 
         let mut sequences: Vec<Vec<u8>> = vec![Vec::new(); n_genomes];

src/obikindex/src/siblings/cardinality.rs

@@ -1,5 +1,4 @@
 use ndarray::Array2;
-use rayon::prelude::*;
 
 use obikpartitionner::KmerPartition;
 use obisys::progress_bar;
@@ -77,9 +76,11 @@ impl KmerIndex {
         let layer_dirs = self.sibling_layer_dirs()?;
 
         let pb = progress_bar("cardinality_tally", layer_dirs.len() as u64, "layers");
-        let partials: Vec<[[u64; 5]; 5]> = layer_dirs
-            .par_iter()
-            .map(|layer_dir| -> OKIResult<[[u64; 5]; 5]> {
+        // One layer at a time — see `snp_pseudo_alignment`'s comment for why
+        // `par_iter()` over layers would defeat `scan_layer_families`'s
+        // partition-grouped locality.
+        let mut partials: Vec<[[u64; 5]; 5]> = Vec::with_capacity(layer_dirs.len());
+        for layer_dir in &layer_dirs {
             let families = scan_layer_families(layer_dir, n_parts, n_genomes, with_counts, k, &cache)?;
             let mut counts = [[0u64; 5]; 5];
 
@@ -112,10 +113,9 @@ impl KmerIndex {
                 }
             }
 
+            partials.push(counts);
             pb.inc(1);
-                Ok(counts)
-            })
-            .collect::<OKIResult<Vec<_>>>()?;
+        }
         pb.finish_and_clear();
 
         let mut total = [[0u64; 5]; 5];

src/obikindex/src/siblings/distance.rs

@@ -1,5 +1,4 @@
 use ndarray::Array2;
-use rayon::prelude::*;
 
 use obikpartitionner::KmerPartition;
 use obisys::progress_bar;
@@ -50,8 +49,11 @@ impl KmerIndex {
     /// should only reflect genuine SNP-adjacent agreement, not the
     /// genome-wide invariant background, need it (see
     /// [`base_pair_tally`](Self::base_pair_tally)'s `same` field). Layers
-    /// are processed in parallel (rayon); each gets its own accumulator
-    /// from `zero()`, combined pairwise via `combine`.
+    /// are processed one at a time, not in parallel — see
+    /// `snp_pseudo_alignment`'s comment for why `par_iter()` over layers
+    /// would defeat `scan_layer_families`'s partition-grouped locality;
+    /// each layer gets its own accumulator from `zero()`, combined
+    /// pairwise via `combine`.
     fn scan_family_pairs<Acc, F, C>(
         &self,
         label: &str,
@@ -81,11 +83,13 @@ impl KmerIndex {
         let layer_dirs = self.sibling_layer_dirs()?;
 
         let pb = progress_bar(label, layer_dirs.len() as u64, "layers");
-        let partials: Vec<Acc> = layer_dirs
-            .par_iter()
-            .map(|layer_dir| -> OKIResult<Acc> {
-                let mut acc = zero();
+        // One layer at a time — see `snp_pseudo_alignment`'s comment for why
+        // `par_iter()` over layers would defeat `scan_layer_families`'s
+        // partition-grouped locality.
+        let mut total = zero();
+        for layer_dir in &layer_dirs {
             let families = scan_layer_families(layer_dir, n_parts, n_genomes, with_counts, k, &cache)?;
+            let mut acc = zero();
 
             for family in &families {
                 let variable = family.mask.family_size() >= 2;
@@ -109,16 +113,11 @@ impl KmerIndex {
                 }
             }
 
+            total = combine(total, acc);
             pb.inc(1);
-                Ok(acc)
-            })
-            .collect::<OKIResult<Vec<_>>>()?;
+        }
         pb.finish_and_clear();
 
-        let mut total = zero();
-        for partial in partials {
-            total = combine(total, partial);
-        }
         Ok(total)
     }
 

src/obikindex/src/siblings/stats.rs

@@ -1,5 +1,3 @@
-use rayon::prelude::*;
-
 use obikpartitionner::KmerPartition;
 use obisys::progress_bar;
 
@@ -58,18 +56,15 @@ impl KmerIndex {
         let cache = PartitionCache::build(&partition, n_parts, with_counts)?;
         let layer_dirs = self.sibling_layer_dirs()?;
 
-        // One layer's worth of work, parallelised across layers with Rayon
-        // — independent, read-only, each producing its own partial tally
-        // merged at the end.
+        // One layer at a time, not parallelised across layers — see
+        // `snp_pseudo_alignment`'s comment for why `par_iter()` over layers
+        // would defeat `scan_layer_families`'s partition-grouped locality.
         let pb = progress_bar("sibling_annex_stats", layer_dirs.len() as u64, "layers");
-        let partials: Vec<SiblingAnnexStats> = layer_dirs
-            .par_iter()
-            .map(|layer_dir| -> OKIResult<SiblingAnnexStats> {
         let mut stats = SiblingAnnexStats {
             per_genome: vec![[0u64; 4]; n_genomes],
             ..Default::default()
         };
-
+        for layer_dir in &layer_dirs {
             let families = scan_layer_families(layer_dir, n_parts, n_genomes, with_counts, k, &cache)?;
             for family in &families {
                 // "Genome g represents this family" means g carries
@@ -83,27 +78,10 @@ impl KmerIndex {
                     }
                 }
             }
-
             pb.inc(1);
-                Ok(stats)
-            })
-            .collect::<OKIResult<Vec<_>>>()?;
+        }
         pb.finish_and_clear();
 
-        let mut stats = SiblingAnnexStats {
-            per_genome: vec![[0u64; 4]; n_genomes],
-            ..Default::default()
-        };
-        for part in partials {
-            for s in 0..4 {
-                stats.counts[s] += part.counts[s];
-            }
-            for g in 0..n_genomes {
-                for s in 0..4 {
-                    stats.per_genome[g][s] += part.per_genome[g][s];
-                }
-            }
-        }
         Ok(stats)
     }
 }