Switch sibling modules to sequential layer directory processing

The sibling calculation modules now process layer directories sequentially instead of in parallel. This eliminates concurrent processing overhead and prevents interleaved cache sweeps, improving disk I/O and page-cache locality for partition-grouped data access. Progress bar updates and result accumulation have been adapted to the sequential control flow, while core filtering logic and output structures remain unchanged.

src/obikindex/src/siblings/alignment.rs

@@ -1,5 +1,3 @@
-use rayon::prelude::*;
-
 use obikpartitionner::KmerPartition;
 use obisys::progress_bar;
 
@@ -74,24 +72,26 @@ impl KmerIndex {
         let layer_dirs = self.sibling_layer_dirs()?;
 
         let pb = progress_bar("snp_pseudo_alignment", layer_dirs.len() as u64, "layers");
-        // `Vec<Vec<u8>>` per layer, one entry (column) per variable family;
+        // One layer at a time, not `par_iter()` over layers — same
-        // `par_iter().map(...).collect()` on this indexed source preserves
+        // rationale as `build_sibling_annex`: running many layers'
-        // input order, so concatenating the results below in order gives a
+        // `scan_layer_families` concurrently would each group their own
-        // single deterministic column order across the whole index.
+        // lookups by partition internally, but interleave those sweeps
-        let partials: Vec<Vec<Vec<u8>>> = layer_dirs
+        // across layers at the OS level, scattering page-cache access over
-            .par_iter()
+        // every partition at once again and defeating the whole point of
-            .map(|layer_dir| -> OKIResult<Vec<Vec<u8>>> {
+        // the grouping. `Vec<Vec<u8>>` per layer, one entry (column) per
+        // variable family, appended in layer order for a single
+        // deterministic column order across the whole index.
+        let mut partials: Vec<Vec<Vec<u8>>> = Vec::with_capacity(layer_dirs.len());
+        for layer_dir in &layer_dirs {
             let families = scan_layer_families(layer_dir, n_parts, n_genomes, with_counts, k, &cache)?;
             let columns = families
                 .into_iter()
                 .filter(|f| f.mask.family_size() >= 2) // monomorphic family — no signal, skip
                 .map(|f| f.genome_mask.iter().map(|&m| iupac_code(m)).collect())
                 .collect();
-
+            partials.push(columns);
             pb.inc(1);
-                Ok(columns)
+        }
-            })
-            .collect::<OKIResult<Vec<_>>>()?;
         pb.finish_and_clear();
 
         let mut sequences: Vec<Vec<u8>> = vec![Vec::new(); n_genomes];

src/obikindex/src/siblings/cardinality.rs

@@ -1,5 +1,4 @@
 use ndarray::Array2;
-use rayon::prelude::*;
 
 use obikpartitionner::KmerPartition;
 use obisys::progress_bar;
@@ -77,9 +76,11 @@ impl KmerIndex {
         let layer_dirs = self.sibling_layer_dirs()?;
 
         let pb = progress_bar("cardinality_tally", layer_dirs.len() as u64, "layers");
-        let partials: Vec<[[u64; 5]; 5]> = layer_dirs
+        // One layer at a time — see `snp_pseudo_alignment`'s comment for why
-            .par_iter()
+        // `par_iter()` over layers would defeat `scan_layer_families`'s
-            .map(|layer_dir| -> OKIResult<[[u64; 5]; 5]> {
+        // partition-grouped locality.
+        let mut partials: Vec<[[u64; 5]; 5]> = Vec::with_capacity(layer_dirs.len());
+        for layer_dir in &layer_dirs {
             let families = scan_layer_families(layer_dir, n_parts, n_genomes, with_counts, k, &cache)?;
             let mut counts = [[0u64; 5]; 5];
 
@@ -112,10 +113,9 @@ impl KmerIndex {
                 }
             }
 
+            partials.push(counts);
             pb.inc(1);
-                Ok(counts)
+        }
-            })
-            .collect::<OKIResult<Vec<_>>>()?;
         pb.finish_and_clear();
 
         let mut total = [[0u64; 5]; 5];

src/obikindex/src/siblings/distance.rs

@@ -1,5 +1,4 @@
 use ndarray::Array2;
-use rayon::prelude::*;
 
 use obikpartitionner::KmerPartition;
 use obisys::progress_bar;
@@ -50,8 +49,11 @@ impl KmerIndex {
     /// should only reflect genuine SNP-adjacent agreement, not the
     /// genome-wide invariant background, need it (see
     /// [`base_pair_tally`](Self::base_pair_tally)'s `same` field). Layers
-    /// are processed in parallel (rayon); each gets its own accumulator
+    /// are processed one at a time, not in parallel — see
-    /// from `zero()`, combined pairwise via `combine`.
+    /// `snp_pseudo_alignment`'s comment for why `par_iter()` over layers
+    /// would defeat `scan_layer_families`'s partition-grouped locality;
+    /// each layer gets its own accumulator from `zero()`, combined
+    /// pairwise via `combine`.
     fn scan_family_pairs<Acc, F, C>(
         &self,
         label: &str,
@@ -81,11 +83,13 @@ impl KmerIndex {
         let layer_dirs = self.sibling_layer_dirs()?;
 
         let pb = progress_bar(label, layer_dirs.len() as u64, "layers");
-        let partials: Vec<Acc> = layer_dirs
+        // One layer at a time — see `snp_pseudo_alignment`'s comment for why
-            .par_iter()
+        // `par_iter()` over layers would defeat `scan_layer_families`'s
-            .map(|layer_dir| -> OKIResult<Acc> {
+        // partition-grouped locality.
-                let mut acc = zero();
+        let mut total = zero();
+        for layer_dir in &layer_dirs {
             let families = scan_layer_families(layer_dir, n_parts, n_genomes, with_counts, k, &cache)?;
+            let mut acc = zero();
 
             for family in &families {
                 let variable = family.mask.family_size() >= 2;
@@ -109,16 +113,11 @@ impl KmerIndex {
                 }
             }
 
+            total = combine(total, acc);
             pb.inc(1);
-                Ok(acc)
+        }
-            })
-            .collect::<OKIResult<Vec<_>>>()?;
         pb.finish_and_clear();
 
-        let mut total = zero();
-        for partial in partials {
-            total = combine(total, partial);
-        }
         Ok(total)
     }
 

src/obikindex/src/siblings/stats.rs

@@ -1,5 +1,3 @@
-use rayon::prelude::*;
-
 use obikpartitionner::KmerPartition;
 use obisys::progress_bar;
 
@@ -58,18 +56,15 @@ impl KmerIndex {
         let cache = PartitionCache::build(&partition, n_parts, with_counts)?;
         let layer_dirs = self.sibling_layer_dirs()?;
 
-        // One layer's worth of work, parallelised across layers with Rayon
+        // One layer at a time, not parallelised across layers — see
-        // — independent, read-only, each producing its own partial tally
+        // `snp_pseudo_alignment`'s comment for why `par_iter()` over layers
-        // merged at the end.
+        // would defeat `scan_layer_families`'s partition-grouped locality.
         let pb = progress_bar("sibling_annex_stats", layer_dirs.len() as u64, "layers");
-        let partials: Vec<SiblingAnnexStats> = layer_dirs
-            .par_iter()
-            .map(|layer_dir| -> OKIResult<SiblingAnnexStats> {
         let mut stats = SiblingAnnexStats {
             per_genome: vec![[0u64; 4]; n_genomes],
             ..Default::default()
         };
-
+        for layer_dir in &layer_dirs {
             let families = scan_layer_families(layer_dir, n_parts, n_genomes, with_counts, k, &cache)?;
             for family in &families {
                 // "Genome g represents this family" means g carries
@@ -83,27 +78,10 @@ impl KmerIndex {
                     }
                 }
             }
-
             pb.inc(1);
-                Ok(stats)
+        }
-            })
-            .collect::<OKIResult<Vec<_>>>()?;
         pb.finish_and_clear();
 
-        let mut stats = SiblingAnnexStats {
-            per_genome: vec![[0u64; 4]; n_genomes],
-            ..Default::default()
-        };
-        for part in partials {
-            for s in 0..4 {
-                stats.counts[s] += part.counts[s];
-            }
-            for g in 0..n_genomes {
-                for s in 0..4 {
-                    stats.per_genome[g][s] += part.per_genome[g][s];
-                }
-            }
-        }
         Ok(stats)
     }
 }