♻️ refactor pipeline architecture and fix macOS memory detection

- Replace WorkerPool-based pipelines with typed `Pipe` abstraction in obipipeline
  - Introduce Pipe/PipeIter for composable, sourceless/sink-less pipelines
- Update partition and superkmer commands to use new Pipe API via make_pipe!
  - Remove Arc<Mutex<...>> patterns; simplify state management
- Fix macOS available_memory() returning 0 by falling back to half total memory in dereplicate()
- Remove unused `format: "zstd"` field from partition.meta
This commit is contained in:
Eric Coissac
2026-04-28 08:40:07 +02:00
parent 4c19882f03
commit 97e65bd831
8 changed files with 264 additions and 48 deletions
+7 -1
View File
@@ -186,7 +186,13 @@ impl KmerPartition {
pub fn dereplicate(&self) -> SKResult<()> {
let level = self.level;
let root = &self.root_path;
let available = System::new_all().available_memory();
let sys = System::new_all();
// available_memory() can return 0 on macOS when the compressor page count exceeds
// free+inactive+purgeable pages (sysinfo saturating_sub). Fall back to half of total.
let available = match sys.available_memory() {
0 => sys.total_memory() / 2,
n => n,
};
let n_threads = rayon::current_num_threads().max(1) as u64;
let available_per_thread = available / n_threads;