OBIKmers/obikmer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

.gitignore: ignore zstandard-compressed files

Browse Source 
- Add *.zst pattern to .gitignore
- Prevents tracking of zstandard-compressed archives
This commit is contained in:
Eric Coissac
2026-04-27 16:53:42 +02:00
parent 1f466bf113
commit 7efec54b27
12 changed files with 281 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/obipipeline/src/scheduler.rs
+38 -32
View File
@@ -78,8 +78,8 @@ impl<D> Clone for Stage<D> {
// ── Worker task ───────────────────────────────────────────────────────────────
enum WorkerTask<D> {
Transform(D, SharedFn<D>, Sender<Result<D, PipelineError>>),
Flat(D, SharedFlatFn<D>, Sender<Result<D, PipelineError>>, Sender<isize>),
Transform(D, usize),
Flat(D, usize),
}
// ── Thread runners ────────────────────────────────────────────────────────────
@@ -121,18 +121,27 @@ where
/// - `Transform` : applique `f(data)` et envoie le résultat dans `result_tx`.
/// - `Flat` : appelle `f(data, &push_tx, &delta_tx)` ; la fonction elle-même
/// pousse ses items dans `push_tx` et envoie `N-1` dans `delta_tx`.
fn transform_runner<DATA>(task_rx: Receiver<WorkerTask<DATA>>) -> thread::JoinHandle<()>
fn transform_runner<DATA>(
task_rx: Receiver<WorkerTask<DATA>>,
stages: Vec<Stage<DATA>>,
stage_txs: Vec<Sender<Result<DATA, PipelineError>>>,
flat_delta_tx: Sender<isize>,
) -> thread::JoinHandle<()>
where
DATA: Send + Sync + 'static,
{
thread::spawn(move || {
while let Ok(task) = task_rx.recv() {
match task {
WorkerTask::Transform(data, f, result_tx) => {
let _ = result_tx.send(f(data));
WorkerTask::Transform(data, idx) => {
if let Stage::Transform(f) = &stages[idx] {
let _ = stage_txs[idx].send(f(data));
}
}
WorkerTask::Flat(data, f, push_tx, delta_tx) => {
f(data, &push_tx, &delta_tx);
WorkerTask::Flat(data, idx) => {
if let Stage::Flat(f) = &stages[idx] {
f(data, &stage_txs[idx], &flat_delta_tx);
}
}
}
}
@@ -223,23 +232,31 @@ where
let stages = self.pipeline.stages;
// ── Canal delta pour les flat stages ───────────────────────────────
// Chaque flat worker envoie `N-1` ici après avoir poussé N items.
// Le scheduler ajuste `in_flight` en conséquence.
let (flat_delta_tx, flat_delta_rx) = bounded::<isize>(self.capacity);
// ── Worker pool ────────────────────────────────────────────────────
let (worker_tx, worker_rx): (Sender<WorkerTask<DATA>>, Receiver<WorkerTask<DATA>>) =
bounded(self.capacity);
for _ in 0..self.n_workers {
self.handles.push(transform_runner(worker_rx.clone()));
self.handles.push(transform_runner(
worker_rx.clone(),
stages.iter().map(Stage::clone).collect(),
stage_txs.clone(),
flat_delta_tx.clone(),
));
}
// Le scheduler ne tient plus flat_delta_tx : les workers le détiennent.
// On le drop ici pour que le canal se ferme quand les workers terminent.
drop(flat_delta_tx);
// ── Sink thread ────────────────────────────────────────────────────
let (sink_tx, sink_err_rx, sink_handle) = sink_runner(self.pipeline.sink, self.capacity);
self.handles.push(sink_handle);
// ── Canal delta pour les flat stages ───────────────────────────────
// Chaque flat worker envoie `N-1` ici après avoir poussé N items.
// Le scheduler ajuste `in_flight` en conséquence.
let (flat_delta_tx, flat_delta_rx) = bounded::<isize>(self.capacity);
// ── Boucle principale ──────────────────────────────────────────────
//
// `in_flight` (isize) = nb d'items qui doivent encore atteindre le sink.
@@ -313,8 +330,8 @@ where
in_flight += 1;
dispatch(
data, 0,
&stages, &stage_txs, &worker_tx,
&flat_delta_tx, &mut flat_workers_active,
&stages, &worker_tx,
&mut flat_workers_active,
);
}
}
@@ -333,8 +350,8 @@ where
} else {
dispatch(
data, stage + 1,
&stages, &stage_txs, &worker_tx,
&flat_delta_tx, &mut flat_workers_active,
&stages, &worker_tx,
&mut flat_workers_active,
);
}
}
@@ -362,27 +379,16 @@ fn dispatch<DATA>(
data: DATA,
stage_idx: usize,
stages: &[Stage<DATA>],
stage_txs: &[Sender<Result<DATA, PipelineError>>],
worker_tx: &Sender<WorkerTask<DATA>>,
flat_delta_tx: &Sender<isize>,
flat_workers_active: &mut usize,
) {
match &stages[stage_idx] {
Stage::Transform(f) => {
let _ = worker_tx.send(WorkerTask::Transform(
data,
Arc::clone(f),
stage_txs[stage_idx].clone(),
));
Stage::Transform(_) => {
let _ = worker_tx.send(WorkerTask::Transform(data, stage_idx));
}
Stage::Flat(f) => {
Stage::Flat(_) => {
*flat_workers_active += 1;
let _ = worker_tx.send(WorkerTask::Flat(
data,
Arc::clone(f),
stage_txs[stage_idx].clone(),
flat_delta_tx.clone(),
));
let _ = worker_tx.send(WorkerTask::Flat(data, stage_idx));
}
}
}