swadhinbiswas WIP save-point: production hardening, H3 removal, fmt gate, and accumulated phase work · 7/15/2026
2080d40
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
use std::future::Future;
use std::time::Duration;
use tokio::sync::{mpsc, oneshot};
/// A request wrapped with a oneshot channel to return the result.
pub struct BatchRequest<Req, Res> {
pub payload: Req,
pub responder: oneshot::Sender<Res>,
}
/// A handle to push items into the batcher.
#[derive(Clone)]
pub struct Batcher<Req, Res> {
sender: mpsc::Sender<BatchRequest<Req, Res>>,
}
impl<Req, Res> Batcher<Req, Res>
where
Req: Send + 'static,
Res: Send + 'static,
{
/// Send a request to the batching queue and wait for the response.
pub async fn execute(&self, payload: Req) -> Result<Res, &'static str> {
let (tx, rx) = oneshot::channel();
let req = BatchRequest { payload, responder: tx };
if self.sender.send(req).await.is_err() {
return Err("Batcher is closed");
}
rx.await.map_err(|_| "Batcher dropped the response")
}
}
/// Start an adaptive batching loop.
///
/// `max_size` is the maximum number of items in a single batch.
/// `window` is the maximum time to wait for items before flushing a partial batch.
/// `processor` is an async closure that takes a `Vec<Req>` and returns a `Vec<Res>`.
/// The returned vector must have the exact same length as the input vector.
pub fn start_batcher<Req, Res, F, Fut>(
max_size: usize,
window: Duration,
mut processor: F,
) -> Batcher<Req, Res>
where
Req: Send + 'static,
Res: Send + 'static,
F: FnMut(Vec<Req>) -> Fut + Send + 'static,
Fut: Future<Output = Vec<Res>> + Send + 'static,
{
let (tx, mut rx) = mpsc::channel::<BatchRequest<Req, Res>>(1024);
tokio::spawn(async move {
loop {
let mut batch = Vec::with_capacity(max_size);
let mut responders = Vec::with_capacity(max_size);
// Wait for the FIRST item in the batch.
// If the channel is closed, we exit the loop.
let first_req = match rx.recv().await {
Some(req) => req,
None => break,
};
batch.push(first_req.payload);
responders.push(first_req.responder);
// Now we have the first item, we wait for more up to `max_size` or `window`.
let deadline = tokio::time::sleep(window);
tokio::pin!(deadline);
loop {
if batch.len() >= max_size {
break;
}
tokio::select! {
_ = &mut deadline => {
// Timeout reached, flush what we have.
break;
}
opt_req = rx.recv() => {
match opt_req {
Some(req) => {
batch.push(req.payload);
responders.push(req.responder);
}
None => {
// Channel closed, break and flush the remaining batch.
break;
}
}
}
}
}
// Process the batch
let results = processor(batch).await;
// Scatter the results back to the callers
for (responder, res) in responders.into_iter().zip(results) {
let _ = responder.send(res);
}
}
});
Batcher { sender: tx }
}