swadhinbiswas WIP save-point: production hardening, H3 removal, fmt gate, and accumulated phase work · 7/15/2026
2080d40
342 lines · 12.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
//! Multi-replica supervisor — the lifecycle/orchestration layer.
//!
//! Ties the [`Autoscaler`] (how many replicas we want) and the [`Router`]
//! (which replicas can serve a request) to actual replicas. It is pure decision
//! logic: rather than spawning processes, it emits [`SupervisorAction`] intents
//! (start/stop a replica) that an external executor fulfills — that executor is
//! where real process/GPU management lives.
//!
//! Lifecycle:
//! 1. `reconcile(metrics)` asks the autoscaler for a desired replica count and
//!    emits `StartReplica` / `StopReplica` intents to close the gap with the
//!    live set, then syncs the router's replica view.
//! 2. Newly started replicas register in the router as **not-yet-healthy**; the
//!    executor calls `on_replica_started` once the replica is actually serving,
//!    flipping it healthy so the router will route to it.
//! 3. `on_replica_report` feeds live load metrics back in; `on_replica_stopped`
//!    removes a replica the executor terminated.

use std::collections::HashSet;

use crate::engine::EngineDevice;
use crate::{Autoscaler, AutoscalerConfig, LlmMetrics, Replica, Router, RoutingStrategy};

/// The supervisor's view of a single live replica.
#[derive(Debug, Clone)]
pub struct LiveReplica {
    pub id: String,
    /// Whether the replica is healthy and accepting traffic.
    pub healthy: bool,
    /// Current active sequences (drives router load-aware selection).
    pub active_sequences: usize,
    /// KV-cache pressure (0–100).
    pub kv_pressure_pct: f32,
}

/// An intent the supervisor emits for the executor to fulfill.
#[derive(Debug, Clone)]
pub enum SupervisorAction {
    /// Start a new replica for `model` @ `version`.
    StartReplica { id: String, model: String, version: String },
    /// Terminate the replica with `id`.
    StopReplica { id: String },
}

/// Tuning for the supervisor (mirrors the autoscaler knobs plus replica shape).
#[derive(Debug, Clone)]
pub struct SupervisorConfig {
    /// Model this supervisor manages.
    pub model: String,
    /// Version of the model this supervisor manages.
    pub version: String,
    /// Device new replicas are pinned to.
    pub device: EngineDevice,
    pub min_replicas: usize,
    pub max_replicas: usize,
    /// Max concurrent sequences per replica (advertised to the router).
    pub max_concurrency_per_replica: usize,
    pub target_ttft_ms: f32,
    pub target_kv_pressure_pct: f32,
    pub queue_depth_scale_up: usize,
    pub queue_depth_scale_down: usize,
    /// Ticks to wait after a scaling action before another (flap protection).
    pub cooldown_ticks: usize,
}

/// Multi-replica supervisor: owns an [`Autoscaler`] + [`Router`] and drives a
/// live replica set toward the autoscaler's desired count.
pub struct Supervisor {
    config: SupervisorConfig,
    autoscaler: Autoscaler,
    router: Router,
    live: Vec<LiveReplica>,
    next_index: usize,
}

impl Supervisor {
    /// Create a supervisor for `config.model`@`config.version`. The embedded
    /// autoscaler seeds at `min_replicas`.
    pub fn new(config: SupervisorConfig) -> Self {
        let autoscaler = Autoscaler::new(AutoscalerConfig {
            min_replicas: config.min_replicas,
            max_replicas: config.max_replicas,
            target_ttft_ms: config.target_ttft_ms,
            target_kv_pressure_pct: config.target_kv_pressure_pct,
            queue_depth_scale_up: config.queue_depth_scale_up,
            queue_depth_scale_down: config.queue_depth_scale_down,
            cooldown_ticks: config.cooldown_ticks,
        });
        Self {
            config,
            autoscaler,
            router: Router::new(RoutingStrategy::LeastLoaded),
            live: Vec::new(),
            next_index: 0,
        }
    }

    /// Current desired replica count (from the autoscaler).
    pub fn desired(&self) -> usize {
        self.autoscaler.current()
    }

    /// Number of live replicas (healthy or starting).
    pub fn live_count(&self) -> usize {
        self.live.len()
    }

    /// Number of healthy live replicas.
    pub fn healthy_count(&self) -> usize {
        self.live.iter().filter(|r| r.healthy).count()
    }

    /// Access the supervisor's router (so requests can be routed against the
    /// live replica set).
    pub fn router(&self) -> &Router {
        &self.router
    }

    /// Reconcile the live replica set with the desired count derived from
    /// `metrics`. Emits start/stop intents and syncs the router.
    pub fn reconcile(&mut self, metrics: &LlmMetrics) -> Vec<SupervisorAction> {
        self.autoscaler.decide(metrics);
        let desired = self.autoscaler.current();
        let mut actions = Vec::new();

        // Scale up: register not-yet-healthy replicas and emit start intents.
        while self.live.len() < desired && self.live.len() < self.config.max_replicas {
            let id = format!("{}-{}-{}", self.config.model, self.config.version, self.next_index);
            self.next_index += 1;
            self.live.push(LiveReplica {
                id: id.clone(),
                healthy: false,
                active_sequences: 0,
                kv_pressure_pct: 0.0,
            });
            actions.push(SupervisorAction::StartReplica {
                id,
                model: self.config.model.clone(),
                version: self.config.version.clone(),
            });
        }

        // Scale down: prefer removing stuck (unhealthy) replicas first.
        while self.live.len() > desired {
            let idx = self
                .live
                .iter()
                .position(|r| !r.healthy)
                .or_else(|| Some(self.live.len().saturating_sub(1)));
            if let Some(idx) = idx {
                let removed = self.live.remove(idx);
                actions.push(SupervisorAction::StopReplica { id: removed.id });
            } else {
                break;
            }
        }

        self.sync_router();
        actions
    }

    /// Executor reports a replica has finished starting and is now healthy.
    pub fn on_replica_started(&mut self, id: &str) {
        if let Some(r) = self.live.iter_mut().find(|r| r.id == id) {
            r.healthy = true;
        }
        self.sync_router();
    }

    /// Executor reports live load for a replica.
    pub fn on_replica_report(&mut self, id: &str, active_sequences: usize, kv_pressure_pct: f32) {
        if let Some(r) = self.live.iter_mut().find(|r| r.id == id) {
            r.active_sequences = active_sequences;
            r.kv_pressure_pct = kv_pressure_pct;
        }
        self.sync_router();
    }

    /// Executor reports a replica was terminated.
    pub fn on_replica_stopped(&mut self, id: &str) {
        self.live.retain(|r| r.id != id);
        self.router.remove_replica(id);
    }

    /// Push the live replica set into the router (add/update live, drop gone).
    fn sync_router(&mut self) {
        let live_ids: HashSet<String> = self.live.iter().map(|r| r.id.clone()).collect();
        for r in self.router.replicas() {
            if !live_ids.contains(&r.id) {
                self.router.remove_replica(&r.id);
            }
        }
        for lr in &self.live {
            self.router.register_replica(Replica {
                id: lr.id.clone(),
                model: self.config.model.clone(),
                version: self.config.version.clone(),
                device: self.config.device,
                max_concurrency: self.config.max_concurrency_per_replica,
                active_sequences: lr.active_sequences,
                kv_pressure_pct: lr.kv_pressure_pct,
                healthy: lr.healthy,
                loaded_adapters: Vec::new(),
            });
        }
    }
}

// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------

#[cfg(test)]
mod tests {
    use super::*;

    fn cfg() -> SupervisorConfig {
        SupervisorConfig {
            model: "llama-7b".into(),
            version: "v1".into(),
            device: EngineDevice::Cuda(0),
            min_replicas: 1,
            max_replicas: 4,
            max_concurrency_per_replica: 8,
            target_ttft_ms: 500.0,
            target_kv_pressure_pct: 80.0,
            queue_depth_scale_up: 16,
            queue_depth_scale_down: 2,
            cooldown_ticks: 2,
        }
    }

    fn metrics(kv: f32, ttft: f32, q: usize) -> LlmMetrics {
        LlmMetrics { tokens_per_sec: 50.0, ttft_ms: ttft, queue_depth: q, kv_pressure_pct: kv }
    }

    #[test]
    fn seeds_min_replicas_on_first_reconcile() {
        let mut s = Supervisor::new(cfg());
        // Healthy metrics → desired stays at min (1). No live yet → start one.
        let actions = s.reconcile(&metrics(10.0, 100.0, 0));
        assert_eq!(s.desired(), 1);
        assert_eq!(s.live_count(), 1);
        assert_eq!(actions.len(), 1);
        match &actions[0] {
            SupervisorAction::StartReplica { id, .. } => assert!(id.contains("llama-7b-v1")),
            _ => panic!("expected StartReplica"),
        }
        // Router sees the replica (not yet healthy → not routable).
        assert_eq!(s.router().replicas().len(), 1);
        assert_eq!(s.healthy_count(), 0);
    }

    #[test]
    fn stays_stable_once_replicas_healthy() {
        let mut s = Supervisor::new(cfg());
        let actions = s.reconcile(&metrics(10.0, 100.0, 0));
        let id = match &actions[0] {
            SupervisorAction::StartReplica { id, .. } => id.clone(),
            _ => panic!(),
        };
        s.on_replica_started(&id);
        assert_eq!(s.healthy_count(), 1);
        // Another healthy reconcile → no new actions.
        let actions = s.reconcile(&metrics(10.0, 100.0, 0));
        assert!(actions.is_empty());
        assert_eq!(s.live_count(), 1);
    }

    #[test]
    fn scales_up_under_load() {
        let mut s = Supervisor::new(cfg());
        // First establish the baseline (1 replica, mark healthy).
        let a = s.reconcile(&metrics(10.0, 100.0, 0));
        let id = match &a[0] {
            SupervisorAction::StartReplica { id, .. } => id.clone(),
            _ => panic!(),
        };
        s.on_replica_started(&id);
        // Now saturate: KV pressure high → autoscaler wants more.
        // Cooldown ticks pass between reconciles.
        let _ = s.reconcile(&metrics(90.0, 100.0, 0));
        let _ = s.reconcile(&metrics(90.0, 100.0, 0));
        let actions = s.reconcile(&metrics(90.0, 100.0, 0));
        // Either we already scaled up via the cooldown window or this emits starts.
        assert!(s.desired() >= 2);
        if !actions.is_empty() {
            assert!(actions.iter().all(|a| matches!(a, SupervisorAction::StartReplica { .. })));
        }
        assert!(s.live_count() >= 2);
    }

    #[test]
    fn never_exceeds_max() {
        let mut s = Supervisor::new(cfg());
        // Hammer with saturated metrics across many ticks; live must cap at max.
        for _ in 0..12 {
            let _ = s.reconcile(&metrics(99.0, 2000.0, 100));
        }
        assert_eq!(s.live_count(), 4);
        assert_eq!(s.desired(), 4);
    }

    #[test]
    fn scales_down_under_low_load() {
        let mut c = cfg();
        c.max_replicas = 3;
        let mut s = Supervisor::new(c);
        // Bring up to 3 replicas.
        for _ in 0..12 {
            let _ = s.reconcile(&metrics(99.0, 2000.0, 100));
        }
        // Mark them all healthy.
        for r in s.router().replicas() {
            s.on_replica_started(&r.id);
        }
        assert_eq!(s.live_count(), 3);
        assert_eq!(s.healthy_count(), 3);
        // Now underused for many ticks → scale down to min.
        for _ in 0..12 {
            let _ = s.reconcile(&metrics(5.0, 50.0, 0));
        }
        assert_eq!(s.live_count(), 1);
        assert_eq!(s.healthy_count(), 1);
    }

    #[test]
    fn router_sync_reflects_live_set() {
        let mut s = Supervisor::new(cfg());
        let _ = s.reconcile(&metrics(99.0, 2000.0, 100));
        let _ = s.reconcile(&metrics(99.0, 2000.0, 100));
        let _ = s.reconcile(&metrics(99.0, 2000.0, 100));
        // Router replica count must equal the live count (synced each reconcile).
        assert_eq!(s.router().replicas().len(), s.live_count());
        // After stopping one via the executor, the router drops it too.
        let id = s.router().replicas()[0].id.clone();
        s.on_replica_stopped(&id);
        assert_eq!(s.router().replicas().len(), s.live_count());
        assert!(!s.router().replicas().iter().any(|r| r.id == id));
    }
}