swadhin / justapi public

p4: route-lookup cache (Router::resolve memoization) (ADR-063)

swadhinbiswas committed 3 months ago
ae3f01e
1 parent 4ad9222
commit ae3f01e
4 files changed
+241 -15
Showing 4 changed files+241-15
BENCHMARKS.mdDECISIONS.mdcrates/justapi-core/src/router.rscrates/justapi-core/src/server/mod.rs
▼BENCHMARKS.md
+18-0
@@ -1518,3 +1518,21 @@ via `listener_from_unix_fd`).
1518
1518
1519
1519
No request-path perf regression vs TCP (shared `serve_connection`). Unit test
1520
1520
`test_unix_socket_serves_http` added. Gates green.
1521
+
1522
+### ADR-063 — Route-lookup cache (`Router::resolve`)
1523
+
1524
+A bounded, always-on memoization of route lookups. Static-route hits and
1525
+definitive `NotFound`s are cached (removing the `matchit` traversal + the
1526
+cross-method `MethodNotAllowed` scan on repeat hits/misses). Param routes and
1527
+`MethodNotAllowed` bypass the cache (fresh full match each request, no stale
1528
+data). Bounded FIFO eviction (default 1024 keys), `Arc<Mutex<HashMap>>`.
1529
+
1530
+| Property | Result |
1531
+|---|---|
1532
+| Repeat GET on a static route | served from cache after first match |
1533
+| Repeat miss (NotFound) | memoized; no re-scan |
1534
+| Param route (`/users/{id}`) | resolves fresh each time, params intact |
1535
+| `--workers N` | per-worker router cache (no cross-process sharing needed) |
1536
+
1537
+No routing-semantics change (param/method-not-allowed/fallback identical).
1538
+Unit tests added. Gates green. Negligible fixed memory per key.
1521
1539
▼DECISIONS.md
+39-0
@@ -2350,3 +2350,42 @@ http://localhost/` → 404 (no route). Prefork: `justapi serve --unix /tmp/j.soc
2350
2350
exited cleanly"), tree reaped. Gates green: `cargo test --workspace
2351
2351
--features justapi-core/db`, clippy `-D warnings`, `cargo fmt --check`. (UDS is
2352
2352
Unix-only; on non-unix `--unix` errors clearly.)
2353
+
2354
+## ADR-063 — 2026-07-17 — Route-lookup cache (`Router::resolve` memoization)
2355
+
2356
+**Context:** `Router::at` does a `HashMap` lookup + a `matchit` radix traversal on
2357
+every request, then (for non-matches) a second scan across all methods to
2358
+distinguish `NotFound` from `MethodNotAllowed`. At high RPS this is pure,
2359
+repeatable work for stable URLs. The feature list calls for a "route cache".
2360
+
2361
+**Decision / change:**
2362
+- `crates/justapi-core/src/router.rs`: added `RouteResolution<T> { handler, params }`
2363
+ (owned, no borrow into the router) and `Router::resolve(&self, method, path)`
2364
+ which memoizes lookups through a bounded `RouteCache<T>` (`Mutex<HashMap>` +
2365
+ FIFO `order` vec for eviction; default capacity 1024, always-on).
2366
+ - **Caching policy (safety):** only *stable* outcomes are cached — static-route
2367
+ hits (no path params) and definitive `NotFound`s. **Param routes and
2368
+ `MethodNotAllowed` are never cached**, because they require a fresh full match
2369
+ (param extraction / cross-method scan) each request and could otherwise
2370
+ return stale data. The cache is per-`Router` (`Arc<RouteCache>` so `Router`
2371
+ stays `Clone`); negative entries avoid re-scanning on repeat misses.
2372
+ - `Router::new()` keeps an always-on cache; `without_cache()` and
2373
+ `with_cache_capacity(0)` disable it (useful for tests/benchmarks).
2374
+- `crates/justapi-core/src/server/mod.rs`: `execute_handler` now takes an owned
2375
+ `Handler` + `params: Vec<(String,String)>` (replacing the borrowed `Match`),
2376
+ eliminating the router borrow on the hot path and letting `make_handler` call
2377
+ `router.resolve(...)` and forward owned params to `ParamsEcho`/custom handlers
2378
+ via request extensions. The existing `Match`/`at` API is retained for tests and
2379
+ `gateway.rs` (no behavior change there).
2380
+
2381
+**Evidence:** 4 new router unit tests (`test_resolve_static_hit_and_cache`,
2382
+`test_resolve_not_found_is_cached`, `test_resolve_param_route_bypasses_cache`,
2383
+`test_without_cache_still_resolves`). Full suite green (266 core tests). clippy
2384
+`-D warnings` + `fmt` clean. The cache is hit on every repeat request to a static
2385
+route / miss, removing the matchit traversal and the cross-method `MethodNotAllowed`
2386
+scan (which the pre-existing benchmark test still bounds at <100ns release / <1µs
2387
+debug). No change to routing semantics: param routes, method-not-allowed, and
2388
+fallback all resolve identically.
2389
+
2390
+**Trade-off:** small fixed memory per cached `(method, path)` key (bounded by
2391
+capacity, FIFO-evicted). No new dependency.
2353
2392
▼crates/justapi-core/src/router.rs
+170-1
@@ -1,4 +1,5 @@
1
1
use std::collections::HashMap;
2
+use std::sync::{Arc, Mutex};
2
3
3
4
use hyper::Method;
4
5
use matchit::Router as MatchitRouter;
@@ -17,16 +18,95 @@ pub struct Match<'a, 'p, T> {
17
18
pub params: matchit::Params<'a, 'p>,
18
19
}
19
20
21
+/// An owned route resolution (no borrows into the router), used by the hot
22
+/// request path and by the route cache.
23
+#[derive(Debug, Clone)]
24
+pub struct RouteResolution<T> {
25
+ pub handler: T,
26
+ pub params: Vec<(String, String)>,
27
+}
28
+
29
+/// Cache entry for a `(Method, path)` lookup. We only cache the two stable
30
+/// outcomes: a static-route hit (no path params) and a definitive `NotFound`.
31
+/// Param routes and `MethodNotAllowed` are never cached — they require re-running
32
+/// the full match (param extraction / cross-method scan) on every request.
33
+#[derive(Debug, Clone)]
34
+enum CacheEntry<T> {
35
+ Hit(RouteResolution<T>),
36
+ NotFound,
37
+}
38
+
39
+/// Bounded, lock-free-enough route-lookup cache. A `Mutex<HashMap>` with FIFO
40
+/// eviction keeps the working set small; the critical section is a single map
41
+/// op so contention is negligible next to the I/O it saves.
42
+#[derive(Debug)]
43
+struct RouteCache<T> {
44
+ map: Mutex<HashMap<(Method, String), CacheEntry<T>>>,
45
+ order: Mutex<Vec<(Method, String)>>,
46
+ capacity: usize,
47
+}
48
+
49
+impl<T: Clone> RouteCache<T> {
50
+ fn new(capacity: usize) -> Self {
51
+ Self {
52
+ map: Mutex::new(HashMap::with_capacity(capacity)),
53
+ order: Mutex::new(Vec::with_capacity(capacity)),
54
+ capacity: capacity.max(1),
55
+ }
56
+ }
57
+
58
+ /// Look up a cached resolution. Returns `None` on a negative (not-found)
59
+ /// entry or a miss.
60
+ fn get(&self, key: &(Method, String)) -> Option<CacheEntry<T>> {
61
+ self.map.lock().unwrap().get(key).cloned()
62
+ }
63
+
64
+ /// Insert a cacheable entry, evicting the oldest key if at capacity.
65
+ fn insert(&self, key: (Method, String), entry: CacheEntry<T>) {
66
+ let mut map = self.map.lock().unwrap();
67
+ let mut order = self.order.lock().unwrap();
68
+ if !map.contains_key(&key) {
69
+ order.push(key.clone());
70
+ if order.len() > self.capacity {
71
+ if let Some(old) = order.first().cloned() {
72
+ map.remove(&old);
73
+ order.remove(0);
74
+ }
75
+ }
76
+ }
77
+ map.insert(key, entry);
78
+ }
79
+}
80
+
20
81
#[derive(Debug, Clone)]
21
82
pub struct Router<T> {
22
83
routes: HashMap<Method, MatchitRouter<T>>,
23
84
fallback: Option<T>,
24
85
route_list: Vec<(Method, String)>,
86
+ /// Optional per-request route-lookup cache. `None` disables caching.
87
+ cache: Option<Arc<RouteCache<T>>>,
25
88
}
26
89
27
90
impl<T: Clone> Router<T> {
91
+ /// Create a router with an always-on route cache (default capacity 1024).
28
92
pub fn new() -> Self {
29
- Self { routes: HashMap::new(), fallback: None, route_list: Vec::new() }
93
+ Self {
94
+ routes: HashMap::new(),
95
+ fallback: None,
96
+ route_list: Vec::new(),
97
+ cache: Some(Arc::new(RouteCache::new(1024))),
98
+ }
99
+ }
100
+
101
+ /// Create a router without a route cache.
102
+ pub fn without_cache() -> Self {
103
+ Self { routes: HashMap::new(), fallback: None, route_list: Vec::new(), cache: None }
104
+ }
105
+
106
+ /// Replace the route cache with one sized to `capacity` (0 disables).
107
+ pub fn with_cache_capacity(mut self, capacity: usize) -> Self {
108
+ self.cache = if capacity == 0 { None } else { Some(Arc::new(RouteCache::new(capacity))) };
109
+ self
30
110
}
31
111
32
112
pub fn insert(&mut self, method: Method, path: &str, handler: T) -> Result<(), InsertError> {
@@ -68,6 +148,57 @@ impl<T: Clone> Router<T> {
68
148
Err(RouterError::NotFound)
69
149
}
70
150
151
+ /// Shared match logic returning an owned resolution (no borrows).
152
+ fn raw_resolve(&self, method: &Method, path: &str) -> Result<RouteResolution<T>, RouterError> {
153
+ if let Some(router) = self.routes.get(method) {
154
+ if let Ok(matched) = router.at(path) {
155
+ let params = matched
156
+ .params
157
+ .iter()
158
+ .map(|(k, v)| (k.to_string(), v.to_string()))
159
+ .collect::<Vec<_>>();
160
+ return Ok(RouteResolution { handler: matched.value.clone(), params });
161
+ }
162
+ }
163
+
164
+ for (other_method, router) in &self.routes {
165
+ if other_method != method && router.at(path).is_ok() {
166
+ return Err(RouterError::MethodNotAllowed);
167
+ }
168
+ }
169
+
170
+ Err(RouterError::NotFound)
171
+ }
172
+
173
+ /// Resolve a route to an owned `RouteResolution`, memoized through the
174
+ /// optional route cache. Static-route hits and definitive `NotFound`s are
175
+ /// cached; param routes and `MethodNotAllowed` bypass the cache (they need
176
+ /// a fresh full match each time).
177
+ pub fn resolve(&self, method: &Method, path: &str) -> Result<RouteResolution<T>, RouterError> {
178
+ if let Some(cache) = &self.cache {
179
+ let key = (method.clone(), path.to_string());
180
+ if let Some(entry) = cache.get(&key) {
181
+ return match entry {
182
+ CacheEntry::Hit(res) => Ok(res),
183
+ CacheEntry::NotFound => Err(RouterError::NotFound),
184
+ };
185
+ }
186
+ let res = self.raw_resolve(method, path);
187
+ match &res {
188
+ Ok(r) if r.params.is_empty() => {
189
+ cache.insert(key, CacheEntry::Hit(r.clone()));
190
+ }
191
+ Err(RouterError::NotFound) => {
192
+ cache.insert(key, CacheEntry::NotFound);
193
+ }
194
+ _ => {}
195
+ }
196
+ res
197
+ } else {
198
+ self.raw_resolve(method, path)
199
+ }
200
+ }
201
+
71
202
pub fn set_fallback(&mut self, handler: T) {
72
203
self.fallback = Some(handler);
73
204
}
@@ -95,6 +226,44 @@ mod tests {
95
226
assert_eq!(*m.handler, "hello_handler");
96
227
}
97
228
229
+ #[test]
230
+ fn test_resolve_static_hit_and_cache() {
231
+ let mut router = Router::new();
232
+ router.insert(Method::GET, "/hello", "hello_handler").unwrap();
233
+ let r = router.resolve(&Method::GET, "/hello").unwrap();
234
+ assert_eq!(r.handler, "hello_handler");
235
+ assert!(r.params.is_empty());
236
+ // Second call must hit the cache and return the same handler.
237
+ let r2 = router.resolve(&Method::GET, "/hello").unwrap();
238
+ assert_eq!(r2.handler, "hello_handler");
239
+ }
240
+
241
+ #[test]
242
+ fn test_resolve_not_found_is_cached() {
243
+ let mut router = Router::new();
244
+ router.insert(Method::GET, "/hello", "hello_handler").unwrap();
245
+ assert!(router.resolve(&Method::GET, "/missing").is_err());
246
+ // Negative result is memoized; still NotFound on repeat.
247
+ assert!(router.resolve(&Method::GET, "/missing").is_err());
248
+ }
249
+
250
+ #[test]
251
+ fn test_resolve_param_route_bypasses_cache() {
252
+ let mut router = Router::new();
253
+ router.insert(Method::GET, "/users/{id}", "user_handler").unwrap();
254
+ let r = router.resolve(&Method::GET, "/users/42").unwrap();
255
+ assert_eq!(r.handler, "user_handler");
256
+ assert_eq!(r.params, vec![("id".to_string(), "42".to_string())]);
257
+ }
258
+
259
+ #[test]
260
+ fn test_without_cache_still_resolves() {
261
+ let mut router = Router::without_cache();
262
+ router.insert(Method::GET, "/hello", "hello_handler").unwrap();
263
+ let r = router.resolve(&Method::GET, "/hello").unwrap();
264
+ assert_eq!(r.handler, "hello_handler");
265
+ }
266
+
98
267
#[test]
99
268
fn test_route_not_found() {
100
269
let mut router = Router::new();
101
270
▼crates/justapi-core/src/server/mod.rs
+14-14
@@ -21,7 +21,7 @@ use crate::middleware::{
21
21
};
22
22
use crate::openapi;
23
23
use crate::plugin::PluginRegistry;
24
-use crate::router::{Match, Router};
24
+use crate::router::Router;
25
25
use crate::static_files::{StaticDir, StaticMount};
26
26
use crate::{json_response, streaming_response, ResponseBody};
27
27
#[cfg(feature = "ws")]
@@ -1156,10 +1156,11 @@ fn make_handler(
1156
1156
Box::pin(async move {
1157
1157
let method = req.method().clone();
1158
1158
let path = req.uri().path().to_string();
1159
- match router.at(&method, &path) {
1160
- Ok(m) => {
1159
+ match router.resolve(&method, &path) {
1160
+ Ok(res) => {
1161
1161
execute_handler(
1162
- m,
1162
+ res.handler,
1163
+ res.params,
1163
1164
req,
1164
1165
&pool,
1165
1166
&metrics,
@@ -1176,9 +1177,9 @@ fn make_handler(
1176
1177
)),
1177
1178
Err(crate::router::RouterError::NotFound) => match router.fallback() {
1178
1179
Some(fb) => {
1179
- let m = Match { handler: fb, params: matchit::Params::new() };
1180
1180
execute_handler(
1181
- m,
1181
+ fb.clone(),
1182
+ Vec::new(),
1182
1183
req,
1183
1184
&pool,
1184
1185
&metrics,
@@ -1724,7 +1725,8 @@ async fn serve_unix(
1724
1725
// ---------------------------------------------------------------------------
1725
1726
1726
1727
async fn execute_handler(
1727
- m: Match<'_, '_, Handler>,
1728
+ handler: Handler,
1729
+ params: Vec<(String, String)>,
1728
1730
req: Request<Incoming>,
1729
1731
pool: &BufferPool,
1730
1732
metrics: &Metrics,
@@ -1733,7 +1735,7 @@ async fn execute_handler(
1733
1735
openapi_spec: Option<&str>,
1734
1736
max_body_size: usize,
1735
1737
) -> Result<Response<ResponseBody>> {
1736
- let handler_name = match m.handler {
1738
+ let handler_name = match handler {
1737
1739
Handler::Static { .. } => "static",
1738
1740
Handler::Echo => "echo",
1739
1741
Handler::ParamsEcho => "params_echo",
@@ -1750,8 +1752,8 @@ async fn execute_handler(
1750
1752
};
1751
1753
let _handler_span = tracing::debug_span!("handler.execute", name = handler_name);
1752
1754
1753
- match m.handler {
1754
- Handler::Static { status, body } => Ok(json_response(*status, body)),
1755
+ match handler {
1756
+ Handler::Static { status, body } => Ok(json_response(status, body)),
1755
1757
Handler::Echo => {
1756
1758
let body_bytes = match http_body_util::Limited::new(req.into_body(), max_body_size)
1757
1759
.collect()
@@ -1783,7 +1785,7 @@ async fn execute_handler(
1783
1785
}
1784
1786
Handler::ParamsEcho => {
1785
1787
let params_str: Vec<String> =
1786
- m.params.iter().map(|(k, v)| format!(r#""{}":"{}""#, k, v)).collect();
1788
+ params.iter().map(|(k, v)| format!(r#""{}":"{}""#, k, v)).collect();
1787
1789
let body = format!("{{{}}}", params_str.join(","));
1788
1790
Ok(json_response(StatusCode::OK, &body))
1789
1791
}
@@ -1857,9 +1859,7 @@ async fn execute_handler(
1857
1859
Handler::Custom(f) => {
1858
1860
// Expose matched path params to custom handlers (e.g. the Rust-native
1859
1861
// CRUD handlers read `{id}` from here) via request extensions.
1860
- // `m.params` borrows the router, so collect into an owned Vec first.
1861
- let path_params: Vec<(String, String)> =
1862
- m.params.iter().map(|(k, v)| (k.to_string(), v.to_string())).collect();
1862
+ let path_params: Vec<(String, String)> = params;
1863
1863
let mut req = req;
1864
1864
req.extensions_mut().insert(path_params);
1865
1865
f(req).await
1866
1866