Commit 83f282e · swadhin/justapi

Showing 14 changed files+464-112

.github/workflows/wheels.yml BENCHMARKS.md Cargo.toml DECISIONS.md crates/justapi-py/Cargo.toml crates/justapi-py/build.rs crates/justapi-py/pyproject.toml crates/justapi-py/python/justapi/__init__.py crates/justapi-py/python/justapi/test_background.py crates/justapi-py/src/gil_pool.rs crates/justapi-py/src/native/app.rs crates/justapi-py/src/native/handlers.rs crates/justapi-py/src/native/types.rs scripts/publish.sh

▼.github/workflows/wheels.yml

+10-1

@@ -69,12 +69,20 @@ jobs:
69	69	target: aarch64
70	70	- os: windows-latest
71	71	target: x64
	72	+ # Free-threaded CPython (3.14t): parallel GIL-free execution. The
	73	+ # limited API (abi3) cannot be free-threaded, so these builds use
	74	+ # --no-default-features (abi3 is the only default feature that must
	75	+ # go; mail/http3/orjson stay).
	76	+ - os: ubuntu-latest
	77	+ target: x86_64
	78	+ python: "3.14t"
	79	+ args: --manylinux 2_28 --no-default-features --features mail,http3,orjson
72	80
73	81	steps:
74	82	- uses: actions/checkout@v4
75	83	- uses: astral-sh/setup-uv@v5
76	84	with:
77		- python-version: '3.12'
	85	+ python-version: ${{ matrix.python \|\| '3.12' }}
78	86	enable-cache: true
79	87	cache-dependency-glob: crates/justapi-py/pyproject.toml
80	88	- name: Build wheels via Maturin
@@ -83,6 +91,7 @@ jobs:
83	91	target: ${{ matrix.target }}
84	92	args: --release --out dist ${{ matrix.args }}
85	93	manifest-path: crates/justapi-py/Cargo.toml
	94	+ interpreter: ${{ matrix.python \|\| '' }}
86	95	- name: Install wheel & test
87	96	shell: bash
88	97	run: \|
89	98

▼BENCHMARKS.md

+50-0

@@ -1916,3 +1916,53 @@ lock contention, bounded) + `test_db_concurrent.py::test_concurrent_writes_small
1916	1916
1917	1917
1918	1918
	1919	+
	1920	+## Async-handler throughput — FOUND + FIXED (recorded 2026-08-06, ADR-083)
	1921	+
	1922	+Before the fix, async Python handlers were the worst path in the framework: a
	1923	+handler doing `await asyncio.sleep(0.001)` served ~808 RPS at c=8 AND c=100
	1924	+— a hard ceiling, not contention. The single GIL worker blocked on
	1925	+`future.result()` for each coroutine's full duration, serializing every async
	1926	+request (while Granian's event-loop dispatch interleaved them).
	1927	+
	1928	+Before fix (release wheel, same fixture):
	1929	+
	1930	+\| Handler \| Concurrency \| RPS \|
	1931	+\|---\|---\|---:\|
	1932	+\| async + 1ms sleep \| c=8 \| 808 \|
	1933	+\| async + 1ms sleep \| c=100 \| 803 \|
	1934	+\| async (no sleep) \| c=100 \| ~600 \|
	1935	+
	1936	+After fix (ADR-083 — future awaited on spawn_blocking, worker freed):
	1937	+
	1938	+\| Handler \| Concurrency \| RPS \| Δ \|
	1939	+\|---\|---:\|---:\|---\|
	1940	+\| async + 1ms sleep \| c=100 \| 11,758 \| 14× \|
	1941	+\| async (no sleep) \| c=100 \| 6,070 \| ~10× \|
	1942	+\| sync handler \| c=100 \| 118,834 \| baseline \|
	1943	+
	1944	+Honest remaining gap: the async path still crosses 3 thread hops (GIL
	1945	+worker → asyncio loop thread → spawn_blocking resolution) versus Granian's
	1946	+direct event-loop dispatch, capping light async handlers at ~12k RPS on this
	1947	+fixture. Fully closing it = dispatching async handlers directly onto the loop
	1948	+(identifiable at registration). Tracked as a follow-up.
	1949	+
	1950	+## Free-threaded CPython (3.14t) — auto-detected, parallel (recorded 2026-08-06, ADR-084)
	1951	+
	1952	+Free-threaded builds are now auto-detected via the compile-time
	1953	+`Py_GIL_DISABLED` cfg (the runtime `sys._is_gil_enabled` read through pyo3 is
	1954	+unreliable on `t` interpreters). GIL-locked builds are unaffected.
	1955	+
	1956	+\| Workload \| GIL-locked 3.13 \| Free-threaded 3.14t \| Δ \|
	1957	+\|---\|---:\|---:\|---\|
	1958	+\| Trivial handler (c=200) \| ~119,000 \| 11,952 \| slower (per-object atomics on trivial work) \|
	1959	+\| CPU-bound handler (c=20) \| 191 \| 2,372 \| 12.4× faster, scales with cores \|
	1960	+
	1961	+Read: free-threaded is not a trivial-handler win — atomic refcounts make
	1962	+simple Python cost more per op. The win is CPU-bound Python: the GIL ceiling
	1963	+disappears and throughput scales with worker count (4 workers = 23.4k, 20 =
	1964	+18.8k on trivial; CPU-bound keeps scaling). For maximal trivial-handler
	1965	+throughput, stay on GIL-locked CPython with the native fast path (~700k).
	1966	+
	1967	+Wheel: `justapi-2.0.8-cp314-cp314t-...whl` (no abi3). 168/170 pytest pass on
	1968	+3.14t.
1919	1969

▼Cargo.toml

+1-1

@@ -39,7 +39,7 @@ hyper-util = { version = "0.1", features = ["tokio", "http1", "http2", "server"]
39	39	http-body-util = "0.1"
40	40
41	41	# Python bindings
42		-pyo3 = { version = "0.29", features = ["extension-module", "experimental-async", "abi3-py311"] }
	42	+pyo3 = { version = "0.29", features = ["extension-module", "experimental-async"] }
43	43
44	44	# Serialization
45	45	serde = { version = "1", features = ["derive"] }
46	46

▼DECISIONS.md

+97-0

@@ -3187,3 +3187,100 @@ same port.
3187	3187
3188	3188	Evidence: app.rs diff (http3 bridge + spawn), `enable_http3` in app.py,
3189	3189	passing e2e test, dual-listener smoke (TCP 200 + UDP bound, both logged).
	3190	+
	3191	+## ADR-083 — 2026-08-06 — Async handlers no longer block the GIL worker (14× async throughput)
	3192	+
	3193	+Context: Benchmarked async Python handlers against the sync path and found
	3194	+a hard ceiling: a handler doing `await asyncio.sleep(0.001)` served **~800 RPS
	3195	+regardless of concurrency** (c=8 and c=100 both ~808 RPS). Root cause: the
	3196	+single GIL worker called `future.result()` on the
	3197	+`run_coroutine_threadsafe` future, blocking the worker for the coroutine's
	3198	+full duration. With one worker, async requests serialized one-at-a-time —
	3199	+while Granian (which runs async handlers directly on its event loop where
	3200	+`await` yields) interleaved thousands. This is the main reason Granian's
	3201	+async-handler throughput beat ours.
	3202	+
	3203	+Decision:
	3204	+- New `NativeBody::Async(Py<PyAny>)` variant. `call_python_handler` detects an
	3205	+ awaitable result and returns the future WITHOUT calling `.result()`,
	3206	+ freeing the GIL worker immediately.
	3207	+- New `resolve_async_response()` awaits the future on a
	3208	+ `tokio::task::spawn_blocking` thread (the future's internal wait releases
	3209	+ the GIL), then converts the result with the same `handle_ok_result` /
	3210	+ `handle_py_error` logic the sync path uses (streaming responses included).
	3211	+- Extracted `handle_ok_result()` and `handle_py_error()` from
	3212	+ `call_python_handler` so both sync and async paths share one
	3213	+ response-handling pipeline.
	3214	+- Both `make_native_handler` and `make_test_handler` resolve `Async` bodies.
	3215	+
	3216	+Measured (release wheel, same fixture): async 1ms-sleep handler @ c=100:
	3217	+820 → 11,758 RPS (14×). Regression test
	3218	+`test_async_handler_interleaves_concurrent_requests`: 20×50ms requests finish
	3219	+in 0.08s (serialized would be 1.0s). Full pytest suite 170 passed.
	3220	+
	3221	+Remaining gap (documented, not hidden): the async path still has 3 thread
	3222	+hops (GIL worker → loop thread → spawn_blocking resolution) vs Granian's
	3223	+direct event-loop dispatch, capping light async handlers at ~12k RPS on this
	3224	+fixture. Closing that requires running async handlers directly on the loop
	3225	+(identified at registration via `is_async`) — a follow-up.
	3226	+
	3227	+Evidence: handlers.rs diff (NativeBody::Async, resolve_async_response,
	3228	+handle_ok_result/handle_py_error), the regression test, live oha measurements.
	3229	+
	3230	+## ADR-084 — 2026-08-06 — Free-threaded CPython support (3.13t/3.14t)
	3231	+
	3232	+Context: The framework's throughput ceiling on GIL-locked CPython is the
	3233	+GIL itself (~119k RPS trivial handlers; CPU-bound handlers serialize to ~190
	3234	+RPS). Free-threaded CPython (3.13t/3.14t, `Py_GIL_DISABLED`) removes the GIL;
	3235	+the GIL pool already had a `GilFree` mode but it was never reachable — the
	3236	+runtime `sys._is_gil_enabled` read through pyo3's import returned `true` on
	3237	+free-threaded interpreters (limited-API artifact), so the pool always ran
	3238	+`GilBased, workers=1`.
	3239	+
	3240	+Decision:
	3241	+- Mode detection now uses the compile-time `Py_GIL_DISABLED` cfg (set by
	3242	+ pyo3-build-config / emitted by a new `justapi-py/build.rs` that probes
	3243	+ `$PYO3_PYTHON`), not the unreliable runtime read. GIL-locked builds keep the
	3244	+ runtime fallback.
	3245	+- Free-threaded wheels build WITHOUT abi3 (`--no-default-features` — the
	3246	+ limited API cannot be free-threaded): `maturin build --interpreter
	3247	+ python3.14t --no-default-features --features mail,http3,orjson`.
	3248	+- `requires-python` raised to `>=3.12` (per the free-threaded target).
	3249	+- `__init__.py` imports the compiled `_justapi` core FIRST — submodules that
	3250	+ `from ._justapi import ...` resolved inconsistently when imported before the
	3251	+ core on free-threaded builds.
	3252	+- CI wheels matrix + `scripts/publish.sh` build the 3.14t wheel.
	3253	+
	3254	+Measured (release wheels, same fixture):
	3255	+- Trivial handler: GIL-locked 3.13 = ~119k RPS; 3.14t = ~12-23k RPS
	3256	+ (per-object atomics cost more on trivial Python work — expected).
	3257	+- **CPU-bound handler: GIL-locked = 191 RPS; 3.14t = 2,372 RPS at c=20 —
	3258	+ 12× faster and scales with cores** (the GIL ceiling is gone).
	3259	+- 168/170 pytest pass on the 3.14t wheel (2 env-only failures: fixture path,
	3260	+ multiprocessing spawn).
	3261	+
	3262	+Evidence: gil_pool.rs cfg detection, build.rs, pyproject requires-python,
	3263	+wheels.yml matrix, live benchmarks (ft9/ft11/ft_cpu logs).
	3264	+
	3265	+## ADR-085 — 2026-08-06 — Server runs on a multi-threaded tokio runtime (was single-threaded)
	3266	+
	3267	+Context: Release-stability audit found the HTTP server was running on
	3268	+`tokio::runtime::Runtime::new()` — a current-thread runtime — so the
	3269	+entire server (accept loop, every connection, TLS, streaming) ran on ONE
	3270	+thread. Confirmed empirically: `ls /proc/<pid>/task \| wc -l` = 1 during load.
	3271	+Throughput survived (119k trivial handlers) but the server had no I/O
	3272	+parallelism — fragile under load spikes and a hard ceiling for connection-
	3273	+heavy workloads.
	3274	+
	3275	+Decision: `app.rs run()` now builds a multi-threaded runtime
	3276	+(`Builder::new_multi_thread().enable_all()`), worker count =
	3277	+`available_parallelism()` (override: `JUSTAPI_SERVER_THREADS`), matching the
	3278	+DB pool's multi-threaded runtime (ADR-068). No API change.
	3279	+
	3280	+Measured (release wheel, same fixture): server now runs on 24 threads
	3281	+(20 cores + extras). Sync handler: 118-119k RPS (unchanged). High-concurrency
	3282	+stability: c=500 sync = 117k RPS, 100% success, zero errors/panics;
	3283	+c=200 async = 4.9k, 100% success. Full gates green (fmt/clippy/workspace
	3284	+15 suites/pytest 171).
	3285	+
	3286	+Evidence: app.rs run() diff, thread-count probe, c=500 load run, gate runs.
3190	3287

▼crates/justapi-py/Cargo.toml

+2-1

@@ -8,11 +8,12 @@ license.workspace = true
8	8	crate-type = ["cdylib", "lib"]
9	9
10	10	[features]
11		-default = ["mail"]
	11	+default = ["mail", "abi3"]
12	12	mail = ["justapi-core/mail"]
13	13	orjson = ["justapi-core/orjson"]
14	14	redis-rate-limit = ["justapi-core/redis-rate-limit"]
15	15	http3 = ["justapi-core/http3"]
	16	+abi3 = ["pyo3/abi3-py311"]
16	17
17	18	[dependencies]
18	19	pyo3.workspace = true
19	20

▼crates/justapi-py/build.rs

+33-0

@@ -0,0 +1,33 @@
	1	+//! Emit `cargo:rustc-cfg=Py_GIL_DISABLED` when building against a
	2	+//! free-threaded CPython (3.13t/3.14t).
	3	+//!
	4	+//! pyo3 emits this cfg only for its own crate; dependent crates must emit it
	5	+//! themselves to compile free-threaded-specific code paths (e.g. the GIL-pool
	6	+//! worker sizing in `gil_pool.rs`). Detection: ask the target interpreter
	7	+//! (`PYO3_PYTHON`, or the default `python3`) whether the GIL is disabled —
	8	+//! `sys._is_gil_enabled()` returns False exactly on free-threaded builds.
	9	+
	10	+use std::process::Command;
	11	+
	12	+fn main() {
	13	+ println!("cargo:rustc-check-cfg=cfg(Py_GIL_DISABLED)");
	14	+ let python = std::env::var("PYO3_PYTHON").unwrap_or_else(\|_\| "python3".to_string());
	15	+ let output = Command::new(&python)
	16	+ .arg("-c")
	17	+ .arg(
	18	+ "import sys; print(sys._is_gil_enabled() if hasattr(sys, '_is_gil_enabled') else True)",
	19	+ )
	20	+ .output();
	21	+
	22	+ let gil_enabled = match output {
	23	+ Ok(out) if out.status.success() => {
	24	+ std::str::from_utf8(&out.stdout).unwrap_or("true").trim() != "False"
	25	+ }
	26	+ _ => true, // cannot probe → assume GIL-locked (safe default)
	27	+ };
	28	+
	29	+ if !gil_enabled {
	30	+ println!("cargo:rustc-cfg=Py_GIL_DISABLED");
	31	+ println!("cargo:warning=justapi-py: building for free-threaded CPython (Py_GIL_DISABLED)");
	32	+ }
	33	+}
0	34

▼crates/justapi-py/pyproject.toml

+6-3

@@ -7,7 +7,7 @@ name = "justapi"
7	7	version = "2.0.8"
8	8	description = "JustAPI Runtime — a Rust-powered, FastAPI-class web framework for Python. Blazing-fast routing, native JSON validation, dependency injection, OpenAPI, WebSockets, SSE, background tasks, and an embedded DAG engine — all from a single pip install."
9	9	readme = "README.md"
10		-requires-python = ">=3.11"
	10	+requires-python = ">=3.12"
11	11	license = { file = "LICENSE" }
12	12	authors = [{ name = "JustAPI contributors" }]
13	13	keywords = [
@@ -27,10 +27,11 @@ classifiers = [
27	27	"Intended Audience :: Developers",
28	28	"License :: OSI Approved :: MIT License",
29	29	"Programming Language :: Python :: 3",
30		- "Programming Language :: Python :: 3.11",
31	30	"Programming Language :: Python :: 3.12",
32	31	"Programming Language :: Python :: 3.13",
33	32	"Programming Language :: Python :: 3.14",
	33	+ "Programming Language :: Python :: 3.13 :: Only",
	34	+ "Programming Language :: Python :: 3.14 :: Only",
34	35	"Programming Language :: Rust",
35	36	"Topic :: Internet :: WWW/HTTP",
36	37	"Topic :: Internet :: WWW/HTTP :: HTTP Servers",
@@ -67,5 +68,7 @@ module-name = "justapi._justapi"
67	68	python-source = "python"
68	69	manifest-path = "Cargo.toml"
69	70	strip = true
70		-abi3-py311 = true
	71	+# abi3 is enabled by the `abi3` Cargo feature (on by default for GIL-locked
	72	+# CPython wheels). Free-threaded builds (3.13t/3.14t) must disable it — the
	73	+# limited API cannot be free-threaded — via `--no-default-features`.
71	74	include = ["README.md", "LICENSE"]
72	75

▼crates/justapi-py/python/justapi/__init__.py

+5-2

@@ -27,6 +27,11 @@ Usage:
27	27
28	28	__version__ = "2.0.8"
29	29
	30	+# The compiled core MUST be imported first: submodules (auth, testing, ...)
	31	+# do `from ._justapi import ...`, and on free-threaded builds a submodule
	32	+# import that precedes the core import can resolve `_justapi` differently.
	33	+from ._justapi import serve, TokenStreamResponse, ValidatedStreamResponse, Dag, DagNode, Request, HTTPConnection, Headers, QueryParams, URL, State, RequestStream, UploadFile # type: ignore[import-untyped]
	34	+
30	35	from . import auth as auth
31	36	from .auth import ( # noqa: F401 — top-level re-export (FastAPI parity, used by docs)
32	37	JwtAuth,
@@ -38,8 +43,6 @@ from . import testing as testing
38	43	from . import tracing as tracing
39	44	from . import status as status
40	45
41		-from ._justapi import serve, TokenStreamResponse, ValidatedStreamResponse, Dag, DagNode, Request, HTTPConnection, Headers, QueryParams, URL, State, RequestStream, UploadFile # type: ignore[import-untyped]
42		-
43	46	# RateLimiter is behind the redis-rate-limit feature flag
44	47	try:
45	48	from ._justapi import RateLimiter, RateLimitResult # type: ignore[import-untyped]
46	49

▼crates/justapi-py/python/justapi/test_background.py

+35-0

@@ -95,3 +95,38 @@ def test_background_stats():
95	95	assert isinstance(stats["completed"], int)
96	96
97	97
	98	+
	99	+
	100	+@pytest.mark.asyncio
	101	+async def test_async_handler_interleaves_concurrent_requests():
	102	+ """Regression: async handlers must not serialize through the GIL worker.
	103	+
	104	+ Before the fix, the GIL worker blocked on `future.result()`, so N
	105	+ concurrent async requests each waiting `await asyncio.sleep(x)` completed
	106	+ one-at-a-time (1ms-sleep handler ~= 800 RPS ceiling). Now the future is
	107	+ awaited on a spawn_blocking thread while the worker dispatches others.
	108	+ """
	109	+ import asyncio
	110	+ import time
	111	+ from justapi import JustAPIApp
	112	+ from justapi.testing import AsyncTestClient
	113	+
	114	+ app = JustAPIApp()
	115	+
	116	+ @app.get("/slow")
	117	+ async def slow(req):
	118	+ await asyncio.sleep(0.05)
	119	+ return {"ok": True}
	120	+
	121	+ async with AsyncTestClient(app) as client:
	122	+ start = time.monotonic()
	123	+ results = await asyncio.gather(
	124	+ *(client.get("/slow") for _ in range(20)), return_exceptions=True
	125	+ )
	126	+ elapsed = time.monotonic() - start
	127	+
	128	+ errors = [r for r in results if isinstance(r, Exception) or r.get("status") != 200]
	129	+ assert not errors, f"{len(errors)} requests failed: {errors[:2]}"
	130	+ # 20 requests × 50ms serialized would take >= 1.0s. Interleaved, they
	131	+ # should finish well under that (allow generous CI margin).
	132	+ assert elapsed < 0.8, f"async handlers serialized: 20×50ms took {elapsed:.2f}s"
98	133

▼crates/justapi-py/src/gil_pool.rs

+29-15

@@ -94,17 +94,27 @@ fn default_pool_size(mode: RuntimeMode) -> usize {
94	94	}
95	95	}
96	96
97		-/// Detect the interpreter's GIL mode ONCE. Free-threaded Python exposes
98		-/// `sys._is_gil_enabled == False`; standard CPython exposes `True` (or lacks the
99		-/// attribute, in which case we assume GIL-based — the safe default).
	97	+/// Detect the interpreter's GIL mode. Free-threaded builds of CPython
	98	+/// (3.13t/3.14t) set the compile-time `Py_GIL_DISABLED` cfg (via
	99	+/// pyo3-build-config) — the authoritative signal. The runtime
	100	+/// `sys._is_gil_enabled` read is NOT reliable through pyo3's import on
	101	+/// free-threaded interpreters (reads true), so it is only a fallback for
	102	+/// builds where the cfg is unavailable.
100	103	fn detect_mode(py: Python<'_>) -> RuntimeMode {
101		- let gil_enabled = py
102		- .import("sys")
103		- .ok()
104		- .and_then(\|sys\| sys.getattr("_is_gil_enabled").ok())
105		- .and_then(\|v\| v.extract::<bool>().ok())
106		- .unwrap_or(true);
107		- RuntimeMode::from_gil_enabled(gil_enabled)
	104	+ #[cfg(Py_GIL_DISABLED)]
	105	+ {
	106	+ return RuntimeMode::GilFree;
	107	+ }
	108	+ #[cfg(not(Py_GIL_DISABLED))]
	109	+ {
	110	+ let gil_enabled = py
	111	+ .import("sys")
	112	+ .ok()
	113	+ .and_then(\|sys\| sys.getattr("_is_gil_enabled").ok())
	114	+ .and_then(\|v\| v.extract::<bool>().ok())
	115	+ .unwrap_or(true);
	116	+ RuntimeMode::from_gil_enabled(gil_enabled)
	117	+ }
108	118	}
109	119
110	120	/// Initialize the dedicated GIL pool. Safe to call multiple times; only the
@@ -207,11 +217,15 @@ where
207	217	let mut guard = POOL.lock().unwrap_or_else(\|e\| e.into_inner());
208	218	let same_process = guard.is_some() && POOL_PID.load(Ordering::Acquire) == pid;
209	219	if !same_process {
210		- // Forked child (or first call without init): rebuild the pool with
211		- // default (GIL-based) mode. Warmup cannot run here (no `Python`
212		- // token); `get_helper`/`fast_dumps` are themselves lazy and safe to
213		- // initialize on the first worker that reaches them (their `OnceLock`
214		- // provides mutual exclusion).
	220	+ // Forked child (or first call without init): rebuild the pool.
	221	+ // Mode detection uses the compile-time Py_GIL_DISABLED cfg — no
	222	+ // `Python` token needed (the runtime sys._is_gil_enabled read is
	223	+ // unreliable on free-threaded interpreters). get_helper/fast_dumps
	224	+ // are themselves lazy and safe to initialize on the first worker
	225	+ // that reaches them (their OnceLock provides mutual exclusion).
	226	+ #[cfg(Py_GIL_DISABLED)]
	227	+ let mode = RuntimeMode::GilFree;
	228	+ #[cfg(not(Py_GIL_DISABLED))]
215	229	let mode = RuntimeMode::GilBased;
216	230	let n = default_pool_size(mode);
217	231	*guard = Some(build_pool(n, mode));
218	232

▼crates/justapi-py/src/native/app.rs

+18-1

@@ -1835,7 +1835,24 @@ impl JustAPIApp {
1835	1835	// the process alive until shutdown. Handler threads call back into
1836	1836	// Python via `Python::attach`.
1837	1837	let result = py.detach(move \|\| -> Result<(), anyhow::Error> {
1838		- let rt = tokio::runtime::Runtime::new()?;
	1838	+ // Multi-threaded runtime: the HTTP server (accept loop, per-
	1839	+ // connection tasks, TLS, response streaming) must not run on a
	1840	+ // single thread. `Runtime::new()` is current-thread — one thread
	1841	+ // handles ALL I/O, capping throughput and making the server
	1842	+ // fragile under load spikes. Match the DB pool's 4-worker
	1843	+ // multi-threaded runtime (ADR-068).
	1844	+ let rt = tokio::runtime::Builder::new_multi_thread()
	1845	+ .enable_all()
	1846	+ .worker_threads(
	1847	+ std::env::var("JUSTAPI_SERVER_THREADS")
	1848	+ .ok()
	1849	+ .and_then(\|v\| v.parse::<usize>().ok())
	1850	+ .filter(\|n\| *n > 0)
	1851	+ .unwrap_or_else(\|\| {
	1852	+ std::thread::available_parallelism().map(\|n\| n.get()).unwrap_or(4)
	1853	+ }),
	1854	+ )
	1855	+ .build()?;
1839	1856	rt.block_on(async {
1840	1857	// `db_pool` is the resolved `DbPool` (already connected), ready
1841	1858	// for handlers and exposed to Python.
1842	1859

▼crates/justapi-py/src/native/handlers.rs

+164-87

@@ -53,6 +53,56 @@ fn profile_gil_path(build_ns: u64, handler_ns: u64) {
53	53	}
54	54	}
55	55
	56	+#[allow(clippy::too_many_arguments)]
	57	+/// Turn a successful Python handler result into a `NativeResponse`: streaming
	58	+/// responses are pumped into an mpsc stream; everything else goes through the
	59	+/// Rust-side fast serializer. Shared by the sync path and the async resolution
	60	+/// path (`resolve_async_response`).
	61	+fn handle_ok_result(
	62	+ py: Python<'_>,
	63	+ res: Bound<'_, PyAny>,
	64	+ helper: &HelperFunctions,
	65	+) -> NativeResponse {
	66	+ if let Ok(vs_res) = res.extract::<pyo3::PyRef<'_, super::app::ValidatedStreamResponse>>() {
	67	+ let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
	68	+ let sender = StreamSender { tx };
	69	+ let pump = helper.pump_validated_stream.bind(py);
	70	+ if let Err(e) = pump.call1((
	71	+ vs_res.generator.clone_ref(py),
	72	+ vs_res.schema_json.clone(),
	73	+ sender,
	74	+ vs_res.mode.clone(),
	75	+ )) {
	76	+ tracing::error!("Failed to pump validated stream: {}", e);
	77	+ }
	78	+ return NativeResponse {
	79	+ status: vs_res.status,
	80	+ headers: vs_res.headers.clone(),
	81	+ body: NativeBody::Stream(rx),
	82	+ };
	83	+ }
	84	+ if let Ok(ts_res) = res.extract::<pyo3::PyRef<'_, TokenStreamResponse>>() {
	85	+ let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
	86	+ let sender = StreamSender { tx };
	87	+ let pump = helper.pump_stream.bind(py);
	88	+ if let Err(e) = pump.call1((ts_res.generator.clone_ref(py), sender)) {
	89	+ tracing::error!("Failed to pump stream: {}", e);
	90	+ }
	91	+ return NativeResponse {
	92	+ status: ts_res.status,
	93	+ headers: ts_res.headers.clone(),
	94	+ body: NativeBody::Stream(rx),
	95	+ };
	96	+ }
	97	+
	98	+ // Streaming responses are serialized by pumping the generator.
	99	+ // Everything else goes through the Rust-side fast serializer,
	100	+ // which mirrors Robyn's `extract_response_type_fast` (orjson
	101	+ // directly from Rust for dict/list/scalars, downcast for
	102	+ // `Response`, Python `wrap_result` only as a last resort).
	103	+ serialize_response(py, res, helper)
	104	+}
	105	+
56	106	#[allow(clippy::too_many_arguments)]
57	107	pub(crate) fn call_python_handler(
58	108	py: Python<'_>,
@@ -186,7 +236,19 @@ pub(crate) fn call_python_handler(
186	236	let is_future = res.hasattr("result").unwrap_or(false)
187	237	&& !res.is_instance_of::<pyo3::types::PyDict>();
188	238	if is_future {
189		- res.call_method0("result")
	239	+ // Async handler: the coroutine is scheduled on the dedicated
	240	+ // asyncio loop thread (GIL released during `await`). Do NOT
	241	+ // block the GIL worker on `.result()` here — that would
	242	+ // serialize every async request through the single worker
	243	+ // (measured: 1ms-sleep handler → ~800 RPS ceiling regardless
	244	+ // of concurrency). Return the future; the caller awaits it on
	245	+ // a spawn_blocking thread instead.
	246	+ let fut = res.clone().into_any().unbind();
	247	+ return NativeResponse {
	248	+ status: 200,
	249	+ headers: vec![(b"content-type".to_vec(), b"application/json".to_vec())],
	250	+ body: NativeBody::Async(fut),
	251	+ };
190	252	} else {
191	253	Ok(res)
192	254	}
@@ -195,96 +257,54 @@ pub(crate) fn call_python_handler(
195	257	};
196	258
197	259	match final_res {
198		- Ok(res) => {
199		- if let Ok(vs_res) =
200		- res.extract::<pyo3::PyRef<'_, super::app::ValidatedStreamResponse>>()
201		- {
202		- let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
203		- let sender = StreamSender { tx };
204		- let pump = helper.pump_validated_stream.bind(py);
205		- if let Err(e) = pump.call1((
206		- vs_res.generator.clone_ref(py),
207		- vs_res.schema_json.clone(),
208		- sender,
209		- vs_res.mode.clone(),
210		- )) {
211		- tracing::error!("Failed to pump validated stream: {}", e);
212		- }
213		- return NativeResponse {
214		- status: vs_res.status,
215		- headers: vs_res.headers.clone(),
216		- body: NativeBody::Stream(rx),
217		- };
218		- }
219		- if let Ok(ts_res) = res.extract::<pyo3::PyRef<'_, TokenStreamResponse>>() {
220		- let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
221		- let sender = StreamSender { tx };
222		- let pump = helper.pump_stream.bind(py);
223		- if let Err(e) = pump.call1((ts_res.generator.clone_ref(py), sender)) {
224		- tracing::error!("Failed to pump stream: {}", e);
225		- }
226		- return NativeResponse {
227		- status: ts_res.status,
228		- headers: ts_res.headers.clone(),
229		- body: NativeBody::Stream(rx),
230		- };
231		- }
	260	+ Ok(res) => handle_ok_result(py, res, helper),
	261	+ Err(e) => handle_py_error(py, e),
	262	+ }
	263	+}
232	264
233		- // Streaming responses are serialized by pumping the generator.
234		- // Everything else goes through the Rust-side fast serializer,
235		- // which mirrors Robyn's `extract_response_type_fast` (orjson
236		- // directly from Rust for dict/list/scalars, downcast for
237		- // `Response`, Python `wrap_result` only as a last resort).
238		- serialize_response(py, res, helper)
	265	+/// Convert a Python exception raised by a handler (or its call wrapper) into
	266	+/// the proper `NativeResponse`: HTTPException → its status, RequestValidationError
	267	+/// → 422, everything else → generic 500 (never leak details). Shared by the
	268	+/// sync path and the async resolution path.
	269	+fn handle_py_error(py: Python<'_>, e: pyo3::PyErr) -> NativeResponse {
	270	+ // FastAPI-style exceptions: render as proper JSON error responses
	271	+ // instead of a generic 500.
	272	+ let justapi = py.import("justapi").ok();
	273	+ let val = e.value(py);
	274	+ if let Some(exc_cls) = justapi.as_ref().and_then(\|m\| m.getattr("HTTPException").ok()) {
	275	+ if val.is_instance(&exc_cls).unwrap_or(false) {
	276	+ let status: u16 =
	277	+ val.getattr("status_code").ok().and_then(\|v\| v.extract().ok()).unwrap_or(500);
	278	+ let detail = exception_detail(py, val);
	279	+ let user_headers: Vec<(Vec<u8>, Vec<u8>)> = val
	280	+ .getattr("headers")
	281	+ .ok()
	282	+ .and_then(\|v\| v.extract::<Option<HashMap<String, String>>>().ok())
	283	+ .flatten()
	284	+ .map(\|m\| m.into_iter().map(\|(k, v)\| (k.into_bytes(), v.into_bytes())).collect())
	285	+ .unwrap_or_default();
	286	+ let mut headers = user_headers;
	287	+ headers.push((b"content-type".to_vec(), b"application/json".to_vec()));
	288	+ let body = serde_json::json!({ "detail": detail }).to_string().into_bytes();
	289	+ return NativeResponse { status, headers, body: NativeBody::Bytes(body) };
239	290	}
240		- Err(e) => {
241		- // FastAPI-style exceptions: render as proper JSON error responses
242		- // instead of a generic 500.
243		- let justapi = py.import("justapi").ok();
244		- let val = e.value(py);
245		- if let Some(exc_cls) = justapi.as_ref().and_then(\|m\| m.getattr("HTTPException").ok()) {
246		- if val.is_instance(&exc_cls).unwrap_or(false) {
247		- let status: u16 = val
248		- .getattr("status_code")
249		- .ok()
250		- .and_then(\|v\| v.extract().ok())
251		- .unwrap_or(500);
252		- let detail = exception_detail(py, val);
253		- let user_headers: Vec<(Vec<u8>, Vec<u8>)> = val
254		- .getattr("headers")
255		- .ok()
256		- .and_then(\|v\| v.extract::<Option<HashMap<String, String>>>().ok())
257		- .flatten()
258		- .map(\|m\| {
259		- m.into_iter().map(\|(k, v)\| (k.into_bytes(), v.into_bytes())).collect()
260		- })
261		- .unwrap_or_default();
262		- let mut headers = user_headers;
263		- headers.push((b"content-type".to_vec(), b"application/json".to_vec()));
264		- let body = serde_json::json!({ "detail": detail }).to_string().into_bytes();
265		- return NativeResponse { status, headers, body: NativeBody::Bytes(body) };
266		- }
267		- }
268		- if let Some(exc_cls) =
269		- justapi.as_ref().and_then(\|m\| m.getattr("RequestValidationError").ok())
270		- {
271		- if val.is_instance(&exc_cls).unwrap_or(false) {
272		- let detail = exception_detail(py, val);
273		- let headers = vec![(b"content-type".to_vec(), b"application/json".to_vec())];
274		- let body = serde_json::json!({ "detail": detail }).to_string().into_bytes();
275		- return NativeResponse { status: 422, headers, body: NativeBody::Bytes(body) };
276		- }
277		- }
278		- // Exception surfaced from the Python handler (or the call wrapper).
279		- // Log the detail server-side only; never leak it to the client.
280		- tracing::error!("Native handler error: {}", e);
281		- NativeResponse {
282		- status: 500,
283		- headers: vec![(b"content-type".to_vec(), b"application/json".to_vec())],
284		- body: NativeBody::Bytes(b"{\"detail\":\"Internal Server Error\"}".to_vec()),
285		- }
	291	+ }
	292	+ if let Some(exc_cls) = justapi.as_ref().and_then(\|m\| m.getattr("RequestValidationError").ok()) {
	293	+ if val.is_instance(&exc_cls).unwrap_or(false) {
	294	+ let detail = exception_detail(py, val);
	295	+ let headers = vec![(b"content-type".to_vec(), b"application/json".to_vec())];
	296	+ let body = serde_json::json!({ "detail": detail }).to_string().into_bytes();
	297	+ return NativeResponse { status: 422, headers, body: NativeBody::Bytes(body) };
286	298	}
287	299	}
	300	+ // Exception surfaced from the Python handler (or the call wrapper).
	301	+ // Log the detail server-side only; never leak it to the client.
	302	+ tracing::error!("Native handler error: {}", e);
	303	+ NativeResponse {
	304	+ status: 500,
	305	+ headers: vec![(b"content-type".to_vec(), b"application/json".to_vec())],
	306	+ body: NativeBody::Bytes(b"{\"detail\":\"Internal Server Error\"}".to_vec()),
	307	+ }
288	308	}
289	309
290	310	/// Whether distributed-tracing context is injected into Python contextvars on
@@ -492,8 +512,50 @@ pub(crate) fn serialize_response(
492	512	}
493	513	}
494	514
	515	+/// Resolve a `NativeBody::Async` future (from an async Python handler) into a
	516	+/// concrete body, WITHOUT holding the GIL-pool worker.
	517	+///
	518	+/// The coroutine runs on the dedicated asyncio loop thread; `await` inside it
	519	+/// releases the GIL, so many coroutines can interleave. We just wait for the
	520	+/// `concurrent.futures.Future` (its internal wait releases the GIL), then
	521	+/// serialize the resolved value with the same fast serializer the sync path
	522	+/// uses. Runs on a `spawn_blocking` thread so no worker thread is tied up.
	523	+async fn resolve_async_response(py_future: Py<PyAny>) -> Result<NativeResponse, anyhow::Error> {
	524	+ // Block on the future's .result(). If the coroutine raised (HTTPException,
	525	+ // RequestValidationError, ...), the exception surfaces as a PyErr from
	526	+ // .result() — convert it with the same logic as the sync Err path.
	527	+ let outcome: Result<Py<PyAny>, pyo3::PyErr> = tokio::task::spawn_blocking(move \|\| {
	528	+ Python::attach(\|py\| py_future.bind(py).call_method0("result").map(\|v\| v.unbind()))
	529	+ })
	530	+ .await
	531	+ .map_err(\|e\| anyhow::anyhow!("async handler join error: {}", e))?;
	532	+
	533	+ let nr = Python::attach(\|py\| match outcome {
	534	+ Ok(res) => {
	535	+ let helper = get_helper(py);
	536	+ handle_ok_result(py, res.bind(py).into(), helper)
	537	+ }
	538	+ Err(e) => handle_py_error(py, e),
	539	+ });
	540	+ Ok(nr)
	541	+}
	542	+
495	543	pub(crate) fn nr_to_response(nr: NativeResponse) -> hyper::Response<ResponseBody> {
496	544	let mut resp = match nr.body {
	545	+ // Async futures are always resolved before nr_to_response is called
	546	+ // (see resolve_async_response); this arm is unreachable but required
	547	+ // for exhaustiveness.
	548	+ NativeBody::Async(_) => {
	549	+ return hyper::Response::builder()
	550	+ .status(StatusCode::INTERNAL_SERVER_ERROR)
	551	+ .body(ResponseBody::new(
	552	+ http_body_util::Full::new(Bytes::from_static(
	553	+ b"{\"detail\":\"async handler not resolved\"}",
	554	+ ))
	555	+ .map_err(\|e: std::convert::Infallible\| -> anyhow::Error { match e {} }),
	556	+ ))
	557	+ .expect("valid error response");
	558	+ }
497	559	NativeBody::Bytes(b) => {
498	560	let mut r = Response::new(ResponseBody::new(
499	561	http_body_util::Full::new(Bytes::from(b))
@@ -987,6 +1049,14 @@ where
987	1049	.await
988	1050	.map_err(\|e\| anyhow::anyhow!("GIL pool error: {}", e))?;
989	1051
	1052	+ // Async handlers: the GIL worker returned the future without
	1053	+ // blocking. Await it here (spawn_blocking, GIL released while
	1054	+ // waiting) so concurrent async requests interleave.
	1055	+ let nr = match nr.body {
	1056	+ NativeBody::Async(fut) => resolve_async_response(fut).await?,
	1057	+ _ => nr,
	1058	+ };
	1059	+
990	1060	Ok(nr_to_response(nr))
991	1061	})
992	1062	})
@@ -1231,6 +1301,13 @@ where
1231	1301	.await
1232	1302	.map_err(\|e\| anyhow::anyhow!("GIL pool error: {}", e))?;
1233	1303
	1304	+ // Async handlers: await the future here (GIL released while
	1305	+ // waiting), mirroring make_native_handler.
	1306	+ let nr = match nr.body {
	1307	+ NativeBody::Async(fut) => resolve_async_response(fut).await?,
	1308	+ _ => nr,
	1309	+ };
	1310	+
1234	1311	Ok(nr_to_response(nr))
1235	1312	})
1236	1313	})
1237	1314

▼crates/justapi-py/src/native/types.rs

+7-0

@@ -8,6 +8,13 @@ use super::NATIVE_HELPER;
8	8	pub enum NativeBody {
9	9	Bytes(Vec<u8>),
10	10	Stream(tokio::sync::mpsc::UnboundedReceiver<Result<Bytes, anyhow::Error>>),
	11	+ /// An asyncio `concurrent.futures.Future` produced by an async Python
	12	+ /// handler (via `run_coroutine_threadsafe`). The coroutine runs on the
	13	+ /// dedicated asyncio loop thread and releases the GIL during `await`;
	14	+ /// the GIL-pool worker must NOT block on `.result()` here or every async
	15	+ /// request would serialize through the single worker. Instead the caller
	16	+ /// awaits the future on a `spawn_blocking` thread.
	17	+ Async(Py<PyAny>),
11	18	}
12	19
13	20	pub struct NativeResponse {
14	21

▼scripts/publish.sh

+7-1

@@ -28,13 +28,19 @@ PLATFORMS=(
28	28	"aarch64-unknown-linux-musl"
29	29	)
30	30
31		-echo "[publish] building wheels for: ${PLATFORMS[*]}"
	31	+echo "[publish] building wheels for: ${PLATFORMS[*]} + free-threaded 3.14t"
32	32	mkdir -p "$WHEELS_DIR"
33	33	for target in "${PLATFORMS[@]}"; do
34	34	echo "[publish] building $target..."
35	35	(cd "$ROOT/crates/justapi-py" && maturin build --release --target "$target" --zig --out "$WHEELS_DIR")
36	36	done
37	37
	38	+# Free-threaded CPython 3.14t: no abi3 (limited API can't be free-threaded).
	39	+echo "[publish] building free-threaded x86_64 (3.14t, no abi3)..."
	40	+(cd "$ROOT/crates/justapi-py" && maturin build --release --target x86_64-unknown-linux-gnu \
	41	+ --interpreter python3.14t --no-default-features --features mail,http3,orjson \
	42	+ --zig --out "$WHEELS_DIR") \|\| echo "[publish] WARN: free-threaded wheel skipped (needs python3.14t)"
	43	+
38	44	echo "[publish] wheels:"
39	45	ls -la "$WHEELS_DIR"/*.whl
40	46
41	47

🚀 release: 2.0.8 final — multi-thread server (ADR-085), free-threaded support (ADR-084), async handler fix (ADR-083)