import asyncio
import time
from justapi.app import JustAPIApp, adaptive_batch
import urllib.request
import threading
app = JustAPIApp()
@app.post("/predict")
@adaptive_batch(max_size=3, window_ms=50)
def predict(requests):
# Log the size of the batch
print(f"Received batch of size {len(requests)}")
# Simulate ML model computation
time.sleep(0.1)
results = []
for req in requests:
body = req.get("body", b"").decode("utf-8")
results.append({"status": 200, "body": f"Processed {body}"})
return results
def test_batching():
def run_server():
try:
app.run("127.0.0.1:8081")
except Exception:
pass
server_thread = threading.Thread(target=run_server, daemon=True)
server_thread.start()
time.sleep(1)
def make_request(idx):
req = urllib.request.Request("http://127.0.0.1:8081/predict", data=f"req-{idx}".encode("utf-8"), method="POST")
try:
with urllib.request.urlopen(req) as response:
return response.read().decode("utf-8")
except Exception as e:
return str(e)
threads = []
for i in range(5):
t = threading.Thread(target=lambda i=i: print(f"Response {i}: {make_request(i)}"))
threads.append(t)
t.start()
for t in threads:
t.join()
print("Done")