swadhin / justapi public
/ justapi / crates / justapi-inference / Cargo.toml
swadhinbiswas feat(inference): add ML inference engine with scheduling and autoscaling · 7/9/2026
d743f45
32 lines · 940 Bytes
Raw Download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
[package]
name = "justapi-inference"
version.workspace = true
edition.workspace = true
license.workspace = true

[dependencies]
tokio = { workspace = true }
anyhow.workspace = true
thiserror.workspace = true
tracing.workspace = true
serde = { workspace = true }
serde_json = { workspace = true }
futures = "0.3"
bytes = "1.12.0"
async-stream = "0.3"
rand = "0.8"
candle-core = "0.11.0"
candle-nn = "0.11.0"
candle-transformers = { version = "0.11.0", optional = true }
tokenizers = { version = "0.22", optional = true }

[features]
default = []
# GPU backends (feature-gated; require the matching toolkit to COMPILE).
# CPU is the default backend and always available.
cuda = ["candle-core/cuda", "candle-nn/cuda"]
# Real weight loading + tokenizer (GGUF/safetensors). Heavy: pulls in
# candle-transformers + tokenizers. NOT in default features so the lean
# build/test path stays fast.
real = ["dep:candle-transformers", "dep:tokenizers"]