swadhinbiswas Wire Hugging Face dataset docs, card generator and repo defaults · 9/16/2026
556d677
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
# ═══════════════════════════════════════════════════════════════════════════
# Air Traffic Platform — Environment Configuration
# Copy to `.env` and fill in values. NEVER commit `.env`.
#
# Three planes:
# VPS collector → Kafka + live API (AIVEN_KAFKA_*, LIVE_API_*)
# Hugging Face → Bronze/Silver lake (HF_TOKEN, HF_REPO)
# MotherDuck → Gold warehouse (dbt) (MOTHERDUCK_TOKEN)
# Turso → serving copy for the site (TURSO_*)
# ═══════════════════════════════════════════════════════════════════════════
# ── Runtime ──────────────────────────────────────────────────────────────────
ENVIRONMENT=development # development | test | production
LOG_LEVEL=INFO # DEBUG | INFO | WARNING | ERROR
MOCK_MODE=false # true => synthetic data when API keys missing
# ── External API credentials (optional in dev, required in production) ─────
AVIATIONSTACK_API_KEY= # fuel prices
OPENWEATHER_API_KEY=
AIRPORTDB_API_TOKEN=
OPENSKY_USERNAME= # legacy; OAuth2 client below is preferred
OPENSKY_PASSWORD=
OPENSKY_CLIENT_ID= # OAuth2 client credentials (required now)
OPENSKY_CLIENT_SECRET=
AIRLABS_API_KEY= # schedules/delays; free key = 1000 calls/month
# ── Live API served by the VPS collector ─────────────────────────────────────
LIVE_API_HOST=0.0.0.0
LIVE_API_PORT=8090
# Public base URL the dashboard is built against (Cloudflare Tunnel / Caddy / DNS):
LIVE_API_PUBLIC_URL=
LIVE_API_TOKEN= # optional bearer for non-read endpoints
LIVE_SNAPSHOT_MAX_AGE_SECONDS=120
# ── Collector cadence (seconds) ──────────────────────────────────────────────
# Live API (dashboard) updates every tick; the lake only needs a coarser feed.
POSITIONS_INTERVAL_SECONDS=15
# How often positions are published to Kafka / the lake. 15s would be ~17M
# rows/day; 300s is the sane default while the live map stays 15s fresh.
POSITIONS_PUBLISH_INTERVAL_SECONDS=300
FLIGHTS_INTERVAL_SECONDS=300
METAR_INTERVAL_SECONDS=300
TAF_INTERVAL_SECONDS=900
FORECAST_INTERVAL_SECONDS=3600
FUEL_INTERVAL_SECONDS=86400
REFERENCE_INTERVAL_SECONDS=86400
# Concurrent upstream fetches (keep modest on a 2-core VPS).
ADSB_MAX_WORKERS=4
FLIGHTS_MAX_WORKERS=4
FLIGHTS_AIRPORTS=12
KAFKA_FLUSH_TIMEOUT_SECONDS=15
# ── Kafka event bus (Aiven) ──────────────────────────────────────────────────
AIVEN_KAFKA_HOST=
AIVEN_KAFKA_PORT=12345
AIVEN_KAFKA_USERNAME=
AIVEN_KAFKA_PASSWORD=
AIVEN_KAFKA_CA_CERT= # path to the Aiven CA .pem (optional)
KAFKA_SECURITY_PROTOCOL=SASL_SSL # SASL_SSL (Aiven) | PLAINTEXT (local Redpanda)
KAFKA_SASL_MECHANISM=PLAIN
# One topic per data product (defaults shown):
# KAFKA_TOPIC_POSITIONS=eu-positions
# KAFKA_TOPIC_FLIGHTS=eu-flights
# KAFKA_TOPIC_WEATHER=eu-weather
# KAFKA_TOPIC_FUEL=eu-fuel
# KAFKA_TOPIC_REFERENCE=eu-reference
# ── Kafka sink (Kafka → Bronze Parquet → Hugging Face) ───────────────────────
SINK_CONSUMER_GROUP=eu-air-traffic-sink
SINK_BATCH_SIZE=5000
SINK_POLL_TIMEOUT_MS=10000
# GitHub Actions drains a ~9-minute window each run. Kafka topic retention MUST
# exceed this (the pipeline sets it at startup; verify in Aiven).
LAKE_WINDOW_SECONDS=540
KAFKA_RETENTION_HOURS=24
# ── Hugging Face Hub (Bronze/Silver data lake) ───────────────────────────────
HF_TOKEN=
HF_REPO=swadhinbiswas/air-traffic
HF_BRONZE_PREFIX=bronze
HF_SILVER_PREFIX=silver
HF_GOLD_PREFIX=gold
HF_PRIVATE=true
# ── MotherDuck (Gold serving layer / dashboard analytics) ────────────────────
MOTHERDUCK_TOKEN=
MOTHERDUCK_DATABASE=air_traffic
# Postgres wire endpoint (BI tools / psql); optional, same token as password.
# MOTHERDUCK_PG_URL=postgresql://postgres:<token>@pg.eu-central-1-aws.motherduck.com:5432/<db>?sslmode=verify-full&sslrootcert=system
# Where the pipeline builds: local (reproducible, then publish) | motherduck
WAREHOUSE_TARGET=local
# dbt target: dev (local DuckDB) | motherduck
DBT_TARGET=dev
# ── Turso (libSQL) serving layer read by the website ─────────────────────────
# The GitHub Action copies Gold into Turso; the site queries it with a
# READ-ONLY token (VITE_TURSO_TOKEN), which is safe to expose in the bundle.
TURSO_DATABASE_URL=
TURSO_AUTH_TOKEN=
VITE_TURSO_TOKEN=
VITE_TURSO_URL=
# ── HTTP / rate limiting ────────────────────────────────────────────────────
REQUEST_TIMEOUT_SECONDS=15
MAX_RETRIES=3
RETRY_BACKOFF_BASE=2.0
RATE_LIMIT_DELAY_SECONDS=0.25
OPENWEATHER_LIMIT=60
# ── Pipeline ─────────────────────────────────────────────────────────────────
PIPELINE_BATCH_SIZE=50000
DELAY_THRESHOLD_MINUTES=15
# ── Dashboard build (Vite, build-time) ───────────────────────────────────────
VITE_LIVE_URL= # e.g. https://live.example.com (the VPS live API)
VITE_API_URL=http://localhost:8000
# ── Storage paths (defaults resolve under ./warehouse) ─────────────────────
# DUCKDB_PATH=warehouse/air_traffic.duckdb
# AIR_TRAFFIC_DUCKDB_PATH=warehouse/air_traffic.duckdb # dbt only