swadhin / eu-air-traffic public

swadhinbiswas

[email protected]
2026-09-14 13:03:10 +0600 committed NaN years ago
76836cb
commit 76836cb
5 files changed
+148 -0
Showing 5 changed files+148-0
dbt/macros/generate_schema_name.sqldeploy.shdocker-compose.ymldocker/Caddyfiledocker/Dockerfile.collector
▼dbt/macros/generate_schema_name.sql
+1-0
@@ -13,3 +13,4 @@
13
13
{{ custom_schema_name | trim }}
14
14
{%- endif -%}
15
15
{%- endmacro %}
16
+
16
17
▼deploy.sh
+59-0
@@ -0,0 +1,59 @@
1
+#!/usr/bin/env bash
2
+# Deploy the full platform with Docker Compose.
3
+#
4
+# ./deploy.sh build and start collector + lake (+ TLS with --tls)
5
+# ./deploy.sh --logs follow logs
6
+# ./deploy.sh --down stop and remove containers
7
+# ./deploy.sh --ps status
8
+set -euo pipefail
9
+
10
+cd "$(dirname "$0")"
11
+COMPOSE=(docker compose)
12
+PROFILES=()
13
+
14
+case "${1:-}" in
15
+ --logs) exec "${COMPOSE[@]}" logs -f --tail=100 ;;
16
+ --down) exec "${COMPOSE[@]}" down ;;
17
+ --ps|--status) exec "${COMPOSE[@]}" ps ;;
18
+ --tls) PROFILES=(--profile tls) ;;
19
+ --help|-h)
20
+ sed -n '2,8p' "$0"
21
+ exit 0
22
+ ;;
23
+ "") ;;
24
+ *) echo "unknown option: $1 (try --help)" >&2; exit 2 ;;
25
+esac
26
+
27
+command -v docker >/dev/null || { echo "docker is required" >&2; exit 1; }
28
+docker info >/dev/null 2>&1 || { echo "docker daemon is not running" >&2; exit 1; }
29
+
30
+if [ ! -f .env ]; then
31
+ echo "creating .env from .env.example — fill in your credentials, then re-run" >&2
32
+ cp .env.example .env
33
+ exit 1
34
+fi
35
+
36
+if [ ! -f secrets/aiven-ca.pem ]; then
37
+ echo "warning: secrets/aiven-ca.pem is missing; mount it or set KAFKA_SECURITY_PROTOCOL=PLAINTEXT" >&2
38
+fi
39
+
40
+echo "building images…"
41
+"${COMPOSE[@]}" "${PROFILES[@]}" build
42
+
43
+echo "starting containers…"
44
+"${COMPOSE[@]}" "${PROFILES[@]}" up -d
45
+
46
+echo "waiting for the live API…"
47
+for _ in $(seq 1 30); do
48
+ if curl -fsS http://127.0.0.1:8090/health >/dev/null 2>&1; then
49
+ echo
50
+ echo "collector is healthy: http://127.0.0.1:8090/health"
51
+ [ ${#PROFILES[@]} -gt 0 ] && echo "TLS front: https://${SITE_ADDRESS:-localhost}"
52
+ echo "lake loop runs every ${LAKE_INTERVAL_SECONDS:-900}s; ./deploy.sh --logs to watch it"
53
+ exit 0
54
+ fi
55
+ sleep 2
56
+done
57
+
58
+echo "collector did not become healthy in 60s — check ./deploy.sh --logs" >&2
59
+exit 1
0
60
▼docker-compose.yml
+64-0
@@ -0,0 +1,64 @@
1
+# Full platform in containers: collector + live API, the lake cycle on a timer,
2
+# and an optional Caddy TLS front. Point .env at real services (Aiven Kafka,
3
+# Hugging Face, MotherDuck, Turso) — nothing here needs a local Kafka.
4
+name: eu-air-traffic
5
+
6
+services:
7
+ collector:
8
+ build:
9
+ context: .
10
+ dockerfile: docker/Dockerfile.collector
11
+ env_file: .env
12
+ environment:
13
+ LIVE_API_HOST: 0.0.0.0
14
+ volumes:
15
+ - ./warehouse:/app/warehouse
16
+ - ./secrets:/app/secrets:ro
17
+ ports:
18
+ - "8090:8090"
19
+ restart: unless-stopped
20
+
21
+ lake:
22
+ build:
23
+ context: .
24
+ dockerfile: docker/lake-job.Dockerfile
25
+ env_file: .env
26
+ environment:
27
+ # One cycle per interval; the runner keeps its own DuckDB state on disk.
28
+ LAKE_INTERVAL_SECONDS: ${LAKE_INTERVAL_SECONDS:-900}
29
+ HF_SYNC: ${HF_SYNC:-1}
30
+ AIR_TRAFFIC_DUCKDB_PATH: /app/warehouse/air_traffic.duckdb
31
+ entrypoint:
32
+ - bash
33
+ - -c
34
+ - |
35
+ while true; do
36
+ echo "[lake] cycle starting $(date -u +%FT%TZ)"
37
+ bash scripts/run_lake.sh || echo "[lake] cycle failed — retrying next interval"
38
+ sleep "$$LAKE_INTERVAL_SECONDS"
39
+ done
40
+ volumes:
41
+ - ./warehouse:/app/warehouse
42
+ - ./secrets:/app/secrets:ro
43
+ depends_on:
44
+ collector:
45
+ condition: service_healthy
46
+ restart: unless-stopped
47
+
48
+ caddy:
49
+ image: caddy:2-alpine
50
+ profiles: ["tls"]
51
+ environment:
52
+ SITE_ADDRESS: ${SITE_ADDRESS:-localhost}
53
+ volumes:
54
+ - ./docker/Caddyfile:/etc/caddy/Caddyfile:ro
55
+ - caddy_data:/data
56
+ ports:
57
+ - "80:80"
58
+ - "443:443"
59
+ depends_on:
60
+ - collector
61
+ restart: unless-stopped
62
+
63
+volumes:
64
+ caddy_data:
0
65
▼docker/Caddyfile
+5-0
@@ -0,0 +1,5 @@
1
+# TLS termination for the live API, addressed by SITE_ADDRESS.
2
+{$SITE_ADDRESS:localhost} {
3
+ encode gzip
4
+ reverse_proxy collector:8090
5
+}
0
6
▼docker/Dockerfile.collector
+19-0
@@ -0,0 +1,19 @@
1
+# VPS collector + live API: positions, flights, weather, fuel, reference, AirLabs.
2
+FROM python:3.12-slim
3
+
4
+RUN apt-get update \
5
+ && apt-get install -y --no-install-recommends ca-certificates curl \
6
+ && rm -rf /var/lib/apt/lists/*
7
+RUN curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR=/usr/local/bin sh
8
+
9
+WORKDIR /app
10
+COPY pyproject.toml uv.lock ./
11
+RUN uv sync --extra stream --no-install-project
12
+COPY . .
13
+RUN uv sync --extra stream
14
+
15
+EXPOSE 8090
16
+HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \
17
+ CMD curl -fsS http://127.0.0.1:8090/health || exit 1
18
+
19
+CMD ["uv", "run", "python", "-m", "services.collector"]
0
20