8a8becd
commit 8a8becd
2 files changed
+46 -0
Showing 2 changed files+46-0
▼docker/lake-job.Dockerfile
+16-0
| @@ -0,0 +1,16 @@ | ||
1 | +# Lake pipeline image: built by CI, executed by Hugging Face Jobs. | |
2 | +# Nothing here runs on the collector VPS. | |
3 | +FROM python:3.12-slim | |
4 | + | |
5 | +RUN apt-get update \ | |
6 | + && apt-get install -y --no-install-recommends ca-certificates git curl \ | |
7 | + && rm -rf /var/lib/apt/lists/* | |
8 | +RUN curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR=/usr/local/bin sh | |
9 | + | |
10 | +WORKDIR /app | |
11 | +COPY pyproject.toml uv.lock ./ | |
12 | +RUN uv sync --extra dev --extra stream --extra turso --no-install-project | |
13 | +COPY . . | |
14 | +RUN uv sync --extra dev --extra stream --extra turso | |
15 | + | |
16 | +ENTRYPOINT ["bash", "scripts/run_lake.sh"] | |
0 | 17 | |
▼scripts/run_lake.sh
+30-0
| @@ -0,0 +1,30 @@ | ||
1 | +#!/usr/bin/env bash | |
2 | +# One lake cycle: the same sequence .github/workflows/lake.yml runs, packaged | |
3 | +# for a Hugging Face Job (or any container). When the dataset is mounted | |
4 | +# locally, set HF_SYNC=0 and the HF pull/push round-trip is skipped entirely. | |
5 | +set -euo pipefail | |
6 | + | |
7 | +export WAREHOUSE_TARGET="${WAREHOUSE_TARGET:-local}" | |
8 | +export DBT_TARGET="${DBT_TARGET:-dev}" | |
9 | +export AIR_TRAFFIC_DUCKDB_PATH="${AIR_TRAFFIC_DUCKDB_PATH:-$PWD/warehouse/air_traffic.duckdb}" | |
10 | +export LAKE_WINDOW_SECONDS="${LAKE_WINDOW_SECONDS:-420}" | |
11 | +HF_SYNC="${HF_SYNC:-1}" | |
12 | + | |
13 | +if [ "$HF_SYNC" = "1" ]; then | |
14 | + uv run python -m scripts.lake_sync pull-silver | |
15 | +fi | |
16 | + | |
17 | +uv run python -m services.sink --window-seconds "$LAKE_WINDOW_SECONDS" | |
18 | +uv run python -m pipelines.silver | |
19 | + | |
20 | +if [ "$HF_SYNC" = "1" ]; then | |
21 | + uv run python -m scripts.lake_sync push-silver | |
22 | +fi | |
23 | + | |
24 | +uv run python -m pipelines.warehouse | |
25 | +uv run dbt build --project-dir dbt --profiles-dir dbt | |
26 | +uv run python -m pipelines.warehouse --register-gold | |
27 | +uv run python -m scripts.publish_motherduck | |
28 | +uv run python -m pipelines.quality | |
29 | +uv run python -m scripts.publish_site_tables | |
30 | +uv run python -m scripts.publish_turso | |
0 | 31 | |