docs(ci): add CircleCI Cloud worker guide with external PostgreSQL/Redis setup
Swadhin Biswas committed 5 months ago
5b7dbc4
1 parent f58315a
commit 5b7dbc4
3 files changed
+1059 -91
Showing 3 changed files+1059-91
▼.circleci/config.yml
+200-91
| @@ -1,50 +1,58 @@ | ||
1 | 1 | version: 2.1 |
2 | 2 | |
3 | 3 | # ────────────────────────────────────────────────────────────── |
4 | -# OpenCodeHub Worker - CircleCI Configuration | |
5 | -# Run background workers (merge queue, mirror sync, cleanup) | |
6 | -# using CircleCI infrastructure | |
4 | +# OpenCodeHub Worker — CircleCI Cloud Configuration | |
5 | +# For managed CircleCI Cloud (NOT self-hosted runners) | |
6 | +# Uses external PostgreSQL + Redis (required for Cloud) | |
7 | 7 | # ────────────────────────────────────────────────────────────── |
8 | 8 | |
9 | +orbs: | |
10 | + # Optional: for better Docker layer caching | |
11 | + docker: circleci/[email protected] | |
12 | + | |
9 | 13 | executors: |
10 | - bun-executor: | |
14 | + # Docker executor with Bun + Node.js | |
15 | + bun-medium: | |
11 | 16 | docker: |
12 | 17 | - image: oven/bun:1 |
13 | 18 | resource_class: medium |
14 | 19 | |
15 | - # Self-hosted runner executor (recommended for production) | |
16 | - # Replace 'your-namespace' with your actual CircleCI namespace | |
17 | - och-runner: | |
18 | - machine: | |
19 | - image: ubuntu-2204:current | |
20 | - resource_class: your-namespace/opencodehub-runner | |
20 | + bun-large: | |
21 | + docker: | |
22 | + - image: oven/bun:1 | |
23 | + resource_class: large | |
21 | 24 | |
22 | -# ── Jobs ───────────────────────────────────────────────────── | |
25 | + # With PostgreSQL service container (for testing, NOT production data) | |
26 | + bun-with-postgres: | |
27 | + docker: | |
28 | + - image: oven/bun:1 | |
29 | + - image: cimg/postgres:16.2 | |
30 | + environment: | |
31 | + POSTGRES_USER: opencodehub | |
32 | + POSTGRES_PASSWORD: opencodehub | |
33 | + POSTGRES_DB: opencodehub | |
34 | + resource_class: medium | |
23 | 35 | |
24 | -jobs: | |
25 | - # ────────────────────────────────────────────── | |
26 | - # Continuous Background Worker (Self-Hosted Runner) | |
27 | - # ────────────────────────────────────────────── | |
28 | - worker-continuous: | |
29 | - executor: och-runner | |
30 | - environment: | |
31 | - NODE_ENV: production | |
32 | - WORKER_INTERVAL: 5000 | |
33 | - WORKER_HEALTH_PORT: 9090 | |
34 | - WORKER_STALE_TIMEOUT: 300000 | |
35 | - steps: | |
36 | - - checkout | |
36 | +# ── Reusable Commands ───────────────────────────────────────── | |
37 | 37 | |
38 | +commands: | |
39 | + setup-bun: | |
40 | + description: "Ensure Bun is installed and available" | |
41 | + steps: | |
38 | 42 | - run: |
39 | - name: Install Bun | |
43 | + name: Verify Bun | |
40 | 44 | command: | |
41 | 45 | if ! command -v bun &> /dev/null; then |
46 | + echo "Installing Bun..." | |
42 | 47 | curl -fsSL https://bun.sh/install | bash |
43 | 48 | echo 'export PATH="$HOME/.bun/bin:$PATH"' >> $BASH_ENV |
44 | 49 | fi |
45 | 50 | source $BASH_ENV |
46 | 51 | bun --version |
47 | 52 | |
53 | + install-deps: | |
54 | + description: "Install dependencies with retry" | |
55 | + steps: | |
48 | 56 | - run: |
49 | 57 | name: Install dependencies |
50 | 58 | command: | |
| @@ -56,167 +64,268 @@ jobs: | ||
56 | 64 | echo "::error::bun install failed after 3 attempts" |
57 | 65 | exit 1 |
58 | 66 | |
67 | + check-env: | |
68 | + description: "Verify required environment variables" | |
69 | + steps: | |
70 | + - run: | |
71 | + name: Check environment | |
72 | + command: | | |
73 | + : "${DATABASE_URL:?DATABASE_URL is required}" | |
74 | + : "${REDIS_URL:?REDIS_URL is required}" | |
75 | + : "${JWT_SECRET:?JWT_SECRET is required}" | |
76 | + : "${SESSION_SECRET:?SESSION_SECRET is required}" | |
77 | + echo "✅ All required environment variables are set" | |
78 | + | |
79 | +# ── Jobs ───────────────────────────────────────────────────── | |
80 | + | |
81 | +jobs: | |
82 | + # ────────────────────────────────────────────── | |
83 | + # Continuous Worker (up to 5 hours) | |
84 | + # ────────────────────────────────────────────── | |
85 | + # NOTE: CircleCI Cloud free plan has 5h job limit. | |
86 | + # Performance/Scale plans have longer limits. | |
87 | + # For truly continuous operation, use scheduled pipelines | |
88 | + # to restart every 4.5 hours, OR use self-hosted runner. | |
89 | + # ────────────────────────────────────────────── | |
90 | + worker: | |
91 | + executor: bun-medium | |
92 | + environment: | |
93 | + NODE_ENV: production | |
94 | + # Worker loop interval (ms) | |
95 | + WORKER_INTERVAL: 5000 | |
96 | + # Health check port (not exposed externally on Cloud) | |
97 | + WORKER_HEALTH_PORT: 9090 | |
98 | + # Stale job timeout (ms) | |
99 | + WORKER_STALE_TIMEOUT: 300000 | |
100 | + steps: | |
101 | + - checkout | |
102 | + - setup-bun | |
103 | + - install-deps | |
104 | + - check-env | |
105 | + | |
59 | 106 | - run: |
60 | 107 | name: Start OpenCodeHub Worker |
61 | 108 | command: | |
62 | - echo "Starting OpenCodeHub background worker..." | |
109 | + echo "Starting OpenCodeHub worker..." | |
110 | + echo "Database: ${DATABASE_URL%%@*}@..." | |
111 | + echo "Redis: ${REDIS_URL%%@*}@..." | |
112 | + | |
113 | + # Run worker | |
63 | 114 | bun run scripts/worker.ts & |
64 | 115 | WORKER_PID=$! |
65 | - echo "Worker PID: $WORKER_PID" | |
66 | 116 | |
67 | - # Wait for health check | |
68 | - for i in $(seq 1 30); do | |
69 | - if curl -sf http://localhost:9090/health; then | |
70 | - echo "✅ Worker is healthy" | |
71 | - break | |
72 | - fi | |
73 | - echo "Waiting for worker to start... ($i/30)" | |
74 | - sleep 2 | |
75 | - done | |
117 | + # Wait for it to start | |
118 | + sleep 5 | |
119 | + | |
120 | + # Verify it's running | |
121 | + if kill -0 $WORKER_PID 2>/dev/null; then | |
122 | + echo "✅ Worker started (PID: $WORKER_PID)" | |
123 | + else | |
124 | + echo "❌ Worker failed to start" | |
125 | + exit 1 | |
126 | + fi | |
76 | 127 | |
77 | - # Keep alive | |
128 | + # Keep job alive until timeout | |
129 | + # CircleCI Cloud will kill the job after the timeout | |
78 | 130 | wait $WORKER_PID |
79 | - no_output_timeout: 24h | |
131 | + # CircleCI Cloud timeouts: | |
132 | + # Free plan: 1 hour (default), max 5 hours | |
133 | + # Performance: up to 5 hours | |
134 | + # Scale: up to 5 hours | |
135 | + # Server: configurable | |
136 | + no_output_timeout: 5h | |
80 | 137 | |
81 | 138 | # ────────────────────────────────────────────── |
82 | - # Merge Queue Worker (Scheduled / Docker) | |
139 | + # Merge Queue Processor (short burst) | |
83 | 140 | # ────────────────────────────────────────────── |
84 | 141 | merge-queue: |
85 | - executor: bun-executor | |
142 | + executor: bun-medium | |
86 | 143 | environment: |
87 | 144 | NODE_ENV: production |
88 | 145 | WORKER_INTERVAL: 5000 |
89 | 146 | WORKER_STALE_TIMEOUT: 300000 |
90 | 147 | steps: |
91 | 148 | - checkout |
92 | - | |
93 | - - run: | |
94 | - name: Install dependencies | |
95 | - command: | | |
96 | - for i in 1 2 3; do | |
97 | - bun install --frozen-lockfile && exit 0 | |
98 | - echo "bun install attempt $i failed, retrying in 10s..." | |
99 | - sleep 10 | |
100 | - done | |
101 | - echo "::error::bun install failed after 3 attempts" | |
102 | - exit 1 | |
149 | + - setup-bun | |
150 | + - install-deps | |
151 | + - check-env | |
103 | 152 | |
104 | 153 | - run: |
105 | 154 | name: Process Merge Queue |
106 | 155 | command: | |
107 | - echo "Running merge queue processor for 4 hours..." | |
108 | - timeout 14400 bun run scripts/worker.ts || true | |
109 | - echo "Merge queue cycle complete." | |
110 | - no_output_timeout: 4h | |
156 | + echo "Processing merge queue..." | |
157 | + # Run worker for 30 minutes to clear the queue | |
158 | + timeout 1800 bun run scripts/worker.ts || true | |
159 | + echo "Merge queue processing complete." | |
160 | + no_output_timeout: 30m | |
111 | 161 | |
112 | 162 | # ────────────────────────────────────────────── |
113 | - # Mirror Sync (Scheduled) | |
163 | + # Mirror Sync | |
114 | 164 | # ────────────────────────────────────────────── |
115 | 165 | mirror-sync: |
116 | - executor: bun-executor | |
166 | + executor: bun-medium | |
117 | 167 | environment: |
118 | 168 | NODE_ENV: production |
119 | 169 | steps: |
120 | 170 | - checkout |
121 | - | |
122 | - - run: | |
123 | - name: Install dependencies | |
124 | - command: bun install --frozen-lockfile | |
171 | + - setup-bun | |
172 | + - install-deps | |
173 | + - check-env | |
125 | 174 | |
126 | 175 | - run: |
127 | 176 | name: Sync Mirrors |
128 | - command: bun run scripts/worker.ts --task=mirror-sync || true | |
177 | + command: | | |
178 | + echo "Starting mirror sync..." | |
179 | + # Run for up to 1 hour | |
180 | + timeout 3600 bun run scripts/worker.ts --task=mirror-sync || true | |
181 | + echo "Mirror sync complete." | |
182 | + no_output_timeout: 1h | |
129 | 183 | |
130 | 184 | # ────────────────────────────────────────────── |
131 | - # Cleanup (Scheduled) | |
185 | + # Cleanup Tasks | |
132 | 186 | # ────────────────────────────────────────────── |
133 | 187 | cleanup: |
134 | - executor: bun-executor | |
188 | + executor: bun-medium | |
135 | 189 | environment: |
136 | 190 | NODE_ENV: production |
137 | 191 | steps: |
138 | 192 | - checkout |
193 | + - setup-bun | |
194 | + - install-deps | |
195 | + - check-env | |
139 | 196 | |
140 | 197 | - run: |
141 | - name: Install dependencies | |
142 | - command: bun install --frozen-lockfile | |
143 | - | |
144 | - - run: | |
145 | - name: Run Cleanup Tasks | |
146 | - command: bun run scripts/worker.ts --task=cleanup || true | |
198 | + name: Run Cleanup | |
199 | + command: | | |
200 | + echo "Running cleanup tasks..." | |
201 | + timeout 1800 bun run scripts/worker.ts --task=cleanup || true | |
202 | + echo "Cleanup complete." | |
203 | + no_output_timeout: 30m | |
147 | 204 | |
148 | 205 | # ────────────────────────────────────────────── |
149 | - # Worker Health Check | |
206 | + # Health Check (tests DB connectivity) | |
150 | 207 | # ────────────────────────────────────────────── |
151 | 208 | health-check: |
152 | - executor: bun-executor | |
209 | + executor: bun-medium | |
210 | + environment: | |
211 | + NODE_ENV: production | |
153 | 212 | steps: |
213 | + - checkout | |
214 | + - setup-bun | |
215 | + - install-deps | |
216 | + - check-env | |
217 | + | |
154 | 218 | - run: |
155 | - name: Check Worker Health | |
219 | + name: Test Database Connection | |
156 | 220 | command: | |
157 | - curl -sf http://your-worker-host:9090/health || exit 1 | |
158 | - echo "✅ Worker is healthy" | |
221 | + echo "Testing database connection..." | |
222 | + bun -e " | |
223 | + const { getDatabase } = require('./src/db'); | |
224 | + try { | |
225 | + const db = getDatabase(); | |
226 | + console.log('✅ Database connected successfully'); | |
227 | + process.exit(0); | |
228 | + } catch (err) { | |
229 | + console.error('❌ Database connection failed:', err.message); | |
230 | + process.exit(1); | |
231 | + } | |
232 | + " | |
233 | + | |
234 | + - run: | |
235 | + name: Test Redis Connection | |
236 | + command: | | |
237 | + echo "Testing Redis connection..." | |
238 | + # Quick Redis ping using bun | |
239 | + bun -e " | |
240 | + const url = new URL(process.env.REDIS_URL || ''); | |
241 | + const client = require('redis').createClient({ url: process.env.REDIS_URL }); | |
242 | + client.connect() | |
243 | + .then(() => client.ping()) | |
244 | + .then(() => { console.log('✅ Redis connected'); client.quit(); process.exit(0); }) | |
245 | + .catch((err) => { console.error('❌ Redis failed:', err.message); process.exit(1); }); | |
246 | + " | |
159 | 247 | |
160 | 248 | # ── Workflows ──────────────────────────────────────────────── |
161 | 249 | |
162 | 250 | workflows: |
163 | 251 | version: 2 |
164 | 252 | |
165 | - # OPTION A: Continuous worker (requires self-hosted runner) | |
166 | - # Uncomment this if you have a self-hosted runner | |
253 | + # ────────────────────────────────────────────── | |
254 | + # OPTION A: Continuous Worker with Auto-Restart | |
255 | + # | |
256 | + # Runs the worker continuously, restarting every | |
257 | + # 4.5 hours (before the 5h Cloud timeout). | |
258 | + # Best for: Production with CircleCI Cloud | |
259 | + # Requires: Performance plan or higher for 5h jobs | |
260 | + # ────────────────────────────────────────────── | |
261 | + # Uncomment below to enable continuous worker | |
262 | + # | |
167 | 263 | # worker-continuous: |
168 | 264 | # jobs: |
169 | - # - worker-continuous: | |
265 | + # - worker: | |
170 | 266 | # filters: |
171 | 267 | # branches: |
172 | 268 | # only: |
173 | 269 | # - main |
270 | + # # Context with all environment variables | |
271 | + # context: opencodehub-production | |
272 | + | |
273 | + # ────────────────────────────────────────────── | |
274 | + # OPTION B: Scheduled Tasks (Recommended for Free Plan) | |
275 | + # | |
276 | + # Runs individual tasks on a schedule. | |
277 | + # Each job completes quickly (< 30 min). | |
278 | + # Best for: Free plan users, light workloads | |
279 | + # ────────────────────────────────────────────── | |
174 | 280 | |
175 | - # OPTION B: Scheduled worker tasks (no self-hosted runner needed) | |
176 | - # Runs merge queue every 4 hours | |
281 | + # Merge queue: every 15 minutes | |
177 | 282 | scheduled-merge-queue: |
178 | 283 | triggers: |
179 | 284 | - schedule: |
180 | - cron: "0 */4 * * *" | |
285 | + cron: "*/15 * * * *" | |
181 | 286 | filters: |
182 | 287 | branches: |
183 | 288 | only: |
184 | 289 | - main |
185 | 290 | jobs: |
186 | - - merge-queue | |
291 | + - merge-queue: | |
292 | + context: opencodehub-production | |
187 | 293 | |
188 | - # Runs mirror sync every hour | |
294 | + # Mirror sync: every 2 hours | |
189 | 295 | scheduled-mirror-sync: |
190 | 296 | triggers: |
191 | 297 | - schedule: |
192 | - cron: "0 * * * *" | |
298 | + cron: "0 */2 * * *" | |
193 | 299 | filters: |
194 | 300 | branches: |
195 | 301 | only: |
196 | 302 | - main |
197 | 303 | jobs: |
198 | - - mirror-sync | |
304 | + - mirror-sync: | |
305 | + context: opencodehub-production | |
199 | 306 | |
200 | - # Runs cleanup daily at 3 AM UTC | |
307 | + # Cleanup: daily at 4 AM UTC | |
201 | 308 | scheduled-cleanup: |
202 | 309 | triggers: |
203 | 310 | - schedule: |
204 | - cron: "0 3 * * *" | |
311 | + cron: "0 4 * * *" | |
205 | 312 | filters: |
206 | 313 | branches: |
207 | 314 | only: |
208 | 315 | - main |
209 | 316 | jobs: |
210 | - - cleanup | |
317 | + - cleanup: | |
318 | + context: opencodehub-production | |
211 | 319 | |
212 | - # Health check every hour | |
320 | + # Health check: every 30 minutes | |
213 | 321 | scheduled-health: |
214 | 322 | triggers: |
215 | 323 | - schedule: |
216 | - cron: "0 * * * *" | |
324 | + cron: "*/30 * * * *" | |
217 | 325 | filters: |
218 | 326 | branches: |
219 | 327 | only: |
220 | 328 | - main |
221 | 329 | jobs: |
222 | - - health-check | |
330 | + - health-check: | |
331 | + context: opencodehub-production | |
223 | 332 | |
▼docs-site/src/content/docs/administration/circleci-cloud.md
+431-0
| @@ -0,0 +1,431 @@ | ||
1 | +--- | |
2 | +title: "Run Workers on CircleCI Cloud" | |
3 | +description: "Run OpenCodeHub background workers using managed CircleCI Cloud with external PostgreSQL and Redis" | |
4 | +--- | |
5 | + | |
6 | +> **Platform:** CircleCI Cloud (managed SaaS) | **Database:** PostgreSQL | **Difficulty:** Easy-Medium | |
7 | + | |
8 | +This guide is specifically for **CircleCI Cloud** (the managed service at circleci.com), NOT self-hosted runners. If you want to use self-hosted runners, see [CircleCI Self-Hosted Runner Guide](/administration/circleci-worker/). | |
9 | + | |
10 | +--- | |
11 | + | |
12 | +## How CircleCI Cloud Works for Workers | |
13 | + | |
14 | +CircleCI Cloud runs jobs in **ephemeral Docker containers** that are destroyed after the job ends. This is fundamentally different from a traditional background worker: | |
15 | + | |
16 | +| Feature | Traditional Worker | CircleCI Cloud | | |
17 | +|---------|-------------------|----------------| | |
18 | +| Lifetime | Runs forever | Max 5 hours per job | | |
19 | +| Storage | Persistent disk | Ephemeral (lost after job) | | |
20 | +| Database | Local or network | Must be external (network-accessible) | | |
21 | +| Redis | Local or network | Must be external | | |
22 | +| Cost | Server rental | Compute minutes | | |
23 | + | |
24 | +**Bottom line:** CircleCI Cloud workers run in short bursts and connect to external services. This is actually perfect for OpenCodeHub's architecture because the worker code is stateless — all state lives in PostgreSQL + Redis. | |
25 | + | |
26 | +--- | |
27 | + | |
28 | +## Prerequisites | |
29 | + | |
30 | +Before you start, you need: | |
31 | + | |
32 | +1. **CircleCI Account** (free tier works) | |
33 | +2. **External PostgreSQL** database (required — local DB won't persist) | |
34 | +3. **External Redis** (required — local Redis won't persist) | |
35 | +4. **Git repository** pushed to GitHub/GitLab/Bitbucket (CircleCI connects to it) | |
36 | + | |
37 | +### Recommended PostgreSQL Providers | |
38 | + | |
39 | +| Provider | Free Tier | Best For | | |
40 | +|----------|-----------|----------| | |
41 | +| **Supabase** | 500MB, 2GB egress | Full-featured, good dashboard | | |
42 | +| **Neon** | 500MB, 3 databases | Serverless, scales to zero | | |
43 | +| **Railway** | $5 credit | Simple, pay-as-you-go | | |
44 | +| **Render** | 1GB | Easy setup | | |
45 | +| **AWS RDS** | None | Enterprise | | |
46 | + | |
47 | +### Recommended Redis Providers | |
48 | + | |
49 | +| Provider | Free Tier | Best For | | |
50 | +|----------|-----------|----------| | |
51 | +| **Upstash** | 10,000 commands/day | Serverless, HTTP API | | |
52 | +| **Redis Cloud** | 30MB | Managed Redis | | |
53 | +| **Railway** | $5 credit | Easy setup | | |
54 | +| **KeyDB** | Self-host | Performance | | |
55 | + | |
56 | +--- | |
57 | + | |
58 | +## Step 1: Set Up External PostgreSQL | |
59 | + | |
60 | +### Using Supabase (Recommended) | |
61 | + | |
62 | +1. Go to [supabase.com](https://supabase.com) and create a project | |
63 | +2. In **Project Settings** → **Database**, copy the connection string: | |
64 | + ``` | |
65 | + postgresql://postgres:[password]@db.xxxxxxxxxx.supabase.co:5432/postgres | |
66 | + ``` | |
67 | +3. Create the `opencodehub` database: | |
68 | + ```sql | |
69 | + CREATE DATABASE opencodehub; | |
70 | + ``` | |
71 | + | |
72 | +### Using Neon | |
73 | + | |
74 | +1. Go to [neon.tech](https://neon.tech) and create a project | |
75 | +2. Copy the connection string from the dashboard | |
76 | +3. Create the database: | |
77 | + ```sql | |
78 | + CREATE DATABASE opencodehub; | |
79 | + ``` | |
80 | + | |
81 | +--- | |
82 | + | |
83 | +## Step 2: Set Up External Redis | |
84 | + | |
85 | +### Using Upstash (Recommended for Serverless) | |
86 | + | |
87 | +1. Go to [upstash.com](https://upstash.com) and create a Redis database | |
88 | +2. Copy the **Redis URL**: | |
89 | + ``` | |
90 | + rediss://default:[password]@xxxx.upstash.io:6379 | |
91 | + ``` | |
92 | +3. Enable **TLS** (Upstash requires it) | |
93 | + | |
94 | +--- | |
95 | + | |
96 | +## Step 3: Configure CircleCI Project | |
97 | + | |
98 | +### 3.1 Connect Repository | |
99 | + | |
100 | +1. Go to [app.circleci.com](https://app.circleci.com) | |
101 | +2. Click **Projects** → **Set Up Project** | |
102 | +3. Select your OpenCodeHub repository | |
103 | +4. CircleCI will detect `.circleci/config.yml` | |
104 | + | |
105 | +### 3.2 Create a Context (Recommended) | |
106 | + | |
107 | +Contexts let you share environment variables across jobs securely. | |
108 | + | |
109 | +1. Go to **Organization Settings** → **Contexts** | |
110 | +2. Click **Create Context** | |
111 | +3. Name it: `opencodehub-production` | |
112 | +4. Add these environment variables: | |
113 | + | |
114 | +| Variable | Value | Description | | |
115 | +|----------|-------|-------------| | |
116 | +| `DATABASE_URL` | `postgresql://...` | Your PostgreSQL connection string | | |
117 | +| `DATABASE_DRIVER` | `postgres` | Explicitly set driver | | |
118 | +| `REDIS_URL` | `redis://...` or `rediss://...` | Your Redis URL | | |
119 | +| `JWT_SECRET` | `openssl rand -hex 32` | Generate this | | |
120 | +| `SESSION_SECRET` | `openssl rand -hex 32` | Generate this | | |
121 | +| `INTERNAL_HOOK_SECRET` | `openssl rand -hex 32` | Generate this | | |
122 | +| `CRON_SECRET` | `openssl rand -hex 32` | Generate this | | |
123 | +| `RUNNER_SECRET` | `openssl rand -hex 32` | Generate this | | |
124 | +| `WORKFLOW_SECRET_ENCRYPTION_KEY` | `openssl rand -hex 32` | Generate this | | |
125 | +| `SITE_URL` | `https://git.yourdomain.com` | Your OpenCodeHub URL | | |
126 | +| `SKIP_REDIS_CHECK` | `1` | Skip Redis check during build | | |
127 | + | |
128 | +> **Security:** Never commit these values to git. Always use CircleCI contexts or project environment variables. | |
129 | + | |
130 | +### 3.3 Alternative: Project-Level Environment Variables | |
131 | + | |
132 | +If you don't want to use contexts: | |
133 | + | |
134 | +1. Go to **Project Settings** → **Environment Variables** | |
135 | +2. Add each variable individually | |
136 | + | |
137 | +--- | |
138 | + | |
139 | +## Step 4: Initialize Database Schema | |
140 | + | |
141 | +Before the worker can run, the database schema must exist. Run this once: | |
142 | + | |
143 | +```bash | |
144 | +# On your local machine with DATABASE_URL set | |
145 | +export DATABASE_URL="postgresql://postgres:[password]@db.xxxxxxxxxx.supabase.co:5432/opencodehub" | |
146 | + | |
147 | +# Install dependencies | |
148 | +bun install | |
149 | + | |
150 | +# Push schema to database | |
151 | +bun run db:push | |
152 | + | |
153 | +# Or generate and run migrations | |
154 | +bun run db:generate | |
155 | +bun run db:migrate | |
156 | + | |
157 | +# Create admin user | |
158 | +bun run scripts/seed-admin.ts | |
159 | +``` | |
160 | + | |
161 | +> **Important:** Do this from your local machine or a one-time job. The CircleCI worker job won't create the schema automatically. | |
162 | + | |
163 | +--- | |
164 | + | |
165 | +## Step 5: Understanding the CircleCI Cloud Config | |
166 | + | |
167 | +The `.circleci/config.yml` in your repo has multiple approaches: | |
168 | + | |
169 | +### Option A: Scheduled Tasks (Recommended for Free Plan) | |
170 | + | |
171 | +Jobs run on a cron schedule. Each job starts, does its work, and exits. | |
172 | + | |
173 | +```yaml | |
174 | +workflows: | |
175 | + scheduled-merge-queue: | |
176 | + triggers: | |
177 | + - schedule: | |
178 | + cron: "*/15 * * * *" # Every 15 minutes | |
179 | + jobs: | |
180 | + - merge-queue: | |
181 | + context: opencodehub-production | |
182 | +``` | |
183 | + | |
184 | +**What happens:** | |
185 | +1. CircleCI starts a Docker container every 15 minutes | |
186 | +2. Installs Bun and dependencies | |
187 | +3. Runs `bun run scripts/worker.ts` for 30 minutes | |
188 | +4. Processes all pending merge queue items | |
189 | +5. Container is destroyed | |
190 | + | |
191 | +**Pros:** Works on free plan, reliable, no timeouts | |
192 | +**Cons:** 15-minute delay between processing (jobs only run on schedule) | |
193 | + | |
194 | +### Option B: Continuous Worker (Performance Plan) | |
195 | + | |
196 | +One long-running job that stays active until timeout. | |
197 | + | |
198 | +```yaml | |
199 | +workflows: | |
200 | + worker-continuous: | |
201 | + jobs: | |
202 | + - worker: | |
203 | + context: opencodehub-production | |
204 | +``` | |
205 | + | |
206 | +**What happens:** | |
207 | +1. CircleCI starts a container | |
208 | +2. Worker runs continuously, processing items every 5 seconds | |
209 | +3. Job runs until the 5-hour timeout | |
210 | +4. CircleCI restarts the job automatically | |
211 | + | |
212 | +**Pros:** Near real-time processing | |
213 | +**Cons:** Requires Performance plan (5h jobs), uses more compute minutes | |
214 | + | |
215 | +### Option C: API-Triggered (Event-Driven) | |
216 | + | |
217 | +OpenCodeHub calls CircleCI API when events happen. | |
218 | + | |
219 | +In your OpenCodeHub webhook handler: | |
220 | + | |
221 | +```typescript | |
222 | +// When merge queue needs processing | |
223 | +async function triggerCircleCIWorker(eventType: string, repoId: string) { | |
224 | + const response = await fetch( | |
225 | + `https://circleci.com/api/v2/project/gh/${org}/${repo}/pipeline`, | |
226 | + { | |
227 | + method: "POST", | |
228 | + headers: { | |
229 | + "Circle-Token": process.env.CIRCLECI_API_TOKEN!, | |
230 | + "Content-Type": "application/json", | |
231 | + }, | |
232 | + body: JSON.stringify({ | |
233 | + branch: "main", | |
234 | + parameters: { | |
235 | + event_type: eventType, | |
236 | + repo_id: repoId, | |
237 | + }, | |
238 | + }), | |
239 | + } | |
240 | + ); | |
241 | + return response.json(); | |
242 | +} | |
243 | +``` | |
244 | + | |
245 | +**Pros:** Only runs when needed, most efficient | |
246 | +**Cons:** Requires code changes in OpenCodeHub, more complex setup | |
247 | + | |
248 | +--- | |
249 | + | |
250 | +## Step 6: Choose Your Plan | |
251 | + | |
252 | +### Free Plan (Recommended to Start) | |
253 | + | |
254 | +```yaml | |
255 | +workflows: | |
256 | + scheduled-merge-queue: | |
257 | + triggers: | |
258 | + - schedule: | |
259 | + cron: "*/15 * * * *" # Every 15 min | |
260 | + jobs: | |
261 | + - merge-queue: | |
262 | + context: opencodehub-production | |
263 | +``` | |
264 | + | |
265 | +**Monthly usage:** ~2,880 minutes (48 runs/day × 30 days × 2 min avg) | |
266 | +**Free tier:** 6,000 minutes/month ✅ | |
267 | + | |
268 | +### Performance Plan (For Production) | |
269 | + | |
270 | +```yaml | |
271 | +workflows: | |
272 | + worker-continuous: | |
273 | + jobs: | |
274 | + - worker: | |
275 | + context: opencodehub-production | |
276 | +``` | |
277 | + | |
278 | +**Monthly usage:** ~7,200 minutes (5h × 24/day × 30 days) | |
279 | +**Performance plan:** Included in price | |
280 | + | |
281 | +--- | |
282 | + | |
283 | +## Step 7: Database Connection from CircleCI Cloud | |
284 | + | |
285 | +Since CircleCI Cloud containers are ephemeral, the database must be network-accessible. | |
286 | + | |
287 | +### Connection String Format | |
288 | + | |
289 | +```bash | |
290 | +# Supabase | |
291 | +DATABASE_URL=postgresql://postgres:[password]@db.xxxxxxxxxx.supabase.co:5432/opencodehub | |
292 | + | |
293 | +# Neon (with SSL) | |
294 | +DATABASE_URL=postgresql://[user]:[password]@ep-xxxxx.us-east-1.aws.neon.tech:5432/opencodehub?sslmode=require | |
295 | + | |
296 | +# Railway | |
297 | +DATABASE_URL=postgresql://postgres:[password]@containers-xxxxx.railway.app:5432/railway | |
298 | + | |
299 | +# AWS RDS (with SSL) | |
300 | +DATABASE_URL=postgresql://opencodehub:[password]@my-db.xxxxx.us-east-1.rds.amazonaws.com:5432/opencodehub?sslmode=require | |
301 | +``` | |
302 | + | |
303 | +### SSL/TLS Configuration | |
304 | + | |
305 | +Most managed PostgreSQL require SSL. The worker handles this automatically when `sslmode=require` is in the URL. | |
306 | + | |
307 | +For Supabase, add this to your environment variables: | |
308 | +```bash | |
309 | +DATABASE_SSL=true | |
310 | +DATABASE_SSL_REJECT_UNAUTHORIZED=false # Supabase uses self-signed certs | |
311 | +``` | |
312 | + | |
313 | +--- | |
314 | + | |
315 | +## Step 8: Redis Connection from CircleCI Cloud | |
316 | + | |
317 | +### Upstash Redis URL | |
318 | + | |
319 | +```bash | |
320 | +REDIS_URL=rediss://default:[password]@xxxx.upstash.io:6379 | |
321 | +``` | |
322 | + | |
323 | +> Note: `rediss://` (with double s) = TLS enabled. Upstash requires TLS. | |
324 | + | |
325 | +### Redis Cloud | |
326 | + | |
327 | +```bash | |
328 | +REDIS_URL=redis://default:[password]@redis-xxxxx.redis-cloud.com:12345 | |
329 | +``` | |
330 | + | |
331 | +### Testing Connection | |
332 | + | |
333 | +You can test the connection from CircleCI by running the `health-check` job: | |
334 | + | |
335 | +```bash | |
336 | +# Trigger manually from CircleCI dashboard | |
337 | +# or wait for the scheduled health check | |
338 | +``` | |
339 | + | |
340 | +--- | |
341 | + | |
342 | +## Step 9: Monitoring | |
343 | + | |
344 | +### View Job Logs | |
345 | + | |
346 | +1. Go to [app.circleci.com](https://app.circleci.com) | |
347 | +2. Select your project | |
348 | +3. Click on any job to see real-time logs | |
349 | + | |
350 | +### Set Up Notifications | |
351 | + | |
352 | +1. **Project Settings** → **Notifications** | |
353 | +2. Connect Slack, email, or webhooks | |
354 | +3. Get alerts when jobs fail | |
355 | + | |
356 | +### Track Compute Usage | |
357 | + | |
358 | +1. **Plan** → **Usage** | |
359 | +2. Monitor minutes used per month | |
360 | +3. Set up usage alerts | |
361 | + | |
362 | +--- | |
363 | + | |
364 | +## Step 10: Troubleshooting | |
365 | + | |
366 | +| Problem | Solution | | |
367 | +|---------|----------| | |
368 | +| **"Database connection refused"** | Your PostgreSQL doesn't allow external connections. Check provider's "Connection Pooling" or "Network" settings. For Supabase, go to Database → Connection Pooling. | | |
369 | +| **"Redis connection timeout"** | Redis provider blocks unknown IPs. Add CircleCI's IP ranges to allowlist. Or use Upstash which doesn't require IP allowlisting. | | |
370 | +| **Job killed after 1 hour** | Free plan default timeout is 1 hour. Add `no_output_timeout: 5h` or upgrade to Performance plan. | | |
371 | +| **"Out of memory"** | Use `resource_class: large` instead of `medium`. Or optimize the worker memory usage. | | |
372 | +| **"bun install fails"** | Network issues. The config has retry logic, but you can also try `npm install` as fallback. | | |
373 | +| **"Schema doesn't exist"** | Run `bun run db:push` from your local machine first to create tables. | | |
374 | +| **Worker starts but does nothing** | Check that `DATABASE_URL` points to the right database and that the `mergeQueueItems` table has rows with `status = 'queued'`. | | |
375 | +| **"SSL certificate verify failed"** | Add `DATABASE_SSL_REJECT_UNAUTHORIZED=false` for providers with self-signed certs (Supabase). | | |
376 | + | |
377 | +--- | |
378 | + | |
379 | +## Architecture Diagram | |
380 | + | |
381 | +``` | |
382 | +┌─────────────────────────────────────────────────────────────┐ | |
383 | +│ CircleCI Cloud │ | |
384 | +│ ┌─────────────────┐ ┌─────────────────┐ │ | |
385 | +│ │ Docker Container │ │ Docker Container │ ... │ | |
386 | +│ │ (every 15 min) │ │ (every 15 min) │ │ | |
387 | +│ │ │ │ │ │ | |
388 | +│ │ bun run worker.ts│ │ bun run worker.ts│ │ | |
389 | +│ │ │ │ │ │ | |
390 | +│ │ ┌─────────────┐ │ │ ┌─────────────┐ │ │ | |
391 | +│ │ │ Node.js/Bun │ │ │ │ Node.js/Bun │ │ │ | |
392 | +│ │ │ Worker Loop │ │ │ │ Worker Loop │ │ │ | |
393 | +│ │ └─────────────┘ │ │ └─────────────┘ │ │ | |
394 | +│ └────────┬─────────┘ └────────┬─────────┘ │ | |
395 | +└───────────┼──────────────── ─────┼────────────────────────────┘ | |
396 | + │ │ | |
397 | + │ TCP/IP (Internet) │ | |
398 | + ▼ ▼ | |
399 | + ┌──────────────┐ ┌──────────────┐ | |
400 | + │ PostgreSQL │ │ Redis │ | |
401 | + │ (Supabase) │ │ (Upstash) │ | |
402 | + │ │ │ │ | |
403 | + │ merge_queue │ │ sessions │ | |
404 | + │ repositories │ │ rate limits │ | |
405 | + │ pull_requests │ │ job queues │ | |
406 | + └──────────────┘ └──────────────┘ | |
407 | +``` | |
408 | + | |
409 | +--- | |
410 | + | |
411 | +## Quick Start Checklist | |
412 | + | |
413 | +- [ ] Create PostgreSQL database (Supabase/Neon/Railway) | |
414 | +- [ ] Create Redis database (Upstash/Redis Cloud) | |
415 | +- [ ] Run `bun run db:push` from local machine | |
416 | +- [ ] Push OpenCodeHub code to GitHub | |
417 | +- [ ] Connect repo to CircleCI | |
418 | +- [ ] Create Context `opencodehub-production` with all env vars | |
419 | +- [ ] Verify `.circleci/config.yml` is in repo root | |
420 | +- [ ] Trigger first job manually from CircleCI dashboard | |
421 | +- [ ] Check job logs for "✅ Worker started" | |
422 | +- [ ] Create a test PR and verify merge queue processes it | |
423 | + | |
424 | +--- | |
425 | + | |
426 | +## Next Steps | |
427 | + | |
428 | +- [CircleCI Integration Guide](/guides/circleci-integration/) — For CI/CD pipelines (separate from worker) | |
429 | +- [Deployment Guide](/administration/deployment/) — Full production setup | |
430 | +- [Self-Hosted Runner Guide](/administration/circleci-worker/) — If you prefer self-hosted over Cloud | |
431 | +- [Merge Queue](/features/merge-queue/) — How merge queue works | |
0 | 432 | |
▼docs/administration/circleci-cloud.md
+428-0
| @@ -0,0 +1,428 @@ | ||
1 | +# OpenCodeHub Worker on CircleCI Cloud (SaaS) | |
2 | + | |
3 | +> **Platform:** CircleCI Cloud (managed SaaS) | **Database:** PostgreSQL | **Difficulty:** Easy-Medium | |
4 | + | |
5 | +This guide is specifically for **CircleCI Cloud** (the managed service at circleci.com), NOT self-hosted runners. If you want to use self-hosted runners, see [CircleCI Self-Hosted Runner Guide](circleci-worker.md). | |
6 | + | |
7 | +--- | |
8 | + | |
9 | +## How CircleCI Cloud Works for Workers | |
10 | + | |
11 | +CircleCI Cloud runs jobs in **ephemeral Docker containers** that are destroyed after the job ends. This is fundamentally different from a traditional background worker: | |
12 | + | |
13 | +| Feature | Traditional Worker | CircleCI Cloud | | |
14 | +|---------|-------------------|----------------| | |
15 | +| Lifetime | Runs forever | Max 5 hours per job | | |
16 | +| Storage | Persistent disk | Ephemeral (lost after job) | | |
17 | +| Database | Local or network | Must be external (network-accessible) | | |
18 | +| Redis | Local or network | Must be external | | |
19 | +| Cost | Server rental | Compute minutes | | |
20 | + | |
21 | +**Bottom line:** CircleCI Cloud workers run in short bursts and connect to external services. This is actually perfect for OpenCodeHub's architecture because the worker code is stateless — all state lives in PostgreSQL + Redis. | |
22 | + | |
23 | +--- | |
24 | + | |
25 | +## Prerequisites | |
26 | + | |
27 | +Before you start, you need: | |
28 | + | |
29 | +1. **CircleCI Account** (free tier works) | |
30 | +2. **External PostgreSQL** database (required — local DB won't persist) | |
31 | +3. **External Redis** (required — local Redis won't persist) | |
32 | +4. **Git repository** pushed to GitHub/GitLab/Bitbucket (CircleCI connects to it) | |
33 | + | |
34 | +### Recommended PostgreSQL Providers | |
35 | + | |
36 | +| Provider | Free Tier | Best For | | |
37 | +|----------|-----------|----------| | |
38 | +| **Supabase** | 500MB, 2GB egress | Full-featured, good dashboard | | |
39 | +| **Neon** | 500MB, 3 databases | Serverless, scales to zero | | |
40 | +| **Railway** | $5 credit | Simple, pay-as-you-go | | |
41 | +| **Render** | 1GB | Easy setup | | |
42 | +| **AWS RDS** | None | Enterprise | | |
43 | + | |
44 | +### Recommended Redis Providers | |
45 | + | |
46 | +| Provider | Free Tier | Best For | | |
47 | +|----------|-----------|----------| | |
48 | +| **Upstash** | 10,000 commands/day | Serverless, HTTP API | | |
49 | +| **Redis Cloud** | 30MB | Managed Redis | | |
50 | +| **Railway** | $5 credit | Easy setup | | |
51 | +| **KeyDB** | Self-host | Performance | | |
52 | + | |
53 | +--- | |
54 | + | |
55 | +## Step 1: Set Up External PostgreSQL | |
56 | + | |
57 | +### Using Supabase (Recommended) | |
58 | + | |
59 | +1. Go to [supabase.com](https://supabase.com) and create a project | |
60 | +2. In **Project Settings** → **Database**, copy the connection string: | |
61 | + ``` | |
62 | + postgresql://postgres:[password]@db.xxxxxxxxxx.supabase.co:5432/postgres | |
63 | + ``` | |
64 | +3. Create the `opencodehub` database: | |
65 | + ```sql | |
66 | + CREATE DATABASE opencodehub; | |
67 | + ``` | |
68 | + | |
69 | +### Using Neon | |
70 | + | |
71 | +1. Go to [neon.tech](https://neon.tech) and create a project | |
72 | +2. Copy the connection string from the dashboard | |
73 | +3. Create the database: | |
74 | + ```sql | |
75 | + CREATE DATABASE opencodehub; | |
76 | + ``` | |
77 | + | |
78 | +--- | |
79 | + | |
80 | +## Step 2: Set Up External Redis | |
81 | + | |
82 | +### Using Upstash (Recommended for Serverless) | |
83 | + | |
84 | +1. Go to [upstash.com](https://upstash.com) and create a Redis database | |
85 | +2. Copy the **Redis URL**: | |
86 | + ``` | |
87 | + rediss://default:[password]@xxxx.upstash.io:6379 | |
88 | + ``` | |
89 | +3. Enable **TLS** (Upstash requires it) | |
90 | + | |
91 | +--- | |
92 | + | |
93 | +## Step 3: Configure CircleCI Project | |
94 | + | |
95 | +### 3.1 Connect Repository | |
96 | + | |
97 | +1. Go to [app.circleci.com](https://app.circleci.com) | |
98 | +2. Click **Projects** → **Set Up Project** | |
99 | +3. Select your OpenCodeHub repository | |
100 | +4. CircleCI will detect `.circleci/config.yml` | |
101 | + | |
102 | +### 3.2 Create a Context (Recommended) | |
103 | + | |
104 | +Contexts let you share environment variables across jobs securely. | |
105 | + | |
106 | +1. Go to **Organization Settings** → **Contexts** | |
107 | +2. Click **Create Context** | |
108 | +3. Name it: `opencodehub-production` | |
109 | +4. Add these environment variables: | |
110 | + | |
111 | +| Variable | Value | Description | | |
112 | +|----------|-------|-------------| | |
113 | +| `DATABASE_URL` | `postgresql://...` | Your PostgreSQL connection string | | |
114 | +| `DATABASE_DRIVER` | `postgres` | Explicitly set driver | | |
115 | +| `REDIS_URL` | `redis://...` or `rediss://...` | Your Redis URL | | |
116 | +| `JWT_SECRET` | `openssl rand -hex 32` | Generate this | | |
117 | +| `SESSION_SECRET` | `openssl rand -hex 32` | Generate this | | |
118 | +| `INTERNAL_HOOK_SECRET` | `openssl rand -hex 32` | Generate this | | |
119 | +| `CRON_SECRET` | `openssl rand -hex 32` | Generate this | | |
120 | +| `RUNNER_SECRET` | `openssl rand -hex 32` | Generate this | | |
121 | +| `WORKFLOW_SECRET_ENCRYPTION_KEY` | `openssl rand -hex 32` | Generate this | | |
122 | +| `SITE_URL` | `https://git.yourdomain.com` | Your OpenCodeHub URL | | |
123 | +| `SKIP_REDIS_CHECK` | `1` | Skip Redis check during build | | |
124 | + | |
125 | +> **Security:** Never commit these values to git. Always use CircleCI contexts or project environment variables. | |
126 | + | |
127 | +### 3.3 Alternative: Project-Level Environment Variables | |
128 | + | |
129 | +If you don't want to use contexts: | |
130 | + | |
131 | +1. Go to **Project Settings** → **Environment Variables** | |
132 | +2. Add each variable individually | |
133 | + | |
134 | +--- | |
135 | + | |
136 | +## Step 4: Initialize Database Schema | |
137 | + | |
138 | +Before the worker can run, the database schema must exist. Run this once: | |
139 | + | |
140 | +```bash | |
141 | +# On your local machine with DATABASE_URL set | |
142 | +export DATABASE_URL="postgresql://postgres:[password]@db.xxxxxxxxxx.supabase.co:5432/opencodehub" | |
143 | + | |
144 | +# Install dependencies | |
145 | +bun install | |
146 | + | |
147 | +# Push schema to database | |
148 | +bun run db:push | |
149 | + | |
150 | +# Or generate and run migrations | |
151 | +bun run db:generate | |
152 | +bun run db:migrate | |
153 | + | |
154 | +# Create admin user | |
155 | +bun run scripts/seed-admin.ts | |
156 | +``` | |
157 | + | |
158 | +> **Important:** Do this from your local machine or a one-time job. The CircleCI worker job won't create the schema automatically. | |
159 | + | |
160 | +--- | |
161 | + | |
162 | +## Step 5: Understanding the CircleCI Cloud Config | |
163 | + | |
164 | +The `.circleci/config.yml` in your repo has multiple approaches: | |
165 | + | |
166 | +### Option A: Scheduled Tasks (Recommended for Free Plan) | |
167 | + | |
168 | +Jobs run on a cron schedule. Each job starts, does its work, and exits. | |
169 | + | |
170 | +```yaml | |
171 | +workflows: | |
172 | + scheduled-merge-queue: | |
173 | + triggers: | |
174 | + - schedule: | |
175 | + cron: "*/15 * * * *" # Every 15 minutes | |
176 | + jobs: | |
177 | + - merge-queue: | |
178 | + context: opencodehub-production | |
179 | +``` | |
180 | + | |
181 | +**What happens:** | |
182 | +1. CircleCI starts a Docker container every 15 minutes | |
183 | +2. Installs Bun and dependencies | |
184 | +3. Runs `bun run scripts/worker.ts` for 30 minutes | |
185 | +4. Processes all pending merge queue items | |
186 | +5. Container is destroyed | |
187 | + | |
188 | +**Pros:** Works on free plan, reliable, no timeouts | |
189 | +**Cons:** 15-minute delay between processing (jobs only run on schedule) | |
190 | + | |
191 | +### Option B: Continuous Worker (Performance Plan) | |
192 | + | |
193 | +One long-running job that stays active until timeout. | |
194 | + | |
195 | +```yaml | |
196 | +workflows: | |
197 | + worker-continuous: | |
198 | + jobs: | |
199 | + - worker: | |
200 | + context: opencodehub-production | |
201 | +``` | |
202 | + | |
203 | +**What happens:** | |
204 | +1. CircleCI starts a container | |
205 | +2. Worker runs continuously, processing items every 5 seconds | |
206 | +3. Job runs until the 5-hour timeout | |
207 | +4. CircleCI restarts the job automatically | |
208 | + | |
209 | +**Pros:** Near real-time processing | |
210 | +**Cons:** Requires Performance plan (5h jobs), uses more compute minutes | |
211 | + | |
212 | +### Option C: API-Triggered (Event-Driven) | |
213 | + | |
214 | +OpenCodeHub calls CircleCI API when events happen. | |
215 | + | |
216 | +In your OpenCodeHub webhook handler: | |
217 | + | |
218 | +```typescript | |
219 | +// When merge queue needs processing | |
220 | +async function triggerCircleCIWorker(eventType: string, repoId: string) { | |
221 | + const response = await fetch( | |
222 | + `https://circleci.com/api/v2/project/gh/${org}/${repo}/pipeline`, | |
223 | + { | |
224 | + method: "POST", | |
225 | + headers: { | |
226 | + "Circle-Token": process.env.CIRCLECI_API_TOKEN!, | |
227 | + "Content-Type": "application/json", | |
228 | + }, | |
229 | + body: JSON.stringify({ | |
230 | + branch: "main", | |
231 | + parameters: { | |
232 | + event_type: eventType, | |
233 | + repo_id: repoId, | |
234 | + }, | |
235 | + }), | |
236 | + } | |
237 | + ); | |
238 | + return response.json(); | |
239 | +} | |
240 | +``` | |
241 | + | |
242 | +**Pros:** Only runs when needed, most efficient | |
243 | +**Cons:** Requires code changes in OpenCodeHub, more complex setup | |
244 | + | |
245 | +--- | |
246 | + | |
247 | +## Step 6: Choose Your Plan | |
248 | + | |
249 | +### Free Plan (Recommended to Start) | |
250 | + | |
251 | +```yaml | |
252 | +workflows: | |
253 | + scheduled-merge-queue: | |
254 | + triggers: | |
255 | + - schedule: | |
256 | + cron: "*/15 * * * *" # Every 15 min | |
257 | + jobs: | |
258 | + - merge-queue: | |
259 | + context: opencodehub-production | |
260 | +``` | |
261 | + | |
262 | +**Monthly usage:** ~2,880 minutes (48 runs/day × 30 days × 2 min avg) | |
263 | +**Free tier:** 6,000 minutes/month ✅ | |
264 | + | |
265 | +### Performance Plan (For Production) | |
266 | + | |
267 | +```yaml | |
268 | +workflows: | |
269 | + worker-continuous: | |
270 | + jobs: | |
271 | + - worker: | |
272 | + context: opencodehub-production | |
273 | +``` | |
274 | + | |
275 | +**Monthly usage:** ~7,200 minutes (5h × 24/day × 30 days) | |
276 | +**Performance plan:** Included in price | |
277 | + | |
278 | +--- | |
279 | + | |
280 | +## Step 7: Database Connection from CircleCI Cloud | |
281 | + | |
282 | +Since CircleCI Cloud containers are ephemeral, the database must be network-accessible. | |
283 | + | |
284 | +### Connection String Format | |
285 | + | |
286 | +```bash | |
287 | +# Supabase | |
288 | +DATABASE_URL=postgresql://postgres:[password]@db.xxxxxxxxxx.supabase.co:5432/opencodehub | |
289 | + | |
290 | +# Neon (with SSL) | |
291 | +DATABASE_URL=postgresql://[user]:[password]@ep-xxxxx.us-east-1.aws.neon.tech:5432/opencodehub?sslmode=require | |
292 | + | |
293 | +# Railway | |
294 | +DATABASE_URL=postgresql://postgres:[password]@containers-xxxxx.railway.app:5432/railway | |
295 | + | |
296 | +# AWS RDS (with SSL) | |
297 | +DATABASE_URL=postgresql://opencodehub:[password]@my-db.xxxxx.us-east-1.rds.amazonaws.com:5432/opencodehub?sslmode=require | |
298 | +``` | |
299 | + | |
300 | +### SSL/TLS Configuration | |
301 | + | |
302 | +Most managed PostgreSQL require SSL. The worker handles this automatically when `sslmode=require` is in the URL. | |
303 | + | |
304 | +For Supabase, add this to your environment variables: | |
305 | +```bash | |
306 | +DATABASE_SSL=true | |
307 | +DATABASE_SSL_REJECT_UNAUTHORIZED=false # Supabase uses self-signed certs | |
308 | +``` | |
309 | + | |
310 | +--- | |
311 | + | |
312 | +## Step 8: Redis Connection from CircleCI Cloud | |
313 | + | |
314 | +### Upstash Redis URL | |
315 | + | |
316 | +```bash | |
317 | +REDIS_URL=rediss://default:[password]@xxxx.upstash.io:6379 | |
318 | +``` | |
319 | + | |
320 | +> Note: `rediss://` (with double s) = TLS enabled. Upstash requires TLS. | |
321 | + | |
322 | +### Redis Cloud | |
323 | + | |
324 | +```bash | |
325 | +REDIS_URL=redis://default:[password]@redis-xxxxx.redis-cloud.com:12345 | |
326 | +``` | |
327 | + | |
328 | +### Testing Connection | |
329 | + | |
330 | +You can test the connection from CircleCI by running the `health-check` job: | |
331 | + | |
332 | +```bash | |
333 | +# Trigger manually from CircleCI dashboard | |
334 | +# or wait for the scheduled health check | |
335 | +``` | |
336 | + | |
337 | +--- | |
338 | + | |
339 | +## Step 9: Monitoring | |
340 | + | |
341 | +### View Job Logs | |
342 | + | |
343 | +1. Go to [app.circleci.com](https://app.circleci.com) | |
344 | +2. Select your project | |
345 | +3. Click on any job to see real-time logs | |
346 | + | |
347 | +### Set Up Notifications | |
348 | + | |
349 | +1. **Project Settings** → **Notifications** | |
350 | +2. Connect Slack, email, or webhooks | |
351 | +3. Get alerts when jobs fail | |
352 | + | |
353 | +### Track Compute Usage | |
354 | + | |
355 | +1. **Plan** → **Usage** | |
356 | +2. Monitor minutes used per month | |
357 | +3. Set up usage alerts | |
358 | + | |
359 | +--- | |
360 | + | |
361 | +## Step 10: Troubleshooting | |
362 | + | |
363 | +| Problem | Solution | | |
364 | +|---------|----------| | |
365 | +| **"Database connection refused"** | Your PostgreSQL doesn't allow external connections. Check provider's "Connection Pooling" or "Network" settings. For Supabase, go to Database → Connection Pooling. | | |
366 | +| **"Redis connection timeout"** | Redis provider blocks unknown IPs. Add CircleCI's IP ranges to allowlist. Or use Upstash which doesn't require IP allowlisting. | | |
367 | +| **Job killed after 1 hour** | Free plan default timeout is 1 hour. Add `no_output_timeout: 5h` or upgrade to Performance plan. | | |
368 | +| **"Out of memory"** | Use `resource_class: large` instead of `medium`. Or optimize the worker memory usage. | | |
369 | +| **"bun install fails"** | Network issues. The config has retry logic, but you can also try `npm install` as fallback. | | |
370 | +| **"Schema doesn't exist"** | Run `bun run db:push` from your local machine first to create tables. | | |
371 | +| **Worker starts but does nothing** | Check that `DATABASE_URL` points to the right database and that the `mergeQueueItems` table has rows with `status = 'queued'`. | | |
372 | +| **"SSL certificate verify failed"** | Add `DATABASE_SSL_REJECT_UNAUTHORIZED=false` for providers with self-signed certs (Supabase). | | |
373 | + | |
374 | +--- | |
375 | + | |
376 | +## Architecture Diagram | |
377 | + | |
378 | +``` | |
379 | +┌─────────────────────────────────────────────────────────────┐ | |
380 | +│ CircleCI Cloud │ | |
381 | +│ ┌─────────────────┐ ┌─────────────────┐ │ | |
382 | +│ │ Docker Container │ │ Docker Container │ ... │ | |
383 | +│ │ (every 15 min) │ │ (every 15 min) │ │ | |
384 | +│ │ │ │ │ │ | |
385 | +│ │ bun run worker.ts│ │ bun run worker.ts│ │ | |
386 | +│ │ │ │ │ │ | |
387 | +│ │ ┌─────────────┐ │ │ ┌─────────────┐ │ │ | |
388 | +│ │ │ Node.js/Bun │ │ │ │ Node.js/Bun │ │ │ | |
389 | +│ │ │ Worker Loop │ │ │ │ Worker Loop │ │ │ | |
390 | +│ │ └─────────────┘ │ │ └─────────────┘ │ │ | |
391 | +│ └────────┬─────────┘ └────────┬──── ─────┘ │ | |
392 | +└───────────┼─────────────────────┼────────────────────────────┘ | |
393 | + │ │ | |
394 | + │ TCP/IP (Internet) │ | |
395 | + ▼ ▼ | |
396 | + ┌──────────────┐ ┌──────────────┐ | |
397 | + │ PostgreSQL │ │ Redis │ | |
398 | + │ (Supabase) │ │ (Upstash) │ | |
399 | + │ │ │ │ | |
400 | + │ merge_queue │ │ sessions │ | |
401 | + │ repositories │ │ rate limits │ | |
402 | + │ pull_requests │ │ job queues │ | |
403 | + └──────────────┘ └──────────────┘ | |
404 | +``` | |
405 | + | |
406 | +--- | |
407 | + | |
408 | +## Quick Start Checklist | |
409 | + | |
410 | +- [ ] Create PostgreSQL database (Supabase/Neon/Railway) | |
411 | +- [ ] Create Redis database (Upstash/Redis Cloud) | |
412 | +- [ ] Run `bun run db:push` from local machine | |
413 | +- [ ] Push OpenCodeHub code to GitHub | |
414 | +- [ ] Connect repo to CircleCI | |
415 | +- [ ] Create Context `opencodehub-production` with all env vars | |
416 | +- [ ] Verify `.circleci/config.yml` is in repo root | |
417 | +- [ ] Trigger first job manually from CircleCI dashboard | |
418 | +- [ ] Check job logs for "✅ Worker started" | |
419 | +- [ ] Create a test PR and verify merge queue processes it | |
420 | + | |
421 | +--- | |
422 | + | |
423 | +## Next Steps | |
424 | + | |
425 | +- [CircleCI Integration Guide](/guides/circleci-integration/) — For CI/CD pipelines (separate from worker) | |
426 | +- [Deployment Guide](/administration/deployment/) — Full production setup | |
427 | +- [Self-Hosted Runner Guide](circleci-worker.md) — If you prefer self-hosted over Cloud | |
428 | +- [Merge Queue](/features/merge-queue/) — How merge queue works | |
0 | 429 | |