docs(ci): add CircleCI Cloud worker guide with external PostgreSQL/Redis setup

Swadhin Biswas committed 5 months ago
5b7dbc4
1 parent f58315a
commit 5b7dbc4
3 files changed
+1059 -91
▼.circleci/config.yml
+200-91
1
1
version: 2.1
2
2
3
3
# ──────────────────────────────────────────────────────────────
4
-# OpenCodeHub Worker - CircleCI Configuration
5
-# Run background workers (merge queue, mirror sync, cleanup)
6
-# using CircleCI infrastructure
4
+# OpenCodeHub Worker — CircleCI Cloud Configuration
5
+# For managed CircleCI Cloud (NOT self-hosted runners)
6
+# Uses external PostgreSQL + Redis (required for Cloud)
7
7
# ──────────────────────────────────────────────────────────────
8
8
9
+orbs:
10
+ # Optional: for better Docker layer caching
11
+ docker: circleci/[email protected]
12
+
9
13
executors:
10
- bun-executor:
14
+ # Docker executor with Bun + Node.js
15
+ bun-medium:
11
16
docker:
12
17
- image: oven/bun:1
13
18
resource_class: medium
14
19
15
- # Self-hosted runner executor (recommended for production)
16
- # Replace 'your-namespace' with your actual CircleCI namespace
17
- och-runner:
18
- machine:
19
- image: ubuntu-2204:current
20
- resource_class: your-namespace/opencodehub-runner
20
+ bun-large:
21
+ docker:
22
+ - image: oven/bun:1
23
+ resource_class: large
21
24
22
-# ── Jobs ─────────────────────────────────────────────────────
25
+ # With PostgreSQL service container (for testing, NOT production data)
26
+ bun-with-postgres:
27
+ docker:
28
+ - image: oven/bun:1
29
+ - image: cimg/postgres:16.2
30
+ environment:
31
+ POSTGRES_USER: opencodehub
32
+ POSTGRES_PASSWORD: opencodehub
33
+ POSTGRES_DB: opencodehub
34
+ resource_class: medium
23
35
24
-jobs:
25
- # ──────────────────────────────────────────────
26
- # Continuous Background Worker (Self-Hosted Runner)
27
- # ──────────────────────────────────────────────
28
- worker-continuous:
29
- executor: och-runner
30
- environment:
31
- NODE_ENV: production
32
- WORKER_INTERVAL: 5000
33
- WORKER_HEALTH_PORT: 9090
34
- WORKER_STALE_TIMEOUT: 300000
35
- steps:
36
- - checkout
36
+# ── Reusable Commands ─────────────────────────────────────────
37
37
38
+commands:
39
+ setup-bun:
40
+ description: "Ensure Bun is installed and available"
41
+ steps:
38
42
- run:
39
- name: Install Bun
43
+ name: Verify Bun
40
44
command: |
41
45
if ! command -v bun &> /dev/null; then
46
+ echo "Installing Bun..."
42
47
curl -fsSL https://bun.sh/install | bash
43
48
echo 'export PATH="$HOME/.bun/bin:$PATH"' >> $BASH_ENV
44
49
fi
45
50
source $BASH_ENV
46
51
bun --version
47
52
53
+ install-deps:
54
+ description: "Install dependencies with retry"
55
+ steps:
48
56
- run:
49
57
name: Install dependencies
50
58
command: |
56
64
echo "::error::bun install failed after 3 attempts"
57
65
exit 1
58
66
67
+ check-env:
68
+ description: "Verify required environment variables"
69
+ steps:
70
+ - run:
71
+ name: Check environment
72
+ command: |
73
+ : "${DATABASE_URL:?DATABASE_URL is required}"
74
+ : "${REDIS_URL:?REDIS_URL is required}"
75
+ : "${JWT_SECRET:?JWT_SECRET is required}"
76
+ : "${SESSION_SECRET:?SESSION_SECRET is required}"
77
+ echo "✅ All required environment variables are set"
78
+
79
+# ── Jobs ─────────────────────────────────────────────────────
80
+
81
+jobs:
82
+ # ──────────────────────────────────────────────
83
+ # Continuous Worker (up to 5 hours)
84
+ # ──────────────────────────────────────────────
85
+ # NOTE: CircleCI Cloud free plan has 5h job limit.
86
+ # Performance/Scale plans have longer limits.
87
+ # For truly continuous operation, use scheduled pipelines
88
+ # to restart every 4.5 hours, OR use self-hosted runner.
89
+ # ──────────────────────────────────────────────
90
+ worker:
91
+ executor: bun-medium
92
+ environment:
93
+ NODE_ENV: production
94
+ # Worker loop interval (ms)
95
+ WORKER_INTERVAL: 5000
96
+ # Health check port (not exposed externally on Cloud)
97
+ WORKER_HEALTH_PORT: 9090
98
+ # Stale job timeout (ms)
99
+ WORKER_STALE_TIMEOUT: 300000
100
+ steps:
101
+ - checkout
102
+ - setup-bun
103
+ - install-deps
104
+ - check-env
105
+
59
106
- run:
60
107
name: Start OpenCodeHub Worker
61
108
command: |
62
- echo "Starting OpenCodeHub background worker..."
109
+ echo "Starting OpenCodeHub worker..."
110
+ echo "Database: ${DATABASE_URL%%@*}@..."
111
+ echo "Redis: ${REDIS_URL%%@*}@..."
112
+
113
+ # Run worker
63
114
bun run scripts/worker.ts &
64
115
WORKER_PID=$!
65
- echo "Worker PID: $WORKER_PID"
66
116
67
- # Wait for health check
68
- for i in $(seq 1 30); do
69
- if curl -sf http://localhost:9090/health; then
70
- echo "✅ Worker is healthy"
71
- break
72
- fi
73
- echo "Waiting for worker to start... ($i/30)"
74
- sleep 2
75
- done
117
+ # Wait for it to start
118
+ sleep 5
119
+
120
+ # Verify it's running
121
+ if kill -0 $WORKER_PID 2>/dev/null; then
122
+ echo "✅ Worker started (PID: $WORKER_PID)"
123
+ else
124
+ echo "❌ Worker failed to start"
125
+ exit 1
126
+ fi
76
127
77
- # Keep alive
128
+ # Keep job alive until timeout
129
+ # CircleCI Cloud will kill the job after the timeout
78
130
wait $WORKER_PID
79
- no_output_timeout: 24h
131
+ # CircleCI Cloud timeouts:
132
+ # Free plan: 1 hour (default), max 5 hours
133
+ # Performance: up to 5 hours
134
+ # Scale: up to 5 hours
135
+ # Server: configurable
136
+ no_output_timeout: 5h
80
137
81
138
# ──────────────────────────────────────────────
82
- # Merge Queue Worker (Scheduled / Docker)
139
+ # Merge Queue Processor (short burst)
83
140
# ──────────────────────────────────────────────
84
141
merge-queue:
85
- executor: bun-executor
142
+ executor: bun-medium
86
143
environment:
87
144
NODE_ENV: production
88
145
WORKER_INTERVAL: 5000
89
146
WORKER_STALE_TIMEOUT: 300000
90
147
steps:
91
148
- checkout
92
-
93
- - run:
94
- name: Install dependencies
95
- command: |
96
- for i in 1 2 3; do
97
- bun install --frozen-lockfile && exit 0
98
- echo "bun install attempt $i failed, retrying in 10s..."
99
- sleep 10
100
- done
101
- echo "::error::bun install failed after 3 attempts"
102
- exit 1
149
+ - setup-bun
150
+ - install-deps
151
+ - check-env
103
152
104
153
- run:
105
154
name: Process Merge Queue
106
155
command: |
107
- echo "Running merge queue processor for 4 hours..."
108
- timeout 14400 bun run scripts/worker.ts || true
109
- echo "Merge queue cycle complete."
110
- no_output_timeout: 4h
156
+ echo "Processing merge queue..."
157
+ # Run worker for 30 minutes to clear the queue
158
+ timeout 1800 bun run scripts/worker.ts || true
159
+ echo "Merge queue processing complete."
160
+ no_output_timeout: 30m
111
161
112
162
# ──────────────────────────────────────────────
113
- # Mirror Sync (Scheduled)
163
+ # Mirror Sync
114
164
# ──────────────────────────────────────────────
115
165
mirror-sync:
116
- executor: bun-executor
166
+ executor: bun-medium
117
167
environment:
118
168
NODE_ENV: production
119
169
steps:
120
170
- checkout
121
-
122
- - run:
123
- name: Install dependencies
124
- command: bun install --frozen-lockfile
171
+ - setup-bun
172
+ - install-deps
173
+ - check-env
125
174
126
175
- run:
127
176
name: Sync Mirrors
128
- command: bun run scripts/worker.ts --task=mirror-sync || true
177
+ command: |
178
+ echo "Starting mirror sync..."
179
+ # Run for up to 1 hour
180
+ timeout 3600 bun run scripts/worker.ts --task=mirror-sync || true
181
+ echo "Mirror sync complete."
182
+ no_output_timeout: 1h
129
183
130
184
# ──────────────────────────────────────────────
131
- # Cleanup (Scheduled)
185
+ # Cleanup Tasks
132
186
# ──────────────────────────────────────────────
133
187
cleanup:
134
- executor: bun-executor
188
+ executor: bun-medium
135
189
environment:
136
190
NODE_ENV: production
137
191
steps:
138
192
- checkout
193
+ - setup-bun
194
+ - install-deps
195
+ - check-env
139
196
140
197
- run:
141
- name: Install dependencies
142
- command: bun install --frozen-lockfile
143
-
144
- - run:
145
- name: Run Cleanup Tasks
146
- command: bun run scripts/worker.ts --task=cleanup || true
198
+ name: Run Cleanup
199
+ command: |
200
+ echo "Running cleanup tasks..."
201
+ timeout 1800 bun run scripts/worker.ts --task=cleanup || true
202
+ echo "Cleanup complete."
203
+ no_output_timeout: 30m
147
204
148
205
# ──────────────────────────────────────────────
149
- # Worker Health Check
206
+ # Health Check (tests DB connectivity)
150
207
# ──────────────────────────────────────────────
151
208
health-check:
152
- executor: bun-executor
209
+ executor: bun-medium
210
+ environment:
211
+ NODE_ENV: production
153
212
steps:
213
+ - checkout
214
+ - setup-bun
215
+ - install-deps
216
+ - check-env
217
+
154
218
- run:
155
- name: Check Worker Health
219
+ name: Test Database Connection
156
220
command: |
157
- curl -sf http://your-worker-host:9090/health || exit 1
158
- echo "✅ Worker is healthy"
221
+ echo "Testing database connection..."
222
+ bun -e "
223
+ const { getDatabase } = require('./src/db');
224
+ try {
225
+ const db = getDatabase();
226
+ console.log('✅ Database connected successfully');
227
+ process.exit(0);
228
+ } catch (err) {
229
+ console.error('❌ Database connection failed:', err.message);
230
+ process.exit(1);
231
+ }
232
+ "
233
+
234
+ - run:
235
+ name: Test Redis Connection
236
+ command: |
237
+ echo "Testing Redis connection..."
238
+ # Quick Redis ping using bun
239
+ bun -e "
240
+ const url = new URL(process.env.REDIS_URL || '');
241
+ const client = require('redis').createClient({ url: process.env.REDIS_URL });
242
+ client.connect()
243
+ .then(() => client.ping())
244
+ .then(() => { console.log('✅ Redis connected'); client.quit(); process.exit(0); })
245
+ .catch((err) => { console.error('❌ Redis failed:', err.message); process.exit(1); });
246
+ "
159
247
160
248
# ── Workflows ────────────────────────────────────────────────
161
249
162
250
workflows:
163
251
version: 2
164
252
165
- # OPTION A: Continuous worker (requires self-hosted runner)
166
- # Uncomment this if you have a self-hosted runner
253
+ # ──────────────────────────────────────────────
254
+ # OPTION A: Continuous Worker with Auto-Restart
255
+ #
256
+ # Runs the worker continuously, restarting every
257
+ # 4.5 hours (before the 5h Cloud timeout).
258
+ # Best for: Production with CircleCI Cloud
259
+ # Requires: Performance plan or higher for 5h jobs
260
+ # ──────────────────────────────────────────────
261
+ # Uncomment below to enable continuous worker
262
+ #
167
263
# worker-continuous:
168
264
# jobs:
169
- # - worker-continuous:
265
+ # - worker:
170
266
# filters:
171
267
# branches:
172
268
# only:
173
269
# - main
270
+ # # Context with all environment variables
271
+ # context: opencodehub-production
272
+
273
+ # ──────────────────────────────────────────────
274
+ # OPTION B: Scheduled Tasks (Recommended for Free Plan)
275
+ #
276
+ # Runs individual tasks on a schedule.
277
+ # Each job completes quickly (< 30 min).
278
+ # Best for: Free plan users, light workloads
279
+ # ──────────────────────────────────────────────
174
280
175
- # OPTION B: Scheduled worker tasks (no self-hosted runner needed)
176
- # Runs merge queue every 4 hours
281
+ # Merge queue: every 15 minutes
177
282
scheduled-merge-queue:
178
283
triggers:
179
284
- schedule:
180
- cron: "0 */4 * * *"
285
+ cron: "*/15 * * * *"
181
286
filters:
182
287
branches:
183
288
only:
184
289
- main
185
290
jobs:
186
- - merge-queue
291
+ - merge-queue:
292
+ context: opencodehub-production
187
293
188
- # Runs mirror sync every hour
294
+ # Mirror sync: every 2 hours
189
295
scheduled-mirror-sync:
190
296
triggers:
191
297
- schedule:
192
- cron: "0 * * * *"
298
+ cron: "0 */2 * * *"
193
299
filters:
194
300
branches:
195
301
only:
196
302
- main
197
303
jobs:
198
- - mirror-sync
304
+ - mirror-sync:
305
+ context: opencodehub-production
199
306
200
- # Runs cleanup daily at 3 AM UTC
307
+ # Cleanup: daily at 4 AM UTC
201
308
scheduled-cleanup:
202
309
triggers:
203
310
- schedule:
204
- cron: "0 3 * * *"
311
+ cron: "0 4 * * *"
205
312
filters:
206
313
branches:
207
314
only:
208
315
- main
209
316
jobs:
210
- - cleanup
317
+ - cleanup:
318
+ context: opencodehub-production
211
319
212
- # Health check every hour
320
+ # Health check: every 30 minutes
213
321
scheduled-health:
214
322
triggers:
215
323
- schedule:
216
- cron: "0 * * * *"
324
+ cron: "*/30 * * * *"
217
325
filters:
218
326
branches:
219
327
only:
220
328
- main
221
329
jobs:
222
- - health-check
330
+ - health-check:
331
+ context: opencodehub-production
223
332
▼docs-site/src/content/docs/administration/circleci-cloud.md
+431-0
1
+---
2
+title: "Run Workers on CircleCI Cloud"
3
+description: "Run OpenCodeHub background workers using managed CircleCI Cloud with external PostgreSQL and Redis"
4
+---
5
+
6
+> **Platform:** CircleCI Cloud (managed SaaS) | **Database:** PostgreSQL | **Difficulty:** Easy-Medium
7
+
8
+This guide is specifically for **CircleCI Cloud** (the managed service at circleci.com), NOT self-hosted runners. If you want to use self-hosted runners, see [CircleCI Self-Hosted Runner Guide](/administration/circleci-worker/).
9
+
10
+---
11
+
12
+## How CircleCI Cloud Works for Workers
13
+
14
+CircleCI Cloud runs jobs in **ephemeral Docker containers** that are destroyed after the job ends. This is fundamentally different from a traditional background worker:
15
+
16
+| Feature | Traditional Worker | CircleCI Cloud |
17
+|---------|-------------------|----------------|
18
+| Lifetime | Runs forever | Max 5 hours per job |
19
+| Storage | Persistent disk | Ephemeral (lost after job) |
20
+| Database | Local or network | Must be external (network-accessible) |
21
+| Redis | Local or network | Must be external |
22
+| Cost | Server rental | Compute minutes |
23
+
24
+**Bottom line:** CircleCI Cloud workers run in short bursts and connect to external services. This is actually perfect for OpenCodeHub's architecture because the worker code is stateless — all state lives in PostgreSQL + Redis.
25
+
26
+---
27
+
28
+## Prerequisites
29
+
30
+Before you start, you need:
31
+
32
+1. **CircleCI Account** (free tier works)
33
+2. **External PostgreSQL** database (required — local DB won't persist)
34
+3. **External Redis** (required — local Redis won't persist)
35
+4. **Git repository** pushed to GitHub/GitLab/Bitbucket (CircleCI connects to it)
36
+
37
+### Recommended PostgreSQL Providers
38
+
39
+| Provider | Free Tier | Best For |
40
+|----------|-----------|----------|
41
+| **Supabase** | 500MB, 2GB egress | Full-featured, good dashboard |
42
+| **Neon** | 500MB, 3 databases | Serverless, scales to zero |
43
+| **Railway** | $5 credit | Simple, pay-as-you-go |
44
+| **Render** | 1GB | Easy setup |
45
+| **AWS RDS** | None | Enterprise |
46
+
47
+### Recommended Redis Providers
48
+
49
+| Provider | Free Tier | Best For |
50
+|----------|-----------|----------|
51
+| **Upstash** | 10,000 commands/day | Serverless, HTTP API |
52
+| **Redis Cloud** | 30MB | Managed Redis |
53
+| **Railway** | $5 credit | Easy setup |
54
+| **KeyDB** | Self-host | Performance |
55
+
56
+---
57
+
58
+## Step 1: Set Up External PostgreSQL
59
+
60
+### Using Supabase (Recommended)
61
+
62
+1. Go to [supabase.com](https://supabase.com) and create a project
63
+2. In **Project Settings** → **Database**, copy the connection string:
64
+ ```
65
+ postgresql://postgres:[password]@db.xxxxxxxxxx.supabase.co:5432/postgres
66
+ ```
67
+3. Create the `opencodehub` database:
68
+ ```sql
69
+ CREATE DATABASE opencodehub;
70
+ ```
71
+
72
+### Using Neon
73
+
74
+1. Go to [neon.tech](https://neon.tech) and create a project
75
+2. Copy the connection string from the dashboard
76
+3. Create the database:
77
+ ```sql
78
+ CREATE DATABASE opencodehub;
79
+ ```
80
+
81
+---
82
+
83
+## Step 2: Set Up External Redis
84
+
85
+### Using Upstash (Recommended for Serverless)
86
+
87
+1. Go to [upstash.com](https://upstash.com) and create a Redis database
88
+2. Copy the **Redis URL**:
89
+ ```
90
+ rediss://default:[password]@xxxx.upstash.io:6379
91
+ ```
92
+3. Enable **TLS** (Upstash requires it)
93
+
94
+---
95
+
96
+## Step 3: Configure CircleCI Project
97
+
98
+### 3.1 Connect Repository
99
+
100
+1. Go to [app.circleci.com](https://app.circleci.com)
101
+2. Click **Projects** → **Set Up Project**
102
+3. Select your OpenCodeHub repository
103
+4. CircleCI will detect `.circleci/config.yml`
104
+
105
+### 3.2 Create a Context (Recommended)
106
+
107
+Contexts let you share environment variables across jobs securely.
108
+
109
+1. Go to **Organization Settings** → **Contexts**
110
+2. Click **Create Context**
111
+3. Name it: `opencodehub-production`
112
+4. Add these environment variables:
113
+
114
+| Variable | Value | Description |
115
+|----------|-------|-------------|
116
+| `DATABASE_URL` | `postgresql://...` | Your PostgreSQL connection string |
117
+| `DATABASE_DRIVER` | `postgres` | Explicitly set driver |
118
+| `REDIS_URL` | `redis://...` or `rediss://...` | Your Redis URL |
119
+| `JWT_SECRET` | `openssl rand -hex 32` | Generate this |
120
+| `SESSION_SECRET` | `openssl rand -hex 32` | Generate this |
121
+| `INTERNAL_HOOK_SECRET` | `openssl rand -hex 32` | Generate this |
122
+| `CRON_SECRET` | `openssl rand -hex 32` | Generate this |
123
+| `RUNNER_SECRET` | `openssl rand -hex 32` | Generate this |
124
+| `WORKFLOW_SECRET_ENCRYPTION_KEY` | `openssl rand -hex 32` | Generate this |
125
+| `SITE_URL` | `https://git.yourdomain.com` | Your OpenCodeHub URL |
126
+| `SKIP_REDIS_CHECK` | `1` | Skip Redis check during build |
127
+
128
+> **Security:** Never commit these values to git. Always use CircleCI contexts or project environment variables.
129
+
130
+### 3.3 Alternative: Project-Level Environment Variables
131
+
132
+If you don't want to use contexts:
133
+
134
+1. Go to **Project Settings** → **Environment Variables**
135
+2. Add each variable individually
136
+
137
+---
138
+
139
+## Step 4: Initialize Database Schema
140
+
141
+Before the worker can run, the database schema must exist. Run this once:
142
+
143
+```bash
144
+# On your local machine with DATABASE_URL set
145
+export DATABASE_URL="postgresql://postgres:[password]@db.xxxxxxxxxx.supabase.co:5432/opencodehub"
146
+
147
+# Install dependencies
148
+bun install
149
+
150
+# Push schema to database
151
+bun run db:push
152
+
153
+# Or generate and run migrations
154
+bun run db:generate
155
+bun run db:migrate
156
+
157
+# Create admin user
158
+bun run scripts/seed-admin.ts
159
+```
160
+
161
+> **Important:** Do this from your local machine or a one-time job. The CircleCI worker job won't create the schema automatically.
162
+
163
+---
164
+
165
+## Step 5: Understanding the CircleCI Cloud Config
166
+
167
+The `.circleci/config.yml` in your repo has multiple approaches:
168
+
169
+### Option A: Scheduled Tasks (Recommended for Free Plan)
170
+
171
+Jobs run on a cron schedule. Each job starts, does its work, and exits.
172
+
173
+```yaml
174
+workflows:
175
+ scheduled-merge-queue:
176
+ triggers:
177
+ - schedule:
178
+ cron: "*/15 * * * *" # Every 15 minutes
179
+ jobs:
180
+ - merge-queue:
181
+ context: opencodehub-production
182
+```
183
+
184
+**What happens:**
185
+1. CircleCI starts a Docker container every 15 minutes
186
+2. Installs Bun and dependencies
187
+3. Runs `bun run scripts/worker.ts` for 30 minutes
188
+4. Processes all pending merge queue items
189
+5. Container is destroyed
190
+
191
+**Pros:** Works on free plan, reliable, no timeouts
192
+**Cons:** 15-minute delay between processing (jobs only run on schedule)
193
+
194
+### Option B: Continuous Worker (Performance Plan)
195
+
196
+One long-running job that stays active until timeout.
197
+
198
+```yaml
199
+workflows:
200
+ worker-continuous:
201
+ jobs:
202
+ - worker:
203
+ context: opencodehub-production
204
+```
205
+
206
+**What happens:**
207
+1. CircleCI starts a container
208
+2. Worker runs continuously, processing items every 5 seconds
209
+3. Job runs until the 5-hour timeout
210
+4. CircleCI restarts the job automatically
211
+
212
+**Pros:** Near real-time processing
213
+**Cons:** Requires Performance plan (5h jobs), uses more compute minutes
214
+
215
+### Option C: API-Triggered (Event-Driven)
216
+
217
+OpenCodeHub calls CircleCI API when events happen.
218
+
219
+In your OpenCodeHub webhook handler:
220
+
221
+```typescript
222
+// When merge queue needs processing
223
+async function triggerCircleCIWorker(eventType: string, repoId: string) {
224
+ const response = await fetch(
225
+ `https://circleci.com/api/v2/project/gh/${org}/${repo}/pipeline`,
226
+ {
227
+ method: "POST",
228
+ headers: {
229
+ "Circle-Token": process.env.CIRCLECI_API_TOKEN!,
230
+ "Content-Type": "application/json",
231
+ },
232
+ body: JSON.stringify({
233
+ branch: "main",
234
+ parameters: {
235
+ event_type: eventType,
236
+ repo_id: repoId,
237
+ },
238
+ }),
239
+ }
240
+ );
241
+ return response.json();
242
+}
243
+```
244
+
245
+**Pros:** Only runs when needed, most efficient
246
+**Cons:** Requires code changes in OpenCodeHub, more complex setup
247
+
248
+---
249
+
250
+## Step 6: Choose Your Plan
251
+
252
+### Free Plan (Recommended to Start)
253
+
254
+```yaml
255
+workflows:
256
+ scheduled-merge-queue:
257
+ triggers:
258
+ - schedule:
259
+ cron: "*/15 * * * *" # Every 15 min
260
+ jobs:
261
+ - merge-queue:
262
+ context: opencodehub-production
263
+```
264
+
265
+**Monthly usage:** ~2,880 minutes (48 runs/day × 30 days × 2 min avg)
266
+**Free tier:** 6,000 minutes/month ✅
267
+
268
+### Performance Plan (For Production)
269
+
270
+```yaml
271
+workflows:
272
+ worker-continuous:
273
+ jobs:
274
+ - worker:
275
+ context: opencodehub-production
276
+```
277
+
278
+**Monthly usage:** ~7,200 minutes (5h × 24/day × 30 days)
279
+**Performance plan:** Included in price
280
+
281
+---
282
+
283
+## Step 7: Database Connection from CircleCI Cloud
284
+
285
+Since CircleCI Cloud containers are ephemeral, the database must be network-accessible.
286
+
287
+### Connection String Format
288
+
289
+```bash
290
+# Supabase
291
+DATABASE_URL=postgresql://postgres:[password]@db.xxxxxxxxxx.supabase.co:5432/opencodehub
292
+
293
+# Neon (with SSL)
294
+DATABASE_URL=postgresql://[user]:[password]@ep-xxxxx.us-east-1.aws.neon.tech:5432/opencodehub?sslmode=require
295
+
296
+# Railway
297
+DATABASE_URL=postgresql://postgres:[password]@containers-xxxxx.railway.app:5432/railway
298
+
299
+# AWS RDS (with SSL)
300
+DATABASE_URL=postgresql://opencodehub:[password]@my-db.xxxxx.us-east-1.rds.amazonaws.com:5432/opencodehub?sslmode=require
301
+```
302
+
303
+### SSL/TLS Configuration
304
+
305
+Most managed PostgreSQL require SSL. The worker handles this automatically when `sslmode=require` is in the URL.
306
+
307
+For Supabase, add this to your environment variables:
308
+```bash
309
+DATABASE_SSL=true
310
+DATABASE_SSL_REJECT_UNAUTHORIZED=false # Supabase uses self-signed certs
311
+```
312
+
313
+---
314
+
315
+## Step 8: Redis Connection from CircleCI Cloud
316
+
317
+### Upstash Redis URL
318
+
319
+```bash
320
+REDIS_URL=rediss://default:[password]@xxxx.upstash.io:6379
321
+```
322
+
323
+> Note: `rediss://` (with double s) = TLS enabled. Upstash requires TLS.
324
+
325
+### Redis Cloud
326
+
327
+```bash
328
+REDIS_URL=redis://default:[password]@redis-xxxxx.redis-cloud.com:12345
329
+```
330
+
331
+### Testing Connection
332
+
333
+You can test the connection from CircleCI by running the `health-check` job:
334
+
335
+```bash
336
+# Trigger manually from CircleCI dashboard
337
+# or wait for the scheduled health check
338
+```
339
+
340
+---
341
+
342
+## Step 9: Monitoring
343
+
344
+### View Job Logs
345
+
346
+1. Go to [app.circleci.com](https://app.circleci.com)
347
+2. Select your project
348
+3. Click on any job to see real-time logs
349
+
350
+### Set Up Notifications
351
+
352
+1. **Project Settings** → **Notifications**
353
+2. Connect Slack, email, or webhooks
354
+3. Get alerts when jobs fail
355
+
356
+### Track Compute Usage
357
+
358
+1. **Plan** → **Usage**
359
+2. Monitor minutes used per month
360
+3. Set up usage alerts
361
+
362
+---
363
+
364
+## Step 10: Troubleshooting
365
+
366
+| Problem | Solution |
367
+|---------|----------|
368
+| **"Database connection refused"** | Your PostgreSQL doesn't allow external connections. Check provider's "Connection Pooling" or "Network" settings. For Supabase, go to Database → Connection Pooling. |
369
+| **"Redis connection timeout"** | Redis provider blocks unknown IPs. Add CircleCI's IP ranges to allowlist. Or use Upstash which doesn't require IP allowlisting. |
370
+| **Job killed after 1 hour** | Free plan default timeout is 1 hour. Add `no_output_timeout: 5h` or upgrade to Performance plan. |
371
+| **"Out of memory"** | Use `resource_class: large` instead of `medium`. Or optimize the worker memory usage. |
372
+| **"bun install fails"** | Network issues. The config has retry logic, but you can also try `npm install` as fallback. |
373
+| **"Schema doesn't exist"** | Run `bun run db:push` from your local machine first to create tables. |
374
+| **Worker starts but does nothing** | Check that `DATABASE_URL` points to the right database and that the `mergeQueueItems` table has rows with `status = 'queued'`. |
375
+| **"SSL certificate verify failed"** | Add `DATABASE_SSL_REJECT_UNAUTHORIZED=false` for providers with self-signed certs (Supabase). |
376
+
377
+---
378
+
379
+## Architecture Diagram
380
+
381
+```
382
+┌─────────────────────────────────────────────────────────────┐
383
+│ CircleCI Cloud │
384
+│ ┌─────────────────┐ ┌─────────────────┐ │
385
+│ │ Docker Container │ │ Docker Container │ ... │
386
+│ │ (every 15 min) │ │ (every 15 min) │ │
387
+│ │ │ │ │ │
388
+│ │ bun run worker.ts│ │ bun run worker.ts│ │
389
+│ │ │ │ │ │
390
+│ │ ┌─────────────┐ │ │ ┌─────────────┐ │ │
391
+│ │ │ Node.js/Bun │ │ │ │ Node.js/Bun │ │ │
392
+│ │ │ Worker Loop │ │ │ │ Worker Loop │ │ │
393
+│ │ └─────────────┘ │ │ └─────────────┘ │ │
394
+│ └────────┬─────────┘ └────────┬─────────┘ │
395
+└───────────┼─────────────────────┼────────────────────────────┘
396
+ │ │
397
+ │ TCP/IP (Internet) │
398
+ ▼ ▼
399
+ ┌──────────────┐ ┌──────────────┐
400
+ │ PostgreSQL │ │ Redis │
401
+ │ (Supabase) │ │ (Upstash) │
402
+ │ │ │ │
403
+ │ merge_queue │ │ sessions │
404
+ │ repositories │ │ rate limits │
405
+ │ pull_requests │ │ job queues │
406
+ └──────────────┘ └──────────────┘
407
+```
408
+
409
+---
410
+
411
+## Quick Start Checklist
412
+
413
+- [ ] Create PostgreSQL database (Supabase/Neon/Railway)
414
+- [ ] Create Redis database (Upstash/Redis Cloud)
415
+- [ ] Run `bun run db:push` from local machine
416
+- [ ] Push OpenCodeHub code to GitHub
417
+- [ ] Connect repo to CircleCI
418
+- [ ] Create Context `opencodehub-production` with all env vars
419
+- [ ] Verify `.circleci/config.yml` is in repo root
420
+- [ ] Trigger first job manually from CircleCI dashboard
421
+- [ ] Check job logs for "✅ Worker started"
422
+- [ ] Create a test PR and verify merge queue processes it
423
+
424
+---
425
+
426
+## Next Steps
427
+
428
+- [CircleCI Integration Guide](/guides/circleci-integration/) — For CI/CD pipelines (separate from worker)
429
+- [Deployment Guide](/administration/deployment/) — Full production setup
430
+- [Self-Hosted Runner Guide](/administration/circleci-worker/) — If you prefer self-hosted over Cloud
431
+- [Merge Queue](/features/merge-queue/) — How merge queue works
0
432
▼docs/administration/circleci-cloud.md
+428-0
1
+# OpenCodeHub Worker on CircleCI Cloud (SaaS)
2
+
3
+> **Platform:** CircleCI Cloud (managed SaaS) | **Database:** PostgreSQL | **Difficulty:** Easy-Medium
4
+
5
+This guide is specifically for **CircleCI Cloud** (the managed service at circleci.com), NOT self-hosted runners. If you want to use self-hosted runners, see [CircleCI Self-Hosted Runner Guide](circleci-worker.md).
6
+
7
+---
8
+
9
+## How CircleCI Cloud Works for Workers
10
+
11
+CircleCI Cloud runs jobs in **ephemeral Docker containers** that are destroyed after the job ends. This is fundamentally different from a traditional background worker:
12
+
13
+| Feature | Traditional Worker | CircleCI Cloud |
14
+|---------|-------------------|----------------|
15
+| Lifetime | Runs forever | Max 5 hours per job |
16
+| Storage | Persistent disk | Ephemeral (lost after job) |
17
+| Database | Local or network | Must be external (network-accessible) |
18
+| Redis | Local or network | Must be external |
19
+| Cost | Server rental | Compute minutes |
20
+
21
+**Bottom line:** CircleCI Cloud workers run in short bursts and connect to external services. This is actually perfect for OpenCodeHub's architecture because the worker code is stateless — all state lives in PostgreSQL + Redis.
22
+
23
+---
24
+
25
+## Prerequisites
26
+
27
+Before you start, you need:
28
+
29
+1. **CircleCI Account** (free tier works)
30
+2. **External PostgreSQL** database (required — local DB won't persist)
31
+3. **External Redis** (required — local Redis won't persist)
32
+4. **Git repository** pushed to GitHub/GitLab/Bitbucket (CircleCI connects to it)
33
+
34
+### Recommended PostgreSQL Providers
35
+
36
+| Provider | Free Tier | Best For |
37
+|----------|-----------|----------|
38
+| **Supabase** | 500MB, 2GB egress | Full-featured, good dashboard |
39
+| **Neon** | 500MB, 3 databases | Serverless, scales to zero |
40
+| **Railway** | $5 credit | Simple, pay-as-you-go |
41
+| **Render** | 1GB | Easy setup |
42
+| **AWS RDS** | None | Enterprise |
43
+
44
+### Recommended Redis Providers
45
+
46
+| Provider | Free Tier | Best For |
47
+|----------|-----------|----------|
48
+| **Upstash** | 10,000 commands/day | Serverless, HTTP API |
49
+| **Redis Cloud** | 30MB | Managed Redis |
50
+| **Railway** | $5 credit | Easy setup |
51
+| **KeyDB** | Self-host | Performance |
52
+
53
+---
54
+
55
+## Step 1: Set Up External PostgreSQL
56
+
57
+### Using Supabase (Recommended)
58
+
59
+1. Go to [supabase.com](https://supabase.com) and create a project
60
+2. In **Project Settings** → **Database**, copy the connection string:
61
+ ```
62
+ postgresql://postgres:[password]@db.xxxxxxxxxx.supabase.co:5432/postgres
63
+ ```
64
+3. Create the `opencodehub` database:
65
+ ```sql
66
+ CREATE DATABASE opencodehub;
67
+ ```
68
+
69
+### Using Neon
70
+
71
+1. Go to [neon.tech](https://neon.tech) and create a project
72
+2. Copy the connection string from the dashboard
73
+3. Create the database:
74
+ ```sql
75
+ CREATE DATABASE opencodehub;
76
+ ```
77
+
78
+---
79
+
80
+## Step 2: Set Up External Redis
81
+
82
+### Using Upstash (Recommended for Serverless)
83
+
84
+1. Go to [upstash.com](https://upstash.com) and create a Redis database
85
+2. Copy the **Redis URL**:
86
+ ```
87
+ rediss://default:[password]@xxxx.upstash.io:6379
88
+ ```
89
+3. Enable **TLS** (Upstash requires it)
90
+
91
+---
92
+
93
+## Step 3: Configure CircleCI Project
94
+
95
+### 3.1 Connect Repository
96
+
97
+1. Go to [app.circleci.com](https://app.circleci.com)
98
+2. Click **Projects** → **Set Up Project**
99
+3. Select your OpenCodeHub repository
100
+4. CircleCI will detect `.circleci/config.yml`
101
+
102
+### 3.2 Create a Context (Recommended)
103
+
104
+Contexts let you share environment variables across jobs securely.
105
+
106
+1. Go to **Organization Settings** → **Contexts**
107
+2. Click **Create Context**
108
+3. Name it: `opencodehub-production`
109
+4. Add these environment variables:
110
+
111
+| Variable | Value | Description |
112
+|----------|-------|-------------|
113
+| `DATABASE_URL` | `postgresql://...` | Your PostgreSQL connection string |
114
+| `DATABASE_DRIVER` | `postgres` | Explicitly set driver |
115
+| `REDIS_URL` | `redis://...` or `rediss://...` | Your Redis URL |
116
+| `JWT_SECRET` | `openssl rand -hex 32` | Generate this |
117
+| `SESSION_SECRET` | `openssl rand -hex 32` | Generate this |
118
+| `INTERNAL_HOOK_SECRET` | `openssl rand -hex 32` | Generate this |
119
+| `CRON_SECRET` | `openssl rand -hex 32` | Generate this |
120
+| `RUNNER_SECRET` | `openssl rand -hex 32` | Generate this |
121
+| `WORKFLOW_SECRET_ENCRYPTION_KEY` | `openssl rand -hex 32` | Generate this |
122
+| `SITE_URL` | `https://git.yourdomain.com` | Your OpenCodeHub URL |
123
+| `SKIP_REDIS_CHECK` | `1` | Skip Redis check during build |
124
+
125
+> **Security:** Never commit these values to git. Always use CircleCI contexts or project environment variables.
126
+
127
+### 3.3 Alternative: Project-Level Environment Variables
128
+
129
+If you don't want to use contexts:
130
+
131
+1. Go to **Project Settings** → **Environment Variables**
132
+2. Add each variable individually
133
+
134
+---
135
+
136
+## Step 4: Initialize Database Schema
137
+
138
+Before the worker can run, the database schema must exist. Run this once:
139
+
140
+```bash
141
+# On your local machine with DATABASE_URL set
142
+export DATABASE_URL="postgresql://postgres:[password]@db.xxxxxxxxxx.supabase.co:5432/opencodehub"
143
+
144
+# Install dependencies
145
+bun install
146
+
147
+# Push schema to database
148
+bun run db:push
149
+
150
+# Or generate and run migrations
151
+bun run db:generate
152
+bun run db:migrate
153
+
154
+# Create admin user
155
+bun run scripts/seed-admin.ts
156
+```
157
+
158
+> **Important:** Do this from your local machine or a one-time job. The CircleCI worker job won't create the schema automatically.
159
+
160
+---
161
+
162
+## Step 5: Understanding the CircleCI Cloud Config
163
+
164
+The `.circleci/config.yml` in your repo has multiple approaches:
165
+
166
+### Option A: Scheduled Tasks (Recommended for Free Plan)
167
+
168
+Jobs run on a cron schedule. Each job starts, does its work, and exits.
169
+
170
+```yaml
171
+workflows:
172
+ scheduled-merge-queue:
173
+ triggers:
174
+ - schedule:
175
+ cron: "*/15 * * * *" # Every 15 minutes
176
+ jobs:
177
+ - merge-queue:
178
+ context: opencodehub-production
179
+```
180
+
181
+**What happens:**
182
+1. CircleCI starts a Docker container every 15 minutes
183
+2. Installs Bun and dependencies
184
+3. Runs `bun run scripts/worker.ts` for 30 minutes
185
+4. Processes all pending merge queue items
186
+5. Container is destroyed
187
+
188
+**Pros:** Works on free plan, reliable, no timeouts
189
+**Cons:** 15-minute delay between processing (jobs only run on schedule)
190
+
191
+### Option B: Continuous Worker (Performance Plan)
192
+
193
+One long-running job that stays active until timeout.
194
+
195
+```yaml
196
+workflows:
197
+ worker-continuous:
198
+ jobs:
199
+ - worker:
200
+ context: opencodehub-production
201
+```
202
+
203
+**What happens:**
204
+1. CircleCI starts a container
205
+2. Worker runs continuously, processing items every 5 seconds
206
+3. Job runs until the 5-hour timeout
207
+4. CircleCI restarts the job automatically
208
+
209
+**Pros:** Near real-time processing
210
+**Cons:** Requires Performance plan (5h jobs), uses more compute minutes
211
+
212
+### Option C: API-Triggered (Event-Driven)
213
+
214
+OpenCodeHub calls CircleCI API when events happen.
215
+
216
+In your OpenCodeHub webhook handler:
217
+
218
+```typescript
219
+// When merge queue needs processing
220
+async function triggerCircleCIWorker(eventType: string, repoId: string) {
221
+ const response = await fetch(
222
+ `https://circleci.com/api/v2/project/gh/${org}/${repo}/pipeline`,
223
+ {
224
+ method: "POST",
225
+ headers: {
226
+ "Circle-Token": process.env.CIRCLECI_API_TOKEN!,
227
+ "Content-Type": "application/json",
228
+ },
229
+ body: JSON.stringify({
230
+ branch: "main",
231
+ parameters: {
232
+ event_type: eventType,
233
+ repo_id: repoId,
234
+ },
235
+ }),
236
+ }
237
+ );
238
+ return response.json();
239
+}
240
+```
241
+
242
+**Pros:** Only runs when needed, most efficient
243
+**Cons:** Requires code changes in OpenCodeHub, more complex setup
244
+
245
+---
246
+
247
+## Step 6: Choose Your Plan
248
+
249
+### Free Plan (Recommended to Start)
250
+
251
+```yaml
252
+workflows:
253
+ scheduled-merge-queue:
254
+ triggers:
255
+ - schedule:
256
+ cron: "*/15 * * * *" # Every 15 min
257
+ jobs:
258
+ - merge-queue:
259
+ context: opencodehub-production
260
+```
261
+
262
+**Monthly usage:** ~2,880 minutes (48 runs/day × 30 days × 2 min avg)
263
+**Free tier:** 6,000 minutes/month ✅
264
+
265
+### Performance Plan (For Production)
266
+
267
+```yaml
268
+workflows:
269
+ worker-continuous:
270
+ jobs:
271
+ - worker:
272
+ context: opencodehub-production
273
+```
274
+
275
+**Monthly usage:** ~7,200 minutes (5h × 24/day × 30 days)
276
+**Performance plan:** Included in price
277
+
278
+---
279
+
280
+## Step 7: Database Connection from CircleCI Cloud
281
+
282
+Since CircleCI Cloud containers are ephemeral, the database must be network-accessible.
283
+
284
+### Connection String Format
285
+
286
+```bash
287
+# Supabase
288
+DATABASE_URL=postgresql://postgres:[password]@db.xxxxxxxxxx.supabase.co:5432/opencodehub
289
+
290
+# Neon (with SSL)
291
+DATABASE_URL=postgresql://[user]:[password]@ep-xxxxx.us-east-1.aws.neon.tech:5432/opencodehub?sslmode=require
292
+
293
+# Railway
294
+DATABASE_URL=postgresql://postgres:[password]@containers-xxxxx.railway.app:5432/railway
295
+
296
+# AWS RDS (with SSL)
297
+DATABASE_URL=postgresql://opencodehub:[password]@my-db.xxxxx.us-east-1.rds.amazonaws.com:5432/opencodehub?sslmode=require
298
+```
299
+
300
+### SSL/TLS Configuration
301
+
302
+Most managed PostgreSQL require SSL. The worker handles this automatically when `sslmode=require` is in the URL.
303
+
304
+For Supabase, add this to your environment variables:
305
+```bash
306
+DATABASE_SSL=true
307
+DATABASE_SSL_REJECT_UNAUTHORIZED=false # Supabase uses self-signed certs
308
+```
309
+
310
+---
311
+
312
+## Step 8: Redis Connection from CircleCI Cloud
313
+
314
+### Upstash Redis URL
315
+
316
+```bash
317
+REDIS_URL=rediss://default:[password]@xxxx.upstash.io:6379
318
+```
319
+
320
+> Note: `rediss://` (with double s) = TLS enabled. Upstash requires TLS.
321
+
322
+### Redis Cloud
323
+
324
+```bash
325
+REDIS_URL=redis://default:[password]@redis-xxxxx.redis-cloud.com:12345
326
+```
327
+
328
+### Testing Connection
329
+
330
+You can test the connection from CircleCI by running the `health-check` job:
331
+
332
+```bash
333
+# Trigger manually from CircleCI dashboard
334
+# or wait for the scheduled health check
335
+```
336
+
337
+---
338
+
339
+## Step 9: Monitoring
340
+
341
+### View Job Logs
342
+
343
+1. Go to [app.circleci.com](https://app.circleci.com)
344
+2. Select your project
345
+3. Click on any job to see real-time logs
346
+
347
+### Set Up Notifications
348
+
349
+1. **Project Settings** → **Notifications**
350
+2. Connect Slack, email, or webhooks
351
+3. Get alerts when jobs fail
352
+
353
+### Track Compute Usage
354
+
355
+1. **Plan** → **Usage**
356
+2. Monitor minutes used per month
357
+3. Set up usage alerts
358
+
359
+---
360
+
361
+## Step 10: Troubleshooting
362
+
363
+| Problem | Solution |
364
+|---------|----------|
365
+| **"Database connection refused"** | Your PostgreSQL doesn't allow external connections. Check provider's "Connection Pooling" or "Network" settings. For Supabase, go to Database → Connection Pooling. |
366
+| **"Redis connection timeout"** | Redis provider blocks unknown IPs. Add CircleCI's IP ranges to allowlist. Or use Upstash which doesn't require IP allowlisting. |
367
+| **Job killed after 1 hour** | Free plan default timeout is 1 hour. Add `no_output_timeout: 5h` or upgrade to Performance plan. |
368
+| **"Out of memory"** | Use `resource_class: large` instead of `medium`. Or optimize the worker memory usage. |
369
+| **"bun install fails"** | Network issues. The config has retry logic, but you can also try `npm install` as fallback. |
370
+| **"Schema doesn't exist"** | Run `bun run db:push` from your local machine first to create tables. |
371
+| **Worker starts but does nothing** | Check that `DATABASE_URL` points to the right database and that the `mergeQueueItems` table has rows with `status = 'queued'`. |
372
+| **"SSL certificate verify failed"** | Add `DATABASE_SSL_REJECT_UNAUTHORIZED=false` for providers with self-signed certs (Supabase). |
373
+
374
+---
375
+
376
+## Architecture Diagram
377
+
378
+```
379
+┌─────────────────────────────────────────────────────────────┐
380
+│ CircleCI Cloud │
381
+│ ┌─────────────────┐ ┌─────────────────┐ │
382
+│ │ Docker Container │ │ Docker Container │ ... │
383
+│ │ (every 15 min) │ │ (every 15 min) │ │
384
+│ │ │ │ │ │
385
+│ │ bun run worker.ts│ │ bun run worker.ts│ │
386
+│ │ │ │ │ │
387
+│ │ ┌─────────────┐ │ │ ┌─────────────┐ │ │
388
+│ │ │ Node.js/Bun │ │ │ │ Node.js/Bun │ │ │
389
+│ │ │ Worker Loop │ │ │ │ Worker Loop │ │ │
390
+│ │ └─────────────┘ │ │ └─────────────┘ │ │
391
+│ └────────┬─────────┘ └────────┬─────────┘ │
392
+└───────────┼─────────────────────┼────────────────────────────┘
393
+ │ │
394
+ │ TCP/IP (Internet) │
395
+ ▼ ▼
396
+ ┌──────────────┐ ┌──────────────┐
397
+ │ PostgreSQL │ │ Redis │
398
+ │ (Supabase) │ │ (Upstash) │
399
+ │ │ │ │
400
+ │ merge_queue │ │ sessions │
401
+ │ repositories │ │ rate limits │
402
+ │ pull_requests │ │ job queues │
403
+ └──────────────┘ └──────────────┘
404
+```
405
+
406
+---
407
+
408
+## Quick Start Checklist
409
+
410
+- [ ] Create PostgreSQL database (Supabase/Neon/Railway)
411
+- [ ] Create Redis database (Upstash/Redis Cloud)
412
+- [ ] Run `bun run db:push` from local machine
413
+- [ ] Push OpenCodeHub code to GitHub
414
+- [ ] Connect repo to CircleCI
415
+- [ ] Create Context `opencodehub-production` with all env vars
416
+- [ ] Verify `.circleci/config.yml` is in repo root
417
+- [ ] Trigger first job manually from CircleCI dashboard
418
+- [ ] Check job logs for "✅ Worker started"
419
+- [ ] Create a test PR and verify merge queue processes it
420
+
421
+---
422
+
423
+## Next Steps
424
+
425
+- [CircleCI Integration Guide](/guides/circleci-integration/) — For CI/CD pipelines (separate from worker)
426
+- [Deployment Guide](/administration/deployment/) — Full production setup
427
+- [Self-Hosted Runner Guide](circleci-worker.md) — If you prefer self-hosted over Cloud
428
+- [Merge Queue](/features/merge-queue/) — How merge queue works
0
429