Swadhin Biswas docs(ci): add CircleCI Cloud worker guide with external PostgreSQL/Redis setup · 5/2/2026
5b7dbc4
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
version: 2.1
# ──────────────────────────────────────────────────────────────
# OpenCodeHub Worker — CircleCI Cloud Configuration
# For managed CircleCI Cloud (NOT self-hosted runners)
# Uses external PostgreSQL + Redis (required for Cloud)
# ──────────────────────────────────────────────────────────────
orbs:
# Optional: for better Docker layer caching
docker: circleci/[email protected]
executors:
# Docker executor with Bun + Node.js
bun-medium:
docker:
- image: oven/bun:1
resource_class: medium
bun-large:
docker:
- image: oven/bun:1
resource_class: large
# With PostgreSQL service container (for testing, NOT production data)
bun-with-postgres:
docker:
- image: oven/bun:1
- image: cimg/postgres:16.2
environment:
POSTGRES_USER: opencodehub
POSTGRES_PASSWORD: opencodehub
POSTGRES_DB: opencodehub
resource_class: medium
# ── Reusable Commands ─────────────────────────────────────────
commands:
setup-bun:
description: "Ensure Bun is installed and available"
steps:
- run:
name: Verify Bun
command: |
if ! command -v bun &> /dev/null; then
echo "Installing Bun..."
curl -fsSL https://bun.sh/install | bash
echo 'export PATH="$HOME/.bun/bin:$PATH"' >> $BASH_ENV
fi
source $BASH_ENV
bun --version
install-deps:
description: "Install dependencies with retry"
steps:
- run:
name: Install dependencies
command: |
for i in 1 2 3; do
bun install --frozen-lockfile && exit 0
echo "bun install attempt $i failed, retrying in 10s..."
sleep 10
done
echo "::error::bun install failed after 3 attempts"
exit 1
check-env:
description: "Verify required environment variables"
steps:
- run:
name: Check environment
command: |
: "${DATABASE_URL:?DATABASE_URL is required}"
: "${REDIS_URL:?REDIS_URL is required}"
: "${JWT_SECRET:?JWT_SECRET is required}"
: "${SESSION_SECRET:?SESSION_SECRET is required}"
echo "✅ All required environment variables are set"
# ── Jobs ─────────────────────────────────────────────────────
jobs:
# ──────────────────────────────────────────────
# Continuous Worker (up to 5 hours)
# ──────────────────────────────────────────────
# NOTE: CircleCI Cloud free plan has 5h job limit.
# Performance/Scale plans have longer limits.
# For truly continuous operation, use scheduled pipelines
# to restart every 4.5 hours, OR use self-hosted runner.
# ──────────────────────────────────────────────
worker:
executor: bun-medium
environment:
NODE_ENV: production
# Worker loop interval (ms)
WORKER_INTERVAL: 5000
# Health check port (not exposed externally on Cloud)
WORKER_HEALTH_PORT: 9090
# Stale job timeout (ms)
WORKER_STALE_TIMEOUT: 300000
steps:
- checkout
- setup-bun
- install-deps
- check-env
- run:
name: Start OpenCodeHub Worker
command: |
echo "Starting OpenCodeHub worker..."
echo "Database: ${DATABASE_URL%%@*}@..."
echo "Redis: ${REDIS_URL%%@*}@..."
# Run worker
bun run scripts/worker.ts &
WORKER_PID=$!
# Wait for it to start
sleep 5
# Verify it's running
if kill -0 $WORKER_PID 2>/dev/null; then
echo "✅ Worker started (PID: $WORKER_PID)"
else
echo "❌ Worker failed to start"
exit 1
fi
# Keep job alive until timeout
# CircleCI Cloud will kill the job after the timeout
wait $WORKER_PID
# CircleCI Cloud timeouts:
# Free plan: 1 hour (default), max 5 hours
# Performance: up to 5 hours
# Scale: up to 5 hours
# Server: configurable
no_output_timeout: 5h
# ──────────────────────────────────────────────
# Merge Queue Processor (short burst)
# ──────────────────────────────────────────────
merge-queue:
executor: bun-medium
environment:
NODE_ENV: production
WORKER_INTERVAL: 5000
WORKER_STALE_TIMEOUT: 300000
steps:
- checkout
- setup-bun
- install-deps
- check-env
- run:
name: Process Merge Queue
command: |
echo "Processing merge queue..."
# Run worker for 30 minutes to clear the queue
timeout 1800 bun run scripts/worker.ts || true
echo "Merge queue processing complete."
no_output_timeout: 30m
# ──────────────────────────────────────────────
# Mirror Sync
# ──────────────────────────────────────────────
mirror-sync:
executor: bun-medium
environment:
NODE_ENV: production
steps:
- checkout
- setup-bun
- install-deps
- check-env
- run:
name: Sync Mirrors
command: |
echo "Starting mirror sync..."
# Run for up to 1 hour
timeout 3600 bun run scripts/worker.ts --task=mirror-sync || true
echo "Mirror sync complete."
no_output_timeout: 1h
# ──────────────────────────────────────────────
# Cleanup Tasks
# ──────────────────────────────────────────────
cleanup:
executor: bun-medium
environment:
NODE_ENV: production
steps:
- checkout
- setup-bun
- install-deps
- check-env
- run:
name: Run Cleanup
command: |
echo "Running cleanup tasks..."
timeout 1800 bun run scripts/worker.ts --task=cleanup || true
echo "Cleanup complete."
no_output_timeout: 30m
# ──────────────────────────────────────────────
# Health Check (tests DB connectivity)
# ──────────────────────────────────────────────
health-check:
executor: bun-medium
environment:
NODE_ENV: production
steps:
- checkout
- setup-bun
- install-deps
- check-env
- run:
name: Test Database Connection
command: |
echo "Testing database connection..."
bun -e "
const { getDatabase } = require('./src/db');
try {
const db = getDatabase();
console.log('✅ Database connected successfully');
process.exit(0);
} catch (err) {
console.error('❌ Database connection failed:', err.message);
process.exit(1);
}
"
- run:
name: Test Redis Connection
command: |
echo "Testing Redis connection..."
# Quick Redis ping using bun
bun -e "
const url = new URL(process.env.REDIS_URL || '');
const client = require('redis').createClient({ url: process.env.REDIS_URL });
client.connect()
.then(() => client.ping())
.then(() => { console.log('✅ Redis connected'); client.quit(); process.exit(0); })
.catch((err) => { console.error('❌ Redis failed:', err.message); process.exit(1); });
"
# ── Workflows ────────────────────────────────────────────────
workflows:
version: 2
# ──────────────────────────────────────────────
# OPTION A: Continuous Worker with Auto-Restart
#
# Runs the worker continuously, restarting every
# 4.5 hours (before the 5h Cloud timeout).
# Best for: Production with CircleCI Cloud
# Requires: Performance plan or higher for 5h jobs
# ──────────────────────────────────────────────
# Uncomment below to enable continuous worker
#
# worker-continuous:
# jobs:
# - worker:
# filters:
# branches:
# only:
# - main
# # Context with all environment variables
# context: opencodehub-production
# ──────────────────────────────────────────────
# OPTION B: Scheduled Tasks (Recommended for Free Plan)
#
# Runs individual tasks on a schedule.
# Each job completes quickly (< 30 min).
# Best for: Free plan users, light workloads
# ──────────────────────────────────────────────
# Merge queue: every 15 minutes
scheduled-merge-queue:
triggers:
- schedule:
cron: "*/15 * * * *"
filters:
branches:
only:
- main
jobs:
- merge-queue:
context: opencodehub-production
# Mirror sync: every 2 hours
scheduled-mirror-sync:
triggers:
- schedule:
cron: "0 */2 * * *"
filters:
branches:
only:
- main
jobs:
- mirror-sync:
context: opencodehub-production
# Cleanup: daily at 4 AM UTC
scheduled-cleanup:
triggers:
- schedule:
cron: "0 4 * * *"
filters:
branches:
only:
- main
jobs:
- cleanup:
context: opencodehub-production
# Health check: every 30 minutes
scheduled-health:
triggers:
- schedule:
cron: "*/30 * * * *"
filters:
branches:
only:
- main
jobs:
- health-check:
context: opencodehub-production