/ opencodehub / .circleci / config.yml
Swadhin Biswas docs(ci): add CircleCI Cloud worker guide with external PostgreSQL/Redis setup · 5/2/2026
5b7dbc4
332 lines · 11.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
version: 2.1

# ──────────────────────────────────────────────────────────────
# OpenCodeHub Worker — CircleCI Cloud Configuration
# For managed CircleCI Cloud (NOT self-hosted runners)
# Uses external PostgreSQL + Redis (required for Cloud)
# ──────────────────────────────────────────────────────────────

orbs:
  # Optional: for better Docker layer caching
  docker: circleci/[email protected]

executors:
  # Docker executor with Bun + Node.js
  bun-medium:
    docker:
      - image: oven/bun:1
    resource_class: medium

  bun-large:
    docker:
      - image: oven/bun:1
    resource_class: large

  # With PostgreSQL service container (for testing, NOT production data)
  bun-with-postgres:
    docker:
      - image: oven/bun:1
      - image: cimg/postgres:16.2
        environment:
          POSTGRES_USER: opencodehub
          POSTGRES_PASSWORD: opencodehub
          POSTGRES_DB: opencodehub
    resource_class: medium

# ── Reusable Commands ─────────────────────────────────────────

commands:
  setup-bun:
    description: "Ensure Bun is installed and available"
    steps:
      - run:
          name: Verify Bun
          command: |
            if ! command -v bun &> /dev/null; then
              echo "Installing Bun..."
              curl -fsSL https://bun.sh/install | bash
              echo 'export PATH="$HOME/.bun/bin:$PATH"' >> $BASH_ENV
            fi
            source $BASH_ENV
            bun --version

  install-deps:
    description: "Install dependencies with retry"
    steps:
      - run:
          name: Install dependencies
          command: |
            for i in 1 2 3; do
              bun install --frozen-lockfile && exit 0
              echo "bun install attempt $i failed, retrying in 10s..."
              sleep 10
            done
            echo "::error::bun install failed after 3 attempts"
            exit 1

  check-env:
    description: "Verify required environment variables"
    steps:
      - run:
          name: Check environment
          command: |
            : "${DATABASE_URL:?DATABASE_URL is required}"
            : "${REDIS_URL:?REDIS_URL is required}"
            : "${JWT_SECRET:?JWT_SECRET is required}"
            : "${SESSION_SECRET:?SESSION_SECRET is required}"
            echo "✅ All required environment variables are set"

# ── Jobs ─────────────────────────────────────────────────────

jobs:
  # ──────────────────────────────────────────────
  # Continuous Worker (up to 5 hours)
  # ──────────────────────────────────────────────
  # NOTE: CircleCI Cloud free plan has 5h job limit.
  # Performance/Scale plans have longer limits.
  # For truly continuous operation, use scheduled pipelines
  # to restart every 4.5 hours, OR use self-hosted runner.
  # ──────────────────────────────────────────────
  worker:
    executor: bun-medium
    environment:
      NODE_ENV: production
      # Worker loop interval (ms)
      WORKER_INTERVAL: 5000
      # Health check port (not exposed externally on Cloud)
      WORKER_HEALTH_PORT: 9090
      # Stale job timeout (ms)
      WORKER_STALE_TIMEOUT: 300000
    steps:
      - checkout
      - setup-bun
      - install-deps
      - check-env

      - run:
          name: Start OpenCodeHub Worker
          command: |
            echo "Starting OpenCodeHub worker..."
            echo "Database: ${DATABASE_URL%%@*}@..."
            echo "Redis: ${REDIS_URL%%@*}@..."
            
            # Run worker
            bun run scripts/worker.ts &
            WORKER_PID=$!
            
            # Wait for it to start
            sleep 5
            
            # Verify it's running
            if kill -0 $WORKER_PID 2>/dev/null; then
              echo "✅ Worker started (PID: $WORKER_PID)"
            else
              echo "❌ Worker failed to start"
              exit 1
            fi
            
            # Keep job alive until timeout
            # CircleCI Cloud will kill the job after the timeout
            wait $WORKER_PID
          # CircleCI Cloud timeouts:
          # Free plan: 1 hour (default), max 5 hours
          # Performance: up to 5 hours
          # Scale: up to 5 hours  
          # Server: configurable
          no_output_timeout: 5h

  # ──────────────────────────────────────────────
  # Merge Queue Processor (short burst)
  # ──────────────────────────────────────────────
  merge-queue:
    executor: bun-medium
    environment:
      NODE_ENV: production
      WORKER_INTERVAL: 5000
      WORKER_STALE_TIMEOUT: 300000
    steps:
      - checkout
      - setup-bun
      - install-deps
      - check-env

      - run:
          name: Process Merge Queue
          command: |
            echo "Processing merge queue..."
            # Run worker for 30 minutes to clear the queue
            timeout 1800 bun run scripts/worker.ts || true
            echo "Merge queue processing complete."
          no_output_timeout: 30m

  # ──────────────────────────────────────────────
  # Mirror Sync
  # ──────────────────────────────────────────────
  mirror-sync:
    executor: bun-medium
    environment:
      NODE_ENV: production
    steps:
      - checkout
      - setup-bun
      - install-deps
      - check-env

      - run:
          name: Sync Mirrors
          command: |
            echo "Starting mirror sync..."
            # Run for up to 1 hour
            timeout 3600 bun run scripts/worker.ts --task=mirror-sync || true
            echo "Mirror sync complete."
          no_output_timeout: 1h

  # ──────────────────────────────────────────────
  # Cleanup Tasks
  # ──────────────────────────────────────────────
  cleanup:
    executor: bun-medium
    environment:
      NODE_ENV: production
    steps:
      - checkout
      - setup-bun
      - install-deps
      - check-env

      - run:
          name: Run Cleanup
          command: |
            echo "Running cleanup tasks..."
            timeout 1800 bun run scripts/worker.ts --task=cleanup || true
            echo "Cleanup complete."
          no_output_timeout: 30m

  # ──────────────────────────────────────────────
  # Health Check (tests DB connectivity)
  # ──────────────────────────────────────────────
  health-check:
    executor: bun-medium
    environment:
      NODE_ENV: production
    steps:
      - checkout
      - setup-bun
      - install-deps
      - check-env

      - run:
          name: Test Database Connection
          command: |
            echo "Testing database connection..."
            bun -e "
              const { getDatabase } = require('./src/db');
              try {
                const db = getDatabase();
                console.log('✅ Database connected successfully');
                process.exit(0);
              } catch (err) {
                console.error('❌ Database connection failed:', err.message);
                process.exit(1);
              }
            "

      - run:
          name: Test Redis Connection
          command: |
            echo "Testing Redis connection..."
            # Quick Redis ping using bun
            bun -e "
              const url = new URL(process.env.REDIS_URL || '');
              const client = require('redis').createClient({ url: process.env.REDIS_URL });
              client.connect()
                .then(() => client.ping())
                .then(() => { console.log('✅ Redis connected'); client.quit(); process.exit(0); })
                .catch((err) => { console.error('❌ Redis failed:', err.message); process.exit(1); });
            "

# ── Workflows ────────────────────────────────────────────────

workflows:
  version: 2

  # ──────────────────────────────────────────────
  # OPTION A: Continuous Worker with Auto-Restart
  # 
  # Runs the worker continuously, restarting every 
  # 4.5 hours (before the 5h Cloud timeout).
  # Best for: Production with CircleCI Cloud
  # Requires: Performance plan or higher for 5h jobs
  # ──────────────────────────────────────────────
  # Uncomment below to enable continuous worker
  # 
  # worker-continuous:
  #   jobs:
  #     - worker:
  #         filters:
  #           branches:
  #             only:
  #               - main
  #         # Context with all environment variables
  #         context: opencodehub-production

  # ──────────────────────────────────────────────
  # OPTION B: Scheduled Tasks (Recommended for Free Plan)
  # 
  # Runs individual tasks on a schedule.
  # Each job completes quickly (< 30 min).
  # Best for: Free plan users, light workloads
  # ──────────────────────────────────────────────

  # Merge queue: every 15 minutes
  scheduled-merge-queue:
    triggers:
      - schedule:
          cron: "*/15 * * * *"
          filters:
            branches:
              only:
                - main
    jobs:
      - merge-queue:
          context: opencodehub-production

  # Mirror sync: every 2 hours
  scheduled-mirror-sync:
    triggers:
      - schedule:
          cron: "0 */2 * * *"
          filters:
            branches:
              only:
                - main
    jobs:
      - mirror-sync:
          context: opencodehub-production

  # Cleanup: daily at 4 AM UTC
  scheduled-cleanup:
    triggers:
      - schedule:
          cron: "0 4 * * *"
          filters:
            branches:
              only:
                - main
    jobs:
      - cleanup:
          context: opencodehub-production

  # Health check: every 30 minutes
  scheduled-health:
    triggers:
      - schedule:
          cron: "*/30 * * * *"
          filters:
            branches:
              only:
                - main
    jobs:
      - health-check:
          context: opencodehub-production