Swadhin Biswas chore(infra): harden deployment, backups, and production docs · 3/6/2026
16d8399
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
# OpenCodeHub Alerting Rules for Prometheus/Alertmanager
#
# Deploy to Prometheus rules directory or configure via Grafana alerting.
groups:
- name: opencodehub.request
rules:
- alert: HighErrorRate
expr: |
sum(rate(http_requests_total{app="opencodehub", code=~"5.."}[5m]))
/ sum(rate(http_requests_total{app="opencodehub"}[5m])) > 0.05
for: 5m
labels:
severity: critical
annotations:
summary: "High HTTP error rate (>5%)"
description: "{{ $value | humanizePercentage }} of requests returning 5xx"
- alert: HighLatencyP99
expr: |
histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{app="opencodehub"}[5m])) by (le)) > 3
for: 5m
labels:
severity: warning
annotations:
summary: "P99 latency exceeds 3 seconds"
description: "P99 latency is {{ $value | humanizeDuration }}"
- alert: HighLatencyP95
expr: |
histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{app="opencodehub"}[5m])) by (le)) > 1
for: 10m
labels:
severity: warning
annotations:
summary: "P95 latency exceeds 1 second"
- name: opencodehub.infrastructure
rules:
- alert: HighCPUUsage
expr: |
rate(process_cpu_seconds_total{app="opencodehub"}[5m]) > 0.8
for: 10m
labels:
severity: warning
annotations:
summary: "High CPU usage detected"
- alert: HighMemoryUsage
expr: |
process_resident_memory_bytes{app="opencodehub"} / 1024 / 1024 > 1500
for: 10m
labels:
severity: warning
annotations:
summary: "Memory usage exceeds 1.5GB"
- alert: DatabaseSlowQueries
expr: |
histogram_quantile(0.95, sum(rate(db_query_duration_seconds_bucket{app="opencodehub"}[5m])) by (le)) > 1
for: 5m
labels:
severity: warning
annotations:
summary: "P95 DB query duration exceeds 1 second"
- alert: DatabaseErrors
expr: |
sum(rate(db_query_errors_total{app="opencodehub"}[5m])) > 0.1
for: 5m
labels:
severity: critical
annotations:
summary: "Database query errors detected"
- name: opencodehub.cicd
rules:
- alert: CIQueueBacklog
expr: ci_queue_depth{app="opencodehub"} > 20
for: 10m
labels:
severity: warning
annotations:
summary: "CI queue depth exceeds 20 jobs"
- alert: NoActiveRunners
expr: active_runners{app="opencodehub"} == 0
for: 5m
labels:
severity: critical
annotations:
summary: "No active CI/CD runners"
- alert: HighJobFailureRate
expr: |
sum(rate(ci_jobs_total{app="opencodehub", status="failed"}[30m]))
/ sum(rate(ci_jobs_total{app="opencodehub"}[30m])) > 0.3
for: 15m
labels:
severity: warning
annotations:
summary: "CI job failure rate exceeds 30%"
- name: opencodehub.auth
rules:
- alert: HighAuthFailureRate
expr: |
sum(rate(auth_attempts_total{app="opencodehub", result="failure"}[5m]))
/ sum(rate(auth_attempts_total{app="opencodehub"}[5m])) > 0.3
for: 5m
labels:
severity: warning
annotations:
summary: "High authentication failure rate (>30%)"
- alert: ExcessiveRateLimiting
expr: |
sum(rate(rate_limit_hits_total{app="opencodehub"}[5m])) > 10
for: 5m
labels:
severity: info
annotations:
summary: "Excessive rate limiting activity"
- name: opencodehub.storage
rules:
- alert: SlowStorageOperations
expr: |
histogram_quantile(0.95, sum(rate(storage_operation_duration_seconds_bucket{app="opencodehub"}[5m])) by (le, adapter)) > 5
for: 10m
labels:
severity: warning
annotations:
summary: "P95 storage operation duration exceeds 5 seconds"
- name: opencodehub.git
rules:
- alert: SlowGitOperations
expr: |
histogram_quantile(0.95, sum(rate(git_operation_duration_seconds_bucket{app="opencodehub"}[5m])) by (le, operation)) > 30
for: 10m
labels:
severity: warning
annotations:
summary: "P95 git operation duration exceeds 30 seconds"