swadhin / opencodehub public

Polish mirror sync scheduling and cron observability

swadhinbiswas committed 8 months ago
75ea0df
1 parent c8e1d4f
commit 75ea0df
4 files changed
+228 -8
Showing 4 changed files+228-8
doc/feature_audit.mdsrc/lib/mirror-sync.tssrc/pages/api/cron/mirror-sync.tstests/integration/cron-mirror-sync-route.test.ts
▼doc/feature_audit.md
+1-1
@@ -28,7 +28,7 @@
28
28
| :--- | :---: | :--- |
29
29
| Full Git protocol (SSH/HTTPS) | ✅ | `ssh.ts`, `git-server.ts` fully implemented |
30
30
| Repository hosting | ✅ | Multiple storage adapters (S3, R2, local, GDrive) |
31
-| Repository mirroring | ⚠️ | Background sync cron plus repository mirror config/manual-sync APIs are implemented (`settings/mirror`, `settings/mirror/sync`) with derived health telemetry (`isHealthy`, `isStale`, `lastSyncAgeMinutes`); scheduling/monitoring automation depth remains partial |
31
+| Repository mirroring | ⚠️ | Background sync cron plus repository mirror config/manual-sync APIs are implemented (`settings/mirror`, `settings/mirror/sync`) with derived health telemetry (`isHealthy`, `isStale`, `lastSyncAgeMinutes`); scheduled sync automation now supports stale-only windows, sync-interval/max-repo controls, and batch monitoring metrics (`eligible`, `skipped`, `stale`, `failedRepoIds`, `durationMs`) via `cron/mirror-sync`, while advanced alerting/runbook workflows remain partial |
32
32
| Forks & pull-based workflows | ✅ | `fork.ts` API, UI implemented |
33
33
| Monorepo support | ⚠️ | Basic support, no advanced path scoping |
34
34
| Git LFS | ✅ | `lfs.ts` with batch API (upload/download) |
35
35
▼src/lib/mirror-sync.ts
+90-3
@@ -17,6 +17,24 @@ interface SyncResult {
17
17
error?: string;
18
18
}
19
19
20
+export interface SyncAllMirrorsOptions {
21
+ staleOnly?: boolean;
22
+ minSyncIntervalMinutes?: number;
23
+ maxRepos?: number;
24
+ staleAfterMinutes?: number;
25
+}
26
+
27
+export interface SyncAllMirrorsResult {
28
+ synced: number;
29
+ failed: number;
30
+ total: number;
31
+ eligible: number;
32
+ skipped: number;
33
+ stale: number;
34
+ failedRepoIds: string[];
35
+ durationMs: number;
36
+}
37
+
20
38
async function ensureUpstreamRemote(git: SimpleGit, mirrorUrl: string): Promise<void> {
21
39
const remotes = await git.getRemotes(true);
22
40
const upstream = remotes.find((remote) => remote.name === "upstream");
@@ -115,22 +133,91 @@ export async function syncAllMirrors(): Promise<{ synced: number; failed: number
115
133
const mirrors = await db.query.repositories.findMany({
116
134
where: eq(schema.repositories.isMirror, true),
117
135
});
136
+ return syncAllMirrorsWithOptions(mirrors, {});
137
+}
138
+
139
+async function syncAllMirrorsWithOptions(
140
+ mirrors: Array<typeof schema.repositories.$inferSelect>,
141
+ options: SyncAllMirrorsOptions
142
+): Promise<SyncAllMirrorsResult> {
143
+ const startedAt = Date.now();
144
+ const staleAfterMinutes = options.staleAfterMinutes ?? 24 * 60;
145
+ const minSyncIntervalMinutes = Math.max(0, options.minSyncIntervalMinutes ?? 30);
146
+ const staleOnly = options.staleOnly ?? true;
147
+ const maxRepos = options.maxRepos && options.maxRepos > 0 ? options.maxRepos : mirrors.length;
118
148
119
149
let synced = 0;
120
150
let failed = 0;
151
+ let skipped = 0;
152
+ let stale = 0;
153
+ const failedRepoIds: string[] = [];
154
+
155
+ const nowMs = Date.now();
156
+ const eligibleMirrors = mirrors.filter((mirror) => {
157
+ const lastSyncMs = mirror.lastMirrorSyncAt?.getTime() ?? null;
158
+ const ageMinutes = lastSyncMs === null ? null : Math.max(0, Math.floor((nowMs - lastSyncMs) / 60000));
159
+ const isStale = ageMinutes === null ? true : ageMinutes > staleAfterMinutes;
160
+ if (isStale) stale += 1;
161
+
162
+ if (mirror.mirrorSyncStatus === "syncing") {
163
+ skipped += 1;
164
+ return false;
165
+ }
166
+ if (staleOnly && !isStale) {
167
+ skipped += 1;
168
+ return false;
169
+ }
170
+ if (!isStale && ageMinutes !== null && ageMinutes < minSyncIntervalMinutes) {
171
+ skipped += 1;
172
+ return false;
173
+ }
174
+ return true;
175
+ }).slice(0, maxRepos);
121
176
122
- for (const mirror of mirrors) {
177
+ for (const mirror of eligibleMirrors) {
123
178
const result = await syncMirrorRepository(mirror.id);
124
179
if (result.success) {
125
180
synced++;
126
181
} else {
127
182
failed++;
183
+ failedRepoIds.push(mirror.id);
128
184
}
129
185
}
130
186
131
- logger.info({ synced, failed, total: mirrors.length }, "Mirror sync batch completed");
187
+ const durationMs = Date.now() - startedAt;
188
+ logger.info(
189
+ {
190
+ synced,
191
+ failed,
192
+ total: mirrors.length,
193
+ eligible: eligibleMirrors.length,
194
+ skipped,
195
+ stale,
196
+ durationMs,
197
+ },
198
+ "Mirror sync batch completed"
199
+ );
200
+
201
+ return {
202
+ synced,
203
+ failed,
204
+ total: mirrors.length,
205
+ eligible: eligibleMirrors.length,
206
+ skipped,
207
+ stale,
208
+ failedRepoIds,
209
+ durationMs,
210
+ };
211
+}
132
212
133
- return { synced, failed };
213
+export async function syncAllMirrorsScheduled(
214
+ options: SyncAllMirrorsOptions = {}
215
+): Promise<SyncAllMirrorsResult> {
216
+ const db = getDatabase();
217
+ const mirrors = await db.query.repositories.findMany({
218
+ where: eq(schema.repositories.isMirror, true),
219
+ });
220
+ return syncAllMirrorsWithOptions(mirrors, options);
134
221
}
135
222
136
223
/**
137
224
▼src/pages/api/cron/mirror-sync.ts
+39-4
@@ -4,12 +4,32 @@
4
4
*/
5
5
6
6
import type { APIRoute } from "astro";
7
-import { syncAllMirrors } from "@/lib/mirror-sync";
7
+import { syncAllMirrorsScheduled } from "@/lib/mirror-sync";
8
8
import { logger } from "@/lib/logger";
9
9
10
10
// Secret for cron authentication (prevents unauthorized triggers)
11
11
const CRON_SECRET = process.env.CRON_SECRET;
12
12
13
+function parseBooleanParam(value: string | null, defaultValue: boolean): boolean {
14
+ if (value === null) return defaultValue;
15
+ if (value === "true" || value === "1") return true;
16
+ if (value === "false" || value === "0") return false;
17
+ return defaultValue;
18
+}
19
+
20
+function parseNonNegativeIntParam(value: string | null): number | undefined {
21
+ if (!value) return undefined;
22
+ const parsed = Number.parseInt(value, 10);
23
+ if (!Number.isFinite(parsed) || parsed < 0) return undefined;
24
+ return parsed;
25
+}
26
+
27
+function parsePositiveIntParam(value: string | null): number | undefined {
28
+ const parsed = parseNonNegativeIntParam(value);
29
+ if (parsed === undefined || parsed === 0) return undefined;
30
+ return parsed;
31
+}
32
+
13
33
export const POST: APIRoute = async ({ request }) => {
14
34
// Verify cron secret
15
35
const authHeader = request.headers.get("Authorization");
@@ -22,12 +42,27 @@ export const POST: APIRoute = async ({ request }) => {
22
42
try {
23
43
logger.info("Starting scheduled mirror sync");
24
44
25
- const result = await syncAllMirrors();
45
+ const url = new URL(request.url);
46
+ const staleOnlyParam = url.searchParams.get("staleOnly");
47
+ const minSyncIntervalParam = url.searchParams.get("minSyncIntervalMinutes");
48
+ const maxReposParam = url.searchParams.get("maxRepos");
49
+ const staleAfterParam = url.searchParams.get("staleAfterMinutes");
50
+
51
+ const staleOnly = parseBooleanParam(staleOnlyParam, true);
52
+ const minSyncIntervalMinutes = parseNonNegativeIntParam(minSyncIntervalParam);
53
+ const maxRepos = parsePositiveIntParam(maxReposParam);
54
+ const staleAfterMinutes = parseNonNegativeIntParam(staleAfterParam);
55
+
56
+ const result = await syncAllMirrorsScheduled({
57
+ staleOnly,
58
+ minSyncIntervalMinutes,
59
+ maxRepos,
60
+ staleAfterMinutes,
61
+ });
26
62
27
63
return new Response(JSON.stringify({
28
64
success: true,
29
- synced: result.synced,
30
- failed: result.failed,
65
+ ...result,
31
66
}), {
32
67
status: 200,
33
68
headers: { "Content-Type": "application/json" },
34
69
▼tests/integration/cron-mirror-sync-route.test.ts
+98-0
@@ -0,0 +1,98 @@
1
+import { beforeEach, describe, expect, it, vi } from "vitest";
2
+
3
+const { syncAllMirrorsScheduledMock } = vi.hoisted(() => ({
4
+ syncAllMirrorsScheduledMock: vi.fn(async () => ({
5
+ synced: 2,
6
+ failed: 1,
7
+ total: 5,
8
+ eligible: 3,
9
+ skipped: 2,
10
+ stale: 4,
11
+ failedRepoIds: ["repo-failed-1"],
12
+ durationMs: 123,
13
+ })),
14
+}));
15
+
16
+vi.mock("@/lib/mirror-sync", () => ({
17
+ syncAllMirrorsScheduled: syncAllMirrorsScheduledMock,
18
+}));
19
+
20
+describe("cron mirror sync route", () => {
21
+ beforeEach(() => {
22
+ vi.resetModules();
23
+ vi.clearAllMocks();
24
+ vi.stubEnv("CRON_SECRET", "cron-secret");
25
+ });
26
+
27
+ it("rejects requests with invalid cron secret", async () => {
28
+ const { POST: mirrorCronPost } = await import("@/pages/api/cron/mirror-sync");
29
+ const response = await mirrorCronPost({
30
+ request: new Request("http://localhost/api/cron/mirror-sync", {
31
+ method: "POST",
32
+ headers: { Authorization: "Bearer wrong-secret" },
33
+ }),
34
+ } as any);
35
+
36
+ expect(response.status).toBe(401);
37
+ expect(syncAllMirrorsScheduledMock).not.toHaveBeenCalled();
38
+ });
39
+
40
+ it("runs scheduled sync with query options and returns monitoring metrics", async () => {
41
+ const { POST: mirrorCronPost } = await import("@/pages/api/cron/mirror-sync");
42
+ const response = await mirrorCronPost({
43
+ request: new Request(
44
+ "http://localhost/api/cron/mirror-sync?staleOnly=false&minSyncIntervalMinutes=15&maxRepos=10&staleAfterMinutes=60",
45
+ {
46
+ method: "POST",
47
+ headers: { Authorization: "Bearer cron-secret" },
48
+ }
49
+ ),
50
+ } as any);
51
+
52
+ const body = await response.json();
53
+ expect(response.status).toBe(200);
54
+ expect(body?.success).toBe(true);
55
+ expect(body?.synced).toBe(2);
56
+ expect(body?.eligible).toBe(3);
57
+ expect(body?.failedRepoIds).toEqual(["repo-failed-1"]);
58
+ expect(syncAllMirrorsScheduledMock).toHaveBeenCalledWith({
59
+ staleOnly: false,
60
+ minSyncIntervalMinutes: 15,
61
+ maxRepos: 10,
62
+ staleAfterMinutes: 60,
63
+ });
64
+ });
65
+
66
+ it("supports GET requests", async () => {
67
+ const { GET: mirrorCronGet } = await import("@/pages/api/cron/mirror-sync");
68
+ const response = await mirrorCronGet({
69
+ request: new Request("http://localhost/api/cron/mirror-sync", {
70
+ method: "GET",
71
+ headers: { Authorization: "Bearer cron-secret" },
72
+ }),
73
+ } as any);
74
+
75
+ expect(response.status).toBe(200);
76
+ });
77
+
78
+ it("falls back to safe defaults for invalid query options", async () => {
79
+ const { POST: mirrorCronPost } = await import("@/pages/api/cron/mirror-sync");
80
+ const response = await mirrorCronPost({
81
+ request: new Request(
82
+ "http://localhost/api/cron/mirror-sync?staleOnly=invalid&minSyncIntervalMinutes=-5&maxRepos=0&staleAfterMinutes=abc",
83
+ {
84
+ method: "POST",
85
+ headers: { Authorization: "Bearer cron-secret" },
86
+ }
87
+ ),
88
+ } as any);
89
+
90
+ expect(response.status).toBe(200);
91
+ expect(syncAllMirrorsScheduledMock).toHaveBeenCalledWith({
92
+ staleOnly: true,
93
+ minSyncIntervalMinutes: undefined,
94
+ maxRepos: undefined,
95
+ staleAfterMinutes: undefined,
96
+ });
97
+ });
98
+});
0
99