swadhin / opencodehub public

Add repository mirror management APIs and sync hardening

swadhinbiswas committed 8 months ago
2ce335e
1 parent 733b31d
commit 2ce335e
6 files changed
+438 -7
Showing 6 changed files+438-7
doc/feature_audit.mdsrc/lib/mirror-sync.tssrc/lib/openapi.tssrc/pages/api/repos/[owner]/[repo]/settings/mirror.tssrc/pages/api/repos/[owner]/[repo]/settings/mirror/sync.tstests/integration/repo-mirror-settings-route.test.ts
▼doc/feature_audit.md
+1-1
@@ -28,7 +28,7 @@
28
28
| :--- | :---: | :--- |
29
29
| Full Git protocol (SSH/HTTPS) | ✅ | `ssh.ts`, `git-server.ts` fully implemented |
30
30
| Repository hosting | ✅ | Multiple storage adapters (S3, R2, local, GDrive) |
31
-| Repository mirroring | ⚠️ | Schema exists, background sync incomplete |
31
+| Repository mirroring | ⚠️ | Background sync cron plus repository mirror config/manual-sync APIs are implemented (`settings/mirror`, `settings/mirror/sync`); scheduling/monitoring depth remains partial |
32
32
| Forks & pull-based workflows | ✅ | `fork.ts` API, UI implemented |
33
33
| Monorepo support | ⚠️ | Basic support, no advanced path scoping |
34
34
| Git LFS | ✅ | `lfs.ts` with batch API (upload/download) |
35
35
▼src/lib/mirror-sync.ts
+62-6
@@ -17,6 +17,20 @@ interface SyncResult {
17
17
error?: string;
18
18
}
19
19
20
+async function ensureUpstreamRemote(git: SimpleGit, mirrorUrl: string): Promise<void> {
21
+ const remotes = await git.getRemotes(true);
22
+ const upstream = remotes.find((remote) => remote.name === "upstream");
23
+ if (!upstream) {
24
+ await git.addRemote("upstream", mirrorUrl);
25
+ return;
26
+ }
27
+
28
+ const fetchUrl = upstream.refs.fetch;
29
+ if (fetchUrl !== mirrorUrl) {
30
+ await git.remote(["set-url", "upstream", mirrorUrl]);
31
+ }
32
+}
33
+
20
34
/**
21
35
* Sync a single mirrored repository with its upstream
22
36
*/
@@ -46,14 +60,24 @@ export async function syncMirrorRepository(repoId: string): Promise<SyncResult>
46
60
.where(eq(schema.repositories.id, repoId));
47
61
48
62
const git: SimpleGit = simpleGit(repoPath);
63
+ await ensureUpstreamRemote(git, repo.mirrorUrl);
49
64
50
- // Fetch all refs from upstream with prune
65
+ // Mirror upstream refs directly into local heads/tags.
51
66
logger.info({ repoId, mirrorUrl: repo.mirrorUrl }, "Starting mirror sync");
52
67
53
- const fetchResult = await git.fetch(["--all", "--prune", "--tags"]);
68
+ const fetchResult = await git.raw([
69
+ "fetch",
70
+ "upstream",
71
+ "+refs/heads/*:refs/heads/*",
72
+ "+refs/tags/*:refs/tags/*",
73
+ "--prune",
74
+ ]);
54
75
55
- // Count updated refs (simplified - fetch doesn't return detailed info easily)
56
- const refsUpdated = fetchResult.updated?.length || 0;
76
+ // Approximate updated refs from fetch output lines.
77
+ const refsUpdated = fetchResult
78
+ .split("\n")
79
+ .filter((line) => line.includes("->"))
80
+ .length;
57
81
58
82
// Update sync status
59
83
// @ts-expect-error - Drizzle multi-db union type issue
@@ -128,8 +152,7 @@ export async function initializeMirror(repoId: string, mirrorUrl: string): Promi
128
152
try {
129
153
const git: SimpleGit = simpleGit(repoPath);
130
154
131
- // Add upstream remote
132
- await git.addRemote("upstream", mirrorUrl);
155
+ await ensureUpstreamRemote(git, mirrorUrl);
133
156
134
157
// Mark as mirror in DB
135
158
// @ts-expect-error - Drizzle multi-db union type issue
@@ -148,3 +171,36 @@ export async function initializeMirror(repoId: string, mirrorUrl: string): Promi
148
171
return { success: false, refsUpdated: 0, error: errorMessage };
149
172
}
150
173
}
174
+
175
+export async function disableMirror(repoId: string): Promise<{ success: boolean; error?: string }> {
176
+ const db = getDatabase();
177
+ const repo = await db.query.repositories.findFirst({
178
+ where: eq(schema.repositories.id, repoId),
179
+ });
180
+ if (!repo) return { success: false, error: "Repository not found" };
181
+
182
+ const repoPath = path.join(REPOS_BASE_PATH, repo.diskPath);
183
+ try {
184
+ const git: SimpleGit = simpleGit(repoPath);
185
+ const remotes = await git.getRemotes(true);
186
+ if (remotes.some((remote) => remote.name === "upstream")) {
187
+ await git.removeRemote("upstream");
188
+ }
189
+
190
+ // @ts-expect-error - Drizzle multi-db union type issue
191
+ await db.update(schema.repositories)
192
+ .set({
193
+ isMirror: false,
194
+ mirrorUrl: null,
195
+ mirrorSyncStatus: null,
196
+ updatedAt: new Date(),
197
+ })
198
+ .where(eq(schema.repositories.id, repoId));
199
+
200
+ return { success: true };
201
+ } catch (error) {
202
+ const errorMessage = error instanceof Error ? error.message : "Unknown error";
203
+ logger.error({ repoId, error: errorMessage }, "Failed to disable mirror");
204
+ return { success: false, error: errorMessage };
205
+ }
206
+}
151
207
▼src/lib/openapi.ts
+82-0
@@ -257,6 +257,13 @@ export const openApiSpec = {
257
257
},
258
258
required: ["prIds"],
259
259
},
260
+ MirrorConfigRequest: {
261
+ type: "object",
262
+ properties: {
263
+ mirrorUrl: { type: "string", format: "uri" },
264
+ },
265
+ required: ["mirrorUrl"],
266
+ },
260
267
},
261
268
},
262
269
paths: {
@@ -826,6 +833,81 @@ export const openApiSpec = {
826
833
},
827
834
},
828
835
},
836
+ "/repos/{owner}/{repo}/settings/mirror": {
837
+ get: {
838
+ tags: ["Repositories"],
839
+ summary: "Get repository mirror settings",
840
+ security: [{ bearerAuth: [] }],
841
+ parameters: [
842
+ { name: "owner", in: "path", required: true, schema: { type: "string" } },
843
+ { name: "repo", in: "path", required: true, schema: { type: "string" } },
844
+ ],
845
+ responses: {
846
+ 200: { description: "Mirror settings returned" },
847
+ 401: { description: "Unauthorized", content: { "application/json": { schema: { $ref: "#/components/schemas/Error" } } } },
848
+ 404: { description: "Repository not found", content: { "application/json": { schema: { $ref: "#/components/schemas/Error" } } } },
849
+ },
850
+ },
851
+ post: {
852
+ tags: ["Repositories"],
853
+ summary: "Configure repository mirror",
854
+ description: "Configures upstream mirror URL and performs initial synchronization.",
855
+ security: [{ bearerAuth: [] }],
856
+ parameters: [
857
+ { name: "owner", in: "path", required: true, schema: { type: "string" } },
858
+ { name: "repo", in: "path", required: true, schema: { type: "string" } },
859
+ ],
860
+ requestBody: {
861
+ required: true,
862
+ content: {
863
+ "application/json": {
864
+ schema: { $ref: "#/components/schemas/MirrorConfigRequest" },
865
+ },
866
+ },
867
+ },
868
+ responses: {
869
+ 200: { description: "Mirror configured" },
870
+ 400: { description: "Invalid mirror URL or sync failure", content: { "application/json": { schema: { $ref: "#/components/schemas/Error" } } } },
871
+ 401: { description: "Unauthorized", content: { "application/json": { schema: { $ref: "#/components/schemas/Error" } } } },
872
+ 403: { description: "Forbidden", content: { "application/json": { schema: { $ref: "#/components/schemas/Error" } } } },
873
+ 404: { description: "Repository not found", content: { "application/json": { schema: { $ref: "#/components/schemas/Error" } } } },
874
+ },
875
+ },
876
+ delete: {
877
+ tags: ["Repositories"],
878
+ summary: "Disable repository mirror",
879
+ security: [{ bearerAuth: [] }],
880
+ parameters: [
881
+ { name: "owner", in: "path", required: true, schema: { type: "string" } },
882
+ { name: "repo", in: "path", required: true, schema: { type: "string" } },
883
+ ],
884
+ responses: {
885
+ 200: { description: "Mirror disabled" },
886
+ 400: { description: "Disable failed", content: { "application/json": { schema: { $ref: "#/components/schemas/Error" } } } },
887
+ 401: { description: "Unauthorized", content: { "application/json": { schema: { $ref: "#/components/schemas/Error" } } } },
888
+ 403: { description: "Forbidden", content: { "application/json": { schema: { $ref: "#/components/schemas/Error" } } } },
889
+ 404: { description: "Repository not found", content: { "application/json": { schema: { $ref: "#/components/schemas/Error" } } } },
890
+ },
891
+ },
892
+ },
893
+ "/repos/{owner}/{repo}/settings/mirror/sync": {
894
+ post: {
895
+ tags: ["Repositories"],
896
+ summary: "Trigger manual mirror sync",
897
+ security: [{ bearerAuth: [] }],
898
+ parameters: [
899
+ { name: "owner", in: "path", required: true, schema: { type: "string" } },
900
+ { name: "repo", in: "path", required: true, schema: { type: "string" } },
901
+ ],
902
+ responses: {
903
+ 200: { description: "Sync completed" },
904
+ 400: { description: "Sync failed", content: { "application/json": { schema: { $ref: "#/components/schemas/Error" } } } },
905
+ 401: { description: "Unauthorized", content: { "application/json": { schema: { $ref: "#/components/schemas/Error" } } } },
906
+ 403: { description: "Forbidden", content: { "application/json": { schema: { $ref: "#/components/schemas/Error" } } } },
907
+ 404: { description: "Repository not found", content: { "application/json": { schema: { $ref: "#/components/schemas/Error" } } } },
908
+ },
909
+ },
910
+ },
829
911
"/repos/{owner}/{repo}/workflow/states": {
830
912
get: {
831
913
tags: ["Workflow"],
832
914
▼src/pages/api/repos/[owner]/[repo]/settings/mirror.ts
+104-0
@@ -0,0 +1,104 @@
1
+import type { APIRoute } from "astro";
2
+import type { NodePgDatabase } from "drizzle-orm/node-postgres";
3
+import { and, eq } from "drizzle-orm";
4
+import { z } from "zod";
5
+import { getDatabase, schema } from "@/db";
6
+import { getUserFromRequest } from "@/lib/auth";
7
+import { canAdminRepo, canReadRepo } from "@/lib/permissions";
8
+import { badRequest, forbidden, notFound, parseBody, success, unauthorized } from "@/lib/api";
9
+import { withErrorHandler } from "@/lib/errors";
10
+import { disableMirror, initializeMirror } from "@/lib/mirror-sync";
11
+
12
+const configureMirrorSchema = z.object({
13
+ mirrorUrl: z.string().url(),
14
+});
15
+
16
+async function resolveRepository(owner: string, repoName: string) {
17
+ const db = getDatabase() as NodePgDatabase<typeof schema>;
18
+ const ownerUser = await db.query.users.findFirst({
19
+ where: eq(schema.users.username, owner),
20
+ });
21
+ if (!ownerUser) return null;
22
+ return db.query.repositories.findFirst({
23
+ where: and(
24
+ eq(schema.repositories.ownerId, ownerUser.id),
25
+ eq(schema.repositories.name, repoName)
26
+ ),
27
+ });
28
+}
29
+
30
+export const GET: APIRoute = withErrorHandler(async ({ params, request }) => {
31
+ const owner = params.owner;
32
+ const repoName = params.repo;
33
+ if (!owner || !repoName) return badRequest("Missing route parameters");
34
+
35
+ const user = await getUserFromRequest(request);
36
+ if (!user) return unauthorized();
37
+
38
+ const repository = await resolveRepository(owner, repoName);
39
+ if (!repository) return notFound("Repository not found");
40
+
41
+ if (!(await canReadRepo(user.userId, repository, { isAdmin: user.isAdmin }))) {
42
+ return notFound("Repository not found");
43
+ }
44
+
45
+ return success({
46
+ isMirror: repository.isMirror,
47
+ mirrorUrl: repository.mirrorUrl,
48
+ mirrorSyncStatus: repository.mirrorSyncStatus,
49
+ lastMirrorSyncAt: repository.lastMirrorSyncAt,
50
+ });
51
+});
52
+
53
+export const POST: APIRoute = withErrorHandler(async ({ params, request }) => {
54
+ const owner = params.owner;
55
+ const repoName = params.repo;
56
+ if (!owner || !repoName) return badRequest("Missing route parameters");
57
+
58
+ const user = await getUserFromRequest(request);
59
+ if (!user) return unauthorized();
60
+
61
+ const repository = await resolveRepository(owner, repoName);
62
+ if (!repository) return notFound("Repository not found");
63
+
64
+ if (!(await canAdminRepo(user.userId, repository, { isAdmin: user.isAdmin }))) {
65
+ return forbidden();
66
+ }
67
+
68
+ const parsed = await parseBody(request, configureMirrorSchema);
69
+ if ("error" in parsed) return parsed.error;
70
+
71
+ const result = await initializeMirror(repository.id, parsed.data.mirrorUrl);
72
+ if (!result.success) {
73
+ return badRequest(result.error || "Failed to initialize mirror");
74
+ }
75
+
76
+ return success({
77
+ configured: true,
78
+ refsUpdated: result.refsUpdated,
79
+ });
80
+});
81
+
82
+export const DELETE: APIRoute = withErrorHandler(async ({ params, request }) => {
83
+ const owner = params.owner;
84
+ const repoName = params.repo;
85
+ if (!owner || !repoName) return badRequest("Missing route parameters");
86
+
87
+ const user = await getUserFromRequest(request);
88
+ if (!user) return unauthorized();
89
+
90
+ const repository = await resolveRepository(owner, repoName);
91
+ if (!repository) return notFound("Repository not found");
92
+
93
+ if (!(await canAdminRepo(user.userId, repository, { isAdmin: user.isAdmin }))) {
94
+ return forbidden();
95
+ }
96
+
97
+ const result = await disableMirror(repository.id);
98
+ if (!result.success) {
99
+ return badRequest(result.error || "Failed to disable mirror");
100
+ }
101
+
102
+ return success({ configured: false });
103
+});
104
+
0
105
▼src/pages/api/repos/[owner]/[repo]/settings/mirror/sync.ts
+47-0
@@ -0,0 +1,47 @@
1
+import type { APIRoute } from "astro";
2
+import type { NodePgDatabase } from "drizzle-orm/node-postgres";
3
+import { and, eq } from "drizzle-orm";
4
+import { getDatabase, schema } from "@/db";
5
+import { getUserFromRequest } from "@/lib/auth";
6
+import { canWriteRepo } from "@/lib/permissions";
7
+import { badRequest, forbidden, notFound, success, unauthorized } from "@/lib/api";
8
+import { withErrorHandler } from "@/lib/errors";
9
+import { syncMirrorRepository } from "@/lib/mirror-sync";
10
+
11
+async function resolveRepository(owner: string, repoName: string) {
12
+ const db = getDatabase() as NodePgDatabase<typeof schema>;
13
+ const ownerUser = await db.query.users.findFirst({
14
+ where: eq(schema.users.username, owner),
15
+ });
16
+ if (!ownerUser) return null;
17
+ return db.query.repositories.findFirst({
18
+ where: and(
19
+ eq(schema.repositories.ownerId, ownerUser.id),
20
+ eq(schema.repositories.name, repoName)
21
+ ),
22
+ });
23
+}
24
+
25
+export const POST: APIRoute = withErrorHandler(async ({ params, request }) => {
26
+ const owner = params.owner;
27
+ const repoName = params.repo;
28
+ if (!owner || !repoName) return badRequest("Missing route parameters");
29
+
30
+ const user = await getUserFromRequest(request);
31
+ if (!user) return unauthorized();
32
+
33
+ const repository = await resolveRepository(owner, repoName);
34
+ if (!repository) return notFound("Repository not found");
35
+
36
+ if (!(await canWriteRepo(user.userId, repository, { isAdmin: user.isAdmin }))) {
37
+ return forbidden();
38
+ }
39
+
40
+ const result = await syncMirrorRepository(repository.id);
41
+ if (!result.success) {
42
+ return badRequest(result.error || "Mirror sync failed");
43
+ }
44
+
45
+ return success(result);
46
+});
47
+
0
48
▼tests/integration/repo-mirror-settings-route.test.ts
+142-0
@@ -0,0 +1,142 @@
1
+import { beforeEach, describe, expect, it, vi } from "vitest";
2
+
3
+const {
4
+ getUserFromRequestMock,
5
+ canReadRepoMock,
6
+ canAdminRepoMock,
7
+ canWriteRepoMock,
8
+ initializeMirrorMock,
9
+ disableMirrorMock,
10
+ syncMirrorRepositoryMock,
11
+ fakeSchema,
12
+} = vi.hoisted(() => ({
13
+ getUserFromRequestMock: vi.fn(async () => ({ userId: "user-1", isAdmin: false })),
14
+ canReadRepoMock: vi.fn(async () => true),
15
+ canAdminRepoMock: vi.fn(async () => true),
16
+ canWriteRepoMock: vi.fn(async () => true),
17
+ initializeMirrorMock: vi.fn(async () => ({ success: true, refsUpdated: 3 })),
18
+ disableMirrorMock: vi.fn(async () => ({ success: true })),
19
+ syncMirrorRepositoryMock: vi.fn(async () => ({ success: true, refsUpdated: 2 })),
20
+ fakeSchema: {
21
+ users: { username: {} },
22
+ repositories: { ownerId: {}, name: {}, id: {} },
23
+ } as any,
24
+}));
25
+
26
+let mockDb: any;
27
+
28
+vi.mock("@/db", () => ({
29
+ getDatabase: () => mockDb,
30
+ schema: fakeSchema,
31
+}));
32
+
33
+vi.mock("@/lib/auth", () => ({
34
+ getUserFromRequest: getUserFromRequestMock,
35
+}));
36
+
37
+vi.mock("@/lib/permissions", () => ({
38
+ canReadRepo: canReadRepoMock,
39
+ canAdminRepo: canAdminRepoMock,
40
+ canWriteRepo: canWriteRepoMock,
41
+}));
42
+
43
+vi.mock("@/lib/mirror-sync", () => ({
44
+ initializeMirror: initializeMirrorMock,
45
+ disableMirror: disableMirrorMock,
46
+ syncMirrorRepository: syncMirrorRepositoryMock,
47
+}));
48
+
49
+import { GET as mirrorGet, POST as mirrorPost, DELETE as mirrorDelete } from "@/pages/api/repos/[owner]/[repo]/settings/mirror";
50
+import { POST as mirrorSyncPost } from "@/pages/api/repos/[owner]/[repo]/settings/mirror/sync";
51
+
52
+function makeDb() {
53
+ return {
54
+ query: {
55
+ users: {
56
+ findFirst: vi.fn(async () => ({ id: "owner-1" })),
57
+ },
58
+ repositories: {
59
+ findFirst: vi.fn(async () => ({
60
+ id: "repo-1",
61
+ ownerId: "owner-1",
62
+ name: "demo",
63
+ isMirror: true,
64
+ mirrorUrl: "https://example.com/upstream.git",
65
+ mirrorSyncStatus: "success",
66
+ lastMirrorSyncAt: new Date("2026-02-19T00:00:00Z"),
67
+ })),
68
+ },
69
+ },
70
+ };
71
+}
72
+
73
+async function readJson(response: Response): Promise<any> {
74
+ return response.json();
75
+}
76
+
77
+describe("repository mirror settings routes", () => {
78
+ beforeEach(() => {
79
+ mockDb = makeDb();
80
+ getUserFromRequestMock.mockResolvedValue({ userId: "user-1", isAdmin: false });
81
+ canReadRepoMock.mockResolvedValue(true);
82
+ canAdminRepoMock.mockResolvedValue(true);
83
+ canWriteRepoMock.mockResolvedValue(true);
84
+ initializeMirrorMock.mockResolvedValue({ success: true, refsUpdated: 3 });
85
+ disableMirrorMock.mockResolvedValue({ success: true });
86
+ syncMirrorRepositoryMock.mockResolvedValue({ success: true, refsUpdated: 2 });
87
+ });
88
+
89
+ it("returns mirror settings for readers", async () => {
90
+ const response = await mirrorGet({
91
+ params: { owner: "owner-1", repo: "demo" },
92
+ request: new Request("http://localhost/api/repos/owner-1/demo/settings/mirror"),
93
+ } as any);
94
+
95
+ const body = await readJson(response);
96
+ expect(response.status).toBe(200);
97
+ expect(body?.data?.isMirror).toBe(true);
98
+ });
99
+
100
+ it("configures mirror for repo admins", async () => {
101
+ const response = await mirrorPost({
102
+ params: { owner: "owner-1", repo: "demo" },
103
+ request: new Request("http://localhost/api/repos/owner-1/demo/settings/mirror", {
104
+ method: "POST",
105
+ headers: { "content-type": "application/json" },
106
+ body: JSON.stringify({ mirrorUrl: "https://example.com/new-upstream.git" }),
107
+ }),
108
+ } as any);
109
+
110
+ const body = await readJson(response);
111
+ expect(response.status).toBe(200);
112
+ expect(body?.data?.configured).toBe(true);
113
+ expect(initializeMirrorMock).toHaveBeenCalledWith("repo-1", "https://example.com/new-upstream.git");
114
+ });
115
+
116
+ it("denies manual mirror sync without write access", async () => {
117
+ canWriteRepoMock.mockResolvedValue(false);
118
+ const response = await mirrorSyncPost({
119
+ params: { owner: "owner-1", repo: "demo" },
120
+ request: new Request("http://localhost/api/repos/owner-1/demo/settings/mirror/sync", {
121
+ method: "POST",
122
+ }),
123
+ } as any);
124
+
125
+ expect(response.status).toBe(403);
126
+ });
127
+
128
+ it("disables mirror for admins", async () => {
129
+ const response = await mirrorDelete({
130
+ params: { owner: "owner-1", repo: "demo" },
131
+ request: new Request("http://localhost/api/repos/owner-1/demo/settings/mirror", {
132
+ method: "DELETE",
133
+ }),
134
+ } as any);
135
+
136
+ const body = await readJson(response);
137
+ expect(response.status).toBe(200);
138
+ expect(body?.data?.configured).toBe(false);
139
+ expect(disableMirrorMock).toHaveBeenCalledWith("repo-1");
140
+ });
141
+});
142
+
0
143