/** * OpenCodeHub Storage System * * Two storage backends are supported: * - local : filesystem storage rooted at STORAGE_PATH (default ./data/storage) * - s3 : any S3-compatible object store (AWS S3, MinIO, Cloudflare R2, * Garage, SeaweedFS, Ceph RGW, Wasabi, Backblaze B2, etc.) * * The S3 adapter speaks the S3 v4 API; any vendor that implements it works * by setting STORAGE_ENDPOINT to the vendor's endpoint URL and STORAGE_REGION * to its region string. No vendor-specific code paths are required. */ import { logger } from "@/lib/logger"; import crypto from "crypto"; import type { NodePgDatabase } from "drizzle-orm/node-postgres"; import { createReadStream, createWriteStream } from "fs"; import fs from "fs/promises"; import path from "path"; import { Readable } from "stream"; import { pipeline } from "stream/promises"; export type StorageType = "local" | "s3"; export interface StorageConfig { type: StorageType; basePath: string; bucket?: string; region?: string; endpoint?: string; accessKeyId?: string; secretAccessKey?: string; } export interface StorageObject { key: string; size: number; lastModified: Date; etag?: string; contentType?: string; metadata?: Record; } export interface ListOptions { prefix?: string; delimiter?: string; maxKeys?: number; continuationToken?: string; } export interface ListResult { objects: StorageObject[]; prefixes?: string[]; isTruncated: boolean; continuationToken?: string; } export interface PutOptions { contentType?: string; metadata?: Record; cacheControl?: string; contentDisposition?: string; } export interface GetOptions { range?: { start: number; end: number }; } /** * Abstract storage adapter */ export abstract class StorageAdapter { protected config: StorageConfig; constructor(config: StorageConfig) { this.config = config; } abstract put( key: string, data: Buffer | Readable | ReadableStream, options?: PutOptions, ): Promise; async writeStream( key: string, stream: Readable | ReadableStream, options?: PutOptions, ): Promise { return this.put(key, stream, options); } abstract get(key: string, options?: GetOptions): Promise; abstract getStream(key: string, options?: GetOptions): Promise; abstract delete(key: string): Promise; abstract exists(key: string): Promise; abstract list(options?: ListOptions): Promise; abstract copy(sourceKey: string, destKey: string): Promise; abstract move(sourceKey: string, destKey: string): Promise; abstract getSignedUrl(key: string, expiresIn?: number): Promise; abstract getSignedUploadUrl(key: string, expiresIn?: number): Promise; abstract stat(key: string): Promise; } /** * Local filesystem storage adapter */ export class LocalStorageAdapter extends StorageAdapter { private getFullPath(key: string): string { return path.join(this.config.basePath, key); } async put( key: string, data: Buffer | Readable | ReadableStream, options?: PutOptions, ): Promise { const fullPath = this.getFullPath(key); await fs.mkdir(path.dirname(fullPath), { recursive: true }); if (Buffer.isBuffer(data)) { await fs.writeFile(fullPath, data); } else { const writeStream = createWriteStream(fullPath); // @ts-ignore await pipeline(data, writeStream); } // Store metadata in .meta file if (options?.metadata) { const metaPath = `${fullPath}.meta`; await fs.writeFile( metaPath, JSON.stringify({ contentType: options.contentType, metadata: options.metadata, cacheControl: options.cacheControl, }), ); } } async get(key: string, options?: GetOptions): Promise { const fullPath = this.getFullPath(key); if (options?.range) { const fd = await fs.open(fullPath, "r"); try { const buffer = Buffer.alloc(options.range.end - options.range.start + 1); await fd.read(buffer, 0, buffer.length, options.range.start); return buffer; } finally { await fd.close(); } } return fs.readFile(fullPath); } async getStream(key: string, options?: GetOptions): Promise { const fullPath = this.getFullPath(key); return createReadStream( fullPath, options?.range ? { start: options.range.start, end: options.range.end, } : undefined, ); } async delete(key: string): Promise { const fullPath = this.getFullPath(key); await fs.unlink(fullPath).catch(() => {}); await fs.unlink(`${fullPath}.meta`).catch(() => {}); } async exists(key: string): Promise { try { await fs.access(this.getFullPath(key)); return true; } catch { return false; } } async list(options?: ListOptions): Promise { const basePath = path.join(this.config.basePath, options?.prefix || ""); const objects: StorageObject[] = []; const prefixes = new Set(); const walk = async (dir: string, currentPrefix: string) => { try { const entries = await fs.readdir(dir, { withFileTypes: true }); for (const entry of entries) { const fullPath = path.join(dir, entry.name); const key = path.join(currentPrefix, entry.name); if (entry.isDirectory()) { if (options?.delimiter === "/") { prefixes.add(key + "/"); } else { await walk(fullPath, key); } } else if (!entry.name.endsWith(".meta")) { const stats = await fs.stat(fullPath); objects.push({ key, size: stats.size, lastModified: stats.mtime, etag: crypto.createHash("md5").update(key).digest("hex"), }); } } } catch {} }; await walk(basePath, options?.prefix || ""); return { objects: objects.slice(0, options?.maxKeys || 1000), prefixes: options?.delimiter ? Array.from(prefixes) : undefined, isTruncated: objects.length > (options?.maxKeys || 1000), }; } async copy(sourceKey: string, destKey: string): Promise { const sourcePath = this.getFullPath(sourceKey); const destPath = this.getFullPath(destKey); await fs.mkdir(path.dirname(destPath), { recursive: true }); await fs.copyFile(sourcePath, destPath); } async move(sourceKey: string, destKey: string): Promise { const sourcePath = this.getFullPath(sourceKey); const destPath = this.getFullPath(destKey); await fs.mkdir(path.dirname(destPath), { recursive: true }); await fs.rename(sourcePath, destPath); } async getSignedUrl(key: string, _expiresIn?: number): Promise { return `file://${this.getFullPath(key)}`; } async stat(key: string): Promise { const fullPath = this.getFullPath(key); const stats = await fs.stat(fullPath); let metadata: Record | undefined; let contentType: string | undefined; try { const meta = JSON.parse(await fs.readFile(`${fullPath}.meta`, "utf-8")); metadata = meta.metadata; contentType = meta.contentType; } catch {} // Stream-based hash to avoid reading entire file into memory const etag = await new Promise((resolve, reject) => { const hash = crypto.createHash("md5"); const stream = createReadStream(fullPath); stream.on("data", (chunk) => hash.update(chunk)); stream.on("end", () => resolve(hash.digest("hex"))); stream.on("error", reject); }); return { key, size: stats.size, lastModified: stats.mtime, etag, contentType, metadata, }; } async getSignedUploadUrl( key: string, _expiresIn: number = 3600, ): Promise { return `/api/storage/upload/${key}`; } } /** * S3-compatible storage adapter. * Works with AWS S3, MinIO, Cloudflare R2, Garage, SeaweedFS, Ceph RGW, * Wasabi, Backblaze B2, and any other S3 v4 implementation. * * Set STORAGE_ENDPOINT to the vendor endpoint (omit for AWS S3). * Set STORAGE_REGION to the region string the vendor documents. * forcePathStyle is enabled automatically when STORAGE_ENDPOINT is set, * which is required by MinIO, Garage, SeaweedFS, and most self-hosted stacks. */ export class S3StorageAdapter extends StorageAdapter { private client: any; // S3Client from @aws-sdk/client-s3 private async getClient() { if (!this.client) { const { S3Client } = await import("@aws-sdk/client-s3"); this.client = new S3Client({ region: this.config.region || "us-east-1", endpoint: this.config.endpoint, credentials: this.config.accessKeyId ? { accessKeyId: this.config.accessKeyId, secretAccessKey: this.config.secretAccessKey!, } : undefined, forcePathStyle: !!this.config.endpoint, }); } return this.client; } async put( key: string, data: Buffer | Readable, options?: PutOptions, ): Promise { const { PutObjectCommand } = await import("@aws-sdk/client-s3"); const client = await this.getClient(); await client.send( new PutObjectCommand({ Bucket: this.config.bucket, Key: path.join(this.config.basePath, key), Body: data, ContentType: options?.contentType, Metadata: options?.metadata, CacheControl: options?.cacheControl, }), ); } async writeStream( key: string, stream: Readable | ReadableStream, options?: PutOptions, ): Promise { const { Upload } = await import("@aws-sdk/lib-storage"); const client = await this.getClient(); const upload = new Upload({ client, params: { Bucket: this.config.bucket, Key: path.join(this.config.basePath, key), Body: stream, ContentType: options?.contentType, Metadata: options?.metadata, CacheControl: options?.cacheControl, ContentDisposition: options?.contentDisposition, }, queueSize: 4, partSize: 5 * 1024 * 1024, }); await upload.done(); } async get(key: string, options?: GetOptions): Promise { const { GetObjectCommand } = await import("@aws-sdk/client-s3"); const client = await this.getClient(); const s3Key = path.join(this.config.basePath, key); logger.info( `[S3StorageAdapter.get] basePath="${this.config.basePath}", key="${key}", s3Key="${s3Key}"`, ); const response = await client.send( new GetObjectCommand({ Bucket: this.config.bucket, Key: s3Key, Range: options?.range ? `bytes=${options.range.start}-${options.range.end}` : undefined, }), ); const chunks: Buffer[] = []; for await (const chunk of response.Body as any) { chunks.push(chunk); } return Buffer.concat(chunks); } async getStream(key: string, options?: GetOptions): Promise { const { GetObjectCommand } = await import("@aws-sdk/client-s3"); const client = await this.getClient(); const response = await client.send( new GetObjectCommand({ Bucket: this.config.bucket, Key: path.join(this.config.basePath, key), Range: options?.range ? `bytes=${options.range.start}-${options.range.end}` : undefined, }), ); return response.Body as Readable; } async delete(key: string): Promise { const { DeleteObjectCommand } = await import("@aws-sdk/client-s3"); const client = await this.getClient(); await client.send( new DeleteObjectCommand({ Bucket: this.config.bucket, Key: path.join(this.config.basePath, key), }), ); } async exists(key: string): Promise { const { HeadObjectCommand } = await import("@aws-sdk/client-s3"); const client = await this.getClient(); try { await client.send( new HeadObjectCommand({ Bucket: this.config.bucket, Key: path.join(this.config.basePath, key), }), ); return true; } catch { return false; } } async list(options?: ListOptions): Promise { const { ListObjectsV2Command } = await import("@aws-sdk/client-s3"); const client = await this.getClient(); const response = await client.send( new ListObjectsV2Command({ Bucket: this.config.bucket, Prefix: path.join(this.config.basePath, options?.prefix || ""), Delimiter: options?.delimiter, MaxKeys: options?.maxKeys, ContinuationToken: options?.continuationToken, }), ); return { objects: (response.Contents || []).map((obj: any) => ({ key: this.config.basePath ? obj.Key!.replace(this.config.basePath + "/", "") : obj.Key!, size: obj.Size!, lastModified: obj.LastModified!, etag: obj.ETag?.replace(/"/g, ""), })), prefixes: response.CommonPrefixes?.map((p: any) => this.config.basePath ? p.Prefix!.replace(this.config.basePath + "/", "") : p.Prefix!, ), isTruncated: response.IsTruncated || false, continuationToken: response.NextContinuationToken, }; } async copy(sourceKey: string, destKey: string): Promise { const { CopyObjectCommand } = await import("@aws-sdk/client-s3"); const client = await this.getClient(); const sourceFullPath = path.join(this.config.basePath, sourceKey); const copySource = `${this.config.bucket}/${encodeURIComponent(sourceFullPath)}`; await client.send( new CopyObjectCommand({ Bucket: this.config.bucket, CopySource: copySource, Key: path.join(this.config.basePath, destKey), }), ); } async move(sourceKey: string, destKey: string): Promise { await this.copy(sourceKey, destKey); await this.delete(sourceKey); } async getSignedUrl(key: string, expiresIn: number = 3600): Promise { const { getSignedUrl } = await import("@aws-sdk/s3-request-presigner"); const { GetObjectCommand } = await import("@aws-sdk/client-s3"); const client = await this.getClient(); const command = new GetObjectCommand({ Bucket: this.config.bucket, Key: path.join(this.config.basePath, key), }); return getSignedUrl(client, command, { expiresIn }); } async getSignedUploadUrl( key: string, expiresIn: number = 3600, ): Promise { const { getSignedUrl } = await import("@aws-sdk/s3-request-presigner"); const { PutObjectCommand } = await import("@aws-sdk/client-s3"); const client = await this.getClient(); const command = new PutObjectCommand({ Bucket: this.config.bucket, Key: path.join(this.config.basePath, key), }); return getSignedUrl(client, command, { expiresIn }); } async stat(key: string): Promise { const { HeadObjectCommand } = await import("@aws-sdk/client-s3"); const client = await this.getClient(); const response = await client.send( new HeadObjectCommand({ Bucket: this.config.bucket, Key: path.join(this.config.basePath, key), }), ); return { key, size: response.ContentLength!, lastModified: response.LastModified!, etag: response.ETag?.replace(/"/g, ""), contentType: response.ContentType, metadata: response.Metadata, }; } } /** * Storage factory. * Throws if an unknown type is requested. Only `local` and `s3` are valid. */ export function createStorageAdapter(config: StorageConfig): StorageAdapter { logger.info( `[createStorageAdapter] Creating adapter for type: ${config.type}`, ); switch (config.type) { case "local": return new LocalStorageAdapter(config); case "s3": return new S3StorageAdapter(config); default: throw new Error( `Unknown storage type: ${(config as { type: string }).type}. ` + `Supported types are: local, s3.`, ); } } // Singleton instance let storageInstance: StorageAdapter | null = null; let lastConfigHash: string = ""; /** * Reset storage singleton - forces re-initialization on next getStorage() call * Useful after changing environment variables or for testing */ export function resetStorage(): void { logger.info("[resetStorage] Clearing storage singleton"); storageInstance = null; lastConfigHash = ""; } /** * Check storage health/connectivity * Returns true if storage is accessible, false otherwise */ export async function checkStorageHealth(): Promise<{ healthy: boolean; type: string; error?: string; }> { try { const storage = await getStorage(); const config = getStorageConfig(); // Try to list root - if this works, storage is accessible await storage.list({ maxKeys: 1 }); return { healthy: true, type: config.type }; } catch (error: any) { return { healthy: false, type: "unknown", error: error.message }; } } /** * Get current storage configuration from environment. * Reads only the variables relevant to the `local` and `s3` drivers. */ export function getStorageConfig(): StorageConfig { const storageType = ((import.meta.env?.STORAGE_TYPE || process.env.STORAGE_TYPE || "local") as StorageType); let basePath = ""; if (storageType === "local") { basePath = import.meta.env?.STORAGE_PATH || process.env.STORAGE_PATH || "./data/storage"; } return { type: storageType, basePath, bucket: import.meta.env?.STORAGE_BUCKET || process.env.STORAGE_BUCKET, region: import.meta.env?.STORAGE_REGION || process.env.STORAGE_REGION, endpoint: import.meta.env?.STORAGE_ENDPOINT || process.env.STORAGE_ENDPOINT, accessKeyId: import.meta.env?.STORAGE_ACCESS_KEY_ID || process.env.STORAGE_ACCESS_KEY_ID, secretAccessKey: import.meta.env?.STORAGE_SECRET_ACCESS_KEY || process.env.STORAGE_SECRET_ACCESS_KEY, }; } export async function getStorage(): Promise { const envPriority = import.meta.env?.STORAGE_ENV_PRIORITY === "true" || process.env.STORAGE_ENV_PRIORITY === "true"; if (!envPriority) { try { const { getDatabase, schema } = await import("@/db"); const { eq } = await import("drizzle-orm"); const db = getDatabase() as NodePgDatabase; const configRow = await db.query.systemConfig.findFirst({ where: eq(schema.systemConfig.key, "storage_config"), }); if (configRow) { if (configRow.value !== lastConfigHash) { const config = JSON.parse(configRow.value) as StorageConfig; logger.info(`[getStorage] Using DB config: type=${config.type}`); storageInstance = createStorageAdapter(config); lastConfigHash = configRow.value; } return storageInstance!; } } catch (e) { logger.warn( "[getStorage] Failed to fetch DB config, using env fallback", e, ); } } else { logger.info("[getStorage] STORAGE_ENV_PRIORITY=true, skipping DB config"); } if (!storageInstance) { const config = getStorageConfig(); logger.info( `[getStorage] Using env config: type=${config.type}, basePath="${config.basePath}"`, ); storageInstance = createStorageAdapter(config); lastConfigHash = JSON.stringify(config); } return storageInstance; } export async function updateStorageConfig( config: StorageConfig, userId?: string, ): Promise { const { getDatabase, schema } = await import("@/db"); const { eq } = await import("drizzle-orm"); const db = getDatabase() as NodePgDatabase; if (!config.type) throw new Error("Storage type required"); if (config.type !== "local" && config.type !== "s3") { throw new Error( `Unsupported storage type '${config.type}'. Supported types are: local, s3.`, ); } const value = JSON.stringify(config); await db .insert(schema.systemConfig) .values({ key: "storage_config", value, updatedById: userId, }) .onConflictDoUpdate({ target: schema.systemConfig.key, set: { value, updatedAt: new Date(), updatedById: userId, }, }); lastConfigHash = ""; storageInstance = null; } /** * Helper functions for common operations */ export async function uploadFile( key: string, filePath: string, options?: PutOptions, ): Promise { const storage = await getStorage(); const stream = createReadStream(filePath); await storage.put(key, stream, options); } export async function downloadFile( key: string, destPath: string, ): Promise { const storage = await getStorage(); const data = await storage.get(key); await fs.mkdir(path.dirname(destPath), { recursive: true }); await fs.writeFile(destPath, data); } export async function getFileAsBase64(key: string): Promise { const storage = await getStorage(); const data = await storage.get(key); return data.toString("base64"); } export async function getFileHash( key: string, algorithm: string = "sha256", ): Promise { const storage = await getStorage(); const data = await storage.get(key); return crypto.createHash(algorithm).update(data).digest("hex"); } export async function getSignedUploadUrl( key: string, expiresIn: number = 3600, ): Promise { const storage = await getStorage(); return storage.getSignedUploadUrl(key, expiresIn); } export async function getSignedDownloadUrl( key: string, expiresIn: number = 3600, ): Promise { const storage = await getStorage(); return storage.getSignedUrl(key, expiresIn); }