Merge pull request #459 from hbjydev/cdn-s3

[CDN] S3 Support

bundle/package.json

@@ -47,7 +47,7 @@
 		"jest": "^27.0.6",
 		"jest-expect-message": "^1.0.2",
 		"jest-runtime": "^27.2.1",
-		"ts-node": "^9.1.1",
+		"ts-node": "^10.2.1",
 		"ts-node-dev": "^1.1.6",
 		"ts-patch": "^1.4.4",
 		"typescript": "^4.2.3",
@@ -93,6 +93,8 @@
 		"typescript": "^4.1.2",
 		"typescript-json-schema": "^0.50.1",
 		"ws": "^7.4.2",
-		"cheerio": "^1.0.0-rc.10"
+		"cheerio": "^1.0.0-rc.10",
+		"@aws-sdk/client-s3": "^3.36.1",
+		"@aws-sdk/node-http-handler": "^3.36.0"
 	}
 }

cdn/package.json

@@ -36,6 +36,8 @@
 		"ts-patch": "^1.4.4"
 	},
 	"dependencies": {
+		"@aws-sdk/client-s3": "^3.36.1",
+		"@aws-sdk/node-http-handler": "^3.36.0",
 		"@fosscord/util": "file:../util",
 		"body-parser": "^1.19.0",
 		"btoa": "^1.2.1",

cdn/src/util/S3Storage.ts

@@ -0,0 +1,60 @@
+import { S3 } from "@aws-sdk/client-s3";
+import { Readable } from "stream";
+import { Storage } from "./Storage";
+
+const readableToBuffer = (readable: Readable): Promise<Buffer> =>
+	new Promise((resolve, reject) => {
+		const chunks: Buffer[] = [];
+		readable.on('data', chunk => chunks.push(chunk));
+		readable.on('error', reject);
+		readable.on('end', () => resolve(Buffer.concat(chunks)));
+	});
+
+export class S3Storage implements Storage {
+	public constructor(
+		private client: S3,
+		private bucket: string,
+		private basePath?: string,
+	) {}
+
+	/**
+	 * Always return a string, to ensure consistency.
+	 */
+	get bucketBasePath() {
+		return this.basePath ?? '';
+	}
+
+	async set(path: string, data: Buffer): Promise<void> {
+		await this.client.putObject({
+			Bucket: this.bucket,
+			Key: `${this.bucketBasePath}${path}`,
+			Body: data
+		});
+	}
+
+	async get(path: string): Promise<Buffer | null> {
+		try {
+			const s3Object = await this.client.getObject({
+				Bucket: this.bucket,
+				Key: `${this.bucketBasePath ?? ''}${path}`
+			});
+
+			if (!s3Object.Body) return null;
+
+			const body = s3Object.Body;
+
+			return await readableToBuffer(<Readable> body);
+		} catch(err) {
+			console.error(`[CDN] Unable to get S3 object at path ${path}.`);
+			console.error(err);
+			return null;
+		}
+	}
+
+	async delete(path: string): Promise<void> {
+		await this.client.deleteObject({
+			Bucket: this.bucket,
+			Key: `${this.bucketBasePath}${path}`
+		});
+	}
+}

cdn/src/util/Storage.ts

@@ -2,6 +2,8 @@ import { FileStorage } from "./FileStorage";
 import path from "path";
 import fse from "fs-extra";
 import { bgCyan, black } from "nanocolors";
+import { S3 } from '@aws-sdk/client-s3';
+import { S3Storage } from "./S3Storage";
 process.cwd();
 
 export interface Storage {
@@ -10,10 +12,10 @@ export interface Storage {
 	delete(path: string): Promise<void>;
 }
 
-var storage: Storage;
+let storage: Storage;
 
 if (process.env.STORAGE_PROVIDER === "file" || !process.env.STORAGE_PROVIDER) {
-	var location = process.env.STORAGE_LOCATION;
+	let location = process.env.STORAGE_LOCATION;
 	if (location) {
 		location = path.resolve(location);
 	} else {
@@ -24,6 +26,32 @@ if (process.env.STORAGE_PROVIDER === "file" || !process.env.STORAGE_PROVIDER) {
 	process.env.STORAGE_LOCATION = location;
 
 	storage = new FileStorage();
+} else if (process.env.STORAGE_PROVIDER === "s3") {
+	const
+		region = process.env.STORAGE_REGION,
+		bucket = process.env.STORAGE_BUCKET;
+
+	if (!region) {
+		console.error(`[CDN] You must provide a region when using the S3 storage provider.`);
+		process.exit(1);
+	}
+
+	if (!bucket) {
+		console.error(`[CDN] You must provide a bucket when using the S3 storage provider.`);
+		process.exit(1);
+	}
+
+	// in the S3 provider, this should be the root path in the bucket
+	let location = process.env.STORAGE_LOCATION;
+
+	if (!location) {
+		console.warn(`[CDN] STORAGE_LOCATION unconfigured for S3 provider, defaulting to the bucket root...`);
+  		location = undefined;
+	}
+
+	const client = new S3({ region });
+
+	storage = new S3Storage(client, bucket, location);
 }
 
 export { storage };