Spacebar/server
1
0
Fork 0
mirror of https://github.com/spacebarchat/server.git synced 2024-11-11 05:02:37 +01:00
Code Issues Releases Activity

feat: implement an S3-based storage API

Browse Source
This commit is contained in:
Hayden Young 2021-10-14 23:04:03 +01:00
parent 2bf3ee4290
commit e496a15f2d
No known key found for this signature in database
GPG Key ID: E5C78A4D88C0BA40
4 changed files with 3737 additions and 6241 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9891
cdn/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

2
cdn/package.json
View File

@ -42,6 +42,8 @@
"ts-patch": "^1.4.4"
},
"dependencies": {
"@aws-sdk/client-s3": "^3.36.1",
"@aws-sdk/node-http-handler": "^3.36.0",
"@fosscord/util": "file:../util",
"body-parser": "^1.19.0",
"btoa": "^1.2.1",

53
cdn/src/util/S3Storage.ts Normal file
View File

@ -0,0 +1,53 @@
import { S3 } from "@aws-sdk/client-s3";
import { Readable, Stream } from "stream";
import { Storage } from "./Storage";
const readableToBuffer = (readable: Readable): Promise<Buffer> =>
new Promise((resolve, reject) => {
const chunks: Buffer[] = [];
readable.on('data', chunk => chunks.push(chunk));
readable.on('error', reject);
readable.on('end', () => resolve(Buffer.concat(chunks)));
});
export class S3Storage implements Storage {
public constructor(
private client: S3,
private basePath: string,
private bucket: string
) {}
async set(path: string, data: Buffer): Promise<void> {
await this.client.putObject({
Bucket: this.bucket,
Key: `${this.basePath}${path}`,
Body: data
});
}
async get(path: string): Promise<Buffer | null> {
try {
const s3Object = await this.client.getObject({
Bucket: this.bucket,
Key: `${this.basePath}${path}`
});
if (!s3Object.Body) return null;
const body = s3Object.Body;
return await readableToBuffer(<Readable> body);
} catch(err) {
console.error(`[CDN] Unable to get S3 object at path ${path}.`);
console.error(err);
return null;
}
}
async delete(path: string): Promise<void> {
await this.client.deleteObject({
Bucket: this.bucket,
Key: `${this.basePath}${path}`
});
}
}

32
cdn/src/util/Storage.ts
View File

@ -2,6 +2,8 @@ import { FileStorage } from "./FileStorage";
import path from "path";
import fse from "fs-extra";
import { bgCyan, black } from "nanocolors";
import { S3 } from '@aws-sdk/client-s3';
import { S3Storage } from "./S3Storage";
process.cwd();
export interface Storage {
@ -10,10 +12,10 @@ export interface Storage {
delete(path: string): Promise<void>;
}
var storage: Storage;
let storage: Storage;
if (process.env.STORAGE_PROVIDER === "file" || !process.env.STORAGE_PROVIDER) {
var location = process.env.STORAGE_LOCATION;
let location = process.env.STORAGE_LOCATION;
if (location) {
location = path.resolve(location);
} else {
@ -24,6 +26,32 @@ if (process.env.STORAGE_PROVIDER === "file" || !process.env.STORAGE_PROVIDER) {
process.env.STORAGE_LOCATION = location;
storage = new FileStorage();
} else if (process.env.STORAGE_PROVIDER === "s3") {
const
region = process.env.STORAGE_REGION,
bucket = process.env.STORAGE_BUCKET;
if (!region) {
console.error(`[CDN] You must provide a region when using the S3 storage provider.`);
process.exit(1);
}
if (!bucket) {
console.error(`[CDN] You must provide a bucket when using the S3 storage provider.`);
process.exit(1);
}
// in the S3 provider, this should be the root path in the bucket
let location = process.env.STORAGE_LOCATION;
if (!location) {
console.warn(`[CDN] STORAGE_LOCATION unconfigured for S3 provider, defaulting to '/'...`);
location = "/";
}
const client = new S3({ region });
storage = new S3Storage(client, location, bucket);
}
export { storage };