✨ added crawler to cdn

.gitignore

@@ -0,0 +1,3 @@
+.vscode/
+node_modules/
+dist/

package.json

@@ -0,0 +1,38 @@
+{
+	"name": "discord-cdn",
+	"version": "1.0.0",
+	"description": "cdn for discord clone",
+	"main": "index.js",
+	"scripts": {
+		"test": "echo \"Error: no test specified\" && exit 1"
+	},
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com/discord-open-source/discord-cdn.git"
+	},
+	"keywords": [],
+	"author": "",
+	"license": "ISC",
+	"bugs": {
+		"url": "https://github.com/discord-open-source/discord-cdn/issues"
+	},
+	"homepage": "https://github.com/discord-open-source/discord-cdn#readme",
+	"dependencies": {
+		"body-parser": "^1.19.0",
+		"btoa": "^1.2.1",
+		"cheerio": "^1.0.0-rc.5",
+		"express": "^4.17.1",
+		"express-async-errors": "^3.1.1",
+		"lambert-db": "^1.0.5",
+		"missing-native-js-functions": "^1.0.8",
+		"multer": "^1.4.2",
+		"node-fetch": "^2.6.1"
+	},
+	"devDependencies": {
+		"@types/btoa": "^1.2.3",
+		"@types/express": "^4.17.9",
+		"@types/multer": "^1.4.5",
+		"@types/node": "^14.14.16",
+		"@types/node-fetch": "^2.5.7"
+	}
+}

src/Server.ts

@@ -0,0 +1,93 @@
+import express, { Application, Router, Request, Response, NextFunction } from "express";
+import { MongoDatabase, Database } from "lambert-db";
+import { Server as HTTPServer } from "http";
+import { traverseDirectory } from "./Util";
+import bodyParser from "body-parser";
+import "express-async-errors";
+
+const log = console.log;
+console.log = (content) => {
+	log(`[${new Date().toTimeString().split(" ")[0]}]`, content);
+};
+
+export type ServerOptions = {
+	db: string;
+	port: number;
+	host: string;
+};
+
+declare global {
+	namespace Express {
+		interface Request {
+			server: Server;
+		}
+	}
+}
+
+export class Server {
+	app: Application;
+	http: HTTPServer;
+	db: Database;
+	routes: Router[];
+	options: ServerOptions;
+
+	constructor(options: Partial<ServerOptions> = { port: 3000, host: "0.0.0.0" }) {
+		this.app = express();
+		this.db = new MongoDatabase(options?.db);
+		this.options = options as ServerOptions;
+	}
+
+	async init() {
+		await this.db.init();
+
+		console.log("[Database] connected...");
+		await new Promise((res, rej) => {
+			this.http = this.app.listen(this.options.port, this.options.host, () => res(null));
+		});
+		this.routes = await this.registerRoutes(__dirname + "/routes/");
+	}
+
+	async registerRoutes(root: string) {
+		this.app.use((req, res, next) => {
+			req.server = this;
+			next();
+		});
+		const routes = await traverseDirectory({ dirname: root, recursive: true }, this.registerRoute.bind(this, root));
+		this.app.use((err: string | Error, req: Request, res: Response, next: NextFunction) => {
+			res.status(400).send(err);
+			next(err);
+		});
+		return routes;
+	}
+
+	registerRoute(root: string, file: string): any {
+		if (root.endsWith("/") || root.endsWith("\\")) root = root.slice(0, -1); // removes slash at the end of the root dir
+		let path = file.replace(root, ""); // remove root from path and
+		path = path.split(".").slice(0, -1).join("."); // trancate .js/.ts file extension of path
+		if (path.endsWith("/index")) path = path.slice(0, -6); // delete index from path
+
+		try {
+			var router = require(file);
+			if (router.router) router = router.router;
+			if (router.default) router = router.default;
+			if (!router || router?.prototype?.constructor?.name !== "router")
+				throw `File doesn't export any default router`;
+			this.app.use(path, <Router>router);
+			console.log(`[Routes] ${path} registerd`);
+
+			return router;
+		} catch (error) {
+			console.error(new Error(`[Server] ¯\\_(ツ)_/¯ Failed to register route ${path}: ${error}`));
+		}
+	}
+
+	async destroy() {
+		await this.db.destroy();
+		await new Promise((res, rej) => {
+			this.http.close((err) => {
+				if (err) return rej(err);
+				return res("");
+			});
+		});
+	}
+}

src/Util.ts

@@ -0,0 +1,38 @@
+import fs from "fs/promises";
+import "missing-native-js-functions";
+
+export interface traverseDirectoryOptions {
+	dirname: string;
+	filter?: RegExp;
+	excludeDirs?: RegExp;
+	recursive?: boolean;
+}
+
+const DEFAULT_EXCLUDE_DIR = /^\./;
+const DEFAULT_FILTER = /^([^\.].*)\.js$/;
+
+export async function traverseDirectory<T>(
+	options: traverseDirectoryOptions,
+	action: (path: string) => T
+): Promise<T[]> {
+	if (!options.filter) options.filter = DEFAULT_FILTER;
+	if (!options.excludeDirs) options.excludeDirs = DEFAULT_EXCLUDE_DIR;
+
+	const routes = await fs.readdir(options.dirname);
+	const promises = <Promise<T | T[] | undefined>[]>routes.map(async (file) => {
+		const path = options.dirname + file;
+		const stat = await fs.lstat(path);
+		if (path.match(<RegExp>options.excludeDirs)) return;
+
+		if (stat.isFile() && path.match(<RegExp>options.filter)) {
+			return action(path);
+		} else if (options.recursive && stat.isDirectory()) {
+			return traverseDirectory({ ...options, dirname: path + "/" }, action);
+		}
+	});
+	const result = await Promise.all(promises);
+
+	const t = <(T | undefined)[]>result.flat();
+
+	return <T[]>t.filter((x) => x != undefined);
+}

src/index.ts

@@ -0,0 +1,14 @@
+import { Server } from "./Server";
+
+const server = new Server();
+server
+	.init()
+	.then(() => {
+		console.log("[Server] started on :" + server.options.port);
+	})
+	.catch((e) => console.error("[Server] Error starting: ", e));
+
+//// server
+//// 	.destroy()
+//// 	.then(() => console.log("[Server] closed."))
+//// .catch((e) => console.log("[Server] Error closing: ", e));

src/routes/attachments.ts.disabled

@@ -0,0 +1,19 @@
+import { Router } from "express";
+import multer from "multer";
+const multer_ = multer();
+
+const router = Router();
+router.post("/:file", multer_.single("attachment"), async (req, res) => {
+	const { buffer } = req.file;
+
+	res.set("Content-Type", "image/png");
+	res.send(buffer);
+});
+router.get("/:hash/:file", async (req, res) => {
+	res.send(`${req.params.hash}/${req.params.file}`);
+});
+router.delete("/:hash/:file", async (req, res) => {
+	res.send("remove");
+});
+
+export default router;

src/routes/external.ts

@@ -0,0 +1,83 @@
+import bodyParser from "body-parser";
+import { Router } from "express";
+import fetch from "node-fetch";
+import cheerio from "cheerio";
+import btoa from "btoa";
+import { URL } from "url";
+
+const router = Router();
+
+type crawled = {
+	title: string;
+	type: string;
+	description: string;
+	url: string;
+	image_url: string;
+};
+
+const DEFAULT_FETCH_OPTIONS: any = {
+	redirect: "follow",
+	follow: 1,
+	headers: {
+		"user-agent": "Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)",
+	},
+	size: 1024 * 1024 * 8,
+	compress: true,
+	method: "GET",
+};
+
+router.post("/", bodyParser.json(), async (req, res) => {
+	if (!req.body) throw new Error("Invalid Body (url missing) \nExample: url:https://discord.com");
+
+	const { db } = req.server;
+	const { url } = req.body;
+
+	const ID = btoa(url);
+
+	const cache = await db.data.crawler({ id: ID }).get();
+	if (cache) return res.send(cache);
+
+	try {
+		const request = await fetch(url, DEFAULT_FETCH_OPTIONS);
+
+		const text = await request.text();
+		const ツ: any = cheerio.load(text);
+
+		const ogTitle = ツ('meta[property="og:title"]').attr("content");
+		const ogDescription = ツ('meta[property="og:description"]').attr("content");
+		const ogImage = ツ('meta[property="og:image"]').attr("content");
+		const ogUrl = ツ('meta[property="og:url"]').attr("content");
+		const ogType = ツ('meta[property="og:type"]').attr("content");
+
+		const filename = new URL(url).host.split(".")[0];
+
+		const ImageResponse = await fetch(ogImage, DEFAULT_FETCH_OPTIONS);
+		const ImageType = ImageResponse.headers.get("content-type");
+		const ImageExtension = ImageType?.split("/")[1];
+		const ImageResponseBuffer = (await ImageResponse.buffer()).toString("base64");
+		const cachedImage = `/external/${ID}/${filename}.${ImageExtension}`;
+
+		await db.data.externals.push({ image: ImageResponseBuffer, id: ID, type: ImageType });
+
+		const new_cache_entry = { id: ID, ogTitle, ogDescription, cachedImage, ogUrl, ogType };
+		await db.data.crawler.push(new_cache_entry);
+
+		res.send(new_cache_entry);
+	} catch (error) {
+		console.log(error);
+
+		throw new Error("Couldn't fetch website");
+	}
+});
+
+router.get("/:id/:filename", async (req, res) => {
+	const { db } = req.server;
+	const { id, filename } = req.params;
+	const { image, type } = await db.data.externals({ id: id }).get();
+	const imageBuffer = Buffer.from(image, "base64");
+
+	res.set("Content-Type", type);
+	res.send(imageBuffer);
+});
+
+export default router;

tsconfig.json

@@ -0,0 +1,69 @@
+{
+	"compilerOptions": {
+		/* Visit https://aka.ms/tsconfig.json to read more about this file */
+
+		/* Basic Options */
+		// "incremental": true,                   /* Enable incremental compilation */
+		"target": "ES6" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */,
+		"module": "commonjs" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */,
+		"lib": ["ES2015"] /* Specify library files to be included in the compilation. */,
+		"allowJs": true /* Allow javascript files to be compiled. */,
+		"checkJs": true /* Report errors in .js files. */,
+		// "jsx": "preserve",                     /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */
+		"declaration": true /* Generates corresponding '.d.ts' file. */,
+		"declarationMap": false /* Generates a sourcemap for each corresponding '.d.ts' file. */,
+		"inlineSourceMap": true /* Emit a single file with source maps instead of having a separate file. */,
+		// "sourceMap": true /* Generates corresponding '.map' file. */,
+		// "outFile": "./",                       /* Concatenate and emit output to single file. */
+		"outDir": "./dist/" /* Redirect output structure to the directory. */,
+		"rootDir": "./src/" /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */,
+		// "composite": true,                     /* Enable project compilation */
+		// "tsBuildInfoFile": "./",               /* Specify file to store incremental compilation information */
+		// "removeComments": true,                /* Do not emit comments to output. */
+		// "noEmit": true,                        /* Do not emit outputs. */
+		// "importHelpers": true,                 /* Import emit helpers from 'tslib'. */
+		// "downlevelIteration": true,            /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */
+		// "isolatedModules": true,               /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */
+
+		/* Strict Type-Checking Options */
+		"strict": true /* Enable all strict type-checking options. */,
+		"noImplicitAny": true /* Raise error on expressions and declarations with an implied 'any' type. */,
+		// "strictNullChecks": true,              /* Enable strict null checks. */
+		// "strictFunctionTypes": true,           /* Enable strict checking of function types. */
+		// "strictBindCallApply": true,           /* Enable strict 'bind', 'call', and 'apply' methods on functions. */
+		"strictPropertyInitialization": false /* Enable strict checking of property initialization in classes. */,
+		// "noImplicitThis": true,                /* Raise error on 'this' expressions with an implied 'any' type. */
+		"alwaysStrict": true /* Parse in strict mode and emit "use strict" for each source file. */,
+
+		/* Additional Checks */
+		// "noUnusedLocals": true,                /* Report errors on unused locals. */
+		// "noUnusedParameters": true,            /* Report errors on unused parameters. */
+		// "noImplicitReturns": true,             /* Report error when not all code paths in function return a value. */
+		// "noFallthroughCasesInSwitch": true,    /* Report errors for fallthrough cases in switch statement. */
+
+		/* Module Resolution Options */
+		// "moduleResolution": "node",            /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */
+		// "baseUrl": "./",                       /* Base directory to resolve non-absolute module names. */
+		// "paths": {},                           /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */
+		// "rootDirs": [],                        /* List of root folders whose combined content represents the structure of the project at runtime. */
+		// "typeRoots": [],                       /* List of folders to include type definitions from. */
+		"types": ["node"] /* Type declaration files to be included in compilation. */,
+		// "allowSyntheticDefaultImports": true,  /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */
+		"esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */,
+		// "preserveSymlinks": true,              /* Do not resolve the real path of symlinks. */
+		// "allowUmdGlobalAccess": true,          /* Allow accessing UMD globals from modules. */
+
+		/* Source Map Options */
+		// "sourceRoot": "",                      /* Specify the location where debugger should locate TypeScript files instead of source locations. */
+		// "mapRoot": "",                         /* Specify the location where debugger should locate map files instead of generated locations. */
+		// "inlineSources": true,                 /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
+
+		/* Experimental Options */
+		// "experimentalDecorators": true,        /* Enables experimental support for ES7 decorators. */
+		// "emitDecoratorMetadata": true,         /* Enables experimental support for emitting type metadata for decorators. */
+
+		/* Advanced Options */
+		"skipLibCheck": true /* Skip type checking of declaration files. */,
+		"forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */
+	}
+}