Spacebar/server
1
0
Fork 0
mirror of https://github.com/spacebarchat/server.git synced 2024-11-08 20:02:39 +01:00
Code Issues Releases Activity

testing stuff

Browse Source
This commit is contained in:
TheArcaneBrony 2022-08-30 17:05:38 +02:00
parent ab1a09591a
commit 6142dcae20
No known key found for this signature in database
GPG Key ID: 32FC5AAADAD75A22
11 changed files with 136 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -34,3 +34,5 @@ dbconf.json
migrations.db
assets/cache_src/
package-lock.json

4
src/api/Server.ts
View File

@ -1,4 +1,4 @@
import { Config, getOrInitialiseDatabase, initEvent, registerRoutes } from "@fosscord/util";
import { Config, getOrInitialiseDatabase, initEvent, Paths, registerRoutes, TestClientPaths } from "@fosscord/util";
import { NextFunction, Request, Response, Router } from "express";
import { Server, ServerOptions } from "lambert-server";
import morgan from "morgan";
@ -44,7 +44,7 @@ export class FosscordServer extends Server {
morgan("combined", {
skip: (req, res) => {
if(req.path.endsWith(".map")) return true;
if(req.path.includes("/assets/") && !fs.existsSync(path.join(__dirname, "..", "..", "..", "assets", req.path.split("/")[0].split('?')[0]))) return true;
if(req.path.includes("/assets/") && !fs.existsSync(path.join(Paths.AssetsPath, req.path.split("/")[0].split('?')[0]))) return true;
let skip = !(process.env["LOG_REQUESTS"]?.includes(res.statusCode.toString()) ?? false);
if (process.env["LOG_REQUESTS"]?.charAt(0) == "-") skip = !skip;
return skip;

53
src/api/middlewares/TestClient.ts
View File

@ -1,4 +1,4 @@
import { Config } from "@fosscord/util";
import { Config, Paths, TestClientPaths } from "@fosscord/util";
import express, { Application, Request, Response } from "express";
import fs from "fs";
import fetch, { Headers, Response as FetchResponse } from "node-fetch";
@ -7,15 +7,15 @@ import { green } from "picocolors";
import ProxyAgent from "proxy-agent";
import { AssetCacheItem } from "../util/entities/AssetCacheItem";
import { patchFile } from "..";
import { createHash } from "crypto";
const prettier = require("prettier");
const AssetsPath = path.join(__dirname, "..", "..", "..", "assets");
export default function TestClient(app: Application) {
const agent = new ProxyAgent();
//build client page
let html = fs.readFileSync(path.join(AssetsPath, "index.html"), { encoding: "utf8" });
let html = fs.readFileSync(TestClientPaths.Index, { encoding: "utf8" });
html = applyEnv(html);
html = applyInlinePlugins(html);
html = applyPlugins(html);
@ -23,19 +23,15 @@ export default function TestClient(app: Application) {
//load asset cache
let newAssetCache: Map<string, AssetCacheItem> = new Map<string, AssetCacheItem>();
let assetCacheDir = path.join(AssetsPath, "cache");
if (process.env.ASSET_CACHE_DIR) assetCacheDir = process.env.ASSET_CACHE_DIR;
console.log(`[TestClient] ${green(`Using asset cache path: ${assetCacheDir}`)}`);
if (!fs.existsSync(assetCacheDir)) {
fs.mkdirSync(assetCacheDir);
}
if (fs.existsSync(path.join(assetCacheDir, "index.json"))) {
let rawdata = fs.readFileSync(path.join(assetCacheDir, "index.json"));
newAssetCache = new Map<string, AssetCacheItem>(Object.entries(JSON.parse(rawdata.toString())));
console.log(`[TestClient] ${green(`Using asset cache path: ${TestClientPaths.CacheDir}`)}`);
if (!fs.existsSync(TestClientPaths.CacheDir)) {
fs.mkdirSync(TestClientPaths.CacheDir);
}
if (fs.existsSync(TestClientPaths.CacheIndex))
newAssetCache = new Map<string, AssetCacheItem>(Object.entries(JSON.parse(fs.readFileSync(TestClientPaths.CacheIndex).toString())));
app.use("/assets", express.static(path.join(AssetsPath)));
app.use("/assets", express.static(path.join(Paths.AssetsPath)));
app.get("/assets/:file", async (req: Request, res: Response) => {
delete req.headers.host;
let response: FetchResponse;
@ -58,6 +54,9 @@ export default function TestClient(app: Application) {
...req.headers
}
});
buffer = await response.buffer();
let hash = createHash("md5").update(buffer).digest('hex');
//set cache info
assetCacheItem.Headers = Object.fromEntries(stripHeaders(response.headers));
assetCacheItem.Key = req.params.file;
@ -67,12 +66,12 @@ export default function TestClient(app: Application) {
if(response.status != 200) {
return res.status(404).send("Not found");
}
assetCacheItem.FilePath = path.join(assetCacheDir, req.params.file);
if(!fs.existsSync(assetCacheDir))
fs.mkdirSync(assetCacheDir);
fs.writeFileSync(path.join(assetCacheDir, "index.json"), JSON.stringify(Object.fromEntries(newAssetCache), null, 4));
assetCacheItem.FilePath = path.join(TestClientPaths.CacheDir, req.params.file);
if(!fs.existsSync(TestClientPaths.CacheDir))
fs.mkdirSync(TestClientPaths.CacheDir);
fs.writeFileSync(TestClientPaths.CacheIndex, JSON.stringify(Object.fromEntries(newAssetCache), null, 4));
//download file
fs.writeFileSync(assetCacheItem.FilePath, /.*\.(js|css)/.test(req.params.file) ? patchFile(assetCacheItem.FilePath, (await response.buffer()).toString()) : await response.buffer());
fs.writeFileSync(assetCacheItem.FilePath, /.*\.(js|css)/.test(req.params.file) ? patchFile(assetCacheItem.FilePath, buffer.toString()) : buffer);
}
assetCacheItem.Headers.forEach((value: string, name: string) => {
@ -87,7 +86,7 @@ export default function TestClient(app: Application) {
if (!useTestClient) return res.send("Test client is disabled on this instance. Use a stand-alone client to connect this instance.");
res.send(fs.readFileSync(path.join(__dirname, "..", "..", "..", "assets", "developers.html"), { encoding: "utf8" }));
res.send(fs.readFileSync(TestClientPaths.Developers, { encoding: "utf8" }));
});
app.get("*", (req: Request, res: Response) => {
const { useTestClient } = Config.get().client;
@ -118,7 +117,7 @@ function applyEnv(html: string): string {
function applyPlugins(html: string): string {
// plugins
let files = fs.readdirSync(path.join(AssetsPath, "plugins"));
let files = fs.readdirSync(TestClientPaths.PluginsDir);
let plugins = "";
files.forEach((x) => {
if (x.endsWith(".js")) plugins += `<script src='/assets/plugins/${x}'></script>\n`;
@ -128,7 +127,7 @@ function applyPlugins(html: string): string {
function applyInlinePlugins(html: string): string {
// inline plugins
let files = fs.readdirSync(path.join(AssetsPath, "inline-plugins"));
let files = fs.readdirSync(TestClientPaths.InlinePluginsDir);
let plugins = "";
files.forEach((x) => {
if (x.endsWith(".js")) plugins += `<script src='/assets/inline-plugins/${x}'></script>\n\n`;
@ -138,10 +137,10 @@ function applyInlinePlugins(html: string): string {
function applyPreloadPlugins(html: string): string {
//preload plugins
let files = fs.readdirSync(path.join(AssetsPath, "preload-plugins"));
let files = fs.readdirSync(TestClientPaths.PreloadPluginsDir);
let plugins = "";
files.forEach((x) => {
if (x.endsWith(".js")) plugins += `<script>${fs.readFileSync(path.join(AssetsPath, "preload-plugins", x))}</script>\n`;
if (x.endsWith(".js")) plugins += `<script>${fs.readFileSync(path.join(TestClientPaths.PreloadPluginsDir, x))}</script>\n`;
});
return html.replaceAll("<!-- preload plugin marker -->", plugins);
}
@ -155,7 +154,13 @@ function stripHeaders(headers: Headers): Headers {
"transfer-encoding",
"expect-ct",
"access-control-allow-origin",
"content-encoding"
"content-encoding",
"cf-cache-status",
"cf-ray",
"server",
"etag",
"nel",
"report-to"
].forEach((headerName) => {
headers.delete(headerName);
});

5
src/api/middlewares/Translation.ts
View File

@ -1,3 +1,4 @@
import { Paths } from "@fosscord/util";
import { Router } from "express";
import fs from "fs";
import i18next from "i18next";
@ -6,8 +7,8 @@ import i18nextBackend from "i18next-node-fs-backend";
import path from "path";
export async function initTranslation(router: Router) {
const languages = fs.readdirSync(path.join(__dirname, "..", "..", "..", "assets", "locales"));
const namespaces = fs.readdirSync(path.join(__dirname, "..", "..", "..", "assets", "locales", "en"));
const languages = fs.readdirSync(path.join(Paths.AssetsPath, "locales"));
const namespaces = fs.readdirSync(path.join(Paths.AssetsPath, "locales", "en"));
const ns = namespaces.filter((x) => x.endsWith(".json")).map((x) => x.slice(0, x.length - 5));
await i18next

4
src/api/routes/guilds/#guild_id/widget.png.ts
View File

@ -1,5 +1,5 @@
import { route } from "@fosscord/api";
import { Guild, HTTPError } from "@fosscord/util";
import { Guild, HTTPError, Paths } from "@fosscord/util";
import { Request, Response, Router } from "express";
import fs from "fs";
import path from "path";
@ -39,7 +39,7 @@ router.get("/", route({}), async (req: Request, res: Response) => {
}
// TODO: Widget style templates need Fosscord branding
const source = path.join(__dirname, "..", "..", "..", "..", "..", "assets", "widget", `${style}.png`);
const source = path.join(Paths.AssetsPath, "widget", `${style}.png`);
if (!fs.existsSync(source)) {
throw new HTTPError("Widget template does not exist.", 400);
}

55
src/api/util/TestClientPatcher.ts
View File

@ -1,30 +1,25 @@
import path from "path";
import fs from "fs";
import { Paths, TestClientPaths } from "@fosscord/util";
console.log('[TestClient] Loading private assets...');
const privateAssetsRoot = path.join(__dirname, "..", "..", "..", "assets", "private");
const iconsRoot = path.join(privateAssetsRoot, "icons");
const icons = new Map<string, Buffer>();
function readAssets(): Map<string, Buffer> {
const icons = new Map<string, Buffer>();
fs.readdirSync(Paths.IconPath).forEach(file => {
const fileName = path.basename(file);
//check if dir
if(fs.lstatSync(path.join(Paths.IconPath, file)).isDirectory()){
return;
}
if(fs.existsSync(path.join(Paths.CustomIconPath, fileName)))
icons.set(fileName,fs.readFileSync(path.join(Paths.CustomIconPath, fileName)) as Buffer);
else
icons.set(fileName,fs.readFileSync(path.join(Paths.IconPath, fileName)) as Buffer);
});
fs.readdirSync(iconsRoot).forEach(file => {
const fileName = path.basename(file);
//check if dir
if(fs.lstatSync(path.join(iconsRoot, file)).isDirectory()){
return;
}
icons.set(fileName,fs.readFileSync(path.join(iconsRoot,file)) as Buffer);
});
fs.readdirSync(path.join(iconsRoot, "custom")).forEach(file => {
const fileName = path.basename(file);
if(fs.lstatSync(path.join(iconsRoot,"custom", file)).isDirectory()){
return;
}
icons.set(fileName,fs.readFileSync(path.join(iconsRoot,"custom",file)) as Buffer);
});
console.log('[TestClient] Patcher ready!');
return icons;
}
export function patchFile(filePath: string, content: string): string {
console.log(`[TestClient] Patching ${filePath}`);
@ -32,10 +27,12 @@ export function patchFile(filePath: string, content: string): string {
content = prettier(filePath, content);
content = autoPatch(filePath, content);
content = applyPatches(filePath, content);
console.log(`[TestClient] Patched ${filePath} in ${Date.now() - startTime}ms`);
return content;
}
function prettier(filePath: string, content: string): string{
let prettier = require("prettier");
let parser;
@ -91,7 +88,7 @@ function autoPatch(filePath: string, content: string): string{
content = content.replaceAll(/--brand-experiment-(\d{1,4}): hsl\(235/g, '--brand-experiment-\$1: hsl(var(--brand-hue)')
//logos
content = content.replace(/d: "M23\.0212.*/, `d: "${icons.get("homeIcon.path")!.toString()}"`);
content = content.replace(/d: "M23\.0212.*/, `d: "${readAssets().get("homeIcon.path")!.toString()}"`);
content = content.replace('width: n, height: o, viewBox: "0 0 28 20"', 'width: 48, height: 48, viewBox: "0 0 48 48"');
//undo webpacking
@ -100,8 +97,18 @@ function autoPatch(filePath: string, content: string): string{
content = content.replace(/!1/g, "false");
// - real esmodule defs
content = content.replace(/Object.defineProperty\((.), "__esModule", { value: (.*) }\);/g, '\$1.__esModule = \$2;');
console.log(`[TestClient] Autopatched ${path.basename(filePath)}!`);
return content;
}
function applyPatches(filePath: string, content: string): string{
//get files in testclient_patches
const patches = fs.readdirSync(TestClientPaths.PatchDir);
for(let patch of patches){
//apply patch with git patch
const patchPath = path.join(TestClientPaths.PatchDir, patch);
}
return content;
}

4
src/api/util/handlers/route.ts
View File

@ -5,6 +5,7 @@ import {
FosscordApiErrors,
getPermission,
getRights,
Paths,
PermissionResolvable,
Permissions,
RightResolvable,
@ -17,8 +18,7 @@ import { NextFunction, Request, Response } from "express";
import fs from "fs";
import path from "path";
const SchemaPath = path.join(__dirname, "..", "..", "..", "..", "assets", "schemas.json");
const schemas = JSON.parse(fs.readFileSync(SchemaPath, { encoding: "utf8" }));
const schemas = JSON.parse(fs.readFileSync(Paths.SchemaPath, { encoding: "utf8" }));
export const ajv = new Ajv({
allErrors: true,

8
src/cdn/util/Storage.ts
View File

@ -5,6 +5,7 @@ import { S3 } from "@aws-sdk/client-s3";
import fs from "fs";
import { bgCyan, black } from "picocolors";
import { S3Storage } from "./S3Storage";
import { Paths } from "@fosscord/util";
process.cwd();
export interface Storage {
@ -16,12 +17,7 @@ export interface Storage {
let storage: Storage;
if (process.env.STORAGE_PROVIDER === "file" || !process.env.STORAGE_PROVIDER) {
let location = process.env.STORAGE_LOCATION;
if (location) {
location = path.resolve(location);
} else {
location = path.join(process.cwd(), "files");
}
let location = Paths.CDNFilePath;
console.log(`[CDN] storage location: ${bgCyan(`${black(location)}`)}`);
//fse.ensureDirSync(location);
fs.mkdirSync(location, { recursive: true });

5
src/util/util/Database.ts
View File

@ -5,6 +5,7 @@ import { green, red, yellow } from "picocolors";
import { exit } from "process";
import "reflect-metadata";
import { DataSource, DataSourceOptions, PrimaryColumn, PrimaryGeneratedColumn } from "typeorm";
import { Paths } from ".";
import * as Models from "../entities";
import { BaseClass, BaseClassWithoutId } from "../entities";
@ -41,7 +42,7 @@ function getDataSourceOptions(): DataSourceOptions {
const dbConnectionString = process.env.DATABASE || path.join(process.cwd(), "database.db");
const type = dbConnectionString.includes("://") ? dbConnectionString.split(":")[0]?.replace("+srv", "") : ("sqlite" as any);
const isSqlite = type.includes("sqlite");
const migrationsExist = fs.existsSync(path.join(__dirname, "..", "migrations", type));
const migrationsExist = fs.existsSync(path.join(Paths.MigrationsRoot, type));
//read env vars
const synchronizeInsteadOfMigrations = "DB_UNSAFE" in process.env;
const verboseDb = "DB_VERBOSE" in process.env;
@ -94,7 +95,7 @@ function getDataSourceOptions(): DataSourceOptions {
bigNumberStrings: false,
supportBigNumbers: true,
name: "default",
migrations: synchronizeInsteadOfMigrations ? [] : [path.join(__dirname, "..", "migrations", type, "*.js")],
migrations: synchronizeInsteadOfMigrations ? [] : [path.join(Paths.MigrationsRoot, type, "*.js")],
migrationsRun: !synchronizeInsteadOfMigrations,
applicationName: `Fosscord Server`,
} as DataSourceOptions;

59
src/util/util/Paths.ts Normal file
View File

@ -0,0 +1,59 @@
import path from "path";
import fs from "fs";
export class ProjectPaths {
public static RuntimePath = path.join(__dirname, "..", "..");
public static ProjectRoot = path.join(ProjectPaths.RuntimePath, "..");
public static DataDir = path.join(ProjectPaths.ProjectRoot, "data");
public static ApiPath = path.join(ProjectPaths.RuntimePath, "api");
public static CdnPath = path.join(ProjectPaths.RuntimePath, "cdn");
public static GatewayPath = path.join(ProjectPaths.RuntimePath, "gateway");
public static UtilPath = path.join(ProjectPaths.RuntimePath, "util");
}
export class Paths {
public static AssetsPath = path.join(ProjectPaths.ProjectRoot, "assets");
public static PrivateAssetsPath = path.join(ProjectPaths.DataDir, "assets");
public static MigrationsRoot = path.join(ProjectPaths.UtilPath, "migrations");
public static CDNFilePath = path.resolve(process.env.STORAGE_LOCATION || path.join(ProjectPaths.ProjectRoot, "files"));
public static SchemaPath = path.join(ProjectPaths.DataDir, "schemas.json");
public static IconPath = path.join(Paths.AssetsPath, "icons");
public static CustomIconPath = path.join(Paths.AssetsPath, "icons", "custom");
}
export class TestClientPaths {
public static TestClientRoot = path.join(ProjectPaths.DataDir, "test-client");
public static TestClientCacheDir = process.env.TEST_CLIENT_CACHE_DIR || process.env.ASSET_CACHE_DIR || path.join(TestClientPaths.TestClientRoot, "cache");
public static Index = path.join(TestClientPaths.TestClientRoot, "index.html");
public static Developers = path.join(TestClientPaths.TestClientRoot, "developers.html");
public static PatchDir = path.join(TestClientPaths.TestClientRoot, "patches");
public static CacheDir = TestClientPaths.TestClientCacheDir;
public static CacheIndex = path.join(TestClientPaths.TestClientCacheDir, "index.json");
public static PluginsDir = path.join(TestClientPaths.TestClientRoot, "plugins");
public static PreloadPluginsDir = path.join(TestClientPaths.TestClientRoot, "preload-plugins");
public static InlinePluginsDir = path.join(TestClientPaths.TestClientRoot, "inline-plugins");
}
//warnings
if(process.env.ASSET_CACHE_DIR) console.log(`[ENV/WARN] ASSET_CACHE_DIR is deprecated, please use TEST_CLIENT_CACHE_DIR instead!`);
for(let key in ProjectPaths) {
if(!fs.existsSync((ProjectPaths as any)[key])) {
console.error(`[ERROR] ${(ProjectPaths as any)[key]} does not exist!`);
}
}
for(let key in Paths) {
if(!fs.existsSync((Paths as any)[key])) {
console.error(`[ERROR] ${(Paths as any)[key]} does not exist!`);
}
}
for(let key in TestClientPaths) {
if(!fs.existsSync((TestClientPaths as any)[key])) {
console.error(`[ERROR] ${(TestClientPaths as any)[key]} does not exist!`);
}
}

1
src/util/util/index.ts
View File

@ -23,3 +23,4 @@ export * from "./Snowflake";
export * from "./String";
export * from "./Token";
export * from "./TraverseDirectory";
export * from "./Paths";