1
0
mirror of https://github.com/spacebarchat/server.git synced 2024-09-20 01:31:34 +02:00

TestClient middleware will now add cache misses to file for generate:client script to fetch. Also patch discord icon top left

This commit is contained in:
Madeline 2022-10-24 16:50:22 +11:00
parent fc2ccc0b25
commit 0f9951bb0e
No known key found for this signature in database
GPG Key ID: 1958E017C36F2E47
3 changed files with 250 additions and 23 deletions

162
assets/cacheMisses Normal file
View File

@ -0,0 +1,162 @@
6864292b0e84deec2d7c.worker.js
40532.ccd7077c8311c798fcff.css
7f18f1d5ab6ded7cf71bbc1f907ee3d4.woff2
f9e7047f6447547781512ec4b977b2ab.woff2
21070f52a8a6a61edef9785eaf303fb8.woff2
24bda95d153a319704c33329f3ab84bb.woff2
24bda95d153a319704c33329f3ab84bb.woff2
df91c98e67b326426141.worker.js
9491a199bd318f1fc38711fb5067f401.woff2
cf4a8a10bbdf9b775fad41e0b9921c84.woff2
ed2007aab2da31a5436e70a28b4d59f9.svg
91dcabd038a2e07ea6fbe7ddb625ecfb.woff2
3c6ccb83716d1e4fb91d3082f6b21d77.png
35827c42a3af54bd169cff0a866eb428.svg
f9c8fcfa7dcc5d6a291a6ebbe3bb847e.woff2
515d0236bce3ba834026209dc78489f0.svg
a532e755c36b75762a75ff0794456efb.svg
7e5013a9afc1404b0b89d99aaec0b398.png
7d66dfcf8e39f27f163fba8d79577fd8.woff2
be71475205180432aaa6a89bd8ee306c.svg
364cec2cc5dfb64873e3a33626a56ba1.svg
40aa3feae87e9745dd56756762029544.svg
112667acf24cbb3d067ccbc54e3638de.svg
0.5a7dcdb1f77d207c0668.css
e6d6b255259ac878d00819a9555072ad.png
9a31e0f65d520cc12d7f42374d59a2d1.svg
b5eb2f7d6b3f8cc9b60be4a5dcf28015.svg
b36de980b174d7b798c89f35c116e5c6.svg
b36c705f790dad253981f1893085015a.svg
8c998f8fb62016fcfb4901e424ff378b.svg
317438b1df555820dfaffad6074dbf72.svg
8e180c91237bcacc8b7ff5663262c1b7.svg
360ca22059b6b674a018547e8e18acdc.svg
2a7b069c2413090d8e17fedbf1647280.png
73eea46fc7df629b7f5b2b6fed926985.png
afc30837a86d8c6aaee02fdfa2787a11.svg
b60ecd82d8fa5a8e21739e7025692429.svg
46f302822fa43628962d28f50bb33b01.svg
6d70d302ed36f6ac6f97341027611e44.svg
39a7fb525b2593689d328ad7367ee4b5.png
487594776f6ebfe57e3ac89d6b3fe437.png
d14825a72be9bc05e41a5579c03d8e94.png
0f67189cc579386633ab41a98088cc8e.png
dbca7d266dfd0c9bcf90013cb4a2814b.svg
1b30c341ef8e07222e5fbb4982b15fed.svg
4f2de8336d160442c074045047924e95.svg
9aaa6213fb0f5a266ceeb9395375ef8e.svg
dced41c080d0a15787074c0c828b094c.svg
f0b5aa3811bdd1f1c230a55a9dae5c55.svg
de09c1a167a87741246b9f3ec239e0ea.svg
acdcdf7b374842a2904df0c9bf4cea9f.svg
4505cadcbd8107f1b72d0321f6800de5.svg
6e59980470e93127806be9304b7dc2ec.svg
3ed9c7cdbea131bc485afa998a5ec93e.svg
1a23068e966efce6fb32ca9fecc65eed.svg
8b7165c3ca8f930bf6a1cf4ad12de85a.svg
6ac195d9605dfb0719e73f002fad38a8.svg
dea5252276408a8bfca6dda585ca5216.svg
f303eeb986430817ee8a52a9b81aea45.svg
fde233b6899a1a0fac64c419724e668b.svg
b7b57e9fa6377409691f498d5d48714e.svg
a053c2e82b0c5e6a4cc853ca846cc000.svg
2cb8e7219e1554e094c4a0316e0ab2c2.svg
45d6946387a0c66f4eb4e62a6e7758ea.svg
5d8898dd9356f25901bae20fc8c980d9.svg
3c6ccb83716d1e4fb91d3082f6b21d77.png
67430d140a79af95585a19d304731dc5.svg
24d05f3b46a110e538674edbac0db4cd.svg
f5de0f917504a81e039f.worklet.js
3c6ccb83716d1e4fb91d3082f6b21d77.png
3be860e37bf5f2df40ad15d2de5cee0e.png
4a1000a95b1aad334e98f9d15b9d0ec4.svg
c115d59ca13c0f942965a82a0f05bf01.svg
db9e2406158133f91e1fe24bb56425c2.svg
37f15a06d062bdd0fc252d2b35d0b0bb.svg
32045743cb7670855ae726743c949e0c.svg
f3224f560c8f5974355596a4f7e9ce19.svg
6a853b4c87fce386cbfef4a2efbacb09.svg
eaeac24163b35f7526704a3d9b3c7722.svg
d897626dfa2016ea3ad0af935acb6070.svg
ca71e0b8818221eea1deebbaf8dc6518.svg
85cf2b49d2a185c98ec8e383ad5a05d6.svg
57ee9535485efae7eb923ed4893abb57.svg
1fa6585cbb8c0fe21188e26d19fc6609.svg
f1e026332f1d6e902bcbbc07892bc849.svg
4da5a1105f56c7344ec198e25e99ce1b.svg
c7f26927db5e7806790f4e968038630a.svg
c4cccc083da07eb95edf161ad5726d00.svg
1f0bfc0865d324c2587920a7d80c609b.png
41a1814ec63404eb722ce88d1c28b4f4.png
ca9e397003df8b46709315d7a248c7de.svg
da3651e59d6006dfa5fa07ec3102d1f3.svg
3590df6f2ae2f7202dab15c0bd3aca9a.png
e6d6b255259ac878d00819a9555072ad.png
7fa2adf98f26db34178bb30a63dabe8c.png
6bf3cf5d375d7561cb3b9bf74212ed45.png
10bb7b2e55f0a34f23d903121de6b9bc.png
8d1d548a64761f0c5b1d7c9e00ae66a6.png
894cceea2dd5b523936930d1d7e333c5.png
2ac1239c26c4ae1d27817a9d7b85dc53.png
d24e290703f8b31b56744a69df613fcb.png
ab02db863b7edeaa46bf4cd49b6646a9.png
d19290ba3158e138bb241ae669a3bc37.png
4db0790f7a81e49025d7fbfb9aeb182c.png
f23c5c28c4429691f7c54af93876d661.png
44a7840b2161275358742595c9257e76.png
8e1cf1f1cf2a1a917002b8b583270c32.png
b2da62f020089ccee92860e4defafdb4.png
b1309f8892f138383d8b0b6ff8e23463.png
da07da4bde6f81f16366b62e8fcc90ec.png
7beab7b17eaa9ff7ceed3e5b1af274c2.png
78cbfbf0381b8c7f5a192c4a46bd0b0e.png
d5073ab2ca9ee7c06c3f4d761968ac44.png
d52ce383ca6d8eb53588bcc042574cae.png
02e75e392ab5d5a8ed5ab4f8fcae9c77.png
c9f51873ae719a6b4b8c6724362e999e.png
64f37efd5319b9b581557604864f042a.png
fcdf14841cd468de3f43704be16fa303.png
08d0c70a708cd25acecb7a8b0cb0eb23.png
cfc9643cb00e44fae64bfeda3556bfd9.png
7eb9487d4dac00095f8ed2d2c80b21a8.png
cbf50c2e3287d2118f741e827a3ddaf5.png
c11b69c74029b5deaca9ca7ce1e987b1.svg
beec347635d7597215747fcaf461e569.svg
5e1649183589023fb24667a369af585d.woff2
46b24114f7276738329be3ae84e22da4.svg
c970061def69b8226617841ca3089219.svg
b13ade832d7f19e30c5d2452b6cbdd53.svg
6864292b0e84deec2d7c.worker.js
40532.ccd7077c8311c798fcff.css
7f18f1d5ab6ded7cf71bbc1f907ee3d4.woff2
f9e7047f6447547781512ec4b977b2ab.woff2
21070f52a8a6a61edef9785eaf303fb8.woff2
24bda95d153a319704c33329f3ab84bb.woff2
df91c98e67b326426141.worker.js
9491a199bd318f1fc38711fb5067f401.woff2
ed2007aab2da31a5436e70a28b4d59f9.svg
cf4a8a10bbdf9b775fad41e0b9921c84.woff2
91dcabd038a2e07ea6fbe7ddb625ecfb.woff2
3c6ccb83716d1e4fb91d3082f6b21d77.png
515d0236bce3ba834026209dc78489f0.svg
35827c42a3af54bd169cff0a866eb428.svg
a532e755c36b75762a75ff0794456efb.svg
7e5013a9afc1404b0b89d99aaec0b398.png
f9c8fcfa7dcc5d6a291a6ebbe3bb847e.woff2
be71475205180432aaa6a89bd8ee306c.svg
7d66dfcf8e39f27f163fba8d79577fd8.woff2
40aa3feae87e9745dd56756762029544.svg
364cec2cc5dfb64873e3a33626a56ba1.svg
112667acf24cbb3d067ccbc54e3638de.svg
3c6ccb83716d1e4fb91d3082f6b21d77.png
24d05f3b46a110e538674edbac0db4cd.svg
3c6ccb83716d1e4fb91d3082f6b21d77.png
2e16625f6f55b96572f79cc35682266f.png
4a1000a95b1aad334e98f9d15b9d0ec4.svg
b5eb2f7d6b3f8cc9b60be4a5dcf28015.svg
750a8ead5eca8745b14658a4ba5b0128.svg
897fb794f533bd92e32b8b06676122ba.gif
65921aa740e7c73ace1a74b54b9e6b82.gif
8cdb96f2d79587fc380142dd132acf45.gif
81f855741b7bdf09d97f8ea644dac150.gif
1dabb8ee5d7cb5137c06f958e0091573.svg

View File

@ -1,8 +1,15 @@
const path = require("path");
const fetch = require("node-fetch");
const http = require('http');
const https = require('https');
const fs = require("fs/promises");
const { existsSync } = require("fs");
// https://stackoverflow.com/a/62500224
const httpAgent = new http.Agent({ keepAlive: true });
const httpsAgent = new https.Agent({ keepAlive: true });
const agent = (_parsedURL) => _parsedURL.protocol == 'http:' ? httpAgent : httpsAgent;
const CACHE_PATH = path.join(__dirname, "..", "assets", "cache");
const BASE_URL = "https://discord.com";
@ -44,14 +51,17 @@ const doPatch = (content) => {
content.replaceAll('"Server', '"Guild');
// //change some vars
// content = content.replace('dsn: "https://fa97a90475514c03a42f80cd36d147c4@sentry.io/140984"', "dsn: (/true/.test(localStorage.sentryOptIn)?'https://6bad92b0175d41a18a037a73d0cff282@sentry.thearcanebrony.net/12':'')");
// content = content.replace('t.DSN = "https://fa97a90475514c03a42f80cd36d147c4@sentry.io/140984"', "t.DSN = (/true/.test(localStorage.sentryOptIn)?'https://6bad92b0175d41a18a037a73d0cff282@sentry.thearcanebrony.net/12':'')");
// content = content.replace('--brand-experiment: hsl(235, calc(var(--saturation-factor, 1) * 85.6%), 64.7%);', '--brand-experiment: hsl(var(--brand-hue), calc(var(--saturation-factor, 1) * 85.6%), 50%);');
// content = content.replaceAll(/--brand-experiment-(\d{1,4}): hsl\(235/g, '--brand-experiment-\$1: hsl(var(--brand-hue)');
content = content.replace('dsn: "https://fa97a90475514c03a42f80cd36d147c4@sentry.io/140984"', "dsn: (/true/.test(localStorage.sentryOptIn)?'https://6bad92b0175d41a18a037a73d0cff282@sentry.thearcanebrony.net/12':'')");
content = content.replace('t.DSN = "https://fa97a90475514c03a42f80cd36d147c4@sentry.io/140984"', "t.DSN = (/true/.test(localStorage.sentryOptIn)?'https://6bad92b0175d41a18a037a73d0cff282@sentry.thearcanebrony.net/12':'')");
content = content.replace('--brand-experiment: hsl(235, calc(var(--saturation-factor, 1) * 85.6%), 64.7%);', '--brand-experiment: hsl(var(--brand-hue), calc(var(--saturation-factor, 1) * 85.6%), 50%);');
content = content.replaceAll(/--brand-experiment-(\d{1,4}): hsl\(235/g, '--brand-experiment-\$1: hsl(var(--brand-hue)');
//logos
// content = content.replace(/d: "M23\.0212.*/, `d: "${icons.get("homeIcon.path")!.toString()}"`);
// content = content.replace('width: n, height: o, viewBox: "0 0 28 20"', 'width: 48, height: 48, viewBox: "0 0 48 48"');
content = content.replace(
"M23.0212 1.67671C21.3107 0.879656 19.5079 0.318797 17.6584 0C17.4062 0.461742 17.1749 0.934541 16.9708 1.4184C15.003 1.12145 12.9974 1.12145 11.0283 1.4184C10.819 0.934541 10.589 0.461744 10.3368 0.00546311C8.48074 0.324393 6.67795 0.885118 4.96746 1.68231C1.56727 6.77853 0.649666 11.7538 1.11108 16.652C3.10102 18.1418 5.3262 19.2743 7.69177 20C8.22338 19.2743 8.69519 18.4993 9.09812 17.691C8.32996 17.3997 7.58522 17.0424 6.87684 16.6135C7.06531 16.4762 7.24726 16.3387 7.42403 16.1847C11.5911 18.1749 16.408 18.1749 20.5763 16.1847C20.7531 16.3332 20.9351 16.4762 21.1171 16.6135C20.41 17.0369 19.6639 17.3997 18.897 17.691C19.3052 18.4993 19.7718 19.2689 20.3021 19.9945C22.6677 19.2689 24.8929 18.1364 26.8828 16.6466H26.8893C27.43 10.9731 25.9665 6.04728 23.0212 1.67671ZM9.68041 13.6383C8.39754 13.6383 7.34085 12.4453 7.34085 10.994C7.34085 9.54272 8.37155 8.34973 9.68041 8.34973C10.9893 8.34973 12.0395 9.54272 12.0187 10.994C12.0187 12.4453 10.9828 13.6383 9.68041 13.6383ZM18.3161 13.6383C17.0332 13.6383 15.9765 12.4453 15.9765 10.994C15.9765 9.54272 17.0124 8.34973 18.3161 8.34973C19.6184 8.34973 20.6751 9.54272 20.6543 10.994C20.6543 12.4453 19.6184 13.6383 18.3161 13.6383Z",
"M 0,0 47.999993,2.7036528e-4 C 48.001796,3.3028172 47.663993,6.5968018 46.991821,9.8301938 43.116101,28.454191 28.452575,43.116441 9.8293509,46.992163 6.5960834,47.664163 3.3023222,48.001868 0,47.999992 Z m 9.8293509,28.735114 v 9.248482 C 22.673599,33.047696 32.857154,22.749268 37.63852,9.829938 H 9.8293509 v 8.679899 H 22.931288 c -3.554489,3.93617 -7.735383,7.257633 -12.373436,9.829938 -0.241031,0.133684 -0.483864,0.265492 -0.7285011,0.395339 z"
);
content = content.replace('width:n,height:o,viewBox:"0 0 28 20"', 'width:48,height:48,viewBox:"0 0 48 48"');
//save some time on load resolving asset urls...
content = content.replaceAll(
@ -72,7 +82,7 @@ const doPatch = (content) => {
content = content.replaceAll(
"https://itunes.apple.com/app/discord/id985746746",
"https://slowcord.understars.dev/api/download?platform=ios"
)
);
// TODO change public test build link
// content = content.replaceAll(
@ -84,15 +94,26 @@ const doPatch = (content) => {
};
const processFile = async (name) => {
const res = await fetch(`${BASE_URL}/assets/${name}.js`);
if (res.status !== 200) return [];
const res = await fetch(`${BASE_URL}/assets/${name}${name.includes(".") ? "" : ".js"}`, {
agent,
});
if (res.status !== 200) {
console.log(name, res.status);
return [];
};
if (name.includes(".") && !name.includes(".js") && !name.includes(".css")) {
await fs.writeFile(path.join(CACHE_PATH, name), await res.buffer());
return [];
}
let text = await res.text();
text = doPatch(text);
await fs.writeFile(path.join(CACHE_PATH, `${name}.js`), text);
await fs.writeFile(path.join(CACHE_PATH, `${name}${name.includes(".") ? "" : ".js"}`), text);
return [...new Set(text.match(/[A-Fa-f0-9]{20}/g))];
return [...new Set(text.match(/\"[A-Fa-f0-9]{20}\"/g))].map(x => x.replaceAll("\"", ""));
};
(async () => {
@ -119,6 +140,45 @@ const processFile = async (name) => {
process.stdout.moveCursor(0, 1);
const CACHE_MISSES = (await fs.readFile(path.join(CACHE_PATH, "..", "cacheMisses"))).toString().split("\n");
while (CACHE_MISSES.length > 0) {
const asset = CACHE_MISSES.shift();
process.stdout.clearLine(0);
process.stdout.cursorTo(0);
process.stdout.write(
`Scraping cache misses ${asset}. Remaining: ${CACHE_MISSES.length}`,
);
if (existsSync(path.join(CACHE_PATH, `${asset}`))) {
continue;
}
const newAssets = await processFile(asset);
assets.push(...newAssets);
}
process.stdout.moveCursor(0, 1);
var existing = await fs.readdir(CACHE_PATH);
while (existing.length > 0) {
var file = existing.shift();
process.stdout.clearLine(0);
process.stdout.cursorTo(0);
process.stdout.write(
`Patching existing ${file}. Remaining: ${existing.length}.`,
);
var text = (await fs.readFile(path.join(CACHE_PATH, file)));
if (file.includes(".js") || file.includes(".css")) {
text = doPatch(text.toString());
await fs.writeFile(path.join(CACHE_PATH, file), text.toString());
assets.push(...[...new Set(text.match(/\"[A-Fa-f0-9]{20}\"/g))].map(x => x.replaceAll("\"", "")));
}
}
process.stdout.moveCursor(0, 1);
let rates = [];
let lastFinished = Date.now();
let previousFinish = Date.now();
@ -144,13 +204,14 @@ const processFile = async (name) => {
`Caching asset ${asset}. ` +
`${i}/${assets.length - 1} = ${Math.floor(
(i / (assets.length - 1)) * 100,
)}% ` +
`Finish at: ${new Date(
Date.now() + finishTime,
).toLocaleTimeString()}`,
)}% `
// + `Finish at: ${new Date(
// Date.now() + finishTime,
// ).toLocaleTimeString()}`,
);
promises.push(processFile(asset));
// await processFile(asset);
if (promises.length > 100) {
const values = await Promise.all(promises);

View File

@ -13,7 +13,7 @@ export default function TestClient(app: Application) {
const agent = new ProxyAgent();
const assetCache = new Map<
string,
{ response: FetchResponse; buffer: Buffer }
{ response: FetchResponse; buffer: Buffer; }
>();
const indexHTML = fs.readFileSync(
path.join(ASSET_FOLDER_PATH, "client_test", "index.html"),
@ -76,13 +76,7 @@ export default function TestClient(app: Application) {
app.use("/assets", express.static(path.join(ASSET_FOLDER_PATH, "cache")));
app.get("/assets/:file", async (req: Request, res: Response) => {
if (!hasWarnedAboutCache) {
hasWarnedAboutCache = true;
if (req.params.file.includes(".js"))
console.warn(
`[TestClient] Cache miss for file ${req.params.file}! Use 'npm run generate:client' to cache and patch.`,
);
}
if (req.params.file.includes(".map")) return res.status(404);
delete req.headers.host;
var response: FetchResponse;
@ -124,6 +118,16 @@ export default function TestClient(app: Application) {
});
assetCache.set(req.params.file, { buffer, response });
if (response.status == 200) {
// if (!hasWarnedAboutCache) {
hasWarnedAboutCache = true;
console.warn(
`[TestClient] Cache miss for file ${req.params.file}! Use 'npm run generate:client' to cache and patch.`,
);
await fs.promises.appendFile(path.join(ASSET_FOLDER_PATH, "cacheMisses"), req.params.file + "\n");
// }
}
return res.send(buffer);
});
app.get("/developers*", (req: Request, res: Response) => {