Implement base logic for index page

Signed-off-by: TheArcaneBrony <myrainbowdash949@gmail.com>

.idea/.gitignore

@@ -0,0 +1,5 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/

.idea/codeStyles/Project.xml

@@ -0,0 +1,6 @@
+<component name="ProjectCodeStyleConfiguration">
+  <code_scheme name="Project" version="173">
+    <option name="WRAP_WHEN_TYPING_REACHES_RIGHT_MARGIN" value="true" />
+    <option name="ENABLE_SECOND_REFORMAT" value="true" />
+  </code_scheme>
+</component>

.idea/codeStyles/codeStyleConfig.xml

@@ -0,0 +1,5 @@
+<component name="ProjectCodeStyleConfiguration">
+  <state>
+    <option name="USE_PER_PROJECT_SETTINGS" value="true" />
+  </state>
+</component>

.idea/copyright/Fosscord.xml

@@ -0,0 +1,8 @@
+<component name="CopyrightManager">
+  <copyright>
+    <option name="allowReplaceRegexp" value="Fosscord: A FOSS" />
+    <option name="keyword" value="(Copyright|@fc-license-skip)" />
+    <option name="notice" value="Fosscord: A FOSS re-implementation and extension of the Discord.com backend.&#10;Copyright (C) &amp;#36;originalComment.match(&quot;Copyright \(c\) (\d+)&quot;, 1, &quot;-&quot;, &quot;&amp;#36;today.year&quot;)&amp;#36;today.year Fosscord and Fosscord Contributors&#10;&#10;This program is free software: you can redistribute it and/or modify&#10;it under the terms of the GNU Affero General Public License as published&#10;by the Free Software Foundation, either version 3 of the License, or&#10;(at your option) any later version.&#10;&#10;This program is distributed in the hope that it will be useful,&#10;but WITHOUT ANY WARRANTY; without even the implied warranty of&#10;MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the&#10;GNU Affero General Public License for more details.&#10;&#10;You should have received a copy of the GNU Affero General Public License&#10;along with this program.  If not, see &lt;https://www.gnu.org/licenses/&gt;." />
+    <option name="myName" value="Fosscord" />
+  </copyright>
+</component>

.idea/copyright/profiles_settings.xml

@@ -0,0 +1,10 @@
+<component name="CopyrightManager">
+  <settings default="Fosscord">
+    <module2copyright>
+      <element module="Project Files" copyright="Fosscord" />
+    </module2copyright>
+    <LanguageOptions name="JavaScript">
+      <option name="fileTypeOverride" value="3" />
+    </LanguageOptions>
+  </settings>
+</component>

.idea/fosscord-server.iml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="WEB_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$">
+      <excludeFolder url="file://$MODULE_DIR$/temp" />
+      <excludeFolder url="file://$MODULE_DIR$/.tmp" />
+      <excludeFolder url="file://$MODULE_DIR$/tmp" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.idea/inspectionProfiles/Project_Default.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="Eslint" enabled="true" level="WARNING" enabled_by_default="true" />
+  </profile>
+</component>

.idea/jsLinters/eslint.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="EslintConfiguration">
+    <option name="fix-on-save" value="true" />
+  </component>
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/fosscord-server.iml" filepath="$PROJECT_DIR$/.idea/fosscord-server.iml" />
+    </modules>
+  </component>
+</project>

.idea/prettier.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="PrettierConfiguration">
+    <option name="myRunOnSave" value="true" />
+    <option name="myRunOnReformat" value="true" />
+  </component>
+</project>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="" vcs="Git" />
+  </component>
+</project>

src/index/Server.ts

@@ -0,0 +1,125 @@
+/*
+ * Fosscord: A FOSS re-implementation and extension of the Discord.com backend.
+ * Copyright (C) 2023 Fosscord and Fosscord Contributors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import {
+	Config,
+	initDatabase,
+	JSONReplacer,
+	registerRoutes,
+	Sentry,
+} from "@fosscord/util";
+import { Request, Response, Router } from "express";
+import { Server, ServerOptions } from "lambert-server";
+import "missing-native-js-functions";
+import morgan from "morgan";
+import path from "path";
+import { red } from "picocolors";
+import { CORS, ErrorHandler, initRateLimits } from "@fosscord/api";
+import { initTranslation } from "../api/middlewares/Translation";
+
+export type FosscordServerOptions = ServerOptions;
+
+declare global {
+	// eslint-disable-next-line @typescript-eslint/no-namespace
+	namespace Express {
+		interface Request {
+			server: FosscordServer;
+		}
+	}
+}
+
+export class FosscordServer extends Server {
+	public declare options: FosscordServerOptions;
+
+	constructor(opts?: Partial<FosscordServerOptions>) {
+		// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+		// @ts-ignore
+		super({ ...opts, errorHandler: false, jsonBody: false });
+	}
+
+	async start() {
+		await initDatabase();
+		await Config.init();
+		await Sentry.init(this.app);
+
+		const logRequests = process.env["LOG_REQUESTS"] != undefined;
+		if (logRequests) {
+			this.app.use(
+				morgan("combined", {
+					skip: (req, res) => {
+						let skip = !(
+							process.env["LOG_REQUESTS"]?.includes(
+								res.statusCode.toString(),
+							) ?? false
+						);
+						if (process.env["LOG_REQUESTS"]?.charAt(0) == "-")
+							skip = !skip;
+						return skip;
+					},
+				}),
+			);
+		}
+
+		this.app.set("json replacer", JSONReplacer);
+
+		this.app.use(CORS);
+
+		const app = this.app;
+		const api = Router();
+		// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+		// @ts-ignore
+		this.app = api;
+
+		await initRateLimits(api);
+		await initTranslation(api);
+
+		this.routes = await registerRoutes(
+			this,
+			path.join(__dirname, "routes", "/"),
+		);
+
+		// 404 is not an error in express, so this should not be an error middleware
+		// this is a fine place to put the 404 handler because its after we register the routes
+		// and since its not an error middleware, our error handler below still works.
+		api.use("*", (req: Request, res: Response) => {
+			res.status(404).json({
+				message: "404 endpoint not found",
+				code: 0,
+			});
+		});
+
+		this.app = app;
+
+		//app.use("/__development", )
+		//app.use("/__internals", )
+		app.use("/index", api);
+
+		this.app.use(ErrorHandler);
+
+		Sentry.errorHandler(this.app);
+
+		if (logRequests)
+			console.log(
+				red(
+					`Warning: Request logging is enabled! This will spam your console!\nTo disable this, unset the 'LOG_REQUESTS' environment variable!`,
+				),
+			);
+
+		return super.start();
+	}
+}

src/index/routes/channels/#channel_id/index.ts

@@ -0,0 +1,116 @@
+/*
+ * Fosscord: A FOSS re-implementation and extension of the Discord.com backend.
+ * Copyright (C) 2023 Fosscord and Fosscord Contributors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import { Channel, Message, User } from "@fosscord/util";
+import { Request, Response, Router } from "express";
+import { route } from "@fosscord/api";
+
+const router: Router = Router();
+// TODO: delete channel
+// TODO: Get channel
+
+router.get("/", route({}), async (req: Request, res: Response) => {
+	const { channel_id } = req.params;
+	const chunk_size = Number(req.query.chunk_size) || 100;
+	if (chunk_size > 1000)
+		return res
+			.status(400)
+			.send({ message: "chunk_size must be <= 1000", code: 50035 });
+
+	// Alicia - get channel
+	const channel = await Channel.findOne({
+		where: { id: channel_id },
+		relations: ["guild"],
+	});
+	// Alicia - if channel not found, or not indexable...
+	if (!channel || !channel.indexable)
+		return res.status(404).send({
+			message: "Unknown Channel (Is it indexable?)",
+			code: 10003,
+		});
+
+	res.setHeader("Transfer-Encoding", "chunked");
+	// Alicia - base body...
+	res.write("context and info here...\n");
+
+	// Alicia - get messages
+	console.time("get channel");
+	console.log("Creating query builders...");
+	const messageQueryBuilder = await Message.createQueryBuilder();
+	const userQueryBuilder = await User.createQueryBuilder();
+	console.log(`Querying messages for channel ${channel_id}...`);
+	const msg_author_ids = await messageQueryBuilder
+		.where("channel_id = :channel_id", { channel_id })
+		.orderBy("id::int8", "DESC")
+		.select("Message.id, Message.author_id")
+		.getRawMany();
+
+	const message_ids = msg_author_ids.map((message) => message.id);
+	const author_ids = msg_author_ids.map((message) => message.author_id);
+
+	console.log(
+		`Got ${message_ids.length} messages for channel ${channel_id}...`,
+	);
+	// Alicia - get authors
+	const author_results = await userQueryBuilder
+		.where("id IN (:...ids)", {
+			ids: author_ids.unique(),
+		})
+		.distinctOn(["id"])
+		.select("User.id, User.username, User.discriminator, User.avatar")
+		.getRawMany();
+	// Alicia - turn result into dictionary
+	const authors: any = {};
+	for (const author of author_results) {
+		authors[author.User_id] = author;
+	}
+
+	console.log(
+		`Got ${author_results.length} authors for channel ${channel_id}...`,
+	);
+
+	// Alicia - write messages
+	while (message_ids.length > 0) {
+		const current_message_ids = message_ids.splice(0, chunk_size);
+		res.write(`<!-- ${message_ids.length} remain... -->\n`);
+		console.log(`${message_ids.length} remain...`);
+		const messages = await messageQueryBuilder
+			.select("*")
+			.where("id IN (:...ids)", {
+				ids: current_message_ids,
+			})
+			.getRawMany();
+		for (const message of messages) {
+			res.write(JSON.stringify(message) + "\n");
+		}
+	}
+	// message_ids.slice();
+	// for (const message_id of message_ids) {
+	// 	const message = await Message.findOne({
+	// 		where: { id: message_id + "" },
+	// 	});
+	// 	res.write(JSON.stringify(message) + "\n");
+	// 	res.write("\n");
+	// }
+
+	res.write(JSON.stringify(channel));
+	console.timeEnd("get channel");
+	res.end();
+});
+
+export default router;

src/index/start.ts

@@ -0,0 +1,57 @@
+/*
+	Fosscord: A FOSS re-implementation and extension of the Discord.com backend.
+	Copyright (C) 2023 Fosscord and Fosscord Contributors
+	
+	This program is free software: you can redistribute it and/or modify
+	it under the terms of the GNU Affero General Public License as published
+	by the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+	
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU Affero General Public License for more details.
+	
+	You should have received a copy of the GNU Affero General Public License
+	along with this program.  If not, see <https://www.gnu.org/licenses/>.
+*/
+
+require("module-alias/register");
+process.on("uncaughtException", console.error);
+process.on("unhandledRejection", console.error);
+
+import "missing-native-js-functions";
+import { config } from "dotenv";
+config();
+import { FosscordServer } from "./Server";
+import cluster from "cluster";
+import os from "os";
+let cores = 1;
+try {
+	cores = Number(process.env.THREADS) || os.cpus().length;
+} catch {
+	console.log("[API] Failed to get thread count! Using 1...");
+}
+
+if (cluster.isPrimary && process.env.NODE_ENV == "production") {
+	console.log(`Primary ${process.pid} is running`);
+
+	// Fork workers.
+	for (let i = 0; i < cores; i++) {
+		cluster.fork();
+	}
+
+	cluster.on("exit", (worker) => {
+		console.log(`worker ${worker.process.pid} died, restart worker`);
+		cluster.fork();
+	});
+} else {
+	const port = Number(process.env.PORT) || 3001;
+
+	const server = new FosscordServer({ port });
+	server.start().catch(console.error);
+
+	// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+	// @ts-ignore
+	global.server = server;
+}

src/util/entities/Application.ts

@@ -1,20 +1,20 @@
 /*
-	Fosscord: A FOSS re-implementation and extension of the Discord.com backend.
-	Copyright (C) 2023 Fosscord and Fosscord Contributors
-	
-	This program is free software: you can redistribute it and/or modify
-	it under the terms of the GNU Affero General Public License as published
-	by the Free Software Foundation, either version 3 of the License, or
-	(at your option) any later version.
-	
-	This program is distributed in the hope that it will be useful,
-	but WITHOUT ANY WARRANTY; without even the implied warranty of
-	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-	GNU Affero General Public License for more details.
-	
-	You should have received a copy of the GNU Affero General Public License
-	along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
+ * Fosscord: A FOSS re-implementation and extension of the Discord.com backend.
+ * Copyright (C) 2023 Fosscord and Fosscord Contributors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
 
 import { Column, Entity, JoinColumn, ManyToOne, OneToOne } from "typeorm";
 import { BaseClass } from "./BaseClass";
@@ -55,8 +55,8 @@ export class Application extends BaseClass {
 	owner: User;
 
 	// TODO: enum this? https://discord.com/developers/docs/resources/application#application-object-application-flags
-	@Column()
-	flags: number = 0;
+	@Column({ default: 0 })
+	flags: number;
 
 	@Column({ type: "simple-array", nullable: true })
 	redirect_uris: string[] = [];

src/util/entities/Channel.ts

@@ -1,20 +1,20 @@
 /*
-	Fosscord: A FOSS re-implementation and extension of the Discord.com backend.
-	Copyright (C) 2023 Fosscord and Fosscord Contributors
-	
-	This program is free software: you can redistribute it and/or modify
-	it under the terms of the GNU Affero General Public License as published
-	by the Free Software Foundation, either version 3 of the License, or
-	(at your option) any later version.
-	
-	This program is distributed in the hope that it will be useful,
-	but WITHOUT ANY WARRANTY; without even the implied warranty of
-	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-	GNU Affero General Public License for more details.
-	
-	You should have received a copy of the GNU Affero General Public License
-	along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
+ * Fosscord: A FOSS re-implementation and extension of the Discord.com backend.
+ * Copyright (C) 2023 Fosscord and Fosscord Contributors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
 
 import {
 	Column,
@@ -32,9 +32,9 @@ import {
 	containsAll,
 	emitEvent,
 	getPermission,
+	InvisibleCharacters,
 	Snowflake,
 	trimSpecial,
-	InvisibleCharacters,
 } from "../util";
 import { ChannelCreateEvent, ChannelRecipientRemoveEvent } from "../interfaces";
 import { Recipient } from "./Recipient";
@@ -193,6 +193,9 @@ export class Channel extends BaseClass {
 	default_thread_rate_limit_per_user: number = 0;
 
 	// TODO: DM channel
+	@Column({ default: false })
+	indexable: boolean;
+
 	static async createChannel(
 		channel: Partial<Channel>,
 		user_id: string = "0",

src/util/util/Database.ts

@@ -1,23 +1,23 @@
 /*
-	Fosscord: A FOSS re-implementation and extension of the Discord.com backend.
-	Copyright (C) 2023 Fosscord and Fosscord Contributors
-	
-	This program is free software: you can redistribute it and/or modify
-	it under the terms of the GNU Affero General Public License as published
-	by the Free Software Foundation, either version 3 of the License, or
-	(at your option) any later version.
-	
-	This program is distributed in the hope that it will be useful,
-	but WITHOUT ANY WARRANTY; without even the implied warranty of
-	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-	GNU Affero General Public License for more details.
-	
-	You should have received a copy of the GNU Affero General Public License
-	along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
+ * Fosscord: A FOSS re-implementation and extension of the Discord.com backend.
+ * Copyright (C) 2023 Fosscord and Fosscord Contributors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
 
 import { DataSource } from "typeorm";
-import { yellow, green, red } from "picocolors";
+import { green, red, yellow } from "picocolors";
 import { Migration } from "../entities/Migration";
 import { ConfigEntity } from "../entities/Config";
 import { config } from "dotenv";
@@ -50,7 +50,7 @@ const DataSourceOptions = new DataSource({
 	database: isSqlite ? dbConnectionString : undefined,
 	entities: [path.join(__dirname, "..", "entities", "*.js")],
 	synchronize: !!process.env.DB_SYNC,
-	logging: false,
+	logging: !!process.env.DB_LOG,
 	bigNumberStrings: false,
 	supportBigNumbers: true,
 	name: "default",