initial bullshit for federation v2

package-lock.json

@@ -13,6 +13,7 @@
 				"@aws-sdk/client-s3": "^3.385.0",
 				"@sentry/integrations": "^7.66.0",
 				"@sentry/node": "^7.66.0",
+				"activitypub-core-types": "^0.3.2",
 				"ajv": "8.6.2",
 				"ajv-formats": "2.1.1",
 				"amqplib": "^0.10.3",
@@ -48,6 +49,7 @@
 				"reflect-metadata": "^0.1.13",
 				"ts-node": "^10.9.1",
 				"tslib": "^2.6.1",
+				"turndown": "^7.1.2",
 				"typeorm": "^0.3.17",
 				"typescript-json-schema": "^0.50.1",
 				"wretch": "^2.6.0",
@@ -71,6 +73,7 @@
 				"@types/nodemailer": "^6.4.9",
 				"@types/probe-image-size": "^7.2.0",
 				"@types/sharp": "^0.31.1",
+				"@types/turndown": "^5.0.2",
 				"@types/ws": "^8.5.5",
 				"@typescript-eslint/eslint-plugin": "^5.62.0",
 				"@typescript-eslint/parser": "^5.62.0",
@@ -2253,6 +2256,12 @@
 				"@types/node": "*"
 			}
 		},
+		"node_modules/@types/turndown": {
+			"version": "5.0.2",
+			"resolved": "https://registry.npmjs.org/@types/turndown/-/turndown-5.0.2.tgz",
+			"integrity": "sha512-ghbjIyvMSQn/UGEuQJD6C4DfbokyYqGRhNAetWH02qnuRfvRZz9qTOG9e0RPkVqGsjv+YsjF3gRp7yFKvc/1PA==",
+			"dev": true
+		},
 		"node_modules/@types/ws": {
 			"version": "8.5.5",
 			"resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.5.tgz",
@@ -2495,6 +2504,14 @@
 				"node": ">=0.4.0"
 			}
 		},
+		"node_modules/activitypub-core-types": {
+			"version": "0.3.2",
+			"resolved": "https://registry.npmjs.org/activitypub-core-types/-/activitypub-core-types-0.3.2.tgz",
+			"integrity": "sha512-hAWCkRIzLJ3eVEjnibPYNHQaM+vD0SCK29gqMEt5pEb/2pbEetkv+4MpVMi0CKtr/GWRCSjIw1C6YAph7yT0pA==",
+			"dependencies": {
+				"formidable": "^2.1.1"
+			}
+		},
 		"node_modules/addressparser": {
 			"version": "1.0.1",
 			"resolved": "https://registry.npmjs.org/addressparser/-/addressparser-1.0.1.tgz",
@@ -2725,8 +2742,7 @@
 		"node_modules/asap": {
 			"version": "2.0.6",
 			"resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz",
-			"integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==",
+			"integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA=="
-			"optional": true
 		},
 		"node_modules/asn1js": {
 			"version": "3.0.5",
@@ -3589,7 +3605,6 @@
 			"version": "1.0.4",
 			"resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.4.tgz",
 			"integrity": "sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==",
-			"optional": true,
 			"dependencies": {
 				"asap": "^2.0.0",
 				"wrappy": "1"
@@ -3665,6 +3680,11 @@
 				"url": "https://github.com/fb55/domhandler?sponsor=1"
 			}
 		},
+		"node_modules/domino": {
+			"version": "2.1.6",
+			"resolved": "https://registry.npmjs.org/domino/-/domino-2.1.6.tgz",
+			"integrity": "sha512-3VdM/SXBZX2omc9JF9nOPCtDaYQ67BGp5CoLpIQlO2KCAPETs8TcDHacF26jXadGbvUteZzRTeos2fhID5+ucQ=="
+		},
 		"node_modules/domutils": {
 			"version": "3.1.0",
 			"resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz",
@@ -4444,7 +4464,6 @@
 			"version": "2.1.2",
 			"resolved": "https://registry.npmjs.org/formidable/-/formidable-2.1.2.tgz",
 			"integrity": "sha512-CM3GuJ57US06mlpQ47YcunuUZ9jpm8Vx+P2CGt2j7HpgkKZO/DJYQ0Bobim8G6PFQmK5lOqOOdUXboU+h73A4g==",
-			"optional": true,
 			"dependencies": {
 				"dezalgo": "^1.0.4",
 				"hexoid": "^1.0.0",
@@ -4723,7 +4742,6 @@
 			"version": "1.0.0",
 			"resolved": "https://registry.npmjs.org/hexoid/-/hexoid-1.0.0.tgz",
 			"integrity": "sha512-QFLV0taWQOZtvIRIAdBChesmogZrtuXvVWsFHZTk2SU+anspqZ2vMnoLg7IE1+Uk16N19APic1BuF8bC8c2m5g==",
-			"optional": true,
 			"engines": {
 				"node": ">=8"
 			}
@@ -7756,6 +7774,14 @@
 			"integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==",
 			"dev": true
 		},
+		"node_modules/turndown": {
+			"version": "7.1.2",
+			"resolved": "https://registry.npmjs.org/turndown/-/turndown-7.1.2.tgz",
+			"integrity": "sha512-ntI9R7fcUKjqBP6QU8rBK2Ehyt8LAzt3UBT9JR9tgo6GtuKvyUzpayWmeMKJw1DPdXzktvtIT8m2mVXz+bL/Qg==",
+			"dependencies": {
+				"domino": "^2.1.6"
+			}
+		},
 		"node_modules/type-check": {
 			"version": "0.4.0",
 			"resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz",

package.json

@@ -54,6 +54,7 @@
 		"@types/nodemailer": "^6.4.9",
 		"@types/probe-image-size": "^7.2.0",
 		"@types/sharp": "^0.31.1",
+		"@types/turndown": "^5.0.2",
 		"@types/ws": "^8.5.5",
 		"@typescript-eslint/eslint-plugin": "^5.62.0",
 		"@typescript-eslint/parser": "^5.62.0",
@@ -68,6 +69,7 @@
 		"@aws-sdk/client-s3": "^3.385.0",
 		"@sentry/integrations": "^7.66.0",
 		"@sentry/node": "^7.66.0",
+		"activitypub-core-types": "^0.3.2",
 		"ajv": "8.6.2",
 		"ajv-formats": "2.1.1",
 		"amqplib": "^0.10.3",
@@ -103,6 +105,7 @@
 		"reflect-metadata": "^0.1.13",
 		"ts-node": "^10.9.1",
 		"tslib": "^2.6.1",
+		"turndown": "^7.1.2",
 		"typeorm": "^0.3.17",
 		"typescript-json-schema": "^0.50.1",
 		"wretch": "^2.6.0",
@@ -112,7 +115,8 @@
 		"@spacebar/api": "dist/api",
 		"@spacebar/cdn": "dist/cdn",
 		"@spacebar/gateway": "dist/gateway",
-		"@spacebar/util": "dist/util"
+		"@spacebar/util": "dist/util",
+		"@spacebar/ap": "dist/activitypub"
 	},
 	"optionalDependencies": {
 		"erlpack": "^0.1.4",

src/activitypub/README.md

@@ -0,0 +1,142 @@
+# Spacebar Activitypub
+
+-   [Activitystreams vocab](https://www.w3.org/TR/activitystreams-vocabulary)
+-   [Activitystreams](https://www.w3.org/TR/activitystreams-core)
+-   [Activitypub spec](https://www.w3.org/TR/activitypub/)
+
+## Supported Types
+
+| Spacebar object | Activitypub                                                                        |
+| --------------- | ---------------------------------------------------------------------------------- |
+| Message         | [Note](https://www.w3.org/TR/activitystreams-vocabulary/#dfn-note)                 |
+| Channel         | [Group](https://www.w3.org/TR/activitystreams-vocabulary/#dfn-group)               |
+| Guild           | [Organisation](https://www.w3.org/TR/activitystreams-vocabulary/#dfn-organization) |
+| User            | [Person](https://www.w3.org/TR/activitystreams-vocabulary/#dfn-person)             |
+| Role            | Spacebar extension: [Role](#role-federation)                                       |
+
+## Message Federation
+
+A message sent by a user. Sent to channels, or directly to users (a DM channel is created on Spacebar instances).
+
+### Supported Activities
+
+| Activity   | Action                                               |
+| ---------- | ---------------------------------------------------- |
+| `Create`   | Transformed from a Note to a Message and saved to db |
+| `Delete`   | Removes a message from db                            |
+| `Update`   | Updates a message and saves to db.                   |
+| `Announce` | Used by Channels to forward to members.              |
+
+### Properties Used
+
+| Property               | Description                                                                  |
+| ---------------------- | ---------------------------------------------------------------------------- |
+| `type`                 | Must be `"Note"`                                                             |
+| `content`              | Message content                                                              |
+| `name`                 | Used as message content if `content` not provided                            |
+| `inReplyTo`            | Reference a previous Message in this guild                                   |
+| `published`            | Timestamp of this Message                                                    |
+| `attributedTo`         | Message author                                                               |
+| `to`                   | The Channel this Message is a part of                                        |
+| `tag`                  | Mentions                                                                     |
+| `tag[].type`           | Must be `Mention`                                                            |
+| `tag[].name`           | Plain-type Webfinger address of a Profile within this Guild OR `@everyone`   |
+| `attachment`           | Message attachments                                                          |
+| `attachment[].url`     | The URL of this media attachment                                             |
+| `attachment[].summary` | The content warning for this media attachment                                |
+| `replies`              | For compatibility with other software: The replies to this message           |
+| `sbType`               | Spacebar extension. Describes the real MessageType. i.e. `GUILD_MEMBER_JOIN` |
+| `embeds`               | Spacebar extension. Describes the attached Embeds                            |
+| `flags`                | Spacebar extension. Message flags as bitfield                                |
+| TODO: reactions        | How does plemora/akkoma/misskey/etc handle reactions?                        |
+| TODO: components       |                                                                              |
+| TODO: stickers         |                                                                              |
+
+## Channel Federation
+
+An automated actor. Users can send messages to it, which the channel forwards to it's followers in an `Announce`.
+Follows/is followed by it's corresponding Guild, if applicable.
+
+### Supported Activities
+
+| Activity       | Action                                                |
+| -------------- | ----------------------------------------------------- |
+| `Create`       | Transformed from a Group to a Channel and saved to db |
+| `Delete`       | Removes a channel from db                             |
+| `Update`       | Updates channel details                               |
+| `Add`/`Remove` | Manage pinned Messages for this Channel               |
+
+### Properties Used
+
+| Property       | Description                                                           |
+| -------------- | --------------------------------------------------------------------- |
+| `type`         | Must be `"Group"`                                                     |
+| `name`         | The Channel name                                                      |
+| `published`    | Creation timestamp of this Channel                                    |
+| `attributedTo` | The Guild this Channel is a part of                                   |
+| `featured`     | Mastodon extension. The pinned Messages in this Channel               |
+| `publicKey`    | The public key used to verify signatures from this actor              |
+| `sbType`       | Spacebar extension. Describes the real ChannelType. i.e. `GUILD_TEXT` |
+
+## Guild Federation
+
+An automated actor. Follows and is followed by it's corresponding Channels.
+Also contains a collection of [roles](#role-federation).
+
+### Supported Activities
+
+| Activity | Action                                                             |
+| -------- | ------------------------------------------------------------------ |
+| `Follow` | Join a guild. Must provide an invite code. Automatically accepted. |
+| `Delete` | Delete a guild.                                                    |
+| `Update` | Update guild details.                                              |
+
+### Properties Used
+
+## User Federation
+
+A person. Sends messages to Channels. May also create, modify, or moderate guilds, channels, or roles.
+Is a partOf a [Role](#role-federation)
+
+### Supported Activities
+
+| Activity          | Action                                                                                         |
+| ----------------- | ---------------------------------------------------------------------------------------------- |
+| `Follow`          | Send a friend request                                                                          |
+| `Accept`/`Reject` | Accept or reject a friend request                                                              |
+| `Undo`            | Unfriend                                                                                       |
+| `Delete`          | Delete a user from the database along with all their messages.                                 |
+| `Block`           | Signal to the remote server that they should hide your profile from that user. Not guaranteed. |
+| `Update`          | Update user details.                                                                           |
+
+## Role Federation
+
+Is a [Collection](https://www.w3.org/TR/activitystreams-vocabulary/#dfn-collection) of Users within this role.
+
+## S2S endpoints
+
+Base url: `/federation`
+
+-   `/.well-known/webfinger?resource=acct@domain` - Returns webfinger response i.e. https://docs.joinmastodon.org/spec/webfinger/
+-   `/.well-known/host-meta` - Returns location of webfinger? Why is this neccessary?
+
+-   `/channels/:channel_id` - Returns specified Channel as AP object ( Group )
+-   `/channels/:channel_id/inbox` - The inbox for this Channel
+-   `/channels/:channel_id/outbox` - The outbox for this Channel
+-   `/channels/:channel_id/followers` - The Users that have access to this Channel
+
+-   `/channels/:channel_id/messages/:message_id` - Returns specified Message in Channel as AP object ( Announce Note )
+-
+-   `/messages/:message_id` - Returns specified Message in Channel as AP object ( Announce Note )
+
+-   `/activities/:activity_id` - Returns the specified activitypub activity. E.g. Announce, Follow, etc.
+-   `/activities/inbox` - Shared inbox.
+
+-   `/users/:user_id` - Returns specified User as AP object (Person)
+-   `/users/:user_id/inbox` - The inbox of this User. POSTing creates a DM channel if it does not exist.
+
+-   `/guilds/:guild_id` - Returns specified Guild as AP object (Organisation)
+
+## notes
+
+-   activitypub responses should be returned if the Accept header is `application/ld+json; profile="https://www.w3.org/ns/activitystreams"` OR `application/activity+json`

src/activitypub/Server.ts

@@ -0,0 +1,69 @@
+import { BodyParser, CORS, ErrorHandler } from "@spacebar/api";
+import {
+	Config,
+	JSONReplacer,
+	Sentry,
+	initDatabase,
+	registerRoutes,
+	setupMorganLogging,
+} from "@spacebar/util";
+import { Request, Response, Router } from "express";
+import { Server, ServerOptions } from "lambert-server";
+import path from "path";
+import wellknown from "./well-known";
+
+export type SpacebarServerOptions = ServerOptions;
+
+export class FederationServer extends Server {
+	public declare options: SpacebarServerOptions;
+
+	constructor(opts?: Partial<SpacebarServerOptions>) {
+		// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+		// @ts-ignore
+		super({ ...opts, errorHandler: false, jsonBody: false });
+	}
+
+	async start() {
+		await initDatabase();
+		await Config.init();
+		await Sentry.init(this.app);
+
+		setupMorganLogging(this.app);
+		this.app.set("json replacer", JSONReplacer);
+
+		this.app.use(CORS);
+		this.app.use(BodyParser({ inflate: true, limit: "10mb" }));
+
+		const app = this.app;
+		const api = Router();
+		// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+		// @ts-ignore
+		this.app = api;
+
+		// TODO: auth
+		// TODO: rate limits
+
+		this.routes = await registerRoutes(
+			this,
+			path.join(__dirname, "routes", "/"),
+		);
+
+		api.use("*", (req: Request, res: Response) => {
+			res.status(404).json({
+				message: "404 endpoint not found",
+				code: 0,
+			});
+		});
+
+		this.app = app;
+
+		this.app.use("/federation", api);
+		this.app.use("/.well-known", wellknown);
+
+		this.app.use(ErrorHandler);
+
+		Sentry.errorHandler(this.app);
+
+		return super.start();
+	}
+}

src/activitypub/federation/OrderedCollection.ts

@@ -0,0 +1,38 @@
+import { AP } from "activitypub-core-types";
+import { ACTIVITYSTREAMS_CONTEXT } from "./utils";
+
+export const makeOrderedCollection = async <T extends AP.CoreObject>(opts: {
+	page: boolean;
+	min_id?: string;
+	max_id?: string;
+	id: URL;
+	getTotalElements: () => Promise<number>;
+	getElements: (before?: string, after?: string) => Promise<T[]>;
+}): Promise<AP.OrderedCollection> => {
+	const { page, min_id, max_id, id, getTotalElements, getElements } = opts;
+
+	if (!page)
+		return {
+			"@context": ACTIVITYSTREAMS_CONTEXT,
+			id: id,
+			type: "OrderedCollection",
+			totalItems: await getTotalElements(),
+			first: new URL(`${id}?page=true`),
+			last: new URL(`${id}?page=true&min_id=0`),
+		};
+
+	const after = min_id ? `${min_id}` : undefined;
+	const before = max_id ? `${max_id}` : undefined;
+
+	const elems = await getElements(before, after);
+
+	return {
+		"@context": ACTIVITYSTREAMS_CONTEXT,
+		id: new URL(`${id}?page=true`),
+		type: "OrderedCollection",
+		first: new URL(`${id}?page=true`),
+		last: new URL(`${id}?page=true&min_id=0`),
+		totalItems: await getTotalElements(),
+		orderedItems: elems,
+	};
+};

src/activitypub/federation/index.ts

@@ -0,0 +1,17 @@
+/**
+ * To be injected into API
+ * Responsible for dispatching activitypub events to external instances
+ */
+
+import { AP } from "activitypub-core-types";
+import { federationQueue } from "./queue";
+
+export * from "./OrderedCollection";
+export * from "./transforms";
+export * from "./utils";
+
+export class Federation {
+	static async distribute(activity: AP.Activity) {
+		await federationQueue.distribute(activity);
+	}
+}

src/activitypub/federation/queue.ts

@@ -0,0 +1,76 @@
+import { Config, FederationKey } from "@spacebar/util";
+import { AP } from "activitypub-core-types";
+import fetch from "node-fetch";
+import {
+	APError,
+	resolveWebfinger,
+	signActivity,
+	splitQualifiedMention,
+} from "./utils";
+
+//
+type Instance = string;
+
+class FederationQueue {
+	// TODO: queue messages and send them to shared inbox
+	private queue: Map<Instance, Array<AP.Activity>> = new Map();
+
+	public async distribute(activity: AP.Activity) {
+		let { to, actor } = activity;
+
+		if (!to)
+			throw new APError("Activity with no `to` field is undeliverable.");
+		if (!Array.isArray(to)) to = [to];
+
+		if (!actor)
+			throw new APError("Activity with no `to` field is undeliverable.");
+		if (Array.isArray(actor)) actor = actor[0];
+
+		// TODO: check if `to` is on our instance?
+		// we shouldn't get to this point if they are, though.
+
+		// if the sender is one of ours, fetch their private key for signing
+		const { user } = splitQualifiedMention(actor.toString());
+		const sender = await FederationKey.findOneOrFail({
+			where: { actorId: user, domain: Config.get().federation.host },
+		});
+
+		if (!sender.privateKey) {
+			console.warn(
+				"tried to federate activity who's sender does not have a private key",
+			);
+			return;
+		}
+
+		for (const receiver of to) {
+			if (!(receiver instanceof URL)) {
+				console.error(receiver);
+				continue;
+			}
+
+			const apReceiver = await resolveWebfinger(receiver.toString());
+			if (!("inbox" in apReceiver)) {
+				console.error(
+					"[federation] receiver doesn't have inbox",
+					apReceiver,
+				);
+				continue;
+			}
+
+			if (typeof apReceiver.inbox != "string") {
+				console.error(apReceiver.inbox);
+				continue;
+			}
+
+			const signedActivity = await signActivity(
+				apReceiver.inbox,
+				sender,
+				activity,
+			);
+
+			await fetch(apReceiver.inbox, signedActivity);
+		}
+	}
+}
+
+export const federationQueue = new FederationQueue();

src/activitypub/federation/transforms.ts

@@ -0,0 +1,267 @@
+import {
+	ActorType,
+	Channel,
+	Config,
+	DmChannelDTO,
+	FederationKey,
+	Member,
+	Message,
+	Snowflake,
+	User,
+	UserSettings,
+} from "@spacebar/util";
+import { AP } from "activitypub-core-types";
+import TurndownService from "turndown";
+import {
+	ACTIVITYSTREAMS_CONTEXT,
+	APError,
+	APObjectIsPerson,
+	resolveAPObject,
+} from "./utils";
+
+export const transformMessageToAnnounceNoce = async (
+	message: Message,
+): Promise<AP.Announce> => {
+	const { host } = Config.get().federation;
+
+	return {
+		"@context": ACTIVITYSTREAMS_CONTEXT,
+		type: "Announce",
+		id: new URL(
+			`https://${host}/federation/channels/${message.channel_id}/messages/${message.id}`,
+		),
+		actor: new URL(`https://${host}/federation/users/${message.author_id}`),
+		published: message.timestamp,
+		to: [
+			new URL(
+				`https://${host}/federation/channels/${message.channel_id}/followers`,
+			),
+		],
+		object: await transformMessageToNote(message),
+	};
+};
+
+export const transformMessageToNote = async (
+	message: Message,
+): Promise<AP.Note> => {
+	const { host } = Config.get().federation;
+
+	const referencedMessage = message.message_reference
+		? await Message.findOne({
+				where: { id: message.message_reference.message_id },
+		  })
+		: null;
+
+	return {
+		id: new URL(`https://${host}/federation/messages/${message.id}`),
+		type: "Note",
+		content: message.content, // TODO: convert markdown to html
+		inReplyTo: referencedMessage
+			? await transformMessageToNote(referencedMessage)
+			: undefined,
+		published: message.timestamp,
+		attributedTo: new URL(
+			`https://${host}/federation/users/${message.author_id}`,
+		),
+		to: [
+			new URL(
+				`https://${host}/federation/channels/${message.channel_id}`,
+			),
+		],
+		tag: message.mentions?.map(
+			(x) => new URL(`https://${host}/federation/users/${x.id}`),
+		),
+		attachment: [],
+		// replies: [],
+		// sbType: message.type,
+		// embeds: [],
+		// flags: message.flags,
+	};
+};
+
+// TODO: this was copied from the previous implemention. refactor it.
+export const transformNoteToMessage = async (note: AP.Note) => {
+	if (!note.id) throw new APError("Note must have ID");
+	if (note.type != "Note") throw new APError("Message must be Note");
+
+	if (!note.attributedTo)
+		throw new APError("Note must have author (attributedTo");
+
+	const attrib = await resolveAPObject(
+		Array.isArray(note.attributedTo)
+			? note.attributedTo[0]
+			: note.attributedTo,
+	);
+
+	if (!APObjectIsPerson(attrib))
+		throw new APError("Note must be attributedTo a Person");
+
+	const user = await transformPersonToUser(attrib);
+
+	const to = Array.isArray(note.to) ? note.to[0] : note.to;
+
+	let channel: Channel | DmChannelDTO;
+	const to_id = to?.toString().split("/").reverse()[0];
+	if (to?.toString().includes("user")) {
+		// this is a DM channel
+		const toUser = await User.findOneOrFail({ where: { id: to_id } });
+
+		// Channel.createDMCHannel does a .save() so the author must be present
+		await user.save();
+
+		// const cache = await Channel.findOne({ where: { recipients: []}})
+
+		channel = await Channel.createDMChannel(
+			[toUser.id, user.id],
+			toUser.id,
+		);
+	} else {
+		channel = await Channel.findOneOrFail({
+			where: { id: to_id },
+			relations: { guild: true },
+		});
+	}
+
+	const member =
+		channel instanceof Channel
+			? await Member.findOneOrFail({
+					where: { id: user.id, guild_id: channel.guild!.id },
+			  })
+			: undefined;
+
+	return Message.create({
+		id: Snowflake.generate(),
+		content: new TurndownService().turndown(note.content),
+		timestamp: note.published,
+		author: user,
+		guild: channel instanceof Channel ? channel.guild : undefined,
+		member,
+		channel_id: channel.id,
+
+		nonce: note.id.toString(),
+		type: 0,
+		sticker_items: [],
+		attachments: [],
+		embeds: [],
+		reactions: [],
+		mentions: [],
+		mention_roles: [],
+		mention_channels: [],
+	});
+};
+
+export const transformChannelToGroup = async (
+	channel: Channel,
+): Promise<AP.Group> => {
+	const { host, accountDomain } = Config.get().federation;
+
+	const keys = await FederationKey.findOneOrFail({
+		where: { actorId: channel.id, domain: accountDomain },
+	});
+
+	return {
+		"@context": "https://www.w3.org/ns/activitystreams",
+		type: "Group",
+		id: new URL(`https://${host}/fed/channels/${channel.id}`),
+		name: channel.name,
+		preferredUsername: channel.id,
+		summary: channel.topic,
+		icon: undefined,
+		// discoverable: true,
+
+		publicKey: {
+			id: `https://${host}/fed/user/${channel.id}#main-key`,
+			owner: `https://${host}/fed/user/${channel.id}`,
+			publicKeyPem: keys.publicKey,
+		},
+
+		inbox: new URL(`https://${host}/fed/channels/${channel.id}/inbox`),
+		outbox: new URL(`https://${host}/fed/channels/${channel.id}/outbox`),
+		followers: new URL(
+			`https://${host}/fed/channels/${channel.id}/followers`,
+		),
+	};
+};
+
+export const transformUserToPerson = async (user: User): Promise<AP.Person> => {
+	const { host, accountDomain } = Config.get().federation;
+
+	const keys = await FederationKey.findOneOrFail({
+		where: { actorId: user.id, domain: accountDomain },
+	});
+
+	return {
+		"@context": ACTIVITYSTREAMS_CONTEXT,
+		type: "Person",
+		id: new URL(`https://${host}/federation/users/${user.id}`),
+
+		name: user.username,
+		preferredUsername: user.id,
+		summary: user.bio,
+		icon: user.avatar
+			? [
+					new URL(
+						`${Config.get().cdn.endpointPublic}/avatars/${
+							user.id
+						}/${user.avatar}`,
+					),
+			  ]
+			: undefined,
+
+		inbox: new URL(`https://${host}/federation/users/${user.id}/inbox`),
+		outbox: new URL(`https://${host}/federation/users/${user.id}/outbox`),
+		followers: new URL(
+			`https://${host}/federation/users/${user.id}/followers`,
+		),
+		publicKey: {
+			id: `https://${host}/federation/users/${user.id}#main-key`,
+			owner: `https://${host}/federation/users/${user.id}`,
+			publicKeyPem: keys.publicKey,
+		},
+	};
+};
+
+// TODO: this was copied from previous implementation. refactor.
+export const transformPersonToUser = async (person: AP.Person) => {
+	if (!person.id) throw new APError("User must have ID");
+
+	const url = new URL(person.id.toString());
+	const email = `${url.pathname.split("/").reverse()[0]}@${url.hostname}`;
+
+	const cachedKeys = await FederationKey.findOne({
+		where: { federatedId: url.toString() },
+	});
+	if (cachedKeys) {
+		return await User.findOneOrFail({ where: { id: cachedKeys.actorId } });
+	}
+
+	await FederationKey.create({
+		actorId: Snowflake.generate(),
+		federatedId: url.toString(),
+		domain: url.hostname,
+		publicKey: person.publicKey?.publicKeyPem,
+		type: ActorType.USER,
+	}).save();
+
+	return User.create({
+		username: person.preferredUsername,
+		discriminator: url.hostname,
+		bio: new TurndownService().turndown(person.summary),
+		email,
+		data: {
+			hash: "#",
+			valid_tokens_since: new Date(),
+		},
+		extended_settings: "{}",
+		settings: UserSettings.create(),
+		premium: false,
+
+		premium_since: Config.get().defaults.user.premium
+			? new Date()
+			: undefined,
+		rights: Config.get().register.defaultRights,
+		premium_type: Config.get().defaults.user.premiumType ?? 0,
+		verified: Config.get().defaults.user.verified ?? true,
+		created_at: new Date(),
+	}).save();
+};

src/activitypub/federation/utils.ts

@@ -0,0 +1,176 @@
+import { DEFAULT_FETCH_OPTIONS } from "@spacebar/api";
+import {
+	Config,
+	FederationKey,
+	OrmUtils,
+	WebfingerResponse,
+} from "@spacebar/util";
+import { AP } from "activitypub-core-types";
+import crypto from "crypto";
+import { HTTPError } from "lambert-server";
+import fetch from "node-fetch";
+import { ProxyAgent } from "proxy-agent";
+
+export const ACTIVITYSTREAMS_CONTEXT = "https://www.w3.org/ns/activitystreams";
+
+export const fetchOpts = OrmUtils.mergeDeep(DEFAULT_FETCH_OPTIONS, {
+	headers: {
+		Accept: "application/activity+json",
+	},
+});
+
+export class APError extends HTTPError {}
+
+export const hasAPContext = (data: object) => {
+	if (!("@context" in data)) return false;
+	const context = data["@context"];
+	const activitystreams = "https://www.w3.org/ns/activitystreams";
+	if (Array.isArray(context))
+		return context.find((x) => x == activitystreams);
+	return context == activitystreams;
+};
+
+export const resolveAPObject = async <T>(data: string | T): Promise<T> => {
+	// we were already given an AP object
+	if (typeof data != "string") return data;
+
+	const agent = new ProxyAgent();
+	const ret = await fetch(data, {
+		...fetchOpts,
+		agent,
+	});
+
+	const json = await ret.json();
+
+	if (!hasAPContext(json)) throw new APError("Object is not APObject");
+
+	return json;
+};
+
+export const splitQualifiedMention = (lookup: string) => {
+	let domain: string, user: string;
+	if (lookup.includes("@")) {
+		// lookup a @handle@domain
+
+		if (lookup[0] == "@") lookup = lookup.slice(1);
+		[user, domain] = lookup.split("@");
+	} else {
+		// lookup was a URL ( hopefully )
+		try {
+			const url = new URL(lookup);
+			domain = url.hostname;
+			user = url.pathname.split("/").reverse()[0];
+		} catch (e) {
+			domain = "";
+			user = "";
+		}
+	}
+
+	return {
+		domain,
+		user,
+	};
+};
+
+export const resolveWebfinger = async (
+	lookup: string,
+): Promise<AP.CoreObject> => {
+	const { domain } = splitQualifiedMention(lookup);
+
+	const agent = new ProxyAgent();
+	const wellknown = (await fetch(
+		`https://${domain}/.well-known/webfinger?resource=${lookup}`,
+		{
+			agent,
+			...fetchOpts,
+		},
+	).then((x) => x.json())) as WebfingerResponse;
+
+	const link = wellknown.links.find((x) => x.rel == "self");
+	if (!link) throw new APError(".well-known did not contain rel=self link");
+
+	return await resolveAPObject<AP.CoreObject>(link.href);
+};
+
+/**
+ * Returns a signed request that can be passed to fetch
+ * ```
+ * const signed = await signActivity(receiver.inbox, sender, activity);
+ * await fetch(receiver.inbox, signed);
+ * ```
+ */
+export const signActivity = async (
+	inbox: string,
+	sender: FederationKey,
+	message: AP.Activity,
+) => {
+	if (!sender.privateKey)
+		throw new APError("cannot sign without private key");
+
+	const digest = crypto
+		.createHash("sha256")
+		.update(JSON.stringify(message))
+		.digest("base64");
+	const signer = crypto.createSign("sha256");
+	const now = new Date();
+
+	const url = new URL(inbox);
+	const inboxFrag = url.pathname;
+	const toSign =
+		`(request-target): post ${inboxFrag}\n` +
+		`host: ${url.hostname}\n` +
+		`date: ${now.toUTCString()}\n` +
+		`digest: SHA-256=${digest}`;
+
+	signer.update(toSign);
+	signer.end();
+
+	const signature = signer.sign(sender.privateKey);
+	const sig_b64 = signature.toString("base64");
+
+	const { host } = Config.get().federation;
+	const header =
+		`keyId="${host}/${sender.type}/${sender.actorId}#main-key",` +
+		`headers="(request-target) host date digest",` +
+		`signature=${sig_b64}`;
+
+	return OrmUtils.mergeDeep(fetchOpts, {
+		method: "POST",
+		body: message,
+		headers: {
+			Host: url.hostname,
+			Date: now.toUTCString(),
+			Digest: `SHA-256=${digest}`,
+			Signature: header,
+		},
+	});
+};
+
+// fetch from remote server?
+export const APObjectIsPerson = (
+	object: AP.EntityReference,
+): object is AP.Person => {
+	return "type" in object && object.type == "Person";
+};
+
+export const APObjectIsGroup = (
+	object: AP.EntityReference,
+): object is AP.Person => {
+	return "type" in object && object.type == "Group";
+};
+
+export const APObjectIsOrganisation = (
+	object: AP.EntityReference,
+): object is AP.Person => {
+	return "type" in object && object.type == "Organization";
+};
+
+export const APObjectIsSpacebarActor = (
+	object: AP.EntityReference,
+): object is AP.Person => {
+	return (
+		APObjectIsGroup(object) ||
+		APObjectIsOrganisation(object) ||
+		APObjectIsPerson(object)
+	);
+};

src/activitypub/index.ts

@@ -0,0 +1,2 @@
+export * from "./Server";
+export * from "./federation";

src/activitypub/routes/channels/#channel_id/inbox.ts

@@ -0,0 +1,35 @@
+import { transformNoteToMessage } from "@spacebar/ap";
+import { route } from "@spacebar/api";
+import { Message, emitEvent } from "@spacebar/util";
+import { AP } from "activitypub-core-types";
+import { Request, Response, Router } from "express";
+import { HTTPError } from "lambert-server";
+const router = Router();
+
+// TODO: check if the activity exists on the remote server
+router.post("/", route({}), async (req: Request, res: Response) => {
+	const body = req.body as AP.Create;
+
+	if (body.type != "Create") throw new HTTPError("not implemented");
+
+	const object = Array.isArray(body.object) ? body.object[0] : body.object;
+	if (!object) return res.status(400);
+	if (!("type" in object) || object.type != "Note")
+		throw new HTTPError("must be Note");
+	const message = await transformNoteToMessage(object as AP.Note);
+
+	if ((await Message.count({ where: { nonce: object.id!.toString() } })) != 0)
+		return res.status(200);
+
+	await message.save();
+
+	await emitEvent({
+		event: "MESSAGE_CREATE",
+		channel_id: message.channel_id,
+		data: message.toJSON(),
+	});
+
+	return res.status(200);
+});
+
+export default router;

src/activitypub/routes/channels/#channel_id/index.ts

@@ -0,0 +1,16 @@
+import { transformChannelToGroup } from "@spacebar/ap";
+import { route } from "@spacebar/api";
+import { Channel } from "@spacebar/util";
+import { Request, Response, Router } from "express";
+const router = Router();
+
+// TODO: auth
+router.get("/", route({}), async (req: Request, res: Response) => {
+	const channel = await Channel.findOneOrFail({
+		where: { id: req.params.channel_id },
+	});
+
+	return res.json(await transformChannelToGroup(channel));
+});
+
+export default router;

src/activitypub/routes/channels/#channel_id/messages/#message_id/index.ts

@@ -0,0 +1,18 @@
+import { transformMessageToAnnounceNoce } from "@spacebar/ap";
+import { route } from "@spacebar/api";
+import { Message } from "@spacebar/util";
+import { Request, Response, Router } from "express";
+const router = Router();
+
+// TODO: auth
+router.get("/", route({}), async (req: Request, res: Response) => {
+	const { channel_id, message_id } = req.params;
+
+	const message = await Message.findOneOrFail({
+		where: { channel_id, id: message_id },
+	});
+
+	return res.json(await transformMessageToAnnounceNoce(message));
+});
+
+export default router;

src/activitypub/routes/channels/#channel_id/outbox.ts

@@ -0,0 +1,53 @@
+import {
+	makeOrderedCollection,
+	transformMessageToAnnounceNoce,
+} from "@spacebar/ap";
+import { route } from "@spacebar/api";
+import { Config, Message, Snowflake } from "@spacebar/util";
+import { AP } from "activitypub-core-types";
+import { Request, Response, Router } from "express";
+import { FindManyOptions, FindOperator, LessThan, MoreThan } from "typeorm";
+const router = Router();
+
+router.get("/", route({}), async (req: Request, res: Response) => {
+	const { channel_id } = req.params;
+	const { page, min_id, max_id } = req.query;
+
+	const { host } = Config.get().federation;
+
+	const ret = await makeOrderedCollection({
+		page: page != undefined,
+		min_id: min_id?.toString(),
+		max_id: max_id?.toString(),
+		id: new URL(`https://${host}/federation/channels/${channel_id}/outbox`),
+		getTotalElements: () => Message.count({ where: { channel_id } }),
+		getElements: async (before, after): Promise<AP.Announce[]> => {
+			const query: FindManyOptions<Message> & {
+				where: { id?: FindOperator<string> | FindOperator<string>[] };
+			} = {
+				order: { timestamp: "DESC" },
+				take: 20,
+				where: { channel_id: channel_id },
+				relations: ["author"],
+			};
+
+			if (after) {
+				if (BigInt(after) > BigInt(Snowflake.generate())) return [];
+				query.where.id = MoreThan(after);
+			} else if (before) {
+				if (BigInt(before) > BigInt(Snowflake.generate())) return [];
+				query.where.id = LessThan(before);
+			}
+
+			const messages = await Message.find(query);
+
+			return await Promise.all(
+				messages.map((x) => transformMessageToAnnounceNoce(x)),
+			);
+		},
+	});
+
+	return res.json(ret);
+});
+
+export default router;

src/activitypub/routes/index.ts

@@ -0,0 +1,9 @@
+import { route } from "@spacebar/api";
+import { Request, Response, Router } from "express";
+const router = Router();
+
+router.get("/", route({}), async (req: Request, res: Response) => {
+	res.send("Online");
+});
+
+export default router;

src/activitypub/routes/users/#user_id/index.ts

@@ -0,0 +1,16 @@
+import { transformUserToPerson } from "@spacebar/ap";
+import { route } from "@spacebar/api";
+import { User } from "@spacebar/util";
+import { Request, Response, Router } from "express";
+const router = Router();
+
+// TODO: auth
+router.get("/", route({}), async (req: Request, res: Response) => {
+	const user = await User.findOneOrFail({
+		where: { id: req.params.user_id },
+	});
+
+	return res.json(await transformUserToPerson(user));
+});
+
+export default router;

src/activitypub/well-known.ts

@@ -0,0 +1,102 @@
+import { route } from "@spacebar/api";
+import {
+	ActorType,
+	Channel,
+	Config,
+	FederationKey,
+	FieldErrors,
+	Guild,
+	User,
+	WebfingerResponse,
+} from "@spacebar/util";
+import { Request, Response, Router } from "express";
+import { HTTPError } from "lambert-server";
+import { splitQualifiedMention } from "./federation";
+const router = Router();
+
+router.get(
+	"/webfinger",
+	route({
+		query: {
+			resource: {
+				type: "string",
+				description: "Resource to locate",
+			},
+		},
+		responses: {
+			200: {
+				body: "WebfingerResponse",
+			},
+		},
+	}),
+	async (req: Request, res: Response<WebfingerResponse>) => {
+		let resource = req.query.resource as string;
+		if (!resource)
+			throw FieldErrors({
+				resource: { message: "Resource must be present" },
+			});
+
+		// We know what you mean
+		resource = resource.replace("acct:", "");
+
+		const { accountDomain, host } = Config.get().federation;
+
+		const { user, domain } = splitQualifiedMention(resource);
+		if (domain != accountDomain)
+			throw new HTTPError("Resource could not be found", 404);
+
+		const keys = await FederationKey.findOneOrFail({
+			where: {
+				actorId: user,
+				domain,
+			},
+			select: ["type"],
+		});
+
+		let entity: User | Channel | Guild;
+		switch (keys.type) {
+			case ActorType.USER:
+				entity = await User.findOneOrFail({ where: { id: user } });
+				break;
+			case ActorType.CHANNEL:
+				entity = await Channel.findOneOrFail({ where: { id: user } });
+				break;
+			case ActorType.GUILD:
+				entity = await Guild.findOneOrFail({ where: { id: user } });
+				break;
+		}
+
+		res.setHeader("Content-Type", "application/jrd+json; charset=utf-8");
+		return res.json({
+			subject: `acct:${user}@${accountDomain}`, // mastodon always returns acct so might as well
+			aliases: [`https://${host}/federation/${keys.type}/${entity.id}`],
+			links: [
+				{
+					rel: "self",
+					type: "application/activity+json",
+					href: `https://${host}/federation/${keys.type}/${entity.id}`,
+				},
+				// {
+				// 	rel: "http://ostatus.org/schema/1.0/subscribe",
+				// 	href: `"https://${host}/fed/authorize-follow?acct={uri}"`,
+				// },
+			],
+		});
+	},
+);
+
+router.get("/host-meta", route({}), (req, res) => {
+	res.setHeader("Content-Type", "application/xrd+xml");
+
+	const { host } = Config.get().federation;
+
+	const ret = `<?xml version="1.0" encoding="UTF-8"?>
+	<XRD
+		xmlns="http://docs.oasis-open.org/ns/xri/xrd-1.0">
+		<Link rel="lrdd" type="application/xrd+xml" template="https://${host}/.well-known/webfinger?resource={uri}"/>
+	</XRD>`;
+
+	return res.send(ret);
+});
+
+export default router;

src/api/Server.ts

@@ -18,22 +18,21 @@
 
 import {
 	Config,
-	Email,
-	initDatabase,
-	initEvent,
-	JSONReplacer,
-	registerRoutes,
-	Sentry,
-	WebAuthn,
 	ConnectionConfig,
 	ConnectionLoader,
+	Email,
+	JSONReplacer,
+	Sentry,
+	WebAuthn,
+	initDatabase,
+	initEvent,
+	registerRoutes,
+	setupMorganLogging,
 } from "@spacebar/util";
 import { Request, Response, Router } from "express";
 import { Server, ServerOptions } from "lambert-server";
 import "missing-native-js-functions";
-import morgan from "morgan";
 import path from "path";
-import { red } from "picocolors";
 import { Authentication, CORS } from "./middlewares/";
 import { BodyParser } from "./middlewares/BodyParser";
 import { ErrorHandler } from "./middlewares/ErrorHandler";
@@ -79,23 +78,7 @@ export class SpacebarServer extends Server {
 		await Sentry.init(this.app);
 		WebAuthn.init();
 
-		const logRequests = process.env["LOG_REQUESTS"] != undefined;
+		setupMorganLogging(this.app);
-		if (logRequests) {
-			this.app.use(
-				morgan("combined", {
-					skip: (req, res) => {
-						let skip = !(
-							process.env["LOG_REQUESTS"]?.includes(
-								res.statusCode.toString(),
-							) ?? false
-						);
-						if (process.env["LOG_REQUESTS"]?.charAt(0) == "-")
-							skip = !skip;
-						return skip;
-					},
-				}),
-			);
-		}
 
 		this.app.set("json replacer", JSONReplacer);
 
@@ -147,13 +130,6 @@ export class SpacebarServer extends Server {
 
 		ConnectionLoader.loadConnections();
 
-		if (logRequests)
-			console.log(
-				red(
-					`Warning: Request logging is enabled! This will spam your console!\nTo disable this, unset the 'LOG_REQUESTS' environment variable!`,
-				),
-			);
-
 		return super.start();
 	}
 }

src/api/routes/-/healthz.ts

@@ -16,14 +16,14 @@
 	along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
 
-import { Router, Response, Request } from "express";
 import { route } from "@spacebar/api";
-import { getDatabase } from "@spacebar/util";
+import { Datasource } from "@spacebar/util";
+import { Request, Response, Router } from "express";
 
 const router = Router();
 
 router.get("/", route({}), (req: Request, res: Response) => {
-	if (!getDatabase()) return res.sendStatus(503);
+	if (!Datasource.isInitialized) return res.sendStatus(503);
 
 	return res.sendStatus(200);
 });

src/api/routes/-/readyz.ts

@@ -16,14 +16,14 @@
 	along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
 
-import { Router, Response, Request } from "express";
 import { route } from "@spacebar/api";
-import { getDatabase } from "@spacebar/util";
+import { Datasource } from "@spacebar/util";
+import { Request, Response, Router } from "express";
 
 const router = Router();
 
 router.get("/", route({}), (req: Request, res: Response) => {
-	if (!getDatabase()) return res.sendStatus(503);
+	if (!Datasource.isInitialized) return res.sendStatus(503);
 
 	return res.sendStatus(200);
 });

src/bundle/Server.ts

@@ -19,13 +19,14 @@
 process.on("unhandledRejection", console.error);
 process.on("uncaughtException", console.error);
 
-import http from "http";
+import { FederationServer } from "@spacebar/ap";
 import * as Api from "@spacebar/api";
-import * as Gateway from "@spacebar/gateway";
 import { CDNServer } from "@spacebar/cdn";
+import * as Gateway from "@spacebar/gateway";
+import { Config, Sentry, initDatabase } from "@spacebar/util";
 import express from "express";
-import { green, bold } from "picocolors";
+import http from "http";
-import { Config, initDatabase, Sentry } from "@spacebar/util";
+import { bold, green } from "picocolors";
 
 const app = express();
 const server = http.createServer();
@@ -36,6 +37,7 @@ server.on("request", app);
 const api = new Api.SpacebarServer({ server, port, production, app });
 const cdn = new CDNServer({ server, port, production, app });
 const gateway = new Gateway.Server({ server, port, production });
+const federation = new FederationServer({ server, port, production, app });
 
 process.on("SIGTERM", async () => {
 	console.log("Shutting down due to SIGTERM");
@@ -54,7 +56,12 @@ async function main() {
 	await new Promise((resolve) =>
 		server.listen({ port }, () => resolve(undefined)),
 	);
-	await Promise.all([api.start(), cdn.start(), gateway.start()]);
+	await Promise.all([
+		api.start(),
+		cdn.start(),
+		gateway.start(),
+		federation.start(),
+	]);
 
 	Sentry.errorHandler(app);
 

src/gateway/opcodes/LazyRequest.ts

@@ -17,26 +17,26 @@
 */
 
 import {
-	getDatabase,
+	OPCODES,
-	getPermission,
+	Payload,
-	listenEvent,
+	Send,
+	WebSocket,
+	handlePresenceUpdate,
+} from "@spacebar/gateway";
+import {
+	Channel,
+	Datasource,
+	LazyRequestSchema,
 	Member,
+	Permissions,
+	Presence,
 	Role,
 	Session,
-	LazyRequestSchema,
 	User,
-	Presence,
+	getPermission,
+	listenEvent,
 	partition,
-	Channel,
-	Permissions,
 } from "@spacebar/util";
-import {
-	WebSocket,
-	Payload,
-	handlePresenceUpdate,
-	OPCODES,
-	Send,
-} from "@spacebar/gateway";
 import murmur from "murmurhash-js/murmurhash3_gc";
 import { check } from "./instanceOf";
 
@@ -73,8 +73,7 @@ async function getMembers(guild_id: string, range: [number, number]) {
 	let members: Member[] = [];
 	try {
 		members =
-			(await getDatabase()
+			(await Datasource?.getRepository(Member)
-				?.getRepository(Member)
 				.createQueryBuilder("member")
 				.where("member.guild_id = :guild_id", { guild_id })
 				.leftJoinAndSelect("member.roles", "role")

src/util/config/Config.ts

@@ -23,6 +23,7 @@ import {
 	EmailConfiguration,
 	EndpointConfiguration,
 	ExternalTokensConfiguration,
+	FederationConfiguration,
 	GeneralConfiguration,
 	GifConfiguration,
 	GuildConfiguration,
@@ -61,4 +62,5 @@ export class ConfigValue {
 	email: EmailConfiguration = new EmailConfiguration();
 	passwordReset: PasswordResetConfiguration =
 		new PasswordResetConfiguration();
+	federation: FederationConfiguration = new FederationConfiguration();
 }

src/util/config/types/FederationConfiguration.ts

@@ -0,0 +1,12 @@
+export class FederationConfiguration {
+	/**
+	 * The S2S api domain, used for federation between instances.
+	 * Must match the DNS record that this instance runs on.
+	 */
+	host: string;
+
+	/** The domain used for account creation. Will appears in user handles, i.e. `@account@spacebar.chat` */
+	accountDomain: string;
+
+	enabled: boolean = false;
+}

src/util/config/types/index.ts

@@ -22,6 +22,7 @@ export * from "./DefaultsConfiguration";
 export * from "./EmailConfiguration";
 export * from "./EndpointConfiguration";
 export * from "./ExternalTokensConfiguration";
+export * from "./FederationConfiguration";
 export * from "./GeneralConfiguration";
 export * from "./GifConfiguration";
 export * from "./GuildConfiguration";
@@ -35,5 +36,5 @@ export * from "./RegionConfiguration";
 export * from "./RegisterConfiguration";
 export * from "./SecurityConfiguration";
 export * from "./SentryConfiguration";
-export * from "./subconfigurations";
 export * from "./TemplateConfiguration";
+export * from "./subconfigurations";

src/util/entities/BaseClass.ts

@@ -24,9 +24,9 @@ import {
 	ObjectIdColumn,
 	PrimaryColumn,
 } from "typeorm";
-import { Snowflake } from "../util/Snowflake";
-import { getDatabase } from "../util/Database";
 import { OrmUtils } from "../imports/OrmUtils";
+import { Datasource } from "../util/Datasource";
+import { Snowflake } from "../util/Snowflake";
 
 export class BaseClassWithoutId extends BaseEntity {
 	private get construct() {
@@ -34,7 +34,7 @@ export class BaseClassWithoutId extends BaseEntity {
 	}
 
 	private get metadata() {
-		return getDatabase()?.getMetadata(this.construct);
+		return Datasource.getMetadata(this.construct);
 	}
 
 	assign(props: object) {

src/util/entities/Channel.ts

@@ -28,6 +28,7 @@ import {
 import { DmChannelDTO } from "../dtos";
 import { ChannelCreateEvent, ChannelRecipientRemoveEvent } from "../interfaces";
 import {
+	Config,
 	InvisibleCharacters,
 	Snowflake,
 	containsAll,
@@ -36,6 +37,7 @@ import {
 	trimSpecial,
 } from "../util";
 import { BaseClass } from "./BaseClass";
+import { ActorType, FederationKey } from "./FederationKeys";
 import { Guild } from "./Guild";
 import { Invite } from "./Invite";
 import { Message } from "./Message";
@@ -193,6 +195,9 @@ export class Channel extends BaseClass {
 	@Column()
 	default_thread_rate_limit_per_user: number = 0;
 
+	@Column({ nullable: true, type: String, select: false })
+	domain: string | null; // federation. if null, we own this channel
+
 	// TODO: DM channel
 	static async createChannel(
 		channel: Partial<Channel>,
@@ -316,6 +321,16 @@ export class Channel extends BaseClass {
 				: Promise.resolve(),
 		]);
 
+		// If federation is enabled, generate signing keys for this actor.
+		setImmediate(
+			async () =>
+				Config.get().federation.enabled &&
+				(await FederationKey.generateSigningKeys(
+					ret.id,
+					ActorType.CHANNEL,
+				)),
+		);
+
 		return ret;
 	}
 

src/util/entities/FederationKeys.ts

@@ -0,0 +1,67 @@
+import { Column, Entity, PrimaryColumn } from "typeorm";
+import { BaseClassWithoutId } from "./BaseClass";
+
+import crypto from "crypto";
+import { promisify } from "util";
+const generateKeyPair = promisify(crypto.generateKeyPair);
+
+export enum ActorType {
+	USER = "users",
+	CHANNEL = "channels",
+	GUILD = "guilds",
+}
+
+@Entity("federation_keys")
+export class FederationKey extends BaseClassWithoutId {
+	/** The ID of this actor. */
+	@PrimaryColumn()
+	actorId: string;
+
+	/** The type of this actor. I.e. User, Channel, Guild */
+	@Column()
+	type: ActorType;
+
+	/** The domain of this actor. I.e. spacebar.chat */
+	@Column()
+	domain: string;
+
+	/** The remote ID ( actor URL ) of this user */
+	@Column()
+	federatedId: string;
+
+	/** The public key of this actor. Public keys of remote actors are cached. */
+	@Column()
+	publicKey: string;
+
+	/** Will only have a private key if this actor is ours */
+	@Column({ nullable: true, type: String })
+	privateKey: string | null;
+
+	/** Create a new FederationKey for an actor */
+	static generateSigningKeys = async (actorId: string, type: ActorType) => {
+		const existing = await FederationKey.findOne({ where: { actorId } });
+		if (existing) return existing;
+
+		// Lazy loading config to prevent circular dep
+		const { Config } = await import("../util/Config");
+
+		const keys = FederationKey.create({
+			actorId,
+			type,
+			domain: Config.get().federation.accountDomain,
+			...(await generateKeyPair("rsa", {
+				modulusLength: 4096,
+				publicKeyEncoding: {
+					type: "spki",
+					format: "pem",
+				},
+				privateKeyEncoding: {
+					type: "pkcs8",
+					format: "pem",
+				},
+			})),
+		});
+
+		return await keys.save();
+	};
+}

src/util/entities/User.ts

@@ -25,7 +25,15 @@ import {
 	OneToMany,
 	OneToOne,
 } from "typeorm";
-import { Config, Email, FieldErrors, Snowflake, trimSpecial } from "..";
+import {
+	ActorType,
+	Config,
+	Email,
+	FederationKey,
+	FieldErrors,
+	Snowflake,
+	trimSpecial,
+} from "..";
 import { BitField } from "../util/BitField";
 import { BaseClass } from "./BaseClass";
 import { ConnectedAccount } from "./ConnectedAccount";
@@ -182,6 +190,15 @@ export class User extends BaseClass {
 	@Column({ type: "bigint" })
 	rights: string;
 
+	@Column({ nullable: true, type: String, select: false })
+	domain: string | null; // Federation. null means this user is our own
+
+	@Column({ nullable: true, type: String, select: false })
+	privateKey: string | null; // No private key if federation is disabled
+
+	@Column({ nullable: true, type: String, select: false })
+	publicKey: string | null; // No public key if federation is disabled
+
 	@OneToMany(() => Session, (session: Session) => session.user)
 	sessions: Session[];
 
@@ -406,6 +423,16 @@ export class User extends BaseClass {
 			}
 		});
 
+		// If federation is enabled, generate signing keys for this actor.
+		setImmediate(
+			async () =>
+				Config.get().federation.enabled &&
+				(await FederationKey.generateSigningKeys(
+					user.id,
+					ActorType.USER,
+				)),
+		);
+
 		return user;
 	}
 }

src/util/entities/index.ts

@@ -31,6 +31,7 @@ export * from "./ConnectionConfigEntity";
 export * from "./EmbedCache";
 export * from "./Emoji";
 export * from "./Encryption";
+export * from "./FederationKeys";
 export * from "./Guild";
 export * from "./Invite";
 export * from "./Member";

src/util/schemas/responses/WebfingerResponse.ts

@@ -0,0 +1,12 @@
+interface WebfingerLink {
+	rel: string;
+	type?: string;
+	href: string;
+	template?: string;
+}
+
+export interface WebfingerResponse {
+	subject: string;
+	aliases: string[];
+	links: WebfingerLink[];
+}

src/util/schemas/responses/index.ts

@@ -28,7 +28,8 @@ export * from "./TypedResponses";
 export * from "./UpdatesResponse";
 export * from "./UserNoteResponse";
 export * from "./UserProfileResponse";
-export * from "./UserRelationshipsResponse";
 export * from "./UserRelationsResponse";
+export * from "./UserRelationshipsResponse";
 export * from "./WebAuthnCreateResponse";
+export * from "./WebfingerResponse";
 export * from "./WebhookCreateResponse";

src/util/util/Database.ts

@@ -16,57 +16,21 @@
 	along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
 
-import { config } from "dotenv";
-import path from "path";
 import { green, red, yellow } from "picocolors";
 import { DataSource } from "typeorm";
 import { ConfigEntity } from "../entities/Config";
 import { Migration } from "../entities/Migration";
+import { Datasource } from "./Datasource";
 
 // UUID extension option is only supported with postgres
 // We want to generate all id's with Snowflakes that's why we have our own BaseEntity class
 
-let dbConnection: DataSource | undefined;
-
-// For typeorm cli
-if (!process.env) {
-	config();
-}
-
-const dbConnectionString =
-	process.env.DATABASE || path.join(process.cwd(), "database.db");
-
-const DatabaseType = dbConnectionString.includes("://")
-	? dbConnectionString.split(":")[0]?.replace("+srv", "")
-	: "sqlite";
-const isSqlite = DatabaseType.includes("sqlite");
-
-const DataSourceOptions = new DataSource({
-	// eslint-disable-next-line @typescript-eslint/ban-ts-comment
-	//@ts-ignore type 'string' is not 'mysql' | 'sqlite' | 'mariadb' | etc etc
-	type: DatabaseType,
-	charset: "utf8mb4",
-	url: isSqlite ? undefined : dbConnectionString,
-	database: isSqlite ? dbConnectionString : undefined,
-	entities: [path.join(__dirname, "..", "entities", "*.js")],
-	synchronize: !!process.env.DB_SYNC,
-	logging: !!process.env.DB_LOGGING,
-	bigNumberStrings: false,
-	supportBigNumbers: true,
-	name: "default",
-	migrations: [path.join(__dirname, "..", "migration", DatabaseType, "*.js")],
-});
-
-// Gets the existing database connection
-export function getDatabase(): DataSource | null {
-	// if (!dbConnection) throw new Error("Tried to get database before it was initialised");
-	if (!dbConnection) return null;
-	return dbConnection;
-}
-
 // Called once on server start
 export async function initDatabase(): Promise<DataSource> {
-	if (dbConnection) return dbConnection;
+	if (Datasource.isInitialized) return Datasource;
+
+	const DatabaseType = Datasource.options.type;
+	const isSqlite = DatabaseType.includes("sqlite");
 
 	if (isSqlite) {
 		console.log(
@@ -92,7 +56,7 @@ export async function initDatabase(): Promise<DataSource> {
 
 	console.log(`[Database] ${yellow(`Connecting to ${DatabaseType} db`)}`);
 
-	dbConnection = await DataSourceOptions.initialize();
+	await Datasource.initialize();
 
 	// Crude way of detecting if the migrations table exists.
 	const dbExists = async () => {
@@ -107,12 +71,12 @@ export async function initDatabase(): Promise<DataSource> {
 		console.log(
 			"[Database] This appears to be a fresh database. Synchronising.",
 		);
-		await dbConnection.synchronize();
+		await Datasource.synchronize();
 
 		// On next start, typeorm will try to run all the migrations again from beginning.
 		// Manually insert every current migration to prevent this:
 		await Promise.all(
-			dbConnection.migrations.map((migration) =>
+			Datasource.migrations.map((migration) =>
 				Migration.insert({
 					name: migration.name,
 					timestamp: Date.now(),
@@ -121,16 +85,14 @@ export async function initDatabase(): Promise<DataSource> {
 		);
 	} else {
 		console.log("[Database] Applying missing migrations, if any.");
-		await dbConnection.runMigrations();
+		await Datasource.runMigrations();
 	}
 
 	console.log(`[Database] ${green("Connected")}`);
 
-	return dbConnection;
+	return Datasource;
 }
 
-export { DataSourceOptions, DatabaseType, dbConnection };
-
 export async function closeDatabase() {
-	await dbConnection?.destroy();
+	await Datasource?.destroy();
 }

src/util/util/Datasource.ts

@@ -0,0 +1,32 @@
+import { config } from "dotenv";
+import path from "path";
+import { DataSource } from "typeorm";
+
+// For typeorm cli
+if (!process.env) {
+	config();
+}
+
+const dbConnectionString =
+	process.env.DATABASE || path.join(process.cwd(), "database.db");
+
+const DatabaseType = dbConnectionString.includes("://")
+	? dbConnectionString.split(":")[0]?.replace("+srv", "")
+	: "sqlite";
+const isSqlite = DatabaseType.includes("sqlite");
+
+export const Datasource = new DataSource({
+	// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+	//@ts-ignore type 'string' is not 'mysql' | 'sqlite' | 'mariadb' | etc etc
+	type: DatabaseType,
+	charset: "utf8mb4",
+	url: isSqlite ? undefined : dbConnectionString,
+	database: isSqlite ? dbConnectionString : undefined,
+	entities: [path.join(__dirname, "..", "entities", "*.js")],
+	synchronize: !!process.env.DB_SYNC,
+	logging: !!process.env.DB_LOGGING,
+	bigNumberStrings: false,
+	supportBigNumbers: true,
+	name: "default",
+	migrations: [path.join(__dirname, "..", "migration", DatabaseType, "*.js")],
+});

src/util/util/index.ts

@@ -20,13 +20,14 @@ export * from "./ApiError";
 export * from "./Array";
 export * from "./BitField";
 //export * from "./Categories";
-export * from "./cdn";
+export * from "./Application";
 export * from "./Config";
 export * from "./Constants";
 export * from "./Database";
-export * from "./email";
+export * from "./Datasource";
 export * from "./Event";
 export * from "./FieldError";
+export * from "./Gifs";
 export * from "./Intents";
 export * from "./InvisibleCharacters";
 export * from "./JSON";
@@ -41,5 +42,6 @@ export * from "./String";
 export * from "./Token";
 export * from "./TraverseDirectory";
 export * from "./WebAuthn";
-export * from "./Gifs";
+export * from "./cdn";
-export * from "./Application";
+export * from "./email";
+export * from "./morgan";

src/util/util/morgan.ts

@@ -0,0 +1,32 @@
+import Express from "express";
+import morgan from "morgan";
+import { red } from "picocolors";
+
+let HAS_WARNED = false;
+export const setupMorganLogging = (app: Express.Application) => {
+	const logRequests = process.env["LOG_REQUESTS"] != undefined;
+	if (!logRequests) return;
+
+	if (!HAS_WARNED)
+		console.log(
+			red(
+				`Warning: Request logging is enabled! This will spam your console!\nTo disable this, unset the 'LOG_REQUESTS' environment variable!`,
+			),
+		);
+
+	HAS_WARNED = true;
+
+	app.use(
+		morgan("combined", {
+			skip: (req, res) => {
+				let skip = !(
+					process.env["LOG_REQUESTS"]?.includes(
+						res.statusCode.toString(),
+					) ?? false
+				);
+				if (process.env["LOG_REQUESTS"]?.charAt(0) == "-") skip = !skip;
+				return skip;
+			},
+		}),
+	);
+};

tsconfig.json

@@ -37,7 +37,8 @@
 			"@spacebar/api*": ["./api"],
 			"@spacebar/gateway*": ["./gateway"],
 			"@spacebar/cdn*": ["./cdn"],
-			"@spacebar/util*": ["./util"]
+			"@spacebar/util*": ["./util"],
+			"@spacebar/ap*": ["./activitypub"]
 		} /* Specify a set of entries that re-map imports to additional lookup locations. */,
 		// "rootDirs": [],                                   /* Allow multiple folders to be treated as one when resolving modules. */
 		// "typeRoots": [],                                  /* Specify multiple folders that act like './node_modules/@types'. */