From c237247f896cce6e7262c13a874fb0a609cb0d86 Mon Sep 17 00:00:00 2001 From: Madeline <46743919+MaddyUnderStars@users.noreply.github.com> Date: Tue, 26 Sep 2023 01:22:40 +1000 Subject: [PATCH] initial bullshit for federation v2 --- package-lock.json | 36 ++- package.json | 6 +- src/activitypub/README.md | 142 ++++++++++ src/activitypub/Server.ts | 69 +++++ .../federation/OrderedCollection.ts | 38 +++ src/activitypub/federation/index.ts | 17 ++ src/activitypub/federation/queue.ts | 76 +++++ src/activitypub/federation/transforms.ts | 267 ++++++++++++++++++ src/activitypub/federation/utils.ts | 176 ++++++++++++ src/activitypub/index.ts | 2 + .../routes/channels/#channel_id/inbox.ts | 35 +++ .../routes/channels/#channel_id/index.ts | 16 ++ .../#channel_id/messages/#message_id/index.ts | 18 ++ .../routes/channels/#channel_id/outbox.ts | 53 ++++ src/activitypub/routes/index.ts | 9 + .../routes/users/#user_id/index.ts | 16 ++ src/activitypub/well-known.ts | 102 +++++++ src/api/Server.ts | 42 +-- src/api/routes/-/healthz.ts | 6 +- src/api/routes/-/readyz.ts | 6 +- src/bundle/Server.ts | 17 +- src/gateway/opcodes/LazyRequest.ts | 31 +- src/util/config/Config.ts | 2 + .../config/types/FederationConfiguration.ts | 12 + src/util/config/types/index.ts | 3 +- src/util/entities/BaseClass.ts | 6 +- src/util/entities/Channel.ts | 15 + src/util/entities/FederationKeys.ts | 67 +++++ src/util/entities/User.ts | 29 +- src/util/entities/index.ts | 1 + .../schemas/responses/WebfingerResponse.ts | 12 + src/util/schemas/responses/index.ts | 3 +- src/util/util/Database.ts | 60 +--- src/util/util/Datasource.ts | 32 +++ src/util/util/index.ts | 10 +- src/util/util/morgan.ts | 32 +++ tsconfig.json | 3 +- 37 files changed, 1341 insertions(+), 126 deletions(-) create mode 100644 src/activitypub/README.md create mode 100644 src/activitypub/Server.ts create mode 100644 src/activitypub/federation/OrderedCollection.ts create mode 100644 src/activitypub/federation/index.ts create mode 100644 src/activitypub/federation/queue.ts create mode 100644 src/activitypub/federation/transforms.ts create mode 100644 src/activitypub/federation/utils.ts create mode 100644 src/activitypub/index.ts create mode 100644 src/activitypub/routes/channels/#channel_id/inbox.ts create mode 100644 src/activitypub/routes/channels/#channel_id/index.ts create mode 100644 src/activitypub/routes/channels/#channel_id/messages/#message_id/index.ts create mode 100644 src/activitypub/routes/channels/#channel_id/outbox.ts create mode 100644 src/activitypub/routes/index.ts create mode 100644 src/activitypub/routes/users/#user_id/index.ts create mode 100644 src/activitypub/well-known.ts create mode 100644 src/util/config/types/FederationConfiguration.ts create mode 100644 src/util/entities/FederationKeys.ts create mode 100644 src/util/schemas/responses/WebfingerResponse.ts create mode 100644 src/util/util/Datasource.ts create mode 100644 src/util/util/morgan.ts diff --git a/package-lock.json b/package-lock.json index aabad644..6c0eb84d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "@aws-sdk/client-s3": "^3.385.0", "@sentry/integrations": "^7.66.0", "@sentry/node": "^7.66.0", + "activitypub-core-types": "^0.3.2", "ajv": "8.6.2", "ajv-formats": "2.1.1", "amqplib": "^0.10.3", @@ -48,6 +49,7 @@ "reflect-metadata": "^0.1.13", "ts-node": "^10.9.1", "tslib": "^2.6.1", + "turndown": "^7.1.2", "typeorm": "^0.3.17", "typescript-json-schema": "^0.50.1", "wretch": "^2.6.0", @@ -71,6 +73,7 @@ "@types/nodemailer": "^6.4.9", "@types/probe-image-size": "^7.2.0", "@types/sharp": "^0.31.1", + "@types/turndown": "^5.0.2", "@types/ws": "^8.5.5", "@typescript-eslint/eslint-plugin": "^5.62.0", "@typescript-eslint/parser": "^5.62.0", @@ -2253,6 +2256,12 @@ "@types/node": "*" } }, + "node_modules/@types/turndown": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/@types/turndown/-/turndown-5.0.2.tgz", + "integrity": "sha512-ghbjIyvMSQn/UGEuQJD6C4DfbokyYqGRhNAetWH02qnuRfvRZz9qTOG9e0RPkVqGsjv+YsjF3gRp7yFKvc/1PA==", + "dev": true + }, "node_modules/@types/ws": { "version": "8.5.5", "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.5.tgz", @@ -2495,6 +2504,14 @@ "node": ">=0.4.0" } }, + "node_modules/activitypub-core-types": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/activitypub-core-types/-/activitypub-core-types-0.3.2.tgz", + "integrity": "sha512-hAWCkRIzLJ3eVEjnibPYNHQaM+vD0SCK29gqMEt5pEb/2pbEetkv+4MpVMi0CKtr/GWRCSjIw1C6YAph7yT0pA==", + "dependencies": { + "formidable": "^2.1.1" + } + }, "node_modules/addressparser": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/addressparser/-/addressparser-1.0.1.tgz", @@ -2725,8 +2742,7 @@ "node_modules/asap": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", - "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==", - "optional": true + "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==" }, "node_modules/asn1js": { "version": "3.0.5", @@ -3589,7 +3605,6 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.4.tgz", "integrity": "sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==", - "optional": true, "dependencies": { "asap": "^2.0.0", "wrappy": "1" @@ -3665,6 +3680,11 @@ "url": "https://github.com/fb55/domhandler?sponsor=1" } }, + "node_modules/domino": { + "version": "2.1.6", + "resolved": "https://registry.npmjs.org/domino/-/domino-2.1.6.tgz", + "integrity": "sha512-3VdM/SXBZX2omc9JF9nOPCtDaYQ67BGp5CoLpIQlO2KCAPETs8TcDHacF26jXadGbvUteZzRTeos2fhID5+ucQ==" + }, "node_modules/domutils": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz", @@ -4444,7 +4464,6 @@ "version": "2.1.2", "resolved": "https://registry.npmjs.org/formidable/-/formidable-2.1.2.tgz", "integrity": "sha512-CM3GuJ57US06mlpQ47YcunuUZ9jpm8Vx+P2CGt2j7HpgkKZO/DJYQ0Bobim8G6PFQmK5lOqOOdUXboU+h73A4g==", - "optional": true, "dependencies": { "dezalgo": "^1.0.4", "hexoid": "^1.0.0", @@ -4723,7 +4742,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/hexoid/-/hexoid-1.0.0.tgz", "integrity": "sha512-QFLV0taWQOZtvIRIAdBChesmogZrtuXvVWsFHZTk2SU+anspqZ2vMnoLg7IE1+Uk16N19APic1BuF8bC8c2m5g==", - "optional": true, "engines": { "node": ">=8" } @@ -7756,6 +7774,14 @@ "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", "dev": true }, + "node_modules/turndown": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/turndown/-/turndown-7.1.2.tgz", + "integrity": "sha512-ntI9R7fcUKjqBP6QU8rBK2Ehyt8LAzt3UBT9JR9tgo6GtuKvyUzpayWmeMKJw1DPdXzktvtIT8m2mVXz+bL/Qg==", + "dependencies": { + "domino": "^2.1.6" + } + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", diff --git a/package.json b/package.json index 3994670b..bfd6afb5 100644 --- a/package.json +++ b/package.json @@ -54,6 +54,7 @@ "@types/nodemailer": "^6.4.9", "@types/probe-image-size": "^7.2.0", "@types/sharp": "^0.31.1", + "@types/turndown": "^5.0.2", "@types/ws": "^8.5.5", "@typescript-eslint/eslint-plugin": "^5.62.0", "@typescript-eslint/parser": "^5.62.0", @@ -68,6 +69,7 @@ "@aws-sdk/client-s3": "^3.385.0", "@sentry/integrations": "^7.66.0", "@sentry/node": "^7.66.0", + "activitypub-core-types": "^0.3.2", "ajv": "8.6.2", "ajv-formats": "2.1.1", "amqplib": "^0.10.3", @@ -103,6 +105,7 @@ "reflect-metadata": "^0.1.13", "ts-node": "^10.9.1", "tslib": "^2.6.1", + "turndown": "^7.1.2", "typeorm": "^0.3.17", "typescript-json-schema": "^0.50.1", "wretch": "^2.6.0", @@ -112,7 +115,8 @@ "@spacebar/api": "dist/api", "@spacebar/cdn": "dist/cdn", "@spacebar/gateway": "dist/gateway", - "@spacebar/util": "dist/util" + "@spacebar/util": "dist/util", + "@spacebar/ap": "dist/activitypub" }, "optionalDependencies": { "erlpack": "^0.1.4", diff --git a/src/activitypub/README.md b/src/activitypub/README.md new file mode 100644 index 00000000..4674d1ee --- /dev/null +++ b/src/activitypub/README.md @@ -0,0 +1,142 @@ +# Spacebar Activitypub + +- [Activitystreams vocab](https://www.w3.org/TR/activitystreams-vocabulary) +- [Activitystreams](https://www.w3.org/TR/activitystreams-core) +- [Activitypub spec](https://www.w3.org/TR/activitypub/) + +## Supported Types + +| Spacebar object | Activitypub | +| --------------- | ---------------------------------------------------------------------------------- | +| Message | [Note](https://www.w3.org/TR/activitystreams-vocabulary/#dfn-note) | +| Channel | [Group](https://www.w3.org/TR/activitystreams-vocabulary/#dfn-group) | +| Guild | [Organisation](https://www.w3.org/TR/activitystreams-vocabulary/#dfn-organization) | +| User | [Person](https://www.w3.org/TR/activitystreams-vocabulary/#dfn-person) | +| Role | Spacebar extension: [Role](#role-federation) | + +## Message Federation + +A message sent by a user. Sent to channels, or directly to users (a DM channel is created on Spacebar instances). + +### Supported Activities + +| Activity | Action | +| ---------- | ---------------------------------------------------- | +| `Create` | Transformed from a Note to a Message and saved to db | +| `Delete` | Removes a message from db | +| `Update` | Updates a message and saves to db. | +| `Announce` | Used by Channels to forward to members. | + +### Properties Used + +| Property | Description | +| ---------------------- | ---------------------------------------------------------------------------- | +| `type` | Must be `"Note"` | +| `content` | Message content | +| `name` | Used as message content if `content` not provided | +| `inReplyTo` | Reference a previous Message in this guild | +| `published` | Timestamp of this Message | +| `attributedTo` | Message author | +| `to` | The Channel this Message is a part of | +| `tag` | Mentions | +| `tag[].type` | Must be `Mention` | +| `tag[].name` | Plain-type Webfinger address of a Profile within this Guild OR `@everyone` | +| `attachment` | Message attachments | +| `attachment[].url` | The URL of this media attachment | +| `attachment[].summary` | The content warning for this media attachment | +| `replies` | For compatibility with other software: The replies to this message | +| `sbType` | Spacebar extension. Describes the real MessageType. i.e. `GUILD_MEMBER_JOIN` | +| `embeds` | Spacebar extension. Describes the attached Embeds | +| `flags` | Spacebar extension. Message flags as bitfield | +| TODO: reactions | How does plemora/akkoma/misskey/etc handle reactions? | +| TODO: components | | +| TODO: stickers | | + +## Channel Federation + +An automated actor. Users can send messages to it, which the channel forwards to it's followers in an `Announce`. +Follows/is followed by it's corresponding Guild, if applicable. + +### Supported Activities + +| Activity | Action | +| -------------- | ----------------------------------------------------- | +| `Create` | Transformed from a Group to a Channel and saved to db | +| `Delete` | Removes a channel from db | +| `Update` | Updates channel details | +| `Add`/`Remove` | Manage pinned Messages for this Channel | + +### Properties Used + +| Property | Description | +| -------------- | --------------------------------------------------------------------- | +| `type` | Must be `"Group"` | +| `name` | The Channel name | +| `published` | Creation timestamp of this Channel | +| `attributedTo` | The Guild this Channel is a part of | +| `featured` | Mastodon extension. The pinned Messages in this Channel | +| `publicKey` | The public key used to verify signatures from this actor | +| `sbType` | Spacebar extension. Describes the real ChannelType. i.e. `GUILD_TEXT` | + +## Guild Federation + +An automated actor. Follows and is followed by it's corresponding Channels. +Also contains a collection of [roles](#role-federation). + +### Supported Activities + +| Activity | Action | +| -------- | ------------------------------------------------------------------ | +| `Follow` | Join a guild. Must provide an invite code. Automatically accepted. | +| `Delete` | Delete a guild. | +| `Update` | Update guild details. | + +### Properties Used + +## User Federation + +A person. Sends messages to Channels. May also create, modify, or moderate guilds, channels, or roles. +Is a partOf a [Role](#role-federation) + +### Supported Activities + +| Activity | Action | +| ----------------- | ---------------------------------------------------------------------------------------------- | +| `Follow` | Send a friend request | +| `Accept`/`Reject` | Accept or reject a friend request | +| `Undo` | Unfriend | +| `Delete` | Delete a user from the database along with all their messages. | +| `Block` | Signal to the remote server that they should hide your profile from that user. Not guaranteed. | +| `Update` | Update user details. | + +## Role Federation + +Is a [Collection](https://www.w3.org/TR/activitystreams-vocabulary/#dfn-collection) of Users within this role. + +## S2S endpoints + +Base url: `/federation` + +- `/.well-known/webfinger?resource=acct@domain` - Returns webfinger response i.e. https://docs.joinmastodon.org/spec/webfinger/ +- `/.well-known/host-meta` - Returns location of webfinger? Why is this neccessary? + +- `/channels/:channel_id` - Returns specified Channel as AP object ( Group ) +- `/channels/:channel_id/inbox` - The inbox for this Channel +- `/channels/:channel_id/outbox` - The outbox for this Channel +- `/channels/:channel_id/followers` - The Users that have access to this Channel + +- `/channels/:channel_id/messages/:message_id` - Returns specified Message in Channel as AP object ( Announce Note ) +- +- `/messages/:message_id` - Returns specified Message in Channel as AP object ( Announce Note ) + +- `/activities/:activity_id` - Returns the specified activitypub activity. E.g. Announce, Follow, etc. +- `/activities/inbox` - Shared inbox. + +- `/users/:user_id` - Returns specified User as AP object (Person) +- `/users/:user_id/inbox` - The inbox of this User. POSTing creates a DM channel if it does not exist. + +- `/guilds/:guild_id` - Returns specified Guild as AP object (Organisation) + +## notes + +- activitypub responses should be returned if the Accept header is `application/ld+json; profile="https://www.w3.org/ns/activitystreams"` OR `application/activity+json` diff --git a/src/activitypub/Server.ts b/src/activitypub/Server.ts new file mode 100644 index 00000000..9b540c68 --- /dev/null +++ b/src/activitypub/Server.ts @@ -0,0 +1,69 @@ +import { BodyParser, CORS, ErrorHandler } from "@spacebar/api"; +import { + Config, + JSONReplacer, + Sentry, + initDatabase, + registerRoutes, + setupMorganLogging, +} from "@spacebar/util"; +import { Request, Response, Router } from "express"; +import { Server, ServerOptions } from "lambert-server"; +import path from "path"; +import wellknown from "./well-known"; + +export type SpacebarServerOptions = ServerOptions; + +export class FederationServer extends Server { + public declare options: SpacebarServerOptions; + + constructor(opts?: Partial) { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + super({ ...opts, errorHandler: false, jsonBody: false }); + } + + async start() { + await initDatabase(); + await Config.init(); + await Sentry.init(this.app); + + setupMorganLogging(this.app); + this.app.set("json replacer", JSONReplacer); + + this.app.use(CORS); + this.app.use(BodyParser({ inflate: true, limit: "10mb" })); + + const app = this.app; + const api = Router(); + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + this.app = api; + + // TODO: auth + // TODO: rate limits + + this.routes = await registerRoutes( + this, + path.join(__dirname, "routes", "/"), + ); + + api.use("*", (req: Request, res: Response) => { + res.status(404).json({ + message: "404 endpoint not found", + code: 0, + }); + }); + + this.app = app; + + this.app.use("/federation", api); + this.app.use("/.well-known", wellknown); + + this.app.use(ErrorHandler); + + Sentry.errorHandler(this.app); + + return super.start(); + } +} diff --git a/src/activitypub/federation/OrderedCollection.ts b/src/activitypub/federation/OrderedCollection.ts new file mode 100644 index 00000000..70a40754 --- /dev/null +++ b/src/activitypub/federation/OrderedCollection.ts @@ -0,0 +1,38 @@ +import { AP } from "activitypub-core-types"; +import { ACTIVITYSTREAMS_CONTEXT } from "./utils"; + +export const makeOrderedCollection = async (opts: { + page: boolean; + min_id?: string; + max_id?: string; + id: URL; + getTotalElements: () => Promise; + getElements: (before?: string, after?: string) => Promise; +}): Promise => { + const { page, min_id, max_id, id, getTotalElements, getElements } = opts; + + if (!page) + return { + "@context": ACTIVITYSTREAMS_CONTEXT, + id: id, + type: "OrderedCollection", + totalItems: await getTotalElements(), + first: new URL(`${id}?page=true`), + last: new URL(`${id}?page=true&min_id=0`), + }; + + const after = min_id ? `${min_id}` : undefined; + const before = max_id ? `${max_id}` : undefined; + + const elems = await getElements(before, after); + + return { + "@context": ACTIVITYSTREAMS_CONTEXT, + id: new URL(`${id}?page=true`), + type: "OrderedCollection", + first: new URL(`${id}?page=true`), + last: new URL(`${id}?page=true&min_id=0`), + totalItems: await getTotalElements(), + orderedItems: elems, + }; +}; diff --git a/src/activitypub/federation/index.ts b/src/activitypub/federation/index.ts new file mode 100644 index 00000000..62c93b18 --- /dev/null +++ b/src/activitypub/federation/index.ts @@ -0,0 +1,17 @@ +/** + * To be injected into API + * Responsible for dispatching activitypub events to external instances + */ + +import { AP } from "activitypub-core-types"; +import { federationQueue } from "./queue"; + +export * from "./OrderedCollection"; +export * from "./transforms"; +export * from "./utils"; + +export class Federation { + static async distribute(activity: AP.Activity) { + await federationQueue.distribute(activity); + } +} diff --git a/src/activitypub/federation/queue.ts b/src/activitypub/federation/queue.ts new file mode 100644 index 00000000..5559b8f5 --- /dev/null +++ b/src/activitypub/federation/queue.ts @@ -0,0 +1,76 @@ +import { Config, FederationKey } from "@spacebar/util"; +import { AP } from "activitypub-core-types"; +import fetch from "node-fetch"; +import { + APError, + resolveWebfinger, + signActivity, + splitQualifiedMention, +} from "./utils"; + +// +type Instance = string; + +class FederationQueue { + // TODO: queue messages and send them to shared inbox + private queue: Map> = new Map(); + + public async distribute(activity: AP.Activity) { + let { to, actor } = activity; + + if (!to) + throw new APError("Activity with no `to` field is undeliverable."); + if (!Array.isArray(to)) to = [to]; + + if (!actor) + throw new APError("Activity with no `to` field is undeliverable."); + if (Array.isArray(actor)) actor = actor[0]; + + // TODO: check if `to` is on our instance? + // we shouldn't get to this point if they are, though. + + // if the sender is one of ours, fetch their private key for signing + const { user } = splitQualifiedMention(actor.toString()); + const sender = await FederationKey.findOneOrFail({ + where: { actorId: user, domain: Config.get().federation.host }, + }); + + if (!sender.privateKey) { + console.warn( + "tried to federate activity who's sender does not have a private key", + ); + return; + } + + for (const receiver of to) { + if (!(receiver instanceof URL)) { + console.error(receiver); + continue; + } + + const apReceiver = await resolveWebfinger(receiver.toString()); + if (!("inbox" in apReceiver)) { + console.error( + "[federation] receiver doesn't have inbox", + apReceiver, + ); + continue; + } + + if (typeof apReceiver.inbox != "string") { + console.error(apReceiver.inbox); + continue; + } + + const signedActivity = await signActivity( + apReceiver.inbox, + sender, + activity, + ); + + await fetch(apReceiver.inbox, signedActivity); + } + } +} + +export const federationQueue = new FederationQueue(); diff --git a/src/activitypub/federation/transforms.ts b/src/activitypub/federation/transforms.ts new file mode 100644 index 00000000..4ce6471a --- /dev/null +++ b/src/activitypub/federation/transforms.ts @@ -0,0 +1,267 @@ +import { + ActorType, + Channel, + Config, + DmChannelDTO, + FederationKey, + Member, + Message, + Snowflake, + User, + UserSettings, +} from "@spacebar/util"; +import { AP } from "activitypub-core-types"; +import TurndownService from "turndown"; +import { + ACTIVITYSTREAMS_CONTEXT, + APError, + APObjectIsPerson, + resolveAPObject, +} from "./utils"; + +export const transformMessageToAnnounceNoce = async ( + message: Message, +): Promise => { + const { host } = Config.get().federation; + + return { + "@context": ACTIVITYSTREAMS_CONTEXT, + type: "Announce", + id: new URL( + `https://${host}/federation/channels/${message.channel_id}/messages/${message.id}`, + ), + actor: new URL(`https://${host}/federation/users/${message.author_id}`), + published: message.timestamp, + to: [ + new URL( + `https://${host}/federation/channels/${message.channel_id}/followers`, + ), + ], + object: await transformMessageToNote(message), + }; +}; + +export const transformMessageToNote = async ( + message: Message, +): Promise => { + const { host } = Config.get().federation; + + const referencedMessage = message.message_reference + ? await Message.findOne({ + where: { id: message.message_reference.message_id }, + }) + : null; + + return { + id: new URL(`https://${host}/federation/messages/${message.id}`), + type: "Note", + content: message.content, // TODO: convert markdown to html + inReplyTo: referencedMessage + ? await transformMessageToNote(referencedMessage) + : undefined, + published: message.timestamp, + attributedTo: new URL( + `https://${host}/federation/users/${message.author_id}`, + ), + to: [ + new URL( + `https://${host}/federation/channels/${message.channel_id}`, + ), + ], + tag: message.mentions?.map( + (x) => new URL(`https://${host}/federation/users/${x.id}`), + ), + attachment: [], + // replies: [], + // sbType: message.type, + // embeds: [], + // flags: message.flags, + }; +}; + +// TODO: this was copied from the previous implemention. refactor it. +export const transformNoteToMessage = async (note: AP.Note) => { + if (!note.id) throw new APError("Note must have ID"); + if (note.type != "Note") throw new APError("Message must be Note"); + + if (!note.attributedTo) + throw new APError("Note must have author (attributedTo"); + + const attrib = await resolveAPObject( + Array.isArray(note.attributedTo) + ? note.attributedTo[0] + : note.attributedTo, + ); + + if (!APObjectIsPerson(attrib)) + throw new APError("Note must be attributedTo a Person"); + + const user = await transformPersonToUser(attrib); + + const to = Array.isArray(note.to) ? note.to[0] : note.to; + + let channel: Channel | DmChannelDTO; + const to_id = to?.toString().split("/").reverse()[0]; + if (to?.toString().includes("user")) { + // this is a DM channel + const toUser = await User.findOneOrFail({ where: { id: to_id } }); + + // Channel.createDMCHannel does a .save() so the author must be present + await user.save(); + + // const cache = await Channel.findOne({ where: { recipients: []}}) + + channel = await Channel.createDMChannel( + [toUser.id, user.id], + toUser.id, + ); + } else { + channel = await Channel.findOneOrFail({ + where: { id: to_id }, + relations: { guild: true }, + }); + } + + const member = + channel instanceof Channel + ? await Member.findOneOrFail({ + where: { id: user.id, guild_id: channel.guild!.id }, + }) + : undefined; + + return Message.create({ + id: Snowflake.generate(), + content: new TurndownService().turndown(note.content), + timestamp: note.published, + author: user, + guild: channel instanceof Channel ? channel.guild : undefined, + member, + channel_id: channel.id, + + nonce: note.id.toString(), + type: 0, + sticker_items: [], + attachments: [], + embeds: [], + reactions: [], + mentions: [], + mention_roles: [], + mention_channels: [], + }); +}; + +export const transformChannelToGroup = async ( + channel: Channel, +): Promise => { + const { host, accountDomain } = Config.get().federation; + + const keys = await FederationKey.findOneOrFail({ + where: { actorId: channel.id, domain: accountDomain }, + }); + + return { + "@context": "https://www.w3.org/ns/activitystreams", + type: "Group", + id: new URL(`https://${host}/fed/channels/${channel.id}`), + name: channel.name, + preferredUsername: channel.id, + summary: channel.topic, + icon: undefined, + // discoverable: true, + + publicKey: { + id: `https://${host}/fed/user/${channel.id}#main-key`, + owner: `https://${host}/fed/user/${channel.id}`, + publicKeyPem: keys.publicKey, + }, + + inbox: new URL(`https://${host}/fed/channels/${channel.id}/inbox`), + outbox: new URL(`https://${host}/fed/channels/${channel.id}/outbox`), + followers: new URL( + `https://${host}/fed/channels/${channel.id}/followers`, + ), + }; +}; + +export const transformUserToPerson = async (user: User): Promise => { + const { host, accountDomain } = Config.get().federation; + + const keys = await FederationKey.findOneOrFail({ + where: { actorId: user.id, domain: accountDomain }, + }); + + return { + "@context": ACTIVITYSTREAMS_CONTEXT, + type: "Person", + id: new URL(`https://${host}/federation/users/${user.id}`), + + name: user.username, + preferredUsername: user.id, + summary: user.bio, + icon: user.avatar + ? [ + new URL( + `${Config.get().cdn.endpointPublic}/avatars/${ + user.id + }/${user.avatar}`, + ), + ] + : undefined, + + inbox: new URL(`https://${host}/federation/users/${user.id}/inbox`), + outbox: new URL(`https://${host}/federation/users/${user.id}/outbox`), + followers: new URL( + `https://${host}/federation/users/${user.id}/followers`, + ), + publicKey: { + id: `https://${host}/federation/users/${user.id}#main-key`, + owner: `https://${host}/federation/users/${user.id}`, + publicKeyPem: keys.publicKey, + }, + }; +}; + +// TODO: this was copied from previous implementation. refactor. +export const transformPersonToUser = async (person: AP.Person) => { + if (!person.id) throw new APError("User must have ID"); + + const url = new URL(person.id.toString()); + const email = `${url.pathname.split("/").reverse()[0]}@${url.hostname}`; + + const cachedKeys = await FederationKey.findOne({ + where: { federatedId: url.toString() }, + }); + if (cachedKeys) { + return await User.findOneOrFail({ where: { id: cachedKeys.actorId } }); + } + + await FederationKey.create({ + actorId: Snowflake.generate(), + federatedId: url.toString(), + domain: url.hostname, + publicKey: person.publicKey?.publicKeyPem, + type: ActorType.USER, + }).save(); + + return User.create({ + username: person.preferredUsername, + discriminator: url.hostname, + bio: new TurndownService().turndown(person.summary), + email, + data: { + hash: "#", + valid_tokens_since: new Date(), + }, + extended_settings: "{}", + settings: UserSettings.create(), + premium: false, + + premium_since: Config.get().defaults.user.premium + ? new Date() + : undefined, + rights: Config.get().register.defaultRights, + premium_type: Config.get().defaults.user.premiumType ?? 0, + verified: Config.get().defaults.user.verified ?? true, + created_at: new Date(), + }).save(); +}; diff --git a/src/activitypub/federation/utils.ts b/src/activitypub/federation/utils.ts new file mode 100644 index 00000000..b69bccf4 --- /dev/null +++ b/src/activitypub/federation/utils.ts @@ -0,0 +1,176 @@ +import { DEFAULT_FETCH_OPTIONS } from "@spacebar/api"; +import { + Config, + FederationKey, + OrmUtils, + WebfingerResponse, +} from "@spacebar/util"; +import { AP } from "activitypub-core-types"; +import crypto from "crypto"; +import { HTTPError } from "lambert-server"; +import fetch from "node-fetch"; +import { ProxyAgent } from "proxy-agent"; + +export const ACTIVITYSTREAMS_CONTEXT = "https://www.w3.org/ns/activitystreams"; + +export const fetchOpts = OrmUtils.mergeDeep(DEFAULT_FETCH_OPTIONS, { + headers: { + Accept: "application/activity+json", + }, +}); + +export class APError extends HTTPError {} + +export const hasAPContext = (data: object) => { + if (!("@context" in data)) return false; + const context = data["@context"]; + const activitystreams = "https://www.w3.org/ns/activitystreams"; + if (Array.isArray(context)) + return context.find((x) => x == activitystreams); + return context == activitystreams; +}; + +export const resolveAPObject = async (data: string | T): Promise => { + // we were already given an AP object + if (typeof data != "string") return data; + + const agent = new ProxyAgent(); + const ret = await fetch(data, { + ...fetchOpts, + agent, + }); + + const json = await ret.json(); + + if (!hasAPContext(json)) throw new APError("Object is not APObject"); + + return json; +}; + +export const splitQualifiedMention = (lookup: string) => { + let domain: string, user: string; + if (lookup.includes("@")) { + // lookup a @handle@domain + + if (lookup[0] == "@") lookup = lookup.slice(1); + [user, domain] = lookup.split("@"); + } else { + // lookup was a URL ( hopefully ) + try { + const url = new URL(lookup); + domain = url.hostname; + user = url.pathname.split("/").reverse()[0]; + } catch (e) { + domain = ""; + user = ""; + } + } + + return { + domain, + user, + }; +}; + +export const resolveWebfinger = async ( + lookup: string, +): Promise => { + const { domain } = splitQualifiedMention(lookup); + + const agent = new ProxyAgent(); + const wellknown = (await fetch( + `https://${domain}/.well-known/webfinger?resource=${lookup}`, + { + agent, + ...fetchOpts, + }, + ).then((x) => x.json())) as WebfingerResponse; + + const link = wellknown.links.find((x) => x.rel == "self"); + if (!link) throw new APError(".well-known did not contain rel=self link"); + + return await resolveAPObject(link.href); +}; + +/** + * Returns a signed request that can be passed to fetch + * ``` + * const signed = await signActivity(receiver.inbox, sender, activity); + * await fetch(receiver.inbox, signed); + * ``` + */ +export const signActivity = async ( + inbox: string, + sender: FederationKey, + message: AP.Activity, +) => { + if (!sender.privateKey) + throw new APError("cannot sign without private key"); + + const digest = crypto + .createHash("sha256") + .update(JSON.stringify(message)) + .digest("base64"); + const signer = crypto.createSign("sha256"); + const now = new Date(); + + const url = new URL(inbox); + const inboxFrag = url.pathname; + const toSign = + `(request-target): post ${inboxFrag}\n` + + `host: ${url.hostname}\n` + + `date: ${now.toUTCString()}\n` + + `digest: SHA-256=${digest}`; + + signer.update(toSign); + signer.end(); + + const signature = signer.sign(sender.privateKey); + const sig_b64 = signature.toString("base64"); + + const { host } = Config.get().federation; + const header = + `keyId="${host}/${sender.type}/${sender.actorId}#main-key",` + + `headers="(request-target) host date digest",` + + `signature=${sig_b64}`; + + return OrmUtils.mergeDeep(fetchOpts, { + method: "POST", + body: message, + headers: { + Host: url.hostname, + Date: now.toUTCString(), + Digest: `SHA-256=${digest}`, + Signature: header, + }, + }); +}; + +// fetch from remote server? +export const APObjectIsPerson = ( + object: AP.EntityReference, +): object is AP.Person => { + return "type" in object && object.type == "Person"; +}; + +export const APObjectIsGroup = ( + object: AP.EntityReference, +): object is AP.Person => { + return "type" in object && object.type == "Group"; +}; + +export const APObjectIsOrganisation = ( + object: AP.EntityReference, +): object is AP.Person => { + return "type" in object && object.type == "Organization"; +}; + +export const APObjectIsSpacebarActor = ( + object: AP.EntityReference, +): object is AP.Person => { + return ( + APObjectIsGroup(object) || + APObjectIsOrganisation(object) || + APObjectIsPerson(object) + ); +}; diff --git a/src/activitypub/index.ts b/src/activitypub/index.ts new file mode 100644 index 00000000..e4de82d7 --- /dev/null +++ b/src/activitypub/index.ts @@ -0,0 +1,2 @@ +export * from "./Server"; +export * from "./federation"; diff --git a/src/activitypub/routes/channels/#channel_id/inbox.ts b/src/activitypub/routes/channels/#channel_id/inbox.ts new file mode 100644 index 00000000..5414ab48 --- /dev/null +++ b/src/activitypub/routes/channels/#channel_id/inbox.ts @@ -0,0 +1,35 @@ +import { transformNoteToMessage } from "@spacebar/ap"; +import { route } from "@spacebar/api"; +import { Message, emitEvent } from "@spacebar/util"; +import { AP } from "activitypub-core-types"; +import { Request, Response, Router } from "express"; +import { HTTPError } from "lambert-server"; +const router = Router(); + +// TODO: check if the activity exists on the remote server +router.post("/", route({}), async (req: Request, res: Response) => { + const body = req.body as AP.Create; + + if (body.type != "Create") throw new HTTPError("not implemented"); + + const object = Array.isArray(body.object) ? body.object[0] : body.object; + if (!object) return res.status(400); + if (!("type" in object) || object.type != "Note") + throw new HTTPError("must be Note"); + const message = await transformNoteToMessage(object as AP.Note); + + if ((await Message.count({ where: { nonce: object.id!.toString() } })) != 0) + return res.status(200); + + await message.save(); + + await emitEvent({ + event: "MESSAGE_CREATE", + channel_id: message.channel_id, + data: message.toJSON(), + }); + + return res.status(200); +}); + +export default router; diff --git a/src/activitypub/routes/channels/#channel_id/index.ts b/src/activitypub/routes/channels/#channel_id/index.ts new file mode 100644 index 00000000..509c6f2f --- /dev/null +++ b/src/activitypub/routes/channels/#channel_id/index.ts @@ -0,0 +1,16 @@ +import { transformChannelToGroup } from "@spacebar/ap"; +import { route } from "@spacebar/api"; +import { Channel } from "@spacebar/util"; +import { Request, Response, Router } from "express"; +const router = Router(); + +// TODO: auth +router.get("/", route({}), async (req: Request, res: Response) => { + const channel = await Channel.findOneOrFail({ + where: { id: req.params.channel_id }, + }); + + return res.json(await transformChannelToGroup(channel)); +}); + +export default router; diff --git a/src/activitypub/routes/channels/#channel_id/messages/#message_id/index.ts b/src/activitypub/routes/channels/#channel_id/messages/#message_id/index.ts new file mode 100644 index 00000000..a5ee5ea7 --- /dev/null +++ b/src/activitypub/routes/channels/#channel_id/messages/#message_id/index.ts @@ -0,0 +1,18 @@ +import { transformMessageToAnnounceNoce } from "@spacebar/ap"; +import { route } from "@spacebar/api"; +import { Message } from "@spacebar/util"; +import { Request, Response, Router } from "express"; +const router = Router(); + +// TODO: auth +router.get("/", route({}), async (req: Request, res: Response) => { + const { channel_id, message_id } = req.params; + + const message = await Message.findOneOrFail({ + where: { channel_id, id: message_id }, + }); + + return res.json(await transformMessageToAnnounceNoce(message)); +}); + +export default router; diff --git a/src/activitypub/routes/channels/#channel_id/outbox.ts b/src/activitypub/routes/channels/#channel_id/outbox.ts new file mode 100644 index 00000000..4eedb210 --- /dev/null +++ b/src/activitypub/routes/channels/#channel_id/outbox.ts @@ -0,0 +1,53 @@ +import { + makeOrderedCollection, + transformMessageToAnnounceNoce, +} from "@spacebar/ap"; +import { route } from "@spacebar/api"; +import { Config, Message, Snowflake } from "@spacebar/util"; +import { AP } from "activitypub-core-types"; +import { Request, Response, Router } from "express"; +import { FindManyOptions, FindOperator, LessThan, MoreThan } from "typeorm"; +const router = Router(); + +router.get("/", route({}), async (req: Request, res: Response) => { + const { channel_id } = req.params; + const { page, min_id, max_id } = req.query; + + const { host } = Config.get().federation; + + const ret = await makeOrderedCollection({ + page: page != undefined, + min_id: min_id?.toString(), + max_id: max_id?.toString(), + id: new URL(`https://${host}/federation/channels/${channel_id}/outbox`), + getTotalElements: () => Message.count({ where: { channel_id } }), + getElements: async (before, after): Promise => { + const query: FindManyOptions & { + where: { id?: FindOperator | FindOperator[] }; + } = { + order: { timestamp: "DESC" }, + take: 20, + where: { channel_id: channel_id }, + relations: ["author"], + }; + + if (after) { + if (BigInt(after) > BigInt(Snowflake.generate())) return []; + query.where.id = MoreThan(after); + } else if (before) { + if (BigInt(before) > BigInt(Snowflake.generate())) return []; + query.where.id = LessThan(before); + } + + const messages = await Message.find(query); + + return await Promise.all( + messages.map((x) => transformMessageToAnnounceNoce(x)), + ); + }, + }); + + return res.json(ret); +}); + +export default router; diff --git a/src/activitypub/routes/index.ts b/src/activitypub/routes/index.ts new file mode 100644 index 00000000..93cad49a --- /dev/null +++ b/src/activitypub/routes/index.ts @@ -0,0 +1,9 @@ +import { route } from "@spacebar/api"; +import { Request, Response, Router } from "express"; +const router = Router(); + +router.get("/", route({}), async (req: Request, res: Response) => { + res.send("Online"); +}); + +export default router; diff --git a/src/activitypub/routes/users/#user_id/index.ts b/src/activitypub/routes/users/#user_id/index.ts new file mode 100644 index 00000000..63c9957f --- /dev/null +++ b/src/activitypub/routes/users/#user_id/index.ts @@ -0,0 +1,16 @@ +import { transformUserToPerson } from "@spacebar/ap"; +import { route } from "@spacebar/api"; +import { User } from "@spacebar/util"; +import { Request, Response, Router } from "express"; +const router = Router(); + +// TODO: auth +router.get("/", route({}), async (req: Request, res: Response) => { + const user = await User.findOneOrFail({ + where: { id: req.params.user_id }, + }); + + return res.json(await transformUserToPerson(user)); +}); + +export default router; diff --git a/src/activitypub/well-known.ts b/src/activitypub/well-known.ts new file mode 100644 index 00000000..c20e1ad3 --- /dev/null +++ b/src/activitypub/well-known.ts @@ -0,0 +1,102 @@ +import { route } from "@spacebar/api"; +import { + ActorType, + Channel, + Config, + FederationKey, + FieldErrors, + Guild, + User, + WebfingerResponse, +} from "@spacebar/util"; +import { Request, Response, Router } from "express"; +import { HTTPError } from "lambert-server"; +import { splitQualifiedMention } from "./federation"; +const router = Router(); + +router.get( + "/webfinger", + route({ + query: { + resource: { + type: "string", + description: "Resource to locate", + }, + }, + responses: { + 200: { + body: "WebfingerResponse", + }, + }, + }), + async (req: Request, res: Response) => { + let resource = req.query.resource as string; + if (!resource) + throw FieldErrors({ + resource: { message: "Resource must be present" }, + }); + + // We know what you mean + resource = resource.replace("acct:", ""); + + const { accountDomain, host } = Config.get().federation; + + const { user, domain } = splitQualifiedMention(resource); + if (domain != accountDomain) + throw new HTTPError("Resource could not be found", 404); + + const keys = await FederationKey.findOneOrFail({ + where: { + actorId: user, + domain, + }, + select: ["type"], + }); + + let entity: User | Channel | Guild; + switch (keys.type) { + case ActorType.USER: + entity = await User.findOneOrFail({ where: { id: user } }); + break; + case ActorType.CHANNEL: + entity = await Channel.findOneOrFail({ where: { id: user } }); + break; + case ActorType.GUILD: + entity = await Guild.findOneOrFail({ where: { id: user } }); + break; + } + + res.setHeader("Content-Type", "application/jrd+json; charset=utf-8"); + return res.json({ + subject: `acct:${user}@${accountDomain}`, // mastodon always returns acct so might as well + aliases: [`https://${host}/federation/${keys.type}/${entity.id}`], + links: [ + { + rel: "self", + type: "application/activity+json", + href: `https://${host}/federation/${keys.type}/${entity.id}`, + }, + // { + // rel: "http://ostatus.org/schema/1.0/subscribe", + // href: `"https://${host}/fed/authorize-follow?acct={uri}"`, + // }, + ], + }); + }, +); + +router.get("/host-meta", route({}), (req, res) => { + res.setHeader("Content-Type", "application/xrd+xml"); + + const { host } = Config.get().federation; + + const ret = ` + + + `; + + return res.send(ret); +}); + +export default router; diff --git a/src/api/Server.ts b/src/api/Server.ts index 472ab1d6..e4b62b5f 100644 --- a/src/api/Server.ts +++ b/src/api/Server.ts @@ -18,22 +18,21 @@ import { Config, - Email, - initDatabase, - initEvent, - JSONReplacer, - registerRoutes, - Sentry, - WebAuthn, ConnectionConfig, ConnectionLoader, + Email, + JSONReplacer, + Sentry, + WebAuthn, + initDatabase, + initEvent, + registerRoutes, + setupMorganLogging, } from "@spacebar/util"; import { Request, Response, Router } from "express"; import { Server, ServerOptions } from "lambert-server"; import "missing-native-js-functions"; -import morgan from "morgan"; import path from "path"; -import { red } from "picocolors"; import { Authentication, CORS } from "./middlewares/"; import { BodyParser } from "./middlewares/BodyParser"; import { ErrorHandler } from "./middlewares/ErrorHandler"; @@ -79,23 +78,7 @@ export class SpacebarServer extends Server { await Sentry.init(this.app); WebAuthn.init(); - const logRequests = process.env["LOG_REQUESTS"] != undefined; - if (logRequests) { - this.app.use( - morgan("combined", { - skip: (req, res) => { - let skip = !( - process.env["LOG_REQUESTS"]?.includes( - res.statusCode.toString(), - ) ?? false - ); - if (process.env["LOG_REQUESTS"]?.charAt(0) == "-") - skip = !skip; - return skip; - }, - }), - ); - } + setupMorganLogging(this.app); this.app.set("json replacer", JSONReplacer); @@ -147,13 +130,6 @@ export class SpacebarServer extends Server { ConnectionLoader.loadConnections(); - if (logRequests) - console.log( - red( - `Warning: Request logging is enabled! This will spam your console!\nTo disable this, unset the 'LOG_REQUESTS' environment variable!`, - ), - ); - return super.start(); } } diff --git a/src/api/routes/-/healthz.ts b/src/api/routes/-/healthz.ts index 6a2f65de..851e8b2c 100644 --- a/src/api/routes/-/healthz.ts +++ b/src/api/routes/-/healthz.ts @@ -16,14 +16,14 @@ along with this program. If not, see . */ -import { Router, Response, Request } from "express"; import { route } from "@spacebar/api"; -import { getDatabase } from "@spacebar/util"; +import { Datasource } from "@spacebar/util"; +import { Request, Response, Router } from "express"; const router = Router(); router.get("/", route({}), (req: Request, res: Response) => { - if (!getDatabase()) return res.sendStatus(503); + if (!Datasource.isInitialized) return res.sendStatus(503); return res.sendStatus(200); }); diff --git a/src/api/routes/-/readyz.ts b/src/api/routes/-/readyz.ts index 6a2f65de..851e8b2c 100644 --- a/src/api/routes/-/readyz.ts +++ b/src/api/routes/-/readyz.ts @@ -16,14 +16,14 @@ along with this program. If not, see . */ -import { Router, Response, Request } from "express"; import { route } from "@spacebar/api"; -import { getDatabase } from "@spacebar/util"; +import { Datasource } from "@spacebar/util"; +import { Request, Response, Router } from "express"; const router = Router(); router.get("/", route({}), (req: Request, res: Response) => { - if (!getDatabase()) return res.sendStatus(503); + if (!Datasource.isInitialized) return res.sendStatus(503); return res.sendStatus(200); }); diff --git a/src/bundle/Server.ts b/src/bundle/Server.ts index d281120d..72d38603 100644 --- a/src/bundle/Server.ts +++ b/src/bundle/Server.ts @@ -19,13 +19,14 @@ process.on("unhandledRejection", console.error); process.on("uncaughtException", console.error); -import http from "http"; +import { FederationServer } from "@spacebar/ap"; import * as Api from "@spacebar/api"; -import * as Gateway from "@spacebar/gateway"; import { CDNServer } from "@spacebar/cdn"; +import * as Gateway from "@spacebar/gateway"; +import { Config, Sentry, initDatabase } from "@spacebar/util"; import express from "express"; -import { green, bold } from "picocolors"; -import { Config, initDatabase, Sentry } from "@spacebar/util"; +import http from "http"; +import { bold, green } from "picocolors"; const app = express(); const server = http.createServer(); @@ -36,6 +37,7 @@ server.on("request", app); const api = new Api.SpacebarServer({ server, port, production, app }); const cdn = new CDNServer({ server, port, production, app }); const gateway = new Gateway.Server({ server, port, production }); +const federation = new FederationServer({ server, port, production, app }); process.on("SIGTERM", async () => { console.log("Shutting down due to SIGTERM"); @@ -54,7 +56,12 @@ async function main() { await new Promise((resolve) => server.listen({ port }, () => resolve(undefined)), ); - await Promise.all([api.start(), cdn.start(), gateway.start()]); + await Promise.all([ + api.start(), + cdn.start(), + gateway.start(), + federation.start(), + ]); Sentry.errorHandler(app); diff --git a/src/gateway/opcodes/LazyRequest.ts b/src/gateway/opcodes/LazyRequest.ts index 4ad1ae7b..dfe27bc0 100644 --- a/src/gateway/opcodes/LazyRequest.ts +++ b/src/gateway/opcodes/LazyRequest.ts @@ -17,26 +17,26 @@ */ import { - getDatabase, - getPermission, - listenEvent, + OPCODES, + Payload, + Send, + WebSocket, + handlePresenceUpdate, +} from "@spacebar/gateway"; +import { + Channel, + Datasource, + LazyRequestSchema, Member, + Permissions, + Presence, Role, Session, - LazyRequestSchema, User, - Presence, + getPermission, + listenEvent, partition, - Channel, - Permissions, } from "@spacebar/util"; -import { - WebSocket, - Payload, - handlePresenceUpdate, - OPCODES, - Send, -} from "@spacebar/gateway"; import murmur from "murmurhash-js/murmurhash3_gc"; import { check } from "./instanceOf"; @@ -73,8 +73,7 @@ async function getMembers(guild_id: string, range: [number, number]) { let members: Member[] = []; try { members = - (await getDatabase() - ?.getRepository(Member) + (await Datasource?.getRepository(Member) .createQueryBuilder("member") .where("member.guild_id = :guild_id", { guild_id }) .leftJoinAndSelect("member.roles", "role") diff --git a/src/util/config/Config.ts b/src/util/config/Config.ts index 90b98b7a..f84773c0 100644 --- a/src/util/config/Config.ts +++ b/src/util/config/Config.ts @@ -23,6 +23,7 @@ import { EmailConfiguration, EndpointConfiguration, ExternalTokensConfiguration, + FederationConfiguration, GeneralConfiguration, GifConfiguration, GuildConfiguration, @@ -61,4 +62,5 @@ export class ConfigValue { email: EmailConfiguration = new EmailConfiguration(); passwordReset: PasswordResetConfiguration = new PasswordResetConfiguration(); + federation: FederationConfiguration = new FederationConfiguration(); } diff --git a/src/util/config/types/FederationConfiguration.ts b/src/util/config/types/FederationConfiguration.ts new file mode 100644 index 00000000..7a6f52ae --- /dev/null +++ b/src/util/config/types/FederationConfiguration.ts @@ -0,0 +1,12 @@ +export class FederationConfiguration { + /** + * The S2S api domain, used for federation between instances. + * Must match the DNS record that this instance runs on. + */ + host: string; + + /** The domain used for account creation. Will appears in user handles, i.e. `@account@spacebar.chat` */ + accountDomain: string; + + enabled: boolean = false; +} diff --git a/src/util/config/types/index.ts b/src/util/config/types/index.ts index 782ebfc3..8ea9819e 100644 --- a/src/util/config/types/index.ts +++ b/src/util/config/types/index.ts @@ -22,6 +22,7 @@ export * from "./DefaultsConfiguration"; export * from "./EmailConfiguration"; export * from "./EndpointConfiguration"; export * from "./ExternalTokensConfiguration"; +export * from "./FederationConfiguration"; export * from "./GeneralConfiguration"; export * from "./GifConfiguration"; export * from "./GuildConfiguration"; @@ -35,5 +36,5 @@ export * from "./RegionConfiguration"; export * from "./RegisterConfiguration"; export * from "./SecurityConfiguration"; export * from "./SentryConfiguration"; -export * from "./subconfigurations"; export * from "./TemplateConfiguration"; +export * from "./subconfigurations"; diff --git a/src/util/entities/BaseClass.ts b/src/util/entities/BaseClass.ts index f4b3cf59..d72c778d 100644 --- a/src/util/entities/BaseClass.ts +++ b/src/util/entities/BaseClass.ts @@ -24,9 +24,9 @@ import { ObjectIdColumn, PrimaryColumn, } from "typeorm"; -import { Snowflake } from "../util/Snowflake"; -import { getDatabase } from "../util/Database"; import { OrmUtils } from "../imports/OrmUtils"; +import { Datasource } from "../util/Datasource"; +import { Snowflake } from "../util/Snowflake"; export class BaseClassWithoutId extends BaseEntity { private get construct() { @@ -34,7 +34,7 @@ export class BaseClassWithoutId extends BaseEntity { } private get metadata() { - return getDatabase()?.getMetadata(this.construct); + return Datasource.getMetadata(this.construct); } assign(props: object) { diff --git a/src/util/entities/Channel.ts b/src/util/entities/Channel.ts index 9f7041d4..63e320a8 100644 --- a/src/util/entities/Channel.ts +++ b/src/util/entities/Channel.ts @@ -28,6 +28,7 @@ import { import { DmChannelDTO } from "../dtos"; import { ChannelCreateEvent, ChannelRecipientRemoveEvent } from "../interfaces"; import { + Config, InvisibleCharacters, Snowflake, containsAll, @@ -36,6 +37,7 @@ import { trimSpecial, } from "../util"; import { BaseClass } from "./BaseClass"; +import { ActorType, FederationKey } from "./FederationKeys"; import { Guild } from "./Guild"; import { Invite } from "./Invite"; import { Message } from "./Message"; @@ -193,6 +195,9 @@ export class Channel extends BaseClass { @Column() default_thread_rate_limit_per_user: number = 0; + @Column({ nullable: true, type: String, select: false }) + domain: string | null; // federation. if null, we own this channel + // TODO: DM channel static async createChannel( channel: Partial, @@ -316,6 +321,16 @@ export class Channel extends BaseClass { : Promise.resolve(), ]); + // If federation is enabled, generate signing keys for this actor. + setImmediate( + async () => + Config.get().federation.enabled && + (await FederationKey.generateSigningKeys( + ret.id, + ActorType.CHANNEL, + )), + ); + return ret; } diff --git a/src/util/entities/FederationKeys.ts b/src/util/entities/FederationKeys.ts new file mode 100644 index 00000000..f3f15de5 --- /dev/null +++ b/src/util/entities/FederationKeys.ts @@ -0,0 +1,67 @@ +import { Column, Entity, PrimaryColumn } from "typeorm"; +import { BaseClassWithoutId } from "./BaseClass"; + +import crypto from "crypto"; +import { promisify } from "util"; +const generateKeyPair = promisify(crypto.generateKeyPair); + +export enum ActorType { + USER = "users", + CHANNEL = "channels", + GUILD = "guilds", +} + +@Entity("federation_keys") +export class FederationKey extends BaseClassWithoutId { + /** The ID of this actor. */ + @PrimaryColumn() + actorId: string; + + /** The type of this actor. I.e. User, Channel, Guild */ + @Column() + type: ActorType; + + /** The domain of this actor. I.e. spacebar.chat */ + @Column() + domain: string; + + /** The remote ID ( actor URL ) of this user */ + @Column() + federatedId: string; + + /** The public key of this actor. Public keys of remote actors are cached. */ + @Column() + publicKey: string; + + /** Will only have a private key if this actor is ours */ + @Column({ nullable: true, type: String }) + privateKey: string | null; + + /** Create a new FederationKey for an actor */ + static generateSigningKeys = async (actorId: string, type: ActorType) => { + const existing = await FederationKey.findOne({ where: { actorId } }); + if (existing) return existing; + + // Lazy loading config to prevent circular dep + const { Config } = await import("../util/Config"); + + const keys = FederationKey.create({ + actorId, + type, + domain: Config.get().federation.accountDomain, + ...(await generateKeyPair("rsa", { + modulusLength: 4096, + publicKeyEncoding: { + type: "spki", + format: "pem", + }, + privateKeyEncoding: { + type: "pkcs8", + format: "pem", + }, + })), + }); + + return await keys.save(); + }; +} diff --git a/src/util/entities/User.ts b/src/util/entities/User.ts index c6582b00..c08ee95f 100644 --- a/src/util/entities/User.ts +++ b/src/util/entities/User.ts @@ -25,7 +25,15 @@ import { OneToMany, OneToOne, } from "typeorm"; -import { Config, Email, FieldErrors, Snowflake, trimSpecial } from ".."; +import { + ActorType, + Config, + Email, + FederationKey, + FieldErrors, + Snowflake, + trimSpecial, +} from ".."; import { BitField } from "../util/BitField"; import { BaseClass } from "./BaseClass"; import { ConnectedAccount } from "./ConnectedAccount"; @@ -182,6 +190,15 @@ export class User extends BaseClass { @Column({ type: "bigint" }) rights: string; + @Column({ nullable: true, type: String, select: false }) + domain: string | null; // Federation. null means this user is our own + + @Column({ nullable: true, type: String, select: false }) + privateKey: string | null; // No private key if federation is disabled + + @Column({ nullable: true, type: String, select: false }) + publicKey: string | null; // No public key if federation is disabled + @OneToMany(() => Session, (session: Session) => session.user) sessions: Session[]; @@ -406,6 +423,16 @@ export class User extends BaseClass { } }); + // If federation is enabled, generate signing keys for this actor. + setImmediate( + async () => + Config.get().federation.enabled && + (await FederationKey.generateSigningKeys( + user.id, + ActorType.USER, + )), + ); + return user; } } diff --git a/src/util/entities/index.ts b/src/util/entities/index.ts index aa943dca..708f2801 100644 --- a/src/util/entities/index.ts +++ b/src/util/entities/index.ts @@ -31,6 +31,7 @@ export * from "./ConnectionConfigEntity"; export * from "./EmbedCache"; export * from "./Emoji"; export * from "./Encryption"; +export * from "./FederationKeys"; export * from "./Guild"; export * from "./Invite"; export * from "./Member"; diff --git a/src/util/schemas/responses/WebfingerResponse.ts b/src/util/schemas/responses/WebfingerResponse.ts new file mode 100644 index 00000000..6b0ab0f9 --- /dev/null +++ b/src/util/schemas/responses/WebfingerResponse.ts @@ -0,0 +1,12 @@ +interface WebfingerLink { + rel: string; + type?: string; + href: string; + template?: string; +} + +export interface WebfingerResponse { + subject: string; + aliases: string[]; + links: WebfingerLink[]; +} diff --git a/src/util/schemas/responses/index.ts b/src/util/schemas/responses/index.ts index d8b7fd57..66b9986b 100644 --- a/src/util/schemas/responses/index.ts +++ b/src/util/schemas/responses/index.ts @@ -28,7 +28,8 @@ export * from "./TypedResponses"; export * from "./UpdatesResponse"; export * from "./UserNoteResponse"; export * from "./UserProfileResponse"; -export * from "./UserRelationshipsResponse"; export * from "./UserRelationsResponse"; +export * from "./UserRelationshipsResponse"; export * from "./WebAuthnCreateResponse"; +export * from "./WebfingerResponse"; export * from "./WebhookCreateResponse"; diff --git a/src/util/util/Database.ts b/src/util/util/Database.ts index 3a45eea0..317dd7eb 100644 --- a/src/util/util/Database.ts +++ b/src/util/util/Database.ts @@ -16,57 +16,21 @@ along with this program. If not, see . */ -import { config } from "dotenv"; -import path from "path"; import { green, red, yellow } from "picocolors"; import { DataSource } from "typeorm"; import { ConfigEntity } from "../entities/Config"; import { Migration } from "../entities/Migration"; +import { Datasource } from "./Datasource"; // UUID extension option is only supported with postgres // We want to generate all id's with Snowflakes that's why we have our own BaseEntity class -let dbConnection: DataSource | undefined; - -// For typeorm cli -if (!process.env) { - config(); -} - -const dbConnectionString = - process.env.DATABASE || path.join(process.cwd(), "database.db"); - -const DatabaseType = dbConnectionString.includes("://") - ? dbConnectionString.split(":")[0]?.replace("+srv", "") - : "sqlite"; -const isSqlite = DatabaseType.includes("sqlite"); - -const DataSourceOptions = new DataSource({ - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - //@ts-ignore type 'string' is not 'mysql' | 'sqlite' | 'mariadb' | etc etc - type: DatabaseType, - charset: "utf8mb4", - url: isSqlite ? undefined : dbConnectionString, - database: isSqlite ? dbConnectionString : undefined, - entities: [path.join(__dirname, "..", "entities", "*.js")], - synchronize: !!process.env.DB_SYNC, - logging: !!process.env.DB_LOGGING, - bigNumberStrings: false, - supportBigNumbers: true, - name: "default", - migrations: [path.join(__dirname, "..", "migration", DatabaseType, "*.js")], -}); - -// Gets the existing database connection -export function getDatabase(): DataSource | null { - // if (!dbConnection) throw new Error("Tried to get database before it was initialised"); - if (!dbConnection) return null; - return dbConnection; -} - // Called once on server start export async function initDatabase(): Promise { - if (dbConnection) return dbConnection; + if (Datasource.isInitialized) return Datasource; + + const DatabaseType = Datasource.options.type; + const isSqlite = DatabaseType.includes("sqlite"); if (isSqlite) { console.log( @@ -92,7 +56,7 @@ export async function initDatabase(): Promise { console.log(`[Database] ${yellow(`Connecting to ${DatabaseType} db`)}`); - dbConnection = await DataSourceOptions.initialize(); + await Datasource.initialize(); // Crude way of detecting if the migrations table exists. const dbExists = async () => { @@ -107,12 +71,12 @@ export async function initDatabase(): Promise { console.log( "[Database] This appears to be a fresh database. Synchronising.", ); - await dbConnection.synchronize(); + await Datasource.synchronize(); // On next start, typeorm will try to run all the migrations again from beginning. // Manually insert every current migration to prevent this: await Promise.all( - dbConnection.migrations.map((migration) => + Datasource.migrations.map((migration) => Migration.insert({ name: migration.name, timestamp: Date.now(), @@ -121,16 +85,14 @@ export async function initDatabase(): Promise { ); } else { console.log("[Database] Applying missing migrations, if any."); - await dbConnection.runMigrations(); + await Datasource.runMigrations(); } console.log(`[Database] ${green("Connected")}`); - return dbConnection; + return Datasource; } -export { DataSourceOptions, DatabaseType, dbConnection }; - export async function closeDatabase() { - await dbConnection?.destroy(); + await Datasource?.destroy(); } diff --git a/src/util/util/Datasource.ts b/src/util/util/Datasource.ts new file mode 100644 index 00000000..d9aa7abb --- /dev/null +++ b/src/util/util/Datasource.ts @@ -0,0 +1,32 @@ +import { config } from "dotenv"; +import path from "path"; +import { DataSource } from "typeorm"; + +// For typeorm cli +if (!process.env) { + config(); +} + +const dbConnectionString = + process.env.DATABASE || path.join(process.cwd(), "database.db"); + +const DatabaseType = dbConnectionString.includes("://") + ? dbConnectionString.split(":")[0]?.replace("+srv", "") + : "sqlite"; +const isSqlite = DatabaseType.includes("sqlite"); + +export const Datasource = new DataSource({ + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + //@ts-ignore type 'string' is not 'mysql' | 'sqlite' | 'mariadb' | etc etc + type: DatabaseType, + charset: "utf8mb4", + url: isSqlite ? undefined : dbConnectionString, + database: isSqlite ? dbConnectionString : undefined, + entities: [path.join(__dirname, "..", "entities", "*.js")], + synchronize: !!process.env.DB_SYNC, + logging: !!process.env.DB_LOGGING, + bigNumberStrings: false, + supportBigNumbers: true, + name: "default", + migrations: [path.join(__dirname, "..", "migration", DatabaseType, "*.js")], +}); diff --git a/src/util/util/index.ts b/src/util/util/index.ts index 10e09b5c..4093ec64 100644 --- a/src/util/util/index.ts +++ b/src/util/util/index.ts @@ -20,13 +20,14 @@ export * from "./ApiError"; export * from "./Array"; export * from "./BitField"; //export * from "./Categories"; -export * from "./cdn"; +export * from "./Application"; export * from "./Config"; export * from "./Constants"; export * from "./Database"; -export * from "./email"; +export * from "./Datasource"; export * from "./Event"; export * from "./FieldError"; +export * from "./Gifs"; export * from "./Intents"; export * from "./InvisibleCharacters"; export * from "./JSON"; @@ -41,5 +42,6 @@ export * from "./String"; export * from "./Token"; export * from "./TraverseDirectory"; export * from "./WebAuthn"; -export * from "./Gifs"; -export * from "./Application"; +export * from "./cdn"; +export * from "./email"; +export * from "./morgan"; diff --git a/src/util/util/morgan.ts b/src/util/util/morgan.ts new file mode 100644 index 00000000..93dc9ce6 --- /dev/null +++ b/src/util/util/morgan.ts @@ -0,0 +1,32 @@ +import Express from "express"; +import morgan from "morgan"; +import { red } from "picocolors"; + +let HAS_WARNED = false; +export const setupMorganLogging = (app: Express.Application) => { + const logRequests = process.env["LOG_REQUESTS"] != undefined; + if (!logRequests) return; + + if (!HAS_WARNED) + console.log( + red( + `Warning: Request logging is enabled! This will spam your console!\nTo disable this, unset the 'LOG_REQUESTS' environment variable!`, + ), + ); + + HAS_WARNED = true; + + app.use( + morgan("combined", { + skip: (req, res) => { + let skip = !( + process.env["LOG_REQUESTS"]?.includes( + res.statusCode.toString(), + ) ?? false + ); + if (process.env["LOG_REQUESTS"]?.charAt(0) == "-") skip = !skip; + return skip; + }, + }), + ); +}; diff --git a/tsconfig.json b/tsconfig.json index 63b5e96c..1ced0e3c 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -37,7 +37,8 @@ "@spacebar/api*": ["./api"], "@spacebar/gateway*": ["./gateway"], "@spacebar/cdn*": ["./cdn"], - "@spacebar/util*": ["./util"] + "@spacebar/util*": ["./util"], + "@spacebar/ap*": ["./activitypub"] } /* Specify a set of entries that re-map imports to additional lookup locations. */, // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ // "typeRoots": [], /* Specify multiple folders that act like './node_modules/@types'. */