Merge branch 'fosscord:master' into master

package-lock.json

@@ -1,12 +1,12 @@
 {
 	"name": "@fosscord/server-util",
-	"version": "1.3.16",
+	"version": "1.3.31",
 	"lockfileVersion": 2,
 	"requires": true,
 	"packages": {
 		"": {
 			"name": "@fosscord/server-util",
-			"version": "1.3.16",
+			"version": "1.3.31",
 			"license": "GPLV3",
 			"dependencies": {
 				"@types/jsonwebtoken": "^8.5.0",

package.json

@@ -1,12 +1,13 @@
 {
 	"name": "@fosscord/server-util",
-	"version": "1.3.16",
+	"version": "1.3.31",
 	"description": "Utility functions for the all server repositories",
 	"main": "dist/index.js",
 	"types": "dist/index.d.ts",
 	"scripts": {
 		"test": "echo \"Error: no test specified\" && exit 1",
-		"build": "tsc -b ."
+		"build": "tsc -b .",
+		"prepublish": "npm run build"
 	},
 	"repository": {
 		"type": "git",

src/models/Message.ts

@@ -342,6 +342,15 @@ MessageSchema.virtual("mention_channels", {
 	autopopulate: { select: { id: true, guild_id: true, type: true, name: true } },
 });
 
+
+MessageSchema.virtual("referenced_message", {
+	ref: "Message",
+	localField: "message_reference.message_id",
+	foreignField: "id",
+	justOne: true,
+	autopopulate: true,
+});
+
 MessageSchema.virtual("created_at").get(function (this: MessageDocument) {
 	return new Date(Snowflake.deconstruct(this.id).timestamp);
 });
@@ -358,3 +367,4 @@ MessageSchema.set("removeResponse", ["mention_channel_ids", "mention_role_ids",
 
 // @ts-ignore
 export const MessageModel = db.model<MessageDocument>("Message", MessageSchema, "messages");
+

src/models/RateLimit.ts

@@ -0,0 +1,25 @@
+import { Schema, Document, Types } from "mongoose";
+import db from "../util/Database";
+
+export interface Bucket {
+	id: "global" | "error" | string; // channel_239842397 | guild_238927349823 | webhook_238923423498
+	user_id: string;
+	hits: number;
+	blocked: boolean;
+	expires_at: Date;
+}
+
+export interface BucketDocument extends Bucket, Document {
+	id: string;
+}
+
+export const BucketSchema = new Schema({
+	id: { type: String, required: true },
+	user_id: { type: String, required: true }, // bot, user, oauth_application, webhook
+	hits: { type: Number, required: true }, // Number of times the user hit this bucket
+	blocked: { type: Boolean, required: true },
+	expires_at: { type: Date, required: true },
+});
+
+// @ts-ignore
+export const BucketModel = db.model<BucketDocument>("Bucket", BucketSchema, "ratelimits");

src/models/index.ts

@@ -1,7 +1,42 @@
-import mongoose from "mongoose";
-import { Schema } from "mongoose";
+import mongoose, { Schema, Document } from "mongoose";
 import mongooseAutoPopulate from "mongoose-autopopulate";
 
+type UpdateWithAggregationPipeline = UpdateAggregationStage[];
+type UpdateAggregationStage =
+	| { $addFields: any }
+	| { $set: any }
+	| { $project: any }
+	| { $unset: any }
+	| { $replaceRoot: any }
+	| { $replaceWith: any };
+type EnforceDocument<T, TMethods> = T extends Document ? T : T & Document & TMethods;
+
+declare module "mongoose" {
+	interface Model<T, TQueryHelpers = {}, TMethods = {}> {
+		// removed null -> always return document -> throw error if it doesn't exist
+		findOne(
+			filter?: FilterQuery<T>,
+			projection?: any | null,
+			options?: QueryOptions | null,
+			callback?: (err: CallbackError, doc: EnforceDocument<T, TMethods>) => void
+		): QueryWithHelpers<EnforceDocument<T, TMethods>, EnforceDocument<T, TMethods>, TQueryHelpers>;
+		findOneAndUpdate(
+			filter?: FilterQuery<T>,
+			update?: UpdateQuery<T> | UpdateWithAggregationPipeline,
+			options?: QueryOptions | null,
+			callback?: (err: any, doc: EnforceDocument<T, TMethods> | null, res: any) => void
+		): QueryWithHelpers<EnforceDocument<T, TMethods>, EnforceDocument<T, TMethods>, TQueryHelpers>;
+	}
+}
+
+var HTTPError: any;
+
+try {
+	HTTPError = require("lambert-server").HTTPError;
+} catch (e) {
+	HTTPError = Error;
+}
+
 mongoose.plugin(mongooseAutoPopulate);
 
 mongoose.plugin((schema: Schema, opts: any) => {
@@ -17,6 +52,18 @@ mongoose.plugin((schema: Schema, opts: any) => {
 			});
 		},
 	});
+	schema.post("findOne", function (doc, next) {
+		try {
+			// @ts-ignore
+			const isExistsQuery = JSON.stringify(this._userProvidedFields) === JSON.stringify({ _id: 1 });
+			if (!doc && !isExistsQuery) return next(new HTTPError("Not found", 404));
+			// @ts-ignore
+			return next();
+		} catch (error) {
+			// @ts-ignore
+			next();
+		}
+	});
 });
 
 export * from "./Activity";
@@ -35,3 +82,4 @@ export * from "./Status";
 export * from "./Role";
 export * from "./User";
 export * from "./VoiceState";
+export * from "./RateLimit";

src/util/Config.ts

@@ -4,12 +4,12 @@ import db, { MongooseCache } from "./Database";
 import { Snowflake } from "./Snowflake";
 import crypto from "crypto";
 
-var Config = new MongooseCache(db.collection("config"), [], { onlyEvents: false });
+var Config = new MongooseCache(db.collection("config"), [], { onlyEvents: false, array: false });
 
 export default {
 	init: async function init(defaultOpts: any = DefaultOptions) {
 		await Config.init();
-		return this.set(Config.data.merge(defaultOpts));
+		return this.set((Config.data || {}).merge(defaultOpts));
 	},
 	get: function get() {
 		return <DefaultOptions>Config.data;
@@ -88,6 +88,7 @@ export interface DefaultOptions {
 			sitekey: string | null;
 			secret: string | null;
 		};
+		ipdataApiKey: string | null;
 	};
 	login: {
 		requireCaptcha: boolean;
@@ -107,6 +108,7 @@ export interface DefaultOptions {
 		requireInvite: boolean;
 		allowNewRegistration: boolean;
 		allowMultipleAccounts: boolean;
+		blockProxies: boolean;
 		password: {
 			minLength: number;
 			minNumbers: number;
@@ -176,6 +178,7 @@ export const DefaultOptions: DefaultOptions = {
 			sitekey: null,
 			secret: null,
 		},
+		ipdataApiKey: "eca677b284b3bac29eb72f5e496aa9047f26543605efe99ff2ce35c9",
 	},
 	login: {
 		requireCaptcha: false,
@@ -196,6 +199,7 @@ export const DefaultOptions: DefaultOptions = {
 		requireCaptcha: true,
 		allowNewRegistration: true,
 		allowMultipleAccounts: true,
+		blockProxies: true,
 		password: {
 			minLength: 8,
 			minNumbers: 2,

src/util/Database.ts

@@ -2,11 +2,10 @@ import "./MongoBigInt";
 import mongoose, { Collection, Connection, LeanDocument } from "mongoose";
 import { ChangeStream, ChangeEvent, Long } from "mongodb";
 import EventEmitter from "events";
-import { Document } from "mongoose";
 const uri = process.env.MONGO_URL || "mongodb://localhost:27017/fosscord?readPreference=secondaryPreferred";
+import { URL } from "url";
 
-// TODO: auto throw error if findOne doesn't find anything
-console.log(`[DB] connect: ${uri}`);
+const url = new URL(uri.replace("mongodb://", "http://"));
 
 const connection = mongoose.createConnection(uri, {
 	autoIndex: true,
@@ -14,6 +13,7 @@ const connection = mongoose.createConnection(uri, {
 	useUnifiedTopology: true,
 	useFindAndModify: false,
 });
+console.log(`[Database] connect: mongodb://${url.username}@${url.host}${url.pathname}${url.search}`);
 
 export default <Connection>connection;
 
@@ -47,29 +47,38 @@ export interface MongooseCache {
 export class MongooseCache extends EventEmitter {
 	public stream: ChangeStream;
 	public data: any;
+	public initalizing?: Promise<void>;
 
 	constructor(
 		public collection: Collection,
 		public pipeline: Array<Record<string, unknown>>,
 		public opts: {
 			onlyEvents: boolean;
+			array?: boolean;
 		}
 	) {
 		super();
+		if (this.opts.array == null) this.opts.array = true;
 	}
 
-	init = async () => {
-		// @ts-ignore
-		this.stream = this.collection.watch(this.pipeline, { fullDocument: "updateLookup" });
+	init = () => {
+		if (this.initalizing) return this.initalizing;
+		this.initalizing = new Promise(async (resolve, reject) => {
+			// @ts-ignore
+			this.stream = this.collection.watch(this.pipeline, { fullDocument: "updateLookup" });
 
-		this.stream.on("change", this.change);
-		this.stream.on("close", this.destroy);
-		this.stream.on("error", console.error);
+			this.stream.on("change", this.change);
+			this.stream.on("close", this.destroy);
+			this.stream.on("error", console.error);
 
-		if (!this.opts.onlyEvents) {
-			const arr = await this.collection.aggregate(this.pipeline).toArray();
-			this.data = arr.length ? arr[0] : arr;
-		}
+			if (!this.opts.onlyEvents) {
+				const arr = await this.collection.aggregate(this.pipeline).toArray();
+				if (this.opts.array) this.data = arr || [];
+				else this.data = arr?.[0];
+			}
+			resolve();
+		});
+		return this.initalizing;
 	};
 
 	changeStream = (pipeline: any) => {
@@ -91,23 +100,34 @@ export class MongooseCache extends EventEmitter {
 
 	change = (doc: ChangeEvent) => {
 		try {
-			// @ts-ignore
-			if (doc.fullDocument) {
-				// @ts-ignore
-				if (!this.opts.onlyEvents) this.data = doc.fullDocument;
-			}
-
 			switch (doc.operationType) {
 				case "dropDatabase":
 					return this.destroy();
 				case "drop":
 					return this.destroy();
 				case "delete":
+					if (!this.opts.onlyEvents) {
+						if (this.opts.array) {
+							this.data = this.data.filter((x: any) => doc.documentKey?._id?.equals(x._id));
+						} else this.data = null;
+					}
 					return this.emit("delete", doc.documentKey._id.toHexString());
 				case "insert":
+					if (!this.opts.onlyEvents) {
+						if (this.opts.array) this.data.push(doc.fullDocument);
+						else this.data = doc.fullDocument;
+					}
 					return this.emit("insert", doc.fullDocument);
 				case "update":
 				case "replace":
+					if (!this.opts.onlyEvents) {
+						if (this.opts.array) {
+							const i = this.data.findIndex((x: any) => doc.fullDocument?._id?.equals(x._id));
+							if (i == -1) this.data.push(doc.fullDocument);
+							else this.data[i] = doc.fullDocument;
+						} else this.data = doc.fullDocument;
+					}
+
 					return this.emit("change", doc.fullDocument);
 				case "invalidate":
 					return this.destroy();
@@ -120,6 +140,7 @@ export class MongooseCache extends EventEmitter {
 	};
 
 	destroy = () => {
+		this.data = null;
 		this.stream?.off("change", this.change);
 		this.emit("close");
 

src/util/checkToken.ts

@@ -4,16 +4,21 @@ import { UserModel } from "../models";
 
 export function checkToken(token: string, jwtSecret: string): Promise<any> {
 	return new Promise((res, rej) => {
+		token = token.replace("Bot ", ""); // TODO: proper bot support
 		jwt.verify(token, jwtSecret, JWTOptions, async (err, decoded: any) => {
 			if (err || !decoded) return rej("Invalid Token");
 
-			const user = await UserModel.findOne({ id: decoded.id }, { "user_data.valid_tokens_since": true }).exec();
+			const user = await UserModel.findOne(
+				{ id: decoded.id },
+				{ "user_data.valid_tokens_since": true, bot: true }
+			).exec();
 			if (!user) return rej("Invalid Token");
-			if (decoded.iat * 1000 < user.user_data.valid_tokens_since.getTime()) return rej("Invalid Token");
+			// we need to round it to seconds as it saved as seconds in jwt iat and valid_tokens_since is stored in milliseconds
+			if (decoded.iat * 1000 < user.user_data.valid_tokens_since.setSeconds(0, 0)) return rej("Invalid Token");
 			if (user.disabled) return rej("User disabled");
 			if (user.deleted) return rej("User not found");
 
-			return res(decoded);
+			return res({ decoded, user });
 		});
 	});
 }