🚧 typeorm

api/src/routes/users/@me/settings.ts

@@ -8,7 +8,8 @@ const router = Router();
 router.patch("/", check(UserSettingsSchema), async (req: Request, res: Response) => {
 	const body = req.body as UserSettings;
 
-	await UserModel.updateOne({ id: req.user_id }, body).exec();
+	// only users can update user settings
+	await UserModel.updateOne({ id: req.user_id, bot: false }, body).exec();
 
 	res.sendStatus(204);
 });

util/oldModels/Event.ts

@@ -3,7 +3,7 @@ import { DMChannel, Channel } from "./Channel";
 import { Guild } from "./Guild";
 import { Member, PublicMember, UserGuildSettings } from "./Member";
 import { Emoji } from "./Emoji";
-import { Presence } from "./Activity";
+import { Presence } from "../models/Activity";
 import { Role } from "./Role";
 import { Invite } from "./Invite";
 import { Message, PartialEmoji } from "./Message";

util/oldModels/index.ts

@@ -0,0 +1,93 @@
+// @ts-nocheck
+import mongoose, { Schema, Document } from "mongoose";
+import mongooseAutoPopulate from "mongoose-autopopulate";
+
+type UpdateWithAggregationPipeline = UpdateAggregationStage[];
+type UpdateAggregationStage =
+	| { $addFields: any }
+	| { $set: any }
+	| { $project: any }
+	| { $unset: any }
+	| { $replaceRoot: any }
+	| { $replaceWith: any };
+type EnforceDocument<T, TMethods> = T extends Document ? T : T & Document & TMethods;
+
+declare module "mongoose" {
+	interface SchemaOptions {
+		removeResponse?: string[];
+	}
+	interface Model<T, TQueryHelpers = {}, TMethods = {}> {
+		// removed null -> always return document -> throw error if it doesn't exist
+		findOne(
+			filter?: FilterQuery<T>,
+			projection?: any | null,
+			options?: QueryOptions | null,
+			callback?: (err: CallbackError, doc: EnforceDocument<T, TMethods>) => void
+		): QueryWithHelpers<EnforceDocument<T, TMethods>, EnforceDocument<T, TMethods>, TQueryHelpers>;
+		findOneAndUpdate(
+			filter?: FilterQuery<T>,
+			update?: UpdateQuery<T> | UpdateWithAggregationPipeline,
+			options?: QueryOptions | null,
+			callback?: (err: any, doc: EnforceDocument<T, TMethods> | null, res: any) => void
+		): QueryWithHelpers<EnforceDocument<T, TMethods>, EnforceDocument<T, TMethods>, TQueryHelpers>;
+	}
+}
+
+var HTTPError: any;
+
+try {
+	HTTPError = require("lambert-server").HTTPError;
+} catch (e) {
+	HTTPError = Error;
+}
+
+mongoose.plugin(mongooseAutoPopulate);
+
+mongoose.plugin((schema: Schema, opts: any) => {
+	schema.set("toObject", {
+		virtuals: true,
+		versionKey: false,
+		transform(doc: any, ret: any) {
+			delete ret._id;
+			delete ret.__v;
+			const props = schema.get("removeResponse") || [];
+			props.forEach((prop: string) => {
+				delete ret[prop];
+			});
+		},
+	});
+	schema.post("findOne", function (doc, next) {
+		try {
+			// @ts-ignore
+			const isExistsQuery = JSON.stringify(this._userProvidedFields) === JSON.stringify({ _id: 1 });
+			if (!doc && !isExistsQuery) {
+				// @ts-ignore
+				return next(new HTTPError(`${this?.mongooseCollection?.name}.${this?._conditions?.id} not found`, 400));
+			}
+			// @ts-ignore
+			return next();
+		} catch (error) {
+			// @ts-ignore
+			next();
+		}
+	});
+});
+
+export * from "../models/Activity";
+export * from "./Application";
+export * from "./Ban";
+export * from "./Channel";
+export * from "./Emoji";
+export * from "./Event";
+export * from "./Template";
+export * from "./Guild";
+export * from "./Invite";
+export * from "./Interaction";
+export * from "./Member";
+export * from "./Message";
+export * from "../models/Status";
+export * from "./Role";
+export * from "./User";
+export * from "./VoiceState";
+export * from "./ReadState";
+export * from "./RateLimit";

util/package.json

@@ -45,6 +45,9 @@
 		"mongoose": "^5.13.7",
 		"mongoose-autopopulate": "^0.12.3",
 		"node-fetch": "^2.6.1",
+		"reflect-metadata": "^0.1.13",
+		"sqlite3": "^5.0.2",
+		"typeorm": "^0.2.37",
 		"typescript": "^4.1.3"
 	}
 }

util/src/index.ts

@@ -1,10 +1,10 @@
-export * from "./util/checkToken";
+import "reflect-metadata";
 
-export * as Constants from "./util/Constants";
+// export * as Constants from "../util/Constants";
 export * from "./models/index";
-export * from "./util/index";
+// export * from "../util/index";
 
-import Config from "./util/Config";
+// import Config from "../util/Config";
-import db, { MongooseCache, toObject } from "./util/Database";
+// import db, { MongooseCache, toObject } from "./util/Database";
 
-export { Config, db, MongooseCache, toObject };
+// export { Config };

util/src/models/Activity.ts

@@ -1,7 +1,5 @@
-import { User } from "..";
+import { User } from "./User";
 import { ClientStatus, Status } from "./Status";
-import { Schema, model, Types, Document } from "mongoose";
-import toBigInt from "../util/toBigInt";
 
 export interface Presence {
 	user: User;
@@ -47,82 +45,6 @@ export interface Activity {
 	flags?: bigint;
 }
 
-export const ActivitySchema = {
-	name: { type: String, required: true },
-	type: { type: Number, required: true },
-	url: String,
-	created_at: Date,
-	timestamps: [
-		{
-			start: Number,
-			end: Number,
-		},
-	],
-	application_id: String,
-	details: String,
-	state: String,
-	emoji: {
-		name: String,
-		id: String,
-		amimated: Boolean,
-	},
-	party: {
-		id: String,
-		size: [Number, Number],
-	},
-	assets: {
-		large_image: String,
-		large_text: String,
-		small_image: String,
-		small_text: String,
-	},
-	secrets: {
-		join: String,
-		spectate: String,
-		match: String,
-	},
-	instance: Boolean,
-	flags: { type: String, get: toBigInt },
-};
-
-export const ActivityBodySchema = {
-	name: String,
-	type: Number,
-	$url: String,
-	$created_at: Date,
-	$timestamps: [
-		{
-			$start: Number,
-			$end: Number,
-		},
-	],
-	$application_id: String,
-	$details: String,
-	$state: String,
-	$emoji: {
-		$name: String,
-		$id: String,
-		$amimated: Boolean,
-	},
-	$party: {
-		$id: String,
-		$size: [Number, Number],
-	},
-	$assets: {
-		$large_image: String,
-		$large_text: String,
-		$small_image: String,
-		$small_text: String,
-	},
-	$secrets: {
-		$join: String,
-		$spectate: String,
-		$match: String,
-	},
-	$instance: Boolean,
-	$flags: BigInt,
-};
-
 export enum ActivityType {
 	GAME = 0,
 	STREAMING = 1,

util/src/models/BaseClass.ts

@@ -0,0 +1,28 @@
+import "reflect-metadata";
+import { BaseEntity, Column } from "typeorm";
+
+export class BaseClass extends BaseEntity {
+	@Column()
+	id?: string;
+
+	constructor(props?: any) {
+		super();
+		BaseClass.assign(props, this, "body.");
+	}
+
+	private static assign(props: any, object: any, path?: string): any {
+		const expectedType = Reflect.getMetadata("design:type", object, props);
+		console.log(expectedType, object, props, path, typeof object);
+
+		if (typeof object !== typeof props) throw new Error(`Property at ${path} must be`);
+		if (typeof object === "object")
+			return Object.keys(object).map((key) => BaseClass.assign(props[key], object[key], `${path}.${key}`));
+	}
+}
+
+// @ts-ignore
+global.BaseClass = BaseClass;
+
+var test = new BaseClass({});
+
+setTimeout(() => {}, 10000 * 1000);

util/src/models/Status.ts

@@ -5,9 +5,3 @@ export interface ClientStatus {
 	mobile?: string; // e.g. iOS/Android
 	web?: string; // e.g. browser, bot account
 }
-
-export const ClientStatus = {
-	desktop: String,
-	mobile: String,
-	web: String,
-};

util/src/models/User.ts

@@ -1,8 +1,7 @@
-import { Activity, ActivitySchema } from "./Activity";
+import { Column, PrimaryColumn, PrimaryGeneratedColumn } from "typeorm";
+import { Activity } from "./Activity";
+import { BaseClass } from "./BaseClass";
 import { ClientStatus, Status } from "./Status";
-import { Schema, Types, Document } from "mongoose";
-import db from "../util/Database";
-import toBigInt from "../util/toBigInt";
 
 export const PublicUserProjection = {
 	username: true,
@@ -16,53 +15,109 @@ export const PublicUserProjection = {
 	bot: true,
 };
 
-export interface User {
+export class User extends BaseClass {
+	@PrimaryGeneratedColumn()
 	id: string;
-	username: string; // username max length 32, min 2
+
+	@Column()
+	username: string; // username max length 32, min 2 (should be configurable)
+
+	@Column()
 	discriminator: string; // #0001 4 digit long string from #0001 - #9999
+
+	@Column()
 	avatar: string | null; // hash of the user avatar
+
+	@Column()
 	accent_color: number | null; // banner color of user
-	banner: string | null;
+
+	@Column()
+	banner: string | null; // hash of the user banner
+
+	@Column()
 	phone: string | null; // phone number of the user
+
+	@Column()
 	desktop: boolean; // if the user has desktop app installed
+
+	@Column()
 	mobile: boolean; // if the user has mobile app installed
+
+	@Column()
 	premium: boolean; // if user bought nitro
+
+	@Column()
 	premium_type: number; // nitro level
+
+	@Column()
 	bot: boolean; // if user is bot
-	bio: string; // short description of the user (max 190 chars)
+
-	system: boolean; // shouldn't be used, the api sents this field type true, if the genetaed message comes from a system generated author
+	@Column()
+	bio: string; // short description of the user (max 190 chars -> should be configurable)
+
+	@Column()
+	system: boolean; // shouldn't be used, the api sents this field type true, if the generated message comes from a system generated author
+
+	@Column()
 	nsfw_allowed: boolean; // if the user is older than 18 (resp. Config)
+
+	@Column()
 	mfa_enabled: boolean; // if multi factor authentication is enabled
+
+	@Column()
 	created_at: Date; // registration date
+
+	@Column()
 	verified: boolean; // if the user is offically verified
+
+	@Column()
 	disabled: boolean; // if the account is disabled
+
+	@Column()
 	deleted: boolean; // if the user was deleted
+
+	@Column()
 	email: string | null; // email of the user
+
+	@Column()
 	flags: bigint; // UserFlags
+
+	@Column()
 	public_flags: bigint;
-	user_settings: UserSettings;
+
+	@Column("simple-array") // string in simple-array must not contain commas
 	guilds: string[]; // array of guild ids the user is part of
+
+	@Column("simple-json")
+	user_settings: UserSettings;
+
+	@Column("simple-json")
 	user_data: UserData;
+
+	@Column("simple-json")
 	presence: {
 		status: Status;
 		activities: Activity[];
 		client_status: ClientStatus;
 	};
+
+	@Column("simple-json")
+	relationships: Relationship[];
+
+	@Column("simple-json")
+	connected_accounts: ConnectedAccount[];
 }
 
-// Private user data:
+// @ts-ignore
+global.User = User;
+
+// Private user data that should never get sent to the client
 export interface UserData {
 	valid_tokens_since: Date; // all tokens with a previous issue date are invalid
-	relationships: Relationship[];
-	connected_accounts: ConnectedAccount[];
 	hash: string; // hash of the password, salt is saved in password (bcrypt)
 	fingerprints: string[]; // array of fingerprints -> used to prevent multiple accounts
 }
 
-export interface UserDocument extends User, Document {
-	id: string;
-}
-
 export interface PublicUser {
 	id: string;
 	discriminator: string;
@@ -143,110 +198,3 @@ export interface UserSettings {
 	theme: "dark" | "white"; // dark
 	timezone_offset: number; // e.g -60
 }
-
-export const UserSchema = new Schema({
-	id: String,
-	username: String,
-	discriminator: String,
-	avatar: String,
-	accent_color: Number,
-	banner: String,
-	phone: String,
-	desktop: Boolean,
-	mobile: Boolean,
-	premium: Boolean,
-	premium_type: Number,
-	bot: Boolean,
-	bio: String,
-	system: Boolean,
-	nsfw_allowed: Boolean,
-	mfa_enabled: Boolean,
-	created_at: Date,
-	verified: Boolean,
-	disabled: Boolean,
-	deleted: Boolean,
-	email: String,
-	flags: { type: String, get: toBigInt }, // TODO: automatically convert Types.Long to BitField of UserFlags
-	public_flags: { type: String, get: toBigInt },
-	guilds: [String], // array of guild ids the user is part of
-	user_data: {
-		fingerprints: [String],
-		hash: String, // hash of the password, salt is saved in password (bcrypt)
-		valid_tokens_since: Date, // all tokens with a previous issue date are invalid
-		relationships: [
-			{
-				id: { type: String, required: true },
-				nickname: String,
-				type: { type: Number },
-			},
-		],
-		connected_accounts: [
-			{
-				access_token: String,
-				friend_sync: Boolean,
-				id: String,
-				name: String,
-				revoked: Boolean,
-				show_activity: Boolean,
-				type: { type: String },
-				verifie: Boolean,
-				visibility: Number,
-			},
-		],
-	},
-	user_settings: {
-		afk_timeout: Number,
-		allow_accessibility_detection: Boolean,
-		animate_emoji: Boolean,
-		animate_stickers: Number,
-		contact_sync_enabled: Boolean,
-		convert_emoticons: Boolean,
-		custom_status: {
-			emoji_id: String,
-			emoji_name: String,
-			expires_at: Number,
-			text: String,
-		},
-		default_guilds_restricted: Boolean,
-		detect_platform_accounts: Boolean,
-		developer_mode: Boolean,
-		disable_games_tab: Boolean,
-		enable_tts_command: Boolean,
-		explicit_content_filter: Number,
-		friend_source_flags: { all: Boolean },
-		gateway_connected: Boolean,
-		gif_auto_play: Boolean,
-		// every top guild is displayed as a "folder"
-		guild_folders: [
-			{
-				color: Number,
-				guild_ids: [String],
-				id: Number,
-				name: String,
-			},
-		],
-		guild_positions: [String], // guild ids ordered by position
-		inline_attachment_media: Boolean,
-		inline_embed_media: Boolean,
-		locale: String, // en_US
-		message_display_compact: Boolean,
-		native_phone_integration_enabled: Boolean,
-		render_embeds: Boolean,
-		render_reactions: Boolean,
-		restricted_guilds: [String],
-		show_current_game: Boolean,
-		status: String,
-		stream_notifications_enabled: Boolean,
-		theme: String, // dark
-		timezone_offset: Number, // e.g -60,
-	},
-
-	presence: {
-		status: String,
-		activities: [ActivitySchema],
-		client_status: ClientStatus,
-	},
-});
-
-// @ts-ignore
-export const UserModel = db.model<UserDocument>("User", UserSchema, "users");

util/src/models/index.ts

@@ -1,93 +1,4 @@
-// @ts-nocheck
-import mongoose, { Schema, Document } from "mongoose";
-import mongooseAutoPopulate from "mongoose-autopopulate";
-
-type UpdateWithAggregationPipeline = UpdateAggregationStage[];
-type UpdateAggregationStage =
-	| { $addFields: any }
-	| { $set: any }
-	| { $project: any }
-	| { $unset: any }
-	| { $replaceRoot: any }
-	| { $replaceWith: any };
-type EnforceDocument<T, TMethods> = T extends Document ? T : T & Document & TMethods;
-
-declare module "mongoose" {
-	interface SchemaOptions {
-		removeResponse?: string[];
-	}
-	interface Model<T, TQueryHelpers = {}, TMethods = {}> {
-		// removed null -> always return document -> throw error if it doesn't exist
-		findOne(
-			filter?: FilterQuery<T>,
-			projection?: any | null,
-			options?: QueryOptions | null,
-			callback?: (err: CallbackError, doc: EnforceDocument<T, TMethods>) => void
-		): QueryWithHelpers<EnforceDocument<T, TMethods>, EnforceDocument<T, TMethods>, TQueryHelpers>;
-		findOneAndUpdate(
-			filter?: FilterQuery<T>,
-			update?: UpdateQuery<T> | UpdateWithAggregationPipeline,
-			options?: QueryOptions | null,
-			callback?: (err: any, doc: EnforceDocument<T, TMethods> | null, res: any) => void
-		): QueryWithHelpers<EnforceDocument<T, TMethods>, EnforceDocument<T, TMethods>, TQueryHelpers>;
-	}
-}
-
-var HTTPError: any;
-
-try {
-	HTTPError = require("lambert-server").HTTPError;
-} catch (e) {
-	HTTPError = Error;
-}
-
-mongoose.plugin(mongooseAutoPopulate);
-
-mongoose.plugin((schema: Schema, opts: any) => {
-	schema.set("toObject", {
-		virtuals: true,
-		versionKey: false,
-		transform(doc: any, ret: any) {
-			delete ret._id;
-			delete ret.__v;
-			const props = schema.get("removeResponse") || [];
-			props.forEach((prop: string) => {
-				delete ret[prop];
-			});
-		},
-	});
-	schema.post("findOne", function (doc, next) {
-		try {
-			// @ts-ignore
-			const isExistsQuery = JSON.stringify(this._userProvidedFields) === JSON.stringify({ _id: 1 });
-			if (!doc && !isExistsQuery) {
-				// @ts-ignore
-				return next(new HTTPError(`${this?.mongooseCollection?.name}.${this?._conditions?.id} not found`, 400));
-			}
-			// @ts-ignore
-			return next();
-		} catch (error) {
-			// @ts-ignore
-			next();
-		}
-	});
-});
-
 export * from "./Activity";
-export * from "./Application";
+export * from "./BaseClass";
-export * from "./Ban";
-export * from "./Channel";
-export * from "./Emoji";
-export * from "./Event";
-export * from "./Template";
-export * from "./Guild";
-export * from "./Invite";
-export * from "./Interaction";
-export * from "./Member";
-export * from "./Message";
 export * from "./Status";
-export * from "./Role";
 export * from "./User";
-export * from "./VoiceState";
-export * from "./ReadState";
-export * from "./RateLimit";

util/src/util/Database.ts

@@ -1,159 +0,0 @@
-// @ts-nocheck
-import "./MongoBigInt";
-import mongoose, { Collection, Connection, LeanDocument } from "mongoose";
-import { ChangeStream, ChangeEvent, Long } from "mongodb";
-import EventEmitter from "events";
-const uri = process.env.MONGO_URL || "mongodb://localhost:27017/fosscord?readPreference=secondaryPreferred";
-import { URL } from "url";
-
-const url = new URL(uri.replace("mongodb://", "http://"));
-
-const connection = mongoose.createConnection(uri, {
-	autoIndex: true,
-	useNewUrlParser: true,
-	useUnifiedTopology: true,
-	useFindAndModify: true,
-});
-
-// this will return the new updated document for findOneAndUpdate
-mongoose.set("returnOriginal", false); // https://mongoosejs.com/docs/api/model.html#model_Model.findOneAndUpdate
-
-console.log(`[Database] connect: mongodb://${url.username}@${url.host}${url.pathname}${url.search}`);
-connection.once("open", () => {
-	console.log("[Database] connected");
-});
-
-export default <Connection>connection;
-
-function transform<T>(document: T) {
-	// @ts-ignore
-	if (!document || !document.toObject) {
-		try {
-			// @ts-ignore
-			delete document._id;
-			// @ts-ignore
-			delete document.__v;
-		} catch (error) {}
-		return document;
-	}
-	// @ts-ignore
-	return document.toObject({ virtuals: true });
-}
-
-export function toObject<T>(document: T): LeanDocument<T> {
-	// @ts-ignore
-	return Array.isArray(document) ? document.map((x) => transform<T>(x)) : transform(document);
-}
-
-export interface MongooseCache {
-	on(event: "delete", listener: (id: string) => void): this;
-	on(event: "change", listener: (data: any) => void): this;
-	on(event: "insert", listener: (data: any) => void): this;
-	on(event: "close", listener: () => void): this;
-}
-
-export class MongooseCache extends EventEmitter {
-	public stream: ChangeStream;
-	public data: any;
-	public initalizing?: Promise<void>;
-
-	constructor(
-		public collection: Collection,
-		public pipeline: Array<Record<string, unknown>>,
-		public opts: {
-			onlyEvents: boolean;
-			array?: boolean;
-		}
-	) {
-		super();
-		if (this.opts.array == null) this.opts.array = true;
-	}
-
-	init = () => {
-		if (this.initalizing) return this.initalizing;
-		this.initalizing = new Promise(async (resolve, reject) => {
-			// @ts-ignore
-			this.stream = this.collection.watch(this.pipeline, { fullDocument: "updateLookup" });
-
-			this.stream.on("change", this.change);
-			this.stream.on("close", this.destroy);
-			this.stream.on("error", console.error);
-
-			if (!this.opts.onlyEvents) {
-				const arr = await this.collection.aggregate(this.pipeline).toArray();
-				if (this.opts.array) this.data = arr || [];
-				else this.data = arr?.[0];
-			}
-			resolve();
-		});
-		return this.initalizing;
-	};
-
-	changeStream = (pipeline: any) => {
-		this.pipeline = pipeline;
-		this.destroy();
-		this.init();
-	};
-
-	convertResult = (obj: any) => {
-		if (obj instanceof Long) return BigInt(obj.toString());
-		if (typeof obj === "object") {
-			Object.keys(obj).forEach((key) => {
-				obj[key] = this.convertResult(obj[key]);
-			});
-		}
-
-		return obj;
-	};
-
-	change = (doc: ChangeEvent) => {
-		try {
-			switch (doc.operationType) {
-				case "dropDatabase":
-					return this.destroy();
-				case "drop":
-					return this.destroy();
-				case "delete":
-					if (!this.opts.onlyEvents) {
-						if (this.opts.array) {
-							this.data = this.data.filter((x: any) => doc.documentKey?._id?.equals(x._id));
-						} else this.data = null;
-					}
-					return this.emit("delete", doc.documentKey._id.toHexString());
-				case "insert":
-					if (!this.opts.onlyEvents) {
-						if (this.opts.array) this.data.push(doc.fullDocument);
-						else this.data = doc.fullDocument;
-					}
-					return this.emit("insert", doc.fullDocument);
-				case "update":
-				case "replace":
-					if (!this.opts.onlyEvents) {
-						if (this.opts.array) {
-							const i = this.data.findIndex((x: any) => doc.fullDocument?._id?.equals(x._id));
-							if (i == -1) this.data.push(doc.fullDocument);
-							else this.data[i] = doc.fullDocument;
-						} else this.data = doc.fullDocument;
-					}
-
-					return this.emit("change", doc.fullDocument);
-				case "invalidate":
-					return this.destroy();
-				default:
-					return;
-			}
-		} catch (error) {
-			this.emit("error", error);
-		}
-	};
-
-	destroy = () => {
-		this.data = null;
-		this.stream?.off("change", this.change);
-		this.emit("close");
-
-		if (this.stream.isClosed()) return;
-
-		return this.stream.close();
-	};
-}

util/tsconfig.json

@@ -65,6 +65,8 @@
 
 		/* Advanced Options */
 		"skipLibCheck": true /* Skip type checking of declaration files. */,
-		"forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */
+		"forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */,
+		"emitDecoratorMetadata": true,
+		"experimentalDecorators": true
 	}
 }

util/util/Database.ts

@@ -0,0 +1,7 @@
+import "reflect-metadata";
+import { createConnection } from "typeorm";
+
+// UUID extension option is only supported with postgres
+// We want to generate all id's with Snowflakes that's why we have our own BaseEntity class
+
+createConnection({ type: "sqlite", database: "database.db", entities: [], synchronize: true, logging: true });

util/util/index.ts

@@ -9,3 +9,4 @@ export * from "./UserFlags";
 export * from "./toBigInt";
 export * from "./RabbitMQ";
 export * from "./Event";
+export * from "./checkToken";