still doesn't work lol

package.json

@@ -109,10 +109,13 @@
 		"@fosscord/api": "dist/api",
 		"@fosscord/cdn": "dist/cdn",
 		"@fosscord/gateway": "dist/gateway",
-		"@fosscord/util": "dist/util"
+		"@fosscord/util": "dist/util",
+		"@fosscord/webrtc": "dist/webrtc"
 	},
 	"optionalDependencies": {
 		"erlpack": "^0.1.4",
+		"medooze-media-server": "^0.129.9",
+		"semantic-sdp": "^3.25.1",
 		"sqlite3": "^5.1.4"
 	}
 }

src/bundle/Server.ts

@@ -23,6 +23,7 @@ import http from "http";
 import * as Api from "@fosscord/api";
 import * as Gateway from "@fosscord/gateway";
 import { CDNServer } from "@fosscord/cdn";
+import * as Webrtc from "@fosscord/webrtc";
 import express from "express";
 import { green, bold } from "picocolors";
 import { Config, initDatabase, Sentry } from "@fosscord/util";
@@ -36,12 +37,14 @@ server.on("request", app);
 const api = new Api.FosscordServer({ server, port, production, app });
 const cdn = new CDNServer({ server, port, production, app });
 const gateway = new Gateway.Server({ server, port, production });
+const webrtc = new Webrtc.Server({ port: 3004, production });
 
 process.on("SIGTERM", async () => {
 	console.log("Shutting down due to SIGTERM");
 	await gateway.stop();
 	await cdn.stop();
 	await api.stop();
+	await webrtc.stop();
 	server.close();
 	Sentry.close();
 });
@@ -52,7 +55,12 @@ async function main() {
 	await Sentry.init(app);
 
 	server.listen(port);
-	await Promise.all([api.start(), cdn.start(), gateway.start()]);
+	await Promise.all([
+		api.start(),
+		cdn.start(),
+		gateway.start(),
+		webrtc.start(),
+	]);
 
 	Sentry.errorHandler(app);
 

src/gateway/util/Constants.ts

@@ -16,7 +16,7 @@
 	along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
 
-// import { VoiceOPCodes } from "@fosscord/webrtc";
+import { VoiceOPCodes } from "@fosscord/webrtc";
 
 export enum OPCODES {
 	Dispatch = 0,
@@ -63,7 +63,7 @@ export enum CLOSECODES {
 }
 
 export interface Payload {
-	op: OPCODES /*  | VoiceOPCodes */;
+	op: OPCODES | VoiceOPCodes;
 	// eslint-disable-next-line @typescript-eslint/no-explicit-any
 	d?: any;
 	s?: number;

src/gateway/util/WebSocket.ts

@@ -19,7 +19,7 @@
 import { Intents, ListenEventOpts, Permissions } from "@fosscord/util";
 import WS from "ws";
 import { Deflate, Inflate } from "fast-zlib";
-// import { Client } from "@fosscord/webrtc";
+import { Client } from "@fosscord/webrtc";
 
 export interface WebSocket extends WS {
 	version: number;
@@ -40,5 +40,5 @@ export interface WebSocket extends WS {
 	events: Record<string, undefined | (() => unknown)>;
 	member_events: Record<string, () => unknown>;
 	listen_options: ListenEventOpts;
-	// client?: Client;
+	webrtcClient?: Client;
 }

src/webrtc/Server.ts

@@ -19,6 +19,7 @@
 import { closeDatabase, Config, initDatabase, initEvent } from "@fosscord/util";
 import dotenv from "dotenv";
 import http from "http";
+import MediaServer from "medooze-media-server";
 import ws from "ws";
 import { Connection } from "./events/Connection";
 dotenv.config();
@@ -77,6 +78,7 @@ export class Server {
 
 	async stop() {
 		closeDatabase();
+		MediaServer.terminate();
 		this.server.close();
 	}
 }

src/webrtc/events/Message.ts

@@ -30,14 +30,12 @@ const PayloadSchema = {
 
 export async function onMessage(this: WebSocket, buffer: Buffer) {
 	try {
-		var data: Payload = JSON.parse(buffer.toString());
+		const data: Payload = JSON.parse(buffer.toString());
 		if (data.op !== VoiceOPCodes.IDENTIFY && !this.user_id)
 			return this.close(CLOSECODES.Not_authenticated);
 
-		// @ts-ignore
 		const OPCodeHandler = OPCodeHandlers[data.op];
 		if (!OPCodeHandler) {
-			// @ts-ignore
 			console.error("[WebRTC] Unkown opcode " + VoiceOPCodes[data.op]);
 			// TODO: if all opcodes are implemented comment this out:
 			// this.close(CloseCodes.Unknown_opcode);
@@ -49,7 +47,6 @@ export async function onMessage(this: WebSocket, buffer: Buffer) {
 				data.op as VoiceOPCodes,
 			)
 		) {
-			// @ts-ignore
 			console.log("[WebRTC] Opcode " + VoiceOPCodes[data.op]);
 		}
 

src/webrtc/opcodes/BackendVersion.ts

@@ -16,10 +16,10 @@
 	along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
 
-import { Payload, Send, WebSocket } from "@fosscord/gateway";
+import { Send, WebSocket } from "@fosscord/gateway";
 import { VoiceOPCodes } from "../util";
 
-export async function onBackendVersion(this: WebSocket, data: Payload) {
+export async function onBackendVersion(this: WebSocket) {
 	await Send(this, {
 		op: VoiceOPCodes.VOICE_BACKEND_VERSION,
 		d: { voice: "0.8.43", rtc_worker: "0.3.26" },

src/webrtc/opcodes/Identify.ts

@@ -24,10 +24,11 @@ import {
 } from "@fosscord/util";
 import { endpoint, getClients, VoiceOPCodes, PublicIP } from "@fosscord/webrtc";
 import SemanticSDP from "semantic-sdp";
-const defaultSDP = require("./sdp.json");
+import defaultSDP from "./sdp.json";
 
 export async function onIdentify(this: WebSocket, data: Payload) {
 	clearTimeout(this.readyTimeout);
+	// eslint-disable-next-line @typescript-eslint/no-unused-vars
 	const { server_id, user_id, session_id, token, streams, video } =
 		validateSchema("VoiceIdentifySchema", data.d) as VoiceIdentifySchema;
 
@@ -47,7 +48,7 @@ export async function onIdentify(this: WebSocket, data: Payload) {
 		}),
 	);
 
-	this.client = {
+	this.webrtcClient = {
 		websocket: this,
 		out: {
 			tracks: new Map(),
@@ -61,20 +62,27 @@ export async function onIdentify(this: WebSocket, data: Payload) {
 		channel_id: voiceState.channel_id,
 	};
 
-	const clients = getClients(voiceState.channel_id)!;
+	const clients = getClients(voiceState.channel_id);
-	clients.add(this.client);
+	clients.add(this.webrtcClient);
 
 	this.on("close", () => {
-		clients.delete(this.client!);
+		if (this.webrtcClient) clients.delete(this.webrtcClient);
 	});
 
 	await Send(this, {
 		op: VoiceOPCodes.READY,
 		d: {
 			streams: [
-				// { type: "video", ssrc: this.ssrc + 1, rtx_ssrc: this.ssrc + 2, rid: "100", quality: 100, active: false }
+				{
+					type: "video",
+					ssrc: this.webrtcClient.in.video_ssrc + 1,
+					rtx_ssrc: this.webrtcClient.in.video_ssrc + 2,
+					rid: "100",
+					quality: 100,
+					active: false,
+				},
 			],
-			ssrc: -1,
+			ssrc: this.webrtcClient.in.audio_ssrc,
 			port: endpoint.getLocalPort(),
 			modes: [
 				"aead_aes256_gcm_rtpsize",

src/webrtc/opcodes/SelectProtocol.ts

@@ -19,24 +19,28 @@
 import { Payload, Send, WebSocket } from "@fosscord/gateway";
 import { SelectProtocolSchema, validateSchema } from "@fosscord/util";
 import { endpoint, PublicIP, VoiceOPCodes } from "@fosscord/webrtc";
-import SemanticSDP, { MediaInfo, SDPInfo } from "semantic-sdp";
+import SemanticSDP, { MediaInfo } from "semantic-sdp";
+import DefaultSDP from "./sdp.json";
 
 export async function onSelectProtocol(this: WebSocket, payload: Payload) {
-	if (!this.client) return;
+	if (!this.webrtcClient) return;
 
 	const data = validateSchema(
 		"SelectProtocolSchema",
 		payload.d,
 	) as SelectProtocolSchema;
 
-	const offer = SemanticSDP.SDPInfo.parse("m=audio\n" + data.sdp!);
+	const offer = SemanticSDP.SDPInfo.parse("m=audio\n" + data.sdp);
-	this.client.sdp!.setICE(offer.getICE());
+	// eslint-disable-next-line @typescript-eslint/ban-ts-comment
-	this.client.sdp!.setDTLS(offer.getDTLS());
+	//@ts-ignore
+	offer.getMedias()[0].type = "audio"; // this is bad, but answer.toString() fails otherwise
+	this.webrtcClient.sdp.setICE(offer.getICE());
+	this.webrtcClient.sdp.setDTLS(offer.getDTLS());
 
-	const transport = endpoint.createTransport(this.client.sdp!);
+	const transport = endpoint.createTransport(this.webrtcClient.sdp);
-	this.client.transport = transport;
+	this.webrtcClient.transport = transport;
-	transport.setRemoteProperties(this.client.sdp!);
+	transport.setRemoteProperties(this.webrtcClient.sdp);
-	transport.setLocalProperties(this.client.sdp!);
+	transport.setLocalProperties(this.webrtcClient.sdp);
 
 	const dtls = transport.getLocalDTLSInfo();
 	const ice = transport.getLocalICEInfo();
@@ -45,21 +49,66 @@ export async function onSelectProtocol(this: WebSocket, payload: Payload) {
 	const candidates = transport.getLocalCandidates();
 	const candidate = candidates[0];
 
-	const answer =
+	// discord answer
-		`m=audio ${port} ICE/SDP` +
+	/*
-		`a=fingerprint:${fingerprint}` +
+		m=audio 50026 ICE/SDP\n
-		`c=IN IP4 ${PublicIP}` +
+		a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87\n
-		`a=rtcp:${port}` +
+		c=IN IP4 66.22.206.174\n
-		`a=ice-ufrag:${ice.getUfrag()}` +
+		a=rtcp:50026\n
-		`a=ice-pwd:${ice.getPwd()}` +
+		a=ice-ufrag:XxnE\n
-		`a=fingerprint:${fingerprint}` +
+		a=ice-pwd:GLQatPT3Q9dCZVVgVf3J1F\n
-		`a=candidate:1 1 ${candidate.getTransport()} ${candidate.getFoundation()} ${candidate.getAddress()} ${candidate.getPort()} typ host`;
+		a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87\n
+		a=candidate:1 1 UDP 4261412862 66.22.206.174 50026 typ host\n
+	*/
+
+	const answer = offer.answer({
+		dtls: dtls,
+		ice: ice,
+		candidates: endpoint.getLocalCandidates(),
+		capabilities: {
+			audio: {
+				codecs: ["opus"],
+				rtx: true,
+				rtcpfbs: [{ id: "transport-cc" }],
+				extensions: [
+					"urn:ietf:params:rtp-hdrext:ssrc-audio-level",
+					"http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time",
+					"http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01",
+					"urn:ietf:params:rtp-hdrext:sdes:mid",
+				],
+			},
+		},
+	});
+
+	// the Video handler creates streams but we need streams now so idk
+	for (const offered of offer.getStreams().values()) {
+		const incomingStream = transport.createIncomingStream(offered);
+		const outgoingStream = transport.createOutgoingStream({
+			audio: true,
+		});
+		outgoingStream.attachTo(incomingStream);
+		this.webrtcClient.in.stream = incomingStream;
+		this.webrtcClient.out.stream = outgoingStream;
+
+		const info = outgoingStream.getStreamInfo();
+		answer.addStream(info);
+	}
+
+	// const answer =
+	// 	`m=audio ${port} ICE/SDP\n` +
+	// 	`a=fingerprint:${fingerprint}\n` +
+	// 	`c=IN IP4 ${PublicIP}\n` +
+	// 	`a=rtcp:${port}\n` +
+	// 	`a=ice-ufrag:${ice.getUfrag()}\n` +
+	// 	`a=ice-pwd:${ice.getPwd()}\n` +
+	// 	`a=fingerprint:${fingerprint}\n` +
+	// 	`a=candidate:1 1 ${candidate.getTransport()} ${candidate.getFoundation()} ${candidate.getAddress()} ${candidate.getPort()} typ host\n`;
 
 	await Send(this, {
 		op: VoiceOPCodes.SELECT_PROTOCOL_ACK,
 		d: {
 			video_codec: "H264",
-			sdp: answer,
+			sdp: answer.toString(),
 			media_session_id: this.session_id,
 			audio_codec: "opus",
 		},

src/webrtc/opcodes/Speaking.ts

@@ -22,11 +22,12 @@ import { getClients, VoiceOPCodes } from "../util";
 // {"speaking":1,"delay":5,"ssrc":2805246727}
 
 export async function onSpeaking(this: WebSocket, data: Payload) {
-	if (!this.client) return;
+	if (!this.webrtcClient) return;
 
-	getClients(this.client.channel_id).forEach((client) => {
+	getClients(this.webrtcClient.channel_id).forEach((client) => {
-		if (client === this.client) return;
+		if (client === this.webrtcClient) return;
-		const ssrc = this.client!.out.tracks.get(client.websocket.user_id);
+		if (!this.webrtcClient) return;
+		const ssrc = this.webrtcClient.out.tracks.get(client.websocket.user_id);
 
 		Send(client.websocket, {
 			op: VoiceOPCodes.SPEAKING,

src/webrtc/opcodes/Video.ts

@@ -19,67 +19,78 @@
 import { Payload, Send, WebSocket } from "@fosscord/gateway";
 import { validateSchema, VoiceVideoSchema } from "@fosscord/util";
 import { channels, getClients, VoiceOPCodes } from "@fosscord/webrtc";
-import { IncomingStreamTrack, SSRCs } from "medooze-media-server";
+import {
+	IncomingStream,
+	IncomingStreamTrack,
+	SSRCs,
+	Transport,
+} from "medooze-media-server";
 import SemanticSDP from "semantic-sdp";
 
+function createStream(
+	this: WebSocket,
+	transport: Transport,
+	channel_id: string,
+) {
+	if (!this.webrtcClient) return;
+	if (!this.webrtcClient.transport) return;
+
+	const id = "stream" + this.user_id;
+
+	const stream = this.webrtcClient.transport.createIncomingStream(
+		SemanticSDP.StreamInfo.expand({
+			id,
+			tracks: [],
+		}),
+	);
+	this.webrtcClient.in.stream = stream;
+
+	const interval = setInterval(() => {
+		for (const track of stream.getTracks()) {
+			for (const layer of Object.values(track.getStats())) {
+				console.log(track.getId(), layer.total);
+			}
+		}
+	}, 5000);
+
+	stream.on("stopped", () => {
+		console.log("stream stopped");
+		clearInterval(interval);
+	});
+	this.on("close", () => {
+		transport.stop();
+	});
+	const out = transport.createOutgoingStream(
+		SemanticSDP.StreamInfo.expand({
+			id: "out" + this.user_id,
+			tracks: [],
+		}),
+	);
+	this.webrtcClient.out.stream = out;
+
+	const clients = channels.get(channel_id);
+	if (!clients) return;
+
+	clients.forEach((client) => {
+		if (client.websocket.user_id === this.user_id) return;
+		if (!client.in.stream) return;
+
+		client.in.stream?.getTracks().forEach((track) => {
+			attachTrack.call(this, track, client.websocket.user_id);
+		});
+	});
+}
+
 export async function onVideo(this: WebSocket, payload: Payload) {
-	if (!this.client) return;
+	if (!this.webrtcClient) return;
-	const { transport, channel_id } = this.client;
+	const { transport, channel_id } = this.webrtcClient;
 	if (!transport) return;
 	const d = validateSchema("VoiceVideoSchema", payload.d) as VoiceVideoSchema;
 
 	await Send(this, { op: VoiceOPCodes.MEDIA_SINK_WANTS, d: { any: 100 } });
 
-	const id = "stream" + this.user_id;
+	if (!this.webrtcClient.in.stream)
-
+		createStream.call(this, transport, channel_id);
-	var stream = this.client.in.stream!;
-	if (!stream) {
-		stream = this.client.transport!.createIncomingStream(
-			// @ts-ignore
-			SemanticSDP.StreamInfo.expand({
-				id,
-				// @ts-ignore
-				tracks: [],
-			}),
-		);
-		this.client.in.stream = stream;
-
-		const interval = setInterval(() => {
-			for (const track of stream.getTracks()) {
-				for (const layer of Object.values(track.getStats())) {
-					console.log(track.getId(), layer.total);
-				}
-			}
-		}, 5000);
-
-		stream.on("stopped", () => {
-			console.log("stream stopped");
-			clearInterval(interval);
-		});
-		this.on("close", () => {
-			transport!.stop();
-		});
-		const out = transport.createOutgoingStream(
-			// @ts-ignore
-			SemanticSDP.StreamInfo.expand({
-				id: "out" + this.user_id,
-				// @ts-ignore
-				tracks: [],
-			}),
-		);
-		this.client.out.stream = out;
-
-		const clients = channels.get(channel_id)!;
-
-		clients.forEach((client) => {
-			if (client.websocket.user_id === this.user_id) return;
-			if (!client.in.stream) return;
-
-			client.in.stream?.getTracks().forEach((track) => {
-				attachTrack.call(this, track, client.websocket.user_id);
-			});
-		});
-	}
 
 	if (d.audio_ssrc) {
 		handleSSRC.call(this, "audio", {
@@ -100,23 +111,32 @@ function attachTrack(
 	track: IncomingStreamTrack,
 	user_id: string,
 ) {
-	if (!this.client) return;
+	if (
-	const outTrack = this.client.transport!.createOutgoingStreamTrack(
+		!this.webrtcClient ||
+		!this.webrtcClient.transport ||
+		!this.webrtcClient.out.stream
+	)
+		return;
+
+	const outTrack = this.webrtcClient.transport.createOutgoingStreamTrack(
 		track.getMedia(),
 	);
 	outTrack.attachTo(track);
-	this.client.out.stream!.addTrack(outTrack);
+
-	var ssrcs = this.client.out.tracks.get(user_id)!;
+	this.webrtcClient.out.stream.addTrack(outTrack);
+	let ssrcs = this.webrtcClient.out.tracks.get(user_id);
 	if (!ssrcs)
-		ssrcs = this.client.out.tracks
+		ssrcs = this.webrtcClient.out.tracks
 			.set(user_id, { audio_ssrc: 0, rtx_ssrc: 0, video_ssrc: 0 })
-			.get(user_id)!;
+			.get(user_id);
+
+	if (!ssrcs) return; // hmm
 
 	if (track.getMedia() === "audio") {
-		ssrcs.audio_ssrc = outTrack.getSSRCs().media!;
+		ssrcs.audio_ssrc = outTrack.getSSRCs().media || 0;
 	} else if (track.getMedia() === "video") {
-		ssrcs.video_ssrc = outTrack.getSSRCs().media!;
+		ssrcs.video_ssrc = outTrack.getSSRCs().media || 0;
-		ssrcs.rtx_ssrc = outTrack.getSSRCs().rtx!;
+		ssrcs.rtx_ssrc = outTrack.getSSRCs().rtx || 0;
 	}
 
 	Send(this, {
@@ -129,18 +149,18 @@ function attachTrack(
 }
 
 function handleSSRC(this: WebSocket, type: "audio" | "video", ssrcs: SSRCs) {
-	if (!this.client) return;
+	if (!this.webrtcClient) return;
-	const stream = this.client.in.stream!;
+	const stream = this.webrtcClient.in.stream as IncomingStream;
-	const transport = this.client.transport!;
+	const transport = this.webrtcClient.transport as Transport;
 
 	const id = type + ssrcs.media;
-	var track = stream.getTrack(id);
+	let track = stream.getTrack(id);
 	if (!track) {
 		console.log("createIncomingStreamTrack", id);
 		track = transport.createIncomingStreamTrack(type, { id, ssrcs });
 		stream.addTrack(track);
 
-		const clients = getClients(this.client.channel_id)!;
+		const clients = getClients(this.webrtcClient.channel_id);
 		clients.forEach((client) => {
 			if (client.websocket.user_id === this.user_id) return;
 			if (!client.out.stream) return;

src/webrtc/opcodes/index.ts

@@ -25,7 +25,7 @@ import { onSelectProtocol } from "./SelectProtocol";
 import { onSpeaking } from "./Speaking";
 import { onVideo } from "./Video";
 
-export type OPCodeHandler = (this: WebSocket, data: Payload) => any;
+export type OPCodeHandler = (this: WebSocket, data: Payload) => unknown;
 
 export default {
 	[VoiceOPCodes.HEARTBEAT]: onHeartbeat,
@@ -34,4 +34,4 @@ export default {
 	[VoiceOPCodes.VIDEO]: onVideo,
 	[VoiceOPCodes.SPEAKING]: onSpeaking,
 	[VoiceOPCodes.SELECT_PROTOCOL]: onSelectProtocol,
-};
+} as { [key: number]: OPCodeHandler };

src/webrtc/util/MediaServer.ts

@@ -28,8 +28,8 @@ MediaServer.enableLog(true);
 export const PublicIP = process.env.PUBLIC_IP || "127.0.0.1";
 
 try {
-	const range = process.env.WEBRTC_PORT_RANGE || "4000";
+	const range = process.env.WEBRTC_PORT_RANGE || "4000-5000";
-	var ports = range.split("-");
+	const ports = range.split("-");
 	const min = Number(ports[0]);
 	const max = Number(ports[1]);
 
@@ -73,5 +73,5 @@ export interface Client {
 
 export function getClients(channel_id: string) {
 	if (!channels.has(channel_id)) channels.set(channel_id, new Set());
-	return channels.get(channel_id)!;
+	return channels.get(channel_id) as Set<Client>;
 }

tsconfig.json

@@ -1,5 +1,5 @@
 {
-	"exclude": ["./src/webrtc"],
+	// "exclude": ["./src/webrtc"],
 	"include": ["./src"],
 	"compilerOptions": {
 		/* Visit https://aka.ms/tsconfig to read more about this file */
@@ -37,7 +37,8 @@
 			"@fosscord/api*": ["./api"],
 			"@fosscord/gateway*": ["./gateway"],
 			"@fosscord/cdn*": ["./cdn"],
-			"@fosscord/util*": ["./util"]
+			"@fosscord/util*": ["./util"],
+			"@fosscord/webrtc*": ["./webrtc"]
 		} /* Specify a set of entries that re-map imports to additional lookup locations. */,
 		// "rootDirs": [],                                   /* Allow multiple folders to be treated as one when resolving modules. */
 		// "typeRoots": [],                                  /* Specify multiple folders that act like './node_modules/@types'. */