VOICE CONNECTS!!! Dtls stuck on "connecting" state + currently no way to edit/inspect packets received or use own packet format in mediasoup ( fork? )

2024-11-23 10:52:30 +01:00 · 2022-03-07 22:57:37 +11:00 · 2022-03-07 22:57:37 +11:00 · e731a369e5
commit e731a369e5
parent b529a37264
5 changed files with 165 additions and 173 deletions
--- a/webrtc/src/Server.ts
+++ b/webrtc/src/Server.ts
@ -6,7 +6,7 @@ import { setHeartbeat } from "./util";
 import * as mediasoup from "mediasoup";
 import { types as MediasoupTypes } from "mediasoup";

-import Net from "net";
+import udp from "dgram";

 var port = Number(process.env.PORT);
 if (isNaN(port)) port = 3004;
@ -16,6 +16,8 @@ export class Server {
 	public mediasoupWorkers: MediasoupTypes.Worker[] = [];
 	public mediasoupRouters: MediasoupTypes.Router[] = [];
 	public mediasoupTransports: MediasoupTypes.WebRtcTransport[] = [];
+	public mediasoupProducers: MediasoupTypes.Producer[] = [];
+	public mediasoupConsumers: MediasoupTypes.Consumer[] = [];

 	constructor() {
 		this.ws = new WebSocketServer({
@ -28,8 +30,6 @@ export class Server {
 			socket.on("message", async (message: string) => {
 				const payload: Payload = JSON.parse(message);

-				// console.log(payload);
-
 				if (OPCodeHandlers[payload.op])
 					try {
 						await OPCodeHandlers[payload.op].call(this, socket, payload);
@ -44,6 +44,7 @@ export class Server {
 				}
 			});
 		});
+
 	}

 	async listen(): Promise<void> {
@ -73,18 +74,26 @@ export class Server {
 				router.observer.on("newtransport", async (transport: MediasoupTypes.WebRtcTransport) => {
 					console.log("new transport created [id:%s]", transport.id);

-					transport.observer.on("sctpstatechange", (state) => {
-						console.log(state)
-					});
-
 					await transport.enableTraceEvent();

+					transport.on("connect", () => {
+						console.log("transport connect")
+					})
+
 					transport.observer.on("newproducer", (producer: MediasoupTypes.Producer) => {
 						console.log("new producer created [id:%s]", producer.id);
+
+						this.mediasoupProducers.push(producer);
 					});

 					transport.observer.on("newconsumer", (consumer: MediasoupTypes.Consumer) => {
 						console.log("new consumer created [id:%s]", consumer.id);
+
+						this.mediasoupConsumers.push(consumer);
+
+						consumer.on("rtp", (rtpPacket) => {
+							console.log(rtpPacket);
+						});
 					});

 					transport.observer.on("newdataproducer", (dataProducer) => {
--- a/webrtc/src/opcodes/Connect.ts
+++ b/webrtc/src/opcodes/Connect.ts
@ -2,8 +2,38 @@ import { WebSocket } from "@fosscord/gateway";
 import { Payload } from "./index";
 import { Server } from "../Server"

+/*
+Sent by client:
+
+{
+    "op": 12,
+    "d": {
+        "audio_ssrc": 0,
+        "video_ssrc": 0,
+        "rtx_ssrc": 0,
+        "streams": [
+            {
+                "type": "video",
+                "rid": "100",
+                "ssrc": 0,
+                "active": false,
+                "quality": 100,
+                "rtx_ssrc": 0,
+                "max_bitrate": 2500000,
+                "max_framerate": 20,
+                "max_resolution": {
+                    "type": "fixed",
+                    "width": 1280,
+                    "height": 720
+                }
+            }
+        ]
+    }
+}
+*/
+
 export async function onConnect(this: Server, socket: WebSocket, data: Payload) {
-	socket.send(JSON.stringify({
+	socket.send(JSON.stringify({	//what is op 15?
 		op: 15,
 		d: { any: 100 }
 	}))
--- a/webrtc/src/opcodes/Identify.ts
+++ b/webrtc/src/opcodes/Identify.ts
@ -34,71 +34,20 @@ export async function onIdentify(this: Server, socket: WebSocket, data: Identify
 		return socket.close(CLOSECODES.Invalid_intent);

 	var transport = this.mediasoupTransports[0] || await this.mediasoupRouters[0].createWebRtcTransport({
-		listenIps: [{ ip: "0.0.0.0", announcedIp: "127.0.0.1" }],
+		listenIps: [{ ip: "10.22.64.69" }],
 		enableUdp: true,
 		enableTcp: true,
 		preferUdp: true,
+		enableSctp: true,
 	});

-	/*
-		//discord proper sends:
-		{ 
-			"streams": [
-				{ "type": "video", "ssrc": 1311885, "rtx_ssrc": 1311886, "rid": "50", "quality": 50, "active": false },
-				{ "type": "video", "ssrc": 1311887, "rtx_ssrc": 1311888, "rid": "100", "quality": 100, "active": false }
-			],
-			"ssrc": 1311884,
-			"port": 50008,
-			"modes": [
-				"aead_aes256_gcm_rtpsize",
-				"aead_aes256_gcm",
-				"xsalsa20_poly1305_lite_rtpsize",
-				"xsalsa20_poly1305_lite",
-				"xsalsa20_poly1305_suffix",
-				"xsalsa20_poly1305"
-			],
-			"ip": "109.200.214.158",
-			"experiments": [
-				"bwe_conservative_link_estimate",
-				"bwe_remote_locus_client",
-				"fixed_keyframe_interval"
-			]
-		}
-	*/
-
-
-
-	/* 
-		{
-			"streams": [
-				{ "type": "video", "ssrc": 129861, "rtx_ssrc": 129862, "rid": "100", "quality": 100, "active": false }
-			],
-			"ssrc": 129860,
-			"port": 50003,
-			"modes": [
-				"aead_aes256_gcm_rtpsize",
-				"aead_aes256_gcm",
-				"xsalsa20_poly1305_lite_rtpsize",
-				"xsalsa20_poly1305_lite",
-				"xsalsa20_poly1305_suffix",
-				"xsalsa20_poly1305"
-			],
-			"ip": "109.200.213.251",
-			"experiments": [
-				"bwe_conservative_link_estimate",
-				"bwe_remote_locus_client",
-				"fixed_keyframe_interval"
-			];
-		};
-	*/
-
 	socket.send(JSON.stringify({
 		op: VoiceOPCodes.READY,
 		d: {
 			streams: [...data.d.streams.map(x => ({ ...x, rtx_ssrc: Math.floor(Math.random() * 10000), ssrc: Math.floor(Math.random() * 10000), active: false, }))],
 			ssrc: Math.floor(Math.random() * 10000),
 			ip: transport.iceCandidates[0].ip,
-			port: "50001",
+			port: transport.iceCandidates[0].port,
 			modes: [
 				"aead_aes256_gcm_rtpsize",
 				"aead_aes256_gcm",
@ -107,7 +56,11 @@ export async function onIdentify(this: Server, socket: WebSocket, data: Identify
 				"xsalsa20_poly1305_suffix",
 				"xsalsa20_poly1305"
 			],
-			experiments: [],
+			experiments: [
+				"bwe_conservative_link_estimate",
+				"bwe_remote_locus_client",
+				"fixed_keyframe_interval"
+			]
 		},
 	}));
 }
--- a/webrtc/src/opcodes/Resume.ts
+++ b/webrtc/src/opcodes/Resume.ts
@ -1,6 +1,24 @@
-import { WebSocket } from "@fosscord/gateway";
+import { CLOSECODES, WebSocket } from "@fosscord/gateway";
 import { Payload } from "./index";
 import { Server } from "../Server"
+import { Guild, Session, VoiceOPCodes } from "@fosscord/util";

 export async function onResume(this: Server, socket: WebSocket, data: Payload) {
+	const session = await Session.findOneOrFail(
+		{ session_id: data.d.session_id, },
+		{
+			where: { user_id: data.d.user_id },
+			relations: ["user"]
+		}
+	);
+	const user = session.user;
+	const guild = await Guild.findOneOrFail({ id: data.d.server_id }, { relations: ["members"] });
+
+	if (!guild.members.find(x => x.id === user.id))
+		return socket.close(CLOSECODES.Invalid_intent);
+
+	socket.send(JSON.stringify({
+		op: VoiceOPCodes.RESUMED,
+		d: null,
+	}))
 }
--- a/webrtc/src/opcodes/SelectProtocol.ts
+++ b/webrtc/src/opcodes/SelectProtocol.ts
@ -6,15 +6,18 @@ import * as mediasoup from "mediasoup";
 import { RtpCodecCapability } from "mediasoup/node/lib/RtpParameters";
 import * as sdpTransform from 'sdp-transform';

+
 /*
-	{
-	op: 1,
-	d: {
-		protocol: "webrtc",
-		data: "
+
+	Sent by client:
+{
+	"op": 1,
+	"d": {
+		"protocol": "webrtc",
+		"data": "
 			a=extmap-allow-mixed
-			a=ice-ufrag:ilWh
-			a=ice-pwd:Mx7TDnPKXDnTgYWC+qMaqspQ
+			a=ice-ufrag:vNxb
+			a=ice-pwd:tZvpbVPYEKcnW0gGRPq0OOnh
 			a=ice-options:trickle
 			a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level
 			a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time
@ -32,43 +35,63 @@ import * as sdpTransform from 'sdp-transform';
 			a=rtpmap:96 VP8/90000
 			a=rtpmap:97 rtx/90000
 		",
-		sdp: "same data as in d.data? also not documented by discord",
-		codecs: [
-		{
-			name: "opus",
-			type: "audio",
-			priority: 1000,
-			payload_type: 111,
-			rtx_payload_type: null,
-		},
-		{
-			name: "H264",
-			type: "video",
-			priority: 1000,
-			payload_type: 102,
-			rtx_payload_type: 121,
-		},
-		{
-			name: "VP8",
-			type: "video",
-			priority: 2000,
-			payload_type: 96,
-			rtx_payload_type: 97,
-		},
-		{
-			name: "VP9",
-			type: "video",
-			priority: 3000,
-			payload_type: 98,
-			rtx_payload_type: 99,
-		},
+		"codecs": [
+			{
+				"name": "opus",
+				"type": "audio",
+				"priority": 1000,
+				"payload_type": 111,
+				"rtx_payload_type": null
+			},
+			{
+				"name": "H264",
+				"type": "video",
+				"priority": 1000,
+				"payload_type": 102,
+				"rtx_payload_type": 121
+			},
+			{
+				"name": "VP8",
+				"type": "video",
+				"priority": 2000,
+				"payload_type": 96,
+				"rtx_payload_type": 97
+			},
+			{
+				"name": "VP9",
+				"type": "video",
+				"priority": 3000,
+				"payload_type": 98,
+				"rtx_payload_type": 99
+			}
 		],
-		rtc_connection_id: "b3c8628a-edb5-49ae-b860-ab0d2842b104",
-	},
+		"rtc_connection_id": "3faa0b80-b3e2-4bae-b291-273801fbb7ab"
 	}
+}
+
+Sent by server:
+
+{
+	"op": 4,
+	"d": {
+		"video_codec": "H264",
+		"sdp": "
+			m=audio 50001 ICE/SDP
+			a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87
+			c=IN IP4 109.200.214.158
+			a=rtcp:50001
+			a=ice-ufrag:CLzn
+			a=ice-pwd:qEmIcNwigd07mu46Ok0XCh
+			a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87
+			a=candidate:1 1 UDP 4261412862 109.200.214.158 50001 typ host
+		",
+		"media_session_id": "807955cb953e98c5b90704cf048e81ec",
+		"audio_codec": "opus"
+	}
+}
+
 */

-var test_hasMadeProducer = false;

 export async function onSelectProtocol(this: Server, socket: WebSocket, data: Payload) {
 	const rtpCapabilities = this.mediasoupRouters[0].rtpCapabilities;
@ -78,87 +101,46 @@ export async function onSelectProtocol(this: Server, socket: WebSocket, data: Pa

 	const res = sdpTransform.parse(data.d.sdp);

-	/*
-	 res.media.map(x => x.rtp).flat(1).map(x => ({
-				codec: x.codec,
-				payloadType: x.payload,
-				clockRate: x.rate as number,
-				mimeType: `audio/${x.codec}`,
+	const videoCodec = this.mediasoupRouters[0].rtpCapabilities.codecs!.find((x: any) => x.kind === "video");
+	const audioCodec = this.mediasoupRouters[0].rtpCapabilities.codecs!.find((x: any) => x.kind === "audio");
+
+	const producer = this.mediasoupProducers[0] || await transport.produce({
+		kind: "audio",
+		rtpParameters: {
+			mid: "audio",
+			codecs: [{
+				clockRate: audioCodec!.clockRate,
+				payloadType: audioCodec!.preferredPayloadType as number,
+				mimeType: audioCodec!.mimeType,
+				channels: audioCodec?.channels,
+			}],
+			headerExtensions: res.ext?.map(x => ({
+				id: x.value,
+				uri: x.uri,
 			})),
-	*/
+		},
+		paused: false,
+	});

-	const videoCodec = this.mediasoupRouters[0].rtpCapabilities.codecs!.find((x: any) => x.kind === "video")?.mimeType
-	const audioCodec = this.mediasoupRouters[0].rtpCapabilities.codecs!.find((x: any) => x.kind === "audio")
-
-	if (!test_hasMadeProducer) {
-		const producer = await transport.produce({
-			kind: "audio",
-			rtpParameters: {
-				mid: "audio",
-				codecs: [{
-					clockRate: audioCodec!.clockRate,
-					payloadType: audioCodec!.preferredPayloadType as number,
-					mimeType: audioCodec!.mimeType,
-					channels: audioCodec?.channels,
-				}],
-				headerExtensions: res.ext?.map(x => ({
-					id: x.value,
-					uri: x.uri,
-				})),
-			},
-			paused: false,
-		});
-
-		const consumer = await transport.consume({
-			producerId: producer.id,
-			paused: true,
-			rtpCapabilities,
-		});
-		
-		test_hasMadeProducer = true;
-	}
-
-	/* server sends sdp:
-
-	m=audio 50021 ICE/SDP		//same port as sent in READY
-	a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87
-	c=IN IP4 109.200.213.132	//same IP as sent in READY
-	a=rtcp:50021				//same port?
-	a=ice-ufrag:rTmX
-	a=ice-pwd:M+ncqWK6SEdHhirOjG2VFA
-	a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87
-	a=candidate:1 1 UDP 4261412862 109.200.213.132 50021 typ host	//same IP and PORT
-
-	*/
-
-
-	var test = {
-		"video_codec": "H264",
-		"sdp": `
-			m=audio 50011 ICE/SDP\n
-			a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87\n
-			c=IN IP4 109.200.214.156\n
-			a=rtcp:50011\n
-			a=ice-ufrag:d0aZ\n
-			a=ice-pwd:51ubWYu7GSkQRqlH/apTSZ\n
-			a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87\n
-			a=candidate:1 1 UDP 4261412862 109.200.214.156 50011 typ host\n`,
-		"media_session_id": "9e18c981687f2de5399edd5cb3f3babf",
-		"audio_codec": "opus"
-	};
+	console.log("can consume: " + this.mediasoupRouters[0].canConsume({ producerId: producer.id, rtpCapabilities: rtpCapabilities }));

+	const consumer = this.mediasoupConsumers[0] || await transport.consume({
+		producerId: producer.id,
+		paused: false,
+		rtpCapabilities,
+	});

 	socket.send(JSON.stringify({
 		op: VoiceOPCodes.SESSION_DESCRIPTION,
 		d: {
-			video_codec: videoCodec?.substring(6) || undefined,
-			// mode: "xsalsa20_poly1305",
+			video_codec: videoCodec?.mimeType?.substring(6) || undefined,
+			mode: "xsalsa20_poly1305_lite",
 			media_session_id: transport.id,
 			audio_codec: audioCodec?.mimeType.substring(6),
 			sdp: `m=audio ${transport.iceCandidates[0].port} ICE/SDP\n`
 				+ `a=fingerprint:sha-256 ${transport.dtlsParameters.fingerprints.find(x => x.algorithm === "sha-256")?.value}\n`
 				+ `c=IN IPV4 ${transport.iceCandidates[0].ip}\n`
-				+ `a=rtcp:${transport.iceCandidates[0].port}\n`
+				+ `a=rtcp: ${transport.iceCandidates[0].port}\n`
 				+ `a=ice-ufrag:${transport.iceParameters.usernameFragment}\n`
 				+ `a=ice-pwd:${transport.iceParameters.password}\n`
 				+ `a=fingerprint:sha-1 ${transport.dtlsParameters.fingerprints[0].value}\n`