diff --git a/webrtc/package-lock.json b/webrtc/package-lock.json index 43bb8cd8..afba7e76 100644 --- a/webrtc/package-lock.json +++ b/webrtc/package-lock.json @@ -12,11 +12,13 @@ "dotenv": "^12.0.4", "mediasoup": "^3.9.5", "node-turn": "^0.0.6", + "sdp-transform": "^2.14.1", "tsconfig-paths": "^3.12.0", "ws": "^7.4.6" }, "devDependencies": { "@types/node": "^15.6.1", + "@types/sdp-transform": "^2.4.5", "@types/ws": "^7.4.4", "ts-node": "^10.4.0", "typescript": "^4.3.2" @@ -78,6 +80,12 @@ "integrity": "sha512-7EIraBEyRHEe7CH+Fm1XvgqU6uwZN8Q7jppJGcqjROMT29qhAuuOxYB1uEY5UMYQKEmA5D+5tBnhdaPXSsLONA==", "dev": true }, + "node_modules/@types/sdp-transform": { + "version": "2.4.5", + "resolved": "https://registry.npmjs.org/@types/sdp-transform/-/sdp-transform-2.4.5.tgz", + "integrity": "sha512-GVO0gnmbyO3Oxm2HdPsYUNcyihZE3GyCY8ysMYHuQGfLhGZq89Nm4lSzULWTzZoyHtg+VO/IdrnxZHPnPSGnAg==", + "dev": true + }, "node_modules/@types/ws": { "version": "7.4.4", "resolved": "https://registry.npmjs.org/@types/ws/-/ws-7.4.4.tgz", @@ -392,6 +400,14 @@ "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.3.0.tgz", "integrity": "sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA==" }, + "node_modules/sdp-transform": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/sdp-transform/-/sdp-transform-2.14.1.tgz", + "integrity": "sha512-RjZyX3nVwJyCuTo5tGPx+PZWkDMCg7oOLpSlhjDdZfwUoNqG1mM8nyj31IGHyaPWXhjbP7cdK3qZ2bmkJ1GzRw==", + "bin": { + "sdp-verify": "checker.js" + } + }, "node_modules/sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", @@ -599,6 +615,12 @@ "integrity": "sha512-7EIraBEyRHEe7CH+Fm1XvgqU6uwZN8Q7jppJGcqjROMT29qhAuuOxYB1uEY5UMYQKEmA5D+5tBnhdaPXSsLONA==", "dev": true }, + "@types/sdp-transform": { + "version": "2.4.5", + "resolved": "https://registry.npmjs.org/@types/sdp-transform/-/sdp-transform-2.4.5.tgz", + "integrity": "sha512-GVO0gnmbyO3Oxm2HdPsYUNcyihZE3GyCY8ysMYHuQGfLhGZq89Nm4lSzULWTzZoyHtg+VO/IdrnxZHPnPSGnAg==", + "dev": true + }, "@types/ws": { "version": "7.4.4", "resolved": "https://registry.npmjs.org/@types/ws/-/ws-7.4.4.tgz", @@ -817,6 +839,11 @@ "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.3.0.tgz", "integrity": "sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA==" }, + "sdp-transform": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/sdp-transform/-/sdp-transform-2.14.1.tgz", + "integrity": "sha512-RjZyX3nVwJyCuTo5tGPx+PZWkDMCg7oOLpSlhjDdZfwUoNqG1mM8nyj31IGHyaPWXhjbP7cdK3qZ2bmkJ1GzRw==" + }, "sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", diff --git a/webrtc/package.json b/webrtc/package.json index d5a994a1..b9bac356 100644 --- a/webrtc/package.json +++ b/webrtc/package.json @@ -13,6 +13,7 @@ "license": "ISC", "devDependencies": { "@types/node": "^15.6.1", + "@types/sdp-transform": "^2.4.5", "@types/ws": "^7.4.4", "ts-node": "^10.4.0", "typescript": "^4.3.2" @@ -21,6 +22,7 @@ "dotenv": "^12.0.4", "mediasoup": "^3.9.5", "node-turn": "^0.0.6", + "sdp-transform": "^2.14.1", "tsconfig-paths": "^3.12.0", "ws": "^7.4.6" } diff --git a/webrtc/src/opcodes/Identify.ts b/webrtc/src/opcodes/Identify.ts index 6bbed04c..c31870c8 100644 --- a/webrtc/src/opcodes/Identify.ts +++ b/webrtc/src/opcodes/Identify.ts @@ -2,10 +2,6 @@ import { WebSocket } from "@fosscord/gateway"; import { Payload } from "./index"; import { VoiceOPCodes } from "@fosscord/util"; import { Server } from "../Server"; -import * as mediasoup from "mediasoup"; -import { RtpCodecCapability } from "mediasoup/node/lib/RtpParameters"; - -const test = "extmap-allow-mixed\na=ice-ufrag:ilWh\na=ice-pwd:Mx7TDnPKXDnTgYWC+qMaqspQ\na=ice-options:trickle\na=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level\na=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\na=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\na=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid\na=rtpmap:111 opus/48000/2\na=extmap:14 urn:ietf:params:rtp-hdrext:toffset\na=extmap:13 urn:3gpp:video-orientation\na=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay\na=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type\na=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing\na=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space\na=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id\na=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id\na=rtpmap:96 VP8/90000\na=rtpmap:97 rtx/90000"; export async function onIdentify(this: Server, socket: WebSocket, data: Payload) { var transport = await this.mediasoupRouters[0].createWebRtcTransport({ @@ -15,52 +11,31 @@ export async function onIdentify(this: Server, socket: WebSocket, data: Payload) preferUdp: true, }); - const rtpCapabilities = this.mediasoupRouters[0].rtpCapabilities; - const codecs = rtpCapabilities.codecs as RtpCodecCapability[]; - - var producer = await transport.produce( - { - kind: "audio", - rtpParameters: - { - mid: "1", - codecs: codecs.filter(x => x.kind === "audio").map((x: RtpCodecCapability) => { - return { - mimeType: x.mimeType, - kind: x.kind, - clockRate: x.clockRate, - channels: x.channels, - payloadType: x.preferredPayloadType as number - }; - }), - headerExtensions: test.split("\na=").map((x, i) => ({ - id: i + 1, - uri: x, - })) - } - }); - - const consumer = await transport.consume( - { - producerId: producer.id, - rtpCapabilities: - { - codecs: codecs.filter(x => x.kind === "audio").map((x: RtpCodecCapability) => { - return { - mimeType: x.mimeType, - kind: x.kind, - clockRate: x.clockRate, - channels: x.channels, - payloadType: x.preferredPayloadType as number - }; - }), - headerExtensions: test.split("\na=").map((x, i) => ({ - kind: "audio", - preferredId: i + 1, - uri: x, - })) - } - }); + /* + //discord proper sends: + { + "streams": [ + { "type": "video", "ssrc": 1311885, "rtx_ssrc": 1311886, "rid": "50", "quality": 50, "active": false }, + { "type": "video", "ssrc": 1311887, "rtx_ssrc": 1311888, "rid": "100", "quality": 100, "active": false } + ], + "ssrc": 1311884, + "port": 50008, + "modes": [ + "aead_aes256_gcm_rtpsize", + "aead_aes256_gcm", + "xsalsa20_poly1305_lite_rtpsize", + "xsalsa20_poly1305_lite", + "xsalsa20_poly1305_suffix", + "xsalsa20_poly1305" + ], + "ip": "109.200.214.158", + "experiments": [ + "bwe_conservative_link_estimate", + "bwe_remote_locus_client", + "fixed_keyframe_interval" + ] + } + */ socket.send(JSON.stringify({ op: VoiceOPCodes.READY, @@ -71,11 +46,13 @@ export async function onIdentify(this: Server, socket: WebSocket, data: Payload) //@ts-ignore port: transport.iceCandidates[0].port, modes: [ - "xsalsa20_poly1305", + "aead_aes256_gcm_rtpsize", + // "xsalsa20_poly1305", // "xsalsa20_poly1305_suffix", // "xsalsa20_poly1305_lite", ], heartbeat_interval: 1, + experiments: [], }, })); } \ No newline at end of file diff --git a/webrtc/src/opcodes/SelectProtocol.ts b/webrtc/src/opcodes/SelectProtocol.ts index 24e8ef5f..08335ade 100644 --- a/webrtc/src/opcodes/SelectProtocol.ts +++ b/webrtc/src/opcodes/SelectProtocol.ts @@ -2,6 +2,9 @@ import { WebSocket } from "@fosscord/gateway"; import { Payload } from "./index"; import { VoiceOPCodes } from "@fosscord/util"; import { Server } from "../Server"; +import * as mediasoup from "mediasoup"; +import { RtpCodecCapability } from "mediasoup/node/lib/RtpParameters"; +import * as sdpTransform from 'sdp-transform'; /* { @@ -66,6 +69,40 @@ import { Server } from "../Server"; */ export async function onSelectProtocol(this: Server, socket: WebSocket, data: Payload) { + const rtpCapabilities = this.mediasoupRouters[0].rtpCapabilities; + const codecs = rtpCapabilities.codecs as RtpCodecCapability[]; + + const transport = this.mediasoupTransports[0]; //whatever + + const res = sdpTransform.parse(data.d.sdp); + + /* + res.media.map(x => x.rtp).flat(1).map(x => ({ + codec: x.codec, + payloadType: x.payload, + clockRate: x.rate as number, + mimeType: `audio/${x.codec}`, + })), + */ + + const producer = await transport.produce({ + kind: "audio", + rtpParameters: { + mid: "audio", + codecs: [{ + clockRate: 48000, + payloadType: 111, + mimeType: "audio/opus", + channels: 2, + }], + headerExtensions: res.ext?.map(x => ({ + id: x.value, + uri: x.uri, + })) + }, + paused: false, + }); + socket.send(JSON.stringify({ op: VoiceOPCodes.SESSION_DESCRIPTION, d: { diff --git a/webrtc/src/opcodes/index.ts b/webrtc/src/opcodes/index.ts index 2fe69c38..36d30e7d 100644 --- a/webrtc/src/opcodes/index.ts +++ b/webrtc/src/opcodes/index.ts @@ -32,4 +32,6 @@ export default { //op 11? [VoiceOPCodes.CLIENT_CONNECT]: onConnect, //op 12 //op 13? + //op 15? + //op 16? empty data on client send but server sends {"voice":"0.8.24+bugfix.voice.streams.opt.branch-ffcefaff7","rtc_worker":"0.3.14-crypto-collision-copy"} }; \ No newline at end of file