Spacebar/server
1
0
Fork 0
mirror of https://github.com/spacebarchat/server.git synced 2024-11-22 02:12:40 +01:00
Code Issues Releases Activity

still doesn't work lol

Browse Source
This commit is contained in:
Madeline 2023-02-04 15:39:37 +11:00
parent a9e50cde7f
commit c170af41ce
No known key found for this signature in database
GPG Key ID: 1958E017C36F2E47
15 changed files with 7143 additions and 1034 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7856
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

5
package.json
View File

@ -109,10 +109,13 @@
"@fosscord/api": "dist/api",
"@fosscord/cdn": "dist/cdn",
"@fosscord/gateway": "dist/gateway",
"@fosscord/util": "dist/util"
"@fosscord/util": "dist/util",
"@fosscord/webrtc": "dist/webrtc"
},
"optionalDependencies": {
"erlpack": "^0.1.4",
"medooze-media-server": "^0.129.9",
"semantic-sdp": "^3.25.1",
"sqlite3": "^5.1.4"
}
}

10
src/bundle/Server.ts
View File

@ -23,6 +23,7 @@ import http from "http";
import * as Api from "@fosscord/api";
import * as Gateway from "@fosscord/gateway";
import { CDNServer } from "@fosscord/cdn";
import * as Webrtc from "@fosscord/webrtc";
import express from "express";
import { green, bold } from "picocolors";
import { Config, initDatabase, Sentry } from "@fosscord/util";
@ -36,12 +37,14 @@ server.on("request", app);
const api = new Api.FosscordServer({ server, port, production, app });
const cdn = new CDNServer({ server, port, production, app });
const gateway = new Gateway.Server({ server, port, production });
const webrtc = new Webrtc.Server({ port: 3004, production });
process.on("SIGTERM", async () => {
console.log("Shutting down due to SIGTERM");
await gateway.stop();
await cdn.stop();
await api.stop();
await webrtc.stop();
server.close();
Sentry.close();
});
@ -52,7 +55,12 @@ async function main() {
await Sentry.init(app);
server.listen(port);
await Promise.all([api.start(), cdn.start(), gateway.start()]);
await Promise.all([
api.start(),
cdn.start(),
gateway.start(),
webrtc.start(),
]);
Sentry.errorHandler(app);

4
src/gateway/util/Constants.ts
View File

@ -16,7 +16,7 @@
along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
// import { VoiceOPCodes } from "@fosscord/webrtc";
import { VoiceOPCodes } from "@fosscord/webrtc";
export enum OPCODES {
Dispatch = 0,
@ -63,7 +63,7 @@ export enum CLOSECODES {
}
export interface Payload {
op: OPCODES /* | VoiceOPCodes */;
op: OPCODES | VoiceOPCodes;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
d?: any;
s?: number;

4
src/gateway/util/WebSocket.ts
View File

@ -19,7 +19,7 @@
import { Intents, ListenEventOpts, Permissions } from "@fosscord/util";
import WS from "ws";
import { Deflate, Inflate } from "fast-zlib";
// import { Client } from "@fosscord/webrtc";
import { Client } from "@fosscord/webrtc";
export interface WebSocket extends WS {
version: number;
@ -40,5 +40,5 @@ export interface WebSocket extends WS {
events: Record<string, undefined | (() => unknown)>;
member_events: Record<string, () => unknown>;
listen_options: ListenEventOpts;
// client?: Client;
webrtcClient?: Client;
}

2
src/webrtc/Server.ts
View File

@ -19,6 +19,7 @@
import { closeDatabase, Config, initDatabase, initEvent } from "@fosscord/util";
import dotenv from "dotenv";
import http from "http";
import MediaServer from "medooze-media-server";
import ws from "ws";
import { Connection } from "./events/Connection";
dotenv.config();
@ -77,6 +78,7 @@ export class Server {
async stop() {
closeDatabase();
MediaServer.terminate();
this.server.close();
}
}

5
src/webrtc/events/Message.ts
View File

@ -30,14 +30,12 @@ const PayloadSchema = {
export async function onMessage(this: WebSocket, buffer: Buffer) {
try {
var data: Payload = JSON.parse(buffer.toString());
const data: Payload = JSON.parse(buffer.toString());
if (data.op !== VoiceOPCodes.IDENTIFY && !this.user_id)
return this.close(CLOSECODES.Not_authenticated);
// @ts-ignore
const OPCodeHandler = OPCodeHandlers[data.op];
if (!OPCodeHandler) {
// @ts-ignore
console.error("[WebRTC] Unkown opcode " + VoiceOPCodes[data.op]);
// TODO: if all opcodes are implemented comment this out:
// this.close(CloseCodes.Unknown_opcode);
@ -49,7 +47,6 @@ export async function onMessage(this: WebSocket, buffer: Buffer) {
data.op as VoiceOPCodes,
)
) {
// @ts-ignore
console.log("[WebRTC] Opcode " + VoiceOPCodes[data.op]);
}

4
src/webrtc/opcodes/BackendVersion.ts
View File

@ -16,10 +16,10 @@
along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import { Payload, Send, WebSocket } from "@fosscord/gateway";
import { Send, WebSocket } from "@fosscord/gateway";
import { VoiceOPCodes } from "../util";
export async function onBackendVersion(this: WebSocket, data: Payload) {
export async function onBackendVersion(this: WebSocket) {
await Send(this, {
op: VoiceOPCodes.VOICE_BACKEND_VERSION,
d: { voice: "0.8.43", rtc_worker: "0.3.26" },

22
src/webrtc/opcodes/Identify.ts
View File

@ -24,10 +24,11 @@ import {
} from "@fosscord/util";
import { endpoint, getClients, VoiceOPCodes, PublicIP } from "@fosscord/webrtc";
import SemanticSDP from "semantic-sdp";
const defaultSDP = require("./sdp.json");
import defaultSDP from "./sdp.json";
export async function onIdentify(this: WebSocket, data: Payload) {
clearTimeout(this.readyTimeout);
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const { server_id, user_id, session_id, token, streams, video } =
validateSchema("VoiceIdentifySchema", data.d) as VoiceIdentifySchema;
@ -47,7 +48,7 @@ export async function onIdentify(this: WebSocket, data: Payload) {
}),
);
this.client = {
this.webrtcClient = {
websocket: this,
out: {
tracks: new Map(),
@ -61,20 +62,27 @@ export async function onIdentify(this: WebSocket, data: Payload) {
channel_id: voiceState.channel_id,
};
const clients = getClients(voiceState.channel_id)!;
clients.add(this.client);
const clients = getClients(voiceState.channel_id);
clients.add(this.webrtcClient);
this.on("close", () => {
clients.delete(this.client!);
if (this.webrtcClient) clients.delete(this.webrtcClient);
});
await Send(this, {
op: VoiceOPCodes.READY,
d: {
streams: [
// { type: "video", ssrc: this.ssrc + 1, rtx_ssrc: this.ssrc + 2, rid: "100", quality: 100, active: false }
{
type: "video",
ssrc: this.webrtcClient.in.video_ssrc + 1,
rtx_ssrc: this.webrtcClient.in.video_ssrc + 2,
rid: "100",
quality: 100,
active: false,
},
],
ssrc: -1,
ssrc: this.webrtcClient.in.audio_ssrc,
port: endpoint.getLocalPort(),
modes: [
"aead_aes256_gcm_rtpsize",

87
src/webrtc/opcodes/SelectProtocol.ts
View File

@ -19,24 +19,28 @@
import { Payload, Send, WebSocket } from "@fosscord/gateway";
import { SelectProtocolSchema, validateSchema } from "@fosscord/util";
import { endpoint, PublicIP, VoiceOPCodes } from "@fosscord/webrtc";
import SemanticSDP, { MediaInfo, SDPInfo } from "semantic-sdp";
import SemanticSDP, { MediaInfo } from "semantic-sdp";
import DefaultSDP from "./sdp.json";
export async function onSelectProtocol(this: WebSocket, payload: Payload) {
if (!this.client) return;
if (!this.webrtcClient) return;
const data = validateSchema(
"SelectProtocolSchema",
payload.d,
) as SelectProtocolSchema;
const offer = SemanticSDP.SDPInfo.parse("m=audio\n" + data.sdp!);
this.client.sdp!.setICE(offer.getICE());
this.client.sdp!.setDTLS(offer.getDTLS());
const offer = SemanticSDP.SDPInfo.parse("m=audio\n" + data.sdp);
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
//@ts-ignore
offer.getMedias()[0].type = "audio"; // this is bad, but answer.toString() fails otherwise
this.webrtcClient.sdp.setICE(offer.getICE());
this.webrtcClient.sdp.setDTLS(offer.getDTLS());
const transport = endpoint.createTransport(this.client.sdp!);
this.client.transport = transport;
transport.setRemoteProperties(this.client.sdp!);
transport.setLocalProperties(this.client.sdp!);
const transport = endpoint.createTransport(this.webrtcClient.sdp);
this.webrtcClient.transport = transport;
transport.setRemoteProperties(this.webrtcClient.sdp);
transport.setLocalProperties(this.webrtcClient.sdp);
const dtls = transport.getLocalDTLSInfo();
const ice = transport.getLocalICEInfo();
@ -45,21 +49,66 @@ export async function onSelectProtocol(this: WebSocket, payload: Payload) {
const candidates = transport.getLocalCandidates();
const candidate = candidates[0];
const answer =
`m=audio ${port} ICE/SDP` +
`a=fingerprint:${fingerprint}` +
`c=IN IP4 ${PublicIP}` +
`a=rtcp:${port}` +
`a=ice-ufrag:${ice.getUfrag()}` +
`a=ice-pwd:${ice.getPwd()}` +
`a=fingerprint:${fingerprint}` +
`a=candidate:1 1 ${candidate.getTransport()} ${candidate.getFoundation()} ${candidate.getAddress()} ${candidate.getPort()} typ host`;
// discord answer
/*
m=audio 50026 ICE/SDP\n
a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87\n
c=IN IP4 66.22.206.174\n
a=rtcp:50026\n
a=ice-ufrag:XxnE\n
a=ice-pwd:GLQatPT3Q9dCZVVgVf3J1F\n
a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87\n
a=candidate:1 1 UDP 4261412862 66.22.206.174 50026 typ host\n
*/
const answer = offer.answer({
dtls: dtls,
ice: ice,
candidates: endpoint.getLocalCandidates(),
capabilities: {
audio: {
codecs: ["opus"],
rtx: true,
rtcpfbs: [{ id: "transport-cc" }],
extensions: [
"urn:ietf:params:rtp-hdrext:ssrc-audio-level",
"http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time",
"http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01",
"urn:ietf:params:rtp-hdrext:sdes:mid",
],
},
},
});
// the Video handler creates streams but we need streams now so idk
for (const offered of offer.getStreams().values()) {
const incomingStream = transport.createIncomingStream(offered);
const outgoingStream = transport.createOutgoingStream({
audio: true,
});
outgoingStream.attachTo(incomingStream);
this.webrtcClient.in.stream = incomingStream;
this.webrtcClient.out.stream = outgoingStream;
const info = outgoingStream.getStreamInfo();
answer.addStream(info);
}
// const answer =
// `m=audio ${port} ICE/SDP\n` +
// `a=fingerprint:${fingerprint}\n` +
// `c=IN IP4 ${PublicIP}\n` +
// `a=rtcp:${port}\n` +
// `a=ice-ufrag:${ice.getUfrag()}\n` +
// `a=ice-pwd:${ice.getPwd()}\n` +
// `a=fingerprint:${fingerprint}\n` +
// `a=candidate:1 1 ${candidate.getTransport()} ${candidate.getFoundation()} ${candidate.getAddress()} ${candidate.getPort()} typ host\n`;
await Send(this, {
op: VoiceOPCodes.SELECT_PROTOCOL_ACK,
d: {
video_codec: "H264",
sdp: answer,
sdp: answer.toString(),
media_session_id: this.session_id,
audio_codec: "opus",
},

9
src/webrtc/opcodes/Speaking.ts
View File

@ -22,11 +22,12 @@ import { getClients, VoiceOPCodes } from "../util";
// {"speaking":1,"delay":5,"ssrc":2805246727}
export async function onSpeaking(this: WebSocket, data: Payload) {
if (!this.client) return;
if (!this.webrtcClient) return;
getClients(this.client.channel_id).forEach((client) => {
if (client === this.client) return;
const ssrc = this.client!.out.tracks.get(client.websocket.user_id);
getClients(this.webrtcClient.channel_id).forEach((client) => {
if (client === this.webrtcClient) return;
if (!this.webrtcClient) return;
const ssrc = this.webrtcClient.out.tracks.get(client.websocket.user_id);
Send(client.websocket, {
op: VoiceOPCodes.SPEAKING,

154
src/webrtc/opcodes/Video.ts
View File

@ -19,67 +19,78 @@
import { Payload, Send, WebSocket } from "@fosscord/gateway";
import { validateSchema, VoiceVideoSchema } from "@fosscord/util";
import { channels, getClients, VoiceOPCodes } from "@fosscord/webrtc";
import { IncomingStreamTrack, SSRCs } from "medooze-media-server";
import {
IncomingStream,
IncomingStreamTrack,
SSRCs,
Transport,
} from "medooze-media-server";
import SemanticSDP from "semantic-sdp";
function createStream(
this: WebSocket,
transport: Transport,
channel_id: string,
) {
if (!this.webrtcClient) return;
if (!this.webrtcClient.transport) return;
const id = "stream" + this.user_id;
const stream = this.webrtcClient.transport.createIncomingStream(
SemanticSDP.StreamInfo.expand({
id,
tracks: [],
}),
);
this.webrtcClient.in.stream = stream;
const interval = setInterval(() => {
for (const track of stream.getTracks()) {
for (const layer of Object.values(track.getStats())) {
console.log(track.getId(), layer.total);
}
}
}, 5000);
stream.on("stopped", () => {
console.log("stream stopped");
clearInterval(interval);
});
this.on("close", () => {
transport.stop();
});
const out = transport.createOutgoingStream(
SemanticSDP.StreamInfo.expand({
id: "out" + this.user_id,
tracks: [],
}),
);
this.webrtcClient.out.stream = out;
const clients = channels.get(channel_id);
if (!clients) return;
clients.forEach((client) => {
if (client.websocket.user_id === this.user_id) return;
if (!client.in.stream) return;
client.in.stream?.getTracks().forEach((track) => {
attachTrack.call(this, track, client.websocket.user_id);
});
});
}
export async function onVideo(this: WebSocket, payload: Payload) {
if (!this.client) return;
const { transport, channel_id } = this.client;
if (!this.webrtcClient) return;
const { transport, channel_id } = this.webrtcClient;
if (!transport) return;
const d = validateSchema("VoiceVideoSchema", payload.d) as VoiceVideoSchema;
await Send(this, { op: VoiceOPCodes.MEDIA_SINK_WANTS, d: { any: 100 } });
const id = "stream" + this.user_id;
var stream = this.client.in.stream!;
if (!stream) {
stream = this.client.transport!.createIncomingStream(
// @ts-ignore
SemanticSDP.StreamInfo.expand({
id,
// @ts-ignore
tracks: [],
}),
);
this.client.in.stream = stream;
const interval = setInterval(() => {
for (const track of stream.getTracks()) {
for (const layer of Object.values(track.getStats())) {
console.log(track.getId(), layer.total);
}
}
}, 5000);
stream.on("stopped", () => {
console.log("stream stopped");
clearInterval(interval);
});
this.on("close", () => {
transport!.stop();
});
const out = transport.createOutgoingStream(
// @ts-ignore
SemanticSDP.StreamInfo.expand({
id: "out" + this.user_id,
// @ts-ignore
tracks: [],
}),
);
this.client.out.stream = out;
const clients = channels.get(channel_id)!;
clients.forEach((client) => {
if (client.websocket.user_id === this.user_id) return;
if (!client.in.stream) return;
client.in.stream?.getTracks().forEach((track) => {
attachTrack.call(this, track, client.websocket.user_id);
});
});
}
if (!this.webrtcClient.in.stream)
createStream.call(this, transport, channel_id);
if (d.audio_ssrc) {
handleSSRC.call(this, "audio", {
@ -100,23 +111,32 @@ function attachTrack(
track: IncomingStreamTrack,
user_id: string,
) {
if (!this.client) return;
const outTrack = this.client.transport!.createOutgoingStreamTrack(
if (
!this.webrtcClient ||
!this.webrtcClient.transport ||
!this.webrtcClient.out.stream
)
return;
const outTrack = this.webrtcClient.transport.createOutgoingStreamTrack(
track.getMedia(),
);
outTrack.attachTo(track);
this.client.out.stream!.addTrack(outTrack);
var ssrcs = this.client.out.tracks.get(user_id)!;
this.webrtcClient.out.stream.addTrack(outTrack);
let ssrcs = this.webrtcClient.out.tracks.get(user_id);
if (!ssrcs)
ssrcs = this.client.out.tracks
ssrcs = this.webrtcClient.out.tracks
.set(user_id, { audio_ssrc: 0, rtx_ssrc: 0, video_ssrc: 0 })
.get(user_id)!;
.get(user_id);
if (!ssrcs) return; // hmm
if (track.getMedia() === "audio") {
ssrcs.audio_ssrc = outTrack.getSSRCs().media!;
ssrcs.audio_ssrc = outTrack.getSSRCs().media || 0;
} else if (track.getMedia() === "video") {
ssrcs.video_ssrc = outTrack.getSSRCs().media!;
ssrcs.rtx_ssrc = outTrack.getSSRCs().rtx!;
ssrcs.video_ssrc = outTrack.getSSRCs().media || 0;
ssrcs.rtx_ssrc = outTrack.getSSRCs().rtx || 0;
}
Send(this, {
@ -129,18 +149,18 @@ function attachTrack(
}
function handleSSRC(this: WebSocket, type: "audio" | "video", ssrcs: SSRCs) {
if (!this.client) return;
const stream = this.client.in.stream!;
const transport = this.client.transport!;
if (!this.webrtcClient) return;
const stream = this.webrtcClient.in.stream as IncomingStream;
const transport = this.webrtcClient.transport as Transport;
const id = type + ssrcs.media;
var track = stream.getTrack(id);
let track = stream.getTrack(id);
if (!track) {
console.log("createIncomingStreamTrack", id);
track = transport.createIncomingStreamTrack(type, { id, ssrcs });
stream.addTrack(track);
const clients = getClients(this.client.channel_id)!;
const clients = getClients(this.webrtcClient.channel_id);
clients.forEach((client) => {
if (client.websocket.user_id === this.user_id) return;
if (!client.out.stream) return;

4
src/webrtc/opcodes/index.ts
View File

@ -25,7 +25,7 @@ import { onSelectProtocol } from "./SelectProtocol";
import { onSpeaking } from "./Speaking";
import { onVideo } from "./Video";
export type OPCodeHandler = (this: WebSocket, data: Payload) => any;
export type OPCodeHandler = (this: WebSocket, data: Payload) => unknown;
export default {
[VoiceOPCodes.HEARTBEAT]: onHeartbeat,
@ -34,4 +34,4 @@ export default {
[VoiceOPCodes.VIDEO]: onVideo,
[VoiceOPCodes.SPEAKING]: onSpeaking,
[VoiceOPCodes.SELECT_PROTOCOL]: onSelectProtocol,
};
} as { [key: number]: OPCodeHandler };

6
src/webrtc/util/MediaServer.ts
View File

@ -28,8 +28,8 @@ MediaServer.enableLog(true);
export const PublicIP = process.env.PUBLIC_IP || "127.0.0.1";
try {
const range = process.env.WEBRTC_PORT_RANGE || "4000";
var ports = range.split("-");
const range = process.env.WEBRTC_PORT_RANGE || "4000-5000";
const ports = range.split("-");
const min = Number(ports[0]);
const max = Number(ports[1]);
@ -73,5 +73,5 @@ export interface Client {
export function getClients(channel_id: string) {
if (!channels.has(channel_id)) channels.set(channel_id, new Set());
return channels.get(channel_id)!;
return channels.get(channel_id) as Set<Client>;
}

5
tsconfig.json
View File

@ -1,5 +1,5 @@
{
"exclude": ["./src/webrtc"],
// "exclude": ["./src/webrtc"],
"include": ["./src"],
"compilerOptions": {
/* Visit https://aka.ms/tsconfig to read more about this file */
@ -37,7 +37,8 @@
"@fosscord/api*": ["./api"],
"@fosscord/gateway*": ["./gateway"],
"@fosscord/cdn*": ["./cdn"],
"@fosscord/util*": ["./util"]
"@fosscord/util*": ["./util"],
"@fosscord/webrtc*": ["./webrtc"]
} /* Specify a set of entries that re-map imports to additional lookup locations. */,
// "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */
// "typeRoots": [], /* Specify multiple folders that act like './node_modules/@types'. */