feat(ssrc-rewriting) Add initial implementation (#2136)

Instead of Jicofo signaling all the remote sources available in the call, the bridge now signals only a limited set of SSRCs and then rewrites the SSRC on the outgoing media streams. The SSRC mapping is done based on the sources requested by the clients through the receiver constraints. This limits the number of m-lines in the remote/local SDPs on the client and therefore results in better performance in large calls.

* Handle source remapping messages from bridge
* Added track_owner_changed events
* don't process an invalid rtx ssrc.
* keep track of remote ssrcs, only renegotiate on new ones.
* Change source name on remote track on ssrc remapping.
* Don't remove tracks on member leave.
* Remove (orphaned) tracks on session terminated.
* Use serial number (per media type) to create msid attribute.
* Update videoType on remapping.

Co-authored-by: James A <jqdrqgnq@users.noreply.github.com>

JitsiConference.js

         return;
     }
 
-    const participant = this.participants[id];
-    const mediaSessions = this.getMediaSessions();
-    let tracksToBeRemoved = [];
+    if (!FeatureFlags.isSsrcRewritingSupported()) {
+        const mediaSessions = this.getMediaSessions();
+        let tracksToBeRemoved = [];
+
+        for (const session of mediaSessions) {
+            const remoteTracks = session.peerconnection.getRemoteTracks(id);
 
-    for (const session of mediaSessions) {
-        const remoteTracks = session.peerconnection.getRemoteTracks(id);
+            remoteTracks && (tracksToBeRemoved = [ ...tracksToBeRemoved, ...remoteTracks ]);
 
-        remoteTracks && (tracksToBeRemoved = [ ...tracksToBeRemoved, ...remoteTracks ]);
+            // Remove the ssrcs from the remote description and renegotiate.
+            session.removeRemoteStreamsOnLeave(id);
+        }
 
-        // Remove the ssrcs from the remote description and renegotiate.
-        session.removeRemoteStreamsOnLeave(id);
+        // Fire the event before renegotiation is done so that the thumbnails can be removed immediately.
+        tracksToBeRemoved.forEach(track => {
+            this.eventEmitter.emit(JitsiConferenceEvents.TRACK_REMOVED, track);
+        });
     }
 
-    // Fire the event before renegotiation is done so that the thumbnails can be removed immediately.
-    tracksToBeRemoved.forEach(track => {
-        this.eventEmitter.emit(JitsiConferenceEvents.TRACK_REMOVED, track);
-    });
+    const participant = this.participants[id];
 
     if (participant) {
         delete this.participants[id];

JitsiConferenceEventManager.js

 
 import * as JitsiConferenceErrors from './JitsiConferenceErrors';
 import * as JitsiConferenceEvents from './JitsiConferenceEvents';
+import * as JitsiTrackEvents from './JitsiTrackEvents';
 import { SPEAKERS_AUDIO_LEVELS } from './modules/statistics/constants';
 import Statistics from './modules/statistics/statistics';
 import EventEmitterForwarder from './modules/util/EventEmitterForwarder';
         }
     });
 
+    chatRoom.addListener(JitsiTrackEvents.TRACK_REMOVED, track => {
+        conference.eventEmitter.emit(JitsiConferenceEvents.TRACK_REMOVED, track);
+    });
+
     this.chatRoomForwarder.forward(XMPPEvents.ROOM_JOIN_ERROR,
         JitsiConferenceEvents.CONFERENCE_FAILED,
         JitsiConferenceErrors.CONNECTION_ERROR);
         conference.eventEmitter.emit(JitsiConferenceEvents.DATA_CHANNEL_OPENED);
     });
 
+    rtc.addListener(RTCEvents.VIDEO_SSRCS_REMAPPED, msg => {
+        const sess = this.conference.getActiveMediaSession();
+
+        sess.videoSsrcsRemapped(msg);
+    });
+
+    rtc.addListener(RTCEvents.AUDIO_SSRCS_REMAPPED, msg => {
+        const sess = this.conference.getActiveMediaSession();
+
+        sess.audioSsrcsRemapped(msg);
+    });
+
     rtc.addListener(RTCEvents.ENDPOINT_MESSAGE_RECEIVED,
         (from, payload) => {
             const participant = conference.getParticipantById(from);

JitsiMeetJS.ts

     externalStorage?: Storage;
     flags?: {
         enableUnifiedOnChrome?: boolean;
+        receiveMultipleVideoStreams?: boolean;
+        runInLiteMode?: boolean;
+        sendMultipleVideoStreams?: boolean;
+        sourceNameSignaling?: boolean;
+        ssrcRewritingEnabled?: boolean;
     }
 }
 

JitsiTrackEvents.spec.ts

         TRACK_VIDEOTYPE_CHANGED,
         NO_DATA_FROM_SOURCE,
         NO_AUDIO_INPUT,
+        TRACK_OWNER_CHANGED,
+        TRACK_REMOVED,
         JitsiTrackEvents,
         ...others
     } = exported;
         expect( TRACK_AUDIO_LEVEL_CHANGED ).toBe( 'track.audioLevelsChanged' );
         expect( TRACK_AUDIO_OUTPUT_CHANGED ).toBe( 'track.audioOutputChanged' );
         expect( TRACK_MUTE_CHANGED ).toBe( 'track.trackMuteChanged' );
+        expect( TRACK_STREAMING_STATUS_CHANGED ).toBe( 'track.streaming_status_changed' );
         expect( TRACK_VIDEOTYPE_CHANGED ).toBe( 'track.videoTypeChanged' );
         expect( NO_DATA_FROM_SOURCE ).toBe( 'track.no_data_from_source' );
         expect( NO_AUDIO_INPUT ).toBe( 'track.no_audio_input' );
+        expect( TRACK_OWNER_CHANGED ).toBe( 'track.owner_changed' );
+        expect( TRACK_REMOVED ).toBe( 'track.removed' );
 
         expect( JitsiTrackEvents ).toBeDefined();
 
         expect( JitsiTrackEvents.TRACK_VIDEOTYPE_CHANGED ).toBe( 'track.videoTypeChanged' );
         expect( JitsiTrackEvents.NO_DATA_FROM_SOURCE ).toBe( 'track.no_data_from_source' );
         expect( JitsiTrackEvents.NO_AUDIO_INPUT ).toBe( 'track.no_audio_input' );
+        expect( JitsiTrackEvents.TRACK_OWNER_CHANGED ).toBe( 'track.owner_changed' );
+        expect( JitsiTrackEvents.TRACK_REMOVED ).toBe( 'track.removed' );
     } );
 
     it( "unknown members", () => {

JitsiTrackEvents.ts

      *
      * The current status value can be obtained by calling JitsiRemoteTrack.getTrackStreamingStatus().
      */
-    TRACK_STREAMING_STATUS_CHANGED = 'track.streaming_status_changed'
+    TRACK_STREAMING_STATUS_CHANGED = 'track.streaming_status_changed',
+
+    /**
+     * An SSRC has been remapped. The track is now associated with a new participant.
+     */
+    TRACK_OWNER_CHANGED = 'track.owner_changed',
+
+    /**
+     * A track is being removed. Fired when a session terminates for tracks
+     * that persist in ssrc-rewriting mode.
+     */
+    TRACK_REMOVED = 'track.removed',
 };
 
 // exported for backward compatibility
 export const NO_DATA_FROM_SOURCE = JitsiTrackEvents.NO_DATA_FROM_SOURCE;
 export const NO_AUDIO_INPUT = JitsiTrackEvents.NO_AUDIO_INPUT;
 export const TRACK_STREAMING_STATUS_CHANGED = JitsiTrackEvents.TRACK_STREAMING_STATUS_CHANGED;
+export const TRACK_OWNER_CHANGED = JitsiTrackEvents.TRACK_OWNER_CHANGED;
+export const TRACK_REMOVED = JitsiTrackEvents.TRACK_REMOVED;

modules/RTC/BridgeChannel.js

                 logger.info(`Received ServerHello, version=${obj.version}.`);
                 break;
             }
+            case 'VideoSourcesMap': {
+                logger.info(`Received VideoSourcesMap: ${JSON.stringify(obj.mappedSources)}`);
+                emitter.emit(RTCEvents.VIDEO_SSRCS_REMAPPED, obj);
+                break;
+            }
+            case 'AudioSourcesMap': {
+                logger.info(`Received AudioSourcesMap: ${JSON.stringify(obj.mappedSources)}`);
+                emitter.emit(RTCEvents.AUDIO_SSRCS_REMAPPED, obj);
+                break;
+            }
             default: {
                 logger.debug('Channel JSON-formatted message: ', obj);
 

modules/RTC/JitsiRemoteTrack.js

         return this._sourceName;
     }
 
+    /**
+     * Update the properties when the track is remapped to another source.
+     *
+     * @param {string} owner The endpoint ID of the new owner.
+     * @param {string} name The new source name.
+     */
+    setNewSource(owner, name) {
+        this.ownerEndpointId = owner;
+        this._sourceName = name;
+        this.emit(JitsiTrackEvents.TRACK_OWNER_CHANGED, owner);
+    }
+
     /**
      * Changes the video type of the track.
      *
      */
     toString() {
         return `RemoteTrack[userID: ${this.getParticipantId()}, type: ${this.getType()}, ssrc: ${
-            this.getSSRC()}, p2p: ${this.isP2P}, sourceName: ${this._sourceName}, status: ${this._getStatus()}]`;
+            this.getSSRC()}, p2p: ${this.isP2P}, sourceName: ${this._sourceName}, status: {${this._getStatus()}}]`;
     }
 }

modules/RTC/TraceablePeerConnection.js

      */
     this.localSSRCs = new Map();
 
+    /**
+     * The set of remote SSRCs seen so far.
+     * Distinguishes new SSRCs from those that have been remapped.
+     */
+    this.remoteSSRCs = new Set();
+
     /**
      * The local ICE username fragment for this session.
      */
             + 'deleting the existing track');
         const existingTrack = Array.from(userTracksByMediaType)[0];
 
-        // The exisiting track needs to be removed here. This happens on Safari sometimes when a SSRC is removed from
+        // The existing track needs to be removed here. This happens on Safari sometimes when an SSRC is removed from
         // the remote description and the browser doesn't fire a 'removetrack' event on the associated MediaStream.
         this._remoteTrackRemoved(existingTrack.getOriginalStream(), existingTrack.getTrack());
     }
     }
 };
 
+/**
+ * Track the SSRCs seen so far.
+ * @param {int} ssrc - SSRC.
+ * @return {boolean} - Whether this is a new SSRC.
+ */
+TraceablePeerConnection.prototype.addRemoteSsrc = function(ssrc) {
+    const existing = this.remoteSSRCs.has(ssrc);
+
+    if (!existing) {
+        this.remoteSSRCs.add(ssrc);
+    }
+
+    return !existing;
+};
+
 TraceablePeerConnection.prototype.addIceCandidate = function(candidate) {
     this.trace('addIceCandidate', JSON.stringify({
         candidate: candidate.candidate,

modules/flags/FeatureFlags.js

     /**
      * Configures the module.
      *
-     * @param {object} flags - The feature flags.
-     * @param {boolean=} flags.enableUnifiedOnChrome - Enable unified plan implementation support on Chromium.
-     * @param {boolean=} flags.runInLiteMode - Enables lite mode for testing to disable media decoding.
-     * @param {boolean=} flags.sourceNameSignaling - Enables source names in the signaling.
-     * @param {boolean=} flags.receiveMultipleVideoStreams - Signal support for receiving multiple video streams.
+     * @param {boolean} flags.runInLiteMode - Enables lite mode for testing to disable media decoding.
+     * @param {boolean} flags.receiveMultipleVideoStreams - Signal support for receiving multiple video streams.
+     * @param {boolean} flags.sendMultipleVideoStreams - Signal support for sending multiple video streams.
+     * @param {boolean} flags.sourceNameSignaling - Enables source names in the signaling.
+     * @param {boolean} flags.ssrcRewritingEnabled - Use SSRC rewriting. Requires sourceNameSignaling to be enabled.
+     * @param {boolean} flags.enableUnifiedOnChrome - Use unified plan signaling on chrome browsers.
      */
     init(flags) {
-        this._receiveMultipleVideoStreams = flags.receiveMultipleVideoStreams ?? true;
         this._runInLiteMode = Boolean(flags.runInLiteMode);
+        this._receiveMultipleVideoStreams = flags.receiveMultipleVideoStreams ?? true;
         this._sendMultipleVideoStreams = flags.sendMultipleVideoStreams ?? true;
         this._sourceNameSignaling = flags.sourceNameSignaling ?? true;
-        this._ssrcRewriting = Boolean(flags.ssrcRewritingEnabled);
+        this._ssrcRewriting = this._sourceNameSignaling && Boolean(flags.ssrcRewritingEnabled);
 
         // For Chromium, check if Unified plan is enabled.
         this._usesUnifiedPlan = browser.supportsUnifiedPlan()
             && (!browser.isChromiumBased() || (flags.enableUnifiedOnChrome ?? true));
 
-        logger.info(`Source name signaling: ${this._sourceNameSignaling},`
-            + ` Send multiple video streams: ${this._sendMultipleVideoStreams},`
-            + ` uses Unified plan: ${this._usesUnifiedPlan}`);
+        logger.info(`Send multiple video streams: ${this._sendMultipleVideoStreams},`
+            + ` Source name signaling: ${this._sourceNameSignaling},`
+            + ` Unified plan: ${this._usesUnifiedPlan}`);
     }
 
     /**

modules/xmpp/JingleSessionPC.js

 import { getLogger } from '@jitsi/logger';
 import $ from 'jquery';
-import { $iq, Strophe } from 'strophe.js';
+import { $build, $iq, Strophe } from 'strophe.js';
 
+import { JitsiTrackEvents } from '../../JitsiTrackEvents';
 import * as CodecMimeType from '../../service/RTC/CodecMimeType';
 import { MediaDirection } from '../../service/RTC/MediaDirection';
 import { MediaType } from '../../service/RTC/MediaType';
     return Strophe.getResourceFromJid(jidOrEndpointId) || jidOrEndpointId;
 }
 
+/**
+ * Add "source" element as a child of "description" element.
+ * @param {Object} description The "description" element to add to.
+ * @param {Object} s Contains properties of the source being added.
+ * @param {Number} ssrc_ The SSRC.
+ * @param {String} msid The "msid" attribute.
+ */
+function _addSourceElement(description, s, ssrc_, msid) {
+
+    description.c('source', {
+        xmlns: 'urn:xmpp:jingle:apps:rtp:ssma:0',
+        ssrc: ssrc_,
+        name: s.source
+    })
+        .c('parameter', {
+            name: 'msid',
+            value: msid
+        })
+        .up()
+        .c('ssrc-info', {
+            xmlns: 'http://jitsi.org/jitmeet',
+            owner: s.owner
+        })
+        .up()
+        .up();
+}
+
 /**
  * @typedef {Object} JingleSessionPCOptions
  * @property {Object} abTesting - A/B testing related options (ask George).
          */
         this.remoteRecvMaxFrameHeight = undefined;
 
+        /**
+         * Number of remote video sources, in SSRC rewriting mode.
+         * Used to generate next unique msid attribute.
+         *
+         * @type {Number}
+         */
+        this.numRemoteVideoSources = 0;
+
+        /**
+         * Number of remote audio sources, in SSRC rewriting mode.
+         * Used to generate next unique msid attribute.
+         *
+         * @type {Number}
+         */
+        this.numRemoteAudioSources = 0;
+
         /**
          * Remote preference for the receive video max frame heights when source-name signaling is enabled.
          *
             this._removeSenderVideoConstraintsChangeListener();
         }
 
+        if (FeatureFlags.isSsrcRewritingSupported() && this.peerconnection) {
+            this.peerconnection.getRemoteTracks().forEach(track => {
+                this.room.eventEmitter.emit(JitsiTrackEvents.TRACK_REMOVED, track);
+            });
+        }
+
         this.close();
     }
 
         this._addOrRemoveRemoteStream(false /* remove */, elem);
     }
 
+    /**
+     * Filter remapped SSRCs.
+     * Process owner change for existing SSRCs.
+     * Return new ones for further processing.
+     */
+    getNewSources(msg) {
+        const newSources = [];
+
+        for (const s of msg.mappedSources) {
+            if (this.peerconnection.addRemoteSsrc(s.ssrc)) {
+                logger.debug(`New SSRC ${s.ssrc}`);
+                newSources[newSources.length] = s;
+            } else {
+                const track = this.peerconnection.getTrackBySSRC(s.ssrc);
+
+                if (track) {
+                    logger.debug(`Existing SSRC ${s.ssrc}: new owner ${s.owner}. name=${s.source}`);
+
+                    if (s.videoType === 'CAMERA') {
+                        track._setVideoType('camera');
+                    } else if (s.videoType === 'DESKTOP') {
+                        track._setVideoType('desktop');
+                    }
+
+                    track.setNewSource(s.owner, s.source);
+                } else {
+                    logger.error(`Remapped SSRC ${s.ssrc} not found`);
+                }
+            }
+        }
+
+        return newSources;
+    }
+
+    /**
+     * Process SSRC remappings for video sources.
+     */
+    videoSsrcsRemapped(msg) {
+        const newSources = this.getNewSources(msg);
+
+        if (newSources.length > 0) {
+
+            let node = $build('content', {
+                xmlns: 'urn:xmpp:jingle:1',
+                name: 'video'
+            }).c('description', {
+                xmlns: 'urn:xmpp:jingle:apps:rtp:1',
+                media: MediaType.VIDEO
+            });
+
+            for (const s of newSources) {
+                const idx = ++this.numRemoteVideoSources;
+                const msid = `remote-video-${idx} remote-video-${idx}`;
+
+                _addSourceElement(node, s, s.ssrc, msid);
+                if (s.rtx !== '-1') {
+                    _addSourceElement(node, s, s.rtx, msid);
+                    node.c('ssrc-group', {
+                        xmlns: 'urn:xmpp:jingle:apps:rtp:ssma:0',
+                        semantics: 'FID'
+                    })
+                        .c('source', {
+                            xmlns: 'urn:xmpp:jingle:apps:rtp:ssma:0',
+                            ssrc: s.ssrc
+                        })
+                        .up()
+                        .c('source', {
+                            xmlns: 'urn:xmpp:jingle:apps:rtp:ssma:0',
+                            ssrc: s.rtx
+                        })
+                        .up()
+                        .up();
+                }
+            }
+
+            node = node.up();
+
+            this._addOrRemoveRemoteStream(true /* add */, node.node);
+        }
+    }
+
+    /**
+     * Process SSRC remappings for audio sources.
+     */
+    audioSsrcsRemapped(msg) {
+        const newSources = this.getNewSources(msg);
+
+        if (newSources.length > 0) {
+
+            let node = $build('content', {
+                xmlns: 'urn:xmpp:jingle:1',
+                name: 'audio'
+            }).c('description', {
+                xmlns: 'urn:xmpp:jingle:apps:rtp:1',
+                media: MediaType.AUDIO
+            });
+
+            for (const s of newSources) {
+                const idx = ++this.numRemoteAudioSources;
+                const msid = `remote-audio-${idx} remote-audio-${idx}`;
+
+                _addSourceElement(node, s, s.ssrc, msid);
+            }
+
+            node = node.up();
+
+            this._addOrRemoveRemoteStream(true /* add */, node.node);
+        }
+    }
+
     /**
      * Handles the deletion of SSRCs associated with a remote user from the remote description when the user leaves.
      *

modules/xmpp/xmpp.js

             logger.info('Receiving multiple video streams is enabled');
             this.caps.addFeature('http://jitsi.org/receive-multiple-video-streams');
         }
-
-        if (FeatureFlags.isSsrcRewritingSupported()) {
-            logger.info('SSRC rewriting is supported');
-            this.caps.addFeature('http://jitsi.org/ssrc-rewriting');
-        }
     }
 
     /**

service/RTC/RTCEvents.spec.ts

         ENDPOINT_STATS_RECEIVED,
         LOCAL_UFRAG_CHANGED,
         REMOTE_UFRAG_CHANGED,
+        VIDEO_SSRCS_REMAPPED,
+        AUDIO_SSRCS_REMAPPED,
         RTCEvents,
         default: RTCEventsDefault,
         ...others
         expect( ENDPOINT_STATS_RECEIVED ).toBe( 'rtc.endpoint_stats_received' );
         expect( LOCAL_UFRAG_CHANGED ).toBe( 'rtc.local_ufrag_changed' );
         expect( REMOTE_UFRAG_CHANGED ).toBe( 'rtc.remote_ufrag_changed' );
+        expect( VIDEO_SSRCS_REMAPPED ).toBe( 'rtc.video_ssrcs_remapped' );
+        expect( AUDIO_SSRCS_REMAPPED ).toBe( 'rtc.audio_ssrcs_remapped' );
 
         if ( RTCEvents ) {
             expect( RTCEvents.CREATE_ANSWER_FAILED ).toBe( 'rtc.create_answer_failed' );
             expect( RTCEvents.ENDPOINT_STATS_RECEIVED ).toBe( 'rtc.endpoint_stats_received' );
             expect( RTCEvents.LOCAL_UFRAG_CHANGED ).toBe( 'rtc.local_ufrag_changed' );
             expect( RTCEvents.REMOTE_UFRAG_CHANGED ).toBe( 'rtc.remote_ufrag_changed' );
+            expect( RTCEvents.VIDEO_SSRCS_REMAPPED ).toBe( 'rtc.video_ssrcs_remapped' );
+            expect( RTCEvents.AUDIO_SSRCS_REMAPPED ).toBe( 'rtc.audio_ssrcs_remapped' );
         }
 
         if ( RTCEventsDefault ) {
             expect( RTCEventsDefault.ENDPOINT_STATS_RECEIVED ).toBe( 'rtc.endpoint_stats_received' );
             expect( RTCEventsDefault.LOCAL_UFRAG_CHANGED ).toBe( 'rtc.local_ufrag_changed' );
             expect( RTCEventsDefault.REMOTE_UFRAG_CHANGED ).toBe( 'rtc.remote_ufrag_changed' );
+            expect( RTCEventsDefault.VIDEO_SSRCS_REMAPPED ).toBe( 'rtc.video_ssrcs_remapped' );
+            expect( RTCEventsDefault.AUDIO_SSRCS_REMAPPED ).toBe( 'rtc.audio_ssrcs_remapped' );
         }
     } );
 

service/RTC/RTCEvents.ts

      * is the source of the event.
      * The second argument is the actual "ufrag" string.
      */
-    REMOTE_UFRAG_CHANGED = 'rtc.remote_ufrag_changed'
+    REMOTE_UFRAG_CHANGED = 'rtc.remote_ufrag_changed',
+
+    /**
+     * Designates an event indicating that some received video SSRCs will now map to
+     * new remote sources.
+     */
+    VIDEO_SSRCS_REMAPPED = 'rtc.video_ssrcs_remapped',
+
+    /**
+     * Designates an event indicating that some received audio SSRCs will now map to
+     * new remote sources.
+     */
+    AUDIO_SSRCS_REMAPPED = 'rtc.audio_ssrcs_remapped'
 };
 
 export const CREATE_ANSWER_FAILED = RTCEvents.CREATE_ANSWER_FAILED;
 export const ENDPOINT_STATS_RECEIVED = RTCEvents.ENDPOINT_STATS_RECEIVED;
 export const LOCAL_UFRAG_CHANGED = RTCEvents.LOCAL_UFRAG_CHANGED;
 export const REMOTE_UFRAG_CHANGED = RTCEvents.REMOTE_UFRAG_CHANGED;
+export const VIDEO_SSRCS_REMAPPED = RTCEvents.VIDEO_SSRCS_REMAPPED;
+export const AUDIO_SSRCS_REMAPPED = RTCEvents.AUDIO_SSRCS_REMAPPED;
 
 // TODO: this was a pre-ES6 module using module.exports = RTCEvents which doesn't translate well
 // it is used in a number of places and should be updated to use the named export