fix(stats) Obtain resolution/fps from 'outbound-rtp' stats. (#2265)

* fix(stats) Obtain resolution/fps from 'outbound-rtp' stats.
'Track' based stats were dropped in Chrome 112, therefore send resolution and fps for the simulcast case needs to be calculated based on the 'outbound-rtp' streams that are currently active.

* squash: remove an unwanted log

* Squash: Address review comments.
Use outbound-rtp stats for both Firefox and Safari.
Firefox - Ignore active field if not present in the stats and calc fps using 'framesSent'.

modules/RTC/TraceablePeerConnection.js

@@ -680,6 +680,33 @@ TraceablePeerConnection.prototype.getAudioLevels = function(speakerList = []) {
     return audioLevels;
 };
 
+/**
+ * Checks if the browser is currently doing true simulcast where in three different media streams are being sent to the
+ * bridge. Currently this happens only when VP8 is the selected codec.
+ * @returns {boolean}
+ */
+TraceablePeerConnection.prototype.doesTrueSimulcast = function() {
+    return this.isSimulcastOn() && this.getConfiguredVideoCodec() === CodecMimeType.VP8;
+};
+
+/**
+ * Returns the SSRCs associated with a given local video track.
+ *
+ * @param {JitsiLocalTrack} localTrack
+ * @returns
+ */
+TraceablePeerConnection.prototype.getLocalVideoSSRCs = function(localTrack) {
+    const ssrcs = [];
+
+    if (!localTrack || !localTrack.isVideoTrack()) {
+        return ssrcs;
+    }
+
+    const ssrcGroup = this.isSimulcastOn() ? 'SIM' : 'FID';
+
+    return this.localSSRCs.get(localTrack.rtcId)?.groups?.find(group => group.semantics === ssrcGroup)?.ssrcs || ssrcs;
+};
+
 /**
  * Obtains local tracks for given {@link MediaType}. If the <tt>mediaType</tt>
  * argument is omitted the list of all local tracks will be returned.
@@ -1237,6 +1264,18 @@ TraceablePeerConnection.prototype._extractSSRCMap = function(desc) {
                     groupsMap.get(primarySSRC).push(group);
                 }
             }
+
+            const simGroup = mLine.ssrcGroups.find(group => group.semantics === 'SIM');
+
+            // Add a SIM group if its missing in the description (happens on Firefox).
+            if (!simGroup) {
+                const groupSsrcs = mLine.ssrcGroups.map(group => group.ssrcs[0]);
+
+                groupsMap.get(groupSsrcs[0]).push({
+                    semantics: 'SIM',
+                    ssrcs: groupSsrcs
+                });
+            }
         }
 
         let ssrcs = mLine.ssrcs;

modules/browser/BrowserCapabilities.js

@@ -219,6 +219,15 @@ export default class BrowserCapabilities extends BrowserDetection {
         return !this.isFirefox();
     }
 
+    /**
+     * Returns true if the browser supports track based statistics for the local video track. Otherwise,
+     * track resolution and framerate will be calculated based on the 'outbound-rtp' statistics.
+     * @returns {boolean}
+     */
+    supportsTrackBasedStats() {
+        return this.isChromiumBased() && this.isVersionLessThan(112);
+    }
+
     /**
      * Returns true if VP9 is supported by the client on the browser. VP9 is currently disabled on Firefox and Safari
      * because of issues with rendering. Please check https://bugzilla.mozilla.org/show_bug.cgi?id=1492500,

modules/statistics/RTPStatsCollector.js

@@ -275,61 +275,86 @@ StatsCollector.prototype._processAndEmitReport = function() {
         bitrateDownload += ssrcStats.bitrate.download;
         bitrateUpload += ssrcStats.bitrate.upload;
 
+        ssrcStats.resetBitrate();
+
         // collect resolutions and framerates
         const track = this.peerconnection.getTrackBySSRC(ssrc);
 
-        if (track) {
-            let audioCodec;
-            let videoCodec;
-
-            if (track.isAudioTrack()) {
-                audioBitrateDownload += ssrcStats.bitrate.download;
-                audioBitrateUpload += ssrcStats.bitrate.upload;
-                audioCodec = ssrcStats.codec;
-            } else {
-                videoBitrateDownload += ssrcStats.bitrate.download;
-                videoBitrateUpload += ssrcStats.bitrate.upload;
-                videoCodec = ssrcStats.codec;
-            }
-
-            const participantId = track.getParticipantId();
-
-            if (participantId) {
-                const resolution = ssrcStats.resolution;
+        if (!track) {
+            continue; // eslint-disable-line no-continue
+        }
 
-                if (resolution.width
-                        && resolution.height
-                        && resolution.width !== -1
-                        && resolution.height !== -1) {
-                    const userResolutions = resolutions[participantId] || {};
+        let audioCodec;
+        let videoCodec;
 
-                    userResolutions[ssrc] = resolution;
-                    resolutions[participantId] = userResolutions;
-                }
+        if (track.isAudioTrack()) {
+            audioBitrateDownload += ssrcStats.bitrate.download;
+            audioBitrateUpload += ssrcStats.bitrate.upload;
+            audioCodec = ssrcStats.codec;
+        } else {
+            videoBitrateDownload += ssrcStats.bitrate.download;
+            videoBitrateUpload += ssrcStats.bitrate.upload;
+            videoCodec = ssrcStats.codec;
+        }
 
-                if (ssrcStats.framerate > 0) {
-                    const userFramerates = framerates[participantId] || {};
+        const participantId = track.getParticipantId();
 
-                    userFramerates[ssrc] = ssrcStats.framerate;
-                    framerates[participantId] = userFramerates;
-                }
+        if (!participantId) {
+            // All tracks in ssrc-rewriting mode need not have a participant associated with it.
+            if (!FeatureFlags.isSsrcRewritingSupported()) {
+                logger.error(`No participant ID returned by ${track}`);
+            }
+            continue; // eslint-disable-line no-continue
+        }
 
-                const userCodecs = codecs[participantId] ?? { };
+        const userCodecs = codecs[participantId] ?? { };
 
-                userCodecs[ssrc] = {
-                    audio: audioCodec,
-                    video: videoCodec
-                };
+        userCodecs[ssrc] = {
+            audio: audioCodec,
+            video: videoCodec
+        };
 
-                codecs[participantId] = userCodecs;
+        codecs[participantId] = userCodecs;
+        const { resolution } = ssrcStats;
 
-            // All tracks in ssrc-rewriting mode need not have a participant associated with it.
-            } else if (!FeatureFlags.isSsrcRewritingSupported()) {
-                logger.error(`No participant ID returned by ${track}`);
+        if (!track.isVideoTrack()
+            || isNaN(resolution?.height)
+            || isNaN(resolution?.width)
+            || resolution.height === -1
+            || resolution.width === -1) {
+            continue; // eslint-disable-line no-continue
+        }
+        const userResolutions = resolutions[participantId] || {};
+
+        // If simulcast (VP8) is used, there will be 3 "outbound-rtp" streams with different resolutions and 3
+        // different SSRCs. Based on the requested resolution and the current cpu and available bandwidth
+        // values, some of the streams might get suspended. Therefore the actual send resolution needs to be
+        // calculated based on the outbound-rtp streams that are currently active for the simulcast case.
+        // However for the SVC case, there will be only 1 "outbound-rtp" stream which will have the correct
+        // send resolution width and height.
+        if (track.isLocal() && !browser.supportsTrackBasedStats() && this.peerconnection.doesTrueSimulcast()) {
+            const localSsrcs = this.peerconnection.getLocalVideoSSRCs(track);
+
+            for (const localSsrc of localSsrcs) {
+                const ssrcResolution = this.ssrc2stats.get(localSsrc)?.resolution;
+
+                // The code processes resolution stats only for 'outbound-rtp' streams that are currently active.
+                if (ssrcResolution?.height && ssrcResolution?.width) {
+                    resolution.height = Math.max(resolution.height, ssrcResolution.height);
+                    resolution.width = Math.max(resolution.width, ssrcResolution.width);
+                }
             }
         }
 
-        ssrcStats.resetBitrate();
+        userResolutions[ssrc] = resolution;
+        resolutions[participantId] = userResolutions;
+
+        if (ssrcStats.framerate > 0) {
+            const userFramerates = framerates[participantId] || {};
+
+            userFramerates[ssrc] = ssrcStats.framerate;
+            framerates[participantId] = userFramerates;
+        }
     }
 
     this.conferenceStats.bitrate = {
@@ -445,6 +470,26 @@ StatsCollector.prototype._calculateBitrate = function(now, before, fieldName) {
     return bitrateKbps;
 };
 
+/**
+ * Calculates the frames per second rate between before and now using a supplied field name and its value in stats.
+ * @param {RTCOutboundRtpStreamStats|RTCSentRtpStreamStats} now the current stats
+ * @param {RTCOutboundRtpStreamStats|RTCSentRtpStreamStats} before the previous stats
+ * @param {string} fieldName the field to use for calculations.
+ * @returns {number} the calculated frame rate between now and before.
+ */
+StatsCollector.prototype._calculateFps = function(now, before, fieldName) {
+    const timeMs = now.timestamp - before.timestamp;
+    let frameRate = 0;
+
+    if (timeMs > 0 && now[fieldName]) {
+        const numberOfFramesSinceBefore = now[fieldName] - before[fieldName];
+
+        frameRate = (numberOfFramesSinceBefore / timeMs) * 1000;
+    }
+
+    return frameRate;
+};
+
 /**
  * Stats processing for spec-compliant RTCPeerConnection#getStats.
  */
@@ -552,29 +597,37 @@ StatsCollector.prototype.processStatsReport = function() {
                 });
             }
 
-            // Get the resolution and framerate for only remote video sources here. For the local video sources,
-            // 'track' stats will be used since they have the updated resolution based on the simulcast streams
-            // currently being sent. Promise based getStats reports three 'outbound-rtp' streams and there will be
-            // more calculations needed to determine what is the highest resolution stream sent by the client if the
-            // 'outbound-rtp' stats are used.
-            if (now.type === 'inbound-rtp') {
-                const resolution = {
-                    height: now.frameHeight,
-                    width: now.frameWidth
-                };
-                const frameRate = now.framesPerSecond;
+            let resolution;
 
-                if (resolution.height && resolution.width) {
-                    ssrcStats.setResolution(resolution);
-                }
-                ssrcStats.setFramerate(Math.round(frameRate || 0));
+            // Process the stats for 'inbound-rtp' streams always and 'outbound-rtp' only if the browser is
+            // Chromium based and version 112 and later since 'track' based stats are no longer available there
+            // for calculating send resolution and frame rate.
+            if (typeof now.frameHeight !== 'undefined' && typeof now.frameWidth !== 'undefined') {
+                // Assume the stream is active if the field is missing in the stats(Firefox)
+                const isStreamActive = now.active ?? true;
 
-                if (before) {
-                    ssrcStats.addBitrate({
-                        'download': this._calculateBitrate(now, before, 'bytesReceived'),
-                        'upload': 0
-                    });
+                if (now.type === 'inbound-rtp' || (!browser.supportsTrackBasedStats() && isStreamActive)) {
+                    resolution = {
+                        height: now.frameHeight,
+                        width: now.frameWidth
+                    };
                 }
+            }
+            ssrcStats.setResolution(resolution);
+
+            let frameRate = now.framesPerSecond;
+
+            if (!frameRate && before) {
+                frameRate = this._calculateFps(now, before, 'framesSent');
+            }
+
+            ssrcStats.setFramerate(Math.round(frameRate || 0));
+
+            if (now.type === 'inbound-rtp' && before) {
+                ssrcStats.addBitrate({
+                    'download': this._calculateBitrate(now, before, 'bytesReceived'),
+                    'upload': 0
+                });
             } else if (before) {
                 byteSentStats[ssrc] = this.getNonNegativeValue(now.bytesSent);
                 ssrcStats.addBitrate({
@@ -596,10 +649,11 @@ StatsCollector.prototype.processStatsReport = function() {
                 codecShortType && ssrcStats.setCodec(codecShortType);
             }
 
-        // Use track stats for resolution and framerate of the local video source.
-        // RTCVideoHandlerStats - https://w3c.github.io/webrtc-stats/#vststats-dict*
-        // RTCMediaHandlerStats - https://w3c.github.io/webrtc-stats/#mststats-dict*
-        } else if (now.type === 'track' && now.kind === MediaType.VIDEO && !now.remoteSource) {
+        // Continue to use the 'track' based stats for Firefox and Safari and older versions of Chromium.
+        } else if (browser.supportsTrackBasedStats()
+            && now.type === 'track'
+            && now.kind === MediaType.VIDEO
+            && !now.remoteSource) {
             const resolution = {
                 height: now.frameHeight,
                 width: now.frameWidth
@@ -629,27 +683,9 @@ StatsCollector.prototype.processStatsReport = function() {
             // Therefore, it needs to be divided by the total number of active simulcast streams.
             let frameRate = now.framesPerSecond;
 
-            if (!frameRate) {
-                if (before) {
-                    const timeMs = now.timestamp - before.timestamp;
-
-                    if (timeMs > 0 && now.framesSent) {
-                        const numberOfFramesSinceBefore = now.framesSent - before.framesSent;
-
-                        frameRate = (numberOfFramesSinceBefore / timeMs) * 1000;
-                    }
-                }
-
-                if (!frameRate) {
-                    return;
-                }
+            if (!frameRate && before) {
+                frameRate = this._calculateFps(now, before, 'framesSent');
             }
-
-            // Get the number of simulcast streams currently enabled from TPC.
-            const numberOfActiveStreams = this.peerconnection.getActiveSimulcastStreams();
-
-            // Reset frame rate to 0 when video is suspended as a result of endpoint falling out of last-n.
-            frameRate = numberOfActiveStreams ? Math.round(frameRate / numberOfActiveStreams) : 0;
             ssrcStats.setFramerate(frameRate);
         }
     });